Merge pull request #2 from yogthos/tumblr

added tumblr support
This commit is contained in:
Dmitri Sotnikov 2018-03-15 09:24:44 -04:00 committed by Sotnikov, Dmitri
commit 5526b70813
3 changed files with 69 additions and 25 deletions

View file

@ -1,7 +1,6 @@
### description ### description
the bot will read the timeline from the specified Twitter accounts, the bot will read the timeline from the specified Twitter/Tumblr accounts, and post it to Mastodon
and post it to Mastodon
### installation ### installation
@ -13,18 +12,28 @@ and post it to Mastodon
* create a Mastodon API key following the instructions [here](https://tinysubversions.com/notes/mastodon-bot/) * create a Mastodon API key following the instructions [here](https://tinysubversions.com/notes/mastodon-bot/)
* create a Twitter API key follwing the instructions [here](https://developer.twitter.com/en/docs/basics/authentication/guides/access-tokens) * create a Twitter API key follwing the instructions [here](https://developer.twitter.com/en/docs/basics/authentication/guides/access-tokens)
* create a Tumblr API key following the instructions [here](http://www.developerdrive.com/2014/05/how-to-get-started-with-the-tumblr-api-part-1/)
* create a file called `config.edn` with the following contents: * create a file called `config.edn` with the following contents:
```clojure ```clojure
{:twitter {:access-keys {;; add Twitter config to mirror Twitter accounts
:twitter {:access-keys
{:consumer_key "XXXX" {:consumer_key "XXXX"
:consumer_secret "XXXX" :consumer_secret "XXXX"
:access_token_key "XXXX" :access_token_key "XXXX"
:access_token_secret "XXXX"} :access_token_secret "XXXX"}
:accounts ["arstechnica" "WIRED"]} ;; accounts you wish to mirror :accounts ["arstechnica" "WIRED"]} ;; accounts you wish to mirror
;; add Tumblr config to mirror Tumblr accounts
:tumblr {:access-keys
{:consumer_key "XXXX"
:consumer_secret "XXXX"
:token "XXXX"
:token_secret "XXXX"}
:accounts ["cyberpunky.tumblr.com" "scipunk.tumblr.com"]}
:mastodon {:access_token "XXXX" :mastodon {:access_token "XXXX"
:api_url "https://botsin.space/api/v1/"}} :api_url "https://botsin.space/api/v1/"}}
``` ```
* the bot looks for `config.edn` at its relative path by default, an alternative location can be specified either using the `MASTODON_BOT_CONFIG` environment variable or passing the path to config as an argument * the bot looks for `config.edn` at its relative path by default, an alternative location can be specified either using the `MASTODON_BOT_CONFIG` environment variable or passing the path to config as an argument
* run the bot: `./mastodon-bot.cljs` * run the bot: `./mastodon-bot.cljs`

View file

@ -3,20 +3,26 @@
(:require (:require
[cljs.core :refer [*command-line-args*]] [cljs.core :refer [*command-line-args*]]
[cljs.reader :as edn] [cljs.reader :as edn]
[clojure.set :refer [rename-keys]]
[clojure.string :as string]
["fs" :as fs] ["fs" :as fs]
["http" :as http]
["https" :as https] ["https" :as https]
["mastodon-api" :as mastodon] ["mastodon-api" :as mastodon]
["tumblr" :as tumblr]
["twitter" :as twitter])) ["twitter" :as twitter]))
(def config (-> (or (first *command-line-args*) (defn find-config []
(-> js/process .-env .-MASTODON_BOT_CONFIG) (or (first *command-line-args*)
"config.edn") (-> js/process .-env .-MASTODON_BOT_CONFIG)
fs/readFileSync "config.edn"))
str
edn/read-string)) (def config (-> (find-config) fs/readFileSync str edn/read-string))
(def mastodon-client (mastodon. (-> config :mastodon clj->js)))
(def twitter-client (twitter. (-> config :twitter :access-keys clj->js))) (def mastodon-client (or (some-> config :mastodon clj->js mastodon.)
(def twitter-accounts (-> config :twitter :accounts)) (do
(js/console.error "missing Mastodon client configuration!")
(js/process.exit 1))))
(defn js->edn [data] (defn js->edn [data]
(js->clj data :keywordize-keys true)) (js->clj data :keywordize-keys true))
@ -41,7 +47,7 @@
(post-status-with-images status-text urls [])) (post-status-with-images status-text urls []))
([status-text [url & urls] ids] ([status-text [url & urls] ids]
(if url (if url
(.get https url (.get (if (string/starts-with? url "https://") https http) url
(fn [image-stream] (fn [image-stream]
(post-image image-stream status-text #(post-status-with-images status-text urls (conj ids %))))) (post-image image-stream status-text #(post-status-with-images status-text urls (conj ids %)))))
(post-status status-text (not-empty ids))))) (post-status status-text (not-empty ids)))))
@ -49,6 +55,12 @@
(defn get-mastodon-timeline [callback] (defn get-mastodon-timeline [callback]
(.then (.get mastodon-client "timelines/home" #js {}) #(-> % .-data js->edn callback))) (.then (.get mastodon-client "timelines/home" #js {}) #(-> % .-data js->edn callback)))
(defn post-items [last-post-time items]
(doseq [{:keys [text media-links]} (filter #(> (:created-at %) last-post-time) items)]
(if media-links
(post-status-with-images text media-links)
(post-status text))))
(defn parse-tweet [{created-at :created_at (defn parse-tweet [{created-at :created_at
text :text text :text
{:keys [media]} :extended_entities {:keys [media]} :extended_entities
@ -57,20 +69,42 @@
:text (str text "\n - " screen_name) :text (str text "\n - " screen_name)
:media-links (keep #(when (= (:type %) "photo") (:media_url_https %)) media)}) :media-links (keep #(when (= (:type %) "photo") (:media_url_https %)) media)})
(defmulti parse-tumblr-post :type)
(defmethod parse-tumblr-post "text" [{:keys [body date short_url]}]
{:created-at (js/Date. date)
:text (str body "\n\n" short_url)})
(defmethod parse-tumblr-post "photo" [{:keys [caption date photos short_url] :as post}]
{:created-at (js/Date. date)
:text (string/join "\n" [(string/replace caption #"<[^>]*>" "") short_url])
:media-links (mapv #(-> % :original_size :url) photos)})
(defmethod parse-tumblr-post :default [post])
(defn post-tumblrs [last-post-time]
(fn [err response]
(->> response
js->edn
:posts
(mapv parse-tumblr-post)
(post-items last-post-time))))
(defn post-tweets [last-post-time] (defn post-tweets [last-post-time]
(fn [error tweets response] (fn [error tweets response]
(when tweets (->> (js->edn tweets)
(doseq [{:keys [text media-links]} (->> (js->edn tweets) (map parse-tweet)
(map parse-tweet) (post-items last-post-time))))
(filter #(> (:created-at %) last-post-time)))]
(if media-links
(post-status-with-images text media-links)
(post-status text))))))
(get-mastodon-timeline (get-mastodon-timeline
(fn [timeline] (fn [timeline]
(doseq [account twitter-accounts] (let [last-post-time (-> timeline first :created_at (js/Date.))]
(.get twitter-client (when-let [twitter-client (some-> config :twitter :access-keys clj->js twitter.)]
"statuses/user_timeline" (doseq [account (-> config :twitter :accounts)]
#js {:screen_name account :include_rts false} (.get twitter-client
(-> timeline first :created_at (js/Date.) post-tweets))))) "statuses/user_timeline"
#js {:screen_name account :include_rts false}
(post-tweets last-post-time))))
(when-let [tumblr-oauth (some-> config :tumblr :access-keys clj->js)]
(when-let [tumblr-client (some-> config :tumblr :accounts first (tumblr/Blog. tumblr-oauth))]
(.posts tumblr-client #js {:limit 5} (post-tumblrs last-post-time)))))))

View file

@ -6,6 +6,7 @@
"dependencies": { "dependencies": {
"lumo-cljs": "^1.8.0", "lumo-cljs": "^1.8.0",
"mastodon-api": "1.3.0", "mastodon-api": "1.3.0",
"tumblr": "0.4.1",
"twitter": "1.7.1" "twitter": "1.7.1"
}, },
"scripts": { "scripts": {