From 44b5bdb3bbb976a3ad9c31b4d74f06dcb84ce876 Mon Sep 17 00:00:00 2001 From: jem Date: Mon, 8 Jun 2020 19:47:04 +0200 Subject: [PATCH] refactored tumblr to transform --- README.md | 31 ++++++----- src/main/mastodon_bot/core.cljs | 64 ++++++++--------------- src/main/mastodon_bot/transform.cljs | 61 +++++++++++++++++---- src/main/mastodon_bot/tumblr_api.cljs | 16 +++++- src/test/mastodon_bot/mytest.cljs | 12 +++++ src/test/mastodon_bot/transform_test.cljs | 4 +- 6 files changed, 118 insertions(+), 70 deletions(-) create mode 100644 src/test/mastodon_bot/mytest.cljs diff --git a/README.md b/README.md index 2145069..4bb9918 100644 --- a/README.md +++ b/README.md @@ -36,18 +36,13 @@ with later timestamps to avoid duplicate posts. On the first run the timestamp w ;; account number you see when you log in and go to your profile ;; e.g: https://mastodon.social/web/accounts/294795 :account-id "XXXX" - :api_url "https://botsin.space/api/v1/"}} - ;; add Tumblr config to mirror Tumblr accounts - :tumblr {:access-keys - {:consumer_key "XXXX" - :consumer_secret "XXXX" - :token "XXXX" - :token_secret "XXXX"} - ;; optional limit for number of posts to retrieve, default: 5 - :limit 10 - :accounts ["cyberpunky.tumblr.com" "scipunk.tumblr.com"]} + :api_url "https://botsin.space/api/v1/"} + :tumblr {:consumer_key "XXXX" + :consumer_secret "XXXX" + :token "XXXX" + :token_secret "XXXX"}} -:transform [{:source {:type :twitter-source +:transform [{:source {:source-type :twitter ;; optional, defaults to false :include-replies? false ;; optional, defaults to false @@ -56,7 +51,7 @@ with later timestamps to avoid duplicate posts. On the first run the timestamp w :nitter-urls? false ;; accounts you wish to mirror :accounts ["arstechnica" "WIRED"]} - :target {:type :mastodon-target + :target {:target-type :mastodon-target ;; optional flag specifying wether the name of the account ;; will be appended in the post, defaults to false :append-screen-name? false @@ -84,11 +79,19 @@ with later timestamps to avoid duplicate posts. On the first run the timestamp w ;; TODO: Description & example missing here :replacements nil} - {:source {:type :rss-source + {:source {:source-type :rss ;; add RSS config to follow feeds :feeds [["Hacker News" "https://hnrss.org/newest"] ["r/Clojure" "https://www.reddit.com/r/clojure/.rss"]]} - :target {:type :mastodon-target + :target {:target-type :mastodon-target + ...} + :resolve-urls? ...} + + {:source {:source-type :tumblr + ;; optional limit for number of posts to retrieve, default: 5 + :limit 10 + :accounts ["cyberpunky.tumblr.com" "scipunk.tumblr.com"] + :target {:target-type :mastodon-target ...} :resolve-urls? ...} ] diff --git a/src/main/mastodon_bot/core.cljs b/src/main/mastodon_bot/core.cljs index 47af81c..aa58e38 100755 --- a/src/main/mastodon_bot/core.cljs +++ b/src/main/mastodon_bot/core.cljs @@ -14,10 +14,9 @@ (s/def ::mastodon masto/mastodon-auth?) (s/def ::twitter twitter/twitter-auth?) +(s/def ::tumblr tumblr/tumblr-auth?) (s/def ::transform transform/transformations?) -(s/def ::tumblr map?) -(s/def ::rss map?) -(s/def ::auth (s/keys :opt-un [::mastodon ::twitter])) +(s/def ::auth (s/keys :opt-un [::mastodon ::twitter ::tumblr])) (def config? (s/keys :req-un [::auth ::transform])) @@ -29,40 +28,16 @@ [config config?] (get-in config [:auth :twitter])) +(defn-spec tumblr-auth ::tumblr + [config config?] + (get-in config [:auth :tumblr])) + (defn-spec transform ::transform [config config?] (:transform config)) (def config (infra/load-config)) -(defn post-tumblrs [last-post-time] - (fn [err response] - (->> response - infra/js->edn - :posts - (mapv tumblr/parse-tumblr-post) - (map #(transform/intermediate-to-mastodon - (mastodon-auth config) - ;todo: fix this - (:target (first (transform config))) %)) - (masto/post-items - (mastodon-auth config) - (:target (first (transform config))) - last-post-time)))) - -(defn parse-feed [last-post-time parser [title url]] - (-> (.parseURL parser url) - (.then #(masto/post-items - (mastodon-auth config) - (:target (first (transform config))) - last-post-time - (for [{:keys [title isoDate pubDate content link]} (-> % infra/js->edn :items)] - {:created-at (js/Date. (or isoDate pubDate)) - :text (str (transform/trim-text - title - (masto/max-post-length (:target (first (transform config))))) - "\n\n" (twitter/strip-utm link))}))))) - (defn -main [] (let [mastodon-auth (mastodon-auth config)] (masto/get-mastodon-timeline @@ -71,12 +46,12 @@ (let [last-post-time (-> timeline first :created_at (js/Date.))] (let [{:keys [transform]} config] (doseq [transformation transform] - (let [source-type (get-in transformation [:source :type]) - target-type (get-in transformation [:target :type])] + (let [source-type (get-in transformation [:source :source-type]) + target-type (get-in transformation [:target :target-type])] (cond ;;post from Twitter - (and (= :twitter-source source-type) - (= :mastodon-target target-type)) + (and (= :twitter source-type) + (= :mastodon target-type)) (when-let [twitter-auth (twitter-auth config)] (transform/tweets-to-mastodon mastodon-auth @@ -84,19 +59,22 @@ transformation last-post-time)) ;;post from RSS - (and (= :rss-source source-type) - (= :mastodon-target target-type)) + (and (= :rss source-type) + (= :mastodon target-type)) (transform/rss-to-mastodon mastodon-auth transformation last-post-time) ;;post from Tumblr - (and (= :tumblr-source source-type) - (= :mastodon-target target-type)) - (when-let [{:keys [access-keys accounts limit]} (:tumblr config)] - (doseq [account accounts] - (let [client (tumblr/tumblr-client access-keys account)] - (.posts client #js {:limit (or limit 5)} (post-tumblrs last-post-time))))))))) + (and (= :tumblr source-type) + (= :mastodon target-type)) + (when-let [tumblr-auth (tumblr-auth config)] + (transform/tumblr-to-mastodon + mastodon-auth + tumblr-auth + transformation + last-post-time)) + )))) ))))) (set! *main-cli-fn* -main) diff --git a/src/main/mastodon_bot/transform.cljs b/src/main/mastodon_bot/transform.cljs index 35e7051..39080d6 100644 --- a/src/main/mastodon_bot/transform.cljs +++ b/src/main/mastodon_bot/transform.cljs @@ -21,21 +21,27 @@ :opt-un [::media-links ::untrimmed-text])) (def mastodon-output? (s/keys :req-un [::created-at ::text] :opt-un [::media-links])) -(s/def ::type keyword?) +(s/def ::source-type #{:twitter :rss :tumblr}) (s/def ::resolve-urls? boolean?) (s/def ::content-filter string?) (s/def ::content-filters (s/* ::content-filter)) (s/def ::keyword-filter string?) (s/def ::keyword-filters (s/* ::keyword-filter)) (s/def ::replacements any?) -(defmulti source-type :type) -(defmethod source-type :twitter-source [_] - (s/merge (s/keys :req-un[::type]) twitter/twitter-source?)) -(s/def ::source (s/multi-spec source-type ::type)) -(defmulti target-type :type) -(defmethod target-type :mastodon-target [_] - (s/merge (s/keys :req-un [::type]) masto/mastodon-target?)) -(s/def ::target (s/multi-spec target-type ::type)) +(defmulti source-type :source-type) +(defmethod source-type :twitter [_] + (s/merge (s/keys :req-un[::source-type]) twitter/twitter-source?)) +(defmethod source-type :rss [_] + (s/merge (s/keys :req-un [::source-type]) rss/rss-source?)) +(defmethod source-type :tumblr [_] + (s/merge (s/keys :req-un [::source-type]) tumblr/tumblr-source?)) +(s/def ::source (s/multi-spec source-type ::source-type)) + +(s/def ::target-type #{:mastodon}) +(defmulti target-type :target-type) +(defmethod target-type :mastodon [_] + (s/merge (s/keys :req-un [::target-type]) masto/mastodon-target?)) +(s/def ::target (s/multi-spec target-type ::target-type)) (s/def ::transformation (s/keys :req-un [::source ::target] :opt-un [::resolve-urls? ::content-filters ::keyword-filters ::replacements])) @@ -129,6 +135,7 @@ untrimmed sname signature_text) + :reblogged true :media-links media-links})) (defn-spec post-tweets-to-mastodon any? @@ -140,6 +147,7 @@ (if error (infra/exit-with-error error) (->> (infra/js->edn tweets) + (debug) (map twitter/parse-tweet) (filter #(> (:created-at %) last-post-time)) (remove #(blocked-content? transformation (:text %))) @@ -165,6 +173,40 @@ transformation last-post-time))))) +(defn-spec post-tumblr-to-mastodon any? + [mastodon-auth masto/mastodon-auth? + transformation ::transformation + last-post-time any?] + (let [{:keys [source target resolve-urls?]} transformation] + (fn [error tweets response] + (if error + (infra/exit-with-error error) + (->> (infra/js->edn tweets) + :posts + (mapv tumblr/parse-tumblr-post) + (filter #(> (:created-at %) last-post-time)) + (remove #(blocked-content? transformation (:text %))) + (map #(perform-replacements transformation %)) + (map #(intermediate-to-mastodon mastodon-auth target %)) + (masto/post-items mastodon-auth target last-post-time)))))) + +(defn-spec tumblr-to-mastodon any? + [mastodon-auth masto/mastodon-auth? + tumblr-auth tumblr/tumblr-auth? + transformation ::transformation + last-post-time any?] + (let [{:keys [accounts limit]} transformation] + (doseq [account accounts] + (let [client (tumblr/tumblr-client tumblr-auth account)] + (.posts client + #js {:limit (or limit 5)} + (post-tumblr-to-mastodon + mastodon-auth + transformation + last-post-time) + ))))) + + (defn-spec post-rss-to-mastodon any? [mastodon-auth masto/mastodon-auth? transformation ::transformation @@ -178,7 +220,6 @@ (remove #(blocked-content? transformation (:text %))) (map #(intermediate-resolve-urls resolve-urls? %)) (map #(perform-replacements transformation %)) - (debug) (map #(intermediate-to-mastodon mastodon-auth target %)) (masto/post-items mastodon-auth target last-post-time))))) diff --git a/src/main/mastodon_bot/tumblr_api.cljs b/src/main/mastodon_bot/tumblr_api.cljs index c149ce3..6af87e6 100755 --- a/src/main/mastodon_bot/tumblr_api.cljs +++ b/src/main/mastodon_bot/tumblr_api.cljs @@ -8,7 +8,21 @@ ["tumblr" :as tumblr] )) -(defn tumblr-client [access-keys account] +(s/def ::consumer_key string?) +(s/def ::consumer_secret string?) +(s/def ::token string?) +(s/def ::token_secret string?) +(def tumblr-auth? (s/keys :req-un [::consumer_key ::consumer_secret ::token + ::token_secret])) + +(s/def ::limit pos?) +(s/def ::account string?) +(s/def ::accounts (s/* ::account)) +(def tumblr-source? (s/keys :req-un [::limit ::accounts])) + +(defn-spec tumblr-client any? + [access-keys tumblr-auth? + account string?] (try (tumblr/Blog. account (clj->js access-keys)) (catch js/Error e diff --git a/src/test/mastodon_bot/mytest.cljs b/src/test/mastodon_bot/mytest.cljs new file mode 100644 index 0000000..678eebc --- /dev/null +++ b/src/test/mastodon_bot/mytest.cljs @@ -0,0 +1,12 @@ +(ns mastodon-bot.mytest + (:require + [mastodon-bot.infra :as infra] + [clojure.pprint :refer [pprint]] + [mastodon-bot.twitter-api :as twitter] + [mastodon-bot.core :as core])) + +(defn myconfig [] + (core/mastodon-auth (infra/load-config))) + +(defn run [] + (core/-main)) diff --git a/src/test/mastodon_bot/transform_test.cljs b/src/test/mastodon_bot/transform_test.cljs index 7244982..13b9f73 100755 --- a/src/test/mastodon_bot/transform_test.cljs +++ b/src/test/mastodon_bot/transform_test.cljs @@ -9,12 +9,12 @@ (is (s/valid? sut/transformations? [])) (is (s/valid? sut/transformations? - [{:source {:type :twitter-source + [{:source {:source-type :twitter :include-replies? false :include-rts? true :nitter-urls? true :accounts ["an-twitter-account"]} - :target {:type :mastodon-target + :target {:target-type :mastodon :append-screen-name? true :media-only? false :max-post-length 500