refactored tumblr to transform

README.md

@@ -36,18 +36,13 @@ with later timestamps to avoid duplicate posts. On the first run the timestamp w
                    ;; account number you see when you log in and go to your profile
                    ;; e.g: https://mastodon.social/web/accounts/294795
                    :account-id "XXXX"
-                   :api_url "https://botsin.space/api/v1/"}}
+                   :api_url "https://botsin.space/api/v1/"}
- ;; add Tumblr config to mirror Tumblr accounts
+        :tumblr {:consumer_key "XXXX"
- :tumblr {:access-keys
-          {:consumer_key "XXXX"
                  :consumer_secret "XXXX"
                  :token "XXXX"
-           :token_secret "XXXX"}
+                 :token_secret "XXXX"}}
-          ;; optional limit for number of posts to retrieve, default: 5
-          :limit 10
-          :accounts ["cyberpunky.tumblr.com" "scipunk.tumblr.com"]}
  
-:transform [{:source {:type :twitter-source
+:transform [{:source {:source-type :twitter
                        ;; optional, defaults to false
                        :include-replies? false
                        ;; optional, defaults to false
@@ -56,7 +51,7 @@ with later timestamps to avoid duplicate posts. On the first run the timestamp w
                        :nitter-urls? false
                        ;; accounts you wish to mirror
                        :accounts ["arstechnica" "WIRED"]}
-             :target {:type :mastodon-target
+             :target {:target-type :mastodon-target
                       ;; optional flag specifying wether the name of the account
                       ;; will be appended in the post, defaults to false
                       :append-screen-name? false
@@ -84,11 +79,19 @@ with later timestamps to avoid duplicate posts. On the first run the timestamp w
              ;; TODO: Description & example missing here
              :replacements nil}
 
-             {:source {:type :rss-source
+             {:source {:source-type :rss
                        ;; add RSS config to follow feeds
                        :feeds [["Hacker News" "https://hnrss.org/newest"]
                                ["r/Clojure" "https://www.reddit.com/r/clojure/.rss"]]}
-             :target {:type :mastodon-target
+             :target {:target-type :mastodon-target
+                      ...}
+             :resolve-urls? ...}
+
+             {:source {:source-type :tumblr
+                       ;; optional limit for number of posts to retrieve, default: 5
+                       :limit 10
+                       :accounts ["cyberpunky.tumblr.com" "scipunk.tumblr.com"]
+             :target {:target-type :mastodon-target
                       ...}
              :resolve-urls? ...}
              ]

src/main/mastodon_bot/core.cljs

@@ -14,10 +14,9 @@
 
 (s/def ::mastodon masto/mastodon-auth?)
 (s/def ::twitter twitter/twitter-auth?)
+(s/def ::tumblr tumblr/tumblr-auth?)
 (s/def ::transform transform/transformations?)
-(s/def ::tumblr map?)
+(s/def ::auth (s/keys :opt-un [::mastodon ::twitter ::tumblr]))
-(s/def ::rss map?)
-(s/def ::auth (s/keys :opt-un [::mastodon ::twitter]))
 (def config? 
   (s/keys :req-un [::auth ::transform]))
 
@@ -29,40 +28,16 @@
   [config config?]
   (get-in config [:auth :twitter]))
 
+(defn-spec tumblr-auth ::tumblr
+  [config config?]
+  (get-in config [:auth :tumblr]))
+
 (defn-spec transform ::transform
   [config config?]
   (:transform config))
 
 (def config (infra/load-config))
 
-(defn post-tumblrs [last-post-time]
-  (fn [err response]
-    (->> response
-         infra/js->edn
-         :posts
-         (mapv tumblr/parse-tumblr-post)
-         (map #(transform/intermediate-to-mastodon
-                (mastodon-auth config)
-                ;todo: fix this
-                (:target (first (transform config))) %))
-         (masto/post-items 
-          (mastodon-auth config)
-          (:target (first (transform config)))
-          last-post-time))))
-
-(defn parse-feed [last-post-time parser [title url]]
-  (-> (.parseURL parser url)
-      (.then #(masto/post-items
-               (mastodon-auth config)
-               (:target (first (transform config)))
-               last-post-time
-               (for [{:keys [title isoDate pubDate content link]} (-> % infra/js->edn :items)]
-                 {:created-at (js/Date. (or isoDate pubDate))
-                  :text (str (transform/trim-text
-                              title
-                              (masto/max-post-length (:target (first (transform config))))) 
-                             "\n\n" (twitter/strip-utm link))})))))
-
 (defn -main []
   (let [mastodon-auth (mastodon-auth config)]
     (masto/get-mastodon-timeline
@@ -71,12 +46,12 @@
        (let [last-post-time (-> timeline first :created_at (js/Date.))]
          (let [{:keys [transform]} config]
            (doseq [transformation transform]
-             (let [source-type (get-in transformation [:source :type])
+             (let [source-type (get-in transformation [:source :source-type])
-                   target-type (get-in transformation [:target :type])]               
+                   target-type (get-in transformation [:target :target-type])]               
                (cond
                ;;post from Twitter
-                 (and (= :twitter-source source-type)
+                 (and (= :twitter source-type)
-                      (= :mastodon-target target-type))
+                      (= :mastodon target-type))
                  (when-let [twitter-auth (twitter-auth config)]
                    (transform/tweets-to-mastodon
                     mastodon-auth
@@ -84,19 +59,22 @@
                     transformation
                     last-post-time))
                ;;post from RSS
-                 (and (= :rss-source source-type)
+                 (and (= :rss source-type)
-                      (= :mastodon-target target-type))
+                      (= :mastodon target-type))
                  (transform/rss-to-mastodon
                   mastodon-auth
                   transformation
                   last-post-time)
                ;;post from Tumblr
-                 (and (= :tumblr-source source-type)
+                 (and (= :tumblr source-type)
-                      (= :mastodon-target target-type))
+                      (= :mastodon target-type))
-                 (when-let [{:keys [access-keys accounts limit]} (:tumblr config)]
+                 (when-let [tumblr-auth (tumblr-auth config)]
-                   (doseq [account accounts]
+                   (transform/tumblr-to-mastodon
-                     (let [client (tumblr/tumblr-client access-keys account)]
+                    mastodon-auth
-                       (.posts client #js {:limit (or limit 5)} (post-tumblrs last-post-time)))))))))
+                    tumblr-auth
+                    transformation
+                    last-post-time))
+                 ))))
 )))))
 
 (set! *main-cli-fn* -main)

src/main/mastodon_bot/transform.cljs

@@ -21,21 +21,27 @@
                      :opt-un [::media-links ::untrimmed-text]))
 (def mastodon-output?  (s/keys :req-un [::created-at ::text]
                                :opt-un [::media-links]))
-(s/def ::type keyword?)
+(s/def ::source-type #{:twitter :rss :tumblr})
 (s/def ::resolve-urls? boolean?)
 (s/def ::content-filter string?)
 (s/def ::content-filters (s/* ::content-filter))
 (s/def ::keyword-filter string?)
 (s/def ::keyword-filters (s/* ::keyword-filter))
 (s/def ::replacements any?)
-(defmulti source-type :type)
+(defmulti source-type :source-type)
-(defmethod source-type :twitter-source [_]
+(defmethod source-type :twitter [_]
-  (s/merge (s/keys :req-un[::type]) twitter/twitter-source?))
+  (s/merge (s/keys :req-un[::source-type]) twitter/twitter-source?))
-(s/def ::source (s/multi-spec source-type ::type))
+(defmethod source-type :rss [_]
-(defmulti target-type :type)
+  (s/merge (s/keys :req-un [::source-type]) rss/rss-source?))
-(defmethod target-type :mastodon-target [_]
+(defmethod source-type :tumblr [_]
-  (s/merge (s/keys :req-un [::type]) masto/mastodon-target?))
+  (s/merge (s/keys :req-un [::source-type]) tumblr/tumblr-source?))
-(s/def ::target (s/multi-spec target-type ::type))
+(s/def ::source (s/multi-spec source-type ::source-type))
+
+(s/def ::target-type #{:mastodon})
+(defmulti target-type :target-type)
+(defmethod target-type :mastodon [_]
+  (s/merge (s/keys :req-un [::target-type]) masto/mastodon-target?))
+(s/def ::target (s/multi-spec target-type ::target-type))
 
 (s/def ::transformation (s/keys :req-un [::source ::target]
                                 :opt-un [::resolve-urls? ::content-filters ::keyword-filters ::replacements]))
@@ -129,6 +135,7 @@
                 untrimmed
                 sname
                 signature_text)
+     :reblogged true
      :media-links media-links}))
 
 (defn-spec post-tweets-to-mastodon any?
@@ -140,6 +147,7 @@
       (if error
         (infra/exit-with-error error)
         (->> (infra/js->edn tweets)
+             (debug)
              (map twitter/parse-tweet)
              (filter #(> (:created-at %) last-post-time))
              (remove #(blocked-content? transformation (:text %)))
@@ -165,6 +173,40 @@
         transformation
         last-post-time)))))
 
+(defn-spec post-tumblr-to-mastodon any?
+  [mastodon-auth masto/mastodon-auth?
+   transformation ::transformation
+   last-post-time any?]
+  (let [{:keys [source target resolve-urls?]} transformation]
+    (fn [error tweets response]
+      (if error
+        (infra/exit-with-error error)
+        (->> (infra/js->edn tweets)
+             :posts
+             (mapv tumblr/parse-tumblr-post)
+             (filter #(> (:created-at %) last-post-time))             
+             (remove #(blocked-content? transformation (:text %)))
+             (map #(perform-replacements transformation %))
+             (map #(intermediate-to-mastodon mastodon-auth target %))
+             (masto/post-items mastodon-auth target last-post-time))))))
+
+(defn-spec tumblr-to-mastodon any?
+  [mastodon-auth masto/mastodon-auth?
+   tumblr-auth tumblr/tumblr-auth?
+   transformation ::transformation
+   last-post-time any?]
+  (let [{:keys [accounts limit]} transformation]
+    (doseq [account accounts]
+      (let [client (tumblr/tumblr-client tumblr-auth account)]
+        (.posts client 
+                #js {:limit (or limit 5)}
+                (post-tumblr-to-mastodon
+                 mastodon-auth
+                 transformation
+                 last-post-time)
+                )))))
+
+
 (defn-spec post-rss-to-mastodon any?
   [mastodon-auth masto/mastodon-auth?
    transformation ::transformation
@@ -178,7 +220,6 @@
            (remove #(blocked-content? transformation (:text %)))
            (map #(intermediate-resolve-urls resolve-urls? %))
            (map #(perform-replacements transformation %))
-           (debug)
            (map #(intermediate-to-mastodon mastodon-auth target %))
            (masto/post-items mastodon-auth target last-post-time)))))
 

src/main/mastodon_bot/tumblr_api.cljs

@@ -8,7 +8,21 @@
    ["tumblr" :as tumblr]
    ))
 
-(defn tumblr-client [access-keys account]
+(s/def ::consumer_key string?)
+(s/def ::consumer_secret string?)
+(s/def ::token string?)
+(s/def ::token_secret string?)
+(def tumblr-auth? (s/keys :req-un [::consumer_key ::consumer_secret ::token
+                                    ::token_secret]))
+
+(s/def ::limit pos?)
+(s/def ::account string?)
+(s/def ::accounts (s/* ::account))
+(def tumblr-source?  (s/keys :req-un [::limit ::accounts]))
+
+(defn-spec tumblr-client any?
+  [access-keys tumblr-auth?
+   account string?]
   (try
     (tumblr/Blog. account (clj->js access-keys))
     (catch js/Error e

src/test/mastodon_bot/mytest.cljs

@@ -0,0 +1,12 @@
+(ns mastodon-bot.mytest
+  (:require
+   [mastodon-bot.infra :as infra]
+   [clojure.pprint :refer [pprint]]   
+   [mastodon-bot.twitter-api :as twitter]
+   [mastodon-bot.core :as core]))
+
+(defn myconfig []
+  (core/mastodon-auth (infra/load-config)))
+
+(defn run []
+  (core/-main))

src/test/mastodon_bot/transform_test.cljs

@@ -9,12 +9,12 @@
   (is (s/valid? sut/transformations?
                 []))
   (is (s/valid? sut/transformations?
-                [{:source {:type :twitter-source
+                [{:source {:source-type :twitter
                            :include-replies? false
                            :include-rts? true
                            :nitter-urls? true
                            :accounts ["an-twitter-account"]}
-                  :target {:type :mastodon-target
+                  :target {:target-type :mastodon
                            :append-screen-name? true
                            :media-only? false
                            :max-post-length 500