move nitter to twitter-api & transform

README.md

@@ -59,13 +59,14 @@ with later timestamps to avoid duplicate posts. On the first run the timestamp w
             ;; optional keyword filter regexes
             ;; any posts not matching the regexes will be filtered out
             :keyword-filters [".*clojure.*"]
-            ;; Replace Twitter links by Nitter
+            }
-            :nitter-urls? false}
 :transform [{:source {:type :twitter-source
                        ;; optional, defaults to false
                        :include-replies? false
                        ;; optional, defaults to false
                        :include-rts? false
+                       ;; Replace Twitter links by Nitter
+                       :nitter-urls? false
                        ;; accounts you wish to mirror
                        :accounts ["arstechnica" "WIRED"]}
              :target {:type :mastodon-target

src/main/mastodon_bot/mastodon_api.cljs

@@ -19,7 +19,6 @@
 (s/def ::sensitive? boolean?)
 (s/def ::media-only? boolean?)
 (s/def ::resolve-urls? boolean?)
-(s/def ::nitter-urls? boolean?)
 (s/def ::visibility #{"direct" "private" "unlisted" "public"})
 (s/def ::replacements string?)
 (s/def ::max-post-length (fn [n] (and
@@ -39,7 +38,7 @@
                                        ::sensitive?
                                        ::media-only?
                                        ;::resolve-urls?
-                                       ;::nitter-urls? ::replacements
+                                       ;::replacements
                                        ]))
 (def mastodon-config? (s/merge mastodon-auth? mastodon-target?))
 
@@ -81,7 +80,7 @@
    status-id string?]
   (.delete (mastodon-client mastodon-config) (str "statuses/" status-id) #js {}))
 
-;; TODO: move to twitter
+;; TODO: move to transform
 (defn resolve-url [[uri]]
   (try
     (or
@@ -95,18 +94,16 @@
      uri)
     (catch js/Error _ uri)))
 
-;; TODO: move to twitter
+;; TODO: move to transform
 (def shortened-url-pattern #"(https?://)?(?:\S+(?::\S*)?@)?(?:(?!(?:10|127)(?:\.\d{1,3}){3})(?!(?:169\.254|192\.168)(?:\.\d{1,3}){2})(?!172\.(?:1[6-9]|2\d|3[0-1])(?:\.\d{1,3}){2})(?:[1-9]\d?|1\d\d|2[01]\d|22[0-3])(?:\.(?:1?\d{1,2}|2[0-4]\d|25[0-5])){2}(?:\.(?:[1-9]\d?|1\d\d|2[0-4]\d|25[0-4]))|(?:(?:[a-z\u00a1-\uffff0-9]-*)*[a-z\u00a1-\uffff0-9]+)(?:\.(?:[a-z\u00a1-\uffff0-9]-*)*[a-z\u00a1-\uffff0-9]+)*(?:\.(?:[a-z\u00a1-\uffff]{2,}))\.?)(?::\d{2,5})?(?:[/?#]\S*)?")
 
-;; TODO: move to twitter
+; TODO: move to transform
 (defn-spec resolve-urls string?
   [mastodon-config mastodon-config?
    text string?]
   (cond-> text
     (:resolve-urls? mastodon-config)
-    (string/replace shortened-url-pattern resolve-url)
+    (string/replace shortened-url-pattern resolve-url)))
-    (:nitter-urls? mastodon-config)
-    (string/replace #"https://twitter.com" "https://nitter.net")))
 
 (defn post-status
   ([mastodon-auth target status-text]

src/main/mastodon_bot/transform.cljs

@@ -74,6 +74,7 @@
 
 (defn-spec post-tweets-to-mastodon any?
   [mastodon-auth masto/mastodon-auth?
+   source twitter/twitter-source?
    target masto/mastodon-target?
    last-post-time any?]
   (fn [error tweets response]
@@ -81,6 +82,7 @@
       (infra/exit-with-error error)
       (->> (infra/js->edn tweets)
            (map twitter/parse-tweet)
+           (map twitter/nitter-url source)
            (map #(intermediate-to-mastodon mastodon-auth target %))
            (masto/post-items mastodon-auth target last-post-time)))))
 
@@ -97,5 +99,6 @@
        account
        (post-tweets-to-mastodon 
         mastodon-auth
+        source
         target
         last-post-time)))))

src/main/mastodon_bot/twitter_api.cljs

@@ -18,9 +18,11 @@
 
 (s/def ::include-rts? boolean?)
 (s/def ::include-replies? boolean?)
+(s/def ::nitter-urls? boolean?)
 (s/def ::account string?)
 (s/def ::accounts (s/* ::account))
-(def twitter-source?  (s/keys :req-un [::include-rts? ::include-replies? ::accounts]))
+(def twitter-source?  (s/keys :req-un [::include-rts? ::include-replies? ::accounts]
+                              :opt-un [::nitter-urls?]))
 
 (defn-spec twitter-client any?
   [twitter-auth twitter-auth?]
@@ -51,6 +53,12 @@
    :screen_name screen_name
    :media-links (keep #(when (= (:type %) "photo") (:media_url_https %)) media)})
 
+(defn-spec nitter-url map?
+  [source twitter-source?
+   parsed-tweet map?]
+  (when (:nitter-urls? source)
+    (update parsed-tweet :text #(string/replace % #"https://twitter.com" "https://nitter.net"))))
+
 (defn-spec user-timeline any?
   [twitter-auth twitter-auth?
    source twitter-source?

src/test/mastodon_bot/transform_test.cljs

@@ -12,6 +12,7 @@
                 [{:source {:type :twitter-source
                            :include-replies? false
                            :include-rts? true
+                           :nitter-urls? true
                            :accounts ["an-twitter-account"]} 
                   :target {:type :mastodon-target
                            :append-screen-name? true