From 0bea240e49ab9d799b32dd795aab5c27af528da2 Mon Sep 17 00:00:00 2001 From: jem Date: Fri, 29 May 2020 22:16:23 +0200 Subject: [PATCH] move nitter to twitter-api & transform --- README.md | 5 +++-- src/main/mastodon_bot/mastodon_api.cljs | 13 +++++-------- src/main/mastodon_bot/transform.cljs | 3 +++ src/main/mastodon_bot/twitter_api.cljs | 10 +++++++++- src/test/mastodon_bot/transform_test.cljs | 1 + 5 files changed, 21 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index e2531e2..51e448a 100644 --- a/README.md +++ b/README.md @@ -59,13 +59,14 @@ with later timestamps to avoid duplicate posts. On the first run the timestamp w ;; optional keyword filter regexes ;; any posts not matching the regexes will be filtered out :keyword-filters [".*clojure.*"] - ;; Replace Twitter links by Nitter - :nitter-urls? false} + } :transform [{:source {:type :twitter-source ;; optional, defaults to false :include-replies? false ;; optional, defaults to false :include-rts? false + ;; Replace Twitter links by Nitter + :nitter-urls? false ;; accounts you wish to mirror :accounts ["arstechnica" "WIRED"]} :target {:type :mastodon-target diff --git a/src/main/mastodon_bot/mastodon_api.cljs b/src/main/mastodon_bot/mastodon_api.cljs index 20e53d6..ce36ad3 100755 --- a/src/main/mastodon_bot/mastodon_api.cljs +++ b/src/main/mastodon_bot/mastodon_api.cljs @@ -19,7 +19,6 @@ (s/def ::sensitive? boolean?) (s/def ::media-only? boolean?) (s/def ::resolve-urls? boolean?) -(s/def ::nitter-urls? boolean?) (s/def ::visibility #{"direct" "private" "unlisted" "public"}) (s/def ::replacements string?) (s/def ::max-post-length (fn [n] (and @@ -39,7 +38,7 @@ ::sensitive? ::media-only? ;::resolve-urls? - ;::nitter-urls? ::replacements + ;::replacements ])) (def mastodon-config? (s/merge mastodon-auth? mastodon-target?)) @@ -81,7 +80,7 @@ status-id string?] (.delete (mastodon-client mastodon-config) (str "statuses/" status-id) #js {})) -;; TODO: move to twitter +;; TODO: move to transform (defn resolve-url [[uri]] (try (or @@ -95,18 +94,16 @@ uri) (catch js/Error _ uri))) -;; TODO: move to twitter +;; TODO: move to transform (def shortened-url-pattern #"(https?://)?(?:\S+(?::\S*)?@)?(?:(?!(?:10|127)(?:\.\d{1,3}){3})(?!(?:169\.254|192\.168)(?:\.\d{1,3}){2})(?!172\.(?:1[6-9]|2\d|3[0-1])(?:\.\d{1,3}){2})(?:[1-9]\d?|1\d\d|2[01]\d|22[0-3])(?:\.(?:1?\d{1,2}|2[0-4]\d|25[0-5])){2}(?:\.(?:[1-9]\d?|1\d\d|2[0-4]\d|25[0-4]))|(?:(?:[a-z\u00a1-\uffff0-9]-*)*[a-z\u00a1-\uffff0-9]+)(?:\.(?:[a-z\u00a1-\uffff0-9]-*)*[a-z\u00a1-\uffff0-9]+)*(?:\.(?:[a-z\u00a1-\uffff]{2,}))\.?)(?::\d{2,5})?(?:[/?#]\S*)?") -;; TODO: move to twitter +; TODO: move to transform (defn-spec resolve-urls string? [mastodon-config mastodon-config? text string?] (cond-> text (:resolve-urls? mastodon-config) - (string/replace shortened-url-pattern resolve-url) - (:nitter-urls? mastodon-config) - (string/replace #"https://twitter.com" "https://nitter.net"))) + (string/replace shortened-url-pattern resolve-url))) (defn post-status ([mastodon-auth target status-text] diff --git a/src/main/mastodon_bot/transform.cljs b/src/main/mastodon_bot/transform.cljs index cc21cb0..179bc7d 100644 --- a/src/main/mastodon_bot/transform.cljs +++ b/src/main/mastodon_bot/transform.cljs @@ -74,6 +74,7 @@ (defn-spec post-tweets-to-mastodon any? [mastodon-auth masto/mastodon-auth? + source twitter/twitter-source? target masto/mastodon-target? last-post-time any?] (fn [error tweets response] @@ -81,6 +82,7 @@ (infra/exit-with-error error) (->> (infra/js->edn tweets) (map twitter/parse-tweet) + (map twitter/nitter-url source) (map #(intermediate-to-mastodon mastodon-auth target %)) (masto/post-items mastodon-auth target last-post-time))))) @@ -97,5 +99,6 @@ account (post-tweets-to-mastodon mastodon-auth + source target last-post-time))))) diff --git a/src/main/mastodon_bot/twitter_api.cljs b/src/main/mastodon_bot/twitter_api.cljs index 312a182..1d2e5bc 100755 --- a/src/main/mastodon_bot/twitter_api.cljs +++ b/src/main/mastodon_bot/twitter_api.cljs @@ -18,9 +18,11 @@ (s/def ::include-rts? boolean?) (s/def ::include-replies? boolean?) +(s/def ::nitter-urls? boolean?) (s/def ::account string?) (s/def ::accounts (s/* ::account)) -(def twitter-source? (s/keys :req-un [::include-rts? ::include-replies? ::accounts])) +(def twitter-source? (s/keys :req-un [::include-rts? ::include-replies? ::accounts] + :opt-un [::nitter-urls?])) (defn-spec twitter-client any? [twitter-auth twitter-auth?] @@ -51,6 +53,12 @@ :screen_name screen_name :media-links (keep #(when (= (:type %) "photo") (:media_url_https %)) media)}) +(defn-spec nitter-url map? + [source twitter-source? + parsed-tweet map?] + (when (:nitter-urls? source) + (update parsed-tweet :text #(string/replace % #"https://twitter.com" "https://nitter.net")))) + (defn-spec user-timeline any? [twitter-auth twitter-auth? source twitter-source? diff --git a/src/test/mastodon_bot/transform_test.cljs b/src/test/mastodon_bot/transform_test.cljs index 5865476..d61e878 100755 --- a/src/test/mastodon_bot/transform_test.cljs +++ b/src/test/mastodon_bot/transform_test.cljs @@ -12,6 +12,7 @@ [{:source {:type :twitter-source :include-replies? false :include-rts? true + :nitter-urls? true :accounts ["an-twitter-account"]} :target {:type :mastodon-target :append-screen-name? true