move nitter to twitter-api & transform

master
jem 4 years ago
parent dd62eab56b
commit 0bea240e49

@ -59,13 +59,14 @@ with later timestamps to avoid duplicate posts. On the first run the timestamp w
;; optional keyword filter regexes ;; optional keyword filter regexes
;; any posts not matching the regexes will be filtered out ;; any posts not matching the regexes will be filtered out
:keyword-filters [".*clojure.*"] :keyword-filters [".*clojure.*"]
;; Replace Twitter links by Nitter }
:nitter-urls? false}
:transform [{:source {:type :twitter-source :transform [{:source {:type :twitter-source
;; optional, defaults to false ;; optional, defaults to false
:include-replies? false :include-replies? false
;; optional, defaults to false ;; optional, defaults to false
:include-rts? false :include-rts? false
;; Replace Twitter links by Nitter
:nitter-urls? false
;; accounts you wish to mirror ;; accounts you wish to mirror
:accounts ["arstechnica" "WIRED"]} :accounts ["arstechnica" "WIRED"]}
:target {:type :mastodon-target :target {:type :mastodon-target

@ -19,7 +19,6 @@
(s/def ::sensitive? boolean?) (s/def ::sensitive? boolean?)
(s/def ::media-only? boolean?) (s/def ::media-only? boolean?)
(s/def ::resolve-urls? boolean?) (s/def ::resolve-urls? boolean?)
(s/def ::nitter-urls? boolean?)
(s/def ::visibility #{"direct" "private" "unlisted" "public"}) (s/def ::visibility #{"direct" "private" "unlisted" "public"})
(s/def ::replacements string?) (s/def ::replacements string?)
(s/def ::max-post-length (fn [n] (and (s/def ::max-post-length (fn [n] (and
@ -39,7 +38,7 @@
::sensitive? ::sensitive?
::media-only? ::media-only?
;::resolve-urls? ;::resolve-urls?
;::nitter-urls? ::replacements ;::replacements
])) ]))
(def mastodon-config? (s/merge mastodon-auth? mastodon-target?)) (def mastodon-config? (s/merge mastodon-auth? mastodon-target?))
@ -81,7 +80,7 @@
status-id string?] status-id string?]
(.delete (mastodon-client mastodon-config) (str "statuses/" status-id) #js {})) (.delete (mastodon-client mastodon-config) (str "statuses/" status-id) #js {}))
;; TODO: move to twitter ;; TODO: move to transform
(defn resolve-url [[uri]] (defn resolve-url [[uri]]
(try (try
(or (or
@ -95,18 +94,16 @@
uri) uri)
(catch js/Error _ uri))) (catch js/Error _ uri)))
;; TODO: move to twitter ;; TODO: move to transform
(def shortened-url-pattern #"(https?://)?(?:\S+(?::\S*)?@)?(?:(?!(?:10|127)(?:\.\d{1,3}){3})(?!(?:169\.254|192\.168)(?:\.\d{1,3}){2})(?!172\.(?:1[6-9]|2\d|3[0-1])(?:\.\d{1,3}){2})(?:[1-9]\d?|1\d\d|2[01]\d|22[0-3])(?:\.(?:1?\d{1,2}|2[0-4]\d|25[0-5])){2}(?:\.(?:[1-9]\d?|1\d\d|2[0-4]\d|25[0-4]))|(?:(?:[a-z\u00a1-\uffff0-9]-*)*[a-z\u00a1-\uffff0-9]+)(?:\.(?:[a-z\u00a1-\uffff0-9]-*)*[a-z\u00a1-\uffff0-9]+)*(?:\.(?:[a-z\u00a1-\uffff]{2,}))\.?)(?::\d{2,5})?(?:[/?#]\S*)?") (def shortened-url-pattern #"(https?://)?(?:\S+(?::\S*)?@)?(?:(?!(?:10|127)(?:\.\d{1,3}){3})(?!(?:169\.254|192\.168)(?:\.\d{1,3}){2})(?!172\.(?:1[6-9]|2\d|3[0-1])(?:\.\d{1,3}){2})(?:[1-9]\d?|1\d\d|2[01]\d|22[0-3])(?:\.(?:1?\d{1,2}|2[0-4]\d|25[0-5])){2}(?:\.(?:[1-9]\d?|1\d\d|2[0-4]\d|25[0-4]))|(?:(?:[a-z\u00a1-\uffff0-9]-*)*[a-z\u00a1-\uffff0-9]+)(?:\.(?:[a-z\u00a1-\uffff0-9]-*)*[a-z\u00a1-\uffff0-9]+)*(?:\.(?:[a-z\u00a1-\uffff]{2,}))\.?)(?::\d{2,5})?(?:[/?#]\S*)?")
;; TODO: move to twitter ; TODO: move to transform
(defn-spec resolve-urls string? (defn-spec resolve-urls string?
[mastodon-config mastodon-config? [mastodon-config mastodon-config?
text string?] text string?]
(cond-> text (cond-> text
(:resolve-urls? mastodon-config) (:resolve-urls? mastodon-config)
(string/replace shortened-url-pattern resolve-url) (string/replace shortened-url-pattern resolve-url)))
(:nitter-urls? mastodon-config)
(string/replace #"https://twitter.com" "https://nitter.net")))
(defn post-status (defn post-status
([mastodon-auth target status-text] ([mastodon-auth target status-text]

@ -74,6 +74,7 @@
(defn-spec post-tweets-to-mastodon any? (defn-spec post-tweets-to-mastodon any?
[mastodon-auth masto/mastodon-auth? [mastodon-auth masto/mastodon-auth?
source twitter/twitter-source?
target masto/mastodon-target? target masto/mastodon-target?
last-post-time any?] last-post-time any?]
(fn [error tweets response] (fn [error tweets response]
@ -81,6 +82,7 @@
(infra/exit-with-error error) (infra/exit-with-error error)
(->> (infra/js->edn tweets) (->> (infra/js->edn tweets)
(map twitter/parse-tweet) (map twitter/parse-tweet)
(map twitter/nitter-url source)
(map #(intermediate-to-mastodon mastodon-auth target %)) (map #(intermediate-to-mastodon mastodon-auth target %))
(masto/post-items mastodon-auth target last-post-time))))) (masto/post-items mastodon-auth target last-post-time)))))
@ -97,5 +99,6 @@
account account
(post-tweets-to-mastodon (post-tweets-to-mastodon
mastodon-auth mastodon-auth
source
target target
last-post-time))))) last-post-time)))))

@ -18,9 +18,11 @@
(s/def ::include-rts? boolean?) (s/def ::include-rts? boolean?)
(s/def ::include-replies? boolean?) (s/def ::include-replies? boolean?)
(s/def ::nitter-urls? boolean?)
(s/def ::account string?) (s/def ::account string?)
(s/def ::accounts (s/* ::account)) (s/def ::accounts (s/* ::account))
(def twitter-source? (s/keys :req-un [::include-rts? ::include-replies? ::accounts])) (def twitter-source? (s/keys :req-un [::include-rts? ::include-replies? ::accounts]
:opt-un [::nitter-urls?]))
(defn-spec twitter-client any? (defn-spec twitter-client any?
[twitter-auth twitter-auth?] [twitter-auth twitter-auth?]
@ -51,6 +53,12 @@
:screen_name screen_name :screen_name screen_name
:media-links (keep #(when (= (:type %) "photo") (:media_url_https %)) media)}) :media-links (keep #(when (= (:type %) "photo") (:media_url_https %)) media)})
(defn-spec nitter-url map?
[source twitter-source?
parsed-tweet map?]
(when (:nitter-urls? source)
(update parsed-tweet :text #(string/replace % #"https://twitter.com" "https://nitter.net"))))
(defn-spec user-timeline any? (defn-spec user-timeline any?
[twitter-auth twitter-auth? [twitter-auth twitter-auth?
source twitter-source? source twitter-source?

@ -12,6 +12,7 @@
[{:source {:type :twitter-source [{:source {:type :twitter-source
:include-replies? false :include-replies? false
:include-rts? true :include-rts? true
:nitter-urls? true
:accounts ["an-twitter-account"]} :accounts ["an-twitter-account"]}
:target {:type :mastodon-target :target {:type :mastodon-target
:append-screen-name? true :append-screen-name? true