move nitter to twitter-api & transform
This commit is contained in:
parent
dd62eab56b
commit
0bea240e49
5 changed files with 21 additions and 11 deletions
|
@ -59,13 +59,14 @@ with later timestamps to avoid duplicate posts. On the first run the timestamp w
|
|||
;; optional keyword filter regexes
|
||||
;; any posts not matching the regexes will be filtered out
|
||||
:keyword-filters [".*clojure.*"]
|
||||
;; Replace Twitter links by Nitter
|
||||
:nitter-urls? false}
|
||||
}
|
||||
:transform [{:source {:type :twitter-source
|
||||
;; optional, defaults to false
|
||||
:include-replies? false
|
||||
;; optional, defaults to false
|
||||
:include-rts? false
|
||||
;; Replace Twitter links by Nitter
|
||||
:nitter-urls? false
|
||||
;; accounts you wish to mirror
|
||||
:accounts ["arstechnica" "WIRED"]}
|
||||
:target {:type :mastodon-target
|
||||
|
|
|
@ -19,7 +19,6 @@
|
|||
(s/def ::sensitive? boolean?)
|
||||
(s/def ::media-only? boolean?)
|
||||
(s/def ::resolve-urls? boolean?)
|
||||
(s/def ::nitter-urls? boolean?)
|
||||
(s/def ::visibility #{"direct" "private" "unlisted" "public"})
|
||||
(s/def ::replacements string?)
|
||||
(s/def ::max-post-length (fn [n] (and
|
||||
|
@ -39,7 +38,7 @@
|
|||
::sensitive?
|
||||
::media-only?
|
||||
;::resolve-urls?
|
||||
;::nitter-urls? ::replacements
|
||||
;::replacements
|
||||
]))
|
||||
(def mastodon-config? (s/merge mastodon-auth? mastodon-target?))
|
||||
|
||||
|
@ -81,7 +80,7 @@
|
|||
status-id string?]
|
||||
(.delete (mastodon-client mastodon-config) (str "statuses/" status-id) #js {}))
|
||||
|
||||
;; TODO: move to twitter
|
||||
;; TODO: move to transform
|
||||
(defn resolve-url [[uri]]
|
||||
(try
|
||||
(or
|
||||
|
@ -95,18 +94,16 @@
|
|||
uri)
|
||||
(catch js/Error _ uri)))
|
||||
|
||||
;; TODO: move to twitter
|
||||
;; TODO: move to transform
|
||||
(def shortened-url-pattern #"(https?://)?(?:\S+(?::\S*)?@)?(?:(?!(?:10|127)(?:\.\d{1,3}){3})(?!(?:169\.254|192\.168)(?:\.\d{1,3}){2})(?!172\.(?:1[6-9]|2\d|3[0-1])(?:\.\d{1,3}){2})(?:[1-9]\d?|1\d\d|2[01]\d|22[0-3])(?:\.(?:1?\d{1,2}|2[0-4]\d|25[0-5])){2}(?:\.(?:[1-9]\d?|1\d\d|2[0-4]\d|25[0-4]))|(?:(?:[a-z\u00a1-\uffff0-9]-*)*[a-z\u00a1-\uffff0-9]+)(?:\.(?:[a-z\u00a1-\uffff0-9]-*)*[a-z\u00a1-\uffff0-9]+)*(?:\.(?:[a-z\u00a1-\uffff]{2,}))\.?)(?::\d{2,5})?(?:[/?#]\S*)?")
|
||||
|
||||
;; TODO: move to twitter
|
||||
; TODO: move to transform
|
||||
(defn-spec resolve-urls string?
|
||||
[mastodon-config mastodon-config?
|
||||
text string?]
|
||||
(cond-> text
|
||||
(:resolve-urls? mastodon-config)
|
||||
(string/replace shortened-url-pattern resolve-url)
|
||||
(:nitter-urls? mastodon-config)
|
||||
(string/replace #"https://twitter.com" "https://nitter.net")))
|
||||
(string/replace shortened-url-pattern resolve-url)))
|
||||
|
||||
(defn post-status
|
||||
([mastodon-auth target status-text]
|
||||
|
|
|
@ -74,6 +74,7 @@
|
|||
|
||||
(defn-spec post-tweets-to-mastodon any?
|
||||
[mastodon-auth masto/mastodon-auth?
|
||||
source twitter/twitter-source?
|
||||
target masto/mastodon-target?
|
||||
last-post-time any?]
|
||||
(fn [error tweets response]
|
||||
|
@ -81,6 +82,7 @@
|
|||
(infra/exit-with-error error)
|
||||
(->> (infra/js->edn tweets)
|
||||
(map twitter/parse-tweet)
|
||||
(map twitter/nitter-url source)
|
||||
(map #(intermediate-to-mastodon mastodon-auth target %))
|
||||
(masto/post-items mastodon-auth target last-post-time)))))
|
||||
|
||||
|
@ -97,5 +99,6 @@
|
|||
account
|
||||
(post-tweets-to-mastodon
|
||||
mastodon-auth
|
||||
source
|
||||
target
|
||||
last-post-time)))))
|
||||
|
|
|
@ -18,9 +18,11 @@
|
|||
|
||||
(s/def ::include-rts? boolean?)
|
||||
(s/def ::include-replies? boolean?)
|
||||
(s/def ::nitter-urls? boolean?)
|
||||
(s/def ::account string?)
|
||||
(s/def ::accounts (s/* ::account))
|
||||
(def twitter-source? (s/keys :req-un [::include-rts? ::include-replies? ::accounts]))
|
||||
(def twitter-source? (s/keys :req-un [::include-rts? ::include-replies? ::accounts]
|
||||
:opt-un [::nitter-urls?]))
|
||||
|
||||
(defn-spec twitter-client any?
|
||||
[twitter-auth twitter-auth?]
|
||||
|
@ -51,6 +53,12 @@
|
|||
:screen_name screen_name
|
||||
:media-links (keep #(when (= (:type %) "photo") (:media_url_https %)) media)})
|
||||
|
||||
(defn-spec nitter-url map?
|
||||
[source twitter-source?
|
||||
parsed-tweet map?]
|
||||
(when (:nitter-urls? source)
|
||||
(update parsed-tweet :text #(string/replace % #"https://twitter.com" "https://nitter.net"))))
|
||||
|
||||
(defn-spec user-timeline any?
|
||||
[twitter-auth twitter-auth?
|
||||
source twitter-source?
|
||||
|
|
|
@ -12,6 +12,7 @@
|
|||
[{:source {:type :twitter-source
|
||||
:include-replies? false
|
||||
:include-rts? true
|
||||
:nitter-urls? true
|
||||
:accounts ["an-twitter-account"]}
|
||||
:target {:type :mastodon-target
|
||||
:append-screen-name? true
|
||||
|
|
Reference in a new issue