refactored tumblr to transform

This commit is contained in:
jem 2020-06-08 19:47:04 +02:00
parent 3f59652d05
commit 44b5bdb3bb
6 changed files with 118 additions and 70 deletions

View file

@ -36,18 +36,13 @@ with later timestamps to avoid duplicate posts. On the first run the timestamp w
;; account number you see when you log in and go to your profile ;; account number you see when you log in and go to your profile
;; e.g: https://mastodon.social/web/accounts/294795 ;; e.g: https://mastodon.social/web/accounts/294795
:account-id "XXXX" :account-id "XXXX"
:api_url "https://botsin.space/api/v1/"}} :api_url "https://botsin.space/api/v1/"}
;; add Tumblr config to mirror Tumblr accounts :tumblr {:consumer_key "XXXX"
:tumblr {:access-keys
{:consumer_key "XXXX"
:consumer_secret "XXXX" :consumer_secret "XXXX"
:token "XXXX" :token "XXXX"
:token_secret "XXXX"} :token_secret "XXXX"}}
;; optional limit for number of posts to retrieve, default: 5
:limit 10
:accounts ["cyberpunky.tumblr.com" "scipunk.tumblr.com"]}
:transform [{:source {:type :twitter-source :transform [{:source {:source-type :twitter
;; optional, defaults to false ;; optional, defaults to false
:include-replies? false :include-replies? false
;; optional, defaults to false ;; optional, defaults to false
@ -56,7 +51,7 @@ with later timestamps to avoid duplicate posts. On the first run the timestamp w
:nitter-urls? false :nitter-urls? false
;; accounts you wish to mirror ;; accounts you wish to mirror
:accounts ["arstechnica" "WIRED"]} :accounts ["arstechnica" "WIRED"]}
:target {:type :mastodon-target :target {:target-type :mastodon-target
;; optional flag specifying wether the name of the account ;; optional flag specifying wether the name of the account
;; will be appended in the post, defaults to false ;; will be appended in the post, defaults to false
:append-screen-name? false :append-screen-name? false
@ -84,11 +79,19 @@ with later timestamps to avoid duplicate posts. On the first run the timestamp w
;; TODO: Description & example missing here ;; TODO: Description & example missing here
:replacements nil} :replacements nil}
{:source {:type :rss-source {:source {:source-type :rss
;; add RSS config to follow feeds ;; add RSS config to follow feeds
:feeds [["Hacker News" "https://hnrss.org/newest"] :feeds [["Hacker News" "https://hnrss.org/newest"]
["r/Clojure" "https://www.reddit.com/r/clojure/.rss"]]} ["r/Clojure" "https://www.reddit.com/r/clojure/.rss"]]}
:target {:type :mastodon-target :target {:target-type :mastodon-target
...}
:resolve-urls? ...}
{:source {:source-type :tumblr
;; optional limit for number of posts to retrieve, default: 5
:limit 10
:accounts ["cyberpunky.tumblr.com" "scipunk.tumblr.com"]
:target {:target-type :mastodon-target
...} ...}
:resolve-urls? ...} :resolve-urls? ...}
] ]

View file

@ -14,10 +14,9 @@
(s/def ::mastodon masto/mastodon-auth?) (s/def ::mastodon masto/mastodon-auth?)
(s/def ::twitter twitter/twitter-auth?) (s/def ::twitter twitter/twitter-auth?)
(s/def ::tumblr tumblr/tumblr-auth?)
(s/def ::transform transform/transformations?) (s/def ::transform transform/transformations?)
(s/def ::tumblr map?) (s/def ::auth (s/keys :opt-un [::mastodon ::twitter ::tumblr]))
(s/def ::rss map?)
(s/def ::auth (s/keys :opt-un [::mastodon ::twitter]))
(def config? (def config?
(s/keys :req-un [::auth ::transform])) (s/keys :req-un [::auth ::transform]))
@ -29,40 +28,16 @@
[config config?] [config config?]
(get-in config [:auth :twitter])) (get-in config [:auth :twitter]))
(defn-spec tumblr-auth ::tumblr
[config config?]
(get-in config [:auth :tumblr]))
(defn-spec transform ::transform (defn-spec transform ::transform
[config config?] [config config?]
(:transform config)) (:transform config))
(def config (infra/load-config)) (def config (infra/load-config))
(defn post-tumblrs [last-post-time]
(fn [err response]
(->> response
infra/js->edn
:posts
(mapv tumblr/parse-tumblr-post)
(map #(transform/intermediate-to-mastodon
(mastodon-auth config)
;todo: fix this
(:target (first (transform config))) %))
(masto/post-items
(mastodon-auth config)
(:target (first (transform config)))
last-post-time))))
(defn parse-feed [last-post-time parser [title url]]
(-> (.parseURL parser url)
(.then #(masto/post-items
(mastodon-auth config)
(:target (first (transform config)))
last-post-time
(for [{:keys [title isoDate pubDate content link]} (-> % infra/js->edn :items)]
{:created-at (js/Date. (or isoDate pubDate))
:text (str (transform/trim-text
title
(masto/max-post-length (:target (first (transform config)))))
"\n\n" (twitter/strip-utm link))})))))
(defn -main [] (defn -main []
(let [mastodon-auth (mastodon-auth config)] (let [mastodon-auth (mastodon-auth config)]
(masto/get-mastodon-timeline (masto/get-mastodon-timeline
@ -71,12 +46,12 @@
(let [last-post-time (-> timeline first :created_at (js/Date.))] (let [last-post-time (-> timeline first :created_at (js/Date.))]
(let [{:keys [transform]} config] (let [{:keys [transform]} config]
(doseq [transformation transform] (doseq [transformation transform]
(let [source-type (get-in transformation [:source :type]) (let [source-type (get-in transformation [:source :source-type])
target-type (get-in transformation [:target :type])] target-type (get-in transformation [:target :target-type])]
(cond (cond
;;post from Twitter ;;post from Twitter
(and (= :twitter-source source-type) (and (= :twitter source-type)
(= :mastodon-target target-type)) (= :mastodon target-type))
(when-let [twitter-auth (twitter-auth config)] (when-let [twitter-auth (twitter-auth config)]
(transform/tweets-to-mastodon (transform/tweets-to-mastodon
mastodon-auth mastodon-auth
@ -84,19 +59,22 @@
transformation transformation
last-post-time)) last-post-time))
;;post from RSS ;;post from RSS
(and (= :rss-source source-type) (and (= :rss source-type)
(= :mastodon-target target-type)) (= :mastodon target-type))
(transform/rss-to-mastodon (transform/rss-to-mastodon
mastodon-auth mastodon-auth
transformation transformation
last-post-time) last-post-time)
;;post from Tumblr ;;post from Tumblr
(and (= :tumblr-source source-type) (and (= :tumblr source-type)
(= :mastodon-target target-type)) (= :mastodon target-type))
(when-let [{:keys [access-keys accounts limit]} (:tumblr config)] (when-let [tumblr-auth (tumblr-auth config)]
(doseq [account accounts] (transform/tumblr-to-mastodon
(let [client (tumblr/tumblr-client access-keys account)] mastodon-auth
(.posts client #js {:limit (or limit 5)} (post-tumblrs last-post-time))))))))) tumblr-auth
transformation
last-post-time))
))))
))))) )))))
(set! *main-cli-fn* -main) (set! *main-cli-fn* -main)

View file

@ -21,21 +21,27 @@
:opt-un [::media-links ::untrimmed-text])) :opt-un [::media-links ::untrimmed-text]))
(def mastodon-output? (s/keys :req-un [::created-at ::text] (def mastodon-output? (s/keys :req-un [::created-at ::text]
:opt-un [::media-links])) :opt-un [::media-links]))
(s/def ::type keyword?) (s/def ::source-type #{:twitter :rss :tumblr})
(s/def ::resolve-urls? boolean?) (s/def ::resolve-urls? boolean?)
(s/def ::content-filter string?) (s/def ::content-filter string?)
(s/def ::content-filters (s/* ::content-filter)) (s/def ::content-filters (s/* ::content-filter))
(s/def ::keyword-filter string?) (s/def ::keyword-filter string?)
(s/def ::keyword-filters (s/* ::keyword-filter)) (s/def ::keyword-filters (s/* ::keyword-filter))
(s/def ::replacements any?) (s/def ::replacements any?)
(defmulti source-type :type) (defmulti source-type :source-type)
(defmethod source-type :twitter-source [_] (defmethod source-type :twitter [_]
(s/merge (s/keys :req-un[::type]) twitter/twitter-source?)) (s/merge (s/keys :req-un[::source-type]) twitter/twitter-source?))
(s/def ::source (s/multi-spec source-type ::type)) (defmethod source-type :rss [_]
(defmulti target-type :type) (s/merge (s/keys :req-un [::source-type]) rss/rss-source?))
(defmethod target-type :mastodon-target [_] (defmethod source-type :tumblr [_]
(s/merge (s/keys :req-un [::type]) masto/mastodon-target?)) (s/merge (s/keys :req-un [::source-type]) tumblr/tumblr-source?))
(s/def ::target (s/multi-spec target-type ::type)) (s/def ::source (s/multi-spec source-type ::source-type))
(s/def ::target-type #{:mastodon})
(defmulti target-type :target-type)
(defmethod target-type :mastodon [_]
(s/merge (s/keys :req-un [::target-type]) masto/mastodon-target?))
(s/def ::target (s/multi-spec target-type ::target-type))
(s/def ::transformation (s/keys :req-un [::source ::target] (s/def ::transformation (s/keys :req-un [::source ::target]
:opt-un [::resolve-urls? ::content-filters ::keyword-filters ::replacements])) :opt-un [::resolve-urls? ::content-filters ::keyword-filters ::replacements]))
@ -129,6 +135,7 @@
untrimmed untrimmed
sname sname
signature_text) signature_text)
:reblogged true
:media-links media-links})) :media-links media-links}))
(defn-spec post-tweets-to-mastodon any? (defn-spec post-tweets-to-mastodon any?
@ -140,6 +147,7 @@
(if error (if error
(infra/exit-with-error error) (infra/exit-with-error error)
(->> (infra/js->edn tweets) (->> (infra/js->edn tweets)
(debug)
(map twitter/parse-tweet) (map twitter/parse-tweet)
(filter #(> (:created-at %) last-post-time)) (filter #(> (:created-at %) last-post-time))
(remove #(blocked-content? transformation (:text %))) (remove #(blocked-content? transformation (:text %)))
@ -165,6 +173,40 @@
transformation transformation
last-post-time))))) last-post-time)))))
(defn-spec post-tumblr-to-mastodon any?
[mastodon-auth masto/mastodon-auth?
transformation ::transformation
last-post-time any?]
(let [{:keys [source target resolve-urls?]} transformation]
(fn [error tweets response]
(if error
(infra/exit-with-error error)
(->> (infra/js->edn tweets)
:posts
(mapv tumblr/parse-tumblr-post)
(filter #(> (:created-at %) last-post-time))
(remove #(blocked-content? transformation (:text %)))
(map #(perform-replacements transformation %))
(map #(intermediate-to-mastodon mastodon-auth target %))
(masto/post-items mastodon-auth target last-post-time))))))
(defn-spec tumblr-to-mastodon any?
[mastodon-auth masto/mastodon-auth?
tumblr-auth tumblr/tumblr-auth?
transformation ::transformation
last-post-time any?]
(let [{:keys [accounts limit]} transformation]
(doseq [account accounts]
(let [client (tumblr/tumblr-client tumblr-auth account)]
(.posts client
#js {:limit (or limit 5)}
(post-tumblr-to-mastodon
mastodon-auth
transformation
last-post-time)
)))))
(defn-spec post-rss-to-mastodon any? (defn-spec post-rss-to-mastodon any?
[mastodon-auth masto/mastodon-auth? [mastodon-auth masto/mastodon-auth?
transformation ::transformation transformation ::transformation
@ -178,7 +220,6 @@
(remove #(blocked-content? transformation (:text %))) (remove #(blocked-content? transformation (:text %)))
(map #(intermediate-resolve-urls resolve-urls? %)) (map #(intermediate-resolve-urls resolve-urls? %))
(map #(perform-replacements transformation %)) (map #(perform-replacements transformation %))
(debug)
(map #(intermediate-to-mastodon mastodon-auth target %)) (map #(intermediate-to-mastodon mastodon-auth target %))
(masto/post-items mastodon-auth target last-post-time))))) (masto/post-items mastodon-auth target last-post-time)))))

View file

@ -8,7 +8,21 @@
["tumblr" :as tumblr] ["tumblr" :as tumblr]
)) ))
(defn tumblr-client [access-keys account] (s/def ::consumer_key string?)
(s/def ::consumer_secret string?)
(s/def ::token string?)
(s/def ::token_secret string?)
(def tumblr-auth? (s/keys :req-un [::consumer_key ::consumer_secret ::token
::token_secret]))
(s/def ::limit pos?)
(s/def ::account string?)
(s/def ::accounts (s/* ::account))
(def tumblr-source? (s/keys :req-un [::limit ::accounts]))
(defn-spec tumblr-client any?
[access-keys tumblr-auth?
account string?]
(try (try
(tumblr/Blog. account (clj->js access-keys)) (tumblr/Blog. account (clj->js access-keys))
(catch js/Error e (catch js/Error e

View file

@ -0,0 +1,12 @@
(ns mastodon-bot.mytest
(:require
[mastodon-bot.infra :as infra]
[clojure.pprint :refer [pprint]]
[mastodon-bot.twitter-api :as twitter]
[mastodon-bot.core :as core]))
(defn myconfig []
(core/mastodon-auth (infra/load-config)))
(defn run []
(core/-main))

View file

@ -9,12 +9,12 @@
(is (s/valid? sut/transformations? (is (s/valid? sut/transformations?
[])) []))
(is (s/valid? sut/transformations? (is (s/valid? sut/transformations?
[{:source {:type :twitter-source [{:source {:source-type :twitter
:include-replies? false :include-replies? false
:include-rts? true :include-rts? true
:nitter-urls? true :nitter-urls? true
:accounts ["an-twitter-account"]} :accounts ["an-twitter-account"]}
:target {:type :mastodon-target :target {:target-type :mastodon
:append-screen-name? true :append-screen-name? true
:media-only? false :media-only? false
:max-post-length 500 :max-post-length 500