meissa/mastodon-bot
Archived
7
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

refactored tumblr to transform

Browse source
This commit is contained in:
jem 2020-06-08 19:47:04 +02:00
parent 3f59652d05
commit 44b5bdb3bb
6 changed files with 118 additions and 70 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

31
README.md
View file

@ -36,18 +36,13 @@ with later timestamps to avoid duplicate posts. On the first run the timestamp w
;; account number you see when you log in and go to your profile
;; e.g: https://mastodon.social/web/accounts/294795
:account-id "XXXX"
:api_url "https://botsin.space/api/v1/"}}
;; add Tumblr config to mirror Tumblr accounts
:tumblr {:access-keys
{:consumer_key "XXXX"
:consumer_secret "XXXX"
:token "XXXX"
:token_secret "XXXX"}
;; optional limit for number of posts to retrieve, default: 5
:limit 10
:accounts ["cyberpunky.tumblr.com" "scipunk.tumblr.com"]}
:api_url "https://botsin.space/api/v1/"}
:tumblr {:consumer_key "XXXX"
:consumer_secret "XXXX"
:token "XXXX"
:token_secret "XXXX"}}
:transform [{:source {:type :twitter-source
:transform [{:source {:source-type :twitter
;; optional, defaults to false
:include-replies? false
;; optional, defaults to false
@ -56,7 +51,7 @@ with later timestamps to avoid duplicate posts. On the first run the timestamp w
:nitter-urls? false
;; accounts you wish to mirror
:accounts ["arstechnica" "WIRED"]}
:target {:type :mastodon-target
:target {:target-type :mastodon-target
;; optional flag specifying wether the name of the account
;; will be appended in the post, defaults to false
:append-screen-name? false
@ -84,11 +79,19 @@ with later timestamps to avoid duplicate posts. On the first run the timestamp w
;; TODO: Description & example missing here
:replacements nil}
{:source {:type :rss-source
{:source {:source-type :rss
;; add RSS config to follow feeds
:feeds [["Hacker News" "https://hnrss.org/newest"]
["r/Clojure" "https://www.reddit.com/r/clojure/.rss"]]}
:target {:type :mastodon-target
:target {:target-type :mastodon-target
...}
:resolve-urls? ...}
{:source {:source-type :tumblr
;; optional limit for number of posts to retrieve, default: 5
:limit 10
:accounts ["cyberpunky.tumblr.com" "scipunk.tumblr.com"]
:target {:target-type :mastodon-target
...}
:resolve-urls? ...}
]

64
src/main/mastodon_bot/core.cljs
View file

@ -14,10 +14,9 @@
(s/def ::mastodon masto/mastodon-auth?)
(s/def ::twitter twitter/twitter-auth?)
(s/def ::tumblr tumblr/tumblr-auth?)
(s/def ::transform transform/transformations?)
(s/def ::tumblr map?)
(s/def ::rss map?)
(s/def ::auth (s/keys :opt-un [::mastodon ::twitter]))
(s/def ::auth (s/keys :opt-un [::mastodon ::twitter ::tumblr]))
(def config?
(s/keys :req-un [::auth ::transform]))
@ -29,40 +28,16 @@
[config config?]
(get-in config [:auth :twitter]))
(defn-spec tumblr-auth ::tumblr
[config config?]
(get-in config [:auth :tumblr]))
(defn-spec transform ::transform
[config config?]
(:transform config))
(def config (infra/load-config))
(defn post-tumblrs [last-post-time]
(fn [err response]
(->> response
infra/js->edn
:posts
(mapv tumblr/parse-tumblr-post)
(map #(transform/intermediate-to-mastodon
(mastodon-auth config)
;todo: fix this
(:target (first (transform config))) %))
(masto/post-items
(mastodon-auth config)
(:target (first (transform config)))
last-post-time))))
(defn parse-feed [last-post-time parser [title url]]
(-> (.parseURL parser url)
(.then #(masto/post-items
(mastodon-auth config)
(:target (first (transform config)))
last-post-time
(for [{:keys [title isoDate pubDate content link]} (-> % infra/js->edn :items)]
{:created-at (js/Date. (or isoDate pubDate))
:text (str (transform/trim-text
title
(masto/max-post-length (:target (first (transform config)))))
"\n\n" (twitter/strip-utm link))})))))
(defn -main []
(let [mastodon-auth (mastodon-auth config)]
(masto/get-mastodon-timeline
@ -71,12 +46,12 @@
(let [last-post-time (-> timeline first :created_at (js/Date.))]
(let [{:keys [transform]} config]
(doseq [transformation transform]
(let [source-type (get-in transformation [:source :type])
target-type (get-in transformation [:target :type])]
(let [source-type (get-in transformation [:source :source-type])
target-type (get-in transformation [:target :target-type])]
(cond
;;post from Twitter
(and (= :twitter-source source-type)
(= :mastodon-target target-type))
(and (= :twitter source-type)
(= :mastodon target-type))
(when-let [twitter-auth (twitter-auth config)]
(transform/tweets-to-mastodon
mastodon-auth
@ -84,19 +59,22 @@
transformation
last-post-time))
;;post from RSS
(and (= :rss-source source-type)
(= :mastodon-target target-type))
(and (= :rss source-type)
(= :mastodon target-type))
(transform/rss-to-mastodon
mastodon-auth
transformation
last-post-time)
;;post from Tumblr
(and (= :tumblr-source source-type)
(= :mastodon-target target-type))
(when-let [{:keys [access-keys accounts limit]} (:tumblr config)]
(doseq [account accounts]
(let [client (tumblr/tumblr-client access-keys account)]
(.posts client #js {:limit (or limit 5)} (post-tumblrs last-post-time)))))))))
(and (= :tumblr source-type)
(= :mastodon target-type))
(when-let [tumblr-auth (tumblr-auth config)]
(transform/tumblr-to-mastodon
mastodon-auth
tumblr-auth
transformation
last-post-time))
))))
)))))
(set! *main-cli-fn* -main)

61
src/main/mastodon_bot/transform.cljs
View file

@ -21,21 +21,27 @@
:opt-un [::media-links ::untrimmed-text]))
(def mastodon-output? (s/keys :req-un [::created-at ::text]
:opt-un [::media-links]))
(s/def ::type keyword?)
(s/def ::source-type #{:twitter :rss :tumblr})
(s/def ::resolve-urls? boolean?)
(s/def ::content-filter string?)
(s/def ::content-filters (s/* ::content-filter))
(s/def ::keyword-filter string?)
(s/def ::keyword-filters (s/* ::keyword-filter))
(s/def ::replacements any?)
(defmulti source-type :type)
(defmethod source-type :twitter-source [_]
(s/merge (s/keys :req-un[::type]) twitter/twitter-source?))
(s/def ::source (s/multi-spec source-type ::type))
(defmulti target-type :type)
(defmethod target-type :mastodon-target [_]
(s/merge (s/keys :req-un [::type]) masto/mastodon-target?))
(s/def ::target (s/multi-spec target-type ::type))
(defmulti source-type :source-type)
(defmethod source-type :twitter [_]
(s/merge (s/keys :req-un[::source-type]) twitter/twitter-source?))
(defmethod source-type :rss [_]
(s/merge (s/keys :req-un [::source-type]) rss/rss-source?))
(defmethod source-type :tumblr [_]
(s/merge (s/keys :req-un [::source-type]) tumblr/tumblr-source?))
(s/def ::source (s/multi-spec source-type ::source-type))
(s/def ::target-type #{:mastodon})
(defmulti target-type :target-type)
(defmethod target-type :mastodon [_]
(s/merge (s/keys :req-un [::target-type]) masto/mastodon-target?))
(s/def ::target (s/multi-spec target-type ::target-type))
(s/def ::transformation (s/keys :req-un [::source ::target]
:opt-un [::resolve-urls? ::content-filters ::keyword-filters ::replacements]))
@ -129,6 +135,7 @@
untrimmed
sname
signature_text)
:reblogged true
:media-links media-links}))
(defn-spec post-tweets-to-mastodon any?
@ -140,6 +147,7 @@
(if error
(infra/exit-with-error error)
(->> (infra/js->edn tweets)
(debug)
(map twitter/parse-tweet)
(filter #(> (:created-at %) last-post-time))
(remove #(blocked-content? transformation (:text %)))
@ -165,6 +173,40 @@
transformation
last-post-time)))))
(defn-spec post-tumblr-to-mastodon any?
[mastodon-auth masto/mastodon-auth?
transformation ::transformation
last-post-time any?]
(let [{:keys [source target resolve-urls?]} transformation]
(fn [error tweets response]
(if error
(infra/exit-with-error error)
(->> (infra/js->edn tweets)
:posts
(mapv tumblr/parse-tumblr-post)
(filter #(> (:created-at %) last-post-time))
(remove #(blocked-content? transformation (:text %)))
(map #(perform-replacements transformation %))
(map #(intermediate-to-mastodon mastodon-auth target %))
(masto/post-items mastodon-auth target last-post-time))))))
(defn-spec tumblr-to-mastodon any?
[mastodon-auth masto/mastodon-auth?
tumblr-auth tumblr/tumblr-auth?
transformation ::transformation
last-post-time any?]
(let [{:keys [accounts limit]} transformation]
(doseq [account accounts]
(let [client (tumblr/tumblr-client tumblr-auth account)]
(.posts client
#js {:limit (or limit 5)}
(post-tumblr-to-mastodon
mastodon-auth
transformation
last-post-time)
)))))
(defn-spec post-rss-to-mastodon any?
[mastodon-auth masto/mastodon-auth?
transformation ::transformation
@ -178,7 +220,6 @@
(remove #(blocked-content? transformation (:text %)))
(map #(intermediate-resolve-urls resolve-urls? %))
(map #(perform-replacements transformation %))
(debug)
(map #(intermediate-to-mastodon mastodon-auth target %))
(masto/post-items mastodon-auth target last-post-time)))))

16
src/main/mastodon_bot/tumblr_api.cljs
View file

@ -8,7 +8,21 @@
["tumblr" :as tumblr]
))
(defn tumblr-client [access-keys account]
(s/def ::consumer_key string?)
(s/def ::consumer_secret string?)
(s/def ::token string?)
(s/def ::token_secret string?)
(def tumblr-auth? (s/keys :req-un [::consumer_key ::consumer_secret ::token
::token_secret]))
(s/def ::limit pos?)
(s/def ::account string?)
(s/def ::accounts (s/* ::account))
(def tumblr-source? (s/keys :req-un [::limit ::accounts]))
(defn-spec tumblr-client any?
[access-keys tumblr-auth?
account string?]
(try
(tumblr/Blog. account (clj->js access-keys))
(catch js/Error e

12
src/test/mastodon_bot/mytest.cljs Normal file
View file

@ -0,0 +1,12 @@
(ns mastodon-bot.mytest
(:require
[mastodon-bot.infra :as infra]
[clojure.pprint :refer [pprint]]
[mastodon-bot.twitter-api :as twitter]
[mastodon-bot.core :as core]))
(defn myconfig []
(core/mastodon-auth (infra/load-config)))
(defn run []
(core/-main))

4
src/test/mastodon_bot/transform_test.cljs
View file

@ -9,12 +9,12 @@
(is (s/valid? sut/transformations?
[]))
(is (s/valid? sut/transformations?
[{:source {:type :twitter-source
[{:source {:source-type :twitter
:include-replies? false
:include-rts? true
:nitter-urls? true
:accounts ["an-twitter-account"]}
:target {:type :mastodon-target
:target {:target-type :mastodon
:append-screen-name? true
:media-only? false
:max-post-length 500