meissa/mastodon-bot
Archived
7
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

introduce rss-api

Browse source
This commit is contained in:
jem 2020-05-31 19:59:20 +02:00
parent d9cb7b5319
commit a6d7c33af7
4 changed files with 109 additions and 24 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

14
README.md
View file

@ -46,9 +46,6 @@ with later timestamps to avoid duplicate posts. On the first run the timestamp w
;; optional limit for number of posts to retrieve, default: 5
:limit 10
:accounts ["cyberpunky.tumblr.com" "scipunk.tumblr.com"]}
;; add RSS config to follow feeds
:rss {"Hacker News" "https://hnrss.org/newest"
"r/Clojure" "https://www.reddit.com/r/clojure/.rss"}
:transform [{:source {:type :twitter-source
;; optional, defaults to false
@ -85,7 +82,16 @@ with later timestamps to avoid duplicate posts. On the first run the timestamp w
;; any posts not matching the regexes will be filtered out
:keyword-filters [".*clojure.*"]
;; TODO: Description & example missing here
:replacements nil}]
:replacements nil}
{:source {:type :rss-source
;; add RSS config to follow feeds
:feeds [["Hacker News" "https://hnrss.org/newest"]
["r/Clojure" "https://www.reddit.com/r/clojure/.rss"]]}
:target {:type :mastodon-target
...}
:resolve-urls? ...}
]
}
```

49
src/main/mastodon_bot/core.cljs
View file

@ -5,7 +5,6 @@
[clojure.spec.alpha :as s]
[clojure.spec.test.alpha :as st]
[orchestra.core :refer-macros [defn-spec]]
["rss-parser" :as rss]
[mastodon-bot.infra :as infra]
[mastodon-bot.transform :as transform]
[mastodon-bot.mastodon-api :as masto]
@ -70,25 +69,35 @@
mastodon-auth
(fn [timeline]
(let [last-post-time (-> timeline first :created_at (js/Date.))]
;;post from Twitter
(when-let [twitter-auth (twitter-auth config)]
(let [{:keys [transform]} config]
(doseq [transformation transform]
(transform/tweets-to-mastodon
mastodon-auth
twitter-auth
transformation
last-post-time))))
;;post from Tumblr
(when-let [{:keys [access-keys accounts limit]} (:tumblr config)]
(doseq [account accounts]
(let [client (tumblr/tumblr-client access-keys account)]
(.posts client #js {:limit (or limit 5)} (post-tumblrs last-post-time)))))
;;post from RSS
(when-let [feeds (some-> config :rss)]
(let [parser (rss.)]
(doseq [feed feeds]
(parse-feed last-post-time parser feed)))))))))
(let [{:keys [transform]} config]
(doseq [transformation transform]
(let [source-type (get-in transformation [:source :type])
target-type (get-in transformation [:target :type])]
(cond
;;post from Twitter
(and (= :twitter-source source-type)
(= :mastodon-target target-type))
(when-let [twitter-auth (twitter-auth config)]
(transform/tweets-to-mastodon
mastodon-auth
twitter-auth
transformation
last-post-time))
;;post from RSS
(and (= :rss-source source-type)
(= :mastodon-target target-type))
(transform/rss-to-mastodon
mastodon-auth
transformation
last-post-time)
;;post from Tumblr
(and (= :tumblr-source source-type)
(= :mastodon-target target-type))
(when-let [{:keys [access-keys accounts limit]} (:tumblr config)]
(doseq [account accounts]
(let [client (tumblr/tumblr-client access-keys account)]
(.posts client #js {:limit (or limit 5)} (post-tumblrs last-post-time)))))))))
)))))
(set! *main-cli-fn* -main)
(st/instrument 'mastodon-auth)

32
src/main/mastodon_bot/rss_api.cljs Executable file
View file

@ -0,0 +1,32 @@
(ns mastodon-bot.rss-api
(:require
[clojure.spec.alpha :as s]
[clojure.spec.test.alpha :as st]
[orchestra.core :refer-macros [defn-spec]]
["rss-parser" :as rss]
[mastodon-bot.infra :as infra]
))
(s/def ::feed (s/cat :name string? :url string?))
(s/def ::feeds (s/* ::feed))
(def rss-source? (s/keys :req-un [::feeds]))
(defn-spec rss-client any?
[]
(rss.))
(defn parse-feed [payload]
(for [{:keys [title isoDate pubDate content link]} (:items payload)]
(do
(println title)
{:created-at (js/Date. (or isoDate pubDate))
:text (str title
"\n\n"
link)})))
(defn-spec get-feed map?
[url string?
callback fn?]
(print url)
(-> (.parseURL (rss-client) url)
(.then callback)))

38
src/main/mastodon_bot/transform.cljs
View file

@ -7,6 +7,7 @@
[mastodon-bot.infra :as infra]
[mastodon-bot.mastodon-api :as masto]
[mastodon-bot.twitter-api :as twitter]
[mastodon-bot.rss-api :as rss]
[mastodon-bot.tumblr-api :as tumblr]
["deasync" :as deasync]
["request" :as request]))
@ -99,6 +100,11 @@
input input?]
(update input :text #(reduce-kv string/replace % (:replacements transformation))))
;TODO: remove in final code
(defn debug[item]
(println item)
item)
; TODO: move this to mastodon-api - seems to belong strongly to mastodon
(defn-spec intermediate-to-mastodon mastodon-output?
@ -135,6 +141,7 @@
(infra/exit-with-error error)
(->> (infra/js->edn tweets)
(map twitter/parse-tweet)
(filter #(> (:created-at %) last-post-time))
(remove #(blocked-content? transformation (:text %)))
(map #(intermediate-resolve-urls resolve-urls? %))
(map #(twitter/nitter-url source %))
@ -157,3 +164,34 @@
mastodon-auth
transformation
last-post-time)))))
(defn-spec post-rss-to-mastodon any?
[mastodon-auth masto/mastodon-auth?
transformation ::transformation
last-post-time any?]
(let [{:keys [source target resolve-urls?]} transformation]
(fn [payload]
(->> (infra/js->edn payload)
(map rss/parse-feed)
(debug)
(filter #(> (:created-at %) last-post-time))
(remove #(blocked-content? transformation (:text %)))
(map #(intermediate-resolve-urls resolve-urls? %))
(map #(twitter/nitter-url source %))
(map #(perform-replacements transformation %))
(map #(intermediate-to-mastodon mastodon-auth target %))
(masto/post-items mastodon-auth target last-post-time)))))
(defn-spec rss-to-mastodon any?
[mastodon-auth masto/mastodon-auth?
transformation ::transformation
last-post-time any?]
(let [{:keys [source target]} transformation]
(doseq [[name url] (:feeds source)]
(rss/get-feed
url
(post-rss-to-mastodon
mastodon-auth
transformation
last-post-time)))))