introduce rss-api
This commit is contained in:
parent
d9cb7b5319
commit
a6d7c33af7
4 changed files with 109 additions and 24 deletions
14
README.md
14
README.md
|
@ -46,9 +46,6 @@ with later timestamps to avoid duplicate posts. On the first run the timestamp w
|
|||
;; optional limit for number of posts to retrieve, default: 5
|
||||
:limit 10
|
||||
:accounts ["cyberpunky.tumblr.com" "scipunk.tumblr.com"]}
|
||||
;; add RSS config to follow feeds
|
||||
:rss {"Hacker News" "https://hnrss.org/newest"
|
||||
"r/Clojure" "https://www.reddit.com/r/clojure/.rss"}
|
||||
|
||||
:transform [{:source {:type :twitter-source
|
||||
;; optional, defaults to false
|
||||
|
@ -85,7 +82,16 @@ with later timestamps to avoid duplicate posts. On the first run the timestamp w
|
|||
;; any posts not matching the regexes will be filtered out
|
||||
:keyword-filters [".*clojure.*"]
|
||||
;; TODO: Description & example missing here
|
||||
:replacements nil}]
|
||||
:replacements nil}
|
||||
|
||||
{:source {:type :rss-source
|
||||
;; add RSS config to follow feeds
|
||||
:feeds [["Hacker News" "https://hnrss.org/newest"]
|
||||
["r/Clojure" "https://www.reddit.com/r/clojure/.rss"]]}
|
||||
:target {:type :mastodon-target
|
||||
...}
|
||||
:resolve-urls? ...}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
|
|
|
@ -5,7 +5,6 @@
|
|||
[clojure.spec.alpha :as s]
|
||||
[clojure.spec.test.alpha :as st]
|
||||
[orchestra.core :refer-macros [defn-spec]]
|
||||
["rss-parser" :as rss]
|
||||
[mastodon-bot.infra :as infra]
|
||||
[mastodon-bot.transform :as transform]
|
||||
[mastodon-bot.mastodon-api :as masto]
|
||||
|
@ -70,25 +69,35 @@
|
|||
mastodon-auth
|
||||
(fn [timeline]
|
||||
(let [last-post-time (-> timeline first :created_at (js/Date.))]
|
||||
;;post from Twitter
|
||||
(when-let [twitter-auth (twitter-auth config)]
|
||||
(let [{:keys [transform]} config]
|
||||
(doseq [transformation transform]
|
||||
(let [source-type (get-in transformation [:source :type])
|
||||
target-type (get-in transformation [:target :type])]
|
||||
(cond
|
||||
;;post from Twitter
|
||||
(and (= :twitter-source source-type)
|
||||
(= :mastodon-target target-type))
|
||||
(when-let [twitter-auth (twitter-auth config)]
|
||||
(transform/tweets-to-mastodon
|
||||
mastodon-auth
|
||||
twitter-auth
|
||||
transformation
|
||||
last-post-time))))
|
||||
last-post-time))
|
||||
;;post from RSS
|
||||
(and (= :rss-source source-type)
|
||||
(= :mastodon-target target-type))
|
||||
(transform/rss-to-mastodon
|
||||
mastodon-auth
|
||||
transformation
|
||||
last-post-time)
|
||||
;;post from Tumblr
|
||||
(and (= :tumblr-source source-type)
|
||||
(= :mastodon-target target-type))
|
||||
(when-let [{:keys [access-keys accounts limit]} (:tumblr config)]
|
||||
(doseq [account accounts]
|
||||
(let [client (tumblr/tumblr-client access-keys account)]
|
||||
(.posts client #js {:limit (or limit 5)} (post-tumblrs last-post-time)))))
|
||||
;;post from RSS
|
||||
(when-let [feeds (some-> config :rss)]
|
||||
(let [parser (rss.)]
|
||||
(doseq [feed feeds]
|
||||
(parse-feed last-post-time parser feed)))))))))
|
||||
(.posts client #js {:limit (or limit 5)} (post-tumblrs last-post-time)))))))))
|
||||
)))))
|
||||
|
||||
(set! *main-cli-fn* -main)
|
||||
(st/instrument 'mastodon-auth)
|
||||
|
|
32
src/main/mastodon_bot/rss_api.cljs
Executable file
32
src/main/mastodon_bot/rss_api.cljs
Executable file
|
@ -0,0 +1,32 @@
|
|||
(ns mastodon-bot.rss-api
|
||||
(:require
|
||||
[clojure.spec.alpha :as s]
|
||||
[clojure.spec.test.alpha :as st]
|
||||
[orchestra.core :refer-macros [defn-spec]]
|
||||
["rss-parser" :as rss]
|
||||
[mastodon-bot.infra :as infra]
|
||||
))
|
||||
|
||||
(s/def ::feed (s/cat :name string? :url string?))
|
||||
(s/def ::feeds (s/* ::feed))
|
||||
(def rss-source? (s/keys :req-un [::feeds]))
|
||||
|
||||
(defn-spec rss-client any?
|
||||
[]
|
||||
(rss.))
|
||||
|
||||
(defn parse-feed [payload]
|
||||
(for [{:keys [title isoDate pubDate content link]} (:items payload)]
|
||||
(do
|
||||
(println title)
|
||||
{:created-at (js/Date. (or isoDate pubDate))
|
||||
:text (str title
|
||||
"\n\n"
|
||||
link)})))
|
||||
|
||||
(defn-spec get-feed map?
|
||||
[url string?
|
||||
callback fn?]
|
||||
(print url)
|
||||
(-> (.parseURL (rss-client) url)
|
||||
(.then callback)))
|
|
@ -7,6 +7,7 @@
|
|||
[mastodon-bot.infra :as infra]
|
||||
[mastodon-bot.mastodon-api :as masto]
|
||||
[mastodon-bot.twitter-api :as twitter]
|
||||
[mastodon-bot.rss-api :as rss]
|
||||
[mastodon-bot.tumblr-api :as tumblr]
|
||||
["deasync" :as deasync]
|
||||
["request" :as request]))
|
||||
|
@ -99,6 +100,11 @@
|
|||
input input?]
|
||||
(update input :text #(reduce-kv string/replace % (:replacements transformation))))
|
||||
|
||||
;TODO: remove in final code
|
||||
(defn debug[item]
|
||||
(println item)
|
||||
item)
|
||||
|
||||
|
||||
; TODO: move this to mastodon-api - seems to belong strongly to mastodon
|
||||
(defn-spec intermediate-to-mastodon mastodon-output?
|
||||
|
@ -135,6 +141,7 @@
|
|||
(infra/exit-with-error error)
|
||||
(->> (infra/js->edn tweets)
|
||||
(map twitter/parse-tweet)
|
||||
(filter #(> (:created-at %) last-post-time))
|
||||
(remove #(blocked-content? transformation (:text %)))
|
||||
(map #(intermediate-resolve-urls resolve-urls? %))
|
||||
(map #(twitter/nitter-url source %))
|
||||
|
@ -157,3 +164,34 @@
|
|||
mastodon-auth
|
||||
transformation
|
||||
last-post-time)))))
|
||||
|
||||
(defn-spec post-rss-to-mastodon any?
|
||||
[mastodon-auth masto/mastodon-auth?
|
||||
transformation ::transformation
|
||||
last-post-time any?]
|
||||
(let [{:keys [source target resolve-urls?]} transformation]
|
||||
(fn [payload]
|
||||
(->> (infra/js->edn payload)
|
||||
(map rss/parse-feed)
|
||||
(debug)
|
||||
(filter #(> (:created-at %) last-post-time))
|
||||
(remove #(blocked-content? transformation (:text %)))
|
||||
(map #(intermediate-resolve-urls resolve-urls? %))
|
||||
(map #(twitter/nitter-url source %))
|
||||
(map #(perform-replacements transformation %))
|
||||
(map #(intermediate-to-mastodon mastodon-auth target %))
|
||||
(masto/post-items mastodon-auth target last-post-time)))))
|
||||
|
||||
|
||||
(defn-spec rss-to-mastodon any?
|
||||
[mastodon-auth masto/mastodon-auth?
|
||||
transformation ::transformation
|
||||
last-post-time any?]
|
||||
(let [{:keys [source target]} transformation]
|
||||
(doseq [[name url] (:feeds source)]
|
||||
(rss/get-feed
|
||||
url
|
||||
(post-rss-to-mastodon
|
||||
mastodon-auth
|
||||
transformation
|
||||
last-post-time)))))
|
Reference in a new issue