From a6d7c33af7e20ca321d396b543f401df4370a2c4 Mon Sep 17 00:00:00 2001 From: jem Date: Sun, 31 May 2020 19:59:20 +0200 Subject: [PATCH] introduce rss-api --- README.md | 14 +++++--- src/main/mastodon_bot/core.cljs | 49 ++++++++++++++++------------ src/main/mastodon_bot/rss_api.cljs | 32 ++++++++++++++++++ src/main/mastodon_bot/transform.cljs | 38 +++++++++++++++++++++ 4 files changed, 109 insertions(+), 24 deletions(-) create mode 100755 src/main/mastodon_bot/rss_api.cljs diff --git a/README.md b/README.md index 43fa316..2145069 100644 --- a/README.md +++ b/README.md @@ -46,9 +46,6 @@ with later timestamps to avoid duplicate posts. On the first run the timestamp w ;; optional limit for number of posts to retrieve, default: 5 :limit 10 :accounts ["cyberpunky.tumblr.com" "scipunk.tumblr.com"]} - ;; add RSS config to follow feeds - :rss {"Hacker News" "https://hnrss.org/newest" - "r/Clojure" "https://www.reddit.com/r/clojure/.rss"} :transform [{:source {:type :twitter-source ;; optional, defaults to false @@ -85,7 +82,16 @@ with later timestamps to avoid duplicate posts. On the first run the timestamp w ;; any posts not matching the regexes will be filtered out :keyword-filters [".*clojure.*"] ;; TODO: Description & example missing here - :replacements nil}] + :replacements nil} + + {:source {:type :rss-source + ;; add RSS config to follow feeds + :feeds [["Hacker News" "https://hnrss.org/newest"] + ["r/Clojure" "https://www.reddit.com/r/clojure/.rss"]]} + :target {:type :mastodon-target + ...} + :resolve-urls? ...} + ] } ``` diff --git a/src/main/mastodon_bot/core.cljs b/src/main/mastodon_bot/core.cljs index d236892..47af81c 100755 --- a/src/main/mastodon_bot/core.cljs +++ b/src/main/mastodon_bot/core.cljs @@ -5,7 +5,6 @@ [clojure.spec.alpha :as s] [clojure.spec.test.alpha :as st] [orchestra.core :refer-macros [defn-spec]] - ["rss-parser" :as rss] [mastodon-bot.infra :as infra] [mastodon-bot.transform :as transform] [mastodon-bot.mastodon-api :as masto] @@ -70,25 +69,35 @@ mastodon-auth (fn [timeline] (let [last-post-time (-> timeline first :created_at (js/Date.))] - ;;post from Twitter - (when-let [twitter-auth (twitter-auth config)] - (let [{:keys [transform]} config] - (doseq [transformation transform] - (transform/tweets-to-mastodon - mastodon-auth - twitter-auth - transformation - last-post-time)))) - ;;post from Tumblr - (when-let [{:keys [access-keys accounts limit]} (:tumblr config)] - (doseq [account accounts] - (let [client (tumblr/tumblr-client access-keys account)] - (.posts client #js {:limit (or limit 5)} (post-tumblrs last-post-time))))) - ;;post from RSS - (when-let [feeds (some-> config :rss)] - (let [parser (rss.)] - (doseq [feed feeds] - (parse-feed last-post-time parser feed))))))))) + (let [{:keys [transform]} config] + (doseq [transformation transform] + (let [source-type (get-in transformation [:source :type]) + target-type (get-in transformation [:target :type])] + (cond + ;;post from Twitter + (and (= :twitter-source source-type) + (= :mastodon-target target-type)) + (when-let [twitter-auth (twitter-auth config)] + (transform/tweets-to-mastodon + mastodon-auth + twitter-auth + transformation + last-post-time)) + ;;post from RSS + (and (= :rss-source source-type) + (= :mastodon-target target-type)) + (transform/rss-to-mastodon + mastodon-auth + transformation + last-post-time) + ;;post from Tumblr + (and (= :tumblr-source source-type) + (= :mastodon-target target-type)) + (when-let [{:keys [access-keys accounts limit]} (:tumblr config)] + (doseq [account accounts] + (let [client (tumblr/tumblr-client access-keys account)] + (.posts client #js {:limit (or limit 5)} (post-tumblrs last-post-time))))))))) +))))) (set! *main-cli-fn* -main) (st/instrument 'mastodon-auth) diff --git a/src/main/mastodon_bot/rss_api.cljs b/src/main/mastodon_bot/rss_api.cljs new file mode 100755 index 0000000..f7bcf04 --- /dev/null +++ b/src/main/mastodon_bot/rss_api.cljs @@ -0,0 +1,32 @@ +(ns mastodon-bot.rss-api + (:require + [clojure.spec.alpha :as s] + [clojure.spec.test.alpha :as st] + [orchestra.core :refer-macros [defn-spec]] + ["rss-parser" :as rss] + [mastodon-bot.infra :as infra] + )) + +(s/def ::feed (s/cat :name string? :url string?)) +(s/def ::feeds (s/* ::feed)) +(def rss-source? (s/keys :req-un [::feeds])) + +(defn-spec rss-client any? + [] + (rss.)) + +(defn parse-feed [payload] + (for [{:keys [title isoDate pubDate content link]} (:items payload)] + (do + (println title) + {:created-at (js/Date. (or isoDate pubDate)) + :text (str title + "\n\n" + link)}))) + +(defn-spec get-feed map? + [url string? + callback fn?] + (print url) + (-> (.parseURL (rss-client) url) + (.then callback))) \ No newline at end of file diff --git a/src/main/mastodon_bot/transform.cljs b/src/main/mastodon_bot/transform.cljs index 365bcbf..8c5ca03 100644 --- a/src/main/mastodon_bot/transform.cljs +++ b/src/main/mastodon_bot/transform.cljs @@ -7,6 +7,7 @@ [mastodon-bot.infra :as infra] [mastodon-bot.mastodon-api :as masto] [mastodon-bot.twitter-api :as twitter] + [mastodon-bot.rss-api :as rss] [mastodon-bot.tumblr-api :as tumblr] ["deasync" :as deasync] ["request" :as request])) @@ -98,6 +99,11 @@ [transformation ::transformation input input?] (update input :text #(reduce-kv string/replace % (:replacements transformation)))) + +;TODO: remove in final code +(defn debug[item] + (println item) + item) ; TODO: move this to mastodon-api - seems to belong strongly to mastodon @@ -135,6 +141,7 @@ (infra/exit-with-error error) (->> (infra/js->edn tweets) (map twitter/parse-tweet) + (filter #(> (:created-at %) last-post-time)) (remove #(blocked-content? transformation (:text %))) (map #(intermediate-resolve-urls resolve-urls? %)) (map #(twitter/nitter-url source %)) @@ -157,3 +164,34 @@ mastodon-auth transformation last-post-time))))) + +(defn-spec post-rss-to-mastodon any? + [mastodon-auth masto/mastodon-auth? + transformation ::transformation + last-post-time any?] + (let [{:keys [source target resolve-urls?]} transformation] + (fn [payload] + (->> (infra/js->edn payload) + (map rss/parse-feed) + (debug) + (filter #(> (:created-at %) last-post-time)) + (remove #(blocked-content? transformation (:text %))) + (map #(intermediate-resolve-urls resolve-urls? %)) + (map #(twitter/nitter-url source %)) + (map #(perform-replacements transformation %)) + (map #(intermediate-to-mastodon mastodon-auth target %)) + (masto/post-items mastodon-auth target last-post-time))))) + + +(defn-spec rss-to-mastodon any? + [mastodon-auth masto/mastodon-auth? + transformation ::transformation + last-post-time any?] + (let [{:keys [source target]} transformation] + (doseq [[name url] (:feeds source)] + (rss/get-feed + url + (post-rss-to-mastodon + mastodon-auth + transformation + last-post-time))))) \ No newline at end of file