introduce rss-api

README.md

@@ -46,9 +46,6 @@ with later timestamps to avoid duplicate posts. On the first run the timestamp w
           ;; optional limit for number of posts to retrieve, default: 5
           :limit 10
           :accounts ["cyberpunky.tumblr.com" "scipunk.tumblr.com"]}
- ;; add RSS config to follow feeds
- :rss {"Hacker News" "https://hnrss.org/newest"
-       "r/Clojure" "https://www.reddit.com/r/clojure/.rss"}
  
 :transform [{:source {:type :twitter-source
                        ;; optional, defaults to false
@@ -85,7 +82,16 @@ with later timestamps to avoid duplicate posts. On the first run the timestamp w
              ;; any posts not matching the regexes will be filtered out
              :keyword-filters [".*clojure.*"]
              ;; TODO: Description & example missing here
-             :replacements nil}]
+             :replacements nil}
+
+             {:source {:type :rss-source
+                       ;; add RSS config to follow feeds
+                       :feeds [["Hacker News" "https://hnrss.org/newest"]
+                               ["r/Clojure" "https://www.reddit.com/r/clojure/.rss"]]}
+             :target {:type :mastodon-target
+                      ...}
+             :resolve-urls? ...}
+             ]
 }
 ```
 

src/main/mastodon_bot/core.cljs

@@ -5,7 +5,6 @@
    [clojure.spec.alpha :as s]
    [clojure.spec.test.alpha :as st]
    [orchestra.core :refer-macros [defn-spec]]
-   ["rss-parser" :as rss]
    [mastodon-bot.infra :as infra]
    [mastodon-bot.transform :as transform]
    [mastodon-bot.mastodon-api :as masto]
@@ -70,25 +69,35 @@
      mastodon-auth
      (fn [timeline]
        (let [last-post-time (-> timeline first :created_at (js/Date.))]
-     ;;post from Twitter
+         (let [{:keys [transform]} config]
-         (when-let [twitter-auth (twitter-auth config)]
+           (doseq [transformation transform]
-           (let [{:keys [transform]} config]
+             (let [source-type (get-in transformation [:source :type])
-             (doseq [transformation transform]
+                   target-type (get-in transformation [:target :type])]               
-               (transform/tweets-to-mastodon
+               (cond
-                mastodon-auth
+               ;;post from Twitter
-                twitter-auth
+                 (and (= :twitter-source source-type)
-                transformation
+                      (= :mastodon-target target-type))
-                last-post-time))))
+                 (when-let [twitter-auth (twitter-auth config)]
-     ;;post from Tumblr
+                   (transform/tweets-to-mastodon
-         (when-let [{:keys [access-keys accounts limit]} (:tumblr config)]
+                    mastodon-auth
-           (doseq [account accounts]
+                    twitter-auth
-             (let [client (tumblr/tumblr-client access-keys account)]
+                    transformation
-               (.posts client #js {:limit (or limit 5)} (post-tumblrs last-post-time)))))
+                    last-post-time))
-     ;;post from RSS
+               ;;post from RSS
-         (when-let [feeds (some-> config :rss)]
+                 (and (= :rss-source source-type)
-           (let [parser (rss.)]
+                      (= :mastodon-target target-type))
-             (doseq [feed feeds]
+                 (transform/rss-to-mastodon
-               (parse-feed last-post-time parser feed)))))))))
+                  mastodon-auth
+                  transformation
+                  last-post-time)
+               ;;post from Tumblr
+                 (and (= :tumblr-source source-type)
+                      (= :mastodon-target target-type))
+                 (when-let [{:keys [access-keys accounts limit]} (:tumblr config)]
+                   (doseq [account accounts]
+                     (let [client (tumblr/tumblr-client access-keys account)]
+                       (.posts client #js {:limit (or limit 5)} (post-tumblrs last-post-time)))))))))
+)))))
 
 (set! *main-cli-fn* -main)
 (st/instrument 'mastodon-auth)

src/main/mastodon_bot/rss_api.cljs

@@ -0,0 +1,32 @@
+(ns mastodon-bot.rss-api
+  (:require
+   [clojure.spec.alpha :as s]
+   [clojure.spec.test.alpha :as st]
+   [orchestra.core :refer-macros [defn-spec]]   
+   ["rss-parser" :as rss]
+   [mastodon-bot.infra :as infra]
+   ))
+
+(s/def ::feed (s/cat :name string? :url string?))
+(s/def ::feeds (s/* ::feed))
+(def rss-source?  (s/keys :req-un [::feeds]))
+
+(defn-spec rss-client any?
+  []
+  (rss.))
+
+(defn parse-feed [payload]
+  (for [{:keys [title isoDate pubDate content link]} (:items payload)]
+    (do 
+      (println title)
+      {:created-at (js/Date. (or isoDate pubDate))
+       :text (str title
+                  "\n\n"
+                  link)})))
+
+(defn-spec get-feed map?
+  [url string?
+   callback fn?]
+  (print url)
+  (-> (.parseURL (rss-client) url)
+      (.then callback)))

src/main/mastodon_bot/transform.cljs

@@ -7,6 +7,7 @@
    [mastodon-bot.infra :as infra]
    [mastodon-bot.mastodon-api :as masto]
    [mastodon-bot.twitter-api :as twitter]
+   [mastodon-bot.rss-api :as rss]
    [mastodon-bot.tumblr-api :as tumblr]
    ["deasync" :as deasync]
    ["request" :as request]))
@@ -98,6 +99,11 @@
   [transformation ::transformation
    input input?]
   (update input :text #(reduce-kv string/replace % (:replacements transformation))))
+
+;TODO: remove in final code
+(defn debug[item]
+  (println item)
+  item)
   
 
 ; TODO: move this to mastodon-api - seems to belong strongly to mastodon
@@ -135,6 +141,7 @@
         (infra/exit-with-error error)
         (->> (infra/js->edn tweets)
              (map twitter/parse-tweet)
+             (filter #(> (:created-at %) last-post-time))
              (remove #(blocked-content? transformation (:text %)))
              (map #(intermediate-resolve-urls resolve-urls? %))
              (map #(twitter/nitter-url source %))
@@ -157,3 +164,34 @@
         mastodon-auth
         transformation
         last-post-time)))))
+
+(defn-spec post-rss-to-mastodon any?
+  [mastodon-auth masto/mastodon-auth?
+   transformation ::transformation
+   last-post-time any?]
+  (let [{:keys [source target resolve-urls?]} transformation]
+    (fn [payload]
+      (->> (infra/js->edn payload)
+           (map rss/parse-feed)
+           (debug)
+           (filter #(> (:created-at %) last-post-time))
+           (remove #(blocked-content? transformation (:text %)))
+           (map #(intermediate-resolve-urls resolve-urls? %))
+           (map #(twitter/nitter-url source %))
+           (map #(perform-replacements transformation %))
+           (map #(intermediate-to-mastodon mastodon-auth target %))
+           (masto/post-items mastodon-auth target last-post-time)))))
+
+
+(defn-spec rss-to-mastodon any?
+  [mastodon-auth masto/mastodon-auth?
+   transformation ::transformation
+   last-post-time any?]
+  (let [{:keys [source target]} transformation]
+    (doseq [[name url] (:feeds source)]
+      (rss/get-feed
+       url
+       (post-rss-to-mastodon
+        mastodon-auth
+        transformation
+        last-post-time)))))