version bump

Improve console log

See merge request yogthos/mastodon-bot!12

.github/workflows/stable.yml

@@ -1,89 +0,0 @@
-name: stable
-on:
-  push:
-    tags: '[0-9]+.[0-9]+.[0-9]+*'
-
-jobs:
-  stable:
-    name: stable
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        node-version: [14.x]
-
-    steps:
-    - uses: actions/checkout@v2
-          
-    - name: Use Node.js ${{ matrix.node-version }}
-      uses: actions/setup-node@v1
-      with:
-        node-version: ${{ matrix.node-version }}
-    
-    - name: test em
-      run: | 
-        npm install
-        npm install -g --save-dev shadow-cljs
-        shadow-cljs compile test
-
-    - name: build em
-      run: |
-        shadow-cljs release app
-        chmod a+x mastodon-bot.js
-        sha256sum mastodon-bot.js > target/mastodon-bot.js.sha256
-        sha512sum mastodon-bot.js > target/mastodon-bot.js.sha512
-
-    - name: Create Release
-      id: create_release
-      uses: actions/create-release@v1
-      env:
-        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      with:
-        tag_name: ${{ github.ref }}
-        release_name: Release ${{ github.ref }}
-        draft: false
-        prerelease: false
-
-    - name: Upload mastodon-bot.js
-      id: upload-mastodon-bot-js
-      uses: actions/upload-release-asset@v1
-      env:
-        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      with:
-        upload_url: ${{ steps.create_release.outputs.upload_url }}
-        asset_path: ./mastodon-bot.js
-        asset_name: mastodon-bot.js
-        asset_content_type: application/javascript
-
-    - name: Upload mastodon-bot.js.sha256
-      id: upload-mastodon-bot-js-sha256
-      uses: actions/upload-release-asset@v1
-      env:
-        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      with:
-        upload_url: ${{ steps.create_release.outputs.upload_url }}
-        asset_path: ./target/mastodon-bot.js.sha256
-        asset_name: mastodon-bot.js.sha256
-        asset_content_type: text/plain
-
-    - name: Upload mastodon-bot.js.sha512
-      id: upload-mastodon-bot-js-sha512
-      uses: actions/upload-release-asset@v1
-      env:
-        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      with:
-        upload_url: ${{ steps.create_release.outputs.upload_url }}
-        asset_path: ./target/mastodon-bot.js.sha512
-        asset_name: mastodon-bot.js.sha512
-        asset_content_type: text/plain
-
-    - name: upload to npm
-      env:
-        NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}
-      run: |
-        mkdir -p target/npm-build/mastodon_bot
-        cp mastodon-bot.js target/npm-build/mastodon_bot/
-        cp target/mastodon-bot.js.sha256 target/npm-build/mastodon_bot/
-        cp target/mastodon-bot.js.sha512 target/npm-build/mastodon_bot/
-        cp package.json target/npm-build/mastodon_bot/
-        cp README.md target/npm-build/mastodon_bot/
-        npm publish ./target/npm-build/mastodon_bot --access public

doc/Releasing.md

@@ -5,6 +5,7 @@
 vi package.json
 
 lein release
+git push --follow-tags
 
 # bump version - increase version and add -SNAPSHOT
 vi package.json

doc/rss.puml

@@ -0,0 +1,10 @@
+@startuml
+"Mastodon-Bot" -> Config: Call rss-sources
+Config --> "Mastodon-Bot": URL list of rss-sources
+"Mastodon-Bot" -> "Mastodon-Bot": Call "get-feed" for every rss-source.\nStart asyncronous Promise.
+activate "Mastodon-Bot"
+"Mastodon-Bot" -> "RSS-Sources": Call URL to parse RSS-Feed
+"RSS-Sources" --> "Mastodon-Bot": RSS-Feed
+"Mastodon-Bot" -> Mastodon: Post feed by post-rss-to-mastodon
+deactivate "Mastodon-Bot"
+@enduml

package.json

@@ -2,17 +2,18 @@
   "name": "mastodon-bot",
   "description": "Bot to publish twitter, tumblr or rss posts to an mastodon account.",
   "author": "Dmitri Sotnikov",
-  "version": "1.10.12-SNAPSHOT",
+  "version": "1.12.1-SNAPSHOT",
   "homepage": "https://github.com/yogthos/mastodon-bot",
   "repository": "https://www.npmjs.com/package/mastodon-bot",
   "license": "MIT",
   "dependencies": {
+    "deasync": "0.1.20",
     "mastodon-api": "1.3.0",
+    "node-fetch": "2.6.1",
+    "request": "2.88.0",
     "rss-parser": "3.7.1",
     "tumblr": "0.4.1",
-    "twitter": "1.7.1",
+    "twitter": "1.7.1"
-    "deasync": "0.1.20",
-    "request": "2.88.0"
   },
   "devDependencies": {
     "shadow-cljs": "^2.8.37"

project.clj

@@ -1,4 +1,4 @@
-(defproject dda/mastodon-bot "1.10.12-SNAPSHOT"
+(defproject dda/mastodon-bot "1.12.1-SNAPSHOT"
   :description "Bot to publish twitter, tumblr or rss posts to an mastodon account."
   :url "https://github.com/yogthos/mastodon-bot"
   :author "Dmitri Sotnikov"

src/main/cljs/mastodon_bot/infra.cljs

@@ -1,15 +1,21 @@
 (ns mastodon-bot.infra
   (:require
    [cljs.reader :as edn]
-   [clojure.pprint :refer [pprint]]
+   [clojure.string :as string]
-   ["fs" :as fs]))
+   ["fs" :as fs]
+   ["deasync" :as deasync]
+   ["node-fetch" :as fetch]))
 
-(defn debug [item]
+(defn log-error [item]
-  (pprint item)
+  (js/console.error item)
   item)
 
-(defn debug-first [item]
+(defn log [item]
-  (pprint (first item))
+  (js/console.log item)
+  item)
+
+(defn log-first [item]
+  (js/console.log (first item))
   item)
 
 (defn js->edn [data]
@@ -28,8 +34,8 @@
   (if config
     (if (fs/existsSync config)
        ;(edn/read-string (fs/readFileSync #js {:encoding "UTF-8"} config))
-       (edn/read-string (fs/readFileSync config "UTF-8"))
+      (edn/read-string (fs/readFileSync config "UTF-8"))
-       (exit-with-error (str "config file does not exist: " config)))
+      (exit-with-error (str "config file does not exist: " config)))
     nil))
 
 (defn load-credentials-config []
@@ -42,3 +48,22 @@
 
 (defn load-config [config-location]
   (merge (load-main-config config-location) (load-credentials-config)))
+
+(defn resolve-promise [promise result-on-error]
+  (let [done (atom false)
+        result (atom nil)]
+    (-> promise
+        (.then #(do (reset! result %) (reset! done true)))
+        (.catch #(do
+                   (log-error %)
+                   (reset! result result-on-error)
+                   (reset! done true))))
+    (.loopWhile deasync (fn [] (not @done)))
+    @result))
+
+(defn resolve-url [[uri]]
+  (let [used-uri (if (string/starts-with? uri "https://") uri (str "https://" uri))
+        location (-> (fetch used-uri #js {:method "GET" :redirect "manual" :timeout "3000"})
+                     (.then #(.get (.-headers %) "Location"))
+                     (.then #(string/replace % "?mbid=social_twitter" "")))]
+    (resolve-promise location uri)))

src/main/cljs/mastodon_bot/rss_api.cljs

@@ -2,11 +2,15 @@
   (:require
    [orchestra.core :refer-macros [defn-spec]]
    [mastodon-bot.rss-domain :as rd]
+   [mastodon-bot.infra :as infra]
+   [clojure.spec.alpha :as s]
    ["rss-parser" :as rss]))
 
+(s/def ::pos-integer (and #(< 0 %) integer?))
 (defn-spec rss-client any?
-  []
+  [& {:keys [timeout]
-  (rss.))
+      :or {timeout 3000}} (s/keys :opt-un [::pos-integer])]
+  (rss. #js {:timeout timeout}))
 
 (defn-spec parse-feed any?
   [item ::rd/feed-item]  
@@ -17,8 +21,5 @@
                   link)}))
 
 (defn-spec get-feed map?
-  [url string?
+  [url string?]
-   callback fn?]
+  (infra/resolve-promise (.parseURL (rss-client) url) []))
-  (print url)
-  (-> (.parseURL (rss-client) url)
-      (.then callback)))

src/main/cljs/mastodon_bot/transform.cljs

@@ -10,22 +10,7 @@
    [mastodon-bot.rss-api :as ra]
    [mastodon-bot.tumblr-domain :as td]
    [mastodon-bot.tumblr-api :as ta]
-   [mastodon-bot.transform-domain :as trd]
+   [mastodon-bot.transform-domain :as trd]))
-   ["deasync" :as deasync]
-   ["request" :as request]))
-
-(defn resolve-url [[uri]]
-  (try
-    (or
-     (some-> ((deasync request)
-              #js {:method "GET"
-                   :uri (if (string/starts-with? uri "https://") uri (str "https://" uri))
-                   :followRedirect false})
-             (.-headers)
-             (.-location)
-             (string/replace "?mbid=social_twitter" ""))
-     uri)
-    (catch js/Error _ uri)))
 
 (def shortened-url-pattern #"(https?://)?(?:\S+(?::\S*)?@)?(?:(?!(?:10|127)(?:\.\d{1,3}){3})(?!(?:169\.254|192\.168)(?:\.\d{1,3}){2})(?!172\.(?:1[6-9]|2\d|3[0-1])(?:\.\d{1,3}){2})(?:[1-9]\d?|1\d\d|2[01]\d|22[0-3])(?:\.(?:1?\d{1,2}|2[0-4]\d|25[0-5])){2}(?:\.(?:[1-9]\d?|1\d\d|2[0-4]\d|25[0-4]))|(?:(?:[a-z\u00a1-\uffff0-9]-*)*[a-z\u00a1-\uffff0-9]+)(?:\.(?:[a-z\u00a1-\uffff0-9]-*)*[a-z\u00a1-\uffff0-9]+)*(?:\.(?:[a-z\u00a1-\uffff]{2,}))\.?)(?::\d{2,5})?(?:[/?#]\S*)?")
 
@@ -33,7 +18,7 @@
   [resolve-urls? ::trd/resolve-urls?
    input trd/intermediate?]
   (if resolve-urls?
-    (update input :text #(string/replace % shortened-url-pattern resolve-url))
+    (update input :text #(string/replace % shortened-url-pattern infra/resolve-url))
     input))
 
 (defn-spec content-filter-regexes ::trd/content-filters
@@ -58,38 +43,33 @@
   (update input :text #(reduce-kv string/replace % (:replacements transformation))))
 
 (defn-spec post-tweets-to-mastodon any?
-  [mastodon-auth md/mastodon-auth?
+  [tweets any?
+   mastodon-auth md/mastodon-auth?
    transformation ::trd/transformation
    last-post-time any?]
   (let [{:keys [source target resolve-urls?]} transformation]
-    (fn [error tweets response]
+    (->> (infra/js->edn tweets)
-      (if error
+         (map twa/parse-tweet)
-        (infra/exit-with-error error)
+         (filter #(> (:created-at %) last-post-time))
-        (->> (infra/js->edn tweets)
+         (remove #(blocked-content? transformation (:text %)))
-             (map twa/parse-tweet)
+         (map #(intermediate-resolve-urls resolve-urls? %))
-             (filter #(> (:created-at %) last-post-time))
+         (map #(twa/nitter-url source %))
-             (remove #(blocked-content? transformation (:text %)))
+         (map #(perform-replacements transformation %))
-             (map #(intermediate-resolve-urls resolve-urls? %))
+         (map #(ma/intermediate-to-mastodon target %))
-             (map #(twa/nitter-url source %))
+         (ma/post-items mastodon-auth target))))
-             (map #(perform-replacements transformation %))
-             (map #(ma/intermediate-to-mastodon target %))
-             (ma/post-items mastodon-auth target))))))
 
 (defn-spec tweets-to-mastodon any?
   [mastodon-auth md/mastodon-auth?
    twitter-auth twd/twitter-auth?
    transformation ::trd/transformation
    last-post-time any?]
-  (let [{:keys [source target resolve-urls?]} transformation]
+  (let [{:keys [source target resolve-urls?]} transformation
-    (doseq [account (:accounts source)]
+        accounts (:accounts source)]
-      (twa/user-timeline
+    (infra/log (str "processing tweets for " accounts))
-       twitter-auth
+    (doseq [account accounts]
-       source
+      (->  (twa/user-timeline twitter-auth source account)
-       account
+           (post-tweets-to-mastodon mastodon-auth transformation last-post-time)))
-       (post-tweets-to-mastodon 
+    (infra/log "done.")))
-        mastodon-auth
-        transformation
-        last-post-time)))))
 
 (defn-spec post-tumblr-to-mastodon any?
   [mastodon-auth md/mastodon-auth?
@@ -113,7 +93,9 @@
    tumblr-auth td/tumblr-auth?
    transformation ::trd/transformation
    last-post-time any?]
-  (let [{:keys [accounts limit]} transformation]
+  (let [{:keys [source target]} transformation
+        {:keys [accounts limit]} source]
+    (infra/log (str "processing tumblr for " accounts))
     (doseq [account accounts]
       (let [client (ta/tumblr-client tumblr-auth account)]
         (.posts client 
@@ -125,31 +107,30 @@
                 )))))
 
 (defn-spec post-rss-to-mastodon any?
-  [mastodon-auth md/mastodon-auth?
+  [payload any?
+   mastodon-auth md/mastodon-auth?
    transformation ::trd/transformation
    last-post-time any?]
   (let [{:keys [source target resolve-urls?]} transformation]
-    (fn [payload]
+    (->> (infra/js->edn payload)
-      (->> (infra/js->edn payload)
+         (:items)
-           (:items)
+         (map ra/parse-feed)
-           (map ra/parse-feed)
+         (filter #(> (:created-at %) last-post-time))
-           (filter #(> (:created-at %) last-post-time))
+         (remove #(blocked-content? transformation (:text %)))
-           (remove #(blocked-content? transformation (:text %)))
+         (map #(intermediate-resolve-urls resolve-urls? %))
-           (map #(intermediate-resolve-urls resolve-urls? %))
+         (map #(perform-replacements transformation %))
-           (map #(perform-replacements transformation %))
+         (map #(ma/intermediate-to-mastodon target %))
-           (map #(ma/intermediate-to-mastodon target %))
+         (ma/post-items mastodon-auth target))))
-           (ma/post-items mastodon-auth target)))))
 
 
 (defn-spec rss-to-mastodon any?
   [mastodon-auth md/mastodon-auth?
    transformation ::trd/transformation
    last-post-time any?]
-  (let [{:keys [source target]} transformation]
+  (let [{:keys [source target]} transformation
-    (doseq [[name url] (:feeds source)]
+        {:keys [feeds]} source]
-      (ra/get-feed
+    (infra/log (str "processing rss for " feeds))
-       url
+    (doseq [[name url] feeds]
-       (post-rss-to-mastodon
+      (-> (ra/get-feed url)
-        mastodon-auth
+          (post-rss-to-mastodon mastodon-auth transformation last-post-time)))
-        transformation
+    (infra/log "done.")))
-        last-post-time)))))

src/main/cljs/mastodon_bot/twitter_api.cljs

@@ -46,14 +46,14 @@
 (defn-spec user-timeline any?
   [twitter-auth td/twitter-auth?
    source td/twitter-source?
-   account ::td/account
+   account ::td/account]
-   callback fn?]
   (let [{:keys [include-rts? include-replies?]} source]
-    (.get (twitter-client twitter-auth)
+    (infra/resolve-promise
-          "statuses/user_timeline"
+     (.get (twitter-client twitter-auth)
-          #js {:screen_name account
+           "statuses/user_timeline"
-               :tweet_mode "extended"
+           #js {:screen_name account
-               :include_rts (boolean include-rts?)
+                :tweet_mode "extended"
-               :exclude_replies (not (boolean include-replies?))}
+                :include_rts (boolean include-rts?)
-          callback)))
+                :exclude_replies (not (boolean include-replies?))})
+     [])))
   

src/test/cljs/mastodon_bot/infra_test.cljs

@@ -0,0 +1,13 @@
+(ns mastodon-bot.infra-test
+  (:require
+   [cljs.test :refer-macros [deftest is testing run-tests]]
+   [mastodon-bot.infra :as sut]))
+
+;TODO: mbid test
+(deftest test-resolve-uri
+  (is (= "https://www.meissa-gmbh.de"
+         (sut/resolve-url ["https://www.meissa-gmbh.de"])))
+  (is (= "https://www.doesnotexist-blablabla.de"
+         (sut/resolve-url ["https://www.doesnotexist-blablabla.de"])))
+  (is (= "http://www.google.de/"
+         (sut/resolve-url ["https://t1p.de/44oo"]))))

src/test/cljs/mastodon_bot/mastodon_api_test.cljs

@@ -49,4 +49,4 @@ https://chrisuehlinger.com/blog/2020/06/16/unshattering-the-audience-building-th
          (sut/intermediate-to-mastodon {:target-type :mastodon
                                         :append-screen-name? false
                                         :max-post-length 500}
-                                       intermediate-rss-item))))
+                                       intermediate-rss-item))))

src/test/cljs/mastodon_bot/transform_test.cljs

@@ -16,4 +16,4 @@
   (is (=
     "💠 Check out what has been going on during March in the world of @ReproBuilds! 💠 https://t.co/k6NsSO115z @opensuse@fosstodon.org @conservancy@mastodon.technology @PrototypeFund@mastodon.social @debian@fosstodon.org "
     (:text (sut/perform-replacements (first (:transform testconfig)) (twitter/parse-tweet (readfile "testdata/twitter/tweet-mentions.edn"))))
-    )))
+    )))