.github/workflows/stable.yml

@@ -0,0 +1,89 @@
+name: stable
+on:
+  push:
+    tags: '[0-9]+.[0-9]+.[0-9]+*'
+
+jobs:
+  stable:
+    name: stable
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        node-version: [14.x]
+
+    steps:
+    - uses: actions/checkout@v2
+          
+    - name: Use Node.js ${{ matrix.node-version }}
+      uses: actions/setup-node@v1
+      with:
+        node-version: ${{ matrix.node-version }}
+    
+    - name: test em
+      run: | 
+        npm install
+        npm install -g --save-dev shadow-cljs
+        shadow-cljs compile test
+
+    - name: build em
+      run: |
+        shadow-cljs release app
+        chmod a+x mastodon-bot.js
+        sha256sum mastodon-bot.js > target/mastodon-bot.js.sha256
+        sha512sum mastodon-bot.js > target/mastodon-bot.js.sha512
+
+    - name: Create Release
+      id: create_release
+      uses: actions/create-release@v1
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      with:
+        tag_name: ${{ github.ref }}
+        release_name: Release ${{ github.ref }}
+        draft: false
+        prerelease: false
+
+    - name: Upload mastodon-bot.js
+      id: upload-mastodon-bot-js
+      uses: actions/upload-release-asset@v1
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      with:
+        upload_url: ${{ steps.create_release.outputs.upload_url }}
+        asset_path: ./mastodon-bot.js
+        asset_name: mastodon-bot.js
+        asset_content_type: application/javascript
+
+    - name: Upload mastodon-bot.js.sha256
+      id: upload-mastodon-bot-js-sha256
+      uses: actions/upload-release-asset@v1
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      with:
+        upload_url: ${{ steps.create_release.outputs.upload_url }}
+        asset_path: ./target/mastodon-bot.js.sha256
+        asset_name: mastodon-bot.js.sha256
+        asset_content_type: text/plain
+
+    - name: Upload mastodon-bot.js.sha512
+      id: upload-mastodon-bot-js-sha512
+      uses: actions/upload-release-asset@v1
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      with:
+        upload_url: ${{ steps.create_release.outputs.upload_url }}
+        asset_path: ./target/mastodon-bot.js.sha512
+        asset_name: mastodon-bot.js.sha512
+        asset_content_type: text/plain
+
+    - name: upload to npm
+      env:
+        NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}
+      run: |
+        mkdir -p target/npm-build/mastodon_bot
+        cp mastodon-bot.js target/npm-build/mastodon_bot/
+        cp target/mastodon-bot.js.sha256 target/npm-build/mastodon_bot/
+        cp target/mastodon-bot.js.sha512 target/npm-build/mastodon_bot/
+        cp package.json target/npm-build/mastodon_bot/
+        cp README.md target/npm-build/mastodon_bot/
+        npm publish ./target/npm-build/mastodon_bot --access public

doc/Releasing.md

@@ -5,7 +5,6 @@
 vi package.json
 
 lein release
-git push --follow-tags
 
 # bump version - increase version and add -SNAPSHOT
 vi package.json

doc/rss.puml

@@ -1,10 +0,0 @@
-@startuml
-"Mastodon-Bot" -> Config: Call rss-sources
-Config --> "Mastodon-Bot": URL list of rss-sources
-"Mastodon-Bot" -> "Mastodon-Bot": Call "get-feed" for every rss-source.\nStart asyncronous Promise.
-activate "Mastodon-Bot"
-"Mastodon-Bot" -> "RSS-Sources": Call URL to parse RSS-Feed
-"RSS-Sources" --> "Mastodon-Bot": RSS-Feed
-"Mastodon-Bot" -> Mastodon: Post feed by post-rss-to-mastodon
-deactivate "Mastodon-Bot"
-@enduml

package.json

@@ -2,18 +2,17 @@
   "name": "mastodon-bot",
   "description": "Bot to publish twitter, tumblr or rss posts to an mastodon account.",
   "author": "Dmitri Sotnikov",
-  "version": "1.12.1-SNAPSHOT",
+  "version": "1.10.12-SNAPSHOT",
   "homepage": "https://github.com/yogthos/mastodon-bot",
   "repository": "https://www.npmjs.com/package/mastodon-bot",
   "license": "MIT",
   "dependencies": {
-    "deasync": "0.1.20",
     "mastodon-api": "1.3.0",
-    "node-fetch": "2.6.1",
-    "request": "2.88.0",
     "rss-parser": "3.7.1",
     "tumblr": "0.4.1",
-    "twitter": "1.7.1"
+    "twitter": "1.7.1",
+    "deasync": "0.1.20",
+    "request": "2.88.0"
   },
   "devDependencies": {
     "shadow-cljs": "^2.8.37"

project.clj

@@ -1,4 +1,4 @@
-(defproject dda/mastodon-bot "1.12.1-SNAPSHOT"
+(defproject dda/mastodon-bot "1.10.12-SNAPSHOT"
   :description "Bot to publish twitter, tumblr or rss posts to an mastodon account."
   :url "https://github.com/yogthos/mastodon-bot"
   :author "Dmitri Sotnikov"

src/main/cljs/mastodon_bot/infra.cljs

@@ -1,21 +1,15 @@
 (ns mastodon-bot.infra
   (:require
    [cljs.reader :as edn]
-   [clojure.string :as string]
+   [clojure.pprint :refer [pprint]]
-   ["fs" :as fs]
+   ["fs" :as fs]))
-   ["deasync" :as deasync]
-   ["node-fetch" :as fetch]))
 
-(defn log-error [item]
+(defn debug [item]
-  (js/console.error item)
+  (pprint item)
   item)
 
-(defn log [item]
+(defn debug-first [item]
-  (js/console.log item)
+  (pprint (first item))
-  item)
-
-(defn log-first [item]
-  (js/console.log (first item))
   item)
 
 (defn js->edn [data]
@@ -34,8 +28,8 @@
   (if config
     (if (fs/existsSync config)
        ;(edn/read-string (fs/readFileSync #js {:encoding "UTF-8"} config))
-      (edn/read-string (fs/readFileSync config "UTF-8"))
+       (edn/read-string (fs/readFileSync config "UTF-8"))
-      (exit-with-error (str "config file does not exist: " config)))
+       (exit-with-error (str "config file does not exist: " config)))
     nil))
 
 (defn load-credentials-config []
@@ -48,22 +42,3 @@
 
 (defn load-config [config-location]
   (merge (load-main-config config-location) (load-credentials-config)))
-
-(defn resolve-promise [promise result-on-error]
-  (let [done (atom false)
-        result (atom nil)]
-    (-> promise
-        (.then #(do (reset! result %) (reset! done true)))
-        (.catch #(do
-                   (log-error %)
-                   (reset! result result-on-error)
-                   (reset! done true))))
-    (.loopWhile deasync (fn [] (not @done)))
-    @result))
-
-(defn resolve-url [[uri]]
-  (let [used-uri (if (string/starts-with? uri "https://") uri (str "https://" uri))
-        location (-> (fetch used-uri #js {:method "GET" :redirect "manual" :timeout "3000"})
-                     (.then #(.get (.-headers %) "Location"))
-                     (.then #(string/replace % "?mbid=social_twitter" "")))]
-    (resolve-promise location uri)))

src/main/cljs/mastodon_bot/rss_api.cljs

@@ -2,15 +2,11 @@
   (:require
    [orchestra.core :refer-macros [defn-spec]]
    [mastodon-bot.rss-domain :as rd]
-   [mastodon-bot.infra :as infra]
-   [clojure.spec.alpha :as s]
    ["rss-parser" :as rss]))
 
-(s/def ::pos-integer (and #(< 0 %) integer?))
 (defn-spec rss-client any?
-  [& {:keys [timeout]
+  []
-      :or {timeout 3000}} (s/keys :opt-un [::pos-integer])]
+  (rss.))
-  (rss. #js {:timeout timeout}))
 
 (defn-spec parse-feed any?
   [item ::rd/feed-item]  
@@ -21,5 +17,8 @@
                   link)}))
 
 (defn-spec get-feed map?
-  [url string?]
+  [url string?
-  (infra/resolve-promise (.parseURL (rss-client) url) []))
+   callback fn?]
+  (print url)
+  (-> (.parseURL (rss-client) url)
+      (.then callback)))

src/main/cljs/mastodon_bot/transform.cljs

@@ -10,7 +10,22 @@
    [mastodon-bot.rss-api :as ra]
    [mastodon-bot.tumblr-domain :as td]
    [mastodon-bot.tumblr-api :as ta]
-   [mastodon-bot.transform-domain :as trd]))
+   [mastodon-bot.transform-domain :as trd]
+   ["deasync" :as deasync]
+   ["request" :as request]))
+
+(defn resolve-url [[uri]]
+  (try
+    (or
+     (some-> ((deasync request)
+              #js {:method "GET"
+                   :uri (if (string/starts-with? uri "https://") uri (str "https://" uri))
+                   :followRedirect false})
+             (.-headers)
+             (.-location)
+             (string/replace "?mbid=social_twitter" ""))
+     uri)
+    (catch js/Error _ uri)))
 
 (def shortened-url-pattern #"(https?://)?(?:\S+(?::\S*)?@)?(?:(?!(?:10|127)(?:\.\d{1,3}){3})(?!(?:169\.254|192\.168)(?:\.\d{1,3}){2})(?!172\.(?:1[6-9]|2\d|3[0-1])(?:\.\d{1,3}){2})(?:[1-9]\d?|1\d\d|2[01]\d|22[0-3])(?:\.(?:1?\d{1,2}|2[0-4]\d|25[0-5])){2}(?:\.(?:[1-9]\d?|1\d\d|2[0-4]\d|25[0-4]))|(?:(?:[a-z\u00a1-\uffff0-9]-*)*[a-z\u00a1-\uffff0-9]+)(?:\.(?:[a-z\u00a1-\uffff0-9]-*)*[a-z\u00a1-\uffff0-9]+)*(?:\.(?:[a-z\u00a1-\uffff]{2,}))\.?)(?::\d{2,5})?(?:[/?#]\S*)?")
 
@@ -18,7 +33,7 @@
   [resolve-urls? ::trd/resolve-urls?
    input trd/intermediate?]
   (if resolve-urls?
-    (update input :text #(string/replace % shortened-url-pattern infra/resolve-url))
+    (update input :text #(string/replace % shortened-url-pattern resolve-url))
     input))
 
 (defn-spec content-filter-regexes ::trd/content-filters
@@ -43,33 +58,38 @@
   (update input :text #(reduce-kv string/replace % (:replacements transformation))))
 
 (defn-spec post-tweets-to-mastodon any?
-  [tweets any?
+  [mastodon-auth md/mastodon-auth?
-   mastodon-auth md/mastodon-auth?
    transformation ::trd/transformation
    last-post-time any?]
   (let [{:keys [source target resolve-urls?]} transformation]
-    (->> (infra/js->edn tweets)
+    (fn [error tweets response]
-         (map twa/parse-tweet)
+      (if error
-         (filter #(> (:created-at %) last-post-time))
+        (infra/exit-with-error error)
-         (remove #(blocked-content? transformation (:text %)))
+        (->> (infra/js->edn tweets)
-         (map #(intermediate-resolve-urls resolve-urls? %))
+             (map twa/parse-tweet)
-         (map #(twa/nitter-url source %))
+             (filter #(> (:created-at %) last-post-time))
-         (map #(perform-replacements transformation %))
+             (remove #(blocked-content? transformation (:text %)))
-         (map #(ma/intermediate-to-mastodon target %))
+             (map #(intermediate-resolve-urls resolve-urls? %))
-         (ma/post-items mastodon-auth target))))
+             (map #(twa/nitter-url source %))
+             (map #(perform-replacements transformation %))
+             (map #(ma/intermediate-to-mastodon target %))
+             (ma/post-items mastodon-auth target))))))
 
 (defn-spec tweets-to-mastodon any?
   [mastodon-auth md/mastodon-auth?
    twitter-auth twd/twitter-auth?
    transformation ::trd/transformation
    last-post-time any?]
-  (let [{:keys [source target resolve-urls?]} transformation
+  (let [{:keys [source target resolve-urls?]} transformation]
-        accounts (:accounts source)]
+    (doseq [account (:accounts source)]
-    (infra/log (str "processing tweets for " accounts))
+      (twa/user-timeline
-    (doseq [account accounts]
+       twitter-auth
-      (->  (twa/user-timeline twitter-auth source account)
+       source
-           (post-tweets-to-mastodon mastodon-auth transformation last-post-time)))
+       account
-    (infra/log "done.")))
+       (post-tweets-to-mastodon 
+        mastodon-auth
+        transformation
+        last-post-time)))))
 
 (defn-spec post-tumblr-to-mastodon any?
   [mastodon-auth md/mastodon-auth?
@@ -93,9 +113,7 @@
    tumblr-auth td/tumblr-auth?
    transformation ::trd/transformation
    last-post-time any?]
-  (let [{:keys [source target]} transformation
+  (let [{:keys [accounts limit]} transformation]
-        {:keys [accounts limit]} source]
-    (infra/log (str "processing tumblr for " accounts))
     (doseq [account accounts]
       (let [client (ta/tumblr-client tumblr-auth account)]
         (.posts client 
@@ -107,30 +125,31 @@
                 )))))
 
 (defn-spec post-rss-to-mastodon any?
-  [payload any?
+  [mastodon-auth md/mastodon-auth?
-   mastodon-auth md/mastodon-auth?
    transformation ::trd/transformation
    last-post-time any?]
   (let [{:keys [source target resolve-urls?]} transformation]
-    (->> (infra/js->edn payload)
+    (fn [payload]
-         (:items)
+      (->> (infra/js->edn payload)
-         (map ra/parse-feed)
+           (:items)
-         (filter #(> (:created-at %) last-post-time))
+           (map ra/parse-feed)
-         (remove #(blocked-content? transformation (:text %)))
+           (filter #(> (:created-at %) last-post-time))
-         (map #(intermediate-resolve-urls resolve-urls? %))
+           (remove #(blocked-content? transformation (:text %)))
-         (map #(perform-replacements transformation %))
+           (map #(intermediate-resolve-urls resolve-urls? %))
-         (map #(ma/intermediate-to-mastodon target %))
+           (map #(perform-replacements transformation %))
-         (ma/post-items mastodon-auth target))))
+           (map #(ma/intermediate-to-mastodon target %))
+           (ma/post-items mastodon-auth target)))))
 
 
 (defn-spec rss-to-mastodon any?
   [mastodon-auth md/mastodon-auth?
    transformation ::trd/transformation
    last-post-time any?]
-  (let [{:keys [source target]} transformation
+  (let [{:keys [source target]} transformation]
-        {:keys [feeds]} source]
+    (doseq [[name url] (:feeds source)]
-    (infra/log (str "processing rss for " feeds))
+      (ra/get-feed
-    (doseq [[name url] feeds]
+       url
-      (-> (ra/get-feed url)
+       (post-rss-to-mastodon
-          (post-rss-to-mastodon mastodon-auth transformation last-post-time)))
+        mastodon-auth
-    (infra/log "done.")))
+        transformation
+        last-post-time)))))

src/main/cljs/mastodon_bot/twitter_api.cljs

@@ -46,14 +46,14 @@
 (defn-spec user-timeline any?
   [twitter-auth td/twitter-auth?
    source td/twitter-source?
-   account ::td/account]
+   account ::td/account
+   callback fn?]
   (let [{:keys [include-rts? include-replies?]} source]
-    (infra/resolve-promise
+    (.get (twitter-client twitter-auth)
-     (.get (twitter-client twitter-auth)
+          "statuses/user_timeline"
-           "statuses/user_timeline"
+          #js {:screen_name account
-           #js {:screen_name account
+               :tweet_mode "extended"
-                :tweet_mode "extended"
+               :include_rts (boolean include-rts?)
-                :include_rts (boolean include-rts?)
+               :exclude_replies (not (boolean include-replies?))}
-                :exclude_replies (not (boolean include-replies?))})
+          callback)))
-     [])))
   

src/test/cljs/mastodon_bot/infra_test.cljs

@@ -1,13 +0,0 @@
-(ns mastodon-bot.infra-test
-  (:require
-   [cljs.test :refer-macros [deftest is testing run-tests]]
-   [mastodon-bot.infra :as sut]))
-
-;TODO: mbid test
-(deftest test-resolve-uri
-  (is (= "https://www.meissa-gmbh.de"
-         (sut/resolve-url ["https://www.meissa-gmbh.de"])))
-  (is (= "https://www.doesnotexist-blablabla.de"
-         (sut/resolve-url ["https://www.doesnotexist-blablabla.de"])))
-  (is (= "http://www.google.de/"
-         (sut/resolve-url ["https://t1p.de/44oo"]))))