Merge branch 'update-dependencies' into 'master'

node-fetch instead of request

See merge request yogthos/mastodon-bot!11

.github/workflows/stable.yml

@@ -1,89 +0,0 @@
-name: stable
-on:
-  push:
-    tags: '[0-9]+.[0-9]+.[0-9]+*'
-
-jobs:
-  stable:
-    name: stable
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        node-version: [14.x]
-
-    steps:
-    - uses: actions/checkout@v2
-          
-    - name: Use Node.js ${{ matrix.node-version }}
-      uses: actions/setup-node@v1
-      with:
-        node-version: ${{ matrix.node-version }}
-    
-    - name: test em
-      run: | 
-        npm install
-        npm install -g --save-dev shadow-cljs
-        shadow-cljs compile test
-
-    - name: build em
-      run: |
-        shadow-cljs release app
-        chmod a+x mastodon-bot.js
-        sha256sum mastodon-bot.js > target/mastodon-bot.js.sha256
-        sha512sum mastodon-bot.js > target/mastodon-bot.js.sha512
-
-    - name: Create Release
-      id: create_release
-      uses: actions/create-release@v1
-      env:
-        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      with:
-        tag_name: ${{ github.ref }}
-        release_name: Release ${{ github.ref }}
-        draft: false
-        prerelease: false
-
-    - name: Upload mastodon-bot.js
-      id: upload-mastodon-bot-js
-      uses: actions/upload-release-asset@v1
-      env:
-        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      with:
-        upload_url: ${{ steps.create_release.outputs.upload_url }}
-        asset_path: ./mastodon-bot.js
-        asset_name: mastodon-bot.js
-        asset_content_type: application/javascript
-
-    - name: Upload mastodon-bot.js.sha256
-      id: upload-mastodon-bot-js-sha256
-      uses: actions/upload-release-asset@v1
-      env:
-        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      with:
-        upload_url: ${{ steps.create_release.outputs.upload_url }}
-        asset_path: ./target/mastodon-bot.js.sha256
-        asset_name: mastodon-bot.js.sha256
-        asset_content_type: text/plain
-
-    - name: Upload mastodon-bot.js.sha512
-      id: upload-mastodon-bot-js-sha512
-      uses: actions/upload-release-asset@v1
-      env:
-        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      with:
-        upload_url: ${{ steps.create_release.outputs.upload_url }}
-        asset_path: ./target/mastodon-bot.js.sha512
-        asset_name: mastodon-bot.js.sha512
-        asset_content_type: text/plain
-
-    - name: upload to npm
-      env:
-        NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}
-      run: |
-        mkdir -p target/npm-build/mastodon_bot
-        cp mastodon-bot.js target/npm-build/mastodon_bot/
-        cp target/mastodon-bot.js.sha256 target/npm-build/mastodon_bot/
-        cp target/mastodon-bot.js.sha512 target/npm-build/mastodon_bot/
-        cp package.json target/npm-build/mastodon_bot/
-        cp README.md target/npm-build/mastodon_bot/
-        npm publish ./target/npm-build/mastodon_bot --access public

doc/rss.puml

@@ -0,0 +1,10 @@
+@startuml
+"Mastodon-Bot" -> Config: Call rss-sources
+Config --> "Mastodon-Bot": URL list of rss-sources
+"Mastodon-Bot" -> "Mastodon-Bot": Call "get-feed" for every rss-source.\nStart asyncronous Promise.
+activate "Mastodon-Bot"
+"Mastodon-Bot" -> "RSS-Sources": Call URL to parse RSS-Feed
+"RSS-Sources" --> "Mastodon-Bot": RSS-Feed
+"Mastodon-Bot" -> Mastodon: Post feed by post-rss-to-mastodon
+deactivate "Mastodon-Bot"
+@enduml

package.json

@@ -7,12 +7,13 @@
   "repository": "https://www.npmjs.com/package/mastodon-bot",
   "license": "MIT",
   "dependencies": {
+    "deasync": "0.1.20",
     "mastodon-api": "1.3.0",
+    "node-fetch": "2.6.1",
+    "request": "2.88.0",
     "rss-parser": "3.7.1",
     "tumblr": "0.4.1",
-    "twitter": "1.7.1",
+    "twitter": "1.7.1"
-    "deasync": "0.1.20",
-    "request": "2.88.0"
   },
   "devDependencies": {
     "shadow-cljs": "^2.8.37"

src/main/cljs/mastodon_bot/infra.cljs

@@ -1,15 +1,17 @@
 (ns mastodon-bot.infra
   (:require
    [cljs.reader :as edn]
-   [clojure.pprint :refer [pprint]]
+   [clojure.string :as string]
-   ["fs" :as fs]))
+   ["fs" :as fs]
+   ["deasync" :as deasync]
+   ["node-fetch" :as fetch]))
 
 (defn debug [item]
-  (pprint item)
+  (js/console.log item)
   item)
 
 (defn debug-first [item]
-  (pprint (first item))
+  (js/console.log (first item))
   item)
 
 (defn js->edn [data]
@@ -28,8 +30,8 @@
   (if config
     (if (fs/existsSync config)
        ;(edn/read-string (fs/readFileSync #js {:encoding "UTF-8"} config))
-       (edn/read-string (fs/readFileSync config "UTF-8"))
+      (edn/read-string (fs/readFileSync config "UTF-8"))
-       (exit-with-error (str "config file does not exist: " config)))
+      (exit-with-error (str "config file does not exist: " config)))
     nil))
 
 (defn load-credentials-config []
@@ -42,3 +44,19 @@
 
 (defn load-config [config-location]
   (merge (load-main-config config-location) (load-credentials-config)))
+
+(defn resolve-promise [promise result-on-error]
+  (let [done (atom false)
+        result (atom nil)
+        promise (-> promise
+                    (.then #(do (reset! result %) (reset! done true)))
+                    (.catch #(do (reset! result result-on-error) (reset! done true))))]
+    (.loopWhile deasync (fn [] (not @done)))
+    @result))
+
+(defn resolve-url [[uri]]
+  (let [used-uri (if (string/starts-with? uri "https://") uri (str "https://" uri))
+        location (-> (fetch used-uri #js {:method "GET" :redirect "manual" :timeout "3000"})
+                     (.then #(.get (.-headers %) "Location"))
+                     (.then #(string/replace % "?mbid=social_twitter" "")))]
+    (resolve-promise location uri)))

src/main/cljs/mastodon_bot/rss_api.cljs

@@ -2,11 +2,14 @@
   (:require
    [orchestra.core :refer-macros [defn-spec]]
    [mastodon-bot.rss-domain :as rd]
+   [clojure.spec.alpha :as s]
    ["rss-parser" :as rss]))
 
+(s/def ::pos-integer (and #(< 0 %) integer?))
 (defn-spec rss-client any?
-  []
+  [& {:keys [timeout]
-  (rss.))
+      :or {timeout 3000}} (s/keys :opt-un [::pos-integer])]
+  (rss. #js {:timeout timeout}))
 
 (defn-spec parse-feed any?
   [item ::rd/feed-item]  
@@ -21,4 +24,5 @@
    callback fn?]
   (print url)
   (-> (.parseURL (rss-client) url)
-      (.then callback)))
+      (.then callback)
+      (.catch #(js/console.log %))))

src/main/cljs/mastodon_bot/transform.cljs

@@ -10,22 +10,7 @@
    [mastodon-bot.rss-api :as ra]
    [mastodon-bot.tumblr-domain :as td]
    [mastodon-bot.tumblr-api :as ta]
-   [mastodon-bot.transform-domain :as trd]
+   [mastodon-bot.transform-domain :as trd]))
-   ["deasync" :as deasync]
-   ["request" :as request]))
-
-(defn resolve-url [[uri]]
-  (try
-    (or
-     (some-> ((deasync request)
-              #js {:method "GET"
-                   :uri (if (string/starts-with? uri "https://") uri (str "https://" uri))
-                   :followRedirect false})
-             (.-headers)
-             (.-location)
-             (string/replace "?mbid=social_twitter" ""))
-     uri)
-    (catch js/Error _ uri)))
 
 (def shortened-url-pattern #"(https?://)?(?:\S+(?::\S*)?@)?(?:(?!(?:10|127)(?:\.\d{1,3}){3})(?!(?:169\.254|192\.168)(?:\.\d{1,3}){2})(?!172\.(?:1[6-9]|2\d|3[0-1])(?:\.\d{1,3}){2})(?:[1-9]\d?|1\d\d|2[01]\d|22[0-3])(?:\.(?:1?\d{1,2}|2[0-4]\d|25[0-5])){2}(?:\.(?:[1-9]\d?|1\d\d|2[0-4]\d|25[0-4]))|(?:(?:[a-z\u00a1-\uffff0-9]-*)*[a-z\u00a1-\uffff0-9]+)(?:\.(?:[a-z\u00a1-\uffff0-9]-*)*[a-z\u00a1-\uffff0-9]+)*(?:\.(?:[a-z\u00a1-\uffff]{2,}))\.?)(?::\d{2,5})?(?:[/?#]\S*)?")
 
@@ -33,7 +18,7 @@
   [resolve-urls? ::trd/resolve-urls?
    input trd/intermediate?]
   (if resolve-urls?
-    (update input :text #(string/replace % shortened-url-pattern resolve-url))
+    (update input :text #(string/replace % shortened-url-pattern infra/resolve-url))
     input))
 
 (defn-spec content-filter-regexes ::trd/content-filters

src/test/cljs/mastodon_bot/infra_test.cljs

@@ -0,0 +1,13 @@
+(ns mastodon-bot.infra-test
+  (:require
+   [cljs.test :refer-macros [deftest is testing run-tests]]
+   [mastodon-bot.infra :as sut]))
+
+;TODO: mbid test
+(deftest test-resolve-uri
+  (is (= "https://www.meissa-gmbh.de"
+         (sut/resolve-url ["https://www.meissa-gmbh.de"])))
+  (is (= "https://www.doesnotexist-blablabla.de"
+         (sut/resolve-url ["https://www.doesnotexist-blablabla.de"])))
+  (is (= "http://www.google.de/"
+         (sut/resolve-url ["https://t1p.de/44oo"]))))

src/test/cljs/mastodon_bot/mastodon_api_test.cljs

@@ -49,4 +49,4 @@ https://chrisuehlinger.com/blog/2020/06/16/unshattering-the-audience-building-th
          (sut/intermediate-to-mastodon {:target-type :mastodon
                                         :append-screen-name? false
                                         :max-post-length 500}
-                                       intermediate-rss-item))))
+                                       intermediate-rss-item))))

src/test/cljs/mastodon_bot/transform_test.cljs

@@ -16,4 +16,4 @@
   (is (=
     "💠 Check out what has been going on during March in the world of @ReproBuilds! 💠 https://t.co/k6NsSO115z @opensuse@fosstodon.org @conservancy@mastodon.technology @PrototypeFund@mastodon.social @debian@fosstodon.org "
     (:text (sut/perform-replacements (first (:transform testconfig)) (twitter/parse-tweet (readfile "testdata/twitter/tweet-mentions.edn"))))
-    )))
+    )))