diff --git a/.github/workflows/stable.yml b/.github/workflows/stable.yml deleted file mode 100644 index 11a63ab..0000000 --- a/.github/workflows/stable.yml +++ /dev/null @@ -1,89 +0,0 @@ -name: stable -on: - push: - tags: '[0-9]+.[0-9]+.[0-9]+*' - -jobs: - stable: - name: stable - runs-on: ubuntu-latest - strategy: - matrix: - node-version: [14.x] - - steps: - - uses: actions/checkout@v2 - - - name: Use Node.js ${{ matrix.node-version }} - uses: actions/setup-node@v1 - with: - node-version: ${{ matrix.node-version }} - - - name: test em - run: | - npm install - npm install -g --save-dev shadow-cljs - shadow-cljs compile test - - - name: build em - run: | - shadow-cljs release app - chmod a+x mastodon-bot.js - sha256sum mastodon-bot.js > target/mastodon-bot.js.sha256 - sha512sum mastodon-bot.js > target/mastodon-bot.js.sha512 - - - name: Create Release - id: create_release - uses: actions/create-release@v1 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - tag_name: ${{ github.ref }} - release_name: Release ${{ github.ref }} - draft: false - prerelease: false - - - name: Upload mastodon-bot.js - id: upload-mastodon-bot-js - uses: actions/upload-release-asset@v1 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - upload_url: ${{ steps.create_release.outputs.upload_url }} - asset_path: ./mastodon-bot.js - asset_name: mastodon-bot.js - asset_content_type: application/javascript - - - name: Upload mastodon-bot.js.sha256 - id: upload-mastodon-bot-js-sha256 - uses: actions/upload-release-asset@v1 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - upload_url: ${{ steps.create_release.outputs.upload_url }} - asset_path: ./target/mastodon-bot.js.sha256 - asset_name: mastodon-bot.js.sha256 - asset_content_type: text/plain - - - name: Upload mastodon-bot.js.sha512 - id: upload-mastodon-bot-js-sha512 - uses: actions/upload-release-asset@v1 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - upload_url: ${{ steps.create_release.outputs.upload_url }} - asset_path: ./target/mastodon-bot.js.sha512 - asset_name: mastodon-bot.js.sha512 - asset_content_type: text/plain - - - name: upload to npm - env: - NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} - run: | - mkdir -p target/npm-build/mastodon_bot - cp mastodon-bot.js target/npm-build/mastodon_bot/ - cp target/mastodon-bot.js.sha256 target/npm-build/mastodon_bot/ - cp target/mastodon-bot.js.sha512 target/npm-build/mastodon_bot/ - cp package.json target/npm-build/mastodon_bot/ - cp README.md target/npm-build/mastodon_bot/ - npm publish ./target/npm-build/mastodon_bot --access public diff --git a/doc/rss.png b/doc/rss.png new file mode 100644 index 0000000..d35bf0b Binary files /dev/null and b/doc/rss.png differ diff --git a/doc/rss.puml b/doc/rss.puml new file mode 100644 index 0000000..53058c1 --- /dev/null +++ b/doc/rss.puml @@ -0,0 +1,10 @@ +@startuml +"Mastodon-Bot" -> Config: Call rss-sources +Config --> "Mastodon-Bot": URL list of rss-sources +"Mastodon-Bot" -> "Mastodon-Bot": Call "get-feed" for every rss-source.\nStart asyncronous Promise. +activate "Mastodon-Bot" +"Mastodon-Bot" -> "RSS-Sources": Call URL to parse RSS-Feed +"RSS-Sources" --> "Mastodon-Bot": RSS-Feed +"Mastodon-Bot" -> Mastodon: Post feed by post-rss-to-mastodon +deactivate "Mastodon-Bot" +@enduml \ No newline at end of file diff --git a/package.json b/package.json index 33ebed5..a40b5b7 100644 --- a/package.json +++ b/package.json @@ -7,12 +7,13 @@ "repository": "https://www.npmjs.com/package/mastodon-bot", "license": "MIT", "dependencies": { + "deasync": "0.1.20", "mastodon-api": "1.3.0", + "node-fetch": "2.6.1", + "request": "2.88.0", "rss-parser": "3.7.1", "tumblr": "0.4.1", - "twitter": "1.7.1", - "deasync": "0.1.20", - "request": "2.88.0" + "twitter": "1.7.1" }, "devDependencies": { "shadow-cljs": "^2.8.37" diff --git a/src/main/cljs/mastodon_bot/infra.cljs b/src/main/cljs/mastodon_bot/infra.cljs index f2f242f..567f9f5 100755 --- a/src/main/cljs/mastodon_bot/infra.cljs +++ b/src/main/cljs/mastodon_bot/infra.cljs @@ -1,15 +1,17 @@ (ns mastodon-bot.infra (:require [cljs.reader :as edn] - [clojure.pprint :refer [pprint]] - ["fs" :as fs])) + [clojure.string :as string] + ["fs" :as fs] + ["deasync" :as deasync] + ["node-fetch" :as fetch])) (defn debug [item] - (pprint item) + (js/console.log item) item) (defn debug-first [item] - (pprint (first item)) + (js/console.log (first item)) item) (defn js->edn [data] @@ -28,8 +30,8 @@ (if config (if (fs/existsSync config) ;(edn/read-string (fs/readFileSync #js {:encoding "UTF-8"} config)) - (edn/read-string (fs/readFileSync config "UTF-8")) - (exit-with-error (str "config file does not exist: " config))) + (edn/read-string (fs/readFileSync config "UTF-8")) + (exit-with-error (str "config file does not exist: " config))) nil)) (defn load-credentials-config [] @@ -42,3 +44,19 @@ (defn load-config [config-location] (merge (load-main-config config-location) (load-credentials-config))) + +(defn resolve-promise [promise result-on-error] + (let [done (atom false) + result (atom nil) + promise (-> promise + (.then #(do (reset! result %) (reset! done true))) + (.catch #(do (reset! result result-on-error) (reset! done true))))] + (.loopWhile deasync (fn [] (not @done))) + @result)) + +(defn resolve-url [[uri]] + (let [used-uri (if (string/starts-with? uri "https://") uri (str "https://" uri)) + location (-> (fetch used-uri #js {:method "GET" :redirect "manual" :timeout "3000"}) + (.then #(.get (.-headers %) "Location")) + (.then #(string/replace % "?mbid=social_twitter" "")))] + (resolve-promise location uri))) diff --git a/src/main/cljs/mastodon_bot/rss_api.cljs b/src/main/cljs/mastodon_bot/rss_api.cljs index 17efb56..a34b771 100755 --- a/src/main/cljs/mastodon_bot/rss_api.cljs +++ b/src/main/cljs/mastodon_bot/rss_api.cljs @@ -2,11 +2,14 @@ (:require [orchestra.core :refer-macros [defn-spec]] [mastodon-bot.rss-domain :as rd] + [clojure.spec.alpha :as s] ["rss-parser" :as rss])) +(s/def ::pos-integer (and #(< 0 %) integer?)) (defn-spec rss-client any? - [] - (rss.)) + [& {:keys [timeout] + :or {timeout 3000}} (s/keys :opt-un [::pos-integer])] + (rss. #js {:timeout timeout})) (defn-spec parse-feed any? [item ::rd/feed-item] @@ -21,4 +24,5 @@ callback fn?] (print url) (-> (.parseURL (rss-client) url) - (.then callback))) + (.then callback) + (.catch #(js/console.log %)))) diff --git a/src/main/cljs/mastodon_bot/transform.cljs b/src/main/cljs/mastodon_bot/transform.cljs index 635f8b1..0422274 100644 --- a/src/main/cljs/mastodon_bot/transform.cljs +++ b/src/main/cljs/mastodon_bot/transform.cljs @@ -10,22 +10,7 @@ [mastodon-bot.rss-api :as ra] [mastodon-bot.tumblr-domain :as td] [mastodon-bot.tumblr-api :as ta] - [mastodon-bot.transform-domain :as trd] - ["deasync" :as deasync] - ["request" :as request])) - -(defn resolve-url [[uri]] - (try - (or - (some-> ((deasync request) - #js {:method "GET" - :uri (if (string/starts-with? uri "https://") uri (str "https://" uri)) - :followRedirect false}) - (.-headers) - (.-location) - (string/replace "?mbid=social_twitter" "")) - uri) - (catch js/Error _ uri))) + [mastodon-bot.transform-domain :as trd])) (def shortened-url-pattern #"(https?://)?(?:\S+(?::\S*)?@)?(?:(?!(?:10|127)(?:\.\d{1,3}){3})(?!(?:169\.254|192\.168)(?:\.\d{1,3}){2})(?!172\.(?:1[6-9]|2\d|3[0-1])(?:\.\d{1,3}){2})(?:[1-9]\d?|1\d\d|2[01]\d|22[0-3])(?:\.(?:1?\d{1,2}|2[0-4]\d|25[0-5])){2}(?:\.(?:[1-9]\d?|1\d\d|2[0-4]\d|25[0-4]))|(?:(?:[a-z\u00a1-\uffff0-9]-*)*[a-z\u00a1-\uffff0-9]+)(?:\.(?:[a-z\u00a1-\uffff0-9]-*)*[a-z\u00a1-\uffff0-9]+)*(?:\.(?:[a-z\u00a1-\uffff]{2,}))\.?)(?::\d{2,5})?(?:[/?#]\S*)?") @@ -33,7 +18,7 @@ [resolve-urls? ::trd/resolve-urls? input trd/intermediate?] (if resolve-urls? - (update input :text #(string/replace % shortened-url-pattern resolve-url)) + (update input :text #(string/replace % shortened-url-pattern infra/resolve-url)) input)) (defn-spec content-filter-regexes ::trd/content-filters diff --git a/src/test/cljs/mastodon_bot/infra_test.cljs b/src/test/cljs/mastodon_bot/infra_test.cljs new file mode 100644 index 0000000..8820cde --- /dev/null +++ b/src/test/cljs/mastodon_bot/infra_test.cljs @@ -0,0 +1,13 @@ +(ns mastodon-bot.infra-test + (:require + [cljs.test :refer-macros [deftest is testing run-tests]] + [mastodon-bot.infra :as sut])) + +;TODO: mbid test +(deftest test-resolve-uri + (is (= "https://www.meissa-gmbh.de" + (sut/resolve-url ["https://www.meissa-gmbh.de"]))) + (is (= "https://www.doesnotexist-blablabla.de" + (sut/resolve-url ["https://www.doesnotexist-blablabla.de"]))) + (is (= "http://www.google.de/" + (sut/resolve-url ["https://t1p.de/44oo"])))) \ No newline at end of file diff --git a/src/test/cljs/mastodon_bot/mastodon_api_test.cljs b/src/test/cljs/mastodon_bot/mastodon_api_test.cljs index c3f8df4..7a6f53d 100755 --- a/src/test/cljs/mastodon_bot/mastodon_api_test.cljs +++ b/src/test/cljs/mastodon_bot/mastodon_api_test.cljs @@ -49,4 +49,4 @@ https://chrisuehlinger.com/blog/2020/06/16/unshattering-the-audience-building-th (sut/intermediate-to-mastodon {:target-type :mastodon :append-screen-name? false :max-post-length 500} - intermediate-rss-item)))) \ No newline at end of file + intermediate-rss-item)))) diff --git a/src/test/cljs/mastodon_bot/transform_test.cljs b/src/test/cljs/mastodon_bot/transform_test.cljs index 95cb03b..845cfa4 100755 --- a/src/test/cljs/mastodon_bot/transform_test.cljs +++ b/src/test/cljs/mastodon_bot/transform_test.cljs @@ -16,4 +16,4 @@ (is (= "💠 Check out what has been going on during March in the world of @ReproBuilds! 💠 https://t.co/k6NsSO115z @opensuse@fosstodon.org @conservancy@mastodon.technology @PrototypeFund@mastodon.social @debian@fosstodon.org " (:text (sut/perform-replacements (first (:transform testconfig)) (twitter/parse-tweet (readfile "testdata/twitter/tweet-mentions.edn")))) - ))) \ No newline at end of file + )))