node-fetch instead of request

This commit is contained in:
Michael Jerger 2021-06-22 06:38:00 +00:00
parent 4ecb09274d
commit c4d85803fb
10 changed files with 62 additions and 120 deletions

View file

@ -1,89 +0,0 @@
name: stable
on:
push:
tags: '[0-9]+.[0-9]+.[0-9]+*'
jobs:
stable:
name: stable
runs-on: ubuntu-latest
strategy:
matrix:
node-version: [14.x]
steps:
- uses: actions/checkout@v2
- name: Use Node.js ${{ matrix.node-version }}
uses: actions/setup-node@v1
with:
node-version: ${{ matrix.node-version }}
- name: test em
run: |
npm install
npm install -g --save-dev shadow-cljs
shadow-cljs compile test
- name: build em
run: |
shadow-cljs release app
chmod a+x mastodon-bot.js
sha256sum mastodon-bot.js > target/mastodon-bot.js.sha256
sha512sum mastodon-bot.js > target/mastodon-bot.js.sha512
- name: Create Release
id: create_release
uses: actions/create-release@v1
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
with:
tag_name: ${{ github.ref }}
release_name: Release ${{ github.ref }}
draft: false
prerelease: false
- name: Upload mastodon-bot.js
id: upload-mastodon-bot-js
uses: actions/upload-release-asset@v1
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
with:
upload_url: ${{ steps.create_release.outputs.upload_url }}
asset_path: ./mastodon-bot.js
asset_name: mastodon-bot.js
asset_content_type: application/javascript
- name: Upload mastodon-bot.js.sha256
id: upload-mastodon-bot-js-sha256
uses: actions/upload-release-asset@v1
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
with:
upload_url: ${{ steps.create_release.outputs.upload_url }}
asset_path: ./target/mastodon-bot.js.sha256
asset_name: mastodon-bot.js.sha256
asset_content_type: text/plain
- name: Upload mastodon-bot.js.sha512
id: upload-mastodon-bot-js-sha512
uses: actions/upload-release-asset@v1
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
with:
upload_url: ${{ steps.create_release.outputs.upload_url }}
asset_path: ./target/mastodon-bot.js.sha512
asset_name: mastodon-bot.js.sha512
asset_content_type: text/plain
- name: upload to npm
env:
NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}
run: |
mkdir -p target/npm-build/mastodon_bot
cp mastodon-bot.js target/npm-build/mastodon_bot/
cp target/mastodon-bot.js.sha256 target/npm-build/mastodon_bot/
cp target/mastodon-bot.js.sha512 target/npm-build/mastodon_bot/
cp package.json target/npm-build/mastodon_bot/
cp README.md target/npm-build/mastodon_bot/
npm publish ./target/npm-build/mastodon_bot --access public

BIN
doc/rss.png Normal file

Binary file not shown.

After

Width:  |  Height:  |  Size: 18 KiB

10
doc/rss.puml Normal file
View file

@ -0,0 +1,10 @@
@startuml
"Mastodon-Bot" -> Config: Call rss-sources
Config --> "Mastodon-Bot": URL list of rss-sources
"Mastodon-Bot" -> "Mastodon-Bot": Call "get-feed" for every rss-source.\nStart asyncronous Promise.
activate "Mastodon-Bot"
"Mastodon-Bot" -> "RSS-Sources": Call URL to parse RSS-Feed
"RSS-Sources" --> "Mastodon-Bot": RSS-Feed
"Mastodon-Bot" -> Mastodon: Post feed by post-rss-to-mastodon
deactivate "Mastodon-Bot"
@enduml

View file

@ -7,12 +7,13 @@
"repository": "https://www.npmjs.com/package/mastodon-bot", "repository": "https://www.npmjs.com/package/mastodon-bot",
"license": "MIT", "license": "MIT",
"dependencies": { "dependencies": {
"deasync": "0.1.20",
"mastodon-api": "1.3.0", "mastodon-api": "1.3.0",
"node-fetch": "2.6.1",
"request": "2.88.0",
"rss-parser": "3.7.1", "rss-parser": "3.7.1",
"tumblr": "0.4.1", "tumblr": "0.4.1",
"twitter": "1.7.1", "twitter": "1.7.1"
"deasync": "0.1.20",
"request": "2.88.0"
}, },
"devDependencies": { "devDependencies": {
"shadow-cljs": "^2.8.37" "shadow-cljs": "^2.8.37"

View file

@ -1,15 +1,17 @@
(ns mastodon-bot.infra (ns mastodon-bot.infra
(:require (:require
[cljs.reader :as edn] [cljs.reader :as edn]
[clojure.pprint :refer [pprint]] [clojure.string :as string]
["fs" :as fs])) ["fs" :as fs]
["deasync" :as deasync]
["node-fetch" :as fetch]))
(defn debug [item] (defn debug [item]
(pprint item) (js/console.log item)
item) item)
(defn debug-first [item] (defn debug-first [item]
(pprint (first item)) (js/console.log (first item))
item) item)
(defn js->edn [data] (defn js->edn [data]
@ -28,8 +30,8 @@
(if config (if config
(if (fs/existsSync config) (if (fs/existsSync config)
;(edn/read-string (fs/readFileSync #js {:encoding "UTF-8"} config)) ;(edn/read-string (fs/readFileSync #js {:encoding "UTF-8"} config))
(edn/read-string (fs/readFileSync config "UTF-8")) (edn/read-string (fs/readFileSync config "UTF-8"))
(exit-with-error (str "config file does not exist: " config))) (exit-with-error (str "config file does not exist: " config)))
nil)) nil))
(defn load-credentials-config [] (defn load-credentials-config []
@ -42,3 +44,19 @@
(defn load-config [config-location] (defn load-config [config-location]
(merge (load-main-config config-location) (load-credentials-config))) (merge (load-main-config config-location) (load-credentials-config)))
(defn resolve-promise [promise result-on-error]
(let [done (atom false)
result (atom nil)
promise (-> promise
(.then #(do (reset! result %) (reset! done true)))
(.catch #(do (reset! result result-on-error) (reset! done true))))]
(.loopWhile deasync (fn [] (not @done)))
@result))
(defn resolve-url [[uri]]
(let [used-uri (if (string/starts-with? uri "https://") uri (str "https://" uri))
location (-> (fetch used-uri #js {:method "GET" :redirect "manual" :timeout "3000"})
(.then #(.get (.-headers %) "Location"))
(.then #(string/replace % "?mbid=social_twitter" "")))]
(resolve-promise location uri)))

View file

@ -2,11 +2,14 @@
(:require (:require
[orchestra.core :refer-macros [defn-spec]] [orchestra.core :refer-macros [defn-spec]]
[mastodon-bot.rss-domain :as rd] [mastodon-bot.rss-domain :as rd]
[clojure.spec.alpha :as s]
["rss-parser" :as rss])) ["rss-parser" :as rss]))
(s/def ::pos-integer (and #(< 0 %) integer?))
(defn-spec rss-client any? (defn-spec rss-client any?
[] [& {:keys [timeout]
(rss.)) :or {timeout 3000}} (s/keys :opt-un [::pos-integer])]
(rss. #js {:timeout timeout}))
(defn-spec parse-feed any? (defn-spec parse-feed any?
[item ::rd/feed-item] [item ::rd/feed-item]
@ -21,4 +24,5 @@
callback fn?] callback fn?]
(print url) (print url)
(-> (.parseURL (rss-client) url) (-> (.parseURL (rss-client) url)
(.then callback))) (.then callback)
(.catch #(js/console.log %))))

View file

@ -10,22 +10,7 @@
[mastodon-bot.rss-api :as ra] [mastodon-bot.rss-api :as ra]
[mastodon-bot.tumblr-domain :as td] [mastodon-bot.tumblr-domain :as td]
[mastodon-bot.tumblr-api :as ta] [mastodon-bot.tumblr-api :as ta]
[mastodon-bot.transform-domain :as trd] [mastodon-bot.transform-domain :as trd]))
["deasync" :as deasync]
["request" :as request]))
(defn resolve-url [[uri]]
(try
(or
(some-> ((deasync request)
#js {:method "GET"
:uri (if (string/starts-with? uri "https://") uri (str "https://" uri))
:followRedirect false})
(.-headers)
(.-location)
(string/replace "?mbid=social_twitter" ""))
uri)
(catch js/Error _ uri)))
(def shortened-url-pattern #"(https?://)?(?:\S+(?::\S*)?@)?(?:(?!(?:10|127)(?:\.\d{1,3}){3})(?!(?:169\.254|192\.168)(?:\.\d{1,3}){2})(?!172\.(?:1[6-9]|2\d|3[0-1])(?:\.\d{1,3}){2})(?:[1-9]\d?|1\d\d|2[01]\d|22[0-3])(?:\.(?:1?\d{1,2}|2[0-4]\d|25[0-5])){2}(?:\.(?:[1-9]\d?|1\d\d|2[0-4]\d|25[0-4]))|(?:(?:[a-z\u00a1-\uffff0-9]-*)*[a-z\u00a1-\uffff0-9]+)(?:\.(?:[a-z\u00a1-\uffff0-9]-*)*[a-z\u00a1-\uffff0-9]+)*(?:\.(?:[a-z\u00a1-\uffff]{2,}))\.?)(?::\d{2,5})?(?:[/?#]\S*)?") (def shortened-url-pattern #"(https?://)?(?:\S+(?::\S*)?@)?(?:(?!(?:10|127)(?:\.\d{1,3}){3})(?!(?:169\.254|192\.168)(?:\.\d{1,3}){2})(?!172\.(?:1[6-9]|2\d|3[0-1])(?:\.\d{1,3}){2})(?:[1-9]\d?|1\d\d|2[01]\d|22[0-3])(?:\.(?:1?\d{1,2}|2[0-4]\d|25[0-5])){2}(?:\.(?:[1-9]\d?|1\d\d|2[0-4]\d|25[0-4]))|(?:(?:[a-z\u00a1-\uffff0-9]-*)*[a-z\u00a1-\uffff0-9]+)(?:\.(?:[a-z\u00a1-\uffff0-9]-*)*[a-z\u00a1-\uffff0-9]+)*(?:\.(?:[a-z\u00a1-\uffff]{2,}))\.?)(?::\d{2,5})?(?:[/?#]\S*)?")
@ -33,7 +18,7 @@
[resolve-urls? ::trd/resolve-urls? [resolve-urls? ::trd/resolve-urls?
input trd/intermediate?] input trd/intermediate?]
(if resolve-urls? (if resolve-urls?
(update input :text #(string/replace % shortened-url-pattern resolve-url)) (update input :text #(string/replace % shortened-url-pattern infra/resolve-url))
input)) input))
(defn-spec content-filter-regexes ::trd/content-filters (defn-spec content-filter-regexes ::trd/content-filters

View file

@ -0,0 +1,13 @@
(ns mastodon-bot.infra-test
(:require
[cljs.test :refer-macros [deftest is testing run-tests]]
[mastodon-bot.infra :as sut]))
;TODO: mbid test
(deftest test-resolve-uri
(is (= "https://www.meissa-gmbh.de"
(sut/resolve-url ["https://www.meissa-gmbh.de"])))
(is (= "https://www.doesnotexist-blablabla.de"
(sut/resolve-url ["https://www.doesnotexist-blablabla.de"])))
(is (= "http://www.google.de/"
(sut/resolve-url ["https://t1p.de/44oo"]))))

View file

@ -49,4 +49,4 @@ https://chrisuehlinger.com/blog/2020/06/16/unshattering-the-audience-building-th
(sut/intermediate-to-mastodon {:target-type :mastodon (sut/intermediate-to-mastodon {:target-type :mastodon
:append-screen-name? false :append-screen-name? false
:max-post-length 500} :max-post-length 500}
intermediate-rss-item)))) intermediate-rss-item))))

View file

@ -16,4 +16,4 @@
(is (= (is (=
"💠 Check out what has been going on during March in the world of @ReproBuilds! 💠 https://t.co/k6NsSO115z @opensuse@fosstodon.org @conservancy@mastodon.technology @PrototypeFund@mastodon.social @debian@fosstodon.org " "💠 Check out what has been going on during March in the world of @ReproBuilds! 💠 https://t.co/k6NsSO115z @opensuse@fosstodon.org @conservancy@mastodon.technology @PrototypeFund@mastodon.social @debian@fosstodon.org "
(:text (sut/perform-replacements (first (:transform testconfig)) (twitter/parse-tweet (readfile "testdata/twitter/tweet-mentions.edn")))) (:text (sut/perform-replacements (first (:transform testconfig)) (twitter/parse-tweet (readfile "testdata/twitter/tweet-mentions.edn"))))
))) )))