node-fetch instead of request
This commit is contained in:
parent
4ecb09274d
commit
c4d85803fb
10 changed files with 62 additions and 120 deletions
89
.github/workflows/stable.yml
vendored
89
.github/workflows/stable.yml
vendored
|
@ -1,89 +0,0 @@
|
||||||
name: stable
|
|
||||||
on:
|
|
||||||
push:
|
|
||||||
tags: '[0-9]+.[0-9]+.[0-9]+*'
|
|
||||||
|
|
||||||
jobs:
|
|
||||||
stable:
|
|
||||||
name: stable
|
|
||||||
runs-on: ubuntu-latest
|
|
||||||
strategy:
|
|
||||||
matrix:
|
|
||||||
node-version: [14.x]
|
|
||||||
|
|
||||||
steps:
|
|
||||||
- uses: actions/checkout@v2
|
|
||||||
|
|
||||||
- name: Use Node.js ${{ matrix.node-version }}
|
|
||||||
uses: actions/setup-node@v1
|
|
||||||
with:
|
|
||||||
node-version: ${{ matrix.node-version }}
|
|
||||||
|
|
||||||
- name: test em
|
|
||||||
run: |
|
|
||||||
npm install
|
|
||||||
npm install -g --save-dev shadow-cljs
|
|
||||||
shadow-cljs compile test
|
|
||||||
|
|
||||||
- name: build em
|
|
||||||
run: |
|
|
||||||
shadow-cljs release app
|
|
||||||
chmod a+x mastodon-bot.js
|
|
||||||
sha256sum mastodon-bot.js > target/mastodon-bot.js.sha256
|
|
||||||
sha512sum mastodon-bot.js > target/mastodon-bot.js.sha512
|
|
||||||
|
|
||||||
- name: Create Release
|
|
||||||
id: create_release
|
|
||||||
uses: actions/create-release@v1
|
|
||||||
env:
|
|
||||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
|
||||||
with:
|
|
||||||
tag_name: ${{ github.ref }}
|
|
||||||
release_name: Release ${{ github.ref }}
|
|
||||||
draft: false
|
|
||||||
prerelease: false
|
|
||||||
|
|
||||||
- name: Upload mastodon-bot.js
|
|
||||||
id: upload-mastodon-bot-js
|
|
||||||
uses: actions/upload-release-asset@v1
|
|
||||||
env:
|
|
||||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
|
||||||
with:
|
|
||||||
upload_url: ${{ steps.create_release.outputs.upload_url }}
|
|
||||||
asset_path: ./mastodon-bot.js
|
|
||||||
asset_name: mastodon-bot.js
|
|
||||||
asset_content_type: application/javascript
|
|
||||||
|
|
||||||
- name: Upload mastodon-bot.js.sha256
|
|
||||||
id: upload-mastodon-bot-js-sha256
|
|
||||||
uses: actions/upload-release-asset@v1
|
|
||||||
env:
|
|
||||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
|
||||||
with:
|
|
||||||
upload_url: ${{ steps.create_release.outputs.upload_url }}
|
|
||||||
asset_path: ./target/mastodon-bot.js.sha256
|
|
||||||
asset_name: mastodon-bot.js.sha256
|
|
||||||
asset_content_type: text/plain
|
|
||||||
|
|
||||||
- name: Upload mastodon-bot.js.sha512
|
|
||||||
id: upload-mastodon-bot-js-sha512
|
|
||||||
uses: actions/upload-release-asset@v1
|
|
||||||
env:
|
|
||||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
|
||||||
with:
|
|
||||||
upload_url: ${{ steps.create_release.outputs.upload_url }}
|
|
||||||
asset_path: ./target/mastodon-bot.js.sha512
|
|
||||||
asset_name: mastodon-bot.js.sha512
|
|
||||||
asset_content_type: text/plain
|
|
||||||
|
|
||||||
- name: upload to npm
|
|
||||||
env:
|
|
||||||
NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}
|
|
||||||
run: |
|
|
||||||
mkdir -p target/npm-build/mastodon_bot
|
|
||||||
cp mastodon-bot.js target/npm-build/mastodon_bot/
|
|
||||||
cp target/mastodon-bot.js.sha256 target/npm-build/mastodon_bot/
|
|
||||||
cp target/mastodon-bot.js.sha512 target/npm-build/mastodon_bot/
|
|
||||||
cp package.json target/npm-build/mastodon_bot/
|
|
||||||
cp README.md target/npm-build/mastodon_bot/
|
|
||||||
npm publish ./target/npm-build/mastodon_bot --access public
|
|
BIN
doc/rss.png
Normal file
BIN
doc/rss.png
Normal file
Binary file not shown.
After Width: | Height: | Size: 18 KiB |
10
doc/rss.puml
Normal file
10
doc/rss.puml
Normal file
|
@ -0,0 +1,10 @@
|
||||||
|
@startuml
|
||||||
|
"Mastodon-Bot" -> Config: Call rss-sources
|
||||||
|
Config --> "Mastodon-Bot": URL list of rss-sources
|
||||||
|
"Mastodon-Bot" -> "Mastodon-Bot": Call "get-feed" for every rss-source.\nStart asyncronous Promise.
|
||||||
|
activate "Mastodon-Bot"
|
||||||
|
"Mastodon-Bot" -> "RSS-Sources": Call URL to parse RSS-Feed
|
||||||
|
"RSS-Sources" --> "Mastodon-Bot": RSS-Feed
|
||||||
|
"Mastodon-Bot" -> Mastodon: Post feed by post-rss-to-mastodon
|
||||||
|
deactivate "Mastodon-Bot"
|
||||||
|
@enduml
|
|
@ -7,12 +7,13 @@
|
||||||
"repository": "https://www.npmjs.com/package/mastodon-bot",
|
"repository": "https://www.npmjs.com/package/mastodon-bot",
|
||||||
"license": "MIT",
|
"license": "MIT",
|
||||||
"dependencies": {
|
"dependencies": {
|
||||||
|
"deasync": "0.1.20",
|
||||||
"mastodon-api": "1.3.0",
|
"mastodon-api": "1.3.0",
|
||||||
|
"node-fetch": "2.6.1",
|
||||||
|
"request": "2.88.0",
|
||||||
"rss-parser": "3.7.1",
|
"rss-parser": "3.7.1",
|
||||||
"tumblr": "0.4.1",
|
"tumblr": "0.4.1",
|
||||||
"twitter": "1.7.1",
|
"twitter": "1.7.1"
|
||||||
"deasync": "0.1.20",
|
|
||||||
"request": "2.88.0"
|
|
||||||
},
|
},
|
||||||
"devDependencies": {
|
"devDependencies": {
|
||||||
"shadow-cljs": "^2.8.37"
|
"shadow-cljs": "^2.8.37"
|
||||||
|
|
|
@ -1,15 +1,17 @@
|
||||||
(ns mastodon-bot.infra
|
(ns mastodon-bot.infra
|
||||||
(:require
|
(:require
|
||||||
[cljs.reader :as edn]
|
[cljs.reader :as edn]
|
||||||
[clojure.pprint :refer [pprint]]
|
[clojure.string :as string]
|
||||||
["fs" :as fs]))
|
["fs" :as fs]
|
||||||
|
["deasync" :as deasync]
|
||||||
|
["node-fetch" :as fetch]))
|
||||||
|
|
||||||
(defn debug [item]
|
(defn debug [item]
|
||||||
(pprint item)
|
(js/console.log item)
|
||||||
item)
|
item)
|
||||||
|
|
||||||
(defn debug-first [item]
|
(defn debug-first [item]
|
||||||
(pprint (first item))
|
(js/console.log (first item))
|
||||||
item)
|
item)
|
||||||
|
|
||||||
(defn js->edn [data]
|
(defn js->edn [data]
|
||||||
|
@ -28,8 +30,8 @@
|
||||||
(if config
|
(if config
|
||||||
(if (fs/existsSync config)
|
(if (fs/existsSync config)
|
||||||
;(edn/read-string (fs/readFileSync #js {:encoding "UTF-8"} config))
|
;(edn/read-string (fs/readFileSync #js {:encoding "UTF-8"} config))
|
||||||
(edn/read-string (fs/readFileSync config "UTF-8"))
|
(edn/read-string (fs/readFileSync config "UTF-8"))
|
||||||
(exit-with-error (str "config file does not exist: " config)))
|
(exit-with-error (str "config file does not exist: " config)))
|
||||||
nil))
|
nil))
|
||||||
|
|
||||||
(defn load-credentials-config []
|
(defn load-credentials-config []
|
||||||
|
@ -42,3 +44,19 @@
|
||||||
|
|
||||||
(defn load-config [config-location]
|
(defn load-config [config-location]
|
||||||
(merge (load-main-config config-location) (load-credentials-config)))
|
(merge (load-main-config config-location) (load-credentials-config)))
|
||||||
|
|
||||||
|
(defn resolve-promise [promise result-on-error]
|
||||||
|
(let [done (atom false)
|
||||||
|
result (atom nil)
|
||||||
|
promise (-> promise
|
||||||
|
(.then #(do (reset! result %) (reset! done true)))
|
||||||
|
(.catch #(do (reset! result result-on-error) (reset! done true))))]
|
||||||
|
(.loopWhile deasync (fn [] (not @done)))
|
||||||
|
@result))
|
||||||
|
|
||||||
|
(defn resolve-url [[uri]]
|
||||||
|
(let [used-uri (if (string/starts-with? uri "https://") uri (str "https://" uri))
|
||||||
|
location (-> (fetch used-uri #js {:method "GET" :redirect "manual" :timeout "3000"})
|
||||||
|
(.then #(.get (.-headers %) "Location"))
|
||||||
|
(.then #(string/replace % "?mbid=social_twitter" "")))]
|
||||||
|
(resolve-promise location uri)))
|
||||||
|
|
|
@ -2,11 +2,14 @@
|
||||||
(:require
|
(:require
|
||||||
[orchestra.core :refer-macros [defn-spec]]
|
[orchestra.core :refer-macros [defn-spec]]
|
||||||
[mastodon-bot.rss-domain :as rd]
|
[mastodon-bot.rss-domain :as rd]
|
||||||
|
[clojure.spec.alpha :as s]
|
||||||
["rss-parser" :as rss]))
|
["rss-parser" :as rss]))
|
||||||
|
|
||||||
|
(s/def ::pos-integer (and #(< 0 %) integer?))
|
||||||
(defn-spec rss-client any?
|
(defn-spec rss-client any?
|
||||||
[]
|
[& {:keys [timeout]
|
||||||
(rss.))
|
:or {timeout 3000}} (s/keys :opt-un [::pos-integer])]
|
||||||
|
(rss. #js {:timeout timeout}))
|
||||||
|
|
||||||
(defn-spec parse-feed any?
|
(defn-spec parse-feed any?
|
||||||
[item ::rd/feed-item]
|
[item ::rd/feed-item]
|
||||||
|
@ -21,4 +24,5 @@
|
||||||
callback fn?]
|
callback fn?]
|
||||||
(print url)
|
(print url)
|
||||||
(-> (.parseURL (rss-client) url)
|
(-> (.parseURL (rss-client) url)
|
||||||
(.then callback)))
|
(.then callback)
|
||||||
|
(.catch #(js/console.log %))))
|
||||||
|
|
|
@ -10,22 +10,7 @@
|
||||||
[mastodon-bot.rss-api :as ra]
|
[mastodon-bot.rss-api :as ra]
|
||||||
[mastodon-bot.tumblr-domain :as td]
|
[mastodon-bot.tumblr-domain :as td]
|
||||||
[mastodon-bot.tumblr-api :as ta]
|
[mastodon-bot.tumblr-api :as ta]
|
||||||
[mastodon-bot.transform-domain :as trd]
|
[mastodon-bot.transform-domain :as trd]))
|
||||||
["deasync" :as deasync]
|
|
||||||
["request" :as request]))
|
|
||||||
|
|
||||||
(defn resolve-url [[uri]]
|
|
||||||
(try
|
|
||||||
(or
|
|
||||||
(some-> ((deasync request)
|
|
||||||
#js {:method "GET"
|
|
||||||
:uri (if (string/starts-with? uri "https://") uri (str "https://" uri))
|
|
||||||
:followRedirect false})
|
|
||||||
(.-headers)
|
|
||||||
(.-location)
|
|
||||||
(string/replace "?mbid=social_twitter" ""))
|
|
||||||
uri)
|
|
||||||
(catch js/Error _ uri)))
|
|
||||||
|
|
||||||
(def shortened-url-pattern #"(https?://)?(?:\S+(?::\S*)?@)?(?:(?!(?:10|127)(?:\.\d{1,3}){3})(?!(?:169\.254|192\.168)(?:\.\d{1,3}){2})(?!172\.(?:1[6-9]|2\d|3[0-1])(?:\.\d{1,3}){2})(?:[1-9]\d?|1\d\d|2[01]\d|22[0-3])(?:\.(?:1?\d{1,2}|2[0-4]\d|25[0-5])){2}(?:\.(?:[1-9]\d?|1\d\d|2[0-4]\d|25[0-4]))|(?:(?:[a-z\u00a1-\uffff0-9]-*)*[a-z\u00a1-\uffff0-9]+)(?:\.(?:[a-z\u00a1-\uffff0-9]-*)*[a-z\u00a1-\uffff0-9]+)*(?:\.(?:[a-z\u00a1-\uffff]{2,}))\.?)(?::\d{2,5})?(?:[/?#]\S*)?")
|
(def shortened-url-pattern #"(https?://)?(?:\S+(?::\S*)?@)?(?:(?!(?:10|127)(?:\.\d{1,3}){3})(?!(?:169\.254|192\.168)(?:\.\d{1,3}){2})(?!172\.(?:1[6-9]|2\d|3[0-1])(?:\.\d{1,3}){2})(?:[1-9]\d?|1\d\d|2[01]\d|22[0-3])(?:\.(?:1?\d{1,2}|2[0-4]\d|25[0-5])){2}(?:\.(?:[1-9]\d?|1\d\d|2[0-4]\d|25[0-4]))|(?:(?:[a-z\u00a1-\uffff0-9]-*)*[a-z\u00a1-\uffff0-9]+)(?:\.(?:[a-z\u00a1-\uffff0-9]-*)*[a-z\u00a1-\uffff0-9]+)*(?:\.(?:[a-z\u00a1-\uffff]{2,}))\.?)(?::\d{2,5})?(?:[/?#]\S*)?")
|
||||||
|
|
||||||
|
@ -33,7 +18,7 @@
|
||||||
[resolve-urls? ::trd/resolve-urls?
|
[resolve-urls? ::trd/resolve-urls?
|
||||||
input trd/intermediate?]
|
input trd/intermediate?]
|
||||||
(if resolve-urls?
|
(if resolve-urls?
|
||||||
(update input :text #(string/replace % shortened-url-pattern resolve-url))
|
(update input :text #(string/replace % shortened-url-pattern infra/resolve-url))
|
||||||
input))
|
input))
|
||||||
|
|
||||||
(defn-spec content-filter-regexes ::trd/content-filters
|
(defn-spec content-filter-regexes ::trd/content-filters
|
||||||
|
|
13
src/test/cljs/mastodon_bot/infra_test.cljs
Normal file
13
src/test/cljs/mastodon_bot/infra_test.cljs
Normal file
|
@ -0,0 +1,13 @@
|
||||||
|
(ns mastodon-bot.infra-test
|
||||||
|
(:require
|
||||||
|
[cljs.test :refer-macros [deftest is testing run-tests]]
|
||||||
|
[mastodon-bot.infra :as sut]))
|
||||||
|
|
||||||
|
;TODO: mbid test
|
||||||
|
(deftest test-resolve-uri
|
||||||
|
(is (= "https://www.meissa-gmbh.de"
|
||||||
|
(sut/resolve-url ["https://www.meissa-gmbh.de"])))
|
||||||
|
(is (= "https://www.doesnotexist-blablabla.de"
|
||||||
|
(sut/resolve-url ["https://www.doesnotexist-blablabla.de"])))
|
||||||
|
(is (= "http://www.google.de/"
|
||||||
|
(sut/resolve-url ["https://t1p.de/44oo"]))))
|
|
@ -49,4 +49,4 @@ https://chrisuehlinger.com/blog/2020/06/16/unshattering-the-audience-building-th
|
||||||
(sut/intermediate-to-mastodon {:target-type :mastodon
|
(sut/intermediate-to-mastodon {:target-type :mastodon
|
||||||
:append-screen-name? false
|
:append-screen-name? false
|
||||||
:max-post-length 500}
|
:max-post-length 500}
|
||||||
intermediate-rss-item))))
|
intermediate-rss-item))))
|
||||||
|
|
|
@ -16,4 +16,4 @@
|
||||||
(is (=
|
(is (=
|
||||||
"💠 Check out what has been going on during March in the world of @ReproBuilds! 💠 https://t.co/k6NsSO115z @opensuse@fosstodon.org @conservancy@mastodon.technology @PrototypeFund@mastodon.social @debian@fosstodon.org "
|
"💠 Check out what has been going on during March in the world of @ReproBuilds! 💠 https://t.co/k6NsSO115z @opensuse@fosstodon.org @conservancy@mastodon.technology @PrototypeFund@mastodon.social @debian@fosstodon.org "
|
||||||
(:text (sut/perform-replacements (first (:transform testconfig)) (twitter/parse-tweet (readfile "testdata/twitter/tweet-mentions.edn"))))
|
(:text (sut/perform-replacements (first (:transform testconfig)) (twitter/parse-tweet (readfile "testdata/twitter/tweet-mentions.edn"))))
|
||||||
)))
|
)))
|
||||||
|
|
Reference in a new issue