node-fetch instead of request
This commit is contained in:
parent
4ecb09274d
commit
c4d85803fb
10 changed files with 62 additions and 120 deletions
89
.github/workflows/stable.yml
vendored
89
.github/workflows/stable.yml
vendored
|
@ -1,89 +0,0 @@
|
|||
name: stable
|
||||
on:
|
||||
push:
|
||||
tags: '[0-9]+.[0-9]+.[0-9]+*'
|
||||
|
||||
jobs:
|
||||
stable:
|
||||
name: stable
|
||||
runs-on: ubuntu-latest
|
||||
strategy:
|
||||
matrix:
|
||||
node-version: [14.x]
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
|
||||
- name: Use Node.js ${{ matrix.node-version }}
|
||||
uses: actions/setup-node@v1
|
||||
with:
|
||||
node-version: ${{ matrix.node-version }}
|
||||
|
||||
- name: test em
|
||||
run: |
|
||||
npm install
|
||||
npm install -g --save-dev shadow-cljs
|
||||
shadow-cljs compile test
|
||||
|
||||
- name: build em
|
||||
run: |
|
||||
shadow-cljs release app
|
||||
chmod a+x mastodon-bot.js
|
||||
sha256sum mastodon-bot.js > target/mastodon-bot.js.sha256
|
||||
sha512sum mastodon-bot.js > target/mastodon-bot.js.sha512
|
||||
|
||||
- name: Create Release
|
||||
id: create_release
|
||||
uses: actions/create-release@v1
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
with:
|
||||
tag_name: ${{ github.ref }}
|
||||
release_name: Release ${{ github.ref }}
|
||||
draft: false
|
||||
prerelease: false
|
||||
|
||||
- name: Upload mastodon-bot.js
|
||||
id: upload-mastodon-bot-js
|
||||
uses: actions/upload-release-asset@v1
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
with:
|
||||
upload_url: ${{ steps.create_release.outputs.upload_url }}
|
||||
asset_path: ./mastodon-bot.js
|
||||
asset_name: mastodon-bot.js
|
||||
asset_content_type: application/javascript
|
||||
|
||||
- name: Upload mastodon-bot.js.sha256
|
||||
id: upload-mastodon-bot-js-sha256
|
||||
uses: actions/upload-release-asset@v1
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
with:
|
||||
upload_url: ${{ steps.create_release.outputs.upload_url }}
|
||||
asset_path: ./target/mastodon-bot.js.sha256
|
||||
asset_name: mastodon-bot.js.sha256
|
||||
asset_content_type: text/plain
|
||||
|
||||
- name: Upload mastodon-bot.js.sha512
|
||||
id: upload-mastodon-bot-js-sha512
|
||||
uses: actions/upload-release-asset@v1
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
with:
|
||||
upload_url: ${{ steps.create_release.outputs.upload_url }}
|
||||
asset_path: ./target/mastodon-bot.js.sha512
|
||||
asset_name: mastodon-bot.js.sha512
|
||||
asset_content_type: text/plain
|
||||
|
||||
- name: upload to npm
|
||||
env:
|
||||
NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}
|
||||
run: |
|
||||
mkdir -p target/npm-build/mastodon_bot
|
||||
cp mastodon-bot.js target/npm-build/mastodon_bot/
|
||||
cp target/mastodon-bot.js.sha256 target/npm-build/mastodon_bot/
|
||||
cp target/mastodon-bot.js.sha512 target/npm-build/mastodon_bot/
|
||||
cp package.json target/npm-build/mastodon_bot/
|
||||
cp README.md target/npm-build/mastodon_bot/
|
||||
npm publish ./target/npm-build/mastodon_bot --access public
|
BIN
doc/rss.png
Normal file
BIN
doc/rss.png
Normal file
Binary file not shown.
After Width: | Height: | Size: 18 KiB |
10
doc/rss.puml
Normal file
10
doc/rss.puml
Normal file
|
@ -0,0 +1,10 @@
|
|||
@startuml
|
||||
"Mastodon-Bot" -> Config: Call rss-sources
|
||||
Config --> "Mastodon-Bot": URL list of rss-sources
|
||||
"Mastodon-Bot" -> "Mastodon-Bot": Call "get-feed" for every rss-source.\nStart asyncronous Promise.
|
||||
activate "Mastodon-Bot"
|
||||
"Mastodon-Bot" -> "RSS-Sources": Call URL to parse RSS-Feed
|
||||
"RSS-Sources" --> "Mastodon-Bot": RSS-Feed
|
||||
"Mastodon-Bot" -> Mastodon: Post feed by post-rss-to-mastodon
|
||||
deactivate "Mastodon-Bot"
|
||||
@enduml
|
|
@ -7,12 +7,13 @@
|
|||
"repository": "https://www.npmjs.com/package/mastodon-bot",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"deasync": "0.1.20",
|
||||
"mastodon-api": "1.3.0",
|
||||
"node-fetch": "2.6.1",
|
||||
"request": "2.88.0",
|
||||
"rss-parser": "3.7.1",
|
||||
"tumblr": "0.4.1",
|
||||
"twitter": "1.7.1",
|
||||
"deasync": "0.1.20",
|
||||
"request": "2.88.0"
|
||||
"twitter": "1.7.1"
|
||||
},
|
||||
"devDependencies": {
|
||||
"shadow-cljs": "^2.8.37"
|
||||
|
|
|
@ -1,15 +1,17 @@
|
|||
(ns mastodon-bot.infra
|
||||
(:require
|
||||
[cljs.reader :as edn]
|
||||
[clojure.pprint :refer [pprint]]
|
||||
["fs" :as fs]))
|
||||
[clojure.string :as string]
|
||||
["fs" :as fs]
|
||||
["deasync" :as deasync]
|
||||
["node-fetch" :as fetch]))
|
||||
|
||||
(defn debug [item]
|
||||
(pprint item)
|
||||
(js/console.log item)
|
||||
item)
|
||||
|
||||
(defn debug-first [item]
|
||||
(pprint (first item))
|
||||
(js/console.log (first item))
|
||||
item)
|
||||
|
||||
(defn js->edn [data]
|
||||
|
@ -28,8 +30,8 @@
|
|||
(if config
|
||||
(if (fs/existsSync config)
|
||||
;(edn/read-string (fs/readFileSync #js {:encoding "UTF-8"} config))
|
||||
(edn/read-string (fs/readFileSync config "UTF-8"))
|
||||
(exit-with-error (str "config file does not exist: " config)))
|
||||
(edn/read-string (fs/readFileSync config "UTF-8"))
|
||||
(exit-with-error (str "config file does not exist: " config)))
|
||||
nil))
|
||||
|
||||
(defn load-credentials-config []
|
||||
|
@ -42,3 +44,19 @@
|
|||
|
||||
(defn load-config [config-location]
|
||||
(merge (load-main-config config-location) (load-credentials-config)))
|
||||
|
||||
(defn resolve-promise [promise result-on-error]
|
||||
(let [done (atom false)
|
||||
result (atom nil)
|
||||
promise (-> promise
|
||||
(.then #(do (reset! result %) (reset! done true)))
|
||||
(.catch #(do (reset! result result-on-error) (reset! done true))))]
|
||||
(.loopWhile deasync (fn [] (not @done)))
|
||||
@result))
|
||||
|
||||
(defn resolve-url [[uri]]
|
||||
(let [used-uri (if (string/starts-with? uri "https://") uri (str "https://" uri))
|
||||
location (-> (fetch used-uri #js {:method "GET" :redirect "manual" :timeout "3000"})
|
||||
(.then #(.get (.-headers %) "Location"))
|
||||
(.then #(string/replace % "?mbid=social_twitter" "")))]
|
||||
(resolve-promise location uri)))
|
||||
|
|
|
@ -2,11 +2,14 @@
|
|||
(:require
|
||||
[orchestra.core :refer-macros [defn-spec]]
|
||||
[mastodon-bot.rss-domain :as rd]
|
||||
[clojure.spec.alpha :as s]
|
||||
["rss-parser" :as rss]))
|
||||
|
||||
(s/def ::pos-integer (and #(< 0 %) integer?))
|
||||
(defn-spec rss-client any?
|
||||
[]
|
||||
(rss.))
|
||||
[& {:keys [timeout]
|
||||
:or {timeout 3000}} (s/keys :opt-un [::pos-integer])]
|
||||
(rss. #js {:timeout timeout}))
|
||||
|
||||
(defn-spec parse-feed any?
|
||||
[item ::rd/feed-item]
|
||||
|
@ -21,4 +24,5 @@
|
|||
callback fn?]
|
||||
(print url)
|
||||
(-> (.parseURL (rss-client) url)
|
||||
(.then callback)))
|
||||
(.then callback)
|
||||
(.catch #(js/console.log %))))
|
||||
|
|
|
@ -10,22 +10,7 @@
|
|||
[mastodon-bot.rss-api :as ra]
|
||||
[mastodon-bot.tumblr-domain :as td]
|
||||
[mastodon-bot.tumblr-api :as ta]
|
||||
[mastodon-bot.transform-domain :as trd]
|
||||
["deasync" :as deasync]
|
||||
["request" :as request]))
|
||||
|
||||
(defn resolve-url [[uri]]
|
||||
(try
|
||||
(or
|
||||
(some-> ((deasync request)
|
||||
#js {:method "GET"
|
||||
:uri (if (string/starts-with? uri "https://") uri (str "https://" uri))
|
||||
:followRedirect false})
|
||||
(.-headers)
|
||||
(.-location)
|
||||
(string/replace "?mbid=social_twitter" ""))
|
||||
uri)
|
||||
(catch js/Error _ uri)))
|
||||
[mastodon-bot.transform-domain :as trd]))
|
||||
|
||||
(def shortened-url-pattern #"(https?://)?(?:\S+(?::\S*)?@)?(?:(?!(?:10|127)(?:\.\d{1,3}){3})(?!(?:169\.254|192\.168)(?:\.\d{1,3}){2})(?!172\.(?:1[6-9]|2\d|3[0-1])(?:\.\d{1,3}){2})(?:[1-9]\d?|1\d\d|2[01]\d|22[0-3])(?:\.(?:1?\d{1,2}|2[0-4]\d|25[0-5])){2}(?:\.(?:[1-9]\d?|1\d\d|2[0-4]\d|25[0-4]))|(?:(?:[a-z\u00a1-\uffff0-9]-*)*[a-z\u00a1-\uffff0-9]+)(?:\.(?:[a-z\u00a1-\uffff0-9]-*)*[a-z\u00a1-\uffff0-9]+)*(?:\.(?:[a-z\u00a1-\uffff]{2,}))\.?)(?::\d{2,5})?(?:[/?#]\S*)?")
|
||||
|
||||
|
@ -33,7 +18,7 @@
|
|||
[resolve-urls? ::trd/resolve-urls?
|
||||
input trd/intermediate?]
|
||||
(if resolve-urls?
|
||||
(update input :text #(string/replace % shortened-url-pattern resolve-url))
|
||||
(update input :text #(string/replace % shortened-url-pattern infra/resolve-url))
|
||||
input))
|
||||
|
||||
(defn-spec content-filter-regexes ::trd/content-filters
|
||||
|
|
13
src/test/cljs/mastodon_bot/infra_test.cljs
Normal file
13
src/test/cljs/mastodon_bot/infra_test.cljs
Normal file
|
@ -0,0 +1,13 @@
|
|||
(ns mastodon-bot.infra-test
|
||||
(:require
|
||||
[cljs.test :refer-macros [deftest is testing run-tests]]
|
||||
[mastodon-bot.infra :as sut]))
|
||||
|
||||
;TODO: mbid test
|
||||
(deftest test-resolve-uri
|
||||
(is (= "https://www.meissa-gmbh.de"
|
||||
(sut/resolve-url ["https://www.meissa-gmbh.de"])))
|
||||
(is (= "https://www.doesnotexist-blablabla.de"
|
||||
(sut/resolve-url ["https://www.doesnotexist-blablabla.de"])))
|
||||
(is (= "http://www.google.de/"
|
||||
(sut/resolve-url ["https://t1p.de/44oo"]))))
|
|
@ -49,4 +49,4 @@ https://chrisuehlinger.com/blog/2020/06/16/unshattering-the-audience-building-th
|
|||
(sut/intermediate-to-mastodon {:target-type :mastodon
|
||||
:append-screen-name? false
|
||||
:max-post-length 500}
|
||||
intermediate-rss-item))))
|
||||
intermediate-rss-item))))
|
||||
|
|
|
@ -16,4 +16,4 @@
|
|||
(is (=
|
||||
"💠 Check out what has been going on during March in the world of @ReproBuilds! 💠 https://t.co/k6NsSO115z @opensuse@fosstodon.org @conservancy@mastodon.technology @PrototypeFund@mastodon.social @debian@fosstodon.org "
|
||||
(:text (sut/perform-replacements (first (:transform testconfig)) (twitter/parse-tweet (readfile "testdata/twitter/tweet-mentions.edn"))))
|
||||
)))
|
||||
)))
|
||||
|
|
Reference in a new issue