switched to use request module instead of curl to resolve urls

This commit is contained in:
Dmitri Sotnikov 2018-08-28 02:15:55 -04:00
parent 8a01e16648
commit 8c004bd876
3 changed files with 20 additions and 14 deletions

View file

@ -64,7 +64,7 @@ If you get a [permission failure](https://github.com/anmonteiro/lumo/issues/206)
;; optional signature for posts ;; optional signature for posts
:signature "#newsbot" :signature "#newsbot"
;; optionally try to resolve URLs in posts to skip URL shorteners ;; optionally try to resolve URLs in posts to skip URL shorteners
;; requires cURL to be installed and defaults to false ;; defaults to false
:resolve-urls? true :resolve-urls? true
;; optional content filter regexes ;; optional content filter regexes
;; any posts matching the regexes will be filtered out ;; any posts matching the regexes will be filtered out

View file

@ -5,10 +5,9 @@
[cljs.reader :as edn] [cljs.reader :as edn]
[clojure.set :refer [rename-keys]] [clojure.set :refer [rename-keys]]
[clojure.string :as string] [clojure.string :as string]
["child_process" :as cp] ["deasync" :as deasync]
["request" :as request]
["fs" :as fs] ["fs" :as fs]
["http" :as http]
["https" :as https]
["mastodon-api" :as mastodon] ["mastodon-api" :as mastodon]
["rss-parser" :as rss] ["rss-parser" :as rss]
["tumblr" :as tumblr] ["tumblr" :as tumblr]
@ -62,15 +61,18 @@
(defn delete-status [status] (defn delete-status [status]
(.delete mastodon-client (str "statuses/" status) #js {})) (.delete mastodon-client (str "statuses/" status) #js {}))
(defn resolve-url [[url]] (defn resolve-url [[uri]]
(try (try
(or (or
(some-> (str (.execSync cp (str "curl -sIL " url) #js{})) (some-> ((deasync request)
(->> (re-seq #"[Ll]ocation: .*")) #js {:method "GET"
(first) :uri (if (string/starts-with? uri "https://") uri (str "https://" uri))
(string/replace #"[Ll]ocation: " "")) :followRedirect false})
url) (.-headers)
(catch js/Error _ url))) (.-location)
(string/replace "?mbid=social_twitter" ""))
uri)
(catch js/Error _ uri)))
(def shortened-url-pattern #"(https?://)?(?:\S+(?::\S*)?@)?(?:(?!(?:10|127)(?:\.\d{1,3}){3})(?!(?:169\.254|192\.168)(?:\.\d{1,3}){2})(?!172\.(?:1[6-9]|2\d|3[0-1])(?:\.\d{1,3}){2})(?:[1-9]\d?|1\d\d|2[01]\d|22[0-3])(?:\.(?:1?\d{1,2}|2[0-4]\d|25[0-5])){2}(?:\.(?:[1-9]\d?|1\d\d|2[0-4]\d|25[0-4]))|(?:(?:[a-z\u00a1-\uffff0-9]-*)*[a-z\u00a1-\uffff0-9]+)(?:\.(?:[a-z\u00a1-\uffff0-9]-*)*[a-z\u00a1-\uffff0-9]+)*(?:\.(?:[a-z\u00a1-\uffff]{2,}))\.?)(?::\d{2,5})?(?:[/?#]\S*)?") (def shortened-url-pattern #"(https?://)?(?:\S+(?::\S*)?@)?(?:(?!(?:10|127)(?:\.\d{1,3}){3})(?!(?:169\.254|192\.168)(?:\.\d{1,3}){2})(?!172\.(?:1[6-9]|2\d|3[0-1])(?:\.\d{1,3}){2})(?:[1-9]\d?|1\d\d|2[01]\d|22[0-3])(?:\.(?:1?\d{1,2}|2[0-4]\d|25[0-5])){2}(?:\.(?:[1-9]\d?|1\d\d|2[0-4]\d|25[0-4]))|(?:(?:[a-z\u00a1-\uffff0-9]-*)*[a-z\u00a1-\uffff0-9]+)(?:\.(?:[a-z\u00a1-\uffff0-9]-*)*[a-z\u00a1-\uffff0-9]+)*(?:\.(?:[a-z\u00a1-\uffff]{2,}))\.?)(?::\d{2,5})?(?:[/?#]\S*)?")
@ -104,9 +106,11 @@
(post-status-with-images status-text urls [])) (post-status-with-images status-text urls []))
([status-text [url & urls] ids] ([status-text [url & urls] ids]
(if url (if url
(.get (if (string/starts-with? url "https://") https http) url (-> request
(.get url)
(.on "response"
(fn [image-stream] (fn [image-stream]
(post-image image-stream status-text #(post-status-with-images status-text urls (conj ids %))))) (post-image image-stream status-text #(post-status-with-images status-text urls (conj ids %))))))
(post-status status-text (not-empty ids))))) (post-status status-text (not-empty ids)))))
(defn get-mastodon-timeline [callback] (defn get-mastodon-timeline [callback]

View file

@ -8,7 +8,9 @@
"mastodon-api": "1.3.0", "mastodon-api": "1.3.0",
"rss-parser": "3.1.2", "rss-parser": "3.1.2",
"tumblr": "0.4.1", "tumblr": "0.4.1",
"twitter": "1.7.1" "twitter": "1.7.1",
"deasync": "0.1.13",
"request": "2.88.0"
}, },
"scripts": { "scripts": {
"start": "./mastodon-bot.cljs" "start": "./mastodon-bot.cljs"