From 7286db72b050aa2475191dcc86ca92599a84cd18 Mon Sep 17 00:00:00 2001 From: jem Date: Thu, 30 Apr 2020 09:08:51 +0200 Subject: [PATCH] initially draft the idea --- mastodon_bot/infra.cljs | 41 +++++++++++++++ mastodon_bot/mastodon_api.cljs | 96 ++++++++++++++++++++++++++++++++++ mastodon_bot/twitter_api.cljs | 58 ++++++++++++++++++++ shadow-cljs.edn | 2 +- 4 files changed, 196 insertions(+), 1 deletion(-) create mode 100755 mastodon_bot/infra.cljs create mode 100755 mastodon_bot/mastodon_api.cljs create mode 100755 mastodon_bot/twitter_api.cljs diff --git a/mastodon_bot/infra.cljs b/mastodon_bot/infra.cljs new file mode 100755 index 0000000..a0fc28e --- /dev/null +++ b/mastodon_bot/infra.cljs @@ -0,0 +1,41 @@ +(ns mastodon-bot.infra + (:require + [cljs.reader :as edn] + [clojure.set :refer [rename-keys]] + [clojure.string :as string])) + +(defn exit-with-error [error] + (js/console.error error) + (js/process.exit 1)) + +(defn find-config [] + (let [config (or (first *command-line-args*) + (-> js/process .-env .-MASTODON_BOT_CONFIG) + "config.edn")] + (if (fs/existsSync config) + config + (exit-with-error (str "failed to read config: " config))))) + +(def config (-> (find-config) (fs/readFileSync #js {:encoding "UTF-8"}) edn/read-string)) + +(def mastodon-config (:mastodon config)) + +(defn js->edn [data] + (js->clj data :keywordize-keys true)) + +(defn trim-text [text] + (cond + + (nil? max-post-length) + text + + (> (count text) max-post-length) + (reduce + (fn [text word] + (if (> (+ (count text) (count word)) (- max-post-length 3)) + (reduced (str text "...")) + (str text " " word))) + "" + (string/split text #" ")) + + :else text)) diff --git a/mastodon_bot/mastodon_api.cljs b/mastodon_bot/mastodon_api.cljs new file mode 100755 index 0000000..15fb1fe --- /dev/null +++ b/mastodon_bot/mastodon_api.cljs @@ -0,0 +1,96 @@ +(ns mastodon-bot.mastodon-api + (:require + [clojure.spec.alpha :as s] + [clojure.spec.test.alpha :as st] + [orchestra.core :refer-macros [defn-spec]] + [cljs.reader :as edn] + [clojure.set :refer [rename-keys]] + [clojure.string :as string] + [mastodon-bot.infra :as infra] + ["mastodon-api" :as mastodon])) + +; Todo: think about how namespaced keywords & clj->js can play nicely together +(s/def :access_token string?) +(s/def :account-id string?) +(s/def :api_url string?) + +(s/def ::mastodon-config (s/keys :req [:access_token :account-id :access_token])) + +(defn-spec mastodon-config ::mastodon-config + [config any?] + (:mastodon config)) + +(defn-spec mastodon-client any? + [mastodon-config ::mastodon-config] + (or (some-> mastodon-config clj->js mastodon.) + (infra/exit-with-error "missing Mastodon client configuration!"))) + +(def content-filter-regexes (mapv re-pattern (:content-filters mastodon-config))) + +(def keyword-filter-regexes (mapv re-pattern (:keyword-filters mastodon-config))) + +(def append-screen-name? (boolean (:append-screen-name? mastodon-config))) + +(def max-post-length (:max-post-length mastodon-config)) + +(defn blocked-content? [text] + (boolean + (or (some #(re-find % text) content-filter-regexes) + (when (not-empty keyword-filter-regexes) + (empty? (some #(re-find % text) keyword-filter-regexes)))))) + +(defn delete-status [status] + (.delete mastodon-client (str "statuses/" status) #js {})) + +(defn set-signature [text] + (if-let [signature (:signature mastodon-config )] + (str text "\n" signature) + text)) + +(defn post-status + ([status-text] + (post-status status-text nil)) + ([status-text media-ids] + (let [{:keys [sensitive? signature visibility]} mastodon-config] + (.post mastodon-client "statuses" + (clj->js (merge {:status (-> status-text resolve-urls set-signature)} + (when media-ids {:media_ids media-ids}) + (when sensitive? {:sensitive sensitive?}) + (when visibility {:visibility visibility}))))))) + +(defn post-image [image-stream description callback] + (-> (.post mastodon-client "media" #js {:file image-stream :description description}) + (.then #(-> % .-data .-id callback)))) + +(defn post-status-with-images + ([status-text urls] + (post-status-with-images status-text urls [])) + ([status-text [url & urls] ids] + (if url + (-> request + (.get url) + (.on "response" + (fn [image-stream] + (post-image image-stream status-text #(post-status-with-images status-text urls (conj ids %)))))) + (post-status status-text (not-empty ids))))) + +(defn get-mastodon-timeline [callback] + (.then (.get mastodon-client (str "accounts/" (:account-id mastodon-config)"/statuses") #js {}) + #(let [response (-> % .-data js->edn)] + (if-let [error (:error response)] + (exit-with-error error) + (callback response))))) + +(defn perform-replacements [post] + (assoc post :text (reduce-kv string/replace (:text post) (:replacements mastodon-config))) + ) + +(defn post-items [last-post-time items] + (doseq [{:keys [text media-links]} (->> items + (remove #(blocked-content? (:text %))) + (filter #(> (:created-at %) last-post-time)) + (map perform-replacements))] + (if media-links + (post-status-with-images text media-links) + (when-not (:media-only? mastodon-config) + (post-status text))))) diff --git a/mastodon_bot/twitter_api.cljs b/mastodon_bot/twitter_api.cljs new file mode 100755 index 0000000..db9c99c --- /dev/null +++ b/mastodon_bot/twitter_api.cljs @@ -0,0 +1,58 @@ +(ns mastodon-bot.twitter-api + (:require + [clojure.set :refer [rename-keys]] + [clojure.string :as string] + ["deasync" :as deasync] + ["request" :as request] + ["twitter" :as twitter])) + + +(defn resolve-url [[uri]] + (try + (or + (some-> ((deasync request) + #js {:method "GET" + :uri (if (string/starts-with? uri "https://") uri (str "https://" uri)) + :followRedirect false}) + (.-headers) + (.-location) + (string/replace "?mbid=social_twitter" "")) + uri) + (catch js/Error _ uri))) + +(def shortened-url-pattern #"(https?://)?(?:\S+(?::\S*)?@)?(?:(?!(?:10|127)(?:\.\d{1,3}){3})(?!(?:169\.254|192\.168)(?:\.\d{1,3}){2})(?!172\.(?:1[6-9]|2\d|3[0-1])(?:\.\d{1,3}){2})(?:[1-9]\d?|1\d\d|2[01]\d|22[0-3])(?:\.(?:1?\d{1,2}|2[0-4]\d|25[0-5])){2}(?:\.(?:[1-9]\d?|1\d\d|2[0-4]\d|25[0-4]))|(?:(?:[a-z\u00a1-\uffff0-9]-*)*[a-z\u00a1-\uffff0-9]+)(?:\.(?:[a-z\u00a1-\uffff0-9]-*)*[a-z\u00a1-\uffff0-9]+)*(?:\.(?:[a-z\u00a1-\uffff]{2,}))\.?)(?::\d{2,5})?(?:[/?#]\S*)?") + +(defn resolve-urls [text] + (cond-> text + (:resolve-urls? mastodon-config) + (string/replace shortened-url-pattern resolve-url) + (:nitter-urls? mastodon-config) + (string/replace #"https://twitter.com" "https://nitter.net"))) + +; If the text ends in a link to the media (which is uploaded anyway), +; chop it off instead of including the link in the toot +(defn chop-tail-media-url [text media] + (string/replace text #" (\S+)$" #(if (in (%1 1) (map :url media)) "" (%1 0)))) + +(defn parse-tweet [{created-at :created_at + text :full_text + {:keys [media]} :extended_entities + {:keys [screen_name]} :user :as tweet}] + {:created-at (js/Date. created-at) + :text (trim-text (str (chop-tail-media-url text media) (if append-screen-name? ("\n - " screen_name) ""))) + :media-links (keep #(when (= (:type %) "photo") (:media_url_https %)) media)}) + +(defn post-tweets [last-post-time] + (fn [error tweets response] + (if error + (exit-with-error error) + (->> (js->edn tweets) + (map parse-tweet) + (post-items last-post-time))))) + +(defn twitter-client [access-keys] + (try + (twitter. (clj->js access-keys)) + (catch js/Error e + (exit-with-error + (str "failed to connect to Twitter: " (.-message e)))))) diff --git a/shadow-cljs.edn b/shadow-cljs.edn index d666e71..14d708a 100644 --- a/shadow-cljs.edn +++ b/shadow-cljs.edn @@ -1,5 +1,5 @@ {:source-paths ["mastodon_bot"] - :dependencies [] + :dependencies [[orchestra "2018.12.06-2"]] :builds {:app {:target :node-script :output-to "target/mastodon-bot.js" :main mastodon-bot.core/main