From 7286db72b050aa2475191dcc86ca92599a84cd18 Mon Sep 17 00:00:00 2001 From: jem Date: Thu, 30 Apr 2020 09:08:51 +0200 Subject: [PATCH 01/12] initially draft the idea --- mastodon_bot/infra.cljs | 41 +++++++++++++++ mastodon_bot/mastodon_api.cljs | 96 ++++++++++++++++++++++++++++++++++ mastodon_bot/twitter_api.cljs | 58 ++++++++++++++++++++ shadow-cljs.edn | 2 +- 4 files changed, 196 insertions(+), 1 deletion(-) create mode 100755 mastodon_bot/infra.cljs create mode 100755 mastodon_bot/mastodon_api.cljs create mode 100755 mastodon_bot/twitter_api.cljs diff --git a/mastodon_bot/infra.cljs b/mastodon_bot/infra.cljs new file mode 100755 index 0000000..a0fc28e --- /dev/null +++ b/mastodon_bot/infra.cljs @@ -0,0 +1,41 @@ +(ns mastodon-bot.infra + (:require + [cljs.reader :as edn] + [clojure.set :refer [rename-keys]] + [clojure.string :as string])) + +(defn exit-with-error [error] + (js/console.error error) + (js/process.exit 1)) + +(defn find-config [] + (let [config (or (first *command-line-args*) + (-> js/process .-env .-MASTODON_BOT_CONFIG) + "config.edn")] + (if (fs/existsSync config) + config + (exit-with-error (str "failed to read config: " config))))) + +(def config (-> (find-config) (fs/readFileSync #js {:encoding "UTF-8"}) edn/read-string)) + +(def mastodon-config (:mastodon config)) + +(defn js->edn [data] + (js->clj data :keywordize-keys true)) + +(defn trim-text [text] + (cond + + (nil? max-post-length) + text + + (> (count text) max-post-length) + (reduce + (fn [text word] + (if (> (+ (count text) (count word)) (- max-post-length 3)) + (reduced (str text "...")) + (str text " " word))) + "" + (string/split text #" ")) + + :else text)) diff --git a/mastodon_bot/mastodon_api.cljs b/mastodon_bot/mastodon_api.cljs new file mode 100755 index 0000000..15fb1fe --- /dev/null +++ b/mastodon_bot/mastodon_api.cljs @@ -0,0 +1,96 @@ +(ns mastodon-bot.mastodon-api + (:require + [clojure.spec.alpha :as s] + [clojure.spec.test.alpha :as st] + [orchestra.core :refer-macros [defn-spec]] + [cljs.reader :as edn] + [clojure.set :refer [rename-keys]] + [clojure.string :as string] + [mastodon-bot.infra :as infra] + ["mastodon-api" :as mastodon])) + +; Todo: think about how namespaced keywords & clj->js can play nicely together +(s/def :access_token string?) +(s/def :account-id string?) +(s/def :api_url string?) + +(s/def ::mastodon-config (s/keys :req [:access_token :account-id :access_token])) + +(defn-spec mastodon-config ::mastodon-config + [config any?] + (:mastodon config)) + +(defn-spec mastodon-client any? + [mastodon-config ::mastodon-config] + (or (some-> mastodon-config clj->js mastodon.) + (infra/exit-with-error "missing Mastodon client configuration!"))) + +(def content-filter-regexes (mapv re-pattern (:content-filters mastodon-config))) + +(def keyword-filter-regexes (mapv re-pattern (:keyword-filters mastodon-config))) + +(def append-screen-name? (boolean (:append-screen-name? mastodon-config))) + +(def max-post-length (:max-post-length mastodon-config)) + +(defn blocked-content? [text] + (boolean + (or (some #(re-find % text) content-filter-regexes) + (when (not-empty keyword-filter-regexes) + (empty? (some #(re-find % text) keyword-filter-regexes)))))) + +(defn delete-status [status] + (.delete mastodon-client (str "statuses/" status) #js {})) + +(defn set-signature [text] + (if-let [signature (:signature mastodon-config )] + (str text "\n" signature) + text)) + +(defn post-status + ([status-text] + (post-status status-text nil)) + ([status-text media-ids] + (let [{:keys [sensitive? signature visibility]} mastodon-config] + (.post mastodon-client "statuses" + (clj->js (merge {:status (-> status-text resolve-urls set-signature)} + (when media-ids {:media_ids media-ids}) + (when sensitive? {:sensitive sensitive?}) + (when visibility {:visibility visibility}))))))) + +(defn post-image [image-stream description callback] + (-> (.post mastodon-client "media" #js {:file image-stream :description description}) + (.then #(-> % .-data .-id callback)))) + +(defn post-status-with-images + ([status-text urls] + (post-status-with-images status-text urls [])) + ([status-text [url & urls] ids] + (if url + (-> request + (.get url) + (.on "response" + (fn [image-stream] + (post-image image-stream status-text #(post-status-with-images status-text urls (conj ids %)))))) + (post-status status-text (not-empty ids))))) + +(defn get-mastodon-timeline [callback] + (.then (.get mastodon-client (str "accounts/" (:account-id mastodon-config)"/statuses") #js {}) + #(let [response (-> % .-data js->edn)] + (if-let [error (:error response)] + (exit-with-error error) + (callback response))))) + +(defn perform-replacements [post] + (assoc post :text (reduce-kv string/replace (:text post) (:replacements mastodon-config))) + ) + +(defn post-items [last-post-time items] + (doseq [{:keys [text media-links]} (->> items + (remove #(blocked-content? (:text %))) + (filter #(> (:created-at %) last-post-time)) + (map perform-replacements))] + (if media-links + (post-status-with-images text media-links) + (when-not (:media-only? mastodon-config) + (post-status text))))) diff --git a/mastodon_bot/twitter_api.cljs b/mastodon_bot/twitter_api.cljs new file mode 100755 index 0000000..db9c99c --- /dev/null +++ b/mastodon_bot/twitter_api.cljs @@ -0,0 +1,58 @@ +(ns mastodon-bot.twitter-api + (:require + [clojure.set :refer [rename-keys]] + [clojure.string :as string] + ["deasync" :as deasync] + ["request" :as request] + ["twitter" :as twitter])) + + +(defn resolve-url [[uri]] + (try + (or + (some-> ((deasync request) + #js {:method "GET" + :uri (if (string/starts-with? uri "https://") uri (str "https://" uri)) + :followRedirect false}) + (.-headers) + (.-location) + (string/replace "?mbid=social_twitter" "")) + uri) + (catch js/Error _ uri))) + +(def shortened-url-pattern #"(https?://)?(?:\S+(?::\S*)?@)?(?:(?!(?:10|127)(?:\.\d{1,3}){3})(?!(?:169\.254|192\.168)(?:\.\d{1,3}){2})(?!172\.(?:1[6-9]|2\d|3[0-1])(?:\.\d{1,3}){2})(?:[1-9]\d?|1\d\d|2[01]\d|22[0-3])(?:\.(?:1?\d{1,2}|2[0-4]\d|25[0-5])){2}(?:\.(?:[1-9]\d?|1\d\d|2[0-4]\d|25[0-4]))|(?:(?:[a-z\u00a1-\uffff0-9]-*)*[a-z\u00a1-\uffff0-9]+)(?:\.(?:[a-z\u00a1-\uffff0-9]-*)*[a-z\u00a1-\uffff0-9]+)*(?:\.(?:[a-z\u00a1-\uffff]{2,}))\.?)(?::\d{2,5})?(?:[/?#]\S*)?") + +(defn resolve-urls [text] + (cond-> text + (:resolve-urls? mastodon-config) + (string/replace shortened-url-pattern resolve-url) + (:nitter-urls? mastodon-config) + (string/replace #"https://twitter.com" "https://nitter.net"))) + +; If the text ends in a link to the media (which is uploaded anyway), +; chop it off instead of including the link in the toot +(defn chop-tail-media-url [text media] + (string/replace text #" (\S+)$" #(if (in (%1 1) (map :url media)) "" (%1 0)))) + +(defn parse-tweet [{created-at :created_at + text :full_text + {:keys [media]} :extended_entities + {:keys [screen_name]} :user :as tweet}] + {:created-at (js/Date. created-at) + :text (trim-text (str (chop-tail-media-url text media) (if append-screen-name? ("\n - " screen_name) ""))) + :media-links (keep #(when (= (:type %) "photo") (:media_url_https %)) media)}) + +(defn post-tweets [last-post-time] + (fn [error tweets response] + (if error + (exit-with-error error) + (->> (js->edn tweets) + (map parse-tweet) + (post-items last-post-time))))) + +(defn twitter-client [access-keys] + (try + (twitter. (clj->js access-keys)) + (catch js/Error e + (exit-with-error + (str "failed to connect to Twitter: " (.-message e)))))) diff --git a/shadow-cljs.edn b/shadow-cljs.edn index d666e71..14d708a 100644 --- a/shadow-cljs.edn +++ b/shadow-cljs.edn @@ -1,5 +1,5 @@ {:source-paths ["mastodon_bot"] - :dependencies [] + :dependencies [[orchestra "2018.12.06-2"]] :builds {:app {:target :node-script :output-to "target/mastodon-bot.js" :main mastodon-bot.core/main From f5fcc142ae93e7a4e4c96c664efe64d8c0c871d2 Mon Sep 17 00:00:00 2001 From: jem Date: Tue, 12 May 2020 18:17:37 +0200 Subject: [PATCH 02/12] compiles after refactoring --- .gitignore | 2 + mastodon_bot/core.cljs | 245 ------------------ mastodon_bot/infra.cljs | 41 --- mastodon_bot/mastodon_api.cljs | 96 ------- mastodon_bot/twitter_api.cljs | 58 ----- shadow-cljs.edn | 10 +- src/main/mastodon_bot/core.cljs | 160 ++++++++++++ src/main/mastodon_bot/infra.cljs | 8 + src/main/mastodon_bot/mastodon_api.cljs | 165 ++++++++++++ .../test/mastodon_bot}/core_test.cljs | 0 10 files changed, 343 insertions(+), 442 deletions(-) delete mode 100755 mastodon_bot/core.cljs delete mode 100755 mastodon_bot/infra.cljs delete mode 100755 mastodon_bot/mastodon_api.cljs delete mode 100755 mastodon_bot/twitter_api.cljs create mode 100755 src/main/mastodon_bot/core.cljs create mode 100755 src/main/mastodon_bot/infra.cljs create mode 100755 src/main/mastodon_bot/mastodon_api.cljs rename {mastodon_bot => src/test/mastodon_bot}/core_test.cljs (100%) diff --git a/.gitignore b/.gitignore index 55ab5df..e3d8124 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,5 @@ +.shadow-cljs config.edn package-lock.json /node_modules +target/mastodon-bot.js diff --git a/mastodon_bot/core.cljs b/mastodon_bot/core.cljs deleted file mode 100755 index 9e02d6a..0000000 --- a/mastodon_bot/core.cljs +++ /dev/null @@ -1,245 +0,0 @@ -#!/usr/bin/env lumo - -(ns mastodon-bot.core - (:require - [cljs.core :refer [*command-line-args*]] - [cljs.reader :as edn] - [clojure.set :refer [rename-keys]] - [clojure.string :as string] - ["deasync" :as deasync] - ["request" :as request] - ["fs" :as fs] - ["mastodon-api" :as mastodon] - ["rss-parser" :as rss] - ["tumblr" :as tumblr] - ["twitter" :as twitter])) - -(defn exit-with-error [error] - (js/console.error error) - (js/process.exit 1)) - -(defn find-config [] - (let [config (or (first *command-line-args*) - (-> js/process .-env .-MASTODON_BOT_CONFIG) - "config.edn")] - (if (fs/existsSync config) - config - (exit-with-error (str "failed to read config: " config))))) - -(def config (-> (find-config) (fs/readFileSync #js {:encoding "UTF-8"}) edn/read-string)) - -(def mastodon-config (:mastodon config)) - -(def mastodon-client (or (some-> mastodon-config clj->js mastodon.) - (exit-with-error "missing Mastodon client configuration!"))) - -(def content-filter-regexes (mapv re-pattern (:content-filters mastodon-config))) - -(def keyword-filter-regexes (mapv re-pattern (:keyword-filters mastodon-config))) - -(def append-screen-name? (boolean (:append-screen-name? mastodon-config))) - -(def max-post-length (:max-post-length mastodon-config)) - -(defn blocked-content? [text] - (boolean - (or (some #(re-find % text) content-filter-regexes) - (when (not-empty keyword-filter-regexes) - (empty? (some #(re-find % text) keyword-filter-regexes)))))) - -(defn js->edn [data] - (js->clj data :keywordize-keys true)) - -(defn trim-text [text] - (cond - - (nil? max-post-length) - text - - (> (count text) max-post-length) - (reduce - (fn [text word] - (if (> (+ (count text) (count word)) (- max-post-length 3)) - (reduced (str text "...")) - (str text " " word))) - "" - (string/split text #" ")) - - :else text)) - -(defn delete-status [status] - (.delete mastodon-client (str "statuses/" status) #js {})) - -(defn resolve-url [[uri]] - (try - (or - (some-> ((deasync request) - #js {:method "GET" - :uri (if (string/starts-with? uri "https://") uri (str "https://" uri)) - :followRedirect false}) - (.-headers) - (.-location) - (string/replace "?mbid=social_twitter" "")) - uri) - (catch js/Error _ uri))) - -(def shortened-url-pattern #"(https?://)?(?:\S+(?::\S*)?@)?(?:(?!(?:10|127)(?:\.\d{1,3}){3})(?!(?:169\.254|192\.168)(?:\.\d{1,3}){2})(?!172\.(?:1[6-9]|2\d|3[0-1])(?:\.\d{1,3}){2})(?:[1-9]\d?|1\d\d|2[01]\d|22[0-3])(?:\.(?:1?\d{1,2}|2[0-4]\d|25[0-5])){2}(?:\.(?:[1-9]\d?|1\d\d|2[0-4]\d|25[0-4]))|(?:(?:[a-z\u00a1-\uffff0-9]-*)*[a-z\u00a1-\uffff0-9]+)(?:\.(?:[a-z\u00a1-\uffff0-9]-*)*[a-z\u00a1-\uffff0-9]+)*(?:\.(?:[a-z\u00a1-\uffff]{2,}))\.?)(?::\d{2,5})?(?:[/?#]\S*)?") - -(defn resolve-urls [text] - (cond-> text - (:resolve-urls? mastodon-config) - (string/replace shortened-url-pattern resolve-url) - (:nitter-urls? mastodon-config) - (string/replace #"https://twitter.com" "https://nitter.net"))) - -(defn set-signature [text] - (if-let [signature (:signature mastodon-config )] - (str text "\n" signature) - text)) - -(defn post-status - ([status-text] - (post-status status-text nil)) - ([status-text media-ids] - (let [{:keys [sensitive? signature visibility]} mastodon-config] - (.post mastodon-client "statuses" - (clj->js (merge {:status (-> status-text resolve-urls set-signature)} - (when media-ids {:media_ids media-ids}) - (when sensitive? {:sensitive sensitive?}) - (when visibility {:visibility visibility}))))))) - -(defn post-image [image-stream description callback] - (-> (.post mastodon-client "media" #js {:file image-stream :description description}) - (.then #(-> % .-data .-id callback)))) - -(defn post-status-with-images - ([status-text urls] - (post-status-with-images status-text urls [])) - ([status-text [url & urls] ids] - (if url - (-> request - (.get url) - (.on "response" - (fn [image-stream] - (post-image image-stream status-text #(post-status-with-images status-text urls (conj ids %)))))) - (post-status status-text (not-empty ids))))) - -(defn get-mastodon-timeline [callback] - (.then (.get mastodon-client (str "accounts/" (:account-id mastodon-config)"/statuses") #js {}) - #(let [response (-> % .-data js->edn)] - (if-let [error (:error response)] - (exit-with-error error) - (callback response))))) - -(defn perform-replacements [post] - (assoc post :text (reduce-kv string/replace (:text post) (:replacements mastodon-config))) - ) - -(defn post-items [last-post-time items] - (doseq [{:keys [text media-links]} (->> items - (remove #(blocked-content? (:text %))) - (filter #(> (:created-at %) last-post-time)) - (map perform-replacements))] - (if media-links - (post-status-with-images text media-links) - (when-not (:media-only? mastodon-config) - (post-status text))))) - -(defn in [needle haystack] - (some (partial = needle) haystack)) - -; If the text ends in a link to the media (which is uploaded anyway), -; chop it off instead of including the link in the toot -(defn chop-tail-media-url [text media] - (string/replace text #" (\S+)$" #(if (in (%1 1) (map :url media)) "" (%1 0)))) - -(defn parse-tweet [{created-at :created_at - text :full_text - {:keys [media]} :extended_entities - {:keys [screen_name]} :user :as tweet}] - {:created-at (js/Date. created-at) - :text (trim-text (str (chop-tail-media-url text media) (if append-screen-name? ("\n - " screen_name) ""))) - :media-links (keep #(when (= (:type %) "photo") (:media_url_https %)) media)}) - -(defmulti parse-tumblr-post :type) - -(defmethod parse-tumblr-post "text" [{:keys [body date short_url]}] - {:created-at (js/Date. date) - :text (str (trim-text body) "\n\n" short_url)}) - -(defmethod parse-tumblr-post "photo" [{:keys [caption date photos short_url] :as post}] - {:created-at (js/Date. date) - :text (string/join "\n" [(string/replace caption #"<[^>]*>" "") short_url]) - :media-links (mapv #(-> % :original_size :url) photos)}) - -(defmethod parse-tumblr-post :default [post]) - -(defn post-tumblrs [last-post-time] - (fn [err response] - (->> response - js->edn - :posts - (mapv parse-tumblr-post) - (post-items last-post-time)))) - -(defn post-tweets [last-post-time] - (fn [error tweets response] - (if error - (exit-with-error error) - (->> (js->edn tweets) - (map parse-tweet) - (post-items last-post-time))))) - -(defn strip-utm [news-link] - (first (string/split news-link #"\?utm"))) - -(defn parse-feed [last-post-time parser [title url]] - (-> (.parseURL parser url) - (.then #(post-items - last-post-time - (for [{:keys [title isoDate pubDate content link]} (-> % js->edn :items)] - {:created-at (js/Date. (or isoDate pubDate)) - :text (str (trim-text title) "\n\n" (strip-utm link))}))))) - -(defn twitter-client [access-keys] - (try - (twitter. (clj->js access-keys)) - (catch js/Error e - (exit-with-error - (str "failed to connect to Twitter: " (.-message e)))))) - -(defn tumblr-client [access-keys account] - (try - (tumblr/Blog. account (clj->js access-keys)) - (catch js/Error e - (exit-with-error - (str "failed to connect to Tumblr account " account ": " (.-message e)))))) - -(defn -main [] - (get-mastodon-timeline - (fn [timeline] - (let [last-post-time (-> timeline first :created_at (js/Date.))] - ;;post from Twitter - (when-let [twitter-config (:twitter config)] - (let [{:keys [access-keys accounts include-replies? include-rts?]} twitter-config - client (twitter-client access-keys)] - (doseq [account accounts] - (.get client - "statuses/user_timeline" - #js {:screen_name account - :tweet_mode "extended" - :include_rts (boolean include-rts?) - :exclude_replies (not (boolean include-replies?))} - (post-tweets last-post-time))))) - ;;post from Tumblr - (when-let [{:keys [access-keys accounts limit]} (:tumblr config)] - (doseq [account accounts] - (let [client (tumblr-client access-keys account)] - (.posts client #js {:limit (or limit 5)} (post-tumblrs last-post-time))))) - ;;post from RSS - (when-let [feeds (some-> config :rss)] - (let [parser (rss.)] - (doseq [feed feeds] - (parse-feed last-post-time parser feed)))))))) - -(set! *main-cli-fn* -main) diff --git a/mastodon_bot/infra.cljs b/mastodon_bot/infra.cljs deleted file mode 100755 index a0fc28e..0000000 --- a/mastodon_bot/infra.cljs +++ /dev/null @@ -1,41 +0,0 @@ -(ns mastodon-bot.infra - (:require - [cljs.reader :as edn] - [clojure.set :refer [rename-keys]] - [clojure.string :as string])) - -(defn exit-with-error [error] - (js/console.error error) - (js/process.exit 1)) - -(defn find-config [] - (let [config (or (first *command-line-args*) - (-> js/process .-env .-MASTODON_BOT_CONFIG) - "config.edn")] - (if (fs/existsSync config) - config - (exit-with-error (str "failed to read config: " config))))) - -(def config (-> (find-config) (fs/readFileSync #js {:encoding "UTF-8"}) edn/read-string)) - -(def mastodon-config (:mastodon config)) - -(defn js->edn [data] - (js->clj data :keywordize-keys true)) - -(defn trim-text [text] - (cond - - (nil? max-post-length) - text - - (> (count text) max-post-length) - (reduce - (fn [text word] - (if (> (+ (count text) (count word)) (- max-post-length 3)) - (reduced (str text "...")) - (str text " " word))) - "" - (string/split text #" ")) - - :else text)) diff --git a/mastodon_bot/mastodon_api.cljs b/mastodon_bot/mastodon_api.cljs deleted file mode 100755 index 15fb1fe..0000000 --- a/mastodon_bot/mastodon_api.cljs +++ /dev/null @@ -1,96 +0,0 @@ -(ns mastodon-bot.mastodon-api - (:require - [clojure.spec.alpha :as s] - [clojure.spec.test.alpha :as st] - [orchestra.core :refer-macros [defn-spec]] - [cljs.reader :as edn] - [clojure.set :refer [rename-keys]] - [clojure.string :as string] - [mastodon-bot.infra :as infra] - ["mastodon-api" :as mastodon])) - -; Todo: think about how namespaced keywords & clj->js can play nicely together -(s/def :access_token string?) -(s/def :account-id string?) -(s/def :api_url string?) - -(s/def ::mastodon-config (s/keys :req [:access_token :account-id :access_token])) - -(defn-spec mastodon-config ::mastodon-config - [config any?] - (:mastodon config)) - -(defn-spec mastodon-client any? - [mastodon-config ::mastodon-config] - (or (some-> mastodon-config clj->js mastodon.) - (infra/exit-with-error "missing Mastodon client configuration!"))) - -(def content-filter-regexes (mapv re-pattern (:content-filters mastodon-config))) - -(def keyword-filter-regexes (mapv re-pattern (:keyword-filters mastodon-config))) - -(def append-screen-name? (boolean (:append-screen-name? mastodon-config))) - -(def max-post-length (:max-post-length mastodon-config)) - -(defn blocked-content? [text] - (boolean - (or (some #(re-find % text) content-filter-regexes) - (when (not-empty keyword-filter-regexes) - (empty? (some #(re-find % text) keyword-filter-regexes)))))) - -(defn delete-status [status] - (.delete mastodon-client (str "statuses/" status) #js {})) - -(defn set-signature [text] - (if-let [signature (:signature mastodon-config )] - (str text "\n" signature) - text)) - -(defn post-status - ([status-text] - (post-status status-text nil)) - ([status-text media-ids] - (let [{:keys [sensitive? signature visibility]} mastodon-config] - (.post mastodon-client "statuses" - (clj->js (merge {:status (-> status-text resolve-urls set-signature)} - (when media-ids {:media_ids media-ids}) - (when sensitive? {:sensitive sensitive?}) - (when visibility {:visibility visibility}))))))) - -(defn post-image [image-stream description callback] - (-> (.post mastodon-client "media" #js {:file image-stream :description description}) - (.then #(-> % .-data .-id callback)))) - -(defn post-status-with-images - ([status-text urls] - (post-status-with-images status-text urls [])) - ([status-text [url & urls] ids] - (if url - (-> request - (.get url) - (.on "response" - (fn [image-stream] - (post-image image-stream status-text #(post-status-with-images status-text urls (conj ids %)))))) - (post-status status-text (not-empty ids))))) - -(defn get-mastodon-timeline [callback] - (.then (.get mastodon-client (str "accounts/" (:account-id mastodon-config)"/statuses") #js {}) - #(let [response (-> % .-data js->edn)] - (if-let [error (:error response)] - (exit-with-error error) - (callback response))))) - -(defn perform-replacements [post] - (assoc post :text (reduce-kv string/replace (:text post) (:replacements mastodon-config))) - ) - -(defn post-items [last-post-time items] - (doseq [{:keys [text media-links]} (->> items - (remove #(blocked-content? (:text %))) - (filter #(> (:created-at %) last-post-time)) - (map perform-replacements))] - (if media-links - (post-status-with-images text media-links) - (when-not (:media-only? mastodon-config) - (post-status text))))) diff --git a/mastodon_bot/twitter_api.cljs b/mastodon_bot/twitter_api.cljs deleted file mode 100755 index db9c99c..0000000 --- a/mastodon_bot/twitter_api.cljs +++ /dev/null @@ -1,58 +0,0 @@ -(ns mastodon-bot.twitter-api - (:require - [clojure.set :refer [rename-keys]] - [clojure.string :as string] - ["deasync" :as deasync] - ["request" :as request] - ["twitter" :as twitter])) - - -(defn resolve-url [[uri]] - (try - (or - (some-> ((deasync request) - #js {:method "GET" - :uri (if (string/starts-with? uri "https://") uri (str "https://" uri)) - :followRedirect false}) - (.-headers) - (.-location) - (string/replace "?mbid=social_twitter" "")) - uri) - (catch js/Error _ uri))) - -(def shortened-url-pattern #"(https?://)?(?:\S+(?::\S*)?@)?(?:(?!(?:10|127)(?:\.\d{1,3}){3})(?!(?:169\.254|192\.168)(?:\.\d{1,3}){2})(?!172\.(?:1[6-9]|2\d|3[0-1])(?:\.\d{1,3}){2})(?:[1-9]\d?|1\d\d|2[01]\d|22[0-3])(?:\.(?:1?\d{1,2}|2[0-4]\d|25[0-5])){2}(?:\.(?:[1-9]\d?|1\d\d|2[0-4]\d|25[0-4]))|(?:(?:[a-z\u00a1-\uffff0-9]-*)*[a-z\u00a1-\uffff0-9]+)(?:\.(?:[a-z\u00a1-\uffff0-9]-*)*[a-z\u00a1-\uffff0-9]+)*(?:\.(?:[a-z\u00a1-\uffff]{2,}))\.?)(?::\d{2,5})?(?:[/?#]\S*)?") - -(defn resolve-urls [text] - (cond-> text - (:resolve-urls? mastodon-config) - (string/replace shortened-url-pattern resolve-url) - (:nitter-urls? mastodon-config) - (string/replace #"https://twitter.com" "https://nitter.net"))) - -; If the text ends in a link to the media (which is uploaded anyway), -; chop it off instead of including the link in the toot -(defn chop-tail-media-url [text media] - (string/replace text #" (\S+)$" #(if (in (%1 1) (map :url media)) "" (%1 0)))) - -(defn parse-tweet [{created-at :created_at - text :full_text - {:keys [media]} :extended_entities - {:keys [screen_name]} :user :as tweet}] - {:created-at (js/Date. created-at) - :text (trim-text (str (chop-tail-media-url text media) (if append-screen-name? ("\n - " screen_name) ""))) - :media-links (keep #(when (= (:type %) "photo") (:media_url_https %)) media)}) - -(defn post-tweets [last-post-time] - (fn [error tweets response] - (if error - (exit-with-error error) - (->> (js->edn tweets) - (map parse-tweet) - (post-items last-post-time))))) - -(defn twitter-client [access-keys] - (try - (twitter. (clj->js access-keys)) - (catch js/Error e - (exit-with-error - (str "failed to connect to Twitter: " (.-message e)))))) diff --git a/shadow-cljs.edn b/shadow-cljs.edn index 14d708a..49ba2c2 100644 --- a/shadow-cljs.edn +++ b/shadow-cljs.edn @@ -1,6 +1,12 @@ -{:source-paths ["mastodon_bot"] +{:source-paths ["src/main" + "src/test"] :dependencies [[orchestra "2018.12.06-2"]] - :builds {:app {:target :node-script + :builds {:dev {:target :node-script + :repl-init-ns mastodon-bot.core + :output-to "target/mastodon-bot.js" + :main mastodon-bot.core/dummy + :repl-pprint true} + :app {:target :node-script :output-to "target/mastodon-bot.js" :main mastodon-bot.core/main :compiler-options {:optimizations :simple}}}} diff --git a/src/main/mastodon_bot/core.cljs b/src/main/mastodon_bot/core.cljs new file mode 100755 index 0000000..fedae9b --- /dev/null +++ b/src/main/mastodon_bot/core.cljs @@ -0,0 +1,160 @@ +#!/usr/bin/env lumo + +(ns mastodon-bot.core + (:require + [clojure.spec.alpha :as s] + [clojure.spec.test.alpha :as st] + [orchestra.core :refer-macros [defn-spec]] + [cljs.core :refer [*command-line-args*]] + [cljs.reader :as edn] + [clojure.string :as string] + ["fs" :as fs] + ["rss-parser" :as rss] + ["tumblr" :as tumblr] + ["twitter" :as twitter] + [mastodon-bot.infra :as infra] + [mastodon-bot.mastodon-api :as masto])) + +(defn dummy []) + +(s/def ::mastodon-config masto/mastodon-config?) + +;this has to stay on top - only ns-keywords can be uses in spec +(defn-spec mastodon-config ::mastodon-config + [config any?] + (::mastodon-config config)) + +(defn find-config [] + (let [config (or (first *command-line-args*) + (-> js/process .-env .-MASTODON_BOT_CONFIG) + "config.edn")] + (if (fs/existsSync config) + config + (infra/exit-with-error (str "failed to read config: " config))))) + +(def config (-> (find-config) (fs/readFileSync #js {:encoding "UTF-8"}) edn/read-string)) + +(defn trim-text [text] + (let [max-post-length (masto/max-post-length (mastodon-config config))] + (cond + + (nil? max-post-length) + text + + (> (count text) max-post-length) + (reduce + (fn [text word] + (if (> (+ (count text) (count word)) (- max-post-length 3)) + (reduced (str text "...")) + (str text " " word))) + "" + (string/split text #" ")) + + :else text))) + +(defn in [needle haystack] + (some (partial = needle) haystack)) + +; If the text ends in a link to the media (which is uploaded anyway), +; chop it off instead of including the link in the toot +(defn chop-tail-media-url [text media] + (string/replace text #" (\S+)$" #(if (in (%1 1) (map :url media)) "" (%1 0)))) + +(defn parse-tweet [{created-at :created_at + text :full_text + {:keys [media]} :extended_entities + {:keys [screen_name]} :user :as tweet}] + {:created-at (js/Date. created-at) + :text (trim-text (str (chop-tail-media-url text media) + (if (masto/append-screen-name? (mastodon-config config)) + ("\n - " screen_name) ""))) + :media-links (keep #(when (= (:type %) "photo") (:media_url_https %)) media)}) + +(defmulti parse-tumblr-post :type) + +(defmethod parse-tumblr-post "text" [{:keys [body date short_url]}] + {:created-at (js/Date. date) + :text (str (trim-text body) "\n\n" short_url)}) + +(defmethod parse-tumblr-post "photo" [{:keys [caption date photos short_url] :as post}] + {:created-at (js/Date. date) + :text (string/join "\n" [(string/replace caption #"<[^>]*>" "") short_url]) + :media-links (mapv #(-> % :original_size :url) photos)}) + +(defmethod parse-tumblr-post :default [post]) + +(defn post-tumblrs [last-post-time] + (fn [err response] + (->> response + infra/js->edn + :posts + (mapv parse-tumblr-post) + (masto/post-items + (mastodon-config config) + last-post-time)))) + +(defn post-tweets [last-post-time] + (fn [error tweets response] + (if error + (infra/exit-with-error error) + (->> (infra/js->edn tweets) + (map parse-tweet) + (masto/post-items + (mastodon-config config) + last-post-time))))) + +(defn strip-utm [news-link] + (first (string/split news-link #"\?utm"))) + +(defn parse-feed [last-post-time parser [title url]] + (-> (.parseURL parser url) + (.then #(masto/post-items + (mastodon-config config) + last-post-time + (for [{:keys [title isoDate pubDate content link]} (-> % infra/js->edn :items)] + {:created-at (js/Date. (or isoDate pubDate)) + :text (str (trim-text title) "\n\n" (strip-utm link))}))))) + +(defn twitter-client [access-keys] + (try + (twitter. (clj->js access-keys)) + (catch js/Error e + (infra/exit-with-error + (str "failed to connect to Twitter: " (.-message e)))))) + +(defn tumblr-client [access-keys account] + (try + (tumblr/Blog. account (clj->js access-keys)) + (catch js/Error e + (infra/exit-with-error + (str "failed to connect to Tumblr account " account ": " (.-message e)))))) + +(defn -main [] + (masto/get-mastodon-timeline + (mastodon-config config) + (fn [timeline] + (let [last-post-time (-> timeline first :created_at (js/Date.))] + ;;post from Twitter + (when-let [twitter-config (:twitter config)] + (let [{:keys [access-keys accounts include-replies? include-rts?]} twitter-config + client (twitter-client access-keys)] + (doseq [account accounts] + (.get client + "statuses/user_timeline" + #js {:screen_name account + :tweet_mode "extended" + :include_rts (boolean include-rts?) + :exclude_replies (not (boolean include-replies?))} + (post-tweets last-post-time))))) + ;;post from Tumblr + (when-let [{:keys [access-keys accounts limit]} (:tumblr config)] + (doseq [account accounts] + (let [client (tumblr-client access-keys account)] + (.posts client #js {:limit (or limit 5)} (post-tumblrs last-post-time))))) + ;;post from RSS + (when-let [feeds (some-> config :rss)] + (let [parser (rss.)] + (doseq [feed feeds] + (parse-feed last-post-time parser feed)))))))) + +(set! *main-cli-fn* -main) diff --git a/src/main/mastodon_bot/infra.cljs b/src/main/mastodon_bot/infra.cljs new file mode 100755 index 0000000..c8ec4b6 --- /dev/null +++ b/src/main/mastodon_bot/infra.cljs @@ -0,0 +1,8 @@ +(ns mastodon-bot.infra) + +(defn js->edn [data] + (js->clj data :keywordize-keys true)) + +(defn exit-with-error [error] + (js/console.error error) + (js/process.exit 1)) diff --git a/src/main/mastodon_bot/mastodon_api.cljs b/src/main/mastodon_bot/mastodon_api.cljs new file mode 100755 index 0000000..65dd487 --- /dev/null +++ b/src/main/mastodon_bot/mastodon_api.cljs @@ -0,0 +1,165 @@ +(ns mastodon-bot.mastodon-api + (:require + [clojure.spec.alpha :as s] + [clojure.spec.test.alpha :as st] + [orchestra.core :refer-macros [defn-spec]] + [clojure.string :as string] + [mastodon-bot.infra :as infra] + ["deasync" :as deasync] + ["request" :as request] + ["mastodon-api" :as mastodon])) + +; Todo: think about how namespaced keywords & clj->js can play nicely together +(s/def ::access_token string?) +(s/def ::api_url string?) +(s/def ::account-id string?) +(s/def ::content-filter string?) +(s/def ::keyword-filter string?) +(s/def ::append-screen-name? boolean?) +(s/def ::signature string?) +(s/def ::sensitive? boolean?) +(s/def ::resolve-urls? boolean?) +(s/def ::nitter-urls? boolean?) +(s/def ::visibility string?) +(s/def ::max-post-length (fn [n] (and + (int? n) + (<= n 600) + (< n 0)))) + +(s/def ::content-filters (s/* ::content-filter)) +(s/def ::keyword-filters (s/* ::keyword-filter)) +(s/def ::mastodon-js-config (s/keys :req [::access_token ::a:pi_url])) +(s/def ::mastodon-clj-config (s/keys :req [::account-id ::content-filters ::keyword-filters + ::max-post-length ::signature ::visibility + ::append-screen-name? ::sensitive? ::resolve-urls? + ::nitter-urls?])) +(def mastodon-config? (s/merge ::mastodon-js-config ::mastodon-clj-config)) + +(defn-spec content-filter-regexes ::content-filters + [mastodon-config mastodon-config?] + (mapv re-pattern (::content-filters mastodon-config))) + +(defn-spec keyword-filter-regexes ::keyword-filters + [mastodon-config mastodon-config?] + (mapv re-pattern (::keyword-filters mastodon-config))) + +(defn-spec append-screen-name? ::append-screen-name? + [mastodon-config mastodon-config?] + (boolean (::append-screen-name? mastodon-config))) + +(defn-spec max-post-length ::max-post-length + [mastodon-config mastodon-config?] + (::max-post-length mastodon-config)) + +(defn-spec mastodon-client any? + [mastodon-config mastodon-config?] + (or (some-> mastodon-config clj->js mastodon.) + (infra/exit-with-error "missing Mastodon client configuration!"))) + +(defn-spec blocked-content? boolean? + [mastodon-config mastodon-config? + text string?] + (boolean + (or (some #(re-find % text) (content-filter-regexes mastodon-config)) + (when (not-empty (keyword-filter-regexes mastodon-config)) + (empty? (some #(re-find % text) (keyword-filter-regexes mastodon-config))))))) + +(defn-spec delete-status any? + [mastodon-config mastodon-config? + status-id string?] + (.delete (mastodon-client mastodon-config) (str "statuses/" status-id) #js {})) + +(defn resolve-url [[uri]] + (try + (or + (some-> ((deasync request) + #js {:method "GET" + :uri (if (string/starts-with? uri "https://") uri (str "https://" uri)) + :followRedirect false}) + (.-headers) + (.-location) + (string/replace "?mbid=social_twitter" "")) + uri) + (catch js/Error _ uri))) + +(def shortened-url-pattern #"(https?://)?(?:\S+(?::\S*)?@)?(?:(?!(?:10|127)(?:\.\d{1,3}){3})(?!(?:169\.254|192\.168)(?:\.\d{1,3}){2})(?!172\.(?:1[6-9]|2\d|3[0-1])(?:\.\d{1,3}){2})(?:[1-9]\d?|1\d\d|2[01]\d|22[0-3])(?:\.(?:1?\d{1,2}|2[0-4]\d|25[0-5])){2}(?:\.(?:[1-9]\d?|1\d\d|2[0-4]\d|25[0-4]))|(?:(?:[a-z\u00a1-\uffff0-9]-*)*[a-z\u00a1-\uffff0-9]+)(?:\.(?:[a-z\u00a1-\uffff0-9]-*)*[a-z\u00a1-\uffff0-9]+)*(?:\.(?:[a-z\u00a1-\uffff]{2,}))\.?)(?::\d{2,5})?(?:[/?#]\S*)?") + +(defn-spec resolve-urls string? + [mastodon-config mastodon-config? + text string?] + (cond-> text + (::resolve-urls? mastodon-config) + (string/replace shortened-url-pattern resolve-url) + (::nitter-urls? mastodon-config) + (string/replace #"https://twitter.com" "https://nitter.net"))) + +(defn-spec set-signature string? + [mastodon-config mastodon-config? + text string?] + (if-let [signature (::signature mastodon-config )] + (str text "\n" signature) + text)) + +(defn post-status + ([mastodon-config status-text] + (post-status mastodon-config status-text nil)) + ([mastodon-config status-text media-ids] + (let [{:keys [sensitive? signature visibility]} mastodon-config] + (.post (mastodon-client mastodon-config) "statuses" + (clj->js (merge {:status (resolve-urls mastodon-config + (set-signature mastodon-config status-text))} + (when media-ids {:media_ids media-ids}) + (when sensitive? {:sensitive sensitive?}) + (when visibility {:visibility visibility}))))))) + +(defn-spec post-image any? + [mastodon-config mastodon-config? + image-stream any? + description string? + callback fn?] + (-> (.post (mastodon-client mastodon-config) "media" + #js {:file image-stream :description description}) + (.then #(-> % .-data .-id callback)))) + +(defn post-status-with-images + ([mastodon-config status-text urls] + (post-status-with-images mastodon-config status-text urls [])) + ([mastodon-config status-text [url & urls] ids] + (if url + (-> request + (.get url) + (.on "response" + (fn [image-stream] + (post-image mastodon-config image-stream status-text + #(post-status-with-images status-text urls (conj ids %)))))) + (post-status mastodon-config status-text (not-empty ids))))) + +(defn-spec get-mastodon-timeline any? + [mastodon-config mastodon-config? + callback fn?] + (.then (.get (mastodon-client mastodon-config) + (str "accounts/" (::account-id mastodon-config)"/statuses") #js {}) + #(let [response (-> % .-data infra/js->edn)] + (if-let [error (::error response)] + (infra/exit-with-error error) + (callback response))))) + +(defn-spec perform-replacements any? + [mastodon-config mastodon-config? + post any?] + (assoc post :text (reduce-kv string/replace (:text post) (::replacements mastodon-config))) + ) + +(defn-spec post-items any? + [mastodon-config mastodon-config? + last-post-time any? + items any?] + (doseq [{:keys [text media-links]} + (->> items + (remove #((blocked-content? mastodon-config (:text %)))) + (filter #(> (:created-at %) last-post-time)) + (map #(perform-replacements mastodon-config %)))] + (if media-links + (post-status-with-images mastodon-config text media-links) + (when-not (::media-only? mastodon-config) + (post-status mastodon-config text))))) diff --git a/mastodon_bot/core_test.cljs b/src/test/mastodon_bot/core_test.cljs similarity index 100% rename from mastodon_bot/core_test.cljs rename to src/test/mastodon_bot/core_test.cljs From 0ae68af441fbf7675c832fb79316c6a3bc7c8caf Mon Sep 17 00:00:00 2001 From: jem Date: Wed, 13 May 2020 17:41:27 +0200 Subject: [PATCH 03/12] repl now works --- .gitignore | 4 +- README.md | 65 +++++++++++++------------ shadow-cljs.edn | 10 ++-- src/main/mastodon_bot/core.cljs | 9 ++-- src/main/mastodon_bot/mastodon_api.cljs | 2 +- 5 files changed, 48 insertions(+), 42 deletions(-) diff --git a/.gitignore b/.gitignore index e3d8124..a87dbc4 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,5 @@ -.shadow-cljs config.edn package-lock.json +/.shadow-cljs /node_modules -target/mastodon-bot.js +/target diff --git a/README.md b/README.md index c5a1d9f..b36df95 100644 --- a/README.md +++ b/README.md @@ -27,6 +27,7 @@ If you get a [permission failure](https://github.com/anmonteiro/lumo/issues/206) with later timestamps to avoid duplicate posts. On the first run the timestamp will default to current time. ```clojure +#:mastodon-bot.core {;; add Twitter config to mirror Twitter accounts :twitter {:access-keys {:consumer_key "XXXX" @@ -51,37 +52,39 @@ with later timestamps to avoid duplicate posts. On the first run the timestamp w ;; add RSS config to follow feeds :rss {"Hacker News" "https://hnrss.org/newest" "r/Clojure" "https://www.reddit.com/r/clojure/.rss"} - :mastodon {:access_token "XXXX" - ;; account number you see when you log in and go to your profile - ;; e.g: https://mastodon.social/web/accounts/294795 - :account-id "XXXX" - :api_url "https://botsin.space/api/v1/" - ;; optional boolean to mark content as sensitive - :sensitive? true - ;; optional boolean defaults to false - ;; only sources containing media will be posted when set to true - :media-only? true - ;; optional visibility flag: direct, private, unlisted, public - ;; defaults to public - :visibility "unlisted" - ;; optional limit for the post length - :max-post-length 300 - ;; optional flag specifying wether the name of the account - ;; will be appended in the post, defaults to false - :append-screen-name? false - ;; optional signature for posts - :signature "#newsbot" - ;; optionally try to resolve URLs in posts to skip URL shorteners - ;; defaults to false - :resolve-urls? true - ;; optional content filter regexes - ;; any posts matching the regexes will be filtered out - :content-filters [".*bannedsite.*"] - ;; optional keyword filter regexes - ;; any posts not matching the regexes will be filtered out - :keyword-filters [".*clojure.*"] - ;; Replace Twitter links by Nitter - :nitter-urls? false}} + :mastodon + #:mastodon-bot.mastodon-api +{:access_token "XXXX" + ;; account number you see when you log in and go to your profile + ;; e.g: https://mastodon.social/web/accounts/294795 + :account-id "XXXX" + :api_url "https://botsin.space/api/v1/" + ;; optional boolean to mark content as sensitive + :sensitive? true + ;; optional boolean defaults to false + ;; only sources containing media will be posted when set to true + :media-only? true + ;; optional visibility flag: direct, private, unlisted, public + ;; defaults to public + :visibility "unlisted" + ;; optional limit for the post length + :max-post-length 300 + ;; optional flag specifying wether the name of the account + ;; will be appended in the post, defaults to false + :append-screen-name? false + ;; optional signature for posts + :signature "#newsbot" + ;; optionally try to resolve URLs in posts to skip URL shorteners + ;; defaults to false + :resolve-urls? true + ;; optional content filter regexes + ;; any posts matching the regexes will be filtered out + :content-filters [".*bannedsite.*"] + ;; optional keyword filter regexes + ;; any posts not matching the regexes will be filtered out + :keyword-filters [".*clojure.*"] + ;; Replace Twitter links by Nitter + :nitter-urls? false}} ``` * the bot looks for `config.edn` at its relative path by default, an alternative location can be specified either using the `MASTODON_BOT_CONFIG` environment variable or passing the path to config as an argument diff --git a/shadow-cljs.edn b/shadow-cljs.edn index 49ba2c2..337ec72 100644 --- a/shadow-cljs.edn +++ b/shadow-cljs.edn @@ -1,11 +1,11 @@ {:source-paths ["src/main" "src/test"] :dependencies [[orchestra "2018.12.06-2"]] - :builds {:dev {:target :node-script - :repl-init-ns mastodon-bot.core - :output-to "target/mastodon-bot.js" - :main mastodon-bot.core/dummy - :repl-pprint true} + :builds {:dev {:target :node-library + :output-to "target/lib-mastodon-bot.js" + :exports {:infra mastodon-bot.infra/js->edn} + :repl-pprint true + } :app {:target :node-script :output-to "target/mastodon-bot.js" :main mastodon-bot.core/main diff --git a/src/main/mastodon_bot/core.cljs b/src/main/mastodon_bot/core.cljs index fedae9b..4e3638e 100755 --- a/src/main/mastodon_bot/core.cljs +++ b/src/main/mastodon_bot/core.cljs @@ -15,9 +15,12 @@ [mastodon-bot.infra :as infra] [mastodon-bot.mastodon-api :as masto])) -(defn dummy []) - (s/def ::mastodon-config masto/mastodon-config?) +(s/def ::twitter map?) +(s/def ::tumblr map?) +(s/def ::rss map?) +(def config? (s/keys :req [::mastodon-config] + :opt [::twitter ::tumblr ::rss])) ;this has to stay on top - only ns-keywords can be uses in spec (defn-spec mastodon-config ::mastodon-config @@ -135,7 +138,7 @@ (fn [timeline] (let [last-post-time (-> timeline first :created_at (js/Date.))] ;;post from Twitter - (when-let [twitter-config (:twitter config)] + (when-let [twitter-config (::twitter config)] (let [{:keys [access-keys accounts include-replies? include-rts?]} twitter-config client (twitter-client access-keys)] (doseq [account accounts] diff --git a/src/main/mastodon_bot/mastodon_api.cljs b/src/main/mastodon_bot/mastodon_api.cljs index 65dd487..16350cd 100755 --- a/src/main/mastodon_bot/mastodon_api.cljs +++ b/src/main/mastodon_bot/mastodon_api.cljs @@ -28,7 +28,7 @@ (s/def ::content-filters (s/* ::content-filter)) (s/def ::keyword-filters (s/* ::keyword-filter)) -(s/def ::mastodon-js-config (s/keys :req [::access_token ::a:pi_url])) +(s/def ::mastodon-js-config (s/keys :req [::access_token ::api_url])) (s/def ::mastodon-clj-config (s/keys :req [::account-id ::content-filters ::keyword-filters ::max-post-length ::signature ::visibility ::append-screen-name? ::sensitive? ::resolve-urls? From fed673f07c96f96d30b46056d588ae3b5e1efea8 Mon Sep 17 00:00:00 2001 From: jem Date: Wed, 13 May 2020 17:49:37 +0200 Subject: [PATCH 04/12] spec now is valid --- src/main/mastodon_bot/core.cljs | 5 ++++- src/main/mastodon_bot/mastodon_api.cljs | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/src/main/mastodon_bot/core.cljs b/src/main/mastodon_bot/core.cljs index 4e3638e..caeb380 100755 --- a/src/main/mastodon_bot/core.cljs +++ b/src/main/mastodon_bot/core.cljs @@ -19,12 +19,14 @@ (s/def ::twitter map?) (s/def ::tumblr map?) (s/def ::rss map?) + (def config? (s/keys :req [::mastodon-config] :opt [::twitter ::tumblr ::rss])) +(s/def ::config config?) ;this has to stay on top - only ns-keywords can be uses in spec (defn-spec mastodon-config ::mastodon-config - [config any?] + [config ::config] (::mastodon-config config)) (defn find-config [] @@ -161,3 +163,4 @@ (parse-feed last-post-time parser feed)))))))) (set! *main-cli-fn* -main) +(st/instrument 'mastodon-config) diff --git a/src/main/mastodon_bot/mastodon_api.cljs b/src/main/mastodon_bot/mastodon_api.cljs index 16350cd..90f314b 100755 --- a/src/main/mastodon_bot/mastodon_api.cljs +++ b/src/main/mastodon_bot/mastodon_api.cljs @@ -24,7 +24,7 @@ (s/def ::max-post-length (fn [n] (and (int? n) (<= n 600) - (< n 0)))) + (> n 0)))) (s/def ::content-filters (s/* ::content-filter)) (s/def ::keyword-filters (s/* ::keyword-filter)) From 8f79814b3815591fa981602abacb65d5a4552652 Mon Sep 17 00:00:00 2001 From: jem Date: Wed, 13 May 2020 18:27:01 +0200 Subject: [PATCH 05/12] keep symmetry only predicate for top level elem --- src/main/mastodon_bot/core.cljs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/main/mastodon_bot/core.cljs b/src/main/mastodon_bot/core.cljs index 0eb8535..855ef3d 100755 --- a/src/main/mastodon_bot/core.cljs +++ b/src/main/mastodon_bot/core.cljs @@ -22,11 +22,10 @@ (def config? (s/keys :req [::mastodon-config] :opt [::twitter ::tumblr ::rss])) -(s/def ::config config?) ;this has to stay on top - only ns-keywords can be uses in spec (defn-spec mastodon-config ::mastodon-config - [config ::config] + [config config?] (::mastodon-config config)) (defn find-config [] From f2a6f15c42619eabcf79f46ec2442dd3873bd85b Mon Sep 17 00:00:00 2001 From: jem Date: Wed, 13 May 2020 19:29:33 +0200 Subject: [PATCH 06/12] mastodon limits posts to 500 --- src/main/mastodon_bot/mastodon_api.cljs | 26 +++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/src/main/mastodon_bot/mastodon_api.cljs b/src/main/mastodon_bot/mastodon_api.cljs index 93d3dd3..8217d28 100755 --- a/src/main/mastodon_bot/mastodon_api.cljs +++ b/src/main/mastodon_bot/mastodon_api.cljs @@ -24,7 +24,7 @@ (s/def ::replacements string?) (s/def ::max-post-length (fn [n] (and (int? n) - (<= n 600) + (<= n 500) (> n 0)))) (s/def ::content-filters (s/* ::content-filter)) @@ -108,17 +108,23 @@ (defn post-status ([mastodon-config status-text] - (post-status mastodon-config status-text nil)) + (post-status mastodon-config status-text nil print)) ([mastodon-config status-text media-ids] + (post-status mastodon-config status-text media-ids print)) + ([mastodon-config status-text media-ids callback] (let [{:keys [sensitive? signature visibility]} mastodon-config] - (.post (mastodon-client mastodon-config) "statuses" - (clj->js (merge {:status (-> status-text - (partial resolve-urls mastodon-config) - (partial perform-replacements mastodon-config) - (partial set-signature mastodon-config))} - (when media-ids {:media_ids media-ids}) - (when sensitive? {:sensitive sensitive?}) - (when visibility {:visibility visibility}))))))) + (println sensitive?) + (println signature) + (println visibility) + (-> (.post (mastodon-client mastodon-config) "statuses" + (clj->js (merge {:status (-> status-text + (partial resolve-urls mastodon-config) + (partial perform-replacements mastodon-config) + (partial set-signature mastodon-config))} + (when media-ids {:media_ids media-ids}) + (when sensitive? {:sensitive sensitive?}) + (when visibility {:visibility visibility})))) + (.then #(-> % callback)))))) (defn-spec post-image any? [mastodon-config mastodon-config? From 418ff67282c3a4122a9e59cd0328ce990c8a9fe2 Mon Sep 17 00:00:00 2001 From: jem Date: Wed, 13 May 2020 19:32:23 +0200 Subject: [PATCH 07/12] learned how to destruct namespaced keywords --- src/main/mastodon_bot/mastodon_api.cljs | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/src/main/mastodon_bot/mastodon_api.cljs b/src/main/mastodon_bot/mastodon_api.cljs index 8217d28..ff46049 100755 --- a/src/main/mastodon_bot/mastodon_api.cljs +++ b/src/main/mastodon_bot/mastodon_api.cljs @@ -112,10 +112,7 @@ ([mastodon-config status-text media-ids] (post-status mastodon-config status-text media-ids print)) ([mastodon-config status-text media-ids callback] - (let [{:keys [sensitive? signature visibility]} mastodon-config] - (println sensitive?) - (println signature) - (println visibility) + (let [{:mastodon-bot.mastodon-api/keys [sensitive? signature visibility]} mastodon-config] (-> (.post (mastodon-client mastodon-config) "statuses" (clj->js (merge {:status (-> status-text (partial resolve-urls mastodon-config) @@ -162,7 +159,7 @@ [mastodon-config mastodon-config? last-post-time any? items any?] - (doseq [{:keys [text media-links]} + (doseq [{:mastodon-bot.mastodon-api/keys [text media-links]} (->> items (remove #((blocked-content? mastodon-config (:text %)))) (filter #(> (:created-at %) last-post-time)))] From 2187df1058a037abd8a7ccf4ae5cc607e3a3cd55 Mon Sep 17 00:00:00 2001 From: jem Date: Wed, 13 May 2020 19:46:51 +0200 Subject: [PATCH 08/12] learn usage of thread-last --- src/main/mastodon_bot/mastodon_api.cljs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/main/mastodon_bot/mastodon_api.cljs b/src/main/mastodon_bot/mastodon_api.cljs index ff46049..f56273f 100755 --- a/src/main/mastodon_bot/mastodon_api.cljs +++ b/src/main/mastodon_bot/mastodon_api.cljs @@ -114,10 +114,10 @@ ([mastodon-config status-text media-ids callback] (let [{:mastodon-bot.mastodon-api/keys [sensitive? signature visibility]} mastodon-config] (-> (.post (mastodon-client mastodon-config) "statuses" - (clj->js (merge {:status (-> status-text - (partial resolve-urls mastodon-config) - (partial perform-replacements mastodon-config) - (partial set-signature mastodon-config))} + (clj->js (merge {:status (->> status-text + (resolve-urls mastodon-config) + (perform-replacements mastodon-config) + (set-signature mastodon-config))} (when media-ids {:media_ids media-ids}) (when sensitive? {:sensitive sensitive?}) (when visibility {:visibility visibility})))) From af85887931023fd9943ce835187e561f4e2e3003 Mon Sep 17 00:00:00 2001 From: jem Date: Thu, 14 May 2020 08:53:49 +0200 Subject: [PATCH 09/12] move more functions to infra for better testability --- src/main/mastodon_bot/infra.cljs | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/src/main/mastodon_bot/infra.cljs b/src/main/mastodon_bot/infra.cljs index c8ec4b6..a2f1bcf 100755 --- a/src/main/mastodon_bot/infra.cljs +++ b/src/main/mastodon_bot/infra.cljs @@ -1,4 +1,7 @@ -(ns mastodon-bot.infra) +(ns mastodon-bot.infra + (:require + [cljs.reader :as edn] + ["fs" :as fs])) (defn js->edn [data] (js->clj data :keywordize-keys true)) @@ -6,3 +9,14 @@ (defn exit-with-error [error] (js/console.error error) (js/process.exit 1)) + +(defn find-config [] + (let [config (or (first *command-line-args*) + (-> js/process .-env .-MASTODON_BOT_CONFIG) + "config.edn")] + (if (fs/existsSync config) + config + (exit-with-error (str "failed to read config: " config))))) + +(defn load-config [] + (-> (find-config) (fs/readFileSync #js {:encoding "UTF-8"}) edn/read-string)) \ No newline at end of file From 1ebb0e71f5ddcc7e599e5a4226ef392fcc5ba8fd Mon Sep 17 00:00:00 2001 From: jem Date: Thu, 14 May 2020 08:56:50 +0200 Subject: [PATCH 10/12] enable unqalified keywords again --- README.md | 65 ++++++++++++------------- src/main/mastodon_bot/core.cljs | 21 ++------ src/main/mastodon_bot/mastodon_api.cljs | 40 +++++++-------- 3 files changed, 57 insertions(+), 69 deletions(-) diff --git a/README.md b/README.md index b36df95..10dea30 100644 --- a/README.md +++ b/README.md @@ -27,7 +27,6 @@ If you get a [permission failure](https://github.com/anmonteiro/lumo/issues/206) with later timestamps to avoid duplicate posts. On the first run the timestamp will default to current time. ```clojure -#:mastodon-bot.core {;; add Twitter config to mirror Twitter accounts :twitter {:access-keys {:consumer_key "XXXX" @@ -52,39 +51,37 @@ with later timestamps to avoid duplicate posts. On the first run the timestamp w ;; add RSS config to follow feeds :rss {"Hacker News" "https://hnrss.org/newest" "r/Clojure" "https://www.reddit.com/r/clojure/.rss"} - :mastodon - #:mastodon-bot.mastodon-api -{:access_token "XXXX" - ;; account number you see when you log in and go to your profile - ;; e.g: https://mastodon.social/web/accounts/294795 - :account-id "XXXX" - :api_url "https://botsin.space/api/v1/" - ;; optional boolean to mark content as sensitive - :sensitive? true - ;; optional boolean defaults to false - ;; only sources containing media will be posted when set to true - :media-only? true - ;; optional visibility flag: direct, private, unlisted, public - ;; defaults to public - :visibility "unlisted" - ;; optional limit for the post length - :max-post-length 300 - ;; optional flag specifying wether the name of the account - ;; will be appended in the post, defaults to false - :append-screen-name? false - ;; optional signature for posts - :signature "#newsbot" - ;; optionally try to resolve URLs in posts to skip URL shorteners - ;; defaults to false - :resolve-urls? true - ;; optional content filter regexes - ;; any posts matching the regexes will be filtered out - :content-filters [".*bannedsite.*"] - ;; optional keyword filter regexes - ;; any posts not matching the regexes will be filtered out - :keyword-filters [".*clojure.*"] - ;; Replace Twitter links by Nitter - :nitter-urls? false}} + :mastodon {:access_token "XXXX" + ;; account number you see when you log in and go to your profile + ;; e.g: https://mastodon.social/web/accounts/294795 + :account-id "XXXX" + :api_url "https://botsin.space/api/v1/" + ;; optional boolean to mark content as sensitive + :sensitive? true + ;; optional boolean defaults to false + ;; only sources containing media will be posted when set to true + :media-only? true + ;; optional visibility flag: direct, private, unlisted, public + ;; defaults to public + :visibility "unlisted" + ;; optional limit for the post length + :max-post-length 300 + ;; optional flag specifying wether the name of the account + ;; will be appended in the post, defaults to false + :append-screen-name? false + ;; optional signature for posts + :signature "#newsbot" + ;; optionally try to resolve URLs in posts to skip URL shorteners + ;; defaults to false + :resolve-urls? true + ;; optional content filter regexes + ;; any posts matching the regexes will be filtered out + :content-filters [".*bannedsite.*"] + ;; optional keyword filter regexes + ;; any posts not matching the regexes will be filtered out + :keyword-filters [".*clojure.*"] + ;; Replace Twitter links by Nitter + :nitter-urls? false}} ``` * the bot looks for `config.edn` at its relative path by default, an alternative location can be specified either using the `MASTODON_BOT_CONFIG` environment variable or passing the path to config as an argument diff --git a/src/main/mastodon_bot/core.cljs b/src/main/mastodon_bot/core.cljs index 855ef3d..3a78cbc 100755 --- a/src/main/mastodon_bot/core.cljs +++ b/src/main/mastodon_bot/core.cljs @@ -6,9 +6,7 @@ [clojure.spec.test.alpha :as st] [orchestra.core :refer-macros [defn-spec]] [cljs.core :refer [*command-line-args*]] - [cljs.reader :as edn] [clojure.string :as string] - ["fs" :as fs] ["rss-parser" :as rss] ["tumblr" :as tumblr] ["twitter" :as twitter] @@ -20,23 +18,14 @@ (s/def ::tumblr map?) (s/def ::rss map?) -(def config? (s/keys :req [::mastodon-config] - :opt [::twitter ::tumblr ::rss])) +(def config? (s/keys :req-un [::mastodon-config] + :opt-un [::twitter ::tumblr ::rss])) -;this has to stay on top - only ns-keywords can be uses in spec (defn-spec mastodon-config ::mastodon-config [config config?] - (::mastodon-config config)) + (:mastodon-config config)) -(defn find-config [] - (let [config (or (first *command-line-args*) - (-> js/process .-env .-MASTODON_BOT_CONFIG) - "config.edn")] - (if (fs/existsSync config) - config - (infra/exit-with-error (str "failed to read config: " config))))) - -(def config (-> (find-config) (fs/readFileSync #js {:encoding "UTF-8"}) edn/read-string)) +(def config (infra/load-config)) (defn trim-text [text] (let [max-post-length (masto/max-post-length (mastodon-config config))] @@ -139,7 +128,7 @@ (fn [timeline] (let [last-post-time (-> timeline first :created_at (js/Date.))] ;;post from Twitter - (when-let [twitter-config (::twitter config)] + (when-let [twitter-config (:twitter config)] (let [{:keys [access-keys accounts include-replies? include-rts?]} twitter-config client (twitter-client access-keys)] (doseq [account accounts] diff --git a/src/main/mastodon_bot/mastodon_api.cljs b/src/main/mastodon_bot/mastodon_api.cljs index f56273f..9a8b839 100755 --- a/src/main/mastodon_bot/mastodon_api.cljs +++ b/src/main/mastodon_bot/mastodon_api.cljs @@ -29,33 +29,33 @@ (s/def ::content-filters (s/* ::content-filter)) (s/def ::keyword-filters (s/* ::keyword-filter)) -(s/def ::mastodon-js-config (s/keys :req [::access_token ::api_url])) -(s/def ::mastodon-clj-config (s/keys :req [::account-id ::content-filters ::keyword-filters - ::max-post-length ::signature ::visibility - ::append-screen-name? ::sensitive? ::resolve-urls? - ::nitter-urls? ::replacements])) +(s/def ::mastodon-js-config (s/keys :req-un [::access_token ::api_url])) +(s/def ::mastodon-clj-config (s/keys :req-un [::account-id ::content-filters ::keyword-filters + ::max-post-length ::signature ::visibility + ::append-screen-name? ::sensitive? ::resolve-urls? + ::nitter-urls? ::replacements])) (def mastodon-config? (s/merge ::mastodon-js-config ::mastodon-clj-config)) (defn-spec content-filter-regexes ::content-filters [mastodon-config mastodon-config?] - (mapv re-pattern (::content-filters mastodon-config))) + (mapv re-pattern (:content-filters mastodon-config))) (defn-spec keyword-filter-regexes ::keyword-filters [mastodon-config mastodon-config?] - (mapv re-pattern (::keyword-filters mastodon-config))) + (mapv re-pattern (:keyword-filters mastodon-config))) (defn-spec append-screen-name? ::append-screen-name? [mastodon-config mastodon-config?] - (boolean (::append-screen-name? mastodon-config))) + (boolean (:append-screen-name? mastodon-config))) (defn-spec max-post-length ::max-post-length [mastodon-config mastodon-config?] - (::max-post-length mastodon-config)) + (:max-post-length mastodon-config)) (defn-spec perform-replacements string? [mastodon-config mastodon-config? text string?] - (reduce-kv string/replace text (::replacements mastodon-config))) + (reduce-kv string/replace text (:replacements mastodon-config))) (defn-spec mastodon-client any? [mastodon-config mastodon-config?] @@ -94,15 +94,15 @@ [mastodon-config mastodon-config? text string?] (cond-> text - (::resolve-urls? mastodon-config) + (:resolve-urls? mastodon-config) (string/replace shortened-url-pattern resolve-url) - (::nitter-urls? mastodon-config) + (:nitter-urls? mastodon-config) (string/replace #"https://twitter.com" "https://nitter.net"))) (defn-spec set-signature string? [mastodon-config mastodon-config? text string?] - (if-let [signature (::signature mastodon-config )] + (if-let [signature (:signature mastodon-config )] (str text "\n" signature) text)) @@ -112,7 +112,7 @@ ([mastodon-config status-text media-ids] (post-status mastodon-config status-text media-ids print)) ([mastodon-config status-text media-ids callback] - (let [{:mastodon-bot.mastodon-api/keys [sensitive? signature visibility]} mastodon-config] + (let [{:keys [sensitive? signature visibility]} mastodon-config] (-> (.post (mastodon-client mastodon-config) "statuses" (clj->js (merge {:status (->> status-text (resolve-urls mastodon-config) @@ -134,22 +134,24 @@ (defn post-status-with-images ([mastodon-config status-text urls] - (post-status-with-images mastodon-config status-text urls [])) - ([mastodon-config status-text [url & urls] ids] + (post-status-with-images mastodon-config status-text urls [] print)) + ([mastodon-config status-text urls ids] + (post-status-with-images mastodon-config status-text urls ids print)) + ([mastodon-config status-text [url & urls] ids callback] (if url (-> request (.get url) (.on "response" (fn [image-stream] (post-image mastodon-config image-stream status-text - #(post-status-with-images status-text urls (conj ids %)))))) - (post-status mastodon-config status-text (not-empty ids))))) + #(post-status-with-images status-text urls (conj ids %) callback))))) + (post-status mastodon-config status-text (not-empty ids) callback)))) (defn-spec get-mastodon-timeline any? [mastodon-config mastodon-config? callback fn?] (.then (.get (mastodon-client mastodon-config) - (str "accounts/" (::account-id mastodon-config)"/statuses") #js {}) + (str "accounts/" (:account-id mastodon-config)"/statuses") #js {}) #(let [response (-> % .-data infra/js->edn)] (if-let [error (::error response)] (infra/exit-with-error error) From 48d0c22b5ad419e4e797a171d1ba6a58c78455b8 Mon Sep 17 00:00:00 2001 From: jem Date: Fri, 15 May 2020 16:42:57 +0200 Subject: [PATCH 11/12] test "post with image" passed --- src/main/mastodon_bot/mastodon_api.cljs | 4 ++-- src/test/mastodon_bot/mytest.cljs | 28 +++++++++++++++++++++++++ 2 files changed, 30 insertions(+), 2 deletions(-) create mode 100644 src/test/mastodon_bot/mytest.cljs diff --git a/src/main/mastodon_bot/mastodon_api.cljs b/src/main/mastodon_bot/mastodon_api.cljs index 9a8b839..0d05c65 100755 --- a/src/main/mastodon_bot/mastodon_api.cljs +++ b/src/main/mastodon_bot/mastodon_api.cljs @@ -144,7 +144,7 @@ (.on "response" (fn [image-stream] (post-image mastodon-config image-stream status-text - #(post-status-with-images status-text urls (conj ids %) callback))))) + #(post-status-with-images mastodon-config status-text urls (conj ids %) callback))))) (post-status mastodon-config status-text (not-empty ids) callback)))) (defn-spec get-mastodon-timeline any? @@ -161,7 +161,7 @@ [mastodon-config mastodon-config? last-post-time any? items any?] - (doseq [{:mastodon-bot.mastodon-api/keys [text media-links]} + (doseq [{:keys [text media-links]} (->> items (remove #((blocked-content? mastodon-config (:text %)))) (filter #(> (:created-at %) last-post-time)))] diff --git a/src/test/mastodon_bot/mytest.cljs b/src/test/mastodon_bot/mytest.cljs new file mode 100644 index 0000000..5f40631 --- /dev/null +++ b/src/test/mastodon_bot/mytest.cljs @@ -0,0 +1,28 @@ +(ns mastodon-bot.mytest + (:require + [mastodon-bot.infra :as infra] + [clojure.pprint :refer [pprint]] + [mastodon-bot.mastodon-api :as masto])) + +(defn myconfig [] + (:mastodon-config (infra/load-config))) + +(defn test-timeline [] + (masto/get-mastodon-timeline + (myconfig) + pprint)) + +(defn test-status [] + (masto/post-status + (myconfig) + "test" + nil + pprint)) + +(defn test-status-image [] + (masto/post-status-with-images + (myconfig) + "test2" + ["https://pbs.twimg.com/media/ER3qX2RW4AEhQfW.jpg"] + [] + pprint)) From 7c73d9aa64b641d34bbba557f8ba4931f83a24a3 Mon Sep 17 00:00:00 2001 From: jem Date: Fri, 15 May 2020 16:44:07 +0200 Subject: [PATCH 12/12] remove useless parts --- src/test/mastodon_bot/mytest.cljs | 28 ---------------------------- 1 file changed, 28 deletions(-) delete mode 100644 src/test/mastodon_bot/mytest.cljs diff --git a/src/test/mastodon_bot/mytest.cljs b/src/test/mastodon_bot/mytest.cljs deleted file mode 100644 index 5f40631..0000000 --- a/src/test/mastodon_bot/mytest.cljs +++ /dev/null @@ -1,28 +0,0 @@ -(ns mastodon-bot.mytest - (:require - [mastodon-bot.infra :as infra] - [clojure.pprint :refer [pprint]] - [mastodon-bot.mastodon-api :as masto])) - -(defn myconfig [] - (:mastodon-config (infra/load-config))) - -(defn test-timeline [] - (masto/get-mastodon-timeline - (myconfig) - pprint)) - -(defn test-status [] - (masto/post-status - (myconfig) - "test" - nil - pprint)) - -(defn test-status-image [] - (masto/post-status-with-images - (myconfig) - "test2" - ["https://pbs.twimg.com/media/ER3qX2RW4AEhQfW.jpg"] - [] - pprint))