From 9139ed7539ae5790c52315c961d0be94d148aa42 Mon Sep 17 00:00:00 2001 From: Michael Jerger Date: Thu, 11 Mar 2021 16:54:54 +0000 Subject: [PATCH] Provide schema as jar --- .gitignore | 15 ++- project.clj | 16 +++ shadow-cljs.edn | 6 +- src/main/cljc/mastodon_bot/core_domain.cljc | 22 +++ .../cljc/mastodon_bot/mastodon_domain.cljc | 29 ++++ src/main/cljc/mastodon_bot/rss_domain.cljc | 16 +++ .../cljc/mastodon_bot/transform_domain.cljc | 42 ++++++ src/main/cljc/mastodon_bot/tumblr_domain.cljc | 16 +++ .../cljc/mastodon_bot/twitter_domain.cljs | 18 +++ src/main/{ => cljs}/mastodon_bot/core.cljs | 44 ++---- src/main/{ => cljs}/mastodon_bot/infra.cljs | 0 .../{ => cljs}/mastodon_bot/mastodon_api.cljs | 52 ++------ src/main/cljs/mastodon_bot/rss_api.cljs | 24 ++++ .../{ => cljs}/mastodon_bot/transform.cljs | 125 +++++++----------- .../{ => cljs}/mastodon_bot/tumblr_api.cljs | 17 +-- .../{ => cljs}/mastodon_bot/twitter_api.cljs | 27 +--- src/main/mastodon_bot/rss_api.cljs | 40 ------ .../cljc/mastodon_bot/rss_domain_test.cljc | 11 ++ .../mastodon_bot/transform_domain_test.cljs} | 19 +-- .../mastodon_bot/mastodon_api_test.cljs | 1 - .../{ => cljs}/mastodon_bot/rss_api_test.cljs | 6 - .../mastodon_bot/transform_rss_test.cljs | 0 .../cljs/mastodon_bot/transform_test.cljs | 19 +++ .../mastodon_bot/twitter_api_test.cljs | 0 24 files changed, 310 insertions(+), 255 deletions(-) create mode 100644 project.clj create mode 100755 src/main/cljc/mastodon_bot/core_domain.cljc create mode 100644 src/main/cljc/mastodon_bot/mastodon_domain.cljc create mode 100644 src/main/cljc/mastodon_bot/rss_domain.cljc create mode 100644 src/main/cljc/mastodon_bot/transform_domain.cljc create mode 100755 src/main/cljc/mastodon_bot/tumblr_domain.cljc create mode 100755 src/main/cljc/mastodon_bot/twitter_domain.cljs rename src/main/{ => cljs}/mastodon_bot/core.cljs (72%) rename src/main/{ => cljs}/mastodon_bot/infra.cljs (100%) rename src/main/{ => cljs}/mastodon_bot/mastodon_api.cljs (75%) create mode 100755 src/main/cljs/mastodon_bot/rss_api.cljs rename src/main/{ => cljs}/mastodon_bot/transform.cljs (55%) rename src/main/{ => cljs}/mastodon_bot/tumblr_api.cljs (65%) rename src/main/{ => cljs}/mastodon_bot/twitter_api.cljs (70%) delete mode 100755 src/main/mastodon_bot/rss_api.cljs create mode 100755 src/test/cljc/mastodon_bot/rss_domain_test.cljc rename src/test/{mastodon_bot/transform_test.cljs => cljc/mastodon_bot/transform_domain_test.cljs} (56%) rename src/test/{ => cljs}/mastodon_bot/mastodon_api_test.cljs (98%) rename src/test/{ => cljs}/mastodon_bot/rss_api_test.cljs (93%) rename src/test/{ => cljs}/mastodon_bot/transform_rss_test.cljs (100%) create mode 100755 src/test/cljs/mastodon_bot/transform_test.cljs rename src/test/{ => cljs}/mastodon_bot/twitter_api_test.cljs (100%) diff --git a/.gitignore b/.gitignore index ceeedd3..106a0e3 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,14 @@ -config.edn -config*.edn +# cljs ignores package-lock.json -/.shadow-cljs /node_modules +/.shadow-cljs + +# mastodon-bot ignores +config*.edn + +# lein ignores /target -/mastodon-bot.js +pom.xml + +# ide ignores +.calva/ diff --git a/project.clj b/project.clj new file mode 100644 index 0000000..250ed03 --- /dev/null +++ b/project.clj @@ -0,0 +1,16 @@ +(defproject dda/mastodon-bot "1.10.5-SNAPSHOT" + :description "Bot to publish twitter, tumblr or rss posts to an mastodon account." + :url "https://github.com/yogthos/mastodon-bot" + :author "Dmitri Sotnikov" + :license {:name "MIT"} + :dependencies [] + :source-paths ["src/main/cljc" + "src/main/clj"] + :resource-paths ["src/main/resources"] + :repositories [["snapshots" :clojars] + ["releases" :clojars]] + :deploy-repositories [["snapshots" :clojars] + ["releases" :clojars]] + :profiles {:test {:test-paths ["src/test/cljc"] + :resource-paths ["src/test/resources"] + :dependencies []}}) diff --git a/shadow-cljs.edn b/shadow-cljs.edn index 62e525d..ef8e219 100644 --- a/shadow-cljs.edn +++ b/shadow-cljs.edn @@ -1,5 +1,7 @@ -{:source-paths ["src/main" - "src/test"] +{:source-paths ["src/main/cljc" + "src/main/cljs" + "src/test/cljc" + "src/test/cljs"] :dependencies [[orchestra "2019.02.06-1"] [expound "0.8.4"]] :builds {:test {:target :node-test diff --git a/src/main/cljc/mastodon_bot/core_domain.cljc b/src/main/cljc/mastodon_bot/core_domain.cljc new file mode 100755 index 0000000..5927870 --- /dev/null +++ b/src/main/cljc/mastodon_bot/core_domain.cljc @@ -0,0 +1,22 @@ +(ns mastodon-bot.core-domain + (:require + [clojure.spec.alpha :as s] + [clojure.string :as cs] + [mastodon-bot.transform-domain :as trd] + [mastodon-bot.mastodon-domain :as md] + [mastodon-bot.twitter-domain :as twd] + [mastodon-bot.tumblr-domain :as td])) + +(s/def ::mastodon md/mastodon-auth?) +(s/def ::twitter twd/twitter-auth?) +(s/def ::tumblr td/tumblr-auth?) +(s/def ::transform trd/transformations?) +(s/def ::auth (s/keys :opt-un [::mastodon ::twitter ::tumblr])) +(def config? + (s/keys :req-un [::auth ::transform])) + +(s/def ::options (s/* #{"-h"})) +(s/def ::config-location (s/? (s/and string? + #(not (cs/starts-with? % "-"))))) +(s/def ::args (s/cat :options ::options + :config-location ::config-location)) diff --git a/src/main/cljc/mastodon_bot/mastodon_domain.cljc b/src/main/cljc/mastodon_bot/mastodon_domain.cljc new file mode 100644 index 0000000..7626129 --- /dev/null +++ b/src/main/cljc/mastodon_bot/mastodon_domain.cljc @@ -0,0 +1,29 @@ +(ns mastodon-bot.mastodon-domain + (:require + [clojure.spec.alpha :as s])) + +(s/def ::access_token string?) +(s/def ::api_url string?) +(s/def ::account-id string?) +(s/def ::append-screen-name? boolean?) +(s/def ::signature string?) +(s/def ::sensitive? boolean?) +(s/def ::media-only? boolean?) +(s/def ::visibility #{"direct" "private" "unlisted" "public"}) +(s/def ::max-post-length (fn [n] (and + (int? n) + (<= n 500) + (> n 0)))) +(def mastodon-auth? (s/keys :req-un [::account-id ::access_token ::api_url])) +(def mastodon-target? (s/keys :opt-un [::max-post-length + ::signature + ::visibility + ::append-screen-name? + ::sensitive? + ::media-only?])) +(s/def ::created-at any?) +(s/def ::text string?) +(s/def ::media-links string?) + +(def mastodon-output? (s/keys :req-un [::created-at ::text] + :opt-un [::media-links])) diff --git a/src/main/cljc/mastodon_bot/rss_domain.cljc b/src/main/cljc/mastodon_bot/rss_domain.cljc new file mode 100644 index 0000000..024b5e8 --- /dev/null +++ b/src/main/cljc/mastodon_bot/rss_domain.cljc @@ -0,0 +1,16 @@ +(ns mastodon-bot.rss-domain + (:require + [clojure.spec.alpha :as s])) + +(s/def ::feed (s/cat :name string? :url string?)) +(s/def ::feeds (s/coll-of ::feed)) +(def rss-source? (s/keys :req-un [::feeds])) + +(s/def ::title string?) +(s/def ::content string?) +(s/def ::link string?) +(s/def ::author string?) +(s/def ::isoDate string?) +(s/def ::pubDate string?) +(s/def ::feed-item (s/keys :req-un [::title ::content ::link] + :opt-un [::author ::isoDate ::pubDate])) diff --git a/src/main/cljc/mastodon_bot/transform_domain.cljc b/src/main/cljc/mastodon_bot/transform_domain.cljc new file mode 100644 index 0000000..99a9ca9 --- /dev/null +++ b/src/main/cljc/mastodon_bot/transform_domain.cljc @@ -0,0 +1,42 @@ +(ns mastodon-bot.transform-domain + (:require + [clojure.spec.alpha :as s] + [mastodon-bot.mastodon-domain :as md] + [mastodon-bot.twitter-domain :as twd] + [mastodon-bot.rss-domain :as rd] + [mastodon-bot.tumblr-domain :as td])) + +(s/def ::created-at any?) +(s/def ::text string?) +(s/def ::untrimmed-text string?) +(s/def ::media-links string?) +(s/def ::screen_name string?) +(def intermediate? (s/keys :req-un [::created-at ::text ::screen_name] + :opt-un [::media-links ::untrimmed-text])) + +(s/def ::source-type #{:twitter :rss :tumblr}) +(s/def ::resolve-urls? boolean?) +(s/def ::content-filter string?) +(s/def ::content-filters (s/* ::content-filter)) +(s/def ::keyword-filter string?) +(s/def ::keyword-filters (s/* ::keyword-filter)) +(s/def ::replacements any?) +(defmulti source-type :source-type) +(defmethod source-type :twitter [_] + (s/merge (s/keys :req-un[::source-type]) twd/twitter-source?)) +(defmethod source-type :rss [_] + (s/merge (s/keys :req-un [::source-type]) rd/rss-source?)) +(defmethod source-type :tumblr [_] + (s/merge (s/keys :req-un [::source-type]) td/tumblr-source?)) +(s/def ::source (s/multi-spec source-type ::source-type)) + +(s/def ::target-type #{:mastodon}) +(defmulti target-type :target-type) +(defmethod target-type :mastodon [_] + (s/merge (s/keys :req-un [::target-type]) md/mastodon-target?)) +(s/def ::target (s/multi-spec target-type ::target-type)) + +(s/def ::transformation (s/keys :req-un [::source ::target] + :opt-un [::resolve-urls? ::content-filters ::keyword-filters + ::replacements])) +(def transformations? (s/* ::transformation)) diff --git a/src/main/cljc/mastodon_bot/tumblr_domain.cljc b/src/main/cljc/mastodon_bot/tumblr_domain.cljc new file mode 100755 index 0000000..881267b --- /dev/null +++ b/src/main/cljc/mastodon_bot/tumblr_domain.cljc @@ -0,0 +1,16 @@ +(ns mastodon-bot.tumblr-domain + (:require + [clojure.spec.alpha :as s] + )) + +(s/def ::consumer_key string?) +(s/def ::consumer_secret string?) +(s/def ::token string?) +(s/def ::token_secret string?) +(def tumblr-auth? (s/keys :req-un [::consumer_key ::consumer_secret ::token + ::token_secret])) + +(s/def ::limit pos?) +(s/def ::account string?) +(s/def ::accounts (s/* ::account)) +(def tumblr-source? (s/keys :req-un [::limit ::accounts])) diff --git a/src/main/cljc/mastodon_bot/twitter_domain.cljs b/src/main/cljc/mastodon_bot/twitter_domain.cljs new file mode 100755 index 0000000..0641866 --- /dev/null +++ b/src/main/cljc/mastodon_bot/twitter_domain.cljs @@ -0,0 +1,18 @@ +(ns mastodon-bot.twitter-domain + (:require + [clojure.spec.alpha :as s])) + +(s/def ::consumer_key string?) +(s/def ::consumer_secret string?) +(s/def ::access_token_key string?) +(s/def ::access_token_secret string?) +(def twitter-auth? (s/keys :req-un [::consumer_key ::consumer_secret ::access_token_key + ::access_token_secret])) + +(s/def ::include-rts? boolean?) +(s/def ::include-replies? boolean?) +(s/def ::nitter-urls? boolean?) +(s/def ::account string?) +(s/def ::accounts (s/* ::account)) +(def twitter-source? (s/keys :req-un [::include-rts? ::include-replies? ::accounts] + :opt-un [::nitter-urls?])) diff --git a/src/main/mastodon_bot/core.cljs b/src/main/cljs/mastodon_bot/core.cljs similarity index 72% rename from src/main/mastodon_bot/core.cljs rename to src/main/cljs/mastodon_bot/core.cljs index dd2eb53..891f43a 100755 --- a/src/main/mastodon_bot/core.cljs +++ b/src/main/cljs/mastodon_bot/core.cljs @@ -2,51 +2,35 @@ (:require [clojure.spec.alpha :as s] [clojure.spec.test.alpha :as st] - [clojure.string :as cs] [orchestra.core :refer-macros [defn-spec]] [expound.alpha :as expound] [mastodon-bot.infra :as infra] + [mastodon-bot.core-domain :as cd] [mastodon-bot.transform :as transform] - [mastodon-bot.mastodon-api :as masto] - [mastodon-bot.twitter-api :as twitter] - [mastodon-bot.tumblr-api :as tumblr])) + [mastodon-bot.mastodon-api :as ma])) (set! s/*explain-out* expound/printer) -(s/def ::mastodon masto/mastodon-auth?) -(s/def ::twitter twitter/twitter-auth?) -(s/def ::tumblr tumblr/tumblr-auth?) -(s/def ::transform transform/transformations?) -(s/def ::auth (s/keys :opt-un [::mastodon ::twitter ::tumblr])) -(def config? - (s/keys :req-un [::auth ::transform])) - -(s/def ::options (s/* #{"-h"})) -(s/def ::config-location (s/? (s/and string? - #(not (cs/starts-with? % "-"))))) -(s/def ::args (s/cat :options ::options - :config-location ::config-location)) - -(defn-spec mastodon-auth ::mastodon - [config config?] +(defn-spec mastodon-auth ::cd/mastodon + [config cd/config?] (get-in config [:auth :mastodon])) -(defn-spec twitter-auth ::twitter - [config config?] +(defn-spec twitter-auth ::cd/twitter + [config cd/config?] (get-in config [:auth :twitter])) -(defn-spec tumblr-auth ::tumblr - [config config?] +(defn-spec tumblr-auth ::cd/tumblr + [config cd/config?] (get-in config [:auth :tumblr])) -(defn-spec transform ::transform - [config config?] +(defn-spec transform ::cd/transform + [config cd/config?] (:transform config)) (defn-spec transform! any? - [config config?] + [config cd/config?] (let [mastodon-auth (mastodon-auth config)] - (masto/get-mastodon-timeline + (ma/get-mastodon-timeline mastodon-auth (fn [timeline] (let [last-post-time (-> timeline first :created_at (js/Date.))] @@ -92,9 +76,9 @@ ") (defn main [& args] - (let [parsed-args (s/conform ::args args)] + (let [parsed-args (s/conform ::cd/args args)] (if (= ::s/invalid parsed-args) - (do (s/explain ::args args) + (do (s/explain ::cd/args args) (infra/exit-with-error (str "Bad commandline arguments\n" usage))) (let [{:keys [options config-location]} parsed-args] (cond diff --git a/src/main/mastodon_bot/infra.cljs b/src/main/cljs/mastodon_bot/infra.cljs similarity index 100% rename from src/main/mastodon_bot/infra.cljs rename to src/main/cljs/mastodon_bot/infra.cljs diff --git a/src/main/mastodon_bot/mastodon_api.cljs b/src/main/cljs/mastodon_bot/mastodon_api.cljs similarity index 75% rename from src/main/mastodon_bot/mastodon_api.cljs rename to src/main/cljs/mastodon_bot/mastodon_api.cljs index b14ff55..5c40ff1 100755 --- a/src/main/mastodon_bot/mastodon_api.cljs +++ b/src/main/cljs/mastodon_bot/mastodon_api.cljs @@ -1,46 +1,18 @@ (ns mastodon-bot.mastodon-api (:require - [clojure.spec.alpha :as s] - [clojure.spec.test.alpha :as st] [orchestra.core :refer-macros [defn-spec]] [clojure.string :as string] + [mastodon-bot.mastodon-domain :as m] [mastodon-bot.infra :as infra] ["request" :as request] ["mastodon-api" :as mastodon])) -(s/def ::access_token string?) -(s/def ::api_url string?) -(s/def ::account-id string?) -(s/def ::append-screen-name? boolean?) -(s/def ::signature string?) -(s/def ::sensitive? boolean?) -(s/def ::media-only? boolean?) -(s/def ::visibility #{"direct" "private" "unlisted" "public"}) -(s/def ::max-post-length (fn [n] (and - (int? n) - (<= n 500) - (> n 0)))) -(def mastodon-auth? (s/keys :req-un [::account-id ::access_token ::api_url])) -(def mastodon-target? (s/keys :opt-un [::max-post-length - ::signature - ::visibility - ::append-screen-name? - ::sensitive? - ::media-only?])) - (def mastodon-target-defaults {:append-screen-name? false :visibility "public" :sensitive? true :media-only? false :max-post-length 300}) -(s/def ::created-at any?) -(s/def ::text string?) -(s/def ::media-links string?) - -(def mastodon-output? (s/keys :req-un [::created-at ::text] - :opt-un [::media-links])) - (defn trim-text [text max-post-length] (cond @@ -58,19 +30,19 @@ :else text)) -(defn-spec max-post-length ::max-post-length - [target mastodon-target?] +(defn-spec max-post-length ::m/max-post-length + [target m/mastodon-target?] (:max-post-length target)) (defn-spec mastodon-client any? - [mastodon-auth mastodon-auth?] + [mastodon-auth m/mastodon-auth?] (or (some-> mastodon-auth clj->js mastodon.) (infra/exit-with-error "missing Mastodon auth configuration!"))) (defn-spec delete-status any? - [mastodon-auth mastodon-auth? + [mastodon-auth m/mastodon-auth? status-id string?] (.delete (mastodon-client mastodon-auth) (str "statuses/" status-id) #js {})) @@ -89,8 +61,8 @@ (.then #(-> % callback)))))) (defn-spec post-image any? - [mastodon-auth mastodon-auth? - target mastodon-target? + [mastodon-auth m/mastodon-auth? + target m/mastodon-target? image-stream any? description string? callback fn?] @@ -119,8 +91,8 @@ (post-status mastodon-auth target status-text (not-empty ids) callback)))) (defn-spec post-items any? - [mastodon-auth mastodon-auth? - target mastodon-target? + [mastodon-auth m/mastodon-auth? + target m/mastodon-target? items any?] (doseq [{:keys [text media-links]} items] (if media-links @@ -129,7 +101,7 @@ (post-status mastodon-auth target text))))) (defn-spec get-mastodon-timeline any? - [mastodon-auth mastodon-auth? + [mastodon-auth m/mastodon-auth? callback fn?] (.then (.get (mastodon-client mastodon-auth) (str "accounts/" (:account-id mastodon-auth) "/statuses") #js {}) @@ -138,8 +110,8 @@ (infra/exit-with-error error) (callback response))))) -(defn-spec intermediate-to-mastodon mastodon-output? - [target mastodon-target? +(defn-spec intermediate-to-mastodon m/mastodon-output? + [target m/mastodon-target? input any?] (let [target-with-defaults (merge mastodon-target-defaults target) diff --git a/src/main/cljs/mastodon_bot/rss_api.cljs b/src/main/cljs/mastodon_bot/rss_api.cljs new file mode 100755 index 0000000..17efb56 --- /dev/null +++ b/src/main/cljs/mastodon_bot/rss_api.cljs @@ -0,0 +1,24 @@ +(ns mastodon-bot.rss-api + (:require + [orchestra.core :refer-macros [defn-spec]] + [mastodon-bot.rss-domain :as rd] + ["rss-parser" :as rss])) + +(defn-spec rss-client any? + [] + (rss.)) + +(defn-spec parse-feed any? + [item ::rd/feed-item] + (let [{:keys [title isoDate pubDate content link]} item] + {:created-at (js/Date. (or isoDate pubDate)) + :text (str title + "\n\n" + link)})) + +(defn-spec get-feed map? + [url string? + callback fn?] + (print url) + (-> (.parseURL (rss-client) url) + (.then callback))) diff --git a/src/main/mastodon_bot/transform.cljs b/src/main/cljs/mastodon_bot/transform.cljs similarity index 55% rename from src/main/mastodon_bot/transform.cljs rename to src/main/cljs/mastodon_bot/transform.cljs index db42f93..635f8b1 100644 --- a/src/main/mastodon_bot/transform.cljs +++ b/src/main/cljs/mastodon_bot/transform.cljs @@ -1,52 +1,19 @@ (ns mastodon-bot.transform (:require - [clojure.spec.alpha :as s] - [clojure.spec.test.alpha :as st] [orchestra.core :refer-macros [defn-spec]] [clojure.string :as string] [mastodon-bot.infra :as infra] - [mastodon-bot.mastodon-api :as masto] - [mastodon-bot.twitter-api :as twitter] - [mastodon-bot.rss-api :as rss] - [mastodon-bot.tumblr-api :as tumblr] + [mastodon-bot.mastodon-domain :as md] + [mastodon-bot.mastodon-api :as ma] + [mastodon-bot.twitter-domain :as twd] + [mastodon-bot.twitter-api :as twa] + [mastodon-bot.rss-api :as ra] + [mastodon-bot.tumblr-domain :as td] + [mastodon-bot.tumblr-api :as ta] + [mastodon-bot.transform-domain :as trd] ["deasync" :as deasync] ["request" :as request])) -(s/def ::created-at any?) -(s/def ::text string?) -(s/def ::untrimmed-text string?) -(s/def ::media-links string?) -(s/def ::screen_name string?) -(def intermediate? (s/keys :req-un [::created-at ::text ::screen_name] - :opt-un [::media-links ::untrimmed-text])) - -(s/def ::source-type #{:twitter :rss :tumblr}) -(s/def ::resolve-urls? boolean?) -(s/def ::content-filter string?) -(s/def ::content-filters (s/* ::content-filter)) -(s/def ::keyword-filter string?) -(s/def ::keyword-filters (s/* ::keyword-filter)) -(s/def ::replacements any?) -(defmulti source-type :source-type) -(defmethod source-type :twitter [_] - (s/merge (s/keys :req-un[::source-type]) twitter/twitter-source?)) -(defmethod source-type :rss [_] - (s/merge (s/keys :req-un [::source-type]) rss/rss-source?)) -(defmethod source-type :tumblr [_] - (s/merge (s/keys :req-un [::source-type]) tumblr/tumblr-source?)) -(s/def ::source (s/multi-spec source-type ::source-type)) - -(s/def ::target-type #{:mastodon}) -(defmulti target-type :target-type) -(defmethod target-type :mastodon [_] - (s/merge (s/keys :req-un [::target-type]) masto/mastodon-target?)) -(s/def ::target (s/multi-spec target-type ::target-type)) - -(s/def ::transformation (s/keys :req-un [::source ::target] - :opt-un [::resolve-urls? ::content-filters ::keyword-filters - ::replacements])) -(def transformations? (s/* ::transformation)) - (defn resolve-url [[uri]] (try (or @@ -62,60 +29,60 @@ (def shortened-url-pattern #"(https?://)?(?:\S+(?::\S*)?@)?(?:(?!(?:10|127)(?:\.\d{1,3}){3})(?!(?:169\.254|192\.168)(?:\.\d{1,3}){2})(?!172\.(?:1[6-9]|2\d|3[0-1])(?:\.\d{1,3}){2})(?:[1-9]\d?|1\d\d|2[01]\d|22[0-3])(?:\.(?:1?\d{1,2}|2[0-4]\d|25[0-5])){2}(?:\.(?:[1-9]\d?|1\d\d|2[0-4]\d|25[0-4]))|(?:(?:[a-z\u00a1-\uffff0-9]-*)*[a-z\u00a1-\uffff0-9]+)(?:\.(?:[a-z\u00a1-\uffff0-9]-*)*[a-z\u00a1-\uffff0-9]+)*(?:\.(?:[a-z\u00a1-\uffff]{2,}))\.?)(?::\d{2,5})?(?:[/?#]\S*)?") -(defn-spec intermediate-resolve-urls intermediate? - [resolve-urls? ::resolve-urls? - input intermediate?] +(defn-spec intermediate-resolve-urls trd/intermediate? + [resolve-urls? ::trd/resolve-urls? + input trd/intermediate?] (if resolve-urls? (update input :text #(string/replace % shortened-url-pattern resolve-url)) input)) -(defn-spec content-filter-regexes ::content-filters - [transformation ::transformation] +(defn-spec content-filter-regexes ::trd/content-filters + [transformation ::trd/transformation] (mapv re-pattern (:content-filters transformation))) -(defn-spec keyword-filter-regexes ::keyword-filters - [transformation ::transformation] +(defn-spec keyword-filter-regexes ::trd/keyword-filters + [transformation ::trd/transformation] (mapv re-pattern (:keyword-filters transformation))) (defn-spec blocked-content? boolean? - [transformation ::transformation + [transformation ::trd/transformation text string?] (boolean (or (some #(re-find % text) (content-filter-regexes transformation)) (when (not-empty (keyword-filter-regexes transformation)) (empty? (some #(re-find % text) (keyword-filter-regexes transformation))))))) -(defn-spec perform-replacements intermediate? - [transformation ::transformation - input intermediate?] +(defn-spec perform-replacements trd/intermediate? + [transformation ::trd/transformation + input trd/intermediate?] (update input :text #(reduce-kv string/replace % (:replacements transformation)))) (defn-spec post-tweets-to-mastodon any? - [mastodon-auth masto/mastodon-auth? - transformation ::transformation + [mastodon-auth md/mastodon-auth? + transformation ::trd/transformation last-post-time any?] (let [{:keys [source target resolve-urls?]} transformation] (fn [error tweets response] (if error (infra/exit-with-error error) (->> (infra/js->edn tweets) - (map twitter/parse-tweet) + (map twa/parse-tweet) (filter #(> (:created-at %) last-post-time)) (remove #(blocked-content? transformation (:text %))) (map #(intermediate-resolve-urls resolve-urls? %)) - (map #(twitter/nitter-url source %)) + (map #(twa/nitter-url source %)) (map #(perform-replacements transformation %)) - (map #(masto/intermediate-to-mastodon target %)) - (masto/post-items mastodon-auth target)))))) + (map #(ma/intermediate-to-mastodon target %)) + (ma/post-items mastodon-auth target)))))) (defn-spec tweets-to-mastodon any? - [mastodon-auth masto/mastodon-auth? - twitter-auth twitter/twitter-auth? - transformation ::transformation + [mastodon-auth md/mastodon-auth? + twitter-auth twd/twitter-auth? + transformation ::trd/transformation last-post-time any?] (let [{:keys [source target resolve-urls?]} transformation] (doseq [account (:accounts source)] - (twitter/user-timeline + (twa/user-timeline twitter-auth source account @@ -125,8 +92,8 @@ last-post-time))))) (defn-spec post-tumblr-to-mastodon any? - [mastodon-auth masto/mastodon-auth? - transformation ::transformation + [mastodon-auth md/mastodon-auth? + transformation ::trd/transformation last-post-time any?] (let [{:keys [source target resolve-urls?]} transformation] (fn [error tweets response] @@ -134,21 +101,21 @@ (infra/exit-with-error error) (->> (infra/js->edn tweets) :posts - (mapv tumblr/parse-tumblr-post) + (mapv ta/parse-tumblr-post) (filter #(> (:created-at %) last-post-time)) (remove #(blocked-content? transformation (:text %))) (map #(perform-replacements transformation %)) - (map #(masto/intermediate-to-mastodon target %)) - (masto/post-items mastodon-auth target)))))) + (map #(ma/intermediate-to-mastodon target %)) + (ma/post-items mastodon-auth target)))))) (defn-spec tumblr-to-mastodon any? - [mastodon-auth masto/mastodon-auth? - tumblr-auth tumblr/tumblr-auth? - transformation ::transformation + [mastodon-auth md/mastodon-auth? + tumblr-auth td/tumblr-auth? + transformation ::trd/transformation last-post-time any?] (let [{:keys [accounts limit]} transformation] (doseq [account accounts] - (let [client (tumblr/tumblr-client tumblr-auth account)] + (let [client (ta/tumblr-client tumblr-auth account)] (.posts client #js {:limit (or limit 5)} (post-tumblr-to-mastodon @@ -158,29 +125,29 @@ ))))) (defn-spec post-rss-to-mastodon any? - [mastodon-auth masto/mastodon-auth? - transformation ::transformation + [mastodon-auth md/mastodon-auth? + transformation ::trd/transformation last-post-time any?] (let [{:keys [source target resolve-urls?]} transformation] (fn [payload] (->> (infra/js->edn payload) (:items) - (map rss/parse-feed) + (map ra/parse-feed) (filter #(> (:created-at %) last-post-time)) (remove #(blocked-content? transformation (:text %))) (map #(intermediate-resolve-urls resolve-urls? %)) (map #(perform-replacements transformation %)) - (map #(masto/intermediate-to-mastodon target %)) - (masto/post-items mastodon-auth target))))) + (map #(ma/intermediate-to-mastodon target %)) + (ma/post-items mastodon-auth target))))) (defn-spec rss-to-mastodon any? - [mastodon-auth masto/mastodon-auth? - transformation ::transformation + [mastodon-auth md/mastodon-auth? + transformation ::trd/transformation last-post-time any?] (let [{:keys [source target]} transformation] (doseq [[name url] (:feeds source)] - (rss/get-feed + (ra/get-feed url (post-rss-to-mastodon mastodon-auth diff --git a/src/main/mastodon_bot/tumblr_api.cljs b/src/main/cljs/mastodon_bot/tumblr_api.cljs similarity index 65% rename from src/main/mastodon_bot/tumblr_api.cljs rename to src/main/cljs/mastodon_bot/tumblr_api.cljs index 6af87e6..bb79784 100755 --- a/src/main/mastodon_bot/tumblr_api.cljs +++ b/src/main/cljs/mastodon_bot/tumblr_api.cljs @@ -1,27 +1,14 @@ (ns mastodon-bot.tumblr-api (:require - [clojure.spec.alpha :as s] - [clojure.spec.test.alpha :as st] [orchestra.core :refer-macros [defn-spec]] [clojure.string :as string] + [mastodon-bot.tumblr-domain :as td] [mastodon-bot.infra :as infra] ["tumblr" :as tumblr] )) -(s/def ::consumer_key string?) -(s/def ::consumer_secret string?) -(s/def ::token string?) -(s/def ::token_secret string?) -(def tumblr-auth? (s/keys :req-un [::consumer_key ::consumer_secret ::token - ::token_secret])) - -(s/def ::limit pos?) -(s/def ::account string?) -(s/def ::accounts (s/* ::account)) -(def tumblr-source? (s/keys :req-un [::limit ::accounts])) - (defn-spec tumblr-client any? - [access-keys tumblr-auth? + [access-keys td/tumblr-auth? account string?] (try (tumblr/Blog. account (clj->js access-keys)) diff --git a/src/main/mastodon_bot/twitter_api.cljs b/src/main/cljs/mastodon_bot/twitter_api.cljs similarity index 70% rename from src/main/mastodon_bot/twitter_api.cljs rename to src/main/cljs/mastodon_bot/twitter_api.cljs index c6d8564..2d66dd2 100755 --- a/src/main/mastodon_bot/twitter_api.cljs +++ b/src/main/cljs/mastodon_bot/twitter_api.cljs @@ -1,30 +1,15 @@ (ns mastodon-bot.twitter-api (:require [clojure.spec.alpha :as s] - [clojure.spec.test.alpha :as st] [orchestra.core :refer-macros [defn-spec]] [clojure.string :as string] ["twitter" :as twitter] [mastodon-bot.infra :as infra] + [mastodon-bot.twitter-domain :as td] )) -(s/def ::consumer_key string?) -(s/def ::consumer_secret string?) -(s/def ::access_token_key string?) -(s/def ::access_token_secret string?) -(def twitter-auth? (s/keys :req-un [::consumer_key ::consumer_secret ::access_token_key - ::access_token_secret])) - -(s/def ::include-rts? boolean?) -(s/def ::include-replies? boolean?) -(s/def ::nitter-urls? boolean?) -(s/def ::account string?) -(s/def ::accounts (s/* ::account)) -(def twitter-source? (s/keys :req-un [::include-rts? ::include-replies? ::accounts] - :opt-un [::nitter-urls?])) - (defn-spec twitter-client any? - [twitter-auth twitter-auth?] + [twitter-auth td/twitter-auth?] (try (twitter. (clj->js twitter-auth)) (catch js/Error e @@ -52,16 +37,16 @@ :media-links (keep #(when (= (:type %) "photo") (:media_url_https %)) media)}) (defn-spec nitter-url map? - [source twitter-source? + [source td/twitter-source? parsed-tweet map?] (if (:nitter-urls? source) (update parsed-tweet :text #(string/replace % #"https://twitter.com" "https://nitter.net")) parsed-tweet)) (defn-spec user-timeline any? - [twitter-auth twitter-auth? - source twitter-source? - account ::account + [twitter-auth td/twitter-auth? + source td/twitter-source? + account ::td/account callback fn?] (let [{:keys [include-rts? include-replies?]} source] (.get (twitter-client twitter-auth) diff --git a/src/main/mastodon_bot/rss_api.cljs b/src/main/mastodon_bot/rss_api.cljs deleted file mode 100755 index 466b80d..0000000 --- a/src/main/mastodon_bot/rss_api.cljs +++ /dev/null @@ -1,40 +0,0 @@ -(ns mastodon-bot.rss-api - (:require - [clojure.spec.alpha :as s] - [clojure.spec.test.alpha :as st] - [orchestra.core :refer-macros [defn-spec]] - ["rss-parser" :as rss] - [mastodon-bot.infra :as infra] - )) - -(s/def ::feed (s/cat :name string? :url string?)) -(s/def ::feeds (s/coll-of ::feed)) -(def rss-source? (s/keys :req-un [::feeds])) - -(s/def ::title string?) -(s/def ::content string?) -(s/def ::link string?) -(s/def ::author string?) -(s/def ::isoDate string?) -(s/def ::pubDate string?) -(s/def ::feed-item (s/keys :req-un [::title ::content ::link] - :opt-un [::author ::isoDate ::pubDate])) - -(defn-spec rss-client any? - [] - (rss.)) - -(defn-spec parse-feed any? - [item ::feed-item] - (let [{:keys [title isoDate pubDate content link]} item] - {:created-at (js/Date. (or isoDate pubDate)) - :text (str title - "\n\n" - link)})) - -(defn-spec get-feed map? - [url string? - callback fn?] - (print url) - (-> (.parseURL (rss-client) url) - (.then callback))) diff --git a/src/test/cljc/mastodon_bot/rss_domain_test.cljc b/src/test/cljc/mastodon_bot/rss_domain_test.cljc new file mode 100755 index 0000000..e20744a --- /dev/null +++ b/src/test/cljc/mastodon_bot/rss_domain_test.cljc @@ -0,0 +1,11 @@ +(ns mastodon-bot.rss-domain-test + (:require + [cljs.test :refer-macros [deftest is testing run-tests]] + [clojure.spec.alpha :as s] + [mastodon-bot.rss-domain :as sut] + )) + +(deftest test-spec + (is (s/valid? sut/rss-source? + {:feeds [["correctiv-blog" "https://news.correctiv.org/news/rss.php"]]} + ))) diff --git a/src/test/mastodon_bot/transform_test.cljs b/src/test/cljc/mastodon_bot/transform_domain_test.cljs similarity index 56% rename from src/test/mastodon_bot/transform_test.cljs rename to src/test/cljc/mastodon_bot/transform_domain_test.cljs index dcb4df8..5958869 100755 --- a/src/test/mastodon_bot/transform_test.cljs +++ b/src/test/cljc/mastodon_bot/transform_domain_test.cljs @@ -1,12 +1,8 @@ -(ns mastodon-bot.transform-test +(ns mastodon-bot.transform-domain-test (:require [cljs.test :refer-macros [deftest is testing run-tests]] [clojure.spec.alpha :as s] - [cljs.reader :as edn] - ["fs" :as fs] - [mastodon-bot.core :as core] - [mastodon-bot.twitter-api :as twitter] - [mastodon-bot.transform :as sut] + [mastodon-bot.transform-domain :as sut] )) (deftest test-spec @@ -28,14 +24,3 @@ :resolve-urls? true :content-filters [".*bannedsite.*"] :keyword-filters [".*"]}]))) - -(defn readfile [filename] - (-> filename (fs/readFileSync #js {:encoding "UTF-8"}) edn/read-string)) - -(def testconfig (readfile "test.edn")) - -(deftest test-replacements - (is (= - "💠 Check out what has been going on during March in the world of @ReproBuilds! 💠 https://t.co/k6NsSO115z @opensuse@fosstodon.org @conservancy@mastodon.technology @PrototypeFund@mastodon.social @debian@fosstodon.org " - (:text (sut/perform-replacements (first (:transform testconfig)) (twitter/parse-tweet (readfile "testdata/twitter/tweet-mentions.edn")))) - ))) \ No newline at end of file diff --git a/src/test/mastodon_bot/mastodon_api_test.cljs b/src/test/cljs/mastodon_bot/mastodon_api_test.cljs similarity index 98% rename from src/test/mastodon_bot/mastodon_api_test.cljs rename to src/test/cljs/mastodon_bot/mastodon_api_test.cljs index 859736e..c3f8df4 100755 --- a/src/test/mastodon_bot/mastodon_api_test.cljs +++ b/src/test/cljs/mastodon_bot/mastodon_api_test.cljs @@ -1,7 +1,6 @@ (ns mastodon-bot.mastodon-api-test (:require [cljs.test :refer-macros [deftest is testing run-tests]] - [clojure.spec.alpha :as s] [mastodon-bot.mastodon-api :as sut] )) diff --git a/src/test/mastodon_bot/rss_api_test.cljs b/src/test/cljs/mastodon_bot/rss_api_test.cljs similarity index 93% rename from src/test/mastodon_bot/rss_api_test.cljs rename to src/test/cljs/mastodon_bot/rss_api_test.cljs index 18e996e..2cfe874 100755 --- a/src/test/mastodon_bot/rss_api_test.cljs +++ b/src/test/cljs/mastodon_bot/rss_api_test.cljs @@ -1,15 +1,9 @@ (ns mastodon-bot.rss-api-test (:require [cljs.test :refer-macros [deftest is testing run-tests]] - [clojure.spec.alpha :as s] [mastodon-bot.rss-api :as sut] )) -(deftest test-spec - (is (s/valid? sut/rss-source? - {:feeds [["correctiv-blog" "https://news.correctiv.org/news/rss.php"]]} - ))) - (def reddit-feed-item {:title "Datahike release 0.3.1" :link "https://www.reddit.com/r/Clojure/comments/hfxotu/datahike_release_031/" diff --git a/src/test/mastodon_bot/transform_rss_test.cljs b/src/test/cljs/mastodon_bot/transform_rss_test.cljs similarity index 100% rename from src/test/mastodon_bot/transform_rss_test.cljs rename to src/test/cljs/mastodon_bot/transform_rss_test.cljs diff --git a/src/test/cljs/mastodon_bot/transform_test.cljs b/src/test/cljs/mastodon_bot/transform_test.cljs new file mode 100755 index 0000000..95cb03b --- /dev/null +++ b/src/test/cljs/mastodon_bot/transform_test.cljs @@ -0,0 +1,19 @@ +(ns mastodon-bot.transform-test + (:require + [cljs.test :refer-macros [deftest is testing run-tests]] + [cljs.reader :as edn] + ["fs" :as fs] + [mastodon-bot.twitter-api :as twitter] + [mastodon-bot.transform :as sut] + )) + +(defn readfile [filename] + (-> filename (fs/readFileSync #js {:encoding "UTF-8"}) edn/read-string)) + +(def testconfig (readfile "test.edn")) + +(deftest test-replacements + (is (= + "💠 Check out what has been going on during March in the world of @ReproBuilds! 💠 https://t.co/k6NsSO115z @opensuse@fosstodon.org @conservancy@mastodon.technology @PrototypeFund@mastodon.social @debian@fosstodon.org " + (:text (sut/perform-replacements (first (:transform testconfig)) (twitter/parse-tweet (readfile "testdata/twitter/tweet-mentions.edn")))) + ))) \ No newline at end of file diff --git a/src/test/mastodon_bot/twitter_api_test.cljs b/src/test/cljs/mastodon_bot/twitter_api_test.cljs similarity index 100% rename from src/test/mastodon_bot/twitter_api_test.cljs rename to src/test/cljs/mastodon_bot/twitter_api_test.cljs