Provide schema as jar

This commit is contained in:
Michael Jerger 2021-03-11 16:54:54 +00:00
parent 72afd80373
commit 9139ed7539
24 changed files with 310 additions and 255 deletions

15
.gitignore vendored
View file

@ -1,7 +1,14 @@
config.edn # cljs ignores
config*.edn
package-lock.json package-lock.json
/.shadow-cljs
/node_modules /node_modules
/.shadow-cljs
# mastodon-bot ignores
config*.edn
# lein ignores
/target /target
/mastodon-bot.js pom.xml
# ide ignores
.calva/

16
project.clj Normal file
View file

@ -0,0 +1,16 @@
(defproject dda/mastodon-bot "1.10.5-SNAPSHOT"
:description "Bot to publish twitter, tumblr or rss posts to an mastodon account."
:url "https://github.com/yogthos/mastodon-bot"
:author "Dmitri Sotnikov"
:license {:name "MIT"}
:dependencies []
:source-paths ["src/main/cljc"
"src/main/clj"]
:resource-paths ["src/main/resources"]
:repositories [["snapshots" :clojars]
["releases" :clojars]]
:deploy-repositories [["snapshots" :clojars]
["releases" :clojars]]
:profiles {:test {:test-paths ["src/test/cljc"]
:resource-paths ["src/test/resources"]
:dependencies []}})

View file

@ -1,5 +1,7 @@
{:source-paths ["src/main" {:source-paths ["src/main/cljc"
"src/test"] "src/main/cljs"
"src/test/cljc"
"src/test/cljs"]
:dependencies [[orchestra "2019.02.06-1"] :dependencies [[orchestra "2019.02.06-1"]
[expound "0.8.4"]] [expound "0.8.4"]]
:builds {:test {:target :node-test :builds {:test {:target :node-test

View file

@ -0,0 +1,22 @@
(ns mastodon-bot.core-domain
(:require
[clojure.spec.alpha :as s]
[clojure.string :as cs]
[mastodon-bot.transform-domain :as trd]
[mastodon-bot.mastodon-domain :as md]
[mastodon-bot.twitter-domain :as twd]
[mastodon-bot.tumblr-domain :as td]))
(s/def ::mastodon md/mastodon-auth?)
(s/def ::twitter twd/twitter-auth?)
(s/def ::tumblr td/tumblr-auth?)
(s/def ::transform trd/transformations?)
(s/def ::auth (s/keys :opt-un [::mastodon ::twitter ::tumblr]))
(def config?
(s/keys :req-un [::auth ::transform]))
(s/def ::options (s/* #{"-h"}))
(s/def ::config-location (s/? (s/and string?
#(not (cs/starts-with? % "-")))))
(s/def ::args (s/cat :options ::options
:config-location ::config-location))

View file

@ -0,0 +1,29 @@
(ns mastodon-bot.mastodon-domain
(:require
[clojure.spec.alpha :as s]))
(s/def ::access_token string?)
(s/def ::api_url string?)
(s/def ::account-id string?)
(s/def ::append-screen-name? boolean?)
(s/def ::signature string?)
(s/def ::sensitive? boolean?)
(s/def ::media-only? boolean?)
(s/def ::visibility #{"direct" "private" "unlisted" "public"})
(s/def ::max-post-length (fn [n] (and
(int? n)
(<= n 500)
(> n 0))))
(def mastodon-auth? (s/keys :req-un [::account-id ::access_token ::api_url]))
(def mastodon-target? (s/keys :opt-un [::max-post-length
::signature
::visibility
::append-screen-name?
::sensitive?
::media-only?]))
(s/def ::created-at any?)
(s/def ::text string?)
(s/def ::media-links string?)
(def mastodon-output? (s/keys :req-un [::created-at ::text]
:opt-un [::media-links]))

View file

@ -0,0 +1,16 @@
(ns mastodon-bot.rss-domain
(:require
[clojure.spec.alpha :as s]))
(s/def ::feed (s/cat :name string? :url string?))
(s/def ::feeds (s/coll-of ::feed))
(def rss-source? (s/keys :req-un [::feeds]))
(s/def ::title string?)
(s/def ::content string?)
(s/def ::link string?)
(s/def ::author string?)
(s/def ::isoDate string?)
(s/def ::pubDate string?)
(s/def ::feed-item (s/keys :req-un [::title ::content ::link]
:opt-un [::author ::isoDate ::pubDate]))

View file

@ -0,0 +1,42 @@
(ns mastodon-bot.transform-domain
(:require
[clojure.spec.alpha :as s]
[mastodon-bot.mastodon-domain :as md]
[mastodon-bot.twitter-domain :as twd]
[mastodon-bot.rss-domain :as rd]
[mastodon-bot.tumblr-domain :as td]))
(s/def ::created-at any?)
(s/def ::text string?)
(s/def ::untrimmed-text string?)
(s/def ::media-links string?)
(s/def ::screen_name string?)
(def intermediate? (s/keys :req-un [::created-at ::text ::screen_name]
:opt-un [::media-links ::untrimmed-text]))
(s/def ::source-type #{:twitter :rss :tumblr})
(s/def ::resolve-urls? boolean?)
(s/def ::content-filter string?)
(s/def ::content-filters (s/* ::content-filter))
(s/def ::keyword-filter string?)
(s/def ::keyword-filters (s/* ::keyword-filter))
(s/def ::replacements any?)
(defmulti source-type :source-type)
(defmethod source-type :twitter [_]
(s/merge (s/keys :req-un[::source-type]) twd/twitter-source?))
(defmethod source-type :rss [_]
(s/merge (s/keys :req-un [::source-type]) rd/rss-source?))
(defmethod source-type :tumblr [_]
(s/merge (s/keys :req-un [::source-type]) td/tumblr-source?))
(s/def ::source (s/multi-spec source-type ::source-type))
(s/def ::target-type #{:mastodon})
(defmulti target-type :target-type)
(defmethod target-type :mastodon [_]
(s/merge (s/keys :req-un [::target-type]) md/mastodon-target?))
(s/def ::target (s/multi-spec target-type ::target-type))
(s/def ::transformation (s/keys :req-un [::source ::target]
:opt-un [::resolve-urls? ::content-filters ::keyword-filters
::replacements]))
(def transformations? (s/* ::transformation))

View file

@ -0,0 +1,16 @@
(ns mastodon-bot.tumblr-domain
(:require
[clojure.spec.alpha :as s]
))
(s/def ::consumer_key string?)
(s/def ::consumer_secret string?)
(s/def ::token string?)
(s/def ::token_secret string?)
(def tumblr-auth? (s/keys :req-un [::consumer_key ::consumer_secret ::token
::token_secret]))
(s/def ::limit pos?)
(s/def ::account string?)
(s/def ::accounts (s/* ::account))
(def tumblr-source? (s/keys :req-un [::limit ::accounts]))

View file

@ -0,0 +1,18 @@
(ns mastodon-bot.twitter-domain
(:require
[clojure.spec.alpha :as s]))
(s/def ::consumer_key string?)
(s/def ::consumer_secret string?)
(s/def ::access_token_key string?)
(s/def ::access_token_secret string?)
(def twitter-auth? (s/keys :req-un [::consumer_key ::consumer_secret ::access_token_key
::access_token_secret]))
(s/def ::include-rts? boolean?)
(s/def ::include-replies? boolean?)
(s/def ::nitter-urls? boolean?)
(s/def ::account string?)
(s/def ::accounts (s/* ::account))
(def twitter-source? (s/keys :req-un [::include-rts? ::include-replies? ::accounts]
:opt-un [::nitter-urls?]))

View file

@ -2,51 +2,35 @@
(:require (:require
[clojure.spec.alpha :as s] [clojure.spec.alpha :as s]
[clojure.spec.test.alpha :as st] [clojure.spec.test.alpha :as st]
[clojure.string :as cs]
[orchestra.core :refer-macros [defn-spec]] [orchestra.core :refer-macros [defn-spec]]
[expound.alpha :as expound] [expound.alpha :as expound]
[mastodon-bot.infra :as infra] [mastodon-bot.infra :as infra]
[mastodon-bot.core-domain :as cd]
[mastodon-bot.transform :as transform] [mastodon-bot.transform :as transform]
[mastodon-bot.mastodon-api :as masto] [mastodon-bot.mastodon-api :as ma]))
[mastodon-bot.twitter-api :as twitter]
[mastodon-bot.tumblr-api :as tumblr]))
(set! s/*explain-out* expound/printer) (set! s/*explain-out* expound/printer)
(s/def ::mastodon masto/mastodon-auth?) (defn-spec mastodon-auth ::cd/mastodon
(s/def ::twitter twitter/twitter-auth?) [config cd/config?]
(s/def ::tumblr tumblr/tumblr-auth?)
(s/def ::transform transform/transformations?)
(s/def ::auth (s/keys :opt-un [::mastodon ::twitter ::tumblr]))
(def config?
(s/keys :req-un [::auth ::transform]))
(s/def ::options (s/* #{"-h"}))
(s/def ::config-location (s/? (s/and string?
#(not (cs/starts-with? % "-")))))
(s/def ::args (s/cat :options ::options
:config-location ::config-location))
(defn-spec mastodon-auth ::mastodon
[config config?]
(get-in config [:auth :mastodon])) (get-in config [:auth :mastodon]))
(defn-spec twitter-auth ::twitter (defn-spec twitter-auth ::cd/twitter
[config config?] [config cd/config?]
(get-in config [:auth :twitter])) (get-in config [:auth :twitter]))
(defn-spec tumblr-auth ::tumblr (defn-spec tumblr-auth ::cd/tumblr
[config config?] [config cd/config?]
(get-in config [:auth :tumblr])) (get-in config [:auth :tumblr]))
(defn-spec transform ::transform (defn-spec transform ::cd/transform
[config config?] [config cd/config?]
(:transform config)) (:transform config))
(defn-spec transform! any? (defn-spec transform! any?
[config config?] [config cd/config?]
(let [mastodon-auth (mastodon-auth config)] (let [mastodon-auth (mastodon-auth config)]
(masto/get-mastodon-timeline (ma/get-mastodon-timeline
mastodon-auth mastodon-auth
(fn [timeline] (fn [timeline]
(let [last-post-time (-> timeline first :created_at (js/Date.))] (let [last-post-time (-> timeline first :created_at (js/Date.))]
@ -92,9 +76,9 @@
") ")
(defn main [& args] (defn main [& args]
(let [parsed-args (s/conform ::args args)] (let [parsed-args (s/conform ::cd/args args)]
(if (= ::s/invalid parsed-args) (if (= ::s/invalid parsed-args)
(do (s/explain ::args args) (do (s/explain ::cd/args args)
(infra/exit-with-error (str "Bad commandline arguments\n" usage))) (infra/exit-with-error (str "Bad commandline arguments\n" usage)))
(let [{:keys [options config-location]} parsed-args] (let [{:keys [options config-location]} parsed-args]
(cond (cond

View file

@ -1,46 +1,18 @@
(ns mastodon-bot.mastodon-api (ns mastodon-bot.mastodon-api
(:require (:require
[clojure.spec.alpha :as s]
[clojure.spec.test.alpha :as st]
[orchestra.core :refer-macros [defn-spec]] [orchestra.core :refer-macros [defn-spec]]
[clojure.string :as string] [clojure.string :as string]
[mastodon-bot.mastodon-domain :as m]
[mastodon-bot.infra :as infra] [mastodon-bot.infra :as infra]
["request" :as request] ["request" :as request]
["mastodon-api" :as mastodon])) ["mastodon-api" :as mastodon]))
(s/def ::access_token string?)
(s/def ::api_url string?)
(s/def ::account-id string?)
(s/def ::append-screen-name? boolean?)
(s/def ::signature string?)
(s/def ::sensitive? boolean?)
(s/def ::media-only? boolean?)
(s/def ::visibility #{"direct" "private" "unlisted" "public"})
(s/def ::max-post-length (fn [n] (and
(int? n)
(<= n 500)
(> n 0))))
(def mastodon-auth? (s/keys :req-un [::account-id ::access_token ::api_url]))
(def mastodon-target? (s/keys :opt-un [::max-post-length
::signature
::visibility
::append-screen-name?
::sensitive?
::media-only?]))
(def mastodon-target-defaults {:append-screen-name? false (def mastodon-target-defaults {:append-screen-name? false
:visibility "public" :visibility "public"
:sensitive? true :sensitive? true
:media-only? false :media-only? false
:max-post-length 300}) :max-post-length 300})
(s/def ::created-at any?)
(s/def ::text string?)
(s/def ::media-links string?)
(def mastodon-output? (s/keys :req-un [::created-at ::text]
:opt-un [::media-links]))
(defn trim-text [text max-post-length] (defn trim-text [text max-post-length]
(cond (cond
@ -58,19 +30,19 @@
:else text)) :else text))
(defn-spec max-post-length ::max-post-length (defn-spec max-post-length ::m/max-post-length
[target mastodon-target?] [target m/mastodon-target?]
(:max-post-length target)) (:max-post-length target))
(defn-spec mastodon-client any? (defn-spec mastodon-client any?
[mastodon-auth mastodon-auth?] [mastodon-auth m/mastodon-auth?]
(or (some-> mastodon-auth (or (some-> mastodon-auth
clj->js clj->js
mastodon.) mastodon.)
(infra/exit-with-error "missing Mastodon auth configuration!"))) (infra/exit-with-error "missing Mastodon auth configuration!")))
(defn-spec delete-status any? (defn-spec delete-status any?
[mastodon-auth mastodon-auth? [mastodon-auth m/mastodon-auth?
status-id string?] status-id string?]
(.delete (mastodon-client mastodon-auth) (str "statuses/" status-id) #js {})) (.delete (mastodon-client mastodon-auth) (str "statuses/" status-id) #js {}))
@ -89,8 +61,8 @@
(.then #(-> % callback)))))) (.then #(-> % callback))))))
(defn-spec post-image any? (defn-spec post-image any?
[mastodon-auth mastodon-auth? [mastodon-auth m/mastodon-auth?
target mastodon-target? target m/mastodon-target?
image-stream any? image-stream any?
description string? description string?
callback fn?] callback fn?]
@ -119,8 +91,8 @@
(post-status mastodon-auth target status-text (not-empty ids) callback)))) (post-status mastodon-auth target status-text (not-empty ids) callback))))
(defn-spec post-items any? (defn-spec post-items any?
[mastodon-auth mastodon-auth? [mastodon-auth m/mastodon-auth?
target mastodon-target? target m/mastodon-target?
items any?] items any?]
(doseq [{:keys [text media-links]} items] (doseq [{:keys [text media-links]} items]
(if media-links (if media-links
@ -129,7 +101,7 @@
(post-status mastodon-auth target text))))) (post-status mastodon-auth target text)))))
(defn-spec get-mastodon-timeline any? (defn-spec get-mastodon-timeline any?
[mastodon-auth mastodon-auth? [mastodon-auth m/mastodon-auth?
callback fn?] callback fn?]
(.then (.get (mastodon-client mastodon-auth) (.then (.get (mastodon-client mastodon-auth)
(str "accounts/" (:account-id mastodon-auth) "/statuses") #js {}) (str "accounts/" (:account-id mastodon-auth) "/statuses") #js {})
@ -138,8 +110,8 @@
(infra/exit-with-error error) (infra/exit-with-error error)
(callback response))))) (callback response)))))
(defn-spec intermediate-to-mastodon mastodon-output? (defn-spec intermediate-to-mastodon m/mastodon-output?
[target mastodon-target? [target m/mastodon-target?
input any?] input any?]
(let [target-with-defaults (merge mastodon-target-defaults (let [target-with-defaults (merge mastodon-target-defaults
target) target)

View file

@ -0,0 +1,24 @@
(ns mastodon-bot.rss-api
(:require
[orchestra.core :refer-macros [defn-spec]]
[mastodon-bot.rss-domain :as rd]
["rss-parser" :as rss]))
(defn-spec rss-client any?
[]
(rss.))
(defn-spec parse-feed any?
[item ::rd/feed-item]
(let [{:keys [title isoDate pubDate content link]} item]
{:created-at (js/Date. (or isoDate pubDate))
:text (str title
"\n\n"
link)}))
(defn-spec get-feed map?
[url string?
callback fn?]
(print url)
(-> (.parseURL (rss-client) url)
(.then callback)))

View file

@ -1,52 +1,19 @@
(ns mastodon-bot.transform (ns mastodon-bot.transform
(:require (:require
[clojure.spec.alpha :as s]
[clojure.spec.test.alpha :as st]
[orchestra.core :refer-macros [defn-spec]] [orchestra.core :refer-macros [defn-spec]]
[clojure.string :as string] [clojure.string :as string]
[mastodon-bot.infra :as infra] [mastodon-bot.infra :as infra]
[mastodon-bot.mastodon-api :as masto] [mastodon-bot.mastodon-domain :as md]
[mastodon-bot.twitter-api :as twitter] [mastodon-bot.mastodon-api :as ma]
[mastodon-bot.rss-api :as rss] [mastodon-bot.twitter-domain :as twd]
[mastodon-bot.tumblr-api :as tumblr] [mastodon-bot.twitter-api :as twa]
[mastodon-bot.rss-api :as ra]
[mastodon-bot.tumblr-domain :as td]
[mastodon-bot.tumblr-api :as ta]
[mastodon-bot.transform-domain :as trd]
["deasync" :as deasync] ["deasync" :as deasync]
["request" :as request])) ["request" :as request]))
(s/def ::created-at any?)
(s/def ::text string?)
(s/def ::untrimmed-text string?)
(s/def ::media-links string?)
(s/def ::screen_name string?)
(def intermediate? (s/keys :req-un [::created-at ::text ::screen_name]
:opt-un [::media-links ::untrimmed-text]))
(s/def ::source-type #{:twitter :rss :tumblr})
(s/def ::resolve-urls? boolean?)
(s/def ::content-filter string?)
(s/def ::content-filters (s/* ::content-filter))
(s/def ::keyword-filter string?)
(s/def ::keyword-filters (s/* ::keyword-filter))
(s/def ::replacements any?)
(defmulti source-type :source-type)
(defmethod source-type :twitter [_]
(s/merge (s/keys :req-un[::source-type]) twitter/twitter-source?))
(defmethod source-type :rss [_]
(s/merge (s/keys :req-un [::source-type]) rss/rss-source?))
(defmethod source-type :tumblr [_]
(s/merge (s/keys :req-un [::source-type]) tumblr/tumblr-source?))
(s/def ::source (s/multi-spec source-type ::source-type))
(s/def ::target-type #{:mastodon})
(defmulti target-type :target-type)
(defmethod target-type :mastodon [_]
(s/merge (s/keys :req-un [::target-type]) masto/mastodon-target?))
(s/def ::target (s/multi-spec target-type ::target-type))
(s/def ::transformation (s/keys :req-un [::source ::target]
:opt-un [::resolve-urls? ::content-filters ::keyword-filters
::replacements]))
(def transformations? (s/* ::transformation))
(defn resolve-url [[uri]] (defn resolve-url [[uri]]
(try (try
(or (or
@ -62,60 +29,60 @@
(def shortened-url-pattern #"(https?://)?(?:\S+(?::\S*)?@)?(?:(?!(?:10|127)(?:\.\d{1,3}){3})(?!(?:169\.254|192\.168)(?:\.\d{1,3}){2})(?!172\.(?:1[6-9]|2\d|3[0-1])(?:\.\d{1,3}){2})(?:[1-9]\d?|1\d\d|2[01]\d|22[0-3])(?:\.(?:1?\d{1,2}|2[0-4]\d|25[0-5])){2}(?:\.(?:[1-9]\d?|1\d\d|2[0-4]\d|25[0-4]))|(?:(?:[a-z\u00a1-\uffff0-9]-*)*[a-z\u00a1-\uffff0-9]+)(?:\.(?:[a-z\u00a1-\uffff0-9]-*)*[a-z\u00a1-\uffff0-9]+)*(?:\.(?:[a-z\u00a1-\uffff]{2,}))\.?)(?::\d{2,5})?(?:[/?#]\S*)?") (def shortened-url-pattern #"(https?://)?(?:\S+(?::\S*)?@)?(?:(?!(?:10|127)(?:\.\d{1,3}){3})(?!(?:169\.254|192\.168)(?:\.\d{1,3}){2})(?!172\.(?:1[6-9]|2\d|3[0-1])(?:\.\d{1,3}){2})(?:[1-9]\d?|1\d\d|2[01]\d|22[0-3])(?:\.(?:1?\d{1,2}|2[0-4]\d|25[0-5])){2}(?:\.(?:[1-9]\d?|1\d\d|2[0-4]\d|25[0-4]))|(?:(?:[a-z\u00a1-\uffff0-9]-*)*[a-z\u00a1-\uffff0-9]+)(?:\.(?:[a-z\u00a1-\uffff0-9]-*)*[a-z\u00a1-\uffff0-9]+)*(?:\.(?:[a-z\u00a1-\uffff]{2,}))\.?)(?::\d{2,5})?(?:[/?#]\S*)?")
(defn-spec intermediate-resolve-urls intermediate? (defn-spec intermediate-resolve-urls trd/intermediate?
[resolve-urls? ::resolve-urls? [resolve-urls? ::trd/resolve-urls?
input intermediate?] input trd/intermediate?]
(if resolve-urls? (if resolve-urls?
(update input :text #(string/replace % shortened-url-pattern resolve-url)) (update input :text #(string/replace % shortened-url-pattern resolve-url))
input)) input))
(defn-spec content-filter-regexes ::content-filters (defn-spec content-filter-regexes ::trd/content-filters
[transformation ::transformation] [transformation ::trd/transformation]
(mapv re-pattern (:content-filters transformation))) (mapv re-pattern (:content-filters transformation)))
(defn-spec keyword-filter-regexes ::keyword-filters (defn-spec keyword-filter-regexes ::trd/keyword-filters
[transformation ::transformation] [transformation ::trd/transformation]
(mapv re-pattern (:keyword-filters transformation))) (mapv re-pattern (:keyword-filters transformation)))
(defn-spec blocked-content? boolean? (defn-spec blocked-content? boolean?
[transformation ::transformation [transformation ::trd/transformation
text string?] text string?]
(boolean (boolean
(or (some #(re-find % text) (content-filter-regexes transformation)) (or (some #(re-find % text) (content-filter-regexes transformation))
(when (not-empty (keyword-filter-regexes transformation)) (when (not-empty (keyword-filter-regexes transformation))
(empty? (some #(re-find % text) (keyword-filter-regexes transformation))))))) (empty? (some #(re-find % text) (keyword-filter-regexes transformation)))))))
(defn-spec perform-replacements intermediate? (defn-spec perform-replacements trd/intermediate?
[transformation ::transformation [transformation ::trd/transformation
input intermediate?] input trd/intermediate?]
(update input :text #(reduce-kv string/replace % (:replacements transformation)))) (update input :text #(reduce-kv string/replace % (:replacements transformation))))
(defn-spec post-tweets-to-mastodon any? (defn-spec post-tweets-to-mastodon any?
[mastodon-auth masto/mastodon-auth? [mastodon-auth md/mastodon-auth?
transformation ::transformation transformation ::trd/transformation
last-post-time any?] last-post-time any?]
(let [{:keys [source target resolve-urls?]} transformation] (let [{:keys [source target resolve-urls?]} transformation]
(fn [error tweets response] (fn [error tweets response]
(if error (if error
(infra/exit-with-error error) (infra/exit-with-error error)
(->> (infra/js->edn tweets) (->> (infra/js->edn tweets)
(map twitter/parse-tweet) (map twa/parse-tweet)
(filter #(> (:created-at %) last-post-time)) (filter #(> (:created-at %) last-post-time))
(remove #(blocked-content? transformation (:text %))) (remove #(blocked-content? transformation (:text %)))
(map #(intermediate-resolve-urls resolve-urls? %)) (map #(intermediate-resolve-urls resolve-urls? %))
(map #(twitter/nitter-url source %)) (map #(twa/nitter-url source %))
(map #(perform-replacements transformation %)) (map #(perform-replacements transformation %))
(map #(masto/intermediate-to-mastodon target %)) (map #(ma/intermediate-to-mastodon target %))
(masto/post-items mastodon-auth target)))))) (ma/post-items mastodon-auth target))))))
(defn-spec tweets-to-mastodon any? (defn-spec tweets-to-mastodon any?
[mastodon-auth masto/mastodon-auth? [mastodon-auth md/mastodon-auth?
twitter-auth twitter/twitter-auth? twitter-auth twd/twitter-auth?
transformation ::transformation transformation ::trd/transformation
last-post-time any?] last-post-time any?]
(let [{:keys [source target resolve-urls?]} transformation] (let [{:keys [source target resolve-urls?]} transformation]
(doseq [account (:accounts source)] (doseq [account (:accounts source)]
(twitter/user-timeline (twa/user-timeline
twitter-auth twitter-auth
source source
account account
@ -125,8 +92,8 @@
last-post-time))))) last-post-time)))))
(defn-spec post-tumblr-to-mastodon any? (defn-spec post-tumblr-to-mastodon any?
[mastodon-auth masto/mastodon-auth? [mastodon-auth md/mastodon-auth?
transformation ::transformation transformation ::trd/transformation
last-post-time any?] last-post-time any?]
(let [{:keys [source target resolve-urls?]} transformation] (let [{:keys [source target resolve-urls?]} transformation]
(fn [error tweets response] (fn [error tweets response]
@ -134,21 +101,21 @@
(infra/exit-with-error error) (infra/exit-with-error error)
(->> (infra/js->edn tweets) (->> (infra/js->edn tweets)
:posts :posts
(mapv tumblr/parse-tumblr-post) (mapv ta/parse-tumblr-post)
(filter #(> (:created-at %) last-post-time)) (filter #(> (:created-at %) last-post-time))
(remove #(blocked-content? transformation (:text %))) (remove #(blocked-content? transformation (:text %)))
(map #(perform-replacements transformation %)) (map #(perform-replacements transformation %))
(map #(masto/intermediate-to-mastodon target %)) (map #(ma/intermediate-to-mastodon target %))
(masto/post-items mastodon-auth target)))))) (ma/post-items mastodon-auth target))))))
(defn-spec tumblr-to-mastodon any? (defn-spec tumblr-to-mastodon any?
[mastodon-auth masto/mastodon-auth? [mastodon-auth md/mastodon-auth?
tumblr-auth tumblr/tumblr-auth? tumblr-auth td/tumblr-auth?
transformation ::transformation transformation ::trd/transformation
last-post-time any?] last-post-time any?]
(let [{:keys [accounts limit]} transformation] (let [{:keys [accounts limit]} transformation]
(doseq [account accounts] (doseq [account accounts]
(let [client (tumblr/tumblr-client tumblr-auth account)] (let [client (ta/tumblr-client tumblr-auth account)]
(.posts client (.posts client
#js {:limit (or limit 5)} #js {:limit (or limit 5)}
(post-tumblr-to-mastodon (post-tumblr-to-mastodon
@ -158,29 +125,29 @@
))))) )))))
(defn-spec post-rss-to-mastodon any? (defn-spec post-rss-to-mastodon any?
[mastodon-auth masto/mastodon-auth? [mastodon-auth md/mastodon-auth?
transformation ::transformation transformation ::trd/transformation
last-post-time any?] last-post-time any?]
(let [{:keys [source target resolve-urls?]} transformation] (let [{:keys [source target resolve-urls?]} transformation]
(fn [payload] (fn [payload]
(->> (infra/js->edn payload) (->> (infra/js->edn payload)
(:items) (:items)
(map rss/parse-feed) (map ra/parse-feed)
(filter #(> (:created-at %) last-post-time)) (filter #(> (:created-at %) last-post-time))
(remove #(blocked-content? transformation (:text %))) (remove #(blocked-content? transformation (:text %)))
(map #(intermediate-resolve-urls resolve-urls? %)) (map #(intermediate-resolve-urls resolve-urls? %))
(map #(perform-replacements transformation %)) (map #(perform-replacements transformation %))
(map #(masto/intermediate-to-mastodon target %)) (map #(ma/intermediate-to-mastodon target %))
(masto/post-items mastodon-auth target))))) (ma/post-items mastodon-auth target)))))
(defn-spec rss-to-mastodon any? (defn-spec rss-to-mastodon any?
[mastodon-auth masto/mastodon-auth? [mastodon-auth md/mastodon-auth?
transformation ::transformation transformation ::trd/transformation
last-post-time any?] last-post-time any?]
(let [{:keys [source target]} transformation] (let [{:keys [source target]} transformation]
(doseq [[name url] (:feeds source)] (doseq [[name url] (:feeds source)]
(rss/get-feed (ra/get-feed
url url
(post-rss-to-mastodon (post-rss-to-mastodon
mastodon-auth mastodon-auth

View file

@ -1,27 +1,14 @@
(ns mastodon-bot.tumblr-api (ns mastodon-bot.tumblr-api
(:require (:require
[clojure.spec.alpha :as s]
[clojure.spec.test.alpha :as st]
[orchestra.core :refer-macros [defn-spec]] [orchestra.core :refer-macros [defn-spec]]
[clojure.string :as string] [clojure.string :as string]
[mastodon-bot.tumblr-domain :as td]
[mastodon-bot.infra :as infra] [mastodon-bot.infra :as infra]
["tumblr" :as tumblr] ["tumblr" :as tumblr]
)) ))
(s/def ::consumer_key string?)
(s/def ::consumer_secret string?)
(s/def ::token string?)
(s/def ::token_secret string?)
(def tumblr-auth? (s/keys :req-un [::consumer_key ::consumer_secret ::token
::token_secret]))
(s/def ::limit pos?)
(s/def ::account string?)
(s/def ::accounts (s/* ::account))
(def tumblr-source? (s/keys :req-un [::limit ::accounts]))
(defn-spec tumblr-client any? (defn-spec tumblr-client any?
[access-keys tumblr-auth? [access-keys td/tumblr-auth?
account string?] account string?]
(try (try
(tumblr/Blog. account (clj->js access-keys)) (tumblr/Blog. account (clj->js access-keys))

View file

@ -1,30 +1,15 @@
(ns mastodon-bot.twitter-api (ns mastodon-bot.twitter-api
(:require (:require
[clojure.spec.alpha :as s] [clojure.spec.alpha :as s]
[clojure.spec.test.alpha :as st]
[orchestra.core :refer-macros [defn-spec]] [orchestra.core :refer-macros [defn-spec]]
[clojure.string :as string] [clojure.string :as string]
["twitter" :as twitter] ["twitter" :as twitter]
[mastodon-bot.infra :as infra] [mastodon-bot.infra :as infra]
[mastodon-bot.twitter-domain :as td]
)) ))
(s/def ::consumer_key string?)
(s/def ::consumer_secret string?)
(s/def ::access_token_key string?)
(s/def ::access_token_secret string?)
(def twitter-auth? (s/keys :req-un [::consumer_key ::consumer_secret ::access_token_key
::access_token_secret]))
(s/def ::include-rts? boolean?)
(s/def ::include-replies? boolean?)
(s/def ::nitter-urls? boolean?)
(s/def ::account string?)
(s/def ::accounts (s/* ::account))
(def twitter-source? (s/keys :req-un [::include-rts? ::include-replies? ::accounts]
:opt-un [::nitter-urls?]))
(defn-spec twitter-client any? (defn-spec twitter-client any?
[twitter-auth twitter-auth?] [twitter-auth td/twitter-auth?]
(try (try
(twitter. (clj->js twitter-auth)) (twitter. (clj->js twitter-auth))
(catch js/Error e (catch js/Error e
@ -52,16 +37,16 @@
:media-links (keep #(when (= (:type %) "photo") (:media_url_https %)) media)}) :media-links (keep #(when (= (:type %) "photo") (:media_url_https %)) media)})
(defn-spec nitter-url map? (defn-spec nitter-url map?
[source twitter-source? [source td/twitter-source?
parsed-tweet map?] parsed-tweet map?]
(if (:nitter-urls? source) (if (:nitter-urls? source)
(update parsed-tweet :text #(string/replace % #"https://twitter.com" "https://nitter.net")) (update parsed-tweet :text #(string/replace % #"https://twitter.com" "https://nitter.net"))
parsed-tweet)) parsed-tweet))
(defn-spec user-timeline any? (defn-spec user-timeline any?
[twitter-auth twitter-auth? [twitter-auth td/twitter-auth?
source twitter-source? source td/twitter-source?
account ::account account ::td/account
callback fn?] callback fn?]
(let [{:keys [include-rts? include-replies?]} source] (let [{:keys [include-rts? include-replies?]} source]
(.get (twitter-client twitter-auth) (.get (twitter-client twitter-auth)

View file

@ -1,40 +0,0 @@
(ns mastodon-bot.rss-api
(:require
[clojure.spec.alpha :as s]
[clojure.spec.test.alpha :as st]
[orchestra.core :refer-macros [defn-spec]]
["rss-parser" :as rss]
[mastodon-bot.infra :as infra]
))
(s/def ::feed (s/cat :name string? :url string?))
(s/def ::feeds (s/coll-of ::feed))
(def rss-source? (s/keys :req-un [::feeds]))
(s/def ::title string?)
(s/def ::content string?)
(s/def ::link string?)
(s/def ::author string?)
(s/def ::isoDate string?)
(s/def ::pubDate string?)
(s/def ::feed-item (s/keys :req-un [::title ::content ::link]
:opt-un [::author ::isoDate ::pubDate]))
(defn-spec rss-client any?
[]
(rss.))
(defn-spec parse-feed any?
[item ::feed-item]
(let [{:keys [title isoDate pubDate content link]} item]
{:created-at (js/Date. (or isoDate pubDate))
:text (str title
"\n\n"
link)}))
(defn-spec get-feed map?
[url string?
callback fn?]
(print url)
(-> (.parseURL (rss-client) url)
(.then callback)))

View file

@ -0,0 +1,11 @@
(ns mastodon-bot.rss-domain-test
(:require
[cljs.test :refer-macros [deftest is testing run-tests]]
[clojure.spec.alpha :as s]
[mastodon-bot.rss-domain :as sut]
))
(deftest test-spec
(is (s/valid? sut/rss-source?
{:feeds [["correctiv-blog" "https://news.correctiv.org/news/rss.php"]]}
)))

View file

@ -1,12 +1,8 @@
(ns mastodon-bot.transform-test (ns mastodon-bot.transform-domain-test
(:require (:require
[cljs.test :refer-macros [deftest is testing run-tests]] [cljs.test :refer-macros [deftest is testing run-tests]]
[clojure.spec.alpha :as s] [clojure.spec.alpha :as s]
[cljs.reader :as edn] [mastodon-bot.transform-domain :as sut]
["fs" :as fs]
[mastodon-bot.core :as core]
[mastodon-bot.twitter-api :as twitter]
[mastodon-bot.transform :as sut]
)) ))
(deftest test-spec (deftest test-spec
@ -28,14 +24,3 @@
:resolve-urls? true :resolve-urls? true
:content-filters [".*bannedsite.*"] :content-filters [".*bannedsite.*"]
:keyword-filters [".*"]}]))) :keyword-filters [".*"]}])))
(defn readfile [filename]
(-> filename (fs/readFileSync #js {:encoding "UTF-8"}) edn/read-string))
(def testconfig (readfile "test.edn"))
(deftest test-replacements
(is (=
"💠 Check out what has been going on during March in the world of @ReproBuilds! 💠 https://t.co/k6NsSO115z @opensuse@fosstodon.org @conservancy@mastodon.technology @PrototypeFund@mastodon.social @debian@fosstodon.org "
(:text (sut/perform-replacements (first (:transform testconfig)) (twitter/parse-tweet (readfile "testdata/twitter/tweet-mentions.edn"))))
)))

View file

@ -1,7 +1,6 @@
(ns mastodon-bot.mastodon-api-test (ns mastodon-bot.mastodon-api-test
(:require (:require
[cljs.test :refer-macros [deftest is testing run-tests]] [cljs.test :refer-macros [deftest is testing run-tests]]
[clojure.spec.alpha :as s]
[mastodon-bot.mastodon-api :as sut] [mastodon-bot.mastodon-api :as sut]
)) ))

View file

@ -1,15 +1,9 @@
(ns mastodon-bot.rss-api-test (ns mastodon-bot.rss-api-test
(:require (:require
[cljs.test :refer-macros [deftest is testing run-tests]] [cljs.test :refer-macros [deftest is testing run-tests]]
[clojure.spec.alpha :as s]
[mastodon-bot.rss-api :as sut] [mastodon-bot.rss-api :as sut]
)) ))
(deftest test-spec
(is (s/valid? sut/rss-source?
{:feeds [["correctiv-blog" "https://news.correctiv.org/news/rss.php"]]}
)))
(def reddit-feed-item {:title "Datahike release 0.3.1" (def reddit-feed-item {:title "Datahike release 0.3.1"
:link :link
"https://www.reddit.com/r/Clojure/comments/hfxotu/datahike_release_031/" "https://www.reddit.com/r/Clojure/comments/hfxotu/datahike_release_031/"

View file

@ -0,0 +1,19 @@
(ns mastodon-bot.transform-test
(:require
[cljs.test :refer-macros [deftest is testing run-tests]]
[cljs.reader :as edn]
["fs" :as fs]
[mastodon-bot.twitter-api :as twitter]
[mastodon-bot.transform :as sut]
))
(defn readfile [filename]
(-> filename (fs/readFileSync #js {:encoding "UTF-8"}) edn/read-string))
(def testconfig (readfile "test.edn"))
(deftest test-replacements
(is (=
"💠 Check out what has been going on during March in the world of @ReproBuilds! 💠 https://t.co/k6NsSO115z @opensuse@fosstodon.org @conservancy@mastodon.technology @PrototypeFund@mastodon.social @debian@fosstodon.org "
(:text (sut/perform-replacements (first (:transform testconfig)) (twitter/parse-tweet (readfile "testdata/twitter/tweet-mentions.edn"))))
)))