mv filters to transform
This commit is contained in:
parent
8719e7fd15
commit
cab5e24845
4 changed files with 48 additions and 52 deletions
17
README.md
17
README.md
|
@ -50,15 +50,6 @@ with later timestamps to avoid duplicate posts. On the first run the timestamp w
|
|||
;; e.g: https://mastodon.social/web/accounts/294795
|
||||
:account-id "XXXX"
|
||||
:api_url "https://botsin.space/api/v1/"
|
||||
;; optionally try to resolve URLs in posts to skip URL shorteners
|
||||
;; defaults to false
|
||||
:resolve-urls? true
|
||||
;; optional content filter regexes
|
||||
;; any posts matching the regexes will be filtered out
|
||||
:content-filters [".*bannedsite.*"]
|
||||
;; optional keyword filter regexes
|
||||
;; any posts not matching the regexes will be filtered out
|
||||
:keyword-filters [".*clojure.*"]
|
||||
}
|
||||
:transform [{:source {:type :twitter-source
|
||||
;; optional, defaults to false
|
||||
|
@ -87,7 +78,13 @@ with later timestamps to avoid duplicate posts. On the first run the timestamp w
|
|||
:signature "#newsbot"}
|
||||
;; optionally try to resolve URLs in posts to skip URL shorteners
|
||||
;; defaults to false
|
||||
:resolve-urls? true}]
|
||||
:resolve-urls? true
|
||||
;; optional content filter regexes
|
||||
;; any posts matching the regexes will be filtered out
|
||||
:content-filters [".*bannedsite.*"]
|
||||
;; optional keyword filter regexes
|
||||
;; any posts not matching the regexes will be filtered out
|
||||
:keyword-filters [".*clojure.*"]}]
|
||||
}
|
||||
```
|
||||
|
||||
|
|
|
@ -11,8 +11,6 @@
|
|||
(s/def ::access_token string?)
|
||||
(s/def ::api_url string?)
|
||||
(s/def ::account-id string?)
|
||||
(s/def ::content-filter string?)
|
||||
(s/def ::keyword-filter string?)
|
||||
(s/def ::append-screen-name? boolean?)
|
||||
(s/def ::signature string?)
|
||||
(s/def ::sensitive? boolean?)
|
||||
|
@ -24,12 +22,9 @@
|
|||
(<= n 500)
|
||||
(> n 0))))
|
||||
|
||||
(s/def ::content-filters (s/* ::content-filter))
|
||||
(s/def ::keyword-filters (s/* ::keyword-filter))
|
||||
|
||||
(def mastodon-auth? (s/keys :req-un [::account-id ::access_token ::api_url]))
|
||||
(def mastodon-target? (s/keys :opt-un [
|
||||
;::content-filters ::keyword-filters
|
||||
::max-post-length
|
||||
(def mastodon-target? (s/keys :opt-un [::max-post-length
|
||||
::signature
|
||||
::visibility
|
||||
::append-screen-name?
|
||||
|
@ -40,14 +35,6 @@
|
|||
(def mastodon-config? (s/merge mastodon-auth? mastodon-target?))
|
||||
|
||||
|
||||
(defn-spec content-filter-regexes ::content-filters
|
||||
[mastodon-config mastodon-config?]
|
||||
(mapv re-pattern (:content-filters mastodon-config)))
|
||||
|
||||
(defn-spec keyword-filter-regexes ::keyword-filters
|
||||
[mastodon-config mastodon-config?]
|
||||
(mapv re-pattern (:keyword-filters mastodon-config)))
|
||||
|
||||
(defn-spec max-post-length ::max-post-length
|
||||
[target mastodon-target?]
|
||||
(:max-post-length target))
|
||||
|
@ -64,14 +51,6 @@
|
|||
mastodon.)
|
||||
(infra/exit-with-error "missing Mastodon auth configuration!")))
|
||||
|
||||
(defn-spec blocked-content? boolean?
|
||||
[mastodon-config mastodon-config?
|
||||
text string?]
|
||||
(boolean
|
||||
(or (some #(re-find % text) (content-filter-regexes mastodon-config))
|
||||
(when (not-empty (keyword-filter-regexes mastodon-config))
|
||||
(empty? (some #(re-find % text) (keyword-filter-regexes mastodon-config)))))))
|
||||
|
||||
(defn-spec delete-status any?
|
||||
[mastodon-config mastodon-config?
|
||||
status-id string?]
|
||||
|
@ -129,7 +108,6 @@
|
|||
items any?]
|
||||
(doseq [{:keys [text media-links]}
|
||||
(->> items
|
||||
(remove #(blocked-content? mastodon-auth (:text %)))
|
||||
(filter #(> (:created-at %) last-post-time)))]
|
||||
(if media-links
|
||||
(post-status-with-images mastodon-auth target text media-links)
|
||||
|
|
|
@ -22,6 +22,10 @@
|
|||
:opt-un [::media-links]))
|
||||
(s/def ::type keyword?)
|
||||
(s/def ::resolve-urls? boolean?)
|
||||
(s/def ::content-filter string?)
|
||||
(s/def ::content-filters (s/* ::content-filter))
|
||||
(s/def ::keyword-filter string?)
|
||||
(s/def ::keyword-filters (s/* ::keyword-filter))
|
||||
(defmulti source-type :type)
|
||||
(defmethod source-type :twitter-source [_]
|
||||
(s/merge (s/keys :req-un[::type]) twitter/twitter-source?))
|
||||
|
@ -32,7 +36,7 @@
|
|||
(s/def ::target (s/multi-spec target-type ::type))
|
||||
|
||||
(s/def ::transformation (s/keys :req-un [::source ::target]
|
||||
:opt-un [::resolve-urls?]))
|
||||
:opt-un [::resolve-urls? ::content-filters ::keyword-filters]))
|
||||
(def transformations? (s/* ::transformation))
|
||||
|
||||
(defn trim-text [text max-post-length]
|
||||
|
@ -73,6 +77,23 @@
|
|||
(when resolve-urls?
|
||||
(update input :text #(string/replace % shortened-url-pattern resolve-url))))
|
||||
|
||||
(defn-spec content-filter-regexes ::content-filters
|
||||
[transformation ::transformation]
|
||||
(mapv re-pattern (:content-filters transformation)))
|
||||
|
||||
(defn-spec keyword-filter-regexes ::keyword-filters
|
||||
[transformation ::transformation]
|
||||
(mapv re-pattern (:keyword-filters transformation)))
|
||||
|
||||
(defn-spec blocked-content? boolean?
|
||||
[transformation ::transformation
|
||||
text string?]
|
||||
(boolean
|
||||
(or (some #(re-find % text) (content-filter-regexes transformation))
|
||||
(when (not-empty (keyword-filter-regexes transformation))
|
||||
(empty? (some #(re-find % text) (keyword-filter-regexes transformation)))))))
|
||||
|
||||
|
||||
; TODO: move this to mastodon-api - seems to belong strongly to mastodon
|
||||
(defn-spec intermediate-to-mastodon mastodon-output?
|
||||
[mastodon-auth masto/mastodon-auth?
|
||||
|
@ -100,19 +121,19 @@
|
|||
|
||||
(defn-spec post-tweets-to-mastodon any?
|
||||
[mastodon-auth masto/mastodon-auth?
|
||||
resolve-urls? ::resolve-urls?
|
||||
source twitter/twitter-source?
|
||||
target masto/mastodon-target?
|
||||
transformation ::transformation
|
||||
last-post-time any?]
|
||||
(fn [error tweets response]
|
||||
(if error
|
||||
(infra/exit-with-error error)
|
||||
(->> (infra/js->edn tweets)
|
||||
(map twitter/parse-tweet)
|
||||
(map #(intermediate-resolve-urls resolve-urls? %))
|
||||
(map #(twitter/nitter-url source %))
|
||||
(map #(intermediate-to-mastodon mastodon-auth target %))
|
||||
(masto/post-items mastodon-auth target last-post-time)))))
|
||||
(let [{:keys [source target resolve-urls?]} transformation]
|
||||
(fn [error tweets response]
|
||||
(if error
|
||||
(infra/exit-with-error error)
|
||||
(->> (infra/js->edn tweets)
|
||||
(map twitter/parse-tweet)
|
||||
(remove #(blocked-content? transformation (:text %)))
|
||||
(map #(intermediate-resolve-urls resolve-urls? %))
|
||||
(map #(twitter/nitter-url source %))
|
||||
(map #(intermediate-to-mastodon mastodon-auth target %))
|
||||
(masto/post-items mastodon-auth target last-post-time))))))
|
||||
|
||||
(defn-spec tweets-to-mastodon any?
|
||||
[mastodon-auth masto/mastodon-auth?
|
||||
|
@ -127,7 +148,5 @@
|
|||
account
|
||||
(post-tweets-to-mastodon
|
||||
mastodon-auth
|
||||
resolve-urls?
|
||||
source
|
||||
target
|
||||
transformation
|
||||
last-post-time)))))
|
||||
|
|
|
@ -13,7 +13,7 @@
|
|||
:include-replies? false
|
||||
:include-rts? true
|
||||
:nitter-urls? true
|
||||
:accounts ["an-twitter-account"]}
|
||||
:accounts ["an-twitter-account"]}
|
||||
:target {:type :mastodon-target
|
||||
:append-screen-name? true
|
||||
:media-only? false
|
||||
|
@ -21,4 +21,6 @@
|
|||
:visibility "unlisted"
|
||||
:sensitive? true
|
||||
:signature "my-bot"}
|
||||
:resolve-urls? true}])))
|
||||
:resolve-urls? true
|
||||
:content-filters [".*bannedsite.*"]
|
||||
:keyword-filters [".*"]}])))
|
||||
|
|
Reference in a new issue