improved RSS parsing

This commit is contained in:
dmitri.sotnikov@gmail.com 2018-03-23 09:14:26 -04:00
parent 67066dae91
commit e4a9fbdecc
2 changed files with 19 additions and 5 deletions

View file

@ -34,7 +34,8 @@ the bot will post the timeline from the specified Twitter/Tumblr accounts and RS
:rss {"Hacker News" "https://hnrss.org/newest" :rss {"Hacker News" "https://hnrss.org/newest"
"r/Clojure" "https://www.reddit.com/r/clojure/.rss"} "r/Clojure" "https://www.reddit.com/r/clojure/.rss"}
:mastodon {:access_token "XXXX" :mastodon {:access_token "XXXX"
:api_url "https://botsin.space/api/v1/"}} :api_url "https://botsin.space/api/v1/"
:max-post-length 300}}
``` ```
* the bot looks for `config.edn` at its relative path by default, an alternative location can be specified either using the `MASTODON_BOT_CONFIG` environment variable or passing the path to config as an argument * the bot looks for `config.edn` at its relative path by default, an alternative location can be specified either using the `MASTODON_BOT_CONFIG` environment variable or passing the path to config as an argument

View file

@ -20,6 +20,8 @@
(def config (-> (find-config) fs/readFileSync str edn/read-string)) (def config (-> (find-config) fs/readFileSync str edn/read-string))
(def max-post-length (or (-> config :mastodon :max-post-length) 300))
(def mastodon-client (or (some-> config :mastodon clj->js mastodon.) (def mastodon-client (or (some-> config :mastodon clj->js mastodon.)
(do (do
(js/console.error "missing Mastodon client configuration!") (js/console.error "missing Mastodon client configuration!")
@ -28,6 +30,17 @@
(defn js->edn [data] (defn js->edn [data]
(js->clj data :keywordize-keys true)) (js->clj data :keywordize-keys true))
(defn trim-text [text]
(if (> (count text) max-post-length)
(reduce
(fn [text word]
(if (> (+ (count text) (count word)) (- max-post-length 3))
(reduced (str text "..."))
(str text " " word)))
""
(clojure.string/split text #" "))
text))
(defn delete-status [status] (defn delete-status [status]
(.delete mastodon-client (str "statuses/" status) #js {})) (.delete mastodon-client (str "statuses/" status) #js {}))
@ -74,7 +87,7 @@
(defmethod parse-tumblr-post "text" [{:keys [body date short_url]}] (defmethod parse-tumblr-post "text" [{:keys [body date short_url]}]
{:created-at (js/Date. date) {:created-at (js/Date. date)
:text (str body "\n\n" short_url)}) :text (str (trim-text body) "\n\n" short_url)})
(defmethod parse-tumblr-post "photo" [{:keys [caption date photos short_url] :as post}] (defmethod parse-tumblr-post "photo" [{:keys [caption date photos short_url] :as post}]
{:created-at (js/Date. date) {:created-at (js/Date. date)
@ -101,9 +114,9 @@
(-> (.parseURL parser url) (-> (.parseURL parser url)
(.then #(post-items (.then #(post-items
last-post-time last-post-time
(for [{:keys [title isoDate content link]} (-> % js->edn :items)] (for [{:keys [title isoDate pubDate content link]} (-> % js->edn :items)]
{:created-at (js/Date. isoDate) {:created-at (js/Date. (or isoDate pubDate))
:text (str title "\n\n" link)}))))) :text (str (trim-text title) "\n\n" link)})))))
(get-mastodon-timeline (get-mastodon-timeline
(fn [timeline] (fn [timeline]