From 68b82be9593e8c8ddbf22f55688c670bccd8d264 Mon Sep 17 00:00:00 2001 From: Adam Tankanow Date: Mon, 29 Dec 2014 11:15:48 -0500 Subject: [PATCH 1/8] add markup ns to abstract markdown vs asciidoc --- src/cryogen_core/markup.clj | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 src/cryogen_core/markup.clj diff --git a/src/cryogen_core/markup.clj b/src/cryogen_core/markup.clj new file mode 100644 index 0000000..f769ce8 --- /dev/null +++ b/src/cryogen_core/markup.clj @@ -0,0 +1,37 @@ +(ns cryogen-core.markup + (:require [markdown.core :refer [md-to-html-string]] + [clojure.string :as s]) + (:import org.asciidoctor.Asciidoctor$Factory + java.util.Collections)) + +(defprotocol Markup + (dir [this]) + (ext [this]) + (render-fn [this])) + +(defn- markdown [] + (reify Markup + (dir [this] "md") + (ext [this] ".md") + (render-fn [this] + (fn [rdr] + (md-to-html-string + (->> (java.io.BufferedReader. rdr) + (line-seq) + (s/join "\n")) + :heading-anchors true))))) + +(defn- asciidoc [] + (reify Markup + (dir [this] "asc") + (ext [this] ".asc") + (render-fn [this] + (fn [rdr] + (.convert (Asciidoctor$Factory/create) + (->> (java.io.BufferedReader. rdr) + (line-seq) + (s/join "\n")) + (Collections/emptyMap)))))) + +(defn markups [] + [(markdown) (asciidoc)]) From 1813848919627d8aa2b1e393606ea88f3211325c Mon Sep 17 00:00:00 2001 From: Adam Tankanow Date: Tue, 30 Dec 2014 07:29:55 -0500 Subject: [PATCH 2/8] Update version to 0.1.9 and add asciidoctorj dependency. --- project.clj | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/project.clj b/project.clj index 015e2fb..4783506 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject cryogen-core "0.1.8" +(defproject cryogen-core "0.1.9" :description "Cryogen's compiler" :url "https://github.com/lacarmen/cryogen-core" :license {:name "Eclipse Public License" @@ -13,4 +13,5 @@ [hiccup "1.0.5"] [selmer "0.7.8"] [markdown-clj "0.9.60" - :exclusions [com.keminglabs/cljx]]]) + :exclusions [com.keminglabs/cljx]] + [org.asciidoctor/asciidoctorj "1.5.2"]]) From b68779fcbe35dc2f342fc5551d23389ad0a3f3a2 Mon Sep 17 00:00:00 2001 From: Adam Tankanow Date: Tue, 30 Dec 2014 07:31:46 -0500 Subject: [PATCH 3/8] Abstract Markup language details and add Asciidoc support: 1. create markup namespace to hold Markup protocol and its Markdown and Asciidoc implementations. 2. Update compiler ns to iterate through available Markups and pass Markup to fns that need Markup info. 3. Refactor compiler ns parse fns to be more granular. --- src/cryogen_core/compiler.clj | 120 ++++++++++++++++++---------------- src/cryogen_core/markup.clj | 54 +++++++++++---- 2 files changed, 105 insertions(+), 69 deletions(-) diff --git a/src/cryogen_core/compiler.clj b/src/cryogen_core/compiler.clj index b089e29..e2a00dc 100644 --- a/src/cryogen_core/compiler.clj +++ b/src/cryogen_core/compiler.clj @@ -9,10 +9,9 @@ [clojure.java.io :refer [copy file reader writer]] [clojure.string :as s] [text-decoration.core :refer :all] - [markdown.core :refer [md-to-html-string]] - [markdown.transformers :refer [transformer-vector]] [cryogen-core.toc :refer [generate-toc]] - [cryogen-core.sass :as sass])) + [cryogen-core.sass :as sass] + [cryogen-core.markup :as m])) (cache-off!) @@ -31,13 +30,13 @@ (defn find-posts "Returns a list of markdown files representing posts under the post root in templates/md" - [{:keys [post-root ignored-files]}] - (find-assets (str "templates/md" post-root) ".md" ignored-files)) + [{:keys [post-root ignored-files]} mu] + (find-assets (str "templates/" (m/dir mu) post-root) (m/ext mu) ignored-files)) (defn find-pages "Returns a list of markdown files representing pages under the page root in templates/md" - [{:keys [page-root ignored-files]}] - (find-assets (str "templates/md" page-root) ".md" ignored-files)) + [{:keys [page-root ignored-files]} mu] + (find-assets (str "templates/" (m/dir mu) page-root) (m/ext mu) ignored-files)) (defn parse-post-date "Parses the post date from the post's file name and returns the corresponding java date object" @@ -47,13 +46,13 @@ (defn post-uri "Creates a post uri from the post file name" - [file-name {:keys [blog-prefix post-root]}] - (str blog-prefix post-root (s/replace file-name #".md" ".html"))) + [file-name {:keys [blog-prefix post-root]} mu] + (str blog-prefix post-root (s/replace file-name (re-pattern (m/ext mu)) ".html"))) (defn page-uri "Creates a page uri from the page file name" - [page-name {:keys [blog-prefix page-root]}] - (str blog-prefix page-root (s/replace page-name #".md" ".html"))) + [page-name {:keys [blog-prefix page-root]} mu] + (str blog-prefix page-root (s/replace page-name (re-pattern (m/ext mu)) ".html"))) (defn read-page-meta "Returns the clojure map from the top of a markdown page/post" @@ -63,65 +62,76 @@ (catch Exception _ (throw (IllegalArgumentException. (str "Malformed metadata on page: " page)))))) -(defn rewrite-hrefs - "Injects the blog prefix in front of any local links +(defn page-content + "Returns a map with the given page's file-name, metadata and content parsed from + the file with the given markup." + [page config markup] + (with-open [rdr (java.io.PushbackReader. (reader page))] + (let [page-name (.getName page) + file-name (s/replace page-name (re-pattern (m/ext markup)) ".html") + page-meta (read-page-meta page-name rdr) + content ((m/render-fn markup) rdr config)] + {:file-name file-name + :page-meta page-meta + :content content}))) - ex. becomes " - [{:keys [blog-prefix]} text state] - [(clojure.string/replace text #"href=.?/|src=.?/" #(str (subs % 0 (dec (count %))) blog-prefix "/")) - state]) - -(defn parse-content - "Parses the markdown content in a post/page into html" - [rdr config] - (md-to-html-string - (->> (java.io.BufferedReader. rdr) - (line-seq) - (s/join "\n")) - :reference-links? true - :heading-anchors true - :replacement-transformers (conj transformer-vector (partial rewrite-hrefs config)))) +(defn merge-meta-and-content + "Merges the page metadata and content maps, adding :toc if necessary." + [file-name page-meta content] + (merge + (update-in page-meta [:layout] #(str (name %) ".html")) + {:file-name file-name + :content content + :toc (if (:toc page-meta) (generate-toc content))})) (defn parse-page "Parses a page/post and returns a map of the content, uri, date etc." - [is-post? page config] - (with-open [rdr (java.io.PushbackReader. (reader page))] - (let [page-name (.getName page) - file-name (s/replace page-name #".md" ".html") - page-meta (read-page-meta page-name rdr) - content (parse-content rdr config)] - (merge - (update-in page-meta [:layout] #(str (name %) ".html")) - {:file-name file-name - :content content - :toc (if (:toc page-meta) (generate-toc content))} - (if is-post? - (let [date (parse-post-date file-name (:post-date-format config)) - archive-fmt (java.text.SimpleDateFormat. "yyyy MMMM" (java.util.Locale. "en")) - formatted-group (.format archive-fmt date)] - {:date date - :formatted-archive-group formatted-group - :parsed-archive-group (.parse archive-fmt formatted-group) - :uri (post-uri file-name config) - :tags (set (:tags page-meta))}) - {:uri (page-uri file-name config) - :page-index (:page-index page-meta)}))))) + [page config markup] + (let [{:keys [file-name page-meta content]} (page-content page config markup)] + (merge + (merge-meta-and-content file-name page-meta content) + {:uri (page-uri file-name config markup) + :page-index (:page-index page-meta)}))) + +(defn parse-post + "Return a map with the given post's information." + [page config markup] + (let [{:keys [file-name page-meta content]} (page-content page config markup)] + (merge + (merge-meta-and-content file-name page-meta content) + (let [date (parse-post-date file-name (:post-date-format config)) + archive-fmt (java.text.SimpleDateFormat. "yyyy MMMM" (java.util.Locale. "en")) + formatted-group (.format archive-fmt date)] + {:date date + :formatted-archive-group formatted-group + :parsed-archive-group (.parse archive-fmt formatted-group) + :uri (post-uri file-name config markup) + :tags (set (:tags page-meta))}) + ))) (defn read-posts "Returns a sequence of maps representing the data from markdown files of posts. Sorts the sequence by post date." [config] - (->> (find-posts config) - (map #(parse-page true % config)) + (->> (mapcat + (fn [mu] + (->> + (find-posts config mu) + (map #(parse-post % config mu)))) + (m/markups)) (sort-by :date) reverse)) (defn read-pages "Returns a sequence of maps representing the data from markdown files of pages. - Sorts the sequence by post date." + Sorts the sequence by post date." [config] - (->> (find-pages config) - (map #(parse-page false % config)) + (->> (mapcat + (fn [mu] + (->> + (find-pages config mu) + (map #(parse-page % config mu)))) + (m/markups)) (sort-by :page-index))) (defn tag-post diff --git a/src/cryogen_core/markup.clj b/src/cryogen_core/markup.clj index f769ce8..d9f60b6 100644 --- a/src/cryogen_core/markup.clj +++ b/src/cryogen_core/markup.clj @@ -1,37 +1,63 @@ (ns cryogen-core.markup (:require [markdown.core :refer [md-to-html-string]] + [markdown.transformers :refer [transformer-vector]] [clojure.string :as s]) (:import org.asciidoctor.Asciidoctor$Factory java.util.Collections)) (defprotocol Markup + "A markup engine comprising a dir(ectory) containing markup files, + an ext(ension) for finding markup file names, and a render-fn that returns + a fn with the signature [java.io.Reader config] -> String (HTML)." (dir [this]) (ext [this]) (render-fn [this])) -(defn- markdown [] +(defn- rewrite-hrefs + "Injects the blog prefix in front of any local links + + ex. becomes " + [{:keys [blog-prefix]} text state] + [(clojure.string/replace text #"href=.?/|src=.?/" #(str (subs % 0 (dec (count %))) blog-prefix "/")) + state]) + +(defn- markdown + "Returns a Markdown (https://daringfireball.net/projects/markdown/) + implementation of the Markup protocol." + [] (reify Markup (dir [this] "md") (ext [this] ".md") (render-fn [this] - (fn [rdr] + (fn [rdr config] (md-to-html-string - (->> (java.io.BufferedReader. rdr) - (line-seq) - (s/join "\n")) - :heading-anchors true))))) + (->> (java.io.BufferedReader. rdr) + (line-seq) + (s/join "\n")) + :reference-links? true + :heading-anchors true + :replacement-transformers (conj transformer-vector (partial rewrite-hrefs config))))))) -(defn- asciidoc [] +(defn- asciidoc + "Returns an Asciidoc (http://asciidoc.org/) implementation of the + Markup protocol." + [] (reify Markup (dir [this] "asc") (ext [this] ".asc") (render-fn [this] - (fn [rdr] - (.convert (Asciidoctor$Factory/create) - (->> (java.io.BufferedReader. rdr) - (line-seq) - (s/join "\n")) - (Collections/emptyMap)))))) + (let [attributes (java.util.HashMap. {"toc" "macro"}) + options (java.util.HashMap. {"attributes" attributes})] + (fn [rdr _] + (.convert (Asciidoctor$Factory/create) + (->> (java.io.BufferedReader. rdr) + (line-seq) + (s/join "\n")) + options)))))) -(defn markups [] +(defn markups + "Return a vector of Markup implementations. This is the primary entry point + for a client of this ns. This vector should be used to iterate over supported + Markups." + [] [(markdown) (asciidoc)]) From 21d825d0cda617ce7bdfb9c2e64a09978236bf0e Mon Sep 17 00:00:00 2001 From: Adam Tankanow Date: Fri, 9 Jan 2015 07:10:49 -0500 Subject: [PATCH 4/8] fix spacing in markdown method --- src/cryogen_core/markup.clj | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/cryogen_core/markup.clj b/src/cryogen_core/markup.clj index d9f60b6..1bff084 100644 --- a/src/cryogen_core/markup.clj +++ b/src/cryogen_core/markup.clj @@ -31,12 +31,12 @@ (render-fn [this] (fn [rdr config] (md-to-html-string - (->> (java.io.BufferedReader. rdr) - (line-seq) - (s/join "\n")) - :reference-links? true - :heading-anchors true - :replacement-transformers (conj transformer-vector (partial rewrite-hrefs config))))))) + (->> (java.io.BufferedReader. rdr) + (line-seq) + (s/join "\n")) + :reference-links? true + :heading-anchors true + :replacement-transformers (conj transformer-vector (partial rewrite-hrefs-transformer config))))))) (defn- asciidoc "Returns an Asciidoc (http://asciidoc.org/) implementation of the From 1cff8eadf22052f2b3d03c49a9cc1d50f0e21ddb Mon Sep 17 00:00:00 2001 From: Adam Tankanow Date: Fri, 9 Jan 2015 07:11:38 -0500 Subject: [PATCH 5/8] refactor rewrite-hrefs into 2 functions - (1) can be reused by the asciidoc processing, it simply does the text replace - (2) is a transformer using (1) for use in markdown.core --- src/cryogen_core/markup.clj | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/cryogen_core/markup.clj b/src/cryogen_core/markup.clj index 1bff084..7b44ae7 100644 --- a/src/cryogen_core/markup.clj +++ b/src/cryogen_core/markup.clj @@ -17,9 +17,14 @@ "Injects the blog prefix in front of any local links ex. becomes " + [blog-prefix text] + (clojure.string/replace text #"href=.?/|src=.?/" #(str (subs % 0 (dec (count %))) blog-prefix "/"))) + +(defn- rewrite-hrefs-transformer + "A :replacement-transformer for use in markdown.core that will inject the + given blog prefix in front of local links." [{:keys [blog-prefix]} text state] - [(clojure.string/replace text #"href=.?/|src=.?/" #(str (subs % 0 (dec (count %))) blog-prefix "/")) - state]) + [(rewrite-hrefs blog-prefix text) state]) (defn- markdown "Returns a Markdown (https://daringfireball.net/projects/markdown/) From eda815a5aea8293df4cce4f32991d4ea7c1dc448 Mon Sep 17 00:00:00 2001 From: Adam Tankanow Date: Fri, 9 Jan 2015 07:12:56 -0500 Subject: [PATCH 6/8] remove asciidoc options/attributes and add rewrite-href post processing --- src/cryogen_core/markup.clj | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/cryogen_core/markup.clj b/src/cryogen_core/markup.clj index 7b44ae7..71b7b41 100644 --- a/src/cryogen_core/markup.clj +++ b/src/cryogen_core/markup.clj @@ -51,14 +51,14 @@ (dir [this] "asc") (ext [this] ".asc") (render-fn [this] - (let [attributes (java.util.HashMap. {"toc" "macro"}) - options (java.util.HashMap. {"attributes" attributes})] - (fn [rdr _] + (fn [rdr config] + (->> (.convert (Asciidoctor$Factory/create) - (->> (java.io.BufferedReader. rdr) - (line-seq) - (s/join "\n")) - options)))))) + (->> (java.io.BufferedReader. rdr) + (line-seq) + (s/join "\n")) + (Collections/emptyMap)) + (rewrite-hrefs (:blog-prefix config))))))) (defn markups "Return a vector of Markup implementations. This is the primary entry point From b3bad2e279292ab5fd875ddf1c148cc008d39f36 Mon Sep 17 00:00:00 2001 From: Adam Tankanow Date: Fri, 9 Jan 2015 07:13:35 -0500 Subject: [PATCH 7/8] add docstring to get-headings function --- src/cryogen_core/toc.clj | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/cryogen_core/toc.clj b/src/cryogen_core/toc.clj index e44c212..a1021d5 100644 --- a/src/cryogen_core/toc.clj +++ b/src/cryogen_core/toc.clj @@ -5,7 +5,10 @@ (def _h [:h1 :h2 :h3 :h4 :h5 :h6]) (defn- compare_index [i1 i2] (- (.indexOf _h i2) (.indexOf _h i1))) -(defn get-headings [content] +(defn- get-headings + "Turn a body of html content into a vector of elements whose tags are + headings." + [content] (reduce (fn [headings {:keys [tag attrs content] :as elm}] (if (some #{tag} _h) From 5609772cd3b43307d13a1b3a0d5f378fc31f816d Mon Sep 17 00:00:00 2001 From: Adam Tankanow Date: Fri, 9 Jan 2015 07:16:22 -0500 Subject: [PATCH 8/8] Add docstring and support for creating links for header id attributes - this method will now look for both of the following references, this commit adds support for (2) (1)

Reference Text

and (2)

Reference Text

--- src/cryogen_core/toc.clj | 28 +++++++++++++++++++--------- 1 file changed, 19 insertions(+), 9 deletions(-) diff --git a/src/cryogen_core/toc.clj b/src/cryogen_core/toc.clj index a1021d5..871dba4 100644 --- a/src/cryogen_core/toc.clj +++ b/src/cryogen_core/toc.clj @@ -18,16 +18,26 @@ headings))) [] content)) -(defn make-links [headings] +(defn make-links + "Create a table of contents from the given headings. This function will look + for either: + (1) headings with a child anchor with a non-nil name attribute, e.g. +

Reference Title

+ or + (2) headings with an id attribute, e.g.

Reference Title

+ In both cases above, the anchor reference becomes \"#reference\" and the + anchor text is \"Reference Title\"." + [headings] (loop [items headings acc nil _last nil] - (if-let [{tag :tag [{{name :name} :attrs} title] :content} (first items)] - (if (nil? name) (recur (rest items) acc nil) - (let [entry [:li [:a {:href (str "#" name)} title]] - jump (compare_index _last tag)] - (cond (> jump 0) (recur (rest items) (str acc "
    " (hiccup/html entry)) tag) - (= jump 0) (recur (rest items) (str acc (hiccup/html entry)) tag) - (< jump 0) (recur (rest items) (str acc (apply str (repeat (* -1 jump) "
")) - (hiccup/html entry)) tag)))) + (if-let [{tag :tag {id :id} :attrs [{{name :name} :attrs} title :as htext] :content} (first items)] + (let [anchor (or id name)] + (if (nil? anchor) (recur (rest items) acc nil) + (let [entry [:li [:a {:href (str "#" anchor)} (or title (first htext))]] + jump (compare_index _last tag)] + (cond (> jump 0) (recur (rest items) (str acc "
    " (hiccup/html entry)) tag) + (= jump 0) (recur (rest items) (str acc (hiccup/html entry)) tag) + (< jump 0) (recur (rest items) (str acc (apply str (repeat (* -1 jump) "
")) + (hiccup/html entry)) tag))))) (str acc "")))) (defn generate-toc [html]