meissa/cryogen-core
6
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Merge branch 'master' into master

Browse source
This commit is contained in:
M.Jerger 2017-02-08 09:13:45 +01:00 committed by GitHub
commit d36acf2316
14 changed files with 500 additions and 315 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

18
project.clj
View file

@ -1,19 +1,19 @@
(defproject cryogen-core "0.1.47"
(defproject cryogen-core "0.1.52"
:description "Cryogen's compiler"
:url "https://github.com/cryogen-project/cryogen-core"
:license {:name "Eclipse Public License"
:url "http://www.eclipse.org/legal/epl-v10.html"}
:dependencies [[org.clojure/clojure "1.8.0"]
[camel-snake-kebab "0.4.0"]
[cheshire "5.7.0"]
[clj-rss "0.2.3"]
[me.raynes/fs "1.4.6"]
[crouton "0.1.2"]
[cheshire "5.6.3"]
[clj-text-decoration "0.0.3"]
[io.aviso/pretty "0.1.33"]
[hiccup "1.0.5"]
[selmer "1.10.3"]
[pandect "0.6.1"]
[enlive "1.1.6"]
[hawk "0.2.11"]
[clj-tagsoup "0.3.0" :exclusions [org.clojure/clojure]]]
[hiccup "1.0.5"]
[io.aviso/pretty "0.1.33"]
[me.raynes/fs "1.4.6"]
[pandect "0.6.1"]
[selmer "1.10.6"]]
:deploy-repositories [["snapshots" :clojars]
["releases" :clojars]])

316
src/cryogen_core/compiler.clj
View file

@ -1,21 +1,20 @@
(ns cryogen-core.compiler
(:require [selmer.parser :refer [cache-off! render-file]]
[selmer.util :refer [set-custom-resource-path!]]
[io.aviso.exception :refer [write-exception]]
[clojure.java.io :refer [copy file reader writer]]
(:require [clojure.java.io :as io]
[clojure.pprint :refer [pprint]]
[clojure.string :as s]
[io.aviso.exception :refer [write-exception]]
[net.cgrand.enlive-html :as enlive]
[selmer.parser :refer [cache-off! render-file]]
[selmer.util :refer [set-custom-resource-path!]]
[text-decoration.core :refer :all]
[pl.danieljanus.tagsoup :as tagsoup]
[hiccup.core :as hiccup]
[cryogen-core.toc :refer [generate-toc]]
[cryogen-core.sass :as sass]
[cryogen-core.io :as cryogen-io]
[cryogen-core.klipse :as klipse]
[cryogen-core.markup :as m]
[cryogen-core.io :refer
[get-resource find-assets create-folder create-file-recursive create-file wipe-public-folder
copy-resources copy-resources-from-theme path]]
[cryogen-core.sitemap :as sitemap]
[cryogen-core.rss :as rss]
[clojure.inspector :as inspector])
[cryogen-core.sass :as sass]
[cryogen-core.sitemap :as sitemap]
[clojure.inspector :as inspector]
[cryogen-core.toc :as toc])
(:import java.util.Locale))
(cache-off!)
@ -38,14 +37,16 @@
looking under the implemented protocol's subdirectory, but fallsback to look
at the templates directory."
[root mu ignored-files]
(let [assets (find-assets (path "templates" (m/dir mu) root)
(m/ext mu)
ignored-files)]
(let [assets (cryogen-io/find-assets
(cryogen-io/path "templates" (m/dir mu) root)
(m/ext mu)
ignored-files)]
(if (seq assets)
assets
(find-assets (path "templates" root)
(m/ext mu)
ignored-files))))
(cryogen-io/find-assets
(cryogen-io/path "templates" root)
(m/ext mu)
ignored-files))))
(defn find-posts
"Returns a list of markdown files representing posts under the post root."
@ -68,9 +69,9 @@
([file-name params]
(page-uri file-name nil params))
([file-name uri-type {:keys [blog-prefix clean-urls?] :as params}]
(let [page-uri (params uri-type)
uri-end (if clean-urls? (s/replace file-name #"(index)?\.html" "/") file-name)]
(path "/" blog-prefix page-uri uri-end))))
(let [page-uri (get params uri-type)
uri-end (if clean-urls? (s/replace file-name #"(index)?\.html" "/") file-name)]
(cryogen-io/path "/" blog-prefix page-uri uri-end))))
(defn read-page-meta
"Returns the clojure map from the top of a markdown page/post"
@ -84,13 +85,13 @@
"Returns a map with the given page's file-name, metadata and content parsed from
the file with the given markup."
[^java.io.File page config markup]
(with-open [rdr (java.io.PushbackReader. (reader page))]
(let [re-root (re-pattern (str "^.*?(" (:page-root config) "|" (:post-root config) ")/"))
page-fwd (s/replace (str page) "\\" "/") ;; make it work on Windows
(with-open [rdr (java.io.PushbackReader. (io/reader page))]
(let [re-root (re-pattern (str "^.*?(" (:page-root config) "|" (:post-root config) ")/"))
page-fwd (s/replace (str page) "\\" "/") ;; make it work on Windows
page-name (s/replace page-fwd re-root "")
file-name (s/replace page-name (re-pattern-from-ext (m/ext markup)) ".html")
page-meta (read-page-meta page-name rdr)
content ((m/render-fn markup) rdr config)]
content ((m/render-fn markup) rdr config)]
{:file-name file-name
:page-meta page-meta
:content content})))
@ -103,7 +104,7 @@
{:file-name file-name
:content content
:toc (if-let [toc (:toc page-meta)]
(generate-toc content :list-type toc))}))
(toc/generate-toc content :list-type toc))}))
(defn parse-page
"Parses a page/post and returns a map of the content, uri, date etc."
@ -112,7 +113,8 @@
(merge
(merge-meta-and-content file-name page-meta content)
{:uri (page-uri file-name :page-root-uri config)
:page-index (:page-index page-meta)})))
:page-index (:page-index page-meta)
:klipse (klipse/merge-configs (:klipse config) (:klipse page-meta))})))
(defn parse-post
"Return a map with the given post's information."
@ -120,28 +122,29 @@
(let [{:keys [file-name page-meta content]} (page-content page config markup)]
(merge
(merge-meta-and-content file-name page-meta content)
(let [date (if (:date page-meta)
(.parse (java.text.SimpleDateFormat. (:post-date-format config)) (:date page-meta))
(parse-post-date file-name (:post-date-format config)))
archive-fmt (java.text.SimpleDateFormat. (get config :archive-group-format "yyyy MMMM") (Locale/getDefault))
(let [date (if (:date page-meta)
(.parse (java.text.SimpleDateFormat. (:post-date-format config)) (:date page-meta))
(parse-post-date file-name (:post-date-format config)))
archive-fmt (java.text.SimpleDateFormat. (:archive-group-format config "yyyy MMMM") (Locale/getDefault))
formatted-group (.format archive-fmt date)]
{:date date
:formatted-archive-group formatted-group
:parsed-archive-group (.parse archive-fmt formatted-group)
:uri (page-uri file-name :post-root-uri config)
:tags (set (:tags page-meta))}))))
:tags (set (:tags page-meta))
:klipse (klipse/merge-configs (:klipse config) (:klipse page-meta))}))))
(defn read-posts
"Returns a sequence of maps representing the data from markdown files of posts.
Sorts the sequence by post date."
[config]
(->> (mapcat
(->> (m/markups)
(mapcat
(fn [mu]
(->>
(find-posts config mu)
(pmap #(parse-post % config mu))
(remove #(= (:draft? %) true))))
(m/markups))
(remove #(= (:draft? %) true)))))
(sort-by :date)
reverse
(drop-while #(and (:hide-future-posts? config) (.after (:date %) (java.util.Date.))))))
@ -150,12 +153,12 @@
"Returns a sequence of maps representing the data from markdown files of pages.
Sorts the sequence by post date."
[config]
(->> (mapcat
(->> (m/markups)
(mapcat
(fn [mu]
(->>
(find-pages config mu)
(map #(parse-page % config mu))))
(m/markups))
(map #(parse-page % config mu)))))
(sort-by :page-index)))
(defn tag-post
@ -163,7 +166,8 @@
[tags post]
(reduce (fn [tags tag]
(update-in tags [tag] (fnil conj []) (select-keys post [:uri :title :content :date :enclosure])))
tags (:tags post)))
tags
(:tags post)))
(defn group-by-tags
"Maps all the tags with a list of posts that contain each tag"
@ -201,13 +205,13 @@
:uri (page-uri (str (name tag) ".html") :tag-root-uri config)})
(defn add-prev-next
"Adds a :prev and :next key to the page/post data containing the title and uri of the prev/next
"Adds a :prev and :next key to the page/post data containing the metadata of the prev/next
post/page if it exists"
[pages]
(map (fn [[prev target next]]
(assoc target
:prev (if prev (select-keys prev [:title :uri]) nil)
:next (if next (select-keys next [:title :uri]) nil)))
:prev (if prev (dissoc prev :content) nil)
:next (if next (dissoc next :content) nil)))
(partition 3 1 (flatten [nil pages nil]))))
(defn group-pages
@ -252,26 +256,30 @@
"When `clean-urls?` is set, appends `/index.html` before spit; otherwise just spits."
[file-uri {:keys [clean-urls?]} data]
(if clean-urls?
(create-file-recursive (path file-uri "index.html") data)
(create-file file-uri data)))
(cryogen-io/create-file-recursive (cryogen-io/path file-uri "index.html") data)
(cryogen-io/create-file file-uri data)))
(defn- print-debug-info [data]
(println "DEBUG:")
(pprint data))
(defn compile-pages
"Compiles all the pages into html and spits them out into the public folder"
[{:keys [blog-prefix page-root-uri debug?] :as params} pages]
(when-not (empty? pages)
(println (blue "compiling pages"))
(create-folder (path "/" blog-prefix page-root-uri))
(cryogen-io/create-folder (cryogen-io/path "/" blog-prefix page-root-uri))
(doseq [{:keys [uri] :as page} pages]
(println "\t-->" (cyan uri))
(println "-->" (cyan uri))
(when debug?
(println "\t-->" (cyan page)))
(print-debug-info page))
(write-html uri
params
(render-file (str "/html/" (:layout page))
(merge params
{:active-page "pages"
:home false
:servlet-context (path "/" blog-prefix "/")
:servlet-context (cryogen-io/path "/" blog-prefix "/")
:page page
:uri uri}))))))
@ -280,74 +288,75 @@
[{:keys [blog-prefix post-root-uri disqus-shortname debug?] :as params} posts]
(when-not (empty? posts)
(println (blue "compiling posts"))
(create-folder (path "/" blog-prefix post-root-uri))
(doseq [post posts]
(println "\t-->" (cyan (:uri post)))
(println "\t-->" (cyan debug?))
(cryogen-io/create-folder (cryogen-io/path "/" blog-prefix post-root-uri))
(doseq [{:keys [uri] :as post} posts]
(println "-->" (cyan uri))
(when debug?
(println "\t-->" (cyan post)))
(write-html (:uri post)
(print-debug-info post))
(write-html uri
params
(render-file (str "/html/" (:layout post))
(merge params
{:active-page "posts"
:servlet-context (path "/" blog-prefix "/")
:servlet-context (cryogen-io/path "/" blog-prefix "/")
:post post
:disqus-shortname disqus-shortname
:uri (:uri post)}))))))
:uri uri}))))))
(defn compile-tags
"Compiles all the tag pages into html and spits them out into the public folder"
[{:keys [blog-prefix tag-root-uri] :as params} posts-by-tag]
(when-not (empty? posts-by-tag)
(println (blue "compiling tags"))
(create-folder (path "/" blog-prefix tag-root-uri))
(cryogen-io/create-folder (cryogen-io/path "/" blog-prefix tag-root-uri))
(doseq [[tag posts] posts-by-tag]
(let [{:keys [name uri]} (tag-info params tag)]
(println "\t-->" (cyan uri))
(println "-->" (cyan uri))
(write-html uri
params
(render-file "/html/tag.html"
(merge params
{:active-page "tags"
:servlet-context (path "/" blog-prefix "/")
:servlet-context (cryogen-io/path "/" blog-prefix "/")
:name name
:posts posts
:uri uri})))))))
(defn compile-tags-page [{:keys [blog-prefix] :as params}]
"Compiles a page with links to each tag page. Spits the page into the public folder"
(println (blue "compiling tags page"))
(let [uri (page-uri "tags.html" params)]
(write-html uri
params
(render-file "/html/tags.html"
(merge params
{:active-page "tags"
:servlet-context (path "/" blog-prefix "/")
:uri uri})))))
{:active-page "tags"
:servlet-context (cryogen-io/path "/" blog-prefix "/")
:uri uri})))))
(defn content-until-more-marker
[^String content]
(let [index (.indexOf content "<!--more-->")]
(if (pos? index)
(let [s (subs content 0 index)]
(->> ((tagsoup/parse-string s) 2)
(drop 2)
hiccup/html)))))
"Returns the content until the <!--more--> special comment,
closing any unclosed tags. Returns nil if there's no such comment."
[content]
(when-let [index (s/index-of content "<!--more-->")]
(->> (subs content 0 index)
enlive/html-snippet
enlive/emit*
(apply str))))
(defn create-preview
"Creates a single post preview"
[blocks-per-preview post]
(merge post
{:content (or (content-until-more-marker (:content post))
(->> ((tagsoup/parse-string (:content post)) 2)
(drop 2)
(take blocks-per-preview)
hiccup/html))}))
(update post :content
#(or (content-until-more-marker %)
(->> (enlive/html-snippet %)
(take blocks-per-preview)
enlive/emit*
(apply str)))))
(defn create-previews
"Returns a sequence of vectors, each containing a set of post previews"
[posts-per-page blocks-per-preview posts]
[posts posts-per-page blocks-per-preview]
(->> posts
(map #(create-preview blocks-per-preview %))
(partition-all posts-per-page)
@ -359,51 +368,53 @@
[previews params]
(mapv (fn [[prev target next]]
(merge target
{:prev (if prev (page-uri (path "p" (str (:index prev) ".html")) params) nil)
:next (if next (page-uri (path "p" (str (:index next) ".html")) params) nil)}))
{:prev (if prev (page-uri (cryogen-io/path "p" (str (:index prev) ".html")) params) nil)
:next (if next (page-uri (cryogen-io/path "p" (str (:index next) ".html")) params) nil)}))
(partition 3 1 (flatten [nil previews nil]))))
(defn compile-preview-pages
"Compiles a series of pages containing 'previews' from each post"
[{:keys [blog-prefix posts-per-page blocks-per-preview] :as params} posts]
(when-not (empty? posts)
(let [previews (-> (create-previews posts-per-page blocks-per-preview posts)
(let [previews (-> posts
(create-previews posts-per-page blocks-per-preview)
(create-preview-links params))
previews (if (> (count previews) 1) (assoc-in previews [1 :prev] (page-uri "index.html" params)) previews)]
(create-folder (path "/" blog-prefix "p"))
previews (if (> (count previews) 1)
(assoc-in previews [1 :prev] (page-uri "index.html" params))
previews)]
(cryogen-io/create-folder (cryogen-io/path "/" blog-prefix "p"))
(doseq [{:keys [index posts prev next]} previews
:let [index-page? (= 1 index)]]
(write-html (if index-page? (page-uri "index.html" params) (page-uri (path "p" (str index ".html")) params))
params
(render-file "/html/previews.html"
(merge params
{:active-page "preview"
:home (when index-page? true)
:servlet-context (path "/" blog-prefix "/")
:posts posts
:prev-uri prev
:next-uri next})))))))
(write-html
(if index-page? (page-uri "index.html" params)
(page-uri (cryogen-io/path "p" (str index ".html")) params))
params
(render-file "/html/previews.html"
(merge params
{:active-page "preview"
:home (when index-page? true)
:servlet-context (cryogen-io/path "/" blog-prefix "/")
:posts posts
:prev-uri prev
:next-uri next})))))))
(defn compile-index
"Compiles the index page into html and spits it out into the public folder"
[{:keys [disqus?] :as params}]
[{:keys [disqus? debug? home-page] :as params}]
(println (blue "compiling index"))
(let [uri (page-uri "index.html" params)
debug? (-> params :debug?)
home-page (-> params :home-page)
meta {:active-page "home"
:home true
:disqus? disqus?
:uri uri
:post home-page
:page home-page}]
(let [uri (page-uri "index.html" params)]
(when debug?
(println "\t-->" (cyan meta)))
(print-debug-info meta))
(write-html uri
params
(render-file (str "/html/" (:layout home-page))
(merge params
meta)))))
{:active-page "home"
:home true
:disqus? disqus?
:uri uri
:post home-page
:page home-page})))))
(defn compile-archives
"Compiles the archives page into html and spits it out into the public folder"
@ -414,28 +425,28 @@
params
(render-file "/html/archives.html"
(merge params
{:active-page "archives"
:archives true
:groups (group-for-archive posts)
:servlet-context (path "/" blog-prefix "/")
:uri uri})))))
{:active-page "archives"
:archives true
:groups (group-for-archive posts)
:servlet-context (cryogen-io/path "/" blog-prefix "/")
:uri uri})))))
(defn compile-authors
"For each author, creates a page with filtered posts."
[{:keys [blog-prefix author-root-uri author] :as params} posts]
(println (blue "compiling authors"))
(create-folder (path "/" blog-prefix author-root-uri))
(cryogen-io/create-folder (cryogen-io/path "/" blog-prefix author-root-uri))
;; if the post author is empty defaults to config's :author
(doseq [{:keys [author posts]} (group-for-author posts author)]
(let [uri (page-uri (str author ".html") :author-root-uri params)]
(println "\t-->" (cyan uri))
(println "-->" (cyan uri))
(write-html uri
params
(render-file "/html/author.html"
(merge params
{:author author
:groups (group-for-archive posts)
:servlet-context (path "/" blog-prefix "/")
:servlet-context (cryogen-io/path "/" blog-prefix "/")
:uri uri}))))))
(defn tag-posts
@ -446,11 +457,11 @@
(defn- template-dir?
"Checks that the dir exists in the templates directory."
[dir]
(.isDirectory (file (str "resources/templates/" dir))))
(.isDirectory (io/file (str "resources/templates/" dir))))
(defn- markup-entries [post-root page-root]
(let [entries (for [mu (m/markups)
t (distinct [post-root page-root])]
t (distinct [post-root page-root])]
[(str (m/dir mu) "/" t) t])]
(apply concat entries)))
@ -459,7 +470,7 @@
[{:keys [post-root page-root] :as config}]
(let [folders (->> (markup-entries post-root page-root)
(filter template-dir?))]
(copy-resources
(cryogen-io/copy-resources
(merge config
{:resources folders
:ignored-files (map #(re-pattern-from-ext (m/ext %)) (m/markups))}))))
@ -469,7 +480,7 @@
[]
(try
(let [config (-> "templates/config.edn"
get-resource
cryogen-io/get-resource
slurp
read-string
(update-in [:blog-prefix] (fnil str ""))
@ -492,20 +503,35 @@
(catch Exception _
(throw (IllegalArgumentException. "Failed to parse config.edn")))))
(defn klipsify
"Add the klipse html under the :klipse key and adds nohighlight
classes to any code blocks that are to be klipsified. Expects
configuration to be under :klipse, if there's none it does nothing."
[{:keys [klipse content] :as post-or-page}]
(-> post-or-page
(update :klipse klipse/emit content)
(update :content klipse/tag-nohighlight (:settings klipse))))
(defn compile-assets
"Generates all the html and copies over resources specified in the config"
[]
(println (green "compiling assets..."))
(let [{:keys [^String site-url blog-prefix rss-name recent-posts sass-src sass-dest sass-path compass-path keep-files ignored-files previews? clean-urls? debug? author-root-uri] :as config} (read-config)
posts (add-prev-next (read-posts config))
pages (add-prev-next (read-pages config))
home-pages (filter #(boolean (:home? %)) pages)
pages-without-home (filter #(boolean (not (:home? %))) pages)
[navbar-pages sidebar-pages] (group-pages pages-without-home)
navmap-pages (build-nav-map pages-without-home)
(let [{:keys [^String site-url blog-prefix rss-name recent-posts sass-dest keep-files ignored-files previews? author-root-uri theme]
:as config} (read-config)
posts (map klipsify (add-prev-next (read-posts config)))
posts-by-tag (group-by-tags posts)
posts (tag-posts posts config)
posts (tag-posts posts config)
latest-posts (->> posts (take recent-posts) vec)
pages (map klipsify (read-pages config))
home-page (->> pages
(filter #(boolean (:home? %)))
(first))
other-pages (->> pages
(remove #{home-page})
(add-prev-next))
[navbar-pages
sidebar-pages] (group-pages other-pages)
navmap-pages (build-nav-map other-pages)
params (merge config
{:today (java.util.Date.)
:title (:site-title config)
@ -515,55 +541,51 @@
:navbar-pages navbar-pages
:navmap-pages navmap-pages
:sidebar-pages sidebar-pages
:home-page (if (not-empty home-pages)
(first home-pages)
(merge (first latest-posts)
{:layout "home.html"}))
:home-page (if home-page
home-page
(assoc (first latest-posts) :layout "home.html"))
:archives-uri (page-uri "archives.html" config)
:index-uri (page-uri "index.html" config)
:tags-uri (page-uri "tags.html" config)
:rss-uri (path "/" blog-prefix rss-name)
:site-url (if (.endsWith site-url "/") (.substring site-url 0 (dec (count site-url))) site-url)
:theme-path (str "file:resources/templates/themes/" (:theme config))})]
:rss-uri (cryogen-io/path "/" blog-prefix rss-name)
:site-url (if (.endsWith site-url "/") (.substring site-url 0 (dec (count site-url))) site-url)})]
(when debug?
(println (blue "debug: navbar-pages:"))
(println "\t-->" (cyan navbar-pages))
(println (blue "debug: navmap-pages:"))
(println "\t-->" (cyan navmap-pages))
)
(set-custom-resource-path! (:theme-path params))
(wipe-public-folder keep-files)
(set-custom-resource-path! (str "file:resources/templates/themes/" theme))
(cryogen-io/wipe-public-folder keep-files)
(println (blue "copying theme resources"))
(copy-resources-from-theme config)
(cryogen-io/copy-resources-from-theme config)
(println (blue "copying resources"))
(copy-resources config)
(cryogen-io/copy-resources config)
(copy-resources-from-markup-folders config)
(compile-pages params pages-without-home)
(compile-pages params other-pages)
(compile-posts params posts)
(compile-tags params posts-by-tag)
(compile-tags-page params)
(when previews?
(compile-preview-pages params posts))
(when (or (not-empty home-pages) (not previews?))
(if previews?
(compile-preview-pages params posts)
(compile-index params))
(compile-archives params posts)
(when author-root-uri
(println (blue "generating authors views"))
(compile-authors params posts))
(println (blue "generating site map"))
(create-file (path "/" blog-prefix "sitemap.xml") (sitemap/generate site-url ignored-files))
(->> (sitemap/generate site-url ignored-files)
(cryogen-io/create-file (cryogen-io/path "/" blog-prefix "sitemap.xml")))
(println (blue "generating main rss"))
(create-file (path "/" blog-prefix rss-name) (rss/make-channel config posts))
(->> (rss/make-channel config posts)
(cryogen-io/create-file (cryogen-io/path "/" blog-prefix rss-name)))
(println (blue "generating filtered rss"))
(rss/make-filtered-channels config posts-by-tag)
(println (blue "compiling sass"))
(sass/compile-sass->css!
{:path-sass sass-path
:path-compass compass-path
:src-sass sass-src
:dest-sass (path ".." "public" blog-prefix sass-dest)
:ignored-files ignored-files
:base-dir "resources/templates/"})))
(merge (select-keys config [:sass-path :compass-path :sass-src :ignored-files])
{:sass-dest (cryogen-io/path ".." "public" blog-prefix sass-dest)
:base-dir "resources/templates/"}))))
(defn compile-assets-timed []
(time

37
src/cryogen_core/github.clj
View file

@ -1,37 +0,0 @@
(ns cryogen-core.github
(:require [cheshire.core :as json])
(:import (org.apache.commons.codec.binary Base64 StringUtils)))
(defn get-gist [gist-uri]
(let [gist-id (last (clojure.string/split gist-uri #"/+")) ;;just need id for git api
gist-resp (try (slurp (str "https://api.github.com/gists/" gist-id))
(catch Exception e {:error (.getMessage e)}))]
(when-not (:error gist-resp)
(if-let [gist (-> (json/parse-string gist-resp)
(get "files")
first ;;todo: optionally get all gist files?
val)]
{:content (get gist "content")
:language (get gist "language")
:name (get gist "filename")
:id gist-id}))))
(defn get-src [git-file]
(let [git-re (re-find #"github.com/(.*)/blob/(.+?)/(.+)" git-file) ;;want second and last now (user/repo,file) for git api
git-res (str "https://api.github.com/repos/" (second git-re) "/contents/" (last git-re))
git-resp (try (slurp git-res)
(catch Exception e {:error (.getMessage e)}))]
(when-not (:error git-resp)
(if-let [git-src (json/parse-string git-resp)]
{:content (String. ^bytes (Base64/decodeBase64 ^String (get git-src "content")) "UTF-8")
:name (get git-src "name")
:uri (get (get git-src "_links") "html")}))))
(defn get-gits-ex []
[(get-gist "https://gist.github.com/viperscape/cec68f0791687f5959f1")
(get-src "https://github.com/viperscape/kuroshio/blob/master/examples/pubsub.clj")])
;(prn (get-gits-ex))

149
src/cryogen_core/klipse.clj Normal file
View file

@ -0,0 +1,149 @@
(ns cryogen-core.klipse
(:require [clojure.string :as str]
[camel-snake-kebab.core :refer [->snake_case_string ->camelCaseString]]
[cheshire.core :as json]
[net.cgrand.enlive-html :as enlive]))
;;;;;;;;;;;
;; utils
(defn map-keys
"Applies f to each key in m"
[f m]
(zipmap (map f (keys m)) (vals m)))
(defn update-existing
"Like clojure.core/update, but returns m untouched if it doesn't contain k"
[m k f & args]
(if (contains? m k) (apply update m k f args) m))
(def map-or-nil? (some-fn map? nil?))
(defn deep-merge
"Like clojure.core/merge, but also merges nested maps under the same key."
[& ms]
(apply merge-with
(fn [v1 v2]
(if (and (map-or-nil? v1) (map-or-nil? v2))
(deep-merge v1 v2)
v2))
ms))
(defn filter-html-elems
"Recursively walks a sequence of enlive-style html elements depth first
and returns a flat sequence of the elements where (pred elem)"
[pred html-elems]
(reduce (fn [acc {:keys [content] :as elem}]
(into (if (pred elem) (conj acc elem) acc)
(filter-html-elems pred content)))
[] html-elems))
(defn code-block-classes
"Takes a string of html and returns a sequence of
all the classes on all code blocks."
[html]
(->> html
enlive/html-snippet
(filter-html-elems (comp #{:code} :tag))
(keep (comp :class :attrs))
(mapcat #(str/split % #" "))))
;;;;;;;;;;;;
;; klipse
(defn eval-classes
"Takes the :settings map and returns all values that are css class selectors."
[settings]
(filter #(str/starts-with? % ".") (vals settings)))
(defn tag-nohighlight
"Takes html as a string and a coll of class-selectors and adds
nohighlight to all code blocks that includes one of them."
[html settings]
(letfn [(tag [h clas]
(enlive/sniptest h
[(keyword (str "code" clas))]
(fn [x]
(update-in x [:attrs :class] #(str % " nohighlight")))))]
(reduce tag html (eval-classes settings))))
(def defaults
{:js-src
{:min "https://storage.googleapis.com/app.klipse.tech/plugin_prod/js/klipse_plugin.min.js"
:non-min "https://storage.googleapis.com/app.klipse.tech/plugin/js/klipse_plugin.js"}
:css-base "https://storage.googleapis.com/app.klipse.tech/css/codemirror.css"})
;; This needs to be updated whenever a new clojure selector is introduced.
;; It should only be necessary for react wrappers and the like, so not very often.
;; When (if?) self hosted cljs becomes compatible with advanced builds
;; this can be removed and we can just always use minified js.
(def clojure-selectors
"A set of selectors that imply clojure evaluation."
#{"selector" "selector_reagent"})
(defn clojure-eval-classes
"Takes settings and returns a set of the html classes that imply clojure eval."
[normalized-settings]
(reduce (fn [classes selector]
(if-let [klass (get normalized-settings selector)]
(conj classes (->> klass rest (apply str))) ;; Strip the leading .
classes))
#{} clojure-selectors))
(defn clojure-eval?
"Takes settings and html and returns whether there is any clojure eval."
[normalized-settings html]
(boolean (some (clojure-eval-classes normalized-settings) (code-block-classes html))))
(defn normalize-settings
"Transform the keys to the correct snake-case or camelCase strings."
[settings]
(-> (map-keys ->snake_case_string settings)
(update-existing "codemirror_options_in" (partial map-keys ->camelCaseString))
(update-existing "codemirror_options_out" (partial map-keys ->camelCaseString))))
(defn merge-configs
"Merges the defaults, global config and post config,
transforms lisp-case keywords into snake_case/camelCase strings
Returns nil if there's no post-config.
A post-config with the value true counts as an empty map."
[global-config post-config]
(when post-config
(let [post-config (if (true? post-config) {} post-config)]
(deep-merge defaults
(update-existing global-config :settings normalize-settings)
(update-existing post-config :settings normalize-settings)))))
(defn infer-clojure-eval
"Infers whether there's clojure eval and returns the config with the
appropriate value assoc'd to :js.
Returns the config untouched if :js is already specified."
[config html]
(if (:js config)
config
(assoc config :js
(if (clojure-eval? (:settings config) html) :non-min :min))))
(defn include-css [href]
(str "<link rel=\"stylesheet\" type=\"text/css\" href=" (pr-str href) ">"))
(defn include-js [src]
(str "<script src=" (pr-str src) "></script>"))
(defn emit
"Takes the :klipse config from config.edn and the :klipse config from the
current post, and returns the html to include on the bottom of the page."
[config html]
(when-let [{:keys [settings js-src js css-base css-theme]}
(infer-clojure-eval config html)]
(assert (#{:min :non-min} js)
(str ":js needs to be one of :min or :non-min but was: " js))
(str (include-css css-base) "\n"
(when css-theme (str (include-css css-theme) "\n"))
"<script>\n"
"window.klipse_settings = " (json/generate-string settings {:pretty true}) ";\n"
"</script>\n"
(include-js (js js-src)))))

5
src/cryogen_core/markup.clj
View file

@ -1,6 +1,5 @@
(ns cryogen-core.markup
(:require [clojure.string :as s])
(:import java.util.Collections))
(:require [clojure.string :as s]))
(defonce markup-registry (atom []))
@ -18,7 +17,7 @@
[blog-prefix text]
(if (s/blank? blog-prefix)
text
(clojure.string/replace text #"href=.?/|src=.?/" #(str (subs % 0 (dec (count %))) blog-prefix "/"))))
(s/replace text #"href=.?/|src=.?/" #(str (subs % 0 (dec (count %))) blog-prefix "/"))))
(defn markups
"Return a vector of Markup implementations. This is the primary entry point

3
src/cryogen_core/plugins.clj
View file

@ -1,6 +1,5 @@
(ns cryogen-core.plugins
(:require [cryogen-core.compiler :refer [compile-assets-timed]]
[clojure.edn :as edn]
(:require [clojure.edn :as edn]
[clojure.string :as s]
[text-decoration.core :refer :all]))

6
src/cryogen_core/rss.clj
View file

@ -1,7 +1,7 @@
(ns cryogen-core.rss
(:require [clj-rss.core :as rss]
[text-decoration.core :refer :all]
[cryogen-core.io :refer [create-file path]])
[cryogen-core.io :as cryogen-io])
(:import java.util.Date))
@ -31,6 +31,6 @@
(defn make-filtered-channels [{:keys [rss-filters blog-prefix] :as config} posts-by-tag]
(doseq [filter rss-filters]
(let [uri (path "/" blog-prefix (str (name filter) ".xml"))]
(let [uri (cryogen-io/path "/" blog-prefix (str (name filter) ".xml"))]
(println "\t-->" (cyan uri))
(create-file uri (make-channel config (get posts-by-tag filter))))))
(cryogen-io/create-file uri (make-channel config (get posts-by-tag filter))))))

63
src/cryogen_core/sass.clj
View file

@ -1,8 +1,8 @@
(ns cryogen-core.sass
(:require [clojure.java.shell :as shell]
[clojure.java.io :as io]
(:require [clojure.java.io :as io]
[clojure.java.shell :as shell]
[text-decoration.core :refer :all]
[cryogen-core.io :refer [ignore match-re-filter]]))
[cryogen-core.io :as cryogen-io]))
(defmacro sh
[& args]
@ -11,64 +11,51 @@
(defn sass-installed?
"Checks for the installation of Sass."
[path-sass]
(= 0 (:exit (sh path-sass "--version"))))
[sass-path]
(zero? (:exit (sh sass-path "--version"))))
(defn compass-installed?
"Checks for the installation of Compass."
[path-compass]
[compass-path]
(try
(= 0 (:exit (sh path-compass "--version")))
(zero? (:exit (sh compass-path "--version")))
(catch java.io.IOException _
false)))
(defn find-sass-files
"Given a Diretory, gets files, Filtered to those having scss or sass
extention. Ignores files matching any ignored regexps."
extention. Ignores files matching any ignored regexps."
[base-dir dir ignored-files]
(let [^java.io.FilenameFilter filename-filter (match-re-filter #"(?i:s[ca]ss$)")]
(let [^java.io.FilenameFilter filename-filter (cryogen-io/match-re-filter #"(?i:s[ca]ss$)")]
(->> (.listFiles (io/file base-dir dir) filename-filter)
(filter #(not (.isDirectory ^java.io.File %)))
(filter (ignore ignored-files))
(filter (cryogen-io/ignore ignored-files))
(map #(.getName ^java.io.File %)))))
(defn compile-sass-file!
"Given a sass file which might be in src-sass directory,
output the resulting css in dest-sass. All error handling is
done by sh / launching the sass command."
[{:keys [src-sass
dest-sass
path-sass
path-compass
base-dir]}]
"Given a sass file which might be in sass-src directory,
output the resulting css in sass-dest. All error handling is
done by sh / launching the sass command."
[{:keys [sass-src sass-dest sass-path compass-path base-dir]}]
(shell/with-sh-dir base-dir
(if (compass-installed? path-compass)
(sh path-sass "--compass" "--update" (str src-sass ":" dest-sass))
(sh path-sass "--update" (str src-sass ":" dest-sass)))))
(if (compass-installed? compass-path)
(sh sass-path "--compass" "--update" (str sass-src ":" sass-dest))
(sh sass-path "--update" (str sass-src ":" sass-dest)))))
(defn compile-sass->css!
"Given a directory src-sass, looks for all sass files and compiles them into
dest-sass. Prompts you to install sass if he finds sass files and can't find
the command. Shows you any problems it comes across when compiling. "
[{:keys [src-sass
dest-sass
path-sass
ignored-files
base-dir] :as opts}]
(when-let [sass-files (seq (find-sass-files base-dir src-sass ignored-files))]
(if (sass-installed? path-sass)
;; I found sass files,
;; If sass is installed
"Given a directory sass-src, looks for all sass files and compiles them into
sass-dest. Prompts you to install sass if he finds sass files and can't find
the command. Shows you any problems it comes across when compiling. "
[{:keys [sass-src sass-dest sass-path ignored-files base-dir] :as opts}]
(when (seq (find-sass-files base-dir sass-src ignored-files))
(if (sass-installed? sass-path)
(do
(println "\t" (cyan src-sass) "-->" (cyan dest-sass))
(println "\t" (cyan sass-src) "-->" (cyan sass-dest))
(let [result (compile-sass-file! opts)]
(if (zero? (:exit result))
;; no problems in sass compilation
(println "Successfully compiled sass files")
;; else I show the error
(println (red (:err result))
(red (:out result))))))
;; Else I prompt to install Sass
(println "Sass seems not to be installed, but you have scss / sass files in "
src-sass
sass-src
" - You might want to install it here: sass-lang.com"))))

4
src/cryogen_core/sitemap.clj
View file

@ -1,6 +1,6 @@
(ns cryogen-core.sitemap
(:require [clojure.xml :refer [emit]]
[cryogen-core.io :refer [get-resource find-assets]])
[cryogen-core.io :as cryogen-io])
(:import java.util.Date))
;;generate sitemaps using the sitemap spec
@ -19,7 +19,7 @@
{:tag :urlset
:attrs {:xmlns "http://www.sitemaps.org/schemas/sitemap/0.9"}
:content
(for [^java.io.File f (find-assets "public" ".html" ignored-files)]
(for [^java.io.File f (cryogen-io/find-assets "public" ".html" ignored-files)]
{:tag :url
:content
[{:tag :loc

9
src/cryogen_core/toc.clj
View file

@ -1,6 +1,6 @@
(ns cryogen-core.toc
(:require [clojure.zip :as z]
[crouton.html :as html]
[net.cgrand.enlive-html :as enlive]
[hiccup.core :as hiccup]))
(def _h [:h1 :h2 :h3 :h4 :h5 :h6])
@ -102,13 +102,10 @@
:ol and true will result in an ordered list being generated for the table of
contents, while :ul will result in an unordered list. The default is an
ordered list."
[^String html & {:keys [list-type] :or {list-type :ol}}]
[html & {:keys [list-type] :or {list-type :ol}}]
(let [list-type (if (true? list-type) :ol list-type)]
(-> html
(.getBytes "UTF-8")
(java.io.ByteArrayInputStream.)
(html/parse)
:content
(enlive/html-snippet)
(get-headings)
(build-toc-tree)
(build-toc list-type)

14
src/cryogen_core/watcher.clj
View file

@ -1,20 +1,20 @@
(ns cryogen-core.watcher
(:require [clojure.java.io :refer [file]]
[cryogen-core.io :refer [ignore]]
[pandect.algo.md5 :refer [md5]]
(:require [clojure.java.io :as io]
[clojure.set :as set]
[hawk.core :as hawk]
[clojure.set :as set]))
[pandect.algo.md5 :as md5]
[cryogen-core.io :as cryogen-io]))
(defn get-assets [path ignored-files]
(->> path
file
io/file
file-seq
(filter #(not (.isDirectory ^java.io.File %)))
(filter (ignore ignored-files))))
(filter (cryogen-io/ignore ignored-files))))
(defn checksums [path ignored-files]
(let [files (get-assets path ignored-files)]
(zipmap (map md5 files) files)))
(zipmap (map md5/md5 files) files)))
(defn find-changes [old-sums new-sums]
(let [old-sum-set (-> old-sums keys set)

8
test/cryogen_core/compiler_test.clj
View file

@ -1,9 +1,8 @@
(ns cryogen-core.compiler-test
(:require [clojure.test :refer :all]
[me.raynes.fs :as fs]
[cryogen-core.compiler :refer :all]
[cryogen-core.io :refer [path]]
[cryogen-core.markup :as m]
[me.raynes.fs :as fs])
[cryogen-core.markup :as m])
(:import [java.io File]))
; Test that the content-until-more-marker return nil or correct html text.
@ -22,7 +21,8 @@
and more content.
</div>
</div>")
"<div id=\"post\"><div class=\"post-content\">
"<div id=\"post\">
<div class=\"post-content\">
this post has more marker
</div></div>")))

75
test/cryogen_core/klipse_test.clj Normal file
View file

@ -0,0 +1,75 @@
(ns cryogen-core.klipse-test
(:require [cryogen-core.klipse :refer :all]
[clojure.test :refer [deftest testing is are]]))
(deftest map-keys-test
(is (= {"a" 1 "b" 2} (map-keys name {:a 1 :b 2}))))
(deftest update-existing-test
(is (= {:a 1 :b 2} (update-existing {:a 1 :b 1} :b inc)))
(is (= {:a 1} (update-existing {:a 1} :b (constantly 2)))))
(deftest deep-merge-test
(is (= {:a {:b 1 :c 2}} (deep-merge {:a {:b 1}} {:a {:c 2}})))
(is (= {:a {:b 1}} (deep-merge {:a {:b 1}} {:a nil})))
(is (= {:a {:b 1 :c 3}} (deep-merge {:a {:b 1 :c 2}} {:a {:c 3}}))))
;; For testing convenience.
(defn elt
"Returns an enlive style html element."
([tag] (elt tag nil))
([tag attrs & content]
{:tag tag, :attrs attrs, :content content}))
(deftest filter-html-elems-test
(is (= [(elt :div {:class "x"} :content [(elt :div {:class "x"} "foo")])
(elt :div {:class "x"} "foo")])
(filter-html-elems (comp #{"x"} :class :attrs)
[(elt :h1 {:class "y"} "things!")
(elt :div {:class "x"} (elt :div {:class "x"} "foo"))])))
(deftest code-block-classes-test
(is (= ["clojure" "ruby"]
(code-block-classes
"<h1>stuff</h1>
<div class=\"not-code\"><pre><code class=\"clojure\">(def x 42)</code></pre></div>
<pre><code class=\"ruby\">123</code><pre>"))))
(deftest clojure-eval-classes-test
(is (= #{"eval-cljs" "eval-reagent"}
(clojure-eval-classes {"selector" ".eval-cljs"
"selector_reagent" ".eval-reagent"
"selector_eval_ruby" ".eval-ruby"}))))
(deftest clojure-eval?-test
(is (clojure-eval? {"selector" ".eval-cljs"}
"<h1>stuff</h1>
<div class=\"not-code\"><pre><code class=\"eval-cljs\">(def x 42)</code></pre></div>
<pre><code class=\"ruby\">123</code><pre>"))
(is (not (clojure-eval? {"selector" ".eval-cljs"
"selector_eval_ruby" ".eval-ruby"}
"<h1>stuff</h1>
<pre><code class=\"eval-ruby\">123</code><pre>"))))
(deftest normalize-settings-test
(is (= {"selector_reagent" ".reagent"
"codemirror_options_in" {"lineNumbers" true}}
(normalize-settings
{:selector-reagent ".reagent"
:codemirror-options-in {:line-numbers true}}))))
(deftest merge-configs-test
(testing "Things are merged correctly"
(is (= (merge defaults
{:settings {"selector" ".clojure-eval"
"codemirror_options_in" {"lineNumbers" true}}})
(merge-configs {:settings {:codemirror-options-in {:line-numbers true}}}
{:settings {:selector ".clojure-eval"}}))))
(testing "If it's all set up in config.edn, in the post it can be just :klipse true"
(is (= (merge defaults {:settings {"selector_js" ".javascript"}})
(merge-configs {:settings {:selector-js ".javascript"}} true))))
(testing "Returns nil if there's nothing in the blog post"
(is (nil? (merge-configs {:settings {:selector ".clojure-eval"}} nil)))))

108
test/cryogen_core/toc_test.clj
View file

@ -1,38 +1,35 @@
(ns cryogen-core.toc-test
(:require [clojure.test :refer :all]
[clojure.string :refer [join]]
[clojure.zip :as z]
[crouton.html :as html]
[clojure.string :as s]
[net.cgrand.enlive-html :as enlive]
[hiccup.core :as hiccup]
[cryogen-core.toc :refer :all]))
; Reimport private functions
(def get-headings #'cryogen-core.toc/get-headings)
(def make-toc-entry #'cryogen-core.toc/make-toc-entry)
(def zip-toc-tree-to-insertion-point #'cryogen-core.toc/zip-toc-tree-to-insertion-point)
(def insert-toc-tree-entry #'cryogen-core.toc/insert-toc-tree-entry)
(def build-toc-tree #'cryogen-core.toc/build-toc-tree)
(def build-toc #'cryogen-core.toc/build-toc)
(defn parse-to-headings
[hiccup-seq]
(-> hiccup-seq hiccup/html html/parse-string :content get-headings))
(-> hiccup-seq hiccup/html enlive/html-snippet get-headings))
; Test that the get-headings function properly filters non-headers
(deftest test-get-headings
(let [noisy-headers [:div [:h1 "First H1"]
[:p "Ignore..."]
[:h2 "First H2"]]]
[:p "Ignore..."]
[:h2 "First H2"]]]
(is (= (parse-to-headings noisy-headers)
[{:tag :h1 :attrs nil :content ["First H1"]}
{:tag :h2 :attrs nil :content ["First H2"]}]))))
[{:tag :h1 :attrs nil :content ["First H1"]}
{:tag :h2 :attrs nil :content ["First H2"]}]))))
; Test that the make-toc-entry ignores invalid input
(deftest test-make-toc-entry
(is (nil?
(make-toc-entry nil "Text")))
(make-toc-entry nil "Text")))
(is (nil?
(make-toc-entry "anchor" nil)))
(make-toc-entry "anchor" nil)))
(is (= [:li [:a {:href "#anchor"} "Text"]]
(make-toc-entry "anchor" "Text"))))
@ -45,65 +42,62 @@
; * h1
(deftest test-build-toc
(let [simplest-header [:div [:h2 [:a {:name "test"}] "Test"]]
no-headers [:div [:p "This is not a header"]]
no-headers [:div [:p "This is not a header"]]
closing-header-larger-than-opening-1
[:div [:h2 [:a {:name "starting_low"}]
"Starting Low"]
[:h1 [:a {:name "finishing_high"}]
"Finishing High"]]
[:div [:h2 [:a {:name "starting_low"}]
"Starting Low"]
[:h1 [:a {:name "finishing_high"}]
"Finishing High"]]
closing-header-larger-than-opening-2
[:div [:h2 [:a {:name "starting_low"}]
"Starting Low"]
[:h4 [:a {:name "jumping_in"}]
"Jumping Right In"]
[:h3 [:a {:name "pulling_back"}]
"But then pull back"]
[:h2 [:a {:name "to_the_top"}]
"To the top"]]]
[:div [:h2 [:a {:name "starting_low"}]
"Starting Low"]
[:h4 [:a {:name "jumping_in"}]
"Jumping Right In"]
[:h3 [:a {:name "pulling_back"}]
"But then pull back"]
[:h2 [:a {:name "to_the_top"}]
"To the top"]]]
(is (= [:ol.content (seq [[:li [:a {:href "#test"} "Test"]]])]
(-> simplest-header parse-to-headings build-toc-tree
(-> simplest-header
(parse-to-headings)
(build-toc-tree)
(build-toc :ol))))
(is (nil?
(-> no-headers parse-to-headings build-toc-tree
(build-toc :ol))))
(is (-> no-headers
(parse-to-headings)
(build-toc-tree)
(build-toc :ol)
(nil?)))
(is (= [:ol.content (seq [[:li [:a {:href "#starting_low"} "Starting Low"]]
[:li [:a {:href "#finishing_high"} "Finishing High"]]])]
(-> closing-header-larger-than-opening-1
parse-to-headings
build-toc-tree
(build-toc :ol)))
(-> closing-header-larger-than-opening-1
(parse-to-headings)
(build-toc-tree)
(build-toc :ol)))
"No outer header should be less indented than the first header tag.")
(is (= [:ul.content
(seq [
(seq [
[:li [:a {:href "#starting_low"} "Starting Low"]]
[:ul
(seq [
[:li [:a {:href "#jumping_in"} "Jumping Right In"]]
[:li [:a {:href "#pulling_back"} "But then pull back"]]
])
] ])
[:li [:a {:href "#to_the_top"} "To the top"]]
])]
(is (= [:ul.content
(seq [(seq [[:li [:a {:href "#starting_low"} "Starting Low"]]
[:ul
(seq [[:li [:a {:href "#jumping_in"} "Jumping Right In"]]
[:li [:a {:href "#pulling_back"} "But then pull back"]]])]])
[:li [:a {:href "#to_the_top"} "To the top"]]])]
(-> closing-header-larger-than-opening-2
parse-to-headings
build-toc-tree
(parse-to-headings)
(build-toc-tree)
(build-toc :ul)))
(join "" ["Inner headers can be more indented, "
"but outer headers cannot be less indented "
"than the original header."]))
))
(s/join "" ["Inner headers can be more indented, "
"but outer headers cannot be less indented "
"than the original header."]))))
(deftest test-generate-toc
(let [htmlString "<div><h2><a name=\"test\"></a>Test</h2></div>"]
(is (= "<ol class=\"content\"><li><a href=\"#test\">Test</a></li></ol>"
(generate-toc htmlString)))
(generate-toc htmlString)))
(is (= "<ol class=\"content\"><li><a href=\"#test\">Test</a></li></ol>"
(generate-toc htmlString :list-type true)))
(generate-toc htmlString :list-type true)))
(is (= "<ol class=\"content\"><li><a href=\"#test\">Test</a></li></ol>"
(generate-toc htmlString :list-type :ol)))
(generate-toc htmlString :list-type :ol)))
(is (= "<ul class=\"content\"><li><a href=\"#test\">Test</a></li></ul>"
(generate-toc htmlString :list-type :ul)))))
(generate-toc htmlString :list-type :ul)))))