diff --git a/deps.edn b/deps.edn index ed56f13..b6810a5 100644 --- a/deps.edn +++ b/deps.edn @@ -7,7 +7,9 @@ :deps {org.clojure/clojure {:mvn/version "1.11.4"} org.clojure/spec.alpha {:mvn/version "0.5.238"} - orchestra/orchestra {:mvn/version "2021.01.01-1"}} + orchestra/orchestra {:mvn/version "2021.01.01-1"} + org.domaindrivenarchitecture/c4k-common-clj {:mvn/version "8.0.1-SNAPSHOT"} + cheshire/cheshire {:mvn/version "5.13.0"}} :aliases { diff --git a/src/dda/build/c4k.clj b/src/dda/build/c4k.clj new file mode 100644 index 0000000..4506341 --- /dev/null +++ b/src/dda/build/c4k.clj @@ -0,0 +1,83 @@ +(ns dda.build.c4k + (:require [orchestra.core :refer [defn-spec]] + [clojure.spec.test.alpha :as st] + [cheshire.core :refer [parse-string generate-string]] + [dda.build.devops :as d] + [dda.build.c4k.domain :as domain] + [dda.build.terragrunt.domain :as tg-domain] + [dda.build.infrastructure :as i])) + +(st/instrument `clean-build-dir!) + +(def default + (merge d/default {:autoapply false + :c4k-output-filename "c4k-app.yaml" + :c4k-config-filename "c4k-config.yaml" + :c4k-auth-filename "c4k-auth.yaml"})) + +(defn-spec clean-build-dir! nil? + [devops ::d/devops] + (let [config (merge default devops)] + (i/execute! (domain/clean-build-dir-command config) config))) + +(defn-spec run-c4k-jar! nil? + [devops ::d/devops] + (let [config (merge default devops)] + (doseq [c (domain/c4k-uberjar-command config)] + (i/execute! c config)))) + +(defn-spec run-c4k-executable! nil? + [devops ::d/devops] + (let [config (merge default devops)] + (doseq [c (domain/c4k-graalvm-command config)] + (i/execute! c config)))) + +; TODO: Generate functions assume that files have already been copied, +; which will happen if this is run after terragrunt +; but it is not guaranteed +(defn-spec generate-jar! nil? + "Generates c4k app yaml using 'c4k-{module}-standalone.jar' + Stores the result in 'c4k-app.yaml' + + Defaults: c4k-config.yaml c4k-auth.yaml c4k-app.yaml + can be changed by adding another value for ':c4k-config-filename', ':c4k-auth-filename', ':c4k-output-filename' + " + [devops ::d/devops] + (let [config (merge default devops)] + (run-c4k-jar! config))) + +(defn-spec generate! nil? + "Generates c4k app yaml using 'c4k-{module}' (graalvm executable) + Stores the result in 'c4k-app.yaml' + + Defaults: c4k-config.yaml c4k-auth.yaml c4k-app.yaml + can be changed by adding another value for ':c4k-config-filename', ':c4k-auth-filename', ':c4k-output-filename' + " + [devops ::d/devops] + (let [config (merge default devops)] + (run-c4k-executable! config))) + +(defn-spec insert-tf-out! nil? + "Inserts relevant values from the tf output into the c4k config + + Default: c4k-config.yaml + can be changed by adding another value for ':c4k-config-filename' + " + [devops ::d/devops + tf-out ::tg-domain/tf-out] + (let [config (merge default devops) + default-c4k-config (parse-string (slurp (domain/config-path config)) + (fn [k] (keyword (.toLowerCase k)))) + tf-out-c4k-config (domain/create-c4k-config config tf-out)] + (->> default-c4k-config + (merge tf-out-c4k-config) + (generate-string) + (spit (domain/config-path config))))) + + +(st/instrument `clean-build-dir!) +(st/instrument `run-c4k-jar!) +(st/instrument `run-c4k-executable!) +(st/instrument `generate-jar!) +(st/instrument `generate!) +(st/instrument `insert-tf-out!) diff --git a/src/dda/build/c4k/domain.clj b/src/dda/build/c4k/domain.clj new file mode 100644 index 0000000..190a1d3 --- /dev/null +++ b/src/dda/build/c4k/domain.clj @@ -0,0 +1,53 @@ +(ns dda.build.c4k.domain + (:require [clojure.spec.alpha :as s] + [orchestra.core :refer [defn-spec]] + [dda.build.devops.domain :as d] + [dda.build.terragrunt.domain :as td])) + +(s/def ::c4k-output-filename string?) +(s/def ::c4k-config-filename string?) +(s/def ::c4k-auth-filename string?) + +(s/def ::config + (s/keys :req-un [::d/name ::d/stage ::d/project-root-path ::d/build-dir-name ::d/debug + ::d/dry-run ::c4k-output-filename ::c4k-config-filename ::c4k-auth-filename] + :opt-un [::d/module])) + +(defn-spec config-path string? + [config ::config] + (let [{:keys [c4k-config-filename]} config] + (str (d/build-path config) "/" c4k-config-filename))) + +(defn-spec auth-path string? + [config ::config] + (let [{:keys [c4k-auth-filename]} config] + (str (d/build-path config) "/" c4k-auth-filename))) + +(defn-spec output-path string? + [config ::config] + (let [{:keys [c4k-output-filename]} config] + (str (d/build-path config) "/" c4k-output-filename))) + +(defn-spec clean-build-dir-command seq? + [config ::config] + ["rm" "-rf" (d/build-path (dissoc config :module))]) + +(defn-spec c4k-uberjar-command seq? + [config ::config] + (let [{:keys [module]} config + executable-name (str "c4k-" module "-standalone.jar")] + [["bash" "-c" (str executable-name " " (config-path config) " " (auth-path config) " > " (output-path config))]])) + +(defn-spec c4k-graalvm-command seq? + [config ::config] + (let [{:keys [module]} config + executable-name (str "c4k-" module)] + [["bash" "-c" (str executable-name " " (config-path config) " " (auth-path config) " > " (output-path config))]])) + +(defn-spec create-c4k-config map? + [config ::config + tf-out ::td/tf-out] + (let [{:keys [stage]} config + issuer (if (= stage "prod") "prod" "staging") + fqdn (:fqdn (:value (:out tf-out)))] + {:issuer issuer :fqdn fqdn})) diff --git a/src/dda/build/devops.clj b/src/dda/build/devops.clj index 8287f23..994ff6e 100644 --- a/src/dda/build/devops.clj +++ b/src/dda/build/devops.clj @@ -22,6 +22,16 @@ :debug false :dry-run false}) +(defn-spec env-or-file string? + [name string?] + (let [from-env (System/getenv name) + name-from-file (System/getenv (str name "_FILE"))] + (cond + (some? from-env) from-env + (some? name-from-file) (slurp name-from-file) + :else (throw (RuntimeException. + (str "Environment: [" name "," name-from-file "] was missing.")))))) + (defn-spec clean-build-dir! nil? [devops ::devops] (let [final (merge default devops)] diff --git a/src/dda/build/gopass.clj b/src/dda/build/gopass.clj new file mode 100644 index 0000000..daf2030 --- /dev/null +++ b/src/dda/build/gopass.clj @@ -0,0 +1,51 @@ +(ns dda.build.gopass + (:require [orchestra.core :refer [defn-spec]] + [clojure.spec.test.alpha :as st] + [cheshire.core :refer [parse-string generate-string]] + [dda.build.devops :as d] + [dda.build.gopass.domain :as domain] + [dda.build.c4k.domain :as c4k-d] + [dda.build.infrastructure :as i])) + +(def default + (merge d/default {:c4k-auth-filename "c4k-auth.yaml"})) + +(defn-spec run-gopass-command! string? + [devops ::d/devops + entry ::domain/gopass-entry] + (let [config (merge default devops) + c (domain/gopass-show-command entry)] + (i/execute-output! c config))) + +(defn-spec resolve-gopass! ::domain/resolved-config + "Resolves gopass values inside a map of key names and entries + + entries may either contain only a path + {:path \"test/path\"} + or a path and a field + {:path \"test/path\" :field \"field\"} + " + [devops ::d/devops + config ::domain/config] + (apply merge (for [[k v] config] {(name k) (run-gopass-command! devops v)}))) + +(defn-spec insert-gopass! nil? + "Inserts values from the resolved auth config into the c4k auth + + Default: c4k-auth.yaml + can be changed by adding another value for ':c4k-auth-filename' + " + [devops ::d/devops + resolved-config ::domain/resolved-config] + (let [config (merge default devops) + default-c4k-auth (parse-string (slurp (c4k-d/auth-path config)) + (fn [k] (keyword (.toLowerCase k))))] + (->> default-c4k-auth + (merge resolved-config) + (generate-string) + (spit (c4k-d/auth-path config))))) + + +(st/instrument `run-gopass-command!) +(st/instrument `resolve-gopass!) +(st/instrument `insert-gopass!) diff --git a/src/dda/build/gopass/domain.clj b/src/dda/build/gopass/domain.clj new file mode 100644 index 0000000..e78fbd8 --- /dev/null +++ b/src/dda/build/gopass/domain.clj @@ -0,0 +1,21 @@ +(ns dda.build.gopass.domain + (:require [clojure.spec.alpha :as s] + [orchestra.core :refer [defn-spec]])) + +(s/def ::path string?) +(s/def ::field string?) +(s/def ::gopass-entry (s/keys :req-un [::path] + :opt-un [::field])) +(s/def ::config (s/map-of keyword? ::gopass-entry)) + +(s/def ::resolved-config (s/map-of string? string?)) + +(s/def ::gopass-command (s/coll-of string?)) +(s/def ::gopass-commands (s/coll-of ::gopass-command)) + +(defn-spec gopass-show-command ::gopass-command + [entry ::gopass-entry] + (let [{:keys [path field] :or {field nil}} entry] + (if (nil? field) + ["gopass" "show" "-y" "-o" path] + ["gopass" "show" "-y" "-o" path field]))) diff --git a/src/dda/build/infrastructure.clj b/src/dda/build/infrastructure.clj index 0adfd8d..e684fb2 100644 --- a/src/dda/build/infrastructure.clj +++ b/src/dda/build/infrastructure.clj @@ -11,3 +11,14 @@ (println command)) (when-not dry-run (apply t/shell command)))) + +(defn-spec execute-output! string? + [command seq? + devops ::d/devops] + (let [{:keys [dry-run debug]} devops] + (when debug + (println command)) + (when-not dry-run + (->> (t/shell command {:out :string}) + :out + clojure.string/trim)))) \ No newline at end of file diff --git a/src/dda/build/provs.clj b/src/dda/build/provs.clj new file mode 100644 index 0000000..840a000 --- /dev/null +++ b/src/dda/build/provs.clj @@ -0,0 +1,40 @@ +(ns dda.build.provs + (:require [orchestra.core :refer [defn-spec]] + [clojure.spec.test.alpha :as st] + [cheshire.core :refer [generate-string]] + [dda.build.devops :as d] + [dda.build.provs.domain :as domain] + [dda.build.terragrunt.domain :as td] + [dda.build.infrastructure :as i])) + +(def default + (merge d/default {:k3s-output-filename "out_k3sServerConfig.yaml" + :k3s-provision-user "root" + :c4k-output-filename "c4k-app.yaml" + :email "default@email.rep" + :echo false})) + +(defn-spec run-provs-server! nil? + [devops ::d/devops + tf-out ::td/tf-out] + (let [config (merge default devops)] + (doseq [c (domain/provs-server-command config tf-out)] + (i/execute! c config)))) + +(defn-spec write-k3s-config! nil? + "Create a server config for provs using tf-out and write it to a file + + Requires ':email' to be set, otherwise certs will not work + Default: out_k3sServerConfig.yaml + can be changed by adding another value for ':k3s-output-filename' + " + [devops ::d/devops + tf-out ::td/tf-out] + (let [config (merge default devops) + tf-out-k3s-config (domain/create-k3s-config config tf-out)] + (->> tf-out-k3s-config + (generate-string) + (spit (domain/output-path config))))) + +(st/instrument `run-provs-server!) +(st/instrument `write-k3s-config!) \ No newline at end of file diff --git a/src/dda/build/provs/domain.clj b/src/dda/build/provs/domain.clj new file mode 100644 index 0000000..77cdd37 --- /dev/null +++ b/src/dda/build/provs/domain.clj @@ -0,0 +1,55 @@ +(ns dda.build.provs.domain + (:require [clojure.spec.alpha :as s] + [orchestra.core :refer [defn-spec]] + [dda.build.devops.domain :as d] + [dda.c4k-common.predicate :as pred] + [dda.build.c4k.domain :as c4k-d] + [dda.build.terragrunt.domain :as td])) + +; TODO: Use a better spec for emails +; should be added to c4k-common, it seems common enough +(s/def ::email pred/bash-env-string?) +(s/def ::fqdn pred/fqdn-string?) +(s/def ::ipv4 pred/ipv4-string?) +(s/def ::ipv6 pred/ipv6-string?) +(s/def ::echo boolean?) +(s/def ::k3s-output-filename string?) +(s/def ::k3s-provision-user pred/bash-env-string?) +(s/def ::config + (s/keys :req-un [::d/name ::d/stage ::d/project-root-path ::d/build-dir-name ::d/debug + ::d/dry-run ::c4k-d/c4k-output-filename + ::email ::echo ::k3s-output-filename ::k3s-provision-user] + :opt-un [::d/module])) +(s/def ::node + (s/keys :req-un [::ipv4 ::ipv6])) +(s/def ::letsencryptEndpoint pred/letsencrypt-issuer?) +(s/def ::certmanager + (s/keys :req-un [::email ::letsencryptEndpoint])) +(s/def ::server-config + (s/keys :req-un [::fqdn ::node ::certmanager ::echo])) + +(defn-spec output-path string? + [config ::config] + (let [{:keys [k3s-output-filename]} config] + (str (d/build-path config) "/" k3s-output-filename))) + +(defn-spec provs-server-command seq? + [config ::config + tf-out ::td/tf-out] + (let [{:keys [k3s-output-filename k3s-provision-user]} config + fqdn (get-in tf-out [:out :value :fqdn])] + [["provs-server.jar" "k3s" (str k3s-provision-user "@" fqdn) "-c" (output-path config) "-a" (c4k-d/output-path config)]])) + +(defn-spec create-k3s-config map? + [config ::config + tf-out ::td/tf-out] + (let [{:keys [stage email echo]} config + letsencrypt-endpoint (if (= stage "prod") "prod" "staging") + values (:value (:out tf-out)) + {:keys [fqdn ipv4 ipv6]} values] + {:fqdn fqdn + :node {:ipv4 ipv4 + :ipv6 ipv6} + :certmanager {:email email + :letsencryptEndpoint letsencrypt-endpoint} + :echo echo})) diff --git a/src/dda/build/terragrunt/domain.clj b/src/dda/build/terragrunt/domain.clj index 48e8665..b2ec251 100644 --- a/src/dda/build/terragrunt/domain.clj +++ b/src/dda/build/terragrunt/domain.clj @@ -1,7 +1,24 @@ (ns dda.build.terragrunt.domain (:require [clojure.spec.alpha :as s] [orchestra.core :refer [defn-spec]] - [dda.build.devops.domain :as d])) + [dda.build.devops.domain :as d] + [dda.c4k-common.predicate :as pred])) + +(s/def ::sensitive boolean?) + +(s/def ::type vector?) + +(s/def ::fqdn pred/fqdn-string?) +(s/def ::ipv4 pred/ipv4-string?) +(s/def ::ipv6 pred/ipv6-string?) +(s/def ::value + (s/keys :req-un [::fqdn ::ipv4 ::ipv6])) + +(s/def ::out + (s/keys :req-un [::sensitive ::type ::value])) + +(s/def ::tf-out + (s/keys :req-un [::out])) (s/def ::tg-output-filenname string?) (s/def ::autoapply boolean?) diff --git a/test/dda/build/c4k/domain_test.clj b/test/dda/build/c4k/domain_test.clj new file mode 100644 index 0000000..646a001 --- /dev/null +++ b/test/dda/build/c4k/domain_test.clj @@ -0,0 +1,129 @@ +(ns dda.build.c4k.domain-test + (:require + [clojure.test :refer [deftest is]] + [clojure.spec.test.alpha :as st] + [dda.build.c4k.domain :as cut])) + +(st/instrument `cut/config-path) +(st/instrument `cut/auth-path) +(st/instrument `cut/output-path) +(st/instrument `cut/clean-build-dir-command) +(st/instrument `cut/c4k-uberjar-command) +(st/instrument `cut/c4k-graalvm-command) +(st/instrument `cut/create-c4k-config) + +(deftest should-calculate-config-path + (is (= "../../target/dda-backup/config.yaml" + (cut/config-path {:name "dda-backup" + :project-root-path "../.." + :build-dir-name "target" + :version "4.11.8-dev" + :stage "dev" + :debug false + :dry-run false + :c4k-config-filename "config.yaml" + :c4k-auth-filename "auth.yaml" + :c4k-output-filename "out.yaml"}))) + (is (= "../../target/dda/backup/config.yaml" + (cut/config-path {:name "dda" + :module "backup" + :project-root-path "../.." + :build-dir-name "target" + :version "4.11.8-dev" + :stage "dev" + :debug false + :dry-run false + :c4k-config-filename "config.yaml" + :c4k-auth-filename "auth.yaml" + :c4k-output-filename "out.yaml"})))) + +(deftest should-calculate-auth-path + (is (= "../../target/dda-backup/auth.yaml" + (cut/auth-path {:name "dda-backup" + :project-root-path "../.." + :build-dir-name "target" + :version "4.11.8-dev" + :stage "dev" + :debug false + :dry-run false + :c4k-config-filename "config.yaml" + :c4k-auth-filename "auth.yaml" + :c4k-output-filename "out.yaml"}))) + (is (= "../../target/dda/backup/auth.yaml" + (cut/auth-path {:name "dda" + :module "backup" + :project-root-path "../.." + :build-dir-name "target" + :version "4.11.8-dev" + :stage "dev" + :debug false + :dry-run false + :c4k-config-filename "config.yaml" + :c4k-auth-filename "auth.yaml" + :c4k-output-filename "out.yaml"})))) + + +(deftest should-calculate-output-path + (is (= "../../target/dda-backup/out.yaml" + (cut/output-path {:name "dda-backup" + :project-root-path "../.." + :build-dir-name "target" + :version "4.11.8-dev" + :stage "dev" + :debug false + :dry-run false + :c4k-config-filename "config.yaml" + :c4k-auth-filename "auth.yaml" + :c4k-output-filename "out.yaml"}))) + (is (= "../../target/dda/backup/out.yaml" + (cut/output-path {:name "dda" + :module "backup" + :project-root-path "../.." + :build-dir-name "target" + :version "4.11.8-dev" + :stage "dev" + :debug false + :dry-run false + :c4k-config-filename "config.yaml" + :c4k-auth-filename "auth.yaml" + :c4k-output-filename "out.yaml"})))) + +(deftest should-calculate-clean-build-dir-command + (is (= ["rm" "-rf" "../../target/dda-backup"] + (cut/clean-build-dir-command {:name "dda-backup" + :project-root-path "../.." + :build-dir-name "target" + :version "4.11.8-dev" + :stage "dev" + :debug false + :dry-run false + :c4k-config-filename "config.yaml" + :c4k-auth-filename "auth.yaml" + :c4k-output-filename "out.yaml"})))) + +(deftest should-calculate-c4k-uberjar-command + (is (= ["bash" "-c" "c4k-dda-backup-standalone.jar ../../target/dda-backup/config.yaml ../../target/dda-backup/auth.yaml > ../../target/dda-backup/out.yaml"] + (cut/c4k-uberjar-command {:name "dda-backup" + :project-root-path "../.." + :build-dir-name "target" + :version "4.11.8-dev" + :stage "dev" + :debug false + :dry-run false + :c4k-config-filename "config.yaml" + :c4k-auth-filename "auth.yaml" + :c4k-output-filename "out.yaml"})))) + +(deftest should-calculate-c4k-graalvm-command + (is (= ["bash" "-c" "c4k-dda-backup ../../target/dda-backup/config.yaml ../../target/dda-backup/auth.yaml > ../../target/dda-backup/out.yaml"] + (cut/c4k-graalvm-command {:name "dda-backup" + :project-root-path "../.." + :build-dir-name "target" + :version "4.11.8-dev" + :stage "dev" + :debug false + :dry-run false + :c4k-config-filename "config.yaml" + :c4k-auth-filename "auth.yaml" + :c4k-output-filename "out.yaml"})))) + diff --git a/test/dda/build/gopass/domain_test.clj b/test/dda/build/gopass/domain_test.clj new file mode 100644 index 0000000..e16294a --- /dev/null +++ b/test/dda/build/gopass/domain_test.clj @@ -0,0 +1,13 @@ +(ns dda.build.gopass.domain-test + (:require + [clojure.test :refer [deftest is]] + [clojure.spec.test.alpha :as st] + [dda.build.gopass.domain :as cut])) + +(st/instrument `cut/gopass-show-command) + +(deftest should-show-gopass-command + (is (= ["gopass" "show" "-y" "-o" "test/pass"] + (cut/gopass-show-command {:path "test/pass"}))) + (is (= ["gopass" "show" "-y" "-o" "test/pass" "field"] + (cut/gopass-show-command {:path "test/pass" :field "field"})))) diff --git a/test/dda/build/provs/domain_test.clj b/test/dda/build/provs/domain_test.clj new file mode 100644 index 0000000..5b23684 --- /dev/null +++ b/test/dda/build/provs/domain_test.clj @@ -0,0 +1,57 @@ +(ns dda.build.provs.domain-test + (:require + [clojure.test :refer [deftest is]] + [clojure.spec.test.alpha :as st] + [dda.build.provs.domain :as cut])) + +(st/instrument `cut/output-path) +(st/instrument `cut/provs-server-command) +(st/instrument `cut/create-k3s-config) + +(deftest should-calculate-output-path + (is (= "../../target/dda-backup/k3s-out.yaml" + (cut/output-path {:name "dda-backup" + :project-root-path "../.." + :build-dir-name "target" + :version "4.11.8-dev" + :stage "dev" + :debug false + :dry-run false + :c4k-output-filename "out.yaml" + :k3s-provision-user "root" + :k3s-output-filename "k3s-out.yaml" + :email "test@test.t" + :echo false}))) + (is (= "../../target/dda/backup/k3s-out.yaml" + (cut/output-path {:name "dda" + :module "backup" + :project-root-path "../.." + :build-dir-name "target" + :version "4.11.8-dev" + :stage "dev" + :debug false + :dry-run false + :c4k-output-filename "out.yaml" + :k3s-provision-user "root" + :k3s-output-filename "k3s-out.yaml" + :email "test@test.t" + :echo false})))) + +(deftest should-calculate-provs-server-command + (is (= ["provs-server.jar" "k3s" "root@test.test.de" "-c" "../../target/dda-backup/k3s-out.yaml" "-a" "../../target/dda-backup/out.yaml"] + (cut/provs-server-command {:name "dda-backup" + :project-root-path "../.." + :build-dir-name "target" + :version "4.11.8-dev" + :stage "dev" + :debug false + :dry-run false + :k3s-output-filename "k3s-out.yaml" + :k3s-provision-user "root" + :c4k-config-filename "config.yaml" + :c4k-auth-filename "auth.yaml" + :c4k-output-filename "out.yaml" + :email "test@test.t" + :echo false} + {:out {:sensitive false :type [] :value {:fqdn "test.test.de" :ipv4 "127.0.0.1" :ipv6 "::"}}})))) +