Fix command functions

Expand infrastructure so we can read output from commands

deps.edn

@@ -7,7 +7,9 @@
  :deps
  {org.clojure/clojure {:mvn/version "1.11.4"}
   org.clojure/spec.alpha {:mvn/version "0.5.238"}
-  orchestra/orchestra {:mvn/version "2021.01.01-1"}}
+  orchestra/orchestra {:mvn/version "2021.01.01-1"}
+  org.domaindrivenarchitecture/c4k-common-clj {:mvn/version "8.0.1-SNAPSHOT"}
+  cheshire/cheshire {:mvn/version "5.13.0"}}
 
  :aliases
  {

src/dda/build/c4k.clj

@@ -0,0 +1,83 @@
+(ns dda.build.c4k
+  (:require [orchestra.core :refer [defn-spec]]
+            [clojure.spec.test.alpha :as st]
+            [cheshire.core :refer [parse-string generate-string]]
+            [dda.build.devops :as d]
+            [dda.build.c4k.domain :as domain]
+            [dda.build.terragrunt.domain :as tg-domain]
+            [dda.build.infrastructure :as i]))
+
+(st/instrument `clean-build-dir!)
+
+(def default
+  (merge d/default {:autoapply false
+                    :c4k-output-filename "c4k-app.yaml"
+                    :c4k-config-filename "c4k-config.yaml"
+                    :c4k-auth-filename "c4k-auth.yaml"}))
+
+(defn-spec clean-build-dir! nil?
+  [devops ::d/devops]
+  (let [config (merge default devops)]
+    (i/execute! (domain/clean-build-dir-command config) config)))
+
+(defn-spec run-c4k-jar! nil?
+  [devops ::d/devops]
+  (let [config (merge default devops)]
+    (doseq [c (domain/c4k-uberjar-command config)]
+      (i/execute! c config))))
+
+(defn-spec run-c4k-executable! nil?
+  [devops ::d/devops]
+  (let [config (merge default devops)]
+    (doseq [c (domain/c4k-graalvm-command config)]
+      (i/execute! c config))))
+
+; TODO: Generate functions assume that files have already been copied,
+;       which will happen if this is run after terragrunt
+;       but it is not guaranteed
+(defn-spec generate-jar! nil?
+  "Generates c4k app yaml using 'c4k-{module}-standalone.jar'
+   Stores the result in 'c4k-app.yaml'
+   
+   Defaults: c4k-config.yaml c4k-auth.yaml c4k-app.yaml
+   can be changed by adding another value for ':c4k-config-filename', ':c4k-auth-filename', ':c4k-output-filename'
+  "
+  [devops ::d/devops]
+  (let [config (merge default devops)]
+    (run-c4k-jar! config)))
+
+(defn-spec generate! nil?
+  "Generates c4k app yaml using 'c4k-{module}' (graalvm executable)
+   Stores the result in 'c4k-app.yaml'
+   
+   Defaults: c4k-config.yaml c4k-auth.yaml c4k-app.yaml
+   can be changed by adding another value for ':c4k-config-filename', ':c4k-auth-filename', ':c4k-output-filename'
+  "
+  [devops ::d/devops]
+  (let [config (merge default devops)]
+    (run-c4k-executable! config)))
+
+(defn-spec insert-tf-out! nil?
+  "Inserts relevant values from the tf output into the c4k config
+   
+   Default: c4k-config.yaml
+   can be changed by adding another value for ':c4k-config-filename'
+   "
+  [devops ::d/devops
+   tf-out ::tg-domain/tf-out]
+  (let [config (merge default devops)
+        default-c4k-config (parse-string (slurp (domain/config-path config))
+                                         (fn [k] (keyword (.toLowerCase k))))
+        tf-out-c4k-config (domain/create-c4k-config config tf-out)]
+    (->> default-c4k-config
+         (merge tf-out-c4k-config)
+         (generate-string)
+         (spit (domain/config-path config)))))
+
+
+(st/instrument `clean-build-dir!)
+(st/instrument `run-c4k-jar!)
+(st/instrument `run-c4k-executable!)
+(st/instrument `generate-jar!)
+(st/instrument `generate!)
+(st/instrument `insert-tf-out!)

src/dda/build/c4k/domain.clj

@@ -0,0 +1,53 @@
+(ns dda.build.c4k.domain
+  (:require [clojure.spec.alpha :as s]
+            [orchestra.core :refer [defn-spec]]
+            [dda.build.devops.domain :as d]
+            [dda.build.terragrunt.domain :as td]))
+
+(s/def ::c4k-output-filename string?)
+(s/def ::c4k-config-filename string?)
+(s/def ::c4k-auth-filename string?)
+
+(s/def ::config
+  (s/keys :req-un [::d/name ::d/stage ::d/project-root-path ::d/build-dir-name ::d/debug
+                   ::d/dry-run ::c4k-output-filename ::c4k-config-filename ::c4k-auth-filename]
+          :opt-un [::d/module]))
+
+(defn-spec config-path string?
+  [config ::config]
+  (let [{:keys [c4k-config-filename]} config]
+    (str (d/build-path config) "/" c4k-config-filename)))
+
+(defn-spec auth-path string?
+  [config ::config]
+  (let [{:keys [c4k-auth-filename]} config]
+    (str (d/build-path config) "/" c4k-auth-filename)))
+
+(defn-spec output-path string?
+  [config ::config]
+  (let [{:keys [c4k-output-filename]} config]
+    (str (d/build-path config) "/" c4k-output-filename)))
+
+(defn-spec clean-build-dir-command seq?
+  [config ::config]
+  ["rm"  "-rf" (d/build-path (dissoc config :module))])
+
+(defn-spec c4k-uberjar-command seq?
+  [config ::config]
+  (let [{:keys [module]} config
+        executable-name (str "c4k-" module "-standalone.jar")]
+    [["bash" "-c" (str executable-name " " (config-path config) " " (auth-path config) " > " (output-path config))]]))
+
+(defn-spec c4k-graalvm-command seq?
+  [config ::config]
+  (let [{:keys [module]} config
+        executable-name (str "c4k-" module)]
+    [["bash" "-c" (str executable-name " " (config-path config) " " (auth-path config) " > " (output-path config))]]))
+
+(defn-spec create-c4k-config map?
+  [config ::config
+   tf-out ::td/tf-out]
+  (let [{:keys [stage]} config
+         issuer (if (= stage "prod") "prod" "staging")
+         fqdn (:fqdn (:value (:out tf-out)))]
+    {:issuer issuer :fqdn fqdn}))

src/dda/build/devops.clj

@@ -22,6 +22,16 @@
               :debug false
               :dry-run false})
 
+(defn-spec env-or-file string?
+  [name string?]
+  (let [from-env (System/getenv name)
+        name-from-file (System/getenv (str name "_FILE"))]
+    (cond
+      (some? from-env) from-env
+      (some? name-from-file) (slurp name-from-file)
+      :else (throw (RuntimeException.
+                    (str "Environment: [" name "," name-from-file "] was missing."))))))
+
 (defn-spec clean-build-dir! nil?
   [devops ::devops]
   (let [final (merge default devops)]

src/dda/build/gopass.clj

@@ -0,0 +1,51 @@
+(ns dda.build.gopass
+  (:require [orchestra.core :refer [defn-spec]]
+            [clojure.spec.test.alpha :as st]
+            [cheshire.core :refer [parse-string generate-string]]
+            [dda.build.devops :as d]
+            [dda.build.gopass.domain :as domain]
+            [dda.build.c4k.domain :as c4k-d]
+            [dda.build.infrastructure :as i]))
+
+(def default
+  (merge d/default {:c4k-auth-filename "c4k-auth.yaml"}))
+
+(defn-spec run-gopass-command! string?
+  [devops ::d/devops
+   entry ::domain/gopass-entry]
+  (let [config (merge default devops)
+        c (domain/gopass-show-command entry)]
+    (i/execute-output! c config)))
+
+(defn-spec resolve-gopass! ::domain/resolved-config
+  "Resolves gopass values inside a map of key names and entries
+   
+   entries may either contain only a path 
+     {:path \"test/path\"} 
+   or a path and a field
+     {:path \"test/path\" :field \"field\"}
+   "
+  [devops ::d/devops
+   config ::domain/config]
+  (apply merge (for [[k v] config] {(name k) (run-gopass-command! devops v)})))
+
+(defn-spec insert-gopass! nil?
+  "Inserts values from the resolved auth config into the c4k auth
+   
+   Default: c4k-auth.yaml
+   can be changed by adding another value for ':c4k-auth-filename'
+   "
+  [devops ::d/devops
+   resolved-config ::domain/resolved-config]
+  (let [config (merge default devops)
+        default-c4k-auth (parse-string (slurp (c4k-d/auth-path config))
+                                       (fn [k] (keyword (.toLowerCase k))))]
+    (->> default-c4k-auth
+         (merge resolved-config)
+         (generate-string)
+         (spit (c4k-d/auth-path config)))))
+
+
+(st/instrument `run-gopass-command!)
+(st/instrument `resolve-gopass!)
+(st/instrument `insert-gopass!)

src/dda/build/gopass/domain.clj

@@ -0,0 +1,21 @@
+(ns dda.build.gopass.domain
+  (:require [clojure.spec.alpha :as s]
+            [orchestra.core :refer [defn-spec]]))
+
+(s/def ::path string?)
+(s/def ::field string?)
+(s/def ::gopass-entry (s/keys :req-un [::path]
+                              :opt-un [::field]))
+(s/def ::config (s/map-of keyword? ::gopass-entry))
+
+(s/def ::resolved-config (s/map-of string? string?))
+
+(s/def ::gopass-command (s/coll-of string?))
+(s/def ::gopass-commands (s/coll-of ::gopass-command))
+
+(defn-spec gopass-show-command ::gopass-command
+  [entry ::gopass-entry]
+  (let [{:keys [path field] :or {field nil}} entry]
+    (if (nil? field)
+      ["gopass" "show" "-y" "-o" path]
+      ["gopass" "show" "-y" "-o" path field])))

src/dda/build/infrastructure.clj

@@ -11,3 +11,14 @@
       (println command))
     (when-not dry-run
       (apply t/shell command))))
+
+(defn-spec execute-output! string?
+  [command seq?
+   devops ::d/devops]
+  (let [{:keys [dry-run debug]} devops]
+    (when debug
+      (println command))
+    (when-not dry-run
+      (->> (t/shell command {:out :string})
+           :out
+           clojure.string/trim))))

src/dda/build/provs.clj

@@ -0,0 +1,40 @@
+(ns dda.build.provs
+  (:require [orchestra.core :refer [defn-spec]]
+            [clojure.spec.test.alpha :as st]
+            [cheshire.core :refer [generate-string]]
+            [dda.build.devops :as d]
+            [dda.build.provs.domain :as domain]
+            [dda.build.terragrunt.domain :as td]
+            [dda.build.infrastructure :as i]))
+
+(def default
+  (merge d/default {:k3s-output-filename "out_k3sServerConfig.yaml"
+                    :k3s-provision-user "root"
+                    :c4k-output-filename "c4k-app.yaml"
+                    :email "default@email.rep"
+                    :echo false}))
+
+(defn-spec run-provs-server! nil?
+  [devops ::d/devops
+   tf-out ::td/tf-out]
+  (let [config (merge default devops)]
+    (doseq [c (domain/provs-server-command config tf-out)]
+      (i/execute! c config))))
+
+(defn-spec write-k3s-config! nil?
+  "Create a server config for provs using tf-out and write it to a file
+   
+   Requires ':email' to be set, otherwise certs will not work
+   Default: out_k3sServerConfig.yaml
+   can be changed by adding another value for ':k3s-output-filename'
+   "
+  [devops ::d/devops
+   tf-out ::td/tf-out]
+  (let [config (merge default devops)
+        tf-out-k3s-config (domain/create-k3s-config config tf-out)]
+    (->> tf-out-k3s-config
+         (generate-string)
+         (spit (domain/output-path config)))))
+
+(st/instrument `run-provs-server!)
+(st/instrument `write-k3s-config!)

src/dda/build/provs/domain.clj

@@ -0,0 +1,55 @@
+(ns dda.build.provs.domain
+  (:require [clojure.spec.alpha :as s]
+            [orchestra.core :refer [defn-spec]]
+            [dda.build.devops.domain :as d]
+            [dda.c4k-common.predicate :as pred]
+            [dda.build.c4k.domain :as c4k-d]
+            [dda.build.terragrunt.domain :as td]))
+
+; TODO: Use a better spec for emails
+;       should be added to c4k-common, it seems common enough
+(s/def ::email pred/bash-env-string?)
+(s/def ::fqdn pred/fqdn-string?)
+(s/def ::ipv4 pred/ipv4-string?)
+(s/def ::ipv6 pred/ipv6-string?)
+(s/def ::echo boolean?)
+(s/def ::k3s-output-filename string?)
+(s/def ::k3s-provision-user pred/bash-env-string?)
+(s/def ::config
+  (s/keys :req-un [::d/name ::d/stage ::d/project-root-path ::d/build-dir-name ::d/debug
+                   ::d/dry-run ::c4k-d/c4k-output-filename 
+                   ::email ::echo ::k3s-output-filename ::k3s-provision-user]
+          :opt-un [::d/module]))
+(s/def ::node
+  (s/keys :req-un [::ipv4 ::ipv6]))
+(s/def ::letsencryptEndpoint pred/letsencrypt-issuer?)
+(s/def ::certmanager
+  (s/keys :req-un [::email ::letsencryptEndpoint]))
+(s/def ::server-config
+  (s/keys :req-un [::fqdn ::node ::certmanager ::echo]))
+
+(defn-spec output-path string?
+  [config ::config]
+  (let [{:keys [k3s-output-filename]} config]
+    (str (d/build-path config) "/" k3s-output-filename)))
+
+(defn-spec provs-server-command seq?
+  [config ::config
+   tf-out ::td/tf-out]
+  (let [{:keys [k3s-output-filename k3s-provision-user]} config
+         fqdn (get-in tf-out [:out :value :fqdn])]
+    [["provs-server.jar" "k3s" (str k3s-provision-user "@" fqdn) "-c" (output-path config) "-a" (c4k-d/output-path config)]]))
+
+(defn-spec create-k3s-config map?
+  [config ::config
+   tf-out ::td/tf-out]
+  (let [{:keys [stage email echo]} config
+        letsencrypt-endpoint (if (= stage "prod") "prod" "staging")
+        values (:value (:out tf-out))
+        {:keys [fqdn ipv4 ipv6]} values]
+    {:fqdn fqdn
+     :node {:ipv4 ipv4
+            :ipv6 ipv6}
+     :certmanager {:email email
+                   :letsencryptEndpoint letsencrypt-endpoint}
+     :echo echo}))

src/dda/build/terragrunt/domain.clj

@@ -1,7 +1,24 @@
 (ns dda.build.terragrunt.domain
   (:require [clojure.spec.alpha :as s]
             [orchestra.core :refer [defn-spec]]
-            [dda.build.devops.domain :as d]))
+            [dda.build.devops.domain :as d]
+            [dda.c4k-common.predicate :as pred]))
+
+(s/def ::sensitive boolean?)
+
+(s/def ::type vector?)
+
+(s/def ::fqdn pred/fqdn-string?)
+(s/def ::ipv4 pred/ipv4-string?)
+(s/def ::ipv6 pred/ipv6-string?)
+(s/def ::value
+  (s/keys :req-un [::fqdn ::ipv4 ::ipv6]))
+
+(s/def ::out
+  (s/keys :req-un [::sensitive ::type ::value]))
+
+(s/def ::tf-out
+  (s/keys :req-un [::out]))
 
 (s/def ::tg-output-filenname string?)
 (s/def ::autoapply boolean?)

test/dda/build/c4k/domain_test.clj

@@ -0,0 +1,129 @@
+(ns dda.build.c4k.domain-test
+  (:require
+   [clojure.test :refer [deftest is]]
+   [clojure.spec.test.alpha :as st]
+   [dda.build.c4k.domain :as cut]))
+
+(st/instrument `cut/config-path)
+(st/instrument `cut/auth-path)
+(st/instrument `cut/output-path)
+(st/instrument `cut/clean-build-dir-command)
+(st/instrument `cut/c4k-uberjar-command)
+(st/instrument `cut/c4k-graalvm-command)
+(st/instrument `cut/create-c4k-config)
+
+(deftest should-calculate-config-path
+  (is (= "../../target/dda-backup/config.yaml"
+         (cut/config-path {:name "dda-backup"
+                           :project-root-path "../.."
+                           :build-dir-name "target"
+                           :version "4.11.8-dev"
+                           :stage "dev"
+                           :debug false
+                           :dry-run false
+                           :c4k-config-filename "config.yaml"
+                           :c4k-auth-filename "auth.yaml"
+                           :c4k-output-filename "out.yaml"})))
+  (is (= "../../target/dda/backup/config.yaml"
+         (cut/config-path {:name "dda"
+                           :module "backup"
+                           :project-root-path "../.."
+                           :build-dir-name "target"
+                           :version "4.11.8-dev"
+                           :stage "dev"
+                           :debug false
+                           :dry-run false
+                           :c4k-config-filename "config.yaml"
+                           :c4k-auth-filename "auth.yaml"
+                           :c4k-output-filename "out.yaml"}))))
+
+(deftest should-calculate-auth-path
+  (is (= "../../target/dda-backup/auth.yaml"
+         (cut/auth-path {:name "dda-backup"
+                         :project-root-path "../.."
+                         :build-dir-name "target"
+                         :version "4.11.8-dev"
+                         :stage "dev"
+                         :debug false
+                         :dry-run false
+                         :c4k-config-filename "config.yaml"
+                         :c4k-auth-filename "auth.yaml"
+                         :c4k-output-filename "out.yaml"})))
+  (is (= "../../target/dda/backup/auth.yaml"
+         (cut/auth-path {:name "dda"
+                         :module "backup"
+                         :project-root-path "../.."
+                         :build-dir-name "target"
+                         :version "4.11.8-dev"
+                         :stage "dev"
+                         :debug false
+                         :dry-run false
+                         :c4k-config-filename "config.yaml"
+                         :c4k-auth-filename "auth.yaml"
+                         :c4k-output-filename "out.yaml"}))))
+
+
+(deftest should-calculate-output-path
+  (is (= "../../target/dda-backup/out.yaml"
+         (cut/output-path {:name "dda-backup"
+                           :project-root-path "../.."
+                           :build-dir-name "target"
+                           :version "4.11.8-dev"
+                           :stage "dev"
+                           :debug false
+                           :dry-run false
+                           :c4k-config-filename "config.yaml"
+                           :c4k-auth-filename "auth.yaml"
+                           :c4k-output-filename "out.yaml"})))
+  (is (= "../../target/dda/backup/out.yaml"
+         (cut/output-path {:name "dda"
+                           :module "backup"
+                           :project-root-path "../.."
+                           :build-dir-name "target"
+                           :version "4.11.8-dev"
+                           :stage "dev"
+                           :debug false
+                           :dry-run false
+                           :c4k-config-filename "config.yaml"
+                           :c4k-auth-filename "auth.yaml"
+                           :c4k-output-filename "out.yaml"}))))
+
+(deftest should-calculate-clean-build-dir-command
+  (is (= ["rm" "-rf" "../../target/dda-backup"]
+         (cut/clean-build-dir-command {:name "dda-backup"
+                                       :project-root-path "../.."
+                                       :build-dir-name "target"
+                                       :version "4.11.8-dev"
+                                       :stage "dev"
+                                       :debug false
+                                       :dry-run false
+                                       :c4k-config-filename "config.yaml"
+                                       :c4k-auth-filename "auth.yaml"
+                                       :c4k-output-filename "out.yaml"}))))
+
+(deftest should-calculate-c4k-uberjar-command
+  (is (= ["bash" "-c" "c4k-dda-backup-standalone.jar ../../target/dda-backup/config.yaml ../../target/dda-backup/auth.yaml > ../../target/dda-backup/out.yaml"]
+         (cut/c4k-uberjar-command {:name "dda-backup"
+                                   :project-root-path "../.."
+                                   :build-dir-name "target"
+                                   :version "4.11.8-dev"
+                                   :stage "dev"
+                                   :debug false
+                                   :dry-run false
+                                   :c4k-config-filename "config.yaml"
+                                   :c4k-auth-filename "auth.yaml"
+                                   :c4k-output-filename "out.yaml"}))))
+
+(deftest should-calculate-c4k-graalvm-command
+  (is (= ["bash" "-c" "c4k-dda-backup ../../target/dda-backup/config.yaml ../../target/dda-backup/auth.yaml > ../../target/dda-backup/out.yaml"]
+         (cut/c4k-graalvm-command {:name "dda-backup"
+                                   :project-root-path "../.."
+                                   :build-dir-name "target"
+                                   :version "4.11.8-dev"
+                                   :stage "dev"
+                                   :debug false
+                                   :dry-run false
+                                   :c4k-config-filename "config.yaml"
+                                   :c4k-auth-filename "auth.yaml"
+                                   :c4k-output-filename "out.yaml"}))))
+

test/dda/build/gopass/domain_test.clj

@@ -0,0 +1,13 @@
+(ns dda.build.gopass.domain-test
+  (:require
+   [clojure.test :refer [deftest is]]
+   [clojure.spec.test.alpha :as st]
+   [dda.build.gopass.domain :as cut]))
+
+(st/instrument `cut/gopass-show-command)
+
+(deftest should-show-gopass-command
+  (is (= ["gopass" "show" "-y" "-o" "test/pass"]
+         (cut/gopass-show-command {:path "test/pass"})))
+  (is (= ["gopass" "show" "-y" "-o" "test/pass" "field"]
+         (cut/gopass-show-command {:path "test/pass" :field "field"}))))

test/dda/build/provs/domain_test.clj

@@ -0,0 +1,57 @@
+(ns dda.build.provs.domain-test
+  (:require
+   [clojure.test :refer [deftest is]]
+   [clojure.spec.test.alpha :as st]
+   [dda.build.provs.domain :as cut]))
+
+(st/instrument `cut/output-path)
+(st/instrument `cut/provs-server-command)
+(st/instrument `cut/create-k3s-config)
+
+(deftest should-calculate-output-path
+  (is (= "../../target/dda-backup/k3s-out.yaml"
+         (cut/output-path {:name "dda-backup"
+                           :project-root-path "../.."
+                           :build-dir-name "target"
+                           :version "4.11.8-dev"
+                           :stage "dev"
+                           :debug false
+                           :dry-run false
+                           :c4k-output-filename "out.yaml"
+                           :k3s-provision-user "root"
+                           :k3s-output-filename "k3s-out.yaml"
+                           :email "test@test.t"
+                           :echo false})))
+  (is (= "../../target/dda/backup/k3s-out.yaml"
+         (cut/output-path {:name "dda"
+                           :module "backup"
+                           :project-root-path "../.."
+                           :build-dir-name "target"
+                           :version "4.11.8-dev"
+                           :stage "dev"
+                           :debug false
+                           :dry-run false
+                           :c4k-output-filename "out.yaml"
+                           :k3s-provision-user "root"
+                           :k3s-output-filename "k3s-out.yaml"
+                           :email "test@test.t"
+                           :echo false}))))
+
+(deftest should-calculate-provs-server-command
+  (is (= ["provs-server.jar" "k3s" "root@test.test.de" "-c" "../../target/dda-backup/k3s-out.yaml" "-a" "../../target/dda-backup/out.yaml"]
+         (cut/provs-server-command {:name "dda-backup"
+                                    :project-root-path "../.."
+                                    :build-dir-name "target"
+                                    :version "4.11.8-dev"
+                                    :stage "dev"
+                                    :debug false
+                                    :dry-run false
+                                    :k3s-output-filename "k3s-out.yaml"
+                                    :k3s-provision-user "root"
+                                    :c4k-config-filename "config.yaml"
+                                    :c4k-auth-filename "auth.yaml"
+                                    :c4k-output-filename "out.yaml"
+                                    :email "test@test.t"
+                                    :echo false}
+                                   {:out {:sensitive false :type [] :value {:fqdn "test.test.de" :ipv4 "127.0.0.1" :ipv6 "::"}}}))))
+