From 866ef84f123b333a9fd021904addf65bc16c2390 Mon Sep 17 00:00:00 2001 From: Michael Jerger Date: Tue, 20 Feb 2024 12:08:49 +0100 Subject: [PATCH 01/15] use load-resource macro --- src/main/cljc/dda/c4k_common/monitoring.cljc | 22 +------------------- src/main/cljc/dda/c4k_common/postgres.cljc | 13 +----------- 2 files changed, 2 insertions(+), 33 deletions(-) diff --git a/src/main/cljc/dda/c4k_common/monitoring.cljc b/src/main/cljc/dda/c4k_common/monitoring.cljc index d966d3e..07893e5 100644 --- a/src/main/cljc/dda/c4k_common/monitoring.cljc +++ b/src/main/cljc/dda/c4k_common/monitoring.cljc @@ -48,27 +48,7 @@ #?(:cljs (defmethod yaml/load-resource :monitoring [resource-name] - (case resource-name - "monitoring/namespace.yaml" (rc/inline "monitoring/namespace.yaml") - - "monitoring/kube-state-metrics/cluster-role-binding.yaml" (rc/inline "monitoring/kube-state-metrics/cluster-role-binding.yaml") - "monitoring/kube-state-metrics/cluster-role.yaml" (rc/inline "monitoring/kube-state-metrics/cluster-role.yaml") - "monitoring/kube-state-metrics/deployment.yaml" (rc/inline "monitoring/kube-state-metrics/deployment.yaml") - "monitoring/kube-state-metrics/service-account.yaml" (rc/inline "monitoring/kube-state-metrics/service-account.yaml") - "monitoring/kube-state-metrics/service.yaml" (rc/inline "monitoring/kube-state-metrics/service.yaml") - "monitoring/node-exporter/cluster-role-binding.yaml" (rc/inline "monitoring/node-exporter/cluster-role-binding.yaml") - "monitoring/node-exporter/cluster-role.yaml" (rc/inline "monitoring/node-exporter/cluster-role.yaml") - "monitoring/node-exporter/daemon-set.yaml" (rc/inline "monitoring/node-exporter/daemon-set.yaml") - "monitoring/node-exporter/service-account.yaml" (rc/inline "monitoring/node-exporter/service-account.yaml") - "monitoring/node-exporter/service.yaml" (rc/inline "monitoring/node-exporter/service.yaml") - "monitoring/prometheus/cluster-role-binding.yaml" (rc/inline "monitoring/prometheus/cluster-role-binding.yaml") - "monitoring/prometheus/cluster-role.yaml" (rc/inline "monitoring/prometheus/cluster-role.yaml") - "monitoring/prometheus/config.yaml" (rc/inline "monitoring/prometheus/config.yaml") - "monitoring/prometheus/deployment.yaml" (rc/inline "monitoring/prometheus/deployment.yaml") - "monitoring/prometheus/prometheus.yaml" (rc/inline "monitoring/prometheus/prometheus.yaml") - "monitoring/prometheus/service-account.yaml" (rc/inline "monitoring/prometheus/service-account.yaml") - "monitoring/prometheus/service.yaml" (rc/inline "monitoring/prometheus/service.yaml") - (throw (js/Error. "Undefined Resource!"))))) + (get (inline-resources "monitoring") resource-name))) (defn-spec generate-stateful-set cp/map-or-seq? [config ::storage] diff --git a/src/main/cljc/dda/c4k_common/postgres.cljc b/src/main/cljc/dda/c4k_common/postgres.cljc index 90b114e..4e81554 100644 --- a/src/main/cljc/dda/c4k_common/postgres.cljc +++ b/src/main/cljc/dda/c4k_common/postgres.cljc @@ -35,17 +35,7 @@ #?(:cljs (defmethod yaml/load-resource :postgres [resource-name] - (case resource-name - "postgres/config-2gb.yaml" (rc/inline "postgres/config-2gb.yaml") - "postgres/config-4gb.yaml" (rc/inline "postgres/config-4gb.yaml") - "postgres/config-8gb.yaml" (rc/inline "postgres/config-8gb.yaml") - "postgres/config-16gb.yaml" (rc/inline "postgres/config-16gb.yaml") - "postgres/deployment.yaml" (rc/inline "postgres/deployment.yaml") - "postgres/persistent-volume.yaml" (rc/inline "postgres/persistent-volume.yaml") - "postgres/pvc.yaml" (rc/inline "postgres/pvc.yaml") - "postgres/secret.yaml" (rc/inline "postgres/secret.yaml") - "postgres/service.yaml" (rc/inline "postgres/service.yaml") - (throw (js/Error. "Undefined Resource!"))))) + (get (inline-resources "postgres") resource-name))) (defn-spec generate-config cp/map-or-seq? [& config (s/? pg-config?)] @@ -57,7 +47,6 @@ (str "postgres/config-" (name postgres-size) ".yaml"))) (assoc-in [:data :postgres-db] db-name)))) -; TODO: why do we need a sequence of configs? (defn-spec generate-deployment cp/map-or-seq? [& config (s/? pg-config?)] (let [{:keys [postgres-image] -- 2.45.2 From 73e2ebd12fc48ef0fda4ddd197f7c84ba8d577da Mon Sep 17 00:00:00 2001 From: Michael Jerger Date: Tue, 20 Feb 2024 15:23:42 +0100 Subject: [PATCH 02/15] introduce namespace --- src/main/cljc/dda/c4k_common/namespace.cljc | 19 +++++++++++++++ .../namespace/namespace_internal.cljc | 23 +++++++++++++++++++ .../cljc/dda/c4k_common/namespace_test.cljc | 18 +++++++++++++++ 3 files changed, 60 insertions(+) create mode 100644 src/main/cljc/dda/c4k_common/namespace.cljc create mode 100644 src/main/cljc/dda/c4k_common/namespace/namespace_internal.cljc create mode 100644 src/test/cljc/dda/c4k_common/namespace_test.cljc diff --git a/src/main/cljc/dda/c4k_common/namespace.cljc b/src/main/cljc/dda/c4k_common/namespace.cljc new file mode 100644 index 0000000..0e8638a --- /dev/null +++ b/src/main/cljc/dda/c4k_common/namespace.cljc @@ -0,0 +1,19 @@ +(ns dda.c4k-common.namespace + (:require + #?(:clj [orchestra.core :refer [defn-spec]] + :cljs [orchestra.core :refer-macros [defn-spec]]) + [clojure.spec.alpha :as s] + [dda.c4k-common.namespace.namespace-internal :as int])) + +(s/def ::namespace ::int/namespace) + +(def config? (s/keys :req-un [] + :opt-un [::namespace])) + +(def default-config {:namespace "default"}) + +(defn-spec generate seq? + [config config?] + (let [final-config (merge default-config + config)] + [(int/generate-namespace final-config)])) diff --git a/src/main/cljc/dda/c4k_common/namespace/namespace_internal.cljc b/src/main/cljc/dda/c4k_common/namespace/namespace_internal.cljc new file mode 100644 index 0000000..5476596 --- /dev/null +++ b/src/main/cljc/dda/c4k_common/namespace/namespace_internal.cljc @@ -0,0 +1,23 @@ +(ns dda.c4k-common.namespace.namespace-internal + (:require + [clojure.spec.alpha :as s] + #?(:clj [orchestra.core :refer [defn-spec]] + :cljs [orchestra.core :refer-macros [defn-spec]]) + #?(:cljs [dda.c4k-common.macros :refer-macros [inline-resources]]) + [dda.c4k-common.yaml :as yaml])) + +(s/def ::namespace string?) + +(def config? (s/keys :req-un [::namespace] + :opt-un [])) + +(defn-spec generate-namespace map? + [config config?] + (let [{:keys [namespace]} config] + (-> + (yaml/load-as-edn "namespace/namespace.yaml") + (assoc-in [:metadata :name] namespace)))) + +#?(:cljs + (defmethod yaml/load-resource :namespace [resource-name] + (get (inline-resources "namespace") resource-name))) \ No newline at end of file diff --git a/src/test/cljc/dda/c4k_common/namespace_test.cljc b/src/test/cljc/dda/c4k_common/namespace_test.cljc new file mode 100644 index 0000000..f400516 --- /dev/null +++ b/src/test/cljc/dda/c4k_common/namespace_test.cljc @@ -0,0 +1,18 @@ +(ns dda.c4k-common.namespace-test + (:require + #?(:clj [clojure.test :refer [deftest is are testing run-tests]] + :cljs [cljs.test :refer-macros [deftest is are testing run-tests]]) + [clojure.spec.test.alpha :as st] + [dda.c4k-common.namespace :as cut])) + +(st/instrument `cut/generate) + +(deftest should-generate-simple-ingress + (is (= [{:apiVersion "v1" + :kind "Namespace" + :metadata {:name "default"}}] + (cut/generate {}))) + (is (= [{:apiVersion "v1" + :kind "Namespace" + :metadata {:name "myapp"}}] + (cut/generate {:namespace "myapp"})))) \ No newline at end of file -- 2.45.2 From 6794a24f0fb16c2d9cc8c2083c8f9b2108c07cef Mon Sep 17 00:00:00 2001 From: Michael Jerger Date: Tue, 20 Feb 2024 15:23:59 +0100 Subject: [PATCH 03/15] add namespace to ingress --- doc/Architecture.md | 17 ++ src/main/cljc/dda/c4k_common/ingress.cljc | 149 +++++++----------- .../c4k_common/ingress/ingress_internal.cljc | 95 +++++++++++ src/main/resources/namespace/namespace.yaml | 4 + .../ingress/ingress_internal_test.cljc | 133 ++++++++++++++++ .../cljc/dda/c4k_common/ingress_test.cljc | 92 ++--------- 6 files changed, 317 insertions(+), 173 deletions(-) create mode 100644 src/main/cljc/dda/c4k_common/ingress/ingress_internal.cljc create mode 100644 src/main/resources/namespace/namespace.yaml create mode 100644 src/test/cljc/dda/c4k_common/ingress/ingress_internal_test.cljc diff --git a/doc/Architecture.md b/doc/Architecture.md index cb8d505..45a8e08 100644 --- a/doc/Architecture.md +++ b/doc/Architecture.md @@ -49,3 +49,20 @@ C4Context Rel(app-backup, app-db-storage, "*dbc") ``` + +# Layout of a component on example of namespace + + +```mermaid +classDiagram + + class namespace { + config? // the external representation + default-config // static defaults + generate(config, auth) seq + } + class namespace-internal { + config? // the internal representation + generate-namespace(config) map + } +``` \ No newline at end of file diff --git a/src/main/cljc/dda/c4k_common/ingress.cljc b/src/main/cljc/dda/c4k_common/ingress.cljc index a0cc521..b963038 100644 --- a/src/main/cljc/dda/c4k_common/ingress.cljc +++ b/src/main/cljc/dda/c4k_common/ingress.cljc @@ -3,117 +3,76 @@ [clojure.spec.alpha :as s] #?(:clj [orchestra.core :refer [defn-spec]] :cljs [orchestra.core :refer-macros [defn-spec]]) - #?(:cljs [dda.c4k-common.macros :refer-macros [inline-resources]]) - [dda.c4k-common.yaml :as yaml] - [dda.c4k-common.common :as cm] - [dda.c4k-common.predicate :as pred])) + [dda.c4k-common.namespace :as ns] + [dda.c4k-common.ingress.ingress-internal :as int])) -(s/def ::issuer pred/letsencrypt-issuer?) -(s/def ::service-name string?) -(s/def ::app-name string?) -(s/def ::ingress-name string?) -(s/def ::cert-name string?) -(s/def ::service-port pos-int?) -(s/def ::fqdns (s/coll-of pred/fqdn-string?)) -(s/def ::average-rate pos-int?) -(s/def ::burst-rate pos-int?) +(s/def ::issuer ::int/issuer) +(s/def ::service-name ::int/service-name) +(s/def ::app-name ::int/app-name) +(s/def ::ingress-name ::int/ingress-name) +(s/def ::cert-name ::int/cert-name) +(s/def ::service-port ::int/service-port) +(s/def ::fqdns ::int/fqdns) +(s/def ::average-rate ::int/average-rate) +(s/def ::burst-rate ::int/burst-rate) (def simple-ingress? (s/keys :req-un [::fqdns ::service-name ::service-port] - :opt-un [::issuer ::average-rate])) + :opt-un [::issuer ::average-rate ::ns/namespace])) (def ingress? (s/keys :req-un [::fqdns ::app-name ::ingress-name ::service-name ::service-port] - :opt-un [::issuer ::cert-name ::rate-limit-name])) + :opt-un [::issuer ::cert-name ::rate-limit-name ::ns/namespace])) (def certificate? (s/keys :req-un [::fqdns ::app-name ::cert-name] - :opt-un [::issuer])) + :opt-un [::issuer ::ns/namespace])) (def rate-limit-config? (s/keys :req-un [::rate-limit-name ::average-rate ::burst-rate])) -(def simple-ingress-defaults {:issuer "staging" - :average-rate 10}) +(def default-config + (merge ns/default-config + {:issuer "staging" + :average-rate 10})) -#?(:cljs - (defmethod yaml/load-resource :ingress [resource-name] - (get (inline-resources "ingress") resource-name))) -(defn-spec generate-host-rule pred/map-or-seq? - [service-name ::service-name - service-port ::service-port - fqdn pred/fqdn-string?] - (-> - (yaml/load-as-edn "ingress/host-rule.yaml") - (cm/replace-all-matching-values-by-new-value "FQDN" fqdn) - (cm/replace-all-matching-values-by-new-value "SERVICE_PORT" service-port) - (cm/replace-all-matching-values-by-new-value "SERVICE_NAME" service-name))) - -(defn-spec generate-rate-limit-middleware pred/map-or-seq? - [config rate-limit-config?] - (let [{:keys [rate-limit-name average-rate burst-rate]} config] - (-> - (yaml/load-as-edn "ingress/middleware-ratelimit.yaml") - (assoc-in [:metadata :name] (str rate-limit-name "-ratelimit")) - (assoc-in [:spec :rateLimit :average] average-rate) - (assoc-in [:spec :rateLimit :burst] burst-rate)))) - -(defn-spec generate-certificate pred/map-or-seq? +(defn-spec generate-certificate map? [config certificate?] - (let [{:keys [cert-name issuer fqdns app-name] - :or {issuer "staging"}} config - letsencrypt-issuer (name issuer)] - (-> - (yaml/load-as-edn "ingress/certificate.yaml") - (assoc-in [:metadata :name] cert-name) - (assoc-in [:metadata :labels :app.kubernetes.part-of] app-name) - (assoc-in [:spec :secretName] cert-name) - (assoc-in [:spec :commonName] (first fqdns)) - (assoc-in [:spec :dnsNames] fqdns) - (assoc-in [:spec :issuerRef :name] letsencrypt-issuer)))) + (let [final-config (merge default-config + config)] + (int/generate-certificate final-config))) -(defn-spec generate-ingress pred/map-or-seq? + +(defn-spec generate-ingress map? [config ingress?] - (let [{:keys [ingress-name cert-name service-name service-port fqdns app-name rate-limit-name]} config] - (-> - (yaml/load-as-edn "ingress/ingress.yaml") - (assoc-in [:metadata :name] ingress-name) - (assoc-in [:metadata :labels :app.kubernetes.part-of] app-name) - (assoc-in [:metadata :annotations] - {:traefik.ingress.kubernetes.io/router.entrypoints - "web, websecure" - :traefik.ingress.kubernetes.io/router.middlewares - (if rate-limit-name - (str "default-redirect-https@kubernetescrd, " rate-limit-name "-ratelimit@kubernetescrd") - "default-redirect-https@kubernetescrd") - :metallb.universe.tf/address-pool "public"}) - (assoc-in [:spec :tls 0 :secretName] cert-name) - (assoc-in [:spec :tls 0 :hosts] fqdns) - (assoc-in [:spec :rules] - (mapv (partial generate-host-rule service-name service-port) fqdns))))) + (let [final-config (merge default-config + config)] + (int/generate-ingress final-config))) -(defn-spec generate-ingress-and-cert any? - [simple-ingress-config simple-ingress?] - (let [{:keys [service-name]} simple-ingress-config - config (merge {:app-name service-name - :ingress-name service-name - :cert-name service-name} - simple-ingress-defaults - simple-ingress-config)] - [(generate-certificate config) - (generate-ingress config)])) -(defn-spec generate-simple-ingress any? - [simple-ingress-config simple-ingress?] - (let [{:keys [service-name]} simple-ingress-config - config (merge {:app-name service-name - :ingress-name service-name - :cert-name service-name - :rate-limit-name service-name} - simple-ingress-defaults - simple-ingress-config) - {:keys [average-rate]} config] - [(generate-certificate config) - (generate-rate-limit-middleware {:rate-limit-name service-name - :average-rate average-rate - :burst-rate average-rate}) - (generate-ingress config)])) \ No newline at end of file +(defn-spec generate-ingress-and-cert seq? + [config simple-ingress?] + (let [{:keys [service-name]} config + final-config (merge {:app-name service-name + :ingress-name service-name + :cert-name service-name} + default-config + config)] + [(int/generate-certificate final-config) + (int/generate-ingress final-config)])) + + +(defn-spec generate-simple-ingress seq? + [config simple-ingress?] + (let [{:keys [service-name]} config + final-config (merge {:app-name service-name + :ingress-name service-name + :cert-name service-name + :rate-limit-name service-name} + default-config + config) + {:keys [average-rate]} final-config] + [(int/generate-certificate final-config) + (int/generate-rate-limit-middleware {:rate-limit-name service-name + :average-rate average-rate + :burst-rate average-rate}) + (int/generate-ingress final-config)])) \ No newline at end of file diff --git a/src/main/cljc/dda/c4k_common/ingress/ingress_internal.cljc b/src/main/cljc/dda/c4k_common/ingress/ingress_internal.cljc new file mode 100644 index 0000000..e725029 --- /dev/null +++ b/src/main/cljc/dda/c4k_common/ingress/ingress_internal.cljc @@ -0,0 +1,95 @@ +(ns dda.c4k-common.ingress.ingress-internal + (:require + [clojure.spec.alpha :as s] + #?(:clj [orchestra.core :refer [defn-spec]] + :cljs [orchestra.core :refer-macros [defn-spec]]) + #?(:cljs [dda.c4k-common.macros :refer-macros [inline-resources]]) + [dda.c4k-common.yaml :as yaml] + [dda.c4k-common.common :as cm] + [dda.c4k-common.namespace :as ns] + [dda.c4k-common.predicate :as pred])) + +(s/def ::issuer pred/letsencrypt-issuer?) +(s/def ::service-name string?) +(s/def ::app-name string?) +(s/def ::ingress-name string?) +(s/def ::cert-name string?) +(s/def ::service-port pos-int?) +(s/def ::fqdns (s/coll-of pred/fqdn-string?)) +(s/def ::average-rate pos-int?) +(s/def ::burst-rate pos-int?) + +(def ingress? (s/keys :req-un [::ingress-name ::app-name + ::ns/namespace + ::service-name ::service-port + ::issuer ::cert-name + ::fqdns] + :opt-un [::rate-limit-name])) + +(def certificate? (s/keys :req-un [::fqdns ::app-name ::cert-name ::issuer ::ns/namespace])) + +(def rate-limit-config? (s/keys :req-un [::rate-limit-name ::average-rate ::burst-rate])) + + +(defn-spec generate-host-rule map? + [service-name ::service-name + service-port ::service-port + fqdn pred/fqdn-string?] + (-> + (yaml/load-as-edn "ingress/host-rule.yaml") + (cm/replace-all-matching-values-by-new-value "FQDN" fqdn) + (cm/replace-all-matching-values-by-new-value "SERVICE_PORT" service-port) + (cm/replace-all-matching-values-by-new-value "SERVICE_NAME" service-name))) + + +(defn-spec generate-certificate map? + [config certificate?] + (let [{:keys [cert-name issuer fqdns app-name namespace]} config + letsencrypt-issuer (name issuer)] + (-> + (yaml/load-as-edn "ingress/certificate.yaml") + (assoc-in [:metadata :name] cert-name) + (assoc-in [:metadata :namespace] namespace) + (assoc-in [:metadata :labels :app.kubernetes.part-of] app-name) + (assoc-in [:spec :secretName] cert-name) + (assoc-in [:spec :commonName] (first fqdns)) + (assoc-in [:spec :dnsNames] fqdns) + (assoc-in [:spec :issuerRef :name] letsencrypt-issuer)))) + + +(defn-spec generate-rate-limit-middleware map? + [config rate-limit-config?] + (let [{:keys [rate-limit-name average-rate burst-rate]} config] + (-> + (yaml/load-as-edn "ingress/middleware-ratelimit.yaml") + (assoc-in [:metadata :name] (str rate-limit-name "-ratelimit")) + (assoc-in [:spec :rateLimit :average] average-rate) + (assoc-in [:spec :rateLimit :burst] burst-rate)))) + + +(defn-spec generate-ingress map? + [config ingress?] + (let [{:keys [ingress-name cert-name service-name service-port + fqdns app-name rate-limit-name namespace]} config] + (-> + (yaml/load-as-edn "ingress/ingress.yaml") + (assoc-in [:metadata :name] ingress-name) + (assoc-in [:metadata :namespace] namespace) + (assoc-in [:metadata :labels :app.kubernetes.part-of] app-name) + (assoc-in [:metadata :annotations] + {:traefik.ingress.kubernetes.io/router.entrypoints + "web, websecure" + :traefik.ingress.kubernetes.io/router.middlewares + (if rate-limit-name + (str "default-redirect-https@kubernetescrd, " rate-limit-name "-ratelimit@kubernetescrd") + "default-redirect-https@kubernetescrd") + :metallb.universe.tf/address-pool "public"}) + (assoc-in [:spec :tls 0 :secretName] cert-name) + (assoc-in [:spec :tls 0 :hosts] fqdns) + (assoc-in [:spec :rules] + (mapv (partial generate-host-rule service-name service-port) fqdns))))) + + +#?(:cljs + (defmethod yaml/load-resource :ingress [resource-name] + (get (inline-resources "ingress") resource-name))) diff --git a/src/main/resources/namespace/namespace.yaml b/src/main/resources/namespace/namespace.yaml new file mode 100644 index 0000000..cf6d1f2 --- /dev/null +++ b/src/main/resources/namespace/namespace.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: default \ No newline at end of file diff --git a/src/test/cljc/dda/c4k_common/ingress/ingress_internal_test.cljc b/src/test/cljc/dda/c4k_common/ingress/ingress_internal_test.cljc new file mode 100644 index 0000000..d7615ff --- /dev/null +++ b/src/test/cljc/dda/c4k_common/ingress/ingress_internal_test.cljc @@ -0,0 +1,133 @@ +(ns dda.c4k-common.ingress.ingress-internal-test + (:require + #?(:clj [clojure.test :refer [deftest is are testing run-tests]] + :cljs [cljs.test :refer-macros [deftest is are testing run-tests]]) + [clojure.spec.test.alpha :as st] + [dda.c4k-common.ingress.ingress-internal :as cut])) + +(st/instrument `cut/generate-host-rule) +(st/instrument `cut/generate-certificate) +(st/instrument `cut/generate-rate-limit-middleware) +(st/instrument `cut/generate-ingress) + + +(deftest should-generate-rule + (is (= {:host "test.com", + :http + {:paths + [{:pathType "Prefix", + :path "/", + :backend + {:service {:name "myservice", :port {:number 3000}}}}]}} + (cut/generate-host-rule "myservice" 3000 "test.com")))) + + +(deftest should-generate-certificate + (is (= {:apiVersion "cert-manager.io/v1", + :kind "Certificate", + :metadata {:name "test-io-cert", + :namespace "default", + :labels {:app.kubernetes.part-of "c4k-common-app"}}, + :spec + {:secretName "test-io-cert", + :commonName "test.de", + :duration "2160h", + :renewBefore "720h", + :dnsNames ["test.de" "test.org" "www.test.de" "www.test.org"], + :issuerRef {:name "prod", :kind "ClusterIssuer"}}} + (cut/generate-certificate {:fqdns ["test.de" "test.org" "www.test.de" "www.test.org"] + :app-name "c4k-common-app" + :cert-name "test-io-cert" + :issuer "prod" + :namespace "default"}))) + (is (= {:apiVersion "cert-manager.io/v1", + :kind "Certificate", + :metadata {:name "test-io-cert", + :namespace "myapp", + :labels {:app.kubernetes.part-of "c4k-common-app"}}, + :spec + {:secretName "test-io-cert", + :commonName "test.de", + :duration "2160h", + :renewBefore "720h", + :dnsNames ["test.de" "test.org" "www.test.de" "www.test.org"], + :issuerRef {:name "prod", :kind "ClusterIssuer"}}} + (cut/generate-certificate {:fqdns ["test.de" "test.org" "www.test.de" "www.test.org"] + :app-name "c4k-common-app" + :cert-name "test-io-cert" + :issuer "prod" + :namespace "myapp"})))) + + +(deftest should-generate-middleware-ratelimit + (is (= {:apiVersion "traefik.containo.us/v1alpha1", + :kind "Middleware", + :metadata {:name "normal-ratelimit"}, + :spec {:rateLimit {:average 10, :burst 5}}} + (cut/generate-rate-limit-middleware {:rate-limit-name "normal" + :average-rate 10, :burst-rate 5})))) + + +(deftest should-generate-ingress + (is (= {:apiVersion "networking.k8s.io/v1", + :kind "Ingress", + :metadata + {:namespace "myapp", + :name "test-io-https-ingress", + :labels {:app.kubernetes.part-of "c4k-common-app"}, + :annotations {:traefik.ingress.kubernetes.io/router.entrypoints + "web, websecure" + :traefik.ingress.kubernetes.io/router.middlewares + "default-redirect-https@kubernetescrd" + :metallb.universe.tf/address-pool "public"}}} + (dissoc (cut/generate-ingress + {:ingress-name "test-io-https-ingress" + :app-name "c4k-common-app" + :namespace "myapp" + :service-name "test-io-service" :service-port 80 + :issuer "prod" :cert-name "noname" + :fqdns ["test.de" "www.test.de" "test-it.de" + "www.test-it.de"]}) :spec))) + (is (= {:name "test-io-https-ingress", + :namespace "default", + :labels {:app.kubernetes.part-of "c4k-common-app"}, + :annotations {:traefik.ingress.kubernetes.io/router.entrypoints + "web, websecure" + :traefik.ingress.kubernetes.io/router.middlewares + "default-redirect-https@kubernetescrd, normal-ratelimit@kubernetescrd", + :metallb.universe.tf/address-pool "public"}} + (:metadata (cut/generate-ingress + { + :ingress-name "test-io-https-ingress" + :app-name "c4k-common-app" + :namespace "default" + :service-name "test-io-service" :service-port 80 + :rate-limit-name "normal" + :issuer "prod" :cert-name "noname" + :fqdns ["test.de"]})))) + (is (= {:tls + [{:hosts + ["test.de" "www.test.de" "test-it.de" "www.test-it.de"], + :secretName "test-io-cert"}] + :rules + [{:host "test.de", + :http + {:paths [{:pathType "Prefix", :path "/", :backend {:service {:name "test-io-service", :port {:number 80}}}}]}} + {:host "www.test.de", + :http + {:paths [{:pathType "Prefix", :path "/", :backend {:service {:name "test-io-service", :port {:number 80}}}}]}} + {:host "test-it.de", + :http + {:paths [{:pathType "Prefix", :path "/", :backend {:service {:name "test-io-service", :port {:number 80}}}}]}} + {:host "www.test-it.de", + :http + {:paths [{:pathType "Prefix", :path "/", :backend {:service {:name "test-io-service", :port {:number 80}}}}]}}]} + (:spec (cut/generate-ingress { + :ingress-name "test-io-https-ingress" + :app-name "c4k-common-app" + :namespace "default" + :service-name "test-io-service" :service-port 80 + :issuer "prod" :cert-name "test-io-cert" + :fqdns ["test.de" "www.test.de" + "test-it.de" + "www.test-it.de"]}))))) \ No newline at end of file diff --git a/src/test/cljc/dda/c4k_common/ingress_test.cljc b/src/test/cljc/dda/c4k_common/ingress_test.cljc index 126152c..fcc106a 100644 --- a/src/test/cljc/dda/c4k_common/ingress_test.cljc +++ b/src/test/cljc/dda/c4k_common/ingress_test.cljc @@ -5,20 +5,10 @@ [clojure.spec.test.alpha :as st] [dda.c4k-common.ingress :as cut])) -(st/instrument `cut/generate-host-rule) (st/instrument `cut/generate-ingress) (st/instrument `cut/generate-certificate) (st/instrument `cut/generate-ingress-and-cert) - -(deftest should-generate-rule - (is (= {:host "test.com", - :http - {:paths - [{:pathType "Prefix", - :path "/", - :backend - {:service {:name "myservice", :port {:number 3000}}}}]}} - (cut/generate-host-rule "myservice" 3000 "test.com")))) +(st/instrument `cut/generate-simple-ingress) (deftest should-generate-certificate (is (= {:apiVersion "cert-manager.io/v1", @@ -31,84 +21,30 @@ :commonName "test.de", :duration "2160h", :renewBefore "720h", - :dnsNames ["test.de" "test.org" "www.test.de" "www.test.org"], - :issuerRef {:name "prod", :kind "ClusterIssuer"}}} - (cut/generate-certificate {:fqdns ["test.de" "test.org" "www.test.de" "www.test.org"] + :dnsNames ["test.de"], + :issuerRef {:name "staging", :kind "ClusterIssuer"}}} + (cut/generate-certificate {:fqdns ["test.de"] :app-name "c4k-common-app" - :cert-name "test-io-cert" - :issuer "prod"})))) + :cert-name "test-io-cert"})))) -(deftest should-generate-middleware-ratelimit - (is (= {:apiVersion "traefik.containo.us/v1alpha1", - :kind "Middleware", - :metadata {:name "normal-ratelimit"}, - :spec {:rateLimit {:average 10, :burst 5}}} - (cut/generate-rate-limit-middleware {:rate-limit-name "normal" - :average-rate 10, :burst-rate 5})))) (deftest should-generate-ingress - (is (= {:apiVersion "networking.k8s.io/v1", - :kind "Ingress", - :metadata - { - :namespace "default", - :name "test-io-https-ingress", - :labels {:app.kubernetes.part-of "c4k-common-app"}, - :annotations {:traefik.ingress.kubernetes.io/router.entrypoints - "web, websecure" - :traefik.ingress.kubernetes.io/router.middlewares - "default-redirect-https@kubernetescrd" - :metallb.universe.tf/address-pool "public"}}} - (dissoc (cut/generate-ingress - {:issuer "prod" - :service-name "test-io-service" - :app-name "c4k-common-app" - :service-port 80 - :ingress-name "test-io-https-ingress" - :fqdns ["test.de" "www.test.de" "test-it.de" - "www.test-it.de"]}) :spec))) - (is (= { - :name "test-io-https-ingress", + (is (= {:name "test-io-https-ingress", :namespace "default", :labels {:app.kubernetes.part-of "c4k-common-app"}, :annotations {:traefik.ingress.kubernetes.io/router.entrypoints "web, websecure" :traefik.ingress.kubernetes.io/router.middlewares - "default-redirect-https@kubernetescrd, normal-ratelimit@kubernetescrd", + "default-redirect-https@kubernetescrd", :metallb.universe.tf/address-pool "public"}} (:metadata (cut/generate-ingress - {:service-name "test-io-service" - :app-name "c4k-common-app" - :service-port 80 - :ingress-name "test-io-https-ingress" - :rate-limit-name "normal" - :fqdns ["test.de"]})))) - (is (= {:tls - [{:hosts - ["test.de" "www.test.de" "test-it.de" "www.test-it.de"], - :secretName "test-io-cert"}] - :rules - [{:host "test.de", - :http - {:paths [{:pathType "Prefix", :path "/", :backend {:service {:name "test-io-service", :port {:number 80}}}}]}} - {:host "www.test.de", - :http - {:paths [{:pathType "Prefix", :path "/", :backend {:service {:name "test-io-service", :port {:number 80}}}}]}} - {:host "test-it.de", - :http - {:paths [{:pathType "Prefix", :path "/", :backend {:service {:name "test-io-service", :port {:number 80}}}}]}} - {:host "www.test-it.de", - :http - {:paths [{:pathType "Prefix", :path "/", :backend {:service {:name "test-io-service", :port {:number 80}}}}]}}]} - (:spec (cut/generate-ingress {:issuer "prod" - :app-name "c4k-common-app" - :service-name "test-io-service" - :service-port 80 - :ingress-name "test-io-https-ingress" - :cert-name "test-io-cert" - :fqdns ["test.de" "www.test.de" - "test-it.de" - "www.test-it.de"]}))))) + {:ingress-name "test-io-https-ingress" + :app-name "c4k-common-app" + :service-name "test-io-service" :service-port 80 + :cert-name "myCert" + :fqdns ["test.de"]}))))) + + (deftest should-generate-ingress-and-cert (is (= [{:apiVersion "cert-manager.io/v1", :kind "Certificate", -- 2.45.2 From b4718500b1e2b4e445d845d65d5e2a6c35a5c033 Mon Sep 17 00:00:00 2001 From: Michael Jerger Date: Tue, 20 Feb 2024 16:03:56 +0100 Subject: [PATCH 04/15] fix cljs for postgresql --- src/main/cljc/dda/c4k_common/postgres.cljc | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/cljc/dda/c4k_common/postgres.cljc b/src/main/cljc/dda/c4k_common/postgres.cljc index 4e81554..48d7d30 100644 --- a/src/main/cljc/dda/c4k_common/postgres.cljc +++ b/src/main/cljc/dda/c4k_common/postgres.cljc @@ -4,6 +4,7 @@ #?(:cljs [shadow.resource :as rc]) #?(:clj [orchestra.core :refer [defn-spec]] :cljs [orchestra.core :refer-macros [defn-spec]]) + #?(:cljs [dda.c4k-common.macros :refer-macros [inline-resources]]) [dda.c4k-common.yaml :as yaml] [dda.c4k-common.base64 :as b64] [dda.c4k-common.predicate :as cp] -- 2.45.2 From ab2473bdf4c4a7c90fafbaada97eeb31eaf6739e Mon Sep 17 00:00:00 2001 From: Michael Jerger Date: Tue, 20 Feb 2024 16:04:28 +0100 Subject: [PATCH 05/15] introduce internal for monitoring --- src/main/cljc/dda/c4k_common/monitoring.cljc | 81 +++++-------------- .../monitoring/monitoring_internal.cljc | 77 ++++++++++++++++++ ...e-state-metrics-cluster-role-binding.yaml} | 0 ...l => kube-state-metrics-cluster-role.yaml} | 0 ...aml => kube-state-metrics-deployment.yaml} | 0 ...> kube-state-metrics-service-account.yaml} | 0 ...e.yaml => kube-state-metrics-service.yaml} | 0 ...> node-exporter-cluster-role-binding.yaml} | 0 ...e.yaml => node-exporter-cluster-role.yaml} | 0 ...set.yaml => node-exporter-daemon-set.yaml} | 0 ...aml => node-exporter-service-account.yaml} | 0 ...ervice.yaml => node-exporter-service.yaml} | 0 ...l => prometheus-cluster-role-binding.yaml} | 0 ...role.yaml => prometheus-cluster-role.yaml} | 0 .../config.yaml => prometheus-config.yaml} | 0 ...oyment.yaml => prometheus-deployment.yaml} | 0 ...etheus.yaml => prometheus-prometheus.yaml} | 0 ...t.yaml => prometheus-service-account.yaml} | 0 .../service.yaml => prometheus-service.yaml} | 0 .../monitoring/monitoring_internal_test.cljc | 59 ++++++++++++++ .../cljc/dda/c4k_common/monitoring_test.cljc | 46 +---------- 21 files changed, 157 insertions(+), 106 deletions(-) create mode 100644 src/main/cljc/dda/c4k_common/monitoring/monitoring_internal.cljc rename src/main/resources/monitoring/{kube-state-metrics/cluster-role-binding.yaml => kube-state-metrics-cluster-role-binding.yaml} (100%) rename src/main/resources/monitoring/{kube-state-metrics/cluster-role.yaml => kube-state-metrics-cluster-role.yaml} (100%) rename src/main/resources/monitoring/{kube-state-metrics/deployment.yaml => kube-state-metrics-deployment.yaml} (100%) rename src/main/resources/monitoring/{kube-state-metrics/service-account.yaml => kube-state-metrics-service-account.yaml} (100%) rename src/main/resources/monitoring/{kube-state-metrics/service.yaml => kube-state-metrics-service.yaml} (100%) rename src/main/resources/monitoring/{node-exporter/cluster-role-binding.yaml => node-exporter-cluster-role-binding.yaml} (100%) rename src/main/resources/monitoring/{node-exporter/cluster-role.yaml => node-exporter-cluster-role.yaml} (100%) rename src/main/resources/monitoring/{node-exporter/daemon-set.yaml => node-exporter-daemon-set.yaml} (100%) rename src/main/resources/monitoring/{node-exporter/service-account.yaml => node-exporter-service-account.yaml} (100%) rename src/main/resources/monitoring/{node-exporter/service.yaml => node-exporter-service.yaml} (100%) rename src/main/resources/monitoring/{prometheus/cluster-role-binding.yaml => prometheus-cluster-role-binding.yaml} (100%) rename src/main/resources/monitoring/{prometheus/cluster-role.yaml => prometheus-cluster-role.yaml} (100%) rename src/main/resources/monitoring/{prometheus/config.yaml => prometheus-config.yaml} (100%) rename src/main/resources/monitoring/{prometheus/deployment.yaml => prometheus-deployment.yaml} (100%) rename src/main/resources/monitoring/{prometheus/prometheus.yaml => prometheus-prometheus.yaml} (100%) rename src/main/resources/monitoring/{prometheus/service-account.yaml => prometheus-service-account.yaml} (100%) rename src/main/resources/monitoring/{prometheus/service.yaml => prometheus-service.yaml} (100%) create mode 100644 src/test/cljc/dda/c4k_common/monitoring/monitoring_internal_test.cljc diff --git a/src/main/cljc/dda/c4k_common/monitoring.cljc b/src/main/cljc/dda/c4k_common/monitoring.cljc index 07893e5..1536443 100644 --- a/src/main/cljc/dda/c4k_common/monitoring.cljc +++ b/src/main/cljc/dda/c4k_common/monitoring.cljc @@ -1,20 +1,18 @@ (ns dda.c4k-common.monitoring (:require [clojure.spec.alpha :as s] - #?(:cljs [shadow.resource :as rc]) #?(:clj [orchestra.core :refer [defn-spec]] :cljs [orchestra.core :refer-macros [defn-spec]]) + [clojure.string :as str] [dda.c4k-common.yaml :as yaml] [dda.c4k-common.predicate :as cp] - [dda.c4k-common.common :as cm] - [clojure.string :as str])) + [dda.c4k-common.monitoring.monitoring-internal :as int])) (s/def ::grafana-cloud-user cp/bash-env-string?) (s/def ::grafana-cloud-password cp/bash-env-string?) (s/def ::grafana-cloud-url string?) (s/def ::cluster-name string?) (s/def ::cluster-stage cp/stage?) -(s/def ::pvc-storage-class-name cp/pvc-storage-class-name?) (s/def ::node-regex string?) (s/def ::traefik-regex string?) (s/def ::kube-state-regex string?) @@ -23,11 +21,11 @@ ::cluster-stage])) (s/def ::mon-auth (s/keys :req-un [::grafana-cloud-user ::grafana-cloud-password])) -(s/def ::storage (s/keys :opt-un [::pvc-storage-class-name])) (s/def ::filter-regex (s/keys :req-un [::node-regex ::traefik-regex ::kube-state-regex])) + (def metric-regex {:node-regex (str "node_cpu_sec.+|node_load[0-9]+|node_memory_Buf.*|node_memory_Mem.*|" "node_memory_Cached.*|node_disk_[r,w,i].*|node_filesystem_[s,a].*|" @@ -46,63 +44,24 @@ (def filter-regex-string (str/join "|" (vals metric-regex))) -#?(:cljs - (defmethod yaml/load-resource :monitoring [resource-name] - (get (inline-resources "monitoring") resource-name))) -(defn-spec generate-stateful-set cp/map-or-seq? - [config ::storage] - (let [{:keys [pvc-storage-class-name] - :or {pvc-storage-class-name :manual}} config] - (-> - (yaml/load-as-edn "monitoring/stateful-set.yaml") - (assoc-in [:spec :volumeClaimTemplates 0 :spec :storageClassName] (name pvc-storage-class-name))))) - -(defn-spec generate-prometheus-config cp/map-or-seq? - [config ::mon-cfg - auth ::mon-auth] - (let [{:keys [grafana-cloud-url cluster-name cluster-stage]} config - {:keys [grafana-cloud-user grafana-cloud-password]} auth] - (-> - (yaml/load-as-edn "monitoring/prometheus/prometheus.yaml") - (assoc-in [:global :external_labels :cluster] - cluster-name) - (assoc-in [:global :external_labels :stage] - cluster-stage) - (assoc-in [:remote_write 0 :url] - grafana-cloud-url) - (assoc-in [:remote_write 0 :basic_auth :username] - grafana-cloud-user) - (assoc-in [:remote_write 0 :basic_auth :password] - grafana-cloud-password) - (cm/replace-all-matching-values-by-new-value "FILTER_REGEX" filter-regex-string)))) - -(defn-spec generate-config cp/map-or-seq? - [config ::mon-cfg - auth ::mon-auth] - (-> - (yaml/load-as-edn "monitoring/prometheus/config.yaml") - (assoc-in [:stringData :prometheus.yaml] - (yaml/to-string - (generate-prometheus-config config auth))))) - -(defn-spec generate cp/map-or-seq? +(defn-spec generate seq? [config ::mon-cfg auth ::mon-auth] [(yaml/load-as-edn "monitoring/namespace.yaml") - (yaml/load-as-edn "monitoring/prometheus/cluster-role.yaml") - (yaml/load-as-edn "monitoring/prometheus/cluster-role-binding.yaml") - (yaml/load-as-edn "monitoring/prometheus/service.yaml") - (yaml/load-as-edn "monitoring/prometheus/service-account.yaml") - (generate-config config auth) - (yaml/load-as-edn "monitoring/prometheus/deployment.yaml") - (yaml/load-as-edn "monitoring/node-exporter/service-account.yaml") - (yaml/load-as-edn "monitoring/node-exporter/cluster-role.yaml") - (yaml/load-as-edn "monitoring/node-exporter/cluster-role-binding.yaml") - (yaml/load-as-edn "monitoring/node-exporter/daemon-set.yaml") - (yaml/load-as-edn "monitoring/node-exporter/service.yaml") - (yaml/load-as-edn "monitoring/kube-state-metrics/cluster-role-binding.yaml") - (yaml/load-as-edn "monitoring/kube-state-metrics/cluster-role.yaml") - (yaml/load-as-edn "monitoring/kube-state-metrics/deployment.yaml") - (yaml/load-as-edn "monitoring/kube-state-metrics/service-account.yaml") - (yaml/load-as-edn "monitoring/kube-state-metrics/service.yaml")]) + (yaml/load-as-edn "monitoring/prometheus-cluster-role.yaml") + (yaml/load-as-edn "monitoring/prometheus-cluster-role-binding.yaml") + (yaml/load-as-edn "monitoring/prometheus-service.yaml") + (yaml/load-as-edn "monitoring/prometheus-service-account.yaml") + (int/generate-config config auth) + (yaml/load-as-edn "monitoring/prometheus-deployment.yaml") + (yaml/load-as-edn "monitoring/node-exporter-service-account.yaml") + (yaml/load-as-edn "monitoring/node-exporter-cluster-role.yaml") + (yaml/load-as-edn "monitoring/node-exporter-cluster-role-binding.yaml") + (yaml/load-as-edn "monitoring/node-exporter-daemon-set.yaml") + (yaml/load-as-edn "monitoring/node-exporter-service.yaml") + (yaml/load-as-edn "monitoring/kube-state-metrics-cluster-role-binding.yaml") + (yaml/load-as-edn "monitoring/kube-state-metrics-cluster-role.yaml") + (yaml/load-as-edn "monitoring/kube-state-metrics-deployment.yaml") + (yaml/load-as-edn "monitoring/kube-state-metrics-service-account.yaml") + (yaml/load-as-edn "monitoring/kube-state-metrics-service.yaml")]) diff --git a/src/main/cljc/dda/c4k_common/monitoring/monitoring_internal.cljc b/src/main/cljc/dda/c4k_common/monitoring/monitoring_internal.cljc new file mode 100644 index 0000000..6dfc3a3 --- /dev/null +++ b/src/main/cljc/dda/c4k_common/monitoring/monitoring_internal.cljc @@ -0,0 +1,77 @@ +(ns dda.c4k-common.monitoring.monitoring-internal + (:require + [clojure.spec.alpha :as s] + #?(:clj [orchestra.core :refer [defn-spec]] + :cljs [orchestra.core :refer-macros [defn-spec]]) + #?(:cljs [dda.c4k-common.macros :refer-macros [inline-resources]]) + [dda.c4k-common.yaml :as yaml] + [dda.c4k-common.predicate :as cp] + [dda.c4k-common.common :as cm] + [clojure.string :as str])) + +(s/def ::grafana-cloud-user cp/bash-env-string?) +(s/def ::grafana-cloud-password cp/bash-env-string?) +(s/def ::grafana-cloud-url string?) +(s/def ::cluster-name string?) +(s/def ::cluster-stage cp/stage?) +(s/def ::node-regex string?) +(s/def ::traefik-regex string?) +(s/def ::kube-state-regex string?) +(s/def ::mon-cfg (s/keys :req-un [::grafana-cloud-url + ::cluster-name + ::cluster-stage])) +(s/def ::mon-auth (s/keys :req-un [::grafana-cloud-user + ::grafana-cloud-password])) +(s/def ::filter-regex (s/keys :req-un [::node-regex + ::traefik-regex + ::kube-state-regex])) + +(def metric-regex {:node-regex + (str "node_cpu_sec.+|node_load[0-9]+|node_memory_Buf.*|node_memory_Mem.*|" + "node_memory_Cached.*|node_disk_[r,w,i].*|node_filesystem_[s,a].*|" + "node_network_receive_bytes_total|node_network_transmit_bytes_total") + :traefik-regex (str "traefik_entrypoint_.*_total|" + "traefik_entrypoint_.*_seconds_count|" + "traefik_router_.*_total|" + "traefik_router_.*_seconds_count|" + "traefik_service_.*_total|" + "traefik_service_.*_seconds_count|" + "traefik_tls_certs_not_after") + :kube-state-regex (str "kube_pod_container_status_restarts_total|" + "kube_pod_status_reason|kube_node_status_capacity|kube_node_status_allocatable|" + "kube_cronjob_status_active|kube_job_status_failed")}) + +(def filter-regex-string + (str/join "|" (vals metric-regex))) + +(defn-spec generate-prometheus-config cp/map-or-seq? + [config ::mon-cfg + auth ::mon-auth] + (let [{:keys [grafana-cloud-url cluster-name cluster-stage]} config + {:keys [grafana-cloud-user grafana-cloud-password]} auth] + (-> + (yaml/load-as-edn "monitoring/prometheus-prometheus.yaml") + (assoc-in [:global :external_labels :cluster] + cluster-name) + (assoc-in [:global :external_labels :stage] + cluster-stage) + (assoc-in [:remote_write 0 :url] + grafana-cloud-url) + (assoc-in [:remote_write 0 :basic_auth :username] + grafana-cloud-user) + (assoc-in [:remote_write 0 :basic_auth :password] + grafana-cloud-password) + (cm/replace-all-matching-values-by-new-value "FILTER_REGEX" filter-regex-string)))) + +(defn-spec generate-config cp/map-or-seq? + [config ::mon-cfg + auth ::mon-auth] + (-> + (yaml/load-as-edn "monitoring/prometheus-config.yaml") + (assoc-in [:stringData :prometheus.yaml] + (yaml/to-string + (generate-prometheus-config config auth))))) + +#?(:cljs + (defmethod yaml/load-resource :monitoring [resource-name] + (get (inline-resources "monitoring") resource-name))) diff --git a/src/main/resources/monitoring/kube-state-metrics/cluster-role-binding.yaml b/src/main/resources/monitoring/kube-state-metrics-cluster-role-binding.yaml similarity index 100% rename from src/main/resources/monitoring/kube-state-metrics/cluster-role-binding.yaml rename to src/main/resources/monitoring/kube-state-metrics-cluster-role-binding.yaml diff --git a/src/main/resources/monitoring/kube-state-metrics/cluster-role.yaml b/src/main/resources/monitoring/kube-state-metrics-cluster-role.yaml similarity index 100% rename from src/main/resources/monitoring/kube-state-metrics/cluster-role.yaml rename to src/main/resources/monitoring/kube-state-metrics-cluster-role.yaml diff --git a/src/main/resources/monitoring/kube-state-metrics/deployment.yaml b/src/main/resources/monitoring/kube-state-metrics-deployment.yaml similarity index 100% rename from src/main/resources/monitoring/kube-state-metrics/deployment.yaml rename to src/main/resources/monitoring/kube-state-metrics-deployment.yaml diff --git a/src/main/resources/monitoring/kube-state-metrics/service-account.yaml b/src/main/resources/monitoring/kube-state-metrics-service-account.yaml similarity index 100% rename from src/main/resources/monitoring/kube-state-metrics/service-account.yaml rename to src/main/resources/monitoring/kube-state-metrics-service-account.yaml diff --git a/src/main/resources/monitoring/kube-state-metrics/service.yaml b/src/main/resources/monitoring/kube-state-metrics-service.yaml similarity index 100% rename from src/main/resources/monitoring/kube-state-metrics/service.yaml rename to src/main/resources/monitoring/kube-state-metrics-service.yaml diff --git a/src/main/resources/monitoring/node-exporter/cluster-role-binding.yaml b/src/main/resources/monitoring/node-exporter-cluster-role-binding.yaml similarity index 100% rename from src/main/resources/monitoring/node-exporter/cluster-role-binding.yaml rename to src/main/resources/monitoring/node-exporter-cluster-role-binding.yaml diff --git a/src/main/resources/monitoring/node-exporter/cluster-role.yaml b/src/main/resources/monitoring/node-exporter-cluster-role.yaml similarity index 100% rename from src/main/resources/monitoring/node-exporter/cluster-role.yaml rename to src/main/resources/monitoring/node-exporter-cluster-role.yaml diff --git a/src/main/resources/monitoring/node-exporter/daemon-set.yaml b/src/main/resources/monitoring/node-exporter-daemon-set.yaml similarity index 100% rename from src/main/resources/monitoring/node-exporter/daemon-set.yaml rename to src/main/resources/monitoring/node-exporter-daemon-set.yaml diff --git a/src/main/resources/monitoring/node-exporter/service-account.yaml b/src/main/resources/monitoring/node-exporter-service-account.yaml similarity index 100% rename from src/main/resources/monitoring/node-exporter/service-account.yaml rename to src/main/resources/monitoring/node-exporter-service-account.yaml diff --git a/src/main/resources/monitoring/node-exporter/service.yaml b/src/main/resources/monitoring/node-exporter-service.yaml similarity index 100% rename from src/main/resources/monitoring/node-exporter/service.yaml rename to src/main/resources/monitoring/node-exporter-service.yaml diff --git a/src/main/resources/monitoring/prometheus/cluster-role-binding.yaml b/src/main/resources/monitoring/prometheus-cluster-role-binding.yaml similarity index 100% rename from src/main/resources/monitoring/prometheus/cluster-role-binding.yaml rename to src/main/resources/monitoring/prometheus-cluster-role-binding.yaml diff --git a/src/main/resources/monitoring/prometheus/cluster-role.yaml b/src/main/resources/monitoring/prometheus-cluster-role.yaml similarity index 100% rename from src/main/resources/monitoring/prometheus/cluster-role.yaml rename to src/main/resources/monitoring/prometheus-cluster-role.yaml diff --git a/src/main/resources/monitoring/prometheus/config.yaml b/src/main/resources/monitoring/prometheus-config.yaml similarity index 100% rename from src/main/resources/monitoring/prometheus/config.yaml rename to src/main/resources/monitoring/prometheus-config.yaml diff --git a/src/main/resources/monitoring/prometheus/deployment.yaml b/src/main/resources/monitoring/prometheus-deployment.yaml similarity index 100% rename from src/main/resources/monitoring/prometheus/deployment.yaml rename to src/main/resources/monitoring/prometheus-deployment.yaml diff --git a/src/main/resources/monitoring/prometheus/prometheus.yaml b/src/main/resources/monitoring/prometheus-prometheus.yaml similarity index 100% rename from src/main/resources/monitoring/prometheus/prometheus.yaml rename to src/main/resources/monitoring/prometheus-prometheus.yaml diff --git a/src/main/resources/monitoring/prometheus/service-account.yaml b/src/main/resources/monitoring/prometheus-service-account.yaml similarity index 100% rename from src/main/resources/monitoring/prometheus/service-account.yaml rename to src/main/resources/monitoring/prometheus-service-account.yaml diff --git a/src/main/resources/monitoring/prometheus/service.yaml b/src/main/resources/monitoring/prometheus-service.yaml similarity index 100% rename from src/main/resources/monitoring/prometheus/service.yaml rename to src/main/resources/monitoring/prometheus-service.yaml diff --git a/src/test/cljc/dda/c4k_common/monitoring/monitoring_internal_test.cljc b/src/test/cljc/dda/c4k_common/monitoring/monitoring_internal_test.cljc new file mode 100644 index 0000000..b763456 --- /dev/null +++ b/src/test/cljc/dda/c4k_common/monitoring/monitoring_internal_test.cljc @@ -0,0 +1,59 @@ +(ns dda.c4k-common.monitoring.monitoring-internal-test + (:require + #?(:clj [clojure.test :refer [deftest is are testing run-tests]] + :cljs [cljs.test :refer-macros [deftest is are testing run-tests]]) + [clojure.string :as str] + [clojure.spec.test.alpha :as st] + [dda.c4k-common.monitoring.monitoring-internal :as cut])) + +(st/instrument `cut/generate-stateful-set) +(st/instrument `cut/generate-agent-config) +(st/instrument `cut/generate-config) + +(def conf {:cluster-name "clustername" + :cluster-stage "test" + :grafana-cloud-url "https://some.url/with/path"}) + +(def auth {:grafana-cloud-user "user" + :grafana-cloud-password "password" + :hetzner-cloud-ro-token "ro-token"}) + +(def invalid-conf {:cluster-name "clustername" + :cluster-stage "test" + :grafana-clud-url "https://some.url/with/path"}) + +(def invalid-auth {:grafana-cloud-user "user" + :grafana-clod-password "password" + :hetzner-cloud-ro-token "ro-token"}) + +(deftest should-not-generate-config + (is (thrown? + #?(:clj Exception :cljs js/Error) + (cut/generate-config invalid-conf auth)))) + +(deftest should-not-generate-auth + (is (thrown? + #?(:clj Exception :cljs js/Error) + (cut/generate-config conf invalid-auth)))) + + +(deftest should-generate-prometheus-remote-write-auth + (is (= {:username "user", + :password "password"} + (get-in + (cut/generate-prometheus-config conf auth) + [:remote_write 0 :basic_auth])))) + +(deftest should-generate-prometheus-external-labels + (is (= {:cluster "clustername", + :stage "test"} + (get-in + (cut/generate-prometheus-config conf auth) + [:global :external_labels])))) + +(deftest should-generate-config + (is (str/starts-with? + (get-in + (cut/generate-config conf auth) + [:stringData :prometheus.yaml]) + "global:\n scrape_interval:"))) \ No newline at end of file diff --git a/src/test/cljc/dda/c4k_common/monitoring_test.cljc b/src/test/cljc/dda/c4k_common/monitoring_test.cljc index 76dfa2c..5e5332b 100644 --- a/src/test/cljc/dda/c4k_common/monitoring_test.cljc +++ b/src/test/cljc/dda/c4k_common/monitoring_test.cljc @@ -2,16 +2,10 @@ (:require #?(:clj [clojure.test :refer [deftest is are testing run-tests]] :cljs [cljs.test :refer-macros [deftest is are testing run-tests]]) - [clojure.string :as s] [clojure.spec.test.alpha :as st] - [dda.c4k-common.monitoring :as cut] - [dda.c4k-common.yaml :as yaml] - [clojure.string :as str])) + [dda.c4k-common.monitoring :as cut])) (st/instrument `cut/generate) -(st/instrument `cut/generate-stateful-set) -(st/instrument `cut/generate-agent-config) -(st/instrument `cut/generate-config) (def conf {:cluster-name "clustername" :cluster-stage "test" @@ -21,45 +15,7 @@ :grafana-cloud-password "password" :hetzner-cloud-ro-token "ro-token"}) -(def invalid-conf {:cluster-name "clustername" - :cluster-stage "test" - :grafana-clud-url "https://some.url/with/path"}) - -(def invalid-auth {:grafana-cloud-user "user" - :grafana-clod-password "password" - :hetzner-cloud-ro-token "ro-token"}) - -(deftest should-not-generate-config - (is (thrown? - #?(:clj Exception :cljs js/Error) - (cut/generate-config invalid-conf auth)))) - -(deftest should-not-generate-auth - (is (thrown? - #?(:clj Exception :cljs js/Error) - (cut/generate-config conf invalid-auth)))) (deftest should-generate (is (= 17 (count (cut/generate conf auth))))) - -(deftest should-generate-prometheus-remote-write-auth - (is (= {:username "user", - :password "password"} - (get-in - (cut/generate-prometheus-config conf auth) - [:remote_write 0 :basic_auth])))) - -(deftest should-generate-prometheus-external-labels - (is (= {:cluster "clustername", - :stage "test"} - (get-in - (cut/generate-prometheus-config conf auth) - [:global :external_labels])))) - -(deftest should-generate-config - (is (s/starts-with? - (get-in - (cut/generate-config conf auth) - [:stringData :prometheus.yaml]) - "global:\n scrape_interval:"))) \ No newline at end of file -- 2.45.2 From a14b7640daa8ca7002eda637ee76e6b8ce7e2959 Mon Sep 17 00:00:00 2001 From: Michael Jerger Date: Tue, 20 Feb 2024 16:13:36 +0100 Subject: [PATCH 06/15] update deps --- project.clj | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/project.clj b/project.clj index e9e3556..5f4215a 100644 --- a/project.clj +++ b/project.clj @@ -4,7 +4,7 @@ :license {:name "Apache License, Version 2.0" :url "https://www.apache.org/licenses/LICENSE-2.0.html"} :dependencies [[org.clojure/clojure "1.11.1" :scope "provided"] - [org.clojure/tools.reader "1.3.7"] + [org.clojure/tools.reader "1.4.0"] [aero "1.1.6"] [orchestra "2021.01.01-1"] [expound "0.9.0"] @@ -25,8 +25,8 @@ :uberjar {:aot :all :main dda.c4k-common.uberjar :uberjar-name "c4k-common-standalone.jar" - :dependencies [[org.clojure/tools.cli "1.0.219"] - [ch.qos.logback/logback-classic "1.4.14" + :dependencies [[org.clojure/tools.cli "1.1.230"] + [ch.qos.logback/logback-classic "1.5.0" :exclusions [com.sun.mail/javax.mail]] [org.slf4j/jcl-over-slf4j "2.0.12"]]}} :release-tasks [["test"] -- 2.45.2 From 8626b91b16fcfa97e85d80b79a04ddf9a3e19e60 Mon Sep 17 00:00:00 2001 From: Michael Jerger Date: Tue, 20 Feb 2024 16:14:15 +0100 Subject: [PATCH 07/15] refactor to dry --- src/main/cljc/dda/c4k_common/monitoring.cljc | 36 ++++--------------- .../monitoring/monitoring_internal.cljc | 10 ++---- .../monitoring_regex_test.clj | 4 +-- .../should_filter_metrik.edn | 0 4 files changed, 10 insertions(+), 40 deletions(-) rename src/test/clj/dda/c4k_common/{ => monitoring}/monitoring_regex_test.clj (88%) rename src/test/resources/dda/c4k_common/{ => monitoring}/monitoring_regex_test/should_filter_metrik.edn (100%) diff --git a/src/main/cljc/dda/c4k_common/monitoring.cljc b/src/main/cljc/dda/c4k_common/monitoring.cljc index 1536443..a3c70a4 100644 --- a/src/main/cljc/dda/c4k_common/monitoring.cljc +++ b/src/main/cljc/dda/c4k_common/monitoring.cljc @@ -3,46 +3,22 @@ [clojure.spec.alpha :as s] #?(:clj [orchestra.core :refer [defn-spec]] :cljs [orchestra.core :refer-macros [defn-spec]]) - [clojure.string :as str] [dda.c4k-common.yaml :as yaml] - [dda.c4k-common.predicate :as cp] [dda.c4k-common.monitoring.monitoring-internal :as int])) -(s/def ::grafana-cloud-user cp/bash-env-string?) -(s/def ::grafana-cloud-password cp/bash-env-string?) -(s/def ::grafana-cloud-url string?) -(s/def ::cluster-name string?) -(s/def ::cluster-stage cp/stage?) -(s/def ::node-regex string?) -(s/def ::traefik-regex string?) -(s/def ::kube-state-regex string?) +(s/def ::grafana-cloud-user ::int/grafana-cloud-user) +(s/def ::grafana-cloud-password ::int/grafana-cloud-password) +(s/def ::grafana-cloud-url ::int/grafana-cloud-url) +(s/def ::cluster-name ::int/cluster-name) +(s/def ::cluster-stage ::int/cluster-stage) (s/def ::mon-cfg (s/keys :req-un [::grafana-cloud-url ::cluster-name ::cluster-stage])) (s/def ::mon-auth (s/keys :req-un [::grafana-cloud-user ::grafana-cloud-password])) -(s/def ::filter-regex (s/keys :req-un [::node-regex - ::traefik-regex - ::kube-state-regex])) -(def metric-regex {:node-regex - (str "node_cpu_sec.+|node_load[0-9]+|node_memory_Buf.*|node_memory_Mem.*|" - "node_memory_Cached.*|node_disk_[r,w,i].*|node_filesystem_[s,a].*|" - "node_network_receive_bytes_total|node_network_transmit_bytes_total") - :traefik-regex (str "traefik_entrypoint_.*_total|" - "traefik_entrypoint_.*_seconds_count|" - "traefik_router_.*_total|" - "traefik_router_.*_seconds_count|" - "traefik_service_.*_total|" - "traefik_service_.*_seconds_count|" - "traefik_tls_certs_not_after") - :kube-state-regex (str "kube_pod_container_status_restarts_total|" - "kube_pod_status_reason|kube_node_status_capacity|kube_node_status_allocatable|" - "kube_cronjob_status_active|kube_job_status_failed")}) - -(def filter-regex-string - (str/join "|" (vals metric-regex))) +(def filter-regex-string int/filter-regex-string) (defn-spec generate seq? diff --git a/src/main/cljc/dda/c4k_common/monitoring/monitoring_internal.cljc b/src/main/cljc/dda/c4k_common/monitoring/monitoring_internal.cljc index 6dfc3a3..023e027 100644 --- a/src/main/cljc/dda/c4k_common/monitoring/monitoring_internal.cljc +++ b/src/main/cljc/dda/c4k_common/monitoring/monitoring_internal.cljc @@ -14,17 +14,11 @@ (s/def ::grafana-cloud-url string?) (s/def ::cluster-name string?) (s/def ::cluster-stage cp/stage?) -(s/def ::node-regex string?) -(s/def ::traefik-regex string?) -(s/def ::kube-state-regex string?) (s/def ::mon-cfg (s/keys :req-un [::grafana-cloud-url ::cluster-name ::cluster-stage])) (s/def ::mon-auth (s/keys :req-un [::grafana-cloud-user ::grafana-cloud-password])) -(s/def ::filter-regex (s/keys :req-un [::node-regex - ::traefik-regex - ::kube-state-regex])) (def metric-regex {:node-regex (str "node_cpu_sec.+|node_load[0-9]+|node_memory_Buf.*|node_memory_Mem.*|" @@ -44,7 +38,7 @@ (def filter-regex-string (str/join "|" (vals metric-regex))) -(defn-spec generate-prometheus-config cp/map-or-seq? +(defn-spec generate-prometheus-config map? [config ::mon-cfg auth ::mon-auth] (let [{:keys [grafana-cloud-url cluster-name cluster-stage]} config @@ -63,7 +57,7 @@ grafana-cloud-password) (cm/replace-all-matching-values-by-new-value "FILTER_REGEX" filter-regex-string)))) -(defn-spec generate-config cp/map-or-seq? +(defn-spec generate-config map? [config ::mon-cfg auth ::mon-auth] (-> diff --git a/src/test/clj/dda/c4k_common/monitoring_regex_test.clj b/src/test/clj/dda/c4k_common/monitoring/monitoring_regex_test.clj similarity index 88% rename from src/test/clj/dda/c4k_common/monitoring_regex_test.clj rename to src/test/clj/dda/c4k_common/monitoring/monitoring_regex_test.clj index 196a911..50f03b1 100644 --- a/src/test/clj/dda/c4k_common/monitoring_regex_test.clj +++ b/src/test/clj/dda/c4k_common/monitoring/monitoring_regex_test.clj @@ -1,8 +1,8 @@ -(ns dda.c4k-common.monitoring-regex-test +(ns dda.c4k-common.monitoring.monitoring-regex-test (:require [clojure.test :refer [deftest is are testing run-tests]] [data-test :refer :all] - [dda.c4k-common.monitoring :as cut])) + [dda.c4k-common.monitoring.monitoring-internal :as cut])) (defn filter-by-regex [regex-str collection] diff --git a/src/test/resources/dda/c4k_common/monitoring_regex_test/should_filter_metrik.edn b/src/test/resources/dda/c4k_common/monitoring/monitoring_regex_test/should_filter_metrik.edn similarity index 100% rename from src/test/resources/dda/c4k_common/monitoring_regex_test/should_filter_metrik.edn rename to src/test/resources/dda/c4k_common/monitoring/monitoring_regex_test/should_filter_metrik.edn -- 2.45.2 From cca93311e44ec858877c60e6a33b8e735791005c Mon Sep 17 00:00:00 2001 From: Michael Jerger Date: Tue, 20 Feb 2024 17:08:39 +0100 Subject: [PATCH 08/15] seperate postgres & postgres-internal --- src/main/cljc/dda/c4k_common/postgres.cljc | 100 +++++++------- .../postgres/postgres_internal.cljc | 89 +++++++++++++ .../postgres/postgres_internal_test.cljc | 122 ++++++++++++++++++ .../cljc/dda/c4k_common/postgres_test.cljc | 101 +-------------- 4 files changed, 259 insertions(+), 153 deletions(-) create mode 100644 src/main/cljc/dda/c4k_common/postgres/postgres_internal.cljc create mode 100644 src/test/cljc/dda/c4k_common/postgres/postgres_internal_test.cljc diff --git a/src/main/cljc/dda/c4k_common/postgres.cljc b/src/main/cljc/dda/c4k_common/postgres.cljc index 48d7d30..1a736cf 100644 --- a/src/main/cljc/dda/c4k_common/postgres.cljc +++ b/src/main/cljc/dda/c4k_common/postgres.cljc @@ -8,23 +8,20 @@ [dda.c4k-common.yaml :as yaml] [dda.c4k-common.base64 :as b64] [dda.c4k-common.predicate :as cp] - [dda.c4k-common.common :as cm])) + [dda.c4k-common.common :as cm] + [dda.c4k-common.postgres.postgres-internal :as int])) -(defn postgres-size? - [input] - (contains? #{:2gb :4gb :8gb :16gb} input)) +(def postgres-size? int/postgres-size?) -(defn postgres-image? - [input] - (contains? #{"postgres:13" "postgres:14" "postgres:15" "postgres:16"} input)) +(def postgres-image? int/postgres-image?) -(s/def ::postgres-db-user cp/bash-env-string?) -(s/def ::postgres-db-password cp/bash-env-string?) -(s/def ::postgres-data-volume-path string?) -(s/def ::postgres-size postgres-size?) -(s/def ::db-name cp/bash-env-string?) -(s/def ::pvc-storage-class-name cp/pvc-storage-class-name?) -(s/def ::pv-storage-size-gb pos?) +(s/def ::postgres-db-user ::int/postgres-db-user) +(s/def ::postgres-db-password ::int/postgres-db-password) +(s/def ::postgres-data-volume-path ::int/postgres-data-volume-path) +(s/def ::postgres-size ::int/postgres-size) +(s/def ::db-name ::int/db-name) +(s/def ::pvc-storage-class-name ::int/pvc-storage-class-name) +(s/def ::pv-storage-size-gb ::int/pv-storage-size-gb) (def pg-config? (s/keys :opt-un [::postgres-size ::db-name ::postgres-data-volume-path @@ -34,56 +31,47 @@ (def postgres-function (s/keys :opt-un [::deserializer ::optional])) -#?(:cljs - (defmethod yaml/load-resource :postgres [resource-name] - (get (inline-resources "postgres") resource-name))) +(def default-config {:postgres-image "postgres:13" + :postgres-size :2gb + :db-name "postgres" + :postgres-data-volume-path "/var/postgres" + :pv-storage-size-gb 10 + :pvc-storage-class-name "manual"}) -(defn-spec generate-config cp/map-or-seq? + +(defn-spec generate-config map? [& config (s/? pg-config?)] - (let [{:keys [postgres-size db-name] - :or {postgres-size :2gb - db-name "postgres"}} (first config)] - (-> - (yaml/from-string (yaml/load-resource - (str "postgres/config-" (name postgres-size) ".yaml"))) - (assoc-in [:data :postgres-db] db-name)))) + (let [final-config (merge default-config + (first config))] + (int/generate-config final-config))) -(defn-spec generate-deployment cp/map-or-seq? + +(defn-spec generate-deployment map? [& config (s/? pg-config?)] - (let [{:keys [postgres-image] - :or {postgres-image "postgres:13"}} (first config)] - (-> - (yaml/from-string (yaml/load-resource "postgres/deployment.yaml")) - (assoc-in [:spec :template :spec :containers 0 :image] postgres-image)))) + (let [final-config (merge default-config + (first config))] + (int/generate-deployment final-config))) -(defn-spec generate-persistent-volume cp/map-or-seq? + +(defn-spec generate-persistent-volume map? [config pg-config?] - (let [{:keys [postgres-data-volume-path pv-storage-size-gb] - :or {postgres-data-volume-path "/var/postgres" - pv-storage-size-gb 10}} config] - (-> - (yaml/from-string (yaml/load-resource "postgres/persistent-volume.yaml")) - (assoc-in [:spec :hostPath :path] postgres-data-volume-path) - (assoc-in [:spec :capacity :storage] (str pv-storage-size-gb "Gi"))))) + (let [final-config (merge default-config + config)] + (int/generate-persistent-volume final-config))) -(defn-spec generate-pvc cp/map-or-seq? + +(defn-spec generate-pvc map? [config pg-config?] - (let [{:keys [pv-storage-size-gb pvc-storage-class-name] - :or {pv-storage-size-gb 10 - pvc-storage-class-name "manual"}} config] - (-> - (yaml/from-string (yaml/load-resource "postgres/pvc.yaml")) - (assoc-in [:spec :resources :requests :storage] (str pv-storage-size-gb "Gi")) - (assoc-in [:spec :storageClassName] (name pvc-storage-class-name))))) + (let [final-config (merge default-config + config)] + (int/generate-pvc final-config))) -(defn-spec generate-secret cp/map-or-seq? - [my-auth any?] - (let [{:keys [postgres-db-user postgres-db-password]} my-auth] - (-> - (yaml/from-string (yaml/load-resource "postgres/secret.yaml")) - (cm/replace-key-value :postgres-user (b64/encode postgres-db-user)) - (cm/replace-key-value :postgres-password (b64/encode postgres-db-password))))) -(defn-spec generate-service cp/map-or-seq? +(defn-spec generate-secret map? + [my-auth pg-auth?] + (int/generate-secret my-auth)) + + +(defn-spec generate-service map? [] - (yaml/from-string (yaml/load-resource "postgres/service.yaml"))) + (int/generate-service)) diff --git a/src/main/cljc/dda/c4k_common/postgres/postgres_internal.cljc b/src/main/cljc/dda/c4k_common/postgres/postgres_internal.cljc new file mode 100644 index 0000000..0fde44c --- /dev/null +++ b/src/main/cljc/dda/c4k_common/postgres/postgres_internal.cljc @@ -0,0 +1,89 @@ +(ns dda.c4k-common.postgres.postgres-internal + (:require + [clojure.spec.alpha :as s] + #?(:clj [orchestra.core :refer [defn-spec]] + :cljs [orchestra.core :refer-macros [defn-spec]]) + #?(:cljs [dda.c4k-common.macros :refer-macros [inline-resources]]) + [dda.c4k-common.yaml :as yaml] + [dda.c4k-common.base64 :as b64] + [dda.c4k-common.predicate :as cp] + [dda.c4k-common.common :as cm] + [dda.c4k-common.postgres.postgres-internal :as int])) + +(defn postgres-size? + [input] + (contains? #{:2gb :4gb :8gb :16gb} input)) + +(defn postgres-image? + [input] + (contains? #{"postgres:13" "postgres:14" "postgres:15" "postgres:16"} input)) + +(s/def ::postgres-db-user cp/bash-env-string?) +(s/def ::postgres-db-password cp/bash-env-string?) +(s/def ::postgres-data-volume-path string?) +(s/def ::postgres-size postgres-size?) +(s/def ::db-name cp/bash-env-string?) +(s/def ::pvc-storage-class-name cp/pvc-storage-class-name?) +(s/def ::pv-storage-size-gb pos?) + +(def pg-config? + (s/keys :req-un [::postgres-size ::db-name ::postgres-data-volume-path + ::pvc-storage-class-name ::pv-storage-size-gb])) +(def pg-auth? + (s/keys :opt-un [::postgres-db-user ::postgres-db-password])) + +(def postgres-function (s/keys :opt-un [::deserializer ::optional])) + + +(defn-spec generate-config map? + [config pg-config?] + (let [{:keys [postgres-size db-name]} config] + (-> + (yaml/from-string (yaml/load-resource + (str "postgres/config-" (name postgres-size) ".yaml"))) + (assoc-in [:data :postgres-db] db-name)))) + + +(defn-spec generate-deployment map? + [config pg-config?] + (let [{:keys [postgres-image]} config] + (-> + (yaml/from-string (yaml/load-resource "postgres/deployment.yaml")) + (assoc-in [:spec :template :spec :containers 0 :image] postgres-image)))) + + +(defn-spec generate-persistent-volume map? + [config pg-config?] + (let [{:keys [postgres-data-volume-path pv-storage-size-gb]} config] + (-> + (yaml/from-string (yaml/load-resource "postgres/persistent-volume.yaml")) + (assoc-in [:spec :hostPath :path] postgres-data-volume-path) + (assoc-in [:spec :capacity :storage] (str pv-storage-size-gb "Gi"))))) + + +(defn-spec generate-pvc map? + [config pg-config?] + (let [{:keys [pv-storage-size-gb pvc-storage-class-name]} config] + (-> + (yaml/from-string (yaml/load-resource "postgres/pvc.yaml")) + (assoc-in [:spec :resources :requests :storage] (str pv-storage-size-gb "Gi")) + (assoc-in [:spec :storageClassName] (name pvc-storage-class-name))))) + + +(defn-spec generate-secret map? + [my-auth pg-auth?] + (let [{:keys [postgres-db-user postgres-db-password]} my-auth] + (-> + (yaml/from-string (yaml/load-resource "postgres/secret.yaml")) + (cm/replace-key-value :postgres-user (b64/encode postgres-db-user)) + (cm/replace-key-value :postgres-password (b64/encode postgres-db-password))))) + + +(defn-spec generate-service map? + [] + (yaml/from-string (yaml/load-resource "postgres/service.yaml"))) + + +#?(:cljs + (defmethod yaml/load-resource :postgres [resource-name] + (get (inline-resources "postgres") resource-name))) diff --git a/src/test/cljc/dda/c4k_common/postgres/postgres_internal_test.cljc b/src/test/cljc/dda/c4k_common/postgres/postgres_internal_test.cljc new file mode 100644 index 0000000..22bdc7e --- /dev/null +++ b/src/test/cljc/dda/c4k_common/postgres/postgres_internal_test.cljc @@ -0,0 +1,122 @@ +(ns dda.c4k-common.postgres.postgres-internal-test + (:require + #?(:clj [clojure.test :refer [deftest is are testing run-tests]] + :cljs [cljs.test :refer-macros [deftest is are testing run-tests]]) + [clojure.spec.test.alpha :as st] + [dda.c4k-common.test-helper :as ct] + [dda.c4k-common.postgres.postgres-internal :as cut])) + +(st/instrument `cut/generate-config) +(st/instrument `cut/generate-deployment) +(st/instrument `cut/generate-persistent-volume) +(st/instrument `cut/generate-pvc) +(st/instrument `cut/generate-secret) +(st/instrument `cut/generate-service) + +(deftest should-generate-config + (is (= {:postgres-db "postgres" + :postgresql.conf + "max_connections = 100\nwork_mem = 4MB\nshared_buffers = 512MB\n"} + (:data (cut/generate-config {:postgres-image "postgres:13" + :postgres-size :2gb + :db-name "postgres" + :postgres-data-volume-path "/var/postgres" + :pv-storage-size-gb 10 + :pvc-storage-class-name "manual"})))) + (is (= {:postgres-db "postgres" + :postgresql.conf + "max_connections = 700\nwork_mem = 3MB\nshared_buffers = 2048MB\n"} + (:data (cut/generate-config {:postgres-image "postgres:13" + :postgres-size :8gb + :db-name "postgres" + :postgres-data-volume-path "/var/postgres" + :pv-storage-size-gb 10 + :pvc-storage-class-name "manual"})))) + (is (= {:postgres-db "test" + :postgresql.conf + "max_connections = 100\nwork_mem = 4MB\nshared_buffers = 512MB\n"} + (:data (cut/generate-config {:postgres-image "postgres:13" + :postgres-size :2gb + :db-name "test" + :postgres-data-volume-path "/var/postgres" + :pv-storage-size-gb 10 + :pvc-storage-class-name "manual"})))) + ) + +(deftest should-generate-deployment + (is (= [{:image "postgres:14" + :name "postgresql" + :env + [{:name "POSTGRES_USER" + :valueFrom + {:secretKeyRef + {:name "postgres-secret", :key "postgres-user"}}} + {:name "POSTGRES_PASSWORD" + :valueFrom + {:secretKeyRef + {:name "postgres-secret", :key "postgres-password"}}} + {:name "POSTGRES_DB" + :valueFrom + {:configMapKeyRef + {:name "postgres-config", :key "postgres-db"}}}] + :ports [{:containerPort 5432, :name "postgresql"}] + :volumeMounts + [{:name "postgres-config-volume" + :mountPath "/etc/postgresql/postgresql.conf" + :subPath "postgresql.conf" + :readOnly true} + {:name "postgre-data-volume" + :mountPath "/var/lib/postgresql/data"}]}] + (get-in (cut/generate-deployment {:postgres-image "postgres:14" + :postgres-size :2gb + :db-name "test" + :postgres-data-volume-path "/var/postgres" + :pv-storage-size-gb 10 + :pvc-storage-class-name "manual"}) + [:spec :template :spec :containers])))) + + + +(deftest should-generate-persistent-volume + (is (= {:kind "PersistentVolume" + :apiVersion "v1" + :metadata + {:name "postgres-pv-volume", :labels {:type "local"}} + :spec + {:storageClassName "manual" + :accessModes ["ReadWriteOnce"] + :capacity {:storage "20Gi"} + :hostPath {:path "xx"}}} + (cut/generate-persistent-volume {:postgres-image "postgres:14" + :postgres-size :2gb + :db-name "test" + :pvc-storage-class-name "manual" + :postgres-data-volume-path "xx" + :pv-storage-size-gb 20})))) + + +(deftest should-generate-persistent-volume-claim + (is (= {:apiVersion "v1" + :kind "PersistentVolumeClaim" + :metadata + {:name "postgres-claim", :labels {:app "postgres"}} + :spec + {:storageClassName "local-path" + :accessModes ["ReadWriteOnce"] + :resources {:requests {:storage "20Gi"}}}} + (cut/generate-pvc {:postgres-image "postgres:13" + :postgres-size :2gb + :db-name "postgres" + :postgres-data-volume-path "/var/postgres" + :pv-storage-size-gb 20 + :pvc-storage-class-name "local-path"})))) + + +(deftest should-generate-secret + (is (= {:apiVersion "v1" + :kind "Secret" + :metadata {:name "postgres-secret"} + :type "Opaque" + :data + {:postgres-user "eHgtdXM=", :postgres-password "eHgtcHc="}} + (cut/generate-secret {:postgres-db-user "xx-us" :postgres-db-password "xx-pw"})))) diff --git a/src/test/cljc/dda/c4k_common/postgres_test.cljc b/src/test/cljc/dda/c4k_common/postgres_test.cljc index 650544d..6ee7c7e 100644 --- a/src/test/cljc/dda/c4k_common/postgres_test.cljc +++ b/src/test/cljc/dda/c4k_common/postgres_test.cljc @@ -7,33 +7,14 @@ [dda.c4k-common.postgres :as cut])) (st/instrument `cut/generate-config) -(st/instrument `cut/generate-deployment) (st/instrument `cut/generate-persistent-volume) -(st/instrument `cut/generate-pvc) -(st/instrument `cut/generate-secret) -(st/instrument `cut/generate-service) +;;(st/instrument `cut/generate) (deftest should-generate-config (is (= {:postgres-db "postgres" :postgresql.conf "max_connections = 100\nwork_mem = 4MB\nshared_buffers = 512MB\n"} - (:data (cut/generate-config)))) - (is (= {:postgres-db "postgres" - :postgresql.conf - "max_connections = 700\nwork_mem = 3MB\nshared_buffers = 2048MB\n"} - (:data (cut/generate-config {:postgres-size :8gb})))) - (is (= {:postgres-db "test" - :postgresql.conf - "max_connections = 100\nwork_mem = 4MB\nshared_buffers = 512MB\n"} - (:data (cut/generate-config {:db-name "test"})))) - ) - -(deftest should-generate-config-diff - (is (= {:postgres-db-c1 "postgres", - :postgres-db-c2 "test", - :postgresql.conf-c1 "max_connections = 100\nwork_mem = 4MB\nshared_buffers = 512MB\n", - :postgresql.conf-c2 "max_connections = 700\nwork_mem = 3MB\nshared_buffers = 2048MB\n"} - (ct/map-diff (cut/generate-config) (cut/generate-config {:db-name "test" :postgres-size :8gb}))))) + (:data (cut/generate-config))))) (deftest should-generate-persistent-volume (is (= {:kind "PersistentVolume" @@ -45,25 +26,8 @@ :accessModes ["ReadWriteOnce"] :capacity {:storage "10Gi"} :hostPath {:path "xx"}}} - (cut/generate-persistent-volume {:postgres-data-volume-path "xx"}))) - (is (= {:kind "PersistentVolume" - :apiVersion "v1" - :metadata - {:name "postgres-pv-volume", :labels {:type "local"}} - :spec - {:storageClassName "manual" - :accessModes ["ReadWriteOnce"] - :capacity {:storage "20Gi"} - :hostPath {:path "xx"}}} - (cut/generate-persistent-volume {:postgres-data-volume-path "xx" - :pv-storage-size-gb 20})))) + (cut/generate-persistent-volume {:postgres-data-volume-path "xx"})))) -(deftest should-generate-persistent-volume-diff - (is (= {:storage-c1 "10Gi", :storage-c2 "20Gi", - :path-c1 "/var/postgres", :path-c2 "xx"} - (ct/map-diff (cut/generate-persistent-volume {}) - (cut/generate-persistent-volume {:postgres-data-volume-path "xx" - :pv-storage-size-gb 20}))))) (deftest should-generate-persistent-volume-claim (is (= {:apiVersion "v1" @@ -74,61 +38,4 @@ {:storageClassName "manual" :accessModes ["ReadWriteOnce"] :resources {:requests {:storage "10Gi"}}}} - (cut/generate-pvc {}))) - (is (= {:apiVersion "v1" - :kind "PersistentVolumeClaim" - :metadata - {:name "postgres-claim", :labels {:app "postgres"}} - :spec - {:storageClassName "local-path" - :accessModes ["ReadWriteOnce"] - :resources {:requests {:storage "20Gi"}}}} - (cut/generate-pvc {:pv-storage-size-gb 20 - :pvc-storage-class-name "local-path"})))) - -(deftest should-generate-persistent-volume-claim-diff - (is (= {:storageClassName-c1 "manual", :storageClassName-c2 "local-path", - :storage-c1 "10Gi", :storage-c2 "20Gi"} - (ct/map-diff (cut/generate-pvc {}) - (cut/generate-pvc {:pv-storage-size-gb 20 - :pvc-storage-class-name "local-path"}))))) - -(deftest should-generate-secret - (is (= {:apiVersion "v1" - :kind "Secret" - :metadata {:name "postgres-secret"} - :type "Opaque" - :data - {:postgres-user "eHgtdXM=", :postgres-password "eHgtcHc="}} - (cut/generate-secret {:postgres-db-user "xx-us" :postgres-db-password "xx-pw"})))) - -(deftest should-generate-deployment - (is (= [{:image "postgres:14" - :name "postgresql" - :env - [{:name "POSTGRES_USER" - :valueFrom - {:secretKeyRef - {:name "postgres-secret", :key "postgres-user"}}} - {:name "POSTGRES_PASSWORD" - :valueFrom - {:secretKeyRef - {:name "postgres-secret", :key "postgres-password"}}} - {:name "POSTGRES_DB" - :valueFrom - {:configMapKeyRef - {:name "postgres-config", :key "postgres-db"}}}] - :ports [{:containerPort 5432, :name "postgresql"}] - :volumeMounts - [{:name "postgres-config-volume" - :mountPath "/etc/postgresql/postgresql.conf" - :subPath "postgresql.conf" - :readOnly true} - {:name "postgre-data-volume" - :mountPath "/var/lib/postgresql/data"}]}] - (get-in (cut/generate-deployment {:postgres-image "postgres:14"}) - [:spec :template :spec :containers])))) - -(deftest should-generate-deployment-diff - (is (= {:image-c1 "postgres:13", :image-c2 "postgres:14"} - (ct/map-diff (cut/generate-deployment) (cut/generate-deployment {:postgres-image "postgres:14"}))))) \ No newline at end of file + (cut/generate-pvc {})))) \ No newline at end of file -- 2.45.2 From 674045eba3c221a79777925188f27add9319c044 Mon Sep 17 00:00:00 2001 From: Michael Jerger Date: Tue, 20 Feb 2024 17:33:22 +0100 Subject: [PATCH 09/15] add test & fixed some issues --- src/main/cljc/dda/c4k_common/postgres.cljc | 19 +++++++++++++------ .../postgres/postgres_internal.cljc | 2 +- .../cljc/dda/c4k_common/postgres_test.cljc | 9 ++++++++- 3 files changed, 22 insertions(+), 8 deletions(-) diff --git a/src/main/cljc/dda/c4k_common/postgres.cljc b/src/main/cljc/dda/c4k_common/postgres.cljc index 1a736cf..99a9cad 100644 --- a/src/main/cljc/dda/c4k_common/postgres.cljc +++ b/src/main/cljc/dda/c4k_common/postgres.cljc @@ -1,14 +1,8 @@ (ns dda.c4k-common.postgres (:require [clojure.spec.alpha :as s] - #?(:cljs [shadow.resource :as rc]) #?(:clj [orchestra.core :refer [defn-spec]] :cljs [orchestra.core :refer-macros [defn-spec]]) - #?(:cljs [dda.c4k-common.macros :refer-macros [inline-resources]]) - [dda.c4k-common.yaml :as yaml] - [dda.c4k-common.base64 :as b64] - [dda.c4k-common.predicate :as cp] - [dda.c4k-common.common :as cm] [dda.c4k-common.postgres.postgres-internal :as int])) (def postgres-size? int/postgres-size?) @@ -75,3 +69,16 @@ (defn-spec generate-service map? [] (int/generate-service)) + + +(defn-spec generate seq? + [config pg-config? + auth pg-auth?] + (let [final-config (merge default-config + config)] + [(int/generate-secret auth) + (int/generate-persistent-volume final-config) + (int/generate-config final-config) + (int/generate-pvc final-config) + (int/generate-deployment final-config) + (int/generate-service)])) diff --git a/src/main/cljc/dda/c4k_common/postgres/postgres_internal.cljc b/src/main/cljc/dda/c4k_common/postgres/postgres_internal.cljc index 0fde44c..89bdbb0 100644 --- a/src/main/cljc/dda/c4k_common/postgres/postgres_internal.cljc +++ b/src/main/cljc/dda/c4k_common/postgres/postgres_internal.cljc @@ -30,7 +30,7 @@ (s/keys :req-un [::postgres-size ::db-name ::postgres-data-volume-path ::pvc-storage-class-name ::pv-storage-size-gb])) (def pg-auth? - (s/keys :opt-un [::postgres-db-user ::postgres-db-password])) + (s/keys :req-un [::postgres-db-user ::postgres-db-password])) (def postgres-function (s/keys :opt-un [::deserializer ::optional])) diff --git a/src/test/cljc/dda/c4k_common/postgres_test.cljc b/src/test/cljc/dda/c4k_common/postgres_test.cljc index 6ee7c7e..b5fffda 100644 --- a/src/test/cljc/dda/c4k_common/postgres_test.cljc +++ b/src/test/cljc/dda/c4k_common/postgres_test.cljc @@ -38,4 +38,11 @@ {:storageClassName "manual" :accessModes ["ReadWriteOnce"] :resources {:requests {:storage "10Gi"}}}} - (cut/generate-pvc {})))) \ No newline at end of file + (cut/generate-pvc {})))) + + +(deftest should-generate + (is (= 6 + (count (cut/generate {} + {:postgres-db-user "user" + :postgres-db-password "password"}))))) \ No newline at end of file -- 2.45.2 From 7e9c813e331f6c7420ce123f445fffd8b51cba76 Mon Sep 17 00:00:00 2001 From: Michael Jerger Date: Tue, 20 Feb 2024 18:05:33 +0100 Subject: [PATCH 10/15] introduce namespace for postgres --- src/main/cljc/dda/c4k_common/postgres.cljc | 31 ++++--- .../postgres/postgres_internal.cljc | 29 +++++-- .../postgres/postgres_internal_test.cljc | 82 ++++++++++++++++--- .../cljc/dda/c4k_common/postgres_test.cljc | 7 +- 4 files changed, 111 insertions(+), 38 deletions(-) diff --git a/src/main/cljc/dda/c4k_common/postgres.cljc b/src/main/cljc/dda/c4k_common/postgres.cljc index 99a9cad..4e59061 100644 --- a/src/main/cljc/dda/c4k_common/postgres.cljc +++ b/src/main/cljc/dda/c4k_common/postgres.cljc @@ -3,6 +3,7 @@ [clojure.spec.alpha :as s] #?(:clj [orchestra.core :refer [defn-spec]] :cljs [orchestra.core :refer-macros [defn-spec]]) + [dda.c4k-common.namespace :as ns] [dda.c4k-common.postgres.postgres-internal :as int])) (def postgres-size? int/postgres-size?) @@ -19,18 +20,19 @@ (def pg-config? (s/keys :opt-un [::postgres-size ::db-name ::postgres-data-volume-path - ::pvc-storage-class-name ::pv-storage-size-gb])) + ::pvc-storage-class-name ::pv-storage-size-gb ::ns/namespace])) (def pg-auth? (s/keys :opt-un [::postgres-db-user ::postgres-db-password])) (def postgres-function (s/keys :opt-un [::deserializer ::optional])) -(def default-config {:postgres-image "postgres:13" - :postgres-size :2gb - :db-name "postgres" - :postgres-data-volume-path "/var/postgres" - :pv-storage-size-gb 10 - :pvc-storage-class-name "manual"}) +(def default-config (merge ns/default-config + {:postgres-image "postgres:13" + :postgres-size :2gb + :db-name "postgres" + :postgres-data-volume-path "/var/postgres" + :pv-storage-size-gb 10 + :pvc-storage-class-name "manual"})) (defn-spec generate-config map? @@ -62,13 +64,16 @@ (defn-spec generate-secret map? - [my-auth pg-auth?] - (int/generate-secret my-auth)) + [config pg-config? + auth pg-auth?] + (int/generate-secret config auth)) (defn-spec generate-service map? - [] - (int/generate-service)) + [config pg-config?] + (let [final-config (merge default-config + config)] + (int/generate-service final-config))) (defn-spec generate seq? @@ -76,9 +81,9 @@ auth pg-auth?] (let [final-config (merge default-config config)] - [(int/generate-secret auth) + [(int/generate-secret final-config auth) (int/generate-persistent-volume final-config) (int/generate-config final-config) (int/generate-pvc final-config) (int/generate-deployment final-config) - (int/generate-service)])) + (int/generate-service final-config)])) diff --git a/src/main/cljc/dda/c4k_common/postgres/postgres_internal.cljc b/src/main/cljc/dda/c4k_common/postgres/postgres_internal.cljc index 89bdbb0..ca980dc 100644 --- a/src/main/cljc/dda/c4k_common/postgres/postgres_internal.cljc +++ b/src/main/cljc/dda/c4k_common/postgres/postgres_internal.cljc @@ -8,6 +8,7 @@ [dda.c4k-common.base64 :as b64] [dda.c4k-common.predicate :as cp] [dda.c4k-common.common :as cm] + [dda.c4k-common.namespace :as ns] [dda.c4k-common.postgres.postgres-internal :as int])) (defn postgres-size? @@ -28,7 +29,7 @@ (def pg-config? (s/keys :req-un [::postgres-size ::db-name ::postgres-data-volume-path - ::pvc-storage-class-name ::pv-storage-size-gb])) + ::pvc-storage-class-name ::pv-storage-size-gb ::ns/namespace])) (def pg-auth? (s/keys :req-un [::postgres-db-user ::postgres-db-password])) @@ -37,51 +38,61 @@ (defn-spec generate-config map? [config pg-config?] - (let [{:keys [postgres-size db-name]} config] + (let [{:keys [postgres-size db-name namespace]} config] (-> (yaml/from-string (yaml/load-resource (str "postgres/config-" (name postgres-size) ".yaml"))) + (assoc-in [:metadata :namespace] namespace) (assoc-in [:data :postgres-db] db-name)))) (defn-spec generate-deployment map? [config pg-config?] - (let [{:keys [postgres-image]} config] + (let [{:keys [postgres-image namespace]} config] (-> (yaml/from-string (yaml/load-resource "postgres/deployment.yaml")) + (assoc-in [:metadata :namespace] namespace) (assoc-in [:spec :template :spec :containers 0 :image] postgres-image)))) (defn-spec generate-persistent-volume map? [config pg-config?] - (let [{:keys [postgres-data-volume-path pv-storage-size-gb]} config] + (let [{:keys [postgres-data-volume-path pv-storage-size-gb namespace]} config] (-> (yaml/from-string (yaml/load-resource "postgres/persistent-volume.yaml")) + (assoc-in [:metadata :namespace] namespace) (assoc-in [:spec :hostPath :path] postgres-data-volume-path) (assoc-in [:spec :capacity :storage] (str pv-storage-size-gb "Gi"))))) (defn-spec generate-pvc map? [config pg-config?] - (let [{:keys [pv-storage-size-gb pvc-storage-class-name]} config] + (let [{:keys [pv-storage-size-gb pvc-storage-class-name namespace]} config] (-> (yaml/from-string (yaml/load-resource "postgres/pvc.yaml")) + (assoc-in [:metadata :namespace] namespace) (assoc-in [:spec :resources :requests :storage] (str pv-storage-size-gb "Gi")) (assoc-in [:spec :storageClassName] (name pvc-storage-class-name))))) (defn-spec generate-secret map? - [my-auth pg-auth?] - (let [{:keys [postgres-db-user postgres-db-password]} my-auth] + [config pg-config? + auth pg-auth?] + (let [{:keys [namespace]} config + {:keys [postgres-db-user postgres-db-password]} auth] (-> (yaml/from-string (yaml/load-resource "postgres/secret.yaml")) + (assoc-in [:metadata :namespace] namespace) (cm/replace-key-value :postgres-user (b64/encode postgres-db-user)) (cm/replace-key-value :postgres-password (b64/encode postgres-db-password))))) (defn-spec generate-service map? - [] - (yaml/from-string (yaml/load-resource "postgres/service.yaml"))) + [config pg-config?] + (let [{:keys [namespace]} config] + (-> + (yaml/from-string (yaml/load-resource "postgres/service.yaml")) + (assoc-in [:metadata :namespace] namespace)))) #?(:cljs diff --git a/src/test/cljc/dda/c4k_common/postgres/postgres_internal_test.cljc b/src/test/cljc/dda/c4k_common/postgres/postgres_internal_test.cljc index 22bdc7e..55116db 100644 --- a/src/test/cljc/dda/c4k_common/postgres/postgres_internal_test.cljc +++ b/src/test/cljc/dda/c4k_common/postgres/postgres_internal_test.cljc @@ -3,7 +3,6 @@ #?(:clj [clojure.test :refer [deftest is are testing run-tests]] :cljs [cljs.test :refer-macros [deftest is are testing run-tests]]) [clojure.spec.test.alpha :as st] - [dda.c4k-common.test-helper :as ct] [dda.c4k-common.postgres.postgres-internal :as cut])) (st/instrument `cut/generate-config) @@ -14,6 +13,26 @@ (st/instrument `cut/generate-service) (deftest should-generate-config + (is (= {:name "postgres-config", + :namespace "default" + :labels {:app "postgres"}} + (:metadata (cut/generate-config {:postgres-image "postgres:13" + :postgres-size :2gb + :db-name "postgres" + :postgres-data-volume-path "/var/postgres" + :pv-storage-size-gb 10 + :pvc-storage-class-name "manual" + :namespace "default"})))) + (is (= {:name "postgres-config", + :namespace "myapp" + :labels {:app "postgres"}} + (:metadata (cut/generate-config {:postgres-image "postgres:13" + :postgres-size :2gb + :db-name "postgres" + :postgres-data-volume-path "/var/postgres" + :pv-storage-size-gb 10 + :pvc-storage-class-name "manual" + :namespace "myapp"})))) (is (= {:postgres-db "postgres" :postgresql.conf "max_connections = 100\nwork_mem = 4MB\nshared_buffers = 512MB\n"} @@ -22,7 +41,8 @@ :db-name "postgres" :postgres-data-volume-path "/var/postgres" :pv-storage-size-gb 10 - :pvc-storage-class-name "manual"})))) + :pvc-storage-class-name "manual" + :namespace "default"})))) (is (= {:postgres-db "postgres" :postgresql.conf "max_connections = 700\nwork_mem = 3MB\nshared_buffers = 2048MB\n"} @@ -31,7 +51,8 @@ :db-name "postgres" :postgres-data-volume-path "/var/postgres" :pv-storage-size-gb 10 - :pvc-storage-class-name "manual"})))) + :pvc-storage-class-name "manual" + :namespace "default"})))) (is (= {:postgres-db "test" :postgresql.conf "max_connections = 100\nwork_mem = 4MB\nshared_buffers = 512MB\n"} @@ -40,7 +61,8 @@ :db-name "test" :postgres-data-volume-path "/var/postgres" :pv-storage-size-gb 10 - :pvc-storage-class-name "manual"})))) + :pvc-storage-class-name "manual" + :namespace "default"})))) ) (deftest should-generate-deployment @@ -72,8 +94,18 @@ :db-name "test" :postgres-data-volume-path "/var/postgres" :pv-storage-size-gb 10 - :pvc-storage-class-name "manual"}) - [:spec :template :spec :containers])))) + :pvc-storage-class-name "manual" + :namespace "default"}) + [:spec :template :spec :containers]))) + (is (= {:name "postgresql", + :namespace "myapp"} + (:metadata (cut/generate-deployment {:postgres-image "postgres:14" + :postgres-size :2gb + :db-name "test" + :postgres-data-volume-path "/var/postgres" + :pv-storage-size-gb 10 + :pvc-storage-class-name "manual" + :namespace "myapp"}))))) @@ -81,7 +113,9 @@ (is (= {:kind "PersistentVolume" :apiVersion "v1" :metadata - {:name "postgres-pv-volume", :labels {:type "local"}} + {:name "postgres-pv-volume", + :namespace "default" + :labels {:type "local"}} :spec {:storageClassName "manual" :accessModes ["ReadWriteOnce"] @@ -92,14 +126,17 @@ :db-name "test" :pvc-storage-class-name "manual" :postgres-data-volume-path "xx" - :pv-storage-size-gb 20})))) + :pv-storage-size-gb 20 + :namespace "default"})))) (deftest should-generate-persistent-volume-claim (is (= {:apiVersion "v1" :kind "PersistentVolumeClaim" :metadata - {:name "postgres-claim", :labels {:app "postgres"}} + {:name "postgres-claim", + :namespace "default" + :labels {:app "postgres"}} :spec {:storageClassName "local-path" :accessModes ["ReadWriteOnce"] @@ -109,14 +146,35 @@ :db-name "postgres" :postgres-data-volume-path "/var/postgres" :pv-storage-size-gb 20 - :pvc-storage-class-name "local-path"})))) + :pvc-storage-class-name "local-path" + :namespace "default"})))) (deftest should-generate-secret (is (= {:apiVersion "v1" :kind "Secret" - :metadata {:name "postgres-secret"} + :metadata {:name "postgres-secret" :namespace "default"} :type "Opaque" :data {:postgres-user "eHgtdXM=", :postgres-password "eHgtcHc="}} - (cut/generate-secret {:postgres-db-user "xx-us" :postgres-db-password "xx-pw"})))) + (cut/generate-secret {:postgres-image "postgres:13" + :postgres-size :2gb + :db-name "postgres" + :postgres-data-volume-path "/var/postgres" + :pv-storage-size-gb 20 + :pvc-storage-class-name "local-path" + :namespace "default"} + {:postgres-db-user "xx-us" :postgres-db-password "xx-pw"})))) + + +(deftest should-generate-service + (is (= {:name "postgresql-service" :namespace "default"} + (:metadata (cut/generate-service + {:postgres-image "postgres:13" + :postgres-size :2gb + :db-name "postgres" + :postgres-data-volume-path "/var/postgres" + :pv-storage-size-gb 20 + :pvc-storage-class-name "local-path" + :namespace "default"}))))) + diff --git a/src/test/cljc/dda/c4k_common/postgres_test.cljc b/src/test/cljc/dda/c4k_common/postgres_test.cljc index b5fffda..bd26fa0 100644 --- a/src/test/cljc/dda/c4k_common/postgres_test.cljc +++ b/src/test/cljc/dda/c4k_common/postgres_test.cljc @@ -3,12 +3,11 @@ #?(:clj [clojure.test :refer [deftest is are testing run-tests]] :cljs [cljs.test :refer-macros [deftest is are testing run-tests]]) [clojure.spec.test.alpha :as st] - [dda.c4k-common.test-helper :as ct] [dda.c4k-common.postgres :as cut])) (st/instrument `cut/generate-config) (st/instrument `cut/generate-persistent-volume) -;;(st/instrument `cut/generate) +(st/instrument `cut/generate) (deftest should-generate-config (is (= {:postgres-db "postgres" @@ -20,7 +19,7 @@ (is (= {:kind "PersistentVolume" :apiVersion "v1" :metadata - {:name "postgres-pv-volume", :labels {:type "local"}} + {:name "postgres-pv-volume", :namespace "default" :labels {:type "local"}} :spec {:storageClassName "manual" :accessModes ["ReadWriteOnce"] @@ -33,7 +32,7 @@ (is (= {:apiVersion "v1" :kind "PersistentVolumeClaim" :metadata - {:name "postgres-claim", :labels {:app "postgres"}} + {:name "postgres-claim", :namespace "default" :labels {:app "postgres"}} :spec {:storageClassName "manual" :accessModes ["ReadWriteOnce"] -- 2.45.2 From 0cd4f2282319fe998e0875b87107b19e0834d3fd Mon Sep 17 00:00:00 2001 From: Michael Jerger Date: Tue, 20 Feb 2024 18:10:05 +0100 Subject: [PATCH 11/15] fix cljs --- src/main/cljc/dda/c4k_common/postgres/postgres_internal.cljc | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/main/cljc/dda/c4k_common/postgres/postgres_internal.cljc b/src/main/cljc/dda/c4k_common/postgres/postgres_internal.cljc index ca980dc..7fa3442 100644 --- a/src/main/cljc/dda/c4k_common/postgres/postgres_internal.cljc +++ b/src/main/cljc/dda/c4k_common/postgres/postgres_internal.cljc @@ -8,8 +8,7 @@ [dda.c4k-common.base64 :as b64] [dda.c4k-common.predicate :as cp] [dda.c4k-common.common :as cm] - [dda.c4k-common.namespace :as ns] - [dda.c4k-common.postgres.postgres-internal :as int])) + [dda.c4k-common.namespace :as ns])) (defn postgres-size? [input] -- 2.45.2 From a1a49e10d0dffec691cfeeaf89b5815ea901ca63 Mon Sep 17 00:00:00 2001 From: Michael Jerger Date: Tue, 20 Feb 2024 18:15:17 +0100 Subject: [PATCH 12/15] update doc --- README.md | 47 ++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 38 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index ab94405..7f6f99d 100644 --- a/README.md +++ b/README.md @@ -25,6 +25,23 @@ By the way, c4k means "convention for kubernetes". c4k-common supports the following use cases: +- [convention 4 kubernetes: c4k-common](#convention-4-kubernetes-c4k-common) + - [Rationale](#rationale) + - [Features](#features) + - [Target Cli and Web Frontend](#target-cli-and-web-frontend) + - [Separate Configuration from Credentials](#separate-configuration-from-credentials) + - [Input as EDN or Yaml](#input-as-edn-or-yaml) + - [Inline k8s resources for versioning \& dependencies](#inline-k8s-resources-for-versioning--dependencies) + - [Work on structured Data instead flat Templating](#work-on-structured-data-instead-flat-templating) + - [Validate your inputs](#validate-your-inputs) + - [Namespaces](#namespaces) + - [Ingress](#ingress) + - [Postgres Database](#postgres-database) + - [Monitoring with Grafana Cloud](#monitoring-with-grafana-cloud) + - [Refactoring \& Module Overview](#refactoring--module-overview) + - [Development \& mirrors](#development--mirrors) + - [License](#license) + #### Target Cli and Web Frontend Set up your cli as follows @@ -152,6 +169,18 @@ Have you recognized the `defn-spec` marco? We use allover validation, e.g. ...) ``` +#### Namespaces + +We support namespaces for ingress & postgres (monitoring lives in it's own namespace `monitoring`). + +```clojure +(deftest should-generate-simple-ingress + (is (= [{:apiVersion "v1" + :kind "Namespace" + :metadata {:name "myapp"}}] + (cut/generate {:namespace "myapp"})))) +``` + #### Ingress In most cases we use 'generate-ingress-and-cert' which generates an ingres in combination with letsencrypt cert for a named service. @@ -218,15 +247,15 @@ You can attach your application to grafana cloud. ## Refactoring & Module Overview -| Module | Version | [common load-as-edn][edn1] | [groups for webview][bgrp1] | [use common ingress][ing1] | [use common monitoring][mon1] | [validate examples][val1] | [ci with pyb][cipyb] | [inline-macro to load resources][macro] |[native build][native] | -| ------------- |---------| :------------------------: | :-------------------------: | :------------------------: | :---------------------------: | :-----------------------: |:--------------------:| :-------------------------------------: |:---------------------:| -| c4k-keycloak | 0.2 | x | x | x | x | x | | | | -| c4k-taiga | 0.1 | | | | | | | | | -| c4k-nextcloud | 4.0 | x | x | x | x | x | | | | -| c4k-jitsi | 1.6 | x | x | x | x | x | x | x | | -| c4k-forgejo | 3.0 | x | x | x | x | x | x | x | x | -| c4k-shynet | 1.0 | | | | | | | | | -| c4k-website | 1.1 | x | x | x | x | x | | | | +| Module | Version | [common load-as-edn][edn1] | [groups for webview][bgrp1] | [use common ingress][ing1] | [use common monitoring][mon1] | [validate examples][val1] | [ci with pyb][cipyb] | [inline-macro to load resources][macro] |[native build][native] | namespaces | +| ------------- |---------| :------------------------: | :-------------------------: | :------------------------: | :---------------------------: | :-----------------------: |:--------------------:| :-------------------------------------: |:---------------------:|:----------:| +| c4k-keycloak | 0.2 | x | x | x | x | x | | | | | +| c4k-taiga | 0.1 | | | | | | | | | | +| c4k-nextcloud | 4.0 | x | x | x | x | x | | | | | +| c4k-jitsi | 1.6 | x | x | x | x | x | x | x | | | +| c4k-forgejo | 3.0 | x | x | x | x | x | x | x | x | | +| c4k-shynet | 1.0 | | | | | | | | | | +| c4k-website | 1.1 | x | x | x | x | x | | | | | [edn1]: https://gitlab.com/domaindrivenarchitecture/c4k-website/-/merge_requests/1 [ing1]: https://repo.prod.meissa.de/meissa/c4k-jitsi/commit/214aa41c28662fbf7a49998e17404e7ac9216430 -- 2.45.2 From c1f2cefd145261512ed58a4e581239895c71cd9c Mon Sep 17 00:00:00 2001 From: Michael Jerger Date: Tue, 20 Feb 2024 18:21:03 +0100 Subject: [PATCH 13/15] keep compatibility --- src/main/cljc/dda/c4k_common/postgres.cljc | 11 ++++++++--- src/test/cljc/dda/c4k_common/postgres_test.cljc | 8 ++++++++ 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/src/main/cljc/dda/c4k_common/postgres.cljc b/src/main/cljc/dda/c4k_common/postgres.cljc index 4e59061..98364d5 100644 --- a/src/main/cljc/dda/c4k_common/postgres.cljc +++ b/src/main/cljc/dda/c4k_common/postgres.cljc @@ -64,9 +64,14 @@ (defn-spec generate-secret map? - [config pg-config? - auth pg-auth?] - (int/generate-secret config auth)) + ([auth pg-auth?] + (let [final-config default-config] + (int/generate-secret final-config auth))) + ([config pg-config? + auth pg-auth?] + (let [final-config (merge default-config + config)] + (int/generate-secret final-config auth)))) (defn-spec generate-service map? diff --git a/src/test/cljc/dda/c4k_common/postgres_test.cljc b/src/test/cljc/dda/c4k_common/postgres_test.cljc index bd26fa0..90be20b 100644 --- a/src/test/cljc/dda/c4k_common/postgres_test.cljc +++ b/src/test/cljc/dda/c4k_common/postgres_test.cljc @@ -39,6 +39,14 @@ :resources {:requests {:storage "10Gi"}}}} (cut/generate-pvc {})))) +(deftest should-generate-secret + (is (= {:apiVersion "v1", + :kind "Secret", + :metadata {:name "postgres-secret", :namespace "default"}, + :type "Opaque", + :data {:postgres-user "eHgtdXM=", :postgres-password "eHgtcHc="}} + (cut/generate-secret {:postgres-db-user "xx-us" :postgres-db-password "xx-pw"})))) + (deftest should-generate (is (= 6 -- 2.45.2 From 73a9965f96e8aa9f608e71be7bebcf3d83bf32bb Mon Sep 17 00:00:00 2001 From: Michael Jerger Date: Tue, 20 Feb 2024 18:21:21 +0100 Subject: [PATCH 14/15] format --- src/test/cljc/dda/c4k_common/postgres_test.cljc | 1 + 1 file changed, 1 insertion(+) diff --git a/src/test/cljc/dda/c4k_common/postgres_test.cljc b/src/test/cljc/dda/c4k_common/postgres_test.cljc index 90be20b..88bc4ce 100644 --- a/src/test/cljc/dda/c4k_common/postgres_test.cljc +++ b/src/test/cljc/dda/c4k_common/postgres_test.cljc @@ -39,6 +39,7 @@ :resources {:requests {:storage "10Gi"}}}} (cut/generate-pvc {})))) + (deftest should-generate-secret (is (= {:apiVersion "v1", :kind "Secret", -- 2.45.2 From b21e3771686ba294daae1ec03586eb65645758d2 Mon Sep 17 00:00:00 2001 From: Michael Jerger Date: Tue, 20 Feb 2024 18:21:55 +0100 Subject: [PATCH 15/15] instrument new test --- src/test/cljc/dda/c4k_common/postgres_test.cljc | 1 + 1 file changed, 1 insertion(+) diff --git a/src/test/cljc/dda/c4k_common/postgres_test.cljc b/src/test/cljc/dda/c4k_common/postgres_test.cljc index 88bc4ce..236928a 100644 --- a/src/test/cljc/dda/c4k_common/postgres_test.cljc +++ b/src/test/cljc/dda/c4k_common/postgres_test.cljc @@ -7,6 +7,7 @@ (st/instrument `cut/generate-config) (st/instrument `cut/generate-persistent-volume) +(st/instrument `cut/generate-secret) (st/instrument `cut/generate) (deftest should-generate-config -- 2.45.2