From cca93311e44ec858877c60e6a33b8e735791005c Mon Sep 17 00:00:00 2001 From: Michael Jerger Date: Tue, 20 Feb 2024 17:08:39 +0100 Subject: [PATCH] seperate postgres & postgres-internal --- src/main/cljc/dda/c4k_common/postgres.cljc | 108 +++++++--------- .../postgres/postgres_internal.cljc | 89 +++++++++++++ .../postgres/postgres_internal_test.cljc | 122 ++++++++++++++++++ .../cljc/dda/c4k_common/postgres_test.cljc | 101 +-------------- 4 files changed, 263 insertions(+), 157 deletions(-) create mode 100644 src/main/cljc/dda/c4k_common/postgres/postgres_internal.cljc create mode 100644 src/test/cljc/dda/c4k_common/postgres/postgres_internal_test.cljc diff --git a/src/main/cljc/dda/c4k_common/postgres.cljc b/src/main/cljc/dda/c4k_common/postgres.cljc index 48d7d30..1a736cf 100644 --- a/src/main/cljc/dda/c4k_common/postgres.cljc +++ b/src/main/cljc/dda/c4k_common/postgres.cljc @@ -8,23 +8,20 @@ [dda.c4k-common.yaml :as yaml] [dda.c4k-common.base64 :as b64] [dda.c4k-common.predicate :as cp] - [dda.c4k-common.common :as cm])) + [dda.c4k-common.common :as cm] + [dda.c4k-common.postgres.postgres-internal :as int])) -(defn postgres-size? - [input] - (contains? #{:2gb :4gb :8gb :16gb} input)) +(def postgres-size? int/postgres-size?) -(defn postgres-image? - [input] - (contains? #{"postgres:13" "postgres:14" "postgres:15" "postgres:16"} input)) +(def postgres-image? int/postgres-image?) -(s/def ::postgres-db-user cp/bash-env-string?) -(s/def ::postgres-db-password cp/bash-env-string?) -(s/def ::postgres-data-volume-path string?) -(s/def ::postgres-size postgres-size?) -(s/def ::db-name cp/bash-env-string?) -(s/def ::pvc-storage-class-name cp/pvc-storage-class-name?) -(s/def ::pv-storage-size-gb pos?) +(s/def ::postgres-db-user ::int/postgres-db-user) +(s/def ::postgres-db-password ::int/postgres-db-password) +(s/def ::postgres-data-volume-path ::int/postgres-data-volume-path) +(s/def ::postgres-size ::int/postgres-size) +(s/def ::db-name ::int/db-name) +(s/def ::pvc-storage-class-name ::int/pvc-storage-class-name) +(s/def ::pv-storage-size-gb ::int/pv-storage-size-gb) (def pg-config? (s/keys :opt-un [::postgres-size ::db-name ::postgres-data-volume-path @@ -34,56 +31,47 @@ (def postgres-function (s/keys :opt-un [::deserializer ::optional])) -#?(:cljs - (defmethod yaml/load-resource :postgres [resource-name] - (get (inline-resources "postgres") resource-name))) +(def default-config {:postgres-image "postgres:13" + :postgres-size :2gb + :db-name "postgres" + :postgres-data-volume-path "/var/postgres" + :pv-storage-size-gb 10 + :pvc-storage-class-name "manual"}) -(defn-spec generate-config cp/map-or-seq? + +(defn-spec generate-config map? [& config (s/? pg-config?)] - (let [{:keys [postgres-size db-name] - :or {postgres-size :2gb - db-name "postgres"}} (first config)] - (-> - (yaml/from-string (yaml/load-resource - (str "postgres/config-" (name postgres-size) ".yaml"))) - (assoc-in [:data :postgres-db] db-name)))) - -(defn-spec generate-deployment cp/map-or-seq? + (let [final-config (merge default-config + (first config))] + (int/generate-config final-config))) + + +(defn-spec generate-deployment map? [& config (s/? pg-config?)] - (let [{:keys [postgres-image] - :or {postgres-image "postgres:13"}} (first config)] - (-> - (yaml/from-string (yaml/load-resource "postgres/deployment.yaml")) - (assoc-in [:spec :template :spec :containers 0 :image] postgres-image)))) + (let [final-config (merge default-config + (first config))] + (int/generate-deployment final-config))) -(defn-spec generate-persistent-volume cp/map-or-seq? + +(defn-spec generate-persistent-volume map? [config pg-config?] - (let [{:keys [postgres-data-volume-path pv-storage-size-gb] - :or {postgres-data-volume-path "/var/postgres" - pv-storage-size-gb 10}} config] - (-> - (yaml/from-string (yaml/load-resource "postgres/persistent-volume.yaml")) - (assoc-in [:spec :hostPath :path] postgres-data-volume-path) - (assoc-in [:spec :capacity :storage] (str pv-storage-size-gb "Gi"))))) - -(defn-spec generate-pvc cp/map-or-seq? + (let [final-config (merge default-config + config)] + (int/generate-persistent-volume final-config))) + + +(defn-spec generate-pvc map? [config pg-config?] - (let [{:keys [pv-storage-size-gb pvc-storage-class-name] - :or {pv-storage-size-gb 10 - pvc-storage-class-name "manual"}} config] - (-> - (yaml/from-string (yaml/load-resource "postgres/pvc.yaml")) - (assoc-in [:spec :resources :requests :storage] (str pv-storage-size-gb "Gi")) - (assoc-in [:spec :storageClassName] (name pvc-storage-class-name))))) - -(defn-spec generate-secret cp/map-or-seq? - [my-auth any?] - (let [{:keys [postgres-db-user postgres-db-password]} my-auth] - (-> - (yaml/from-string (yaml/load-resource "postgres/secret.yaml")) - (cm/replace-key-value :postgres-user (b64/encode postgres-db-user)) - (cm/replace-key-value :postgres-password (b64/encode postgres-db-password))))) - -(defn-spec generate-service cp/map-or-seq? + (let [final-config (merge default-config + config)] + (int/generate-pvc final-config))) + + +(defn-spec generate-secret map? + [my-auth pg-auth?] + (int/generate-secret my-auth)) + + +(defn-spec generate-service map? [] - (yaml/from-string (yaml/load-resource "postgres/service.yaml"))) + (int/generate-service)) diff --git a/src/main/cljc/dda/c4k_common/postgres/postgres_internal.cljc b/src/main/cljc/dda/c4k_common/postgres/postgres_internal.cljc new file mode 100644 index 0000000..0fde44c --- /dev/null +++ b/src/main/cljc/dda/c4k_common/postgres/postgres_internal.cljc @@ -0,0 +1,89 @@ +(ns dda.c4k-common.postgres.postgres-internal + (:require + [clojure.spec.alpha :as s] + #?(:clj [orchestra.core :refer [defn-spec]] + :cljs [orchestra.core :refer-macros [defn-spec]]) + #?(:cljs [dda.c4k-common.macros :refer-macros [inline-resources]]) + [dda.c4k-common.yaml :as yaml] + [dda.c4k-common.base64 :as b64] + [dda.c4k-common.predicate :as cp] + [dda.c4k-common.common :as cm] + [dda.c4k-common.postgres.postgres-internal :as int])) + +(defn postgres-size? + [input] + (contains? #{:2gb :4gb :8gb :16gb} input)) + +(defn postgres-image? + [input] + (contains? #{"postgres:13" "postgres:14" "postgres:15" "postgres:16"} input)) + +(s/def ::postgres-db-user cp/bash-env-string?) +(s/def ::postgres-db-password cp/bash-env-string?) +(s/def ::postgres-data-volume-path string?) +(s/def ::postgres-size postgres-size?) +(s/def ::db-name cp/bash-env-string?) +(s/def ::pvc-storage-class-name cp/pvc-storage-class-name?) +(s/def ::pv-storage-size-gb pos?) + +(def pg-config? + (s/keys :req-un [::postgres-size ::db-name ::postgres-data-volume-path + ::pvc-storage-class-name ::pv-storage-size-gb])) +(def pg-auth? + (s/keys :opt-un [::postgres-db-user ::postgres-db-password])) + +(def postgres-function (s/keys :opt-un [::deserializer ::optional])) + + +(defn-spec generate-config map? + [config pg-config?] + (let [{:keys [postgres-size db-name]} config] + (-> + (yaml/from-string (yaml/load-resource + (str "postgres/config-" (name postgres-size) ".yaml"))) + (assoc-in [:data :postgres-db] db-name)))) + + +(defn-spec generate-deployment map? + [config pg-config?] + (let [{:keys [postgres-image]} config] + (-> + (yaml/from-string (yaml/load-resource "postgres/deployment.yaml")) + (assoc-in [:spec :template :spec :containers 0 :image] postgres-image)))) + + +(defn-spec generate-persistent-volume map? + [config pg-config?] + (let [{:keys [postgres-data-volume-path pv-storage-size-gb]} config] + (-> + (yaml/from-string (yaml/load-resource "postgres/persistent-volume.yaml")) + (assoc-in [:spec :hostPath :path] postgres-data-volume-path) + (assoc-in [:spec :capacity :storage] (str pv-storage-size-gb "Gi"))))) + + +(defn-spec generate-pvc map? + [config pg-config?] + (let [{:keys [pv-storage-size-gb pvc-storage-class-name]} config] + (-> + (yaml/from-string (yaml/load-resource "postgres/pvc.yaml")) + (assoc-in [:spec :resources :requests :storage] (str pv-storage-size-gb "Gi")) + (assoc-in [:spec :storageClassName] (name pvc-storage-class-name))))) + + +(defn-spec generate-secret map? + [my-auth pg-auth?] + (let [{:keys [postgres-db-user postgres-db-password]} my-auth] + (-> + (yaml/from-string (yaml/load-resource "postgres/secret.yaml")) + (cm/replace-key-value :postgres-user (b64/encode postgres-db-user)) + (cm/replace-key-value :postgres-password (b64/encode postgres-db-password))))) + + +(defn-spec generate-service map? + [] + (yaml/from-string (yaml/load-resource "postgres/service.yaml"))) + + +#?(:cljs + (defmethod yaml/load-resource :postgres [resource-name] + (get (inline-resources "postgres") resource-name))) diff --git a/src/test/cljc/dda/c4k_common/postgres/postgres_internal_test.cljc b/src/test/cljc/dda/c4k_common/postgres/postgres_internal_test.cljc new file mode 100644 index 0000000..22bdc7e --- /dev/null +++ b/src/test/cljc/dda/c4k_common/postgres/postgres_internal_test.cljc @@ -0,0 +1,122 @@ +(ns dda.c4k-common.postgres.postgres-internal-test + (:require + #?(:clj [clojure.test :refer [deftest is are testing run-tests]] + :cljs [cljs.test :refer-macros [deftest is are testing run-tests]]) + [clojure.spec.test.alpha :as st] + [dda.c4k-common.test-helper :as ct] + [dda.c4k-common.postgres.postgres-internal :as cut])) + +(st/instrument `cut/generate-config) +(st/instrument `cut/generate-deployment) +(st/instrument `cut/generate-persistent-volume) +(st/instrument `cut/generate-pvc) +(st/instrument `cut/generate-secret) +(st/instrument `cut/generate-service) + +(deftest should-generate-config + (is (= {:postgres-db "postgres" + :postgresql.conf + "max_connections = 100\nwork_mem = 4MB\nshared_buffers = 512MB\n"} + (:data (cut/generate-config {:postgres-image "postgres:13" + :postgres-size :2gb + :db-name "postgres" + :postgres-data-volume-path "/var/postgres" + :pv-storage-size-gb 10 + :pvc-storage-class-name "manual"})))) + (is (= {:postgres-db "postgres" + :postgresql.conf + "max_connections = 700\nwork_mem = 3MB\nshared_buffers = 2048MB\n"} + (:data (cut/generate-config {:postgres-image "postgres:13" + :postgres-size :8gb + :db-name "postgres" + :postgres-data-volume-path "/var/postgres" + :pv-storage-size-gb 10 + :pvc-storage-class-name "manual"})))) + (is (= {:postgres-db "test" + :postgresql.conf + "max_connections = 100\nwork_mem = 4MB\nshared_buffers = 512MB\n"} + (:data (cut/generate-config {:postgres-image "postgres:13" + :postgres-size :2gb + :db-name "test" + :postgres-data-volume-path "/var/postgres" + :pv-storage-size-gb 10 + :pvc-storage-class-name "manual"})))) + ) + +(deftest should-generate-deployment + (is (= [{:image "postgres:14" + :name "postgresql" + :env + [{:name "POSTGRES_USER" + :valueFrom + {:secretKeyRef + {:name "postgres-secret", :key "postgres-user"}}} + {:name "POSTGRES_PASSWORD" + :valueFrom + {:secretKeyRef + {:name "postgres-secret", :key "postgres-password"}}} + {:name "POSTGRES_DB" + :valueFrom + {:configMapKeyRef + {:name "postgres-config", :key "postgres-db"}}}] + :ports [{:containerPort 5432, :name "postgresql"}] + :volumeMounts + [{:name "postgres-config-volume" + :mountPath "/etc/postgresql/postgresql.conf" + :subPath "postgresql.conf" + :readOnly true} + {:name "postgre-data-volume" + :mountPath "/var/lib/postgresql/data"}]}] + (get-in (cut/generate-deployment {:postgres-image "postgres:14" + :postgres-size :2gb + :db-name "test" + :postgres-data-volume-path "/var/postgres" + :pv-storage-size-gb 10 + :pvc-storage-class-name "manual"}) + [:spec :template :spec :containers])))) + + + +(deftest should-generate-persistent-volume + (is (= {:kind "PersistentVolume" + :apiVersion "v1" + :metadata + {:name "postgres-pv-volume", :labels {:type "local"}} + :spec + {:storageClassName "manual" + :accessModes ["ReadWriteOnce"] + :capacity {:storage "20Gi"} + :hostPath {:path "xx"}}} + (cut/generate-persistent-volume {:postgres-image "postgres:14" + :postgres-size :2gb + :db-name "test" + :pvc-storage-class-name "manual" + :postgres-data-volume-path "xx" + :pv-storage-size-gb 20})))) + + +(deftest should-generate-persistent-volume-claim + (is (= {:apiVersion "v1" + :kind "PersistentVolumeClaim" + :metadata + {:name "postgres-claim", :labels {:app "postgres"}} + :spec + {:storageClassName "local-path" + :accessModes ["ReadWriteOnce"] + :resources {:requests {:storage "20Gi"}}}} + (cut/generate-pvc {:postgres-image "postgres:13" + :postgres-size :2gb + :db-name "postgres" + :postgres-data-volume-path "/var/postgres" + :pv-storage-size-gb 20 + :pvc-storage-class-name "local-path"})))) + + +(deftest should-generate-secret + (is (= {:apiVersion "v1" + :kind "Secret" + :metadata {:name "postgres-secret"} + :type "Opaque" + :data + {:postgres-user "eHgtdXM=", :postgres-password "eHgtcHc="}} + (cut/generate-secret {:postgres-db-user "xx-us" :postgres-db-password "xx-pw"})))) diff --git a/src/test/cljc/dda/c4k_common/postgres_test.cljc b/src/test/cljc/dda/c4k_common/postgres_test.cljc index 650544d..6ee7c7e 100644 --- a/src/test/cljc/dda/c4k_common/postgres_test.cljc +++ b/src/test/cljc/dda/c4k_common/postgres_test.cljc @@ -7,33 +7,14 @@ [dda.c4k-common.postgres :as cut])) (st/instrument `cut/generate-config) -(st/instrument `cut/generate-deployment) (st/instrument `cut/generate-persistent-volume) -(st/instrument `cut/generate-pvc) -(st/instrument `cut/generate-secret) -(st/instrument `cut/generate-service) +;;(st/instrument `cut/generate) (deftest should-generate-config (is (= {:postgres-db "postgres" :postgresql.conf "max_connections = 100\nwork_mem = 4MB\nshared_buffers = 512MB\n"} - (:data (cut/generate-config)))) - (is (= {:postgres-db "postgres" - :postgresql.conf - "max_connections = 700\nwork_mem = 3MB\nshared_buffers = 2048MB\n"} - (:data (cut/generate-config {:postgres-size :8gb})))) - (is (= {:postgres-db "test" - :postgresql.conf - "max_connections = 100\nwork_mem = 4MB\nshared_buffers = 512MB\n"} - (:data (cut/generate-config {:db-name "test"})))) - ) - -(deftest should-generate-config-diff - (is (= {:postgres-db-c1 "postgres", - :postgres-db-c2 "test", - :postgresql.conf-c1 "max_connections = 100\nwork_mem = 4MB\nshared_buffers = 512MB\n", - :postgresql.conf-c2 "max_connections = 700\nwork_mem = 3MB\nshared_buffers = 2048MB\n"} - (ct/map-diff (cut/generate-config) (cut/generate-config {:db-name "test" :postgres-size :8gb}))))) + (:data (cut/generate-config))))) (deftest should-generate-persistent-volume (is (= {:kind "PersistentVolume" @@ -45,25 +26,8 @@ :accessModes ["ReadWriteOnce"] :capacity {:storage "10Gi"} :hostPath {:path "xx"}}} - (cut/generate-persistent-volume {:postgres-data-volume-path "xx"}))) - (is (= {:kind "PersistentVolume" - :apiVersion "v1" - :metadata - {:name "postgres-pv-volume", :labels {:type "local"}} - :spec - {:storageClassName "manual" - :accessModes ["ReadWriteOnce"] - :capacity {:storage "20Gi"} - :hostPath {:path "xx"}}} - (cut/generate-persistent-volume {:postgres-data-volume-path "xx" - :pv-storage-size-gb 20})))) + (cut/generate-persistent-volume {:postgres-data-volume-path "xx"})))) -(deftest should-generate-persistent-volume-diff - (is (= {:storage-c1 "10Gi", :storage-c2 "20Gi", - :path-c1 "/var/postgres", :path-c2 "xx"} - (ct/map-diff (cut/generate-persistent-volume {}) - (cut/generate-persistent-volume {:postgres-data-volume-path "xx" - :pv-storage-size-gb 20}))))) (deftest should-generate-persistent-volume-claim (is (= {:apiVersion "v1" @@ -74,61 +38,4 @@ {:storageClassName "manual" :accessModes ["ReadWriteOnce"] :resources {:requests {:storage "10Gi"}}}} - (cut/generate-pvc {}))) - (is (= {:apiVersion "v1" - :kind "PersistentVolumeClaim" - :metadata - {:name "postgres-claim", :labels {:app "postgres"}} - :spec - {:storageClassName "local-path" - :accessModes ["ReadWriteOnce"] - :resources {:requests {:storage "20Gi"}}}} - (cut/generate-pvc {:pv-storage-size-gb 20 - :pvc-storage-class-name "local-path"})))) - -(deftest should-generate-persistent-volume-claim-diff - (is (= {:storageClassName-c1 "manual", :storageClassName-c2 "local-path", - :storage-c1 "10Gi", :storage-c2 "20Gi"} - (ct/map-diff (cut/generate-pvc {}) - (cut/generate-pvc {:pv-storage-size-gb 20 - :pvc-storage-class-name "local-path"}))))) - -(deftest should-generate-secret - (is (= {:apiVersion "v1" - :kind "Secret" - :metadata {:name "postgres-secret"} - :type "Opaque" - :data - {:postgres-user "eHgtdXM=", :postgres-password "eHgtcHc="}} - (cut/generate-secret {:postgres-db-user "xx-us" :postgres-db-password "xx-pw"})))) - -(deftest should-generate-deployment - (is (= [{:image "postgres:14" - :name "postgresql" - :env - [{:name "POSTGRES_USER" - :valueFrom - {:secretKeyRef - {:name "postgres-secret", :key "postgres-user"}}} - {:name "POSTGRES_PASSWORD" - :valueFrom - {:secretKeyRef - {:name "postgres-secret", :key "postgres-password"}}} - {:name "POSTGRES_DB" - :valueFrom - {:configMapKeyRef - {:name "postgres-config", :key "postgres-db"}}}] - :ports [{:containerPort 5432, :name "postgresql"}] - :volumeMounts - [{:name "postgres-config-volume" - :mountPath "/etc/postgresql/postgresql.conf" - :subPath "postgresql.conf" - :readOnly true} - {:name "postgre-data-volume" - :mountPath "/var/lib/postgresql/data"}]}] - (get-in (cut/generate-deployment {:postgres-image "postgres:14"}) - [:spec :template :spec :containers])))) - -(deftest should-generate-deployment-diff - (is (= {:image-c1 "postgres:13", :image-c2 "postgres:14"} - (ct/map-diff (cut/generate-deployment) (cut/generate-deployment {:postgres-image "postgres:14"}))))) \ No newline at end of file + (cut/generate-pvc {})))) \ No newline at end of file