From 2fc6a177ad510576891bc6ff72b75230c5a79404 Mon Sep 17 00:00:00 2001 From: jem Date: Mon, 5 Jul 2021 18:02:25 +0200 Subject: [PATCH] introduce postgres & persistent volumes --- .../docker/image/resources/entrypoint.sh | 51 +++++++++--- src/main/cljc/dda/c4k_jira/core.cljc | 54 ++++++++----- src/main/cljc/dda/c4k_jira/jira.cljc | 31 +++----- src/main/cljc/dda/c4k_jira/postgres.cljc | 35 +++++++++ src/main/resources/jira/pod.yaml | 22 ++++-- src/main/resources/jira/secret.yaml | 8 -- src/main/resources/postgres/config.yaml | 2 +- src/main/resources/postgres/deployment.yaml | 5 ++ .../resources/postgres/persistent-volume.yaml | 14 ++++ src/main/resources/postgres/pvc.yaml | 13 ++++ src/test/cljc/dda/c4k_jira/jira_test.cljc | 78 +++++++++++++++++++ src/test/cljc/dda/c4k_jira/postgres_test.cljc | 26 +++++++ src/test/resources/invalid-auth.edn | 1 - src/test/resources/invalid-config.edn | 27 ------- src/test/resources/valid-auth.edn | 1 - src/test/resources/valid-config.edn | 27 ------- 16 files changed, 279 insertions(+), 116 deletions(-) create mode 100644 src/main/cljc/dda/c4k_jira/postgres.cljc delete mode 100644 src/main/resources/jira/secret.yaml create mode 100644 src/main/resources/postgres/persistent-volume.yaml create mode 100644 src/main/resources/postgres/pvc.yaml create mode 100644 src/test/cljc/dda/c4k_jira/postgres_test.cljc delete mode 100644 src/test/resources/invalid-auth.edn delete mode 100644 src/test/resources/invalid-config.edn delete mode 100644 src/test/resources/valid-auth.edn delete mode 100644 src/test/resources/valid-config.edn diff --git a/infrastrucure/docker/image/resources/entrypoint.sh b/infrastrucure/docker/image/resources/entrypoint.sh index fe0fa87..ee871ca 100644 --- a/infrastrucure/docker/image/resources/entrypoint.sh +++ b/infrastrucure/docker/image/resources/entrypoint.sh @@ -1,12 +1,45 @@ #!/bin/bash -xmlstarlet ed -L -u "/Server/Service/Connector[@proxyName='{subdomain}.{domain}.com']/@proxyName" \ - -v "$1" /opt/atlassian-jira-software-standalone/conf/server.xml -xmlstarlet ed -L -u "/jira-database-config/jdbc-datasource/username" \ - -v "$2" /app/dbconfig.xml -xmlstarlet ed -L -u "/jira-database-config/jdbc-datasource/password" \ - -v "$3" /app/dbconfig.xml +function main() { + local fqdn_value=file_env "FQDN" + local db_username_value=file_env "DB_USERNAME" + local db_username_value=file_env "DB_PASSWORD" -install -ojira -gjira -m660 /app/dbconfig.xml /var/jira/dbconfig.xml -/opt/atlassian-jira-software-standalone/bin/setenv.sh run -/opt/atlassian-jira-software-standalone/bin/start-jira.sh run + xmlstarlet ed -L -u "/Server/Service/Connector[@proxyName='{subdomain}.{domain}.com']/@proxyName" \ + -v "$fqdn_value" /opt/atlassian-jira-software-standalone/conf/server.xml + xmlstarlet ed -L -u "/jira-database-config/jdbc-datasource/username" \ + -v "$db_username_value" /app/dbconfig.xml + xmlstarlet ed -L -u "/jira-database-config/jdbc-datasource/password" \ + -v "$db_username_value" /app/dbconfig.xml + + install -ojira -gjira -m660 /app/dbconfig.xml /var/jira/dbconfig.xml + /opt/atlassian-jira-software-standalone/bin/setenv.sh run + /opt/atlassian-jira-software-standalone/bin/start-jira.sh run +} + + +# usage: file_env VAR [DEFAULT] +# ie: file_env 'XYZ_DB_PASSWORD' 'example' +# (will allow for "$XYZ_DB_PASSWORD_FILE" to fill in the value of +# "$XYZ_DB_PASSWORD" from a file, especially for Docker's secrets feature) +function file_env() { + local var="$1" + local fileVar="${var}_FILE" + local def="${2:-}" + local varValue=$(env | grep -E "^${var}=" | sed -E -e "s/^${var}=//") + local fileVarValue=$(env | grep -E "^${fileVar}=" | sed -E -e "s/^${fileVar}=//") + if [ -n "${varValue}" ] && [ -n "${fileVarValue}" ]; then + echo >&2 "error: both $var and $fileVar are set (but are exclusive)" + exit 1 + fi + if [ -n "${varValue}" ]; then + export "$var"="${varValue}" + elif [ -n "${fileVarValue}" ]; then + export "$var"="$(cat "${fileVarValue}")" + elif [ -n "${def}" ]; then + export "$var"="$def" + fi + unset "$fileVar" +} + +main diff --git a/src/main/cljc/dda/c4k_jira/core.cljc b/src/main/cljc/dda/c4k_jira/core.cljc index 147ff90..78f0cde 100644 --- a/src/main/cljc/dda/c4k_jira/core.cljc +++ b/src/main/cljc/dda/c4k_jira/core.cljc @@ -5,30 +5,48 @@ #?(:clj [orchestra.core :refer [defn-spec]] :cljs [orchestra.core :refer-macros [defn-spec]]) [dda.c4k-common.yaml :as yaml] - [dda.c4k-jira.jira :as jira])) + [dda.c4k-jira.jira :as jira] + [dda.c4k-jira.postgres :as postgres])) (def config-defaults {:issuer :staging}) (def config? (s/keys :req-un [::jira/fqdn] - :opt-un [::jira/issuer])) + :opt-un [::jira/issuer ::jira/jira-data-volume-path + ::postgres/postgres-data-volume-path])) -(def auth? (s/keys :req-un [::jira/db-user-name ::jira/db-user-password])) +(def auth? (s/keys :req-un [::postgres/db-user-name ::postgres/db-user-password])) (defn-spec generate any? [my-config config? my-auth auth?] - (let [resulting-config (merge config-defaults my-config)] - (cs/join "\n" - [(yaml/to-string (jira/generate-secret my-auth)) - "---" - (yaml/to-string (jira/generate-persistent-volume)) - "---" - (yaml/to-string (jira/generate-pvc)) - "---" - (yaml/to-string (jira/generate-certificate resulting-config)) - "---" - (yaml/to-string (jira/generate-ingress resulting-config)) - "---" - (yaml/to-string (jira/generate-service)) - "---" - (yaml/to-string (jira/generate-pod resulting-config))]))) + (let [resulting-config (merge config-defaults my-config my-auth)] + (cs/join + "\n" + (into + [(yaml/to-string (postgres/generate-config)) + "---" + (yaml/to-string (postgres/generate-secret resulting-config))] + (when-some [{:keys [postgres-data-volume-path]} resulting-config] + ["---" + (yaml/to-string (postgres/generate-persistent-volume resulting-config))]) + ["---" + (yaml/to-string (postgres/generate-pvc)) + "---" + (yaml/to-string (postgres/generate-deployment)) + "---" + (yaml/to-string (postgres/generate-service))] + (when-some [{:keys [jira-data-volume-path]} resulting-config] + ["---" + (yaml/to-string (jira/generate-persistent-volume resulting-config))]) + ["---" + (yaml/to-string (jira/generate-pvc)) + "---" + (yaml/to-string (jira/generate-pod resulting-config)) + "---" + (yaml/to-string (jira/generate-service)) + "---" + (yaml/to-string (jira/generate-certificate resulting-config)) + "---" + (yaml/to-string (jira/generate-ingress resulting-config)) + "---" + (yaml/to-string (jira/generate-service))])))) diff --git a/src/main/cljc/dda/c4k_jira/jira.cljc b/src/main/cljc/dda/c4k_jira/jira.cljc index 5d9ab0d..ea2bd2d 100644 --- a/src/main/cljc/dda/c4k_jira/jira.cljc +++ b/src/main/cljc/dda/c4k_jira/jira.cljc @@ -2,24 +2,14 @@ (:require [clojure.spec.alpha :as s] [dda.c4k-common.yaml :as yaml] - [dda.c4k-common.base64 :as b64] [dda.c4k-common.common :as cm])) -(s/def ::db-user-name cm/bash-env-string?) -(s/def ::db-user-password cm/bash-env-string?) (s/def ::fqdn cm/fqdn-string?) (s/def ::issuer cm/letsencrypt-issuer?) - -(defn generate-secret [auth] - (let [{:keys [db-user-name db-user-password]} auth] - (-> - (yaml/from-string (yaml/load-resource "jira/secret.yaml")) - (assoc-in [:data :db-user-name] db-user-name) - (assoc-in [:data :db-user-password] db-user-password)))) +(s/def ::jira-data-volume-path string?) (defn generate-certificate [config] - (let [{:keys [fqdn issuer] - :or {issuer :staging}} config + (let [{:keys [fqdn issuer]} config letsencrypt-issuer (str "letsencrypt-" (name issuer) "-issuer")] (-> (yaml/from-string (yaml/load-resource "jira/certificate.yaml")) @@ -36,16 +26,19 @@ (assoc-in [:metadata :annotations :cert-manager.io/cluster-issuer] letsencrypt-issuer) (cm/replace-all-matching-values-by-new-value "fqdn" fqdn)))) -(defn generate-persistent-volume [] - (yaml/from-string (yaml/load-resource "jira/persistent-volume.yaml"))) +(defn generate-persistent-volume [config] + (let [{:keys [jira-data-volume-path]} config] + (-> + (yaml/from-string (yaml/load-resource "jira/persistent-volume.yaml")) + (assoc-in [:spec :hostPath :path] jira-data-volume-path)))) + +(defn generate-pod [config] + (let [{:keys [fqdn]} config] + (-> (yaml/from-string (yaml/load-resource "jira/pod.yaml")) + (cm/replace-named-value "FQDN" fqdn)))) (defn generate-pvc [] (yaml/from-string (yaml/load-resource "jira/pvc.yaml"))) (defn generate-service [] (yaml/from-string (yaml/load-resource "jira/service.yaml"))) - -(defn generate-pod [config] - (let [{:keys [fqdn db-user-name db-user-password]} config] - (-> (yaml/from-string (yaml/load-resource "jira/pod.yaml")) - (assoc-in [:spec :containers 0 :args] [fqdn, db-user-name, db-user-password])))) diff --git a/src/main/cljc/dda/c4k_jira/postgres.cljc b/src/main/cljc/dda/c4k_jira/postgres.cljc new file mode 100644 index 0000000..b9230e7 --- /dev/null +++ b/src/main/cljc/dda/c4k_jira/postgres.cljc @@ -0,0 +1,35 @@ +(ns dda.c4k-jira.postgres + (:require + [clojure.spec.alpha :as s] + [dda.c4k-common.yaml :as yaml] + [dda.c4k-common.base64 :as b64] + [dda.c4k-common.common :as cm])) + +(s/def ::postgres-db-user cm/bash-env-string?) +(s/def ::postgres-db-password cm/bash-env-string?) +(s/def ::postgres-data-volume-path string?) + +(defn generate-config [] + (yaml/from-string (yaml/load-resource "postgres/config.yaml"))) + +(defn generate-deployment [] + (yaml/from-string (yaml/load-resource "postgres/deployment.yaml"))) + +(defn generate-persistent-volume [config] + (let [{:keys [postgres-data-volume-path]} config] + (-> + (yaml/from-string (yaml/load-resource "postgres/persistent-volume.yaml")) + (assoc-in [:spec :hostPath :path] postgres-data-volume-path)))) + +(defn generate-pvc [] + (yaml/from-string (yaml/load-resource "postgres/pvc.yaml"))) + +(defn generate-secret [my-auth] + (let [{:keys [postgres-db-user postgres-db-password]} my-auth] + (-> + (yaml/from-string (yaml/load-resource "postgres/secret.yaml")) + (cm/replace-key-value :postgres-user (b64/encode postgres-db-user)) + (cm/replace-key-value :postgres-password (b64/encode postgres-db-password))))) + +(defn generate-service [] + (yaml/from-string (yaml/load-resource "postgres/service.yaml"))) diff --git a/src/main/resources/jira/pod.yaml b/src/main/resources/jira/pod.yaml index 826160f..625babf 100644 --- a/src/main/resources/jira/pod.yaml +++ b/src/main/resources/jira/pod.yaml @@ -6,15 +6,27 @@ metadata: app: jira spec: containers: - - name: jira-app - image: domaindrivenarchitecture/c4k-jira + - image: domaindrivenarchitecture/c4k-jira + name: jira-app imagePullPolicy: IfNotPresent + env: + - name: DB_USERNAME_FILE + value: /var/run/secrets/postgres-secret/postgres-user + - name: DB_PASSWORD_FILE + value: /var/run/secrets/postgres-secret/postgres-password + - name: FQDN + value: fqdn command: ["/app/entrypoint.sh"] - args: ["fqdn", "db-user-name", "db-user-password"] volumeMounts: - - mountPath: /var/jira - name: jira-data-volume + - mountPath: /var/jira + name: jira-data-volume + - name: postgres-secret-volume + mountPath: /var/run/secrets/postgres-secret + readOnly: true volumes: - name: jira-data-volume persistentVolumeClaim: claimName: jira-pvc + - name: postgres-secret-volume + secret: + secretName: postgres-secret diff --git a/src/main/resources/jira/secret.yaml b/src/main/resources/jira/secret.yaml deleted file mode 100644 index 75d990c..0000000 --- a/src/main/resources/jira/secret.yaml +++ /dev/null @@ -1,8 +0,0 @@ -apiVersion: v1 -kind: Secret -metadata: - name: jira-secret -type: Opaque -data: - db-user-name: admin - db-user-password: admin diff --git a/src/main/resources/postgres/config.yaml b/src/main/resources/postgres/config.yaml index 4006743..e2c62d5 100644 --- a/src/main/resources/postgres/config.yaml +++ b/src/main/resources/postgres/config.yaml @@ -5,7 +5,7 @@ metadata: labels: app: postgres data: - postgres-db: keycloak + postgres-db: jira postgresql.conf: | max_connections = 1000 shared_buffers = 512MB diff --git a/src/main/resources/postgres/deployment.yaml b/src/main/resources/postgres/deployment.yaml index 5448e3d..e7c0d48 100644 --- a/src/main/resources/postgres/deployment.yaml +++ b/src/main/resources/postgres/deployment.yaml @@ -40,7 +40,12 @@ spec: mountPath: /etc/postgresql/postgresql.conf subPath: postgresql.conf readOnly: true + - name: postgre-data-volume + mountPath: /var/lib/postgresql/data volumes: - name: postgres-config-volume configMap: name: postgres-config + - name: postgre-d + persistentVolumeClaim: + claimName: postgres-claim diff --git a/src/main/resources/postgres/persistent-volume.yaml b/src/main/resources/postgres/persistent-volume.yaml new file mode 100644 index 0000000..acc9b9d --- /dev/null +++ b/src/main/resources/postgres/persistent-volume.yaml @@ -0,0 +1,14 @@ +kind: PersistentVolume +apiVersion: v1 +metadata: + name: postgres-pv-volume + labels: + type: local +spec: + storageClassName: manual + accessModes: + - ReadWriteOnce + capacity: + storage: 10Gi + hostPath: + path: "/var/postgres" \ No newline at end of file diff --git a/src/main/resources/postgres/pvc.yaml b/src/main/resources/postgres/pvc.yaml new file mode 100644 index 0000000..3a127c7 --- /dev/null +++ b/src/main/resources/postgres/pvc.yaml @@ -0,0 +1,13 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: postgres-claim + labels: + app: postgres +spec: + storageClassName: manual + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 10Gi \ No newline at end of file diff --git a/src/test/cljc/dda/c4k_jira/jira_test.cljc b/src/test/cljc/dda/c4k_jira/jira_test.cljc index c5cd440..b635897 100644 --- a/src/test/cljc/dda/c4k_jira/jira_test.cljc +++ b/src/test/cljc/dda/c4k_jira/jira_test.cljc @@ -4,3 +4,81 @@ :cljs [cljs.test :refer-macros [deftest is are testing run-tests]]) [dda.c4k-jira.jira :as cut])) +(deftest should-generate-certificate + (is (= {:apiVersion "cert-manager.io/v1alpha2" + :kind "Certificate" + :metadata {:name "jira-cert", :namespace "default"} + :spec + {:secretName "jira-secret" + :commonName "xx" + :dnsNames ["xx"] + :issuerRef + {:name "letsencrypt-prod-issuer", :kind "ClusterIssuer"}}} + (cut/generate-certificate {:fqdn "xx" :issuer :prod})))) + +(deftest should-generate-ingress + (is (= {:apiVersion "extensions/v1beta1" + :kind "Ingress" + :metadata + {:name "ingress-jira" + :annotations + {:cert-manager.io/cluster-issuer + "letsencrypt-staging-issuer" + :nginx.ingress.kubernetes.io/proxy-body-size "256m" + :nginx.ingress.kubernetes.io/ssl-redirect "true" + :nginx.ingress.kubernetes.io/rewrite-target "/" + :nginx.ingress.kubernetes.io/proxy-connect-timeout "300" + :nginx.ingress.kubernetes.io/proxy-send-timeout "300" + :nginx.ingress.kubernetes.io/proxy-read-timeout "300"} + :namespace "default"} + :spec + {:tls [{:hosts ["xx"], :secretName "jira-secret"}] + :rules + [{:host "xx" + :http + {:paths + [{:path "/" + :backend + {:serviceName "jira-service", :servicePort 8080}}]}}]}} + (cut/generate-ingress {:fqdn "xx"})))) + +(deftest should-generate-persistent-volume + (is (= {:kind "PersistentVolume" + :apiVersion "v1" + :metadata {:name "jira-pv-volume", :labels {:type "local"}} + :spec + {:storageClassName "manual" + :accessModes ["ReadWriteOnce"] + :capacity {:storage "30Gi"} + :hostPath {:path "xx"}}} + (cut/generate-persistent-volume {:jira-data-volume-path "xx"})))) + +(deftest should-generate-pod + (is (= {:kind "Pod" + :apiVersion "v1" + :metadata {:name "jira-app", :labels {:app "jira"}} + :spec + {:containers + [{:image "domaindrivenarchitecture/c4k-jira" + :name "jira-app" + :imagePullPolicy "IfNotPresent" + :env + [{:name "DB_USERNAME_FILE" + :value + "/var/run/secrets/postgres-secret/postgres-user"} + {:name "DB_PASSWORD_FILE" + :value + "/var/run/secrets/postgres-secret/postgres-password"} + {:name "FQDN", :value "xx"}] + :command ["/app/entrypoint.sh"] + :volumeMounts + [{:mountPath "/var/jira", :name "jira-data-volume"} + {:name "postgres-secret-volume" + :mountPath "/var/run/secrets/postgres-secret" + :readOnly true}]}] + :volumes + [{:name "jira-data-volume" + :persistentVolumeClaim {:claimName "jira-pvc"}} + {:name "postgres-secret-volume" + :secret {:secretName "postgres-secret"}}]}} + (cut/generate-pod {:fqdn "xx"})))) diff --git a/src/test/cljc/dda/c4k_jira/postgres_test.cljc b/src/test/cljc/dda/c4k_jira/postgres_test.cljc new file mode 100644 index 0000000..58f639d --- /dev/null +++ b/src/test/cljc/dda/c4k_jira/postgres_test.cljc @@ -0,0 +1,26 @@ +(ns dda.c4k-jira.postgres-test + (:require + #?(:clj [clojure.test :refer [deftest is are testing run-tests]] + :cljs [cljs.test :refer-macros [deftest is are testing run-tests]]) + [dda.c4k-jira.postgres :as cut])) + +(deftest should-generate-persistent-volume + (is (= {:kind "PersistentVolume" + :apiVersion "v1" + :metadata + {:name "postgres-pv-volume", :labels {:type "local"}} + :spec + {:storageClassName "manual" + :accessModes ["ReadWriteOnce"] + :capacity {:storage "10Gi"} + :hostPath {:path "xx"}}} + (cut/generate-persistent-volume {:postgres-data-volume-path "xx"})))) + +(deftest should-generate-secret + (is (= {:apiVersion "v1" + :kind "Secret" + :metadata {:name "postgres-secret"} + :type "Opaque" + :data + {:postgres-user "eHgtdXM=", :postgres-password "eHgtcHc="}} + (cut/generate-secret {:postgres-db-user "xx-us" :postgres-db-password "xx-pw"})))) diff --git a/src/test/resources/invalid-auth.edn b/src/test/resources/invalid-auth.edn deleted file mode 100644 index db507d4..0000000 --- a/src/test/resources/invalid-auth.edn +++ /dev/null @@ -1 +0,0 @@ -{:xx {}} \ No newline at end of file diff --git a/src/test/resources/invalid-config.edn b/src/test/resources/invalid-config.edn deleted file mode 100644 index 4e2bda4..0000000 --- a/src/test/resources/invalid-config.edn +++ /dev/null @@ -1,27 +0,0 @@ -{:transform [{:source {:source-type :twitter - ;; optional, defaults to false - :include-replies? false - ;; optional, defaults to false - :include-rts? false - ;; Replace Twitter links by Nitter - :nitter-urls? 42 - ;; accounts you wish to mirror - :accounts ["arstechnica" "WIRED"]} - :target {:target-type :mastodon - ;; optional flag specifying wether the name of the account - ;; will be appended in the post, defaults to false - :append-screen-name? false - ;; optional visibility flag: direct, private, unlisted, public - ;; defaults to public - :visibility "unlisted" - ;; optional boolean to mark content as sensitive. Defaults to true. - :sensitive? true - ;; optional boolean defaults to false - ;; only sources containing media will be posted when set to true - :media-only? true - ;; optional limit for the post length. Defaults to 300. - :max-post-length 300 - ;; optional signature for posts. Defaults to "not present". - :signature "#newsbot"} - }] - :auth {}} \ No newline at end of file diff --git a/src/test/resources/valid-auth.edn b/src/test/resources/valid-auth.edn deleted file mode 100644 index df4843c..0000000 --- a/src/test/resources/valid-auth.edn +++ /dev/null @@ -1 +0,0 @@ -{:auth {}} \ No newline at end of file diff --git a/src/test/resources/valid-config.edn b/src/test/resources/valid-config.edn deleted file mode 100644 index 71187c8..0000000 --- a/src/test/resources/valid-config.edn +++ /dev/null @@ -1,27 +0,0 @@ -{:transform [{:source {:source-type :twitter - ;; optional, defaults to false - :include-replies? false - ;; optional, defaults to false - :include-rts? false - ;; Replace Twitter links by Nitter - :nitter-urls? false - ;; accounts you wish to mirror - :accounts ["arstechnica" "WIRED"]} - :target {:target-type :mastodon - ;; optional flag specifying wether the name of the account - ;; will be appended in the post, defaults to false - :append-screen-name? false - ;; optional visibility flag: direct, private, unlisted, public - ;; defaults to public - :visibility "unlisted" - ;; optional boolean to mark content as sensitive. Defaults to true. - :sensitive? true - ;; optional boolean defaults to false - ;; only sources containing media will be posted when set to true - :media-only? true - ;; optional limit for the post length. Defaults to 300. - :max-post-length 300 - ;; optional signature for posts. Defaults to "not present". - :signature "#newsbot"} - }] - :auth {}} \ No newline at end of file