From e5a8f9d9b446cffef5f470d2c41740033ef5861a Mon Sep 17 00:00:00 2001 From: Clemens Geibel Date: Fri, 9 Jul 2021 15:35:05 +0200 Subject: [PATCH] Added backup yamls --- .../docker-backup/test/serverspec.edn | 1 - src/main/cljc/dda/c4k_jira/backup.cljc | 28 ++++++ src/main/cljc/dda/c4k_jira/core.cljc | 14 ++- src/main/resources/backup/config.yaml | 9 ++ src/main/resources/backup/cron.yaml | 68 ++++++++++++++ src/main/resources/backup/secret.yaml | 9 ++ src/test/cljc/dda/c4k_jira/backup_test.cljc | 93 +++++++++++++++++++ src/test/cljc/dda/c4k_jira/core_test.cljc | 16 +++- valid-auth.edn | 5 +- valid-config.edn | 3 +- 10 files changed, 235 insertions(+), 11 deletions(-) create mode 100644 src/main/cljc/dda/c4k_jira/backup.cljc create mode 100644 src/main/resources/backup/config.yaml create mode 100644 src/main/resources/backup/cron.yaml create mode 100644 src/main/resources/backup/secret.yaml create mode 100644 src/test/cljc/dda/c4k_jira/backup_test.cljc diff --git a/infrastrucure/docker-backup/test/serverspec.edn b/infrastrucure/docker-backup/test/serverspec.edn index 37bbbfb..09623c7 100644 --- a/infrastrucure/docker-backup/test/serverspec.edn +++ b/infrastrucure/docker-backup/test/serverspec.edn @@ -1,7 +1,6 @@ {:file [{:path "/usr/local/bin/init.sh" :mod "700"} {:path "/usr/local/bin/backup.sh" :mod "700"} {:path "/usr/local/bin/restore.sh" :mod "700"} - {:path "/usr/local/bin/export-db.sh" :mod "700"} {:path "/usr/local/bin/restic-snapshots.sh" :mod "700"} {:path "/entrypoint.sh" :mod "700"} {:path "/entrypoint-start-and-wait.sh" :mod "700"}]} diff --git a/src/main/cljc/dda/c4k_jira/backup.cljc b/src/main/cljc/dda/c4k_jira/backup.cljc new file mode 100644 index 0000000..ee09ef1 --- /dev/null +++ b/src/main/cljc/dda/c4k_jira/backup.cljc @@ -0,0 +1,28 @@ +(ns dda.c4k-jira.backup + (:require + [clojure.spec.alpha :as s] + [dda.c4k-common.yaml :as yaml] + [dda.c4k-common.base64 :as b64] + [dda.c4k-common.common :as cm])) + +(s/def ::aws-access-key-id cm/bash-env-string?) +(s/def ::aws-secret-access-key cm/bash-env-string?) +(s/def ::restic-password cm/bash-env-string?) +(s/def ::restic-repository cm/bash-env-string?) + +(defn generate-config [my-conf] + (let [{:keys [restic-repository]} my-conf] + (-> + (yaml/from-string (yaml/load-resource "backup/config.yaml")) + (cm/replace-key-value :restic-repository (b64/encode restic-repository))))) + +(defn generate-cron [] + (yaml/from-string (yaml/load-resource "backup/cron.yaml"))) + +(defn generate-secret [my-auth] + (let [{:keys [aws-access-key-id aws-secret-access-key restic-password]} my-auth] + (-> + (yaml/from-string (yaml/load-resource "backup/secret.yaml")) + (cm/replace-key-value :aws-access-key-id (b64/encode aws-access-key-id)) + (cm/replace-key-value :aws-secret-access-key (b64/encode aws-secret-access-key)) + (cm/replace-key-value :restic-password (b64/encode restic-password))))) diff --git a/src/main/cljc/dda/c4k_jira/core.cljc b/src/main/cljc/dda/c4k_jira/core.cljc index 5ce7381..9e7a393 100644 --- a/src/main/cljc/dda/c4k_jira/core.cljc +++ b/src/main/cljc/dda/c4k_jira/core.cljc @@ -6,15 +6,18 @@ :cljs [orchestra.core :refer-macros [defn-spec]]) [dda.c4k-common.yaml :as yaml] [dda.c4k-jira.jira :as jira] - [dda.c4k-jira.postgres :as postgres])) + [dda.c4k-jira.postgres :as postgres] + [dda.c4k-jira.backup :as backup])) (def config-defaults {:issuer :staging}) -(def config? (s/keys :req-un [::jira/fqdn] +(def config? (s/keys :req-un [::jira/fqdn ::restic-repository] :opt-un [::jira/issuer ::jira/jira-data-volume-path ::postgres/postgres-data-volume-path])) -(def auth? (s/keys :req-un [::postgres/postgres-db-user ::postgres/postgres-db-password])) +(def auth? (s/keys :req-un [::postgres/postgres-db-user ::postgres/postgres-db-password + ::aws-access-key-id ::aws-secret-access-key + ::restic-password])) (defn k8s-objects [config] (into @@ -33,7 +36,10 @@ (yaml/to-string (jira/generate-service)) (yaml/to-string (jira/generate-certificate config)) (yaml/to-string (jira/generate-ingress config)) - (yaml/to-string (jira/generate-service))]))) + (yaml/to-string (jira/generate-service))] + [(yaml/to-string (backup/generate-config config)) + (yaml/to-string (backup/generate-secret config)) + (yaml/to-string (backup/generate-cron))]))) (defn-spec generate any? [my-config config? diff --git a/src/main/resources/backup/config.yaml b/src/main/resources/backup/config.yaml new file mode 100644 index 0000000..64b775d --- /dev/null +++ b/src/main/resources/backup/config.yaml @@ -0,0 +1,9 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: backup-config + labels: + app.kubernetes.io/name: backup + app.kubernetes.io/part-of: jira +data: + restic-repository: restic-repository \ No newline at end of file diff --git a/src/main/resources/backup/cron.yaml b/src/main/resources/backup/cron.yaml new file mode 100644 index 0000000..a98bcec --- /dev/null +++ b/src/main/resources/backup/cron.yaml @@ -0,0 +1,68 @@ +apiVersion: batch/v1beta1 +kind: CronJob +metadata: + name: jira-backup + labels: + app.kubernetes.part-of: jira +spec: + schedule: "10 23 * * *" + successfulJobsHistoryLimit: 0 + failedJobsHistoryLimit: 0 + jobTemplate: + spec: + template: + spec: + containers: + - name: backup-app + image: domaindrivenarchitecture/c4k-jira-backup + imagePullPolicy: IfNotPresent + command: ["/entrypoint.sh"] + env: + - name: POSTGRES_USER + valueFrom: + secretKeyRef: + name: postgres-secret + key: postgres-user + - name: POSTGRES_PASSWORD + valueFrom: + secretKeyRef: + name: postgres-secret + key: postgres-password + - name: POSTGRES_DB + valueFrom: + configMapKeyRef: + name: postgres-config + key: postgres-db + - name: POSTGRES_HOST + value: "postgresql-service:5432" + - name: POSTGRES_SERVICE + value: "postgresql-service" + - name: POSTGRES_PORT + value: "5432" + - name: AWS_DEFAULT_REGION + value: eu-central-1 + - name: AWS_ACCESS_KEY_ID_FILE + value: /var/run/secrets/backup-secrets/aws-access-key-id + - name: AWS_SECRET_ACCESS_KEY_FILE + value: /var/run/secrets/backup-secrets/aws-secret-access-key + - name: RESTIC_REPOSITORY + valueFrom: + configMapKeyRef: + name: backup-config + key: restic-repository + - name: RESTIC_PASSWORD_FILE + value: /var/run/secrets/backup-secrets/restic-password + volumeMounts: + - name: jira-data-volume + mountPath: /var/backups + - name: backup-secret-volume + mountPath: /var/run/secrets/backup-secrets + readOnly: true + volumes: + - name: jira-data-volume + persistentVolumeClaim: + claimName: jira-pvc + - name: backup-secret-volume + secret: + secretName: backup-secret + restartPolicy: OnFailure \ No newline at end of file diff --git a/src/main/resources/backup/secret.yaml b/src/main/resources/backup/secret.yaml new file mode 100644 index 0000000..c5809e0 --- /dev/null +++ b/src/main/resources/backup/secret.yaml @@ -0,0 +1,9 @@ +apiVersion: v1 +kind: Secret +metadata: + name: backup-secret +type: Opaque +data: + aws-access-key-id: aws-access-key-id + aws-secret-access-key: aws-secret-access-key + restic-password: restic-password \ No newline at end of file diff --git a/src/test/cljc/dda/c4k_jira/backup_test.cljc b/src/test/cljc/dda/c4k_jira/backup_test.cljc new file mode 100644 index 0000000..46b6133 --- /dev/null +++ b/src/test/cljc/dda/c4k_jira/backup_test.cljc @@ -0,0 +1,93 @@ +(ns dda.c4k-jira.backup-test + (:require + #?(:clj [clojure.test :refer [deftest is are testing run-tests]] + :cljs [cljs.test :refer-macros [deftest is are testing run-tests]]) + [dda.c4k-jira.backup :as cut])) + + +(deftest should-generate-secret + (is (= {:apiVersion "v1" + :kind "Secret" + :metadata {:name "backup-secret"} + :type "Opaque" + :data + {:aws-access-key-id "YXdzLWlk", :aws-secret-access-key "YXdzLXNlY3JldA==", :restic-password "cmVzdGljLXB3"}} + (cut/generate-secret {:aws-access-key-id "aws-id" :aws-secret-access-key "aws-secret" :restic-password "restic-pw"})))) + +(deftest should-generate-config + (is (= {:apiVersion "v1" + :kind "ConfigMap" + :metadata {:name "backup-config" + :labels {:app.kubernetes.io/name "backup" + :app.kubernetes.io/part-of "jira"}} + :data + {:restic-repository "cmVzdGljLXJlcG9zaXRvcnk="}} + (cut/generate-config {:restic-repository "restic-repository"})))) + +(deftest should-generate-cron + (is (= {:apiVersion "batch/v1beta1" + :kind "CronJob" + :metadata {:name "jira-backup" + :labels {:app.kubernetes.part-of "jira"}} + :spec {:schedule "10 23 * * *" + :successfulJobsHistoryLimit 0 + :failedJobsHistoryLimit 0 + :jobTemplate + {:spec + {:template + {:spec + {:containers + [{:name "backup-app" + :image "domaindrivenarchitecture/c4k-jira-backup" + :imagePullPolicy "IfNotPresent" + :command ["/entrypoint.sh"] + :env + [{:name "POSTGRES_USER" + :valueFrom + {:secretKeyRef + {:name "postgres-secret" + :key "postgres-user"}}} + {:name "POSTGRES_PASSWORD" + :valueFrom + {:secretKeyRef + {:name "postgres-secret" + :key "postgres-password"}}} + {:name "POSTGRES_DB" + :valueFrom + {:configMapKeyRef + {:name "postgres-config" + :key "postgres-db"}}} + {:name "POSTGRES_HOST" + :value "postgresql-service:5432"} + {:name "POSTGRES_SERVICE" + :value "postgresql-service"} + {:name "POSTGRES_PORT" + :value "5432"} + {:name "AWS_DEFAULT_REGION" + :value "eu-central-1"} + {:name "AWS_ACCESS_KEY_ID_FILE" + :value "/var/run/secrets/backup-secrets/aws-access-key-id"} + {:name "AWS_SECRET_ACCESS_KEY_FILE" + :value "/var/run/secrets/backup-secrets/aws-secret-access-key"} + {:name "RESTIC_REPOSITORY" + :valueFrom + {:configMapKeyRef + {:name "backup-config" + :key "restic-repository"}}} + {:name "RESTIC_PASSWORD_FILE" + :value "/var/run/secrets/backup-secrets/restic-password"}] + :volumeMounts + [{:name "jira-data-volume" + :mountPath "/var/backups"} + {:name "backup-secret-volume" + :mountPath "/var/run/secrets/backup-secrets" + :readOnly true}]}] + :volumes + [{:name "jira-data-volume" + :persistentVolumeClaim + {:claimName "jira-pvc"}} + {:name "backup-secret-volume" + :secret + {:secretName "backup-secret"}}] + :restartPolicy "OnFailure"}}}}}} + (cut/generate-cron)))) diff --git a/src/test/cljc/dda/c4k_jira/core_test.cljc b/src/test/cljc/dda/c4k_jira/core_test.cljc index 10b9c47..557c308 100644 --- a/src/test/cljc/dda/c4k_jira/core_test.cljc +++ b/src/test/cljc/dda/c4k_jira/core_test.cljc @@ -5,15 +5,23 @@ [dda.c4k-jira.core :as cut])) (deftest should-k8s-objects - (is (= 13 + (is (= 16 (count (cut/k8s-objects {:fqdn "jira-neu.prod.meissa-gmbh.de" :postgres-db-user "jira" :postgres-db-password "jira-db-password" :issuer :prod :jira-data-volume-path "/var/jira" - :postgres-data-volume-path "/var/postgres"})))) - (is (= 11 + :postgres-data-volume-path "/var/postgres" + :aws-access-key-id "aws-id" + :aws-secret-access-key "aws-secret" + :restic-password "restic-pw" + :restic-repository "restic-repository"})))) + (is (= 14 (count (cut/k8s-objects {:fqdn "jira-neu.prod.meissa-gmbh.de" :postgres-db-user "jira" :postgres-db-password "jira-db-password" - :issuer :prod}))))) + :issuer :prod + :aws-access-key-id "aws-id" + :aws-secret-access-key "aws-secret" + :restic-password "restic-pw" + :restic-repository "restic-repository"}))))) diff --git a/valid-auth.edn b/valid-auth.edn index a07c585..a017a56 100644 --- a/valid-auth.edn +++ b/valid-auth.edn @@ -1,2 +1,5 @@ {:postgres-db-user "jira" - :postgres-db-password "jira-db-password"} \ No newline at end of file + :postgres-db-password "jira-db-password" + :aws-access-key-id "aws-id" + :aws-secret-access-key "aws-secret" + :restic-password "restic-password"} \ No newline at end of file diff --git a/valid-config.edn b/valid-config.edn index 7a85b54..be16615 100644 --- a/valid-config.edn +++ b/valid-config.edn @@ -1,3 +1,4 @@ {:fqdn "jira-neu.prod.meissa-gmbh.de" :jira-data-volume-path "/var/jira" - :postgres-data-volume-path "/var/postgres"} \ No newline at end of file + :postgres-data-volume-path "/var/postgres" + :restic-repository "restic-repository"} \ No newline at end of file