Merge pull request 'add-backup' () from add-backup into main

Reviewed-on: 
This commit is contained in:
jem 2025-04-16 16:22:24 +00:00
commit bffe5281b7
10 changed files with 543 additions and 10 deletions

View file

@ -374,19 +374,20 @@ You can get a cluster local node-exporter, kube-state-metrics, pushgateway & pro
## Refactoring & Module Overview
| Module | Version | [namespaces][ns] | [split config and auth][split] | [backup monitoring][bak] |
| ------------- | ------- | :--------------: | :----------------------------: | :----------------------: |
| c4k-keycloak | 1.4 | x | x | x |
| c4k-taiga | 2.0 | x | x | |
| c4k-nextcloud | 11.0 | x | x | |
| c4k-jitsi | 3.0 | x | x | - |
| c4k-forgejo | 6.0 | x | x | x |
| c4k-stats | 1.0 | x | x | x |
| c4k-website | 2.0 | | | - |
| Module | Version | [namespaces][ns] | [split config and auth][split] | [backup monitoring][bak] | [commons backup][back-common] |
| ------------- | ------- | :--------------: | :----------------------------: | :----------------------: | :---------------------------: |
| c4k-keycloak | 1.4 | x | x | x | |
| c4k-taiga | 2.0 | x | x | | |
| c4k-nextcloud | 11.0 | x | x | | |
| c4k-jitsi | 3.0 | x | x | - | - |
| c4k-forgejo | 6.0 | x | x | x | |
| c4k-stats | 1.0 | x | x | x | x |
| c4k-website | 2.0 | | | - | - |
[split]: https://repo.prod.meissa.de/meissa/c4k-jitsi/commit/d4fb8ca9e2ab44f9f9923d2e09c81a61e44b39b2
[ns]: https://repo.prod.meissa.de/meissa/c4k-keycloak/commit/3639f3d5e6d5b364822a05b3d5d569bbc556a68b
[bak]: https://repo.prod.meissa.de/meissa/c4k-keycloak/pulls/5/files
[bak-common]:
## Development & Mirrors

View file

@ -1,4 +1,4 @@
(defproject org.domaindrivenarchitecture/c4k-common-cljs "10.0.1-SNAPSHOT"
(defproject org.domaindrivenarchitecture/c4k-common-cljs "11.0.0-SNAPSHOT"
:description "Contains predicates and tools for c4k"
:url "https://domaindrivenarchitecture.org"
:license {:name "Apache License, Version 2.0"

View file

@ -0,0 +1,43 @@
(ns dda.c4k-common.backup
(:require
[clojure.spec.alpha :as s]
#?(:clj [orchestra.core :refer [defn-spec]]
:cljs [orchestra.core :refer-macros [defn-spec]])
[dda.c4k-common.namespace :as ns]
[dda.c4k-common.backup.backup-internal :as int]))
(s/def ::mount-name ::int/mount-name)
(s/def ::pvc-name ::int/pvc-name)
(s/def ::mount-path ::int/mount-path)
(s/def ::app-name ::int/app-name)
(s/def ::backup-image ::int/backup-image)
(s/def ::backup-postgres ::int/backup-postgres)
(s/def ::backup-volume-mount ::int/backup-volume-mount)
(s/def ::aws-access-key-id ::int/aws-access-key-id)
(s/def ::aws-secret-access-key ::int/aws-secret-access-key)
(s/def ::restic-password ::int/restic-password)
(s/def ::restic-new-password ::int/restic-new-password)
(s/def ::restic-repository ::int/restic-repository)
(s/def ::config int/config?)
(s/def ::auth int/auth?)
(def default-config
(merge ns/default-config
{:backup-postgres false}))
(defn-spec config-objects seq?
[config ::config]
(let [resolved-config (merge default-config
config)]
[(int/config resolved-config)
(int/backup-restore-deployment resolved-config)
(int/backup-cron resolved-config)]))
(defn-spec auth-objects seq?
[config ::config
auth ::auth]
(let [resolved-config (merge default-config
config)]
[(int/secret resolved-config auth)]))

View file

@ -0,0 +1,142 @@
(ns dda.c4k-common.backup.backup-internal
(:require
[clojure.spec.alpha :as s]
#?(:cljs [shadow.resource :as rc])
#?(:clj [orchestra.core :refer [defn-spec]]
:cljs [orchestra.core :refer-macros [defn-spec]])
[dda.c4k-common.yaml :as yaml]
[dda.c4k-common.base64 :as b64]
[dda.c4k-common.common :as cm]
[dda.c4k-common.predicate :as p]
[dda.c4k-common.namespace :as ns]))
(s/def ::mount-name string?)
(s/def ::pvc-name string?)
(s/def ::mount-path string?)
(s/def ::app-name string?)
(s/def ::backup-image string?)
(s/def ::backup-postgres boolean?)
(s/def ::backup-volume-mount (s/keys :req-un [::mount-name ::pvc-name ::mount-path]))
(s/def ::aws-access-key-id p/bash-env-string?)
(s/def ::aws-secret-access-key p/bash-env-string?)
(s/def ::restic-password p/bash-env-string?)
(s/def ::restic-new-password p/bash-env-string?)
(s/def ::restic-repository p/bash-env-string?)
(def config? (s/keys :req-un [::ns/namespace
::app-name
::backup-image
::backup-postgres
::restic-repository]
:opt-un [::backup-volume-mount]))
(def auth? (s/keys :req-un [::restic-password ::aws-access-key-id ::aws-secret-access-key]
:opt-un [::restic-new-password]))
#?(:cljs
(defmethod yaml/load-resource :backup [resource-name]
(case resource-name
"backup/backup-restore-deployment.yaml" (rc/inline "backup/backup-restore-deployment.yaml")
"backup/backup-cron.yaml" (rc/inline "backup/backup-cron.yaml")
"backup/secret.yaml" (rc/inline "backup/secret.yaml")
"backup/config.yaml" (rc/inline "backup/config.yaml")
(throw (js/Error. (str "Undefined Resource: " resource-name))))))
(defn-spec backup-volumes seq?
[config config?]
(let [{:keys [backup-volume-mount]} config
backup-secret [{:name "backup-secret-volume",
:secret {:secretName "backup-secret"}}]]
(if (some? backup-volume-mount)
(into backup-secret [{:name (:mount-name backup-volume-mount),
:persistentVolumeClaim {:claimName (:pvc-name backup-volume-mount)}}])
backup-secret)))
(defn-spec backup-volume-mounts seq?
[config config?]
(let [{:keys [backup-volume-mount]} config
backup-secret [{:name "backup-secret-volume",
:mountPath "/var/run/secrets/backup-secrets",
:readOnly true}]]
(if (some? backup-volume-mount)
(into backup-secret [{:name (:mount-name backup-volume-mount),
:mountPath (:mount-path backup-volume-mount)}])
backup-secret)))
(defn-spec backup-env seq?
[config config?]
(let [{:keys [backup-postgres]} config
postgres-env [{:name "POSTGRES_USER",
:valueFrom
{:secretKeyRef {:name "postgres-secret", :key "postgres-user"}}}
{:name "POSTGRES_PASSWORD",
:valueFrom
{:secretKeyRef {:name "postgres-secret", :key "postgres-password"}}}
{:name "POSTGRES_DB",
:valueFrom
{:configMapKeyRef {:name "postgres-config", :key "postgres-db"}}}
{:name "POSTGRES_HOST", :value "postgresql-service:5432"}
{:name "POSTGRES_SERVICE", :value "postgresql-service"}
{:name "POSTGRES_PORT", :value "5432"}]
restic-env [{:name "AWS_DEFAULT_REGION", :value "eu-central-1"}
{:name "AWS_ACCESS_KEY_ID_FILE",
:value "/var/run/secrets/backup-secrets/aws-access-key-id"}
{:name "AWS_SECRET_ACCESS_KEY_FILE",
:value "/var/run/secrets/backup-secrets/aws-secret-access-key"}
{:name "RESTIC_REPOSITORY",
:valueFrom
{:configMapKeyRef {:name "backup-config", :key "restic-repository"}}}
{:name "RESTIC_PASSWORD_FILE",
:value "/var/run/secrets/backup-secrets/restic-password"}
{:name "RESTIC_NEW_PASSWORD_FILE",
:value "/var/run/secrets/backup-secrets/restic-new-password"}]]
(if backup-postgres (into postgres-env restic-env) restic-env)))
(defn-spec backup-restore-deployment map?
[config config?]
(let [{:keys [namespace app-name backup-image]} config]
(->
(ns/load-and-adjust-namespace "backup/backup-restore-deployment.yaml" namespace)
(assoc-in [:metadata :labels :app.kubernetes.io/part-of] app-name)
(assoc-in [:spec :template :metadata :labels :app.kubernetes.io/part-of] app-name)
(assoc-in [:spec :template :spec :containers 0 :image] backup-image)
(assoc-in [:spec :template :spec :containers 0 :env] (backup-env config))
(assoc-in [:spec :template :spec :containers 0 :volumeMounts] (backup-volume-mounts config))
(assoc-in [:spec :template :spec :volumes] (backup-volumes config)))))
(defn-spec backup-cron map?
[config config?]
(let [{:keys [namespace app-name backup-image]} config]
(->
(ns/load-and-adjust-namespace "backup/backup-cron.yaml" namespace)
(assoc-in [:metadata :labels :app.kubernetes.io/part-of] app-name)
(assoc-in [:spec :jobTemplate :spec :template :spec :containers 0 :image] backup-image)
(assoc-in [:spec :jobTemplate :spec :template :spec :containers 0 :env] (backup-env config))
(assoc-in [:spec :jobTemplate :spec :template :spec :containers 0 :volumeMounts] (backup-volume-mounts config))
(assoc-in [:spec :jobTemplate :spec :template :spec :volumes] (backup-volumes config)))))
(defn-spec config map?
[config config?]
(let [{:keys [restic-repository namespace app-name]} config]
(->
(ns/load-and-adjust-namespace "backup/config.yaml" namespace)
(assoc-in [:metadata :labels :app.kubernetes.io/part-of] app-name)
(cm/replace-key-value :restic-repository restic-repository))))
(defn-spec secret map?
[config config?
auth auth?]
(let [{:keys [namespace app-name]} config
{:keys [aws-access-key-id aws-secret-access-key
restic-password restic-new-password]} auth]
(as->
(ns/load-and-adjust-namespace "backup/secret.yaml" namespace) res
(assoc-in res [:metadata :labels :app.kubernetes.io/part-of] app-name)
(cm/replace-key-value res :#:app.kubernetes.io:part-of app-name)
(cm/replace-key-value res :aws-access-key-id (b64/encode aws-access-key-id))
(cm/replace-key-value res :aws-secret-access-key (b64/encode aws-secret-access-key))
(cm/replace-key-value res :restic-password (b64/encode restic-password))
(if (contains? auth :restic-new-password)
(assoc-in res [:data :restic-new-password] (b64/encode restic-new-password))
res))))

View file

@ -0,0 +1,25 @@
apiVersion: batch/v1
kind: CronJob
metadata:
name: backup-cron
namespace: REPLACE_ME
labels:
app.kubernetes.io/name: backup-cron
app.kubernetes.io/part-of: REPLACE_ME
spec:
schedule: "10 23 * * *"
successfulJobsHistoryLimit: 1
failedJobsHistoryLimit: 1
jobTemplate:
spec:
template:
spec:
containers:
- name: backup-cron-job
image: REPLACE_ME
imagePullPolicy: IfNotPresent
command: ["backup.bb"]
env: []
volumeMounts: []
volumes: []
restartPolicy: OnFailure

View file

@ -0,0 +1,29 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: backup-restore
namespace: REPLACE_ME
labels:
app.kubernetes.io/name: backup-restore
app.kubernetes.io/part-of: REPLACE_ME
spec:
replicas: 0
selector:
matchLabels:
app.kubernetes.io/name: backup-restore
strategy:
type: Recreate
template:
metadata:
labels:
app.kubernetes.io/name: backup-restore
app.kubernetes.io/part-of: REPLACE_ME
spec:
containers:
- image: REPLACE_ME
name: backup-restore
imagePullPolicy: IfNotPresent
command: ["wait.bb"]
env: []
volumeMounts: []
volumes: []

View file

@ -0,0 +1,10 @@
apiVersion: v1
kind: ConfigMap
metadata:
name: backup-config
namespace: REPLACE_ME
labels:
app.kubernetes.io/name: backup-config
app.kubernetes.io/part-of: REPLACE_ME
data:
restic-repository: restic-repository

View file

@ -0,0 +1,13 @@
apiVersion: v1
kind: Secret
metadata:
name: backup-secret
namespace: REPLACE_ME
labels:
app.kubernetes.io/name: backup-secret
app.kubernetes.io/part-of: REPLACE_ME
type: Opaque
data:
aws-access-key-id: REPLACE_ME
aws-secret-access-key: REPLACE_ME
restic-password: REPLACE_ME

View file

@ -0,0 +1,241 @@
(ns dda.c4k-common.backup.backup-internal-test
(:require
#?(:clj [clojure.test :refer [deftest is are testing run-tests]]
:cljs [cljs.test :refer-macros [deftest is are testing run-tests]])
[clojure.spec.test.alpha :as st]
[dda.c4k-common.backup.backup-internal :as cut]))
(st/instrument `cut/secret)
(st/instrument `cut/config)
(st/instrument `cut/backup-restore-deployment)
(st/instrument `cut/backup-cron)
(def config {:namespace "ns"
:app-name "app-name"
:backup-image "image"
:backup-postgres false
:restic-repository "repo"})
(deftest should-generate-backup-cron
(is (= {:name "backup-cron",
:namespace "ns",
:labels #:app.kubernetes.io{:name "backup-cron"
:part-of "app-name"}}
(:metadata (cut/backup-cron config))))
(is (= "image"
(get-in (cut/backup-cron config)
[:spec :jobTemplate :spec :template :spec :containers 0 :image]))))
(deftest should-generate-backup-cron-env
(is (= [{:name "AWS_DEFAULT_REGION", :value "eu-central-1"}
{:name "AWS_ACCESS_KEY_ID_FILE",
:value "/var/run/secrets/backup-secrets/aws-access-key-id"}
{:name "AWS_SECRET_ACCESS_KEY_FILE",
:value "/var/run/secrets/backup-secrets/aws-secret-access-key"}
{:name "RESTIC_REPOSITORY",
:valueFrom
{:configMapKeyRef {:name "backup-config", :key "restic-repository"}}}
{:name "RESTIC_PASSWORD_FILE",
:value "/var/run/secrets/backup-secrets/restic-password"}
{:name "RESTIC_NEW_PASSWORD_FILE",
:value "/var/run/secrets/backup-secrets/restic-new-password"}]
(get-in (cut/backup-cron config)
[:spec :jobTemplate :spec :template :spec :containers 0 :env])))
(is (= [{:name "POSTGRES_USER",
:valueFrom
{:secretKeyRef {:name "postgres-secret", :key "postgres-user"}}}
{:name "POSTGRES_PASSWORD",
:valueFrom
{:secretKeyRef {:name "postgres-secret", :key "postgres-password"}}}
{:name "POSTGRES_DB",
:valueFrom
{:configMapKeyRef {:name "postgres-config", :key "postgres-db"}}}
{:name "POSTGRES_HOST", :value "postgresql-service:5432"}
{:name "POSTGRES_SERVICE", :value "postgresql-service"}
{:name "POSTGRES_PORT", :value "5432"}
{:name "AWS_DEFAULT_REGION", :value "eu-central-1"}
{:name "AWS_ACCESS_KEY_ID_FILE",
:value "/var/run/secrets/backup-secrets/aws-access-key-id"}
{:name "AWS_SECRET_ACCESS_KEY_FILE",
:value "/var/run/secrets/backup-secrets/aws-secret-access-key"}
{:name "RESTIC_REPOSITORY",
:valueFrom
{:configMapKeyRef {:name "backup-config", :key "restic-repository"}}}
{:name "RESTIC_PASSWORD_FILE",
:value "/var/run/secrets/backup-secrets/restic-password"}
{:name "RESTIC_NEW_PASSWORD_FILE",
:value "/var/run/secrets/backup-secrets/restic-new-password"}]
(get-in (cut/backup-cron (merge config
{:backup-postgres true}))
[:spec :jobTemplate :spec :template :spec :containers 0 :env]))))
(deftest should-generate-backup-cron-volume-mounts
(is (= [{:name "backup-secret-volume",
:mountPath "/var/run/secrets/backup-secrets",
:readOnly true}]
(get-in (cut/backup-cron config)
[:spec :jobTemplate :spec :template :spec :containers 0 :volumeMounts])))
(is (= [{:name "backup-secret-volume",
:mountPath "/var/run/secrets/backup-secrets",
:readOnly true}
{:name "forgejo-data-volume",
:mountPath "/var/backups"}]
(get-in (cut/backup-cron (merge config
{:backup-volume-mount
{:mount-name "forgejo-data-volume"
:pvc-name "forgejo-data-pvc"
:mount-path "/var/backups"}}))
[:spec :jobTemplate :spec :template :spec :containers 0 :volumeMounts]))))
(deftest should-generate-backup-cron-volumes
(is (= [{:name "backup-secret-volume",
:secret {:secretName "backup-secret"}}]
(get-in (cut/backup-cron config)
[:spec :jobTemplate :spec :template :spec :volumes])))
(is (= [{:name "backup-secret-volume",
:secret {:secretName "backup-secret"}}
{:name "forgejo-data-volume",
:persistentVolumeClaim {:claimName "forgejo-data-pvc"}}]
(get-in (cut/backup-cron (merge config
{:backup-volume-mount
{:mount-name "forgejo-data-volume"
:pvc-name "forgejo-data-pvc"
:mount-path "/var/backups"}}))
[:spec :jobTemplate :spec :template :spec :volumes]))))
(deftest should-generate-backup-restore-deployment
(is (= {:name "backup-restore",
:namespace "ns",
:labels #:app.kubernetes.io{:name "backup-restore"
:part-of "app-name"}}
(:metadata (cut/backup-restore-deployment config))))
(is (= {:labels #:app.kubernetes.io{:name "backup-restore"
:part-of "app-name"}}
(get-in (cut/backup-restore-deployment config)
[:spec :template :metadata])))
(is (= "image"
(get-in (cut/backup-restore-deployment config)
[:spec :template :spec :containers 0 :image]))))
(deftest should-generate-backup-restore-deployment-env
(is (= [{:name "AWS_DEFAULT_REGION", :value "eu-central-1"}
{:name "AWS_ACCESS_KEY_ID_FILE",
:value "/var/run/secrets/backup-secrets/aws-access-key-id"}
{:name "AWS_SECRET_ACCESS_KEY_FILE",
:value "/var/run/secrets/backup-secrets/aws-secret-access-key"}
{:name "RESTIC_REPOSITORY",
:valueFrom
{:configMapKeyRef {:name "backup-config", :key "restic-repository"}}}
{:name "RESTIC_PASSWORD_FILE",
:value "/var/run/secrets/backup-secrets/restic-password"}
{:name "RESTIC_NEW_PASSWORD_FILE",
:value "/var/run/secrets/backup-secrets/restic-new-password"}]
(get-in (cut/backup-restore-deployment config)
[:spec :template :spec :containers 0 :env])))
(is (= [{:name "POSTGRES_USER",
:valueFrom
{:secretKeyRef {:name "postgres-secret", :key "postgres-user"}}}
{:name "POSTGRES_PASSWORD",
:valueFrom
{:secretKeyRef {:name "postgres-secret", :key "postgres-password"}}}
{:name "POSTGRES_DB",
:valueFrom
{:configMapKeyRef {:name "postgres-config", :key "postgres-db"}}}
{:name "POSTGRES_HOST", :value "postgresql-service:5432"}
{:name "POSTGRES_SERVICE", :value "postgresql-service"}
{:name "POSTGRES_PORT", :value "5432"}
{:name "AWS_DEFAULT_REGION", :value "eu-central-1"}
{:name "AWS_ACCESS_KEY_ID_FILE",
:value "/var/run/secrets/backup-secrets/aws-access-key-id"}
{:name "AWS_SECRET_ACCESS_KEY_FILE",
:value "/var/run/secrets/backup-secrets/aws-secret-access-key"}
{:name "RESTIC_REPOSITORY",
:valueFrom
{:configMapKeyRef {:name "backup-config", :key "restic-repository"}}}
{:name "RESTIC_PASSWORD_FILE",
:value "/var/run/secrets/backup-secrets/restic-password"}
{:name "RESTIC_NEW_PASSWORD_FILE",
:value "/var/run/secrets/backup-secrets/restic-new-password"}]
(get-in (cut/backup-restore-deployment (merge config
{:backup-postgres true}))
[:spec :template :spec :containers 0 :env]))))
(deftest should-generate-backup-restore-deployment-volume-mounts
(is (= [{:name "backup-secret-volume",
:mountPath "/var/run/secrets/backup-secrets",
:readOnly true}]
(get-in (cut/backup-restore-deployment config)
[:spec :template :spec :containers 0 :volumeMounts])))
(is (= [{:name "backup-secret-volume",
:mountPath "/var/run/secrets/backup-secrets",
:readOnly true}
{:name "forgejo-data-volume",
:mountPath "/var/backups"}]
(get-in (cut/backup-restore-deployment (merge config
{:backup-volume-mount
{:mount-name "forgejo-data-volume"
:pvc-name "forgejo-data-pvc"
:mount-path "/var/backups"}}))
[:spec :template :spec :containers 0 :volumeMounts]))))
(deftest should-generate-backup-restore-deployment-volumes
(is (= [{:name "backup-secret-volume",
:secret {:secretName "backup-secret"}}]
(get-in (cut/backup-restore-deployment config)
[:spec :template :spec :volumes])))
(is (= [{:name "backup-secret-volume",
:secret {:secretName "backup-secret"}}
{:name "forgejo-data-volume",
:persistentVolumeClaim {:claimName "forgejo-data-pvc"}}]
(get-in (cut/backup-restore-deployment (merge config
{:backup-volume-mount
{:mount-name "forgejo-data-volume"
:pvc-name "forgejo-data-pvc"
:mount-path "/var/backups"}}))
[:spec :template :spec :volumes]))))
(deftest should-generate-config
(is (= {:apiVersion "v1",
:kind "ConfigMap",
:metadata
{:name "backup-config",
:namespace "ns",
:labels #:app.kubernetes.io{:name "backup-config",
:part-of "app-name"}},
:data {:restic-repository "repo"}}
(cut/config config))))
(deftest should-generate-secret
(is (= {:apiVersion "v1",
:kind "Secret",
:metadata {:name "backup-secret", :namespace "ns"
:labels #:app.kubernetes.io{:name "backup-secret",
:part-of "app-name"}},
:type "Opaque",
:data
{:aws-access-key-id "YWtp",
:aws-secret-access-key "YXNhaw==",
:restic-password "cnB3"}}
(cut/secret config
{:restic-password "rpw"
:aws-access-key-id "aki"
:aws-secret-access-key "asak"})))
(is (= {:apiVersion "v1",
:kind "Secret",
:metadata {:name "backup-secret",
:namespace "ns"
:labels #:app.kubernetes.io{:name "backup-secret",
:part-of "app-name"}},
:type "Opaque",
:data
{:aws-access-key-id "YWtp",
:aws-secret-access-key "YXNhaw==",
:restic-new-password "bnJwdw=="
:restic-password "cnB3"}}
(cut/secret config
{:restic-password "rpw"
:restic-new-password "nrpw"
:aws-access-key-id "aki"
:aws-secret-access-key "asak"}))))

View file

@ -0,0 +1,29 @@
(ns dda.c4k-common.backup-test
(:require
#?(:clj [clojure.test :refer [deftest is are testing run-tests]]
:cljs [cljs.test :refer-macros [deftest is are testing run-tests]])
[clojure.spec.test.alpha :as st]
[dda.c4k-common.backup :as cut]))
(st/instrument `cut/config-objects)
(st/instrument `cut/auth-objects)
(def config {:namespace "ns"
:app-name "app-name"
:backup-image "image"
:backup-postgres true
:restic-repository "repo"
:backup-volume-mount
{:mount-name "forgejo-data-volume"
:pvc-name "forgejo-data-pvc"
:mount-path "/var/backups"}})
(deftest should-generate
(is (= 3
(count (cut/config-objects config))))
(is (= 1
(count (cut/auth-objects config
{:restic-password "rpw"
:restic-new-password "nrpw"
:aws-access-key-id "aki"
:aws-secret-access-key "asak"})))))