From dd5d835b21bbd6f6049c4bc49b67025dc79fb8b7 Mon Sep 17 00:00:00 2001 From: Clemens Geibel Date: Fri, 19 Aug 2022 11:59:04 +0200 Subject: [PATCH 01/14] Added backup skeleton --- infrastructure/docker-backup/build.py | 51 +++++++++++++++++++ infrastructure/docker-backup/image/Dockerfile | 5 ++ .../docker-backup/image/resources/backup.sh | 20 ++++++++ .../resources/entrypoint-start-and-wait.sh | 11 ++++ .../image/resources/entrypoint.sh | 9 ++++ .../docker-backup/image/resources/init.sh | 12 +++++ .../docker-backup/image/resources/install.sh | 11 ++++ .../image/resources/restic-snapshots.sh | 15 ++++++ .../docker-backup/image/resources/restore.sh | 39 ++++++++++++++ infrastructure/docker-backup/test/Dockerfile | 11 ++++ .../docker-backup/test/serverspec.edn | 6 +++ .../backup/backup-restore-deployment.yaml | 50 ++++++++++++++++++ src/main/resources/backup/config.yaml | 9 ++++ src/main/resources/backup/cron.yaml | 47 +++++++++++++++++ src/main/resources/backup/secret.yaml | 9 ++++ 15 files changed, 305 insertions(+) create mode 100644 infrastructure/docker-backup/build.py create mode 100644 infrastructure/docker-backup/image/Dockerfile create mode 100755 infrastructure/docker-backup/image/resources/backup.sh create mode 100644 infrastructure/docker-backup/image/resources/entrypoint-start-and-wait.sh create mode 100755 infrastructure/docker-backup/image/resources/entrypoint.sh create mode 100755 infrastructure/docker-backup/image/resources/init.sh create mode 100755 infrastructure/docker-backup/image/resources/install.sh create mode 100755 infrastructure/docker-backup/image/resources/restic-snapshots.sh create mode 100755 infrastructure/docker-backup/image/resources/restore.sh create mode 100644 infrastructure/docker-backup/test/Dockerfile create mode 100644 infrastructure/docker-backup/test/serverspec.edn create mode 100644 src/main/resources/backup/backup-restore-deployment.yaml create mode 100644 src/main/resources/backup/config.yaml create mode 100644 src/main/resources/backup/cron.yaml create mode 100644 src/main/resources/backup/secret.yaml diff --git a/infrastructure/docker-backup/build.py b/infrastructure/docker-backup/build.py new file mode 100644 index 0000000..fbe23c0 --- /dev/null +++ b/infrastructure/docker-backup/build.py @@ -0,0 +1,51 @@ +from os import environ +from pybuilder.core import task, init +from ddadevops import * +import logging + +name = 'c4k-gitea-backup' +MODULE = 'docker' +PROJECT_ROOT_PATH = '../..' + +class MyBuild(DevopsDockerBuild): + pass + +@init +def initialize(project): + project.build_depends_on('ddadevops>=0.12.4') + stage = 'prod' + dockerhub_user = environ.get('DOCKERHUB_USER') + if not dockerhub_user: + dockerhub_user = gopass_field_from_path('meissa/web/docker.com', 'login') + dockerhub_password = environ.get('DOCKERHUB_PASSWORD') + if not dockerhub_password: + dockerhub_password = gopass_password_from_path('meissa/web/docker.com') + tag = environ.get('CI_COMMIT_TAG') + if not tag: + tag = get_tag_from_latest_commit() + config = create_devops_docker_build_config( + stage, PROJECT_ROOT_PATH, MODULE, dockerhub_user, dockerhub_password, docker_publish_tag=tag) + build = MyBuild(project, config) + build.initialize_build_dir() + + +@task +def image(project): + build = get_devops_build(project) + build.image() + +@task +def drun(project): + build = get_devops_build(project) + build.drun() + +@task +def publish(project): + build = get_devops_build(project) + build.dockerhub_login() + build.dockerhub_publish() + +@task +def test(project): + build = get_devops_build(project) + build.test() diff --git a/infrastructure/docker-backup/image/Dockerfile b/infrastructure/docker-backup/image/Dockerfile new file mode 100644 index 0000000..9b6d4f0 --- /dev/null +++ b/infrastructure/docker-backup/image/Dockerfile @@ -0,0 +1,5 @@ +FROM domaindrivenarchitecture/dda-backup:1.0.5 + +# Prepare Entrypoint Script +ADD resources /tmp +RUN /tmp/install.sh diff --git a/infrastructure/docker-backup/image/resources/backup.sh b/infrastructure/docker-backup/image/resources/backup.sh new file mode 100755 index 0000000..9bdbde1 --- /dev/null +++ b/infrastructure/docker-backup/image/resources/backup.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +set -o pipefail + +function main() { + file_env AWS_ACCESS_KEY_ID + file_env AWS_SECRET_ACCESS_KEY + file_env RESTIC_DAYS_TO_KEEP 30 + file_env RESTIC_MONTHS_TO_KEEP 12 + + #backup-roles 'TODO' + backup-db-dump + backup-fs-from-directory '/var/backups/' 'gitea/' 'git/repositories/' +} + +source /usr/local/lib/functions.sh +source /usr/local/lib/pg-functions.sh +source /usr/local/lib/file-functions.sh + +main diff --git a/infrastructure/docker-backup/image/resources/entrypoint-start-and-wait.sh b/infrastructure/docker-backup/image/resources/entrypoint-start-and-wait.sh new file mode 100644 index 0000000..0915071 --- /dev/null +++ b/infrastructure/docker-backup/image/resources/entrypoint-start-and-wait.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +function main() { + + while true; do + sleep 1m + done +} + +source /usr/local/lib/functions.sh +main \ No newline at end of file diff --git a/infrastructure/docker-backup/image/resources/entrypoint.sh b/infrastructure/docker-backup/image/resources/entrypoint.sh new file mode 100755 index 0000000..b25e15f --- /dev/null +++ b/infrastructure/docker-backup/image/resources/entrypoint.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +function main() { + + /usr/local/bin/backup.sh +} + +source /usr/local/lib/functions.sh +main diff --git a/infrastructure/docker-backup/image/resources/init.sh b/infrastructure/docker-backup/image/resources/init.sh new file mode 100755 index 0000000..678f57c --- /dev/null +++ b/infrastructure/docker-backup/image/resources/init.sh @@ -0,0 +1,12 @@ +#!/bin/bash + +function main() { + file_env AWS_ACCESS_KEY_ID + file_env AWS_SECRET_ACCESS_KEY + + init-file-repo +} + +source /usr/local/lib/functions.sh +source /usr/local/lib/file-functions.sh +main diff --git a/infrastructure/docker-backup/image/resources/install.sh b/infrastructure/docker-backup/image/resources/install.sh new file mode 100755 index 0000000..1a8cbd7 --- /dev/null +++ b/infrastructure/docker-backup/image/resources/install.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +apt-get update > /dev/null; + +install -m 0700 /tmp/entrypoint.sh / +install -m 0700 /tmp/entrypoint-start-and-wait.sh / + +install -m 0700 /tmp/init.sh /usr/local/bin/ +install -m 0700 /tmp/backup.sh /usr/local/bin/ +install -m 0700 /tmp/restore.sh /usr/local/bin/ +install -m 0700 /tmp/restic-snapshots.sh /usr/local/bin/ diff --git a/infrastructure/docker-backup/image/resources/restic-snapshots.sh b/infrastructure/docker-backup/image/resources/restic-snapshots.sh new file mode 100755 index 0000000..ca889ce --- /dev/null +++ b/infrastructure/docker-backup/image/resources/restic-snapshots.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +set -o pipefail + +function main() { + file_env AWS_ACCESS_KEY_ID + file_env AWS_SECRET_ACCESS_KEY + + restic -r ${RESTIC_REPOSITORY}/files snapshots +} + +source /usr/local/lib/functions.sh +source /usr/local/lib/file-functions.sh + +main diff --git a/infrastructure/docker-backup/image/resources/restore.sh b/infrastructure/docker-backup/image/resources/restore.sh new file mode 100755 index 0000000..264ebd6 --- /dev/null +++ b/infrastructure/docker-backup/image/resources/restore.sh @@ -0,0 +1,39 @@ +#!/bin/bash + +set -Eeo pipefail + +function main() { + + file_env AWS_ACCESS_KEY_ID + file_env AWS_SECRET_ACCESS_KEY + + file_env POSTGRES_DB + file_env POSTGRES_PASSWORD + file_env POSTGRES_USER + + # Restore latest snapshot into /var/backups/restore + rm -rf /var/backups/restore + restore-directory '/var/backups/restore' + + rm -rf /data/gitea/* + rm -rf /data/git/repositories/* + cp /var/backups/restore/gitea/* /data/gitea/ + cp /var/backups/restore/git/repositories/* /data/git/repositories/ + + # adjust file permissions + #chown -R git:git /data + + # Regenerate Git Hooks + /usr/local/bin/gitea -c '/data/gitea/conf/app.ini' admin regenerate hooks + + # Restore db + drop-create-db + #restore-roles + restore-db +} + +source /usr/local/lib/functions.sh +source /usr/local/lib/pg-functions.sh +source /usr/local/lib/file-functions.sh + +main diff --git a/infrastructure/docker-backup/test/Dockerfile b/infrastructure/docker-backup/test/Dockerfile new file mode 100644 index 0000000..f2e19b6 --- /dev/null +++ b/infrastructure/docker-backup/test/Dockerfile @@ -0,0 +1,11 @@ +FROM c4k-jira-backup + +RUN apt update +RUN apt -yqq --no-install-recommends --yes install curl default-jre-headless + +RUN curl -L -o /tmp/serverspec.jar \ + https://github.com/DomainDrivenArchitecture/dda-serverspec-crate/releases/download/2.0.0/dda-serverspec-standalone.jar + +COPY serverspec.edn /tmp/serverspec.edn + +RUN java -jar /tmp/serverspec.jar /tmp/serverspec.edn -v \ No newline at end of file diff --git a/infrastructure/docker-backup/test/serverspec.edn b/infrastructure/docker-backup/test/serverspec.edn new file mode 100644 index 0000000..09623c7 --- /dev/null +++ b/infrastructure/docker-backup/test/serverspec.edn @@ -0,0 +1,6 @@ +{:file [{:path "/usr/local/bin/init.sh" :mod "700"} + {:path "/usr/local/bin/backup.sh" :mod "700"} + {:path "/usr/local/bin/restore.sh" :mod "700"} + {:path "/usr/local/bin/restic-snapshots.sh" :mod "700"} + {:path "/entrypoint.sh" :mod "700"} + {:path "/entrypoint-start-and-wait.sh" :mod "700"}]} diff --git a/src/main/resources/backup/backup-restore-deployment.yaml b/src/main/resources/backup/backup-restore-deployment.yaml new file mode 100644 index 0000000..c74145b --- /dev/null +++ b/src/main/resources/backup/backup-restore-deployment.yaml @@ -0,0 +1,50 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: backup-restore +spec: + replicas: 0 + selector: + matchLabels: + app: backup-restore + strategy: + type: Recreate + template: + metadata: + labels: + app: backup-restore + app.kubernetes.io/name: backup-restore + app.kubernetes.io/part-of: gitea + spec: + containers: + - image: domaindrivenarchitecture/c4k-gitea-backup + name: backup-app + imagePullPolicy: IfNotPresent + command: ["/entrypoint-start-and-wait.sh"] + env: + - name: AWS_DEFAULT_REGION + value: eu-central-1 + - name: AWS_ACCESS_KEY_ID_FILE + value: /var/run/secrets/backup-secrets/aws-access-key-id + - name: AWS_SECRET_ACCESS_KEY_FILE + value: /var/run/secrets/backup-secrets/aws-secret-access-key + - name: RESTIC_REPOSITORY + valueFrom: + configMapKeyRef: + name: backup-config + key: restic-repository + - name: RESTIC_PASSWORD_FILE + value: /var/run/secrets/backup-secrets/restic-password + volumeMounts: + - name: gitea-data-volume + mountPath: /var/backups + - name: backup-secret-volume + mountPath: /var/run/secrets/backup-secrets + readOnly: true + volumes: + - name: gitea-data-volume + persistentVolumeClaim: + claimName: gitea-data-pvc + - name: backup-secret-volume + secret: + secretName: backup-secret \ No newline at end of file diff --git a/src/main/resources/backup/config.yaml b/src/main/resources/backup/config.yaml new file mode 100644 index 0000000..2d60d3c --- /dev/null +++ b/src/main/resources/backup/config.yaml @@ -0,0 +1,9 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: backup-config + labels: + app.kubernetes.io/name: backup + app.kubernetes.io/part-of: gitea +data: + restic-repository: restic-repository \ No newline at end of file diff --git a/src/main/resources/backup/cron.yaml b/src/main/resources/backup/cron.yaml new file mode 100644 index 0000000..7ee682e --- /dev/null +++ b/src/main/resources/backup/cron.yaml @@ -0,0 +1,47 @@ +apiVersion: batch/v1beta1 +kind: CronJob +metadata: + name: gitea-backup + labels: + app.kubernetes.part-of: gitea +spec: + schedule: "10 23 * * *" + successfulJobsHistoryLimit: 1 + failedJobsHistoryLimit: 1 + jobTemplate: + spec: + template: + spec: + containers: + - name: backup-app + image: domaindrivenarchitecture/c4k-gitea-backup + imagePullPolicy: IfNotPresent + command: ["/entrypoint.sh"] + env: + - name: AWS_DEFAULT_REGION + value: eu-central-1 + - name: AWS_ACCESS_KEY_ID_FILE + value: /var/run/secrets/backup-secrets/aws-access-key-id + - name: AWS_SECRET_ACCESS_KEY_FILE + value: /var/run/secrets/backup-secrets/aws-secret-access-key + - name: RESTIC_REPOSITORY + valueFrom: + configMapKeyRef: + name: backup-config + key: restic-repository + - name: RESTIC_PASSWORD_FILE + value: /var/run/secrets/backup-secrets/restic-password + volumeMounts: + - name: gitea-data-volume + mountPath: /var/backups + - name: backup-secret-volume + mountPath: /var/run/secrets/backup-secrets + readOnly: true + volumes: + - name: gitea-data-volume + persistentVolumeClaim: + claimName: gitea-data-pvc + - name: backup-secret-volume + secret: + secretName: backup-secret + restartPolicy: OnFailure \ No newline at end of file diff --git a/src/main/resources/backup/secret.yaml b/src/main/resources/backup/secret.yaml new file mode 100644 index 0000000..c5809e0 --- /dev/null +++ b/src/main/resources/backup/secret.yaml @@ -0,0 +1,9 @@ +apiVersion: v1 +kind: Secret +metadata: + name: backup-secret +type: Opaque +data: + aws-access-key-id: aws-access-key-id + aws-secret-access-key: aws-secret-access-key + restic-password: restic-password \ No newline at end of file From bbf0e17004e5a47482df433040db5eda8fd2c3a8 Mon Sep 17 00:00:00 2001 From: Clemens Geibel Date: Fri, 19 Aug 2022 12:56:52 +0200 Subject: [PATCH 02/14] Added backup ns and moved spec to core.cljc --- src/main/clj/dda/c4k_gitea/uberjar.clj | 2 +- src/main/cljc/dda/c4k_gitea/backup.cljc | 46 +++++++++++++++++++++++++ src/main/cljc/dda/c4k_gitea/core.cljc | 32 ++++++++++++++--- src/main/cljc/dda/c4k_gitea/gitea.cljc | 43 ++++++++--------------- 4 files changed, 89 insertions(+), 34 deletions(-) create mode 100644 src/main/cljc/dda/c4k_gitea/backup.cljc diff --git a/src/main/clj/dda/c4k_gitea/uberjar.clj b/src/main/clj/dda/c4k_gitea/uberjar.clj index f23b20a..28e15a9 100644 --- a/src/main/clj/dda/c4k_gitea/uberjar.clj +++ b/src/main/clj/dda/c4k_gitea/uberjar.clj @@ -6,4 +6,4 @@ [dda.c4k-common.uberjar :as uberjar])) (defn -main [& cmd-args] - (uberjar/main-common "c4k-gitea" gitea/config? gitea/auth? gitea/config-defaults core/k8s-objects cmd-args)) + (uberjar/main-common "c4k-gitea" core/config? core/auth? core/config-defaults core/k8s-objects cmd-args)) diff --git a/src/main/cljc/dda/c4k_gitea/backup.cljc b/src/main/cljc/dda/c4k_gitea/backup.cljc new file mode 100644 index 0000000..b5b32f1 --- /dev/null +++ b/src/main/cljc/dda/c4k_gitea/backup.cljc @@ -0,0 +1,46 @@ +(ns dda.c4k-gitea.backup + (:require + [clojure.spec.alpha :as s] + #?(:cljs [shadow.resource :as rc]) + [dda.c4k-common.yaml :as yaml] + [dda.c4k-common.base64 :as b64] + [dda.c4k-common.common :as cm])) + +(s/def ::aws-access-key-id cm/bash-env-string?) +(s/def ::aws-secret-access-key cm/bash-env-string?) +(s/def ::restic-password cm/bash-env-string?) +(s/def ::restic-repository cm/bash-env-string?) + +(def auth? (s/keys :req-un [::aws-access-key-id ::aws-secret-access-key ::restic-password ::restic-repository])) + +#?(:cljs + (defmethod yaml/load-resource :backup [resource-name] + (case resource-name + "backup/config.yaml" (rc/inline "backup/config.yaml") + "backup/cron.yaml" (rc/inline "backup/cron.yaml") + "backup/secret.yaml" (rc/inline "backup/secret.yaml") + "backup/backup-restore-deployment.yaml" (rc/inline "backup/backup-restore-deployment.yaml") + (throw (js/Error. "Undefined Resource!"))))) + +(defn generate-config [my-conf] + (let [{:keys [restic-repository]} my-conf] + (-> + (yaml/from-string (yaml/load-resource "backup/config.yaml")) + (cm/replace-key-value :restic-repository restic-repository)))) + +(defn generate-cron [] + (yaml/from-string (yaml/load-resource "backup/cron.yaml"))) + +(defn generate-backup-restore-deployment [my-conf] + (let [backup-restore-yaml (yaml/from-string (yaml/load-resource "backup/backup-restore-deployment.yaml"))] + (if (and (contains? my-conf :local-integration-test) (= true (:local-integration-test my-conf))) + (cm/replace-named-value backup-restore-yaml "CERTIFICATE_FILE" "/var/run/secrets/localstack-secrets/ca.crt") + backup-restore-yaml))) + +(defn generate-secret [my-auth] + (let [{:keys [aws-access-key-id aws-secret-access-key restic-password]} my-auth] + (-> + (yaml/from-string (yaml/load-resource "backup/secret.yaml")) + (cm/replace-key-value :aws-access-key-id (b64/encode aws-access-key-id)) + (cm/replace-key-value :aws-secret-access-key (b64/encode aws-secret-access-key)) + (cm/replace-key-value :restic-password (b64/encode restic-password))))) diff --git a/src/main/cljc/dda/c4k_gitea/core.cljc b/src/main/cljc/dda/c4k_gitea/core.cljc index 66d0022..5692dd2 100644 --- a/src/main/cljc/dda/c4k_gitea/core.cljc +++ b/src/main/cljc/dda/c4k_gitea/core.cljc @@ -3,13 +3,32 @@ [dda.c4k-common.yaml :as yaml] [dda.c4k-common.common :as cm] [dda.c4k-gitea.gitea :as gitea] + [dda.c4k-gitea.gitea :as backup] [dda.c4k-common.postgres :as postgres])) +(def config-defaults {:issuer "staging"}) + +(def config? (s/keys :req-un [::gitea/fqdn + ::gitea/mailer-from + ::gitea/mailer-host-port + ::gitea/service-noreply-address] + :opt-un [::gitea/issuer + ::gitea/default-app-name + ::gitea/service-domain-whitelist + ::backup/restic-repository])) + +(def auth? (s/keys :req-un [::postgres/postgres-db-user ::postgres/postgres-db-password + ::gitea/mailer-user ::gitea/mailer-pw + ::backup/aws-access-key-id ::backup/aws-secret-access-key + ::backup/restic-password])) + +(def vol? (s/keys :req-un [::gitea/volume-total-storage-size])) + (defn k8s-objects [config] (let [storage-class (if (contains? config :postgres-data-volume-path) :manual :local-path)] - (cm/concat-vec - (map yaml/to-string - (filter #(not (nil? %)) + (map yaml/to-string + (filter #(not (nil? %)) + (cm/concat-vec [(postgres/generate-config {:postgres-size :2gb :db-name "gitea"}) (postgres/generate-secret config) (when (contains? config :postgres-data-volume-path) @@ -27,4 +46,9 @@ (gitea/generate-appini-env config) (gitea/generate-secrets config) (gitea/generate-ingress config) - (gitea/generate-certificate config)]))))) + (gitea/generate-certificate config)] + (when (contains? config :restic-repository) + [(backup/generate-config config) + (backup/generate-secret config) + (backup/generate-cron) + (backup/generate-backup-restore-deployment config)])))))) diff --git a/src/main/cljc/dda/c4k_gitea/gitea.cljc b/src/main/cljc/dda/c4k_gitea/gitea.cljc index 8e55138..2cf0d62 100644 --- a/src/main/cljc/dda/c4k_gitea/gitea.cljc +++ b/src/main/cljc/dda/c4k_gitea/gitea.cljc @@ -10,8 +10,7 @@ [dda.c4k-common.yaml :as yaml] [dda.c4k-common.common :as cm] [dda.c4k-common.base64 :as b64] - [dda.c4k-common.predicate :as pred] - [dda.c4k-common.postgres :as postgres])) + [dda.c4k-common.predicate :as pred])) (defn domain-list? [input] @@ -30,20 +29,6 @@ (s/def ::issuer pred/letsencrypt-issuer?) (s/def ::volume-total-storage-size (partial pred/int-gt-n? 5)) -(def config-defaults {:issuer "staging"}) - -(def config? (s/keys :req-un [::fqdn - ::mailer-from - ::mailer-host-port - ::service-noreply-address] - :opt-un [::issuer - ::default-app-name - ::service-domain-whitelist])) - -(def auth? (s/keys :req-un [::postgres/postgres-db-user ::postgres/postgres-db-password ::mailer-user ::mailer-pw])) - -(def vol? (s/keys :req-un [::volume-total-storage-size])) - (defn-spec root-storage-by-volume-size int? [volume-total-storage-size ::volume-total-storage-size] (cond @@ -74,8 +59,8 @@ (defmethod yaml/load-as-edn :gitea [resource-name] (yaml/from-string (yaml/load-resource resource-name)))) -(defn-spec generate-appini-env pred/map-or-seq? - [config config?] +(defn generate-appini-env + [config] (let [{:keys [default-app-name fqdn mailer-from @@ -95,8 +80,8 @@ (cm/replace-all-matching-values-by-new-value "WHITELISTDOMAINS" service-domain-whitelist) (cm/replace-all-matching-values-by-new-value "NOREPLY" service-noreply-address)))) -(defn-spec generate-secrets pred/map-or-seq? - [auth auth?] +(defn generate-secrets + [auth] (let [{:keys [postgres-db-user postgres-db-password mailer-user @@ -115,8 +100,8 @@ (yaml/load-as-edn "gitea/ingress.yaml") (cm/replace-all-matching-values-by-new-value "FQDN" fqdn)))) -(defn-spec generate-certificate pred/map-or-seq? - [config config?] +(defn generate-certificate + [config] (let [{:keys [fqdn issuer] :or {issuer "staging"}} config letsencrypt-issuer (name issuer)] @@ -125,16 +110,16 @@ (assoc-in [:spec :issuerRef :name] letsencrypt-issuer) (cm/replace-all-matching-values-by-new-value "FQDN" fqdn)))) -(defn-spec generate-root-volume pred/map-or-seq? - [config vol?] +(defn generate-root-volume + [config] (let [{:keys [volume-total-storage-size]} config root-storage-size (root-storage-by-volume-size volume-total-storage-size)] (-> (yaml/load-as-edn "gitea/rootvolume.yaml") (cm/replace-all-matching-values-by-new-value "ROOTSTORAGESIZE" (str (str root-storage-size) "Gi"))))) -(defn-spec generate-data-volume pred/map-or-seq? - [config vol?] +(defn generate-data-volume + [config] (let [{:keys [volume-total-storage-size]} config root-storage-size (root-storage-by-volume-size volume-total-storage-size) data-storage-size (data-storage-by-volume-size volume-total-storage-size root-storage-size)] @@ -142,14 +127,14 @@ (yaml/load-as-edn "gitea/datavolume.yaml") (cm/replace-all-matching-values-by-new-value "DATASTORAGESIZE" (str (str data-storage-size) "Gi"))))) -(defn-spec generate-deployment pred/map-or-seq? +(defn generate-deployment [] (yaml/load-as-edn "gitea/deployment.yaml")) -(defn-spec generate-service pred/map-or-seq? +(defn generate-service [] (yaml/load-as-edn "gitea/service.yaml")) -(defn-spec generate-service-ssh pred/map-or-seq? +(defn generate-service-ssh [] (yaml/load-as-edn "gitea/service-ssh.yaml")) From e7a2b0e577319be0c4949e6d1a9a5666fea8efc9 Mon Sep 17 00:00:00 2001 From: Clemens Geibel Date: Fri, 19 Aug 2022 13:02:07 +0200 Subject: [PATCH 03/14] Minor adjustments --- src/main/cljc/dda/c4k_gitea/backup.cljc | 2 -- src/main/cljc/dda/c4k_gitea/gitea.cljc | 4 ++-- src/main/cljs/dda/c4k_gitea/browser.cljs | 2 +- 3 files changed, 3 insertions(+), 5 deletions(-) diff --git a/src/main/cljc/dda/c4k_gitea/backup.cljc b/src/main/cljc/dda/c4k_gitea/backup.cljc index b5b32f1..fb44fa0 100644 --- a/src/main/cljc/dda/c4k_gitea/backup.cljc +++ b/src/main/cljc/dda/c4k_gitea/backup.cljc @@ -11,8 +11,6 @@ (s/def ::restic-password cm/bash-env-string?) (s/def ::restic-repository cm/bash-env-string?) -(def auth? (s/keys :req-un [::aws-access-key-id ::aws-secret-access-key ::restic-password ::restic-repository])) - #?(:cljs (defmethod yaml/load-resource :backup [resource-name] (case resource-name diff --git a/src/main/cljc/dda/c4k_gitea/gitea.cljc b/src/main/cljc/dda/c4k_gitea/gitea.cljc index 2cf0d62..8b6e8b6 100644 --- a/src/main/cljc/dda/c4k_gitea/gitea.cljc +++ b/src/main/cljc/dda/c4k_gitea/gitea.cljc @@ -93,8 +93,8 @@ (cm/replace-all-matching-values-by-new-value "MAILERUSER" (b64/encode mailer-user)) (cm/replace-all-matching-values-by-new-value "MAILERPW" (b64/encode mailer-pw))))) -(defn-spec generate-ingress pred/map-or-seq? - [config config?] +(defn generate-ingress + [config] (let [{:keys [fqdn issuer]} config] (-> (yaml/load-as-edn "gitea/ingress.yaml") diff --git a/src/main/cljs/dda/c4k_gitea/browser.cljs b/src/main/cljs/dda/c4k_gitea/browser.cljs index 810f80c..ec41aa4 100644 --- a/src/main/cljs/dda/c4k_gitea/browser.cljs +++ b/src/main/cljs/dda/c4k_gitea/browser.cljs @@ -93,7 +93,7 @@ (br/validate! "domain-whitelist" ::gitea/service-domain-whitelist :optional true) (br/validate! "postgres-data-volume-path" ::pgc/postgres-data-volume-path :optional true) (br/validate! "volume-total-storage-size" ::gitea/volume-total-storage-size :deserializer js/parseInt) - (br/validate! "auth" gitea/auth? :deserializer edn/read-string) + (br/validate! "auth" core/auth? :deserializer edn/read-string) (br/set-form-validated!)) (defn add-validate-listener [name] From 9e4eaa7dbe5142fc6e6c4e48f52dbd2ae9a9c50d Mon Sep 17 00:00:00 2001 From: Clemens Geibel Date: Fri, 19 Aug 2022 13:33:06 +0200 Subject: [PATCH 04/14] Fix ns error --- src/main/cljc/dda/c4k_gitea/core.cljc | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/main/cljc/dda/c4k_gitea/core.cljc b/src/main/cljc/dda/c4k_gitea/core.cljc index 5692dd2..5befcdd 100644 --- a/src/main/cljc/dda/c4k_gitea/core.cljc +++ b/src/main/cljc/dda/c4k_gitea/core.cljc @@ -1,9 +1,10 @@ (ns dda.c4k-gitea.core (:require + [clojure.spec.alpha :as s] [dda.c4k-common.yaml :as yaml] [dda.c4k-common.common :as cm] [dda.c4k-gitea.gitea :as gitea] - [dda.c4k-gitea.gitea :as backup] + [dda.c4k-gitea.backup :as backup] [dda.c4k-common.postgres :as postgres])) (def config-defaults {:issuer "staging"}) @@ -19,8 +20,8 @@ (def auth? (s/keys :req-un [::postgres/postgres-db-user ::postgres/postgres-db-password ::gitea/mailer-user ::gitea/mailer-pw - ::backup/aws-access-key-id ::backup/aws-secret-access-key - ::backup/restic-password])) + ::backup/aws-access-key-id ::backup/aws-secret-access-key] + :opt-un [::backup/restic-password])) ; TODO gec: Is restic password opt or req? (def vol? (s/keys :req-un [::gitea/volume-total-storage-size])) From aae7f82ad6e3100c905588dda1004025aaa8fe7a Mon Sep 17 00:00:00 2001 From: Clemens Geibel Date: Fri, 19 Aug 2022 13:39:59 +0200 Subject: [PATCH 05/14] Fixed test-schema ci-script --- valid-auth.edn | 5 ++++- valid-config.edn | 19 +++++++++---------- 2 files changed, 13 insertions(+), 11 deletions(-) diff --git a/valid-auth.edn b/valid-auth.edn index a2bf50e..f484e2a 100644 --- a/valid-auth.edn +++ b/valid-auth.edn @@ -1,4 +1,7 @@ {:postgres-db-user "gitea" :postgres-db-password "gitea-db-password" :mailer-user "" - :mailer-pw ""} + :mailer-pw "" + :aws-access-key-id "AWS_KEY_ID" + :aws-secret-access-key "AWS_KEY_SECRET" + :restic-password ""} diff --git a/valid-config.edn b/valid-config.edn index cf37f4d..056f3de 100644 --- a/valid-config.edn +++ b/valid-config.edn @@ -1,10 +1,9 @@ -{ -:default-app-name "Meissas awesome gitea" -:fqdn "test.de" -:issuer "staging" -:mailer-from "test@test.de" -:mailer-host-port "test.de:123" -:service-whitelist-domains "test.de" -:service-noreply-address "noreply@test.de" -:volume-total-storage-size 6 - } +{:default-app-name "Meissas awesome gitea" + :fqdn "test.de" + :issuer "staging" + :mailer-from "test@test.de" + :mailer-host-port "test.de:123" + :service-whitelist-domains "test.de" + :service-noreply-address "noreply@test.de" + :volume-total-storage-size 6 + :restic-repository "repo-path"} From 2430e7483adf86cd3261d2d5959f816a8ab23191 Mon Sep 17 00:00:00 2001 From: Clemens Geibel Date: Fri, 26 Aug 2022 15:06:39 +0200 Subject: [PATCH 06/14] Added init-role and init-database to init.sh --- infrastructure/docker-backup/image/resources/init.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/infrastructure/docker-backup/image/resources/init.sh b/infrastructure/docker-backup/image/resources/init.sh index 678f57c..322b35d 100755 --- a/infrastructure/docker-backup/image/resources/init.sh +++ b/infrastructure/docker-backup/image/resources/init.sh @@ -4,6 +4,8 @@ function main() { file_env AWS_ACCESS_KEY_ID file_env AWS_SECRET_ACCESS_KEY + init-role-repo + init-database-repo init-file-repo } From 8c67f86aa79b6f9c12e07461f1f6086cabff79d7 Mon Sep 17 00:00:00 2001 From: Clemens Geibel Date: Fri, 26 Aug 2022 15:57:16 +0200 Subject: [PATCH 07/14] enable file permission adjustments for restore --- infrastructure/docker-backup/image/resources/restore.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infrastructure/docker-backup/image/resources/restore.sh b/infrastructure/docker-backup/image/resources/restore.sh index 264ebd6..b56b97d 100755 --- a/infrastructure/docker-backup/image/resources/restore.sh +++ b/infrastructure/docker-backup/image/resources/restore.sh @@ -21,7 +21,7 @@ function main() { cp /var/backups/restore/git/repositories/* /data/git/repositories/ # adjust file permissions - #chown -R git:git /data + chown -R git:git /data # Regenerate Git Hooks /usr/local/bin/gitea -c '/data/gitea/conf/app.ini' admin regenerate hooks From 965d2b463919278ff135894ae835713e46c313ab Mon Sep 17 00:00:00 2001 From: erik Date: Fri, 2 Sep 2022 09:51:20 +0200 Subject: [PATCH 08/14] Update CI to show failing tests --- .gitlab-ci.yml | 1 + shadow-cljs.edn | 3 +-- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 50fe61e..d17556d 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -35,6 +35,7 @@ test-cljs: stage: build_and_test script: - shadow-cljs compile test + - node target/node-tests.js test-clj: <<: *clj diff --git a/shadow-cljs.edn b/shadow-cljs.edn index be071ef..bce1936 100644 --- a/shadow-cljs.edn +++ b/shadow-cljs.edn @@ -11,6 +11,5 @@ :release {} :compiler-options {:optimizations :advanced}} :test {:target :node-test - :output-to "target/node-tests.js" - :autorun true + :output-to "target/node-tests.js" :repl-pprint true}}} \ No newline at end of file From 4a00286b20eb54ceafc2406a7c0c055eea05c17f Mon Sep 17 00:00:00 2001 From: Clemens Geibel Date: Fri, 16 Sep 2022 11:57:24 +0200 Subject: [PATCH 09/14] Added backup doc and fixed init.sh --- doc/BackupAndRestore.md | 37 +++++++++++++++++++ .../docker-backup/image/resources/init.sh | 1 + 2 files changed, 38 insertions(+) create mode 100644 doc/BackupAndRestore.md diff --git a/doc/BackupAndRestore.md b/doc/BackupAndRestore.md new file mode 100644 index 0000000..68222bb --- /dev/null +++ b/doc/BackupAndRestore.md @@ -0,0 +1,37 @@ +# Backup Architecture details + +![](backup.svg) + +* we use restic to produce small & encrypted backups +* backup is scheduled at `schedule: "10 23 * * *"` +* Gitea stores files in `/data/gitea` and `/data/git/repositories`, these files are backed up. +* The postgres db is also backed up + +## Manual init the restic repository for the first time + +1. apply backup-and-restore pod: + `kubectl scale deployment backup-restore --replicas=1` +2. exec into pod and execute restore pod (press tab to get your exact pod name) + `kubectl exec -it backup-restore-... -- /usr/local/bin/init.sh` +3. remove backup-and-restore pod: + `kubectl scale deployment backup-restore --replicas=0` + + +## Manual backup the restic repository for the first time + +1. apply backup-and-restore pod: + `kubectl scale deployment backup-restore --replicas=1` +2. exec into pod and execute restore pod (press tab to get your exact pod name) + `kubectl exec -it backup-restore-... -- /usr/local/bin/backup.sh` +3. remove backup-and-restore pod: + `kubectl scale deployment backup-restore --replicas=0` + + +## Manual restore + +1. apply backup-and-restore pod: + `kubectl scale deployment backup-restore --replicas=1` +2. exec into pod and execute restore pod (press tab to get your exact pod name) + `kubectl exec -it backup-restore-... -- /usr/local/bin/restore.sh` +1. remove backup-and-restore pod: + `kubectl scale deployment backup-restore --replicas=0` diff --git a/infrastructure/docker-backup/image/resources/init.sh b/infrastructure/docker-backup/image/resources/init.sh index 322b35d..5767e69 100755 --- a/infrastructure/docker-backup/image/resources/init.sh +++ b/infrastructure/docker-backup/image/resources/init.sh @@ -10,5 +10,6 @@ function main() { } source /usr/local/lib/functions.sh +source /usr/local/lib/pg-functions.sh source /usr/local/lib/file-functions.sh main From f8b4137c60010f554854b74b03ad073121264e5d Mon Sep 17 00:00:00 2001 From: Clemens Geibel Date: Fri, 16 Sep 2022 12:26:18 +0200 Subject: [PATCH 10/14] Added more necessary env variables to backup-restore-deployment --- .../backup/backup-restore-deployment.yaml | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/src/main/resources/backup/backup-restore-deployment.yaml b/src/main/resources/backup/backup-restore-deployment.yaml index c74145b..79f4ec7 100644 --- a/src/main/resources/backup/backup-restore-deployment.yaml +++ b/src/main/resources/backup/backup-restore-deployment.yaml @@ -22,6 +22,27 @@ spec: imagePullPolicy: IfNotPresent command: ["/entrypoint-start-and-wait.sh"] env: + - name: POSTGRES_USER + valueFrom: + secretKeyRef: + name: postgres-secret + key: postgres-user + - name: POSTGRES_PASSWORD + valueFrom: + secretKeyRef: + name: postgres-secret + key: postgres-password + - name: POSTGRES_DB + valueFrom: + configMapKeyRef: + name: postgres-config + key: postgres-db + - name: POSTGRES_HOST + value: "postgresql-service:5432" + - name: POSTGRES_SERVICE + value: "postgresql-service" + - name: POSTGRES_PORT + value: "5432" - name: AWS_DEFAULT_REGION value: eu-central-1 - name: AWS_ACCESS_KEY_ID_FILE @@ -35,6 +56,8 @@ spec: key: restic-repository - name: RESTIC_PASSWORD_FILE value: /var/run/secrets/backup-secrets/restic-password + - name: CERTIFICATE_FILE + value: "" volumeMounts: - name: gitea-data-volume mountPath: /var/backups From 372f5c7a40e09d0c5de3ae0039b8196a5d2c9b26 Mon Sep 17 00:00:00 2001 From: Clemens Geibel Date: Fri, 16 Sep 2022 15:23:25 +0200 Subject: [PATCH 11/14] More fixed for backup image --- doc/BackupAndRestore.md | 8 ++++++-- .../image/resources/entrypoint-start-and-wait.sh | 2 ++ .../docker-backup/image/resources/entrypoint.sh | 4 +++- .../docker-backup/image/resources/init.sh | 2 +- .../image/resources/restic-snapshots.sh | 1 + .../docker-backup/image/resources/restore.sh | 16 ++++++++-------- 6 files changed, 21 insertions(+), 12 deletions(-) diff --git a/doc/BackupAndRestore.md b/doc/BackupAndRestore.md index 68222bb..b15d4a1 100644 --- a/doc/BackupAndRestore.md +++ b/doc/BackupAndRestore.md @@ -31,7 +31,11 @@ 1. apply backup-and-restore pod: `kubectl scale deployment backup-restore --replicas=1` -2. exec into pod and execute restore pod (press tab to get your exact pod name) +2. Scale down gitea deployment: + `kubectl scale deployment gitea --replicas=0` +3. exec into pod and execute restore pod (press tab to get your exact pod name) `kubectl exec -it backup-restore-... -- /usr/local/bin/restore.sh` -1. remove backup-and-restore pod: +4. Start gitea again: + `kubectl scale deployment gitea --replicas=1` +5. remove backup-and-restore pod: `kubectl scale deployment backup-restore --replicas=0` diff --git a/infrastructure/docker-backup/image/resources/entrypoint-start-and-wait.sh b/infrastructure/docker-backup/image/resources/entrypoint-start-and-wait.sh index 0915071..c6addac 100644 --- a/infrastructure/docker-backup/image/resources/entrypoint-start-and-wait.sh +++ b/infrastructure/docker-backup/image/resources/entrypoint-start-and-wait.sh @@ -1,6 +1,7 @@ #!/bin/bash function main() { + create-pg-pass while true; do sleep 1m @@ -8,4 +9,5 @@ function main() { } source /usr/local/lib/functions.sh +source /usr/local/lib/pg-functions.sh main \ No newline at end of file diff --git a/infrastructure/docker-backup/image/resources/entrypoint.sh b/infrastructure/docker-backup/image/resources/entrypoint.sh index b25e15f..96df4f3 100755 --- a/infrastructure/docker-backup/image/resources/entrypoint.sh +++ b/infrastructure/docker-backup/image/resources/entrypoint.sh @@ -1,9 +1,11 @@ #!/bin/bash function main() { - + create-pg-pass + /usr/local/bin/backup.sh } source /usr/local/lib/functions.sh +source /usr/local/lib/pg-functions.sh main diff --git a/infrastructure/docker-backup/image/resources/init.sh b/infrastructure/docker-backup/image/resources/init.sh index 5767e69..baac041 100755 --- a/infrastructure/docker-backup/image/resources/init.sh +++ b/infrastructure/docker-backup/image/resources/init.sh @@ -4,7 +4,7 @@ function main() { file_env AWS_ACCESS_KEY_ID file_env AWS_SECRET_ACCESS_KEY - init-role-repo + #init-role-repo init-database-repo init-file-repo } diff --git a/infrastructure/docker-backup/image/resources/restic-snapshots.sh b/infrastructure/docker-backup/image/resources/restic-snapshots.sh index ca889ce..1d97a2c 100755 --- a/infrastructure/docker-backup/image/resources/restic-snapshots.sh +++ b/infrastructure/docker-backup/image/resources/restic-snapshots.sh @@ -7,6 +7,7 @@ function main() { file_env AWS_SECRET_ACCESS_KEY restic -r ${RESTIC_REPOSITORY}/files snapshots + restic -r ${RESTIC_REPOSITORY}/pg-database snapshots } source /usr/local/lib/functions.sh diff --git a/infrastructure/docker-backup/image/resources/restore.sh b/infrastructure/docker-backup/image/resources/restore.sh index b56b97d..0d9d10d 100755 --- a/infrastructure/docker-backup/image/resources/restore.sh +++ b/infrastructure/docker-backup/image/resources/restore.sh @@ -15,16 +15,16 @@ function main() { rm -rf /var/backups/restore restore-directory '/var/backups/restore' - rm -rf /data/gitea/* - rm -rf /data/git/repositories/* - cp /var/backups/restore/gitea/* /data/gitea/ - cp /var/backups/restore/git/repositories/* /data/git/repositories/ + rm -rf /var/backups/gitea/* + rm -rf /var/backups/git/repositories/* + cp -r /var/backups/restore/gitea/* /var/backups/gitea/ + cp -r /var/backups/restore/git/repositories/* /var/backups/git/repositories/ - # adjust file permissions - chown -R git:git /data + # adjust file permissions for the git user + chown -R 1000:1000 /var/backups - # Regenerate Git Hooks - /usr/local/bin/gitea -c '/data/gitea/conf/app.ini' admin regenerate hooks + # TODO: Regenerate Git Hooks + #/usr/local/bin/gitea -c '/data/gitea/conf/app.ini' admin regenerate hooks # Restore db drop-create-db From 4dda50ec694fa4f77e47a3350c7a34da58a849ab Mon Sep 17 00:00:00 2001 From: Clemens Geibel Date: Fri, 16 Sep 2022 15:45:04 +0200 Subject: [PATCH 12/14] Use dda-backup:1.0.6 --- infrastructure/docker-backup/image/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infrastructure/docker-backup/image/Dockerfile b/infrastructure/docker-backup/image/Dockerfile index 9b6d4f0..02f5ca9 100644 --- a/infrastructure/docker-backup/image/Dockerfile +++ b/infrastructure/docker-backup/image/Dockerfile @@ -1,4 +1,4 @@ -FROM domaindrivenarchitecture/dda-backup:1.0.5 +FROM domaindrivenarchitecture/dda-backup:1.0.6 # Prepare Entrypoint Script ADD resources /tmp From 5ff22f425aa2ac520a0e75fba10a96304650a762 Mon Sep 17 00:00:00 2001 From: Clemens Geibel Date: Fri, 16 Sep 2022 16:44:14 +0200 Subject: [PATCH 13/14] Remove unused pg-role functions --- infrastructure/docker-backup/image/resources/backup.sh | 1 - infrastructure/docker-backup/image/resources/init.sh | 1 - infrastructure/docker-backup/image/resources/restore.sh | 3 +-- 3 files changed, 1 insertion(+), 4 deletions(-) diff --git a/infrastructure/docker-backup/image/resources/backup.sh b/infrastructure/docker-backup/image/resources/backup.sh index 9bdbde1..d2549fd 100755 --- a/infrastructure/docker-backup/image/resources/backup.sh +++ b/infrastructure/docker-backup/image/resources/backup.sh @@ -8,7 +8,6 @@ function main() { file_env RESTIC_DAYS_TO_KEEP 30 file_env RESTIC_MONTHS_TO_KEEP 12 - #backup-roles 'TODO' backup-db-dump backup-fs-from-directory '/var/backups/' 'gitea/' 'git/repositories/' } diff --git a/infrastructure/docker-backup/image/resources/init.sh b/infrastructure/docker-backup/image/resources/init.sh index baac041..1f47fa5 100755 --- a/infrastructure/docker-backup/image/resources/init.sh +++ b/infrastructure/docker-backup/image/resources/init.sh @@ -4,7 +4,6 @@ function main() { file_env AWS_ACCESS_KEY_ID file_env AWS_SECRET_ACCESS_KEY - #init-role-repo init-database-repo init-file-repo } diff --git a/infrastructure/docker-backup/image/resources/restore.sh b/infrastructure/docker-backup/image/resources/restore.sh index 0d9d10d..2289ab2 100755 --- a/infrastructure/docker-backup/image/resources/restore.sh +++ b/infrastructure/docker-backup/image/resources/restore.sh @@ -23,12 +23,11 @@ function main() { # adjust file permissions for the git user chown -R 1000:1000 /var/backups - # TODO: Regenerate Git Hooks + # TODO: Regenerate Git Hooks? Do we need this? #/usr/local/bin/gitea -c '/data/gitea/conf/app.ini' admin regenerate hooks # Restore db drop-create-db - #restore-roles restore-db } From fe2a2282f435af2639f4bc86fddacd0654205ba1 Mon Sep 17 00:00:00 2001 From: erik Date: Tue, 4 Oct 2022 13:03:30 +0200 Subject: [PATCH 14/14] Add require postgres to gitea --- src/main/cljc/dda/c4k_gitea/gitea.cljc | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/main/cljc/dda/c4k_gitea/gitea.cljc b/src/main/cljc/dda/c4k_gitea/gitea.cljc index 1a75db7..1e9dbbf 100644 --- a/src/main/cljc/dda/c4k_gitea/gitea.cljc +++ b/src/main/cljc/dda/c4k_gitea/gitea.cljc @@ -10,7 +10,8 @@ [dda.c4k-common.yaml :as yaml] [dda.c4k-common.common :as cm] [dda.c4k-common.base64 :as b64] - [dda.c4k-common.predicate :as pred])) + [dda.c4k-common.predicate :as pred] + [dda.c4k-common.postgres :as postgres])) (defn domain-list? [input] @@ -101,7 +102,7 @@ (defn generate-ingress [config] - (let [{:keys [fqdn issuer]} config] + (let [{:keys [fqdn]} config] (-> (yaml/load-as-edn "gitea/ingress.yaml") (cm/replace-all-matching-values-by-new-value "FQDN" fqdn))))