diff --git a/README.md b/README.md index 8f247c8..a19768f 100644 --- a/README.md +++ b/README.md @@ -31,6 +31,13 @@ You need: Apply this file on your cluster with `kubectl apply -f yourApp.yaml`. Done. +## Backup + +- Taiga DB: https://docs.taiga.io/backup-and-restore.html +- Taiga Media: ? +- Taiga Protected: ? +- Taiga Static: ? + ## Development & mirrors Development happens at: https://repo.prod.meissa.de/meissa/c4k-taiga diff --git a/src/main/resources/taiga/donotapply/.env b/doc/donotapply/.env similarity index 100% rename from src/main/resources/taiga/donotapply/.env rename to doc/donotapply/.env diff --git a/src/main/resources/taiga/donotapply/changes-made.md b/doc/donotapply/changes-made.md similarity index 100% rename from src/main/resources/taiga/donotapply/changes-made.md rename to doc/donotapply/changes-made.md diff --git a/src/main/resources/taiga/donotapply/conf.json b/doc/donotapply/conf.json similarity index 100% rename from src/main/resources/taiga/donotapply/conf.json rename to doc/donotapply/conf.json diff --git a/src/main/resources/taiga/donotapply/config.py b/doc/donotapply/config.py similarity index 100% rename from src/main/resources/taiga/donotapply/config.py rename to doc/donotapply/config.py diff --git a/src/main/resources/taiga/donotapply/old-docker-compose-inits.yml b/doc/donotapply/old-docker-compose-inits.yml similarity index 100% rename from src/main/resources/taiga/donotapply/old-docker-compose-inits.yml rename to doc/donotapply/old-docker-compose-inits.yml diff --git a/src/main/resources/taiga/donotapply/old-docker-compose.yml b/doc/donotapply/old-docker-compose.yml similarity index 100% rename from src/main/resources/taiga/donotapply/old-docker-compose.yml rename to doc/donotapply/old-docker-compose.yml diff --git a/infrastructure/docker-backup/build.py b/infrastructure/docker-backup/build.py new file mode 100644 index 0000000..2247dd4 --- /dev/null +++ b/infrastructure/docker-backup/build.py @@ -0,0 +1,49 @@ +from os import environ +from pybuilder.core import task, init +from ddadevops import * + +name = "c4k-forgejo-backup" +MODULE = "docker" +PROJECT_ROOT_PATH = "../.." + + +@init +def initialize(project): + input = { + "name": name, + "module": MODULE, + "stage": "notused", + "project_root_path": PROJECT_ROOT_PATH, + "build_types": ["IMAGE"], + "mixin_types": [], + } + + project.build_depends_on("ddadevops>=4.0.0-dev") + + build = DevopsImageBuild(project, input) + build.initialize_build_dir() + + +@task +def image(project): + build = get_devops_build(project) + build.image() + + +@task +def drun(project): + build = get_devops_build(project) + build.drun() + + +@task +def publish(project): + build = get_devops_build(project) + build.dockerhub_login() + build.dockerhub_publish() + + +@task +def test(project): + build = get_devops_build(project) + build.test() diff --git a/infrastructure/docker-backup/image/Dockerfile b/infrastructure/docker-backup/image/Dockerfile new file mode 100644 index 0000000..02f5ca9 --- /dev/null +++ b/infrastructure/docker-backup/image/Dockerfile @@ -0,0 +1,5 @@ +FROM domaindrivenarchitecture/dda-backup:1.0.6 + +# Prepare Entrypoint Script +ADD resources /tmp +RUN /tmp/install.sh diff --git a/infrastructure/docker-backup/image/resources/backup.sh b/infrastructure/docker-backup/image/resources/backup.sh new file mode 100755 index 0000000..3de1fce --- /dev/null +++ b/infrastructure/docker-backup/image/resources/backup.sh @@ -0,0 +1,18 @@ +#!/bin/bash + +set -o pipefail + +function main() { + file_env AWS_ACCESS_KEY_ID + file_env AWS_SECRET_ACCESS_KEY + file_env RESTIC_DAYS_TO_KEEP 30 + file_env RESTIC_MONTHS_TO_KEEP 12 + + backup-db-dump +} + +source /usr/local/lib/functions.sh +source /usr/local/lib/pg-functions.sh +source /usr/local/lib/file-functions.sh + +main diff --git a/infrastructure/docker-backup/image/resources/entrypoint-start-and-wait.sh b/infrastructure/docker-backup/image/resources/entrypoint-start-and-wait.sh new file mode 100644 index 0000000..c6addac --- /dev/null +++ b/infrastructure/docker-backup/image/resources/entrypoint-start-and-wait.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +function main() { + create-pg-pass + + while true; do + sleep 1m + done +} + +source /usr/local/lib/functions.sh +source /usr/local/lib/pg-functions.sh +main \ No newline at end of file diff --git a/infrastructure/docker-backup/image/resources/entrypoint.sh b/infrastructure/docker-backup/image/resources/entrypoint.sh new file mode 100755 index 0000000..96df4f3 --- /dev/null +++ b/infrastructure/docker-backup/image/resources/entrypoint.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +function main() { + create-pg-pass + + /usr/local/bin/backup.sh +} + +source /usr/local/lib/functions.sh +source /usr/local/lib/pg-functions.sh +main diff --git a/infrastructure/docker-backup/image/resources/init.sh b/infrastructure/docker-backup/image/resources/init.sh new file mode 100755 index 0000000..1f47fa5 --- /dev/null +++ b/infrastructure/docker-backup/image/resources/init.sh @@ -0,0 +1,14 @@ +#!/bin/bash + +function main() { + file_env AWS_ACCESS_KEY_ID + file_env AWS_SECRET_ACCESS_KEY + + init-database-repo + init-file-repo +} + +source /usr/local/lib/functions.sh +source /usr/local/lib/pg-functions.sh +source /usr/local/lib/file-functions.sh +main diff --git a/infrastructure/docker-backup/image/resources/install.sh b/infrastructure/docker-backup/image/resources/install.sh new file mode 100755 index 0000000..1a8cbd7 --- /dev/null +++ b/infrastructure/docker-backup/image/resources/install.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +apt-get update > /dev/null; + +install -m 0700 /tmp/entrypoint.sh / +install -m 0700 /tmp/entrypoint-start-and-wait.sh / + +install -m 0700 /tmp/init.sh /usr/local/bin/ +install -m 0700 /tmp/backup.sh /usr/local/bin/ +install -m 0700 /tmp/restore.sh /usr/local/bin/ +install -m 0700 /tmp/restic-snapshots.sh /usr/local/bin/ diff --git a/infrastructure/docker-backup/image/resources/restic-snapshots.sh b/infrastructure/docker-backup/image/resources/restic-snapshots.sh new file mode 100755 index 0000000..1d97a2c --- /dev/null +++ b/infrastructure/docker-backup/image/resources/restic-snapshots.sh @@ -0,0 +1,16 @@ +#!/bin/bash + +set -o pipefail + +function main() { + file_env AWS_ACCESS_KEY_ID + file_env AWS_SECRET_ACCESS_KEY + + restic -r ${RESTIC_REPOSITORY}/files snapshots + restic -r ${RESTIC_REPOSITORY}/pg-database snapshots +} + +source /usr/local/lib/functions.sh +source /usr/local/lib/file-functions.sh + +main diff --git a/infrastructure/docker-backup/image/resources/restore.sh b/infrastructure/docker-backup/image/resources/restore.sh new file mode 100755 index 0000000..6ed2fc0 --- /dev/null +++ b/infrastructure/docker-backup/image/resources/restore.sh @@ -0,0 +1,38 @@ +#!/bin/bash + +set -Eeo pipefail + +function main() { + + file_env AWS_ACCESS_KEY_ID + file_env AWS_SECRET_ACCESS_KEY + + file_env POSTGRES_DB + file_env POSTGRES_PASSWORD + file_env POSTGRES_USER + + # Restore latest snapshot into /var/backups/restore + rm -rf /var/backups/restore + restore-directory '/var/backups/restore' + + rm -rf /var/backups/gitea/* + rm -rf /var/backups/git/repositories/* + cp -r /var/backups/restore/gitea /var/backups/ + cp -r /var/backups/restore/git/repositories /var/backups/git/ + + # adjust file permissions for the git user + chown -R 1000:1000 /var/backups + + # TODO: Regenerate Git Hooks? Do we need this? + #/usr/local/bin/gitea -c '/data/gitea/conf/app.ini' admin regenerate hooks + + # Restore db + drop-create-db + restore-db +} + +source /usr/local/lib/functions.sh +source /usr/local/lib/pg-functions.sh +source /usr/local/lib/file-functions.sh + +main diff --git a/infrastructure/docker-backup/test/Dockerfile b/infrastructure/docker-backup/test/Dockerfile new file mode 100644 index 0000000..0db1c6c --- /dev/null +++ b/infrastructure/docker-backup/test/Dockerfile @@ -0,0 +1,11 @@ +FROM c4k-forgejo-backup + +RUN apt update +RUN apt -yqq --no-install-recommends --yes install curl default-jre-headless + +RUN curl -L -o /tmp/serverspec.jar \ + https://github.com/DomainDrivenArchitecture/dda-serverspec-crate/releases/download/2.0.0/dda-serverspec-standalone.jar + +COPY serverspec.edn /tmp/serverspec.edn + +RUN java -jar /tmp/serverspec.jar /tmp/serverspec.edn -v \ No newline at end of file diff --git a/infrastructure/docker-backup/test/serverspec.edn b/infrastructure/docker-backup/test/serverspec.edn new file mode 100644 index 0000000..09623c7 --- /dev/null +++ b/infrastructure/docker-backup/test/serverspec.edn @@ -0,0 +1,6 @@ +{:file [{:path "/usr/local/bin/init.sh" :mod "700"} + {:path "/usr/local/bin/backup.sh" :mod "700"} + {:path "/usr/local/bin/restore.sh" :mod "700"} + {:path "/usr/local/bin/restic-snapshots.sh" :mod "700"} + {:path "/entrypoint.sh" :mod "700"} + {:path "/entrypoint-start-and-wait.sh" :mod "700"}]} diff --git a/infrastructure/c4k-website-build/build.py b/infrastructure/docker-taiga/build.py similarity index 100% rename from infrastructure/c4k-website-build/build.py rename to infrastructure/docker-taiga/build.py diff --git a/infrastructure/c4k-website-build/image/Dockerfile b/infrastructure/docker-taiga/image/Dockerfile similarity index 100% rename from infrastructure/c4k-website-build/image/Dockerfile rename to infrastructure/docker-taiga/image/Dockerfile diff --git a/infrastructure/c4k-website-build/image/resources/entrypoint.sh b/infrastructure/docker-taiga/image/resources/entrypoint.sh similarity index 100% rename from infrastructure/c4k-website-build/image/resources/entrypoint.sh rename to infrastructure/docker-taiga/image/resources/entrypoint.sh diff --git a/infrastructure/c4k-website-build/image/resources/exclude.pattern b/infrastructure/docker-taiga/image/resources/exclude.pattern similarity index 100% rename from infrastructure/c4k-website-build/image/resources/exclude.pattern rename to infrastructure/docker-taiga/image/resources/exclude.pattern diff --git a/infrastructure/c4k-website-build/image/resources/functions.sh b/infrastructure/docker-taiga/image/resources/functions.sh similarity index 100% rename from infrastructure/c4k-website-build/image/resources/functions.sh rename to infrastructure/docker-taiga/image/resources/functions.sh diff --git a/infrastructure/c4k-website-build/image/resources/install.sh b/infrastructure/docker-taiga/image/resources/install.sh similarity index 100% rename from infrastructure/c4k-website-build/image/resources/install.sh rename to infrastructure/docker-taiga/image/resources/install.sh diff --git a/infrastructure/c4k-website-build/image/resources/project.clj b/infrastructure/docker-taiga/image/resources/project.clj similarity index 100% rename from infrastructure/c4k-website-build/image/resources/project.clj rename to infrastructure/docker-taiga/image/resources/project.clj diff --git a/infrastructure/c4k-website-build/test/Dockerfile b/infrastructure/docker-taiga/test/Dockerfile similarity index 100% rename from infrastructure/c4k-website-build/test/Dockerfile rename to infrastructure/docker-taiga/test/Dockerfile diff --git a/infrastructure/c4k-website-build/test/serverspec.edn b/infrastructure/docker-taiga/test/serverspec.edn similarity index 100% rename from infrastructure/c4k-website-build/test/serverspec.edn rename to infrastructure/docker-taiga/test/serverspec.edn diff --git a/src/main/cljc/dda/c4k_taiga/backup.cljc b/src/main/cljc/dda/c4k_taiga/backup.cljc new file mode 100644 index 0000000..eb55172 --- /dev/null +++ b/src/main/cljc/dda/c4k_taiga/backup.cljc @@ -0,0 +1,44 @@ +(ns dda.c4k-forgejo.backup + (:require + [clojure.spec.alpha :as s] + #?(:cljs [shadow.resource :as rc]) + [dda.c4k-common.yaml :as yaml] + [dda.c4k-common.base64 :as b64] + [dda.c4k-common.common :as cm])) + +(s/def ::aws-access-key-id cm/bash-env-string?) +(s/def ::aws-secret-access-key cm/bash-env-string?) +(s/def ::restic-password cm/bash-env-string?) +(s/def ::restic-repository cm/bash-env-string?) + +#?(:cljs + (defmethod yaml/load-resource :backup [resource-name] + (case resource-name + "backup/config.yaml" (rc/inline "backup/config.yaml") + "backup/cron.yaml" (rc/inline "backup/cron.yaml") + "backup/secret.yaml" (rc/inline "backup/secret.yaml") + "backup/backup-restore-deployment.yaml" (rc/inline "backup/backup-restore-deployment.yaml") + (throw (js/Error. "Undefined Resource!"))))) + +(defn generate-config [my-conf] + (let [{:keys [restic-repository]} my-conf] + (-> + (yaml/from-string (yaml/load-resource "backup/config.yaml")) + (cm/replace-key-value :restic-repository restic-repository)))) + +(defn generate-cron [] + (yaml/from-string (yaml/load-resource "backup/cron.yaml"))) + +(defn generate-backup-restore-deployment [my-conf] + (let [backup-restore-yaml (yaml/from-string (yaml/load-resource "backup/backup-restore-deployment.yaml"))] + (if (and (contains? my-conf :local-integration-test) (= true (:local-integration-test my-conf))) + (cm/replace-named-value backup-restore-yaml "CERTIFICATE_FILE" "/var/run/secrets/localstack-secrets/ca.crt") + backup-restore-yaml))) + +(defn generate-secret [my-auth] + (let [{:keys [aws-access-key-id aws-secret-access-key restic-password]} my-auth] + (-> + (yaml/from-string (yaml/load-resource "backup/secret.yaml")) + (cm/replace-key-value :aws-access-key-id (b64/encode aws-access-key-id)) + (cm/replace-key-value :aws-secret-access-key (b64/encode aws-secret-access-key)) + (cm/replace-key-value :restic-password (b64/encode restic-password))))) diff --git a/src/main/resources/backup/backup-restore-deployment.yaml b/src/main/resources/backup/backup-restore-deployment.yaml new file mode 100644 index 0000000..163bc14 --- /dev/null +++ b/src/main/resources/backup/backup-restore-deployment.yaml @@ -0,0 +1,73 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: backup-restore +spec: + replicas: 0 + selector: + matchLabels: + app: backup-restore + strategy: + type: Recreate + template: + metadata: + labels: + app: backup-restore + app.kubernetes.io/name: backup-restore + app.kubernetes.io/part-of: forgejo + spec: + containers: + - image: domaindrivenarchitecture/c4k-forgejo-backup + name: backup-app + imagePullPolicy: IfNotPresent + command: ["/entrypoint-start-and-wait.sh"] + env: + - name: POSTGRES_USER + valueFrom: + secretKeyRef: + name: postgres-secret + key: postgres-user + - name: POSTGRES_PASSWORD + valueFrom: + secretKeyRef: + name: postgres-secret + key: postgres-password + - name: POSTGRES_DB + valueFrom: + configMapKeyRef: + name: postgres-config + key: postgres-db + - name: POSTGRES_HOST + value: "postgresql-service:5432" + - name: POSTGRES_SERVICE + value: "postgresql-service" + - name: POSTGRES_PORT + value: "5432" + - name: AWS_DEFAULT_REGION + value: eu-central-1 + - name: AWS_ACCESS_KEY_ID_FILE + value: /var/run/secrets/backup-secrets/aws-access-key-id + - name: AWS_SECRET_ACCESS_KEY_FILE + value: /var/run/secrets/backup-secrets/aws-secret-access-key + - name: RESTIC_REPOSITORY + valueFrom: + configMapKeyRef: + name: backup-config + key: restic-repository + - name: RESTIC_PASSWORD_FILE + value: /var/run/secrets/backup-secrets/restic-password + - name: CERTIFICATE_FILE + value: "" + volumeMounts: + - name: forgejo-data-volume + mountPath: /var/backups + - name: backup-secret-volume + mountPath: /var/run/secrets/backup-secrets + readOnly: true + volumes: + - name: forgejo-data-volume + persistentVolumeClaim: + claimName: forgejo-data-pvc + - name: backup-secret-volume + secret: + secretName: backup-secret \ No newline at end of file diff --git a/src/main/resources/backup/config.yaml b/src/main/resources/backup/config.yaml new file mode 100644 index 0000000..f7252a2 --- /dev/null +++ b/src/main/resources/backup/config.yaml @@ -0,0 +1,9 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: backup-config + labels: + app.kubernetes.io/name: backup + app.kubernetes.io/part-of: forgejo +data: + restic-repository: restic-repository \ No newline at end of file diff --git a/src/main/resources/backup/cron.yaml b/src/main/resources/backup/cron.yaml new file mode 100644 index 0000000..349b04a --- /dev/null +++ b/src/main/resources/backup/cron.yaml @@ -0,0 +1,70 @@ +apiVersion: batch/v1beta1 +kind: CronJob +metadata: + name: forgejo-backup + labels: + app.kubernetes.part-of: forgejo +spec: + schedule: "10 23 * * *" + successfulJobsHistoryLimit: 1 + failedJobsHistoryLimit: 1 + jobTemplate: + spec: + template: + spec: + containers: + - name: backup-app + image: domaindrivenarchitecture/c4k-forgejo-backup + imagePullPolicy: IfNotPresent + command: ["/entrypoint.sh"] + env: + - name: POSTGRES_USER + valueFrom: + secretKeyRef: + name: postgres-secret + key: postgres-user + - name: POSTGRES_PASSWORD + valueFrom: + secretKeyRef: + name: postgres-secret + key: postgres-password + - name: POSTGRES_DB + valueFrom: + configMapKeyRef: + name: postgres-config + key: postgres-db + - name: POSTGRES_HOST + value: "postgresql-service:5432" + - name: POSTGRES_SERVICE + value: "postgresql-service" + - name: POSTGRES_PORT + value: "5432" + - name: AWS_DEFAULT_REGION + value: eu-central-1 + - name: AWS_ACCESS_KEY_ID_FILE + value: /var/run/secrets/backup-secrets/aws-access-key-id + - name: AWS_SECRET_ACCESS_KEY_FILE + value: /var/run/secrets/backup-secrets/aws-secret-access-key + - name: RESTIC_REPOSITORY + valueFrom: + configMapKeyRef: + name: backup-config + key: restic-repository + - name: RESTIC_PASSWORD_FILE + value: /var/run/secrets/backup-secrets/restic-password + - name: CERTIFICATE_FILE + value: "" + volumeMounts: + - name: forgejo-data-volume + mountPath: /var/backups + - name: backup-secret-volume + mountPath: /var/run/secrets/backup-secrets + readOnly: true + volumes: + - name: forgejo-data-volume + persistentVolumeClaim: + claimName: forgejo-data-pvc + - name: backup-secret-volume + secret: + secretName: backup-secret + restartPolicy: OnFailure \ No newline at end of file diff --git a/src/main/resources/backup/secret.yaml b/src/main/resources/backup/secret.yaml new file mode 100644 index 0000000..c5809e0 --- /dev/null +++ b/src/main/resources/backup/secret.yaml @@ -0,0 +1,9 @@ +apiVersion: v1 +kind: Secret +metadata: + name: backup-secret +type: Opaque +data: + aws-access-key-id: aws-access-key-id + aws-secret-access-key: aws-secret-access-key + restic-password: restic-password \ No newline at end of file diff --git a/src/test/resources/taiga-test/valid-auth.yaml b/src/test/resources/taiga-test/valid-auth.yaml index 6ed8858..29515f0 100644 --- a/src/test/resources/taiga-test/valid-auth.yaml +++ b/src/test/resources/taiga-test/valid-auth.yaml @@ -9,6 +9,9 @@ django-superuser-email: "some@example.com" rabbitmq-user: "rabbit-user" rabbitmq-pw: "rabbit-pw" rabbitmq-erlang-cookie: "rabbit-erlang" +aws-access-key-id: "AWS_KEY_ID" +aws-secret-access-key: "AWS_KEY_SECRET" +restic-password: "" mon-auth: grafana-cloud-user: "user" grafana-cloud-password: "password" diff --git a/src/test/resources/taiga-test/valid-config.yaml b/src/test/resources/taiga-test/valid-config.yaml index a0a21f7..766cde0 100644 --- a/src/test/resources/taiga-test/valid-config.yaml +++ b/src/test/resources/taiga-test/valid-config.yaml @@ -7,6 +7,7 @@ storage-media-size: 2 storage-static-size: 3 storage-async-rabbitmq-size: 4 storage-events-rabbitmq-size: 5 +restic-repository: "repo-path" mon-cfg: grafana-cloud-url: "url-for-your-prom-remote-write-endpoint" cluster-name: "jitsi"