Start integrating backup from forgejo template

This commit is contained in:
erik 2023-10-18 11:05:16 +02:00
parent 63ba3c41f9
commit 206a860810
34 changed files with 408 additions and 0 deletions

View file

@ -31,6 +31,13 @@ You need:
Apply this file on your cluster with `kubectl apply -f yourApp.yaml`.
Done.
## Backup
- Taiga DB: https://docs.taiga.io/backup-and-restore.html
- Taiga Media: ?
- Taiga Protected: ?
- Taiga Static: ?
## Development & mirrors
Development happens at: https://repo.prod.meissa.de/meissa/c4k-taiga

View file

@ -0,0 +1,49 @@
from os import environ
from pybuilder.core import task, init
from ddadevops import *
name = "c4k-forgejo-backup"
MODULE = "docker"
PROJECT_ROOT_PATH = "../.."
@init
def initialize(project):
input = {
"name": name,
"module": MODULE,
"stage": "notused",
"project_root_path": PROJECT_ROOT_PATH,
"build_types": ["IMAGE"],
"mixin_types": [],
}
project.build_depends_on("ddadevops>=4.0.0-dev")
build = DevopsImageBuild(project, input)
build.initialize_build_dir()
@task
def image(project):
build = get_devops_build(project)
build.image()
@task
def drun(project):
build = get_devops_build(project)
build.drun()
@task
def publish(project):
build = get_devops_build(project)
build.dockerhub_login()
build.dockerhub_publish()
@task
def test(project):
build = get_devops_build(project)
build.test()

View file

@ -0,0 +1,5 @@
FROM domaindrivenarchitecture/dda-backup:1.0.6
# Prepare Entrypoint Script
ADD resources /tmp
RUN /tmp/install.sh

View file

@ -0,0 +1,18 @@
#!/bin/bash
set -o pipefail
function main() {
file_env AWS_ACCESS_KEY_ID
file_env AWS_SECRET_ACCESS_KEY
file_env RESTIC_DAYS_TO_KEEP 30
file_env RESTIC_MONTHS_TO_KEEP 12
backup-db-dump
}
source /usr/local/lib/functions.sh
source /usr/local/lib/pg-functions.sh
source /usr/local/lib/file-functions.sh
main

View file

@ -0,0 +1,13 @@
#!/bin/bash
function main() {
create-pg-pass
while true; do
sleep 1m
done
}
source /usr/local/lib/functions.sh
source /usr/local/lib/pg-functions.sh
main

View file

@ -0,0 +1,11 @@
#!/bin/bash
function main() {
create-pg-pass
/usr/local/bin/backup.sh
}
source /usr/local/lib/functions.sh
source /usr/local/lib/pg-functions.sh
main

View file

@ -0,0 +1,14 @@
#!/bin/bash
function main() {
file_env AWS_ACCESS_KEY_ID
file_env AWS_SECRET_ACCESS_KEY
init-database-repo
init-file-repo
}
source /usr/local/lib/functions.sh
source /usr/local/lib/pg-functions.sh
source /usr/local/lib/file-functions.sh
main

View file

@ -0,0 +1,11 @@
#!/bin/bash
apt-get update > /dev/null;
install -m 0700 /tmp/entrypoint.sh /
install -m 0700 /tmp/entrypoint-start-and-wait.sh /
install -m 0700 /tmp/init.sh /usr/local/bin/
install -m 0700 /tmp/backup.sh /usr/local/bin/
install -m 0700 /tmp/restore.sh /usr/local/bin/
install -m 0700 /tmp/restic-snapshots.sh /usr/local/bin/

View file

@ -0,0 +1,16 @@
#!/bin/bash
set -o pipefail
function main() {
file_env AWS_ACCESS_KEY_ID
file_env AWS_SECRET_ACCESS_KEY
restic -r ${RESTIC_REPOSITORY}/files snapshots
restic -r ${RESTIC_REPOSITORY}/pg-database snapshots
}
source /usr/local/lib/functions.sh
source /usr/local/lib/file-functions.sh
main

View file

@ -0,0 +1,38 @@
#!/bin/bash
set -Eeo pipefail
function main() {
file_env AWS_ACCESS_KEY_ID
file_env AWS_SECRET_ACCESS_KEY
file_env POSTGRES_DB
file_env POSTGRES_PASSWORD
file_env POSTGRES_USER
# Restore latest snapshot into /var/backups/restore
rm -rf /var/backups/restore
restore-directory '/var/backups/restore'
rm -rf /var/backups/gitea/*
rm -rf /var/backups/git/repositories/*
cp -r /var/backups/restore/gitea /var/backups/
cp -r /var/backups/restore/git/repositories /var/backups/git/
# adjust file permissions for the git user
chown -R 1000:1000 /var/backups
# TODO: Regenerate Git Hooks? Do we need this?
#/usr/local/bin/gitea -c '/data/gitea/conf/app.ini' admin regenerate hooks
# Restore db
drop-create-db
restore-db
}
source /usr/local/lib/functions.sh
source /usr/local/lib/pg-functions.sh
source /usr/local/lib/file-functions.sh
main

View file

@ -0,0 +1,11 @@
FROM c4k-forgejo-backup
RUN apt update
RUN apt -yqq --no-install-recommends --yes install curl default-jre-headless
RUN curl -L -o /tmp/serverspec.jar \
https://github.com/DomainDrivenArchitecture/dda-serverspec-crate/releases/download/2.0.0/dda-serverspec-standalone.jar
COPY serverspec.edn /tmp/serverspec.edn
RUN java -jar /tmp/serverspec.jar /tmp/serverspec.edn -v

View file

@ -0,0 +1,6 @@
{:file [{:path "/usr/local/bin/init.sh" :mod "700"}
{:path "/usr/local/bin/backup.sh" :mod "700"}
{:path "/usr/local/bin/restore.sh" :mod "700"}
{:path "/usr/local/bin/restic-snapshots.sh" :mod "700"}
{:path "/entrypoint.sh" :mod "700"}
{:path "/entrypoint-start-and-wait.sh" :mod "700"}]}

View file

@ -0,0 +1,44 @@
(ns dda.c4k-forgejo.backup
(:require
[clojure.spec.alpha :as s]
#?(:cljs [shadow.resource :as rc])
[dda.c4k-common.yaml :as yaml]
[dda.c4k-common.base64 :as b64]
[dda.c4k-common.common :as cm]))
(s/def ::aws-access-key-id cm/bash-env-string?)
(s/def ::aws-secret-access-key cm/bash-env-string?)
(s/def ::restic-password cm/bash-env-string?)
(s/def ::restic-repository cm/bash-env-string?)
#?(:cljs
(defmethod yaml/load-resource :backup [resource-name]
(case resource-name
"backup/config.yaml" (rc/inline "backup/config.yaml")
"backup/cron.yaml" (rc/inline "backup/cron.yaml")
"backup/secret.yaml" (rc/inline "backup/secret.yaml")
"backup/backup-restore-deployment.yaml" (rc/inline "backup/backup-restore-deployment.yaml")
(throw (js/Error. "Undefined Resource!")))))
(defn generate-config [my-conf]
(let [{:keys [restic-repository]} my-conf]
(->
(yaml/from-string (yaml/load-resource "backup/config.yaml"))
(cm/replace-key-value :restic-repository restic-repository))))
(defn generate-cron []
(yaml/from-string (yaml/load-resource "backup/cron.yaml")))
(defn generate-backup-restore-deployment [my-conf]
(let [backup-restore-yaml (yaml/from-string (yaml/load-resource "backup/backup-restore-deployment.yaml"))]
(if (and (contains? my-conf :local-integration-test) (= true (:local-integration-test my-conf)))
(cm/replace-named-value backup-restore-yaml "CERTIFICATE_FILE" "/var/run/secrets/localstack-secrets/ca.crt")
backup-restore-yaml)))
(defn generate-secret [my-auth]
(let [{:keys [aws-access-key-id aws-secret-access-key restic-password]} my-auth]
(->
(yaml/from-string (yaml/load-resource "backup/secret.yaml"))
(cm/replace-key-value :aws-access-key-id (b64/encode aws-access-key-id))
(cm/replace-key-value :aws-secret-access-key (b64/encode aws-secret-access-key))
(cm/replace-key-value :restic-password (b64/encode restic-password)))))

View file

@ -0,0 +1,73 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: backup-restore
spec:
replicas: 0
selector:
matchLabels:
app: backup-restore
strategy:
type: Recreate
template:
metadata:
labels:
app: backup-restore
app.kubernetes.io/name: backup-restore
app.kubernetes.io/part-of: forgejo
spec:
containers:
- image: domaindrivenarchitecture/c4k-forgejo-backup
name: backup-app
imagePullPolicy: IfNotPresent
command: ["/entrypoint-start-and-wait.sh"]
env:
- name: POSTGRES_USER
valueFrom:
secretKeyRef:
name: postgres-secret
key: postgres-user
- name: POSTGRES_PASSWORD
valueFrom:
secretKeyRef:
name: postgres-secret
key: postgres-password
- name: POSTGRES_DB
valueFrom:
configMapKeyRef:
name: postgres-config
key: postgres-db
- name: POSTGRES_HOST
value: "postgresql-service:5432"
- name: POSTGRES_SERVICE
value: "postgresql-service"
- name: POSTGRES_PORT
value: "5432"
- name: AWS_DEFAULT_REGION
value: eu-central-1
- name: AWS_ACCESS_KEY_ID_FILE
value: /var/run/secrets/backup-secrets/aws-access-key-id
- name: AWS_SECRET_ACCESS_KEY_FILE
value: /var/run/secrets/backup-secrets/aws-secret-access-key
- name: RESTIC_REPOSITORY
valueFrom:
configMapKeyRef:
name: backup-config
key: restic-repository
- name: RESTIC_PASSWORD_FILE
value: /var/run/secrets/backup-secrets/restic-password
- name: CERTIFICATE_FILE
value: ""
volumeMounts:
- name: forgejo-data-volume
mountPath: /var/backups
- name: backup-secret-volume
mountPath: /var/run/secrets/backup-secrets
readOnly: true
volumes:
- name: forgejo-data-volume
persistentVolumeClaim:
claimName: forgejo-data-pvc
- name: backup-secret-volume
secret:
secretName: backup-secret

View file

@ -0,0 +1,9 @@
apiVersion: v1
kind: ConfigMap
metadata:
name: backup-config
labels:
app.kubernetes.io/name: backup
app.kubernetes.io/part-of: forgejo
data:
restic-repository: restic-repository

View file

@ -0,0 +1,70 @@
apiVersion: batch/v1beta1
kind: CronJob
metadata:
name: forgejo-backup
labels:
app.kubernetes.part-of: forgejo
spec:
schedule: "10 23 * * *"
successfulJobsHistoryLimit: 1
failedJobsHistoryLimit: 1
jobTemplate:
spec:
template:
spec:
containers:
- name: backup-app
image: domaindrivenarchitecture/c4k-forgejo-backup
imagePullPolicy: IfNotPresent
command: ["/entrypoint.sh"]
env:
- name: POSTGRES_USER
valueFrom:
secretKeyRef:
name: postgres-secret
key: postgres-user
- name: POSTGRES_PASSWORD
valueFrom:
secretKeyRef:
name: postgres-secret
key: postgres-password
- name: POSTGRES_DB
valueFrom:
configMapKeyRef:
name: postgres-config
key: postgres-db
- name: POSTGRES_HOST
value: "postgresql-service:5432"
- name: POSTGRES_SERVICE
value: "postgresql-service"
- name: POSTGRES_PORT
value: "5432"
- name: AWS_DEFAULT_REGION
value: eu-central-1
- name: AWS_ACCESS_KEY_ID_FILE
value: /var/run/secrets/backup-secrets/aws-access-key-id
- name: AWS_SECRET_ACCESS_KEY_FILE
value: /var/run/secrets/backup-secrets/aws-secret-access-key
- name: RESTIC_REPOSITORY
valueFrom:
configMapKeyRef:
name: backup-config
key: restic-repository
- name: RESTIC_PASSWORD_FILE
value: /var/run/secrets/backup-secrets/restic-password
- name: CERTIFICATE_FILE
value: ""
volumeMounts:
- name: forgejo-data-volume
mountPath: /var/backups
- name: backup-secret-volume
mountPath: /var/run/secrets/backup-secrets
readOnly: true
volumes:
- name: forgejo-data-volume
persistentVolumeClaim:
claimName: forgejo-data-pvc
- name: backup-secret-volume
secret:
secretName: backup-secret
restartPolicy: OnFailure

View file

@ -0,0 +1,9 @@
apiVersion: v1
kind: Secret
metadata:
name: backup-secret
type: Opaque
data:
aws-access-key-id: aws-access-key-id
aws-secret-access-key: aws-secret-access-key
restic-password: restic-password

View file

@ -9,6 +9,9 @@ django-superuser-email: "some@example.com"
rabbitmq-user: "rabbit-user"
rabbitmq-pw: "rabbit-pw"
rabbitmq-erlang-cookie: "rabbit-erlang"
aws-access-key-id: "AWS_KEY_ID"
aws-secret-access-key: "AWS_KEY_SECRET"
restic-password: ""
mon-auth:
grafana-cloud-user: "user"
grafana-cloud-password: "password"

View file

@ -7,6 +7,7 @@ storage-media-size: 2
storage-static-size: 3
storage-async-rabbitmq-size: 4
storage-events-rabbitmq-size: 5
restic-repository: "repo-path"
mon-cfg:
grafana-cloud-url: "url-for-your-prom-remote-write-endpoint"
cluster-name: "jitsi"