Start integrating backup from forgejo template
This commit is contained in:
parent
63ba3c41f9
commit
206a860810
34 changed files with 408 additions and 0 deletions
|
@ -31,6 +31,13 @@ You need:
|
|||
Apply this file on your cluster with `kubectl apply -f yourApp.yaml`.
|
||||
Done.
|
||||
|
||||
## Backup
|
||||
|
||||
- Taiga DB: https://docs.taiga.io/backup-and-restore.html
|
||||
- Taiga Media: ?
|
||||
- Taiga Protected: ?
|
||||
- Taiga Static: ?
|
||||
|
||||
## Development & mirrors
|
||||
|
||||
Development happens at: https://repo.prod.meissa.de/meissa/c4k-taiga
|
||||
|
|
49
infrastructure/docker-backup/build.py
Normal file
49
infrastructure/docker-backup/build.py
Normal file
|
@ -0,0 +1,49 @@
|
|||
from os import environ
|
||||
from pybuilder.core import task, init
|
||||
from ddadevops import *
|
||||
|
||||
name = "c4k-forgejo-backup"
|
||||
MODULE = "docker"
|
||||
PROJECT_ROOT_PATH = "../.."
|
||||
|
||||
|
||||
@init
|
||||
def initialize(project):
|
||||
input = {
|
||||
"name": name,
|
||||
"module": MODULE,
|
||||
"stage": "notused",
|
||||
"project_root_path": PROJECT_ROOT_PATH,
|
||||
"build_types": ["IMAGE"],
|
||||
"mixin_types": [],
|
||||
}
|
||||
|
||||
project.build_depends_on("ddadevops>=4.0.0-dev")
|
||||
|
||||
build = DevopsImageBuild(project, input)
|
||||
build.initialize_build_dir()
|
||||
|
||||
|
||||
@task
|
||||
def image(project):
|
||||
build = get_devops_build(project)
|
||||
build.image()
|
||||
|
||||
|
||||
@task
|
||||
def drun(project):
|
||||
build = get_devops_build(project)
|
||||
build.drun()
|
||||
|
||||
|
||||
@task
|
||||
def publish(project):
|
||||
build = get_devops_build(project)
|
||||
build.dockerhub_login()
|
||||
build.dockerhub_publish()
|
||||
|
||||
|
||||
@task
|
||||
def test(project):
|
||||
build = get_devops_build(project)
|
||||
build.test()
|
5
infrastructure/docker-backup/image/Dockerfile
Normal file
5
infrastructure/docker-backup/image/Dockerfile
Normal file
|
@ -0,0 +1,5 @@
|
|||
FROM domaindrivenarchitecture/dda-backup:1.0.6
|
||||
|
||||
# Prepare Entrypoint Script
|
||||
ADD resources /tmp
|
||||
RUN /tmp/install.sh
|
18
infrastructure/docker-backup/image/resources/backup.sh
Executable file
18
infrastructure/docker-backup/image/resources/backup.sh
Executable file
|
@ -0,0 +1,18 @@
|
|||
#!/bin/bash
|
||||
|
||||
set -o pipefail
|
||||
|
||||
function main() {
|
||||
file_env AWS_ACCESS_KEY_ID
|
||||
file_env AWS_SECRET_ACCESS_KEY
|
||||
file_env RESTIC_DAYS_TO_KEEP 30
|
||||
file_env RESTIC_MONTHS_TO_KEEP 12
|
||||
|
||||
backup-db-dump
|
||||
}
|
||||
|
||||
source /usr/local/lib/functions.sh
|
||||
source /usr/local/lib/pg-functions.sh
|
||||
source /usr/local/lib/file-functions.sh
|
||||
|
||||
main
|
|
@ -0,0 +1,13 @@
|
|||
#!/bin/bash
|
||||
|
||||
function main() {
|
||||
create-pg-pass
|
||||
|
||||
while true; do
|
||||
sleep 1m
|
||||
done
|
||||
}
|
||||
|
||||
source /usr/local/lib/functions.sh
|
||||
source /usr/local/lib/pg-functions.sh
|
||||
main
|
11
infrastructure/docker-backup/image/resources/entrypoint.sh
Executable file
11
infrastructure/docker-backup/image/resources/entrypoint.sh
Executable file
|
@ -0,0 +1,11 @@
|
|||
#!/bin/bash
|
||||
|
||||
function main() {
|
||||
create-pg-pass
|
||||
|
||||
/usr/local/bin/backup.sh
|
||||
}
|
||||
|
||||
source /usr/local/lib/functions.sh
|
||||
source /usr/local/lib/pg-functions.sh
|
||||
main
|
14
infrastructure/docker-backup/image/resources/init.sh
Executable file
14
infrastructure/docker-backup/image/resources/init.sh
Executable file
|
@ -0,0 +1,14 @@
|
|||
#!/bin/bash
|
||||
|
||||
function main() {
|
||||
file_env AWS_ACCESS_KEY_ID
|
||||
file_env AWS_SECRET_ACCESS_KEY
|
||||
|
||||
init-database-repo
|
||||
init-file-repo
|
||||
}
|
||||
|
||||
source /usr/local/lib/functions.sh
|
||||
source /usr/local/lib/pg-functions.sh
|
||||
source /usr/local/lib/file-functions.sh
|
||||
main
|
11
infrastructure/docker-backup/image/resources/install.sh
Executable file
11
infrastructure/docker-backup/image/resources/install.sh
Executable file
|
@ -0,0 +1,11 @@
|
|||
#!/bin/bash
|
||||
|
||||
apt-get update > /dev/null;
|
||||
|
||||
install -m 0700 /tmp/entrypoint.sh /
|
||||
install -m 0700 /tmp/entrypoint-start-and-wait.sh /
|
||||
|
||||
install -m 0700 /tmp/init.sh /usr/local/bin/
|
||||
install -m 0700 /tmp/backup.sh /usr/local/bin/
|
||||
install -m 0700 /tmp/restore.sh /usr/local/bin/
|
||||
install -m 0700 /tmp/restic-snapshots.sh /usr/local/bin/
|
16
infrastructure/docker-backup/image/resources/restic-snapshots.sh
Executable file
16
infrastructure/docker-backup/image/resources/restic-snapshots.sh
Executable file
|
@ -0,0 +1,16 @@
|
|||
#!/bin/bash
|
||||
|
||||
set -o pipefail
|
||||
|
||||
function main() {
|
||||
file_env AWS_ACCESS_KEY_ID
|
||||
file_env AWS_SECRET_ACCESS_KEY
|
||||
|
||||
restic -r ${RESTIC_REPOSITORY}/files snapshots
|
||||
restic -r ${RESTIC_REPOSITORY}/pg-database snapshots
|
||||
}
|
||||
|
||||
source /usr/local/lib/functions.sh
|
||||
source /usr/local/lib/file-functions.sh
|
||||
|
||||
main
|
38
infrastructure/docker-backup/image/resources/restore.sh
Executable file
38
infrastructure/docker-backup/image/resources/restore.sh
Executable file
|
@ -0,0 +1,38 @@
|
|||
#!/bin/bash
|
||||
|
||||
set -Eeo pipefail
|
||||
|
||||
function main() {
|
||||
|
||||
file_env AWS_ACCESS_KEY_ID
|
||||
file_env AWS_SECRET_ACCESS_KEY
|
||||
|
||||
file_env POSTGRES_DB
|
||||
file_env POSTGRES_PASSWORD
|
||||
file_env POSTGRES_USER
|
||||
|
||||
# Restore latest snapshot into /var/backups/restore
|
||||
rm -rf /var/backups/restore
|
||||
restore-directory '/var/backups/restore'
|
||||
|
||||
rm -rf /var/backups/gitea/*
|
||||
rm -rf /var/backups/git/repositories/*
|
||||
cp -r /var/backups/restore/gitea /var/backups/
|
||||
cp -r /var/backups/restore/git/repositories /var/backups/git/
|
||||
|
||||
# adjust file permissions for the git user
|
||||
chown -R 1000:1000 /var/backups
|
||||
|
||||
# TODO: Regenerate Git Hooks? Do we need this?
|
||||
#/usr/local/bin/gitea -c '/data/gitea/conf/app.ini' admin regenerate hooks
|
||||
|
||||
# Restore db
|
||||
drop-create-db
|
||||
restore-db
|
||||
}
|
||||
|
||||
source /usr/local/lib/functions.sh
|
||||
source /usr/local/lib/pg-functions.sh
|
||||
source /usr/local/lib/file-functions.sh
|
||||
|
||||
main
|
11
infrastructure/docker-backup/test/Dockerfile
Normal file
11
infrastructure/docker-backup/test/Dockerfile
Normal file
|
@ -0,0 +1,11 @@
|
|||
FROM c4k-forgejo-backup
|
||||
|
||||
RUN apt update
|
||||
RUN apt -yqq --no-install-recommends --yes install curl default-jre-headless
|
||||
|
||||
RUN curl -L -o /tmp/serverspec.jar \
|
||||
https://github.com/DomainDrivenArchitecture/dda-serverspec-crate/releases/download/2.0.0/dda-serverspec-standalone.jar
|
||||
|
||||
COPY serverspec.edn /tmp/serverspec.edn
|
||||
|
||||
RUN java -jar /tmp/serverspec.jar /tmp/serverspec.edn -v
|
6
infrastructure/docker-backup/test/serverspec.edn
Normal file
6
infrastructure/docker-backup/test/serverspec.edn
Normal file
|
@ -0,0 +1,6 @@
|
|||
{:file [{:path "/usr/local/bin/init.sh" :mod "700"}
|
||||
{:path "/usr/local/bin/backup.sh" :mod "700"}
|
||||
{:path "/usr/local/bin/restore.sh" :mod "700"}
|
||||
{:path "/usr/local/bin/restic-snapshots.sh" :mod "700"}
|
||||
{:path "/entrypoint.sh" :mod "700"}
|
||||
{:path "/entrypoint-start-and-wait.sh" :mod "700"}]}
|
44
src/main/cljc/dda/c4k_taiga/backup.cljc
Normal file
44
src/main/cljc/dda/c4k_taiga/backup.cljc
Normal file
|
@ -0,0 +1,44 @@
|
|||
(ns dda.c4k-forgejo.backup
|
||||
(:require
|
||||
[clojure.spec.alpha :as s]
|
||||
#?(:cljs [shadow.resource :as rc])
|
||||
[dda.c4k-common.yaml :as yaml]
|
||||
[dda.c4k-common.base64 :as b64]
|
||||
[dda.c4k-common.common :as cm]))
|
||||
|
||||
(s/def ::aws-access-key-id cm/bash-env-string?)
|
||||
(s/def ::aws-secret-access-key cm/bash-env-string?)
|
||||
(s/def ::restic-password cm/bash-env-string?)
|
||||
(s/def ::restic-repository cm/bash-env-string?)
|
||||
|
||||
#?(:cljs
|
||||
(defmethod yaml/load-resource :backup [resource-name]
|
||||
(case resource-name
|
||||
"backup/config.yaml" (rc/inline "backup/config.yaml")
|
||||
"backup/cron.yaml" (rc/inline "backup/cron.yaml")
|
||||
"backup/secret.yaml" (rc/inline "backup/secret.yaml")
|
||||
"backup/backup-restore-deployment.yaml" (rc/inline "backup/backup-restore-deployment.yaml")
|
||||
(throw (js/Error. "Undefined Resource!")))))
|
||||
|
||||
(defn generate-config [my-conf]
|
||||
(let [{:keys [restic-repository]} my-conf]
|
||||
(->
|
||||
(yaml/from-string (yaml/load-resource "backup/config.yaml"))
|
||||
(cm/replace-key-value :restic-repository restic-repository))))
|
||||
|
||||
(defn generate-cron []
|
||||
(yaml/from-string (yaml/load-resource "backup/cron.yaml")))
|
||||
|
||||
(defn generate-backup-restore-deployment [my-conf]
|
||||
(let [backup-restore-yaml (yaml/from-string (yaml/load-resource "backup/backup-restore-deployment.yaml"))]
|
||||
(if (and (contains? my-conf :local-integration-test) (= true (:local-integration-test my-conf)))
|
||||
(cm/replace-named-value backup-restore-yaml "CERTIFICATE_FILE" "/var/run/secrets/localstack-secrets/ca.crt")
|
||||
backup-restore-yaml)))
|
||||
|
||||
(defn generate-secret [my-auth]
|
||||
(let [{:keys [aws-access-key-id aws-secret-access-key restic-password]} my-auth]
|
||||
(->
|
||||
(yaml/from-string (yaml/load-resource "backup/secret.yaml"))
|
||||
(cm/replace-key-value :aws-access-key-id (b64/encode aws-access-key-id))
|
||||
(cm/replace-key-value :aws-secret-access-key (b64/encode aws-secret-access-key))
|
||||
(cm/replace-key-value :restic-password (b64/encode restic-password)))))
|
73
src/main/resources/backup/backup-restore-deployment.yaml
Normal file
73
src/main/resources/backup/backup-restore-deployment.yaml
Normal file
|
@ -0,0 +1,73 @@
|
|||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: backup-restore
|
||||
spec:
|
||||
replicas: 0
|
||||
selector:
|
||||
matchLabels:
|
||||
app: backup-restore
|
||||
strategy:
|
||||
type: Recreate
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: backup-restore
|
||||
app.kubernetes.io/name: backup-restore
|
||||
app.kubernetes.io/part-of: forgejo
|
||||
spec:
|
||||
containers:
|
||||
- image: domaindrivenarchitecture/c4k-forgejo-backup
|
||||
name: backup-app
|
||||
imagePullPolicy: IfNotPresent
|
||||
command: ["/entrypoint-start-and-wait.sh"]
|
||||
env:
|
||||
- name: POSTGRES_USER
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: postgres-secret
|
||||
key: postgres-user
|
||||
- name: POSTGRES_PASSWORD
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: postgres-secret
|
||||
key: postgres-password
|
||||
- name: POSTGRES_DB
|
||||
valueFrom:
|
||||
configMapKeyRef:
|
||||
name: postgres-config
|
||||
key: postgres-db
|
||||
- name: POSTGRES_HOST
|
||||
value: "postgresql-service:5432"
|
||||
- name: POSTGRES_SERVICE
|
||||
value: "postgresql-service"
|
||||
- name: POSTGRES_PORT
|
||||
value: "5432"
|
||||
- name: AWS_DEFAULT_REGION
|
||||
value: eu-central-1
|
||||
- name: AWS_ACCESS_KEY_ID_FILE
|
||||
value: /var/run/secrets/backup-secrets/aws-access-key-id
|
||||
- name: AWS_SECRET_ACCESS_KEY_FILE
|
||||
value: /var/run/secrets/backup-secrets/aws-secret-access-key
|
||||
- name: RESTIC_REPOSITORY
|
||||
valueFrom:
|
||||
configMapKeyRef:
|
||||
name: backup-config
|
||||
key: restic-repository
|
||||
- name: RESTIC_PASSWORD_FILE
|
||||
value: /var/run/secrets/backup-secrets/restic-password
|
||||
- name: CERTIFICATE_FILE
|
||||
value: ""
|
||||
volumeMounts:
|
||||
- name: forgejo-data-volume
|
||||
mountPath: /var/backups
|
||||
- name: backup-secret-volume
|
||||
mountPath: /var/run/secrets/backup-secrets
|
||||
readOnly: true
|
||||
volumes:
|
||||
- name: forgejo-data-volume
|
||||
persistentVolumeClaim:
|
||||
claimName: forgejo-data-pvc
|
||||
- name: backup-secret-volume
|
||||
secret:
|
||||
secretName: backup-secret
|
9
src/main/resources/backup/config.yaml
Normal file
9
src/main/resources/backup/config.yaml
Normal file
|
@ -0,0 +1,9 @@
|
|||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: backup-config
|
||||
labels:
|
||||
app.kubernetes.io/name: backup
|
||||
app.kubernetes.io/part-of: forgejo
|
||||
data:
|
||||
restic-repository: restic-repository
|
70
src/main/resources/backup/cron.yaml
Normal file
70
src/main/resources/backup/cron.yaml
Normal file
|
@ -0,0 +1,70 @@
|
|||
apiVersion: batch/v1beta1
|
||||
kind: CronJob
|
||||
metadata:
|
||||
name: forgejo-backup
|
||||
labels:
|
||||
app.kubernetes.part-of: forgejo
|
||||
spec:
|
||||
schedule: "10 23 * * *"
|
||||
successfulJobsHistoryLimit: 1
|
||||
failedJobsHistoryLimit: 1
|
||||
jobTemplate:
|
||||
spec:
|
||||
template:
|
||||
spec:
|
||||
containers:
|
||||
- name: backup-app
|
||||
image: domaindrivenarchitecture/c4k-forgejo-backup
|
||||
imagePullPolicy: IfNotPresent
|
||||
command: ["/entrypoint.sh"]
|
||||
env:
|
||||
- name: POSTGRES_USER
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: postgres-secret
|
||||
key: postgres-user
|
||||
- name: POSTGRES_PASSWORD
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: postgres-secret
|
||||
key: postgres-password
|
||||
- name: POSTGRES_DB
|
||||
valueFrom:
|
||||
configMapKeyRef:
|
||||
name: postgres-config
|
||||
key: postgres-db
|
||||
- name: POSTGRES_HOST
|
||||
value: "postgresql-service:5432"
|
||||
- name: POSTGRES_SERVICE
|
||||
value: "postgresql-service"
|
||||
- name: POSTGRES_PORT
|
||||
value: "5432"
|
||||
- name: AWS_DEFAULT_REGION
|
||||
value: eu-central-1
|
||||
- name: AWS_ACCESS_KEY_ID_FILE
|
||||
value: /var/run/secrets/backup-secrets/aws-access-key-id
|
||||
- name: AWS_SECRET_ACCESS_KEY_FILE
|
||||
value: /var/run/secrets/backup-secrets/aws-secret-access-key
|
||||
- name: RESTIC_REPOSITORY
|
||||
valueFrom:
|
||||
configMapKeyRef:
|
||||
name: backup-config
|
||||
key: restic-repository
|
||||
- name: RESTIC_PASSWORD_FILE
|
||||
value: /var/run/secrets/backup-secrets/restic-password
|
||||
- name: CERTIFICATE_FILE
|
||||
value: ""
|
||||
volumeMounts:
|
||||
- name: forgejo-data-volume
|
||||
mountPath: /var/backups
|
||||
- name: backup-secret-volume
|
||||
mountPath: /var/run/secrets/backup-secrets
|
||||
readOnly: true
|
||||
volumes:
|
||||
- name: forgejo-data-volume
|
||||
persistentVolumeClaim:
|
||||
claimName: forgejo-data-pvc
|
||||
- name: backup-secret-volume
|
||||
secret:
|
||||
secretName: backup-secret
|
||||
restartPolicy: OnFailure
|
9
src/main/resources/backup/secret.yaml
Normal file
9
src/main/resources/backup/secret.yaml
Normal file
|
@ -0,0 +1,9 @@
|
|||
apiVersion: v1
|
||||
kind: Secret
|
||||
metadata:
|
||||
name: backup-secret
|
||||
type: Opaque
|
||||
data:
|
||||
aws-access-key-id: aws-access-key-id
|
||||
aws-secret-access-key: aws-secret-access-key
|
||||
restic-password: restic-password
|
|
@ -9,6 +9,9 @@ django-superuser-email: "some@example.com"
|
|||
rabbitmq-user: "rabbit-user"
|
||||
rabbitmq-pw: "rabbit-pw"
|
||||
rabbitmq-erlang-cookie: "rabbit-erlang"
|
||||
aws-access-key-id: "AWS_KEY_ID"
|
||||
aws-secret-access-key: "AWS_KEY_SECRET"
|
||||
restic-password: ""
|
||||
mon-auth:
|
||||
grafana-cloud-user: "user"
|
||||
grafana-cloud-password: "password"
|
||||
|
|
|
@ -7,6 +7,7 @@ storage-media-size: 2
|
|||
storage-static-size: 3
|
||||
storage-async-rabbitmq-size: 4
|
||||
storage-events-rabbitmq-size: 5
|
||||
restic-repository: "repo-path"
|
||||
mon-cfg:
|
||||
grafana-cloud-url: "url-for-your-prom-remote-write-endpoint"
|
||||
cluster-name: "jitsi"
|
||||
|
|
Loading…
Reference in a new issue