Merge branch 'backup' into 'main'
Backup See merge request domaindrivenarchitecture/c4k-gitea!2merge-requests/3/head
commit
fbc3a660c3
@ -0,0 +1,41 @@
|
|||||||
|
# Backup Architecture details
|
||||||
|
|
||||||
|
![](backup.svg)
|
||||||
|
|
||||||
|
* we use restic to produce small & encrypted backups
|
||||||
|
* backup is scheduled at `schedule: "10 23 * * *"`
|
||||||
|
* Gitea stores files in `/data/gitea` and `/data/git/repositories`, these files are backed up.
|
||||||
|
* The postgres db is also backed up
|
||||||
|
|
||||||
|
## Manual init the restic repository for the first time
|
||||||
|
|
||||||
|
1. apply backup-and-restore pod:
|
||||||
|
`kubectl scale deployment backup-restore --replicas=1`
|
||||||
|
2. exec into pod and execute restore pod (press tab to get your exact pod name)
|
||||||
|
`kubectl exec -it backup-restore-... -- /usr/local/bin/init.sh`
|
||||||
|
3. remove backup-and-restore pod:
|
||||||
|
`kubectl scale deployment backup-restore --replicas=0`
|
||||||
|
|
||||||
|
|
||||||
|
## Manual backup the restic repository for the first time
|
||||||
|
|
||||||
|
1. apply backup-and-restore pod:
|
||||||
|
`kubectl scale deployment backup-restore --replicas=1`
|
||||||
|
2. exec into pod and execute restore pod (press tab to get your exact pod name)
|
||||||
|
`kubectl exec -it backup-restore-... -- /usr/local/bin/backup.sh`
|
||||||
|
3. remove backup-and-restore pod:
|
||||||
|
`kubectl scale deployment backup-restore --replicas=0`
|
||||||
|
|
||||||
|
|
||||||
|
## Manual restore
|
||||||
|
|
||||||
|
1. apply backup-and-restore pod:
|
||||||
|
`kubectl scale deployment backup-restore --replicas=1`
|
||||||
|
2. Scale down gitea deployment:
|
||||||
|
`kubectl scale deployment gitea --replicas=0`
|
||||||
|
3. exec into pod and execute restore pod (press tab to get your exact pod name)
|
||||||
|
`kubectl exec -it backup-restore-... -- /usr/local/bin/restore.sh`
|
||||||
|
4. Start gitea again:
|
||||||
|
`kubectl scale deployment gitea --replicas=1`
|
||||||
|
5. remove backup-and-restore pod:
|
||||||
|
`kubectl scale deployment backup-restore --replicas=0`
|
@ -0,0 +1,51 @@
|
|||||||
|
from os import environ
|
||||||
|
from pybuilder.core import task, init
|
||||||
|
from ddadevops import *
|
||||||
|
import logging
|
||||||
|
|
||||||
|
name = 'c4k-gitea-backup'
|
||||||
|
MODULE = 'docker'
|
||||||
|
PROJECT_ROOT_PATH = '../..'
|
||||||
|
|
||||||
|
class MyBuild(DevopsDockerBuild):
|
||||||
|
pass
|
||||||
|
|
||||||
|
@init
|
||||||
|
def initialize(project):
|
||||||
|
project.build_depends_on('ddadevops>=0.12.4')
|
||||||
|
stage = 'prod'
|
||||||
|
dockerhub_user = environ.get('DOCKERHUB_USER')
|
||||||
|
if not dockerhub_user:
|
||||||
|
dockerhub_user = gopass_field_from_path('meissa/web/docker.com', 'login')
|
||||||
|
dockerhub_password = environ.get('DOCKERHUB_PASSWORD')
|
||||||
|
if not dockerhub_password:
|
||||||
|
dockerhub_password = gopass_password_from_path('meissa/web/docker.com')
|
||||||
|
tag = environ.get('CI_COMMIT_TAG')
|
||||||
|
if not tag:
|
||||||
|
tag = get_tag_from_latest_commit()
|
||||||
|
config = create_devops_docker_build_config(
|
||||||
|
stage, PROJECT_ROOT_PATH, MODULE, dockerhub_user, dockerhub_password, docker_publish_tag=tag)
|
||||||
|
build = MyBuild(project, config)
|
||||||
|
build.initialize_build_dir()
|
||||||
|
|
||||||
|
|
||||||
|
@task
|
||||||
|
def image(project):
|
||||||
|
build = get_devops_build(project)
|
||||||
|
build.image()
|
||||||
|
|
||||||
|
@task
|
||||||
|
def drun(project):
|
||||||
|
build = get_devops_build(project)
|
||||||
|
build.drun()
|
||||||
|
|
||||||
|
@task
|
||||||
|
def publish(project):
|
||||||
|
build = get_devops_build(project)
|
||||||
|
build.dockerhub_login()
|
||||||
|
build.dockerhub_publish()
|
||||||
|
|
||||||
|
@task
|
||||||
|
def test(project):
|
||||||
|
build = get_devops_build(project)
|
||||||
|
build.test()
|
@ -0,0 +1,5 @@
|
|||||||
|
FROM domaindrivenarchitecture/dda-backup:1.0.6
|
||||||
|
|
||||||
|
# Prepare Entrypoint Script
|
||||||
|
ADD resources /tmp
|
||||||
|
RUN /tmp/install.sh
|
@ -0,0 +1,19 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
|
||||||
|
set -o pipefail
|
||||||
|
|
||||||
|
function main() {
|
||||||
|
file_env AWS_ACCESS_KEY_ID
|
||||||
|
file_env AWS_SECRET_ACCESS_KEY
|
||||||
|
file_env RESTIC_DAYS_TO_KEEP 30
|
||||||
|
file_env RESTIC_MONTHS_TO_KEEP 12
|
||||||
|
|
||||||
|
backup-db-dump
|
||||||
|
backup-fs-from-directory '/var/backups/' 'gitea/' 'git/repositories/'
|
||||||
|
}
|
||||||
|
|
||||||
|
source /usr/local/lib/functions.sh
|
||||||
|
source /usr/local/lib/pg-functions.sh
|
||||||
|
source /usr/local/lib/file-functions.sh
|
||||||
|
|
||||||
|
main
|
@ -0,0 +1,13 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
|
||||||
|
function main() {
|
||||||
|
create-pg-pass
|
||||||
|
|
||||||
|
while true; do
|
||||||
|
sleep 1m
|
||||||
|
done
|
||||||
|
}
|
||||||
|
|
||||||
|
source /usr/local/lib/functions.sh
|
||||||
|
source /usr/local/lib/pg-functions.sh
|
||||||
|
main
|
@ -0,0 +1,11 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
|
||||||
|
function main() {
|
||||||
|
create-pg-pass
|
||||||
|
|
||||||
|
/usr/local/bin/backup.sh
|
||||||
|
}
|
||||||
|
|
||||||
|
source /usr/local/lib/functions.sh
|
||||||
|
source /usr/local/lib/pg-functions.sh
|
||||||
|
main
|
@ -0,0 +1,14 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
|
||||||
|
function main() {
|
||||||
|
file_env AWS_ACCESS_KEY_ID
|
||||||
|
file_env AWS_SECRET_ACCESS_KEY
|
||||||
|
|
||||||
|
init-database-repo
|
||||||
|
init-file-repo
|
||||||
|
}
|
||||||
|
|
||||||
|
source /usr/local/lib/functions.sh
|
||||||
|
source /usr/local/lib/pg-functions.sh
|
||||||
|
source /usr/local/lib/file-functions.sh
|
||||||
|
main
|
@ -0,0 +1,11 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
|
||||||
|
apt-get update > /dev/null;
|
||||||
|
|
||||||
|
install -m 0700 /tmp/entrypoint.sh /
|
||||||
|
install -m 0700 /tmp/entrypoint-start-and-wait.sh /
|
||||||
|
|
||||||
|
install -m 0700 /tmp/init.sh /usr/local/bin/
|
||||||
|
install -m 0700 /tmp/backup.sh /usr/local/bin/
|
||||||
|
install -m 0700 /tmp/restore.sh /usr/local/bin/
|
||||||
|
install -m 0700 /tmp/restic-snapshots.sh /usr/local/bin/
|
@ -0,0 +1,16 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
|
||||||
|
set -o pipefail
|
||||||
|
|
||||||
|
function main() {
|
||||||
|
file_env AWS_ACCESS_KEY_ID
|
||||||
|
file_env AWS_SECRET_ACCESS_KEY
|
||||||
|
|
||||||
|
restic -r ${RESTIC_REPOSITORY}/files snapshots
|
||||||
|
restic -r ${RESTIC_REPOSITORY}/pg-database snapshots
|
||||||
|
}
|
||||||
|
|
||||||
|
source /usr/local/lib/functions.sh
|
||||||
|
source /usr/local/lib/file-functions.sh
|
||||||
|
|
||||||
|
main
|
@ -0,0 +1,38 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
|
||||||
|
set -Eeo pipefail
|
||||||
|
|
||||||
|
function main() {
|
||||||
|
|
||||||
|
file_env AWS_ACCESS_KEY_ID
|
||||||
|
file_env AWS_SECRET_ACCESS_KEY
|
||||||
|
|
||||||
|
file_env POSTGRES_DB
|
||||||
|
file_env POSTGRES_PASSWORD
|
||||||
|
file_env POSTGRES_USER
|
||||||
|
|
||||||
|
# Restore latest snapshot into /var/backups/restore
|
||||||
|
rm -rf /var/backups/restore
|
||||||
|
restore-directory '/var/backups/restore'
|
||||||
|
|
||||||
|
rm -rf /var/backups/gitea/*
|
||||||
|
rm -rf /var/backups/git/repositories/*
|
||||||
|
cp -r /var/backups/restore/gitea/* /var/backups/gitea/
|
||||||
|
cp -r /var/backups/restore/git/repositories/* /var/backups/git/repositories/
|
||||||
|
|
||||||
|
# adjust file permissions for the git user
|
||||||
|
chown -R 1000:1000 /var/backups
|
||||||
|
|
||||||
|
# TODO: Regenerate Git Hooks? Do we need this?
|
||||||
|
#/usr/local/bin/gitea -c '/data/gitea/conf/app.ini' admin regenerate hooks
|
||||||
|
|
||||||
|
# Restore db
|
||||||
|
drop-create-db
|
||||||
|
restore-db
|
||||||
|
}
|
||||||
|
|
||||||
|
source /usr/local/lib/functions.sh
|
||||||
|
source /usr/local/lib/pg-functions.sh
|
||||||
|
source /usr/local/lib/file-functions.sh
|
||||||
|
|
||||||
|
main
|
@ -0,0 +1,11 @@
|
|||||||
|
FROM c4k-jira-backup
|
||||||
|
|
||||||
|
RUN apt update
|
||||||
|
RUN apt -yqq --no-install-recommends --yes install curl default-jre-headless
|
||||||
|
|
||||||
|
RUN curl -L -o /tmp/serverspec.jar \
|
||||||
|
https://github.com/DomainDrivenArchitecture/dda-serverspec-crate/releases/download/2.0.0/dda-serverspec-standalone.jar
|
||||||
|
|
||||||
|
COPY serverspec.edn /tmp/serverspec.edn
|
||||||
|
|
||||||
|
RUN java -jar /tmp/serverspec.jar /tmp/serverspec.edn -v
|
@ -0,0 +1,6 @@
|
|||||||
|
{:file [{:path "/usr/local/bin/init.sh" :mod "700"}
|
||||||
|
{:path "/usr/local/bin/backup.sh" :mod "700"}
|
||||||
|
{:path "/usr/local/bin/restore.sh" :mod "700"}
|
||||||
|
{:path "/usr/local/bin/restic-snapshots.sh" :mod "700"}
|
||||||
|
{:path "/entrypoint.sh" :mod "700"}
|
||||||
|
{:path "/entrypoint-start-and-wait.sh" :mod "700"}]}
|
@ -0,0 +1,44 @@
|
|||||||
|
(ns dda.c4k-gitea.backup
|
||||||
|
(:require
|
||||||
|
[clojure.spec.alpha :as s]
|
||||||
|
#?(:cljs [shadow.resource :as rc])
|
||||||
|
[dda.c4k-common.yaml :as yaml]
|
||||||
|
[dda.c4k-common.base64 :as b64]
|
||||||
|
[dda.c4k-common.common :as cm]))
|
||||||
|
|
||||||
|
(s/def ::aws-access-key-id cm/bash-env-string?)
|
||||||
|
(s/def ::aws-secret-access-key cm/bash-env-string?)
|
||||||
|
(s/def ::restic-password cm/bash-env-string?)
|
||||||
|
(s/def ::restic-repository cm/bash-env-string?)
|
||||||
|
|
||||||
|
#?(:cljs
|
||||||
|
(defmethod yaml/load-resource :backup [resource-name]
|
||||||
|
(case resource-name
|
||||||
|
"backup/config.yaml" (rc/inline "backup/config.yaml")
|
||||||
|
"backup/cron.yaml" (rc/inline "backup/cron.yaml")
|
||||||
|
"backup/secret.yaml" (rc/inline "backup/secret.yaml")
|
||||||
|
"backup/backup-restore-deployment.yaml" (rc/inline "backup/backup-restore-deployment.yaml")
|
||||||
|
(throw (js/Error. "Undefined Resource!")))))
|
||||||
|
|
||||||
|
(defn generate-config [my-conf]
|
||||||
|
(let [{:keys [restic-repository]} my-conf]
|
||||||
|
(->
|
||||||
|
(yaml/from-string (yaml/load-resource "backup/config.yaml"))
|
||||||
|
(cm/replace-key-value :restic-repository restic-repository))))
|
||||||
|
|
||||||
|
(defn generate-cron []
|
||||||
|
(yaml/from-string (yaml/load-resource "backup/cron.yaml")))
|
||||||
|
|
||||||
|
(defn generate-backup-restore-deployment [my-conf]
|
||||||
|
(let [backup-restore-yaml (yaml/from-string (yaml/load-resource "backup/backup-restore-deployment.yaml"))]
|
||||||
|
(if (and (contains? my-conf :local-integration-test) (= true (:local-integration-test my-conf)))
|
||||||
|
(cm/replace-named-value backup-restore-yaml "CERTIFICATE_FILE" "/var/run/secrets/localstack-secrets/ca.crt")
|
||||||
|
backup-restore-yaml)))
|
||||||
|
|
||||||
|
(defn generate-secret [my-auth]
|
||||||
|
(let [{:keys [aws-access-key-id aws-secret-access-key restic-password]} my-auth]
|
||||||
|
(->
|
||||||
|
(yaml/from-string (yaml/load-resource "backup/secret.yaml"))
|
||||||
|
(cm/replace-key-value :aws-access-key-id (b64/encode aws-access-key-id))
|
||||||
|
(cm/replace-key-value :aws-secret-access-key (b64/encode aws-secret-access-key))
|
||||||
|
(cm/replace-key-value :restic-password (b64/encode restic-password)))))
|
@ -0,0 +1,73 @@
|
|||||||
|
apiVersion: apps/v1
|
||||||
|
kind: Deployment
|
||||||
|
metadata:
|
||||||
|
name: backup-restore
|
||||||
|
spec:
|
||||||
|
replicas: 0
|
||||||
|
selector:
|
||||||
|
matchLabels:
|
||||||
|
app: backup-restore
|
||||||
|
strategy:
|
||||||
|
type: Recreate
|
||||||
|
template:
|
||||||
|
metadata:
|
||||||
|
labels:
|
||||||
|
app: backup-restore
|
||||||
|
app.kubernetes.io/name: backup-restore
|
||||||
|
app.kubernetes.io/part-of: gitea
|
||||||
|
spec:
|
||||||
|
containers:
|
||||||
|
- image: domaindrivenarchitecture/c4k-gitea-backup
|
||||||
|
name: backup-app
|
||||||
|
imagePullPolicy: IfNotPresent
|
||||||
|
command: ["/entrypoint-start-and-wait.sh"]
|
||||||
|
env:
|
||||||
|
- name: POSTGRES_USER
|
||||||
|
valueFrom:
|
||||||
|
secretKeyRef:
|
||||||
|
name: postgres-secret
|
||||||
|
key: postgres-user
|
||||||
|
- name: POSTGRES_PASSWORD
|
||||||
|
valueFrom:
|
||||||
|
secretKeyRef:
|
||||||
|
name: postgres-secret
|
||||||
|
key: postgres-password
|
||||||
|
- name: POSTGRES_DB
|
||||||
|
valueFrom:
|
||||||
|
configMapKeyRef:
|
||||||
|
name: postgres-config
|
||||||
|
key: postgres-db
|
||||||
|
- name: POSTGRES_HOST
|
||||||
|
value: "postgresql-service:5432"
|
||||||
|
- name: POSTGRES_SERVICE
|
||||||
|
value: "postgresql-service"
|
||||||
|
- name: POSTGRES_PORT
|
||||||
|
value: "5432"
|
||||||
|
- name: AWS_DEFAULT_REGION
|
||||||
|
value: eu-central-1
|
||||||
|
- name: AWS_ACCESS_KEY_ID_FILE
|
||||||
|
value: /var/run/secrets/backup-secrets/aws-access-key-id
|
||||||
|
- name: AWS_SECRET_ACCESS_KEY_FILE
|
||||||
|
value: /var/run/secrets/backup-secrets/aws-secret-access-key
|
||||||
|
- name: RESTIC_REPOSITORY
|
||||||
|
valueFrom:
|
||||||
|
configMapKeyRef:
|
||||||
|
name: backup-config
|
||||||
|
key: restic-repository
|
||||||
|
- name: RESTIC_PASSWORD_FILE
|
||||||
|
value: /var/run/secrets/backup-secrets/restic-password
|
||||||
|
- name: CERTIFICATE_FILE
|
||||||
|
value: ""
|
||||||
|
volumeMounts:
|
||||||
|
- name: gitea-data-volume
|
||||||
|
mountPath: /var/backups
|
||||||
|
- name: backup-secret-volume
|
||||||
|
mountPath: /var/run/secrets/backup-secrets
|
||||||
|
readOnly: true
|
||||||
|
volumes:
|
||||||
|
- name: gitea-data-volume
|
||||||
|
persistentVolumeClaim:
|
||||||
|
claimName: gitea-data-pvc
|
||||||
|
- name: backup-secret-volume
|
||||||
|
secret:
|
||||||
|
secretName: backup-secret
|
@ -0,0 +1,9 @@
|
|||||||
|
apiVersion: v1
|
||||||
|
kind: ConfigMap
|
||||||
|
metadata:
|
||||||
|
name: backup-config
|
||||||
|
labels:
|
||||||
|
app.kubernetes.io/name: backup
|
||||||
|
app.kubernetes.io/part-of: gitea
|
||||||
|
data:
|
||||||
|
restic-repository: restic-repository
|
@ -0,0 +1,47 @@
|
|||||||
|
apiVersion: batch/v1beta1
|
||||||
|
kind: CronJob
|
||||||
|
metadata:
|
||||||
|
name: gitea-backup
|
||||||
|
labels:
|
||||||
|
app.kubernetes.part-of: gitea
|
||||||
|
spec:
|
||||||
|
schedule: "10 23 * * *"
|
||||||
|
successfulJobsHistoryLimit: 1
|
||||||
|
failedJobsHistoryLimit: 1
|
||||||
|
jobTemplate:
|
||||||
|
spec:
|
||||||
|
template:
|
||||||
|
spec:
|
||||||
|
containers:
|
||||||
|
- name: backup-app
|
||||||
|
image: domaindrivenarchitecture/c4k-gitea-backup
|
||||||
|
imagePullPolicy: IfNotPresent
|
||||||
|
command: ["/entrypoint.sh"]
|
||||||
|
env:
|
||||||
|
- name: AWS_DEFAULT_REGION
|
||||||
|
value: eu-central-1
|
||||||
|
- name: AWS_ACCESS_KEY_ID_FILE
|
||||||
|
value: /var/run/secrets/backup-secrets/aws-access-key-id
|
||||||
|
- name: AWS_SECRET_ACCESS_KEY_FILE
|
||||||
|
value: /var/run/secrets/backup-secrets/aws-secret-access-key
|
||||||
|
- name: RESTIC_REPOSITORY
|
||||||
|
valueFrom:
|
||||||
|
configMapKeyRef:
|
||||||
|
name: backup-config
|
||||||
|
key: restic-repository
|
||||||
|
- name: RESTIC_PASSWORD_FILE
|
||||||
|
value: /var/run/secrets/backup-secrets/restic-password
|
||||||
|
volumeMounts:
|
||||||
|
- name: gitea-data-volume
|
||||||
|
mountPath: /var/backups
|
||||||
|
- name: backup-secret-volume
|
||||||
|
mountPath: /var/run/secrets/backup-secrets
|
||||||
|
readOnly: true
|
||||||
|
volumes:
|
||||||
|
- name: gitea-data-volume
|
||||||
|
persistentVolumeClaim:
|
||||||
|
claimName: gitea-data-pvc
|
||||||
|
- name: backup-secret-volume
|
||||||
|
secret:
|
||||||
|
secretName: backup-secret
|
||||||
|
restartPolicy: OnFailure
|
@ -0,0 +1,9 @@
|
|||||||
|
apiVersion: v1
|
||||||
|
kind: Secret
|
||||||
|
metadata:
|
||||||
|
name: backup-secret
|
||||||
|
type: Opaque
|
||||||
|
data:
|
||||||
|
aws-access-key-id: aws-access-key-id
|
||||||
|
aws-secret-access-key: aws-secret-access-key
|
||||||
|
restic-password: restic-password
|
@ -1,4 +1,7 @@
|
|||||||
{:postgres-db-user "gitea"
|
{:postgres-db-user "gitea"
|
||||||
:postgres-db-password "gitea-db-password"
|
:postgres-db-password "gitea-db-password"
|
||||||
:mailer-user ""
|
:mailer-user ""
|
||||||
:mailer-pw ""}
|
:mailer-pw ""
|
||||||
|
:aws-access-key-id "AWS_KEY_ID"
|
||||||
|
:aws-secret-access-key "AWS_KEY_SECRET"
|
||||||
|
:restic-password ""}
|
||||||
|
@ -1,10 +1,9 @@
|
|||||||
{
|
{:default-app-name "Meissas awesome gitea"
|
||||||
:default-app-name "Meissas awesome gitea"
|
:fqdn "test.de"
|
||||||
:fqdn "test.de"
|
:issuer "staging"
|
||||||
:issuer "staging"
|
:mailer-from "test@test.de"
|
||||||
:mailer-from "test@test.de"
|
:mailer-host-port "test.de:123"
|
||||||
:mailer-host-port "test.de:123"
|
:service-whitelist-domains "test.de"
|
||||||
:service-whitelist-domains "test.de"
|
:service-noreply-address "noreply@test.de"
|
||||||
:service-noreply-address "noreply@test.de"
|
:volume-total-storage-size 6
|
||||||
:volume-total-storage-size 6
|
:restic-repository "repo-path"}
|
||||||
}
|
|
||||||
|
Loading…
Reference in New Issue