Merge branch 'backup' into 'main'

Backup

See merge request domaindrivenarchitecture/c4k-gitea!2
This commit is contained in:
Clemens Geibel 2022-10-07 07:04:08 +00:00
commit fbc3a660c3
23 changed files with 473 additions and 28 deletions

41
doc/BackupAndRestore.md Normal file
View file

@ -0,0 +1,41 @@
# Backup Architecture details
![](backup.svg)
* we use restic to produce small & encrypted backups
* backup is scheduled at `schedule: "10 23 * * *"`
* Gitea stores files in `/data/gitea` and `/data/git/repositories`, these files are backed up.
* The postgres db is also backed up
## Manual init the restic repository for the first time
1. apply backup-and-restore pod:
`kubectl scale deployment backup-restore --replicas=1`
2. exec into pod and execute restore pod (press tab to get your exact pod name)
`kubectl exec -it backup-restore-... -- /usr/local/bin/init.sh`
3. remove backup-and-restore pod:
`kubectl scale deployment backup-restore --replicas=0`
## Manual backup the restic repository for the first time
1. apply backup-and-restore pod:
`kubectl scale deployment backup-restore --replicas=1`
2. exec into pod and execute restore pod (press tab to get your exact pod name)
`kubectl exec -it backup-restore-... -- /usr/local/bin/backup.sh`
3. remove backup-and-restore pod:
`kubectl scale deployment backup-restore --replicas=0`
## Manual restore
1. apply backup-and-restore pod:
`kubectl scale deployment backup-restore --replicas=1`
2. Scale down gitea deployment:
`kubectl scale deployment gitea --replicas=0`
3. exec into pod and execute restore pod (press tab to get your exact pod name)
`kubectl exec -it backup-restore-... -- /usr/local/bin/restore.sh`
4. Start gitea again:
`kubectl scale deployment gitea --replicas=1`
5. remove backup-and-restore pod:
`kubectl scale deployment backup-restore --replicas=0`

View file

@ -0,0 +1,51 @@
from os import environ
from pybuilder.core import task, init
from ddadevops import *
import logging
name = 'c4k-gitea-backup'
MODULE = 'docker'
PROJECT_ROOT_PATH = '../..'
class MyBuild(DevopsDockerBuild):
pass
@init
def initialize(project):
project.build_depends_on('ddadevops>=0.12.4')
stage = 'prod'
dockerhub_user = environ.get('DOCKERHUB_USER')
if not dockerhub_user:
dockerhub_user = gopass_field_from_path('meissa/web/docker.com', 'login')
dockerhub_password = environ.get('DOCKERHUB_PASSWORD')
if not dockerhub_password:
dockerhub_password = gopass_password_from_path('meissa/web/docker.com')
tag = environ.get('CI_COMMIT_TAG')
if not tag:
tag = get_tag_from_latest_commit()
config = create_devops_docker_build_config(
stage, PROJECT_ROOT_PATH, MODULE, dockerhub_user, dockerhub_password, docker_publish_tag=tag)
build = MyBuild(project, config)
build.initialize_build_dir()
@task
def image(project):
build = get_devops_build(project)
build.image()
@task
def drun(project):
build = get_devops_build(project)
build.drun()
@task
def publish(project):
build = get_devops_build(project)
build.dockerhub_login()
build.dockerhub_publish()
@task
def test(project):
build = get_devops_build(project)
build.test()

View file

@ -0,0 +1,5 @@
FROM domaindrivenarchitecture/dda-backup:1.0.6
# Prepare Entrypoint Script
ADD resources /tmp
RUN /tmp/install.sh

View file

@ -0,0 +1,19 @@
#!/bin/bash
set -o pipefail
function main() {
file_env AWS_ACCESS_KEY_ID
file_env AWS_SECRET_ACCESS_KEY
file_env RESTIC_DAYS_TO_KEEP 30
file_env RESTIC_MONTHS_TO_KEEP 12
backup-db-dump
backup-fs-from-directory '/var/backups/' 'gitea/' 'git/repositories/'
}
source /usr/local/lib/functions.sh
source /usr/local/lib/pg-functions.sh
source /usr/local/lib/file-functions.sh
main

View file

@ -0,0 +1,13 @@
#!/bin/bash
function main() {
create-pg-pass
while true; do
sleep 1m
done
}
source /usr/local/lib/functions.sh
source /usr/local/lib/pg-functions.sh
main

View file

@ -0,0 +1,11 @@
#!/bin/bash
function main() {
create-pg-pass
/usr/local/bin/backup.sh
}
source /usr/local/lib/functions.sh
source /usr/local/lib/pg-functions.sh
main

View file

@ -0,0 +1,14 @@
#!/bin/bash
function main() {
file_env AWS_ACCESS_KEY_ID
file_env AWS_SECRET_ACCESS_KEY
init-database-repo
init-file-repo
}
source /usr/local/lib/functions.sh
source /usr/local/lib/pg-functions.sh
source /usr/local/lib/file-functions.sh
main

View file

@ -0,0 +1,11 @@
#!/bin/bash
apt-get update > /dev/null;
install -m 0700 /tmp/entrypoint.sh /
install -m 0700 /tmp/entrypoint-start-and-wait.sh /
install -m 0700 /tmp/init.sh /usr/local/bin/
install -m 0700 /tmp/backup.sh /usr/local/bin/
install -m 0700 /tmp/restore.sh /usr/local/bin/
install -m 0700 /tmp/restic-snapshots.sh /usr/local/bin/

View file

@ -0,0 +1,16 @@
#!/bin/bash
set -o pipefail
function main() {
file_env AWS_ACCESS_KEY_ID
file_env AWS_SECRET_ACCESS_KEY
restic -r ${RESTIC_REPOSITORY}/files snapshots
restic -r ${RESTIC_REPOSITORY}/pg-database snapshots
}
source /usr/local/lib/functions.sh
source /usr/local/lib/file-functions.sh
main

View file

@ -0,0 +1,38 @@
#!/bin/bash
set -Eeo pipefail
function main() {
file_env AWS_ACCESS_KEY_ID
file_env AWS_SECRET_ACCESS_KEY
file_env POSTGRES_DB
file_env POSTGRES_PASSWORD
file_env POSTGRES_USER
# Restore latest snapshot into /var/backups/restore
rm -rf /var/backups/restore
restore-directory '/var/backups/restore'
rm -rf /var/backups/gitea/*
rm -rf /var/backups/git/repositories/*
cp -r /var/backups/restore/gitea/* /var/backups/gitea/
cp -r /var/backups/restore/git/repositories/* /var/backups/git/repositories/
# adjust file permissions for the git user
chown -R 1000:1000 /var/backups
# TODO: Regenerate Git Hooks? Do we need this?
#/usr/local/bin/gitea -c '/data/gitea/conf/app.ini' admin regenerate hooks
# Restore db
drop-create-db
restore-db
}
source /usr/local/lib/functions.sh
source /usr/local/lib/pg-functions.sh
source /usr/local/lib/file-functions.sh
main

View file

@ -0,0 +1,11 @@
FROM c4k-jira-backup
RUN apt update
RUN apt -yqq --no-install-recommends --yes install curl default-jre-headless
RUN curl -L -o /tmp/serverspec.jar \
https://github.com/DomainDrivenArchitecture/dda-serverspec-crate/releases/download/2.0.0/dda-serverspec-standalone.jar
COPY serverspec.edn /tmp/serverspec.edn
RUN java -jar /tmp/serverspec.jar /tmp/serverspec.edn -v

View file

@ -0,0 +1,6 @@
{:file [{:path "/usr/local/bin/init.sh" :mod "700"}
{:path "/usr/local/bin/backup.sh" :mod "700"}
{:path "/usr/local/bin/restore.sh" :mod "700"}
{:path "/usr/local/bin/restic-snapshots.sh" :mod "700"}
{:path "/entrypoint.sh" :mod "700"}
{:path "/entrypoint-start-and-wait.sh" :mod "700"}]}

View file

@ -6,4 +6,4 @@
[dda.c4k-common.uberjar :as uberjar])) [dda.c4k-common.uberjar :as uberjar]))
(defn -main [& cmd-args] (defn -main [& cmd-args]
(uberjar/main-common "c4k-gitea" gitea/config? gitea/auth? gitea/config-defaults core/k8s-objects cmd-args)) (uberjar/main-common "c4k-gitea" core/config? core/auth? core/config-defaults core/k8s-objects cmd-args))

View file

@ -0,0 +1,44 @@
(ns dda.c4k-gitea.backup
(:require
[clojure.spec.alpha :as s]
#?(:cljs [shadow.resource :as rc])
[dda.c4k-common.yaml :as yaml]
[dda.c4k-common.base64 :as b64]
[dda.c4k-common.common :as cm]))
(s/def ::aws-access-key-id cm/bash-env-string?)
(s/def ::aws-secret-access-key cm/bash-env-string?)
(s/def ::restic-password cm/bash-env-string?)
(s/def ::restic-repository cm/bash-env-string?)
#?(:cljs
(defmethod yaml/load-resource :backup [resource-name]
(case resource-name
"backup/config.yaml" (rc/inline "backup/config.yaml")
"backup/cron.yaml" (rc/inline "backup/cron.yaml")
"backup/secret.yaml" (rc/inline "backup/secret.yaml")
"backup/backup-restore-deployment.yaml" (rc/inline "backup/backup-restore-deployment.yaml")
(throw (js/Error. "Undefined Resource!")))))
(defn generate-config [my-conf]
(let [{:keys [restic-repository]} my-conf]
(->
(yaml/from-string (yaml/load-resource "backup/config.yaml"))
(cm/replace-key-value :restic-repository restic-repository))))
(defn generate-cron []
(yaml/from-string (yaml/load-resource "backup/cron.yaml")))
(defn generate-backup-restore-deployment [my-conf]
(let [backup-restore-yaml (yaml/from-string (yaml/load-resource "backup/backup-restore-deployment.yaml"))]
(if (and (contains? my-conf :local-integration-test) (= true (:local-integration-test my-conf)))
(cm/replace-named-value backup-restore-yaml "CERTIFICATE_FILE" "/var/run/secrets/localstack-secrets/ca.crt")
backup-restore-yaml)))
(defn generate-secret [my-auth]
(let [{:keys [aws-access-key-id aws-secret-access-key restic-password]} my-auth]
(->
(yaml/from-string (yaml/load-resource "backup/secret.yaml"))
(cm/replace-key-value :aws-access-key-id (b64/encode aws-access-key-id))
(cm/replace-key-value :aws-secret-access-key (b64/encode aws-secret-access-key))
(cm/replace-key-value :restic-password (b64/encode restic-password)))))

View file

@ -1,15 +1,35 @@
(ns dda.c4k-gitea.core (ns dda.c4k-gitea.core
(:require (:require
[clojure.spec.alpha :as s]
[dda.c4k-common.yaml :as yaml] [dda.c4k-common.yaml :as yaml]
[dda.c4k-common.common :as cm] [dda.c4k-common.common :as cm]
[dda.c4k-gitea.gitea :as gitea] [dda.c4k-gitea.gitea :as gitea]
[dda.c4k-gitea.backup :as backup]
[dda.c4k-common.postgres :as postgres])) [dda.c4k-common.postgres :as postgres]))
(def config-defaults {:issuer "staging"})
(def config? (s/keys :req-un [::gitea/fqdn
::gitea/mailer-from
::gitea/mailer-host-port
::gitea/service-noreply-address]
:opt-un [::gitea/issuer
::gitea/default-app-name
::gitea/service-domain-whitelist
::backup/restic-repository]))
(def auth? (s/keys :req-un [::postgres/postgres-db-user ::postgres/postgres-db-password
::gitea/mailer-user ::gitea/mailer-pw
::backup/aws-access-key-id ::backup/aws-secret-access-key]
:opt-un [::backup/restic-password])) ; TODO gec: Is restic password opt or req?
(def vol? (s/keys :req-un [::gitea/volume-total-storage-size]))
(defn k8s-objects [config] (defn k8s-objects [config]
(let [storage-class (if (contains? config :postgres-data-volume-path) :manual :local-path)] (let [storage-class (if (contains? config :postgres-data-volume-path) :manual :local-path)]
(cm/concat-vec (map yaml/to-string
(map yaml/to-string (filter #(not (nil? %))
(filter #(not (nil? %)) (cm/concat-vec
[(postgres/generate-config {:postgres-size :2gb :db-name "gitea"}) [(postgres/generate-config {:postgres-size :2gb :db-name "gitea"})
(postgres/generate-secret config) (postgres/generate-secret config)
(when (contains? config :postgres-data-volume-path) (when (contains? config :postgres-data-volume-path)
@ -26,4 +46,9 @@
(gitea/generate-appini-env config) (gitea/generate-appini-env config)
(gitea/generate-secrets config) (gitea/generate-secrets config)
(gitea/generate-ingress config) (gitea/generate-ingress config)
(gitea/generate-certificate config)]))))) (gitea/generate-certificate config)]
(when (contains? config :restic-repository)
[(backup/generate-config config)
(backup/generate-secret config)
(backup/generate-cron)
(backup/generate-backup-restore-deployment config)]))))))

View file

@ -66,8 +66,8 @@
(defmethod yaml/load-as-edn :gitea [resource-name] (defmethod yaml/load-as-edn :gitea [resource-name]
(yaml/from-string (yaml/load-resource resource-name)))) (yaml/from-string (yaml/load-resource resource-name))))
(defn-spec generate-appini-env pred/map-or-seq? (defn generate-appini-env
[config config?] [config]
(let [{:keys [default-app-name (let [{:keys [default-app-name
fqdn fqdn
mailer-from mailer-from
@ -87,8 +87,8 @@
(cm/replace-all-matching-values-by-new-value "WHITELISTDOMAINS" service-domain-whitelist) (cm/replace-all-matching-values-by-new-value "WHITELISTDOMAINS" service-domain-whitelist)
(cm/replace-all-matching-values-by-new-value "NOREPLY" service-noreply-address)))) (cm/replace-all-matching-values-by-new-value "NOREPLY" service-noreply-address))))
(defn-spec generate-secrets pred/map-or-seq? (defn generate-secrets
[auth auth?] [auth]
(let [{:keys [postgres-db-user (let [{:keys [postgres-db-user
postgres-db-password postgres-db-password
mailer-user mailer-user
@ -100,15 +100,15 @@
(cm/replace-all-matching-values-by-new-value "MAILERUSER" (b64/encode mailer-user)) (cm/replace-all-matching-values-by-new-value "MAILERUSER" (b64/encode mailer-user))
(cm/replace-all-matching-values-by-new-value "MAILERPW" (b64/encode mailer-pw))))) (cm/replace-all-matching-values-by-new-value "MAILERPW" (b64/encode mailer-pw)))))
(defn-spec generate-ingress pred/map-or-seq? (defn generate-ingress
[config config?] [config]
(let [{:keys [fqdn]} config] (let [{:keys [fqdn]} config]
(-> (->
(yaml/load-as-edn "gitea/ingress.yaml") (yaml/load-as-edn "gitea/ingress.yaml")
(cm/replace-all-matching-values-by-new-value "FQDN" fqdn)))) (cm/replace-all-matching-values-by-new-value "FQDN" fqdn))))
(defn-spec generate-certificate pred/map-or-seq? (defn generate-certificate
[config config?] [config]
(let [{:keys [fqdn issuer] (let [{:keys [fqdn issuer]
:or {issuer "staging"}} config :or {issuer "staging"}} config
letsencrypt-issuer (name issuer)] letsencrypt-issuer (name issuer)]
@ -125,14 +125,14 @@
(yaml/load-as-edn "gitea/datavolume.yaml") (yaml/load-as-edn "gitea/datavolume.yaml")
(cm/replace-all-matching-values-by-new-value "DATASTORAGESIZE" (str (str data-storage-size) "Gi"))))) (cm/replace-all-matching-values-by-new-value "DATASTORAGESIZE" (str (str data-storage-size) "Gi")))))
(defn-spec generate-deployment pred/map-or-seq? (defn generate-deployment
[] []
(yaml/load-as-edn "gitea/deployment.yaml")) (yaml/load-as-edn "gitea/deployment.yaml"))
(defn-spec generate-service pred/map-or-seq? (defn generate-service
[] []
(yaml/load-as-edn "gitea/service.yaml")) (yaml/load-as-edn "gitea/service.yaml"))
(defn-spec generate-service-ssh pred/map-or-seq? (defn generate-service-ssh
[] []
(yaml/load-as-edn "gitea/service-ssh.yaml")) (yaml/load-as-edn "gitea/service-ssh.yaml"))

View file

@ -87,7 +87,7 @@
(br/validate! "app-name" ::gitea/default-app-name :optional true) (br/validate! "app-name" ::gitea/default-app-name :optional true)
(br/validate! "domain-whitelist" ::gitea/service-domain-whitelist :optional true) (br/validate! "domain-whitelist" ::gitea/service-domain-whitelist :optional true)
(br/validate! "volume-total-storage-size" ::gitea/volume-total-storage-size :deserializer js/parseInt) (br/validate! "volume-total-storage-size" ::gitea/volume-total-storage-size :deserializer js/parseInt)
(br/validate! "auth" gitea/auth? :deserializer edn/read-string) (br/validate! "auth" core/auth? :deserializer edn/read-string)
(br/set-form-validated!)) (br/set-form-validated!))
(defn add-validate-listener [name] (defn add-validate-listener [name]

View file

@ -0,0 +1,73 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: backup-restore
spec:
replicas: 0
selector:
matchLabels:
app: backup-restore
strategy:
type: Recreate
template:
metadata:
labels:
app: backup-restore
app.kubernetes.io/name: backup-restore
app.kubernetes.io/part-of: gitea
spec:
containers:
- image: domaindrivenarchitecture/c4k-gitea-backup
name: backup-app
imagePullPolicy: IfNotPresent
command: ["/entrypoint-start-and-wait.sh"]
env:
- name: POSTGRES_USER
valueFrom:
secretKeyRef:
name: postgres-secret
key: postgres-user
- name: POSTGRES_PASSWORD
valueFrom:
secretKeyRef:
name: postgres-secret
key: postgres-password
- name: POSTGRES_DB
valueFrom:
configMapKeyRef:
name: postgres-config
key: postgres-db
- name: POSTGRES_HOST
value: "postgresql-service:5432"
- name: POSTGRES_SERVICE
value: "postgresql-service"
- name: POSTGRES_PORT
value: "5432"
- name: AWS_DEFAULT_REGION
value: eu-central-1
- name: AWS_ACCESS_KEY_ID_FILE
value: /var/run/secrets/backup-secrets/aws-access-key-id
- name: AWS_SECRET_ACCESS_KEY_FILE
value: /var/run/secrets/backup-secrets/aws-secret-access-key
- name: RESTIC_REPOSITORY
valueFrom:
configMapKeyRef:
name: backup-config
key: restic-repository
- name: RESTIC_PASSWORD_FILE
value: /var/run/secrets/backup-secrets/restic-password
- name: CERTIFICATE_FILE
value: ""
volumeMounts:
- name: gitea-data-volume
mountPath: /var/backups
- name: backup-secret-volume
mountPath: /var/run/secrets/backup-secrets
readOnly: true
volumes:
- name: gitea-data-volume
persistentVolumeClaim:
claimName: gitea-data-pvc
- name: backup-secret-volume
secret:
secretName: backup-secret

View file

@ -0,0 +1,9 @@
apiVersion: v1
kind: ConfigMap
metadata:
name: backup-config
labels:
app.kubernetes.io/name: backup
app.kubernetes.io/part-of: gitea
data:
restic-repository: restic-repository

View file

@ -0,0 +1,47 @@
apiVersion: batch/v1beta1
kind: CronJob
metadata:
name: gitea-backup
labels:
app.kubernetes.part-of: gitea
spec:
schedule: "10 23 * * *"
successfulJobsHistoryLimit: 1
failedJobsHistoryLimit: 1
jobTemplate:
spec:
template:
spec:
containers:
- name: backup-app
image: domaindrivenarchitecture/c4k-gitea-backup
imagePullPolicy: IfNotPresent
command: ["/entrypoint.sh"]
env:
- name: AWS_DEFAULT_REGION
value: eu-central-1
- name: AWS_ACCESS_KEY_ID_FILE
value: /var/run/secrets/backup-secrets/aws-access-key-id
- name: AWS_SECRET_ACCESS_KEY_FILE
value: /var/run/secrets/backup-secrets/aws-secret-access-key
- name: RESTIC_REPOSITORY
valueFrom:
configMapKeyRef:
name: backup-config
key: restic-repository
- name: RESTIC_PASSWORD_FILE
value: /var/run/secrets/backup-secrets/restic-password
volumeMounts:
- name: gitea-data-volume
mountPath: /var/backups
- name: backup-secret-volume
mountPath: /var/run/secrets/backup-secrets
readOnly: true
volumes:
- name: gitea-data-volume
persistentVolumeClaim:
claimName: gitea-data-pvc
- name: backup-secret-volume
secret:
secretName: backup-secret
restartPolicy: OnFailure

View file

@ -0,0 +1,9 @@
apiVersion: v1
kind: Secret
metadata:
name: backup-secret
type: Opaque
data:
aws-access-key-id: aws-access-key-id
aws-secret-access-key: aws-secret-access-key
restic-password: restic-password

View file

@ -1,4 +1,7 @@
{:postgres-db-user "gitea" {:postgres-db-user "gitea"
:postgres-db-password "gitea-db-password" :postgres-db-password "gitea-db-password"
:mailer-user "" :mailer-user ""
:mailer-pw ""} :mailer-pw ""
:aws-access-key-id "AWS_KEY_ID"
:aws-secret-access-key "AWS_KEY_SECRET"
:restic-password ""}

View file

@ -1,10 +1,9 @@
{ {:default-app-name "Meissas awesome gitea"
:default-app-name "Meissas awesome gitea" :fqdn "test.de"
:fqdn "test.de" :issuer "staging"
:issuer "staging" :mailer-from "test@test.de"
:mailer-from "test@test.de" :mailer-host-port "test.de:123"
:mailer-host-port "test.de:123" :service-whitelist-domains "test.de"
:service-whitelist-domains "test.de" :service-noreply-address "noreply@test.de"
:service-noreply-address "noreply@test.de" :volume-total-storage-size 6
:volume-total-storage-size 6 :restic-repository "repo-path"}
}