meissa/c4k-nextcloud
6
0
Fork 0
Code Issues Pull requests Projects Releases27 Packages Wiki Activity

implement backup & snapshot

Browse source
This commit is contained in:
Michael Jerger 2024-12-28 18:30:49 +01:00
parent d7ce373d87
commit 67ec0f58eb
19 changed files with 251 additions and 77 deletions

4
infrastructure/backup/image/Dockerfile
View file

@ -2,4 +2,6 @@ FROM domaindrivenarchitecture/dda-backup:latest
# Prepare Entrypoint Script
ADD resources /tmp
RUN /tmp/install.sh
RUN /tmp/install.bb
RUN init.bb
ADD resources2 /tmp

35
infrastructure/backup/image/resources/backup.bb Executable file
View file

@ -0,0 +1,35 @@
#!/usr/bin/env bb
(require
'[babashka.fs :as fs])
(-> "/usr/local/bin/config.clj" fs/file load-file)
(require
'[babashka.tasks :as t]
'[dda.backup.core :as bc]
'[dda.backup.restic :as rc]
'[dda.backup.postgresql :as pg]
'[dda.backup.backup :as bak]
'[config :as cf])
(defn prepare!
[]
(bc/create-aws-credentials! cf/aws-config)
(pg/create-pg-pass! cf/db-config))
(defn restic-repo-init!
[]
(rc/init! cf/file-config)
(rc/init! cf/db-role-config)
(rc/init! cf/db-config))
(defn restic-backup!
[]
(bak/backup-file! cf/file-config)
(bak/backup-db-roles! cf/db-role-config)
(bak/backup-db! cf/db-config))
(t/shell "start-maintenance.sh")
(prepare!)
(restic-repo-init!)
(restic-backup!)
(t/shell "end-maintenance.sh")

28
infrastructure/backup/image/resources/backup.sh
View file

@ -1,28 +0,0 @@
#!/bin/bash
set -Eexo pipefail
function main() {
start-maintenance.sh
file_env AWS_ACCESS_KEY_ID
file_env AWS_SECRET_ACCESS_KEY
file_env POSTGRES_DB
file_env POSTGRES_PASSWORD
file_env POSTGRES_USER
file_env RESTIC_DAYS_TO_KEEP 30
file_env RESTIC_MONTHS_TO_KEEP 12
backup-roles 'oc_'
backup-db-dump
backup-directory '/var/backups/'
end-maintenance.sh
}
source /usr/local/lib/functions.sh
source /usr/local/lib/pg-functions.sh
source /usr/local/lib/file-functions.sh
main

3
infrastructure/backup/image/resources/bb-backup.edn Normal file
View file

@ -0,0 +1,3 @@
{:deps {org.clojure/spec.alpha {:mvn/version "0.4.233"}
orchestra/orchestra {:mvn/version "2021.01.01-1"}
org.domaindrivenarchitecture/dda-backup {:local/root "/usr/local/lib/dda-backup"}}}

3
infrastructure/backup/image/resources/bb.edn Normal file
View file

@ -0,0 +1,3 @@
{:deps {org.clojure/spec.alpha {:mvn/version "0.4.233"}
orchestra/orchestra {:mvn/version "2021.01.01-1"}
org.domaindrivenarchitecture/dda-build {:mvn/version "0.2.0"}}}

34
infrastructure/backup/image/resources/config.clj Normal file
View file

@ -0,0 +1,34 @@
(ns config
(:require
[dda.backup.core :as bc]))
(def restic-repo {:password-file (bc/env-or-file "RESTIC_PASSWORD_FILE")
:restic-repository (bc/env-or-file "RESTIC_REPOSITORY")})
(def file-config (merge restic-repo {:backup-path "files"
:execution-directory "/var/backups"
:restore-target-directory "/var/backups/"
:files ["."]}))
(def file-restore-config (merge restic-repo {:backup-path "files"
:restore-target-directory "/var/backups/"}))
(def db-config (merge restic-repo {:backup-path "pg-database"
:pg-host (bc/env-or-file "POSTGRES_SERVICE")
:pg-port (bc/env-or-file "POSTGRES_PORT")
:pg-db (bc/env-or-file "POSTGRES_DB")
:pg-user (bc/env-or-file "POSTGRES_USER")
:pg-password (bc/env-or-file "POSTGRES_PASSWORD")}))
(def db-role-config (merge restic-repo {:backup-path "pg-role"
:pg-role-prefix "oc_"
:pg-host (bc/env-or-file "POSTGRES_SERVICE")
:pg-port (bc/env-or-file "POSTGRES_PORT")
:pg-db (bc/env-or-file "POSTGRES_DB")
:pg-user (bc/env-or-file "POSTGRES_USER")
:pg-password (bc/env-or-file "POSTGRES_PASSWORD")}))
(def aws-config {:aws-access-key-id (bc/env-or-file "AWS_ACCESS_KEY_ID")
:aws-secret-access-key (bc/env-or-file "AWS_SECRET_ACCESS_KEY")})
(def dry-run {:dry-run true :debug true})

3
infrastructure/backup/image/resources/init.bb Executable file
View file

@ -0,0 +1,3 @@
#!/usr/bin/env bb
(println "initialized")

17
infrastructure/backup/image/resources/init.sh
View file

@ -1,17 +0,0 @@
#!/bin/bash
set -Eexo pipefail
function main() {
file_env AWS_ACCESS_KEY_ID
file_env AWS_SECRET_ACCESS_KEY
init-role-repo
init-database-repo
init-file-repo
}
source /usr/local/lib/functions.sh
source /usr/local/lib/pg-functions.sh
source /usr/local/lib/file-functions.sh
main

23
infrastructure/backup/image/resources/install.bb Executable file
View file

@ -0,0 +1,23 @@
#!/usr/bin/env bb
(require
'[dda.image.ubuntu :as ub]
'[dda.image.install :as in])
(ub/upgrade-system!)
(in/install! "entrypoint.sh")
(in/install! "entrypoint-start-and-wait.sh")
(in/install! "bb-backup.edn" :target-name "bb.edn" :mod "0440")
(in/install! "config.clj" :mod "0440")
(in/install! "init.bb")
(in/install! "backup.bb")
(in/install! "restore.sh")
(in/install! "list-snapshots.bb")
(in/install! "start-maintenance.sh")
(in/install! "end-maintenance.sh")
#(in/install! "restore.bb")
#(in/install! "wait.bb")
(ub/cleanup-container!)

25
infrastructure/backup/image/resources/list-snapshots.bb Executable file
View file

@ -0,0 +1,25 @@
#!/usr/bin/env bb
(require
'[babashka.fs :as fs])
(println (-> "/usr/local/bin/config.clj" fs/file load-file))
(-> "/usr/local/bin/config.clj" fs/file load-file)
(require
'[dda.backup.core :as bc]
'[dda.backup.restic :as rc]
'[config.clj :as cf])
(defn prepare!
[]
(bc/create-aws-credentials! cf/aws-config))
(defn list-snapshots!
[]
(rc/list-snapshots! cf/file-config)
(rc/list-snapshots! cf/db-role-config)
(rc/list-snapshots! cf/db-config))
(prepare!)
(list-snapshots!)

31
infrastructure/backup/image/resources/list-snapshots.sh
View file

@ -1,31 +0,0 @@
#!/bin/bash
set -exo pipefail
function list-snapshot-files() {
if [ -z ${CERTIFICATE_FILE} ];
then
restic -r ${RESTIC_REPOSITORY}/${backup_file_path} snapshots
else
restic -r ${RESTIC_REPOSITORY}/${backup_file_path} snapshots --cacert ${CERTIFICATE_FILE}
fi
}
function main() {
file_env AWS_ACCESS_KEY_ID
file_env AWS_SECRET_ACCESS_KEY
file_env POSTGRES_DB
file_env POSTGRES_PASSWORD
file_env POSTGRES_USER
list-snapshot-roles
list-snapshot-db
list-snapshot-files
}
source /usr/local/lib/functions.sh
source /usr/local/lib/file-functions.sh
source /usr/local/lib/pg-functions.sh
main

3
infrastructure/backup/image/resources2/bb.edn Normal file
View file

@ -0,0 +1,3 @@
{:deps {org.clojure/spec.alpha {:mvn/version "0.4.233"}
orchestra/orchestra {:mvn/version "2021.01.01-1"}
org.domaindrivenarchitecture/dda-backup {:local/root "/usr/local/lib/dda-backup"}}}

10
infrastructure/backup/image/resources2/exports.sh Normal file
View file

@ -0,0 +1,10 @@
export ENV_PASSWORD=env-password
export RESTIC_PASSWORD_FILE=/tmp/file_password
export RESTIC_REPOSITORY=/var/restic-repo
export POSTGRES_SERVICE=dummy
export POSTGRES_PORT=dummy
export POSTGRES_DB=dummy
export POSTGRES_USER=dummy
export POSTGRES_PASSWORD=dummy
export AWS_ACCESS_KEY_ID=dummy
export AWS_SECRET_ACCESS_KEY=dummy

1
infrastructure/backup/image/resources2/file_password Normal file
View file

@ -0,0 +1 @@
oldPassword

52
infrastructure/backup/image/resources2/test.bb Executable file
View file

@ -0,0 +1,52 @@
#!/usr/bin/env bb
(require
'[babashka.fs :as fs])
(-> "/usr/local/bin/config.clj" fs/file load-file)
(require '[babashka.tasks :as tasks]
'[dda.backup.core :as bc]
'[dda.backup.restic :as rc]
'[dda.backup.postgresql :as pg]
'[dda.backup.backup :as bak]
'[dda.backup.restore :as rs]
'[config :as cf])
(defn prepare!
[]
(println (bc/env-or-file "RESTIC_PASSWORD_FILE"))
(println (bc/env-or-file "ENV_PASSWORD"))
(tasks/shell "mkdir" "-p" "/var/backups/")
(tasks/shell "mkdir" "-p" "/var/restic-repo/")
(spit "/var/backups/file" "I was here"))
(defn restic-repo-init!
[]
(rc/init! cf/file-config)
(rc/init! (merge cf/db-config cf/dry-run))
(rc/init! (merge cf/db-role-config cf/dry-run)))
(defn restic-backup!
[]
(bak/backup-file! cf/file-config)
(bak/backup-db-roles! (merge cf/db-role-config cf/dry-run))
(bak/backup-db! (merge cf/db-config cf/dry-run)))
(defn list-snapshots!
[]
(rc/list-snapshots! cf/file-config)
(rc/list-snapshots! (merge cf/db-role-config cf/dry-run))
(rc/list-snapshots! (merge cf/db-config cf/dry-run)))
(defn restic-restore!
[]
(rs/restore-file! (merge {:debug true} cf/file-restore-config))
(pg/drop-create-db! (merge cf/db-config cf/dry-run))
;(rs/restore-db-roles! (merge cf/db-role-config cf/dry-run))
(rs/restore-db! (merge cf/db-config cf/dry-run)))
(prepare!)
(restic-repo-init!)
(restic-backup!)
(list-snapshots!)
(restic-restore!)

4
infrastructure/backup/test/Dockerfile Normal file
View file

@ -0,0 +1,4 @@
FROM c4k-cloud-backup:latest
ADD resources /tmp/
RUN ENV_PASSWORD=env-password RESTIC_PASSWORD_FILE_FILE=/tmp/file_password RESTIC_REPOSITORY=restic-repo POSTGRES_SERVICE=dummy POSTGRES_PORT=dummy POSTGRES_DB=dummy POSTGRES_USER=dummy POSTGRES_PASSWORD=dummy AWS_ACCESS_KEY_ID=dummy AWS_SECRET_ACCESS_KEY=dummy /tmp/test.bb

3
infrastructure/backup/test/resources/bb.edn Normal file
View file

@ -0,0 +1,3 @@
{:deps {org.clojure/spec.alpha {:mvn/version "0.4.233"}
orchestra/orchestra {:mvn/version "2021.01.01-1"}
org.domaindrivenarchitecture/dda-backup {:local/root "/usr/local/lib/dda-backup"}}}

1
infrastructure/backup/test/resources/file_password Normal file
View file

@ -0,0 +1 @@
oldPassword

48
infrastructure/backup/test/resources/test.bb Executable file
View file

@ -0,0 +1,48 @@
#!/usr/bin/env bb
(require
'[babashka.fs :as fs])
(-> "/usr/local/bin/config.clj" fs/file load-file)
(require '[babashka.tasks :as tasks]
'[dda.backup.core :as bc]
'[dda.backup.restic :as rc]
'[dda.backup.postgresql :as pg]
'[dda.backup.backup :as bak]
'[dda.backup.restore :as rs]
'[config :as cf])
(defn prepare!
[]
(println (bc/env-or-file "RESTIC_PASSWORD_FILE"))
(println (bc/env-or-file "ENV_PASSWORD"))
(tasks/shell "mkdir" "-p" "/var/backups/")
(spit "/var/backups/file" "I was here")
(tasks/shell "mkdir" "-p" "/var/restore"))
(defn restic-repo-init!
[]
(rc/init! (merge cf/file-config cf/dry-run))
(rc/init! (merge cf/db-config cf/dry-run)))
(defn restic-backup!
[]
(bak/backup-file! cf/file-config)
(bak/backup-db! (merge cf/db-config cf/dry-run)))
(defn list-snapshots!
[]
(rc/list-snapshots! cf/file-config)
(rc/list-snapshots! (merge cf/db-config cf/dry-run)))
(defn restic-restore!
[]
(rs/restore-file! cf/file-config)
(pg/drop-create-db! (merge cf/db-config cf/dry-run))
(rs/restore-db! (merge cf/db-config cf/dry-run)))
(prepare!)
(restic-repo-init!)
#(restic-backup!)
#(list-snapshots!)
#(restic-restore!)