Compare commits
1 commit
main
...
remove-the
| Author | SHA1 | Date | |
|---|---|---|---|
| a55fead350 |
41 changed files with 255 additions and 853 deletions
.gitlab-ci.ymlMakefileREADME.mddeps.edn
docs
infrastructure/backup
src/dda/backup
backup.clj
backup
config.cljcore.cljcore
infrastructure.cljmonitoring.cljmonitoring
postgresql.cljpostgresql
restic.cljrestic
restore.cljrestore
test/dda/backup
tests.edn|
|
@ -5,7 +5,7 @@ stages:
|
|||
- image
|
||||
|
||||
.img: &img
|
||||
image: "domaindrivenarchitecture/ddadevops-dind:4.14.0"
|
||||
image: "domaindrivenarchitecture/ddadevops-dind:4.13.0"
|
||||
services:
|
||||
- docker:dind
|
||||
before_script:
|
||||
|
|
@ -15,15 +15,14 @@ stages:
|
|||
- export IMAGE_TAG=$CI_COMMIT_TAG
|
||||
|
||||
.clj-job: &clj
|
||||
image: "domaindrivenarchitecture/ddadevops-clj:4.14.0"
|
||||
image: "domaindrivenarchitecture/ddadevops-clj:4.13.0"
|
||||
cache:
|
||||
key: ${CI_COMMIT_REF_SLUG}
|
||||
paths:
|
||||
- .m2
|
||||
before_script:
|
||||
- export RELEASE_ARTIFACT_TOKEN=$MEISSA_REPO_BUERO_RW
|
||||
- export CLOJARS_USERNAME=$CLOJARS_USER
|
||||
- export CLOJARS_PASSWORD=$CLOJARS_TOKEN_DOMAINDRIVENARCHITECTURE
|
||||
- mkdir -p /root/.lein
|
||||
- echo "{:auth {:repository-auth {#\"clojars\" {:username \"${CLOJARS_USER}\" :password \"${CLOJARS_TOKEN_DOMAINDRIVENARCHITECTURE}\" }}}}" > ~/.lein/profiles.clj
|
||||
|
||||
.tag_only: &tag_only
|
||||
|
|
|
|||
1
Makefile
1
Makefile
|
|
@ -176,7 +176,6 @@ swagger-editor-down: ## Stop Swagger Editor in Docker
|
|||
# https://makefiletutorial.com/#delete_on_error
|
||||
|
||||
# TODO: focus runner on ^:integration` tests
|
||||
# This is a pending decision about continue drop bb in favor of rust
|
||||
test-ci: deps ## Test runner for integration tests
|
||||
$(info --------- Runner for integration tests ---------)
|
||||
clojure -P -X:test/env:test/run
|
||||
|
|
|
|||
86
README.md
86
README.md
|
|
@ -4,91 +4,9 @@
|
|||
|
||||
[<img src="https://domaindrivenarchitecture.org/img/delta-chat.svg" width=20 alt="DeltaChat"> chat over e-mail](mailto:buero@meissa-gmbh.de?subject=community-chat) | [<img src="https://meissa.de/images/parts/contact/mastodon36_hue9b2464f10b18e134322af482b9c915e_5501_filter_14705073121015236177.png" width=20 alt="M"> meissa@social.meissa-gmbh.de](https://social.meissa-gmbh.de/@meissa) | [Blog](https://domaindrivenarchitecture.org) | [Website](https://meissa.de)
|
||||
|
||||
|
||||
Images are available at https://hub.docker.com/r/domaindrivenarchitecture/dda-backup
|
||||
|
||||
## Usage example
|
||||
|
||||
```clojure
|
||||
#!/usr/bin/env bb
|
||||
|
||||
(require '[babashka.tasks :as tasks]
|
||||
'[dda.backup.restic :as rc]
|
||||
'[dda.backup.backup :as bak]
|
||||
'[dda.backup.restore :as rs])
|
||||
|
||||
(def config {:restic-repo
|
||||
:debug true
|
||||
:restic-repository "/restic-repo"
|
||||
:password-file "/tmp/file_password"
|
||||
:new-password-file "/tmp/file_new_password"
|
||||
:backup-path "files"
|
||||
:execution-directory "/var/backups/"
|
||||
:files ["test-backup"]
|
||||
:restore-target-directory "test-restore"})
|
||||
|
||||
(defn prepare!
|
||||
[]
|
||||
(tasks/shell "mkdir" "-p" "/var/backups/test-backup")
|
||||
(spit "/var/backups/test-backup/file" "I was here")
|
||||
(tasks/shell "mkdir" "-p" "/var/backups/test-restore"))
|
||||
|
||||
(defn restic-repo-init!
|
||||
[]
|
||||
(println "\nrestic-repo-init!")
|
||||
(rc/init! config))
|
||||
|
||||
(defn restic-backup!
|
||||
[]
|
||||
(println "\nrestic-backup!")
|
||||
(bak/backup-file! config))
|
||||
|
||||
(defn list-snapshots!
|
||||
[]
|
||||
(println "\nlist-snapshots!")
|
||||
(rc/list-snapshots! config))
|
||||
|
||||
(defn restic-restore!
|
||||
[]
|
||||
(println "\nrestic-restore!")
|
||||
(rs/restore-file! config))
|
||||
|
||||
(defn change-password!
|
||||
[]
|
||||
(println "\nchange-password!")
|
||||
(rc/change-password! config))
|
||||
|
||||
(defn restic-backup-with-new!
|
||||
[]
|
||||
(println "\nrestic-backup with new!")
|
||||
(bak/backup-file! (merge {:password-file "/tmp/file_new_password"} config)))
|
||||
|
||||
(defn list-snapshots-with-new!
|
||||
[]
|
||||
(println "\nlist-snapshots with new!")
|
||||
(rc/list-snapshots! (merge {:password-file "/tmp/file_new_password"} config)))
|
||||
|
||||
|
||||
(prepare!)
|
||||
(restic-repo-init!)
|
||||
(restic-backup!)
|
||||
(list-snapshots!)
|
||||
(restic-restore!)
|
||||
(change-password!)
|
||||
(restic-backup!)
|
||||
(list-snapshots!)
|
||||
(restic-restore!)
|
||||
(restic-backup-with-new!)
|
||||
(list-snapshots-with-new!)
|
||||
|
||||
```
|
||||
|
||||
## Real Life Examples
|
||||
|
||||
see:
|
||||
|
||||
* https://repo.prod.meissa.de/meissa/c4k-taiga/src/branch/main/infrastructure/backup/image/resources
|
||||
* https://repo.prod.meissa.de/meissa/c4k-forgejo/src/branch/main/infrastructure/backup/image/resources
|
||||
|
||||
## Development & mirrors
|
||||
|
||||
Development happens at: https://repo.prod.meissa.de/meissa/dda-backup
|
||||
|
|
@ -101,6 +19,6 @@ Mirrors are:
|
|||
For more details about our repository model see: https://repo.prod.meissa.de/meissa/federate-your-repos
|
||||
|
||||
## License
|
||||
Copyright © 2024, 2025 meissa GmbH
|
||||
Copyright © 2024 meissa GmbH
|
||||
Published under [apache2.0 license](LICENSE.md)
|
||||
Pls. find licenses of our subcomponents [here](doc/SUBCOMPONENT_LICENSE)
|
||||
|
|
|
|||
18
deps.edn
18
deps.edn
|
|
@ -1,5 +1,5 @@
|
|||
{:project {:name org.domaindrivenarchitecture/dda-backup
|
||||
:version "5.4.1-SNAPSHOT"}
|
||||
:version "5.0.0-SNAPSHOT"}
|
||||
|
||||
;; ---------------------------------------------------------
|
||||
:paths
|
||||
|
|
@ -9,27 +9,21 @@
|
|||
;; ---------------------------------------------------------
|
||||
:deps
|
||||
{;; Application
|
||||
org.clojure/clojure {:mvn/version "1.11.4"}
|
||||
org.clojure/spec.alpha {:mvn/version "0.5.238"}
|
||||
orchestra/orchestra {:mvn/version "2021.01.01-1"}
|
||||
aero/aero {:mvn/version "1.1.6"}
|
||||
cheshire/cheshire {:mvn/version "5.13.0"}
|
||||
com.widdindustries/cljc.java-time {:mvn/version "0.1.21"}
|
||||
org.babashka/http-client {:mvn/version "0.3.11"}}
|
||||
orchestra/orchestra {:mvn/version "2021.01.01-1"}}
|
||||
;; ---------------------------------------------------------
|
||||
|
||||
;; ---------------------------------------------------------
|
||||
:aliases
|
||||
{
|
||||
;; ------------
|
||||
;; Add libraries and paths to support additional test tools
|
||||
:test/env
|
||||
{}
|
||||
|
||||
|
||||
;; Test runner - local and CI
|
||||
;; call with :watch? true to start file watcher and re-run tests on saved changes
|
||||
:test/run
|
||||
{:extra-paths ["test"]
|
||||
:extra-deps {lambdaisland/kaocha {:mvn/version "1.91.1392"}}
|
||||
:extra-deps {lambdaisland/kaocha {:mvn/version "1.91.1392"}
|
||||
expound/expound {:mvn/version "0.9.0"}}
|
||||
:main-opts ["-m" "kaocha.runner"]
|
||||
:exec-fn kaocha.runner/exec-fn
|
||||
:exec-args {:randomize? false
|
||||
|
|
|
|||
132
docs/Backup.md
132
docs/Backup.md
|
|
@ -1,132 +0,0 @@
|
|||
# Application Backup for k8s
|
||||
|
||||
# Runtime View
|
||||
|
||||
```mermaid
|
||||
C4Context
|
||||
title Backup
|
||||
Enterprise_Boundary(b0, "Infrastructure") {
|
||||
|
||||
Container_Boundary(k3s, "K3S") {
|
||||
Container(wl, "workload")
|
||||
Container(br, "backup", "cron job")
|
||||
SystemDb(db, "postgres database")
|
||||
SystemDb(pv, "persistent volume")
|
||||
|
||||
Rel(br, wl, "stop/start")
|
||||
Rel(br, db, "read")
|
||||
Rel(br, pv, "read")
|
||||
}
|
||||
|
||||
|
||||
System(s3, "S3", "")
|
||||
|
||||
Rel(br, s3, "backup & manage retention")
|
||||
}
|
||||
|
||||
UpdateLayoutConfig($c4ShapeInRow="2", $c4BoundaryInRow="1")
|
||||
```
|
||||
|
||||
```mermaid
|
||||
C4Context
|
||||
title Restore
|
||||
Enterprise_Boundary(b0, "Infrastructure") {
|
||||
|
||||
Container_Boundary(k3s, "K3S") {
|
||||
Container(wl, "workload")
|
||||
Container(br, "backup-restore", "deployment")
|
||||
SystemDb(db, "postgres database")
|
||||
SystemDb(pv, "persistent volume")
|
||||
|
||||
Rel(br, db, "drop & import")
|
||||
Rel(br, pv, "delete & write")
|
||||
}
|
||||
|
||||
|
||||
System(s3, "S3", "")
|
||||
|
||||
Rel(br, s3, "restore")
|
||||
}
|
||||
|
||||
UpdateLayoutConfig($c4ShapeInRow="2", $c4BoundaryInRow="1")
|
||||
```
|
||||
|
||||
* we use restic to produce small & encrypted backups
|
||||
* backups are stored in s3
|
||||
* backup is scheduled at `schedule: "10 23 * * *"`
|
||||
* backup supports file or postgres-db-dump backup.
|
||||
* supported restic usecases are backup, restore, change password, list snapshots, support retentions
|
||||
|
||||
## Parameter determined by k8s-application
|
||||
|
||||
1. name of application deployments **deployment-name** to scale down during backup / restore
|
||||
2. namespace of application deployments **deployment-namespace**
|
||||
|
||||
## backup / restore
|
||||
|
||||
### Scale application down
|
||||
|
||||
1. Scale [deployment-name] deployment down:
|
||||
`kubectl -n [deployment-namespace] scale deployment [deployment-name] --replicas=0`
|
||||
|
||||
### Manual backup
|
||||
|
||||
1. Scale backup-restore deployment up:
|
||||
`kubectl -n [deployment-namespace] scale deployment backup-restore --replicas=1`
|
||||
2. exec into pod and execute restore pod
|
||||
`kubectl -n [deployment-namespace] exec -it backup-restore -- backup.bb`
|
||||
3. Scale backup-restore deployment down:
|
||||
`kubectl -n [deployment-namespace] scale deployment backup-restore --replicas=0`
|
||||
|
||||
### Manual restore
|
||||
|
||||
1. Scale backup-restore deployment up:
|
||||
`kubectl -n [deployment-namespace] scale deployment backup-restore --replicas=1`
|
||||
3. exec into pod and execute restore pod
|
||||
`kubectl -n [deployment-namespace] exec -it backup-restore -- restore.bb`
|
||||
4. Scale backup-restore deployment down:
|
||||
`kubectl -n [deployment-namespace] scale deployment backup-restore --replicas=0`
|
||||
|
||||
### Scale application up
|
||||
|
||||
1. Scale [deployment-name] deployment up:
|
||||
`kubectl -n [deployment-namespace] scale deployment [deployment-name] --replicas=1`
|
||||
|
||||
## Change Password
|
||||
|
||||
1. Check restic-new-password env is set in backup deployment
|
||||
```
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: backup-restore
|
||||
spec:
|
||||
spec:
|
||||
containers:
|
||||
- name: backup-app
|
||||
env:
|
||||
- name: RESTIC_NEW_PASSWORD_FILE
|
||||
value: /var/run/secrets/backup-secrets/restic-new-password
|
||||
```
|
||||
2. Add restic-new-password to secret
|
||||
```
|
||||
kind: Secret
|
||||
metadata:
|
||||
name: backup-secret
|
||||
data:
|
||||
restic-password: old
|
||||
restic-new-password: new
|
||||
```
|
||||
3. Scale backup-restore deployment up:
|
||||
`kubectl -n [deployment-namespace] scale deployment backup-restore --replicas=1`
|
||||
4. exec into pod and execute restore pod
|
||||
`kubectl -n [deployment-namespace] exec -it backup-restore -- change-password.bb`
|
||||
5. Scale backup-restore deployment down:
|
||||
`kubectl -n [deployment-namespace] scale deployment backup-restore --replicas=0`
|
||||
6. Replace restic-password with restic-new-password in secret
|
||||
```
|
||||
kind: Secret
|
||||
metadata:
|
||||
name: backup-secret
|
||||
data:
|
||||
restic-password: new
|
||||
```
|
||||
|
|
@ -1,3 +0,0 @@
|
|||
# Setup
|
||||
|
||||
See: https://clojure.org/guides/install_clojure#_linux_instructions
|
||||
|
|
@ -8,7 +8,7 @@ import logging
|
|||
name = 'dda-backup'
|
||||
MODULE = 'NOT_SET'
|
||||
PROJECT_ROOT_PATH = '../..'
|
||||
version = "5.4.1-dev"
|
||||
version = "5.0.0-dev"
|
||||
|
||||
|
||||
@init
|
||||
|
|
|
|||
|
|
@ -1,50 +1,79 @@
|
|||
# changing password on restic repository
|
||||
## Init Statemachine
|
||||
|
||||
## config to use
|
||||
### Inputs
|
||||
1. `restic-password: ""`
|
||||
2. `restic-password-to-rotate: ""`
|
||||
|
||||
To change the password add new-password-file to config.
|
||||
### Manual init the restic repository for the first time
|
||||
|
||||
```clojure
|
||||
{:password-file "/restic-pwd"
|
||||
:new-password-file "/new-restic-pwd"}
|
||||
```
|
||||
1. apply backup-and-restore pod:
|
||||
`kubectl scale deployment backup-restore --replicas=1`
|
||||
2. exec into pod and execute restore pod (press tab to get your exact pod name)
|
||||
`kubectl exec -it backup-restore-... -- /usr/local/bin/init.sh`
|
||||
3. remove backup-and-restore pod:
|
||||
`kubectl scale deployment backup-restore --replicas=0`
|
||||
|
||||
## restic: decide which password to choose
|
||||
### Password Rotation
|
||||
|
||||
1. apply backup-and-restore pod:
|
||||
`kubectl scale deployment backup-restore --replicas=1`
|
||||
2. add new password to restic repository
|
||||
`restic key add ....`
|
||||
=> Trigger ::
|
||||
field (1) credential current
|
||||
filed (2) credential new
|
||||
3. replace field (1) with (2) & clear (2)
|
||||
4. remove old key - ???
|
||||
`restic remove ....`
|
||||
|
||||
If there is a new-password-file defined, decide witch to use:
|
||||
|
||||
```mermaid
|
||||
stateDiagram-v2
|
||||
state new? <<choice>>
|
||||
state "restic --password-file /new-pwd check" as check_new
|
||||
state new_works? <<choice>>
|
||||
|
||||
[*] --> containsNewPassword?
|
||||
containsNewPassword? --> new?
|
||||
new? --> check_new: yes
|
||||
new? --> use_old: no
|
||||
check_new --> new_works?
|
||||
new_works? --> use_new: yes
|
||||
new_works? --> use_old: no
|
||||
[*] --> init
|
||||
init --> backup_ready: trigger, restic-password !empty
|
||||
backup_ready --> new_password_added: restic-password !empty && restic-password-to-rotate !empty
|
||||
new_password_added --> backup_ready: restic-password !empty && restic-password-to-rotate empty
|
||||
```
|
||||
|
||||
# Process to change password in k8s
|
||||
### First Steps
|
||||
|
||||
1. Cloud Testserver hochfahren
|
||||
2. Dort backup-restore deployment (leeres Secret mgl.?), neues Secret "rotation-credential-secret" als Daten
|
||||
3. mounten von angelegtem Secret in Pod backup-restore
|
||||
4. ba*bash*ka Skript in pod starten -> liest Secret ?leer
|
||||
5. Micha cons.
|
||||
|
||||
```mermaid
|
||||
sequenceDiagram
|
||||
actor d as DevOps
|
||||
participant s as BackupSecret
|
||||
participant a as ApplicationDeployment
|
||||
participant b as BackupDeployment
|
||||
|
||||
d ->> s: add new-password-file
|
||||
d ->> a: scale down
|
||||
d ->> b: scale up
|
||||
d ->> b: shell into
|
||||
activate b
|
||||
b ->> b: call change password
|
||||
deactivate b
|
||||
d ->> s: replace password-file with new-password-file
|
||||
d ->> b: scale down
|
||||
d ->> a: scale up
|
||||
```
|
||||
participant k8s
|
||||
participant e as entrypoint.sh
|
||||
participant rm as restic-management.clj
|
||||
|
||||
k8s ->> e: cronjob calls
|
||||
e ->> rm: start-file
|
||||
rm ->> rm: rotate
|
||||
activate rm
|
||||
rm ->> rm: read-backup-repository-state (state)
|
||||
rm ->> rm: read-secret (backup-secret/restic-password, rotation-credential-secret/rotation-credential)
|
||||
rm ->> rm: switch
|
||||
activate rm
|
||||
rm ->> rm: if init && restic-password != null
|
||||
activate rm
|
||||
rm ->> rm: init.sh
|
||||
rm ->> rm: state init -> backup-ready
|
||||
deactivate rm
|
||||
rm ->> rm: if backup-ready && rotation-credential != null
|
||||
activate rm
|
||||
rm ->> rm: add-new-password-to-restic-repository.sh
|
||||
rm ->> rm: state backup-ready -> new-password-added
|
||||
deactivate rm
|
||||
rm ->> rm: if new-password-added && rotation-credential == null
|
||||
activate rm
|
||||
rm ->> rm: remove-old-password-from-restic-repository.sh
|
||||
rm ->> rm: state new-password-added -> backup-ready
|
||||
deactivate rm
|
||||
deactivate rm
|
||||
|
||||
rm ->> rm: store-repository-state (state)
|
||||
deactivate rm
|
||||
```
|
||||
|
|
@ -5,6 +5,3 @@ ADD resources /tmp/
|
|||
RUN /tmp/install.sh
|
||||
ADD local/ /usr/local/lib/dda-backup
|
||||
RUN init-bb.bb
|
||||
#ADD resources2 /tmp/
|
||||
#RUN install -m 0700 -o root -g root /tmp/test.bb /usr/local/bin/
|
||||
#RUN FILE_PASSWORD_FILE=/tmp/file_password test.bb
|
||||
|
|
|
|||
|
|
@ -18,12 +18,12 @@ function main() {
|
|||
apt-get install -qqy ca-certificates curl gnupg postgresql-client-16 restic openjdk-21-jre-headless nano
|
||||
curl -Ss --fail https://www.postgresql.org/media/keys/ACCC4CF8.asc | gpg --dearmor | tee /etc/apt/trusted.gpg.d/postgresql-common_pgdg_archive_keyring.gpg
|
||||
sh -c 'echo "deb [signed-by=/etc/apt/trusted.gpg.d/postgresql-common_pgdg_archive_keyring.gpg] https://apt.postgresql.org/pub/repos/apt jammy-pgdg main" > /etc/apt/sources.list.d/pgdg.list'
|
||||
upgradeSystem
|
||||
babashka_install
|
||||
} > /dev/null
|
||||
|
||||
update-ca-certificates
|
||||
install -m 0700 -o root -g root /tmp/init-bb.bb /usr/local/bin/
|
||||
install -m 0600 -o root -g root /tmp/bb.edn /usr/local/bin/
|
||||
cleanupDocker
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -1,24 +0,0 @@
|
|||
{:restic-repo {:password-file #env-or-file "FILE_PASSWORD_FILE"
|
||||
:restic-repository "/restic-repo"
|
||||
:debug true}
|
||||
:file-config #merge [#ref [:restic-repo]
|
||||
{:new-password-file "/tmp/file_new_password"
|
||||
:backup-path "files"
|
||||
:execution-directory "/var/backups/"
|
||||
:files ["test-backup"]
|
||||
:restore-target-directory "test-restore"}]
|
||||
:file-config-with-new #merge [#ref [:file-config]
|
||||
{:password-file "/tmp/file_new_password"}]
|
||||
:db-config #merge [#ref [:restic-repo]
|
||||
{:new-password-file "/tmp/file_new_password"
|
||||
:backup-path "db"
|
||||
:pg-db "mydb"
|
||||
:pg-user "user"
|
||||
:pg-password "password"}]
|
||||
:db-roles-config #merge [#ref [:restic-repo]
|
||||
{:new-password-file "/tmp/file_new_password"
|
||||
:backup-path "db-roles"
|
||||
:pg-db "mydb"
|
||||
:pg-user "user"
|
||||
:pg-password "password"}]
|
||||
:dry-run {:dry-run true :debug true}}
|
||||
|
|
@ -1 +0,0 @@
|
|||
new
|
||||
|
|
@ -1 +0,0 @@
|
|||
old
|
||||
|
|
@ -1,75 +0,0 @@
|
|||
#!/usr/bin/env bb
|
||||
|
||||
(require '[babashka.tasks :as tasks]
|
||||
'[dda.backup.config :as cfg]
|
||||
'[dda.backup.restic :as rc]
|
||||
'[dda.backup.postgresql :as pg]
|
||||
'[dda.backup.backup :as bak]
|
||||
'[dda.backup.restore :as rs])
|
||||
|
||||
(def config (cfg/read-config "/tmp/config.edn"))
|
||||
|
||||
(defn prepare!
|
||||
[]
|
||||
(println config)
|
||||
(tasks/shell "mkdir" "-p" "/var/backups/test-backup")
|
||||
(spit "/var/backups/test-backup/file" "I was here")
|
||||
(tasks/shell "mkdir" "-p" "/var/backups/test-restore")
|
||||
(pg/create-pg-pass! (:db-config config)))
|
||||
|
||||
(defn restic-repo-init!
|
||||
[]
|
||||
(println "\nrestic-repo-init!")
|
||||
(rc/init! (:file-config config))
|
||||
(rc/init! (merge (:db-config config) (:dry-run config)))
|
||||
(rc/init! (merge (:db-roles-config config) (:dry-run config))))
|
||||
|
||||
(defn restic-backup!
|
||||
[]
|
||||
(println "\nrestic-backup!")
|
||||
(bak/backup-file! (:file-config config))
|
||||
(bak/backup-db-roles! (merge (:db-roles-config config) (:dry-run config)))
|
||||
(bak/backup-db! (merge (:db-config config) (:dry-run config))))
|
||||
|
||||
(defn list-snapshots!
|
||||
[]
|
||||
(println "\nlist-snapshots!")
|
||||
(rc/list-snapshots! (:file-config config))
|
||||
(rc/list-snapshots! (merge (:db-roles-config config) (:dry-run config)))
|
||||
(rc/list-snapshots! (merge (:db-config config) (:dry-run config))))
|
||||
|
||||
(defn restic-restore!
|
||||
[]
|
||||
(println "\nrestic-restore!")
|
||||
(rs/restore-file! (:file-config config))
|
||||
(pg/drop-create-db! (merge (:db-config config) (:dry-run config)))
|
||||
(rs/restore-db-roles! (merge (:db-roles-config config) (:dry-run config)))
|
||||
(rs/restore-db! (merge (:db-config config) (:dry-run config))))
|
||||
|
||||
(defn change-password!
|
||||
[]
|
||||
(println "\nchange-password!")
|
||||
(rc/change-password! (:file-config config)))
|
||||
|
||||
(defn restic-backup-with-new!
|
||||
[]
|
||||
(println "\nrestic-backup with new!")
|
||||
(bak/backup-file! (:file-config-with-new config)))
|
||||
|
||||
(defn list-snapshots-with-new!
|
||||
[]
|
||||
(println "\nlist-snapshots with new!")
|
||||
(rc/list-snapshots! (:file-config-with-new config)))
|
||||
|
||||
|
||||
(prepare!)
|
||||
(restic-repo-init!)
|
||||
(restic-backup!)
|
||||
(list-snapshots!)
|
||||
(restic-restore!)
|
||||
(change-password!)
|
||||
(restic-backup!)
|
||||
(list-snapshots!)
|
||||
(restic-restore!)
|
||||
(restic-backup-with-new!)
|
||||
(list-snapshots-with-new!)
|
||||
|
|
@ -1,5 +1,6 @@
|
|||
FROM dda-backup:latest
|
||||
|
||||
# install it
|
||||
#ADD local/ /usr/local/lib/dda-backup
|
||||
ADD resources /tmp/
|
||||
RUN install -m 0700 -o root -g root /tmp/test.bb /usr/local/bin/
|
||||
RUN FILE_PASSWORD_FILE=/tmp/file_password test.bb
|
||||
RUN ENV_PASSWORD=env-password FILE_PASSWORD_FILE=/tmp/file_password /tmp/test.bb
|
||||
|
|
|
|||
4
infrastructure/backup/test/resources/bb.edn
Normal file
4
infrastructure/backup/test/resources/bb.edn
Normal file
|
|
@ -0,0 +1,4 @@
|
|||
{:deps {org.babashka/spec.alpha {:git/url "https://github.com/babashka/spec.alpha"
|
||||
:git/sha "1a841c4cc1d4f6dab7505a98ed2d532dd9d56b78"}
|
||||
orchestra/orchestra {:mvn/version "2021.01.01-1"}
|
||||
org.domaindrivenarchitecture/dda-backup {:local/root "/usr/local/lib/dda-backup"}}}
|
||||
|
|
@ -1,24 +0,0 @@
|
|||
{:restic-repo {:password-file #env-or-file "FILE_PASSWORD_FILE"
|
||||
:restic-repository "/restic-repo"
|
||||
:debug true}
|
||||
:file-config #merge [#ref [:restic-repo]
|
||||
{:new-password-file "/tmp/file_new_password"
|
||||
:backup-path "files"
|
||||
:execution-directory "/var/backups/"
|
||||
:files ["test-backup"]
|
||||
:restore-target-directory "test-restore"}]
|
||||
:file-config-with-new #merge [#ref [:file-config]
|
||||
{:password-file "/tmp/file_new_password"}]
|
||||
:db-config #merge [#ref [:restic-repo]
|
||||
{:new-password-file "/tmp/file_new_password"
|
||||
:backup-path "db"
|
||||
:pg-db "mydb"
|
||||
:pg-user "user"
|
||||
:pg-password "password"}]
|
||||
:db-roles-config #merge [#ref [:restic-repo]
|
||||
{:new-password-file "/tmp/file_new_password"
|
||||
:backup-path "db-roles"
|
||||
:pg-db "mydb"
|
||||
:pg-user "user"
|
||||
:pg-password "password"}]
|
||||
:dry-run {:dry-run true :debug true}}
|
||||
|
|
@ -1 +0,0 @@
|
|||
new
|
||||
|
|
@ -1 +0,0 @@
|
|||
old
|
||||
29
infrastructure/backup/test/resources/install-test.bb
Executable file
29
infrastructure/backup/test/resources/install-test.bb
Executable file
|
|
@ -0,0 +1,29 @@
|
|||
#!/usr/bin/env bb
|
||||
|
||||
(require '[babashka.tasks :as tasks])
|
||||
|
||||
(defn curl-and-check!
|
||||
[filename artifact-url sha256-url]
|
||||
(let [filepath (str "/tmp/" filename)]
|
||||
(tasks/shell "curl" "-SsLo" filepath artifact-url)
|
||||
(tasks/shell "curl" "-SsLo" "/tmp/checksum" sha256-url)
|
||||
(tasks/shell "bash" "-c" (str "echo \" " filepath "\"|tee -a /tmp/checksum"))
|
||||
;(tasks/shell "sha256sum" "-c" "--status" "/tmp/checksum")
|
||||
))
|
||||
|
||||
(defn tar-install!
|
||||
[filename binname]
|
||||
(let [filepath (str "/tmp/" filename)]
|
||||
(tasks/shell "tar" "-C" "/tmp" "-xzf" filepath)
|
||||
(tasks/shell "install" "-m" "0700" "-o" "root" "-g" "root" (str "/tmp/" binname) "/usr/local/bin/")))
|
||||
|
||||
(defn install!
|
||||
[filename]
|
||||
(tasks/shell "install" "-m" "0700" "-o" "root" "-g" "root" (str "/tmp/" filename) "/usr/local/bin/"))
|
||||
|
||||
(curl-and-check!
|
||||
"provs-syspec.jar"
|
||||
"https://repo.prod.meissa.de/attachments/0a1da41e-aa5b-4a3e-a3b1-215cf2d5b021"
|
||||
"https://repo.prod.meissa.de/attachments/f227cf65-cb0f-46a7-a6cd-28f46917412a")
|
||||
(install! "provs-syspec.jar")
|
||||
(tasks/shell "java" "-jar" "/usr/local/bin/provs-syspec.jar" "local" "-c" "/tmp/spec.yml" )
|
||||
7
infrastructure/backup/test/resources/spec.yml
Normal file
7
infrastructure/backup/test/resources/spec.yml
Normal file
|
|
@ -0,0 +1,7 @@
|
|||
package:
|
||||
- name: "restic"
|
||||
|
||||
command:
|
||||
- command: "bb -h"
|
||||
- command: "/tmp/test.bb"
|
||||
|
||||
|
|
@ -1,75 +1,62 @@
|
|||
#!/usr/bin/env bb
|
||||
|
||||
(require '[babashka.tasks :as tasks]
|
||||
'[dda.backup.config :as cfg]
|
||||
'[dda.backup.core :as bc]
|
||||
'[dda.backup.restic :as rc]
|
||||
'[dda.backup.postgresql :as pg]
|
||||
'[dda.backup.backup :as bak]
|
||||
'[dda.backup.restore :as rs])
|
||||
|
||||
(def config (cfg/read-config "/tmp/config.edn"))
|
||||
(def restic-repo {:password-file "restic-pwd"
|
||||
:restic-repository "restic-repo"})
|
||||
|
||||
(def file-config (merge restic-repo {:backup-path "files"
|
||||
:files ["test-backup"]
|
||||
:restore-target-directory "test-restore"}))
|
||||
|
||||
|
||||
(def db-config (merge restic-repo {:backup-path "db"
|
||||
:pg-db "mydb"
|
||||
:pg-user "user"
|
||||
:pg-password "password"}))
|
||||
|
||||
(def dry-run {:dry-run true :debug true})
|
||||
|
||||
(defn prepare!
|
||||
[]
|
||||
(println config)
|
||||
(tasks/shell "mkdir" "-p" "/var/backups/test-backup")
|
||||
(spit "/var/backups/test-backup/file" "I was here")
|
||||
(tasks/shell "mkdir" "-p" "/var/backups/test-restore")
|
||||
(pg/create-pg-pass! (:db-config config)))
|
||||
(spit "/tmp/file_password" "file-password")
|
||||
(println (bc/env-or-file "FILE_PASSWORD"))
|
||||
(println (bc/env-or-file "ENV_PASSWORD"))
|
||||
(spit "restic-pwd" "ThePassword")
|
||||
(tasks/shell "mkdir" "-p" "test-backup")
|
||||
(spit "test-backup/file" "I was here")
|
||||
(tasks/shell "mkdir" "-p" "test-restore")
|
||||
(pg/create-pg-pass! db-config))
|
||||
|
||||
(defn restic-repo-init!
|
||||
[]
|
||||
(println "\nrestic-repo-init!")
|
||||
(rc/init! (:file-config config))
|
||||
(rc/init! (merge (:db-config config) (:dry-run config)))
|
||||
(rc/init! (merge (:db-roles-config config) (:dry-run config))))
|
||||
[]
|
||||
(rc/init! file-config)
|
||||
(rc/init! (merge db-config dry-run)))
|
||||
|
||||
(defn restic-backup!
|
||||
[]
|
||||
(println "\nrestic-backup!")
|
||||
(bak/backup-file! (:file-config config))
|
||||
(bak/backup-db-roles! (merge (:db-roles-config config) (:dry-run config)))
|
||||
(bak/backup-db! (merge (:db-config config) (:dry-run config))))
|
||||
[]
|
||||
(bak/backup-file! file-config)
|
||||
(bak/backup-db! (merge db-config dry-run)))
|
||||
|
||||
(defn list-snapshots!
|
||||
[]
|
||||
(println "\nlist-snapshots!")
|
||||
(rc/list-snapshots! (:file-config config))
|
||||
(rc/list-snapshots! (merge (:db-roles-config config) (:dry-run config)))
|
||||
(rc/list-snapshots! (merge (:db-config config) (:dry-run config))))
|
||||
(rc/list-snapshots! file-config)
|
||||
(rc/list-snapshots! (merge db-config dry-run)))
|
||||
|
||||
|
||||
(defn restic-restore!
|
||||
[]
|
||||
(println "\nrestic-restore!")
|
||||
(rs/restore-file! (:file-config config))
|
||||
(pg/drop-create-db! (merge (:db-config config) (:dry-run config)))
|
||||
(rs/restore-db-roles! (merge (:db-roles-config config) (:dry-run config)))
|
||||
(rs/restore-db! (merge (:db-config config) (:dry-run config))))
|
||||
|
||||
(defn change-password!
|
||||
[]
|
||||
(println "\nchange-password!")
|
||||
(rc/change-password! (:file-config config)))
|
||||
|
||||
(defn restic-backup-with-new!
|
||||
[]
|
||||
(println "\nrestic-backup with new!")
|
||||
(bak/backup-file! (:file-config-with-new config)))
|
||||
|
||||
(defn list-snapshots-with-new!
|
||||
[]
|
||||
(println "\nlist-snapshots with new!")
|
||||
(rc/list-snapshots! (:file-config-with-new config)))
|
||||
|
||||
[]
|
||||
(rs/restore-file! file-config)
|
||||
(pg/drop-create-db! (merge db-config dry-run))
|
||||
(rs/restore-db! (merge db-config dry-run)))
|
||||
|
||||
(prepare!)
|
||||
(restic-repo-init!)
|
||||
(restic-backup!)
|
||||
(list-snapshots!)
|
||||
(restic-restore!)
|
||||
(change-password!)
|
||||
(restic-backup!)
|
||||
(list-snapshots!)
|
||||
(restic-restore!)
|
||||
(restic-backup-with-new!)
|
||||
(list-snapshots-with-new!)
|
||||
|
|
|
|||
|
|
@ -1,5 +1,4 @@
|
|||
(ns dda.backup.backup
|
||||
"Part of API, represents the application service layer."
|
||||
(:require
|
||||
[orchestra.core :refer [defn-spec]]
|
||||
[clojure.spec.alpha :as s]
|
||||
|
|
@ -27,31 +26,25 @@
|
|||
(s/merge ::pg/pg-config
|
||||
::restic/restic-config))
|
||||
|
||||
(defn- config-w-defaults
|
||||
[config]
|
||||
(if (restic/use-new-password? config)
|
||||
(merge default config {:password-file (:new-password-file config)})
|
||||
(merge default config)))
|
||||
|
||||
(defn-spec backup-file! nil?
|
||||
[config ::backup-file-config]
|
||||
(let [config-2-use (config-w-defaults config)]
|
||||
(restic/unlock! config-2-use)
|
||||
(let [config-w-defaults (merge default config)]
|
||||
(restic/unlock! config-w-defaults)
|
||||
(i/execute!
|
||||
(domain/backup-files-command config-2-use)
|
||||
config-2-use)
|
||||
(restic/forget! config-2-use)))
|
||||
(domain/backup-files-command config-w-defaults)
|
||||
config-w-defaults)
|
||||
(restic/forget! config-w-defaults)))
|
||||
|
||||
(defn-spec backup-db-roles! nil?
|
||||
[config ::pg-role-dump-config]
|
||||
(let [config-2-use (config-w-defaults config)]
|
||||
(restic/unlock! config-2-use)
|
||||
(i/execute! (domain/backup-role-command config-2-use) config-2-use)
|
||||
(restic/forget! config-2-use)))
|
||||
(let [config-w-defaults (merge default config)]
|
||||
(restic/unlock! config-w-defaults)
|
||||
(i/execute! (domain/backup-role-command config-w-defaults) config-w-defaults)
|
||||
(restic/forget! config-w-defaults)))
|
||||
|
||||
(defn-spec backup-db! nil?
|
||||
[config ::pg-db-dump-config]
|
||||
(let [config-2-use (config-w-defaults config)]
|
||||
(restic/unlock! config-2-use)
|
||||
(i/execute! (domain/backup-db-command config-2-use) config-2-use)
|
||||
(restic/forget! config-2-use)))
|
||||
(let [config-w-defaults (merge default config)]
|
||||
(restic/unlock! config-w-defaults)
|
||||
(i/execute! (domain/backup-db-command config-w-defaults) config-w-defaults)
|
||||
(restic/forget! config-w-defaults)))
|
||||
|
|
@ -1,5 +1,4 @@
|
|||
(ns dda.backup.backup.domain
|
||||
"Intended for internal use only, represents the domain layer."
|
||||
(:require
|
||||
[orchestra.core :refer [defn-spec]]
|
||||
[clojure.spec.alpha :as s]
|
||||
|
|
@ -18,7 +17,7 @@
|
|||
(defn-spec backup-files-command ::cd/commands
|
||||
[config ::backup-file-config]
|
||||
(let [{:keys [files]} config]
|
||||
[(rd/repo-command config (into ["backup"] files) false)]))
|
||||
[(rd/repo-command config (into ["backup"] files))]))
|
||||
|
||||
(defn-spec backup-role-command ::cd/commands
|
||||
[config ::pd/pg-role-dump-config]
|
||||
|
|
|
|||
|
|
@ -1,22 +0,0 @@
|
|||
(ns dda.backup.config
|
||||
"Part of API, represents the application service layer."
|
||||
(:require [aero.core :as aero]
|
||||
[dda.backup.core :as bc]
|
||||
[dda.backup.infrastructure :as i]))
|
||||
|
||||
(defmethod aero/reader 'env-or-file
|
||||
[{:keys [profile] :as opts} tag value]
|
||||
(bc/env-or-file value))
|
||||
|
||||
(defmethod aero/reader 'gopass
|
||||
[{:keys [profile] :as opts} tag value]
|
||||
(i/execute-out! (into ["gopass" "show" "-y" "-o"] value) {}))
|
||||
|
||||
(defn read-config
|
||||
[file]
|
||||
(try
|
||||
(aero/read-config file)
|
||||
(catch Exception e
|
||||
(do (println (str "Warn: " e))
|
||||
{}))
|
||||
))
|
||||
|
|
@ -1,5 +1,4 @@
|
|||
(ns dda.backup.core
|
||||
"Part of API, represents the application service layer."
|
||||
(:require
|
||||
[orchestra.core :refer [defn-spec]]
|
||||
[clojure.spec.alpha :as s]
|
||||
|
|
|
|||
|
|
@ -1,5 +1,4 @@
|
|||
(ns dda.backup.core.domain
|
||||
"Intended for internal use only, represents the domain layer."
|
||||
(:require
|
||||
[clojure.spec.alpha :as s]))
|
||||
|
||||
|
|
|
|||
|
|
@ -1,29 +1,8 @@
|
|||
(ns dda.backup.infrastructure
|
||||
"Intenden for internal use, represents the infrastructure layer."
|
||||
(:require [orchestra.core :refer [defn-spec]]
|
||||
[babashka.tasks :as t]
|
||||
[babashka.http-client :as http]
|
||||
[dda.backup.core.domain :as core]))
|
||||
|
||||
(defn-spec execute-out! string?
|
||||
[command ::core/command
|
||||
config ::core/execution]
|
||||
(let [{:keys [dry-run debug]} config]
|
||||
(when debug
|
||||
(println command))
|
||||
(when-not dry-run
|
||||
(:out (t/shell {:out :string :err :string} (clojure.string/join " " command))))))
|
||||
|
||||
(defn-spec execute-single! string?
|
||||
[command ::core/command
|
||||
config ::core/execution]
|
||||
(let [{:keys [dry-run debug]} config]
|
||||
(when debug
|
||||
(println command))
|
||||
(when-not dry-run
|
||||
(:out (t/shell {:err :string} (clojure.string/join " " command))))))
|
||||
|
||||
|
||||
(defn-spec execute! nil?
|
||||
[commands ::core/commands
|
||||
config ::core/execution]
|
||||
|
|
@ -33,8 +12,3 @@
|
|||
(println c))
|
||||
(when-not dry-run
|
||||
(apply t/shell c)))))
|
||||
|
||||
(defn-spec post! nil?
|
||||
[url string?
|
||||
content string?]
|
||||
(http/post url {:body content}))
|
||||
|
|
@ -1,49 +0,0 @@
|
|||
(ns dda.backup.monitoring
|
||||
(:require
|
||||
[orchestra.core :refer [defn-spec]]
|
||||
[clojure.spec.alpha :as s]
|
||||
[dda.backup.monitoring.domain :as domain]
|
||||
[dda.backup.infrastructure :as i]))
|
||||
|
||||
(def default {:url "http://prometheus-pushgateway.monitoring.svc.cluster.local:9091/metrics/job"
|
||||
:namespace "default"
|
||||
:metrics {:kube_job_status_active 0
|
||||
:kube_job_status_failed 0
|
||||
:kube_job_status_succeeded 0}})
|
||||
|
||||
(s/def ::config (s/keys :req-un [::name]
|
||||
:opt-un [::url ::metrics ::namespace]))
|
||||
|
||||
(defn- config-w-defaults
|
||||
[config]
|
||||
(merge default config))
|
||||
|
||||
(defn-spec send-metrics! nil?
|
||||
[config ::config]
|
||||
(let [config-2-use (config-w-defaults config)
|
||||
{:keys [url name]} config-2-use]
|
||||
(try
|
||||
(i/post! (str url "/" name) (domain/collect-metrics config-2-use))
|
||||
(catch Exception e (println (str "Warn: unable to send log" (.getMessage e)))))))
|
||||
|
||||
(defn-spec backup-start-metrics! nil?
|
||||
[config ::config]
|
||||
(send-metrics!
|
||||
{:name "backup"
|
||||
:metrics {:kube_job_status_active 1
|
||||
:kube_job_status_start_time (long (/ (System/currentTimeMillis) 1000))}}))
|
||||
|
||||
(defn-spec backup-success-metrics! nil?
|
||||
[config ::config]
|
||||
(send-metrics!
|
||||
{:name "backup"
|
||||
:metrics {:kube_job_status_succeeded 1
|
||||
:kube_job_status_completion_time (long (/ (System/currentTimeMillis) 1000))}}))
|
||||
|
||||
(defn-spec backup-fail-metrics! nil?
|
||||
[config ::config]
|
||||
(send-metrics!
|
||||
{:name "backup"
|
||||
:metrics {:kube_job_status_failed 1
|
||||
:kube_job_status_completion_time (long (/ (System/currentTimeMillis) 1000))}}))
|
||||
|
||||
|
|
@ -1,23 +0,0 @@
|
|||
(ns dda.backup.monitoring.domain
|
||||
(:require
|
||||
[orchestra.core :refer [defn-spec]]
|
||||
[clojure.spec.alpha :as s]
|
||||
[clojure.string :as st]))
|
||||
|
||||
(s/def ::url string?)
|
||||
(s/def ::name string?)
|
||||
(s/def ::namespace string?)
|
||||
(s/def ::metrics map?)
|
||||
|
||||
(s/def ::config (s/keys :req-un [::url ::name ::metrics ::namespace]))
|
||||
|
||||
(defn-spec collect-metrics string?
|
||||
[config ::config]
|
||||
(let [{:keys [metrics namespace]} config]
|
||||
(str
|
||||
(->> metrics
|
||||
(map (fn [entry] (str (name (key entry))
|
||||
"{namespace=\"" namespace "\"}" " "
|
||||
(val entry))))
|
||||
(st/join "\n"))
|
||||
"\n")))
|
||||
|
|
@ -1,5 +1,4 @@
|
|||
(ns dda.backup.postgresql
|
||||
"Part of API, represents the application service layer."
|
||||
(:require
|
||||
[orchestra.core :refer [defn-spec]]
|
||||
[clojure.spec.alpha :as s]
|
||||
|
|
|
|||
|
|
@ -1,5 +1,4 @@
|
|||
(ns dda.backup.postgresql.domain
|
||||
"Intended for internal use only, represents the domain layer."
|
||||
(:require
|
||||
[orchestra.core :refer [defn-spec]]
|
||||
[clojure.spec.alpha :as s]
|
||||
|
|
@ -51,7 +50,7 @@
|
|||
"--no-password"]
|
||||
command
|
||||
["|" "grep" pg-role-prefix "|"]
|
||||
(rd/repo-command config ["backup" "--stdin"] false)))))
|
||||
(rd/repo-command config ["backup" "--stdin"])))))
|
||||
|
||||
(defn-spec pgdump-command ::cd/command
|
||||
[config ::pg-db-dump-config
|
||||
|
|
@ -64,7 +63,7 @@
|
|||
"--no-password"]
|
||||
command
|
||||
["|"]
|
||||
(rd/repo-command config ["backup" "--stdin"] false)))))
|
||||
(rd/repo-command config ["backup" "--stdin"])))))
|
||||
|
||||
(defn-spec pgpass string?
|
||||
[config ::pg-config]
|
||||
|
|
|
|||
|
|
@ -1,5 +1,4 @@
|
|||
(ns dda.backup.restic
|
||||
"Part of API, represents the application service layer."
|
||||
(:require
|
||||
[orchestra.core :refer [defn-spec]]
|
||||
[clojure.spec.alpha :as s]
|
||||
|
|
@ -13,81 +12,39 @@
|
|||
:months-to-keep 12}))
|
||||
|
||||
(s/def ::restic-config
|
||||
(s/merge ::core/execution
|
||||
(s/merge ::core/execution
|
||||
(s/keys :req-un [::domain/restic-repository
|
||||
::domain/backup-path]
|
||||
:opt-un [::domain/certificate-file
|
||||
::domain/password-file
|
||||
::domain/new-password-file
|
||||
::domain/days-to-keep
|
||||
::domain/months-to-keep])))
|
||||
|
||||
(s/def ::check-result #{:initialized :wrong-password :not-initialized :error})
|
||||
|
||||
(defn-spec check ::check-result
|
||||
"Check a restic repository whether it is initialized and there are correct credentials given."
|
||||
(defn-spec initalized? boolean?
|
||||
[restic-config ::restic-config]
|
||||
(let [config-w-defaults (merge core/default restic-config)]
|
||||
(try
|
||||
(i/execute! (domain/check-repo-command config-w-defaults) config-w-defaults)
|
||||
:initialized
|
||||
(catch Exception e
|
||||
(let [data (ex-data e)
|
||||
parsed-error (domain/parse-check-error (get-in data [:proc :err]))]
|
||||
(cond
|
||||
(= parsed-error :not-initialized) :not-initialized
|
||||
(= parsed-error :wrong-password) :wrong-password
|
||||
:default :error))))))
|
||||
|
||||
(defn-spec use-new-password? boolean?
|
||||
"Check, whether a given new password should b used instead of the normal one."
|
||||
[restic-config ::restic-config]
|
||||
(if (contains? restic-config :new-password-file)
|
||||
(= :initialized (check (merge restic-config {:password-file (:new-password-file restic-config)})))
|
||||
false))
|
||||
|
||||
(defn- config-w-defaults
|
||||
"Merge given configuration with defaults & replace restic-password with restic-new-password if necessary."
|
||||
[restic-config]
|
||||
(if (use-new-password? restic-config)
|
||||
(merge default restic-config {:password-file (:new-password-file restic-config)})
|
||||
(merge default restic-config)))
|
||||
|
||||
(defn-spec
|
||||
^{:deprecated "0.2.0"
|
||||
:superseded-by "check"}
|
||||
initalized? boolean?
|
||||
[restic-config ::restic-config]
|
||||
(let [config-2-use (config-w-defaults restic-config)]
|
||||
(= :initialized (check config-2-use))))
|
||||
true
|
||||
(catch Exception e false))))
|
||||
|
||||
(defn-spec init! nil?
|
||||
[restic-config ::restic-config]
|
||||
(let [config-2-use (config-w-defaults restic-config)]
|
||||
(when (= :not-initialized (check config-2-use))
|
||||
(i/execute! (domain/init-repo-command config-2-use) config-2-use))))
|
||||
(let [config-w-defaults (merge core/default restic-config)]
|
||||
(when (not (initalized? config-w-defaults))
|
||||
(i/execute! (domain/init-repo-command config-w-defaults) config-w-defaults))))
|
||||
|
||||
(defn-spec unlock! nil?
|
||||
[restic-config ::restic-config]
|
||||
(let [config-2-use (config-w-defaults restic-config)]
|
||||
(i/execute! (domain/unlock-repo-command config-2-use) config-2-use)))
|
||||
(let [config-w-defaults (merge core/default restic-config)]
|
||||
(i/execute! (domain/unlock-repo-command config-w-defaults) config-w-defaults)))
|
||||
|
||||
(defn-spec forget! nil?
|
||||
[restic-config ::restic-config]
|
||||
(let [config-2-use (config-w-defaults restic-config)]
|
||||
(i/execute! (domain/forget-command config-2-use) config-2-use)))
|
||||
(let [config-w-defaults (merge core/default restic-config)]
|
||||
(i/execute! (domain/forget-command config-w-defaults) config-w-defaults)))
|
||||
|
||||
(defn-spec list-snapshots! nil?
|
||||
[restic-config ::restic-config]
|
||||
(let [config-2-use (config-w-defaults restic-config)]
|
||||
(i/execute! (domain/list-snapshot-command config-2-use) config-2-use)))
|
||||
|
||||
(defn-spec change-password! nil?
|
||||
[restic-config ::restic-config]
|
||||
(when (contains? restic-config :new-password-file)
|
||||
(let [config-2-use (merge core/default restic-config)]
|
||||
(when (= :initialized (check config-2-use))
|
||||
(do
|
||||
(i/execute! (domain/change-password-command config-2-use) config-2-use)
|
||||
(when-not (= :wrong-password (check config-2-use))
|
||||
(throw (Exception. "password-change did not work!"))))))))
|
||||
(let [config-w-defaults (merge core/default restic-config)]
|
||||
(i/execute! (domain/list-snapshot-command config-w-defaults) config-w-defaults)))
|
||||
|
|
|
|||
|
|
@ -1,5 +1,4 @@
|
|||
(ns dda.backup.restic.domain
|
||||
"Intended for internal use only, represents the domain layer."
|
||||
(:require
|
||||
[orchestra.core :refer [defn-spec]]
|
||||
[clojure.spec.alpha :as s]
|
||||
|
|
@ -7,40 +6,31 @@
|
|||
|
||||
(s/def ::certificate-file string?)
|
||||
(s/def ::password-file string?)
|
||||
(s/def ::new-password-file string?)
|
||||
(s/def ::restic-repository string?)
|
||||
(s/def ::backup-path string?)
|
||||
(s/def ::days-to-keep pos?)
|
||||
(s/def ::months-to-keep pos?)
|
||||
|
||||
(s/def ::restic-config
|
||||
(s/keys :req-un [::restic-repository
|
||||
::backup-path
|
||||
::days-to-keep
|
||||
(s/keys :req-un [::restic-repository
|
||||
::backup-path
|
||||
::days-to-keep
|
||||
::months-to-keep]
|
||||
:opt-un [::certificate-file
|
||||
::password-file
|
||||
::new-password-file
|
||||
:opt-un [::certificate-file
|
||||
::password-file
|
||||
::cd/execution-directory]))
|
||||
|
||||
(s/def ::check-error #{:not-initialized :wrong-password :no-password :unknown})
|
||||
|
||||
(defn-spec repo-command ::cd/command
|
||||
[config ::restic-config
|
||||
command ::cd/command
|
||||
read-error-stream boolean?]
|
||||
(let [{:keys [certificate-file password-file execution-directory
|
||||
restic-repository backup-path]} config
|
||||
shell-option (if read-error-stream {:err :string} {})]
|
||||
command ::cd/command]
|
||||
(let [{:keys [certificate-file password-file execution-directory
|
||||
restic-repository backup-path]} config]
|
||||
(into
|
||||
[]
|
||||
(concat
|
||||
(cond
|
||||
(some? execution-directory)
|
||||
[(merge shell-option {:dir execution-directory})]
|
||||
read-error-stream
|
||||
[shell-option]
|
||||
:default [])
|
||||
(if (some? execution-directory)
|
||||
[{:dir execution-directory}]
|
||||
[])
|
||||
["restic" "-r" (str restic-repository "/" backup-path) "-v"]
|
||||
(cond
|
||||
(some? certificate-file)
|
||||
|
|
@ -53,19 +43,19 @@
|
|||
|
||||
(defn-spec check-repo-command ::cd/commands
|
||||
[config ::restic-config]
|
||||
[(repo-command config ["check"] true)])
|
||||
[(repo-command config ["check"])])
|
||||
|
||||
(defn-spec init-repo-command ::cd/commands
|
||||
[config ::restic-config]
|
||||
[(repo-command config ["init"] false)])
|
||||
[(repo-command config ["init"])])
|
||||
|
||||
(defn-spec unlock-repo-command ::cd/commands
|
||||
[config ::restic-config]
|
||||
[(repo-command config ["--cleanup-cache" "unlock"] false)])
|
||||
[(repo-command config ["--cleanup-cache" "unlock"])])
|
||||
|
||||
(defn-spec list-snapshot-command ::cd/commands
|
||||
[config ::restic-config]
|
||||
[(repo-command config ["snapshots"] false)])
|
||||
[(repo-command config ["snapshots"])])
|
||||
|
||||
(defn-spec forget-command ::cd/commands
|
||||
[config ::restic-config]
|
||||
|
|
@ -73,20 +63,4 @@
|
|||
[(repo-command config ["forget" "--group-by" ""
|
||||
"--keep-last" "1"
|
||||
"--keep-daily" (str days-to-keep)
|
||||
"--keep-monthly" (str months-to-keep) "--prune"] false)]))
|
||||
|
||||
(defn-spec change-password-command ::cd/command
|
||||
[config ::restic-config]
|
||||
(if (contains? config :new-password-file)
|
||||
(let [{:keys [new-password-file]} config]
|
||||
[(repo-command config ["--new-password-file" new-password-file
|
||||
"key" "passwd"] false)])
|
||||
(throw (Exception. "change-password: new password required"))))
|
||||
|
||||
(defn-spec parse-check-error ::check-error
|
||||
[error string?]
|
||||
(cond
|
||||
(clojure.string/includes? error "Fatal: unable to open config file") :not-initialized
|
||||
(clojure.string/includes? error "Fatal: wrong password or no key found") :wrong-password
|
||||
(clojure.string/includes? error "Resolving password failed") :no-password
|
||||
:default :unknown))
|
||||
"--keep-monthly" (str months-to-keep) "--prune"])]))
|
||||
|
|
@ -1,5 +1,4 @@
|
|||
(ns dda.backup.restore
|
||||
"Part of API, represents the application service layer."
|
||||
(:require
|
||||
[orchestra.core :refer [defn-spec]]
|
||||
[clojure.spec.alpha :as s]
|
||||
|
|
@ -21,29 +20,34 @@
|
|||
(s/merge ::pg/pg-config
|
||||
(s/keys :req-un [::domain/snapshot-id])))
|
||||
|
||||
(defn- config-w-defaults
|
||||
[config]
|
||||
(if (restic/use-new-password? config)
|
||||
(merge default config {:password-file (:new-password-file config)})
|
||||
(merge default config)))
|
||||
|
||||
|
||||
(defn-spec restore-file! nil?
|
||||
[config ::restore-file-config]
|
||||
(let [config-2-use (config-w-defaults config)]
|
||||
(restic/unlock! config-2-use)
|
||||
(let [config-w-defaults (merge default config)]
|
||||
(restic/unlock! config-w-defaults)
|
||||
(i/execute!
|
||||
(domain/restore-dir-command config-2-use)
|
||||
config-2-use)))
|
||||
(domain/restore-dir-command config-w-defaults)
|
||||
config-w-defaults)))
|
||||
|
||||
(defn-spec restore-db! nil?
|
||||
[config ::restore-db-config]
|
||||
(let [config-2-use (config-w-defaults config)]
|
||||
(restic/unlock! config-2-use)
|
||||
(i/execute! (domain/restore-db-command config-2-use) config-2-use)))
|
||||
(let [config-w-defaults (merge default config)]
|
||||
(restic/unlock! config-w-defaults)
|
||||
(i/execute! (domain/restore-db-command config-w-defaults) config-w-defaults)))
|
||||
|
||||
(defn-spec restore-db-roles! nil?
|
||||
[config ::restore-db-config]
|
||||
(let [config-2-use (config-w-defaults config)]
|
||||
(restic/unlock! config-2-use)
|
||||
(i/execute! (domain/restore-db-roles-command config-2-use) config-2-use)))
|
||||
|
||||
;; function restore-roles() {
|
||||
;; local snapshot_id="${1:-latest}"; shift
|
||||
|
||||
;; if [ -z ${CERTIFICATE_FILE} ];
|
||||
;; then
|
||||
;; roles-unlock-command
|
||||
;; restic -r ${RESTIC_REPOSITORY}/${backup_pg_role_path} dump ${snapshot_id} stdin | \
|
||||
;; psql -d template1 -h ${POSTGRES_SERVICE} -p ${POSTGRES_PORT} -U ${POSTGRES_USER} \
|
||||
;; --no-password
|
||||
;; else
|
||||
;; roles-unlock-command --cacert ${CERTIFICATE_FILE}
|
||||
;; restic -r ${RESTIC_REPOSITORY}/${backup_pg_role_path} dump ${snapshot_id} stdin --cacert ${CERTIFICATE_FILE} | \
|
||||
;; psql -d template1 -h ${POSTGRES_SERVICE} -p ${POSTGRES_PORT} -U ${POSTGRES_USER} \
|
||||
;; --no-password
|
||||
;; fi
|
||||
;; }
|
||||
|
|
|
|||
|
|
@ -1,5 +1,4 @@
|
|||
(ns dda.backup.restore.domain
|
||||
"Intended for internal use only, represents the domain layer."
|
||||
(:require
|
||||
[orchestra.core :refer [defn-spec]]
|
||||
[clojure.spec.alpha :as s]
|
||||
|
|
@ -10,13 +9,11 @@
|
|||
|
||||
(s/def ::restore-target-directory string?)
|
||||
(s/def ::snapshot-id string?)
|
||||
(s/def ::clean-up-element string?)
|
||||
(s/def ::clean-up-elements (s/coll-of ::clean-up-element))
|
||||
|
||||
(s/def ::restore-file-config
|
||||
(s/merge ::rd/restic-config
|
||||
(s/keys :req-un [::restore-target-directory ::snapshot-id]
|
||||
:opt-un [::clean-up-elements])))
|
||||
(s/keys :req-un [::restore-target-directory
|
||||
::snapshot-id])))
|
||||
|
||||
(s/def ::restore-db-config
|
||||
(s/merge ::pd/pg-config
|
||||
|
|
@ -25,13 +22,9 @@
|
|||
|
||||
(defn-spec restore-dir-command ::cd/commands
|
||||
[config ::restore-file-config]
|
||||
(let [{:keys [restore-target-directory snapshot-id clean-up-elements]} config]
|
||||
[(if (contains? config :clean-up-elements)
|
||||
(into
|
||||
["rm" "-rf"]
|
||||
(map #(str restore-target-directory "/" %) clean-up-elements))
|
||||
["rm" "-rf" restore-target-directory])
|
||||
(rd/repo-command config ["restore" snapshot-id "--target" restore-target-directory] false)]))
|
||||
(let [{:keys [restore-target-directory snapshot-id]} config]
|
||||
[["rm" "-rf" restore-target-directory]
|
||||
(rd/repo-command config ["restore" snapshot-id "--target" restore-target-directory])]))
|
||||
|
||||
(defn-spec restore-db-command ::cd/commands
|
||||
[config ::restore-db-config]
|
||||
|
|
@ -42,19 +35,7 @@
|
|||
(into
|
||||
[]
|
||||
(concat
|
||||
(rd/repo-command config ["dump" snapshot-id "stdin"] false)
|
||||
(rd/repo-command config ["dump" snapshot-id "stdin"])
|
||||
["|"]
|
||||
(pd/psql-command config []))))]]))
|
||||
|
||||
(defn-spec restore-db-roles-command ::cd/commands
|
||||
[config ::restore-db-config]
|
||||
(let [{:keys [snapshot-id]} config]
|
||||
[["bash" "-c"
|
||||
(st/join
|
||||
" "
|
||||
(into
|
||||
[]
|
||||
(concat
|
||||
(rd/repo-command config ["dump" snapshot-id "stdin"] false)
|
||||
["|"]
|
||||
(pd/psql-command (merge config {:pg-db "template1"}) []))))]]))
|
||||
|
||||
|
|
|
|||
|
|
@ -18,7 +18,7 @@
|
|||
"file2"
|
||||
"file2"]]
|
||||
(cut/backup-files-command {:restic-repository "repo"
|
||||
:backup-path "dir-at-repo"
|
||||
;:backup-path "dir-at-repo"
|
||||
:execution-directory "dir-to-backup"
|
||||
:days-to-keep 39
|
||||
:months-to-keep 3
|
||||
|
|
|
|||
|
|
@ -1,20 +0,0 @@
|
|||
(ns dda.backup.monitoring.domain-test
|
||||
(:require
|
||||
[clojure.test :refer [deftest is are testing run-tests]]
|
||||
[clojure.spec.test.alpha :as st]
|
||||
[dda.backup.monitoring.domain :as cut]))
|
||||
|
||||
(st/instrument `cut/collect-metrics)
|
||||
|
||||
(deftest should-collect-metrics
|
||||
(is (= "\n"
|
||||
(cut/collect-metrics {:url "url"
|
||||
:name "name"
|
||||
:namespace "default"
|
||||
:metrics {}})))
|
||||
(is (= "metric1{namespace=\"default\"} 1\nmetric2{namespace=\"default\"} text\n"
|
||||
(cut/collect-metrics {:url "url"
|
||||
:name "name"
|
||||
:namespace "default"
|
||||
:metrics {:metric1 1
|
||||
:metric2 "text"}}))))
|
||||
|
|
@ -10,7 +10,7 @@
|
|||
(st/instrument `cut/forget-command)
|
||||
|
||||
(deftest should-calculate-repo-command
|
||||
(is (= [{:dir "dir" :err :string}
|
||||
(is (= [{:dir "dir"}
|
||||
"restic"
|
||||
"-r"
|
||||
"repo/dir"
|
||||
|
|
@ -26,45 +26,27 @@
|
|||
:backup-path "dir"
|
||||
:days-to-keep 39
|
||||
:months-to-keep 3}
|
||||
["backup" "file1" "file2"]
|
||||
true)))
|
||||
(is (= [{:dir "dir"}
|
||||
"restic"
|
||||
"-r"
|
||||
"repo/dir"
|
||||
"-v"
|
||||
"--cacert"
|
||||
"ca"
|
||||
"backup"
|
||||
"file1"
|
||||
"file2"]
|
||||
(cut/repo-command {:certificate-file "ca"
|
||||
:execution-directory "dir"
|
||||
:restic-repository "repo"
|
||||
:backup-path "dir"
|
||||
:days-to-keep 39
|
||||
:months-to-keep 3}
|
||||
["backup" "file1" "file2"] false)))
|
||||
(is (= [{:err :string} "restic" "-r" "repo/dir" "-v" "--cacert" "ca" "snapshots"]
|
||||
["backup" "file1" "file2"])))
|
||||
(is (= ["restic" "-r" "repo/dir" "-v" "--cacert" "ca" "snapshots"]
|
||||
(cut/repo-command {:certificate-file "ca"
|
||||
:restic-repository "repo"
|
||||
:backup-path "dir"
|
||||
:days-to-keep 39
|
||||
:months-to-keep 3}
|
||||
["snapshots"] true)))
|
||||
["snapshots"])))
|
||||
(is (= ["restic" "-r" "repo/dir" "-v" "--password-file" "password" "snapshots"]
|
||||
(cut/repo-command {:password-file "password"
|
||||
:restic-repository "repo"
|
||||
:backup-path "dir"
|
||||
:days-to-keep 39
|
||||
:months-to-keep 3}
|
||||
["snapshots"] false)))
|
||||
["snapshots"])))
|
||||
(is (= ["restic" "-r" "repo/dir" "-v" "snapshots"]
|
||||
(cut/repo-command {:restic-repository "repo"
|
||||
:backup-path "dir"
|
||||
:days-to-keep 39
|
||||
:months-to-keep 3}
|
||||
["snapshots"] false))))
|
||||
["snapshots"]))))
|
||||
|
||||
(deftest should-calculate-init-repo-command
|
||||
(is (= [["restic" "-r" "repo/dir" "-v" "init"]]
|
||||
|
|
@ -87,29 +69,4 @@
|
|||
(cut/forget-command {:restic-repository "repo"
|
||||
:backup-path "dir"
|
||||
:days-to-keep 39
|
||||
:months-to-keep 3}))))
|
||||
|
||||
(deftest should-calculate-change-password-command
|
||||
(is (= [["restic"
|
||||
"-r"
|
||||
"repo/dir"
|
||||
"-v"
|
||||
"--new-password-file"
|
||||
"/new-pwd"
|
||||
"key"
|
||||
"passwd"]]
|
||||
(cut/change-password-command {:restic-repository "repo"
|
||||
:new-password-file "/new-pwd"
|
||||
:backup-path "dir"
|
||||
:days-to-keep 39
|
||||
:months-to-keep 3}))))
|
||||
|
||||
(deftest should-parse-check-error
|
||||
(is (= :not-initialized
|
||||
(cut/parse-check-error "Fatal: unable to open config file: stat /restic-repo/files/config: no such file or directory\nIs there a repository at the following location?\n/restic-repo/files" )
|
||||
))
|
||||
(is (= :wrong-password
|
||||
(cut/parse-check-error "Fatal: wrong password or no key found\n")))
|
||||
(is (= :no-password
|
||||
(cut/parse-check-error "Resolving password failed: Fatal: /restic-pwd does not exist\n")))
|
||||
)
|
||||
:months-to-keep 3}))))
|
||||
|
|
@ -7,7 +7,7 @@
|
|||
(st/instrument `cut/restore-dir-command)
|
||||
(st/instrument `cut/restore-db-command)
|
||||
|
||||
(deftest should-calculate-restore-dir-command
|
||||
(deftest should-calculate-restore-dir
|
||||
(is (= [["rm" "-rf" "dir-to-backup"]
|
||||
["restic"
|
||||
"-r"
|
||||
|
|
@ -22,22 +22,6 @@
|
|||
:restore-target-directory "dir-to-backup"
|
||||
:days-to-keep 39
|
||||
:months-to-keep 3
|
||||
:snapshot-id "latest"})))
|
||||
(is (= [["rm" "-rf" "dir-to-backup/file" "dir-to-backup/folder/"]
|
||||
["restic"
|
||||
"-r"
|
||||
"repo/dir-at-repo"
|
||||
"-v"
|
||||
"restore"
|
||||
"latest"
|
||||
"--target"
|
||||
"dir-to-backup"]]
|
||||
(cut/restore-dir-command {:restic-repository "repo"
|
||||
:backup-path "dir-at-repo"
|
||||
:restore-target-directory "dir-to-backup"
|
||||
:clean-up-elements ["file" "folder/"]
|
||||
:days-to-keep 39
|
||||
:months-to-keep 3
|
||||
:snapshot-id "latest"}))))
|
||||
|
||||
(deftest should-calculate-restore-db
|
||||
|
|
@ -55,19 +39,3 @@
|
|||
:days-to-keep 39
|
||||
:months-to-keep 3
|
||||
:snapshot-id "latest"}))))
|
||||
|
||||
(deftest should-calculate-restore-db-roles
|
||||
(is (= [["bash"
|
||||
"-c"
|
||||
(str "restic -r repo/dir-at-repo -v dump latest stdin | "
|
||||
"psql -d template1 -h localhost -p 5432 -U user --no-password")]]
|
||||
(cut/restore-db-roles-command {:restic-repository "repo"
|
||||
:backup-path "dir-at-repo"
|
||||
:pg-host "localhost"
|
||||
:pg-port 5432
|
||||
:pg-db "mydb"
|
||||
:pg-user "user"
|
||||
:pg-password "password"
|
||||
:days-to-keep 39
|
||||
:months-to-keep 3
|
||||
:snapshot-id "latest"}))))
|
||||
|
|
|
|||
13
tests.edn
13
tests.edn
|
|
@ -10,4 +10,15 @@
|
|||
|
||||
;; ---------------------------------------------------------
|
||||
|
||||
#kaocha/v1 {}
|
||||
#kaocha/v1 {:plugins [:orchestra
|
||||
:kaocha.plugin.alpha/info
|
||||
:profiling
|
||||
:print-invocations
|
||||
:hooks
|
||||
:notifier
|
||||
:kaocha.plugin/version-filter]
|
||||
|
||||
:kaocha/bindings {kaocha.stacktrace/*stacktrace-filters* []
|
||||
kaocha.stacktrace/*stacktrace-stop-list* []}
|
||||
|
||||
:reporter kaocha.report/documentation}
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue
