Compare commits
96 commits
Author | SHA1 | Date | |
---|---|---|---|
6a04396984 | |||
7b3143beca | |||
a0e47d74fc | |||
bd22b84a32 | |||
a06409fadf | |||
27987dfc8b | |||
b22ace7c6a | |||
ad16cb52bd | |||
58eccda7a7 | |||
242ebdc2ff | |||
5f69b54f15 | |||
78af31036e | |||
1c596889ad | |||
2aec56fbd6 | |||
d748a36559 | |||
71bd701d13 | |||
d489eee954 | |||
939523bc2e | |||
29fc04830a | |||
0b9739fdb6 | |||
164b691bc9 | |||
9685f07e46 | |||
6e1cc7bda2 | |||
304bb3c967 | |||
e26d67a40c | |||
8cb6bf6b28 | |||
9fc9687a08 | |||
20c66235c0 | |||
741b720632 | |||
394959d158 | |||
711ceecefc | |||
dad69a180b | |||
6de3e47bd2 | |||
03b0b85247 | |||
67ec0f58eb | |||
d7ce373d87 | |||
d1a8479598 | |||
c4832b1107 | |||
7e3312e285 | |||
f636f7ffc3 | |||
bba6bbe830 | |||
8388d72517 | |||
f2b583c060 | |||
5a3aca38cf | |||
4764d1db67 | |||
8aef785bdc | |||
5eb83f78a0 | |||
d997e470a0 | |||
dc86531454 | |||
ca0d4ac7b2 | |||
e4666a592e | |||
fa48d9762a | |||
6a278ece0d | |||
dda45d92d6 | |||
c69c9da659 | |||
de53b9b7a5 | |||
cc845af696 | |||
f5aa2295f0 | |||
1d344abc27 | |||
7cd4e4101d | |||
5d2a65079e | |||
b36808de7c | |||
3e588c082c | |||
4a74a1bec0 | |||
1ff8a0dc13 | |||
fd27c15ec7 | |||
192e053afc | |||
cab9b573c1 | |||
226046d278 | |||
f108b67e62 | |||
96343b9af5 | |||
f072ff027d | |||
e396880365 | |||
83080b30a8 | |||
4a78a35424 | |||
8727f16c75 | |||
ac3f2a455d | |||
c05ecfa427 | |||
fff2c939d9 | |||
e8cec9de8a | |||
1fb309f213 | |||
70d41ca532 | |||
a96cba8cb1 | |||
3bdf2ea553 | |||
fcf6d7783e | |||
ab4c6e0d76 | |||
ef8bfa11fc | |||
7d9ca203bb | |||
f8bcbe63ba | |||
7219533c86 | |||
9b13b96ff6 | |||
351b2295e3 | |||
3a9694d9a1 | |||
3b458b980b | |||
5f38fb7526 | |||
fa81908791 |
67 changed files with 585 additions and 771 deletions
.gitlab-ci.ymlREADME.mdauth-local.ednbuild.pyconfig-local.ednpackage.jsonproject.cljshadow-cljs.edn
doc
infrastructure
backup
build.py
image
Dockerfile
resources
backup.bbbackup.shbb-backup.ednbb.ednconfig.cljentrypoint-start-and-wait.shentrypoint.shinit.bbinit.shinstall.bbinstall.shlist-snapshots.bblist-snapshots.shrestore.bbrestore.shwait.bb
resources2
test
nextcloud
src
main
clj/dda/c4k_nextcloud
cljc/dda/c4k_nextcloud
cljs/dda/c4k_nextcloud
resources
test
cljc/dda/c4k_nextcloud
resources/local-integration-test
|
@ -4,7 +4,6 @@ stages:
|
||||||
- security
|
- security
|
||||||
- upload
|
- upload
|
||||||
- image
|
- image
|
||||||
#- integrationtest
|
|
||||||
|
|
||||||
.img: &img
|
.img: &img
|
||||||
image: "domaindrivenarchitecture/ddadevops-dind:4.11.3"
|
image: "domaindrivenarchitecture/ddadevops-dind:4.11.3"
|
||||||
|
@ -130,23 +129,3 @@ nextcloud-image-publish:
|
||||||
stage: image
|
stage: image
|
||||||
script:
|
script:
|
||||||
- cd infrastructure/nextcloud && pyb image publish
|
- cd infrastructure/nextcloud && pyb image publish
|
||||||
|
|
||||||
#.nextcloud-integrationtest:
|
|
||||||
# stage: integrationtest
|
|
||||||
# image: registry.gitlab.com/gitlab-org/cluster-integration/helm-install-image/releases/3.7.1-kube-1.20.11-alpine-3.14
|
|
||||||
# services:
|
|
||||||
# - name: registry.gitlab.com/gitlab-org/cluster-integration/test-utils/k3s-gitlab-ci/releases/v1.22.2-k3s2
|
|
||||||
# alias: k3s
|
|
||||||
# script:
|
|
||||||
# - apk add curl sudo bash
|
|
||||||
# - apk add wget curl bash sudo openjdk8
|
|
||||||
# - wget -P /etc/apk/keys/ https://alpine-pkgs.sgerrand.com/sgerrand.rsa.pub
|
|
||||||
# - apk add --no-cache --repository=https://apkproxy.herokuapp.com/sgerrand/alpine-pkg-leiningen leiningen
|
|
||||||
#
|
|
||||||
# - mkdir -p ${HOME}/.kube/
|
|
||||||
# - curl -f k3s:8081 > ${HOME}/.kube/config
|
|
||||||
# - kubectl version
|
|
||||||
# - kubectl cluster-info
|
|
||||||
# - echo "---------- Integration test -------------"
|
|
||||||
# - pwd
|
|
||||||
# - cd ./src/test/resources/local-integration-test/ && ./setup-local-s3-on-k3d.sh
|
|
|
@ -41,7 +41,8 @@ Development happens at: https://repo.prod.meissa.de/meissa/c4k-nextcloud
|
||||||
|
|
||||||
Mirrors are:
|
Mirrors are:
|
||||||
|
|
||||||
* https://gitlab.com/domaindrivenarchitecture/c4k-nextcloud (issues and PR, CI)
|
* https://codeberg.org/meissa/c4k-nextcloud (Issues and PR)
|
||||||
|
* https://gitlab.com/domaindrivenarchitecture/c4k-nextcloud (CI)
|
||||||
* https://github.com/DomainDrivenArchitecture/c4k-nextcloud
|
* https://github.com/DomainDrivenArchitecture/c4k-nextcloud
|
||||||
|
|
||||||
For more details about our repository model see: https://repo.prod.meissa.de/meissa/federate-your-repos
|
For more details about our repository model see: https://repo.prod.meissa.de/meissa/federate-your-repos
|
||||||
|
@ -49,6 +50,6 @@ For more details about our repository model see: https://repo.prod.meissa.de/mei
|
||||||
|
|
||||||
## License
|
## License
|
||||||
|
|
||||||
Copyright © 2021 meissa GmbH
|
Copyright © 2021, 2022, 2023, 2024 meissa GmbH
|
||||||
Licensed under the [Apache License, Version 2.0](LICENSE) (the "License")
|
Licensed under the [Apache License, Version 2.0](LICENSE) (the "License")
|
||||||
Pls. find licenses of our subcomponents [here](doc/SUBCOMPONENT_LICENSE)
|
Pls. find licenses of our subcomponents [here](doc/SUBCOMPONENT_LICENSE)
|
||||||
|
|
|
@ -1,7 +0,0 @@
|
||||||
{:postgres-db-user "nextcloud"
|
|
||||||
:postgres-db-password "dbpass"
|
|
||||||
:nextcloud-admin-user "cloudadmin"
|
|
||||||
:nextcloud-admin-password "cloudpassword"
|
|
||||||
:aws-access-key-id ""
|
|
||||||
:aws-secret-access-key ""
|
|
||||||
:restic-password "test-password"}
|
|
29
build.py
29
build.py
|
@ -33,7 +33,7 @@ def initialize(project):
|
||||||
f"target/uberjar/{name}-standalone.jar",
|
f"target/uberjar/{name}-standalone.jar",
|
||||||
f"target/frontend-build/{name}.js",
|
f"target/frontend-build/{name}.js",
|
||||||
],
|
],
|
||||||
"release_main_branch": "master",
|
"release_main_branch": "main",
|
||||||
}
|
}
|
||||||
|
|
||||||
build = ReleaseMixin(project, input)
|
build = ReleaseMixin(project, input)
|
||||||
|
@ -41,18 +41,18 @@ def initialize(project):
|
||||||
|
|
||||||
|
|
||||||
@task
|
@task
|
||||||
def test_clj(project):
|
def test_clj():
|
||||||
run("lein test", shell=True, check=True)
|
run("lein test", shell=True, check=True)
|
||||||
|
|
||||||
|
|
||||||
@task
|
@task
|
||||||
def test_cljs(project):
|
def test_cljs():
|
||||||
run("shadow-cljs compile test", shell=True, check=True)
|
run("shadow-cljs compile test", shell=True, check=True)
|
||||||
run("node target/node-tests.js", shell=True, check=True)
|
run("node target/node-tests.js", shell=True, check=True)
|
||||||
|
|
||||||
|
|
||||||
@task
|
@task
|
||||||
def test_schema(project):
|
def test_schema():
|
||||||
run("lein uberjar", shell=True, check=True)
|
run("lein uberjar", shell=True, check=True)
|
||||||
run(
|
run(
|
||||||
"java -jar target/uberjar/c4k-nextcloud-standalone.jar "
|
"java -jar target/uberjar/c4k-nextcloud-standalone.jar "
|
||||||
|
@ -63,6 +63,11 @@ def test_schema(project):
|
||||||
check=True,
|
check=True,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@task
|
||||||
|
def test():
|
||||||
|
test_clj()
|
||||||
|
test_cljs()
|
||||||
|
test_schema()
|
||||||
|
|
||||||
@task
|
@task
|
||||||
def report_frontend(project):
|
def report_frontend(project):
|
||||||
|
@ -148,11 +153,11 @@ def upload_clj(project):
|
||||||
|
|
||||||
@task
|
@task
|
||||||
def lint(project):
|
def lint(project):
|
||||||
#run(
|
run(
|
||||||
# "lein eastwood",
|
"lein eastwood",
|
||||||
# shell=True,
|
shell=True,
|
||||||
# check=True,
|
check=True,
|
||||||
#)
|
)
|
||||||
run(
|
run(
|
||||||
"lein ancient check",
|
"lein ancient check",
|
||||||
shell=True,
|
shell=True,
|
||||||
|
@ -222,7 +227,7 @@ def release(project):
|
||||||
def linttest(project, release_type):
|
def linttest(project, release_type):
|
||||||
build = get_devops_build(project)
|
build = get_devops_build(project)
|
||||||
build.update_release_type(release_type)
|
build.update_release_type(release_type)
|
||||||
test_clj(project)
|
test_clj()
|
||||||
test_cljs(project)
|
test_cljs()
|
||||||
test_schema(project)
|
test_schema()
|
||||||
lint(project)
|
lint(project)
|
||||||
|
|
|
@ -1,6 +0,0 @@
|
||||||
{:fqdn "cloudhost"
|
|
||||||
:issuer :staging
|
|
||||||
:nextcloud-data-volume-path "/var/cloud"
|
|
||||||
:postgres-data-volume-path "/var/postgres"
|
|
||||||
:restic-repository "s3://k3stesthost/mybucket"
|
|
||||||
:local-integration-test true}
|
|
|
@ -7,40 +7,28 @@
|
||||||
* Cloud stores files on `/var/jira`, these files are backuped. If you create a jira xml backup located in /var/jira this file will also be backed up.
|
* Cloud stores files on `/var/jira`, these files are backuped. If you create a jira xml backup located in /var/jira this file will also be backed up.
|
||||||
* postgres db is backed up as pgdump
|
* postgres db is backed up as pgdump
|
||||||
|
|
||||||
## Manual init the restic repository for the first time
|
## Manual backup
|
||||||
|
|
||||||
1. Scale backup-restore deployment up:
|
|
||||||
`kubectl scale deployment backup-restore --replicas=1`
|
|
||||||
1. exec into pod and execute restore pod
|
|
||||||
`kubectl exec -it backup-restore -- /usr/local/bin/init.sh`
|
|
||||||
1. Scale backup-restore deployment down:
|
|
||||||
`kubectl scale deployment backup-restore --replicas=0`
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
## Manual backup the restic repository for the first time
|
|
||||||
|
|
||||||
1. Scale Cloud deployment down:
|
1. Scale Cloud deployment down:
|
||||||
`kubectl scale deployment cloud-deployment --replicas=0`
|
`kubectl -n nextcloud scale deployment cloud-deployment --replicas=0`
|
||||||
1. Scale backup-restore deployment up:
|
1. Scale backup-restore deployment up:
|
||||||
`kubectl scale deployment backup-restore --replicas=1`
|
`kubectl -n nextcloud scale deployment backup-restore --replicas=1`
|
||||||
1. exec into pod and execute restore pod
|
1. exec into pod and execute restore pod
|
||||||
`kubectl exec -it backup-restore -- /usr/local/bin/backup.sh`
|
`kubectl -n nextcloud exec -it backup-restore -- backup.bb`
|
||||||
1. Scale backup-restore deployment down:
|
1. Scale backup-restore deployment down:
|
||||||
`kubectl scale deployment backup-restore --replicas=0`
|
`kubectl -n nextcloud scale deployment backup-restore --replicas=0`
|
||||||
1. Scale Cloud deployment up:
|
1. Scale Cloud deployment up:
|
||||||
`kubectl scale deployment cloud-deployment --replicas=1`
|
`kubectl -n nextcloud scale deployment cloud-deployment --replicas=1`
|
||||||
|
|
||||||
|
|
||||||
## Manual restore
|
## Manual restore
|
||||||
|
|
||||||
1. Scale Cloud deployment down:
|
1. Scale Cloud deployment down:
|
||||||
`kubectl scale deployment cloud-deployment --replicas=0`
|
`kubectl -n nextcloud scale deployment cloud-deployment --replicas=0`
|
||||||
2. Scale backup-restore deployment up:
|
2. Scale backup-restore deployment up:
|
||||||
`kubectl scale deployment backup-restore --replicas=1`
|
`kubectl -n nextcloud scale deployment backup-restore --replicas=1`
|
||||||
3. exec into pod and execute restore pod
|
3. exec into pod and execute restore pod
|
||||||
`kubectl exec -it backup-restore -- /usr/local/bin/restore.sh`
|
`kubectl -n nextcloud exec -it backup-restore -- restore.bb`
|
||||||
4. Scale backup-restore deployment down:
|
4. Scale backup-restore deployment down:
|
||||||
`kubectl scale deployment backup-restore --replicas=0`
|
`kubectl -n nextcloud scale deployment backup-restore --replicas=0`
|
||||||
5. Scale Cloud deployment up:
|
5. Scale Cloud deployment up:
|
||||||
`kubectl scale deployment cloud-deployment --replicas=1`
|
`kubectl -n nextcloud scale deployment cloud-deployment --replicas=1`
|
||||||
|
|
|
@ -5,12 +5,18 @@
|
||||||
- 4.0.3: nextcloud 22
|
- 4.0.3: nextcloud 22
|
||||||
- 5.0.0: nextcloud 23
|
- 5.0.0: nextcloud 23
|
||||||
- 6.0.0: nextcloud 24
|
- 6.0.0: nextcloud 24
|
||||||
- 7.0.0: nextcloud 25
|
- 7.0.7: nextcloud 25.0.13
|
||||||
|
- 7.1.1: nextcloud 26.0.0 (manual publish) => attention - only upgrade to 26.0.0 is working
|
||||||
|
- 7.1.0: nextcloud 26.0.13 (manual publish)
|
||||||
|
- 7.2.0: nextcloud 27 (manual publish)
|
||||||
|
- 10.0.0: nextcloud 28.0.5
|
||||||
|
- 10.1.0: nextcloud 29.0.0
|
||||||
|
- 10.4.2: nextcloud 30
|
||||||
|
|
||||||
## Uprgrading process
|
## Uprgrading process
|
||||||
|
|
||||||
1. Change the version of the docker image in the deployment to the next major version
|
1. Change the version of the docker image in the deployment to the next major version
|
||||||
- `kubectl edit deploy cloud-deployment`
|
- `kubectl -n=nextcloud edit deploy cloud-deployment`
|
||||||
- change `image: domaindrivenarchitecture/c4k-cloud:4.0.3`
|
- change `image: domaindrivenarchitecture/c4k-cloud:4.0.3`
|
||||||
2. Wait for the pod to finish restarting
|
2. Wait for the pod to finish restarting
|
||||||
3. Verify the website is working and https://URL/settings/admin/overview shows the correct version
|
3. Verify the website is working and https://URL/settings/admin/overview shows the correct version
|
||||||
|
|
41
doc/RenameDatabase.md
Normal file
41
doc/RenameDatabase.md
Normal file
|
@ -0,0 +1,41 @@
|
||||||
|
# Rename Database
|
||||||
|
|
||||||
|
## Start
|
||||||
|
|
||||||
|
1. Scale down cloud deployment
|
||||||
|
`k -n nextcloud scale deployment cloud-deployment --replicas 0`
|
||||||
|
|
||||||
|
## Change db-name in postgres
|
||||||
|
|
||||||
|
1. Connect to postgres-pod
|
||||||
|
`k -n nextcloud exec -it postgresql-... -- bash`
|
||||||
|
2. Connect to a database
|
||||||
|
`PGPASSWORD=$POSTGRES_PASSWORD psql -h postgresql-service -U $POSTGRES_USER postgres`
|
||||||
|
3. List available databases
|
||||||
|
`\l`
|
||||||
|
4. Rename database
|
||||||
|
`ALTER DATABASE cloud RENAME TO nextcloud;`
|
||||||
|
5. Verify
|
||||||
|
`\l`
|
||||||
|
6. Quit
|
||||||
|
`\q`
|
||||||
|
|
||||||
|
## Update postgres-config
|
||||||
|
|
||||||
|
1. Edit configmap
|
||||||
|
`k -n nextcloud edit configmap postgres-config`
|
||||||
|
2. Update postgres-db value
|
||||||
|
3. Save
|
||||||
|
|
||||||
|
## Update nextcloud db-name
|
||||||
|
|
||||||
|
1. Scale up nextcloud
|
||||||
|
`k -n nextcloud scale deployment cloud-deployment --replicas 1`
|
||||||
|
2. Connect
|
||||||
|
`k -n nextcloud exec -it cloud-deployment-... -- bash`
|
||||||
|
3. Update db value in config.php
|
||||||
|
`apt update`
|
||||||
|
`apt install vim`
|
||||||
|
`vim config/config.php`
|
||||||
|
4. Update dbname field
|
||||||
|
5. Verify server+website is working
|
|
@ -6,7 +6,7 @@ from ddadevops import *
|
||||||
name = "c4k-cloud"
|
name = "c4k-cloud"
|
||||||
MODULE = "backup"
|
MODULE = "backup"
|
||||||
PROJECT_ROOT_PATH = "../.."
|
PROJECT_ROOT_PATH = "../.."
|
||||||
version = "8.0.1"
|
version = "10.4.5-dev"
|
||||||
|
|
||||||
|
|
||||||
@init
|
@init
|
||||||
|
|
|
@ -1,5 +1,7 @@
|
||||||
FROM domaindrivenarchitecture/dda-backup:latest
|
FROM domaindrivenarchitecture/dda-backup:5.2.1
|
||||||
|
|
||||||
# Prepare Entrypoint Script
|
# Prepare Entrypoint Script
|
||||||
ADD resources /tmp
|
ADD resources /tmp
|
||||||
RUN /tmp/install.sh
|
RUN /tmp/install.bb
|
||||||
|
RUN init.bb
|
||||||
|
#ADD resources2 /tmp
|
||||||
|
|
35
infrastructure/backup/image/resources/backup.bb
Executable file
35
infrastructure/backup/image/resources/backup.bb
Executable file
|
@ -0,0 +1,35 @@
|
||||||
|
#!/usr/bin/env bb
|
||||||
|
(require
|
||||||
|
'[babashka.fs :as fs])
|
||||||
|
(-> "/usr/local/bin/config.clj" fs/file load-file)
|
||||||
|
|
||||||
|
(require
|
||||||
|
'[babashka.tasks :as t]
|
||||||
|
'[dda.backup.core :as bc]
|
||||||
|
'[dda.backup.restic :as rc]
|
||||||
|
'[dda.backup.postgresql :as pg]
|
||||||
|
'[dda.backup.backup :as bak]
|
||||||
|
'[config :as cf])
|
||||||
|
|
||||||
|
(defn prepare!
|
||||||
|
[]
|
||||||
|
(bc/create-aws-credentials! cf/aws-config)
|
||||||
|
(pg/create-pg-pass! cf/db-config))
|
||||||
|
|
||||||
|
(defn restic-repo-init!
|
||||||
|
[]
|
||||||
|
(rc/init! cf/file-config)
|
||||||
|
(rc/init! cf/db-role-config)
|
||||||
|
(rc/init! cf/db-config))
|
||||||
|
|
||||||
|
(defn restic-backup!
|
||||||
|
[]
|
||||||
|
(bak/backup-file! cf/file-config)
|
||||||
|
(bak/backup-db-roles! cf/db-role-config)
|
||||||
|
(bak/backup-db! cf/db-config))
|
||||||
|
|
||||||
|
(t/shell "start-maintenance.sh")
|
||||||
|
(prepare!)
|
||||||
|
(restic-repo-init!)
|
||||||
|
(restic-backup!)
|
||||||
|
(t/shell "end-maintenance.sh")
|
|
@ -1,28 +0,0 @@
|
||||||
#!/bin/bash
|
|
||||||
|
|
||||||
set -Eexo pipefail
|
|
||||||
|
|
||||||
function main() {
|
|
||||||
|
|
||||||
start-maintenance.sh
|
|
||||||
|
|
||||||
file_env AWS_ACCESS_KEY_ID
|
|
||||||
file_env AWS_SECRET_ACCESS_KEY
|
|
||||||
file_env POSTGRES_DB
|
|
||||||
file_env POSTGRES_PASSWORD
|
|
||||||
file_env POSTGRES_USER
|
|
||||||
file_env RESTIC_DAYS_TO_KEEP 30
|
|
||||||
file_env RESTIC_MONTHS_TO_KEEP 12
|
|
||||||
|
|
||||||
backup-roles 'oc_'
|
|
||||||
backup-db-dump
|
|
||||||
backup-directory '/var/backups/'
|
|
||||||
|
|
||||||
end-maintenance.sh
|
|
||||||
}
|
|
||||||
|
|
||||||
source /usr/local/lib/functions.sh
|
|
||||||
source /usr/local/lib/pg-functions.sh
|
|
||||||
source /usr/local/lib/file-functions.sh
|
|
||||||
|
|
||||||
main
|
|
3
infrastructure/backup/image/resources/bb-backup.edn
Normal file
3
infrastructure/backup/image/resources/bb-backup.edn
Normal file
|
@ -0,0 +1,3 @@
|
||||||
|
{:deps {org.clojure/spec.alpha {:mvn/version "0.4.233"}
|
||||||
|
orchestra/orchestra {:mvn/version "2021.01.01-1"}
|
||||||
|
org.domaindrivenarchitecture/dda-backup {:local/root "/usr/local/lib/dda-backup"}}}
|
3
infrastructure/backup/image/resources/bb.edn
Normal file
3
infrastructure/backup/image/resources/bb.edn
Normal file
|
@ -0,0 +1,3 @@
|
||||||
|
{:deps {org.clojure/spec.alpha {:mvn/version "0.4.233"}
|
||||||
|
orchestra/orchestra {:mvn/version "2021.01.01-1"}
|
||||||
|
org.domaindrivenarchitecture/dda-build {:mvn/version "0.2.0"}}}
|
61
infrastructure/backup/image/resources/config.clj
Normal file
61
infrastructure/backup/image/resources/config.clj
Normal file
|
@ -0,0 +1,61 @@
|
||||||
|
(ns config
|
||||||
|
(:require
|
||||||
|
[dda.backup.core :as bc]))
|
||||||
|
|
||||||
|
(def restic-repo {:password-file (bc/env-or-file "RESTIC_PASSWORD_FILE")
|
||||||
|
:restic-repository (bc/env-or-file "RESTIC_REPOSITORY")})
|
||||||
|
|
||||||
|
(def file-config (merge restic-repo {:backup-path "files"
|
||||||
|
:execution-directory "/var/backups"
|
||||||
|
:restore-target-directory "/var/backups/"
|
||||||
|
:files ["."]}))
|
||||||
|
|
||||||
|
(def file-restore-config (merge restic-repo {:backup-path "files"
|
||||||
|
:restore-target-directory "/var/backups/"
|
||||||
|
:clean-up-elements [".htaccess"
|
||||||
|
".reuse/"
|
||||||
|
".user.ini"
|
||||||
|
"3rdparty/"
|
||||||
|
"apps/"
|
||||||
|
"composer.json"
|
||||||
|
"composer.lock"
|
||||||
|
"console.php"
|
||||||
|
"core/"
|
||||||
|
"cron.php"
|
||||||
|
"custom_apps/"
|
||||||
|
"data/"
|
||||||
|
"dist/"
|
||||||
|
"index.html"
|
||||||
|
"index.php"
|
||||||
|
"lib/"
|
||||||
|
"ocs/"
|
||||||
|
"ocs-provider/"
|
||||||
|
"package-lock.json"
|
||||||
|
"package.json"
|
||||||
|
"public.php"
|
||||||
|
"remote.php"
|
||||||
|
"resources"
|
||||||
|
"robots.txt"
|
||||||
|
"status.php"
|
||||||
|
"themes/"
|
||||||
|
"version.php"]}))
|
||||||
|
|
||||||
|
(def db-config (merge restic-repo {:backup-path "pg-database"
|
||||||
|
:pg-host (bc/env-or-file "POSTGRES_SERVICE")
|
||||||
|
:pg-port (bc/env-or-file "POSTGRES_PORT")
|
||||||
|
:pg-db (bc/env-or-file "POSTGRES_DB")
|
||||||
|
:pg-user (bc/env-or-file "POSTGRES_USER")
|
||||||
|
:pg-password (bc/env-or-file "POSTGRES_PASSWORD")}))
|
||||||
|
|
||||||
|
(def db-role-config (merge restic-repo {:backup-path "pg-role"
|
||||||
|
:pg-role-prefix "oc_"
|
||||||
|
:pg-host (bc/env-or-file "POSTGRES_SERVICE")
|
||||||
|
:pg-port (bc/env-or-file "POSTGRES_PORT")
|
||||||
|
:pg-db (bc/env-or-file "POSTGRES_DB")
|
||||||
|
:pg-user (bc/env-or-file "POSTGRES_USER")
|
||||||
|
:pg-password (bc/env-or-file "POSTGRES_PASSWORD")}))
|
||||||
|
|
||||||
|
(def aws-config {:aws-access-key-id (bc/env-or-file "AWS_ACCESS_KEY_ID")
|
||||||
|
:aws-secret-access-key (bc/env-or-file "AWS_SECRET_ACCESS_KEY")})
|
||||||
|
|
||||||
|
(def dry-run {:dry-run true :debug true})
|
|
@ -1,19 +0,0 @@
|
||||||
#!/bin/bash
|
|
||||||
|
|
||||||
set -exo pipefail
|
|
||||||
|
|
||||||
function main() {
|
|
||||||
file_env POSTGRES_DB
|
|
||||||
file_env POSTGRES_PASSWORD
|
|
||||||
file_env POSTGRES_USER
|
|
||||||
|
|
||||||
create-pg-pass
|
|
||||||
|
|
||||||
while true; do
|
|
||||||
sleep 1m
|
|
||||||
done
|
|
||||||
}
|
|
||||||
|
|
||||||
source /usr/local/lib/functions.sh
|
|
||||||
source /usr/local/lib/pg-functions.sh
|
|
||||||
main
|
|
|
@ -1,17 +0,0 @@
|
||||||
#!/bin/bash
|
|
||||||
|
|
||||||
set -Eexo pipefail
|
|
||||||
|
|
||||||
function main() {
|
|
||||||
file_env POSTGRES_DB
|
|
||||||
file_env POSTGRES_PASSWORD
|
|
||||||
file_env POSTGRES_USER
|
|
||||||
|
|
||||||
create-pg-pass
|
|
||||||
|
|
||||||
/usr/local/bin/backup.sh
|
|
||||||
}
|
|
||||||
|
|
||||||
source /usr/local/lib/functions.sh
|
|
||||||
source /usr/local/lib/pg-functions.sh
|
|
||||||
main
|
|
3
infrastructure/backup/image/resources/init.bb
Executable file
3
infrastructure/backup/image/resources/init.bb
Executable file
|
@ -0,0 +1,3 @@
|
||||||
|
#!/usr/bin/env bb
|
||||||
|
|
||||||
|
(println "initialized")
|
|
@ -1,17 +0,0 @@
|
||||||
#!/bin/bash
|
|
||||||
|
|
||||||
set -Eexo pipefail
|
|
||||||
|
|
||||||
function main() {
|
|
||||||
file_env AWS_ACCESS_KEY_ID
|
|
||||||
file_env AWS_SECRET_ACCESS_KEY
|
|
||||||
|
|
||||||
init-role-repo
|
|
||||||
init-database-repo
|
|
||||||
init-file-repo
|
|
||||||
}
|
|
||||||
|
|
||||||
source /usr/local/lib/functions.sh
|
|
||||||
source /usr/local/lib/pg-functions.sh
|
|
||||||
source /usr/local/lib/file-functions.sh
|
|
||||||
main
|
|
19
infrastructure/backup/image/resources/install.bb
Executable file
19
infrastructure/backup/image/resources/install.bb
Executable file
|
@ -0,0 +1,19 @@
|
||||||
|
#!/usr/bin/env bb
|
||||||
|
|
||||||
|
(require
|
||||||
|
'[dda.image.ubuntu :as ub]
|
||||||
|
'[dda.image.install :as in])
|
||||||
|
|
||||||
|
(ub/upgrade-system!)
|
||||||
|
(in/install! "bb-backup.edn" :target-name "bb.edn" :mod "0440")
|
||||||
|
(in/install! "config.clj" :mod "0440")
|
||||||
|
(in/install! "init.bb")
|
||||||
|
(in/install! "backup.bb")
|
||||||
|
(in/install! "restore.bb")
|
||||||
|
(in/install! "list-snapshots.bb")
|
||||||
|
(in/install! "start-maintenance.sh")
|
||||||
|
(in/install! "end-maintenance.sh")
|
||||||
|
(in/install! "restore.bb")
|
||||||
|
(in/install! "wait.bb")
|
||||||
|
|
||||||
|
(ub/cleanup-container!)
|
|
@ -1,21 +0,0 @@
|
||||||
#!/bin/bash
|
|
||||||
|
|
||||||
set -exo pipefail
|
|
||||||
|
|
||||||
function main() {
|
|
||||||
{
|
|
||||||
install -m 0700 /tmp/entrypoint.sh /
|
|
||||||
install -m 0700 /tmp/entrypoint-start-and-wait.sh /
|
|
||||||
|
|
||||||
install -m 0700 /tmp/init.sh /usr/local/bin/
|
|
||||||
install -m 0700 /tmp/backup.sh /usr/local/bin/
|
|
||||||
install -m 0700 /tmp/restore.sh /usr/local/bin/
|
|
||||||
install -m 0700 /tmp/list-snapshots.sh /usr/local/bin/
|
|
||||||
install -m 0700 /tmp/start-maintenance.sh /usr/local/bin/
|
|
||||||
install -m 0700 /tmp/end-maintenance.sh /usr/local/bin/
|
|
||||||
cleanupDocker
|
|
||||||
} > /dev/null
|
|
||||||
}
|
|
||||||
|
|
||||||
source /tmp/install_functions_debian.sh
|
|
||||||
main
|
|
22
infrastructure/backup/image/resources/list-snapshots.bb
Executable file
22
infrastructure/backup/image/resources/list-snapshots.bb
Executable file
|
@ -0,0 +1,22 @@
|
||||||
|
#!/usr/bin/env bb
|
||||||
|
(require
|
||||||
|
'[babashka.fs :as fs])
|
||||||
|
(-> "/usr/local/bin/config.clj" fs/file load-file)
|
||||||
|
|
||||||
|
(require
|
||||||
|
'[dda.backup.core :as bc]
|
||||||
|
'[dda.backup.restic :as rc]
|
||||||
|
'[config :as cf])
|
||||||
|
|
||||||
|
(defn prepare!
|
||||||
|
[]
|
||||||
|
(bc/create-aws-credentials! cf/aws-config))
|
||||||
|
|
||||||
|
(defn list-snapshots!
|
||||||
|
[]
|
||||||
|
(rc/list-snapshots! cf/file-config)
|
||||||
|
(rc/list-snapshots! cf/db-role-config)
|
||||||
|
(rc/list-snapshots! cf/db-config))
|
||||||
|
|
||||||
|
(prepare!)
|
||||||
|
(list-snapshots!)
|
|
@ -1,31 +0,0 @@
|
||||||
#!/bin/bash
|
|
||||||
|
|
||||||
set -exo pipefail
|
|
||||||
|
|
||||||
function list-snapshot-files() {
|
|
||||||
if [ -z ${CERTIFICATE_FILE} ];
|
|
||||||
then
|
|
||||||
restic -r ${RESTIC_REPOSITORY}/${backup_file_path} snapshots
|
|
||||||
else
|
|
||||||
restic -r ${RESTIC_REPOSITORY}/${backup_file_path} snapshots --cacert ${CERTIFICATE_FILE}
|
|
||||||
fi
|
|
||||||
}
|
|
||||||
|
|
||||||
function main() {
|
|
||||||
file_env AWS_ACCESS_KEY_ID
|
|
||||||
file_env AWS_SECRET_ACCESS_KEY
|
|
||||||
|
|
||||||
file_env POSTGRES_DB
|
|
||||||
file_env POSTGRES_PASSWORD
|
|
||||||
file_env POSTGRES_USER
|
|
||||||
|
|
||||||
list-snapshot-roles
|
|
||||||
list-snapshot-db
|
|
||||||
list-snapshot-files
|
|
||||||
}
|
|
||||||
|
|
||||||
source /usr/local/lib/functions.sh
|
|
||||||
source /usr/local/lib/file-functions.sh
|
|
||||||
source /usr/local/lib/pg-functions.sh
|
|
||||||
|
|
||||||
main
|
|
29
infrastructure/backup/image/resources/restore.bb
Executable file
29
infrastructure/backup/image/resources/restore.bb
Executable file
|
@ -0,0 +1,29 @@
|
||||||
|
#!/usr/bin/env bb
|
||||||
|
(require
|
||||||
|
'[babashka.fs :as fs])
|
||||||
|
(-> "/usr/local/bin/config.clj" fs/file load-file)
|
||||||
|
|
||||||
|
(require
|
||||||
|
'[babashka.tasks :as t]
|
||||||
|
'[dda.backup.core :as bc]
|
||||||
|
'[dda.backup.postgresql :as pg]
|
||||||
|
'[dda.backup.restore :as rs]
|
||||||
|
'[config :as cf])
|
||||||
|
|
||||||
|
(defn prepare!
|
||||||
|
[]
|
||||||
|
(bc/create-aws-credentials! cf/aws-config)
|
||||||
|
(pg/create-pg-pass! cf/db-config))
|
||||||
|
|
||||||
|
(defn restic-restore!
|
||||||
|
[]
|
||||||
|
(pg/drop-create-db! cf/db-config)
|
||||||
|
(rs/restore-db-roles! cf/db-role-config)
|
||||||
|
(rs/restore-db! cf/db-config)
|
||||||
|
(rs/restore-file! cf/file-restore-config)
|
||||||
|
)
|
||||||
|
|
||||||
|
(t/shell "start-maintenance.sh")
|
||||||
|
(prepare!)
|
||||||
|
(restic-restore!)
|
||||||
|
(t/shell "end-maintenance.sh")
|
|
@ -1,33 +0,0 @@
|
||||||
#!/bin/bash
|
|
||||||
|
|
||||||
set -Eexo pipefail
|
|
||||||
|
|
||||||
function main() {
|
|
||||||
local role_snapshot_id="${1:-latest}"
|
|
||||||
local db_snapshot_id="${2:-latest}"
|
|
||||||
local file_snapshot_id="${3:-latest}"
|
|
||||||
|
|
||||||
|
|
||||||
start-maintenance.sh
|
|
||||||
|
|
||||||
file_env AWS_ACCESS_KEY_ID
|
|
||||||
file_env AWS_SECRET_ACCESS_KEY
|
|
||||||
|
|
||||||
file_env POSTGRES_DB
|
|
||||||
file_env POSTGRES_PASSWORD
|
|
||||||
file_env POSTGRES_USER
|
|
||||||
|
|
||||||
drop-create-db
|
|
||||||
|
|
||||||
restore-roles ${role_snapshot_id}
|
|
||||||
restore-db ${db_snapshot_id}
|
|
||||||
restore-directory '/var/backups/' ${file_snapshot_id}
|
|
||||||
|
|
||||||
end-maintenance.sh
|
|
||||||
}
|
|
||||||
|
|
||||||
source /usr/local/lib/functions.sh
|
|
||||||
source /usr/local/lib/pg-functions.sh
|
|
||||||
source /usr/local/lib/file-functions.sh
|
|
||||||
|
|
||||||
main "$@"
|
|
21
infrastructure/backup/image/resources/wait.bb
Executable file
21
infrastructure/backup/image/resources/wait.bb
Executable file
|
@ -0,0 +1,21 @@
|
||||||
|
#!/usr/bin/env bb
|
||||||
|
(require
|
||||||
|
'[babashka.fs :as fs])
|
||||||
|
(-> "/usr/local/bin/config.clj" fs/file load-file)
|
||||||
|
|
||||||
|
(require
|
||||||
|
'[dda.backup.core :as bc]
|
||||||
|
'[dda.backup.postgresql :as pg]
|
||||||
|
'[config :as cf])
|
||||||
|
|
||||||
|
(defn prepare!
|
||||||
|
[]
|
||||||
|
(bc/create-aws-credentials! cf/aws-config)
|
||||||
|
(pg/create-pg-pass! cf/db-config))
|
||||||
|
|
||||||
|
(defn wait! []
|
||||||
|
(while true
|
||||||
|
(Thread/sleep 1000)))
|
||||||
|
|
||||||
|
(prepare!)
|
||||||
|
(wait!)
|
3
infrastructure/backup/image/resources2/bb.edn
Normal file
3
infrastructure/backup/image/resources2/bb.edn
Normal file
|
@ -0,0 +1,3 @@
|
||||||
|
{:deps {org.clojure/spec.alpha {:mvn/version "0.4.233"}
|
||||||
|
orchestra/orchestra {:mvn/version "2021.01.01-1"}
|
||||||
|
org.domaindrivenarchitecture/dda-backup {:local/root "/usr/local/lib/dda-backup"}}}
|
9
infrastructure/backup/image/resources2/exports.sh
Normal file
9
infrastructure/backup/image/resources2/exports.sh
Normal file
|
@ -0,0 +1,9 @@
|
||||||
|
export RESTIC_PASSWORD_FILE=/tmp/file_password
|
||||||
|
export RESTIC_REPOSITORY=/var/restic-repo
|
||||||
|
export POSTGRES_SERVICE=dummy
|
||||||
|
export POSTGRES_PORT=dummy
|
||||||
|
export POSTGRES_DB=dummy
|
||||||
|
export POSTGRES_USER=dummy
|
||||||
|
export POSTGRES_PASSWORD=dummy
|
||||||
|
export AWS_ACCESS_KEY_ID=dummy
|
||||||
|
export AWS_SECRET_ACCESS_KEY=dummy
|
1
infrastructure/backup/image/resources2/file_password
Normal file
1
infrastructure/backup/image/resources2/file_password
Normal file
|
@ -0,0 +1 @@
|
||||||
|
oldPassword
|
51
infrastructure/backup/image/resources2/test.bb
Executable file
51
infrastructure/backup/image/resources2/test.bb
Executable file
|
@ -0,0 +1,51 @@
|
||||||
|
#!/usr/bin/env bb
|
||||||
|
(require
|
||||||
|
'[babashka.fs :as fs])
|
||||||
|
(-> "/usr/local/bin/config.clj" fs/file load-file)
|
||||||
|
|
||||||
|
(require '[babashka.tasks :as tasks]
|
||||||
|
'[dda.backup.core :as bc]
|
||||||
|
'[dda.backup.restic :as rc]
|
||||||
|
'[dda.backup.postgresql :as pg]
|
||||||
|
'[dda.backup.backup :as bak]
|
||||||
|
'[dda.backup.restore :as rs]
|
||||||
|
'[config :as cf])
|
||||||
|
|
||||||
|
(defn prepare!
|
||||||
|
[]
|
||||||
|
(tasks/shell "mkdir" "-p" "/var/backups/")
|
||||||
|
(tasks/shell "mkdir" "-p" "/var/restic-repo/")
|
||||||
|
(spit "/var/backups/file" "I was here"))
|
||||||
|
|
||||||
|
(defn restic-repo-init!
|
||||||
|
[]
|
||||||
|
(rc/init! cf/file-config)
|
||||||
|
(rc/init! (merge cf/db-role-config cf/dry-run))
|
||||||
|
(rc/init! (merge cf/db-config cf/dry-run)))
|
||||||
|
|
||||||
|
(defn restic-backup!
|
||||||
|
[]
|
||||||
|
(bak/backup-file! cf/file-config)
|
||||||
|
(bak/backup-db-roles! (merge cf/db-role-config cf/dry-run))
|
||||||
|
(bak/backup-db! (merge cf/db-config cf/dry-run)))
|
||||||
|
|
||||||
|
(defn list-snapshots!
|
||||||
|
[]
|
||||||
|
(rc/list-snapshots! cf/file-config)
|
||||||
|
(rc/list-snapshots! (merge cf/db-role-config cf/dry-run))
|
||||||
|
(rc/list-snapshots! (merge cf/db-config cf/dry-run)))
|
||||||
|
|
||||||
|
|
||||||
|
(defn restic-restore!
|
||||||
|
[]
|
||||||
|
(pg/drop-create-db! (merge cf/db-config cf/dry-run))
|
||||||
|
(rs/restore-db-roles! (merge cf/db-role-config cf/dry-run))
|
||||||
|
(rs/restore-db! (merge cf/db-config cf/dry-run))
|
||||||
|
(rs/restore-file! cf/file-restore-config))
|
||||||
|
|
||||||
|
(prepare!)
|
||||||
|
(restic-repo-init!)
|
||||||
|
(restic-backup!)
|
||||||
|
(list-snapshots!)
|
||||||
|
(restic-restore!)
|
||||||
|
|
4
infrastructure/backup/test/Dockerfile
Normal file
4
infrastructure/backup/test/Dockerfile
Normal file
|
@ -0,0 +1,4 @@
|
||||||
|
FROM c4k-cloud-backup:latest
|
||||||
|
|
||||||
|
ADD resources /tmp/
|
||||||
|
RUN RESTIC_PASSWORD_FILE=/tmp/file_password RESTIC_REPOSITORY=restic-repo POSTGRES_SERVICE=dummy POSTGRES_PORT=dummy POSTGRES_DB=dummy POSTGRES_USER=dummy POSTGRES_PASSWORD=dummy AWS_ACCESS_KEY_ID=dummy AWS_SECRET_ACCESS_KEY=dummy /tmp/test.bb
|
3
infrastructure/backup/test/resources/bb.edn
Normal file
3
infrastructure/backup/test/resources/bb.edn
Normal file
|
@ -0,0 +1,3 @@
|
||||||
|
{:deps {org.clojure/spec.alpha {:mvn/version "0.4.233"}
|
||||||
|
orchestra/orchestra {:mvn/version "2021.01.01-1"}
|
||||||
|
org.domaindrivenarchitecture/dda-backup {:local/root "/usr/local/lib/dda-backup"}}}
|
1
infrastructure/backup/test/resources/file_password
Normal file
1
infrastructure/backup/test/resources/file_password
Normal file
|
@ -0,0 +1 @@
|
||||||
|
oldPassword
|
49
infrastructure/backup/test/resources/test.bb
Executable file
49
infrastructure/backup/test/resources/test.bb
Executable file
|
@ -0,0 +1,49 @@
|
||||||
|
#!/usr/bin/env bb
|
||||||
|
(require
|
||||||
|
'[babashka.fs :as fs])
|
||||||
|
(-> "/usr/local/bin/config.clj" fs/file load-file)
|
||||||
|
|
||||||
|
(require '[babashka.tasks :as tasks]
|
||||||
|
'[dda.backup.restic :as rc]
|
||||||
|
'[dda.backup.postgresql :as pg]
|
||||||
|
'[dda.backup.backup :as bak]
|
||||||
|
'[dda.backup.restore :as rs]
|
||||||
|
'[config :as cf])
|
||||||
|
|
||||||
|
(defn prepare!
|
||||||
|
[]
|
||||||
|
(tasks/shell "mkdir" "-p" "/var/backups/")
|
||||||
|
(spit "/var/backups/file" "I was here")
|
||||||
|
(tasks/shell "mkdir" "-p" "/var/restore"))
|
||||||
|
|
||||||
|
(defn restic-repo-init!
|
||||||
|
[]
|
||||||
|
(rc/init! cf/file-config)
|
||||||
|
(rc/init! (merge cf/db-role-config cf/dry-run))
|
||||||
|
(rc/init! (merge cf/db-config cf/dry-run)))
|
||||||
|
|
||||||
|
(defn restic-backup!
|
||||||
|
[]
|
||||||
|
(bak/backup-file! cf/file-config)
|
||||||
|
(bak/backup-db-roles! (merge cf/db-role-config cf/dry-run))
|
||||||
|
(bak/backup-db! (merge cf/db-config cf/dry-run)))
|
||||||
|
|
||||||
|
(defn list-snapshots!
|
||||||
|
[]
|
||||||
|
(rc/list-snapshots! cf/file-config)
|
||||||
|
(rc/list-snapshots! (merge cf/db-role-config cf/dry-run))
|
||||||
|
(rc/list-snapshots! (merge cf/db-config cf/dry-run)))
|
||||||
|
|
||||||
|
|
||||||
|
(defn restic-restore!
|
||||||
|
[]
|
||||||
|
(pg/drop-create-db! (merge cf/db-config cf/dry-run))
|
||||||
|
(rs/restore-db-roles! (merge cf/db-role-config cf/dry-run))
|
||||||
|
(rs/restore-db! (merge cf/db-config cf/dry-run))
|
||||||
|
(rs/restore-file! (merge cf/file-restore-config cf/dry-run)))
|
||||||
|
|
||||||
|
(prepare!)
|
||||||
|
(restic-repo-init!)
|
||||||
|
(restic-backup!)
|
||||||
|
(list-snapshots!)
|
||||||
|
(restic-restore!)
|
|
@ -6,7 +6,7 @@ from ddadevops import *
|
||||||
name = 'c4k-cloud'
|
name = 'c4k-cloud'
|
||||||
MODULE = 'not_set'
|
MODULE = 'not_set'
|
||||||
PROJECT_ROOT_PATH = '../..'
|
PROJECT_ROOT_PATH = '../..'
|
||||||
version = "8.0.1"
|
version = "10.4.5-dev"
|
||||||
|
|
||||||
@init
|
@init
|
||||||
def initialize(project):
|
def initialize(project):
|
||||||
|
|
|
@ -1,4 +1,4 @@
|
||||||
FROM nextcloud:28
|
FROM nextcloud:30
|
||||||
|
|
||||||
# REQUIRES docker >= 2.10.10
|
# REQUIRES docker >= 2.10.10
|
||||||
# https://docs.docker.com/engine/release-notes/20.10/#201010
|
# https://docs.docker.com/engine/release-notes/20.10/#201010
|
||||||
|
|
10
package.json
10
package.json
|
@ -2,7 +2,7 @@
|
||||||
"name": "c4k-nextcloud",
|
"name": "c4k-nextcloud",
|
||||||
"description": "Generate c4k yaml for a nextcloud deployment.",
|
"description": "Generate c4k yaml for a nextcloud deployment.",
|
||||||
"author": "meissa GmbH",
|
"author": "meissa GmbH",
|
||||||
"version": "8.0.1",
|
"version": "10.4.5-SNAPSHOT",
|
||||||
"homepage": "https://gitlab.com/domaindrivenarchitecture/c4k-nextcloud#readme",
|
"homepage": "https://gitlab.com/domaindrivenarchitecture/c4k-nextcloud#readme",
|
||||||
"repository": "https://www.npmjs.com/package/c4k-nextcloud",
|
"repository": "https://www.npmjs.com/package/c4k-nextcloud",
|
||||||
"license": "APACHE2",
|
"license": "APACHE2",
|
||||||
|
@ -23,11 +23,11 @@
|
||||||
"url": "https://gitlab.com/domaindrivenarchitecture/c4k-nextcloud/issues"
|
"url": "https://gitlab.com/domaindrivenarchitecture/c4k-nextcloud/issues"
|
||||||
},
|
},
|
||||||
"dependencies": {
|
"dependencies": {
|
||||||
"js-base64": "^3.6.1",
|
"js-base64": "^3.7.7",
|
||||||
"js-yaml": "^4.0.0"
|
"js-yaml": "^4.1.0"
|
||||||
},
|
},
|
||||||
"devDependencies": {
|
"devDependencies": {
|
||||||
"shadow-cljs": "^2.11.18",
|
"shadow-cljs": "^2.28.20",
|
||||||
"source-map-support": "^0.5.19"
|
"source-map-support": "^0.5.21"
|
||||||
}
|
}
|
||||||
}
|
}
|
12
project.clj
12
project.clj
|
@ -1,11 +1,11 @@
|
||||||
(defproject org.domaindrivenarchitecture/c4k-nextcloud "8.0.1"
|
(defproject org.domaindrivenarchitecture/c4k-nextcloud "10.4.5-SNAPSHOT"
|
||||||
:description "nextcloud c4k-installation package"
|
:description "nextcloud c4k-installation package"
|
||||||
:url "https://domaindrivenarchitecture.org"
|
:url "https://domaindrivenarchitecture.org"
|
||||||
:license {:name "Apache License, Version 2.0"
|
:license {:name "Apache License, Version 2.0"
|
||||||
:url "https://www.apache.org/licenses/LICENSE-2.0.html"}
|
:url "https://www.apache.org/licenses/LICENSE-2.0.html"}
|
||||||
:dependencies [[org.clojure/clojure "1.11.2"]
|
:dependencies [[org.clojure/clojure "1.12.0"]
|
||||||
[org.clojure/tools.reader "1.4.1"]
|
[org.clojure/tools.reader "1.5.0"]
|
||||||
[org.domaindrivenarchitecture/c4k-common-clj "6.2.2"]
|
[org.domaindrivenarchitecture/c4k-common-clj "9.0.1"]
|
||||||
[hickory "0.7.1" :exclusions [viebel/codox-klipse-theme]]]
|
[hickory "0.7.1" :exclusions [viebel/codox-klipse-theme]]]
|
||||||
:target-path "target/%s/"
|
:target-path "target/%s/"
|
||||||
:source-paths ["src/main/cljc"
|
:source-paths ["src/main/cljc"
|
||||||
|
@ -23,9 +23,9 @@
|
||||||
:main dda.c4k-nextcloud.uberjar
|
:main dda.c4k-nextcloud.uberjar
|
||||||
:uberjar-name "c4k-nextcloud-standalone.jar"
|
:uberjar-name "c4k-nextcloud-standalone.jar"
|
||||||
:dependencies [[org.clojure/tools.cli "1.1.230"]
|
:dependencies [[org.clojure/tools.cli "1.1.230"]
|
||||||
[ch.qos.logback/logback-classic "1.5.3"
|
[ch.qos.logback/logback-classic "1.5.16"
|
||||||
:exclusions [com.sun.mail/javax.mail]]
|
:exclusions [com.sun.mail/javax.mail]]
|
||||||
[org.slf4j/jcl-over-slf4j "2.0.12"]
|
[org.slf4j/jcl-over-slf4j "2.0.16"]
|
||||||
[com.github.clj-easy/graal-build-time "1.0.5"]]}}
|
[com.github.clj-easy/graal-build-time "1.0.5"]]}}
|
||||||
:release-tasks [["test"]
|
:release-tasks [["test"]
|
||||||
["vcs" "assert-committed"]
|
["vcs" "assert-committed"]
|
||||||
|
|
|
@ -4,7 +4,7 @@
|
||||||
"src/test/cljc"
|
"src/test/cljc"
|
||||||
"src/test/cljs"
|
"src/test/cljs"
|
||||||
"src/test/resources"]
|
"src/test/resources"]
|
||||||
:dependencies [[org.domaindrivenarchitecture/c4k-common-cljs "6.1.3"]
|
:dependencies [[org.domaindrivenarchitecture/c4k-common-cljs "9.0.1"]
|
||||||
[hickory "0.7.1"]]
|
[hickory "0.7.1"]]
|
||||||
:builds {:frontend {:target :browser
|
:builds {:frontend {:target :browser
|
||||||
:modules {:main {:init-fn dda.c4k-nextcloud.browser/init}}
|
:modules {:main {:init-fn dda.c4k-nextcloud.browser/init}}
|
||||||
|
|
|
@ -2,14 +2,14 @@
|
||||||
(:gen-class)
|
(:gen-class)
|
||||||
(:require
|
(:require
|
||||||
[dda.c4k-common.uberjar :as uberjar]
|
[dda.c4k-common.uberjar :as uberjar]
|
||||||
[dda.c4k-nextcloud.nextcloud :as nextcloud]
|
|
||||||
[dda.c4k-nextcloud.core :as core]))
|
[dda.c4k-nextcloud.core :as core]))
|
||||||
|
|
||||||
(defn -main [& cmd-args]
|
(defn -main [& cmd-args]
|
||||||
(uberjar/main-common
|
(uberjar/main-cm
|
||||||
"c4k-nextcloud"
|
"c4k-nextcloud"
|
||||||
nextcloud/config?
|
core/config?
|
||||||
nextcloud/auth?
|
core/auth?
|
||||||
core/config-defaults
|
core/config-defaults
|
||||||
core/k8s-objects
|
core/config-objects
|
||||||
|
core/auth-objects
|
||||||
cmd-args))
|
cmd-args))
|
||||||
|
|
|
@ -1,6 +1,8 @@
|
||||||
(ns dda.c4k-nextcloud.backup
|
(ns dda.c4k-nextcloud.backup
|
||||||
(:require
|
(:require
|
||||||
[clojure.spec.alpha :as s]
|
[clojure.spec.alpha :as s]
|
||||||
|
#?(:clj [orchestra.core :refer [defn-spec]]
|
||||||
|
:cljs [orchestra.core :refer-macros [defn-spec]])
|
||||||
[dda.c4k-common.yaml :as yaml]
|
[dda.c4k-common.yaml :as yaml]
|
||||||
[dda.c4k-common.base64 :as b64]
|
[dda.c4k-common.base64 :as b64]
|
||||||
[dda.c4k-common.common :as cm]
|
[dda.c4k-common.common :as cm]
|
||||||
|
@ -12,26 +14,35 @@
|
||||||
(s/def ::restic-password cp/bash-env-string?)
|
(s/def ::restic-password cp/bash-env-string?)
|
||||||
(s/def ::restic-repository cp/bash-env-string?)
|
(s/def ::restic-repository cp/bash-env-string?)
|
||||||
|
|
||||||
|
(s/def ::config (s/keys :req-un [::restic-repository]))
|
||||||
|
|
||||||
|
(s/def ::auth (s/keys :req-un [::restic-password ::aws-access-key-id ::aws-secret-access-key]))
|
||||||
|
|
||||||
|
|
||||||
#?(:cljs
|
#?(:cljs
|
||||||
(defmethod yaml/load-resource :backup [resource-name]
|
(defmethod yaml/load-resource :backup [resource-name]
|
||||||
(get (inline-resources "backup") resource-name)))
|
(get (inline-resources "backup") resource-name)))
|
||||||
|
|
||||||
(defn generate-config [my-conf]
|
(defn-spec generate-config map?
|
||||||
|
[my-conf ::config]
|
||||||
(let [{:keys [restic-repository]} my-conf]
|
(let [{:keys [restic-repository]} my-conf]
|
||||||
(->
|
(->
|
||||||
(yaml/load-as-edn "backup/config.yaml")
|
(yaml/load-as-edn "backup/config.yaml")
|
||||||
(cm/replace-key-value :restic-repository restic-repository))))
|
(cm/replace-key-value :restic-repository restic-repository))))
|
||||||
|
|
||||||
(defn generate-cron []
|
(defn-spec generate-cron map?
|
||||||
|
[]
|
||||||
(yaml/from-string (yaml/load-resource "backup/cron.yaml")))
|
(yaml/from-string (yaml/load-resource "backup/cron.yaml")))
|
||||||
|
|
||||||
(defn generate-backup-restore-deployment [my-conf]
|
(defn-spec generate-backup-restore-deployment map?
|
||||||
|
[my-conf ::config]
|
||||||
(let [backup-restore-yaml (yaml/load-as-edn "backup/backup-restore-deployment.yaml")]
|
(let [backup-restore-yaml (yaml/load-as-edn "backup/backup-restore-deployment.yaml")]
|
||||||
(if (and (contains? my-conf :local-integration-test) (= true (:local-integration-test my-conf)))
|
(if (and (contains? my-conf :local-integration-test) (= true (:local-integration-test my-conf)))
|
||||||
(cm/replace-named-value backup-restore-yaml "CERTIFICATE_FILE" "/var/run/secrets/localstack-secrets/ca.crt")
|
(cm/replace-named-value backup-restore-yaml "CERTIFICATE_FILE" "/var/run/secrets/localstack-secrets/ca.crt")
|
||||||
backup-restore-yaml)))
|
backup-restore-yaml)))
|
||||||
|
|
||||||
(defn generate-secret [my-auth]
|
(defn-spec generate-secret map?
|
||||||
|
[my-auth ::auth]
|
||||||
(let [{:keys [aws-access-key-id aws-secret-access-key restic-password]} my-auth]
|
(let [{:keys [aws-access-key-id aws-secret-access-key restic-password]} my-auth]
|
||||||
(->
|
(->
|
||||||
(yaml/load-as-edn "backup/secret.yaml")
|
(yaml/load-as-edn "backup/secret.yaml")
|
||||||
|
|
|
@ -1,5 +1,6 @@
|
||||||
(ns dda.c4k-nextcloud.core
|
(ns dda.c4k-nextcloud.core
|
||||||
(:require
|
(:require
|
||||||
|
[clojure.spec.alpha :as s]
|
||||||
#?(:clj [orchestra.core :refer [defn-spec]]
|
#?(:clj [orchestra.core :refer [defn-spec]]
|
||||||
:cljs [orchestra.core :refer-macros [defn-spec]])
|
:cljs [orchestra.core :refer-macros [defn-spec]])
|
||||||
[dda.c4k-common.common :as cm]
|
[dda.c4k-common.common :as cm]
|
||||||
|
@ -8,36 +9,58 @@
|
||||||
[dda.c4k-common.postgres :as postgres]
|
[dda.c4k-common.postgres :as postgres]
|
||||||
[dda.c4k-nextcloud.nextcloud :as nextcloud]
|
[dda.c4k-nextcloud.nextcloud :as nextcloud]
|
||||||
[dda.c4k-nextcloud.backup :as backup]
|
[dda.c4k-nextcloud.backup :as backup]
|
||||||
[dda.c4k-common.monitoring :as mon]))
|
[dda.c4k-common.monitoring :as mon]
|
||||||
|
[dda.c4k-common.namespace :as ns]))
|
||||||
|
|
||||||
(def default-storage-class :local-path)
|
(def config-defaults {:namespace "nextcloud"
|
||||||
|
:issuer "staging"
|
||||||
|
:pvc-storage-class-name "hcloud-volumes-encrypted"
|
||||||
|
:pv-storage-size-gb 200})
|
||||||
|
|
||||||
(def config-defaults {:issuer "staging"})
|
(def config? (s/merge ::nextcloud/config
|
||||||
|
::backup/config))
|
||||||
|
|
||||||
(defn-spec k8s-objects cp/map-or-seq?
|
(def auth? (s/merge ::nextcloud/auth
|
||||||
[config nextcloud/config?
|
::backup/auth))
|
||||||
auth nextcloud/auth?]
|
|
||||||
(let [nextcloud-default-storage-config {:pvc-storage-class-name default-storage-class
|
|
||||||
:pv-storage-size-gb 200}]
|
(defn-spec config-objects cp/map-or-seq?
|
||||||
|
[config config?]
|
||||||
|
(let [resolved-config (merge config-defaults config)]
|
||||||
(map yaml/to-string
|
(map yaml/to-string
|
||||||
(filter
|
(filter
|
||||||
#(not (nil? %))
|
#(not (nil? %))
|
||||||
(cm/concat-vec
|
(cm/concat-vec
|
||||||
[(postgres/generate-config {:postgres-size :8gb :db-name "nextcloud"})
|
(ns/generate resolved-config)
|
||||||
(postgres/generate-secret auth)
|
(postgres/generate-config (merge resolved-config {:postgres-image "postgres:16"
|
||||||
(postgres/generate-pvc {:pv-storage-size-gb 50
|
:postgres-size :8gb
|
||||||
:pvc-storage-class-name default-storage-class})
|
:db-name "cloud"
|
||||||
(postgres/generate-deployment)
|
:pv-storage-size-gb 50}))
|
||||||
(postgres/generate-service)
|
[(nextcloud/generate-pvc resolved-config)
|
||||||
(nextcloud/generate-secret auth)
|
(nextcloud/generate-deployment resolved-config)
|
||||||
(nextcloud/generate-pvc (merge nextcloud-default-storage-config config))
|
|
||||||
(nextcloud/generate-deployment config)
|
|
||||||
(nextcloud/generate-service)]
|
(nextcloud/generate-service)]
|
||||||
(nextcloud/generate-ingress-and-cert config)
|
(nextcloud/generate-ingress-and-cert resolved-config)
|
||||||
(when (:contains? config :restic-repository)
|
(when (:contains? resolved-config :restic-repository)
|
||||||
[(backup/generate-config config)
|
[(backup/generate-config resolved-config)
|
||||||
(backup/generate-secret auth)
|
|
||||||
(backup/generate-cron)
|
(backup/generate-cron)
|
||||||
(backup/generate-backup-restore-deployment config)])
|
(backup/generate-backup-restore-deployment resolved-config)])
|
||||||
(when (:contains? config :mon-cfg)
|
(when (:contains? resolved-config :mon-cfg)
|
||||||
(mon/generate (:mon-cfg config) (:mon-auth auth))))))))
|
(mon/generate-config)))))))
|
||||||
|
|
||||||
|
(defn-spec auth-objects cp/map-or-seq?
|
||||||
|
[config config?
|
||||||
|
auth auth?]
|
||||||
|
(let [resolved-config (merge config-defaults config)]
|
||||||
|
(map yaml/to-string
|
||||||
|
(filter
|
||||||
|
#(not (nil? %))
|
||||||
|
(cm/concat-vec
|
||||||
|
(postgres/generate-auth (merge resolved-config {:postgres-size :8gb
|
||||||
|
:db-name "cloud"
|
||||||
|
:pv-storage-size-gb 50})
|
||||||
|
auth)
|
||||||
|
[(nextcloud/generate-secret auth)]
|
||||||
|
(when (:contains? resolved-config :restic-repository)
|
||||||
|
[(backup/generate-secret auth)])
|
||||||
|
(when (:contains? resolved-config :mon-cfg)
|
||||||
|
(mon/generate-auth (:mon-cfg resolved-config) (:mon-auth auth))))))))
|
|
@ -14,23 +14,20 @@
|
||||||
|
|
||||||
(s/def ::fqdn cp/fqdn-string?)
|
(s/def ::fqdn cp/fqdn-string?)
|
||||||
(s/def ::issuer cp/letsencrypt-issuer?)
|
(s/def ::issuer cp/letsencrypt-issuer?)
|
||||||
(s/def ::restic-repository string?)
|
|
||||||
(s/def ::nextcloud-admin-user cp/bash-env-string?)
|
(s/def ::nextcloud-admin-user cp/bash-env-string?)
|
||||||
(s/def ::nextcloud-admin-password cp/bash-env-string?)
|
(s/def ::nextcloud-admin-password cp/bash-env-string?)
|
||||||
(s/def ::pvc-storage-class-name cp/pvc-storage-class-name?)
|
(s/def ::pvc-storage-class-name cp/pvc-storage-class-name?)
|
||||||
(s/def ::pv-storage-size-gb pos?)
|
(s/def ::pv-storage-size-gb pos?)
|
||||||
|
|
||||||
(def config? (s/keys :req-un [::fqdn]
|
(s/def ::config (s/keys :req-un [::fqdn]
|
||||||
:opt-un [::issuer
|
:opt-un [::issuer
|
||||||
::restic-repository
|
|
||||||
::pv-storage-size-gb
|
::pv-storage-size-gb
|
||||||
::pvc-storage-class-name
|
::pvc-storage-class-name
|
||||||
::mon/mon-cfg]))
|
::mon/mon-cfg]))
|
||||||
|
|
||||||
(def auth? (s/keys :req-un [::postgres/postgres-db-user ::postgres/postgres-db-password
|
(s/def ::auth (s/keys :req-un [::postgres/postgres-db-user ::postgres/postgres-db-password
|
||||||
::nextcloud-admin-user ::nextcloud-admin-password
|
::nextcloud-admin-user ::nextcloud-admin-password
|
||||||
::aws-access-key-id ::aws-secret-access-key
|
::aws-access-key-id ::aws-secret-access-key]
|
||||||
::restic-password]
|
|
||||||
:opt-un [::mon/mon-auth]))
|
:opt-un [::mon/mon-auth]))
|
||||||
|
|
||||||
#?(:cljs
|
#?(:cljs
|
||||||
|
@ -38,13 +35,13 @@
|
||||||
(get (inline-resources "nextcloud") resource-name)))
|
(get (inline-resources "nextcloud") resource-name)))
|
||||||
|
|
||||||
(defn-spec generate-deployment cp/map-or-seq?
|
(defn-spec generate-deployment cp/map-or-seq?
|
||||||
[config config?]
|
[config ::config]
|
||||||
(let [{:keys [fqdn]} config]
|
(let [{:keys [fqdn]} config]
|
||||||
(-> (yaml/load-as-edn "nextcloud/deployment.yaml")
|
(-> (yaml/load-as-edn "nextcloud/deployment.yaml")
|
||||||
(cm/replace-all-matching-values-by-new-value "fqdn" fqdn))))
|
(cm/replace-all-matching "fqdn" fqdn))))
|
||||||
|
|
||||||
(defn-spec generate-ingress-and-cert cp/map-or-seq?
|
(defn-spec generate-ingress-and-cert cp/map-or-seq?
|
||||||
[config config?]
|
[config ::config]
|
||||||
(ing/generate-ingress-and-cert
|
(ing/generate-ingress-and-cert
|
||||||
(merge
|
(merge
|
||||||
{:service-name "cloud-service"
|
{:service-name "cloud-service"
|
||||||
|
@ -64,7 +61,7 @@
|
||||||
(yaml/load-as-edn "nextcloud/service.yaml"))
|
(yaml/load-as-edn "nextcloud/service.yaml"))
|
||||||
|
|
||||||
(defn-spec generate-secret cp/map-or-seq?
|
(defn-spec generate-secret cp/map-or-seq?
|
||||||
[auth auth?]
|
[auth ::auth]
|
||||||
(let [{:keys [nextcloud-admin-user nextcloud-admin-password]} auth]
|
(let [{:keys [nextcloud-admin-user nextcloud-admin-password]} auth]
|
||||||
(->
|
(->
|
||||||
(yaml/load-as-edn "nextcloud/secret.yaml")
|
(yaml/load-as-edn "nextcloud/secret.yaml")
|
||||||
|
|
|
@ -2,28 +2,27 @@
|
||||||
(:require
|
(:require
|
||||||
[clojure.tools.reader.edn :as edn]
|
[clojure.tools.reader.edn :as edn]
|
||||||
[dda.c4k-common.common :as cm]
|
[dda.c4k-common.common :as cm]
|
||||||
[dda.c4k-common.monitoring :as mon]
|
|
||||||
[dda.c4k-nextcloud.core :as core]
|
[dda.c4k-nextcloud.core :as core]
|
||||||
[dda.c4k-nextcloud.nextcloud :as nextcloud]
|
[dda.c4k-common.browser :as br]))
|
||||||
[dda.c4k-common.browser :as br]
|
|
||||||
[dda.c4k-common.postgres :as pgc]))
|
|
||||||
|
|
||||||
(defn generate-content []
|
(defn generate-content []
|
||||||
(cm/concat-vec
|
(cm/concat-vec
|
||||||
[(assoc
|
[(assoc
|
||||||
(br/generate-needs-validation) :content
|
(br/generate-needs-validation) :content
|
||||||
(cm/concat-vec
|
(cm/concat-vec
|
||||||
(br/generate-group "domain"
|
(br/generate-group "config"
|
||||||
(cm/concat-vec (br/generate-input-field "fqdn" "Your fqdn:" "nextcloud-neu.prod.meissa-gmbh.de")
|
(br/generate-text-area "config" "Your config.edn:" "{:fqdn \"cloud.your.domain\"
|
||||||
(br/generate-input-field "issuer" "(Optional) Your issuer prod/staging:" "")
|
:issuer \"staging\"
|
||||||
(br/generate-input-field "pv-storage-size-gb" "(Optional) Your nextcloud storage size in GB" "8")
|
:pv-storage-size-gb \"400\"
|
||||||
(br/generate-input-field "pvc-storage-class-name" "(Optional) Your storage class type (manual / local-path):" "local-path")
|
:pvc-storage-class-name \"local-path\"
|
||||||
(br/generate-input-field "postgres-data-volume-path" "(Optional) Your postgres-data-volume-path:" "/var/postgres")
|
:postgres-data-volume-path \"/var/postgres\"
|
||||||
(br/generate-input-field "restic-repository" "(Optional) Your restic-repository:" "restic-repository")
|
:restic-repository \"s3://yourbucket/your-repo\"
|
||||||
(br/generate-input-field "mon-cluster-name" "(Optional) monitoring cluster name:" "keycloak")
|
:restic-password \"restic-password\"}
|
||||||
(br/generate-input-field "mon-cluster-stage" "(Optional) monitoring cluster stage:" "test")
|
:mon-cfg {:cluster-name \"cloud\"
|
||||||
(br/generate-input-field "mon-cloud-url" "(Optional) grafana cloud url:" "https://prometheus-prod-01-eu-west-0.grafana.net/api/prom/push")))
|
:cluster-stage \"test\"
|
||||||
(br/generate-group "credentials"
|
:cloud-url \"https://prometheus-prod-01-eu-west-0.grafana.net/api/prom/push\"}"
|
||||||
|
"5"))
|
||||||
|
(br/generate-group "auth"
|
||||||
(br/generate-text-area "auth" "Your auth.edn:" "{:postgres-db-user \"nextcloud\"
|
(br/generate-text-area "auth" "Your auth.edn:" "{:postgres-db-user \"nextcloud\"
|
||||||
:postgres-db-password \"nextcloud-db-password\"
|
:postgres-db-password \"nextcloud-db-password\"
|
||||||
:nextcloud-admin-password \"nextcloud-admin-password\"
|
:nextcloud-admin-password \"nextcloud-admin-password\"
|
||||||
|
@ -45,41 +44,9 @@
|
||||||
:content
|
:content
|
||||||
(generate-content)})
|
(generate-content)})
|
||||||
|
|
||||||
(defn config-from-document []
|
|
||||||
(let [pv-storage-size-gb (br/get-content-from-element "pv-storage-size-gb" :optional true)
|
|
||||||
pvc-storage-class-name (br/get-content-from-element "pvc-storage-class-name" :optional true)
|
|
||||||
postgres-data-volume-path (br/get-content-from-element "postgres-data-volume-path" :optional true)
|
|
||||||
restic-repository (br/get-content-from-element "restic-repository" :optional true)
|
|
||||||
issuer (br/get-content-from-element "issuer" :optional true)
|
|
||||||
mon-cluster-name (br/get-content-from-element "mon-cluster-name" :optional true)
|
|
||||||
mon-cluster-stage (br/get-content-from-element "mon-cluster-stage" :optional true)
|
|
||||||
mon-cloud-url (br/get-content-from-element "mon-cloud-url" :optional true)]
|
|
||||||
(merge
|
|
||||||
{:fqdn (br/get-content-from-element "fqdn")}
|
|
||||||
(when (and (some? pv-storage-size-gb) (some? pvc-storage-class-name))
|
|
||||||
{:pv-storage-size-gb pv-storage-size-gb :pvc-storage-class-name pvc-storage-class-name})
|
|
||||||
(when (some? postgres-data-volume-path)
|
|
||||||
{:postgres-data-volume-path postgres-data-volume-path})
|
|
||||||
(when (some? restic-repository)
|
|
||||||
{:restic-repository restic-repository})
|
|
||||||
(when (some? issuer)
|
|
||||||
{:issuer issuer})
|
|
||||||
(when (some? mon-cluster-name)
|
|
||||||
{:mon-cfg {:cluster-name mon-cluster-name
|
|
||||||
:cluster-stage (keyword mon-cluster-stage)
|
|
||||||
:grafana-cloud-url mon-cloud-url}}))))
|
|
||||||
|
|
||||||
(defn validate-all! []
|
(defn validate-all! []
|
||||||
(br/validate! "fqdn" ::nextcloud/fqdn)
|
(br/validate! "config" core/config? :deserializer edn/read-string)
|
||||||
(br/validate! "pv-storage-size-gb" ::nextcloud/pv-storage-size-gb :optional true)
|
(br/validate! "auth" core/auth? :deserializer edn/read-string)
|
||||||
(br/validate! "pvc-storage-class-name" ::nextcloud/pvc-storage-class-name :optional true)
|
|
||||||
(br/validate! "postgres-data-volume-path" ::pgc/postgres-data-volume-path :optional true)
|
|
||||||
(br/validate! "restic-repository" ::nextcloud/restic-repository :optional true)
|
|
||||||
(br/validate! "issuer" ::nextcloud/issuer :optional true)
|
|
||||||
(br/validate! "mon-cluster-name" ::mon/cluster-name :optional true)
|
|
||||||
(br/validate! "mon-cluster-stage" ::mon/cluster-stage :optional true)
|
|
||||||
(br/validate! "mon-cloud-url" ::mon/grafana-cloud-url :optional true)
|
|
||||||
(br/validate! "auth" nextcloud/auth? :deserializer edn/read-string)
|
|
||||||
(br/set-validated!))
|
(br/set-validated!))
|
||||||
|
|
||||||
(defn add-validate-listener [name]
|
(defn add-validate-listener [name]
|
||||||
|
@ -92,19 +59,14 @@
|
||||||
(.getElementById "generate-button")
|
(.getElementById "generate-button")
|
||||||
(.addEventListener "click"
|
(.addEventListener "click"
|
||||||
#(do (validate-all!)
|
#(do (validate-all!)
|
||||||
(-> (cm/generate-common
|
(-> (cm/generate-cm
|
||||||
(config-from-document)
|
(br/get-content-from-element "config" :deserializer edn/read-string)
|
||||||
(br/get-content-from-element "auth" :deserializer edn/read-string)
|
(br/get-content-from-element "auth" :deserializer edn/read-string)
|
||||||
{}
|
core/config-defaults
|
||||||
core/k8s-objects)
|
core/config-objects
|
||||||
|
core/auth-objects
|
||||||
|
false
|
||||||
|
false)
|
||||||
(br/set-output!)))))
|
(br/set-output!)))))
|
||||||
(add-validate-listener "fqdn")
|
(add-validate-listener "config")
|
||||||
(add-validate-listener "pv-storage-size-gb")
|
|
||||||
(add-validate-listener "pvc-storage-class-name")
|
|
||||||
(add-validate-listener "postgres-data-volume-path")
|
|
||||||
(add-validate-listener "restic-repository")
|
|
||||||
(add-validate-listener "issuer")
|
|
||||||
(add-validate-listener "mon-cluster-name")
|
|
||||||
(add-validate-listener "mon-cluster-stage")
|
|
||||||
(add-validate-listener "mon-cloud-url")
|
|
||||||
(add-validate-listener "auth"))
|
(add-validate-listener "auth"))
|
||||||
|
|
|
@ -2,6 +2,7 @@ apiVersion: apps/v1
|
||||||
kind: Deployment
|
kind: Deployment
|
||||||
metadata:
|
metadata:
|
||||||
name: backup-restore
|
name: backup-restore
|
||||||
|
namespace: nextcloud
|
||||||
spec:
|
spec:
|
||||||
replicas: 0
|
replicas: 0
|
||||||
selector:
|
selector:
|
||||||
|
@ -20,7 +21,7 @@ spec:
|
||||||
- name: backup-app
|
- name: backup-app
|
||||||
image: domaindrivenarchitecture/c4k-cloud-backup
|
image: domaindrivenarchitecture/c4k-cloud-backup
|
||||||
imagePullPolicy: IfNotPresent
|
imagePullPolicy: IfNotPresent
|
||||||
command: ["/entrypoint-start-and-wait.sh"]
|
command: ["wait.bb"]
|
||||||
env:
|
env:
|
||||||
- name: POSTGRES_USER
|
- name: POSTGRES_USER
|
||||||
valueFrom:
|
valueFrom:
|
||||||
|
@ -67,6 +68,9 @@ spec:
|
||||||
- name: cloud-secret-volume
|
- name: cloud-secret-volume
|
||||||
mountPath: /var/run/secrets/cloud-secrets
|
mountPath: /var/run/secrets/cloud-secrets
|
||||||
readOnly: true
|
readOnly: true
|
||||||
|
- name: rotation-credential-secret-volume
|
||||||
|
mountPath: /var/run/secrets/rotation-credential-secret
|
||||||
|
readOnly: true
|
||||||
volumes:
|
volumes:
|
||||||
- name: cloud-data-volume
|
- name: cloud-data-volume
|
||||||
persistentVolumeClaim:
|
persistentVolumeClaim:
|
||||||
|
@ -77,3 +81,7 @@ spec:
|
||||||
- name: backup-secret-volume
|
- name: backup-secret-volume
|
||||||
secret:
|
secret:
|
||||||
secretName: backup-secret
|
secretName: backup-secret
|
||||||
|
- name: rotation-credential-secret-volume
|
||||||
|
secret:
|
||||||
|
secretName: rotation-credential-secret
|
||||||
|
optional: true
|
||||||
|
|
|
@ -2,6 +2,7 @@ apiVersion: v1
|
||||||
kind: ConfigMap
|
kind: ConfigMap
|
||||||
metadata:
|
metadata:
|
||||||
name: backup-config
|
name: backup-config
|
||||||
|
namespace: nextcloud
|
||||||
labels:
|
labels:
|
||||||
app.kubernetes.io/name: backup
|
app.kubernetes.io/name: backup
|
||||||
app.kubernetes.io/part-of: cloud
|
app.kubernetes.io/part-of: cloud
|
||||||
|
|
8
src/main/resources/backup/credential-rotation.yaml
Normal file
8
src/main/resources/backup/credential-rotation.yaml
Normal file
|
@ -0,0 +1,8 @@
|
||||||
|
apiVersion: v1
|
||||||
|
kind: Secret
|
||||||
|
metadata:
|
||||||
|
name: rotation-credential-secret
|
||||||
|
namespace: nextcloud
|
||||||
|
type: Opaque
|
||||||
|
data:
|
||||||
|
rotation-credential: "dGVzdAo="
|
|
@ -2,6 +2,7 @@ apiVersion: batch/v1
|
||||||
kind: CronJob
|
kind: CronJob
|
||||||
metadata:
|
metadata:
|
||||||
name: cloud-backup
|
name: cloud-backup
|
||||||
|
namespace: nextcloud
|
||||||
labels:
|
labels:
|
||||||
app.kubernetes.part-of: cloud
|
app.kubernetes.part-of: cloud
|
||||||
spec:
|
spec:
|
||||||
|
@ -16,7 +17,7 @@ spec:
|
||||||
- name: backup-app
|
- name: backup-app
|
||||||
image: domaindrivenarchitecture/c4k-cloud-backup
|
image: domaindrivenarchitecture/c4k-cloud-backup
|
||||||
imagePullPolicy: IfNotPresent
|
imagePullPolicy: IfNotPresent
|
||||||
command: ["/entrypoint.sh"]
|
command: ["backup.bb"]
|
||||||
env:
|
env:
|
||||||
- name: POSTGRES_USER
|
- name: POSTGRES_USER
|
||||||
valueFrom:
|
valueFrom:
|
||||||
|
|
|
@ -2,6 +2,7 @@ apiVersion: v1
|
||||||
kind: Secret
|
kind: Secret
|
||||||
metadata:
|
metadata:
|
||||||
name: backup-secret
|
name: backup-secret
|
||||||
|
namespace: nextcloud
|
||||||
type: Opaque
|
type: Opaque
|
||||||
data:
|
data:
|
||||||
aws-access-key-id: "aws-access-key-id"
|
aws-access-key-id: "aws-access-key-id"
|
||||||
|
|
|
@ -2,6 +2,7 @@ apiVersion: apps/v1
|
||||||
kind: Deployment
|
kind: Deployment
|
||||||
metadata:
|
metadata:
|
||||||
name: cloud-deployment
|
name: cloud-deployment
|
||||||
|
namespace: nextcloud
|
||||||
spec:
|
spec:
|
||||||
selector:
|
selector:
|
||||||
matchLabels:
|
matchLabels:
|
||||||
|
@ -18,7 +19,7 @@ spec:
|
||||||
redeploy: v3
|
redeploy: v3
|
||||||
spec:
|
spec:
|
||||||
containers:
|
containers:
|
||||||
- image: domaindrivenarchitecture/c4k-cloud:8.0.0
|
- image: domaindrivenarchitecture/c4k-cloud:10.4.2
|
||||||
name: cloud-app
|
name: cloud-app
|
||||||
imagePullPolicy: IfNotPresent
|
imagePullPolicy: IfNotPresent
|
||||||
ports:
|
ports:
|
||||||
|
|
|
@ -2,6 +2,7 @@ apiVersion: v1
|
||||||
kind: PersistentVolumeClaim
|
kind: PersistentVolumeClaim
|
||||||
metadata:
|
metadata:
|
||||||
name: cloud-pvc
|
name: cloud-pvc
|
||||||
|
namespace: nextcloud
|
||||||
labels:
|
labels:
|
||||||
app.kubernetes.io/application: cloud
|
app.kubernetes.io/application: cloud
|
||||||
spec:
|
spec:
|
||||||
|
|
|
@ -2,6 +2,7 @@ apiVersion: v1
|
||||||
kind: Secret
|
kind: Secret
|
||||||
metadata:
|
metadata:
|
||||||
name: cloud-secret
|
name: cloud-secret
|
||||||
|
namespace: nextcloud
|
||||||
type: Opaque
|
type: Opaque
|
||||||
data:
|
data:
|
||||||
nextcloud-admin-user: "admin-user"
|
nextcloud-admin-user: "admin-user"
|
||||||
|
|
|
@ -2,6 +2,7 @@ apiVersion: v1
|
||||||
kind: Service
|
kind: Service
|
||||||
metadata:
|
metadata:
|
||||||
name: cloud-service
|
name: cloud-service
|
||||||
|
namespace: nextcloud
|
||||||
labels:
|
labels:
|
||||||
app.kubernetes.io/name: cloud-service
|
app.kubernetes.io/name: cloud-service
|
||||||
app.kubernetes.io/application: cloud
|
app.kubernetes.io/application: cloud
|
||||||
|
|
|
@ -2,13 +2,17 @@
|
||||||
(:require
|
(:require
|
||||||
#?(:clj [clojure.test :refer [deftest is are testing run-tests]]
|
#?(:clj [clojure.test :refer [deftest is are testing run-tests]]
|
||||||
:cljs [cljs.test :refer-macros [deftest is are testing run-tests]])
|
:cljs [cljs.test :refer-macros [deftest is are testing run-tests]])
|
||||||
|
[clojure.spec.test.alpha :as st]
|
||||||
[dda.c4k-nextcloud.backup :as cut]))
|
[dda.c4k-nextcloud.backup :as cut]))
|
||||||
|
|
||||||
|
(st/instrument `cut/generate-secret)
|
||||||
|
(st/instrument `cut/generate-config)
|
||||||
|
(st/instrument `cut/generate-cron)
|
||||||
|
|
||||||
(deftest should-generate-secret
|
(deftest should-generate-secret
|
||||||
(is (= {:apiVersion "v1"
|
(is (= {:apiVersion "v1"
|
||||||
:kind "Secret"
|
:kind "Secret"
|
||||||
:metadata {:name "backup-secret"}
|
:metadata {:name "backup-secret", :namespace "nextcloud"}
|
||||||
:type "Opaque"
|
:type "Opaque"
|
||||||
:data
|
:data
|
||||||
{:aws-access-key-id "YXdzLWlk", :aws-secret-access-key "YXdzLXNlY3JldA==", :restic-password "cmVzdGljLXB3"}}
|
{:aws-access-key-id "YXdzLWlk", :aws-secret-access-key "YXdzLXNlY3JldA==", :restic-password "cmVzdGljLXB3"}}
|
||||||
|
@ -18,6 +22,7 @@
|
||||||
(is (= {:apiVersion "v1"
|
(is (= {:apiVersion "v1"
|
||||||
:kind "ConfigMap"
|
:kind "ConfigMap"
|
||||||
:metadata {:name "backup-config"
|
:metadata {:name "backup-config"
|
||||||
|
:namespace "nextcloud"
|
||||||
:labels {:app.kubernetes.io/name "backup"
|
:labels {:app.kubernetes.io/name "backup"
|
||||||
:app.kubernetes.io/part-of "cloud"}}
|
:app.kubernetes.io/part-of "cloud"}}
|
||||||
:data
|
:data
|
||||||
|
@ -27,7 +32,7 @@
|
||||||
(deftest should-generate-cron
|
(deftest should-generate-cron
|
||||||
(is (= {:apiVersion "batch/v1"
|
(is (= {:apiVersion "batch/v1"
|
||||||
:kind "CronJob"
|
:kind "CronJob"
|
||||||
:metadata {:name "cloud-backup", :labels {:app.kubernetes.part-of "cloud"}}
|
:metadata {:name "cloud-backup", :namespace "nextcloud", :labels {:app.kubernetes.part-of "cloud"}}
|
||||||
:spec
|
:spec
|
||||||
{:schedule "10 23 * * *"
|
{:schedule "10 23 * * *"
|
||||||
:successfulJobsHistoryLimit 1
|
:successfulJobsHistoryLimit 1
|
||||||
|
@ -40,7 +45,7 @@
|
||||||
[{:name "backup-app"
|
[{:name "backup-app"
|
||||||
:image "domaindrivenarchitecture/c4k-cloud-backup"
|
:image "domaindrivenarchitecture/c4k-cloud-backup"
|
||||||
:imagePullPolicy "IfNotPresent"
|
:imagePullPolicy "IfNotPresent"
|
||||||
:command ["/entrypoint.sh"]
|
:command ["backup.bb"]
|
||||||
:env
|
:env
|
||||||
[{:valueFrom
|
[{:valueFrom
|
||||||
{:secretKeyRef
|
{:secretKeyRef
|
||||||
|
|
19
src/test/cljc/dda/c4k_nextcloud/core_test.cljc
Normal file
19
src/test/cljc/dda/c4k_nextcloud/core_test.cljc
Normal file
|
@ -0,0 +1,19 @@
|
||||||
|
(ns dda.c4k-nextcloud.core-test
|
||||||
|
(:require
|
||||||
|
#?(:clj [clojure.test :refer [deftest is are testing run-tests]]
|
||||||
|
:cljs [cljs.test :refer-macros [deftest is are testing run-tests]])
|
||||||
|
[clojure.spec.alpha :as s]
|
||||||
|
[dda.c4k-common.yaml :as yaml]
|
||||||
|
[dda.c4k-nextcloud.core :as cut]
|
||||||
|
#?(:cljs [dda.c4k-common.macros :refer-macros [inline-resources]])))
|
||||||
|
|
||||||
|
|
||||||
|
#?(:cljs
|
||||||
|
(defmethod yaml/load-resource :nextcloud-test [resource-name]
|
||||||
|
(get (inline-resources "nextcloud-test") resource-name)))
|
||||||
|
|
||||||
|
(deftest validate-valid-resources
|
||||||
|
(is (s/valid? cut/config? (yaml/load-as-edn "nextcloud-test/valid-config.yaml")))
|
||||||
|
(is (s/valid? cut/auth? (yaml/load-as-edn "nextcloud-test/valid-auth.yaml")))
|
||||||
|
(is (not (s/valid? cut/config? (yaml/load-as-edn "nextcloud-test/invalid-config.yaml"))))
|
||||||
|
(is (not (s/valid? cut/auth? (yaml/load-as-edn "nextcloud-test/invalid-auth.yaml")))))
|
|
@ -2,28 +2,24 @@
|
||||||
(:require
|
(:require
|
||||||
#?(:clj [clojure.test :refer [deftest is are testing run-tests]]
|
#?(:clj [clojure.test :refer [deftest is are testing run-tests]]
|
||||||
:cljs [cljs.test :refer-macros [deftest is are testing run-tests]])
|
:cljs [cljs.test :refer-macros [deftest is are testing run-tests]])
|
||||||
[clojure.spec.alpha :as s]
|
|
||||||
[clojure.spec.test.alpha :as st]
|
[clojure.spec.test.alpha :as st]
|
||||||
[dda.c4k-common.yaml :as yaml]
|
[dda.c4k-common.yaml :as yaml]
|
||||||
[dda.c4k-nextcloud.nextcloud :as cut]
|
[dda.c4k-nextcloud.nextcloud :as cut]
|
||||||
#?(:cljs [dda.c4k-common.macros :refer-macros [inline-resources]])))
|
#?(:cljs [dda.c4k-common.macros :refer-macros [inline-resources]])))
|
||||||
|
|
||||||
(st/instrument)
|
(st/instrument `cut/generate-secret)
|
||||||
|
(st/instrument `cut/generate-ingress-and-cert)
|
||||||
|
(st/instrument `cut/generate-pvc)
|
||||||
|
(st/instrument `cut/generate-deployment)
|
||||||
|
|
||||||
#?(:cljs
|
#?(:cljs
|
||||||
(defmethod yaml/load-resource :nextcloud-test [resource-name]
|
(defmethod yaml/load-resource :nextcloud-test [resource-name]
|
||||||
(get (inline-resources "nextcloud-test") resource-name)))
|
(get (inline-resources "nextcloud-test") resource-name)))
|
||||||
|
|
||||||
(deftest validate-valid-resources
|
|
||||||
(is (s/valid? cut/config? (yaml/load-as-edn "nextcloud-test/valid-config.yaml")))
|
|
||||||
(is (s/valid? cut/auth? (yaml/load-as-edn "nextcloud-test/valid-auth.yaml")))
|
|
||||||
(is (not (s/valid? cut/config? (yaml/load-as-edn "nextcloud-test/invalid-config.yaml"))))
|
|
||||||
(is (not (s/valid? cut/auth? (yaml/load-as-edn "nextcloud-test/invalid-auth.yaml")))))
|
|
||||||
|
|
||||||
(deftest should-generate-secret
|
(deftest should-generate-secret
|
||||||
(is (= {:apiVersion "v1"
|
(is (= {:apiVersion "v1"
|
||||||
:kind "Secret"
|
:kind "Secret"
|
||||||
:metadata {:name "cloud-secret"}
|
:metadata {:name "cloud-secret", :namespace "nextcloud"}
|
||||||
:type "Opaque"
|
:type "Opaque"
|
||||||
:data
|
:data
|
||||||
{:nextcloud-admin-user "Y2xvdWRhZG1pbg=="
|
{:nextcloud-admin-user "Y2xvdWRhZG1pbg=="
|
||||||
|
@ -77,6 +73,7 @@
|
||||||
(is (= {:apiVersion "v1"
|
(is (= {:apiVersion "v1"
|
||||||
:kind "PersistentVolumeClaim"
|
:kind "PersistentVolumeClaim"
|
||||||
:metadata {:name "cloud-pvc"
|
:metadata {:name "cloud-pvc"
|
||||||
|
:namespace "nextcloud"
|
||||||
:labels {:app.kubernetes.io/application "cloud"}}
|
:labels {:app.kubernetes.io/application "cloud"}}
|
||||||
:spec {:storageClassName "local-path"
|
:spec {:storageClassName "local-path"
|
||||||
:accessModes ["ReadWriteOnce"]
|
:accessModes ["ReadWriteOnce"]
|
||||||
|
@ -86,7 +83,7 @@
|
||||||
(deftest should-generate-deployment
|
(deftest should-generate-deployment
|
||||||
(is (= {:apiVersion "apps/v1"
|
(is (= {:apiVersion "apps/v1"
|
||||||
:kind "Deployment"
|
:kind "Deployment"
|
||||||
:metadata {:name "cloud-deployment"}
|
:metadata {:name "cloud-deployment", :namespace "nextcloud"}
|
||||||
:spec
|
:spec
|
||||||
{:selector {:matchLabels #:app.kubernetes.io{:name "cloud-pod", :application "cloud"}}
|
{:selector {:matchLabels #:app.kubernetes.io{:name "cloud-pod", :application "cloud"}}
|
||||||
:strategy {:type "Recreate"}
|
:strategy {:type "Recreate"}
|
||||||
|
@ -94,7 +91,7 @@
|
||||||
{:metadata {:labels {:app "cloud-app", :app.kubernetes.io/name "cloud-pod", :app.kubernetes.io/application "cloud", :redeploy "v3"}}
|
{:metadata {:labels {:app "cloud-app", :app.kubernetes.io/name "cloud-pod", :app.kubernetes.io/application "cloud", :redeploy "v3"}}
|
||||||
:spec
|
:spec
|
||||||
{:containers
|
{:containers
|
||||||
[{:image "domaindrivenarchitecture/c4k-cloud:8.0.0"
|
[{:image "domaindrivenarchitecture/c4k-cloud:10.4.2"
|
||||||
:name "cloud-app"
|
:name "cloud-app"
|
||||||
:imagePullPolicy "IfNotPresent"
|
:imagePullPolicy "IfNotPresent"
|
||||||
:ports [{:containerPort 80}]
|
:ports [{:containerPort 80}]
|
||||||
|
|
|
@ -1,84 +0,0 @@
|
||||||
# Usage
|
|
||||||
|
|
||||||
`setup-local-s3.sh [BUCKET_NAME]`:
|
|
||||||
- [BUCKET_NAME] is optional, "mybucket" will be used if not specified
|
|
||||||
- sets up a k3s instance
|
|
||||||
- installs a localstack pod
|
|
||||||
- creates http and https routing to localstack via localhost
|
|
||||||
- saves the self-signed certificate as ca.crt
|
|
||||||
- uses the certificate to initialize a restic repo at `https://k3stesthost/BUCKET_NAME`
|
|
||||||
|
|
||||||
Note: In case of not being able to connect to "k3stesthost/health", you might need to ensure that the ingress' ip matches with the required host names: k3stesthost and cloudhost. With `sudo k3s kubectl get ingress` you can view the ingress' ip (e.g. 10.0.2.15), then add a line to file "/etc/hosts" e.g. `10.0.2.15 k3stesthost cloudhost`
|
|
||||||
|
|
||||||
`start-k3s.sh`:
|
|
||||||
- creates and starts a k3s instance
|
|
||||||
|
|
||||||
`k3s-uninstall.sh`:
|
|
||||||
- deletes everything k3s related
|
|
||||||
|
|
||||||
## Other useful commands
|
|
||||||
- `sudo k3s kubectl get pods`
|
|
||||||
- `curl k3stesthost/health`
|
|
||||||
expected: `{"services": {"s3": "running"}, "features": {"persistence": "disabled", "initScripts": "initialized"}}`
|
|
||||||
|
|
||||||
#### Requires AWS-CLI
|
|
||||||
- create bucket `aws --endpoint-url=http://k3stesthost s3 mb s3://mybucket`
|
|
||||||
- list buckets `aws --endpoint-url=http://k3stesthost s3 ls`
|
|
||||||
- upload something `aws --endpoint-url=http://k3stesthost s3 cp test.txt s3://mybucket`
|
|
||||||
- check files `aws --endpoint-url=http://k3stesthost s3 ls s3://mybucket`
|
|
||||||
|
|
||||||
## Run docker locally
|
|
||||||
|
|
||||||
|
|
||||||
```
|
|
||||||
docker pull docker:19.03.12-dind
|
|
||||||
docker run -d --privileged --name integration-test docker:19.03.12-dind
|
|
||||||
docker exec integration-test sh -c "apk add bash"
|
|
||||||
|
|
||||||
```
|
|
||||||
|
|
||||||
Set up docker container integration-test:
|
|
||||||
|
|
||||||
```
|
|
||||||
docker cp ../../../../../c4k-nextcloud/ integration-test:/
|
|
||||||
docker exec -it integration-test sh
|
|
||||||
cd /c4k-nextcloud/src/test/resources/local-integration-test
|
|
||||||
./setup-docker.sh
|
|
||||||
```
|
|
||||||
|
|
||||||
## Deploy nextcloud
|
|
||||||
|
|
||||||
### Requirements
|
|
||||||
|
|
||||||
* leiningen (install with: `sudo apt install leiningen` )
|
|
||||||
* In the project's root execute: `lein uberjar`
|
|
||||||
* Change file "valid-config.edn" according to your settings (e.g. `:fqdn "cloudhost"` and `:restic-repository "s3://k3stesthost:mybucket"`).
|
|
||||||
|
|
||||||
### Deploy to k3s
|
|
||||||
|
|
||||||
* Create and deploy the k8s yaml:
|
|
||||||
`java -jar target/uberjar/c4k-nextcloud-standalone.jar valid-config.edn valid-auth.edn | sudo k3s kubectl apply -f -`
|
|
||||||
|
|
||||||
Some of the steps may take some min to be effective, but eventually nextcloud should be available at: https://cloudhost
|
|
||||||
|
|
||||||
### Deploy to k3d
|
|
||||||
|
|
||||||
k3d is a k3s system which is running inside of a container. To install k3d run `curl -s https://raw.githubusercontent.com/rancher/k3d/main/install.sh | bash` or have a look at https://k3d.io/v5.0.3/ .
|
|
||||||
|
|
||||||
* Start a k3d cluster to deploy s3, nextcloud and test backup and restore on it: `./setup-local-s3-on-k3d.sh`
|
|
||||||
|
|
||||||
Some steps may take a couple of minutes to be effective, but eventually nextcloud should be available at: https://cloudhost
|
|
||||||
|
|
||||||
#### Remove k3d cluster
|
|
||||||
|
|
||||||
`k3d cluster delete nextcloud`
|
|
||||||
|
|
||||||
## Test in local gitlab runner
|
|
||||||
|
|
||||||
See https://stackoverflow.com/questions/32933174/use-gitlab-ci-to-run-tests-locally
|
|
||||||
|
|
||||||
This needs to be done in the project root
|
|
||||||
|
|
||||||
`docker run -d --name gitlab-runner --restart always -v $PWD:$PWD -v /var/run/docker.sock:/var/run/docker.sock gitlab/gitlab-runner:latest`
|
|
||||||
|
|
||||||
`docker exec -it -w $PWD gitlab-runner gitlab-runner exec docker nextcloud-integrationtest --docker-privileged --docker-volumes '/var/run/docker.sock:/var/run/docker.sock'`
|
|
|
@ -1,20 +0,0 @@
|
||||||
apiVersion: cert-manager.io/v1
|
|
||||||
kind: Certificate
|
|
||||||
metadata:
|
|
||||||
name: localstack-cert
|
|
||||||
namespace: default
|
|
||||||
spec:
|
|
||||||
secretName: localstack-secret
|
|
||||||
commonName: k3stesthost
|
|
||||||
dnsNames:
|
|
||||||
- k3stesthost
|
|
||||||
issuerRef:
|
|
||||||
name: selfsigning-issuer
|
|
||||||
kind: ClusterIssuer
|
|
||||||
---
|
|
||||||
apiVersion: cert-manager.io/v1
|
|
||||||
kind: ClusterIssuer
|
|
||||||
metadata:
|
|
||||||
name: selfsigning-issuer
|
|
||||||
spec:
|
|
||||||
selfSigned: {}
|
|
|
@ -1,44 +0,0 @@
|
||||||
@startuml
|
|
||||||
|
|
||||||
autonumber
|
|
||||||
|
|
||||||
skinparam sequenceBox {
|
|
||||||
borderColor White
|
|
||||||
}
|
|
||||||
|
|
||||||
participant gitlab_runner
|
|
||||||
|
|
||||||
box "outer container" #LightBlue
|
|
||||||
|
|
||||||
participant .gitlab_ci
|
|
||||||
participant PreparingCommands
|
|
||||||
participant test_script
|
|
||||||
|
|
||||||
end box
|
|
||||||
|
|
||||||
|
|
||||||
box "k3s" #CornSilk
|
|
||||||
|
|
||||||
participant k3s_api_server
|
|
||||||
participant backup_pod
|
|
||||||
|
|
||||||
end box
|
|
||||||
|
|
||||||
|
|
||||||
gitlab_runner -> k3s_api_server: run k3s as container
|
|
||||||
gitlab_runner -> .gitlab_ci : run
|
|
||||||
|
|
||||||
.gitlab_ci -> PreparingCommands : Install packages (curl bash ...)
|
|
||||||
.gitlab_ci -> PreparingCommands : get k3s_api_server config for k3s_api_server
|
|
||||||
|
|
||||||
.gitlab_ci -> test_script : run
|
|
||||||
|
|
||||||
test_script -> k3s_api_server: apply cert-manager
|
|
||||||
test_script -> k3s_api_server: apply localstack
|
|
||||||
test_script -> k3s_api_server: enable tls / create certificates
|
|
||||||
test_script -> k3s_api_server: apply cloud
|
|
||||||
test_script -> k3s_api_server: create backup_pod (by scale to 1)
|
|
||||||
test_script -> backup_pod: backup
|
|
||||||
test_script -> backup_pod: restore
|
|
||||||
|
|
||||||
@enduml
|
|
|
@ -1,17 +0,0 @@
|
||||||
# Set the default kube context if present
|
|
||||||
DEFAULT_KUBE_CONTEXTS="$HOME/.kube/config"
|
|
||||||
if test -f "${DEFAULT_KUBE_CONTEXTS}"
|
|
||||||
then
|
|
||||||
export KUBECONFIG="$DEFAULT_KUBE_CONTEXTS"
|
|
||||||
fi
|
|
||||||
|
|
||||||
# Additional contexts should be in ~/.kube/custom-contexts/
|
|
||||||
CUSTOM_KUBE_CONTEXTS="$HOME/.kube/custom-contexts"
|
|
||||||
mkdir -p "${CUSTOM_KUBE_CONTEXTS}"
|
|
||||||
OIFS="$IFS"
|
|
||||||
IFS=$'\n'
|
|
||||||
for contextFile in `find "${CUSTOM_KUBE_CONTEXTS}" -type f -name "*.yml"`
|
|
||||||
do
|
|
||||||
export KUBECONFIG="$contextFile:$KUBECONFIG"
|
|
||||||
done
|
|
||||||
IFS="$OIFS"
|
|
|
@ -1,65 +0,0 @@
|
||||||
apiVersion: apps/v1
|
|
||||||
kind: Deployment
|
|
||||||
metadata:
|
|
||||||
name: localstack
|
|
||||||
spec:
|
|
||||||
selector:
|
|
||||||
matchLabels:
|
|
||||||
app: localstack
|
|
||||||
strategy:
|
|
||||||
type: Recreate
|
|
||||||
template:
|
|
||||||
metadata:
|
|
||||||
labels:
|
|
||||||
app: localstack
|
|
||||||
spec:
|
|
||||||
containers:
|
|
||||||
- image: localstack/localstack
|
|
||||||
name: localstack-app
|
|
||||||
imagePullPolicy: IfNotPresent
|
|
||||||
env:
|
|
||||||
- name: SERVICES
|
|
||||||
value: s3
|
|
||||||
---
|
|
||||||
# service
|
|
||||||
apiVersion: v1
|
|
||||||
kind: Service
|
|
||||||
metadata:
|
|
||||||
name: localstack-service
|
|
||||||
spec:
|
|
||||||
selector:
|
|
||||||
app: localstack
|
|
||||||
ports:
|
|
||||||
- port: 4566
|
|
||||||
---
|
|
||||||
apiVersion: v1
|
|
||||||
kind: Secret
|
|
||||||
metadata:
|
|
||||||
name: localstack-secret
|
|
||||||
type: Opaque
|
|
||||||
---
|
|
||||||
apiVersion: networking.k8s.io/v1
|
|
||||||
kind: Ingress
|
|
||||||
metadata:
|
|
||||||
name: ingress-localstack
|
|
||||||
annotations:
|
|
||||||
cert-manager.io/cluster-issuer: selfsigning-issuer
|
|
||||||
kubernetes.io/ingress.class: traefik
|
|
||||||
traefik.ingress.kubernetes.io/redirect-entry-point: https
|
|
||||||
namespace: default
|
|
||||||
spec:
|
|
||||||
tls:
|
|
||||||
- hosts:
|
|
||||||
- k3stesthost
|
|
||||||
secretName: localstack-secret
|
|
||||||
rules:
|
|
||||||
- host: k3stesthost
|
|
||||||
http:
|
|
||||||
paths:
|
|
||||||
- path: /
|
|
||||||
pathType: Prefix
|
|
||||||
backend:
|
|
||||||
service:
|
|
||||||
name: localstack-service
|
|
||||||
port:
|
|
||||||
number: 4566
|
|
|
@ -1,48 +0,0 @@
|
||||||
#!/bin/bash
|
|
||||||
|
|
||||||
set -x
|
|
||||||
|
|
||||||
docker volume create k3s-server
|
|
||||||
|
|
||||||
name='inttst'
|
|
||||||
|
|
||||||
[[ $(docker ps -f "name=$name" --format '{{.Names}}') == $name ]] || docker run --name $name -d --privileged --tmpfs /run --tmpfs /var/run --restart always -e K3S_TOKEN=12345678901234 -e K3S_KUBECONFIG_OUTPUT=./kubeconfig.yaml -e K3S_KUBECONFIG_MODE=666 -v k3s-server:/var/lib/rancher/k3s:z -v $(pwd):/output:z -p 6443:6443 -p 80:80 -p 443:443 rancher/k3s server --cluster-init --tls-san k3stesthost --tls-san cloudhost
|
|
||||||
|
|
||||||
docker ps
|
|
||||||
|
|
||||||
export timeout=30; while ! docker exec $name sh -c "test -f /var/lib/rancher/k3s/server/kubeconfig.yaml"; do if [ "$timeout" == 0 ]; then echo "ERROR: Timeout while waiting for file."; break; fi; sleep 1; ((timeout--)); done
|
|
||||||
|
|
||||||
mkdir -p $HOME/.kube/
|
|
||||||
|
|
||||||
docker cp $name:/var/lib/rancher/k3s/server/kubeconfig.yaml $HOME/.kube/config
|
|
||||||
|
|
||||||
if [ "$timeout" == 0 ]
|
|
||||||
then
|
|
||||||
echo -------------------------------------------------------
|
|
||||||
find / -name "kubeconfig.yaml";
|
|
||||||
echo -------------------------------------------------------
|
|
||||||
docker ps -a
|
|
||||||
echo -------------------------------------------------------
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
|
|
||||||
echo "127.0.0.1 kubernetes" >> /etc/hosts
|
|
||||||
|
|
||||||
apk add wget curl bash sudo openjdk8
|
|
||||||
|
|
||||||
wget -P /etc/apk/keys/ https://alpine-pkgs.sgerrand.com/sgerrand.rsa.pub
|
|
||||||
apk add --no-cache --repository=https://apkproxy.herokuapp.com/sgerrand/alpine-pkg-leiningen leiningen
|
|
||||||
|
|
||||||
curl -LO https://storage.googleapis.com/kubernetes-release/release/v1.22.0/bin/linux/amd64/kubectl
|
|
||||||
chmod +x ./kubectl
|
|
||||||
mv ./kubectl /usr/local/bin/kubectl
|
|
||||||
|
|
||||||
sleep 20 #allow some time to startup k3s
|
|
||||||
docker ps -a
|
|
||||||
|
|
||||||
swapoff -a # can this be removed ?
|
|
||||||
|
|
||||||
export KUBECONFIG=$HOME/.kube/config
|
|
||||||
|
|
||||||
pwd
|
|
||||||
cd ./c4k-nextcloud/src/test/resources/local-integration-test && ./setup-local-s3-on-k3d.sh
|
|
|
@ -1,60 +0,0 @@
|
||||||
#!/bin/bash
|
|
||||||
|
|
||||||
set -x
|
|
||||||
|
|
||||||
function main()
|
|
||||||
{
|
|
||||||
# enable tls for k3s with cert-manager
|
|
||||||
kubectl apply -f https://github.com/jetstack/cert-manager/releases/download/v1.5.4/cert-manager.yaml
|
|
||||||
|
|
||||||
kubectl apply -f localstack.yaml
|
|
||||||
|
|
||||||
until kubectl apply -f certificate.yaml
|
|
||||||
do
|
|
||||||
echo "[INFO] Waiting for certificate ..."
|
|
||||||
sleep 30
|
|
||||||
done
|
|
||||||
|
|
||||||
# wait for ingress to be ready
|
|
||||||
bash -c 'external_ip=""; while [ -z $external_ip ]; do echo "[INFO] Waiting for end point..."; external_ip=$(kubectl get ingress -o jsonpath="{$.items[*].status.loadBalancer.ingress[*].ip}"); [ -z "$external_ip" ] && sleep 10; done; echo "End point ready - $external_ip";'
|
|
||||||
|
|
||||||
export INGRESS_IP=$(kubectl get ingress ingress-localstack -o=jsonpath="{.status.loadBalancer.ingress[0].ip}")
|
|
||||||
|
|
||||||
cd ../../../../ # c4k-nextcloud project root
|
|
||||||
lein uberjar
|
|
||||||
java -jar target/uberjar/c4k-nextcloud-standalone.jar config-local.edn auth-local.edn | kubectl apply -f -
|
|
||||||
|
|
||||||
CLOUD_POD=$(kubectl get pod -l app=cloud-app -o name)
|
|
||||||
kubectl wait $CLOUD_POD --for=condition=Ready --timeout=240s
|
|
||||||
|
|
||||||
# wait for nextcloud config file available
|
|
||||||
timeout 180 bash -c "kubectl exec -t $POD -- bash -c \"until [ -f /var/www/html/config/config.php ]; do sleep 10; done\""
|
|
||||||
|
|
||||||
# ensure an instance of pod backup-restore
|
|
||||||
kubectl scale deployment backup-restore --replicas 1
|
|
||||||
|
|
||||||
# wait for localstack health endpoint
|
|
||||||
echo "$INGRESS_IP k3stesthost cloudhost" >> /etc/hosts
|
|
||||||
until curl --fail --silent k3stesthost/health | grep -oe '"s3": "available"' -oe '"s3": "running"'
|
|
||||||
do
|
|
||||||
curl --fail k3stesthost/health
|
|
||||||
echo "[INFO] Waiting for s3 running"
|
|
||||||
sleep 10
|
|
||||||
done
|
|
||||||
|
|
||||||
BACKUP_POD=$(kubectl get pod -l app=backup-restore -o name)
|
|
||||||
kubectl wait $BACKUP_POD --for=condition=Ready --timeout=240s
|
|
||||||
|
|
||||||
kubectl exec -t $BACKUP_POD -- bash -c "echo \"$INGRESS_IP k3stesthost cloudhost\" >> /etc/hosts"
|
|
||||||
kubectl exec -t $BACKUP_POD -- /usr/local/bin/init.sh
|
|
||||||
|
|
||||||
echo ================= BACKUP =================
|
|
||||||
kubectl exec -t $BACKUP_POD -- /usr/local/bin/backup.sh
|
|
||||||
|
|
||||||
sleep 10 # avoid race conditions
|
|
||||||
|
|
||||||
echo ================= RESTORE =================
|
|
||||||
kubectl exec -t $BACKUP_POD -- /usr/local/bin/restore.sh
|
|
||||||
}
|
|
||||||
|
|
||||||
main "$@"
|
|
|
@ -1,34 +0,0 @@
|
||||||
function main()
|
|
||||||
{
|
|
||||||
local bucket_name="${1:-mybucket}"; shift
|
|
||||||
|
|
||||||
./start-k3s.sh
|
|
||||||
|
|
||||||
sudo k3s kubectl apply -f https://github.com/jetstack/cert-manager/releases/download/v1.5.4/cert-manager.yaml
|
|
||||||
|
|
||||||
sudo k3s kubectl apply -f localstack.yaml
|
|
||||||
|
|
||||||
until sudo k3s kubectl apply -f certificate.yaml
|
|
||||||
do
|
|
||||||
echo "*** Waiting for certificate ... ***"
|
|
||||||
sleep 10
|
|
||||||
done
|
|
||||||
echo
|
|
||||||
|
|
||||||
echo
|
|
||||||
echo "[INFO] Waiting for localstack health endpoint"
|
|
||||||
until curl --connect-timeout 3 -s -f -o /dev/null "k3stesthost/health"
|
|
||||||
do
|
|
||||||
sleep 5
|
|
||||||
done
|
|
||||||
echo
|
|
||||||
|
|
||||||
sudo k3s kubectl get secret localstack-secret -o jsonpath="{.data.ca\.crt}" | base64 --decode > ca.crt
|
|
||||||
|
|
||||||
#aws --endpoint-url=http://localhost s3 mb s3://$bucket_name
|
|
||||||
export RESTIC_PASSWORD="test-password"
|
|
||||||
restic init --cacert ca.crt -r s3://k3stesthost/$bucket_name
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
main $@
|
|
|
@ -1,9 +0,0 @@
|
||||||
function main()
|
|
||||||
{
|
|
||||||
./start-k3s.sh
|
|
||||||
|
|
||||||
sudo k3s kubectl apply -f https://github.com/jetstack/cert-manager/releases/download/v1.5.4/cert-manager.yaml
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
main
|
|
|
@ -1 +0,0 @@
|
||||||
KUBECONFIG=~/.kube/custom-contexts/k3d-config.yml k3d cluster create nextcloud --k3s-arg '--tls-san cloudhost@loadbalancer' --port 80:80@loadbalancer --port 443:443@loadbalancer --api-port 6443 --kubeconfig-update-default
|
|
|
@ -1 +0,0 @@
|
||||||
curl -sfL https://get.k3s.io | K3S_NODE_NAME=k3stesthost INSTALL_K3S_EXEC='--tls-san cloudhost' sh -
|
|
Loading…
Reference in a new issue