Compare commits

...

96 commits
8.0.1 ... main

Author SHA1 Message Date
6a04396984 bump version to: 10.4.5-SNAPSHOT 2025-01-11 15:13:18 +01:00
7b3143beca release: 10.4.4 2025-01-11 15:13:18 +01:00
a0e47d74fc fix browser 2025-01-11 15:11:44 +01:00
bd22b84a32 bump version to: 10.4.4-SNAPSHOT 2025-01-11 14:48:25 +01:00
a06409fadf release: 10.4.3 2025-01-11 14:48:25 +01:00
27987dfc8b refactor to fix spec location 2025-01-11 14:27:12 +01:00
b22ace7c6a update version 2025-01-11 13:17:10 +01:00
ad16cb52bd bump version to: 10.4.3-SNAPSHOT 2025-01-11 11:36:12 +01:00
58eccda7a7 release: 10.4.2 2025-01-11 11:36:12 +01:00
242ebdc2ff cleanup image & update test 2025-01-11 11:32:17 +01:00
5f69b54f15 use newest version 2025-01-11 11:07:43 +01:00
78af31036e fix config 2025-01-11 11:07:17 +01:00
1c596889ad fine grained restore config 2025-01-10 08:34:59 +01:00
2aec56fbd6 one step back - ubuntu24 does not contain pg17 2025-01-09 20:19:38 +01:00
d748a36559 adjust doc 2025-01-09 20:10:18 +01:00
71bd701d13 bump version to: 10.4.2-SNAPSHOT 2025-01-09 18:53:22 +01:00
d489eee954 release: 10.4.1 2025-01-09 18:53:22 +01:00
939523bc2e use new nextcloud 2025-01-09 18:34:31 +01:00
29fc04830a adjust upgrade doc 2025-01-09 15:15:57 +01:00
0b9739fdb6 bump version to: 10.4.1-SNAPSHOT 2025-01-09 14:21:29 +01:00
164b691bc9 release: 10.4.0 2025-01-09 14:21:28 +01:00
9685f07e46 bump version to: 10.3.1-SNAPSHOT 2025-01-09 14:07:56 +01:00
6e1cc7bda2 release: 10.3.0 2025-01-09 14:07:56 +01:00
304bb3c967 update dep 2025-01-09 14:03:43 +01:00
jem
e26d67a40c Merge pull request 'refactor for new backup' () from new-backup into main
Reviewed-on: 
2025-01-09 12:58:39 +00:00
8cb6bf6b28 Merge branch 'main' into new-backup 2025-01-09 13:35:41 +01:00
jem
9fc9687a08 Merge pull request 'feat: upgrade-and-fix-postgres' () from upgrade-and-fix-postgres into main
Reviewed-on: 
2025-01-09 12:11:44 +00:00
20c66235c0 upgrade nextcloud 2025-01-09 12:55:51 +01:00
741b720632 upgrade versions 2025-01-09 12:55:40 +01:00
394959d158 update db 2025-01-09 12:54:27 +01:00
711ceecefc upgrade versions 2025-01-09 09:24:46 +01:00
dad69a180b versions update 2025-01-08 16:52:21 +01:00
6de3e47bd2 update refactoring 2024-12-31 11:09:22 +01:00
03b0b85247 refactor to babashka driven backup 2024-12-30 14:21:27 +01:00
67ec0f58eb implement backup & snapshot 2024-12-28 18:30:49 +01:00
d7ce373d87 refactoring - split config & auth 2024-12-28 09:59:43 +01:00
d1a8479598 update federation 2024-08-05 08:57:40 +02:00
c4832b1107 remove unused 2024-06-07 17:19:07 +02:00
7e3312e285 switch to main 2024-05-31 17:17:16 +02:00
f636f7ffc3 update deps 2024-05-31 17:17:07 +02:00
bba6bbe830 bump version to: 10.2.1-SNAPSHOT 2024-05-31 17:03:56 +02:00
8388d72517 release: 10.2.0 2024-05-31 17:03:56 +02:00
f2b583c060 adjust version 2024-05-31 16:58:27 +02:00
5a3aca38cf use new fkt 2024-05-31 16:48:33 +02:00
4764d1db67 use the new pcv defaults 2024-05-31 16:43:06 +02:00
8aef785bdc reenable lint & native build 2024-05-31 16:42:51 +02:00
5eb83f78a0 update deps 2024-05-31 16:14:26 +02:00
d997e470a0 remove integration tests 2024-05-31 16:14:12 +02:00
bom
dc86531454 Fix default storage class
Our default was not actually valid
2024-05-17 15:15:16 +02:00
ca0d4ac7b2 add the upgrade experience 2024-04-26 15:38:28 +02:00
bom
e4666a592e bump version to: 10.1.1-SNAPSHOT 2024-04-26 14:53:08 +02:00
bom
fa48d9762a release: 10.1.0 2024-04-26 14:53:08 +02:00
bom
6a278ece0d Upgrade nextcloud 2024-04-26 14:52:19 +02:00
dda45d92d6 update instructions 2024-04-26 13:07:05 +02:00
bom
c69c9da659 bump version to: 10.0.1-SNAPSHOT 2024-04-26 10:27:14 +02:00
bom
de53b9b7a5 release: 10.0.0 2024-04-26 10:27:14 +02:00
bom
cc845af696 Bump clojure version 2024-04-26 10:26:33 +02:00
bom
f5aa2295f0 bump version to: 9.0.1-SNAPSHOT 2024-04-26 10:25:57 +02:00
bom
1d344abc27 release: 9.0.0 2024-04-26 10:25:57 +02:00
bom
7cd4e4101d No longer package native if we dont use it 2024-04-25 14:29:59 +02:00
bom
5d2a65079e Add Doc to rename the cloud DB 2024-04-25 14:23:38 +02:00
bom
b36808de7c Change db-name to cloud 2024-04-25 14:23:06 +02:00
bom
3e588c082c Mark rotation-credential as optional 2024-04-25 14:22:43 +02:00
bom
4a74a1bec0 bump version to: 8.0.9-SNAPSHOT 2024-04-19 13:03:32 +02:00
bom
1ff8a0dc13 release: 8.0.8 2024-04-19 13:03:32 +02:00
bom
fd27c15ec7 lein ancient upgrade 2024-04-19 13:02:52 +02:00
bom
192e053afc Update upgrading docs 2024-04-19 13:00:37 +02:00
cab9b573c1 add namespace to metadata in credential-rotation.yaml 2024-04-17 11:37:52 +02:00
226046d278 test credential for rotation-credential 2024-04-17 09:20:03 +02:00
f108b67e62 lint fix use current versions 2024-04-12 11:50:21 +02:00
96343b9af5 Merge branch 'master' of ssh://repo.prod.meissa.de:2222/meissa/c4k-nextcloud 2024-04-12 11:27:28 +02:00
f072ff027d add rotation-credential-volume to backup-restore-deployment 2024-04-12 11:27:23 +02:00
bom
e396880365 bump version to: 8.0.8-SNAPSHOT 2024-03-22 15:30:08 +01:00
bom
83080b30a8 release: 8.0.7 2024-03-22 15:30:08 +01:00
bom
4a78a35424 Pass namespace to ingress-and-cert 2024-03-22 15:29:17 +01:00
bom
8727f16c75 Add namespace to backup/restore commands 2024-03-22 15:29:01 +01:00
bom
ac3f2a455d Move namespace before postgres 2024-03-22 15:08:10 +01:00
bom
c05ecfa427 Pass namespace to postgres 2024-03-22 15:02:24 +01:00
bom
fff2c939d9 bump version to: 8.0.7-SNAPSHOT 2024-03-22 14:05:09 +01:00
bom
e8cec9de8a release: 8.0.6 2024-03-22 14:05:08 +01:00
bom
1fb309f213 [skip-ci] Remove project parameters 2024-03-22 14:04:25 +01:00
bom
70d41ca532 Bump c4k-common cljs version 2024-03-22 13:56:35 +01:00
bom
a96cba8cb1 Add test task that runs every test 2024-03-22 13:53:02 +01:00
bom
3bdf2ea553 Require namespace namespace 2024-03-22 13:52:45 +01:00
bom
fcf6d7783e Add namespace 2024-03-22 13:42:31 +01:00
bom
ab4c6e0d76 bump version to: 8.0.6-SNAPSHOT 2024-03-15 15:47:46 +01:00
bom
ef8bfa11fc release: 8.0.5 2024-03-15 15:47:46 +01:00
bom
7d9ca203bb bump version to: 8.0.5-SNAPSHOT 2024-03-15 15:36:34 +01:00
bom
f8bcbe63ba release: 8.0.4 2024-03-15 15:36:34 +01:00
bom
7219533c86 bump version to: 8.0.4-SNAPSHOT 2024-03-15 14:29:49 +01:00
bom
9b13b96ff6 release: 8.0.3 2024-03-15 14:29:49 +01:00
bom
351b2295e3 Add auth to postgres generate 2024-03-15 14:29:06 +01:00
bom
3a9694d9a1 bump version to: 8.0.3-SNAPSHOT 2024-03-15 14:27:40 +01:00
bom
3b458b980b release: 8.0.2 2024-03-15 14:27:40 +01:00
bom
5f38fb7526 Adapt core to changes in c4k-common 2024-03-15 14:26:55 +01:00
bom
fa81908791 bump version to: 8.0.2-SNAPSHOT 2024-03-15 14:14:52 +01:00
67 changed files with 585 additions and 771 deletions

View file

@ -4,7 +4,6 @@ stages:
- security - security
- upload - upload
- image - image
#- integrationtest
.img: &img .img: &img
image: "domaindrivenarchitecture/ddadevops-dind:4.11.3" image: "domaindrivenarchitecture/ddadevops-dind:4.11.3"
@ -130,23 +129,3 @@ nextcloud-image-publish:
stage: image stage: image
script: script:
- cd infrastructure/nextcloud && pyb image publish - cd infrastructure/nextcloud && pyb image publish
#.nextcloud-integrationtest:
# stage: integrationtest
# image: registry.gitlab.com/gitlab-org/cluster-integration/helm-install-image/releases/3.7.1-kube-1.20.11-alpine-3.14
# services:
# - name: registry.gitlab.com/gitlab-org/cluster-integration/test-utils/k3s-gitlab-ci/releases/v1.22.2-k3s2
# alias: k3s
# script:
# - apk add curl sudo bash
# - apk add wget curl bash sudo openjdk8
# - wget -P /etc/apk/keys/ https://alpine-pkgs.sgerrand.com/sgerrand.rsa.pub
# - apk add --no-cache --repository=https://apkproxy.herokuapp.com/sgerrand/alpine-pkg-leiningen leiningen
#
# - mkdir -p ${HOME}/.kube/
# - curl -f k3s:8081 > ${HOME}/.kube/config
# - kubectl version
# - kubectl cluster-info
# - echo "---------- Integration test -------------"
# - pwd
# - cd ./src/test/resources/local-integration-test/ && ./setup-local-s3-on-k3d.sh

View file

@ -41,7 +41,8 @@ Development happens at: https://repo.prod.meissa.de/meissa/c4k-nextcloud
Mirrors are: Mirrors are:
* https://gitlab.com/domaindrivenarchitecture/c4k-nextcloud (issues and PR, CI) * https://codeberg.org/meissa/c4k-nextcloud (Issues and PR)
* https://gitlab.com/domaindrivenarchitecture/c4k-nextcloud (CI)
* https://github.com/DomainDrivenArchitecture/c4k-nextcloud * https://github.com/DomainDrivenArchitecture/c4k-nextcloud
For more details about our repository model see: https://repo.prod.meissa.de/meissa/federate-your-repos For more details about our repository model see: https://repo.prod.meissa.de/meissa/federate-your-repos
@ -49,6 +50,6 @@ For more details about our repository model see: https://repo.prod.meissa.de/mei
## License ## License
Copyright © 2021 meissa GmbH Copyright © 2021, 2022, 2023, 2024 meissa GmbH
Licensed under the [Apache License, Version 2.0](LICENSE) (the "License") Licensed under the [Apache License, Version 2.0](LICENSE) (the "License")
Pls. find licenses of our subcomponents [here](doc/SUBCOMPONENT_LICENSE) Pls. find licenses of our subcomponents [here](doc/SUBCOMPONENT_LICENSE)

View file

@ -1,7 +0,0 @@
{:postgres-db-user "nextcloud"
:postgres-db-password "dbpass"
:nextcloud-admin-user "cloudadmin"
:nextcloud-admin-password "cloudpassword"
:aws-access-key-id ""
:aws-secret-access-key ""
:restic-password "test-password"}

View file

@ -33,7 +33,7 @@ def initialize(project):
f"target/uberjar/{name}-standalone.jar", f"target/uberjar/{name}-standalone.jar",
f"target/frontend-build/{name}.js", f"target/frontend-build/{name}.js",
], ],
"release_main_branch": "master", "release_main_branch": "main",
} }
build = ReleaseMixin(project, input) build = ReleaseMixin(project, input)
@ -41,18 +41,18 @@ def initialize(project):
@task @task
def test_clj(project): def test_clj():
run("lein test", shell=True, check=True) run("lein test", shell=True, check=True)
@task @task
def test_cljs(project): def test_cljs():
run("shadow-cljs compile test", shell=True, check=True) run("shadow-cljs compile test", shell=True, check=True)
run("node target/node-tests.js", shell=True, check=True) run("node target/node-tests.js", shell=True, check=True)
@task @task
def test_schema(project): def test_schema():
run("lein uberjar", shell=True, check=True) run("lein uberjar", shell=True, check=True)
run( run(
"java -jar target/uberjar/c4k-nextcloud-standalone.jar " "java -jar target/uberjar/c4k-nextcloud-standalone.jar "
@ -63,6 +63,11 @@ def test_schema(project):
check=True, check=True,
) )
@task
def test():
test_clj()
test_cljs()
test_schema()
@task @task
def report_frontend(project): def report_frontend(project):
@ -148,11 +153,11 @@ def upload_clj(project):
@task @task
def lint(project): def lint(project):
#run( run(
# "lein eastwood", "lein eastwood",
# shell=True, shell=True,
# check=True, check=True,
#) )
run( run(
"lein ancient check", "lein ancient check",
shell=True, shell=True,
@ -222,7 +227,7 @@ def release(project):
def linttest(project, release_type): def linttest(project, release_type):
build = get_devops_build(project) build = get_devops_build(project)
build.update_release_type(release_type) build.update_release_type(release_type)
test_clj(project) test_clj()
test_cljs(project) test_cljs()
test_schema(project) test_schema()
lint(project) lint(project)

View file

@ -1,6 +0,0 @@
{:fqdn "cloudhost"
:issuer :staging
:nextcloud-data-volume-path "/var/cloud"
:postgres-data-volume-path "/var/postgres"
:restic-repository "s3://k3stesthost/mybucket"
:local-integration-test true}

View file

@ -7,40 +7,28 @@
* Cloud stores files on `/var/jira`, these files are backuped. If you create a jira xml backup located in /var/jira this file will also be backed up. * Cloud stores files on `/var/jira`, these files are backuped. If you create a jira xml backup located in /var/jira this file will also be backed up.
* postgres db is backed up as pgdump * postgres db is backed up as pgdump
## Manual init the restic repository for the first time ## Manual backup
1. Scale backup-restore deployment up:
`kubectl scale deployment backup-restore --replicas=1`
1. exec into pod and execute restore pod
`kubectl exec -it backup-restore -- /usr/local/bin/init.sh`
1. Scale backup-restore deployment down:
`kubectl scale deployment backup-restore --replicas=0`
## Manual backup the restic repository for the first time
1. Scale Cloud deployment down: 1. Scale Cloud deployment down:
`kubectl scale deployment cloud-deployment --replicas=0` `kubectl -n nextcloud scale deployment cloud-deployment --replicas=0`
1. Scale backup-restore deployment up: 1. Scale backup-restore deployment up:
`kubectl scale deployment backup-restore --replicas=1` `kubectl -n nextcloud scale deployment backup-restore --replicas=1`
1. exec into pod and execute restore pod 1. exec into pod and execute restore pod
`kubectl exec -it backup-restore -- /usr/local/bin/backup.sh` `kubectl -n nextcloud exec -it backup-restore -- backup.bb`
1. Scale backup-restore deployment down: 1. Scale backup-restore deployment down:
`kubectl scale deployment backup-restore --replicas=0` `kubectl -n nextcloud scale deployment backup-restore --replicas=0`
1. Scale Cloud deployment up: 1. Scale Cloud deployment up:
`kubectl scale deployment cloud-deployment --replicas=1` `kubectl -n nextcloud scale deployment cloud-deployment --replicas=1`
## Manual restore ## Manual restore
1. Scale Cloud deployment down: 1. Scale Cloud deployment down:
`kubectl scale deployment cloud-deployment --replicas=0` `kubectl -n nextcloud scale deployment cloud-deployment --replicas=0`
2. Scale backup-restore deployment up: 2. Scale backup-restore deployment up:
`kubectl scale deployment backup-restore --replicas=1` `kubectl -n nextcloud scale deployment backup-restore --replicas=1`
3. exec into pod and execute restore pod 3. exec into pod and execute restore pod
`kubectl exec -it backup-restore -- /usr/local/bin/restore.sh` `kubectl -n nextcloud exec -it backup-restore -- restore.bb`
4. Scale backup-restore deployment down: 4. Scale backup-restore deployment down:
`kubectl scale deployment backup-restore --replicas=0` `kubectl -n nextcloud scale deployment backup-restore --replicas=0`
5. Scale Cloud deployment up: 5. Scale Cloud deployment up:
`kubectl scale deployment cloud-deployment --replicas=1` `kubectl -n nextcloud scale deployment cloud-deployment --replicas=1`

View file

@ -5,12 +5,18 @@
- 4.0.3: nextcloud 22 - 4.0.3: nextcloud 22
- 5.0.0: nextcloud 23 - 5.0.0: nextcloud 23
- 6.0.0: nextcloud 24 - 6.0.0: nextcloud 24
- 7.0.0: nextcloud 25 - 7.0.7: nextcloud 25.0.13
- 7.1.1: nextcloud 26.0.0 (manual publish) => attention - only upgrade to 26.0.0 is working
- 7.1.0: nextcloud 26.0.13 (manual publish)
- 7.2.0: nextcloud 27 (manual publish)
- 10.0.0: nextcloud 28.0.5
- 10.1.0: nextcloud 29.0.0
- 10.4.2: nextcloud 30
## Uprgrading process ## Uprgrading process
1. Change the version of the docker image in the deployment to the next major version 1. Change the version of the docker image in the deployment to the next major version
- `kubectl edit deploy cloud-deployment` - `kubectl -n=nextcloud edit deploy cloud-deployment`
- change `image: domaindrivenarchitecture/c4k-cloud:4.0.3` - change `image: domaindrivenarchitecture/c4k-cloud:4.0.3`
2. Wait for the pod to finish restarting 2. Wait for the pod to finish restarting
3. Verify the website is working and https://URL/settings/admin/overview shows the correct version 3. Verify the website is working and https://URL/settings/admin/overview shows the correct version

41
doc/RenameDatabase.md Normal file
View file

@ -0,0 +1,41 @@
# Rename Database
## Start
1. Scale down cloud deployment
`k -n nextcloud scale deployment cloud-deployment --replicas 0`
## Change db-name in postgres
1. Connect to postgres-pod
`k -n nextcloud exec -it postgresql-... -- bash`
2. Connect to a database
`PGPASSWORD=$POSTGRES_PASSWORD psql -h postgresql-service -U $POSTGRES_USER postgres`
3. List available databases
`\l`
4. Rename database
`ALTER DATABASE cloud RENAME TO nextcloud;`
5. Verify
`\l`
6. Quit
`\q`
## Update postgres-config
1. Edit configmap
`k -n nextcloud edit configmap postgres-config`
2. Update postgres-db value
3. Save
## Update nextcloud db-name
1. Scale up nextcloud
`k -n nextcloud scale deployment cloud-deployment --replicas 1`
2. Connect
`k -n nextcloud exec -it cloud-deployment-... -- bash`
3. Update db value in config.php
`apt update`
`apt install vim`
`vim config/config.php`
4. Update dbname field
5. Verify server+website is working

View file

@ -6,7 +6,7 @@ from ddadevops import *
name = "c4k-cloud" name = "c4k-cloud"
MODULE = "backup" MODULE = "backup"
PROJECT_ROOT_PATH = "../.." PROJECT_ROOT_PATH = "../.."
version = "8.0.1" version = "10.4.5-dev"
@init @init

View file

@ -1,5 +1,7 @@
FROM domaindrivenarchitecture/dda-backup:latest FROM domaindrivenarchitecture/dda-backup:5.2.1
# Prepare Entrypoint Script # Prepare Entrypoint Script
ADD resources /tmp ADD resources /tmp
RUN /tmp/install.sh RUN /tmp/install.bb
RUN init.bb
#ADD resources2 /tmp

View file

@ -0,0 +1,35 @@
#!/usr/bin/env bb
(require
'[babashka.fs :as fs])
(-> "/usr/local/bin/config.clj" fs/file load-file)
(require
'[babashka.tasks :as t]
'[dda.backup.core :as bc]
'[dda.backup.restic :as rc]
'[dda.backup.postgresql :as pg]
'[dda.backup.backup :as bak]
'[config :as cf])
(defn prepare!
[]
(bc/create-aws-credentials! cf/aws-config)
(pg/create-pg-pass! cf/db-config))
(defn restic-repo-init!
[]
(rc/init! cf/file-config)
(rc/init! cf/db-role-config)
(rc/init! cf/db-config))
(defn restic-backup!
[]
(bak/backup-file! cf/file-config)
(bak/backup-db-roles! cf/db-role-config)
(bak/backup-db! cf/db-config))
(t/shell "start-maintenance.sh")
(prepare!)
(restic-repo-init!)
(restic-backup!)
(t/shell "end-maintenance.sh")

View file

@ -1,28 +0,0 @@
#!/bin/bash
set -Eexo pipefail
function main() {
start-maintenance.sh
file_env AWS_ACCESS_KEY_ID
file_env AWS_SECRET_ACCESS_KEY
file_env POSTGRES_DB
file_env POSTGRES_PASSWORD
file_env POSTGRES_USER
file_env RESTIC_DAYS_TO_KEEP 30
file_env RESTIC_MONTHS_TO_KEEP 12
backup-roles 'oc_'
backup-db-dump
backup-directory '/var/backups/'
end-maintenance.sh
}
source /usr/local/lib/functions.sh
source /usr/local/lib/pg-functions.sh
source /usr/local/lib/file-functions.sh
main

View file

@ -0,0 +1,3 @@
{:deps {org.clojure/spec.alpha {:mvn/version "0.4.233"}
orchestra/orchestra {:mvn/version "2021.01.01-1"}
org.domaindrivenarchitecture/dda-backup {:local/root "/usr/local/lib/dda-backup"}}}

View file

@ -0,0 +1,3 @@
{:deps {org.clojure/spec.alpha {:mvn/version "0.4.233"}
orchestra/orchestra {:mvn/version "2021.01.01-1"}
org.domaindrivenarchitecture/dda-build {:mvn/version "0.2.0"}}}

View file

@ -0,0 +1,61 @@
(ns config
(:require
[dda.backup.core :as bc]))
(def restic-repo {:password-file (bc/env-or-file "RESTIC_PASSWORD_FILE")
:restic-repository (bc/env-or-file "RESTIC_REPOSITORY")})
(def file-config (merge restic-repo {:backup-path "files"
:execution-directory "/var/backups"
:restore-target-directory "/var/backups/"
:files ["."]}))
(def file-restore-config (merge restic-repo {:backup-path "files"
:restore-target-directory "/var/backups/"
:clean-up-elements [".htaccess"
".reuse/"
".user.ini"
"3rdparty/"
"apps/"
"composer.json"
"composer.lock"
"console.php"
"core/"
"cron.php"
"custom_apps/"
"data/"
"dist/"
"index.html"
"index.php"
"lib/"
"ocs/"
"ocs-provider/"
"package-lock.json"
"package.json"
"public.php"
"remote.php"
"resources"
"robots.txt"
"status.php"
"themes/"
"version.php"]}))
(def db-config (merge restic-repo {:backup-path "pg-database"
:pg-host (bc/env-or-file "POSTGRES_SERVICE")
:pg-port (bc/env-or-file "POSTGRES_PORT")
:pg-db (bc/env-or-file "POSTGRES_DB")
:pg-user (bc/env-or-file "POSTGRES_USER")
:pg-password (bc/env-or-file "POSTGRES_PASSWORD")}))
(def db-role-config (merge restic-repo {:backup-path "pg-role"
:pg-role-prefix "oc_"
:pg-host (bc/env-or-file "POSTGRES_SERVICE")
:pg-port (bc/env-or-file "POSTGRES_PORT")
:pg-db (bc/env-or-file "POSTGRES_DB")
:pg-user (bc/env-or-file "POSTGRES_USER")
:pg-password (bc/env-or-file "POSTGRES_PASSWORD")}))
(def aws-config {:aws-access-key-id (bc/env-or-file "AWS_ACCESS_KEY_ID")
:aws-secret-access-key (bc/env-or-file "AWS_SECRET_ACCESS_KEY")})
(def dry-run {:dry-run true :debug true})

View file

@ -1,19 +0,0 @@
#!/bin/bash
set -exo pipefail
function main() {
file_env POSTGRES_DB
file_env POSTGRES_PASSWORD
file_env POSTGRES_USER
create-pg-pass
while true; do
sleep 1m
done
}
source /usr/local/lib/functions.sh
source /usr/local/lib/pg-functions.sh
main

View file

@ -1,17 +0,0 @@
#!/bin/bash
set -Eexo pipefail
function main() {
file_env POSTGRES_DB
file_env POSTGRES_PASSWORD
file_env POSTGRES_USER
create-pg-pass
/usr/local/bin/backup.sh
}
source /usr/local/lib/functions.sh
source /usr/local/lib/pg-functions.sh
main

View file

@ -0,0 +1,3 @@
#!/usr/bin/env bb
(println "initialized")

View file

@ -1,17 +0,0 @@
#!/bin/bash
set -Eexo pipefail
function main() {
file_env AWS_ACCESS_KEY_ID
file_env AWS_SECRET_ACCESS_KEY
init-role-repo
init-database-repo
init-file-repo
}
source /usr/local/lib/functions.sh
source /usr/local/lib/pg-functions.sh
source /usr/local/lib/file-functions.sh
main

View file

@ -0,0 +1,19 @@
#!/usr/bin/env bb
(require
'[dda.image.ubuntu :as ub]
'[dda.image.install :as in])
(ub/upgrade-system!)
(in/install! "bb-backup.edn" :target-name "bb.edn" :mod "0440")
(in/install! "config.clj" :mod "0440")
(in/install! "init.bb")
(in/install! "backup.bb")
(in/install! "restore.bb")
(in/install! "list-snapshots.bb")
(in/install! "start-maintenance.sh")
(in/install! "end-maintenance.sh")
(in/install! "restore.bb")
(in/install! "wait.bb")
(ub/cleanup-container!)

View file

@ -1,21 +0,0 @@
#!/bin/bash
set -exo pipefail
function main() {
{
install -m 0700 /tmp/entrypoint.sh /
install -m 0700 /tmp/entrypoint-start-and-wait.sh /
install -m 0700 /tmp/init.sh /usr/local/bin/
install -m 0700 /tmp/backup.sh /usr/local/bin/
install -m 0700 /tmp/restore.sh /usr/local/bin/
install -m 0700 /tmp/list-snapshots.sh /usr/local/bin/
install -m 0700 /tmp/start-maintenance.sh /usr/local/bin/
install -m 0700 /tmp/end-maintenance.sh /usr/local/bin/
cleanupDocker
} > /dev/null
}
source /tmp/install_functions_debian.sh
main

View file

@ -0,0 +1,22 @@
#!/usr/bin/env bb
(require
'[babashka.fs :as fs])
(-> "/usr/local/bin/config.clj" fs/file load-file)
(require
'[dda.backup.core :as bc]
'[dda.backup.restic :as rc]
'[config :as cf])
(defn prepare!
[]
(bc/create-aws-credentials! cf/aws-config))
(defn list-snapshots!
[]
(rc/list-snapshots! cf/file-config)
(rc/list-snapshots! cf/db-role-config)
(rc/list-snapshots! cf/db-config))
(prepare!)
(list-snapshots!)

View file

@ -1,31 +0,0 @@
#!/bin/bash
set -exo pipefail
function list-snapshot-files() {
if [ -z ${CERTIFICATE_FILE} ];
then
restic -r ${RESTIC_REPOSITORY}/${backup_file_path} snapshots
else
restic -r ${RESTIC_REPOSITORY}/${backup_file_path} snapshots --cacert ${CERTIFICATE_FILE}
fi
}
function main() {
file_env AWS_ACCESS_KEY_ID
file_env AWS_SECRET_ACCESS_KEY
file_env POSTGRES_DB
file_env POSTGRES_PASSWORD
file_env POSTGRES_USER
list-snapshot-roles
list-snapshot-db
list-snapshot-files
}
source /usr/local/lib/functions.sh
source /usr/local/lib/file-functions.sh
source /usr/local/lib/pg-functions.sh
main

View file

@ -0,0 +1,29 @@
#!/usr/bin/env bb
(require
'[babashka.fs :as fs])
(-> "/usr/local/bin/config.clj" fs/file load-file)
(require
'[babashka.tasks :as t]
'[dda.backup.core :as bc]
'[dda.backup.postgresql :as pg]
'[dda.backup.restore :as rs]
'[config :as cf])
(defn prepare!
[]
(bc/create-aws-credentials! cf/aws-config)
(pg/create-pg-pass! cf/db-config))
(defn restic-restore!
[]
(pg/drop-create-db! cf/db-config)
(rs/restore-db-roles! cf/db-role-config)
(rs/restore-db! cf/db-config)
(rs/restore-file! cf/file-restore-config)
)
(t/shell "start-maintenance.sh")
(prepare!)
(restic-restore!)
(t/shell "end-maintenance.sh")

View file

@ -1,33 +0,0 @@
#!/bin/bash
set -Eexo pipefail
function main() {
local role_snapshot_id="${1:-latest}"
local db_snapshot_id="${2:-latest}"
local file_snapshot_id="${3:-latest}"
start-maintenance.sh
file_env AWS_ACCESS_KEY_ID
file_env AWS_SECRET_ACCESS_KEY
file_env POSTGRES_DB
file_env POSTGRES_PASSWORD
file_env POSTGRES_USER
drop-create-db
restore-roles ${role_snapshot_id}
restore-db ${db_snapshot_id}
restore-directory '/var/backups/' ${file_snapshot_id}
end-maintenance.sh
}
source /usr/local/lib/functions.sh
source /usr/local/lib/pg-functions.sh
source /usr/local/lib/file-functions.sh
main "$@"

View file

@ -0,0 +1,21 @@
#!/usr/bin/env bb
(require
'[babashka.fs :as fs])
(-> "/usr/local/bin/config.clj" fs/file load-file)
(require
'[dda.backup.core :as bc]
'[dda.backup.postgresql :as pg]
'[config :as cf])
(defn prepare!
[]
(bc/create-aws-credentials! cf/aws-config)
(pg/create-pg-pass! cf/db-config))
(defn wait! []
(while true
(Thread/sleep 1000)))
(prepare!)
(wait!)

View file

@ -0,0 +1,3 @@
{:deps {org.clojure/spec.alpha {:mvn/version "0.4.233"}
orchestra/orchestra {:mvn/version "2021.01.01-1"}
org.domaindrivenarchitecture/dda-backup {:local/root "/usr/local/lib/dda-backup"}}}

View file

@ -0,0 +1,9 @@
export RESTIC_PASSWORD_FILE=/tmp/file_password
export RESTIC_REPOSITORY=/var/restic-repo
export POSTGRES_SERVICE=dummy
export POSTGRES_PORT=dummy
export POSTGRES_DB=dummy
export POSTGRES_USER=dummy
export POSTGRES_PASSWORD=dummy
export AWS_ACCESS_KEY_ID=dummy
export AWS_SECRET_ACCESS_KEY=dummy

View file

@ -0,0 +1 @@
oldPassword

View file

@ -0,0 +1,51 @@
#!/usr/bin/env bb
(require
'[babashka.fs :as fs])
(-> "/usr/local/bin/config.clj" fs/file load-file)
(require '[babashka.tasks :as tasks]
'[dda.backup.core :as bc]
'[dda.backup.restic :as rc]
'[dda.backup.postgresql :as pg]
'[dda.backup.backup :as bak]
'[dda.backup.restore :as rs]
'[config :as cf])
(defn prepare!
[]
(tasks/shell "mkdir" "-p" "/var/backups/")
(tasks/shell "mkdir" "-p" "/var/restic-repo/")
(spit "/var/backups/file" "I was here"))
(defn restic-repo-init!
[]
(rc/init! cf/file-config)
(rc/init! (merge cf/db-role-config cf/dry-run))
(rc/init! (merge cf/db-config cf/dry-run)))
(defn restic-backup!
[]
(bak/backup-file! cf/file-config)
(bak/backup-db-roles! (merge cf/db-role-config cf/dry-run))
(bak/backup-db! (merge cf/db-config cf/dry-run)))
(defn list-snapshots!
[]
(rc/list-snapshots! cf/file-config)
(rc/list-snapshots! (merge cf/db-role-config cf/dry-run))
(rc/list-snapshots! (merge cf/db-config cf/dry-run)))
(defn restic-restore!
[]
(pg/drop-create-db! (merge cf/db-config cf/dry-run))
(rs/restore-db-roles! (merge cf/db-role-config cf/dry-run))
(rs/restore-db! (merge cf/db-config cf/dry-run))
(rs/restore-file! cf/file-restore-config))
(prepare!)
(restic-repo-init!)
(restic-backup!)
(list-snapshots!)
(restic-restore!)

View file

@ -0,0 +1,4 @@
FROM c4k-cloud-backup:latest
ADD resources /tmp/
RUN RESTIC_PASSWORD_FILE=/tmp/file_password RESTIC_REPOSITORY=restic-repo POSTGRES_SERVICE=dummy POSTGRES_PORT=dummy POSTGRES_DB=dummy POSTGRES_USER=dummy POSTGRES_PASSWORD=dummy AWS_ACCESS_KEY_ID=dummy AWS_SECRET_ACCESS_KEY=dummy /tmp/test.bb

View file

@ -0,0 +1,3 @@
{:deps {org.clojure/spec.alpha {:mvn/version "0.4.233"}
orchestra/orchestra {:mvn/version "2021.01.01-1"}
org.domaindrivenarchitecture/dda-backup {:local/root "/usr/local/lib/dda-backup"}}}

View file

@ -0,0 +1 @@
oldPassword

View file

@ -0,0 +1,49 @@
#!/usr/bin/env bb
(require
'[babashka.fs :as fs])
(-> "/usr/local/bin/config.clj" fs/file load-file)
(require '[babashka.tasks :as tasks]
'[dda.backup.restic :as rc]
'[dda.backup.postgresql :as pg]
'[dda.backup.backup :as bak]
'[dda.backup.restore :as rs]
'[config :as cf])
(defn prepare!
[]
(tasks/shell "mkdir" "-p" "/var/backups/")
(spit "/var/backups/file" "I was here")
(tasks/shell "mkdir" "-p" "/var/restore"))
(defn restic-repo-init!
[]
(rc/init! cf/file-config)
(rc/init! (merge cf/db-role-config cf/dry-run))
(rc/init! (merge cf/db-config cf/dry-run)))
(defn restic-backup!
[]
(bak/backup-file! cf/file-config)
(bak/backup-db-roles! (merge cf/db-role-config cf/dry-run))
(bak/backup-db! (merge cf/db-config cf/dry-run)))
(defn list-snapshots!
[]
(rc/list-snapshots! cf/file-config)
(rc/list-snapshots! (merge cf/db-role-config cf/dry-run))
(rc/list-snapshots! (merge cf/db-config cf/dry-run)))
(defn restic-restore!
[]
(pg/drop-create-db! (merge cf/db-config cf/dry-run))
(rs/restore-db-roles! (merge cf/db-role-config cf/dry-run))
(rs/restore-db! (merge cf/db-config cf/dry-run))
(rs/restore-file! (merge cf/file-restore-config cf/dry-run)))
(prepare!)
(restic-repo-init!)
(restic-backup!)
(list-snapshots!)
(restic-restore!)

View file

@ -6,7 +6,7 @@ from ddadevops import *
name = 'c4k-cloud' name = 'c4k-cloud'
MODULE = 'not_set' MODULE = 'not_set'
PROJECT_ROOT_PATH = '../..' PROJECT_ROOT_PATH = '../..'
version = "8.0.1" version = "10.4.5-dev"
@init @init
def initialize(project): def initialize(project):

View file

@ -1,4 +1,4 @@
FROM nextcloud:28 FROM nextcloud:30
# REQUIRES docker >= 2.10.10 # REQUIRES docker >= 2.10.10
# https://docs.docker.com/engine/release-notes/20.10/#201010 # https://docs.docker.com/engine/release-notes/20.10/#201010

View file

@ -2,7 +2,7 @@
"name": "c4k-nextcloud", "name": "c4k-nextcloud",
"description": "Generate c4k yaml for a nextcloud deployment.", "description": "Generate c4k yaml for a nextcloud deployment.",
"author": "meissa GmbH", "author": "meissa GmbH",
"version": "8.0.1", "version": "10.4.5-SNAPSHOT",
"homepage": "https://gitlab.com/domaindrivenarchitecture/c4k-nextcloud#readme", "homepage": "https://gitlab.com/domaindrivenarchitecture/c4k-nextcloud#readme",
"repository": "https://www.npmjs.com/package/c4k-nextcloud", "repository": "https://www.npmjs.com/package/c4k-nextcloud",
"license": "APACHE2", "license": "APACHE2",
@ -23,11 +23,11 @@
"url": "https://gitlab.com/domaindrivenarchitecture/c4k-nextcloud/issues" "url": "https://gitlab.com/domaindrivenarchitecture/c4k-nextcloud/issues"
}, },
"dependencies": { "dependencies": {
"js-base64": "^3.6.1", "js-base64": "^3.7.7",
"js-yaml": "^4.0.0" "js-yaml": "^4.1.0"
}, },
"devDependencies": { "devDependencies": {
"shadow-cljs": "^2.11.18", "shadow-cljs": "^2.28.20",
"source-map-support": "^0.5.19" "source-map-support": "^0.5.21"
} }
} }

View file

@ -1,11 +1,11 @@
(defproject org.domaindrivenarchitecture/c4k-nextcloud "8.0.1" (defproject org.domaindrivenarchitecture/c4k-nextcloud "10.4.5-SNAPSHOT"
:description "nextcloud c4k-installation package" :description "nextcloud c4k-installation package"
:url "https://domaindrivenarchitecture.org" :url "https://domaindrivenarchitecture.org"
:license {:name "Apache License, Version 2.0" :license {:name "Apache License, Version 2.0"
:url "https://www.apache.org/licenses/LICENSE-2.0.html"} :url "https://www.apache.org/licenses/LICENSE-2.0.html"}
:dependencies [[org.clojure/clojure "1.11.2"] :dependencies [[org.clojure/clojure "1.12.0"]
[org.clojure/tools.reader "1.4.1"] [org.clojure/tools.reader "1.5.0"]
[org.domaindrivenarchitecture/c4k-common-clj "6.2.2"] [org.domaindrivenarchitecture/c4k-common-clj "9.0.1"]
[hickory "0.7.1" :exclusions [viebel/codox-klipse-theme]]] [hickory "0.7.1" :exclusions [viebel/codox-klipse-theme]]]
:target-path "target/%s/" :target-path "target/%s/"
:source-paths ["src/main/cljc" :source-paths ["src/main/cljc"
@ -23,9 +23,9 @@
:main dda.c4k-nextcloud.uberjar :main dda.c4k-nextcloud.uberjar
:uberjar-name "c4k-nextcloud-standalone.jar" :uberjar-name "c4k-nextcloud-standalone.jar"
:dependencies [[org.clojure/tools.cli "1.1.230"] :dependencies [[org.clojure/tools.cli "1.1.230"]
[ch.qos.logback/logback-classic "1.5.3" [ch.qos.logback/logback-classic "1.5.16"
:exclusions [com.sun.mail/javax.mail]] :exclusions [com.sun.mail/javax.mail]]
[org.slf4j/jcl-over-slf4j "2.0.12"] [org.slf4j/jcl-over-slf4j "2.0.16"]
[com.github.clj-easy/graal-build-time "1.0.5"]]}} [com.github.clj-easy/graal-build-time "1.0.5"]]}}
:release-tasks [["test"] :release-tasks [["test"]
["vcs" "assert-committed"] ["vcs" "assert-committed"]

View file

@ -4,7 +4,7 @@
"src/test/cljc" "src/test/cljc"
"src/test/cljs" "src/test/cljs"
"src/test/resources"] "src/test/resources"]
:dependencies [[org.domaindrivenarchitecture/c4k-common-cljs "6.1.3"] :dependencies [[org.domaindrivenarchitecture/c4k-common-cljs "9.0.1"]
[hickory "0.7.1"]] [hickory "0.7.1"]]
:builds {:frontend {:target :browser :builds {:frontend {:target :browser
:modules {:main {:init-fn dda.c4k-nextcloud.browser/init}} :modules {:main {:init-fn dda.c4k-nextcloud.browser/init}}

View file

@ -2,14 +2,14 @@
(:gen-class) (:gen-class)
(:require (:require
[dda.c4k-common.uberjar :as uberjar] [dda.c4k-common.uberjar :as uberjar]
[dda.c4k-nextcloud.nextcloud :as nextcloud]
[dda.c4k-nextcloud.core :as core])) [dda.c4k-nextcloud.core :as core]))
(defn -main [& cmd-args] (defn -main [& cmd-args]
(uberjar/main-common (uberjar/main-cm
"c4k-nextcloud" "c4k-nextcloud"
nextcloud/config? core/config?
nextcloud/auth? core/auth?
core/config-defaults core/config-defaults
core/k8s-objects core/config-objects
core/auth-objects
cmd-args)) cmd-args))

View file

@ -1,6 +1,8 @@
(ns dda.c4k-nextcloud.backup (ns dda.c4k-nextcloud.backup
(:require (:require
[clojure.spec.alpha :as s] [clojure.spec.alpha :as s]
#?(:clj [orchestra.core :refer [defn-spec]]
:cljs [orchestra.core :refer-macros [defn-spec]])
[dda.c4k-common.yaml :as yaml] [dda.c4k-common.yaml :as yaml]
[dda.c4k-common.base64 :as b64] [dda.c4k-common.base64 :as b64]
[dda.c4k-common.common :as cm] [dda.c4k-common.common :as cm]
@ -12,26 +14,35 @@
(s/def ::restic-password cp/bash-env-string?) (s/def ::restic-password cp/bash-env-string?)
(s/def ::restic-repository cp/bash-env-string?) (s/def ::restic-repository cp/bash-env-string?)
(s/def ::config (s/keys :req-un [::restic-repository]))
(s/def ::auth (s/keys :req-un [::restic-password ::aws-access-key-id ::aws-secret-access-key]))
#?(:cljs #?(:cljs
(defmethod yaml/load-resource :backup [resource-name] (defmethod yaml/load-resource :backup [resource-name]
(get (inline-resources "backup") resource-name))) (get (inline-resources "backup") resource-name)))
(defn generate-config [my-conf] (defn-spec generate-config map?
[my-conf ::config]
(let [{:keys [restic-repository]} my-conf] (let [{:keys [restic-repository]} my-conf]
(-> (->
(yaml/load-as-edn "backup/config.yaml") (yaml/load-as-edn "backup/config.yaml")
(cm/replace-key-value :restic-repository restic-repository)))) (cm/replace-key-value :restic-repository restic-repository))))
(defn generate-cron [] (defn-spec generate-cron map?
[]
(yaml/from-string (yaml/load-resource "backup/cron.yaml"))) (yaml/from-string (yaml/load-resource "backup/cron.yaml")))
(defn generate-backup-restore-deployment [my-conf] (defn-spec generate-backup-restore-deployment map?
[my-conf ::config]
(let [backup-restore-yaml (yaml/load-as-edn "backup/backup-restore-deployment.yaml")] (let [backup-restore-yaml (yaml/load-as-edn "backup/backup-restore-deployment.yaml")]
(if (and (contains? my-conf :local-integration-test) (= true (:local-integration-test my-conf))) (if (and (contains? my-conf :local-integration-test) (= true (:local-integration-test my-conf)))
(cm/replace-named-value backup-restore-yaml "CERTIFICATE_FILE" "/var/run/secrets/localstack-secrets/ca.crt") (cm/replace-named-value backup-restore-yaml "CERTIFICATE_FILE" "/var/run/secrets/localstack-secrets/ca.crt")
backup-restore-yaml))) backup-restore-yaml)))
(defn generate-secret [my-auth] (defn-spec generate-secret map?
[my-auth ::auth]
(let [{:keys [aws-access-key-id aws-secret-access-key restic-password]} my-auth] (let [{:keys [aws-access-key-id aws-secret-access-key restic-password]} my-auth]
(-> (->
(yaml/load-as-edn "backup/secret.yaml") (yaml/load-as-edn "backup/secret.yaml")

View file

@ -1,5 +1,6 @@
(ns dda.c4k-nextcloud.core (ns dda.c4k-nextcloud.core
(:require (:require
[clojure.spec.alpha :as s]
#?(:clj [orchestra.core :refer [defn-spec]] #?(:clj [orchestra.core :refer [defn-spec]]
:cljs [orchestra.core :refer-macros [defn-spec]]) :cljs [orchestra.core :refer-macros [defn-spec]])
[dda.c4k-common.common :as cm] [dda.c4k-common.common :as cm]
@ -8,36 +9,58 @@
[dda.c4k-common.postgres :as postgres] [dda.c4k-common.postgres :as postgres]
[dda.c4k-nextcloud.nextcloud :as nextcloud] [dda.c4k-nextcloud.nextcloud :as nextcloud]
[dda.c4k-nextcloud.backup :as backup] [dda.c4k-nextcloud.backup :as backup]
[dda.c4k-common.monitoring :as mon])) [dda.c4k-common.monitoring :as mon]
[dda.c4k-common.namespace :as ns]))
(def default-storage-class :local-path) (def config-defaults {:namespace "nextcloud"
:issuer "staging"
:pvc-storage-class-name "hcloud-volumes-encrypted"
:pv-storage-size-gb 200})
(def config-defaults {:issuer "staging"}) (def config? (s/merge ::nextcloud/config
::backup/config))
(defn-spec k8s-objects cp/map-or-seq? (def auth? (s/merge ::nextcloud/auth
[config nextcloud/config? ::backup/auth))
auth nextcloud/auth?]
(let [nextcloud-default-storage-config {:pvc-storage-class-name default-storage-class
:pv-storage-size-gb 200}] (defn-spec config-objects cp/map-or-seq?
[config config?]
(let [resolved-config (merge config-defaults config)]
(map yaml/to-string (map yaml/to-string
(filter (filter
#(not (nil? %)) #(not (nil? %))
(cm/concat-vec (cm/concat-vec
[(postgres/generate-config {:postgres-size :8gb :db-name "nextcloud"}) (ns/generate resolved-config)
(postgres/generate-secret auth) (postgres/generate-config (merge resolved-config {:postgres-image "postgres:16"
(postgres/generate-pvc {:pv-storage-size-gb 50 :postgres-size :8gb
:pvc-storage-class-name default-storage-class}) :db-name "cloud"
(postgres/generate-deployment) :pv-storage-size-gb 50}))
(postgres/generate-service) [(nextcloud/generate-pvc resolved-config)
(nextcloud/generate-secret auth) (nextcloud/generate-deployment resolved-config)
(nextcloud/generate-pvc (merge nextcloud-default-storage-config config))
(nextcloud/generate-deployment config)
(nextcloud/generate-service)] (nextcloud/generate-service)]
(nextcloud/generate-ingress-and-cert config) (nextcloud/generate-ingress-and-cert resolved-config)
(when (:contains? config :restic-repository) (when (:contains? resolved-config :restic-repository)
[(backup/generate-config config) [(backup/generate-config resolved-config)
(backup/generate-secret auth)
(backup/generate-cron) (backup/generate-cron)
(backup/generate-backup-restore-deployment config)]) (backup/generate-backup-restore-deployment resolved-config)])
(when (:contains? config :mon-cfg) (when (:contains? resolved-config :mon-cfg)
(mon/generate (:mon-cfg config) (:mon-auth auth)))))))) (mon/generate-config)))))))
(defn-spec auth-objects cp/map-or-seq?
[config config?
auth auth?]
(let [resolved-config (merge config-defaults config)]
(map yaml/to-string
(filter
#(not (nil? %))
(cm/concat-vec
(postgres/generate-auth (merge resolved-config {:postgres-size :8gb
:db-name "cloud"
:pv-storage-size-gb 50})
auth)
[(nextcloud/generate-secret auth)]
(when (:contains? resolved-config :restic-repository)
[(backup/generate-secret auth)])
(when (:contains? resolved-config :mon-cfg)
(mon/generate-auth (:mon-cfg resolved-config) (:mon-auth auth))))))))

View file

@ -14,23 +14,20 @@
(s/def ::fqdn cp/fqdn-string?) (s/def ::fqdn cp/fqdn-string?)
(s/def ::issuer cp/letsencrypt-issuer?) (s/def ::issuer cp/letsencrypt-issuer?)
(s/def ::restic-repository string?)
(s/def ::nextcloud-admin-user cp/bash-env-string?) (s/def ::nextcloud-admin-user cp/bash-env-string?)
(s/def ::nextcloud-admin-password cp/bash-env-string?) (s/def ::nextcloud-admin-password cp/bash-env-string?)
(s/def ::pvc-storage-class-name cp/pvc-storage-class-name?) (s/def ::pvc-storage-class-name cp/pvc-storage-class-name?)
(s/def ::pv-storage-size-gb pos?) (s/def ::pv-storage-size-gb pos?)
(def config? (s/keys :req-un [::fqdn] (s/def ::config (s/keys :req-un [::fqdn]
:opt-un [::issuer :opt-un [::issuer
::restic-repository
::pv-storage-size-gb ::pv-storage-size-gb
::pvc-storage-class-name ::pvc-storage-class-name
::mon/mon-cfg])) ::mon/mon-cfg]))
(def auth? (s/keys :req-un [::postgres/postgres-db-user ::postgres/postgres-db-password (s/def ::auth (s/keys :req-un [::postgres/postgres-db-user ::postgres/postgres-db-password
::nextcloud-admin-user ::nextcloud-admin-password ::nextcloud-admin-user ::nextcloud-admin-password
::aws-access-key-id ::aws-secret-access-key ::aws-access-key-id ::aws-secret-access-key]
::restic-password]
:opt-un [::mon/mon-auth])) :opt-un [::mon/mon-auth]))
#?(:cljs #?(:cljs
@ -38,13 +35,13 @@
(get (inline-resources "nextcloud") resource-name))) (get (inline-resources "nextcloud") resource-name)))
(defn-spec generate-deployment cp/map-or-seq? (defn-spec generate-deployment cp/map-or-seq?
[config config?] [config ::config]
(let [{:keys [fqdn]} config] (let [{:keys [fqdn]} config]
(-> (yaml/load-as-edn "nextcloud/deployment.yaml") (-> (yaml/load-as-edn "nextcloud/deployment.yaml")
(cm/replace-all-matching-values-by-new-value "fqdn" fqdn)))) (cm/replace-all-matching "fqdn" fqdn))))
(defn-spec generate-ingress-and-cert cp/map-or-seq? (defn-spec generate-ingress-and-cert cp/map-or-seq?
[config config?] [config ::config]
(ing/generate-ingress-and-cert (ing/generate-ingress-and-cert
(merge (merge
{:service-name "cloud-service" {:service-name "cloud-service"
@ -64,7 +61,7 @@
(yaml/load-as-edn "nextcloud/service.yaml")) (yaml/load-as-edn "nextcloud/service.yaml"))
(defn-spec generate-secret cp/map-or-seq? (defn-spec generate-secret cp/map-or-seq?
[auth auth?] [auth ::auth]
(let [{:keys [nextcloud-admin-user nextcloud-admin-password]} auth] (let [{:keys [nextcloud-admin-user nextcloud-admin-password]} auth]
(-> (->
(yaml/load-as-edn "nextcloud/secret.yaml") (yaml/load-as-edn "nextcloud/secret.yaml")

View file

@ -2,28 +2,27 @@
(:require (:require
[clojure.tools.reader.edn :as edn] [clojure.tools.reader.edn :as edn]
[dda.c4k-common.common :as cm] [dda.c4k-common.common :as cm]
[dda.c4k-common.monitoring :as mon]
[dda.c4k-nextcloud.core :as core] [dda.c4k-nextcloud.core :as core]
[dda.c4k-nextcloud.nextcloud :as nextcloud] [dda.c4k-common.browser :as br]))
[dda.c4k-common.browser :as br]
[dda.c4k-common.postgres :as pgc]))
(defn generate-content [] (defn generate-content []
(cm/concat-vec (cm/concat-vec
[(assoc [(assoc
(br/generate-needs-validation) :content (br/generate-needs-validation) :content
(cm/concat-vec (cm/concat-vec
(br/generate-group "domain" (br/generate-group "config"
(cm/concat-vec (br/generate-input-field "fqdn" "Your fqdn:" "nextcloud-neu.prod.meissa-gmbh.de") (br/generate-text-area "config" "Your config.edn:" "{:fqdn \"cloud.your.domain\"
(br/generate-input-field "issuer" "(Optional) Your issuer prod/staging:" "") :issuer \"staging\"
(br/generate-input-field "pv-storage-size-gb" "(Optional) Your nextcloud storage size in GB" "8") :pv-storage-size-gb \"400\"
(br/generate-input-field "pvc-storage-class-name" "(Optional) Your storage class type (manual / local-path):" "local-path") :pvc-storage-class-name \"local-path\"
(br/generate-input-field "postgres-data-volume-path" "(Optional) Your postgres-data-volume-path:" "/var/postgres") :postgres-data-volume-path \"/var/postgres\"
(br/generate-input-field "restic-repository" "(Optional) Your restic-repository:" "restic-repository") :restic-repository \"s3://yourbucket/your-repo\"
(br/generate-input-field "mon-cluster-name" "(Optional) monitoring cluster name:" "keycloak") :restic-password \"restic-password\"}
(br/generate-input-field "mon-cluster-stage" "(Optional) monitoring cluster stage:" "test") :mon-cfg {:cluster-name \"cloud\"
(br/generate-input-field "mon-cloud-url" "(Optional) grafana cloud url:" "https://prometheus-prod-01-eu-west-0.grafana.net/api/prom/push"))) :cluster-stage \"test\"
(br/generate-group "credentials" :cloud-url \"https://prometheus-prod-01-eu-west-0.grafana.net/api/prom/push\"}"
"5"))
(br/generate-group "auth"
(br/generate-text-area "auth" "Your auth.edn:" "{:postgres-db-user \"nextcloud\" (br/generate-text-area "auth" "Your auth.edn:" "{:postgres-db-user \"nextcloud\"
:postgres-db-password \"nextcloud-db-password\" :postgres-db-password \"nextcloud-db-password\"
:nextcloud-admin-password \"nextcloud-admin-password\" :nextcloud-admin-password \"nextcloud-admin-password\"
@ -45,41 +44,9 @@
:content :content
(generate-content)}) (generate-content)})
(defn config-from-document []
(let [pv-storage-size-gb (br/get-content-from-element "pv-storage-size-gb" :optional true)
pvc-storage-class-name (br/get-content-from-element "pvc-storage-class-name" :optional true)
postgres-data-volume-path (br/get-content-from-element "postgres-data-volume-path" :optional true)
restic-repository (br/get-content-from-element "restic-repository" :optional true)
issuer (br/get-content-from-element "issuer" :optional true)
mon-cluster-name (br/get-content-from-element "mon-cluster-name" :optional true)
mon-cluster-stage (br/get-content-from-element "mon-cluster-stage" :optional true)
mon-cloud-url (br/get-content-from-element "mon-cloud-url" :optional true)]
(merge
{:fqdn (br/get-content-from-element "fqdn")}
(when (and (some? pv-storage-size-gb) (some? pvc-storage-class-name))
{:pv-storage-size-gb pv-storage-size-gb :pvc-storage-class-name pvc-storage-class-name})
(when (some? postgres-data-volume-path)
{:postgres-data-volume-path postgres-data-volume-path})
(when (some? restic-repository)
{:restic-repository restic-repository})
(when (some? issuer)
{:issuer issuer})
(when (some? mon-cluster-name)
{:mon-cfg {:cluster-name mon-cluster-name
:cluster-stage (keyword mon-cluster-stage)
:grafana-cloud-url mon-cloud-url}}))))
(defn validate-all! [] (defn validate-all! []
(br/validate! "fqdn" ::nextcloud/fqdn) (br/validate! "config" core/config? :deserializer edn/read-string)
(br/validate! "pv-storage-size-gb" ::nextcloud/pv-storage-size-gb :optional true) (br/validate! "auth" core/auth? :deserializer edn/read-string)
(br/validate! "pvc-storage-class-name" ::nextcloud/pvc-storage-class-name :optional true)
(br/validate! "postgres-data-volume-path" ::pgc/postgres-data-volume-path :optional true)
(br/validate! "restic-repository" ::nextcloud/restic-repository :optional true)
(br/validate! "issuer" ::nextcloud/issuer :optional true)
(br/validate! "mon-cluster-name" ::mon/cluster-name :optional true)
(br/validate! "mon-cluster-stage" ::mon/cluster-stage :optional true)
(br/validate! "mon-cloud-url" ::mon/grafana-cloud-url :optional true)
(br/validate! "auth" nextcloud/auth? :deserializer edn/read-string)
(br/set-validated!)) (br/set-validated!))
(defn add-validate-listener [name] (defn add-validate-listener [name]
@ -92,19 +59,14 @@
(.getElementById "generate-button") (.getElementById "generate-button")
(.addEventListener "click" (.addEventListener "click"
#(do (validate-all!) #(do (validate-all!)
(-> (cm/generate-common (-> (cm/generate-cm
(config-from-document) (br/get-content-from-element "config" :deserializer edn/read-string)
(br/get-content-from-element "auth" :deserializer edn/read-string) (br/get-content-from-element "auth" :deserializer edn/read-string)
{} core/config-defaults
core/k8s-objects) core/config-objects
core/auth-objects
false
false)
(br/set-output!))))) (br/set-output!)))))
(add-validate-listener "fqdn") (add-validate-listener "config")
(add-validate-listener "pv-storage-size-gb")
(add-validate-listener "pvc-storage-class-name")
(add-validate-listener "postgres-data-volume-path")
(add-validate-listener "restic-repository")
(add-validate-listener "issuer")
(add-validate-listener "mon-cluster-name")
(add-validate-listener "mon-cluster-stage")
(add-validate-listener "mon-cloud-url")
(add-validate-listener "auth")) (add-validate-listener "auth"))

View file

@ -2,6 +2,7 @@ apiVersion: apps/v1
kind: Deployment kind: Deployment
metadata: metadata:
name: backup-restore name: backup-restore
namespace: nextcloud
spec: spec:
replicas: 0 replicas: 0
selector: selector:
@ -20,7 +21,7 @@ spec:
- name: backup-app - name: backup-app
image: domaindrivenarchitecture/c4k-cloud-backup image: domaindrivenarchitecture/c4k-cloud-backup
imagePullPolicy: IfNotPresent imagePullPolicy: IfNotPresent
command: ["/entrypoint-start-and-wait.sh"] command: ["wait.bb"]
env: env:
- name: POSTGRES_USER - name: POSTGRES_USER
valueFrom: valueFrom:
@ -67,6 +68,9 @@ spec:
- name: cloud-secret-volume - name: cloud-secret-volume
mountPath: /var/run/secrets/cloud-secrets mountPath: /var/run/secrets/cloud-secrets
readOnly: true readOnly: true
- name: rotation-credential-secret-volume
mountPath: /var/run/secrets/rotation-credential-secret
readOnly: true
volumes: volumes:
- name: cloud-data-volume - name: cloud-data-volume
persistentVolumeClaim: persistentVolumeClaim:
@ -77,3 +81,7 @@ spec:
- name: backup-secret-volume - name: backup-secret-volume
secret: secret:
secretName: backup-secret secretName: backup-secret
- name: rotation-credential-secret-volume
secret:
secretName: rotation-credential-secret
optional: true

View file

@ -2,6 +2,7 @@ apiVersion: v1
kind: ConfigMap kind: ConfigMap
metadata: metadata:
name: backup-config name: backup-config
namespace: nextcloud
labels: labels:
app.kubernetes.io/name: backup app.kubernetes.io/name: backup
app.kubernetes.io/part-of: cloud app.kubernetes.io/part-of: cloud

View file

@ -0,0 +1,8 @@
apiVersion: v1
kind: Secret
metadata:
name: rotation-credential-secret
namespace: nextcloud
type: Opaque
data:
rotation-credential: "dGVzdAo="

View file

@ -2,6 +2,7 @@ apiVersion: batch/v1
kind: CronJob kind: CronJob
metadata: metadata:
name: cloud-backup name: cloud-backup
namespace: nextcloud
labels: labels:
app.kubernetes.part-of: cloud app.kubernetes.part-of: cloud
spec: spec:
@ -16,7 +17,7 @@ spec:
- name: backup-app - name: backup-app
image: domaindrivenarchitecture/c4k-cloud-backup image: domaindrivenarchitecture/c4k-cloud-backup
imagePullPolicy: IfNotPresent imagePullPolicy: IfNotPresent
command: ["/entrypoint.sh"] command: ["backup.bb"]
env: env:
- name: POSTGRES_USER - name: POSTGRES_USER
valueFrom: valueFrom:

View file

@ -2,6 +2,7 @@ apiVersion: v1
kind: Secret kind: Secret
metadata: metadata:
name: backup-secret name: backup-secret
namespace: nextcloud
type: Opaque type: Opaque
data: data:
aws-access-key-id: "aws-access-key-id" aws-access-key-id: "aws-access-key-id"

View file

@ -2,6 +2,7 @@ apiVersion: apps/v1
kind: Deployment kind: Deployment
metadata: metadata:
name: cloud-deployment name: cloud-deployment
namespace: nextcloud
spec: spec:
selector: selector:
matchLabels: matchLabels:
@ -18,7 +19,7 @@ spec:
redeploy: v3 redeploy: v3
spec: spec:
containers: containers:
- image: domaindrivenarchitecture/c4k-cloud:8.0.0 - image: domaindrivenarchitecture/c4k-cloud:10.4.2
name: cloud-app name: cloud-app
imagePullPolicy: IfNotPresent imagePullPolicy: IfNotPresent
ports: ports:

View file

@ -2,6 +2,7 @@ apiVersion: v1
kind: PersistentVolumeClaim kind: PersistentVolumeClaim
metadata: metadata:
name: cloud-pvc name: cloud-pvc
namespace: nextcloud
labels: labels:
app.kubernetes.io/application: cloud app.kubernetes.io/application: cloud
spec: spec:

View file

@ -2,6 +2,7 @@ apiVersion: v1
kind: Secret kind: Secret
metadata: metadata:
name: cloud-secret name: cloud-secret
namespace: nextcloud
type: Opaque type: Opaque
data: data:
nextcloud-admin-user: "admin-user" nextcloud-admin-user: "admin-user"

View file

@ -2,6 +2,7 @@ apiVersion: v1
kind: Service kind: Service
metadata: metadata:
name: cloud-service name: cloud-service
namespace: nextcloud
labels: labels:
app.kubernetes.io/name: cloud-service app.kubernetes.io/name: cloud-service
app.kubernetes.io/application: cloud app.kubernetes.io/application: cloud

View file

@ -2,13 +2,17 @@
(:require (:require
#?(:clj [clojure.test :refer [deftest is are testing run-tests]] #?(:clj [clojure.test :refer [deftest is are testing run-tests]]
:cljs [cljs.test :refer-macros [deftest is are testing run-tests]]) :cljs [cljs.test :refer-macros [deftest is are testing run-tests]])
[clojure.spec.test.alpha :as st]
[dda.c4k-nextcloud.backup :as cut])) [dda.c4k-nextcloud.backup :as cut]))
(st/instrument `cut/generate-secret)
(st/instrument `cut/generate-config)
(st/instrument `cut/generate-cron)
(deftest should-generate-secret (deftest should-generate-secret
(is (= {:apiVersion "v1" (is (= {:apiVersion "v1"
:kind "Secret" :kind "Secret"
:metadata {:name "backup-secret"} :metadata {:name "backup-secret", :namespace "nextcloud"}
:type "Opaque" :type "Opaque"
:data :data
{:aws-access-key-id "YXdzLWlk", :aws-secret-access-key "YXdzLXNlY3JldA==", :restic-password "cmVzdGljLXB3"}} {:aws-access-key-id "YXdzLWlk", :aws-secret-access-key "YXdzLXNlY3JldA==", :restic-password "cmVzdGljLXB3"}}
@ -18,6 +22,7 @@
(is (= {:apiVersion "v1" (is (= {:apiVersion "v1"
:kind "ConfigMap" :kind "ConfigMap"
:metadata {:name "backup-config" :metadata {:name "backup-config"
:namespace "nextcloud"
:labels {:app.kubernetes.io/name "backup" :labels {:app.kubernetes.io/name "backup"
:app.kubernetes.io/part-of "cloud"}} :app.kubernetes.io/part-of "cloud"}}
:data :data
@ -27,7 +32,7 @@
(deftest should-generate-cron (deftest should-generate-cron
(is (= {:apiVersion "batch/v1" (is (= {:apiVersion "batch/v1"
:kind "CronJob" :kind "CronJob"
:metadata {:name "cloud-backup", :labels {:app.kubernetes.part-of "cloud"}} :metadata {:name "cloud-backup", :namespace "nextcloud", :labels {:app.kubernetes.part-of "cloud"}}
:spec :spec
{:schedule "10 23 * * *" {:schedule "10 23 * * *"
:successfulJobsHistoryLimit 1 :successfulJobsHistoryLimit 1
@ -40,7 +45,7 @@
[{:name "backup-app" [{:name "backup-app"
:image "domaindrivenarchitecture/c4k-cloud-backup" :image "domaindrivenarchitecture/c4k-cloud-backup"
:imagePullPolicy "IfNotPresent" :imagePullPolicy "IfNotPresent"
:command ["/entrypoint.sh"] :command ["backup.bb"]
:env :env
[{:valueFrom [{:valueFrom
{:secretKeyRef {:secretKeyRef

View file

@ -0,0 +1,19 @@
(ns dda.c4k-nextcloud.core-test
(:require
#?(:clj [clojure.test :refer [deftest is are testing run-tests]]
:cljs [cljs.test :refer-macros [deftest is are testing run-tests]])
[clojure.spec.alpha :as s]
[dda.c4k-common.yaml :as yaml]
[dda.c4k-nextcloud.core :as cut]
#?(:cljs [dda.c4k-common.macros :refer-macros [inline-resources]])))
#?(:cljs
(defmethod yaml/load-resource :nextcloud-test [resource-name]
(get (inline-resources "nextcloud-test") resource-name)))
(deftest validate-valid-resources
(is (s/valid? cut/config? (yaml/load-as-edn "nextcloud-test/valid-config.yaml")))
(is (s/valid? cut/auth? (yaml/load-as-edn "nextcloud-test/valid-auth.yaml")))
(is (not (s/valid? cut/config? (yaml/load-as-edn "nextcloud-test/invalid-config.yaml"))))
(is (not (s/valid? cut/auth? (yaml/load-as-edn "nextcloud-test/invalid-auth.yaml")))))

View file

@ -2,28 +2,24 @@
(:require (:require
#?(:clj [clojure.test :refer [deftest is are testing run-tests]] #?(:clj [clojure.test :refer [deftest is are testing run-tests]]
:cljs [cljs.test :refer-macros [deftest is are testing run-tests]]) :cljs [cljs.test :refer-macros [deftest is are testing run-tests]])
[clojure.spec.alpha :as s]
[clojure.spec.test.alpha :as st] [clojure.spec.test.alpha :as st]
[dda.c4k-common.yaml :as yaml] [dda.c4k-common.yaml :as yaml]
[dda.c4k-nextcloud.nextcloud :as cut] [dda.c4k-nextcloud.nextcloud :as cut]
#?(:cljs [dda.c4k-common.macros :refer-macros [inline-resources]]))) #?(:cljs [dda.c4k-common.macros :refer-macros [inline-resources]])))
(st/instrument) (st/instrument `cut/generate-secret)
(st/instrument `cut/generate-ingress-and-cert)
(st/instrument `cut/generate-pvc)
(st/instrument `cut/generate-deployment)
#?(:cljs #?(:cljs
(defmethod yaml/load-resource :nextcloud-test [resource-name] (defmethod yaml/load-resource :nextcloud-test [resource-name]
(get (inline-resources "nextcloud-test") resource-name))) (get (inline-resources "nextcloud-test") resource-name)))
(deftest validate-valid-resources
(is (s/valid? cut/config? (yaml/load-as-edn "nextcloud-test/valid-config.yaml")))
(is (s/valid? cut/auth? (yaml/load-as-edn "nextcloud-test/valid-auth.yaml")))
(is (not (s/valid? cut/config? (yaml/load-as-edn "nextcloud-test/invalid-config.yaml"))))
(is (not (s/valid? cut/auth? (yaml/load-as-edn "nextcloud-test/invalid-auth.yaml")))))
(deftest should-generate-secret (deftest should-generate-secret
(is (= {:apiVersion "v1" (is (= {:apiVersion "v1"
:kind "Secret" :kind "Secret"
:metadata {:name "cloud-secret"} :metadata {:name "cloud-secret", :namespace "nextcloud"}
:type "Opaque" :type "Opaque"
:data :data
{:nextcloud-admin-user "Y2xvdWRhZG1pbg==" {:nextcloud-admin-user "Y2xvdWRhZG1pbg=="
@ -77,6 +73,7 @@
(is (= {:apiVersion "v1" (is (= {:apiVersion "v1"
:kind "PersistentVolumeClaim" :kind "PersistentVolumeClaim"
:metadata {:name "cloud-pvc" :metadata {:name "cloud-pvc"
:namespace "nextcloud"
:labels {:app.kubernetes.io/application "cloud"}} :labels {:app.kubernetes.io/application "cloud"}}
:spec {:storageClassName "local-path" :spec {:storageClassName "local-path"
:accessModes ["ReadWriteOnce"] :accessModes ["ReadWriteOnce"]
@ -86,7 +83,7 @@
(deftest should-generate-deployment (deftest should-generate-deployment
(is (= {:apiVersion "apps/v1" (is (= {:apiVersion "apps/v1"
:kind "Deployment" :kind "Deployment"
:metadata {:name "cloud-deployment"} :metadata {:name "cloud-deployment", :namespace "nextcloud"}
:spec :spec
{:selector {:matchLabels #:app.kubernetes.io{:name "cloud-pod", :application "cloud"}} {:selector {:matchLabels #:app.kubernetes.io{:name "cloud-pod", :application "cloud"}}
:strategy {:type "Recreate"} :strategy {:type "Recreate"}
@ -94,7 +91,7 @@
{:metadata {:labels {:app "cloud-app", :app.kubernetes.io/name "cloud-pod", :app.kubernetes.io/application "cloud", :redeploy "v3"}} {:metadata {:labels {:app "cloud-app", :app.kubernetes.io/name "cloud-pod", :app.kubernetes.io/application "cloud", :redeploy "v3"}}
:spec :spec
{:containers {:containers
[{:image "domaindrivenarchitecture/c4k-cloud:8.0.0" [{:image "domaindrivenarchitecture/c4k-cloud:10.4.2"
:name "cloud-app" :name "cloud-app"
:imagePullPolicy "IfNotPresent" :imagePullPolicy "IfNotPresent"
:ports [{:containerPort 80}] :ports [{:containerPort 80}]

View file

@ -1,84 +0,0 @@
# Usage
`setup-local-s3.sh [BUCKET_NAME]`:
- [BUCKET_NAME] is optional, "mybucket" will be used if not specified
- sets up a k3s instance
- installs a localstack pod
- creates http and https routing to localstack via localhost
- saves the self-signed certificate as ca.crt
- uses the certificate to initialize a restic repo at `https://k3stesthost/BUCKET_NAME`
Note: In case of not being able to connect to "k3stesthost/health", you might need to ensure that the ingress' ip matches with the required host names: k3stesthost and cloudhost. With `sudo k3s kubectl get ingress` you can view the ingress' ip (e.g. 10.0.2.15), then add a line to file "/etc/hosts" e.g. `10.0.2.15 k3stesthost cloudhost`
`start-k3s.sh`:
- creates and starts a k3s instance
`k3s-uninstall.sh`:
- deletes everything k3s related
## Other useful commands
- `sudo k3s kubectl get pods`
- `curl k3stesthost/health`
expected: `{"services": {"s3": "running"}, "features": {"persistence": "disabled", "initScripts": "initialized"}}`
#### Requires AWS-CLI
- create bucket `aws --endpoint-url=http://k3stesthost s3 mb s3://mybucket`
- list buckets `aws --endpoint-url=http://k3stesthost s3 ls`
- upload something `aws --endpoint-url=http://k3stesthost s3 cp test.txt s3://mybucket`
- check files `aws --endpoint-url=http://k3stesthost s3 ls s3://mybucket`
## Run docker locally
```
docker pull docker:19.03.12-dind
docker run -d --privileged --name integration-test docker:19.03.12-dind
docker exec integration-test sh -c "apk add bash"
```
Set up docker container integration-test:
```
docker cp ../../../../../c4k-nextcloud/ integration-test:/
docker exec -it integration-test sh
cd /c4k-nextcloud/src/test/resources/local-integration-test
./setup-docker.sh
```
## Deploy nextcloud
### Requirements
* leiningen (install with: `sudo apt install leiningen` )
* In the project's root execute: `lein uberjar`
* Change file "valid-config.edn" according to your settings (e.g. `:fqdn "cloudhost"` and `:restic-repository "s3://k3stesthost:mybucket"`).
### Deploy to k3s
* Create and deploy the k8s yaml:
`java -jar target/uberjar/c4k-nextcloud-standalone.jar valid-config.edn valid-auth.edn | sudo k3s kubectl apply -f -`
Some of the steps may take some min to be effective, but eventually nextcloud should be available at: https://cloudhost
### Deploy to k3d
k3d is a k3s system which is running inside of a container. To install k3d run `curl -s https://raw.githubusercontent.com/rancher/k3d/main/install.sh | bash` or have a look at https://k3d.io/v5.0.3/ .
* Start a k3d cluster to deploy s3, nextcloud and test backup and restore on it: `./setup-local-s3-on-k3d.sh`
Some steps may take a couple of minutes to be effective, but eventually nextcloud should be available at: https://cloudhost
#### Remove k3d cluster
`k3d cluster delete nextcloud`
## Test in local gitlab runner
See https://stackoverflow.com/questions/32933174/use-gitlab-ci-to-run-tests-locally
This needs to be done in the project root
`docker run -d --name gitlab-runner --restart always -v $PWD:$PWD -v /var/run/docker.sock:/var/run/docker.sock gitlab/gitlab-runner:latest`
`docker exec -it -w $PWD gitlab-runner gitlab-runner exec docker nextcloud-integrationtest --docker-privileged --docker-volumes '/var/run/docker.sock:/var/run/docker.sock'`

View file

@ -1,20 +0,0 @@
apiVersion: cert-manager.io/v1
kind: Certificate
metadata:
name: localstack-cert
namespace: default
spec:
secretName: localstack-secret
commonName: k3stesthost
dnsNames:
- k3stesthost
issuerRef:
name: selfsigning-issuer
kind: ClusterIssuer
---
apiVersion: cert-manager.io/v1
kind: ClusterIssuer
metadata:
name: selfsigning-issuer
spec:
selfSigned: {}

View file

@ -1,44 +0,0 @@
@startuml
autonumber
skinparam sequenceBox {
borderColor White
}
participant gitlab_runner
box "outer container" #LightBlue
participant .gitlab_ci
participant PreparingCommands
participant test_script
end box
box "k3s" #CornSilk
participant k3s_api_server
participant backup_pod
end box
gitlab_runner -> k3s_api_server: run k3s as container
gitlab_runner -> .gitlab_ci : run
.gitlab_ci -> PreparingCommands : Install packages (curl bash ...)
.gitlab_ci -> PreparingCommands : get k3s_api_server config for k3s_api_server
.gitlab_ci -> test_script : run
test_script -> k3s_api_server: apply cert-manager
test_script -> k3s_api_server: apply localstack
test_script -> k3s_api_server: enable tls / create certificates
test_script -> k3s_api_server: apply cloud
test_script -> k3s_api_server: create backup_pod (by scale to 1)
test_script -> backup_pod: backup
test_script -> backup_pod: restore
@enduml

View file

@ -1,17 +0,0 @@
# Set the default kube context if present
DEFAULT_KUBE_CONTEXTS="$HOME/.kube/config"
if test -f "${DEFAULT_KUBE_CONTEXTS}"
then
export KUBECONFIG="$DEFAULT_KUBE_CONTEXTS"
fi
# Additional contexts should be in ~/.kube/custom-contexts/
CUSTOM_KUBE_CONTEXTS="$HOME/.kube/custom-contexts"
mkdir -p "${CUSTOM_KUBE_CONTEXTS}"
OIFS="$IFS"
IFS=$'\n'
for contextFile in `find "${CUSTOM_KUBE_CONTEXTS}" -type f -name "*.yml"`
do
export KUBECONFIG="$contextFile:$KUBECONFIG"
done
IFS="$OIFS"

View file

@ -1,65 +0,0 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: localstack
spec:
selector:
matchLabels:
app: localstack
strategy:
type: Recreate
template:
metadata:
labels:
app: localstack
spec:
containers:
- image: localstack/localstack
name: localstack-app
imagePullPolicy: IfNotPresent
env:
- name: SERVICES
value: s3
---
# service
apiVersion: v1
kind: Service
metadata:
name: localstack-service
spec:
selector:
app: localstack
ports:
- port: 4566
---
apiVersion: v1
kind: Secret
metadata:
name: localstack-secret
type: Opaque
---
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
name: ingress-localstack
annotations:
cert-manager.io/cluster-issuer: selfsigning-issuer
kubernetes.io/ingress.class: traefik
traefik.ingress.kubernetes.io/redirect-entry-point: https
namespace: default
spec:
tls:
- hosts:
- k3stesthost
secretName: localstack-secret
rules:
- host: k3stesthost
http:
paths:
- path: /
pathType: Prefix
backend:
service:
name: localstack-service
port:
number: 4566

View file

@ -1,48 +0,0 @@
#!/bin/bash
set -x
docker volume create k3s-server
name='inttst'
[[ $(docker ps -f "name=$name" --format '{{.Names}}') == $name ]] || docker run --name $name -d --privileged --tmpfs /run --tmpfs /var/run --restart always -e K3S_TOKEN=12345678901234 -e K3S_KUBECONFIG_OUTPUT=./kubeconfig.yaml -e K3S_KUBECONFIG_MODE=666 -v k3s-server:/var/lib/rancher/k3s:z -v $(pwd):/output:z -p 6443:6443 -p 80:80 -p 443:443 rancher/k3s server --cluster-init --tls-san k3stesthost --tls-san cloudhost
docker ps
export timeout=30; while ! docker exec $name sh -c "test -f /var/lib/rancher/k3s/server/kubeconfig.yaml"; do if [ "$timeout" == 0 ]; then echo "ERROR: Timeout while waiting for file."; break; fi; sleep 1; ((timeout--)); done
mkdir -p $HOME/.kube/
docker cp $name:/var/lib/rancher/k3s/server/kubeconfig.yaml $HOME/.kube/config
if [ "$timeout" == 0 ]
then
echo -------------------------------------------------------
find / -name "kubeconfig.yaml";
echo -------------------------------------------------------
docker ps -a
echo -------------------------------------------------------
exit 1
fi
echo "127.0.0.1 kubernetes" >> /etc/hosts
apk add wget curl bash sudo openjdk8
wget -P /etc/apk/keys/ https://alpine-pkgs.sgerrand.com/sgerrand.rsa.pub
apk add --no-cache --repository=https://apkproxy.herokuapp.com/sgerrand/alpine-pkg-leiningen leiningen
curl -LO https://storage.googleapis.com/kubernetes-release/release/v1.22.0/bin/linux/amd64/kubectl
chmod +x ./kubectl
mv ./kubectl /usr/local/bin/kubectl
sleep 20 #allow some time to startup k3s
docker ps -a
swapoff -a # can this be removed ?
export KUBECONFIG=$HOME/.kube/config
pwd
cd ./c4k-nextcloud/src/test/resources/local-integration-test && ./setup-local-s3-on-k3d.sh

View file

@ -1,60 +0,0 @@
#!/bin/bash
set -x
function main()
{
# enable tls for k3s with cert-manager
kubectl apply -f https://github.com/jetstack/cert-manager/releases/download/v1.5.4/cert-manager.yaml
kubectl apply -f localstack.yaml
until kubectl apply -f certificate.yaml
do
echo "[INFO] Waiting for certificate ..."
sleep 30
done
# wait for ingress to be ready
bash -c 'external_ip=""; while [ -z $external_ip ]; do echo "[INFO] Waiting for end point..."; external_ip=$(kubectl get ingress -o jsonpath="{$.items[*].status.loadBalancer.ingress[*].ip}"); [ -z "$external_ip" ] && sleep 10; done; echo "End point ready - $external_ip";'
export INGRESS_IP=$(kubectl get ingress ingress-localstack -o=jsonpath="{.status.loadBalancer.ingress[0].ip}")
cd ../../../../ # c4k-nextcloud project root
lein uberjar
java -jar target/uberjar/c4k-nextcloud-standalone.jar config-local.edn auth-local.edn | kubectl apply -f -
CLOUD_POD=$(kubectl get pod -l app=cloud-app -o name)
kubectl wait $CLOUD_POD --for=condition=Ready --timeout=240s
# wait for nextcloud config file available
timeout 180 bash -c "kubectl exec -t $POD -- bash -c \"until [ -f /var/www/html/config/config.php ]; do sleep 10; done\""
# ensure an instance of pod backup-restore
kubectl scale deployment backup-restore --replicas 1
# wait for localstack health endpoint
echo "$INGRESS_IP k3stesthost cloudhost" >> /etc/hosts
until curl --fail --silent k3stesthost/health | grep -oe '"s3": "available"' -oe '"s3": "running"'
do
curl --fail k3stesthost/health
echo "[INFO] Waiting for s3 running"
sleep 10
done
BACKUP_POD=$(kubectl get pod -l app=backup-restore -o name)
kubectl wait $BACKUP_POD --for=condition=Ready --timeout=240s
kubectl exec -t $BACKUP_POD -- bash -c "echo \"$INGRESS_IP k3stesthost cloudhost\" >> /etc/hosts"
kubectl exec -t $BACKUP_POD -- /usr/local/bin/init.sh
echo ================= BACKUP =================
kubectl exec -t $BACKUP_POD -- /usr/local/bin/backup.sh
sleep 10 # avoid race conditions
echo ================= RESTORE =================
kubectl exec -t $BACKUP_POD -- /usr/local/bin/restore.sh
}
main "$@"

View file

@ -1,34 +0,0 @@
function main()
{
local bucket_name="${1:-mybucket}"; shift
./start-k3s.sh
sudo k3s kubectl apply -f https://github.com/jetstack/cert-manager/releases/download/v1.5.4/cert-manager.yaml
sudo k3s kubectl apply -f localstack.yaml
until sudo k3s kubectl apply -f certificate.yaml
do
echo "*** Waiting for certificate ... ***"
sleep 10
done
echo
echo
echo "[INFO] Waiting for localstack health endpoint"
until curl --connect-timeout 3 -s -f -o /dev/null "k3stesthost/health"
do
sleep 5
done
echo
sudo k3s kubectl get secret localstack-secret -o jsonpath="{.data.ca\.crt}" | base64 --decode > ca.crt
#aws --endpoint-url=http://localhost s3 mb s3://$bucket_name
export RESTIC_PASSWORD="test-password"
restic init --cacert ca.crt -r s3://k3stesthost/$bucket_name
}
main $@

View file

@ -1,9 +0,0 @@
function main()
{
./start-k3s.sh
sudo k3s kubectl apply -f https://github.com/jetstack/cert-manager/releases/download/v1.5.4/cert-manager.yaml
}
main

View file

@ -1 +0,0 @@
KUBECONFIG=~/.kube/custom-contexts/k3d-config.yml k3d cluster create nextcloud --k3s-arg '--tls-san cloudhost@loadbalancer' --port 80:80@loadbalancer --port 443:443@loadbalancer --api-port 6443 --kubeconfig-update-default

View file

@ -1 +0,0 @@
curl -sfL https://get.k3s.io | K3S_NODE_NAME=k3stesthost INSTALL_K3S_EXEC='--tls-san cloudhost' sh -