From c5e5de16e8796728a0791dc2eddb9ddb7b04c6ee Mon Sep 17 00:00:00 2001 From: leo Date: Fri, 6 Aug 2021 17:36:59 +0200 Subject: [PATCH 01/12] added base structure of c4k-nextcloud --- .gitignore | 28 +++ .gitlab-ci.yml | 141 ++++++++++++ LICENSE | 201 +++++++++++++++++ README.md | 41 +++- doc/Development.md | 110 ++++++++++ doc/Releasing.md | 14 ++ doc/SUBCOMPONENT_LICENSE | 206 ++++++++++++++++++ infrastructure/docker-backup/build.py | 51 +++++ infrastructure/docker-backup/image/Dockerfile | 5 + .../docker-backup/image/resources/backup.sh | 22 ++ .../resources/entrypoint-start-and-wait.sh | 17 ++ .../image/resources/entrypoint.sh | 15 ++ .../docker-backup/image/resources/init.sh | 15 ++ .../docker-backup/image/resources/install.sh | 11 + .../image/resources/restic-snapshots.sh | 17 ++ .../docker-backup/image/resources/restore.sh | 36 +++ infrastructure/docker-backup/test/Dockerfile | 8 + .../docker-backup/test/serverspec.edn | 6 + infrastructure/docker-cloud/build.py | 51 +++++ infrastructure/docker-cloud/image/Dockerfile | 10 + .../docker-cloud/image/resources/dbconfig.xml | 25 +++ .../image/resources/entrypoint.sh | 44 ++++ .../docker-cloud/image/resources/install.sh | 30 +++ .../image/resources/logging.properties | 69 ++++++ .../docker-cloud/image/resources/server.xml | 114 ++++++++++ .../docker-cloud/image/resources/setenv.sh | 121 ++++++++++ .../docker-cloud/image/resources/user.sh | 5 + infrastructure/docker-cloud/test/Dockerfile | 8 + .../docker-cloud/test/serverspec.edn | 3 + package.json | 33 +++ project.clj | 41 ++++ public/index.html | 69 ++++++ shadow-cljs.edn | 15 ++ src/main/clj/dda/c4k_cloud/uberjar.clj | 56 +++++ src/main/cljc/dda/c4k_cloud/backup.cljc | 37 ++++ src/main/cljc/dda/c4k_cloud/cloud.cljc | 56 +++++ src/main/cljc/dda/c4k_cloud/core.cljc | 51 +++++ src/main/cljs/dda/c4k_cloud/browser.cljs | 61 ++++++ src/main/resources/backup/backup-restore.yaml | 62 ++++++ src/main/resources/backup/config.yaml | 9 + src/main/resources/backup/cron.yaml | 68 ++++++ src/main/resources/backup/secret.yaml | 9 + src/main/resources/cloud/certificate.yaml | 13 ++ src/main/resources/cloud/deployment.yaml | 40 ++++ src/main/resources/cloud/ingress.yaml | 26 +++ .../resources/cloud/persistent-volume.yaml | 14 ++ src/main/resources/cloud/pvc.yaml | 13 ++ src/main/resources/cloud/service.yaml | 9 + src/main/resources/logback.xml | 50 +++++ src/test/cljc/dda/c4k_cloud/backup_test.cljc | 93 ++++++++ src/test/cljc/dda/c4k_cloud/cloud_test.cljc | 80 +++++++ src/test/cljc/dda/c4k_cloud/core_test.cljc | 35 +++ valid-auth.edn | 5 + valid-config.edn | 4 + 54 files changed, 2372 insertions(+), 1 deletion(-) create mode 100644 .gitignore create mode 100644 .gitlab-ci.yml create mode 100644 LICENSE create mode 100644 doc/Development.md create mode 100644 doc/Releasing.md create mode 100644 doc/SUBCOMPONENT_LICENSE create mode 100644 infrastructure/docker-backup/build.py create mode 100644 infrastructure/docker-backup/image/Dockerfile create mode 100755 infrastructure/docker-backup/image/resources/backup.sh create mode 100644 infrastructure/docker-backup/image/resources/entrypoint-start-and-wait.sh create mode 100755 infrastructure/docker-backup/image/resources/entrypoint.sh create mode 100755 infrastructure/docker-backup/image/resources/init.sh create mode 100755 infrastructure/docker-backup/image/resources/install.sh create mode 100755 infrastructure/docker-backup/image/resources/restic-snapshots.sh create mode 100755 infrastructure/docker-backup/image/resources/restore.sh create mode 100644 infrastructure/docker-backup/test/Dockerfile create mode 100644 infrastructure/docker-backup/test/serverspec.edn create mode 100644 infrastructure/docker-cloud/build.py create mode 100644 infrastructure/docker-cloud/image/Dockerfile create mode 100644 infrastructure/docker-cloud/image/resources/dbconfig.xml create mode 100644 infrastructure/docker-cloud/image/resources/entrypoint.sh create mode 100755 infrastructure/docker-cloud/image/resources/install.sh create mode 100644 infrastructure/docker-cloud/image/resources/logging.properties create mode 100644 infrastructure/docker-cloud/image/resources/server.xml create mode 100644 infrastructure/docker-cloud/image/resources/setenv.sh create mode 100644 infrastructure/docker-cloud/image/resources/user.sh create mode 100644 infrastructure/docker-cloud/test/Dockerfile create mode 100644 infrastructure/docker-cloud/test/serverspec.edn create mode 100644 package.json create mode 100644 project.clj create mode 100644 public/index.html create mode 100644 shadow-cljs.edn create mode 100644 src/main/clj/dda/c4k_cloud/uberjar.clj create mode 100644 src/main/cljc/dda/c4k_cloud/backup.cljc create mode 100644 src/main/cljc/dda/c4k_cloud/cloud.cljc create mode 100644 src/main/cljc/dda/c4k_cloud/core.cljc create mode 100644 src/main/cljs/dda/c4k_cloud/browser.cljs create mode 100644 src/main/resources/backup/backup-restore.yaml create mode 100644 src/main/resources/backup/config.yaml create mode 100644 src/main/resources/backup/cron.yaml create mode 100644 src/main/resources/backup/secret.yaml create mode 100644 src/main/resources/cloud/certificate.yaml create mode 100644 src/main/resources/cloud/deployment.yaml create mode 100644 src/main/resources/cloud/ingress.yaml create mode 100644 src/main/resources/cloud/persistent-volume.yaml create mode 100644 src/main/resources/cloud/pvc.yaml create mode 100644 src/main/resources/cloud/service.yaml create mode 100644 src/main/resources/logback.xml create mode 100644 src/test/cljc/dda/c4k_cloud/backup_test.cljc create mode 100644 src/test/cljc/dda/c4k_cloud/cloud_test.cljc create mode 100644 src/test/cljc/dda/c4k_cloud/core_test.cljc create mode 100644 valid-auth.edn create mode 100644 valid-config.edn diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..667b168 --- /dev/null +++ b/.gitignore @@ -0,0 +1,28 @@ +# pybuilder +.pybuilder/ +__pycache__/ + +# lein +target/ +.lein-repl-history +.lein-failures +pom.* + +# cljs +.shadow-cljs +.nrepl-* +package-lock.json +node_modules/ +public/js/ + +# ide +.calva + +*.iml +.idea/ + +#valid-auth.edn +#valid-config.edn +my-auth.edn +auth.edn +config.edn diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml new file mode 100644 index 0000000..81a9d2d --- /dev/null +++ b/.gitlab-ci.yml @@ -0,0 +1,141 @@ +stages: + - build_and_test + - package + - security + - upload + - image + +services: + - docker:19.03.12-dind + +.cljs-job: &cljs + image: domaindrivenarchitecture/shadow-cljs + cache: + key: ${CI_COMMIT_REF_SLUG} + paths: + - node_modules/ + - .shadow-cljs/ + - .m2 + before_script: + - echo "//registry.npmjs.org/:_authToken=$NPM_TOKEN" > ~/.npmrc + - npm install + +.clj-uploadjob: &clj + image: domaindrivenarchitecture/lein + cache: + key: ${CI_COMMIT_REF_SLUG} + paths: + - .m2 + before_script: + - mkdir -p /root/.lein + - echo "{:auth {:repository-auth {#\"clojars\" {:username \"${CLOJARS_USER}\" :password \"${CLOJARS_TOKEN_DOMAINDRIVENARCHITECTURE}\" }}}}" > ~/.lein/profiles.clj + +test-cljs: + <<: *cljs + stage: build_and_test + script: + - shadow-cljs compile test + +test-clj: + <<: *clj + stage: build_and_test + script: + - lein test + +test-schema: + <<: *clj + stage: build_and_test + script: + - lein uberjar + - java -jar target/uberjar/c4k-cloud-standalone.jar valid-config.edn valid-auth.edn | kubeconform --kubernetes-version 1.19.0 --strict --skip Certificate - + artifacts: + paths: + - target/uberjar + +.report-frontend: + <<: *cljs + stage: package + script: + - mkdir -p target/frontend-build + - shadow-cljs run shadow.cljs.build-report frontend target/frontend-build/build-report.html + artifacts: + paths: + - target/frontend-build/build-report.html + +.package-frontend: + <<: *cljs + stage: package + script: + - mkdir -p target/frontend-build + - shadow-cljs release frontend + - cp public/js/main.js target/frontend-build/c4k-cloud.js + - sha256sum target/frontend-build/c4k-cloud.js > target/frontend-build/c4k-cloud.js.sha256 + - sha512sum target/frontend-build/c4k-cloud.js > target/frontend-build/c4k-cloud.js.sha512 + artifacts: + paths: + - target/frontend-build + +package-uberjar: + <<: *clj + stage: package + script: + - sha256sum target/uberjar/c4k-cloud-standalone.jar > target/uberjar/c4k-cloud-standalone.jar.sha256 + - sha512sum target/uberjar/c4k-cloud-standalone.jar > target/uberjar/c4k-cloud-standalone.jar.sha512 + artifacts: + paths: + - target/uberjar + +sast: + variables: + SAST_EXCLUDED_ANALYZERS: + bandit, brakeman, flawfinder, gosec, kubesec, phpcs-security-audit, + pmd-apex, security-code-scan, sobelow, spotbugs + stage: security + before_script: + - mkdir -p builds && cp -r target/ builds/ +include: + - template: Security/SAST.gitlab-ci.yml + +upload-clj-prerelease: + <<: *clj + stage: upload + rules: + - if: '$CI_COMMIT_BRANCH == "master" && $CI_COMMIT_TAG == null' + script: + - lein deploy clojars + +release: + image: registry.gitlab.com/gitlab-org/release-cli:latest + stage: upload + rules: + - if: '$CI_COMMIT_TAG != null' + artifacts: + paths: + - target/uberjar + - target/frontend-build + script: + - apk --no-cache add curl + - | + release-cli create --name "Release $CI_COMMIT_TAG" --tag-name $CI_COMMIT_TAG \ + --assets-link "{\"name\":\"c4k-cloud-standalone.jar\",\"url\":\"https://gitlab.com/domaindrivenarchitecture/c4k-nextcloud/-/jobs/${CI_JOB_ID}/artifacts/file/target/uberjar/c4k-cloud-standalone.jar\"}" \ + --assets-link "{\"name\":\"c4k-cloud-standalone.jar.sha256\",\"url\":\"https://gitlab.com/domaindrivenarchitecture/c4k-nextcloud/-/jobs/${CI_JOB_ID}/artifacts/file/target/uberjar/c4k-cloud-standalone.jar.sha256\"}" \ + --assets-link "{\"name\":\"c4k-cloud-standalone.jar.sha512\",\"url\":\"https://gitlab.com/domaindrivenarchitecture/c4k-nextcloud/-/jobs/${CI_JOB_ID}/artifacts/file/target/uberjar/c4k-cloud-standalone.jar.sha512\"}" \ + --assets-link "{\"name\":\"c4k-cloud.js\",\"url\":\"https://gitlab.com/domaindrivenarchitecture/c4k-nextcloud/-/jobs/${CI_JOB_ID}/artifacts/file/target/frontend-build/c4k-cloud.js\"}" \ + --assets-link "{\"name\":\"c4k-cloud.js.sha256\",\"url\":\"https://gitlab.com/domaindrivenarchitecture/c4k-nextcloud/-/jobs/${CI_JOB_ID}/artifacts/file/target/frontend-build/c4k-cloud.js.sha256\"}" \ + --assets-link "{\"name\":\"c4k-cloud.js.sha512\",\"url\":\"https://gitlab.com/domaindrivenarchitecture/c4k-nextcloud/-/jobs/${CI_JOB_ID}/artifacts/file/target/frontend-build/c4k-cloud.js.sha512\"}" \ + +cloud-image-test-publish: + image: domaindrivenarchitecture/devops-build:latest + stage: image + rules: + - if: '$CI_COMMIT_TAG != null' + script: + - cd infrastructure/docker-cloud && pyb image test publish + +backup-image-test-publish: + image: domaindrivenarchitecture/devops-build:latest + stage: image + rules: + - if: '$CI_COMMIT_TAG != null' + script: + - cd infrastructure/docker-backup && pyb image test publish \ No newline at end of file diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..261eeb9 --- /dev/null +++ b/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/README.md b/README.md index 1b2df0c..6b16f0d 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,41 @@ -# c4k-nextcloud +# convention 4 kubernetes: c4k-nextcloud +[![Clojars Project](https://img.shields.io/clojars/v/org.domaindrivenarchitecture/c4k-nextcloud.svg)](https://clojars.org/org.domaindrivenarchitecture/c4k-nextcloud) [![pipeline status](https://gitlab.com/domaindrivenarchitecture/c4k-nextcloud/badges/master/pipeline.svg)](https://gitlab.com/domaindrivenarchitecture/c4k-nextcloud/-/commits/master) + +[DeltaChat chat over e-mail](mailto:buero@meissa-gmbh.de?subject=community-chat) | [team@social.meissa-gmbh.de team@social.meissa-gmbh.de](https://social.meissa-gmbh.de/@team) | [Website & Blog](https://domaindrivenarchitecture.org) + +## Purpose + +c4k-nextcloud provides a k8s deployment for nextcloud containing: +* adjusted nextcloud docker image +* nextcloud +??? * ingress having a letsencrypt managed certificate +??? * postgres database + +The package aims to a low load sceanrio. + +## Status + +This is under development. + +## Manual restore + +1) Scale Nextcloud deployment down: +kubectl scale deployment nextcloud --replicas=0 + +2)apply backup and restore pod: +kubectl apply -f src/main/resources/backup/backup-restore.yaml + +3) exec into pod and execute restore pod +kubectl exec -it backup-restore -- /usr/local/bin/restore.sh + +4) Scale Nextcloud deployment up: +kubectl scale deployment nextcloud --replicas=1 + +5) Update index of Nextcloud: +Nextcloud > Settings > System > Advanced > Indexing +## License + +Copyright © 2021 meissa GmbH +Licensed under the [Apache License, Version 2.0](LICENSE) (the "License") +Pls. find licenses of our subcomponents [here](doc/SUBCOMPONENT_LICENSE) \ No newline at end of file diff --git a/doc/Development.md b/doc/Development.md new file mode 100644 index 0000000..461f3d6 --- /dev/null +++ b/doc/Development.md @@ -0,0 +1,110 @@ +# Project Setup + +## clj setup + +### install leiningen +``` +sudo apt install leiningen +``` +or manually using Instructions on https://leiningen.org/#install + +### install vscode + extensions +``` +sudo snap install code +``` +or with packages from https://code.visualstudio.com/Download + +install extension "Calva: Clojure & ClojureScript Interactive Programming" + +## cljs / js-dev setup + +``` +sudo apt install npm +sudo npm install -g npx + +# maybe +sudo npm install -g shadow-cljs + +# in project root to retrieve all dependencies +npm install --ignore-scripts +npx shadow-cljs compile test +``` + +### create frontend script + +``` +npx shadow-cljs release frontend +``` + +## graalvm-setup + +``` +curl -LO https://github.com/graalvm/graalvm-ce-builds/releases/download/vm-21.0.0.2/graalvm-ce-java11-linux-amd64-21.0.0.2.tar.gz + +# unpack +tar -xzf graalvm-ce-java11-linux-amd64-21.0.0.2.tar.gz + +sudo mv graalvm-ce-java11-21.0.0.2 /usr/lib/jvm/ +sudo ln -s /usr/lib/jvm/graalvm-ce-java11-21.0.0.2 /usr/lib/jvm/graalvm +sudo ln -s /usr/lib/jvm/graalvm/bin/gu /usr/local/bin +sudo update-alternatives --install /usr/bin/java java /usr/lib/jvm/graalvm/bin/java 2 +sudo update-alternatives --config java + +# install native-image in graalvm-ce-java11-linux-amd64-21.0.0.2/bin +sudo gu install native-image +sudo ln -s /usr/lib/jvm/graalvm/bin/native-image /usr/local/bin + +# deps +sudo apt-get install build-essential libz-dev zlib1g-dev + +# build +cd ~/repo/dda/c4k-cloud +lein uberjar +mkdir -p target/graalvm +lein native + +# execute +./target/graalvm/c4k-cloud -h +./target/graalvm/c4k-cloud src/test/resources/valid-config.edn src/test/resources/valid-auth.edn +./target/graalvm/c4k-cloud src/test/resources/invalid-config.edn src/test/resources/invalid-auth.edn +``` + +## c4k-setup +### install kubectl + +``` +sudo -i +curl -s https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add - +echo "deb http://apt.kubernetes.io/ kubernetes-xenial main" \ + | tee -a /etc/apt/sources.list.d/kubernetes.list +apt update && apt install kubectl +kubectl completion bash >> /etc/bash_completion.d/kubernetes +``` + +### install kubeconform + +``` +curl -Lo /tmp/kubeconform.tar.gz https://github.com/yannh/kubeconform/releases/download/v0.4.7/kubeconform-linux-amd64.tar.gz +tar -xf /tmp/kubeconform.tar.gz +sudo cp kubeconform /usr/local/bin +``` + +### remote access to c4k + +``` +scp -r root@devops.test.meissa-gmbh.de:/home/c4k/.kube ~/ +ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no root@devops.test.meissa-gmbh.de -L 8002:localhost:8002 -L 6443:192.168.5.1:6443 + +# add in /etc/hosts "127.0.0.1 kubernetes" + +# change in ~/.kube/config 192.168.5.1 -> kubernetes + +kubectl get pods +``` + +### deploy cloud + +``` +java -jar target/uberjar/c4k-cloud-standalone.jar valid-config.edn valid-auth.edn | kubeconform --kubernetes-version 1.19.0 --strict --skip Certificate - +java -jar target/uberjar/c4k-cloud-standalone.jar valid-config.edn my-auth.edn | kubectl apply -f - +``` diff --git a/doc/Releasing.md b/doc/Releasing.md new file mode 100644 index 0000000..e3aa755 --- /dev/null +++ b/doc/Releasing.md @@ -0,0 +1,14 @@ +# stable release (should be done from master) + +``` +#adjust [version] +vi package.json + +lein release +git push --follow-tags + +# bump version - increase version and add -SNAPSHOT +vi package.json +git commit -am "version bump" +git push +``` \ No newline at end of file diff --git a/doc/SUBCOMPONENT_LICENSE b/doc/SUBCOMPONENT_LICENSE new file mode 100644 index 0000000..581c523 --- /dev/null +++ b/doc/SUBCOMPONENT_LICENSE @@ -0,0 +1,206 @@ +SUBCOMPONENTS: + +This Software includes a number of subcomponents with +separate copyright notices and license terms. Your use of the source +code for the these subcomponents is subject to the terms and +conditions of the following licenses. + +lein with-profile uberjar licenses +... + + + +Eclipse Public License, Version 1.0 (EPL-1.0) + +THE ACCOMPANYING PROGRAM IS PROVIDED UNDER THE TERMS OF THIS ECLIPSE PUBLIC LICENSE ("AGREEMENT"). ANY USE, REPRODUCTION OR DISTRIBUTION OF THE PROGRAM CONSTITUTES RECIPIENT'S ACCEPTANCE OF THIS AGREEMENT. + +1. DEFINITIONS + +"Contribution" means: + + a) in the case of the initial Contributor, the initial code and documentation distributed under this Agreement, and + b) in the case of each subsequent Contributor: + i) changes to the Program, and + ii) additions to the Program; + where such changes and/or additions to the Program originate from and are distributed by that particular Contributor. A Contribution 'originates' from a Contributor if it was added to the Program by such Contributor itself or anyone acting on such Contributor's behalf. Contributions do not include additions to the Program which: (i) are separate modules of software distributed in conjunction with the Program under their own license agreement, and (ii) are not derivative works of the Program. + +"Contributor" means any person or entity that distributes the Program. + +"Licensed Patents" mean patent claims licensable by a Contributor which are necessarily infringed by the use or sale of its Contribution alone or when combined with the Program. + +"Program" means the Contributions distributed in accordance with this Agreement. + +"Recipient" means anyone who receives the Program under this Agreement, including all Contributors. + +2. GRANT OF RIGHTS + + a) Subject to the terms of this Agreement, each Contributor hereby grants Recipient a non-exclusive, worldwide, royalty-free copyright license to reproduce, prepare derivative works of, publicly display, publicly perform, distribute and sublicense the Contribution of such Contributor, if any, and such derivative works, in source code and object code form. + b) Subject to the terms of this Agreement, each Contributor hereby grants Recipient a non-exclusive, worldwide, royalty-free patent license under Licensed Patents to make, use, sell, offer to sell, import and otherwise transfer the Contribution of such Contributor, if any, in source code and object code form. This patent license shall apply to the combination of the Contribution and the Program if, at the time the Contribution is added by the Contributor, such addition of the Contribution causes such combination to be covered by the Licensed Patents. The patent license shall not apply to any other combinations which include the Contribution. No hardware per se is licensed hereunder. + c) Recipient understands that although each Contributor grants the licenses to its Contributions set forth herein, no assurances are provided by any Contributor that the Program does not infringe the patent or other intellectual property rights of any other entity. Each Contributor disclaims any liability to Recipient for claims brought by any other entity based on infringement of intellectual property rights or otherwise. As a condition to exercising the rights and licenses granted hereunder, each Recipient hereby assumes sole responsibility to secure any other intellectual property rights needed, if any. For example, if a third party patent license is required to allow Recipient to distribute the Program, it is Recipient's responsibility to acquire that license before distributing the Program. + d) Each Contributor represents that to its knowledge it has sufficient copyright rights in its Contribution, if any, to grant the copyright license set forth in this Agreement. + +3. REQUIREMENTS + +A Contributor may choose to distribute the Program in object code form under its own license agreement, provided that: + + a) it complies with the terms and conditions of this Agreement; and + b) its license agreement: + i) effectively disclaims on behalf of all Contributors all warranties and conditions, express and implied, including warranties or conditions of title and non-infringement, and implied warranties or conditions of merchantability and fitness for a particular purpose; + ii) effectively excludes on behalf of all Contributors all liability for damages, including direct, indirect, special, incidental and consequential damages, such as lost profits; + iii) states that any provisions which differ from this Agreement are offered by that Contributor alone and not by any other party; and + iv) states that source code for the Program is available from such Contributor, and informs licensees how to obtain it in a reasonable manner on or through a medium customarily used for software exchange. + +When the Program is made available in source code form: + + a) it must be made available under this Agreement; and + b) a copy of this Agreement must be included with each copy of the Program. + +Contributors may not remove or alter any copyright notices contained within the Program. + +Each Contributor must identify itself as the originator of its Contribution, if any, in a manner that reasonably allows subsequent Recipients to identify the originator of the Contribution. + +4. COMMERCIAL DISTRIBUTION + +Commercial distributors of software may accept certain responsibilities with respect to end users, business partners and the like. While this license is intended to facilitate the commercial use of the Program, the Contributor who includes the Program in a commercial product offering should do so in a manner which does not create potential liability for other Contributors. Therefore, if a Contributor includes the Program in a commercial product offering, such Contributor ("Commercial Contributor") hereby agrees to defend and indemnify every other Contributor ("Indemnified Contributor") against any losses, damages and costs (collectively "Losses") arising from claims, lawsuits and other legal actions brought by a third party against the Indemnified Contributor to the extent caused by the acts or omissions of such Commercial Contributor in connection with its distribution of the Program in a commercial product offering. The obligations in this section do not apply to any claims or Losses relating to any actual or alleged intellectual property infringement. In order to qualify, an Indemnified Contributor must: a) promptly notify the Commercial Contributor in writing of such claim, and b) allow the Commercial Contributor to control, and cooperate with the Commercial Contributor in, the defense and any related settlement negotiations. The Indemnified Contributor may participate in any such claim at its own expense. + +For example, a Contributor might include the Program in a commercial product offering, Product X. That Contributor is then a Commercial Contributor. If that Commercial Contributor then makes performance claims, or offers warranties related to Product X, those performance claims and warranties are such Commercial Contributor's responsibility alone. Under this section, the Commercial Contributor would have to defend claims against the other Contributors related to those performance claims and warranties, and if a court requires any other Contributor to pay any damages as a result, the Commercial Contributor must pay those damages. + +5. NO WARRANTY + +EXCEPT AS EXPRESSLY SET FORTH IN THIS AGREEMENT, THE PROGRAM IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED INCLUDING, WITHOUT LIMITATION, ANY WARRANTIES OR CONDITIONS OF TITLE, NON-INFRINGEMENT, MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE. Each Recipient is solely responsible for determining the appropriateness of using and distributing the Program and assumes all risks associated with its exercise of rights under this Agreement , including but not limited to the risks and costs of program errors, compliance with applicable laws, damage to or loss of data, programs or equipment, and unavailability or interruption of operations. + +6. DISCLAIMER OF LIABILITY + +EXCEPT AS EXPRESSLY SET FORTH IN THIS AGREEMENT, NEITHER RECIPIENT NOR ANY CONTRIBUTORS SHALL HAVE ANY LIABILITY FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING WITHOUT LIMITATION LOST PROFITS), HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OR DISTRIBUTION OF THE PROGRAM OR THE EXERCISE OF ANY RIGHTS GRANTED HEREUNDER, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. + +7. GENERAL + +If any provision of this Agreement is invalid or unenforceable under applicable law, it shall not affect the validity or enforceability of the remainder of the terms of this Agreement, and without further action by the parties hereto, such provision shall be reformed to the minimum extent necessary to make such provision valid and enforceable. + +If Recipient institutes patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Program itself (excluding combinations of the Program with other software or hardware) infringes such Recipient's patent(s), then such Recipient's rights granted under Section 2(b) shall terminate as of the date such litigation is filed. + +All Recipient's rights under this Agreement shall terminate if it fails to comply with any of the material terms or conditions of this Agreement and does not cure such failure in a reasonable period of time after becoming aware of such noncompliance. If all Recipient's rights under this Agreement terminate, Recipient agrees to cease use and distribution of the Program as soon as reasonably practicable. However, Recipient's obligations under this Agreement and any licenses granted by Recipient relating to the Program shall continue and survive. + +Everyone is permitted to copy and distribute copies of this Agreement, but in order to avoid inconsistency the Agreement is copyrighted and may only be modified in the following manner. The Agreement Steward reserves the right to publish new versions (including revisions) of this Agreement from time to time. No one other than the Agreement Steward has the right to modify this Agreement. The Eclipse Foundation is the initial Agreement Steward. The Eclipse Foundation may assign the responsibility to serve as the Agreement Steward to a suitable separate entity. Each new version of the Agreement will be given a distinguishing version number. The Program (including Contributions) may always be distributed subject to the version of the Agreement under which it was received. In addition, after a new version of the Agreement is published, Contributor may elect to distribute the Program (including its Contributions) under the new version. Except as expressly stated in Sections 2(a) and 2(b) above, Recipient receives no rights or licenses to the intellectual property of any Contributor under this Agreement, whether expressly, by implication, estoppel or otherwise. All rights in the Program not expressly granted under this Agreement are reserved. + +This Agreement is governed by the laws of the State of New York and the intellectual property laws of the United States of America. No party to this Agreement will bring a legal action under this Agreement more than one year after the cause of action arose. Each party waives its rights to a jury trial in any resulting litigation. +------------------------------------------------------------------------------- + +The MIT License (MIT) + +Copyright © 2015 Stuart Sierra + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +------------------------------------------------------------------------------- + +The BSD 3-Clause License The following is a BSD 3-Clause ("BSD New" or "BSD Simplified") license template. To generate your own license, change the values of OWNER, ORGANIZATION and YEAR from their original values as given here, and substitute your own. + +Note: You may omit clause 3 and still be OSD-conformant. Despite its colloquial name "BSD New", this is not the newest version of the BSD license; it was followed by the even newer BSD-2-Clause version, sometimes known as the "Simplified BSD License". On January 9th, 2008 the OSI Board approved BSD-2-Clause, which is used by FreeBSD and others. It omits the final "no-endorsement" clause and is thus roughly equivalent to the MIT License. + +Historical Background: The original license used on BSD Unix had four clauses. The advertising clause (the third of four clauses) required you to acknowledge use of U.C. Berkeley code in your advertising of any product using that code. It was officially rescinded by the Director of the Office of Technology Licensing of the University of California on July 22nd, 1999. He states that clause 3 is "hereby deleted in its entirety." The four clause license has not been approved by OSI. The license below does not contain the advertising clause. + +This prelude is not part of the license. + +OWNER = Regents of the University of California + +ORGANIZATION = University of California, Berkeley + +YEAR = 1998 + +In the original BSD license, both occurrences of the phrase "COPYRIGHT HOLDERS AND CONTRIBUTORS" in the disclaimer read "REGENTS AND CONTRIBUTORS". +License template + +Copyright (c) $YEAR $OWNER, All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: + + Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. + Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. + Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +------------------------------------------------------------------------------ + + +GNU LESSER GENERAL PUBLIC LICENSE + +Version 3, 29 June 2007 + +Copyright © 2007 Free Software Foundation, Inc. + +Everyone is permitted to copy and distribute verbatim copies of this license document, but changing it is not allowed. + +This version of the GNU Lesser General Public License incorporates the terms and conditions of version 3 of the GNU General Public License, supplemented by the additional permissions listed below. +0. Additional Definitions. + +As used herein, “this License” refers to version 3 of the GNU Lesser General Public License, and the “GNU GPL” refers to version 3 of the GNU General Public License. + +“The Library” refers to a covered work governed by this License, other than an Application or a Combined Work as defined below. + +An “Application” is any work that makes use of an interface provided by the Library, but which is not otherwise based on the Library. Defining a subclass of a class defined by the Library is deemed a mode of using an interface provided by the Library. + +A “Combined Work” is a work produced by combining or linking an Application with the Library. The particular version of the Library with which the Combined Work was made is also called the “Linked Version”. + +The “Minimal Corresponding Source” for a Combined Work means the Corresponding Source for the Combined Work, excluding any source code for portions of the Combined Work that, considered in isolation, are based on the Application, and not on the Linked Version. + +The “Corresponding Application Code” for a Combined Work means the object code and/or source code for the Application, including any data and utility programs needed for reproducing the Combined Work from the Application, but excluding the System Libraries of the Combined Work. +1. Exception to Section 3 of the GNU GPL. + +You may convey a covered work under sections 3 and 4 of this License without being bound by section 3 of the GNU GPL. +2. Conveying Modified Versions. + +If you modify a copy of the Library, and, in your modifications, a facility refers to a function or data to be supplied by an Application that uses the facility (other than as an argument passed when the facility is invoked), then you may convey a copy of the modified version: + + a) under this License, provided that you make a good faith effort to ensure that, in the event an Application does not supply the function or data, the facility still operates, and performs whatever part of its purpose remains meaningful, or + b) under the GNU GPL, with none of the additional permissions of this License applicable to that copy. + +3. Object Code Incorporating Material from Library Header Files. + +The object code form of an Application may incorporate material from a header file that is part of the Library. You may convey such object code under terms of your choice, provided that, if the incorporated material is not limited to numerical parameters, data structure layouts and accessors, or small macros, inline functions and templates (ten or fewer lines in length), you do both of the following: + + a) Give prominent notice with each copy of the object code that the Library is used in it and that the Library and its use are covered by this License. + b) Accompany the object code with a copy of the GNU GPL and this license document. + +4. Combined Works. + +You may convey a Combined Work under terms of your choice that, taken together, effectively do not restrict modification of the portions of the Library contained in the Combined Work and reverse engineering for debugging such modifications, if you also do each of the following: + + a) Give prominent notice with each copy of the Combined Work that the Library is used in it and that the Library and its use are covered by this License. + b) Accompany the Combined Work with a copy of the GNU GPL and this license document. + c) For a Combined Work that displays copyright notices during execution, include the copyright notice for the Library among these notices, as well as a reference directing the user to the copies of the GNU GPL and this license document. + d) Do one of the following: + 0) Convey the Minimal Corresponding Source under the terms of this License, and the Corresponding Application Code in a form suitable for, and under terms that permit, the user to recombine or relink the Application with a modified version of the Linked Version to produce a modified Combined Work, in the manner specified by section 6 of the GNU GPL for conveying Corresponding Source. + 1) Use a suitable shared library mechanism for linking with the Library. A suitable mechanism is one that (a) uses at run time a copy of the Library already present on the user's computer system, and (b) will operate properly with a modified version of the Library that is interface-compatible with the Linked Version. + e) Provide Installation Information, but only if you would otherwise be required to provide such information under section 6 of the GNU GPL, and only to the extent that such information is necessary to install and execute a modified version of the Combined Work produced by recombining or relinking the Application with a modified version of the Linked Version. (If you use option 4d0, the Installation Information must accompany the Minimal Corresponding Source and Corresponding Application Code. If you use option 4d1, you must provide the Installation Information in the manner specified by section 6 of the GNU GPL for conveying Corresponding Source.) + +5. Combined Libraries. + +You may place library facilities that are a work based on the Library side by side in a single library together with other library facilities that are not Applications and are not covered by this License, and convey such a combined library under terms of your choice, if you do both of the following: + + a) Accompany the combined library with a copy of the same work based on the Library, uncombined with any other library facilities, conveyed under the terms of this License. + b) Give prominent notice with the combined library that part of it is a work based on the Library, and explaining where to find the accompanying uncombined form of the same work. + +6. Revised Versions of the GNU Lesser General Public License. + +The Free Software Foundation may publish revised and/or new versions of the GNU Lesser General Public License from time to time. Such new versions will be similar in spirit to the present version, but may differ in detail to address new problems or concerns. + +Each version is given a distinguishing version number. If the Library as you received it specifies that a certain numbered version of the GNU Lesser General Public License “or any later version” applies to it, you have the option of following the terms and conditions either of that published version or of any later version published by the Free Software Foundation. If the Library as you received it does not specify a version number of the GNU Lesser General Public License, you may choose any version of the GNU Lesser General Public License ever published by the Free Software Foundation. + +If the Library as you received it specifies that a proxy can decide whether future versions of the GNU Lesser General Public License shall apply, that proxy's public statement of acceptance of any version is permanent authorization for you to choose that version for the Library. + +------------------------------------------------------------------------------ +License + +Copyright (c) 2000 - 2018 The Legion of the Bouncy Castle Inc. (https://www.bouncycastle.org) + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/infrastructure/docker-backup/build.py b/infrastructure/docker-backup/build.py new file mode 100644 index 0000000..68ac1ae --- /dev/null +++ b/infrastructure/docker-backup/build.py @@ -0,0 +1,51 @@ +from os import environ +from pybuilder.core import task, init +from ddadevops import * +import logging + +name = 'c4k-cloud-backup' +MODULE = 'docker' +PROJECT_ROOT_PATH = '../..' + +class MyBuild(DevopsDockerBuild): + pass + +@init +def initialize(project): + project.build_depends_on('ddadevops>=0.12.4') + stage = 'prod' + dockerhub_user = environ.get('DOCKERHUB_USER') + if not dockerhub_user: + dockerhub_user = gopass_field_from_path('meissa/web/docker.com', 'login') + dockerhub_password = environ.get('DOCKERHUB_PASSWORD') + if not dockerhub_password: + dockerhub_password = gopass_password_from_path('meissa/web/docker.com') + tag = environ.get('CI_COMMIT_TAG') + if not tag: + tag = get_tag_from_latest_commit() + config = create_devops_docker_build_config( + stage, PROJECT_ROOT_PATH, MODULE, dockerhub_user, dockerhub_password, docker_publish_tag=tag) + build = MyBuild(project, config) + build.initialize_build_dir() + + +@task +def image(project): + build = get_devops_build(project) + build.image() + +@task +def drun(project): + build = get_devops_build(project) + build.drun() + +@task +def publish(project): + build = get_devops_build(project) + build.dockerhub_login() + build.dockerhub_publish() + +@task +def test(project): + build = get_devops_build(project) + build.test() diff --git a/infrastructure/docker-backup/image/Dockerfile b/infrastructure/docker-backup/image/Dockerfile new file mode 100644 index 0000000..20e4c73 --- /dev/null +++ b/infrastructure/docker-backup/image/Dockerfile @@ -0,0 +1,5 @@ +FROM domaindrivenarchitecture/dda-backup + +# Prepare Entrypoint Script +ADD resources /tmp +RUN /tmp/install.sh diff --git a/infrastructure/docker-backup/image/resources/backup.sh b/infrastructure/docker-backup/image/resources/backup.sh new file mode 100755 index 0000000..d01affd --- /dev/null +++ b/infrastructure/docker-backup/image/resources/backup.sh @@ -0,0 +1,22 @@ +#!/bin/bash + +set -o pipefail + +function main() { + file_env AWS_ACCESS_KEY_ID + file_env AWS_SECRET_ACCESS_KEY + file_env POSTGRES_DB + file_env POSTGRES_PASSWORD + file_env POSTGRES_USER + file_env RESTIC_DAYS_TO_KEEP 14 + + backup-roles "" + backup-db-dump + backup-fs-from-directory '/var/backups/' 'data/' +} + +source /usr/local/lib/functions.sh +source /usr/local/lib/file-functions.sh +source /usr/local/lib/pg-functions.sh + +main diff --git a/infrastructure/docker-backup/image/resources/entrypoint-start-and-wait.sh b/infrastructure/docker-backup/image/resources/entrypoint-start-and-wait.sh new file mode 100644 index 0000000..67a8f3b --- /dev/null +++ b/infrastructure/docker-backup/image/resources/entrypoint-start-and-wait.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +function main() { + file_env POSTGRES_DB + file_env POSTGRES_PASSWORD + file_env POSTGRES_USER + + create-pg-pass + + while true; do + sleep 1m + done +} + +source /usr/local/lib/functions.sh +source /usr/local/lib/pg-functions.sh +main \ No newline at end of file diff --git a/infrastructure/docker-backup/image/resources/entrypoint.sh b/infrastructure/docker-backup/image/resources/entrypoint.sh new file mode 100755 index 0000000..924f24c --- /dev/null +++ b/infrastructure/docker-backup/image/resources/entrypoint.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +function main() { + file_env POSTGRES_DB + file_env POSTGRES_PASSWORD + file_env POSTGRES_USER + + create-pg-pass + + /usr/local/bin/backup.sh +} + +source /usr/local/lib/functions.sh +source /usr/local/lib/pg-functions.sh +main diff --git a/infrastructure/docker-backup/image/resources/init.sh b/infrastructure/docker-backup/image/resources/init.sh new file mode 100755 index 0000000..5693dcb --- /dev/null +++ b/infrastructure/docker-backup/image/resources/init.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +function main() { + file_env AWS_ACCESS_KEY_ID + file_env AWS_SECRET_ACCESS_KEY + + init-role-repo + init-database-repo + init-file-repo +} + +source /usr/local/lib/functions.sh +source /usr/local/lib/file-functions.sh +source /usr/local/lib/pg-functions.sh +main diff --git a/infrastructure/docker-backup/image/resources/install.sh b/infrastructure/docker-backup/image/resources/install.sh new file mode 100755 index 0000000..1a8cbd7 --- /dev/null +++ b/infrastructure/docker-backup/image/resources/install.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +apt-get update > /dev/null; + +install -m 0700 /tmp/entrypoint.sh / +install -m 0700 /tmp/entrypoint-start-and-wait.sh / + +install -m 0700 /tmp/init.sh /usr/local/bin/ +install -m 0700 /tmp/backup.sh /usr/local/bin/ +install -m 0700 /tmp/restore.sh /usr/local/bin/ +install -m 0700 /tmp/restic-snapshots.sh /usr/local/bin/ diff --git a/infrastructure/docker-backup/image/resources/restic-snapshots.sh b/infrastructure/docker-backup/image/resources/restic-snapshots.sh new file mode 100755 index 0000000..a428fe2 --- /dev/null +++ b/infrastructure/docker-backup/image/resources/restic-snapshots.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +set -o pipefail + +function main() { + file_env AWS_ACCESS_KEY_ID + file_env AWS_SECRET_ACCESS_KEY + + restic -r ${RESTIC_REPOSITORY}/pg-role snapshots + restic -r ${RESTIC_REPOSITORY}/pg-database snapshots + restic -r ${RESTIC_REPOSITORY}/files snapshots +} + +source /usr/local/lib/functions.sh +source /usr/local/lib/file-functions.sh + +main diff --git a/infrastructure/docker-backup/image/resources/restore.sh b/infrastructure/docker-backup/image/resources/restore.sh new file mode 100755 index 0000000..ba1e091 --- /dev/null +++ b/infrastructure/docker-backup/image/resources/restore.sh @@ -0,0 +1,36 @@ +#!/bin/bash + +set -Eeo pipefail + +function main() { + + file_env AWS_ACCESS_KEY_ID + file_env AWS_SECRET_ACCESS_KEY + + file_env POSTGRES_DB + file_env POSTGRES_PASSWORD + file_env POSTGRES_USER + + # Im Nextcloud pod: /opt/atlassian-cloud-software-standalone/bin/stop-cloud.sh + + # Restore latest snapshot into /var/backups/restic-restore + rm -rf /var/backups/restic-restore + restore-directory '/var/backups/restic-restore' + + # Restore data dir backup + rm -rf /var/backups/data/* + cp -a /var/backups/restic-restore/data/* /var/backups/data + + # Restore db + drop-create-db + restore-roles + restore-db + + # /opt/atlassian-cloud-software-standalone/bin/start-cloud.sh +} + +source /usr/local/lib/functions.sh +source /usr/local/lib/file-functions.sh +source /usr/local/lib/pg-functions.sh +main + diff --git a/infrastructure/docker-backup/test/Dockerfile b/infrastructure/docker-backup/test/Dockerfile new file mode 100644 index 0000000..a18ebee --- /dev/null +++ b/infrastructure/docker-backup/test/Dockerfile @@ -0,0 +1,8 @@ +FROM c4k-cloud-backup + +RUN curl -L -o /tmp/serverspec.jar \ + https://github.com/DomainDrivenArchitecture/dda-serverspec-crate/releases/download/2.0.0/dda-serverspec-standalone.jar + +COPY serverspec.edn /tmp/serverspec.edn + +RUN java -jar /tmp/serverspec.jar /tmp/serverspec.edn -v \ No newline at end of file diff --git a/infrastructure/docker-backup/test/serverspec.edn b/infrastructure/docker-backup/test/serverspec.edn new file mode 100644 index 0000000..09623c7 --- /dev/null +++ b/infrastructure/docker-backup/test/serverspec.edn @@ -0,0 +1,6 @@ +{:file [{:path "/usr/local/bin/init.sh" :mod "700"} + {:path "/usr/local/bin/backup.sh" :mod "700"} + {:path "/usr/local/bin/restore.sh" :mod "700"} + {:path "/usr/local/bin/restic-snapshots.sh" :mod "700"} + {:path "/entrypoint.sh" :mod "700"} + {:path "/entrypoint-start-and-wait.sh" :mod "700"}]} diff --git a/infrastructure/docker-cloud/build.py b/infrastructure/docker-cloud/build.py new file mode 100644 index 0000000..c192b2a --- /dev/null +++ b/infrastructure/docker-cloud/build.py @@ -0,0 +1,51 @@ +from os import environ +from pybuilder.core import task, init +from ddadevops import * +import logging + +name = 'c4k-cloud' +MODULE = 'docker' +PROJECT_ROOT_PATH = '../..' + +class MyBuild(DevopsDockerBuild): + pass + +@init +def initialize(project): + project.build_depends_on('ddadevops>=0.12.4') + stage = 'prod' + dockerhub_user = environ.get('DOCKERHUB_USER') + if not dockerhub_user: + dockerhub_user = gopass_field_from_path('meissa/web/docker.com', 'login') + dockerhub_password = environ.get('DOCKERHUB_PASSWORD') + if not dockerhub_password: + dockerhub_password = gopass_password_from_path('meissa/web/docker.com') + tag = environ.get('CI_COMMIT_TAG') + if not tag: + tag = get_tag_from_latest_commit() + config = create_devops_docker_build_config( + stage, PROJECT_ROOT_PATH, MODULE, dockerhub_user, dockerhub_password, docker_publish_tag=tag) + build = MyBuild(project, config) + build.initialize_build_dir() + + +@task +def image(project): + build = get_devops_build(project) + build.image() + +@task +def drun(project): + build = get_devops_build(project) + build.drun() + +@task +def publish(project): + build = get_devops_build(project) + build.dockerhub_login() + build.dockerhub_publish() + +@task +def test(project): + build = get_devops_build(project) + build.test() diff --git a/infrastructure/docker-cloud/image/Dockerfile b/infrastructure/docker-cloud/image/Dockerfile new file mode 100644 index 0000000..9551640 --- /dev/null +++ b/infrastructure/docker-cloud/image/Dockerfile @@ -0,0 +1,10 @@ +FROM ubuntu:20.04 + +ENV CLOUD_HOME="/var/cloud" \ + DOWNLOAD_URL="https://product-downloads.atlassian.com/software/nextcloud/downloads" \ + CLOUD_RELEASE="8.8.0" + # CLOUD_RELEASE="20.1.0"??? +ADD resources /tmp/resources +RUN /tmp/resources/install.sh + +USER 901:901 diff --git a/infrastructure/docker-cloud/image/resources/dbconfig.xml b/infrastructure/docker-cloud/image/resources/dbconfig.xml new file mode 100644 index 0000000..7ff0887 --- /dev/null +++ b/infrastructure/docker-cloud/image/resources/dbconfig.xml @@ -0,0 +1,25 @@ + + + + defaultDS + default + postgres72 + public + + jdbc:postgresql://postgresql-service:5432/cloud + org.postgresql.Driver + $CLOUD_DB_USER + $CLOUD_DB_PASSWORD + 20 + 20 + 30000 + select 1 + 60000 + 300000 + 20 + true + 300 + false + true + + \ No newline at end of file diff --git a/infrastructure/docker-cloud/image/resources/entrypoint.sh b/infrastructure/docker-cloud/image/resources/entrypoint.sh new file mode 100644 index 0000000..115e905 --- /dev/null +++ b/infrastructure/docker-cloud/image/resources/entrypoint.sh @@ -0,0 +1,44 @@ +#!/bin/bash + +# usage: file_env VAR [DEFAULT] +# ie: file_env 'XYZ_DB_PASSWORD' 'example' +# (will allow for "$XYZ_DB_PASSWORD_FILE" to fill in the value of +# "$XYZ_DB_PASSWORD" from a file, especially for Docker's secrets feature) +function file_env() { + local var="$1" + local fileVar="${var}_FILE" + local def="${2:-}" + local varValue=$(env | grep -E "^${var}=" | sed -E -e "s/^${var}=//") + local fileVarValue=$(env | grep -E "^${fileVar}=" | sed -E -e "s/^${fileVar}=//") + if [ -n "${varValue}" ] && [ -n "${fileVarValue}" ]; then + echo >&2 "error: both $var and $fileVar are set (but are exclusive)" + exit 1 + fi + if [ -n "${varValue}" ]; then + export "$var"="${varValue}" + elif [ -n "${fileVarValue}" ]; then + export "$var"="$(cat "${fileVarValue}")" + elif [ -n "${def}" ]; then + export "$var"="$def" + fi + unset "$fileVar" +} + +function main() { + file_env "FQDN" + file_env "DB_USERNAME" + file_env "DB_PASSWORD" + + xmlstarlet ed -L -u "/Server/Service/Connector[@proxyName='{subdomain}.{domain}.com']/@proxyName" \ + -v "$FQDN" /opt/atlassian-cloud-software-standalone/conf/server.xml + xmlstarlet ed -L -u "/cloud-database-config/jdbc-datasource/username" \ + -v "$DB_USERNAME" /app/dbconfig.xml + xmlstarlet ed -L -u "/cloud-database-config/jdbc-datasource/password" \ + -v "$DB_PASSWORD" /app/dbconfig.xml + + install -ocloud -gcloud -m660 /app/dbconfig.xml /var/cloud/dbconfig.xml + /opt/atlassian-cloud-software-standalone/bin/setenv.sh run + /opt/atlassian-cloud-software-standalone/bin/start-cloud.sh run +} + +main diff --git a/infrastructure/docker-cloud/image/resources/install.sh b/infrastructure/docker-cloud/image/resources/install.sh new file mode 100755 index 0000000..4927a6b --- /dev/null +++ b/infrastructure/docker-cloud/image/resources/install.sh @@ -0,0 +1,30 @@ +#!/bin/bash + +function main() { + + upgradeSystem + apt-get -qqy install curl openjdk-11-jre-headless xmlstarlet > /dev/null + + adduser --system --disabled-login --home ${CLOUD_HOME} --uid 901 --group cloud + + mkdir -p /app + + cd /tmp + curl --location ${DOWNLOAD_URL}/atlassian-cloud-software-${CLOUD_RELEASE}.tar.gz \ + -o atlassian-cloud-software-${CLOUD_RELEASE}.tar.gz + tar -xvf atlassian-cloud-software-${CLOUD_RELEASE}.tar.gz -C /tmp/ + mv /tmp/atlassian-cloud-software-${CLOUD_RELEASE}-standalone \ + /opt/atlassian-cloud-software-standalone + chown -R cloud:cloud /opt/atlassian-cloud-software-standalone + + install -ocloud -gcloud -m744 /tmp/resources/entrypoint.sh /app/entrypoint.sh + install -ocloud -gcloud -m744 /tmp/resources/setenv.sh /opt/atlassian-cloud-software-standalone/bin/setenv.sh + install -ocloud -gcloud -m660 /tmp/resources/dbconfig.xml /app/dbconfig.xml + install -ocloud -gcloud -m660 /tmp/resources/server.xml /opt/atlassian-cloud-software-standalone/conf/server.xml + install -ocloud -gcloud -m660 /tmp/resources/logging.properties /opt/atlassian-cloud-software-standalone/conf/logging.properties + + cleanupDocker +} + +source /tmp/resources/install_functions.sh +main \ No newline at end of file diff --git a/infrastructure/docker-cloud/image/resources/logging.properties b/infrastructure/docker-cloud/image/resources/logging.properties new file mode 100644 index 0000000..267d805 --- /dev/null +++ b/infrastructure/docker-cloud/image/resources/logging.properties @@ -0,0 +1,69 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +handlers = java.util.logging.ConsoleHandler + +.handlers = java.util.logging.ConsoleHandler + +############################################################ +# Handler specific properties. +# Describes specific configuration info for Handlers. +############################################################ + +java.util.logging.ConsoleHandler.level = FINE +java.util.logging.ConsoleHandler.formatter = org.apache.juli.OneLineFormatter +java.util.logging.ConsoleHandler.encoding = UTF-8 + + +############################################################ +# Facility specific properties. +# Provides extra control for each logger. +############################################################ + +org.apache.catalina.core.ContainerBase.[Catalina].[localhost].level = INFO +org.apache.catalina.core.ContainerBase.[Catalina].[localhost].handlers = 2localhost.org.apache.juli.AsyncFileHandler + +org.apache.catalina.core.ContainerBase.[Catalina].[localhost].[/manager].level = INFO +org.apache.catalina.core.ContainerBase.[Catalina].[localhost].[/manager].handlers = 3manager.org.apache.juli.AsyncFileHandler + +org.apache.catalina.core.ContainerBase.[Catalina].[localhost].[/host-manager].level = INFO +org.apache.catalina.core.ContainerBase.[Catalina].[localhost].[/host-manager].handlers = 4host-manager.org.apache.juli.AsyncFileHandler + +# For example, set the org.apache.catalina.util.LifecycleBase logger to log +# each component that extends LifecycleBase changing state: +#org.apache.catalina.util.LifecycleBase.level = FINE + +# To see debug messages in TldLocationsCache, uncomment the following line: +#org.apache.jasper.compiler.TldLocationsCache.level = FINE + +# To see debug messages for HTTP/2 handling, uncomment the following line: +#org.apache.coyote.http2.level = FINE + +# To see debug messages for WebSocket handling, uncomment the following line: +#org.apache.tomcat.websocket.level = FINE +# Suppress clearReferencesThreads and clearThreadLocalMap notifications on shutdown +org.apache.catalina.loader.WebappClassLoader.level = OFF + +# Suppress some useless REST messages. See JRADEV-12212. +com.sun.jersey.server.impl.application.WebApplicationImpl.level = WARNING +com.atlassian.plugins.rest.doclet.level = WARNING +com.sun.jersey.api.wadl.config.WadlGeneratorLoader.level = WARNING + +# Suppress log spam about the request body - https://cloud.atlassian.com/browse/JRASERVER-30406 +com.sun.jersey.spi.container.servlet.WebComponent.level = SEVERE +com.sun.jersey.spi.container.servlet.WebComponent.filterFormParameters.level = SEVERE + +# Suppress resources cache full warnings +org.apache.catalina.webresources.Cache.level = OFF \ No newline at end of file diff --git a/infrastructure/docker-cloud/image/resources/server.xml b/infrastructure/docker-cloud/image/resources/server.xml new file mode 100644 index 0000000..6e057fd --- /dev/null +++ b/infrastructure/docker-cloud/image/resources/server.xml @@ -0,0 +1,114 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/infrastructure/docker-cloud/image/resources/setenv.sh b/infrastructure/docker-cloud/image/resources/setenv.sh new file mode 100644 index 0000000..d587952 --- /dev/null +++ b/infrastructure/docker-cloud/image/resources/setenv.sh @@ -0,0 +1,121 @@ +# +# If the limit of files that Nextcloud can open is too low, it will be set to this value. +# +MIN_NOFILES_LIMIT=16384 + +# +# One way to set the Nextcloud HOME path is here via this variable. Simply uncomment it and set a valid path like /cloud/home. You can of course set it outside in the command terminal. That will also work. +# +CLOUD_HOME="/var/cloud" + +# +# Occasionally Atlassian Support may recommend that you set some specific JVM arguments. You can use this variable below to do that. +# +JVM_SUPPORT_RECOMMENDED_ARGS="" + +# +# You can use variable below to modify garbage collector settings. +# For Java 8 we recommend default settings +# For Java 11 and relatively small heaps we recommend: -XX:+UseParallelGC +# For Java 11 and larger heaps we recommend: -XX:+UseG1GC -XX:+ExplicitGCInvokesConcurrent +# +JVM_GC_ARGS="-XX:+UseG1GC -XX:+ExplicitGCInvokesConcurrent" + +# +# The following 2 settings control the minimum and maximum given to the Nextcloud Java virtual machine. In larger Nextcloud instances, the maximum amount will need to be increased. +# +JVM_MINIMUM_MEMORY="4096m" +JVM_MAXIMUM_MEMORY="4096m" + +# +# The following setting configures the size of JVM code cache. A high value of reserved size allows Nextcloud to work with more installed apps. +# +JVM_CODE_CACHE_ARGS='-XX:InitialCodeCacheSize=32m -XX:ReservedCodeCacheSize=512m' + +# +# The following are the required arguments for Nextcloud. +# +JVM_REQUIRED_ARGS='-Djava.awt.headless=true -Datlassian.standalone=CLOUD -Dorg.apache.jasper.runtime.BodyContentImpl.LIMIT_BUFFER=true -Dmail.mime.decodeparameters=true -Dorg.dom4j.factory=com.atlassian.core.xml.InterningDocumentFactory' + +# Uncomment this setting if you want to import data without notifications +# +#DISABLE_NOTIFICATIONS=" -Datlassian.mail.senddisabled=true -Datlassian.mail.fetchdisabled=true -Datlassian.mail.popdisabled=true" + + +#----------------------------------------------------------------------------------- +# +# In general don't make changes below here +# +#----------------------------------------------------------------------------------- + +#----------------------------------------------------------------------------------- +# Prevents the JVM from suppressing stack traces if a given type of exception +# occurs frequently, which could make it harder for support to diagnose a problem. +#----------------------------------------------------------------------------------- +JVM_EXTRA_ARGS="-XX:-OmitStackTraceInFastThrow -Djava.locale.providers=COMPAT" + +CURRENT_NOFILES_LIMIT=$( ulimit -Hn ) +ulimit -Sn $CURRENT_NOFILES_LIMIT +ulimit -n $(( CURRENT_NOFILES_LIMIT > MIN_NOFILES_LIMIT ? CURRENT_NOFILES_LIMIT : MIN_NOFILES_LIMIT )) + +PRGDIR=`dirname "$0"` +cat "${PRGDIR}"/cloudbanner.txt + +CLOUD_HOME_MINUSD="" +if [ "$CLOUD_HOME" != "" ]; then + echo $CLOUD_HOME | grep -q " " + if [ $? -eq 0 ]; then + echo "" + echo "--------------------------------------------------------------------------------------------------------------------" + echo " WARNING : You cannot have a CLOUD_HOME environment variable set with spaces in it. This variable is being ignored" + echo "--------------------------------------------------------------------------------------------------------------------" + else + CLOUD_HOME_MINUSD=-Dcloud.home=$CLOUD_HOME + fi +fi + +JAVA_OPTS="-Xms${JVM_MINIMUM_MEMORY} -Xmx${JVM_MAXIMUM_MEMORY} ${JVM_CODE_CACHE_ARGS} ${JAVA_OPTS} ${JVM_REQUIRED_ARGS} ${DISABLE_NOTIFICATIONS} ${JVM_SUPPORT_RECOMMENDED_ARGS} ${JVM_EXTRA_ARGS} ${CLOUD_HOME_MINUSD} ${START_CLOUD_JAVA_OPTS}" + +export JAVA_OPTS + +# DO NOT remove the following line +# !INSTALLER SET JAVA_HOME + +echo "" +echo "If you encounter issues starting or stopping Nextcloud, please see the Troubleshooting guide at https://docs.atlassian.com/cloud/jadm-docs-088/Troubleshooting+installation" +echo "" +if [ "$CLOUD_HOME_MINUSD" != "" ]; then + echo "Using CLOUD_HOME: $CLOUD_HOME" +fi + +# set the location of the pid file +if [ -z "$CATALINA_PID" ] ; then + if [ -n "$CATALINA_BASE" ] ; then + CATALINA_PID="$CATALINA_BASE"/work/catalina.pid + elif [ -n "$CATALINA_HOME" ] ; then + CATALINA_PID="$CATALINA_HOME"/work/catalina.pid + fi +fi +export CATALINA_PID + +if [ -z "$CATALINA_BASE" ]; then + if [ -z "$CATALINA_HOME" ]; then + LOGBASE=$PRGDIR + LOGTAIL=.. + else + LOGBASE=$CATALINA_HOME + LOGTAIL=. + fi +else + LOGBASE=$CATALINA_BASE + LOGTAIL=. +fi + +PUSHED_DIR=`pwd` +cd $LOGBASE +cd $LOGTAIL +LOGBASEABS=`pwd` +cd $PUSHED_DIR + +echo "" +echo "Server startup logs are located in $LOGBASEABS/logs/catalina.out" diff --git a/infrastructure/docker-cloud/image/resources/user.sh b/infrastructure/docker-cloud/image/resources/user.sh new file mode 100644 index 0000000..6ae8a6d --- /dev/null +++ b/infrastructure/docker-cloud/image/resources/user.sh @@ -0,0 +1,5 @@ +# START INSTALLER MAGIC ! DO NOT EDIT ! +CLOUD_USER="cloud" ## +# END INSTALLER MAGIC ! DO NOT EDIT ! + +export CLOUD_USER diff --git a/infrastructure/docker-cloud/test/Dockerfile b/infrastructure/docker-cloud/test/Dockerfile new file mode 100644 index 0000000..693ed79 --- /dev/null +++ b/infrastructure/docker-cloud/test/Dockerfile @@ -0,0 +1,8 @@ +FROM c4k-cloud + +RUN curl -L -o /tmp/serverspec.jar \ + https://github.com/DomainDrivenArchitecture/dda-serverspec-crate/releases/download/1.3.4/dda-serverspec-standalone.jar + +COPY serverspec.edn /tmp/serverspec.edn + +RUN java -jar /tmp/serverspec.jar /tmp/serverspec.edn -v diff --git a/infrastructure/docker-cloud/test/serverspec.edn b/infrastructure/docker-cloud/test/serverspec.edn new file mode 100644 index 0000000..a462c1a --- /dev/null +++ b/infrastructure/docker-cloud/test/serverspec.edn @@ -0,0 +1,3 @@ +{:file [{:path "/app/entrypoint.sh"} + {:path "/var/cloud"} + {:path "/opt/atlassian-cloud-software-standalone"}]} diff --git a/package.json b/package.json new file mode 100644 index 0000000..f29d34b --- /dev/null +++ b/package.json @@ -0,0 +1,33 @@ +{ + "name": "c4k-cloud", + "description": "Generate c4k yaml for a nextcloud deployment.", + "author": "meissa GmbH", + "version": "0.1.3-SNAPSHOT", + "homepage": "https://gitlab.com/domaindrivenarchitecture/c4k-nextcloud#readme", + "repository": "https://www.npmjs.com/package/c4k-nextcloud", + "license": "APACHE2", + "main": "c4k-cloud.js", + "bin": { + "c4k-cloud": "./c4k-cloud.js" + }, + "keywords": [ + "cljs", + "cloud", + "k8s", + "c4k", + "deployment", + "yaml", + "convention4kubernetes" + ], + "bugs": { + "url": "https://gitlab.com/domaindrivenarchitecture/c4k-nextcloud/issues" + }, + "dependencies": { + "js-base64": "^3.6.1", + "js-yaml": "^4.0.0" + }, + "devDependencies": { + "shadow-cljs": "^2.11.18", + "source-map-support": "^0.5.19" + } +} diff --git a/project.clj b/project.clj new file mode 100644 index 0000000..c6dc3f6 --- /dev/null +++ b/project.clj @@ -0,0 +1,41 @@ +(defproject org.domaindrivenarchitecture/c4k-cloud "0.1.3-SNAPSHOT" + :description "cloud c4k-installation package" + :url "https://domaindrivenarchitecture.org" + :license {:name "Apache License, Version 2.0" + :url "https://www.apache.org/licenses/LICENSE-2.0.html"} + :dependencies [[org.clojure/clojure "1.10.3"] + [org.clojure/tools.reader "1.3.4"] + [org.domaindrivenarchitecture/c4k-common-clj "0.2.8"]] + :target-path "target/%s/" + :source-paths ["src/main/cljc" + "src/main/clj"] + :resource-paths ["src/main/resources"] + :repositories [["snapshots" :clojars] + ["releases" :clojars]] + :deploy-repositories [["snapshots" :clojars] + ["releases" :clojars]] + :profiles {:test {:test-paths ["src/test/cljc"] + :resource-paths ["src/test/resources"] + :dependencies [[dda/data-test "0.1.1"]]} + :dev {:plugins [[lein-shell "0.5.0"]]} + :uberjar {:aot :all + :main dda.c4k-cloud.uberjar + :uberjar-name "c4k-cloud-standalone.jar" + :dependencies [[org.clojure/tools.cli "1.0.206"] + [ch.qos.logback/logback-classic "1.3.0-alpha4" + :exclusions [com.sun.mail/javax.mail]] + [org.slf4j/jcl-over-slf4j "2.0.0-alpha1"]]}} + :release-tasks [["test"] + ["vcs" "assert-committed"] + ["change" "version" "leiningen.release/bump-version" "release"] + ["vcs" "commit"] + ["vcs" "tag"] + ["change" "version" "leiningen.release/bump-version"]] + :aliases {"native" ["shell" + "native-image" + "--report-unsupported-elements-at-runtime" + "--initialize-at-build-time" + "-jar" "target/uberjar/c4k-cloud-standalone.jar" + "-H:ResourceConfigurationFiles=graalvm-resource-config.json" + "-H:Log=registerResource" + "-H:Name=target/graalvm/${:name}"]}) diff --git a/public/index.html b/public/index.html new file mode 100644 index 0000000..1e2614d --- /dev/null +++ b/public/index.html @@ -0,0 +1,69 @@ + + + + + + c4k-keycloak + + + + + + +
+
+ + +
+

+      
+ + +
+

+      
+ + +
+

+      
+ + +
+

+      
+
+ + +
+

+      
+

+ + +
+

+      
+

+ +


+
+ + +
+ +
+ + + + \ No newline at end of file diff --git a/shadow-cljs.edn b/shadow-cljs.edn new file mode 100644 index 0000000..1e3d86f --- /dev/null +++ b/shadow-cljs.edn @@ -0,0 +1,15 @@ +{:source-paths ["src/main/cljc" + "src/main/cljs" + "src/main/resources" + "src/test/cljc" + "src/test/cljs" + "src/test/resources"] + :dependencies [[org.domaindrivenarchitecture/c4k-common-cljs "0.2.8"]] + :builds {:frontend {:target :browser + :modules {:main {:init-fn dda.c4k-cloud.browser/init}} + :release {} + :compiler-options {:optimizations :advanced}} + :test {:target :node-test + :output-to "target/node-tests.js" + :autorun true + :repl-pprint true}}} \ No newline at end of file diff --git a/src/main/clj/dda/c4k_cloud/uberjar.clj b/src/main/clj/dda/c4k_cloud/uberjar.clj new file mode 100644 index 0000000..49f37b0 --- /dev/null +++ b/src/main/clj/dda/c4k_cloud/uberjar.clj @@ -0,0 +1,56 @@ +(ns dda.c4k-cloud.uberjar + (:gen-class) + (:require + [clojure.spec.alpha :as s] + [clojure.string :as cs] + [clojure.tools.reader.edn :as edn] + [expound.alpha :as expound] + [dda.c4k-cloud.core :as core])) + +(def usage + "usage: + + c4k-cloud {your configuraton file} {your authorization file}") + +(s/def ::options (s/* #{"-h"})) +(s/def ::filename (s/and string? + #(not (cs/starts-with? % "-")))) +(s/def ::cmd-args (s/cat :options ::options + :args (s/? + (s/cat :config ::filename + :auth ::filename)))) + +(defn expound-config + [config] + (expound/expound ::core/config config)) + +(defn invalid-args-msg + [spec args] + (s/explain spec args) + (println (str "Bad commandline arguments\n" usage))) + +(defn -main [& cmd-args] + (let [parsed-args-cmd (s/conform ::cmd-args cmd-args)] + (if (= ::s/invalid parsed-args-cmd) + (invalid-args-msg ::cmd-args cmd-args) + (let [{:keys [options args]} parsed-args-cmd + {:keys [config auth]} args] + (cond + (some #(= "-h" %) options) + (println usage) + :default + (let [config-str (slurp config) + auth-str (slurp auth) + config-edn (edn/read-string config-str) + auth-edn (edn/read-string auth-str) + config-valid? (s/valid? core/config? config-edn) + auth-valid? (s/valid? core/auth? auth-edn)] + (if (and config-valid? auth-valid?) + (println (core/generate config-edn auth-edn)) + (do + (when (not config-valid?) + (println + (expound/expound-str core/config? config-edn {:print-specs? false}))) + (when (not auth-valid?) + (println + (expound/expound-str core/auth? auth-edn {:print-specs? false}))))))))))) diff --git a/src/main/cljc/dda/c4k_cloud/backup.cljc b/src/main/cljc/dda/c4k_cloud/backup.cljc new file mode 100644 index 0000000..1876e10 --- /dev/null +++ b/src/main/cljc/dda/c4k_cloud/backup.cljc @@ -0,0 +1,37 @@ +(ns dda.c4k-cloud.backup + (:require + [clojure.spec.alpha :as s] + #?(:cljs [shadow.resource :as rc]) + [dda.c4k-common.yaml :as yaml] + [dda.c4k-common.base64 :as b64] + [dda.c4k-common.common :as cm])) + +(s/def ::aws-access-key-id cm/bash-env-string?) +(s/def ::aws-secret-access-key cm/bash-env-string?) +(s/def ::restic-password cm/bash-env-string?) +(s/def ::restic-repository cm/bash-env-string?) + +#?(:cljs + (defmethod yaml/load-resource :backup [resource-name] + (case resource-name + "backup/config.yaml" (rc/inline "backup/config.yaml") + "backup/cron.yaml" (rc/inline "backup/cron.yaml") + "backup/secret.yaml" (rc/inline "backup/secret.yaml") + (throw (js/Error. "Undefined Resource!"))))) + +(defn generate-config [my-conf] + (let [{:keys [restic-repository]} my-conf] + (-> + (yaml/from-string (yaml/load-resource "backup/config.yaml")) + (cm/replace-key-value :restic-repository restic-repository)))) + +(defn generate-cron [] + (yaml/from-string (yaml/load-resource "backup/cron.yaml"))) + +(defn generate-secret [my-auth] + (let [{:keys [aws-access-key-id aws-secret-access-key restic-password]} my-auth] + (-> + (yaml/from-string (yaml/load-resource "backup/secret.yaml")) + (cm/replace-key-value :aws-access-key-id (b64/encode aws-access-key-id)) + (cm/replace-key-value :aws-secret-access-key (b64/encode aws-secret-access-key)) + (cm/replace-key-value :restic-password (b64/encode restic-password))))) diff --git a/src/main/cljc/dda/c4k_cloud/cloud.cljc b/src/main/cljc/dda/c4k_cloud/cloud.cljc new file mode 100644 index 0000000..79b74f8 --- /dev/null +++ b/src/main/cljc/dda/c4k_cloud/cloud.cljc @@ -0,0 +1,56 @@ +(ns dda.c4k-cloud.cloud + (:require + [clojure.spec.alpha :as s] + #?(:cljs [shadow.resource :as rc]) + [dda.c4k-common.yaml :as yaml] + [dda.c4k-common.common :as cm])) + +(s/def ::fqdn cm/fqdn-string?) +(s/def ::issuer cm/letsencrypt-issuer?) +(s/def ::cloud-data-volume-path string?) + +#?(:cljs + (defmethod yaml/load-resource :cloud [resource-name] + (case resource-name + "cloud/certificate.yaml" (rc/inline "cloud/certificate.yaml") + "cloud/deployment.yaml" (rc/inline "cloud/deployment.yaml") + "cloud/ingress.yaml" (rc/inline "cloud/ingress.yaml") + "cloud/persistent-volume.yaml" (rc/inline "cloud/persistent-volume.yaml") + "cloud/pvc.yaml" (rc/inline "cloud/pvc.yaml") + "cloud/service.yaml" (rc/inline "cloud/service.yaml") + (throw (js/Error. "Undefined Resource!"))))) + +(defn generate-certificate [config] + (let [{:keys [fqdn issuer]} config + letsencrypt-issuer (str "letsencrypt-" (name issuer) "-issuer")] + (-> + (yaml/from-string (yaml/load-resource "cloud/certificate.yaml")) + (assoc-in [:spec :commonName] fqdn) + (assoc-in [:spec :dnsNames] [fqdn]) + (assoc-in [:spec :issuerRef :name] letsencrypt-issuer)))) + +(defn generate-deployment [config] + (let [{:keys [fqdn]} config] + (-> (yaml/from-string (yaml/load-resource "cloud/deployment.yaml")) + (cm/replace-named-value "FQDN" fqdn)))) + +(defn generate-ingress [config] + (let [{:keys [fqdn issuer] + :or {issuer :staging}} config + letsencrypt-issuer (str "letsencrypt-" (name issuer) "-issuer")] + (-> + (yaml/from-string (yaml/load-resource "cloud/ingress.yaml")) + (assoc-in [:metadata :annotations :cert-manager.io/cluster-issuer] letsencrypt-issuer) + (cm/replace-all-matching-values-by-new-value "fqdn" fqdn)))) + +(defn generate-persistent-volume [config] + (let [{:keys [cloud-data-volume-path]} config] + (-> + (yaml/from-string (yaml/load-resource "cloud/persistent-volume.yaml")) + (assoc-in [:spec :hostPath :path] cloud-data-volume-path)))) + +(defn generate-pvc [] + (yaml/from-string (yaml/load-resource "cloud/pvc.yaml"))) + +(defn generate-service [] + (yaml/from-string (yaml/load-resource "cloud/service.yaml"))) diff --git a/src/main/cljc/dda/c4k_cloud/core.cljc b/src/main/cljc/dda/c4k_cloud/core.cljc new file mode 100644 index 0000000..3397657 --- /dev/null +++ b/src/main/cljc/dda/c4k_cloud/core.cljc @@ -0,0 +1,51 @@ +(ns dda.c4k-cloud.core + (:require + [clojure.string :as cs] + [clojure.spec.alpha :as s] + #?(:clj [orchestra.core :refer [defn-spec]] + :cljs [orchestra.core :refer-macros [defn-spec]]) + [dda.c4k-common.yaml :as yaml] + [dda.c4k-common.postgres :as postgres] + [dda.c4k-cloud.cloud :as cloud] + [dda.c4k-cloud.backup :as backup])) + +(def config-defaults {:issuer :staging}) + +(def config? (s/keys :req-un [::cloud/fqdn] + :opt-un [::cloud/issuer ::cloud/cloud-data-volume-path + ::postgres/postgres-data-volume-path ::restic-repository])) + +(def auth? (s/keys :req-un [::postgres/postgres-db-user ::postgres/postgres-db-password + ::aws-access-key-id ::aws-secret-access-key + ::restic-password])) + +(defn k8s-objects [config] + (into + [] + (concat [(yaml/to-string (postgres/generate-config)) + (yaml/to-string (postgres/generate-secret config))] + (when (contains? config :postgres-data-volume-path) + [(yaml/to-string (postgres/generate-persistent-volume config))]) + [(yaml/to-string (postgres/generate-pvc)) + (yaml/to-string (postgres/generate-deployment)) + (yaml/to-string (postgres/generate-service))] + (when (contains? config :cloud-data-volume-path) + [(yaml/to-string (cloud/generate-persistent-volume config))]) + [(yaml/to-string (cloud/generate-pvc)) + (yaml/to-string (cloud/generate-deployment config)) + (yaml/to-string (cloud/generate-service)) + (yaml/to-string (cloud/generate-certificate config)) + (yaml/to-string (cloud/generate-ingress config)) + (yaml/to-string (cloud/generate-service))] + (when (contains? config :restic-repository) + [(yaml/to-string (backup/generate-config config)) + (yaml/to-string (backup/generate-secret config)) + (yaml/to-string (backup/generate-cron))])))) + +(defn-spec generate any? + [my-config config? + my-auth auth?] + (let [resulting-config (merge config-defaults my-config my-auth)] + (cs/join + "\n---\n" + (k8s-objects resulting-config)))) diff --git a/src/main/cljs/dda/c4k_cloud/browser.cljs b/src/main/cljs/dda/c4k_cloud/browser.cljs new file mode 100644 index 0000000..1380b19 --- /dev/null +++ b/src/main/cljs/dda/c4k_cloud/browser.cljs @@ -0,0 +1,61 @@ +(ns dda.c4k-cloud.browser + (:require + [clojure.tools.reader.edn :as edn] + [dda.c4k-cloud.core :as core] + [dda.c4k-cloud.cloud :as cloud] + [dda.c4k-common.browser :as br])) + +(defn config-from-document [] + (let [cloud-data-volume-path (br/get-content-from-element "cloud-data-volume-path" :optional true :deserializer keyword) + postgres-data-volume-path (br/get-content-from-element "postgres-data-volume-path" :optional true :deserializer keyword) + restic-repository (br/get-content-from-element "restic-repository" :optional true :deserializer keyword) + issuer (br/get-content-from-element "issuer" :optional true :deserializer keyword)] + (merge + {:fqdn (br/get-content-from-element "fqdn")} + (when (some? cloud-data-volume-path) + {:cloud-data-volume-path cloud-data-volume-path}) + (when (some? postgres-data-volume-path) + {:postgres-data-volume-path postgres-data-volume-path}) + (when (some? restic-repository) + {:restic-repository restic-repository}) + (when (some? issuer) + {:issuer issuer}) + ))) + +(defn validate-all! [] + (br/validate! "fqdn" ::cloud/fqdn) + (br/validate! "cloud-data-volume-path" ::cloud/cloud-data-volume-path :optional true :deserializer keyword) + (br/validate! "postgres-data-volume-path" ::cloud/cloud-data-volume-path :optional true :deserializer keyword) + (br/validate! "restic-repository" ::cloud/restic-repository :optional true :deserializer keyword) + (br/validate! "issuer" ::cloud/issuer :optional true :deserializer keyword) + (br/validate! "auth" core/auth? :deserializer edn/read-string) + (br/set-validated!)) + +(defn init [] + (-> js/document + (.getElementById "generate-button") + (.addEventListener "click" + #(do (validate-all!) + (-> (core/generate + (config-from-document) + (br/get-content-from-element "auth" :deserializer edn/read-string)) + (br/set-output!))))) + (-> (br/get-element-by-id "fqdn") + (.addEventListener "blur" + #(do (validate-all!)))) + (-> (br/get-element-by-id "cloud-data-volume-path") + (.addEventListener "blur" + #(do (validate-all!)))) + (-> (br/get-element-by-id "postgres-data-volume-path") + (.addEventListener "blur" + #(do (validate-all!)))) + (-> (br/get-element-by-id "restic-repository") + (.addEventListener "blur" + #(do (validate-all!)))) + (-> (br/get-element-by-id "issuer") + (.addEventListener "blur" + #(do (validate-all!)))) + (-> (br/get-element-by-id "auth") + (.addEventListener "blur" + #(do (validate-all!)))) + ) \ No newline at end of file diff --git a/src/main/resources/backup/backup-restore.yaml b/src/main/resources/backup/backup-restore.yaml new file mode 100644 index 0000000..bcce528 --- /dev/null +++ b/src/main/resources/backup/backup-restore.yaml @@ -0,0 +1,62 @@ +kind: Pod +apiVersion: v1 +metadata: + name: backup-restore + labels: + app.kubernetes.io/name: backup-restore + app.kubernetes.io/part-of: cloud +spec: + containers: + - name: backup-app + image: domaindrivenarchitecture/c4k-cloud-backup + imagePullPolicy: IfNotPresent + command: ["/entrypoint-start-and-wait.sh"] + env: + - name: POSTGRES_USER + valueFrom: + secretKeyRef: + name: postgres-secret + key: postgres-user + - name: POSTGRES_DB + valueFrom: + configMapKeyRef: + name: postgres-config + key: postgres-db + - name: POSTGRES_PASSWORD + valueFrom: + secretKeyRef: + name: postgres-secret + key: postgres-password + - name: POSTGRES_HOST + value: "postgresql-service:5432" + - name: POSTGRES_SERVICE + value: "postgresql-service" + - name: POSTGRES_PORT + value: "5432" + - name: AWS_DEFAULT_REGION + value: eu-central-1 + - name: AWS_ACCESS_KEY_ID_FILE + value: /var/run/secrets/backup-secrets/aws-access-key-id + - name: AWS_SECRET_ACCESS_KEY_FILE + value: /var/run/secrets/backup-secrets/aws-secret-access-key + - name: RESTIC_REPOSITORY + valueFrom: + configMapKeyRef: + name: backup-config + key: restic-repository + - name: RESTIC_PASSWORD_FILE + value: /var/run/secrets/backup-secrets/restic-password + volumeMounts: + - name: cloud-data-volume + mountPath: /var/backups + - name: backup-secret-volume + mountPath: /var/run/secrets/backup-secrets + readOnly: true + volumes: + - name: cloud-data-volume + persistentVolumeClaim: + claimName: cloud-pvc + - name: backup-secret-volume + secret: + secretName: backup-secret + restartPolicy: OnFailure \ No newline at end of file diff --git a/src/main/resources/backup/config.yaml b/src/main/resources/backup/config.yaml new file mode 100644 index 0000000..17aa35c --- /dev/null +++ b/src/main/resources/backup/config.yaml @@ -0,0 +1,9 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: backup-config + labels: + app.kubernetes.io/name: backup + app.kubernetes.io/part-of: cloud +data: + restic-repository: restic-repository \ No newline at end of file diff --git a/src/main/resources/backup/cron.yaml b/src/main/resources/backup/cron.yaml new file mode 100644 index 0000000..a914d37 --- /dev/null +++ b/src/main/resources/backup/cron.yaml @@ -0,0 +1,68 @@ +apiVersion: batch/v1beta1 +kind: CronJob +metadata: + name: cloud-backup + labels: + app.kubernetes.part-of: cloud +spec: + schedule: "10 23 * * *" + successfulJobsHistoryLimit: 1 + failedJobsHistoryLimit: 1 + jobTemplate: + spec: + template: + spec: + containers: + - name: backup-app + image: domaindrivenarchitecture/c4k-cloud-backup + imagePullPolicy: IfNotPresent + command: ["/entrypoint.sh"] + env: + - name: POSTGRES_USER + valueFrom: + secretKeyRef: + name: postgres-secret + key: postgres-user + - name: POSTGRES_PASSWORD + valueFrom: + secretKeyRef: + name: postgres-secret + key: postgres-password + - name: POSTGRES_DB + valueFrom: + configMapKeyRef: + name: postgres-config + key: postgres-db + - name: POSTGRES_HOST + value: "postgresql-service:5432" + - name: POSTGRES_SERVICE + value: "postgresql-service" + - name: POSTGRES_PORT + value: "5432" + - name: AWS_DEFAULT_REGION + value: eu-central-1 + - name: AWS_ACCESS_KEY_ID_FILE + value: /var/run/secrets/backup-secrets/aws-access-key-id + - name: AWS_SECRET_ACCESS_KEY_FILE + value: /var/run/secrets/backup-secrets/aws-secret-access-key + - name: RESTIC_REPOSITORY + valueFrom: + configMapKeyRef: + name: backup-config + key: restic-repository + - name: RESTIC_PASSWORD_FILE + value: /var/run/secrets/backup-secrets/restic-password + volumeMounts: + - name: cloud-data-volume + mountPath: /var/backups + - name: backup-secret-volume + mountPath: /var/run/secrets/backup-secrets + readOnly: true + volumes: + - name: cloud-data-volume + persistentVolumeClaim: + claimName: cloud-pvc + - name: backup-secret-volume + secret: + secretName: backup-secret + restartPolicy: OnFailure \ No newline at end of file diff --git a/src/main/resources/backup/secret.yaml b/src/main/resources/backup/secret.yaml new file mode 100644 index 0000000..c5809e0 --- /dev/null +++ b/src/main/resources/backup/secret.yaml @@ -0,0 +1,9 @@ +apiVersion: v1 +kind: Secret +metadata: + name: backup-secret +type: Opaque +data: + aws-access-key-id: aws-access-key-id + aws-secret-access-key: aws-secret-access-key + restic-password: restic-password \ No newline at end of file diff --git a/src/main/resources/cloud/certificate.yaml b/src/main/resources/cloud/certificate.yaml new file mode 100644 index 0000000..054965b --- /dev/null +++ b/src/main/resources/cloud/certificate.yaml @@ -0,0 +1,13 @@ +apiVersion: cert-manager.io/v1alpha2 +kind: Certificate +metadata: + name: cloud-cert + namespace: default +spec: + secretName: cloud-secret + commonName: fqdn + dnsNames: + - fqdn + issuerRef: + name: letsencrypt-staging-issuer + kind: ClusterIssuer \ No newline at end of file diff --git a/src/main/resources/cloud/deployment.yaml b/src/main/resources/cloud/deployment.yaml new file mode 100644 index 0000000..bc719c9 --- /dev/null +++ b/src/main/resources/cloud/deployment.yaml @@ -0,0 +1,40 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: cloud +spec: + selector: + matchLabels: + app: cloud + strategy: + type: Recreate + template: + metadata: + labels: + app: cloud + spec: + containers: + - image: domaindrivenarchitecture/c4k-cloud + name: cloud-app + imagePullPolicy: IfNotPresent + env: + - name: DB_USERNAME_FILE + value: /var/run/secrets/postgres-secret/postgres-user + - name: DB_PASSWORD_FILE + value: /var/run/secrets/postgres-secret/postgres-password + - name: FQDN + value: fqdn + command: ["/app/entrypoint.sh"] + volumeMounts: + - mountPath: /var/cloud + name: cloud-data-volume + - name: postgres-secret-volume + mountPath: /var/run/secrets/postgres-secret + readOnly: true + volumes: + - name: cloud-data-volume + persistentVolumeClaim: + claimName: cloud-pvc + - name: postgres-secret-volume + secret: + secretName: postgres-secret diff --git a/src/main/resources/cloud/ingress.yaml b/src/main/resources/cloud/ingress.yaml new file mode 100644 index 0000000..f206da2 --- /dev/null +++ b/src/main/resources/cloud/ingress.yaml @@ -0,0 +1,26 @@ +apiVersion: extensions/v1beta1 +kind: Ingress +metadata: + name: ingress-cloud + annotations: + cert-manager.io/cluster-issuer: letsencrypt-staging-issuer + nginx.ingress.kubernetes.io/proxy-body-size: "256m" + nginx.ingress.kubernetes.io/ssl-redirect: "true" + nginx.ingress.kubernetes.io/rewrite-target: / + nginx.ingress.kubernetes.io/proxy-connect-timeout: "300" + nginx.ingress.kubernetes.io/proxy-send-timeout: "300" + nginx.ingress.kubernetes.io/proxy-read-timeout: "300" + namespace: default +spec: + tls: + - hosts: + - fqdn + secretName: cloud-secret + rules: + - host: fqdn + http: + paths: + - path: / + backend: + serviceName: cloud-service + servicePort: 8080 diff --git a/src/main/resources/cloud/persistent-volume.yaml b/src/main/resources/cloud/persistent-volume.yaml new file mode 100644 index 0000000..f39a2ec --- /dev/null +++ b/src/main/resources/cloud/persistent-volume.yaml @@ -0,0 +1,14 @@ +kind: PersistentVolume +apiVersion: v1 +metadata: + name: cloud-pv-volume + labels: + type: local +spec: + storageClassName: manual + accessModes: + - ReadWriteOnce + capacity: + storage: 30Gi + hostPath: + path: "/var/cloud" diff --git a/src/main/resources/cloud/pvc.yaml b/src/main/resources/cloud/pvc.yaml new file mode 100644 index 0000000..3285ef6 --- /dev/null +++ b/src/main/resources/cloud/pvc.yaml @@ -0,0 +1,13 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: cloud-pvc + labels: + app: cloud +spec: + storageClassName: manual + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 30Gi \ No newline at end of file diff --git a/src/main/resources/cloud/service.yaml b/src/main/resources/cloud/service.yaml new file mode 100644 index 0000000..2c05a15 --- /dev/null +++ b/src/main/resources/cloud/service.yaml @@ -0,0 +1,9 @@ +apiVersion: v1 +kind: Service +metadata: + name: cloud-service +spec: + selector: + app: cloud + ports: + - port: 8080 diff --git a/src/main/resources/logback.xml b/src/main/resources/logback.xml new file mode 100644 index 0000000..8985f2b --- /dev/null +++ b/src/main/resources/logback.xml @@ -0,0 +1,50 @@ + + + + %d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n + + + INFO + + + + + logs/pallet.log + + logs/old/pallet.%d{yyyy-MM-dd}.log + 3 + + + %date %level [%thread] %logger{10} %msg%n + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/test/cljc/dda/c4k_cloud/backup_test.cljc b/src/test/cljc/dda/c4k_cloud/backup_test.cljc new file mode 100644 index 0000000..e9eec96 --- /dev/null +++ b/src/test/cljc/dda/c4k_cloud/backup_test.cljc @@ -0,0 +1,93 @@ +(ns dda.c4k-cloud.backup-test + (:require + #?(:clj [clojure.test :refer [deftest is are testing run-tests]] + :cljs [cljs.test :refer-macros [deftest is are testing run-tests]]) + [dda.c4k-cloud.backup :as cut])) + + +(deftest should-generate-secret + (is (= {:apiVersion "v1" + :kind "Secret" + :metadata {:name "backup-secret"} + :type "Opaque" + :data + {:aws-access-key-id "YXdzLWlk", :aws-secret-access-key "YXdzLXNlY3JldA==", :restic-password "cmVzdGljLXB3"}} + (cut/generate-secret {:aws-access-key-id "aws-id" :aws-secret-access-key "aws-secret" :restic-password "restic-pw"})))) + +(deftest should-generate-config + (is (= {:apiVersion "v1" + :kind "ConfigMap" + :metadata {:name "backup-config" + :labels {:app.kubernetes.io/name "backup" + :app.kubernetes.io/part-of "cloud"}} + :data + {:restic-repository "s3:restic-repository"}} + (cut/generate-config {:restic-repository "s3:restic-repository"})))) + +(deftest should-generate-cron + (is (= {:apiVersion "batch/v1beta1" + :kind "CronJob" + :metadata {:name "cloud-backup" + :labels {:app.kubernetes.part-of "cloud"}} + :spec {:schedule "10 23 * * *" + :successfulJobsHistoryLimit 1 + :failedJobsHistoryLimit 1 + :jobTemplate + {:spec + {:template + {:spec + {:containers + [{:name "backup-app" + :image "domaindrivenarchitecture/c4k-cloud-backup" + :imagePullPolicy "IfNotPresent" + :command ["/entrypoint.sh"] + :env + [{:name "POSTGRES_USER" + :valueFrom + {:secretKeyRef + {:name "postgres-secret" + :key "postgres-user"}}} + {:name "POSTGRES_PASSWORD" + :valueFrom + {:secretKeyRef + {:name "postgres-secret" + :key "postgres-password"}}} + {:name "POSTGRES_DB" + :valueFrom + {:configMapKeyRef + {:name "postgres-config" + :key "postgres-db"}}} + {:name "POSTGRES_HOST" + :value "postgresql-service:5432"} + {:name "POSTGRES_SERVICE" + :value "postgresql-service"} + {:name "POSTGRES_PORT" + :value "5432"} + {:name "AWS_DEFAULT_REGION" + :value "eu-central-1"} + {:name "AWS_ACCESS_KEY_ID_FILE" + :value "/var/run/secrets/backup-secrets/aws-access-key-id"} + {:name "AWS_SECRET_ACCESS_KEY_FILE" + :value "/var/run/secrets/backup-secrets/aws-secret-access-key"} + {:name "RESTIC_REPOSITORY" + :valueFrom + {:configMapKeyRef + {:name "backup-config" + :key "restic-repository"}}} + {:name "RESTIC_PASSWORD_FILE" + :value "/var/run/secrets/backup-secrets/restic-password"}] + :volumeMounts + [{:name "cloud-data-volume" + :mountPath "/var/backups"} + {:name "backup-secret-volume" + :mountPath "/var/run/secrets/backup-secrets" + :readOnly true}]}] + :volumes + [{:name "cloud-data-volume" + :persistentVolumeClaim + {:claimName "cloud-pvc"}} + {:name "backup-secret-volume" + :secret + {:secretName "backup-secret"}}] + :restartPolicy "OnFailure"}}}}}} + (cut/generate-cron)))) diff --git a/src/test/cljc/dda/c4k_cloud/cloud_test.cljc b/src/test/cljc/dda/c4k_cloud/cloud_test.cljc new file mode 100644 index 0000000..b29c541 --- /dev/null +++ b/src/test/cljc/dda/c4k_cloud/cloud_test.cljc @@ -0,0 +1,80 @@ +(ns dda.c4k-cloud.cloud-test + (:require + #?(:clj [clojure.test :refer [deftest is are testing run-tests]] + :cljs [cljs.test :refer-macros [deftest is are testing run-tests]]) + [dda.c4k-cloud.cloud :as cut])) + +(deftest should-generate-certificate + (is (= {:apiVersion "cert-manager.io/v1alpha2" + :kind "Certificate" + :metadata {:name "cloud-cert", :namespace "default"} + :spec + {:secretName "cloud-secret" + :commonName "xx" + :dnsNames ["xx"] + :issuerRef + {:name "letsencrypt-prod-issuer", :kind "ClusterIssuer"}}} + (cut/generate-certificate {:fqdn "xx" :issuer :prod})))) + +(deftest should-generate-ingress + (is (= {:apiVersion "extensions/v1beta1" + :kind "Ingress" + :metadata + {:name "ingress-cloud" + :annotations + {:cert-manager.io/cluster-issuer + "letsencrypt-staging-issuer" + :nginx.ingress.kubernetes.io/proxy-body-size "256m" + :nginx.ingress.kubernetes.io/ssl-redirect "true" + :nginx.ingress.kubernetes.io/rewrite-target "/" + :nginx.ingress.kubernetes.io/proxy-connect-timeout "300" + :nginx.ingress.kubernetes.io/proxy-send-timeout "300" + :nginx.ingress.kubernetes.io/proxy-read-timeout "300"} + :namespace "default"} + :spec + {:tls [{:hosts ["xx"], :secretName "cloud-secret"}] + :rules + [{:host "xx" + :http + {:paths + [{:path "/" + :backend + {:serviceName "cloud-service", :servicePort 8080}}]}}]}} + (cut/generate-ingress {:fqdn "xx"})))) + +(deftest should-generate-persistent-volume + (is (= {:kind "PersistentVolume" + :apiVersion "v1" + :metadata {:name "cloud-pv-volume", :labels {:type "local"}} + :spec + {:storageClassName "manual" + :accessModes ["ReadWriteOnce"] + :capacity {:storage "30Gi"} + :hostPath {:path "xx"}}} + (cut/generate-persistent-volume {:cloud-data-volume-path "xx"})))) + +(deftest should-generate-deployment + (is (= {:containers + [{:image "domaindrivenarchitecture/c4k-cloud" + :name "cloud-app" + :imagePullPolicy "IfNotPresent" + :env + [{:name "DB_USERNAME_FILE" + :value + "/var/run/secrets/postgres-secret/postgres-user"} + {:name "DB_PASSWORD_FILE" + :value + "/var/run/secrets/postgres-secret/postgres-password"} + {:name "FQDN", :value "xx"}] + :command ["/app/entrypoint.sh"] + :volumeMounts + [{:mountPath "/var/cloud", :name "cloud-data-volume"} + {:name "postgres-secret-volume" + :mountPath "/var/run/secrets/postgres-secret" + :readOnly true}]}] + :volumes + [{:name "cloud-data-volume" + :persistentVolumeClaim {:claimName "cloud-pvc"}} + {:name "postgres-secret-volume" + :secret {:secretName "postgres-secret"}}]} + (get-in (cut/generate-deployment {:fqdn "xx"}) [:spec :template :spec])))) diff --git a/src/test/cljc/dda/c4k_cloud/core_test.cljc b/src/test/cljc/dda/c4k_cloud/core_test.cljc new file mode 100644 index 0000000..2fd1634 --- /dev/null +++ b/src/test/cljc/dda/c4k_cloud/core_test.cljc @@ -0,0 +1,35 @@ +(ns dda.c4k-cloud.core-test + (:require + #?(:clj [clojure.test :refer [deftest is are testing run-tests]] + :cljs [cljs.test :refer-macros [deftest is are testing run-tests]]) + [dda.c4k-cloud.core :as cut])) + +(deftest should-k8s-objects + (is (= 16 + (count (cut/k8s-objects {:fqdn "cloud-neu.prod.meissa-gmbh.de" + :postgres-db-user "cloud" + :postgres-db-password "cloud-db-password" + :issuer :prod + :cloud-data-volume-path "/var/cloud" + :postgres-data-volume-path "/var/postgres" + :aws-access-key-id "aws-id" + :aws-secret-access-key "aws-secret" + :restic-password "restic-pw" + :restic-repository "restic-repository"})))) + (is (= 14 + (count (cut/k8s-objects {:fqdn "cloud-neu.prod.meissa-gmbh.de" + :postgres-db-user "cloud" + :postgres-db-password "cloud-db-password" + :issuer :prod + :aws-access-key-id "aws-id" + :aws-secret-access-key "aws-secret" + :restic-password "restic-pw" + :restic-repository "restic-repository"})))) + (is (= 11 + (count (cut/k8s-objects {:fqdn "cloud-neu.prod.meissa-gmbh.de" + :postgres-db-user "cloud" + :postgres-db-password "cloud-db-password" + :issuer :prod + :aws-access-key-id "aws-id" + :aws-secret-access-key "aws-secret" + :restic-password "restic-pw"}))))) diff --git a/valid-auth.edn b/valid-auth.edn new file mode 100644 index 0000000..42ac8a7 --- /dev/null +++ b/valid-auth.edn @@ -0,0 +1,5 @@ +{:postgres-db-user "cloud" + :postgres-db-password "cloud-db-password" + :aws-access-key-id "aws-id" + :aws-secret-access-key "aws-secret" + :restic-password "restic-password"} \ No newline at end of file diff --git a/valid-config.edn b/valid-config.edn new file mode 100644 index 0000000..49a40d8 --- /dev/null +++ b/valid-config.edn @@ -0,0 +1,4 @@ +{:fqdn "cloud-neu.prod.meissa-gmbh.de" + :cloud-data-volume-path "/var/cloud" + :postgres-data-volume-path "/var/postgres" + :restic-repository "restic-repository"} \ No newline at end of file From 6732be46a5d6f0461feac6c7cc10a6a76044ab7b Mon Sep 17 00:00:00 2001 From: leo Date: Fri, 6 Aug 2021 17:38:56 +0200 Subject: [PATCH 02/12] fixed typo --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 6b16f0d..e32ed90 100644 --- a/README.md +++ b/README.md @@ -9,8 +9,8 @@ c4k-nextcloud provides a k8s deployment for nextcloud containing: * adjusted nextcloud docker image * nextcloud -??? * ingress having a letsencrypt managed certificate -??? * postgres database +* ingress having a letsencrypt managed certificate +* postgres database The package aims to a low load sceanrio. @@ -23,7 +23,7 @@ This is under development. 1) Scale Nextcloud deployment down: kubectl scale deployment nextcloud --replicas=0 -2)apply backup and restore pod: +2) apply backup and restore pod: kubectl apply -f src/main/resources/backup/backup-restore.yaml 3) exec into pod and execute restore pod From 1c2f6b89f20a60ce7c3c8f5f19c63ccdab22c563 Mon Sep 17 00:00:00 2001 From: leo Date: Tue, 10 Aug 2021 00:41:53 +0200 Subject: [PATCH 03/12] added initial cloud files to c4k-nextcloud --- README.md | 42 +--- infrastructure/docker-backup/build.py | 22 +- .../docker-backup/image/resources/backup.sh | 11 +- .../image/resources/end-maintenance.sh | 10 + .../resources/entrypoint-start-and-wait.sh | 2 +- .../image/resources/entrypoint.sh | 6 +- .../docker-backup/image/resources/init.sh | 2 +- .../docker-backup/image/resources/install.sh | 3 +- .../image/resources/restic-snapshots.sh | 17 -- .../docker-backup/image/resources/restore.sh | 18 +- .../image/resources/start-maintenance.sh | 11 + infrastructure/docker-backup/test/Dockerfile | 8 +- .../docker-backup/test/serverspec.edn | 3 +- infrastructure/docker-cloud/image/Dockerfile | 10 - .../docker-cloud/image/resources/dbconfig.xml | 25 --- .../image/resources/entrypoint.sh | 44 ---- .../docker-cloud/image/resources/install.sh | 30 --- .../image/resources/logging.properties | 69 ------- .../docker-cloud/image/resources/server.xml | 114 ----------- .../docker-cloud/image/resources/setenv.sh | 121 ----------- .../docker-cloud/image/resources/user.sh | 5 - infrastructure/docker-cloud/test/Dockerfile | 8 - .../docker-cloud/test/serverspec.edn | 3 - .../build.py | 22 +- .../docker-nextcloud/image/Dockerfile | 8 + .../image/resources/entrypoint.sh | 191 ++++++++++++++++++ .../image/resources/install-debug.sh | 3 + .../image/resources/install.sh | 9 + .../image/resources/memory-limit.ini | 2 + .../image/resources/upload-max-limit.ini | 3 + .../docker-nextcloud/test/Dockerfile | 11 + .../docker-nextcloud/test/serverspec.edn | 2 + main/resources/backup/backup-restore.yaml | 59 ++++++ main/resources/backup/config.yaml | 9 + main/resources/backup/configure-as-user.sh | 9 + main/resources/backup/cron.yaml | 65 ++++++ main/resources/backup/secret.yaml | 9 + main/resources/cloud/certificate.yaml | 13 ++ main/resources/cloud/cloud-pod.yml.template | 45 +++++ main/resources/cloud/configure-as-user.sh | 22 ++ main/resources/cloud/ingress.yaml | 26 +++ .../cloud/install-as-root.sh.template | 4 + main/resources/cloud/persistent-volume.yaml | 15 ++ main/resources/cloud/pod-running.sh | 25 +++ main/resources/cloud/pvc.yaml | 16 ++ main/resources/cloud/secret.yaml | 11 + main/resources/cloud/service.yaml | 9 + main/resources/cloud/verify.sh.template | 15 ++ main/resources/postgres/configure-as-user.sh | 15 ++ .../postgres/install-as-root.sh.template | 4 + main/resources/postgres/postgres-config.yml | 10 + .../postgres/postgres-deployment.yml.template | 49 +++++ .../postgres/postgres-persistent-volume.yml | 15 ++ main/resources/postgres/postgres-pvc.yml | 16 ++ .../postgres/postgres-secret.yml.template | 9 + main/resources/postgres/postgres-service.yml | 10 + main/resources/postgres/verify.sh | 8 + main/src/meissa/pallet/meissa_cloud/app.clj | 61 ++++++ .../meissa/pallet/meissa_cloud/convention.clj | 93 +++++++++ .../pallet/meissa_cloud/convention/bash.clj | 10 + .../meissa_cloud/convention/bash_php.clj | 11 + main/src/meissa/pallet/meissa_cloud/infra.clj | 51 +++++ .../pallet/meissa_cloud/infra/backup.clj | 39 ++++ .../pallet/meissa_cloud/infra/cloud.clj | 57 ++++++ .../pallet/meissa_cloud/infra/postgres.clj | 47 +++++ project.clj | 86 ++++---- targets.edn | 3 + .../should_generate_infra_for_convention.edn | 30 +++ .../should_generate_k8s_convention.edn | 30 +++ .../should_validate_input.0.edn | 14 ++ .../should_validate_input.1.edn | 14 ++ .../should_validate_input.2.edn | 14 ++ .../meissa/pallet/meissa_cloud/app_test.clj | 31 +++ .../meissa_cloud/convention/bash_php_test.clj | 20 ++ .../meissa_cloud/convention/bash_test.clj | 22 ++ .../pallet/meissa_cloud/convention_test.clj | 18 ++ uberjar/resources/localhost-target.edn | 2 + uberjar/resources/logback.xml | 50 +++++ .../src/meissa/pallet/meissa_cloud/main.clj | 57 ++++++ 79 files changed, 1512 insertions(+), 571 deletions(-) create mode 100644 infrastructure/docker-backup/image/resources/end-maintenance.sh delete mode 100755 infrastructure/docker-backup/image/resources/restic-snapshots.sh create mode 100644 infrastructure/docker-backup/image/resources/start-maintenance.sh delete mode 100644 infrastructure/docker-cloud/image/Dockerfile delete mode 100644 infrastructure/docker-cloud/image/resources/dbconfig.xml delete mode 100644 infrastructure/docker-cloud/image/resources/entrypoint.sh delete mode 100755 infrastructure/docker-cloud/image/resources/install.sh delete mode 100644 infrastructure/docker-cloud/image/resources/logging.properties delete mode 100644 infrastructure/docker-cloud/image/resources/server.xml delete mode 100644 infrastructure/docker-cloud/image/resources/setenv.sh delete mode 100644 infrastructure/docker-cloud/image/resources/user.sh delete mode 100644 infrastructure/docker-cloud/test/Dockerfile delete mode 100644 infrastructure/docker-cloud/test/serverspec.edn rename infrastructure/{docker-cloud => docker-nextcloud}/build.py (83%) create mode 100644 infrastructure/docker-nextcloud/image/Dockerfile create mode 100644 infrastructure/docker-nextcloud/image/resources/entrypoint.sh create mode 100644 infrastructure/docker-nextcloud/image/resources/install-debug.sh create mode 100755 infrastructure/docker-nextcloud/image/resources/install.sh create mode 100644 infrastructure/docker-nextcloud/image/resources/memory-limit.ini create mode 100644 infrastructure/docker-nextcloud/image/resources/upload-max-limit.ini create mode 100644 infrastructure/docker-nextcloud/test/Dockerfile create mode 100644 infrastructure/docker-nextcloud/test/serverspec.edn create mode 100644 main/resources/backup/backup-restore.yaml create mode 100644 main/resources/backup/config.yaml create mode 100644 main/resources/backup/configure-as-user.sh create mode 100644 main/resources/backup/cron.yaml create mode 100644 main/resources/backup/secret.yaml create mode 100644 main/resources/cloud/certificate.yaml create mode 100644 main/resources/cloud/cloud-pod.yml.template create mode 100644 main/resources/cloud/configure-as-user.sh create mode 100644 main/resources/cloud/ingress.yaml create mode 100644 main/resources/cloud/install-as-root.sh.template create mode 100644 main/resources/cloud/persistent-volume.yaml create mode 100755 main/resources/cloud/pod-running.sh create mode 100644 main/resources/cloud/pvc.yaml create mode 100644 main/resources/cloud/secret.yaml create mode 100644 main/resources/cloud/service.yaml create mode 100644 main/resources/cloud/verify.sh.template create mode 100644 main/resources/postgres/configure-as-user.sh create mode 100644 main/resources/postgres/install-as-root.sh.template create mode 100644 main/resources/postgres/postgres-config.yml create mode 100644 main/resources/postgres/postgres-deployment.yml.template create mode 100644 main/resources/postgres/postgres-persistent-volume.yml create mode 100644 main/resources/postgres/postgres-pvc.yml create mode 100644 main/resources/postgres/postgres-secret.yml.template create mode 100644 main/resources/postgres/postgres-service.yml create mode 100644 main/resources/postgres/verify.sh create mode 100644 main/src/meissa/pallet/meissa_cloud/app.clj create mode 100644 main/src/meissa/pallet/meissa_cloud/convention.clj create mode 100644 main/src/meissa/pallet/meissa_cloud/convention/bash.clj create mode 100644 main/src/meissa/pallet/meissa_cloud/convention/bash_php.clj create mode 100644 main/src/meissa/pallet/meissa_cloud/infra.clj create mode 100644 main/src/meissa/pallet/meissa_cloud/infra/backup.clj create mode 100644 main/src/meissa/pallet/meissa_cloud/infra/cloud.clj create mode 100644 main/src/meissa/pallet/meissa_cloud/infra/postgres.clj create mode 100644 targets.edn create mode 100644 test/resources/meissa/pallet/meissa_cloud/convention_test/should_generate_infra_for_convention.edn create mode 100644 test/resources/meissa/pallet/meissa_cloud/convention_test/should_generate_k8s_convention.edn create mode 100644 test/resources/meissa/pallet/meissa_cloud/convention_test/should_validate_input.0.edn create mode 100644 test/resources/meissa/pallet/meissa_cloud/convention_test/should_validate_input.1.edn create mode 100644 test/resources/meissa/pallet/meissa_cloud/convention_test/should_validate_input.2.edn create mode 100644 test/src/meissa/pallet/meissa_cloud/app_test.clj create mode 100644 test/src/meissa/pallet/meissa_cloud/convention/bash_php_test.clj create mode 100644 test/src/meissa/pallet/meissa_cloud/convention/bash_test.clj create mode 100644 test/src/meissa/pallet/meissa_cloud/convention_test.clj create mode 100644 uberjar/resources/localhost-target.edn create mode 100644 uberjar/resources/logback.xml create mode 100644 uberjar/src/meissa/pallet/meissa_cloud/main.clj diff --git a/README.md b/README.md index e32ed90..9af9e4c 100644 --- a/README.md +++ b/README.md @@ -1,41 +1,5 @@ -# convention 4 kubernetes: c4k-nextcloud +# meissa-cloud -[![Clojars Project](https://img.shields.io/clojars/v/org.domaindrivenarchitecture/c4k-nextcloud.svg)](https://clojars.org/org.domaindrivenarchitecture/c4k-nextcloud) [![pipeline status](https://gitlab.com/domaindrivenarchitecture/c4k-nextcloud/badges/master/pipeline.svg)](https://gitlab.com/domaindrivenarchitecture/c4k-nextcloud/-/commits/master) +# backup manuell triggern -[DeltaChat chat over e-mail](mailto:buero@meissa-gmbh.de?subject=community-chat) | [team@social.meissa-gmbh.de team@social.meissa-gmbh.de](https://social.meissa-gmbh.de/@team) | [Website & Blog](https://domaindrivenarchitecture.org) - -## Purpose - -c4k-nextcloud provides a k8s deployment for nextcloud containing: -* adjusted nextcloud docker image -* nextcloud -* ingress having a letsencrypt managed certificate -* postgres database - -The package aims to a low load sceanrio. - -## Status - -This is under development. - -## Manual restore - -1) Scale Nextcloud deployment down: -kubectl scale deployment nextcloud --replicas=0 - -2) apply backup and restore pod: -kubectl apply -f src/main/resources/backup/backup-restore.yaml - -3) exec into pod and execute restore pod -kubectl exec -it backup-restore -- /usr/local/bin/restore.sh - -4) Scale Nextcloud deployment up: -kubectl scale deployment nextcloud --replicas=1 - -5) Update index of Nextcloud: -Nextcloud > Settings > System > Advanced > Indexing -## License - -Copyright © 2021 meissa GmbH -Licensed under the [Apache License, Version 2.0](LICENSE) (the "License") -Pls. find licenses of our subcomponents [here](doc/SUBCOMPONENT_LICENSE) \ No newline at end of file +# restore manuell triggern \ No newline at end of file diff --git a/infrastructure/docker-backup/build.py b/infrastructure/docker-backup/build.py index 68ac1ae..48d702a 100644 --- a/infrastructure/docker-backup/build.py +++ b/infrastructure/docker-backup/build.py @@ -3,28 +3,26 @@ from pybuilder.core import task, init from ddadevops import * import logging -name = 'c4k-cloud-backup' +name = 'meissa-cloud-backup' MODULE = 'docker' PROJECT_ROOT_PATH = '../..' + class MyBuild(DevopsDockerBuild): pass @init def initialize(project): - project.build_depends_on('ddadevops>=0.12.4') - stage = 'prod' + project.build_depends_on('ddadevops>=0.6.1') + stage = 'notused' dockerhub_user = environ.get('DOCKERHUB_USER') if not dockerhub_user: dockerhub_user = gopass_field_from_path('meissa/web/docker.com', 'login') dockerhub_password = environ.get('DOCKERHUB_PASSWORD') if not dockerhub_password: dockerhub_password = gopass_password_from_path('meissa/web/docker.com') - tag = environ.get('CI_COMMIT_TAG') - if not tag: - tag = get_tag_from_latest_commit() config = create_devops_docker_build_config( - stage, PROJECT_ROOT_PATH, MODULE, dockerhub_user, dockerhub_password, docker_publish_tag=tag) + stage, PROJECT_ROOT_PATH, MODULE, dockerhub_user, dockerhub_password) build = MyBuild(project, config) build.initialize_build_dir() @@ -40,12 +38,12 @@ def drun(project): build.drun() @task -def publish(project): +def test(project): build = get_devops_build(project) - build.dockerhub_login() - build.dockerhub_publish() + build.test() @task -def test(project): +def publish(project): build = get_devops_build(project) - build.test() + build.dockerhub_login() + build.dockerhub_publish() diff --git a/infrastructure/docker-backup/image/resources/backup.sh b/infrastructure/docker-backup/image/resources/backup.sh index d01affd..c28cf98 100755 --- a/infrastructure/docker-backup/image/resources/backup.sh +++ b/infrastructure/docker-backup/image/resources/backup.sh @@ -3,6 +3,9 @@ set -o pipefail function main() { + + start-maintenance.sh + file_env AWS_ACCESS_KEY_ID file_env AWS_SECRET_ACCESS_KEY file_env POSTGRES_DB @@ -10,13 +13,15 @@ function main() { file_env POSTGRES_USER file_env RESTIC_DAYS_TO_KEEP 14 - backup-roles "" + backup-roles 'oc_' backup-db-dump - backup-fs-from-directory '/var/backups/' 'data/' + backup-directory '/var/backups/' + + end-maintenance.sh } source /usr/local/lib/functions.sh -source /usr/local/lib/file-functions.sh source /usr/local/lib/pg-functions.sh +source /usr/local/lib/file-functions.sh main diff --git a/infrastructure/docker-backup/image/resources/end-maintenance.sh b/infrastructure/docker-backup/image/resources/end-maintenance.sh new file mode 100644 index 0000000..0013509 --- /dev/null +++ b/infrastructure/docker-backup/image/resources/end-maintenance.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +if test -f "/var/backups/config/config.orig"; then + + rm /var/backups/config/config.php + mv /var/backups/config/config.orig /var/backups/config/config.php + chown www-data:root /var/backups/config/config.php + touch /var/backups/config/config.php + +fi \ No newline at end of file diff --git a/infrastructure/docker-backup/image/resources/entrypoint-start-and-wait.sh b/infrastructure/docker-backup/image/resources/entrypoint-start-and-wait.sh index 67a8f3b..60075b5 100644 --- a/infrastructure/docker-backup/image/resources/entrypoint-start-and-wait.sh +++ b/infrastructure/docker-backup/image/resources/entrypoint-start-and-wait.sh @@ -4,7 +4,7 @@ function main() { file_env POSTGRES_DB file_env POSTGRES_PASSWORD file_env POSTGRES_USER - + create-pg-pass while true; do diff --git a/infrastructure/docker-backup/image/resources/entrypoint.sh b/infrastructure/docker-backup/image/resources/entrypoint.sh index 924f24c..480c971 100755 --- a/infrastructure/docker-backup/image/resources/entrypoint.sh +++ b/infrastructure/docker-backup/image/resources/entrypoint.sh @@ -1,13 +1,13 @@ #!/bin/bash function main() { - file_env POSTGRES_DB + file_env POSTGRES_DB file_env POSTGRES_PASSWORD file_env POSTGRES_USER - + create-pg-pass - /usr/local/bin/backup.sh + /usr/local/bin/backup.sh } source /usr/local/lib/functions.sh diff --git a/infrastructure/docker-backup/image/resources/init.sh b/infrastructure/docker-backup/image/resources/init.sh index 5693dcb..5767e69 100755 --- a/infrastructure/docker-backup/image/resources/init.sh +++ b/infrastructure/docker-backup/image/resources/init.sh @@ -10,6 +10,6 @@ function main() { } source /usr/local/lib/functions.sh -source /usr/local/lib/file-functions.sh source /usr/local/lib/pg-functions.sh +source /usr/local/lib/file-functions.sh main diff --git a/infrastructure/docker-backup/image/resources/install.sh b/infrastructure/docker-backup/image/resources/install.sh index 1a8cbd7..7c58fce 100755 --- a/infrastructure/docker-backup/image/resources/install.sh +++ b/infrastructure/docker-backup/image/resources/install.sh @@ -8,4 +8,5 @@ install -m 0700 /tmp/entrypoint-start-and-wait.sh / install -m 0700 /tmp/init.sh /usr/local/bin/ install -m 0700 /tmp/backup.sh /usr/local/bin/ install -m 0700 /tmp/restore.sh /usr/local/bin/ -install -m 0700 /tmp/restic-snapshots.sh /usr/local/bin/ +install -m 0700 /tmp/start-maintenance.sh /usr/local/bin/ +install -m 0700 /tmp/end-maintenance.sh /usr/local/bin/ diff --git a/infrastructure/docker-backup/image/resources/restic-snapshots.sh b/infrastructure/docker-backup/image/resources/restic-snapshots.sh deleted file mode 100755 index a428fe2..0000000 --- a/infrastructure/docker-backup/image/resources/restic-snapshots.sh +++ /dev/null @@ -1,17 +0,0 @@ -#!/bin/bash - -set -o pipefail - -function main() { - file_env AWS_ACCESS_KEY_ID - file_env AWS_SECRET_ACCESS_KEY - - restic -r ${RESTIC_REPOSITORY}/pg-role snapshots - restic -r ${RESTIC_REPOSITORY}/pg-database snapshots - restic -r ${RESTIC_REPOSITORY}/files snapshots -} - -source /usr/local/lib/functions.sh -source /usr/local/lib/file-functions.sh - -main diff --git a/infrastructure/docker-backup/image/resources/restore.sh b/infrastructure/docker-backup/image/resources/restore.sh index ba1e091..1ebef16 100755 --- a/infrastructure/docker-backup/image/resources/restore.sh +++ b/infrastructure/docker-backup/image/resources/restore.sh @@ -4,6 +4,8 @@ set -Eeo pipefail function main() { + start-maintenance.sh + file_env AWS_ACCESS_KEY_ID file_env AWS_SECRET_ACCESS_KEY @@ -11,26 +13,16 @@ function main() { file_env POSTGRES_PASSWORD file_env POSTGRES_USER - # Im Nextcloud pod: /opt/atlassian-cloud-software-standalone/bin/stop-cloud.sh - - # Restore latest snapshot into /var/backups/restic-restore - rm -rf /var/backups/restic-restore - restore-directory '/var/backups/restic-restore' - - # Restore data dir backup - rm -rf /var/backups/data/* - cp -a /var/backups/restic-restore/data/* /var/backups/data - - # Restore db drop-create-db + restore-roles restore-db + restore-directory '/var/backups/' - # /opt/atlassian-cloud-software-standalone/bin/start-cloud.sh } source /usr/local/lib/functions.sh -source /usr/local/lib/file-functions.sh source /usr/local/lib/pg-functions.sh +source /usr/local/lib/file-functions.sh main diff --git a/infrastructure/docker-backup/image/resources/start-maintenance.sh b/infrastructure/docker-backup/image/resources/start-maintenance.sh new file mode 100644 index 0000000..474ede1 --- /dev/null +++ b/infrastructure/docker-backup/image/resources/start-maintenance.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +if [ ! -f "/var/backups/config/config.orig" ]; then + + rm -f /var/backups/config/config.orig + cp /var/backups/config/config.php /var/backups/config/config.orig + sed -i "s/);/ \'maintenance\' => true,\n);/g" /var/backups/config/config.php + chown www-data:root /var/backups/config/config.php + touch /var/backups/config/config.php + +fi diff --git a/infrastructure/docker-backup/test/Dockerfile b/infrastructure/docker-backup/test/Dockerfile index a18ebee..eb92122 100644 --- a/infrastructure/docker-backup/test/Dockerfile +++ b/infrastructure/docker-backup/test/Dockerfile @@ -1,7 +1,9 @@ -FROM c4k-cloud-backup +FROM meissa-cloud-backup -RUN curl -L -o /tmp/serverspec.jar \ - https://github.com/DomainDrivenArchitecture/dda-serverspec-crate/releases/download/2.0.0/dda-serverspec-standalone.jar +RUN apt update +RUN apt -yqq --no-install-recommends --yes install curl default-jre-headless + +RUN curl -L -o /tmp/serverspec.jar https://github.com/DomainDrivenArchitecture/dda-serverspec-crate/releases/download/2.0.0/dda-serverspec-standalone.jar COPY serverspec.edn /tmp/serverspec.edn diff --git a/infrastructure/docker-backup/test/serverspec.edn b/infrastructure/docker-backup/test/serverspec.edn index 09623c7..bc4277f 100644 --- a/infrastructure/docker-backup/test/serverspec.edn +++ b/infrastructure/docker-backup/test/serverspec.edn @@ -1,6 +1,7 @@ {:file [{:path "/usr/local/bin/init.sh" :mod "700"} {:path "/usr/local/bin/backup.sh" :mod "700"} {:path "/usr/local/bin/restore.sh" :mod "700"} - {:path "/usr/local/bin/restic-snapshots.sh" :mod "700"} + {:path "/usr/local/bin/start-maintenance.sh" :mod "700"} + {:path "/usr/local/bin/end-maintenance.sh" :mod "700"} {:path "/entrypoint.sh" :mod "700"} {:path "/entrypoint-start-and-wait.sh" :mod "700"}]} diff --git a/infrastructure/docker-cloud/image/Dockerfile b/infrastructure/docker-cloud/image/Dockerfile deleted file mode 100644 index 9551640..0000000 --- a/infrastructure/docker-cloud/image/Dockerfile +++ /dev/null @@ -1,10 +0,0 @@ -FROM ubuntu:20.04 - -ENV CLOUD_HOME="/var/cloud" \ - DOWNLOAD_URL="https://product-downloads.atlassian.com/software/nextcloud/downloads" \ - CLOUD_RELEASE="8.8.0" - # CLOUD_RELEASE="20.1.0"??? -ADD resources /tmp/resources -RUN /tmp/resources/install.sh - -USER 901:901 diff --git a/infrastructure/docker-cloud/image/resources/dbconfig.xml b/infrastructure/docker-cloud/image/resources/dbconfig.xml deleted file mode 100644 index 7ff0887..0000000 --- a/infrastructure/docker-cloud/image/resources/dbconfig.xml +++ /dev/null @@ -1,25 +0,0 @@ - - - - defaultDS - default - postgres72 - public - - jdbc:postgresql://postgresql-service:5432/cloud - org.postgresql.Driver - $CLOUD_DB_USER - $CLOUD_DB_PASSWORD - 20 - 20 - 30000 - select 1 - 60000 - 300000 - 20 - true - 300 - false - true - - \ No newline at end of file diff --git a/infrastructure/docker-cloud/image/resources/entrypoint.sh b/infrastructure/docker-cloud/image/resources/entrypoint.sh deleted file mode 100644 index 115e905..0000000 --- a/infrastructure/docker-cloud/image/resources/entrypoint.sh +++ /dev/null @@ -1,44 +0,0 @@ -#!/bin/bash - -# usage: file_env VAR [DEFAULT] -# ie: file_env 'XYZ_DB_PASSWORD' 'example' -# (will allow for "$XYZ_DB_PASSWORD_FILE" to fill in the value of -# "$XYZ_DB_PASSWORD" from a file, especially for Docker's secrets feature) -function file_env() { - local var="$1" - local fileVar="${var}_FILE" - local def="${2:-}" - local varValue=$(env | grep -E "^${var}=" | sed -E -e "s/^${var}=//") - local fileVarValue=$(env | grep -E "^${fileVar}=" | sed -E -e "s/^${fileVar}=//") - if [ -n "${varValue}" ] && [ -n "${fileVarValue}" ]; then - echo >&2 "error: both $var and $fileVar are set (but are exclusive)" - exit 1 - fi - if [ -n "${varValue}" ]; then - export "$var"="${varValue}" - elif [ -n "${fileVarValue}" ]; then - export "$var"="$(cat "${fileVarValue}")" - elif [ -n "${def}" ]; then - export "$var"="$def" - fi - unset "$fileVar" -} - -function main() { - file_env "FQDN" - file_env "DB_USERNAME" - file_env "DB_PASSWORD" - - xmlstarlet ed -L -u "/Server/Service/Connector[@proxyName='{subdomain}.{domain}.com']/@proxyName" \ - -v "$FQDN" /opt/atlassian-cloud-software-standalone/conf/server.xml - xmlstarlet ed -L -u "/cloud-database-config/jdbc-datasource/username" \ - -v "$DB_USERNAME" /app/dbconfig.xml - xmlstarlet ed -L -u "/cloud-database-config/jdbc-datasource/password" \ - -v "$DB_PASSWORD" /app/dbconfig.xml - - install -ocloud -gcloud -m660 /app/dbconfig.xml /var/cloud/dbconfig.xml - /opt/atlassian-cloud-software-standalone/bin/setenv.sh run - /opt/atlassian-cloud-software-standalone/bin/start-cloud.sh run -} - -main diff --git a/infrastructure/docker-cloud/image/resources/install.sh b/infrastructure/docker-cloud/image/resources/install.sh deleted file mode 100755 index 4927a6b..0000000 --- a/infrastructure/docker-cloud/image/resources/install.sh +++ /dev/null @@ -1,30 +0,0 @@ -#!/bin/bash - -function main() { - - upgradeSystem - apt-get -qqy install curl openjdk-11-jre-headless xmlstarlet > /dev/null - - adduser --system --disabled-login --home ${CLOUD_HOME} --uid 901 --group cloud - - mkdir -p /app - - cd /tmp - curl --location ${DOWNLOAD_URL}/atlassian-cloud-software-${CLOUD_RELEASE}.tar.gz \ - -o atlassian-cloud-software-${CLOUD_RELEASE}.tar.gz - tar -xvf atlassian-cloud-software-${CLOUD_RELEASE}.tar.gz -C /tmp/ - mv /tmp/atlassian-cloud-software-${CLOUD_RELEASE}-standalone \ - /opt/atlassian-cloud-software-standalone - chown -R cloud:cloud /opt/atlassian-cloud-software-standalone - - install -ocloud -gcloud -m744 /tmp/resources/entrypoint.sh /app/entrypoint.sh - install -ocloud -gcloud -m744 /tmp/resources/setenv.sh /opt/atlassian-cloud-software-standalone/bin/setenv.sh - install -ocloud -gcloud -m660 /tmp/resources/dbconfig.xml /app/dbconfig.xml - install -ocloud -gcloud -m660 /tmp/resources/server.xml /opt/atlassian-cloud-software-standalone/conf/server.xml - install -ocloud -gcloud -m660 /tmp/resources/logging.properties /opt/atlassian-cloud-software-standalone/conf/logging.properties - - cleanupDocker -} - -source /tmp/resources/install_functions.sh -main \ No newline at end of file diff --git a/infrastructure/docker-cloud/image/resources/logging.properties b/infrastructure/docker-cloud/image/resources/logging.properties deleted file mode 100644 index 267d805..0000000 --- a/infrastructure/docker-cloud/image/resources/logging.properties +++ /dev/null @@ -1,69 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -handlers = java.util.logging.ConsoleHandler - -.handlers = java.util.logging.ConsoleHandler - -############################################################ -# Handler specific properties. -# Describes specific configuration info for Handlers. -############################################################ - -java.util.logging.ConsoleHandler.level = FINE -java.util.logging.ConsoleHandler.formatter = org.apache.juli.OneLineFormatter -java.util.logging.ConsoleHandler.encoding = UTF-8 - - -############################################################ -# Facility specific properties. -# Provides extra control for each logger. -############################################################ - -org.apache.catalina.core.ContainerBase.[Catalina].[localhost].level = INFO -org.apache.catalina.core.ContainerBase.[Catalina].[localhost].handlers = 2localhost.org.apache.juli.AsyncFileHandler - -org.apache.catalina.core.ContainerBase.[Catalina].[localhost].[/manager].level = INFO -org.apache.catalina.core.ContainerBase.[Catalina].[localhost].[/manager].handlers = 3manager.org.apache.juli.AsyncFileHandler - -org.apache.catalina.core.ContainerBase.[Catalina].[localhost].[/host-manager].level = INFO -org.apache.catalina.core.ContainerBase.[Catalina].[localhost].[/host-manager].handlers = 4host-manager.org.apache.juli.AsyncFileHandler - -# For example, set the org.apache.catalina.util.LifecycleBase logger to log -# each component that extends LifecycleBase changing state: -#org.apache.catalina.util.LifecycleBase.level = FINE - -# To see debug messages in TldLocationsCache, uncomment the following line: -#org.apache.jasper.compiler.TldLocationsCache.level = FINE - -# To see debug messages for HTTP/2 handling, uncomment the following line: -#org.apache.coyote.http2.level = FINE - -# To see debug messages for WebSocket handling, uncomment the following line: -#org.apache.tomcat.websocket.level = FINE -# Suppress clearReferencesThreads and clearThreadLocalMap notifications on shutdown -org.apache.catalina.loader.WebappClassLoader.level = OFF - -# Suppress some useless REST messages. See JRADEV-12212. -com.sun.jersey.server.impl.application.WebApplicationImpl.level = WARNING -com.atlassian.plugins.rest.doclet.level = WARNING -com.sun.jersey.api.wadl.config.WadlGeneratorLoader.level = WARNING - -# Suppress log spam about the request body - https://cloud.atlassian.com/browse/JRASERVER-30406 -com.sun.jersey.spi.container.servlet.WebComponent.level = SEVERE -com.sun.jersey.spi.container.servlet.WebComponent.filterFormParameters.level = SEVERE - -# Suppress resources cache full warnings -org.apache.catalina.webresources.Cache.level = OFF \ No newline at end of file diff --git a/infrastructure/docker-cloud/image/resources/server.xml b/infrastructure/docker-cloud/image/resources/server.xml deleted file mode 100644 index 6e057fd..0000000 --- a/infrastructure/docker-cloud/image/resources/server.xml +++ /dev/null @@ -1,114 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/infrastructure/docker-cloud/image/resources/setenv.sh b/infrastructure/docker-cloud/image/resources/setenv.sh deleted file mode 100644 index d587952..0000000 --- a/infrastructure/docker-cloud/image/resources/setenv.sh +++ /dev/null @@ -1,121 +0,0 @@ -# -# If the limit of files that Nextcloud can open is too low, it will be set to this value. -# -MIN_NOFILES_LIMIT=16384 - -# -# One way to set the Nextcloud HOME path is here via this variable. Simply uncomment it and set a valid path like /cloud/home. You can of course set it outside in the command terminal. That will also work. -# -CLOUD_HOME="/var/cloud" - -# -# Occasionally Atlassian Support may recommend that you set some specific JVM arguments. You can use this variable below to do that. -# -JVM_SUPPORT_RECOMMENDED_ARGS="" - -# -# You can use variable below to modify garbage collector settings. -# For Java 8 we recommend default settings -# For Java 11 and relatively small heaps we recommend: -XX:+UseParallelGC -# For Java 11 and larger heaps we recommend: -XX:+UseG1GC -XX:+ExplicitGCInvokesConcurrent -# -JVM_GC_ARGS="-XX:+UseG1GC -XX:+ExplicitGCInvokesConcurrent" - -# -# The following 2 settings control the minimum and maximum given to the Nextcloud Java virtual machine. In larger Nextcloud instances, the maximum amount will need to be increased. -# -JVM_MINIMUM_MEMORY="4096m" -JVM_MAXIMUM_MEMORY="4096m" - -# -# The following setting configures the size of JVM code cache. A high value of reserved size allows Nextcloud to work with more installed apps. -# -JVM_CODE_CACHE_ARGS='-XX:InitialCodeCacheSize=32m -XX:ReservedCodeCacheSize=512m' - -# -# The following are the required arguments for Nextcloud. -# -JVM_REQUIRED_ARGS='-Djava.awt.headless=true -Datlassian.standalone=CLOUD -Dorg.apache.jasper.runtime.BodyContentImpl.LIMIT_BUFFER=true -Dmail.mime.decodeparameters=true -Dorg.dom4j.factory=com.atlassian.core.xml.InterningDocumentFactory' - -# Uncomment this setting if you want to import data without notifications -# -#DISABLE_NOTIFICATIONS=" -Datlassian.mail.senddisabled=true -Datlassian.mail.fetchdisabled=true -Datlassian.mail.popdisabled=true" - - -#----------------------------------------------------------------------------------- -# -# In general don't make changes below here -# -#----------------------------------------------------------------------------------- - -#----------------------------------------------------------------------------------- -# Prevents the JVM from suppressing stack traces if a given type of exception -# occurs frequently, which could make it harder for support to diagnose a problem. -#----------------------------------------------------------------------------------- -JVM_EXTRA_ARGS="-XX:-OmitStackTraceInFastThrow -Djava.locale.providers=COMPAT" - -CURRENT_NOFILES_LIMIT=$( ulimit -Hn ) -ulimit -Sn $CURRENT_NOFILES_LIMIT -ulimit -n $(( CURRENT_NOFILES_LIMIT > MIN_NOFILES_LIMIT ? CURRENT_NOFILES_LIMIT : MIN_NOFILES_LIMIT )) - -PRGDIR=`dirname "$0"` -cat "${PRGDIR}"/cloudbanner.txt - -CLOUD_HOME_MINUSD="" -if [ "$CLOUD_HOME" != "" ]; then - echo $CLOUD_HOME | grep -q " " - if [ $? -eq 0 ]; then - echo "" - echo "--------------------------------------------------------------------------------------------------------------------" - echo " WARNING : You cannot have a CLOUD_HOME environment variable set with spaces in it. This variable is being ignored" - echo "--------------------------------------------------------------------------------------------------------------------" - else - CLOUD_HOME_MINUSD=-Dcloud.home=$CLOUD_HOME - fi -fi - -JAVA_OPTS="-Xms${JVM_MINIMUM_MEMORY} -Xmx${JVM_MAXIMUM_MEMORY} ${JVM_CODE_CACHE_ARGS} ${JAVA_OPTS} ${JVM_REQUIRED_ARGS} ${DISABLE_NOTIFICATIONS} ${JVM_SUPPORT_RECOMMENDED_ARGS} ${JVM_EXTRA_ARGS} ${CLOUD_HOME_MINUSD} ${START_CLOUD_JAVA_OPTS}" - -export JAVA_OPTS - -# DO NOT remove the following line -# !INSTALLER SET JAVA_HOME - -echo "" -echo "If you encounter issues starting or stopping Nextcloud, please see the Troubleshooting guide at https://docs.atlassian.com/cloud/jadm-docs-088/Troubleshooting+installation" -echo "" -if [ "$CLOUD_HOME_MINUSD" != "" ]; then - echo "Using CLOUD_HOME: $CLOUD_HOME" -fi - -# set the location of the pid file -if [ -z "$CATALINA_PID" ] ; then - if [ -n "$CATALINA_BASE" ] ; then - CATALINA_PID="$CATALINA_BASE"/work/catalina.pid - elif [ -n "$CATALINA_HOME" ] ; then - CATALINA_PID="$CATALINA_HOME"/work/catalina.pid - fi -fi -export CATALINA_PID - -if [ -z "$CATALINA_BASE" ]; then - if [ -z "$CATALINA_HOME" ]; then - LOGBASE=$PRGDIR - LOGTAIL=.. - else - LOGBASE=$CATALINA_HOME - LOGTAIL=. - fi -else - LOGBASE=$CATALINA_BASE - LOGTAIL=. -fi - -PUSHED_DIR=`pwd` -cd $LOGBASE -cd $LOGTAIL -LOGBASEABS=`pwd` -cd $PUSHED_DIR - -echo "" -echo "Server startup logs are located in $LOGBASEABS/logs/catalina.out" diff --git a/infrastructure/docker-cloud/image/resources/user.sh b/infrastructure/docker-cloud/image/resources/user.sh deleted file mode 100644 index 6ae8a6d..0000000 --- a/infrastructure/docker-cloud/image/resources/user.sh +++ /dev/null @@ -1,5 +0,0 @@ -# START INSTALLER MAGIC ! DO NOT EDIT ! -CLOUD_USER="cloud" ## -# END INSTALLER MAGIC ! DO NOT EDIT ! - -export CLOUD_USER diff --git a/infrastructure/docker-cloud/test/Dockerfile b/infrastructure/docker-cloud/test/Dockerfile deleted file mode 100644 index 693ed79..0000000 --- a/infrastructure/docker-cloud/test/Dockerfile +++ /dev/null @@ -1,8 +0,0 @@ -FROM c4k-cloud - -RUN curl -L -o /tmp/serverspec.jar \ - https://github.com/DomainDrivenArchitecture/dda-serverspec-crate/releases/download/1.3.4/dda-serverspec-standalone.jar - -COPY serverspec.edn /tmp/serverspec.edn - -RUN java -jar /tmp/serverspec.jar /tmp/serverspec.edn -v diff --git a/infrastructure/docker-cloud/test/serverspec.edn b/infrastructure/docker-cloud/test/serverspec.edn deleted file mode 100644 index a462c1a..0000000 --- a/infrastructure/docker-cloud/test/serverspec.edn +++ /dev/null @@ -1,3 +0,0 @@ -{:file [{:path "/app/entrypoint.sh"} - {:path "/var/cloud"} - {:path "/opt/atlassian-cloud-software-standalone"}]} diff --git a/infrastructure/docker-cloud/build.py b/infrastructure/docker-nextcloud/build.py similarity index 83% rename from infrastructure/docker-cloud/build.py rename to infrastructure/docker-nextcloud/build.py index c192b2a..323f4bc 100644 --- a/infrastructure/docker-cloud/build.py +++ b/infrastructure/docker-nextcloud/build.py @@ -3,28 +3,26 @@ from pybuilder.core import task, init from ddadevops import * import logging -name = 'c4k-cloud' +name = 'meissa-cloud-app' MODULE = 'docker' PROJECT_ROOT_PATH = '../..' + class MyBuild(DevopsDockerBuild): pass @init def initialize(project): - project.build_depends_on('ddadevops>=0.12.4') - stage = 'prod' + project.build_depends_on('ddadevops>=0.6.1') + stage = 'notused' dockerhub_user = environ.get('DOCKERHUB_USER') if not dockerhub_user: dockerhub_user = gopass_field_from_path('meissa/web/docker.com', 'login') dockerhub_password = environ.get('DOCKERHUB_PASSWORD') if not dockerhub_password: dockerhub_password = gopass_password_from_path('meissa/web/docker.com') - tag = environ.get('CI_COMMIT_TAG') - if not tag: - tag = get_tag_from_latest_commit() config = create_devops_docker_build_config( - stage, PROJECT_ROOT_PATH, MODULE, dockerhub_user, dockerhub_password, docker_publish_tag=tag) + stage, PROJECT_ROOT_PATH, MODULE, dockerhub_user, dockerhub_password) build = MyBuild(project, config) build.initialize_build_dir() @@ -40,12 +38,12 @@ def drun(project): build.drun() @task -def publish(project): +def test(project): build = get_devops_build(project) - build.dockerhub_login() - build.dockerhub_publish() + build.test() @task -def test(project): +def publish(project): build = get_devops_build(project) - build.test() + build.dockerhub_login() + build.dockerhub_publish() diff --git a/infrastructure/docker-nextcloud/image/Dockerfile b/infrastructure/docker-nextcloud/image/Dockerfile new file mode 100644 index 0000000..8b3d9e9 --- /dev/null +++ b/infrastructure/docker-nextcloud/image/Dockerfile @@ -0,0 +1,8 @@ +FROM nextcloud:19 + +# Prepare Entrypoint Script +ADD resources /tmp +RUN /tmp/install.sh + +ENTRYPOINT ["/entrypoint.sh"] +CMD ["apache2-foreground"] diff --git a/infrastructure/docker-nextcloud/image/resources/entrypoint.sh b/infrastructure/docker-nextcloud/image/resources/entrypoint.sh new file mode 100644 index 0000000..aedef7d --- /dev/null +++ b/infrastructure/docker-nextcloud/image/resources/entrypoint.sh @@ -0,0 +1,191 @@ +#!/bin/sh +set -eu + +# version_greater A B returns whether A > B +version_greater() { + [ "$(printf '%s\n' "$@" | sort -t '.' -n -k1,1 -k2,2 -k3,3 -k4,4 | head -n 1)" != "$1" ] +} + +# return true if specified directory is empty +directory_empty() { + [ -z "$(ls -A "$1/")" ] +} + +run_as() { + if [ "$(id -u)" = 0 ]; then + su -p www-data -s /bin/sh -c "$1" + else + sh -c "$1" + fi +} + +# usage: file_env VAR [DEFAULT] +# ie: file_env 'XYZ_DB_PASSWORD' 'example' +# (will allow for "$XYZ_DB_PASSWORD_FILE" to fill in the value of +# "$XYZ_DB_PASSWORD" from a file, especially for Docker's secrets feature) +file_env() { + local var="$1" + local fileVar="${var}_FILE" + local def="${2:-}" + local varValue=$(env | grep -E "^${var}=" | sed -E -e "s/^${var}=//") + local fileVarValue=$(env | grep -E "^${fileVar}=" | sed -E -e "s/^${fileVar}=//") + if [ -n "${varValue}" ] && [ -n "${fileVarValue}" ]; then + echo >&2 "error: both $var and $fileVar are set (but are exclusive)" + exit 1 + fi + if [ -n "${varValue}" ]; then + export "$var"="${varValue}" + elif [ -n "${fileVarValue}" ]; then + export "$var"="$(cat "${fileVarValue}")" + elif [ -n "${def}" ]; then + export "$var"="$def" + fi + unset "$fileVar" +} + +if expr "$1" : "apache" 1>/dev/null; then + if [ -n "${APACHE_DISABLE_REWRITE_IP+x}" ]; then + a2disconf remoteip + fi +fi + +if expr "$1" : "apache" 1>/dev/null || [ "$1" = "php-fpm" ] || [ "${NEXTCLOUD_UPDATE:-0}" -eq 1 ]; then + if [ -n "${REDIS_HOST+x}" ]; then + + echo "Configuring Redis as session handler" + { + echo 'session.save_handler = redis' + # check if redis host is an unix socket path + if [ "$(echo "$REDIS_HOST" | cut -c1-1)" = "/" ]; then + if [ -n "${REDIS_HOST_PASSWORD+x}" ]; then + echo "session.save_path = \"unix://${REDIS_HOST}?auth=${REDIS_HOST_PASSWORD}\"" + else + echo "session.save_path = \"unix://${REDIS_HOST}\"" + fi + # check if redis password has been set + elif [ -n "${REDIS_HOST_PASSWORD+x}" ]; then + echo "session.save_path = \"tcp://${REDIS_HOST}:${REDIS_HOST_PORT:=6379}?auth=${REDIS_HOST_PASSWORD}\"" + else + echo "session.save_path = \"tcp://${REDIS_HOST}:${REDIS_HOST_PORT:=6379}\"" + fi + } > /usr/local/etc/php/conf.d/redis-session.ini + fi + + installed_version="0.0.0.0" + if [ -f /var/www/html/version.php ]; then + # shellcheck disable=SC2016 + installed_version="$(php -r 'require "/var/www/html/version.php"; echo implode(".", $OC_Version);')" + fi + # shellcheck disable=SC2016 + image_version="$(php -r 'require "/usr/src/nextcloud/version.php"; echo implode(".", $OC_Version);')" + + if version_greater "$installed_version" "$image_version"; then + echo "Can't start Nextcloud because the version of the data ($installed_version) is higher than the docker image version ($image_version) and downgrading is not supported. Are you sure you have pulled the newest image version?" + exit 1 + fi + + if version_greater "$image_version" "$installed_version"; then + echo "Initializing nextcloud $image_version (image) over $installed_version (installed) ..." + if [ "$installed_version" != "0.0.0.0" ]; then + echo "Upgrading nextcloud from $installed_version ..." + run_as 'php /var/www/html/occ app:list' | sed -n "/Enabled:/,/Disabled:/p" > /tmp/list_before + fi + if [ "$(id -u)" = 0 ]; then + rsync_options="-rlDog --chown www-data:root" + else + rsync_options="-rlD" + fi + rsync $rsync_options --delete --exclude-from=/upgrade.exclude /usr/src/nextcloud/ /var/www/html/ + + for dir in config data custom_apps themes; do + if [ ! -d "/var/www/html/$dir" ] || directory_empty "/var/www/html/$dir"; then + rsync $rsync_options --include "/$dir/" --exclude '/*' /usr/src/nextcloud/ /var/www/html/ + fi + done + rsync $rsync_options --include '/version.php' --exclude '/*' /usr/src/nextcloud/ /var/www/html/ + echo "Initializing finished" + + #install + if [ "$installed_version" = "0.0.0.0" ]; then + echo "New nextcloud instance" + + file_env NEXTCLOUD_ADMIN_PASSWORD + file_env NEXTCLOUD_ADMIN_USER + + if [ -n "${NEXTCLOUD_ADMIN_USER+x}" ] && [ -n "${NEXTCLOUD_ADMIN_PASSWORD+x}" ]; then + # shellcheck disable=SC2016 + install_options='-n --admin-user "$NEXTCLOUD_ADMIN_USER" --admin-pass "$NEXTCLOUD_ADMIN_PASSWORD"' + if [ -n "${NEXTCLOUD_DATA_DIR+x}" ]; then + # shellcheck disable=SC2016 + install_options=$install_options' --data-dir "$NEXTCLOUD_DATA_DIR"' + fi + + file_env MYSQL_DATABASE + file_env MYSQL_PASSWORD + file_env MYSQL_USER + file_env POSTGRES_DB + file_env POSTGRES_PASSWORD + file_env POSTGRES_USER + + install=false + if [ -n "${SQLITE_DATABASE+x}" ]; then + echo "Installing with SQLite database" + # shellcheck disable=SC2016 + install_options=$install_options' --database-name "$SQLITE_DATABASE"' + install=true + elif [ -n "${MYSQL_DATABASE+x}" ] && [ -n "${MYSQL_USER+x}" ] && [ -n "${MYSQL_PASSWORD+x}" ] && [ -n "${MYSQL_HOST+x}" ]; then + echo "Installing with MySQL database" + # shellcheck disable=SC2016 + install_options=$install_options' --database mysql --database-name "$MYSQL_DATABASE" --database-user "$MYSQL_USER" --database-pass "$MYSQL_PASSWORD" --database-host "$MYSQL_HOST"' + install=true + elif [ -n "${POSTGRES_DB+x}" ] && [ -n "${POSTGRES_USER+x}" ] && [ -n "${POSTGRES_PASSWORD+x}" ] && [ -n "${POSTGRES_HOST+x}" ]; then + echo "Installing with PostgreSQL database" + # shellcheck disable=SC2016 + install_options=$install_options' --database pgsql --database-name "$POSTGRES_DB" --database-user "$POSTGRES_USER" --database-pass "$POSTGRES_PASSWORD" --database-host "$POSTGRES_HOST"' + install=true + fi + + if [ "$install" = true ]; then + echo "starting nextcloud installation" + echo "$install_options" + max_retries=10 + try=0 + until run_as "php /var/www/html/occ maintenance:install $install_options" || [ "$try" -gt "$max_retries" ] + do + echo "retrying install..." + try=$((try+1)) + sleep 20s + done + if [ "$try" -gt "$max_retries" ]; then + echo "installing of nextcloud failed!" + exit 1 + fi + if [ -n "${NEXTCLOUD_TRUSTED_DOMAINS+x}" ]; then + echo "setting trusted domains…" + NC_TRUSTED_DOMAIN_IDX=1 + for DOMAIN in $NEXTCLOUD_TRUSTED_DOMAINS ; do + DOMAIN=$(echo "$DOMAIN" | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//') + run_as "php /var/www/html/occ config:system:set trusted_domains $NC_TRUSTED_DOMAIN_IDX --value=$DOMAIN" + run_as "php /var/www/html/occ config:system:set overwritehost --value=$DOMAIN" + NC_TRUSTED_DOMAIN_IDX=$(($NC_TRUSTED_DOMAIN_IDX+1)) + done + fi + run_as "php /var/www/html/occ config:system:set overwriteprotocol --value=https" + else + echo "running web-based installer on first connect!" + fi + fi + #upgrade + else + run_as 'php /var/www/html/occ upgrade' + + run_as 'php /var/www/html/occ app:list' | sed -n "/Enabled:/,/Disabled:/p" > /tmp/list_after + echo "The following apps have been disabled:" + diff /tmp/list_before /tmp/list_after | grep '<' | cut -d- -f2 | cut -d: -f1 + rm -f /tmp/list_before /tmp/list_after + + fi + fi +fi + +exec "$@" diff --git a/infrastructure/docker-nextcloud/image/resources/install-debug.sh b/infrastructure/docker-nextcloud/image/resources/install-debug.sh new file mode 100644 index 0000000..a5b8ce8 --- /dev/null +++ b/infrastructure/docker-nextcloud/image/resources/install-debug.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +apt update && apt -qqy install vim bash-completion less diff --git a/infrastructure/docker-nextcloud/image/resources/install.sh b/infrastructure/docker-nextcloud/image/resources/install.sh new file mode 100755 index 0000000..b92fc18 --- /dev/null +++ b/infrastructure/docker-nextcloud/image/resources/install.sh @@ -0,0 +1,9 @@ +#!/bin/bash +set -Eeo pipefail + +mkdir /var/data + +install -m 0700 /tmp/install-debug.sh /usr/local/bin/ +install -m 0544 /tmp/upload-max-limit.ini /usr/local/etc/php/conf.d/ +install -m 0544 /tmp/memory-limit.ini /usr/local/etc/php/conf.d/ +install -m 0755 /tmp/entrypoint.sh / \ No newline at end of file diff --git a/infrastructure/docker-nextcloud/image/resources/memory-limit.ini b/infrastructure/docker-nextcloud/image/resources/memory-limit.ini new file mode 100644 index 0000000..3dc87bc --- /dev/null +++ b/infrastructure/docker-nextcloud/image/resources/memory-limit.ini @@ -0,0 +1,2 @@ + +memory_limit=5120M diff --git a/infrastructure/docker-nextcloud/image/resources/upload-max-limit.ini b/infrastructure/docker-nextcloud/image/resources/upload-max-limit.ini new file mode 100644 index 0000000..61a407e --- /dev/null +++ b/infrastructure/docker-nextcloud/image/resources/upload-max-limit.ini @@ -0,0 +1,3 @@ + +post_max_size = 512M +upload_max_filesize = 512M diff --git a/infrastructure/docker-nextcloud/test/Dockerfile b/infrastructure/docker-nextcloud/test/Dockerfile new file mode 100644 index 0000000..891c1ee --- /dev/null +++ b/infrastructure/docker-nextcloud/test/Dockerfile @@ -0,0 +1,11 @@ +FROM meissa-cloud-app + +RUN apt update +RUN mkdir /usr/share/man/man1/ +RUN apt -yqq install --no-install-recommends --yes curl default-jre-headless + +RUN curl -L -o /tmp/serverspec.jar https://github.com/DomainDrivenArchitecture/dda-serverspec-crate/releases/download/2.0.0/dda-serverspec-standalone.jar + +COPY serverspec.edn /tmp/serverspec.edn + +RUN java -jar /tmp/serverspec.jar /tmp/serverspec.edn -v diff --git a/infrastructure/docker-nextcloud/test/serverspec.edn b/infrastructure/docker-nextcloud/test/serverspec.edn new file mode 100644 index 0000000..767e01c --- /dev/null +++ b/infrastructure/docker-nextcloud/test/serverspec.edn @@ -0,0 +1,2 @@ +{:file [{:path "/var/data"} + {:path "/entrypoint.sh" :mod "755"}]} diff --git a/main/resources/backup/backup-restore.yaml b/main/resources/backup/backup-restore.yaml new file mode 100644 index 0000000..c13e166 --- /dev/null +++ b/main/resources/backup/backup-restore.yaml @@ -0,0 +1,59 @@ +kind: Pod +apiVersion: v1 +metadata: + name: backup-restore + labels: + app.kubernetes.io/name: backup-restore + app.kubernetes.io/part-of: cloud +spec: + containers: + - name: backup-app + image: domaindrivenarchitecture/c4k-cloud-backup + imagePullPolicy: IfNotPresent + command: ["/entrypoint-start-and-wait.sh"] + env: + - name: POSTGRES_USER_FILE + value: /var/run/secrets/cloud-secrets/postgres-user + - name: POSTGRES_DB_FILE + value: /var/run/secrets/cloud-secrets/postgres-db + - name: POSTGRES_PASSWORD_FILE + value: /var/run/secrets/cloud-secrets/postgres-password + - name: POSTGRES_HOST + value: "postgresql-service:5432" + - name: POSTGRES_SERVICE + value: "postgresql-service" + - name: POSTGRES_PORT + value: "5432" + - name: AWS_DEFAULT_REGION + value: eu-central-1 + - name: AWS_ACCESS_KEY_ID_FILE + value: /var/run/secrets/backup-secrets/aws-access-key-id + - name: AWS_SECRET_ACCESS_KEY_FILE + value: /var/run/secrets/backup-secrets/aws-secret-access-key + - name: RESTIC_REPOSITORY + valueFrom: + configMapKeyRef: + name: backup-config + key: restic-repository + - name: RESTIC_PASSWORD_FILE + value: /var/run/secrets/backup-secrets/restic-password + volumeMounts: + - name: cloud-data-volume + mountPath: /var/backups + - name: backup-secret-volume + mountPath: /var/run/secrets/backup-secrets + readOnly: true + - name: cloud-secret-volume + mountPath: /var/run/secrets/cloud-secrets + readOnly: true + volumes: + - name: cloud-data-volume + persistentVolumeClaim: + claimName: cloud-pvc + - name: cloud-secret-volume + secret: + secretName: cloud-secret + - name: backup-secret-volume + secret: + secretName: backup-secret + restartPolicy: OnFailure \ No newline at end of file diff --git a/main/resources/backup/config.yaml b/main/resources/backup/config.yaml new file mode 100644 index 0000000..17aa35c --- /dev/null +++ b/main/resources/backup/config.yaml @@ -0,0 +1,9 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: backup-config + labels: + app.kubernetes.io/name: backup + app.kubernetes.io/part-of: cloud +data: + restic-repository: restic-repository \ No newline at end of file diff --git a/main/resources/backup/configure-as-user.sh b/main/resources/backup/configure-as-user.sh new file mode 100644 index 0000000..a5a099c --- /dev/null +++ b/main/resources/backup/configure-as-user.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +kubectl delete --ignore-not-found=true -f backup-secret.yml +kubectl delete --ignore-not-found=true -f backup-config.yml +kubectl delete --ignore-not-found=true -f backup-cron.yml + +kubectl apply -f backup-secret.yml +kubectl apply -f backup-config.yml +kubectl apply -f backup-cron.yml diff --git a/main/resources/backup/cron.yaml b/main/resources/backup/cron.yaml new file mode 100644 index 0000000..8bb54bc --- /dev/null +++ b/main/resources/backup/cron.yaml @@ -0,0 +1,65 @@ +apiVersion: batch/v1beta1 +kind: CronJob +metadata: + name: cloud-backup + labels: + app.kubernetes.part-of: cloud +spec: + schedule: "10 23 * * *" + successfulJobsHistoryLimit: 0 + failedJobsHistoryLimit: 0 + jobTemplate: + spec: + template: + spec: + containers: + - name: backup-app + image: domaindrivenarchitecture/meissa-cloud-backup + imagePullPolicy: IfNotPresent + command: ["/entrypoint.sh"] + env: + - name: POSTGRES_USER_FILE + value: /var/run/secrets/cloud-secrets/postgres-user + - name: POSTGRES_DB_FILE + value: /var/run/secrets/cloud-secrets/postgres-db + - name: POSTGRES_PASSWORD_FILE + value: /var/run/secrets/cloud-secrets/postgres-password + - name: POSTGRES_HOST + value: "postgresql-service:5432" + - name: POSTGRES_SERVICE + value: "postgresql-service" + - name: POSTGRES_PORT + value: "5432" + - name: AWS_DEFAULT_REGION + value: eu-central-1 + - name: AWS_ACCESS_KEY_ID_FILE + value: /var/run/secrets/backup-secrets/aws-access-key-id + - name: AWS_SECRET_ACCESS_KEY_FILE + value: /var/run/secrets/backup-secrets/aws-secret-access-key + - name: RESTIC_REPOSITORY + valueFrom: + configMapKeyRef: + name: backup-config + key: restic-repository + - name: RESTIC_PASSWORD_FILE + value: /var/run/secrets/backup-secrets/restic-password + volumeMounts: + - name: cloud-data-volume + mountPath: /var/backups + - name: backup-secret-volume + mountPath: /var/run/secrets/backup-secrets + readOnly: true + - name: cloud-secret-volume + mountPath: /var/run/secrets/cloud-secrets + readOnly: true + volumes: + - name: cloud-data-volume + persistentVolumeClaim: + claimName: cloud-pvc + - name: cloud-secret-volume + secret: + secretName: cloud-secret + - name: backup-secret-volume + secret: + secretName: backup-secret + restartPolicy: OnFailure \ No newline at end of file diff --git a/main/resources/backup/secret.yaml b/main/resources/backup/secret.yaml new file mode 100644 index 0000000..4b68578 --- /dev/null +++ b/main/resources/backup/secret.yaml @@ -0,0 +1,9 @@ +apiVersion: v1 +kind: Secret +metadata: + name: backup-secret +type: Opaque +stringData: + aws-access-key-id: aws-access-key-id + aws-secret-access-key: aws-secret-access-key + restic-password: restic-password \ No newline at end of file diff --git a/main/resources/cloud/certificate.yaml b/main/resources/cloud/certificate.yaml new file mode 100644 index 0000000..054965b --- /dev/null +++ b/main/resources/cloud/certificate.yaml @@ -0,0 +1,13 @@ +apiVersion: cert-manager.io/v1alpha2 +kind: Certificate +metadata: + name: cloud-cert + namespace: default +spec: + secretName: cloud-secret + commonName: fqdn + dnsNames: + - fqdn + issuerRef: + name: letsencrypt-staging-issuer + kind: ClusterIssuer \ No newline at end of file diff --git a/main/resources/cloud/cloud-pod.yml.template b/main/resources/cloud/cloud-pod.yml.template new file mode 100644 index 0000000..eac26ec --- /dev/null +++ b/main/resources/cloud/cloud-pod.yml.template @@ -0,0 +1,45 @@ +kind: Pod +apiVersion: v1 +metadata: + name: cloud + labels: + app.kubernetes.io/name: cloud +spec: + shareProcessNamespace: true + containers: + - name: cloud-app + image: domaindrivenarchitecture/meissa-cloud-app + imagePullPolicy: IfNotPresent + ports: + - containerPort: 80 + env: + - name: NEXTCLOUD_ADMIN_USER_FILE + value: /var/run/secrets/cloud-secrets/nextcloud-admin-user + - name: NEXTCLOUD_ADMIN_PASSWORD_FILE + value: /var/run/secrets/cloud-secrets/nextcloud-admin-password + - name: NEXTCLOUD_TRUSTED_DOMAINS + value: "{{fqdn}}" + - name: POSTGRES_USER_FILE + value: /var/run/secrets/cloud-secrets/postgres-user + - name: POSTGRES_PASSWORD_FILE + value: /var/run/secrets/cloud-secrets/postgres-password + - name: POSTGRES_DB_FILE + value: /var/run/secrets/cloud-secrets/postgres-db + - name: POSTGRES_HOST + value: "postgresql-service:5432" + volumeMounts: + - name: cloud-data-volume + mountPath: /var/www/html + - name: cloud-secret-volume + mountPath: /var/run/secrets/cloud-secrets + readOnly: true + volumes: + - name: cloud-data-volume + persistentVolumeClaim: + claimName: cloud-pvc + - name: cloud-secret-volume + secret: + secretName: cloud-secret + - name: backup-secret-volume + secret: + secretName: backup-secret diff --git a/main/resources/cloud/configure-as-user.sh b/main/resources/cloud/configure-as-user.sh new file mode 100644 index 0000000..6e46558 --- /dev/null +++ b/main/resources/cloud/configure-as-user.sh @@ -0,0 +1,22 @@ +#!/bin/bash + +kubectl delete --ignore-not-found=true -f cloud-ingress.yml +kubectl delete --ignore-not-found=true -f cloud-pod.yml +kubectl delete --ignore-not-found=true -f cloud-pvc.yml +kubectl delete --ignore-not-found=true -f cloud-service.yml +kubectl delete --ignore-not-found=true -f cloud-secret.yml +kubectl delete --ignore-not-found=true -f cloud-persistent-volume.yml + +#Wait for postgres to be running +while [$POSTGRES = ""] +do + POSTGRES=$(kubectl get pods --selector=app=postgresql -o jsonpath='{.items[*].metadata.name}') +done +kubectl wait --for=condition=ready pod/$POSTGRES + +kubectl apply -f cloud-persistent-volume.yml +kubectl apply -f cloud-secret.yml +kubectl apply -f cloud-service.yml +kubectl apply -f cloud-pvc.yml +kubectl apply -f cloud-pod.yml +kubectl apply -f cloud-ingress.yml diff --git a/main/resources/cloud/ingress.yaml b/main/resources/cloud/ingress.yaml new file mode 100644 index 0000000..cc5a0df --- /dev/null +++ b/main/resources/cloud/ingress.yaml @@ -0,0 +1,26 @@ +apiVersion: extensions/v1beta1 +kind: Ingress +metadata: + name: ingress-cloud + annotations: + cert-manager.io/cluster-issuer: letsencrypt-staging-issuer + nginx.ingress.kubernetes.io/proxy-body-size: "256m" + nginx.ingress.kubernetes.io/ssl-redirect: "true" + nginx.ingress.kubernetes.io/rewrite-target: / + nginx.ingress.kubernetes.io/proxy-connect-timeout: "300" + nginx.ingress.kubernetes.io/proxy-send-timeout: "300" + nginx.ingress.kubernetes.io/proxy-read-timeout: "300" + namespace: default +spec: + tls: + - hosts: + - fqdn + secretName: cloud-secret + rules: + - host: fqdn + http: + paths: + - path: / + backend: + serviceName: cloud-service + servicePort: 80 diff --git a/main/resources/cloud/install-as-root.sh.template b/main/resources/cloud/install-as-root.sh.template new file mode 100644 index 0000000..1492df6 --- /dev/null +++ b/main/resources/cloud/install-as-root.sh.template @@ -0,0 +1,4 @@ +#!/bin/bash + +mkdir -p /var/cloud +install -d -m 0777 -o {{user}} -g {{user}} /var/cloud diff --git a/main/resources/cloud/persistent-volume.yaml b/main/resources/cloud/persistent-volume.yaml new file mode 100644 index 0000000..7c3f89e --- /dev/null +++ b/main/resources/cloud/persistent-volume.yaml @@ -0,0 +1,15 @@ +kind: PersistentVolume +apiVersion: v1 +metadata: + name: cloud-pv-volume + labels: + type: local + app: cloud +spec: + storageClassName: manual + accessModes: + - ReadWriteOnce + capacity: + storage: {{storage-size}}Gi #??? 30Gi? + hostPath: + path: "/var/cloud" diff --git a/main/resources/cloud/pod-running.sh b/main/resources/cloud/pod-running.sh new file mode 100755 index 0000000..61e8cb2 --- /dev/null +++ b/main/resources/cloud/pod-running.sh @@ -0,0 +1,25 @@ +#!/bin/bash +SECONDS=0 + +while true; do + + POD_STATUS="$(kubectl get pods --all-namespaces --field-selector=status.phase=Running | grep $1 )"; + if [ ! -z "$POD_STATUS" ] + then + # pod = ready is not enough + sleep $4 + break + fi + + let duration=$SECONDS/60 + # pallet needs a regular action, otherwise unwanted timeout after 5 min + echo "Seconds waited: ${SECONDS}" + if [ "$duration" -ge "$2" ] + then + exit 1 + fi + + sleep $3 +done + +exit 0 diff --git a/main/resources/cloud/pvc.yaml b/main/resources/cloud/pvc.yaml new file mode 100644 index 0000000..d4f7e04 --- /dev/null +++ b/main/resources/cloud/pvc.yaml @@ -0,0 +1,16 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: cloud-pvc + labels: + app: cloud +spec: + storageClassName: manual + accessModes: + - ReadWriteOnce + resources: + requests: + storage: {{storage-size}}Gi #??? 30Gi? + selector: + matchLabels: + app: cloud diff --git a/main/resources/cloud/secret.yaml b/main/resources/cloud/secret.yaml new file mode 100644 index 0000000..2429d16 --- /dev/null +++ b/main/resources/cloud/secret.yaml @@ -0,0 +1,11 @@ +apiVersion: v1 +kind: Secret +metadata: + name: cloud-secret +type: Opaque +stringData: + postgres-db: db-name + postgres-user: db-user-name + postgres-password: db-user-password + nextcloud-admin-user: admin-user + nextcloud-admin-password: admin-password diff --git a/main/resources/cloud/service.yaml b/main/resources/cloud/service.yaml new file mode 100644 index 0000000..7fcd0d7 --- /dev/null +++ b/main/resources/cloud/service.yaml @@ -0,0 +1,9 @@ +apiVersion: v1 +kind: Service +metadata: + name: cloud-service +spec: + selector: + app.kubernetes.io/name: cloud #??? + ports: + - port: 80 diff --git a/main/resources/cloud/verify.sh.template b/main/resources/cloud/verify.sh.template new file mode 100644 index 0000000..ad362e5 --- /dev/null +++ b/main/resources/cloud/verify.sh.template @@ -0,0 +1,15 @@ +#!/bin/bash + +echo -e "\n====================\n" +echo -e "cloud is running, ingress exists" +echo -e "\n====================\n" +kubectl get all + +echo -e "\n====================\n" +echo -e "shows certificate with subject" +echo -e "CN={{fqdn}}" +echo -e "issuer: CN=Fake LE Intermediate X1" +echo -e "\n====================\n" +curl --insecure -v https://{{fqdn}} + +echo -e "\n" diff --git a/main/resources/postgres/configure-as-user.sh b/main/resources/postgres/configure-as-user.sh new file mode 100644 index 0000000..3d76fbe --- /dev/null +++ b/main/resources/postgres/configure-as-user.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +kubectl delete --ignore-not-found=true -f postgres-deployment.yml +kubectl delete --ignore-not-found=true -f postgres-pvc.yml +kubectl delete --ignore-not-found=true -f postgres-service.yml +kubectl delete --ignore-not-found=true -f postgres-config.yml +kubectl delete --ignore-not-found=true -f postgres-secret.yml +kubectl delete --ignore-not-found=true -f postgres-persistent-volume.yml + +kubectl apply -f postgres-persistent-volume.yml +kubectl apply -f postgres-secret.yml +kubectl apply -f postgres-config.yml +kubectl apply -f postgres-service.yml +kubectl apply -f postgres-pvc.yml +kubectl apply -f postgres-deployment.yml diff --git a/main/resources/postgres/install-as-root.sh.template b/main/resources/postgres/install-as-root.sh.template new file mode 100644 index 0000000..d8899f6 --- /dev/null +++ b/main/resources/postgres/install-as-root.sh.template @@ -0,0 +1,4 @@ +#!/bin/bash + +mkdir -p /var/postgres +install -d -m 0777 -o {{user}} -g {{user}} /var/postgres diff --git a/main/resources/postgres/postgres-config.yml b/main/resources/postgres/postgres-config.yml new file mode 100644 index 0000000..799cc45 --- /dev/null +++ b/main/resources/postgres/postgres-config.yml @@ -0,0 +1,10 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: postgres-config + labels: + app: postgres +data: + postgresql.conf: | + max_connections = 1000 + shared_buffers = 512MB diff --git a/main/resources/postgres/postgres-deployment.yml.template b/main/resources/postgres/postgres-deployment.yml.template new file mode 100644 index 0000000..55d9f65 --- /dev/null +++ b/main/resources/postgres/postgres-deployment.yml.template @@ -0,0 +1,49 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: postgresql +spec: + selector: + matchLabels: + app: postgresql + strategy: + type: Recreate + template: + metadata: + labels: + app: postgresql + spec: + containers: + - image: postgres + name: postgresql + env: + - name: POSTGRES_USER_FILE + value: /var/run/secrets/postgres-secrets/postgres-user + - name: POSTGRES_DB_FILE + value: /var/run/secrets/postgres-secrets/postgres-db + - name: POSTGRES_PASSWORD_FILE + value: /var/run/secrets/postgres-secrets/postgres-password + ports: + - containerPort: 5432 + name: postgresql + cmd: + volumeMounts: + - name: postgresql + mountPath: /var/lib/postgresql/data + - name: postgres-secret-volume + mountPath: /var/run/secrets/postgres-secrets + readOnly: true + - name: postgres-config-volume + mountPath: /etc/postgresql/postgresql.conf + subPath: postgresql.conf + readOnly: true + volumes: + - name: postgresql + persistentVolumeClaim: + claimName: postgres-claim + - name: postgres-secret-volume + secret: + secretName: postgres-secret + - name: postgres-config-volume + configMap: + name: postgres-config diff --git a/main/resources/postgres/postgres-persistent-volume.yml b/main/resources/postgres/postgres-persistent-volume.yml new file mode 100644 index 0000000..0c7b710 --- /dev/null +++ b/main/resources/postgres/postgres-persistent-volume.yml @@ -0,0 +1,15 @@ +kind: PersistentVolume +apiVersion: v1 +metadata: + name: postgres-pv-volume + labels: + type: local + app: postgresql +spec: + storageClassName: manual + accessModes: + - ReadWriteOnce + capacity: + storage: 10Gi + hostPath: + path: "/var/postgres" diff --git a/main/resources/postgres/postgres-pvc.yml b/main/resources/postgres/postgres-pvc.yml new file mode 100644 index 0000000..1a43337 --- /dev/null +++ b/main/resources/postgres/postgres-pvc.yml @@ -0,0 +1,16 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: postgres-claim + labels: + app: postgresql +spec: + storageClassName: manual + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 10Gi + selector: + matchLabels: + app: postgresql \ No newline at end of file diff --git a/main/resources/postgres/postgres-secret.yml.template b/main/resources/postgres/postgres-secret.yml.template new file mode 100644 index 0000000..d3579d3 --- /dev/null +++ b/main/resources/postgres/postgres-secret.yml.template @@ -0,0 +1,9 @@ +apiVersion: v1 +kind: Secret +metadata: + name: postgres-secret +type: Opaque +stringData: + postgres-db: cloud + postgres-user: {{db-user-name}} + postgres-password: {{db-user-password}} diff --git a/main/resources/postgres/postgres-service.yml b/main/resources/postgres/postgres-service.yml new file mode 100644 index 0000000..c3cadf1 --- /dev/null +++ b/main/resources/postgres/postgres-service.yml @@ -0,0 +1,10 @@ +--- +apiVersion: v1 +kind: Service +metadata: + name: postgresql-service +spec: + selector: + app: postgresql + ports: + - port: 5432 diff --git a/main/resources/postgres/verify.sh b/main/resources/postgres/verify.sh new file mode 100644 index 0000000..b9f0730 --- /dev/null +++ b/main/resources/postgres/verify.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +echo -e "\n====================\n" +echo -e "postgres is running" +echo -e "\n====================\n" +kubectl get all + +echo -e "\n" diff --git a/main/src/meissa/pallet/meissa_cloud/app.clj b/main/src/meissa/pallet/meissa_cloud/app.clj new file mode 100644 index 0000000..1547082 --- /dev/null +++ b/main/src/meissa/pallet/meissa_cloud/app.clj @@ -0,0 +1,61 @@ +(ns meissa.pallet.meissa-cloud.app + (:require + [schema.core :as s] + [dda.pallet.commons.secret :as secret] + [dda.config.commons.map-utils :as mu] + [dda.pallet.core.app :as core-app] + [dda.pallet.dda-config-crate.infra :as config-crate] + [dda.pallet.dda-user-crate.app :as user] + [dda.pallet.dda-k8s-crate.app :as k8s] + [meissa.pallet.meissa-cloud.convention :as convention] + [meissa.pallet.meissa-cloud.infra :as infra])) + +(def with-cloud infra/with-cloud) + +(def CloudConvention convention/CloudConvention) + +(def CloudConventionResolved convention/CloudConventionResolved) + +(def InfraResult convention/InfraResult) + +(def CloudApp + {:group-specific-config + {s/Keyword (merge InfraResult + user/InfraResult + k8s/InfraResult)}}) + +(s/defn ^:always-validate + app-configuration-resolved :- CloudApp + [resolved-convention-config :- CloudConventionResolved + & options] + (let [{:keys [group-key] :or {group-key infra/facility}} options] + (mu/deep-merge + (k8s/app-configuration-resolved + (convention/k8s-convention-configuration resolved-convention-config) :group-key group-key) + {:group-specific-config + {group-key + (convention/infra-configuration resolved-convention-config)}}))) + +(s/defn ^:always-validate + app-configuration :- CloudApp + [convention-config :- CloudConvention + & options] + (let [resolved-convention-config (secret/resolve-secrets convention-config CloudConvention)] + (apply app-configuration-resolved resolved-convention-config options))) + +(s/defmethod ^:always-validate + core-app/group-spec infra/facility + [crate-app + convention-config :- CloudConventionResolved] + (let [app-config (app-configuration-resolved convention-config)] + (core-app/pallet-group-spec + app-config [(config-crate/with-config app-config) + user/with-user + k8s/with-k8s + with-cloud]))) + +(def crate-app (core-app/make-dda-crate-app + :facility infra/facility + :convention-schema CloudConvention + :convention-schema-resolved CloudConventionResolved + :default-convention-file "cloud.edn")) diff --git a/main/src/meissa/pallet/meissa_cloud/convention.clj b/main/src/meissa/pallet/meissa_cloud/convention.clj new file mode 100644 index 0000000..17223f4 --- /dev/null +++ b/main/src/meissa/pallet/meissa_cloud/convention.clj @@ -0,0 +1,93 @@ +(ns meissa.pallet.meissa-cloud.convention + (:require + [schema.core :as s] + [dda.pallet.commons.secret :as secret] + [dda.config.commons.map-utils :as mu] + [clojure.spec.alpha :as sp] + [clojure.spec.test.alpha :as st] + [dda.pallet.dda-k8s-crate.convention :as k8s-convention] + [meissa.pallet.meissa-cloud.infra :as infra] + [clojure.string :as str] + [meissa.pallet.meissa-cloud.convention.bash :as bash] + [meissa.pallet.meissa-cloud.convention.bash-php :as bash-php])) + +(def InfraResult {infra/facility infra/MeissaCloudInfra}) + +(s/def CloudConvention + {:user s/Keyword + :external-ip s/Str + :fqdn s/Str + :cert-manager (s/enum :letsencrypt-prod-issuer :letsencrypt-staging-issuer) + :db-user-password secret/Secret + :admin-user s/Str + :admin-password secret/Secret + :storage-size s/Int + :restic-repository s/Str + :aws-access-key-id secret/Secret + :aws-secret-access-key secret/Secret + :restic-password secret/Secret + (s/optional-key :u18-04) (s/enum true)}) + +(def CloudConventionResolved (secret/create-resolved-schema CloudConvention)) + +(sp/def ::user keyword?) +(sp/def ::external-ip string?) +(sp/def ::fqdn string?) +(sp/def ::cert-manager #{:letsencrypt-prod-issuer :letsencrypt-staging-issuer}) +(sp/def ::db-user-password bash-php/bash-php-env-string?) +(sp/def ::admin-user bash-php/bash-php-env-string?) +(sp/def ::admin-password bash-php/bash-php-env-string?) +(sp/def ::storage-size int?) +(sp/def ::restic-repository string?) +(sp/def ::restic-password bash/bash-env-string?) +(sp/def ::aws-access-key-id bash/bash-env-string?) +(sp/def ::aws-secret-access-key bash/bash-env-string?) +(sp/def ::u18-04 #{true}) +(def cloud-convention-resolved? (sp/keys :req-un [::user ::external-ip ::fqdn ::cert-manager + ::db-user-password ::admin-user ::admin-password + ::storage-size ::restic-repository ::restic-password + ::aws-access-key-id ::aws-secret-access-key ] + :opt-un [::u18-04])) + +(def cloud-spec-resolved nil) + +(s/defn k8s-convention-configuration :- k8s-convention/k8sConventionResolved + [convention-config :- CloudConventionResolved] + {:pre [(sp/valid? cloud-convention-resolved? convention-config)]} + (let [{:keys [cert-manager external-ip user u18-04]} convention-config + cluster-issuer (name cert-manager)] + (if u18-04 + {:user user + :k8s {:external-ip external-ip + :u18-04 true} + :cert-manager cert-manager} + {:user user + :k8s {:external-ip external-ip} + :cert-manager cert-manager}))) + + +(s/defn ^:always-validate + infra-configuration :- InfraResult + [convention-config :- CloudConventionResolved] + (let [{:keys [cert-manager fqdn user db-user-password admin-user admin-password storage-size + restic-repository aws-access-key-id aws-secret-access-key restic-password]} convention-config + cluster-issuer (name cert-manager) + db-user-name "cloud"] + {infra/facility + {:user user + :backup {:restic-repository restic-repository + :aws-access-key-id aws-access-key-id + :aws-secret-access-key aws-secret-access-key + :restic-password restic-password} + :cloud {:fqdn fqdn + :secret-name (str/replace fqdn #"\." "-") + :cluster-issuer cluster-issuer + :db-name "cloud" + :db-user-password db-user-password + :db-user-name db-user-name + :admin-user admin-user + :admin-password admin-password + :storage-size (str storage-size)} + :postgres {:db-user-password db-user-password + :db-user-name db-user-name}}})) + diff --git a/main/src/meissa/pallet/meissa_cloud/convention/bash.clj b/main/src/meissa/pallet/meissa_cloud/convention/bash.clj new file mode 100644 index 0000000..5d1ef2c --- /dev/null +++ b/main/src/meissa/pallet/meissa_cloud/convention/bash.clj @@ -0,0 +1,10 @@ +(ns meissa.pallet.meissa-cloud.convention.bash + (:require + [clojure.spec.alpha :as s])) + +(defn bash-env-string? + [input] + (and (string? input) + (not (re-matches #".*['\"\$]+.*" input)))) + +(s/def ::plain bash-env-string?) diff --git a/main/src/meissa/pallet/meissa_cloud/convention/bash_php.clj b/main/src/meissa/pallet/meissa_cloud/convention/bash_php.clj new file mode 100644 index 0000000..d065a66 --- /dev/null +++ b/main/src/meissa/pallet/meissa_cloud/convention/bash_php.clj @@ -0,0 +1,11 @@ +(ns meissa.pallet.meissa-cloud.convention.bash-php + (:require + [clojure.spec.alpha :as s] + [meissa.pallet.meissa-cloud.convention.bash :as bash])) + +(defn bash-php-env-string? + [input] + (and (bash/bash-env-string? input) + (not (re-matches #".*[\-\\\\]+.*" input)))) + +(s/def ::plain bash-php-env-string?) diff --git a/main/src/meissa/pallet/meissa_cloud/infra.clj b/main/src/meissa/pallet/meissa_cloud/infra.clj new file mode 100644 index 0000000..a964dc3 --- /dev/null +++ b/main/src/meissa/pallet/meissa_cloud/infra.clj @@ -0,0 +1,51 @@ +(ns meissa.pallet.meissa-cloud.infra + (:require + [schema.core :as s] + [dda.pallet.core.infra :as core-infra] + [meissa.pallet.meissa-cloud.infra.backup :as backup] + [meissa.pallet.meissa-cloud.infra.cloud :as cloud] + [meissa.pallet.meissa-cloud.infra.postgres :as postgres])) + +(def facility :meissa-cloud) + +(def MeissaCloudInfra + (merge + {:user s/Keyword} + backup/MeissaBackupInfra + cloud/MeissaCloudInfra + postgres/MeissaPostgresInfra)) + +(s/defmethod core-infra/dda-init facility + [dda-crate config] + (let [facility (:facility dda-crate) + {:keys [user backup postgres cloud]} config + user-str (name user)] + (postgres/init facility user-str postgres) + (cloud/init facility user-str cloud) + (backup/init facility user-str backup))) + +(s/defmethod core-infra/dda-install facility + [dda-crate config] + (let [facility (:facility dda-crate) + {:keys [user backup postgres cloud]} config + user-str (name user)] + (postgres/install facility user-str postgres) + (cloud/install facility user-str cloud) + (backup/install facility user-str backup))) + +(s/defmethod core-infra/dda-configure facility + [dda-crate config] + (let [facility (:facility dda-crate) + {:keys [user backup postgres cloud]} config + user-str (name user)] + (postgres/configure facility user-str postgres) + (cloud/configure facility user-str cloud) + (backup/configure facility user-str backup))) + +(def meissa-cloud + (core-infra/make-dda-crate-infra + :facility facility + :infra-schema MeissaCloudInfra)) + +(def with-cloud + (core-infra/create-infra-plan meissa-cloud)) diff --git a/main/src/meissa/pallet/meissa_cloud/infra/backup.clj b/main/src/meissa/pallet/meissa_cloud/infra/backup.clj new file mode 100644 index 0000000..c80ead6 --- /dev/null +++ b/main/src/meissa/pallet/meissa_cloud/infra/backup.clj @@ -0,0 +1,39 @@ +(ns meissa.pallet.meissa-cloud.infra.backup + (:require + [schema.core :as s] + [dda.provision :as p] + [dda.provision.pallet :as pp])) + +(s/def Backup + {:restic-repository s/Str + :aws-access-key-id s/Str + :aws-secret-access-key s/Str + :restic-password s/Str}) + +(def MeissaBackupInfra {:backup Backup}) + +(def backup "backup") + +(defn init [facility user config]) + +(defn install + [facility user config] + (let [facility-name (name facility)] + (p/provision-log ::pp/pallet facility-name backup + ::p/info "install") + (p/copy-resources-to-user + ::pp/pallet user facility-name backup + [{:filename "backup-secret.yml" :config config} + {:filename "backup-config.yml" :config config} + {:filename "configure-as-user.sh"} + {:filename "backup-restore.yml"} + {:filename "backup-cron.yml"}]))) + +(defn configure + [facility user config] + (let [facility-name (name facility)] + (p/provision-log ::pp/pallet facility-name backup + ::p/info "configure") + (p/exec-file-on-target-as-user + ::pp/pallet user facility-name backup "configure-as-user.sh") + )) diff --git a/main/src/meissa/pallet/meissa_cloud/infra/cloud.clj b/main/src/meissa/pallet/meissa_cloud/infra/cloud.clj new file mode 100644 index 0000000..ed11188 --- /dev/null +++ b/main/src/meissa/pallet/meissa_cloud/infra/cloud.clj @@ -0,0 +1,57 @@ +(ns meissa.pallet.meissa-cloud.infra.cloud + (:require + [schema.core :as s] + [dda.provision :as p] + [dda.provision.pallet :as pp])) + +(s/def Cloud + {:fqdn s/Str + :secret-name s/Str + :cluster-issuer s/Str + :db-name s/Str + :db-user-name s/Str + :db-user-password s/Str + :admin-user s/Str + :admin-password s/Str + :storage-size s/Str}) + +(def MeissaCloudInfra {:cloud Cloud}) + +(def cloud "cloud") + +(defn init + [facility user config] + (let [facility-name (name facility)] + (p/provision-log ::pp/pallet facility-name cloud + ::p/info "init") + (p/copy-resources-to-tmp + ::pp/pallet facility-name cloud + [{:filename "install-as-root.sh" :config {:user user}}]))) + + +(defn install + [facility user config] + (let [facility-name (name facility)] + (p/provision-log ::pp/pallet facility-name cloud + ::p/info "install") + (p/copy-resources-to-user + ::pp/pallet user facility-name cloud + [{:filename "pod-running.sh"} + {:filename "cloud-persistent-volume.yml" :config config} + {:filename "cloud-secret.yml" :config config} + {:filename "cloud-service.yml"} + {:filename "cloud-pvc.yml" :config config} + {:filename "cloud-pod.yml" :config config} + {:filename "cloud-ingress.yml" :config config} + {:filename "configure-as-user.sh"} + {:filename "verify.sh" :config config}]) + (p/exec-file-on-target-as-root + ::pp/pallet facility-name cloud "install-as-root.sh"))) + +(defn configure + [facility user config] + (let [facility-name (name facility)] + (p/provision-log ::pp/pallet facility-name cloud + ::p/info "configure") + (p/exec-file-on-target-as-user + ::pp/pallet user facility-name cloud "configure-as-user.sh"))) diff --git a/main/src/meissa/pallet/meissa_cloud/infra/postgres.clj b/main/src/meissa/pallet/meissa_cloud/infra/postgres.clj new file mode 100644 index 0000000..60a3e8b --- /dev/null +++ b/main/src/meissa/pallet/meissa_cloud/infra/postgres.clj @@ -0,0 +1,47 @@ +(ns meissa.pallet.meissa-cloud.infra.postgres + (:require + [schema.core :as s] + [dda.provision :as p] + [dda.provision.pallet :as pp])) + +(s/def Postgres {:db-user-name s/Str :db-user-password s/Str}) + +(def MeissaPostgresInfra {:postgres Postgres}) + +(def postgres "postgres") + +(defn init + [facility user config] + (let [facility-name (name facility)] + (p/provision-log ::pp/pallet facility-name postgres + ::p/info "init") + (p/copy-resources-to-tmp + ::pp/pallet facility-name postgres + [{:filename "install-as-root.sh" :config {:user user}}]))) + + +(defn install + [facility user config] + (let [facility-name (name facility)] + (p/provision-log ::pp/pallet facility-name postgres + ::p/info "install") + (p/copy-resources-to-user + ::pp/pallet user facility-name postgres + [{:filename "postgres-persistent-volume.yml"} + {:filename "postgres-secret.yml" :config config} + {:filename "postgres-config.yml"} + {:filename "postgres-service.yml"} + {:filename "postgres-pvc.yml"} + {:filename "postgres-deployment.yml" :config config} + {:filename "configure-as-user.sh"} + {:filename "verify.sh"}]) + (p/exec-file-on-target-as-root + ::pp/pallet facility-name postgres "install-as-root.sh"))) + +(defn configure + [facility user config] + (let [facility-name (name facility)] + (p/provision-log ::pp/pallet facility-name postgres + ::p/info "configure") + (p/exec-file-on-target-as-user + ::pp/pallet user facility-name postgres "configure-as-user.sh"))) diff --git a/project.clj b/project.clj index c6dc3f6..4d4e476 100644 --- a/project.clj +++ b/project.clj @@ -1,41 +1,55 @@ -(defproject org.domaindrivenarchitecture/c4k-cloud "0.1.3-SNAPSHOT" - :description "cloud c4k-installation package" - :url "https://domaindrivenarchitecture.org" - :license {:name "Apache License, Version 2.0" - :url "https://www.apache.org/licenses/LICENSE-2.0.html"} - :dependencies [[org.clojure/clojure "1.10.3"] - [org.clojure/tools.reader "1.3.4"] - [org.domaindrivenarchitecture/c4k-common-clj "0.2.8"]] +(defproject meissa/meissa-cloud "1.0.2-SNAPSHOT" + :description "Crate to install cloud" + :url "https://meissa-gmbh.de" + :license {:name "meissa commercial license" + :url "https://www.meissa-gmbh.de"} + :dependencies [[dda/dda-pallet "4.0.3"] + [dda/dda-k8s-crate "1.0.1"] + [orchestra "2021.01.01-1"]] :target-path "target/%s/" - :source-paths ["src/main/cljc" - "src/main/clj"] - :resource-paths ["src/main/resources"] - :repositories [["snapshots" :clojars] - ["releases" :clojars]] - :deploy-repositories [["snapshots" :clojars] - ["releases" :clojars]] - :profiles {:test {:test-paths ["src/test/cljc"] - :resource-paths ["src/test/resources"] - :dependencies [[dda/data-test "0.1.1"]]} - :dev {:plugins [[lein-shell "0.5.0"]]} - :uberjar {:aot :all - :main dda.c4k-cloud.uberjar - :uberjar-name "c4k-cloud-standalone.jar" - :dependencies [[org.clojure/tools.cli "1.0.206"] - [ch.qos.logback/logback-classic "1.3.0-alpha4" - :exclusions [com.sun.mail/javax.mail]] + :source-paths ["main/src"] + :resource-paths ["main/resources"] + :repositories [["clojars" "https://clojars.org/repo"] + ["snapshots" + "https://artifact.prod.meissa-gmbh.de/repository/maven-snapshots/"] + ["releases" + "https://artifact.prod.meissa-gmbh.de/repository/maven-releases/"]] + :deploy-repositories [["snapshots" "https://artifact.prod.meissa-gmbh.de/repository/maven-snapshots/"] + ["releases" "https://artifact.prod.meissa-gmbh.de/repository/maven-releases/"]] + :profiles {:dev {:source-paths ["integration/src" + "test/src" + "uberjar/src"] + :resource-paths ["integration/resources" + "test/resources"] + :dependencies + [[org.clojure/test.check "1.1.0"] + [dda/data-test "0.1.1"] + [dda/pallet "0.9.1" :classifier "tests"] + [ch.qos.logback/logback-classic "1.3.0-alpha5"] + [org.slf4j/jcl-over-slf4j "2.0.0-alpha1"]] + :plugins + [[lein-sub "0.3.0"]] + :leiningen/reply + {:dependencies [[org.slf4j/jcl-over-slf4j "1.8.0-beta2"]] + :exclusions [commons-logging]}} + :test {:test-paths ["test/src"] + :resource-paths ["test/resources"] + :dependencies [[dda/pallet "0.9.1" :classifier "tests"]]} + :uberjar {:source-paths ["uberjar/src"] + :resource-paths ["uberjar/resources"] + :aot :all + :main meissa.pallet.meissa-cloud.main + :uberjar-name "meissa-cloud-standalone.jar" + :dependencies [[org.clojure/tools.cli "1.0.194"] + [ch.qos.logback/logback-classic "1.3.0-alpha5"] [org.slf4j/jcl-over-slf4j "2.0.0-alpha1"]]}} - :release-tasks [["test"] - ["vcs" "assert-committed"] + :release-tasks [["vcs" "assert-committed"] ["change" "version" "leiningen.release/bump-version" "release"] ["vcs" "commit"] ["vcs" "tag"] - ["change" "version" "leiningen.release/bump-version"]] - :aliases {"native" ["shell" - "native-image" - "--report-unsupported-elements-at-runtime" - "--initialize-at-build-time" - "-jar" "target/uberjar/c4k-cloud-standalone.jar" - "-H:ResourceConfigurationFiles=graalvm-resource-config.json" - "-H:Log=registerResource" - "-H:Name=target/graalvm/${:name}"]}) + ["deploy"] + ["uberjar"] + ["change" "version" "leiningen.release/bump-version"] + ["vcs" "commit"] + ["vcs" "push"]] + :local-repo-classpath true) diff --git a/targets.edn b/targets.edn new file mode 100644 index 0000000..515f51a --- /dev/null +++ b/targets.edn @@ -0,0 +1,3 @@ +{:existing [{:node-name "cloud" + :node-ip "168.119.190.126"}] + :provisioning-user {:login "root"}} diff --git a/test/resources/meissa/pallet/meissa_cloud/convention_test/should_generate_infra_for_convention.edn b/test/resources/meissa/pallet/meissa_cloud/convention_test/should_generate_infra_for_convention.edn new file mode 100644 index 0000000..9b4e226 --- /dev/null +++ b/test/resources/meissa/pallet/meissa_cloud/convention_test/should_generate_infra_for_convention.edn @@ -0,0 +1,30 @@ +{:input {:user :k8s + :external-ip "95.217.221.140" + :fqdn "cloud.test.meissa-gmbh.de" + :cert-manager :letsencrypt-staging-issuer + :db-user-password "test1234" + :admin-user "root" + :admin-password "test1234" + :storage-size 50 + :restic-repository "test4321" + :aws-access-key-id "10" + :aws-secret-access-key "secret" + :restic-password "test4321"} + :expected {:meissa-cloud + {:user :k8s + :backup {:restic-repository "test4321" + :aws-access-key-id "10" + :aws-secret-access-key "secret" + :restic-password "test4321"} + :cloud {:fqdn "cloud.test.meissa-gmbh.de" + :secret-name "cloud-test-meissa-gmbh-de" + :cluster-issuer "letsencrypt-staging-issuer" + :db-name "cloud" + :db-user-password "test1234" + :db-user-name "cloud" + :admin-user "root" + :admin-password "test1234" + :storage-size "50"} + :postgres {:db-user-password "test1234" + :db-user-name "cloud"}}} + } diff --git a/test/resources/meissa/pallet/meissa_cloud/convention_test/should_generate_k8s_convention.edn b/test/resources/meissa/pallet/meissa_cloud/convention_test/should_generate_k8s_convention.edn new file mode 100644 index 0000000..0478a77 --- /dev/null +++ b/test/resources/meissa/pallet/meissa_cloud/convention_test/should_generate_k8s_convention.edn @@ -0,0 +1,30 @@ +{:input {:user :k8s + :external-ip "95.217.221.140" + :fqdn "cloud.test.meissa-gmbh.de" + :cert-manager :letsencrypt-staging-issuer + :db-user-password "test1234" + :admin-user "root" + :admin-password "test1234" + :storage-size 50 + :restic-repository "cloud" + :aws-access-key-id "10" + :aws-secret-access-key "secret" + :restic-password "test4321"} + :expected {:user :k8s, :k8s {:external-ip "95.217.221.140"}, + :cert-manager :letsencrypt-staging-issuer}} + +{:input {:user :k8s + :external-ip "95.217.221.140" + :fqdn "cloud.test.meissa-gmbh.de" + :cert-manager :letsencrypt-staging-issuer + :db-user-password "test1234" + :admin-user "root" + :admin-password "test1234" + :storage-size 50 + :restic-repository "cloud" + :aws-access-key-id "10" + :aws-secret-access-key "secret" + :restic-password "test4321" + :u18-04 true} + :expected {:user :k8s, :k8s {:external-ip "95.217.221.140", :u18-04 true} + :cert-manager :letsencrypt-staging-issuer}} diff --git a/test/resources/meissa/pallet/meissa_cloud/convention_test/should_validate_input.0.edn b/test/resources/meissa/pallet/meissa_cloud/convention_test/should_validate_input.0.edn new file mode 100644 index 0000000..7dda0b7 --- /dev/null +++ b/test/resources/meissa/pallet/meissa_cloud/convention_test/should_validate_input.0.edn @@ -0,0 +1,14 @@ +{:input {:user :k8s + :external-ip "95.217.221.140" + :fqdn "cloud.test.meissa-gmbh.de" + :cert-manager :letsencrypt-staging-issuer + :db-user-password "test1234" + :admin-user "root" + :admin-password "test1234" + :storage-size 50 + :restic-repository "test4321" + :aws-access-key-id "10" + :aws-secret-access-key "secret" + :restic-password "test4321"} + :expected true + } diff --git a/test/resources/meissa/pallet/meissa_cloud/convention_test/should_validate_input.1.edn b/test/resources/meissa/pallet/meissa_cloud/convention_test/should_validate_input.1.edn new file mode 100644 index 0000000..b8aec50 --- /dev/null +++ b/test/resources/meissa/pallet/meissa_cloud/convention_test/should_validate_input.1.edn @@ -0,0 +1,14 @@ +{:input {:user :k8s + :external-ip "95.217.221.140" + :fqdn "cloud.test.meissa-gmbh.de" + :cert-manager :letsencrypt-staging-issuer + :db-user-password "test-1234" + :admin-user "root" + :admin-password "test1234" + :storage-size 50 + :restic-repository "test4321" + :aws-access-key-id "10" + :aws-secret-access-key "secret" + :restic-password "test4321"} + :expected false + } diff --git a/test/resources/meissa/pallet/meissa_cloud/convention_test/should_validate_input.2.edn b/test/resources/meissa/pallet/meissa_cloud/convention_test/should_validate_input.2.edn new file mode 100644 index 0000000..ada2a9e --- /dev/null +++ b/test/resources/meissa/pallet/meissa_cloud/convention_test/should_validate_input.2.edn @@ -0,0 +1,14 @@ +{:input {:user :k8s + :external-ip "95.217.221.140" + :fqdn "cloud.test.meissa-gmbh.de" + :cert-manager :letsencrypt-staging-issuer + :db-user-password "test1234" + :admin-user "root" + :admin-password "test1234" + :storage-size 50 + :restic-repository "test4321" + :aws-access-key-id "1$0" + :aws-secret-access-key "secret" + :restic-password "test4321"} + :expected false + } diff --git a/test/src/meissa/pallet/meissa_cloud/app_test.clj b/test/src/meissa/pallet/meissa_cloud/app_test.clj new file mode 100644 index 0000000..9daafa9 --- /dev/null +++ b/test/src/meissa/pallet/meissa_cloud/app_test.clj @@ -0,0 +1,31 @@ +(ns meissa.pallet.meissa-cloud.app-test + (:require + [clojure.test :refer :all] + [schema.core :as s] + [meissa.pallet.meissa-cloud.app :as sut])) + +(s/set-fn-validation! true) + +(s/def test-convention-conf + {:user :k8s + :external-ip "12.121.111.121" + :fqdn "some.domain.de" + :cert-manager :letsencrypt-staging-issuer + :db-user-password "test1234" + :admin-user "root" + :admin-password "test1234" + :storage-size 50 + :restic-repository "cloud" + :aws-access-key-id "10" + :aws-secret-access-key "secret" + :restic-password "test4321"}) + +(deftest app-config + (testing + "test plan-def" + (is (map? (sut/app-configuration-resolved test-convention-conf))))) + +(deftest plan-def + (testing + "test plan-def" + (is (map? sut/with-cloud)))) diff --git a/test/src/meissa/pallet/meissa_cloud/convention/bash_php_test.clj b/test/src/meissa/pallet/meissa_cloud/convention/bash_php_test.clj new file mode 100644 index 0000000..787fd6c --- /dev/null +++ b/test/src/meissa/pallet/meissa_cloud/convention/bash_php_test.clj @@ -0,0 +1,20 @@ +(ns meissa.pallet.meissa-cloud.convention.bash-php-test + (:require + [clojure.test :refer :all] + [meissa.pallet.meissa-cloud.convention.bash-php :as sut])) + + +(deftest test-it + (is (= false + (sut/bash-php-env-string? 4))) + (is (= false + (sut/bash-php-env-string? "hal-lo"))) + (is (= false + (sut/bash-php-env-string? "hal--lo"))) + (is (= false + (sut/bash-php-env-string? "hal\\lo"))) + (is (= true + (sut/bash-php-env-string? "test"))) + (is (= true + (sut/bash-php-env-string? "test123"))) + ) \ No newline at end of file diff --git a/test/src/meissa/pallet/meissa_cloud/convention/bash_test.clj b/test/src/meissa/pallet/meissa_cloud/convention/bash_test.clj new file mode 100644 index 0000000..ce51ba6 --- /dev/null +++ b/test/src/meissa/pallet/meissa_cloud/convention/bash_test.clj @@ -0,0 +1,22 @@ +(ns meissa.pallet.meissa-cloud.convention.bash-test + (:require + [clojure.test :refer :all] + [meissa.pallet.meissa-cloud.convention.bash :as sut])) + + +(deftest test-it + (is (= false + (sut/bash-env-string? 4))) + (is (= false + (sut/bash-env-string? "1$0"))) + (is (= false + (sut/bash-env-string? "'hallo"))) + (is (= false + (sut/bash-env-string? "hallo\""))) + (is (= false + (sut/bash-env-string? "hall$o"))) + (is (= true + (sut/bash-env-string? "test"))) + (is (= true + (sut/bash-env-string? "test123"))) + ) \ No newline at end of file diff --git a/test/src/meissa/pallet/meissa_cloud/convention_test.clj b/test/src/meissa/pallet/meissa_cloud/convention_test.clj new file mode 100644 index 0000000..e5e30c1 --- /dev/null +++ b/test/src/meissa/pallet/meissa_cloud/convention_test.clj @@ -0,0 +1,18 @@ +(ns meissa.pallet.meissa-cloud.convention-test + (:require + [clojure.test :refer :all] + [data-test :refer :all] + [meissa.pallet.meissa-cloud.convention :as sut] + [clojure.spec.alpha :as sp])) + +(defdatatest should-generate-infra-for-convention [input expected] + (is (= expected + (sut/infra-configuration input)))) + +(defdatatest should-generate-k8s-convention [input expected] + (is (= expected + (sut/k8s-convention-configuration input)))) + +(defdatatest should-validate-input [input expected] + (is (= expected + (sp/valid? sut/cloud-convention-resolved? input)))) diff --git a/uberjar/resources/localhost-target.edn b/uberjar/resources/localhost-target.edn new file mode 100644 index 0000000..d49f818 --- /dev/null +++ b/uberjar/resources/localhost-target.edn @@ -0,0 +1,2 @@ +{:existing [{:node-name "localhost" + :node-ip "127.0.0.1"}]} diff --git a/uberjar/resources/logback.xml b/uberjar/resources/logback.xml new file mode 100644 index 0000000..8985f2b --- /dev/null +++ b/uberjar/resources/logback.xml @@ -0,0 +1,50 @@ + + + + %d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n + + + INFO + + + + + logs/pallet.log + + logs/old/pallet.%d{yyyy-MM-dd}.log + 3 + + + %date %level [%thread] %logger{10} %msg%n + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/uberjar/src/meissa/pallet/meissa_cloud/main.clj b/uberjar/src/meissa/pallet/meissa_cloud/main.clj new file mode 100644 index 0000000..60a33f2 --- /dev/null +++ b/uberjar/src/meissa/pallet/meissa_cloud/main.clj @@ -0,0 +1,57 @@ +(ns meissa.pallet.meissa-cloud.main + (:gen-class) + (:require + [clojure.string :as str] + [clojure.tools.cli :as cli] + [dda.pallet.core.main-helper :as mh] + [dda.pallet.core.app :as core-app] + [meissa.pallet.meissa-cloud.app :as app])) + +(def cli-options + [["-h" "--help"] + ["-c" "--configure"] + ["-t" "--targets example-targets.edn" "edn file containing the targets to install on." + :default "localhost-target.edn"] + ["-v" "--verbose"]]) + +(defn usage [options-summary] + (str/join + \newline + ["meissa-cloud installs & configures a single host kubernetes cluster with Cloud installed" + "" + "Usage: java -jar meissa-cloud-standalone.jar [options] cloud.edn" + "" + "Options:" + options-summary + "" + "cloud.edn" + " - follows the edn format." + " - has to be a valid CloudConventionConfig" + ""])) + +(defn -main [& args] + (let [{:keys [options arguments errors summary help]} (cli/parse-opts args cli-options) + verbose (if (contains? options :verbose) 1 0)] + (cond + help (mh/exit 0 (usage summary)) + errors (mh/exit 1 (mh/error-msg errors)) + (not= (count arguments) 1) (mh/exit 1 (usage summary)) + (:serverspec options) (if (core-app/existing-serverspec + app/crate-app + {:convention (first arguments) + :targets (:targets options) + :verbosity verbose}) + (mh/exit-test-passed) + (mh/exit-test-failed)) + (:configure options) (if (core-app/existing-configure + app/crate-app + {:convention (first arguments) + :targets (:targets options)}) + (mh/exit-default-success) + (mh/exit-default-error)) + :default (if (core-app/existing-install + app/crate-app + {:convention (first arguments) + :targets (:targets options)}) + (mh/exit-default-success) + (mh/exit-default-error))))) From 303f7ae5f2a64ff2bf81862bea939120da51a6d6 Mon Sep 17 00:00:00 2001 From: leo Date: Tue, 10 Aug 2021 10:43:04 +0200 Subject: [PATCH 04/12] renamed namespaces to nextcloud --- main/resources/backup/backup-restore.yaml | 59 ------------ main/resources/backup/config.yaml | 9 -- main/resources/backup/configure-as-user.sh | 9 -- main/resources/backup/cron.yaml | 65 ------------- main/resources/backup/secret.yaml | 9 -- main/resources/cloud/certificate.yaml | 13 --- main/resources/cloud/cloud-pod.yml.template | 45 --------- main/resources/cloud/ingress.yaml | 26 ------ main/resources/cloud/persistent-volume.yaml | 15 --- main/resources/cloud/pvc.yaml | 16 ---- main/resources/cloud/service.yaml | 9 -- main/resources/cloud/verify.sh.template | 15 --- main/resources/postgres/configure-as-user.sh | 15 --- main/resources/postgres/verify.sh | 8 -- main/src/meissa/pallet/meissa_cloud/app.clj | 61 ------------ .../meissa/pallet/meissa_cloud/convention.clj | 93 ------------------- .../pallet/meissa_cloud/convention/bash.clj | 10 -- .../meissa_cloud/convention/bash_php.clj | 11 --- main/src/meissa/pallet/meissa_cloud/infra.clj | 51 ---------- .../pallet/meissa_cloud/infra/backup.clj | 39 -------- .../pallet/meissa_cloud/infra/cloud.clj | 57 ------------ .../pallet/meissa_cloud/infra/postgres.clj | 47 ---------- project.clj | 4 +- .../{c4k_cloud => c4k_nextcloud}/uberjar.clj | 6 +- .../{c4k_cloud => c4k_nextcloud}/backup.cljc | 2 +- .../{c4k_cloud => c4k_nextcloud}/cloud.cljc | 35 +++---- .../{c4k_cloud => c4k_nextcloud}/core.cljc | 26 +++--- .../{c4k_cloud => c4k_nextcloud}/browser.cljs | 31 ++++--- src/main/resources/backup/backup-restore.yaml | 27 +++--- src/main/resources/backup/cron.yaml | 33 +++---- src/main/resources/backup/secret.yaml | 2 +- .../resources/cloud/configure-as-user.sh | 0 src/main/resources/cloud/deployment.yaml | 50 ++++++---- src/main/resources/cloud/ingress.yaml | 2 +- .../cloud/install-as-root.sh.template | 0 .../resources/cloud/persistent-volume.yaml | 3 +- .../main}/resources/cloud/pod-running.sh | 0 src/main/resources/cloud/pvc.yaml | 5 +- .../main}/resources/cloud/secret.yaml | 0 src/main/resources/cloud/service.yaml | 4 +- src/main/resources/logback.xml | 50 ---------- .../postgres/install-as-root.sh.template | 0 .../resources/postgres/postgres-config.yaml | 0 .../postgres/postgres-deployment.yaml | 0 .../postgres/postgres-persistent-volume.yaml | 0 .../main/resources/postgres/postgres-pvc.yaml | 0 .../resources/postgres/postgres-secret.yaml | 0 .../resources/postgres/postgres-service.yaml | 0 src/test/cljc/dda/c4k_cloud/backup_test.cljc | 93 ------------------- src/test/cljc/dda/c4k_cloud/cloud_test.cljc | 80 ---------------- src/test/cljc/dda/c4k_cloud/core_test.cljc | 35 ------- .../meissa/pallet/meissa_cloud/app_test.clj | 8 +- .../src/meissa/pallet/meissa_cloud/main.clj | 12 +-- 53 files changed, 131 insertions(+), 1059 deletions(-) delete mode 100644 main/resources/backup/backup-restore.yaml delete mode 100644 main/resources/backup/config.yaml delete mode 100644 main/resources/backup/configure-as-user.sh delete mode 100644 main/resources/backup/cron.yaml delete mode 100644 main/resources/backup/secret.yaml delete mode 100644 main/resources/cloud/certificate.yaml delete mode 100644 main/resources/cloud/cloud-pod.yml.template delete mode 100644 main/resources/cloud/ingress.yaml delete mode 100644 main/resources/cloud/persistent-volume.yaml delete mode 100644 main/resources/cloud/pvc.yaml delete mode 100644 main/resources/cloud/service.yaml delete mode 100644 main/resources/cloud/verify.sh.template delete mode 100644 main/resources/postgres/configure-as-user.sh delete mode 100644 main/resources/postgres/verify.sh delete mode 100644 main/src/meissa/pallet/meissa_cloud/app.clj delete mode 100644 main/src/meissa/pallet/meissa_cloud/convention.clj delete mode 100644 main/src/meissa/pallet/meissa_cloud/convention/bash.clj delete mode 100644 main/src/meissa/pallet/meissa_cloud/convention/bash_php.clj delete mode 100644 main/src/meissa/pallet/meissa_cloud/infra.clj delete mode 100644 main/src/meissa/pallet/meissa_cloud/infra/backup.clj delete mode 100644 main/src/meissa/pallet/meissa_cloud/infra/cloud.clj delete mode 100644 main/src/meissa/pallet/meissa_cloud/infra/postgres.clj rename src/main/clj/dda/{c4k_cloud => c4k_nextcloud}/uberjar.clj (93%) rename src/main/cljc/dda/{c4k_cloud => c4k_nextcloud}/backup.cljc (98%) rename src/main/cljc/dda/{c4k_cloud => c4k_nextcloud}/cloud.cljc (50%) rename src/main/cljc/dda/{c4k_cloud => c4k_nextcloud}/core.cljc (66%) rename src/main/cljs/dda/{c4k_cloud => c4k_nextcloud}/browser.cljs (65%) rename {main => src/main}/resources/cloud/configure-as-user.sh (100%) rename {main => src/main}/resources/cloud/install-as-root.sh.template (100%) rename {main => src/main}/resources/cloud/pod-running.sh (100%) mode change 100755 => 100644 rename {main => src/main}/resources/cloud/secret.yaml (100%) delete mode 100644 src/main/resources/logback.xml rename {main => src/main}/resources/postgres/install-as-root.sh.template (100%) rename main/resources/postgres/postgres-config.yml => src/main/resources/postgres/postgres-config.yaml (100%) rename main/resources/postgres/postgres-deployment.yml.template => src/main/resources/postgres/postgres-deployment.yaml (100%) rename main/resources/postgres/postgres-persistent-volume.yml => src/main/resources/postgres/postgres-persistent-volume.yaml (100%) rename main/resources/postgres/postgres-pvc.yml => src/main/resources/postgres/postgres-pvc.yaml (100%) rename main/resources/postgres/postgres-secret.yml.template => src/main/resources/postgres/postgres-secret.yaml (100%) rename main/resources/postgres/postgres-service.yml => src/main/resources/postgres/postgres-service.yaml (100%) delete mode 100644 src/test/cljc/dda/c4k_cloud/backup_test.cljc delete mode 100644 src/test/cljc/dda/c4k_cloud/cloud_test.cljc delete mode 100644 src/test/cljc/dda/c4k_cloud/core_test.cljc diff --git a/main/resources/backup/backup-restore.yaml b/main/resources/backup/backup-restore.yaml deleted file mode 100644 index c13e166..0000000 --- a/main/resources/backup/backup-restore.yaml +++ /dev/null @@ -1,59 +0,0 @@ -kind: Pod -apiVersion: v1 -metadata: - name: backup-restore - labels: - app.kubernetes.io/name: backup-restore - app.kubernetes.io/part-of: cloud -spec: - containers: - - name: backup-app - image: domaindrivenarchitecture/c4k-cloud-backup - imagePullPolicy: IfNotPresent - command: ["/entrypoint-start-and-wait.sh"] - env: - - name: POSTGRES_USER_FILE - value: /var/run/secrets/cloud-secrets/postgres-user - - name: POSTGRES_DB_FILE - value: /var/run/secrets/cloud-secrets/postgres-db - - name: POSTGRES_PASSWORD_FILE - value: /var/run/secrets/cloud-secrets/postgres-password - - name: POSTGRES_HOST - value: "postgresql-service:5432" - - name: POSTGRES_SERVICE - value: "postgresql-service" - - name: POSTGRES_PORT - value: "5432" - - name: AWS_DEFAULT_REGION - value: eu-central-1 - - name: AWS_ACCESS_KEY_ID_FILE - value: /var/run/secrets/backup-secrets/aws-access-key-id - - name: AWS_SECRET_ACCESS_KEY_FILE - value: /var/run/secrets/backup-secrets/aws-secret-access-key - - name: RESTIC_REPOSITORY - valueFrom: - configMapKeyRef: - name: backup-config - key: restic-repository - - name: RESTIC_PASSWORD_FILE - value: /var/run/secrets/backup-secrets/restic-password - volumeMounts: - - name: cloud-data-volume - mountPath: /var/backups - - name: backup-secret-volume - mountPath: /var/run/secrets/backup-secrets - readOnly: true - - name: cloud-secret-volume - mountPath: /var/run/secrets/cloud-secrets - readOnly: true - volumes: - - name: cloud-data-volume - persistentVolumeClaim: - claimName: cloud-pvc - - name: cloud-secret-volume - secret: - secretName: cloud-secret - - name: backup-secret-volume - secret: - secretName: backup-secret - restartPolicy: OnFailure \ No newline at end of file diff --git a/main/resources/backup/config.yaml b/main/resources/backup/config.yaml deleted file mode 100644 index 17aa35c..0000000 --- a/main/resources/backup/config.yaml +++ /dev/null @@ -1,9 +0,0 @@ -apiVersion: v1 -kind: ConfigMap -metadata: - name: backup-config - labels: - app.kubernetes.io/name: backup - app.kubernetes.io/part-of: cloud -data: - restic-repository: restic-repository \ No newline at end of file diff --git a/main/resources/backup/configure-as-user.sh b/main/resources/backup/configure-as-user.sh deleted file mode 100644 index a5a099c..0000000 --- a/main/resources/backup/configure-as-user.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/bash - -kubectl delete --ignore-not-found=true -f backup-secret.yml -kubectl delete --ignore-not-found=true -f backup-config.yml -kubectl delete --ignore-not-found=true -f backup-cron.yml - -kubectl apply -f backup-secret.yml -kubectl apply -f backup-config.yml -kubectl apply -f backup-cron.yml diff --git a/main/resources/backup/cron.yaml b/main/resources/backup/cron.yaml deleted file mode 100644 index 8bb54bc..0000000 --- a/main/resources/backup/cron.yaml +++ /dev/null @@ -1,65 +0,0 @@ -apiVersion: batch/v1beta1 -kind: CronJob -metadata: - name: cloud-backup - labels: - app.kubernetes.part-of: cloud -spec: - schedule: "10 23 * * *" - successfulJobsHistoryLimit: 0 - failedJobsHistoryLimit: 0 - jobTemplate: - spec: - template: - spec: - containers: - - name: backup-app - image: domaindrivenarchitecture/meissa-cloud-backup - imagePullPolicy: IfNotPresent - command: ["/entrypoint.sh"] - env: - - name: POSTGRES_USER_FILE - value: /var/run/secrets/cloud-secrets/postgres-user - - name: POSTGRES_DB_FILE - value: /var/run/secrets/cloud-secrets/postgres-db - - name: POSTGRES_PASSWORD_FILE - value: /var/run/secrets/cloud-secrets/postgres-password - - name: POSTGRES_HOST - value: "postgresql-service:5432" - - name: POSTGRES_SERVICE - value: "postgresql-service" - - name: POSTGRES_PORT - value: "5432" - - name: AWS_DEFAULT_REGION - value: eu-central-1 - - name: AWS_ACCESS_KEY_ID_FILE - value: /var/run/secrets/backup-secrets/aws-access-key-id - - name: AWS_SECRET_ACCESS_KEY_FILE - value: /var/run/secrets/backup-secrets/aws-secret-access-key - - name: RESTIC_REPOSITORY - valueFrom: - configMapKeyRef: - name: backup-config - key: restic-repository - - name: RESTIC_PASSWORD_FILE - value: /var/run/secrets/backup-secrets/restic-password - volumeMounts: - - name: cloud-data-volume - mountPath: /var/backups - - name: backup-secret-volume - mountPath: /var/run/secrets/backup-secrets - readOnly: true - - name: cloud-secret-volume - mountPath: /var/run/secrets/cloud-secrets - readOnly: true - volumes: - - name: cloud-data-volume - persistentVolumeClaim: - claimName: cloud-pvc - - name: cloud-secret-volume - secret: - secretName: cloud-secret - - name: backup-secret-volume - secret: - secretName: backup-secret - restartPolicy: OnFailure \ No newline at end of file diff --git a/main/resources/backup/secret.yaml b/main/resources/backup/secret.yaml deleted file mode 100644 index 4b68578..0000000 --- a/main/resources/backup/secret.yaml +++ /dev/null @@ -1,9 +0,0 @@ -apiVersion: v1 -kind: Secret -metadata: - name: backup-secret -type: Opaque -stringData: - aws-access-key-id: aws-access-key-id - aws-secret-access-key: aws-secret-access-key - restic-password: restic-password \ No newline at end of file diff --git a/main/resources/cloud/certificate.yaml b/main/resources/cloud/certificate.yaml deleted file mode 100644 index 054965b..0000000 --- a/main/resources/cloud/certificate.yaml +++ /dev/null @@ -1,13 +0,0 @@ -apiVersion: cert-manager.io/v1alpha2 -kind: Certificate -metadata: - name: cloud-cert - namespace: default -spec: - secretName: cloud-secret - commonName: fqdn - dnsNames: - - fqdn - issuerRef: - name: letsencrypt-staging-issuer - kind: ClusterIssuer \ No newline at end of file diff --git a/main/resources/cloud/cloud-pod.yml.template b/main/resources/cloud/cloud-pod.yml.template deleted file mode 100644 index eac26ec..0000000 --- a/main/resources/cloud/cloud-pod.yml.template +++ /dev/null @@ -1,45 +0,0 @@ -kind: Pod -apiVersion: v1 -metadata: - name: cloud - labels: - app.kubernetes.io/name: cloud -spec: - shareProcessNamespace: true - containers: - - name: cloud-app - image: domaindrivenarchitecture/meissa-cloud-app - imagePullPolicy: IfNotPresent - ports: - - containerPort: 80 - env: - - name: NEXTCLOUD_ADMIN_USER_FILE - value: /var/run/secrets/cloud-secrets/nextcloud-admin-user - - name: NEXTCLOUD_ADMIN_PASSWORD_FILE - value: /var/run/secrets/cloud-secrets/nextcloud-admin-password - - name: NEXTCLOUD_TRUSTED_DOMAINS - value: "{{fqdn}}" - - name: POSTGRES_USER_FILE - value: /var/run/secrets/cloud-secrets/postgres-user - - name: POSTGRES_PASSWORD_FILE - value: /var/run/secrets/cloud-secrets/postgres-password - - name: POSTGRES_DB_FILE - value: /var/run/secrets/cloud-secrets/postgres-db - - name: POSTGRES_HOST - value: "postgresql-service:5432" - volumeMounts: - - name: cloud-data-volume - mountPath: /var/www/html - - name: cloud-secret-volume - mountPath: /var/run/secrets/cloud-secrets - readOnly: true - volumes: - - name: cloud-data-volume - persistentVolumeClaim: - claimName: cloud-pvc - - name: cloud-secret-volume - secret: - secretName: cloud-secret - - name: backup-secret-volume - secret: - secretName: backup-secret diff --git a/main/resources/cloud/ingress.yaml b/main/resources/cloud/ingress.yaml deleted file mode 100644 index cc5a0df..0000000 --- a/main/resources/cloud/ingress.yaml +++ /dev/null @@ -1,26 +0,0 @@ -apiVersion: extensions/v1beta1 -kind: Ingress -metadata: - name: ingress-cloud - annotations: - cert-manager.io/cluster-issuer: letsencrypt-staging-issuer - nginx.ingress.kubernetes.io/proxy-body-size: "256m" - nginx.ingress.kubernetes.io/ssl-redirect: "true" - nginx.ingress.kubernetes.io/rewrite-target: / - nginx.ingress.kubernetes.io/proxy-connect-timeout: "300" - nginx.ingress.kubernetes.io/proxy-send-timeout: "300" - nginx.ingress.kubernetes.io/proxy-read-timeout: "300" - namespace: default -spec: - tls: - - hosts: - - fqdn - secretName: cloud-secret - rules: - - host: fqdn - http: - paths: - - path: / - backend: - serviceName: cloud-service - servicePort: 80 diff --git a/main/resources/cloud/persistent-volume.yaml b/main/resources/cloud/persistent-volume.yaml deleted file mode 100644 index 7c3f89e..0000000 --- a/main/resources/cloud/persistent-volume.yaml +++ /dev/null @@ -1,15 +0,0 @@ -kind: PersistentVolume -apiVersion: v1 -metadata: - name: cloud-pv-volume - labels: - type: local - app: cloud -spec: - storageClassName: manual - accessModes: - - ReadWriteOnce - capacity: - storage: {{storage-size}}Gi #??? 30Gi? - hostPath: - path: "/var/cloud" diff --git a/main/resources/cloud/pvc.yaml b/main/resources/cloud/pvc.yaml deleted file mode 100644 index d4f7e04..0000000 --- a/main/resources/cloud/pvc.yaml +++ /dev/null @@ -1,16 +0,0 @@ -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: cloud-pvc - labels: - app: cloud -spec: - storageClassName: manual - accessModes: - - ReadWriteOnce - resources: - requests: - storage: {{storage-size}}Gi #??? 30Gi? - selector: - matchLabels: - app: cloud diff --git a/main/resources/cloud/service.yaml b/main/resources/cloud/service.yaml deleted file mode 100644 index 7fcd0d7..0000000 --- a/main/resources/cloud/service.yaml +++ /dev/null @@ -1,9 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: cloud-service -spec: - selector: - app.kubernetes.io/name: cloud #??? - ports: - - port: 80 diff --git a/main/resources/cloud/verify.sh.template b/main/resources/cloud/verify.sh.template deleted file mode 100644 index ad362e5..0000000 --- a/main/resources/cloud/verify.sh.template +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/bash - -echo -e "\n====================\n" -echo -e "cloud is running, ingress exists" -echo -e "\n====================\n" -kubectl get all - -echo -e "\n====================\n" -echo -e "shows certificate with subject" -echo -e "CN={{fqdn}}" -echo -e "issuer: CN=Fake LE Intermediate X1" -echo -e "\n====================\n" -curl --insecure -v https://{{fqdn}} - -echo -e "\n" diff --git a/main/resources/postgres/configure-as-user.sh b/main/resources/postgres/configure-as-user.sh deleted file mode 100644 index 3d76fbe..0000000 --- a/main/resources/postgres/configure-as-user.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/bash - -kubectl delete --ignore-not-found=true -f postgres-deployment.yml -kubectl delete --ignore-not-found=true -f postgres-pvc.yml -kubectl delete --ignore-not-found=true -f postgres-service.yml -kubectl delete --ignore-not-found=true -f postgres-config.yml -kubectl delete --ignore-not-found=true -f postgres-secret.yml -kubectl delete --ignore-not-found=true -f postgres-persistent-volume.yml - -kubectl apply -f postgres-persistent-volume.yml -kubectl apply -f postgres-secret.yml -kubectl apply -f postgres-config.yml -kubectl apply -f postgres-service.yml -kubectl apply -f postgres-pvc.yml -kubectl apply -f postgres-deployment.yml diff --git a/main/resources/postgres/verify.sh b/main/resources/postgres/verify.sh deleted file mode 100644 index b9f0730..0000000 --- a/main/resources/postgres/verify.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/bash - -echo -e "\n====================\n" -echo -e "postgres is running" -echo -e "\n====================\n" -kubectl get all - -echo -e "\n" diff --git a/main/src/meissa/pallet/meissa_cloud/app.clj b/main/src/meissa/pallet/meissa_cloud/app.clj deleted file mode 100644 index 1547082..0000000 --- a/main/src/meissa/pallet/meissa_cloud/app.clj +++ /dev/null @@ -1,61 +0,0 @@ -(ns meissa.pallet.meissa-cloud.app - (:require - [schema.core :as s] - [dda.pallet.commons.secret :as secret] - [dda.config.commons.map-utils :as mu] - [dda.pallet.core.app :as core-app] - [dda.pallet.dda-config-crate.infra :as config-crate] - [dda.pallet.dda-user-crate.app :as user] - [dda.pallet.dda-k8s-crate.app :as k8s] - [meissa.pallet.meissa-cloud.convention :as convention] - [meissa.pallet.meissa-cloud.infra :as infra])) - -(def with-cloud infra/with-cloud) - -(def CloudConvention convention/CloudConvention) - -(def CloudConventionResolved convention/CloudConventionResolved) - -(def InfraResult convention/InfraResult) - -(def CloudApp - {:group-specific-config - {s/Keyword (merge InfraResult - user/InfraResult - k8s/InfraResult)}}) - -(s/defn ^:always-validate - app-configuration-resolved :- CloudApp - [resolved-convention-config :- CloudConventionResolved - & options] - (let [{:keys [group-key] :or {group-key infra/facility}} options] - (mu/deep-merge - (k8s/app-configuration-resolved - (convention/k8s-convention-configuration resolved-convention-config) :group-key group-key) - {:group-specific-config - {group-key - (convention/infra-configuration resolved-convention-config)}}))) - -(s/defn ^:always-validate - app-configuration :- CloudApp - [convention-config :- CloudConvention - & options] - (let [resolved-convention-config (secret/resolve-secrets convention-config CloudConvention)] - (apply app-configuration-resolved resolved-convention-config options))) - -(s/defmethod ^:always-validate - core-app/group-spec infra/facility - [crate-app - convention-config :- CloudConventionResolved] - (let [app-config (app-configuration-resolved convention-config)] - (core-app/pallet-group-spec - app-config [(config-crate/with-config app-config) - user/with-user - k8s/with-k8s - with-cloud]))) - -(def crate-app (core-app/make-dda-crate-app - :facility infra/facility - :convention-schema CloudConvention - :convention-schema-resolved CloudConventionResolved - :default-convention-file "cloud.edn")) diff --git a/main/src/meissa/pallet/meissa_cloud/convention.clj b/main/src/meissa/pallet/meissa_cloud/convention.clj deleted file mode 100644 index 17223f4..0000000 --- a/main/src/meissa/pallet/meissa_cloud/convention.clj +++ /dev/null @@ -1,93 +0,0 @@ -(ns meissa.pallet.meissa-cloud.convention - (:require - [schema.core :as s] - [dda.pallet.commons.secret :as secret] - [dda.config.commons.map-utils :as mu] - [clojure.spec.alpha :as sp] - [clojure.spec.test.alpha :as st] - [dda.pallet.dda-k8s-crate.convention :as k8s-convention] - [meissa.pallet.meissa-cloud.infra :as infra] - [clojure.string :as str] - [meissa.pallet.meissa-cloud.convention.bash :as bash] - [meissa.pallet.meissa-cloud.convention.bash-php :as bash-php])) - -(def InfraResult {infra/facility infra/MeissaCloudInfra}) - -(s/def CloudConvention - {:user s/Keyword - :external-ip s/Str - :fqdn s/Str - :cert-manager (s/enum :letsencrypt-prod-issuer :letsencrypt-staging-issuer) - :db-user-password secret/Secret - :admin-user s/Str - :admin-password secret/Secret - :storage-size s/Int - :restic-repository s/Str - :aws-access-key-id secret/Secret - :aws-secret-access-key secret/Secret - :restic-password secret/Secret - (s/optional-key :u18-04) (s/enum true)}) - -(def CloudConventionResolved (secret/create-resolved-schema CloudConvention)) - -(sp/def ::user keyword?) -(sp/def ::external-ip string?) -(sp/def ::fqdn string?) -(sp/def ::cert-manager #{:letsencrypt-prod-issuer :letsencrypt-staging-issuer}) -(sp/def ::db-user-password bash-php/bash-php-env-string?) -(sp/def ::admin-user bash-php/bash-php-env-string?) -(sp/def ::admin-password bash-php/bash-php-env-string?) -(sp/def ::storage-size int?) -(sp/def ::restic-repository string?) -(sp/def ::restic-password bash/bash-env-string?) -(sp/def ::aws-access-key-id bash/bash-env-string?) -(sp/def ::aws-secret-access-key bash/bash-env-string?) -(sp/def ::u18-04 #{true}) -(def cloud-convention-resolved? (sp/keys :req-un [::user ::external-ip ::fqdn ::cert-manager - ::db-user-password ::admin-user ::admin-password - ::storage-size ::restic-repository ::restic-password - ::aws-access-key-id ::aws-secret-access-key ] - :opt-un [::u18-04])) - -(def cloud-spec-resolved nil) - -(s/defn k8s-convention-configuration :- k8s-convention/k8sConventionResolved - [convention-config :- CloudConventionResolved] - {:pre [(sp/valid? cloud-convention-resolved? convention-config)]} - (let [{:keys [cert-manager external-ip user u18-04]} convention-config - cluster-issuer (name cert-manager)] - (if u18-04 - {:user user - :k8s {:external-ip external-ip - :u18-04 true} - :cert-manager cert-manager} - {:user user - :k8s {:external-ip external-ip} - :cert-manager cert-manager}))) - - -(s/defn ^:always-validate - infra-configuration :- InfraResult - [convention-config :- CloudConventionResolved] - (let [{:keys [cert-manager fqdn user db-user-password admin-user admin-password storage-size - restic-repository aws-access-key-id aws-secret-access-key restic-password]} convention-config - cluster-issuer (name cert-manager) - db-user-name "cloud"] - {infra/facility - {:user user - :backup {:restic-repository restic-repository - :aws-access-key-id aws-access-key-id - :aws-secret-access-key aws-secret-access-key - :restic-password restic-password} - :cloud {:fqdn fqdn - :secret-name (str/replace fqdn #"\." "-") - :cluster-issuer cluster-issuer - :db-name "cloud" - :db-user-password db-user-password - :db-user-name db-user-name - :admin-user admin-user - :admin-password admin-password - :storage-size (str storage-size)} - :postgres {:db-user-password db-user-password - :db-user-name db-user-name}}})) - diff --git a/main/src/meissa/pallet/meissa_cloud/convention/bash.clj b/main/src/meissa/pallet/meissa_cloud/convention/bash.clj deleted file mode 100644 index 5d1ef2c..0000000 --- a/main/src/meissa/pallet/meissa_cloud/convention/bash.clj +++ /dev/null @@ -1,10 +0,0 @@ -(ns meissa.pallet.meissa-cloud.convention.bash - (:require - [clojure.spec.alpha :as s])) - -(defn bash-env-string? - [input] - (and (string? input) - (not (re-matches #".*['\"\$]+.*" input)))) - -(s/def ::plain bash-env-string?) diff --git a/main/src/meissa/pallet/meissa_cloud/convention/bash_php.clj b/main/src/meissa/pallet/meissa_cloud/convention/bash_php.clj deleted file mode 100644 index d065a66..0000000 --- a/main/src/meissa/pallet/meissa_cloud/convention/bash_php.clj +++ /dev/null @@ -1,11 +0,0 @@ -(ns meissa.pallet.meissa-cloud.convention.bash-php - (:require - [clojure.spec.alpha :as s] - [meissa.pallet.meissa-cloud.convention.bash :as bash])) - -(defn bash-php-env-string? - [input] - (and (bash/bash-env-string? input) - (not (re-matches #".*[\-\\\\]+.*" input)))) - -(s/def ::plain bash-php-env-string?) diff --git a/main/src/meissa/pallet/meissa_cloud/infra.clj b/main/src/meissa/pallet/meissa_cloud/infra.clj deleted file mode 100644 index a964dc3..0000000 --- a/main/src/meissa/pallet/meissa_cloud/infra.clj +++ /dev/null @@ -1,51 +0,0 @@ -(ns meissa.pallet.meissa-cloud.infra - (:require - [schema.core :as s] - [dda.pallet.core.infra :as core-infra] - [meissa.pallet.meissa-cloud.infra.backup :as backup] - [meissa.pallet.meissa-cloud.infra.cloud :as cloud] - [meissa.pallet.meissa-cloud.infra.postgres :as postgres])) - -(def facility :meissa-cloud) - -(def MeissaCloudInfra - (merge - {:user s/Keyword} - backup/MeissaBackupInfra - cloud/MeissaCloudInfra - postgres/MeissaPostgresInfra)) - -(s/defmethod core-infra/dda-init facility - [dda-crate config] - (let [facility (:facility dda-crate) - {:keys [user backup postgres cloud]} config - user-str (name user)] - (postgres/init facility user-str postgres) - (cloud/init facility user-str cloud) - (backup/init facility user-str backup))) - -(s/defmethod core-infra/dda-install facility - [dda-crate config] - (let [facility (:facility dda-crate) - {:keys [user backup postgres cloud]} config - user-str (name user)] - (postgres/install facility user-str postgres) - (cloud/install facility user-str cloud) - (backup/install facility user-str backup))) - -(s/defmethod core-infra/dda-configure facility - [dda-crate config] - (let [facility (:facility dda-crate) - {:keys [user backup postgres cloud]} config - user-str (name user)] - (postgres/configure facility user-str postgres) - (cloud/configure facility user-str cloud) - (backup/configure facility user-str backup))) - -(def meissa-cloud - (core-infra/make-dda-crate-infra - :facility facility - :infra-schema MeissaCloudInfra)) - -(def with-cloud - (core-infra/create-infra-plan meissa-cloud)) diff --git a/main/src/meissa/pallet/meissa_cloud/infra/backup.clj b/main/src/meissa/pallet/meissa_cloud/infra/backup.clj deleted file mode 100644 index c80ead6..0000000 --- a/main/src/meissa/pallet/meissa_cloud/infra/backup.clj +++ /dev/null @@ -1,39 +0,0 @@ -(ns meissa.pallet.meissa-cloud.infra.backup - (:require - [schema.core :as s] - [dda.provision :as p] - [dda.provision.pallet :as pp])) - -(s/def Backup - {:restic-repository s/Str - :aws-access-key-id s/Str - :aws-secret-access-key s/Str - :restic-password s/Str}) - -(def MeissaBackupInfra {:backup Backup}) - -(def backup "backup") - -(defn init [facility user config]) - -(defn install - [facility user config] - (let [facility-name (name facility)] - (p/provision-log ::pp/pallet facility-name backup - ::p/info "install") - (p/copy-resources-to-user - ::pp/pallet user facility-name backup - [{:filename "backup-secret.yml" :config config} - {:filename "backup-config.yml" :config config} - {:filename "configure-as-user.sh"} - {:filename "backup-restore.yml"} - {:filename "backup-cron.yml"}]))) - -(defn configure - [facility user config] - (let [facility-name (name facility)] - (p/provision-log ::pp/pallet facility-name backup - ::p/info "configure") - (p/exec-file-on-target-as-user - ::pp/pallet user facility-name backup "configure-as-user.sh") - )) diff --git a/main/src/meissa/pallet/meissa_cloud/infra/cloud.clj b/main/src/meissa/pallet/meissa_cloud/infra/cloud.clj deleted file mode 100644 index ed11188..0000000 --- a/main/src/meissa/pallet/meissa_cloud/infra/cloud.clj +++ /dev/null @@ -1,57 +0,0 @@ -(ns meissa.pallet.meissa-cloud.infra.cloud - (:require - [schema.core :as s] - [dda.provision :as p] - [dda.provision.pallet :as pp])) - -(s/def Cloud - {:fqdn s/Str - :secret-name s/Str - :cluster-issuer s/Str - :db-name s/Str - :db-user-name s/Str - :db-user-password s/Str - :admin-user s/Str - :admin-password s/Str - :storage-size s/Str}) - -(def MeissaCloudInfra {:cloud Cloud}) - -(def cloud "cloud") - -(defn init - [facility user config] - (let [facility-name (name facility)] - (p/provision-log ::pp/pallet facility-name cloud - ::p/info "init") - (p/copy-resources-to-tmp - ::pp/pallet facility-name cloud - [{:filename "install-as-root.sh" :config {:user user}}]))) - - -(defn install - [facility user config] - (let [facility-name (name facility)] - (p/provision-log ::pp/pallet facility-name cloud - ::p/info "install") - (p/copy-resources-to-user - ::pp/pallet user facility-name cloud - [{:filename "pod-running.sh"} - {:filename "cloud-persistent-volume.yml" :config config} - {:filename "cloud-secret.yml" :config config} - {:filename "cloud-service.yml"} - {:filename "cloud-pvc.yml" :config config} - {:filename "cloud-pod.yml" :config config} - {:filename "cloud-ingress.yml" :config config} - {:filename "configure-as-user.sh"} - {:filename "verify.sh" :config config}]) - (p/exec-file-on-target-as-root - ::pp/pallet facility-name cloud "install-as-root.sh"))) - -(defn configure - [facility user config] - (let [facility-name (name facility)] - (p/provision-log ::pp/pallet facility-name cloud - ::p/info "configure") - (p/exec-file-on-target-as-user - ::pp/pallet user facility-name cloud "configure-as-user.sh"))) diff --git a/main/src/meissa/pallet/meissa_cloud/infra/postgres.clj b/main/src/meissa/pallet/meissa_cloud/infra/postgres.clj deleted file mode 100644 index 60a3e8b..0000000 --- a/main/src/meissa/pallet/meissa_cloud/infra/postgres.clj +++ /dev/null @@ -1,47 +0,0 @@ -(ns meissa.pallet.meissa-cloud.infra.postgres - (:require - [schema.core :as s] - [dda.provision :as p] - [dda.provision.pallet :as pp])) - -(s/def Postgres {:db-user-name s/Str :db-user-password s/Str}) - -(def MeissaPostgresInfra {:postgres Postgres}) - -(def postgres "postgres") - -(defn init - [facility user config] - (let [facility-name (name facility)] - (p/provision-log ::pp/pallet facility-name postgres - ::p/info "init") - (p/copy-resources-to-tmp - ::pp/pallet facility-name postgres - [{:filename "install-as-root.sh" :config {:user user}}]))) - - -(defn install - [facility user config] - (let [facility-name (name facility)] - (p/provision-log ::pp/pallet facility-name postgres - ::p/info "install") - (p/copy-resources-to-user - ::pp/pallet user facility-name postgres - [{:filename "postgres-persistent-volume.yml"} - {:filename "postgres-secret.yml" :config config} - {:filename "postgres-config.yml"} - {:filename "postgres-service.yml"} - {:filename "postgres-pvc.yml"} - {:filename "postgres-deployment.yml" :config config} - {:filename "configure-as-user.sh"} - {:filename "verify.sh"}]) - (p/exec-file-on-target-as-root - ::pp/pallet facility-name postgres "install-as-root.sh"))) - -(defn configure - [facility user config] - (let [facility-name (name facility)] - (p/provision-log ::pp/pallet facility-name postgres - ::p/info "configure") - (p/exec-file-on-target-as-user - ::pp/pallet user facility-name postgres "configure-as-user.sh"))) diff --git a/project.clj b/project.clj index 4d4e476..be010d3 100644 --- a/project.clj +++ b/project.clj @@ -38,8 +38,8 @@ :uberjar {:source-paths ["uberjar/src"] :resource-paths ["uberjar/resources"] :aot :all - :main meissa.pallet.meissa-cloud.main - :uberjar-name "meissa-cloud-standalone.jar" + :main dda.c4k-nextcloud.uberjar + :uberjar-name "c4k-nextcloud-standalone.jar" :dependencies [[org.clojure/tools.cli "1.0.194"] [ch.qos.logback/logback-classic "1.3.0-alpha5"] [org.slf4j/jcl-over-slf4j "2.0.0-alpha1"]]}} diff --git a/src/main/clj/dda/c4k_cloud/uberjar.clj b/src/main/clj/dda/c4k_nextcloud/uberjar.clj similarity index 93% rename from src/main/clj/dda/c4k_cloud/uberjar.clj rename to src/main/clj/dda/c4k_nextcloud/uberjar.clj index 49f37b0..4c5a766 100644 --- a/src/main/clj/dda/c4k_cloud/uberjar.clj +++ b/src/main/clj/dda/c4k_nextcloud/uberjar.clj @@ -1,16 +1,16 @@ -(ns dda.c4k-cloud.uberjar +(ns dda.c4k-nextcloud.uberjar (:gen-class) (:require [clojure.spec.alpha :as s] [clojure.string :as cs] [clojure.tools.reader.edn :as edn] [expound.alpha :as expound] - [dda.c4k-cloud.core :as core])) + [dda.c4k-nextcloud.core :as core])) (def usage "usage: - c4k-cloud {your configuraton file} {your authorization file}") + c4k-nextcloud {your configuraton file} {your authorization file}") (s/def ::options (s/* #{"-h"})) (s/def ::filename (s/and string? diff --git a/src/main/cljc/dda/c4k_cloud/backup.cljc b/src/main/cljc/dda/c4k_nextcloud/backup.cljc similarity index 98% rename from src/main/cljc/dda/c4k_cloud/backup.cljc rename to src/main/cljc/dda/c4k_nextcloud/backup.cljc index 1876e10..19973c1 100644 --- a/src/main/cljc/dda/c4k_cloud/backup.cljc +++ b/src/main/cljc/dda/c4k_nextcloud/backup.cljc @@ -1,4 +1,4 @@ -(ns dda.c4k-cloud.backup +(ns dda.c4k-nextcloud.backup (:require [clojure.spec.alpha :as s] #?(:cljs [shadow.resource :as rc]) diff --git a/src/main/cljc/dda/c4k_cloud/cloud.cljc b/src/main/cljc/dda/c4k_nextcloud/cloud.cljc similarity index 50% rename from src/main/cljc/dda/c4k_cloud/cloud.cljc rename to src/main/cljc/dda/c4k_nextcloud/cloud.cljc index 79b74f8..4f715ed 100644 --- a/src/main/cljc/dda/c4k_cloud/cloud.cljc +++ b/src/main/cljc/dda/c4k_nextcloud/cloud.cljc @@ -1,4 +1,4 @@ -(ns dda.c4k-cloud.cloud +(ns dda.c4k-nextcloud.nextcloud (:require [clojure.spec.alpha :as s] #?(:cljs [shadow.resource :as rc]) @@ -7,31 +7,32 @@ (s/def ::fqdn cm/fqdn-string?) (s/def ::issuer cm/letsencrypt-issuer?) -(s/def ::cloud-data-volume-path string?) +(s/def ::restic-repository string?) +(s/def ::nextcloud-data-volume-path string?) #?(:cljs - (defmethod yaml/load-resource :cloud [resource-name] + (defmethod yaml/load-resource :nextcloud [resource-name] (case resource-name - "cloud/certificate.yaml" (rc/inline "cloud/certificate.yaml") - "cloud/deployment.yaml" (rc/inline "cloud/deployment.yaml") - "cloud/ingress.yaml" (rc/inline "cloud/ingress.yaml") - "cloud/persistent-volume.yaml" (rc/inline "cloud/persistent-volume.yaml") - "cloud/pvc.yaml" (rc/inline "cloud/pvc.yaml") - "cloud/service.yaml" (rc/inline "cloud/service.yaml") + "nextcloud/certificate.yaml" (rc/inline "nextcloud/certificate.yaml") + "nextcloud/deployment.yaml" (rc/inline "nextcloud/deployment.yaml") + "nextcloud/ingress.yaml" (rc/inline "nextcloud/ingress.yaml") + "nextcloud/persistent-volume.yaml" (rc/inline "nextcloud/persistent-volume.yaml") + "nextcloud/pvc.yaml" (rc/inline "nextcloud/pvc.yaml") + "nextcloud/service.yaml" (rc/inline "nextcloud/service.yaml") (throw (js/Error. "Undefined Resource!"))))) (defn generate-certificate [config] (let [{:keys [fqdn issuer]} config letsencrypt-issuer (str "letsencrypt-" (name issuer) "-issuer")] (-> - (yaml/from-string (yaml/load-resource "cloud/certificate.yaml")) + (yaml/from-string (yaml/load-resource "nextcloud/certificate.yaml")) (assoc-in [:spec :commonName] fqdn) (assoc-in [:spec :dnsNames] [fqdn]) (assoc-in [:spec :issuerRef :name] letsencrypt-issuer)))) (defn generate-deployment [config] (let [{:keys [fqdn]} config] - (-> (yaml/from-string (yaml/load-resource "cloud/deployment.yaml")) + (-> (yaml/from-string (yaml/load-resource "nextcloud/deployment.yaml")) (cm/replace-named-value "FQDN" fqdn)))) (defn generate-ingress [config] @@ -39,18 +40,18 @@ :or {issuer :staging}} config letsencrypt-issuer (str "letsencrypt-" (name issuer) "-issuer")] (-> - (yaml/from-string (yaml/load-resource "cloud/ingress.yaml")) + (yaml/from-string (yaml/load-resource "nextcloud/ingress.yaml")) (assoc-in [:metadata :annotations :cert-manager.io/cluster-issuer] letsencrypt-issuer) (cm/replace-all-matching-values-by-new-value "fqdn" fqdn)))) (defn generate-persistent-volume [config] - (let [{:keys [cloud-data-volume-path]} config] + (let [{:keys [nextcloud-data-volume-path]} config] (-> - (yaml/from-string (yaml/load-resource "cloud/persistent-volume.yaml")) - (assoc-in [:spec :hostPath :path] cloud-data-volume-path)))) + (yaml/from-string (yaml/load-resource "nextcloud/persistent-volume.yaml")) + (assoc-in [:spec :hostPath :path] nextcloud-data-volume-path)))) (defn generate-pvc [] - (yaml/from-string (yaml/load-resource "cloud/pvc.yaml"))) + (yaml/from-string (yaml/load-resource "nextcloud/pvc.yaml"))) (defn generate-service [] - (yaml/from-string (yaml/load-resource "cloud/service.yaml"))) + (yaml/from-string (yaml/load-resource "nextcloud/service.yaml"))) diff --git a/src/main/cljc/dda/c4k_cloud/core.cljc b/src/main/cljc/dda/c4k_nextcloud/core.cljc similarity index 66% rename from src/main/cljc/dda/c4k_cloud/core.cljc rename to src/main/cljc/dda/c4k_nextcloud/core.cljc index 3397657..047a6ad 100644 --- a/src/main/cljc/dda/c4k_cloud/core.cljc +++ b/src/main/cljc/dda/c4k_nextcloud/core.cljc @@ -1,4 +1,4 @@ -(ns dda.c4k-cloud.core +(ns dda.c4k-nextcloud.core (:require [clojure.string :as cs] [clojure.spec.alpha :as s] @@ -6,13 +6,13 @@ :cljs [orchestra.core :refer-macros [defn-spec]]) [dda.c4k-common.yaml :as yaml] [dda.c4k-common.postgres :as postgres] - [dda.c4k-cloud.cloud :as cloud] - [dda.c4k-cloud.backup :as backup])) + [dda.c4k-nextcloud.nextcloud :as nextcloud] + [dda.c4k-nextcloud.backup :as backup])) (def config-defaults {:issuer :staging}) -(def config? (s/keys :req-un [::cloud/fqdn] - :opt-un [::cloud/issuer ::cloud/cloud-data-volume-path +(def config? (s/keys :req-un [::nextcloud/fqdn] + :opt-un [::nextcloud/issuer ::nextcloud/nextcloud-data-volume-path ::postgres/postgres-data-volume-path ::restic-repository])) (def auth? (s/keys :req-un [::postgres/postgres-db-user ::postgres/postgres-db-password @@ -29,14 +29,14 @@ [(yaml/to-string (postgres/generate-pvc)) (yaml/to-string (postgres/generate-deployment)) (yaml/to-string (postgres/generate-service))] - (when (contains? config :cloud-data-volume-path) - [(yaml/to-string (cloud/generate-persistent-volume config))]) - [(yaml/to-string (cloud/generate-pvc)) - (yaml/to-string (cloud/generate-deployment config)) - (yaml/to-string (cloud/generate-service)) - (yaml/to-string (cloud/generate-certificate config)) - (yaml/to-string (cloud/generate-ingress config)) - (yaml/to-string (cloud/generate-service))] + (when (contains? config :nextcloud-data-volume-path) + [(yaml/to-string (nextcloud/generate-persistent-volume config))]) + [(yaml/to-string (nextcloud/generate-pvc)) + (yaml/to-string (nextcloud/generate-deployment config)) + (yaml/to-string (nextcloud/generate-service)) + (yaml/to-string (nextcloud/generate-certificate config)) + (yaml/to-string (nextcloud/generate-ingress config)) + (yaml/to-string (nextcloud/generate-service))] (when (contains? config :restic-repository) [(yaml/to-string (backup/generate-config config)) (yaml/to-string (backup/generate-secret config)) diff --git a/src/main/cljs/dda/c4k_cloud/browser.cljs b/src/main/cljs/dda/c4k_nextcloud/browser.cljs similarity index 65% rename from src/main/cljs/dda/c4k_cloud/browser.cljs rename to src/main/cljs/dda/c4k_nextcloud/browser.cljs index 1380b19..94012b6 100644 --- a/src/main/cljs/dda/c4k_cloud/browser.cljs +++ b/src/main/cljs/dda/c4k_nextcloud/browser.cljs @@ -1,19 +1,20 @@ -(ns dda.c4k-cloud.browser +(ns dda.c4k-nextcloud.browser (:require [clojure.tools.reader.edn :as edn] - [dda.c4k-cloud.core :as core] - [dda.c4k-cloud.cloud :as cloud] - [dda.c4k-common.browser :as br])) + [dda.c4k-nextcloud.core :as core] + [dda.c4k-nextcloud.nextcloud :as nextcloud] + [dda.c4k-common.browser :as br] + [dda.c4k-common.postgres :as pgc])) (defn config-from-document [] - (let [cloud-data-volume-path (br/get-content-from-element "cloud-data-volume-path" :optional true :deserializer keyword) - postgres-data-volume-path (br/get-content-from-element "postgres-data-volume-path" :optional true :deserializer keyword) - restic-repository (br/get-content-from-element "restic-repository" :optional true :deserializer keyword) + (let [nextcloud-data-volume-path (br/get-content-from-element "nextcloud-data-volume-path" :optional true) + postgres-data-volume-path (br/get-content-from-element "postgres-data-volume-path" :optional true) + restic-repository (br/get-content-from-element "restic-repository" :optional true) issuer (br/get-content-from-element "issuer" :optional true :deserializer keyword)] (merge {:fqdn (br/get-content-from-element "fqdn")} - (when (some? cloud-data-volume-path) - {:cloud-data-volume-path cloud-data-volume-path}) + (when (some? nextcloud-data-volume-path) + {:nextcloud-data-volume-path nextcloud-data-volume-path}) (when (some? postgres-data-volume-path) {:postgres-data-volume-path postgres-data-volume-path}) (when (some? restic-repository) @@ -23,11 +24,11 @@ ))) (defn validate-all! [] - (br/validate! "fqdn" ::cloud/fqdn) - (br/validate! "cloud-data-volume-path" ::cloud/cloud-data-volume-path :optional true :deserializer keyword) - (br/validate! "postgres-data-volume-path" ::cloud/cloud-data-volume-path :optional true :deserializer keyword) - (br/validate! "restic-repository" ::cloud/restic-repository :optional true :deserializer keyword) - (br/validate! "issuer" ::cloud/issuer :optional true :deserializer keyword) + (br/validate! "fqdn" ::nextcloud/fqdn) + (br/validate! "nextcloud-data-volume-path" ::nextcloud/nextcloud-data-volume-path :optional true) + (br/validate! "postgres-data-volume-path" ::pgc/postgres-data-volume-path :optional true) + (br/validate! "restic-repository" ::nextcloud/restic-repository :optional true) + (br/validate! "issuer" ::nextcloud/issuer :optional true :deserializer keyword) (br/validate! "auth" core/auth? :deserializer edn/read-string) (br/set-validated!)) @@ -43,7 +44,7 @@ (-> (br/get-element-by-id "fqdn") (.addEventListener "blur" #(do (validate-all!)))) - (-> (br/get-element-by-id "cloud-data-volume-path") + (-> (br/get-element-by-id "nextcloud-data-volume-path") (.addEventListener "blur" #(do (validate-all!)))) (-> (br/get-element-by-id "postgres-data-volume-path") diff --git a/src/main/resources/backup/backup-restore.yaml b/src/main/resources/backup/backup-restore.yaml index bcce528..c13e166 100644 --- a/src/main/resources/backup/backup-restore.yaml +++ b/src/main/resources/backup/backup-restore.yaml @@ -12,21 +12,12 @@ spec: imagePullPolicy: IfNotPresent command: ["/entrypoint-start-and-wait.sh"] env: - - name: POSTGRES_USER - valueFrom: - secretKeyRef: - name: postgres-secret - key: postgres-user - - name: POSTGRES_DB - valueFrom: - configMapKeyRef: - name: postgres-config - key: postgres-db - - name: POSTGRES_PASSWORD - valueFrom: - secretKeyRef: - name: postgres-secret - key: postgres-password + - name: POSTGRES_USER_FILE + value: /var/run/secrets/cloud-secrets/postgres-user + - name: POSTGRES_DB_FILE + value: /var/run/secrets/cloud-secrets/postgres-db + - name: POSTGRES_PASSWORD_FILE + value: /var/run/secrets/cloud-secrets/postgres-password - name: POSTGRES_HOST value: "postgresql-service:5432" - name: POSTGRES_SERVICE @@ -52,10 +43,16 @@ spec: - name: backup-secret-volume mountPath: /var/run/secrets/backup-secrets readOnly: true + - name: cloud-secret-volume + mountPath: /var/run/secrets/cloud-secrets + readOnly: true volumes: - name: cloud-data-volume persistentVolumeClaim: claimName: cloud-pvc + - name: cloud-secret-volume + secret: + secretName: cloud-secret - name: backup-secret-volume secret: secretName: backup-secret diff --git a/src/main/resources/backup/cron.yaml b/src/main/resources/backup/cron.yaml index a914d37..8bb54bc 100644 --- a/src/main/resources/backup/cron.yaml +++ b/src/main/resources/backup/cron.yaml @@ -6,33 +6,24 @@ metadata: app.kubernetes.part-of: cloud spec: schedule: "10 23 * * *" - successfulJobsHistoryLimit: 1 - failedJobsHistoryLimit: 1 + successfulJobsHistoryLimit: 0 + failedJobsHistoryLimit: 0 jobTemplate: spec: template: spec: containers: - name: backup-app - image: domaindrivenarchitecture/c4k-cloud-backup + image: domaindrivenarchitecture/meissa-cloud-backup imagePullPolicy: IfNotPresent command: ["/entrypoint.sh"] env: - - name: POSTGRES_USER - valueFrom: - secretKeyRef: - name: postgres-secret - key: postgres-user - - name: POSTGRES_PASSWORD - valueFrom: - secretKeyRef: - name: postgres-secret - key: postgres-password - - name: POSTGRES_DB - valueFrom: - configMapKeyRef: - name: postgres-config - key: postgres-db + - name: POSTGRES_USER_FILE + value: /var/run/secrets/cloud-secrets/postgres-user + - name: POSTGRES_DB_FILE + value: /var/run/secrets/cloud-secrets/postgres-db + - name: POSTGRES_PASSWORD_FILE + value: /var/run/secrets/cloud-secrets/postgres-password - name: POSTGRES_HOST value: "postgresql-service:5432" - name: POSTGRES_SERVICE @@ -58,10 +49,16 @@ spec: - name: backup-secret-volume mountPath: /var/run/secrets/backup-secrets readOnly: true + - name: cloud-secret-volume + mountPath: /var/run/secrets/cloud-secrets + readOnly: true volumes: - name: cloud-data-volume persistentVolumeClaim: claimName: cloud-pvc + - name: cloud-secret-volume + secret: + secretName: cloud-secret - name: backup-secret-volume secret: secretName: backup-secret diff --git a/src/main/resources/backup/secret.yaml b/src/main/resources/backup/secret.yaml index c5809e0..4b68578 100644 --- a/src/main/resources/backup/secret.yaml +++ b/src/main/resources/backup/secret.yaml @@ -3,7 +3,7 @@ kind: Secret metadata: name: backup-secret type: Opaque -data: +stringData: aws-access-key-id: aws-access-key-id aws-secret-access-key: aws-secret-access-key restic-password: restic-password \ No newline at end of file diff --git a/main/resources/cloud/configure-as-user.sh b/src/main/resources/cloud/configure-as-user.sh similarity index 100% rename from main/resources/cloud/configure-as-user.sh rename to src/main/resources/cloud/configure-as-user.sh diff --git a/src/main/resources/cloud/deployment.yaml b/src/main/resources/cloud/deployment.yaml index bc719c9..1e05dda 100644 --- a/src/main/resources/cloud/deployment.yaml +++ b/src/main/resources/cloud/deployment.yaml @@ -14,27 +14,39 @@ spec: app: cloud spec: containers: - - image: domaindrivenarchitecture/c4k-cloud + - image: domaindrivenarchitecture/meissa-cloud-app name: cloud-app imagePullPolicy: IfNotPresent + ports: + - containerPort: 80 env: - - name: DB_USERNAME_FILE - value: /var/run/secrets/postgres-secret/postgres-user - - name: DB_PASSWORD_FILE - value: /var/run/secrets/postgres-secret/postgres-password - - name: FQDN - value: fqdn - command: ["/app/entrypoint.sh"] - volumeMounts: - - mountPath: /var/cloud - name: cloud-data-volume - - name: postgres-secret-volume - mountPath: /var/run/secrets/postgres-secret + - name: NEXTCLOUD_ADMIN_USER_FILE + value: /var/run/secrets/cloud-secrets/nextcloud-admin-user + - name: NEXTCLOUD_ADMIN_PASSWORD_FILE + value: /var/run/secrets/cloud-secrets/nextcloud-admin-password + - name: NEXTCLOUD_TRUSTED_DOMAINS + value: "{{fqdn}}" + - name: POSTGRES_USER_FILE + value: /var/run/secrets/cloud-secrets/postgres-user + - name: POSTGRES_PASSWORD_FILE + value: /var/run/secrets/cloud-secrets/postgres-password + - name: POSTGRES_DB_FILE + value: /var/run/secrets/cloud-secrets/postgres-db + - name: POSTGRES_HOST + value: "postgresql-service:5432" + volumeMounts: + - name: cloud-data-volume + mountPath: /var/www/html + - name: cloud-secret-volume + mountPath: /var/run/secrets/cloud-secrets readOnly: true volumes: - - name: cloud-data-volume - persistentVolumeClaim: - claimName: cloud-pvc - - name: postgres-secret-volume - secret: - secretName: postgres-secret + - name: cloud-data-volume + persistentVolumeClaim: + claimName: cloud-pvc + - name: cloud-secret-volume + secret: + secretName: cloud-secret + - name: backup-secret-volume + secret: + secretName: backup-secret diff --git a/src/main/resources/cloud/ingress.yaml b/src/main/resources/cloud/ingress.yaml index f206da2..cc5a0df 100644 --- a/src/main/resources/cloud/ingress.yaml +++ b/src/main/resources/cloud/ingress.yaml @@ -23,4 +23,4 @@ spec: - path: / backend: serviceName: cloud-service - servicePort: 8080 + servicePort: 80 diff --git a/main/resources/cloud/install-as-root.sh.template b/src/main/resources/cloud/install-as-root.sh.template similarity index 100% rename from main/resources/cloud/install-as-root.sh.template rename to src/main/resources/cloud/install-as-root.sh.template diff --git a/src/main/resources/cloud/persistent-volume.yaml b/src/main/resources/cloud/persistent-volume.yaml index f39a2ec..7c3f89e 100644 --- a/src/main/resources/cloud/persistent-volume.yaml +++ b/src/main/resources/cloud/persistent-volume.yaml @@ -4,11 +4,12 @@ metadata: name: cloud-pv-volume labels: type: local + app: cloud spec: storageClassName: manual accessModes: - ReadWriteOnce capacity: - storage: 30Gi + storage: {{storage-size}}Gi #??? 30Gi? hostPath: path: "/var/cloud" diff --git a/main/resources/cloud/pod-running.sh b/src/main/resources/cloud/pod-running.sh old mode 100755 new mode 100644 similarity index 100% rename from main/resources/cloud/pod-running.sh rename to src/main/resources/cloud/pod-running.sh diff --git a/src/main/resources/cloud/pvc.yaml b/src/main/resources/cloud/pvc.yaml index 3285ef6..d4f7e04 100644 --- a/src/main/resources/cloud/pvc.yaml +++ b/src/main/resources/cloud/pvc.yaml @@ -10,4 +10,7 @@ spec: - ReadWriteOnce resources: requests: - storage: 30Gi \ No newline at end of file + storage: {{storage-size}}Gi #??? 30Gi? + selector: + matchLabels: + app: cloud diff --git a/main/resources/cloud/secret.yaml b/src/main/resources/cloud/secret.yaml similarity index 100% rename from main/resources/cloud/secret.yaml rename to src/main/resources/cloud/secret.yaml diff --git a/src/main/resources/cloud/service.yaml b/src/main/resources/cloud/service.yaml index 2c05a15..7fcd0d7 100644 --- a/src/main/resources/cloud/service.yaml +++ b/src/main/resources/cloud/service.yaml @@ -4,6 +4,6 @@ metadata: name: cloud-service spec: selector: - app: cloud + app.kubernetes.io/name: cloud #??? ports: - - port: 8080 + - port: 80 diff --git a/src/main/resources/logback.xml b/src/main/resources/logback.xml deleted file mode 100644 index 8985f2b..0000000 --- a/src/main/resources/logback.xml +++ /dev/null @@ -1,50 +0,0 @@ - - - - %d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n - - - INFO - - - - - logs/pallet.log - - logs/old/pallet.%d{yyyy-MM-dd}.log - 3 - - - %date %level [%thread] %logger{10} %msg%n - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/main/resources/postgres/install-as-root.sh.template b/src/main/resources/postgres/install-as-root.sh.template similarity index 100% rename from main/resources/postgres/install-as-root.sh.template rename to src/main/resources/postgres/install-as-root.sh.template diff --git a/main/resources/postgres/postgres-config.yml b/src/main/resources/postgres/postgres-config.yaml similarity index 100% rename from main/resources/postgres/postgres-config.yml rename to src/main/resources/postgres/postgres-config.yaml diff --git a/main/resources/postgres/postgres-deployment.yml.template b/src/main/resources/postgres/postgres-deployment.yaml similarity index 100% rename from main/resources/postgres/postgres-deployment.yml.template rename to src/main/resources/postgres/postgres-deployment.yaml diff --git a/main/resources/postgres/postgres-persistent-volume.yml b/src/main/resources/postgres/postgres-persistent-volume.yaml similarity index 100% rename from main/resources/postgres/postgres-persistent-volume.yml rename to src/main/resources/postgres/postgres-persistent-volume.yaml diff --git a/main/resources/postgres/postgres-pvc.yml b/src/main/resources/postgres/postgres-pvc.yaml similarity index 100% rename from main/resources/postgres/postgres-pvc.yml rename to src/main/resources/postgres/postgres-pvc.yaml diff --git a/main/resources/postgres/postgres-secret.yml.template b/src/main/resources/postgres/postgres-secret.yaml similarity index 100% rename from main/resources/postgres/postgres-secret.yml.template rename to src/main/resources/postgres/postgres-secret.yaml diff --git a/main/resources/postgres/postgres-service.yml b/src/main/resources/postgres/postgres-service.yaml similarity index 100% rename from main/resources/postgres/postgres-service.yml rename to src/main/resources/postgres/postgres-service.yaml diff --git a/src/test/cljc/dda/c4k_cloud/backup_test.cljc b/src/test/cljc/dda/c4k_cloud/backup_test.cljc deleted file mode 100644 index e9eec96..0000000 --- a/src/test/cljc/dda/c4k_cloud/backup_test.cljc +++ /dev/null @@ -1,93 +0,0 @@ -(ns dda.c4k-cloud.backup-test - (:require - #?(:clj [clojure.test :refer [deftest is are testing run-tests]] - :cljs [cljs.test :refer-macros [deftest is are testing run-tests]]) - [dda.c4k-cloud.backup :as cut])) - - -(deftest should-generate-secret - (is (= {:apiVersion "v1" - :kind "Secret" - :metadata {:name "backup-secret"} - :type "Opaque" - :data - {:aws-access-key-id "YXdzLWlk", :aws-secret-access-key "YXdzLXNlY3JldA==", :restic-password "cmVzdGljLXB3"}} - (cut/generate-secret {:aws-access-key-id "aws-id" :aws-secret-access-key "aws-secret" :restic-password "restic-pw"})))) - -(deftest should-generate-config - (is (= {:apiVersion "v1" - :kind "ConfigMap" - :metadata {:name "backup-config" - :labels {:app.kubernetes.io/name "backup" - :app.kubernetes.io/part-of "cloud"}} - :data - {:restic-repository "s3:restic-repository"}} - (cut/generate-config {:restic-repository "s3:restic-repository"})))) - -(deftest should-generate-cron - (is (= {:apiVersion "batch/v1beta1" - :kind "CronJob" - :metadata {:name "cloud-backup" - :labels {:app.kubernetes.part-of "cloud"}} - :spec {:schedule "10 23 * * *" - :successfulJobsHistoryLimit 1 - :failedJobsHistoryLimit 1 - :jobTemplate - {:spec - {:template - {:spec - {:containers - [{:name "backup-app" - :image "domaindrivenarchitecture/c4k-cloud-backup" - :imagePullPolicy "IfNotPresent" - :command ["/entrypoint.sh"] - :env - [{:name "POSTGRES_USER" - :valueFrom - {:secretKeyRef - {:name "postgres-secret" - :key "postgres-user"}}} - {:name "POSTGRES_PASSWORD" - :valueFrom - {:secretKeyRef - {:name "postgres-secret" - :key "postgres-password"}}} - {:name "POSTGRES_DB" - :valueFrom - {:configMapKeyRef - {:name "postgres-config" - :key "postgres-db"}}} - {:name "POSTGRES_HOST" - :value "postgresql-service:5432"} - {:name "POSTGRES_SERVICE" - :value "postgresql-service"} - {:name "POSTGRES_PORT" - :value "5432"} - {:name "AWS_DEFAULT_REGION" - :value "eu-central-1"} - {:name "AWS_ACCESS_KEY_ID_FILE" - :value "/var/run/secrets/backup-secrets/aws-access-key-id"} - {:name "AWS_SECRET_ACCESS_KEY_FILE" - :value "/var/run/secrets/backup-secrets/aws-secret-access-key"} - {:name "RESTIC_REPOSITORY" - :valueFrom - {:configMapKeyRef - {:name "backup-config" - :key "restic-repository"}}} - {:name "RESTIC_PASSWORD_FILE" - :value "/var/run/secrets/backup-secrets/restic-password"}] - :volumeMounts - [{:name "cloud-data-volume" - :mountPath "/var/backups"} - {:name "backup-secret-volume" - :mountPath "/var/run/secrets/backup-secrets" - :readOnly true}]}] - :volumes - [{:name "cloud-data-volume" - :persistentVolumeClaim - {:claimName "cloud-pvc"}} - {:name "backup-secret-volume" - :secret - {:secretName "backup-secret"}}] - :restartPolicy "OnFailure"}}}}}} - (cut/generate-cron)))) diff --git a/src/test/cljc/dda/c4k_cloud/cloud_test.cljc b/src/test/cljc/dda/c4k_cloud/cloud_test.cljc deleted file mode 100644 index b29c541..0000000 --- a/src/test/cljc/dda/c4k_cloud/cloud_test.cljc +++ /dev/null @@ -1,80 +0,0 @@ -(ns dda.c4k-cloud.cloud-test - (:require - #?(:clj [clojure.test :refer [deftest is are testing run-tests]] - :cljs [cljs.test :refer-macros [deftest is are testing run-tests]]) - [dda.c4k-cloud.cloud :as cut])) - -(deftest should-generate-certificate - (is (= {:apiVersion "cert-manager.io/v1alpha2" - :kind "Certificate" - :metadata {:name "cloud-cert", :namespace "default"} - :spec - {:secretName "cloud-secret" - :commonName "xx" - :dnsNames ["xx"] - :issuerRef - {:name "letsencrypt-prod-issuer", :kind "ClusterIssuer"}}} - (cut/generate-certificate {:fqdn "xx" :issuer :prod})))) - -(deftest should-generate-ingress - (is (= {:apiVersion "extensions/v1beta1" - :kind "Ingress" - :metadata - {:name "ingress-cloud" - :annotations - {:cert-manager.io/cluster-issuer - "letsencrypt-staging-issuer" - :nginx.ingress.kubernetes.io/proxy-body-size "256m" - :nginx.ingress.kubernetes.io/ssl-redirect "true" - :nginx.ingress.kubernetes.io/rewrite-target "/" - :nginx.ingress.kubernetes.io/proxy-connect-timeout "300" - :nginx.ingress.kubernetes.io/proxy-send-timeout "300" - :nginx.ingress.kubernetes.io/proxy-read-timeout "300"} - :namespace "default"} - :spec - {:tls [{:hosts ["xx"], :secretName "cloud-secret"}] - :rules - [{:host "xx" - :http - {:paths - [{:path "/" - :backend - {:serviceName "cloud-service", :servicePort 8080}}]}}]}} - (cut/generate-ingress {:fqdn "xx"})))) - -(deftest should-generate-persistent-volume - (is (= {:kind "PersistentVolume" - :apiVersion "v1" - :metadata {:name "cloud-pv-volume", :labels {:type "local"}} - :spec - {:storageClassName "manual" - :accessModes ["ReadWriteOnce"] - :capacity {:storage "30Gi"} - :hostPath {:path "xx"}}} - (cut/generate-persistent-volume {:cloud-data-volume-path "xx"})))) - -(deftest should-generate-deployment - (is (= {:containers - [{:image "domaindrivenarchitecture/c4k-cloud" - :name "cloud-app" - :imagePullPolicy "IfNotPresent" - :env - [{:name "DB_USERNAME_FILE" - :value - "/var/run/secrets/postgres-secret/postgres-user"} - {:name "DB_PASSWORD_FILE" - :value - "/var/run/secrets/postgres-secret/postgres-password"} - {:name "FQDN", :value "xx"}] - :command ["/app/entrypoint.sh"] - :volumeMounts - [{:mountPath "/var/cloud", :name "cloud-data-volume"} - {:name "postgres-secret-volume" - :mountPath "/var/run/secrets/postgres-secret" - :readOnly true}]}] - :volumes - [{:name "cloud-data-volume" - :persistentVolumeClaim {:claimName "cloud-pvc"}} - {:name "postgres-secret-volume" - :secret {:secretName "postgres-secret"}}]} - (get-in (cut/generate-deployment {:fqdn "xx"}) [:spec :template :spec])))) diff --git a/src/test/cljc/dda/c4k_cloud/core_test.cljc b/src/test/cljc/dda/c4k_cloud/core_test.cljc deleted file mode 100644 index 2fd1634..0000000 --- a/src/test/cljc/dda/c4k_cloud/core_test.cljc +++ /dev/null @@ -1,35 +0,0 @@ -(ns dda.c4k-cloud.core-test - (:require - #?(:clj [clojure.test :refer [deftest is are testing run-tests]] - :cljs [cljs.test :refer-macros [deftest is are testing run-tests]]) - [dda.c4k-cloud.core :as cut])) - -(deftest should-k8s-objects - (is (= 16 - (count (cut/k8s-objects {:fqdn "cloud-neu.prod.meissa-gmbh.de" - :postgres-db-user "cloud" - :postgres-db-password "cloud-db-password" - :issuer :prod - :cloud-data-volume-path "/var/cloud" - :postgres-data-volume-path "/var/postgres" - :aws-access-key-id "aws-id" - :aws-secret-access-key "aws-secret" - :restic-password "restic-pw" - :restic-repository "restic-repository"})))) - (is (= 14 - (count (cut/k8s-objects {:fqdn "cloud-neu.prod.meissa-gmbh.de" - :postgres-db-user "cloud" - :postgres-db-password "cloud-db-password" - :issuer :prod - :aws-access-key-id "aws-id" - :aws-secret-access-key "aws-secret" - :restic-password "restic-pw" - :restic-repository "restic-repository"})))) - (is (= 11 - (count (cut/k8s-objects {:fqdn "cloud-neu.prod.meissa-gmbh.de" - :postgres-db-user "cloud" - :postgres-db-password "cloud-db-password" - :issuer :prod - :aws-access-key-id "aws-id" - :aws-secret-access-key "aws-secret" - :restic-password "restic-pw"}))))) diff --git a/test/src/meissa/pallet/meissa_cloud/app_test.clj b/test/src/meissa/pallet/meissa_cloud/app_test.clj index 9daafa9..1cbda1b 100644 --- a/test/src/meissa/pallet/meissa_cloud/app_test.clj +++ b/test/src/meissa/pallet/meissa_cloud/app_test.clj @@ -1,8 +1,8 @@ -(ns meissa.pallet.meissa-cloud.app-test +(ns meissa.pallet.meissa-nextcloud.app-test (:require [clojure.test :refer :all] [schema.core :as s] - [meissa.pallet.meissa-cloud.app :as sut])) + [meissa.pallet.meissa-nextcloud.app :as sut])) (s/set-fn-validation! true) @@ -15,7 +15,7 @@ :admin-user "root" :admin-password "test1234" :storage-size 50 - :restic-repository "cloud" + :restic-repository "nextcloud" :aws-access-key-id "10" :aws-secret-access-key "secret" :restic-password "test4321"}) @@ -28,4 +28,4 @@ (deftest plan-def (testing "test plan-def" - (is (map? sut/with-cloud)))) + (is (map? sut/with-nextcloud)))) diff --git a/uberjar/src/meissa/pallet/meissa_cloud/main.clj b/uberjar/src/meissa/pallet/meissa_cloud/main.clj index 60a33f2..4ba35d6 100644 --- a/uberjar/src/meissa/pallet/meissa_cloud/main.clj +++ b/uberjar/src/meissa/pallet/meissa_cloud/main.clj @@ -1,11 +1,11 @@ -(ns meissa.pallet.meissa-cloud.main +(ns meissa.pallet.meissa-nextcloud.main (:gen-class) (:require [clojure.string :as str] [clojure.tools.cli :as cli] [dda.pallet.core.main-helper :as mh] [dda.pallet.core.app :as core-app] - [meissa.pallet.meissa-cloud.app :as app])) + [meissa.pallet.meissa-nextcloud.app :as app])) (def cli-options [["-h" "--help"] @@ -17,16 +17,16 @@ (defn usage [options-summary] (str/join \newline - ["meissa-cloud installs & configures a single host kubernetes cluster with Cloud installed" + ["meissa-nextcloud installs & configures a single host kubernetes cluster with nextcloud installed" "" - "Usage: java -jar meissa-cloud-standalone.jar [options] cloud.edn" + "Usage: java -jar meissa-nextcloud-standalone.jar [options] nextcloud.edn" "" "Options:" options-summary "" - "cloud.edn" + "nextcloud.edn" " - follows the edn format." - " - has to be a valid CloudConventionConfig" + " - has to be a valid nextcloudConventionConfig" ""])) (defn -main [& args] From 74f4a2928987b01b410ded446783149c51a104a0 Mon Sep 17 00:00:00 2001 From: Jan Krebs Date: Tue, 10 Aug 2021 10:56:04 +0200 Subject: [PATCH 05/12] remove uberjar folder and remove tests for now --- .../{cloud.cljc => nextcloud.cljc} | 0 .../should_generate_infra_for_convention.edn | 30 ---------- .../should_generate_k8s_convention.edn | 30 ---------- .../should_validate_input.0.edn | 14 ----- .../should_validate_input.1.edn | 14 ----- .../should_validate_input.2.edn | 14 ----- .../meissa/pallet/meissa_cloud/app_test.clj | 31 ---------- .../meissa_cloud/convention/bash_php_test.clj | 20 ------- .../meissa_cloud/convention/bash_test.clj | 22 ------- .../pallet/meissa_cloud/convention_test.clj | 18 ------ uberjar/resources/localhost-target.edn | 2 - uberjar/resources/logback.xml | 50 ---------------- .../src/meissa/pallet/meissa_cloud/main.clj | 57 ------------------- 13 files changed, 302 deletions(-) rename src/main/cljc/dda/c4k_nextcloud/{cloud.cljc => nextcloud.cljc} (100%) delete mode 100644 test/resources/meissa/pallet/meissa_cloud/convention_test/should_generate_infra_for_convention.edn delete mode 100644 test/resources/meissa/pallet/meissa_cloud/convention_test/should_generate_k8s_convention.edn delete mode 100644 test/resources/meissa/pallet/meissa_cloud/convention_test/should_validate_input.0.edn delete mode 100644 test/resources/meissa/pallet/meissa_cloud/convention_test/should_validate_input.1.edn delete mode 100644 test/resources/meissa/pallet/meissa_cloud/convention_test/should_validate_input.2.edn delete mode 100644 test/src/meissa/pallet/meissa_cloud/app_test.clj delete mode 100644 test/src/meissa/pallet/meissa_cloud/convention/bash_php_test.clj delete mode 100644 test/src/meissa/pallet/meissa_cloud/convention/bash_test.clj delete mode 100644 test/src/meissa/pallet/meissa_cloud/convention_test.clj delete mode 100644 uberjar/resources/localhost-target.edn delete mode 100644 uberjar/resources/logback.xml delete mode 100644 uberjar/src/meissa/pallet/meissa_cloud/main.clj diff --git a/src/main/cljc/dda/c4k_nextcloud/cloud.cljc b/src/main/cljc/dda/c4k_nextcloud/nextcloud.cljc similarity index 100% rename from src/main/cljc/dda/c4k_nextcloud/cloud.cljc rename to src/main/cljc/dda/c4k_nextcloud/nextcloud.cljc diff --git a/test/resources/meissa/pallet/meissa_cloud/convention_test/should_generate_infra_for_convention.edn b/test/resources/meissa/pallet/meissa_cloud/convention_test/should_generate_infra_for_convention.edn deleted file mode 100644 index 9b4e226..0000000 --- a/test/resources/meissa/pallet/meissa_cloud/convention_test/should_generate_infra_for_convention.edn +++ /dev/null @@ -1,30 +0,0 @@ -{:input {:user :k8s - :external-ip "95.217.221.140" - :fqdn "cloud.test.meissa-gmbh.de" - :cert-manager :letsencrypt-staging-issuer - :db-user-password "test1234" - :admin-user "root" - :admin-password "test1234" - :storage-size 50 - :restic-repository "test4321" - :aws-access-key-id "10" - :aws-secret-access-key "secret" - :restic-password "test4321"} - :expected {:meissa-cloud - {:user :k8s - :backup {:restic-repository "test4321" - :aws-access-key-id "10" - :aws-secret-access-key "secret" - :restic-password "test4321"} - :cloud {:fqdn "cloud.test.meissa-gmbh.de" - :secret-name "cloud-test-meissa-gmbh-de" - :cluster-issuer "letsencrypt-staging-issuer" - :db-name "cloud" - :db-user-password "test1234" - :db-user-name "cloud" - :admin-user "root" - :admin-password "test1234" - :storage-size "50"} - :postgres {:db-user-password "test1234" - :db-user-name "cloud"}}} - } diff --git a/test/resources/meissa/pallet/meissa_cloud/convention_test/should_generate_k8s_convention.edn b/test/resources/meissa/pallet/meissa_cloud/convention_test/should_generate_k8s_convention.edn deleted file mode 100644 index 0478a77..0000000 --- a/test/resources/meissa/pallet/meissa_cloud/convention_test/should_generate_k8s_convention.edn +++ /dev/null @@ -1,30 +0,0 @@ -{:input {:user :k8s - :external-ip "95.217.221.140" - :fqdn "cloud.test.meissa-gmbh.de" - :cert-manager :letsencrypt-staging-issuer - :db-user-password "test1234" - :admin-user "root" - :admin-password "test1234" - :storage-size 50 - :restic-repository "cloud" - :aws-access-key-id "10" - :aws-secret-access-key "secret" - :restic-password "test4321"} - :expected {:user :k8s, :k8s {:external-ip "95.217.221.140"}, - :cert-manager :letsencrypt-staging-issuer}} - -{:input {:user :k8s - :external-ip "95.217.221.140" - :fqdn "cloud.test.meissa-gmbh.de" - :cert-manager :letsencrypt-staging-issuer - :db-user-password "test1234" - :admin-user "root" - :admin-password "test1234" - :storage-size 50 - :restic-repository "cloud" - :aws-access-key-id "10" - :aws-secret-access-key "secret" - :restic-password "test4321" - :u18-04 true} - :expected {:user :k8s, :k8s {:external-ip "95.217.221.140", :u18-04 true} - :cert-manager :letsencrypt-staging-issuer}} diff --git a/test/resources/meissa/pallet/meissa_cloud/convention_test/should_validate_input.0.edn b/test/resources/meissa/pallet/meissa_cloud/convention_test/should_validate_input.0.edn deleted file mode 100644 index 7dda0b7..0000000 --- a/test/resources/meissa/pallet/meissa_cloud/convention_test/should_validate_input.0.edn +++ /dev/null @@ -1,14 +0,0 @@ -{:input {:user :k8s - :external-ip "95.217.221.140" - :fqdn "cloud.test.meissa-gmbh.de" - :cert-manager :letsencrypt-staging-issuer - :db-user-password "test1234" - :admin-user "root" - :admin-password "test1234" - :storage-size 50 - :restic-repository "test4321" - :aws-access-key-id "10" - :aws-secret-access-key "secret" - :restic-password "test4321"} - :expected true - } diff --git a/test/resources/meissa/pallet/meissa_cloud/convention_test/should_validate_input.1.edn b/test/resources/meissa/pallet/meissa_cloud/convention_test/should_validate_input.1.edn deleted file mode 100644 index b8aec50..0000000 --- a/test/resources/meissa/pallet/meissa_cloud/convention_test/should_validate_input.1.edn +++ /dev/null @@ -1,14 +0,0 @@ -{:input {:user :k8s - :external-ip "95.217.221.140" - :fqdn "cloud.test.meissa-gmbh.de" - :cert-manager :letsencrypt-staging-issuer - :db-user-password "test-1234" - :admin-user "root" - :admin-password "test1234" - :storage-size 50 - :restic-repository "test4321" - :aws-access-key-id "10" - :aws-secret-access-key "secret" - :restic-password "test4321"} - :expected false - } diff --git a/test/resources/meissa/pallet/meissa_cloud/convention_test/should_validate_input.2.edn b/test/resources/meissa/pallet/meissa_cloud/convention_test/should_validate_input.2.edn deleted file mode 100644 index ada2a9e..0000000 --- a/test/resources/meissa/pallet/meissa_cloud/convention_test/should_validate_input.2.edn +++ /dev/null @@ -1,14 +0,0 @@ -{:input {:user :k8s - :external-ip "95.217.221.140" - :fqdn "cloud.test.meissa-gmbh.de" - :cert-manager :letsencrypt-staging-issuer - :db-user-password "test1234" - :admin-user "root" - :admin-password "test1234" - :storage-size 50 - :restic-repository "test4321" - :aws-access-key-id "1$0" - :aws-secret-access-key "secret" - :restic-password "test4321"} - :expected false - } diff --git a/test/src/meissa/pallet/meissa_cloud/app_test.clj b/test/src/meissa/pallet/meissa_cloud/app_test.clj deleted file mode 100644 index 1cbda1b..0000000 --- a/test/src/meissa/pallet/meissa_cloud/app_test.clj +++ /dev/null @@ -1,31 +0,0 @@ -(ns meissa.pallet.meissa-nextcloud.app-test - (:require - [clojure.test :refer :all] - [schema.core :as s] - [meissa.pallet.meissa-nextcloud.app :as sut])) - -(s/set-fn-validation! true) - -(s/def test-convention-conf - {:user :k8s - :external-ip "12.121.111.121" - :fqdn "some.domain.de" - :cert-manager :letsencrypt-staging-issuer - :db-user-password "test1234" - :admin-user "root" - :admin-password "test1234" - :storage-size 50 - :restic-repository "nextcloud" - :aws-access-key-id "10" - :aws-secret-access-key "secret" - :restic-password "test4321"}) - -(deftest app-config - (testing - "test plan-def" - (is (map? (sut/app-configuration-resolved test-convention-conf))))) - -(deftest plan-def - (testing - "test plan-def" - (is (map? sut/with-nextcloud)))) diff --git a/test/src/meissa/pallet/meissa_cloud/convention/bash_php_test.clj b/test/src/meissa/pallet/meissa_cloud/convention/bash_php_test.clj deleted file mode 100644 index 787fd6c..0000000 --- a/test/src/meissa/pallet/meissa_cloud/convention/bash_php_test.clj +++ /dev/null @@ -1,20 +0,0 @@ -(ns meissa.pallet.meissa-cloud.convention.bash-php-test - (:require - [clojure.test :refer :all] - [meissa.pallet.meissa-cloud.convention.bash-php :as sut])) - - -(deftest test-it - (is (= false - (sut/bash-php-env-string? 4))) - (is (= false - (sut/bash-php-env-string? "hal-lo"))) - (is (= false - (sut/bash-php-env-string? "hal--lo"))) - (is (= false - (sut/bash-php-env-string? "hal\\lo"))) - (is (= true - (sut/bash-php-env-string? "test"))) - (is (= true - (sut/bash-php-env-string? "test123"))) - ) \ No newline at end of file diff --git a/test/src/meissa/pallet/meissa_cloud/convention/bash_test.clj b/test/src/meissa/pallet/meissa_cloud/convention/bash_test.clj deleted file mode 100644 index ce51ba6..0000000 --- a/test/src/meissa/pallet/meissa_cloud/convention/bash_test.clj +++ /dev/null @@ -1,22 +0,0 @@ -(ns meissa.pallet.meissa-cloud.convention.bash-test - (:require - [clojure.test :refer :all] - [meissa.pallet.meissa-cloud.convention.bash :as sut])) - - -(deftest test-it - (is (= false - (sut/bash-env-string? 4))) - (is (= false - (sut/bash-env-string? "1$0"))) - (is (= false - (sut/bash-env-string? "'hallo"))) - (is (= false - (sut/bash-env-string? "hallo\""))) - (is (= false - (sut/bash-env-string? "hall$o"))) - (is (= true - (sut/bash-env-string? "test"))) - (is (= true - (sut/bash-env-string? "test123"))) - ) \ No newline at end of file diff --git a/test/src/meissa/pallet/meissa_cloud/convention_test.clj b/test/src/meissa/pallet/meissa_cloud/convention_test.clj deleted file mode 100644 index e5e30c1..0000000 --- a/test/src/meissa/pallet/meissa_cloud/convention_test.clj +++ /dev/null @@ -1,18 +0,0 @@ -(ns meissa.pallet.meissa-cloud.convention-test - (:require - [clojure.test :refer :all] - [data-test :refer :all] - [meissa.pallet.meissa-cloud.convention :as sut] - [clojure.spec.alpha :as sp])) - -(defdatatest should-generate-infra-for-convention [input expected] - (is (= expected - (sut/infra-configuration input)))) - -(defdatatest should-generate-k8s-convention [input expected] - (is (= expected - (sut/k8s-convention-configuration input)))) - -(defdatatest should-validate-input [input expected] - (is (= expected - (sp/valid? sut/cloud-convention-resolved? input)))) diff --git a/uberjar/resources/localhost-target.edn b/uberjar/resources/localhost-target.edn deleted file mode 100644 index d49f818..0000000 --- a/uberjar/resources/localhost-target.edn +++ /dev/null @@ -1,2 +0,0 @@ -{:existing [{:node-name "localhost" - :node-ip "127.0.0.1"}]} diff --git a/uberjar/resources/logback.xml b/uberjar/resources/logback.xml deleted file mode 100644 index 8985f2b..0000000 --- a/uberjar/resources/logback.xml +++ /dev/null @@ -1,50 +0,0 @@ - - - - %d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n - - - INFO - - - - - logs/pallet.log - - logs/old/pallet.%d{yyyy-MM-dd}.log - 3 - - - %date %level [%thread] %logger{10} %msg%n - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/uberjar/src/meissa/pallet/meissa_cloud/main.clj b/uberjar/src/meissa/pallet/meissa_cloud/main.clj deleted file mode 100644 index 4ba35d6..0000000 --- a/uberjar/src/meissa/pallet/meissa_cloud/main.clj +++ /dev/null @@ -1,57 +0,0 @@ -(ns meissa.pallet.meissa-nextcloud.main - (:gen-class) - (:require - [clojure.string :as str] - [clojure.tools.cli :as cli] - [dda.pallet.core.main-helper :as mh] - [dda.pallet.core.app :as core-app] - [meissa.pallet.meissa-nextcloud.app :as app])) - -(def cli-options - [["-h" "--help"] - ["-c" "--configure"] - ["-t" "--targets example-targets.edn" "edn file containing the targets to install on." - :default "localhost-target.edn"] - ["-v" "--verbose"]]) - -(defn usage [options-summary] - (str/join - \newline - ["meissa-nextcloud installs & configures a single host kubernetes cluster with nextcloud installed" - "" - "Usage: java -jar meissa-nextcloud-standalone.jar [options] nextcloud.edn" - "" - "Options:" - options-summary - "" - "nextcloud.edn" - " - follows the edn format." - " - has to be a valid nextcloudConventionConfig" - ""])) - -(defn -main [& args] - (let [{:keys [options arguments errors summary help]} (cli/parse-opts args cli-options) - verbose (if (contains? options :verbose) 1 0)] - (cond - help (mh/exit 0 (usage summary)) - errors (mh/exit 1 (mh/error-msg errors)) - (not= (count arguments) 1) (mh/exit 1 (usage summary)) - (:serverspec options) (if (core-app/existing-serverspec - app/crate-app - {:convention (first arguments) - :targets (:targets options) - :verbosity verbose}) - (mh/exit-test-passed) - (mh/exit-test-failed)) - (:configure options) (if (core-app/existing-configure - app/crate-app - {:convention (first arguments) - :targets (:targets options)}) - (mh/exit-default-success) - (mh/exit-default-error)) - :default (if (core-app/existing-install - app/crate-app - {:convention (first arguments) - :targets (:targets options)}) - (mh/exit-default-success) - (mh/exit-default-error))))) From 23619a7a61c5e23da8cd9eb6ea92d6fef6e21821 Mon Sep 17 00:00:00 2001 From: leo Date: Tue, 10 Aug 2021 11:22:45 +0200 Subject: [PATCH 06/12] fixed project.clj, compiles now --- project.clj | 80 ++++++++++++++++++++++------------------------------- 1 file changed, 33 insertions(+), 47 deletions(-) diff --git a/project.clj b/project.clj index be010d3..f5a8d48 100644 --- a/project.clj +++ b/project.clj @@ -1,55 +1,41 @@ (defproject meissa/meissa-cloud "1.0.2-SNAPSHOT" - :description "Crate to install cloud" - :url "https://meissa-gmbh.de" - :license {:name "meissa commercial license" - :url "https://www.meissa-gmbh.de"} - :dependencies [[dda/dda-pallet "4.0.3"] - [dda/dda-k8s-crate "1.0.1"] - [orchestra "2021.01.01-1"]] + :description "nextcloud c4k-installation package" + :url "https://domaindrivenarchitecture.org" + :license {:name "Apache License, Version 2.0" + :url "https://www.apache.org/licenses/LICENSE-2.0.html"} + :dependencies [[org.clojure/clojure "1.10.3"] + [org.clojure/tools.reader "1.3.4"] + [org.domaindrivenarchitecture/c4k-common-clj "0.2.8"]] :target-path "target/%s/" - :source-paths ["main/src"] - :resource-paths ["main/resources"] - :repositories [["clojars" "https://clojars.org/repo"] - ["snapshots" - "https://artifact.prod.meissa-gmbh.de/repository/maven-snapshots/"] - ["releases" - "https://artifact.prod.meissa-gmbh.de/repository/maven-releases/"]] - :deploy-repositories [["snapshots" "https://artifact.prod.meissa-gmbh.de/repository/maven-snapshots/"] - ["releases" "https://artifact.prod.meissa-gmbh.de/repository/maven-releases/"]] - :profiles {:dev {:source-paths ["integration/src" - "test/src" - "uberjar/src"] - :resource-paths ["integration/resources" - "test/resources"] - :dependencies - [[org.clojure/test.check "1.1.0"] - [dda/data-test "0.1.1"] - [dda/pallet "0.9.1" :classifier "tests"] - [ch.qos.logback/logback-classic "1.3.0-alpha5"] - [org.slf4j/jcl-over-slf4j "2.0.0-alpha1"]] - :plugins - [[lein-sub "0.3.0"]] - :leiningen/reply - {:dependencies [[org.slf4j/jcl-over-slf4j "1.8.0-beta2"]] - :exclusions [commons-logging]}} - :test {:test-paths ["test/src"] - :resource-paths ["test/resources"] - :dependencies [[dda/pallet "0.9.1" :classifier "tests"]]} - :uberjar {:source-paths ["uberjar/src"] - :resource-paths ["uberjar/resources"] - :aot :all + :source-paths ["src/main/cljc" + "src/main/clj"] + :resource-paths ["src/main/resources"] + :repositories [["snapshots" :clojars] + ["releases" :clojars]] + :deploy-repositories [["snapshots" :clojars] + ["releases" :clojars]] + :profiles {:test {:test-paths ["src/test/cljc"] + :resource-paths ["src/test/resources"] + :dependencies [[dda/data-test "0.1.1"]]} + :dev {:plugins [[lein-shell "0.5.0"]]} + :uberjar {:aot :all :main dda.c4k-nextcloud.uberjar :uberjar-name "c4k-nextcloud-standalone.jar" - :dependencies [[org.clojure/tools.cli "1.0.194"] - [ch.qos.logback/logback-classic "1.3.0-alpha5"] + :dependencies [[org.clojure/tools.cli "1.0.206"] + [ch.qos.logback/logback-classic "1.3.0-alpha4" + :exclusions [com.sun.mail/javax.mail]] [org.slf4j/jcl-over-slf4j "2.0.0-alpha1"]]}} - :release-tasks [["vcs" "assert-committed"] + :release-tasks [["test"] + ["vcs" "assert-committed"] ["change" "version" "leiningen.release/bump-version" "release"] ["vcs" "commit"] ["vcs" "tag"] - ["deploy"] - ["uberjar"] - ["change" "version" "leiningen.release/bump-version"] - ["vcs" "commit"] - ["vcs" "push"]] - :local-repo-classpath true) + ["change" "version" "leiningen.release/bump-version"]] + :aliases {"native" ["shell" + "native-image" + "--report-unsupported-elements-at-runtime" + "--initialize-at-build-time" + "-jar" "target/uberjar/c4k-jira-standalone.jar" + "-H:ResourceConfigurationFiles=graalvm-resource-config.json" + "-H:Log=registerResource" + "-H:Name=target/graalvm/${:name}"]}) \ No newline at end of file From 655a84a5fc396bd7fd723b65632bcbcfff8a2e05 Mon Sep 17 00:00:00 2001 From: leo Date: Tue, 10 Aug 2021 14:34:17 +0200 Subject: [PATCH 07/12] first working version --- .gitlab-ci.yml | 28 +++++------ ...PONENT_LICENSE => SUBCOMPONENT_LICENSE.md} | 0 package.json | 8 +-- project.clj | 4 +- public/index.html | 16 +++--- shadow-cljs.edn | 2 +- src/main/cljc/dda/c4k_nextcloud/core.cljc | 3 +- .../cljc/dda/c4k_nextcloud/nextcloud.cljc | 7 +-- .../{cloud => nextcloud}/certificate.yaml | 0 .../{cloud => nextcloud}/configure-as-user.sh | 0 .../{cloud => nextcloud}/deployment.yaml | 4 +- .../{cloud => nextcloud}/ingress.yaml | 0 .../install-as-root.sh.template | 0 .../persistent-volume.yaml | 2 +- .../{cloud => nextcloud}/pod-running.sh | 0 .../resources/{cloud => nextcloud}/pvc.yaml | 2 +- .../{cloud => nextcloud}/secret.yaml | 0 .../{cloud => nextcloud}/service.yaml | 2 +- .../postgres/install-as-root.sh.template | 4 -- .../resources/postgres/postgres-config.yaml | 10 ---- .../postgres/postgres-deployment.yaml | 49 ------------------- .../postgres/postgres-persistent-volume.yaml | 15 ------ src/main/resources/postgres/postgres-pvc.yaml | 16 ------ .../resources/postgres/postgres-secret.yaml | 9 ---- .../resources/postgres/postgres-service.yaml | 10 ---- targets.edn | 3 -- valid-auth.edn | 4 +- valid-config.edn | 3 +- 28 files changed, 44 insertions(+), 157 deletions(-) rename doc/{SUBCOMPONENT_LICENSE => SUBCOMPONENT_LICENSE.md} (100%) rename src/main/resources/{cloud => nextcloud}/certificate.yaml (100%) rename src/main/resources/{cloud => nextcloud}/configure-as-user.sh (100%) rename src/main/resources/{cloud => nextcloud}/deployment.yaml (96%) rename src/main/resources/{cloud => nextcloud}/ingress.yaml (100%) rename src/main/resources/{cloud => nextcloud}/install-as-root.sh.template (100%) rename src/main/resources/{cloud => nextcloud}/persistent-volume.yaml (83%) rename src/main/resources/{cloud => nextcloud}/pod-running.sh (100%) rename src/main/resources/{cloud => nextcloud}/pvc.yaml (83%) rename src/main/resources/{cloud => nextcloud}/secret.yaml (100%) rename src/main/resources/{cloud => nextcloud}/service.yaml (71%) delete mode 100644 src/main/resources/postgres/install-as-root.sh.template delete mode 100644 src/main/resources/postgres/postgres-config.yaml delete mode 100644 src/main/resources/postgres/postgres-deployment.yaml delete mode 100644 src/main/resources/postgres/postgres-persistent-volume.yaml delete mode 100644 src/main/resources/postgres/postgres-pvc.yaml delete mode 100644 src/main/resources/postgres/postgres-secret.yaml delete mode 100644 src/main/resources/postgres/postgres-service.yaml delete mode 100644 targets.edn diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 81a9d2d..adca075 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -47,7 +47,7 @@ test-schema: stage: build_and_test script: - lein uberjar - - java -jar target/uberjar/c4k-cloud-standalone.jar valid-config.edn valid-auth.edn | kubeconform --kubernetes-version 1.19.0 --strict --skip Certificate - + - java -jar target/uberjar/c4k-nextcloud-standalone.jar valid-config.edn valid-auth.edn | kubeconform --kubernetes-version 1.19.0 --strict --skip Certificate - artifacts: paths: - target/uberjar @@ -68,9 +68,9 @@ test-schema: script: - mkdir -p target/frontend-build - shadow-cljs release frontend - - cp public/js/main.js target/frontend-build/c4k-cloud.js - - sha256sum target/frontend-build/c4k-cloud.js > target/frontend-build/c4k-cloud.js.sha256 - - sha512sum target/frontend-build/c4k-cloud.js > target/frontend-build/c4k-cloud.js.sha512 + - cp public/js/main.js target/frontend-build/c4k-nextcloud.js + - sha256sum target/frontend-build/c4k-nextcloud.js > target/frontend-build/c4k-nextcloud.js.sha256 + - sha512sum target/frontend-build/c4k-nextcloud.js > target/frontend-build/c4k-nextcloud.js.sha512 artifacts: paths: - target/frontend-build @@ -79,8 +79,8 @@ package-uberjar: <<: *clj stage: package script: - - sha256sum target/uberjar/c4k-cloud-standalone.jar > target/uberjar/c4k-cloud-standalone.jar.sha256 - - sha512sum target/uberjar/c4k-cloud-standalone.jar > target/uberjar/c4k-cloud-standalone.jar.sha512 + - sha256sum target/uberjar/c4k-nextcloud-standalone.jar > target/uberjar/c4k-nextcloud-standalone.jar.sha256 + - sha512sum target/uberjar/c4k-nextcloud-standalone.jar > target/uberjar/c4k-nextcloud-standalone.jar.sha512 artifacts: paths: - target/uberjar @@ -117,20 +117,20 @@ release: - apk --no-cache add curl - | release-cli create --name "Release $CI_COMMIT_TAG" --tag-name $CI_COMMIT_TAG \ - --assets-link "{\"name\":\"c4k-cloud-standalone.jar\",\"url\":\"https://gitlab.com/domaindrivenarchitecture/c4k-nextcloud/-/jobs/${CI_JOB_ID}/artifacts/file/target/uberjar/c4k-cloud-standalone.jar\"}" \ - --assets-link "{\"name\":\"c4k-cloud-standalone.jar.sha256\",\"url\":\"https://gitlab.com/domaindrivenarchitecture/c4k-nextcloud/-/jobs/${CI_JOB_ID}/artifacts/file/target/uberjar/c4k-cloud-standalone.jar.sha256\"}" \ - --assets-link "{\"name\":\"c4k-cloud-standalone.jar.sha512\",\"url\":\"https://gitlab.com/domaindrivenarchitecture/c4k-nextcloud/-/jobs/${CI_JOB_ID}/artifacts/file/target/uberjar/c4k-cloud-standalone.jar.sha512\"}" \ - --assets-link "{\"name\":\"c4k-cloud.js\",\"url\":\"https://gitlab.com/domaindrivenarchitecture/c4k-nextcloud/-/jobs/${CI_JOB_ID}/artifacts/file/target/frontend-build/c4k-cloud.js\"}" \ - --assets-link "{\"name\":\"c4k-cloud.js.sha256\",\"url\":\"https://gitlab.com/domaindrivenarchitecture/c4k-nextcloud/-/jobs/${CI_JOB_ID}/artifacts/file/target/frontend-build/c4k-cloud.js.sha256\"}" \ - --assets-link "{\"name\":\"c4k-cloud.js.sha512\",\"url\":\"https://gitlab.com/domaindrivenarchitecture/c4k-nextcloud/-/jobs/${CI_JOB_ID}/artifacts/file/target/frontend-build/c4k-cloud.js.sha512\"}" \ + --assets-link "{\"name\":\"c4k-nextcloud-standalone.jar\",\"url\":\"https://gitlab.com/domaindrivenarchitecture/c4k-nextcloud/-/jobs/${CI_JOB_ID}/artifacts/file/target/uberjar/c4k-nextcloud-standalone.jar\"}" \ + --assets-link "{\"name\":\"c4k-nextcloud-standalone.jar.sha256\",\"url\":\"https://gitlab.com/domaindrivenarchitecture/c4k-nextcloud/-/jobs/${CI_JOB_ID}/artifacts/file/target/uberjar/c4k-nextcloud-standalone.jar.sha256\"}" \ + --assets-link "{\"name\":\"c4k-nextcloud-standalone.jar.sha512\",\"url\":\"https://gitlab.com/domaindrivenarchitecture/c4k-nextcloud/-/jobs/${CI_JOB_ID}/artifacts/file/target/uberjar/c4k-nextcloud-standalone.jar.sha512\"}" \ + --assets-link "{\"name\":\"c4k-nextcloud.js\",\"url\":\"https://gitlab.com/domaindrivenarchitecture/c4k-nextcloud/-/jobs/${CI_JOB_ID}/artifacts/file/target/frontend-build/c4k-nextcloud.js\"}" \ + --assets-link "{\"name\":\"c4k-nextcloud.js.sha256\",\"url\":\"https://gitlab.com/domaindrivenarchitecture/c4k-nextcloud/-/jobs/${CI_JOB_ID}/artifacts/file/target/frontend-build/c4k-nextcloud.js.sha256\"}" \ + --assets-link "{\"name\":\"c4k-nextcloud.js.sha512\",\"url\":\"https://gitlab.com/domaindrivenarchitecture/c4k-nextcloud/-/jobs/${CI_JOB_ID}/artifacts/file/target/frontend-build/c4k-nextcloud.js.sha512\"}" \ -cloud-image-test-publish: +nextcloud-image-test-publish: image: domaindrivenarchitecture/devops-build:latest stage: image rules: - if: '$CI_COMMIT_TAG != null' script: - - cd infrastructure/docker-cloud && pyb image test publish + - cd infrastructure/docker-nextcloud && pyb image test publish backup-image-test-publish: image: domaindrivenarchitecture/devops-build:latest diff --git a/doc/SUBCOMPONENT_LICENSE b/doc/SUBCOMPONENT_LICENSE.md similarity index 100% rename from doc/SUBCOMPONENT_LICENSE rename to doc/SUBCOMPONENT_LICENSE.md diff --git a/package.json b/package.json index f29d34b..a79ac7d 100644 --- a/package.json +++ b/package.json @@ -1,18 +1,18 @@ { - "name": "c4k-cloud", + "name": "c4k-nextcloud", "description": "Generate c4k yaml for a nextcloud deployment.", "author": "meissa GmbH", "version": "0.1.3-SNAPSHOT", "homepage": "https://gitlab.com/domaindrivenarchitecture/c4k-nextcloud#readme", "repository": "https://www.npmjs.com/package/c4k-nextcloud", "license": "APACHE2", - "main": "c4k-cloud.js", + "main": "c4k-nextcloud.js", "bin": { - "c4k-cloud": "./c4k-cloud.js" + "c4k-nextcloud": "./c4k-nextcloud.js" }, "keywords": [ "cljs", - "cloud", + "nextcloud", "k8s", "c4k", "deployment", diff --git a/project.clj b/project.clj index f5a8d48..538f3ce 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject meissa/meissa-cloud "1.0.2-SNAPSHOT" +(defproject org.domaindrivenarchitecture/c4k-nextcloud "1.0.2-SNAPSHOT" :description "nextcloud c4k-installation package" :url "https://domaindrivenarchitecture.org" :license {:name "Apache License, Version 2.0" @@ -35,7 +35,7 @@ "native-image" "--report-unsupported-elements-at-runtime" "--initialize-at-build-time" - "-jar" "target/uberjar/c4k-jira-standalone.jar" + "-jar" "target/uberjar/c4k-nextcloud-standalone.jar" "-H:ResourceConfigurationFiles=graalvm-resource-config.json" "-H:Log=registerResource" "-H:Name=target/graalvm/${:name}"]}) \ No newline at end of file diff --git a/public/index.html b/public/index.html index 1e2614d..0f0d145 100644 --- a/public/index.html +++ b/public/index.html @@ -14,24 +14,24 @@
- +

       
- - + +
-

+        

       
-

+        

       
-

+        

       

@@ -42,8 +42,8 @@

-
-

-      
-

- -


-
- - -
- -
- - - - \ No newline at end of file diff --git a/src/test/cljc/dda/c4k_nextcloud/backup_test.cljc b/src/test/cljc/dda/c4k_nextcloud/backup_test.cljc new file mode 100644 index 0000000..d9b5f38 --- /dev/null +++ b/src/test/cljc/dda/c4k_nextcloud/backup_test.cljc @@ -0,0 +1,93 @@ +(ns dda.c4k-nextcloud.backup-test + (:require + #?(:clj [clojure.test :refer [deftest is are testing run-tests]] + :cljs [cljs.test :refer-macros [deftest is are testing run-tests]]) + [dda.c4k-nextcloud.backup :as cut])) + + +(deftest should-generate-secret + (is (= {:apiVersion "v1" + :kind "Secret" + :metadata {:name "backup-secret"} + :type "Opaque" + :data + {:aws-access-key-id "YXdzLWlk", :aws-secret-access-key "YXdzLXNlY3JldA==", :restic-password "cmVzdGljLXB3"}} + (cut/generate-secret {:aws-access-key-id "aws-id" :aws-secret-access-key "aws-secret" :restic-password "restic-pw"})))) + +(deftest should-generate-config + (is (= {:apiVersion "v1" + :kind "ConfigMap" + :metadata {:name "backup-config" + :labels {:app.kubernetes.io/name "backup" + :app.kubernetes.io/part-of "cloud"}} + :data + {:restic-repository "s3:restic-repository"}} + (cut/generate-config {:restic-repository "s3:restic-repository"})))) + +(deftest should-generate-cron + (is (= {:apiVersion "batch/v1beta1" + :kind "CronJob" + :metadata {:name "nextcloud-backup" + :labels {:app.kubernetes.part-of "cloud"}} + :spec {:schedule "10 23 * * *" + :successfulJobsHistoryLimit 1 + :failedJobsHistoryLimit 1 + :jobTemplate + {:spec + {:template + {:spec + {:containers + [{:name "backup-app" + :image "domaindrivenarchitecture/c4k-nextcloud-backup" + :imagePullPolicy "IfNotPresent" + :command ["/entrypoint.sh"] + :env + [{:name "POSTGRES_USER" + :valueFrom + {:secretKeyRef + {:name "postgres-secret" + :key "postgres-user"}}} + {:name "POSTGRES_PASSWORD" + :valueFrom + {:secretKeyRef + {:name "postgres-secret" + :key "postgres-password"}}} + {:name "POSTGRES_DB" + :valueFrom + {:configMapKeyRef + {:name "postgres-config" + :key "postgres-db"}}} + {:name "POSTGRES_HOST" + :value "postgresql-service:5432"} + {:name "POSTGRES_SERVICE" + :value "postgresql-service"} + {:name "POSTGRES_PORT" + :value "5432"} + {:name "AWS_DEFAULT_REGION" + :value "eu-central-1"} + {:name "AWS_ACCESS_KEY_ID_FILE" + :value "/var/run/secrets/backup-secrets/aws-access-key-id"} + {:name "AWS_SECRET_ACCESS_KEY_FILE" + :value "/var/run/secrets/backup-secrets/aws-secret-access-key"} + {:name "RESTIC_REPOSITORY" + :valueFrom + {:configMapKeyRef + {:name "backup-config" + :key "restic-repository"}}} + {:name "RESTIC_PASSWORD_FILE" + :value "/var/run/secrets/backup-secrets/restic-password"}] + :volumeMounts + [{:name "nextcloud-data-volume" + :mountPath "/var/backups"} + {:name "backup-secret-volume" + :mountPath "/var/run/secrets/backup-secrets" + :readOnly true}]}] + :volumes + [{:name "nextcloud-data-volume" + :persistentVolumeClaim + {:claimName "nextcloud-pvc"}} + {:name "backup-secret-volume" + :secret + {:secretName "backup-secret"}}] + :restartPolicy "OnFailure"}}}}}} + (cut/generate-cron)))) diff --git a/src/test/cljc/dda/c4k_nextcloud/core_test.cljc b/src/test/cljc/dda/c4k_nextcloud/core_test.cljc new file mode 100644 index 0000000..c841770 --- /dev/null +++ b/src/test/cljc/dda/c4k_nextcloud/core_test.cljc @@ -0,0 +1,35 @@ +(ns dda.c4k-nextcloud.core-test + (:require + #?(:clj [clojure.test :refer [deftest is are testing run-tests]] + :cljs [cljs.test :refer-macros [deftest is are testing run-tests]]) + [dda.c4k-nextcloud.core :as cut])) + +(deftest should-k8s-objects + (is (= 16 + (count (cut/k8s-objects {:fqdn "nextcloud-neu.prod.meissa-gmbh.de" + :postgres-db-user "nextcloud" + :postgres-db-password "nextcloud-db-password" + :issuer :prod + :nextcloud-data-volume-path "/var/nextcloud" + :postgres-data-volume-path "/var/postgres" + :aws-access-key-id "aws-id" + :aws-secret-access-key "aws-secret" + :restic-password "restic-pw" + :restic-repository "restic-repository"})))) + (is (= 14 + (count (cut/k8s-objects {:fqdn "nextcloud-neu.prod.meissa-gmbh.de" + :postgres-db-user "nextcloud" + :postgres-db-password "nextcloud-db-password" + :issuer :prod + :aws-access-key-id "aws-id" + :aws-secret-access-key "aws-secret" + :restic-password "restic-pw" + :restic-repository "restic-repository"})))) + (is (= 11 + (count (cut/k8s-objects {:fqdn "nextcloud-neu.prod.meissa-gmbh.de" + :postgres-db-user "nextcloud" + :postgres-db-password "nextcloud-db-password" + :issuer :prod + :aws-access-key-id "aws-id" + :aws-secret-access-key "aws-secret" + :restic-password "restic-pw"}))))) diff --git a/src/test/cljc/dda/c4k_nextcloud/nextcloud_test.cljc b/src/test/cljc/dda/c4k_nextcloud/nextcloud_test.cljc new file mode 100644 index 0000000..b48bd0f --- /dev/null +++ b/src/test/cljc/dda/c4k_nextcloud/nextcloud_test.cljc @@ -0,0 +1,81 @@ +(ns dda.c4k-nextcloud.nextcloud-test + (:require + #?(:clj [clojure.test :refer [deftest is are testing run-tests]] + :cljs [cljs.test :refer-macros [deftest is are testing run-tests]]) + [dda.c4k-nextcloud.nextcloud :as cut])) + +(deftest should-generate-certificate + (is (= {:apiVersion "cert-manager.io/v1alpha2" + :kind "Certificate" + :metadata {:name "cloud-cert", :namespace "default"} + :spec + {:secretName "cloud-secret" + :commonName "xx" + :dnsNames ["xx"] + :issuerRef + {:name "letsencrypt-prod-issuer", :kind "ClusterIssuer"}}} + (cut/generate-certificate {:fqdn "xx" :issuer :prod})))) + +(deftest should-generate-ingress + (is (= {:apiVersion "extensions/v1beta1" + :kind "Ingress" + :metadata + {:name "ingress-cloud" + :annotations + {:cert-manager.io/cluster-issuer + "letsencrypt-staging-issuer" + :nginx.ingress.kubernetes.io/proxy-body-size "256m" + :nginx.ingress.kubernetes.io/ssl-redirect "true" + :nginx.ingress.kubernetes.io/rewrite-target "/" + :nginx.ingress.kubernetes.io/proxy-connect-timeout "300" + :nginx.ingress.kubernetes.io/proxy-send-timeout "300" + :nginx.ingress.kubernetes.io/proxy-read-timeout "300"} + :namespace "default"} + :spec + {:tls [{:hosts ["xx"], :secretName "nextcloud-secret"}] + :rules + [{:host "xx" + :http + {:paths + [{:path "/" + :backend + {:serviceName "nextcloud-service", :servicePort 80}}]}}]}} + (cut/generate-ingress {:fqdn "xx"})))) + +(deftest should-generate-persistent-volume + (is (= {:kind "PersistentVolume" + :apiVersion "v1" + :metadata {:name "cloud-pv-volume", :labels {:type "local" :app "cloud"}} + :spec + {:storageClassName "manual" + :accessModes ["ReadWriteOnce"] + :capacity {:storage "200Gi"} + :hostPath {:path "xx"}}} + (cut/generate-persistent-volume {:nextcloud-data-volume-path "xx"})))) + +(deftest should-generate-deployment + (is (= {:containers + [{:image "domaindrivenarchitecture/meissa-cloud-app" + :name "cloud-app" + :imagePullPolicy "IfNotPresent" + :ports [{:containerPort 80}] + :env + [{:name "DB_USERNAME_FILE" + :value + "/var/run/secrets/postgres-secret/postgres-user"} + {:name "DB_PASSWORD_FILE" + :value + "/var/run/secrets/postgres-secret/postgres-password"} + {:name "FQDN", :value "xx"}] + :command ["/app/entrypoint.sh"] + :volumeMounts + [{:mountPath "/var/nextcloud", :name "cloud-data-volume"} + {:name "postgres-secret-volume" + :mountPath "/var/run/secrets/postgres-secret" + :readOnly true}]}] + :volumes + [{:name "cloud-data-volume" + :persistentVolumeClaim {:claimName "cloud-pvc"}} + {:name "postgres-secret-volume" + :secret {:secretName "postgres-secret"}}]} + (get-in (cut/generate-deployment {:fqdn "xx"}) [:spec :template :spec])))) From 70e3b967f1de76505c5318c1b57e59dc6c0db45c Mon Sep 17 00:00:00 2001 From: leo Date: Tue, 10 Aug 2021 16:01:43 +0200 Subject: [PATCH 10/12] fixed some test errors --- src/test/cljc/dda/c4k_nextcloud/backup_test.cljc | 6 +++--- src/test/cljc/dda/c4k_nextcloud/nextcloud_test.cljc | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/test/cljc/dda/c4k_nextcloud/backup_test.cljc b/src/test/cljc/dda/c4k_nextcloud/backup_test.cljc index d9b5f38..8fb9179 100644 --- a/src/test/cljc/dda/c4k_nextcloud/backup_test.cljc +++ b/src/test/cljc/dda/c4k_nextcloud/backup_test.cljc @@ -10,7 +10,7 @@ :kind "Secret" :metadata {:name "backup-secret"} :type "Opaque" - :data + :stringData {:aws-access-key-id "YXdzLWlk", :aws-secret-access-key "YXdzLXNlY3JldA==", :restic-password "cmVzdGljLXB3"}} (cut/generate-secret {:aws-access-key-id "aws-id" :aws-secret-access-key "aws-secret" :restic-password "restic-pw"})))) @@ -27,7 +27,7 @@ (deftest should-generate-cron (is (= {:apiVersion "batch/v1beta1" :kind "CronJob" - :metadata {:name "nextcloud-backup" + :metadata {:name "cloud-backup" :labels {:app.kubernetes.part-of "cloud"}} :spec {:schedule "10 23 * * *" :successfulJobsHistoryLimit 1 @@ -85,7 +85,7 @@ :volumes [{:name "nextcloud-data-volume" :persistentVolumeClaim - {:claimName "nextcloud-pvc"}} + {:claimName "cloud-pvc"}} {:name "backup-secret-volume" :secret {:secretName "backup-secret"}}] diff --git a/src/test/cljc/dda/c4k_nextcloud/nextcloud_test.cljc b/src/test/cljc/dda/c4k_nextcloud/nextcloud_test.cljc index b48bd0f..47f2b9f 100644 --- a/src/test/cljc/dda/c4k_nextcloud/nextcloud_test.cljc +++ b/src/test/cljc/dda/c4k_nextcloud/nextcloud_test.cljc @@ -32,14 +32,14 @@ :nginx.ingress.kubernetes.io/proxy-read-timeout "300"} :namespace "default"} :spec - {:tls [{:hosts ["xx"], :secretName "nextcloud-secret"}] + {:tls [{:hosts ["xx"], :secretName "cloud-secret"}] :rules [{:host "xx" :http {:paths [{:path "/" :backend - {:serviceName "nextcloud-service", :servicePort 80}}]}}]}} + {:serviceName "cloud-service", :servicePort 80}}]}}]}} (cut/generate-ingress {:fqdn "xx"})))) (deftest should-generate-persistent-volume From bc546e33d82b0f27adcede4c919b050f8a92cb00 Mon Sep 17 00:00:00 2001 From: bom Date: Tue, 10 Aug 2021 18:00:43 +0200 Subject: [PATCH 11/12] fixed lein test --- .../cljc/dda/c4k_nextcloud/backup_test.cljc | 98 +++++++------------ .../dda/c4k_nextcloud/nextcloud_test.cljc | 55 ++++++----- 2 files changed, 65 insertions(+), 88 deletions(-) diff --git a/src/test/cljc/dda/c4k_nextcloud/backup_test.cljc b/src/test/cljc/dda/c4k_nextcloud/backup_test.cljc index 8fb9179..4aee6b2 100644 --- a/src/test/cljc/dda/c4k_nextcloud/backup_test.cljc +++ b/src/test/cljc/dda/c4k_nextcloud/backup_test.cljc @@ -27,67 +27,39 @@ (deftest should-generate-cron (is (= {:apiVersion "batch/v1beta1" :kind "CronJob" - :metadata {:name "cloud-backup" - :labels {:app.kubernetes.part-of "cloud"}} - :spec {:schedule "10 23 * * *" - :successfulJobsHistoryLimit 1 - :failedJobsHistoryLimit 1 - :jobTemplate - {:spec - {:template - {:spec - {:containers - [{:name "backup-app" - :image "domaindrivenarchitecture/c4k-nextcloud-backup" - :imagePullPolicy "IfNotPresent" - :command ["/entrypoint.sh"] - :env - [{:name "POSTGRES_USER" - :valueFrom - {:secretKeyRef - {:name "postgres-secret" - :key "postgres-user"}}} - {:name "POSTGRES_PASSWORD" - :valueFrom - {:secretKeyRef - {:name "postgres-secret" - :key "postgres-password"}}} - {:name "POSTGRES_DB" - :valueFrom - {:configMapKeyRef - {:name "postgres-config" - :key "postgres-db"}}} - {:name "POSTGRES_HOST" - :value "postgresql-service:5432"} - {:name "POSTGRES_SERVICE" - :value "postgresql-service"} - {:name "POSTGRES_PORT" - :value "5432"} - {:name "AWS_DEFAULT_REGION" - :value "eu-central-1"} - {:name "AWS_ACCESS_KEY_ID_FILE" - :value "/var/run/secrets/backup-secrets/aws-access-key-id"} - {:name "AWS_SECRET_ACCESS_KEY_FILE" - :value "/var/run/secrets/backup-secrets/aws-secret-access-key"} - {:name "RESTIC_REPOSITORY" - :valueFrom - {:configMapKeyRef - {:name "backup-config" - :key "restic-repository"}}} - {:name "RESTIC_PASSWORD_FILE" - :value "/var/run/secrets/backup-secrets/restic-password"}] - :volumeMounts - [{:name "nextcloud-data-volume" - :mountPath "/var/backups"} - {:name "backup-secret-volume" - :mountPath "/var/run/secrets/backup-secrets" - :readOnly true}]}] - :volumes - [{:name "nextcloud-data-volume" - :persistentVolumeClaim - {:claimName "cloud-pvc"}} - {:name "backup-secret-volume" - :secret - {:secretName "backup-secret"}}] - :restartPolicy "OnFailure"}}}}}} + :metadata {:name "cloud-backup", :labels {:app.kubernetes.part-of "cloud"}} + :spec + {:schedule "10 23 * * *" + :successfulJobsHistoryLimit 0 + :failedJobsHistoryLimit 0 + :jobTemplate + {:spec + {:template + {:spec + {:containers + [{:name "backup-app" + :image "domaindrivenarchitecture/meissa-cloud-backup" + :imagePullPolicy "IfNotPresent" + :command ["/entrypoint.sh"] + :env + [{:name "POSTGRES_USER_FILE", :value "/var/run/secrets/cloud-secrets/postgres-user"} + {:name "POSTGRES_DB_FILE", :value "/var/run/secrets/cloud-secrets/postgres-db"} + {:name "POSTGRES_PASSWORD_FILE", :value "/var/run/secrets/cloud-secrets/postgres-password"} + {:name "POSTGRES_HOST", :value "postgresql-service:5432"} + {:name "POSTGRES_SERVICE", :value "postgresql-service"} + {:name "POSTGRES_PORT", :value "5432"} + {:name "AWS_DEFAULT_REGION", :value "eu-central-1"} + {:name "AWS_ACCESS_KEY_ID_FILE", :value "/var/run/secrets/backup-secrets/aws-access-key-id"} + {:name "AWS_SECRET_ACCESS_KEY_FILE", :value "/var/run/secrets/backup-secrets/aws-secret-access-key"} + {:name "RESTIC_REPOSITORY", :valueFrom {:configMapKeyRef {:name "backup-config", :key "restic-repository"}}} + {:name "RESTIC_PASSWORD_FILE", :value "/var/run/secrets/backup-secrets/restic-password"}] + :volumeMounts + [{:name "cloud-data-volume", :mountPath "/var/backups"} + {:name "backup-secret-volume", :mountPath "/var/run/secrets/backup-secrets", :readOnly true} + {:name "cloud-secret-volume", :mountPath "/var/run/secrets/cloud-secrets", :readOnly true}]}] + :volumes + [{:name "cloud-data-volume", :persistentVolumeClaim {:claimName "cloud-pvc"}} + {:name "cloud-secret-volume", :secret {:secretName "cloud-secret"}} + {:name "backup-secret-volume", :secret {:secretName "backup-secret"}}] + :restartPolicy "OnFailure"}}}}}} (cut/generate-cron)))) diff --git a/src/test/cljc/dda/c4k_nextcloud/nextcloud_test.cljc b/src/test/cljc/dda/c4k_nextcloud/nextcloud_test.cljc index 47f2b9f..ce6c7cc 100644 --- a/src/test/cljc/dda/c4k_nextcloud/nextcloud_test.cljc +++ b/src/test/cljc/dda/c4k_nextcloud/nextcloud_test.cljc @@ -54,28 +54,33 @@ (cut/generate-persistent-volume {:nextcloud-data-volume-path "xx"})))) (deftest should-generate-deployment - (is (= {:containers - [{:image "domaindrivenarchitecture/meissa-cloud-app" - :name "cloud-app" - :imagePullPolicy "IfNotPresent" - :ports [{:containerPort 80}] - :env - [{:name "DB_USERNAME_FILE" - :value - "/var/run/secrets/postgres-secret/postgres-user"} - {:name "DB_PASSWORD_FILE" - :value - "/var/run/secrets/postgres-secret/postgres-password"} - {:name "FQDN", :value "xx"}] - :command ["/app/entrypoint.sh"] - :volumeMounts - [{:mountPath "/var/nextcloud", :name "cloud-data-volume"} - {:name "postgres-secret-volume" - :mountPath "/var/run/secrets/postgres-secret" - :readOnly true}]}] - :volumes - [{:name "cloud-data-volume" - :persistentVolumeClaim {:claimName "cloud-pvc"}} - {:name "postgres-secret-volume" - :secret {:secretName "postgres-secret"}}]} - (get-in (cut/generate-deployment {:fqdn "xx"}) [:spec :template :spec])))) + (is (= {:apiVersion "apps/v1" + :kind "Deployment" + :metadata {:name "cloud"} + :spec + {:selector {:matchLabels {:app "cloud"}} + :strategy {:type "Recreate"} + :template + {:metadata {:labels {:app "cloud"}} + :spec + {:containers + [{:image "domaindrivenarchitecture/meissa-cloud-app" + :name "cloud-app" + :imagePullPolicy "IfNotPresent" + :ports [{:containerPort 80}] + :env + [{:name "NEXTCLOUD_ADMIN_USER_FILE", :value "/var/run/secrets/cloud-secrets/nextcloud-admin-user"} + {:name "NEXTCLOUD_ADMIN_PASSWORD_FILE", :value "/var/run/secrets/cloud-secrets/nextcloud-admin-password"} + {:name "NEXTCLOUD_TRUSTED_DOMAINS", :value "xx"} + {:name "POSTGRES_USER_FILE", :value "/var/run/secrets/cloud-secrets/postgres-user"} + {:name "POSTGRES_PASSWORD_FILE", :value "/var/run/secrets/cloud-secrets/postgres-password"} + {:name "POSTGRES_DB_FILE", :value "/var/run/secrets/cloud-secrets/postgres-db"} + {:name "POSTGRES_HOST", :value "postgresql-service:5432"}] + :volumeMounts + [{:name "cloud-data-volume", :mountPath "/var/www/html"} + {:name "cloud-secret-volume", :mountPath "/var/run/secrets/cloud-secrets", :readOnly true}]}] + :volumes + [{:name "cloud-data-volume", :persistentVolumeClaim {:claimName "cloud-pvc"}} + {:name "cloud-secret-volume", :secret {:secretName "cloud-secret"}} + {:name "backup-secret-volume", :secret {:secretName "backup-secret"}}]}}}} + (cut/generate-deployment {:fqdn "xx"})))) From f140b989b2ec81da9e88f23616db1399d45e3db7 Mon Sep 17 00:00:00 2001 From: leo Date: Fri, 10 Sep 2021 16:55:39 +0200 Subject: [PATCH 12/12] updated ddadevops version --- infrastructure/docker-backup/build.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infrastructure/docker-backup/build.py b/infrastructure/docker-backup/build.py index 48d702a..8b2b41c 100644 --- a/infrastructure/docker-backup/build.py +++ b/infrastructure/docker-backup/build.py @@ -13,7 +13,7 @@ class MyBuild(DevopsDockerBuild): @init def initialize(project): - project.build_depends_on('ddadevops>=0.6.1') + project.build_depends_on('ddadevops>=0.12.4') stage = 'notused' dockerhub_user = environ.get('DOCKERHUB_USER') if not dockerhub_user: