diff --git a/README.md b/README.md index e32ed90..9af9e4c 100644 --- a/README.md +++ b/README.md @@ -1,41 +1,5 @@ -# convention 4 kubernetes: c4k-nextcloud +# meissa-cloud -[![Clojars Project](https://img.shields.io/clojars/v/org.domaindrivenarchitecture/c4k-nextcloud.svg)](https://clojars.org/org.domaindrivenarchitecture/c4k-nextcloud) [![pipeline status](https://gitlab.com/domaindrivenarchitecture/c4k-nextcloud/badges/master/pipeline.svg)](https://gitlab.com/domaindrivenarchitecture/c4k-nextcloud/-/commits/master) +# backup manuell triggern -[DeltaChat chat over e-mail](mailto:buero@meissa-gmbh.de?subject=community-chat) | [team@social.meissa-gmbh.de team@social.meissa-gmbh.de](https://social.meissa-gmbh.de/@team) | [Website & Blog](https://domaindrivenarchitecture.org) - -## Purpose - -c4k-nextcloud provides a k8s deployment for nextcloud containing: -* adjusted nextcloud docker image -* nextcloud -* ingress having a letsencrypt managed certificate -* postgres database - -The package aims to a low load sceanrio. - -## Status - -This is under development. - -## Manual restore - -1) Scale Nextcloud deployment down: -kubectl scale deployment nextcloud --replicas=0 - -2) apply backup and restore pod: -kubectl apply -f src/main/resources/backup/backup-restore.yaml - -3) exec into pod and execute restore pod -kubectl exec -it backup-restore -- /usr/local/bin/restore.sh - -4) Scale Nextcloud deployment up: -kubectl scale deployment nextcloud --replicas=1 - -5) Update index of Nextcloud: -Nextcloud > Settings > System > Advanced > Indexing -## License - -Copyright © 2021 meissa GmbH -Licensed under the [Apache License, Version 2.0](LICENSE) (the "License") -Pls. find licenses of our subcomponents [here](doc/SUBCOMPONENT_LICENSE) \ No newline at end of file +# restore manuell triggern \ No newline at end of file diff --git a/infrastructure/docker-backup/build.py b/infrastructure/docker-backup/build.py index 68ac1ae..48d702a 100644 --- a/infrastructure/docker-backup/build.py +++ b/infrastructure/docker-backup/build.py @@ -3,28 +3,26 @@ from pybuilder.core import task, init from ddadevops import * import logging -name = 'c4k-cloud-backup' +name = 'meissa-cloud-backup' MODULE = 'docker' PROJECT_ROOT_PATH = '../..' + class MyBuild(DevopsDockerBuild): pass @init def initialize(project): - project.build_depends_on('ddadevops>=0.12.4') - stage = 'prod' + project.build_depends_on('ddadevops>=0.6.1') + stage = 'notused' dockerhub_user = environ.get('DOCKERHUB_USER') if not dockerhub_user: dockerhub_user = gopass_field_from_path('meissa/web/docker.com', 'login') dockerhub_password = environ.get('DOCKERHUB_PASSWORD') if not dockerhub_password: dockerhub_password = gopass_password_from_path('meissa/web/docker.com') - tag = environ.get('CI_COMMIT_TAG') - if not tag: - tag = get_tag_from_latest_commit() config = create_devops_docker_build_config( - stage, PROJECT_ROOT_PATH, MODULE, dockerhub_user, dockerhub_password, docker_publish_tag=tag) + stage, PROJECT_ROOT_PATH, MODULE, dockerhub_user, dockerhub_password) build = MyBuild(project, config) build.initialize_build_dir() @@ -40,12 +38,12 @@ def drun(project): build.drun() @task -def publish(project): +def test(project): build = get_devops_build(project) - build.dockerhub_login() - build.dockerhub_publish() + build.test() @task -def test(project): +def publish(project): build = get_devops_build(project) - build.test() + build.dockerhub_login() + build.dockerhub_publish() diff --git a/infrastructure/docker-backup/image/resources/backup.sh b/infrastructure/docker-backup/image/resources/backup.sh index d01affd..c28cf98 100755 --- a/infrastructure/docker-backup/image/resources/backup.sh +++ b/infrastructure/docker-backup/image/resources/backup.sh @@ -3,6 +3,9 @@ set -o pipefail function main() { + + start-maintenance.sh + file_env AWS_ACCESS_KEY_ID file_env AWS_SECRET_ACCESS_KEY file_env POSTGRES_DB @@ -10,13 +13,15 @@ function main() { file_env POSTGRES_USER file_env RESTIC_DAYS_TO_KEEP 14 - backup-roles "" + backup-roles 'oc_' backup-db-dump - backup-fs-from-directory '/var/backups/' 'data/' + backup-directory '/var/backups/' + + end-maintenance.sh } source /usr/local/lib/functions.sh -source /usr/local/lib/file-functions.sh source /usr/local/lib/pg-functions.sh +source /usr/local/lib/file-functions.sh main diff --git a/infrastructure/docker-backup/image/resources/end-maintenance.sh b/infrastructure/docker-backup/image/resources/end-maintenance.sh new file mode 100644 index 0000000..0013509 --- /dev/null +++ b/infrastructure/docker-backup/image/resources/end-maintenance.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +if test -f "/var/backups/config/config.orig"; then + + rm /var/backups/config/config.php + mv /var/backups/config/config.orig /var/backups/config/config.php + chown www-data:root /var/backups/config/config.php + touch /var/backups/config/config.php + +fi \ No newline at end of file diff --git a/infrastructure/docker-backup/image/resources/entrypoint-start-and-wait.sh b/infrastructure/docker-backup/image/resources/entrypoint-start-and-wait.sh index 67a8f3b..60075b5 100644 --- a/infrastructure/docker-backup/image/resources/entrypoint-start-and-wait.sh +++ b/infrastructure/docker-backup/image/resources/entrypoint-start-and-wait.sh @@ -4,7 +4,7 @@ function main() { file_env POSTGRES_DB file_env POSTGRES_PASSWORD file_env POSTGRES_USER - + create-pg-pass while true; do diff --git a/infrastructure/docker-backup/image/resources/entrypoint.sh b/infrastructure/docker-backup/image/resources/entrypoint.sh index 924f24c..480c971 100755 --- a/infrastructure/docker-backup/image/resources/entrypoint.sh +++ b/infrastructure/docker-backup/image/resources/entrypoint.sh @@ -1,13 +1,13 @@ #!/bin/bash function main() { - file_env POSTGRES_DB + file_env POSTGRES_DB file_env POSTGRES_PASSWORD file_env POSTGRES_USER - + create-pg-pass - /usr/local/bin/backup.sh + /usr/local/bin/backup.sh } source /usr/local/lib/functions.sh diff --git a/infrastructure/docker-backup/image/resources/init.sh b/infrastructure/docker-backup/image/resources/init.sh index 5693dcb..5767e69 100755 --- a/infrastructure/docker-backup/image/resources/init.sh +++ b/infrastructure/docker-backup/image/resources/init.sh @@ -10,6 +10,6 @@ function main() { } source /usr/local/lib/functions.sh -source /usr/local/lib/file-functions.sh source /usr/local/lib/pg-functions.sh +source /usr/local/lib/file-functions.sh main diff --git a/infrastructure/docker-backup/image/resources/install.sh b/infrastructure/docker-backup/image/resources/install.sh index 1a8cbd7..7c58fce 100755 --- a/infrastructure/docker-backup/image/resources/install.sh +++ b/infrastructure/docker-backup/image/resources/install.sh @@ -8,4 +8,5 @@ install -m 0700 /tmp/entrypoint-start-and-wait.sh / install -m 0700 /tmp/init.sh /usr/local/bin/ install -m 0700 /tmp/backup.sh /usr/local/bin/ install -m 0700 /tmp/restore.sh /usr/local/bin/ -install -m 0700 /tmp/restic-snapshots.sh /usr/local/bin/ +install -m 0700 /tmp/start-maintenance.sh /usr/local/bin/ +install -m 0700 /tmp/end-maintenance.sh /usr/local/bin/ diff --git a/infrastructure/docker-backup/image/resources/restic-snapshots.sh b/infrastructure/docker-backup/image/resources/restic-snapshots.sh deleted file mode 100755 index a428fe2..0000000 --- a/infrastructure/docker-backup/image/resources/restic-snapshots.sh +++ /dev/null @@ -1,17 +0,0 @@ -#!/bin/bash - -set -o pipefail - -function main() { - file_env AWS_ACCESS_KEY_ID - file_env AWS_SECRET_ACCESS_KEY - - restic -r ${RESTIC_REPOSITORY}/pg-role snapshots - restic -r ${RESTIC_REPOSITORY}/pg-database snapshots - restic -r ${RESTIC_REPOSITORY}/files snapshots -} - -source /usr/local/lib/functions.sh -source /usr/local/lib/file-functions.sh - -main diff --git a/infrastructure/docker-backup/image/resources/restore.sh b/infrastructure/docker-backup/image/resources/restore.sh index ba1e091..1ebef16 100755 --- a/infrastructure/docker-backup/image/resources/restore.sh +++ b/infrastructure/docker-backup/image/resources/restore.sh @@ -4,6 +4,8 @@ set -Eeo pipefail function main() { + start-maintenance.sh + file_env AWS_ACCESS_KEY_ID file_env AWS_SECRET_ACCESS_KEY @@ -11,26 +13,16 @@ function main() { file_env POSTGRES_PASSWORD file_env POSTGRES_USER - # Im Nextcloud pod: /opt/atlassian-cloud-software-standalone/bin/stop-cloud.sh - - # Restore latest snapshot into /var/backups/restic-restore - rm -rf /var/backups/restic-restore - restore-directory '/var/backups/restic-restore' - - # Restore data dir backup - rm -rf /var/backups/data/* - cp -a /var/backups/restic-restore/data/* /var/backups/data - - # Restore db drop-create-db + restore-roles restore-db + restore-directory '/var/backups/' - # /opt/atlassian-cloud-software-standalone/bin/start-cloud.sh } source /usr/local/lib/functions.sh -source /usr/local/lib/file-functions.sh source /usr/local/lib/pg-functions.sh +source /usr/local/lib/file-functions.sh main diff --git a/infrastructure/docker-backup/image/resources/start-maintenance.sh b/infrastructure/docker-backup/image/resources/start-maintenance.sh new file mode 100644 index 0000000..474ede1 --- /dev/null +++ b/infrastructure/docker-backup/image/resources/start-maintenance.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +if [ ! -f "/var/backups/config/config.orig" ]; then + + rm -f /var/backups/config/config.orig + cp /var/backups/config/config.php /var/backups/config/config.orig + sed -i "s/);/ \'maintenance\' => true,\n);/g" /var/backups/config/config.php + chown www-data:root /var/backups/config/config.php + touch /var/backups/config/config.php + +fi diff --git a/infrastructure/docker-backup/test/Dockerfile b/infrastructure/docker-backup/test/Dockerfile index a18ebee..eb92122 100644 --- a/infrastructure/docker-backup/test/Dockerfile +++ b/infrastructure/docker-backup/test/Dockerfile @@ -1,7 +1,9 @@ -FROM c4k-cloud-backup +FROM meissa-cloud-backup -RUN curl -L -o /tmp/serverspec.jar \ - https://github.com/DomainDrivenArchitecture/dda-serverspec-crate/releases/download/2.0.0/dda-serverspec-standalone.jar +RUN apt update +RUN apt -yqq --no-install-recommends --yes install curl default-jre-headless + +RUN curl -L -o /tmp/serverspec.jar https://github.com/DomainDrivenArchitecture/dda-serverspec-crate/releases/download/2.0.0/dda-serverspec-standalone.jar COPY serverspec.edn /tmp/serverspec.edn diff --git a/infrastructure/docker-backup/test/serverspec.edn b/infrastructure/docker-backup/test/serverspec.edn index 09623c7..bc4277f 100644 --- a/infrastructure/docker-backup/test/serverspec.edn +++ b/infrastructure/docker-backup/test/serverspec.edn @@ -1,6 +1,7 @@ {:file [{:path "/usr/local/bin/init.sh" :mod "700"} {:path "/usr/local/bin/backup.sh" :mod "700"} {:path "/usr/local/bin/restore.sh" :mod "700"} - {:path "/usr/local/bin/restic-snapshots.sh" :mod "700"} + {:path "/usr/local/bin/start-maintenance.sh" :mod "700"} + {:path "/usr/local/bin/end-maintenance.sh" :mod "700"} {:path "/entrypoint.sh" :mod "700"} {:path "/entrypoint-start-and-wait.sh" :mod "700"}]} diff --git a/infrastructure/docker-cloud/image/Dockerfile b/infrastructure/docker-cloud/image/Dockerfile deleted file mode 100644 index 9551640..0000000 --- a/infrastructure/docker-cloud/image/Dockerfile +++ /dev/null @@ -1,10 +0,0 @@ -FROM ubuntu:20.04 - -ENV CLOUD_HOME="/var/cloud" \ - DOWNLOAD_URL="https://product-downloads.atlassian.com/software/nextcloud/downloads" \ - CLOUD_RELEASE="8.8.0" - # CLOUD_RELEASE="20.1.0"??? -ADD resources /tmp/resources -RUN /tmp/resources/install.sh - -USER 901:901 diff --git a/infrastructure/docker-cloud/image/resources/dbconfig.xml b/infrastructure/docker-cloud/image/resources/dbconfig.xml deleted file mode 100644 index 7ff0887..0000000 --- a/infrastructure/docker-cloud/image/resources/dbconfig.xml +++ /dev/null @@ -1,25 +0,0 @@ - - - - defaultDS - default - postgres72 - public - - jdbc:postgresql://postgresql-service:5432/cloud - org.postgresql.Driver - $CLOUD_DB_USER - $CLOUD_DB_PASSWORD - 20 - 20 - 30000 - select 1 - 60000 - 300000 - 20 - true - 300 - false - true - - \ No newline at end of file diff --git a/infrastructure/docker-cloud/image/resources/entrypoint.sh b/infrastructure/docker-cloud/image/resources/entrypoint.sh deleted file mode 100644 index 115e905..0000000 --- a/infrastructure/docker-cloud/image/resources/entrypoint.sh +++ /dev/null @@ -1,44 +0,0 @@ -#!/bin/bash - -# usage: file_env VAR [DEFAULT] -# ie: file_env 'XYZ_DB_PASSWORD' 'example' -# (will allow for "$XYZ_DB_PASSWORD_FILE" to fill in the value of -# "$XYZ_DB_PASSWORD" from a file, especially for Docker's secrets feature) -function file_env() { - local var="$1" - local fileVar="${var}_FILE" - local def="${2:-}" - local varValue=$(env | grep -E "^${var}=" | sed -E -e "s/^${var}=//") - local fileVarValue=$(env | grep -E "^${fileVar}=" | sed -E -e "s/^${fileVar}=//") - if [ -n "${varValue}" ] && [ -n "${fileVarValue}" ]; then - echo >&2 "error: both $var and $fileVar are set (but are exclusive)" - exit 1 - fi - if [ -n "${varValue}" ]; then - export "$var"="${varValue}" - elif [ -n "${fileVarValue}" ]; then - export "$var"="$(cat "${fileVarValue}")" - elif [ -n "${def}" ]; then - export "$var"="$def" - fi - unset "$fileVar" -} - -function main() { - file_env "FQDN" - file_env "DB_USERNAME" - file_env "DB_PASSWORD" - - xmlstarlet ed -L -u "/Server/Service/Connector[@proxyName='{subdomain}.{domain}.com']/@proxyName" \ - -v "$FQDN" /opt/atlassian-cloud-software-standalone/conf/server.xml - xmlstarlet ed -L -u "/cloud-database-config/jdbc-datasource/username" \ - -v "$DB_USERNAME" /app/dbconfig.xml - xmlstarlet ed -L -u "/cloud-database-config/jdbc-datasource/password" \ - -v "$DB_PASSWORD" /app/dbconfig.xml - - install -ocloud -gcloud -m660 /app/dbconfig.xml /var/cloud/dbconfig.xml - /opt/atlassian-cloud-software-standalone/bin/setenv.sh run - /opt/atlassian-cloud-software-standalone/bin/start-cloud.sh run -} - -main diff --git a/infrastructure/docker-cloud/image/resources/install.sh b/infrastructure/docker-cloud/image/resources/install.sh deleted file mode 100755 index 4927a6b..0000000 --- a/infrastructure/docker-cloud/image/resources/install.sh +++ /dev/null @@ -1,30 +0,0 @@ -#!/bin/bash - -function main() { - - upgradeSystem - apt-get -qqy install curl openjdk-11-jre-headless xmlstarlet > /dev/null - - adduser --system --disabled-login --home ${CLOUD_HOME} --uid 901 --group cloud - - mkdir -p /app - - cd /tmp - curl --location ${DOWNLOAD_URL}/atlassian-cloud-software-${CLOUD_RELEASE}.tar.gz \ - -o atlassian-cloud-software-${CLOUD_RELEASE}.tar.gz - tar -xvf atlassian-cloud-software-${CLOUD_RELEASE}.tar.gz -C /tmp/ - mv /tmp/atlassian-cloud-software-${CLOUD_RELEASE}-standalone \ - /opt/atlassian-cloud-software-standalone - chown -R cloud:cloud /opt/atlassian-cloud-software-standalone - - install -ocloud -gcloud -m744 /tmp/resources/entrypoint.sh /app/entrypoint.sh - install -ocloud -gcloud -m744 /tmp/resources/setenv.sh /opt/atlassian-cloud-software-standalone/bin/setenv.sh - install -ocloud -gcloud -m660 /tmp/resources/dbconfig.xml /app/dbconfig.xml - install -ocloud -gcloud -m660 /tmp/resources/server.xml /opt/atlassian-cloud-software-standalone/conf/server.xml - install -ocloud -gcloud -m660 /tmp/resources/logging.properties /opt/atlassian-cloud-software-standalone/conf/logging.properties - - cleanupDocker -} - -source /tmp/resources/install_functions.sh -main \ No newline at end of file diff --git a/infrastructure/docker-cloud/image/resources/logging.properties b/infrastructure/docker-cloud/image/resources/logging.properties deleted file mode 100644 index 267d805..0000000 --- a/infrastructure/docker-cloud/image/resources/logging.properties +++ /dev/null @@ -1,69 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -handlers = java.util.logging.ConsoleHandler - -.handlers = java.util.logging.ConsoleHandler - -############################################################ -# Handler specific properties. -# Describes specific configuration info for Handlers. -############################################################ - -java.util.logging.ConsoleHandler.level = FINE -java.util.logging.ConsoleHandler.formatter = org.apache.juli.OneLineFormatter -java.util.logging.ConsoleHandler.encoding = UTF-8 - - -############################################################ -# Facility specific properties. -# Provides extra control for each logger. -############################################################ - -org.apache.catalina.core.ContainerBase.[Catalina].[localhost].level = INFO -org.apache.catalina.core.ContainerBase.[Catalina].[localhost].handlers = 2localhost.org.apache.juli.AsyncFileHandler - -org.apache.catalina.core.ContainerBase.[Catalina].[localhost].[/manager].level = INFO -org.apache.catalina.core.ContainerBase.[Catalina].[localhost].[/manager].handlers = 3manager.org.apache.juli.AsyncFileHandler - -org.apache.catalina.core.ContainerBase.[Catalina].[localhost].[/host-manager].level = INFO -org.apache.catalina.core.ContainerBase.[Catalina].[localhost].[/host-manager].handlers = 4host-manager.org.apache.juli.AsyncFileHandler - -# For example, set the org.apache.catalina.util.LifecycleBase logger to log -# each component that extends LifecycleBase changing state: -#org.apache.catalina.util.LifecycleBase.level = FINE - -# To see debug messages in TldLocationsCache, uncomment the following line: -#org.apache.jasper.compiler.TldLocationsCache.level = FINE - -# To see debug messages for HTTP/2 handling, uncomment the following line: -#org.apache.coyote.http2.level = FINE - -# To see debug messages for WebSocket handling, uncomment the following line: -#org.apache.tomcat.websocket.level = FINE -# Suppress clearReferencesThreads and clearThreadLocalMap notifications on shutdown -org.apache.catalina.loader.WebappClassLoader.level = OFF - -# Suppress some useless REST messages. See JRADEV-12212. -com.sun.jersey.server.impl.application.WebApplicationImpl.level = WARNING -com.atlassian.plugins.rest.doclet.level = WARNING -com.sun.jersey.api.wadl.config.WadlGeneratorLoader.level = WARNING - -# Suppress log spam about the request body - https://cloud.atlassian.com/browse/JRASERVER-30406 -com.sun.jersey.spi.container.servlet.WebComponent.level = SEVERE -com.sun.jersey.spi.container.servlet.WebComponent.filterFormParameters.level = SEVERE - -# Suppress resources cache full warnings -org.apache.catalina.webresources.Cache.level = OFF \ No newline at end of file diff --git a/infrastructure/docker-cloud/image/resources/server.xml b/infrastructure/docker-cloud/image/resources/server.xml deleted file mode 100644 index 6e057fd..0000000 --- a/infrastructure/docker-cloud/image/resources/server.xml +++ /dev/null @@ -1,114 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/infrastructure/docker-cloud/image/resources/setenv.sh b/infrastructure/docker-cloud/image/resources/setenv.sh deleted file mode 100644 index d587952..0000000 --- a/infrastructure/docker-cloud/image/resources/setenv.sh +++ /dev/null @@ -1,121 +0,0 @@ -# -# If the limit of files that Nextcloud can open is too low, it will be set to this value. -# -MIN_NOFILES_LIMIT=16384 - -# -# One way to set the Nextcloud HOME path is here via this variable. Simply uncomment it and set a valid path like /cloud/home. You can of course set it outside in the command terminal. That will also work. -# -CLOUD_HOME="/var/cloud" - -# -# Occasionally Atlassian Support may recommend that you set some specific JVM arguments. You can use this variable below to do that. -# -JVM_SUPPORT_RECOMMENDED_ARGS="" - -# -# You can use variable below to modify garbage collector settings. -# For Java 8 we recommend default settings -# For Java 11 and relatively small heaps we recommend: -XX:+UseParallelGC -# For Java 11 and larger heaps we recommend: -XX:+UseG1GC -XX:+ExplicitGCInvokesConcurrent -# -JVM_GC_ARGS="-XX:+UseG1GC -XX:+ExplicitGCInvokesConcurrent" - -# -# The following 2 settings control the minimum and maximum given to the Nextcloud Java virtual machine. In larger Nextcloud instances, the maximum amount will need to be increased. -# -JVM_MINIMUM_MEMORY="4096m" -JVM_MAXIMUM_MEMORY="4096m" - -# -# The following setting configures the size of JVM code cache. A high value of reserved size allows Nextcloud to work with more installed apps. -# -JVM_CODE_CACHE_ARGS='-XX:InitialCodeCacheSize=32m -XX:ReservedCodeCacheSize=512m' - -# -# The following are the required arguments for Nextcloud. -# -JVM_REQUIRED_ARGS='-Djava.awt.headless=true -Datlassian.standalone=CLOUD -Dorg.apache.jasper.runtime.BodyContentImpl.LIMIT_BUFFER=true -Dmail.mime.decodeparameters=true -Dorg.dom4j.factory=com.atlassian.core.xml.InterningDocumentFactory' - -# Uncomment this setting if you want to import data without notifications -# -#DISABLE_NOTIFICATIONS=" -Datlassian.mail.senddisabled=true -Datlassian.mail.fetchdisabled=true -Datlassian.mail.popdisabled=true" - - -#----------------------------------------------------------------------------------- -# -# In general don't make changes below here -# -#----------------------------------------------------------------------------------- - -#----------------------------------------------------------------------------------- -# Prevents the JVM from suppressing stack traces if a given type of exception -# occurs frequently, which could make it harder for support to diagnose a problem. -#----------------------------------------------------------------------------------- -JVM_EXTRA_ARGS="-XX:-OmitStackTraceInFastThrow -Djava.locale.providers=COMPAT" - -CURRENT_NOFILES_LIMIT=$( ulimit -Hn ) -ulimit -Sn $CURRENT_NOFILES_LIMIT -ulimit -n $(( CURRENT_NOFILES_LIMIT > MIN_NOFILES_LIMIT ? CURRENT_NOFILES_LIMIT : MIN_NOFILES_LIMIT )) - -PRGDIR=`dirname "$0"` -cat "${PRGDIR}"/cloudbanner.txt - -CLOUD_HOME_MINUSD="" -if [ "$CLOUD_HOME" != "" ]; then - echo $CLOUD_HOME | grep -q " " - if [ $? -eq 0 ]; then - echo "" - echo "--------------------------------------------------------------------------------------------------------------------" - echo " WARNING : You cannot have a CLOUD_HOME environment variable set with spaces in it. This variable is being ignored" - echo "--------------------------------------------------------------------------------------------------------------------" - else - CLOUD_HOME_MINUSD=-Dcloud.home=$CLOUD_HOME - fi -fi - -JAVA_OPTS="-Xms${JVM_MINIMUM_MEMORY} -Xmx${JVM_MAXIMUM_MEMORY} ${JVM_CODE_CACHE_ARGS} ${JAVA_OPTS} ${JVM_REQUIRED_ARGS} ${DISABLE_NOTIFICATIONS} ${JVM_SUPPORT_RECOMMENDED_ARGS} ${JVM_EXTRA_ARGS} ${CLOUD_HOME_MINUSD} ${START_CLOUD_JAVA_OPTS}" - -export JAVA_OPTS - -# DO NOT remove the following line -# !INSTALLER SET JAVA_HOME - -echo "" -echo "If you encounter issues starting or stopping Nextcloud, please see the Troubleshooting guide at https://docs.atlassian.com/cloud/jadm-docs-088/Troubleshooting+installation" -echo "" -if [ "$CLOUD_HOME_MINUSD" != "" ]; then - echo "Using CLOUD_HOME: $CLOUD_HOME" -fi - -# set the location of the pid file -if [ -z "$CATALINA_PID" ] ; then - if [ -n "$CATALINA_BASE" ] ; then - CATALINA_PID="$CATALINA_BASE"/work/catalina.pid - elif [ -n "$CATALINA_HOME" ] ; then - CATALINA_PID="$CATALINA_HOME"/work/catalina.pid - fi -fi -export CATALINA_PID - -if [ -z "$CATALINA_BASE" ]; then - if [ -z "$CATALINA_HOME" ]; then - LOGBASE=$PRGDIR - LOGTAIL=.. - else - LOGBASE=$CATALINA_HOME - LOGTAIL=. - fi -else - LOGBASE=$CATALINA_BASE - LOGTAIL=. -fi - -PUSHED_DIR=`pwd` -cd $LOGBASE -cd $LOGTAIL -LOGBASEABS=`pwd` -cd $PUSHED_DIR - -echo "" -echo "Server startup logs are located in $LOGBASEABS/logs/catalina.out" diff --git a/infrastructure/docker-cloud/image/resources/user.sh b/infrastructure/docker-cloud/image/resources/user.sh deleted file mode 100644 index 6ae8a6d..0000000 --- a/infrastructure/docker-cloud/image/resources/user.sh +++ /dev/null @@ -1,5 +0,0 @@ -# START INSTALLER MAGIC ! DO NOT EDIT ! -CLOUD_USER="cloud" ## -# END INSTALLER MAGIC ! DO NOT EDIT ! - -export CLOUD_USER diff --git a/infrastructure/docker-cloud/test/Dockerfile b/infrastructure/docker-cloud/test/Dockerfile deleted file mode 100644 index 693ed79..0000000 --- a/infrastructure/docker-cloud/test/Dockerfile +++ /dev/null @@ -1,8 +0,0 @@ -FROM c4k-cloud - -RUN curl -L -o /tmp/serverspec.jar \ - https://github.com/DomainDrivenArchitecture/dda-serverspec-crate/releases/download/1.3.4/dda-serverspec-standalone.jar - -COPY serverspec.edn /tmp/serverspec.edn - -RUN java -jar /tmp/serverspec.jar /tmp/serverspec.edn -v diff --git a/infrastructure/docker-cloud/test/serverspec.edn b/infrastructure/docker-cloud/test/serverspec.edn deleted file mode 100644 index a462c1a..0000000 --- a/infrastructure/docker-cloud/test/serverspec.edn +++ /dev/null @@ -1,3 +0,0 @@ -{:file [{:path "/app/entrypoint.sh"} - {:path "/var/cloud"} - {:path "/opt/atlassian-cloud-software-standalone"}]} diff --git a/infrastructure/docker-cloud/build.py b/infrastructure/docker-nextcloud/build.py similarity index 83% rename from infrastructure/docker-cloud/build.py rename to infrastructure/docker-nextcloud/build.py index c192b2a..323f4bc 100644 --- a/infrastructure/docker-cloud/build.py +++ b/infrastructure/docker-nextcloud/build.py @@ -3,28 +3,26 @@ from pybuilder.core import task, init from ddadevops import * import logging -name = 'c4k-cloud' +name = 'meissa-cloud-app' MODULE = 'docker' PROJECT_ROOT_PATH = '../..' + class MyBuild(DevopsDockerBuild): pass @init def initialize(project): - project.build_depends_on('ddadevops>=0.12.4') - stage = 'prod' + project.build_depends_on('ddadevops>=0.6.1') + stage = 'notused' dockerhub_user = environ.get('DOCKERHUB_USER') if not dockerhub_user: dockerhub_user = gopass_field_from_path('meissa/web/docker.com', 'login') dockerhub_password = environ.get('DOCKERHUB_PASSWORD') if not dockerhub_password: dockerhub_password = gopass_password_from_path('meissa/web/docker.com') - tag = environ.get('CI_COMMIT_TAG') - if not tag: - tag = get_tag_from_latest_commit() config = create_devops_docker_build_config( - stage, PROJECT_ROOT_PATH, MODULE, dockerhub_user, dockerhub_password, docker_publish_tag=tag) + stage, PROJECT_ROOT_PATH, MODULE, dockerhub_user, dockerhub_password) build = MyBuild(project, config) build.initialize_build_dir() @@ -40,12 +38,12 @@ def drun(project): build.drun() @task -def publish(project): +def test(project): build = get_devops_build(project) - build.dockerhub_login() - build.dockerhub_publish() + build.test() @task -def test(project): +def publish(project): build = get_devops_build(project) - build.test() + build.dockerhub_login() + build.dockerhub_publish() diff --git a/infrastructure/docker-nextcloud/image/Dockerfile b/infrastructure/docker-nextcloud/image/Dockerfile new file mode 100644 index 0000000..8b3d9e9 --- /dev/null +++ b/infrastructure/docker-nextcloud/image/Dockerfile @@ -0,0 +1,8 @@ +FROM nextcloud:19 + +# Prepare Entrypoint Script +ADD resources /tmp +RUN /tmp/install.sh + +ENTRYPOINT ["/entrypoint.sh"] +CMD ["apache2-foreground"] diff --git a/infrastructure/docker-nextcloud/image/resources/entrypoint.sh b/infrastructure/docker-nextcloud/image/resources/entrypoint.sh new file mode 100644 index 0000000..aedef7d --- /dev/null +++ b/infrastructure/docker-nextcloud/image/resources/entrypoint.sh @@ -0,0 +1,191 @@ +#!/bin/sh +set -eu + +# version_greater A B returns whether A > B +version_greater() { + [ "$(printf '%s\n' "$@" | sort -t '.' -n -k1,1 -k2,2 -k3,3 -k4,4 | head -n 1)" != "$1" ] +} + +# return true if specified directory is empty +directory_empty() { + [ -z "$(ls -A "$1/")" ] +} + +run_as() { + if [ "$(id -u)" = 0 ]; then + su -p www-data -s /bin/sh -c "$1" + else + sh -c "$1" + fi +} + +# usage: file_env VAR [DEFAULT] +# ie: file_env 'XYZ_DB_PASSWORD' 'example' +# (will allow for "$XYZ_DB_PASSWORD_FILE" to fill in the value of +# "$XYZ_DB_PASSWORD" from a file, especially for Docker's secrets feature) +file_env() { + local var="$1" + local fileVar="${var}_FILE" + local def="${2:-}" + local varValue=$(env | grep -E "^${var}=" | sed -E -e "s/^${var}=//") + local fileVarValue=$(env | grep -E "^${fileVar}=" | sed -E -e "s/^${fileVar}=//") + if [ -n "${varValue}" ] && [ -n "${fileVarValue}" ]; then + echo >&2 "error: both $var and $fileVar are set (but are exclusive)" + exit 1 + fi + if [ -n "${varValue}" ]; then + export "$var"="${varValue}" + elif [ -n "${fileVarValue}" ]; then + export "$var"="$(cat "${fileVarValue}")" + elif [ -n "${def}" ]; then + export "$var"="$def" + fi + unset "$fileVar" +} + +if expr "$1" : "apache" 1>/dev/null; then + if [ -n "${APACHE_DISABLE_REWRITE_IP+x}" ]; then + a2disconf remoteip + fi +fi + +if expr "$1" : "apache" 1>/dev/null || [ "$1" = "php-fpm" ] || [ "${NEXTCLOUD_UPDATE:-0}" -eq 1 ]; then + if [ -n "${REDIS_HOST+x}" ]; then + + echo "Configuring Redis as session handler" + { + echo 'session.save_handler = redis' + # check if redis host is an unix socket path + if [ "$(echo "$REDIS_HOST" | cut -c1-1)" = "/" ]; then + if [ -n "${REDIS_HOST_PASSWORD+x}" ]; then + echo "session.save_path = \"unix://${REDIS_HOST}?auth=${REDIS_HOST_PASSWORD}\"" + else + echo "session.save_path = \"unix://${REDIS_HOST}\"" + fi + # check if redis password has been set + elif [ -n "${REDIS_HOST_PASSWORD+x}" ]; then + echo "session.save_path = \"tcp://${REDIS_HOST}:${REDIS_HOST_PORT:=6379}?auth=${REDIS_HOST_PASSWORD}\"" + else + echo "session.save_path = \"tcp://${REDIS_HOST}:${REDIS_HOST_PORT:=6379}\"" + fi + } > /usr/local/etc/php/conf.d/redis-session.ini + fi + + installed_version="0.0.0.0" + if [ -f /var/www/html/version.php ]; then + # shellcheck disable=SC2016 + installed_version="$(php -r 'require "/var/www/html/version.php"; echo implode(".", $OC_Version);')" + fi + # shellcheck disable=SC2016 + image_version="$(php -r 'require "/usr/src/nextcloud/version.php"; echo implode(".", $OC_Version);')" + + if version_greater "$installed_version" "$image_version"; then + echo "Can't start Nextcloud because the version of the data ($installed_version) is higher than the docker image version ($image_version) and downgrading is not supported. Are you sure you have pulled the newest image version?" + exit 1 + fi + + if version_greater "$image_version" "$installed_version"; then + echo "Initializing nextcloud $image_version (image) over $installed_version (installed) ..." + if [ "$installed_version" != "0.0.0.0" ]; then + echo "Upgrading nextcloud from $installed_version ..." + run_as 'php /var/www/html/occ app:list' | sed -n "/Enabled:/,/Disabled:/p" > /tmp/list_before + fi + if [ "$(id -u)" = 0 ]; then + rsync_options="-rlDog --chown www-data:root" + else + rsync_options="-rlD" + fi + rsync $rsync_options --delete --exclude-from=/upgrade.exclude /usr/src/nextcloud/ /var/www/html/ + + for dir in config data custom_apps themes; do + if [ ! -d "/var/www/html/$dir" ] || directory_empty "/var/www/html/$dir"; then + rsync $rsync_options --include "/$dir/" --exclude '/*' /usr/src/nextcloud/ /var/www/html/ + fi + done + rsync $rsync_options --include '/version.php' --exclude '/*' /usr/src/nextcloud/ /var/www/html/ + echo "Initializing finished" + + #install + if [ "$installed_version" = "0.0.0.0" ]; then + echo "New nextcloud instance" + + file_env NEXTCLOUD_ADMIN_PASSWORD + file_env NEXTCLOUD_ADMIN_USER + + if [ -n "${NEXTCLOUD_ADMIN_USER+x}" ] && [ -n "${NEXTCLOUD_ADMIN_PASSWORD+x}" ]; then + # shellcheck disable=SC2016 + install_options='-n --admin-user "$NEXTCLOUD_ADMIN_USER" --admin-pass "$NEXTCLOUD_ADMIN_PASSWORD"' + if [ -n "${NEXTCLOUD_DATA_DIR+x}" ]; then + # shellcheck disable=SC2016 + install_options=$install_options' --data-dir "$NEXTCLOUD_DATA_DIR"' + fi + + file_env MYSQL_DATABASE + file_env MYSQL_PASSWORD + file_env MYSQL_USER + file_env POSTGRES_DB + file_env POSTGRES_PASSWORD + file_env POSTGRES_USER + + install=false + if [ -n "${SQLITE_DATABASE+x}" ]; then + echo "Installing with SQLite database" + # shellcheck disable=SC2016 + install_options=$install_options' --database-name "$SQLITE_DATABASE"' + install=true + elif [ -n "${MYSQL_DATABASE+x}" ] && [ -n "${MYSQL_USER+x}" ] && [ -n "${MYSQL_PASSWORD+x}" ] && [ -n "${MYSQL_HOST+x}" ]; then + echo "Installing with MySQL database" + # shellcheck disable=SC2016 + install_options=$install_options' --database mysql --database-name "$MYSQL_DATABASE" --database-user "$MYSQL_USER" --database-pass "$MYSQL_PASSWORD" --database-host "$MYSQL_HOST"' + install=true + elif [ -n "${POSTGRES_DB+x}" ] && [ -n "${POSTGRES_USER+x}" ] && [ -n "${POSTGRES_PASSWORD+x}" ] && [ -n "${POSTGRES_HOST+x}" ]; then + echo "Installing with PostgreSQL database" + # shellcheck disable=SC2016 + install_options=$install_options' --database pgsql --database-name "$POSTGRES_DB" --database-user "$POSTGRES_USER" --database-pass "$POSTGRES_PASSWORD" --database-host "$POSTGRES_HOST"' + install=true + fi + + if [ "$install" = true ]; then + echo "starting nextcloud installation" + echo "$install_options" + max_retries=10 + try=0 + until run_as "php /var/www/html/occ maintenance:install $install_options" || [ "$try" -gt "$max_retries" ] + do + echo "retrying install..." + try=$((try+1)) + sleep 20s + done + if [ "$try" -gt "$max_retries" ]; then + echo "installing of nextcloud failed!" + exit 1 + fi + if [ -n "${NEXTCLOUD_TRUSTED_DOMAINS+x}" ]; then + echo "setting trusted domains…" + NC_TRUSTED_DOMAIN_IDX=1 + for DOMAIN in $NEXTCLOUD_TRUSTED_DOMAINS ; do + DOMAIN=$(echo "$DOMAIN" | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//') + run_as "php /var/www/html/occ config:system:set trusted_domains $NC_TRUSTED_DOMAIN_IDX --value=$DOMAIN" + run_as "php /var/www/html/occ config:system:set overwritehost --value=$DOMAIN" + NC_TRUSTED_DOMAIN_IDX=$(($NC_TRUSTED_DOMAIN_IDX+1)) + done + fi + run_as "php /var/www/html/occ config:system:set overwriteprotocol --value=https" + else + echo "running web-based installer on first connect!" + fi + fi + #upgrade + else + run_as 'php /var/www/html/occ upgrade' + + run_as 'php /var/www/html/occ app:list' | sed -n "/Enabled:/,/Disabled:/p" > /tmp/list_after + echo "The following apps have been disabled:" + diff /tmp/list_before /tmp/list_after | grep '<' | cut -d- -f2 | cut -d: -f1 + rm -f /tmp/list_before /tmp/list_after + + fi + fi +fi + +exec "$@" diff --git a/infrastructure/docker-nextcloud/image/resources/install-debug.sh b/infrastructure/docker-nextcloud/image/resources/install-debug.sh new file mode 100644 index 0000000..a5b8ce8 --- /dev/null +++ b/infrastructure/docker-nextcloud/image/resources/install-debug.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +apt update && apt -qqy install vim bash-completion less diff --git a/infrastructure/docker-nextcloud/image/resources/install.sh b/infrastructure/docker-nextcloud/image/resources/install.sh new file mode 100755 index 0000000..b92fc18 --- /dev/null +++ b/infrastructure/docker-nextcloud/image/resources/install.sh @@ -0,0 +1,9 @@ +#!/bin/bash +set -Eeo pipefail + +mkdir /var/data + +install -m 0700 /tmp/install-debug.sh /usr/local/bin/ +install -m 0544 /tmp/upload-max-limit.ini /usr/local/etc/php/conf.d/ +install -m 0544 /tmp/memory-limit.ini /usr/local/etc/php/conf.d/ +install -m 0755 /tmp/entrypoint.sh / \ No newline at end of file diff --git a/infrastructure/docker-nextcloud/image/resources/memory-limit.ini b/infrastructure/docker-nextcloud/image/resources/memory-limit.ini new file mode 100644 index 0000000..3dc87bc --- /dev/null +++ b/infrastructure/docker-nextcloud/image/resources/memory-limit.ini @@ -0,0 +1,2 @@ + +memory_limit=5120M diff --git a/infrastructure/docker-nextcloud/image/resources/upload-max-limit.ini b/infrastructure/docker-nextcloud/image/resources/upload-max-limit.ini new file mode 100644 index 0000000..61a407e --- /dev/null +++ b/infrastructure/docker-nextcloud/image/resources/upload-max-limit.ini @@ -0,0 +1,3 @@ + +post_max_size = 512M +upload_max_filesize = 512M diff --git a/infrastructure/docker-nextcloud/test/Dockerfile b/infrastructure/docker-nextcloud/test/Dockerfile new file mode 100644 index 0000000..891c1ee --- /dev/null +++ b/infrastructure/docker-nextcloud/test/Dockerfile @@ -0,0 +1,11 @@ +FROM meissa-cloud-app + +RUN apt update +RUN mkdir /usr/share/man/man1/ +RUN apt -yqq install --no-install-recommends --yes curl default-jre-headless + +RUN curl -L -o /tmp/serverspec.jar https://github.com/DomainDrivenArchitecture/dda-serverspec-crate/releases/download/2.0.0/dda-serverspec-standalone.jar + +COPY serverspec.edn /tmp/serverspec.edn + +RUN java -jar /tmp/serverspec.jar /tmp/serverspec.edn -v diff --git a/infrastructure/docker-nextcloud/test/serverspec.edn b/infrastructure/docker-nextcloud/test/serverspec.edn new file mode 100644 index 0000000..767e01c --- /dev/null +++ b/infrastructure/docker-nextcloud/test/serverspec.edn @@ -0,0 +1,2 @@ +{:file [{:path "/var/data"} + {:path "/entrypoint.sh" :mod "755"}]} diff --git a/main/resources/backup/backup-restore.yaml b/main/resources/backup/backup-restore.yaml new file mode 100644 index 0000000..c13e166 --- /dev/null +++ b/main/resources/backup/backup-restore.yaml @@ -0,0 +1,59 @@ +kind: Pod +apiVersion: v1 +metadata: + name: backup-restore + labels: + app.kubernetes.io/name: backup-restore + app.kubernetes.io/part-of: cloud +spec: + containers: + - name: backup-app + image: domaindrivenarchitecture/c4k-cloud-backup + imagePullPolicy: IfNotPresent + command: ["/entrypoint-start-and-wait.sh"] + env: + - name: POSTGRES_USER_FILE + value: /var/run/secrets/cloud-secrets/postgres-user + - name: POSTGRES_DB_FILE + value: /var/run/secrets/cloud-secrets/postgres-db + - name: POSTGRES_PASSWORD_FILE + value: /var/run/secrets/cloud-secrets/postgres-password + - name: POSTGRES_HOST + value: "postgresql-service:5432" + - name: POSTGRES_SERVICE + value: "postgresql-service" + - name: POSTGRES_PORT + value: "5432" + - name: AWS_DEFAULT_REGION + value: eu-central-1 + - name: AWS_ACCESS_KEY_ID_FILE + value: /var/run/secrets/backup-secrets/aws-access-key-id + - name: AWS_SECRET_ACCESS_KEY_FILE + value: /var/run/secrets/backup-secrets/aws-secret-access-key + - name: RESTIC_REPOSITORY + valueFrom: + configMapKeyRef: + name: backup-config + key: restic-repository + - name: RESTIC_PASSWORD_FILE + value: /var/run/secrets/backup-secrets/restic-password + volumeMounts: + - name: cloud-data-volume + mountPath: /var/backups + - name: backup-secret-volume + mountPath: /var/run/secrets/backup-secrets + readOnly: true + - name: cloud-secret-volume + mountPath: /var/run/secrets/cloud-secrets + readOnly: true + volumes: + - name: cloud-data-volume + persistentVolumeClaim: + claimName: cloud-pvc + - name: cloud-secret-volume + secret: + secretName: cloud-secret + - name: backup-secret-volume + secret: + secretName: backup-secret + restartPolicy: OnFailure \ No newline at end of file diff --git a/main/resources/backup/config.yaml b/main/resources/backup/config.yaml new file mode 100644 index 0000000..17aa35c --- /dev/null +++ b/main/resources/backup/config.yaml @@ -0,0 +1,9 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: backup-config + labels: + app.kubernetes.io/name: backup + app.kubernetes.io/part-of: cloud +data: + restic-repository: restic-repository \ No newline at end of file diff --git a/main/resources/backup/configure-as-user.sh b/main/resources/backup/configure-as-user.sh new file mode 100644 index 0000000..a5a099c --- /dev/null +++ b/main/resources/backup/configure-as-user.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +kubectl delete --ignore-not-found=true -f backup-secret.yml +kubectl delete --ignore-not-found=true -f backup-config.yml +kubectl delete --ignore-not-found=true -f backup-cron.yml + +kubectl apply -f backup-secret.yml +kubectl apply -f backup-config.yml +kubectl apply -f backup-cron.yml diff --git a/main/resources/backup/cron.yaml b/main/resources/backup/cron.yaml new file mode 100644 index 0000000..8bb54bc --- /dev/null +++ b/main/resources/backup/cron.yaml @@ -0,0 +1,65 @@ +apiVersion: batch/v1beta1 +kind: CronJob +metadata: + name: cloud-backup + labels: + app.kubernetes.part-of: cloud +spec: + schedule: "10 23 * * *" + successfulJobsHistoryLimit: 0 + failedJobsHistoryLimit: 0 + jobTemplate: + spec: + template: + spec: + containers: + - name: backup-app + image: domaindrivenarchitecture/meissa-cloud-backup + imagePullPolicy: IfNotPresent + command: ["/entrypoint.sh"] + env: + - name: POSTGRES_USER_FILE + value: /var/run/secrets/cloud-secrets/postgres-user + - name: POSTGRES_DB_FILE + value: /var/run/secrets/cloud-secrets/postgres-db + - name: POSTGRES_PASSWORD_FILE + value: /var/run/secrets/cloud-secrets/postgres-password + - name: POSTGRES_HOST + value: "postgresql-service:5432" + - name: POSTGRES_SERVICE + value: "postgresql-service" + - name: POSTGRES_PORT + value: "5432" + - name: AWS_DEFAULT_REGION + value: eu-central-1 + - name: AWS_ACCESS_KEY_ID_FILE + value: /var/run/secrets/backup-secrets/aws-access-key-id + - name: AWS_SECRET_ACCESS_KEY_FILE + value: /var/run/secrets/backup-secrets/aws-secret-access-key + - name: RESTIC_REPOSITORY + valueFrom: + configMapKeyRef: + name: backup-config + key: restic-repository + - name: RESTIC_PASSWORD_FILE + value: /var/run/secrets/backup-secrets/restic-password + volumeMounts: + - name: cloud-data-volume + mountPath: /var/backups + - name: backup-secret-volume + mountPath: /var/run/secrets/backup-secrets + readOnly: true + - name: cloud-secret-volume + mountPath: /var/run/secrets/cloud-secrets + readOnly: true + volumes: + - name: cloud-data-volume + persistentVolumeClaim: + claimName: cloud-pvc + - name: cloud-secret-volume + secret: + secretName: cloud-secret + - name: backup-secret-volume + secret: + secretName: backup-secret + restartPolicy: OnFailure \ No newline at end of file diff --git a/main/resources/backup/secret.yaml b/main/resources/backup/secret.yaml new file mode 100644 index 0000000..4b68578 --- /dev/null +++ b/main/resources/backup/secret.yaml @@ -0,0 +1,9 @@ +apiVersion: v1 +kind: Secret +metadata: + name: backup-secret +type: Opaque +stringData: + aws-access-key-id: aws-access-key-id + aws-secret-access-key: aws-secret-access-key + restic-password: restic-password \ No newline at end of file diff --git a/main/resources/cloud/certificate.yaml b/main/resources/cloud/certificate.yaml new file mode 100644 index 0000000..054965b --- /dev/null +++ b/main/resources/cloud/certificate.yaml @@ -0,0 +1,13 @@ +apiVersion: cert-manager.io/v1alpha2 +kind: Certificate +metadata: + name: cloud-cert + namespace: default +spec: + secretName: cloud-secret + commonName: fqdn + dnsNames: + - fqdn + issuerRef: + name: letsencrypt-staging-issuer + kind: ClusterIssuer \ No newline at end of file diff --git a/main/resources/cloud/cloud-pod.yml.template b/main/resources/cloud/cloud-pod.yml.template new file mode 100644 index 0000000..eac26ec --- /dev/null +++ b/main/resources/cloud/cloud-pod.yml.template @@ -0,0 +1,45 @@ +kind: Pod +apiVersion: v1 +metadata: + name: cloud + labels: + app.kubernetes.io/name: cloud +spec: + shareProcessNamespace: true + containers: + - name: cloud-app + image: domaindrivenarchitecture/meissa-cloud-app + imagePullPolicy: IfNotPresent + ports: + - containerPort: 80 + env: + - name: NEXTCLOUD_ADMIN_USER_FILE + value: /var/run/secrets/cloud-secrets/nextcloud-admin-user + - name: NEXTCLOUD_ADMIN_PASSWORD_FILE + value: /var/run/secrets/cloud-secrets/nextcloud-admin-password + - name: NEXTCLOUD_TRUSTED_DOMAINS + value: "{{fqdn}}" + - name: POSTGRES_USER_FILE + value: /var/run/secrets/cloud-secrets/postgres-user + - name: POSTGRES_PASSWORD_FILE + value: /var/run/secrets/cloud-secrets/postgres-password + - name: POSTGRES_DB_FILE + value: /var/run/secrets/cloud-secrets/postgres-db + - name: POSTGRES_HOST + value: "postgresql-service:5432" + volumeMounts: + - name: cloud-data-volume + mountPath: /var/www/html + - name: cloud-secret-volume + mountPath: /var/run/secrets/cloud-secrets + readOnly: true + volumes: + - name: cloud-data-volume + persistentVolumeClaim: + claimName: cloud-pvc + - name: cloud-secret-volume + secret: + secretName: cloud-secret + - name: backup-secret-volume + secret: + secretName: backup-secret diff --git a/main/resources/cloud/configure-as-user.sh b/main/resources/cloud/configure-as-user.sh new file mode 100644 index 0000000..6e46558 --- /dev/null +++ b/main/resources/cloud/configure-as-user.sh @@ -0,0 +1,22 @@ +#!/bin/bash + +kubectl delete --ignore-not-found=true -f cloud-ingress.yml +kubectl delete --ignore-not-found=true -f cloud-pod.yml +kubectl delete --ignore-not-found=true -f cloud-pvc.yml +kubectl delete --ignore-not-found=true -f cloud-service.yml +kubectl delete --ignore-not-found=true -f cloud-secret.yml +kubectl delete --ignore-not-found=true -f cloud-persistent-volume.yml + +#Wait for postgres to be running +while [$POSTGRES = ""] +do + POSTGRES=$(kubectl get pods --selector=app=postgresql -o jsonpath='{.items[*].metadata.name}') +done +kubectl wait --for=condition=ready pod/$POSTGRES + +kubectl apply -f cloud-persistent-volume.yml +kubectl apply -f cloud-secret.yml +kubectl apply -f cloud-service.yml +kubectl apply -f cloud-pvc.yml +kubectl apply -f cloud-pod.yml +kubectl apply -f cloud-ingress.yml diff --git a/main/resources/cloud/ingress.yaml b/main/resources/cloud/ingress.yaml new file mode 100644 index 0000000..cc5a0df --- /dev/null +++ b/main/resources/cloud/ingress.yaml @@ -0,0 +1,26 @@ +apiVersion: extensions/v1beta1 +kind: Ingress +metadata: + name: ingress-cloud + annotations: + cert-manager.io/cluster-issuer: letsencrypt-staging-issuer + nginx.ingress.kubernetes.io/proxy-body-size: "256m" + nginx.ingress.kubernetes.io/ssl-redirect: "true" + nginx.ingress.kubernetes.io/rewrite-target: / + nginx.ingress.kubernetes.io/proxy-connect-timeout: "300" + nginx.ingress.kubernetes.io/proxy-send-timeout: "300" + nginx.ingress.kubernetes.io/proxy-read-timeout: "300" + namespace: default +spec: + tls: + - hosts: + - fqdn + secretName: cloud-secret + rules: + - host: fqdn + http: + paths: + - path: / + backend: + serviceName: cloud-service + servicePort: 80 diff --git a/main/resources/cloud/install-as-root.sh.template b/main/resources/cloud/install-as-root.sh.template new file mode 100644 index 0000000..1492df6 --- /dev/null +++ b/main/resources/cloud/install-as-root.sh.template @@ -0,0 +1,4 @@ +#!/bin/bash + +mkdir -p /var/cloud +install -d -m 0777 -o {{user}} -g {{user}} /var/cloud diff --git a/main/resources/cloud/persistent-volume.yaml b/main/resources/cloud/persistent-volume.yaml new file mode 100644 index 0000000..7c3f89e --- /dev/null +++ b/main/resources/cloud/persistent-volume.yaml @@ -0,0 +1,15 @@ +kind: PersistentVolume +apiVersion: v1 +metadata: + name: cloud-pv-volume + labels: + type: local + app: cloud +spec: + storageClassName: manual + accessModes: + - ReadWriteOnce + capacity: + storage: {{storage-size}}Gi #??? 30Gi? + hostPath: + path: "/var/cloud" diff --git a/main/resources/cloud/pod-running.sh b/main/resources/cloud/pod-running.sh new file mode 100755 index 0000000..61e8cb2 --- /dev/null +++ b/main/resources/cloud/pod-running.sh @@ -0,0 +1,25 @@ +#!/bin/bash +SECONDS=0 + +while true; do + + POD_STATUS="$(kubectl get pods --all-namespaces --field-selector=status.phase=Running | grep $1 )"; + if [ ! -z "$POD_STATUS" ] + then + # pod = ready is not enough + sleep $4 + break + fi + + let duration=$SECONDS/60 + # pallet needs a regular action, otherwise unwanted timeout after 5 min + echo "Seconds waited: ${SECONDS}" + if [ "$duration" -ge "$2" ] + then + exit 1 + fi + + sleep $3 +done + +exit 0 diff --git a/main/resources/cloud/pvc.yaml b/main/resources/cloud/pvc.yaml new file mode 100644 index 0000000..d4f7e04 --- /dev/null +++ b/main/resources/cloud/pvc.yaml @@ -0,0 +1,16 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: cloud-pvc + labels: + app: cloud +spec: + storageClassName: manual + accessModes: + - ReadWriteOnce + resources: + requests: + storage: {{storage-size}}Gi #??? 30Gi? + selector: + matchLabels: + app: cloud diff --git a/main/resources/cloud/secret.yaml b/main/resources/cloud/secret.yaml new file mode 100644 index 0000000..2429d16 --- /dev/null +++ b/main/resources/cloud/secret.yaml @@ -0,0 +1,11 @@ +apiVersion: v1 +kind: Secret +metadata: + name: cloud-secret +type: Opaque +stringData: + postgres-db: db-name + postgres-user: db-user-name + postgres-password: db-user-password + nextcloud-admin-user: admin-user + nextcloud-admin-password: admin-password diff --git a/main/resources/cloud/service.yaml b/main/resources/cloud/service.yaml new file mode 100644 index 0000000..7fcd0d7 --- /dev/null +++ b/main/resources/cloud/service.yaml @@ -0,0 +1,9 @@ +apiVersion: v1 +kind: Service +metadata: + name: cloud-service +spec: + selector: + app.kubernetes.io/name: cloud #??? + ports: + - port: 80 diff --git a/main/resources/cloud/verify.sh.template b/main/resources/cloud/verify.sh.template new file mode 100644 index 0000000..ad362e5 --- /dev/null +++ b/main/resources/cloud/verify.sh.template @@ -0,0 +1,15 @@ +#!/bin/bash + +echo -e "\n====================\n" +echo -e "cloud is running, ingress exists" +echo -e "\n====================\n" +kubectl get all + +echo -e "\n====================\n" +echo -e "shows certificate with subject" +echo -e "CN={{fqdn}}" +echo -e "issuer: CN=Fake LE Intermediate X1" +echo -e "\n====================\n" +curl --insecure -v https://{{fqdn}} + +echo -e "\n" diff --git a/main/resources/postgres/configure-as-user.sh b/main/resources/postgres/configure-as-user.sh new file mode 100644 index 0000000..3d76fbe --- /dev/null +++ b/main/resources/postgres/configure-as-user.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +kubectl delete --ignore-not-found=true -f postgres-deployment.yml +kubectl delete --ignore-not-found=true -f postgres-pvc.yml +kubectl delete --ignore-not-found=true -f postgres-service.yml +kubectl delete --ignore-not-found=true -f postgres-config.yml +kubectl delete --ignore-not-found=true -f postgres-secret.yml +kubectl delete --ignore-not-found=true -f postgres-persistent-volume.yml + +kubectl apply -f postgres-persistent-volume.yml +kubectl apply -f postgres-secret.yml +kubectl apply -f postgres-config.yml +kubectl apply -f postgres-service.yml +kubectl apply -f postgres-pvc.yml +kubectl apply -f postgres-deployment.yml diff --git a/main/resources/postgres/install-as-root.sh.template b/main/resources/postgres/install-as-root.sh.template new file mode 100644 index 0000000..d8899f6 --- /dev/null +++ b/main/resources/postgres/install-as-root.sh.template @@ -0,0 +1,4 @@ +#!/bin/bash + +mkdir -p /var/postgres +install -d -m 0777 -o {{user}} -g {{user}} /var/postgres diff --git a/main/resources/postgres/postgres-config.yml b/main/resources/postgres/postgres-config.yml new file mode 100644 index 0000000..799cc45 --- /dev/null +++ b/main/resources/postgres/postgres-config.yml @@ -0,0 +1,10 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: postgres-config + labels: + app: postgres +data: + postgresql.conf: | + max_connections = 1000 + shared_buffers = 512MB diff --git a/main/resources/postgres/postgres-deployment.yml.template b/main/resources/postgres/postgres-deployment.yml.template new file mode 100644 index 0000000..55d9f65 --- /dev/null +++ b/main/resources/postgres/postgres-deployment.yml.template @@ -0,0 +1,49 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: postgresql +spec: + selector: + matchLabels: + app: postgresql + strategy: + type: Recreate + template: + metadata: + labels: + app: postgresql + spec: + containers: + - image: postgres + name: postgresql + env: + - name: POSTGRES_USER_FILE + value: /var/run/secrets/postgres-secrets/postgres-user + - name: POSTGRES_DB_FILE + value: /var/run/secrets/postgres-secrets/postgres-db + - name: POSTGRES_PASSWORD_FILE + value: /var/run/secrets/postgres-secrets/postgres-password + ports: + - containerPort: 5432 + name: postgresql + cmd: + volumeMounts: + - name: postgresql + mountPath: /var/lib/postgresql/data + - name: postgres-secret-volume + mountPath: /var/run/secrets/postgres-secrets + readOnly: true + - name: postgres-config-volume + mountPath: /etc/postgresql/postgresql.conf + subPath: postgresql.conf + readOnly: true + volumes: + - name: postgresql + persistentVolumeClaim: + claimName: postgres-claim + - name: postgres-secret-volume + secret: + secretName: postgres-secret + - name: postgres-config-volume + configMap: + name: postgres-config diff --git a/main/resources/postgres/postgres-persistent-volume.yml b/main/resources/postgres/postgres-persistent-volume.yml new file mode 100644 index 0000000..0c7b710 --- /dev/null +++ b/main/resources/postgres/postgres-persistent-volume.yml @@ -0,0 +1,15 @@ +kind: PersistentVolume +apiVersion: v1 +metadata: + name: postgres-pv-volume + labels: + type: local + app: postgresql +spec: + storageClassName: manual + accessModes: + - ReadWriteOnce + capacity: + storage: 10Gi + hostPath: + path: "/var/postgres" diff --git a/main/resources/postgres/postgres-pvc.yml b/main/resources/postgres/postgres-pvc.yml new file mode 100644 index 0000000..1a43337 --- /dev/null +++ b/main/resources/postgres/postgres-pvc.yml @@ -0,0 +1,16 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: postgres-claim + labels: + app: postgresql +spec: + storageClassName: manual + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 10Gi + selector: + matchLabels: + app: postgresql \ No newline at end of file diff --git a/main/resources/postgres/postgres-secret.yml.template b/main/resources/postgres/postgres-secret.yml.template new file mode 100644 index 0000000..d3579d3 --- /dev/null +++ b/main/resources/postgres/postgres-secret.yml.template @@ -0,0 +1,9 @@ +apiVersion: v1 +kind: Secret +metadata: + name: postgres-secret +type: Opaque +stringData: + postgres-db: cloud + postgres-user: {{db-user-name}} + postgres-password: {{db-user-password}} diff --git a/main/resources/postgres/postgres-service.yml b/main/resources/postgres/postgres-service.yml new file mode 100644 index 0000000..c3cadf1 --- /dev/null +++ b/main/resources/postgres/postgres-service.yml @@ -0,0 +1,10 @@ +--- +apiVersion: v1 +kind: Service +metadata: + name: postgresql-service +spec: + selector: + app: postgresql + ports: + - port: 5432 diff --git a/main/resources/postgres/verify.sh b/main/resources/postgres/verify.sh new file mode 100644 index 0000000..b9f0730 --- /dev/null +++ b/main/resources/postgres/verify.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +echo -e "\n====================\n" +echo -e "postgres is running" +echo -e "\n====================\n" +kubectl get all + +echo -e "\n" diff --git a/main/src/meissa/pallet/meissa_cloud/app.clj b/main/src/meissa/pallet/meissa_cloud/app.clj new file mode 100644 index 0000000..1547082 --- /dev/null +++ b/main/src/meissa/pallet/meissa_cloud/app.clj @@ -0,0 +1,61 @@ +(ns meissa.pallet.meissa-cloud.app + (:require + [schema.core :as s] + [dda.pallet.commons.secret :as secret] + [dda.config.commons.map-utils :as mu] + [dda.pallet.core.app :as core-app] + [dda.pallet.dda-config-crate.infra :as config-crate] + [dda.pallet.dda-user-crate.app :as user] + [dda.pallet.dda-k8s-crate.app :as k8s] + [meissa.pallet.meissa-cloud.convention :as convention] + [meissa.pallet.meissa-cloud.infra :as infra])) + +(def with-cloud infra/with-cloud) + +(def CloudConvention convention/CloudConvention) + +(def CloudConventionResolved convention/CloudConventionResolved) + +(def InfraResult convention/InfraResult) + +(def CloudApp + {:group-specific-config + {s/Keyword (merge InfraResult + user/InfraResult + k8s/InfraResult)}}) + +(s/defn ^:always-validate + app-configuration-resolved :- CloudApp + [resolved-convention-config :- CloudConventionResolved + & options] + (let [{:keys [group-key] :or {group-key infra/facility}} options] + (mu/deep-merge + (k8s/app-configuration-resolved + (convention/k8s-convention-configuration resolved-convention-config) :group-key group-key) + {:group-specific-config + {group-key + (convention/infra-configuration resolved-convention-config)}}))) + +(s/defn ^:always-validate + app-configuration :- CloudApp + [convention-config :- CloudConvention + & options] + (let [resolved-convention-config (secret/resolve-secrets convention-config CloudConvention)] + (apply app-configuration-resolved resolved-convention-config options))) + +(s/defmethod ^:always-validate + core-app/group-spec infra/facility + [crate-app + convention-config :- CloudConventionResolved] + (let [app-config (app-configuration-resolved convention-config)] + (core-app/pallet-group-spec + app-config [(config-crate/with-config app-config) + user/with-user + k8s/with-k8s + with-cloud]))) + +(def crate-app (core-app/make-dda-crate-app + :facility infra/facility + :convention-schema CloudConvention + :convention-schema-resolved CloudConventionResolved + :default-convention-file "cloud.edn")) diff --git a/main/src/meissa/pallet/meissa_cloud/convention.clj b/main/src/meissa/pallet/meissa_cloud/convention.clj new file mode 100644 index 0000000..17223f4 --- /dev/null +++ b/main/src/meissa/pallet/meissa_cloud/convention.clj @@ -0,0 +1,93 @@ +(ns meissa.pallet.meissa-cloud.convention + (:require + [schema.core :as s] + [dda.pallet.commons.secret :as secret] + [dda.config.commons.map-utils :as mu] + [clojure.spec.alpha :as sp] + [clojure.spec.test.alpha :as st] + [dda.pallet.dda-k8s-crate.convention :as k8s-convention] + [meissa.pallet.meissa-cloud.infra :as infra] + [clojure.string :as str] + [meissa.pallet.meissa-cloud.convention.bash :as bash] + [meissa.pallet.meissa-cloud.convention.bash-php :as bash-php])) + +(def InfraResult {infra/facility infra/MeissaCloudInfra}) + +(s/def CloudConvention + {:user s/Keyword + :external-ip s/Str + :fqdn s/Str + :cert-manager (s/enum :letsencrypt-prod-issuer :letsencrypt-staging-issuer) + :db-user-password secret/Secret + :admin-user s/Str + :admin-password secret/Secret + :storage-size s/Int + :restic-repository s/Str + :aws-access-key-id secret/Secret + :aws-secret-access-key secret/Secret + :restic-password secret/Secret + (s/optional-key :u18-04) (s/enum true)}) + +(def CloudConventionResolved (secret/create-resolved-schema CloudConvention)) + +(sp/def ::user keyword?) +(sp/def ::external-ip string?) +(sp/def ::fqdn string?) +(sp/def ::cert-manager #{:letsencrypt-prod-issuer :letsencrypt-staging-issuer}) +(sp/def ::db-user-password bash-php/bash-php-env-string?) +(sp/def ::admin-user bash-php/bash-php-env-string?) +(sp/def ::admin-password bash-php/bash-php-env-string?) +(sp/def ::storage-size int?) +(sp/def ::restic-repository string?) +(sp/def ::restic-password bash/bash-env-string?) +(sp/def ::aws-access-key-id bash/bash-env-string?) +(sp/def ::aws-secret-access-key bash/bash-env-string?) +(sp/def ::u18-04 #{true}) +(def cloud-convention-resolved? (sp/keys :req-un [::user ::external-ip ::fqdn ::cert-manager + ::db-user-password ::admin-user ::admin-password + ::storage-size ::restic-repository ::restic-password + ::aws-access-key-id ::aws-secret-access-key ] + :opt-un [::u18-04])) + +(def cloud-spec-resolved nil) + +(s/defn k8s-convention-configuration :- k8s-convention/k8sConventionResolved + [convention-config :- CloudConventionResolved] + {:pre [(sp/valid? cloud-convention-resolved? convention-config)]} + (let [{:keys [cert-manager external-ip user u18-04]} convention-config + cluster-issuer (name cert-manager)] + (if u18-04 + {:user user + :k8s {:external-ip external-ip + :u18-04 true} + :cert-manager cert-manager} + {:user user + :k8s {:external-ip external-ip} + :cert-manager cert-manager}))) + + +(s/defn ^:always-validate + infra-configuration :- InfraResult + [convention-config :- CloudConventionResolved] + (let [{:keys [cert-manager fqdn user db-user-password admin-user admin-password storage-size + restic-repository aws-access-key-id aws-secret-access-key restic-password]} convention-config + cluster-issuer (name cert-manager) + db-user-name "cloud"] + {infra/facility + {:user user + :backup {:restic-repository restic-repository + :aws-access-key-id aws-access-key-id + :aws-secret-access-key aws-secret-access-key + :restic-password restic-password} + :cloud {:fqdn fqdn + :secret-name (str/replace fqdn #"\." "-") + :cluster-issuer cluster-issuer + :db-name "cloud" + :db-user-password db-user-password + :db-user-name db-user-name + :admin-user admin-user + :admin-password admin-password + :storage-size (str storage-size)} + :postgres {:db-user-password db-user-password + :db-user-name db-user-name}}})) + diff --git a/main/src/meissa/pallet/meissa_cloud/convention/bash.clj b/main/src/meissa/pallet/meissa_cloud/convention/bash.clj new file mode 100644 index 0000000..5d1ef2c --- /dev/null +++ b/main/src/meissa/pallet/meissa_cloud/convention/bash.clj @@ -0,0 +1,10 @@ +(ns meissa.pallet.meissa-cloud.convention.bash + (:require + [clojure.spec.alpha :as s])) + +(defn bash-env-string? + [input] + (and (string? input) + (not (re-matches #".*['\"\$]+.*" input)))) + +(s/def ::plain bash-env-string?) diff --git a/main/src/meissa/pallet/meissa_cloud/convention/bash_php.clj b/main/src/meissa/pallet/meissa_cloud/convention/bash_php.clj new file mode 100644 index 0000000..d065a66 --- /dev/null +++ b/main/src/meissa/pallet/meissa_cloud/convention/bash_php.clj @@ -0,0 +1,11 @@ +(ns meissa.pallet.meissa-cloud.convention.bash-php + (:require + [clojure.spec.alpha :as s] + [meissa.pallet.meissa-cloud.convention.bash :as bash])) + +(defn bash-php-env-string? + [input] + (and (bash/bash-env-string? input) + (not (re-matches #".*[\-\\\\]+.*" input)))) + +(s/def ::plain bash-php-env-string?) diff --git a/main/src/meissa/pallet/meissa_cloud/infra.clj b/main/src/meissa/pallet/meissa_cloud/infra.clj new file mode 100644 index 0000000..a964dc3 --- /dev/null +++ b/main/src/meissa/pallet/meissa_cloud/infra.clj @@ -0,0 +1,51 @@ +(ns meissa.pallet.meissa-cloud.infra + (:require + [schema.core :as s] + [dda.pallet.core.infra :as core-infra] + [meissa.pallet.meissa-cloud.infra.backup :as backup] + [meissa.pallet.meissa-cloud.infra.cloud :as cloud] + [meissa.pallet.meissa-cloud.infra.postgres :as postgres])) + +(def facility :meissa-cloud) + +(def MeissaCloudInfra + (merge + {:user s/Keyword} + backup/MeissaBackupInfra + cloud/MeissaCloudInfra + postgres/MeissaPostgresInfra)) + +(s/defmethod core-infra/dda-init facility + [dda-crate config] + (let [facility (:facility dda-crate) + {:keys [user backup postgres cloud]} config + user-str (name user)] + (postgres/init facility user-str postgres) + (cloud/init facility user-str cloud) + (backup/init facility user-str backup))) + +(s/defmethod core-infra/dda-install facility + [dda-crate config] + (let [facility (:facility dda-crate) + {:keys [user backup postgres cloud]} config + user-str (name user)] + (postgres/install facility user-str postgres) + (cloud/install facility user-str cloud) + (backup/install facility user-str backup))) + +(s/defmethod core-infra/dda-configure facility + [dda-crate config] + (let [facility (:facility dda-crate) + {:keys [user backup postgres cloud]} config + user-str (name user)] + (postgres/configure facility user-str postgres) + (cloud/configure facility user-str cloud) + (backup/configure facility user-str backup))) + +(def meissa-cloud + (core-infra/make-dda-crate-infra + :facility facility + :infra-schema MeissaCloudInfra)) + +(def with-cloud + (core-infra/create-infra-plan meissa-cloud)) diff --git a/main/src/meissa/pallet/meissa_cloud/infra/backup.clj b/main/src/meissa/pallet/meissa_cloud/infra/backup.clj new file mode 100644 index 0000000..c80ead6 --- /dev/null +++ b/main/src/meissa/pallet/meissa_cloud/infra/backup.clj @@ -0,0 +1,39 @@ +(ns meissa.pallet.meissa-cloud.infra.backup + (:require + [schema.core :as s] + [dda.provision :as p] + [dda.provision.pallet :as pp])) + +(s/def Backup + {:restic-repository s/Str + :aws-access-key-id s/Str + :aws-secret-access-key s/Str + :restic-password s/Str}) + +(def MeissaBackupInfra {:backup Backup}) + +(def backup "backup") + +(defn init [facility user config]) + +(defn install + [facility user config] + (let [facility-name (name facility)] + (p/provision-log ::pp/pallet facility-name backup + ::p/info "install") + (p/copy-resources-to-user + ::pp/pallet user facility-name backup + [{:filename "backup-secret.yml" :config config} + {:filename "backup-config.yml" :config config} + {:filename "configure-as-user.sh"} + {:filename "backup-restore.yml"} + {:filename "backup-cron.yml"}]))) + +(defn configure + [facility user config] + (let [facility-name (name facility)] + (p/provision-log ::pp/pallet facility-name backup + ::p/info "configure") + (p/exec-file-on-target-as-user + ::pp/pallet user facility-name backup "configure-as-user.sh") + )) diff --git a/main/src/meissa/pallet/meissa_cloud/infra/cloud.clj b/main/src/meissa/pallet/meissa_cloud/infra/cloud.clj new file mode 100644 index 0000000..ed11188 --- /dev/null +++ b/main/src/meissa/pallet/meissa_cloud/infra/cloud.clj @@ -0,0 +1,57 @@ +(ns meissa.pallet.meissa-cloud.infra.cloud + (:require + [schema.core :as s] + [dda.provision :as p] + [dda.provision.pallet :as pp])) + +(s/def Cloud + {:fqdn s/Str + :secret-name s/Str + :cluster-issuer s/Str + :db-name s/Str + :db-user-name s/Str + :db-user-password s/Str + :admin-user s/Str + :admin-password s/Str + :storage-size s/Str}) + +(def MeissaCloudInfra {:cloud Cloud}) + +(def cloud "cloud") + +(defn init + [facility user config] + (let [facility-name (name facility)] + (p/provision-log ::pp/pallet facility-name cloud + ::p/info "init") + (p/copy-resources-to-tmp + ::pp/pallet facility-name cloud + [{:filename "install-as-root.sh" :config {:user user}}]))) + + +(defn install + [facility user config] + (let [facility-name (name facility)] + (p/provision-log ::pp/pallet facility-name cloud + ::p/info "install") + (p/copy-resources-to-user + ::pp/pallet user facility-name cloud + [{:filename "pod-running.sh"} + {:filename "cloud-persistent-volume.yml" :config config} + {:filename "cloud-secret.yml" :config config} + {:filename "cloud-service.yml"} + {:filename "cloud-pvc.yml" :config config} + {:filename "cloud-pod.yml" :config config} + {:filename "cloud-ingress.yml" :config config} + {:filename "configure-as-user.sh"} + {:filename "verify.sh" :config config}]) + (p/exec-file-on-target-as-root + ::pp/pallet facility-name cloud "install-as-root.sh"))) + +(defn configure + [facility user config] + (let [facility-name (name facility)] + (p/provision-log ::pp/pallet facility-name cloud + ::p/info "configure") + (p/exec-file-on-target-as-user + ::pp/pallet user facility-name cloud "configure-as-user.sh"))) diff --git a/main/src/meissa/pallet/meissa_cloud/infra/postgres.clj b/main/src/meissa/pallet/meissa_cloud/infra/postgres.clj new file mode 100644 index 0000000..60a3e8b --- /dev/null +++ b/main/src/meissa/pallet/meissa_cloud/infra/postgres.clj @@ -0,0 +1,47 @@ +(ns meissa.pallet.meissa-cloud.infra.postgres + (:require + [schema.core :as s] + [dda.provision :as p] + [dda.provision.pallet :as pp])) + +(s/def Postgres {:db-user-name s/Str :db-user-password s/Str}) + +(def MeissaPostgresInfra {:postgres Postgres}) + +(def postgres "postgres") + +(defn init + [facility user config] + (let [facility-name (name facility)] + (p/provision-log ::pp/pallet facility-name postgres + ::p/info "init") + (p/copy-resources-to-tmp + ::pp/pallet facility-name postgres + [{:filename "install-as-root.sh" :config {:user user}}]))) + + +(defn install + [facility user config] + (let [facility-name (name facility)] + (p/provision-log ::pp/pallet facility-name postgres + ::p/info "install") + (p/copy-resources-to-user + ::pp/pallet user facility-name postgres + [{:filename "postgres-persistent-volume.yml"} + {:filename "postgres-secret.yml" :config config} + {:filename "postgres-config.yml"} + {:filename "postgres-service.yml"} + {:filename "postgres-pvc.yml"} + {:filename "postgres-deployment.yml" :config config} + {:filename "configure-as-user.sh"} + {:filename "verify.sh"}]) + (p/exec-file-on-target-as-root + ::pp/pallet facility-name postgres "install-as-root.sh"))) + +(defn configure + [facility user config] + (let [facility-name (name facility)] + (p/provision-log ::pp/pallet facility-name postgres + ::p/info "configure") + (p/exec-file-on-target-as-user + ::pp/pallet user facility-name postgres "configure-as-user.sh"))) diff --git a/project.clj b/project.clj index c6dc3f6..4d4e476 100644 --- a/project.clj +++ b/project.clj @@ -1,41 +1,55 @@ -(defproject org.domaindrivenarchitecture/c4k-cloud "0.1.3-SNAPSHOT" - :description "cloud c4k-installation package" - :url "https://domaindrivenarchitecture.org" - :license {:name "Apache License, Version 2.0" - :url "https://www.apache.org/licenses/LICENSE-2.0.html"} - :dependencies [[org.clojure/clojure "1.10.3"] - [org.clojure/tools.reader "1.3.4"] - [org.domaindrivenarchitecture/c4k-common-clj "0.2.8"]] +(defproject meissa/meissa-cloud "1.0.2-SNAPSHOT" + :description "Crate to install cloud" + :url "https://meissa-gmbh.de" + :license {:name "meissa commercial license" + :url "https://www.meissa-gmbh.de"} + :dependencies [[dda/dda-pallet "4.0.3"] + [dda/dda-k8s-crate "1.0.1"] + [orchestra "2021.01.01-1"]] :target-path "target/%s/" - :source-paths ["src/main/cljc" - "src/main/clj"] - :resource-paths ["src/main/resources"] - :repositories [["snapshots" :clojars] - ["releases" :clojars]] - :deploy-repositories [["snapshots" :clojars] - ["releases" :clojars]] - :profiles {:test {:test-paths ["src/test/cljc"] - :resource-paths ["src/test/resources"] - :dependencies [[dda/data-test "0.1.1"]]} - :dev {:plugins [[lein-shell "0.5.0"]]} - :uberjar {:aot :all - :main dda.c4k-cloud.uberjar - :uberjar-name "c4k-cloud-standalone.jar" - :dependencies [[org.clojure/tools.cli "1.0.206"] - [ch.qos.logback/logback-classic "1.3.0-alpha4" - :exclusions [com.sun.mail/javax.mail]] + :source-paths ["main/src"] + :resource-paths ["main/resources"] + :repositories [["clojars" "https://clojars.org/repo"] + ["snapshots" + "https://artifact.prod.meissa-gmbh.de/repository/maven-snapshots/"] + ["releases" + "https://artifact.prod.meissa-gmbh.de/repository/maven-releases/"]] + :deploy-repositories [["snapshots" "https://artifact.prod.meissa-gmbh.de/repository/maven-snapshots/"] + ["releases" "https://artifact.prod.meissa-gmbh.de/repository/maven-releases/"]] + :profiles {:dev {:source-paths ["integration/src" + "test/src" + "uberjar/src"] + :resource-paths ["integration/resources" + "test/resources"] + :dependencies + [[org.clojure/test.check "1.1.0"] + [dda/data-test "0.1.1"] + [dda/pallet "0.9.1" :classifier "tests"] + [ch.qos.logback/logback-classic "1.3.0-alpha5"] + [org.slf4j/jcl-over-slf4j "2.0.0-alpha1"]] + :plugins + [[lein-sub "0.3.0"]] + :leiningen/reply + {:dependencies [[org.slf4j/jcl-over-slf4j "1.8.0-beta2"]] + :exclusions [commons-logging]}} + :test {:test-paths ["test/src"] + :resource-paths ["test/resources"] + :dependencies [[dda/pallet "0.9.1" :classifier "tests"]]} + :uberjar {:source-paths ["uberjar/src"] + :resource-paths ["uberjar/resources"] + :aot :all + :main meissa.pallet.meissa-cloud.main + :uberjar-name "meissa-cloud-standalone.jar" + :dependencies [[org.clojure/tools.cli "1.0.194"] + [ch.qos.logback/logback-classic "1.3.0-alpha5"] [org.slf4j/jcl-over-slf4j "2.0.0-alpha1"]]}} - :release-tasks [["test"] - ["vcs" "assert-committed"] + :release-tasks [["vcs" "assert-committed"] ["change" "version" "leiningen.release/bump-version" "release"] ["vcs" "commit"] ["vcs" "tag"] - ["change" "version" "leiningen.release/bump-version"]] - :aliases {"native" ["shell" - "native-image" - "--report-unsupported-elements-at-runtime" - "--initialize-at-build-time" - "-jar" "target/uberjar/c4k-cloud-standalone.jar" - "-H:ResourceConfigurationFiles=graalvm-resource-config.json" - "-H:Log=registerResource" - "-H:Name=target/graalvm/${:name}"]}) + ["deploy"] + ["uberjar"] + ["change" "version" "leiningen.release/bump-version"] + ["vcs" "commit"] + ["vcs" "push"]] + :local-repo-classpath true) diff --git a/targets.edn b/targets.edn new file mode 100644 index 0000000..515f51a --- /dev/null +++ b/targets.edn @@ -0,0 +1,3 @@ +{:existing [{:node-name "cloud" + :node-ip "168.119.190.126"}] + :provisioning-user {:login "root"}} diff --git a/test/resources/meissa/pallet/meissa_cloud/convention_test/should_generate_infra_for_convention.edn b/test/resources/meissa/pallet/meissa_cloud/convention_test/should_generate_infra_for_convention.edn new file mode 100644 index 0000000..9b4e226 --- /dev/null +++ b/test/resources/meissa/pallet/meissa_cloud/convention_test/should_generate_infra_for_convention.edn @@ -0,0 +1,30 @@ +{:input {:user :k8s + :external-ip "95.217.221.140" + :fqdn "cloud.test.meissa-gmbh.de" + :cert-manager :letsencrypt-staging-issuer + :db-user-password "test1234" + :admin-user "root" + :admin-password "test1234" + :storage-size 50 + :restic-repository "test4321" + :aws-access-key-id "10" + :aws-secret-access-key "secret" + :restic-password "test4321"} + :expected {:meissa-cloud + {:user :k8s + :backup {:restic-repository "test4321" + :aws-access-key-id "10" + :aws-secret-access-key "secret" + :restic-password "test4321"} + :cloud {:fqdn "cloud.test.meissa-gmbh.de" + :secret-name "cloud-test-meissa-gmbh-de" + :cluster-issuer "letsencrypt-staging-issuer" + :db-name "cloud" + :db-user-password "test1234" + :db-user-name "cloud" + :admin-user "root" + :admin-password "test1234" + :storage-size "50"} + :postgres {:db-user-password "test1234" + :db-user-name "cloud"}}} + } diff --git a/test/resources/meissa/pallet/meissa_cloud/convention_test/should_generate_k8s_convention.edn b/test/resources/meissa/pallet/meissa_cloud/convention_test/should_generate_k8s_convention.edn new file mode 100644 index 0000000..0478a77 --- /dev/null +++ b/test/resources/meissa/pallet/meissa_cloud/convention_test/should_generate_k8s_convention.edn @@ -0,0 +1,30 @@ +{:input {:user :k8s + :external-ip "95.217.221.140" + :fqdn "cloud.test.meissa-gmbh.de" + :cert-manager :letsencrypt-staging-issuer + :db-user-password "test1234" + :admin-user "root" + :admin-password "test1234" + :storage-size 50 + :restic-repository "cloud" + :aws-access-key-id "10" + :aws-secret-access-key "secret" + :restic-password "test4321"} + :expected {:user :k8s, :k8s {:external-ip "95.217.221.140"}, + :cert-manager :letsencrypt-staging-issuer}} + +{:input {:user :k8s + :external-ip "95.217.221.140" + :fqdn "cloud.test.meissa-gmbh.de" + :cert-manager :letsencrypt-staging-issuer + :db-user-password "test1234" + :admin-user "root" + :admin-password "test1234" + :storage-size 50 + :restic-repository "cloud" + :aws-access-key-id "10" + :aws-secret-access-key "secret" + :restic-password "test4321" + :u18-04 true} + :expected {:user :k8s, :k8s {:external-ip "95.217.221.140", :u18-04 true} + :cert-manager :letsencrypt-staging-issuer}} diff --git a/test/resources/meissa/pallet/meissa_cloud/convention_test/should_validate_input.0.edn b/test/resources/meissa/pallet/meissa_cloud/convention_test/should_validate_input.0.edn new file mode 100644 index 0000000..7dda0b7 --- /dev/null +++ b/test/resources/meissa/pallet/meissa_cloud/convention_test/should_validate_input.0.edn @@ -0,0 +1,14 @@ +{:input {:user :k8s + :external-ip "95.217.221.140" + :fqdn "cloud.test.meissa-gmbh.de" + :cert-manager :letsencrypt-staging-issuer + :db-user-password "test1234" + :admin-user "root" + :admin-password "test1234" + :storage-size 50 + :restic-repository "test4321" + :aws-access-key-id "10" + :aws-secret-access-key "secret" + :restic-password "test4321"} + :expected true + } diff --git a/test/resources/meissa/pallet/meissa_cloud/convention_test/should_validate_input.1.edn b/test/resources/meissa/pallet/meissa_cloud/convention_test/should_validate_input.1.edn new file mode 100644 index 0000000..b8aec50 --- /dev/null +++ b/test/resources/meissa/pallet/meissa_cloud/convention_test/should_validate_input.1.edn @@ -0,0 +1,14 @@ +{:input {:user :k8s + :external-ip "95.217.221.140" + :fqdn "cloud.test.meissa-gmbh.de" + :cert-manager :letsencrypt-staging-issuer + :db-user-password "test-1234" + :admin-user "root" + :admin-password "test1234" + :storage-size 50 + :restic-repository "test4321" + :aws-access-key-id "10" + :aws-secret-access-key "secret" + :restic-password "test4321"} + :expected false + } diff --git a/test/resources/meissa/pallet/meissa_cloud/convention_test/should_validate_input.2.edn b/test/resources/meissa/pallet/meissa_cloud/convention_test/should_validate_input.2.edn new file mode 100644 index 0000000..ada2a9e --- /dev/null +++ b/test/resources/meissa/pallet/meissa_cloud/convention_test/should_validate_input.2.edn @@ -0,0 +1,14 @@ +{:input {:user :k8s + :external-ip "95.217.221.140" + :fqdn "cloud.test.meissa-gmbh.de" + :cert-manager :letsencrypt-staging-issuer + :db-user-password "test1234" + :admin-user "root" + :admin-password "test1234" + :storage-size 50 + :restic-repository "test4321" + :aws-access-key-id "1$0" + :aws-secret-access-key "secret" + :restic-password "test4321"} + :expected false + } diff --git a/test/src/meissa/pallet/meissa_cloud/app_test.clj b/test/src/meissa/pallet/meissa_cloud/app_test.clj new file mode 100644 index 0000000..9daafa9 --- /dev/null +++ b/test/src/meissa/pallet/meissa_cloud/app_test.clj @@ -0,0 +1,31 @@ +(ns meissa.pallet.meissa-cloud.app-test + (:require + [clojure.test :refer :all] + [schema.core :as s] + [meissa.pallet.meissa-cloud.app :as sut])) + +(s/set-fn-validation! true) + +(s/def test-convention-conf + {:user :k8s + :external-ip "12.121.111.121" + :fqdn "some.domain.de" + :cert-manager :letsencrypt-staging-issuer + :db-user-password "test1234" + :admin-user "root" + :admin-password "test1234" + :storage-size 50 + :restic-repository "cloud" + :aws-access-key-id "10" + :aws-secret-access-key "secret" + :restic-password "test4321"}) + +(deftest app-config + (testing + "test plan-def" + (is (map? (sut/app-configuration-resolved test-convention-conf))))) + +(deftest plan-def + (testing + "test plan-def" + (is (map? sut/with-cloud)))) diff --git a/test/src/meissa/pallet/meissa_cloud/convention/bash_php_test.clj b/test/src/meissa/pallet/meissa_cloud/convention/bash_php_test.clj new file mode 100644 index 0000000..787fd6c --- /dev/null +++ b/test/src/meissa/pallet/meissa_cloud/convention/bash_php_test.clj @@ -0,0 +1,20 @@ +(ns meissa.pallet.meissa-cloud.convention.bash-php-test + (:require + [clojure.test :refer :all] + [meissa.pallet.meissa-cloud.convention.bash-php :as sut])) + + +(deftest test-it + (is (= false + (sut/bash-php-env-string? 4))) + (is (= false + (sut/bash-php-env-string? "hal-lo"))) + (is (= false + (sut/bash-php-env-string? "hal--lo"))) + (is (= false + (sut/bash-php-env-string? "hal\\lo"))) + (is (= true + (sut/bash-php-env-string? "test"))) + (is (= true + (sut/bash-php-env-string? "test123"))) + ) \ No newline at end of file diff --git a/test/src/meissa/pallet/meissa_cloud/convention/bash_test.clj b/test/src/meissa/pallet/meissa_cloud/convention/bash_test.clj new file mode 100644 index 0000000..ce51ba6 --- /dev/null +++ b/test/src/meissa/pallet/meissa_cloud/convention/bash_test.clj @@ -0,0 +1,22 @@ +(ns meissa.pallet.meissa-cloud.convention.bash-test + (:require + [clojure.test :refer :all] + [meissa.pallet.meissa-cloud.convention.bash :as sut])) + + +(deftest test-it + (is (= false + (sut/bash-env-string? 4))) + (is (= false + (sut/bash-env-string? "1$0"))) + (is (= false + (sut/bash-env-string? "'hallo"))) + (is (= false + (sut/bash-env-string? "hallo\""))) + (is (= false + (sut/bash-env-string? "hall$o"))) + (is (= true + (sut/bash-env-string? "test"))) + (is (= true + (sut/bash-env-string? "test123"))) + ) \ No newline at end of file diff --git a/test/src/meissa/pallet/meissa_cloud/convention_test.clj b/test/src/meissa/pallet/meissa_cloud/convention_test.clj new file mode 100644 index 0000000..e5e30c1 --- /dev/null +++ b/test/src/meissa/pallet/meissa_cloud/convention_test.clj @@ -0,0 +1,18 @@ +(ns meissa.pallet.meissa-cloud.convention-test + (:require + [clojure.test :refer :all] + [data-test :refer :all] + [meissa.pallet.meissa-cloud.convention :as sut] + [clojure.spec.alpha :as sp])) + +(defdatatest should-generate-infra-for-convention [input expected] + (is (= expected + (sut/infra-configuration input)))) + +(defdatatest should-generate-k8s-convention [input expected] + (is (= expected + (sut/k8s-convention-configuration input)))) + +(defdatatest should-validate-input [input expected] + (is (= expected + (sp/valid? sut/cloud-convention-resolved? input)))) diff --git a/uberjar/resources/localhost-target.edn b/uberjar/resources/localhost-target.edn new file mode 100644 index 0000000..d49f818 --- /dev/null +++ b/uberjar/resources/localhost-target.edn @@ -0,0 +1,2 @@ +{:existing [{:node-name "localhost" + :node-ip "127.0.0.1"}]} diff --git a/uberjar/resources/logback.xml b/uberjar/resources/logback.xml new file mode 100644 index 0000000..8985f2b --- /dev/null +++ b/uberjar/resources/logback.xml @@ -0,0 +1,50 @@ + + + + %d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n + + + INFO + + + + + logs/pallet.log + + logs/old/pallet.%d{yyyy-MM-dd}.log + 3 + + + %date %level [%thread] %logger{10} %msg%n + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/uberjar/src/meissa/pallet/meissa_cloud/main.clj b/uberjar/src/meissa/pallet/meissa_cloud/main.clj new file mode 100644 index 0000000..60a33f2 --- /dev/null +++ b/uberjar/src/meissa/pallet/meissa_cloud/main.clj @@ -0,0 +1,57 @@ +(ns meissa.pallet.meissa-cloud.main + (:gen-class) + (:require + [clojure.string :as str] + [clojure.tools.cli :as cli] + [dda.pallet.core.main-helper :as mh] + [dda.pallet.core.app :as core-app] + [meissa.pallet.meissa-cloud.app :as app])) + +(def cli-options + [["-h" "--help"] + ["-c" "--configure"] + ["-t" "--targets example-targets.edn" "edn file containing the targets to install on." + :default "localhost-target.edn"] + ["-v" "--verbose"]]) + +(defn usage [options-summary] + (str/join + \newline + ["meissa-cloud installs & configures a single host kubernetes cluster with Cloud installed" + "" + "Usage: java -jar meissa-cloud-standalone.jar [options] cloud.edn" + "" + "Options:" + options-summary + "" + "cloud.edn" + " - follows the edn format." + " - has to be a valid CloudConventionConfig" + ""])) + +(defn -main [& args] + (let [{:keys [options arguments errors summary help]} (cli/parse-opts args cli-options) + verbose (if (contains? options :verbose) 1 0)] + (cond + help (mh/exit 0 (usage summary)) + errors (mh/exit 1 (mh/error-msg errors)) + (not= (count arguments) 1) (mh/exit 1 (usage summary)) + (:serverspec options) (if (core-app/existing-serverspec + app/crate-app + {:convention (first arguments) + :targets (:targets options) + :verbosity verbose}) + (mh/exit-test-passed) + (mh/exit-test-failed)) + (:configure options) (if (core-app/existing-configure + app/crate-app + {:convention (first arguments) + :targets (:targets options)}) + (mh/exit-default-success) + (mh/exit-default-error)) + :default (if (core-app/existing-install + app/crate-app + {:convention (first arguments) + :targets (:targets options)}) + (mh/exit-default-success) + (mh/exit-default-error)))))