From 76f5b87ec9a182dd7641d81151bdf42f5bf5c6dc Mon Sep 17 00:00:00 2001 From: erik Date: Fri, 23 Sep 2022 15:39:17 +0200 Subject: [PATCH] [skip-ci] WIP Add docker image Add website-build-deployment and cron skeleton. Add shadow-cljs.edn - needs config for cljs. --- README.md | 49 +++---------- infrastructure/docker-website-build/build.py | 51 +++++++++++++ .../docker-website-build/image/Dockerfile | 9 +++ .../image/resources/entrypoint.sh | 15 ++++ .../image/resources/exclude.pattern | 2 + .../image/resources/functions.sh | 18 +++++ .../image/resources/install.sh | 10 +++ .../image/resources/project.clj | 2 + .../docker-website-build/test/Dockerfile | 11 +++ .../docker-website-build/test/serverspec.edn | 5 ++ shadow-cljs.edn | 16 ++++ .../resources/website/nginx-configmap.yaml | 4 +- .../resources/website/website-build-cron.yaml | 47 ++++++++++++ .../website/website-build-deployment.yaml | 73 +++++++++++++++++++ 14 files changed, 274 insertions(+), 38 deletions(-) create mode 100644 infrastructure/docker-website-build/build.py create mode 100644 infrastructure/docker-website-build/image/Dockerfile create mode 100755 infrastructure/docker-website-build/image/resources/entrypoint.sh create mode 100644 infrastructure/docker-website-build/image/resources/exclude.pattern create mode 100644 infrastructure/docker-website-build/image/resources/functions.sh create mode 100755 infrastructure/docker-website-build/image/resources/install.sh create mode 100644 infrastructure/docker-website-build/image/resources/project.clj create mode 100644 infrastructure/docker-website-build/test/Dockerfile create mode 100644 infrastructure/docker-website-build/test/serverspec.edn create mode 100644 shadow-cljs.edn create mode 100644 src/main/resources/website/website-build-cron.yaml create mode 100644 src/main/resources/website/website-build-deployment.yaml diff --git a/README.md b/README.md index cfbffef..b992f17 100644 --- a/README.md +++ b/README.md @@ -2,10 +2,6 @@ ## Requirements -Unsere Website soll mit k8s laufen. - -Unsere erzeugten images incl. Web-Inhalt sollen nicht öffentlich zugänglich sein. - A.C.: wir haben ein konzept erstellt, wie wir statische Inhalte ausliefern (git-pull & cron // cicid // manueller pybuilder-run). @@ -15,33 +11,18 @@ A.C.: c4k holt sich stets die neuesten website-container unterbrechungsfrei. - unsere website läuft mit dem neuen setup und ist unter ipv4 und ipv6 erreichbar. - website ist unter 1 oder 2 fqdns erreichbar. + Unsere website läuft mit dem neuen Setup, ist unter ipv4/v6 und unter 1 oder 2 FQDNs erreichbar. ## Was passiert? -_Frage_ Sollen später Webserver auf den gleichen, laufenden Cluster hinzugefügt werden können? +Webserver sollen auf den gleichen, laufenden Cluster hinzugefügt werden können. -**Ja:** - -Ein Cluster, 1 Pod für eine Website (WS) --> mehrere FQDNs müssen auf einen Cluster Zeigen --> viele Pods für viele WS --> Package einen Webserver mit einer Website --> DNS Routen so einrichten, dass verschiedene FQDNs auf selbe cluster-id zeigen --> Ingress muss die angefragten FQDNS auflösen und weiterleiten - -Anforderungen an Website-Container: -nginx-deployment -nginx-config -> wie -serviceType: LoadBalancer -PVC + volume-mount zeigt auf website-data-vol +**c4k** +Pro Website je: Ingress -> Service -> Webserver Pod -> Volume Mount -> PVC. +Ingress muss die angefragten FQDNS auflösen und weiterleiten. **Terraform:** -FQDNs für verschiedene Websites hinzufügen? -Verbindung mit c4k? -c4k-Modulname als Teil des FQDNs wird später noch problematisch. - +Mehrere DNS Einträge auf Terraform Seite für einen Cluster. ### Reading: @@ -54,20 +35,14 @@ Deploying Nginx: https://kubernetes.io/docs/tasks/run-application/run-stateless-application-deployment/ https://www.tecmint.com/deploy-nginx-on-a-kubernetes-cluster/ +## +Definition von Env-Vars im Deployment: +AUTHTOKEN - muss konfigurierbar sein +REPOZIPURL - muss konfigurierbar sein +TARGETDIR - wird aus dem FQDN zusammengesetzt -## Struktur - -1. Baue die statische website - -2. Schieb die Website auf den Server - -3. Liefere die Website mit einem Webserver aus - -## Fragen - -Soll Cryogen auch in einem container laufen? - +Ingress, Service, Certificate, Deployment, Configmap und Volume Mount sollen durch eindeutige, auf FQDN basierenden Namen verbunden sein. ## Getting started diff --git a/infrastructure/docker-website-build/build.py b/infrastructure/docker-website-build/build.py new file mode 100644 index 0000000..b42b47c --- /dev/null +++ b/infrastructure/docker-website-build/build.py @@ -0,0 +1,51 @@ +from os import environ +from pybuilder.core import task, init +from ddadevops import * +import logging + +name = 'c4k-website-build' +MODULE = 'docker' +PROJECT_ROOT_PATH = '../..' + +class MyBuild(DevopsDockerBuild): + pass + +@init +def initialize(project): + project.build_depends_on('ddadevops>=0.12.4') + stage = 'prod' + dockerhub_user = environ.get('DOCKERHUB_USER') + if not dockerhub_user: + dockerhub_user = gopass_field_from_path('meissa/web/docker.com', 'login') + dockerhub_password = environ.get('DOCKERHUB_PASSWORD') + if not dockerhub_password: + dockerhub_password = gopass_password_from_path('meissa/web/docker.com') + tag = environ.get('CI_COMMIT_TAG') + if not tag: + tag = get_tag_from_latest_commit() + config = create_devops_docker_build_config( + stage, PROJECT_ROOT_PATH, MODULE, dockerhub_user, dockerhub_password, docker_publish_tag=tag) + build = MyBuild(project, config) + build.initialize_build_dir() + + +@task +def image(project): + build = get_devops_build(project) + build.image() + +@task +def drun(project): + build = get_devops_build(project) + build.drun() + +@task +def publish(project): + build = get_devops_build(project) + build.dockerhub_login() + build.dockerhub_publish() + +@task +def test(project): + build = get_devops_build(project) + build.test() diff --git a/infrastructure/docker-website-build/image/Dockerfile b/infrastructure/docker-website-build/image/Dockerfile new file mode 100644 index 0000000..00eaedf --- /dev/null +++ b/infrastructure/docker-website-build/image/Dockerfile @@ -0,0 +1,9 @@ +FROM clojure:temurin-18-jammy + +# Prepare Entrypoint Script +ADD resources /tmp + +ENV BUILDDIR="website" + +RUN /tmp/install.sh +RUN /tmp/entrypoint.sh diff --git a/infrastructure/docker-website-build/image/resources/entrypoint.sh b/infrastructure/docker-website-build/image/resources/entrypoint.sh new file mode 100755 index 0000000..4a4da8e --- /dev/null +++ b/infrastructure/docker-website-build/image/resources/entrypoint.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +source /usr/local/bin/functions.sh + +function main() { + get-and-unzip-website-data + build-and-extract-website + move-website-files-to-target +} + +main + +while true; do + sleep 1m +done diff --git a/infrastructure/docker-website-build/image/resources/exclude.pattern b/infrastructure/docker-website-build/image/resources/exclude.pattern new file mode 100644 index 0000000..3978a0f --- /dev/null +++ b/infrastructure/docker-website-build/image/resources/exclude.pattern @@ -0,0 +1,2 @@ +.git +.gitignore diff --git a/infrastructure/docker-website-build/image/resources/functions.sh b/infrastructure/docker-website-build/image/resources/functions.sh new file mode 100644 index 0000000..fac8337 --- /dev/null +++ b/infrastructure/docker-website-build/image/resources/functions.sh @@ -0,0 +1,18 @@ +#!/bin/bash + +# "Authorization: token YOURAUTHTOKEN" +function get-and-unzip-website-data() { + curl -H "$AUTHTOKEN" -O $REPOZIPURL # REPOZIPURL = https://your.gitea.host/api/v1/repos///archive/main.zip + mkdir $BUILDDIR + unzip main.zip -D $BUILDDIR +} + +function build-and-extract-website() { + (cd $BUILDDIR; lein ring war; websiteartifactname=$(ls | grep -o *.war); unzip target/$websiteartifactname "WEB-INF/classes/public/*") +} + +# set variables from environment +# read write zugriff sicherstellen +function move-website-files-to-target() { + rsync -ru --exclude-from "/home/$USER/exclude.pattern" --delete WEB-INF/classes/public/* $TARGETDIR # TARGETDIR = mount/path/to/website-content-vol with write permission +} diff --git a/infrastructure/docker-website-build/image/resources/install.sh b/infrastructure/docker-website-build/image/resources/install.sh new file mode 100755 index 0000000..903b473 --- /dev/null +++ b/infrastructure/docker-website-build/image/resources/install.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +apt update > /dev/null; + +install -m 0700 /tmp/entrypoint.sh / + +install -m 0700 /tmp/functions.sh /usr/local/bin/ +install -m 0700 /tmp/exclude.pattern /home/$USER +install -m 0700 /tmp/project.clj /home/$USER/.lein/ +(cd /home/$USER/.lein; lein deps) diff --git a/infrastructure/docker-website-build/image/resources/project.clj b/infrastructure/docker-website-build/image/resources/project.clj new file mode 100644 index 0000000..8f5e9a0 --- /dev/null +++ b/infrastructure/docker-website-build/image/resources/project.clj @@ -0,0 +1,2 @@ +{:user + {:plugins [[lein-ring "0.12.5"]]}} \ No newline at end of file diff --git a/infrastructure/docker-website-build/test/Dockerfile b/infrastructure/docker-website-build/test/Dockerfile new file mode 100644 index 0000000..f2e19b6 --- /dev/null +++ b/infrastructure/docker-website-build/test/Dockerfile @@ -0,0 +1,11 @@ +FROM c4k-jira-backup + +RUN apt update +RUN apt -yqq --no-install-recommends --yes install curl default-jre-headless + +RUN curl -L -o /tmp/serverspec.jar \ + https://github.com/DomainDrivenArchitecture/dda-serverspec-crate/releases/download/2.0.0/dda-serverspec-standalone.jar + +COPY serverspec.edn /tmp/serverspec.edn + +RUN java -jar /tmp/serverspec.jar /tmp/serverspec.edn -v \ No newline at end of file diff --git a/infrastructure/docker-website-build/test/serverspec.edn b/infrastructure/docker-website-build/test/serverspec.edn new file mode 100644 index 0000000..000622f --- /dev/null +++ b/infrastructure/docker-website-build/test/serverspec.edn @@ -0,0 +1,5 @@ +{:file [{:path "/usr/local/bin/install.sh" :mod "700"} + {:path "/usr/local/bin/functions.sh" :mod "700"} + {:path "/usr/local/bin/exclude.pattern" :mod "700"} + {:path "/usr/local/bin/project.clj" :mod "700"} + {:path "/entrypoint.sh" :mod "700"}]} diff --git a/shadow-cljs.edn b/shadow-cljs.edn new file mode 100644 index 0000000..ed56bf2 --- /dev/null +++ b/shadow-cljs.edn @@ -0,0 +1,16 @@ +{:source-paths ["src/main/cljc" + "src/main/cljs" + "src/main/resources" + "src/test/cljc" + "src/test/cljs" + "src/test/resources"] + :dependencies [[org.domaindrivenarchitecture/c4k-common-cljs "3.0.1"] + [hickory "0.7.1"] + [org.clojure/math.numeric-tower "0.0.5"]] + :builds {:frontend {:target :browser + :modules {:main {:init-fn dda.c4k-website.browser/init}} + :release {} + :compiler-options {:optimizations :advanced}} + :test {:target :node-test + :output-to "target/node-tests.js" + :repl-pprint true}}} \ No newline at end of file diff --git a/src/main/resources/website/nginx-configmap.yaml b/src/main/resources/website/nginx-configmap.yaml index 6ea0179..7510a1b 100644 --- a/src/main/resources/website/nginx-configmap.yaml +++ b/src/main/resources/website/nginx-configmap.yaml @@ -95,5 +95,7 @@ data: root /var/www/html/website/; # root /usr/share/nginx/html/; # testing purposes index index.html; - try_files $uri /index.html; + location / { + try_files $uri $uri/ /index.html =404; + } } \ No newline at end of file diff --git a/src/main/resources/website/website-build-cron.yaml b/src/main/resources/website/website-build-cron.yaml new file mode 100644 index 0000000..5d57e69 --- /dev/null +++ b/src/main/resources/website/website-build-cron.yaml @@ -0,0 +1,47 @@ +apiVersion: batch/v1beta1 +kind: CronJob +metadata: + name: website-builder + labels: + app.kubernetes.part-of: website # correct name? +spec: + schedule: "10 23 * * *" + successfulJobsHistoryLimit: 1 + failedJobsHistoryLimit: 1 + jobTemplate: + spec: + template: + spec: + containers: + - name: backup-app + image: domaindrivenarchitecture/c4k-gitea-backup + imagePullPolicy: IfNotPresent + command: ["/entrypoint.sh"] + env: + - name: AWS_DEFAULT_REGION + value: eu-central-1 + - name: AWS_ACCESS_KEY_ID_FILE + value: /var/run/secrets/backup-secrets/aws-access-key-id + - name: AWS_SECRET_ACCESS_KEY_FILE + value: /var/run/secrets/backup-secrets/aws-secret-access-key + - name: RESTIC_REPOSITORY + valueFrom: + configMapKeyRef: + name: backup-config + key: restic-repository + - name: RESTIC_PASSWORD_FILE + value: /var/run/secrets/backup-secrets/restic-password + volumeMounts: + - name: gitea-data-volume + mountPath: /var/backups + - name: backup-secret-volume + mountPath: /var/run/secrets/backup-secrets + readOnly: true + volumes: + - name: gitea-data-volume + persistentVolumeClaim: + claimName: gitea-data-pvc + - name: backup-secret-volume + secret: + secretName: backup-secret + restartPolicy: OnFailure \ No newline at end of file diff --git a/src/main/resources/website/website-build-deployment.yaml b/src/main/resources/website/website-build-deployment.yaml new file mode 100644 index 0000000..79f4ec7 --- /dev/null +++ b/src/main/resources/website/website-build-deployment.yaml @@ -0,0 +1,73 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: backup-restore +spec: + replicas: 0 + selector: + matchLabels: + app: backup-restore + strategy: + type: Recreate + template: + metadata: + labels: + app: backup-restore + app.kubernetes.io/name: backup-restore + app.kubernetes.io/part-of: gitea + spec: + containers: + - image: domaindrivenarchitecture/c4k-gitea-backup + name: backup-app + imagePullPolicy: IfNotPresent + command: ["/entrypoint-start-and-wait.sh"] + env: + - name: POSTGRES_USER + valueFrom: + secretKeyRef: + name: postgres-secret + key: postgres-user + - name: POSTGRES_PASSWORD + valueFrom: + secretKeyRef: + name: postgres-secret + key: postgres-password + - name: POSTGRES_DB + valueFrom: + configMapKeyRef: + name: postgres-config + key: postgres-db + - name: POSTGRES_HOST + value: "postgresql-service:5432" + - name: POSTGRES_SERVICE + value: "postgresql-service" + - name: POSTGRES_PORT + value: "5432" + - name: AWS_DEFAULT_REGION + value: eu-central-1 + - name: AWS_ACCESS_KEY_ID_FILE + value: /var/run/secrets/backup-secrets/aws-access-key-id + - name: AWS_SECRET_ACCESS_KEY_FILE + value: /var/run/secrets/backup-secrets/aws-secret-access-key + - name: RESTIC_REPOSITORY + valueFrom: + configMapKeyRef: + name: backup-config + key: restic-repository + - name: RESTIC_PASSWORD_FILE + value: /var/run/secrets/backup-secrets/restic-password + - name: CERTIFICATE_FILE + value: "" + volumeMounts: + - name: gitea-data-volume + mountPath: /var/backups + - name: backup-secret-volume + mountPath: /var/run/secrets/backup-secrets + readOnly: true + volumes: + - name: gitea-data-volume + persistentVolumeClaim: + claimName: gitea-data-pvc + - name: backup-secret-volume + secret: + secretName: backup-secret \ No newline at end of file