From de88a5d8075f7571cc04c167a51f7c2189079ae4 Mon Sep 17 00:00:00 2001 From: erik Date: Tue, 27 Sep 2022 13:53:04 +0200 Subject: [PATCH] [skip-ci] Make unique name for each website deploy Add function for replacing values in the beginning of a string. Add build secret. --- README.md | 4 +- .../build.py | 0 .../image/Dockerfile | 1 + .../image/resources/entrypoint.sh | 0 .../image/resources/exclude.pattern | 0 .../image/resources/functions.sh | 0 .../image/resources/install.sh | 0 .../image/resources/project.clj | 0 .../test/Dockerfile | 0 .../test/serverspec.edn | 0 src/main/cljc/dda/c4k_website/core.cljc | 8 +- src/main/cljc/dda/c4k_website/website.cljc | 76 +++++++++++++++---- src/main/resources/website/certificate.yaml | 4 +- src/main/resources/website/ingress.yaml | 6 +- .../resources/website/nginx-configmap.yaml | 2 +- .../resources/website/nginx-deployment.yaml | 18 ++--- src/main/resources/website/nginx-service.yaml | 6 +- .../resources/website/website-build-cron.yaml | 37 +++------ .../website/website-build-deployment.yaml | 70 ++++------------- .../website/website-build-secret.yaml | 8 ++ .../website/website-content-volume.yaml | 7 +- 21 files changed, 126 insertions(+), 121 deletions(-) rename infrastructure/{docker-website-build => c4k-website-build}/build.py (100%) rename infrastructure/{docker-website-build => c4k-website-build}/image/Dockerfile (78%) rename infrastructure/{docker-website-build => c4k-website-build}/image/resources/entrypoint.sh (100%) rename infrastructure/{docker-website-build => c4k-website-build}/image/resources/exclude.pattern (100%) rename infrastructure/{docker-website-build => c4k-website-build}/image/resources/functions.sh (100%) rename infrastructure/{docker-website-build => c4k-website-build}/image/resources/install.sh (100%) rename infrastructure/{docker-website-build => c4k-website-build}/image/resources/project.clj (100%) rename infrastructure/{docker-website-build => c4k-website-build}/test/Dockerfile (100%) rename infrastructure/{docker-website-build => c4k-website-build}/test/serverspec.edn (100%) create mode 100644 src/main/resources/website/website-build-secret.yaml diff --git a/README.md b/README.md index b992f17..4b11b1e 100644 --- a/README.md +++ b/README.md @@ -40,9 +40,9 @@ https://www.tecmint.com/deploy-nginx-on-a-kubernetes-cluster/ Definition von Env-Vars im Deployment: AUTHTOKEN - muss konfigurierbar sein REPOZIPURL - muss konfigurierbar sein -TARGETDIR - wird aus dem FQDN zusammengesetzt +WEBSITEROOT - hardcoded, da containerspezifisch -Ingress, Service, Certificate, Deployment, Configmap und Volume Mount sollen durch eindeutige, auf FQDN basierenden Namen verbunden sein. +Certificate, Ingress, Configmap, Deployment, Service, Volume Claim, Website-deployment sollen durch eindeutige, auf FQDN basierenden Namen verbunden sein. ## Getting started diff --git a/infrastructure/docker-website-build/build.py b/infrastructure/c4k-website-build/build.py similarity index 100% rename from infrastructure/docker-website-build/build.py rename to infrastructure/c4k-website-build/build.py diff --git a/infrastructure/docker-website-build/image/Dockerfile b/infrastructure/c4k-website-build/image/Dockerfile similarity index 78% rename from infrastructure/docker-website-build/image/Dockerfile rename to infrastructure/c4k-website-build/image/Dockerfile index 00eaedf..9236670 100644 --- a/infrastructure/docker-website-build/image/Dockerfile +++ b/infrastructure/c4k-website-build/image/Dockerfile @@ -4,6 +4,7 @@ FROM clojure:temurin-18-jammy ADD resources /tmp ENV BUILDDIR="website" +ENV WEBSITEROOT="/var/www/html/website/" RUN /tmp/install.sh RUN /tmp/entrypoint.sh diff --git a/infrastructure/docker-website-build/image/resources/entrypoint.sh b/infrastructure/c4k-website-build/image/resources/entrypoint.sh similarity index 100% rename from infrastructure/docker-website-build/image/resources/entrypoint.sh rename to infrastructure/c4k-website-build/image/resources/entrypoint.sh diff --git a/infrastructure/docker-website-build/image/resources/exclude.pattern b/infrastructure/c4k-website-build/image/resources/exclude.pattern similarity index 100% rename from infrastructure/docker-website-build/image/resources/exclude.pattern rename to infrastructure/c4k-website-build/image/resources/exclude.pattern diff --git a/infrastructure/docker-website-build/image/resources/functions.sh b/infrastructure/c4k-website-build/image/resources/functions.sh similarity index 100% rename from infrastructure/docker-website-build/image/resources/functions.sh rename to infrastructure/c4k-website-build/image/resources/functions.sh diff --git a/infrastructure/docker-website-build/image/resources/install.sh b/infrastructure/c4k-website-build/image/resources/install.sh similarity index 100% rename from infrastructure/docker-website-build/image/resources/install.sh rename to infrastructure/c4k-website-build/image/resources/install.sh diff --git a/infrastructure/docker-website-build/image/resources/project.clj b/infrastructure/c4k-website-build/image/resources/project.clj similarity index 100% rename from infrastructure/docker-website-build/image/resources/project.clj rename to infrastructure/c4k-website-build/image/resources/project.clj diff --git a/infrastructure/docker-website-build/test/Dockerfile b/infrastructure/c4k-website-build/test/Dockerfile similarity index 100% rename from infrastructure/docker-website-build/test/Dockerfile rename to infrastructure/c4k-website-build/test/Dockerfile diff --git a/infrastructure/docker-website-build/test/serverspec.edn b/infrastructure/c4k-website-build/test/serverspec.edn similarity index 100% rename from infrastructure/docker-website-build/test/serverspec.edn rename to infrastructure/c4k-website-build/test/serverspec.edn diff --git a/src/main/cljc/dda/c4k_website/core.cljc b/src/main/cljc/dda/c4k_website/core.cljc index a37e767..da7947c 100644 --- a/src/main/cljc/dda/c4k_website/core.cljc +++ b/src/main/cljc/dda/c4k_website/core.cljc @@ -7,10 +7,14 @@ (defn k8s-objects [config] (cm/concat-vec (map yaml/to-string - [(website/generate-nginx-deployment) + [(website/generate-nginx-deployment config) (website/generate-nginx-configmap config) - (website/generate-nginx-service) + (website/generate-nginx-service config) (website/generate-website-content-volume config) (website/generate-ingress config) (website/generate-certificate config) + (website/generate-website-build-cron config) + (website/generate-website-build-deployment config) + (website/generate-website-build-secret config) + ]))) diff --git a/src/main/cljc/dda/c4k_website/website.cljc b/src/main/cljc/dda/c4k_website/website.cljc index a4f8be1..8ce478d 100644 --- a/src/main/cljc/dda/c4k_website/website.cljc +++ b/src/main/cljc/dda/c4k_website/website.cljc @@ -9,7 +9,7 @@ #?(:clj [clojure.edn :as edn] :cljs [cljs.reader :as edn]) [dda.c4k-common.yaml :as yaml] - [dda.c4k-common.common :as cm] + [dda.c4k-common.common :as cm] [dda.c4k-common.predicate :as pred])) (defn domain-list? @@ -27,15 +27,29 @@ (def config? (s/keys :req-un [::fqdn] :opt-un [::issuer])) -(def auth? (s/keys :req-un [::none])) +(def auth? (s/keys :req-un [::token ::url])) (def vol? (s/keys :req-un [::volume-total-storage-size ::number-of-websites])) -(defn data-storage-by-volume-size +(defn volume-size-by-total-available-space [total number-of-websites-on-node] (m/floor (/ total number-of-websites-on-node))) ; ToDo: This might be a terrible idea +(defn unique-name-from-fqdn + [fqdn] + (st/replace fqdn #"\." "-")) + +; ToDo: Move to common +(defn-spec replace-all-matching-subvalues-in-string-start pred/map-or-seq? + [col string? + value-to-partly-match string? + value-to-inplace string?] + (clojure.walk/postwalk #(if (and (= (type value-to-partly-match) (type %)) + (re-matches (re-pattern (str value-to-partly-match ".*")) %)) + (st/replace % value-to-partly-match value-to-inplace) %) + col)) + #?(:cljs (defmethod yaml/load-resource :website [resource-name] (case resource-name @@ -45,6 +59,9 @@ "website/nginx-deployment.yaml" (rc/inline "website/nginx-deployment.yaml") "website/nginx-service.yaml" (rc/inline "website/nginx-service.yaml") "website/website-content-volume.yaml" (rc/inline "website/website-content-volume.yaml") + "website/website-build-cron.yaml" (rc/inline "website/website-build-cron.yaml") + "website/website-build-deployment.yaml" (rc/inline "website/website-build-deployment.yaml") + "website/website-build-secret.yaml" (rc/inline "website/website-build-secret.yaml") (throw (js/Error. "Undefined Resource!"))))) #?(:cljs @@ -59,6 +76,7 @@ (-> (yaml/load-as-edn "website/certificate.yaml") (assoc-in [:spec :issuerRef :name] letsencrypt-issuer) + (replace-all-matching-subvalues-in-string-start "NAME" (unique-name-from-fqdn fqdn)) (cm/replace-all-matching-values-by-new-value "FQDN" fqdn)))) (defn-spec generate-ingress pred/map-or-seq? @@ -66,30 +84,60 @@ (let [{:keys [fqdn]} config] (-> (yaml/load-as-edn "website/ingress.yaml") + (replace-all-matching-subvalues-in-string-start "NAME" (unique-name-from-fqdn fqdn)) (cm/replace-all-matching-values-by-new-value "FQDN" fqdn)))) -(defn-spec generate-nginx-configmap pred/map-or-seq? +(defn-spec generate-nginx-configmap pred/map-or-seq? [config config?] (let [{:keys [fqdn]} config - configmap (yaml/load-as-edn "website/nginx-configmap.yaml")] - (-> + configmap (yaml/load-as-edn "website/nginx-configmap.yaml")] + (-> configmap (assoc-in [:data :website.conf] (st/replace (-> configmap :data :website.conf) #"FQDN" (str fqdn ";"))) - ) - )) + (replace-all-matching-subvalues-in-string-start "NAME" (unique-name-from-fqdn fqdn))))) (defn-spec generate-nginx-deployment pred/map-or-seq? - [] - (yaml/load-as-edn "website/nginx-deployment.yaml")) + [config config?] + (let [{:keys [fqdn]} config] + (-> + (yaml/load-as-edn "website/nginx-deployment.yaml") + (replace-all-matching-subvalues-in-string-start "NAME" (unique-name-from-fqdn fqdn))))) (defn-spec generate-nginx-service pred/map-or-seq? - [] - (yaml/load-as-edn "website/nginx-service.yaml")) + [config config?] + (let [{:keys [fqdn]} config] + (-> + (yaml/load-as-edn "website/nginx-service.yaml") + (replace-all-matching-subvalues-in-string-start "NAME" (unique-name-from-fqdn fqdn))))) (defn-spec generate-website-content-volume pred/map-or-seq? [config vol?] - (let [{:keys [volume-total-storage-size number-of-websites]} config - data-storage-size (data-storage-by-volume-size volume-total-storage-size number-of-websites)] + (let [{:keys [volume-total-storage-size number-of-websites fqdn]} config + data-storage-size (volume-size-by-total-available-space volume-total-storage-size number-of-websites)] (-> (yaml/load-as-edn "website/website-content-volume.yaml") + (replace-all-matching-subvalues-in-string-start "NAME" (unique-name-from-fqdn fqdn)) (cm/replace-all-matching-values-by-new-value "WEBSITESTORAGESIZE" (str (str data-storage-size) "Gi"))))) + +(defn-spec generate-website-build-cron pred/map-or-seq? + [config config?] + (let [{:keys [fqdn]} config] + (-> + (yaml/load-as-edn "website/website-build-cron.yaml") + (replace-all-matching-subvalues-in-string-start "NAME" (unique-name-from-fqdn fqdn))))) + +(defn-spec generate-website-build-deployment pred/map-or-seq? + [config config?] + (let [{:keys [fqdn]} config] + (-> + (yaml/load-as-edn "website/website-build-deployment.yaml") + (replace-all-matching-subvalues-in-string-start "NAME" (unique-name-from-fqdn fqdn))))) + +(defn-spec generate-website-build-secret pred/map-or-seq? + [auth auth?] + (let [{:keys [fqdn token url]} config] + (-> + (yaml/load-as-edn "website/website-build-secrets.yaml") + (replace-all-matching-subvalues-in-string-start "NAME" (unique-name-from-fqdn fqdn)) + (cm/replace-all-matching-values-by-new-value "TOKEN" (b64/encode token)) + (cm/replace-all-matching-values-by-new-value "URL" (b64/encode url))))) \ No newline at end of file diff --git a/src/main/resources/website/certificate.yaml b/src/main/resources/website/certificate.yaml index a9dacca..003e8e9 100644 --- a/src/main/resources/website/certificate.yaml +++ b/src/main/resources/website/certificate.yaml @@ -1,10 +1,10 @@ apiVersion: cert-manager.io/v1 kind: Certificate metadata: - name: website-cert + name: NAME-cert namespace: default spec: - secretName: website-cert + secretName: NAME-cert commonName: FQDN duration: 2160h # 90d renewBefore: 360h # 15d diff --git a/src/main/resources/website/ingress.yaml b/src/main/resources/website/ingress.yaml index 80647ca..c1003cf 100644 --- a/src/main/resources/website/ingress.yaml +++ b/src/main/resources/website/ingress.yaml @@ -1,7 +1,7 @@ apiVersion: networking.k8s.io/v1 kind: Ingress metadata: - name: ingress-website + name: NAME-ingress namespace: default annotations: ingress.kubernetes.io/ssl-redirect: "true" @@ -10,7 +10,7 @@ spec: tls: - hosts: - FQDN - secretName: website-cert + secretName: NAME-cert rules: - host: FQDN http: @@ -19,6 +19,6 @@ spec: path: "/" backend: service: - name: nginx-service + name: NAME-service port: number: 80 diff --git a/src/main/resources/website/nginx-configmap.yaml b/src/main/resources/website/nginx-configmap.yaml index 7510a1b..6e6b6f5 100644 --- a/src/main/resources/website/nginx-configmap.yaml +++ b/src/main/resources/website/nginx-configmap.yaml @@ -1,7 +1,7 @@ apiVersion: v1 kind: ConfigMap metadata: - name: nginxconf + name: NAME-configmap namespace: default data: nginx.conf: | diff --git a/src/main/resources/website/nginx-deployment.yaml b/src/main/resources/website/nginx-deployment.yaml index 87b7cf1..bce23a3 100644 --- a/src/main/resources/website/nginx-deployment.yaml +++ b/src/main/resources/website/nginx-deployment.yaml @@ -1,19 +1,19 @@ apiVersion: apps/v1 kind: Deployment metadata: - name: nginx + name: NAME-deployment spec: replicas: 1 selector: matchLabels: - app: nginx + app: NAME-nginx template: metadata: labels: - app: nginx + app: NAME-nginx spec: containers: - - name: nginx + - name: NAME-nginx image: nginx:latest imagePullPolicy: "Always" ports: @@ -21,7 +21,7 @@ spec: volumeMounts: - mountPath: /etc/nginx readOnly: true - name: nginxconfigvol + name: nginx-config-volume - mountPath: /var/log/nginx name: log - mountPath: /var/www/html/website @@ -30,9 +30,9 @@ spec: name: website-cert readOnly: true volumes: - - name: nginxconfigvol + - name: nginx-config-volume configMap: - name: nginxconf + name: NAME-configmap items: - key: nginx.conf path: nginx.conf @@ -44,10 +44,10 @@ spec: emptyDir: {} - name: website-content-volume persistentVolumeClaim: - claimName: website-content-pvc + claimName: NAME-content-pvc - name: website-cert secret: - secretName: website-cert + secretName: NAME-cert items: - key: tls.crt path: tls.crt diff --git a/src/main/resources/website/nginx-service.yaml b/src/main/resources/website/nginx-service.yaml index 643f13a..fe87eb9 100644 --- a/src/main/resources/website/nginx-service.yaml +++ b/src/main/resources/website/nginx-service.yaml @@ -1,13 +1,13 @@ kind: Service apiVersion: v1 metadata: - name: nginx-service + name: NAME-service labels: - app: nginx + app: NAME-nginx namespace: default spec: selector: - app: nginx + app: NAME-nginx ports: - name: nginx-http port: 80 diff --git a/src/main/resources/website/website-build-cron.yaml b/src/main/resources/website/website-build-cron.yaml index 5d57e69..356c82b 100644 --- a/src/main/resources/website/website-build-cron.yaml +++ b/src/main/resources/website/website-build-cron.yaml @@ -1,7 +1,7 @@ apiVersion: batch/v1beta1 kind: CronJob metadata: - name: website-builder + name: NAME-build-cron labels: app.kubernetes.part-of: website # correct name? spec: @@ -13,35 +13,18 @@ spec: template: spec: containers: - - name: backup-app - image: domaindrivenarchitecture/c4k-gitea-backup + - image: domaindrivenarchitecture/c4k-website-build + name: NAME-build-app imagePullPolicy: IfNotPresent command: ["/entrypoint.sh"] - env: - - name: AWS_DEFAULT_REGION - value: eu-central-1 - - name: AWS_ACCESS_KEY_ID_FILE - value: /var/run/secrets/backup-secrets/aws-access-key-id - - name: AWS_SECRET_ACCESS_KEY_FILE - value: /var/run/secrets/backup-secrets/aws-secret-access-key - - name: RESTIC_REPOSITORY - valueFrom: - configMapKeyRef: - name: backup-config - key: restic-repository - - name: RESTIC_PASSWORD_FILE - value: /var/run/secrets/backup-secrets/restic-password + envFrom: + - secretRef: + name: NAME-secret volumeMounts: - - name: gitea-data-volume - mountPath: /var/backups - - name: backup-secret-volume - mountPath: /var/run/secrets/backup-secrets - readOnly: true + - name: content-volume + mountPath: /var/www/html/website volumes: - - name: gitea-data-volume + - name: content-volume persistentVolumeClaim: - claimName: gitea-data-pvc - - name: backup-secret-volume - secret: - secretName: backup-secret + claimName: NAME-content-pvc restartPolicy: OnFailure \ No newline at end of file diff --git a/src/main/resources/website/website-build-deployment.yaml b/src/main/resources/website/website-build-deployment.yaml index 79f4ec7..a1f9831 100644 --- a/src/main/resources/website/website-build-deployment.yaml +++ b/src/main/resources/website/website-build-deployment.yaml @@ -1,73 +1,33 @@ apiVersion: apps/v1 kind: Deployment metadata: - name: backup-restore + name: NAME-build-deployment spec: replicas: 0 selector: matchLabels: - app: backup-restore + app: NAME-builder strategy: type: Recreate template: metadata: labels: - app: backup-restore - app.kubernetes.io/name: backup-restore - app.kubernetes.io/part-of: gitea + app: NAME-builder + app.kubernetes.io/name: NAME-builder + app.kubernetes.io/part-of: website spec: containers: - - image: domaindrivenarchitecture/c4k-gitea-backup - name: backup-app + - image: domaindrivenarchitecture/c4k-website-build + name: NAME-build-app imagePullPolicy: IfNotPresent - command: ["/entrypoint-start-and-wait.sh"] - env: - - name: POSTGRES_USER - valueFrom: - secretKeyRef: - name: postgres-secret - key: postgres-user - - name: POSTGRES_PASSWORD - valueFrom: - secretKeyRef: - name: postgres-secret - key: postgres-password - - name: POSTGRES_DB - valueFrom: - configMapKeyRef: - name: postgres-config - key: postgres-db - - name: POSTGRES_HOST - value: "postgresql-service:5432" - - name: POSTGRES_SERVICE - value: "postgresql-service" - - name: POSTGRES_PORT - value: "5432" - - name: AWS_DEFAULT_REGION - value: eu-central-1 - - name: AWS_ACCESS_KEY_ID_FILE - value: /var/run/secrets/backup-secrets/aws-access-key-id - - name: AWS_SECRET_ACCESS_KEY_FILE - value: /var/run/secrets/backup-secrets/aws-secret-access-key - - name: RESTIC_REPOSITORY - valueFrom: - configMapKeyRef: - name: backup-config - key: restic-repository - - name: RESTIC_PASSWORD_FILE - value: /var/run/secrets/backup-secrets/restic-password - - name: CERTIFICATE_FILE - value: "" + command: ["/entrypoint.sh"] + envFrom: + - secretRef: + name: NAME-secret volumeMounts: - - name: gitea-data-volume - mountPath: /var/backups - - name: backup-secret-volume - mountPath: /var/run/secrets/backup-secrets - readOnly: true + - name: content-volume + mountPath: /var/www/html/website volumes: - - name: gitea-data-volume + - name: content-volume persistentVolumeClaim: - claimName: gitea-data-pvc - - name: backup-secret-volume - secret: - secretName: backup-secret \ No newline at end of file + claimName: NAME-content-pvc diff --git a/src/main/resources/website/website-build-secret.yaml b/src/main/resources/website/website-build-secret.yaml new file mode 100644 index 0000000..8a57cdb --- /dev/null +++ b/src/main/resources/website/website-build-secret.yaml @@ -0,0 +1,8 @@ +apiVersion: v1 +kind: Secret +metadata: + name: NAME-secret +data: + AUTHTOKEN: TOKEN + REPOZIPURL: URL + \ No newline at end of file diff --git a/src/main/resources/website/website-content-volume.yaml b/src/main/resources/website/website-content-volume.yaml index 4f44a1f..1589ea4 100644 --- a/src/main/resources/website/website-content-volume.yaml +++ b/src/main/resources/website/website-content-volume.yaml @@ -1,14 +1,15 @@ apiVersion: v1 kind: PersistentVolumeClaim metadata: - name: website-content-pvc + name: NAME-content-volume namespace: default labels: - app: nginx + app: NAME-nginx spec: storageClassName: local-path accessModes: - ReadWriteOnce resources: requests: - storage: WEBSITESTORAGESIZE \ No newline at end of file + storage: WEBSITESTORAGESIZE + \ No newline at end of file