From 13218b395ea3e3de6817392dc3232f6bc909b954 Mon Sep 17 00:00:00 2001 From: Mirco Date: Thu, 21 Dec 2023 19:56:34 +0100 Subject: [PATCH 01/12] Improvements docker image building --- infrastructure/backup/image/Dockerfile | 2 +- .../backup/image/resources/install.sh | 24 ++++++++++++------- infrastructure/backup/test/Dockerfile | 11 --------- 3 files changed, 17 insertions(+), 20 deletions(-) delete mode 100644 infrastructure/backup/test/Dockerfile diff --git a/infrastructure/backup/image/Dockerfile b/infrastructure/backup/image/Dockerfile index a58ad24..e6e11b4 100644 --- a/infrastructure/backup/image/Dockerfile +++ b/infrastructure/backup/image/Dockerfile @@ -1,4 +1,4 @@ -FROM domaindrivenarchitecture/dda-backup:1.0.10 +FROM domaindrivenarchitecture/dda-backup:latest # Prepare Entrypoint Script ADD resources /tmp diff --git a/infrastructure/backup/image/resources/install.sh b/infrastructure/backup/image/resources/install.sh index b366fbf..f9c0484 100755 --- a/infrastructure/backup/image/resources/install.sh +++ b/infrastructure/backup/image/resources/install.sh @@ -1,13 +1,21 @@ #!/bin/bash -set -eux pipefail +set -exo pipefail -apt-get update > /dev/null; +function main() +{ + { + install -m 0700 /tmp/entrypoint.sh / + install -m 0700 /tmp/entrypoint-start-and-wait.sh / -install -m 0700 /tmp/entrypoint.sh / -install -m 0700 /tmp/entrypoint-start-and-wait.sh / + install -m 0700 /tmp/init.sh /usr/local/bin/ + install -m 0700 /tmp/backup.sh /usr/local/bin/ + install -m 0700 /tmp/restore.sh /usr/local/bin/ + install -m 0700 /tmp/restic-snapshots.sh /usr/local/bin/ + + cleanupDocker + } > /dev/null +} -install -m 0700 /tmp/init.sh /usr/local/bin/ -install -m 0700 /tmp/backup.sh /usr/local/bin/ -install -m 0700 /tmp/restore.sh /usr/local/bin/ -install -m 0700 /tmp/restic-snapshots.sh /usr/local/bin/ +source /tmp/install_functions_debian.sh +DEBIAN_FRONTEND=noninteractive DEBCONF_NOWARNINGS=yes main diff --git a/infrastructure/backup/test/Dockerfile b/infrastructure/backup/test/Dockerfile deleted file mode 100644 index 667bca5..0000000 --- a/infrastructure/backup/test/Dockerfile +++ /dev/null @@ -1,11 +0,0 @@ -FROM c4k-taiga-backup - -RUN apt update -RUN apt -yqq --no-install-recommends --yes install curl default-jre-headless - -RUN curl -L -o /tmp/serverspec.jar \ - https://github.com/DomainDrivenArchitecture/dda-serverspec-crate/releases/download/2.0.0/dda-serverspec-standalone.jar - -COPY serverspec.edn /tmp/serverspec.edn - -RUN java -jar /tmp/serverspec.jar /tmp/serverspec.edn -v \ No newline at end of file From 568262760676233745f4ab5da8c60386d3b4eb02 Mon Sep 17 00:00:00 2001 From: Mirco Date: Thu, 21 Dec 2023 21:17:14 +0100 Subject: [PATCH 02/12] lift versions ddadevops dind/clj/clj-cljs --- .gitlab-ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index dd2bddb..5894a77 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -6,7 +6,7 @@ stages: - image .img: &img - image: "domaindrivenarchitecture/ddadevops-dind:4.9.0" + image: "domaindrivenarchitecture/ddadevops-dind:4.10.5" services: - docker:dind before_script: @@ -16,7 +16,7 @@ stages: - export IMAGE_TAG=$CI_COMMIT_TAG .cljs-job: &cljs - image: "domaindrivenarchitecture/ddadevops-clj-cljs:4.9.0" + image: "domaindrivenarchitecture/ddadevops-clj-cljs:4.10.5" cache: key: ${CI_COMMIT_REF_SLUG} paths: @@ -29,7 +29,7 @@ stages: - npm install .clj-job: &clj - image: "domaindrivenarchitecture/ddadevops-clj-cljs:4.9.0" + image: "domaindrivenarchitecture/ddadevops-clj-cljs:4.10.5" cache: key: ${CI_COMMIT_REF_SLUG} paths: From c76f7e58f0d92bb0833b44bcb22793383212584a Mon Sep 17 00:00:00 2001 From: Mirco Date: Fri, 22 Dec 2023 16:55:49 +0100 Subject: [PATCH 03/12] [skip-ci] infrastructure/../test folder removed --- infrastructure/backup/test/serverspec.edn | 6 ------ 1 file changed, 6 deletions(-) delete mode 100644 infrastructure/backup/test/serverspec.edn diff --git a/infrastructure/backup/test/serverspec.edn b/infrastructure/backup/test/serverspec.edn deleted file mode 100644 index 09623c7..0000000 --- a/infrastructure/backup/test/serverspec.edn +++ /dev/null @@ -1,6 +0,0 @@ -{:file [{:path "/usr/local/bin/init.sh" :mod "700"} - {:path "/usr/local/bin/backup.sh" :mod "700"} - {:path "/usr/local/bin/restore.sh" :mod "700"} - {:path "/usr/local/bin/restic-snapshots.sh" :mod "700"} - {:path "/entrypoint.sh" :mod "700"} - {:path "/entrypoint-start-and-wait.sh" :mod "700"}]} From 3ad9937ff4350dc32cad6692ba0cbbdb27a0c9d5 Mon Sep 17 00:00:00 2001 From: bom Date: Fri, 19 Jan 2024 13:20:47 +0100 Subject: [PATCH 04/12] Bump versions --- .gitlab-ci.yml | 6 +++--- project.clj | 8 ++++---- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 5894a77..3ea3b69 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -6,7 +6,7 @@ stages: - image .img: &img - image: "domaindrivenarchitecture/ddadevops-dind:4.10.5" + image: "domaindrivenarchitecture/ddadevops-dind:4.10.7" services: - docker:dind before_script: @@ -16,7 +16,7 @@ stages: - export IMAGE_TAG=$CI_COMMIT_TAG .cljs-job: &cljs - image: "domaindrivenarchitecture/ddadevops-clj-cljs:4.10.5" + image: "domaindrivenarchitecture/ddadevops-clj-cljs:4.10.7" cache: key: ${CI_COMMIT_REF_SLUG} paths: @@ -29,7 +29,7 @@ stages: - npm install .clj-job: &clj - image: "domaindrivenarchitecture/ddadevops-clj-cljs:4.10.5" + image: "domaindrivenarchitecture/ddadevops-clj-cljs:4.10.7" cache: key: ${CI_COMMIT_REF_SLUG} paths: diff --git a/project.clj b/project.clj index 249d4dc..4f420f2 100644 --- a/project.clj +++ b/project.clj @@ -4,8 +4,8 @@ :license {:name "Apache License, Version 2.0" :url "https://www.apache.org/licenses/LICENSE-2.0.html"} :dependencies [[org.clojure/clojure "1.11.1"] - [org.clojure/tools.reader "1.3.6"] - [org.domaindrivenarchitecture/c4k-common-clj "6.1.0"] + [org.clojure/tools.reader "1.3.7"] + [org.domaindrivenarchitecture/c4k-common-clj "6.1.2"] [hickory "0.7.1" :exclusions [viebel/codox-klipse-theme]]] :target-path "target/%s/" :source-paths ["src/main/cljc" @@ -23,9 +23,9 @@ :main dda.c4k-taiga.uberjar :uberjar-name "c4k-taiga-standalone.jar" :dependencies [[org.clojure/tools.cli "1.0.219"] - [ch.qos.logback/logback-classic "1.4.11" + [ch.qos.logback/logback-classic "1.4.14" :exclusions [com.sun.mail/javax.mail]] - [org.slf4j/jcl-over-slf4j "2.0.9"]]}} + [org.slf4j/jcl-over-slf4j "2.0.11"]]}} :release-tasks [["test"] ["vcs" "assert-committed"] ["change" "version" "leiningen.release/bump-version" "release"] From dfb5fd8f08395054d51dfa26bb764018c9a1ad66 Mon Sep 17 00:00:00 2001 From: bom Date: Fri, 19 Jan 2024 13:21:35 +0100 Subject: [PATCH 05/12] release: 1.1.2 --- package.json | 2 +- project.clj | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/package.json b/package.json index 861ae56..3d5002b 100644 --- a/package.json +++ b/package.json @@ -2,7 +2,7 @@ "name": "c4k-taiga", "description": "Generate c4k yaml for a taiga project management deployment.", "author": "meissa GmbH", - "version": "1.1.2-SNAPSHOT", + "version": "1.1.2", "homepage": "https://gitlab.com/domaindrivenarchitecture/c4k-taiga#readme", "repository": "https://www.npmjs.com/package/c4k-taiga", "license": "APACHE2", diff --git a/project.clj b/project.clj index 4f420f2..034085e 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject org.domaindrivenarchitecture/c4k-taiga "1.1.2-SNAPSHOT" +(defproject org.domaindrivenarchitecture/c4k-taiga "1.1.2" :description "taiga c4k-installation package" :url "https://domaindrivenarchitecture.org" :license {:name "Apache License, Version 2.0" From 671c12b650bd97cce79ce0f6fc9de681ee3c1684 Mon Sep 17 00:00:00 2001 From: bom Date: Fri, 19 Jan 2024 13:21:35 +0100 Subject: [PATCH 06/12] bump version to: 1.1.3-SNAPSHOT --- package.json | 2 +- project.clj | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/package.json b/package.json index 3d5002b..74abe2c 100644 --- a/package.json +++ b/package.json @@ -2,7 +2,7 @@ "name": "c4k-taiga", "description": "Generate c4k yaml for a taiga project management deployment.", "author": "meissa GmbH", - "version": "1.1.2", + "version": "1.1.3-SNAPSHOT", "homepage": "https://gitlab.com/domaindrivenarchitecture/c4k-taiga#readme", "repository": "https://www.npmjs.com/package/c4k-taiga", "license": "APACHE2", diff --git a/project.clj b/project.clj index 034085e..c92762e 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject org.domaindrivenarchitecture/c4k-taiga "1.1.2" +(defproject org.domaindrivenarchitecture/c4k-taiga "1.1.3-SNAPSHOT" :description "taiga c4k-installation package" :url "https://domaindrivenarchitecture.org" :license {:name "Apache License, Version 2.0" From 424a7bee74a4180272f265c930cde4c3108585f4 Mon Sep 17 00:00:00 2001 From: bom Date: Fri, 23 Feb 2024 16:11:22 +0100 Subject: [PATCH 07/12] Refactor common yaml loading --- src/main/cljc/dda/c4k_taiga/backup.cljc | 8 ++--- src/main/cljc/dda/c4k_taiga/taiga.cljc | 46 ++++++++++++------------- 2 files changed, 27 insertions(+), 27 deletions(-) diff --git a/src/main/cljc/dda/c4k_taiga/backup.cljc b/src/main/cljc/dda/c4k_taiga/backup.cljc index 50bb114..3d8cba3 100644 --- a/src/main/cljc/dda/c4k_taiga/backup.cljc +++ b/src/main/cljc/dda/c4k_taiga/backup.cljc @@ -24,14 +24,14 @@ (defn generate-config [my-conf] (let [{:keys [restic-repository]} my-conf] (-> - (yaml/from-string (yaml/load-resource "backup/config.yaml")) + (yaml/load-as-edn "backup/config.yaml") (cm/replace-key-value :restic-repository restic-repository)))) (defn generate-cron [] - (yaml/from-string (yaml/load-resource "backup/cron.yaml"))) + (yaml/load-as-edn "backup/cron.yaml")) (defn generate-backup-restore-deployment [my-conf] - (let [backup-restore-yaml (yaml/from-string (yaml/load-resource "backup/backup-restore-deployment.yaml"))] + (let [backup-restore-yaml (yaml/load-as-edn "backup/backup-restore-deployment.yaml")] (if (and (contains? my-conf :local-integration-test) (= true (:local-integration-test my-conf))) (cm/replace-named-value backup-restore-yaml "CERTIFICATE_FILE" "/var/run/secrets/localstack-secrets/ca.crt") backup-restore-yaml))) @@ -39,7 +39,7 @@ (defn generate-secret [my-auth] (let [{:keys [aws-access-key-id aws-secret-access-key restic-password]} my-auth] (-> - (yaml/from-string (yaml/load-resource "backup/secret.yaml")) + (yaml/load-as-edn "backup/secret.yaml") (cm/replace-key-value :aws-access-key-id (b64/encode aws-access-key-id)) (cm/replace-key-value :aws-secret-access-key (b64/encode aws-secret-access-key)) (cm/replace-key-value :restic-password (b64/encode restic-password))))) diff --git a/src/main/cljc/dda/c4k_taiga/taiga.cljc b/src/main/cljc/dda/c4k_taiga/taiga.cljc index 8ca3f6c..4ba018f 100644 --- a/src/main/cljc/dda/c4k_taiga/taiga.cljc +++ b/src/main/cljc/dda/c4k_taiga/taiga.cljc @@ -112,60 +112,60 @@ config)))) (defn-spec generate-async-deployment cp/map-or-seq? [] - (yaml/from-string (yaml/load-resource "taiga/async-deployment.yaml"))) + (yaml/load-as-edn "taiga/async-deployment.yaml")) (defn-spec generate-async-service cp/map-or-seq? [] - (yaml/from-string (yaml/load-resource "taiga/async-service.yaml"))) + (yaml/load-as-edn "taiga/async-service.yaml")) (defn-spec generate-async-rabbitmq-deployment cp/map-or-seq? [] - (yaml/from-string (yaml/load-resource "taiga/async-rabbitmq-deployment.yaml"))) + (yaml/load-as-edn "taiga/async-rabbitmq-deployment.yaml")) (defn-spec generate-events-rabbitmq-service cp/map-or-seq? [] - (yaml/from-string (yaml/load-resource "taiga/events-rabbitmq-service.yaml"))) + (yaml/load-as-edn "taiga/events-rabbitmq-service.yaml")) (defn-spec generate-async-rabbitmq-service cp/map-or-seq? [] - (yaml/from-string (yaml/load-resource "taiga/async-rabbitmq-service.yaml"))) + (yaml/load-as-edn "taiga/async-rabbitmq-service.yaml")) (defn-spec generate-back-deployment cp/map-or-seq? [] - (yaml/from-string (yaml/load-resource "taiga/back-deployment.yaml"))) + (yaml/load-as-edn "taiga/back-deployment.yaml")) (defn-spec generate-back-service cp/map-or-seq? [] - (yaml/from-string (yaml/load-resource "taiga/back-service.yaml"))) + (yaml/load-as-edn "taiga/back-service.yaml")) (defn-spec generate-events-rabbitmq-deployment cp/map-or-seq? [] - (yaml/from-string (yaml/load-resource "taiga/events-rabbitmq-deployment.yaml"))) + (yaml/load-as-edn "taiga/events-rabbitmq-deployment.yaml")) (defn-spec generate-events-deployment cp/map-or-seq? [] - (yaml/from-string (yaml/load-resource "taiga/events-deployment.yaml"))) + (yaml/load-as-edn "taiga/events-deployment.yaml")) (defn-spec generate-events-service cp/map-or-seq? [] - (yaml/from-string (yaml/load-resource "taiga/events-service.yaml"))) + (yaml/load-as-edn "taiga/events-service.yaml")) (defn-spec generate-front-deployment cp/map-or-seq? [] - (yaml/from-string (yaml/load-resource "taiga/front-deployment.yaml"))) + (yaml/load-as-edn "taiga/front-deployment.yaml")) (defn-spec generate-front-service cp/map-or-seq? [] - (yaml/from-string (yaml/load-resource "taiga/front-service.yaml"))) + (yaml/load-as-edn "taiga/front-service.yaml")) (defn-spec generate-gateway-configmap cp/map-or-seq? [] - (yaml/from-string (yaml/load-resource "taiga/gateway-configmap.yaml"))) + (yaml/load-as-edn "taiga/gateway-configmap.yaml")) (defn-spec generate-gateway-deployment cp/map-or-seq? [] - (yaml/from-string (yaml/load-resource "taiga/gateway-deployment.yaml"))) + (yaml/load-as-edn "taiga/gateway-deployment.yaml")) (defn-spec generate-gateway-service cp/map-or-seq? [] - (yaml/from-string (yaml/load-resource "taiga/gateway-service.yaml"))) + (yaml/load-as-edn "taiga/gateway-service.yaml")) (defn-spec generate-protected-deployment cp/map-or-seq? [] - (yaml/from-string (yaml/load-resource "taiga/protected-deployment.yaml"))) + (yaml/load-as-edn "taiga/protected-deployment.yaml")) (defn-spec generate-protected-service cp/map-or-seq? [] - (yaml/from-string (yaml/load-resource "taiga/protected-service.yaml"))) + (yaml/load-as-edn "taiga/protected-service.yaml")) (defn-spec generate-configmap cp/map-or-seq? [config config?] (let [{:keys [fqdn enable-telemetry public-register-enabled]} (merge config-defaults config)] - (-> (yaml/load-as-edn "taiga/configmap.yaml") + (-> (yaml/load-as-edn "taiga/configmap.yaml") (cm/replace-key-value :TAIGA_SITES_DOMAIN fqdn) (cm/replace-key-value :TAIGA_URL (str "https://" fqdn)) (cm/replace-key-value :TAIGA_WEBSOCKETS_URL (str "wss://" fqdn)) @@ -176,7 +176,7 @@ [config config?] (let [{:keys [storage-class-name storage-media-size]} (merge config-defaults config)] (-> - (yaml/from-string (yaml/load-resource "taiga/pvc-taiga-media-data.yaml")) + (yaml/load-as-edn "taiga/pvc-taiga-media-data.yaml") (assoc-in [:spec :storageClassName] storage-class-name) (assoc-in [:spec :resources :requests :storage] (str storage-media-size "Gi"))))) @@ -184,7 +184,7 @@ [config config?] (let [{:keys [storage-class-name storage-static-size]} (merge config-defaults config)] (-> - (yaml/from-string (yaml/load-resource "taiga/pvc-taiga-static-data.yaml")) + (yaml/load-as-edn "taiga/pvc-taiga-static-data.yaml") (assoc-in [:spec :storageClassName] storage-class-name) (assoc-in [:spec :resources :requests :storage] (str storage-static-size "Gi"))))) @@ -195,7 +195,7 @@ rabbitmq-user rabbitmq-pw rabbitmq-erlang-cookie django-superuser-username django-superuser-password django-superuser-email]} auth] (-> - (yaml/from-string (yaml/load-resource "taiga/secret.yaml")) + (yaml/load-as-edn "taiga/secret.yaml") (cm/replace-key-value :TAIGA_SECRET_KEY (b64/encode taiga-secret-key)) (cm/replace-key-value :EMAIL_HOST_USER (b64/encode mailer-user)) (cm/replace-key-value :EMAIL_HOST_PASSWORD (b64/encode mailer-pw)) @@ -210,7 +210,7 @@ [config config?] (let [{:keys [storage-class-name storage-async-rabbitmq-size]} (merge config-defaults config)] (-> - (yaml/from-string (yaml/load-resource "taiga/rabbitmq-pvc-async.yaml")) + (yaml/load-as-edn "taiga/rabbitmq-pvc-async.yaml") (assoc-in [:spec :storageClassName] storage-class-name) (assoc-in [:spec :resources :requests :storage] (str storage-async-rabbitmq-size "Gi"))))) @@ -218,7 +218,7 @@ [config config?] (let [{:keys [storage-class-name storage-events-rabbitmq-size]} (merge config-defaults config)] (-> - (yaml/from-string (yaml/load-resource "taiga/rabbitmq-pvc-events.yaml")) + (yaml/load-as-edn "taiga/rabbitmq-pvc-events.yaml") (assoc-in [:spec :storageClassName] storage-class-name) (assoc-in [:spec :resources :requests :storage] (str storage-events-rabbitmq-size "Gi"))))) From dbffa50dab0108585866331494a2b2283cb342f3 Mon Sep 17 00:00:00 2001 From: bom Date: Fri, 23 Feb 2024 16:11:32 +0100 Subject: [PATCH 08/12] Add generic test task to build --- build.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/build.py b/build.py index d8197e5..7438b93 100644 --- a/build.py +++ b/build.py @@ -39,6 +39,12 @@ def initialize(project): build.initialize_build_dir() +@task +def test(project): + test_clj(project) + test_cljs(project) + test_schema(project) + @task def test_clj(project): run("lein test", shell=True, check=True) From 1c3088630f9c8a29700c068859a32982f5fb8525 Mon Sep 17 00:00:00 2001 From: bom Date: Fri, 23 Feb 2024 16:20:46 +0100 Subject: [PATCH 09/12] Use inline-resources macro --- project.clj | 2 +- shadow-cljs.edn | 2 +- src/main/cljc/dda/c4k_taiga/taiga.cljc | 29 +++----------------------- 3 files changed, 5 insertions(+), 28 deletions(-) diff --git a/project.clj b/project.clj index c92762e..ac3550a 100644 --- a/project.clj +++ b/project.clj @@ -5,7 +5,7 @@ :url "https://www.apache.org/licenses/LICENSE-2.0.html"} :dependencies [[org.clojure/clojure "1.11.1"] [org.clojure/tools.reader "1.3.7"] - [org.domaindrivenarchitecture/c4k-common-clj "6.1.2"] + [org.domaindrivenarchitecture/c4k-common-clj "6.1.3"] [hickory "0.7.1" :exclusions [viebel/codox-klipse-theme]]] :target-path "target/%s/" :source-paths ["src/main/cljc" diff --git a/shadow-cljs.edn b/shadow-cljs.edn index b47c277..f9ee9c9 100644 --- a/shadow-cljs.edn +++ b/shadow-cljs.edn @@ -4,7 +4,7 @@ "src/test/cljc" "src/test/cljs" "src/test/resources"] - :dependencies [[org.domaindrivenarchitecture/c4k-common-cljs "6.0.1"] + :dependencies [[org.domaindrivenarchitecture/c4k-common-cljs "6.1.3"] [hickory "0.7.1"]] :builds {:frontend {:target :browser :modules {:main {:init-fn dda.c4k-taiga.browser/init}} diff --git a/src/main/cljc/dda/c4k_taiga/taiga.cljc b/src/main/cljc/dda/c4k_taiga/taiga.cljc index 4ba018f..ba8dfb7 100644 --- a/src/main/cljc/dda/c4k_taiga/taiga.cljc +++ b/src/main/cljc/dda/c4k_taiga/taiga.cljc @@ -13,7 +13,8 @@ [dda.c4k-common.monitoring :as mon] [dda.c4k-common.postgres :as postgres] [dda.c4k-common.ingress :as ing] - [clojure.string :as str])) + [clojure.string :as str] + #?(:cljs [dda.c4k-common.macros :refer-macros [inline-resources]]))) (def config-defaults {:issuer "staging" @@ -75,31 +76,7 @@ #?(:cljs (defmethod yaml/load-resource :taiga [resource-name] - (case resource-name - "taiga/events-rabbitmq-deployment.yaml" (rc/inline "taiga/events-rabbitmq-deployment.yaml") - "taiga/gateway-deployment.yaml" (rc/inline "taiga/gateway-deployment.yaml") - "taiga/protected-deployment.yaml" (rc/inline "taiga/protected-deployment.yaml") - "taiga/gateway-configmap.yaml" (rc/inline "taiga/gateway-configmap.yaml") - "taiga/configmap.yaml" (rc/inline "taiga/configmap.yaml") - "taiga/async-service.yaml" (rc/inline "taiga/async-service.yaml") - "taiga/events-deployment.yaml" (rc/inline "taiga/events-deployment.yaml") - "taiga/async-deployment.yaml" (rc/inline "taiga/async-deployment.yaml") - "taiga/back-deployment.yaml" (rc/inline "taiga/back-deployment.yaml") - "taiga/front-deployment.yaml" (rc/inline "taiga/front-deployment.yaml") - "taiga/front-service.yaml" (rc/inline "taiga/front-service.yaml") - "taiga/gateway-service.yaml" (rc/inline "taiga/gateway-service.yaml") - "taiga/pvc-taiga-media-data.yaml" (rc/inline "taiga/pvc-taiga-media-data.yaml") - "taiga/pvc-taiga-static-data.yaml" (rc/inline "taiga/pvc-taiga-static-data.yaml") - "taiga/async-rabbitmq-deployment.yaml" (rc/inline "taiga/async-rabbitmq-deployment.yaml") - "taiga/protected-service.yaml" (rc/inline "taiga/protected-service.yaml") - "taiga/secret.yaml" (rc/inline "taiga/secret.yaml") - "taiga/async-rabbitmq-service.yaml" (rc/inline "taiga/async-rabbitmq-service.yaml") - "taiga/events-service.yaml" (rc/inline "taiga/events-service.yaml") - "taiga/back-service.yaml" (rc/inline "taiga/back-service.yaml") - "taiga/events-rabbitmq-service.yaml" (rc/inline "taiga/events-rabbitmq-service.yaml") - "taiga/rabbitmq-pvc-async.yaml" (rc/inline "taiga/rabbitmq-pvc-async.yaml") - "taiga/rabbitmq-pvc-events.yaml" (rc/inline "taiga/rabbitmq-pvc-events.yaml") - (throw (js/Error. "Undefined Resource!"))))) + (get (inline-resources "taiga") resource-name))) (defn-spec generate-ingress-and-cert cp/map-or-seq? [config config?] From 24b1abf7afdbf6e31632977477dc0c454b2702a8 Mon Sep 17 00:00:00 2001 From: bom Date: Fri, 23 Feb 2024 16:47:24 +0100 Subject: [PATCH 10/12] Add native build --- .gitignore | 1 + .gitlab-ci.yml | 15 +++++-- build.py | 47 ++++++++++++++++++++ doc/Development.md | 107 +++++++++++++++++++++++++++++++++++++++++++++ project.clj | 23 +++------- 5 files changed, 173 insertions(+), 20 deletions(-) create mode 100644 doc/Development.md diff --git a/.gitignore b/.gitignore index 2eabbd1..33d8fb7 100644 --- a/.gitignore +++ b/.gitignore @@ -12,6 +12,7 @@ target/ .lein-repl-history .lein-failures pom.* +reports/* # cljs .shadow-cljs diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 3ea3b69..fdb70af 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -6,7 +6,7 @@ stages: - image .img: &img - image: "domaindrivenarchitecture/ddadevops-dind:4.10.7" + image: "domaindrivenarchitecture/ddadevops-dind:4.11.3" services: - docker:dind before_script: @@ -16,7 +16,7 @@ stages: - export IMAGE_TAG=$CI_COMMIT_TAG .cljs-job: &cljs - image: "domaindrivenarchitecture/ddadevops-clj-cljs:4.10.7" + image: "domaindrivenarchitecture/ddadevops-clj-cljs:4.11.3" cache: key: ${CI_COMMIT_REF_SLUG} paths: @@ -29,7 +29,7 @@ stages: - npm install .clj-job: &clj - image: "domaindrivenarchitecture/ddadevops-clj-cljs:4.10.7" + image: "domaindrivenarchitecture/ddadevops-clj-cljs:4.11.3" cache: key: ${CI_COMMIT_REF_SLUG} paths: @@ -93,6 +93,15 @@ package-uberjar: paths: - target/uberjar +package-native: + <<: *clj + stage: package + script: + - pyb package_native + artifacts: + paths: + - target/graalvm + release-to-clojars: <<: *clj <<: *tag_only diff --git a/build.py b/build.py index 7438b93..ba479d4 100644 --- a/build.py +++ b/build.py @@ -29,6 +29,7 @@ def initialize(project): "release_organisation": "meissa", "release_repository_name": name, "release_artifacts": [ + f"target/graalvm/{name}", f"target/uberjar/{name}-standalone.jar", f"target/frontend-build/{name}.js", ], @@ -113,11 +114,57 @@ def package_uberjar(project): check=True, ) +@task +def package_native(project): + run( + "mkdir -p target/graalvm", + shell=True, + check=True, + ) + run( + "native-image " + + "--native-image-info " + + "--report-unsupported-elements-at-runtime " + + "--no-server " + + "--no-fallback " + + "--features=clj_easy.graal_build_time.InitClojureClasses " + + f"-jar target/uberjar/{project.name}-standalone.jar " + + "-H:IncludeResources=.*.yaml " + + "-H:Log=registerResource:verbose " + + f"-H:Name=target/graalvm/{project.name}", + shell=True, + check=True, + ) + run( + f"sha256sum target/graalvm/{project.name} > target/graalvm/{project.name}.sha256", + shell=True, + check=True, + ) + run( + f"sha512sum target/graalvm/{project.name} > target/graalvm/{project.name}.sha512", + shell=True, + check=True, + ) @task def upload_clj(project): run("lein deploy", shell=True, check=True) +@task +def inst(project): + package_uberjar(project) + package_native(project) + run( + f"sudo install -m=755 target/uberjar/{project.name}-standalone.jar /usr/local/bin/{project.name}-standalone.jar", + shell=True, + check=True, + ) + run( + f"sudo install -m=755 target/graalvm/{project.name} /usr/local/bin/{project.name}", + shell=True, + check=True, + ) + @task def lint(project): diff --git a/doc/Development.md b/doc/Development.md new file mode 100644 index 0000000..08c8546 --- /dev/null +++ b/doc/Development.md @@ -0,0 +1,107 @@ +# Project Setup + +## clj setup + +### install leiningen +``` +sudo apt install leiningen +``` +or manually using Instructions on https://leiningen.org/#install + +### install vscode + extensions +``` +sudo snap install code +``` +or with packages from https://code.visualstudio.com/Download + +install extension "Calva: Clojure & ClojureScript Interactive Programming" + +## cljs / js-dev setup + +``` +sudo apt install npm +sudo npm install -g npx + +# maybe +sudo npm install -g shadow-cljs + +# in project root to retrieve all dependencies +npm install --ignore-scripts +npx shadow-cljs compile test +``` + +### create frontend script + +``` +npx shadow-cljs release frontend +``` + +## graalvm-setup + +``` +curl -LO https://github.com/graalvm/graalvm-ce-builds/releases/download/jdk-21.0.2/graalvm-community-jdk-21.0.2_linux-x64_bin.tar.gz + +# unpack +tar -xzf graalvm-community-jdk-21.0.2_linux-x64_bin.tar.gz + +sudo mv graalvm-community-openjdk-21.0.2+13.1 /usr/lib/jvm/ +sudo ln -s /usr/lib/jvm/graalvm-community-openjdk-21.0.2+13.1 /usr/lib/jvm/graalvm-21 +sudo ln -s /usr/lib/jvm/graalvm-21/bin/gu /usr/local/bin +sudo update-alternatives --install /usr/bin/java java /usr/lib/jvm/graalvm-21/bin/java 2 +sudo update-alternatives --config java +sudo ln -s /usr/lib/jvm/graalvm-21/bin/native-image /usr/local/bin + +# deps +sudo apt-get install build-essential libz-dev zlib1g-dev + +# build +cd ~/repo/c4k/c4k-forgejo +lein uberjar +mkdir -p target/graalvm +lein native + +# execute +./target/graalvm/c4k-cloud -h +./target/graalvm/c4k-cloud src/test/resources/valid-config.edn src/test/resources/valid-auth.edn +./target/graalvm/c4k-cloud src/test/resources/invalid-config.edn src/test/resources/invalid-auth.edn +``` + +## c4k-setup +### install kubectl + +``` +sudo -i +curl -s https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add - +echo "deb http://apt.kubernetes.io/ kubernetes-xenial main" \ + | tee -a /etc/apt/sources.list.d/kubernetes.list +apt update && apt install kubectl +kubectl completion bash >> /etc/bash_completion.d/kubernetes +``` + +### install kubeconform + +``` +curl -Lo /tmp/kubeconform.tar.gz https://github.com/yannh/kubeconform/releases/download/v0.4.7/kubeconform-linux-amd64.tar.gz +tar -xf /tmp/kubeconform.tar.gz +sudo cp kubeconform /usr/local/bin +``` + +### remote access to c4k + +``` +scp -r root@devops.test.meissa-gmbh.de:/home/c4k/.kube ~/ +ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no root@devops.test.meissa-gmbh.de -L 8002:localhost:8002 -L 6443:192.168.5.1:6443 + +# add in /etc/hosts "127.0.0.1 kubernetes" + +# change in ~/.kube/config 192.168.5.1 -> kubernetes + +kubectl get pods +``` + +### deploy cloud + +``` +java -jar target/uberjar/c4k-cloud-standalone.jar valid-config.edn valid-auth.edn | kubeconform --kubernetes-version 1.19.0 --strict --skip Certificate - +java -jar target/uberjar/c4k-cloud-standalone.jar valid-config.edn my-auth.edn | kubectl apply -f - +``` diff --git a/project.clj b/project.clj index ac3550a..f81cafa 100644 --- a/project.clj +++ b/project.clj @@ -4,7 +4,7 @@ :license {:name "Apache License, Version 2.0" :url "https://www.apache.org/licenses/LICENSE-2.0.html"} :dependencies [[org.clojure/clojure "1.11.1"] - [org.clojure/tools.reader "1.3.7"] + [org.clojure/tools.reader "1.4.0"] [org.domaindrivenarchitecture/c4k-common-clj "6.1.3"] [hickory "0.7.1" :exclusions [viebel/codox-klipse-theme]]] :target-path "target/%s/" @@ -22,25 +22,14 @@ :uberjar {:aot :all :main dda.c4k-taiga.uberjar :uberjar-name "c4k-taiga-standalone.jar" - :dependencies [[org.clojure/tools.cli "1.0.219"] - [ch.qos.logback/logback-classic "1.4.14" + :dependencies [[org.clojure/tools.cli "1.1.230"] + [ch.qos.logback/logback-classic "1.5.0" :exclusions [com.sun.mail/javax.mail]] - [org.slf4j/jcl-over-slf4j "2.0.11"]]}} + [org.slf4j/jcl-over-slf4j "2.0.12"] + [com.github.clj-easy/graal-build-time "1.0.5"]]}} :release-tasks [["test"] ["vcs" "assert-committed"] ["change" "version" "leiningen.release/bump-version" "release"] ["vcs" "commit"] ["vcs" "tag" "v" "--no-sign"] - ["change" "version" "leiningen.release/bump-version"]] - :aliases {"native" ["shell" - "native-image" - "--report-unsupported-elements-at-runtime" - "--initialize-at-build-time" - "-jar" "target/uberjar/c4k-taiga-standalone.jar" - "-H:ResourceConfigurationFiles=graalvm-resource-config.json" - "-H:Log=registerResource" - "-H:Name=target/graalvm/${:name}"] - "inst" ["shell" - "sh" - "-c" - "lein uberjar && sudo install -m=755 target/uberjar/c4k-taiga-standalone.jar /usr/local/bin/c4k-taiga-standalone.jar"]}) + ["change" "version" "leiningen.release/bump-version"]]) From f8abf9a414634c779933b4256c518688cf4af74d Mon Sep 17 00:00:00 2001 From: bom Date: Fri, 23 Feb 2024 16:53:11 +0100 Subject: [PATCH 11/12] Use correct image for clj build --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index fdb70af..05b8c7f 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -29,7 +29,7 @@ stages: - npm install .clj-job: &clj - image: "domaindrivenarchitecture/ddadevops-clj-cljs:4.11.3" + image: "domaindrivenarchitecture/ddadevops-clj:4.11.3" cache: key: ${CI_COMMIT_REF_SLUG} paths: From 9201aba498a59e9c2a80ef65712a476c9f742eba Mon Sep 17 00:00:00 2001 From: Clemens Date: Mon, 3 Jun 2024 10:05:22 +0200 Subject: [PATCH 12/12] make superuser creation manual --- README.md | 8 ++++++++ src/main/resources/taiga/back-deployment.yaml | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 3389fb1..e858ee6 100644 --- a/README.md +++ b/README.md @@ -63,10 +63,18 @@ To set up you need: Apply this file on your cluster with `kubectl apply -f application.yaml`. Done. +## Setup + +`python manage.py createsuperuser --noinput` + ## Administration You can access the administration of the taiga installation via: your.taiga.url/admin/ +In order to login, you first have to create a superuser. +1. Connect to taiga-back pod: `kubectl exec -it taiga-back-deployment-... -- bash` +2. `source /opt/venv/bin/activate && python manage.py createsuperuser --noinput` + ## Backup You need some form of cloud storage like AWS buckets and the respective access credentials diff --git a/src/main/resources/taiga/back-deployment.yaml b/src/main/resources/taiga/back-deployment.yaml index f389e23..96b994f 100644 --- a/src/main/resources/taiga/back-deployment.yaml +++ b/src/main/resources/taiga/back-deployment.yaml @@ -20,7 +20,7 @@ spec: image: taigaio/taiga-back:latest imagePullPolicy: IfNotPresent command: ["/bin/bash"] - args: ["-c", "source /opt/venv/bin/activate && CELERY_ENABLE=true python manage.py migrate && sleep 15 && python manage.py createsuperuser --noinput"] + args: ["-c", "source /opt/venv/bin/activate && CELERY_ENABLE=true python manage.py migrate && sleep 15"] ports: - name: http containerPort: 8000