Merge branch 'main' of ssh://repo.prod.meissa.de:2222/meissa/c4k-taiga

This commit is contained in:
patdyn 2024-08-06 13:14:57 +02:00
commit faeeb1c9f7
15 changed files with 239 additions and 104 deletions

1
.gitignore vendored
View file

@ -12,6 +12,7 @@ target/
.lein-repl-history
.lein-failures
pom.*
reports/*
# cljs
.shadow-cljs

View file

@ -6,7 +6,7 @@ stages:
- image
.img: &img
image: "domaindrivenarchitecture/ddadevops-dind:4.9.0"
image: "domaindrivenarchitecture/ddadevops-dind:4.11.3"
services:
- docker:dind
before_script:
@ -16,7 +16,7 @@ stages:
- export IMAGE_TAG=$CI_COMMIT_TAG
.cljs-job: &cljs
image: "domaindrivenarchitecture/ddadevops-clj-cljs:4.9.0"
image: "domaindrivenarchitecture/ddadevops-clj-cljs:4.11.3"
cache:
key: ${CI_COMMIT_REF_SLUG}
paths:
@ -29,7 +29,7 @@ stages:
- npm install
.clj-job: &clj
image: "domaindrivenarchitecture/ddadevops-clj-cljs:4.9.0"
image: "domaindrivenarchitecture/ddadevops-clj:4.11.3"
cache:
key: ${CI_COMMIT_REF_SLUG}
paths:
@ -93,6 +93,15 @@ package-uberjar:
paths:
- target/uberjar
package-native:
<<: *clj
stage: package
script:
- pyb package_native
artifacts:
paths:
- target/graalvm
release-to-clojars:
<<: *clj
<<: *tag_only

View file

@ -63,10 +63,18 @@ To set up you need:
Apply this file on your cluster with `kubectl apply -f application.yaml`.
Done.
## Setup
`python manage.py createsuperuser --noinput`
## Administration
You can access the administration of the taiga installation via: your.taiga.url/admin/
In order to login, you first have to create a superuser.
1. Connect to taiga-back pod: `kubectl exec -it taiga-back-deployment-... -- bash`
2. `source /opt/venv/bin/activate && python manage.py createsuperuser --noinput`
## Backup
You need some form of cloud storage like AWS buckets and the respective access credentials

View file

@ -29,6 +29,7 @@ def initialize(project):
"release_organisation": "meissa",
"release_repository_name": name,
"release_artifacts": [
f"target/graalvm/{name}",
f"target/uberjar/{name}-standalone.jar",
f"target/frontend-build/{name}.js",
],
@ -39,6 +40,12 @@ def initialize(project):
build.initialize_build_dir()
@task
def test(project):
test_clj(project)
test_cljs(project)
test_schema(project)
@task
def test_clj(project):
run("lein test", shell=True, check=True)
@ -107,11 +114,57 @@ def package_uberjar(project):
check=True,
)
@task
def package_native(project):
run(
"mkdir -p target/graalvm",
shell=True,
check=True,
)
run(
"native-image " +
"--native-image-info " +
"--report-unsupported-elements-at-runtime " +
"--no-server " +
"--no-fallback " +
"--features=clj_easy.graal_build_time.InitClojureClasses " +
f"-jar target/uberjar/{project.name}-standalone.jar " +
"-H:IncludeResources=.*.yaml " +
"-H:Log=registerResource:verbose " +
f"-H:Name=target/graalvm/{project.name}",
shell=True,
check=True,
)
run(
f"sha256sum target/graalvm/{project.name} > target/graalvm/{project.name}.sha256",
shell=True,
check=True,
)
run(
f"sha512sum target/graalvm/{project.name} > target/graalvm/{project.name}.sha512",
shell=True,
check=True,
)
@task
def upload_clj(project):
run("lein deploy", shell=True, check=True)
@task
def inst(project):
package_uberjar(project)
package_native(project)
run(
f"sudo install -m=755 target/uberjar/{project.name}-standalone.jar /usr/local/bin/{project.name}-standalone.jar",
shell=True,
check=True,
)
run(
f"sudo install -m=755 target/graalvm/{project.name} /usr/local/bin/{project.name}",
shell=True,
check=True,
)
@task
def lint(project):

107
doc/Development.md Normal file
View file

@ -0,0 +1,107 @@
# Project Setup
## clj setup
### install leiningen
```
sudo apt install leiningen
```
or manually using Instructions on https://leiningen.org/#install
### install vscode + extensions
```
sudo snap install code
```
or with packages from https://code.visualstudio.com/Download
install extension "Calva: Clojure & ClojureScript Interactive Programming"
## cljs / js-dev setup
```
sudo apt install npm
sudo npm install -g npx
# maybe
sudo npm install -g shadow-cljs
# in project root to retrieve all dependencies
npm install --ignore-scripts
npx shadow-cljs compile test
```
### create frontend script
```
npx shadow-cljs release frontend
```
## graalvm-setup
```
curl -LO https://github.com/graalvm/graalvm-ce-builds/releases/download/jdk-21.0.2/graalvm-community-jdk-21.0.2_linux-x64_bin.tar.gz
# unpack
tar -xzf graalvm-community-jdk-21.0.2_linux-x64_bin.tar.gz
sudo mv graalvm-community-openjdk-21.0.2+13.1 /usr/lib/jvm/
sudo ln -s /usr/lib/jvm/graalvm-community-openjdk-21.0.2+13.1 /usr/lib/jvm/graalvm-21
sudo ln -s /usr/lib/jvm/graalvm-21/bin/gu /usr/local/bin
sudo update-alternatives --install /usr/bin/java java /usr/lib/jvm/graalvm-21/bin/java 2
sudo update-alternatives --config java
sudo ln -s /usr/lib/jvm/graalvm-21/bin/native-image /usr/local/bin
# deps
sudo apt-get install build-essential libz-dev zlib1g-dev
# build
cd ~/repo/c4k/c4k-forgejo
lein uberjar
mkdir -p target/graalvm
lein native
# execute
./target/graalvm/c4k-cloud -h
./target/graalvm/c4k-cloud src/test/resources/valid-config.edn src/test/resources/valid-auth.edn
./target/graalvm/c4k-cloud src/test/resources/invalid-config.edn src/test/resources/invalid-auth.edn
```
## c4k-setup
### install kubectl
```
sudo -i
curl -s https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add -
echo "deb http://apt.kubernetes.io/ kubernetes-xenial main" \
| tee -a /etc/apt/sources.list.d/kubernetes.list
apt update && apt install kubectl
kubectl completion bash >> /etc/bash_completion.d/kubernetes
```
### install kubeconform
```
curl -Lo /tmp/kubeconform.tar.gz https://github.com/yannh/kubeconform/releases/download/v0.4.7/kubeconform-linux-amd64.tar.gz
tar -xf /tmp/kubeconform.tar.gz
sudo cp kubeconform /usr/local/bin
```
### remote access to c4k
```
scp -r root@devops.test.meissa-gmbh.de:/home/c4k/.kube ~/
ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no root@devops.test.meissa-gmbh.de -L 8002:localhost:8002 -L 6443:192.168.5.1:6443
# add in /etc/hosts "127.0.0.1 kubernetes"
# change in ~/.kube/config 192.168.5.1 -> kubernetes
kubectl get pods
```
### deploy cloud
```
java -jar target/uberjar/c4k-cloud-standalone.jar valid-config.edn valid-auth.edn | kubeconform --kubernetes-version 1.19.0 --strict --skip Certificate -
java -jar target/uberjar/c4k-cloud-standalone.jar valid-config.edn my-auth.edn | kubectl apply -f -
```

View file

@ -1,4 +1,4 @@
FROM domaindrivenarchitecture/dda-backup:1.0.10
FROM domaindrivenarchitecture/dda-backup:latest
# Prepare Entrypoint Script
ADD resources /tmp

View file

@ -1,9 +1,10 @@
#!/bin/bash
set -eux pipefail
apt-get update > /dev/null;
set -exo pipefail
function main()
{
{
install -m 0700 /tmp/entrypoint.sh /
install -m 0700 /tmp/entrypoint-start-and-wait.sh /
@ -11,3 +12,10 @@ install -m 0700 /tmp/init.sh /usr/local/bin/
install -m 0700 /tmp/backup.sh /usr/local/bin/
install -m 0700 /tmp/restore.sh /usr/local/bin/
install -m 0700 /tmp/restic-snapshots.sh /usr/local/bin/
cleanupDocker
} > /dev/null
}
source /tmp/install_functions_debian.sh
DEBIAN_FRONTEND=noninteractive DEBCONF_NOWARNINGS=yes main

View file

@ -1,11 +0,0 @@
FROM c4k-taiga-backup
RUN apt update
RUN apt -yqq --no-install-recommends --yes install curl default-jre-headless
RUN curl -L -o /tmp/serverspec.jar \
https://github.com/DomainDrivenArchitecture/dda-serverspec-crate/releases/download/2.0.0/dda-serverspec-standalone.jar
COPY serverspec.edn /tmp/serverspec.edn
RUN java -jar /tmp/serverspec.jar /tmp/serverspec.edn -v

View file

@ -1,6 +0,0 @@
{:file [{:path "/usr/local/bin/init.sh" :mod "700"}
{:path "/usr/local/bin/backup.sh" :mod "700"}
{:path "/usr/local/bin/restore.sh" :mod "700"}
{:path "/usr/local/bin/restic-snapshots.sh" :mod "700"}
{:path "/entrypoint.sh" :mod "700"}
{:path "/entrypoint-start-and-wait.sh" :mod "700"}]}

View file

@ -2,7 +2,7 @@
"name": "c4k-taiga",
"description": "Generate c4k yaml for a taiga project management deployment.",
"author": "meissa GmbH",
"version": "1.1.2-SNAPSHOT",
"version": "1.1.3-SNAPSHOT",
"homepage": "https://gitlab.com/domaindrivenarchitecture/c4k-taiga#readme",
"repository": "https://www.npmjs.com/package/c4k-taiga",
"license": "APACHE2",

View file

@ -1,11 +1,11 @@
(defproject org.domaindrivenarchitecture/c4k-taiga "1.1.2-SNAPSHOT"
(defproject org.domaindrivenarchitecture/c4k-taiga "1.1.3-SNAPSHOT"
:description "taiga c4k-installation package"
:url "https://domaindrivenarchitecture.org"
:license {:name "Apache License, Version 2.0"
:url "https://www.apache.org/licenses/LICENSE-2.0.html"}
:dependencies [[org.clojure/clojure "1.11.1"]
[org.clojure/tools.reader "1.3.6"]
[org.domaindrivenarchitecture/c4k-common-clj "6.1.0"]
[org.clojure/tools.reader "1.4.0"]
[org.domaindrivenarchitecture/c4k-common-clj "6.1.3"]
[hickory "0.7.1" :exclusions [viebel/codox-klipse-theme]]]
:target-path "target/%s/"
:source-paths ["src/main/cljc"
@ -22,25 +22,14 @@
:uberjar {:aot :all
:main dda.c4k-taiga.uberjar
:uberjar-name "c4k-taiga-standalone.jar"
:dependencies [[org.clojure/tools.cli "1.0.219"]
[ch.qos.logback/logback-classic "1.4.11"
:dependencies [[org.clojure/tools.cli "1.1.230"]
[ch.qos.logback/logback-classic "1.5.0"
:exclusions [com.sun.mail/javax.mail]]
[org.slf4j/jcl-over-slf4j "2.0.9"]]}}
[org.slf4j/jcl-over-slf4j "2.0.12"]
[com.github.clj-easy/graal-build-time "1.0.5"]]}}
:release-tasks [["test"]
["vcs" "assert-committed"]
["change" "version" "leiningen.release/bump-version" "release"]
["vcs" "commit"]
["vcs" "tag" "v" "--no-sign"]
["change" "version" "leiningen.release/bump-version"]]
:aliases {"native" ["shell"
"native-image"
"--report-unsupported-elements-at-runtime"
"--initialize-at-build-time"
"-jar" "target/uberjar/c4k-taiga-standalone.jar"
"-H:ResourceConfigurationFiles=graalvm-resource-config.json"
"-H:Log=registerResource"
"-H:Name=target/graalvm/${:name}"]
"inst" ["shell"
"sh"
"-c"
"lein uberjar && sudo install -m=755 target/uberjar/c4k-taiga-standalone.jar /usr/local/bin/c4k-taiga-standalone.jar"]})
["change" "version" "leiningen.release/bump-version"]])

View file

@ -4,7 +4,7 @@
"src/test/cljc"
"src/test/cljs"
"src/test/resources"]
:dependencies [[org.domaindrivenarchitecture/c4k-common-cljs "6.0.1"]
:dependencies [[org.domaindrivenarchitecture/c4k-common-cljs "6.1.3"]
[hickory "0.7.1"]]
:builds {:frontend {:target :browser
:modules {:main {:init-fn dda.c4k-taiga.browser/init}}

View file

@ -24,14 +24,14 @@
(defn generate-config [my-conf]
(let [{:keys [restic-repository]} my-conf]
(->
(yaml/from-string (yaml/load-resource "backup/config.yaml"))
(yaml/load-as-edn "backup/config.yaml")
(cm/replace-key-value :restic-repository restic-repository))))
(defn generate-cron []
(yaml/from-string (yaml/load-resource "backup/cron.yaml")))
(yaml/load-as-edn "backup/cron.yaml"))
(defn generate-backup-restore-deployment [my-conf]
(let [backup-restore-yaml (yaml/from-string (yaml/load-resource "backup/backup-restore-deployment.yaml"))]
(let [backup-restore-yaml (yaml/load-as-edn "backup/backup-restore-deployment.yaml")]
(if (and (contains? my-conf :local-integration-test) (= true (:local-integration-test my-conf)))
(cm/replace-named-value backup-restore-yaml "CERTIFICATE_FILE" "/var/run/secrets/localstack-secrets/ca.crt")
backup-restore-yaml)))
@ -39,7 +39,7 @@
(defn generate-secret [my-auth]
(let [{:keys [aws-access-key-id aws-secret-access-key restic-password]} my-auth]
(->
(yaml/from-string (yaml/load-resource "backup/secret.yaml"))
(yaml/load-as-edn "backup/secret.yaml")
(cm/replace-key-value :aws-access-key-id (b64/encode aws-access-key-id))
(cm/replace-key-value :aws-secret-access-key (b64/encode aws-secret-access-key))
(cm/replace-key-value :restic-password (b64/encode restic-password)))))

View file

@ -13,7 +13,8 @@
[dda.c4k-common.monitoring :as mon]
[dda.c4k-common.postgres :as postgres]
[dda.c4k-common.ingress :as ing]
[clojure.string :as str]))
[clojure.string :as str]
#?(:cljs [dda.c4k-common.macros :refer-macros [inline-resources]])))
(def config-defaults {:issuer "staging"
@ -75,31 +76,7 @@
#?(:cljs
(defmethod yaml/load-resource :taiga [resource-name]
(case resource-name
"taiga/events-rabbitmq-deployment.yaml" (rc/inline "taiga/events-rabbitmq-deployment.yaml")
"taiga/gateway-deployment.yaml" (rc/inline "taiga/gateway-deployment.yaml")
"taiga/protected-deployment.yaml" (rc/inline "taiga/protected-deployment.yaml")
"taiga/gateway-configmap.yaml" (rc/inline "taiga/gateway-configmap.yaml")
"taiga/configmap.yaml" (rc/inline "taiga/configmap.yaml")
"taiga/async-service.yaml" (rc/inline "taiga/async-service.yaml")
"taiga/events-deployment.yaml" (rc/inline "taiga/events-deployment.yaml")
"taiga/async-deployment.yaml" (rc/inline "taiga/async-deployment.yaml")
"taiga/back-deployment.yaml" (rc/inline "taiga/back-deployment.yaml")
"taiga/front-deployment.yaml" (rc/inline "taiga/front-deployment.yaml")
"taiga/front-service.yaml" (rc/inline "taiga/front-service.yaml")
"taiga/gateway-service.yaml" (rc/inline "taiga/gateway-service.yaml")
"taiga/pvc-taiga-media-data.yaml" (rc/inline "taiga/pvc-taiga-media-data.yaml")
"taiga/pvc-taiga-static-data.yaml" (rc/inline "taiga/pvc-taiga-static-data.yaml")
"taiga/async-rabbitmq-deployment.yaml" (rc/inline "taiga/async-rabbitmq-deployment.yaml")
"taiga/protected-service.yaml" (rc/inline "taiga/protected-service.yaml")
"taiga/secret.yaml" (rc/inline "taiga/secret.yaml")
"taiga/async-rabbitmq-service.yaml" (rc/inline "taiga/async-rabbitmq-service.yaml")
"taiga/events-service.yaml" (rc/inline "taiga/events-service.yaml")
"taiga/back-service.yaml" (rc/inline "taiga/back-service.yaml")
"taiga/events-rabbitmq-service.yaml" (rc/inline "taiga/events-rabbitmq-service.yaml")
"taiga/rabbitmq-pvc-async.yaml" (rc/inline "taiga/rabbitmq-pvc-async.yaml")
"taiga/rabbitmq-pvc-events.yaml" (rc/inline "taiga/rabbitmq-pvc-events.yaml")
(throw (js/Error. "Undefined Resource!")))))
(get (inline-resources "taiga") resource-name)))
(defn-spec generate-ingress-and-cert cp/map-or-seq?
[config config?]
@ -112,55 +89,55 @@
config))))
(defn-spec generate-async-deployment cp/map-or-seq? []
(yaml/from-string (yaml/load-resource "taiga/async-deployment.yaml")))
(yaml/load-as-edn "taiga/async-deployment.yaml"))
(defn-spec generate-async-service cp/map-or-seq? []
(yaml/from-string (yaml/load-resource "taiga/async-service.yaml")))
(yaml/load-as-edn "taiga/async-service.yaml"))
(defn-spec generate-async-rabbitmq-deployment cp/map-or-seq? []
(yaml/from-string (yaml/load-resource "taiga/async-rabbitmq-deployment.yaml")))
(yaml/load-as-edn "taiga/async-rabbitmq-deployment.yaml"))
(defn-spec generate-events-rabbitmq-service cp/map-or-seq? []
(yaml/from-string (yaml/load-resource "taiga/events-rabbitmq-service.yaml")))
(yaml/load-as-edn "taiga/events-rabbitmq-service.yaml"))
(defn-spec generate-async-rabbitmq-service cp/map-or-seq? []
(yaml/from-string (yaml/load-resource "taiga/async-rabbitmq-service.yaml")))
(yaml/load-as-edn "taiga/async-rabbitmq-service.yaml"))
(defn-spec generate-back-deployment cp/map-or-seq? []
(yaml/from-string (yaml/load-resource "taiga/back-deployment.yaml")))
(yaml/load-as-edn "taiga/back-deployment.yaml"))
(defn-spec generate-back-service cp/map-or-seq? []
(yaml/from-string (yaml/load-resource "taiga/back-service.yaml")))
(yaml/load-as-edn "taiga/back-service.yaml"))
(defn-spec generate-events-rabbitmq-deployment cp/map-or-seq? []
(yaml/from-string (yaml/load-resource "taiga/events-rabbitmq-deployment.yaml")))
(yaml/load-as-edn "taiga/events-rabbitmq-deployment.yaml"))
(defn-spec generate-events-deployment cp/map-or-seq? []
(yaml/from-string (yaml/load-resource "taiga/events-deployment.yaml")))
(yaml/load-as-edn "taiga/events-deployment.yaml"))
(defn-spec generate-events-service cp/map-or-seq? []
(yaml/from-string (yaml/load-resource "taiga/events-service.yaml")))
(yaml/load-as-edn "taiga/events-service.yaml"))
(defn-spec generate-front-deployment cp/map-or-seq? []
(yaml/from-string (yaml/load-resource "taiga/front-deployment.yaml")))
(yaml/load-as-edn "taiga/front-deployment.yaml"))
(defn-spec generate-front-service cp/map-or-seq? []
(yaml/from-string (yaml/load-resource "taiga/front-service.yaml")))
(yaml/load-as-edn "taiga/front-service.yaml"))
(defn-spec generate-gateway-configmap cp/map-or-seq? []
(yaml/from-string (yaml/load-resource "taiga/gateway-configmap.yaml")))
(yaml/load-as-edn "taiga/gateway-configmap.yaml"))
(defn-spec generate-gateway-deployment cp/map-or-seq? []
(yaml/from-string (yaml/load-resource "taiga/gateway-deployment.yaml")))
(yaml/load-as-edn "taiga/gateway-deployment.yaml"))
(defn-spec generate-gateway-service cp/map-or-seq? []
(yaml/from-string (yaml/load-resource "taiga/gateway-service.yaml")))
(yaml/load-as-edn "taiga/gateway-service.yaml"))
(defn-spec generate-protected-deployment cp/map-or-seq? []
(yaml/from-string (yaml/load-resource "taiga/protected-deployment.yaml")))
(yaml/load-as-edn "taiga/protected-deployment.yaml"))
(defn-spec generate-protected-service cp/map-or-seq? []
(yaml/from-string (yaml/load-resource "taiga/protected-service.yaml")))
(yaml/load-as-edn "taiga/protected-service.yaml"))
(defn-spec generate-configmap cp/map-or-seq?
[config config?]
@ -176,7 +153,7 @@
[config config?]
(let [{:keys [storage-class-name storage-media-size]} (merge config-defaults config)]
(->
(yaml/from-string (yaml/load-resource "taiga/pvc-taiga-media-data.yaml"))
(yaml/load-as-edn "taiga/pvc-taiga-media-data.yaml")
(assoc-in [:spec :storageClassName] storage-class-name)
(assoc-in [:spec :resources :requests :storage] (str storage-media-size "Gi")))))
@ -184,7 +161,7 @@
[config config?]
(let [{:keys [storage-class-name storage-static-size]} (merge config-defaults config)]
(->
(yaml/from-string (yaml/load-resource "taiga/pvc-taiga-static-data.yaml"))
(yaml/load-as-edn "taiga/pvc-taiga-static-data.yaml")
(assoc-in [:spec :storageClassName] storage-class-name)
(assoc-in [:spec :resources :requests :storage] (str storage-static-size "Gi")))))
@ -195,7 +172,7 @@
rabbitmq-user rabbitmq-pw rabbitmq-erlang-cookie
django-superuser-username django-superuser-password django-superuser-email]} auth]
(->
(yaml/from-string (yaml/load-resource "taiga/secret.yaml"))
(yaml/load-as-edn "taiga/secret.yaml")
(cm/replace-key-value :TAIGA_SECRET_KEY (b64/encode taiga-secret-key))
(cm/replace-key-value :EMAIL_HOST_USER (b64/encode mailer-user))
(cm/replace-key-value :EMAIL_HOST_PASSWORD (b64/encode mailer-pw))
@ -210,7 +187,7 @@
[config config?]
(let [{:keys [storage-class-name storage-async-rabbitmq-size]} (merge config-defaults config)]
(->
(yaml/from-string (yaml/load-resource "taiga/rabbitmq-pvc-async.yaml"))
(yaml/load-as-edn "taiga/rabbitmq-pvc-async.yaml")
(assoc-in [:spec :storageClassName] storage-class-name)
(assoc-in [:spec :resources :requests :storage] (str storage-async-rabbitmq-size "Gi")))))
@ -218,7 +195,7 @@
[config config?]
(let [{:keys [storage-class-name storage-events-rabbitmq-size]} (merge config-defaults config)]
(->
(yaml/from-string (yaml/load-resource "taiga/rabbitmq-pvc-events.yaml"))
(yaml/load-as-edn "taiga/rabbitmq-pvc-events.yaml")
(assoc-in [:spec :storageClassName] storage-class-name)
(assoc-in [:spec :resources :requests :storage] (str storage-events-rabbitmq-size "Gi")))))

View file

@ -20,7 +20,7 @@ spec:
image: taigaio/taiga-back:latest
imagePullPolicy: IfNotPresent
command: ["/bin/bash"]
args: ["-c", "source /opt/venv/bin/activate && CELERY_ENABLE=true python manage.py migrate && sleep 15 && python manage.py createsuperuser --noinput"]
args: ["-c", "source /opt/venv/bin/activate && CELERY_ENABLE=true python manage.py migrate && sleep 15"]
ports:
- name: http
containerPort: 8000