Merge pull request 'Add the inline macro, native build and namespaces refactorings' (#1) from refactorings into master

Reviewed-on: #1
This commit is contained in:
patdyn 2024-07-04 14:02:43 +00:00
commit e8eea1e646
13 changed files with 331 additions and 116 deletions

1
.gitignore vendored
View file

@ -7,6 +7,7 @@ target/
.lein-repl-history
.lein-failures
pom.*
reports/*
# cljs
.shadow-cljs

View file

@ -5,7 +5,7 @@ stages:
- upload
.cljs-job: &cljs
image: domaindrivenarchitecture/shadow-cljs
image: "domaindrivenarchitecture/ddadevops-clj-cljs:4.11.3"
cache:
key: ${CI_COMMIT_REF_SLUG}
paths:
@ -13,38 +13,44 @@ stages:
- .shadow-cljs/
- .m2
before_script:
- echo "//registry.npmjs.org/:_authToken=$NPM_TOKEN" > ~/.npmrc
- npm install
- export RELEASE_ARTIFACT_TOKEN=$MEISSA_REPO_BUERO_RW
- echo "//registry.npmjs.org/:_authToken=$NPM_TOKEN" > ~/.npmrc
- npm install
.clj-uploadjob: &clj
image: domaindrivenarchitecture/lein
image: "domaindrivenarchitecture/ddadevops-clj:4.11.3"
cache:
key: ${CI_COMMIT_REF_SLUG}
paths:
- .m2
before_script:
- mkdir -p /root/.lein
- echo "{:auth {:repository-auth {#\"clojars\" {:username \"${CLOJARS_USER}\" :password \"${CLOJARS_TOKEN_DOMAINDRIVENARCHITECTURE}\" }}}}" > ~/.lein/profiles.clj
- export RELEASE_ARTIFACT_TOKEN=$MEISSA_REPO_BUERO_RW
- mkdir -p /root/.lein
- echo "{:auth {:repository-auth {#\"clojars\" {:username \"${CLOJARS_USER}\" :password \"${CLOJARS_TOKEN_DOMAINDRIVENARCHITECTURE}\" }}}}" > ~/.lein/profiles.clj
test-cljs:
<<: *cljs
stage: build_and_test
script:
- shadow-cljs compile test
- node target/node-tests.js
.tag_only: &tag_only
rules:
- if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
when: never
- if: '$CI_COMMIT_TAG =~ /^[0-9]+\.[0-9]+\.[0-9]+$/'
test-clj:
<<: *clj
stage: build_and_test
script:
- lein test
- pyb test_clj
test-cljs:
<<: *cljs
stage: build_and_test
script:
- pyb test_cljs
test-schema:
<<: *clj
stage: build_and_test
script:
- lein uberjar
- java -jar target/uberjar/c4k-keycloak-standalone.jar src/test/resources/keycloak-test/valid-config.yaml src/test/resources/keycloak-test/valid-auth.yaml | kubeconform --kubernetes-version 1.19.0 --strict --skip Certificate -
- pyb test_schema
artifacts:
paths:
- target/uberjar
@ -53,8 +59,7 @@ report-frontend:
<<: *cljs
stage: package
script:
- mkdir -p target/frontend-build
- shadow-cljs run shadow.cljs.build-report frontend target/frontend-build/build-report.html
- pyb report_frontend
artifacts:
paths:
- target/frontend-build/build-report.html
@ -63,11 +68,7 @@ package-frontend:
<<: *cljs
stage: package
script:
- mkdir -p target/frontend-build
- shadow-cljs release frontend
- cp public/js/main.js target/frontend-build/c4k-keycloak.js
- sha256sum target/frontend-build/c4k-keycloak.js > target/frontend-build/c4k-keycloak.js.sha256
- sha512sum target/frontend-build/c4k-keycloak.js > target/frontend-build/c4k-keycloak.js.sha512
- pyb package_frontend
artifacts:
paths:
- target/frontend-build
@ -76,47 +77,30 @@ package-uberjar:
<<: *clj
stage: package
script:
- sha256sum target/uberjar/c4k-keycloak-standalone.jar > target/uberjar/c4k-keycloak-standalone.jar.sha256
- sha512sum target/uberjar/c4k-keycloak-standalone.jar > target/uberjar/c4k-keycloak-standalone.jar.sha512
- pyb package_uberjar
artifacts:
paths:
- target/uberjar
sast:
variables:
SAST_EXCLUDED_ANALYZERS:
bandit, brakeman, flawfinder, gosec, kubesec, phpcs-security-audit,
pmd-apex, security-code-scan, sobelow, spotbugs
stage: security
before_script:
- mkdir -p builds && cp -r target/ builds/
include:
- template: Security/SAST.gitlab-ci.yml
upload-clj-release:
package-native:
<<: *clj
stage: upload
rules:
- if: '$CI_COMMIT_TAG != null'
stage: package
script:
- lein deploy
release:
image: registry.gitlab.com/gitlab-org/release-cli:latest
stage: upload
rules:
- if: '$CI_COMMIT_TAG != null'
- pyb package_native
artifacts:
paths:
- target/uberjar
- target/frontend-build
- target/graalvm
release-to-clojars:
<<: *clj
<<: *tag_only
stage: upload
script:
- apk --no-cache add curl
- |
release-cli create --name "Release $CI_COMMIT_TAG" --tag-name $CI_COMMIT_TAG \
--assets-link "{\"name\":\"c4k-keycloak-standalone.jar\",\"url\":\"https://gitlab.com/domaindrivenarchitecture/c4k-keycloak/-/jobs/${CI_JOB_ID}/artifacts/file/target/uberjar/c4k-keycloak-standalone.jar\"}" \
--assets-link "{\"name\":\"c4k-keycloak-standalone.jar.sha256\",\"url\":\"https://gitlab.com/domaindrivenarchitecture/c4k-keycloak/-/jobs/${CI_JOB_ID}/artifacts/file/target/uberjar/c4k-keycloak-standalone.jar.sha256\"}" \
--assets-link "{\"name\":\"c4k-keycloak-standalone.jar.sha512\",\"url\":\"https://gitlab.com/domaindrivenarchitecture/c4k-keycloak/-/jobs/${CI_JOB_ID}/artifacts/file/target/uberjar/c4k-keycloak-standalone.jar.sha512\"}" \
--assets-link "{\"name\":\"c4k-keycloak.js\",\"url\":\"https://gitlab.com/domaindrivenarchitecture/c4k-keycloak/-/jobs/${CI_JOB_ID}/artifacts/file/target/frontend-build/c4k-keycloak.js\"}" \
--assets-link "{\"name\":\"c4k-keycloak.js.sha256\",\"url\":\"https://gitlab.com/domaindrivenarchitecture/c4k-keycloak/-/jobs/${CI_JOB_ID}/artifacts/file/target/frontend-build/c4k-keycloak.js.sha256\"}" \
--assets-link "{\"name\":\"c4k-keycloak.js.sha512\",\"url\":\"https://gitlab.com/domaindrivenarchitecture/c4k-keycloak/-/jobs/${CI_JOB_ID}/artifacts/file/target/frontend-build/c4k-keycloak.js.sha512\"}" \
- pyb upload_clj
release-to-forgejo:
<<: *clj
<<: *tag_only
stage: upload
script:
- pyb publish_artifacts

241
build.py Normal file
View file

@ -0,0 +1,241 @@
from os import environ
from subprocess import run
from pybuilder.core import init, task
from ddadevops import *
default_task = "dev"
name = 'c4k-keycloak'
MODULE = 'not-used'
PROJECT_ROOT_PATH = '.'
@init
def initialize(project):
project.build_depends_on("ddadevops>=4.7.0")
input = {
"name": name,
"module": MODULE,
"stage": "notused",
"project_root_path": PROJECT_ROOT_PATH,
"build_types": [],
"mixin_types": ["RELEASE"],
"release_primary_build_file": "project.clj",
"release_secondary_build_files": [
"package.json",
"infrastructure/backup/build.py",
"infrastructure/federated/build.py",
],
"release_artifact_server_url": "https://repo.prod.meissa.de",
"release_organisation": "meissa",
"release_repository_name": name,
"release_artifacts": [
"target/graalvm/" + name,
"target/uberjar/" + name + "-standalone.jar",
"target/frontend-build/" + name + ".js",
],
}
build = ReleaseMixin(project, input)
build.initialize_build_dir()
@task
def test_clj():
run("lein test", shell=True, check=True)
@task
def test_cljs():
run("shadow-cljs compile test", shell=True, check=True)
run("node target/node-tests.js", shell=True, check=True)
@task
def test_schema():
run("lein uberjar", shell=True, check=True)
run(
"java -jar target/uberjar/c4k-keycloak-standalone.jar "
+ "src/test/resources/keycloak-test/valid-config.yaml "
+ "src/test/resources/keycloak-test/valid-auth.yaml | "
+ """kubeconform --kubernetes-version 1.23.0 --strict --skip "Certificate,Middleware" -""",
shell=True,
check=True,
)
@task
def test():
test_clj()
test_cljs()
test_schema()
@task
def report_frontend(project):
run("mkdir -p target/frontend-build", shell=True, check=True)
run(
"shadow-cljs run shadow.cljs.build-report frontend target/frontend-build/build-report.html",
shell=True,
check=True,
)
@task
def package_frontend(project):
run("mkdir -p target/frontend-build", shell=True, check=True)
run("shadow-cljs release frontend", shell=True, check=True)
run(
"cp public/js/main.js target/frontend-build/" + project.name + ".js",
shell=True,
check=True,
)
run(
"sha256sum target/frontend-build/c4k-keycloak.js > target/frontend-build/" + project.name + ".js.sha256",
shell=True,
check=True,
)
run(
"sha512sum target/frontend-build/c4k-keycloak.js > target/frontend-build/" + project.name + ".js.sha512",
shell=True,
check=True,
)
@task
def package_uberjar(project):
run(
"sha256sum target/uberjar/c4k-keycloak-standalone.jar > target/uberjar/" + project.name + "-standalone.jar.sha256",
shell=True,
check=True,
)
run(
"sha512sum target/uberjar/c4k-keycloak-standalone.jar > target/uberjar/" + project.name + "-standalone.jar.sha512",
shell=True,
check=True,
)
@task
def package_native(project):
run(
"mkdir -p target/graalvm",
shell=True,
check=True,
)
run(
"native-image " +
"--native-image-info " +
"--report-unsupported-elements-at-runtime " +
"--no-server " +
"--no-fallback " +
"--features=clj_easy.graal_build_time.InitClojureClasses " +
"-jar target/uberjar/" + project.name + "-standalone.jar " +
"-march=compatibility " +
"-H:+UnlockExperimentalVMOptions " +
"-H:IncludeResources=.*.yaml " +
"-H:IncludeResources=logback.xml " +
"-H:Log=registerResource:verbose " +
"-H:Name=target/graalvm/" + project.name + "",
shell=True,
check=True,
)
run(
"sha256sum target/graalvm/c4k-keycloak > target/graalvm/" + project.name + ".sha256",
shell=True,
check=True,
)
run(
"sha512sum target/graalvm/c4k-keycloak > target/graalvm/" + project.name + ".sha512",
shell=True,
check=True,
)
@task
def inst(project):
run(
"lein uberjar",
shell=True,
check=True,
)
package_native(project)
run(
"sudo install -m=755 target/uberjar/" + project.name + "-standalone.jar /usr/local/bin/" + project.name + "-standalone.jar",
shell=True,
check=True,
)
run(
"sudo install -m=755 target/graalvm/" + project.name + " /usr/local/bin/" + project.name + "",
shell=True,
check=True,
)
@task
def upload_clj(project):
run("lein deploy", shell=True, check=True)
@task
def lint(project):
#run(
# "lein eastwood",
# shell=True,
# check=True,
#)
run(
"lein ancient check",
shell=True,
check=True,
)
@task
def patch(project):
linttest(project, "PATCH")
release(project)
@task
def minor(project):
linttest(project, "MINOR")
release(project)
@task
def major(project):
linttest(project, "MAJOR")
release(project)
@task
def dev(project):
linttest(project, "NONE")
@task
def prepare(project):
build = get_devops_build(project)
build.prepare_release()
@task
def tag(project):
build = get_devops_build(project)
build.tag_bump_and_push_release()
@task
def publish_artifacts(project):
build = get_devops_build(project)
build.publish_artifacts()
def release(project):
prepare(project)
tag(project)
def linttest(project, release_type):
build = get_devops_build(project)
build.update_release_type(release_type)
test_clj(project)
test_cljs(project)
test_schema(project)
lint(project)

View file

@ -3,13 +3,13 @@
## clj setup
### install leiningen
```
```bash
sudo apt install leiningen
```
or manually using Instructions on https://leiningen.org/#install
### install vscode + extensions
```
```bash
sudo snap install code
```
or with packages from https://code.visualstudio.com/Download
@ -18,7 +18,7 @@ install extension "Calva: Clojure & ClojureScript Interactive Programming"
## cljs / js-dev setup
```
```bash
sudo apt install npm
sudo npm install -g npx
@ -32,47 +32,44 @@ npx shadow-cljs compile test
### create frontend script
```
```bash
npx shadow-cljs release frontend
```
## graalvm-setup
```
curl -LO https://github.com/graalvm/graalvm-ce-builds/releases/download/vm-21.0.0.2/graalvm-ce-java11-linux-amd64-21.0.0.2.tar.gz
```bash
curl -LO https://github.com/graalvm/graalvm-ce-builds/releases/download/jdk-21.0.2/graalvm-community-jdk-21.0.2_linux-x64_bin.tar.gz
# unpack
tar -xzf graalvm-ce-java11-linux-amd64-21.0.0.2.tar.gz
tar -xzf graalvm-community-jdk-21.0.2_linux-x64_bin.tar.gz
sudo mv graalvm-ce-java11-21.0.0.2 /usr/lib/jvm/
sudo ln -s /usr/lib/jvm/graalvm-ce-java11-21.0.0.2 /usr/lib/jvm/graalvm
sudo ln -s /usr/lib/jvm/graalvm/bin/gu /usr/local/bin
sudo update-alternatives --install /usr/bin/java java /usr/lib/jvm/graalvm/bin/java 2
sudo mv graalvm-community-openjdk-21.0.2+13.1 /usr/lib/jvm/
sudo ln -s /usr/lib/jvm/graalvm-community-openjdk-21.0.2+13.1 /usr/lib/jvm/graalvm-21
sudo ln -s /usr/lib/jvm/graalvm-21/bin/gu /usr/local/bin
sudo update-alternatives --install /usr/bin/java java /usr/lib/jvm/graalvm-21/bin/java 2
sudo update-alternatives --config java
# install native-image in graalvm-ce-java11-linux-amd64-21.0.0.2/bin
sudo gu install native-image
sudo ln -s /usr/lib/jvm/graalvm/bin/native-image /usr/local/bin
sudo ln -s /usr/lib/jvm/graalvm-21/bin/native-image /usr/local/bin
# deps
sudo apt-get install build-essential libz-dev zlib1g-dev
# build
cd ~/repo/dda/c4k-keycloak
cd ~/repo/c4k/c4k-keycloak
lein uberjar
mkdir -p target/graalvm
lein native
# execute
./target/graalvm/c4k-keycloak -h
./target/graalvm/c4k-keycloak src/test/resources/valid-config.edn src/test/resources/valid-auth.edn
./target/graalvm/c4k-keycloak src/test/resources/invalid-config.edn src/test/resources/invalid-auth.edn
./target/graalvm/c4k-cloud -h
./target/graalvm/c4k-cloud src/test/resources/valid-config.edn src/test/resources/valid-auth.edn
./target/graalvm/c4k-cloud src/test/resources/invalid-config.edn src/test/resources/invalid-auth.edn
```
## c4k-setup
### install kubectl
```
```bash
sudo -i
curl -s https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add -
echo "deb http://apt.kubernetes.io/ kubernetes-xenial main" \
@ -83,7 +80,7 @@ kubectl completion bash >> /etc/bash_completion.d/kubernetes
### install kubeconform
```
```bash
curl -Lo /tmp/kubeconform.tar.gz https://github.com/yannh/kubeconform/releases/download/v0.4.7/kubeconform-linux-amd64.tar.gz
tar -xf /tmp/kubeconform.tar.gz
sudo cp kubeconform /usr/local/bin
@ -91,7 +88,7 @@ sudo cp kubeconform /usr/local/bin
### remote access to c4k
```
```bash
scp -r root@devops.test.meissa-gmbh.de:/home/c4k/.kube ~/
ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no root@devops.test.meissa-gmbh.de -L 8002:localhost:8002 -L 6443:192.168.5.1:6443
@ -104,7 +101,7 @@ kubectl get pods
### deploy keycloak
```
```bash
java -jar target/uberjar/c4k-keycloak-standalone.jar valid-config.edn valid-auth.edn | kubeconform --kubernetes-version 1.19.0 --strict --skip Certificate -
java -jar target/uberjar/c4k-keycloak-standalone.jar valid-config.edn my-auth.edn | kubectl apply -f -
```

View file

@ -5,7 +5,7 @@
:url "https://www.apache.org/licenses/LICENSE-2.0.html"}
:dependencies [[org.clojure/clojure "1.11.1"]
[org.clojure/tools.reader "1.3.6"]
[org.domaindrivenarchitecture/c4k-common-clj "6.0.1"]]
[org.domaindrivenarchitecture/c4k-common-clj "6.3.1"]]
:target-path "target/%s/"
:source-paths ["src/main/cljc"
"src/main/clj"]
@ -21,25 +21,14 @@
:uberjar {:aot :all
:main dda.c4k-keycloak.uberjar
:uberjar-name "c4k-keycloak-standalone.jar"
:dependencies [[org.clojure/tools.cli "1.0.214"]
[ch.qos.logback/logback-classic "1.4.5"
:dependencies [[org.clojure/tools.cli "1.0.219"]
[ch.qos.logback/logback-classic "1.4.14"
:exclusions [com.sun.mail/javax.mail]]
[org.slf4j/jcl-over-slf4j "2.0.6"]]}}
[org.slf4j/jcl-over-slf4j "2.0.12"]
[com.github.clj-easy/graal-build-time "1.0.5"]]}}
:release-tasks [["test"]
["vcs" "assert-committed"]
["change" "version" "leiningen.release/bump-version" "release"]
["vcs" "commit"]
["vcs" "tag" "--no-sign"]
["change" "version" "leiningen.release/bump-version"]]
:aliases {"native" ["shell"
"native-image"
"--report-unsupported-elements-at-runtime"
"--initialize-at-build-time"
"-jar" "target/uberjar/c4k-keycloak-standalone.jar"
"-H:ResourceConfigurationFiles=graalvm-resource-config.json"
"-H:Log=registerResource"
"-H:Name=target/graalvm/${:name}"]
"inst" ["shell"
"sh"
"-c"
"lein uberjar && sudo install -m=755 target/uberjar/c4k-keycloak-standalone.jar /usr/local/bin/c4k-keycloak-standalone.jar"]})
["change" "version" "leiningen.release/bump-version"]])

View file

@ -4,7 +4,7 @@
"src/test/cljc"
"src/test/cljs"
"src/test/resources"]
:dependencies [[org.domaindrivenarchitecture/c4k-common-cljs "6.0.1"]
:dependencies [[org.domaindrivenarchitecture/c4k-common-cljs "6.3.1"]
[hickory "0.7.1"]]
:builds {:frontend {:target :browser
:modules {:main {:init-fn dda.c4k-keycloak.browser/init}}

View file

@ -4,6 +4,8 @@
[dda.c4k-common.uberjar :as uberjar]
[dda.c4k-keycloak.core :as core]))
(set! *warn-on-reflection* true)
(defn -main [& cmd-args]
(uberjar/main-common
"c4k-keycloak"

View file

@ -8,7 +8,8 @@
[dda.c4k-common.monitoring :as mon]
[dda.c4k-common.yaml :as yaml]
[dda.c4k-common.postgres :as postgres]
[dda.c4k-keycloak.keycloak :as kc]))
[dda.c4k-keycloak.keycloak :as kc]
[dda.c4k-common.namespace :as ns]))
(def default-storage-class :local-path)
@ -29,15 +30,17 @@
(filter
#(not (nil? %))
(cm/concat-vec
[(postgres/generate-config {:postgres-size :2gb :db-name "keycloak"})
(postgres/generate-secret auth)
(postgres/generate-pvc {:pv-storage-size-gb 30
:pvc-storage-class-name default-storage-class})
(postgres/generate-deployment :postgres-image "postgres:14")
(postgres/generate-service)
(kc/generate-secret auth)
(ns/generate (merge {:namespace "keycloak"} config))
(postgres/generate (merge {:postgres-image "postgres:14"
:postgres-size :2gb
:db-name "keycloak"
:pv-storage-size-gb 30
:pvc-storage-class-name default-storage-class
:namespace "keycloak"})
auth)
[(kc/generate-secret auth)
(kc/generate-service)
(kc/generate-deployment config)]
(kc/generate-ingress config)
(kc/generate-ingress (merge {:namespace "keycloak"} config))
(when (:contains? config :mon-cfg)
(mon/generate (:mon-cfg config) (:mon-auth auth)))))))

View file

@ -1,7 +1,7 @@
(ns dda.c4k-keycloak.keycloak
(:require
[clojure.spec.alpha :as s]
#?(:cljs [shadow.resource :as rc])
#?(:cljs [dda.c4k-common.macros :refer-macros [inline-resources]])
#?(:clj [orchestra.core :refer [defn-spec]]
:cljs [orchestra.core :refer-macros [defn-spec]])
[dda.c4k-common.yaml :as yaml]
@ -23,11 +23,7 @@
#?(:cljs
(defmethod yaml/load-resource :keycloak [resource-name]
(case resource-name
"keycloak/deployment.yaml" (rc/inline "keycloak/deployment.yaml")
"keycloak/secret.yaml" (rc/inline "keycloak/secret.yaml")
"keycloak/service.yaml" (rc/inline "keycloak/service.yaml")
(throw (js/Error. "Undefined Resource!")))))
(get (inline-resources "keycloak") resource-name)))
(defn-spec generate-ingress cp/map-or-seq?
[config config?]

View file

@ -2,7 +2,7 @@ apiVersion: apps/v1
kind: Deployment
metadata:
name: keycloak
namespace: default
namespace: keycloak
labels:
app: keycloak
spec:

View file

@ -2,6 +2,7 @@ apiVersion: v1
kind: Secret
metadata:
name: keycloak-secret
namespace: keycloak
type: Opaque
data:
keycloak-user: admin

View file

@ -4,6 +4,7 @@ metadata:
name: keycloak
labels:
service: keycloak
namespace: keycloak
spec:
ports:
- name: "http"

View file

@ -10,7 +10,7 @@
(deftest should-generate-secret
(is (= {:apiVersion "v1"
:kind "Secret"
:metadata {:name "keycloak-secret"}
:metadata {:name "keycloak-secret", :namespace "keycloak"}
:type "Opaque"
:data
{:keycloak-user "dXNlcg=="
@ -21,7 +21,7 @@
(is (= {:apiVersion "apps/v1",
:kind "Deployment",
:metadata
{:name "keycloak", :namespace "default", :labels {:app "keycloak"}},
{:name "keycloak", :namespace "keycloak", :labels {:app "keycloak"}},
:spec
{:replicas 1,
:selector {:matchLabels {:app "keycloak"}},