meissa
/
c4k-website
6
5
Fork 0
Code Issues Pull Requests Packages Projects Releases 8 Wiki Activity

WIP nginx config generation

Browse Source 
Also make non essential paths generic.
Less need to fiddle in config map strings.
merge-requests/1/merge
erik 2 years ago
parent 0340b8d4e7
commit f0a69e7fda
8 changed files with 96 additions and 221 deletions
Show Stats Download Patch File Download Diff File

24
src/main/cljc/dda/c4k_website/core.cljc
Unescape Escape View File

@ -2,28 +2,16 @@
(:require
[dda.c4k-common.yaml :as yaml]
[dda.c4k-common.common :as cm]
[dda.c4k-website.website :as website]
[dda.c4k-common.postgres :as postgres]))
[dda.c4k-website.website :as website]))
(defn k8s-objects [config]
(let [storage-class (if (contains? config :postgres-data-volume-path) :manual :local-path)]
(cm/concat-vec
(map yaml/to-string
(filter #(not (nil? %))
[(postgres/generate-config {:postgres-size :2gb :db-name "website"})
(postgres/generate-secret config)
(when (contains? config :postgres-data-volume-path)
(postgres/generate-persistent-volume (select-keys config [:postgres-data-volume-path :pv-storage-size-gb])))
(postgres/generate-pvc {:pv-storage-size-gb 5
:pvc-storage-class-name storage-class})
(postgres/generate-deployment {:postgres-image "postgres:14"
:postgres-size :2gb})
(postgres/generate-service)
(website/generate-deployment)
(website/generate-service)
(website/generate-service-ssh)
(website/generate-data-volume config)
(website/generate-appini-env config)
(website/generate-secrets config)
[(website/generate-certificate config)
(website/generate-ingress config)
(website/generate-certificate config)])))))
(website/generate-nginx-configmap config)
(website/generate-nginx-deployment)
(website/generate-nginx-service)
(website/generate-website-content-volume config)])))))

119
src/main/cljc/dda/c4k_website/gitea.cljc
Unescape Escape View File

@ -1,6 +1,7 @@
(ns dda.c4k-website.website
(:require
[clojure.spec.alpha :as s]
[clojure.math :as m]
[clojure.string :as st]
#?(:cljs [shadow.resource :as rc])
#?(:clj [orchestra.core :refer [defn-spec]]
@ -8,10 +9,8 @@
#?(:clj [clojure.edn :as edn]
:cljs [cljs.reader :as edn])
[dda.c4k-common.yaml :as yaml]
[dda.c4k-common.common :as cm]
[dda.c4k-common.base64 :as b64]
[dda.c4k-common.predicate :as pred]
[dda.c4k-common.postgres :as postgres]))
[dda.c4k-common.common :as cm]
[dda.c4k-common.predicate :as pred]))
(defn domain-list?
[input]
@ -19,86 +18,46 @@
(st/blank? input)
(pred/string-of-separated-by? pred/fqdn-string? #"," input)))
(s/def ::default-app-name string?)
(s/def ::fqdn pred/fqdn-string?)
(s/def ::mailer-from pred/bash-env-string?)
(s/def ::mailer-host-port pred/host-and-port-string?)
(s/def ::service-domain-whitelist domain-list?)
(s/def ::service-noreply-address string?)
(s/def ::mailer-user pred/bash-env-string?)
(s/def ::mailer-pw pred/bash-env-string?)
(s/def ::issuer pred/letsencrypt-issuer?)
(s/def ::volume-total-storage-size (partial pred/int-gt-n? 5))
(def config-defaults {:issuer "staging"})
(def config? (s/keys :req-un [::fqdn
::mailer-from
::mailer-host-port
::service-noreply-address]
:opt-un [::issuer
::default-app-name
::service-domain-whitelist]))
(def config? (s/keys :req-un [::fqdn]
:opt-un [::issuer]))
(def auth? (s/keys :req-un [::postgres/postgres-db-user ::postgres/postgres-db-password ::mailer-user ::mailer-pw]))
(def vol? (s/keys :req-un [::volume-total-storage-size]))
(def vol? (s/keys :req-un [::volume-total-storage-size
::number-of-websites]))
(defn data-storage-by-volume-size
[total]
total)
[total number-of-websites-on-node]
(m/floor (/ total number-of-websites-on-node))) ; ToDo: This might be a terrible idea
#?(:cljs
(defmethod yaml/load-resource :website [resource-name]
(case resource-name
"website/appini-env-configmap.yaml" (rc/inline "website/appini-env-configmap.yaml")
"website/deployment.yaml" (rc/inline "website/deployment.yaml")
"website/certificate.yaml" (rc/inline "website/certificate.yaml")
"website/ingress.yaml" (rc/inline "website/ingress.yaml")
"website/secrets.yaml" (rc/inline "website/secrets.yaml")
"website/service.yaml" (rc/inline "website/service.yaml")
"website/service-ssh.yaml" (rc/inline "website/service-ssh.yaml")
"website/datavolume.yaml" (rc/inline "website/datavolume.yaml")
"website/nginx-configmap.yaml" (rc/inline "website/nginx-configmap.yaml")
"website/nginx-deployment.yaml" (rc/inline "website/nginx-deployment.yaml")
"website/nginx-service.yaml" (rc/inline "website/nginx-service.yaml")
"website/website-content-volume.yaml" (rc/inline "website/website-content-volume.yaml")
(throw (js/Error. "Undefined Resource!")))))
#?(:cljs
(defmethod yaml/load-as-edn :website [resource-name]
(yaml/from-string (yaml/load-resource resource-name))))
(defn-spec generate-appini-env pred/map-or-seq?
(defn-spec generate-certificate pred/map-or-seq?
[config config?]
(let [{:keys [default-app-name
fqdn
mailer-from
mailer-host-port
service-domain-whitelist
service-noreply-address]
:or {default-app-name "website instance"
service-domain-whitelist fqdn}}
config]
(->
(yaml/load-as-edn "website/appini-env-configmap.yaml")
(cm/replace-all-matching-values-by-new-value "APPNAME" default-app-name)
(cm/replace-all-matching-values-by-new-value "FQDN" fqdn)
(cm/replace-all-matching-values-by-new-value "URL" (str "https://" fqdn))
(cm/replace-all-matching-values-by-new-value "FROM" mailer-from)
(cm/replace-all-matching-values-by-new-value "HOSTANDPORT" mailer-host-port)
(cm/replace-all-matching-values-by-new-value "WHITELISTDOMAINS" service-domain-whitelist)
(cm/replace-all-matching-values-by-new-value "NOREPLY" service-noreply-address))))
(defn-spec generate-secrets pred/map-or-seq?
[auth auth?]
(let [{:keys [postgres-db-user
postgres-db-password
mailer-user
mailer-pw]} auth]
(let [{:keys [fqdn issuer]
:or {issuer "staging"}} config
letsencrypt-issuer (name issuer)]
(->
(yaml/load-as-edn "website/secrets.yaml")
(cm/replace-all-matching-values-by-new-value "DBUSER" (b64/encode postgres-db-user))
(cm/replace-all-matching-values-by-new-value "DBPW" (b64/encode postgres-db-password))
(cm/replace-all-matching-values-by-new-value "MAILERUSER" (b64/encode mailer-user))
(cm/replace-all-matching-values-by-new-value "MAILERPW" (b64/encode mailer-pw)))))
(yaml/load-as-edn "website/certificate.yaml")
(assoc-in [:spec :issuerRef :name] letsencrypt-issuer)
(cm/replace-all-matching-values-by-new-value "FQDN" fqdn))))
(defn-spec generate-ingress pred/map-or-seq?
[config config?]
@ -107,32 +66,26 @@
(yaml/load-as-edn "website/ingress.yaml")
(cm/replace-all-matching-values-by-new-value "FQDN" fqdn))))
(defn-spec generate-certificate pred/map-or-seq?
(defn-spec generate-nginx-configmap pred/map-or-seq?
[config config?]
(let [{:keys [fqdn issuer]
:or {issuer "staging"}} config
letsencrypt-issuer (name issuer)]
(let [{:keys [fqdn]} config]
(->
(yaml/load-as-edn "website/certificate.yaml")
(assoc-in [:spec :issuerRef :name] letsencrypt-issuer)
(cm/replace-all-matching-values-by-new-value "FQDN" fqdn))))
(yaml/load-as-edn "website/nginx-configmap.yaml")
(cm/replace-all-matching-values-by-new-value "FQDN" (str fqdn ";"))
)))
(defn-spec generate-data-volume pred/map-or-seq?
[config vol?]
(let [{:keys [volume-total-storage-size]} config
data-storage-size (data-storage-by-volume-size volume-total-storage-size)]
(->
(yaml/load-as-edn "website/datavolume.yaml")
(cm/replace-all-matching-values-by-new-value "DATASTORAGESIZE" (str (str data-storage-size) "Gi")))))
(defn-spec generate-deployment pred/map-or-seq?
(defn-spec generate-nginx-deployment pred/map-or-seq?
[]
(yaml/load-as-edn "website/deployment.yaml"))
(yaml/load-as-edn "website/nginx-deployment.yaml"))
(defn-spec generate-service pred/map-or-seq?
(defn-spec generate-nginx-service pred/map-or-seq?
[]
(yaml/load-as-edn "website/service.yaml"))
(yaml/load-as-edn "website/nginx-service.yaml"))
(defn-spec generate-service-ssh pred/map-or-seq?
[]
(yaml/load-as-edn "website/service-ssh.yaml"))
(defn-spec generate-website-content-volume pred/map-or-seq?
[config vol?]
(let [{:keys [volume-total-storage-size number-of-websites]} config
data-storage-size (data-storage-by-volume-size volume-total-storage-size number-of-websites)]
(->
(yaml/load-as-edn "website/website-content-volume.yaml")
(cm/replace-all-matching-values-by-new-value "WEBSITESTORAGESIZE" (str (str data-storage-size) "Gi")))))

64
src/main/cljs/dda/c4k_website/browser.cljs
Unescape Escape View File

@ -4,8 +4,7 @@
[clojure.tools.reader.edn :as edn]
[dda.c4k-website.core :as core]
[dda.c4k-website.website :as website]
[dda.c4k-common.browser :as br]
[dda.c4k-common.postgres :as pgc]
[dda.c4k-common.browser :as br]
[dda.c4k-common.common :as cm]))
(defn generate-group
@ -30,27 +29,13 @@
(generate-group
"domain"
(cm/concat-vec
(br/generate-input-field "fqdn" "Your fqdn:" "repo.test.de")
(br/generate-input-field "mailer-from" "Your mailer email address:" "test@test.de")
(br/generate-input-field "mailer-host-port" "Your mailer host with port:" "test.de:123")
(br/generate-input-field "service-noreply-address" "Your noreply domain:" "test.de")
(br/generate-input-field "issuer" "(Optional) Your issuer prod/staging:" "")
(br/generate-input-field "app-name" "(Optional) Your app name:" "")
(br/generate-input-field "domain-whitelist" "(Optional) Domain whitelist for registration email-addresses:" "")))
(br/generate-input-field "fqdn" "Your fqdn:" "deineWebsite.de")
(br/generate-input-field "issuer" "(Optional) Your issuer prod/staging:" "")))
(generate-group
"provider"
(cm/concat-vec
(br/generate-input-field "volume-total-storage-size" "Your website volume-total-storage-size:" "20")
(br/generate-input-field "postgres-data-volume-path" "(Optional) Your postgres-data-volume-path if Persistent Volumes are not generated by an Operator:" "")))
(generate-group
"credentials"
(br/generate-text-area
"auth" "Your auth.edn:"
"{:postgres-db-user \"website\"
:postgres-db-password \"website-db-password\"
:mailer-user \"test@test.de\"
:mailer-pw \"mail-test-password\"}"
"5"))
(br/generate-input-field "number-of-websites" "The Number of websites running on your cluster" "5")))
[(br/generate-br)]
(br/generate-button "generate-button" "Generate c4k yaml")))]
(br/generate-output "c4k-website-output" "Your c4k deployment.yaml:" "25")))
@ -63,37 +48,20 @@
(generate-content)})
(defn config-from-document []
(let [postgres-data-volume-path (br/get-content-from-element "postgres-data-volume-path" :optional true)
issuer (br/get-content-from-element "issuer" :optional true)
app-name (br/get-content-from-element "app-name" :optional true)
domain-whitelist (br/get-content-from-element "domain-whitelist" :optional true)]
(let [issuer (br/get-content-from-element "issuer" :optional true)]
(merge
{:fqdn (br/get-content-from-element "fqdn")
:mailer-from (br/get-content-from-element "mailer-from")
:mailer-host-port (br/get-content-from-element "mailer-host-port")
:service-noreply-address (br/get-content-from-element "service-noreply-address")
:volume-total-storage-size (br/get-content-from-element "volume-total-storage-size" :deserializer js/parseInt)}
(when (not (st/blank? postgres-data-volume-path))
{:postgres-data-volume-path postgres-data-volume-path})
{:fqdn (br/get-content-from-element "fqdn")
:volume-total-storage-size (br/get-content-from-element "volume-total-storage-size" :deserializer js/parseInt)
:number-of-websites (br/get-content-from-element "number-of-websites" :deserializer js/parseInt)}
(when (not (st/blank? issuer))
{:issuer issuer})
(when (not (st/blank? app-name))
{:default-app-name app-name})
(when (not (st/blank? domain-whitelist))
{:service-domain-whitelist domain-whitelist})
{:issuer issuer})
)))
(defn validate-all! []
(br/validate! "fqdn" ::website/fqdn)
(br/validate! "mailer-from" ::website/mailer-from)
(br/validate! "mailer-host-port" ::website/mailer-host-port)
(br/validate! "service-noreply-address" ::website/service-noreply-address)
(br/validate! "fqdn" ::website/fqdn)
(br/validate! "issuer" ::website/issuer :optional true)
(br/validate! "app-name" ::website/default-app-name :optional true)
(br/validate! "domain-whitelist" ::website/service-domain-whitelist :optional true)
(br/validate! "postgres-data-volume-path" ::pgc/postgres-data-volume-path :optional true)
(br/validate! "volume-total-storage-size" ::website/volume-total-storage-size :deserializer js/parseInt)
(br/validate! "auth" website/auth? :deserializer edn/read-string)
(br/validate! "number-of-websites" ::website/number-of-websites :deserializer js/parseInt)
(br/set-form-validated!))
(defn add-validate-listener [name]
@ -113,13 +81,7 @@
website/config-defaults
core/k8s-objects)
(br/set-output!)))))
(add-validate-listener "fqdn")
(add-validate-listener "mailer-from")
(add-validate-listener "mailer-host-port")
(add-validate-listener "service-noreply-address")
(add-validate-listener "app-name")
(add-validate-listener "domain-whitelist")
(add-validate-listener "postgres-data-volume-path")
(add-validate-listener "fqdn")
(add-validate-listener "volume-total-storage-size")
(add-validate-listener "issuer")
(add-validate-listener "auth"))
(add-validate-listener "number-of-websites"))

1
src/main/resources/website/ingress.yaml
Unescape Escape View File

@ -22,4 +22,3 @@ spec:
name: website-service
port:
number: 80

8
src/main/resources/website/nginx-configmap.yaml
Unescape Escape View File

@ -44,7 +44,7 @@ data:
# it might be a good idea to set a common reverse proxy
# which points to the ingress?
include /etc/nginx/conf.d/FQDN.conf; # should be replaced by c4k
include /etc/nginx/conf.d/website.conf;
}
mime.types: |
@ -96,7 +96,7 @@ data:
video/x-ms-asf asx asf;
video/x-mng mng;
}
FQDN.conf: |
website.conf: |
server {
listen 80 default_server;
@ -107,7 +107,7 @@ data:
ssl_certificate /etc/certs/tls.crt;
ssl_certificate_key /etc/certs/tls.key;
server_name FQDN;
server_name FQDN
# security headers
add_header Strict-Transport-Security 'max-age=31536000; includeSubDomains; preload';
@ -119,7 +119,7 @@ data:
# maybe need to add:
# add_header Permissions-Policy "permissions here";
root /var/www/html/FQDN;
root /var/www/html/website/;
# root /usr/share/nginx/html/; # testing purposes
index index.html;

10
src/main/resources/website/nginx-deployment.yaml
Unescape Escape View File

@ -24,7 +24,7 @@ spec:
name: nginx-conf
- mountPath: /var/log/nginx
name: log
- mountPath: /var/www/html/FQDN
- mountPath: /var/www/html/website
name: website-content-volume
- mountPath: /etc/certs
name: website-cert
@ -32,14 +32,14 @@ spec:
volumes:
- name: nginx-conf
configMap:
name: nginx-conf # place ConfigMap `nginx-conf` on /etc/nginx
name: nginx-conf
items:
- key: nginx.conf
path: nginx.conf
- key: FQDN.conf
path: conf.d/FQDN.conf
- key: website.conf
path: conf.d/website.conf
- key: mime.types
path: mime.types # dig directory
path: mime.types
- name: log
emptyDir: {}
- name: website-content-volume

2
src/main/resources/website/testconfig.yaml
Unescape Escape View File

@ -259,4 +259,4 @@ spec:
- repo.test.meissa.de
issuerRef:
name: staging
kind: ClusterIssuer
kind: ClusterIssuer

89
src/test/cljc/dda/c4k_website/website_test.cljc
Unescape Escape View File

@ -7,67 +7,40 @@
[dda.c4k-common.base64 :as b64]
[dda.c4k-website.website :as cut]))
(st/instrument `cut/generate-appini-env)
(st/instrument `cut/generate-ingress)
(st/instrument `cut/generate-secrets)
(deftest should-generate-appini-env
(is (= {:APP_NAME-c1 "",
:APP_NAME-c2 "test website",
:website__mailer__FROM-c1 "",
:website__mailer__FROM-c2 "test@test.com",
:website__mailer__HOST-c1 "m.t.de:123",
:website__mailer__HOST-c2 "mail.test.com:123",
:website__server__DOMAIN-c1 "test.de",
:website__server__DOMAIN-c2 "test.com",
:website__server__ROOT_URL-c1 "https://test.de",
:website__server__ROOT_URL-c2 "https://test.com",
:website__server__SSH_DOMAIN-c1 "test.de",
:website__server__SSH_DOMAIN-c2 "test.com",
:website__service__EMAIL_DOMAIN_WHITELIST-c1 "adb.de",
:website__service__EMAIL_DOMAIN_WHITELIST-c2 "test.com,test.net",
:website__service__NO_REPLY_ADDRESS-c1 "",
:website__service__NO_REPLY_ADDRESS-c2 "noreply@test.com"}
(th/map-diff (cut/generate-appini-env {:default-app-name ""
:fqdn "test.de"
:mailer-from ""
:mailer-host-port "m.t.de:123"
:service-domain-whitelist "adb.de"
:service-noreply-address ""
})
(cut/generate-appini-env {:default-app-name "test website"
:fqdn "test.com"
:mailer-from "test@test.com"
:mailer-host-port "mail.test.com:123"
:service-domain-whitelist "test.com,test.net"
:service-noreply-address "noreply@test.com"
})))))
(st/instrument `cut/generate-certificate)
(st/instrument `cut/generate-ingress)
(st/instrument `cut/generate-nginx-configmap)
(st/instrument `cut/generate-website-content-volume)
(deftest should-generate-certificate
(is (= {:name-c2 "prod", :name-c1 "staging"}
(th/map-diff (cut/generate-certificate {})
(cut/generate-certificate {:issuer "prod"})))))
(th/map-diff (cut/generate-certificate {:fqdn "test.de"})
(cut/generate-certificate {:issuer "prod"
:fqdn "test.de"})))))
(deftest should-generate-secret
(is (= {:website__database__USER-c1 "",
:website__database__USER-c2 (b64/encode "pg-user"),
:website__database__PASSWD-c1 "",
:website__database__PASSWD-c2 (b64/encode "pg-pw"),
:website__mailer__USER-c1 "",
:website__mailer__USER-c2 (b64/encode "maileruser"),
:website__mailer__PASSWD-c1 "",
:website__mailer__PASSWD-c2 (b64/encode "mailerpw")}
(th/map-diff (cut/generate-secrets {:postgres-db-user ""
:postgres-db-password ""
:mailer-user ""
:mailer-pw ""})
(cut/generate-secrets {:postgres-db-user "pg-user"
:postgres-db-password "pg-pw"
:mailer-user "maileruser"
:mailer-pw "mailerpw"})))))
(deftest should-generate-ingress
(is (= {:hosts-c1 "test.de",
:hosts-c2 "test.com",
:host-c1 "test.de",
:host-c2 "test.com"}
(th/map-diff (cut/generate-ingress {:fqdn "test.de"
})
(cut/generate-ingress {:fqdn "test.com"
})))))
(deftest should-generate-nginx-configmap
(is (= {:server_name-c1 "test.de",
:server_name-c2 "test.com"}
(th/map-diff (cut/generate-appini-env {:fqdn "test.de"
})
(cut/generate-appini-env {:fqdn "test.com"
})))))
(deftest should-generate-data-volume
(is (= {:storage-c1 "1Gi",
:storage-c2 "15Gi"}
(th/map-diff (cut/generate-data-volume {:volume-total-storage-size 1})
(cut/generate-data-volume {:volume-total-storage-size 15})))))
(deftest should-generate-website-content-volume
(is (= {:storage-c1 "2Gi",
:storage-c2 "10Gi"}
(th/map-diff (cut/generate-website-content-volume {:volume-total-storage-size 10
:number-of-websites 5})
(cut/generate-website-content-volume {:volume-total-storage-size 50
:number-of-websites 5})))))
Write Preview
Loading…
Cancel
Save