WIP nginx config generation

Also make non essential paths generic.
Less need to fiddle in config map strings.

src/main/cljc/dda/c4k_website/core.cljc

@@ -2,28 +2,16 @@
  (:require
   [dda.c4k-common.yaml :as yaml]
   [dda.c4k-common.common :as cm]
-  [dda.c4k-website.website :as website]
+  [dda.c4k-website.website :as website]))
-  [dda.c4k-common.postgres :as postgres]))
 
 (defn k8s-objects [config]
   (let [storage-class (if (contains? config :postgres-data-volume-path) :manual :local-path)]
     (cm/concat-vec
      (map yaml/to-string
           (filter #(not (nil? %))
-                  [(postgres/generate-config {:postgres-size :2gb :db-name "website"})
+                  [(website/generate-certificate config)
-                   (postgres/generate-secret config)
-                   (when (contains? config :postgres-data-volume-path)
-                     (postgres/generate-persistent-volume (select-keys config [:postgres-data-volume-path :pv-storage-size-gb])))
-                   (postgres/generate-pvc {:pv-storage-size-gb 5
-                                           :pvc-storage-class-name storage-class})
-                   (postgres/generate-deployment {:postgres-image "postgres:14"
-                                                  :postgres-size :2gb})
-                   (postgres/generate-service)
-                   (website/generate-deployment)
-                   (website/generate-service)
-                   (website/generate-service-ssh)                   
-                   (website/generate-data-volume config)
-                   (website/generate-appini-env config)
-                   (website/generate-secrets config)
                    (website/generate-ingress config)
-                   (website/generate-certificate config)])))))
+                   (website/generate-nginx-configmap config)
+                   (website/generate-nginx-deployment)
+                   (website/generate-nginx-service)                                  
+                   (website/generate-website-content-volume config)])))))

src/main/cljc/dda/c4k_website/gitea.cljc

@@ -1,6 +1,7 @@
 (ns dda.c4k-website.website
   (:require
    [clojure.spec.alpha :as s]
+   [clojure.math :as m]
    [clojure.string :as st]
    #?(:cljs [shadow.resource :as rc])
    #?(:clj [orchestra.core :refer [defn-spec]]
@@ -8,10 +9,8 @@
    #?(:clj [clojure.edn :as edn]
       :cljs [cljs.reader :as edn])
    [dda.c4k-common.yaml :as yaml]
-   [dda.c4k-common.common :as cm]
+   [dda.c4k-common.common :as cm]   
-   [dda.c4k-common.base64 :as b64]
+   [dda.c4k-common.predicate :as pred]))
-   [dda.c4k-common.predicate :as pred]
-   [dda.c4k-common.postgres :as postgres]))
 
 (defn domain-list?
   [input]
@@ -19,94 +18,37 @@
    (st/blank? input)
    (pred/string-of-separated-by? pred/fqdn-string? #"," input)))
 
-(s/def ::default-app-name string?)
 (s/def ::fqdn pred/fqdn-string?)
-(s/def ::mailer-from pred/bash-env-string?)
-(s/def ::mailer-host-port pred/host-and-port-string?)
-(s/def ::service-domain-whitelist domain-list?)
-(s/def ::service-noreply-address string?)
-(s/def ::mailer-user pred/bash-env-string?)
-(s/def ::mailer-pw pred/bash-env-string?)
 (s/def ::issuer pred/letsencrypt-issuer?)
 (s/def ::volume-total-storage-size (partial pred/int-gt-n? 5))
 
 (def config-defaults {:issuer "staging"})
 
-(def config? (s/keys :req-un [::fqdn 
+(def config? (s/keys :req-un [::fqdn]
-                              ::mailer-from 
+                     :opt-un [::issuer]))
-                              ::mailer-host-port 
-                              ::service-noreply-address]
-                     :opt-un [::issuer 
-                              ::default-app-name 
-                              ::service-domain-whitelist]))
 
-(def auth? (s/keys :req-un [::postgres/postgres-db-user ::postgres/postgres-db-password ::mailer-user ::mailer-pw]))
+(def vol? (s/keys :req-un [::volume-total-storage-size
-
+                           ::number-of-websites]))
-(def vol? (s/keys :req-un [::volume-total-storage-size]))
 
 (defn data-storage-by-volume-size
-  [total]
+  [total number-of-websites-on-node]
-  total)
+  (m/floor (/ total number-of-websites-on-node))) ; ToDo: This might be a terrible idea
-
 
 #?(:cljs
    (defmethod yaml/load-resource :website [resource-name]
      (case resource-name
-       "website/appini-env-configmap.yaml" (rc/inline "website/appini-env-configmap.yaml")
-       "website/deployment.yaml" (rc/inline "website/deployment.yaml")
        "website/certificate.yaml" (rc/inline "website/certificate.yaml")
        "website/ingress.yaml" (rc/inline "website/ingress.yaml")
-       "website/secrets.yaml" (rc/inline "website/secrets.yaml")
+       "website/nginx-configmap.yaml" (rc/inline "website/nginx-configmap.yaml")
-       "website/service.yaml" (rc/inline "website/service.yaml")
+       "website/nginx-deployment.yaml" (rc/inline "website/nginx-deployment.yaml")
-       "website/service-ssh.yaml" (rc/inline "website/service-ssh.yaml")       
+       "website/nginx-service.yaml" (rc/inline "website/nginx-service.yaml")              
-       "website/datavolume.yaml" (rc/inline "website/datavolume.yaml")
+       "website/website-content-volume.yaml" (rc/inline "website/website-content-volume.yaml")
        (throw (js/Error. "Undefined Resource!")))))
 
 #?(:cljs
    (defmethod yaml/load-as-edn :website [resource-name]
      (yaml/from-string (yaml/load-resource resource-name))))
 
-(defn-spec generate-appini-env pred/map-or-seq?  
-  [config config?]
-  (let [{:keys [default-app-name
-                fqdn
-                mailer-from
-                mailer-host-port
-                service-domain-whitelist
-                service-noreply-address]
-         :or {default-app-name "website instance"
-              service-domain-whitelist fqdn}}
-        config]
-    (->
-     (yaml/load-as-edn "website/appini-env-configmap.yaml")
-     (cm/replace-all-matching-values-by-new-value "APPNAME" default-app-name)
-     (cm/replace-all-matching-values-by-new-value "FQDN" fqdn)
-     (cm/replace-all-matching-values-by-new-value "URL" (str "https://" fqdn))
-     (cm/replace-all-matching-values-by-new-value "FROM" mailer-from)
-     (cm/replace-all-matching-values-by-new-value "HOSTANDPORT" mailer-host-port)
-     (cm/replace-all-matching-values-by-new-value "WHITELISTDOMAINS" service-domain-whitelist)
-     (cm/replace-all-matching-values-by-new-value "NOREPLY" service-noreply-address))))
-
-(defn-spec generate-secrets pred/map-or-seq?
-  [auth auth?]
-  (let [{:keys [postgres-db-user 
-                postgres-db-password 
-                mailer-user 
-                mailer-pw]} auth]
-    (->
-     (yaml/load-as-edn "website/secrets.yaml")
-     (cm/replace-all-matching-values-by-new-value "DBUSER" (b64/encode postgres-db-user))
-     (cm/replace-all-matching-values-by-new-value "DBPW" (b64/encode postgres-db-password))
-     (cm/replace-all-matching-values-by-new-value "MAILERUSER" (b64/encode mailer-user))
-     (cm/replace-all-matching-values-by-new-value "MAILERPW" (b64/encode mailer-pw)))))
-
-(defn-spec generate-ingress pred/map-or-seq?
-  [config config?]
-  (let [{:keys [fqdn issuer]} config]
-    (->
-     (yaml/load-as-edn "website/ingress.yaml")
-     (cm/replace-all-matching-values-by-new-value "FQDN" fqdn))))
-
 (defn-spec generate-certificate pred/map-or-seq?
   [config config?]
   (let [{:keys [fqdn issuer]
@@ -117,22 +59,33 @@
      (assoc-in [:spec :issuerRef :name] letsencrypt-issuer)
      (cm/replace-all-matching-values-by-new-value "FQDN" fqdn))))
 
-(defn-spec generate-data-volume pred/map-or-seq?
+(defn-spec generate-ingress pred/map-or-seq?
+  [config config?]
+  (let [{:keys [fqdn issuer]} config]
+    (->
+     (yaml/load-as-edn "website/ingress.yaml")
+     (cm/replace-all-matching-values-by-new-value "FQDN" fqdn))))
+
+(defn-spec generate-nginx-configmap pred/map-or-seq?  
+  [config config?]
+  (let [{:keys [fqdn]} config]
+    (->
+     (yaml/load-as-edn "website/nginx-configmap.yaml")     
+     (cm/replace-all-matching-values-by-new-value "FQDN" (str fqdn ";"))     
+     )))
+
+(defn-spec generate-nginx-deployment pred/map-or-seq?
+  []
+  (yaml/load-as-edn "website/nginx-deployment.yaml"))
+
+(defn-spec generate-nginx-service pred/map-or-seq?
+  []
+  (yaml/load-as-edn "website/nginx-service.yaml"))
+
+(defn-spec generate-website-content-volume pred/map-or-seq?
   [config vol?]
-  (let [{:keys [volume-total-storage-size]} config        
+  (let [{:keys [volume-total-storage-size number-of-websites]} config
-        data-storage-size (data-storage-by-volume-size volume-total-storage-size)]
+        data-storage-size (data-storage-by-volume-size volume-total-storage-size number-of-websites)]
-    (->     
+    (->
-     (yaml/load-as-edn "website/datavolume.yaml")
+     (yaml/load-as-edn "website/website-content-volume.yaml")
-     (cm/replace-all-matching-values-by-new-value "DATASTORAGESIZE" (str (str data-storage-size) "Gi")))))
+     (cm/replace-all-matching-values-by-new-value "WEBSITESTORAGESIZE" (str (str data-storage-size) "Gi")))))
-
-(defn-spec generate-deployment pred/map-or-seq?
-  []
-  (yaml/load-as-edn "website/deployment.yaml"))
-
-(defn-spec generate-service pred/map-or-seq?
-  []
-  (yaml/load-as-edn "website/service.yaml"))
-
-(defn-spec generate-service-ssh pred/map-or-seq?
-  []
-  (yaml/load-as-edn "website/service-ssh.yaml"))

src/main/cljs/dda/c4k_website/browser.cljs

@@ -4,8 +4,7 @@
    [clojure.tools.reader.edn :as edn]
    [dda.c4k-website.core :as core]
    [dda.c4k-website.website :as website]
-   [dda.c4k-common.browser :as br]
+   [dda.c4k-common.browser :as br]   
-   [dda.c4k-common.postgres :as pgc]
    [dda.c4k-common.common :as cm]))
 
 (defn generate-group
@@ -30,27 +29,13 @@
       (generate-group
        "domain"
        (cm/concat-vec
-        (br/generate-input-field "fqdn" "Your fqdn:" "repo.test.de")
+        (br/generate-input-field "fqdn" "Your fqdn:" "deineWebsite.de")
-        (br/generate-input-field "mailer-from" "Your mailer email address:" "test@test.de")
+        (br/generate-input-field "issuer" "(Optional) Your issuer prod/staging:" "")))
-        (br/generate-input-field "mailer-host-port" "Your mailer host with port:" "test.de:123")
-        (br/generate-input-field "service-noreply-address" "Your noreply domain:" "test.de")
-        (br/generate-input-field "issuer" "(Optional) Your issuer prod/staging:" "")
-        (br/generate-input-field "app-name" "(Optional) Your app name:" "")
-        (br/generate-input-field "domain-whitelist" "(Optional) Domain whitelist for registration email-addresses:" "")))
       (generate-group
        "provider"
        (cm/concat-vec
         (br/generate-input-field "volume-total-storage-size" "Your website volume-total-storage-size:" "20")
-        (br/generate-input-field "postgres-data-volume-path" "(Optional) Your postgres-data-volume-path if Persistent Volumes are not generated by an Operator:" "")))
+        (br/generate-input-field "number-of-websites" "The Number of websites running on your cluster" "5")))
-      (generate-group
-       "credentials"
-       (br/generate-text-area
-        "auth" "Your auth.edn:"
-        "{:postgres-db-user \"website\"
-         :postgres-db-password \"website-db-password\"
-         :mailer-user \"test@test.de\"
-         :mailer-pw \"mail-test-password\"}"
-        "5"))
       [(br/generate-br)]
       (br/generate-button "generate-button" "Generate c4k yaml")))]
    (br/generate-output "c4k-website-output" "Your c4k deployment.yaml:" "25")))
@@ -63,37 +48,20 @@
    (generate-content)})
 
 (defn config-from-document []
-  (let [postgres-data-volume-path (br/get-content-from-element "postgres-data-volume-path" :optional true)
+  (let [issuer (br/get-content-from-element "issuer" :optional true)]
-        issuer (br/get-content-from-element "issuer" :optional true)
-        app-name (br/get-content-from-element "app-name" :optional true)
-        domain-whitelist (br/get-content-from-element "domain-whitelist" :optional true)]
     (merge
-     {:fqdn (br/get-content-from-element "fqdn")
+     {:fqdn (br/get-content-from-element "fqdn")      
-      :mailer-from (br/get-content-from-element "mailer-from")
+      :volume-total-storage-size (br/get-content-from-element "volume-total-storage-size" :deserializer js/parseInt)
-      :mailer-host-port (br/get-content-from-element "mailer-host-port")
+      :number-of-websites (br/get-content-from-element "number-of-websites" :deserializer js/parseInt)}     
-      :service-noreply-address (br/get-content-from-element "service-noreply-address")
-      :volume-total-storage-size (br/get-content-from-element "volume-total-storage-size" :deserializer js/parseInt)}
-     (when (not (st/blank? postgres-data-volume-path))
-       {:postgres-data-volume-path postgres-data-volume-path})
      (when (not (st/blank? issuer))
-       {:issuer issuer})
+       {:issuer issuer})          
-     (when (not (st/blank? app-name))
-       {:default-app-name app-name})
-     (when (not (st/blank? domain-whitelist))
-       {:service-domain-whitelist domain-whitelist})
      )))
 
 (defn validate-all! []
-  (br/validate! "fqdn" ::website/fqdn)
+  (br/validate! "fqdn" ::website/fqdn)  
-  (br/validate! "mailer-from" ::website/mailer-from)
-  (br/validate! "mailer-host-port" ::website/mailer-host-port)
-  (br/validate! "service-noreply-address" ::website/service-noreply-address)
   (br/validate! "issuer" ::website/issuer :optional true)
-  (br/validate! "app-name" ::website/default-app-name :optional true)
-  (br/validate! "domain-whitelist" ::website/service-domain-whitelist :optional true)
-  (br/validate! "postgres-data-volume-path" ::pgc/postgres-data-volume-path :optional true)
   (br/validate! "volume-total-storage-size" ::website/volume-total-storage-size :deserializer js/parseInt)
-  (br/validate! "auth" website/auth? :deserializer edn/read-string)
+  (br/validate! "number-of-websites" ::website/number-of-websites :deserializer js/parseInt)  
   (br/set-form-validated!))
 
 (defn add-validate-listener [name]
@@ -113,13 +81,7 @@
                                    website/config-defaults
                                    core/k8s-objects)
                                (br/set-output!)))))
-  (add-validate-listener "fqdn")
+  (add-validate-listener "fqdn")  
-  (add-validate-listener "mailer-from")
-  (add-validate-listener "mailer-host-port")
-  (add-validate-listener "service-noreply-address")
-  (add-validate-listener "app-name")
-  (add-validate-listener "domain-whitelist")
-  (add-validate-listener "postgres-data-volume-path")
   (add-validate-listener "volume-total-storage-size")
   (add-validate-listener "issuer")
-  (add-validate-listener "auth"))
+  (add-validate-listener "number-of-websites"))

src/main/resources/website/ingress.yaml

@@ -22,4 +22,3 @@ spec:
                 name: website-service
                 port:
                   number: 80
-                  

src/main/resources/website/nginx-configmap.yaml

@@ -44,7 +44,7 @@ data:
       # it might be a good idea to set a common reverse proxy 
       # which points to the ingress?
 
-      include /etc/nginx/conf.d/FQDN.conf; # should be replaced by c4k
+      include /etc/nginx/conf.d/website.conf;
     }
     
   mime.types: |
@@ -96,7 +96,7 @@ data:
       video/x-ms-asf                        asx asf;
       video/x-mng                           mng;
     }
-  FQDN.conf: | 
+  website.conf: | 
     server {
 
       listen 80 default_server;
@@ -107,7 +107,7 @@ data:
       ssl_certificate /etc/certs/tls.crt;
       ssl_certificate_key /etc/certs/tls.key;
 
-      server_name FQDN;
+      server_name FQDN
 
       # security headers
       add_header Strict-Transport-Security 'max-age=31536000; includeSubDomains; preload';
@@ -119,7 +119,7 @@ data:
       # maybe need to add:
       # add_header Permissions-Policy "permissions here";
 
-      root /var/www/html/FQDN;
+      root /var/www/html/website/;
       # root /usr/share/nginx/html/; # testing purposes
 
       index index.html;

src/main/resources/website/nginx-deployment.yaml

@@ -24,7 +24,7 @@ spec:
           name: nginx-conf
         - mountPath: /var/log/nginx
           name: log                    
-        - mountPath: /var/www/html/FQDN
+        - mountPath: /var/www/html/website
           name: website-content-volume
         - mountPath: /etc/certs
           name: website-cert
@@ -32,14 +32,14 @@ spec:
       volumes:
         - name: nginx-conf
           configMap:
-            name: nginx-conf # place ConfigMap `nginx-conf` on /etc/nginx
+            name: nginx-conf 
             items:
               - key: nginx.conf
                 path: nginx.conf
-              - key: FQDN.conf
+              - key: website.conf
-                path: conf.d/FQDN.conf
+                path: conf.d/website.conf
               - key: mime.types
-                path: mime.types # dig directory
+                path: mime.types 
         - name: log
           emptyDir: {}
         - name: website-content-volume

src/main/resources/website/testconfig.yaml

@@ -259,4 +259,4 @@ spec:
   - repo.test.meissa.de
   issuerRef:
     name: staging
-    kind: ClusterIssuer
+    kind: ClusterIssuer

src/test/cljc/dda/c4k_website/website_test.cljc

@@ -7,67 +7,40 @@
    [dda.c4k-common.base64 :as b64]
    [dda.c4k-website.website :as cut]))
 
-(st/instrument `cut/generate-appini-env)
-(st/instrument `cut/generate-ingress)
-(st/instrument `cut/generate-secrets)
 
-(deftest should-generate-appini-env
+(st/instrument `cut/generate-certificate)
-  (is (= {:APP_NAME-c1 "",
+(st/instrument `cut/generate-ingress)
-          :APP_NAME-c2 "test website",
+(st/instrument `cut/generate-nginx-configmap)
-          :website__mailer__FROM-c1 "",
+(st/instrument `cut/generate-website-content-volume)
-          :website__mailer__FROM-c2 "test@test.com",
-          :website__mailer__HOST-c1 "m.t.de:123",
-          :website__mailer__HOST-c2 "mail.test.com:123",
-          :website__server__DOMAIN-c1 "test.de",
-          :website__server__DOMAIN-c2 "test.com",
-          :website__server__ROOT_URL-c1 "https://test.de",
-          :website__server__ROOT_URL-c2 "https://test.com",
-          :website__server__SSH_DOMAIN-c1 "test.de",
-          :website__server__SSH_DOMAIN-c2 "test.com",
-          :website__service__EMAIL_DOMAIN_WHITELIST-c1 "adb.de",
-          :website__service__EMAIL_DOMAIN_WHITELIST-c2 "test.com,test.net",
-          :website__service__NO_REPLY_ADDRESS-c1 "",
-          :website__service__NO_REPLY_ADDRESS-c2 "noreply@test.com"}
-         (th/map-diff (cut/generate-appini-env {:default-app-name ""
-                                                :fqdn "test.de"                                                
-                                                :mailer-from ""
-                                                :mailer-host-port "m.t.de:123"                                                
-                                                :service-domain-whitelist "adb.de"
-                                                :service-noreply-address ""
-                                                })
-                      (cut/generate-appini-env {:default-app-name "test website"
-                                                :fqdn "test.com"                                                 
-                                                :mailer-from "test@test.com"
-                                                :mailer-host-port "mail.test.com:123"                                                
-                                                :service-domain-whitelist "test.com,test.net"
-                                                :service-noreply-address "noreply@test.com"
-                                                })))))
 
 (deftest should-generate-certificate
   (is (= {:name-c2 "prod", :name-c1 "staging"}
-         (th/map-diff (cut/generate-certificate {}) 
+         (th/map-diff (cut/generate-certificate {:fqdn "test.de"})
-                      (cut/generate-certificate {:issuer "prod"})))))
+                      (cut/generate-certificate {:issuer "prod"                                                 
+                                                 :fqdn "test.de"})))))
 
-(deftest should-generate-secret
+(deftest should-generate-ingress
-  (is (= {:website__database__USER-c1 "",
+  (is (= {:hosts-c1 "test.de",
-          :website__database__USER-c2 (b64/encode "pg-user"),
+          :hosts-c2 "test.com",
-          :website__database__PASSWD-c1 "",
+          :host-c1 "test.de", 
-          :website__database__PASSWD-c2 (b64/encode "pg-pw"),
+          :host-c2 "test.com"}
-          :website__mailer__USER-c1 "",
+         (th/map-diff (cut/generate-ingress {:fqdn "test.de"
-          :website__mailer__USER-c2 (b64/encode "maileruser"),
+                                             })
-          :website__mailer__PASSWD-c1 "",
+                      (cut/generate-ingress {:fqdn "test.com"
-          :website__mailer__PASSWD-c2 (b64/encode "mailerpw")}
+                                             })))))
-         (th/map-diff (cut/generate-secrets {:postgres-db-user ""
-                                             :postgres-db-password ""
-                                             :mailer-user ""
-                                             :mailer-pw ""})
-                      (cut/generate-secrets {:postgres-db-user "pg-user"
-                                             :postgres-db-password "pg-pw"
-                                             :mailer-user "maileruser"
-                                             :mailer-pw "mailerpw"})))))
 
-(deftest should-generate-data-volume
+(deftest should-generate-nginx-configmap
-  (is (= {:storage-c1 "1Gi",
+  (is (= {:server_name-c1 "test.de",
-          :storage-c2 "15Gi"}
+          :server_name-c2 "test.com"}
-         (th/map-diff (cut/generate-data-volume {:volume-total-storage-size 1})
+         (th/map-diff (cut/generate-appini-env {:fqdn "test.de"                                                
-                      (cut/generate-data-volume {:volume-total-storage-size 15})))))
+                                                })
+                      (cut/generate-appini-env {:fqdn "test.com"                                                
+                                                })))))
+
+(deftest should-generate-website-content-volume
+  (is (= {:storage-c1 "2Gi",
+          :storage-c2 "10Gi"}
+         (th/map-diff (cut/generate-website-content-volume {:volume-total-storage-size 10
+                                                            :number-of-websites 5})
+                      (cut/generate-website-content-volume {:volume-total-storage-size 50
+                                                            :number-of-websites 5})))))