Work on db schema handling

.gitignore

@@ -12,3 +12,4 @@ resources/public/ui
 .store
 deps.local.edn
 .#*
+.pgdata

config.local.edn

@@ -0,0 +1 @@
+{:dev/start-keys [:storage/schema]}

deps.edn

@@ -32,7 +32,8 @@
          ch.qos.logback/logback-classic {:exclusions [org.slf4j/slf4j-api org.slf4j/slf4j-nop] :mvn/version "1.4.4"}
 
          djblue/portal {:mvn/version "0.35.0"}
-         lambdaisland/uri {:mvn/version "1.13.95"}}
+         lambdaisland/uri {:mvn/version "1.13.95"}
+         com.lambdaisland/facai {:mvn/version "0.7.59-alpha"}}
 
  :aliases
  {:dev  {:extra-paths ["dev"]}

dev/user.clj

@@ -1,6 +1,11 @@
 (ns user)
 
+(try
   (alter-var-root #'*print-namespace-maps* (constantly false))
+  (catch Exception _))
+(try
+  (set! *print-namespace-maps* false)
+  (catch Exception _))
 
 (defmacro jit [sym]
   `(requiring-resolve '~sym))

docker-compose.yml

@@ -0,0 +1,19 @@
+version: '3'
+services:
+  postgres:
+    image: postgres:latest
+    container_name: souk-postgres
+    command: postgres -c log_statement='all'
+    restart: always
+    ports:
+      - "55432:5432"
+    volumes:
+      - .pgdata:/var/lib/postgresql/data
+    environment:
+      POSTGRES_DB: postgres
+      POSTGRES_USER: postgres
+      POSTGRES_PASSWORD: postgres
+      POSTGRES_HOST_AUTH_METHOD: trust
+    logging:
+      options:
+        max-size: 50m

repl_sessions/second_stream.clj

@@ -30,3 +30,43 @@
   :ldp/inbox "https://toot.cat/users/plexus/inbox"})
 
 (jdbc/execute! (:ds (user/value :storage/db)) [(sql/sql 'drop-table :activitystreams/Actor)])
+
+(ap/GET "https://plexus.osrx.chat/users/plexus")
+(ap/GET "https://plexus.osrx.chat/users/plexus/outbox")
+(ap/GET "https://plexus.osrx.chat/users/plexus/outbox?page=true")
+
+{:rdf/id "https://plexus.osrx.chat/users/plexus/outbox",
+ :rdf/type :activitystreams/OrderedCollection,
+ :activitystreams/totalItems 1,
+ :activitystreams/first
+ "https://plexus.osrx.chat/users/plexus/outbox?page=true",
+ :activitystreams/last
+ "https://plexus.osrx.chat/users/plexus/outbox?min_id=0&page=true"}
+
+
+{:rdf/type :activitystreams/Note,
+ :rdf/id "https://plexus.osrx.chat/users/plexus/statuses/109495602955086656",
+ :activitystreams/inReplyTo nil,
+ :activitystreams/published #inst "2022-12-11T14:51:48Z"
+ :activitystreams/to ["https://www.w3.org/ns/activitystreams#Public"],
+ :activitystreams/sensitive false,
+ :activitystreams/cc ["https://plexus.osrx.chat/users/plexus/followers"],
+ :activitystreams/attributedTo "https://plexus.osrx.chat/users/plexus",
+ :activitystreams/summary nil,
+ :activitystreams/tag [],
+ :ostatus/conversation "https://www.w3.org/ns/activitystreams#tagplexus.osrx.chat,2022-12-11",
+ :activitystreams/replies {:rdf/id
+                           "https://plexus.osrx.chat/users/plexus/statuses/109495602955086656/replies",
+                           :rdf/type :activitystreams/Collection,
+                           :activitystreams/first
+                           {:rdf/type :activitystreams/CollectionPage,
+                            :activitystreams/next
+                            "https://plexus.osrx.chat/users/plexus/statuses/109495602955086656/replies?only_other_accounts=true&page=true",
+                            :activitystreams/partOf
+                            "https://plexus.osrx.chat/users/plexus/statuses/109495602955086656/replies",
+                            :activitystreams/items []}},
+ :ostatus/inReplyToAtomUri nil,
+ :ostatus/atomUri "https://plexus.osrx.chat/users/plexus/statuses/109495602955086656",
+ :activitystreams/url "https://plexus.osrx.chat/@plexus/109495602955086656",
+ :activitystreams/attachment [],
+ :activitystreams/content {"en" "<p>Hello, world!</p>"}}

resources/lambdaisland/souk/ActivityStreams.edn

@@ -0,0 +1,59 @@
+{:activitystreams/Actor
+ {:properties
+  [[:activitystreams/name text]
+   [:activitystreams/preferredUsername text]
+   [:activitystreams/url rdf/iri]
+   [:activitystreams/summary text]
+   [:ldp/inbox rdf/iri]
+   [:activitystreams/outbox rdf/iri]
+   [:activitystreams/published datetime]]}
+
+ :activitystreams/Person
+ {:store-as :activitystreams/Actor}
+
+ :activitystreams/Service
+ {:store-as :activitystreams/Actor}
+
+ :activitystreams/Note
+ {:properties
+  [[:activitystreams/summary text]
+   [:activitystreams/content text]
+   ]}
+
+ #_
+
+ {:rdf/type :activitystreams/Note,
+  :rdf/id "https://plexus.osrx.chat/users/plexus/statuses/109495602955086656",
+  :activitystreams/inReplyTo nil,
+  :activitystreams/published #inst "2022-12-11T14:51:48Z"
+  :activitystreams/to ["https://www.w3.org/ns/activitystreams#Public"],
+  :activitystreams/sensitive false,
+  :activitystreams/cc ["https://plexus.osrx.chat/users/plexus/followers"],
+  :activitystreams/attributedTo "https://plexus.osrx.chat/users/plexus",
+  :activitystreams/summary nil,
+  :activitystreams/tag [],
+  :ostatus/conversation "https://www.w3.org/ns/activitystreams#tagplexus.osrx.chat,2022-12-11",
+  :activitystreams/replies {:rdf/id
+                            "https://plexus.osrx.chat/users/plexus/statuses/109495602955086656/replies",
+                            :rdf/type :activitystreams/Collection,
+                            :activitystreams/first
+                            {:rdf/type :activitystreams/CollectionPage,
+                             :activitystreams/next
+                             "https://plexus.osrx.chat/users/plexus/statuses/109495602955086656/replies?only_other_accounts=true&page=true",
+                             :activitystreams/partOf
+                             "https://plexus.osrx.chat/users/plexus/statuses/109495602955086656/replies",
+                             :activitystreams/items []}},
+  :ostatus/inReplyToAtomUri nil,
+  :ostatus/atomUri "https://plexus.osrx.chat/users/plexus/statuses/109495602955086656",
+  :activitystreams/url "https://plexus.osrx.chat/@plexus/109495602955086656",
+  :activitystreams/attachment [],
+  :activitystreams/content {"en" "<p>Hello, world!</p>"}}
+
+ :activitystreams/Article
+ {:store-as :activitystreams/Note}
+
+ :activitystreams/Link
+ {:store-as :activitystreams/Note}
+
+ :activitystreams/Question
+ {:store-as :activitystreams/Note}}

resources/lambdaisland/souk/config.edn

@@ -1,13 +1,21 @@
 {:http/router
  {:gx/component lambdaisland.souk.components.router/component
-  :gx/props     {:dev-router? #setting :dev/reload-routes?}}
+  :gx/props     {:dev-router?     #setting :dev/reload-routes?
+                 :storage/db      (gx/ref :storage/db)
+                 :instance/domain #setting :instance/domain}}
 
  :http/server
  {:gx/component lambdaisland.souk.components.jetty/component
   :gx/props     {:jetty-options {:port #setting :port}
-                 :router        (gx/ref :http/router)
+                 :http/router   (gx/ref :http/router)}}
-                 :db            (gx/ref :storage/db)}}
 
  :storage/db
- {:gx/component lambdaisland.souk.db/component
+ {:gx/component lambdaisland.souk.components.db/component
-  :gx/props     {:url #setting :jdbc/url}}}
+  :gx/props     {:url #setting :jdbc/url
+                 :schema (gx/ref :storage/schema)}}
+
+ :storage/schema
+ {:gx/component lambdaisland.souk.components.db-schema/component
+  :gx/props     {:url #setting :jdbc/url
+                 :admin-url #setting :jdbc/admin-url
+                 :schemas [#resource "lambdaisland/souk/ActivityStreams.edn"]}}}

resources/lambdaisland/souk/settings-dev.edn

@@ -1,2 +1,4 @@
 {:dev/reload-routes? true
- :jdbc/url "jdbc:pgsql://localhost:5432/souk?user=postgres"}
+ :jdbc/url "jdbc:pgsql://localhost:55432/souk?user=postgres"
+ :jdbc/admin-url "jdbc:pgsql://localhost:55432/postgres?user=postgres"
+ :instance/domain "dev.squid.casa"}

src/lambdaisland/souk/activitypub.clj

@@ -12,9 +12,8 @@
    "activitystreams" "https://www.w3.org/ns/activitystreams#"
    "xsd" "http://www.w3.org/2001/XMLSchema#"
    "owl" "http://www.w3.org/2002/07/owl#"
-   "rdfs" "http://www.w3.org/2000/01/rdf-schema#"})
+   "rdfs" "http://www.w3.org/2000/01/rdf-schema#"
+   "ostatus" "http://ostatus.org#"})
 
 (defn GET [url]
   (ld/internalize (ld/expand (:body (ld/json-get url))) common-prefixes))
-
-(GET "https://toot.cat/users/plexus")

src/lambdaisland/souk/components/db.clj

@@ -0,0 +1,64 @@
+(ns lambdaisland.souk.components.db
+  (:require [lambdaisland.souk.sql :as sql]
+            [next.jdbc :as jdbc]
+            [next.jdbc.date-time :as jdbc-date-time]
+            [next.jdbc.plan]
+            [next.jdbc.result-set :as rs]
+            [next.jdbc.sql :as nsql]
+            [cheshire.core :as json]
+            [lambdaisland.glogc :as log])
+  (:import (com.mchange.v2.c3p0 ComboPooledDataSource)))
+
+(defn pg-coerce [val]
+  (cond
+    (instance? java.time.ZonedDateTime val)
+    (.toOffsetDateTime val)
+    :else
+    val))
+
+(defn insert-sql [table entity props]
+  (into [(sql/sql 'insert-into table
+                  (cons :rdf/props
+                        (map key entity))
+                  'values
+                  (repeat (inc (count entity)) '?)
+                  'on-conflict [:raw "(\"rdf/id\")"]
+                  'do
+                  'update-set
+                  (into [:commas]
+                        (map (fn [[k]]
+                               [k '= '?]))
+                        entity))]
+        (cons
+         (json/encode props)
+         (concat
+          (map (comp pg-coerce val) entity)
+          (map (comp pg-coerce val) entity)))))
+
+(defn start! [{:keys [props schema]}]
+  (let [ds (doto (ComboPooledDataSource.)
+             (.setDriverClass "com.impossibl.postgres.jdbc.PGDriver")
+             (.setJdbcUrl (:url props)))]
+    (let [table-columns
+          (into {}
+                (with-open [con (jdbc/get-connection ds {})]
+                  (let [md (.getMetaData con)]
+                    (doall
+                     (for [{:keys [pg_class/TABLE_NAME]}
+                           (-> md
+                               (.getTables nil nil nil (into-array ["TABLE" "VIEW"]))
+                               (rs/datafiable-result-set ds {}))]
+                       [(keyword TABLE_NAME)
+                        (map (comp keyword :COLUMN_NAME)
+                             (rs/datafiable-result-set (.getColumns md nil nil TABLE_NAME nil) ds {}))])))))]
+      {:schema table-columns
+       :ds ds})))
+
+(defn stop! [{ds :value}]
+  #_(.close ds))
+
+;; cpds.setUser("dbuser");
+;; cpds.setPassword("dbpassword");
+(def component
+  {:gx/start {:gx/processor #'start!}
+   :gx/stop {:gx/processor #'stop!}})

src/lambdaisland/souk/components/db_schema.clj

@@ -0,0 +1,110 @@
+(ns lambdaisland.souk.components.db-schema
+  (:require
+   [aero.core :as aero]
+   [lambdaisland.glogc :as log]
+   [lambdaisland.souk.sql :as sql]
+   [lambdaisland.uri :as uri]
+   [next.jdbc :as jdbc]
+   [next.jdbc.result-set :as rs]))
+
+(set! *warn-on-reflection* true)
+
+(def default-properties
+  [[:rdf/id 'text 'primary-key]
+   [:rdf/props 'jsonb 'default "{}"]
+   [:meta/created-at 'timestamp-with-time-zone 'default [:fn 'now] 'not-null]
+   [:meta/updated-at 'timestamp-with-time-zone]])
+
+(def set-ts-trigger-def "CREATE OR REPLACE FUNCTION trigger_set_timestamp()\nRETURNS TRIGGER AS $$\nBEGIN\n  NEW.\"meta/updated-at\" = NOW();\n  RETURN NEW;\nEND;\n$$ LANGUAGE plpgsql;")
+
+(def pg-types
+  '{text text
+    rdf/iri text
+    datetime timestamp-with-time-zone})
+
+(defn jdbc-url->db-name [url]
+  (-> url
+      uri/uri
+      :path
+      uri/uri
+      :path
+      (subs 1)))
+
+(defn table-columns
+  ([ds]
+   (table-columns ds nil))
+  ([ds opts]
+   (with-open [conn (jdbc/get-connection ds opts)]
+     (let [md     (.getMetaData conn)
+           cols   #(rs/datafiable-result-set
+                    (.getColumns md nil nil % nil) ds)
+           tables #(rs/datafiable-result-set
+                    (.getTables md nil nil nil
+                                (into-array ["TABLE" "VIEW"])) ds opts)]
+       (into {}
+             (map (fn [{:pg_class/keys [TABLE_NAME]}]
+                    [(keyword TABLE_NAME)
+                     (into #{} (map (comp keyword :COLUMN_NAME)) (cols TABLE_NAME))]))
+             (tables))))))
+
+(defn create-table! [ds table-name columns]
+  (log/info :table/creating {:name table-name :columns columns})
+  (jdbc/execute! ds [(sql/sql 'create-table 'if-not-exists table-name
+                              (concat
+                               default-properties
+                               columns))])
+  (jdbc/execute! ds [(sql/sql 'drop-trigger 'if-exists :set-timestamp
+                              'on table-name)]))
+
+(defn create-table-sql [table properties]
+  [(sql/sql 'create-table table properties)])
+
+(defn add-columns-sql [table properties]
+  [(sql/sql 'alter-table table
+            (into [:bare-list]
+                  (map (fn [[col type]]
+                         ['add col type]))
+                  properties))])
+
+(defn migrate-tables! [url schemas]
+  (doseq [schema schemas
+          [table {:keys [properties store-as]}] (aero/read-config schema)
+          :when (not store-as)]
+    (let [ds (jdbc/get-datasource url)
+          table-cols (table-columns ds nil)
+          all-props (concat
+                     default-properties
+                     (for [[column type] properties]
+                       [column (get pg-types type)]))]
+      (if (contains? table-cols table)
+        (when-let [new-props (seq (remove (fn [[col]]
+                                            (get-in table-cols [table col]))
+                                          all-props))]
+          (jdbc/execute! ds (add-columns-sql table new-props))
+          (log/info :table/altered {:table table :new-props (map first new-props)}))
+        (do
+          (jdbc/execute! ds (create-table-sql table all-props))
+          (jdbc/execute! ds [(sql/sql 'create-trigger  :set-timestamp
+                                      'before-update
+                                      'on table
+                                      'for-each-row
+                                      'execute-procedure [:fn 'trigger_set_timestamp])])
+          (log/info :table/created {:table table :properties (map first all-props)}))))))
+
+(defn start! [{{:keys [url admin-url schemas]} :props}]
+  (try
+    (jdbc/execute! (jdbc/get-datasource admin-url)
+                   [(sql/sql ['create-database [:ident (jdbc-url->db-name url)]])])
+    (log/info :database/created {:url url})
+    (catch Exception e
+      ;; as a poor-man's CREATE IF NOT EXISTS, we swallow this particular error,
+      ;; and rethrow anything else.
+      (when-not (re-find #"database.*already exists" (.getMessage e))
+        (throw e))))
+  (let [ds (jdbc/get-datasource url)]
+    (jdbc/execute! ds [set-ts-trigger-def])
+    (migrate-tables! url schemas)
+    (table-columns ds)))
+
+(def component
+  {:gx/start {:gx/processor #'start!}})

src/lambdaisland/souk/components/jetty.clj

@@ -3,8 +3,6 @@
             [reitit.ring :as reitit-ring])
   (:import (org.eclipse.jetty.server Server)))
 
-(def defaults {:join? false})
-
 (def ?JettyOptions
   "Start a Jetty webserver to serve the given handler according to the
   supplied options:
@@ -67,6 +65,8 @@
   [:map
    [:jetty-options ?JettyOptions]])
 
+(def defaults {:join? false})
+
 (defn ring-handler [reitit-router]
   (reitit-ring/ring-handler
    reitit-router
@@ -74,7 +74,8 @@
     (reitit-ring/create-default-handler))))
 
 (defn http-start [{:keys [props]}]
-  (let [{:keys [router jetty-options]} props
+  (let [{:keys [jetty-options]
+         :http/keys [router]} props
         jetty-options (merge defaults jetty-options)]
     (ring.jetty/run-jetty (ring-handler router) jetty-options)))
 

src/lambdaisland/souk/components/template.clj

@@ -0,0 +1,12 @@
+(ns lambdaisland.souk.components.template)
+
+(defn start! [{:keys [props]}]
+  )
+
+(defn stop! [{:keys [value]}]
+  )
+
+(def component
+  {:gx/start {:gx/processor #'start!
+              :gx/props-schema [:map]}
+   :gx/stop {:gx/processor #'stop!}})

src/lambdaisland/souk/db.clj

@@ -1,100 +1 @@
-(ns lambdaisland.souk.db
+(ns lambdaisland.souk.db)
-  (:require [lambdaisland.souk.sql :as sql]
-            [next.jdbc :as jdbc]
-            [next.jdbc.date-time :as jdbc-date-time]
-            [next.jdbc.plan]
-            [next.jdbc.result-set :as rs]
-            [next.jdbc.sql :as nsql]
-            [cheshire.core :as json]
-            [lambdaisland.glogc :as log])
-  (:import (com.mchange.v2.c3p0 ComboPooledDataSource)))
-
-(def default-properties
-  [[:rdf/id 'text 'primary-key]
-   [:rdf/type 'text]
-   [:rdf/props 'jsonb 'default "{}"]
-   [:meta/created-at 'timestamp-with-time-zone 'default [:fn 'now] 'not-null]
-   [:meta/updated-at 'timestamp-with-time-zone]])
-
-(def set-ts-trigger-def "CREATE OR REPLACE FUNCTION trigger_set_timestamp()\nRETURNS TRIGGER AS $$\nBEGIN\n  NEW.\"meta/updated-at\" = NOW();\n  RETURN NEW;\nEND;\n$$ LANGUAGE plpgsql;")
-
-(def tables
-  [[:activitystreams/Actor
-    [[:activitystreams/name 'text 'not-null]
-     [:activitystreams/preferredUsername 'text 'not-null]
-     [:activitystreams/url 'text 'not-null]
-     [:activitystreams/summary 'text]
-     [:ldp/inbox 'text 'not-null]
-     [:activitystreams/outbox 'text 'not-null]
-     [:activitystreams/published 'timestamp-with-time-zone]]]])
-
-(defn create-table! [ds table-name columns]
-  (log/info :table/creating {:name table-name :columns columns})
-  (jdbc/execute! ds [(sql/sql 'create-table 'if-not-exists table-name
-                              (concat
-                               default-properties
-                               columns))])
-  (jdbc/execute! ds [(sql/sql 'drop-trigger 'if-exists :set-timestamp
-                              'on table-name)])
-  (jdbc/execute! ds [(sql/sql 'create-trigger  :set-timestamp
-                              'before-update
-                              'on table-name
-                              'for-each-row
-                              'execute-procedure [:fn 'trigger_set_timestamp])]))
-
-(defn pg-coerce [val]
-  (cond
-    (instance? java.time.ZonedDateTime val)
-    (.toOffsetDateTime val)
-    :else
-    val))
-
-(defn insert-sql [table entity props]
-  (into [(sql/sql 'insert-into table
-                  (cons :rdf/props
-                        (map key entity))
-                  'values
-                  (repeat (inc (count entity)) '?)
-                  'on-conflict [:raw "(\"rdf/id\")"]
-                  'do
-                  'update-set
-                  (into [:commas]
-                        (map (fn [[k]]
-                               [k '= '?]))
-                        entity))]
-        (cons
-         (json/encode props)
-         (concat
-          (map (comp pg-coerce val) entity)
-          (map (comp pg-coerce val) entity)))))
-
-(defn start! [{:keys [props]}]
-  (let [ds (doto (ComboPooledDataSource.)
-             (.setDriverClass "com.impossibl.postgres.jdbc.PGDriver")
-             (.setJdbcUrl (:url props)))]
-    (jdbc/execute! ds [set-ts-trigger-def])
-    (doseq [[table columns] tables]
-      (create-table! ds table columns))
-    (let [table-columns
-          (into {}
-                (with-open [con (jdbc/get-connection ds {})]
-                  (let [md (.getMetaData con)]
-                    (doall
-                     (for [{:keys [pg_class/TABLE_NAME]}
-                           (-> md
-                               (.getTables nil nil nil (into-array ["TABLE" "VIEW"]))
-                               (rs/datafiable-result-set ds {}))]
-                       [(keyword TABLE_NAME)
-                        (map (comp keyword :COLUMN_NAME)
-                             (rs/datafiable-result-set (.getColumns md nil nil TABLE_NAME nil) ds {}))])))))]
-      {:schema table-columns
-       :ds ds})))
-
-(defn stop! [{ds :value}]
-  #_(.close ds))
-
-;; cpds.setUser("dbuser");
-;; cpds.setPassword("dbpassword");
-(def component
-  {:gx/start {:gx/processor #'start!}
-   :gx/stop {:gx/processor #'stop!}})

src/lambdaisland/souk/json_ld.clj

@@ -110,8 +110,6 @@
       (map? v)
       (-> v
           (cond-> (contains? v "@type")
-            (doto prn)
-            (contains? v "@type")
             (update "@type" shorten))
           (update-keys (fn [k]
                          (case k

src/lambdaisland/souk/sql.clj

@@ -41,7 +41,7 @@
                     :str (sql-str (second x))
                     :raw (second x)
                     :list (sql-list (map sql (next x)))
-                    :commas (sql-list "" "" ", " (map sql (next x)))
+                    :bare-list (sql-list "" "" ", " (map sql (next x)))
                     :fn (str (second x)
                              (sql-list (map sql (nnext x))))
                     (apply sql x))