Work on db schema handling
Arne Brasseur
1 year ago
parent
bf25b247d9
commit
8184ee40da
@ -0,0 +1 @@
|
||||
{:dev/start-keys [:storage/schema]}
|
||||
@ -0,0 +1,19 @@
|
||||
version: '3'
|
||||
services:
|
||||
postgres:
|
||||
image: postgres:latest
|
||||
container_name: souk-postgres
|
||||
command: postgres -c log_statement='all'
|
||||
restart: always
|
||||
ports:
|
||||
- "55432:5432"
|
||||
volumes:
|
||||
- .pgdata:/var/lib/postgresql/data
|
||||
environment:
|
||||
POSTGRES_DB: postgres
|
||||
POSTGRES_USER: postgres
|
||||
POSTGRES_PASSWORD: postgres
|
||||
POSTGRES_HOST_AUTH_METHOD: trust
|
||||
logging:
|
||||
options:
|
||||
max-size: 50m
|
||||
@ -0,0 +1,59 @@
|
||||
{:activitystreams/Actor
|
||||
{:properties
|
||||
[[:activitystreams/name text]
|
||||
[:activitystreams/preferredUsername text]
|
||||
[:activitystreams/url rdf/iri]
|
||||
[:activitystreams/summary text]
|
||||
[:ldp/inbox rdf/iri]
|
||||
[:activitystreams/outbox rdf/iri]
|
||||
[:activitystreams/published datetime]]}
|
||||
|
||||
:activitystreams/Person
|
||||
{:store-as :activitystreams/Actor}
|
||||
|
||||
:activitystreams/Service
|
||||
{:store-as :activitystreams/Actor}
|
||||
|
||||
:activitystreams/Note
|
||||
{:properties
|
||||
[[:activitystreams/summary text]
|
||||
[:activitystreams/content text]
|
||||
]}
|
||||
|
||||
#_
|
||||
|
||||
{:rdf/type :activitystreams/Note,
|
||||
:rdf/id "https://plexus.osrx.chat/users/plexus/statuses/109495602955086656",
|
||||
:activitystreams/inReplyTo nil,
|
||||
:activitystreams/published #inst "2022-12-11T14:51:48Z"
|
||||
:activitystreams/to ["https://www.w3.org/ns/activitystreams#Public"],
|
||||
:activitystreams/sensitive false,
|
||||
:activitystreams/cc ["https://plexus.osrx.chat/users/plexus/followers"],
|
||||
:activitystreams/attributedTo "https://plexus.osrx.chat/users/plexus",
|
||||
:activitystreams/summary nil,
|
||||
:activitystreams/tag [],
|
||||
:ostatus/conversation "https://www.w3.org/ns/activitystreams#tagplexus.osrx.chat,2022-12-11",
|
||||
:activitystreams/replies {:rdf/id
|
||||
"https://plexus.osrx.chat/users/plexus/statuses/109495602955086656/replies",
|
||||
:rdf/type :activitystreams/Collection,
|
||||
:activitystreams/first
|
||||
{:rdf/type :activitystreams/CollectionPage,
|
||||
:activitystreams/next
|
||||
"https://plexus.osrx.chat/users/plexus/statuses/109495602955086656/replies?only_other_accounts=true&page=true",
|
||||
:activitystreams/partOf
|
||||
"https://plexus.osrx.chat/users/plexus/statuses/109495602955086656/replies",
|
||||
:activitystreams/items []}},
|
||||
:ostatus/inReplyToAtomUri nil,
|
||||
:ostatus/atomUri "https://plexus.osrx.chat/users/plexus/statuses/109495602955086656",
|
||||
:activitystreams/url "https://plexus.osrx.chat/@plexus/109495602955086656",
|
||||
:activitystreams/attachment [],
|
||||
:activitystreams/content {"en" "<p>Hello, world!</p>"}}
|
||||
|
||||
:activitystreams/Article
|
||||
{:store-as :activitystreams/Note}
|
||||
|
||||
:activitystreams/Link
|
||||
{:store-as :activitystreams/Note}
|
||||
|
||||
:activitystreams/Question
|
||||
{:store-as :activitystreams/Note}}
|
||||
@ -1,13 +1,21 @@
|
||||
{:http/router
|
||||
{:gx/component lambdaisland.souk.components.router/component
|
||||
:gx/props {:dev-router? #setting :dev/reload-routes?}}
|
||||
:gx/props {:dev-router? #setting :dev/reload-routes?
|
||||
:storage/db (gx/ref :storage/db)
|
||||
:instance/domain #setting :instance/domain}}
|
||||
|
||||
:http/server
|
||||
{:gx/component lambdaisland.souk.components.jetty/component
|
||||
:gx/props {:jetty-options {:port #setting :port}
|
||||
:router (gx/ref :http/router)
|
||||
:db (gx/ref :storage/db)}}
|
||||
:http/router (gx/ref :http/router)}}
|
||||
|
||||
:storage/db
|
||||
{:gx/component lambdaisland.souk.db/component
|
||||
:gx/props {:url #setting :jdbc/url}}}
|
||||
{:gx/component lambdaisland.souk.components.db/component
|
||||
:gx/props {:url #setting :jdbc/url
|
||||
:schema (gx/ref :storage/schema)}}
|
||||
|
||||
:storage/schema
|
||||
{:gx/component lambdaisland.souk.components.db-schema/component
|
||||
:gx/props {:url #setting :jdbc/url
|
||||
:admin-url #setting :jdbc/admin-url
|
||||
:schemas [#resource "lambdaisland/souk/ActivityStreams.edn"]}}}
|
||||
|
||||
@ -1,2 +1,4 @@
|
||||
{:dev/reload-routes? true
|
||||
:jdbc/url "jdbc:pgsql://localhost:5432/souk?user=postgres"}
|
||||
:jdbc/url "jdbc:pgsql://localhost:55432/souk?user=postgres"
|
||||
:jdbc/admin-url "jdbc:pgsql://localhost:55432/postgres?user=postgres"
|
||||
:instance/domain "dev.squid.casa"}
|
||||
|
||||
@ -0,0 +1,64 @@
|
||||
(ns lambdaisland.souk.components.db
|
||||
(:require [lambdaisland.souk.sql :as sql]
|
||||
[next.jdbc :as jdbc]
|
||||
[next.jdbc.date-time :as jdbc-date-time]
|
||||
[next.jdbc.plan]
|
||||
[next.jdbc.result-set :as rs]
|
||||
[next.jdbc.sql :as nsql]
|
||||
[cheshire.core :as json]
|
||||
[lambdaisland.glogc :as log])
|
||||
(:import (com.mchange.v2.c3p0 ComboPooledDataSource)))
|
||||
|
||||
(defn pg-coerce [val]
|
||||
(cond
|
||||
(instance? java.time.ZonedDateTime val)
|
||||
(.toOffsetDateTime val)
|
||||
:else
|
||||
val))
|
||||
|
||||
(defn insert-sql [table entity props]
|
||||
(into [(sql/sql 'insert-into table
|
||||
(cons :rdf/props
|
||||
(map key entity))
|
||||
'values
|
||||
(repeat (inc (count entity)) '?)
|
||||
'on-conflict [:raw "(\"rdf/id\")"]
|
||||
'do
|
||||
'update-set
|
||||
(into [:commas]
|
||||
(map (fn [[k]]
|
||||
[k '= '?]))
|
||||
entity))]
|
||||
(cons
|
||||
(json/encode props)
|
||||
(concat
|
||||
(map (comp pg-coerce val) entity)
|
||||
(map (comp pg-coerce val) entity)))))
|
||||
|
||||
(defn start! [{:keys [props schema]}]
|
||||
(let [ds (doto (ComboPooledDataSource.)
|
||||
(.setDriverClass "com.impossibl.postgres.jdbc.PGDriver")
|
||||
(.setJdbcUrl (:url props)))]
|
||||
(let [table-columns
|
||||
(into {}
|
||||
(with-open [con (jdbc/get-connection ds {})]
|
||||
(let [md (.getMetaData con)]
|
||||
(doall
|
||||
(for [{:keys [pg_class/TABLE_NAME]}
|
||||
(-> md
|
||||
(.getTables nil nil nil (into-array ["TABLE" "VIEW"]))
|
||||
(rs/datafiable-result-set ds {}))]
|
||||
[(keyword TABLE_NAME)
|
||||
(map (comp keyword :COLUMN_NAME)
|
||||
(rs/datafiable-result-set (.getColumns md nil nil TABLE_NAME nil) ds {}))])))))]
|
||||
{:schema table-columns
|
||||
:ds ds})))
|
||||
|
||||
(defn stop! [{ds :value}]
|
||||
#_(.close ds))
|
||||
|
||||
;; cpds.setUser("dbuser");
|
||||
;; cpds.setPassword("dbpassword");
|
||||
(def component
|
||||
{:gx/start {:gx/processor #'start!}
|
||||
:gx/stop {:gx/processor #'stop!}})
|
||||
@ -0,0 +1,110 @@
|
||||
(ns lambdaisland.souk.components.db-schema
|
||||
(:require
|
||||
[aero.core :as aero]
|
||||
[lambdaisland.glogc :as log]
|
||||
[lambdaisland.souk.sql :as sql]
|
||||
[lambdaisland.uri :as uri]
|
||||
[next.jdbc :as jdbc]
|
||||
[next.jdbc.result-set :as rs]))
|
||||
|
||||
(set! *warn-on-reflection* true)
|
||||
|
||||
(def default-properties
|
||||
[[:rdf/id 'text 'primary-key]
|
||||
[:rdf/props 'jsonb 'default "{}"]
|
||||
[:meta/created-at 'timestamp-with-time-zone 'default [:fn 'now] 'not-null]
|
||||
[:meta/updated-at 'timestamp-with-time-zone]])
|
||||
|
||||
(def set-ts-trigger-def "CREATE OR REPLACE FUNCTION trigger_set_timestamp()\nRETURNS TRIGGER AS $$\nBEGIN\n NEW.\"meta/updated-at\" = NOW();\n RETURN NEW;\nEND;\n$$ LANGUAGE plpgsql;")
|
||||
|
||||
(def pg-types
|
||||
'{text text
|
||||
rdf/iri text
|
||||
datetime timestamp-with-time-zone})
|
||||
|
||||
(defn jdbc-url->db-name [url]
|
||||
(-> url
|
||||
uri/uri
|
||||
:path
|
||||
uri/uri
|
||||
:path
|
||||
(subs 1)))
|
||||
|
||||
(defn table-columns
|
||||
([ds]
|
||||
(table-columns ds nil))
|
||||
([ds opts]
|
||||
(with-open [conn (jdbc/get-connection ds opts)]
|
||||
(let [md (.getMetaData conn)
|
||||
cols #(rs/datafiable-result-set
|
||||
(.getColumns md nil nil % nil) ds)
|
||||
tables #(rs/datafiable-result-set
|
||||
(.getTables md nil nil nil
|
||||
(into-array ["TABLE" "VIEW"])) ds opts)]
|
||||
(into {}
|
||||
(map (fn [{:pg_class/keys [TABLE_NAME]}]
|
||||
[(keyword TABLE_NAME)
|
||||
(into #{} (map (comp keyword :COLUMN_NAME)) (cols TABLE_NAME))]))
|
||||
(tables))))))
|
||||
|
||||
(defn create-table! [ds table-name columns]
|
||||
(log/info :table/creating {:name table-name :columns columns})
|
||||
(jdbc/execute! ds [(sql/sql 'create-table 'if-not-exists table-name
|
||||
(concat
|
||||
default-properties
|
||||
columns))])
|
||||
(jdbc/execute! ds [(sql/sql 'drop-trigger 'if-exists :set-timestamp
|
||||
'on table-name)]))
|
||||
|
||||
(defn create-table-sql [table properties]
|
||||
[(sql/sql 'create-table table properties)])
|
||||
|
||||
(defn add-columns-sql [table properties]
|
||||
[(sql/sql 'alter-table table
|
||||
(into [:bare-list]
|
||||
(map (fn [[col type]]
|
||||
['add col type]))
|
||||
properties))])
|
||||
|
||||
(defn migrate-tables! [url schemas]
|
||||
(doseq [schema schemas
|
||||
[table {:keys [properties store-as]}] (aero/read-config schema)
|
||||
:when (not store-as)]
|
||||
(let [ds (jdbc/get-datasource url)
|
||||
table-cols (table-columns ds nil)
|
||||
all-props (concat
|
||||
default-properties
|
||||
(for [[column type] properties]
|
||||
[column (get pg-types type)]))]
|
||||
(if (contains? table-cols table)
|
||||
(when-let [new-props (seq (remove (fn [[col]]
|
||||
(get-in table-cols [table col]))
|
||||
all-props))]
|
||||
(jdbc/execute! ds (add-columns-sql table new-props))
|
||||
(log/info :table/altered {:table table :new-props (map first new-props)}))
|
||||
(do
|
||||
(jdbc/execute! ds (create-table-sql table all-props))
|
||||
(jdbc/execute! ds [(sql/sql 'create-trigger :set-timestamp
|
||||
'before-update
|
||||
'on table
|
||||
'for-each-row
|
||||
'execute-procedure [:fn 'trigger_set_timestamp])])
|
||||
(log/info :table/created {:table table :properties (map first all-props)}))))))
|
||||
|
||||
(defn start! [{{:keys [url admin-url schemas]} :props}]
|
||||
(try
|
||||
(jdbc/execute! (jdbc/get-datasource admin-url)
|
||||
[(sql/sql ['create-database [:ident (jdbc-url->db-name url)]])])
|
||||
(log/info :database/created {:url url})
|
||||
(catch Exception e
|
||||
;; as a poor-man's CREATE IF NOT EXISTS, we swallow this particular error,
|
||||
;; and rethrow anything else.
|
||||
(when-not (re-find #"database.*already exists" (.getMessage e))
|
||||
(throw e))))
|
||||
(let [ds (jdbc/get-datasource url)]
|
||||
(jdbc/execute! ds [set-ts-trigger-def])
|
||||
(migrate-tables! url schemas)
|
||||
(table-columns ds)))
|
||||
|
||||
(def component
|
||||
{:gx/start {:gx/processor #'start!}})
|
||||
@ -0,0 +1,12 @@
|
||||
(ns lambdaisland.souk.components.template)
|
||||
|
||||
(defn start! [{:keys [props]}]
|
||||
)
|
||||
|
||||
(defn stop! [{:keys [value]}]
|
||||
)
|
||||
|
||||
(def component
|
||||
{:gx/start {:gx/processor #'start!
|
||||
:gx/props-schema [:map]}
|
||||
:gx/stop {:gx/processor #'stop!}})
|
||||
@ -1,100 +1 @@
|
||||
(ns lambdaisland.souk.db
|
||||
(:require [lambdaisland.souk.sql :as sql]
|
||||
[next.jdbc :as jdbc]
|
||||
[next.jdbc.date-time :as jdbc-date-time]
|
||||
[next.jdbc.plan]
|
||||
[next.jdbc.result-set :as rs]
|
||||
[next.jdbc.sql :as nsql]
|
||||
[cheshire.core :as json]
|
||||
[lambdaisland.glogc :as log])
|
||||
(:import (com.mchange.v2.c3p0 ComboPooledDataSource)))
|
||||
|
||||
(def default-properties
|
||||
[[:rdf/id 'text 'primary-key]
|
||||
[:rdf/type 'text]
|
||||
[:rdf/props 'jsonb 'default "{}"]
|
||||
[:meta/created-at 'timestamp-with-time-zone 'default [:fn 'now] 'not-null]
|
||||
[:meta/updated-at 'timestamp-with-time-zone]])
|
||||
|
||||
(def set-ts-trigger-def "CREATE OR REPLACE FUNCTION trigger_set_timestamp()\nRETURNS TRIGGER AS $$\nBEGIN\n NEW.\"meta/updated-at\" = NOW();\n RETURN NEW;\nEND;\n$$ LANGUAGE plpgsql;")
|
||||
|
||||
(def tables
|
||||
[[:activitystreams/Actor
|
||||
[[:activitystreams/name 'text 'not-null]
|
||||
[:activitystreams/preferredUsername 'text 'not-null]
|
||||
[:activitystreams/url 'text 'not-null]
|
||||
[:activitystreams/summary 'text]
|
||||
[:ldp/inbox 'text 'not-null]
|
||||
[:activitystreams/outbox 'text 'not-null]
|
||||
[:activitystreams/published 'timestamp-with-time-zone]]]])
|
||||
|
||||
(defn create-table! [ds table-name columns]
|
||||
(log/info :table/creating {:name table-name :columns columns})
|
||||
(jdbc/execute! ds [(sql/sql 'create-table 'if-not-exists table-name
|
||||
(concat
|
||||
default-properties
|
||||
columns))])
|
||||
(jdbc/execute! ds [(sql/sql 'drop-trigger 'if-exists :set-timestamp
|
||||
'on table-name)])
|
||||
(jdbc/execute! ds [(sql/sql 'create-trigger :set-timestamp
|
||||
'before-update
|
||||
'on table-name
|
||||
'for-each-row
|
||||
'execute-procedure [:fn 'trigger_set_timestamp])]))
|
||||
|
||||
(defn pg-coerce [val]
|
||||
(cond
|
||||
(instance? java.time.ZonedDateTime val)
|
||||
(.toOffsetDateTime val)
|
||||
:else
|
||||
val))
|
||||
|
||||
(defn insert-sql [table entity props]
|
||||
(into [(sql/sql 'insert-into table
|
||||
(cons :rdf/props
|
||||
(map key entity))
|
||||
'values
|
||||
(repeat (inc (count entity)) '?)
|
||||
'on-conflict [:raw "(\"rdf/id\")"]
|
||||
'do
|
||||
'update-set
|
||||
(into [:commas]
|
||||
(map (fn [[k]]
|
||||
[k '= '?]))
|
||||
entity))]
|
||||
(cons
|
||||
(json/encode props)
|
||||
(concat
|
||||
(map (comp pg-coerce val) entity)
|
||||
(map (comp pg-coerce val) entity)))))
|
||||
|
||||
(defn start! [{:keys [props]}]
|
||||
(let [ds (doto (ComboPooledDataSource.)
|
||||
(.setDriverClass "com.impossibl.postgres.jdbc.PGDriver")
|
||||
(.setJdbcUrl (:url props)))]
|
||||
(jdbc/execute! ds [set-ts-trigger-def])
|
||||
(doseq [[table columns] tables]
|
||||
(create-table! ds table columns))
|
||||
(let [table-columns
|
||||
(into {}
|
||||
(with-open [con (jdbc/get-connection ds {})]
|
||||
(let [md (.getMetaData con)]
|
||||
(doall
|
||||
(for [{:keys [pg_class/TABLE_NAME]}
|
||||
(-> md
|
||||
(.getTables nil nil nil (into-array ["TABLE" "VIEW"]))
|
||||
(rs/datafiable-result-set ds {}))]
|
||||
[(keyword TABLE_NAME)
|
||||
(map (comp keyword :COLUMN_NAME)
|
||||
(rs/datafiable-result-set (.getColumns md nil nil TABLE_NAME nil) ds {}))])))))]
|
||||
{:schema table-columns
|
||||
:ds ds})))
|
||||
|
||||
(defn stop! [{ds :value}]
|
||||
#_(.close ds))
|
||||
|
||||
;; cpds.setUser("dbuser");
|
||||
;; cpds.setPassword("dbpassword");
|
||||
(def component
|
||||
{:gx/start {:gx/processor #'start!}
|
||||
:gx/stop {:gx/processor #'stop!}})
|
||||
(ns lambdaisland.souk.db)
|
||||
|
||||
Loading…
Reference in New Issue