meissa/souk
7
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Work on db schema handling

Browse source
This commit is contained in:
Arne Brasseur 2022-12-14 11:03:57 +01:00
parent bf25b247d9
commit 8184ee40da
17 changed files with 338 additions and 117 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
.gitignore vendored
View file

@ -12,3 +12,4 @@ resources/public/ui
.store
deps.local.edn
.#*
.pgdata

1
config.local.edn Normal file
View file

@ -0,0 +1 @@
{:dev/start-keys [:storage/schema]}

3
deps.edn
View file

@ -32,7 +32,8 @@
ch.qos.logback/logback-classic {:exclusions [org.slf4j/slf4j-api org.slf4j/slf4j-nop] :mvn/version "1.4.4"}
djblue/portal {:mvn/version "0.35.0"}
lambdaisland/uri {:mvn/version "1.13.95"}}
lambdaisland/uri {:mvn/version "1.13.95"}
com.lambdaisland/facai {:mvn/version "0.7.59-alpha"}}
:aliases
{:dev {:extra-paths ["dev"]}

7
dev/user.clj
View file

@ -1,6 +1,11 @@
(ns user)
(alter-var-root #'*print-namespace-maps* (constantly false))
(try
(alter-var-root #'*print-namespace-maps* (constantly false))
(catch Exception _))
(try
(set! *print-namespace-maps* false)
(catch Exception _))
(defmacro jit [sym]
`(requiring-resolve '~sym))

19
docker-compose.yml Normal file
View file

@ -0,0 +1,19 @@
version: '3'
services:
postgres:
image: postgres:latest
container_name: souk-postgres
command: postgres -c log_statement='all'
restart: always
ports:
- "55432:5432"
volumes:
- .pgdata:/var/lib/postgresql/data
environment:
POSTGRES_DB: postgres
POSTGRES_USER: postgres
POSTGRES_PASSWORD: postgres
POSTGRES_HOST_AUTH_METHOD: trust
logging:
options:
max-size: 50m

40
repl_sessions/second_stream.clj
View file

@ -30,3 +30,43 @@
:ldp/inbox "https://toot.cat/users/plexus/inbox"})
(jdbc/execute! (:ds (user/value :storage/db)) [(sql/sql 'drop-table :activitystreams/Actor)])
(ap/GET "https://plexus.osrx.chat/users/plexus")
(ap/GET "https://plexus.osrx.chat/users/plexus/outbox")
(ap/GET "https://plexus.osrx.chat/users/plexus/outbox?page=true")
{:rdf/id "https://plexus.osrx.chat/users/plexus/outbox",
:rdf/type :activitystreams/OrderedCollection,
:activitystreams/totalItems 1,
:activitystreams/first
"https://plexus.osrx.chat/users/plexus/outbox?page=true",
:activitystreams/last
"https://plexus.osrx.chat/users/plexus/outbox?min_id=0&page=true"}
{:rdf/type :activitystreams/Note,
:rdf/id "https://plexus.osrx.chat/users/plexus/statuses/109495602955086656",
:activitystreams/inReplyTo nil,
:activitystreams/published #inst "2022-12-11T14:51:48Z"
:activitystreams/to ["https://www.w3.org/ns/activitystreams#Public"],
:activitystreams/sensitive false,
:activitystreams/cc ["https://plexus.osrx.chat/users/plexus/followers"],
:activitystreams/attributedTo "https://plexus.osrx.chat/users/plexus",
:activitystreams/summary nil,
:activitystreams/tag [],
:ostatus/conversation "https://www.w3.org/ns/activitystreams#tagplexus.osrx.chat,2022-12-11",
:activitystreams/replies {:rdf/id
"https://plexus.osrx.chat/users/plexus/statuses/109495602955086656/replies",
:rdf/type :activitystreams/Collection,
:activitystreams/first
{:rdf/type :activitystreams/CollectionPage,
:activitystreams/next
"https://plexus.osrx.chat/users/plexus/statuses/109495602955086656/replies?only_other_accounts=true&page=true",
:activitystreams/partOf
"https://plexus.osrx.chat/users/plexus/statuses/109495602955086656/replies",
:activitystreams/items []}},
:ostatus/inReplyToAtomUri nil,
:ostatus/atomUri "https://plexus.osrx.chat/users/plexus/statuses/109495602955086656",
:activitystreams/url "https://plexus.osrx.chat/@plexus/109495602955086656",
:activitystreams/attachment [],
:activitystreams/content {"en" "<p>Hello, world!</p>"}}

59
resources/lambdaisland/souk/ActivityStreams.edn Normal file
View file

@ -0,0 +1,59 @@
{:activitystreams/Actor
{:properties
[[:activitystreams/name text]
[:activitystreams/preferredUsername text]
[:activitystreams/url rdf/iri]
[:activitystreams/summary text]
[:ldp/inbox rdf/iri]
[:activitystreams/outbox rdf/iri]
[:activitystreams/published datetime]]}
:activitystreams/Person
{:store-as :activitystreams/Actor}
:activitystreams/Service
{:store-as :activitystreams/Actor}
:activitystreams/Note
{:properties
[[:activitystreams/summary text]
[:activitystreams/content text]
]}
#_
{:rdf/type :activitystreams/Note,
:rdf/id "https://plexus.osrx.chat/users/plexus/statuses/109495602955086656",
:activitystreams/inReplyTo nil,
:activitystreams/published #inst "2022-12-11T14:51:48Z"
:activitystreams/to ["https://www.w3.org/ns/activitystreams#Public"],
:activitystreams/sensitive false,
:activitystreams/cc ["https://plexus.osrx.chat/users/plexus/followers"],
:activitystreams/attributedTo "https://plexus.osrx.chat/users/plexus",
:activitystreams/summary nil,
:activitystreams/tag [],
:ostatus/conversation "https://www.w3.org/ns/activitystreams#tagplexus.osrx.chat,2022-12-11",
:activitystreams/replies {:rdf/id
"https://plexus.osrx.chat/users/plexus/statuses/109495602955086656/replies",
:rdf/type :activitystreams/Collection,
:activitystreams/first
{:rdf/type :activitystreams/CollectionPage,
:activitystreams/next
"https://plexus.osrx.chat/users/plexus/statuses/109495602955086656/replies?only_other_accounts=true&page=true",
:activitystreams/partOf
"https://plexus.osrx.chat/users/plexus/statuses/109495602955086656/replies",
:activitystreams/items []}},
:ostatus/inReplyToAtomUri nil,
:ostatus/atomUri "https://plexus.osrx.chat/users/plexus/statuses/109495602955086656",
:activitystreams/url "https://plexus.osrx.chat/@plexus/109495602955086656",
:activitystreams/attachment [],
:activitystreams/content {"en" "<p>Hello, world!</p>"}}
:activitystreams/Article
{:store-as :activitystreams/Note}
:activitystreams/Link
{:store-as :activitystreams/Note}
:activitystreams/Question
{:store-as :activitystreams/Note}}

18
resources/lambdaisland/souk/config.edn
View file

@ -1,13 +1,21 @@
{:http/router
{:gx/component lambdaisland.souk.components.router/component
:gx/props {:dev-router? #setting :dev/reload-routes?}}
:gx/props {:dev-router? #setting :dev/reload-routes?
:storage/db (gx/ref :storage/db)
:instance/domain #setting :instance/domain}}
:http/server
{:gx/component lambdaisland.souk.components.jetty/component
:gx/props {:jetty-options {:port #setting :port}
:router (gx/ref :http/router)
:db (gx/ref :storage/db)}}
:http/router (gx/ref :http/router)}}
:storage/db
{:gx/component lambdaisland.souk.db/component
:gx/props {:url #setting :jdbc/url}}}
{:gx/component lambdaisland.souk.components.db/component
:gx/props {:url #setting :jdbc/url
:schema (gx/ref :storage/schema)}}
:storage/schema
{:gx/component lambdaisland.souk.components.db-schema/component
:gx/props {:url #setting :jdbc/url
:admin-url #setting :jdbc/admin-url
:schemas [#resource "lambdaisland/souk/ActivityStreams.edn"]}}}

4
resources/lambdaisland/souk/settings-dev.edn
View file

@ -1,2 +1,4 @@
{:dev/reload-routes? true
:jdbc/url "jdbc:pgsql://localhost:5432/souk?user=postgres"}
:jdbc/url "jdbc:pgsql://localhost:55432/souk?user=postgres"
:jdbc/admin-url "jdbc:pgsql://localhost:55432/postgres?user=postgres"
:instance/domain "dev.squid.casa"}

5
src/lambdaisland/souk/activitypub.clj
View file

@ -12,9 +12,8 @@
"activitystreams" "https://www.w3.org/ns/activitystreams#"
"xsd" "http://www.w3.org/2001/XMLSchema#"
"owl" "http://www.w3.org/2002/07/owl#"
"rdfs" "http://www.w3.org/2000/01/rdf-schema#"})
"rdfs" "http://www.w3.org/2000/01/rdf-schema#"
"ostatus" "http://ostatus.org#"})
(defn GET [url]
(ld/internalize (ld/expand (:body (ld/json-get url))) common-prefixes))
(GET "https://toot.cat/users/plexus")

64
src/lambdaisland/souk/components/db.clj Normal file
View file

@ -0,0 +1,64 @@
(ns lambdaisland.souk.components.db
(:require [lambdaisland.souk.sql :as sql]
[next.jdbc :as jdbc]
[next.jdbc.date-time :as jdbc-date-time]
[next.jdbc.plan]
[next.jdbc.result-set :as rs]
[next.jdbc.sql :as nsql]
[cheshire.core :as json]
[lambdaisland.glogc :as log])
(:import (com.mchange.v2.c3p0 ComboPooledDataSource)))
(defn pg-coerce [val]
(cond
(instance? java.time.ZonedDateTime val)
(.toOffsetDateTime val)
:else
val))
(defn insert-sql [table entity props]
(into [(sql/sql 'insert-into table
(cons :rdf/props
(map key entity))
'values
(repeat (inc (count entity)) '?)
'on-conflict [:raw "(\"rdf/id\")"]
'do
'update-set
(into [:commas]
(map (fn [[k]]
[k '= '?]))
entity))]
(cons
(json/encode props)
(concat
(map (comp pg-coerce val) entity)
(map (comp pg-coerce val) entity)))))
(defn start! [{:keys [props schema]}]
(let [ds (doto (ComboPooledDataSource.)
(.setDriverClass "com.impossibl.postgres.jdbc.PGDriver")
(.setJdbcUrl (:url props)))]
(let [table-columns
(into {}
(with-open [con (jdbc/get-connection ds {})]
(let [md (.getMetaData con)]
(doall
(for [{:keys [pg_class/TABLE_NAME]}
(-> md
(.getTables nil nil nil (into-array ["TABLE" "VIEW"]))
(rs/datafiable-result-set ds {}))]
[(keyword TABLE_NAME)
(map (comp keyword :COLUMN_NAME)
(rs/datafiable-result-set (.getColumns md nil nil TABLE_NAME nil) ds {}))])))))]
{:schema table-columns
:ds ds})))
(defn stop! [{ds :value}]
#_(.close ds))
;; cpds.setUser("dbuser");
;; cpds.setPassword("dbpassword");
(def component
{:gx/start {:gx/processor #'start!}
:gx/stop {:gx/processor #'stop!}})

110
src/lambdaisland/souk/components/db_schema.clj Normal file
View file

@ -0,0 +1,110 @@
(ns lambdaisland.souk.components.db-schema
(:require
[aero.core :as aero]
[lambdaisland.glogc :as log]
[lambdaisland.souk.sql :as sql]
[lambdaisland.uri :as uri]
[next.jdbc :as jdbc]
[next.jdbc.result-set :as rs]))
(set! *warn-on-reflection* true)
(def default-properties
[[:rdf/id 'text 'primary-key]
[:rdf/props 'jsonb 'default "{}"]
[:meta/created-at 'timestamp-with-time-zone 'default [:fn 'now] 'not-null]
[:meta/updated-at 'timestamp-with-time-zone]])
(def set-ts-trigger-def "CREATE OR REPLACE FUNCTION trigger_set_timestamp()\nRETURNS TRIGGER AS $$\nBEGIN\n NEW.\"meta/updated-at\" = NOW();\n RETURN NEW;\nEND;\n$$ LANGUAGE plpgsql;")
(def pg-types
'{text text
rdf/iri text
datetime timestamp-with-time-zone})
(defn jdbc-url->db-name [url]
(-> url
uri/uri
:path
uri/uri
:path
(subs 1)))
(defn table-columns
([ds]
(table-columns ds nil))
([ds opts]
(with-open [conn (jdbc/get-connection ds opts)]
(let [md (.getMetaData conn)
cols #(rs/datafiable-result-set
(.getColumns md nil nil % nil) ds)
tables #(rs/datafiable-result-set
(.getTables md nil nil nil
(into-array ["TABLE" "VIEW"])) ds opts)]
(into {}
(map (fn [{:pg_class/keys [TABLE_NAME]}]
[(keyword TABLE_NAME)
(into #{} (map (comp keyword :COLUMN_NAME)) (cols TABLE_NAME))]))
(tables))))))
(defn create-table! [ds table-name columns]
(log/info :table/creating {:name table-name :columns columns})
(jdbc/execute! ds [(sql/sql 'create-table 'if-not-exists table-name
(concat
default-properties
columns))])
(jdbc/execute! ds [(sql/sql 'drop-trigger 'if-exists :set-timestamp
'on table-name)]))
(defn create-table-sql [table properties]
[(sql/sql 'create-table table properties)])
(defn add-columns-sql [table properties]
[(sql/sql 'alter-table table
(into [:bare-list]
(map (fn [[col type]]
['add col type]))
properties))])
(defn migrate-tables! [url schemas]
(doseq [schema schemas
[table {:keys [properties store-as]}] (aero/read-config schema)
:when (not store-as)]
(let [ds (jdbc/get-datasource url)
table-cols (table-columns ds nil)
all-props (concat
default-properties
(for [[column type] properties]
[column (get pg-types type)]))]
(if (contains? table-cols table)
(when-let [new-props (seq (remove (fn [[col]]
(get-in table-cols [table col]))
all-props))]
(jdbc/execute! ds (add-columns-sql table new-props))
(log/info :table/altered {:table table :new-props (map first new-props)}))
(do
(jdbc/execute! ds (create-table-sql table all-props))
(jdbc/execute! ds [(sql/sql 'create-trigger :set-timestamp
'before-update
'on table
'for-each-row
'execute-procedure [:fn 'trigger_set_timestamp])])
(log/info :table/created {:table table :properties (map first all-props)}))))))
(defn start! [{{:keys [url admin-url schemas]} :props}]
(try
(jdbc/execute! (jdbc/get-datasource admin-url)
[(sql/sql ['create-database [:ident (jdbc-url->db-name url)]])])
(log/info :database/created {:url url})
(catch Exception e
;; as a poor-man's CREATE IF NOT EXISTS, we swallow this particular error,
;; and rethrow anything else.
(when-not (re-find #"database.*already exists" (.getMessage e))
(throw e))))
(let [ds (jdbc/get-datasource url)]
(jdbc/execute! ds [set-ts-trigger-def])
(migrate-tables! url schemas)
(table-columns ds)))
(def component
{:gx/start {:gx/processor #'start!}})

7
src/lambdaisland/souk/components/jetty.clj
View file

@ -3,8 +3,6 @@
[reitit.ring :as reitit-ring])
(:import (org.eclipse.jetty.server Server)))
(def defaults {:join? false})
(def ?JettyOptions
"Start a Jetty webserver to serve the given handler according to the
supplied options:
@ -67,6 +65,8 @@
[:map
[:jetty-options ?JettyOptions]])
(def defaults {:join? false})
(defn ring-handler [reitit-router]
(reitit-ring/ring-handler
reitit-router
@ -74,7 +74,8 @@
(reitit-ring/create-default-handler))))
(defn http-start [{:keys [props]}]
(let [{:keys [router jetty-options]} props
(let [{:keys [jetty-options]
:http/keys [router]} props
jetty-options (merge defaults jetty-options)]
(ring.jetty/run-jetty (ring-handler router) jetty-options)))

12
src/lambdaisland/souk/components/template.clj Normal file
View file

@ -0,0 +1,12 @@
(ns lambdaisland.souk.components.template)
(defn start! [{:keys [props]}]
)
(defn stop! [{:keys [value]}]
)
(def component
{:gx/start {:gx/processor #'start!
:gx/props-schema [:map]}
:gx/stop {:gx/processor #'stop!}})

101
src/lambdaisland/souk/db.clj
View file

@ -1,100 +1 @@
(ns lambdaisland.souk.db
(:require [lambdaisland.souk.sql :as sql]
[next.jdbc :as jdbc]
[next.jdbc.date-time :as jdbc-date-time]
[next.jdbc.plan]
[next.jdbc.result-set :as rs]
[next.jdbc.sql :as nsql]
[cheshire.core :as json]
[lambdaisland.glogc :as log])
(:import (com.mchange.v2.c3p0 ComboPooledDataSource)))
(def default-properties
[[:rdf/id 'text 'primary-key]
[:rdf/type 'text]
[:rdf/props 'jsonb 'default "{}"]
[:meta/created-at 'timestamp-with-time-zone 'default [:fn 'now] 'not-null]
[:meta/updated-at 'timestamp-with-time-zone]])
(def set-ts-trigger-def "CREATE OR REPLACE FUNCTION trigger_set_timestamp()\nRETURNS TRIGGER AS $$\nBEGIN\n NEW.\"meta/updated-at\" = NOW();\n RETURN NEW;\nEND;\n$$ LANGUAGE plpgsql;")
(def tables
[[:activitystreams/Actor
[[:activitystreams/name 'text 'not-null]
[:activitystreams/preferredUsername 'text 'not-null]
[:activitystreams/url 'text 'not-null]
[:activitystreams/summary 'text]
[:ldp/inbox 'text 'not-null]
[:activitystreams/outbox 'text 'not-null]
[:activitystreams/published 'timestamp-with-time-zone]]]])
(defn create-table! [ds table-name columns]
(log/info :table/creating {:name table-name :columns columns})
(jdbc/execute! ds [(sql/sql 'create-table 'if-not-exists table-name
(concat
default-properties
columns))])
(jdbc/execute! ds [(sql/sql 'drop-trigger 'if-exists :set-timestamp
'on table-name)])
(jdbc/execute! ds [(sql/sql 'create-trigger :set-timestamp
'before-update
'on table-name
'for-each-row
'execute-procedure [:fn 'trigger_set_timestamp])]))
(defn pg-coerce [val]
(cond
(instance? java.time.ZonedDateTime val)
(.toOffsetDateTime val)
:else
val))
(defn insert-sql [table entity props]
(into [(sql/sql 'insert-into table
(cons :rdf/props
(map key entity))
'values
(repeat (inc (count entity)) '?)
'on-conflict [:raw "(\"rdf/id\")"]
'do
'update-set
(into [:commas]
(map (fn [[k]]
[k '= '?]))
entity))]
(cons
(json/encode props)
(concat
(map (comp pg-coerce val) entity)
(map (comp pg-coerce val) entity)))))
(defn start! [{:keys [props]}]
(let [ds (doto (ComboPooledDataSource.)
(.setDriverClass "com.impossibl.postgres.jdbc.PGDriver")
(.setJdbcUrl (:url props)))]
(jdbc/execute! ds [set-ts-trigger-def])
(doseq [[table columns] tables]
(create-table! ds table columns))
(let [table-columns
(into {}
(with-open [con (jdbc/get-connection ds {})]
(let [md (.getMetaData con)]
(doall
(for [{:keys [pg_class/TABLE_NAME]}
(-> md
(.getTables nil nil nil (into-array ["TABLE" "VIEW"]))
(rs/datafiable-result-set ds {}))]
[(keyword TABLE_NAME)
(map (comp keyword :COLUMN_NAME)
(rs/datafiable-result-set (.getColumns md nil nil TABLE_NAME nil) ds {}))])))))]
{:schema table-columns
:ds ds})))
(defn stop! [{ds :value}]
#_(.close ds))
;; cpds.setUser("dbuser");
;; cpds.setPassword("dbpassword");
(def component
{:gx/start {:gx/processor #'start!}
:gx/stop {:gx/processor #'stop!}})
(ns lambdaisland.souk.db)

2
src/lambdaisland/souk/json_ld.clj
View file

@ -110,8 +110,6 @@
(map? v)
(-> v
(cond-> (contains? v "@type")
(doto prn)
(contains? v "@type")
(update "@type" shorten))
(update-keys (fn [k]
(case k

2
src/lambdaisland/souk/sql.clj
View file

@ -41,7 +41,7 @@
:str (sql-str (second x))
:raw (second x)
:list (sql-list (map sql (next x)))
:commas (sql-list "" "" ", " (map sql (next x)))
:bare-list (sql-list "" "" ", " (map sql (next x)))
:fn (str (second x)
(sql-list (map sql (nnext x))))
(apply sql x))