Hello, world!
"}} diff --git a/resources/lambdaisland/souk/ActivityStreams.edn b/resources/lambdaisland/souk/ActivityStreams.edn new file mode 100644 index 0000000..fd59e65 --- /dev/null +++ b/resources/lambdaisland/souk/ActivityStreams.edn @@ -0,0 +1,59 @@ +{:activitystreams/Actor + {:properties + [[:activitystreams/name text] + [:activitystreams/preferredUsername text] + [:activitystreams/url rdf/iri] + [:activitystreams/summary text] + [:ldp/inbox rdf/iri] + [:activitystreams/outbox rdf/iri] + [:activitystreams/published datetime]]} + + :activitystreams/Person + {:store-as :activitystreams/Actor} + + :activitystreams/Service + {:store-as :activitystreams/Actor} + + :activitystreams/Note + {:properties + [[:activitystreams/summary text] + [:activitystreams/content text] + ]} + + #_ + + {:rdf/type :activitystreams/Note, + :rdf/id "https://plexus.osrx.chat/users/plexus/statuses/109495602955086656", + :activitystreams/inReplyTo nil, + :activitystreams/published #inst "2022-12-11T14:51:48Z" + :activitystreams/to ["https://www.w3.org/ns/activitystreams#Public"], + :activitystreams/sensitive false, + :activitystreams/cc ["https://plexus.osrx.chat/users/plexus/followers"], + :activitystreams/attributedTo "https://plexus.osrx.chat/users/plexus", + :activitystreams/summary nil, + :activitystreams/tag [], + :ostatus/conversation "https://www.w3.org/ns/activitystreams#tagplexus.osrx.chat,2022-12-11", + :activitystreams/replies {:rdf/id + "https://plexus.osrx.chat/users/plexus/statuses/109495602955086656/replies", + :rdf/type :activitystreams/Collection, + :activitystreams/first + {:rdf/type :activitystreams/CollectionPage, + :activitystreams/next + "https://plexus.osrx.chat/users/plexus/statuses/109495602955086656/replies?only_other_accounts=true&page=true", + :activitystreams/partOf + "https://plexus.osrx.chat/users/plexus/statuses/109495602955086656/replies", + :activitystreams/items []}}, + :ostatus/inReplyToAtomUri nil, + :ostatus/atomUri "https://plexus.osrx.chat/users/plexus/statuses/109495602955086656", + :activitystreams/url "https://plexus.osrx.chat/@plexus/109495602955086656", + :activitystreams/attachment [], + :activitystreams/content {"en" "Hello, world!
"}} + + :activitystreams/Article + {:store-as :activitystreams/Note} + + :activitystreams/Link + {:store-as :activitystreams/Note} + + :activitystreams/Question + {:store-as :activitystreams/Note}} diff --git a/resources/lambdaisland/souk/config.edn b/resources/lambdaisland/souk/config.edn index f236f76..094efc9 100644 --- a/resources/lambdaisland/souk/config.edn +++ b/resources/lambdaisland/souk/config.edn @@ -1,13 +1,21 @@ {:http/router {:gx/component lambdaisland.souk.components.router/component - :gx/props {:dev-router? #setting :dev/reload-routes?}} + :gx/props {:dev-router? #setting :dev/reload-routes? + :storage/db (gx/ref :storage/db) + :instance/domain #setting :instance/domain}} :http/server {:gx/component lambdaisland.souk.components.jetty/component :gx/props {:jetty-options {:port #setting :port} - :router (gx/ref :http/router) - :db (gx/ref :storage/db)}} + :http/router (gx/ref :http/router)}} :storage/db - {:gx/component lambdaisland.souk.db/component - :gx/props {:url #setting :jdbc/url}}} + {:gx/component lambdaisland.souk.components.db/component + :gx/props {:url #setting :jdbc/url + :schema (gx/ref :storage/schema)}} + + :storage/schema + {:gx/component lambdaisland.souk.components.db-schema/component + :gx/props {:url #setting :jdbc/url + :admin-url #setting :jdbc/admin-url + :schemas [#resource "lambdaisland/souk/ActivityStreams.edn"]}}} diff --git a/resources/lambdaisland/souk/settings-dev.edn b/resources/lambdaisland/souk/settings-dev.edn index a7633f7..24a7682 100644 --- a/resources/lambdaisland/souk/settings-dev.edn +++ b/resources/lambdaisland/souk/settings-dev.edn @@ -1,2 +1,4 @@ {:dev/reload-routes? true - :jdbc/url "jdbc:pgsql://localhost:5432/souk?user=postgres"} + :jdbc/url "jdbc:pgsql://localhost:55432/souk?user=postgres" + :jdbc/admin-url "jdbc:pgsql://localhost:55432/postgres?user=postgres" + :instance/domain "dev.squid.casa"} diff --git a/src/lambdaisland/souk/activitypub.clj b/src/lambdaisland/souk/activitypub.clj index 3f4465a..b638350 100644 --- a/src/lambdaisland/souk/activitypub.clj +++ b/src/lambdaisland/souk/activitypub.clj @@ -12,9 +12,8 @@ "activitystreams" "https://www.w3.org/ns/activitystreams#" "xsd" "http://www.w3.org/2001/XMLSchema#" "owl" "http://www.w3.org/2002/07/owl#" - "rdfs" "http://www.w3.org/2000/01/rdf-schema#"}) + "rdfs" "http://www.w3.org/2000/01/rdf-schema#" + "ostatus" "http://ostatus.org#"}) (defn GET [url] (ld/internalize (ld/expand (:body (ld/json-get url))) common-prefixes)) - -(GET "https://toot.cat/users/plexus") diff --git a/src/lambdaisland/souk/components/db.clj b/src/lambdaisland/souk/components/db.clj new file mode 100644 index 0000000..d4ff6e3 --- /dev/null +++ b/src/lambdaisland/souk/components/db.clj @@ -0,0 +1,64 @@ +(ns lambdaisland.souk.components.db + (:require [lambdaisland.souk.sql :as sql] + [next.jdbc :as jdbc] + [next.jdbc.date-time :as jdbc-date-time] + [next.jdbc.plan] + [next.jdbc.result-set :as rs] + [next.jdbc.sql :as nsql] + [cheshire.core :as json] + [lambdaisland.glogc :as log]) + (:import (com.mchange.v2.c3p0 ComboPooledDataSource))) + +(defn pg-coerce [val] + (cond + (instance? java.time.ZonedDateTime val) + (.toOffsetDateTime val) + :else + val)) + +(defn insert-sql [table entity props] + (into [(sql/sql 'insert-into table + (cons :rdf/props + (map key entity)) + 'values + (repeat (inc (count entity)) '?) + 'on-conflict [:raw "(\"rdf/id\")"] + 'do + 'update-set + (into [:commas] + (map (fn [[k]] + [k '= '?])) + entity))] + (cons + (json/encode props) + (concat + (map (comp pg-coerce val) entity) + (map (comp pg-coerce val) entity))))) + +(defn start! [{:keys [props schema]}] + (let [ds (doto (ComboPooledDataSource.) + (.setDriverClass "com.impossibl.postgres.jdbc.PGDriver") + (.setJdbcUrl (:url props)))] + (let [table-columns + (into {} + (with-open [con (jdbc/get-connection ds {})] + (let [md (.getMetaData con)] + (doall + (for [{:keys [pg_class/TABLE_NAME]} + (-> md + (.getTables nil nil nil (into-array ["TABLE" "VIEW"])) + (rs/datafiable-result-set ds {}))] + [(keyword TABLE_NAME) + (map (comp keyword :COLUMN_NAME) + (rs/datafiable-result-set (.getColumns md nil nil TABLE_NAME nil) ds {}))])))))] + {:schema table-columns + :ds ds}))) + +(defn stop! [{ds :value}] + #_(.close ds)) + +;; cpds.setUser("dbuser"); +;; cpds.setPassword("dbpassword"); +(def component + {:gx/start {:gx/processor #'start!} + :gx/stop {:gx/processor #'stop!}}) diff --git a/src/lambdaisland/souk/components/db_schema.clj b/src/lambdaisland/souk/components/db_schema.clj new file mode 100644 index 0000000..f184fbc --- /dev/null +++ b/src/lambdaisland/souk/components/db_schema.clj @@ -0,0 +1,110 @@ +(ns lambdaisland.souk.components.db-schema + (:require + [aero.core :as aero] + [lambdaisland.glogc :as log] + [lambdaisland.souk.sql :as sql] + [lambdaisland.uri :as uri] + [next.jdbc :as jdbc] + [next.jdbc.result-set :as rs])) + +(set! *warn-on-reflection* true) + +(def default-properties + [[:rdf/id 'text 'primary-key] + [:rdf/props 'jsonb 'default "{}"] + [:meta/created-at 'timestamp-with-time-zone 'default [:fn 'now] 'not-null] + [:meta/updated-at 'timestamp-with-time-zone]]) + +(def set-ts-trigger-def "CREATE OR REPLACE FUNCTION trigger_set_timestamp()\nRETURNS TRIGGER AS $$\nBEGIN\n NEW.\"meta/updated-at\" = NOW();\n RETURN NEW;\nEND;\n$$ LANGUAGE plpgsql;") + +(def pg-types + '{text text + rdf/iri text + datetime timestamp-with-time-zone}) + +(defn jdbc-url->db-name [url] + (-> url + uri/uri + :path + uri/uri + :path + (subs 1))) + +(defn table-columns + ([ds] + (table-columns ds nil)) + ([ds opts] + (with-open [conn (jdbc/get-connection ds opts)] + (let [md (.getMetaData conn) + cols #(rs/datafiable-result-set + (.getColumns md nil nil % nil) ds) + tables #(rs/datafiable-result-set + (.getTables md nil nil nil + (into-array ["TABLE" "VIEW"])) ds opts)] + (into {} + (map (fn [{:pg_class/keys [TABLE_NAME]}] + [(keyword TABLE_NAME) + (into #{} (map (comp keyword :COLUMN_NAME)) (cols TABLE_NAME))])) + (tables)))))) + +(defn create-table! [ds table-name columns] + (log/info :table/creating {:name table-name :columns columns}) + (jdbc/execute! ds [(sql/sql 'create-table 'if-not-exists table-name + (concat + default-properties + columns))]) + (jdbc/execute! ds [(sql/sql 'drop-trigger 'if-exists :set-timestamp + 'on table-name)])) + +(defn create-table-sql [table properties] + [(sql/sql 'create-table table properties)]) + +(defn add-columns-sql [table properties] + [(sql/sql 'alter-table table + (into [:bare-list] + (map (fn [[col type]] + ['add col type])) + properties))]) + +(defn migrate-tables! [url schemas] + (doseq [schema schemas + [table {:keys [properties store-as]}] (aero/read-config schema) + :when (not store-as)] + (let [ds (jdbc/get-datasource url) + table-cols (table-columns ds nil) + all-props (concat + default-properties + (for [[column type] properties] + [column (get pg-types type)]))] + (if (contains? table-cols table) + (when-let [new-props (seq (remove (fn [[col]] + (get-in table-cols [table col])) + all-props))] + (jdbc/execute! ds (add-columns-sql table new-props)) + (log/info :table/altered {:table table :new-props (map first new-props)})) + (do + (jdbc/execute! ds (create-table-sql table all-props)) + (jdbc/execute! ds [(sql/sql 'create-trigger :set-timestamp + 'before-update + 'on table + 'for-each-row + 'execute-procedure [:fn 'trigger_set_timestamp])]) + (log/info :table/created {:table table :properties (map first all-props)})))))) + +(defn start! [{{:keys [url admin-url schemas]} :props}] + (try + (jdbc/execute! (jdbc/get-datasource admin-url) + [(sql/sql ['create-database [:ident (jdbc-url->db-name url)]])]) + (log/info :database/created {:url url}) + (catch Exception e + ;; as a poor-man's CREATE IF NOT EXISTS, we swallow this particular error, + ;; and rethrow anything else. + (when-not (re-find #"database.*already exists" (.getMessage e)) + (throw e)))) + (let [ds (jdbc/get-datasource url)] + (jdbc/execute! ds [set-ts-trigger-def]) + (migrate-tables! url schemas) + (table-columns ds))) + +(def component + {:gx/start {:gx/processor #'start!}}) diff --git a/src/lambdaisland/souk/components/jetty.clj b/src/lambdaisland/souk/components/jetty.clj index 9075c99..c0ae582 100644 --- a/src/lambdaisland/souk/components/jetty.clj +++ b/src/lambdaisland/souk/components/jetty.clj @@ -3,8 +3,6 @@ [reitit.ring :as reitit-ring]) (:import (org.eclipse.jetty.server Server))) -(def defaults {:join? false}) - (def ?JettyOptions "Start a Jetty webserver to serve the given handler according to the supplied options: @@ -67,6 +65,8 @@ [:map [:jetty-options ?JettyOptions]]) +(def defaults {:join? false}) + (defn ring-handler [reitit-router] (reitit-ring/ring-handler reitit-router @@ -74,7 +74,8 @@ (reitit-ring/create-default-handler)))) (defn http-start [{:keys [props]}] - (let [{:keys [router jetty-options]} props + (let [{:keys [jetty-options] + :http/keys [router]} props jetty-options (merge defaults jetty-options)] (ring.jetty/run-jetty (ring-handler router) jetty-options))) diff --git a/src/lambdaisland/souk/components/template.clj b/src/lambdaisland/souk/components/template.clj new file mode 100644 index 0000000..65f2e54 --- /dev/null +++ b/src/lambdaisland/souk/components/template.clj @@ -0,0 +1,12 @@ +(ns lambdaisland.souk.components.template) + +(defn start! [{:keys [props]}] + ) + +(defn stop! [{:keys [value]}] + ) + +(def component + {:gx/start {:gx/processor #'start! + :gx/props-schema [:map]} + :gx/stop {:gx/processor #'stop!}}) diff --git a/src/lambdaisland/souk/db.clj b/src/lambdaisland/souk/db.clj index c817129..0bb8c86 100644 --- a/src/lambdaisland/souk/db.clj +++ b/src/lambdaisland/souk/db.clj @@ -1,100 +1 @@ -(ns lambdaisland.souk.db - (:require [lambdaisland.souk.sql :as sql] - [next.jdbc :as jdbc] - [next.jdbc.date-time :as jdbc-date-time] - [next.jdbc.plan] - [next.jdbc.result-set :as rs] - [next.jdbc.sql :as nsql] - [cheshire.core :as json] - [lambdaisland.glogc :as log]) - (:import (com.mchange.v2.c3p0 ComboPooledDataSource))) - -(def default-properties - [[:rdf/id 'text 'primary-key] - [:rdf/type 'text] - [:rdf/props 'jsonb 'default "{}"] - [:meta/created-at 'timestamp-with-time-zone 'default [:fn 'now] 'not-null] - [:meta/updated-at 'timestamp-with-time-zone]]) - -(def set-ts-trigger-def "CREATE OR REPLACE FUNCTION trigger_set_timestamp()\nRETURNS TRIGGER AS $$\nBEGIN\n NEW.\"meta/updated-at\" = NOW();\n RETURN NEW;\nEND;\n$$ LANGUAGE plpgsql;") - -(def tables - [[:activitystreams/Actor - [[:activitystreams/name 'text 'not-null] - [:activitystreams/preferredUsername 'text 'not-null] - [:activitystreams/url 'text 'not-null] - [:activitystreams/summary 'text] - [:ldp/inbox 'text 'not-null] - [:activitystreams/outbox 'text 'not-null] - [:activitystreams/published 'timestamp-with-time-zone]]]]) - -(defn create-table! [ds table-name columns] - (log/info :table/creating {:name table-name :columns columns}) - (jdbc/execute! ds [(sql/sql 'create-table 'if-not-exists table-name - (concat - default-properties - columns))]) - (jdbc/execute! ds [(sql/sql 'drop-trigger 'if-exists :set-timestamp - 'on table-name)]) - (jdbc/execute! ds [(sql/sql 'create-trigger :set-timestamp - 'before-update - 'on table-name - 'for-each-row - 'execute-procedure [:fn 'trigger_set_timestamp])])) - -(defn pg-coerce [val] - (cond - (instance? java.time.ZonedDateTime val) - (.toOffsetDateTime val) - :else - val)) - -(defn insert-sql [table entity props] - (into [(sql/sql 'insert-into table - (cons :rdf/props - (map key entity)) - 'values - (repeat (inc (count entity)) '?) - 'on-conflict [:raw "(\"rdf/id\")"] - 'do - 'update-set - (into [:commas] - (map (fn [[k]] - [k '= '?])) - entity))] - (cons - (json/encode props) - (concat - (map (comp pg-coerce val) entity) - (map (comp pg-coerce val) entity))))) - -(defn start! [{:keys [props]}] - (let [ds (doto (ComboPooledDataSource.) - (.setDriverClass "com.impossibl.postgres.jdbc.PGDriver") - (.setJdbcUrl (:url props)))] - (jdbc/execute! ds [set-ts-trigger-def]) - (doseq [[table columns] tables] - (create-table! ds table columns)) - (let [table-columns - (into {} - (with-open [con (jdbc/get-connection ds {})] - (let [md (.getMetaData con)] - (doall - (for [{:keys [pg_class/TABLE_NAME]} - (-> md - (.getTables nil nil nil (into-array ["TABLE" "VIEW"])) - (rs/datafiable-result-set ds {}))] - [(keyword TABLE_NAME) - (map (comp keyword :COLUMN_NAME) - (rs/datafiable-result-set (.getColumns md nil nil TABLE_NAME nil) ds {}))])))))] - {:schema table-columns - :ds ds}))) - -(defn stop! [{ds :value}] - #_(.close ds)) - -;; cpds.setUser("dbuser"); -;; cpds.setPassword("dbpassword"); -(def component - {:gx/start {:gx/processor #'start!} - :gx/stop {:gx/processor #'stop!}}) +(ns lambdaisland.souk.db) diff --git a/src/lambdaisland/souk/json_ld.clj b/src/lambdaisland/souk/json_ld.clj index e6abff3..8a5a8a0 100644 --- a/src/lambdaisland/souk/json_ld.clj +++ b/src/lambdaisland/souk/json_ld.clj @@ -110,8 +110,6 @@ (map? v) (-> v (cond-> (contains? v "@type") - (doto prn) - (contains? v "@type") (update "@type" shorten)) (update-keys (fn [k] (case k diff --git a/src/lambdaisland/souk/sql.clj b/src/lambdaisland/souk/sql.clj index ccb16fb..134e9b3 100644 --- a/src/lambdaisland/souk/sql.clj +++ b/src/lambdaisland/souk/sql.clj @@ -41,7 +41,7 @@ :str (sql-str (second x)) :raw (second x) :list (sql-list (map sql (next x))) - :commas (sql-list "" "" ", " (map sql (next x))) + :bare-list (sql-list "" "" ", " (map sql (next x))) :fn (str (second x) (sql-list (map sql (nnext x)))) (apply sql x))