diff --git a/.gitignore b/.gitignore index d963ec7..71683c7 100644 --- a/.gitignore +++ b/.gitignore @@ -218,6 +218,7 @@ dist # Stores VSCode versions used for testing VSCode extensions .vscode-test +.vscode/ # yarn v2 .yarn/cache diff --git a/data/sql/dsm.sql b/data/sql/dsm.sql index 97cb2b3..1738ed9 100644 --- a/data/sql/dsm.sql +++ b/data/sql/dsm.sql @@ -1,3 +1,8 @@ +-- TODO: This variable should be obtained from the system ENV +\set DB dnet_dsm +CREATE DATABASE :DB; +\c :DB + BEGIN; CREATE TABLE dsm_services ( @@ -195,4 +200,4 @@ FROM dsm_services s LEFT JOIN dsm_api a ON (s.id = a.service); -COMMIT; \ No newline at end of file +COMMIT; diff --git a/data/sql/mdstore_schema.sql b/data/sql/mdstores.sql similarity index 100% rename from data/sql/mdstore_schema.sql rename to data/sql/mdstores.sql diff --git a/data/sql/schema.sql b/data/sql/resources.sql similarity index 99% rename from data/sql/schema.sql rename to data/sql/resources.sql index 41ac895..fb70c90 100644 --- a/data/sql/schema.sql +++ b/data/sql/resources.sql @@ -1,3 +1,7 @@ +-- This is the main DB, it should be automatically created -- + +BEGIN; + -- Vocabularies CREATE TABLE vocabularies ( @@ -244,3 +248,4 @@ CREATE TABLE emails ( message text NOT NULL ); +COMMIT; diff --git a/data/sql/wfs.sql b/data/sql/wfs.sql index 555ca39..7744ff9 100644 --- a/data/sql/wfs.sql +++ b/data/sql/wfs.sql @@ -1,3 +1,10 @@ +-- TODO: This variable should be obtained from the system ENV +\set DB dnet_wfs +CREATE DATABASE :DB; +\c :DB + +BEGIN; + -- WF History CREATE TABLE wf_history ( @@ -59,3 +66,4 @@ CREATE TABLE wf_subscriptions ( PRIMARY KEY (wf_conf_id, condition, email) ); +COMMIT; diff --git a/data/sql/wfs.sql~ b/data/sql/wfs.sql~ deleted file mode 100644 index 656e2bb..0000000 --- a/data/sql/wfs.sql~ +++ /dev/null @@ -1,307 +0,0 @@ --- Vocabularies - -CREATE TABLE vocabularies ( - id text PRIMARY KEY, - name text NOT NULL, - description text -); - -CREATE TABLE vocabulary_terms ( - vocabulary text NOT NULL REFERENCES vocabularies(id) ON UPDATE CASCADE ON DELETE CASCADE, - code text NOT NULL, - name text NOT NULL, - encoding text DEFAULT 'OPENAIRE', - synonyms jsonb, - PRIMARY KEY (vocabulary, code) -); - -CREATE INDEX ON vocabulary_terms (vocabulary); - -CREATE TABLE protocols ( - id text PRIMARY KEY -); -INSERT INTO protocols(id) VALUES ('oai'),('oai_sets'),('http'),('file'),('classpath'),('fileCSV'),('httpCSV'),('ftp'),('sftp'),('filesystem'),('files_from_metadata'),('files_from_mdstore'),('mongoDump'),('targz'),('zip'),('fileGzip'),('httpList'),('remoteMdstore'); - -CREATE TABLE protocol_params ( - protocol text NOT NULL REFERENCES protocols(id) ON UPDATE CASCADE ON DELETE CASCADE, - param_name text NOT NULL, - param_label text NOT NULL, - param_type text NOT NULL DEFAULT 'TEXT', - optional boolean NOT NULL default false, - has_sel_function boolean NOT NULL default false, - PRIMARY KEY (protocol, param_name) -); -INSERT INTO protocol_params(protocol, param_name, param_label, param_type, optional, has_sel_function) VALUES -('oai', 'set', 'OAI set', 'LIST', true, true), -('oai', 'format', 'OAI Metadata Format', 'TEXT', false, false), -('http', 'splitOnElement', 'splitOnElement', 'TEXT', false, false), -('file', 'splitOnElement', 'splitOnElement', 'TEXT', false, false), -('classpath', 'splitOnElement', 'splitOnElement', 'TEXT', false, false), -('fileCSV', 'header', 'header', 'TEXT', false, false), -('fileCSV', 'separator', 'separator', 'TEXT', false, false), -('fileCSV', 'identifier', 'identifier', 'TEXT', false, false), -('fileCSV', 'quote', 'quote', 'TEXT', false, false), -('httpCSV', 'separator', 'separator', 'TEXT', false, false), -('httpCSV', 'identifier', 'identifier', 'TEXT', false, false), -('httpCSV', 'quote', 'quote', 'TEXT', false, false), -('ftp', 'username', 'username', 'TEXT', false, false), -('ftp', 'password', 'password', 'TEXT', false, false), -('ftp', 'recursive', 'recursive', 'BOOLEAN', false, false), -('ftp', 'extensions', 'extensions', 'LIST', false, false), -('sftp', 'username', 'username', 'TEXT', false, false), -('sftp', 'password', 'password', 'TEXT', true, false), -('sftp', 'authMethod', 'authMethod', 'TEXT', true, false), -('sftp', 'privateKeyPath', 'privateKeyPath', 'TEXT', true, false), -('sftp', 'port', 'port', 'TEXT', true, false), -('sftp', 'recursive', 'recursive', 'BOOLEAN', false, false), -('sftp', 'extensions', 'extensions', 'LIST', false, false); - --- Contexts - -CREATE TABLE contexts ( - id text PRIMARY KEY, - label text NOT NULL, - type text NOT NULL, - params jsonb -); - -CREATE TABLE context_categories ( - id text NOT NULL PRIMARY KEY, - parent text NOT NULL REFERENCES contexts(id) ON UPDATE CASCADE ON DELETE CASCADE, - label text NOT NULL, - claim boolean NOT NULL, - params jsonb -); - -CREATE TABLE context_cat_concepts_lvl_0 ( - id text NOT NULL PRIMARY KEY, - parent text NOT NULL REFERENCES context_categories(id) ON UPDATE CASCADE ON DELETE CASCADE, - label text NOT NULL, - claim boolean NOT NULL, - params jsonb -); - -CREATE TABLE context_cat_concepts_lvl_1 ( - id text NOT NULL PRIMARY KEY, - parent text NOT NULL REFERENCES context_cat_concepts_lvl_0(id) ON UPDATE CASCADE ON DELETE CASCADE, - label text NOT NULL, - claim boolean NOT NULL, - params jsonb -); - -CREATE TABLE context_cat_concepts_lvl_2 ( - id text NOT NULL PRIMARY KEY, - parent text NOT NULL REFERENCES context_cat_concepts_lvl_1(id) ON UPDATE CASCADE ON DELETE CASCADE, - label text NOT NULL, - claim boolean NOT NULL, - params jsonb -); - -CREATE INDEX ON context_categories (parent); -CREATE INDEX ON context_cat_concepts_lvl_0 (parent); -CREATE INDEX ON context_cat_concepts_lvl_1 (parent); -CREATE INDEX ON context_cat_concepts_lvl_2 (parent); - --- WF History - -CREATE TABLE wf_history ( - process_id text PRIMARY KEY, - wf_conf_id text NOT NULL, - name text NOT NULL, - family text NOT NULL, - status text NOT NULL, - start_date timestamp NOT NULL, - end_date timestamp NOT NULL, - ds_id text, - ds_name text, - ds_api text, - details jsonb -); - --- Other Resources - -CREATE TABLE resource_types( - id text PRIMARY KEY, - name text NOT NULL, - content_type text NOT NULL DEFAULT 'text/plain' -); - -INSERT INTO resource_types(id, name, content_type) VALUES - ('transformation_rule_xslt', 'Transformation Rules (xslt)', 'application/xml'), - ('transformation_rule_legacy', 'Transformation Rules (legacy)', 'text/plain'), - ('cleaning_rule', 'Cleaning Rules', 'application/xml'), - ('hadoop_job_configuration', 'Hadoop Job Configurations', 'application/xml'), - ('dedup_configuration', 'Dedup Configurations', 'application/json'), - ('wf_template', 'Workflow Templates', 'application/json'); - -CREATE TABLE resources ( - id text PRIMARY KEY, - name text NOT NULL, - description text, - content text NOT NULL DEFAULT '', - type text NOT NULL REFERENCES resource_types(id), - subtype text, - creation_date timestamp NOT NULL DEFAULT now(), - modification_date timestamp NOT NULL DEFAULT now() -); - -CREATE VIEW resource_types_view AS ( - SELECT - t.id AS id, - t.name AS name, - t.content_type AS content_type, - count(r.id) AS count, - true AS simple - FROM resource_types t - LEFT OUTER JOIN resources r ON (r.type = t.id) - GROUP BY t.id, t.name - ORDER BY t.name -) UNION ALL ( - SELECT - 'vocabulary' AS id, - 'Vocabularies' AS name, - 'text/plain' AS content_type, - count(*) AS count, - false AS simple - FROM vocabularies -) UNION ALL ( - SELECT - 'context' AS id, - 'Contexts' AS name, - 'text/plain' AS content_type, - count(*) AS count, - false AS simple - FROM contexts -) UNION ALL ( - SELECT - 'protocol' AS id, - 'Protocols' AS name, - 'text/plain' AS content_type, - count(*) AS count, - false AS simple - FROM protocols -) UNION ALL ( - SELECT - 'email' AS id, - 'Email templates' AS name, - 'text/plain' AS content_type, - count(*) AS count, - false AS simple - FROM emails -); - -CREATE TABLE mdstores ( - id text PRIMARY KEY, - format text NOT NULL, - layout text NOT NULL, - interpretation text NOT NULL, - type text NOT NULL, - datasource_name text, - datasource_id text, - api_id text, - creation_date timestamp NOT NULL DEFAULT now(), - params jsonb -); - -CREATE TABLE mdstore_versions ( - id text PRIMARY KEY, - mdstore text NOT NULL REFERENCES mdstores(id), - writing boolean NOT NULL, - readcount int NOT NULL DEFAULT 0, - lastupdate timestamp NOT NULL DEFAULT now(), - size bigint NOT NULL DEFAULT 0, - params jsonb -); - -CREATE TABLE mdstore_current_versions ( - mdstore text PRIMARY KEY REFERENCES mdstores(id), - current_version text NOT NULL REFERENCES mdstore_versions(id) -); - -CREATE VIEW mdstores_with_info AS SELECT - md.id AS id, - md.format AS format, - md.layout AS layout, - md.type AS type, - md.interpretation AS interpretation, - md.datasource_name AS datasource_name, - md.datasource_id AS datasource_id, - md.api_id AS api_id, - md.params AS params, - md.creation_date as creation_date, - cv.current_version AS current_version, - v1.lastupdate AS lastupdate, - v1.size AS size, - count(v2.id) AS n_versions -FROM - mdstores md - LEFT OUTER JOIN mdstore_current_versions cv ON (md.id = cv.mdstore) - LEFT OUTER JOIN mdstore_versions v1 ON (cv.current_version = v1.id) - LEFT OUTER JOIN mdstore_versions v2 ON (md.id = v2.mdstore) -GROUP BY md.id, - md.format, - md.layout, - md.interpretation, - md.type, - md.datasource_name, - md.datasource_id, - md.params, - md.creation_date, - md.api_id, - cv.current_version, - v1.lastupdate, - v1.size; - --- Email Templates -CREATE TABLE emails ( - id text PRIMARY KEY, - description text NOT NULL, - subject text NOT NULL, - message text NOT NULL -); - --- Workflows - -CREATE TABLE wf_sections ( - id text PRIMARY KEY, - name text NOT NULL -); - -INSERT INTO wf_sections(id, name) VALUES -('GC', 'Garbage Collection'), -('CONSISTENCY', 'InfoSpace Consistency'), -('DEDUP', 'InfoSpace Deduplication'), -('INFERENCE', 'InfoSpace Inference'), -('MONITOR', 'InfoSpace Monitoring'), -('PROVISION', 'InfoSpace Provision'), -('IS', 'Information Service'), -('BROKER', 'Notification Broker'); - -CREATE TABLE wf_configurations ( - id text PRIMARY KEY, - name text NOT NULL, - section text REFERENCES wf_sections(id), - details jsonb NOT NULL DEFAULT '{}', - priority int, - dsid text, - dsname text, - apiid text, - enabled boolean NOT NULL DEFAULT false, - configured boolean NOT NULL DEFAULT false, - scheduling_enabled boolean NOT NULL DEFAULT false, - scheduling_cron text, - scheduling_min_interval int, - workflow text REFERENCES resources(id), - destroy_wf text REFERENCES resources(id), - system_params jsonb NOT NULL DEFAULT '{}', - user_params jsonb NOT NULL DEFAULT '{}' -); - -CREATE TABLE wf_subscriptions ( - wf_conf_id text NOT NULL REFERENCES wf_configurations(id), - condition text NOT NULL, - email text NOT NULL, - message_id text NOT NULL REFERENCES emails(id), - PRIMARY KEY (wf_conf_id, condition, email) -); - diff --git a/docker-build-and-start.sh b/docker-build-and-start.sh new file mode 100755 index 0000000..fbaca51 --- /dev/null +++ b/docker-build-and-start.sh @@ -0,0 +1,3 @@ +#!/bin/sh + +(cd dnet-app && mvn clean install) && ./docker_start.sh diff --git a/docker-compose.yml b/docker-compose.yml index 0fb24bc..b9264ff 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -2,7 +2,7 @@ version: "3" services: context-manager: build: ./dnet-app/apps/dnet-context-manager - expose: + expose: - ${SPRING_BOOT_PORT} networks: - backend @@ -25,23 +25,23 @@ services: - SPRING_DATASOURCE_URL=jdbc:postgresql://db-main:${PG_PORT}/${PG_MAIN_DB} - SPRING_DATASOURCE_USERNAME=${PG_USER} - SPRING_DATASOURCE_PASSWORD=${PG_PASSWORD} - + dsm: build: ./dnet-app/apps/dnet-datasource-manager - expose: + expose: - ${SPRING_BOOT_PORT} networks: - backend depends_on: - - db-dsm + - db-main environment: - - SPRING_DATASOURCE_URL=jdbc:postgresql://db-dsm:${PG_PORT}/${PG_DSM_DB} + - SPRING_DATASOURCE_URL=jdbc:postgresql://db-main:${PG_PORT}/${PG_DSM_DB} - SPRING_DATASOURCE_USERNAME=${PG_USER} - SPRING_DATASOURCE_PASSWORD=${PG_PASSWORD} - + indexer: build: ./dnet-app/apps/dnet-indexer-solr - expose: + expose: - ${SPRING_BOOT_PORT} networks: - backend @@ -54,7 +54,7 @@ services: mdsm: build: ./dnet-app/apps/dnet-mdstore-manager - expose: + expose: - ${SPRING_BOOT_PORT} networks: - backend @@ -67,7 +67,7 @@ services: oai-exporter: build: ./dnet-app/apps/dnet-oai-exporter - expose: + expose: - ${SPRING_BOOT_PORT} networks: - backend @@ -80,7 +80,7 @@ services: resource-manager: build: ./dnet-app/apps/dnet-resource-manager - expose: + expose: - ${SPRING_BOOT_PORT} networks: - backend @@ -93,33 +93,33 @@ services: wf-exec-postgres: build: ./dnet-app/apps/dnet-wf-executor-postgres - expose: + expose: - ${SPRING_BOOT_PORT} networks: - backend depends_on: - - db-wfs + - db-main environment: - - SPRING_DATASOURCE_URL=jdbc:postgresql://db-wfs:${PG_PORT}/${PG_WFS_DB} + - SPRING_DATASOURCE_URL=jdbc:postgresql://db-main:${PG_PORT}/${PG_WFS_DB} - SPRING_DATASOURCE_USERNAME=${PG_USER} - SPRING_DATASOURCE_PASSWORD=${PG_PASSWORD} wf-manager: build: ./dnet-app/apps/dnet-wf-manager - expose: + expose: - ${SPRING_BOOT_PORT} networks: - backend depends_on: - - db-wfs + - db-main environment: - - SPRING_DATASOURCE_URL=jdbc:postgresql://db-wfs:${PG_PORT}/${PG_WFS_DB} + - SPRING_DATASOURCE_URL=jdbc:postgresql://db-main:${PG_PORT}/${PG_WFS_DB} - SPRING_DATASOURCE_USERNAME=${PG_USER} - SPRING_DATASOURCE_PASSWORD=${PG_PASSWORD} mail-sender: build: ./dnet-app/apps/dnet-email-sender - expose: + expose: - ${SPRING_BOOT_PORT} networks: - backend @@ -138,27 +138,16 @@ services: networks: - backend environment: - POSTGRES_USER: ${PG_USER} - POSTGRES_PASSWORD: ${PG_PASSWORD} POSTGRES_DB: ${PG_MAIN_DB} - volumes: - - ./data/sql/schema.sql:/docker-entrypoint-initdb.d/init.sql - - pg_main_data:/var/lib/postgresql/data - - db-dsm: - image: postgres:15.4 - restart: always - expose: - - ${PG_PORT} - networks: - - backend - environment: + POSTGRES_DSM_DB: ${PG_DSM_DB} + POSTGRES_WFS_DB: ${PG_WFS_DB} POSTGRES_USER: ${PG_USER} POSTGRES_PASSWORD: ${PG_PASSWORD} - POSTGRES_DB: ${PG_DSM_DB} volumes: - - ./data/sql/dsm.sql:/docker-entrypoint-initdb.d/init.sql - - pg_dsm_data:/var/lib/postgresql/data + - pg_main_data:/var/lib/postgresql/data + - ./data/sql/resources.sql:/docker-entrypoint-initdb.d/init_resources.sql + - ./data/sql/dsm.sql:/docker-entrypoint-initdb.d/init_dsm.sql + - ./data/sql/wfs.sql:/docker-entrypoint-initdb.d/init_wfs.sql db-mdstores: image: postgres:15.4 @@ -172,28 +161,13 @@ services: POSTGRES_PASSWORD: ${PG_PASSWORD} POSTGRES_DB: ${PG_MDSTORE_DB} volumes: - - ./data/sql/mdstore_schema.sql:/docker-entrypoint-initdb.d/init.sql - - pg_mdstore_data:/var/lib/postgresql/data + - ./data/sql/mdstores:/docker-entrypoint-initdb.d/init.sql + - pg_mdstore_data:/var/lib/postgresql/data - db-wfs: - image: postgres:15.4 - restart: always - expose: - - ${PG_PORT} - networks: - - backend - environment: - POSTGRES_USER: ${PG_USER} - POSTGRES_PASSWORD: ${PG_PASSWORD} - POSTGRES_DB: ${PG_WFS_DB} - volumes: - - ./data/sql/wfs.sql:/docker-entrypoint-initdb.d/init.sql - - pg_wfs_data:/var/lib/postgresql/data - solr: image: solr:9.3.0 expose: - - ${SOLR_PORT} + - ${SOLR_PORT} networks: - backend volumes: @@ -204,7 +178,7 @@ services: web: build: ./dnet-app/frontends/is - ports: + ports: - ${PUBLIC_PORT}:${PUBLIC_PORT} volumes: - /app/node_modules @@ -231,10 +205,8 @@ networks: backend: frontend: + volumes: solrdata: pg_main_data: - pg_dsm_data: pg_mdstore_data: - pg_wfs_data: - \ No newline at end of file diff --git a/docker-start.sh b/docker-start.sh index c847ca2..ed8993b 100755 --- a/docker-start.sh +++ b/docker-start.sh @@ -1,7 +1,5 @@ #!/bin/sh -(cd dnet-app && mvn clean install) - export PUBLIC_PORT=8888 export SPRING_BOOT_PORT=8080 export SOLR_PORT=8983 @@ -15,4 +13,3 @@ export PG_MDSTORE_DB=dnet_mdstores export PG_WFS_DB=dnet_wfs docker-compose up --force-recreate --build -