michele.artini
/
dnet-docker
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

unique db

This commit is contained in:
Michele Artini 2023-09-28 10:28:33 +02:00
parent 29dd244fe2
commit a59aa01aac
9 changed files with 51 additions and 367 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@ -218,6 +218,7 @@ dist
# Stores VSCode versions used for testing VSCode extensions
.vscode-test
.vscode/
# yarn v2
.yarn/cache

7
data/sql/dsm.sql
View File

@ -1,3 +1,8 @@
-- TODO: This variable should be obtained from the system ENV
\set DB dnet_dsm
CREATE DATABASE :DB;
\c :DB
BEGIN;
CREATE TABLE dsm_services (
@ -195,4 +200,4 @@ FROM
dsm_services s
LEFT JOIN dsm_api a ON (s.id = a.service);
COMMIT;
COMMIT;

0
data/sql/mdstore_schema.sql → data/sql/mdstores.sql
View File

5
data/sql/schema.sql → data/sql/resources.sql
View File

@ -1,3 +1,7 @@
-- This is the main DB, it should be automatically created --
BEGIN;
-- Vocabularies
CREATE TABLE vocabularies (
@ -244,3 +248,4 @@ CREATE TABLE emails (
message text NOT NULL
);
COMMIT;

8
data/sql/wfs.sql
View File

@ -1,3 +1,10 @@
-- TODO: This variable should be obtained from the system ENV
\set DB dnet_wfs
CREATE DATABASE :DB;
\c :DB
BEGIN;
-- WF History
CREATE TABLE wf_history (
@ -59,3 +66,4 @@ CREATE TABLE wf_subscriptions (
PRIMARY KEY (wf_conf_id, condition, email)
);
COMMIT;

307
data/sql/wfs.sql~
View File

@ -1,307 +0,0 @@
-- Vocabularies
CREATE TABLE vocabularies (
id text PRIMARY KEY,
name text NOT NULL,
description text
);
CREATE TABLE vocabulary_terms (
vocabulary text NOT NULL REFERENCES vocabularies(id) ON UPDATE CASCADE ON DELETE CASCADE,
code text NOT NULL,
name text NOT NULL,
encoding text DEFAULT 'OPENAIRE',
synonyms jsonb,
PRIMARY KEY (vocabulary, code)
);
CREATE INDEX ON vocabulary_terms (vocabulary);
CREATE TABLE protocols (
id text PRIMARY KEY
);
INSERT INTO protocols(id) VALUES ('oai'),('oai_sets'),('http'),('file'),('classpath'),('fileCSV'),('httpCSV'),('ftp'),('sftp'),('filesystem'),('files_from_metadata'),('files_from_mdstore'),('mongoDump'),('targz'),('zip'),('fileGzip'),('httpList'),('remoteMdstore');
CREATE TABLE protocol_params (
protocol text NOT NULL REFERENCES protocols(id) ON UPDATE CASCADE ON DELETE CASCADE,
param_name text NOT NULL,
param_label text NOT NULL,
param_type text NOT NULL DEFAULT 'TEXT',
optional boolean NOT NULL default false,
has_sel_function boolean NOT NULL default false,
PRIMARY KEY (protocol, param_name)
);
INSERT INTO protocol_params(protocol, param_name, param_label, param_type, optional, has_sel_function) VALUES
('oai', 'set', 'OAI set', 'LIST', true, true),
('oai', 'format', 'OAI Metadata Format', 'TEXT', false, false),
('http', 'splitOnElement', 'splitOnElement', 'TEXT', false, false),
('file', 'splitOnElement', 'splitOnElement', 'TEXT', false, false),
('classpath', 'splitOnElement', 'splitOnElement', 'TEXT', false, false),
('fileCSV', 'header', 'header', 'TEXT', false, false),
('fileCSV', 'separator', 'separator', 'TEXT', false, false),
('fileCSV', 'identifier', 'identifier', 'TEXT', false, false),
('fileCSV', 'quote', 'quote', 'TEXT', false, false),
('httpCSV', 'separator', 'separator', 'TEXT', false, false),
('httpCSV', 'identifier', 'identifier', 'TEXT', false, false),
('httpCSV', 'quote', 'quote', 'TEXT', false, false),
('ftp', 'username', 'username', 'TEXT', false, false),
('ftp', 'password', 'password', 'TEXT', false, false),
('ftp', 'recursive', 'recursive', 'BOOLEAN', false, false),
('ftp', 'extensions', 'extensions', 'LIST', false, false),
('sftp', 'username', 'username', 'TEXT', false, false),
('sftp', 'password', 'password', 'TEXT', true, false),
('sftp', 'authMethod', 'authMethod', 'TEXT', true, false),
('sftp', 'privateKeyPath', 'privateKeyPath', 'TEXT', true, false),
('sftp', 'port', 'port', 'TEXT', true, false),
('sftp', 'recursive', 'recursive', 'BOOLEAN', false, false),
('sftp', 'extensions', 'extensions', 'LIST', false, false);
-- Contexts
CREATE TABLE contexts (
id text PRIMARY KEY,
label text NOT NULL,
type text NOT NULL,
params jsonb
);
CREATE TABLE context_categories (
id text NOT NULL PRIMARY KEY,
parent text NOT NULL REFERENCES contexts(id) ON UPDATE CASCADE ON DELETE CASCADE,
label text NOT NULL,
claim boolean NOT NULL,
params jsonb
);
CREATE TABLE context_cat_concepts_lvl_0 (
id text NOT NULL PRIMARY KEY,
parent text NOT NULL REFERENCES context_categories(id) ON UPDATE CASCADE ON DELETE CASCADE,
label text NOT NULL,
claim boolean NOT NULL,
params jsonb
);
CREATE TABLE context_cat_concepts_lvl_1 (
id text NOT NULL PRIMARY KEY,
parent text NOT NULL REFERENCES context_cat_concepts_lvl_0(id) ON UPDATE CASCADE ON DELETE CASCADE,
label text NOT NULL,
claim boolean NOT NULL,
params jsonb
);
CREATE TABLE context_cat_concepts_lvl_2 (
id text NOT NULL PRIMARY KEY,
parent text NOT NULL REFERENCES context_cat_concepts_lvl_1(id) ON UPDATE CASCADE ON DELETE CASCADE,
label text NOT NULL,
claim boolean NOT NULL,
params jsonb
);
CREATE INDEX ON context_categories (parent);
CREATE INDEX ON context_cat_concepts_lvl_0 (parent);
CREATE INDEX ON context_cat_concepts_lvl_1 (parent);
CREATE INDEX ON context_cat_concepts_lvl_2 (parent);
-- WF History
CREATE TABLE wf_history (
process_id text PRIMARY KEY,
wf_conf_id text NOT NULL,
name text NOT NULL,
family text NOT NULL,
status text NOT NULL,
start_date timestamp NOT NULL,
end_date timestamp NOT NULL,
ds_id text,
ds_name text,
ds_api text,
details jsonb
);
-- Other Resources
CREATE TABLE resource_types(
id text PRIMARY KEY,
name text NOT NULL,
content_type text NOT NULL DEFAULT 'text/plain'
);
INSERT INTO resource_types(id, name, content_type) VALUES
('transformation_rule_xslt', 'Transformation Rules (xslt)', 'application/xml'),
('transformation_rule_legacy', 'Transformation Rules (legacy)', 'text/plain'),
('cleaning_rule', 'Cleaning Rules', 'application/xml'),
('hadoop_job_configuration', 'Hadoop Job Configurations', 'application/xml'),
('dedup_configuration', 'Dedup Configurations', 'application/json'),
('wf_template', 'Workflow Templates', 'application/json');
CREATE TABLE resources (
id text PRIMARY KEY,
name text NOT NULL,
description text,
content text NOT NULL DEFAULT '',
type text NOT NULL REFERENCES resource_types(id),
subtype text,
creation_date timestamp NOT NULL DEFAULT now(),
modification_date timestamp NOT NULL DEFAULT now()
);
CREATE VIEW resource_types_view AS (
SELECT
t.id AS id,
t.name AS name,
t.content_type AS content_type,
count(r.id) AS count,
true AS simple
FROM resource_types t
LEFT OUTER JOIN resources r ON (r.type = t.id)
GROUP BY t.id, t.name
ORDER BY t.name
) UNION ALL (
SELECT
'vocabulary' AS id,
'Vocabularies' AS name,
'text/plain' AS content_type,
count(*) AS count,
false AS simple
FROM vocabularies
) UNION ALL (
SELECT
'context' AS id,
'Contexts' AS name,
'text/plain' AS content_type,
count(*) AS count,
false AS simple
FROM contexts
) UNION ALL (
SELECT
'protocol' AS id,
'Protocols' AS name,
'text/plain' AS content_type,
count(*) AS count,
false AS simple
FROM protocols
) UNION ALL (
SELECT
'email' AS id,
'Email templates' AS name,
'text/plain' AS content_type,
count(*) AS count,
false AS simple
FROM emails
);
CREATE TABLE mdstores (
id text PRIMARY KEY,
format text NOT NULL,
layout text NOT NULL,
interpretation text NOT NULL,
type text NOT NULL,
datasource_name text,
datasource_id text,
api_id text,
creation_date timestamp NOT NULL DEFAULT now(),
params jsonb
);
CREATE TABLE mdstore_versions (
id text PRIMARY KEY,
mdstore text NOT NULL REFERENCES mdstores(id),
writing boolean NOT NULL,
readcount int NOT NULL DEFAULT 0,
lastupdate timestamp NOT NULL DEFAULT now(),
size bigint NOT NULL DEFAULT 0,
params jsonb
);
CREATE TABLE mdstore_current_versions (
mdstore text PRIMARY KEY REFERENCES mdstores(id),
current_version text NOT NULL REFERENCES mdstore_versions(id)
);
CREATE VIEW mdstores_with_info AS SELECT
md.id AS id,
md.format AS format,
md.layout AS layout,
md.type AS type,
md.interpretation AS interpretation,
md.datasource_name AS datasource_name,
md.datasource_id AS datasource_id,
md.api_id AS api_id,
md.params AS params,
md.creation_date as creation_date,
cv.current_version AS current_version,
v1.lastupdate AS lastupdate,
v1.size AS size,
count(v2.id) AS n_versions
FROM
mdstores md
LEFT OUTER JOIN mdstore_current_versions cv ON (md.id = cv.mdstore)
LEFT OUTER JOIN mdstore_versions v1 ON (cv.current_version = v1.id)
LEFT OUTER JOIN mdstore_versions v2 ON (md.id = v2.mdstore)
GROUP BY md.id,
md.format,
md.layout,
md.interpretation,
md.type,
md.datasource_name,
md.datasource_id,
md.params,
md.creation_date,
md.api_id,
cv.current_version,
v1.lastupdate,
v1.size;
-- Email Templates
CREATE TABLE emails (
id text PRIMARY KEY,
description text NOT NULL,
subject text NOT NULL,
message text NOT NULL
);
-- Workflows
CREATE TABLE wf_sections (
id text PRIMARY KEY,
name text NOT NULL
);
INSERT INTO wf_sections(id, name) VALUES
('GC', 'Garbage Collection'),
('CONSISTENCY', 'InfoSpace Consistency'),
('DEDUP', 'InfoSpace Deduplication'),
('INFERENCE', 'InfoSpace Inference'),
('MONITOR', 'InfoSpace Monitoring'),
('PROVISION', 'InfoSpace Provision'),
('IS', 'Information Service'),
('BROKER', 'Notification Broker');
CREATE TABLE wf_configurations (
id text PRIMARY KEY,
name text NOT NULL,
section text REFERENCES wf_sections(id),
details jsonb NOT NULL DEFAULT '{}',
priority int,
dsid text,
dsname text,
apiid text,
enabled boolean NOT NULL DEFAULT false,
configured boolean NOT NULL DEFAULT false,
scheduling_enabled boolean NOT NULL DEFAULT false,
scheduling_cron text,
scheduling_min_interval int,
workflow text REFERENCES resources(id),
destroy_wf text REFERENCES resources(id),
system_params jsonb NOT NULL DEFAULT '{}',
user_params jsonb NOT NULL DEFAULT '{}'
);
CREATE TABLE wf_subscriptions (
wf_conf_id text NOT NULL REFERENCES wf_configurations(id),
condition text NOT NULL,
email text NOT NULL,
message_id text NOT NULL REFERENCES emails(id),
PRIMARY KEY (wf_conf_id, condition, email)
);

3
docker-build-and-start.sh Executable file
View File

@ -0,0 +1,3 @@
#!/bin/sh
(cd dnet-app && mvn clean install) && ./docker_start.sh

84
docker-compose.yml
View File

@ -2,7 +2,7 @@ version: "3"
services:
context-manager:
build: ./dnet-app/apps/dnet-context-manager
expose:
expose:
- ${SPRING_BOOT_PORT}
networks:
- backend
@ -25,23 +25,23 @@ services:
- SPRING_DATASOURCE_URL=jdbc:postgresql://db-main:${PG_PORT}/${PG_MAIN_DB}
- SPRING_DATASOURCE_USERNAME=${PG_USER}
- SPRING_DATASOURCE_PASSWORD=${PG_PASSWORD}
dsm:
build: ./dnet-app/apps/dnet-datasource-manager
expose:
expose:
- ${SPRING_BOOT_PORT}
networks:
- backend
depends_on:
- db-dsm
- db-main
environment:
- SPRING_DATASOURCE_URL=jdbc:postgresql://db-dsm:${PG_PORT}/${PG_DSM_DB}
- SPRING_DATASOURCE_URL=jdbc:postgresql://db-main:${PG_PORT}/${PG_DSM_DB}
- SPRING_DATASOURCE_USERNAME=${PG_USER}
- SPRING_DATASOURCE_PASSWORD=${PG_PASSWORD}
indexer:
build: ./dnet-app/apps/dnet-indexer-solr
expose:
expose:
- ${SPRING_BOOT_PORT}
networks:
- backend
@ -54,7 +54,7 @@ services:
mdsm:
build: ./dnet-app/apps/dnet-mdstore-manager
expose:
expose:
- ${SPRING_BOOT_PORT}
networks:
- backend
@ -67,7 +67,7 @@ services:
oai-exporter:
build: ./dnet-app/apps/dnet-oai-exporter
expose:
expose:
- ${SPRING_BOOT_PORT}
networks:
- backend
@ -80,7 +80,7 @@ services:
resource-manager:
build: ./dnet-app/apps/dnet-resource-manager
expose:
expose:
- ${SPRING_BOOT_PORT}
networks:
- backend
@ -93,33 +93,33 @@ services:
wf-exec-postgres:
build: ./dnet-app/apps/dnet-wf-executor-postgres
expose:
expose:
- ${SPRING_BOOT_PORT}
networks:
- backend
depends_on:
- db-wfs
- db-main
environment:
- SPRING_DATASOURCE_URL=jdbc:postgresql://db-wfs:${PG_PORT}/${PG_WFS_DB}
- SPRING_DATASOURCE_URL=jdbc:postgresql://db-main:${PG_PORT}/${PG_WFS_DB}
- SPRING_DATASOURCE_USERNAME=${PG_USER}
- SPRING_DATASOURCE_PASSWORD=${PG_PASSWORD}
wf-manager:
build: ./dnet-app/apps/dnet-wf-manager
expose:
expose:
- ${SPRING_BOOT_PORT}
networks:
- backend
depends_on:
- db-wfs
- db-main
environment:
- SPRING_DATASOURCE_URL=jdbc:postgresql://db-wfs:${PG_PORT}/${PG_WFS_DB}
- SPRING_DATASOURCE_URL=jdbc:postgresql://db-main:${PG_PORT}/${PG_WFS_DB}
- SPRING_DATASOURCE_USERNAME=${PG_USER}
- SPRING_DATASOURCE_PASSWORD=${PG_PASSWORD}
mail-sender:
build: ./dnet-app/apps/dnet-email-sender
expose:
expose:
- ${SPRING_BOOT_PORT}
networks:
- backend
@ -138,27 +138,16 @@ services:
networks:
- backend
environment:
POSTGRES_USER: ${PG_USER}
POSTGRES_PASSWORD: ${PG_PASSWORD}
POSTGRES_DB: ${PG_MAIN_DB}
volumes:
- ./data/sql/schema.sql:/docker-entrypoint-initdb.d/init.sql
- pg_main_data:/var/lib/postgresql/data
db-dsm:
image: postgres:15.4
restart: always
expose:
- ${PG_PORT}
networks:
- backend
environment:
POSTGRES_DSM_DB: ${PG_DSM_DB}
POSTGRES_WFS_DB: ${PG_WFS_DB}
POSTGRES_USER: ${PG_USER}
POSTGRES_PASSWORD: ${PG_PASSWORD}
POSTGRES_DB: ${PG_DSM_DB}
volumes:
- ./data/sql/dsm.sql:/docker-entrypoint-initdb.d/init.sql
- pg_dsm_data:/var/lib/postgresql/data
- pg_main_data:/var/lib/postgresql/data
- ./data/sql/resources.sql:/docker-entrypoint-initdb.d/init_resources.sql
- ./data/sql/dsm.sql:/docker-entrypoint-initdb.d/init_dsm.sql
- ./data/sql/wfs.sql:/docker-entrypoint-initdb.d/init_wfs.sql
db-mdstores:
image: postgres:15.4
@ -172,28 +161,13 @@ services:
POSTGRES_PASSWORD: ${PG_PASSWORD}
POSTGRES_DB: ${PG_MDSTORE_DB}
volumes:
- ./data/sql/mdstore_schema.sql:/docker-entrypoint-initdb.d/init.sql
- pg_mdstore_data:/var/lib/postgresql/data
- ./data/sql/mdstores:/docker-entrypoint-initdb.d/init.sql
- pg_mdstore_data:/var/lib/postgresql/data
db-wfs:
image: postgres:15.4
restart: always
expose:
- ${PG_PORT}
networks:
- backend
environment:
POSTGRES_USER: ${PG_USER}
POSTGRES_PASSWORD: ${PG_PASSWORD}
POSTGRES_DB: ${PG_WFS_DB}
volumes:
- ./data/sql/wfs.sql:/docker-entrypoint-initdb.d/init.sql
- pg_wfs_data:/var/lib/postgresql/data
solr:
image: solr:9.3.0
expose:
- ${SOLR_PORT}
- ${SOLR_PORT}
networks:
- backend
volumes:
@ -204,7 +178,7 @@ services:
web:
build: ./dnet-app/frontends/is
ports:
ports:
- ${PUBLIC_PORT}:${PUBLIC_PORT}
volumes:
- /app/node_modules
@ -231,10 +205,8 @@ networks:
backend:
frontend:
volumes:
solrdata:
pg_main_data:
pg_dsm_data:
pg_mdstore_data:
pg_wfs_data:

3
docker-start.sh
View File

@ -1,7 +1,5 @@
#!/bin/sh
(cd dnet-app && mvn clean install)
export PUBLIC_PORT=8888
export SPRING_BOOT_PORT=8080
export SOLR_PORT=8983
@ -15,4 +13,3 @@ export PG_MDSTORE_DB=dnet_mdstores
export PG_WFS_DB=dnet_wfs
docker-compose up --force-recreate --build