diff --git a/dnet-ariadneplus/pom.xml b/dnet-ariadneplus/pom.xml index 3ba99bd..3abf3a6 100644 --- a/dnet-ariadneplus/pom.xml +++ b/dnet-ariadneplus/pom.xml @@ -10,9 +10,9 @@ eu.dnetlib dnet-ariadneplus jar - 1.0.3-SNAPSHOT + 1.1.0-SNAPSHOT - scm:svn:https://svn.driver.research-infrastructures.eu/driver/dnet50/modules/dnet-ariadneplus/trunk + scm:git:https://code-repo.d4science.org/D-Net/AriadnePlus.git diff --git a/dnet-ariadneplus/src/main/resources/eu/dnetlib/ariadneplus/workflows/repo-hi/NEW_publish_wf.xml.st b/dnet-ariadneplus/src/main/resources/eu/dnetlib/ariadneplus/workflows/repo-hi/NEW_publish_wf.xml.st deleted file mode 100644 index 2747394..0000000 --- a/dnet-ariadneplus/src/main/resources/eu/dnetlib/ariadneplus/workflows/repo-hi/NEW_publish_wf.xml.st +++ /dev/null @@ -1,79 +0,0 @@ - - -
- - - - - -
- - $name$ - $desc$ - - publishing - $priority$ - - - - - - - public - REFRESH - http://localhost:8080/ariadneplus/publish - - - - Store RDF files in Virtuoso - - - - - - - - - - - - - - - - - - - - - - - - - - 9 9 9 ? * * - 10080 - - - - diff --git a/dnet-ariadneplus/src/main/resources/eu/dnetlib/ariadneplus/workflows/repo-hi/full_aggregation_wf.xml.st b/dnet-ariadneplus/src/main/resources/eu/dnetlib/ariadneplus/workflows/repo-hi/full_aggregation_wf.xml.st new file mode 100644 index 0000000..752194a --- /dev/null +++ b/dnet-ariadneplus/src/main/resources/eu/dnetlib/ariadneplus/workflows/repo-hi/full_aggregation_wf.xml.st @@ -0,0 +1,205 @@ + + +
+ + + + + +
+ + $name$ + $desc$ + + aggregator + $priority$ + + + + + INCREMENTAL + + + false + REFRESH + + + + + + + + + transformed + REFRESH + + https://aggregator.ariadne.d4science.org/ariadneplus-graphdb + https://aggregator.ariadne.d4science.org/ariadneplus-graphdb-public + + + + + Collect metadata + + + + + + + + + + + + + + + + + + + Transform records + + + + + + + + + + + + + + + + + + + + + + + + + + Index record + + + + + + + + + + + + + + + + + + + Publish records to the Staging GraphDB + + + + + + + + + + + + + + + + + + Enrich GraphDB with sparql update query + + + + + + + + + + + + + + + + + + Index on Staging Elasticsearch all records of an API from GraphDB + + + + + + + + + + + + + + + + + Publish records to the public GraphDB and Elasticsearch + + + + + + + + + + + + + + + + + + + A Fake node + + + + + + + + + + + + + + + + + + + + + + 9 9 9 ? * * + 10080 + + + + diff --git a/dnet-ariadneplus/src/main/resources/eu/dnetlib/ariadneplus/workflows/repo-hi/enrich_graphdb_wf.xml.st b/dnet-ariadneplus/src/main/resources/eu/dnetlib/ariadneplus/workflows/repo-hi/old/enrich_graphdb_wf.xml.st similarity index 100% rename from dnet-ariadneplus/src/main/resources/eu/dnetlib/ariadneplus/workflows/repo-hi/enrich_graphdb_wf.xml.st rename to dnet-ariadneplus/src/main/resources/eu/dnetlib/ariadneplus/workflows/repo-hi/old/enrich_graphdb_wf.xml.st diff --git a/dnet-ariadneplus/src/main/resources/eu/dnetlib/ariadneplus/workflows/repo-hi/index_on_es_wf.xml.st b/dnet-ariadneplus/src/main/resources/eu/dnetlib/ariadneplus/workflows/repo-hi/old/index_on_es_wf.xml.st similarity index 100% rename from dnet-ariadneplus/src/main/resources/eu/dnetlib/ariadneplus/workflows/repo-hi/index_on_es_wf.xml.st rename to dnet-ariadneplus/src/main/resources/eu/dnetlib/ariadneplus/workflows/repo-hi/old/index_on_es_wf.xml.st diff --git a/dnet-ariadneplus/src/main/resources/eu/dnetlib/ariadneplus/workflows/repo-hi/x3m_aggregation_wf.xml.st b/dnet-ariadneplus/src/main/resources/eu/dnetlib/ariadneplus/workflows/repo-hi/old/x3m_aggregation_wf.xml.st similarity index 100% rename from dnet-ariadneplus/src/main/resources/eu/dnetlib/ariadneplus/workflows/repo-hi/x3m_aggregation_wf.xml.st rename to dnet-ariadneplus/src/main/resources/eu/dnetlib/ariadneplus/workflows/repo-hi/old/x3m_aggregation_wf.xml.st diff --git a/dnet-ariadneplus/src/main/resources/eu/dnetlib/bootstrap/profiles/workflows/repo_hi_enrich_graphdb.xml b/dnet-ariadneplus/src/main/resources/eu/dnetlib/bootstrap/profiles/workflows/old/repo_hi_enrich_graphdb.xml similarity index 100% rename from dnet-ariadneplus/src/main/resources/eu/dnetlib/bootstrap/profiles/workflows/repo_hi_enrich_graphdb.xml rename to dnet-ariadneplus/src/main/resources/eu/dnetlib/bootstrap/profiles/workflows/old/repo_hi_enrich_graphdb.xml diff --git a/dnet-ariadneplus/src/main/resources/eu/dnetlib/bootstrap/profiles/workflows/repo_hi_index_on_es.xml b/dnet-ariadneplus/src/main/resources/eu/dnetlib/bootstrap/profiles/workflows/old/repo_hi_index_on_es.xml similarity index 100% rename from dnet-ariadneplus/src/main/resources/eu/dnetlib/bootstrap/profiles/workflows/repo_hi_index_on_es.xml rename to dnet-ariadneplus/src/main/resources/eu/dnetlib/bootstrap/profiles/workflows/old/repo_hi_index_on_es.xml diff --git a/dnet-ariadneplus/src/main/resources/eu/dnetlib/bootstrap/profiles/workflows/publish/virtuoso_publish_template.xml b/dnet-ariadneplus/src/main/resources/eu/dnetlib/bootstrap/profiles/workflows/publish/virtuoso_publish_template.xml deleted file mode 100644 index ab67154..0000000 --- a/dnet-ariadneplus/src/main/resources/eu/dnetlib/bootstrap/profiles/workflows/publish/virtuoso_publish_template.xml +++ /dev/null @@ -1,41 +0,0 @@ - - -
- - - - - -
- - - - - - - - - Fetch records from MDStore - - - - - - - - - - - Feed records into Virtuoso - - - - - - - - - - - - diff --git a/dnet-ariadneplus/src/main/resources/eu/dnetlib/bootstrap/profiles/workflows/repo_hi_import_periodo_into_graphdb.xml b/dnet-ariadneplus/src/main/resources/eu/dnetlib/bootstrap/profiles/workflows/repo-hi/repo_hi_import_periodo_into_graphdb.xml similarity index 100% rename from dnet-ariadneplus/src/main/resources/eu/dnetlib/bootstrap/profiles/workflows/repo_hi_import_periodo_into_graphdb.xml rename to dnet-ariadneplus/src/main/resources/eu/dnetlib/bootstrap/profiles/workflows/repo-hi/repo_hi_import_periodo_into_graphdb.xml diff --git a/dnet-ariadneplus/src/main/resources/eu/dnetlib/bootstrap/profiles/workflows/NEW_repo_hi_x3m.xml b/dnet-ariadneplus/src/main/resources/eu/dnetlib/bootstrap/profiles/workflows/repo-hi/repo_hi_main.xml similarity index 90% rename from dnet-ariadneplus/src/main/resources/eu/dnetlib/bootstrap/profiles/workflows/NEW_repo_hi_x3m.xml rename to dnet-ariadneplus/src/main/resources/eu/dnetlib/bootstrap/profiles/workflows/repo-hi/repo_hi_main.xml index 5bc901b..c903e38 100644 --- a/dnet-ariadneplus/src/main/resources/eu/dnetlib/bootstrap/profiles/workflows/NEW_repo_hi_x3m.xml +++ b/dnet-ariadneplus/src/main/resources/eu/dnetlib/bootstrap/profiles/workflows/repo-hi/repo_hi_main.xml @@ -8,8 +8,8 @@ - Aggregate Metadata (X3M) - Collect, transform with X3M, and export metadata + Aggregate and Publish Metadata + Aggregate, enrich and publish metadata Aggregate Metadata Content Provider @@ -40,9 +40,9 @@ Create Workflow - - - + + + diff --git a/dnet-ariadneplus/src/main/resources/eu/dnetlib/bootstrap/profiles/workflows/repo_bye_template.xml b/dnet-ariadneplus/src/main/resources/eu/dnetlib/bootstrap/profiles/workflows/repo_bye_template.xml index 391160f..4a3d493 100644 --- a/dnet-ariadneplus/src/main/resources/eu/dnetlib/bootstrap/profiles/workflows/repo_bye_template.xml +++ b/dnet-ariadneplus/src/main/resources/eu/dnetlib/bootstrap/profiles/workflows/repo_bye_template.xml @@ -18,6 +18,7 @@ + Delete the mdstore of collected records diff --git a/dnet-ariadneplus/src/main/resources/eu/dnetlib/bootstrap/profiles/workflows/collection_template.xml b/dnet-ariadneplus/src/main/resources/eu/dnetlib/bootstrap/profiles/workflows/templates/collection_template.xml similarity index 100% rename from dnet-ariadneplus/src/main/resources/eu/dnetlib/bootstrap/profiles/workflows/collection_template.xml rename to dnet-ariadneplus/src/main/resources/eu/dnetlib/bootstrap/profiles/workflows/templates/collection_template.xml diff --git a/dnet-ariadneplus/src/main/resources/eu/dnetlib/bootstrap/profiles/workflows/enrich_graphdb_template.xml b/dnet-ariadneplus/src/main/resources/eu/dnetlib/bootstrap/profiles/workflows/templates/enrich_graphdb_template.xml similarity index 100% rename from dnet-ariadneplus/src/main/resources/eu/dnetlib/bootstrap/profiles/workflows/enrich_graphdb_template.xml rename to dnet-ariadneplus/src/main/resources/eu/dnetlib/bootstrap/profiles/workflows/templates/enrich_graphdb_template.xml diff --git a/dnet-ariadneplus/src/main/resources/eu/dnetlib/bootstrap/profiles/workflows/graphdb_template.xml b/dnet-ariadneplus/src/main/resources/eu/dnetlib/bootstrap/profiles/workflows/templates/graphdb_template.xml similarity index 91% rename from dnet-ariadneplus/src/main/resources/eu/dnetlib/bootstrap/profiles/workflows/graphdb_template.xml rename to dnet-ariadneplus/src/main/resources/eu/dnetlib/bootstrap/profiles/workflows/templates/graphdb_template.xml index 1f1e054..dc8fe0d 100644 --- a/dnet-ariadneplus/src/main/resources/eu/dnetlib/bootstrap/profiles/workflows/graphdb_template.xml +++ b/dnet-ariadneplus/src/main/resources/eu/dnetlib/bootstrap/profiles/workflows/templates/graphdb_template.xml @@ -13,7 +13,7 @@ - http://localhost:8080/ariadneplus/publish + diff --git a/dnet-ariadneplus/src/main/resources/eu/dnetlib/bootstrap/profiles/workflows/import_periodo_into_graphdb_template.xml b/dnet-ariadneplus/src/main/resources/eu/dnetlib/bootstrap/profiles/workflows/templates/import_periodo_into_graphdb_template.xml similarity index 100% rename from dnet-ariadneplus/src/main/resources/eu/dnetlib/bootstrap/profiles/workflows/import_periodo_into_graphdb_template.xml rename to dnet-ariadneplus/src/main/resources/eu/dnetlib/bootstrap/profiles/workflows/templates/import_periodo_into_graphdb_template.xml diff --git a/dnet-ariadneplus/src/main/resources/eu/dnetlib/bootstrap/profiles/workflows/index_on_es_template.xml b/dnet-ariadneplus/src/main/resources/eu/dnetlib/bootstrap/profiles/workflows/templates/index_on_es_template.xml similarity index 92% rename from dnet-ariadneplus/src/main/resources/eu/dnetlib/bootstrap/profiles/workflows/index_on_es_template.xml rename to dnet-ariadneplus/src/main/resources/eu/dnetlib/bootstrap/profiles/workflows/templates/index_on_es_template.xml index ef24ed1..e8af7a4 100644 --- a/dnet-ariadneplus/src/main/resources/eu/dnetlib/bootstrap/profiles/workflows/index_on_es_template.xml +++ b/dnet-ariadneplus/src/main/resources/eu/dnetlib/bootstrap/profiles/workflows/templates/index_on_es_template.xml @@ -12,7 +12,7 @@ - http://localhost:8281/ariadneplus-graphdb + diff --git a/dnet-ariadneplus/src/main/resources/eu/dnetlib/bootstrap/profiles/workflows/index_template.xml b/dnet-ariadneplus/src/main/resources/eu/dnetlib/bootstrap/profiles/workflows/templates/index_template.xml similarity index 100% rename from dnet-ariadneplus/src/main/resources/eu/dnetlib/bootstrap/profiles/workflows/index_template.xml rename to dnet-ariadneplus/src/main/resources/eu/dnetlib/bootstrap/profiles/workflows/templates/index_template.xml diff --git a/dnet-ariadneplus/src/main/resources/eu/dnetlib/bootstrap/profiles/workflows/templates/public_publishing_template.xml b/dnet-ariadneplus/src/main/resources/eu/dnetlib/bootstrap/profiles/workflows/templates/public_publishing_template.xml new file mode 100644 index 0000000..0a8732f --- /dev/null +++ b/dnet-ariadneplus/src/main/resources/eu/dnetlib/bootstrap/profiles/workflows/templates/public_publishing_template.xml @@ -0,0 +1,76 @@ + +
+ + + + + +
+ + + + + + + + + + + + + Publish records to the public GraphDB + + + + + + + + + + + + + + + + + + Enrich public GraphDB with sparql update query + + + + + + + + + + + + + + + + + + Index on public Elasticsearch all records of an API from GraphDB + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/dnet-ariadneplus/src/main/resources/eu/dnetlib/bootstrap/profiles/workflows/transform_x3m_template.xml b/dnet-ariadneplus/src/main/resources/eu/dnetlib/bootstrap/profiles/workflows/templates/transform_x3m_template.xml similarity index 100% rename from dnet-ariadneplus/src/main/resources/eu/dnetlib/bootstrap/profiles/workflows/transform_x3m_template.xml rename to dnet-ariadneplus/src/main/resources/eu/dnetlib/bootstrap/profiles/workflows/templates/transform_x3m_template.xml diff --git a/dnet-ariadneplus/src/test/resources/eu/dnetlib/wf_migration/new_agg_wf.xml b/dnet-ariadneplus/src/test/resources/eu/dnetlib/wf_migration/new_agg_wf.xml new file mode 100644 index 0000000..e899548 --- /dev/null +++ b/dnet-ariadneplus/src/test/resources/eu/dnetlib/wf_migration/new_agg_wf.xml @@ -0,0 +1,195 @@ + +
+ + + + + +
+ + + Aggregate, enrich and index AriadnePlus content + + Aggregate, enrich and index AriadnePlus content on shadow and public instance + + aggregator + 50 + + + + INCREMENTAL + 36aadfcc-648b-43f7-96f3-b4ae612b3fa4_TURTdG9yZURTUmVzb3VyY2VzL01EU3RvcmVEU1Jlc291cmNlVHlwZQ== + false + REFRESH + + + + + 971d3cf8-b720-4f29-a16d-f4453c962343_TURTdG9yZURTUmVzb3VyY2VzL01EU3RvcmVEU1Jlc291cmNlVHlwZQ== + null + transformed + REFRESH + + https://aggregator.ariadne.d4science.org/ariadneplus-graphdb + https://aggregator.ariadne.d4science.org/ariadneplus-graphdb-public + + + + + Collect metadata + + + + + + + + + + + + + + + + + + + Transform records + + + + + + + + + + + + + + + + + + + + + + + + + + Index record + + + + + + + + + + + + + + + + + + + + + Publish records to the Staging GraphDB + + + + + + + + + + + + + + + + + + Enrich GraphDB with sparql update query + + + + + + + + + + + + + + + + + + Index on Staging Elasticsearch all records of an API from GraphDB + + + + + + + + + + + + + + + + + Publish records to the public GraphDB and Elasticsearch + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 9 9 9 ? * * + 10080 + + + + \ No newline at end of file diff --git a/dnet-ariadneplus/src/test/resources/eu/dnetlib/wf_migration/old_agg_wf.xml b/dnet-ariadneplus/src/test/resources/eu/dnetlib/wf_migration/old_agg_wf.xml new file mode 100644 index 0000000..a41b9c9 --- /dev/null +++ b/dnet-ariadneplus/src/test/resources/eu/dnetlib/wf_migration/old_agg_wf.xml @@ -0,0 +1,139 @@ + +
+ + + + + +
+ + Aggregate Metadata (X3M) + Collect, transform with X3M, and export metadata + + aggregator + 50 + + + + REFRESH + 6f7c21a0-a7be-4724-be89-833b33ad5e0a_TURTdG9yZURTUmVzb3VyY2VzL01EU3RvcmVEU1Jlc291cmNlVHlwZQ== + false + REFRESH + false + e1a85324-8f54-45c1-8513-208693a41666_VHJhbnNmb3JtYXRpb25SdWxlRFNSZXNvdXJjZXMvVHJhbnNmb3JtYXRpb25SdWxlRFNSZXNvdXJjZVR5cGU= + e508ddc7-ff5e-45dc-9116-96a2201d0bc9_Q2xlYW5lckRTUmVzb3VyY2VzL0NsZWFuZXJEU1Jlc291cmNlVHlwZQ== + af07e923-424b-4d5e-9677-66582db7efdd_TURTdG9yZURTUmVzb3VyY2VzL01EU3RvcmVEU1Jlc291cmNlVHlwZQ== + 84c5cd33-b152-4fae-8796-fea0685a6c83_SW5kZXhEU1Jlc291cmNlcy9JbmRleERTUmVzb3VyY2VUeXBl + transformed + REFRESH + https://mapping.d4science.org/3MEditor/Services?id=591&output=text/xml&method=export + https://aggregator.ariadne.d4science.org/ariadneplus-graphdb + + + + Collect metadata + + + + + + + + + + + + + + + + + + + Transform records + + + + + + + + + + + + + + + + + + + + + + + + + + Index record + + + + + + + + + + + + + + + + + + + Publish records to GraphDB + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 9 9 9 ? * * + 10080 + + + wf_20200605_160619_405 + 2020-06-05T16:07:19+02:00 + SUCCESS + + + + \ No newline at end of file