D-Net
/
dnet-hadoop
13
4
Fork 10
Code Issues 14 Pull Requests 8 Projects 2 Releases 1 Wiki Activity

EOSC Services - ongoing update

This commit is contained in:
Claudio Atzori 2022-04-26 16:18:41 +02:00
parent c96a8613f8
commit 5ffc24d1ba
3 changed files with 40 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/MigrateDbEntitiesApplication.java
View File

@ -143,8 +143,8 @@ public class MigrateDbEntitiesApplication extends AbstractMigrationApplication i
smdbe.execute("queryClaims.sql", smdbe::processClaims); smdbe.execute("queryClaims.sql", smdbe::processClaims);
break; break;
case openaire: case openaire:
log.info("Processing datasources..."); log.info("Processing services...");
smdbe.execute("queryDatasources.sql", smdbe::processDatasource, verifyNamespacePrefix); smdbe.execute("queryServices.sql", smdbe::processService, verifyNamespacePrefix);
log.info("Processing projects..."); log.info("Processing projects...");
if (dbSchema.equalsIgnoreCase("beta")) { if (dbSchema.equalsIgnoreCase("beta")) {
@ -235,7 +235,7 @@ public class MigrateDbEntitiesApplication extends AbstractMigrationApplication i
dbClient.processResults(sql, consumer); dbClient.processResults(sql, consumer);
} }
public List<Oaf> processDatasource(final ResultSet rs) { public List<Oaf> processService(final ResultSet rs) {
try { try {
final DataInfo info = prepareDataInfo(rs); final DataInfo info = prepareDataInfo(rs);

51
dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/sql/queryServices.sql
View File

@ -1,6 +1,6 @@
SELECT SELECT
d.id AS datasourceid, d.id AS datasourceid,
d.id || array_agg(distinct di.pid) AS identities, array_remove(d.id || array_agg(distinct di.pid) filter (where di.pid like 'piwik%') || array_agg(distinct dds.duplicate), NULL) AS identities,
d.officialname AS officialname, d.officialname AS officialname,
d.englishname AS englishname, d.englishname AS englishname,
d.contactemail AS contactemail, d.contactemail AS contactemail,
@ -57,7 +57,8 @@ SELECT
NULL AS odpolicies, NULL AS odpolicies,
ARRAY(SELECT trim(s) ARRAY(SELECT trim(s)
FROM unnest(string_to_array(d.languages, ',')) AS s) AS odlanguages, FROM unnest(string_to_array(d.languages, ',')) AS s) AS odlanguages,
ARRAY(SELECT trim(s)
FROM unnest(string_to_array(d.languages, ',')) AS s) AS languages,
-- Term provided only by OpenDOAR: -- Term provided only by OpenDOAR:
-- probably updating the TR it could be replaced by research_entity_types[] -- probably updating the TR it could be replaced by research_entity_types[]
-- But a study on the vocabulary terms is needed -- But a study on the vocabulary terms is needed
@ -74,40 +75,54 @@ SELECT
d.releaseenddate AS releaseenddate, d.releaseenddate AS releaseenddate,
d.missionstatementurl AS missionstatementurl, d.missionstatementurl AS missionstatementurl,
-- the following 2 fields (provided by re3data) have been replaced by research_entity_types[] -- the following 2 fields (provided by re3data) have been replaced by research_entity_types[]
-- VALUE 'Research Data' : d.dataprovider AS dataprovider, -- VALUE 'Research Data' : d.dataprovider AS dataprovider,
-- VALUE 'Services' : d.serviceprovider AS serviceprovider, -- VALUE 'Services' : d.serviceprovider AS serviceprovider,
d.databaseaccesstype AS databaseaccesstype, d.databaseaccesstype AS databaseaccesstype,
d.datauploadtype AS datauploadtype, d.datauploadtype AS datauploadtype,
d.databaseaccessrestriction AS databaseaccessrestriction, d.databaseaccessrestriction AS databaseaccessrestriction,
d.datauploadrestriction AS datauploadrestriction, d.datauploadrestriction AS datauploadrestriction,
-- REPLACED BY version_control : d.versioning AS versioning, -- REPLACED BY version_control : d.versioning AS versioning,
d.version_control AS versioning, d.version_control AS versioning,
d.version_control AS versioncontrol,
d.citationguidelineurl AS citationguidelineurl, d.citationguidelineurl AS citationguidelineurl,
-- REMOVED (it was provided only by re3data: yes, no, unknown): d.qualitymanagementkind AS qualitymanagementkind,
d.pidsystems AS pidsystems, d.pidsystems AS pidsystems,
d.certificates AS certificates, d.certificates AS certificates,
ARRAY[]::text[] AS policies, ARRAY[]::text[] AS policies,
dc.id AS collectedfromid, array_remove(dc.id||'@@@'||dc.officialname || array_agg(distinct dds_cf.id||'@@@'||dds_cf.officialname), NULL) AS collectedfrom,
dc.officialname AS collectedfromname,
d._typology_to_remove_||'@@@dnet:datasource_typologies' AS datasourcetype, d._typology_to_remove_||'@@@dnet:datasource_typologies' AS datasourcetype,
d.eosc_type||'@@@dnet:eosc_types' AS eosc_type, d.eosc_type||'@@@dnet:eosc_types' AS eosctype,
d.eosc_datasource_type||'@@@dnet:eosc_datasource_types' AS eosc_datasoorce_type, d.eosc_datasource_type||'@@@dnet:eosc_datasource_types' AS eoscdatasourcetype,
'sysimport:crosswalk:entityregistry@@@dnet:provenance_actions' AS provenanceaction, 'sysimport:crosswalk:entityregistry@@@dnet:provenance_actions' AS provenanceaction,
d.issn AS issnPrinted, d.issn AS issnPrinted,
d.eissn AS issnOnline, d.eissn AS issnOnline,
d.lissn AS issnLinking, d.lissn AS issnLinking,
d.research_entity_types AS researchentitytypes,
d.consenttermsofuse AS consenttermsofuse, d.consenttermsofuse AS consenttermsofuse,
d.fulltextdownload AS fulltextdownload, d.fulltextdownload AS fulltextdownload,
d.consenttermsofusedate AS consenttermsofusedate, d.consenttermsofusedate AS consenttermsofusedate,
d.jurisdiction||'@@@eosc:jurisdictions' AS jurisdiction, d.lastconsenttermsofusedate AS lastconsenttermsofusedate,
d.thematic AS thematic, d.jurisdiction||'@@@eosc:jurisdictions' AS jurisdiction,
-- REMOVED ???: d.knowledge_graph AS knowledgegraph, d.thematic AS thematic,
array(select unnest(d.content_policies)||'@@@eosc:contentpolicies') AS contentpolicies -- REMOVED ???: d.knowledge_graph AS knowledgegraph,
array(select unnest(d.content_policies)||'@@@eosc:contentpolicies') AS contentpolicies,
d.submission_policy_url AS submissionpolicyurl,
d.preservation_policy_url AS preservationpolicyurl,
d.research_product_access_policies AS researchproductaccesspolicies,
d.research_product_metadata_access_policies AS researchproductmetadataaccesspolicies
FROM dsm_services d FROM dsm_services d
LEFT OUTER JOIN dsm_services dc on (d.collectedfrom = dc.id) LEFT OUTER JOIN dsm_services dc on (d.collectedfrom = dc.id)
LEFT OUTER JOIN dsm_api a ON (d.id = a.service) LEFT OUTER JOIN dsm_api a ON (d.id = a.service)
LEFT OUTER JOIN dsm_servicepids di ON (d.id = di.service) LEFT OUTER JOIN dsm_servicepids di ON (d.id = di.service)
LEFT OUTER JOIN dsm_dedup_services dds ON (d.id = dds.id)
LEFT OUTER JOIN dsm_services dds_dup ON (dds.duplicate = dds_dup.id)
LEFT OUTER JOIN dsm_services dds_cf ON (dds_dup.collectedfrom = dds_cf.id)
WHERE
d.dedup_main_service = true
GROUP BY GROUP BY
d.id, d.id,
@ -132,7 +147,7 @@ GROUP BY
d.databaseaccessrestriction, d.databaseaccessrestriction,
d.datauploadrestriction, d.datauploadrestriction,
-- REPLACED BY version_control : d.versioning, -- REPLACED BY version_control : d.versioning,
d.version_control d.version_control,
d.citationguidelineurl, d.citationguidelineurl,
-- REMOVED: d.qualitymanagementkind, -- REMOVED: d.qualitymanagementkind,
d.pidsystems, d.pidsystems,
@ -145,4 +160,8 @@ GROUP BY
d.jurisdiction, d.jurisdiction,
d.thematic, d.thematic,
-- REMOVED ???: de.knowledge_graph, -- REMOVED ???: de.knowledge_graph,
d.content_policies d.content_policies,
d.submission_policy_url,
d.preservation_policy_url,
d.research_product_access_policies,
d.research_product_metadata_access_policies

4
dhp-workflows/dhp-graph-mapper/src/test/java/eu/dnetlib/dhp/oa/graph/raw/MigrateDbEntitiesApplicationTest.java
View File

@ -63,10 +63,10 @@ public class MigrateDbEntitiesApplicationTest {
} }
@Test @Test
public void testProcessDatasource() throws Exception { public void testProcessService() throws Exception {
final List<TypedField> fields = prepareMocks("datasources_resultset_entry.json"); final List<TypedField> fields = prepareMocks("datasources_resultset_entry.json");
final List<Oaf> list = app.processDatasource(rs); final List<Oaf> list = app.processService(rs);
assertEquals(1, list.size()); assertEquals(1, list.size());
verifyMocks(fields); verifyMocks(fields);