D-Net
/
dnet-hadoop
14
5
Fork 11
Code Issues 14 Pull Requests 12 Projects 2 Releases 2 Wiki Activity

[stats wf] indicators across stats dbs & updates in the org ids #248

Closed
dimitris.pierrakos wants to merge 1742 commits from beta into beta2master_sept_2022
pull from: beta
merge into: D-Net:beta2master_sept_2022
Conversation 3 Commits 1742 Files Changed 1525 +40 -33
6 changed files with 40 additions and 33 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files
Showing only changes of commit c96a8613f8 - Show all commits

2
dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/sql/queryOrganizations.sql
View File

@ -27,7 +27,7 @@ SELECT
'sysimport:crosswalk:entityregistry@@@dnet:provenance_actions' AS provenanceaction, 'sysimport:crosswalk:entityregistry@@@dnet:provenance_actions' AS provenanceaction,
array_remove(array_agg(DISTINCT i.pid || '###' || i.issuertype || '@@@' || i.issuertype), NULL) AS pid array_remove(array_agg(DISTINCT i.pid || '###' || i.issuertype || '@@@' || i.issuertype), NULL) AS pid
FROM dsm_organizations o FROM dsm_organizations o
LEFT OUTER JOIN dsm_datasources d ON (d.id = o.collectedfrom) LEFT OUTER JOIN dsm_services d ON (d.id = o.collectedfrom)
LEFT OUTER JOIN dsm_organizationpids p ON (p.organization = o.id) LEFT OUTER JOIN dsm_organizationpids p ON (p.organization = o.id)
LEFT OUTER JOIN dsm_identities i ON (i.pid = p.pid) LEFT OUTER JOIN dsm_identities i ON (i.pid = p.pid)
GROUP BY GROUP BY

2
dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/sql/queryProjectOrganization.sql
View File

@ -16,4 +16,4 @@ SELECT
FROM project_organization po FROM project_organization po
LEFT OUTER JOIN projects p ON (p.id = po.project) LEFT OUTER JOIN projects p ON (p.id = po.project)
LEFT OUTER JOIN dsm_datasources dc ON (dc.id = p.collectedfrom); LEFT OUTER JOIN dsm_services dc ON (dc.id = p.collectedfrom);

2
dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/sql/queryProjects.sql
View File

@ -42,7 +42,7 @@ SELECT
LEFT OUTER JOIN projectpids pp ON (pp.project = p.id) LEFT OUTER JOIN projectpids pp ON (pp.project = p.id)
LEFT OUTER JOIN dsm_identities i ON (i.pid = pp.pid) LEFT OUTER JOIN dsm_identities i ON (i.pid = pp.pid)
LEFT OUTER JOIN dsm_datasources dc ON (dc.id = p.collectedfrom) LEFT OUTER JOIN dsm_services dc ON (dc.id = p.collectedfrom)
LEFT OUTER JOIN project_fundingpath pf ON (pf.project = p.id) LEFT OUTER JOIN project_fundingpath pf ON (pf.project = p.id)
LEFT OUTER JOIN fundingpaths fp ON (fp.id = pf.funding) LEFT OUTER JOIN fundingpaths fp ON (fp.id = pf.funding)

2
dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/sql/queryProjects_production.sql
View File

@ -40,7 +40,7 @@ SELECT
LEFT OUTER JOIN projectpids pp ON (pp.project = p.id) LEFT OUTER JOIN projectpids pp ON (pp.project = p.id)
LEFT OUTER JOIN dsm_identities i ON (i.pid = pp.pid) LEFT OUTER JOIN dsm_identities i ON (i.pid = pp.pid)
LEFT OUTER JOIN dsm_datasources dc ON (dc.id = p.collectedfrom) LEFT OUTER JOIN dsm_services dc ON (dc.id = p.collectedfrom)
LEFT OUTER JOIN project_fundingpath pf ON (pf.project = p.id) LEFT OUTER JOIN project_fundingpath pf ON (pf.project = p.id)
LEFT OUTER JOIN fundingpaths fp ON (fp.id = pf.funding) LEFT OUTER JOIN fundingpaths fp ON (fp.id = pf.funding)

8
dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/sql/queryServiceOrganization.sql
View File

@ -1,5 +1,5 @@
SELECT SELECT
dor.datasource AS datasource, dor.service AS service,
dor.organization AS organization, dor.organization AS organization,
NULL AS startdate, NULL AS startdate,
NULL AS enddate, NULL AS enddate,
@ -11,6 +11,6 @@ SELECT
dc.officialname AS collectedfromname, dc.officialname AS collectedfromname,
'providedBy@@@dnet:datasources_organizations_typologies' AS semantics, 'providedBy@@@dnet:datasources_organizations_typologies' AS semantics,
d.provenanceaction || '@@@dnet:provenanceActions' AS provenanceaction d.provenanceaction || '@@@dnet:provenanceActions' AS provenanceaction
FROM dsm_datasource_organization dor FROM dsm_service_organization dor
LEFT OUTER JOIN dsm_datasources d ON (dor.datasource = d.id) LEFT OUTER JOIN dsm_services d ON (dor.service = d.id)
LEFT OUTER JOIN dsm_datasources dc ON (dc.id = d.collectedfrom); LEFT OUTER JOIN dsm_services dc ON (dc.id = d.collectedfrom);

57
dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/sql/queryServices.sql
View File

@ -57,8 +57,12 @@ SELECT
NULL AS odpolicies, NULL AS odpolicies,
ARRAY(SELECT trim(s) ARRAY(SELECT trim(s)
FROM unnest(string_to_array(d.languages, ',')) AS s) AS odlanguages, FROM unnest(string_to_array(d.languages, ',')) AS s) AS odlanguages,
ARRAY(SELECT trim(s)
FROM unnest(string_to_array(d.od_contenttypes, '-')) AS s) AS odcontenttypes, -- Term provided only by OpenDOAR:
-- probably updating the TR it could be replaced by research_entity_types[]
-- But a study on the vocabulary terms is needed
-- REMOVED: ARRAY(SELECT trim(s) FROM unnest(string_to_array(d.od_contenttypes, '-')) AS s) AS odcontenttypes,
false AS inferred, false AS inferred,
false AS deletedbyinference, false AS deletedbyinference,
0.9 AS trust, 0.9 AS trust,
@ -69,22 +73,25 @@ SELECT
d.releasestartdate AS releasestartdate, d.releasestartdate AS releasestartdate,
d.releaseenddate AS releaseenddate, d.releaseenddate AS releaseenddate,
d.missionstatementurl AS missionstatementurl, d.missionstatementurl AS missionstatementurl,
d.dataprovider AS dataprovider, -- the following 2 fields (provided by re3data) have been replaced by research_entity_types[]
d.serviceprovider AS serviceprovider, -- VALUE 'Research Data' : d.dataprovider AS dataprovider,
-- VALUE 'Services' : d.serviceprovider AS serviceprovider,
d.databaseaccesstype AS databaseaccesstype, d.databaseaccesstype AS databaseaccesstype,
d.datauploadtype AS datauploadtype, d.datauploadtype AS datauploadtype,
d.databaseaccessrestriction AS databaseaccessrestriction, d.databaseaccessrestriction AS databaseaccessrestriction,
d.datauploadrestriction AS datauploadrestriction, d.datauploadrestriction AS datauploadrestriction,
d.versioning AS versioning, -- REPLACED BY version_control : d.versioning AS versioning,
d.version_control AS versioning,
d.citationguidelineurl AS citationguidelineurl, d.citationguidelineurl AS citationguidelineurl,
d.qualitymanagementkind AS qualitymanagementkind, -- REMOVED (it was provided only by re3data: yes, no, unknown): d.qualitymanagementkind AS qualitymanagementkind,
d.pidsystems AS pidsystems, d.pidsystems AS pidsystems,
d.certificates AS certificates, d.certificates AS certificates,
ARRAY[]::text[] AS policies, ARRAY[]::text[] AS policies,
dc.id AS collectedfromid, dc.id AS collectedfromid,
dc.officialname AS collectedfromname, dc.officialname AS collectedfromname,
d.typology||'@@@dnet:datasource_typologies' AS datasourcetype, d._typology_to_remove_||'@@@dnet:datasource_typologies' AS datasourcetype,
d.typology||'@@@dnet:datasource_typologies_ui' AS datasourcetypeui, d.eosc_type||'@@@dnet:eosc_types' AS eosc_type,
d.eosc_datasource_type||'@@@dnet:eosc_datasource_types' AS eosc_datasoorce_type,
'sysimport:crosswalk:entityregistry@@@dnet:provenance_actions' AS provenanceaction, 'sysimport:crosswalk:entityregistry@@@dnet:provenance_actions' AS provenanceaction,
d.issn AS issnPrinted, d.issn AS issnPrinted,
d.eissn AS issnOnline, d.eissn AS issnOnline,
@ -92,16 +99,15 @@ SELECT
d.consenttermsofuse AS consenttermsofuse, d.consenttermsofuse AS consenttermsofuse,
d.fulltextdownload AS fulltextdownload, d.fulltextdownload AS fulltextdownload,
d.consenttermsofusedate AS consenttermsofusedate, d.consenttermsofusedate AS consenttermsofusedate,
de.jurisdiction||'@@@eosc:jurisdictions' AS jurisdiction, d.jurisdiction||'@@@eosc:jurisdictions' AS jurisdiction,
de.thematic AS thematic, d.thematic AS thematic,
de.knowledge_graph AS knowledgegraph, -- REMOVED ???: d.knowledge_graph AS knowledgegraph,
array(select unnest(de.content_policies)||'@@@eosc:contentpolicies') AS contentpolicies array(select unnest(d.content_policies)||'@@@eosc:contentpolicies') AS contentpolicies
FROM dsm_datasources d FROM dsm_services d
LEFT OUTER JOIN dsm_datasources_eosc de on (d.id = de.id) LEFT OUTER JOIN dsm_services dc on (d.collectedfrom = dc.id)
LEFT OUTER JOIN dsm_datasources dc on (d.collectedfrom = dc.id) LEFT OUTER JOIN dsm_api a ON (d.id = a.service)
LEFT OUTER JOIN dsm_api a ON (d.id = a.datasource) LEFT OUTER JOIN dsm_servicepids di ON (d.id = di.service)
LEFT OUTER JOIN dsm_datasourcepids di ON (d.id = di.datasource)
GROUP BY GROUP BY
d.id, d.id,
@ -119,15 +125,16 @@ GROUP BY
d.releasestartdate, d.releasestartdate,
d.releaseenddate, d.releaseenddate,
d.missionstatementurl, d.missionstatementurl,
d.dataprovider, -- TODO REMOVED ???: d.dataprovider,
d.serviceprovider, -- TODO REMOVED ???: d.serviceprovider,
d.databaseaccesstype, d.databaseaccesstype,
d.datauploadtype, d.datauploadtype,
d.databaseaccessrestriction, d.databaseaccessrestriction,
d.datauploadrestriction, d.datauploadrestriction,
d.versioning, -- REPLACED BY version_control : d.versioning,
d.version_control
d.citationguidelineurl, d.citationguidelineurl,
d.qualitymanagementkind, -- REMOVED: d.qualitymanagementkind,
d.pidsystems, d.pidsystems,
d.certificates, d.certificates,
dc.id, dc.id,
@ -135,7 +142,7 @@ GROUP BY
d.issn, d.issn,
d.eissn, d.eissn,
d.lissn, d.lissn,
de.jurisdiction, d.jurisdiction,
de.thematic, d.thematic,
de.knowledge_graph, -- REMOVED ???: de.knowledge_graph,
de.content_policies d.content_policies