diff --git a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/migration/step1/MigrateDbEntitiesApplication.java b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/migration/step1/MigrateDbEntitiesApplication.java index 1af1908f5..7db2b1772 100644 --- a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/migration/step1/MigrateDbEntitiesApplication.java +++ b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/migration/step1/MigrateDbEntitiesApplication.java @@ -20,6 +20,7 @@ import java.util.Arrays; import java.util.Date; import java.util.List; import java.util.function.Consumer; +import java.util.function.Function; import org.apache.commons.io.IOUtils; import org.apache.commons.lang3.StringUtils; @@ -36,6 +37,7 @@ import eu.dnetlib.dhp.schema.oaf.Datasource; import eu.dnetlib.dhp.schema.oaf.Field; import eu.dnetlib.dhp.schema.oaf.Journal; import eu.dnetlib.dhp.schema.oaf.KeyValue; +import eu.dnetlib.dhp.schema.oaf.Oaf; import eu.dnetlib.dhp.schema.oaf.Organization; import eu.dnetlib.dhp.schema.oaf.OtherResearchProduct; import eu.dnetlib.dhp.schema.oaf.Project; @@ -95,6 +97,12 @@ public class MigrateDbEntitiesApplication extends AbstractMigrationApplication i } } + protected MigrateDbEntitiesApplication() { // ONLY FOR UNIT TEST + super(); + this.dbClient = null; + this.lastUpdateTimestamp = new Date().getTime(); + } + public MigrateDbEntitiesApplication(final String hdfsPath, final String dbUrl, final String dbUser, final String dbPassword) throws Exception { super(hdfsPath); @@ -102,12 +110,15 @@ public class MigrateDbEntitiesApplication extends AbstractMigrationApplication i this.lastUpdateTimestamp = new Date().getTime(); } - public void execute(final String sqlFile, final Consumer consumer) throws Exception { + public void execute(final String sqlFile, final Function> producer) throws Exception { final String sql = IOUtils.toString(getClass().getResourceAsStream("/eu/dnetlib/dhp/migration/sql/" + sqlFile)); + + final Consumer consumer = rs -> producer.apply(rs).forEach(oaf -> emitOaf(oaf)); + dbClient.processResults(sql, consumer); } - public void processDatasource(final ResultSet rs) { + public List processDatasource(final ResultSet rs) { try { @@ -161,61 +172,13 @@ public class MigrateDbEntitiesApplication extends AbstractMigrationApplication i ds.setDataInfo(info); ds.setLastupdatetimestamp(lastUpdateTimestamp); - // rs.getString("datasourceid"); - // rs.getArray("identities"); - // rs.getString("officialname"); - // rs.getString("englishname"); - // rs.getString("contactemail"); - // rs.getString("openairecompatibility"); // COMPLEX ...@@@... - // rs.getString("websiteurl"); - // rs.getString("logourl"); - // rs.getArray("accessinfopackage"); - // rs.getDouble("latitude"); - // rs.getDouble("longitude"); - // rs.getString("namespaceprefix"); - // rs.getInt("odnumberofitems"); // NULL - // rs.getDate("odnumberofitemsdate"); // NULL - // rs.getArray("subjects"); - // rs.getString("description"); - // rs.getString("odpolicies"); // NULL - // rs.getArray("odlanguages"); - // rs.getArray("odcontenttypes"); - // rs.getBoolean("inferred"); // false - // rs.getBoolean("deletedbyinference");// false - // rs.getDouble("trust"); // 0.9 - // rs.getString("inferenceprovenance"); // NULL - // rs.getDate("dateofcollection"); - // rs.getDate("dateofvalidation"); - // rs.getDate("releasestartdate"); - // rs.getDate("releaseenddate"); - // rs.getString("missionstatementurl"); - // rs.getBoolean("dataprovider"); - // rs.getBoolean("serviceprovider"); - // rs.getString("databaseaccesstype"); - // rs.getString("datauploadtype"); - // rs.getString("databaseaccessrestriction"); - // rs.getString("datauploadrestriction"); - // rs.getBoolean("versioning"); - // rs.getString("citationguidelineurl"); - // rs.getString("qualitymanagementkind"); - // rs.getString("pidsystems"); - // rs.getString("certificates"); - // rs.getArray("policies"); - // rs.getString("collectedfromid"); - // rs.getString("collectedfromname"); - // rs.getString("datasourcetype"); // COMPLEX - // rs.getString("provenanceaction"); // - // 'sysimport:crosswalk:entityregistry@@@sysimport:crosswalk:entityregistry@@@dnet:provenance_actions@@@dnet:provenance_actions' - // AS provenanceaction, - // rs.getString("journal"); // CONCAT(d.issn, '@@@', d.eissn, '@@@', d.lissn) AS journal - - emitOaf(ds); + return Arrays.asList(ds); } catch (final Exception e) { throw new RuntimeException(e); } } - public void processProject(final ResultSet rs) { + public List processProject(final ResultSet rs) { try { final DataInfo info = prepareDataInfo(rs); @@ -259,52 +222,14 @@ public class MigrateDbEntitiesApplication extends AbstractMigrationApplication i p.setDataInfo(info); p.setLastupdatetimestamp(lastUpdateTimestamp); - // rs.getString("projectid"); - // rs.getString("code"); - // rs.getString("websiteurl"); - // rs.getString("acronym"); - // rs.getString("title"); - // rs.getDate("startdate"); - // rs.getDate("enddate"); - // rs.getString("callidentifier"); - // rs.getString("keywords"); - // rs.getInt("duration"); - // rs.getBoolean("ecsc39"); - // rs.getBoolean("oamandatepublications"); - // rs.getBoolean("ecarticle29_3"); - // rs.getDate("dateofcollection"); - // rs.getDate("dateoftransformation"); - // rs.getBoolean("inferred"); - // rs.getBoolean("deletedbyinference"); - // rs.getDouble("trust"); - // rs.getString("inferenceprovenance"); - // rs.getString("optional1"); - // rs.getString("optional2"); - // rs.getString("jsonextrainfo"); - // rs.getString("contactfullname"); - // rs.getString("contactfax"); - // rs.getString("contactphone"); - // rs.getString("contactemail"); - // rs.getString("summary"); - // rs.getString("currency"); - // rs.getDouble("totalcost"); - // rs.getDouble("fundedamount"); - // rs.getString("collectedfromid"); - // rs.getString("collectedfromname"); - // rs.getString("contracttype"); // COMPLEX - // rs.getString("provenanceaction"); // COMPLEX - // rs.getArray("pid"); - // rs.getArray("subjects"); - // rs.getArray("fundingtree"); - - emitOaf(p); + return Arrays.asList(p); } catch (final Exception e) { throw new RuntimeException(e); } } - public void processOrganization(final ResultSet rs) { + public List processOrganization(final ResultSet rs) { try { @@ -339,41 +264,13 @@ public class MigrateDbEntitiesApplication extends AbstractMigrationApplication i o.setDataInfo(info); o.setLastupdatetimestamp(lastUpdateTimestamp); - // rs.getString("organizationid"); - // rs.getString("legalshortname"); - // rs.getString("legalname"); - // rs.getString("websiteurl"); - // rs.getString("logourl"); - // rs.getBoolean("eclegalbody"); - // rs.getBoolean("eclegalperson"); - // rs.getBoolean("ecnonprofit"); - // rs.getBoolean("ecresearchorganization"); - // rs.getBoolean("echighereducation"); - // rs.getBoolean("ecinternationalorganizationeurinterests"); - // rs.getBoolean("ecinternationalorganization"); - // rs.getBoolean("ecenterprise"); - // rs.getBoolean("ecsmevalidated"); - // rs.getBoolean("ecnutscode"); - // rs.getDate("dateofcollection"); - // rs.getDate("dateoftransformation"); - // rs.getBoolean("inferred"); - // rs.getBoolean("deletedbyinference"); - // rs.getDouble("trust"); - // rs.getString("inferenceprovenance"); - // rs.getString("collectedfromid"); - // rs.getString("collectedfromname"); - // rs.getString("country"); - // rs.getString("provenanceaction"); - // rs.getArray("pid"); - - emitOaf(o); + return Arrays.asList(o); } catch (final Exception e) { throw new RuntimeException(e); } } - public void processDatasourceOrganization(final ResultSet rs) { - + public List processDatasourceOrganization(final ResultSet rs) { try { final DataInfo info = prepareDataInfo(rs); final String orgId = createOpenaireId(20, rs.getString("organization"), true); @@ -389,7 +286,6 @@ public class MigrateDbEntitiesApplication extends AbstractMigrationApplication i r1.setCollectedFrom(collectedFrom); r1.setDataInfo(info); r1.setLastupdatetimestamp(lastUpdateTimestamp); - emitOaf(r1); final Relation r2 = new Relation(); r2.setRelType("datasourceOrganization"); @@ -400,29 +296,14 @@ public class MigrateDbEntitiesApplication extends AbstractMigrationApplication i r2.setCollectedFrom(collectedFrom); r2.setDataInfo(info); r2.setLastupdatetimestamp(lastUpdateTimestamp); - emitOaf(r2); - - // rs.getString("datasource"); - // rs.getString("organization"); - // rs.getDate("startdate"); // NULL - // rs.getDate("enddate"); // NULL - // rs.getBoolean("inferred"); // false - // rs.getBoolean("deletedbyinference"); // false - // rs.getDouble("trust"); // 0.9 - // rs.getString("inferenceprovenance"); // NULL - // rs.getString("semantics"); // 'providedBy@@@provided - // by@@@dnet:datasources_organizations_typologies@@@dnet:datasources_organizations_typologies' AS - // semantics, - // rs.getString("provenanceaction"); // d.provenanceaction || '@@@' || d.provenanceaction || - // '@@@dnet:provenanceActions@@@dnet:provenanceActions' AS provenanceaction + return Arrays.asList(r1, r2); } catch (final Exception e) { throw new RuntimeException(e); } } - public void processProjectOrganization(final ResultSet rs) { - + public List processProjectOrganization(final ResultSet rs) { try { final DataInfo info = prepareDataInfo(rs); final String orgId = createOpenaireId(20, rs.getString("resporganization"), true); @@ -438,7 +319,6 @@ public class MigrateDbEntitiesApplication extends AbstractMigrationApplication i r1.setCollectedFrom(collectedFrom); r1.setDataInfo(info); r1.setLastupdatetimestamp(lastUpdateTimestamp); - emitOaf(r1); final Relation r2 = new Relation(); r2.setRelType("projectOrganization"); @@ -449,30 +329,14 @@ public class MigrateDbEntitiesApplication extends AbstractMigrationApplication i r2.setCollectedFrom(collectedFrom); r2.setDataInfo(info); r2.setLastupdatetimestamp(lastUpdateTimestamp); - emitOaf(r2); - - // rs.getString("project"); - // rs.getString("resporganization"); - // rs.getInt("participantnumber"); - // rs.getDouble("contribution"); - // rs.getDate("startdate");// null - // rs.getDate("enddate");// null - // rs.getBoolean("inferred");// false - // rs.getBoolean("deletedbyinference"); // false - // rs.getDouble("trust"); - // rs.getString("inferenceprovenance"); // NULL - // rs.getString("semantics"); // po.semanticclass || '@@@' || po.semanticclass || - // '@@@dnet:project_organization_relations@@@dnet:project_organization_relations' AS semantics, - // rs.getString("provenanceaction"); // - // 'sysimport:crosswalk:entityregistry@@@sysimport:crosswalk:entityregistry@@@dnet:provenance_actions@@@dnet:provenance_actions' - // AS provenanceaction + return Arrays.asList(r1, r2); } catch (final Exception e) { throw new RuntimeException(e); } } - public void processClaims(final ResultSet rs) { + public List processClaims(final ResultSet rs) { final DataInfo info = dataInfo(false, null, false, false, qualifier("user:claim", "user:claim", "dnet:provenanceActions", "dnet:provenanceActions"), "0.9"); @@ -495,7 +359,8 @@ public class MigrateDbEntitiesApplication extends AbstractMigrationApplication i r.setLastupdatetimestamp(lastUpdateTimestamp); r.setContext(prepareContext(rs.getString("source_id"), info)); r.setDataInfo(info); - emitOaf(r); + + return Arrays.asList(r); } else { final String sourceId = createOpenaireId(rs.getString("source_type"), rs.getString("source_id"), false); final String targetId = createOpenaireId(rs.getString("target_type"), rs.getString("target_id"), false); @@ -525,14 +390,13 @@ public class MigrateDbEntitiesApplication extends AbstractMigrationApplication i r1.setTarget(targetId); r1.setDataInfo(info); r1.setLastupdatetimestamp(lastUpdateTimestamp); - emitOaf(r1); r2.setSource(targetId); r2.setTarget(sourceId); r2.setDataInfo(info); r2.setLastupdatetimestamp(lastUpdateTimestamp); - emitOaf(r2); + return Arrays.asList(r1, r2); } } catch (final Exception e) { @@ -563,7 +427,7 @@ public class MigrateDbEntitiesApplication extends AbstractMigrationApplication i private List> prepareListFields(final Array array, final DataInfo info) { try { - return listFields(info, (String[]) array.getArray()); + return array != null ? listFields(info, (String[]) array.getArray()) : new ArrayList<>(); } catch (final SQLException e) { throw new RuntimeException("Invalid SQL array", e); } diff --git a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/migration/utils/AbstractMigrationApplication.java b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/migration/utils/AbstractMigrationApplication.java index 8eb444562..e1a5e5fa7 100644 --- a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/migration/utils/AbstractMigrationApplication.java +++ b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/migration/utils/AbstractMigrationApplication.java @@ -28,6 +28,10 @@ public class AbstractMigrationApplication implements Closeable { private static final Log log = LogFactory.getLog(AbstractMigrationApplication.class); + protected AbstractMigrationApplication() { // ONLY FOR UNIT TEST + this.writer = null; + } + public AbstractMigrationApplication(final String hdfsPath) throws Exception { log.info(String.format("Creating SequenceFile Writer, hdfsPath=%s", hdfsPath)); diff --git a/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/migration/wfs/regular_step1_onlydb/oozie_app/workflow.xml b/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/migration/wfs/regular_step1_onlydb/oozie_app/workflow.xml index e81289161..0730f3a1f 100644 --- a/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/migration/wfs/regular_step1_onlydb/oozie_app/workflow.xml +++ b/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/migration/wfs/regular_step1_onlydb/oozie_app/workflow.xml @@ -1,4 +1,4 @@ - + migrationPathStep1 diff --git a/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/migration/step1/MigrateDbEntitiesApplicationTest.java b/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/migration/step1/MigrateDbEntitiesApplicationTest.java new file mode 100644 index 000000000..45d39dfd5 --- /dev/null +++ b/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/migration/step1/MigrateDbEntitiesApplicationTest.java @@ -0,0 +1,238 @@ +package eu.dnetlib.dhp.migration.step1; + +import static org.junit.Assert.assertEquals; + +import java.io.IOException; +import java.sql.Array; +import java.sql.Date; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.List; +import java.util.Objects; + +import org.apache.commons.io.IOUtils; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.junit.MockitoJUnitRunner; + +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; + +import eu.dnetlib.dhp.schema.oaf.Oaf; + +@RunWith(MockitoJUnitRunner.class) +public class MigrateDbEntitiesApplicationTest { + + private MigrateDbEntitiesApplication app; + + @Mock + private ResultSet rs; + + @Before + public void setUp() throws Exception { + this.app = new MigrateDbEntitiesApplication(); + } + + @Test + public void testProcessDatasource() throws Exception { + final List fields = prepareMocks("datasources_resultset_entry.json"); + + final List list = app.processDatasource(rs); + assertEquals(1, list.size()); + + verifyMocks(fields); + } + + @Test + public void testProcessProject() throws Exception { + final List fields = prepareMocks("projects_resultset_entry.json"); + + final List list = app.processProject(rs); + assertEquals(1, list.size()); + + verifyMocks(fields); + } + + @Test + public void testProcessOrganization() throws Exception { + final List fields = prepareMocks("organizations_resultset_entry.json"); + + final List list = app.processOrganization(rs); + + assertEquals(1, list.size()); + + verifyMocks(fields); + } + + @Test + public void testProcessDatasourceOrganization() throws Exception { + final List fields = prepareMocks("datasourceorganization_resultset_entry.json"); + + final List list = app.processDatasourceOrganization(rs); + + assertEquals(2, list.size()); + verifyMocks(fields); + } + + @Test + public void testProcessProjectOrganization() throws Exception { + final List fields = prepareMocks("projectorganization_resultset_entry.json"); + + final List list = app.processProjectOrganization(rs); + + assertEquals(2, list.size()); + verifyMocks(fields); + } + + @Test + public void testProcessClaims_context() throws Exception { + final List fields = prepareMocks("claimscontext_resultset_entry.json"); + + final List list = app.processClaims(rs); + + assertEquals(1, list.size()); + verifyMocks(fields); + } + + @Test + public void testProcessClaims_rels() throws Exception { + final List fields = prepareMocks("claimsrel_resultset_entry.json"); + + final List list = app.processClaims(rs); + + assertEquals(2, list.size()); + verifyMocks(fields); + } + + private List prepareMocks(final String jsonFile) throws IOException, SQLException { + final String json = IOUtils.toString(getClass().getResourceAsStream(jsonFile)); + final ObjectMapper mapper = new ObjectMapper(); + final List list = mapper.readValue(json, new TypeReference>() {}); + + for (final TypedField tf : list) { + if (tf.getValue() == null) { + switch (tf.getType()) { + case "not_used": + break; + case "boolean": + Mockito.when(rs.getBoolean(tf.getField())).thenReturn(false); + break; + case "date": + Mockito.when(rs.getDate(tf.getField())).thenReturn(null); + break; + case "int": + Mockito.when(rs.getInt(tf.getField())).thenReturn(0); + break; + case "double": + Mockito.when(rs.getDouble(tf.getField())).thenReturn(0.0); + break; + case "array": + Mockito.when(rs.getArray(tf.getField())).thenReturn(null); + break; + case "string": + default: + Mockito.when(rs.getString(tf.getField())).thenReturn(null); + break; + } + } else { + switch (tf.getType()) { + case "not_used": + break; + case "boolean": + Mockito.when(rs.getBoolean(tf.getField())).thenReturn(Boolean.parseBoolean(tf.getValue().toString())); + break; + case "date": + Mockito.when(rs.getDate(tf.getField())).thenReturn(Date.valueOf(tf.getValue().toString())); + break; + case "int": + Mockito.when(rs.getInt(tf.getField())).thenReturn(new Integer(tf.getValue().toString())); + break; + case "double": + Mockito.when(rs.getDouble(tf.getField())).thenReturn(new Double(tf.getValue().toString())); + break; + case "array": + final Array arr = Mockito.mock(Array.class); + final String[] values = ((List) tf.getValue()).stream() + .filter(Objects::nonNull) + .map(o -> o.toString()) + .toArray(String[]::new); + + Mockito.when(arr.getArray()).thenReturn(values); + Mockito.when(rs.getArray(tf.getField())).thenReturn(arr); + break; + case "string": + default: + Mockito.when(rs.getString(tf.getField())).thenReturn(tf.getValue().toString()); + break; + } + } + } + + return list; + } + + private void verifyMocks(final List list) throws SQLException { + for (final TypedField tf : list) { + + switch (tf.getType()) { + case "not_used": + break; + case "boolean": + Mockito.verify(rs, Mockito.atLeastOnce()).getBoolean(tf.getField()); + break; + case "date": + Mockito.verify(rs, Mockito.atLeastOnce()).getDate(tf.getField()); + break; + case "int": + Mockito.verify(rs, Mockito.atLeastOnce()).getInt(tf.getField()); + break; + case "double": + Mockito.verify(rs, Mockito.atLeastOnce()).getDouble(tf.getField()); + break; + case "array": + Mockito.verify(rs, Mockito.atLeastOnce()).getArray(tf.getField()); + break; + case "string": + default: + Mockito.verify(rs, Mockito.atLeastOnce()).getString(tf.getField()); + break; + } + } + + } +} + +class TypedField { + + private String field; + private String type; + private Object value; + + public String getField() { + return field; + } + + public void setField(final String field) { + this.field = field; + } + + public String getType() { + return type; + } + + public void setType(final String type) { + this.type = type; + } + + public Object getValue() { + return value; + } + + public void setValue(final Object value) { + this.value = value; + } + +} diff --git a/dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/migration/step1/claimscontext_resultset_entry.json b/dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/migration/step1/claimscontext_resultset_entry.json new file mode 100644 index 000000000..72bd01a96 --- /dev/null +++ b/dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/migration/step1/claimscontext_resultset_entry.json @@ -0,0 +1,27 @@ +[ + { + "field": "source_type", + "type": "string", + "value": "context" + }, + { + "field": "source_id", + "type": "string", + "value": "oa-pg" + }, + { + "field": "target_type", + "type": "string", + "value": "publication" + }, + { + "field": "target_id", + "type": "string", + "value": "userclaim___::d99de49026e79d271f3e7451d8de18b6" + }, + { + "field": "semantics", + "type": "not_used", + "value": "isRelevantTo" + } +] \ No newline at end of file diff --git a/dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/migration/step1/claimsrel_resultset_entry.json b/dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/migration/step1/claimsrel_resultset_entry.json new file mode 100644 index 000000000..28fa70035 --- /dev/null +++ b/dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/migration/step1/claimsrel_resultset_entry.json @@ -0,0 +1,27 @@ +[ + { + "field": "source_type", + "type": "string", + "value": "project" + }, + { + "field": "source_id", + "type": "string", + "value": "corda__h2020::b38a638a93b505d670fcacc47a0283d6" + }, + { + "field": "target_type", + "type": "string", + "value": "publication" + }, + { + "field": "target_id", + "type": "string", + "value": "userclaim___::5b5117253d3c64c79809d0b92fa287b4" + }, + { + "field": "semantics", + "type": "not_used", + "value": "resultProject_outcome_produces" + } +] \ No newline at end of file diff --git a/dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/migration/step1/datasourceorganization_resultset_entry.json b/dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/migration/step1/datasourceorganization_resultset_entry.json new file mode 100644 index 000000000..3a0318ed7 --- /dev/null +++ b/dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/migration/step1/datasourceorganization_resultset_entry.json @@ -0,0 +1,62 @@ +[ + { + "field": "datasource", + "type": "string", + "value": "openaire____::revistasunicauca" + }, + { + "field": "organization", + "type": "string", + "value": "openaire____::openaire____::revistasunicauca" + }, + { + "field": "startdate", + "type": "not_used", + "value": null + }, + { + "field": "enddate", + "type": "not_used", + "value": null + }, + { + "field": "inferred", + "type": "boolean", + "value": false + }, + { + "field": "deletedbyinference", + "type": "boolean", + "value": false + }, + { + "field": "trust", + "type": "string", + "value": "0.9" + }, + { + "field": "inferenceprovenance", + "type": "string", + "value": null + }, + { + "field": "collectedfromid", + "type": "string", + "value": null + }, + { + "field": "collectedfromname", + "type": "string", + "value": null + }, + { + "field": "semantics", + "type": "not_used", + "value": "providedBy@@@provided by@@@dnet:datasources_organizations_typologies@@@dnet:datasources_organizations_typologies" + }, + { + "field": "provenanceaction", + "type": "not_used", + "value": null + } +] \ No newline at end of file diff --git a/dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/migration/step1/datasources_resultset_entry.json b/dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/migration/step1/datasources_resultset_entry.json new file mode 100644 index 000000000..b60443d58 --- /dev/null +++ b/dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/migration/step1/datasources_resultset_entry.json @@ -0,0 +1,239 @@ +[ + { + "field": "datasourceid", + "type": "string", + "value": "274269ac6f3b::2579-5449" + }, + { + "field": "identities", + "type": "not_used", + "value": [ + "274269ac6f3b::2579-5449", + null + ] + }, + { + "field": "officialname", + "type": "string", + "value": "Jurnal Ilmiah Pendidikan Scholastic" + }, + { + "field": "englishname", + "type": "string", + "value": "Jurnal Ilmiah Pendidikan Scholastic" + }, + { + "field": "contactemail", + "type": "string", + "value": null + }, + { + "field": "openairecompatibility", + "type": "string", + "value": "hostedBy@@@collected from a compatible aggregator@@@dnet:datasourceCompatibilityLevel@@@dnet:datasourceCompatibilityLevel" + }, + { + "field": "websiteurl", + "type": "string", + "value": "http://e-journal.sastra-unes.com/index.php/JIPS/index" + }, + { + "field": "logourl", + "type": "string", + "value": null + }, + { + "field": "accessinfopackage", + "type": "array", + "value": [ + null + ] + }, + { + "field": "latitude", + "type": "double", + "value": 0 + }, + { + "field": "longitude", + "type": "double", + "value": 0 + }, + { + "field": "namespaceprefix", + "type": "string", + "value": "ojs_25795449" + }, + { + "field": "odnumberofitems", + "type": "int", + "value": null + }, + { + "field": "odnumberofitemsdate", + "type": "date", + "value": null + }, + { + "field": "subjects", + "type": "array", + "value": null + }, + { + "field": "description", + "type": "string", + "value": null + }, + { + "field": "odpolicies", + "type": "string", + "value": null + }, + { + "field": "odlanguages", + "type": "array", + "value": [] + }, + { + "field": "odcontenttypes", + "type": "array", + "value": [ + "Journal articles" + ] + }, + { + "field": "inferred", + "type": "boolean", + "value": false + }, + { + "field": "deletedbyinference", + "type": "boolean", + "value": false + }, + { + "field": "trust", + "type": "string", + "value": "0.9" + }, + { + "field": "inferenceprovenance", + "type": "string", + "value": null + }, + { + "field": "dateofcollection", + "type": "date", + "value": "2020-01-21" + }, + { + "field": "dateofvalidation", + "type": "date", + "value": null + }, + { + "field": "releasestartdate", + "type": "date", + "value": null + }, + { + "field": "releaseenddate", + "type": "date", + "value": null + }, + { + "field": "missionstatementurl", + "type": "string", + "value": null + }, + { + "field": "dataprovider", + "type": "boolean", + "value": null + }, + { + "field": "serviceprovider", + "type": "boolean", + "value": null + }, + { + "field": "databaseaccesstype", + "type": "string", + "value": null + }, + { + "field": "datauploadtype", + "type": "string", + "value": null + }, + { + "field": "databaseaccessrestriction", + "type": "string", + "value": null + }, + { + "field": "datauploadrestriction", + "type": "string", + "value": null + }, + { + "field": "versioning", + "type": "boolean", + "value": null + }, + { + "field": "citationguidelineurl", + "type": "string", + "value": null + }, + { + "field": "qualitymanagementkind", + "type": "string", + "value": null + }, + { + "field": "pidsystems", + "type": "string", + "value": null + }, + { + "field": "certificates", + "type": "string", + "value": null + }, + { + "field": "policies", + "type": "not_used", + "value": [] + }, + { + "field": "collectedfromid", + "type": "string", + "value": "openaire____::SnVybmFsIEZha3VsdGFzIFNhc3RyYSBVbml2ZXJzaXRhcyBFa2FzYWt0aQ==" + }, + { + "field": "collectedfromname", + "type": "string", + "value": "Jurnal Fakultas Sastra Universitas Ekasakti" + }, + { + "field": "datasourcetype", + "type": "string", + "value": "pubsrepository::journal@@@Journal@@@dnet:datasource_typologies@@@dnet:datasource_typologies" + }, + { + "field": "provenanceaction", + "type": "not_used", + "value": "sysimport:crosswalk:entityregistry@@@sysimport:crosswalk:entityregistry@@@dnet:provenance_actions@@@dnet:provenance_actions" + }, + { + "field": "journal", + "type": "string", + "value": "2579-5449@@@2597-6540@@@" + }, + { + "field": "collectedfromid", + "type": "string", + "value": "openaire____::SnVybmFsIEZha3VsdGFzIFNhc3RyYSBVbml2ZXJzaXRhcyBFa2FzYWt0aQ==" + } +] diff --git a/dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/migration/step1/organizations_resultset_entry.json b/dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/migration/step1/organizations_resultset_entry.json new file mode 100644 index 000000000..ed1379784 --- /dev/null +++ b/dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/migration/step1/organizations_resultset_entry.json @@ -0,0 +1,127 @@ +[ + { + "field": "organizationid", + "type": "string", + "value": "openaire____::openaire____::microsoft" + }, + { + "field": "legalshortname", + "type": "string", + "value": null + }, + { + "field": "legalname", + "type": "string", + "value": "Microsoft Research" + }, + { + "field": "websiteurl", + "type": "string", + "value": null + }, + { + "field": "logourl", + "type": "string", + "value": null + }, + { + "field": "eclegalbody", + "type": "boolean", + "value": false + }, + { + "field": "eclegalperson", + "type": "boolean", + "value": false + }, + { + "field": "ecnonprofit", + "type": "boolean", + "value": false + }, + { + "field": "ecresearchorganization", + "type": "boolean", + "value": false + }, + { + "field": "echighereducation", + "type": "boolean", + "value": false + }, + { + "field": "ecinternationalorganizationeurinterests", + "type": "boolean", + "value": false + }, + { + "field": "ecinternationalorganization", + "type": "boolean", + "value": false + }, + { + "field": "ecenterprise", + "type": "boolean", + "value": false + }, + { + "field": "ecsmevalidated", + "type": "boolean", + "value": false + }, + { + "field": "ecnutscode", + "type": "boolean", + "value": false + }, + { + "field": "dateofcollection", + "type": "date", + "value": "2018-10-19" + }, + { + "field": "dateoftransformation", + "type": "date", + "value": "2018-10-19" + }, + { + "field": "inferred", + "type": "boolean", + "value": false + }, + { + "field": "deletedbyinference", + "type": "boolean", + "value": false + }, + { + "field": "trust", + "type": "string", + "value": "0.9" + }, + { + "field": "inferenceprovenance", + "type": "string", + "value": "" + }, + { + "field": "collectedfromid", + "type": "string", + "value": null + }, + { + "field": "collectedfromname", + "type": "string", + "value": null + }, + { + "field": "country", + "type": "string", + "value": "US@@@US@@@dnet:countries@@@dnet:countries" + }, + { + "field": "provenanceaction", + "type": "not_used", + "value": "sysimport:crosswalk:entityregistry@@@sysimport:crosswalk:entityregistry@@@dnet:provenance_actions@@@dnet:provenance_actions" + } +] \ No newline at end of file diff --git a/dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/migration/step1/projectorganization_resultset_entry.json b/dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/migration/step1/projectorganization_resultset_entry.json new file mode 100644 index 000000000..855e1a483 --- /dev/null +++ b/dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/migration/step1/projectorganization_resultset_entry.json @@ -0,0 +1,72 @@ +[ + { + "field": "project", + "type": "string", + "value": "nsf_________::1700003" + }, + { + "field": "resporganization", + "type": "string", + "value": "nsf_________::University_of_Notre_Dame" + }, + { + "field": "participantnumber", + "type": "not_used", + "value": 1 + }, + { + "field": "contribution", + "type": "not_used", + "value": null + }, + { + "field": "startdate", + "type": "not_used", + "value": null + }, + { + "field": "enddate", + "type": "not_used", + "value": null + }, + { + "field": "inferred", + "type": "boolean", + "value": false + }, + { + "field": "deletedbyinference", + "type": "boolean", + "value": false + }, + { + "field": "trust", + "type": "string", + "value": "0.9" + }, + { + "field": "inferenceprovenance", + "type": "string", + "value": null + }, + { + "field": "collectedfromid", + "type": "string", + "value": "openaire____::nsf" + }, + { + "field": "collectedfromname", + "type": "string", + "value": "NSF - National Science Foundation" + }, + { + "field": "semantics", + "type": "not_used", + "value": "coordinator@@@coordinator@@@dnet:project_organization_relations@@@dnet:project_organization_relations" + }, + { + "field": "provenanceaction", + "type": "not_used", + "value": "sysimport:crosswalk:entityregistry@@@sysimport:crosswalk:entityregistry@@@dnet:provenance_actions@@@dnet:provenance_actions" + } +] \ No newline at end of file diff --git a/dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/migration/step1/projects_resultset_entry.json b/dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/migration/step1/projects_resultset_entry.json new file mode 100644 index 000000000..7d6ebffbe --- /dev/null +++ b/dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/migration/step1/projects_resultset_entry.json @@ -0,0 +1,193 @@ +[ + { + "field": "projectid", + "type": "string", + "value": "aka_________::100469" + }, + { + "field": "code", + "type": "string", + "value": "100469" + }, + { + "field": "websiteurl", + "type": "string", + "value": "http://test" + }, + { + "field": "acronym", + "type": "string", + "value": "RMCAG" + }, + { + "field": "title", + "type": "string", + "value": "Regulation of melanoma cell autonomous growth" + }, + { + "field": "startdate", + "type": "date", + "value": null + }, + { + "field": "enddate", + "type": "date", + "value": null + }, + { + "field": "callidentifier", + "type": "string", + "value": "Tutkijankoulutus ja työskentely ulkomailla/kevät TT" + }, + { + "field": "keywords", + "type": "string", + "value": null + }, + { + "field": "duration", + "type": "int", + "value": null + }, + { + "field": "ecsc39", + "type": "boolean", + "value": null + }, + { + "field": "oamandatepublications", + "type": "boolean", + "value": false + }, + { + "field": "ecarticle29_3", + "type": "boolean", + "value": null + }, + { + "field": "dateofcollection", + "type": "date", + "value": "2019-01-25" + }, + { + "field": "dateoftransformation", + "type": "date", + "value": "2019-04-16" + }, + { + "field": "inferred", + "type": "boolean", + "value": false + }, + { + "field": "deletedbyinference", + "type": "boolean", + "value": false + }, + { + "field": "trust", + "type": "string", + "value": "0.9" + }, + { + "field": "inferenceprovenance", + "type": "string", + "value": null + }, + { + "field": "optional1", + "type": "string", + "value": "9,284 €" + }, + { + "field": "optional2", + "type": "string", + "value": null + }, + { + "field": "jsonextrainfo", + "type": "string", + "value": "{}" + }, + { + "field": "contactfullname", + "type": "string", + "value": null + }, + { + "field": "contactfax", + "type": "string", + "value": null + }, + { + "field": "contactphone", + "type": "string", + "value": null + }, + { + "field": "contactemail", + "type": "string", + "value": null + }, + { + "field": "summary", + "type": "string", + "value": null + }, + { + "field": "currency", + "type": "string", + "value": null + }, + { + "field": "totalcost", + "type": "double", + "value": null + }, + { + "field": "fundedamount", + "type": "double", + "value": null + }, + { + "field": "collectedfromid", + "type": "string", + "value": "openaire____::aka" + }, + { + "field": "collectedfromname", + "type": "string", + "value": "Academy of Finland" + }, + { + "field": "contracttype", + "type": "string", + "value": null + }, + { + "field": "provenanceaction", + "type": "not_used", + "value": "sysimport:crosswalk:entityregistry@@@Harvested@@@dnet:provenanceActions@@@dnet:provenanceActions" + }, + { + "field": "pid", + "type": "not_used", + "value": [ + null + ] + }, + { + "field": "subjects", + "type": "array", + "value": [ + null + ] + }, + { + "field": "fundingtree", + "type": "array", + "value": [ + "\n aka_________::AKA\n AKA\n Academy of Finland\n Academy of Finland\n FI\n " + ] + } +] \ No newline at end of file