spyros
/
dnet-hadoop
forked from D-Net/dnet-hadoop
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

some tests on db records

This commit is contained in:
Michele Artini 2020-03-25 09:43:58 +01:00
parent e3760c7f39
commit 0fda2c3a30
11 changed files with 1017 additions and 164 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

190
dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/migration/step1/MigrateDbEntitiesApplication.java
View File

@ -20,6 +20,7 @@ import java.util.Arrays;
import java.util.Date; import java.util.Date;
import java.util.List; import java.util.List;
import java.util.function.Consumer; import java.util.function.Consumer;
import java.util.function.Function;
import org.apache.commons.io.IOUtils; import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.StringUtils;
@ -36,6 +37,7 @@ import eu.dnetlib.dhp.schema.oaf.Datasource;
import eu.dnetlib.dhp.schema.oaf.Field; import eu.dnetlib.dhp.schema.oaf.Field;
import eu.dnetlib.dhp.schema.oaf.Journal; import eu.dnetlib.dhp.schema.oaf.Journal;
import eu.dnetlib.dhp.schema.oaf.KeyValue; import eu.dnetlib.dhp.schema.oaf.KeyValue;
import eu.dnetlib.dhp.schema.oaf.Oaf;
import eu.dnetlib.dhp.schema.oaf.Organization; import eu.dnetlib.dhp.schema.oaf.Organization;
import eu.dnetlib.dhp.schema.oaf.OtherResearchProduct; import eu.dnetlib.dhp.schema.oaf.OtherResearchProduct;
import eu.dnetlib.dhp.schema.oaf.Project; import eu.dnetlib.dhp.schema.oaf.Project;
@ -95,6 +97,12 @@ public class MigrateDbEntitiesApplication extends AbstractMigrationApplication i
} }
} }
protected MigrateDbEntitiesApplication() { // ONLY FOR UNIT TEST
super();
this.dbClient = null;
this.lastUpdateTimestamp = new Date().getTime();
}
public MigrateDbEntitiesApplication(final String hdfsPath, final String dbUrl, final String dbUser, public MigrateDbEntitiesApplication(final String hdfsPath, final String dbUrl, final String dbUser,
final String dbPassword) throws Exception { final String dbPassword) throws Exception {
super(hdfsPath); super(hdfsPath);
@ -102,12 +110,15 @@ public class MigrateDbEntitiesApplication extends AbstractMigrationApplication i
this.lastUpdateTimestamp = new Date().getTime(); this.lastUpdateTimestamp = new Date().getTime();
} }
public void execute(final String sqlFile, final Consumer<ResultSet> consumer) throws Exception { public void execute(final String sqlFile, final Function<ResultSet, List<Oaf>> producer) throws Exception {
final String sql = IOUtils.toString(getClass().getResourceAsStream("/eu/dnetlib/dhp/migration/sql/" + sqlFile)); final String sql = IOUtils.toString(getClass().getResourceAsStream("/eu/dnetlib/dhp/migration/sql/" + sqlFile));
final Consumer<ResultSet> consumer = rs -> producer.apply(rs).forEach(oaf -> emitOaf(oaf));
dbClient.processResults(sql, consumer); dbClient.processResults(sql, consumer);
} }
public void processDatasource(final ResultSet rs) { public List<Oaf> processDatasource(final ResultSet rs) {
try { try {
@ -161,61 +172,13 @@ public class MigrateDbEntitiesApplication extends AbstractMigrationApplication i
ds.setDataInfo(info); ds.setDataInfo(info);
ds.setLastupdatetimestamp(lastUpdateTimestamp); ds.setLastupdatetimestamp(lastUpdateTimestamp);
// rs.getString("datasourceid"); return Arrays.asList(ds);
// rs.getArray("identities");
// rs.getString("officialname");
// rs.getString("englishname");
// rs.getString("contactemail");
// rs.getString("openairecompatibility"); // COMPLEX ...@@@...
// rs.getString("websiteurl");
// rs.getString("logourl");
// rs.getArray("accessinfopackage");
// rs.getDouble("latitude");
// rs.getDouble("longitude");
// rs.getString("namespaceprefix");
// rs.getInt("odnumberofitems"); // NULL
// rs.getDate("odnumberofitemsdate"); // NULL
// rs.getArray("subjects");
// rs.getString("description");
// rs.getString("odpolicies"); // NULL
// rs.getArray("odlanguages");
// rs.getArray("odcontenttypes");
// rs.getBoolean("inferred"); // false
// rs.getBoolean("deletedbyinference");// false
// rs.getDouble("trust"); // 0.9
// rs.getString("inferenceprovenance"); // NULL
// rs.getDate("dateofcollection");
// rs.getDate("dateofvalidation");
// rs.getDate("releasestartdate");
// rs.getDate("releaseenddate");
// rs.getString("missionstatementurl");
// rs.getBoolean("dataprovider");
// rs.getBoolean("serviceprovider");
// rs.getString("databaseaccesstype");
// rs.getString("datauploadtype");
// rs.getString("databaseaccessrestriction");
// rs.getString("datauploadrestriction");
// rs.getBoolean("versioning");
// rs.getString("citationguidelineurl");
// rs.getString("qualitymanagementkind");
// rs.getString("pidsystems");
// rs.getString("certificates");
// rs.getArray("policies");
// rs.getString("collectedfromid");
// rs.getString("collectedfromname");
// rs.getString("datasourcetype"); // COMPLEX
// rs.getString("provenanceaction"); //
// 'sysimport:crosswalk:entityregistry@@@sysimport:crosswalk:entityregistry@@@dnet:provenance_actions@@@dnet:provenance_actions'
// AS provenanceaction,
// rs.getString("journal"); // CONCAT(d.issn, '@@@', d.eissn, '@@@', d.lissn) AS journal
emitOaf(ds);
} catch (final Exception e) { } catch (final Exception e) {
throw new RuntimeException(e); throw new RuntimeException(e);
} }
} }
public void processProject(final ResultSet rs) { public List<Oaf> processProject(final ResultSet rs) {
try { try {
final DataInfo info = prepareDataInfo(rs); final DataInfo info = prepareDataInfo(rs);
@ -259,52 +222,14 @@ public class MigrateDbEntitiesApplication extends AbstractMigrationApplication i
p.setDataInfo(info); p.setDataInfo(info);
p.setLastupdatetimestamp(lastUpdateTimestamp); p.setLastupdatetimestamp(lastUpdateTimestamp);
// rs.getString("projectid"); return Arrays.asList(p);
// rs.getString("code");
// rs.getString("websiteurl");
// rs.getString("acronym");
// rs.getString("title");
// rs.getDate("startdate");
// rs.getDate("enddate");
// rs.getString("callidentifier");
// rs.getString("keywords");
// rs.getInt("duration");
// rs.getBoolean("ecsc39");
// rs.getBoolean("oamandatepublications");
// rs.getBoolean("ecarticle29_3");
// rs.getDate("dateofcollection");
// rs.getDate("dateoftransformation");
// rs.getBoolean("inferred");
// rs.getBoolean("deletedbyinference");
// rs.getDouble("trust");
// rs.getString("inferenceprovenance");
// rs.getString("optional1");
// rs.getString("optional2");
// rs.getString("jsonextrainfo");
// rs.getString("contactfullname");
// rs.getString("contactfax");
// rs.getString("contactphone");
// rs.getString("contactemail");
// rs.getString("summary");
// rs.getString("currency");
// rs.getDouble("totalcost");
// rs.getDouble("fundedamount");
// rs.getString("collectedfromid");
// rs.getString("collectedfromname");
// rs.getString("contracttype"); // COMPLEX
// rs.getString("provenanceaction"); // COMPLEX
// rs.getArray("pid");
// rs.getArray("subjects");
// rs.getArray("fundingtree");
emitOaf(p);
} catch (final Exception e) { } catch (final Exception e) {
throw new RuntimeException(e); throw new RuntimeException(e);
} }
} }
public void processOrganization(final ResultSet rs) { public List<Oaf> processOrganization(final ResultSet rs) {
try { try {
@ -339,41 +264,13 @@ public class MigrateDbEntitiesApplication extends AbstractMigrationApplication i
o.setDataInfo(info); o.setDataInfo(info);
o.setLastupdatetimestamp(lastUpdateTimestamp); o.setLastupdatetimestamp(lastUpdateTimestamp);
// rs.getString("organizationid"); return Arrays.asList(o);
// rs.getString("legalshortname");
// rs.getString("legalname");
// rs.getString("websiteurl");
// rs.getString("logourl");
// rs.getBoolean("eclegalbody");
// rs.getBoolean("eclegalperson");
// rs.getBoolean("ecnonprofit");
// rs.getBoolean("ecresearchorganization");
// rs.getBoolean("echighereducation");
// rs.getBoolean("ecinternationalorganizationeurinterests");
// rs.getBoolean("ecinternationalorganization");
// rs.getBoolean("ecenterprise");
// rs.getBoolean("ecsmevalidated");
// rs.getBoolean("ecnutscode");
// rs.getDate("dateofcollection");
// rs.getDate("dateoftransformation");
// rs.getBoolean("inferred");
// rs.getBoolean("deletedbyinference");
// rs.getDouble("trust");
// rs.getString("inferenceprovenance");
// rs.getString("collectedfromid");
// rs.getString("collectedfromname");
// rs.getString("country");
// rs.getString("provenanceaction");
// rs.getArray("pid");
emitOaf(o);
} catch (final Exception e) { } catch (final Exception e) {
throw new RuntimeException(e); throw new RuntimeException(e);
} }
} }
public void processDatasourceOrganization(final ResultSet rs) { public List<Oaf> processDatasourceOrganization(final ResultSet rs) {
try { try {
final DataInfo info = prepareDataInfo(rs); final DataInfo info = prepareDataInfo(rs);
final String orgId = createOpenaireId(20, rs.getString("organization"), true); final String orgId = createOpenaireId(20, rs.getString("organization"), true);
@ -389,7 +286,6 @@ public class MigrateDbEntitiesApplication extends AbstractMigrationApplication i
r1.setCollectedFrom(collectedFrom); r1.setCollectedFrom(collectedFrom);
r1.setDataInfo(info); r1.setDataInfo(info);
r1.setLastupdatetimestamp(lastUpdateTimestamp); r1.setLastupdatetimestamp(lastUpdateTimestamp);
emitOaf(r1);
final Relation r2 = new Relation(); final Relation r2 = new Relation();
r2.setRelType("datasourceOrganization"); r2.setRelType("datasourceOrganization");
@ -400,29 +296,14 @@ public class MigrateDbEntitiesApplication extends AbstractMigrationApplication i
r2.setCollectedFrom(collectedFrom); r2.setCollectedFrom(collectedFrom);
r2.setDataInfo(info); r2.setDataInfo(info);
r2.setLastupdatetimestamp(lastUpdateTimestamp); r2.setLastupdatetimestamp(lastUpdateTimestamp);
emitOaf(r2);
// rs.getString("datasource");
// rs.getString("organization");
// rs.getDate("startdate"); // NULL
// rs.getDate("enddate"); // NULL
// rs.getBoolean("inferred"); // false
// rs.getBoolean("deletedbyinference"); // false
// rs.getDouble("trust"); // 0.9
// rs.getString("inferenceprovenance"); // NULL
// rs.getString("semantics"); // 'providedBy@@@provided
// by@@@dnet:datasources_organizations_typologies@@@dnet:datasources_organizations_typologies' AS
// semantics,
// rs.getString("provenanceaction"); // d.provenanceaction || '@@@' || d.provenanceaction ||
// '@@@dnet:provenanceActions@@@dnet:provenanceActions' AS provenanceaction
return Arrays.asList(r1, r2);
} catch (final Exception e) { } catch (final Exception e) {
throw new RuntimeException(e); throw new RuntimeException(e);
} }
} }
public void processProjectOrganization(final ResultSet rs) { public List<Oaf> processProjectOrganization(final ResultSet rs) {
try { try {
final DataInfo info = prepareDataInfo(rs); final DataInfo info = prepareDataInfo(rs);
final String orgId = createOpenaireId(20, rs.getString("resporganization"), true); final String orgId = createOpenaireId(20, rs.getString("resporganization"), true);
@ -438,7 +319,6 @@ public class MigrateDbEntitiesApplication extends AbstractMigrationApplication i
r1.setCollectedFrom(collectedFrom); r1.setCollectedFrom(collectedFrom);
r1.setDataInfo(info); r1.setDataInfo(info);
r1.setLastupdatetimestamp(lastUpdateTimestamp); r1.setLastupdatetimestamp(lastUpdateTimestamp);
emitOaf(r1);
final Relation r2 = new Relation(); final Relation r2 = new Relation();
r2.setRelType("projectOrganization"); r2.setRelType("projectOrganization");
@ -449,30 +329,14 @@ public class MigrateDbEntitiesApplication extends AbstractMigrationApplication i
r2.setCollectedFrom(collectedFrom); r2.setCollectedFrom(collectedFrom);
r2.setDataInfo(info); r2.setDataInfo(info);
r2.setLastupdatetimestamp(lastUpdateTimestamp); r2.setLastupdatetimestamp(lastUpdateTimestamp);
emitOaf(r2);
// rs.getString("project");
// rs.getString("resporganization");
// rs.getInt("participantnumber");
// rs.getDouble("contribution");
// rs.getDate("startdate");// null
// rs.getDate("enddate");// null
// rs.getBoolean("inferred");// false
// rs.getBoolean("deletedbyinference"); // false
// rs.getDouble("trust");
// rs.getString("inferenceprovenance"); // NULL
// rs.getString("semantics"); // po.semanticclass || '@@@' || po.semanticclass ||
// '@@@dnet:project_organization_relations@@@dnet:project_organization_relations' AS semantics,
// rs.getString("provenanceaction"); //
// 'sysimport:crosswalk:entityregistry@@@sysimport:crosswalk:entityregistry@@@dnet:provenance_actions@@@dnet:provenance_actions'
// AS provenanceaction
return Arrays.asList(r1, r2);
} catch (final Exception e) { } catch (final Exception e) {
throw new RuntimeException(e); throw new RuntimeException(e);
} }
} }
public void processClaims(final ResultSet rs) { public List<Oaf> processClaims(final ResultSet rs) {
final DataInfo info = final DataInfo info =
dataInfo(false, null, false, false, qualifier("user:claim", "user:claim", "dnet:provenanceActions", "dnet:provenanceActions"), "0.9"); dataInfo(false, null, false, false, qualifier("user:claim", "user:claim", "dnet:provenanceActions", "dnet:provenanceActions"), "0.9");
@ -495,7 +359,8 @@ public class MigrateDbEntitiesApplication extends AbstractMigrationApplication i
r.setLastupdatetimestamp(lastUpdateTimestamp); r.setLastupdatetimestamp(lastUpdateTimestamp);
r.setContext(prepareContext(rs.getString("source_id"), info)); r.setContext(prepareContext(rs.getString("source_id"), info));
r.setDataInfo(info); r.setDataInfo(info);
emitOaf(r);
return Arrays.asList(r);
} else { } else {
final String sourceId = createOpenaireId(rs.getString("source_type"), rs.getString("source_id"), false); final String sourceId = createOpenaireId(rs.getString("source_type"), rs.getString("source_id"), false);
final String targetId = createOpenaireId(rs.getString("target_type"), rs.getString("target_id"), false); final String targetId = createOpenaireId(rs.getString("target_type"), rs.getString("target_id"), false);
@ -525,14 +390,13 @@ public class MigrateDbEntitiesApplication extends AbstractMigrationApplication i
r1.setTarget(targetId); r1.setTarget(targetId);
r1.setDataInfo(info); r1.setDataInfo(info);
r1.setLastupdatetimestamp(lastUpdateTimestamp); r1.setLastupdatetimestamp(lastUpdateTimestamp);
emitOaf(r1);
r2.setSource(targetId); r2.setSource(targetId);
r2.setTarget(sourceId); r2.setTarget(sourceId);
r2.setDataInfo(info); r2.setDataInfo(info);
r2.setLastupdatetimestamp(lastUpdateTimestamp); r2.setLastupdatetimestamp(lastUpdateTimestamp);
emitOaf(r2);
return Arrays.asList(r1, r2);
} }
} catch (final Exception e) { } catch (final Exception e) {
@ -563,7 +427,7 @@ public class MigrateDbEntitiesApplication extends AbstractMigrationApplication i
private List<Field<String>> prepareListFields(final Array array, final DataInfo info) { private List<Field<String>> prepareListFields(final Array array, final DataInfo info) {
try { try {
return listFields(info, (String[]) array.getArray()); return array != null ? listFields(info, (String[]) array.getArray()) : new ArrayList<>();
} catch (final SQLException e) { } catch (final SQLException e) {
throw new RuntimeException("Invalid SQL array", e); throw new RuntimeException("Invalid SQL array", e);
} }

4
dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/migration/utils/AbstractMigrationApplication.java
View File

@ -28,6 +28,10 @@ public class AbstractMigrationApplication implements Closeable {
private static final Log log = LogFactory.getLog(AbstractMigrationApplication.class); private static final Log log = LogFactory.getLog(AbstractMigrationApplication.class);
protected AbstractMigrationApplication() { // ONLY FOR UNIT TEST
this.writer = null;
}
public AbstractMigrationApplication(final String hdfsPath) throws Exception { public AbstractMigrationApplication(final String hdfsPath) throws Exception {
log.info(String.format("Creating SequenceFile Writer, hdfsPath=%s", hdfsPath)); log.info(String.format("Creating SequenceFile Writer, hdfsPath=%s", hdfsPath));

2
dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/migration/wfs/regular_step1_onlydb/oozie_app/workflow.xml
View File

@ -1,4 +1,4 @@
<workflow-app name="import regular entities as Graph (step 1)" xmlns="uri:oozie:workflow:0.5"> <workflow-app name="import db entities (step 1)" xmlns="uri:oozie:workflow:0.5">
<parameters> <parameters>
<property> <property>
<name>migrationPathStep1</name> <name>migrationPathStep1</name>

238
dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/migration/step1/MigrateDbEntitiesApplicationTest.java Normal file
View File

@ -0,0 +1,238 @@
package eu.dnetlib.dhp.migration.step1;
import static org.junit.Assert.assertEquals;
import java.io.IOException;
import java.sql.Array;
import java.sql.Date;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.List;
import java.util.Objects;
import org.apache.commons.io.IOUtils;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.Mock;
import org.mockito.Mockito;
import org.mockito.junit.MockitoJUnitRunner;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
import eu.dnetlib.dhp.schema.oaf.Oaf;
@RunWith(MockitoJUnitRunner.class)
public class MigrateDbEntitiesApplicationTest {
private MigrateDbEntitiesApplication app;
@Mock
private ResultSet rs;
@Before
public void setUp() throws Exception {
this.app = new MigrateDbEntitiesApplication();
}
@Test
public void testProcessDatasource() throws Exception {
final List<TypedField> fields = prepareMocks("datasources_resultset_entry.json");
final List<Oaf> list = app.processDatasource(rs);
assertEquals(1, list.size());
verifyMocks(fields);
}
@Test
public void testProcessProject() throws Exception {
final List<TypedField> fields = prepareMocks("projects_resultset_entry.json");
final List<Oaf> list = app.processProject(rs);
assertEquals(1, list.size());
verifyMocks(fields);
}
@Test
public void testProcessOrganization() throws Exception {
final List<TypedField> fields = prepareMocks("organizations_resultset_entry.json");
final List<Oaf> list = app.processOrganization(rs);
assertEquals(1, list.size());
verifyMocks(fields);
}
@Test
public void testProcessDatasourceOrganization() throws Exception {
final List<TypedField> fields = prepareMocks("datasourceorganization_resultset_entry.json");
final List<Oaf> list = app.processDatasourceOrganization(rs);
assertEquals(2, list.size());
verifyMocks(fields);
}
@Test
public void testProcessProjectOrganization() throws Exception {
final List<TypedField> fields = prepareMocks("projectorganization_resultset_entry.json");
final List<Oaf> list = app.processProjectOrganization(rs);
assertEquals(2, list.size());
verifyMocks(fields);
}
@Test
public void testProcessClaims_context() throws Exception {
final List<TypedField> fields = prepareMocks("claimscontext_resultset_entry.json");
final List<Oaf> list = app.processClaims(rs);
assertEquals(1, list.size());
verifyMocks(fields);
}
@Test
public void testProcessClaims_rels() throws Exception {
final List<TypedField> fields = prepareMocks("claimsrel_resultset_entry.json");
final List<Oaf> list = app.processClaims(rs);
assertEquals(2, list.size());
verifyMocks(fields);
}
private List<TypedField> prepareMocks(final String jsonFile) throws IOException, SQLException {
final String json = IOUtils.toString(getClass().getResourceAsStream(jsonFile));
final ObjectMapper mapper = new ObjectMapper();
final List<TypedField> list = mapper.readValue(json, new TypeReference<List<TypedField>>() {});
for (final TypedField tf : list) {
if (tf.getValue() == null) {
switch (tf.getType()) {
case "not_used":
break;
case "boolean":
Mockito.when(rs.getBoolean(tf.getField())).thenReturn(false);
break;
case "date":
Mockito.when(rs.getDate(tf.getField())).thenReturn(null);
break;
case "int":
Mockito.when(rs.getInt(tf.getField())).thenReturn(0);
break;
case "double":
Mockito.when(rs.getDouble(tf.getField())).thenReturn(0.0);
break;
case "array":
Mockito.when(rs.getArray(tf.getField())).thenReturn(null);
break;
case "string":
default:
Mockito.when(rs.getString(tf.getField())).thenReturn(null);
break;
}
} else {
switch (tf.getType()) {
case "not_used":
break;
case "boolean":
Mockito.when(rs.getBoolean(tf.getField())).thenReturn(Boolean.parseBoolean(tf.getValue().toString()));
break;
case "date":
Mockito.when(rs.getDate(tf.getField())).thenReturn(Date.valueOf(tf.getValue().toString()));
break;
case "int":
Mockito.when(rs.getInt(tf.getField())).thenReturn(new Integer(tf.getValue().toString()));
break;
case "double":
Mockito.when(rs.getDouble(tf.getField())).thenReturn(new Double(tf.getValue().toString()));
break;
case "array":
final Array arr = Mockito.mock(Array.class);
final String[] values = ((List<?>) tf.getValue()).stream()
.filter(Objects::nonNull)
.map(o -> o.toString())
.toArray(String[]::new);
Mockito.when(arr.getArray()).thenReturn(values);
Mockito.when(rs.getArray(tf.getField())).thenReturn(arr);
break;
case "string":
default:
Mockito.when(rs.getString(tf.getField())).thenReturn(tf.getValue().toString());
break;
}
}
}
return list;
}
private void verifyMocks(final List<TypedField> list) throws SQLException {
for (final TypedField tf : list) {
switch (tf.getType()) {
case "not_used":
break;
case "boolean":
Mockito.verify(rs, Mockito.atLeastOnce()).getBoolean(tf.getField());
break;
case "date":
Mockito.verify(rs, Mockito.atLeastOnce()).getDate(tf.getField());
break;
case "int":
Mockito.verify(rs, Mockito.atLeastOnce()).getInt(tf.getField());
break;
case "double":
Mockito.verify(rs, Mockito.atLeastOnce()).getDouble(tf.getField());
break;
case "array":
Mockito.verify(rs, Mockito.atLeastOnce()).getArray(tf.getField());
break;
case "string":
default:
Mockito.verify(rs, Mockito.atLeastOnce()).getString(tf.getField());
break;
}
}
}
}
class TypedField {
private String field;
private String type;
private Object value;
public String getField() {
return field;
}
public void setField(final String field) {
this.field = field;
}
public String getType() {
return type;
}
public void setType(final String type) {
this.type = type;
}
public Object getValue() {
return value;
}
public void setValue(final Object value) {
this.value = value;
}
}

27
dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/migration/step1/claimscontext_resultset_entry.json Normal file
View File

@ -0,0 +1,27 @@
[
{
"field": "source_type",
"type": "string",
"value": "context"
},
{
"field": "source_id",
"type": "string",
"value": "oa-pg"
},
{
"field": "target_type",
"type": "string",
"value": "publication"
},
{
"field": "target_id",
"type": "string",
"value": "userclaim___::d99de49026e79d271f3e7451d8de18b6"
},
{
"field": "semantics",
"type": "not_used",
"value": "isRelevantTo"
}
]

27
dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/migration/step1/claimsrel_resultset_entry.json Normal file
View File

@ -0,0 +1,27 @@
[
{
"field": "source_type",
"type": "string",
"value": "project"
},
{
"field": "source_id",
"type": "string",
"value": "corda__h2020::b38a638a93b505d670fcacc47a0283d6"
},
{
"field": "target_type",
"type": "string",
"value": "publication"
},
{
"field": "target_id",
"type": "string",
"value": "userclaim___::5b5117253d3c64c79809d0b92fa287b4"
},
{
"field": "semantics",
"type": "not_used",
"value": "resultProject_outcome_produces"
}
]

62
dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/migration/step1/datasourceorganization_resultset_entry.json Normal file
View File

@ -0,0 +1,62 @@
[
{
"field": "datasource",
"type": "string",
"value": "openaire____::revistasunicauca"
},
{
"field": "organization",
"type": "string",
"value": "openaire____::openaire____::revistasunicauca"
},
{
"field": "startdate",
"type": "not_used",
"value": null
},
{
"field": "enddate",
"type": "not_used",
"value": null
},
{
"field": "inferred",
"type": "boolean",
"value": false
},
{
"field": "deletedbyinference",
"type": "boolean",
"value": false
},
{
"field": "trust",
"type": "string",
"value": "0.9"
},
{
"field": "inferenceprovenance",
"type": "string",
"value": null
},
{
"field": "collectedfromid",
"type": "string",
"value": null
},
{
"field": "collectedfromname",
"type": "string",
"value": null
},
{
"field": "semantics",
"type": "not_used",
"value": "providedBy@@@provided by@@@dnet:datasources_organizations_typologies@@@dnet:datasources_organizations_typologies"
},
{
"field": "provenanceaction",
"type": "not_used",
"value": null
}
]

239
dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/migration/step1/datasources_resultset_entry.json Normal file
View File

@ -0,0 +1,239 @@
[
{
"field": "datasourceid",
"type": "string",
"value": "274269ac6f3b::2579-5449"
},
{
"field": "identities",
"type": "not_used",
"value": [
"274269ac6f3b::2579-5449",
null
]
},
{
"field": "officialname",
"type": "string",
"value": "Jurnal Ilmiah Pendidikan Scholastic"
},
{
"field": "englishname",
"type": "string",
"value": "Jurnal Ilmiah Pendidikan Scholastic"
},
{
"field": "contactemail",
"type": "string",
"value": null
},
{
"field": "openairecompatibility",
"type": "string",
"value": "hostedBy@@@collected from a compatible aggregator@@@dnet:datasourceCompatibilityLevel@@@dnet:datasourceCompatibilityLevel"
},
{
"field": "websiteurl",
"type": "string",
"value": "http://e-journal.sastra-unes.com/index.php/JIPS/index"
},
{
"field": "logourl",
"type": "string",
"value": null
},
{
"field": "accessinfopackage",
"type": "array",
"value": [
null
]
},
{
"field": "latitude",
"type": "double",
"value": 0
},
{
"field": "longitude",
"type": "double",
"value": 0
},
{
"field": "namespaceprefix",
"type": "string",
"value": "ojs_25795449"
},
{
"field": "odnumberofitems",
"type": "int",
"value": null
},
{
"field": "odnumberofitemsdate",
"type": "date",
"value": null
},
{
"field": "subjects",
"type": "array",
"value": null
},
{
"field": "description",
"type": "string",
"value": null
},
{
"field": "odpolicies",
"type": "string",
"value": null
},
{
"field": "odlanguages",
"type": "array",
"value": []
},
{
"field": "odcontenttypes",
"type": "array",
"value": [
"Journal articles"
]
},
{
"field": "inferred",
"type": "boolean",
"value": false
},
{
"field": "deletedbyinference",
"type": "boolean",
"value": false
},
{
"field": "trust",
"type": "string",
"value": "0.9"
},
{
"field": "inferenceprovenance",
"type": "string",
"value": null
},
{
"field": "dateofcollection",
"type": "date",
"value": "2020-01-21"
},
{
"field": "dateofvalidation",
"type": "date",
"value": null
},
{
"field": "releasestartdate",
"type": "date",
"value": null
},
{
"field": "releaseenddate",
"type": "date",
"value": null
},
{
"field": "missionstatementurl",
"type": "string",
"value": null
},
{
"field": "dataprovider",
"type": "boolean",
"value": null
},
{
"field": "serviceprovider",
"type": "boolean",
"value": null
},
{
"field": "databaseaccesstype",
"type": "string",
"value": null
},
{
"field": "datauploadtype",
"type": "string",
"value": null
},
{
"field": "databaseaccessrestriction",
"type": "string",
"value": null
},
{
"field": "datauploadrestriction",
"type": "string",
"value": null
},
{
"field": "versioning",
"type": "boolean",
"value": null
},
{
"field": "citationguidelineurl",
"type": "string",
"value": null
},
{
"field": "qualitymanagementkind",
"type": "string",
"value": null
},
{
"field": "pidsystems",
"type": "string",
"value": null
},
{
"field": "certificates",
"type": "string",
"value": null
},
{
"field": "policies",
"type": "not_used",
"value": []
},
{
"field": "collectedfromid",
"type": "string",
"value": "openaire____::SnVybmFsIEZha3VsdGFzIFNhc3RyYSBVbml2ZXJzaXRhcyBFa2FzYWt0aQ=="
},
{
"field": "collectedfromname",
"type": "string",
"value": "Jurnal Fakultas Sastra Universitas Ekasakti"
},
{
"field": "datasourcetype",
"type": "string",
"value": "pubsrepository::journal@@@Journal@@@dnet:datasource_typologies@@@dnet:datasource_typologies"
},
{
"field": "provenanceaction",
"type": "not_used",
"value": "sysimport:crosswalk:entityregistry@@@sysimport:crosswalk:entityregistry@@@dnet:provenance_actions@@@dnet:provenance_actions"
},
{
"field": "journal",
"type": "string",
"value": "2579-5449@@@2597-6540@@@"
},
{
"field": "collectedfromid",
"type": "string",
"value": "openaire____::SnVybmFsIEZha3VsdGFzIFNhc3RyYSBVbml2ZXJzaXRhcyBFa2FzYWt0aQ=="
}
]

127
dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/migration/step1/organizations_resultset_entry.json Normal file
View File

@ -0,0 +1,127 @@
[
{
"field": "organizationid",
"type": "string",
"value": "openaire____::openaire____::microsoft"
},
{
"field": "legalshortname",
"type": "string",
"value": null
},
{
"field": "legalname",
"type": "string",
"value": "Microsoft Research"
},
{
"field": "websiteurl",
"type": "string",
"value": null
},
{
"field": "logourl",
"type": "string",
"value": null
},
{
"field": "eclegalbody",
"type": "boolean",
"value": false
},
{
"field": "eclegalperson",
"type": "boolean",
"value": false
},
{
"field": "ecnonprofit",
"type": "boolean",
"value": false
},
{
"field": "ecresearchorganization",
"type": "boolean",
"value": false
},
{
"field": "echighereducation",
"type": "boolean",
"value": false
},
{
"field": "ecinternationalorganizationeurinterests",
"type": "boolean",
"value": false
},
{
"field": "ecinternationalorganization",
"type": "boolean",
"value": false
},
{
"field": "ecenterprise",
"type": "boolean",
"value": false
},
{
"field": "ecsmevalidated",
"type": "boolean",
"value": false
},
{
"field": "ecnutscode",
"type": "boolean",
"value": false
},
{
"field": "dateofcollection",
"type": "date",
"value": "2018-10-19"
},
{
"field": "dateoftransformation",
"type": "date",
"value": "2018-10-19"
},
{
"field": "inferred",
"type": "boolean",
"value": false
},
{
"field": "deletedbyinference",
"type": "boolean",
"value": false
},
{
"field": "trust",
"type": "string",
"value": "0.9"
},
{
"field": "inferenceprovenance",
"type": "string",
"value": ""
},
{
"field": "collectedfromid",
"type": "string",
"value": null
},
{
"field": "collectedfromname",
"type": "string",
"value": null
},
{
"field": "country",
"type": "string",
"value": "US@@@US@@@dnet:countries@@@dnet:countries"
},
{
"field": "provenanceaction",
"type": "not_used",
"value": "sysimport:crosswalk:entityregistry@@@sysimport:crosswalk:entityregistry@@@dnet:provenance_actions@@@dnet:provenance_actions"
}
]

72
dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/migration/step1/projectorganization_resultset_entry.json Normal file
View File

@ -0,0 +1,72 @@
[
{
"field": "project",
"type": "string",
"value": "nsf_________::1700003"
},
{
"field": "resporganization",
"type": "string",
"value": "nsf_________::University_of_Notre_Dame"
},
{
"field": "participantnumber",
"type": "not_used",
"value": 1
},
{
"field": "contribution",
"type": "not_used",
"value": null
},
{
"field": "startdate",
"type": "not_used",
"value": null
},
{
"field": "enddate",
"type": "not_used",
"value": null
},
{
"field": "inferred",
"type": "boolean",
"value": false
},
{
"field": "deletedbyinference",
"type": "boolean",
"value": false
},
{
"field": "trust",
"type": "string",
"value": "0.9"
},
{
"field": "inferenceprovenance",
"type": "string",
"value": null
},
{
"field": "collectedfromid",
"type": "string",
"value": "openaire____::nsf"
},
{
"field": "collectedfromname",
"type": "string",
"value": "NSF - National Science Foundation"
},
{
"field": "semantics",
"type": "not_used",
"value": "coordinator@@@coordinator@@@dnet:project_organization_relations@@@dnet:project_organization_relations"
},
{
"field": "provenanceaction",
"type": "not_used",
"value": "sysimport:crosswalk:entityregistry@@@sysimport:crosswalk:entityregistry@@@dnet:provenance_actions@@@dnet:provenance_actions"
}
]

193
dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/migration/step1/projects_resultset_entry.json Normal file
View File

@ -0,0 +1,193 @@
[
{
"field": "projectid",
"type": "string",
"value": "aka_________::100469"
},
{
"field": "code",
"type": "string",
"value": "100469"
},
{
"field": "websiteurl",
"type": "string",
"value": "http://test"
},
{
"field": "acronym",
"type": "string",
"value": "RMCAG"
},
{
"field": "title",
"type": "string",
"value": "Regulation of melanoma cell autonomous growth"
},
{
"field": "startdate",
"type": "date",
"value": null
},
{
"field": "enddate",
"type": "date",
"value": null
},
{
"field": "callidentifier",
"type": "string",
"value": "Tutkijankoulutus ja työskentely ulkomailla/kevät TT"
},
{
"field": "keywords",
"type": "string",
"value": null
},
{
"field": "duration",
"type": "int",
"value": null
},
{
"field": "ecsc39",
"type": "boolean",
"value": null
},
{
"field": "oamandatepublications",
"type": "boolean",
"value": false
},
{
"field": "ecarticle29_3",
"type": "boolean",
"value": null
},
{
"field": "dateofcollection",
"type": "date",
"value": "2019-01-25"
},
{
"field": "dateoftransformation",
"type": "date",
"value": "2019-04-16"
},
{
"field": "inferred",
"type": "boolean",
"value": false
},
{
"field": "deletedbyinference",
"type": "boolean",
"value": false
},
{
"field": "trust",
"type": "string",
"value": "0.9"
},
{
"field": "inferenceprovenance",
"type": "string",
"value": null
},
{
"field": "optional1",
"type": "string",
"value": "9,284 €"
},
{
"field": "optional2",
"type": "string",
"value": null
},
{
"field": "jsonextrainfo",
"type": "string",
"value": "{}"
},
{
"field": "contactfullname",
"type": "string",
"value": null
},
{
"field": "contactfax",
"type": "string",
"value": null
},
{
"field": "contactphone",
"type": "string",
"value": null
},
{
"field": "contactemail",
"type": "string",
"value": null
},
{
"field": "summary",
"type": "string",
"value": null
},
{
"field": "currency",
"type": "string",
"value": null
},
{
"field": "totalcost",
"type": "double",
"value": null
},
{
"field": "fundedamount",
"type": "double",
"value": null
},
{
"field": "collectedfromid",
"type": "string",
"value": "openaire____::aka"
},
{
"field": "collectedfromname",
"type": "string",
"value": "Academy of Finland"
},
{
"field": "contracttype",
"type": "string",
"value": null
},
{
"field": "provenanceaction",
"type": "not_used",
"value": "sysimport:crosswalk:entityregistry@@@Harvested@@@dnet:provenanceActions@@@dnet:provenanceActions"
},
{
"field": "pid",
"type": "not_used",
"value": [
null
]
},
{
"field": "subjects",
"type": "array",
"value": [
null
]
},
{
"field": "fundingtree",
"type": "array",
"value": [
"<fundingtree><funder>\n <id>aka_________::AKA</id>\n <shortname>AKA</shortname>\n <name>Academy of Finland</name>\n <originalname>Academy of Finland</originalname>\n <jurisdiction>FI</jurisdiction>\n </funder></fundingtree>"
]
}
]