Fix relations in migration

Fix pom.xml in dhp-stats-update
This commit is contained in:
Michele Artini 2020-04-29 12:05:41 +02:00
parent 3616d0f88d
commit a5d7007005
1 changed files with 506 additions and 511 deletions

View File

@ -1,4 +1,3 @@
package eu.dnetlib.dhp.oa.graph.raw; package eu.dnetlib.dhp.oa.graph.raw;
import static eu.dnetlib.dhp.oa.graph.raw.common.OafMapperUtils.asString; import static eu.dnetlib.dhp.oa.graph.raw.common.OafMapperUtils.asString;
@ -11,23 +10,6 @@ import static eu.dnetlib.dhp.oa.graph.raw.common.OafMapperUtils.listKeyValues;
import static eu.dnetlib.dhp.oa.graph.raw.common.OafMapperUtils.qualifier; import static eu.dnetlib.dhp.oa.graph.raw.common.OafMapperUtils.qualifier;
import static eu.dnetlib.dhp.oa.graph.raw.common.OafMapperUtils.structuredProperty; import static eu.dnetlib.dhp.oa.graph.raw.common.OafMapperUtils.structuredProperty;
import java.io.Closeable;
import java.io.IOException;
import java.sql.Array;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Date;
import java.util.List;
import java.util.function.Consumer;
import java.util.function.Function;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import eu.dnetlib.dhp.application.ArgumentApplicationParser; import eu.dnetlib.dhp.application.ArgumentApplicationParser;
import eu.dnetlib.dhp.oa.graph.raw.common.AbstractMigrationApplication; import eu.dnetlib.dhp.oa.graph.raw.common.AbstractMigrationApplication;
import eu.dnetlib.dhp.oa.graph.raw.common.DbClient; import eu.dnetlib.dhp.oa.graph.raw.common.DbClient;
@ -49,6 +31,21 @@ import eu.dnetlib.dhp.schema.oaf.Relation;
import eu.dnetlib.dhp.schema.oaf.Result; import eu.dnetlib.dhp.schema.oaf.Result;
import eu.dnetlib.dhp.schema.oaf.Software; import eu.dnetlib.dhp.schema.oaf.Software;
import eu.dnetlib.dhp.schema.oaf.StructuredProperty; import eu.dnetlib.dhp.schema.oaf.StructuredProperty;
import java.io.Closeable;
import java.io.IOException;
import java.sql.Array;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Date;
import java.util.List;
import java.util.function.Consumer;
import java.util.function.Function;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
public class MigrateDbEntitiesApplication extends AbstractMigrationApplication public class MigrateDbEntitiesApplication extends AbstractMigrationApplication
implements Closeable { implements Closeable {
@ -60,11 +57,10 @@ public class MigrateDbEntitiesApplication extends AbstractMigrationApplication
private final long lastUpdateTimestamp; private final long lastUpdateTimestamp;
public static void main(final String[] args) throws Exception { public static void main(final String[] args) throws Exception {
final ArgumentApplicationParser parser = new ArgumentApplicationParser( final ArgumentApplicationParser parser =
IOUtils new ArgumentApplicationParser(
.toString( IOUtils.toString(
MigrateDbEntitiesApplication.class MigrateDbEntitiesApplication.class.getResourceAsStream(
.getResourceAsStream(
"/eu/dnetlib/dhp/oa/graph/migrate_db_entities_parameters.json"))); "/eu/dnetlib/dhp/oa/graph/migrate_db_entities_parameters.json")));
parser.parseArgument(args); parser.parseArgument(args);
@ -75,10 +71,11 @@ public class MigrateDbEntitiesApplication extends AbstractMigrationApplication
final String hdfsPath = parser.get("hdfsPath"); final String hdfsPath = parser.get("hdfsPath");
final boolean processClaims = parser.get("action") != null && parser.get("action").equalsIgnoreCase("claims"); final boolean processClaims =
parser.get("action") != null && parser.get("action").equalsIgnoreCase("claims");
try (final MigrateDbEntitiesApplication smdbe = new MigrateDbEntitiesApplication(hdfsPath, dbUrl, dbUser, try (final MigrateDbEntitiesApplication smdbe =
dbPassword)) { new MigrateDbEntitiesApplication(hdfsPath, dbUrl, dbUser, dbPassword)) {
if (processClaims) { if (processClaims) {
log.info("Processing claims..."); log.info("Processing claims...");
smdbe.execute("queryClaims.sql", smdbe::processClaims); smdbe.execute("queryClaims.sql", smdbe::processClaims);
@ -118,7 +115,8 @@ public class MigrateDbEntitiesApplication extends AbstractMigrationApplication
public void execute(final String sqlFile, final Function<ResultSet, List<Oaf>> producer) public void execute(final String sqlFile, final Function<ResultSet, List<Oaf>> producer)
throws Exception { throws Exception {
final String sql = IOUtils.toString(getClass().getResourceAsStream("/eu/dnetlib/dhp/oa/graph/sql/" + sqlFile)); final String sql =
IOUtils.toString(getClass().getResourceAsStream("/eu/dnetlib/dhp/oa/graph/sql/" + sqlFile));
final Consumer<ResultSet> consumer = rs -> producer.apply(rs).forEach(oaf -> emitOaf(oaf)); final Consumer<ResultSet> consumer = rs -> producer.apply(rs).forEach(oaf -> emitOaf(oaf));
@ -135,8 +133,7 @@ public class MigrateDbEntitiesApplication extends AbstractMigrationApplication
ds.setId(createOpenaireId(10, rs.getString("datasourceid"), true)); ds.setId(createOpenaireId(10, rs.getString("datasourceid"), true));
ds.setOriginalId(Arrays.asList(rs.getString("datasourceid"))); ds.setOriginalId(Arrays.asList(rs.getString("datasourceid")));
ds ds.setCollectedfrom(
.setCollectedfrom(
listKeyValues( listKeyValues(
createOpenaireId(10, rs.getString("collectedfromid"), true), createOpenaireId(10, rs.getString("collectedfromid"), true),
rs.getString("collectedfromname"))); rs.getString("collectedfromname")));
@ -179,8 +176,7 @@ public class MigrateDbEntitiesApplication extends AbstractMigrationApplication
ds.setPidsystems(field(rs.getString("pidsystems"), info)); ds.setPidsystems(field(rs.getString("pidsystems"), info));
ds.setCertificates(field(rs.getString("certificates"), info)); ds.setCertificates(field(rs.getString("certificates"), info));
ds.setPolicies(new ArrayList<>()); // The sql query returns an empty array ds.setPolicies(new ArrayList<>()); // The sql query returns an empty array
ds ds.setJournal(
.setJournal(
prepareJournal(rs.getString("officialname"), rs.getString("journal"), info)); // Journal prepareJournal(rs.getString("officialname"), rs.getString("journal"), info)); // Journal
ds.setDataInfo(info); ds.setDataInfo(info);
ds.setLastupdatetimestamp(lastUpdateTimestamp); ds.setLastupdatetimestamp(lastUpdateTimestamp);
@ -200,8 +196,7 @@ public class MigrateDbEntitiesApplication extends AbstractMigrationApplication
p.setId(createOpenaireId(40, rs.getString("projectid"), true)); p.setId(createOpenaireId(40, rs.getString("projectid"), true));
p.setOriginalId(Arrays.asList(rs.getString("projectid"))); p.setOriginalId(Arrays.asList(rs.getString("projectid")));
p p.setCollectedfrom(
.setCollectedfrom(
listKeyValues( listKeyValues(
createOpenaireId(10, rs.getString("collectedfromid"), true), createOpenaireId(10, rs.getString("collectedfromid"), true),
rs.getString("collectedfromname"))); rs.getString("collectedfromname")));
@ -220,8 +215,7 @@ public class MigrateDbEntitiesApplication extends AbstractMigrationApplication
p.setKeywords(field(rs.getString("keywords"), info)); p.setKeywords(field(rs.getString("keywords"), info));
p.setDuration(field(Integer.toString(rs.getInt("duration")), info)); p.setDuration(field(Integer.toString(rs.getInt("duration")), info));
p.setEcsc39(field(Boolean.toString(rs.getBoolean("ecsc39")), info)); p.setEcsc39(field(Boolean.toString(rs.getBoolean("ecsc39")), info));
p p.setOamandatepublications(
.setOamandatepublications(
field(Boolean.toString(rs.getBoolean("oamandatepublications")), info)); field(Boolean.toString(rs.getBoolean("oamandatepublications")), info));
p.setEcarticle29_3(field(Boolean.toString(rs.getBoolean("ecarticle29_3")), info)); p.setEcarticle29_3(field(Boolean.toString(rs.getBoolean("ecarticle29_3")), info));
p.setSubjects(prepareListOfStructProps(rs.getArray("subjects"), info)); p.setSubjects(prepareListOfStructProps(rs.getArray("subjects"), info));
@ -258,8 +252,7 @@ public class MigrateDbEntitiesApplication extends AbstractMigrationApplication
o.setId(createOpenaireId(20, rs.getString("organizationid"), true)); o.setId(createOpenaireId(20, rs.getString("organizationid"), true));
o.setOriginalId(Arrays.asList(rs.getString("organizationid"))); o.setOriginalId(Arrays.asList(rs.getString("organizationid")));
o o.setCollectedfrom(
.setCollectedfrom(
listKeyValues( listKeyValues(
createOpenaireId(10, rs.getString("collectedfromid"), true), createOpenaireId(10, rs.getString("collectedfromid"), true),
rs.getString("collectedfromname"))); rs.getString("collectedfromname")));
@ -276,15 +269,12 @@ public class MigrateDbEntitiesApplication extends AbstractMigrationApplication
o.setEclegalbody(field(Boolean.toString(rs.getBoolean("eclegalbody")), info)); o.setEclegalbody(field(Boolean.toString(rs.getBoolean("eclegalbody")), info));
o.setEclegalperson(field(Boolean.toString(rs.getBoolean("eclegalperson")), info)); o.setEclegalperson(field(Boolean.toString(rs.getBoolean("eclegalperson")), info));
o.setEcnonprofit(field(Boolean.toString(rs.getBoolean("ecnonprofit")), info)); o.setEcnonprofit(field(Boolean.toString(rs.getBoolean("ecnonprofit")), info));
o o.setEcresearchorganization(
.setEcresearchorganization(
field(Boolean.toString(rs.getBoolean("ecresearchorganization")), info)); field(Boolean.toString(rs.getBoolean("ecresearchorganization")), info));
o.setEchighereducation(field(Boolean.toString(rs.getBoolean("echighereducation")), info)); o.setEchighereducation(field(Boolean.toString(rs.getBoolean("echighereducation")), info));
o o.setEcinternationalorganizationeurinterests(
.setEcinternationalorganizationeurinterests(
field(Boolean.toString(rs.getBoolean("ecinternationalorganizationeurinterests")), info)); field(Boolean.toString(rs.getBoolean("ecinternationalorganizationeurinterests")), info));
o o.setEcinternationalorganization(
.setEcinternationalorganization(
field(Boolean.toString(rs.getBoolean("ecinternationalorganization")), info)); field(Boolean.toString(rs.getBoolean("ecinternationalorganization")), info));
o.setEcenterprise(field(Boolean.toString(rs.getBoolean("ecenterprise")), info)); o.setEcenterprise(field(Boolean.toString(rs.getBoolean("ecenterprise")), info));
o.setEcsmevalidated(field(Boolean.toString(rs.getBoolean("ecsmevalidated")), info)); o.setEcsmevalidated(field(Boolean.toString(rs.getBoolean("ecsmevalidated")), info));
@ -304,7 +294,8 @@ public class MigrateDbEntitiesApplication extends AbstractMigrationApplication
final DataInfo info = prepareDataInfo(rs); final DataInfo info = prepareDataInfo(rs);
final String orgId = createOpenaireId(20, rs.getString("organization"), true); final String orgId = createOpenaireId(20, rs.getString("organization"), true);
final String dsId = createOpenaireId(10, rs.getString("datasource"), true); final String dsId = createOpenaireId(10, rs.getString("datasource"), true);
final List<KeyValue> collectedFrom = listKeyValues( final List<KeyValue> collectedFrom =
listKeyValues(
createOpenaireId(10, rs.getString("collectedfromid"), true), createOpenaireId(10, rs.getString("collectedfromid"), true),
rs.getString("collectedfromname")); rs.getString("collectedfromname"));
@ -339,14 +330,15 @@ public class MigrateDbEntitiesApplication extends AbstractMigrationApplication
final DataInfo info = prepareDataInfo(rs); final DataInfo info = prepareDataInfo(rs);
final String orgId = createOpenaireId(20, rs.getString("resporganization"), true); final String orgId = createOpenaireId(20, rs.getString("resporganization"), true);
final String projectId = createOpenaireId(40, rs.getString("project"), true); final String projectId = createOpenaireId(40, rs.getString("project"), true);
final List<KeyValue> collectedFrom = listKeyValues( final List<KeyValue> collectedFrom =
listKeyValues(
createOpenaireId(10, rs.getString("collectedfromid"), true), createOpenaireId(10, rs.getString("collectedfromid"), true),
rs.getString("collectedfromname")); rs.getString("collectedfromname"));
final Relation r1 = new Relation(); final Relation r1 = new Relation();
r1.setRelType("projectOrganization"); r1.setRelType("projectOrganization");
r1.setSubRelType("participation"); r1.setSubRelType("participation");
r1.setRelClass("isParticipant"); r1.setRelClass("hasParticipant");
r1.setSource(projectId); r1.setSource(projectId);
r1.setTarget(orgId); r1.setTarget(orgId);
r1.setCollectedfrom(collectedFrom); r1.setCollectedfrom(collectedFrom);
@ -356,7 +348,7 @@ public class MigrateDbEntitiesApplication extends AbstractMigrationApplication
final Relation r2 = new Relation(); final Relation r2 = new Relation();
r2.setRelType("projectOrganization"); r2.setRelType("projectOrganization");
r2.setSubRelType("participation"); r2.setSubRelType("participation");
r2.setRelClass("hasParticipant"); r2.setRelClass("isParticipant");
r2.setSource(orgId); r2.setSource(orgId);
r2.setTarget(projectId); r2.setTarget(projectId);
r2.setCollectedfrom(collectedFrom); r2.setCollectedfrom(collectedFrom);
@ -371,7 +363,8 @@ public class MigrateDbEntitiesApplication extends AbstractMigrationApplication
public List<Oaf> processClaims(final ResultSet rs) { public List<Oaf> processClaims(final ResultSet rs) {
final DataInfo info = dataInfo( final DataInfo info =
dataInfo(
false, false,
null, null,
false, false,
@ -380,8 +373,8 @@ public class MigrateDbEntitiesApplication extends AbstractMigrationApplication
"user:claim", "user:claim", "dnet:provenanceActions", "dnet:provenanceActions"), "user:claim", "user:claim", "dnet:provenanceActions", "dnet:provenanceActions"),
"0.9"); "0.9");
final List<KeyValue> collectedFrom = listKeyValues( final List<KeyValue> collectedFrom =
createOpenaireId(10, "infrastruct_::openaire", true), "OpenAIRE"); listKeyValues(createOpenaireId(10, "infrastruct_::openaire", true), "OpenAIRE");
try { try {
@ -409,8 +402,10 @@ public class MigrateDbEntitiesApplication extends AbstractMigrationApplication
return Arrays.asList(r); return Arrays.asList(r);
} else { } else {
final String sourceId = createOpenaireId(rs.getString("source_type"), rs.getString("source_id"), false); final String sourceId =
final String targetId = createOpenaireId(rs.getString("target_type"), rs.getString("target_id"), false); createOpenaireId(rs.getString("source_type"), rs.getString("source_id"), false);
final String targetId =
createOpenaireId(rs.getString("target_type"), rs.getString("target_id"), false);
final Relation r1 = new Relation(); final Relation r1 = new Relation();
final Relation r2 = new Relation(); final Relation r2 = new Relation();