pispis
/
dnet-hadoop
forked from antonis.lempesis/dnet-hadoop
1
0
Fork 0
Code Pull Requests Activity

partial implementation

This commit is contained in:
Michele Artini 2020-01-21 14:17:05 +01:00
parent cd114f1c3b
commit f6eccdde33
2 changed files with 230 additions and 84 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

147
dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/migration/AbstractMigrateApplication.java
View File

@ -3,8 +3,13 @@ package eu.dnetlib.dhp.migration;
import java.io.Closeable; import java.io.Closeable;
import java.io.IOException; import java.io.IOException;
import java.net.URI; import java.net.URI;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicInteger;
import java.util.stream.Collectors;
import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.Path;
@ -13,7 +18,17 @@ import org.apache.hadoop.io.SequenceFile;
import org.apache.hadoop.io.Text; import org.apache.hadoop.io.Text;
import org.codehaus.jackson.map.ObjectMapper; import org.codehaus.jackson.map.ObjectMapper;
import eu.dnetlib.dhp.schema.oaf.DataInfo;
import eu.dnetlib.dhp.schema.oaf.ExtraInfo;
import eu.dnetlib.dhp.schema.oaf.Field;
import eu.dnetlib.dhp.schema.oaf.Journal;
import eu.dnetlib.dhp.schema.oaf.KeyValue;
import eu.dnetlib.dhp.schema.oaf.OAIProvenance;
import eu.dnetlib.dhp.schema.oaf.Oaf; import eu.dnetlib.dhp.schema.oaf.Oaf;
import eu.dnetlib.dhp.schema.oaf.OriginDescription;
import eu.dnetlib.dhp.schema.oaf.Qualifier;
import eu.dnetlib.dhp.schema.oaf.StructuredProperty;
import eu.dnetlib.dhp.utils.DHPUtils;
public class AbstractMigrateApplication implements Closeable { public class AbstractMigrateApplication implements Closeable {
@ -58,4 +73,136 @@ public class AbstractMigrateApplication implements Closeable {
writer.close(); writer.close();
} }
public static KeyValue keyValue(final String k, final String v) {
final KeyValue kv = new KeyValue();
kv.setKey(k);
kv.setValue(v);
return kv;
}
public static List<KeyValue> listKeyValues(final String... s) {
if (s.length % 2 > 0) { throw new RuntimeException("Invalid number of parameters (k,v,k,v,....)"); }
final List<KeyValue> list = new ArrayList<>();
for (int i = 0; i < s.length; i += 2) {
list.add(keyValue(s[i], s[i + 1]));
}
return list;
}
public static <T> Field<T> field(final T value, final DataInfo info) {
final Field<T> field = new Field<>();
field.setValue(value);
field.setDataInfo(info);
return field;
}
public static List<Field<String>> listFields(final DataInfo info, final String... values) {
return Arrays.stream(values).map(v -> field(v, info)).collect(Collectors.toList());
}
public static Qualifier qualifier(final String classid, final String classname, final String schemeid, final String schemename) {
final Qualifier q = new Qualifier();
q.setClassid(classid);
q.setClassname(classname);
q.setSchemeid(schemeid);
q.setSchemename(schemename);
return q;
}
public static StructuredProperty structuredProperty(final String value,
final String classid,
final String classname,
final String schemeid,
final String schemename,
final DataInfo dataInfo) {
final StructuredProperty sp = new StructuredProperty();
sp.setValue(value);
sp.setQualifier(qualifier(classid, classname, schemeid, schemename));
sp.setDataInfo(dataInfo);
return sp;
}
public static ExtraInfo extraInfo(final String name, final String value, final String typology, final String provenance, final String trust) {
final ExtraInfo info = new ExtraInfo();
info.setName(name);
info.setValue(value);
info.setTypology(typology);
info.setProvenance(provenance);
info.setTrust(trust);
return info;
}
public static OAIProvenance oaiIProvenance(final String identifier,
final String baseURL,
final String metadataNamespace,
final Boolean altered,
final String datestamp,
final String harvestDate) {
final OriginDescription desc = new OriginDescription();
desc.setIdentifier(identifier);
desc.setBaseURL(baseURL);
desc.setMetadataNamespace(metadataNamespace);
desc.setAltered(altered);
desc.setDatestamp(datestamp);
desc.setHarvestDate(harvestDate);
final OAIProvenance p = new OAIProvenance();
p.setOriginDescription(desc);
return p;
}
public static Journal journal(final String name,
final String issnPrinted,
final String issnOnline,
final String issnLinking,
final String ep,
final String iss,
final String sp,
final String vol,
final String edition,
final String conferenceplace,
final String conferencedate,
final DataInfo dataInfo) {
final Journal j = new Journal();
j.setName(name);
j.setIssnPrinted(issnPrinted);
j.setIssnOnline(issnOnline);
j.setIssnLinking(issnLinking);
j.setEp(ep);
j.setIss(iss);
j.setSp(sp);
j.setVol(vol);
j.setEdition(edition);
j.setConferenceplace(conferenceplace);
j.setConferencedate(conferencedate);
j.setDataInfo(dataInfo);
return j;
}
public static DataInfo dataInfo(final Boolean deletedbyinference,
final String inferenceprovenance,
final Boolean inferred,
final Boolean invisible,
final Qualifier provenanceaction,
final String trust) {
final DataInfo d = new DataInfo();
d.setDeletedbyinference(deletedbyinference);
d.setInferenceprovenance(inferenceprovenance);
d.setInferred(inferred);
d.setInvisible(invisible);
d.setProvenanceaction(provenanceaction);
d.setTrust(trust);
return d;
}
public static String createOpenaireId(final String prefix, final String originalId) {
final String nsPrefix = StringUtils.substringBefore(originalId, "::");
final String rest = StringUtils.substringAfter(originalId, "::");
return String.format("%s|%s::%s", prefix, nsPrefix, DHPUtils.md5(rest));
}
} }

167
dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/migration/MigrateDbEntitiesApplication.java
View File

@ -30,8 +30,8 @@ import eu.dnetlib.dhp.schema.oaf.StructuredProperty;
public class MigrateDbEntitiesApplication extends AbstractMigrateApplication implements Closeable { public class MigrateDbEntitiesApplication extends AbstractMigrateApplication implements Closeable {
private static final Qualifier ENTITYREGISTRY_PROVENANCE_ACTION = MigrationUtils private static final Qualifier ENTITYREGISTRY_PROVENANCE_ACTION =
.qualifier("sysimport:crosswalk:entityregistry", "sysimport:crosswalk:entityregistry", "dnet:provenance_actions", "dnet:provenance_actions"); qualifier("sysimport:crosswalk:entityregistry", "sysimport:crosswalk:entityregistry", "dnet:provenance_actions", "dnet:provenance_actions");
private static final Log log = LogFactory.getLog(MigrateDbEntitiesApplication.class); private static final Log log = LogFactory.getLog(MigrateDbEntitiesApplication.class);
@ -82,9 +82,9 @@ public class MigrateDbEntitiesApplication extends AbstractMigrateApplication imp
final Datasource ds = new Datasource(); final Datasource ds = new Datasource();
ds.setId(MigrationUtils.createOpenaireId("10", rs.getString("datasourceid"))); ds.setId(createOpenaireId("10", rs.getString("datasourceid")));
ds.setOriginalId(Arrays.asList(rs.getString("datasourceid"))); ds.setOriginalId(Arrays.asList(rs.getString("datasourceid")));
ds.setCollectedfrom(MigrationUtils.listKeyValues(rs.getString("collectedfromid"), rs.getString("collectedfromname"))); ds.setCollectedfrom(listKeyValues(rs.getString("collectedfromid"), rs.getString("collectedfromname")));
ds.setPid(null); // List<StructuredProperty> // TODO ds.setPid(null); // List<StructuredProperty> // TODO
ds.setDateofcollection(rs.getDate("dateofcollection").toString()); ds.setDateofcollection(rs.getDate("dateofcollection").toString());
ds.setDateoftransformation(null); // Value not returned by the SQL query ds.setDateoftransformation(null); // Value not returned by the SQL query
@ -92,37 +92,37 @@ public class MigrateDbEntitiesApplication extends AbstractMigrateApplication imp
ds.setOaiprovenance(null); // Values not present in the DB ds.setOaiprovenance(null); // Values not present in the DB
ds.setDatasourcetype(prepareQualifierSplitting(rs.getString("datasourcetype"))); ds.setDatasourcetype(prepareQualifierSplitting(rs.getString("datasourcetype")));
ds.setOpenairecompatibility(prepareQualifierSplitting(rs.getString("openairecompatibility"))); ds.setOpenairecompatibility(prepareQualifierSplitting(rs.getString("openairecompatibility")));
ds.setOfficialname(MigrationUtils.field(rs.getString("officialname"), info)); ds.setOfficialname(field(rs.getString("officialname"), info));
ds.setEnglishname(MigrationUtils.field(rs.getString("englishname"), info)); ds.setEnglishname(field(rs.getString("englishname"), info));
ds.setWebsiteurl(MigrationUtils.field(rs.getString("websiteurl"), info)); ds.setWebsiteurl(field(rs.getString("websiteurl"), info));
ds.setLogourl(MigrationUtils.field(rs.getString("logourl"), info)); ds.setLogourl(field(rs.getString("logourl"), info));
ds.setContactemail(MigrationUtils.field(rs.getString("contactemail"), info)); ds.setContactemail(field(rs.getString("contactemail"), info));
ds.setNamespaceprefix(MigrationUtils.field(rs.getString("namespaceprefix"), info)); ds.setNamespaceprefix(field(rs.getString("namespaceprefix"), info));
ds.setLatitude(MigrationUtils.field(Double.toString(rs.getDouble("latitude")), info)); ds.setLatitude(field(Double.toString(rs.getDouble("latitude")), info));
ds.setLongitude(MigrationUtils.field(Double.toString(rs.getDouble("longitude")), info)); ds.setLongitude(field(Double.toString(rs.getDouble("longitude")), info));
ds.setDateofvalidation(MigrationUtils.field(rs.getDate("dateofvalidation").toString(), info)); ds.setDateofvalidation(field(rs.getDate("dateofvalidation").toString(), info));
ds.setDescription(MigrationUtils.field(rs.getString("description"), info)); ds.setDescription(field(rs.getString("description"), info));
ds.setSubjects(prepareListOfStructProps(rs.getArray("subjects"), info)); ds.setSubjects(prepareListOfStructProps(rs.getArray("subjects"), info));
ds.setOdnumberofitems(MigrationUtils.field(Double.toString(rs.getInt("odnumberofitems")), info)); ds.setOdnumberofitems(field(Double.toString(rs.getInt("odnumberofitems")), info));
ds.setOdnumberofitemsdate(MigrationUtils.field(rs.getDate("odnumberofitemsdate").toString(), info)); ds.setOdnumberofitemsdate(field(rs.getDate("odnumberofitemsdate").toString(), info));
ds.setOdpolicies(MigrationUtils.field(rs.getString("odpolicies"), info)); ds.setOdpolicies(field(rs.getString("odpolicies"), info));
ds.setOdlanguages(prepareListFields(rs.getArray("odlanguages"), info)); ds.setOdlanguages(prepareListFields(rs.getArray("odlanguages"), info));
ds.setOdcontenttypes(prepareListFields(rs.getArray("odcontenttypes"), info)); ds.setOdcontenttypes(prepareListFields(rs.getArray("odcontenttypes"), info));
ds.setAccessinfopackage(prepareListFields(rs.getArray("accessinfopackage"), info)); ds.setAccessinfopackage(prepareListFields(rs.getArray("accessinfopackage"), info));
ds.setReleasestartdate(MigrationUtils.field(rs.getDate("releasestartdate").toString(), info)); ds.setReleasestartdate(field(rs.getDate("releasestartdate").toString(), info));
ds.setReleaseenddate(MigrationUtils.field(rs.getDate("releaseenddate").toString(), info)); ds.setReleaseenddate(field(rs.getDate("releaseenddate").toString(), info));
ds.setMissionstatementurl(MigrationUtils.field(rs.getString("missionstatementurl"), info)); ds.setMissionstatementurl(field(rs.getString("missionstatementurl"), info));
ds.setDataprovider(MigrationUtils.field(rs.getBoolean("dataprovider"), info)); ds.setDataprovider(field(rs.getBoolean("dataprovider"), info));
ds.setServiceprovider(MigrationUtils.field(rs.getBoolean("serviceprovider"), info)); ds.setServiceprovider(field(rs.getBoolean("serviceprovider"), info));
ds.setDatabaseaccesstype(MigrationUtils.field(rs.getString("databaseaccesstype"), info)); ds.setDatabaseaccesstype(field(rs.getString("databaseaccesstype"), info));
ds.setDatauploadtype(MigrationUtils.field(rs.getString("datauploadtype"), info)); ds.setDatauploadtype(field(rs.getString("datauploadtype"), info));
ds.setDatabaseaccessrestriction(MigrationUtils.field(rs.getString("databaseaccessrestriction"), info)); ds.setDatabaseaccessrestriction(field(rs.getString("databaseaccessrestriction"), info));
ds.setDatauploadrestriction(MigrationUtils.field(rs.getString("datauploadrestriction"), info)); ds.setDatauploadrestriction(field(rs.getString("datauploadrestriction"), info));
ds.setVersioning(MigrationUtils.field(rs.getBoolean("versioning"), info)); ds.setVersioning(field(rs.getBoolean("versioning"), info));
ds.setCitationguidelineurl(MigrationUtils.field(rs.getString("citationguidelineurl"), info)); ds.setCitationguidelineurl(field(rs.getString("citationguidelineurl"), info));
ds.setQualitymanagementkind(MigrationUtils.field(rs.getString("qualitymanagementkind"), info)); ds.setQualitymanagementkind(field(rs.getString("qualitymanagementkind"), info));
ds.setPidsystems(MigrationUtils.field(rs.getString("pidsystems"), info)); ds.setPidsystems(field(rs.getString("pidsystems"), info));
ds.setCertificates(MigrationUtils.field(rs.getString("certificates"), info)); ds.setCertificates(field(rs.getString("certificates"), info));
ds.setPolicies(new ArrayList<>()); // The sql query returns an empty array ds.setPolicies(new ArrayList<>()); // The sql query returns an empty array
ds.setJournal(prepareJournal(rs.getString("officialname"), rs.getString("journal"), info)); // Journal ds.setJournal(prepareJournal(rs.getString("officialname"), rs.getString("journal"), info)); // Journal
ds.setDataInfo(info); ds.setDataInfo(info);
@ -189,38 +189,38 @@ public class MigrateDbEntitiesApplication extends AbstractMigrateApplication imp
final Project p = new Project(); final Project p = new Project();
p.setId(MigrationUtils.createOpenaireId("40", rs.getString("projectid"))); p.setId(createOpenaireId("40", rs.getString("projectid")));
p.setOriginalId(Arrays.asList(rs.getString("projectid"))); p.setOriginalId(Arrays.asList(rs.getString("projectid")));
p.setCollectedfrom(MigrationUtils.listKeyValues(rs.getString("collectedfromid"), rs.getString("collectedfromname"))); p.setCollectedfrom(listKeyValues(rs.getString("collectedfromid"), rs.getString("collectedfromname")));
p.setPid(null); // List<StructuredProperty> // TODO p.setPid(null); // List<StructuredProperty> // TODO
p.setDateofcollection(rs.getDate("dateofcollection").toString()); p.setDateofcollection(rs.getDate("dateofcollection").toString());
p.setDateoftransformation(rs.getDate("dateoftransformation").toString()); p.setDateoftransformation(rs.getDate("dateoftransformation").toString());
p.setExtraInfo(null); // List<ExtraInfo> //TODO p.setExtraInfo(null); // List<ExtraInfo> //TODO
p.setOaiprovenance(null); // Values not present in the DB p.setOaiprovenance(null); // Values not present in the DB
p.setWebsiteurl(MigrationUtils.field(rs.getString("websiteurl"), info)); p.setWebsiteurl(field(rs.getString("websiteurl"), info));
p.setCode(MigrationUtils.field(rs.getString("code"), info)); p.setCode(field(rs.getString("code"), info));
p.setAcronym(MigrationUtils.field(rs.getString("acronym"), info)); p.setAcronym(field(rs.getString("acronym"), info));
p.setTitle(MigrationUtils.field(rs.getString("title"), info)); p.setTitle(field(rs.getString("title"), info));
p.setStartdate(MigrationUtils.field(rs.getDate("startdate").toString(), info)); p.setStartdate(field(rs.getDate("startdate").toString(), info));
p.setEnddate(MigrationUtils.field(rs.getDate("enddate").toString(), info)); p.setEnddate(field(rs.getDate("enddate").toString(), info));
p.setCallidentifier(MigrationUtils.field(rs.getString("callidentifier"), info)); p.setCallidentifier(field(rs.getString("callidentifier"), info));
p.setKeywords(MigrationUtils.field(rs.getString("keywords"), info)); p.setKeywords(field(rs.getString("keywords"), info));
p.setDuration(MigrationUtils.field(Integer.toString(rs.getInt("duration")), info)); p.setDuration(field(Integer.toString(rs.getInt("duration")), info));
p.setEcsc39(MigrationUtils.field(Boolean.toString(rs.getBoolean("ecsc39")), info)); p.setEcsc39(field(Boolean.toString(rs.getBoolean("ecsc39")), info));
p.setOamandatepublications(MigrationUtils.field(Boolean.toString(rs.getBoolean("oamandatepublications")), info)); p.setOamandatepublications(field(Boolean.toString(rs.getBoolean("oamandatepublications")), info));
p.setEcarticle29_3(MigrationUtils.field(Boolean.toString(rs.getBoolean("ecarticle29_3")), info)); p.setEcarticle29_3(field(Boolean.toString(rs.getBoolean("ecarticle29_3")), info));
p.setSubjects(prepareListOfStructProps(rs.getArray("subjects"), info)); p.setSubjects(prepareListOfStructProps(rs.getArray("subjects"), info));
p.setFundingtree(prepareListFields(rs.getArray("fundingtree"), info)); p.setFundingtree(prepareListFields(rs.getArray("fundingtree"), info));
p.setContracttype(prepareQualifierSplitting(rs.getString("contracttype"))); p.setContracttype(prepareQualifierSplitting(rs.getString("contracttype")));
p.setOptional1(MigrationUtils.field(rs.getString("optional1"), info)); p.setOptional1(field(rs.getString("optional1"), info));
p.setOptional2(MigrationUtils.field(rs.getString("optional2"), info)); p.setOptional2(field(rs.getString("optional2"), info));
p.setJsonextrainfo(MigrationUtils.field(rs.getString("jsonextrainfo"), info)); p.setJsonextrainfo(field(rs.getString("jsonextrainfo"), info));
p.setContactfullname(MigrationUtils.field(rs.getString("contactfullname"), info)); p.setContactfullname(field(rs.getString("contactfullname"), info));
p.setContactfax(MigrationUtils.field(rs.getString("contactfax"), info)); p.setContactfax(field(rs.getString("contactfax"), info));
p.setContactphone(MigrationUtils.field(rs.getString("contactphone"), info)); p.setContactphone(field(rs.getString("contactphone"), info));
p.setContactemail(MigrationUtils.field(rs.getString("contactemail"), info)); p.setContactemail(field(rs.getString("contactemail"), info));
p.setSummary(MigrationUtils.field(rs.getString("summary"), info)); p.setSummary(field(rs.getString("summary"), info));
p.setCurrency(MigrationUtils.field(rs.getString("currency"), info)); p.setCurrency(field(rs.getString("currency"), info));
p.setTotalcost(new Float(rs.getDouble("totalcost"))); p.setTotalcost(new Float(rs.getDouble("totalcost")));
p.setFundedamount(new Float(rs.getDouble("fundedamount"))); p.setFundedamount(new Float(rs.getDouble("fundedamount")));
p.setDataInfo(info); p.setDataInfo(info);
@ -278,30 +278,29 @@ public class MigrateDbEntitiesApplication extends AbstractMigrateApplication imp
final Organization o = new Organization(); final Organization o = new Organization();
o.setId(MigrationUtils.createOpenaireId("20", rs.getString("organizationid"))); // String id) { o.setId(createOpenaireId("20", rs.getString("organizationid"))); // String id) {
o.setOriginalId(Arrays.asList(rs.getString("organizationid"))); o.setOriginalId(Arrays.asList(rs.getString("organizationid")));
o.setCollectedfrom(MigrationUtils.listKeyValues(rs.getString("collectedfromid"), rs.getString("collectedfromname"))); o.setCollectedfrom(listKeyValues(rs.getString("collectedfromid"), rs.getString("collectedfromname")));
o.setPid(null); // List<StructuredProperty> // TODO o.setPid(null); // List<StructuredProperty> // TODO
o.setDateofcollection(rs.getDate("dateofcollection").toString()); o.setDateofcollection(rs.getDate("dateofcollection").toString());
o.setDateoftransformation(rs.getDate("dateoftransformation").toString()); o.setDateoftransformation(rs.getDate("dateoftransformation").toString());
o.setExtraInfo(null); // List<ExtraInfo> // TODO o.setExtraInfo(null); // List<ExtraInfo> // TODO
o.setOaiprovenance(null); // Values not present in the DB o.setOaiprovenance(null); // Values not present in the DB
o.setLegalshortname(MigrationUtils.field("legalshortname", info)); o.setLegalshortname(field("legalshortname", info));
o.setLegalname(MigrationUtils.field("legalname", info)); o.setLegalname(field("legalname", info));
o.setAlternativeNames(new ArrayList<>()); // Values not returned by the SQL query o.setAlternativeNames(new ArrayList<>()); // Values not returned by the SQL query
o.setWebsiteurl(MigrationUtils.field("websiteurl", info)); o.setWebsiteurl(field("websiteurl", info));
o.setLogourl(MigrationUtils.field("logourl", info)); o.setLogourl(field("logourl", info));
o.setEclegalbody(MigrationUtils.field(Boolean.toString(rs.getBoolean("eclegalbody")), info)); o.setEclegalbody(field(Boolean.toString(rs.getBoolean("eclegalbody")), info));
o.setEclegalperson(MigrationUtils.field(Boolean.toString(rs.getBoolean("eclegalperson")), info)); o.setEclegalperson(field(Boolean.toString(rs.getBoolean("eclegalperson")), info));
o.setEcnonprofit(MigrationUtils.field(Boolean.toString(rs.getBoolean("ecnonprofit")), info)); o.setEcnonprofit(field(Boolean.toString(rs.getBoolean("ecnonprofit")), info));
o.setEcresearchorganization(MigrationUtils.field(Boolean.toString(rs.getBoolean("ecresearchorganization")), info)); o.setEcresearchorganization(field(Boolean.toString(rs.getBoolean("ecresearchorganization")), info));
o.setEchighereducation(MigrationUtils.field(Boolean.toString(rs.getBoolean("echighereducation")), info)); o.setEchighereducation(field(Boolean.toString(rs.getBoolean("echighereducation")), info));
o.setEcinternationalorganizationeurinterests(MigrationUtils o.setEcinternationalorganizationeurinterests(field(Boolean.toString(rs.getBoolean("ecinternationalorganizationeurinterests")), info));
.field(Boolean.toString(rs.getBoolean("ecinternationalorganizationeurinterests")), info)); o.setEcinternationalorganization(field(Boolean.toString(rs.getBoolean("ecinternationalorganization")), info));
o.setEcinternationalorganization(MigrationUtils.field(Boolean.toString(rs.getBoolean("ecinternationalorganization")), info)); o.setEcenterprise(field(Boolean.toString(rs.getBoolean("ecenterprise")), info));
o.setEcenterprise(MigrationUtils.field(Boolean.toString(rs.getBoolean("ecenterprise")), info)); o.setEcsmevalidated(field(Boolean.toString(rs.getBoolean("ecsmevalidated")), info));
o.setEcsmevalidated(MigrationUtils.field(Boolean.toString(rs.getBoolean("ecsmevalidated")), info)); o.setEcnutscode(field(Boolean.toString(rs.getBoolean("ecnutscode")), info));
o.setEcnutscode(MigrationUtils.field(Boolean.toString(rs.getBoolean("ecnutscode")), info));
o.setCountry(prepareQualifierSplitting(rs.getString("country"))); o.setCountry(prepareQualifierSplitting(rs.getString("country")));
o.setDataInfo(info); o.setDataInfo(info);
o.setLastupdatetimestamp(lastUpdateTimestamp); o.setLastupdatetimestamp(lastUpdateTimestamp);
@ -343,9 +342,9 @@ public class MigrateDbEntitiesApplication extends AbstractMigrateApplication imp
try { try {
final DataInfo info = prepareDataInfo(rs); final DataInfo info = prepareDataInfo(rs);
final String orgId = MigrationUtils.createOpenaireId("20", rs.getString("organization")); final String orgId = createOpenaireId("20", rs.getString("organization"));
final String dsId = MigrationUtils.createOpenaireId("10", rs.getString("datasource")); final String dsId = createOpenaireId("10", rs.getString("datasource"));
final List<KeyValue> collectedFrom = MigrationUtils.listKeyValues(rs.getString("collectedfromid"), rs.getString("collectedfromname")); final List<KeyValue> collectedFrom = listKeyValues(rs.getString("collectedfromid"), rs.getString("collectedfromname"));
final Relation r1 = new Relation(); final Relation r1 = new Relation();
r1.setRelType("datasourceOrganization"); r1.setRelType("datasourceOrganization");
@ -391,9 +390,9 @@ public class MigrateDbEntitiesApplication extends AbstractMigrateApplication imp
public void processProjectOrganization(final ResultSet rs) { public void processProjectOrganization(final ResultSet rs) {
try { try {
final DataInfo info = prepareDataInfo(rs); final DataInfo info = prepareDataInfo(rs);
final String orgId = MigrationUtils.createOpenaireId("20", rs.getString("resporganization")); final String orgId = createOpenaireId("20", rs.getString("resporganization"));
final String projectId = MigrationUtils.createOpenaireId("40", rs.getString("project")); final String projectId = createOpenaireId("40", rs.getString("project"));
final List<KeyValue> collectedFrom = MigrationUtils.listKeyValues(rs.getString("collectedfromid"), rs.getString("collectedfromname")); final List<KeyValue> collectedFrom = listKeyValues(rs.getString("collectedfromid"), rs.getString("collectedfromname"));
final Relation r1 = new Relation(); final Relation r1 = new Relation();
r1.setRelType("projectOrganization"); r1.setRelType("projectOrganization");
@ -443,18 +442,18 @@ public class MigrateDbEntitiesApplication extends AbstractMigrateApplication imp
final String inferenceprovenance = rs.getString("inferenceprovenance"); final String inferenceprovenance = rs.getString("inferenceprovenance");
final Boolean inferred = rs.getBoolean("inferred"); final Boolean inferred = rs.getBoolean("inferred");
final String trust = rs.getString("trust"); final String trust = rs.getString("trust");
return MigrationUtils.dataInfo(deletedbyinference, inferenceprovenance, inferred, false, ENTITYREGISTRY_PROVENANCE_ACTION, trust); return dataInfo(deletedbyinference, inferenceprovenance, inferred, false, ENTITYREGISTRY_PROVENANCE_ACTION, trust);
} }
private Qualifier prepareQualifierSplitting(final String s) { private Qualifier prepareQualifierSplitting(final String s) {
if (StringUtils.isBlank(s)) { return null; } if (StringUtils.isBlank(s)) { return null; }
final String[] arr = s.split("@@@"); final String[] arr = s.split("@@@");
return arr.length == 4 ? MigrationUtils.qualifier(arr[0], arr[1], arr[2], arr[3]) : null; return arr.length == 4 ? qualifier(arr[0], arr[1], arr[2], arr[3]) : null;
} }
public static List<Field<String>> prepareListFields(final Array array, final DataInfo info) { private static List<Field<String>> prepareListFields(final Array array, final DataInfo info) {
try { try {
return MigrationUtils.listFields(info, (String[]) array.getArray()); return listFields(info, (String[]) array.getArray());
} catch (final SQLException e) { } catch (final SQLException e) {
throw new RuntimeException("Invalid SQL array", e); throw new RuntimeException("Invalid SQL array", e);
} }
@ -466,7 +465,7 @@ public class MigrateDbEntitiesApplication extends AbstractMigrateApplication imp
if (parts.length == 2) { if (parts.length == 2) {
final String value = parts[0]; final String value = parts[0];
final String[] arr = parts[1].split("@@@"); final String[] arr = parts[1].split("@@@");
if (arr.length == 4) { return MigrationUtils.structuredProperty(value, arr[0], arr[1], arr[2], arr[3], dataInfo); } if (arr.length == 4) { return structuredProperty(value, arr[0], arr[1], arr[2], arr[3], dataInfo); }
} }
return null; return null;
} }
@ -492,8 +491,8 @@ public class MigrateDbEntitiesApplication extends AbstractMigrateApplication imp
final String issn = StringUtils.isNotBlank(arr[0]) ? arr[0] : null; final String issn = StringUtils.isNotBlank(arr[0]) ? arr[0] : null;
final String eissn = StringUtils.isNotBlank(arr[1]) ? arr[1] : null;; final String eissn = StringUtils.isNotBlank(arr[1]) ? arr[1] : null;;
final String lissn = StringUtils.isNotBlank(arr[2]) ? arr[2] : null;; final String lissn = StringUtils.isNotBlank(arr[2]) ? arr[2] : null;;
if (issn != null || eissn != null || lissn != null) { return MigrationUtils if (issn != null || eissn != null
.journal(name, issn, eissn, eissn, null, null, null, null, null, null, null, info); } || lissn != null) { return journal(name, issn, eissn, eissn, null, null, null, null, null, null, null, info); }
} }
} }
return null; return null;