Merge branch 'master' of https://code-repo.d4science.org/D-Net/dnet-hadoop

dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/AbstractMdRecordToOafMapper.java

@@ -10,6 +10,20 @@ import static eu.dnetlib.dhp.oa.graph.raw.common.OafMapperUtils.oaiIProvenance;
 import static eu.dnetlib.dhp.oa.graph.raw.common.OafMapperUtils.qualifier;
 import static eu.dnetlib.dhp.oa.graph.raw.common.OafMapperUtils.structuredProperty;
 
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.commons.lang3.StringUtils;
+import org.dom4j.Document;
+import org.dom4j.DocumentFactory;
+import org.dom4j.DocumentHelper;
+import org.dom4j.Node;
+
+import eu.dnetlib.dhp.oa.graph.raw.common.MigrationConstants;
 import eu.dnetlib.dhp.schema.oaf.Author;
 import eu.dnetlib.dhp.schema.oaf.DataInfo;
 import eu.dnetlib.dhp.schema.oaf.Dataset;
@@ -27,17 +41,6 @@ import eu.dnetlib.dhp.schema.oaf.Relation;
 import eu.dnetlib.dhp.schema.oaf.Result;
 import eu.dnetlib.dhp.schema.oaf.Software;
 import eu.dnetlib.dhp.schema.oaf.StructuredProperty;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Date;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import org.apache.commons.lang3.StringUtils;
-import org.dom4j.Document;
-import org.dom4j.DocumentFactory;
-import org.dom4j.DocumentHelper;
-import org.dom4j.Node;
 
 public abstract class AbstractMdRecordToOafMapper {
 
@@ -46,25 +49,6 @@ public abstract class AbstractMdRecordToOafMapper {
 	protected static final Qualifier MAIN_TITLE_QUALIFIER =
 			qualifier("main title", "main title", "dnet:dataCite_title", "dnet:dataCite_title");
 
-    protected static final Qualifier PUBLICATION_RESULTTYPE_QUALIFIER =
-            qualifier(
-                    "publication",
-                    "publication",
-                    "dnet:result_typologies",
-                    "dnet:result_typologies");
-    protected static final Qualifier DATASET_RESULTTYPE_QUALIFIER =
-            qualifier("dataset", "dataset", "dnet:result_typologies", "dnet:result_typologies");
-    protected static final Qualifier SOFTWARE_RESULTTYPE_QUALIFIER =
-            qualifier("software", "software", "dnet:result_typologies", "dnet:result_typologies");
-    protected static final Qualifier OTHER_RESULTTYPE_QUALIFIER =
-            qualifier("other", "other", "dnet:result_typologies", "dnet:result_typologies");
-    protected static final Qualifier REPOSITORY_QUALIFIER =
-            qualifier(
-                    "sysimport:crosswalk:repository",
-                    "sysimport:crosswalk:repository",
-                    "dnet:provenanceActions",
-                    "dnet:provenanceActions");
-
 	protected AbstractMdRecordToOafMapper(final Map<String, String> code2name) {
 		this.code2name = code2name;
 	}
@@ -82,22 +66,15 @@ public abstract class AbstractMdRecordToOafMapper {
 			DocumentFactory.getInstance().setXPathNamespaceURIs(nsContext);
 
 			final Document doc =
-                    DocumentHelper.parseText(
+					DocumentHelper.parseText(xml.replaceAll("http://datacite.org/schema/kernel-4", "http://datacite.org/schema/kernel-3"));
-                            xml.replaceAll(
-                                    "http://datacite.org/schema/kernel-4",
-                                    "http://datacite.org/schema/kernel-3"));
 
 			final String type = doc.valueOf("//dr:CobjCategory/@type");
 			final KeyValue collectedFrom =
-                    keyValue(
+					keyValue(createOpenaireId(10, doc.valueOf("//oaf:collectedFrom/@id"), true), doc.valueOf("//oaf:collectedFrom/@name"));
-                            createOpenaireId(10, doc.valueOf("//oaf:collectedFrom/@id"), true),
-                            doc.valueOf("//oaf:collectedFrom/@name"));
 			final KeyValue hostedBy =
 					StringUtils.isBlank(doc.valueOf("//oaf:hostedBy/@id"))
 							? collectedFrom
-                            : keyValue(
+							: keyValue(createOpenaireId(10, doc.valueOf("//oaf:hostedBy/@id"), true), doc.valueOf("//oaf:hostedBy/@name"));
-                                    createOpenaireId(10, doc.valueOf("//oaf:hostedBy/@id"), true),
-                                    doc.valueOf("//oaf:hostedBy/@name"));
 
 			final DataInfo info = prepareDataInfo(doc);
 			final long lastUpdateTimestamp = new Date().getTime();
@@ -123,14 +100,14 @@ public abstract class AbstractMdRecordToOafMapper {
 		case "publication":
 			final Publication p = new Publication();
 			populateResultFields(p, doc, collectedFrom, hostedBy, info, lastUpdateTimestamp);
-                p.setResulttype(PUBLICATION_RESULTTYPE_QUALIFIER);
+			p.setResulttype(MigrationConstants.PUBLICATION_RESULTTYPE_QUALIFIER);
 			p.setJournal(prepareJournal(doc, info));
 			oafs.add(p);
 			break;
 		case "dataset":
 			final Dataset d = new Dataset();
 			populateResultFields(d, doc, collectedFrom, hostedBy, info, lastUpdateTimestamp);
-                d.setResulttype(DATASET_RESULTTYPE_QUALIFIER);
+			d.setResulttype(MigrationConstants.DATASET_RESULTTYPE_QUALIFIER);
 			d.setStoragedate(prepareDatasetStorageDate(doc, info));
 			d.setDevice(prepareDatasetDevice(doc, info));
 			d.setSize(prepareDatasetSize(doc, info));
@@ -143,7 +120,7 @@ public abstract class AbstractMdRecordToOafMapper {
 		case "software":
 			final Software s = new Software();
 			populateResultFields(s, doc, collectedFrom, hostedBy, info, lastUpdateTimestamp);
-                s.setResulttype(SOFTWARE_RESULTTYPE_QUALIFIER);
+			s.setResulttype(MigrationConstants.SOFTWARE_RESULTTYPE_QUALIFIER);
 			s.setDocumentationUrl(prepareSoftwareDocumentationUrls(doc, info));
 			s.setLicense(prepareSoftwareLicenses(doc, info));
 			s.setCodeRepositoryUrl(prepareSoftwareCodeRepositoryUrl(doc, info));
@@ -154,7 +131,7 @@ public abstract class AbstractMdRecordToOafMapper {
 		default:
 			final OtherResearchProduct o = new OtherResearchProduct();
 			populateResultFields(o, doc, collectedFrom, hostedBy, info, lastUpdateTimestamp);
-                o.setResulttype(OTHER_RESULTTYPE_QUALIFIER);
+			o.setResulttype(MigrationConstants.OTHER_RESULTTYPE_QUALIFIER);
 			o.setContactperson(prepareOtherResearchProductContactPersons(doc, info));
 			o.setContactgroup(prepareOtherResearchProductContactGroups(doc, info));
 			o.setTool(prepareOtherResearchProductTools(doc, info));
@@ -227,14 +204,7 @@ public abstract class AbstractMdRecordToOafMapper {
 		r.setId(createOpenaireId(50, doc.valueOf("//dri:objIdentifier"), false));
 		r.setOriginalId(Arrays.asList(doc.valueOf("//dri:objIdentifier")));
 		r.setCollectedfrom(Arrays.asList(collectedFrom));
-        r.setPid(
+		r.setPid(prepareListStructProps(doc, "//oaf:identifier", "@identifierType", "dnet:pid_types", "dnet:pid_types", info));
-                prepareListStructProps(
-                        doc,
-                        "//oaf:identifier",
-                        "@identifierType",
-                        "dnet:pid_types",
-                        "dnet:pid_types",
-                        info));
 		r.setDateofcollection(doc.valueOf("//dr:dateOfCollection"));
 		r.setDateoftransformation(doc.valueOf("//dr:dateOfTransformation"));
 		r.setExtraInfo(new ArrayList<>()); // NOT PRESENT IN MDSTORES
@@ -263,7 +233,10 @@ public abstract class AbstractMdRecordToOafMapper {
 	protected abstract Qualifier prepareResourceType(Document doc, DataInfo info);
 
 	protected abstract List<Instance> prepareInstances(
-            Document doc, DataInfo info, KeyValue collectedfrom, KeyValue hostedby);
+			Document doc,
+			DataInfo info,
+			KeyValue collectedfrom,
+			KeyValue hostedby);
 
 	protected abstract List<Field<String>> prepareSources(Document doc, DataInfo info);
 
@@ -288,28 +261,34 @@ public abstract class AbstractMdRecordToOafMapper {
 	protected abstract List<Author> prepareAuthors(Document doc, DataInfo info);
 
 	protected abstract List<Field<String>> prepareOtherResearchProductTools(
-            Document doc, DataInfo info);
+			Document doc,
+			DataInfo info);
 
 	protected abstract List<Field<String>> prepareOtherResearchProductContactGroups(
-            Document doc, DataInfo info);
+			Document doc,
+			DataInfo info);
 
 	protected abstract List<Field<String>> prepareOtherResearchProductContactPersons(
-            Document doc, DataInfo info);
+			Document doc,
+			DataInfo info);
 
 	protected abstract Qualifier prepareSoftwareProgrammingLanguage(Document doc, DataInfo info);
 
 	protected abstract Field<String> prepareSoftwareCodeRepositoryUrl(Document doc, DataInfo info);
 
 	protected abstract List<StructuredProperty> prepareSoftwareLicenses(
-            Document doc, DataInfo info);
+			Document doc,
+			DataInfo info);
 
 	protected abstract List<Field<String>> prepareSoftwareDocumentationUrls(
-            Document doc, DataInfo info);
+			Document doc,
+			DataInfo info);
 
 	protected abstract List<GeoLocation> prepareDatasetGeoLocations(Document doc, DataInfo info);
 
 	protected abstract Field<String> prepareDatasetMetadataVersionNumber(
-            Document doc, DataInfo info);
+			Document doc,
+			DataInfo info);
 
 	protected abstract Field<String> prepareDatasetLastMetadataUpdate(Document doc, DataInfo info);
 
@@ -333,27 +312,16 @@ public abstract class AbstractMdRecordToOafMapper {
 			final String sp = n.valueOf("@sp");
 			final String vol = n.valueOf("@vol");
 			final String edition = n.valueOf("@edition");
-            if (StringUtils.isNotBlank(name)) {
+			if (StringUtils.isNotBlank(name)) { return journal(name, issnPrinted, issnOnline, issnLinking, ep, iss, sp, vol, edition, null, null, info); }
-                return journal(
-                        name,
-                        issnPrinted,
-                        issnOnline,
-                        issnLinking,
-                        ep,
-                        iss,
-                        sp,
-                        vol,
-                        edition,
-                        null,
-                        null,
-                        info);
-            }
 		}
 		return null;
 	}
 
 	protected Qualifier prepareQualifier(
-            final Node node, final String xpath, final String schemeId, final String schemeName) {
+			final Node node,
+			final String xpath,
+			final String schemeId,
+			final String schemeName) {
 		final String classId = node.valueOf(xpath);
 		final String className = code2name.get(classId);
 		return qualifier(classId, className, schemeId, schemeName);
@@ -371,15 +339,16 @@ public abstract class AbstractMdRecordToOafMapper {
 			final Node n = (Node) o;
 			final String classId = n.valueOf(xpathClassId);
 			final String className = code2name.get(classId);
-            res.add(
+			res.add(structuredProperty(n.getText(), classId, className, schemeId, schemeName, info));
-                    structuredProperty(
-                            n.getText(), classId, className, schemeId, schemeName, info));
 		}
 		return res;
 	}
 
 	protected List<StructuredProperty> prepareListStructProps(
-            final Node node, final String xpath, final Qualifier qualifier, final DataInfo info) {
+			final Node node,
+			final String xpath,
+			final Qualifier qualifier,
+			final DataInfo info) {
 		final List<StructuredProperty> res = new ArrayList<>();
 		for (final Object o : node.selectNodes(xpath)) {
 			final Node n = (Node) o;
@@ -389,52 +358,37 @@ public abstract class AbstractMdRecordToOafMapper {
 	}
 
 	protected List<StructuredProperty> prepareListStructProps(
-            final Node node, final String xpath, final DataInfo info) {
+			final Node node,
+			final String xpath,
+			final DataInfo info) {
 		final List<StructuredProperty> res = new ArrayList<>();
 		for (final Object o : node.selectNodes(xpath)) {
 			final Node n = (Node) o;
-            res.add(
+			res.add(structuredProperty(n.getText(), n.valueOf("@classid"), n.valueOf("@classname"), n.valueOf("@schemeid"), n.valueOf("@schemename"), info));
-                    structuredProperty(
-                            n.getText(),
-                            n.valueOf("@classid"),
-                            n.valueOf("@classname"),
-                            n.valueOf("@schemeid"),
-                            n.valueOf("@schemename"),
-                            info));
 		}
 		return res;
 	}
 
 	protected OAIProvenance prepareOAIprovenance(final Document doc) {
 		final Node n =
-                doc.selectSingleNode(
+				doc.selectSingleNode("//*[local-name()='provenance']/*[local-name()='originDescription']");
-                        "//*[local-name()='provenance']/*[local-name()='originDescription']");
 
-        if (n == null) {
+		if (n == null) { return null; }
-            return null;
-        }
 
 		final String identifier = n.valueOf("./*[local-name()='identifier']");
-        final String baseURL = n.valueOf("./*[local-name()='baseURL']");
+		final String baseURL = n.valueOf("./*[local-name()='baseURL']");;
-        ;
+		final String metadataNamespace = n.valueOf("./*[local-name()='metadataNamespace']");;
-        final String metadataNamespace = n.valueOf("./*[local-name()='metadataNamespace']");
-        ;
 		final boolean altered = n.valueOf("@altered").equalsIgnoreCase("true");
-        final String datestamp = n.valueOf("./*[local-name()='datestamp']");
+		final String datestamp = n.valueOf("./*[local-name()='datestamp']");;
-        ;
+		final String harvestDate = n.valueOf("@harvestDate");;
-        final String harvestDate = n.valueOf("@harvestDate");
-        ;
 
-        return oaiIProvenance(
+		return oaiIProvenance(identifier, baseURL, metadataNamespace, altered, datestamp, harvestDate);
-                identifier, baseURL, metadataNamespace, altered, datestamp, harvestDate);
 	}
 
 	protected DataInfo prepareDataInfo(final Document doc) {
 		final Node n = doc.selectSingleNode("//oaf:datainfo");
 
-        if (n == null) {
+		if (n == null) { return dataInfo(false, null, false, false, MigrationConstants.REPOSITORY_PROVENANCE_ACTIONS, "0.9"); }
-            return dataInfo(false, null, false, false, REPOSITORY_QUALIFIER, "0.9");
-        }
 
 		final String paClassId = n.valueOf("./oaf:provenanceaction/@classid");
 		final String paClassName = n.valueOf("./oaf:provenanceaction/@classname");
@@ -447,13 +401,7 @@ public abstract class AbstractMdRecordToOafMapper {
 		final Boolean inferred = Boolean.parseBoolean(n.valueOf("./oaf:inferred"));
 		final String trust = n.valueOf("./oaf:trust");
 
-        return dataInfo(
+		return dataInfo(deletedbyinference, inferenceprovenance, inferred, false, qualifier(paClassId, paClassName, paSchemeId, paSchemeName), trust);
-                deletedbyinference,
-                inferenceprovenance,
-                inferred,
-                false,
-                qualifier(paClassId, paClassName, paSchemeId, paSchemeName),
-                trust);
 	}
 
 	protected Field<String> prepareField(final Node node, final String xpath, final DataInfo info) {
@@ -461,7 +409,9 @@ public abstract class AbstractMdRecordToOafMapper {
 	}
 
 	protected List<Field<String>> prepareListFields(
-            final Node node, final String xpath, final DataInfo info) {
+			final Node node,
+			final String xpath,
+			final DataInfo info) {
 		return listFields(info, prepareListString(node, xpath));
 	}
 

dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/MigrateDbEntitiesApplication.java

@@ -10,9 +10,27 @@ import static eu.dnetlib.dhp.oa.graph.raw.common.OafMapperUtils.listKeyValues;
 import static eu.dnetlib.dhp.oa.graph.raw.common.OafMapperUtils.qualifier;
 import static eu.dnetlib.dhp.oa.graph.raw.common.OafMapperUtils.structuredProperty;
 
+import java.io.Closeable;
+import java.io.IOException;
+import java.sql.Array;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Date;
+import java.util.List;
+import java.util.function.Consumer;
+import java.util.function.Function;
+
+import org.apache.commons.io.IOUtils;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
 import eu.dnetlib.dhp.application.ArgumentApplicationParser;
 import eu.dnetlib.dhp.oa.graph.raw.common.AbstractMigrationApplication;
 import eu.dnetlib.dhp.oa.graph.raw.common.DbClient;
+import eu.dnetlib.dhp.oa.graph.raw.common.MigrationConstants;
 import eu.dnetlib.dhp.schema.oaf.Context;
 import eu.dnetlib.dhp.schema.oaf.DataInfo;
 import eu.dnetlib.dhp.schema.oaf.Dataset;
@@ -30,32 +48,10 @@ import eu.dnetlib.dhp.schema.oaf.Relation;
 import eu.dnetlib.dhp.schema.oaf.Result;
 import eu.dnetlib.dhp.schema.oaf.Software;
 import eu.dnetlib.dhp.schema.oaf.StructuredProperty;
-import java.io.Closeable;
-import java.io.IOException;
-import java.sql.Array;
-import java.sql.ResultSet;
-import java.sql.SQLException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Date;
-import java.util.List;
-import java.util.function.Consumer;
-import java.util.function.Function;
-import org.apache.commons.io.IOUtils;
-import org.apache.commons.lang3.StringUtils;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
 
 public class MigrateDbEntitiesApplication extends AbstractMigrationApplication
 		implements Closeable {
 
-    private static final Qualifier ENTITYREGISTRY_PROVENANCE_ACTION =
-            qualifier(
-                    "sysimport:crosswalk:entityregistry",
-                    "sysimport:crosswalk:entityregistry",
-                    "dnet:provenance_actions",
-                    "dnet:provenance_actions");
-
 	private static final Log log = LogFactory.getLog(MigrateDbEntitiesApplication.class);
 
 	private final DbClient dbClient;
@@ -65,9 +61,8 @@ public class MigrateDbEntitiesApplication extends AbstractMigrationApplication
 	public static void main(final String[] args) throws Exception {
 		final ArgumentApplicationParser parser =
 				new ArgumentApplicationParser(
-                        IOUtils.toString(
+						IOUtils.toString(MigrateDbEntitiesApplication.class
-                                MigrateDbEntitiesApplication.class.getResourceAsStream(
+								.getResourceAsStream("/eu/dnetlib/dhp/oa/graph/migrate_db_entities_parameters.json")));
-                                        "/eu/dnetlib/dhp/oa/graph/migrate_db_entities_parameters.json")));
 
 		parser.parseArgument(args);
 
@@ -96,8 +91,7 @@ public class MigrateDbEntitiesApplication extends AbstractMigrationApplication
 				smdbe.execute("queryOrganizations.sql", smdbe::processOrganization);
 
 				log.info("Processing relations ds <-> orgs ...");
-                smdbe.execute(
+				smdbe.execute("queryDatasourceOrganization.sql", smdbe::processDatasourceOrganization);
-                        "queryDatasourceOrganization.sql", smdbe::processDatasourceOrganization);
 
 				log.info("Processing projects <-> orgs ...");
 				smdbe.execute("queryProjectOrganization.sql", smdbe::processProjectOrganization);
@@ -123,8 +117,7 @@ public class MigrateDbEntitiesApplication extends AbstractMigrationApplication
 	public void execute(final String sqlFile, final Function<ResultSet, List<Oaf>> producer)
 			throws Exception {
 		final String sql =
-                IOUtils.toString(
+				IOUtils.toString(getClass().getResourceAsStream("/eu/dnetlib/dhp/oa/graph/sql/" + sqlFile));
-                        getClass().getResourceAsStream("/eu/dnetlib/dhp/oa/graph/sql/" + sqlFile));
 
 		final Consumer<ResultSet> consumer = rs -> producer.apply(rs).forEach(oaf -> emitOaf(oaf));
 
@@ -141,18 +134,14 @@ public class MigrateDbEntitiesApplication extends AbstractMigrationApplication
 
 			ds.setId(createOpenaireId(10, rs.getString("datasourceid"), true));
 			ds.setOriginalId(Arrays.asList(rs.getString("datasourceid")));
-            ds.setCollectedfrom(
+			ds.setCollectedfrom(listKeyValues(createOpenaireId(10, rs.getString("collectedfromid"), true), rs.getString("collectedfromname")));
-                    listKeyValues(
-                            createOpenaireId(10, rs.getString("collectedfromid"), true),
-                            rs.getString("collectedfromname")));
 			ds.setPid(new ArrayList<>());
 			ds.setDateofcollection(asString(rs.getDate("dateofcollection")));
 			ds.setDateoftransformation(null); // Value not returned by the SQL query
 			ds.setExtraInfo(new ArrayList<>()); // Values not present in the DB
 			ds.setOaiprovenance(null); // Values not present in the DB
 			ds.setDatasourcetype(prepareQualifierSplitting(rs.getString("datasourcetype")));
-            ds.setOpenairecompatibility(
+			ds.setOpenairecompatibility(prepareQualifierSplitting(rs.getString("openairecompatibility")));
-                    prepareQualifierSplitting(rs.getString("openairecompatibility")));
 			ds.setOfficialname(field(rs.getString("officialname"), info));
 			ds.setEnglishname(field(rs.getString("englishname"), info));
 			ds.setWebsiteurl(field(rs.getString("websiteurl"), info));
@@ -185,11 +174,7 @@ public class MigrateDbEntitiesApplication extends AbstractMigrationApplication
 			ds.setPidsystems(field(rs.getString("pidsystems"), info));
 			ds.setCertificates(field(rs.getString("certificates"), info));
 			ds.setPolicies(new ArrayList<>()); // The sql query returns an empty array
-            ds.setJournal(
+			ds.setJournal(prepareJournal(rs.getString("officialname"), rs.getString("journal"), info)); // Journal
-                    prepareJournal(
-                            rs.getString("officialname"),
-                            rs.getString("journal"),
-                            info)); // Journal
 			ds.setDataInfo(info);
 			ds.setLastupdatetimestamp(lastUpdateTimestamp);
 
@@ -208,10 +193,7 @@ public class MigrateDbEntitiesApplication extends AbstractMigrationApplication
 
 			p.setId(createOpenaireId(40, rs.getString("projectid"), true));
 			p.setOriginalId(Arrays.asList(rs.getString("projectid")));
-            p.setCollectedfrom(
+			p.setCollectedfrom(listKeyValues(createOpenaireId(10, rs.getString("collectedfromid"), true), rs.getString("collectedfromname")));
-                    listKeyValues(
-                            createOpenaireId(10, rs.getString("collectedfromid"), true),
-                            rs.getString("collectedfromname")));
 			p.setPid(new ArrayList<>());
 			p.setDateofcollection(asString(rs.getDate("dateofcollection")));
 			p.setDateoftransformation(asString(rs.getDate("dateoftransformation")));
@@ -227,8 +209,7 @@ public class MigrateDbEntitiesApplication extends AbstractMigrationApplication
 			p.setKeywords(field(rs.getString("keywords"), info));
 			p.setDuration(field(Integer.toString(rs.getInt("duration")), info));
 			p.setEcsc39(field(Boolean.toString(rs.getBoolean("ecsc39")), info));
-            p.setOamandatepublications(
+			p.setOamandatepublications(field(Boolean.toString(rs.getBoolean("oamandatepublications")), info));
-                    field(Boolean.toString(rs.getBoolean("oamandatepublications")), info));
 			p.setEcarticle29_3(field(Boolean.toString(rs.getBoolean("ecarticle29_3")), info));
 			p.setSubjects(prepareListOfStructProps(rs.getArray("subjects"), info));
 			p.setFundingtree(prepareListFields(rs.getArray("fundingtree"), info));
@@ -264,10 +245,7 @@ public class MigrateDbEntitiesApplication extends AbstractMigrationApplication
 
 			o.setId(createOpenaireId(20, rs.getString("organizationid"), true));
 			o.setOriginalId(Arrays.asList(rs.getString("organizationid")));
-            o.setCollectedfrom(
+			o.setCollectedfrom(listKeyValues(createOpenaireId(10, rs.getString("collectedfromid"), true), rs.getString("collectedfromname")));
-                    listKeyValues(
-                            createOpenaireId(10, rs.getString("collectedfromid"), true),
-                            rs.getString("collectedfromname")));
 			o.setPid(new ArrayList<>());
 			o.setDateofcollection(asString(rs.getDate("dateofcollection")));
 			o.setDateoftransformation(asString(rs.getDate("dateoftransformation")));
@@ -281,17 +259,10 @@ public class MigrateDbEntitiesApplication extends AbstractMigrationApplication
 			o.setEclegalbody(field(Boolean.toString(rs.getBoolean("eclegalbody")), info));
 			o.setEclegalperson(field(Boolean.toString(rs.getBoolean("eclegalperson")), info));
 			o.setEcnonprofit(field(Boolean.toString(rs.getBoolean("ecnonprofit")), info));
-            o.setEcresearchorganization(
+			o.setEcresearchorganization(field(Boolean.toString(rs.getBoolean("ecresearchorganization")), info));
-                    field(Boolean.toString(rs.getBoolean("ecresearchorganization")), info));
+			o.setEchighereducation(field(Boolean.toString(rs.getBoolean("echighereducation")), info));
-            o.setEchighereducation(
+			o.setEcinternationalorganizationeurinterests(field(Boolean.toString(rs.getBoolean("ecinternationalorganizationeurinterests")), info));
-                    field(Boolean.toString(rs.getBoolean("echighereducation")), info));
+			o.setEcinternationalorganization(field(Boolean.toString(rs.getBoolean("ecinternationalorganization")), info));
-            o.setEcinternationalorganizationeurinterests(
-                    field(
-                            Boolean.toString(
-                                    rs.getBoolean("ecinternationalorganizationeurinterests")),
-                            info));
-            o.setEcinternationalorganization(
-                    field(Boolean.toString(rs.getBoolean("ecinternationalorganization")), info));
 			o.setEcenterprise(field(Boolean.toString(rs.getBoolean("ecenterprise")), info));
 			o.setEcsmevalidated(field(Boolean.toString(rs.getBoolean("ecsmevalidated")), info));
 			o.setEcnutscode(field(Boolean.toString(rs.getBoolean("ecnutscode")), info));
@@ -311,9 +282,7 @@ public class MigrateDbEntitiesApplication extends AbstractMigrationApplication
 			final String orgId = createOpenaireId(20, rs.getString("organization"), true);
 			final String dsId = createOpenaireId(10, rs.getString("datasource"), true);
 			final List<KeyValue> collectedFrom =
-                    listKeyValues(
+					listKeyValues(createOpenaireId(10, rs.getString("collectedfromid"), true), rs.getString("collectedfromname"));
-                            createOpenaireId(10, rs.getString("collectedfromid"), true),
-                            rs.getString("collectedfromname"));
 
 			final Relation r1 = new Relation();
 			r1.setRelType("datasourceOrganization");
@@ -347,9 +316,7 @@ public class MigrateDbEntitiesApplication extends AbstractMigrationApplication
 			final String orgId = createOpenaireId(20, rs.getString("resporganization"), true);
 			final String projectId = createOpenaireId(40, rs.getString("project"), true);
 			final List<KeyValue> collectedFrom =
-                    listKeyValues(
+					listKeyValues(createOpenaireId(10, rs.getString("collectedfromid"), true), rs.getString("collectedfromname"));
-                            createOpenaireId(10, rs.getString("collectedfromid"), true),
-                            rs.getString("collectedfromname"));
 
 			final Relation r1 = new Relation();
 			r1.setRelType("projectOrganization");
@@ -380,17 +347,7 @@ public class MigrateDbEntitiesApplication extends AbstractMigrationApplication
 	public List<Oaf> processClaims(final ResultSet rs) {
 
 		final DataInfo info =
-                dataInfo(
+				dataInfo(false, null, false, false, qualifier("user:claim", "user:claim", "dnet:provenanceActions", "dnet:provenanceActions"), "0.9");
-                        false,
-                        null,
-                        false,
-                        false,
-                        qualifier(
-                                "user:claim",
-                                "user:claim",
-                                "dnet:provenanceActions",
-                                "dnet:provenanceActions"),
-                        "0.9");
 
 		final List<KeyValue> collectedFrom =
 				listKeyValues(createOpenaireId(10, "infrastruct_::openaire", true), "OpenAIRE");
@@ -402,12 +359,16 @@ public class MigrateDbEntitiesApplication extends AbstractMigrationApplication
 
 				if (rs.getString("target_type").equals("dataset")) {
 					r = new Dataset();
+					r.setResulttype(MigrationConstants.DATASET_RESULTTYPE_QUALIFIER);
 				} else if (rs.getString("target_type").equals("software")) {
 					r = new Software();
+					r.setResulttype(MigrationConstants.SOFTWARE_RESULTTYPE_QUALIFIER);
 				} else if (rs.getString("target_type").equals("other")) {
 					r = new OtherResearchProduct();
+					r.setResulttype(MigrationConstants.OTHER_RESULTTYPE_QUALIFIER);
 				} else {
 					r = new Publication();
+					r.setResulttype(MigrationConstants.PUBLICATION_RESULTTYPE_QUALIFIER);
 				}
 				r.setId(createOpenaireId(50, rs.getString("target_id"), false));
 				r.setLastupdatetimestamp(lastUpdateTimestamp);
@@ -418,11 +379,9 @@ public class MigrateDbEntitiesApplication extends AbstractMigrationApplication
 				return Arrays.asList(r);
 			} else {
 				final String sourceId =
-                        createOpenaireId(
+						createOpenaireId(rs.getString("source_type"), rs.getString("source_id"), false);
-                                rs.getString("source_type"), rs.getString("source_id"), false);
 				final String targetId =
-                        createOpenaireId(
+						createOpenaireId(rs.getString("target_type"), rs.getString("target_id"), false);
-                                rs.getString("target_type"), rs.getString("target_id"), false);
 
 				final Relation r1 = new Relation();
 				final Relation r2 = new Relation();
@@ -479,19 +438,11 @@ public class MigrateDbEntitiesApplication extends AbstractMigrationApplication
 		final String inferenceprovenance = rs.getString("inferenceprovenance");
 		final Boolean inferred = rs.getBoolean("inferred");
 		final String trust = rs.getString("trust");
-        return dataInfo(
+		return dataInfo(deletedbyinference, inferenceprovenance, inferred, false, MigrationConstants.ENTITYREGISTRY_PROVENANCE_ACTION, trust);
-                deletedbyinference,
-                inferenceprovenance,
-                inferred,
-                false,
-                ENTITYREGISTRY_PROVENANCE_ACTION,
-                trust);
 	}
 
 	private Qualifier prepareQualifierSplitting(final String s) {
-        if (StringUtils.isBlank(s)) {
+		if (StringUtils.isBlank(s)) { return null; }
-            return null;
-        }
 		final String[] arr = s.split("@@@");
 		return arr.length == 4 ? qualifier(arr[0], arr[1], arr[2], arr[3]) : null;
 	}
@@ -507,22 +458,19 @@ public class MigrateDbEntitiesApplication extends AbstractMigrationApplication
 	}
 
 	private StructuredProperty prepareStructProp(final String s, final DataInfo dataInfo) {
-        if (StringUtils.isBlank(s)) {
+		if (StringUtils.isBlank(s)) { return null; }
-            return null;
-        }
 		final String[] parts = s.split("###");
 		if (parts.length == 2) {
 			final String value = parts[0];
 			final String[] arr = parts[1].split("@@@");
-            if (arr.length == 4) {
+			if (arr.length == 4) { return structuredProperty(value, arr[0], arr[1], arr[2], arr[3], dataInfo); }
-                return structuredProperty(value, arr[0], arr[1], arr[2], arr[3], dataInfo);
-            }
 		}
 		return null;
 	}
 
 	private List<StructuredProperty> prepareListOfStructProps(
-            final Array array, final DataInfo dataInfo) throws SQLException {
+			final Array array,
+			final DataInfo dataInfo) throws SQLException {
 		final List<StructuredProperty> res = new ArrayList<>();
 		if (array != null) {
 			for (final String s : (String[]) array.getArray()) {
@@ -541,14 +489,10 @@ public class MigrateDbEntitiesApplication extends AbstractMigrationApplication
 			final String[] arr = sj.split("@@@");
 			if (arr.length == 3) {
 				final String issn = StringUtils.isNotBlank(arr[0]) ? arr[0] : null;
-                final String eissn = StringUtils.isNotBlank(arr[1]) ? arr[1] : null;
+				final String eissn = StringUtils.isNotBlank(arr[1]) ? arr[1] : null;;
-                ;
+				final String lissn = StringUtils.isNotBlank(arr[2]) ? arr[2] : null;;
-                final String lissn = StringUtils.isNotBlank(arr[2]) ? arr[2] : null;
-                ;
 				if (issn != null || eissn != null || lissn != null) {
-                    return journal(
+					return journal(name, issn, eissn, eissn, null, null, null, null, null, null, null, info);
-                            name, issn, eissn, eissn, null, null, null, null, null, null, null,
-                            info);
 				}
 			}
 		}

dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/common/MigrationConstants.java

@@ -0,0 +1,22 @@
+package eu.dnetlib.dhp.oa.graph.raw.common;
+
+import static eu.dnetlib.dhp.oa.graph.raw.common.OafMapperUtils.qualifier;
+
+import eu.dnetlib.dhp.schema.oaf.Qualifier;
+
+public class MigrationConstants {
+
+	public static final Qualifier PUBLICATION_RESULTTYPE_QUALIFIER =
+			qualifier("publication", "publication", "dnet:result_typologies", "dnet:result_typologies");
+	public static final Qualifier DATASET_RESULTTYPE_QUALIFIER =
+			qualifier("dataset", "dataset", "dnet:result_typologies", "dnet:result_typologies");
+	public static final Qualifier SOFTWARE_RESULTTYPE_QUALIFIER =
+			qualifier("software", "software", "dnet:result_typologies", "dnet:result_typologies");
+	public static final Qualifier OTHER_RESULTTYPE_QUALIFIER =
+			qualifier("other", "other", "dnet:result_typologies", "dnet:result_typologies");
+	public static final Qualifier REPOSITORY_PROVENANCE_ACTIONS =
+			qualifier("sysimport:crosswalk:repository", "sysimport:crosswalk:repository", "dnet:provenanceActions", "dnet:provenanceActions");
+	public static final Qualifier ENTITYREGISTRY_PROVENANCE_ACTION =
+			qualifier("sysimport:crosswalk:entityregistry", "sysimport:crosswalk:entityregistry", "dnet:provenanceActions", "dnet:provenanceActions");
+
+}