partial implementation

dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/migration/AbstractMigrateApplication.java

@@ -116,9 +116,13 @@ public class AbstractMigrateApplication implements Closeable {
 			final String schemeid,
 			final String schemename,
 			final DataInfo dataInfo) {
+		return structuredProperty(value, qualifier(classid, classname, schemeid, schemename), dataInfo);
+	}
+
+	public static StructuredProperty structuredProperty(final String value, final Qualifier qualifier, final DataInfo dataInfo) {
 		final StructuredProperty sp = new StructuredProperty();
 		sp.setValue(value);
-		sp.setQualifier(qualifier(classid, classname, schemeid, schemename));
+		sp.setQualifier(qualifier);
 		sp.setDataInfo(dataInfo);
 		return sp;
 	}
@@ -198,7 +202,7 @@ public class AbstractMigrateApplication implements Closeable {
 		return d;
 	}
 
-	public static String createOpenaireId(final String prefix, final String originalId) {
+	public static String createOpenaireId(final int prefix, final String originalId) {
 		final String nsPrefix = StringUtils.substringBefore(originalId, "::");
 		final String rest = StringUtils.substringAfter(originalId, "::");
 		return String.format("%s|%s::%s", prefix, nsPrefix, DHPUtils.md5(rest));

dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/migration/MigrateDbEntitiesApplication.java

@@ -60,7 +60,6 @@ public class MigrateDbEntitiesApplication extends AbstractMigrateApplication imp
 			smdbe.execute("queryDatasourceOrganization.sql", smdbe::processDatasourceOrganization);
 			smdbe.execute("queryProjectOrganization.sql", smdbe::processProjectOrganization);
 		}
-
 	}
 
 	public MigrateDbEntitiesApplication(final String hdfsPath, final String hdfsNameNode, final String hdfsUser, final String dbUrl, final String dbUser,
@@ -82,13 +81,13 @@ public class MigrateDbEntitiesApplication extends AbstractMigrateApplication imp
 
 			final Datasource ds = new Datasource();
 
-			ds.setId(createOpenaireId("10", rs.getString("datasourceid")));
+			ds.setId(createOpenaireId(10, rs.getString("datasourceid")));
 			ds.setOriginalId(Arrays.asList(rs.getString("datasourceid")));
 			ds.setCollectedfrom(listKeyValues(rs.getString("collectedfromid"), rs.getString("collectedfromname")));
-			ds.setPid(null); // List<StructuredProperty> // TODO
+			ds.setPid(new ArrayList<>());
 			ds.setDateofcollection(rs.getDate("dateofcollection").toString());
 			ds.setDateoftransformation(null);   // Value not returned by the SQL query
-			ds.setExtraInfo(null); // TODO
+			ds.setExtraInfo(new ArrayList<>());  // Values not present in the DB
 			ds.setOaiprovenance(null); // Values not present in the DB
 			ds.setDatasourcetype(prepareQualifierSplitting(rs.getString("datasourcetype")));
 			ds.setOpenairecompatibility(prepareQualifierSplitting(rs.getString("openairecompatibility")));
@@ -189,13 +188,13 @@ public class MigrateDbEntitiesApplication extends AbstractMigrateApplication imp
 
 			final Project p = new Project();
 
-			p.setId(createOpenaireId("40", rs.getString("projectid")));
+			p.setId(createOpenaireId(40, rs.getString("projectid")));
 			p.setOriginalId(Arrays.asList(rs.getString("projectid")));
 			p.setCollectedfrom(listKeyValues(rs.getString("collectedfromid"), rs.getString("collectedfromname")));
-			p.setPid(null); // List<StructuredProperty> // TODO
+			p.setPid(new ArrayList<>());
 			p.setDateofcollection(rs.getDate("dateofcollection").toString());
 			p.setDateoftransformation(rs.getDate("dateoftransformation").toString());
-			p.setExtraInfo(null); // List<ExtraInfo> //TODO
+			p.setExtraInfo(new ArrayList<>());  // Values not present in the DB
 			p.setOaiprovenance(null); // Values not present in the DB
 			p.setWebsiteurl(field(rs.getString("websiteurl"), info));
 			p.setCode(field(rs.getString("code"), info));
@@ -278,13 +277,13 @@ public class MigrateDbEntitiesApplication extends AbstractMigrateApplication imp
 
 			final Organization o = new Organization();
 
-			o.setId(createOpenaireId("20", rs.getString("organizationid"))); // String id) {
+			o.setId(createOpenaireId(20, rs.getString("organizationid")));
 			o.setOriginalId(Arrays.asList(rs.getString("organizationid")));
 			o.setCollectedfrom(listKeyValues(rs.getString("collectedfromid"), rs.getString("collectedfromname")));
-			o.setPid(null); // List<StructuredProperty> // TODO
+			o.setPid(new ArrayList<>());
 			o.setDateofcollection(rs.getDate("dateofcollection").toString());
 			o.setDateoftransformation(rs.getDate("dateoftransformation").toString());
-			o.setExtraInfo(null); // List<ExtraInfo> // TODO
+			o.setExtraInfo(new ArrayList<>());  // Values not present in the DB
 			o.setOaiprovenance(null); // Values not present in the DB
 			o.setLegalshortname(field("legalshortname", info));
 			o.setLegalname(field("legalname", info));
@@ -342,8 +341,8 @@ public class MigrateDbEntitiesApplication extends AbstractMigrateApplication imp
 
 		try {
 			final DataInfo info = prepareDataInfo(rs);
-			final String orgId = createOpenaireId("20", rs.getString("organization"));
+			final String orgId = createOpenaireId(20, rs.getString("organization"));
-			final String dsId = createOpenaireId("10", rs.getString("datasource"));
+			final String dsId = createOpenaireId(10, rs.getString("datasource"));
 			final List<KeyValue> collectedFrom = listKeyValues(rs.getString("collectedfromid"), rs.getString("collectedfromname"));
 
 			final Relation r1 = new Relation();
@@ -390,8 +389,8 @@ public class MigrateDbEntitiesApplication extends AbstractMigrateApplication imp
 	public void processProjectOrganization(final ResultSet rs) {
 		try {
 			final DataInfo info = prepareDataInfo(rs);
-			final String orgId = createOpenaireId("20", rs.getString("resporganization"));
+			final String orgId = createOpenaireId(20, rs.getString("resporganization"));
-			final String projectId = createOpenaireId("40", rs.getString("project"));
+			final String projectId = createOpenaireId(40, rs.getString("project"));
 			final List<KeyValue> collectedFrom = listKeyValues(rs.getString("collectedfromid"), rs.getString("collectedfromname"));
 
 			final Relation r1 = new Relation();
@@ -451,7 +450,7 @@ public class MigrateDbEntitiesApplication extends AbstractMigrateApplication imp
 		return arr.length == 4 ? qualifier(arr[0], arr[1], arr[2], arr[3]) : null;
 	}
 
-	private static List<Field<String>> prepareListFields(final Array array, final DataInfo info) {
+	private List<Field<String>> prepareListFields(final Array array, final DataInfo info) {
 		try {
 			return listFields(info, (String[]) array.getArray());
 		} catch (final SQLException e) {

dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/migration/MigrateMongoMdstoresApplication.java

@@ -2,34 +2,56 @@ package eu.dnetlib.dhp.migration;
 
 import java.io.Closeable;
 import java.io.IOException;
-import java.io.StringReader;
+import java.sql.SQLException;
 import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Date;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
 import java.util.Map.Entry;
 
 import org.apache.commons.io.IOUtils;
+import org.apache.commons.lang3.StringUtils;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.dom4j.Document;
 import org.dom4j.DocumentException;
+import org.dom4j.DocumentFactory;
+import org.dom4j.DocumentHelper;
 import org.dom4j.Node;
-import org.dom4j.io.SAXReader;
 
 import eu.dnetlib.dhp.application.ArgumentApplicationParser;
+import eu.dnetlib.dhp.schema.oaf.DataInfo;
 import eu.dnetlib.dhp.schema.oaf.Dataset;
+import eu.dnetlib.dhp.schema.oaf.Field;
+import eu.dnetlib.dhp.schema.oaf.KeyValue;
+import eu.dnetlib.dhp.schema.oaf.OAIProvenance;
 import eu.dnetlib.dhp.schema.oaf.Oaf;
 import eu.dnetlib.dhp.schema.oaf.OtherResearchProduct;
 import eu.dnetlib.dhp.schema.oaf.Publication;
+import eu.dnetlib.dhp.schema.oaf.Qualifier;
 import eu.dnetlib.dhp.schema.oaf.Relation;
 import eu.dnetlib.dhp.schema.oaf.Result;
 import eu.dnetlib.dhp.schema.oaf.Software;
+import eu.dnetlib.dhp.schema.oaf.StructuredProperty;
 
 public class MigrateMongoMdstoresApplication extends AbstractMigrateApplication implements Closeable {
 
 	private static final Log log = LogFactory.getLog(MigrateMongoMdstoresApplication.class);
 
+	private final Map<String, String> code2name = new HashMap<>();
+
 	private final MdstoreClient mdstoreClient;
 
+	private static final Qualifier MAIN_TITLE_QUALIFIER = qualifier("main title", "main title", "dnet:dataCite_title", "dnet:dataCite_title");
+
+	private static final Qualifier PUBLICATION_RESULTTYPE_QUALIFIER =
+			qualifier("publication", "publication", "dnet:result_typologies", "dnet:result_typologies");
+	private static final Qualifier DATASET_RESULTTYPE_QUALIFIER = qualifier("dataset", "dataset", "dnet:result_typologies", "dnet:result_typologies");
+	private static final Qualifier SOFTWARE_RESULTTYPE_QUALIFIER = qualifier("software", "software", "dnet:result_typologies", "dnet:result_typologies");
+	private static final Qualifier OTHER_RESULTTYPE_QUALIFIER = qualifier("other", "other", "dnet:result_typologies", "dnet:result_typologies");
+
 	public static void main(final String[] args) throws Exception {
 		final ArgumentApplicationParser parser = new ArgumentApplicationParser(
 				IOUtils.toString(MigrateMongoMdstoresApplication.class.getResourceAsStream("/eu/dnetlib/dhp/migration/migrate_mongo_mstores_parameters.json")));
@@ -46,16 +68,46 @@ public class MigrateMongoMdstoresApplication extends AbstractMigrateApplication
 		final String hdfsNameNode = parser.get("namenode");
 		final String hdfsUser = parser.get("hdfsUser");
 
-		try (final MigrateMongoMdstoresApplication mig = new MigrateMongoMdstoresApplication(hdfsPath, hdfsNameNode, hdfsUser, mongoBaseUrl, mongoDb)) {
+		final String dbUrl = parser.get("postgresUrl");
+		final String dbUser = parser.get("postgresUser");
+		final String dbPassword = parser.get("postgresPassword");
+
+		try (final MigrateMongoMdstoresApplication mig =
+				new MigrateMongoMdstoresApplication(hdfsPath, hdfsNameNode, hdfsUser, mongoBaseUrl, mongoDb, dbUrl, dbUser, dbPassword)) {
 			mig.processMdRecords(mdFormat, mdLayout, mdInterpretation);
 		}
 
 	}
 
 	public MigrateMongoMdstoresApplication(final String hdfsPath, final String hdfsNameNode, final String hdfsUser, final String mongoBaseUrl,
-			final String mongoDb) throws Exception {
+			final String mongoDb, final String dbUrl, final String dbUser,
+			final String dbPassword) throws Exception {
 		super(hdfsPath, hdfsNameNode, hdfsUser);
+
 		this.mdstoreClient = new MdstoreClient(mongoBaseUrl, mongoDb);
+		loadClassNames(dbUrl, dbUser, dbPassword);
+
+		final Map<String, String> nsContext = new HashMap<>();
+		nsContext.put("dc", "http://purl.org/dc/elements/1.1/");
+		nsContext.put("dr", "http://www.driver-repository.eu/namespace/dr");
+		nsContext.put("dri", "http://www.driver-repository.eu/namespace/dri");
+		nsContext.put("oaf", "http://namespace.openaire.eu/oaf");
+		nsContext.put("oai", "http://www.openarchives.org/OAI/2.0/");
+		nsContext.put("prov", "http://www.openarchives.org/OAI/2.0/provenance");
+		DocumentFactory.getInstance().setXPathNamespaceURIs(nsContext);
+	}
+
+	private void loadClassNames(final String dbUrl, final String dbUser, final String dbPassword) throws IOException {
+		try (DbClient dbClient = new DbClient(dbUrl, dbUser, dbPassword)) {
+			code2name.clear();
+			dbClient.processResults("select code, name from class", rs -> {
+				try {
+					code2name.put(rs.getString("code"), rs.getString("name"));
+				} catch (final SQLException e) {
+					e.printStackTrace();
+				}
+			});
+		}
 
 	}
 
@@ -74,23 +126,29 @@ public class MigrateMongoMdstoresApplication extends AbstractMigrateApplication
 	}
 
 	private List<Oaf> createOafs(final String xml) throws DocumentException {
-		final SAXReader reader = new SAXReader();
-		final Document doc = reader.read(new StringReader(xml));
 
-		final String type = doc.valueOf(""); // TODO
+		final Document doc = DocumentHelper.parseText(xml);
+
+		final String type = doc.valueOf("//dr:CobjCategory/@type");
+		final KeyValue collectedFrom = keyValue(doc.valueOf("//oaf:collectedFrom/@id"), doc.valueOf("//oaf:collectedFrom/@name"));
+		final DataInfo info = prepareDataInfo(doc);
+		final long lastUpdateTimestamp = new Date().getTime();
 
 		final List<Oaf> oafs = new ArrayList<>();
 
 		switch (type.toLowerCase()) {
+		case "":
 		case "publication":
 			final Publication p = new Publication();
-			populateResultFields(p, doc);
+			populateResultFields(p, doc, collectedFrom, info, lastUpdateTimestamp);
+			p.setResulttype(PUBLICATION_RESULTTYPE_QUALIFIER);
 			p.setJournal(null); // TODO
 			oafs.add(p);
 			break;
 		case "dataset":
 			final Dataset d = new Dataset();
-			populateResultFields(d, doc);
+			populateResultFields(d, doc, collectedFrom, info, lastUpdateTimestamp);
+			d.setResulttype(DATASET_RESULTTYPE_QUALIFIER);
 			d.setStoragedate(null); // TODO
 			d.setDevice(null); // TODO
 			d.setSize(null); // TODO
@@ -101,16 +159,11 @@ public class MigrateMongoMdstoresApplication extends AbstractMigrateApplication
 			oafs.add(d);
 			break;
 		case "otherresearchproducts":
-			final OtherResearchProduct o = new OtherResearchProduct();
+
-			populateResultFields(o, doc);
-			o.setContactperson(null); // TODO
-			o.setContactgroup(null); // TODO
-			o.setTool(null); // TODO
-			oafs.add(o);
-			break;
 		case "software":
 			final Software s = new Software();
-			populateResultFields(s, doc);
+			populateResultFields(s, doc, collectedFrom, info, lastUpdateTimestamp);
+			s.setResulttype(SOFTWARE_RESULTTYPE_QUALIFIER);
 			s.setDocumentationUrl(null); // TODO
 			s.setLicense(null); // TODO
 			s.setCodeRepositoryUrl(null); // TODO
@@ -118,20 +171,32 @@ public class MigrateMongoMdstoresApplication extends AbstractMigrateApplication
 			oafs.add(s);
 			break;
 		default:
-			log.error("Inavlid type: " + type);
+			final OtherResearchProduct o = new OtherResearchProduct();
+			populateResultFields(o, doc, collectedFrom, info, lastUpdateTimestamp);
+			o.setResulttype(OTHER_RESULTTYPE_QUALIFIER);
+			o.setContactperson(null); // TODO
+			o.setContactgroup(null); // TODO
+			o.setTool(null); // TODO
+			oafs.add(o);
 			break;
 		}
 
 		if (!oafs.isEmpty()) {
-			addRelations(oafs, doc, "//*", "TYPE");
+			addRelations(oafs, doc, "//*", "TYPE", collectedFrom, info, lastUpdateTimestamp); // TODO
-			addRelations(oafs, doc, "//*", "TYPE");
+			addRelations(oafs, doc, "//*", "TYPE", collectedFrom, info, lastUpdateTimestamp); // TODO
-			addRelations(oafs, doc, "//*", "TYPE");
+			addRelations(oafs, doc, "//*", "TYPE", collectedFrom, info, lastUpdateTimestamp); // TODO
 		}
 
 		return oafs;
 	}
 
-	private void addRelations(final List<Oaf> oafs, final Document doc, final String xpath, final String type) {
+	private void addRelations(final List<Oaf> oafs,
+			final Document doc,
+			final String xpath,
+			final String type,
+			final KeyValue collectedFrom,
+			final DataInfo info,
+			final long lastUpdateTimestamp) {
 		for (final Object o : doc.selectNodes(xpath)) {
 			final Node n = (Node) o;
 			final Relation r = new Relation();
@@ -140,40 +205,42 @@ public class MigrateMongoMdstoresApplication extends AbstractMigrateApplication
 			r.setRelClass(null); // TODO
 			r.setSource(null); // TODO
 			r.setTarget(null); // TODO
-			r.setCollectedFrom(null); // TODO
+			r.setCollectedFrom(Arrays.asList(collectedFrom));
+			r.setDataInfo(info);
+			r.setLastupdatetimestamp(lastUpdateTimestamp);
 			oafs.add(r);
 		}
 
 	}
 
-	private void populateResultFields(final Result r, final Document doc) {
+	private void populateResultFields(final Result r, final Document doc, final KeyValue collectedFrom, final DataInfo info, final long lastUpdateTimestamp) {
-		r.setDataInfo(null); // TODO
+
-		r.setLastupdatetimestamp(null); // TODO
+		r.setDataInfo(info);
-		r.setId(null); // TODO
+		r.setLastupdatetimestamp(lastUpdateTimestamp);
-		r.setOriginalId(null); // TODO
+		r.setId(createOpenaireId(50, doc.valueOf("//dri:objIdentifier")));
-		r.setCollectedfrom(null); // TODO
+		r.setOriginalId(Arrays.asList(doc.valueOf("//dri:objIdentifier")));
-		r.setPid(null); // TODO
+		r.setCollectedfrom(Arrays.asList(collectedFrom));
-		r.setDateofcollection(null); // TODO
+		r.setPid(prepareListStructProps(doc, "//oaf:identifier", "@identifierType", "dnet:pid_types", "dnet:pid_types", info));
-		r.setDateoftransformation(null); // TODO
+		r.setDateofcollection(doc.valueOf("//dr:dateOfCollection"));
-		r.setExtraInfo(null); // TODO
+		r.setDateoftransformation(doc.valueOf("//dr:dateOfTransformation"));
-		r.setOaiprovenance(null); // TODO
+		r.setExtraInfo(new ArrayList<>()); // NOT PRESENT IN MDSTORES
+		r.setOaiprovenance(prepareOAIprovenance(doc));
 		r.setAuthor(null); // TODO
-		r.setResulttype(null); // TODO
+		r.setLanguage(prepareQualifier(doc, "//dc:language", "dnet:languages", "dnet:languages"));
-		r.setLanguage(null); // TODO
+		r.setCountry(new ArrayList<>()); // NOT PRESENT IN MDSTORES
-		r.setCountry(null); // TODO
+		r.setSubject(prepareListStructProps(doc, "//dc:subject", info));
-		r.setSubject(null); // TODO
+		r.setTitle(prepareListStructProps(doc, "//dc:title", MAIN_TITLE_QUALIFIER, info));
-		r.setTitle(null); // TODO
 		r.setRelevantdate(null); // TODO
-		r.setDescription(null); // TODO
+		r.setDescription(prepareListFields(doc, "//dc:description", info));
-		r.setDateofacceptance(null); // TODO
+		r.setDateofacceptance(prepareField(doc, "//oaf:dateAccepted", info));
-		r.setPublisher(null); // TODO
+		r.setPublisher(prepareField(doc, "//dc:publisher", info));
 		r.setEmbargoenddate(null); // TODO
 		r.setSource(null); // TODO
 		r.setFulltext(null); // TODO
-		r.setFormat(null); // TODO
+		r.setFormat(prepareListFields(doc, "//dc:format", info));
-		r.setContributor(null); // TODO
+		r.setContributor(prepareListFields(doc, "//dc:contributor", info));
 		r.setResourcetype(null); // TODO
-		r.setCoverage(null); // TODO
+		r.setCoverage(prepareListFields(doc, "//dc:coverage", info));
 		r.setRefereed(null); // TODO
 		r.setContext(null); // TODO
 		r.setExternalReference(null); // TODO
@@ -182,9 +249,117 @@ public class MigrateMongoMdstoresApplication extends AbstractMigrateApplication
 		r.setProcessingchargecurrency(null); // TODO
 	}
 
+	private Qualifier prepareQualifier(final Document doc, final String xpath, final String schemeId, final String schemeName) {
+		final String classId = doc.valueOf(xpath);
+		final String className = code2name.get(classId);
+		return qualifier(classId, className, schemeId, schemeName);
+	}
+
+	private List<StructuredProperty> prepareListStructProps(final Document doc,
+			final String xpath,
+			final String xpathClassId,
+			final String schemeId,
+			final String schemeName,
+			final DataInfo info) {
+		final List<StructuredProperty> res = new ArrayList<>();
+		for (final Object o : doc.selectNodes(xpath)) {
+			final Node n = (Node) o;
+			final String classId = n.valueOf(xpathClassId);
+			final String className = code2name.get(classId);
+			res.add(structuredProperty(n.getText(), classId, className, schemeId, schemeName, info));
+		}
+		return res;
+	}
+
+	private List<StructuredProperty> prepareListStructProps(final Document doc, final String xpath, final Qualifier qualifier, final DataInfo info) {
+		final List<StructuredProperty> res = new ArrayList<>();
+		for (final Object o : doc.selectNodes(xpath)) {
+			final Node n = (Node) o;
+			res.add(structuredProperty(n.getText(), qualifier, info));
+		}
+		return res;
+	}
+
+	private List<StructuredProperty> prepareListStructProps(final Document doc, final String xpath, final DataInfo info) {
+		final List<StructuredProperty> res = new ArrayList<>();
+		for (final Object o : doc.selectNodes(xpath)) {
+			final Node n = (Node) o;
+			res.add(structuredProperty(n.getText(), n.valueOf("@classid"), n.valueOf("@classname"), n.valueOf("@schemeid"), n
+					.valueOf("@schemename"), info));
+		}
+		return res;
+	}
+
+	private OAIProvenance prepareOAIprovenance(final Document doc) {
+		final Node n = doc.selectSingleNode("//*[local-name()='provenance']/*[local-name()='originDescription']");
+
+		final String identifier = n.valueOf("./*[local-name()='identifier']");
+		final String baseURL = n.valueOf("./*[local-name()='baseURL']");;
+		final String metadataNamespace = n.valueOf("./*[local-name()='metadataNamespace']");;
+		final boolean altered = n.valueOf("@altered").equalsIgnoreCase("true");
+		final String datestamp = n.valueOf("./*[local-name()='datestamp']");;
+		final String harvestDate = n.valueOf("@harvestDate");;
+
+		return oaiIProvenance(identifier, baseURL, metadataNamespace, altered, datestamp, harvestDate);
+	}
+
+	private DataInfo prepareDataInfo(final Document doc) {
+		final Node n = doc.selectSingleNode("//oaf:datainfo");
+
+		final String paClassId = n.valueOf("./oaf:provenanceaction/@classid");
+		final String paClassName = n.valueOf("./oaf:provenanceaction/@classname");
+		final String paSchemeId = n.valueOf("./oaf:provenanceaction/@schemeid");
+		final String paSchemeName = n.valueOf("./oaf:provenanceaction/@schemename");
+
+		final boolean deletedbyinference = Boolean.parseBoolean(n.valueOf("./oaf:deletedbyinference"));
+		final String inferenceprovenance = n.valueOf("./oaf:inferenceprovenance");
+		final Boolean inferred = Boolean.parseBoolean(n.valueOf("./oaf:inferred"));
+		final String trust = n.valueOf("./oaf:trust");
+
+		return dataInfo(deletedbyinference, inferenceprovenance, inferred, false, qualifier(paClassId, paClassName, paSchemeId, paSchemeName), trust);
+	}
+
+	private Field<String> prepareField(final Document doc, final String xpath, final DataInfo info) {
+		return field(doc.valueOf(xpath), info);
+	}
+
+	private List<Field<String>> prepareListFields(final Document doc, final String xpath, final DataInfo info) {
+		return listFields(info, (String[]) prepareListString(doc, xpath).toArray());
+	}
+
+	private List<String> prepareListString(final Document doc, final String xpath) {
+		final List<String> res = new ArrayList<>();
+		for (final Object o : doc.selectNodes(xpath)) {
+			final String s = ((Node) o).getText().trim();
+			if (StringUtils.isNotBlank(s)) {
+				res.add(s);
+			}
+		}
+		return res;
+	}
+	/*
+	 * private StructuredProperty prepareStructProp(final Document doc, final String xpath, final DataInfo dataInfo) { if
+	 * (StringUtils.isBlank(s)) { return null; } final String[] parts = s.split("###"); if (parts.length == 2) { final String value =
+	 * parts[0]; final String[] arr = parts[1].split("@@@"); if (arr.length == 4) { return structuredProperty(value, arr[0], arr[1], arr[2],
+	 * arr[3], dataInfo); } } return null; }
+	 *
+	 * private List<StructuredProperty> prepareListOfStructProps(final Document doc, final String xpath, final DataInfo dataInfo) { final
+	 * List<StructuredProperty> res = new ArrayList<>(); if (array != null) { for (final String s : (String[]) array.getArray()) { final
+	 * StructuredProperty sp = prepareStructProp(s, dataInfo); if (sp != null) { res.add(sp); } } }
+	 *
+	 * return res; }
+	 *
+	 * private Journal prepareJournal(final Document doc, final String xpath, final DataInfo info) { if (StringUtils.isNotBlank(sj)) { final
+	 * String[] arr = sj.split("@@@"); if (arr.length == 3) { final String issn = StringUtils.isNotBlank(arr[0]) ? arr[0] : null; final
+	 * String eissn = StringUtils.isNotBlank(arr[1]) ? arr[1] : null;; final String lissn = StringUtils.isNotBlank(arr[2]) ? arr[2] : null;;
+	 * if (issn != null || eissn != null || lissn != null) { return journal(name, issn, eissn, eissn, null, null, null, null, null, null,
+	 * null, info); } } } return null; }
+	 */
+
 	@Override
 	public void close() throws IOException {
 		super.close();
 		mdstoreClient.close();
 	}
+
 }

dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/migration/migrate_mongo_mstores_parameters.json

@@ -46,5 +46,23 @@
 		"paramLongName": "mdInterpretation",
 		"paramDescription": "metadata interpretation",
 		"paramRequired": true
+	},
+	{
+		"paramName": "postgresUrl",
+		"paramLongName": "postgresUrl",
+		"paramDescription": "postgres url, example: jdbc:postgresql://localhost:5432/testdb",
+		"paramRequired": true
+	},
+	{
+		"paramName": "postgresUser",
+		"paramLongName": "postgresUser",
+		"paramDescription": "postgres user",
+		"paramRequired": true
+	},
+	{
+		"paramName": "postgresPassword",
+		"paramLongName": "postgresPassword",
+		"paramDescription": "postgres password",
+		"paramRequired": true
 	}
 ]