From fe36895c53abd194a3dfc30b8edf4506ba8463e7 Mon Sep 17 00:00:00 2001 From: "miriam.baglioni" Date: Fri, 22 Jan 2021 11:55:10 +0100 Subject: [PATCH 1/9] added datasource blacklist for the organization to result propagation through institutional repositories --- .../PrepareResultInstRepoAssociation.java | 23 ++++++++++++++++--- .../input_prepareresultorg_parameters.json | 7 +++++- .../oozie_app/workflow.xml | 1 + 3 files changed, 27 insertions(+), 4 deletions(-) diff --git a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/resulttoorganizationfrominstrepo/PrepareResultInstRepoAssociation.java b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/resulttoorganizationfrominstrepo/PrepareResultInstRepoAssociation.java index fe5889c53..92c09fb28 100644 --- a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/resulttoorganizationfrominstrepo/PrepareResultInstRepoAssociation.java +++ b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/resulttoorganizationfrominstrepo/PrepareResultInstRepoAssociation.java @@ -22,6 +22,11 @@ import eu.dnetlib.dhp.schema.oaf.Datasource; import eu.dnetlib.dhp.schema.oaf.Organization; import eu.dnetlib.dhp.schema.oaf.Relation; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Optional; + public class PrepareResultInstRepoAssociation { private static final Logger log = LoggerFactory.getLogger(PrepareResultInstRepoAssociation.class); @@ -51,6 +56,10 @@ public class PrepareResultInstRepoAssociation { final String alreadyLinkedPath = parser.get("alreadyLinkedPath"); log.info("alreadyLinkedPath {}: ", alreadyLinkedPath); + List blacklist = Optional.ofNullable(parser.get("blacklist")) + .map(v -> Arrays.asList(v.split(";"))) + .orElse(new ArrayList<>()); + SparkConf conf = new SparkConf(); conf.set("hive.metastore.uris", parser.get("hive_metastore_uris")); @@ -61,7 +70,7 @@ public class PrepareResultInstRepoAssociation { readNeededResources(spark, inputPath); removeOutputDir(spark, datasourceOrganizationPath); - prepareDatasourceOrganization(spark, datasourceOrganizationPath); + prepareDatasourceOrganization(spark, datasourceOrganizationPath, blacklist); removeOutputDir(spark, alreadyLinkedPath); prepareAlreadyLinkedAssociation(spark, alreadyLinkedPath); @@ -80,7 +89,15 @@ public class PrepareResultInstRepoAssociation { } private static void prepareDatasourceOrganization( - SparkSession spark, String datasourceOrganizationPath) { + SparkSession spark, String datasourceOrganizationPath, List blacklist) { + String blacklisted = ""; + if(blacklist.size() > 0 ){ + blacklisted = " AND d.id != '" + blacklist.get(0) + "'"; + for (int i = 1; i < blacklist.size(); i++) { + blacklisted += " AND d.id != '" + blacklist.get(i) + "'"; + } + } + String query = "SELECT source datasourceId, target organizationId " + "FROM ( SELECT id " @@ -88,7 +105,7 @@ public class PrepareResultInstRepoAssociation { + "WHERE datasourcetype.classid = '" + INSTITUTIONAL_REPO_TYPE + "' " - + "AND datainfo.deletedbyinference = false ) d " + + "AND datainfo.deletedbyinference = false " + blacklisted + " ) d " + "JOIN ( SELECT source, target " + "FROM relation " + "WHERE lower(relclass) = '" diff --git a/dhp-workflows/dhp-enrichment/src/main/resources/eu/dnetlib/dhp/resulttoorganizationfrominstrepo/input_prepareresultorg_parameters.json b/dhp-workflows/dhp-enrichment/src/main/resources/eu/dnetlib/dhp/resulttoorganizationfrominstrepo/input_prepareresultorg_parameters.json index c74496350..2f00bacae 100644 --- a/dhp-workflows/dhp-enrichment/src/main/resources/eu/dnetlib/dhp/resulttoorganizationfrominstrepo/input_prepareresultorg_parameters.json +++ b/dhp-workflows/dhp-enrichment/src/main/resources/eu/dnetlib/dhp/resulttoorganizationfrominstrepo/input_prepareresultorg_parameters.json @@ -28,5 +28,10 @@ "paramLongName": "isSparkSessionManaged", "paramDescription": "the path where prepared info have been stored", "paramRequired": false - } + },{ + "paramName": "bl", + "paramLongName": "blacklist", + "paramDescription": "institutional repositories that should not be considered for the propagation", + "paramRequired": false +} ] \ No newline at end of file diff --git a/dhp-workflows/dhp-enrichment/src/main/resources/eu/dnetlib/dhp/resulttoorganizationfrominstrepo/oozie_app/workflow.xml b/dhp-workflows/dhp-enrichment/src/main/resources/eu/dnetlib/dhp/resulttoorganizationfrominstrepo/oozie_app/workflow.xml index 2fe9a4256..edfff8817 100644 --- a/dhp-workflows/dhp-enrichment/src/main/resources/eu/dnetlib/dhp/resulttoorganizationfrominstrepo/oozie_app/workflow.xml +++ b/dhp-workflows/dhp-enrichment/src/main/resources/eu/dnetlib/dhp/resulttoorganizationfrominstrepo/oozie_app/workflow.xml @@ -141,6 +141,7 @@ --hive_metastore_uris${hive_metastore_uris} --datasourceOrganizationPath${workingDir}/preparedInfo/datasourceOrganization --alreadyLinkedPath${workingDir}/preparedInfo/alreadyLinked + --blacklist${blacklist} From 4ae6fba01d40f53ad09b7cf29768abce8a170374 Mon Sep 17 00:00:00 2001 From: "miriam.baglioni" Date: Fri, 23 Apr 2021 12:09:19 +0200 Subject: [PATCH 2/9] refactoring --- .../main/java/eu/dnetlib/dhp/schema/oaf/Relation.java | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/Relation.java b/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/Relation.java index 8825d7137..adfc6af95 100644 --- a/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/Relation.java +++ b/dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/Relation.java @@ -1,8 +1,6 @@ package eu.dnetlib.dhp.schema.oaf; -import eu.dnetlib.dhp.schema.common.ModelSupport; - import static com.google.common.base.Preconditions.checkArgument; import java.text.ParseException; @@ -10,6 +8,8 @@ import java.util.*; import java.util.stream.Collectors; import java.util.stream.Stream; +import eu.dnetlib.dhp.schema.common.ModelSupport; + /** * Relation models any edge between two nodes in the OpenAIRE graph. It has a source id and a target id pointing to * graph node identifiers and it is further characterised by the semantic of the link through the fields relType, @@ -137,7 +137,10 @@ public class Relation extends Oaf { try { setValidationDate(ModelSupport.oldest(getValidationDate(), r.getValidationDate())); } catch (ParseException e) { - throw new IllegalArgumentException(String.format("invalid validation date format in relation [s:%s, t:%s]: %s", getSource(), getTarget(), getValidationDate())); + throw new IllegalArgumentException(String + .format( + "invalid validation date format in relation [s:%s, t:%s]: %s", getSource(), getTarget(), + getValidationDate())); } super.mergeFrom(r); From 72e5aa3b42ff8011d19d81eb11805ee8d5f594e1 Mon Sep 17 00:00:00 2001 From: "miriam.baglioni" Date: Fri, 23 Apr 2021 12:10:30 +0200 Subject: [PATCH 3/9] refactoring --- .../PrepareResultInstRepoAssociation.java | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/resulttoorganizationfrominstrepo/PrepareResultInstRepoAssociation.java b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/resulttoorganizationfrominstrepo/PrepareResultInstRepoAssociation.java index 92c09fb28..a41399627 100644 --- a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/resulttoorganizationfrominstrepo/PrepareResultInstRepoAssociation.java +++ b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/resulttoorganizationfrominstrepo/PrepareResultInstRepoAssociation.java @@ -4,6 +4,11 @@ package eu.dnetlib.dhp.resulttoorganizationfrominstrepo; import static eu.dnetlib.dhp.PropagationConstant.*; import static eu.dnetlib.dhp.common.SparkSessionSupport.runWithSparkHiveSession; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Optional; + import org.apache.commons.io.IOUtils; import org.apache.hadoop.io.compress.GzipCodec; import org.apache.spark.SparkConf; @@ -22,11 +27,6 @@ import eu.dnetlib.dhp.schema.oaf.Datasource; import eu.dnetlib.dhp.schema.oaf.Organization; import eu.dnetlib.dhp.schema.oaf.Relation; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.Optional; - public class PrepareResultInstRepoAssociation { private static final Logger log = LoggerFactory.getLogger(PrepareResultInstRepoAssociation.class); @@ -56,9 +56,10 @@ public class PrepareResultInstRepoAssociation { final String alreadyLinkedPath = parser.get("alreadyLinkedPath"); log.info("alreadyLinkedPath {}: ", alreadyLinkedPath); - List blacklist = Optional.ofNullable(parser.get("blacklist")) - .map(v -> Arrays.asList(v.split(";"))) - .orElse(new ArrayList<>()); + List blacklist = Optional + .ofNullable(parser.get("blacklist")) + .map(v -> Arrays.asList(v.split(";"))) + .orElse(new ArrayList<>()); SparkConf conf = new SparkConf(); conf.set("hive.metastore.uris", parser.get("hive_metastore_uris")); @@ -91,14 +92,13 @@ public class PrepareResultInstRepoAssociation { private static void prepareDatasourceOrganization( SparkSession spark, String datasourceOrganizationPath, List blacklist) { String blacklisted = ""; - if(blacklist.size() > 0 ){ + if (blacklist.size() > 0) { blacklisted = " AND d.id != '" + blacklist.get(0) + "'"; for (int i = 1; i < blacklist.size(); i++) { blacklisted += " AND d.id != '" + blacklist.get(i) + "'"; } } - String query = "SELECT source datasourceId, target organizationId " + "FROM ( SELECT id " + "FROM datasource " From eeb8bcf07569c6ebd6fb6c7fb13f39acd256dbde Mon Sep 17 00:00:00 2001 From: Claudio Atzori Date: Tue, 18 May 2021 11:10:07 +0200 Subject: [PATCH 4/9] using constants from ModelConstants --- .../java/eu/dnetlib/dhp/export/DLIToOAF.scala | 57 ++++++++++--------- 1 file changed, 29 insertions(+), 28 deletions(-) diff --git a/dhp-workflows/dhp-graph-provision-scholexplorer/src/main/java/eu/dnetlib/dhp/export/DLIToOAF.scala b/dhp-workflows/dhp-graph-provision-scholexplorer/src/main/java/eu/dnetlib/dhp/export/DLIToOAF.scala index 8043236e0..996b4a821 100644 --- a/dhp-workflows/dhp-graph-provision-scholexplorer/src/main/java/eu/dnetlib/dhp/export/DLIToOAF.scala +++ b/dhp-workflows/dhp-graph-provision-scholexplorer/src/main/java/eu/dnetlib/dhp/export/DLIToOAF.scala @@ -6,6 +6,7 @@ import java.time.LocalDateTime import java.time.format.DateTimeFormatter import eu.dnetlib.dhp.common.PacePerson import eu.dnetlib.dhp.schema.action.AtomicAction +import eu.dnetlib.dhp.schema.common.ModelConstants import eu.dnetlib.dhp.schema.oaf.{Author, Dataset, ExternalReference, Field, Instance, KeyValue, Oaf, Publication, Qualifier, Relation, Result, StructuredProperty} import eu.dnetlib.dhp.schema.scholexplorer.{DLIDataset, DLIPublication} import eu.dnetlib.dhp.utils.DHPUtils @@ -43,18 +44,18 @@ object DLIToOAF { val relationTypeMapping: Map[String, (String, String)] = Map( - "IsReferencedBy" -> ("isRelatedTo", "relationship"), - "References" -> ("isRelatedTo", "relationship"), - "IsRelatedTo" -> ("isRelatedTo", "relationship"), - "IsSupplementedBy" -> ("isSupplementedBy", "supplement"), - "Documents"-> ("isRelatedTo", "relationship"), - "Cites" -> ("cites", "citation"), - "Unknown" -> ("isRelatedTo", "relationship"), - "IsSourceOf" -> ("isRelatedTo", "relationship"), - "IsCitedBy" -> ("IsCitedBy", "citation"), - "Reviews" -> ("reviews", "review"), - "Describes" -> ("isRelatedTo", "relationship"), - "HasAssociationWith" -> ("isRelatedTo", "relationship") + "IsReferencedBy" -> (ModelConstants.IS_RELATED_TO, ModelConstants.RELATIONSHIP), + "References" -> (ModelConstants.IS_RELATED_TO, ModelConstants.RELATIONSHIP), + "IsRelatedTo" -> (ModelConstants.IS_RELATED_TO, ModelConstants.RELATIONSHIP), + "IsSupplementedBy" -> (ModelConstants.IS_SUPPLEMENTED_BY, ModelConstants.SUPPLEMENT), + "Documents"-> (ModelConstants.IS_RELATED_TO, ModelConstants.RELATIONSHIP), + "Cites" -> (ModelConstants.CITES, ModelConstants.CITATION), + "Unknown" -> (ModelConstants.IS_RELATED_TO, ModelConstants.RELATIONSHIP), + "IsSourceOf" -> (ModelConstants.IS_RELATED_TO, ModelConstants.RELATIONSHIP), + "IsCitedBy" -> (ModelConstants.IS_CITED_BY, ModelConstants.CITATION), + "Reviews" -> (ModelConstants.REVIEWS, ModelConstants.REVIEW), + "Describes" -> (ModelConstants.IS_RELATED_TO, ModelConstants.RELATIONSHIP), + "HasAssociationWith" -> (ModelConstants.IS_RELATED_TO, ModelConstants.RELATIONSHIP) ) val expectecdPidType = List("uniprot", "ena", "chembl", "ncbi-n", "ncbi-p", "genbank", "pdb", "url") @@ -83,11 +84,11 @@ object DLIToOAF { val rel_inverse: Map[String, String] = Map( - "isRelatedTo" -> "isRelatedTo", - "isSupplementedBy" -> "isSupplementTo", - "cites" -> "IsCitedBy", - "IsCitedBy" -> "cites", - "reviews" -> "IsReviewedBy" + ModelConstants.IS_RELATED_TO -> ModelConstants.IS_RELATED_TO, + ModelConstants.IS_SUPPLEMENTED_BY -> ModelConstants.IS_SUPPLEMENT_TO, + ModelConstants.CITES -> ModelConstants.IS_CITED_BY, + ModelConstants.IS_CITED_BY -> ModelConstants.CITES, + ModelConstants.REVIEWS -> ModelConstants.IS_REVIEWED_BY ) @@ -158,7 +159,7 @@ object DLIToOAF { result.setUrl(e.url) result.setRefidentifier(e.pid) result.setDataInfo(generateDataInfo()) - result.setQualifier(createQualifier(e.classId, "dnet:externalReference_typologies")) + result.setQualifier(createQualifier(e.classId, ModelConstants.DNET_EXTERNAL_REFERENCE_TYPE)) result }) publication.setExternalReference(eRefs.asJava) @@ -237,7 +238,7 @@ object DLIToOAF { if (inputPublication.getAuthor == null || inputPublication.getAuthor.isEmpty) return null result.setAuthor(inputPublication.getAuthor.asScala.map(convertAuthor).asJava) - result.setResulttype(createQualifier(inputPublication.getResulttype.getClassid, inputPublication.getResulttype.getClassname, "dnet:result_typologies", "dnet:result_typologies")) + result.setResulttype(createQualifier(inputPublication.getResulttype.getClassid, inputPublication.getResulttype.getClassname, ModelConstants.DNET_RESULT_TYPOLOGIES, ModelConstants.DNET_RESULT_TYPOLOGIES)) if (inputPublication.getSubject != null) result.setSubject(inputPublication.getSubject.asScala.map(convertSubject).asJava) @@ -258,7 +259,7 @@ object DLIToOAF { result.setDateofacceptance(asField(inputPublication.getRelevantdate.get(0).getValue)) result.setPublisher(inputPublication.getPublisher) result.setSource(inputPublication.getSource) - result.setBestaccessright(createQualifier("UNKNOWN", "not available", "dnet:access_modes", "dnet:access_modes")) + result.setBestaccessright(createQualifier("UNKNOWN", "not available", ModelConstants.DNET_ACCESS_MODES, ModelConstants.DNET_ACCESS_MODES)) val dois = result.getPid.asScala.filter(p => "doi".equalsIgnoreCase(p.getQualifier.getClassname)).map(p => p.getValue) if (dois.isEmpty) @@ -316,7 +317,7 @@ object DLIToOAF { if (d.getAuthor == null || d.getAuthor.isEmpty) return null result.setAuthor(d.getAuthor.asScala.map(convertAuthor).asJava) - result.setResulttype(createQualifier(d.getResulttype.getClassid, d.getResulttype.getClassname, "dnet:result_typologies", "dnet:result_typologies")) + result.setResulttype(createQualifier(d.getResulttype.getClassid, d.getResulttype.getClassname, ModelConstants.DNET_RESULT_TYPOLOGIES, ModelConstants.DNET_RESULT_TYPOLOGIES)) if (d.getSubject != null) result.setSubject(d.getSubject.asScala.map(convertSubject).asJava) @@ -337,7 +338,7 @@ object DLIToOAF { result.setDateofacceptance(asField(d.getRelevantdate.get(0).getValue)) result.setPublisher(d.getPublisher) result.setSource(d.getSource) - result.setBestaccessright(createQualifier("UNKNOWN", "not available", "dnet:access_modes", "dnet:access_modes")) + result.setBestaccessright(createQualifier("UNKNOWN", "not available", ModelConstants.DNET_ACCESS_MODES, ModelConstants.DNET_ACCESS_MODES)) val instance_urls = if (fpids.head.length < 5) s"https://www.rcsb.org/structure/${fpids.head}" else s"https://dx.doi.org/${fpids.head}" @@ -364,13 +365,13 @@ object DLIToOAF { val i = new Instance i.setUrl(List(url).asJava) if (dataset) - i.setInstancetype(createQualifier("0021", "Dataset", "dnet:publication_resource", "dnet:publication_resource")) + i.setInstancetype(createQualifier("0021", "Dataset", ModelConstants.DNET_PUBLICATION_RESOURCE, ModelConstants.DNET_PUBLICATION_RESOURCE)) else - i.setInstancetype(createQualifier("0000", "Unknown", "dnet:publication_resource", "dnet:publication_resource")) + i.setInstancetype(createQualifier("0000", "Unknown", ModelConstants.DNET_PUBLICATION_RESOURCE, ModelConstants.DNET_PUBLICATION_RESOURCE)) if (originalInstance != null && originalInstance.getHostedby != null) i.setHostedby(originalInstance.getHostedby) - i.setAccessright(createQualifier("UNKNOWN", "not available", "dnet:access_modes", "dnet:access_modes")) + i.setAccessright(createQualifier("UNKNOWN", "not available", ModelConstants.DNET_ACCESS_MODES, ModelConstants.DNET_ACCESS_MODES)) i.setDateofacceptance(doa) i @@ -380,19 +381,19 @@ object DLIToOAF { def patchRelevantDate(d: StructuredProperty): StructuredProperty = { - d.setQualifier(createQualifier("UNKNOWN", "dnet:dataCite_date")) + d.setQualifier(createQualifier("UNKNOWN", ModelConstants.DNET_DATA_CITE_DATE)) d } def patchTitle(t: StructuredProperty): StructuredProperty = { - t.setQualifier(createQualifier("main title", "dnet:dataCite_title")) + t.setQualifier(createQualifier("main title","dnet:dataCite_title")) t } def convertSubject(s: StructuredProperty): StructuredProperty = { - s.setQualifier(createQualifier("keyword", "dnet:subject_classification_typologies")) + s.setQualifier(createQualifier("keyword", ModelConstants.DNET_SUBJECT_TYPOLOGIES)) s From 239d0f0a9af89cbbf695a838629410c6c22e203d Mon Sep 17 00:00:00 2001 From: Claudio Atzori Date: Tue, 18 May 2021 16:12:11 +0200 Subject: [PATCH 5/9] ROR actionset import workflow backported from branch stable_ids --- .../ror/GenerateRorActionSetJob.java | 215 ++++++++++++++++++ .../dhp/actionmanager/ror/model/Address.java | 122 ++++++++++ .../dhp/actionmanager/ror/model/Country.java | 34 +++ .../ror/model/ExternalIdType.java | 42 ++++ .../ror/model/ExternalIdTypeDeserializer.java | 38 ++++ .../ror/model/GeonamesAdmin.java | 56 +++++ .../actionmanager/ror/model/GeonamesCity.java | 100 ++++++++ .../dhp/actionmanager/ror/model/Label.java | 34 +++ .../dhp/actionmanager/ror/model/License.java | 34 +++ .../actionmanager/ror/model/NameAndCode.java | 34 +++ .../actionmanager/ror/model/Relationship.java | 45 ++++ .../ror/model/RorOrganization.java | 192 ++++++++++++++++ .../ror/action_set_parameters.json | 14 ++ .../ror/oozie_app/config-default.xml | 58 +++++ .../actionmanager/ror/oozie_app/workflow.xml | 55 +++++ .../ror/GenerateRorActionSetJobTest.java | 46 ++++ .../dhp/actionmanager/ror/ror_org.json | 123 ++++++++++ 17 files changed, 1242 insertions(+) create mode 100644 dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/ror/GenerateRorActionSetJob.java create mode 100644 dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/ror/model/Address.java create mode 100644 dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/ror/model/Country.java create mode 100644 dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/ror/model/ExternalIdType.java create mode 100644 dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/ror/model/ExternalIdTypeDeserializer.java create mode 100644 dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/ror/model/GeonamesAdmin.java create mode 100644 dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/ror/model/GeonamesCity.java create mode 100644 dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/ror/model/Label.java create mode 100644 dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/ror/model/License.java create mode 100644 dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/ror/model/NameAndCode.java create mode 100644 dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/ror/model/Relationship.java create mode 100644 dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/ror/model/RorOrganization.java create mode 100644 dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/ror/action_set_parameters.json create mode 100644 dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/ror/oozie_app/config-default.xml create mode 100644 dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/ror/oozie_app/workflow.xml create mode 100644 dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/ror/GenerateRorActionSetJobTest.java create mode 100644 dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/actionmanager/ror/ror_org.json diff --git a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/ror/GenerateRorActionSetJob.java b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/ror/GenerateRorActionSetJob.java new file mode 100644 index 000000000..6e25f1654 --- /dev/null +++ b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/ror/GenerateRorActionSetJob.java @@ -0,0 +1,215 @@ + +package eu.dnetlib.dhp.actionmanager.ror; + +import static eu.dnetlib.dhp.common.SparkSessionSupport.runWithSparkSession; +import static eu.dnetlib.dhp.schema.common.ModelConstants.ENTITYREGISTRY_PROVENANCE_ACTION; +import static eu.dnetlib.dhp.schema.oaf.OafMapperUtils.dataInfo; +import static eu.dnetlib.dhp.schema.oaf.OafMapperUtils.field; +import static eu.dnetlib.dhp.schema.oaf.OafMapperUtils.listKeyValues; +import static eu.dnetlib.dhp.schema.oaf.OafMapperUtils.qualifier; +import static eu.dnetlib.dhp.schema.oaf.OafMapperUtils.structuredProperty; + +import java.io.InputStream; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Date; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; + +import org.apache.commons.io.IOUtils; +import org.apache.commons.lang3.StringUtils; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.io.Text; +import org.apache.hadoop.mapred.SequenceFileOutputFormat; +import org.apache.spark.SparkConf; +import org.apache.spark.api.java.function.MapFunction; +import org.apache.spark.sql.Dataset; +import org.apache.spark.sql.Encoders; +import org.apache.spark.sql.SparkSession; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.fasterxml.jackson.databind.ObjectMapper; + +import eu.dnetlib.dhp.actionmanager.project.SparkAtomicActionJob; +import eu.dnetlib.dhp.actionmanager.ror.model.ExternalIdType; +import eu.dnetlib.dhp.actionmanager.ror.model.RorOrganization; +import eu.dnetlib.dhp.application.ArgumentApplicationParser; +import eu.dnetlib.dhp.common.HdfsSupport; +import eu.dnetlib.dhp.schema.action.AtomicAction; +import eu.dnetlib.dhp.schema.common.ModelConstants; +import eu.dnetlib.dhp.schema.oaf.DataInfo; +import eu.dnetlib.dhp.schema.oaf.Field; +import eu.dnetlib.dhp.schema.oaf.KeyValue; +import eu.dnetlib.dhp.schema.oaf.Organization; +import eu.dnetlib.dhp.schema.oaf.Qualifier; +import eu.dnetlib.dhp.schema.oaf.StructuredProperty; +import eu.dnetlib.dhp.utils.DHPUtils; +import scala.Tuple2; + +public class GenerateRorActionSetJob { + + private static final Logger log = LoggerFactory.getLogger(GenerateRorActionSetJob.class); + + private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); + + private static final String ROR_NS_PREFIX = "ror_________"; + + private static final List ROR_COLLECTED_FROM = listKeyValues( + "10|openaire____::993a7ae7a863813cf95028b50708e222", "ROR"); + + private static final DataInfo ROR_DATA_INFO = dataInfo( + false, "", false, false, ENTITYREGISTRY_PROVENANCE_ACTION, "0.92"); + + private static final Qualifier ROR_PID_TYPE = qualifier( + "ROR", "ROR", ModelConstants.DNET_PID_TYPES, ModelConstants.DNET_PID_TYPES); + + public static void main(final String[] args) throws Exception { + + final String jsonConfiguration = IOUtils + .toString( + SparkAtomicActionJob.class + .getResourceAsStream("/eu/dnetlib/dhp/actionmanager/ror/action_set_parameters.json")); + + final ArgumentApplicationParser parser = new ArgumentApplicationParser(jsonConfiguration); + + parser.parseArgument(args); + + final Boolean isSparkSessionManaged = Optional + .ofNullable(parser.get("isSparkSessionManaged")) + .map(Boolean::valueOf) + .orElse(Boolean.TRUE); + + log.info("isSparkSessionManaged: {}", isSparkSessionManaged); + + final String inputPath = parser.get("inputPath"); + log.info("inputPath: {}", inputPath); + + final String outputPath = parser.get("outputPath"); + log.info("outputPath {}: ", outputPath); + + final SparkConf conf = new SparkConf(); + + runWithSparkSession(conf, isSparkSessionManaged, spark -> { + removeOutputDir(spark, outputPath); + processRorOrganizations(spark, inputPath, outputPath); + }); + } + + private static void removeOutputDir(final SparkSession spark, final String path) { + HdfsSupport.remove(path, spark.sparkContext().hadoopConfiguration()); + } + + private static void processRorOrganizations(final SparkSession spark, + final String inputPath, + final String outputPath) throws Exception { + + readInputPath(spark, inputPath) + .map( + (MapFunction) GenerateRorActionSetJob::convertRorOrg, + Encoders.bean(Organization.class)) + .toJavaRDD() + .map(o -> new AtomicAction<>(Organization.class, o)) + .mapToPair( + aa -> new Tuple2<>(new Text(aa.getClazz().getCanonicalName()), + new Text(OBJECT_MAPPER.writeValueAsString(aa)))) + .saveAsHadoopFile(outputPath, Text.class, Text.class, SequenceFileOutputFormat.class); + } + + protected static Organization convertRorOrg(final RorOrganization r) { + + final Date now = new Date(); + + final Organization o = new Organization(); + + o.setId(String.format("20|%s::%s", ROR_NS_PREFIX, DHPUtils.md5(r.getId()))); + o.setOriginalId(Arrays.asList(String.format("%s::%s", ROR_NS_PREFIX, r.getId()))); + o.setCollectedfrom(ROR_COLLECTED_FROM); + o.setPid(pids(r)); + o.setDateofcollection(now.toString()); + o.setDateoftransformation(now.toString()); + o.setExtraInfo(new ArrayList<>()); // Values not present in the file + o.setOaiprovenance(null); // Values not present in the file + o.setLegalshortname(field(r.getAcronyms().stream().findFirst().orElse(r.getName()), ROR_DATA_INFO)); + o.setLegalname(field(r.getName(), ROR_DATA_INFO)); + o.setAlternativeNames(alternativeNames(r)); + o.setWebsiteurl(field(r.getLinks().stream().findFirst().orElse(null), ROR_DATA_INFO)); + o.setLogourl(null); + o.setEclegalbody(null); + o.setEclegalperson(null); + o.setEcnonprofit(null); + o.setEcresearchorganization(null); + o.setEchighereducation(null); + o.setEcinternationalorganizationeurinterests(null); + o.setEcinternationalorganization(null); + o.setEcenterprise(null); + o.setEcsmevalidated(null); + o.setEcnutscode(null); + if (r.getCountry() != null) { + o + .setCountry( + qualifier( + r.getCountry().getCountryCode(), r + .getCountry() + .getCountryName(), + ModelConstants.DNET_COUNTRY_TYPE, ModelConstants.DNET_COUNTRY_TYPE)); + } else { + o.setCountry(null); + } + o.setDataInfo(ROR_DATA_INFO); + o.setLastupdatetimestamp(now.getTime()); + + return o; + } + + private static List pids(final RorOrganization r) { + final List pids = new ArrayList<>(); + pids.add(structuredProperty(r.getId(), ROR_PID_TYPE, ROR_DATA_INFO)); + + for (final Map.Entry e : r.getExternalIds().entrySet()) { + final String type = e.getKey(); + final List all = e.getValue().getAll(); + if (all != null) { + final Qualifier qualifier = qualifier( + type, type, ModelConstants.DNET_PID_TYPES, ModelConstants.DNET_PID_TYPES); + for (final String pid : all) { + pids + .add(structuredProperty(pid, qualifier, ROR_DATA_INFO)); + } + } + } + + return pids; + } + + private static List> alternativeNames(final RorOrganization r) { + final Set names = new LinkedHashSet<>(); + names.addAll(r.getAliases()); + names.addAll(r.getAcronyms()); + r.getLabels().forEach(l -> names.add(l.getLabel())); + + return names + .stream() + .filter(StringUtils::isNotBlank) + .map(s -> field(s, ROR_DATA_INFO)) + .collect(Collectors.toList()); + } + + private static Dataset readInputPath( + final SparkSession spark, + final String path) throws Exception { + + try (final FileSystem fileSystem = FileSystem.get(new Configuration()); + final InputStream is = fileSystem.open(new Path(path))) { + final RorOrganization[] arr = OBJECT_MAPPER.readValue(is, RorOrganization[].class); + return spark.createDataset(Arrays.asList(arr), Encoders.bean(RorOrganization.class)); + } + } + +} diff --git a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/ror/model/Address.java b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/ror/model/Address.java new file mode 100644 index 000000000..b566a5501 --- /dev/null +++ b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/ror/model/Address.java @@ -0,0 +1,122 @@ + +package eu.dnetlib.dhp.actionmanager.ror.model; + +import java.io.Serializable; + +import com.fasterxml.jackson.annotation.JsonProperty; + +public class Address implements Serializable { + + @JsonProperty("lat") + private Float lat; + + @JsonProperty("state_code") + private String stateCode; + + @JsonProperty("country_geonames_id") + private Integer countryGeonamesId; + + @JsonProperty("lng") + private Float lng; + + @JsonProperty("state") + private String state; + + @JsonProperty("city") + private String city; + + @JsonProperty("geonames_city") + private GeonamesCity geonamesCity; + + @JsonProperty("postcode") + private String postcode; + + @JsonProperty("primary") + private Boolean primary; + + @JsonProperty("line") + private String line; + + private final static long serialVersionUID = 2444635485253443195L; + + public Float getLat() { + return lat; + } + + public void setLat(final Float lat) { + this.lat = lat; + } + + public String getStateCode() { + return stateCode; + } + + public void setStateCode(final String stateCode) { + this.stateCode = stateCode; + } + + public Integer getCountryGeonamesId() { + return countryGeonamesId; + } + + public void setCountryGeonamesId(final Integer countryGeonamesId) { + this.countryGeonamesId = countryGeonamesId; + } + + public Float getLng() { + return lng; + } + + public void setLng(final Float lng) { + this.lng = lng; + } + + public String getState() { + return state; + } + + public void setState(final String state) { + this.state = state; + } + + public String getCity() { + return city; + } + + public void setCity(final String city) { + this.city = city; + } + + public GeonamesCity getGeonamesCity() { + return geonamesCity; + } + + public void setGeonamesCity(final GeonamesCity geonamesCity) { + this.geonamesCity = geonamesCity; + } + + public String getPostcode() { + return postcode; + } + + public void setPostcode(final String postcode) { + this.postcode = postcode; + } + + public Boolean getPrimary() { + return primary; + } + + public void setPrimary(final Boolean primary) { + this.primary = primary; + } + + public String getLine() { + return line; + } + + public void setLine(final String line) { + this.line = line; + } + +} diff --git a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/ror/model/Country.java b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/ror/model/Country.java new file mode 100644 index 000000000..3dab60a9f --- /dev/null +++ b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/ror/model/Country.java @@ -0,0 +1,34 @@ + +package eu.dnetlib.dhp.actionmanager.ror.model; + +import java.io.Serializable; + +import com.fasterxml.jackson.annotation.JsonProperty; + +public class Country implements Serializable { + + @JsonProperty("country_code") + private String countryCode; + + @JsonProperty("country_name") + private String countryName; + + private final static long serialVersionUID = 4357848706229493627L; + + public String getCountryCode() { + return countryCode; + } + + public void setCountryCode(final String countryCode) { + this.countryCode = countryCode; + } + + public String getCountryName() { + return countryName; + } + + public void setCountryName(final String countryName) { + this.countryName = countryName; + } + +} diff --git a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/ror/model/ExternalIdType.java b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/ror/model/ExternalIdType.java new file mode 100644 index 000000000..406bfd82c --- /dev/null +++ b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/ror/model/ExternalIdType.java @@ -0,0 +1,42 @@ + +package eu.dnetlib.dhp.actionmanager.ror.model; + +import java.io.Serializable; +import java.util.List; + +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; + +@JsonDeserialize(using = ExternalIdTypeDeserializer.class) +public class ExternalIdType implements Serializable { + + private List all; + + private String preferred; + + private final static long serialVersionUID = 2616688352998387611L; + + public ExternalIdType() { + } + + public ExternalIdType(final List all, final String preferred) { + this.all = all; + this.preferred = preferred; + } + + public List getAll() { + return all; + } + + public void setAll(final List all) { + this.all = all; + } + + public String getPreferred() { + return preferred; + } + + public void setPreferred(final String preferred) { + this.preferred = preferred; + } + +} diff --git a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/ror/model/ExternalIdTypeDeserializer.java b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/ror/model/ExternalIdTypeDeserializer.java new file mode 100644 index 000000000..3fd0c9250 --- /dev/null +++ b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/ror/model/ExternalIdTypeDeserializer.java @@ -0,0 +1,38 @@ + +package eu.dnetlib.dhp.actionmanager.ror.model; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.core.ObjectCodec; +import com.fasterxml.jackson.databind.DeserializationContext; +import com.fasterxml.jackson.databind.JsonDeserializer; +import com.fasterxml.jackson.databind.JsonNode; + +public class ExternalIdTypeDeserializer extends JsonDeserializer { + + @Override + public ExternalIdType deserialize(final JsonParser p, final DeserializationContext ctxt) + throws IOException, JsonProcessingException { + final ObjectCodec oc = p.getCodec(); + final JsonNode node = oc.readTree(p); + + final JsonNode allNode = node.get("all"); + + final String preferred = node.get("preferred").asText(); + + final List all = new ArrayList<>(); + + if (allNode.isArray()) { + allNode.elements().forEachRemaining(x -> all.add(x.asText())); + } else { + all.add(allNode.asText()); + } + + return new ExternalIdType(all, preferred); + } + +} diff --git a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/ror/model/GeonamesAdmin.java b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/ror/model/GeonamesAdmin.java new file mode 100644 index 000000000..9616db447 --- /dev/null +++ b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/ror/model/GeonamesAdmin.java @@ -0,0 +1,56 @@ + +package eu.dnetlib.dhp.actionmanager.ror.model; + +import java.io.Serializable; + +import com.fasterxml.jackson.annotation.JsonProperty; + +public class GeonamesAdmin implements Serializable { + + @JsonProperty("ascii_name") + private String asciiName; + + @JsonProperty("id") + private Integer id; + + @JsonProperty("name") + private String name; + + @JsonProperty("code") + private String code; + + private final static long serialVersionUID = 7294958526269195673L; + + public String getAsciiName() { + return asciiName; + } + + public void setAsciiName(final String asciiName) { + this.asciiName = asciiName; + } + + public Integer getId() { + return id; + } + + public void setId(final Integer id) { + this.id = id; + } + + public String getName() { + return name; + } + + public void setName(final String name) { + this.name = name; + } + + public String getCode() { + return code; + } + + public void setCode(final String code) { + this.code = code; + } + +} diff --git a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/ror/model/GeonamesCity.java b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/ror/model/GeonamesCity.java new file mode 100644 index 000000000..2b0487168 --- /dev/null +++ b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/ror/model/GeonamesCity.java @@ -0,0 +1,100 @@ + +package eu.dnetlib.dhp.actionmanager.ror.model; + +import java.io.Serializable; + +import com.fasterxml.jackson.annotation.JsonProperty; + +public class GeonamesCity implements Serializable { + + @JsonProperty("geonames_admin1") + private GeonamesAdmin geonamesAdmin1; + + @JsonProperty("geonames_admin2") + private GeonamesAdmin geonamesAdmin2; + + @JsonProperty("city") + private String city; + + @JsonProperty("id") + private Integer id; + + @JsonProperty("nuts_level1") + private NameAndCode nutsLevel1; + + @JsonProperty("nuts_level2") + private NameAndCode nutsLevel2; + + @JsonProperty("nuts_level3") + private NameAndCode nutsLevel3; + + @JsonProperty("license") + private License license; + + private final static long serialVersionUID = -8389480201526252955L; + + public NameAndCode getNutsLevel2() { + return nutsLevel2; + } + + public void setNutsLevel2(final NameAndCode nutsLevel2) { + this.nutsLevel2 = nutsLevel2; + } + + public GeonamesAdmin getGeonamesAdmin2() { + return geonamesAdmin2; + } + + public void setGeonamesAdmin2(final GeonamesAdmin geonamesAdmin2) { + this.geonamesAdmin2 = geonamesAdmin2; + } + + public GeonamesAdmin getGeonamesAdmin1() { + return geonamesAdmin1; + } + + public void setGeonamesAdmin1(final GeonamesAdmin geonamesAdmin1) { + this.geonamesAdmin1 = geonamesAdmin1; + } + + public String getCity() { + return city; + } + + public void setCity(final String city) { + this.city = city; + } + + public Integer getId() { + return id; + } + + public void setId(final Integer id) { + this.id = id; + } + + public NameAndCode getNutsLevel1() { + return nutsLevel1; + } + + public void setNutsLevel1(final NameAndCode nutsLevel1) { + this.nutsLevel1 = nutsLevel1; + } + + public NameAndCode getNutsLevel3() { + return nutsLevel3; + } + + public void setNutsLevel3(final NameAndCode nutsLevel3) { + this.nutsLevel3 = nutsLevel3; + } + + public License getLicense() { + return license; + } + + public void setLicense(final License license) { + this.license = license; + } + +} diff --git a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/ror/model/Label.java b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/ror/model/Label.java new file mode 100644 index 000000000..61eb0339d --- /dev/null +++ b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/ror/model/Label.java @@ -0,0 +1,34 @@ + +package eu.dnetlib.dhp.actionmanager.ror.model; + +import java.io.Serializable; + +import com.fasterxml.jackson.annotation.JsonProperty; + +public class Label implements Serializable { + + @JsonProperty("iso639") + private String iso639; + + @JsonProperty("label") + private String label; + + private final static long serialVersionUID = -6576156103297850809L; + + public String getIso639() { + return iso639; + } + + public void setIso639(final String iso639) { + this.iso639 = iso639; + } + + public String getLabel() { + return label; + } + + public void setLabel(final String label) { + this.label = label; + } + +} diff --git a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/ror/model/License.java b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/ror/model/License.java new file mode 100644 index 000000000..bdc8f4c42 --- /dev/null +++ b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/ror/model/License.java @@ -0,0 +1,34 @@ + +package eu.dnetlib.dhp.actionmanager.ror.model; + +import java.io.Serializable; + +import com.fasterxml.jackson.annotation.JsonProperty; + +public class License implements Serializable { + + @JsonProperty("attribution") + private String attribution; + + @JsonProperty("license") + private String license; + + private final static long serialVersionUID = -194308261058176439L; + + public String getAttribution() { + return attribution; + } + + public void setAttribution(final String attribution) { + this.attribution = attribution; + } + + public String getLicense() { + return license; + } + + public void setLicense(final String license) { + this.license = license; + } + +} diff --git a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/ror/model/NameAndCode.java b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/ror/model/NameAndCode.java new file mode 100644 index 000000000..61d7eb8e6 --- /dev/null +++ b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/ror/model/NameAndCode.java @@ -0,0 +1,34 @@ + +package eu.dnetlib.dhp.actionmanager.ror.model; + +import java.io.Serializable; + +import com.fasterxml.jackson.annotation.JsonProperty; + +public class NameAndCode implements Serializable { + + @JsonProperty("name") + private String name; + + @JsonProperty("code") + private String code; + + private final static long serialVersionUID = 5459836979206140843L; + + public String getName() { + return name; + } + + public void setName(final String name) { + this.name = name; + } + + public String getCode() { + return code; + } + + public void setCode(final String code) { + this.code = code; + } + +} diff --git a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/ror/model/Relationship.java b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/ror/model/Relationship.java new file mode 100644 index 000000000..8b73db98f --- /dev/null +++ b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/ror/model/Relationship.java @@ -0,0 +1,45 @@ + +package eu.dnetlib.dhp.actionmanager.ror.model; + +import java.io.Serializable; + +import com.fasterxml.jackson.annotation.JsonProperty; + +public class Relationship implements Serializable { + + @JsonProperty("type") + private String type; + + @JsonProperty("id") + private String id; + + @JsonProperty("label") + private String label; + + private final static long serialVersionUID = 7847399503395576960L; + + public String getType() { + return type; + } + + public void setType(final String type) { + this.type = type; + } + + public String getId() { + return id; + } + + public void setId(final String id) { + this.id = id; + } + + public String getLabel() { + return label; + } + + public void setLabel(final String label) { + this.label = label; + } + +} diff --git a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/ror/model/RorOrganization.java b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/ror/model/RorOrganization.java new file mode 100644 index 000000000..94de34fee --- /dev/null +++ b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/ror/model/RorOrganization.java @@ -0,0 +1,192 @@ + +package eu.dnetlib.dhp.actionmanager.ror.model; + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +import com.fasterxml.jackson.annotation.JsonProperty; + +public class RorOrganization implements Serializable { + + @JsonProperty("ip_addresses") + private List ipAddresses = new ArrayList<>(); + + @JsonProperty("aliases") + private List aliases = new ArrayList<>(); + + @JsonProperty("acronyms") + private List acronyms = new ArrayList<>(); + + @JsonProperty("links") + private List links = new ArrayList<>(); + + @JsonProperty("country") + private Country country; + + @JsonProperty("name") + private String name; + + @JsonProperty("wikipedia_url") + private String wikipediaUrl; + + @JsonProperty("addresses") + private List
addresses = new ArrayList<>(); + + @JsonProperty("types") + private List types = new ArrayList<>(); + + @JsonProperty("established") + private Integer established; + + @JsonProperty("relationships") + private List relationships = new ArrayList<>(); + + @JsonProperty("email_address") + private String emailAddress; + + @JsonProperty("external_ids") + private Map externalIds = new LinkedHashMap<>(); + + @JsonProperty("id") + private String id; + + @JsonProperty("labels") + private List