From b1b48a90dc814f43a5a574c3c174ecfe428cb296 Mon Sep 17 00:00:00 2001 From: "miriam.baglioni" Date: Mon, 18 Sep 2023 08:59:02 +0200 Subject: [PATCH] first entities --- .../dnetlib/dhp/skgif/model/AccessRight.java | 19 ++ .../dnetlib/dhp/skgif/model/Affiliation.java | 41 +++ .../eu/dnetlib/dhp/skgif/model/Biblio.java | 86 ++++++ .../dnetlib/dhp/skgif/model/Contribution.java | 50 ++++ .../eu/dnetlib/dhp/skgif/model/Dates.java | 28 ++ .../dnetlib/dhp/skgif/model/Identifier.java | 29 ++ .../dhp/skgif/model/Manifestation.java | 138 +++++++++ .../dhp/skgif/model/MetadataCuration.java | 17 ++ .../dnetlib/dhp/skgif/model/PeerReview.java | 19 ++ .../eu/dnetlib/dhp/skgif/model/Persons.java | 71 +++++ .../dnetlib/dhp/skgif/model/Provenance.java | 28 ++ .../dnetlib/dhp/skgif/model/RelationType.java | 22 ++ .../eu/dnetlib/dhp/skgif/model/Relations.java | 33 ++ .../dhp/skgif/model/ResearchProduct.java | 116 +++++++ .../dhp/skgif/model/ResearchTypes.java | 19 ++ .../eu/dnetlib/dhp/skgif/model/Topic.java | 28 ++ .../dhp/{oa => }/common/MakeTarArchive.java | 2 +- .../zenodoapi/InputStreamRequestBody.java | 2 +- .../zenodoapi/MissingConceptDoiException.java | 2 +- .../zenodoapi/ZenodoAPIClient.java | 17 +- .../zenodoapi/model/Community.java | 2 +- .../zenodoapi/model/Creator.java | 2 +- .../{oa => common}/zenodoapi/model/File.java | 2 +- .../{oa => common}/zenodoapi/model/Grant.java | 2 +- .../{oa => common}/zenodoapi/model/Links.java | 2 +- .../zenodoapi/model/Metadata.java | 2 +- .../zenodoapi/model/PrereserveDoi.java | 2 +- .../zenodoapi/model/RelatedIdentifier.java | 2 +- .../zenodoapi/model/ZenodoModel.java | 2 +- .../zenodoapi/model/ZenodoModelList.java | 2 +- .../eu/dnetlib/dhp/oa/graph/dump/MakeTar.java | 2 +- .../eu/dnetlib/dhp/skgif/ExtendResult.java | 126 ++++++++ .../dhp/skgif/PrepareResultRelation.java | 93 ++++++ .../eu/dnetlib/dhp/skgif/ResultMapper.java | 282 ++++++++++++++++++ .../exception/NoAllowedTypeException.java | 29 ++ .../exception/NoTitleFoundException.java | 29 ++ .../dhp/oa/graph/dump/ZenodoUploadTest.java | 4 +- 37 files changed, 1322 insertions(+), 30 deletions(-) create mode 100644 dump-schema/src/main/java/eu/dnetlib/dhp/skgif/model/AccessRight.java create mode 100644 dump-schema/src/main/java/eu/dnetlib/dhp/skgif/model/Affiliation.java create mode 100644 dump-schema/src/main/java/eu/dnetlib/dhp/skgif/model/Biblio.java create mode 100644 dump-schema/src/main/java/eu/dnetlib/dhp/skgif/model/Contribution.java create mode 100644 dump-schema/src/main/java/eu/dnetlib/dhp/skgif/model/Dates.java create mode 100644 dump-schema/src/main/java/eu/dnetlib/dhp/skgif/model/Identifier.java create mode 100644 dump-schema/src/main/java/eu/dnetlib/dhp/skgif/model/Manifestation.java create mode 100644 dump-schema/src/main/java/eu/dnetlib/dhp/skgif/model/MetadataCuration.java create mode 100644 dump-schema/src/main/java/eu/dnetlib/dhp/skgif/model/PeerReview.java create mode 100644 dump-schema/src/main/java/eu/dnetlib/dhp/skgif/model/Persons.java create mode 100644 dump-schema/src/main/java/eu/dnetlib/dhp/skgif/model/Provenance.java create mode 100644 dump-schema/src/main/java/eu/dnetlib/dhp/skgif/model/RelationType.java create mode 100644 dump-schema/src/main/java/eu/dnetlib/dhp/skgif/model/Relations.java create mode 100644 dump-schema/src/main/java/eu/dnetlib/dhp/skgif/model/ResearchProduct.java create mode 100644 dump-schema/src/main/java/eu/dnetlib/dhp/skgif/model/ResearchTypes.java create mode 100644 dump-schema/src/main/java/eu/dnetlib/dhp/skgif/model/Topic.java rename dump/src/main/java/eu/dnetlib/dhp/{oa => }/common/MakeTarArchive.java (99%) rename dump/src/main/java/eu/dnetlib/dhp/{oa => common}/zenodoapi/InputStreamRequestBody.java (96%) rename dump/src/main/java/eu/dnetlib/dhp/{oa => common}/zenodoapi/MissingConceptDoiException.java (77%) rename dump/src/main/java/eu/dnetlib/dhp/{oa => common}/zenodoapi/ZenodoAPIClient.java (95%) rename dump/src/main/java/eu/dnetlib/dhp/{oa => common}/zenodoapi/model/Community.java (84%) rename dump/src/main/java/eu/dnetlib/dhp/{oa => common}/zenodoapi/model/Creator.java (94%) rename dump/src/main/java/eu/dnetlib/dhp/{oa => common}/zenodoapi/model/File.java (94%) rename dump/src/main/java/eu/dnetlib/dhp/{oa => common}/zenodoapi/model/Grant.java (89%) rename dump/src/main/java/eu/dnetlib/dhp/{oa => common}/zenodoapi/model/Links.java (97%) rename dump/src/main/java/eu/dnetlib/dhp/{oa => common}/zenodoapi/model/Metadata.java (98%) rename dump/src/main/java/eu/dnetlib/dhp/{oa => common}/zenodoapi/model/PrereserveDoi.java (90%) rename dump/src/main/java/eu/dnetlib/dhp/{oa => common}/zenodoapi/model/RelatedIdentifier.java (94%) rename dump/src/main/java/eu/dnetlib/dhp/{oa => common}/zenodoapi/model/ZenodoModel.java (97%) rename dump/src/main/java/eu/dnetlib/dhp/{oa => common}/zenodoapi/model/ZenodoModelList.java (81%) create mode 100644 dump/src/main/java/eu/dnetlib/dhp/skgif/ExtendResult.java create mode 100644 dump/src/main/java/eu/dnetlib/dhp/skgif/PrepareResultRelation.java create mode 100644 dump/src/main/java/eu/dnetlib/dhp/skgif/ResultMapper.java create mode 100644 dump/src/main/java/eu/dnetlib/dhp/skgif/exception/NoAllowedTypeException.java create mode 100644 dump/src/main/java/eu/dnetlib/dhp/skgif/exception/NoTitleFoundException.java diff --git a/dump-schema/src/main/java/eu/dnetlib/dhp/skgif/model/AccessRight.java b/dump-schema/src/main/java/eu/dnetlib/dhp/skgif/model/AccessRight.java new file mode 100644 index 0000000..106d2c4 --- /dev/null +++ b/dump-schema/src/main/java/eu/dnetlib/dhp/skgif/model/AccessRight.java @@ -0,0 +1,19 @@ +package eu.dnetlib.dhp.skgif.model; + +/** + * @author miriam.baglioni + * @Date 04/09/23 + */ +public enum AccessRight { + OPEN("open"), + CLOSED("closed"), + EMBARGO("embargo"), + RESTRICTED("restricted"), + UNAVAILABLE("unavailable"); + + public final String label; + + private AccessRight(String label) { + this.label = label; + } +} diff --git a/dump-schema/src/main/java/eu/dnetlib/dhp/skgif/model/Affiliation.java b/dump-schema/src/main/java/eu/dnetlib/dhp/skgif/model/Affiliation.java new file mode 100644 index 0000000..47e54cb --- /dev/null +++ b/dump-schema/src/main/java/eu/dnetlib/dhp/skgif/model/Affiliation.java @@ -0,0 +1,41 @@ +package eu.dnetlib.dhp.skgif.model; + +import com.fasterxml.jackson.annotation.JsonProperty; + +import java.io.Serializable; + +/** + * @author miriam.baglioni + * @Date 05/09/23 + */ +public class Affiliation implements Serializable { + private String organization; + @JsonProperty("start_date") + private String startDate; + @JsonProperty("end_date") + private String endDate; + + public String getOrganization() { + return organization; + } + + public void setOrganization(String organization) { + this.organization = organization; + } + + public String getStartDate() { + return startDate; + } + + public void setStartDate(String startDate) { + this.startDate = startDate; + } + + public String getEndDate() { + return endDate; + } + + public void setEndDate(String endDate) { + this.endDate = endDate; + } +} diff --git a/dump-schema/src/main/java/eu/dnetlib/dhp/skgif/model/Biblio.java b/dump-schema/src/main/java/eu/dnetlib/dhp/skgif/model/Biblio.java new file mode 100644 index 0000000..70ffdd0 --- /dev/null +++ b/dump-schema/src/main/java/eu/dnetlib/dhp/skgif/model/Biblio.java @@ -0,0 +1,86 @@ +package eu.dnetlib.dhp.skgif.model; + +import com.fasterxml.jackson.annotation.JsonProperty; + +import java.io.Serializable; + +/** + * @author miriam.baglioni + * @Date 01/09/23 + */ +public class Biblio implements Serializable { + private String issue; + @JsonProperty("start_page") + private String startPage; + @JsonProperty("end_page") + private String endPage; + private String volume; + private String edition; + private String number; + private String publisher; + private String series; + + public String getIssue() { + return issue; + } + + public void setIssue(String issue) { + this.issue = issue; + } + + public String getStartPage() { + return startPage; + } + + public void setStartPage(String startPage) { + this.startPage = startPage; + } + + public String getEndPage() { + return endPage; + } + + public void setEndPage(String endPage) { + this.endPage = endPage; + } + + public String getVolume() { + return volume; + } + + public void setVolume(String volume) { + this.volume = volume; + } + + public String getEdition() { + return edition; + } + + public void setEdition(String edition) { + this.edition = edition; + } + + public String getNumber() { + return number; + } + + public void setNumber(String number) { + this.number = number; + } + + public String getPublisher() { + return publisher; + } + + public void setPublisher(String publisher) { + this.publisher = publisher; + } + + public String getSeries() { + return series; + } + + public void setSeries(String series) { + this.series = series; + } +} diff --git a/dump-schema/src/main/java/eu/dnetlib/dhp/skgif/model/Contribution.java b/dump-schema/src/main/java/eu/dnetlib/dhp/skgif/model/Contribution.java new file mode 100644 index 0000000..2e040d2 --- /dev/null +++ b/dump-schema/src/main/java/eu/dnetlib/dhp/skgif/model/Contribution.java @@ -0,0 +1,50 @@ +package eu.dnetlib.dhp.skgif.model; + +import com.fasterxml.jackson.annotation.JsonProperty; + +import java.io.Serializable; +import java.util.List; + +/** + * @author miriam.baglioni + * @Date 01/09/23 + */ +public class Contribution implements Serializable { + private String person; + @JsonProperty("declared_affiliations") + private List declaredAffiliation; + private List roles; + private Integer rank; + + public String getPerson() { + return person; + } + + public void setPerson(String person) { + this.person = person; + } + + public List getDeclaredAffiliation() { + return declaredAffiliation; + } + + public void setDeclaredAffiliation(List declaredAffiliation) { + this.declaredAffiliation = declaredAffiliation; + } + + public List getRoles() { + return roles; + } + + public void setRoles(List roles) { + this.roles = roles; + } + + public Integer getRank() { + return rank; + } + + public void setRank(Integer rank) { + this.rank = rank; + } +} diff --git a/dump-schema/src/main/java/eu/dnetlib/dhp/skgif/model/Dates.java b/dump-schema/src/main/java/eu/dnetlib/dhp/skgif/model/Dates.java new file mode 100644 index 0000000..0c4e4f9 --- /dev/null +++ b/dump-schema/src/main/java/eu/dnetlib/dhp/skgif/model/Dates.java @@ -0,0 +1,28 @@ +package eu.dnetlib.dhp.skgif.model; + +import java.io.Serializable; + +/** + * @author miriam.baglioni + * @Date 01/09/23 + */ +public class Dates implements Serializable { + private String value; + private String type; + + public String getValue() { + return value; + } + + public void setValue(String value) { + this.value = value; + } + + public String getType() { + return type; + } + + public void setType(String type) { + this.type = type; + } +} diff --git a/dump-schema/src/main/java/eu/dnetlib/dhp/skgif/model/Identifier.java b/dump-schema/src/main/java/eu/dnetlib/dhp/skgif/model/Identifier.java new file mode 100644 index 0000000..3b87aed --- /dev/null +++ b/dump-schema/src/main/java/eu/dnetlib/dhp/skgif/model/Identifier.java @@ -0,0 +1,29 @@ +package eu.dnetlib.dhp.skgif.model; + +import java.io.Serializable; + +/** + * @author miriam.baglioni + * @Date 01/09/23 + */ +public class Identifier implements Serializable { + private String scheme; + private String value; + + public String getScheme() { + return scheme; + } + + public void setScheme(String scheme) { + this.scheme = scheme; + } + + public String getValue() { + return value; + } + + public void setValue(String value) { + this.value = value; + } + +} diff --git a/dump-schema/src/main/java/eu/dnetlib/dhp/skgif/model/Manifestation.java b/dump-schema/src/main/java/eu/dnetlib/dhp/skgif/model/Manifestation.java new file mode 100644 index 0000000..f530d6b --- /dev/null +++ b/dump-schema/src/main/java/eu/dnetlib/dhp/skgif/model/Manifestation.java @@ -0,0 +1,138 @@ +package eu.dnetlib.dhp.skgif.model; + +import com.fasterxml.jackson.annotation.JsonProperty; + +import java.io.Serializable; +import java.net.URL; +import java.util.List; + +/** + * @author miriam.baglioni + * @Date 01/09/23 + */ +public class Manifestation implements Serializable { + @JsonProperty("product_local_type") + private String productLocalType; + @JsonProperty("product_local_type_schema") + private String productLocalTypeSchema; + private List dates; + @JsonProperty("peer_review") + private String peerReview; + @JsonProperty("metadata_curation") + private String metadataCuration; + private URL url; + private String pid; + @JsonProperty("access_right") + private String accessRight; + private String licence; + @JsonProperty("licance_schema") + private String licenceSchema; + private Biblio biblio; + private String venue; + @JsonProperty("hosting_datasource") + private String hostingDatasource; + + public String getProductLocalType() { + return productLocalType; + } + + public void setProductLocalType(String productLocalType) { + this.productLocalType = productLocalType; + } + + public String getProductLocalTypeSchema() { + return productLocalTypeSchema; + } + + public void setProductLocalTypeSchema(String productLocalTypeSchema) { + this.productLocalTypeSchema = productLocalTypeSchema; + } + + public List getDates() { + return dates; + } + + public void setDates(List dates) { + this.dates = dates; + } + + public String getPeerReview() { + return peerReview; + } + + public void setPeerReview(String peerReview) { + this.peerReview = peerReview; + } + + public String getMetadataCuration() { + return metadataCuration; + } + + public void setMetadataCuration(String metadataCuration) { + this.metadataCuration = metadataCuration; + } + + public URL getUrl() { + return url; + } + + public void setUrl(URL url) { + this.url = url; + } + + public String getPid() { + return pid; + } + + public void setPid(String pid) { + this.pid = pid; + } + + public String getAccessRight() { + return accessRight; + } + + public void setAccessRight(String accessRight) { + this.accessRight = accessRight; + } + + public String getLicence() { + return licence; + } + + public void setLicence(String licence) { + this.licence = licence; + } + + public String getLicenceSchema() { + return licenceSchema; + } + + public void setLicenceSchema(String licenceSchema) { + this.licenceSchema = licenceSchema; + } + + public Biblio getBiblio() { + return biblio; + } + + public void setBiblio(Biblio biblio) { + this.biblio = biblio; + } + + public String getVenue() { + return venue; + } + + public void setVenue(String venue) { + this.venue = venue; + } + + public String getHostingDatasource() { + return hostingDatasource; + } + + public void setHostingDatasource(String hostingDatasource) { + this.hostingDatasource = hostingDatasource; + } +} diff --git a/dump-schema/src/main/java/eu/dnetlib/dhp/skgif/model/MetadataCuration.java b/dump-schema/src/main/java/eu/dnetlib/dhp/skgif/model/MetadataCuration.java new file mode 100644 index 0000000..975c4e6 --- /dev/null +++ b/dump-schema/src/main/java/eu/dnetlib/dhp/skgif/model/MetadataCuration.java @@ -0,0 +1,17 @@ +package eu.dnetlib.dhp.skgif.model; + +/** + * @author miriam.baglioni + * @Date 04/09/23 + */ +public enum MetadataCuration { + YES("yes"), + NO("no"), + UNAVAILABLE("unavailable"); + + public final String label; + + private MetadataCuration(String label) { + this.label = label; + } +} diff --git a/dump-schema/src/main/java/eu/dnetlib/dhp/skgif/model/PeerReview.java b/dump-schema/src/main/java/eu/dnetlib/dhp/skgif/model/PeerReview.java new file mode 100644 index 0000000..a7856c3 --- /dev/null +++ b/dump-schema/src/main/java/eu/dnetlib/dhp/skgif/model/PeerReview.java @@ -0,0 +1,19 @@ +package eu.dnetlib.dhp.skgif.model; + +/** + * @author miriam.baglioni + * @Date 04/09/23 + */ +public enum PeerReview { + PEER_REVIEWED("peer reviewed"), + NON_PEER_REVIEWED("open"), + DOUBLE_BLIND("double-blind"), + SINGLE_BLIND("single-blind"), + UNAVAILABLE("unavailable"); + + public final String label; + + private PeerReview(String label) { + this.label = label; + } +} diff --git a/dump-schema/src/main/java/eu/dnetlib/dhp/skgif/model/Persons.java b/dump-schema/src/main/java/eu/dnetlib/dhp/skgif/model/Persons.java new file mode 100644 index 0000000..04b2dd7 --- /dev/null +++ b/dump-schema/src/main/java/eu/dnetlib/dhp/skgif/model/Persons.java @@ -0,0 +1,71 @@ +package eu.dnetlib.dhp.skgif.model; + +import org.codehaus.jackson.annotate.JsonProperty; + +import java.io.Serializable; +import java.util.List; + +/** + * @author miriam.baglioni + * @Date 05/09/23 + */ +public class Persons implements Serializable { + @JsonProperty("local_identifier") + private String localIdentifier; + private List identifiers; + @JsonProperty("given_name") + private String givenName; + @JsonProperty("family_name") + private String familyName; + private String agent; + @JsonProperty("declared_affiliations") + private ListdeclaredAffiliations; + + public String getLocalIdentifier() { + return localIdentifier; + } + + public void setLocalIdentifier(String localIdentifier) { + this.localIdentifier = localIdentifier; + } + + public List getIdentifiers() { + return identifiers; + } + + public void setIdentifiers(List identifiers) { + this.identifiers = identifiers; + } + + public String getGivenName() { + return givenName; + } + + public void setGivenName(String givenName) { + this.givenName = givenName; + } + + public String getFamilyName() { + return familyName; + } + + public void setFamilyName(String familyName) { + this.familyName = familyName; + } + + public String getAgent() { + return agent; + } + + public void setAgent(String agent) { + this.agent = agent; + } + + public List getDeclaredAffiliations() { + return declaredAffiliations; + } + + public void setDeclaredAffiliations(List declaredAffiliations) { + this.declaredAffiliations = declaredAffiliations; + } +} diff --git a/dump-schema/src/main/java/eu/dnetlib/dhp/skgif/model/Provenance.java b/dump-schema/src/main/java/eu/dnetlib/dhp/skgif/model/Provenance.java new file mode 100644 index 0000000..a8618f1 --- /dev/null +++ b/dump-schema/src/main/java/eu/dnetlib/dhp/skgif/model/Provenance.java @@ -0,0 +1,28 @@ +package eu.dnetlib.dhp.skgif.model; + +import java.io.Serializable; + +/** + * @author miriam.baglioni + * @Date 01/09/23 + */ +public class Provenance implements Serializable { + private String type; + private double trust; + + public String getType() { + return type; + } + + public void setType(String type) { + this.type = type; + } + + public double getTrust() { + return trust; + } + + public void setTrust(double trust) { + this.trust = trust; + } +} diff --git a/dump-schema/src/main/java/eu/dnetlib/dhp/skgif/model/RelationType.java b/dump-schema/src/main/java/eu/dnetlib/dhp/skgif/model/RelationType.java new file mode 100644 index 0000000..5ad8843 --- /dev/null +++ b/dump-schema/src/main/java/eu/dnetlib/dhp/skgif/model/RelationType.java @@ -0,0 +1,22 @@ +package eu.dnetlib.dhp.skgif.model; + +import java.io.Serializable; + +/** + * @author miriam.baglioni + * @Date 05/09/23 + */ +public enum RelationType implements Serializable { + OUTCOME("outcome"), + AFFILIATION("hasAuthorInstitution"), + SUPPLEMENT("IsSupplementedBy"), + DOCUMENTS("IsDocumentedBy"), + PART("IsPartOf"), + VERSION("IsNewVersioneOf"); + + public final String label; + + private RelationType(String label) { + this.label = label; + } +} diff --git a/dump-schema/src/main/java/eu/dnetlib/dhp/skgif/model/Relations.java b/dump-schema/src/main/java/eu/dnetlib/dhp/skgif/model/Relations.java new file mode 100644 index 0000000..49ab3b2 --- /dev/null +++ b/dump-schema/src/main/java/eu/dnetlib/dhp/skgif/model/Relations.java @@ -0,0 +1,33 @@ +package eu.dnetlib.dhp.skgif.model; + +import com.fasterxml.jackson.annotation.JsonProperty; + +import java.io.Serializable; +import java.util.List; + +/** + * @author miriam.baglioni + * @Date 01/09/23 + */ +public class Relations implements Serializable { + @JsonProperty("relation_type") + private String relationType; + @JsonProperty("product_list") + private List productList; + + public String getRelationType() { + return relationType; + } + + public void setRelationType(String relationType) { + this.relationType = relationType; + } + + public List getProductList() { + return productList; + } + + public void setProductList(List productList) { + this.productList = productList; + } +} diff --git a/dump-schema/src/main/java/eu/dnetlib/dhp/skgif/model/ResearchProduct.java b/dump-schema/src/main/java/eu/dnetlib/dhp/skgif/model/ResearchProduct.java new file mode 100644 index 0000000..1ff45c9 --- /dev/null +++ b/dump-schema/src/main/java/eu/dnetlib/dhp/skgif/model/ResearchProduct.java @@ -0,0 +1,116 @@ +package eu.dnetlib.dhp.skgif.model; + +import com.fasterxml.jackson.annotation.JsonProperty; + +import java.io.Serializable; +import java.util.List; + +/** + * @author miriam.baglioni + * @Date 01/09/23 + */ +public class ResearchProduct implements Serializable { + @JsonProperty("local_identifier") + private String localIdentifier; + private List identifiers; + private List titles; + private List abstracts; + @JsonProperty("product_type") + private String productType; + private List topics; + private List contributions; + private List manifestations; + @JsonProperty("relevant_organizations") + private List relevantOrganizations; + private List funding; + @JsonProperty("related_products") + private List relatedProducts; + + public String getLocalIdentifier() { + return localIdentifier; + } + + public void setLocalIdentifier(String localIdentifier) { + this.localIdentifier = localIdentifier; + } + + public List getIdentifiers() { + return identifiers; + } + + public void setIdentifiers(List identifiers) { + this.identifiers = identifiers; + } + + public List getTitles() { + return titles; + } + + public void setTitles(List titles) { + this.titles = titles; + } + + public List getAbstracts() { + return abstracts; + } + + public void setAbstracts(List abstracts) { + this.abstracts = abstracts; + } + + public String getProductType() { + return productType; + } + + public void setProductType(String productType) { + this.productType = productType; + } + + public List getTopics() { + return topics; + } + + public void setTopics(List topics) { + this.topics = topics; + } + + public List getContributions() { + return contributions; + } + + public void setContributions(List contributions) { + this.contributions = contributions; + } + + public List getManifestations() { + return manifestations; + } + + public void setManifestations(List manifestations) { + this.manifestations = manifestations; + } + + public List getRelevantOrganizations() { + return relevantOrganizations; + } + + public void setRelevantOrganizations(List relevantOrganizations) { + this.relevantOrganizations = relevantOrganizations; + } + + public List getFunding() { + return funding; + } + + public void setFunding(List funding) { + this.funding = funding; + } + + public List getRelatedProducts() { + return relatedProducts; + } + + public void setRelatedProducts(List relatedProducts) { + this.relatedProducts = relatedProducts; + } +} diff --git a/dump-schema/src/main/java/eu/dnetlib/dhp/skgif/model/ResearchTypes.java b/dump-schema/src/main/java/eu/dnetlib/dhp/skgif/model/ResearchTypes.java new file mode 100644 index 0000000..95ab7e9 --- /dev/null +++ b/dump-schema/src/main/java/eu/dnetlib/dhp/skgif/model/ResearchTypes.java @@ -0,0 +1,19 @@ +package eu.dnetlib.dhp.skgif.model; + +/** + * @author miriam.baglioni + * @Date 01/09/23 + */ +public enum ResearchTypes { + LITERATURE("literature"), + RESEARCH_DATA("research data"), + RESEARCH_SOFTWARE("research software"), + OTHER("other"); + + public final String label; + + private ResearchTypes(String label) { + this.label = label; + } + +} diff --git a/dump-schema/src/main/java/eu/dnetlib/dhp/skgif/model/Topic.java b/dump-schema/src/main/java/eu/dnetlib/dhp/skgif/model/Topic.java new file mode 100644 index 0000000..a4ff9b6 --- /dev/null +++ b/dump-schema/src/main/java/eu/dnetlib/dhp/skgif/model/Topic.java @@ -0,0 +1,28 @@ +package eu.dnetlib.dhp.skgif.model; + +import java.io.Serializable; + +/** + * @author miriam.baglioni + * @Date 01/09/23 + */ +public class Topic implements Serializable { + private String topic; + private Provenance provenance; + + public String getTopic() { + return topic; + } + + public void setTopic(String topic) { + this.topic = topic; + } + + public Provenance getProvenance() { + return provenance; + } + + public void setProvenance(Provenance provenance) { + this.provenance = provenance; + } +} diff --git a/dump/src/main/java/eu/dnetlib/dhp/oa/common/MakeTarArchive.java b/dump/src/main/java/eu/dnetlib/dhp/common/MakeTarArchive.java similarity index 99% rename from dump/src/main/java/eu/dnetlib/dhp/oa/common/MakeTarArchive.java rename to dump/src/main/java/eu/dnetlib/dhp/common/MakeTarArchive.java index 20058bc..43e61e5 100644 --- a/dump/src/main/java/eu/dnetlib/dhp/oa/common/MakeTarArchive.java +++ b/dump/src/main/java/eu/dnetlib/dhp/common/MakeTarArchive.java @@ -1,5 +1,5 @@ -package eu.dnetlib.dhp.oa.common; +package eu.dnetlib.dhp.common; import java.io.BufferedInputStream; import java.io.IOException; diff --git a/dump/src/main/java/eu/dnetlib/dhp/oa/zenodoapi/InputStreamRequestBody.java b/dump/src/main/java/eu/dnetlib/dhp/common/zenodoapi/InputStreamRequestBody.java similarity index 96% rename from dump/src/main/java/eu/dnetlib/dhp/oa/zenodoapi/InputStreamRequestBody.java rename to dump/src/main/java/eu/dnetlib/dhp/common/zenodoapi/InputStreamRequestBody.java index 0f37da0..bbb1e70 100644 --- a/dump/src/main/java/eu/dnetlib/dhp/oa/zenodoapi/InputStreamRequestBody.java +++ b/dump/src/main/java/eu/dnetlib/dhp/common/zenodoapi/InputStreamRequestBody.java @@ -1,5 +1,5 @@ -package eu.dnetlib.dhp.oa.zenodoapi; +package eu.dnetlib.dhp.common.zenodoapi; /** * @author miriam.baglioni diff --git a/dump/src/main/java/eu/dnetlib/dhp/oa/zenodoapi/MissingConceptDoiException.java b/dump/src/main/java/eu/dnetlib/dhp/common/zenodoapi/MissingConceptDoiException.java similarity index 77% rename from dump/src/main/java/eu/dnetlib/dhp/oa/zenodoapi/MissingConceptDoiException.java rename to dump/src/main/java/eu/dnetlib/dhp/common/zenodoapi/MissingConceptDoiException.java index 6bfdba6..dc3819c 100644 --- a/dump/src/main/java/eu/dnetlib/dhp/oa/zenodoapi/MissingConceptDoiException.java +++ b/dump/src/main/java/eu/dnetlib/dhp/common/zenodoapi/MissingConceptDoiException.java @@ -1,5 +1,5 @@ -package eu.dnetlib.dhp.oa.zenodoapi; +package eu.dnetlib.dhp.common.zenodoapi; public class MissingConceptDoiException extends Throwable { public MissingConceptDoiException(String message) { diff --git a/dump/src/main/java/eu/dnetlib/dhp/oa/zenodoapi/ZenodoAPIClient.java b/dump/src/main/java/eu/dnetlib/dhp/common/zenodoapi/ZenodoAPIClient.java similarity index 95% rename from dump/src/main/java/eu/dnetlib/dhp/oa/zenodoapi/ZenodoAPIClient.java rename to dump/src/main/java/eu/dnetlib/dhp/common/zenodoapi/ZenodoAPIClient.java index 59f6055..192ba8c 100644 --- a/dump/src/main/java/eu/dnetlib/dhp/oa/zenodoapi/ZenodoAPIClient.java +++ b/dump/src/main/java/eu/dnetlib/dhp/common/zenodoapi/ZenodoAPIClient.java @@ -1,31 +1,20 @@ -package eu.dnetlib.dhp.oa.zenodoapi; +package eu.dnetlib.dhp.common.zenodoapi; import java.io.*; import java.net.HttpURLConnection; -import java.net.URI; import java.net.URL; -import java.nio.charset.StandardCharsets; import java.util.concurrent.TimeUnit; import org.apache.http.HttpHeaders; -import org.apache.http.client.methods.CloseableHttpResponse; -import org.apache.http.client.methods.HttpGet; -import org.apache.http.client.methods.HttpPost; -import org.apache.http.client.methods.HttpPut; -import org.apache.http.client.utils.URIBuilder; import org.apache.http.entity.ContentType; -import org.apache.http.entity.InputStreamEntity; -import org.apache.http.entity.StringEntity; -import org.apache.http.impl.client.CloseableHttpClient; // import org.apache.http.impl.client.HttpClients; -import org.apache.http.util.EntityUtils; import org.jetbrains.annotations.NotNull; import com.google.gson.Gson; -import eu.dnetlib.dhp.oa.zenodoapi.model.ZenodoModel; -import eu.dnetlib.dhp.oa.zenodoapi.model.ZenodoModelList; +import eu.dnetlib.dhp.common.zenodoapi.model.ZenodoModel; +import eu.dnetlib.dhp.common.zenodoapi.model.ZenodoModelList; import okhttp3.*; public class ZenodoAPIClient implements Serializable { diff --git a/dump/src/main/java/eu/dnetlib/dhp/oa/zenodoapi/model/Community.java b/dump/src/main/java/eu/dnetlib/dhp/common/zenodoapi/model/Community.java similarity index 84% rename from dump/src/main/java/eu/dnetlib/dhp/oa/zenodoapi/model/Community.java rename to dump/src/main/java/eu/dnetlib/dhp/common/zenodoapi/model/Community.java index 008737b..8ab58b2 100644 --- a/dump/src/main/java/eu/dnetlib/dhp/oa/zenodoapi/model/Community.java +++ b/dump/src/main/java/eu/dnetlib/dhp/common/zenodoapi/model/Community.java @@ -1,5 +1,5 @@ -package eu.dnetlib.dhp.oa.zenodoapi.model; +package eu.dnetlib.dhp.common.zenodoapi.model; /** * @author miriam.baglioni diff --git a/dump/src/main/java/eu/dnetlib/dhp/oa/zenodoapi/model/Creator.java b/dump/src/main/java/eu/dnetlib/dhp/common/zenodoapi/model/Creator.java similarity index 94% rename from dump/src/main/java/eu/dnetlib/dhp/oa/zenodoapi/model/Creator.java rename to dump/src/main/java/eu/dnetlib/dhp/common/zenodoapi/model/Creator.java index 9e6eab8..dd74f40 100644 --- a/dump/src/main/java/eu/dnetlib/dhp/oa/zenodoapi/model/Creator.java +++ b/dump/src/main/java/eu/dnetlib/dhp/common/zenodoapi/model/Creator.java @@ -1,5 +1,5 @@ -package eu.dnetlib.dhp.oa.zenodoapi.model; +package eu.dnetlib.dhp.common.zenodoapi.model; /** * @author miriam.baglioni diff --git a/dump/src/main/java/eu/dnetlib/dhp/oa/zenodoapi/model/File.java b/dump/src/main/java/eu/dnetlib/dhp/common/zenodoapi/model/File.java similarity index 94% rename from dump/src/main/java/eu/dnetlib/dhp/oa/zenodoapi/model/File.java rename to dump/src/main/java/eu/dnetlib/dhp/common/zenodoapi/model/File.java index 540bc7c..1931345 100644 --- a/dump/src/main/java/eu/dnetlib/dhp/oa/zenodoapi/model/File.java +++ b/dump/src/main/java/eu/dnetlib/dhp/common/zenodoapi/model/File.java @@ -1,5 +1,5 @@ -package eu.dnetlib.dhp.oa.zenodoapi.model; +package eu.dnetlib.dhp.common.zenodoapi.model; /** * @author miriam.baglioni diff --git a/dump/src/main/java/eu/dnetlib/dhp/oa/zenodoapi/model/Grant.java b/dump/src/main/java/eu/dnetlib/dhp/common/zenodoapi/model/Grant.java similarity index 89% rename from dump/src/main/java/eu/dnetlib/dhp/oa/zenodoapi/model/Grant.java rename to dump/src/main/java/eu/dnetlib/dhp/common/zenodoapi/model/Grant.java index 3a9ddd2..185054e 100644 --- a/dump/src/main/java/eu/dnetlib/dhp/oa/zenodoapi/model/Grant.java +++ b/dump/src/main/java/eu/dnetlib/dhp/common/zenodoapi/model/Grant.java @@ -1,5 +1,5 @@ -package eu.dnetlib.dhp.oa.zenodoapi.model; +package eu.dnetlib.dhp.common.zenodoapi.model; /** * @author miriam.baglioni diff --git a/dump/src/main/java/eu/dnetlib/dhp/oa/zenodoapi/model/Links.java b/dump/src/main/java/eu/dnetlib/dhp/common/zenodoapi/model/Links.java similarity index 97% rename from dump/src/main/java/eu/dnetlib/dhp/oa/zenodoapi/model/Links.java rename to dump/src/main/java/eu/dnetlib/dhp/common/zenodoapi/model/Links.java index e2c3a74..7d5aa13 100644 --- a/dump/src/main/java/eu/dnetlib/dhp/oa/zenodoapi/model/Links.java +++ b/dump/src/main/java/eu/dnetlib/dhp/common/zenodoapi/model/Links.java @@ -1,5 +1,5 @@ -package eu.dnetlib.dhp.oa.zenodoapi.model; +package eu.dnetlib.dhp.common.zenodoapi.model; /** * @author miriam.baglioni diff --git a/dump/src/main/java/eu/dnetlib/dhp/oa/zenodoapi/model/Metadata.java b/dump/src/main/java/eu/dnetlib/dhp/common/zenodoapi/model/Metadata.java similarity index 98% rename from dump/src/main/java/eu/dnetlib/dhp/oa/zenodoapi/model/Metadata.java rename to dump/src/main/java/eu/dnetlib/dhp/common/zenodoapi/model/Metadata.java index 0aaf9b6..74d6d2c 100644 --- a/dump/src/main/java/eu/dnetlib/dhp/oa/zenodoapi/model/Metadata.java +++ b/dump/src/main/java/eu/dnetlib/dhp/common/zenodoapi/model/Metadata.java @@ -1,5 +1,5 @@ -package eu.dnetlib.dhp.oa.zenodoapi.model; +package eu.dnetlib.dhp.common.zenodoapi.model; /** * @author miriam.baglioni diff --git a/dump/src/main/java/eu/dnetlib/dhp/oa/zenodoapi/model/PrereserveDoi.java b/dump/src/main/java/eu/dnetlib/dhp/common/zenodoapi/model/PrereserveDoi.java similarity index 90% rename from dump/src/main/java/eu/dnetlib/dhp/oa/zenodoapi/model/PrereserveDoi.java rename to dump/src/main/java/eu/dnetlib/dhp/common/zenodoapi/model/PrereserveDoi.java index a38b338..05d523a 100644 --- a/dump/src/main/java/eu/dnetlib/dhp/oa/zenodoapi/model/PrereserveDoi.java +++ b/dump/src/main/java/eu/dnetlib/dhp/common/zenodoapi/model/PrereserveDoi.java @@ -1,5 +1,5 @@ -package eu.dnetlib.dhp.oa.zenodoapi.model; +package eu.dnetlib.dhp.common.zenodoapi.model; /** * @author miriam.baglioni diff --git a/dump/src/main/java/eu/dnetlib/dhp/oa/zenodoapi/model/RelatedIdentifier.java b/dump/src/main/java/eu/dnetlib/dhp/common/zenodoapi/model/RelatedIdentifier.java similarity index 94% rename from dump/src/main/java/eu/dnetlib/dhp/oa/zenodoapi/model/RelatedIdentifier.java rename to dump/src/main/java/eu/dnetlib/dhp/common/zenodoapi/model/RelatedIdentifier.java index d128993..3eae11a 100644 --- a/dump/src/main/java/eu/dnetlib/dhp/oa/zenodoapi/model/RelatedIdentifier.java +++ b/dump/src/main/java/eu/dnetlib/dhp/common/zenodoapi/model/RelatedIdentifier.java @@ -1,5 +1,5 @@ -package eu.dnetlib.dhp.oa.zenodoapi.model; +package eu.dnetlib.dhp.common.zenodoapi.model; /** * @author miriam.baglioni diff --git a/dump/src/main/java/eu/dnetlib/dhp/oa/zenodoapi/model/ZenodoModel.java b/dump/src/main/java/eu/dnetlib/dhp/common/zenodoapi/model/ZenodoModel.java similarity index 97% rename from dump/src/main/java/eu/dnetlib/dhp/oa/zenodoapi/model/ZenodoModel.java rename to dump/src/main/java/eu/dnetlib/dhp/common/zenodoapi/model/ZenodoModel.java index 1a0a1a9..4b2aa13 100644 --- a/dump/src/main/java/eu/dnetlib/dhp/oa/zenodoapi/model/ZenodoModel.java +++ b/dump/src/main/java/eu/dnetlib/dhp/common/zenodoapi/model/ZenodoModel.java @@ -1,5 +1,5 @@ -package eu.dnetlib.dhp.oa.zenodoapi.model; +package eu.dnetlib.dhp.common.zenodoapi.model; /** * @author miriam.baglioni diff --git a/dump/src/main/java/eu/dnetlib/dhp/oa/zenodoapi/model/ZenodoModelList.java b/dump/src/main/java/eu/dnetlib/dhp/common/zenodoapi/model/ZenodoModelList.java similarity index 81% rename from dump/src/main/java/eu/dnetlib/dhp/oa/zenodoapi/model/ZenodoModelList.java rename to dump/src/main/java/eu/dnetlib/dhp/common/zenodoapi/model/ZenodoModelList.java index 858eff8..2f1229e 100644 --- a/dump/src/main/java/eu/dnetlib/dhp/oa/zenodoapi/model/ZenodoModelList.java +++ b/dump/src/main/java/eu/dnetlib/dhp/common/zenodoapi/model/ZenodoModelList.java @@ -1,5 +1,5 @@ -package eu.dnetlib.dhp.oa.zenodoapi.model; +package eu.dnetlib.dhp.common.zenodoapi.model; /** * @author miriam.baglioni diff --git a/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/MakeTar.java b/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/MakeTar.java index 44f4ef6..cb2e29b 100644 --- a/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/MakeTar.java +++ b/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/MakeTar.java @@ -15,7 +15,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import eu.dnetlib.dhp.application.ArgumentApplicationParser; -import eu.dnetlib.dhp.oa.common.MakeTarArchive; +import eu.dnetlib.dhp.common.MakeTarArchive; public class MakeTar implements Serializable { diff --git a/dump/src/main/java/eu/dnetlib/dhp/skgif/ExtendResult.java b/dump/src/main/java/eu/dnetlib/dhp/skgif/ExtendResult.java new file mode 100644 index 0000000..d7e3d69 --- /dev/null +++ b/dump/src/main/java/eu/dnetlib/dhp/skgif/ExtendResult.java @@ -0,0 +1,126 @@ +package eu.dnetlib.dhp.skgif; + +import com.fasterxml.jackson.databind.ObjectMapper; +import eu.dnetlib.dhp.application.ArgumentApplicationParser; +import eu.dnetlib.dhp.oa.graph.dump.Utils; +import eu.dnetlib.dhp.skgif.model.RelationType; +import eu.dnetlib.dhp.skgif.model.Relations; +import eu.dnetlib.dhp.skgif.model.ResearchProduct; +import org.apache.commons.io.IOUtils; +import org.apache.spark.SparkConf; +import org.apache.spark.api.java.function.MapFunction; +import org.apache.spark.api.java.function.MapGroupsFunction; +import org.apache.spark.sql.*; +import org.apache.spark.sql.types.StructType; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import scala.Tuple2; + +import java.io.Serializable; +import java.util.*; + +import static eu.dnetlib.dhp.common.SparkSessionSupport.runWithSparkSession; + +/** + * @author miriam.baglioni + * @Date 05/09/23 + */ +public class ExtendResult implements Serializable { + private static final Logger log = LoggerFactory.getLogger(ExtendResult.class); + + public static void main(String[] args) throws Exception { + String jsonConfiguration = IOUtils + .toString( + PrepareResultRelation.class + .getResourceAsStream( + "/eu/dnetlib/dhp/oa/graph/dump/extend_result_parameters.json")); + + final ArgumentApplicationParser parser = new ArgumentApplicationParser(jsonConfiguration); + parser.parseArgument(args); + + Boolean isSparkSessionManaged = Optional + .ofNullable(parser.get("isSparkSessionManaged")) + .map(Boolean::valueOf) + .orElse(Boolean.TRUE); + + + log.info("isSparkSessionManaged: {}", isSparkSessionManaged); + + final String inputPath = parser.get("sourcePath"); + log.info("inputPath: {}", inputPath); + + final String outputPath = parser.get("outputPath"); + log.info("outputPath: {}", outputPath); + + SparkConf conf = new SparkConf(); + + runWithSparkSession( + conf, + isSparkSessionManaged, + spark -> { + Utils.removeOutputDir(spark, outputPath); + extendResult(spark, inputPath, outputPath); + }); + } + + private static void extendResult(SparkSession spark, String inputPath, String outputPath) { + ObjectMapper mapper = new ObjectMapper(); + Dataset result = spark.read().json(inputPath + "/result") + .as(Encoders.bean(ResearchProduct.class)); + final StructType structureSchema = new StructType() + .fromDDL("`resultId` STRING, `target` STRING, `resultClass` STRING"); + + Dataset relations = spark + .read() + .schema(structureSchema) + .json(inputPath + "/preparedRelations"); + + result.joinWith(relations, result.col("localIdentifier") + .equalTo(relations.col("resultId")), "left") + .groupByKey((MapFunction, String>)t2 -> t2._1().getLocalIdentifier(),Encoders.STRING() ) + .mapGroups((MapGroupsFunction, ResearchProduct>) (key,it) -> { + Tuple2 first = it.next(); + ResearchProduct rp = first._1(); + addRels(rp, first._2()); + it.forEachRemaining(t2 -> addRels(rp, t2._2())); + return rp; + }, Encoders.bean(ResearchProduct.class)) + .map((MapFunction) r -> mapper.writeValueAsString(r), Encoders.STRING()) + .write() + .mode(SaveMode.Overwrite) + .option("compression","gzip") + .text(outputPath); + + } + + private static void addRels(ResearchProduct rp, Row row) { + String relClass = row.getAs("relClass"); + Map> relations = new HashMap<>(); + if(relClass.equals(RelationType.OUTCOME.label)){ + if(!Optional.ofNullable(rp.getFunding()).isPresent()){ + rp.setFunding(new ArrayList<>()); + } + rp.getFunding().add(row.getAs("target")); + }else if(relClass.equals(RelationType.AFFILIATION)){ + if(!Optional.ofNullable(rp.getRelevantOrganizations()).isPresent()) + rp.setRelevantOrganizations(new ArrayList<>()); + rp.getRelevantOrganizations().add(row.getAs("target")); + }else{ + if(!relations.containsKey(relClass)){ + relations.put(relClass, new ArrayList<>()); + } + relations.get(relClass).add(row.getAs("target")); + } + if(relations.size() > 0) { + rp.setRelatedProducts(new ArrayList<>()); + for (String key: relations.keySet()){ + Relations rel = new Relations(); + rel.setRelationType(key); + rel.setProductList(relations.get(key)); + } + } + + + + } +} diff --git a/dump/src/main/java/eu/dnetlib/dhp/skgif/PrepareResultRelation.java b/dump/src/main/java/eu/dnetlib/dhp/skgif/PrepareResultRelation.java new file mode 100644 index 0000000..634efcb --- /dev/null +++ b/dump/src/main/java/eu/dnetlib/dhp/skgif/PrepareResultRelation.java @@ -0,0 +1,93 @@ +package eu.dnetlib.dhp.skgif; + +import eu.dnetlib.dhp.application.ArgumentApplicationParser; +import eu.dnetlib.dhp.oa.graph.dump.Utils; +import eu.dnetlib.dhp.schema.oaf.Relation; +import org.apache.commons.io.IOUtils; +import org.apache.spark.SparkConf; +import org.apache.spark.sql.*; +import org.apache.spark.sql.types.StructType; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import java.io.Serializable; +import java.util.*; + +import static eu.dnetlib.dhp.common.SparkSessionSupport.runWithSparkSession; + +/** + * @author miriam.baglioni + * @Date 04/09/23 + */ +public class PrepareResultRelation implements Serializable { + private static final Logger log = LoggerFactory.getLogger(PrepareResultRelation.class); + + public static void main(String[] args) throws Exception { + String jsonConfiguration = IOUtils + .toString( + PrepareResultRelation.class + .getResourceAsStream( + "/eu/dnetlib/dhp/oa/graph/dump/result_relation_parameters.json")); + + final ArgumentApplicationParser parser = new ArgumentApplicationParser(jsonConfiguration); + parser.parseArgument(args); + + Boolean isSparkSessionManaged = Optional + .ofNullable(parser.get("isSparkSessionManaged")) + .map(Boolean::valueOf) + .orElse(Boolean.TRUE); + log.info("isSparkSessionManaged: {}", isSparkSessionManaged); + + final String inputPath = parser.get("sourcePath"); + log.info("inputPath: {}", inputPath); + + final String outputPath = parser.get("outputPath"); + log.info("outputPath: {}", outputPath); + + SparkConf conf = new SparkConf(); + + runWithSparkSession( + conf, + isSparkSessionManaged, + spark -> { + Utils.removeOutputDir(spark, outputPath); + prepareResultRelationList(spark, inputPath, outputPath); + }); + } + + private static void prepareResultRelationList(SparkSession spark, String inputPath, String outputPath) { + final StructType structureSchema = new StructType() + .fromDDL("`id` STRING, `dataInfo` STRUCT<`deletedbyinference`:BOOLEAN,`invisible`:BOOLEAN>"); + Dataset relation = spark.read().json(inputPath).as(Encoders.bean(Relation.class)) + .filter("dataInfo.deletedbyinference != true and dataInfo.invisible != true") + .filter("relClass == 'hasAuthorInstitution' or relClass == 'outcome' or " + + "relClass == 'IsSupplementedBy' or relClass == 'IsDocumentedBy' or relClass == 'IsPartOf' " + + "relClass == IsNewVersionOf"); + + org.apache.spark.sql.Dataset df = spark.createDataFrame(new ArrayList(), structureSchema); + List entities = Arrays + .asList( + "publication", "dataset", "otherresearchproduct", "software"); + for (String e : entities) + df = df + .union( + spark + .read() + .schema(structureSchema) + .json(inputPath + "/" + e) + .filter("dataInfo.deletedbyinference != true and dataInfo.invisible != true")); + + relation.joinWith(df, relation.col("source").equalTo(df.col("id"))) + .select( + new Column("id").as("resultId"), + new Column("target"), + new Column("relClass")) + .write() + .mode(SaveMode.Overwrite) + .option("compression","gzip") + .json(outputPath); + + + } + + +} diff --git a/dump/src/main/java/eu/dnetlib/dhp/skgif/ResultMapper.java b/dump/src/main/java/eu/dnetlib/dhp/skgif/ResultMapper.java new file mode 100644 index 0000000..e5e927a --- /dev/null +++ b/dump/src/main/java/eu/dnetlib/dhp/skgif/ResultMapper.java @@ -0,0 +1,282 @@ +package eu.dnetlib.dhp.skgif; + + +import eu.dnetlib.dhp.oa.graph.dump.csv.AuthorResult; +import eu.dnetlib.dhp.oa.model.ResultPid; +import eu.dnetlib.dhp.schema.common.ModelConstants; +import eu.dnetlib.dhp.schema.oaf.*; +import eu.dnetlib.dhp.skgif.exception.NoAllowedTypeException; +import eu.dnetlib.dhp.skgif.exception.NoTitleFoundException; +import eu.dnetlib.dhp.skgif.model.*; +import eu.dnetlib.dhp.skgif.model.AccessRight; +import eu.dnetlib.dhp.utils.DHPUtils; +import scala.Tuple2; +import scala.reflect.internal.Trees; + +import javax.management.RuntimeErrorException; +import java.io.Serializable; +import java.net.MalformedURLException; +import java.net.URL; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Optional; +import java.util.stream.Collectors; + +import static eu.dnetlib.dhp.oa.graph.dump.ResultMapper.*; + + +/** + * @author miriam.baglioni + * @Date 01/09/23 + */ +public class ResultMapper implements Serializable { + + public static ResearchProduct map( + E input) + throws Exception{ + + ResearchProduct out = new ResearchProduct(); + + Optional ort = Optional.ofNullable(input.getResulttype()); + if (ort.isPresent()) { + try { + out.setLocalIdentifier(input.getId()); + mapPid(out, input); + mapTitle(out, input); + mapAbstract(out, input); + mapType(out, input); + mapTopic(out, input); + mapContribution(out, input); + + if(!Optional.ofNullable(out.getTitles()).isPresent() || + !Optional.ofNullable(out.getContributions()).isPresent()) + return null; + //TODO map the manifestation directly from the instances + //it is not completed + mapManifestation(out, input); + + + //TODO extend the mapping to consider relations between these entities and the results +// private List relevantOrganizations; +// private List funding; +// private List relatedProducts; + + } catch (ClassCastException cce) { + return null; + } + } + + return null; + + } + + private static void mapManifestation(ResearchProduct out, E input) { + out.setManifestations( input.getInstance().stream().parallel() + .map(i -> { + try { + return getManifestation(i); + } catch (MalformedURLException e) { + throw new RuntimeException(e); + } + }).collect(Collectors.toList())); + } + + private static Manifestation getManifestation(Instance i) throws MalformedURLException { + Manifestation manifestation = new Manifestation(); + manifestation.setProductLocalType(i.getInstancetype().getClassname()); + manifestation.setProductLocalTypeSchema(i.getInstancetype().getSchemename()); + Dates dates = new Dates(); + dates.setType("publishing"); + dates.setValue(i.getDateofacceptance().getValue()); + manifestation.setDates(Arrays.asList(dates)); + switch(i.getRefereed().getClassid()){ + case "0000": + manifestation.setPeerReview(PeerReview.UNAVAILABLE.label); + break; + case "0001": + manifestation.setPeerReview(PeerReview.PEER_REVIEWED.label); + break; + case "0002": + manifestation.setPeerReview(PeerReview.NON_PEER_REVIEWED.label); + break; + } + manifestation.setMetadataCuration(MetadataCuration.UNAVAILABLE.label); + //TODO filter out the URL that refer to pids. If nothing remains, decide what to do + manifestation.setUrl(new URL(i.getUrl().get(0))); + if(Optional.ofNullable(i.getPid()).isPresent()){ + manifestation.setPid(i.getPid().get(0).getValue()); + } + switch(i.getAccessright().getClassid()){ + case"OPEN": + case"OPEN DATA": + case "OPEN SOURCE": + manifestation.setAccessRight(AccessRight.OPEN.label); + break; + case "CLOSED": + manifestation.setAccessRight(AccessRight.CLOSED.label); + break; + case "RESTRICTED": + manifestation.setAccessRight(AccessRight.RESTRICTED.label); + break; + case"EMBARGO": + case "12MONTHS": + case "6MONTHS": + manifestation.setAccessRight(AccessRight.EMBARGO.label); + break; + default: + manifestation.setAccessRight(AccessRight.UNAVAILABLE.label); + } + if(Optional.ofNullable(i.getLicense()).isPresent()) + manifestation.setLicence(i.getLicense().getValue()); + //TODO to fill the biblio in case it is a journal, we need to join with the datasource and verify the type + Biblio biblio = null; + manifestation.setHostingDatasource(i.getHostedby().getKey()); + //TODO verify if the result is published in ojournal or conferences. In that case the venue is the identifier + //of the journal/conference. In case it is not, the venue is the datasource + if(biblio == null){ + manifestation.setVenue(i.getHostedby().getKey()); + }else{ + manifestation.setVenue("insert the id of the venue"); + } + return manifestation; + } + + private static Tuple2 getOrcid(List pid) { + if (!Optional.ofNullable(pid).isPresent()) + return null; + if (pid.size() == 0) + return null; + for (StructuredProperty p : pid) { + if (p.getQualifier().getClassid().equals(ModelConstants.ORCID)) { + return new Tuple2<>(p.getValue(), Boolean.TRUE); + } + } + for (StructuredProperty p : pid) { + if (p.getQualifier().getClassid().equals(ModelConstants.ORCID_PENDING)) { + return new Tuple2<>(p.getValue(), Boolean.FALSE); + } + } + return null; + } + private static void mapContribution(ResearchProduct out, E input) { + if(Optional.ofNullable(input.getAuthor()).isPresent()){ + int count = 0; + for (Author a : input.getAuthor()) { + count += 1; + Contribution contribution = new Contribution(); + if(Optional.ofNullable(a.getPid()).isPresent()){ + Tuple2 orcid = getOrcid(a.getPid()); + if(orcid != null){ + contribution.setPerson(DHPUtils.md5(orcid._1() + orcid._2())); + }else{ + if(Optional.ofNullable(a.getRank()).isPresent()){ + contribution.setPerson(DHPUtils.md5(input.getId() + a.getRank())); + }else{ + contribution.setPerson(DHPUtils.md5(input.getId() + count)); + } + + } + } + } + } + +// "contributions": [ +// { +// "person": "person_123", +// "declared_affiliations": ["org_1", "org_3"], +// "rank": 1, +// "roles": ["writing-original-draft", "conceptualization"] +// } +// ] + } + + private static void mapTopic(ResearchProduct out, E input) { + if(Optional.ofNullable(input.getSubject()).isPresent()){ + out.setTopics(input.getSubject().stream().parallel().map(s -> { + Topic topic = new Topic(); + topic.setTopic(getIdentifier(s)); + Provenance provenance = new Provenance(); + provenance.setTrust(Double.valueOf(s.getDataInfo().getTrust())); + provenance.setType(s.getDataInfo().getInferenceprovenance()); + topic.setProvenance(provenance); + return topic; + }).collect(Collectors.toList())); + } + } + + private static String getIdentifier(StructuredProperty s) { + return DHPUtils.md5(s.getQualifier().getClassid() + s.getValue()); + } + + private static void mapType(ResearchProduct out, E input) throws NoAllowedTypeException{ + switch (input.getResulttype().getClassid()){ + case "publication": + out.setProductType(ResearchTypes.LITERATURE.label); + break; + case"dataset": + out.setProductType(ResearchTypes.RESEARCH_DATA.label); + break; + case"software": + out.setProductType(ResearchTypes.RESEARCH_SOFTWARE.label); + break; + case"other": + out.setProductType(ResearchTypes.OTHER.label); + break; + default: + throw new ClassCastException("Result type not present or not allowed"); + } + } + + private static void mapPid(ResearchProduct out, Result input) { + Optional + .ofNullable(input.getPid()) + .ifPresent( + value -> out + .setIdentifiers( + value + .stream() + .map( + p -> { + Identifier identifier = new Identifier(); + identifier.setValue(p.getValue()); + identifier.setScheme(p.getQualifier().getSchemeid()); + return identifier; + }) + .collect(Collectors.toList()))); + } + + private static void mapTitle(ResearchProduct out, Result input) throws NoTitleFoundException { + Optional> otitle = Optional.ofNullable(input.getTitle()); + if (otitle.isPresent()) { + List iTitle = otitle + .get() + .stream() + .filter(t -> t.getQualifier().getClassid().equalsIgnoreCase("main title")) + .collect(Collectors.toList()); + if (!iTitle.isEmpty()) { + out.setTitles(Arrays.asList(iTitle.get(0).getValue())); + return; + } + + iTitle = otitle + .get() + .stream() + .filter(t -> t.getQualifier().getClassid().equalsIgnoreCase("subtitle")) + .collect(Collectors.toList()); + if (!iTitle.isEmpty()) { + out.setTitles(Arrays.asList(iTitle.get(0).getValue())); + } + + + } + } + + private static void mapAbstract(ResearchProduct out, Result input) { + final List descriptionList = new ArrayList<>(); + Optional + .ofNullable(input.getDescription()) + .ifPresent(value -> value.forEach(d -> descriptionList.add(d.getValue()))); + out.setAbstracts(descriptionList); + } +} diff --git a/dump/src/main/java/eu/dnetlib/dhp/skgif/exception/NoAllowedTypeException.java b/dump/src/main/java/eu/dnetlib/dhp/skgif/exception/NoAllowedTypeException.java new file mode 100644 index 0000000..fac91bf --- /dev/null +++ b/dump/src/main/java/eu/dnetlib/dhp/skgif/exception/NoAllowedTypeException.java @@ -0,0 +1,29 @@ + +package eu.dnetlib.dhp.skgif.exception; + +public class NoAllowedTypeException extends Exception { + public NoAllowedTypeException() { + super(); + } + + public NoAllowedTypeException( + final String message, + final Throwable cause, + final boolean enableSuppression, + final boolean writableStackTrace) { + super(message, cause, enableSuppression, writableStackTrace); + } + + public NoAllowedTypeException(final String message, final Throwable cause) { + super(message, cause); + } + + public NoAllowedTypeException(final String message) { + super(message); + } + + public NoAllowedTypeException(final Throwable cause) { + super(cause); + } + +} diff --git a/dump/src/main/java/eu/dnetlib/dhp/skgif/exception/NoTitleFoundException.java b/dump/src/main/java/eu/dnetlib/dhp/skgif/exception/NoTitleFoundException.java new file mode 100644 index 0000000..5cc7591 --- /dev/null +++ b/dump/src/main/java/eu/dnetlib/dhp/skgif/exception/NoTitleFoundException.java @@ -0,0 +1,29 @@ + +package eu.dnetlib.dhp.skgif.exception; + +public class NoTitleFoundException extends Exception { + public NoTitleFoundException() { + super(); + } + + public NoTitleFoundException( + final String message, + final Throwable cause, + final boolean enableSuppression, + final boolean writableStackTrace) { + super(message, cause, enableSuppression, writableStackTrace); + } + + public NoTitleFoundException(final String message, final Throwable cause) { + super(message, cause); + } + + public NoTitleFoundException(final String message) { + super(message); + } + + public NoTitleFoundException(final Throwable cause) { + super(cause); + } + +} diff --git a/dump/src/test/java/eu/dnetlib/dhp/oa/graph/dump/ZenodoUploadTest.java b/dump/src/test/java/eu/dnetlib/dhp/oa/graph/dump/ZenodoUploadTest.java index 25aec29..1c326eb 100644 --- a/dump/src/test/java/eu/dnetlib/dhp/oa/graph/dump/ZenodoUploadTest.java +++ b/dump/src/test/java/eu/dnetlib/dhp/oa/graph/dump/ZenodoUploadTest.java @@ -17,8 +17,8 @@ import org.junit.jupiter.api.Test; import com.google.gson.Gson; import eu.dnetlib.dhp.oa.graph.dump.community.CommunityMap; -import eu.dnetlib.dhp.oa.zenodoapi.MissingConceptDoiException; -import eu.dnetlib.dhp.oa.zenodoapi.ZenodoAPIClient; +import eu.dnetlib.dhp.common.zenodoapi.MissingConceptDoiException; +import eu.dnetlib.dhp.common.zenodoapi.ZenodoAPIClient; @Disabled public class ZenodoUploadTest {