From eb407ba0d3c0b19a289112d8b5f7cff8424162ea Mon Sep 17 00:00:00 2001 From: "miriam.baglioni" Date: Wed, 25 Oct 2023 14:58:19 +0200 Subject: [PATCH] first try to remove the IS FROM THE CODE --- api/pom.xml | 44 +++++++++ .../dhp/communityapi/QueryCommunityAPI.java | 59 ++++++++++++ .../eu/dnetlib/dhp/communityapi/Utils.java | 93 +++++++++++++++++++ .../model/CommunityContentprovider.java | 30 ++++++ .../model/CommunityEntityMap.java | 21 +++++ .../communityapi/model/CommunityModel.java | 62 +++++++++++++ .../communityapi/model/CommunitySummary.java | 15 +++ .../dhp/communityapi/model/ContentModel.java | 51 ++++++++++ .../communityapi/model/DatasourceList.java | 11 +++ .../communityapi/model/OrganizationList.java | 16 ++++ .../dhp/communityapi/model/ProjectModel.java | 24 +++++ .../dhp/oa/graph/dump/SendToZenodoHDFS.java | 4 +- pom.xml | 1 + 13 files changed, 429 insertions(+), 2 deletions(-) create mode 100644 api/pom.xml create mode 100644 api/src/main/java/eu/dnetlib/dhp/communityapi/QueryCommunityAPI.java create mode 100644 api/src/main/java/eu/dnetlib/dhp/communityapi/Utils.java create mode 100644 api/src/main/java/eu/dnetlib/dhp/communityapi/model/CommunityContentprovider.java create mode 100644 api/src/main/java/eu/dnetlib/dhp/communityapi/model/CommunityEntityMap.java create mode 100644 api/src/main/java/eu/dnetlib/dhp/communityapi/model/CommunityModel.java create mode 100644 api/src/main/java/eu/dnetlib/dhp/communityapi/model/CommunitySummary.java create mode 100644 api/src/main/java/eu/dnetlib/dhp/communityapi/model/ContentModel.java create mode 100644 api/src/main/java/eu/dnetlib/dhp/communityapi/model/DatasourceList.java create mode 100644 api/src/main/java/eu/dnetlib/dhp/communityapi/model/OrganizationList.java create mode 100644 api/src/main/java/eu/dnetlib/dhp/communityapi/model/ProjectModel.java diff --git a/api/pom.xml b/api/pom.xml new file mode 100644 index 0000000..0109977 --- /dev/null +++ b/api/pom.xml @@ -0,0 +1,44 @@ + + + 4.0.0 + + + eu.dnetlib.dhp + dhp-graph-dump + 1.2.5-SNAPSHOT + + + eu.dnetlib.dhp + api + 1.2.5-SNAPSHOT + + + 8 + 8 + + + + + + + dom4j + dom4j + + + + jaxen + jaxen + + + + eu.dnetlib.dhp + dhp-common + ${project.version} + + + + + + \ No newline at end of file diff --git a/api/src/main/java/eu/dnetlib/dhp/communityapi/QueryCommunityAPI.java b/api/src/main/java/eu/dnetlib/dhp/communityapi/QueryCommunityAPI.java new file mode 100644 index 0000000..a5148ea --- /dev/null +++ b/api/src/main/java/eu/dnetlib/dhp/communityapi/QueryCommunityAPI.java @@ -0,0 +1,59 @@ + +package eu.dnetlib.dhp.communityapi; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStreamReader; +import java.net.HttpURLConnection; +import java.net.URL; + +/** + * @author miriam.baglioni + * @Date 06/10/23 + */ +public class QueryCommunityAPI { + private static final String PRODUCTION_BASE_URL = "https://services.openaire.eu/openaire/"; + + private static String get(String geturl) throws IOException { + URL url = new URL(geturl); + HttpURLConnection conn = (HttpURLConnection) url.openConnection(); + conn.setDoOutput(true); + conn.setRequestMethod("GET"); + + int responseCode = conn.getResponseCode(); + String body = getBody(conn); + conn.disconnect(); + if (responseCode != HttpURLConnection.HTTP_OK) + throw new IOException("Unexpected code " + responseCode + body); + + return body; + } + + public static String communities() throws IOException { + + return get(PRODUCTION_BASE_URL + "community/communities"); + } + + public static String community(String id) throws IOException { + + return get(PRODUCTION_BASE_URL + "community/" + id); + + } + + private static String getBody(HttpURLConnection conn) throws IOException { + String body = "{}"; + try (BufferedReader br = new BufferedReader( + new InputStreamReader(conn.getInputStream(), "utf-8"))) { + StringBuilder response = new StringBuilder(); + String responseLine = null; + while ((responseLine = br.readLine()) != null) { + response.append(responseLine.trim()); + } + + body = response.toString(); + + } + return body; + } + +} diff --git a/api/src/main/java/eu/dnetlib/dhp/communityapi/Utils.java b/api/src/main/java/eu/dnetlib/dhp/communityapi/Utils.java new file mode 100644 index 0000000..30cb46b --- /dev/null +++ b/api/src/main/java/eu/dnetlib/dhp/communityapi/Utils.java @@ -0,0 +1,93 @@ + +package eu.dnetlib.dhp.communityapi; + +import java.io.IOException; +import java.io.Serializable; +import java.util.List; +import java.util.stream.Collectors; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.fasterxml.jackson.databind.ObjectMapper; + +import eu.dnetlib.dhp.communityapi.model.CommunitySummary; + +/** + * @author miriam.baglioni + * @Date 09/10/23 + */ +public class Utils implements Serializable { + private static final Logger log = LoggerFactory.getLogger(Utils.class); + + public static List getValidCommunities() throws IOException { + ObjectMapper mapper = new ObjectMapper(); + return mapper + .readValue(QueryCommunityAPI.communities(), CommunitySummary.class) + .stream() + .filter( + community -> community.getStatus().equals("all") && + (community.getType().equals("ri") || community.getType().equals("community"))) + .collect(Collectors.toList()); + + } + + /** + * it returns for each organization the list of associated communities + */ +// public static CommunityEntityMap getCommunityOrganization() throws IOException { +// CommunityEntityMap organizationMap = new CommunityEntityMap(); +// getValidCommunities(production) +// .forEach(community -> { +// String id = community.getId(); +// try { +// List associatedOrgs = MAPPER +// .readValue( +// QueryCommunityAPI.communityPropagationOrganization(id, production), OrganizationList.class); +// associatedOrgs.forEach(o -> { +// if (!organizationMap +// .keySet() +// .contains( +// "20|" + o)) +// organizationMap.put("20|" + o, new ArrayList<>()); +// organizationMap.get("20|" + o).add(community.getId()); +// }); +// } catch (IOException e) { +// throw new RuntimeException(e); +// } +// }); +// +// return organizationMap; +// } +// +// public static CommunityEntityMap getCommunityProjects(boolean production) throws IOException { +// CommunityEntityMap projectMap = new CommunityEntityMap(); +// getValidCommunities(production) +// .forEach(community -> { +// int page = -1; +// int size = 100; +// ContentModel cm = new ContentModel(); +// do { +// page++; +// try { +// cm = MAPPER +// .readValue( +// QueryCommunityAPI +// .communityProjects( +// community.getId(), String.valueOf(page), String.valueOf(size), production), +// ContentModel.class); +// if (cm.getContent().size() > 0) { +// cm.getContent().forEach(p -> { +// if (!projectMap.keySet().contains("40|" + p.getOpenaireId())) +// projectMap.put("40|" + p.getOpenaireId(), new ArrayList<>()); +// projectMap.get("40|" + p.getOpenaireId()).add(community.getId()); +// }); +// } +// } catch (IOException e) { +// throw new RuntimeException(e); +// } +// } while (!cm.getLast()); +// }); +// return projectMap; +// } +} diff --git a/api/src/main/java/eu/dnetlib/dhp/communityapi/model/CommunityContentprovider.java b/api/src/main/java/eu/dnetlib/dhp/communityapi/model/CommunityContentprovider.java new file mode 100644 index 0000000..52c65b1 --- /dev/null +++ b/api/src/main/java/eu/dnetlib/dhp/communityapi/model/CommunityContentprovider.java @@ -0,0 +1,30 @@ + +package eu.dnetlib.dhp.communityapi.model; + +import com.fasterxml.jackson.annotation.JsonAutoDetect; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; + +@JsonAutoDetect +@JsonIgnoreProperties(ignoreUnknown = true) +public class CommunityContentprovider { + private String openaireId; + + private String enabled; + + public String getEnabled() { + return enabled; + } + + public void setEnabled(String enabled) { + this.enabled = enabled; + } + + public String getOpenaireId() { + return openaireId; + } + + public void setOpenaireId(final String openaireId) { + this.openaireId = openaireId; + } + +} diff --git a/api/src/main/java/eu/dnetlib/dhp/communityapi/model/CommunityEntityMap.java b/api/src/main/java/eu/dnetlib/dhp/communityapi/model/CommunityEntityMap.java new file mode 100644 index 0000000..91e17e5 --- /dev/null +++ b/api/src/main/java/eu/dnetlib/dhp/communityapi/model/CommunityEntityMap.java @@ -0,0 +1,21 @@ + +package eu.dnetlib.dhp.communityapi.model; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; + +public class CommunityEntityMap extends HashMap> { + + public CommunityEntityMap() { + super(); + } + + public List get(String key) { + + if (super.get(key) == null) { + return new ArrayList<>(); + } + return super.get(key); + } +} diff --git a/api/src/main/java/eu/dnetlib/dhp/communityapi/model/CommunityModel.java b/api/src/main/java/eu/dnetlib/dhp/communityapi/model/CommunityModel.java new file mode 100644 index 0000000..1d86308 --- /dev/null +++ b/api/src/main/java/eu/dnetlib/dhp/communityapi/model/CommunityModel.java @@ -0,0 +1,62 @@ + +package eu.dnetlib.dhp.communityapi.model; + +import java.io.Serializable; +import java.util.List; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; + +/** + * @author miriam.baglioni + * @Date 06/10/23 + */ +@JsonIgnoreProperties(ignoreUnknown = true) +public class CommunityModel implements Serializable { + private String id; + private String label; + private String description; + + private String status; + + private String type; + + public String getType() { + return type; + } + + public void setType(String type) { + this.type = type; + } + + public String getStatus() { + return status; + } + + public void setStatus(String status) { + this.status = status; + } + + public String getId() { + return id; + } + + public void setId(String id) { + this.id = id; + } + + public String getLabel() { + return label; + } + + public void setLabel(String label) { + this.label = label; + } + + public String getDescription() { + return description; + } + + public void setDescription(String description) { + this.description = description; + } +} diff --git a/api/src/main/java/eu/dnetlib/dhp/communityapi/model/CommunitySummary.java b/api/src/main/java/eu/dnetlib/dhp/communityapi/model/CommunitySummary.java new file mode 100644 index 0000000..47d4d1b --- /dev/null +++ b/api/src/main/java/eu/dnetlib/dhp/communityapi/model/CommunitySummary.java @@ -0,0 +1,15 @@ + +package eu.dnetlib.dhp.communityapi.model; + +import java.io.Serializable; +import java.util.ArrayList; + +/** + * @author miriam.baglioni + * @Date 06/10/23 + */ +public class CommunitySummary extends ArrayList implements Serializable { + public CommunitySummary() { + super(); + } +} diff --git a/api/src/main/java/eu/dnetlib/dhp/communityapi/model/ContentModel.java b/api/src/main/java/eu/dnetlib/dhp/communityapi/model/ContentModel.java new file mode 100644 index 0000000..9d7245d --- /dev/null +++ b/api/src/main/java/eu/dnetlib/dhp/communityapi/model/ContentModel.java @@ -0,0 +1,51 @@ + +package eu.dnetlib.dhp.communityapi.model; + +import java.io.Serializable; +import java.util.List; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; + +/** + * @author miriam.baglioni + * @Date 09/10/23 + */ +@JsonIgnoreProperties(ignoreUnknown = true) +public class ContentModel implements Serializable { + private List content; + private Integer totalPages; + private Boolean last; + private Integer number; + + public List getContent() { + return content; + } + + public void setContent(List content) { + this.content = content; + } + + public Integer getTotalPages() { + return totalPages; + } + + public void setTotalPages(Integer totalPages) { + this.totalPages = totalPages; + } + + public Boolean getLast() { + return last; + } + + public void setLast(Boolean last) { + this.last = last; + } + + public Integer getNumber() { + return number; + } + + public void setNumber(Integer number) { + this.number = number; + } +} diff --git a/api/src/main/java/eu/dnetlib/dhp/communityapi/model/DatasourceList.java b/api/src/main/java/eu/dnetlib/dhp/communityapi/model/DatasourceList.java new file mode 100644 index 0000000..0cd98a7 --- /dev/null +++ b/api/src/main/java/eu/dnetlib/dhp/communityapi/model/DatasourceList.java @@ -0,0 +1,11 @@ + +package eu.dnetlib.dhp.communityapi.model; + +import java.io.Serializable; +import java.util.ArrayList; + +public class DatasourceList extends ArrayList implements Serializable { + public DatasourceList() { + super(); + } +} diff --git a/api/src/main/java/eu/dnetlib/dhp/communityapi/model/OrganizationList.java b/api/src/main/java/eu/dnetlib/dhp/communityapi/model/OrganizationList.java new file mode 100644 index 0000000..33ccfd5 --- /dev/null +++ b/api/src/main/java/eu/dnetlib/dhp/communityapi/model/OrganizationList.java @@ -0,0 +1,16 @@ + +package eu.dnetlib.dhp.communityapi.model; + +import java.io.Serializable; +import java.util.ArrayList; + +/** + * @author miriam.baglioni + * @Date 09/10/23 + */ +public class OrganizationList extends ArrayList implements Serializable { + + public OrganizationList() { + super(); + } +} diff --git a/api/src/main/java/eu/dnetlib/dhp/communityapi/model/ProjectModel.java b/api/src/main/java/eu/dnetlib/dhp/communityapi/model/ProjectModel.java new file mode 100644 index 0000000..f5aa967 --- /dev/null +++ b/api/src/main/java/eu/dnetlib/dhp/communityapi/model/ProjectModel.java @@ -0,0 +1,24 @@ + +package eu.dnetlib.dhp.communityapi.model; + +import java.io.Serializable; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; + +/** + * @author miriam.baglioni + * @Date 09/10/23 + */ +@JsonIgnoreProperties(ignoreUnknown = true) +public class ProjectModel implements Serializable { + + private String openaireId; + + public String getOpenaireId() { + return openaireId; + } + + public void setOpenaireId(String openaireId) { + this.openaireId = openaireId; + } +} diff --git a/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/SendToZenodoHDFS.java b/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/SendToZenodoHDFS.java index 502bb5c..29a3ab5 100644 --- a/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/SendToZenodoHDFS.java +++ b/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/SendToZenodoHDFS.java @@ -101,8 +101,8 @@ public class SendToZenodoHDFS implements Serializable { try { response_code = zenodoApiClient - .uploadIS3(inputStream, name, fileSystem.getFileStatus(p).getLen()); - }catch (Exception e){ + .uploadIS3(inputStream, name, fileSystem.getFileStatus(p).getLen()); + } catch (Exception e) { log.info(e.getMessage()); throw new RuntimeException("Error while uploading on Zenodo"); } diff --git a/pom.xml b/pom.xml index 00b5e97..eb45552 100644 --- a/pom.xml +++ b/pom.xml @@ -6,6 +6,7 @@ dump-schema dump + api