diff --git a/api/src/main/java/eu/dnetlib/dhp/communityapi/QueryCommunityAPI.java b/api/src/main/java/eu/dnetlib/dhp/communityapi/QueryCommunityAPI.java index fca6406..0b7cdaf 100644 --- a/api/src/main/java/eu/dnetlib/dhp/communityapi/QueryCommunityAPI.java +++ b/api/src/main/java/eu/dnetlib/dhp/communityapi/QueryCommunityAPI.java @@ -1,5 +1,5 @@ -package eu.dnetlib.dhp.communityapi; +package eu.dnetlib.dhp.communityapi; import java.io.BufferedReader; import java.io.IOException; @@ -12,64 +12,64 @@ import java.net.URL; * @Date 06/10/23 */ public class QueryCommunityAPI { - private static final String PRODUCTION_BASE_URL = "https://services.openaire.eu/openaire/"; + private static final String PRODUCTION_BASE_URL = "https://services.openaire.eu/openaire/"; - private static String get(String geturl) throws IOException { - URL url = new URL(geturl); - HttpURLConnection conn = (HttpURLConnection) url.openConnection(); - conn.setDoOutput(true); - conn.setRequestMethod("GET"); + private static String get(String geturl) throws IOException { + URL url = new URL(geturl); + HttpURLConnection conn = (HttpURLConnection) url.openConnection(); + conn.setDoOutput(true); + conn.setRequestMethod("GET"); - int responseCode = conn.getResponseCode(); - String body = getBody(conn); - conn.disconnect(); - if (responseCode != HttpURLConnection.HTTP_OK) - throw new IOException("Unexpected code " + responseCode + body); + int responseCode = conn.getResponseCode(); + String body = getBody(conn); + conn.disconnect(); + if (responseCode != HttpURLConnection.HTTP_OK) + throw new IOException("Unexpected code " + responseCode + body); - return body; - } + return body; + } - public static String communities() throws IOException { + public static String communities() throws IOException { - return get(PRODUCTION_BASE_URL + "community/communities"); - } + return get(PRODUCTION_BASE_URL + "community/communities"); + } - public static String community(String id) throws IOException { + public static String community(String id) throws IOException { - return get(PRODUCTION_BASE_URL + "community/" + id); + return get(PRODUCTION_BASE_URL + "community/" + id); - } + } - public static String communityDatasource(String id) throws IOException { + public static String communityDatasource(String id) throws IOException { - return get(PRODUCTION_BASE_URL + "community/" + id + "/contentproviders"); + return get(PRODUCTION_BASE_URL + "community/" + id + "/contentproviders"); - } + } - public static String communityPropagationOrganization(String id) throws IOException { + public static String communityPropagationOrganization(String id) throws IOException { - return get(PRODUCTION_BASE_URL + "community/" + id + "/propagationOrganizations"); - } + return get(PRODUCTION_BASE_URL + "community/" + id + "/propagationOrganizations"); + } - public static String communityProjects(String id, String page, String size) throws IOException { + public static String communityProjects(String id, String page, String size) throws IOException { - return get(PRODUCTION_BASE_URL + "community/" + id + "/projects/" + page + "/" + size); - } + return get(PRODUCTION_BASE_URL + "community/" + id + "/projects/" + page + "/" + size); + } - private static String getBody(HttpURLConnection conn) throws IOException { - String body = "{}"; - try (BufferedReader br = new BufferedReader( - new InputStreamReader(conn.getInputStream(), "utf-8"))) { - StringBuilder response = new StringBuilder(); - String responseLine = null; - while ((responseLine = br.readLine()) != null) { - response.append(responseLine.trim()); - } + private static String getBody(HttpURLConnection conn) throws IOException { + String body = "{}"; + try (BufferedReader br = new BufferedReader( + new InputStreamReader(conn.getInputStream(), "utf-8"))) { + StringBuilder response = new StringBuilder(); + String responseLine = null; + while ((responseLine = br.readLine()) != null) { + response.append(responseLine.trim()); + } - body = response.toString(); + body = response.toString(); - } - return body; - } + } + return body; + } } diff --git a/api/src/main/java/eu/dnetlib/dhp/communityapi/model/CommunityContentprovider.java b/api/src/main/java/eu/dnetlib/dhp/communityapi/model/CommunityContentprovider.java index a1de823..52c65b1 100644 --- a/api/src/main/java/eu/dnetlib/dhp/communityapi/model/CommunityContentprovider.java +++ b/api/src/main/java/eu/dnetlib/dhp/communityapi/model/CommunityContentprovider.java @@ -7,24 +7,24 @@ import com.fasterxml.jackson.annotation.JsonIgnoreProperties; @JsonAutoDetect @JsonIgnoreProperties(ignoreUnknown = true) public class CommunityContentprovider { - private String openaireId; + private String openaireId; - private String enabled; + private String enabled; - public String getEnabled() { - return enabled; - } + public String getEnabled() { + return enabled; + } - public void setEnabled(String enabled) { - this.enabled = enabled; - } + public void setEnabled(String enabled) { + this.enabled = enabled; + } - public String getOpenaireId() { - return openaireId; - } + public String getOpenaireId() { + return openaireId; + } - public void setOpenaireId(final String openaireId) { - this.openaireId = openaireId; - } + public void setOpenaireId(final String openaireId) { + this.openaireId = openaireId; + } } diff --git a/api/src/main/java/eu/dnetlib/dhp/communityapi/model/CommunityEntityMap.java b/api/src/main/java/eu/dnetlib/dhp/communityapi/model/CommunityEntityMap.java index efc0399..91e17e5 100644 --- a/api/src/main/java/eu/dnetlib/dhp/communityapi/model/CommunityEntityMap.java +++ b/api/src/main/java/eu/dnetlib/dhp/communityapi/model/CommunityEntityMap.java @@ -7,15 +7,15 @@ import java.util.List; public class CommunityEntityMap extends HashMap> { - public CommunityEntityMap() { - super(); - } + public CommunityEntityMap() { + super(); + } - public List get(String key) { + public List get(String key) { - if (super.get(key) == null) { - return new ArrayList<>(); - } - return super.get(key); - } + if (super.get(key) == null) { + return new ArrayList<>(); + } + return super.get(key); + } } diff --git a/api/src/main/java/eu/dnetlib/dhp/communityapi/model/CommunityModel.java b/api/src/main/java/eu/dnetlib/dhp/communityapi/model/CommunityModel.java index 144dfd7..f942eea 100644 --- a/api/src/main/java/eu/dnetlib/dhp/communityapi/model/CommunityModel.java +++ b/api/src/main/java/eu/dnetlib/dhp/communityapi/model/CommunityModel.java @@ -12,71 +12,71 @@ import com.fasterxml.jackson.annotation.JsonIgnoreProperties; */ @JsonIgnoreProperties(ignoreUnknown = true) public class CommunityModel implements Serializable { - private String id; - private String name; - private String description; + private String id; + private String name; + private String description; - private String status; + private String status; - private String type; + private String type; - private List subjects; + private List subjects; - private String zenodoCommunity; + private String zenodoCommunity; - public List getSubjects() { - return subjects; - } + public List getSubjects() { + return subjects; + } - public void setSubjects(List subjects) { - this.subjects = subjects; - } + public void setSubjects(List subjects) { + this.subjects = subjects; + } - public String getZenodoCommunity() { - return zenodoCommunity; - } + public String getZenodoCommunity() { + return zenodoCommunity; + } - public void setZenodoCommunity(String zenodoCommunity) { - this.zenodoCommunity = zenodoCommunity; - } + public void setZenodoCommunity(String zenodoCommunity) { + this.zenodoCommunity = zenodoCommunity; + } - public String getType() { - return type; - } + public String getType() { + return type; + } - public void setType(String type) { - this.type = type; - } + public void setType(String type) { + this.type = type; + } - public String getStatus() { - return status; - } + public String getStatus() { + return status; + } - public void setStatus(String status) { - this.status = status; - } + public void setStatus(String status) { + this.status = status; + } - public String getId() { - return id; - } + public String getId() { + return id; + } - public void setId(String id) { - this.id = id; - } + public void setId(String id) { + this.id = id; + } - public String getName() { - return name; - } + public String getName() { + return name; + } - public void setName(String name) { - this.name = name; - } + public void setName(String name) { + this.name = name; + } - public String getDescription() { - return description; - } + public String getDescription() { + return description; + } - public void setDescription(String description) { - this.description = description; - } + public void setDescription(String description) { + this.description = description; + } } diff --git a/api/src/main/java/eu/dnetlib/dhp/communityapi/model/CommunitySummary.java b/api/src/main/java/eu/dnetlib/dhp/communityapi/model/CommunitySummary.java index 93bbe83..47d4d1b 100644 --- a/api/src/main/java/eu/dnetlib/dhp/communityapi/model/CommunitySummary.java +++ b/api/src/main/java/eu/dnetlib/dhp/communityapi/model/CommunitySummary.java @@ -9,7 +9,7 @@ import java.util.ArrayList; * @Date 06/10/23 */ public class CommunitySummary extends ArrayList implements Serializable { - public CommunitySummary() { - super(); - } + public CommunitySummary() { + super(); + } } diff --git a/api/src/main/java/eu/dnetlib/dhp/communityapi/model/ContentModel.java b/api/src/main/java/eu/dnetlib/dhp/communityapi/model/ContentModel.java index ea0ed33..9d7245d 100644 --- a/api/src/main/java/eu/dnetlib/dhp/communityapi/model/ContentModel.java +++ b/api/src/main/java/eu/dnetlib/dhp/communityapi/model/ContentModel.java @@ -12,40 +12,40 @@ import com.fasterxml.jackson.annotation.JsonIgnoreProperties; */ @JsonIgnoreProperties(ignoreUnknown = true) public class ContentModel implements Serializable { - private List content; - private Integer totalPages; - private Boolean last; - private Integer number; + private List content; + private Integer totalPages; + private Boolean last; + private Integer number; - public List getContent() { - return content; - } + public List getContent() { + return content; + } - public void setContent(List content) { - this.content = content; - } + public void setContent(List content) { + this.content = content; + } - public Integer getTotalPages() { - return totalPages; - } + public Integer getTotalPages() { + return totalPages; + } - public void setTotalPages(Integer totalPages) { - this.totalPages = totalPages; - } + public void setTotalPages(Integer totalPages) { + this.totalPages = totalPages; + } - public Boolean getLast() { - return last; - } + public Boolean getLast() { + return last; + } - public void setLast(Boolean last) { - this.last = last; - } + public void setLast(Boolean last) { + this.last = last; + } - public Integer getNumber() { - return number; - } + public Integer getNumber() { + return number; + } - public void setNumber(Integer number) { - this.number = number; - } + public void setNumber(Integer number) { + this.number = number; + } } diff --git a/api/src/main/java/eu/dnetlib/dhp/communityapi/model/DatasourceList.java b/api/src/main/java/eu/dnetlib/dhp/communityapi/model/DatasourceList.java index 9a2f44a..0cd98a7 100644 --- a/api/src/main/java/eu/dnetlib/dhp/communityapi/model/DatasourceList.java +++ b/api/src/main/java/eu/dnetlib/dhp/communityapi/model/DatasourceList.java @@ -5,7 +5,7 @@ import java.io.Serializable; import java.util.ArrayList; public class DatasourceList extends ArrayList implements Serializable { - public DatasourceList() { - super(); - } + public DatasourceList() { + super(); + } } diff --git a/api/src/main/java/eu/dnetlib/dhp/communityapi/model/OrganizationList.java b/api/src/main/java/eu/dnetlib/dhp/communityapi/model/OrganizationList.java index 96305ff..33ccfd5 100644 --- a/api/src/main/java/eu/dnetlib/dhp/communityapi/model/OrganizationList.java +++ b/api/src/main/java/eu/dnetlib/dhp/communityapi/model/OrganizationList.java @@ -10,7 +10,7 @@ import java.util.ArrayList; */ public class OrganizationList extends ArrayList implements Serializable { - public OrganizationList() { - super(); - } + public OrganizationList() { + super(); + } } diff --git a/api/src/main/java/eu/dnetlib/dhp/communityapi/model/ProjectModel.java b/api/src/main/java/eu/dnetlib/dhp/communityapi/model/ProjectModel.java index 94b6114..72429ea 100644 --- a/api/src/main/java/eu/dnetlib/dhp/communityapi/model/ProjectModel.java +++ b/api/src/main/java/eu/dnetlib/dhp/communityapi/model/ProjectModel.java @@ -12,33 +12,33 @@ import com.fasterxml.jackson.annotation.JsonIgnoreProperties; @JsonIgnoreProperties(ignoreUnknown = true) public class ProjectModel implements Serializable { - private String openaireId; + private String openaireId; - private String funder; + private String funder; - private String gratId; + private String gratId; - public String getFunder() { - return funder; - } + public String getFunder() { + return funder; + } - public void setFunder(String funder) { - this.funder = funder; - } + public void setFunder(String funder) { + this.funder = funder; + } - public String getGratId() { - return gratId; - } + public String getGratId() { + return gratId; + } - public void setGratId(String gratId) { - this.gratId = gratId; - } + public void setGratId(String gratId) { + this.gratId = gratId; + } - public String getOpenaireId() { - return openaireId; - } + public String getOpenaireId() { + return openaireId; + } - public void setOpenaireId(String openaireId) { - this.openaireId = openaireId; - } + public void setOpenaireId(String openaireId) { + this.openaireId = openaireId; + } } diff --git a/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/UtilCommunityAPI.java b/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/UtilCommunityAPI.java index 22eca32..97d4bea 100644 --- a/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/UtilCommunityAPI.java +++ b/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/UtilCommunityAPI.java @@ -1,13 +1,9 @@ package eu.dnetlib.dhp.oa.graph.dump; -import static eu.dnetlib.dhp.utils.DHPUtils.MAPPER; - import java.io.IOException; -import java.util.ArrayList; import java.util.Arrays; import java.util.List; -import java.util.Optional; import java.util.stream.Collectors; import org.slf4j.Logger; @@ -16,129 +12,44 @@ import org.slf4j.LoggerFactory; import com.fasterxml.jackson.databind.ObjectMapper; import eu.dnetlib.dhp.communityapi.model.*; - -import eu.dnetlib.dhp.utils.DHPUtils; +import eu.dnetlib.dhp.oa.graph.dump.eosc.CommunityMap; public class UtilCommunityAPI { - private static final Logger log = LoggerFactory.getLogger(UtilCommunityAPI.class); + private static final Logger log = LoggerFactory.getLogger(UtilCommunityAPI.class); - public CommunityMap getCommunityMap(boolean singleCommunity, String communityId) - throws IOException { - if (singleCommunity) - return getMap(Arrays.asList(getCommunity(communityId))); - return getMap(getValidCommunities()); + public CommunityMap getCommunityMap(boolean singleCommunity, String communityId) + throws IOException { + if (singleCommunity) + return getMap(Arrays.asList(getCommunity(communityId))); + return getMap(getValidCommunities()); - } + } - private CommunityMap getMap(List communities) { - final CommunityMap map = new CommunityMap(); - communities.forEach(c -> map.put(c.getId(), c.getName())); - return map; - } - + private CommunityMap getMap(List communities) { + final CommunityMap map = new CommunityMap(); + communities.forEach(c -> map.put(c.getId(), c.getName())); + return map; + } - private List getValidCommunities() throws IOException { - ObjectMapper mapper = new ObjectMapper(); - return mapper - .readValue(eu.dnetlib.dhp.communityapi.QueryCommunityAPI.communities(), CommunitySummary.class) - .stream() - .filter( - community -> (community.getStatus().equals("all") || community.getStatus().equalsIgnoreCase("public")) - && - (community.getType().equals("ri") || community.getType().equals("community"))) - .collect(Collectors.toList()); + private List getValidCommunities() throws IOException { + ObjectMapper mapper = new ObjectMapper(); + return mapper + .readValue(eu.dnetlib.dhp.communityapi.QueryCommunityAPI.communities(), CommunitySummary.class) + .stream() + .filter( + community -> (community.getStatus().equals("all") || community.getStatus().equalsIgnoreCase("public")) + && + (community.getType().equals("ri") || community.getType().equals("community"))) + .collect(Collectors.toList()); - } + } - private CommunityModel getCommunity(String id) throws IOException { - ObjectMapper mapper = new ObjectMapper(); - return mapper - .readValue(eu.dnetlib.dhp.communityapi.QueryCommunityAPI.community(id), CommunityModel.class); + private CommunityModel getCommunity(String id) throws IOException { + ObjectMapper mapper = new ObjectMapper(); + return mapper + .readValue(eu.dnetlib.dhp.communityapi.QueryCommunityAPI.community(id), CommunityModel.class); - } - - - - private List getDatasourceList(String id) { - List datasourceList = new ArrayList<>(); - try { - - new ObjectMapper() - .readValue( - eu.dnetlib.dhp.communityapi.QueryCommunityAPI.communityDatasource(id), - DatasourceList.class) - .stream() - .forEach(ds -> { - if (Optional.ofNullable(ds.getOpenaireId()).isPresent()) { - - datasourceList.add(ds.getOpenaireId()); - } - - }); - - } catch (IOException e) { - throw new RuntimeException(e); - } - return datasourceList; - } - - private List getProjectList(String id) { - int page = -1; - int size = 100; - ContentModel cm = null; - ; - ArrayList projectList = new ArrayList<>(); - do { - page++; - try { - cm = new ObjectMapper() - .readValue( - eu.dnetlib.dhp.communityapi.QueryCommunityAPI - .communityProjects( - id, String.valueOf(page), String.valueOf(size)), - ContentModel.class); - if (cm.getContent().size() > 0) { - cm.getContent().forEach(p -> { - if (Optional.ofNullable(p.getOpenaireId()).isPresent()) - projectList.add(p.getOpenaireId()); - - }); - } - } catch (IOException e) { - throw new RuntimeException(e); - } - } while (!cm.getLast()); - - return projectList; - } - - /** - * it returns for each organization the list of associated communities - */ - public CommunityEntityMap getCommunityOrganization() throws IOException { - CommunityEntityMap organizationMap = new CommunityEntityMap(); - getValidCommunities() - .forEach(community -> { - String id = community.getId(); - try { - List associatedOrgs = MAPPER - .readValue( - eu.dnetlib.dhp.communityapi.QueryCommunityAPI.communityPropagationOrganization(id), - OrganizationList.class); - associatedOrgs.forEach(o -> { - if (!organizationMap - .keySet() - .contains(o)) - organizationMap.put(o, new ArrayList<>()); - organizationMap.get(o).add(community.getId()); - }); - } catch (IOException e) { - throw new RuntimeException(e); - } - }); - - return organizationMap; - } + } } diff --git a/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/eosc/SaveCommunityMap.java b/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/eosc/SaveCommunityMap.java index 5caedf4..ce07028 100644 --- a/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/eosc/SaveCommunityMap.java +++ b/dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/eosc/SaveCommunityMap.java @@ -8,7 +8,6 @@ import java.io.Serializable; import java.nio.charset.StandardCharsets; import java.util.Optional; -import eu.dnetlib.dhp.oa.graph.dump.UtilCommunityAPI; import org.apache.commons.io.IOUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSDataOutputStream; @@ -18,6 +17,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import eu.dnetlib.dhp.application.ArgumentApplicationParser; +import eu.dnetlib.dhp.oa.graph.dump.UtilCommunityAPI; /** * This class connects with the community APIs for production. It saves the information about the @@ -51,10 +51,10 @@ public class SaveCommunityMap implements Serializable { public static void main(String[] args) throws Exception { String jsonConfiguration = IOUtils - .toString( - SaveCommunityMap.class - .getResourceAsStream( - "/eu/dnetlib/dhp/oa/graph/dump/input_cm_parameters.json")); + .toString( + SaveCommunityMap.class + .getResourceAsStream( + "/eu/dnetlib/dhp/oa/graph/dump/eosc_cm_parameters.json")); final ArgumentApplicationParser parser = new ArgumentApplicationParser(jsonConfiguration); parser.parseArgument(args); @@ -66,9 +66,9 @@ public class SaveCommunityMap implements Serializable { log.info("outputPath: {}", outputPath); final Boolean singleCommunity = Optional - .ofNullable(parser.get("singleDeposition")) - .map(Boolean::valueOf) - .orElse(false); + .ofNullable(parser.get("singleDeposition")) + .map(Boolean::valueOf) + .orElse(false); final String community_id = Optional.ofNullable(parser.get("communityId")).orElse(null); @@ -79,15 +79,14 @@ public class SaveCommunityMap implements Serializable { } private void saveCommunityMap(boolean singleCommunity, String communityId) - throws IOException { + throws IOException { final String communityMapString = Utils.OBJECT_MAPPER - .writeValueAsString(queryInformationSystem.getCommunityMap(singleCommunity, communityId)); + .writeValueAsString(queryInformationSystem.getCommunityMap(singleCommunity, communityId)); log.info("communityMap {} ", communityMapString); writer - .write( - communityMapString); + .write( + communityMapString); writer.close(); } } - diff --git a/dump/src/main/resources/eu/dnetlib/dhp/oa/graph/dump/eosc/oozie_app/workflow.xml b/dump/src/main/resources/eu/dnetlib/dhp/oa/graph/dump/eosc/oozie_app/workflow.xml index 9b34569..843cffc 100644 --- a/dump/src/main/resources/eu/dnetlib/dhp/oa/graph/dump/eosc/oozie_app/workflow.xml +++ b/dump/src/main/resources/eu/dnetlib/dhp/oa/graph/dump/eosc/oozie_app/workflow.xml @@ -95,7 +95,6 @@ eu.dnetlib.dhp.oa.graph.dump.eosc.SaveCommunityMap --outputPath${workingDir}/communityMap --nameNode${nameNode} - --isLookUpUrl${isLookUpUrl} diff --git a/dump/src/main/resources/eu/dnetlib/dhp/oa/graph/dump/eosc_cm_parameters.json b/dump/src/main/resources/eu/dnetlib/dhp/oa/graph/dump/eosc_cm_parameters.json index 6e42bfa..a33ec9d 100644 --- a/dump/src/main/resources/eu/dnetlib/dhp/oa/graph/dump/eosc_cm_parameters.json +++ b/dump/src/main/resources/eu/dnetlib/dhp/oa/graph/dump/eosc_cm_parameters.json @@ -1,12 +1,7 @@ [ - { - "paramName":"is", - "paramLongName":"isLookUpUrl", - "paramDescription": "URL of the isLookUp Service", - "paramRequired": true - }, + { "paramName":"nn", "paramLongName":"nameNode",