D-Net
/
dhp-graph-dump
4
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

refactoring

This commit is contained in:
Miriam Baglioni 2024-01-04 12:17:28 +01:00
parent f6f734bf5e
commit 9435c6756e
13 changed files with 214 additions and 310 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

84
api/src/main/java/eu/dnetlib/dhp/communityapi/QueryCommunityAPI.java
View File

@ -1,5 +1,5 @@
package eu.dnetlib.dhp.communityapi;
package eu.dnetlib.dhp.communityapi;
import java.io.BufferedReader; import java.io.BufferedReader;
import java.io.IOException; import java.io.IOException;
@ -12,64 +12,64 @@ import java.net.URL;
* @Date 06/10/23 * @Date 06/10/23
*/ */
public class QueryCommunityAPI { public class QueryCommunityAPI {
private static final String PRODUCTION_BASE_URL = "https://services.openaire.eu/openaire/"; private static final String PRODUCTION_BASE_URL = "https://services.openaire.eu/openaire/";
private static String get(String geturl) throws IOException { private static String get(String geturl) throws IOException {
URL url = new URL(geturl); URL url = new URL(geturl);
HttpURLConnection conn = (HttpURLConnection) url.openConnection(); HttpURLConnection conn = (HttpURLConnection) url.openConnection();
conn.setDoOutput(true); conn.setDoOutput(true);
conn.setRequestMethod("GET"); conn.setRequestMethod("GET");
int responseCode = conn.getResponseCode(); int responseCode = conn.getResponseCode();
String body = getBody(conn); String body = getBody(conn);
conn.disconnect(); conn.disconnect();
if (responseCode != HttpURLConnection.HTTP_OK) if (responseCode != HttpURLConnection.HTTP_OK)
throw new IOException("Unexpected code " + responseCode + body); throw new IOException("Unexpected code " + responseCode + body);
return body; return body;
} }
public static String communities() throws IOException { public static String communities() throws IOException {
return get(PRODUCTION_BASE_URL + "community/communities"); return get(PRODUCTION_BASE_URL + "community/communities");
} }
public static String community(String id) throws IOException { public static String community(String id) throws IOException {
return get(PRODUCTION_BASE_URL + "community/" + id); return get(PRODUCTION_BASE_URL + "community/" + id);
} }
public static String communityDatasource(String id) throws IOException { public static String communityDatasource(String id) throws IOException {
return get(PRODUCTION_BASE_URL + "community/" + id + "/contentproviders"); return get(PRODUCTION_BASE_URL + "community/" + id + "/contentproviders");
} }
public static String communityPropagationOrganization(String id) throws IOException { public static String communityPropagationOrganization(String id) throws IOException {
return get(PRODUCTION_BASE_URL + "community/" + id + "/propagationOrganizations"); return get(PRODUCTION_BASE_URL + "community/" + id + "/propagationOrganizations");
} }
public static String communityProjects(String id, String page, String size) throws IOException { public static String communityProjects(String id, String page, String size) throws IOException {
return get(PRODUCTION_BASE_URL + "community/" + id + "/projects/" + page + "/" + size); return get(PRODUCTION_BASE_URL + "community/" + id + "/projects/" + page + "/" + size);
} }
private static String getBody(HttpURLConnection conn) throws IOException { private static String getBody(HttpURLConnection conn) throws IOException {
String body = "{}"; String body = "{}";
try (BufferedReader br = new BufferedReader( try (BufferedReader br = new BufferedReader(
new InputStreamReader(conn.getInputStream(), "utf-8"))) { new InputStreamReader(conn.getInputStream(), "utf-8"))) {
StringBuilder response = new StringBuilder(); StringBuilder response = new StringBuilder();
String responseLine = null; String responseLine = null;
while ((responseLine = br.readLine()) != null) { while ((responseLine = br.readLine()) != null) {
response.append(responseLine.trim()); response.append(responseLine.trim());
} }
body = response.toString(); body = response.toString();
} }
return body; return body;
} }
} }

28
api/src/main/java/eu/dnetlib/dhp/communityapi/model/CommunityContentprovider.java
View File

@ -7,24 +7,24 @@ import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
@JsonAutoDetect @JsonAutoDetect
@JsonIgnoreProperties(ignoreUnknown = true) @JsonIgnoreProperties(ignoreUnknown = true)
public class CommunityContentprovider { public class CommunityContentprovider {
private String openaireId; private String openaireId;
private String enabled; private String enabled;
public String getEnabled() { public String getEnabled() {
return enabled; return enabled;
} }
public void setEnabled(String enabled) { public void setEnabled(String enabled) {
this.enabled = enabled; this.enabled = enabled;
} }
public String getOpenaireId() { public String getOpenaireId() {
return openaireId; return openaireId;
} }
public void setOpenaireId(final String openaireId) { public void setOpenaireId(final String openaireId) {
this.openaireId = openaireId; this.openaireId = openaireId;
} }
} }

18
api/src/main/java/eu/dnetlib/dhp/communityapi/model/CommunityEntityMap.java
View File

@ -7,15 +7,15 @@ import java.util.List;
public class CommunityEntityMap extends HashMap<String, List<String>> { public class CommunityEntityMap extends HashMap<String, List<String>> {
public CommunityEntityMap() { public CommunityEntityMap() {
super(); super();
} }
public List<String> get(String key) { public List<String> get(String key) {
if (super.get(key) == null) { if (super.get(key) == null) {
return new ArrayList<>(); return new ArrayList<>();
} }
return super.get(key); return super.get(key);
} }
} }

98
api/src/main/java/eu/dnetlib/dhp/communityapi/model/CommunityModel.java
View File

@ -12,71 +12,71 @@ import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
*/ */
@JsonIgnoreProperties(ignoreUnknown = true) @JsonIgnoreProperties(ignoreUnknown = true)
public class CommunityModel implements Serializable { public class CommunityModel implements Serializable {
private String id; private String id;
private String name; private String name;
private String description; private String description;
private String status; private String status;
private String type; private String type;
private List<String> subjects; private List<String> subjects;
private String zenodoCommunity; private String zenodoCommunity;
public List<String> getSubjects() { public List<String> getSubjects() {
return subjects; return subjects;
} }
public void setSubjects(List<String> subjects) { public void setSubjects(List<String> subjects) {
this.subjects = subjects; this.subjects = subjects;
} }
public String getZenodoCommunity() { public String getZenodoCommunity() {
return zenodoCommunity; return zenodoCommunity;
} }
public void setZenodoCommunity(String zenodoCommunity) { public void setZenodoCommunity(String zenodoCommunity) {
this.zenodoCommunity = zenodoCommunity; this.zenodoCommunity = zenodoCommunity;
} }
public String getType() { public String getType() {
return type; return type;
} }
public void setType(String type) { public void setType(String type) {
this.type = type; this.type = type;
} }
public String getStatus() { public String getStatus() {
return status; return status;
} }
public void setStatus(String status) { public void setStatus(String status) {
this.status = status; this.status = status;
} }
public String getId() { public String getId() {
return id; return id;
} }
public void setId(String id) { public void setId(String id) {
this.id = id; this.id = id;
} }
public String getName() { public String getName() {
return name; return name;
} }
public void setName(String name) { public void setName(String name) {
this.name = name; this.name = name;
} }
public String getDescription() { public String getDescription() {
return description; return description;
} }
public void setDescription(String description) { public void setDescription(String description) {
this.description = description; this.description = description;
} }
} }

6
api/src/main/java/eu/dnetlib/dhp/communityapi/model/CommunitySummary.java
View File

@ -9,7 +9,7 @@ import java.util.ArrayList;
* @Date 06/10/23 * @Date 06/10/23
*/ */
public class CommunitySummary extends ArrayList<CommunityModel> implements Serializable { public class CommunitySummary extends ArrayList<CommunityModel> implements Serializable {
public CommunitySummary() { public CommunitySummary() {
super(); super();
} }
} }

56
api/src/main/java/eu/dnetlib/dhp/communityapi/model/ContentModel.java
View File

@ -12,40 +12,40 @@ import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
*/ */
@JsonIgnoreProperties(ignoreUnknown = true) @JsonIgnoreProperties(ignoreUnknown = true)
public class ContentModel implements Serializable { public class ContentModel implements Serializable {
private List<ProjectModel> content; private List<ProjectModel> content;
private Integer totalPages; private Integer totalPages;
private Boolean last; private Boolean last;
private Integer number; private Integer number;
public List<ProjectModel> getContent() { public List<ProjectModel> getContent() {
return content; return content;
} }
public void setContent(List<ProjectModel> content) { public void setContent(List<ProjectModel> content) {
this.content = content; this.content = content;
} }
public Integer getTotalPages() { public Integer getTotalPages() {
return totalPages; return totalPages;
} }
public void setTotalPages(Integer totalPages) { public void setTotalPages(Integer totalPages) {
this.totalPages = totalPages; this.totalPages = totalPages;
} }
public Boolean getLast() { public Boolean getLast() {
return last; return last;
} }
public void setLast(Boolean last) { public void setLast(Boolean last) {
this.last = last; this.last = last;
} }
public Integer getNumber() { public Integer getNumber() {
return number; return number;
} }
public void setNumber(Integer number) { public void setNumber(Integer number) {
this.number = number; this.number = number;
} }
} }

6
api/src/main/java/eu/dnetlib/dhp/communityapi/model/DatasourceList.java
View File

@ -5,7 +5,7 @@ import java.io.Serializable;
import java.util.ArrayList; import java.util.ArrayList;
public class DatasourceList extends ArrayList<CommunityContentprovider> implements Serializable { public class DatasourceList extends ArrayList<CommunityContentprovider> implements Serializable {
public DatasourceList() { public DatasourceList() {
super(); super();
} }
} }

6
api/src/main/java/eu/dnetlib/dhp/communityapi/model/OrganizationList.java
View File

@ -10,7 +10,7 @@ import java.util.ArrayList;
*/ */
public class OrganizationList extends ArrayList<String> implements Serializable { public class OrganizationList extends ArrayList<String> implements Serializable {
public OrganizationList() { public OrganizationList() {
super(); super();
} }
} }

42
api/src/main/java/eu/dnetlib/dhp/communityapi/model/ProjectModel.java
View File

@ -12,33 +12,33 @@ import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
@JsonIgnoreProperties(ignoreUnknown = true) @JsonIgnoreProperties(ignoreUnknown = true)
public class ProjectModel implements Serializable { public class ProjectModel implements Serializable {
private String openaireId; private String openaireId;
private String funder; private String funder;
private String gratId; private String gratId;
public String getFunder() { public String getFunder() {
return funder; return funder;
} }
public void setFunder(String funder) { public void setFunder(String funder) {
this.funder = funder; this.funder = funder;
} }
public String getGratId() { public String getGratId() {
return gratId; return gratId;
} }
public void setGratId(String gratId) { public void setGratId(String gratId) {
this.gratId = gratId; this.gratId = gratId;
} }
public String getOpenaireId() { public String getOpenaireId() {
return openaireId; return openaireId;
} }
public void setOpenaireId(String openaireId) { public void setOpenaireId(String openaireId) {
this.openaireId = openaireId; this.openaireId = openaireId;
} }
} }

147
dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/UtilCommunityAPI.java
View File

@ -1,13 +1,9 @@
package eu.dnetlib.dhp.oa.graph.dump; package eu.dnetlib.dhp.oa.graph.dump;
import static eu.dnetlib.dhp.utils.DHPUtils.MAPPER;
import java.io.IOException; import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays; import java.util.Arrays;
import java.util.List; import java.util.List;
import java.util.Optional;
import java.util.stream.Collectors; import java.util.stream.Collectors;
import org.slf4j.Logger; import org.slf4j.Logger;
@ -16,129 +12,44 @@ import org.slf4j.LoggerFactory;
import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectMapper;
import eu.dnetlib.dhp.communityapi.model.*; import eu.dnetlib.dhp.communityapi.model.*;
import eu.dnetlib.dhp.oa.graph.dump.eosc.CommunityMap;
import eu.dnetlib.dhp.utils.DHPUtils;
public class UtilCommunityAPI { public class UtilCommunityAPI {
private static final Logger log = LoggerFactory.getLogger(UtilCommunityAPI.class); private static final Logger log = LoggerFactory.getLogger(UtilCommunityAPI.class);
public CommunityMap getCommunityMap(boolean singleCommunity, String communityId) public CommunityMap getCommunityMap(boolean singleCommunity, String communityId)
throws IOException { throws IOException {
if (singleCommunity) if (singleCommunity)
return getMap(Arrays.asList(getCommunity(communityId))); return getMap(Arrays.asList(getCommunity(communityId)));
return getMap(getValidCommunities()); return getMap(getValidCommunities());
} }
private CommunityMap getMap(List<CommunityModel> communities) { private CommunityMap getMap(List<CommunityModel> communities) {
final CommunityMap map = new CommunityMap(); final CommunityMap map = new CommunityMap();
communities.forEach(c -> map.put(c.getId(), c.getName())); communities.forEach(c -> map.put(c.getId(), c.getName()));
return map; return map;
} }
private List<CommunityModel> getValidCommunities() throws IOException { private List<CommunityModel> getValidCommunities() throws IOException {
ObjectMapper mapper = new ObjectMapper(); ObjectMapper mapper = new ObjectMapper();
return mapper return mapper
.readValue(eu.dnetlib.dhp.communityapi.QueryCommunityAPI.communities(), CommunitySummary.class) .readValue(eu.dnetlib.dhp.communityapi.QueryCommunityAPI.communities(), CommunitySummary.class)
.stream() .stream()
.filter( .filter(
community -> (community.getStatus().equals("all") || community.getStatus().equalsIgnoreCase("public")) community -> (community.getStatus().equals("all") || community.getStatus().equalsIgnoreCase("public"))
&& &&
(community.getType().equals("ri") || community.getType().equals("community"))) (community.getType().equals("ri") || community.getType().equals("community")))
.collect(Collectors.toList()); .collect(Collectors.toList());
} }
private CommunityModel getCommunity(String id) throws IOException { private CommunityModel getCommunity(String id) throws IOException {
ObjectMapper mapper = new ObjectMapper(); ObjectMapper mapper = new ObjectMapper();
return mapper return mapper
.readValue(eu.dnetlib.dhp.communityapi.QueryCommunityAPI.community(id), CommunityModel.class); .readValue(eu.dnetlib.dhp.communityapi.QueryCommunityAPI.community(id), CommunityModel.class);
} }
private List<String> getDatasourceList(String id) {
List<String> datasourceList = new ArrayList<>();
try {
new ObjectMapper()
.readValue(
eu.dnetlib.dhp.communityapi.QueryCommunityAPI.communityDatasource(id),
DatasourceList.class)
.stream()
.forEach(ds -> {
if (Optional.ofNullable(ds.getOpenaireId()).isPresent()) {
datasourceList.add(ds.getOpenaireId());
}
});
} catch (IOException e) {
throw new RuntimeException(e);
}
return datasourceList;
}
private List<String> getProjectList(String id) {
int page = -1;
int size = 100;
ContentModel cm = null;
;
ArrayList<String> projectList = new ArrayList<>();
do {
page++;
try {
cm = new ObjectMapper()
.readValue(
eu.dnetlib.dhp.communityapi.QueryCommunityAPI
.communityProjects(
id, String.valueOf(page), String.valueOf(size)),
ContentModel.class);
if (cm.getContent().size() > 0) {
cm.getContent().forEach(p -> {
if (Optional.ofNullable(p.getOpenaireId()).isPresent())
projectList.add(p.getOpenaireId());
});
}
} catch (IOException e) {
throw new RuntimeException(e);
}
} while (!cm.getLast());
return projectList;
}
/**
* it returns for each organization the list of associated communities
*/
public CommunityEntityMap getCommunityOrganization() throws IOException {
CommunityEntityMap organizationMap = new CommunityEntityMap();
getValidCommunities()
.forEach(community -> {
String id = community.getId();
try {
List<String> associatedOrgs = MAPPER
.readValue(
eu.dnetlib.dhp.communityapi.QueryCommunityAPI.communityPropagationOrganization(id),
OrganizationList.class);
associatedOrgs.forEach(o -> {
if (!organizationMap
.keySet()
.contains(o))
organizationMap.put(o, new ArrayList<>());
organizationMap.get(o).add(community.getId());
});
} catch (IOException e) {
throw new RuntimeException(e);
}
});
return organizationMap;
}
} }

25
dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/eosc/SaveCommunityMap.java
View File

@ -8,7 +8,6 @@ import java.io.Serializable;
import java.nio.charset.StandardCharsets; import java.nio.charset.StandardCharsets;
import java.util.Optional; import java.util.Optional;
import eu.dnetlib.dhp.oa.graph.dump.UtilCommunityAPI;
import org.apache.commons.io.IOUtils; import org.apache.commons.io.IOUtils;
import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FSDataOutputStream;
@ -18,6 +17,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import eu.dnetlib.dhp.application.ArgumentApplicationParser; import eu.dnetlib.dhp.application.ArgumentApplicationParser;
import eu.dnetlib.dhp.oa.graph.dump.UtilCommunityAPI;
/** /**
* This class connects with the community APIs for production. It saves the information about the * This class connects with the community APIs for production. It saves the information about the
@ -51,10 +51,10 @@ public class SaveCommunityMap implements Serializable {
public static void main(String[] args) throws Exception { public static void main(String[] args) throws Exception {
String jsonConfiguration = IOUtils String jsonConfiguration = IOUtils
.toString( .toString(
SaveCommunityMap.class SaveCommunityMap.class
.getResourceAsStream( .getResourceAsStream(
"/eu/dnetlib/dhp/oa/graph/dump/input_cm_parameters.json")); "/eu/dnetlib/dhp/oa/graph/dump/eosc_cm_parameters.json"));
final ArgumentApplicationParser parser = new ArgumentApplicationParser(jsonConfiguration); final ArgumentApplicationParser parser = new ArgumentApplicationParser(jsonConfiguration);
parser.parseArgument(args); parser.parseArgument(args);
@ -66,9 +66,9 @@ public class SaveCommunityMap implements Serializable {
log.info("outputPath: {}", outputPath); log.info("outputPath: {}", outputPath);
final Boolean singleCommunity = Optional final Boolean singleCommunity = Optional
.ofNullable(parser.get("singleDeposition")) .ofNullable(parser.get("singleDeposition"))
.map(Boolean::valueOf) .map(Boolean::valueOf)
.orElse(false); .orElse(false);
final String community_id = Optional.ofNullable(parser.get("communityId")).orElse(null); final String community_id = Optional.ofNullable(parser.get("communityId")).orElse(null);
@ -79,15 +79,14 @@ public class SaveCommunityMap implements Serializable {
} }
private void saveCommunityMap(boolean singleCommunity, String communityId) private void saveCommunityMap(boolean singleCommunity, String communityId)
throws IOException { throws IOException {
final String communityMapString = Utils.OBJECT_MAPPER final String communityMapString = Utils.OBJECT_MAPPER
.writeValueAsString(queryInformationSystem.getCommunityMap(singleCommunity, communityId)); .writeValueAsString(queryInformationSystem.getCommunityMap(singleCommunity, communityId));
log.info("communityMap {} ", communityMapString); log.info("communityMap {} ", communityMapString);
writer writer
.write( .write(
communityMapString); communityMapString);
writer.close(); writer.close();
} }
} }

1
dump/src/main/resources/eu/dnetlib/dhp/oa/graph/dump/eosc/oozie_app/workflow.xml
View File

@ -95,7 +95,6 @@
<main-class>eu.dnetlib.dhp.oa.graph.dump.eosc.SaveCommunityMap</main-class> <main-class>eu.dnetlib.dhp.oa.graph.dump.eosc.SaveCommunityMap</main-class>
<arg>--outputPath</arg><arg>${workingDir}/communityMap</arg> <arg>--outputPath</arg><arg>${workingDir}/communityMap</arg>
<arg>--nameNode</arg><arg>${nameNode}</arg> <arg>--nameNode</arg><arg>${nameNode}</arg>
<arg>--isLookUpUrl</arg><arg>${isLookUpUrl}</arg>
</java> </java>
<ok to="get_ds_master_duplicate"/> <ok to="get_ds_master_duplicate"/>
<error to="Kill"/> <error to="Kill"/>

7
dump/src/main/resources/eu/dnetlib/dhp/oa/graph/dump/eosc_cm_parameters.json
View File

@ -1,12 +1,7 @@
[ [
{
"paramName":"is",
"paramLongName":"isLookUpUrl",
"paramDescription": "URL of the isLookUp Service",
"paramRequired": true
},
{ {
"paramName":"nn", "paramName":"nn",
"paramLongName":"nameNode", "paramLongName":"nameNode",