refactoring
This commit is contained in:
parent
f6f734bf5e
commit
9435c6756e
|
|
@ -1,5 +1,5 @@
|
|||
package eu.dnetlib.dhp.communityapi;
|
||||
|
||||
package eu.dnetlib.dhp.communityapi;
|
||||
|
||||
import java.io.BufferedReader;
|
||||
import java.io.IOException;
|
||||
|
|
|
|||
|
|
@ -1,13 +1,9 @@
|
|||
|
||||
package eu.dnetlib.dhp.oa.graph.dump;
|
||||
|
||||
import static eu.dnetlib.dhp.utils.DHPUtils.MAPPER;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.List;
|
||||
import java.util.Optional;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import org.slf4j.Logger;
|
||||
|
|
@ -16,8 +12,7 @@ import org.slf4j.LoggerFactory;
|
|||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
|
||||
import eu.dnetlib.dhp.communityapi.model.*;
|
||||
|
||||
import eu.dnetlib.dhp.utils.DHPUtils;
|
||||
import eu.dnetlib.dhp.oa.graph.dump.eosc.CommunityMap;
|
||||
|
||||
public class UtilCommunityAPI {
|
||||
|
||||
|
|
@ -37,7 +32,6 @@ public class UtilCommunityAPI {
|
|||
return map;
|
||||
}
|
||||
|
||||
|
||||
private List<CommunityModel> getValidCommunities() throws IOException {
|
||||
ObjectMapper mapper = new ObjectMapper();
|
||||
return mapper
|
||||
|
|
@ -58,87 +52,4 @@ public class UtilCommunityAPI {
|
|||
|
||||
}
|
||||
|
||||
|
||||
|
||||
private List<String> getDatasourceList(String id) {
|
||||
List<String> datasourceList = new ArrayList<>();
|
||||
try {
|
||||
|
||||
new ObjectMapper()
|
||||
.readValue(
|
||||
eu.dnetlib.dhp.communityapi.QueryCommunityAPI.communityDatasource(id),
|
||||
DatasourceList.class)
|
||||
.stream()
|
||||
.forEach(ds -> {
|
||||
if (Optional.ofNullable(ds.getOpenaireId()).isPresent()) {
|
||||
|
||||
datasourceList.add(ds.getOpenaireId());
|
||||
}
|
||||
|
||||
});
|
||||
|
||||
} catch (IOException e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
return datasourceList;
|
||||
}
|
||||
|
||||
private List<String> getProjectList(String id) {
|
||||
int page = -1;
|
||||
int size = 100;
|
||||
ContentModel cm = null;
|
||||
;
|
||||
ArrayList<String> projectList = new ArrayList<>();
|
||||
do {
|
||||
page++;
|
||||
try {
|
||||
cm = new ObjectMapper()
|
||||
.readValue(
|
||||
eu.dnetlib.dhp.communityapi.QueryCommunityAPI
|
||||
.communityProjects(
|
||||
id, String.valueOf(page), String.valueOf(size)),
|
||||
ContentModel.class);
|
||||
if (cm.getContent().size() > 0) {
|
||||
cm.getContent().forEach(p -> {
|
||||
if (Optional.ofNullable(p.getOpenaireId()).isPresent())
|
||||
projectList.add(p.getOpenaireId());
|
||||
|
||||
});
|
||||
}
|
||||
} catch (IOException e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
} while (!cm.getLast());
|
||||
|
||||
return projectList;
|
||||
}
|
||||
|
||||
/**
|
||||
* it returns for each organization the list of associated communities
|
||||
*/
|
||||
public CommunityEntityMap getCommunityOrganization() throws IOException {
|
||||
CommunityEntityMap organizationMap = new CommunityEntityMap();
|
||||
getValidCommunities()
|
||||
.forEach(community -> {
|
||||
String id = community.getId();
|
||||
try {
|
||||
List<String> associatedOrgs = MAPPER
|
||||
.readValue(
|
||||
eu.dnetlib.dhp.communityapi.QueryCommunityAPI.communityPropagationOrganization(id),
|
||||
OrganizationList.class);
|
||||
associatedOrgs.forEach(o -> {
|
||||
if (!organizationMap
|
||||
.keySet()
|
||||
.contains(o))
|
||||
organizationMap.put(o, new ArrayList<>());
|
||||
organizationMap.get(o).add(community.getId());
|
||||
});
|
||||
} catch (IOException e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
});
|
||||
|
||||
return organizationMap;
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -8,7 +8,6 @@ import java.io.Serializable;
|
|||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.Optional;
|
||||
|
||||
import eu.dnetlib.dhp.oa.graph.dump.UtilCommunityAPI;
|
||||
import org.apache.commons.io.IOUtils;
|
||||
import org.apache.hadoop.conf.Configuration;
|
||||
import org.apache.hadoop.fs.FSDataOutputStream;
|
||||
|
|
@ -18,6 +17,7 @@ import org.slf4j.Logger;
|
|||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import eu.dnetlib.dhp.application.ArgumentApplicationParser;
|
||||
import eu.dnetlib.dhp.oa.graph.dump.UtilCommunityAPI;
|
||||
|
||||
/**
|
||||
* This class connects with the community APIs for production. It saves the information about the
|
||||
|
|
@ -54,7 +54,7 @@ public class SaveCommunityMap implements Serializable {
|
|||
.toString(
|
||||
SaveCommunityMap.class
|
||||
.getResourceAsStream(
|
||||
"/eu/dnetlib/dhp/oa/graph/dump/input_cm_parameters.json"));
|
||||
"/eu/dnetlib/dhp/oa/graph/dump/eosc_cm_parameters.json"));
|
||||
|
||||
final ArgumentApplicationParser parser = new ArgumentApplicationParser(jsonConfiguration);
|
||||
parser.parseArgument(args);
|
||||
|
|
@ -90,4 +90,3 @@ public class SaveCommunityMap implements Serializable {
|
|||
}
|
||||
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -95,7 +95,6 @@
|
|||
<main-class>eu.dnetlib.dhp.oa.graph.dump.eosc.SaveCommunityMap</main-class>
|
||||
<arg>--outputPath</arg><arg>${workingDir}/communityMap</arg>
|
||||
<arg>--nameNode</arg><arg>${nameNode}</arg>
|
||||
<arg>--isLookUpUrl</arg><arg>${isLookUpUrl}</arg>
|
||||
</java>
|
||||
<ok to="get_ds_master_duplicate"/>
|
||||
<error to="Kill"/>
|
||||
|
|
|
|||
|
|
@ -1,12 +1,7 @@
|
|||
|
||||
[
|
||||
|
||||
{
|
||||
"paramName":"is",
|
||||
"paramLongName":"isLookUpUrl",
|
||||
"paramDescription": "URL of the isLookUp Service",
|
||||
"paramRequired": true
|
||||
},
|
||||
|
||||
{
|
||||
"paramName":"nn",
|
||||
"paramLongName":"nameNode",
|
||||
|
|
|
|||
Loading…
Reference in New Issue