D-Net
/
dnet-hadoop
14
5
Fork 11
Code Issues 14 Pull Requests 13 Projects 2 Releases 2 Wiki Activity

[SubCommunity] Extention of the Utils methods to add also the associations between the subcommunities and organization/project/datasources

This commit is contained in:
Miriam Baglioni 2024-11-20 10:59:49 +01:00
parent 9dbcf19efb
commit 6beb94adee
1 changed files with 70 additions and 46 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

116
dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/api/Utils.java
View File

@ -8,8 +8,6 @@ import java.util.stream.Collectors;
import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.StringUtils;
import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.NotNull;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.collect.Maps; import com.google.common.collect.Maps;
@ -93,15 +91,14 @@ public class Utils implements Serializable {
private static @NotNull Community getSubCommunityConfiguration(String baseURL, String communityId, SubCommunityModel sc) { private static @NotNull Community getSubCommunityConfiguration(String baseURL, String communityId, SubCommunityModel sc) {
Community c = getCommunity(sc); Community c = getCommunity(sc);
c.setProviders(getSubcommunityDatasources(baseURL, communityId, sc.getSubCommunityId())); c.setProviders(getRelevantDatasources(baseURL, communityId, sc.getSubCommunityId()));
return c; return c;
} }
private static List<Provider> getSubcommunityDatasources(String baseURL, String communityId, String subcommunityId) { private static List<Provider> getRelevantDatasources(String baseURL, String communityId, String subcommunityId) {
try { try {
DatasourceList dl = null; DatasourceList dl = MAPPER
dl = MAPPER
.readValue( .readValue(
QueryCommunityAPI.subcommunityDatasource(communityId, subcommunityId, baseURL), DatasourceList.class); QueryCommunityAPI.subcommunityDatasource(communityId, subcommunityId, baseURL), DatasourceList.class);
return dl.stream().map(d -> { return dl.stream().map(d -> {
@ -239,50 +236,77 @@ public class Utils implements Serializable {
return projectMap; return projectMap;
} }
private static void addRelevantProjects(String communityId, String baseURL, CommunityEntityMap communityEntityMap){ private static void addRelevantProjects(
int page = -1; String communityId,
int size = 100; String baseURL,
ContentModel cm = new ContentModel(); CommunityEntityMap communityEntityMap
do { ) {
page++; fetchAndProcessProjects(
try { (page, size) -> {
cm = MAPPER try {
.readValue( return QueryCommunityAPI.communityProjects(communityId, String.valueOf(page), String.valueOf(size), baseURL);
QueryCommunityAPI } catch (IOException e) {
.communityProjects( throw new RuntimeException(e);
communityId, String.valueOf(page), String.valueOf(size), baseURL), }
ContentModel.class); },
if (!cm.getContent().isEmpty()) { communityId,
cm.getContent().forEach(p -> updateEntityMap(communityId, p.getOpenaireId(),communityEntityMap, ModelSupport.getIdPrefix(Project.class))); communityEntityMap
} );
} catch (IOException e) {
throw new RuntimeException(e);
}
} while (!cm.getLast());
} }
private static void addRelevantProjects(String communityId, String subcommunityId, String baseURL, CommunityEntityMap communityEntityMap){ private static void addRelevantProjects(
int page = -1; String communityId,
int size = 100; String subcommunityId,
ContentModel cm = new ContentModel(); String baseURL,
do { CommunityEntityMap communityEntityMap
page++; ) {
try { fetchAndProcessProjects(
cm = MAPPER (page, size) -> {
.readValue( try {
QueryCommunityAPI return QueryCommunityAPI.subcommunityProjects(communityId, subcommunityId, String.valueOf(page), String.valueOf(size), baseURL);
.subcommunityProjects( } catch (IOException e) {
communityId, subcommunityId , String.valueOf(page), String.valueOf(size), baseURL), throw new RuntimeException(e);
ContentModel.class); }
if (!cm.getContent().isEmpty()) { },
cm.getContent().forEach(p -> updateEntityMap(communityId, p.getOpenaireId(),communityEntityMap, ModelSupport.getIdPrefix(Project.class))); communityId,
} communityEntityMap
} catch (IOException e) { );
throw new RuntimeException(e);
}
} while (!cm.getLast());
} }
@FunctionalInterface
private interface ProjectQueryFunction {
String query(int page, int size);
}
private static void fetchAndProcessProjects(
ProjectQueryFunction projectQueryFunction,
String communityId,
CommunityEntityMap communityEntityMap
) {
int page = 0;
final int size = 100;
ContentModel contentModel;
do {
try {
String response = projectQueryFunction.query(page, size);
contentModel = MAPPER.readValue(response, ContentModel.class);
if (!contentModel.getContent().isEmpty()) {
contentModel.getContent().forEach(project ->
updateEntityMap(
communityId,
project.getOpenaireId(),
communityEntityMap,
ModelSupport.getIdPrefix(Project.class)
)
);
}
} catch (IOException e) {
throw new RuntimeException("Error processing projects for community: " + communityId, e);
}
page++;
} while (!contentModel.getLast());
}
public static List<String> getCommunityIdList(String baseURL) throws IOException { public static List<String> getCommunityIdList(String baseURL) throws IOException {
return getValidCommunities(baseURL) return getValidCommunities(baseURL)
.stream() .stream()