[SubCommunity] Extention of CommunityAPIs fro bulk tagging

dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/api/QueryCommunityAPI.java

@@ -43,9 +43,16 @@ public class QueryCommunityAPI {
 
 	}
 
+	public static String subcommunities(String communityId, String baseURL) throws IOException {
+
+		return get(baseURL + communityId + "/subcommunities");
+
+	}
+
+
 	public static String communityDatasource(String id, String baseURL) throws IOException {
 
-		return get(baseURL + id + "/contentproviders");
+		return get(baseURL + id + "/datasources");
 
 	}
 
@@ -78,4 +85,7 @@ public class QueryCommunityAPI {
 		return body;
 	}
 
+	public static String subcommunityDatasource(String communityId, String subcommunityId, String baseURL) throws IOException {
+		return get(baseURL + communityId + "/subcommunities/datasources?subCommunityId=" + subcommunityId);
+	}
 }

dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/api/Utils.java

@@ -33,17 +33,17 @@ public class Utils implements Serializable {
 	private static final ObjectMapper MAPPER = new ObjectMapper();
 	private static final VerbResolver resolver = VerbResolverFactory.newInstance();
 
-	private static final Logger log = LoggerFactory.getLogger(Utils.class);
-
 	public static CommunityConfiguration getCommunityConfiguration(String baseURL) throws IOException {
 		final Map<String, Community> communities = Maps.newHashMap();
+		List<CommunityModel> communityList = getValidCommunities(baseURL);
 		List<Community> validCommunities = new ArrayList<>();
-		getValidCommunities(baseURL)
+		communityList
 			.forEach(community -> {
 				try {
 					CommunityModel cm = MAPPER
 						.readValue(QueryCommunityAPI.community(community.getId(), baseURL), CommunityModel.class);
 					validCommunities.add(getCommunity(cm));
+
 				} catch (IOException e) {
 					throw new RuntimeException(e);
 				}
@@ -70,31 +70,90 @@ public class Utils implements Serializable {
 			}
 		});
 
+		//add subcommunities information if any
+		communityList.forEach(community -> {
+			try {
+				List<SubCommunityModel> subcommunities = getSubcommunities(community.getId(), baseURL);
+				subcommunities.forEach(sc ->
+						validCommunities.add(getSubCommunityConfiguration(baseURL, community.getId(), sc)));
+			} catch (IOException e) {
+				throw new RuntimeException(e);
+			}
+
+		});
+
 		validCommunities.forEach(community -> {
 			if (community.isValid())
 				communities.put(community.getId(), community);
 		});
+
+
 		return new CommunityConfiguration(communities);
 	}
 
-	private static Community getCommunity(CommunityModel cm) {
+	private static @NotNull Community getSubCommunityConfiguration(String baseURL, String communityId, SubCommunityModel sc) {
+		Community c = getCommunity(sc);
+		c.setProviders(getSubcommunityDatasources(baseURL, communityId, sc.getSubCommunityId()));
+
+		return c;
+	}
+
+	private static List<Provider> getSubcommunityDatasources(String baseURL, String communityId, String subcommunityId) {
+		try {
+		DatasourceList dl = null;
+			dl = MAPPER
+					.readValue(
+							QueryCommunityAPI.subcommunityDatasource(communityId, subcommunityId, baseURL), DatasourceList.class);
+			return dl.stream().map(d -> {
+						if (d.getEnabled() == null || Boolean.FALSE.equals(d.getEnabled()))
+							return null;
+						Provider p = new Provider();
+						p.setOpenaireId(ModelSupport.getIdPrefix(Datasource.class) + "|" + d.getOpenaireId());
+						p.setSelectionConstraints(d.getSelectioncriteria());
+						if (p.getSelectionConstraints() != null)
+							p.getSelectionConstraints().setSelection(resolver);
+						return p;
+					})
+					.filter(Objects::nonNull)
+					.collect(Collectors.toList());
+		} catch (IOException e) {
+			throw new RuntimeException(e);
+		}
+	}
+
+	private static <C extends CommonConfigurationModel> Community getCommonConfiguration(C input){
 		Community c = new Community();
-		c.setId(cm.getId());
+		c.setZenodoCommunities(input.getOtherZenodoCommunities());
-		c.setZenodoCommunities(cm.getOtherZenodoCommunities());
+		if (StringUtils.isNotBlank(input.getZenodoCommunity()))
-		if (StringUtils.isNotBlank(cm.getZenodoCommunity()))
+			c.getZenodoCommunities().add(input.getZenodoCommunity());
-			c.getZenodoCommunities().add(cm.getZenodoCommunity());
+		c.setSubjects(input.getSubjects());
-		c.setSubjects(cm.getSubjects());
+		c.getSubjects().addAll(input.getFos());
-		c.getSubjects().addAll(cm.getFos());
+		c.getSubjects().addAll(input.getSdg());
-		c.getSubjects().addAll(cm.getSdg());
+		if (input.getAdvancedConstraints() != null) {
-		if (cm.getAdvancedConstraints() != null) {
+			c.setConstraints(input.getAdvancedConstraints());
-			c.setConstraints(cm.getAdvancedConstraints());
 			c.getConstraints().setSelection(resolver);
 		}
-		if (cm.getRemoveConstraints() != null) {
+		if (input.getRemoveConstraints() != null) {
-			c.setRemoveConstraints(cm.getRemoveConstraints());
+			c.setRemoveConstraints(input.getRemoveConstraints());
 			c.getRemoveConstraints().setSelection(resolver);
 		}
 		return c;
+
+	}
+
+	private static Community getCommunity(SubCommunityModel sc) {
+		Community c = getCommonConfiguration(sc);
+		c.setId(sc.getSubCommunityId());
+		return c;
+	}
+
+
+
+	private static Community getCommunity(CommunityModel cm) {
+		Community c = getCommonConfiguration(cm);
+		c.setId(cm.getId());
+
+		return c;
 	}
 
 	public static List<CommunityModel> getValidCommunities(String baseURL) throws IOException {
@@ -107,6 +166,10 @@ public class Utils implements Serializable {
 			.collect(Collectors.toList());
 	}
 
+	public static List<SubCommunityModel> getSubcommunities(String communityId, String baseURL) throws IOException {
+		return MAPPER.readValue(QueryCommunityAPI.subcommunities(communityId, baseURL), SubCommunitySummary.class);
+	}
+
 	/**
 	 * it returns for each organization the list of associated communities
 	 */

dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/api/model/CommonConfigurationModel.java

@@ -0,0 +1,74 @@
+package eu.dnetlib.dhp.api.model;
+
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import eu.dnetlib.dhp.bulktag.community.SelectionConstraints;
+
+import java.io.Serializable;
+import java.util.List;
+
+@JsonIgnoreProperties(ignoreUnknown = true)
+public class CommonConfigurationModel implements Serializable {
+    private String zenodoCommunity;
+    private List<String> subjects;
+    private List<String> otherZenodoCommunities;
+    private List<String> fos;
+    private List<String> sdg;
+    private SelectionConstraints advancedConstraints;
+    private SelectionConstraints removeConstraints;
+
+    public String getZenodoCommunity() {
+        return zenodoCommunity;
+    }
+
+    public void setZenodoCommunity(String zenodoCommunity) {
+        this.zenodoCommunity = zenodoCommunity;
+    }
+
+    public List<String> getSubjects() {
+        return subjects;
+    }
+
+    public void setSubjects(List<String> subjects) {
+        this.subjects = subjects;
+    }
+
+    public List<String> getOtherZenodoCommunities() {
+        return otherZenodoCommunities;
+    }
+
+    public void setOtherZenodoCommunities(List<String> otherZenodoCommunities) {
+        this.otherZenodoCommunities = otherZenodoCommunities;
+    }
+
+    public List<String> getFos() {
+        return fos;
+    }
+
+    public void setFos(List<String> fos) {
+        this.fos = fos;
+    }
+
+    public List<String> getSdg() {
+        return sdg;
+    }
+
+    public void setSdg(List<String> sdg) {
+        this.sdg = sdg;
+    }
+
+    public SelectionConstraints getRemoveConstraints() {
+        return removeConstraints;
+    }
+
+    public void setRemoveConstraints(SelectionConstraints removeConstraints) {
+        this.removeConstraints = removeConstraints;
+    }
+
+    public SelectionConstraints getAdvancedConstraints() {
+        return advancedConstraints;
+    }
+
+    public void setAdvancedConstraints(SelectionConstraints advancedConstraints) {
+        this.advancedConstraints = advancedConstraints;
+    }
+}

dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/api/model/CommunityModel.java

@@ -13,75 +13,11 @@ import eu.dnetlib.dhp.bulktag.community.SelectionConstraints;
  * @Date 06/10/23
  */
 @JsonIgnoreProperties(ignoreUnknown = true)
-public class CommunityModel implements Serializable {
+public class CommunityModel extends CommonConfigurationModel implements Serializable {
 	private String id;
 	private String type;
 	private String status;
 
-	private String zenodoCommunity;
-	private List<String> subjects;
-	private List<String> otherZenodoCommunities;
-	private List<String> fos;
-	private List<String> sdg;
-	private SelectionConstraints advancedConstraints;
-	private SelectionConstraints removeConstraints;
-
-	public String getZenodoCommunity() {
-		return zenodoCommunity;
-	}
-
-	public void setZenodoCommunity(String zenodoCommunity) {
-		this.zenodoCommunity = zenodoCommunity;
-	}
-
-	public List<String> getSubjects() {
-		return subjects;
-	}
-
-	public void setSubjects(List<String> subjects) {
-		this.subjects = subjects;
-	}
-
-	public List<String> getOtherZenodoCommunities() {
-		return otherZenodoCommunities;
-	}
-
-	public void setOtherZenodoCommunities(List<String> otherZenodoCommunities) {
-		this.otherZenodoCommunities = otherZenodoCommunities;
-	}
-
-	public List<String> getFos() {
-		return fos;
-	}
-
-	public void setFos(List<String> fos) {
-		this.fos = fos;
-	}
-
-	public List<String> getSdg() {
-		return sdg;
-	}
-
-	public void setSdg(List<String> sdg) {
-		this.sdg = sdg;
-	}
-
-	public SelectionConstraints getRemoveConstraints() {
-		return removeConstraints;
-	}
-
-	public void setRemoveConstraints(SelectionConstraints removeConstraints) {
-		this.removeConstraints = removeConstraints;
-	}
-
-	public SelectionConstraints getAdvancedConstraints() {
-		return advancedConstraints;
-	}
-
-	public void setAdvancedConstraints(SelectionConstraints advancedConstraints) {
-		this.advancedConstraints = advancedConstraints;
-	}
-
 	public String getId() {
 		return id;
 	}

dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/api/model/SubCommunityModel.java

@@ -0,0 +1,18 @@
+package eu.dnetlib.dhp.api.model;
+
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+
+import java.io.Serializable;
+
+@JsonIgnoreProperties(ignoreUnknown = true)
+public class SubCommunityModel extends CommonConfigurationModel implements Serializable {
+    private String subCommunityId;
+
+    public String getSubCommunityId() {
+        return subCommunityId;
+    }
+
+    public void setSubCommunityId(String subCommunityId) {
+        this.subCommunityId = subCommunityId;
+    }
+}

dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/api/model/SubCommunitySummary.java

@@ -0,0 +1,10 @@
+package eu.dnetlib.dhp.api.model;
+
+import java.io.Serializable;
+import java.util.ArrayList;
+
+public class SubCommunitySummary extends ArrayList<SubCommunityModel> implements Serializable {
+    public SubCommunitySummary(){
+        super();
+    }
+}

dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/bulktag/SparkBulkTagJob.java

@@ -101,7 +101,7 @@ public class SparkBulkTagJob {
 			cc = CommunityConfigurationFactory.newInstance(taggingConf);
 		} else {
 			cc = Utils.getCommunityConfiguration(baseURL);
-			log.info(OBJECT_MAPPER.writeValueAsString(cc));
+
 		}
 
 		runWithSparkSession(

dhp-workflows/dhp-enrichment/src/test/java/eu/dnetlib/dhp/bulktag/BulkTagJobTest.java

@@ -9,6 +9,13 @@ import java.nio.file.Path;
 import java.util.Arrays;
 import java.util.List;
 
+import com.fasterxml.jackson.core.JsonProcessingException;
+import eu.dnetlib.dhp.api.QueryCommunityAPI;
+import eu.dnetlib.dhp.api.Utils;
+import eu.dnetlib.dhp.api.model.CommunityModel;
+import eu.dnetlib.dhp.api.model.SubCommunityModel;
+import eu.dnetlib.dhp.api.model.SubCommunitySummary;
+import eu.dnetlib.dhp.bulktag.community.CommunityConfiguration;
 import org.apache.commons.io.FileUtils;
 import org.apache.commons.io.IOUtils;
 import org.apache.hadoop.conf.Configuration;
@@ -1949,4 +1956,19 @@ public class BulkTagJobTest {
 
 	}
 
+	@Test
+	public void testApi() throws IOException {
+		String baseURL = "https://dev-openaire.d4science.org/openaire/community/";
+		List<SubCommunityModel> subcommunities = Utils.getSubcommunities("clarin", baseURL);
+
+		CommunityConfiguration tmp = Utils.getCommunityConfiguration(baseURL);
+				tmp.getCommunities().keySet().forEach(c -> {
+                    try {
+                        System.out.println(new ObjectMapper().writeValueAsString(tmp.getCommunities().get(c)));
+                    } catch (JsonProcessingException e) {
+                        throw new RuntimeException(e);
+                    }
+                });
+	}
+
 }