adding the new code to use the API instead of the Information Service

dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/api/QueryCommunityAPI.java

@@ -0,0 +1,83 @@
+
+package eu.dnetlib.dhp.api;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.net.HttpURLConnection;
+import java.net.URL;
+
+import org.jetbrains.annotations.NotNull;
+
+/**
+ * @author miriam.baglioni
+ * @Date 06/10/23
+ */
+public class QueryCommunityAPI {
+	private static final String PRODUCTION_BASE_URL = "https://services.openaire.eu/openaire/";
+	private static final String BETA_BASE_URL = "https://beta.services.openaire.eu/openaire/";
+
+	private static String get(String geturl) throws IOException {
+		URL url = new URL(geturl);
+		HttpURLConnection conn = (HttpURLConnection) url.openConnection();
+		conn.setDoOutput(true);
+		conn.setRequestMethod("GET");
+
+		int responseCode = conn.getResponseCode();
+		String body = getBody(conn);
+		conn.disconnect();
+		if (responseCode != HttpURLConnection.HTTP_OK)
+			throw new IOException("Unexpected code " + responseCode + body);
+
+		return body;
+	}
+
+	public static String communities(boolean production) throws IOException {
+		if (production)
+			return get(PRODUCTION_BASE_URL + "community/communities");
+		return get(BETA_BASE_URL + "community/communities");
+	}
+
+	public static String community(String id, boolean production) throws IOException {
+		if (production)
+			return get(PRODUCTION_BASE_URL + "community/" + id);
+		return get(BETA_BASE_URL + "community/" + id);
+	}
+
+	public static String communityDatasource(String id, boolean production) throws IOException {
+		if (production)
+			return get(PRODUCTION_BASE_URL + "community/" + id + "/contentproviders");
+		return (BETA_BASE_URL + "community/" + id + "/contentproviders");
+
+	}
+
+	public static String communityPropagationOrganization(String id, boolean production) throws IOException {
+		if (production)
+			return get(PRODUCTION_BASE_URL + "community/" + id + "/propagationOrganizations");
+		return get(BETA_BASE_URL + "community/" + id + "/propagationOrganizations");
+	}
+
+	public static String communityProjects(String id, String page, String size, boolean production) throws IOException {
+		if (production)
+			return get(PRODUCTION_BASE_URL + "community/" + id + "/projects/" + page + "/" + size);
+		return get(BETA_BASE_URL + "community/" + id + "/projects/" + page + "/" + size);
+	}
+
+	@NotNull
+	private static String getBody(HttpURLConnection conn) throws IOException {
+		String body = "{}";
+		try (BufferedReader br = new BufferedReader(
+			new InputStreamReader(conn.getInputStream(), "utf-8"))) {
+			StringBuilder response = new StringBuilder();
+			String responseLine = null;
+			while ((responseLine = br.readLine()) != null) {
+				response.append(responseLine.trim());
+			}
+
+			body = response.toString();
+
+		}
+		return body;
+	}
+
+}

dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/api/Utils.java

@@ -0,0 +1,169 @@
+
+package eu.dnetlib.dhp.api;
+
+import java.io.IOException;
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.stream.Collectors;
+
+import javax.management.Query;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.amazonaws.util.StringUtils;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.google.common.collect.Maps;
+
+import eu.dnetlib.dhp.api.model.*;
+import eu.dnetlib.dhp.bulktag.community.Community;
+import eu.dnetlib.dhp.bulktag.community.CommunityConfiguration;
+import eu.dnetlib.dhp.bulktag.community.Provider;
+import eu.dnetlib.dhp.bulktag.criteria.VerbResolver;
+import eu.dnetlib.dhp.bulktag.criteria.VerbResolverFactory;
+import eu.dnetlib.dhp.resulttocommunityfromorganization.SparkResultToCommunityFromOrganizationJob;
+
+/**
+ * @author miriam.baglioni
+ * @Date 09/10/23
+ */
+public class Utils implements Serializable {
+	private static final ObjectMapper MAPPER = new ObjectMapper();
+	private static final VerbResolver resolver = VerbResolverFactory.newInstance();
+
+	private static final Logger log = LoggerFactory.getLogger(Utils.class);
+
+	public static CommunityConfiguration getCommunityConfiguration(boolean production) throws IOException {
+		final Map<String, Community> communities = Maps.newHashMap();
+		List<Community> validCommunities = new ArrayList<>();
+		getValidCommunities(production)
+			.forEach(community -> {
+				try {
+					CommunityModel cm = MAPPER
+						.readValue(QueryCommunityAPI.community(community.getId(), production), CommunityModel.class);
+					validCommunities.add(getCommunity(cm));
+				} catch (IOException e) {
+					throw new RuntimeException(e);
+				}
+			});
+		validCommunities.forEach(community -> {
+			try {
+				DatasourceList dl = MAPPER
+					.readValue(
+						QueryCommunityAPI.communityDatasource(community.getId(), production), DatasourceList.class);
+				community.setProviders(dl.stream().map(d -> {
+					if (d.getEnabled() == null || Boolean.FALSE.equals(d.getEnabled()))
+						return null;
+					Provider p = new Provider();
+					p.setOpenaireId("10|" + d.getOpenaireId());
+					p.setSelectionConstraints(d.getSelectioncriteria());
+					if (p.getSelectionConstraints() != null)
+						p.getSelectionConstraints().setSelection(resolver);
+					return p;
+				})
+					.filter(Objects::nonNull)
+					.collect(Collectors.toList()));
+			} catch (IOException e) {
+				throw new RuntimeException(e);
+			}
+		});
+
+		validCommunities.forEach(community -> {
+			if (community.isValid())
+				communities.put(community.getId(), community);
+		});
+		return new CommunityConfiguration(communities);
+	}
+
+	private static Community getCommunity(CommunityModel cm) {
+		Community c = new Community();
+		c.setId(cm.getId());
+		c.setZenodoCommunities(cm.getOtherZenodoCommunities());
+		if (!StringUtils.isNullOrEmpty(cm.getZenodoCommunity()))
+			c.getZenodoCommunities().add(cm.getZenodoCommunity());
+		c.setSubjects(cm.getSubjects());
+		c.getSubjects().addAll(cm.getFos());
+		c.getSubjects().addAll(cm.getSdg());
+		if (cm.getAdvancedConstraints() != null) {
+			c.setConstraints(cm.getAdvancedConstraints());
+			c.getConstraints().setSelection(resolver);
+		}
+		if (cm.getRemoveConstraints() != null) {
+			c.setRemoveConstraints(cm.getRemoveConstraints());
+			c.getRemoveConstraints().setSelection(resolver);
+		}
+		return c;
+	}
+
+	public static List<CommunityModel> getValidCommunities(boolean production) throws IOException {
+		return MAPPER
+			.readValue(QueryCommunityAPI.communities(production), CommunitySummary.class)
+			.stream()
+			.filter(
+				community -> !community.getStatus().equals("hidden") &&
+					(community.getType().equals("ri") || community.getType().equals("community")))
+			.collect(Collectors.toList());
+	}
+
+	/**
+	 * it returns for each organization the list of associated communities
+	 */
+	public static CommunityEntityMap getCommunityOrganization(boolean production) throws IOException {
+		CommunityEntityMap organizationMap = new CommunityEntityMap();
+		getValidCommunities(production)
+			.forEach(community -> {
+				String id = community.getId();
+				try {
+					List<String> associatedOrgs = MAPPER
+						.readValue(
+							QueryCommunityAPI.communityPropagationOrganization(id, production), OrganizationList.class);
+					associatedOrgs.forEach(o -> {
+						if (!organizationMap
+							.keySet()
+							.contains(
+								"20|" + o))
+							organizationMap.put("20|" + o, new ArrayList<>());
+						organizationMap.get("20|" + o).add(community.getId());
+					});
+				} catch (IOException e) {
+					throw new RuntimeException(e);
+				}
+			});
+
+		return organizationMap;
+	}
+
+	public static CommunityEntityMap getCommunityProjects(boolean production) throws IOException {
+		CommunityEntityMap projectMap = new CommunityEntityMap();
+		getValidCommunities(production)
+			.forEach(community -> {
+				int page = -1;
+				int size = 100;
+				ContentModel cm = new ContentModel();
+				do {
+					page++;
+					try {
+						cm = MAPPER
+							.readValue(
+								QueryCommunityAPI
+									.communityProjects(
+										community.getId(), String.valueOf(page), String.valueOf(size), production),
+								ContentModel.class);
+						if (cm.getContent().size() > 0) {
+							cm.getContent().forEach(p -> {
+								if (!projectMap.keySet().contains("40|" + p.getOpenaireId()))
+									projectMap.put("40|" + p.getOpenaireId(), new ArrayList<>());
+								projectMap.get("40|" + p.getOpenaireId()).add(community.getId());
+							});
+						}
+					} catch (IOException e) {
+						throw new RuntimeException(e);
+					}
+				} while (!cm.getLast());
+			});
+		return projectMap;
+	}
+}

dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/api/model/CommunityContentprovider.java

@@ -0,0 +1,43 @@
+
+package eu.dnetlib.dhp.api.model;
+
+import com.fasterxml.jackson.annotation.JsonAutoDetect;
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import com.google.gson.Gson;
+
+import eu.dnetlib.dhp.bulktag.community.SelectionConstraints;
+
+@JsonAutoDetect
+@JsonIgnoreProperties(ignoreUnknown = true)
+public class CommunityContentprovider {
+	private String openaireId;
+	private SelectionConstraints selectioncriteria;
+
+	private String enabled;
+
+	public String getEnabled() {
+		return enabled;
+	}
+
+	public void setEnabled(String enabled) {
+		this.enabled = enabled;
+	}
+
+	public String getOpenaireId() {
+		return openaireId;
+	}
+
+	public void setOpenaireId(final String openaireId) {
+		this.openaireId = openaireId;
+	}
+
+	public SelectionConstraints getSelectioncriteria() {
+
+		return this.selectioncriteria;
+	}
+
+	public void setSelectioncriteria(SelectionConstraints selectioncriteria) {
+		this.selectioncriteria = selectioncriteria;
+
+	}
+}

dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/api/model/CommunityEntityMap.java

@@ -0,0 +1,21 @@
+
+package eu.dnetlib.dhp.api.model;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+
+public class CommunityEntityMap extends HashMap<String, List<String>> {
+
+	public CommunityEntityMap() {
+		super();
+	}
+
+	public List<String> get(String key) {
+
+		if (super.get(key) == null) {
+			return new ArrayList<>();
+		}
+		return super.get(key);
+	}
+}

dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/api/model/CommunityModel.java

@@ -0,0 +1,108 @@
+
+package eu.dnetlib.dhp.api.model;
+
+import java.io.Serializable;
+import java.util.List;
+
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+
+import eu.dnetlib.dhp.bulktag.community.SelectionConstraints;
+
+/**
+ * @author miriam.baglioni
+ * @Date 06/10/23
+ */
+@JsonIgnoreProperties(ignoreUnknown = true)
+public class CommunityModel implements Serializable {
+	private String id;
+	private String type;
+	private String status;
+
+	private String zenodoCommunity;
+	private List<String> subjects;
+	private List<String> otherZenodoCommunities;
+	private List<String> fos;
+	private List<String> sdg;
+	private SelectionConstraints advancedConstraints;
+	private SelectionConstraints removeConstraints;
+
+	public String getZenodoCommunity() {
+		return zenodoCommunity;
+	}
+
+	public void setZenodoCommunity(String zenodoCommunity) {
+		this.zenodoCommunity = zenodoCommunity;
+	}
+
+	public List<String> getSubjects() {
+		return subjects;
+	}
+
+	public void setSubjects(List<String> subjects) {
+		this.subjects = subjects;
+	}
+
+	public List<String> getOtherZenodoCommunities() {
+		return otherZenodoCommunities;
+	}
+
+	public void setOtherZenodoCommunities(List<String> otherZenodoCommunities) {
+		this.otherZenodoCommunities = otherZenodoCommunities;
+	}
+
+	public List<String> getFos() {
+		return fos;
+	}
+
+	public void setFos(List<String> fos) {
+		this.fos = fos;
+	}
+
+	public List<String> getSdg() {
+		return sdg;
+	}
+
+	public void setSdg(List<String> sdg) {
+		this.sdg = sdg;
+	}
+
+	public SelectionConstraints getRemoveConstraints() {
+		return removeConstraints;
+	}
+
+	public void setRemoveConstraints(SelectionConstraints removeConstraints) {
+		this.removeConstraints = removeConstraints;
+	}
+
+	public SelectionConstraints getAdvancedConstraints() {
+		return advancedConstraints;
+	}
+
+	public void setAdvancedConstraints(SelectionConstraints advancedConstraints) {
+		this.advancedConstraints = advancedConstraints;
+	}
+
+	public String getId() {
+		return id;
+	}
+
+	public void setId(String id) {
+		this.id = id;
+	}
+
+	public String getType() {
+		return type;
+	}
+
+	public void setType(String type) {
+		this.type = type;
+	}
+
+	public String getStatus() {
+		return status;
+	}
+
+	public void setStatus(String status) {
+		this.status = status;
+	}
+}

dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/api/model/CommunitySummary.java

@@ -0,0 +1,15 @@
+
+package eu.dnetlib.dhp.api.model;
+
+import java.io.Serializable;
+import java.util.ArrayList;
+
+/**
+ * @author miriam.baglioni
+ * @Date 06/10/23
+ */
+public class CommunitySummary extends ArrayList<CommunityModel> implements Serializable {
+	public CommunitySummary() {
+		super();
+	}
+}

dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/api/model/ContentModel.java

@@ -0,0 +1,51 @@
+
+package eu.dnetlib.dhp.api.model;
+
+import java.io.Serializable;
+import java.util.List;
+
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+
+/**
+ * @author miriam.baglioni
+ * @Date 09/10/23
+ */
+@JsonIgnoreProperties(ignoreUnknown = true)
+public class ContentModel implements Serializable {
+	private List<ProjectModel> content;
+	private Integer totalPages;
+	private Boolean last;
+	private Integer number;
+
+	public List<ProjectModel> getContent() {
+		return content;
+	}
+
+	public void setContent(List<ProjectModel> content) {
+		this.content = content;
+	}
+
+	public Integer getTotalPages() {
+		return totalPages;
+	}
+
+	public void setTotalPages(Integer totalPages) {
+		this.totalPages = totalPages;
+	}
+
+	public Boolean getLast() {
+		return last;
+	}
+
+	public void setLast(Boolean last) {
+		this.last = last;
+	}
+
+	public Integer getNumber() {
+		return number;
+	}
+
+	public void setNumber(Integer number) {
+		this.number = number;
+	}
+}

dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/api/model/DatasourceList.java

@@ -0,0 +1,13 @@
+
+package eu.dnetlib.dhp.api.model;
+
+import java.io.Serializable;
+import java.util.ArrayList;
+
+import eu.dnetlib.dhp.api.model.CommunityContentprovider;
+
+public class DatasourceList extends ArrayList<CommunityContentprovider> implements Serializable {
+	public DatasourceList() {
+		super();
+	}
+}

dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/api/model/OrganizationList.java

@@ -0,0 +1,16 @@
+
+package eu.dnetlib.dhp.api.model;
+
+import java.io.Serializable;
+import java.util.ArrayList;
+
+/**
+ * @author miriam.baglioni
+ * @Date 09/10/23
+ */
+public class OrganizationList extends ArrayList<String> implements Serializable {
+
+	public OrganizationList() {
+		super();
+	}
+}

dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/api/model/ProjectModel.java

@@ -0,0 +1,24 @@
+
+package eu.dnetlib.dhp.api.model;
+
+import java.io.Serializable;
+
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+
+/**
+ * @author miriam.baglioni
+ * @Date 09/10/23
+ */
+@JsonIgnoreProperties(ignoreUnknown = true)
+public class ProjectModel implements Serializable {
+
+	private String openaireId;
+
+	public String getOpenaireId() {
+		return openaireId;
+	}
+
+	public void setOpenaireId(String openaireId) {
+		this.openaireId = openaireId;
+	}
+}

dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/bulktag/community/QueryInformationSystem.java

@@ -1,34 +0,0 @@
-
-package eu.dnetlib.dhp.bulktag.community;
-
-import java.io.IOException;
-import java.util.List;
-
-import org.apache.commons.io.IOUtils;
-import org.dom4j.DocumentException;
-import org.xml.sax.SAXException;
-
-import com.google.common.base.Joiner;
-
-import eu.dnetlib.dhp.utils.ISLookupClientFactory;
-import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpException;
-import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpService;
-
-public class QueryInformationSystem {
-
-	public static CommunityConfiguration getCommunityConfiguration(final String isLookupUrl)
-		throws ISLookUpException, DocumentException, SAXException, IOException {
-		ISLookUpService isLookUp = ISLookupClientFactory.getLookUpService(isLookupUrl);
-		final List<String> res = isLookUp
-			.quickSearchProfile(
-				IOUtils
-					.toString(
-						QueryInformationSystem.class
-							.getResourceAsStream(
-								"/eu/dnetlib/dhp/bulktag/query.xq")));
-
-		final String xmlConf = "<communities>" + Joiner.on(" ").join(res) + "</communities>";
-
-		return CommunityConfigurationFactory.newInstance(xmlConf);
-	}
-}

dhp-workflows/dhp-enrichment/src/test/java/eu/dnetlib/dhp/api/QueryCommunityAPITest.java

@@ -0,0 +1,9 @@
+package eu.dnetlib.dhp.api;/**
+
+ * @author miriam.baglioni
+
+ * @Date 20/10/23 
+
+ */
+public class QueryCommunityAPITest {
+}