[maven-release-plugin] prepare for next development iteration

Reviewed-on: #20

apps/bioschemas-api/pom.xml

@@ -3,7 +3,7 @@
     <parent>
         <groupId>eu.dnetlib.dhp</groupId>
         <artifactId>apps</artifactId>
-        <version>3.4.3-SNAPSHOT</version>
+        <version>3.5.5-SNAPSHOT</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
 

apps/dhp-broker-application/pom.xml

@@ -4,7 +4,7 @@
 	<parent>
 		<groupId>eu.dnetlib.dhp</groupId>
 		<artifactId>apps</artifactId>
-		<version>3.4.3-SNAPSHOT</version>
+		<version>3.5.5-SNAPSHOT</version>
 		<relativePath>../pom.xml</relativePath>
 	</parent>
 

apps/dhp-broker-public-application/pom.xml

@@ -4,7 +4,7 @@
 	<parent>
 		<groupId>eu.dnetlib.dhp</groupId>
 		<artifactId>apps</artifactId>
-		<version>3.4.3-SNAPSHOT</version>
+		<version>3.5.5-SNAPSHOT</version>
 		<relativePath>../pom.xml</relativePath>
 	</parent>
 

apps/dhp-broker-public-application/src/main/java/eu/dnetlib/broker/oa/controllers/EventFeedback.java

@@ -2,6 +2,8 @@ package eu.dnetlib.broker.oa.controllers;
 
 import java.io.Serializable;
 
+import eu.dnetlib.broker.common.feedbacks.FeedbackStatus;
+
 public class EventFeedback implements Serializable {
 
 	/**
@@ -10,27 +12,22 @@ public class EventFeedback implements Serializable {
 	private static final long serialVersionUID = -6967719685282712195L;
 
 	private String eventId;
-	private String status;
 
-	// TOOD status should be an enum having the following values:
-	// * DISCARDED: the event was not processable by the system. OpenAIRE should not interpret such status in a negative or positive sense
-	// with regard to the accuracy of the notification
-	// * REJECTED: a human takes the decision to reject the suggestion as it was wrong
-	// * ACCEPTED: a human takes the decision to apply the suggested enrichment to the local record
+	private FeedbackStatus status;
 
-	protected String getEventId() {
+	public String getEventId() {
 		return eventId;
 	}
 
-	protected void setEventId(final String eventId) {
+	public void setEventId(final String eventId) {
 		this.eventId = eventId;
 	}
 
-	protected String getStatus() {
+	public FeedbackStatus getStatus() {
 		return status;
 	}
 
-	protected void setStatus(final String status) {
+	public void setStatus(final FeedbackStatus status) {
 		this.status = status;
 	}
 

apps/dhp-broker-public-application/src/main/java/eu/dnetlib/broker/oa/controllers/OpenairePublicController.java

@@ -57,6 +57,8 @@ import eu.dnetlib.broker.api.ShortEventMessage;
 import eu.dnetlib.broker.common.elasticsearch.EventRepository;
 import eu.dnetlib.broker.common.elasticsearch.Notification;
 import eu.dnetlib.broker.common.elasticsearch.NotificationRepository;
+import eu.dnetlib.broker.common.feedbacks.DbEventFeedback;
+import eu.dnetlib.broker.common.feedbacks.DbEventFeedbackRepository;
 import eu.dnetlib.broker.common.properties.ElasticSearchProperties;
 import eu.dnetlib.broker.common.stats.OpenaireDsStatRepository;
 import eu.dnetlib.broker.common.subscriptions.Subscription;
@@ -87,6 +89,9 @@ public class OpenairePublicController extends AbstractDnetController {
 	@Autowired
 	private OpenaireDsStatRepository openaireDsStatRepository;
 
+	@Autowired
+	private DbEventFeedbackRepository feedbackRepository;
+
 	@Autowired
 	private ElasticSearchProperties props;
 
@@ -103,31 +108,27 @@ public class OpenairePublicController extends AbstractDnetController {
 
 		final Optional<Subscription> optSub = subscriptionRepo.findById(subscrId);
 
-		if (optSub.isPresent()) {
+		if (!optSub.isPresent()) {
+			log.warn("Invalid subscription: " + subscrId);
+			return new ScrollPage<>(null, true, new ArrayList<>());
+		}
+		final ElasticsearchRestTemplate esTemplate = (ElasticsearchRestTemplate) esOperations;
 
-			final ElasticsearchRestTemplate esTemplate = (ElasticsearchRestTemplate) esOperations;
-
-			final NativeSearchQuery searchQuery = new NativeSearchQueryBuilder()
+		final NativeSearchQuery searchQuery = new NativeSearchQueryBuilder()
 				.withQuery(QueryBuilders.termQuery("subscriptionId.keyword", subscrId))
 				.withSearchType(SearchType.DEFAULT)
 				.withFields("topic", "payload", "eventId")
 				.withPageable(PageRequest.of(0, 100))
 				.build();
 
-			final SearchScrollHits<Notification> scroll =
+		final SearchScrollHits<Notification> scroll =
 				esTemplate.searchScrollStart(SCROLL_TIMEOUT_IN_MILLIS, searchQuery, Notification.class, IndexCoordinates.of(props.getNotificationsIndexName()));
-			if (scroll.hasSearchHits()) {
-				final List<ShortEventMessage> values = calculateNotificationMessages(scroll);
-				return new ScrollPage<>(scroll.getScrollId(), values.isEmpty() || scroll.getScrollId() == null, values);
-			} else {
-				esTemplate.searchScrollClear(Arrays.asList(scroll.getScrollId()));
-				return new ScrollPage<>(null, true, new ArrayList<>());
-			}
-
-		} else {
-			log.warn("Invalid subscription: " + subscrId);
-			return new ScrollPage<>(null, true, new ArrayList<>());
+		if (scroll.hasSearchHits()) {
+			final List<ShortEventMessage> values = calculateNotificationMessages(scroll);
+			return new ScrollPage<>(scroll.getScrollId(), values.isEmpty() || scroll.getScrollId() == null, values);
 		}
+		esTemplate.searchScrollClear(Arrays.asList(scroll.getScrollId()));
+		return new ScrollPage<>(null, true, new ArrayList<>());
 	}
 
 	@Operation(summary = "Returns notifications using scrolls (other pages)")
@@ -137,14 +138,13 @@ public class OpenairePublicController extends AbstractDnetController {
 		final ElasticsearchRestTemplate esTemplate = (ElasticsearchRestTemplate) esOperations;
 
 		final SearchScrollHits<Notification> scroll =
-			esTemplate.searchScrollContinue(scrollId, SCROLL_TIMEOUT_IN_MILLIS, Notification.class, IndexCoordinates.of(props.getNotificationsIndexName()));
+				esTemplate.searchScrollContinue(scrollId, SCROLL_TIMEOUT_IN_MILLIS, Notification.class, IndexCoordinates.of(props.getNotificationsIndexName()));
 		if (scroll.hasSearchHits()) {
 			final List<ShortEventMessage> values = calculateNotificationMessages(scroll);
 			return new ScrollPage<>(scroll.getScrollId(), values.isEmpty() || scroll.getScrollId() == null, values);
-		} else {
-			esTemplate.searchScrollClear(Arrays.asList(scroll.getScrollId()));
-			return new ScrollPage<>(null, true, new ArrayList<>());
 		}
+		esTemplate.searchScrollClear(Arrays.asList(scroll.getScrollId()));
+		return new ScrollPage<>(null, true, new ArrayList<>());
 	}
 
 	@Operation(summary = "Returns notifications as file")
@@ -197,8 +197,8 @@ public class OpenairePublicController extends AbstractDnetController {
 		final Path pathDir = new Path(opendoarEventsPath + "/" + DigestUtils.md5Hex(id));
 
 		try (final FileSystem fs = FileSystem.get(conf);
-			final ServletOutputStream out = res.getOutputStream();
-			final GZIPOutputStream gzOut = new GZIPOutputStream(out)) {
+				final ServletOutputStream out = res.getOutputStream();
+				final GZIPOutputStream gzOut = new GZIPOutputStream(out)) {
 			boolean first = true;
 
 			IOUtils.write("[\n", gzOut, StandardCharsets.UTF_8);
@@ -209,16 +209,16 @@ public class OpenairePublicController extends AbstractDnetController {
 						final Path path = fileStatus.getPath();
 						if (path.getName().endsWith(".json")) {
 							try (final FSDataInputStream fis = fs.open(path);
-								final InputStreamReader isr = new InputStreamReader(fis);
-								final BufferedReader br = new BufferedReader(isr)) {
+									final InputStreamReader isr = new InputStreamReader(fis);
+									final BufferedReader br = new BufferedReader(isr)) {
 
 								first = processLine(gzOut, first, br);
 							}
 						} else if (path.getName().endsWith(".json.gz")) {
 							try (final FSDataInputStream fis = fs.open(path);
-								final GZIPInputStream gzIn = new GZIPInputStream(fis);
-								final InputStreamReader isr = new InputStreamReader(gzIn);
-								final BufferedReader br = new BufferedReader(isr)) {
+									final GZIPInputStream gzIn = new GZIPInputStream(fis);
+									final InputStreamReader isr = new InputStreamReader(gzIn);
+									final BufferedReader br = new BufferedReader(isr)) {
 
 								first = processLine(gzOut, first, br);
 							}
@@ -271,24 +271,30 @@ public class OpenairePublicController extends AbstractDnetController {
 
 	@Operation(summary = "Store the feedback of an event (MOCK)")
 	@RequestMapping(value = "/feedback/events", method = {
-		RequestMethod.POST, RequestMethod.PATCH
+			RequestMethod.POST, RequestMethod.PATCH
 	})
-	private Map<String, String> feedbackEvent(@RequestBody final EventFeedback feedback) {
-		// TOOD
-		final Map<String, String> res = new HashMap<>();
+	private Map<String, Object> feedbackEvent(@RequestBody final EventFeedback feedback) {
+
+		final DbEventFeedback dbEntry = new DbEventFeedback();
+		dbEntry.setEventId(feedback.getEventId());
+		dbEntry.setStatus(feedback.getStatus());
+
+		feedbackRepository.save(dbEntry);
+
+		final Map<String, Object> res = new HashMap<>();
 		res.put("status", "done");
+		res.put("feedback", dbEntry);
 		return res;
 	}
 
 	private List<ShortEventMessage> calculateNotificationMessages(final SearchScrollHits<Notification> scroll) {
 		if (scroll.getSearchHits().size() > 0) {
 			return scroll.stream()
-				.map(SearchHit::getContent)
-				.map(this::messageFromNotification)
-				.collect(Collectors.toList());
-		} else {
-			return new ArrayList<>();
+					.map(SearchHit::getContent)
+					.map(this::messageFromNotification)
+					.collect(Collectors.toList());
 		}
+		return new ArrayList<>();
 	}
 
 	private ShortEventMessage messageFromNotification(final Notification n) {

apps/dhp-broker-public-application/ssh_tunnel_postgres.sh

@@ -1,3 +0,0 @@
-#!/bin/bash
-
-ssh -vNL 5432:10.19.65.40:5432 michele.artini@iis-cdh5-test-gw.ocean.icm.edu.pl

apps/dhp-mdstore-manager/pom.xml

@@ -4,7 +4,7 @@
 	<parent>
 		<groupId>eu.dnetlib.dhp</groupId>
 		<artifactId>apps</artifactId>
-		<version>3.4.3-SNAPSHOT</version>
+		<version>3.5.5-SNAPSHOT</version>
 		<relativePath>../pom.xml</relativePath>
 	</parent>
 

apps/dnet-exporter-api/pom.xml

@@ -3,7 +3,7 @@
 	<parent>
 		<groupId>eu.dnetlib.dhp</groupId>
 		<artifactId>apps</artifactId>
-		<version>3.4.3-SNAPSHOT</version>
+		<version>3.5.5-SNAPSHOT</version>
 		<relativePath>../</relativePath>
 	</parent>
 
@@ -113,6 +113,10 @@
 			<groupId>com.fasterxml.jackson.datatype</groupId>
 			<artifactId>jackson-datatype-joda</artifactId>
 		</dependency>
+		<dependency>
+			<groupId>com.fasterxml.jackson.datatype</groupId>
+			<artifactId>jackson-datatype-jsr310</artifactId>
+		</dependency>
 		<dependency>
 			<groupId>joda-time</groupId>
 			<artifactId>joda-time</artifactId>
@@ -141,6 +145,12 @@
             <artifactId>spring-web</artifactId>
             <version>5.3.8</version>
         </dependency>
+        
+		<dependency>
+			<groupId>com.vladmihalcea</groupId>
+			<artifactId>hibernate-types-52</artifactId>
+		</dependency>
+        
     </dependencies>
 	
 	<build>

apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/common/AbstractExporterController.java

@@ -16,7 +16,9 @@ import org.springframework.web.bind.annotation.ResponseStatus;
 
 import com.fasterxml.jackson.annotation.JsonAutoDetect;
 
+import eu.dnetlib.openaire.exporter.exceptions.CommunityException;
 import eu.dnetlib.openaire.exporter.exceptions.DsmApiException;
+import eu.dnetlib.openaire.exporter.exceptions.ResourceNotFoundException;
 import eu.dnetlib.openaire.exporter.model.dsm.Response;
 
 /**
@@ -28,17 +30,26 @@ public abstract class AbstractExporterController {
 
 	@ResponseBody
 	@ExceptionHandler({
-		DsmApiException.class
+		DsmApiException.class, CommunityException.class, Exception.class
 	})
 	@ResponseStatus(value = HttpStatus.INTERNAL_SERVER_ERROR)
-	public ErrorMessage handleDSMException(final Exception e) {
+	public ErrorMessage handle500(final Exception e) {
+		return _handleError(e);
+	}
+
+	@ResponseBody
+	@ExceptionHandler({
+		ResourceNotFoundException.class
+	})
+	@ResponseStatus(value = HttpStatus.NOT_FOUND)
+	public ErrorMessage handle404(final Exception e) {
 		return _handleError(e);
 	}
 
 	@ResponseBody
 	@ExceptionHandler(MethodArgumentNotValidException.class)
 	@ResponseStatus(HttpStatus.BAD_REQUEST)
-	public List<ErrorMessage> processValidationError(final MethodArgumentNotValidException e) {
+	public List<ErrorMessage> handle400(final MethodArgumentNotValidException e) {
 		return e.getBindingResult()
 			.getFieldErrors()
 			.stream()

apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/common/ExporterConstants.java

@@ -10,6 +10,7 @@ public class ExporterConstants {
 	public final static String C_PJ = "Community projects";
 	public final static String C_ZC = "Community Zenodo Communities";
 	public final static String C_O = "Community Organizations";
+	public final static String C_SUB = "Subcommunities";
 
 	public final static String DS = "Datasource";
 	public final static String API = "Interface";

apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/common/ISClient.java

@@ -4,6 +4,7 @@ import java.io.IOException;
 import java.util.List;
 import java.util.Map;
 
+import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpException;
 import eu.dnetlib.openaire.dsm.dao.utils.IndexDsInfo;
 import eu.dnetlib.openaire.exporter.model.context.Context;
 
@@ -13,18 +14,25 @@ public interface ISClient {
 
 	String getObjectStoreId(String dsId) throws Exception;
 
+	@Deprecated
 	Map<String, Context> getFunderContextMap() throws IOException;
 
+	@Deprecated
 	Map<String, Context> getCommunityContextMap() throws IOException;
 
+	@Deprecated
 	Map<String, Context> getContextMap(final List<String> type) throws IOException;
 
+	@Deprecated
 	void updateContextParam(String id, String name, String value, boolean toEscape);
 
+	@Deprecated
 	void updateContextAttribute(String id, String name, String value);
 
+	@Deprecated
 	void addConcept(String id, String categoryId, String data);
 
+	@Deprecated
 	void removeConcept(String id, String categoryId, String conceptId);
 
 	void dropCache();
@@ -38,10 +46,15 @@ public interface ISClient {
 	 * @param value
 	 *            new value for the attribute
 	 */
+	@Deprecated
 	void updateConceptAttribute(String id, String name, String value);
 
+	@Deprecated
 	void updateConceptParam(String id, String name, String value);
 
+	@Deprecated
 	void updateConceptParamNoEscape(String id, String name, String value);
 
+	String getProfile(String profileId) throws ISLookUpException;
+
 }

apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/common/ISClientImpl.java

@@ -79,29 +79,37 @@ public class ISClientImpl implements ISClient {
 
 	@Override
 	@Cacheable("context-cache-funder")
+	@Deprecated
 	public Map<String, Context> getFunderContextMap() throws IOException {
 		return _processContext(_getQuery(config.getFindFunderContexts()));
 	}
 
 	@Override
 	@Cacheable("context-cache-community")
+	@Deprecated
 	public Map<String, Context> getCommunityContextMap() throws IOException {
 		return _processContext(_getQuery(config.getFindCommunityContexts()));
 	}
 
 	@Override
 	@Cacheable("context-cache")
+	@Deprecated
 	public Map<String, Context> getContextMap(final List<String> type) throws IOException {
 		if (Objects.isNull(type) || type.isEmpty()) {
 			return _processContext(_getQuery(config.getFindContextProfiles()));
 		} else {
-			final String xqueryTemplate = _getQuery(config.getFindContextProfilesByType());
+			try {
+				final String xqueryTemplate = _getQuery(config.getFindContextProfilesByType());
+				final String xquery = String.format(xqueryTemplate, type.stream()
+					.map(t -> String.format("./RESOURCE_PROFILE/BODY/CONFIGURATION/context/@type = '%s'", t))
+					.collect(Collectors.joining(" or ")));
 
-			final String xquery = String.format(xqueryTemplate, type.stream()
-				.map(t -> String.format("./RESOURCE_PROFILE/BODY/CONFIGURATION/context/@type = '%s'", t))
-				.collect(Collectors.joining(" or ")));
+				return _processContext(xquery);
+			} catch (final Exception e) {
+				log.error(e.getMessage());
+				throw new RuntimeException(e);
+			}
 
-			return _processContext(xquery);
 		}
 	}
 
@@ -109,6 +117,7 @@ public class ISClientImpl implements ISClient {
 	@CacheEvict(value = {
 		"context-cache", "context-cache-funder"
 	}, allEntries = true)
+	@Deprecated
 	public void updateContextParam(final String id, final String name, final String value, final boolean toEscape) {
 		if (getSize(id, name) > 0) {
 			try {
@@ -142,6 +151,7 @@ public class ISClientImpl implements ISClient {
 	@CacheEvict(value = {
 		"context-cache", "context-cache-funder"
 	}, allEntries = true)
+	@Deprecated
 	public void updateContextAttribute(final String id, final String name, final String value) {
 		final Escaper esc = XmlEscapers.xmlAttributeEscaper();
 		try {
@@ -156,6 +166,7 @@ public class ISClientImpl implements ISClient {
 	@CacheEvict(value = {
 		"context-cache", "context-cache-funder"
 	}, allEntries = true)
+	@Deprecated
 	public void addConcept(final String id, final String categoryId, final String data) {
 		try {
 			_quickSeachProfile(String.format("update insert %s into collection('/db/DRIVER/ContextDSResources/ContextDSResourceType')" +
@@ -169,6 +180,7 @@ public class ISClientImpl implements ISClient {
 	@CacheEvict(value = {
 		"context-cache", "context-cache-funder"
 	}, allEntries = true)
+	@Deprecated
 	public void removeConcept(final String id, final String categoryId, final String conceptId) {
 		try {
 			_quickSeachProfile(String.format("for $concept in collection('/db/DRIVER/ContextDSResources/ContextDSResourceType')" +
@@ -184,6 +196,7 @@ public class ISClientImpl implements ISClient {
 	@CacheEvict(value = {
 		"context-cache", "context-cache-community", "context-cache-funder"
 	}, allEntries = true)
+	@Deprecated
 	public void updateConceptAttribute(final String id, final String name, final String value) {
 		final Escaper esc = XmlEscapers.xmlAttributeEscaper();
 		try {
@@ -199,6 +212,7 @@ public class ISClientImpl implements ISClient {
 	@CacheEvict(value = {
 		"context-cache", "context-cache-funder"
 	}, allEntries = true)
+	@Deprecated
 	public void updateConceptParam(final String id, final String name, final String value) {
 		try {
 			_quickSeachProfile(getConceptXQuery(id, name, value));
@@ -211,6 +225,7 @@ public class ISClientImpl implements ISClient {
 	@CacheEvict(value = {
 		"context-cache", "context-cache-funder"
 	}, allEntries = true)
+	@Deprecated
 	public void updateConceptParamNoEscape(final String id, final String name, final String value) {
 		try {
 			_quickSeachProfile(getConceptXQueryNoEscape(id, name, value));
@@ -221,6 +236,7 @@ public class ISClientImpl implements ISClient {
 
 	/// HELPERS
 
+	@Deprecated
 	private String getInsertXQuery(final String id, final String paramName, final String paramValue, final boolean toEscape) {
 		String value;
 		if (toEscape) {
@@ -237,6 +253,7 @@ public class ISClientImpl implements ISClient {
 		}
 	}
 
+	@Deprecated
 	private String getXQuery(final String id, final String name, final String paramValue, final boolean toEscape) {
 		String value = paramValue;
 		if (toEscape) {
@@ -263,6 +280,7 @@ public class ISClientImpl implements ISClient {
 	// }
 	// }
 
+	@Deprecated
 	private String getConceptXQuery(final String id, final String name, final String value) {
 		final Escaper esc = XmlEscapers.xmlContentEscaper();
 		if (StringUtils.isNotBlank(value)) {
@@ -274,6 +292,7 @@ public class ISClientImpl implements ISClient {
 		}
 	}
 
+	@Deprecated
 	private String getConceptXQueryNoEscape(final String id, final String name, final String value) {
 
 		if (StringUtils.isNotBlank(value)) {
@@ -285,10 +304,12 @@ public class ISClientImpl implements ISClient {
 		}
 	}
 
+	@Deprecated
 	private Map<String, Context> _processContext(final String xquery) throws IOException {
 		return _processContext(new LinkedBlockingQueue<>(), xquery);
 	}
 
+	@Deprecated
 	private Map<String, Context> _processContext(final Queue<Throwable> errors, final String xquery) throws IOException {
 		try {
 			return getContextProfiles(errors, xquery).stream()
@@ -306,6 +327,7 @@ public class ISClientImpl implements ISClient {
 		}
 	}
 
+	@Deprecated
 	private List<String> getContextProfiles(final Queue<Throwable> errors, final String xquery) throws IOException {
 		log.warn("getContextProfiles(): not using cache");
 		try {
@@ -329,13 +351,13 @@ public class ISClientImpl implements ISClient {
 		final List<String> res = Lists.newArrayList();
 
 		log.debug(String.format("running xquery:\n%s", xquery));
-		try{
+		try {
 			final List<String> list = isLookUpService.quickSearchProfile(xquery);
 			if (list != null) {
 				res.addAll(list);
 			}
 			log.debug(String.format("query result size: %s", res.size()));
-		}catch(Exception ex){
+		} catch (final Exception ex) {
 			log.error(ex.getMessage());
 			throw new ISLookUpException("");
 		}
@@ -352,4 +374,9 @@ public class ISClientImpl implements ISClient {
 		log.debug("dropped dsManager IS cache");
 	}
 
+	@Override
+	public String getProfile(final String profileId) throws ISLookUpException {
+		return isLookUpService.getResourceProfile(profileId);
+	}
+
 }

apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/common/ReadOnlyRepository.java

@@ -0,0 +1,22 @@
+package eu.dnetlib.openaire.common;
+
+import java.util.Optional;
+
+import org.springframework.data.domain.Page;
+import org.springframework.data.domain.Pageable;
+import org.springframework.data.repository.NoRepositoryBean;
+import org.springframework.data.repository.Repository;
+
+@NoRepositoryBean
+public interface ReadOnlyRepository<T, ID> extends Repository<T, ID> {
+
+	Optional<T> findById(ID id);
+
+	boolean existsById(ID id);
+
+	Page<T> findAll(Pageable pageable);
+
+	Iterable<T> findAll();
+
+	long count();
+}

apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityApiCore.java

@@ -1,517 +0,0 @@
-package eu.dnetlib.openaire.community;
-
-import java.util.*;
-import java.util.stream.Collectors;
-
-import com.google.gson.Gson;
-
-import eu.dnetlib.openaire.exporter.exceptions.CommunityException;
-import eu.dnetlib.openaire.exporter.exceptions.ResourceNotFoundException;
-import eu.dnetlib.openaire.exporter.model.community.CommunityContentprovider;
-import eu.dnetlib.openaire.exporter.model.community.CommunityDetails;
-import eu.dnetlib.openaire.exporter.model.community.CommunityOpenAIRECommunities;
-import eu.dnetlib.openaire.exporter.model.community.CommunityOrganization;
-import eu.dnetlib.openaire.exporter.model.community.CommunityProject;
-import eu.dnetlib.openaire.exporter.model.community.CommunitySummary;
-import eu.dnetlib.openaire.exporter.model.community.CommunityWritableProperties;
-import eu.dnetlib.openaire.exporter.model.community.CommunityZenodoCommunity;
-import eu.dnetlib.openaire.exporter.model.community.selectioncriteria.SelectionCriteria;
-
-import org.apache.commons.lang3.StringUtils;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
-import org.springframework.cache.annotation.CacheEvict;
-import org.springframework.stereotype.Component;
-
-import com.google.common.base.Functions;
-import com.google.common.base.Joiner;
-import com.google.common.collect.Lists;
-import com.google.common.collect.Sets;
-
-import eu.dnetlib.openaire.common.ISClient;
-
-import static eu.dnetlib.openaire.community.CommunityConstants.*;
-
-@Component
-@ConditionalOnProperty(value = "openaire.exporter.enable.community", havingValue = "true")
-public class CommunityApiCore {// implements CommunityClient{
-
-	private static final Log log = LogFactory.getLog(CommunityApiCore.class);
-
-	@Autowired
-	private CommunityClient cci;
-
-	@Autowired
-	private ISClient isClient;
-
-	@Autowired
-	private CommunityCommon cc;
-
-	public List<CommunitySummary> listCommunities() throws CommunityException {
-		return cc.listCommunities();
-
-	}
-
-	public CommunityDetails getCommunity(final String id) throws CommunityException, ResourceNotFoundException {
-		return cc.getCommunity(id);
-
-	}
-
-	private void removeAdvancedConstraint(String id) throws ResourceNotFoundException, CommunityException {
-		cc.getCommunity(id);
-		isClient.updateContextParam(id, CPROFILE_ADVANCED_CONSTRAINT, "", false);
-		cc.removeAdvancedConstraint(id);
-
-	}
-
-	public void setCommunity(final String id, final CommunityWritableProperties details) throws CommunityException, ResourceNotFoundException {
-
-		cc.getCommunity(id); // ensure the community exists.
-
-		if (details.getShortName() != null) {
-			isClient.updateContextAttribute(id, CLABEL, details.getShortName());
-
-		}
-		if (details.getName() != null) {
-			isClient.updateContextParam(id, CSUMMARY_NAME, details.getName(), true);
-
-		}
-		if (details.getDescription() != null) {
-			isClient.updateContextParam(id, CSUMMARY_DESCRIPTION, details.getDescription(), true);
-
-		}
-		if (details.getLogoUrl() != null) {
-			isClient.updateContextParam(id, CSUMMARY_LOGOURL, details.getLogoUrl(), true);
-
-		}
-		if (details.getStatus() != null) {
-			isClient.updateContextParam(id, CSUMMARY_STATUS, details.getStatus().name(), true);
-
-		}
-		if (details.getSubjects() != null) {
-			isClient.updateContextParam(id, CPROFILE_SUBJECT, Joiner.on(CSV_DELIMITER).join(details.getSubjects()), true);
-
-		}
-		if (details.getFos() != null) {
-			isClient.updateContextParam(id, CPROFILE_FOS, Joiner.on(CSV_DELIMITER).join(details.getFos()), true);
-
-		}
-		if (details.getSdg() != null) {
-			isClient.updateContextParam(id, CPROFILE_SDG, Joiner.on(CSV_DELIMITER).join(details.getSdg()), true);
-
-		}
-		if (details.getAdvancedConstraints() != null) {
-			isClient.updateContextParam(id, CPROFILE_ADVANCED_CONSTRAINT, "<![CDATA[" + new Gson().toJson(details.getAdvancedConstraints()) + "]]>", false);
-
-		}
-		if (details.getMainZenodoCommunity() != null) {
-			isClient.updateContextParam(id, CSUMMARY_ZENODOC, details.getMainZenodoCommunity(), true);
-		}
-
-		cc.updateCommunity(id, details);
-	}
-
-	public List<CommunityProject> getCommunityProjects(final String id) throws CommunityException, ResourceNotFoundException {
-		cc.getCommunity(id); // ensure the community exists.
-		return cc.getCommunityInfo(id, PROJECTS_ID_SUFFIX, c -> CommunityMappingUtils.asCommunityProject(id, c));
-	}
-
-	public CommunityProject addCommunityProject(final String id, final CommunityProject project) throws CommunityException, ResourceNotFoundException {
-		if (!StringUtils.equalsIgnoreCase(id, project.getCommunityId())) {
-			throw new CommunityException("parameters 'id' and project.communityId must be coherent");
-		}
-
-		return updateProject(id, project);
-
-	}
-
-	private CommunityProject updateProject(String id, CommunityProject project) throws CommunityException, ResourceNotFoundException {
-		final TreeMap<Integer, CommunityProject> projects = getCommunityProjectMap(id);
-		String project_id = project.getId();
-
-		if (project_id != null && projects.keySet().contains(Integer.valueOf(project_id))){
-			if (project.getName() != null) {
-				isClient.updateConceptParam(id + PROJECTS_ID_SUFFIX + ID_SEPARATOR + project_id, CPROJECT_FULLNAME, project.getName());
-
-			}
-			if(project.getAcronym()!= null){
-				isClient.updateConceptParam(id + PROJECTS_ID_SUFFIX + ID_SEPARATOR + project_id, CPROJECT_ACRONYM, project.getAcronym());
-
-			}
-			if (project.getOpenaireId() != null){
-				isClient.updateConceptParam(id + PROJECTS_ID_SUFFIX + ID_SEPARATOR + project_id, OPENAIRE_ID, project.getOpenaireId());
-
-			}
-			if (project.getFunder() != null){
-				isClient.updateConceptParam(id + PROJECTS_ID_SUFFIX + ID_SEPARATOR + project_id, CPROJECT_FUNDER, project.getFunder());
-
-			}
-			if(project.getGrantId() != null){
-				isClient.updateConceptParam(id + PROJECTS_ID_SUFFIX + ID_SEPARATOR + project_id, CPROJECT_NUMBER, project.getGrantId());
-
-			}
-		}else {
-			project.setId(nextId(projects != null && !projects.isEmpty() ? projects.lastKey() : 0));
-
-			isClient.addConcept(id, id + PROJECTS_ID_SUFFIX, CommunityMappingUtils.asProjectXML(id, project));
-
-		}
-		cc.updateProject(id, project );
-		return project;
-	}
-
-	public List<CommunityProject> addCommunityProjectList(final String id, final List<CommunityProject> projectList) throws CommunityException, ResourceNotFoundException {
-		if(projectList == null || projectList.size() == 0){
-			throw new CommunityException("parameter 'projectList' must be present and should contain at least one project");
-		}
-		if (!StringUtils.equalsIgnoreCase(id, projectList.get(0).getCommunityId())) {
-			throw new CommunityException("parameters 'id' and project.communityId must be coherent");
-		}
-
-		List<CommunityProject> projects = new ArrayList();
-
-		for(CommunityProject project : projectList){
-			projects.add(updateProject(id, project));
-		}
-
-
-		return projects;
-	}
-
-	private String nextId(final Integer id) {
-		return String.valueOf(id + 1);
-	}
-
-	public void removeCommunityProject(final String id, final Integer projectId) throws CommunityException, ResourceNotFoundException {
-		final Map<Integer, CommunityProject> projects = getCommunityProjectMap(id);
-		if (!projects.containsKey(projectId)) {
-			throw new ResourceNotFoundException(String.format("project '%s' doesn't exist within context '%s'", projectId, id));
-		}
-		isClient.removeConcept(id, id + PROJECTS_ID_SUFFIX, id + PROJECTS_ID_SUFFIX + ID_SEPARATOR + projectId);
-		cc.removeFromCategory(id, PROJECTS_ID_SUFFIX, String.valueOf(projectId));
-	}
-
-	public void removeCommunityProjectList(final String id, final List<Integer> projectIdList) throws CommunityException, ResourceNotFoundException {
-		for(Integer projectId: projectIdList){
-			removeCommunityProject(id, projectId);
-		}
-	}
-
-	public List<CommunityContentprovider> getCommunityContentproviders(final String id) throws CommunityException, ResourceNotFoundException {
-		cc.getCommunity(id); // ensure the community exists.
-		return cc.getCommunityInfo(id, CONTENTPROVIDERS_ID_SUFFIX, c -> CommunityMappingUtils.asCommunityDataprovider(id, c));
-	}
-
-	public CommunityContentprovider addCommunityContentprovider(final String id, final CommunityContentprovider cp)
-		throws CommunityException, ResourceNotFoundException {
-		log.info("content provider to add " + cp.toString());
-		if (!StringUtils.equalsIgnoreCase(id, cp.getCommunityId())) { throw new CommunityException("parameters 'id' and cp.communityId must be coherent"); }
-
-		return updateContentprovider(id, cp);
-	}
-
-	private CommunityContentprovider updateContentprovider(String id, CommunityContentprovider cp) throws CommunityException, ResourceNotFoundException {
-		final TreeMap<Integer, CommunityContentprovider> cps = getCommunityContentproviderMap(id);
-		final String concept_id = cp.getId();
-		if (concept_id != null && cps.keySet().contains(Integer.valueOf(concept_id))) {
-			if (cp.getName() != null) {
-				isClient.updateConceptParam(id + CONTENTPROVIDERS_ID_SUFFIX + ID_SEPARATOR + concept_id, CCONTENTPROVIDER_NAME, cp.getName());
-			}
-			if (cp.getOfficialname() != null) {
-				isClient.updateConceptParam(id + CONTENTPROVIDERS_ID_SUFFIX + ID_SEPARATOR + concept_id, CCONTENTPROVIDER_OFFICIALNAME, cp.getOfficialname());
-			}
-			if (cp.getOpenaireId() != null) {
-				isClient.updateConceptParam(id + CONTENTPROVIDERS_ID_SUFFIX + ID_SEPARATOR + concept_id, OPENAIRE_ID, cp.getOpenaireId());
-			}
-			if (cp.getSelectioncriteria() != null) {
-				isClient.updateConceptParamNoEscape(id + CONTENTPROVIDERS_ID_SUFFIX + ID_SEPARATOR + concept_id, CCONTENTPROVIDER_SELCRITERIA, cp.toXML());
-			}
-		} else {
-			log.info("adding new concept for community " + id);
-			cp.setId(nextId(!cps.isEmpty() ? cps.lastKey() : 0));
-
-			isClient.addConcept(id, id + CONTENTPROVIDERS_ID_SUFFIX, CommunityMappingUtils.asContentProviderXML(id, cp));
-		}
-
-		cc.updateDatasource(id, cp);
-		return cp;
-	}
-
-	public void removeCommunityContentProvider(final String id, final Integer contentproviderId) throws CommunityException, ResourceNotFoundException {
-		final Map<Integer, CommunityContentprovider> providers = getCommunityContentproviderMap(id);
-		if (!providers.containsKey(contentproviderId)) {
-			throw new ResourceNotFoundException(String.format("content provider '%s' doesn't exist within context '%s'", contentproviderId, id));
-		}
-		isClient.removeConcept(id, id + CONTENTPROVIDERS_ID_SUFFIX, id + CONTENTPROVIDERS_ID_SUFFIX + ID_SEPARATOR + contentproviderId);
-		cc.removeFromCategory(id, CONTENTPROVIDERS_ID_SUFFIX, String.valueOf(contentproviderId));
-	}
-
-	public List<CommunityContentprovider> addCommunityContentProvidersList(String id, List<CommunityContentprovider> contentprovidersList) throws CommunityException, ResourceNotFoundException {
-		if(contentprovidersList == null || contentprovidersList.size() == 0){
-			throw new CommunityException("parameter 'contentprovidersList' must be present and should contain at least one content provider");
-		}
-		if (!StringUtils.equalsIgnoreCase(id, contentprovidersList.get(0).getCommunityId())) {
-			throw new CommunityException("parameters 'id' and contentprovider.communityId must be coherent");
-		}
-
-		List<CommunityContentprovider> contentproviders = new ArrayList();
-
-		for(CommunityContentprovider contentProvider : contentprovidersList){
-			contentproviders.add(updateContentprovider(id, contentProvider));
-		}
-
-
-		return contentproviders;
-	}
-
-	public void removeCommunityContentProviderList(final String id, final List<Integer> contentProviderIdList) throws CommunityException, ResourceNotFoundException {
-		for(Integer contentProviderId: contentProviderIdList){
-			removeCommunityContentProvider(id, contentProviderId);
-		}
-	}
-
-	public void removeCommunityOrganization(final String id, final Integer organizationId) throws CommunityException, ResourceNotFoundException {
-		final Map<Integer, CommunityOrganization> organizations = getCommunityOrganizationMap(id);
-		if (!organizations.containsKey(organizationId)) {
-			throw new ResourceNotFoundException(String.format("organization '%s' doesn't exist within context '%s'", organizationId, id));
-		}
-		isClient.removeConcept(id, id + ORGANIZATION_ID_SUFFIX, id + ORGANIZATION_ID_SUFFIX + ID_SEPARATOR + organizationId);
-		cc.removeFromCategory(id, ORGANIZATION_ID_SUFFIX, String.valueOf(organizationId));
-	}
-
-	public List<CommunityZenodoCommunity> getCommunityZenodoCommunities(final String id) throws CommunityException, ResourceNotFoundException {
-
-		return cc.getCommunityZenodoCommunities(id);
-	}
-
-	public List<CommunityOrganization> getCommunityOrganizations(final String id) throws CommunityException, ResourceNotFoundException {
-		cc.getCommunity(id);
-		return cc.getCommunityInfo(id, ORGANIZATION_ID_SUFFIX, c -> CommunityMappingUtils.asCommunityOrganization(id, c));
-	}
-
-	public CommunityDetails addCommunitySubjects(final String id, final List<String> subjects) throws CommunityException, ResourceNotFoundException {
-
-		final CommunityDetails cd = new CommunityDetails();
-
-		final Set<String> current = Sets.newHashSet(cc.getCommunity(id).getSubjects());
-
-		current.addAll(subjects);
-
-		cd.setSubjects(Lists.newArrayList(current));
-
-		setCommunity(id, CommunityWritableProperties.fromDetails(cd));
-
-		return cd;
-	}
-
-	public CommunityDetails removeCommunitySubjects(final String id, final List<String> subjects) throws CommunityException, ResourceNotFoundException {
-
-		final CommunityDetails cd = new CommunityDetails();
-
-		final Set<String> current = Sets.newHashSet(cc.getCommunity(id).getSubjects());
-
-		current.removeAll(subjects);
-
-		cd.setSubjects(Lists.newArrayList(current));
-
-		setCommunity(id, CommunityWritableProperties.fromDetails(cd));
-
-		return cd;
-	}
-
-	public CommunityDetails addCommunityFOS(final String id, final List<String> foss) throws CommunityException, ResourceNotFoundException {
-
-		final CommunityDetails cd = new CommunityDetails();
-
-		final Set<String> current = Sets.newHashSet();
-		if(Optional.ofNullable(cc.getCommunity(id).getFos()).isPresent()){
-			current.addAll(cc.getCommunity(id).getFos());
-		}
-
-		current.addAll(foss);
-
-		cd.setFos(Lists.newArrayList(current));
-
-		setCommunity(id, CommunityWritableProperties.fromDetails(cd));
-
-		return cd;
-	}
-
-	public CommunityDetails removeCommunityFOS(final String id, final List<String> foss) throws CommunityException, ResourceNotFoundException {
-
-		final CommunityDetails cd = new CommunityDetails();
-
-		final Set<String> current = Sets.newHashSet(cc.getCommunity(id).getFos());
-
-		current.removeAll(foss);
-
-		cd.setFos(Lists.newArrayList(current));
-
-		setCommunity(id, CommunityWritableProperties.fromDetails(cd));
-
-		return cd;
-	}
-
-	public CommunityDetails addCommunitySDG(final String id, final List<String> sdgs) throws CommunityException, ResourceNotFoundException {
-
-		final CommunityDetails cd = new CommunityDetails();
-		final Set<String> current = Sets.newHashSet();
-		if(Optional.ofNullable(cc.getCommunity(id).getSdg()).isPresent()){
-			current.addAll(cc.getCommunity(id).getSdg());
-		}
-
-		current.addAll(sdgs);
-
-		cd.setSdg(Lists.newArrayList(current));
-
-		setCommunity(id, CommunityWritableProperties.fromDetails(cd));
-
-		return cd;
-	}
-
-	public CommunityDetails removeCommunitySDG(final String id, final List<String> sdgs) throws CommunityException, ResourceNotFoundException {
-
-		final CommunityDetails cd = new CommunityDetails();
-
-		final Set<String> current = Sets.newHashSet(cc.getCommunity(id).getSdg());
-
-		current.removeAll(sdgs);
-
-		cd.setSdg(Lists.newArrayList(current));
-
-		setCommunity(id, CommunityWritableProperties.fromDetails(cd));
-
-		return cd;
-	}
-
-	public CommunityDetails addCommunityAdvancedConstraint(final String id, final SelectionCriteria advancedCosntraint) throws CommunityException, ResourceNotFoundException {
-
-		final CommunityDetails cd = new CommunityDetails();
-
-		cd.setAdvancedConstraints(advancedCosntraint);
-
-		setCommunity(id, CommunityWritableProperties.fromDetails(cd));
-
-		return cd;
-	}
-
-	public CommunityDetails removeCommunityAdvancedConstraint(final String id) throws ResourceNotFoundException, CommunityException {
-
-		removeAdvancedConstraint(id);
-
-		return new CommunityDetails();
-	}
-
-
-
-	@CacheEvict(value = "community-cache", allEntries = true)
-	public void removeCommunityZenodoCommunity(final String id, final Integer zenodoCommId) throws CommunityException, ResourceNotFoundException {
-
-		final Map<Integer, CommunityZenodoCommunity> zcomms = getZenodoCommunityMap(id);
-		if (!zcomms.containsKey(zenodoCommId)) {
-			throw new ResourceNotFoundException(String.format("Zenodo community '%s' doesn't exist within context '%s'", zenodoCommId, id));
-		}
-		isClient.removeConcept(id, id + ZENODOCOMMUNITY_ID_SUFFIX, id + ZENODOCOMMUNITY_ID_SUFFIX + ID_SEPARATOR + zenodoCommId);
-
-		cc.removeFromCategory(id, ZENODOCOMMUNITY_ID_SUFFIX, String.valueOf(zenodoCommId));
-
-	}
-
-	@CacheEvict(value = "community-cache", allEntries = true)
-	public CommunityZenodoCommunity addCommunityZenodoCommunity(final String id, final CommunityZenodoCommunity zc)
-		throws CommunityException, ResourceNotFoundException {
-		if (!StringUtils.equalsIgnoreCase(id, zc.getCommunityId())) { throw new CommunityException("parameters 'id' and zc.communityId must be coherent"); }
-		if (!StringUtils.isNotBlank(zc.getZenodoid())) { throw new CommunityException("parameter zenodoid cannot be null or empty"); }
-		final TreeMap<Integer, CommunityZenodoCommunity> zcs = getZenodoCommunityMap(id);
-
-		for (final CommunityZenodoCommunity czc : zcs.values()) {
-			if (czc.getZenodoid().equals(zc.getZenodoid())) { throw new CommunityException("Zenodo community already associated to the RCD"); }
-		}
-
-		zc.setId(nextId(!zcs.isEmpty() ? zcs.lastKey() : 0));
-
-		isClient.addConcept(id, id + ZENODOCOMMUNITY_ID_SUFFIX, CommunityMappingUtils.asZenodoCommunityXML(id, zc));
-		cc.updateZenodoCommunity(id, zc);
-
-		return zc;
-	}
-
-	public CommunityOpenAIRECommunities getOpenAIRECommunities(final String zenodoId) throws CommunityException, ResourceNotFoundException {
-
-		if (cci.getInverseZenodoCommunityMap().containsKey(zenodoId)) {
-			return new CommunityOpenAIRECommunities().setZenodoid(zenodoId)
-				.setOpenAirecommunitylist(cci.getInverseZenodoCommunityMap().get(zenodoId).stream().collect(Collectors.toList()));
-		}
-		return new CommunityOpenAIRECommunities();
-
-	}
-
-	// HELPERS
-
-	private TreeMap<Integer, CommunityProject> getCommunityProjectMap(final String id) throws CommunityException, ResourceNotFoundException {
-		return getCommunityProjects(id).stream()
-			.collect(Collectors.toMap(p -> Integer.valueOf(p.getId()), Functions.identity(), (p1, p2) -> {
-				log.warn(String.format("duplicate project found: '%s'", p1.getId()));
-				return p2;
-			}, TreeMap::new));
-	}
-
-	private TreeMap<Integer, CommunityContentprovider> getCommunityContentproviderMap(final String id) throws CommunityException, ResourceNotFoundException {
-		log.info("getting community content provider map");
-		return getCommunityContentproviders(id).stream()
-			.collect(Collectors.toMap(cp -> Integer.valueOf(cp.getId()), Functions.identity(), (cp1, cp2) -> {
-				log.warn(String.format("duplicate content provider found: '%s'", cp1.getId()));
-				return cp2;
-			}, TreeMap::new));
-	}
-
-	private TreeMap<Integer, CommunityZenodoCommunity> getZenodoCommunityMap(final String id) throws CommunityException, ResourceNotFoundException {
-		return getCommunityZenodoCommunities(id).stream()
-			.collect(Collectors.toMap(cp -> Integer.valueOf(cp.getId()), Functions.identity(), (cp1, cp2) -> {
-				log.warn(String.format("duplicate Zenodo community found: '%s'", cp1.getId()));
-				return cp2;
-			}, TreeMap::new));
-	}
-
-	private TreeMap<Integer, CommunityOrganization> getCommunityOrganizationMap(final String id) throws CommunityException, ResourceNotFoundException {
-		return getCommunityOrganizations(id).stream()
-			.collect(Collectors.toMap(o -> Integer.valueOf(o.getId()), Functions.identity(), (o1, o2) -> {
-				log.warn(String.format("duplicate content provider found: '%s'", o1.getId()));
-				return o2;
-			}, TreeMap::new));
-	}
-
-	public CommunityOrganization addCommunityOrganization(final String id, final CommunityOrganization organization)
-		throws CommunityException, ResourceNotFoundException {
-		if (!StringUtils.equalsIgnoreCase(id, organization.getCommunityId())) {
-			throw new CommunityException("parameters 'id' and organization.communityId must be coherent");
-		}
-
-		final TreeMap<Integer, CommunityOrganization> cps = getCommunityOrganizationMap(id);
-
-		final String organization_id = organization.getId();
-		if (organization_id != null && cps.keySet().contains(Integer.valueOf(organization_id))) {
-			if (organization.getName() != null) {
-				isClient.updateConceptParam(id + ORGANIZATION_ID_SUFFIX + ID_SEPARATOR + organization_id, CORGANIZATION_NAME, organization.getName());
-			}
-			if (organization.getLogo_url() != null) {
-				isClient.updateConceptParam(id + ORGANIZATION_ID_SUFFIX + ID_SEPARATOR + organization_id, CORGANIZATION_LOGOURL, Base64.getEncoder()
-					.encodeToString(organization.getLogo_url().getBytes()));
-			}
-			if (organization.getWebsite_url() != null) {
-				isClient.updateConceptParam(id + ORGANIZATION_ID_SUFFIX + ID_SEPARATOR + organization_id, CORGANIZATION_WEBSITEURL, Base64.getEncoder()
-					.encodeToString(organization.getWebsite_url().getBytes()));
-			}
-		} else {
-			organization.setId(nextId(!cps.isEmpty() ? cps.lastKey() : 0));
-			isClient.addConcept(id, id + ORGANIZATION_ID_SUFFIX, CommunityMappingUtils.asOrganizationXML(id, organization));
-		}
-
-		cc.updateOrganization(id, organization);
-		return organization;
-	}
-
-
-}

apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityClient.java

@@ -1,15 +0,0 @@
-package eu.dnetlib.openaire.community;
-
-import java.util.Map;
-import java.util.Set;
-
-import eu.dnetlib.openaire.exporter.exceptions.CommunityException;
-import eu.dnetlib.openaire.exporter.exceptions.ResourceNotFoundException;
-
-public interface CommunityClient {
-
-    Map<String, Set<String>> getInverseZenodoCommunityMap() throws CommunityException, ResourceNotFoundException;
-
-    void dropCache();
-
-}

apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityClientImpl.java

@@ -1,63 +0,0 @@
-package eu.dnetlib.openaire.community;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.cache.annotation.CacheEvict;
-import org.springframework.cache.annotation.Cacheable;
-import org.springframework.scheduling.annotation.Scheduled;
-import org.springframework.stereotype.Component;
-
-import eu.dnetlib.openaire.exporter.exceptions.CommunityException;
-import eu.dnetlib.openaire.exporter.exceptions.ResourceNotFoundException;
-import eu.dnetlib.openaire.exporter.model.community.CommunitySummary;
-import eu.dnetlib.openaire.exporter.model.community.CommunityZenodoCommunity;
-
-import java.util.*;
-
-@Component
-public class CommunityClientImpl implements CommunityClient {
-
-    private static final Log log = LogFactory.getLog(CommunityClient.class);
-
-    @Autowired
-    private CommunityCommon communityCommon;
-
-    @Override
-    @Cacheable("community-cache")
-    public Map<String, Set<String>> getInverseZenodoCommunityMap () throws CommunityException, ResourceNotFoundException {
-        log.info("Creating the data structure. Not using cache");
-        final Map<String, Set<String>> inverseListMap = new HashMap<>();
-
-        final List<CommunitySummary> communityList = communityCommon.listCommunities();
-
-        for(CommunitySummary cs :communityList){
-            final String communityId =  cs.getId();
-            List<CommunityZenodoCommunity> czc = communityCommon.getCommunityZenodoCommunities(communityId);
-            for(CommunityZenodoCommunity zc:czc){
-                final String zenodoId = zc.getZenodoid();
-                if(!inverseListMap.containsKey(zenodoId)) {
-                    inverseListMap.put(zc.getZenodoid(),new HashSet<>());
-                }
-                inverseListMap.get(zc.getZenodoid()).add(communityId);
-            }
-            final String zenodoMainCommunity = communityCommon.getCommunity(communityId).getZenodoCommunity();
-            if(!inverseListMap.containsKey(zenodoMainCommunity)) {
-                inverseListMap.put(zenodoMainCommunity,new HashSet<>());
-            }
-
-            inverseListMap.get(zenodoMainCommunity).add(communityId);
-        }
-        return inverseListMap;
-    }
-
-
-
-    @Override
-    @CacheEvict(cacheNames = { "community-cache", "context-cache-community"}, allEntries = true)
-    @Scheduled(fixedDelayString = "${openaire.exporter.cache.ttl}")
-    public void dropCache(){
-        log.debug("dropped community cache");
-    }
-
-}

apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityCommon.java

@@ -1,489 +0,0 @@
-package eu.dnetlib.openaire.community;
-
-import com.google.common.base.Joiner;
-import com.google.common.collect.Lists;
-import com.google.gson.Gson;
-import eu.dnetlib.openaire.common.ISClient;
-import eu.dnetlib.openaire.exporter.exceptions.CommunityException;
-import eu.dnetlib.openaire.exporter.exceptions.ResourceNotFoundException;
-import eu.dnetlib.openaire.exporter.model.community.CommunityContentprovider;
-import eu.dnetlib.openaire.exporter.model.community.CommunityDetails;
-import eu.dnetlib.openaire.exporter.model.community.CommunityOrganization;
-import eu.dnetlib.openaire.exporter.model.community.CommunityProject;
-import eu.dnetlib.openaire.exporter.model.community.CommunitySummary;
-import eu.dnetlib.openaire.exporter.model.community.CommunityWritableProperties;
-import eu.dnetlib.openaire.exporter.model.community.CommunityZenodoCommunity;
-import eu.dnetlib.openaire.exporter.model.context.Category;
-import eu.dnetlib.openaire.exporter.model.context.Concept;
-import eu.dnetlib.openaire.exporter.model.context.Context;
-import eu.dnetlib.openaire.exporter.model.context.Param;
-
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.stereotype.Component;
-
-import java.io.IOException;
-import java.util.*;
-import java.util.function.Function;
-import java.util.stream.Collectors;
-
-import static eu.dnetlib.openaire.community.CommunityConstants.*;
-
-
-@Component
-public class CommunityCommon {
-
-    @Autowired
-    private ISClient isClient;
-
-    public Map<String, Context> getContextMap() throws CommunityException {
-        try {
-            return isClient.getCommunityContextMap();
-        } catch (IOException e) {
-            throw new CommunityException(e);
-        }
-    }
-
-    public List<CommunitySummary> listCommunities()  throws CommunityException {
-        return getContextMap().values().stream()
-                .filter(context -> !communityBlackList.contains(context.getId()))
-                .map(CommunityMappingUtils::asCommunitySummary)
-                .collect(Collectors.toList());
-    }
-
-
-    public <R> List<R> getCommunityInfo(final String id, final String idSuffix, final Function<Concept, R> mapping) throws CommunityException {
-        final Map<String, Context> contextMap = getContextMap();
-        final Context context = contextMap.get(id);
-        if (context != null) {
-            final Map<String, Category> categories = context.getCategories();
-            final Category category = categories.get(id + idSuffix);
-            if (category != null) {
-                return category.getConcepts().stream()
-                        .map(mapping)
-                        .collect(Collectors.toList());
-            }
-        }
-        return Lists.newArrayList();
-    }
-
-    public CommunityDetails getCommunity(final String id) throws CommunityException, ResourceNotFoundException {
-        final Context context = getContextMap().get(id);
-        if (context == null || CommunityConstants.communityBlackList.contains(id)) {
-            //ResponseStatusException(NOT_FOUND, "Unable to find resource");
-            throw new ResourceNotFoundException();
-        }
-        return CommunityMappingUtils.asCommunityProfile(context);
-    }
-
-    public List<CommunityZenodoCommunity> getCommunityZenodoCommunities(final String id) throws CommunityException, ResourceNotFoundException {
-        getCommunity(id); // ensure the community exists.
-        return getCommunityInfo(id, ZENODOCOMMUNITY_ID_SUFFIX, c -> CommunityMappingUtils.asCommunityZenodoCommunity(id, c));
-    }
-
-
-    public void updateProject(String communityId, CommunityProject project) throws CommunityException {
-        final Context context = getContextMap().get(communityId);
-        Category prj = context.getCategories().get(communityId + PROJECTS_ID_SUFFIX);
-        if (prj.getConcepts().stream().map(c -> c.getId()).collect(Collectors.toList())
-                .contains(communityId + PROJECTS_ID_SUFFIX + ID_SEPARATOR + project.getId())){
-            prj.getConcepts().forEach(concept -> {
-                if (concept.getId().equals(communityId + PROJECTS_ID_SUFFIX + ID_SEPARATOR + project.getId())) {
-                    if (project.getName() != null) {
-
-                        concept.getParams().replace(CPROJECT_FULLNAME, Arrays.asList(new Param()
-                                .setName(CPROJECT_FULLNAME).setValue(project.getName())));
-                    }
-                    if (project.getAcronym() != null) {
-                        if(concept.getParams().keySet().contains(CPROJECT_ACRONYM)){
-                            concept.getParams().replace(CPROJECT_ACRONYM, Arrays.asList(new Param()
-                                    .setName(CPROJECT_ACRONYM).setValue(project.getAcronym())));
-                        }
-                        else{
-                            concept.getParams().put(CPROJECT_ACRONYM, Arrays.asList(new Param()
-                                    .setName(CPROJECT_ACRONYM).setValue(project.getAcronym())));
-                        }
-
-                    }
-                    if (project.getOpenaireId() != null) {
-                        if(concept.getParams().keySet().contains(OPENAIRE_ID)){
-                            concept.getParams().replace(OPENAIRE_ID, Arrays.asList(new Param()
-                                    .setName(OPENAIRE_ID).setValue(project.getOpenaireId())));
-                        }
-                        else{
-                            concept.getParams().put(OPENAIRE_ID, Arrays.asList(new Param()
-                            .setName(OPENAIRE_ID).setValue(project.getOpenaireId())));
-                        }
-
-                    }
-                    if (project.getFunder() != null) {
-                        concept.getParams().replace(CPROJECT_FUNDER, Arrays.asList(new Param()
-                                .setName(CPROJECT_FUNDER).setValue(project.getFunder())));
-                    }
-                    if (project.getGrantId() != null) {
-                        concept.getParams().replace(CPROJECT_NUMBER, Arrays.asList(new Param()
-                                .setName(CPROJECT_NUMBER).setValue(project.getGrantId())));
-
-                    }
-                }
-            });
-        }
-        else{
-            Concept concept = new Concept();
-            concept.setId(communityId + PROJECTS_ID_SUFFIX + ID_SEPARATOR + project.getId());
-            concept.setClaim(false);
-            if(project.getAcronym() != null)
-                concept.setLabel(project.getAcronym());
-            else
-                concept.setLabel("");
-
-             Map<String, List<Param>> params = new TreeMap<>();
-
-             if(project.getAcronym() != null){
-                 params.put(CPROJECT_ACRONYM, Arrays.asList(new Param().setName(CPROJECT_ACRONYM)
-                         .setValue(project.getAcronym())));
-             }
-
-             if (project.getName() != null){
-                 params.put(CPROJECT_FULLNAME, Arrays.asList(new Param()
-                         .setName(CPROJECT_FULLNAME)
-                         .setValue(project.getName())
-                 ));
-             }
-
-             if (project.getOpenaireId() != null){
-                 params.put(OPENAIRE_ID, Arrays.asList(new Param()
-                         .setName(OPENAIRE_ID)
-                         .setValue(project.getOpenaireId())
-                 ));
-             }
-
-             if(project.getFunder() != null){
-                 params.put(CPROJECT_FUNDER, Arrays.asList(new Param()
-                         .setName(CPROJECT_FUNDER)
-                         .setValue(project.getFunder())
-                 ));
-             }
-
-             if (project.getGrantId()!=null){
-                 params.put(CPROJECT_NUMBER, Arrays.asList(new Param()
-                         .setName(CPROJECT_NUMBER)
-                         .setValue(project.getGrantId())
-                 ));
-             }
-
-            concept.setParams(params);
-            prj.getConcepts().add(concept);
-
-
-        }
-
-    }
-
-    public void removeAdvancedConstraint(String id) throws CommunityException {
-        final Context context = getContextMap().get(id);
-        context.getParams()
-                .replace(CPROFILE_ADVANCED_CONSTRAINT, Arrays.asList(new Param()
-                        .setName(CPROFILE_ADVANCED_CONSTRAINT).setValue(null)));
-    }
-
-    public void updateCommunity(String id, CommunityWritableProperties community) throws CommunityException {
-        final Context context = getContextMap().get(id);
-
-        if(community.getShortName() != null) {
-            context.setLabel(community.getShortName());
-        }
-
-        if (community.getName() != null){
-            context.getParams().replace(CSUMMARY_NAME, Arrays.asList(new Param()
-            .setValue(community.getName()).setName(CSUMMARY_NAME)));
-        }
-        if(community.getDescription() != null) {
-            context.getParams()
-                    .replace(CSUMMARY_DESCRIPTION, Arrays.asList(new Param()
-                            .setName(CSUMMARY_DESCRIPTION).setValue(community.getDescription())));
-        }
-        if(community.getLogoUrl() != null){
-            context.getParams()
-                    .replace(CSUMMARY_LOGOURL, Arrays.asList(new Param()
-                    .setName(CSUMMARY_LOGOURL).setValue(community.getLogoUrl())));
-
-        }
-        if (community.getStatus() != null) {
-            context.getParams()
-                    .replace(CSUMMARY_STATUS, Arrays.asList(new Param()
-                    .setName(CSUMMARY_STATUS).setValue(community.getStatus().name())));
-        }
-        if (community.getSubjects() != null) {
-            context.getParams()
-                    .replace(CPROFILE_SUBJECT, Arrays.asList(new Param().setName(CPROFILE_SUBJECT)
-                            .setValue(Joiner.on(CSV_DELIMITER)
-                                    .join(community.getSubjects()))));
-        }
-        if(community.getFos() != null){
-            if (context.getParams().containsKey(CPROFILE_FOS))
-            context.getParams()
-                    .replace(CPROFILE_FOS, Arrays.asList(new Param().setName(CPROFILE_FOS)
-                            .setValue(Joiner.on(CSV_DELIMITER)
-                                    .join(community.getFos()))));
-            else
-                context.getParams().put(CPROFILE_FOS, Arrays.asList(new Param().setName(CPROFILE_FOS)
-                        .setValue(Joiner.on(CSV_DELIMITER)
-                                .join(community.getFos()))));
-        }
-        if(community.getSdg() != null){
-            if(context.getParams().containsKey(CPROFILE_SDG))
-            context.getParams()
-                    .replace(CPROFILE_SDG, Arrays.asList(new Param().setName(CPROFILE_SDG)
-                            .setValue(Joiner.on(CSV_DELIMITER)
-                                    .join(community.getSdg()))));
-            else
-                context.getParams().put(CPROFILE_SDG, Arrays.asList(new Param().setName(CPROFILE_SDG)
-                        .setValue(Joiner.on(CSV_DELIMITER)
-                                .join(community.getSdg()))));
-        }
-        if (community.getAdvancedConstraints() != null) {
-            if(context.getParams().containsKey(CPROFILE_ADVANCED_CONSTRAINT))
-            context.getParams()
-                    .replace(CPROFILE_ADVANCED_CONSTRAINT, Arrays.asList(new Param()
-                            .setName(CPROFILE_ADVANCED_CONSTRAINT).setValue(new Gson().toJson(community.getAdvancedConstraints()))));
-            else
-                context.getParams().put(CPROFILE_ADVANCED_CONSTRAINT, Arrays.asList(new Param()
-                        .setName(CPROFILE_ADVANCED_CONSTRAINT).setValue(new Gson().toJson(community.getAdvancedConstraints()))));
-        }
-        if(community.getMainZenodoCommunity() != null){
-            context.getParams()
-                    .replace(CSUMMARY_ZENODOC, Arrays.asList(new Param()
-                            .setName(CSUMMARY_ZENODOC).setValue(community.getMainZenodoCommunity())));
-
-        }
-
-    }
-
-
-    public void removeFromCategory(String communityId, String category, String conceptId) throws CommunityException {
-        Map<String, Context> cmap = getContextMap();
-        Context context = cmap.get(communityId);
-        Map<String, Category> cat = context.getCategories();
-
-        List<Concept> concepts = cat.get(communityId + category).getConcepts()
-                .stream().filter(c -> !c.getId().equals(communityId + category + ID_SEPARATOR + conceptId)).collect(Collectors.toList());
-
-         cat.get(communityId + category).setConcepts(concepts);
-    }
-
-    public void updateDatasource(String communityId, CommunityContentprovider cp) throws CommunityException {
-        final Context context = getContextMap().get(communityId);
-        Category dts = context.getCategories().get(communityId + CONTENTPROVIDERS_ID_SUFFIX);
-        if (dts.getConcepts().stream().map(c -> c.getId()).collect(Collectors.toList())
-                .contains(communityId + CONTENTPROVIDERS_ID_SUFFIX + ID_SEPARATOR + cp.getId())){
-            dts.getConcepts().forEach(concept -> {
-                if (concept.getId().equals(communityId + CONTENTPROVIDERS_ID_SUFFIX + ID_SEPARATOR + cp.getId())) {
-
-
-                    if (cp.getName() != null) {
-                        if(concept.getParams().keySet().contains(CCONTENTPROVIDER_NAME)){
-                            concept.getParams().replace(CCONTENTPROVIDER_NAME, Arrays.asList(new Param()
-                                    .setName(CCONTENTPROVIDER_NAME).setValue(cp.getName())));
-                        }
-                        else{
-                            concept.getParams().put(CCONTENTPROVIDER_NAME, Arrays.asList(new Param()
-                                    .setName(CCONTENTPROVIDER_NAME).setValue(cp.getName())));
-                        }
-
-                    }
-                    if (cp.getOfficialname() != null) {
-                        if(concept.getParams().keySet().contains(CCONTENTPROVIDER_OFFICIALNAME)){
-                            concept.getParams().replace(CCONTENTPROVIDER_OFFICIALNAME, Arrays.asList(new Param()
-                                    .setName(CCONTENTPROVIDER_OFFICIALNAME).setValue(cp.getOfficialname())));
-                        }
-                        else{
-                            concept.getParams().put(CCONTENTPROVIDER_OFFICIALNAME, Arrays.asList(new Param()
-                                    .setName(CCONTENTPROVIDER_OFFICIALNAME).setValue(cp.getOfficialname())));
-                        }
-
-                    }
-
-
-                    if (cp.getOpenaireId() != null) {
-                        if(concept.getParams().keySet().contains(OPENAIRE_ID)){
-                            concept.getParams().replace(OPENAIRE_ID, Arrays.asList(new Param()
-                                    .setName(OPENAIRE_ID).setValue(cp.getOpenaireId())));
-                        }
-                        else{
-                            concept.getParams().put(OPENAIRE_ID, Arrays.asList(new Param()
-                                    .setName(OPENAIRE_ID).setValue(cp.getOpenaireId())));
-                        }
-
-                    }
-
-                    if (cp.getSelectioncriteria()  != null) {
-                        if(concept.getParams().keySet().contains(CCONTENTPROVIDER_SELCRITERIA)){
-                            concept.getParams().replace(CCONTENTPROVIDER_SELCRITERIA, Arrays.asList(new Param()
-                                    .setName(CCONTENTPROVIDER_SELCRITERIA).setValue(cp.toJson())));
-                        }
-                        else{
-                            concept.getParams().put(CCONTENTPROVIDER_SELCRITERIA, Arrays.asList(new Param()
-                                    .setName(CCONTENTPROVIDER_SELCRITERIA).setValue(cp.toJson())));
-                        }
-
-                    }
-                }
-            });
-        }
-        else{
-            Concept concept = new Concept();
-            concept.setId(communityId + CONTENTPROVIDERS_ID_SUFFIX + ID_SEPARATOR + cp.getId());
-            concept.setClaim(false);
-            concept.setLabel("");
-
-            Map<String, List<Param>> params = new TreeMap<>();
-
-            if (cp.getName() != null) {
-                params.put( CCONTENTPROVIDER_NAME, Arrays.asList(new Param().setValue(cp.getName()).setName(CCONTENTPROVIDER_NAME)));
-            }
-            if(cp.getOfficialname()!= null){
-                params.put( CCONTENTPROVIDER_OFFICIALNAME, Arrays.asList(new Param().setValue(cp.getOfficialname()).setName(CCONTENTPROVIDER_OFFICIALNAME)));
-            }
-            if (cp.getOpenaireId() != null){
-                params.put( OPENAIRE_ID, Arrays.asList(new Param().setValue(cp.getOpenaireId()).setName(OPENAIRE_ID)));
-            }
-            if(cp.getSelectioncriteria() != null){
-                params.put( CCONTENTPROVIDER_SELCRITERIA, Arrays.asList(new Param().setValue(cp.toJson()).setName(CCONTENTPROVIDER_SELCRITERIA)));
-
-            }
-
-            concept.setParams(params);
-            dts.getConcepts().add(concept);
-
-
-        }
-
-    }
-
-    public void updateOrganization(String communityId, CommunityOrganization organization) throws CommunityException {
-
-        final Context context = getContextMap().get(communityId);
-        Category orgs = context.getCategories().get(communityId + ORGANIZATION_ID_SUFFIX);
-        if (orgs.getConcepts().stream().map(c -> c.getId()).collect(Collectors.toList())
-                .contains(communityId + ORGANIZATION_ID_SUFFIX + ID_SEPARATOR + organization.getId())){
-            orgs.getConcepts().forEach(concept -> {
-                if (concept.getId().equals(communityId + ORGANIZATION_ID_SUFFIX + ID_SEPARATOR + organization.getId())) {
-
-
-                    if (organization.getName() != null) {
-                        if(concept.getParams().keySet().contains(CORGANIZATION_NAME)){
-                            concept.getParams().replace(CORGANIZATION_NAME, Arrays.asList(new Param()
-                                    .setName(CORGANIZATION_NAME).setValue(organization.getName())));
-                        }
-                        else{
-                            concept.getParams().put(CORGANIZATION_NAME, Arrays.asList(new Param()
-                                    .setName(CORGANIZATION_NAME).setValue(organization.getName())));
-                        }
-
-                    }
-                    if (organization.getLogo_url() != null) {
-                        if(concept.getParams().keySet().contains(CORGANIZATION_LOGOURL)){
-                            concept.getParams().replace(CORGANIZATION_LOGOURL, Arrays.asList(new Param()
-                                    .setName(CORGANIZATION_LOGOURL).setValue(Base64.getEncoder().encodeToString(organization.getLogo_url().getBytes()))));
-                        }
-                        else{
-                            concept.getParams().put(CORGANIZATION_LOGOURL, Arrays.asList(new Param()
-                                    .setName(CORGANIZATION_LOGOURL).setValue(Base64.getEncoder().encodeToString(organization.getLogo_url().getBytes()))));
-                        }
-
-                    }
-
-
-                    if (organization.getWebsite_url() != null) {
-                        if(concept.getParams().keySet().contains(CORGANIZATION_WEBSITEURL)){
-                            concept.getParams().replace(CORGANIZATION_WEBSITEURL, Arrays.asList(new Param()
-                                    .setName(CORGANIZATION_WEBSITEURL).setValue(Base64.getEncoder().encodeToString(organization.getWebsite_url().getBytes()))));
-                        }
-                        else{
-                            concept.getParams().put(CORGANIZATION_WEBSITEURL, Arrays.asList(new Param()
-                                    .setName(CORGANIZATION_WEBSITEURL).setValue(Base64.getEncoder().encodeToString(organization.getWebsite_url().getBytes()))));
-                        }
-
-                    }
-
-                    }
-
-            });
-        }
-        else{
-
-            Concept concept = new Concept();
-            concept.setId(communityId + ORGANIZATION_ID_SUFFIX + ID_SEPARATOR + organization.getId());
-            concept.setClaim(false);
-            concept.setLabel("");
-
-            Map<String, List<Param>> params = new TreeMap<>();
-
-            if (organization.getName() != null) {
-                params.put( CORGANIZATION_NAME, Arrays.asList(new Param().setValue(organization.getName()).setName(CORGANIZATION_NAME)));
-            }
-            if(organization.getLogo_url()!= null){
-
-                params.put( CORGANIZATION_LOGOURL, Arrays.asList(new Param().setValue(Base64.getEncoder().encodeToString(organization.getLogo_url().getBytes())).setName(CORGANIZATION_LOGOURL)));
-            }
-            if (organization.getWebsite_url() != null){
-                params.put( CORGANIZATION_WEBSITEURL, Arrays.asList(new Param().setValue(Base64.getEncoder().encodeToString(organization.getWebsite_url().getBytes())).setName(CORGANIZATION_WEBSITEURL)));
-            }
-
-
-            concept.setParams(params);
-            orgs.getConcepts().add(concept);
-
-
-        }
-
-    }
-
-    public void updateZenodoCommunity(String communityId, CommunityZenodoCommunity zc) throws CommunityException {
-        final Context context = getContextMap().get(communityId);
-        Category zcs = context.getCategories().get(communityId + ZENODOCOMMUNITY_ID_SUFFIX);
-        if (zcs.getConcepts().stream().map(c -> c.getId()).collect(Collectors.toList())
-                .contains(communityId + ZENODOCOMMUNITY_ID_SUFFIX + ID_SEPARATOR + zc.getId())){
-            zcs.getConcepts().forEach(concept -> {
-                if (concept.getId().equals(communityId + ZENODOCOMMUNITY_ID_SUFFIX + ID_SEPARATOR + zc.getId())) {
-
-
-                    if (zc.getZenodoid() != null) {
-                        if(concept.getParams().keySet().contains(CZENODOCOMMUNITY_ID)){
-                            concept.getParams().replace(CZENODOCOMMUNITY_ID, Arrays.asList(new Param()
-                                    .setName(CZENODOCOMMUNITY_ID).setValue(zc.getZenodoid())));
-                        }
-                        else{
-                            concept.getParams().put(CZENODOCOMMUNITY_ID, Arrays.asList(new Param()
-                                    .setName(CZENODOCOMMUNITY_ID).setValue(zc.getZenodoid())));
-                        }
-
-                    }
-
-                }
-
-            });
-        }
-        else{
-
-            Concept concept = new Concept();
-            concept.setId(communityId + ZENODOCOMMUNITY_ID_SUFFIX + ID_SEPARATOR + zc.getId());
-            concept.setClaim(false);
-
-
-            Map<String, List<Param>> params = new TreeMap<>();
-
-            if (zc.getZenodoid() != null) {
-                params.put( CZENODOCOMMUNITY_ID, Arrays.asList(new Param().setValue(zc.getZenodoid()).setName(CZENODOCOMMUNITY_ID)));
-                concept.setLabel(zc.getZenodoid());
-            }else{
-                concept.setLabel("");
-            }
-            concept.setParams(params);
-            zcs.getConcepts().add(concept);
-
-        }
-
-    }
-
-}

apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityConstants.java

@@ -1,61 +0,0 @@
-package eu.dnetlib.openaire.community;
-
-import java.util.Set;
-
-import com.google.common.collect.Sets;
-
-public class CommunityConstants {
-
-	public final static Set<String> communityBlackList = Sets.newHashSet("fet-fp7", "fet-h2020");
-
-	// common
-	public final static String OPENAIRE_ID = "openaireId";
-	public final static String PIPE_SEPARATOR = "||";
-	public final static String ID_SEPARATOR = "::";
-	public final static String CSV_DELIMITER = ",";
-	public final static String CLABEL = "label";
-
-	// id suffixes
-	public final static String PROJECTS_ID_SUFFIX = ID_SEPARATOR + "projects";
-	public final static String CONTENTPROVIDERS_ID_SUFFIX = ID_SEPARATOR + "contentproviders";
-	public final static String ZENODOCOMMUNITY_ID_SUFFIX = ID_SEPARATOR + "zenodocommunities";
-	public final static String ORGANIZATION_ID_SUFFIX = ID_SEPARATOR + "organizations";
-
-	// community summary
-	public final static String CSUMMARY_DESCRIPTION = "description";
-	public final static String CSUMMARY_LOGOURL = "logourl";
-	public final static String CSUMMARY_STATUS = "status";
-	public final static String CSUMMARY_NAME = "name";
-	public final static String CSUMMARY_MANAGER = "manager";
-	public final static String CSUMMARY_ZENODOC = "zenodoCommunity";
-
-	// community profile
-	public final static String CPROFILE_SUBJECT = "subject";
-	public final static String CPROFILE_CREATIONDATE = "creationdate";
-	public final static String CPROFILE_FOS = "fos";
-	public final static String CPROFILE_SDG = "sdg";
-	public final static String CPROFILE_ADVANCED_CONSTRAINT = "advancedConstraints";
-
-	// community project
-	public final static String CPROJECT_FUNDER = "funder";
-	public final static String CPROJECT_NUMBER = "CD_PROJECT_NUMBER";
-	public final static String CPROJECT_FULLNAME = "projectfullname";
-	public final static String CPROJECT_ACRONYM = "acronym";
-
-	// community content provider
-	public final static String CCONTENTPROVIDER_NAME = "name";
-	public final static String CCONTENTPROVIDER_OFFICIALNAME = "officialname";
-	public final static String CCONTENTPROVIDER_ENABLED = "enabled";
-	public final static String CCONTENTPROVIDERENABLED_DEFAULT = "true";
-	public final static String CCONTENTPROVIDER_SELCRITERIA = "selcriteria";
-
-	//community zenodo community
-	public final static String CZENODOCOMMUNITY_ID = "zenodoid";
-
-	//community organization
-	public final static String CORGANIZATION_NAME = "name";
-	public final static String CORGANIZATION_LOGOURL = "logourl";
-	public final static String CORGANIZATION_WEBSITEURL = "websiteurl";
-
-
-}

apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityMappingUtils.java

@@ -1,251 +0,0 @@
-package eu.dnetlib.openaire.community;
-
-import java.text.ParseException;
-import java.util.*;
-import java.util.stream.Collectors;
-import java.util.stream.Stream;
-
-import com.google.common.escape.Escaper;
-import com.google.common.xml.XmlEscapers;
-
-import eu.dnetlib.openaire.exporter.model.community.CommunityContentprovider;
-import eu.dnetlib.openaire.exporter.model.community.CommunityDetails;
-import eu.dnetlib.openaire.exporter.model.community.CommunityOrganization;
-import eu.dnetlib.openaire.exporter.model.community.CommunityProject;
-import eu.dnetlib.openaire.exporter.model.community.CommunityStatus;
-import eu.dnetlib.openaire.exporter.model.community.CommunitySummary;
-import eu.dnetlib.openaire.exporter.model.community.CommunityZenodoCommunity;
-import eu.dnetlib.openaire.exporter.model.community.selectioncriteria.SelectionCriteria;
-import eu.dnetlib.openaire.exporter.model.context.Concept;
-import eu.dnetlib.openaire.exporter.model.context.Context;
-import eu.dnetlib.openaire.exporter.model.context.Param;
-
-import org.apache.commons.lang3.StringUtils;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-
-import static eu.dnetlib.openaire.common.Utils.escape;
-import static eu.dnetlib.openaire.community.CommunityConstants.*;
-
-public class CommunityMappingUtils {
-
-	private final static String pattern = "yyyy-MM-dd'T'hh:mm:ss";
-
-	private static final Log log = LogFactory.getLog(CommunityMappingUtils.class);
-
-	public static CommunitySummary asCommunitySummary(final Context c) {
-		final CommunitySummary summary = new CommunitySummary();
-
-		summary.setId(c.getId());
-		summary.setShortName(c.getLabel());
-		summary.setLastUpdateDate(c.getLastUpdateDate());
-		summary.setCreationDate(c.getCreationDate());
-		summary.setQueryId(c.getId() + PIPE_SEPARATOR + c.getLabel());
-		summary.setType(c.getType());
-
-		final Map<String, List<Param>> params = c.getParams();
-		if (params.containsKey(CSUMMARY_DESCRIPTION)) {
-			summary.setDescription(asCsv(params.get(CSUMMARY_DESCRIPTION)));
-		}
-		if (params.containsKey(CSUMMARY_LOGOURL)) {
-			summary.setLogoUrl(asCsv(params.get(CSUMMARY_LOGOURL)));
-		}
-		if (params.containsKey(CSUMMARY_STATUS)) {
-			summary.setStatus(CommunityStatus.valueOf(firstValue(params, CSUMMARY_STATUS)));
-		}
-		if (params.containsKey(CSUMMARY_NAME)) {
-			summary.setName(asCsv(params.get(CSUMMARY_NAME)));
-		}
-		if (params.containsKey(CSUMMARY_ZENODOC)) {
-			summary.setZenodoCommunity(asCsv(params.get(CSUMMARY_ZENODOC)));
-		}
-
-		return summary;
-	}
-
-	public static CommunityDetails asCommunityProfile(final Context c)  {
-
-		final CommunityDetails p = new CommunityDetails(asCommunitySummary(c));
-		p.setLastUpdateDate(c.getLastUpdateDate());
-		final Map<String, List<Param>> params = c.getParams();
-		if (params.containsKey(CPROFILE_SUBJECT)) {
-			p.setSubjects(splitValues(asValues(params.get(CPROFILE_SUBJECT)), CSV_DELIMITER));
-		}
-		if (params.containsKey(CPROFILE_FOS)) {
-			p.setFos(splitValues(asValues(params.get(CPROFILE_FOS)), CSV_DELIMITER));
-		}
-		if (params.containsKey(CPROFILE_SDG)) {
-			p.setSdg(splitValues(asValues(params.get(CPROFILE_SDG)), CSV_DELIMITER));
-		}
-		if (params.containsKey(CPROFILE_ADVANCED_CONSTRAINT)) {
-			//In the map the string is the serialization of the json representing the selection criteria so it is a valid json
-			p.setAdvancedConstraints(SelectionCriteria.fromJson(asCsv(params.get(CPROFILE_ADVANCED_CONSTRAINT))));
-
-		}
-		if (params.containsKey(CPROFILE_CREATIONDATE)){
-			try {
-				p.setCreationDate(org.apache.commons.lang3.time.DateUtils.parseDate(asCsv(params.get(CPROFILE_CREATIONDATE)), pattern));
-			}catch(ParseException e) {
-				log.debug("Exception on date format: " + e.getMessage());
-			}
-		}
-
-		return p;
-	}
-
-	public static CommunityProject asCommunityProject(final String communityId, final Concept c) {
-
-		final Map<String, List<Param>> p = c.getParams();
-		final CommunityProject project = new CommunityProject();
-		project.setCommunityId(communityId);
-		project.setId(StringUtils.substringAfterLast(c.getId(), ID_SEPARATOR));
-		project.setOpenaireId(firstValue(p, OPENAIRE_ID));
-		project.setFunder(firstValue(p, CPROJECT_FUNDER));
-		project.setGrantId(firstValue(p, CPROJECT_NUMBER));
-		project.setName(firstValue(p, CPROJECT_FULLNAME));
-		project.setAcronym(firstValue(p, CPROJECT_ACRONYM));
-
-		return project;
-	}
-
-	public static CommunityContentprovider asCommunityDataprovider(final String communityId, final Concept c) {
-
-		final Map<String, List<Param>> p = c.getParams();
-		final CommunityContentprovider d = new CommunityContentprovider();
-		d.setCommunityId(communityId);
-		d.setId(StringUtils.substringAfterLast(c.getId(), ID_SEPARATOR));
-		d.setOpenaireId(firstValue(p, OPENAIRE_ID));
-		d.setName(firstValue(p, CCONTENTPROVIDER_NAME));
-		d.setOfficialname(firstValue(p, CCONTENTPROVIDER_OFFICIALNAME));
-		d.setSelectioncriteria(SelectionCriteria.fromJson(firstValue(p, CCONTENTPROVIDER_SELCRITERIA)));
-		return d;
-	}
-
-	public static CommunityZenodoCommunity asCommunityZenodoCommunity(final String communityId, final Concept c){
-		final CommunityZenodoCommunity z = new CommunityZenodoCommunity();
-		final Map<String, List<Param>> p = c.getParams();
-		z.setCommunityId(communityId);
-		z.setId(StringUtils.substringAfterLast(c.getId(), ID_SEPARATOR));
-		z.setZenodoid(firstValue(p,CZENODOCOMMUNITY_ID));
-		//z.setName(c.getLabel());
-		return z;
-	}
-
-
-	public static CommunityOrganization asCommunityOrganization(String id, Concept c) {
-		final Map<String, List<Param>> p = c.getParams();
-		final CommunityOrganization o = new CommunityOrganization();
-		o.setCommunityId(id);
-		o.setId(StringUtils.substringAfterLast(c.getId(), ID_SEPARATOR));
-		o.setName(firstValue(p,CORGANIZATION_NAME));
-		o.setLogo_url(getDecodedUrl(firstValue(p,CORGANIZATION_LOGOURL)));
-		o.setWebsite_url(getDecodedUrl(firstValue(p,CORGANIZATION_WEBSITEURL)));
-
-		return o;
-	}
-
-	private static String getDecodedUrl(final String encoded_url){
-		if(encoded_url == null){
-			return encoded_url;
-		}
-		return new String(Base64.getDecoder().decode(encoded_url));
-	}
-
-
-	private static List<String> splitValues(final Stream<String> stream, final String separator) {
-		return stream.map(s -> s.split(separator))
-				.map(Arrays::asList)
-				.flatMap(List::stream)
-				.filter(StringUtils::isNotBlank)
-				.map(StringUtils::trim)
-				.collect(Collectors.toList());
-	}
-
-	private static String firstValue(final Map<String, List<Param>> p, final String paramName) {
-		return asValues(p.get(paramName)).findFirst().orElse(null);
-	}
-
-	private static String asCsv(final List<Param> params) {
-		return asValues(params)
-				.collect(Collectors.joining(CSV_DELIMITER));
-	}
-
-	private static Stream<String> asValues(final List<Param> params) {
-		return params == null ? Stream.empty() : params.stream()
-				.map(Param::getValue)
-				.map(StringUtils::trim)
-				.distinct();
-	}
-
-	public static String asProjectXML(final String contextId, final CommunityProject project) {
-		final Escaper esc = XmlEscapers.xmlAttributeEscaper();
-		final StringBuilder sb = new StringBuilder();
-		sb.append(
-				String.format(
-						"<concept claim='false' id='%s%s%s%s' label='%s'>\n",
-						escape(esc, contextId), PROJECTS_ID_SUFFIX, ID_SEPARATOR, escape(esc, String.valueOf(project.getId())), escape(esc, project.getAcronym())));
-		sb.append(paramXML(CPROJECT_FULLNAME, project.getName()));
-		sb.append(paramXML(CPROJECT_ACRONYM, project.getAcronym()));
-		sb.append(paramXML(CPROJECT_NUMBER, project.getGrantId()));
-		sb.append(paramXML(CPROJECT_FUNDER, project.getFunder()));
-		sb.append(paramXML(OPENAIRE_ID, project.getOpenaireId()));
-		sb.append("</concept>\n");
-		return sb.toString();
-	}
-
-	public static String asContentProviderXML(final String contextId, final CommunityContentprovider ccp) {
-		log.info("creating the XML for the content provider");
-		final Escaper esc = XmlEscapers.xmlAttributeEscaper();
-		final StringBuilder sb = new StringBuilder();
-		sb.append(
-				String.format(
-						"<concept claim='false' id='%s%s%s%s' label='%s'>\n",
-						escape(esc, contextId), CONTENTPROVIDERS_ID_SUFFIX, ID_SEPARATOR, escape(esc, String.valueOf(ccp.getId())), escape(esc, ccp.getName())));
-		sb.append(paramXML(OPENAIRE_ID, ccp.getOpenaireId()));
-		sb.append(paramXML(CCONTENTPROVIDER_NAME, ccp.getName()));
-		sb.append(paramXML(CCONTENTPROVIDER_OFFICIALNAME, ccp.getOfficialname()));
-		sb.append(paramXML(CCONTENTPROVIDER_ENABLED,CCONTENTPROVIDERENABLED_DEFAULT));
-		sb.append(paramXMLNoEscape(CCONTENTPROVIDER_SELCRITERIA, ccp.toXML()));
-		sb.append("</concept>\n");
-		log.info(sb.toString());
-		return sb.toString();
-	}
-
-	public static String asZenodoCommunityXML(final String contextId, final CommunityZenodoCommunity zc) {
-		final Escaper esc = XmlEscapers.xmlAttributeEscaper();
-		final StringBuilder sb = new StringBuilder();
-		sb.append(
-				String.format(
-						"<concept claim='false' id='%s%s%s%s' label='%s'>\n",
-						escape(esc, contextId), ZENODOCOMMUNITY_ID_SUFFIX, ID_SEPARATOR, escape(esc, String.valueOf(zc.getId())), escape(esc, zc.getZenodoid())));
-
-		sb.append(paramXML(CZENODOCOMMUNITY_ID, zc.getZenodoid()));
-		sb.append("</concept>\n");
-		return sb.toString();
-	}
-
-
-	public static String asOrganizationXML(final String contextId, CommunityOrganization organization) {
-		final Escaper esc = XmlEscapers.xmlAttributeEscaper();
-		final StringBuilder sb = new StringBuilder();
-		sb.append(
-				String.format(
-						"<concept claim='false' id='%s%s%s%s' label='%s'>\n",
-						escape(esc, contextId), ORGANIZATION_ID_SUFFIX, ID_SEPARATOR, escape(esc, String.valueOf(organization.getId())), escape(esc, organization.getName())));
-		sb.append(paramXML(CORGANIZATION_NAME, organization.getName()));
-		sb.append(paramXML(CORGANIZATION_LOGOURL, Base64.getEncoder().encodeToString(organization.getLogo_url().getBytes())));
-		sb.append(paramXML(CORGANIZATION_WEBSITEURL,Base64.getEncoder().encodeToString(organization.getWebsite_url().getBytes())));
-		sb.append("</concept>\n");
-		return sb.toString();
-	}
-
-
-	private static String paramXML(final String paramName, final String value) {
-		return String.format("<param name='%s'>%s</param>\n", paramName, escape(XmlEscapers.xmlContentEscaper(), value));
-	}
-
-	private static String paramXMLNoEscape(final String paramName, final String value) {
-		return String.format("<param name='%s'>%s</param>\n", paramName, value);
-	}
-
-}

apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityService.java

@@ -0,0 +1,479 @@
+package eu.dnetlib.openaire.community;
+
+import java.time.LocalDateTime;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.function.BiConsumer;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+
+import javax.persistence.criteria.Predicate;
+import javax.transaction.Transactional;
+
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
+import org.springframework.data.domain.Page;
+import org.springframework.data.domain.PageRequest;
+import org.springframework.data.domain.Sort;
+import org.springframework.data.jpa.domain.Specification;
+import org.springframework.stereotype.Service;
+
+import eu.dnetlib.openaire.community.model.DbCommunity;
+import eu.dnetlib.openaire.community.model.DbDatasource;
+import eu.dnetlib.openaire.community.model.DbDatasourcePK;
+import eu.dnetlib.openaire.community.model.DbOrganization;
+import eu.dnetlib.openaire.community.model.DbProject;
+import eu.dnetlib.openaire.community.model.DbProjectPK;
+import eu.dnetlib.openaire.community.model.DbSubCommunity;
+import eu.dnetlib.openaire.community.model.DbSupportOrg;
+import eu.dnetlib.openaire.community.model.DbSupportOrgPK;
+import eu.dnetlib.openaire.community.repository.DbCommunityRepository;
+import eu.dnetlib.openaire.community.repository.DbDatasourceRepository;
+import eu.dnetlib.openaire.community.repository.DbOrganizationRepository;
+import eu.dnetlib.openaire.community.repository.DbProjectRepository;
+import eu.dnetlib.openaire.community.repository.DbSubCommunityRepository;
+import eu.dnetlib.openaire.community.repository.DbSupportOrgRepository;
+import eu.dnetlib.openaire.community.utils.CommunityMappingUtils;
+import eu.dnetlib.openaire.exporter.exceptions.CommunityException;
+import eu.dnetlib.openaire.exporter.exceptions.ResourceNotFoundException;
+import eu.dnetlib.openaire.exporter.model.community.CommunityContentprovider;
+import eu.dnetlib.openaire.exporter.model.community.CommunityDetails;
+import eu.dnetlib.openaire.exporter.model.community.CommunityOrganization;
+import eu.dnetlib.openaire.exporter.model.community.CommunityProject;
+import eu.dnetlib.openaire.exporter.model.community.CommunitySummary;
+import eu.dnetlib.openaire.exporter.model.community.CommunityWritableProperties;
+import eu.dnetlib.openaire.exporter.model.community.SubCommunity;
+import eu.dnetlib.openaire.exporter.model.community.selectioncriteria.SelectionCriteria;
+import eu.dnetlib.openaire.exporter.model.context.IISConfigurationEntry;
+
+@Service
+@ConditionalOnProperty(value = "openaire.exporter.enable.community", havingValue = "true")
+public class CommunityService {
+
+	@Autowired
+	private DbCommunityRepository dbCommunityRepository;
+	@Autowired
+	private DbProjectRepository dbProjectRepository;
+	@Autowired
+	private DbDatasourceRepository dbDatasourceRepository;
+	@Autowired
+	private DbOrganizationRepository dbOrganizationRepository;
+	@Autowired
+	private DbSupportOrgRepository dbSupportOrgRepository;
+	@Autowired
+	private DbSubCommunityRepository dbSubCommunityRepository;
+
+	private static final Log log = LogFactory.getLog(CommunityService.class);
+
+	public List<CommunitySummary> listCommunities() {
+		return dbCommunityRepository.findAll()
+				.stream()
+				.map(CommunityMappingUtils::toCommunitySummary)
+				.collect(Collectors.toList());
+	}
+
+	@Transactional
+	public CommunityDetails newCommunity(final CommunityDetails details) throws CommunityException {
+		if (StringUtils.isBlank(details.getId())) { throw new CommunityException("Empty Id"); }
+		if (dbCommunityRepository.existsById(details.getId())) { throw new CommunityException("Community already exists: " + details.getId()); }
+		details.setCreationDate(LocalDateTime.now());
+		return saveCommunity(details);
+
+	}
+
+	@Transactional
+	public CommunityDetails saveCommunity(final CommunityDetails details) {
+		details.setLastUpdateDate(LocalDateTime.now());
+		dbCommunityRepository.save(CommunityMappingUtils.toCommunity(details));
+		return getCommunity(details.getId());
+	}
+
+	@Transactional
+	public CommunityDetails getCommunity(final String id) {
+		final DbCommunity c = dbCommunityRepository.findById(id).orElseThrow(() -> new ResourceNotFoundException("Community not found: " + id));
+		return CommunityMappingUtils.toCommunityDetails(c);
+	}
+
+	@Transactional
+	public void setCommunity(final String id, final CommunityWritableProperties details) {
+		final DbCommunity c = dbCommunityRepository.findById(id).orElseThrow(() -> new ResourceNotFoundException("Community not found: " + id));
+		CommunityMappingUtils.populateCommunity(c, details);
+		c.setLastUpdateDate(LocalDateTime.now());
+		dbCommunityRepository.save(c);
+	}
+
+	@Transactional
+	public Page<CommunityProject> getCommunityProjects(final String id,
+			final String funder,
+			final String filter,
+			final int page,
+			final int size,
+			final String orderBy) throws CommunityException {
+		if (StringUtils.isBlank(id)) { throw new CommunityException("Empty ID"); }
+		try {
+			final Sort sort;
+			if (StringUtils.isBlank(orderBy)) {
+				sort = Sort.by("projectName");
+			} else if ("funder".equalsIgnoreCase(orderBy)) {
+				sort = Sort.by("projectFunder").and(Sort.by("projectName"));
+			} else if ("grantId".equalsIgnoreCase(orderBy)) {
+				sort = Sort.by("projectCode");
+			} else if ("acronym".equalsIgnoreCase(orderBy)) {
+				sort = Sort.by("projectAcronym");
+			} else if ("openaireId".equalsIgnoreCase(orderBy)) {
+				sort = Sort.by("projectId");
+			} else {
+				sort = Sort.by("projectName");
+			}
+
+			final PageRequest pageable = PageRequest.of(page, size, sort);
+			if (StringUtils.isAllBlank(filter, funder)) {
+				return dbProjectRepository.findByCommunity(id, pageable).map(CommunityMappingUtils::toCommunityProject);
+			}
+			final Specification<DbProject> projSpec = prepareProjectSpec(id, funder, filter);
+			return dbProjectRepository.findAll(projSpec, pageable).map(CommunityMappingUtils::toCommunityProject);
+		} catch (final Throwable e) {
+			log.error(e);
+			throw new CommunityException(e);
+		}
+	}
+
+	private Specification<DbProject> prepareProjectSpec(final String community, final String funder, final String other) {
+		return (project, query, cb) -> {
+
+			final List<Predicate> andConds = new ArrayList<>();
+			andConds.add(cb.equal(project.get("community"), community));
+
+			if (StringUtils.isNotBlank(funder)) {
+				andConds.add(cb.equal(project.get("projectFunder"), funder));
+			}
+
+			if (StringUtils.isNotBlank(other)) {
+				final String s = other.toLowerCase().trim();
+
+				final List<Predicate> orConds = new ArrayList<>();
+				orConds.add(cb.equal(cb.lower(project.get("projectId")), s));
+				orConds.add(cb.equal(cb.lower(project.get("projectCode")), s));
+				orConds.add(cb.equal(cb.lower(project.get("projectAcronym")), s));
+				orConds.add(cb.like(cb.lower(project.get("projectName")), "%" + s + "%"));
+				if (StringUtils.isBlank(funder)) {
+					orConds.add(cb.equal(cb.lower(project.get("projectFunder")), s));
+				}
+
+				andConds.add(cb.or(orConds.toArray(new Predicate[orConds.size()])));
+			}
+
+			return cb.and(andConds.toArray(new Predicate[andConds.size()]));
+		};
+
+	}
+
+	@Transactional
+	public CommunityProject addCommunityProject(final String id, final CommunityProject project) {
+		final DbProject p = CommunityMappingUtils.toDbProject(id, project);
+		dbProjectRepository.save(p);
+		return project;
+	}
+
+	@Transactional
+	public void addCommunityProjects(final String id, final CommunityProject... projects) throws CommunityException {
+		try {
+			final List<DbProject> list = Arrays.stream(projects)
+					.map(p -> CommunityMappingUtils.toDbProject(id, p))
+					.collect(Collectors.toList());
+
+			dbProjectRepository.saveAll(list);
+		} catch (final Throwable e) {
+			log.error(e);
+			throw new CommunityException(e);
+		}
+	}
+
+	@Transactional
+	public void removeCommunityProjects(final String id, final String... ids) {
+		final List<DbProjectPK> list = Arrays.stream(ids)
+				.map(projectId -> new DbProjectPK(id, projectId))
+				.collect(Collectors.toList());
+		dbProjectRepository.deleteAllById(list);
+	}
+
+	public List<CommunityContentprovider> getCommunityDatasources(final String id) {
+		return dbDatasourceRepository.findByCommunity(id)
+				.stream()
+				.map(CommunityMappingUtils::toCommunityContentprovider)
+				.collect(Collectors.toList());
+	}
+
+	public List<CommunityContentprovider> getCommunityDatasourcesWithDeposit(final String id, final boolean deposit) {
+		return dbDatasourceRepository.findByCommunityAndDeposit(id, deposit)
+				.stream()
+				.map(CommunityMappingUtils::toCommunityContentprovider)
+				.collect(Collectors.toList());
+	}
+
+	@Transactional
+	public CommunityContentprovider updateCommunityDatasourcesDeposit(final String id, final String dsId, final Boolean deposit, final String message) {
+		return dbDatasourceRepository.findById(new DbDatasourcePK(id, dsId))
+				.map(ds -> {
+					ds.setDeposit(deposit != null ? deposit : false);
+					ds.setMessage(message);
+					return ds;
+				})
+				.map(CommunityMappingUtils::toCommunityContentprovider)
+				.orElseThrow(() -> new ResourceNotFoundException("Community and/or Datasource not found"));
+	}
+
+	@Transactional
+	public void addCommunityDatasources(final String id, final CommunityContentprovider... contentproviders) {
+		final List<DbDatasource> list = Arrays.stream(contentproviders)
+				.map(cp -> CommunityMappingUtils.toDbDatasource(id, cp))
+				.collect(Collectors.toList());
+
+		dbDatasourceRepository.saveAll(list);
+	}
+
+	@Transactional
+	public void removeCommunityDatasources(final String id, final String... ids) {
+		final List<DbDatasourcePK> list = Arrays.stream(ids)
+				.map(dsId -> new DbDatasourcePK(id, dsId))
+				.collect(Collectors.toList());
+		dbDatasourceRepository.deleteAllById(list);
+	}
+
+	@Transactional
+	public void removeCommunityOrganizations(final String id, final String... orgNames) {
+		final List<DbSupportOrgPK> list = Arrays.stream(orgNames)
+				.map(name -> new DbSupportOrgPK(id, name))
+				.collect(Collectors.toList());
+		dbSupportOrgRepository.deleteAllById(list);
+	}
+
+	@Transactional
+	public List<CommunityOrganization> getCommunityOrganizations(final String id) {
+		return dbSupportOrgRepository.findByCommunity(id)
+				.stream()
+				.map(CommunityMappingUtils::toCommunityOrganization)
+				.collect(Collectors.toList());
+	}
+
+	@Transactional
+	public void addCommunityOrganizations(final String id, final CommunityOrganization... orgs) {
+		final List<DbSupportOrg> list = Arrays.stream(orgs)
+				.map(o -> CommunityMappingUtils.toDbSupportOrg(id, o))
+				.collect(Collectors.toList());
+
+		dbSupportOrgRepository.saveAll(list);
+	}
+
+	@Transactional
+	public void removeSubCommunities(final String id, final String... subCommunityIds) {
+		dbSubCommunityRepository.deleteAllById(Arrays.asList(subCommunityIds));
+	}
+
+	@Transactional
+	public List<SubCommunity> getSubCommunities(final String id) {
+		return dbSubCommunityRepository.findByCommunity(id)
+				.stream()
+				.map(CommunityMappingUtils::toSubCommunity)
+				.collect(Collectors.toList());
+	}
+
+	@Transactional
+	public void addSubCommunities(final String id, final SubCommunity... subs) {
+		final List<DbSubCommunity> list = Arrays.stream(subs)
+				.map(s -> CommunityMappingUtils.toDbSubCommunity(id, s))
+				.collect(Collectors.toList());
+
+		dbSubCommunityRepository.saveAll(list);
+	}
+
+	@Transactional
+	public CommunityDetails addCommunitySubjects(final String id, final String... subjects) {
+		return modifyElementToArrayField(id, DbCommunity::getSubjects, DbCommunity::setSubjects, false, subjects);
+	}
+
+	public CommunityDetails removeCommunitySubjects(final String id, final String... subjects) {
+		return modifyElementToArrayField(id, DbCommunity::getSubjects, DbCommunity::setSubjects, true, subjects);
+	}
+
+	public CommunityDetails addCommunityFOS(final String id, final String... foss) {
+		return modifyElementToArrayField(id, DbCommunity::getFos, DbCommunity::setFos, false, foss);
+	}
+
+	public CommunityDetails removeCommunityFOS(final String id, final String... foss) {
+		return modifyElementToArrayField(id, DbCommunity::getFos, DbCommunity::setFos, true, foss);
+	}
+
+	public CommunityDetails addCommunitySDG(final String id, final String... sdgs) {
+		return modifyElementToArrayField(id, DbCommunity::getSdg, DbCommunity::setSdg, false, sdgs);
+	}
+
+	public CommunityDetails removeCommunitySDG(final String id, final String... sdgs) {
+		return modifyElementToArrayField(id, DbCommunity::getSdg, DbCommunity::setSdg, true, sdgs);
+	}
+
+	@Transactional
+	public CommunityDetails addCommunityAdvancedConstraint(final String id, final SelectionCriteria advancedCosntraint) {
+		final DbCommunity dbEntry = dbCommunityRepository.findById(id).orElseThrow(() -> new ResourceNotFoundException("Community not found: " + id));
+		dbEntry.setAdvancedConstraints(advancedCosntraint);
+		dbEntry.setLastUpdateDate(LocalDateTime.now());
+		dbCommunityRepository.save(dbEntry);
+		return getCommunity(id);
+	}
+
+	@Transactional
+	public CommunityDetails removeCommunityAdvancedConstraint(final String id) {
+		final DbCommunity dbEntry = dbCommunityRepository.findById(id).orElseThrow(() -> new ResourceNotFoundException("Community not found: " + id));
+		dbEntry.setAdvancedConstraints(null);
+		dbEntry.setLastUpdateDate(LocalDateTime.now());
+		dbCommunityRepository.save(dbEntry);
+		return getCommunity(id);
+	}
+
+	@Transactional
+	public CommunityDetails addCommunityRemoveConstraint(final String id, final SelectionCriteria removeConstraint) {
+		final DbCommunity dbEntry = dbCommunityRepository.findById(id).orElseThrow(() -> new ResourceNotFoundException("Community not found: " + id));
+		dbEntry.setRemoveConstraints(removeConstraint);
+		dbEntry.setLastUpdateDate(LocalDateTime.now());
+		dbCommunityRepository.save(dbEntry);
+		return getCommunity(id);
+	}
+
+	@Transactional
+	public CommunityDetails removeCommunityRemoveConstraint(final String id) {
+		final DbCommunity dbEntry = dbCommunityRepository.findById(id).orElseThrow(() -> new ResourceNotFoundException("Community not found: " + id));
+		dbEntry.setRemoveConstraints(null);
+		dbEntry.setLastUpdateDate(LocalDateTime.now());
+		dbCommunityRepository.save(dbEntry);
+		return getCommunity(id);
+	}
+
+	public CommunityDetails removeCommunityZenodoCommunity(final String id, final String zenodoCommunity, final boolean isMain) {
+		if (isMain) { return updateElementToSimpleField(id, DbCommunity::setMainZenodoCommunity, null); }
+		return modifyElementToArrayField(id, DbCommunity::getOtherZenodoCommunities, DbCommunity::setOtherZenodoCommunities, true, zenodoCommunity);
+	}
+
+	public CommunityDetails addCommunityZenodoCommunity(final String id, final String zenodoCommunity, final boolean isMain) {
+		if (isMain) { return updateElementToSimpleField(id, DbCommunity::setMainZenodoCommunity, zenodoCommunity); }
+		return modifyElementToArrayField(id, DbCommunity::getOtherZenodoCommunities, DbCommunity::setOtherZenodoCommunities, false, zenodoCommunity);
+	}
+
+	@Transactional
+	private CommunityDetails updateElementToSimpleField(final String id,
+			final BiConsumer<DbCommunity, String> setter,
+			final String value) {
+		final DbCommunity dbEntry = dbCommunityRepository.findById(id).orElseThrow(() -> new ResourceNotFoundException("Community not found: " + id));
+		setter.accept(dbEntry, value);
+		dbEntry.setLastUpdateDate(LocalDateTime.now());
+		dbCommunityRepository.save(dbEntry);
+		return getCommunity(id);
+	}
+
+	@Transactional
+	private CommunityDetails modifyElementToArrayField(final String id,
+			final Function<DbCommunity, String[]> getter,
+			final BiConsumer<DbCommunity, String[]> setter,
+			final boolean remove,
+			final String... values) {
+
+		final DbCommunity dbEntry = dbCommunityRepository.findById(id).orElseThrow(() -> new ResourceNotFoundException("Community not found: " + id));
+
+		final Set<String> tmpList = new LinkedHashSet<>();
+		final String[] oldValues = getter.apply(dbEntry);
+		if (oldValues != null) {
+			Collections.addAll(tmpList, oldValues);
+		}
+		if (remove) {
+			tmpList.removeAll(Arrays.asList(values));
+		} else {
+			tmpList.addAll(Arrays.asList(values));
+		}
+
+		setter.accept(dbEntry, tmpList.toArray(new String[tmpList.size()]));
+
+		dbEntry.setLastUpdateDate(LocalDateTime.now());
+
+		dbCommunityRepository.save(dbEntry);
+
+		return getCommunity(id);
+	}
+
+	@Transactional
+	public List<String> getOpenAIRECommunitiesByZenodoId(final String zenodoId) {
+		return dbCommunityRepository.findByZenodoId(zenodoId);
+	}
+
+	@Transactional
+	public Map<String, Set<String>> getPropagationOrganizationCommunityMap() {
+		return dbOrganizationRepository.findAll()
+				.stream()
+				.collect(Collectors.groupingBy(DbOrganization::getOrgId, Collectors.mapping(DbOrganization::getCommunity, Collectors.toSet())));
+	}
+
+	@Transactional
+	public Set<String> getPropagationOrganizationsForCommunity(final String communityId) {
+		return dbOrganizationRepository.findByCommunity(communityId)
+				.stream()
+				.map(DbOrganization::getOrgId)
+				.collect(Collectors.toSet());
+	}
+
+	@Transactional
+	public Set<String> addPropagationOrganizationForCommunity(final String communityId, final String... organizationIds) {
+		for (final String orgId : organizationIds) {
+			final DbOrganization o = new DbOrganization(communityId.trim(), orgId.trim());
+			dbOrganizationRepository.save(o);
+		}
+		return getPropagationOrganizationsForCommunity(communityId);
+	}
+
+	@Transactional
+	public Set<String> removePropagationOrganizationForCommunity(final String communityId, final String... organizationIds) {
+		for (final String orgId : organizationIds) {
+			final DbOrganization o = new DbOrganization(communityId.trim(), orgId.trim());
+			dbOrganizationRepository.delete(o);
+		}
+		return getPropagationOrganizationsForCommunity(communityId);
+	}
+
+	@Transactional
+	public void deleteCommunity(final String id, final boolean recursive) {
+		if (recursive) {
+			dbProjectRepository.deleteByCommunity(id);
+			dbDatasourceRepository.deleteByCommunity(id);
+			dbOrganizationRepository.deleteByCommunity(id);
+			dbSupportOrgRepository.deleteByCommunity(id);
+			dbSubCommunityRepository.deleteByCommunity(id);
+		}
+		dbCommunityRepository.deleteById(id);
+	}
+
+	@Transactional
+	public List<IISConfigurationEntry> getIISConfiguration(final String id) {
+		final List<IISConfigurationEntry> res = new ArrayList<>();
+
+		res.add(dbCommunityRepository.findById(id)
+				.map(CommunityMappingUtils::asIISConfigurationEntry)
+				.orElseThrow(() -> new ResourceNotFoundException("Community not found: " + id)));
+
+		for (final DbSubCommunity subc : dbSubCommunityRepository.findByCommunity(id)) {
+			res.add(CommunityMappingUtils.asIISConfigurationEntry(subc));
+		}
+
+		return res;
+	}
+
+	@Transactional
+	public List<String> getCommunityFunders(final String id) {
+		return dbProjectRepository.findFundersByCommunity(id);
+	}
+
+}

apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/importer/CommunityImporterController.java

@@ -0,0 +1,87 @@
+package eu.dnetlib.openaire.community.importer;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
+import org.springframework.web.bind.annotation.CrossOrigin;
+import org.springframework.web.bind.annotation.GetMapping;
+import org.springframework.web.bind.annotation.RequestParam;
+import org.springframework.web.bind.annotation.RestController;
+
+import com.google.common.collect.Sets;
+
+import eu.dnetlib.common.controller.AbstractDnetController;
+import eu.dnetlib.openaire.common.ISClient;
+import eu.dnetlib.openaire.community.model.DbOrganization;
+import eu.dnetlib.openaire.exporter.exceptions.CommunityException;
+import eu.dnetlib.openaire.exporter.model.context.Context;
+import io.swagger.v3.oas.annotations.tags.Tag;
+
+@RestController
+@CrossOrigin(origins = {
+	"*"
+})
+@ConditionalOnProperty(value = "openaire.exporter.enable.community.import", havingValue = "true")
+@Tag(name = "OpenAIRE Communities: Migration API", description = "OpenAIRE Communities: Migration API")
+public class CommunityImporterController extends AbstractDnetController {
+
+	// public final static Set<String> communityBlackList = Sets.newHashSet("fet-fp7", "fet-h2020");
+	public final static Set<String> communityBlackList = Sets.newHashSet();
+
+	@Autowired
+	private CommunityImporterService importer;
+
+	@Autowired
+	private ISClient isClient;
+
+	private static final Log log = LogFactory.getLog(CommunityImporterController.class);
+
+	@GetMapping("/community_importer/communities")
+	public List<String> importProfiles() throws CommunityException {
+		try {
+			final Map<String, Context> contextMap = getContextMap();
+
+			final List<String> list = contextMap.keySet()
+				.stream()
+				.filter(id -> !communityBlackList.contains(id))
+				.collect(Collectors.toList());
+
+			list.forEach(id -> {
+				importer.importCommunity(contextMap.get(id));
+			});
+
+			return list;
+		} catch (final Throwable e) {
+			log.error("Error importing communities", e);
+			throw new CommunityException(e.getMessage());
+		}
+	}
+
+	@GetMapping("/community_importer/propagationOrgs")
+	public List<DbOrganization> importPropagationOrgs(@RequestParam final String profileId,
+		@RequestParam(required = false, defaultValue = "false") final boolean simulation) throws Exception {
+		try {
+			final String xml = isClient.getProfile(profileId);
+			return importer.importPropagationOrganizationsFromProfile(xml, simulation);
+		} catch (final Throwable e) {
+			log.error("Error importing communities", e);
+			throw new CommunityException(e.getMessage());
+		}
+	}
+
+	private Map<String, Context> getContextMap() throws CommunityException {
+		try {
+			return isClient.getCommunityContextMap();
+		} catch (final IOException e) {
+			throw new CommunityException(e);
+		}
+	}
+
+}

apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/importer/CommunityImporterService.java

@@ -0,0 +1,411 @@
+package eu.dnetlib.openaire.community.importer;
+
+import java.time.LocalDate;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Base64;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+import javax.transaction.Transactional;
+
+import org.apache.commons.lang3.BooleanUtils;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.dom4j.DocumentHelper;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
+import org.springframework.jdbc.core.JdbcTemplate;
+import org.springframework.stereotype.Service;
+
+import com.fasterxml.jackson.core.type.TypeReference;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.google.common.collect.Lists;
+
+import eu.dnetlib.miscutils.functional.hash.Hashing;
+import eu.dnetlib.openaire.community.CommunityService;
+import eu.dnetlib.openaire.community.model.DbOrganization;
+import eu.dnetlib.openaire.community.repository.DbOrganizationRepository;
+import eu.dnetlib.openaire.community.utils.CommunityMappingUtils;
+import eu.dnetlib.openaire.exporter.exceptions.CommunityException;
+import eu.dnetlib.openaire.exporter.model.community.CommunityClaimType;
+import eu.dnetlib.openaire.exporter.model.community.CommunityContentprovider;
+import eu.dnetlib.openaire.exporter.model.community.CommunityDetails;
+import eu.dnetlib.openaire.exporter.model.community.CommunityMembershipType;
+import eu.dnetlib.openaire.exporter.model.community.CommunityOrganization;
+import eu.dnetlib.openaire.exporter.model.community.CommunityPlanType;
+import eu.dnetlib.openaire.exporter.model.community.CommunityProject;
+import eu.dnetlib.openaire.exporter.model.community.CommunityStatus;
+import eu.dnetlib.openaire.exporter.model.community.CommunityType;
+import eu.dnetlib.openaire.exporter.model.community.SubCommunity;
+import eu.dnetlib.openaire.exporter.model.community.selectioncriteria.SelectionCriteria;
+import eu.dnetlib.openaire.exporter.model.context.Category;
+import eu.dnetlib.openaire.exporter.model.context.Concept;
+import eu.dnetlib.openaire.exporter.model.context.Context;
+import eu.dnetlib.openaire.exporter.model.context.Param;
+
+@Service
+@ConditionalOnProperty(value = "openaire.exporter.enable.community.import", havingValue = "true")
+public class CommunityImporterService {
+
+	// common
+	public final static String OPENAIRE_ID = "openaireId";
+	public final static String PIPE_SEPARATOR = "||";
+	public final static String ID_SEPARATOR = "::";
+	public final static String CSV_DELIMITER = ",";
+	public final static String CLABEL = "label";
+
+	// id suffixes
+	public final static String PROJECTS_ID_SUFFIX = ID_SEPARATOR + "projects";
+	public final static String CONTENTPROVIDERS_ID_SUFFIX = ID_SEPARATOR + "contentproviders";
+	public final static String ZENODOCOMMUNITY_ID_SUFFIX = ID_SEPARATOR + "zenodocommunities";
+	public final static String ORGANIZATION_ID_SUFFIX = ID_SEPARATOR + "organizations";
+
+	// community summary
+	public final static String CSUMMARY_DESCRIPTION = "description";
+	public final static String CSUMMARY_LOGOURL = "logourl";
+	public final static String CSUMMARY_STATUS = "status";
+	public final static String CSUMMARY_NAME = "name";
+	public final static String CSUMMARY_MANAGER = "manager";
+	public final static String CSUMMARY_ZENODOC = "zenodoCommunity";
+
+	// community profile
+	public final static String CPROFILE_SUBJECT = "subject";
+	public final static String CPROFILE_CREATIONDATE = "creationdate";
+	public final static String CPROFILE_FOS = "fos";
+	public final static String CPROFILE_SDG = "sdg";
+	public final static String CPROFILE_ADVANCED_CONSTRAINT = "advancedConstraints";
+	public final static String CPROFILE_REMOVE_CONSTRAINT = "removeConstraints";
+	public final static String CPROFILE_SUGGESTED_ACKNOWLEDGEMENT = "suggestedAcknowledgement";
+
+	// community project
+	public final static String CPROJECT_FUNDER = "funder";
+	public final static String CPROJECT_NUMBER = "CD_PROJECT_NUMBER";
+	public final static String CPROJECT_FULLNAME = "projectfullname";
+	public final static String CPROJECT_ACRONYM = "acronym";
+
+	// community content provider
+	public final static String CCONTENTPROVIDER_NAME = "name";
+	public final static String CCONTENTPROVIDER_OFFICIALNAME = "officialname";
+	public final static String CCONTENTPROVIDER_ENABLED = "enabled";
+	public final static String CCONTENTPROVIDERENABLED_DEFAULT = "true";
+	public final static String CCONTENTPROVIDER_SELCRITERIA = "selcriteria";
+
+	// community zenodo community
+	public final static String CZENODOCOMMUNITY_ID = "zenodoid";
+
+	// community organization
+	public final static String CORGANIZATION_NAME = "name";
+	public final static String CORGANIZATION_LOGOURL = "logourl";
+	public final static String CORGANIZATION_WEBSITEURL = "websiteurl";
+
+	@Autowired
+	private DbOrganizationRepository dbOrganizationRepository;
+
+	@Autowired
+	private CommunityService service;
+
+	@Autowired
+	private JdbcTemplate jdbcTemplate;
+
+	private static final Log log = LogFactory.getLog(CommunityImporterService.class);
+
+	public List<DbOrganization> importPropagationOrganizationsFromProfile(final String xml, final boolean simulation) throws Exception {
+		final String json = DocumentHelper.parseText(xml)
+				.selectSingleNode("//NODE[@name='setPropagationOrganizationCommunityMap']//PARAM[@name='parameterValue']")
+				.getText();
+
+		final List<DbOrganization> list = new ObjectMapper()
+				.readValue(json, new TypeReference<Map<String, List<String>>>() {})
+				.entrySet()
+				.stream()
+				.flatMap(e -> e.getValue()
+						.stream()
+						.map(community -> {
+							if (e.getKey().contains("|")) { return new DbOrganization(community, StringUtils.substringAfter(e.getKey(), "|")); }
+							return new DbOrganization(community, e.getKey());
+						}))
+				.collect(Collectors.toList());
+
+		if (!simulation) {
+			list.forEach(o -> {
+				try {
+					dbOrganizationRepository.save(o);
+				} catch (final Throwable e) {
+					log.error("ERROR saving org: " + o);
+				}
+			});
+		}
+
+		return list;
+	}
+
+	@Transactional
+	public void importCommunity(final Context context) {
+		try {
+
+			final CommunityDetails community = asCommunityDetails(context);
+
+			final List<CommunityContentprovider> datasources =
+					getCommunityInfo(context, CONTENTPROVIDERS_ID_SUFFIX, c -> asCommunityDataprovider(context.getId(), c))
+							.stream()
+							.map(o -> {
+								if (o.getOpenaireId() == null) {
+									log.warn("Openaire ID is missing, organization: " + o.getOfficialname());
+								} else if (o.getOpenaireId().contains("|")) {
+									o.setOpenaireId(StringUtils.substringAfter(o.getOpenaireId(), "|"));
+								}
+								return o;
+							})
+							.filter(o -> o.getOpenaireId() != null)
+							.collect(Collectors.toList());
+
+			final List<CommunityProject> projects =
+					getCommunityInfo(context, PROJECTS_ID_SUFFIX, c -> asCommunityProject(context.getId(), c))
+							.stream()
+							.map(p -> {
+								if (p.getOpenaireId() == null) {
+									if ("EC".equalsIgnoreCase(p.getFunder())) {
+										final String ns = findNamespaceForECProject(p.getGrantId());
+										if (ns != null) {
+											p.setOpenaireId(ns + "::" + Hashing.md5(p.getGrantId()));
+										} else {
+											log.warn("EC project not in the db: " + p.getGrantId());
+										}
+									} else if ("NSF".equalsIgnoreCase(p.getFunder())) {
+										p.setOpenaireId("nsf_________::" + Hashing.md5(p.getGrantId()));
+									} else if ("NIH".equalsIgnoreCase(p.getFunder())) {
+										p.setOpenaireId("nih_________::" + Hashing.md5(p.getGrantId()));
+									} else {
+										log.warn("Openaire ID is missing, funder: " + p.getFunder());
+									}
+								} else if (p.getOpenaireId().contains("|")) {
+									p.setOpenaireId(StringUtils.substringAfter(p.getOpenaireId(), "|"));
+								}
+								return p;
+							})
+							.filter(p -> p.getOpenaireId() != null)
+							.collect(Collectors.toList());
+
+			final List<CommunityOrganization> orgs =
+					getCommunityInfo(context, ORGANIZATION_ID_SUFFIX, c -> asCommunityOrganization(context.getId(), c));
+
+			final List<String> otherZenodoCommunities =
+					getCommunityInfo(context, ZENODOCOMMUNITY_ID_SUFFIX, CommunityImporterService::asZenodoCommunity);
+
+			community.setOtherZenodoCommunities(otherZenodoCommunities);
+
+			final List<SubCommunity> subs = context.getCategories()
+					.entrySet()
+					.stream()
+					.filter(e -> !(context.getId() + CONTENTPROVIDERS_ID_SUFFIX).equals(e.getKey()))
+					.filter(e -> !(context.getId() + PROJECTS_ID_SUFFIX).equals(e.getKey()))
+					.filter(e -> !(context.getId() + ORGANIZATION_ID_SUFFIX).equals(e.getKey()))
+					.filter(e -> !(context.getId() + ZENODOCOMMUNITY_ID_SUFFIX).equals(e.getKey()))
+					.map(Entry::getValue)
+					.map(cat -> asSubCommunities(context.getId(), null, cat.getLabel(), cat.getConcepts()))
+					.flatMap(List::stream)
+					.collect(Collectors.toList());
+
+			service.saveCommunity(community);
+			service.addCommunityProjects(context.getId(), projects.toArray(new CommunityProject[projects.size()]));
+			service.addCommunityDatasources(context.getId(), datasources.toArray(new CommunityContentprovider[datasources.size()]));
+			service.addCommunityOrganizations(context.getId(), orgs.toArray(new CommunityOrganization[orgs.size()]));
+			service.addSubCommunities(context.getId(), subs.toArray(new SubCommunity[subs.size()]));
+		} catch (
+
+		final Exception e) {
+			throw new RuntimeException("Error importing community: " + context.getId(), e);
+		}
+	}
+
+	private <R> List<R> getCommunityInfo(final Context context, final String idSuffix, final Function<Concept, R> mapping)
+			throws CommunityException {
+		if (context != null) {
+			final Map<String, Category> categories = context.getCategories();
+			final Category category = categories.get(context.getId() + idSuffix);
+			if (category != null) { return category.getConcepts()
+					.stream()
+					.map(mapping)
+					.collect(Collectors.toList()); }
+		}
+		return Lists.newArrayList();
+	}
+
+	private static CommunityDetails asCommunityDetails(final Context c) {
+
+		final CommunityDetails details = new CommunityDetails();
+
+		details.setId(c.getId());
+		details.setShortName(c.getLabel());
+		details.setDisplayShortName(c.getLabel());
+		details.setLastUpdateDate(CommunityMappingUtils.asLocalDateTime(c.getLastUpdateDate()));
+		details.setCreationDate(CommunityMappingUtils.asLocalDateTime(c.getCreationDate()));
+		details.setQueryId(c.getId() + PIPE_SEPARATOR + c.getLabel());
+		details.setType(CommunityType.valueOf(c.getType()));
+		details.setMembership(CommunityMembershipType.open);
+		details.setClaim(CommunityClaimType.all);
+		details.setDescription(asCsv(CSUMMARY_DESCRIPTION, c.getParams()));
+		details.setLogoUrl(asCsv(CSUMMARY_LOGOURL, c.getParams()));
+
+		final String status = firstValue(CSUMMARY_STATUS, c.getParams());
+		if (StringUtils.isNotBlank(status)) {
+			details.setStatus(CommunityStatus.valueOf(status));
+		} else {
+			details.setStatus(CommunityStatus.hidden);
+		}
+
+		details.setName(StringUtils.firstNonBlank(asCsv(CSUMMARY_NAME, c.getParams()), c.getLabel()));
+		details.setDisplayName(StringUtils.firstNonBlank(asCsv(CSUMMARY_NAME, c.getParams()), c.getLabel()));
+
+		details.setZenodoCommunity(asCsv(CSUMMARY_ZENODOC, c.getParams()));
+		details.setSubjects(splitValues(asValues(CPROFILE_SUBJECT, c.getParams()), CSV_DELIMITER));
+		details.setFos(splitValues(asValues(CPROFILE_FOS, c.getParams()), CSV_DELIMITER));
+		details.setSdg(splitValues(asValues(CPROFILE_SDG, c.getParams()), CSV_DELIMITER));
+		// In the map the string is the serialization of the json representing the selection criteria so it is a valid json
+		details.setAdvancedConstraints(SelectionCriteria.fromJson(asCsv(CPROFILE_ADVANCED_CONSTRAINT, c.getParams())));
+		// In the map the string is the serialization of the json representing the selection criteria so it is a valid json
+		details.setRemoveConstraints(SelectionCriteria.fromJson(asCsv(CPROFILE_REMOVE_CONSTRAINT, c.getParams())));
+		details.setSuggestedAcknowledgements(splitValues(asValues(CPROFILE_SUGGESTED_ACKNOWLEDGEMENT, c.getParams()), CSV_DELIMITER));
+		details.setPlan(CommunityPlanType.Default);
+		try {
+			details.setCreationDate(CommunityMappingUtils.asLocalDateTime(asCsv(CPROFILE_CREATIONDATE, c.getParams())));
+		} catch (final Exception e) {
+			log.debug("Exception on date format: " + e.getMessage());
+		}
+
+		return details;
+	}
+
+	private static CommunityProject asCommunityProject(final String communityId, final Concept c) {
+		final List<Param> p = c.getParams();
+		final CommunityProject project = new CommunityProject();
+		project.setCommunityId(communityId);
+		project.setOpenaireId(firstValue(OPENAIRE_ID, p));
+		project.setFunder(firstValue(CPROJECT_FUNDER, p));
+		project.setGrantId(firstValue(CPROJECT_NUMBER, p));
+		project.setName(firstValue(CPROJECT_FULLNAME, p));
+		project.setAcronym(firstValue(CPROJECT_ACRONYM, p));
+		project.setAvailableSince(LocalDate.of(2017, 2, 25)); // Birillo Birth Date
+		return project;
+	}
+
+	private static CommunityContentprovider asCommunityDataprovider(final String communityId, final Concept c) {
+		final List<Param> p = c.getParams();
+		final CommunityContentprovider d = new CommunityContentprovider();
+		d.setCommunityId(communityId);
+		d.setOpenaireId(firstValue(OPENAIRE_ID, p));
+		d.setName(firstValue(CCONTENTPROVIDER_NAME, p));
+		d.setOfficialname(firstValue(CCONTENTPROVIDER_OFFICIALNAME, p));
+		d.setEnabled(BooleanUtils.toBoolean(firstValue(CCONTENTPROVIDER_ENABLED, p)));
+		d.setSelectioncriteria(SelectionCriteria.fromJson(firstValue(CCONTENTPROVIDER_SELCRITERIA, p)));
+		d.setDeposit(false);
+		d.setMessage(null);
+		return d;
+	}
+
+	private static CommunityOrganization asCommunityOrganization(final String id, final Concept c) {
+		final List<Param> p = c.getParams();
+		final CommunityOrganization o = new CommunityOrganization();
+		o.setCommunityId(id);
+		o.setName(firstValue(CORGANIZATION_NAME, p));
+		o.setLogo_url(getDecodedUrl(firstValue(CORGANIZATION_LOGOURL, p)));
+		o.setWebsite_url(getDecodedUrl(firstValue(CORGANIZATION_WEBSITEURL, p)));
+		return o;
+	}
+
+	private static String asZenodoCommunity(final Concept c) {
+		return firstValue(CZENODOCOMMUNITY_ID, c.getParams());
+	}
+
+	private static List<SubCommunity> asSubCommunities(final String communityId, final String parent, final String category, final List<Concept> concepts) {
+		final List<SubCommunity> list = new ArrayList<>();
+		for (final Concept c : concepts) {
+			final SubCommunity sc = new SubCommunity();
+			sc.setSubCommunityId(c.getId());
+			sc.setCommunityId(communityId);
+			sc.setParent(parent);
+			sc.setCategory(category);
+			sc.setLabel(c.getLabel());
+			sc.setParams(c.getParams());
+			sc.setClaim(c.isClaim());
+			sc.setBrowsable(false);
+			list.add(sc);
+			list.addAll(asSubCommunities(communityId, c.getId(), category, c.getConcepts()));
+		}
+		return list;
+	}
+
+	private String findNamespaceForECProject(final String code) {
+		final List<String> list =
+				jdbcTemplate.queryForList("SELECT substr(id, 1, 12) from projects where code = ? and id like 'corda%'", String.class, code);
+		return list.isEmpty() ? null : list.get(0);
+	}
+
+	private static String getDecodedUrl(final String encoded_url) {
+		if (encoded_url == null || encoded_url.startsWith("http")) { return encoded_url; }
+		try {
+			return new String(Base64.getDecoder().decode(encoded_url));
+		} catch (final Exception e) {
+			log.warn("Invalid base64: " + encoded_url);
+			return encoded_url;
+		}
+	}
+
+	private static List<String> splitValues(final Stream<String> stream, final String separator) {
+		return stream.map(s -> s.split(separator))
+				.map(Arrays::asList)
+				.flatMap(List::stream)
+				.filter(StringUtils::isNotBlank)
+				.map(StringUtils::trim)
+				.collect(Collectors.toList());
+	}
+
+	private static String firstValue(final String name, final List<Param> params) {
+		return asValues(name, params).findFirst().orElse(null);
+	}
+
+	private static String asCsv(final String name, final List<Param> params) {
+		return asValues(name, params).collect(Collectors.joining(CSV_DELIMITER));
+	}
+
+	private static Stream<String> asValues(final String name, final List<Param> params) {
+		return params == null ? Stream.empty()
+				: params.stream()
+						.filter(p -> p != null)
+						.filter(p -> StringUtils.isNotBlank(p.getName()))
+						.filter(p -> p.getName().trim().equals(name.trim()))
+						.map(Param::getValue)
+						.map(StringUtils::trim)
+						.distinct();
+	}
+
+	protected DbOrganizationRepository getDbOrganizationRepository() {
+		return dbOrganizationRepository;
+	}
+
+	protected void setDbOrganizationRepository(final DbOrganizationRepository dbOrganizationRepository) {
+		this.dbOrganizationRepository = dbOrganizationRepository;
+	}
+
+	protected CommunityService getService() {
+		return service;
+	}
+
+	protected void setService(final CommunityService service) {
+		this.service = service;
+	}
+
+	protected JdbcTemplate getJdbcTemplate() {
+		return jdbcTemplate;
+	}
+
+	protected void setJdbcTemplate(final JdbcTemplate jdbcTemplate) {
+		this.jdbcTemplate = jdbcTemplate;
+	}
+}

apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/model/DbCommunity.java

@@ -0,0 +1,304 @@
+package eu.dnetlib.openaire.community.model;
+
+import java.io.Serializable;
+import java.time.LocalDateTime;
+
+import javax.persistence.Column;
+import javax.persistence.Convert;
+import javax.persistence.Entity;
+import javax.persistence.EnumType;
+import javax.persistence.Enumerated;
+import javax.persistence.GeneratedValue;
+import javax.persistence.GenerationType;
+import javax.persistence.Id;
+import javax.persistence.Table;
+
+import org.hibernate.annotations.Type;
+import org.hibernate.annotations.TypeDef;
+import org.hibernate.annotations.TypeDefs;
+import org.springframework.data.annotation.CreatedDate;
+import org.springframework.data.annotation.LastModifiedDate;
+
+import com.vladmihalcea.hibernate.type.array.StringArrayType;
+import com.vladmihalcea.hibernate.type.json.JsonBinaryType;
+import com.vladmihalcea.hibernate.type.json.JsonStringType;
+
+import eu.dnetlib.openaire.community.utils.CommunityClaimTypeConverter;
+import eu.dnetlib.openaire.community.utils.CommunityMembershipTypeConverter;
+import eu.dnetlib.openaire.exporter.model.community.CommunityClaimType;
+import eu.dnetlib.openaire.exporter.model.community.CommunityMembershipType;
+import eu.dnetlib.openaire.exporter.model.community.CommunityPlanType;
+import eu.dnetlib.openaire.exporter.model.community.CommunityStatus;
+import eu.dnetlib.openaire.exporter.model.community.CommunityType;
+import eu.dnetlib.openaire.exporter.model.community.selectioncriteria.SelectionCriteria;
+
+@Entity
+@Table(name = "communities")
+@TypeDefs({
+		@TypeDef(name = "string-array", typeClass = StringArrayType.class),
+		@TypeDef(name = "json", typeClass = JsonStringType.class),
+		@TypeDef(name = "jsonb", typeClass = JsonBinaryType.class)
+})
+public class DbCommunity implements Serializable {
+
+	private static final long serialVersionUID = 4315597783109726539L;
+
+	@Id
+	@Column(name = "id")
+	private String id;
+
+	@Column(name = "name")
+	private String name;
+
+	@Column(name = "shortname")
+	private String shortName;
+
+	@Column(name = "displayname")
+	private String displayName;
+
+	@Column(name = "displayshortname")
+	private String displayShortName;
+
+	@Column(name = "description")
+	private String description;
+
+	@Column(name = "status")
+	@Enumerated(EnumType.STRING)
+	private CommunityStatus status = CommunityStatus.hidden;
+
+	@Column(name = "membership")
+	@Convert(converter = CommunityMembershipTypeConverter.class)
+	private CommunityMembershipType membership = CommunityMembershipType.byInvitation;
+
+	@Column(name = "type")
+	@Enumerated(EnumType.STRING)
+	private CommunityType type;
+
+	@Column(name = "claim")
+	@Convert(converter = CommunityClaimTypeConverter.class)
+	private CommunityClaimType claim;
+
+	@Type(type = "string-array")
+	@Column(name = "subjects", columnDefinition = "text[]")
+	private String[] subjects;
+
+	@Type(type = "string-array")
+	@Column(name = "fos", columnDefinition = "text[]")
+	private String[] fos;
+
+	@Type(type = "string-array")
+	@Column(name = "sdg", columnDefinition = "text[]")
+	private String[] sdg;
+
+	@Type(type = "jsonb")
+	@Column(name = "adv_constraints")
+	private SelectionCriteria advancedConstraints;
+
+	@Type(type = "jsonb")
+	@Column(name = "remove_constraints")
+	private SelectionCriteria removeConstraints;
+
+	@Column(name = "main_zenodo_community")
+	private String mainZenodoCommunity;
+
+	@Type(type = "string-array")
+	@Column(name = "other_zenodo_communities", columnDefinition = "text[]")
+	private String[] otherZenodoCommunities;
+
+	@CreatedDate
+	@Column(name = "creation_date")
+	private LocalDateTime creationDate;
+
+	@LastModifiedDate
+	@Column(name = "last_update")
+	@GeneratedValue(strategy = GenerationType.IDENTITY)
+	private LocalDateTime lastUpdateDate;
+
+	@Column(name = "logo_url")
+	private String logoUrl;
+
+	@Type(type = "string-array")
+	@Column(name = "suggested_acknowledgements", columnDefinition = "text[]")
+	private String[] suggestedAcknowledgements;
+
+	@Column(name = "plan")
+	@Enumerated(EnumType.STRING)
+	private CommunityPlanType plan;
+
+	public String getId() {
+		return id;
+	}
+
+	public void setId(final String id) {
+		this.id = id;
+	}
+
+	public String getName() {
+		return name;
+	}
+
+	public void setName(final String name) {
+		this.name = name;
+	}
+
+	public String getShortName() {
+		return shortName;
+	}
+
+	public void setShortName(final String shortName) {
+		this.shortName = shortName;
+	}
+
+	public String getDisplayName() {
+		return displayName;
+	}
+
+	public void setDisplayName(final String displayName) {
+		this.displayName = displayName;
+	}
+
+	public String getDisplayShortName() {
+		return displayShortName;
+	}
+
+	public void setDisplayShortName(final String displayShortName) {
+		this.displayShortName = displayShortName;
+	}
+
+	public String getDescription() {
+		return description;
+	}
+
+	public void setDescription(final String description) {
+		this.description = description;
+	}
+
+	public CommunityStatus getStatus() {
+		return status;
+	}
+
+	public void setStatus(final CommunityStatus status) {
+		this.status = status;
+	}
+
+	public CommunityMembershipType getMembership() {
+		return membership;
+	}
+
+	public void setMembership(final CommunityMembershipType membership) {
+		this.membership = membership;
+	}
+
+	public CommunityType getType() {
+		return type;
+	}
+
+	public void setType(final CommunityType type) {
+		this.type = type;
+	}
+
+	public CommunityClaimType getClaim() {
+		return claim;
+	}
+
+	public void setClaim(final CommunityClaimType claim) {
+		this.claim = claim;
+	}
+
+	public String[] getSubjects() {
+		return subjects;
+	}
+
+	public void setSubjects(final String[] subjects) {
+		this.subjects = subjects;
+	}
+
+	public String[] getFos() {
+		return fos;
+	}
+
+	public void setFos(final String[] fos) {
+		this.fos = fos;
+	}
+
+	public String[] getSdg() {
+		return sdg;
+	}
+
+	public void setSdg(final String[] sdg) {
+		this.sdg = sdg;
+	}
+
+	public SelectionCriteria getAdvancedConstraints() {
+		return advancedConstraints;
+	}
+
+	public void setAdvancedConstraints(final SelectionCriteria advancedConstraints) {
+		this.advancedConstraints = advancedConstraints;
+	}
+
+	public SelectionCriteria getRemoveConstraints() {
+		return removeConstraints;
+	}
+
+	public void setRemoveConstraints(final SelectionCriteria removeConstraints) {
+		this.removeConstraints = removeConstraints;
+	}
+
+	public String getMainZenodoCommunity() {
+		return mainZenodoCommunity;
+	}
+
+	public void setMainZenodoCommunity(final String mainZenodoCommunity) {
+		this.mainZenodoCommunity = mainZenodoCommunity;
+	}
+
+	public String[] getOtherZenodoCommunities() {
+		return otherZenodoCommunities;
+	}
+
+	public void setOtherZenodoCommunities(final String[] otherZenodoCommunities) {
+		this.otherZenodoCommunities = otherZenodoCommunities;
+	}
+
+	public LocalDateTime getCreationDate() {
+		return creationDate;
+	}
+
+	public void setCreationDate(final LocalDateTime creationDate) {
+		this.creationDate = creationDate;
+	}
+
+	public LocalDateTime getLastUpdateDate() {
+		return lastUpdateDate;
+	}
+
+	public void setLastUpdateDate(final LocalDateTime lastUpdateDate) {
+		this.lastUpdateDate = lastUpdateDate;
+	}
+
+	public String getLogoUrl() {
+		return logoUrl;
+	}
+
+	public void setLogoUrl(final String logoUrl) {
+		this.logoUrl = logoUrl;
+	}
+
+	public String[] getSuggestedAcknowledgements() {
+		return suggestedAcknowledgements;
+	}
+
+	public void setSuggestedAcknowledgements(final String[] suggestedAcknowledgements) {
+		this.suggestedAcknowledgements = suggestedAcknowledgements;
+	}
+
+	public CommunityPlanType getPlan() {
+		return plan;
+	}
+
+	public void setPlan(final CommunityPlanType plan) {
+		this.plan = plan;
+	}
+
+}

apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/model/DbDatasource.java

@@ -0,0 +1,123 @@
+package eu.dnetlib.openaire.community.model;
+
+import java.io.Serializable;
+
+import javax.persistence.Column;
+import javax.persistence.Entity;
+import javax.persistence.Id;
+import javax.persistence.IdClass;
+import javax.persistence.Table;
+
+import org.hibernate.annotations.Type;
+
+import eu.dnetlib.openaire.exporter.model.community.selectioncriteria.SelectionCriteria;
+
+@Entity
+@Table(name = "community_datasources")
+@IdClass(DbDatasourcePK.class)
+public class DbDatasource implements Serializable {
+
+	private static final long serialVersionUID = -8782576185861694228L;
+
+	@Id
+	@Column(name = "community")
+	private String community;
+
+	@Id
+	@Column(name = "ds_id")
+	private String dsId;
+
+	@Column(name = "ds_name")
+	private String dsName;
+
+	@Column(name = "ds_officialname")
+	private String dsOfficialName;
+
+	@Column(name = "enabled")
+	private Boolean enabled;
+
+	@Type(type = "jsonb")
+	@Column(name = "constraints")
+	private SelectionCriteria constraints;
+
+	@Column(name = "deposit")
+	private Boolean deposit;
+
+	@Column(name = "message")
+	private String message;
+
+	public DbDatasource() {}
+
+	public DbDatasource(final String community, final String dsId, final String dsName, final String dsOfficialName, final SelectionCriteria constraints) {
+		this.community = community;
+		this.dsId = dsId;
+		this.dsName = dsName;
+		this.dsOfficialName = dsOfficialName;
+		this.constraints = constraints;
+	}
+
+	public String getCommunity() {
+		return community;
+	}
+
+	public void setCommunity(final String community) {
+		this.community = community;
+	}
+
+	public String getDsId() {
+		return dsId;
+	}
+
+	public void setDsId(final String dsId) {
+		this.dsId = dsId;
+	}
+
+	public String getDsName() {
+		return dsName;
+	}
+
+	public void setDsName(final String dsName) {
+		this.dsName = dsName;
+	}
+
+	public String getDsOfficialName() {
+		return dsOfficialName;
+	}
+
+	public void setDsOfficialName(final String dsOfficialName) {
+		this.dsOfficialName = dsOfficialName;
+	}
+
+	public Boolean getEnabled() {
+		return enabled;
+	}
+
+	public void setEnabled(final Boolean enabled) {
+		this.enabled = enabled;
+	}
+
+	public SelectionCriteria getConstraints() {
+		return constraints;
+	}
+
+	public void setConstraints(final SelectionCriteria constraints) {
+		this.constraints = constraints;
+	}
+
+	public Boolean getDeposit() {
+		return deposit;
+	}
+
+	public void setDeposit(final Boolean deposit) {
+		this.deposit = deposit;
+	}
+
+	public String getMessage() {
+		return message;
+	}
+
+	public void setMessage(final String message) {
+		this.message = message;
+	}
+
+}

apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/model/DbDatasourcePK.java

@@ -0,0 +1,55 @@
+package eu.dnetlib.openaire.community.model;
+
+import java.io.Serializable;
+import java.util.Objects;
+
+public class DbDatasourcePK implements Serializable {
+
+	private static final long serialVersionUID = -8073510491611213955L;
+
+	private String community;
+
+	private String dsId;
+
+	public DbDatasourcePK() {}
+
+	public DbDatasourcePK(final String community, final String dsId) {
+		this.community = community;
+		this.dsId = dsId;
+	}
+
+	public String getCommunity() {
+		return community;
+	}
+
+	public void setCommunity(final String community) {
+		this.community = community;
+	}
+
+	public String getDsId() {
+		return dsId;
+	}
+
+	public void setDsId(final String dsId) {
+		this.dsId = dsId;
+	}
+
+	@Override
+	public int hashCode() {
+		return Objects.hash(community, dsId);
+	}
+
+	@Override
+	public boolean equals(final Object obj) {
+		if (this == obj) { return true; }
+		if (!(obj instanceof DbDatasourcePK)) { return false; }
+		final DbDatasourcePK other = (DbDatasourcePK) obj;
+		return Objects.equals(community, other.community) && Objects.equals(dsId, other.dsId);
+	}
+
+	@Override
+	public String toString() {
+		return String.format("CommunityDatasourcePK [community=%s, dsId=%s]", community, dsId);
+	}
+
+}

apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/model/DbOrganization.java

@@ -0,0 +1,54 @@
+package eu.dnetlib.openaire.community.model;
+
+import java.io.Serializable;
+
+import javax.persistence.Column;
+import javax.persistence.Entity;
+import javax.persistence.Id;
+import javax.persistence.IdClass;
+import javax.persistence.Table;
+
+@Entity
+@Table(name = "community_orgs")
+@IdClass(DbOrganizationPK.class)
+public class DbOrganization implements Serializable {
+
+	private static final long serialVersionUID = -602114117980437763L;
+
+	@Id
+	@Column(name = "community")
+	private String community;
+
+	@Id
+	@Column(name = "org_id")
+	private String orgId;
+
+	public DbOrganization() {}
+
+	public DbOrganization(final String community, final String orgId) {
+		this.community = community;
+		this.orgId = orgId;
+	}
+
+	public String getCommunity() {
+		return community;
+	}
+
+	public void setCommunity(final String community) {
+		this.community = community;
+	}
+
+	public String getOrgId() {
+		return orgId;
+	}
+
+	public void setOrgId(final String orgId) {
+		this.orgId = orgId;
+	}
+
+	@Override
+	public String toString() {
+		return String.format("DbOrganization [community=%s, orgId=%s]", community, orgId);
+	}
+
+}

apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/model/DbOrganizationPK.java

@@ -0,0 +1,54 @@
+package eu.dnetlib.openaire.community.model;
+
+import java.io.Serializable;
+import java.util.Objects;
+
+public class DbOrganizationPK implements Serializable {
+
+	private static final long serialVersionUID = -6720182815397534837L;
+
+	private String community;
+
+	private String orgId;
+
+	public DbOrganizationPK() {}
+
+	public DbOrganizationPK(final String community, final String orgId) {
+		this.community = community;
+		this.orgId = orgId;
+	}
+
+	public String getCommunity() {
+		return community;
+	}
+
+	public void setCommunity(final String community) {
+		this.community = community;
+	}
+
+	public String getOrgId() {
+		return orgId;
+	}
+
+	public void setOrgId(final String orgId) {
+		this.orgId = orgId;
+	}
+
+	@Override
+	public int hashCode() {
+		return Objects.hash(community, orgId);
+	}
+
+	@Override
+	public boolean equals(final Object obj) {
+		if (this == obj) { return true; }
+		if (!(obj instanceof DbDatasourcePK)) { return false; }
+		final DbOrganizationPK other = (DbOrganizationPK) obj;
+		return Objects.equals(community, other.community) && Objects.equals(orgId, other.orgId);
+	}
+
+	@Override
+	public String toString() {
+		return String.format("CommunityOrgPK [community=%s, orgId=%s]", community, orgId);
+	}
+}

apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/model/DbProject.java

@@ -0,0 +1,113 @@
+package eu.dnetlib.openaire.community.model;
+
+import java.io.Serializable;
+import java.time.LocalDate;
+
+import javax.persistence.Column;
+import javax.persistence.Entity;
+import javax.persistence.Id;
+import javax.persistence.IdClass;
+import javax.persistence.Table;
+
+import org.springframework.data.annotation.CreatedDate;
+
+@Entity
+@Table(name = "community_projects")
+@IdClass(DbProjectPK.class)
+public class DbProject implements Serializable {
+
+	private static final long serialVersionUID = 1649065971750517925L;
+
+	@Id
+	@Column(name = "community")
+	private String community;
+
+	@Id
+	@Column(name = "project_id")
+	private String projectId;
+
+	@Column(name = "project_code")
+	private String projectCode;
+
+	@Column(name = "project_name")
+	private String projectName;
+
+	@Column(name = "project_acronym")
+	private String projectAcronym;
+
+	@Column(name = "project_funder")
+	private String projectFunder;
+
+	@CreatedDate
+	@Column(name = "available_since")
+	private LocalDate availableSince;
+
+	public DbProject() {}
+
+	public DbProject(final String community, final String projectId, final String projectCode, final String projectName, final String projectAcronym,
+		final String projectFunder, final LocalDate availableSince) {
+		this.community = community;
+		this.projectId = projectId;
+		this.projectCode = projectCode;
+		this.projectName = projectName;
+		this.projectAcronym = projectAcronym;
+		this.projectFunder = projectFunder;
+		this.availableSince = availableSince;
+	}
+
+	public String getCommunity() {
+		return community;
+	}
+
+	public void setCommunity(final String community) {
+		this.community = community;
+	}
+
+	public String getProjectId() {
+		return projectId;
+	}
+
+	public void setProjectId(final String projectId) {
+		this.projectId = projectId;
+	}
+
+	public String getProjectCode() {
+		return projectCode;
+	}
+
+	public void setProjectCode(final String projectCode) {
+		this.projectCode = projectCode;
+	}
+
+	public String getProjectName() {
+		return projectName;
+	}
+
+	public void setProjectName(final String projectName) {
+		this.projectName = projectName;
+	}
+
+	public String getProjectAcronym() {
+		return projectAcronym;
+	}
+
+	public void setProjectAcronym(final String projectAcronym) {
+		this.projectAcronym = projectAcronym;
+	}
+
+	public String getProjectFunder() {
+		return projectFunder;
+	}
+
+	public void setProjectFunder(final String projectFunder) {
+		this.projectFunder = projectFunder;
+	}
+
+	public LocalDate getAvailableSince() {
+		return availableSince;
+	}
+
+	public void setAvailableSince(final LocalDate availableSince) {
+		this.availableSince = availableSince;
+	}
+}

apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/model/DbProjectPK.java

@@ -0,0 +1,54 @@
+package eu.dnetlib.openaire.community.model;
+
+import java.io.Serializable;
+import java.util.Objects;
+
+public class DbProjectPK implements Serializable {
+
+	private static final long serialVersionUID = -4236577148534835803L;
+
+	private String community;
+
+	private String projectId;
+
+	public DbProjectPK() {}
+
+	public DbProjectPK(final String community, final String projectId) {
+		this.community = community;
+		this.projectId = projectId;
+	}
+
+	public String getCommunity() {
+		return community;
+	}
+
+	public void setCommunity(final String community) {
+		this.community = community;
+	}
+
+	public String getProjectId() {
+		return projectId;
+	}
+
+	public void setProjectId(final String projectId) {
+		this.projectId = projectId;
+	}
+
+	@Override
+	public int hashCode() {
+		return Objects.hash(community, projectId);
+	}
+
+	@Override
+	public boolean equals(final Object obj) {
+		if (this == obj) { return true; }
+		if (!(obj instanceof DbProjectPK)) { return false; }
+		final DbProjectPK other = (DbProjectPK) obj;
+		return Objects.equals(community, other.community) && Objects.equals(projectId, other.projectId);
+	}
+
+	@Override
+	public String toString() {
+		return String.format("CommunityProjectPK [community=%s, projectId=%s]", community, projectId);
+	}
+}

apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/model/DbSubCommunity.java

@@ -0,0 +1,122 @@
+package eu.dnetlib.openaire.community.model;
+
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.List;
+
+import javax.persistence.Column;
+import javax.persistence.Entity;
+import javax.persistence.Id;
+import javax.persistence.Table;
+
+import org.hibernate.annotations.Type;
+import org.hibernate.annotations.TypeDef;
+import org.hibernate.annotations.TypeDefs;
+
+import com.vladmihalcea.hibernate.type.array.StringArrayType;
+import com.vladmihalcea.hibernate.type.json.JsonBinaryType;
+import com.vladmihalcea.hibernate.type.json.JsonStringType;
+
+import eu.dnetlib.openaire.exporter.model.context.Param;
+
+@Entity
+@Table(name = "community_subs")
+@TypeDefs({
+	@TypeDef(name = "string-array", typeClass = StringArrayType.class),
+	@TypeDef(name = "json", typeClass = JsonStringType.class),
+	@TypeDef(name = "jsonb", typeClass = JsonBinaryType.class)
+})
+public class DbSubCommunity implements Serializable {
+
+	private static final long serialVersionUID = 7104936574383307358L;
+
+	@Id
+	@Column(name = "sub_id")
+	private String id;
+
+	@Column(name = "community")
+	private String community;
+
+	@Column(name = "label")
+	private String label;
+
+	@Column(name = "category")
+	private String category;
+
+	@Type(type = "jsonb")
+	@Column(name = "params")
+	private List<Param> params = new ArrayList<>();
+
+	@Column(name = "parent")
+	private String parent;
+
+	@Column(name = "claim")
+	private boolean claim = false;
+
+	@Column(name = "browsable")
+	private boolean browsable = false;
+
+	public String getId() {
+		return id;
+	}
+
+	public void setId(final String id) {
+		this.id = id;
+	}
+
+	public String getCommunity() {
+		return community;
+	}
+
+	public void setCommunity(final String community) {
+		this.community = community;
+	}
+
+	public String getLabel() {
+		return label;
+	}
+
+	public void setLabel(final String label) {
+		this.label = label;
+	}
+
+	public String getCategory() {
+		return category;
+	}
+
+	public void setCategory(final String category) {
+		this.category = category;
+	}
+
+	public List<Param> getParams() {
+		return params;
+	}
+
+	public void setParams(final List<Param> params) {
+		this.params = params;
+	}
+
+	public String getParent() {
+		return parent;
+	}
+
+	public void setParent(final String parent) {
+		this.parent = parent;
+	}
+
+	public boolean isClaim() {
+		return claim;
+	}
+
+	public void setClaim(final boolean claim) {
+		this.claim = claim;
+	}
+
+	public boolean isBrowsable() {
+		return browsable;
+	}
+
+	public void setBrowsable(final boolean browsable) {
+		this.browsable = browsable;
+	}
+}

apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/model/DbSupportOrg.java

@@ -0,0 +1,64 @@
+package eu.dnetlib.openaire.community.model;
+
+import java.io.Serializable;
+
+import javax.persistence.Column;
+import javax.persistence.Entity;
+import javax.persistence.Id;
+import javax.persistence.IdClass;
+import javax.persistence.Table;
+
+@Entity
+@Table(name = "community_support_orgs")
+@IdClass(DbSupportOrgPK.class)
+public class DbSupportOrg implements Serializable {
+
+	private static final long serialVersionUID = 1308759097276753411L;
+
+	@Id
+	@Column(name = "community")
+	private String community;
+
+	@Id
+	@Column(name = "org_name")
+	private String orgName;
+
+	@Column(name = "org_url")
+	private String orgUrl;
+
+	@Column(name = "org_logourl")
+	private String orgLogoUrl;
+
+	public String getCommunity() {
+		return community;
+	}
+
+	public void setCommunity(final String community) {
+		this.community = community;
+	}
+
+	public String getOrgName() {
+		return orgName;
+	}
+
+	public void setOrgName(final String orgName) {
+		this.orgName = orgName;
+	}
+
+	public String getOrgUrl() {
+		return orgUrl;
+	}
+
+	public void setOrgUrl(final String orgUrl) {
+		this.orgUrl = orgUrl;
+	}
+
+	public String getOrgLogoUrl() {
+		return orgLogoUrl;
+	}
+
+	public void setOrgLogoUrl(final String orgLogoUrl) {
+		this.orgLogoUrl = orgLogoUrl;
+	}
+
+}

apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/model/DbSupportOrgPK.java

@@ -0,0 +1,55 @@
+package eu.dnetlib.openaire.community.model;
+
+import java.io.Serializable;
+import java.util.Objects;
+
+public class DbSupportOrgPK implements Serializable {
+
+	private static final long serialVersionUID = -4117154543803798310L;
+
+	private String community;
+
+	private String orgName;
+
+	public DbSupportOrgPK() {}
+
+	public DbSupportOrgPK(final String community, final String orgName) {
+		this.community = community;
+		this.orgName = orgName;
+	}
+
+	public String getCommunity() {
+		return community;
+	}
+
+	public void setCommunity(final String community) {
+		this.community = community;
+	}
+
+	public String getOrgName() {
+		return orgName;
+	}
+
+	public void setOrgName(final String orgName) {
+		this.orgName = orgName;
+	}
+
+	@Override
+	public int hashCode() {
+		return Objects.hash(community, orgName);
+	}
+
+	@Override
+	public boolean equals(final Object obj) {
+		if (this == obj) { return true; }
+		if (!(obj instanceof DbSupportOrgPK)) { return false; }
+		final DbSupportOrgPK other = (DbSupportOrgPK) obj;
+		return Objects.equals(community, other.community) && Objects.equals(orgName, other.orgName);
+	}
+
+	@Override
+	public String toString() {
+		return String.format("CommunitySupportOrgPK [community=%s, orgName=%s]", community, orgName);
+	}
+
+}

apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/model/DepositionInfo.java

@@ -0,0 +1,39 @@
+package eu.dnetlib.openaire.community.model;
+
+import java.io.Serializable;
+
+public class DepositionInfo implements Serializable {
+
+	private static final long serialVersionUID = 7538663287451167904L;
+
+	private String openaireId;
+
+	private Boolean deposit;
+
+	private String message;
+
+	public String getOpenaireId() {
+		return openaireId;
+	}
+
+	public void setOpenaireId(final String openaireId) {
+		this.openaireId = openaireId;
+	}
+
+	public Boolean getDeposit() {
+		return deposit;
+	}
+
+	public void setDeposit(final Boolean deposit) {
+		this.deposit = deposit;
+	}
+
+	public String getMessage() {
+		return message;
+	}
+
+	public void setMessage(final String message) {
+		this.message = message;
+	}
+
+}

apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/repository/DbCommunityRepository.java

@@ -0,0 +1,17 @@
+package eu.dnetlib.openaire.community.repository;
+
+import java.util.List;
+
+import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
+import org.springframework.data.jpa.repository.JpaRepository;
+import org.springframework.data.jpa.repository.Query;
+
+import eu.dnetlib.openaire.community.model.DbCommunity;
+
+@ConditionalOnProperty(value = "openaire.exporter.enable.community", havingValue = "true")
+public interface DbCommunityRepository extends JpaRepository<DbCommunity, String> {
+
+	@Query(value = "select id from communities where ?1 = ANY(array_append(other_zenodo_communities, main_zenodo_community))", nativeQuery = true)
+	List<String> findByZenodoId(String zenodoId);
+
+}

apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/repository/DbDatasourceRepository.java

@@ -0,0 +1,20 @@
+package eu.dnetlib.openaire.community.repository;
+
+import java.util.List;
+
+import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
+import org.springframework.data.jpa.repository.JpaRepository;
+
+import eu.dnetlib.openaire.community.model.DbDatasource;
+import eu.dnetlib.openaire.community.model.DbDatasourcePK;
+
+@ConditionalOnProperty(value = "openaire.exporter.enable.community", havingValue = "true")
+public interface DbDatasourceRepository extends JpaRepository<DbDatasource, DbDatasourcePK> {
+
+	List<DbDatasource> findByCommunity(String community);
+
+	List<DbDatasource> findByCommunityAndDeposit(String community, boolean deposit);
+
+	void deleteByCommunity(String id);
+
+}

apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/repository/DbOrganizationRepository.java

@@ -0,0 +1,18 @@
+package eu.dnetlib.openaire.community.repository;
+
+import java.util.List;
+
+import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
+import org.springframework.data.jpa.repository.JpaRepository;
+
+import eu.dnetlib.openaire.community.model.DbOrganization;
+import eu.dnetlib.openaire.community.model.DbOrganizationPK;
+
+@ConditionalOnProperty(value = "openaire.exporter.enable.community", havingValue = "true")
+public interface DbOrganizationRepository extends JpaRepository<DbOrganization, DbOrganizationPK> {
+
+	List<DbOrganization> findByCommunity(String community);
+
+	void deleteByCommunity(String id);
+
+}

apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/repository/DbProjectRepository.java

@@ -0,0 +1,25 @@
+package eu.dnetlib.openaire.community.repository;
+
+import java.util.List;
+
+import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
+import org.springframework.data.domain.Page;
+import org.springframework.data.domain.Pageable;
+import org.springframework.data.jpa.repository.JpaRepository;
+import org.springframework.data.jpa.repository.JpaSpecificationExecutor;
+import org.springframework.data.jpa.repository.Query;
+
+import eu.dnetlib.openaire.community.model.DbProject;
+import eu.dnetlib.openaire.community.model.DbProjectPK;
+
+@ConditionalOnProperty(value = "openaire.exporter.enable.community", havingValue = "true")
+public interface DbProjectRepository extends JpaRepository<DbProject, DbProjectPK>, JpaSpecificationExecutor<DbProject> {
+
+	Page<DbProject> findByCommunity(String community, Pageable page);
+
+	void deleteByCommunity(String id);
+
+	@Query(value = "select distinct project_funder from community_projects where community = ?1 order by project_funder", nativeQuery = true)
+	List<String> findFundersByCommunity(String id);
+
+}

apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/repository/DbSubCommunityRepository.java

@@ -0,0 +1,19 @@
+package eu.dnetlib.openaire.community.repository;
+
+import java.util.List;
+
+import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
+import org.springframework.data.jpa.repository.JpaRepository;
+
+import eu.dnetlib.openaire.community.model.DbSubCommunity;
+
+@ConditionalOnProperty(value = "openaire.exporter.enable.community", havingValue = "true")
+public interface DbSubCommunityRepository extends JpaRepository<DbSubCommunity, String> {
+
+	List<DbSubCommunity> findByCommunity(String community);
+
+	List<DbSubCommunity> findByCommunityAndParent(String community, String parent);
+
+	void deleteByCommunity(String id);
+
+}

apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/repository/DbSupportOrgRepository.java

@@ -0,0 +1,18 @@
+package eu.dnetlib.openaire.community.repository;
+
+import java.util.List;
+
+import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
+import org.springframework.data.jpa.repository.JpaRepository;
+
+import eu.dnetlib.openaire.community.model.DbSupportOrg;
+import eu.dnetlib.openaire.community.model.DbSupportOrgPK;
+
+@ConditionalOnProperty(value = "openaire.exporter.enable.community", havingValue = "true")
+public interface DbSupportOrgRepository extends JpaRepository<DbSupportOrg, DbSupportOrgPK> {
+
+	List<DbSupportOrg> findByCommunity(String community);
+
+	void deleteByCommunity(String id);
+
+}

apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/utils/CommunityClaimTypeConverter.java

@@ -0,0 +1,29 @@
+package eu.dnetlib.openaire.community.utils;
+
+import javax.persistence.AttributeConverter;
+
+import org.apache.commons.lang3.StringUtils;
+
+import eu.dnetlib.openaire.exporter.model.community.CommunityClaimType;
+
+public class CommunityClaimTypeConverter implements AttributeConverter<CommunityClaimType, String> {
+
+	@Override
+	public String convertToDatabaseColumn(final CommunityClaimType attribute) {
+		if (attribute == null) {
+			return null;
+		} else {
+			return attribute.getDescription();
+		}
+	}
+
+	@Override
+	public CommunityClaimType convertToEntityAttribute(final String dbData) {
+		if (StringUtils.isBlank(dbData)) {
+			return null;
+		} else {
+			return CommunityClaimType.fromDescription(dbData);
+		}
+	}
+
+}

apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/utils/CommunityMappingUtils.java

@@ -0,0 +1,315 @@
+package eu.dnetlib.openaire.community.utils;
+
+import java.text.ParseException;
+import java.time.LocalDate;
+import java.time.LocalDateTime;
+import java.time.ZoneId;
+import java.util.Arrays;
+import java.util.Date;
+import java.util.List;
+
+import org.apache.commons.lang3.ObjectUtils;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.lang3.time.DateUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+import com.google.common.collect.Lists;
+
+import eu.dnetlib.openaire.community.importer.CommunityImporterService;
+import eu.dnetlib.openaire.community.model.DbCommunity;
+import eu.dnetlib.openaire.community.model.DbDatasource;
+import eu.dnetlib.openaire.community.model.DbProject;
+import eu.dnetlib.openaire.community.model.DbSubCommunity;
+import eu.dnetlib.openaire.community.model.DbSupportOrg;
+import eu.dnetlib.openaire.exporter.model.community.CommunityContentprovider;
+import eu.dnetlib.openaire.exporter.model.community.CommunityDetails;
+import eu.dnetlib.openaire.exporter.model.community.CommunityOrganization;
+import eu.dnetlib.openaire.exporter.model.community.CommunityProject;
+import eu.dnetlib.openaire.exporter.model.community.CommunitySummary;
+import eu.dnetlib.openaire.exporter.model.community.CommunityWritableProperties;
+import eu.dnetlib.openaire.exporter.model.community.SubCommunity;
+import eu.dnetlib.openaire.exporter.model.context.IISConfigurationEntry;
+
+public class CommunityMappingUtils {
+
+	public final static String PIPE_SEPARATOR = "||";
+
+	private static final List<String> DATE_PATTERN = Lists.newArrayList("yyyy-MM-dd'T'hh:mm:ss", "yyyy-MM-dd'T'hh:mm:ssXXX", "yyyy-MM-dd'T'hh:mm:ss+00:00");
+
+	private static final Log log = LogFactory.getLog(CommunityMappingUtils.class);
+
+	public static CommunitySummary toCommunitySummary(final DbCommunity c) {
+		final CommunitySummary summary = new CommunitySummary();
+		populateSummary(summary, c);
+		return summary;
+	}
+
+	public static DbCommunity toCommunity(final CommunityDetails details) {
+		final DbCommunity c = new DbCommunity();
+		c.setId(details.getId());
+		c.setName(details.getName());
+		c.setShortName(details.getShortName());
+		c.setDisplayName(details.getDisplayName());
+		c.setDisplayShortName(details.getDisplayShortName());
+		c.setDescription(details.getDescription());
+		c.setStatus(details.getStatus());
+		c.setLogoUrl(details.getLogoUrl());
+		c.setMembership(details.getMembership());
+		c.setType(details.getType());
+		c.setClaim(details.getClaim());
+		c.setSubjects(toStringArray(details.getSubjects()));
+		c.setFos(toStringArray(details.getFos()));
+		c.setSdg(toStringArray(details.getSdg()));
+		c.setAdvancedConstraints(details.getAdvancedConstraints());
+		c.setRemoveConstraints(details.getRemoveConstraints());
+		c.setMainZenodoCommunity(details.getZenodoCommunity());
+		c.setOtherZenodoCommunities(toStringArray(details.getOtherZenodoCommunities()));
+		c.setSuggestedAcknowledgements(toStringArray(details.getSuggestedAcknowledgements()));
+		c.setPlan(details.getPlan());
+		c.setCreationDate(ObjectUtils.firstNonNull(details.getCreationDate(), LocalDateTime.now()));
+		c.setLastUpdateDate(LocalDateTime.now());
+		return c;
+	}
+
+	public static void populateCommunity(final DbCommunity c, final CommunityWritableProperties details) {
+		if (StringUtils.isNotBlank(details.getName())) {
+			c.setName(details.getName());
+		}
+		if (StringUtils.isNotBlank(details.getShortName())) {
+			c.setShortName(details.getShortName());
+		}
+		if (StringUtils.isNotBlank(details.getDisplayName())) {
+			c.setDisplayName(details.getDisplayName());
+		}
+		if (StringUtils.isNotBlank(details.getDisplayShortName())) {
+			c.setDisplayShortName(details.getDisplayShortName());
+		}
+		if (StringUtils.isNotBlank(details.getDescription())) {
+			c.setDescription(details.getDescription());
+		}
+		if (details.getStatus() != null) {
+			c.setStatus(details.getStatus());
+		}
+		if (details.getMembership() != null) {
+			c.setMembership(details.getMembership());
+		}
+		if (details.getType() != null) {
+			c.setType(details.getType());
+		}
+		if (details.getClaim() != null) {
+			c.setClaim(details.getClaim());
+		}
+		if (StringUtils.isNotBlank(details.getLogoUrl())) {
+			c.setLogoUrl(details.getLogoUrl());
+		}
+		if (details.getFos() != null) {
+			c.setFos(toStringArray(details.getFos()));
+		}
+		if (details.getSdg() != null) {
+			c.setSdg(toStringArray(details.getSdg()));
+		}
+		if (details.getSubjects() != null) {
+			c.setSubjects(toStringArray(details.getSubjects()));
+		}
+		if (details.getAdvancedConstraints() != null) {
+			c.setAdvancedConstraints(details.getAdvancedConstraints());
+		}
+		if (details.getRemoveConstraints() != null) {
+			c.setRemoveConstraints(details.getRemoveConstraints());
+		}
+		if (StringUtils.isNotBlank(details.getMainZenodoCommunity())) {
+			c.setMainZenodoCommunity(details.getMainZenodoCommunity());
+		}
+		if (details.getOtherZenodoCommunities() != null) {
+			c.setOtherZenodoCommunities(toStringArray(details.getOtherZenodoCommunities()));
+		}
+		if (details.getPlan() != null) {
+			c.setPlan(details.getPlan());
+		}
+
+		c.setLastUpdateDate(LocalDateTime.now());
+	}
+
+	public static CommunityDetails toCommunityDetails(final DbCommunity c) {
+		final CommunityDetails details = new CommunityDetails();
+		populateSummary(details, c);
+		details.setAdvancedConstraints(c.getAdvancedConstraints());
+		details.setRemoveConstraints(c.getRemoveConstraints());
+		details.setFos(Arrays.asList(c.getFos()));
+		details.setSdg(Arrays.asList(c.getSdg()));
+		details.setSubjects(Arrays.asList(c.getSubjects()));
+		details.setOtherZenodoCommunities(Arrays.asList(c.getOtherZenodoCommunities()));
+		details.setSuggestedAcknowledgements(Arrays.asList(c.getSuggestedAcknowledgements()));
+		return details;
+	}
+
+	private static void populateSummary(final CommunitySummary summary, final DbCommunity c) {
+		summary.setId(c.getId());
+		summary.setName(c.getName());
+		summary.setShortName(c.getShortName());
+		summary.setDisplayName(c.getDisplayName());
+		summary.setDisplayShortName(c.getDisplayShortName());
+		summary.setLastUpdateDate(c.getLastUpdateDate());
+		summary.setCreationDate(c.getCreationDate());
+		summary.setQueryId(c.getId() + PIPE_SEPARATOR + c.getShortName());
+		summary.setType(c.getType());
+		summary.setDescription(c.getDescription());
+		summary.setLogoUrl(c.getLogoUrl());
+		summary.setStatus(c.getStatus());
+		summary.setClaim(c.getClaim());
+		summary.setMembership(c.getMembership());
+		summary.setZenodoCommunity(c.getMainZenodoCommunity());
+		summary.setPlan(c.getPlan());
+	}
+
+	public static CommunityProject toCommunityProject(final DbProject dbEntry) {
+		final CommunityProject cp = new CommunityProject();
+		cp.setCommunityId(dbEntry.getCommunity());
+		cp.setOpenaireId(dbEntry.getProjectId());
+		cp.setName(dbEntry.getProjectName());
+		cp.setAcronym(dbEntry.getProjectAcronym());
+		cp.setFunder(dbEntry.getProjectFunder());
+		cp.setGrantId(dbEntry.getProjectCode());
+		cp.setAvailableSince(dbEntry.getAvailableSince());
+		return cp;
+	}
+
+	public static DbProject toDbProject(final String id, final CommunityProject project) {
+		final DbProject p = new DbProject();
+		p.setCommunity(id);
+		p.setProjectId(project.getOpenaireId());
+		p.setProjectName(project.getName());
+		p.setProjectAcronym(project.getAcronym());
+		p.setProjectCode(project.getGrantId());
+		p.setProjectFunder(project.getFunder());
+		if (project.getAvailableSince() != null) {
+			p.setAvailableSince(project.getAvailableSince());
+		} else {
+			p.setAvailableSince(LocalDate.now());
+		}
+		return p;
+	}
+
+	public static CommunityContentprovider toCommunityContentprovider(final DbDatasource dbEntry) {
+		final CommunityContentprovider ccp = new CommunityContentprovider();
+		ccp.setCommunityId(dbEntry.getCommunity());
+		ccp.setOpenaireId(dbEntry.getDsId());
+		ccp.setName(dbEntry.getDsName());
+		ccp.setOfficialname(dbEntry.getDsOfficialName());
+		ccp.setSelectioncriteria(dbEntry.getConstraints());
+		ccp.setEnabled(dbEntry.getEnabled() != null ? dbEntry.getEnabled() : true);
+		ccp.setDeposit(dbEntry.getDeposit() != null ? dbEntry.getDeposit() : false);
+		ccp.setMessage(dbEntry.getMessage());
+		return ccp;
+	}
+
+	public static DbDatasource toDbDatasource(final String id, final CommunityContentprovider provider) {
+		final DbDatasource ds = new DbDatasource();
+		ds.setCommunity(id);
+		ds.setDsId(provider.getOpenaireId());
+		ds.setDsName(provider.getName());
+		ds.setDsOfficialName(provider.getOfficialname());
+		ds.setConstraints(provider.getSelectioncriteria());
+		ds.setEnabled(provider.isEnabled());
+		ds.setDeposit(provider.getDeposit() != null ? provider.getDeposit() : false);
+		ds.setMessage(provider.getMessage());
+		return ds;
+	}
+
+	public static CommunityOrganization toCommunityOrganization(final DbSupportOrg dbEntry) {
+		final CommunityOrganization co = new CommunityOrganization();
+		co.setCommunityId(dbEntry.getCommunity());
+		co.setName(dbEntry.getOrgName());
+		co.setWebsite_url(dbEntry.getOrgUrl());
+		co.setLogo_url(dbEntry.getOrgLogoUrl());
+		return co;
+	}
+
+	public static DbSupportOrg toDbSupportOrg(final String id, final CommunityOrganization org) {
+		final DbSupportOrg dbo = new DbSupportOrg();
+		dbo.setCommunity(id);
+		dbo.setOrgName(org.getName());
+		dbo.setOrgUrl(org.getWebsite_url());
+		dbo.setOrgLogoUrl(org.getLogo_url());
+		return dbo;
+	}
+
+	public static DbSubCommunity toDbSubCommunity(final String id, final SubCommunity sub) {
+		final DbSubCommunity dbsc = new DbSubCommunity();
+		dbsc.setCommunity(id);
+		dbsc.setId(sub.getSubCommunityId());
+		dbsc.setCategory(sub.getCategory());
+		dbsc.setLabel(sub.getLabel());
+		dbsc.setParams(sub.getParams());
+		dbsc.setParent(sub.getParent());
+		dbsc.setClaim(sub.isClaim());
+		dbsc.setBrowsable(sub.isBrowsable());
+		return dbsc;
+	}
+
+	public static SubCommunity toSubCommunity(final DbSubCommunity sub) {
+		final SubCommunity sc = new SubCommunity();
+		sc.setSubCommunityId(sub.getId());
+		sc.setCategory(sub.getCategory());
+		sc.setCommunityId(sub.getCommunity());
+		sc.setLabel(sub.getLabel());
+		sc.setParams(sub.getParams());
+		sc.setParent(sub.getParent());
+		sc.setClaim(sub.isClaim());
+		sc.setBrowsable(sub.isBrowsable());
+		return sc;
+	}
+
+	public static LocalDateTime asLocalDateTime(final String s) {
+		if (StringUtils.isBlank(s)) { return null; }
+
+		for (final String pattern : DATE_PATTERN) {
+			try {
+				final Date res = DateUtils.parseDate(s, pattern);
+				if (res != null) { return asLocalDateTime(res); }
+			} catch (final ParseException e) {}
+		}
+		log.warn("Invalid Date: " + s);
+		return null;
+	}
+
+	public static LocalDateTime asLocalDateTime(final Date date) {
+		return date.toInstant()
+				.atZone(ZoneId.systemDefault())
+				.toLocalDateTime();
+	}
+
+	private static String[] toStringArray(final List<String> list) {
+		return list != null ? list.toArray(new String[list.size()]) : new String[0];
+	}
+
+	public static IISConfigurationEntry asIISConfigurationEntry(final DbCommunity c) {
+		final IISConfigurationEntry conf = new IISConfigurationEntry();
+		conf.setId(c.getId());
+		conf.setLabel(c.getName());
+		conf.addParams(CommunityImporterService.CSUMMARY_DESCRIPTION, c.getDescription());
+		conf.addParams(CommunityImporterService.CSUMMARY_LOGOURL, c.getLogoUrl());
+		conf.addParams(CommunityImporterService.CSUMMARY_STATUS, c.getStatus().toString());
+		conf.addParams(CommunityImporterService.CSUMMARY_NAME, c.getName());
+		conf.addParams(CommunityImporterService.CSUMMARY_ZENODOC, c.getMainZenodoCommunity());
+		conf.addParams(CommunityImporterService.CPROFILE_SUBJECT, c.getSubjects());
+		conf.addParams(CommunityImporterService.CPROFILE_FOS, c.getFos());
+		conf.addParams(CommunityImporterService.CPROFILE_SDG, c.getSdg());
+		conf.addParams(CommunityImporterService.CPROFILE_ADVANCED_CONSTRAINT, c.getAdvancedConstraints() != null ? c.getAdvancedConstraints().toJson() : null);
+		conf.addParams(CommunityImporterService.CPROFILE_REMOVE_CONSTRAINT, c.getRemoveConstraints() != null ? c.getRemoveConstraints().toJson() : null);
+		conf.addParams(CommunityImporterService.CPROFILE_SUGGESTED_ACKNOWLEDGEMENT, c.getSuggestedAcknowledgements());
+		conf.addParams(CommunityImporterService.CPROFILE_CREATIONDATE, c.getCreationDate() != null ? c.getCreationDate().toString() : null);
+		return conf;
+	}
+
+	public static IISConfigurationEntry asIISConfigurationEntry(final DbSubCommunity subc) {
+		final IISConfigurationEntry conf = new IISConfigurationEntry();
+		conf.setId(subc.getId());
+		conf.setLabel(subc.getLabel());
+		if (subc.getParams() != null) {
+			conf.getParams().addAll(subc.getParams());
+		}
+		return conf;
+	}
+}

apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/utils/CommunityMembershipTypeConverter.java

@@ -0,0 +1,29 @@
+package eu.dnetlib.openaire.community.utils;
+
+import javax.persistence.AttributeConverter;
+
+import org.apache.commons.lang3.StringUtils;
+
+import eu.dnetlib.openaire.exporter.model.community.CommunityMembershipType;
+
+public class CommunityMembershipTypeConverter implements AttributeConverter<CommunityMembershipType, String> {
+
+	@Override
+	public String convertToDatabaseColumn(final CommunityMembershipType attribute) {
+		if (attribute == null) {
+			return null;
+		} else {
+			return attribute.getDescription();
+		}
+	}
+
+	@Override
+	public CommunityMembershipType convertToEntityAttribute(final String dbData) {
+		if (StringUtils.isBlank(dbData)) {
+			return null;
+		} else {
+			return CommunityMembershipType.fromDescription(dbData);
+		}
+	}
+
+}

apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/context/ContextApiController.java

@@ -1,8 +1,13 @@
 package eu.dnetlib.openaire.context;
 
 import java.util.List;
-import java.util.Optional;
+import java.util.Objects;
+import java.util.Set;
+import java.util.stream.Collectors;
 
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
 import org.springframework.web.bind.annotation.CrossOrigin;
@@ -12,10 +17,16 @@ import org.springframework.web.bind.annotation.RequestMethod;
 import org.springframework.web.bind.annotation.RequestParam;
 import org.springframework.web.bind.annotation.RestController;
 
-import eu.dnetlib.openaire.exporter.exceptions.ContextException;
+import eu.dnetlib.openaire.common.AbstractExporterController;
+import eu.dnetlib.openaire.community.CommunityService;
+import eu.dnetlib.openaire.exporter.exceptions.CommunityException;
+import eu.dnetlib.openaire.exporter.exceptions.ResourceNotFoundException;
+import eu.dnetlib.openaire.exporter.model.community.CommunityType;
+import eu.dnetlib.openaire.exporter.model.community.SubCommunity;
 import eu.dnetlib.openaire.exporter.model.context.CategorySummary;
 import eu.dnetlib.openaire.exporter.model.context.ConceptSummary;
 import eu.dnetlib.openaire.exporter.model.context.ContextSummary;
+import eu.dnetlib.openaire.exporter.model.context.IISConfigurationEntry;
 import io.swagger.v3.oas.annotations.Operation;
 import io.swagger.v3.oas.annotations.responses.ApiResponse;
 import io.swagger.v3.oas.annotations.responses.ApiResponses;
@@ -25,12 +36,14 @@ import io.swagger.v3.oas.annotations.tags.Tag;
 @CrossOrigin(origins = {
 	"*"
 })
-@ConditionalOnProperty(value = "openaire.exporter.enable.context", havingValue = "true")
+@ConditionalOnProperty(value = "openaire.exporter.enable.community", havingValue = "true")
 @Tag(name = "OpenAIRE Context API", description = "the OpenAIRE Context API")
-public class ContextApiController {
+public class ContextApiController extends AbstractExporterController {
 
 	@Autowired
-	private ContextApiCore contextApiCore;
+	private CommunityService communityService;
+
+	private static final Log log = LogFactory.getLog(ContextApiController.class);
 
 	@RequestMapping(value = "/contexts", produces = {
 		"application/json"
@@ -38,10 +51,28 @@ public class ContextApiController {
 	@Operation(summary = "list brief information about all the context profiles", description = "list brief information about all the context profiles")
 	@ApiResponses(value = {
 		@ApiResponse(responseCode = "200", description = "OK"),
+		@ApiResponse(responseCode = "404", description = "not found"),
 		@ApiResponse(responseCode = "500", description = "unexpected error")
 	})
-	public List<ContextSummary> listContexts(@RequestParam(required = false, defaultValue = "") final List<String> type) throws ContextException {
-		return contextApiCore.listContexts(type);
+	public List<ContextSummary> listContexts(@RequestParam(required = false) final Set<CommunityType> type) throws CommunityException {
+		try {
+			return communityService.listCommunities()
+				.stream()
+				.filter(c -> type == null || type.contains(c.getType()))
+				.map(c -> {
+					final ContextSummary ctx = new ContextSummary();
+					ctx.setId(c.getId());
+					ctx.setLabel(c.getName());
+					ctx.setStatus(c.getStatus().toString());
+					ctx.setType(c.getType().toString());
+					return ctx;
+				})
+				.collect(Collectors.toList());
+		} catch (final ResourceNotFoundException e) {
+			throw e;
+		} catch (final Throwable e) {
+			throw new CommunityException(e);
+		}
 	}
 
 	@RequestMapping(value = "/context/{contextId}", produces = {
@@ -50,14 +81,52 @@ public class ContextApiController {
 	@Operation(summary = "list the categories defined within a context", description = "list the categories defined within a context")
 	@ApiResponses(value = {
 		@ApiResponse(responseCode = "200", description = "OK"),
+		@ApiResponse(responseCode = "404", description = "not found"),
 		@ApiResponse(responseCode = "500", description = "unexpected error")
 	})
 	public List<CategorySummary> listCategories(
 		@PathVariable final String contextId,
-		@RequestParam(required = false, defaultValue = "false") final Boolean all) throws ContextException {
+		@RequestParam(required = false, defaultValue = "false") final boolean all) throws CommunityException {
 
-		final Boolean allFilter = Optional.ofNullable(all).orElse(false);
-		return contextApiCore.listCategories(contextId, allFilter);
+		try {
+			return communityService.getSubCommunities(contextId)
+				.stream()
+				.filter(sc -> all || sc.isClaim())
+				.map(sc -> {
+					final String[] parts = StringUtils.split(sc.getSubCommunityId(), "::");
+					if (parts.length < 3) { throw new RuntimeException("Invalid conceptId (It should have 3 (or more) parts): " + sc.getSubCommunityId()); }
+					final CategorySummary cat = new CategorySummary();
+					cat.setId(parts[0] + "::" + parts[1]);
+					cat.setLabel(sc.getCategory());
+					cat.setHasConcept(true);
+					return cat;
+				})
+				.distinct()
+				.collect(Collectors.toList());
+		} catch (final ResourceNotFoundException e) {
+			throw e;
+		} catch (final Throwable e) {
+			throw new CommunityException(e);
+		}
+	}
+
+	@RequestMapping(value = "/context/iis/conf/{contextId}", produces = {
+		"application/json"
+	}, method = RequestMethod.GET)
+	@Operation(summary = "return a list of entries for IIS", description = "return a list of entries for IIS")
+	@ApiResponses(value = {
+		@ApiResponse(responseCode = "200", description = "OK"),
+		@ApiResponse(responseCode = "404", description = "not found"),
+		@ApiResponse(responseCode = "500", description = "unexpected error")
+	})
+	public List<IISConfigurationEntry> getIISConfiguration(@PathVariable final String contextId) throws CommunityException {
+		try {
+			return communityService.getIISConfiguration(contextId);
+		} catch (final ResourceNotFoundException e) {
+			throw e;
+		} catch (final Throwable e) {
+			throw new CommunityException(e);
+		}
 	}
 
 	@RequestMapping(value = "/context/category/{categoryId}", produces = {
@@ -66,14 +135,30 @@ public class ContextApiController {
 	@Operation(summary = "list the concepts defined within a category", description = "list the concepts defined within a category")
 	@ApiResponses(value = {
 		@ApiResponse(responseCode = "200", description = "OK"),
+		@ApiResponse(responseCode = "404", description = "not found"),
 		@ApiResponse(responseCode = "500", description = "unexpected error")
 	})
 	public List<ConceptSummary> listConcepts(
 		@PathVariable final String categoryId,
-		@RequestParam(required = false, defaultValue = "false") final Boolean all) throws ContextException {
+		@RequestParam(required = false, defaultValue = "false") final boolean all) throws CommunityException {
+		try {
+			final String[] parts = StringUtils.split(categoryId, "::");
+			if (parts.length != 2) {
+				log.error("Invalid category id (it should have 2 parts): " + categoryId);
+				throw new CommunityException("Invalid category id (it should have 2 parts): " + categoryId);
+			}
+
+			final String contextId = parts[0];
+
+			final List<SubCommunity> list = findSubCommunities(categoryId + "::", all, contextId);
+
+			return processSubCommunities(null, list);
+		} catch (final ResourceNotFoundException e) {
+			throw e;
+		} catch (final Throwable e) {
+			throw new CommunityException(e);
+		}
 
-		final Boolean allFilter = Optional.ofNullable(all).orElse(false);
-		return contextApiCore.listConcepts(categoryId, allFilter);
 	}
 
 	@RequestMapping(value = "/context/category/concept/{conceptId}", produces = {
@@ -82,14 +167,53 @@ public class ContextApiController {
 	@Operation(summary = "list the concepts defined within a category", description = "list the concepts defined within a category")
 	@ApiResponses(value = {
 		@ApiResponse(responseCode = "200", description = "OK"),
+		@ApiResponse(responseCode = "404", description = "not found"),
 		@ApiResponse(responseCode = "500", description = "unexpected error")
 	})
 	public List<ConceptSummary> listSubConcepts(
 		@PathVariable final String conceptId,
-		@RequestParam(required = false, defaultValue = "false") final Boolean all) throws ContextException {
+		@RequestParam(required = false, defaultValue = "false") final boolean all) throws CommunityException {
+		try {
+			final String[] parts = StringUtils.split(conceptId, "::");
+			if (parts.length < 3) {
+				log.error("Invalid concept id (it should have 3 (or more) parts): " + conceptId);
+				throw new CommunityException("Invalid concept id (it should have 3 (or more) parts): " + conceptId);
+			}
 
-		final Boolean allFilter = Optional.ofNullable(all).orElse(false);
-		return contextApiCore.listSubConcepts(conceptId, allFilter);
+			final String contextId = parts[0];
+
+			final List<SubCommunity> list = findSubCommunities(conceptId + "::", all, contextId);
+
+			return processSubCommunities(conceptId, list);
+		} catch (final ResourceNotFoundException e) {
+			throw e;
+		} catch (final Throwable e) {
+			throw new CommunityException(e);
+		}
+
+	}
+
+	private List<SubCommunity> findSubCommunities(final String prefix, final boolean all, final String contextId) throws CommunityException {
+		return communityService.getSubCommunities(contextId)
+			.stream()
+			.filter(sc -> all || sc.isClaim())
+			.filter(sc -> sc.getSubCommunityId().startsWith(prefix))
+			.collect(Collectors.toList());
+	}
+
+	private List<ConceptSummary> processSubCommunities(final String parent, final List<SubCommunity> list) {
+		return list.stream()
+			.filter(sc -> Objects.equals(sc.getParent(), parent))
+			.map(sc -> {
+				final List<ConceptSummary> childs = processSubCommunities(sc.getSubCommunityId(), list);
+				final ConceptSummary concept = new ConceptSummary();
+				concept.setId(sc.getSubCommunityId());
+				concept.setLabel(sc.getLabel());
+				concept.setHasSubConcept(!childs.isEmpty());
+				concept.setConcept(childs);
+				return concept;
+			})
+			.collect(Collectors.toList());
 	}
 
 }

apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/context/ContextApiCore.java

@@ -8,14 +8,12 @@ import java.util.stream.Stream;
 
 import org.apache.commons.lang3.StringUtils;
 import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
-import org.springframework.stereotype.Component;
 
 import com.google.common.base.Splitter;
 import com.google.common.collect.Lists;
 
 import eu.dnetlib.openaire.common.ISClient;
-import eu.dnetlib.openaire.exporter.exceptions.ContextException;
+import eu.dnetlib.openaire.exporter.exceptions.CommunityException;
 import eu.dnetlib.openaire.exporter.model.context.Category;
 import eu.dnetlib.openaire.exporter.model.context.CategorySummary;
 import eu.dnetlib.openaire.exporter.model.context.Concept;
@@ -23,8 +21,9 @@ import eu.dnetlib.openaire.exporter.model.context.ConceptSummary;
 import eu.dnetlib.openaire.exporter.model.context.Context;
 import eu.dnetlib.openaire.exporter.model.context.ContextSummary;
 
-@Component
-@ConditionalOnProperty(value = "openaire.exporter.enable.context", havingValue = "true")
+// @Component
+// @ConditionalOnProperty(value = "openaire.exporter.enable.context", havingValue = "true")
+@Deprecated
 public class ContextApiCore {
 
 	private static final String SEPARATOR = "::";
@@ -32,18 +31,24 @@ public class ContextApiCore {
 	@Autowired
 	private ISClient isClient;
 
-	public List<ContextSummary> listContexts(final List<String> type) throws ContextException {
+	public List<ContextSummary> listContexts(final List<String> type) throws CommunityException {
+
 		return getContextMap(type).values()
 			.stream()
 			.map(c -> new ContextSummary()
 				.setId(c.getId())
 				.setType(c.getType())
 				.setLabel(c.getLabel())
-				.setStatus(c.getParams().containsKey("status") ? c.getParams().get("status").get(0).getValue() : ""))
+				.setStatus(c.getParams()
+					.stream()
+					.filter(p -> p.getName().equals("status"))
+					.map(p -> p.getValue())
+					.findFirst()
+					.orElse("")))
 			.collect(Collectors.toList());
 	}
 
-	public List<CategorySummary> listCategories(final String contextId, final Boolean all) throws ContextException {
+	public List<CategorySummary> listCategories(final String contextId, final Boolean all) throws CommunityException {
 		final Stream<Category> categories = getContextMap().get(contextId).getCategories().values().stream();
 		return all ? asCategorySummaries(categories) : asCategorySummaries(categories.filter(Category::isClaim));
 	}
@@ -57,7 +62,7 @@ public class ContextApiCore {
 			.collect(Collectors.toList());
 	}
 
-	public List<ConceptSummary> listConcepts(final String categoryId, final Boolean all) throws ContextException {
+	public List<ConceptSummary> listConcepts(final String categoryId, final Boolean all) throws CommunityException {
 		final String contextId = StringUtils.substringBefore(categoryId, SEPARATOR);
 		final Stream<Concept> concepts = getContextMap().get(contextId)
 			.getCategories()
@@ -77,9 +82,9 @@ public class ContextApiCore {
 			.collect(Collectors.toList());
 	}
 
-	public List<ConceptSummary> listSubConcepts(final String conceptId, final Boolean all) throws ContextException {
+	public List<ConceptSummary> listSubConcepts(final String conceptId, final Boolean all) throws CommunityException {
 		final List<String> ids = Splitter.on(SEPARATOR).splitToList(conceptId);
-		if (ids.size() < 3) { throw new ContextException(""); }
+		if (ids.size() < 3) { throw new CommunityException(""); }
 
 		final String contextId = ids.get(0);
 		final String categoryId = contextId + SEPARATOR + ids.get(1);
@@ -105,15 +110,15 @@ public class ContextApiCore {
 			.collect(Collectors.toList());
 	}
 
-	private Map<String, Context> getContextMap() throws ContextException {
+	private Map<String, Context> getContextMap() throws CommunityException {
 		return getContextMap(Lists.newArrayList());
 	}
 
-	private Map<String, Context> getContextMap(final List<String> type) throws ContextException {
+	private Map<String, Context> getContextMap(final List<String> type) throws CommunityException {
 		try {
 			return isClient.getContextMap(type);
 		} catch (final IOException e) {
-			throw new ContextException(e);
+			throw new CommunityException(e);
 		}
 	}
 

apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/context/ContextMappingUtils.java

@@ -1,7 +1,6 @@
 package eu.dnetlib.openaire.context;
 
 import java.text.ParseException;
-import java.util.ArrayList;
 import java.util.Date;
 import java.util.List;
 import java.util.Map;
@@ -11,6 +10,8 @@ import java.util.stream.Collectors;
 import org.apache.commons.lang3.BooleanUtils;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.commons.lang3.time.DateUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
 import org.dom4j.Document;
 import org.dom4j.DocumentException;
 import org.dom4j.DocumentHelper;
@@ -26,25 +27,31 @@ import eu.dnetlib.openaire.exporter.model.context.Context;
 import eu.dnetlib.openaire.exporter.model.context.Param;
 import eu.dnetlib.openaire.exporter.model.funders.FunderDetails;
 
+@Deprecated
 public class ContextMappingUtils {
 
 	private static final List<String> DATE_PATTERN = Lists.newArrayList("yyyy-MM-dd'T'hh:mm:ss", "yyyy-MM-dd'T'hh:mm:ssXXX", "yyyy-MM-dd'T'hh:mm:ss+00:00");
 
+	private static final Log log = LogFactory.getLog(ContextMappingUtils.class);
+
 	public static Context parseContext(final String s, final Queue<Throwable> errors) {
 		try {
 			final Document doc = DocumentHelper.parseText(s);
 			final Element eContext = (Element) doc.selectSingleNode("/RESOURCE_PROFILE/BODY/CONFIGURATION/context");
 			final String creationDate = eContext.valueOf("./param[./@name='creationdate']/text()");
+			final String otherDate = doc.valueOf("/RESOURCE_PROFILE/HEADER/DATE_OF_CREATION/@value");
 			final Context c = new Context()
 				.setId(eContext.attributeValue("id"))
 				.setLabel(eContext.attributeValue("label"))
 				.setType(eContext.attributeValue("type"))
-				.setLastUpdateDate(asDate(doc.valueOf("/RESOURCE_PROFILE/HEADER/DATE_OF_CREATION/@value")))
+				.setLastUpdateDate(asDate(otherDate))
 				.setParams(parseParams(eContext))
 				.setCategories(parseCategories(eContext));
 			// the creation date will be added in the param elements of the community profile. Funders may not have it, hence the check.
 			if (StringUtils.isNotBlank(creationDate)) {
 				c.setCreationDate(asDate(creationDate));
+			} else {
+				c.setCreationDate(asDate(otherDate));
 			}
 			return c;
 		} catch (final DocumentException e) {
@@ -54,11 +61,15 @@ public class ContextMappingUtils {
 	}
 
 	private static Date asDate(final String s) {
+		if (StringUtils.isBlank(s)) { return null; }
+
 		for (final String pattern : DATE_PATTERN) {
 			try {
-				return DateUtils.parseDate(s, pattern);
+				final Date res = DateUtils.parseDate(s, pattern);
+				if (res != null) { return res; }
 			} catch (final ParseException e) {}
 		}
+		log.warn("Invalid Date: " + s);
 		return null;
 	}
 
@@ -93,18 +104,12 @@ public class ContextMappingUtils {
 		return BooleanUtils.toBooleanObject(StringUtils.isNotBlank(claim) ? claim : "false");
 	}
 
-	private static Map<String, List<Param>> parseParams(final Element e) {
+	private static List<Param> parseParams(final Element e) {
 		final List<Node> params = e.selectNodes("./param");
 		return params.stream()
 			.map(n -> (Element) n)
-			.map(p -> new Param()
-				.setName(p.attributeValue("name"))
-				.setValue(p.getTextTrim()))
-			.collect(Collectors.toMap(Param::getName, Lists::newArrayList, (p1, p2) -> {
-				final List<Param> p = new ArrayList<>(p1);
-				p.addAll(p2);
-				return p;
-			}));
+			.map(p -> new Param().setName(p.attributeValue("name")).setValue(p.getTextTrim()))
+			.collect(Collectors.toList());
 	}
 
 	public static FunderDetails asFunderDetails(final Context c) {

apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/DsmCore.java

@@ -36,7 +36,6 @@ import eu.dnetlib.enabling.datasources.common.DsmException;
 import eu.dnetlib.enabling.datasources.common.DsmForbiddenException;
 import eu.dnetlib.enabling.datasources.common.DsmNotFoundException;
 import eu.dnetlib.openaire.common.ISClient;
-import eu.dnetlib.openaire.community.CommunityClient;
 import eu.dnetlib.openaire.dsm.dao.CountryTermRepository;
 import eu.dnetlib.openaire.dsm.dao.DatasourceDao;
 import eu.dnetlib.openaire.dsm.dao.MongoLoggerClient;
@@ -91,9 +90,6 @@ public class DsmCore {
 	@Autowired
 	private JdbcTemplate jdbcTemplate;
 
-	@Autowired
-	private CommunityClient communityClient;
-
 	public List<Country> listCountries() throws DsmApiException {
 		final List<Country> countries = Lists.newArrayList();
 		final Vocabulary v = vocabularyClient.getCountries();
@@ -321,7 +317,6 @@ public class DsmCore {
 		mongoLoggerClient.dropCache();
 		isClient.dropCache();
 		vocabularyClient.dropCache();
-		communityClient.dropCache();
 	}
 
 	// HELPERS //////////////

apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/MongoLoggerClientImpl.java

@@ -48,11 +48,7 @@ import eu.dnetlib.openaire.exporter.exceptions.DsmApiException;
 import eu.dnetlib.openaire.exporter.model.dsm.AggregationInfo;
 import eu.dnetlib.openaire.exporter.model.dsm.AggregationInfoV1;
 import eu.dnetlib.openaire.exporter.model.dsm.AggregationStage;
-import eu.dnetlib.openaire.exporter.model.dsm.CollectionInfoV1;
-import eu.dnetlib.openaire.exporter.model.dsm.CollectionInfoV2;
 import eu.dnetlib.openaire.exporter.model.dsm.CollectionMode;
-import eu.dnetlib.openaire.exporter.model.dsm.TransformationInfoV1;
-import eu.dnetlib.openaire.exporter.model.dsm.TransformationInfoV2;
 import eu.dnetlib.openaire.info.JdbcInfoDao;
 
 /**
@@ -75,16 +71,16 @@ public class MongoLoggerClientImpl implements MongoLoggerClient {
 
 	private final static String LOADTIME = "loadtime";
 	private final LoadingCache<String, Instant> loadingCache =
-		CacheBuilder.newBuilder().maximumSize(1).expireAfterWrite(60, TimeUnit.MINUTES).build(new CacheLoader<String, Instant>() {
+			CacheBuilder.newBuilder().maximumSize(1).expireAfterWrite(60, TimeUnit.MINUTES).build(new CacheLoader<String, Instant>() {
 
-			// The only cached value is associated to "loadtime"
-			@Override
-			public Instant load(final String key) {
-				final Instant loadTime = getLoadTime();
-				log.debug("found load time: " + loadTime.toString());
-				return loadTime;
-			}
-		});
+				// The only cached value is associated to "loadtime"
+				@Override
+				public Instant load(final String key) {
+					final Instant loadTime = getLoadTime();
+					log.debug("found load time: " + loadTime.toString());
+					return loadTime;
+				}
+			});
 
 	private static final Bson fields = getFields();
 
@@ -94,30 +90,64 @@ public class MongoLoggerClientImpl implements MongoLoggerClient {
 	@Cacheable("dsm-aggregationhistory-cache-v1")
 	@Deprecated
 	public List<AggregationInfoV1> getAggregationHistoryV1(final String dsId) throws DsmApiException {
-		return getAggregationHistory(dsId, queryForAggregationHistoryV1(dsId, "(collect|transform)"), getMapperV1());
+		return getAggregationHistoryV1(dsId, queryForAggregationHistoryV1(dsId, "(collect|transform)"), getMapperV1());
 	}
 
 	@Override
 	@Cacheable("dsm-aggregationhistory-cache-v2")
 	public List<AggregationInfo> getAggregationHistoryV2(final String dsId) throws DsmApiException {
-		return getAggregationHistory(dsId, queryForAggregationHistoryV2(dsId, "(collect|transform)"), getMapperV2());
+		return getAggregationHistoryV2(dsId, queryForAggregationHistoryV2(dsId, "(collect|transform)"), getMapperV2());
 	}
 
-	private <T extends AggregationInfo> List<T> getAggregationHistory(final String dsId,
-		final Bson queryForAggregationHistory,
-		final Function<Document, T> mapper) throws DsmApiException {
+	@Deprecated
+	private List<AggregationInfoV1> getAggregationHistoryV1(final String dsId,
+			final Bson queryForAggregationHistory,
+			final Function<Document, AggregationInfoV1> mapper) throws DsmApiException {
 		log.warn(String.format("getAggregationHistory(dsId = %s): not using cache", dsId));
 		final Datasource conf = config.getDatasource();
 		try {
 			final FindIterable<Document> aggregationDocs = getCollection().find(queryForAggregationHistory)
-				.projection(fields)
-				.limit(conf.getMongoQueryLimit())
-				.sort(dbo("system:startHumanDate", -1));
+					.projection(fields)
+					.limit(conf.getMongoQueryLimit())
+					.sort(dbo("system:startHumanDate", -1));
 
-			final List<T> aggregationInfos = Utils.stream(aggregationDocs.iterator())
-				.map(mapper)
-				.filter(ai -> ai.getNumberOfRecords() >= 0 && StringUtils.isNotBlank(ai.getDate()))
-				.collect(Collectors.toList());
+			final List<AggregationInfoV1> aggregationInfos = Utils.stream(aggregationDocs.iterator())
+					.map(mapper)
+					.filter(ai -> ai.getNumberOfRecords() >= 0 && StringUtils.isNotBlank(ai.getDate()))
+					.collect(Collectors.toList());
+
+			final Instant loadTime = loadingCache.get(LOADTIME);
+
+			if (!Objects.equals(Instant.MIN, loadTime)) {
+				for (final AggregationInfoV1 a : aggregationInfos) {
+					if (asInstant(a).isBefore(loadTime) && AggregationStage.COLLECT.equals(a.getAggregationStage())) {
+						a.setIndexedVersion(true);
+						break;
+					}
+				}
+			}
+
+			return aggregationInfos;
+		} catch (final Throwable e) {
+			throw new DsmApiException(HttpStatus.SC_INTERNAL_SERVER_ERROR, String.format("error reading aggregation history for '%s'", dsId), e);
+		}
+	}
+
+	private List<AggregationInfo> getAggregationHistoryV2(final String dsId,
+			final Bson queryForAggregationHistory,
+			final Function<Document, AggregationInfo> mapper) throws DsmApiException {
+		log.warn(String.format("getAggregationHistory(dsId = %s): not using cache", dsId));
+		final Datasource conf = config.getDatasource();
+		try {
+			final FindIterable<Document> aggregationDocs = getCollection().find(queryForAggregationHistory)
+					.projection(fields)
+					.limit(conf.getMongoQueryLimit())
+					.sort(dbo("system:startHumanDate", -1));
+
+			final List<AggregationInfo> aggregationInfos = Utils.stream(aggregationDocs.iterator())
+					.map(mapper)
+					.filter(ai -> ai.getNumberOfRecords() >= 0 && StringUtils.isNotBlank(ai.getDate()))
+					.collect(Collectors.toList());
 
 			final Instant loadTime = loadingCache.get(LOADTIME);
 
@@ -146,9 +176,14 @@ public class MongoLoggerClientImpl implements MongoLoggerClient {
 		return Instant.parse(a.getDate() + "T00:00:00Z");
 	}
 
+	@Deprecated
+	private Instant asInstant(final AggregationInfoV1 a) {
+		return Instant.parse(a.getDate() + "T00:00:00Z");
+	}
+
 	@Override
 	@CacheEvict(cacheNames = {
-		"dsm-aggregationhistory-cache-v1", "dsm-aggregationhistory-cache-v2", "dsm-firstharvestdate-cache"
+			"dsm-aggregationhistory-cache-v1", "dsm-aggregationhistory-cache-v2", "dsm-firstharvestdate-cache"
 	}, allEntries = true)
 	@Scheduled(fixedDelayString = "${openaire.exporter.cache.ttl}")
 	public void dropCache() {
@@ -166,20 +201,18 @@ public class MongoLoggerClientImpl implements MongoLoggerClient {
 			switch (stage) {
 
 			case COLLECT:
-				final CollectionInfoV1 cInfo = new CollectionInfoV1();
+				final AggregationInfoV1 cInfo = new AggregationInfoV1();
 				cInfo.setAggregationStage(stage);
 				cInfo.setCollectionMode(getCollectionMode(d));
 				cInfo.setNumberOfRecords(success ? getNumberOfRecords(d) : 0);
 				cInfo.setDate(getDate(d));
-				cInfo.setCompletedSuccessfully(success);
 				info = cInfo;
 				break;
 			case TRANSFORM:
-				final TransformationInfoV1 tInfo = new TransformationInfoV1();
+				final AggregationInfoV1 tInfo = new AggregationInfoV1();
 				tInfo.setAggregationStage(stage);
 				tInfo.setNumberOfRecords(success ? getNumberOfRecords(d) : 0);
 				tInfo.setDate(getDate(d));
-				tInfo.setCompletedSuccessfully(success);
 				info = tInfo;
 				break;
 			}
@@ -197,7 +230,7 @@ public class MongoLoggerClientImpl implements MongoLoggerClient {
 			switch (stage) {
 
 			case COLLECT:
-				final CollectionInfoV2 cInfo = new CollectionInfoV2();
+				final AggregationInfo cInfo = new AggregationInfo();
 				cInfo.setAggregationStage(stage);
 				cInfo.setCollectionMode(getCollectionMode(d));
 				cInfo.setNumberOfRecords(success ? getNumberOfRecords(d) : 0);
@@ -206,7 +239,7 @@ public class MongoLoggerClientImpl implements MongoLoggerClient {
 				info = cInfo;
 				break;
 			case TRANSFORM:
-				final TransformationInfoV2 tInfo = new TransformationInfoV2();
+				final AggregationInfo tInfo = new AggregationInfo();
 				tInfo.setAggregationStage(stage);
 				tInfo.setNumberOfRecords(success ? getNumberOfRecords(d) : 0);
 				tInfo.setDate(getDate(d));
@@ -220,23 +253,19 @@ public class MongoLoggerClientImpl implements MongoLoggerClient {
 
 	private CollectionMode getCollectionMode(final Document d) {
 		return Optional.ofNullable(d.getString("system:node:SELECT_MODE:selection"))
-			.map(CollectionMode::valueOf)
-			.orElseGet(() -> Optional.ofNullable(d.getString("collectionMode"))
 				.map(CollectionMode::valueOf)
-				.orElse(null));
+				.orElseGet(() -> Optional.ofNullable(d.getString("collectionMode"))
+						.map(CollectionMode::valueOf)
+						.orElse(null));
 	}
 
 	private Integer getNumberOfRecords(final Document d) {
 		final String sinkSize = d.getString("mainlog:sinkSize");
 		final String total = d.getString("mainlog:total");
 
-		if (StringUtils.isNotBlank(sinkSize)) {
-			return Ints.tryParse(sinkSize);
-		} else if (StringUtils.isNotBlank(total)) {
-			return Ints.tryParse(total);
-		} else {
-			return -1;
-		}
+		if (StringUtils.isNotBlank(sinkSize)) { return Ints.tryParse(sinkSize); }
+		if (StringUtils.isNotBlank(total)) { return Ints.tryParse(total); }
+		return -1;
 	}
 
 	private String getDate(final Document d) {

apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/ConversionUtils.java

@@ -1,91 +0,0 @@
-package eu.dnetlib.openaire.funders;
-
-import java.io.StringReader;
-import java.util.stream.Collectors;
-
-import eu.dnetlib.openaire.exporter.model.funders.ExtendedFunderDetails;
-import eu.dnetlib.openaire.exporter.model.funders.FunderDetails;
-import eu.dnetlib.openaire.exporter.model.funders.FundingStream;
-import eu.dnetlib.openaire.funders.domain.db.FunderDbEntry;
-import eu.dnetlib.openaire.funders.domain.db.FundingPathDbEntry;
-import org.dom4j.Document;
-import org.dom4j.DocumentException;
-import org.dom4j.io.SAXReader;
-
-import static org.apache.commons.lang3.StringUtils.*;
-
-public class ConversionUtils {
-
-	public static final String SEPARATOR = "::";
-
-	public static FunderDetails asFunderDetails(final FunderDbEntry fdb) {
-		final FunderDetails f = new FunderDetails();
-
-		f.setId(fdb.getId());
-		f.setName(fdb.getName());
-		f.setShortname(fdb.getShortname());
-		f.setJurisdiction(fdb.getJurisdiction());
-		f.setRegistrationDate(fdb.getRegistrationdate());
-		f.setLastUpdateDate(fdb.getLastupdatedate());
-
-		return f;
-	}
-
-	public static ExtendedFunderDetails asExtendedFunderDetails(final FunderDbEntry fdb) {
-		final ExtendedFunderDetails f = new ExtendedFunderDetails(asFunderDetails(fdb));
-
-		if (fdb.getFundingpaths() != null) {
-			f.setFundingStreams(
-					fdb.getFundingpaths().stream()
-						.map(ConversionUtils::asFundingStream)
-						.collect(Collectors.toList()));
-		}
-
-		return f;
-	}
-
-	private static FundingStream asFundingStream(final FundingPathDbEntry pathDbEntry) {
-		final FundingStream f = new FundingStream();
-
-		try {
-			final Document xml = new SAXReader().read(new StringReader(pathDbEntry.getPath()));
-
-			for(int i=2;i>=0;i--) {
-				if (hasFundingLevel(i, xml)) {
-					f.setId(getId(i, xml));
-					f.setName(getName(i, xml));
-					break;
-				}
-			}
-
-			if (isBlank(f.getId()) && isNoneBlank(xml.valueOf("//funder/id/text()"))) {
-				f.setId(xml.valueOf("//funder/shortname/text()"));
-				f.setName(xml.valueOf("//funder/name/text()"));
-			}
-
-			if (isBlank(f.getId())) {
-				throw new IllegalStateException("invalid funding path:\n" + xml.asXML());
-			}
-
-			return f;
-
-		} catch (DocumentException e) {
-			throw new IllegalStateException("unable to parse funding path:\n" + pathDbEntry.getPath());
-		}
-	}
-
-	private static String getName(int level, final Document d) {
-		return d.valueOf(String.format("//funding_level_%s/description/text()", level));
-	}
-
-	private static String getId(int level, final Document d) {
-		return substringAfter(
-				d.valueOf(
-					String.format("//funding_level_%s/id/text()", level)
-				), SEPARATOR);
-	}
-
-	private static boolean hasFundingLevel(int level, Document d) {
-		return isNotBlank(getId(level, d));
-	}
-}

apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/FunderDao.java

@@ -1,43 +0,0 @@
-package eu.dnetlib.openaire.funders;
-
-import java.util.List;
-import java.util.stream.Collectors;
-
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
-import org.springframework.data.domain.PageRequest;
-import org.springframework.stereotype.Component;
-
-import eu.dnetlib.openaire.exporter.exceptions.FundersApiException;
-import eu.dnetlib.openaire.exporter.model.funders.ExtendedFunderDetails;
-import eu.dnetlib.openaire.exporter.model.funders.FunderDetails;
-
-@Component
-@ConditionalOnProperty(value = "openaire.exporter.enable.funders", havingValue = "true")
-public class FunderDao {
-
-	@Autowired
-	private FunderRepository funderRepository;
-
-	public ExtendedFunderDetails getExtendedFunderDetails(final String funderId) throws FundersApiException {
-		return ConversionUtils
-			.asExtendedFunderDetails(funderRepository.findById(funderId).orElseThrow(() -> new FundersApiException("Funder not found. ID:  " + funderId)));
-	}
-
-	public List<FunderDetails> listFunderDetails(final int page, final int size) throws FundersApiException {
-		return funderRepository.findAll(PageRequest.of(page, size))
-			.getContent()
-			.stream()
-			.map(ConversionUtils::asFunderDetails)
-			.collect(Collectors.toList());
-	}
-
-	public List<String> listFunderIds(final int page, final int size) throws FundersApiException {
-		return funderRepository.findAll(PageRequest.of(page, size))
-			.getContent()
-			.stream()
-			.map(f -> f.getId())
-			.collect(Collectors.toList());
-	}
-
-}

apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/FunderRepository.java

@@ -1,10 +1,11 @@
 package eu.dnetlib.openaire.funders;
 
-import eu.dnetlib.openaire.funders.domain.db.FunderDbEntry;
-import org.springframework.data.jpa.repository.JpaRepository;
 import org.springframework.stereotype.Repository;
 
+import eu.dnetlib.openaire.common.ReadOnlyRepository;
+import eu.dnetlib.openaire.funders.domain.db.FunderDbEntry;
+
 @Repository
-public interface FunderRepository extends JpaRepository<FunderDbEntry, String> {
+public interface FunderRepository extends ReadOnlyRepository<FunderDbEntry, String> {
 
 }

apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/FunderService.java

@@ -0,0 +1,183 @@
+package eu.dnetlib.openaire.funders;
+
+import java.io.File;
+import java.io.FileWriter;
+import java.io.FilenameFilter;
+import java.time.LocalDate;
+import java.time.format.DateTimeFormatter;
+import java.util.Arrays;
+import java.util.Comparator;
+import java.util.List;
+import java.util.Objects;
+import java.util.stream.Collectors;
+
+import javax.annotation.PostConstruct;
+
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
+import org.springframework.scheduling.annotation.Scheduled;
+import org.springframework.stereotype.Component;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.SerializationFeature;
+import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
+
+import eu.dnetlib.openaire.dsm.dao.MongoLoggerClient;
+import eu.dnetlib.openaire.exporter.exceptions.DsmApiException;
+import eu.dnetlib.openaire.exporter.model.dsm.AggregationInfo;
+import eu.dnetlib.openaire.exporter.model.dsm.AggregationStage;
+import eu.dnetlib.openaire.funders.domain.db.FunderDatasource;
+import eu.dnetlib.openaire.funders.domain.db.FunderDbEntry;
+import eu.dnetlib.openaire.funders.domain.db.FunderPid;
+
+@Component
+@ConditionalOnProperty(value = "openaire.exporter.enable.funders", havingValue = "true")
+public class FunderService {
+
+	private static final String TEMP_FILE_SUFFIX = ".funds.tmp";
+	private static final String SEPARATOR = "@=@";
+
+	@Autowired
+	private FunderRepository funderRepository;
+
+	@Autowired
+	private MongoLoggerClient mongoLoggerClient;
+
+	private File tempDir;
+
+	private File tempFile;
+
+	private final DateTimeFormatter DATEFORMATTER = DateTimeFormatter.ofPattern("yyyy-MM-dd");
+
+	private static final Log log = LogFactory.getLog(FunderService.class);
+
+	@PostConstruct
+	public void init() {
+
+		tempDir = new File(System.getProperty("java.io.tmpdir", "/tmp"));
+
+		for (final File f : tempDir.listFiles((FilenameFilter) (dir, name) -> name.endsWith(TEMP_FILE_SUFFIX))) {
+			deleteFile(f);
+		}
+
+		new Thread(this::updateFunders).start();
+	}
+
+	private void deleteFile(final File f) {
+		if (f != null && f.exists()) {
+			log.info("Deleting file: " + f.getAbsolutePath());
+			f.delete();
+		}
+	}
+
+	@Scheduled(cron = "${openaire.exporter.funders.cron}")
+	public void updateFunders() {
+		try {
+			final ObjectMapper mapper = new ObjectMapper();
+			mapper.registerModule(new JavaTimeModule());
+			mapper.configure(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS, false);
+
+			final File tmp = File.createTempFile("funders-api-", TEMP_FILE_SUFFIX, tempDir);
+
+			log.info("Generating funders file: " + tmp.getAbsolutePath());
+
+			try (final FileWriter writer = new FileWriter(tmp)) {
+				writer.write("[");
+				boolean first = true;
+				for (final FunderDbEntry funder : funderRepository.findAll()) {
+					log.info("  - adding: " + funder.getId());
+
+					// THIS PATCH IS NECESSARY FOR COMPATIBILITY WITH POSTGRES 9.3 (PARTIAL SUPPORT OF THE JSON LIBRARY)
+
+					final List<FunderDatasource> datasources = Arrays.stream(funder.getDatasourcesPostgres())
+							.filter(Objects::nonNull)
+							.map(s -> s.split(SEPARATOR))
+							.filter(arr -> arr.length == 3)
+							.map(arr -> {
+								final FunderDatasource ds = new FunderDatasource();
+								ds.setId(arr[0].trim());
+								ds.setName(arr[1].trim());
+								ds.setType(arr[2].trim());
+								return ds;
+							})
+							.filter(ds -> StringUtils.isNotBlank(ds.getId()))
+							.collect(Collectors.toList());
+
+					funder.setDatasources(datasources);
+
+					final List<FunderPid> pids = Arrays.stream(funder.getPidsPostgres())
+							.filter(Objects::nonNull)
+							.map(s -> s.split(SEPARATOR))
+							.filter(arr -> arr.length == 2)
+							.map(arr -> {
+								final FunderPid pid = new FunderPid();
+								pid.setType(arr[0].trim());
+								pid.setValue(arr[1].trim());
+								return pid;
+							})
+							.filter(pid -> StringUtils.isNotBlank(pid.getValue()))
+							.collect(Collectors.toList());
+
+					funder.setPids(pids);
+
+					// END PATCH
+
+					addAggregationHistory(funder);
+
+					if (first) {
+						first = false;
+					} else {
+						writer.write(",");
+					}
+					writer.write(mapper.writeValueAsString(funder));
+				}
+				writer.write("]");
+				log.info("Publish funders file: " + tmp.getAbsolutePath());
+
+				deleteFile(tempFile);
+				setTempFile(tmp);
+			}
+		} catch (final Throwable e) {
+			log.error("Error generating funders file", e);
+			throw new RuntimeException("Error generating funders file", e);
+		}
+	}
+
+	private void addAggregationHistory(final FunderDbEntry funder) {
+
+		final List<LocalDate> dates = funder.getDatasources()
+				.stream()
+				.map(FunderDatasource::getId)
+				.map(id -> {
+					try {
+						return mongoLoggerClient.getAggregationHistoryV2(id);
+					} catch (final DsmApiException e) {
+						log.error("Error retrieving the aggregation history", e);
+						throw new RuntimeException("Error retrieving the aggregation history", e);
+					}
+				})
+				.flatMap(List::stream)
+				.filter(AggregationInfo::isCompletedSuccessfully)
+				.filter(info -> info.getAggregationStage() == AggregationStage.TRANSFORM)
+				.map(AggregationInfo::getDate)
+				.distinct()
+				.map(s -> LocalDate.parse(s, DATEFORMATTER))
+				.sorted(Comparator.reverseOrder())
+				.limit(10)
+				.collect(Collectors.toList());
+
+		funder.setAggregationDates(dates);
+	}
+
+	public File getTempFile() {
+		return tempFile;
+	}
+
+	public void setTempFile(final File tempFile) {
+		this.tempFile = tempFile;
+	}
+
+}

apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/FundersApiController.java

@@ -1,21 +1,25 @@
 package eu.dnetlib.openaire.funders;
 
-import java.util.List;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.InputStream;
+import java.io.OutputStream;
 
+import javax.servlet.http.HttpServletResponse;
+
+import org.apache.commons.io.IOUtils;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
+import org.springframework.http.MediaType;
 import org.springframework.web.bind.annotation.CrossOrigin;
-import org.springframework.web.bind.annotation.PathVariable;
 import org.springframework.web.bind.annotation.RequestMapping;
 import org.springframework.web.bind.annotation.RequestMethod;
 import org.springframework.web.bind.annotation.RestController;
 
 import eu.dnetlib.openaire.common.AbstractExporterController;
 import eu.dnetlib.openaire.exporter.exceptions.FundersApiException;
-import eu.dnetlib.openaire.exporter.model.funders.ExtendedFunderDetails;
-import eu.dnetlib.openaire.exporter.model.funders.FunderDetails;
 import io.swagger.v3.oas.annotations.Operation;
 import io.swagger.v3.oas.annotations.responses.ApiResponse;
 import io.swagger.v3.oas.annotations.responses.ApiResponses;
@@ -23,59 +27,48 @@ import io.swagger.v3.oas.annotations.tags.Tag;
 
 @RestController
 @CrossOrigin(origins = {
-	"*"
+		"*"
 })
 @ConditionalOnProperty(value = "openaire.exporter.enable.funders", havingValue = "true")
 @Tag(name = "OpenAIRE funders API", description = "the OpenAIRE funders API")
 public class FundersApiController extends AbstractExporterController {
 
+	@Autowired
+	private FunderService service;
+
 	private static final Log log = LogFactory.getLog(FundersApiController.class);
 
-	@Autowired
-	private FunderDao fDao;
-
 	@RequestMapping(value = "/funders", produces = {
-		"application/json"
+			"application/json"
 	}, method = RequestMethod.GET)
-	@Operation(summary = "get basic information about funders", description = "basic information about funders: id, name, shortname, last update date, registration date")
+	@Operation(summary = "get all funders", description = "get all funders")
 	@ApiResponses(value = {
-		@ApiResponse(responseCode = "200", description = "OK"),
-		@ApiResponse(responseCode = "500", description = "unexpected error")
+			@ApiResponse(responseCode = "200", description = "OK"),
+			@ApiResponse(responseCode = "500", description = "unexpected error")
 	})
-	public List<FunderDetails> getFunders(
-		@PathVariable final int page,
-		@PathVariable final int size) throws FundersApiException {
+	public void getFunders(final HttpServletResponse res) throws FundersApiException {
 
-		return fDao.listFunderDetails(page, size);
-	}
+		res.setContentType(MediaType.APPLICATION_JSON_VALUE);
 
-	@RequestMapping(value = "/funder/{id}", produces = {
-		"application/json"
-	}, method = RequestMethod.GET)
-	@Operation(summary = "get the funder details", description = "complete funder information")
-	@ApiResponses(value = {
-		@ApiResponse(responseCode = "200", description = "OK"),
-		@ApiResponse(responseCode = "500", description = "unexpected error")
-	})
-	public ExtendedFunderDetails getFunderDetails(
-		@PathVariable final String id) throws FundersApiException {
+		final File file = service.getTempFile();
 
-		return fDao.getExtendedFunderDetails(id);
-	}
+		if (file == null) {
+			log.error("Missing temp file (NULL)");
+			throw new FundersApiException("Missing temp file (NULL)");
+		}
 
-	@RequestMapping(value = "/funder/ids", produces = {
-		"application/json"
-	}, method = RequestMethod.GET)
-	@Operation(summary = "get the list of funder ids", description = "get the list of funder ids")
-	@ApiResponses(value = {
-		@ApiResponse(responseCode = "200", description = "OK"),
-		@ApiResponse(responseCode = "500", description = "unexpected error")
-	})
-	public List<String> getFunderIds(
-		@PathVariable final int page,
-		@PathVariable final int size) throws FundersApiException {
+		if (!file.exists()) {
+			log.error("Missing temp file " + service.getTempFile());
+			throw new FundersApiException("Missing temp file " + service.getTempFile());
+		}
 
-		return fDao.listFunderIds(page, size);
+		try (final InputStream in = new FileInputStream(file); OutputStream out = res.getOutputStream()) {
+			IOUtils.copy(in, out);
+			return;
+		} catch (final Exception e) {
+			log.error("Error reading file " + service.getTempFile(), e);
+			throw new FundersApiException("Error reading file " + service.getTempFile(), e);
+		}
 	}
 
 }

apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/domain/db/FunderDatasource.java

@@ -0,0 +1,39 @@
+package eu.dnetlib.openaire.funders.domain.db;
+
+import java.io.Serializable;
+
+public class FunderDatasource implements Serializable {
+
+	private static final long serialVersionUID = 2145493560459874509L;
+
+	private String id;
+
+	private String name;
+
+	private String type;
+
+	public String getId() {
+		return id;
+	}
+
+	public void setId(final String id) {
+		this.id = id;
+	}
+
+	public String getName() {
+		return name;
+	}
+
+	public void setName(final String name) {
+		this.name = name;
+	}
+
+	public String getType() {
+		return type;
+	}
+
+	public void setType(final String type) {
+		this.type = type;
+	}
+
+}

apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/domain/db/FunderDbEntry.java

@@ -1,38 +1,75 @@
 package eu.dnetlib.openaire.funders.domain.db;
 
-import java.sql.Date;
-import java.util.Set;
-import javax.persistence.*;
+import java.io.Serializable;
+import java.time.LocalDate;
+import java.util.ArrayList;
+import java.util.List;
 
-import eu.dnetlib.openaire.dsm.domain.db.IdentityDbEntry;
+import javax.persistence.Column;
+import javax.persistence.Entity;
+import javax.persistence.Id;
+import javax.persistence.Table;
+import javax.persistence.Transient;
+
+import org.hibernate.annotations.Type;
+import org.hibernate.annotations.TypeDef;
+import org.hibernate.annotations.TypeDefs;
+
+import com.fasterxml.jackson.annotation.JsonIgnore;
+import com.vladmihalcea.hibernate.type.array.StringArrayType;
 
 @Entity
-@Table(name = "funders")
-public class FunderDbEntry {
+@Table(name = "funders_view")
+@TypeDefs({
+		@TypeDef(name = "string-array", typeClass = StringArrayType.class)
+})
+public class FunderDbEntry implements Serializable {
+
+	private static final long serialVersionUID = 1290088460508203016L;
 
 	@Id
+	@Column(name = "id")
 	private String id;
-	private String name;
-	private String shortname;
-	private String jurisdiction;
-	private String websiteurl;
-	private String policy;
-	private Date registrationdate;
-	private Date lastupdatedate;
 
-	@OneToMany(
-			cascade = { CascadeType.PERSIST, CascadeType.MERGE },
-			fetch = FetchType.LAZY )
-	@JoinTable(
-			name = "funder_identity",
-			joinColumns = @JoinColumn(name = "funder"),
-			inverseJoinColumns = @JoinColumn(name = "pid"))
-	private Set<IdentityDbEntry> pids;
+	@Column(name = "legalshortname")
+	private String legalShortName;
 
-	@OneToMany(mappedBy = "funderid", cascade = CascadeType.ALL, fetch = FetchType.LAZY)
-	private Set<FundingPathDbEntry> fundingpaths;
+	@Column(name = "legalname")
+	private String legalName;
 
-	public FunderDbEntry() {}
+	@Column(name = "websiteurl")
+	private String websiteUrl;
+
+	@Column(name = "logourl")
+	private String logoUrl;
+
+	@Column(name = "country")
+	private String country;
+
+	@Column(name = "registrationdate")
+	private LocalDate registrationDate;
+
+	@Column(name = "registered")
+	private Boolean registered;
+
+	@JsonIgnore
+	@Type(type = "string-array")
+	@Column(name = "pids", columnDefinition = "text[]")
+	private String[] pidsPostgres;
+
+	@Transient
+	private List<FunderPid> pids = new ArrayList<FunderPid>();
+
+	@JsonIgnore
+	@Type(type = "string-array")
+	@Column(name = "datasources", columnDefinition = "text[]")
+	private String[] datasourcesPostgres;
+
+	@Transient
+	private List<FunderDatasource> datasources = new ArrayList<FunderDatasource>();
+
+	@Transient
+	private List<LocalDate> aggregationDates;
 
 	public String getId() {
 		return id;
@@ -42,75 +79,100 @@ public class FunderDbEntry {
 		this.id = id;
 	}
 
-	public String getName() {
-		return name;
+	public String getLegalShortName() {
+		return legalShortName;
 	}
 
-	public void setName(final String name) {
-		this.name = name;
+	public void setLegalShortName(final String legalShortName) {
+		this.legalShortName = legalShortName;
 	}
 
-	public String getShortname() {
-		return shortname;
+	public String getLegalName() {
+		return legalName;
 	}
 
-	public void setShortname(final String shortname) {
-		this.shortname = shortname;
+	public void setLegalName(final String legalName) {
+		this.legalName = legalName;
 	}
 
-	public String getJurisdiction() {
-		return jurisdiction;
+	public String getWebsiteUrl() {
+		return websiteUrl;
 	}
 
-	public void setJurisdiction(final String jurisdiction) {
-		this.jurisdiction = jurisdiction;
+	public void setWebsiteUrl(final String websiteUrl) {
+		this.websiteUrl = websiteUrl;
 	}
 
-	public String getWebsiteurl() {
-		return websiteurl;
+	public String getLogoUrl() {
+		return logoUrl;
 	}
 
-	public void setWebsiteurl(final String websiteurl) {
-		this.websiteurl = websiteurl;
+	public void setLogoUrl(final String logoUrl) {
+		this.logoUrl = logoUrl;
 	}
 
-	public String getPolicy() {
-		return policy;
+	public String getCountry() {
+		return country;
 	}
 
-	public void setPolicy(final String policy) {
-		this.policy = policy;
+	public void setCountry(final String country) {
+		this.country = country;
 	}
 
-	public Date getRegistrationdate() {
-		return registrationdate;
+	public LocalDate getRegistrationDate() {
+		return registrationDate;
 	}
 
-	public void setRegistrationdate(final Date registrationdate) {
-		this.registrationdate = registrationdate;
+	public void setRegistrationDate(final LocalDate registrationDate) {
+		this.registrationDate = registrationDate;
 	}
 
-	public Date getLastupdatedate() {
-		return lastupdatedate;
+	public Boolean getRegistered() {
+		return registered;
 	}
 
-	public void setLastupdatedate(final Date lastupdatedate) {
-		this.lastupdatedate = lastupdatedate;
+	public void setRegistered(final Boolean registered) {
+		this.registered = registered;
 	}
 
-	public Set<IdentityDbEntry> getPids() {
+	public String[] getPidsPostgres() {
+		return pidsPostgres;
+	}
+
+	public void setPidsPostgres(final String[] pidsPostgres) {
+		this.pidsPostgres = pidsPostgres;
+	}
+
+	public List<FunderPid> getPids() {
 		return pids;
 	}
 
-	public void setPids(final Set<IdentityDbEntry> pids) {
+	public void setPids(final List<FunderPid> pids) {
 		this.pids = pids;
 	}
 
-	public Set<FundingPathDbEntry> getFundingpaths() {
-		return fundingpaths;
+	public String[] getDatasourcesPostgres() {
+		return datasourcesPostgres;
 	}
 
-	public void setFundingpaths(final Set<FundingPathDbEntry> fundingpaths) {
-		this.fundingpaths = fundingpaths;
+	public void setDatasourcesPostgres(final String[] datasourcesPostgres) {
+		this.datasourcesPostgres = datasourcesPostgres;
 	}
+
+	public List<FunderDatasource> getDatasources() {
+		return datasources;
+	}
+
+	public void setDatasources(final List<FunderDatasource> datasources) {
+		this.datasources = datasources;
+	}
+
+	public List<LocalDate> getAggregationDates() {
+		return aggregationDates;
+	}
+
+	public void setAggregationDates(final List<LocalDate> aggregationDates) {
+		this.aggregationDates = aggregationDates;
+	}
+
 }

apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/domain/db/FunderPid.java

@@ -0,0 +1,29 @@
+package eu.dnetlib.openaire.funders.domain.db;
+
+import java.io.Serializable;
+
+public class FunderPid implements Serializable {
+
+	private static final long serialVersionUID = 2145493560459874509L;
+
+	private String type;
+
+	private String value;
+
+	public String getType() {
+		return type;
+	}
+
+	public void setType(final String type) {
+		this.type = type;
+	}
+
+	public String getValue() {
+		return value;
+	}
+
+	public void setValue(final String value) {
+		this.value = value;
+	}
+
+}

apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/domain/db/OldFunderDbEntry.java

@@ -0,0 +1,118 @@
+package eu.dnetlib.openaire.funders.domain.db;
+
+import java.sql.Date;
+import java.util.Set;
+
+import javax.persistence.CascadeType;
+import javax.persistence.FetchType;
+import javax.persistence.Id;
+import javax.persistence.JoinColumn;
+import javax.persistence.JoinTable;
+import javax.persistence.OneToMany;
+
+import eu.dnetlib.openaire.dsm.domain.db.IdentityDbEntry;
+
+// @Entity
+// @Table(name = "funders")
+@Deprecated
+public class OldFunderDbEntry {
+
+	@Id
+	private String id;
+	private String name;
+	private String shortname;
+	private String jurisdiction;
+	private String websiteurl;
+	private String policy;
+	private Date registrationdate;
+	private Date lastupdatedate;
+
+	@OneToMany(cascade = { CascadeType.PERSIST, CascadeType.MERGE }, fetch = FetchType.LAZY)
+	@JoinTable(name = "funder_identity", joinColumns = @JoinColumn(name = "funder"), inverseJoinColumns = @JoinColumn(name = "pid"))
+	private Set<IdentityDbEntry> pids;
+
+	@OneToMany(mappedBy = "funderid", cascade = CascadeType.ALL, fetch = FetchType.LAZY)
+	private Set<OldFundingPathDbEntry> fundingpaths;
+
+	public OldFunderDbEntry() {}
+
+	public String getId() {
+		return id;
+	}
+
+	public void setId(final String id) {
+		this.id = id;
+	}
+
+	public String getName() {
+		return name;
+	}
+
+	public void setName(final String name) {
+		this.name = name;
+	}
+
+	public String getShortname() {
+		return shortname;
+	}
+
+	public void setShortname(final String shortname) {
+		this.shortname = shortname;
+	}
+
+	public String getJurisdiction() {
+		return jurisdiction;
+	}
+
+	public void setJurisdiction(final String jurisdiction) {
+		this.jurisdiction = jurisdiction;
+	}
+
+	public String getWebsiteurl() {
+		return websiteurl;
+	}
+
+	public void setWebsiteurl(final String websiteurl) {
+		this.websiteurl = websiteurl;
+	}
+
+	public String getPolicy() {
+		return policy;
+	}
+
+	public void setPolicy(final String policy) {
+		this.policy = policy;
+	}
+
+	public Date getRegistrationdate() {
+		return registrationdate;
+	}
+
+	public void setRegistrationdate(final Date registrationdate) {
+		this.registrationdate = registrationdate;
+	}
+
+	public Date getLastupdatedate() {
+		return lastupdatedate;
+	}
+
+	public void setLastupdatedate(final Date lastupdatedate) {
+		this.lastupdatedate = lastupdatedate;
+	}
+
+	public Set<IdentityDbEntry> getPids() {
+		return pids;
+	}
+
+	public void setPids(final Set<IdentityDbEntry> pids) {
+		this.pids = pids;
+	}
+
+	public Set<OldFundingPathDbEntry> getFundingpaths() {
+		return fundingpaths;
+	}
+
+	public void setFundingpaths(final Set<OldFundingPathDbEntry> fundingpaths) {
+		this.fundingpaths = fundingpaths;
+	}
+}

apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/domain/db/OldFundingPathDbEntry.java

@@ -1,12 +1,15 @@
 package eu.dnetlib.openaire.funders.domain.db;
 
-import javax.persistence.*;
+import javax.persistence.Column;
+import javax.persistence.Id;
+import javax.persistence.Transient;
 
 import com.fasterxml.jackson.annotation.JsonIgnore;
 
-@Entity
-@Table(name = "fundingpaths")
-public class FundingPathDbEntry {
+// @Entity
+// @Table(name = "fundingpaths")
+@Deprecated
+public class OldFundingPathDbEntry {
 
 	@JsonIgnore
 	@Transient
@@ -28,7 +31,7 @@ public class FundingPathDbEntry {
 
 	private String funderid;
 
-	public FundingPathDbEntry() {}
+	public OldFundingPathDbEntry() {}
 
 	public String getDnetresourceidentifier() {
 		return dnetresourceidentifier;

apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/info/JdbcInfoDao.java

@@ -8,6 +8,7 @@ public interface JdbcInfoDao {
 		claim_load_date,
 		oaf_load_date,
 		odf_load_date,
+		db_load_date,
 		inference_date,
 		stats_update_date,
 		crossref_update_date,

apps/dnet-exporter-api/src/main/resources/application.properties

@@ -25,8 +25,9 @@ management.endpoints.web.path-mapping.health        =   health
 # ENABLE / DISABLE CONTROLLERS
 openaire.exporter.enable.dsm            =   true
 openaire.exporter.enable.community      =   true
+openaire.exporter.enable.community.import =   false
 openaire.exporter.enable.context        =   true
-openaire.exporter.enable.funders        =   false
+openaire.exporter.enable.funders        =   true
 openaire.exporter.enable.project        =   true
 openaire.exporter.enable.info           =   true
 
@@ -35,3 +36,4 @@ openaire.exporter.cache.ttl             =   43200000
 
 maven.pom.path = /META-INF/maven/eu.dnetlib.dhp/dnet-exporter-api/effective-pom.xml
 
+openaire.exporter.funders.cron = 0 30 2 * * *

apps/dnet-exporter-api/src/main/resources/global.properties

@@ -1,12 +1,8 @@
-#services.is.host                    =   localhost
-services.is.host                    =   dev-openaire.d4science.org
-#services.is.port                    =   8280
-services.is.port                    =   443
-#services.is.protocol                =   http
-services.is.protocol                =   https
-#services.is.context                 =   app
-services.is.context                 =   is
-#services.is.baseurl                 =   ${services.is.protocol}://${services.is.host}:${services.is.port}/${services.is.context}/services
+services.is.protocol                =   http
+services.is.host                    =   localhost
+services.is.port                    =   8280
+services.is.context                 =   app
+
 services.is.baseurl                 =   ${services.is.protocol}://${services.is.host}:${services.is.port}/${services.is.context}/services
 
 openaire.exporter.isLookupUrl           =   ${services.is.baseurl}/isLookUp
@@ -20,8 +16,8 @@ openaire.exporter.cxfClientConnectTimeout    =   60000
 openaire.exporter.cxfClientReceiveTimeout    =   120000
 
 # JDBC
-#openaire.exporter.jdbc.url          =   jdbc:postgresql://localhost:5432/dnet_openaire
-openaire.exporter.jdbc.url          =   jdbc:postgresql://localhost:5432/dev_openaire_8280
+openaire.exporter.jdbc.url          =   jdbc:postgresql://localhost:5432/dnet_openaireplus
+#openaire.exporter.jdbc.url          =   jdbc:postgresql://localhost:5432/dev_openaire_8280
 openaire.exporter.jdbc.user         =   dnetapi
 openaire.exporter.jdbc.pwd          =   dnetPwd
 openaire.exporter.jdbc.minIdle      =   1
@@ -51,10 +47,9 @@ openaire.exporter.findObjectStore           =   /eu/dnetlib/openaire/xquery/find
 openaire.exporter.findFunderContexts        =   /eu/dnetlib/openaire/xquery/findFunderContexts.xquery
 openaire.exporter.findCommunityContexts     =   /eu/dnetlib/openaire/xquery/findCommunityContexts.xquery
 openaire.exporter.findContextProfiles       =   /eu/dnetlib/openaire/xquery/findContextProfiles.xquery
+openaire.exporter.findContextProfilesByType =   /eu/dnetlib/openaire/xquery/findContextProfilesByType.xquery
 openaire.exporter.getRepoProfile            =   /eu/dnetlib/openaire/xquery/getRepoProfile.xquery
 
-openaire.exporter.contentLoadQuery          =   { "$and" : [ { "system:profileName" : "Graph construction [PROD]" }, { "system:isCompletedSuccessfully" : "true" }, { "reuseContent" : "false" } ] }
-
 # REST API CONFIGURATION
 openaire.exporter.swaggerDsm.apiTitle          =   OpenAIRE aggregator REST API
 openaire.exporter.swaggerDsm.apiDescription    =   The OpenAIRE data provision REST API allows developers to access the metadata information space of OpenAIRE programmatically.
@@ -107,4 +102,4 @@ openaire.exporter.swaggerInfo.apiContactEmail   =   ${openaire.exporter.swaggerD
 # VOCABULARIES
 openaire.exporter.vocabularies.baseUrl                      = http://localhost:8980/provision/mvc/vocabularies
 openaire.exporter.vocabularies.countriesEndpoint            = ${openaire.exporter.vocabularies.baseUrl}/dnet:countries.json
-openaire.exporter.vocabularies.datasourceTypologiesEndpoint = ${openaire.exporter.vocabularies.baseUrl}/dnet:datasource_typologies.json
+openaire.exporter.vocabularies.datasourceTypologiesEndpoint = ${openaire.exporter.vocabularies.baseUrl}/dnet:eosc_datasource_types.json

apps/dnet-exporter-api/src/main/resources/global.properties.locale

@@ -1,7 +1,12 @@
-services.is.host                    =   localhost
-services.is.port                    =   8280
-services.is.protocol                =   http
-services.is.context                 =   app
+#services.is.host                    =   localhost
+services.is.host                    =   dev-openaire.d4science.org
+#services.is.port                    =   8280
+services.is.port                    =   443
+#services.is.protocol                =   http
+services.is.protocol                =   https
+#services.is.context                 =   app
+services.is.context                 =   is
+#services.is.baseurl                 =   ${services.is.protocol}://${services.is.host}:${services.is.port}/${services.is.context}/services
 services.is.baseurl                 =   ${services.is.protocol}://${services.is.host}:${services.is.port}/${services.is.context}/services
 
 openaire.exporter.isLookupUrl           =   ${services.is.baseurl}/isLookUp
@@ -15,7 +20,7 @@ openaire.exporter.cxfClientConnectTimeout    =   60000
 openaire.exporter.cxfClientReceiveTimeout    =   120000
 
 # JDBC
-#openaire.exporter.jdbc.url          =   jdbc:postgresql://localhost:5432/dnet_openaireplus
+#openaire.exporter.jdbc.url          =   jdbc:postgresql://localhost:5432/dnet_openaire
 openaire.exporter.jdbc.url          =   jdbc:postgresql://localhost:5432/dev_openaire_8280
 openaire.exporter.jdbc.user         =   dnetapi
 openaire.exporter.jdbc.pwd          =   dnetPwd
@@ -49,6 +54,8 @@ openaire.exporter.findContextProfiles       =   /eu/dnetlib/openaire/xquery/find
 openaire.exporter.findContextProfilesByType =   /eu/dnetlib/openaire/xquery/findContextProfilesByType.xquery
 openaire.exporter.getRepoProfile            =   /eu/dnetlib/openaire/xquery/getRepoProfile.xquery
 
+openaire.exporter.contentLoadQuery          =   { "$and" : [ { "system:profileName" : "Graph construction [PROD]" }, { "system:isCompletedSuccessfully" : "true" }, { "reuseContent" : "false" } ] }
+
 # REST API CONFIGURATION
 openaire.exporter.swaggerDsm.apiTitle          =   OpenAIRE aggregator REST API
 openaire.exporter.swaggerDsm.apiDescription    =   The OpenAIRE data provision REST API allows developers to access the metadata information space of OpenAIRE programmatically.
@@ -101,4 +108,4 @@ openaire.exporter.swaggerInfo.apiContactEmail   =   ${openaire.exporter.swaggerD
 # VOCABULARIES
 openaire.exporter.vocabularies.baseUrl                      = http://localhost:8980/provision/mvc/vocabularies
 openaire.exporter.vocabularies.countriesEndpoint            = ${openaire.exporter.vocabularies.baseUrl}/dnet:countries.json
-openaire.exporter.vocabularies.datasourceTypologiesEndpoint = ${openaire.exporter.vocabularies.baseUrl}/dnet:eosc_datasource_types.json
+openaire.exporter.vocabularies.datasourceTypologiesEndpoint = ${openaire.exporter.vocabularies.baseUrl}/dnet:datasource_typologies.json

apps/dnet-exporter-api/src/main/resources/sql/community-schema.sql

@@ -0,0 +1,87 @@
+DROP TABLE IF EXISTS community_subs;
+DROP TABLE IF EXISTS community_projects;
+DROP TABLE IF EXISTS community_datasources;
+DROP TABLE IF EXISTS community_support_orgs;
+DROP TABLE IF EXISTS community_orgs;
+DROP TABLE IF EXISTS communities;
+
+CREATE TABLE communities (
+	id                       text PRIMARY KEY,
+	name                     text NOT NULL,
+	shortname                text NOT NULL, -- in the profile is label
+	displayname              text,
+	displayshortname         text,
+	description              text NOT NULL DEFAULT '',
+	status                   text NOT NULL DEFAULT 'hidden', -- all, manager, hidden, members
+	membership               text NOT NULL DEFAULT 'by-invitation', -- open, by-invitation
+	type                     text NOT NULL, -- community, ri
+	claim                    text, -- managers-only, members-only, all
+	subjects                 text[],
+	fos                      text[],
+	sdg                      text[],
+	adv_constraints          jsonb,
+	remove_constraints       jsonb,
+	main_zenodo_community    text,
+	other_zenodo_communities text[],
+	creation_date            timestamp NOT NULL DEFAULT now(),
+	last_update              timestamp NOT NULL DEFAULT now(),
+	logo_url                 text,
+	suggested_acknowledgements text[],
+	plan                     text NOT NULL DEFAULT 'Default'
+);
+
+CREATE TABLE community_projects (
+	community       text NOT NULL REFERENCES communities(id),
+	project_id      text NOT NULL,
+	project_code    text NOT NULL,
+	project_name    text NOT NULL,
+	project_acronym text,
+	project_funder  text NOT NULL,
+	available_since date NOT NULL default now(),
+	PRIMARY KEY (community, project_id)
+);
+
+CREATE TABLE community_datasources (
+	community       text NOT NULL REFERENCES communities(id),
+	ds_id           text NOT NULL,
+	ds_name         text NOT NULL,
+	ds_officialname text NOT NULL,
+	enabled         boolean NOT NULL DEFAULT true
+	constraints     jsonb,
+	deposit         boolean NOT NULL DEFAULT false,
+	message         text,
+	PRIMARY KEY (community, ds_id)
+);
+
+CREATE TABLE community_support_orgs (
+	community     text NOT NULL REFERENCES communities(id),
+	org_name      text NOT NULL,
+	org_url       text NOT NULL,
+	org_logourl   text NOT NULL,
+	PRIMARY KEY (community, org_name)
+);
+
+CREATE TABLE community_orgs (
+	community   text NOT NULL REFERENCES communities(id),
+	org_id      text NOT NULL,
+	PRIMARY KEY (community, org_id)
+);
+
+
+CREATE TABLE community_subs (
+	sub_id      text    NOT NULL PRIMARY KEY,
+	community   text    NOT NULL REFERENCES communities(id),
+	label       text    NOT NULL,
+	category    text    NOT NULL,
+	claim       boolean NOT NULL DEFAULT false,
+	browsable   boolean NOT NULL DEFAULT false,
+	params      jsonb,
+	parent      text REFERENCES community_subs(sub_id) -- NULL for the first level
+);
+
+CREATE INDEX community_projects_community ON community_projects(community);
+CREATE INDEX community_datasources_community ON community_datasources(community);
+CREATE INDEX community_support_orgs_community ON community_support_orgs(community);
+CREATE INDEX community_orgs_community ON community_orgs(community);
+CREATE INDEX community_subs_community ON community_subs(community);
+CREATE INDEX community_subs_parent ON community_subs(parent);

apps/dnet-exporter-api/src/main/resources/sql/export_nwo_projects.sql

@@ -0,0 +1,11 @@
+\copy (
+	SELECT 
+		'netherlands'                         AS community,
+		'nwo_________::'||md5(substr(id, 15)) AS project_id,
+		code                                  AS project_code,
+		title                                 AS project_name,
+		acronym                               AS project_acronym,
+		'NWO'                                 AS project_funder
+	FROM projects
+	WHERE id LIKE 'nwo\_\_\_\_\_\_\_\_\_::%'
+) TO '/tmp/nwo_community_projects.csv' CSV HEADER;

apps/dnet-exporter-api/src/main/resources/sql/funders-schema.sql

@@ -0,0 +1,25 @@
+ALTER TABLE dsm_organizations ADD COLUMN registered_funder boolean;
+
+CREATE VIEW funders_view AS SELECT
+	o.id                AS id, 
+	o.legalshortname    AS legalshortname,
+	o.legalname         AS legalname,
+	o.websiteurl        AS websiteurl, 
+	o.logourl           AS logourl, 
+	o.country           AS country,
+	o.dateofcollection  AS registrationdate,
+	o.registered_funder AS registered,
+	array_agg(DISTINCT s.id||' @=@ '||s.officialname||' @=@ '||s.eosc_datasource_type) AS datasources,
+	array_agg(DISTINCT pids.issuertype||' @=@ '||pids.pid) AS pids
+FROM 
+	dsm_organizations o 
+	JOIN dsm_service_organization so           ON (o.id = so.organization) 
+	JOIN dsm_services s                        ON (so.service = s.id) 
+	JOIN projects p                            ON (p.collectedfrom = s.id)
+	LEFT OUTER JOIN dsm_organizationpids opids ON (o.id = opids.organization)
+	LEFT OUTER JOIN dsm_identities pids        ON (opids.pid = pids.pid)
+GROUP BY o.id;
+
+GRANT ALL ON funders_view TO dnetapi;
+
+

apps/dnet-exporter-api/src/main/resources/sql/import_nwo_projects.sql

@@ -0,0 +1 @@
+\copy community_projects FROM '/tmp/nwo_community_projects.csv' CSV HEADER;

apps/dnet-exporter-api/src/test/java/eu/dnetlib/openaire/community/CommunityApiControllerTest.java

@@ -8,7 +8,7 @@ import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.
 import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status;
 
 import java.nio.charset.Charset;
-import java.util.Date;
+import java.time.LocalDateTime;
 import java.util.List;
 
 import org.junit.jupiter.api.Test;
@@ -26,7 +26,7 @@ import eu.dnetlib.openaire.exporter.model.community.CommunitySummary;
  *
  * @author Alessia Bardi
  */
-@SpringBootTest
+//@SpringBootTest
 @WebMvcTest(CommunityApiController.class)
 public class CommunityApiControllerTest {
 
@@ -46,7 +46,7 @@ public class CommunityApiControllerTest {
 		final CommunitySummary cs = new CommunitySummary();
 		cs.setDescription("the description");
 		cs.setId("id1");
-		cs.setLastUpdateDate(new Date());
+		cs.setLastUpdateDate(LocalDateTime.now());
 		cs.setName("X");
 		cs.setShortName("x");
 		final List<CommunitySummary> csList = singletonList(cs);

apps/dnet-exporter-api/src/test/java/eu/dnetlib/openaire/community/importer/CommunityImporterServiceTest.java

@@ -0,0 +1,161 @@
+package eu.dnetlib.openaire.community.importer;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.nio.charset.StandardCharsets;
+import java.util.Arrays;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Queue;
+
+import org.apache.commons.io.IOUtils;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.mockito.ArgumentCaptor;
+import org.mockito.Mock;
+import org.mockito.Mockito;
+import org.mockito.junit.jupiter.MockitoExtension;
+import org.springframework.jdbc.core.JdbcTemplate;
+
+import eu.dnetlib.openaire.community.CommunityService;
+import eu.dnetlib.openaire.community.model.DbOrganization;
+import eu.dnetlib.openaire.community.repository.DbOrganizationRepository;
+import eu.dnetlib.openaire.context.ContextMappingUtils;
+import eu.dnetlib.openaire.exporter.model.community.CommunityContentprovider;
+import eu.dnetlib.openaire.exporter.model.community.CommunityDetails;
+import eu.dnetlib.openaire.exporter.model.community.CommunityOrganization;
+import eu.dnetlib.openaire.exporter.model.community.CommunityProject;
+import eu.dnetlib.openaire.exporter.model.community.SubCommunity;
+import eu.dnetlib.openaire.exporter.model.context.Context;
+
+@ExtendWith(MockitoExtension.class)
+class CommunityImporterServiceTest {
+
+	// Class under test
+	private CommunityImporterService importer;
+
+	@Mock
+	private DbOrganizationRepository dbOrganizationRepository;
+
+	@Mock
+	private CommunityService service;
+
+	@Mock
+	private JdbcTemplate jdbcTemplate;
+
+	@BeforeEach
+	public void setUp() {
+		importer = new CommunityImporterService();
+		importer.setDbOrganizationRepository(dbOrganizationRepository);
+		importer.setService(service);
+		importer.setJdbcTemplate(jdbcTemplate);
+	}
+
+	@Test
+	public void testImportPropagationOrganizationsFromProfile() throws Exception {
+		final String profile = IOUtils.toString(getClass().getResourceAsStream("old_provision_wf.xml"), StandardCharsets.UTF_8.toString());
+		final List<DbOrganization> list = importer.importPropagationOrganizationsFromProfile(profile, true);
+		// list.forEach(System.out::println);
+
+		assertEquals(245, list.size());
+		assertEquals(1, list.stream().filter(o -> "openorgs____::9dd5545aacd3d8019e00c3f837269746".equals(o.getOrgId())).count());
+		assertEquals(2, list.stream().filter(o -> "openorgs____::d11f981828c485cd23d93f7f24f24db1".equals(o.getOrgId())).count());
+		assertEquals(14, list.stream().filter(o -> "beopen".equals(o.getCommunity())).count());
+	}
+
+	@SuppressWarnings("unchecked")
+	@Test
+	public void testImportCommunity() throws Exception {
+		final String profile = IOUtils.toString(getClass().getResourceAsStream("old_community_profile.xml"), StandardCharsets.UTF_8.toString());
+
+		final Queue<Throwable> errors = new LinkedList<>();
+		final Context context = ContextMappingUtils.parseContext(profile, errors);
+		assertTrue(errors.isEmpty());
+
+		Mockito.when(jdbcTemplate.queryForList(Mockito.anyString(), Mockito.any(Class.class), Mockito.anyString())).thenReturn(Arrays.asList("corda_______"));
+		importer.importCommunity(context);
+
+		final ArgumentCaptor<CommunityDetails> detailsCapture = ArgumentCaptor.forClass(CommunityDetails.class);
+		final ArgumentCaptor<CommunityProject> projectsCapture = ArgumentCaptor.forClass(CommunityProject.class);
+		final ArgumentCaptor<CommunityContentprovider> datasourcesCapture = ArgumentCaptor.forClass(CommunityContentprovider.class);
+		final ArgumentCaptor<CommunityOrganization> orgsCapture = ArgumentCaptor.forClass(CommunityOrganization.class);
+		final ArgumentCaptor<SubCommunity> subCommunitiesCapture = ArgumentCaptor.forClass(SubCommunity.class);
+
+		Mockito.verify(service, Mockito.times(1)).saveCommunity(detailsCapture.capture());
+		Mockito.verify(service, Mockito.times(1)).addCommunityProjects(Mockito.anyString(), projectsCapture.capture());
+		Mockito.verify(service, Mockito.times(1)).addCommunityDatasources(Mockito.anyString(), datasourcesCapture.capture());
+		Mockito.verify(service, Mockito.times(1)).addCommunityOrganizations(Mockito.anyString(), orgsCapture.capture());
+		Mockito.verify(service, Mockito.times(1)).addSubCommunities(Mockito.anyString(), subCommunitiesCapture.capture());
+
+		final CommunityDetails details = detailsCapture.getValue();
+		assertEquals("egi", details.getId());
+		// System.out.println(details);
+
+		final List<CommunityProject> projects = projectsCapture.getAllValues();
+		assertEquals(83, projects.size());
+		// projects.forEach(System.out::println);
+
+		final List<CommunityContentprovider> datasources = datasourcesCapture.getAllValues();
+		assertEquals(1, datasources.size());
+		// datasources.forEach(System.out::println);
+
+		final List<CommunityOrganization> orgs = orgsCapture.getAllValues();
+		assertEquals(1, orgs.size());
+		// orgs.forEach(System.out::println);
+
+		final List<SubCommunity> subs = subCommunitiesCapture.getAllValues();
+		assertEquals(688, subs.size());
+		// subs.forEach(System.out::println);
+
+	}
+
+	@SuppressWarnings("unchecked")
+	@Test
+	public void testImportCommunityFetFp7() throws Exception {
+		final String profile = IOUtils.toString(getClass().getResourceAsStream("old_community_profile_fet-fp7.xml"), StandardCharsets.UTF_8.toString());
+
+		final Queue<Throwable> errors = new LinkedList<>();
+		final Context context = ContextMappingUtils.parseContext(profile, errors);
+		assertTrue(errors.isEmpty());
+
+		// Mockito.when(jdbcTemplate.queryForList(Mockito.anyString(), Mockito.any(Class.class),
+		// Mockito.anyString())).thenReturn(Arrays.asList("corda_______"));
+		importer.importCommunity(context);
+
+		final ArgumentCaptor<CommunityDetails> detailsCapture = ArgumentCaptor.forClass(CommunityDetails.class);
+		final ArgumentCaptor<CommunityProject> projectsCapture = ArgumentCaptor.forClass(CommunityProject.class);
+		final ArgumentCaptor<CommunityContentprovider> datasourcesCapture = ArgumentCaptor.forClass(CommunityContentprovider.class);
+		final ArgumentCaptor<CommunityOrganization> orgsCapture = ArgumentCaptor.forClass(CommunityOrganization.class);
+		final ArgumentCaptor<SubCommunity> subCommunitiesCapture = ArgumentCaptor.forClass(SubCommunity.class);
+
+		Mockito.verify(service, Mockito.times(1)).saveCommunity(detailsCapture.capture());
+		Mockito.verify(service, Mockito.times(1)).addCommunityProjects(Mockito.anyString(), projectsCapture.capture());
+		Mockito.verify(service, Mockito.times(1)).addCommunityDatasources(Mockito.anyString(), datasourcesCapture.capture());
+		Mockito.verify(service, Mockito.times(1)).addCommunityOrganizations(Mockito.anyString(), orgsCapture.capture());
+		Mockito.verify(service, Mockito.times(1)).addSubCommunities(Mockito.anyString(), subCommunitiesCapture.capture());
+
+		final CommunityDetails details = detailsCapture.getValue();
+		assertEquals("fet-fp7", details.getId());
+		// System.out.println(details);
+
+		final List<CommunityProject> projects = projectsCapture.getAllValues();
+		assertEquals(0, projects.size());
+		// projects.forEach(System.out::println);
+
+		final List<CommunityContentprovider> datasources = datasourcesCapture.getAllValues();
+		assertEquals(0, datasources.size());
+		// datasources.forEach(System.out::println);
+
+		final List<CommunityOrganization> orgs = orgsCapture.getAllValues();
+		assertEquals(0, orgs.size());
+		// orgs.forEach(System.out::println);
+
+		final List<SubCommunity> subs = subCommunitiesCapture.getAllValues();
+		assertEquals(151, subs.size());
+		subs.forEach(System.out::println);
+
+	}
+
+}

apps/dnet-exporter-api/src/test/java/eu/dnetlib/openaire/dsm/DatasourceApiControllerTest.java

@@ -25,7 +25,6 @@ import eu.dnetlib.openaire.dsm.domain.db.DatasourceDbEntry;
 import eu.dnetlib.openaire.exporter.model.dsm.DatasourceDetails;
 import eu.dnetlib.openaire.exporter.model.vocabularies.Country;
 
-@SpringBootTest
 @WebMvcTest(DsmApiController.class)
 public class DatasourceApiControllerTest {
 

apps/dnet-exporter-api/src/test/java/eu/dnetlib/openaire/funders/FunderContextClientTest.java

@@ -20,11 +20,11 @@ import eu.dnetlib.openaire.exporter.model.funders.FunderDetails;
 
 public class FunderContextClientTest {
 
-	private FunderDao fDao;
+	private FunderService fDao;
 
 	@BeforeEach
 	public void setUp() {
-		fDao = new FunderDao();
+		fDao = new FunderService();
 	}
 
 	@Test

apps/dnet-exporter-api/src/test/java/eu/dnetlib/openaire/info/InfoControllerTest.java

@@ -7,6 +7,7 @@ import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.
 import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status;
 
 import java.nio.charset.Charset;
+import java.nio.charset.StandardCharsets;
 import java.time.LocalDate;
 import java.time.format.DateTimeFormatter;
 import java.util.Arrays;
@@ -14,22 +15,23 @@ import java.util.stream.Collectors;
 
 import org.assertj.core.util.Maps;
 import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Disabled;
 import org.junit.jupiter.api.Test;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.boot.test.autoconfigure.web.servlet.WebMvcTest;
-import org.springframework.boot.test.context.SpringBootTest;
 import org.springframework.boot.test.mock.mockito.MockBean;
 import org.springframework.http.MediaType;
 import org.springframework.test.web.servlet.MockMvc;
 
-@SpringBootTest
+
+@Disabled
 @WebMvcTest(InfoController.class)
 public class InfoControllerTest {
 
 	public static final MediaType APPLICATION_JSON_UTF8 = new MediaType(
 		MediaType.APPLICATION_JSON.getType(),
 		MediaType.APPLICATION_JSON.getSubtype(),
-		Charset.forName("utf8"));
+			StandardCharsets.UTF_8);
 
 	@Autowired
 	private MockMvc mvc;
@@ -37,18 +39,18 @@ public class InfoControllerTest {
 	@MockBean
 	private InfoController infoController;
 
-	private LocalDate expectedDate;
 	private String formattedDate;
 
 	@BeforeEach
 	public void setup() {
-		expectedDate = LocalDate.now();
+		LocalDate expectedDate = LocalDate.now();
 		formattedDate = expectedDate.format(DateTimeFormatter.ofPattern("yyyy-MM-dd"));
 		given(infoController.getDate(JdbcInfoDao.DATE_INFO.claim_load_date.name())).willReturn(expectedDate);
 		given(infoController.getDate(JdbcInfoDao.DATE_INFO.oaf_load_date.name())).willReturn(expectedDate);
 		given(infoController.getDate(JdbcInfoDao.DATE_INFO.odf_load_date.name())).willReturn(expectedDate);
 		given(infoController.getDate(JdbcInfoDao.DATE_INFO.inference_date.name())).willReturn(expectedDate);
 		given(infoController.getDate(JdbcInfoDao.DATE_INFO.stats_update_date.name())).willReturn(expectedDate);
+		given(infoController.getDate(JdbcInfoDao.DATE_INFO.db_load_date.name())).willReturn(expectedDate);
 		given(infoController.listInfo()).willReturn(Maps.newHashMap(JdbcInfoDao.DATE_INFO.inference_date.name(), LocalDate.now()));
 		given(infoController.listInfoKeys())
 			.willReturn(Arrays.stream(JdbcInfoDao.DATE_INFO.values()).map(JdbcInfoDao.DATE_INFO::name).collect(Collectors.toList()));
@@ -116,4 +118,12 @@ public class InfoControllerTest {
 			.getContentAsString());
 	}
 
+	@Test
+	public void testGetDbLoadDate() throws Exception {
+		mvc.perform(get("/info/db_load_date")
+						.contentType(APPLICATION_JSON_UTF8))
+				.andExpect(status().isOk())
+				.andExpect(jsonPath("$", is(formattedDate)));
+	}
+
 }

apps/dnet-orgs-database-application/pom.xml

@@ -4,7 +4,7 @@
 	<parent>
 		<groupId>eu.dnetlib.dhp</groupId>
 		<artifactId>apps</artifactId>
-		<version>3.4.3-SNAPSHOT</version>
+		<version>3.5.5-SNAPSHOT</version>
 		<relativePath>../</relativePath>
 	</parent>
 

apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/MainApplication.java

@@ -23,9 +23,9 @@ public class MainApplication extends AbstractDnetApp {
 	@Bean
 	public GroupedOpenApi publicApi() {
 		return GroupedOpenApi.builder()
-			.group("D-Net Organizations Service APIs")
-			.pathsToMatch("/api/**", "/oa_api/**")
-			.build();
+				.group("D-Net Organizations Service APIs")
+				.pathsToMatch("/api/**", "/oa_api/**", "/public-api/**")
+				.build();
 	}
 
 	@Override

apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/MockSecurityConfig.java

@@ -44,28 +44,28 @@ public class MockSecurityConfig extends WebSecurityConfigurerAdapter {
 		http.headers().frameOptions().sameOrigin();
 
 		http.csrf()
-			.disable()
-			.authorizeRequests()
-			.antMatchers("/", "/api/**")
-			.hasAnyRole(OpenOrgsConstants.VALID_ROLES)
-			.antMatchers("/registration_api/**")
-			.hasRole(OpenOrgsConstants.NOT_AUTORIZED_ROLE)
-			.antMatchers("/common/**", "/resources/**", "/webjars/**", "/metrics", "/health", "/kpis", "/dbmodel/**")
-			.permitAll()
-			.antMatchers("/oa_api/**")
-			.permitAll()
-			.anyRequest()
-			.authenticated()
-			.and()
-			.formLogin()
-			.loginPage("/login")
-			.permitAll()
-			.and()
-			.logout()
-			.permitAll()
-			.and()
-			.exceptionHandling()
-			.accessDeniedHandler(accessDeniedHandler());
+				.disable()
+				.authorizeRequests()
+				.antMatchers("/", "/api/**")
+				.hasAnyRole(OpenOrgsConstants.VALID_ROLES)
+				.antMatchers("/registration_api/**")
+				.hasRole(OpenOrgsConstants.NOT_AUTORIZED_ROLE)
+				.antMatchers("/common/**", "/resources/**", "/webjars/**", "/metrics", "/health", "/kpis", "/dbmodel/**", "/public-api/**")
+				.permitAll()
+				.antMatchers("/oa_api/**")
+				.permitAll()
+				.anyRequest()
+				.authenticated()
+				.and()
+				.formLogin()
+				.loginPage("/login")
+				.permitAll()
+				.and()
+				.logout()
+				.permitAll()
+				.and()
+				.exceptionHandling()
+				.accessDeniedHandler(accessDeniedHandler());
 	}
 
 	private AccessDeniedHandler accessDeniedHandler() {
@@ -74,7 +74,8 @@ public class MockSecurityConfig extends WebSecurityConfigurerAdapter {
 
 			if (auth != null) {
 				logger
-					.warn(String.format("User '%s' (%s) attempted to access the protected URL: %s", auth.getName(), req.getRemoteAddr(), req.getRequestURI()));
+						.warn(String
+								.format("User '%s' (%s) attempted to access the protected URL: %s", auth.getName(), req.getRemoteAddr(), req.getRequestURI()));
 			}
 
 			if (UserInfo.isNotAuthorized(auth)) {
@@ -89,12 +90,12 @@ public class MockSecurityConfig extends WebSecurityConfigurerAdapter {
 	@Autowired
 	public void configureGlobal(final AuthenticationManagerBuilder auth) throws Exception {
 		auth.jdbcAuthentication()
-			.dataSource(dataSource)
-			.usersByUsernameQuery("select ?, '{MD5}" + DigestUtils.md5Hex(DEFAULT_PASSWORD) + "', true")
-			.authoritiesByUsernameQuery("with const as (SELECT ? as email) "
-				+ "select c.email, 'ROLE_" + OpenOrgsConstants.OPENORGS_ROLE_PREFIX + "'||coalesce(u.role, '"
-				+ UserRole.NOT_AUTHORIZED
-				+ "') from const c left outer join users u on (u.email = c.email)");
+				.dataSource(dataSource)
+				.usersByUsernameQuery("select ?, '{MD5}" + DigestUtils.md5Hex(DEFAULT_PASSWORD) + "', true")
+				.authoritiesByUsernameQuery("with const as (SELECT ? as email) "
+						+ "select c.email, 'ROLE_" + OpenOrgsConstants.OPENORGS_ROLE_PREFIX + "'||coalesce(u.role, '"
+						+ UserRole.NOT_AUTHORIZED
+						+ "') from const c left outer join users u on (u.email = c.email)");
 	}
 
 	@Bean

apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/OAuth2WebSecurityConfig.java

@@ -57,29 +57,29 @@ public class OAuth2WebSecurityConfig extends WebSecurityConfigurerAdapter {
 		http.headers().frameOptions().sameOrigin();
 
 		http.csrf()
-			.disable()
-			.authorizeRequests()
-			.antMatchers("/main", "/api/**")
-			.hasAnyRole(OpenOrgsConstants.VALID_ROLES)
-			.antMatchers("/registration_api/**")
-			.hasRole(OpenOrgsConstants.NOT_AUTORIZED_ROLE)
-			.antMatchers("/", "/common/**", "/resources/**", "/webjars/**", "/metrics", "/health", "/kpis", "/dbmodel/**")
-			.permitAll()
-			.antMatchers("/oa_api/**")
-			.hasIpAddress(openaireApiValidSubnet)
-			.anyRequest()
-			.authenticated()
-			.and()
-			.exceptionHandling()
-			.accessDeniedHandler(accessDeniedHandler())
-			.and()
-			.logout()
-			.logoutSuccessHandler(oidcLogoutSuccessHandler())
-			.invalidateHttpSession(true)
-			.clearAuthentication(true)
-			.deleteCookies("JSESSIONID")
-			.and()
-			.oauth2Login(oauth2 -> oauth2.userInfoEndpoint(userInfo -> userInfo.oidcUserService(this.oidcUserService())));
+				.disable()
+				.authorizeRequests()
+				.antMatchers("/main", "/api/**")
+				.hasAnyRole(OpenOrgsConstants.VALID_ROLES)
+				.antMatchers("/registration_api/**")
+				.hasRole(OpenOrgsConstants.NOT_AUTORIZED_ROLE)
+				.antMatchers("/", "/common/**", "/resources/**", "/webjars/**", "/metrics", "/health", "/kpis", "/dbmodel/**", "/public-api/**")
+				.permitAll()
+				.antMatchers("/oa_api/**")
+				.hasIpAddress(openaireApiValidSubnet)
+				.anyRequest()
+				.authenticated()
+				.and()
+				.exceptionHandling()
+				.accessDeniedHandler(accessDeniedHandler())
+				.and()
+				.logout()
+				.logoutSuccessHandler(oidcLogoutSuccessHandler())
+				.invalidateHttpSession(true)
+				.clearAuthentication(true)
+				.deleteCookies("JSESSIONID")
+				.and()
+				.oauth2Login(oauth2 -> oauth2.userInfoEndpoint(userInfo -> userInfo.oidcUserService(oidcUserService())));
 	}
 
 	private AccessDeniedHandler accessDeniedHandler() {
@@ -87,8 +87,8 @@ public class OAuth2WebSecurityConfig extends WebSecurityConfigurerAdapter {
 			final Authentication authentication = SecurityContextHolder.getContext().getAuthentication();
 			if (authentication != null) {
 				log.warn(String
-					.format("User '%s' (%s) attempted to access the protected URL: %s", UserInfo.getEmail(authentication), req
-						.getRemoteAddr(), req.getRequestURI()));
+						.format("User '%s' (%s) attempted to access the protected URL: %s", UserInfo.getEmail(authentication), req
+								.getRemoteAddr(), req.getRequestURI()));
 			}
 
 			if (UserInfo.isNotAuthorized(authentication)) {
@@ -122,7 +122,7 @@ public class OAuth2WebSecurityConfig extends WebSecurityConfigurerAdapter {
 	private OAuth2UserService<OidcUserRequest, OidcUser> oidcUserService() {
 		final OidcUserService delegate = new OidcUserService();
 
-		return (userRequest) -> {
+		return userRequest -> {
 			final OidcUser oidcUser = delegate.loadUser(userRequest);
 
 			log.debug("User attributes:");
@@ -137,9 +137,9 @@ public class OAuth2WebSecurityConfig extends WebSecurityConfigurerAdapter {
 			}
 
 			final String role = "ROLE_" + OpenOrgsConstants.OPENORGS_ROLE_PREFIX + user
-				.map(User::getRole)
-				.filter(StringUtils::isNotBlank)
-				.orElse(UserRole.NOT_AUTHORIZED.toString());
+					.map(User::getRole)
+					.filter(StringUtils::isNotBlank)
+					.orElse(UserRole.NOT_AUTHORIZED.toString());
 
 			final Set<GrantedAuthority> mappedAuthorities = new HashSet<>();
 			mappedAuthorities.add(new SimpleGrantedAuthority(role));

apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/controller/OrganizationController.java

@@ -86,20 +86,16 @@ public class OrganizationController extends AbstractDnetController {
 	@PostMapping("/save")
 	public List<String> save(@RequestBody final OrganizationView org, final Authentication authentication) {
 
-		if (StringUtils.isBlank(org.getName())) {
-			throw new RuntimeException("Missing field: name");
-		} else if (StringUtils.isBlank(org.getCountry())) {
-			throw new RuntimeException("Missing field: country");
-		} else if (StringUtils.isBlank(org.getType())) {
-			throw new RuntimeException("Missing field: type");
-		} else if (UserInfo.isSuperAdmin(authentication)
-			|| userCountryRepository.verifyAuthorizationForCountry(org.getCountry(), UserInfo.getEmail(authentication))) {
-				final String orgId =
+		if (StringUtils.isBlank(org.getName())) { throw new RuntimeException("Missing field: name"); }
+		if (StringUtils.isBlank(org.getCountry())) { throw new RuntimeException("Missing field: country"); }
+		if (StringUtils.isBlank(org.getType())) { throw new RuntimeException("Missing field: type"); }
+		if (UserInfo.isSuperAdmin(authentication)
+				|| userCountryRepository.verifyAuthorizationForCountry(org.getCountry(), UserInfo.getEmail(authentication))) {
+			final String orgId =
 					databaseUtils.insertOrUpdateOrganization(org, UserInfo.getEmail(authentication), UserInfo.isSimpleUser(authentication));
-				return Arrays.asList(orgId);
-			} else {
-				throw new RuntimeException("User not authorized");
-			}
+			return Arrays.asList(orgId);
+		}
+		throw new RuntimeException("User not authorized");
 	}
 
 	@GetMapping("/info")
@@ -115,11 +111,11 @@ public class OrganizationController extends AbstractDnetController {
 			suggestionInfoViewByCountryRepository.findAll().forEach(info::add);
 		} else if (UserInfo.isSimpleUser(authentication) || UserInfo.isNationalAdmin(authentication)) {
 			userCountryRepository.getCountriesForUser(UserInfo.getEmail(authentication))
-				.stream()
-				.map(suggestionInfoViewByCountryRepository::findById)
-				.filter(Optional::isPresent)
-				.map(Optional::get)
-				.forEach(info::add);
+					.stream()
+					.map(suggestionInfoViewByCountryRepository::findById)
+					.filter(Optional::isPresent)
+					.map(Optional::get)
+					.forEach(info::add);
 		}
 		return info;
 	}
@@ -129,29 +125,26 @@ public class OrganizationController extends AbstractDnetController {
 		final OrganizationView org = organizationViewRepository.findById(id).get();
 
 		if (UserInfo.isSuperAdmin(authentication)
-			|| userCountryRepository.verifyAuthorizationForCountry(org.getCountry(), UserInfo.getEmail(authentication))) {
+				|| userCountryRepository.verifyAuthorizationForCountry(org.getCountry(), UserInfo.getEmail(authentication))) {
 			return org;
-		} else {
-			throw new RuntimeException("User not authorized");
 		}
+		throw new RuntimeException("User not authorized");
 	}
 
 	@GetMapping("/conflicts")
 	public List<OrganizationConflict> conflicts(@RequestParam final String id, final Authentication authentication) {
 		if (UserInfo.isSuperAdmin(authentication) || userCountryRepository.verifyAuthorizationForId(id, UserInfo.getEmail(authentication))) {
 			return databaseUtils.listConflictsForId(id);
-		} else {
-			throw new RuntimeException("User not authorized");
 		}
+		throw new RuntimeException("User not authorized");
 	}
 
 	@GetMapping("/duplicates")
 	public List<OpenaireDuplicateView> duplicates(@RequestParam final String id, final Authentication authentication) {
 		if (UserInfo.isSuperAdmin(authentication) || userCountryRepository.verifyAuthorizationForId(id, UserInfo.getEmail(authentication))) {
 			return listDuplicates(id);
-		} else {
-			throw new RuntimeException("User not authorized");
 		}
+		throw new RuntimeException("User not authorized");
 	}
 
 	private List<OpenaireDuplicateView> listDuplicates(final String id) {
@@ -161,43 +154,42 @@ public class OrganizationController extends AbstractDnetController {
 	@GetMapping("/conflicts/byCountry/{country}")
 	public Collection<Set<OrganizationConflict>> findConflictsByCountry(@PathVariable final String country, final Authentication authentication) {
 
-		if (UserInfo.isSuperAdmin(authentication)) {
-			return groupConflicts(conflictGroupViewRepository.findByCountry1OrCountry2(country, country).stream());
-		} else if (UserInfo.isSimpleUser(authentication) || UserInfo.isNationalAdmin(authentication)) {
+		if (UserInfo.isSuperAdmin(authentication)) { return groupConflicts(conflictGroupViewRepository.findByCountry1OrCountry2(country, country).stream()); }
+		if (UserInfo.isSimpleUser(authentication) || UserInfo.isNationalAdmin(authentication)) {
 			final Stream<ConflictGroupView> list = userCountryRepository.getCountriesForUser(UserInfo.getEmail(authentication))
-				.stream()
-				.filter(country::equalsIgnoreCase)
-				.map(c -> conflictGroupViewRepository.findByCountry1OrCountry2(c, c).stream())
-				.findFirst()
-				.orElse(Stream.empty());
+					.stream()
+					.filter(country::equalsIgnoreCase)
+					.map(c -> conflictGroupViewRepository.findByCountry1OrCountry2(c, c).stream())
+					.findFirst()
+					.orElse(Stream.empty());
 			return groupConflicts(list);
-		} else {
-			throw new RuntimeException("User not authorized");
 		}
+		throw new RuntimeException("User not authorized");
 
 	}
 
 	@GetMapping("/duplicates/byCountry/{country}")
-	public Iterable<DuplicateGroupView> findDuplicatesByCountry(@PathVariable final String country, final Authentication authentication) {
+	public Iterable<DuplicateGroupView> findDuplicatesByCountry(@PathVariable final String country,
+			@RequestParam(required = false, defaultValue = "0") final int page,
+			@RequestParam(required = false, defaultValue = "${openorgs.findDuplicatesByCountry.limit.default}") final int size,
+			final Authentication authentication) {
 
-		if (UserInfo.isSuperAdmin(authentication)) {
-			return duplicateGroupViewRepository.findByCountry(country);
-		} else if (UserInfo.isSimpleUser(authentication) || UserInfo.isNationalAdmin(authentication)) {
+		if (UserInfo.isSuperAdmin(authentication)) { return duplicateGroupViewRepository.findByCountry(country, PageRequest.of(page, size)); }
+		if (UserInfo.isSimpleUser(authentication) || UserInfo.isNationalAdmin(authentication)) {
 			return userCountryRepository.getCountriesForUser(UserInfo.getEmail(authentication))
-				.stream()
-				.filter(country::equalsIgnoreCase)
-				.map(duplicateGroupViewRepository::findByCountry)
-				.findFirst()
-				.orElse(new ArrayList<DuplicateGroupView>());
-		} else {
-			throw new RuntimeException("User not authorized");
+					.stream()
+					.filter(country::equalsIgnoreCase)
+					.map(c -> duplicateGroupViewRepository.findByCountry(c, PageRequest.of(page, size)))
+					.findFirst()
+					.orElse(new ArrayList<DuplicateGroupView>());
 		}
+		throw new RuntimeException("User not authorized");
 	}
 
 	@GetMapping(value = "/duplicates/byCountry/{country}/csv", produces = "text/csv")
 	public void findDuplicatesByCountryCSV(@PathVariable final String country, final HttpServletResponse res, final Authentication authentication)
-		throws IOException {
-		final Iterable<DuplicateGroupView> list = findDuplicatesByCountry(country, authentication);
+			throws IOException {
+		final Iterable<DuplicateGroupView> list = findDuplicatesByCountry(country, 0, Integer.MAX_VALUE, authentication);
 		CSVConverter.writeCSV(res.getOutputStream(), list, DuplicateGroupView.class, "id", "name", "city", "country", "numberOfDuplicates");
 	}
 
@@ -219,148 +211,131 @@ public class OrganizationController extends AbstractDnetController {
 		if (simrels.isEmpty()) { return new ArrayList<>(); }
 
 		final boolean b = UserInfo.isSuperAdmin(authentication)
-			|| simrels.stream()
-				.map(OpenaireDuplicate::getLocalId)
-				.distinct()
-				.allMatch(id -> userCountryRepository.verifyAuthorizationForId(id, UserInfo.getEmail(authentication)));
+				|| simrels.stream()
+						.map(OpenaireDuplicate::getLocalId)
+						.distinct()
+						.allMatch(id -> userCountryRepository.verifyAuthorizationForId(id, UserInfo.getEmail(authentication)));
 
 		if (b) {
 			databaseUtils.saveDuplicates(simrels, UserInfo.getEmail(authentication));
 			return listDuplicates(simrels.get(0).getLocalId());
-		} else {
-			throw new RuntimeException("User not authorized");
 		}
+		throw new RuntimeException("User not authorized");
 	}
 
 	@GetMapping("/search/{page}/{size}")
 	public Page<OrganizationSimpleView> search(@PathVariable final int page,
-		@PathVariable final int size,
-		@RequestParam final String q,
-		@RequestParam(required = false, defaultValue = "") final String status,
-		final Authentication authentication) {
+			@PathVariable final int size,
+			@RequestParam final String q,
+			@RequestParam(required = false, defaultValue = "") final String status,
+			final Authentication authentication) {
 
-		if (status.equals(SPECIAL_STATUS_FOR_CANDIDATE_DUP)) {
+		if (SPECIAL_STATUS_FOR_CANDIDATE_DUP.equals(status)) {
 			return UserInfo.isSuperAdmin(authentication)
-				? organizationSimpleViewRepository.searchCandidateDuplicates(q, PageRequest.of(page, size))
-				: organizationSimpleViewRepository.searchCandidateDuplicatesForUser(q, UserInfo.getEmail(authentication), PageRequest.of(page, size));
+					? organizationSimpleViewRepository.searchCandidateDuplicates(q, PageRequest.of(page, size))
+					: organizationSimpleViewRepository.searchCandidateDuplicatesForUser(q, UserInfo.getEmail(authentication), PageRequest.of(page, size));
+		}
+		final List<String> statuses;
+		if (StringUtils.isNotBlank(status)) {
+			statuses = Arrays.asList(status.split(","));
+		} else if (UserInfo.isSimpleUser(authentication)) {
+			statuses = Arrays.asList(OrganizationStatus.approved.toString());
 		} else {
-			final List<String> statuses;
-			if (StringUtils.isNotBlank(status)) {
-				statuses = Arrays.asList(status.split(","));
-			} else if (UserInfo.isSimpleUser(authentication)) {
-				statuses = Arrays.asList(OrganizationStatus.approved.toString());
-			} else {
-				statuses = Arrays.asList(OrganizationStatus.approved.toString(), OrganizationStatus.suggested.toString());
-			}
+			statuses = Arrays.asList(OrganizationStatus.approved.toString(), OrganizationStatus.suggested.toString());
+		}
 
-			return UserInfo.isSuperAdmin(authentication)
+		return UserInfo.isSuperAdmin(authentication)
 				? organizationSimpleViewRepository.search(q, statuses, PageRequest.of(page, size))
 				: organizationSimpleViewRepository.searchForUser(q, UserInfo.getEmail(authentication), statuses, PageRequest.of(page, size));
-		}
 	}
 
 	@GetMapping("/byCountry/{status}/{code}/{page}/{size}")
 	public Page<OrganizationSimpleView> findByCountry(@PathVariable final String status,
-		@PathVariable final String code,
-		@PathVariable final int page,
-		@PathVariable final int size,
-		final Authentication authentication) {
-		if (UserInfo.isSuperAdmin(authentication)
-			|| userCountryRepository.verifyAuthorizationForCountry(code, UserInfo.getEmail(authentication))) {
-			if (status.equalsIgnoreCase("all")) {
-				return organizationSimpleViewRepository.findByCountryOrderByName(code, PageRequest.of(page, size));
-			} else {
-				return organizationSimpleViewRepository.findByCountryAndStatusOrderByName(code, status, PageRequest.of(page, size));
-			}
-		} else {
+			@PathVariable final String code,
+			@PathVariable final int page,
+			@PathVariable final int size,
+			final Authentication authentication) {
+		if (!UserInfo.isSuperAdmin(authentication) && !userCountryRepository.verifyAuthorizationForCountry(code, UserInfo.getEmail(authentication))) {
 			throw new RuntimeException("User not authorized");
 		}
+		if ("all".equalsIgnoreCase(status)) { return organizationSimpleViewRepository.findByCountryOrderByName(code, PageRequest.of(page, size)); }
+		return organizationSimpleViewRepository.findByCountryAndStatusOrderByName(code, status, PageRequest.of(page, size));
 	}
 
 	@GetMapping("/byCountry/{status}/{code}")
-	public Iterable<OrganizationSimpleView> findOrgsByStatusAndCountry(@PathVariable final String status,
-		@PathVariable final String code,
-		final Authentication authentication) {
-		if (UserInfo.isSuperAdmin(authentication)
-			|| userCountryRepository.verifyAuthorizationForCountry(code, UserInfo.getEmail(authentication))) {
-			if (status.equalsIgnoreCase("all")) {
-				return organizationSimpleViewRepository.findByCountryOrderByName(code);
-			} else {
-				return organizationSimpleViewRepository.findByCountryAndStatusOrderByName(code, status);
-			}
-		} else {
+	public List<OrganizationSimpleView> findOrgsByStatusAndCountry(@PathVariable final String status,
+			@PathVariable final String code,
+			@RequestParam(required = false, defaultValue = "0") final int page,
+			@RequestParam(required = false, defaultValue = "${openorgs.findOrgsByStatusAndCountry.limit.default}") final int size,
+			final Authentication authentication) {
+		if (!UserInfo.isSuperAdmin(authentication) && !userCountryRepository.verifyAuthorizationForCountry(code, UserInfo.getEmail(authentication))) {
 			throw new RuntimeException("User not authorized");
 		}
+		if ("all".equalsIgnoreCase(status)) { return organizationSimpleViewRepository.findByCountryOrderByName(code, PageRequest.of(page, size)).getContent(); }
+		return organizationSimpleViewRepository.findByCountryAndStatusOrderByName(code, status, PageRequest.of(page, size)).getContent();
 	}
 
 	@GetMapping(value = "/byCountry/{status}/{code}/csv", produces = "text/csv")
 	public void findOrgsByStatusAndCountryCSV(@PathVariable final String status,
-		@PathVariable final String code,
-		final HttpServletResponse res,
-		final Authentication authentication) throws IOException {
-		final Iterable<OrganizationSimpleView> list = findOrgsByStatusAndCountry(status, code, authentication);
+			@PathVariable final String code,
+			final HttpServletResponse res,
+			final Authentication authentication) throws IOException {
+		final Iterable<OrganizationSimpleView> list = findOrgsByStatusAndCountry(status, code, 0, Integer.MAX_VALUE, authentication);
 		CSVConverter.writeCSV(res
-			.getOutputStream(), list, OrganizationSimpleView.class, "id", "name", "type", "city", "country", "acronyms", "urls", "status", "nSimilarDups", "nSuggestedDups", "nDifferentDups");
+				.getOutputStream(), list, OrganizationSimpleView.class, "id", "name", "type", "city", "country", "acronyms", "urls", "status", "nSimilarDups", "nSuggestedDups", "nDifferentDups");
 	}
 
 	@GetMapping("/byType/{status}/{type}/{page}/{size}")
 	public Page<OrganizationSimpleView> findByType(@PathVariable final String status,
-		@PathVariable final String type,
-		@PathVariable final int page,
-		@PathVariable final int size,
-		final Authentication authentication) {
+			@PathVariable final String type,
+			@PathVariable final int page,
+			@PathVariable final int size,
+			final Authentication authentication) {
 
 		if (UserInfo.isSuperAdmin(authentication)) {
-			if (status.equalsIgnoreCase("all")) {
-				return organizationSimpleViewRepository.findByTypeOrderByName(type, PageRequest.of(page, size));
-			} else {
-				return organizationSimpleViewRepository.findByTypeAndStatusOrderByName(type, status, PageRequest.of(page, size));
-			}
-		} else {
-			if (status.equalsIgnoreCase("all")) {
-				return organizationSimpleViewRepository.findByTypeForUser(type, UserInfo.getEmail(authentication), PageRequest.of(page, size));
-			} else {
-				return organizationSimpleViewRepository
-					.findByTypeAndStatusForUser(type, status, UserInfo.getEmail(authentication), PageRequest.of(page, size));
-			}
+			if ("all".equalsIgnoreCase(status)) { return organizationSimpleViewRepository.findByTypeOrderByName(type, PageRequest.of(page, size)); }
+			return organizationSimpleViewRepository.findByTypeAndStatusOrderByName(type, status, PageRequest.of(page, size));
 		}
+		if ("all".equalsIgnoreCase(status)) {
+			return organizationSimpleViewRepository.findByTypeForUser(type, UserInfo.getEmail(authentication), PageRequest.of(page, size));
+		}
+		return organizationSimpleViewRepository
+				.findByTypeAndStatusForUser(type, status, UserInfo.getEmail(authentication), PageRequest.of(page, size));
 
 	}
 
 	@GetMapping("/browse/countries")
 	public List<BrowseEntry> browseCountries(final Authentication authentication) {
 		return UserInfo.isSuperAdmin(authentication)
-			? databaseUtils.browseCountries()
-			: databaseUtils.browseCountriesForUser(UserInfo.getEmail(authentication));
+				? databaseUtils.browseCountries()
+				: databaseUtils.browseCountriesForUser(UserInfo.getEmail(authentication));
 	}
 
 	@GetMapping("/browse/types")
 	public List<BrowseEntry> browseOrganizationTypes(final Authentication authentication) {
 		return UserInfo.isSuperAdmin(authentication)
-			? databaseUtils.browseTypes()
-			: databaseUtils.browseTypesForUser(UserInfo.getEmail(authentication));
+				? databaseUtils.browseTypes()
+				: databaseUtils.browseTypesForUser(UserInfo.getEmail(authentication));
 	}
 
 	@PostMapping("/conflicts/fix/similar")
 	public List<String> fixConflictSim(final Authentication authentication, @RequestBody final List<String> ids) {
 		if (ids.size() > 1 && UserInfo.isSuperAdmin(authentication)
-			|| userCountryRepository.verifyAuthorizationForId(ids.get(0), UserInfo.getEmail(authentication))) {
+				|| userCountryRepository.verifyAuthorizationForId(ids.get(0), UserInfo.getEmail(authentication))) {
 			final String newOrgId = databaseUtils.fixConflictSimilars(ids, UserInfo.getEmail(authentication));
 			return Arrays.asList(newOrgId);
-		} else {
-			return new ArrayList<>();
 		}
+		return new ArrayList<>();
 	}
 
 	@PostMapping("/conflicts/fix/different")
 	public List<String> fixConflictDiff(final Authentication authentication, @RequestBody final List<String> ids) {
 		if (ids.size() > 1 && UserInfo.isSuperAdmin(authentication)
-			|| userCountryRepository.verifyAuthorizationForId(ids.get(0), UserInfo.getEmail(authentication))) {
+				|| userCountryRepository.verifyAuthorizationForId(ids.get(0), UserInfo.getEmail(authentication))) {
 			databaseUtils.fixConflictDifferents(ids, UserInfo.getEmail(authentication));
 			return ids;
-		} else {
-			return new ArrayList<>();
 		}
+		return new ArrayList<>();
 	}
 
 	@GetMapping("/note")
@@ -368,11 +343,10 @@ public class OrganizationController extends AbstractDnetController {
 		final OrganizationView org = organizationViewRepository.findById(id).get();
 
 		if (UserInfo.isSuperAdmin(authentication)
-			|| userCountryRepository.verifyAuthorizationForCountry(org.getCountry(), UserInfo.getEmail(authentication))) {
+				|| userCountryRepository.verifyAuthorizationForCountry(org.getCountry(), UserInfo.getEmail(authentication))) {
 			return noteRepository.findById(id).orElse(new Note(id, "", null, null));
-		} else {
-			throw new RuntimeException("User not authorized");
 		}
+		throw new RuntimeException("User not authorized");
 	}
 
 	@PostMapping("/note")
@@ -381,20 +355,17 @@ public class OrganizationController extends AbstractDnetController {
 
 		final OrganizationView org = organizationViewRepository.findById(orgId).get();
 
-		if (UserInfo.isSuperAdmin(authentication)
-			|| userCountryRepository.verifyAuthorizationForCountry(org.getCountry(), UserInfo.getEmail(authentication))) {
-
-			if (StringUtils.isNotBlank(note.getNote())) {
-				note.setModifiedBy(UserInfo.getEmail(authentication));
-				note.setModificationDate(OffsetDateTime.now());
-				return noteRepository.save(note);
-			} else {
-				noteRepository.deleteById(orgId);
-				return new Note(orgId, "", null, null);
-			}
-		} else {
+		if (!UserInfo.isSuperAdmin(authentication)
+				&& !userCountryRepository.verifyAuthorizationForCountry(org.getCountry(), UserInfo.getEmail(authentication))) {
 			throw new RuntimeException("User not authorized");
 		}
+		if (StringUtils.isNotBlank(note.getNote())) {
+			note.setModifiedBy(UserInfo.getEmail(authentication));
+			note.setModificationDate(OffsetDateTime.now());
+			return noteRepository.save(note);
+		}
+		noteRepository.deleteById(orgId);
+		return new Note(orgId, "", null, null);
 	}
 
 	@GetMapping("/journal")
@@ -402,11 +373,10 @@ public class OrganizationController extends AbstractDnetController {
 		final OrganizationView org = organizationViewRepository.findById(id).get();
 
 		if (UserInfo.isSuperAdmin(authentication)
-			|| userCountryRepository.verifyAuthorizationForCountry(org.getCountry(), UserInfo.getEmail(authentication))) {
+				|| userCountryRepository.verifyAuthorizationForCountry(org.getCountry(), UserInfo.getEmail(authentication))) {
 			return journalEntryRepository.findByOrgIdOrderByDateDesc(id);
-		} else {
-			throw new RuntimeException("User not authorized");
 		}
+		throw new RuntimeException("User not authorized");
 	}
 
 }

apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/controller/PublicApiController.java

@@ -0,0 +1,69 @@
+package eu.dnetlib.organizations.controller;
+
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+
+import javax.servlet.ServletOutputStream;
+import javax.servlet.http.HttpServletResponse;
+
+import org.apache.commons.io.IOUtils;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.http.HttpStatus;
+import org.springframework.http.MediaType;
+import org.springframework.web.bind.annotation.GetMapping;
+import org.springframework.web.bind.annotation.PathVariable;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RequestParam;
+import org.springframework.web.bind.annotation.RestController;
+
+import eu.dnetlib.organizations.utils.DatabaseUtils;
+
+@RestController
+@RequestMapping("/public-api")
+public class PublicApiController {
+
+	@Autowired
+	private DatabaseUtils dbUtils;
+
+	@GetMapping("/logs")
+	public void findJournalById(@RequestParam final String id,
+			final HttpServletResponse res) throws IOException {
+
+		final String filename = String.format("%s.log", id.replaceAll(":", "_"));
+
+		res.setContentType(MediaType.TEXT_PLAIN.getType());
+		res.setHeader("Content-Disposition", "attachment; filename=" + filename);
+
+		try (final ServletOutputStream out = res.getOutputStream()) {
+			for (final String s : dbUtils.obtainLogEntries(id)) {
+				IOUtils.write(s, out, StandardCharsets.UTF_8);
+			}
+		}
+	}
+
+	@GetMapping("/logs/{year}/{month}/{country}")
+	public void findJournalByCountry(@PathVariable final int year,
+			@PathVariable final int month,
+			@PathVariable final String country,
+			final HttpServletResponse res) throws IOException {
+
+		if (month < 1 || month > 12) {
+			res.sendError(HttpStatus.BAD_REQUEST.value(), "Invalid month");
+		}
+		if (year < 2020 || year > 2100) {
+			res.sendError(HttpStatus.BAD_REQUEST.value(), "Invalid year");
+		}
+
+		final String filename = String.format("%04d_%02d_%s.log", year, month, country);
+
+		res.setContentType(MediaType.TEXT_PLAIN.getType());
+		res.setHeader("Content-Disposition", "attachment; filename=" + filename);
+
+		try (final ServletOutputStream out = res.getOutputStream()) {
+			for (final String s : dbUtils.obtainLogEntries(year, month, country)) {
+				IOUtils.write(s, out, StandardCharsets.UTF_8);
+			}
+		}
+	}
+
+}

apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/repository/readonly/DuplicateGroupViewRepository.java

@@ -2,6 +2,7 @@ package eu.dnetlib.organizations.repository.readonly;
 
 import java.util.List;
 
+import org.springframework.data.domain.Pageable;
 import org.springframework.stereotype.Repository;
 
 import eu.dnetlib.organizations.model.view.DuplicateGroupView;
@@ -9,5 +10,5 @@ import eu.dnetlib.organizations.model.view.DuplicateGroupView;
 @Repository
 public interface DuplicateGroupViewRepository extends ReadOnlyRepository<DuplicateGroupView, String> {
 
-	List<DuplicateGroupView> findByCountry(String country);
+	List<DuplicateGroupView> findByCountry(String country, Pageable page);
 }

apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/repository/readonly/OrganizationSimpleViewRepository.java

@@ -15,60 +15,56 @@ public interface OrganizationSimpleViewRepository extends ReadOnlyRepository<Org
 
 	// SEARCH
 	@Query(value = "SELECT \n"
-		+ "    org.id,\n"
-		+ "    org.name,\n"
-		+ "    org.type,\n"
-		+ "    org.city,\n"
-		+ "    org.country,\n"
-		+ "    org.status,\n"
-		+ "    array_remove(array_agg(DISTINCT a.acronym), NULL) AS acronyms,\n"
-		+ "    array_remove(array_agg(DISTINCT u.url), NULL) AS urls,\n"
-		+ "    count(DISTINCT d1.oa_original_id) FILTER (WHERE d1.reltype = 'is_similar') AS n_similar_dups,\n"
-		+ "    count(DISTINCT d1.oa_original_id) FILTER (WHERE d1.reltype = 'suggested') AS n_suggested_dups,\n"
-		+ "    count(DISTINCT d1.oa_original_id) FILTER (WHERE d1.reltype = 'is_different') AS n_different_dups\n"
-		+ "FROM org_index_search idx "
-		+ "     JOIN organizations org ON (idx.id = org.id) \n"
-		+ "     LEFT OUTER JOIN acronyms a ON org.id = a.id\n"
-		+ "     LEFT OUTER JOIN urls u ON org.id = u.id\n"
-		+ "     LEFT OUTER JOIN oa_duplicates d1 ON org.id = d1.local_id\n"
-		+ "WHERE org.status in :statuses AND (org.name ilike '%'||:text||'%' OR idx.txt @@ plainto_tsquery(:text))\n"
-		+ "GROUP BY org.id, org.name, org.type, org.city, org.country, org.status\n"
-		+ "ORDER BY org.name", nativeQuery = true)
+			+ "    org.id,\n"
+			+ "    org.name,\n"
+			+ "    org.type,\n"
+			+ "    org.city,\n"
+			+ "    org.country,\n"
+			+ "    org.status,\n"
+			+ "    array_remove(array_agg(DISTINCT a.acronym), NULL) AS acronyms,\n"
+			+ "    array_remove(array_agg(DISTINCT u.url), NULL) AS urls,\n"
+			+ "    count(DISTINCT d1.oa_original_id) FILTER (WHERE d1.reltype = 'is_similar') AS n_similar_dups,\n"
+			+ "    count(DISTINCT d1.oa_original_id) FILTER (WHERE d1.reltype = 'suggested') AS n_suggested_dups,\n"
+			+ "    count(DISTINCT d1.oa_original_id) FILTER (WHERE d1.reltype = 'is_different') AS n_different_dups\n"
+			+ "FROM org_index_search idx "
+			+ "     JOIN organizations org ON (idx.id = org.id) \n"
+			+ "     LEFT OUTER JOIN acronyms a ON org.id = a.id\n"
+			+ "     LEFT OUTER JOIN urls u ON org.id = u.id\n"
+			+ "     LEFT OUTER JOIN oa_duplicates d1 ON org.id = d1.local_id\n"
+			+ "WHERE org.status in :statuses AND (org.name ilike '%'||:text||'%' OR idx.txt @@ plainto_tsquery(:text))\n"
+			+ "GROUP BY org.id, org.name, org.type, org.city, org.country, org.status\n"
+			+ "ORDER BY org.name", nativeQuery = true)
 	Page<OrganizationSimpleView> search(@Param("text") String text, @Param("statuses") List<String> statuses, Pageable pageable);
 
 	// SEARCH FOR USER
 	@Query(value = "SELECT\n"
-		+ "    org.id,\n"
-		+ "    org.name,\n"
-		+ "    org.type,\n"
-		+ "    org.city,\n"
-		+ "    org.country,\n"
-		+ "    org.status,\n"
-		+ "    array_remove(array_agg(DISTINCT a.acronym), NULL) AS acronyms,\n"
-		+ "    array_remove(array_agg(DISTINCT u.url), NULL) AS urls,\n"
-		+ "    count(DISTINCT d1.oa_original_id) FILTER (WHERE d1.reltype = 'is_similar'  ) AS n_similar_dups,\n"
-		+ "    count(DISTINCT d1.oa_original_id) FILTER (WHERE d1.reltype = 'suggested'   ) AS n_suggested_dups,\n"
-		+ "    count(DISTINCT d1.oa_original_id) FILTER (WHERE d1.reltype = 'is_different') AS n_different_dups\n"
-		+ "FROM org_index_search idx\n"
-		+ "    JOIN organizations org ON (idx.id = org.id)\n"
-		+ "    LEFT OUTER JOIN acronyms a ON (org.id = a.id)\n"
-		+ "    LEFT OUTER JOIN urls u ON (org.id = u.id)\n"
-		+ "    LEFT OUTER JOIN oa_duplicates d1 ON (org.id = d1.local_id)\n"
-		+ "    LEFT OUTER JOIN user_countries uc ON (uc.country = org.country) \n"
-		+ "WHERE uc.email = :email AND org.status IN :statuses AND (org.name ilike '%'||:text||'%' OR idx.txt @@ plainto_tsquery(:text))\n"
-		+ "GROUP BY org.id, org.name, org.type, org.city, org.country, org.status\n"
-		+ "ORDER BY org.name", nativeQuery = true)
+			+ "    org.id,\n"
+			+ "    org.name,\n"
+			+ "    org.type,\n"
+			+ "    org.city,\n"
+			+ "    org.country,\n"
+			+ "    org.status,\n"
+			+ "    array_remove(array_agg(DISTINCT a.acronym), NULL) AS acronyms,\n"
+			+ "    array_remove(array_agg(DISTINCT u.url), NULL) AS urls,\n"
+			+ "    count(DISTINCT d1.oa_original_id) FILTER (WHERE d1.reltype = 'is_similar'  ) AS n_similar_dups,\n"
+			+ "    count(DISTINCT d1.oa_original_id) FILTER (WHERE d1.reltype = 'suggested'   ) AS n_suggested_dups,\n"
+			+ "    count(DISTINCT d1.oa_original_id) FILTER (WHERE d1.reltype = 'is_different') AS n_different_dups\n"
+			+ "FROM org_index_search idx\n"
+			+ "    JOIN organizations org ON (idx.id = org.id)\n"
+			+ "    LEFT OUTER JOIN acronyms a ON (org.id = a.id)\n"
+			+ "    LEFT OUTER JOIN urls u ON (org.id = u.id)\n"
+			+ "    LEFT OUTER JOIN oa_duplicates d1 ON (org.id = d1.local_id)\n"
+			+ "    LEFT OUTER JOIN user_countries uc ON (uc.country = org.country) \n"
+			+ "WHERE uc.email = :email AND org.status IN :statuses AND (org.name ilike '%'||:text||'%' OR idx.txt @@ plainto_tsquery(:text))\n"
+			+ "GROUP BY org.id, org.name, org.type, org.city, org.country, org.status\n"
+			+ "ORDER BY org.name", nativeQuery = true)
 	Page<OrganizationSimpleView> searchForUser(@Param("text") String text,
-		@Param("email") String email,
-		@Param("statuses") List<String> statuses,
-		Pageable pageable);
+			@Param("email") String email,
+			@Param("statuses") List<String> statuses,
+			Pageable pageable);
 
 	Page<OrganizationSimpleView> findByCountryOrderByName(String country, Pageable pageable);
 
-	Iterable<OrganizationSimpleView> findByCountryOrderByName(String code);
-
-	Iterable<OrganizationSimpleView> findByCountryAndStatusOrderByName(String code, String status);
-
 	Page<OrganizationSimpleView> findByCountryAndStatusOrderByName(String code, String status, Pageable pageable);
 
 	Page<OrganizationSimpleView> findByTypeOrderByName(String type, Pageable pageable);
@@ -83,51 +79,51 @@ public interface OrganizationSimpleViewRepository extends ReadOnlyRepository<Org
 
 	// SEARCH FOR VALID DUPLICATE CANDIDATES
 	@Query(value = "SELECT\n"
-		+ "    org.id,\n"
-		+ "    org.name,\n"
-		+ "    org.type,\n"
-		+ "    org.city,\n"
-		+ "    org.country,\n"
-		+ "    org.status,\n"
-		+ "    array_remove(array_agg(DISTINCT a.acronym), NULL) AS acronyms,\n"
-		+ "    array_remove(array_agg(DISTINCT u.url), NULL) AS urls,\n"
-		+ "    NULL AS n_similar_dups,\n"
-		+ "    NULL AS n_suggested_dups,\n"
-		+ "    NULL AS n_different_dups\n"
-		+ "FROM org_index_search idx\n"
-		+ "    JOIN organizations org ON (idx.id = org.id) \n"
-		+ "    LEFT OUTER JOIN acronyms a ON (org.id = a.id)\n"
-		+ "    LEFT OUTER JOIN urls u ON (org.id = u.id)\n"
-		+ "    LEFT OUTER JOIN oa_duplicates d2 ON (org.id = d2.oa_original_id)\n"
-		+ "WHERE org.status = 'raw' AND (org.name ilike '%'||:text||'%' OR idx.txt @@ plainto_tsquery(:text))\n"
-		+ "GROUP BY org.id, org.name, org.type, org.city, org.country, org.status\n"
-		+ "HAVING  not('is_similar' = ANY(array_agg(d2.reltype)))\n"
-		+ "ORDER BY org.name", nativeQuery = true)
+			+ "    org.id,\n"
+			+ "    org.name,\n"
+			+ "    org.type,\n"
+			+ "    org.city,\n"
+			+ "    org.country,\n"
+			+ "    org.status,\n"
+			+ "    array_remove(array_agg(DISTINCT a.acronym), NULL) AS acronyms,\n"
+			+ "    array_remove(array_agg(DISTINCT u.url), NULL) AS urls,\n"
+			+ "    NULL AS n_similar_dups,\n"
+			+ "    NULL AS n_suggested_dups,\n"
+			+ "    NULL AS n_different_dups\n"
+			+ "FROM org_index_search idx\n"
+			+ "    JOIN organizations org ON (idx.id = org.id) \n"
+			+ "    LEFT OUTER JOIN acronyms a ON (org.id = a.id)\n"
+			+ "    LEFT OUTER JOIN urls u ON (org.id = u.id)\n"
+			+ "    LEFT OUTER JOIN oa_duplicates d2 ON (org.id = d2.oa_original_id)\n"
+			+ "WHERE org.status = 'raw' AND (org.name ilike '%'||:text||'%' OR idx.txt @@ plainto_tsquery(:text))\n"
+			+ "GROUP BY org.id, org.name, org.type, org.city, org.country, org.status\n"
+			+ "HAVING  not('is_similar' = ANY(array_agg(d2.reltype)))\n"
+			+ "ORDER BY org.name", nativeQuery = true)
 	Page<OrganizationSimpleView> searchCandidateDuplicates(@Param("text") String text, Pageable pageable);
 
 	// SEARCH FOR VALID DUPLICATE CANDIDATES FOR USER
 	@Query(value = "SELECT\n"
-		+ "    org.id,\n"
-		+ "    org.name,\n"
-		+ "    org.type,\n"
-		+ "    org.city,\n"
-		+ "    org.country,\n"
-		+ "    org.status,\n"
-		+ "    array_remove(array_agg(DISTINCT a.acronym), NULL) AS acronyms,\n"
-		+ "    array_remove(array_agg(DISTINCT u.url), NULL) AS urls,\n"
-		+ "    NULL AS n_similar_dups,\n"
-		+ "    NULL AS n_suggested_dups,\n"
-		+ "    NULL AS n_different_dups\n"
-		+ "FROM org_index_search idx\n"
-		+ "    JOIN organizations org ON (idx.id = org.id) \n"
-		+ "    LEFT OUTER JOIN acronyms a ON (org.id = a.id)\n"
-		+ "    LEFT OUTER JOIN urls u ON (org.id = u.id)\n"
-		+ "    LEFT OUTER JOIN oa_duplicates d2 ON (org.id = d2.oa_original_id)\n"
-		+ "    LEFT OUTER JOIN user_countries uc ON (uc.country = org.country)\n"
-		+ "WHERE org.status = 'raw' AND uc.email = :email AND (org.name ilike '%'||:text||'%' OR idx.txt @@ plainto_tsquery(:text))\n"
-		+ "GROUP BY org.id, org.name, org.type, org.city, org.country, org.status\n"
-		+ "HAVING  not('is_similar' = ANY(array_agg(d2.reltype)))\n"
-		+ "ORDER BY org.name", nativeQuery = true)
+			+ "    org.id,\n"
+			+ "    org.name,\n"
+			+ "    org.type,\n"
+			+ "    org.city,\n"
+			+ "    org.country,\n"
+			+ "    org.status,\n"
+			+ "    array_remove(array_agg(DISTINCT a.acronym), NULL) AS acronyms,\n"
+			+ "    array_remove(array_agg(DISTINCT u.url), NULL) AS urls,\n"
+			+ "    NULL AS n_similar_dups,\n"
+			+ "    NULL AS n_suggested_dups,\n"
+			+ "    NULL AS n_different_dups\n"
+			+ "FROM org_index_search idx\n"
+			+ "    JOIN organizations org ON (idx.id = org.id) \n"
+			+ "    LEFT OUTER JOIN acronyms a ON (org.id = a.id)\n"
+			+ "    LEFT OUTER JOIN urls u ON (org.id = u.id)\n"
+			+ "    LEFT OUTER JOIN oa_duplicates d2 ON (org.id = d2.oa_original_id)\n"
+			+ "    LEFT OUTER JOIN user_countries uc ON (uc.country = org.country)\n"
+			+ "WHERE org.status = 'raw' AND uc.email = :email AND (org.name ilike '%'||:text||'%' OR idx.txt @@ plainto_tsquery(:text))\n"
+			+ "GROUP BY org.id, org.name, org.type, org.city, org.country, org.status\n"
+			+ "HAVING  not('is_similar' = ANY(array_agg(d2.reltype)))\n"
+			+ "ORDER BY org.name", nativeQuery = true)
 	Page<OrganizationSimpleView> searchCandidateDuplicatesForUser(@Param("text") String text, @Param("email") String email, Pageable pageable);
 
 }

apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/utils/DatabaseUtils.java

@@ -1,5 +1,6 @@
 package eu.dnetlib.organizations.utils;
 
+import java.io.StringWriter;
 import java.time.OffsetDateTime;
 import java.util.ArrayList;
 import java.util.Arrays;
@@ -104,12 +105,7 @@ public class DatabaseUtils {
 	private static final Log log = LogFactory.getLog(DatabaseUtils.class);
 
 	public enum VocabularyTable {
-		languages,
-		countries,
-		org_types,
-		id_types,
-		rel_types,
-		simrel_types
+		languages, countries, org_types, id_types, rel_types, simrel_types
 	}
 
 	@Transactional
@@ -118,8 +114,8 @@ public class DatabaseUtils {
 		final String oldId = StringUtils.isNotBlank(orgView.getId()) ? new String(orgView.getId()) : null;
 
 		final String oldStatus = oldId != null ? organizationRepository.findById(oldId)
-			.map(Organization::getStatus)
-			.orElse(null) : null;
+				.map(Organization::getStatus)
+				.orElse(null) : null;
 
 		final boolean alreadyApproved = StringUtils.equals(oldStatus, OrganizationStatus.approved.toString());
 
@@ -146,15 +142,15 @@ public class DatabaseUtils {
 		}
 
 		final Organization org = new Organization(orgView.getId(),
-			orgView.getName(),
-			orgView.getType(),
-			orgView.getLat(), orgView.getLng(),
-			orgView.getCity(), orgView.getCountry(),
-			newStatus,
-			orgView.getEcLegalBody(),
-			orgView.getEcLegalPerson(), orgView.getEcNonProfit(), orgView.getEcResearchOrganization(), orgView.getEcHigherEducation(),
-			orgView.getEcInternationalOrganizationEurInterests(), orgView.getEcInternationalOrganization(), orgView.getEcEnterprise(),
-			orgView.getEcSmeValidated(), orgView.getEcNutscode());
+				orgView.getName(),
+				orgView.getType(),
+				orgView.getLat(), orgView.getLng(),
+				orgView.getCity(), orgView.getCountry(),
+				newStatus,
+				orgView.getEcLegalBody(),
+				orgView.getEcLegalPerson(), orgView.getEcNonProfit(), orgView.getEcResearchOrganization(), orgView.getEcHigherEducation(),
+				orgView.getEcInternationalOrganizationEurInterests(), orgView.getEcInternationalOrganization(), orgView.getEcEnterprise(),
+				orgView.getEcSmeValidated(), orgView.getEcNutscode());
 
 		final String newId = organizationRepository.save(org).getId();
 
@@ -168,9 +164,9 @@ public class DatabaseUtils {
 			if (oldId != null) {
 
 				final List<OpenaireDuplicate> dups = openaireDuplicateRepository.findByLocalId(oldId)
-					.stream()
-					.map(d -> prepareNewDuplicate(newId, oldId, d))
-					.collect(Collectors.toList());
+						.stream()
+						.map(d -> prepareNewDuplicate(newId, oldId, d))
+						.collect(Collectors.toList());
 
 				openaireDuplicateRepository.saveAll(dups);
 
@@ -194,7 +190,7 @@ public class DatabaseUtils {
 			if (oldStatus == null) {
 				op = JournalOperations.NEW_SUGG_ORG;
 				message = "Created a new suggested org";
-			} else if (oldStatus != null) {
+			} else {
 				op = JournalOperations.EDIT_SUGG_ORG;
 				message = "Metadata updated";
 			}
@@ -227,9 +223,9 @@ public class DatabaseUtils {
 		d.setOaCollectedFrom(old.getOaCollectedFrom());
 
 		if (oldId != null
-			&& newId.startsWith(OpenOrgsConstants.OPENORGS_PREFIX)
-			&& oldId.startsWith(OpenOrgsConstants.OPENORGS_PENDING_PREFIX)
-			&& StringUtils.substringAfter(oldId, OpenOrgsConstants.OPENORGS_PENDING_PREFIX).equalsIgnoreCase(DigestUtils.md5Hex(d.getOaOriginalId()))) {
+				&& newId.startsWith(OpenOrgsConstants.OPENORGS_PREFIX)
+				&& oldId.startsWith(OpenOrgsConstants.OPENORGS_PENDING_PREFIX)
+				&& StringUtils.substringAfter(oldId, OpenOrgsConstants.OPENORGS_PENDING_PREFIX).equalsIgnoreCase(DigestUtils.md5Hex(d.getOaOriginalId()))) {
 			d.setRelType(SimilarityType.is_similar.toString());
 		} else {
 			d.setRelType(SimilarityType.suggested.toString());
@@ -276,16 +272,16 @@ public class DatabaseUtils {
 		// Updating journal
 		toSave.stream().collect(Collectors.groupingBy(OpenaireDuplicate::getLocalId)).forEach((id, list) -> {
 			final long sim = list.stream()
-				.filter(d -> d.getRelType().equals(SimilarityType.is_similar.toString()))
-				.count();
+					.filter(d -> d.getRelType().equals(SimilarityType.is_similar.toString()))
+					.count();
 
 			final long diff = list.stream()
-				.filter(d -> d.getRelType().equals(SimilarityType.is_different.toString()))
-				.count();
+					.filter(d -> d.getRelType().equals(SimilarityType.is_different.toString()))
+					.count();
 
 			final long sugg = list.stream()
-				.filter(d -> d.getRelType().equals(SimilarityType.suggested.toString()))
-				.count();
+					.filter(d -> d.getRelType().equals(SimilarityType.suggested.toString()))
+					.count();
 
 			final String message = String.format("Duplicates updated (%s similars, %s differents, %s suggested)", sim, diff, sugg);
 
@@ -300,7 +296,7 @@ public class DatabaseUtils {
 
 		// delete invalid pending orgs (without simrels)
 		final int n = jdbcTemplate
-			.update("delete from organizations where id in (select o.id from organizations o left outer join oa_duplicates d on (o.id = d.local_id) where o.status = 'suggested' and o.created_by = 'dedupWf' group by o.id having count(d.local_id) = 0)");
+				.update("delete from organizations where id in (select o.id from organizations o left outer join oa_duplicates d on (o.id = d.local_id) where o.status = 'suggested' and o.created_by = 'dedupWf' group by o.id having count(d.local_id) = 0)");
 
 		if (n > 0) {
 			log.info("Invalid pending orgs deleted: " + n);
@@ -335,12 +331,12 @@ public class DatabaseUtils {
 	@Cacheable("countries_for_user")
 	public List<VocabularyTerm> listCountriesForUser(final String name) {
 		final String sql =
-			"select uc.country as value, c.name as name from user_countries uc left outer join countries c on (c.val = uc.country) where uc.email = ? order by c.name";
+				"select uc.country as value, c.name as name from user_countries uc left outer join countries c on (c.val = uc.country) where uc.email = ? order by c.name";
 		return jdbcTemplate.query(sql, new BeanPropertyRowMapper<>(VocabularyTerm.class), name);
 	}
 
 	@CacheEvict(value = {
-		"vocs", "countries_for_user"
+			"vocs", "countries_for_user"
 	}, allEntries = true)
 	public void clearCache() {
 		log.info("All caches cleaned");
@@ -359,7 +355,7 @@ public class DatabaseUtils {
 		userCountryRepository.deleteByEmail(userView.getEmail());
 		if (userView.getCountries() != null) {
 			userCountryRepository
-				.saveAll(Arrays.stream(userView.getCountries()).map(c -> new UserCountry(userView.getEmail(), c)).collect(Collectors.toList()));
+					.saveAll(Arrays.stream(userView.getCountries()).map(c -> new UserCountry(userView.getEmail(), c)).collect(Collectors.toList()));
 		}
 	}
 
@@ -371,11 +367,11 @@ public class DatabaseUtils {
 
 	@Transactional
 	public void newUser(final String email,
-		final String fullname,
-		final String organization,
-		final String referencePerson,
-		final String requestMessage,
-		final List<String> countries) {
+			final String fullname,
+			final String organization,
+			final String referencePerson,
+			final String requestMessage,
+			final List<String> countries) {
 
 		final User user = new User();
 		user.setEmail(email);
@@ -405,32 +401,32 @@ public class DatabaseUtils {
 	// BROWSE BY COUNTRY
 	public List<BrowseEntry> browseCountries() {
 		final String sql =
-			"select o.country as code, c.name as name, o.status as group, count(o.status) as count from organizations o left outer join countries c on (o.country = c.val) group by o.country, c.name, o.status";
+				"select o.country as code, c.name as name, o.status as group, count(o.status) as count from organizations o left outer join countries c on (o.country = c.val) group by o.country, c.name, o.status";
 		return listBrowseEntries(sql);
 	}
 
 	// BROWSE BY COUNTRY FOR USER
 	public List<BrowseEntry> browseCountriesForUser(final String email) {
 		final String sql =
-			"select o.country as code, c.name as name, o.status as group, count(o.status) as count from user_countries uc left outer join organizations o on (uc.country = o.country) left outer join countries c on (o.country = c.val) where uc.email=? group by o.country, c.name, o.status";
+				"select o.country as code, c.name as name, o.status as group, count(o.status) as count from user_countries uc left outer join organizations o on (uc.country = o.country) left outer join countries c on (o.country = c.val) where uc.email=? group by o.country, c.name, o.status";
 		return listBrowseEntries(sql, email);
 	}
 
 	// BROWSE BY ORG TYPE
 	public List<BrowseEntry> browseTypes() {
 		final String sql =
-			"select type as code, type as name, status as group, count(status) as count from organizations group by type, status";
+				"select type as code, type as name, status as group, count(status) as count from organizations group by type, status";
 		return listBrowseEntries(sql);
 	}
 
 	// BROWSE BY ORG TYPE FOR USER
 	public List<BrowseEntry> browseTypesForUser(final String email) {
 		final String sql = "select o.type as code, o.type as name,"
-			+ "o.status as group, count(o.status) as count "
-			+ "from organizations o "
-			+ "left outer join user_countries uc on (uc.country = o.country) "
-			+ "where uc.email=? "
-			+ "group by o.type, o.status";
+				+ "o.status as group, count(o.status) as count "
+				+ "from organizations o "
+				+ "left outer join user_countries uc on (uc.country = o.country) "
+				+ "where uc.email=? "
+				+ "group by o.type, o.status";
 		return listBrowseEntries(sql, email);
 	}
 
@@ -454,7 +450,7 @@ public class DatabaseUtils {
 
 	public List<OrganizationConflict> listConflictsForId(final String id) {
 		final String sql =
-			"select o.id, o.name, o.type, o.city, o.country from oa_conflicts c left outer join organizations o on (c.id2 = o.id) where o.id is not null and c.id1 = ? and c.reltype = 'suggested'";
+				"select o.id, o.name, o.type, o.city, o.country from oa_conflicts c left outer join organizations o on (c.id2 = o.id) where o.id is not null and c.id1 = ? and c.reltype = 'suggested'";
 		return jdbcTemplate.query(sql, new BeanPropertyRowMapper<>(OrganizationConflict.class), id);
 	}
 
@@ -479,15 +475,14 @@ public class DatabaseUtils {
 	public String fixConflictSimilars(final List<String> similarIds, final String user) {
 
 		final List<OrganizationView> views =
-			similarIds.stream().map(organizationViewRepository::findById).filter(Optional::isPresent).map(Optional::get).collect(Collectors.toList());
+				similarIds.stream().map(organizationViewRepository::findById).filter(Optional::isPresent).map(Optional::get).collect(Collectors.toList());
 
-		final List<OrganizationView> persistents = views.stream().filter(v -> v.getPersistent()).collect(Collectors.toList());
+		final List<OrganizationView> persistents = views.stream().filter(OrganizationView::getPersistent).collect(Collectors.toList());
 
 		final OrganizationView masterOrg = new OrganizationView();
 
-		if (persistents.size() > 1) {
-			throw new RuntimeException("Too many persintent organizations");
-		} else if (persistents.size() == 1) {
+		if (persistents.size() > 1) { throw new RuntimeException("Too many persintent organizations"); }
+		if (persistents.size() == 1) {
 			backupOrg(persistents.get(0), user);
 			masterOrg.setId(persistents.get(0).getId());
 			masterOrg.setStatus(OrganizationStatus.approved.toString());
@@ -513,9 +508,9 @@ public class DatabaseUtils {
 			organizationRepository.updateStatus(backupId, OrganizationStatus.hidden.toString());
 
 			journalEntryRepository
-				.save(new JournalEntry(origId, JournalOperations.BACKUP_ORG, "Saved a backup copy: " + backupId, user));
+					.save(new JournalEntry(origId, JournalOperations.BACKUP_ORG, "Saved a backup copy: " + backupId, user));
 			journalEntryRepository
-				.save(new JournalEntry(backupId, JournalOperations.BACKUP_ORG, "Saved a backup copy of " + origId, user));
+					.save(new JournalEntry(backupId, JournalOperations.BACKUP_ORG, "Saved a backup copy of " + origId, user));
 
 			return backupId;
 		} catch (final Exception e) {
@@ -529,9 +524,9 @@ public class DatabaseUtils {
 		final OffsetDateTime now = OffsetDateTime.now();
 
 		final String finalMessage = (masterOrg.getId() == null ? "New org created merging: " : "Merging in persistent org: ") +
-			orgs.stream()
-				.map(OrganizationView::getId)
-				.collect(Collectors.joining(",  "));
+				orgs.stream()
+						.map(OrganizationView::getId)
+						.collect(Collectors.joining(",  "));
 
 		masterOrg.setName(findFirstString(orgs, OrganizationView::getName));
 		masterOrg.setType(findFirstString(orgs, OrganizationView::getType));
@@ -544,35 +539,35 @@ public class DatabaseUtils {
 		masterOrg.setAcronyms(findAll(orgs, OrganizationView::getAcronyms));
 		masterOrg.setUrls(findAll(orgs, OrganizationView::getUrls));
 		masterOrg.setRelations(findAll(orgs, OrganizationView::getRelations, r -> !r.getType().equals(RelationType.Merged_In.toString())
-			&& !r.getType().equals(RelationType.Merged_In.toString())));
+				&& !r.getType().equals(RelationType.Merged_In.toString())));
 
 		masterOrg.getOtherNames()
-			.addAll(orgs.stream()
-				.map(OrganizationView::getName)
-				.filter(StringUtils::isNotBlank)
-				.filter(s -> StringUtils.equalsIgnoreCase(s, masterOrg.getName()))
-				.map(s -> new eu.dnetlib.organizations.model.view.OtherName(s, "UNKNOWN"))
-				.collect(Collectors.toList()));
+				.addAll(orgs.stream()
+						.map(OrganizationView::getName)
+						.filter(StringUtils::isNotBlank)
+						.filter(s -> StringUtils.equalsIgnoreCase(s, masterOrg.getName()))
+						.map(s -> new eu.dnetlib.organizations.model.view.OtherName(s, "UNKNOWN"))
+						.collect(Collectors.toList()));
 
 		final String masterId = insertOrUpdateOrganization(masterOrg, user, false);
 
 		// I hide the merged organizations
 		orgs.stream()
-			.map(OrganizationView::getId)
-			.filter(id -> !id.equals(masterId))
-			.forEach(id -> {
-				hideConflictOrgs(masterId, id);
-				journalEntryRepository
-					.save(new JournalEntry(id, JournalOperations.FIX_CONFLICT, "The org has been hidded and merged in " + masterId, user));
-			});
+				.map(OrganizationView::getId)
+				.filter(id -> !id.equals(masterId))
+				.forEach(id -> {
+					hideConflictOrgs(masterId, id);
+					journalEntryRepository
+							.save(new JournalEntry(id, JournalOperations.FIX_CONFLICT, "The org has been hidded and merged in " + masterId, user));
+				});
 
 		// I reassign the duplicates to the new org
 		final List<OpenaireDuplicate> newDuplicates = orgs.stream()
-			.map(OrganizationView::getId)
-			.map(openaireDuplicateRepository::findByLocalId)
-			.flatMap(l -> l.stream())
-			.map(d -> new OpenaireDuplicate(masterId, d.getOaOriginalId(), d.getRelType(), d.getOaCollectedFrom()))
-			.collect(Collectors.toList());
+				.map(OrganizationView::getId)
+				.map(openaireDuplicateRepository::findByLocalId)
+				.flatMap(List::stream)
+				.map(d -> new OpenaireDuplicate(masterId, d.getOaOriginalId(), d.getRelType(), d.getOaCollectedFrom()))
+				.collect(Collectors.toList());
 
 		openaireDuplicateRepository.saveAll(newDuplicates);
 		newDuplicates.forEach(d -> {
@@ -588,12 +583,12 @@ public class DatabaseUtils {
 		for (int i = 0; i < orgs.size(); i++) {
 			for (int j = i + 1; j < orgs.size(); j++) {
 				openaireConflictRepository
-					.updateStatusAndResetGroup(orgs.get(i).getId(), orgs.get(j).getId(), SimilarityType.is_similar.toString(), user, now);
+						.updateStatusAndResetGroup(orgs.get(i).getId(), orgs.get(j).getId(), SimilarityType.is_similar.toString(), user, now);
 			}
 		}
 
 		journalEntryRepository
-			.save(new JournalEntry(masterId, JournalOperations.FIX_CONFLICT, finalMessage, user));
+				.save(new JournalEntry(masterId, JournalOperations.FIX_CONFLICT, finalMessage, user));
 
 		return masterId;
 	}
@@ -608,7 +603,7 @@ public class DatabaseUtils {
 		for (int i = 0; i < differentsIds.size(); i++) {
 			for (int j = i + 1; j < differentsIds.size(); j++) {
 				openaireConflictRepository
-					.updateStatusAndResetGroup(differentsIds.get(i), differentsIds.get(j), SimilarityType.is_different.toString(), user, now);
+						.updateStatusAndResetGroup(differentsIds.get(i), differentsIds.get(j), SimilarityType.is_different.toString(), user, now);
 			}
 			journalEntryRepository.save(new JournalEntry(differentsIds.get(i), JournalOperations.NO_CONFLICT, message, user));
 		}
@@ -633,11 +628,11 @@ public class DatabaseUtils {
 	}
 
 	private <T> Set<T> findAll(final List<OrganizationView> views, final Function<OrganizationView, Set<T>> mapper) {
-		return views.stream().map(mapper).flatMap(s -> s.stream()).collect(Collectors.toCollection(LinkedHashSet::new));
+		return views.stream().map(mapper).flatMap(Set::stream).collect(Collectors.toCollection(LinkedHashSet::new));
 	}
 
 	private <T> Set<T> findAll(final List<OrganizationView> views, final Function<OrganizationView, Set<T>> mapper, final Predicate<T> filter) {
-		return views.stream().map(mapper).flatMap(s -> s.stream()).filter(filter).collect(Collectors.toCollection(LinkedHashSet::new));
+		return views.stream().map(mapper).flatMap(Set::stream).filter(filter).collect(Collectors.toCollection(LinkedHashSet::new));
 	}
 
 	private List<Relationship> hideConflictOrgs(final String masterId, final String otherId) {
@@ -649,7 +644,7 @@ public class DatabaseUtils {
 
 	public List<String> invalidCountriesInSuggestions() {
 		final String sql =
-			" select distinct t.oa_country from tmp_dedup_events t left outer join countries c on (t.oa_country = c.val) where c.val is null order by t.oa_country";
+				" select distinct t.oa_country from tmp_dedup_events t left outer join countries c on (t.oa_country = c.val) where c.val is null order by t.oa_country";
 		return jdbcTemplate.queryForList(sql, String.class);
 	}
 
@@ -665,27 +660,92 @@ public class DatabaseUtils {
 		if (id.length() == 46) {
 			final Optional<OrganizationView> orgView = organizationViewRepository.findByOpenaireId(id);
 			valid = orgView.map(OrganizationView::getStatus)
-				.filter(s -> s.equals(OrganizationStatus.approved.toString()))
-				.isPresent();
+					.filter(s -> s.equals(OrganizationStatus.approved.toString()))
+					.isPresent();
 			ooid = orgView.get().getId();
 		} else {
 			valid = organizationRepository.findById(id)
-				.map(Organization::getStatus)
-				.filter(s -> s.equals(OrganizationStatus.approved.toString()))
-				.isPresent();
+					.map(Organization::getStatus)
+					.filter(s -> s.equals(OrganizationStatus.approved.toString()))
+					.isPresent();
 			ooid = id;
 		}
 
 		if (valid) {
 			persistentOrganizationRepository.save(new PersistentOrganization(ooid));
 			return ooid;
-		} else {
-			throw new RuntimeException("The ID does not refer to an approved Organization");
 		}
+		throw new RuntimeException("The ID does not refer to an approved Organization");
 	}
 
 	public void deletePersistentOrgs(final String id) {
 		persistentOrganizationRepository.deleteById(id);
 	}
 
+	public List<String> obtainLogEntries(final String id) {
+		final String query = "SELECT o.id, o.name, j.operation, j.description, date(j.op_date) as op_date "
+				+ "FROM organizations o JOIN journal j ON o.id = j.id "
+				+ "WHERE o.id=? "
+				+ "ORDER BY date(j.op_date)";
+		return jdbcTemplate.queryForList(query, id)
+				.stream()
+				.map(this::asLogEntry)
+				.collect(Collectors.toList());
+	}
+
+	public List<String> obtainLogEntries(final int year, final int month, final String country) {
+		final String query = "SELECT o.id, o.name, j.operation, j.description, date(j.op_date) as op_date "
+				+ "FROM organizations o JOIN journal j ON o.id = j.id "
+				+ "WHERE o.country=? AND extract(year FROM j.op_date)=? AND extract(month FROM j.op_date)=? "
+				+ "ORDER BY date(j.op_date), o,id";
+		return jdbcTemplate.queryForList(query, country, year, month)
+				.stream()
+				.map(this::asLogEntry)
+				.collect(Collectors.toList());
+
+	}
+
+	private String asLogEntry(final Map<String, Object> map) {
+		final JournalOperations op = JournalOperations.valueOf("" + map.get("operation"));
+
+		final StringWriter sw = new StringWriter();
+		sw.write("On %s a curator ");
+
+		switch (op) {
+		case APPROVE_SUGG_ORG:
+			sw.write("approved the suggested organisation %s (ID: %s)");
+			break;
+		case NEW_ORG:
+			sw.write("created the organisation %s (ID: %s)");
+			break;
+		case NEW_SUGG_ORG:
+			sw.write("suggested the organisation %s (ID: %s)");
+			break;
+		case EDIT_ORG:
+			sw.write("updated the metadata of %s (ID: %s)");
+			break;
+		case EDIT_SUGG_ORG:
+			sw.write("updated the metadata of the suggested organisation %s (ID: %s)");
+			break;
+		case DUPLICATES:
+			sw.write("updated the list of duplicates for %s (ID: %s)");
+			break;
+		case FIX_CONFLICT:
+			sw.write("chose as master the organisation %s (id: %s), which has been involved in a conflict resolution");
+			break;
+		case NO_CONFLICT:
+			sw.write("stated that the conflict where the organisation %s (id: %s) was involved was a false positive");
+			break;
+		case BACKUP_ORG:
+			sw.write("resolved a conflict and the organisation %s (id: %s) involved has been hidden, because another organisation has been chosen as master");
+			break;
+		case UNKNOWN:
+		default:
+			sw.write("performed an unknown operation on %s (ID: %s)");
+		}
+		sw.write(". \"%s\"\n");
+
+		return String.format(sw.toString(), map.get("op_date"), map.get("name"), map.get("id"), map.get("description"));
+	}
+
 }

apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/utils/MailDispatcher.java

@@ -42,12 +42,12 @@ public class MailDispatcher {
 	}
 
 	public void configure(final SystemConfiguration conf) {
-		this.enabled = conf.getSmtpEnabled();
-		this.fromMail = conf.getSmtpFromMail();
-		this.fromName = conf.getSmtpFromName();
-		this.toMailAdmin = conf.getSmtpToMailAdmin();
-		this.newUserMessage = conf.getSmtpNewUserMessage();
-		this.updateUserMessage = conf.getSmtpUpdateUserMessage();
+		enabled = conf.getSmtpEnabled();
+		fromMail = conf.getSmtpFromMail();
+		fromName = conf.getSmtpFromName();
+		toMailAdmin = conf.getSmtpToMailAdmin();
+		newUserMessage = conf.getSmtpNewUserMessage();
+		updateUserMessage = conf.getSmtpUpdateUserMessage();
 
 		if (StringUtils.isNotBlank(conf.getSmtpHost())) {
 			emailFactory.setSmtpHost(conf.getSmtpHost());
@@ -70,7 +70,7 @@ public class MailDispatcher {
 
 	public void sendRequestRegistrationMail(final UserView user) {
 		if (StringUtils.isNotBlank(newUserMessage)) {
-			final String message = prepareMessage(this.newUserMessage, user);
+			final String message = prepareMessage(newUserMessage, user);
 
 			for (final String to : StringUtils.split(toMailAdmin, ",")) {
 				sendMail("OpenOrgs: new registration request", message, to.trim());
@@ -82,7 +82,7 @@ public class MailDispatcher {
 
 	public void sendUpdatedUserMail(final UserView user) {
 		if (StringUtils.isNotBlank(updateUserMessage)) {
-			final String message = prepareMessage(this.updateUserMessage, user);
+			final String message = prepareMessage(updateUserMessage, user);
 			sendMail("OpenOrgs: user updated", message, user.getEmail());
 		} else {
 			log.warn("Template is empty (updateUserMessage)");
@@ -92,13 +92,13 @@ public class MailDispatcher {
 	private String prepareMessage(final String template, final UserView user) {
 		final String countries = user.getRole().equals(UserRole.ADMIN.toString()) ? "All" : StringUtils.join(user.getCountries(), ", ");
 
-		return template.replaceAll(":email:", user.getEmail())
-			.replaceAll(":fullname:", user.getFullname())
-			.replaceAll(":organization:", user.getOrganization())
-			.replaceAll(":refperson:", user.getReferencePerson())
-			.replaceAll(":reqmessage:", user.getRequestMessage())
-			.replaceAll(":role:", user.getRole())
-			.replaceAll(":countries:", countries);
+		return template.replaceAll(":email:", "" + user.getEmail())
+				.replaceAll(":fullname:", "" + user.getFullname())
+				.replaceAll(":organization:", "" + user.getOrganization())
+				.replaceAll(":refperson:", "" + user.getReferencePerson())
+				.replaceAll(":reqmessage:", "" + user.getRequestMessage())
+				.replaceAll(":role:", "" + user.getRole())
+				.replaceAll(":countries:", countries);
 
 	}
 

apps/dnet-orgs-database-application/src/main/resources/application.properties

@@ -49,6 +49,9 @@ openaire.api.https.proxy = 10.19.65.35
 
 openorgs.support.pages = { "Ask a question": "https://www.openaire.eu/support/helpdesk?view=ticket&layout=open", "FAQ": "https://www.openaire.eu/faqs" }
 
+openorgs.findDuplicatesByCountry.limit.default = 1000
+openorgs.findOrgsByStatusAndCountry.limit.default = 1000
+
 openaire.override.logout.url = 
 
 openaire.explore.organization.baseurl = https://explore.openaire.eu/search/organization?organizationId=%s

apps/dnet-orgs-database-application/src/main/resources/sql/schema.sql

@@ -6,6 +6,7 @@ DROP VIEW IF EXISTS conflict_groups_view;
 DROP VIEW IF EXISTS suggestions_info_by_country_view;
 DROP VIEW IF EXISTS duplicate_groups_view;
 DROP VIEW IF EXISTS persistent_orgs_view;
+DROP VIEW IF EXISTS api_journal_view;
 
 DROP TABLE IF EXISTS sysconf;
 DROP TABLE IF EXISTS other_ids;
@@ -748,3 +749,15 @@ $$;
 CREATE TRIGGER insert_or_update_index_search_trigger AFTER INSERT OR UPDATE ON organizations FOR EACH ROW EXECUTE PROCEDURE insert_or_update_index_search_trigger();
 CREATE TRIGGER delete_index_search_trigger           BEFORE DELETE          ON organizations FOR EACH ROW EXECUTE PROCEDURE delete_index_search();
 
+-- PUBLIC VIEW
+
+CREATE VIEW api_journal_view AS SELECT
+	o.id          AS org_id,
+	o.name        AS org_name,
+	o.country     AS country,
+	j.operation   AS operation,
+	j.description AS description,
+	j.op_date     AS op_date
+FROM organizations o JOIN journal j ON o.id = j.id;
+
+

apps/pom.xml

@@ -3,7 +3,7 @@
 	<parent>
 		<groupId>eu.dnetlib.dhp</groupId>
 		<artifactId>dnet-applications</artifactId>
-		<version>3.4.3-SNAPSHOT</version>
+		<version>3.5.5-SNAPSHOT</version>
 		<relativePath>../pom.xml</relativePath>
 	</parent>
 

apps/scholexplorer-api/pom.xml

@@ -3,7 +3,7 @@
     <parent>
         <groupId>eu.dnetlib.dhp</groupId>
         <artifactId>apps</artifactId>
-        <version>3.4.3-SNAPSHOT</version>
+        <version>3.5.5-SNAPSHOT</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
 

cmd-line-apps/dhp-broker-client/pom.xml

@@ -4,7 +4,7 @@
 	<parent>
 		<groupId>eu.dnetlib.dhp</groupId>
 		<artifactId>cmd-line-apps</artifactId>
-		<version>3.4.3-SNAPSHOT</version>
+		<version>3.5.5-SNAPSHOT</version>
 		<relativePath>../</relativePath>
 	</parent>
 

cmd-line-apps/pom.xml

@@ -3,7 +3,7 @@
 	<parent>
 		<groupId>eu.dnetlib.dhp</groupId>
 		<artifactId>dnet-applications</artifactId>
-		<version>3.4.3-SNAPSHOT</version>
+		<version>3.5.5-SNAPSHOT</version>
 		<relativePath>../</relativePath>
 	</parent>
 

libs/dnet-apps-common/pom.xml

@@ -4,7 +4,7 @@
 	<parent>
 		<groupId>eu.dnetlib.dhp</groupId>
 		<artifactId>libs</artifactId>
-		<version>3.4.3-SNAPSHOT</version>
+		<version>3.5.5-SNAPSHOT</version>
 		<relativePath>../</relativePath>
 	</parent>
 

libs/dnet-broker-apps-common/pom.xml

@@ -4,7 +4,7 @@
 	<parent>
 		<groupId>eu.dnetlib.dhp</groupId>
 		<artifactId>libs</artifactId>
-		<version>3.4.3-SNAPSHOT</version>
+		<version>3.5.5-SNAPSHOT</version>
 		<relativePath>../</relativePath>
 	</parent>
 

libs/dnet-broker-apps-common/src/main/java/eu/dnetlib/broker/common/feedbacks/DbEventFeedback.java

@@ -0,0 +1,69 @@
+package eu.dnetlib.broker.common.feedbacks;
+
+import java.io.Serializable;
+import java.util.Date;
+
+import javax.persistence.Column;
+import javax.persistence.Entity;
+import javax.persistence.EnumType;
+import javax.persistence.Enumerated;
+import javax.persistence.Id;
+import javax.persistence.Table;
+
+import org.springframework.data.annotation.CreatedDate;
+import org.springframework.data.annotation.LastModifiedDate;
+
+@Entity(name = "feedbacks")
+@Table(name = "feedbacks")
+public class DbEventFeedback implements Serializable {
+
+	private static final long serialVersionUID = -1252332235554824748L;
+
+	@Id
+	@Column(name = "eventid")
+	private String eventId;
+
+	@Column(name = "status", nullable = false)
+	@Enumerated(EnumType.STRING)
+	private FeedbackStatus status;
+
+	@CreatedDate
+	@Column(name = "creation_date", nullable = false, updatable = false, columnDefinition = "timestamp without time zone default now()'")
+	private Date creationDate = new Date();
+
+	@LastModifiedDate
+	@Column(name = "modification_date", nullable = false, columnDefinition = "timestamp without time zone default now()")
+	private Date modificationDate = new Date();
+
+	public String getEventId() {
+		return eventId;
+	}
+
+	public void setEventId(final String eventId) {
+		this.eventId = eventId;
+	}
+
+	public FeedbackStatus getStatus() {
+		return status;
+	}
+
+	public void setStatus(final FeedbackStatus status) {
+		this.status = status;
+	}
+
+	public Date getCreationDate() {
+		return creationDate;
+	}
+
+	public void setCreationDate(final Date creationDate) {
+		this.creationDate = creationDate;
+	}
+
+	public Date getModificationDate() {
+		return modificationDate;
+	}
+
+	public void setModificationDate(final Date modificationDate) {
+		this.modificationDate = modificationDate;
+	}
+}

libs/dnet-broker-apps-common/src/main/java/eu/dnetlib/broker/common/feedbacks/DbEventFeedbackRepository.java

@@ -0,0 +1,7 @@
+package eu.dnetlib.broker.common.feedbacks;
+
+import org.springframework.data.repository.CrudRepository;
+
+public interface DbEventFeedbackRepository extends CrudRepository<DbEventFeedback, String> {
+
+}

libs/dnet-broker-apps-common/src/main/java/eu/dnetlib/broker/common/feedbacks/FeedbackStatus.java

@@ -0,0 +1,9 @@
+package eu.dnetlib.broker.common.feedbacks;
+
+public enum FeedbackStatus {
+	DISCARDED,	// the event was not processable by the system. OpenAIRE should not interpret such status in a negative or positive sense
+				// with regard to the accuracy of the notification
+	REJECTED,	// a human takes the decision to reject the suggestion as it was wrong
+	ACCEPTED	// a human takes the decision to apply the suggested enrichment to the local record
+
+}

libs/dnet-broker-apps-common/src/main/resources/config/schema.sql

@@ -18,6 +18,13 @@ CREATE TABLE topic_types (
     regex       text UNIQUE NOT NULL
 );
 
+CREATE TABLE feedbacks (
+       eventid text PRIMARY KEY,
+       status text NOT NULL,
+       creation_date     timestamp without time zone DEFAULT now(),
+       modification_date timestamp without time zone DEFAULT now()
+);
+
 -- curl "http://localhost:8080/api/topic-types/add" -d "name=ENRICH&expression=ENRICH%2F%3Ccond%3E%2F%3Cfield%3E&producerId=OpenAIRE&mapKeys=targetDatasourceName"
 INSERT INTO public.topic_types (id, name, expression, map_keys, producer_id, regex) VALUES ('tt-a739fa2b-fde0-4eb2-bcee-6e7f277347db', 'ENRICH',               'ENRICH/<cond>/<field>',            'targetDatasourceName', 'OpenAIRE', '^ENRICH\/[a-zA-Z0-9._-]+\/[a-zA-Z0-9._-]+$');
 

libs/dnet-exporter-model/pom.xml

@@ -4,7 +4,7 @@
 	<parent>
 		<groupId>eu.dnetlib.dhp</groupId>
 		<artifactId>libs</artifactId>
-		<version>3.4.3-SNAPSHOT</version>
+		<version>3.5.5-SNAPSHOT</version>
 		<relativePath>../</relativePath>
 	</parent>
 

libs/dnet-exporter-model/src/main/java/eu/dnetlib/openaire/exporter/exceptions/CommunityException.java

@@ -1,7 +1,5 @@
 package eu.dnetlib.openaire.exporter.exceptions;
 
-import java.io.IOException;
-
 public class CommunityException extends Exception {
 
 	/**
@@ -13,7 +11,7 @@ public class CommunityException extends Exception {
 		super(message);
 	}
 
-	public CommunityException(final IOException e) {
+	public CommunityException(final Throwable e) {
 		super(e);
 	}
 }

libs/dnet-exporter-model/src/main/java/eu/dnetlib/openaire/exporter/exceptions/ContextException.java

@@ -1,19 +0,0 @@
-package eu.dnetlib.openaire.exporter.exceptions;
-
-import java.io.IOException;
-
-public class ContextException extends Exception {
-
-	/**
-	 *
-	 */
-	private static final long serialVersionUID = -5489369676370127052L;
-
-	public ContextException(final String message) {
-		super(message);
-	}
-
-	public ContextException(final IOException e) {
-		super(e);
-	}
-}

libs/dnet-exporter-model/src/main/java/eu/dnetlib/openaire/exporter/exceptions/ContextNotFoundException.java

@@ -1,17 +0,0 @@
-package eu.dnetlib.openaire.exporter.exceptions;
-
-public class ContextNotFoundException extends Exception {
-
-	/**
-	 *
-	 */
-	private static final long serialVersionUID = -2026506752817353752L;
-
-	public ContextNotFoundException(final String msg) {
-		super(msg);
-	}
-
-	public ContextNotFoundException(final Exception e) {
-		super(e);
-	}
-}