[maven-release-plugin] prepare for next development iteration

Reviewed-on: #20

.gitignore

@@ -7,7 +7,7 @@
 *.iws
 *.java-version
 *~
-/**/ssh_tunnel_*.sh
+/**/*.sh
 /**/my_application.properties
 .vscode
 .classpath

apps/bioschemas-api/pom.xml

@@ -3,7 +3,7 @@
     <parent>
         <groupId>eu.dnetlib.dhp</groupId>
         <artifactId>apps</artifactId>
-        <version>3.4.3-SNAPSHOT</version>
+        <version>3.5.5-SNAPSHOT</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
 

apps/dhp-broker-application/pom.xml

@@ -4,7 +4,7 @@
 	<parent>
 		<groupId>eu.dnetlib.dhp</groupId>
 		<artifactId>apps</artifactId>
-		<version>3.4.3-SNAPSHOT</version>
+		<version>3.5.5-SNAPSHOT</version>
 		<relativePath>../pom.xml</relativePath>
 	</parent>
 

apps/dhp-broker-public-application/pom.xml

@@ -4,7 +4,7 @@
 	<parent>
 		<groupId>eu.dnetlib.dhp</groupId>
 		<artifactId>apps</artifactId>
-		<version>3.4.3-SNAPSHOT</version>
+		<version>3.5.5-SNAPSHOT</version>
 		<relativePath>../pom.xml</relativePath>
 	</parent>
 

apps/dhp-broker-public-application/src/main/java/eu/dnetlib/broker/oa/controllers/EventFeedback.java

@@ -2,6 +2,8 @@ package eu.dnetlib.broker.oa.controllers;
 
 import java.io.Serializable;
 
+import eu.dnetlib.broker.common.feedbacks.FeedbackStatus;
+
 public class EventFeedback implements Serializable {
 
 	/**
@@ -10,27 +12,22 @@ public class EventFeedback implements Serializable {
 	private static final long serialVersionUID = -6967719685282712195L;
 
 	private String eventId;
-	private String status;
 
-	// TOOD status should be an enum having the following values:
-	// * DISCARDED: the event was not processable by the system. OpenAIRE should not interpret such status in a negative or positive sense
-	// with regard to the accuracy of the notification
-	// * REJECTED: a human takes the decision to reject the suggestion as it was wrong
-	// * ACCEPTED: a human takes the decision to apply the suggested enrichment to the local record
+	private FeedbackStatus status;
 
-	protected String getEventId() {
+	public String getEventId() {
 		return eventId;
 	}
 
-	protected void setEventId(final String eventId) {
+	public void setEventId(final String eventId) {
 		this.eventId = eventId;
 	}
 
-	protected String getStatus() {
+	public FeedbackStatus getStatus() {
 		return status;
 	}
 
-	protected void setStatus(final String status) {
+	public void setStatus(final FeedbackStatus status) {
 		this.status = status;
 	}
 

apps/dhp-broker-public-application/src/main/java/eu/dnetlib/broker/oa/controllers/OpenairePublicController.java

@@ -57,6 +57,8 @@ import eu.dnetlib.broker.api.ShortEventMessage;
 import eu.dnetlib.broker.common.elasticsearch.EventRepository;
 import eu.dnetlib.broker.common.elasticsearch.Notification;
 import eu.dnetlib.broker.common.elasticsearch.NotificationRepository;
+import eu.dnetlib.broker.common.feedbacks.DbEventFeedback;
+import eu.dnetlib.broker.common.feedbacks.DbEventFeedbackRepository;
 import eu.dnetlib.broker.common.properties.ElasticSearchProperties;
 import eu.dnetlib.broker.common.stats.OpenaireDsStatRepository;
 import eu.dnetlib.broker.common.subscriptions.Subscription;
@@ -87,6 +89,9 @@ public class OpenairePublicController extends AbstractDnetController {
 	@Autowired
 	private OpenaireDsStatRepository openaireDsStatRepository;
 
+	@Autowired
+	private DbEventFeedbackRepository feedbackRepository;
+
 	@Autowired
 	private ElasticSearchProperties props;
 
@@ -103,31 +108,27 @@ public class OpenairePublicController extends AbstractDnetController {
 
 		final Optional<Subscription> optSub = subscriptionRepo.findById(subscrId);
 
-		if (optSub.isPresent()) {
+		if (!optSub.isPresent()) {
+			log.warn("Invalid subscription: " + subscrId);
+			return new ScrollPage<>(null, true, new ArrayList<>());
+		}
+		final ElasticsearchRestTemplate esTemplate = (ElasticsearchRestTemplate) esOperations;
 
-			final ElasticsearchRestTemplate esTemplate = (ElasticsearchRestTemplate) esOperations;
-
-			final NativeSearchQuery searchQuery = new NativeSearchQueryBuilder()
+		final NativeSearchQuery searchQuery = new NativeSearchQueryBuilder()
 				.withQuery(QueryBuilders.termQuery("subscriptionId.keyword", subscrId))
 				.withSearchType(SearchType.DEFAULT)
 				.withFields("topic", "payload", "eventId")
 				.withPageable(PageRequest.of(0, 100))
 				.build();
 
-			final SearchScrollHits<Notification> scroll =
+		final SearchScrollHits<Notification> scroll =
 				esTemplate.searchScrollStart(SCROLL_TIMEOUT_IN_MILLIS, searchQuery, Notification.class, IndexCoordinates.of(props.getNotificationsIndexName()));
-			if (scroll.hasSearchHits()) {
-				final List<ShortEventMessage> values = calculateNotificationMessages(scroll);
-				return new ScrollPage<>(scroll.getScrollId(), values.isEmpty() || scroll.getScrollId() == null, values);
-			} else {
-				esTemplate.searchScrollClear(Arrays.asList(scroll.getScrollId()));
-				return new ScrollPage<>(null, true, new ArrayList<>());
-			}
-
-		} else {
-			log.warn("Invalid subscription: " + subscrId);
-			return new ScrollPage<>(null, true, new ArrayList<>());
+		if (scroll.hasSearchHits()) {
+			final List<ShortEventMessage> values = calculateNotificationMessages(scroll);
+			return new ScrollPage<>(scroll.getScrollId(), values.isEmpty() || scroll.getScrollId() == null, values);
 		}
+		esTemplate.searchScrollClear(Arrays.asList(scroll.getScrollId()));
+		return new ScrollPage<>(null, true, new ArrayList<>());
 	}
 
 	@Operation(summary = "Returns notifications using scrolls (other pages)")
@@ -137,14 +138,13 @@ public class OpenairePublicController extends AbstractDnetController {
 		final ElasticsearchRestTemplate esTemplate = (ElasticsearchRestTemplate) esOperations;
 
 		final SearchScrollHits<Notification> scroll =
-			esTemplate.searchScrollContinue(scrollId, SCROLL_TIMEOUT_IN_MILLIS, Notification.class, IndexCoordinates.of(props.getNotificationsIndexName()));
+				esTemplate.searchScrollContinue(scrollId, SCROLL_TIMEOUT_IN_MILLIS, Notification.class, IndexCoordinates.of(props.getNotificationsIndexName()));
 		if (scroll.hasSearchHits()) {
 			final List<ShortEventMessage> values = calculateNotificationMessages(scroll);
 			return new ScrollPage<>(scroll.getScrollId(), values.isEmpty() || scroll.getScrollId() == null, values);
-		} else {
-			esTemplate.searchScrollClear(Arrays.asList(scroll.getScrollId()));
-			return new ScrollPage<>(null, true, new ArrayList<>());
 		}
+		esTemplate.searchScrollClear(Arrays.asList(scroll.getScrollId()));
+		return new ScrollPage<>(null, true, new ArrayList<>());
 	}
 
 	@Operation(summary = "Returns notifications as file")
@@ -197,8 +197,8 @@ public class OpenairePublicController extends AbstractDnetController {
 		final Path pathDir = new Path(opendoarEventsPath + "/" + DigestUtils.md5Hex(id));
 
 		try (final FileSystem fs = FileSystem.get(conf);
-			final ServletOutputStream out = res.getOutputStream();
-			final GZIPOutputStream gzOut = new GZIPOutputStream(out)) {
+				final ServletOutputStream out = res.getOutputStream();
+				final GZIPOutputStream gzOut = new GZIPOutputStream(out)) {
 			boolean first = true;
 
 			IOUtils.write("[\n", gzOut, StandardCharsets.UTF_8);
@@ -209,16 +209,16 @@ public class OpenairePublicController extends AbstractDnetController {
 						final Path path = fileStatus.getPath();
 						if (path.getName().endsWith(".json")) {
 							try (final FSDataInputStream fis = fs.open(path);
-								final InputStreamReader isr = new InputStreamReader(fis);
-								final BufferedReader br = new BufferedReader(isr)) {
+									final InputStreamReader isr = new InputStreamReader(fis);
+									final BufferedReader br = new BufferedReader(isr)) {
 
 								first = processLine(gzOut, first, br);
 							}
 						} else if (path.getName().endsWith(".json.gz")) {
 							try (final FSDataInputStream fis = fs.open(path);
-								final GZIPInputStream gzIn = new GZIPInputStream(fis);
-								final InputStreamReader isr = new InputStreamReader(gzIn);
-								final BufferedReader br = new BufferedReader(isr)) {
+									final GZIPInputStream gzIn = new GZIPInputStream(fis);
+									final InputStreamReader isr = new InputStreamReader(gzIn);
+									final BufferedReader br = new BufferedReader(isr)) {
 
 								first = processLine(gzOut, first, br);
 							}
@@ -271,24 +271,30 @@ public class OpenairePublicController extends AbstractDnetController {
 
 	@Operation(summary = "Store the feedback of an event (MOCK)")
 	@RequestMapping(value = "/feedback/events", method = {
-		RequestMethod.POST, RequestMethod.PATCH
+			RequestMethod.POST, RequestMethod.PATCH
 	})
-	private Map<String, String> feedbackEvent(@RequestBody final EventFeedback feedback) {
-		// TOOD
-		final Map<String, String> res = new HashMap<>();
+	private Map<String, Object> feedbackEvent(@RequestBody final EventFeedback feedback) {
+
+		final DbEventFeedback dbEntry = new DbEventFeedback();
+		dbEntry.setEventId(feedback.getEventId());
+		dbEntry.setStatus(feedback.getStatus());
+
+		feedbackRepository.save(dbEntry);
+
+		final Map<String, Object> res = new HashMap<>();
 		res.put("status", "done");
+		res.put("feedback", dbEntry);
 		return res;
 	}
 
 	private List<ShortEventMessage> calculateNotificationMessages(final SearchScrollHits<Notification> scroll) {
 		if (scroll.getSearchHits().size() > 0) {
 			return scroll.stream()
-				.map(SearchHit::getContent)
-				.map(this::messageFromNotification)
-				.collect(Collectors.toList());
-		} else {
-			return new ArrayList<>();
+					.map(SearchHit::getContent)
+					.map(this::messageFromNotification)
+					.collect(Collectors.toList());
 		}
+		return new ArrayList<>();
 	}
 
 	private ShortEventMessage messageFromNotification(final Notification n) {

apps/dhp-broker-public-application/ssh_tunnel_postgres.sh

@@ -1,3 +0,0 @@
-#!/bin/bash
-
-ssh -vNL 5432:10.19.65.40:5432 michele.artini@iis-cdh5-test-gw.ocean.icm.edu.pl

apps/dhp-mdstore-manager/pom.xml

@@ -4,7 +4,7 @@
 	<parent>
 		<groupId>eu.dnetlib.dhp</groupId>
 		<artifactId>apps</artifactId>
-		<version>3.4.3-SNAPSHOT</version>
+		<version>3.5.5-SNAPSHOT</version>
 		<relativePath>../pom.xml</relativePath>
 	</parent>
 

apps/dhp-mdstore-manager/src/main/java/eu/dnetlib/data/mdstore/manager/controller/SwaggerController.java

@@ -1,16 +1,16 @@
-package eu.dnetlib;
+package eu.dnetlib.data.mdstore.manager.controller;
 
 import org.springframework.stereotype.Controller;
 import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RequestMethod;
 
 @Controller
 public class SwaggerController {
 
 	@RequestMapping(value = {
-		"/", "/docs", "swagger-ui.html", "swagger-ui/"
-	})
+		"/apidoc", "/api-doc", "/doc", "/swagger"
+	}, method = RequestMethod.GET)
 	public String apiDoc() {
 		return "redirect:swagger-ui/index.html";
 	}
-
 }

apps/dnet-exporter-api/pom.xml

@@ -3,8 +3,8 @@
 	<parent>
 		<groupId>eu.dnetlib.dhp</groupId>
 		<artifactId>apps</artifactId>
-		<version>3.4.3-SNAPSHOT</version>
-		<relativePath>../pom.xml</relativePath>
+		<version>3.5.5-SNAPSHOT</version>
+		<relativePath>../</relativePath>
 	</parent>
 
 	<modelVersion>4.0.0</modelVersion>
@@ -113,6 +113,10 @@
 			<groupId>com.fasterxml.jackson.datatype</groupId>
 			<artifactId>jackson-datatype-joda</artifactId>
 		</dependency>
+		<dependency>
+			<groupId>com.fasterxml.jackson.datatype</groupId>
+			<artifactId>jackson-datatype-jsr310</artifactId>
+		</dependency>
 		<dependency>
 			<groupId>joda-time</groupId>
 			<artifactId>joda-time</artifactId>
@@ -141,6 +145,12 @@
             <artifactId>spring-web</artifactId>
             <version>5.3.8</version>
         </dependency>
+        
+		<dependency>
+			<groupId>com.vladmihalcea</groupId>
+			<artifactId>hibernate-types-52</artifactId>
+		</dependency>
+        
     </dependencies>
 	
 	<build>

apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/common/AbstractExporterController.java

@@ -16,7 +16,9 @@ import org.springframework.web.bind.annotation.ResponseStatus;
 
 import com.fasterxml.jackson.annotation.JsonAutoDetect;
 
+import eu.dnetlib.openaire.exporter.exceptions.CommunityException;
 import eu.dnetlib.openaire.exporter.exceptions.DsmApiException;
+import eu.dnetlib.openaire.exporter.exceptions.ResourceNotFoundException;
 import eu.dnetlib.openaire.exporter.model.dsm.Response;
 
 /**
@@ -28,17 +30,26 @@ public abstract class AbstractExporterController {
 
 	@ResponseBody
 	@ExceptionHandler({
-		DsmApiException.class
+		DsmApiException.class, CommunityException.class, Exception.class
 	})
 	@ResponseStatus(value = HttpStatus.INTERNAL_SERVER_ERROR)
-	public ErrorMessage handleDSMException(final Exception e) {
+	public ErrorMessage handle500(final Exception e) {
+		return _handleError(e);
+	}
+
+	@ResponseBody
+	@ExceptionHandler({
+		ResourceNotFoundException.class
+	})
+	@ResponseStatus(value = HttpStatus.NOT_FOUND)
+	public ErrorMessage handle404(final Exception e) {
 		return _handleError(e);
 	}
 
 	@ResponseBody
 	@ExceptionHandler(MethodArgumentNotValidException.class)
 	@ResponseStatus(HttpStatus.BAD_REQUEST)
-	public List<ErrorMessage> processValidationError(final MethodArgumentNotValidException e) {
+	public List<ErrorMessage> handle400(final MethodArgumentNotValidException e) {
 		return e.getBindingResult()
 			.getFieldErrors()
 			.stream()

apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/common/ExporterConstants.java

@@ -10,6 +10,7 @@ public class ExporterConstants {
 	public final static String C_PJ = "Community projects";
 	public final static String C_ZC = "Community Zenodo Communities";
 	public final static String C_O = "Community Organizations";
+	public final static String C_SUB = "Subcommunities";
 
 	public final static String DS = "Datasource";
 	public final static String API = "Interface";

apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/common/ISClient.java

@@ -4,6 +4,7 @@ import java.io.IOException;
 import java.util.List;
 import java.util.Map;
 
+import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpException;
 import eu.dnetlib.openaire.dsm.dao.utils.IndexDsInfo;
 import eu.dnetlib.openaire.exporter.model.context.Context;
 
@@ -13,18 +14,25 @@ public interface ISClient {
 
 	String getObjectStoreId(String dsId) throws Exception;
 
+	@Deprecated
 	Map<String, Context> getFunderContextMap() throws IOException;
 
+	@Deprecated
 	Map<String, Context> getCommunityContextMap() throws IOException;
 
+	@Deprecated
 	Map<String, Context> getContextMap(final List<String> type) throws IOException;
 
+	@Deprecated
 	void updateContextParam(String id, String name, String value, boolean toEscape);
 
+	@Deprecated
 	void updateContextAttribute(String id, String name, String value);
 
+	@Deprecated
 	void addConcept(String id, String categoryId, String data);
 
+	@Deprecated
 	void removeConcept(String id, String categoryId, String conceptId);
 
 	void dropCache();
@@ -38,10 +46,15 @@ public interface ISClient {
 	 * @param value
 	 *            new value for the attribute
 	 */
+	@Deprecated
 	void updateConceptAttribute(String id, String name, String value);
 
+	@Deprecated
 	void updateConceptParam(String id, String name, String value);
 
+	@Deprecated
 	void updateConceptParamNoEscape(String id, String name, String value);
 
+	String getProfile(String profileId) throws ISLookUpException;
+
 }

apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/common/ISClientImpl.java

@@ -79,29 +79,37 @@ public class ISClientImpl implements ISClient {
 
 	@Override
 	@Cacheable("context-cache-funder")
+	@Deprecated
 	public Map<String, Context> getFunderContextMap() throws IOException {
 		return _processContext(_getQuery(config.getFindFunderContexts()));
 	}
 
 	@Override
 	@Cacheable("context-cache-community")
+	@Deprecated
 	public Map<String, Context> getCommunityContextMap() throws IOException {
 		return _processContext(_getQuery(config.getFindCommunityContexts()));
 	}
 
 	@Override
 	@Cacheable("context-cache")
+	@Deprecated
 	public Map<String, Context> getContextMap(final List<String> type) throws IOException {
 		if (Objects.isNull(type) || type.isEmpty()) {
 			return _processContext(_getQuery(config.getFindContextProfiles()));
 		} else {
-			final String xqueryTemplate = _getQuery(config.getFindContextProfilesByType());
+			try {
+				final String xqueryTemplate = _getQuery(config.getFindContextProfilesByType());
+				final String xquery = String.format(xqueryTemplate, type.stream()
+					.map(t -> String.format("./RESOURCE_PROFILE/BODY/CONFIGURATION/context/@type = '%s'", t))
+					.collect(Collectors.joining(" or ")));
 
-			final String xquery = String.format(xqueryTemplate, type.stream()
-				.map(t -> String.format("./RESOURCE_PROFILE/BODY/CONFIGURATION/context/@type = '%s'", t))
-				.collect(Collectors.joining(" or ")));
+				return _processContext(xquery);
+			} catch (final Exception e) {
+				log.error(e.getMessage());
+				throw new RuntimeException(e);
+			}
 
-			return _processContext(xquery);
 		}
 	}
 
@@ -109,6 +117,7 @@ public class ISClientImpl implements ISClient {
 	@CacheEvict(value = {
 		"context-cache", "context-cache-funder"
 	}, allEntries = true)
+	@Deprecated
 	public void updateContextParam(final String id, final String name, final String value, final boolean toEscape) {
 		if (getSize(id, name) > 0) {
 			try {
@@ -142,6 +151,7 @@ public class ISClientImpl implements ISClient {
 	@CacheEvict(value = {
 		"context-cache", "context-cache-funder"
 	}, allEntries = true)
+	@Deprecated
 	public void updateContextAttribute(final String id, final String name, final String value) {
 		final Escaper esc = XmlEscapers.xmlAttributeEscaper();
 		try {
@@ -156,6 +166,7 @@ public class ISClientImpl implements ISClient {
 	@CacheEvict(value = {
 		"context-cache", "context-cache-funder"
 	}, allEntries = true)
+	@Deprecated
 	public void addConcept(final String id, final String categoryId, final String data) {
 		try {
 			_quickSeachProfile(String.format("update insert %s into collection('/db/DRIVER/ContextDSResources/ContextDSResourceType')" +
@@ -169,6 +180,7 @@ public class ISClientImpl implements ISClient {
 	@CacheEvict(value = {
 		"context-cache", "context-cache-funder"
 	}, allEntries = true)
+	@Deprecated
 	public void removeConcept(final String id, final String categoryId, final String conceptId) {
 		try {
 			_quickSeachProfile(String.format("for $concept in collection('/db/DRIVER/ContextDSResources/ContextDSResourceType')" +
@@ -184,6 +196,7 @@ public class ISClientImpl implements ISClient {
 	@CacheEvict(value = {
 		"context-cache", "context-cache-community", "context-cache-funder"
 	}, allEntries = true)
+	@Deprecated
 	public void updateConceptAttribute(final String id, final String name, final String value) {
 		final Escaper esc = XmlEscapers.xmlAttributeEscaper();
 		try {
@@ -199,6 +212,7 @@ public class ISClientImpl implements ISClient {
 	@CacheEvict(value = {
 		"context-cache", "context-cache-funder"
 	}, allEntries = true)
+	@Deprecated
 	public void updateConceptParam(final String id, final String name, final String value) {
 		try {
 			_quickSeachProfile(getConceptXQuery(id, name, value));
@@ -211,6 +225,7 @@ public class ISClientImpl implements ISClient {
 	@CacheEvict(value = {
 		"context-cache", "context-cache-funder"
 	}, allEntries = true)
+	@Deprecated
 	public void updateConceptParamNoEscape(final String id, final String name, final String value) {
 		try {
 			_quickSeachProfile(getConceptXQueryNoEscape(id, name, value));
@@ -221,6 +236,7 @@ public class ISClientImpl implements ISClient {
 
 	/// HELPERS
 
+	@Deprecated
 	private String getInsertXQuery(final String id, final String paramName, final String paramValue, final boolean toEscape) {
 		String value;
 		if (toEscape) {
@@ -237,6 +253,7 @@ public class ISClientImpl implements ISClient {
 		}
 	}
 
+	@Deprecated
 	private String getXQuery(final String id, final String name, final String paramValue, final boolean toEscape) {
 		String value = paramValue;
 		if (toEscape) {
@@ -263,6 +280,7 @@ public class ISClientImpl implements ISClient {
 	// }
 	// }
 
+	@Deprecated
 	private String getConceptXQuery(final String id, final String name, final String value) {
 		final Escaper esc = XmlEscapers.xmlContentEscaper();
 		if (StringUtils.isNotBlank(value)) {
@@ -274,6 +292,7 @@ public class ISClientImpl implements ISClient {
 		}
 	}
 
+	@Deprecated
 	private String getConceptXQueryNoEscape(final String id, final String name, final String value) {
 
 		if (StringUtils.isNotBlank(value)) {
@@ -285,10 +304,12 @@ public class ISClientImpl implements ISClient {
 		}
 	}
 
+	@Deprecated
 	private Map<String, Context> _processContext(final String xquery) throws IOException {
 		return _processContext(new LinkedBlockingQueue<>(), xquery);
 	}
 
+	@Deprecated
 	private Map<String, Context> _processContext(final Queue<Throwable> errors, final String xquery) throws IOException {
 		try {
 			return getContextProfiles(errors, xquery).stream()
@@ -306,6 +327,7 @@ public class ISClientImpl implements ISClient {
 		}
 	}
 
+	@Deprecated
 	private List<String> getContextProfiles(final Queue<Throwable> errors, final String xquery) throws IOException {
 		log.warn("getContextProfiles(): not using cache");
 		try {
@@ -329,13 +351,13 @@ public class ISClientImpl implements ISClient {
 		final List<String> res = Lists.newArrayList();
 
 		log.debug(String.format("running xquery:\n%s", xquery));
-		try{
+		try {
 			final List<String> list = isLookUpService.quickSearchProfile(xquery);
 			if (list != null) {
 				res.addAll(list);
 			}
 			log.debug(String.format("query result size: %s", res.size()));
-		}catch(Exception ex){
+		} catch (final Exception ex) {
 			log.error(ex.getMessage());
 			throw new ISLookUpException("");
 		}
@@ -352,4 +374,9 @@ public class ISClientImpl implements ISClient {
 		log.debug("dropped dsManager IS cache");
 	}
 
+	@Override
+	public String getProfile(final String profileId) throws ISLookUpException {
+		return isLookUpService.getResourceProfile(profileId);
+	}
+
 }

apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/common/ReadOnlyRepository.java

@@ -1,4 +1,4 @@
-package eu.dnetlib.utils;
+package eu.dnetlib.openaire.common;
 
 import java.util.Optional;
 
@@ -19,4 +19,4 @@ public interface ReadOnlyRepository<T, ID> extends Repository<T, ID> {
 	Iterable<T> findAll();
 
 	long count();
-}
+}

apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityApiCore.java

@@ -1,517 +0,0 @@
-package eu.dnetlib.openaire.community;
-
-import java.util.*;
-import java.util.stream.Collectors;
-
-import com.google.gson.Gson;
-
-import eu.dnetlib.openaire.exporter.exceptions.CommunityException;
-import eu.dnetlib.openaire.exporter.exceptions.ResourceNotFoundException;
-import eu.dnetlib.openaire.exporter.model.community.CommunityContentprovider;
-import eu.dnetlib.openaire.exporter.model.community.CommunityDetails;
-import eu.dnetlib.openaire.exporter.model.community.CommunityOpenAIRECommunities;
-import eu.dnetlib.openaire.exporter.model.community.CommunityOrganization;
-import eu.dnetlib.openaire.exporter.model.community.CommunityProject;
-import eu.dnetlib.openaire.exporter.model.community.CommunitySummary;
-import eu.dnetlib.openaire.exporter.model.community.CommunityWritableProperties;
-import eu.dnetlib.openaire.exporter.model.community.CommunityZenodoCommunity;
-import eu.dnetlib.openaire.exporter.model.community.selectioncriteria.SelectionCriteria;
-
-import org.apache.commons.lang3.StringUtils;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
-import org.springframework.cache.annotation.CacheEvict;
-import org.springframework.stereotype.Component;
-
-import com.google.common.base.Functions;
-import com.google.common.base.Joiner;
-import com.google.common.collect.Lists;
-import com.google.common.collect.Sets;
-
-import eu.dnetlib.openaire.common.ISClient;
-
-import static eu.dnetlib.openaire.community.CommunityConstants.*;
-
-@Component
-@ConditionalOnProperty(value = "openaire.exporter.enable.community", havingValue = "true")
-public class CommunityApiCore {// implements CommunityClient{
-
-	private static final Log log = LogFactory.getLog(CommunityApiCore.class);
-
-	@Autowired
-	private CommunityClient cci;
-
-	@Autowired
-	private ISClient isClient;
-
-	@Autowired
-	private CommunityCommon cc;
-
-	public List<CommunitySummary> listCommunities() throws CommunityException {
-		return cc.listCommunities();
-
-	}
-
-	public CommunityDetails getCommunity(final String id) throws CommunityException, ResourceNotFoundException {
-		return cc.getCommunity(id);
-
-	}
-
-	private void removeAdvancedConstraint(String id) throws ResourceNotFoundException, CommunityException {
-		cc.getCommunity(id);
-		isClient.updateContextParam(id, CPROFILE_ADVANCED_CONSTRAINT, "", false);
-		cc.removeAdvancedConstraint(id);
-
-	}
-
-	public void setCommunity(final String id, final CommunityWritableProperties details) throws CommunityException, ResourceNotFoundException {
-
-		cc.getCommunity(id); // ensure the community exists.
-
-		if (details.getShortName() != null) {
-			isClient.updateContextAttribute(id, CLABEL, details.getShortName());
-
-		}
-		if (details.getName() != null) {
-			isClient.updateContextParam(id, CSUMMARY_NAME, details.getName(), true);
-
-		}
-		if (details.getDescription() != null) {
-			isClient.updateContextParam(id, CSUMMARY_DESCRIPTION, details.getDescription(), true);
-
-		}
-		if (details.getLogoUrl() != null) {
-			isClient.updateContextParam(id, CSUMMARY_LOGOURL, details.getLogoUrl(), true);
-
-		}
-		if (details.getStatus() != null) {
-			isClient.updateContextParam(id, CSUMMARY_STATUS, details.getStatus().name(), true);
-
-		}
-		if (details.getSubjects() != null) {
-			isClient.updateContextParam(id, CPROFILE_SUBJECT, Joiner.on(CSV_DELIMITER).join(details.getSubjects()), true);
-
-		}
-		if (details.getFos() != null) {
-			isClient.updateContextParam(id, CPROFILE_FOS, Joiner.on(CSV_DELIMITER).join(details.getFos()), true);
-
-		}
-		if (details.getSdg() != null) {
-			isClient.updateContextParam(id, CPROFILE_SDG, Joiner.on(CSV_DELIMITER).join(details.getSdg()), true);
-
-		}
-		if (details.getAdvancedConstraints() != null) {
-			isClient.updateContextParam(id, CPROFILE_ADVANCED_CONSTRAINT, "<![CDATA[" + new Gson().toJson(details.getAdvancedConstraints()) + "]]>", false);
-
-		}
-		if (details.getMainZenodoCommunity() != null) {
-			isClient.updateContextParam(id, CSUMMARY_ZENODOC, details.getMainZenodoCommunity(), true);
-		}
-
-		cc.updateCommunity(id, details);
-	}
-
-	public List<CommunityProject> getCommunityProjects(final String id) throws CommunityException, ResourceNotFoundException {
-		cc.getCommunity(id); // ensure the community exists.
-		return cc.getCommunityInfo(id, PROJECTS_ID_SUFFIX, c -> CommunityMappingUtils.asCommunityProject(id, c));
-	}
-
-	public CommunityProject addCommunityProject(final String id, final CommunityProject project) throws CommunityException, ResourceNotFoundException {
-		if (!StringUtils.equalsIgnoreCase(id, project.getCommunityId())) {
-			throw new CommunityException("parameters 'id' and project.communityId must be coherent");
-		}
-
-		return updateProject(id, project);
-
-	}
-
-	private CommunityProject updateProject(String id, CommunityProject project) throws CommunityException, ResourceNotFoundException {
-		final TreeMap<Integer, CommunityProject> projects = getCommunityProjectMap(id);
-		String project_id = project.getId();
-
-		if (project_id != null && projects.keySet().contains(Integer.valueOf(project_id))){
-			if (project.getName() != null) {
-				isClient.updateConceptParam(id + PROJECTS_ID_SUFFIX + ID_SEPARATOR + project_id, CPROJECT_FULLNAME, project.getName());
-
-			}
-			if(project.getAcronym()!= null){
-				isClient.updateConceptParam(id + PROJECTS_ID_SUFFIX + ID_SEPARATOR + project_id, CPROJECT_ACRONYM, project.getAcronym());
-
-			}
-			if (project.getOpenaireId() != null){
-				isClient.updateConceptParam(id + PROJECTS_ID_SUFFIX + ID_SEPARATOR + project_id, OPENAIRE_ID, project.getOpenaireId());
-
-			}
-			if (project.getFunder() != null){
-				isClient.updateConceptParam(id + PROJECTS_ID_SUFFIX + ID_SEPARATOR + project_id, CPROJECT_FUNDER, project.getFunder());
-
-			}
-			if(project.getGrantId() != null){
-				isClient.updateConceptParam(id + PROJECTS_ID_SUFFIX + ID_SEPARATOR + project_id, CPROJECT_NUMBER, project.getGrantId());
-
-			}
-		}else {
-			project.setId(nextId(projects != null && !projects.isEmpty() ? projects.lastKey() : 0));
-
-			isClient.addConcept(id, id + PROJECTS_ID_SUFFIX, CommunityMappingUtils.asProjectXML(id, project));
-
-		}
-		cc.updateProject(id, project );
-		return project;
-	}
-
-	public List<CommunityProject> addCommunityProjectList(final String id, final List<CommunityProject> projectList) throws CommunityException, ResourceNotFoundException {
-		if(projectList == null || projectList.size() == 0){
-			throw new CommunityException("parameter 'projectList' must be present and should contain at least one project");
-		}
-		if (!StringUtils.equalsIgnoreCase(id, projectList.get(0).getCommunityId())) {
-			throw new CommunityException("parameters 'id' and project.communityId must be coherent");
-		}
-
-		List<CommunityProject> projects = new ArrayList();
-
-		for(CommunityProject project : projectList){
-			projects.add(updateProject(id, project));
-		}
-
-
-		return projects;
-	}
-
-	private String nextId(final Integer id) {
-		return String.valueOf(id + 1);
-	}
-
-	public void removeCommunityProject(final String id, final Integer projectId) throws CommunityException, ResourceNotFoundException {
-		final Map<Integer, CommunityProject> projects = getCommunityProjectMap(id);
-		if (!projects.containsKey(projectId)) {
-			throw new ResourceNotFoundException(String.format("project '%s' doesn't exist within context '%s'", projectId, id));
-		}
-		isClient.removeConcept(id, id + PROJECTS_ID_SUFFIX, id + PROJECTS_ID_SUFFIX + ID_SEPARATOR + projectId);
-		cc.removeFromCategory(id, PROJECTS_ID_SUFFIX, String.valueOf(projectId));
-	}
-
-	public void removeCommunityProjectList(final String id, final List<Integer> projectIdList) throws CommunityException, ResourceNotFoundException {
-		for(Integer projectId: projectIdList){
-			removeCommunityProject(id, projectId);
-		}
-	}
-
-	public List<CommunityContentprovider> getCommunityContentproviders(final String id) throws CommunityException, ResourceNotFoundException {
-		cc.getCommunity(id); // ensure the community exists.
-		return cc.getCommunityInfo(id, CONTENTPROVIDERS_ID_SUFFIX, c -> CommunityMappingUtils.asCommunityDataprovider(id, c));
-	}
-
-	public CommunityContentprovider addCommunityContentprovider(final String id, final CommunityContentprovider cp)
-		throws CommunityException, ResourceNotFoundException {
-		log.info("content provider to add " + cp.toString());
-		if (!StringUtils.equalsIgnoreCase(id, cp.getCommunityId())) { throw new CommunityException("parameters 'id' and cp.communityId must be coherent"); }
-
-		return updateContentprovider(id, cp);
-	}
-
-	private CommunityContentprovider updateContentprovider(String id, CommunityContentprovider cp) throws CommunityException, ResourceNotFoundException {
-		final TreeMap<Integer, CommunityContentprovider> cps = getCommunityContentproviderMap(id);
-		final String concept_id = cp.getId();
-		if (concept_id != null && cps.keySet().contains(Integer.valueOf(concept_id))) {
-			if (cp.getName() != null) {
-				isClient.updateConceptParam(id + CONTENTPROVIDERS_ID_SUFFIX + ID_SEPARATOR + concept_id, CCONTENTPROVIDER_NAME, cp.getName());
-			}
-			if (cp.getOfficialname() != null) {
-				isClient.updateConceptParam(id + CONTENTPROVIDERS_ID_SUFFIX + ID_SEPARATOR + concept_id, CCONTENTPROVIDER_OFFICIALNAME, cp.getOfficialname());
-			}
-			if (cp.getOpenaireId() != null) {
-				isClient.updateConceptParam(id + CONTENTPROVIDERS_ID_SUFFIX + ID_SEPARATOR + concept_id, OPENAIRE_ID, cp.getOpenaireId());
-			}
-			if (cp.getSelectioncriteria() != null) {
-				isClient.updateConceptParamNoEscape(id + CONTENTPROVIDERS_ID_SUFFIX + ID_SEPARATOR + concept_id, CCONTENTPROVIDER_SELCRITERIA, cp.toXML());
-			}
-		} else {
-			log.info("adding new concept for community " + id);
-			cp.setId(nextId(!cps.isEmpty() ? cps.lastKey() : 0));
-
-			isClient.addConcept(id, id + CONTENTPROVIDERS_ID_SUFFIX, CommunityMappingUtils.asContentProviderXML(id, cp));
-		}
-
-		cc.updateDatasource(id, cp);
-		return cp;
-	}
-
-	public void removeCommunityContentProvider(final String id, final Integer contentproviderId) throws CommunityException, ResourceNotFoundException {
-		final Map<Integer, CommunityContentprovider> providers = getCommunityContentproviderMap(id);
-		if (!providers.containsKey(contentproviderId)) {
-			throw new ResourceNotFoundException(String.format("content provider '%s' doesn't exist within context '%s'", contentproviderId, id));
-		}
-		isClient.removeConcept(id, id + CONTENTPROVIDERS_ID_SUFFIX, id + CONTENTPROVIDERS_ID_SUFFIX + ID_SEPARATOR + contentproviderId);
-		cc.removeFromCategory(id, CONTENTPROVIDERS_ID_SUFFIX, String.valueOf(contentproviderId));
-	}
-
-	public List<CommunityContentprovider> addCommunityContentProvidersList(String id, List<CommunityContentprovider> contentprovidersList) throws CommunityException, ResourceNotFoundException {
-		if(contentprovidersList == null || contentprovidersList.size() == 0){
-			throw new CommunityException("parameter 'contentprovidersList' must be present and should contain at least one content provider");
-		}
-		if (!StringUtils.equalsIgnoreCase(id, contentprovidersList.get(0).getCommunityId())) {
-			throw new CommunityException("parameters 'id' and contentprovider.communityId must be coherent");
-		}
-
-		List<CommunityContentprovider> contentproviders = new ArrayList();
-
-		for(CommunityContentprovider contentProvider : contentprovidersList){
-			contentproviders.add(updateContentprovider(id, contentProvider));
-		}
-
-
-		return contentproviders;
-	}
-
-	public void removeCommunityContentProviderList(final String id, final List<Integer> contentProviderIdList) throws CommunityException, ResourceNotFoundException {
-		for(Integer contentProviderId: contentProviderIdList){
-			removeCommunityContentProvider(id, contentProviderId);
-		}
-	}
-
-	public void removeCommunityOrganization(final String id, final Integer organizationId) throws CommunityException, ResourceNotFoundException {
-		final Map<Integer, CommunityOrganization> organizations = getCommunityOrganizationMap(id);
-		if (!organizations.containsKey(organizationId)) {
-			throw new ResourceNotFoundException(String.format("organization '%s' doesn't exist within context '%s'", organizationId, id));
-		}
-		isClient.removeConcept(id, id + ORGANIZATION_ID_SUFFIX, id + ORGANIZATION_ID_SUFFIX + ID_SEPARATOR + organizationId);
-		cc.removeFromCategory(id, ORGANIZATION_ID_SUFFIX, String.valueOf(organizationId));
-	}
-
-	public List<CommunityZenodoCommunity> getCommunityZenodoCommunities(final String id) throws CommunityException, ResourceNotFoundException {
-
-		return cc.getCommunityZenodoCommunities(id);
-	}
-
-	public List<CommunityOrganization> getCommunityOrganizations(final String id) throws CommunityException, ResourceNotFoundException {
-		cc.getCommunity(id);
-		return cc.getCommunityInfo(id, ORGANIZATION_ID_SUFFIX, c -> CommunityMappingUtils.asCommunityOrganization(id, c));
-	}
-
-	public CommunityDetails addCommunitySubjects(final String id, final List<String> subjects) throws CommunityException, ResourceNotFoundException {
-
-		final CommunityDetails cd = new CommunityDetails();
-
-		final Set<String> current = Sets.newHashSet(cc.getCommunity(id).getSubjects());
-
-		current.addAll(subjects);
-
-		cd.setSubjects(Lists.newArrayList(current));
-
-		setCommunity(id, CommunityWritableProperties.fromDetails(cd));
-
-		return cd;
-	}
-
-	public CommunityDetails removeCommunitySubjects(final String id, final List<String> subjects) throws CommunityException, ResourceNotFoundException {
-
-		final CommunityDetails cd = new CommunityDetails();
-
-		final Set<String> current = Sets.newHashSet(cc.getCommunity(id).getSubjects());
-
-		current.removeAll(subjects);
-
-		cd.setSubjects(Lists.newArrayList(current));
-
-		setCommunity(id, CommunityWritableProperties.fromDetails(cd));
-
-		return cd;
-	}
-
-	public CommunityDetails addCommunityFOS(final String id, final List<String> foss) throws CommunityException, ResourceNotFoundException {
-
-		final CommunityDetails cd = new CommunityDetails();
-
-		final Set<String> current = Sets.newHashSet();
-		if(Optional.ofNullable(cc.getCommunity(id).getFos()).isPresent()){
-			current.addAll(cc.getCommunity(id).getFos());
-		}
-
-		current.addAll(foss);
-
-		cd.setFos(Lists.newArrayList(current));
-
-		setCommunity(id, CommunityWritableProperties.fromDetails(cd));
-
-		return cd;
-	}
-
-	public CommunityDetails removeCommunityFOS(final String id, final List<String> foss) throws CommunityException, ResourceNotFoundException {
-
-		final CommunityDetails cd = new CommunityDetails();
-
-		final Set<String> current = Sets.newHashSet(cc.getCommunity(id).getFos());
-
-		current.removeAll(foss);
-
-		cd.setFos(Lists.newArrayList(current));
-
-		setCommunity(id, CommunityWritableProperties.fromDetails(cd));
-
-		return cd;
-	}
-
-	public CommunityDetails addCommunitySDG(final String id, final List<String> sdgs) throws CommunityException, ResourceNotFoundException {
-
-		final CommunityDetails cd = new CommunityDetails();
-		final Set<String> current = Sets.newHashSet();
-		if(Optional.ofNullable(cc.getCommunity(id).getSdg()).isPresent()){
-			current.addAll(cc.getCommunity(id).getSdg());
-		}
-
-		current.addAll(sdgs);
-
-		cd.setSdg(Lists.newArrayList(current));
-
-		setCommunity(id, CommunityWritableProperties.fromDetails(cd));
-
-		return cd;
-	}
-
-	public CommunityDetails removeCommunitySDG(final String id, final List<String> sdgs) throws CommunityException, ResourceNotFoundException {
-
-		final CommunityDetails cd = new CommunityDetails();
-
-		final Set<String> current = Sets.newHashSet(cc.getCommunity(id).getSdg());
-
-		current.removeAll(sdgs);
-
-		cd.setSdg(Lists.newArrayList(current));
-
-		setCommunity(id, CommunityWritableProperties.fromDetails(cd));
-
-		return cd;
-	}
-
-	public CommunityDetails addCommunityAdvancedConstraint(final String id, final SelectionCriteria advancedCosntraint) throws CommunityException, ResourceNotFoundException {
-
-		final CommunityDetails cd = new CommunityDetails();
-
-		cd.setAdvancedConstraints(advancedCosntraint);
-
-		setCommunity(id, CommunityWritableProperties.fromDetails(cd));
-
-		return cd;
-	}
-
-	public CommunityDetails removeCommunityAdvancedConstraint(final String id) throws ResourceNotFoundException, CommunityException {
-
-		removeAdvancedConstraint(id);
-
-		return new CommunityDetails();
-	}
-
-
-
-	@CacheEvict(value = "community-cache", allEntries = true)
-	public void removeCommunityZenodoCommunity(final String id, final Integer zenodoCommId) throws CommunityException, ResourceNotFoundException {
-
-		final Map<Integer, CommunityZenodoCommunity> zcomms = getZenodoCommunityMap(id);
-		if (!zcomms.containsKey(zenodoCommId)) {
-			throw new ResourceNotFoundException(String.format("Zenodo community '%s' doesn't exist within context '%s'", zenodoCommId, id));
-		}
-		isClient.removeConcept(id, id + ZENODOCOMMUNITY_ID_SUFFIX, id + ZENODOCOMMUNITY_ID_SUFFIX + ID_SEPARATOR + zenodoCommId);
-
-		cc.removeFromCategory(id, ZENODOCOMMUNITY_ID_SUFFIX, String.valueOf(zenodoCommId));
-
-	}
-
-	@CacheEvict(value = "community-cache", allEntries = true)
-	public CommunityZenodoCommunity addCommunityZenodoCommunity(final String id, final CommunityZenodoCommunity zc)
-		throws CommunityException, ResourceNotFoundException {
-		if (!StringUtils.equalsIgnoreCase(id, zc.getCommunityId())) { throw new CommunityException("parameters 'id' and zc.communityId must be coherent"); }
-		if (!StringUtils.isNotBlank(zc.getZenodoid())) { throw new CommunityException("parameter zenodoid cannot be null or empty"); }
-		final TreeMap<Integer, CommunityZenodoCommunity> zcs = getZenodoCommunityMap(id);
-
-		for (final CommunityZenodoCommunity czc : zcs.values()) {
-			if (czc.getZenodoid().equals(zc.getZenodoid())) { throw new CommunityException("Zenodo community already associated to the RCD"); }
-		}
-
-		zc.setId(nextId(!zcs.isEmpty() ? zcs.lastKey() : 0));
-
-		isClient.addConcept(id, id + ZENODOCOMMUNITY_ID_SUFFIX, CommunityMappingUtils.asZenodoCommunityXML(id, zc));
-		cc.updateZenodoCommunity(id, zc);
-
-		return zc;
-	}
-
-	public CommunityOpenAIRECommunities getOpenAIRECommunities(final String zenodoId) throws CommunityException, ResourceNotFoundException {
-
-		if (cci.getInverseZenodoCommunityMap().containsKey(zenodoId)) {
-			return new CommunityOpenAIRECommunities().setZenodoid(zenodoId)
-				.setOpenAirecommunitylist(cci.getInverseZenodoCommunityMap().get(zenodoId).stream().collect(Collectors.toList()));
-		}
-		return new CommunityOpenAIRECommunities();
-
-	}
-
-	// HELPERS
-
-	private TreeMap<Integer, CommunityProject> getCommunityProjectMap(final String id) throws CommunityException, ResourceNotFoundException {
-		return getCommunityProjects(id).stream()
-			.collect(Collectors.toMap(p -> Integer.valueOf(p.getId()), Functions.identity(), (p1, p2) -> {
-				log.warn(String.format("duplicate project found: '%s'", p1.getId()));
-				return p2;
-			}, TreeMap::new));
-	}
-
-	private TreeMap<Integer, CommunityContentprovider> getCommunityContentproviderMap(final String id) throws CommunityException, ResourceNotFoundException {
-		log.info("getting community content provider map");
-		return getCommunityContentproviders(id).stream()
-			.collect(Collectors.toMap(cp -> Integer.valueOf(cp.getId()), Functions.identity(), (cp1, cp2) -> {
-				log.warn(String.format("duplicate content provider found: '%s'", cp1.getId()));
-				return cp2;
-			}, TreeMap::new));
-	}
-
-	private TreeMap<Integer, CommunityZenodoCommunity> getZenodoCommunityMap(final String id) throws CommunityException, ResourceNotFoundException {
-		return getCommunityZenodoCommunities(id).stream()
-			.collect(Collectors.toMap(cp -> Integer.valueOf(cp.getId()), Functions.identity(), (cp1, cp2) -> {
-				log.warn(String.format("duplicate Zenodo community found: '%s'", cp1.getId()));
-				return cp2;
-			}, TreeMap::new));
-	}
-
-	private TreeMap<Integer, CommunityOrganization> getCommunityOrganizationMap(final String id) throws CommunityException, ResourceNotFoundException {
-		return getCommunityOrganizations(id).stream()
-			.collect(Collectors.toMap(o -> Integer.valueOf(o.getId()), Functions.identity(), (o1, o2) -> {
-				log.warn(String.format("duplicate content provider found: '%s'", o1.getId()));
-				return o2;
-			}, TreeMap::new));
-	}
-
-	public CommunityOrganization addCommunityOrganization(final String id, final CommunityOrganization organization)
-		throws CommunityException, ResourceNotFoundException {
-		if (!StringUtils.equalsIgnoreCase(id, organization.getCommunityId())) {
-			throw new CommunityException("parameters 'id' and organization.communityId must be coherent");
-		}
-
-		final TreeMap<Integer, CommunityOrganization> cps = getCommunityOrganizationMap(id);
-
-		final String organization_id = organization.getId();
-		if (organization_id != null && cps.keySet().contains(Integer.valueOf(organization_id))) {
-			if (organization.getName() != null) {
-				isClient.updateConceptParam(id + ORGANIZATION_ID_SUFFIX + ID_SEPARATOR + organization_id, CORGANIZATION_NAME, organization.getName());
-			}
-			if (organization.getLogo_url() != null) {
-				isClient.updateConceptParam(id + ORGANIZATION_ID_SUFFIX + ID_SEPARATOR + organization_id, CORGANIZATION_LOGOURL, Base64.getEncoder()
-					.encodeToString(organization.getLogo_url().getBytes()));
-			}
-			if (organization.getWebsite_url() != null) {
-				isClient.updateConceptParam(id + ORGANIZATION_ID_SUFFIX + ID_SEPARATOR + organization_id, CORGANIZATION_WEBSITEURL, Base64.getEncoder()
-					.encodeToString(organization.getWebsite_url().getBytes()));
-			}
-		} else {
-			organization.setId(nextId(!cps.isEmpty() ? cps.lastKey() : 0));
-			isClient.addConcept(id, id + ORGANIZATION_ID_SUFFIX, CommunityMappingUtils.asOrganizationXML(id, organization));
-		}
-
-		cc.updateOrganization(id, organization);
-		return organization;
-	}
-
-
-}

apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityClient.java

@@ -1,15 +0,0 @@
-package eu.dnetlib.openaire.community;
-
-import java.util.Map;
-import java.util.Set;
-
-import eu.dnetlib.openaire.exporter.exceptions.CommunityException;
-import eu.dnetlib.openaire.exporter.exceptions.ResourceNotFoundException;
-
-public interface CommunityClient {
-
-    Map<String, Set<String>> getInverseZenodoCommunityMap() throws CommunityException, ResourceNotFoundException;
-
-    void dropCache();
-
-}

apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityClientImpl.java

@@ -1,63 +0,0 @@
-package eu.dnetlib.openaire.community;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.cache.annotation.CacheEvict;
-import org.springframework.cache.annotation.Cacheable;
-import org.springframework.scheduling.annotation.Scheduled;
-import org.springframework.stereotype.Component;
-
-import eu.dnetlib.openaire.exporter.exceptions.CommunityException;
-import eu.dnetlib.openaire.exporter.exceptions.ResourceNotFoundException;
-import eu.dnetlib.openaire.exporter.model.community.CommunitySummary;
-import eu.dnetlib.openaire.exporter.model.community.CommunityZenodoCommunity;
-
-import java.util.*;
-
-@Component
-public class CommunityClientImpl implements CommunityClient {
-
-    private static final Log log = LogFactory.getLog(CommunityClient.class);
-
-    @Autowired
-    private CommunityCommon communityCommon;
-
-    @Override
-    @Cacheable("community-cache")
-    public Map<String, Set<String>> getInverseZenodoCommunityMap () throws CommunityException, ResourceNotFoundException {
-        log.info("Creating the data structure. Not using cache");
-        final Map<String, Set<String>> inverseListMap = new HashMap<>();
-
-        final List<CommunitySummary> communityList = communityCommon.listCommunities();
-
-        for(CommunitySummary cs :communityList){
-            final String communityId =  cs.getId();
-            List<CommunityZenodoCommunity> czc = communityCommon.getCommunityZenodoCommunities(communityId);
-            for(CommunityZenodoCommunity zc:czc){
-                final String zenodoId = zc.getZenodoid();
-                if(!inverseListMap.containsKey(zenodoId)) {
-                    inverseListMap.put(zc.getZenodoid(),new HashSet<>());
-                }
-                inverseListMap.get(zc.getZenodoid()).add(communityId);
-            }
-            final String zenodoMainCommunity = communityCommon.getCommunity(communityId).getZenodoCommunity();
-            if(!inverseListMap.containsKey(zenodoMainCommunity)) {
-                inverseListMap.put(zenodoMainCommunity,new HashSet<>());
-            }
-
-            inverseListMap.get(zenodoMainCommunity).add(communityId);
-        }
-        return inverseListMap;
-    }
-
-
-
-    @Override
-    @CacheEvict(cacheNames = { "community-cache", "context-cache-community"}, allEntries = true)
-    @Scheduled(fixedDelayString = "${openaire.exporter.cache.ttl}")
-    public void dropCache(){
-        log.debug("dropped community cache");
-    }
-
-}

apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityCommon.java

@@ -1,489 +0,0 @@
-package eu.dnetlib.openaire.community;
-
-import com.google.common.base.Joiner;
-import com.google.common.collect.Lists;
-import com.google.gson.Gson;
-import eu.dnetlib.openaire.common.ISClient;
-import eu.dnetlib.openaire.exporter.exceptions.CommunityException;
-import eu.dnetlib.openaire.exporter.exceptions.ResourceNotFoundException;
-import eu.dnetlib.openaire.exporter.model.community.CommunityContentprovider;
-import eu.dnetlib.openaire.exporter.model.community.CommunityDetails;
-import eu.dnetlib.openaire.exporter.model.community.CommunityOrganization;
-import eu.dnetlib.openaire.exporter.model.community.CommunityProject;
-import eu.dnetlib.openaire.exporter.model.community.CommunitySummary;
-import eu.dnetlib.openaire.exporter.model.community.CommunityWritableProperties;
-import eu.dnetlib.openaire.exporter.model.community.CommunityZenodoCommunity;
-import eu.dnetlib.openaire.exporter.model.context.Category;
-import eu.dnetlib.openaire.exporter.model.context.Concept;
-import eu.dnetlib.openaire.exporter.model.context.Context;
-import eu.dnetlib.openaire.exporter.model.context.Param;
-
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.stereotype.Component;
-
-import java.io.IOException;
-import java.util.*;
-import java.util.function.Function;
-import java.util.stream.Collectors;
-
-import static eu.dnetlib.openaire.community.CommunityConstants.*;
-
-
-@Component
-public class CommunityCommon {
-
-    @Autowired
-    private ISClient isClient;
-
-    public Map<String, Context> getContextMap() throws CommunityException {
-        try {
-            return isClient.getCommunityContextMap();
-        } catch (IOException e) {
-            throw new CommunityException(e);
-        }
-    }
-
-    public List<CommunitySummary> listCommunities()  throws CommunityException {
-        return getContextMap().values().stream()
-                .filter(context -> !communityBlackList.contains(context.getId()))
-                .map(CommunityMappingUtils::asCommunitySummary)
-                .collect(Collectors.toList());
-    }
-
-
-    public <R> List<R> getCommunityInfo(final String id, final String idSuffix, final Function<Concept, R> mapping) throws CommunityException {
-        final Map<String, Context> contextMap = getContextMap();
-        final Context context = contextMap.get(id);
-        if (context != null) {
-            final Map<String, Category> categories = context.getCategories();
-            final Category category = categories.get(id + idSuffix);
-            if (category != null) {
-                return category.getConcepts().stream()
-                        .map(mapping)
-                        .collect(Collectors.toList());
-            }
-        }
-        return Lists.newArrayList();
-    }
-
-    public CommunityDetails getCommunity(final String id) throws CommunityException, ResourceNotFoundException {
-        final Context context = getContextMap().get(id);
-        if (context == null || CommunityConstants.communityBlackList.contains(id)) {
-            //ResponseStatusException(NOT_FOUND, "Unable to find resource");
-            throw new ResourceNotFoundException();
-        }
-        return CommunityMappingUtils.asCommunityProfile(context);
-    }
-
-    public List<CommunityZenodoCommunity> getCommunityZenodoCommunities(final String id) throws CommunityException, ResourceNotFoundException {
-        getCommunity(id); // ensure the community exists.
-        return getCommunityInfo(id, ZENODOCOMMUNITY_ID_SUFFIX, c -> CommunityMappingUtils.asCommunityZenodoCommunity(id, c));
-    }
-
-
-    public void updateProject(String communityId, CommunityProject project) throws CommunityException {
-        final Context context = getContextMap().get(communityId);
-        Category prj = context.getCategories().get(communityId + PROJECTS_ID_SUFFIX);
-        if (prj.getConcepts().stream().map(c -> c.getId()).collect(Collectors.toList())
-                .contains(communityId + PROJECTS_ID_SUFFIX + ID_SEPARATOR + project.getId())){
-            prj.getConcepts().forEach(concept -> {
-                if (concept.getId().equals(communityId + PROJECTS_ID_SUFFIX + ID_SEPARATOR + project.getId())) {
-                    if (project.getName() != null) {
-
-                        concept.getParams().replace(CPROJECT_FULLNAME, Arrays.asList(new Param()
-                                .setName(CPROJECT_FULLNAME).setValue(project.getName())));
-                    }
-                    if (project.getAcronym() != null) {
-                        if(concept.getParams().keySet().contains(CPROJECT_ACRONYM)){
-                            concept.getParams().replace(CPROJECT_ACRONYM, Arrays.asList(new Param()
-                                    .setName(CPROJECT_ACRONYM).setValue(project.getAcronym())));
-                        }
-                        else{
-                            concept.getParams().put(CPROJECT_ACRONYM, Arrays.asList(new Param()
-                                    .setName(CPROJECT_ACRONYM).setValue(project.getAcronym())));
-                        }
-
-                    }
-                    if (project.getOpenaireId() != null) {
-                        if(concept.getParams().keySet().contains(OPENAIRE_ID)){
-                            concept.getParams().replace(OPENAIRE_ID, Arrays.asList(new Param()
-                                    .setName(OPENAIRE_ID).setValue(project.getOpenaireId())));
-                        }
-                        else{
-                            concept.getParams().put(OPENAIRE_ID, Arrays.asList(new Param()
-                            .setName(OPENAIRE_ID).setValue(project.getOpenaireId())));
-                        }
-
-                    }
-                    if (project.getFunder() != null) {
-                        concept.getParams().replace(CPROJECT_FUNDER, Arrays.asList(new Param()
-                                .setName(CPROJECT_FUNDER).setValue(project.getFunder())));
-                    }
-                    if (project.getGrantId() != null) {
-                        concept.getParams().replace(CPROJECT_NUMBER, Arrays.asList(new Param()
-                                .setName(CPROJECT_NUMBER).setValue(project.getGrantId())));
-
-                    }
-                }
-            });
-        }
-        else{
-            Concept concept = new Concept();
-            concept.setId(communityId + PROJECTS_ID_SUFFIX + ID_SEPARATOR + project.getId());
-            concept.setClaim(false);
-            if(project.getAcronym() != null)
-                concept.setLabel(project.getAcronym());
-            else
-                concept.setLabel("");
-
-             Map<String, List<Param>> params = new TreeMap<>();
-
-             if(project.getAcronym() != null){
-                 params.put(CPROJECT_ACRONYM, Arrays.asList(new Param().setName(CPROJECT_ACRONYM)
-                         .setValue(project.getAcronym())));
-             }
-
-             if (project.getName() != null){
-                 params.put(CPROJECT_FULLNAME, Arrays.asList(new Param()
-                         .setName(CPROJECT_FULLNAME)
-                         .setValue(project.getName())
-                 ));
-             }
-
-             if (project.getOpenaireId() != null){
-                 params.put(OPENAIRE_ID, Arrays.asList(new Param()
-                         .setName(OPENAIRE_ID)
-                         .setValue(project.getOpenaireId())
-                 ));
-             }
-
-             if(project.getFunder() != null){
-                 params.put(CPROJECT_FUNDER, Arrays.asList(new Param()
-                         .setName(CPROJECT_FUNDER)
-                         .setValue(project.getFunder())
-                 ));
-             }
-
-             if (project.getGrantId()!=null){
-                 params.put(CPROJECT_NUMBER, Arrays.asList(new Param()
-                         .setName(CPROJECT_NUMBER)
-                         .setValue(project.getGrantId())
-                 ));
-             }
-
-            concept.setParams(params);
-            prj.getConcepts().add(concept);
-
-
-        }
-
-    }
-
-    public void removeAdvancedConstraint(String id) throws CommunityException {
-        final Context context = getContextMap().get(id);
-        context.getParams()
-                .replace(CPROFILE_ADVANCED_CONSTRAINT, Arrays.asList(new Param()
-                        .setName(CPROFILE_ADVANCED_CONSTRAINT).setValue(null)));
-    }
-
-    public void updateCommunity(String id, CommunityWritableProperties community) throws CommunityException {
-        final Context context = getContextMap().get(id);
-
-        if(community.getShortName() != null) {
-            context.setLabel(community.getShortName());
-        }
-
-        if (community.getName() != null){
-            context.getParams().replace(CSUMMARY_NAME, Arrays.asList(new Param()
-            .setValue(community.getName()).setName(CSUMMARY_NAME)));
-        }
-        if(community.getDescription() != null) {
-            context.getParams()
-                    .replace(CSUMMARY_DESCRIPTION, Arrays.asList(new Param()
-                            .setName(CSUMMARY_DESCRIPTION).setValue(community.getDescription())));
-        }
-        if(community.getLogoUrl() != null){
-            context.getParams()
-                    .replace(CSUMMARY_LOGOURL, Arrays.asList(new Param()
-                    .setName(CSUMMARY_LOGOURL).setValue(community.getLogoUrl())));
-
-        }
-        if (community.getStatus() != null) {
-            context.getParams()
-                    .replace(CSUMMARY_STATUS, Arrays.asList(new Param()
-                    .setName(CSUMMARY_STATUS).setValue(community.getStatus().name())));
-        }
-        if (community.getSubjects() != null) {
-            context.getParams()
-                    .replace(CPROFILE_SUBJECT, Arrays.asList(new Param().setName(CPROFILE_SUBJECT)
-                            .setValue(Joiner.on(CSV_DELIMITER)
-                                    .join(community.getSubjects()))));
-        }
-        if(community.getFos() != null){
-            if (context.getParams().containsKey(CPROFILE_FOS))
-            context.getParams()
-                    .replace(CPROFILE_FOS, Arrays.asList(new Param().setName(CPROFILE_FOS)
-                            .setValue(Joiner.on(CSV_DELIMITER)
-                                    .join(community.getFos()))));
-            else
-                context.getParams().put(CPROFILE_FOS, Arrays.asList(new Param().setName(CPROFILE_FOS)
-                        .setValue(Joiner.on(CSV_DELIMITER)
-                                .join(community.getFos()))));
-        }
-        if(community.getSdg() != null){
-            if(context.getParams().containsKey(CPROFILE_SDG))
-            context.getParams()
-                    .replace(CPROFILE_SDG, Arrays.asList(new Param().setName(CPROFILE_SDG)
-                            .setValue(Joiner.on(CSV_DELIMITER)
-                                    .join(community.getSdg()))));
-            else
-                context.getParams().put(CPROFILE_SDG, Arrays.asList(new Param().setName(CPROFILE_SDG)
-                        .setValue(Joiner.on(CSV_DELIMITER)
-                                .join(community.getSdg()))));
-        }
-        if (community.getAdvancedConstraints() != null) {
-            if(context.getParams().containsKey(CPROFILE_ADVANCED_CONSTRAINT))
-            context.getParams()
-                    .replace(CPROFILE_ADVANCED_CONSTRAINT, Arrays.asList(new Param()
-                            .setName(CPROFILE_ADVANCED_CONSTRAINT).setValue(new Gson().toJson(community.getAdvancedConstraints()))));
-            else
-                context.getParams().put(CPROFILE_ADVANCED_CONSTRAINT, Arrays.asList(new Param()
-                        .setName(CPROFILE_ADVANCED_CONSTRAINT).setValue(new Gson().toJson(community.getAdvancedConstraints()))));
-        }
-        if(community.getMainZenodoCommunity() != null){
-            context.getParams()
-                    .replace(CSUMMARY_ZENODOC, Arrays.asList(new Param()
-                            .setName(CSUMMARY_ZENODOC).setValue(community.getMainZenodoCommunity())));
-
-        }
-
-    }
-
-
-    public void removeFromCategory(String communityId, String category, String conceptId) throws CommunityException {
-        Map<String, Context> cmap = getContextMap();
-        Context context = cmap.get(communityId);
-        Map<String, Category> cat = context.getCategories();
-
-        List<Concept> concepts = cat.get(communityId + category).getConcepts()
-                .stream().filter(c -> !c.getId().equals(communityId + category + ID_SEPARATOR + conceptId)).collect(Collectors.toList());
-
-         cat.get(communityId + category).setConcepts(concepts);
-    }
-
-    public void updateDatasource(String communityId, CommunityContentprovider cp) throws CommunityException {
-        final Context context = getContextMap().get(communityId);
-        Category dts = context.getCategories().get(communityId + CONTENTPROVIDERS_ID_SUFFIX);
-        if (dts.getConcepts().stream().map(c -> c.getId()).collect(Collectors.toList())
-                .contains(communityId + CONTENTPROVIDERS_ID_SUFFIX + ID_SEPARATOR + cp.getId())){
-            dts.getConcepts().forEach(concept -> {
-                if (concept.getId().equals(communityId + CONTENTPROVIDERS_ID_SUFFIX + ID_SEPARATOR + cp.getId())) {
-
-
-                    if (cp.getName() != null) {
-                        if(concept.getParams().keySet().contains(CCONTENTPROVIDER_NAME)){
-                            concept.getParams().replace(CCONTENTPROVIDER_NAME, Arrays.asList(new Param()
-                                    .setName(CCONTENTPROVIDER_NAME).setValue(cp.getName())));
-                        }
-                        else{
-                            concept.getParams().put(CCONTENTPROVIDER_NAME, Arrays.asList(new Param()
-                                    .setName(CCONTENTPROVIDER_NAME).setValue(cp.getName())));
-                        }
-
-                    }
-                    if (cp.getOfficialname() != null) {
-                        if(concept.getParams().keySet().contains(CCONTENTPROVIDER_OFFICIALNAME)){
-                            concept.getParams().replace(CCONTENTPROVIDER_OFFICIALNAME, Arrays.asList(new Param()
-                                    .setName(CCONTENTPROVIDER_OFFICIALNAME).setValue(cp.getOfficialname())));
-                        }
-                        else{
-                            concept.getParams().put(CCONTENTPROVIDER_OFFICIALNAME, Arrays.asList(new Param()
-                                    .setName(CCONTENTPROVIDER_OFFICIALNAME).setValue(cp.getOfficialname())));
-                        }
-
-                    }
-
-
-                    if (cp.getOpenaireId() != null) {
-                        if(concept.getParams().keySet().contains(OPENAIRE_ID)){
-                            concept.getParams().replace(OPENAIRE_ID, Arrays.asList(new Param()
-                                    .setName(OPENAIRE_ID).setValue(cp.getOpenaireId())));
-                        }
-                        else{
-                            concept.getParams().put(OPENAIRE_ID, Arrays.asList(new Param()
-                                    .setName(OPENAIRE_ID).setValue(cp.getOpenaireId())));
-                        }
-
-                    }
-
-                    if (cp.getSelectioncriteria()  != null) {
-                        if(concept.getParams().keySet().contains(CCONTENTPROVIDER_SELCRITERIA)){
-                            concept.getParams().replace(CCONTENTPROVIDER_SELCRITERIA, Arrays.asList(new Param()
-                                    .setName(CCONTENTPROVIDER_SELCRITERIA).setValue(cp.toJson())));
-                        }
-                        else{
-                            concept.getParams().put(CCONTENTPROVIDER_SELCRITERIA, Arrays.asList(new Param()
-                                    .setName(CCONTENTPROVIDER_SELCRITERIA).setValue(cp.toJson())));
-                        }
-
-                    }
-                }
-            });
-        }
-        else{
-            Concept concept = new Concept();
-            concept.setId(communityId + CONTENTPROVIDERS_ID_SUFFIX + ID_SEPARATOR + cp.getId());
-            concept.setClaim(false);
-            concept.setLabel("");
-
-            Map<String, List<Param>> params = new TreeMap<>();
-
-            if (cp.getName() != null) {
-                params.put( CCONTENTPROVIDER_NAME, Arrays.asList(new Param().setValue(cp.getName()).setName(CCONTENTPROVIDER_NAME)));
-            }
-            if(cp.getOfficialname()!= null){
-                params.put( CCONTENTPROVIDER_OFFICIALNAME, Arrays.asList(new Param().setValue(cp.getOfficialname()).setName(CCONTENTPROVIDER_OFFICIALNAME)));
-            }
-            if (cp.getOpenaireId() != null){
-                params.put( OPENAIRE_ID, Arrays.asList(new Param().setValue(cp.getOpenaireId()).setName(OPENAIRE_ID)));
-            }
-            if(cp.getSelectioncriteria() != null){
-                params.put( CCONTENTPROVIDER_SELCRITERIA, Arrays.asList(new Param().setValue(cp.toJson()).setName(CCONTENTPROVIDER_SELCRITERIA)));
-
-            }
-
-            concept.setParams(params);
-            dts.getConcepts().add(concept);
-
-
-        }
-
-    }
-
-    public void updateOrganization(String communityId, CommunityOrganization organization) throws CommunityException {
-
-        final Context context = getContextMap().get(communityId);
-        Category orgs = context.getCategories().get(communityId + ORGANIZATION_ID_SUFFIX);
-        if (orgs.getConcepts().stream().map(c -> c.getId()).collect(Collectors.toList())
-                .contains(communityId + ORGANIZATION_ID_SUFFIX + ID_SEPARATOR + organization.getId())){
-            orgs.getConcepts().forEach(concept -> {
-                if (concept.getId().equals(communityId + ORGANIZATION_ID_SUFFIX + ID_SEPARATOR + organization.getId())) {
-
-
-                    if (organization.getName() != null) {
-                        if(concept.getParams().keySet().contains(CORGANIZATION_NAME)){
-                            concept.getParams().replace(CORGANIZATION_NAME, Arrays.asList(new Param()
-                                    .setName(CORGANIZATION_NAME).setValue(organization.getName())));
-                        }
-                        else{
-                            concept.getParams().put(CORGANIZATION_NAME, Arrays.asList(new Param()
-                                    .setName(CORGANIZATION_NAME).setValue(organization.getName())));
-                        }
-
-                    }
-                    if (organization.getLogo_url() != null) {
-                        if(concept.getParams().keySet().contains(CORGANIZATION_LOGOURL)){
-                            concept.getParams().replace(CORGANIZATION_LOGOURL, Arrays.asList(new Param()
-                                    .setName(CORGANIZATION_LOGOURL).setValue(Base64.getEncoder().encodeToString(organization.getLogo_url().getBytes()))));
-                        }
-                        else{
-                            concept.getParams().put(CORGANIZATION_LOGOURL, Arrays.asList(new Param()
-                                    .setName(CORGANIZATION_LOGOURL).setValue(Base64.getEncoder().encodeToString(organization.getLogo_url().getBytes()))));
-                        }
-
-                    }
-
-
-                    if (organization.getWebsite_url() != null) {
-                        if(concept.getParams().keySet().contains(CORGANIZATION_WEBSITEURL)){
-                            concept.getParams().replace(CORGANIZATION_WEBSITEURL, Arrays.asList(new Param()
-                                    .setName(CORGANIZATION_WEBSITEURL).setValue(Base64.getEncoder().encodeToString(organization.getWebsite_url().getBytes()))));
-                        }
-                        else{
-                            concept.getParams().put(CORGANIZATION_WEBSITEURL, Arrays.asList(new Param()
-                                    .setName(CORGANIZATION_WEBSITEURL).setValue(Base64.getEncoder().encodeToString(organization.getWebsite_url().getBytes()))));
-                        }
-
-                    }
-
-                    }
-
-            });
-        }
-        else{
-
-            Concept concept = new Concept();
-            concept.setId(communityId + ORGANIZATION_ID_SUFFIX + ID_SEPARATOR + organization.getId());
-            concept.setClaim(false);
-            concept.setLabel("");
-
-            Map<String, List<Param>> params = new TreeMap<>();
-
-            if (organization.getName() != null) {
-                params.put( CORGANIZATION_NAME, Arrays.asList(new Param().setValue(organization.getName()).setName(CORGANIZATION_NAME)));
-            }
-            if(organization.getLogo_url()!= null){
-
-                params.put( CORGANIZATION_LOGOURL, Arrays.asList(new Param().setValue(Base64.getEncoder().encodeToString(organization.getLogo_url().getBytes())).setName(CORGANIZATION_LOGOURL)));
-            }
-            if (organization.getWebsite_url() != null){
-                params.put( CORGANIZATION_WEBSITEURL, Arrays.asList(new Param().setValue(Base64.getEncoder().encodeToString(organization.getWebsite_url().getBytes())).setName(CORGANIZATION_WEBSITEURL)));
-            }
-
-
-            concept.setParams(params);
-            orgs.getConcepts().add(concept);
-
-
-        }
-
-    }
-
-    public void updateZenodoCommunity(String communityId, CommunityZenodoCommunity zc) throws CommunityException {
-        final Context context = getContextMap().get(communityId);
-        Category zcs = context.getCategories().get(communityId + ZENODOCOMMUNITY_ID_SUFFIX);
-        if (zcs.getConcepts().stream().map(c -> c.getId()).collect(Collectors.toList())
-                .contains(communityId + ZENODOCOMMUNITY_ID_SUFFIX + ID_SEPARATOR + zc.getId())){
-            zcs.getConcepts().forEach(concept -> {
-                if (concept.getId().equals(communityId + ZENODOCOMMUNITY_ID_SUFFIX + ID_SEPARATOR + zc.getId())) {
-
-
-                    if (zc.getZenodoid() != null) {
-                        if(concept.getParams().keySet().contains(CZENODOCOMMUNITY_ID)){
-                            concept.getParams().replace(CZENODOCOMMUNITY_ID, Arrays.asList(new Param()
-                                    .setName(CZENODOCOMMUNITY_ID).setValue(zc.getZenodoid())));
-                        }
-                        else{
-                            concept.getParams().put(CZENODOCOMMUNITY_ID, Arrays.asList(new Param()
-                                    .setName(CZENODOCOMMUNITY_ID).setValue(zc.getZenodoid())));
-                        }
-
-                    }
-
-                }
-
-            });
-        }
-        else{
-
-            Concept concept = new Concept();
-            concept.setId(communityId + ZENODOCOMMUNITY_ID_SUFFIX + ID_SEPARATOR + zc.getId());
-            concept.setClaim(false);
-
-
-            Map<String, List<Param>> params = new TreeMap<>();
-
-            if (zc.getZenodoid() != null) {
-                params.put( CZENODOCOMMUNITY_ID, Arrays.asList(new Param().setValue(zc.getZenodoid()).setName(CZENODOCOMMUNITY_ID)));
-                concept.setLabel(zc.getZenodoid());
-            }else{
-                concept.setLabel("");
-            }
-            concept.setParams(params);
-            zcs.getConcepts().add(concept);
-
-        }
-
-    }
-
-}

apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityConstants.java

@@ -1,61 +0,0 @@
-package eu.dnetlib.openaire.community;
-
-import java.util.Set;
-
-import com.google.common.collect.Sets;
-
-public class CommunityConstants {
-
-	public final static Set<String> communityBlackList = Sets.newHashSet("fet-fp7", "fet-h2020");
-
-	// common
-	public final static String OPENAIRE_ID = "openaireId";
-	public final static String PIPE_SEPARATOR = "||";
-	public final static String ID_SEPARATOR = "::";
-	public final static String CSV_DELIMITER = ",";
-	public final static String CLABEL = "label";
-
-	// id suffixes
-	public final static String PROJECTS_ID_SUFFIX = ID_SEPARATOR + "projects";
-	public final static String CONTENTPROVIDERS_ID_SUFFIX = ID_SEPARATOR + "contentproviders";
-	public final static String ZENODOCOMMUNITY_ID_SUFFIX = ID_SEPARATOR + "zenodocommunities";
-	public final static String ORGANIZATION_ID_SUFFIX = ID_SEPARATOR + "organizations";
-
-	// community summary
-	public final static String CSUMMARY_DESCRIPTION = "description";
-	public final static String CSUMMARY_LOGOURL = "logourl";
-	public final static String CSUMMARY_STATUS = "status";
-	public final static String CSUMMARY_NAME = "name";
-	public final static String CSUMMARY_MANAGER = "manager";
-	public final static String CSUMMARY_ZENODOC = "zenodoCommunity";
-
-	// community profile
-	public final static String CPROFILE_SUBJECT = "subject";
-	public final static String CPROFILE_CREATIONDATE = "creationdate";
-	public final static String CPROFILE_FOS = "fos";
-	public final static String CPROFILE_SDG = "sdg";
-	public final static String CPROFILE_ADVANCED_CONSTRAINT = "advancedConstraints";
-
-	// community project
-	public final static String CPROJECT_FUNDER = "funder";
-	public final static String CPROJECT_NUMBER = "CD_PROJECT_NUMBER";
-	public final static String CPROJECT_FULLNAME = "projectfullname";
-	public final static String CPROJECT_ACRONYM = "acronym";
-
-	// community content provider
-	public final static String CCONTENTPROVIDER_NAME = "name";
-	public final static String CCONTENTPROVIDER_OFFICIALNAME = "officialname";
-	public final static String CCONTENTPROVIDER_ENABLED = "enabled";
-	public final static String CCONTENTPROVIDERENABLED_DEFAULT = "true";
-	public final static String CCONTENTPROVIDER_SELCRITERIA = "selcriteria";
-
-	//community zenodo community
-	public final static String CZENODOCOMMUNITY_ID = "zenodoid";
-
-	//community organization
-	public final static String CORGANIZATION_NAME = "name";
-	public final static String CORGANIZATION_LOGOURL = "logourl";
-	public final static String CORGANIZATION_WEBSITEURL = "websiteurl";
-
-
-}

apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityMappingUtils.java

@@ -1,251 +0,0 @@
-package eu.dnetlib.openaire.community;
-
-import java.text.ParseException;
-import java.util.*;
-import java.util.stream.Collectors;
-import java.util.stream.Stream;
-
-import com.google.common.escape.Escaper;
-import com.google.common.xml.XmlEscapers;
-
-import eu.dnetlib.openaire.exporter.model.community.CommunityContentprovider;
-import eu.dnetlib.openaire.exporter.model.community.CommunityDetails;
-import eu.dnetlib.openaire.exporter.model.community.CommunityOrganization;
-import eu.dnetlib.openaire.exporter.model.community.CommunityProject;
-import eu.dnetlib.openaire.exporter.model.community.CommunityStatus;
-import eu.dnetlib.openaire.exporter.model.community.CommunitySummary;
-import eu.dnetlib.openaire.exporter.model.community.CommunityZenodoCommunity;
-import eu.dnetlib.openaire.exporter.model.community.selectioncriteria.SelectionCriteria;
-import eu.dnetlib.openaire.exporter.model.context.Concept;
-import eu.dnetlib.openaire.exporter.model.context.Context;
-import eu.dnetlib.openaire.exporter.model.context.Param;
-
-import org.apache.commons.lang3.StringUtils;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-
-import static eu.dnetlib.openaire.common.Utils.escape;
-import static eu.dnetlib.openaire.community.CommunityConstants.*;
-
-public class CommunityMappingUtils {
-
-	private final static String pattern = "yyyy-MM-dd'T'hh:mm:ss";
-
-	private static final Log log = LogFactory.getLog(CommunityMappingUtils.class);
-
-	public static CommunitySummary asCommunitySummary(final Context c) {
-		final CommunitySummary summary = new CommunitySummary();
-
-		summary.setId(c.getId());
-		summary.setShortName(c.getLabel());
-		summary.setLastUpdateDate(c.getLastUpdateDate());
-		summary.setCreationDate(c.getCreationDate());
-		summary.setQueryId(c.getId() + PIPE_SEPARATOR + c.getLabel());
-		summary.setType(c.getType());
-
-		final Map<String, List<Param>> params = c.getParams();
-		if (params.containsKey(CSUMMARY_DESCRIPTION)) {
-			summary.setDescription(asCsv(params.get(CSUMMARY_DESCRIPTION)));
-		}
-		if (params.containsKey(CSUMMARY_LOGOURL)) {
-			summary.setLogoUrl(asCsv(params.get(CSUMMARY_LOGOURL)));
-		}
-		if (params.containsKey(CSUMMARY_STATUS)) {
-			summary.setStatus(CommunityStatus.valueOf(firstValue(params, CSUMMARY_STATUS)));
-		}
-		if (params.containsKey(CSUMMARY_NAME)) {
-			summary.setName(asCsv(params.get(CSUMMARY_NAME)));
-		}
-		if (params.containsKey(CSUMMARY_ZENODOC)) {
-			summary.setZenodoCommunity(asCsv(params.get(CSUMMARY_ZENODOC)));
-		}
-
-		return summary;
-	}
-
-	public static CommunityDetails asCommunityProfile(final Context c)  {
-
-		final CommunityDetails p = new CommunityDetails(asCommunitySummary(c));
-		p.setLastUpdateDate(c.getLastUpdateDate());
-		final Map<String, List<Param>> params = c.getParams();
-		if (params.containsKey(CPROFILE_SUBJECT)) {
-			p.setSubjects(splitValues(asValues(params.get(CPROFILE_SUBJECT)), CSV_DELIMITER));
-		}
-		if (params.containsKey(CPROFILE_FOS)) {
-			p.setFos(splitValues(asValues(params.get(CPROFILE_FOS)), CSV_DELIMITER));
-		}
-		if (params.containsKey(CPROFILE_SDG)) {
-			p.setSdg(splitValues(asValues(params.get(CPROFILE_SDG)), CSV_DELIMITER));
-		}
-		if (params.containsKey(CPROFILE_ADVANCED_CONSTRAINT)) {
-			//In the map the string is the serialization of the json representing the selection criteria so it is a valid json
-			p.setAdvancedConstraints(SelectionCriteria.fromJson(asCsv(params.get(CPROFILE_ADVANCED_CONSTRAINT))));
-
-		}
-		if (params.containsKey(CPROFILE_CREATIONDATE)){
-			try {
-				p.setCreationDate(org.apache.commons.lang3.time.DateUtils.parseDate(asCsv(params.get(CPROFILE_CREATIONDATE)), pattern));
-			}catch(ParseException e) {
-				log.debug("Exception on date format: " + e.getMessage());
-			}
-		}
-
-		return p;
-	}
-
-	public static CommunityProject asCommunityProject(final String communityId, final Concept c) {
-
-		final Map<String, List<Param>> p = c.getParams();
-		final CommunityProject project = new CommunityProject();
-		project.setCommunityId(communityId);
-		project.setId(StringUtils.substringAfterLast(c.getId(), ID_SEPARATOR));
-		project.setOpenaireId(firstValue(p, OPENAIRE_ID));
-		project.setFunder(firstValue(p, CPROJECT_FUNDER));
-		project.setGrantId(firstValue(p, CPROJECT_NUMBER));
-		project.setName(firstValue(p, CPROJECT_FULLNAME));
-		project.setAcronym(firstValue(p, CPROJECT_ACRONYM));
-
-		return project;
-	}
-
-	public static CommunityContentprovider asCommunityDataprovider(final String communityId, final Concept c) {
-
-		final Map<String, List<Param>> p = c.getParams();
-		final CommunityContentprovider d = new CommunityContentprovider();
-		d.setCommunityId(communityId);
-		d.setId(StringUtils.substringAfterLast(c.getId(), ID_SEPARATOR));
-		d.setOpenaireId(firstValue(p, OPENAIRE_ID));
-		d.setName(firstValue(p, CCONTENTPROVIDER_NAME));
-		d.setOfficialname(firstValue(p, CCONTENTPROVIDER_OFFICIALNAME));
-		d.setSelectioncriteria(SelectionCriteria.fromJson(firstValue(p, CCONTENTPROVIDER_SELCRITERIA)));
-		return d;
-	}
-
-	public static CommunityZenodoCommunity asCommunityZenodoCommunity(final String communityId, final Concept c){
-		final CommunityZenodoCommunity z = new CommunityZenodoCommunity();
-		final Map<String, List<Param>> p = c.getParams();
-		z.setCommunityId(communityId);
-		z.setId(StringUtils.substringAfterLast(c.getId(), ID_SEPARATOR));
-		z.setZenodoid(firstValue(p,CZENODOCOMMUNITY_ID));
-		//z.setName(c.getLabel());
-		return z;
-	}
-
-
-	public static CommunityOrganization asCommunityOrganization(String id, Concept c) {
-		final Map<String, List<Param>> p = c.getParams();
-		final CommunityOrganization o = new CommunityOrganization();
-		o.setCommunityId(id);
-		o.setId(StringUtils.substringAfterLast(c.getId(), ID_SEPARATOR));
-		o.setName(firstValue(p,CORGANIZATION_NAME));
-		o.setLogo_url(getDecodedUrl(firstValue(p,CORGANIZATION_LOGOURL)));
-		o.setWebsite_url(getDecodedUrl(firstValue(p,CORGANIZATION_WEBSITEURL)));
-
-		return o;
-	}
-
-	private static String getDecodedUrl(final String encoded_url){
-		if(encoded_url == null){
-			return encoded_url;
-		}
-		return new String(Base64.getDecoder().decode(encoded_url));
-	}
-
-
-	private static List<String> splitValues(final Stream<String> stream, final String separator) {
-		return stream.map(s -> s.split(separator))
-				.map(Arrays::asList)
-				.flatMap(List::stream)
-				.filter(StringUtils::isNotBlank)
-				.map(StringUtils::trim)
-				.collect(Collectors.toList());
-	}
-
-	private static String firstValue(final Map<String, List<Param>> p, final String paramName) {
-		return asValues(p.get(paramName)).findFirst().orElse(null);
-	}
-
-	private static String asCsv(final List<Param> params) {
-		return asValues(params)
-				.collect(Collectors.joining(CSV_DELIMITER));
-	}
-
-	private static Stream<String> asValues(final List<Param> params) {
-		return params == null ? Stream.empty() : params.stream()
-				.map(Param::getValue)
-				.map(StringUtils::trim)
-				.distinct();
-	}
-
-	public static String asProjectXML(final String contextId, final CommunityProject project) {
-		final Escaper esc = XmlEscapers.xmlAttributeEscaper();
-		final StringBuilder sb = new StringBuilder();
-		sb.append(
-				String.format(
-						"<concept claim='false' id='%s%s%s%s' label='%s'>\n",
-						escape(esc, contextId), PROJECTS_ID_SUFFIX, ID_SEPARATOR, escape(esc, String.valueOf(project.getId())), escape(esc, project.getAcronym())));
-		sb.append(paramXML(CPROJECT_FULLNAME, project.getName()));
-		sb.append(paramXML(CPROJECT_ACRONYM, project.getAcronym()));
-		sb.append(paramXML(CPROJECT_NUMBER, project.getGrantId()));
-		sb.append(paramXML(CPROJECT_FUNDER, project.getFunder()));
-		sb.append(paramXML(OPENAIRE_ID, project.getOpenaireId()));
-		sb.append("</concept>\n");
-		return sb.toString();
-	}
-
-	public static String asContentProviderXML(final String contextId, final CommunityContentprovider ccp) {
-		log.info("creating the XML for the content provider");
-		final Escaper esc = XmlEscapers.xmlAttributeEscaper();
-		final StringBuilder sb = new StringBuilder();
-		sb.append(
-				String.format(
-						"<concept claim='false' id='%s%s%s%s' label='%s'>\n",
-						escape(esc, contextId), CONTENTPROVIDERS_ID_SUFFIX, ID_SEPARATOR, escape(esc, String.valueOf(ccp.getId())), escape(esc, ccp.getName())));
-		sb.append(paramXML(OPENAIRE_ID, ccp.getOpenaireId()));
-		sb.append(paramXML(CCONTENTPROVIDER_NAME, ccp.getName()));
-		sb.append(paramXML(CCONTENTPROVIDER_OFFICIALNAME, ccp.getOfficialname()));
-		sb.append(paramXML(CCONTENTPROVIDER_ENABLED,CCONTENTPROVIDERENABLED_DEFAULT));
-		sb.append(paramXMLNoEscape(CCONTENTPROVIDER_SELCRITERIA, ccp.toXML()));
-		sb.append("</concept>\n");
-		log.info(sb.toString());
-		return sb.toString();
-	}
-
-	public static String asZenodoCommunityXML(final String contextId, final CommunityZenodoCommunity zc) {
-		final Escaper esc = XmlEscapers.xmlAttributeEscaper();
-		final StringBuilder sb = new StringBuilder();
-		sb.append(
-				String.format(
-						"<concept claim='false' id='%s%s%s%s' label='%s'>\n",
-						escape(esc, contextId), ZENODOCOMMUNITY_ID_SUFFIX, ID_SEPARATOR, escape(esc, String.valueOf(zc.getId())), escape(esc, zc.getZenodoid())));
-
-		sb.append(paramXML(CZENODOCOMMUNITY_ID, zc.getZenodoid()));
-		sb.append("</concept>\n");
-		return sb.toString();
-	}
-
-
-	public static String asOrganizationXML(final String contextId, CommunityOrganization organization) {
-		final Escaper esc = XmlEscapers.xmlAttributeEscaper();
-		final StringBuilder sb = new StringBuilder();
-		sb.append(
-				String.format(
-						"<concept claim='false' id='%s%s%s%s' label='%s'>\n",
-						escape(esc, contextId), ORGANIZATION_ID_SUFFIX, ID_SEPARATOR, escape(esc, String.valueOf(organization.getId())), escape(esc, organization.getName())));
-		sb.append(paramXML(CORGANIZATION_NAME, organization.getName()));
-		sb.append(paramXML(CORGANIZATION_LOGOURL, Base64.getEncoder().encodeToString(organization.getLogo_url().getBytes())));
-		sb.append(paramXML(CORGANIZATION_WEBSITEURL,Base64.getEncoder().encodeToString(organization.getWebsite_url().getBytes())));
-		sb.append("</concept>\n");
-		return sb.toString();
-	}
-
-
-	private static String paramXML(final String paramName, final String value) {
-		return String.format("<param name='%s'>%s</param>\n", paramName, escape(XmlEscapers.xmlContentEscaper(), value));
-	}
-
-	private static String paramXMLNoEscape(final String paramName, final String value) {
-		return String.format("<param name='%s'>%s</param>\n", paramName, value);
-	}
-
-}

apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityService.java

@@ -0,0 +1,479 @@
+package eu.dnetlib.openaire.community;
+
+import java.time.LocalDateTime;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.function.BiConsumer;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+
+import javax.persistence.criteria.Predicate;
+import javax.transaction.Transactional;
+
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
+import org.springframework.data.domain.Page;
+import org.springframework.data.domain.PageRequest;
+import org.springframework.data.domain.Sort;
+import org.springframework.data.jpa.domain.Specification;
+import org.springframework.stereotype.Service;
+
+import eu.dnetlib.openaire.community.model.DbCommunity;
+import eu.dnetlib.openaire.community.model.DbDatasource;
+import eu.dnetlib.openaire.community.model.DbDatasourcePK;
+import eu.dnetlib.openaire.community.model.DbOrganization;
+import eu.dnetlib.openaire.community.model.DbProject;
+import eu.dnetlib.openaire.community.model.DbProjectPK;
+import eu.dnetlib.openaire.community.model.DbSubCommunity;
+import eu.dnetlib.openaire.community.model.DbSupportOrg;
+import eu.dnetlib.openaire.community.model.DbSupportOrgPK;
+import eu.dnetlib.openaire.community.repository.DbCommunityRepository;
+import eu.dnetlib.openaire.community.repository.DbDatasourceRepository;
+import eu.dnetlib.openaire.community.repository.DbOrganizationRepository;
+import eu.dnetlib.openaire.community.repository.DbProjectRepository;
+import eu.dnetlib.openaire.community.repository.DbSubCommunityRepository;
+import eu.dnetlib.openaire.community.repository.DbSupportOrgRepository;
+import eu.dnetlib.openaire.community.utils.CommunityMappingUtils;
+import eu.dnetlib.openaire.exporter.exceptions.CommunityException;
+import eu.dnetlib.openaire.exporter.exceptions.ResourceNotFoundException;
+import eu.dnetlib.openaire.exporter.model.community.CommunityContentprovider;
+import eu.dnetlib.openaire.exporter.model.community.CommunityDetails;
+import eu.dnetlib.openaire.exporter.model.community.CommunityOrganization;
+import eu.dnetlib.openaire.exporter.model.community.CommunityProject;
+import eu.dnetlib.openaire.exporter.model.community.CommunitySummary;
+import eu.dnetlib.openaire.exporter.model.community.CommunityWritableProperties;
+import eu.dnetlib.openaire.exporter.model.community.SubCommunity;
+import eu.dnetlib.openaire.exporter.model.community.selectioncriteria.SelectionCriteria;
+import eu.dnetlib.openaire.exporter.model.context.IISConfigurationEntry;
+
+@Service
+@ConditionalOnProperty(value = "openaire.exporter.enable.community", havingValue = "true")
+public class CommunityService {
+
+	@Autowired
+	private DbCommunityRepository dbCommunityRepository;
+	@Autowired
+	private DbProjectRepository dbProjectRepository;
+	@Autowired
+	private DbDatasourceRepository dbDatasourceRepository;
+	@Autowired
+	private DbOrganizationRepository dbOrganizationRepository;
+	@Autowired
+	private DbSupportOrgRepository dbSupportOrgRepository;
+	@Autowired
+	private DbSubCommunityRepository dbSubCommunityRepository;
+
+	private static final Log log = LogFactory.getLog(CommunityService.class);
+
+	public List<CommunitySummary> listCommunities() {
+		return dbCommunityRepository.findAll()
+				.stream()
+				.map(CommunityMappingUtils::toCommunitySummary)
+				.collect(Collectors.toList());
+	}
+
+	@Transactional
+	public CommunityDetails newCommunity(final CommunityDetails details) throws CommunityException {
+		if (StringUtils.isBlank(details.getId())) { throw new CommunityException("Empty Id"); }
+		if (dbCommunityRepository.existsById(details.getId())) { throw new CommunityException("Community already exists: " + details.getId()); }
+		details.setCreationDate(LocalDateTime.now());
+		return saveCommunity(details);
+
+	}
+
+	@Transactional
+	public CommunityDetails saveCommunity(final CommunityDetails details) {
+		details.setLastUpdateDate(LocalDateTime.now());
+		dbCommunityRepository.save(CommunityMappingUtils.toCommunity(details));
+		return getCommunity(details.getId());
+	}
+
+	@Transactional
+	public CommunityDetails getCommunity(final String id) {
+		final DbCommunity c = dbCommunityRepository.findById(id).orElseThrow(() -> new ResourceNotFoundException("Community not found: " + id));
+		return CommunityMappingUtils.toCommunityDetails(c);
+	}
+
+	@Transactional
+	public void setCommunity(final String id, final CommunityWritableProperties details) {
+		final DbCommunity c = dbCommunityRepository.findById(id).orElseThrow(() -> new ResourceNotFoundException("Community not found: " + id));
+		CommunityMappingUtils.populateCommunity(c, details);
+		c.setLastUpdateDate(LocalDateTime.now());
+		dbCommunityRepository.save(c);
+	}
+
+	@Transactional
+	public Page<CommunityProject> getCommunityProjects(final String id,
+			final String funder,
+			final String filter,
+			final int page,
+			final int size,
+			final String orderBy) throws CommunityException {
+		if (StringUtils.isBlank(id)) { throw new CommunityException("Empty ID"); }
+		try {
+			final Sort sort;
+			if (StringUtils.isBlank(orderBy)) {
+				sort = Sort.by("projectName");
+			} else if ("funder".equalsIgnoreCase(orderBy)) {
+				sort = Sort.by("projectFunder").and(Sort.by("projectName"));
+			} else if ("grantId".equalsIgnoreCase(orderBy)) {
+				sort = Sort.by("projectCode");
+			} else if ("acronym".equalsIgnoreCase(orderBy)) {
+				sort = Sort.by("projectAcronym");
+			} else if ("openaireId".equalsIgnoreCase(orderBy)) {
+				sort = Sort.by("projectId");
+			} else {
+				sort = Sort.by("projectName");
+			}
+
+			final PageRequest pageable = PageRequest.of(page, size, sort);
+			if (StringUtils.isAllBlank(filter, funder)) {
+				return dbProjectRepository.findByCommunity(id, pageable).map(CommunityMappingUtils::toCommunityProject);
+			}
+			final Specification<DbProject> projSpec = prepareProjectSpec(id, funder, filter);
+			return dbProjectRepository.findAll(projSpec, pageable).map(CommunityMappingUtils::toCommunityProject);
+		} catch (final Throwable e) {
+			log.error(e);
+			throw new CommunityException(e);
+		}
+	}
+
+	private Specification<DbProject> prepareProjectSpec(final String community, final String funder, final String other) {
+		return (project, query, cb) -> {
+
+			final List<Predicate> andConds = new ArrayList<>();
+			andConds.add(cb.equal(project.get("community"), community));
+
+			if (StringUtils.isNotBlank(funder)) {
+				andConds.add(cb.equal(project.get("projectFunder"), funder));
+			}
+
+			if (StringUtils.isNotBlank(other)) {
+				final String s = other.toLowerCase().trim();
+
+				final List<Predicate> orConds = new ArrayList<>();
+				orConds.add(cb.equal(cb.lower(project.get("projectId")), s));
+				orConds.add(cb.equal(cb.lower(project.get("projectCode")), s));
+				orConds.add(cb.equal(cb.lower(project.get("projectAcronym")), s));
+				orConds.add(cb.like(cb.lower(project.get("projectName")), "%" + s + "%"));
+				if (StringUtils.isBlank(funder)) {
+					orConds.add(cb.equal(cb.lower(project.get("projectFunder")), s));
+				}
+
+				andConds.add(cb.or(orConds.toArray(new Predicate[orConds.size()])));
+			}
+
+			return cb.and(andConds.toArray(new Predicate[andConds.size()]));
+		};
+
+	}
+
+	@Transactional
+	public CommunityProject addCommunityProject(final String id, final CommunityProject project) {
+		final DbProject p = CommunityMappingUtils.toDbProject(id, project);
+		dbProjectRepository.save(p);
+		return project;
+	}
+
+	@Transactional
+	public void addCommunityProjects(final String id, final CommunityProject... projects) throws CommunityException {
+		try {
+			final List<DbProject> list = Arrays.stream(projects)
+					.map(p -> CommunityMappingUtils.toDbProject(id, p))
+					.collect(Collectors.toList());
+
+			dbProjectRepository.saveAll(list);
+		} catch (final Throwable e) {
+			log.error(e);
+			throw new CommunityException(e);
+		}
+	}
+
+	@Transactional
+	public void removeCommunityProjects(final String id, final String... ids) {
+		final List<DbProjectPK> list = Arrays.stream(ids)
+				.map(projectId -> new DbProjectPK(id, projectId))
+				.collect(Collectors.toList());
+		dbProjectRepository.deleteAllById(list);
+	}
+
+	public List<CommunityContentprovider> getCommunityDatasources(final String id) {
+		return dbDatasourceRepository.findByCommunity(id)
+				.stream()
+				.map(CommunityMappingUtils::toCommunityContentprovider)
+				.collect(Collectors.toList());
+	}
+
+	public List<CommunityContentprovider> getCommunityDatasourcesWithDeposit(final String id, final boolean deposit) {
+		return dbDatasourceRepository.findByCommunityAndDeposit(id, deposit)
+				.stream()
+				.map(CommunityMappingUtils::toCommunityContentprovider)
+				.collect(Collectors.toList());
+	}
+
+	@Transactional
+	public CommunityContentprovider updateCommunityDatasourcesDeposit(final String id, final String dsId, final Boolean deposit, final String message) {
+		return dbDatasourceRepository.findById(new DbDatasourcePK(id, dsId))
+				.map(ds -> {
+					ds.setDeposit(deposit != null ? deposit : false);
+					ds.setMessage(message);
+					return ds;
+				})
+				.map(CommunityMappingUtils::toCommunityContentprovider)
+				.orElseThrow(() -> new ResourceNotFoundException("Community and/or Datasource not found"));
+	}
+
+	@Transactional
+	public void addCommunityDatasources(final String id, final CommunityContentprovider... contentproviders) {
+		final List<DbDatasource> list = Arrays.stream(contentproviders)
+				.map(cp -> CommunityMappingUtils.toDbDatasource(id, cp))
+				.collect(Collectors.toList());
+
+		dbDatasourceRepository.saveAll(list);
+	}
+
+	@Transactional
+	public void removeCommunityDatasources(final String id, final String... ids) {
+		final List<DbDatasourcePK> list = Arrays.stream(ids)
+				.map(dsId -> new DbDatasourcePK(id, dsId))
+				.collect(Collectors.toList());
+		dbDatasourceRepository.deleteAllById(list);
+	}
+
+	@Transactional
+	public void removeCommunityOrganizations(final String id, final String... orgNames) {
+		final List<DbSupportOrgPK> list = Arrays.stream(orgNames)
+				.map(name -> new DbSupportOrgPK(id, name))
+				.collect(Collectors.toList());
+		dbSupportOrgRepository.deleteAllById(list);
+	}
+
+	@Transactional
+	public List<CommunityOrganization> getCommunityOrganizations(final String id) {
+		return dbSupportOrgRepository.findByCommunity(id)
+				.stream()
+				.map(CommunityMappingUtils::toCommunityOrganization)
+				.collect(Collectors.toList());
+	}
+
+	@Transactional
+	public void addCommunityOrganizations(final String id, final CommunityOrganization... orgs) {
+		final List<DbSupportOrg> list = Arrays.stream(orgs)
+				.map(o -> CommunityMappingUtils.toDbSupportOrg(id, o))
+				.collect(Collectors.toList());
+
+		dbSupportOrgRepository.saveAll(list);
+	}
+
+	@Transactional
+	public void removeSubCommunities(final String id, final String... subCommunityIds) {
+		dbSubCommunityRepository.deleteAllById(Arrays.asList(subCommunityIds));
+	}
+
+	@Transactional
+	public List<SubCommunity> getSubCommunities(final String id) {
+		return dbSubCommunityRepository.findByCommunity(id)
+				.stream()
+				.map(CommunityMappingUtils::toSubCommunity)
+				.collect(Collectors.toList());
+	}
+
+	@Transactional
+	public void addSubCommunities(final String id, final SubCommunity... subs) {
+		final List<DbSubCommunity> list = Arrays.stream(subs)
+				.map(s -> CommunityMappingUtils.toDbSubCommunity(id, s))
+				.collect(Collectors.toList());
+
+		dbSubCommunityRepository.saveAll(list);
+	}
+
+	@Transactional
+	public CommunityDetails addCommunitySubjects(final String id, final String... subjects) {
+		return modifyElementToArrayField(id, DbCommunity::getSubjects, DbCommunity::setSubjects, false, subjects);
+	}
+
+	public CommunityDetails removeCommunitySubjects(final String id, final String... subjects) {
+		return modifyElementToArrayField(id, DbCommunity::getSubjects, DbCommunity::setSubjects, true, subjects);
+	}
+
+	public CommunityDetails addCommunityFOS(final String id, final String... foss) {
+		return modifyElementToArrayField(id, DbCommunity::getFos, DbCommunity::setFos, false, foss);
+	}
+
+	public CommunityDetails removeCommunityFOS(final String id, final String... foss) {
+		return modifyElementToArrayField(id, DbCommunity::getFos, DbCommunity::setFos, true, foss);
+	}
+
+	public CommunityDetails addCommunitySDG(final String id, final String... sdgs) {
+		return modifyElementToArrayField(id, DbCommunity::getSdg, DbCommunity::setSdg, false, sdgs);
+	}
+
+	public CommunityDetails removeCommunitySDG(final String id, final String... sdgs) {
+		return modifyElementToArrayField(id, DbCommunity::getSdg, DbCommunity::setSdg, true, sdgs);
+	}
+
+	@Transactional
+	public CommunityDetails addCommunityAdvancedConstraint(final String id, final SelectionCriteria advancedCosntraint) {
+		final DbCommunity dbEntry = dbCommunityRepository.findById(id).orElseThrow(() -> new ResourceNotFoundException("Community not found: " + id));
+		dbEntry.setAdvancedConstraints(advancedCosntraint);
+		dbEntry.setLastUpdateDate(LocalDateTime.now());
+		dbCommunityRepository.save(dbEntry);
+		return getCommunity(id);
+	}
+
+	@Transactional
+	public CommunityDetails removeCommunityAdvancedConstraint(final String id) {
+		final DbCommunity dbEntry = dbCommunityRepository.findById(id).orElseThrow(() -> new ResourceNotFoundException("Community not found: " + id));
+		dbEntry.setAdvancedConstraints(null);
+		dbEntry.setLastUpdateDate(LocalDateTime.now());
+		dbCommunityRepository.save(dbEntry);
+		return getCommunity(id);
+	}
+
+	@Transactional
+	public CommunityDetails addCommunityRemoveConstraint(final String id, final SelectionCriteria removeConstraint) {
+		final DbCommunity dbEntry = dbCommunityRepository.findById(id).orElseThrow(() -> new ResourceNotFoundException("Community not found: " + id));
+		dbEntry.setRemoveConstraints(removeConstraint);
+		dbEntry.setLastUpdateDate(LocalDateTime.now());
+		dbCommunityRepository.save(dbEntry);
+		return getCommunity(id);
+	}
+
+	@Transactional
+	public CommunityDetails removeCommunityRemoveConstraint(final String id) {
+		final DbCommunity dbEntry = dbCommunityRepository.findById(id).orElseThrow(() -> new ResourceNotFoundException("Community not found: " + id));
+		dbEntry.setRemoveConstraints(null);
+		dbEntry.setLastUpdateDate(LocalDateTime.now());
+		dbCommunityRepository.save(dbEntry);
+		return getCommunity(id);
+	}
+
+	public CommunityDetails removeCommunityZenodoCommunity(final String id, final String zenodoCommunity, final boolean isMain) {
+		if (isMain) { return updateElementToSimpleField(id, DbCommunity::setMainZenodoCommunity, null); }
+		return modifyElementToArrayField(id, DbCommunity::getOtherZenodoCommunities, DbCommunity::setOtherZenodoCommunities, true, zenodoCommunity);
+	}
+
+	public CommunityDetails addCommunityZenodoCommunity(final String id, final String zenodoCommunity, final boolean isMain) {
+		if (isMain) { return updateElementToSimpleField(id, DbCommunity::setMainZenodoCommunity, zenodoCommunity); }
+		return modifyElementToArrayField(id, DbCommunity::getOtherZenodoCommunities, DbCommunity::setOtherZenodoCommunities, false, zenodoCommunity);
+	}
+
+	@Transactional
+	private CommunityDetails updateElementToSimpleField(final String id,
+			final BiConsumer<DbCommunity, String> setter,
+			final String value) {
+		final DbCommunity dbEntry = dbCommunityRepository.findById(id).orElseThrow(() -> new ResourceNotFoundException("Community not found: " + id));
+		setter.accept(dbEntry, value);
+		dbEntry.setLastUpdateDate(LocalDateTime.now());
+		dbCommunityRepository.save(dbEntry);
+		return getCommunity(id);
+	}
+
+	@Transactional
+	private CommunityDetails modifyElementToArrayField(final String id,
+			final Function<DbCommunity, String[]> getter,
+			final BiConsumer<DbCommunity, String[]> setter,
+			final boolean remove,
+			final String... values) {
+
+		final DbCommunity dbEntry = dbCommunityRepository.findById(id).orElseThrow(() -> new ResourceNotFoundException("Community not found: " + id));
+
+		final Set<String> tmpList = new LinkedHashSet<>();
+		final String[] oldValues = getter.apply(dbEntry);
+		if (oldValues != null) {
+			Collections.addAll(tmpList, oldValues);
+		}
+		if (remove) {
+			tmpList.removeAll(Arrays.asList(values));
+		} else {
+			tmpList.addAll(Arrays.asList(values));
+		}
+
+		setter.accept(dbEntry, tmpList.toArray(new String[tmpList.size()]));
+
+		dbEntry.setLastUpdateDate(LocalDateTime.now());
+
+		dbCommunityRepository.save(dbEntry);
+
+		return getCommunity(id);
+	}
+
+	@Transactional
+	public List<String> getOpenAIRECommunitiesByZenodoId(final String zenodoId) {
+		return dbCommunityRepository.findByZenodoId(zenodoId);
+	}
+
+	@Transactional
+	public Map<String, Set<String>> getPropagationOrganizationCommunityMap() {
+		return dbOrganizationRepository.findAll()
+				.stream()
+				.collect(Collectors.groupingBy(DbOrganization::getOrgId, Collectors.mapping(DbOrganization::getCommunity, Collectors.toSet())));
+	}
+
+	@Transactional
+	public Set<String> getPropagationOrganizationsForCommunity(final String communityId) {
+		return dbOrganizationRepository.findByCommunity(communityId)
+				.stream()
+				.map(DbOrganization::getOrgId)
+				.collect(Collectors.toSet());
+	}
+
+	@Transactional
+	public Set<String> addPropagationOrganizationForCommunity(final String communityId, final String... organizationIds) {
+		for (final String orgId : organizationIds) {
+			final DbOrganization o = new DbOrganization(communityId.trim(), orgId.trim());
+			dbOrganizationRepository.save(o);
+		}
+		return getPropagationOrganizationsForCommunity(communityId);
+	}
+
+	@Transactional
+	public Set<String> removePropagationOrganizationForCommunity(final String communityId, final String... organizationIds) {
+		for (final String orgId : organizationIds) {
+			final DbOrganization o = new DbOrganization(communityId.trim(), orgId.trim());
+			dbOrganizationRepository.delete(o);
+		}
+		return getPropagationOrganizationsForCommunity(communityId);
+	}
+
+	@Transactional
+	public void deleteCommunity(final String id, final boolean recursive) {
+		if (recursive) {
+			dbProjectRepository.deleteByCommunity(id);
+			dbDatasourceRepository.deleteByCommunity(id);
+			dbOrganizationRepository.deleteByCommunity(id);
+			dbSupportOrgRepository.deleteByCommunity(id);
+			dbSubCommunityRepository.deleteByCommunity(id);
+		}
+		dbCommunityRepository.deleteById(id);
+	}
+
+	@Transactional
+	public List<IISConfigurationEntry> getIISConfiguration(final String id) {
+		final List<IISConfigurationEntry> res = new ArrayList<>();
+
+		res.add(dbCommunityRepository.findById(id)
+				.map(CommunityMappingUtils::asIISConfigurationEntry)
+				.orElseThrow(() -> new ResourceNotFoundException("Community not found: " + id)));
+
+		for (final DbSubCommunity subc : dbSubCommunityRepository.findByCommunity(id)) {
+			res.add(CommunityMappingUtils.asIISConfigurationEntry(subc));
+		}
+
+		return res;
+	}
+
+	@Transactional
+	public List<String> getCommunityFunders(final String id) {
+		return dbProjectRepository.findFundersByCommunity(id);
+	}
+
+}

apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/importer/CommunityImporterController.java

@@ -0,0 +1,87 @@
+package eu.dnetlib.openaire.community.importer;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
+import org.springframework.web.bind.annotation.CrossOrigin;
+import org.springframework.web.bind.annotation.GetMapping;
+import org.springframework.web.bind.annotation.RequestParam;
+import org.springframework.web.bind.annotation.RestController;
+
+import com.google.common.collect.Sets;
+
+import eu.dnetlib.common.controller.AbstractDnetController;
+import eu.dnetlib.openaire.common.ISClient;
+import eu.dnetlib.openaire.community.model.DbOrganization;
+import eu.dnetlib.openaire.exporter.exceptions.CommunityException;
+import eu.dnetlib.openaire.exporter.model.context.Context;
+import io.swagger.v3.oas.annotations.tags.Tag;
+
+@RestController
+@CrossOrigin(origins = {
+	"*"
+})
+@ConditionalOnProperty(value = "openaire.exporter.enable.community.import", havingValue = "true")
+@Tag(name = "OpenAIRE Communities: Migration API", description = "OpenAIRE Communities: Migration API")
+public class CommunityImporterController extends AbstractDnetController {
+
+	// public final static Set<String> communityBlackList = Sets.newHashSet("fet-fp7", "fet-h2020");
+	public final static Set<String> communityBlackList = Sets.newHashSet();
+
+	@Autowired
+	private CommunityImporterService importer;
+
+	@Autowired
+	private ISClient isClient;
+
+	private static final Log log = LogFactory.getLog(CommunityImporterController.class);
+
+	@GetMapping("/community_importer/communities")
+	public List<String> importProfiles() throws CommunityException {
+		try {
+			final Map<String, Context> contextMap = getContextMap();
+
+			final List<String> list = contextMap.keySet()
+				.stream()
+				.filter(id -> !communityBlackList.contains(id))
+				.collect(Collectors.toList());
+
+			list.forEach(id -> {
+				importer.importCommunity(contextMap.get(id));
+			});
+
+			return list;
+		} catch (final Throwable e) {
+			log.error("Error importing communities", e);
+			throw new CommunityException(e.getMessage());
+		}
+	}
+
+	@GetMapping("/community_importer/propagationOrgs")
+	public List<DbOrganization> importPropagationOrgs(@RequestParam final String profileId,
+		@RequestParam(required = false, defaultValue = "false") final boolean simulation) throws Exception {
+		try {
+			final String xml = isClient.getProfile(profileId);
+			return importer.importPropagationOrganizationsFromProfile(xml, simulation);
+		} catch (final Throwable e) {
+			log.error("Error importing communities", e);
+			throw new CommunityException(e.getMessage());
+		}
+	}
+
+	private Map<String, Context> getContextMap() throws CommunityException {
+		try {
+			return isClient.getCommunityContextMap();
+		} catch (final IOException e) {
+			throw new CommunityException(e);
+		}
+	}
+
+}

apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/importer/CommunityImporterService.java

@@ -0,0 +1,411 @@
+package eu.dnetlib.openaire.community.importer;
+
+import java.time.LocalDate;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Base64;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+import javax.transaction.Transactional;
+
+import org.apache.commons.lang3.BooleanUtils;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.dom4j.DocumentHelper;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
+import org.springframework.jdbc.core.JdbcTemplate;
+import org.springframework.stereotype.Service;
+
+import com.fasterxml.jackson.core.type.TypeReference;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.google.common.collect.Lists;
+
+import eu.dnetlib.miscutils.functional.hash.Hashing;
+import eu.dnetlib.openaire.community.CommunityService;
+import eu.dnetlib.openaire.community.model.DbOrganization;
+import eu.dnetlib.openaire.community.repository.DbOrganizationRepository;
+import eu.dnetlib.openaire.community.utils.CommunityMappingUtils;
+import eu.dnetlib.openaire.exporter.exceptions.CommunityException;
+import eu.dnetlib.openaire.exporter.model.community.CommunityClaimType;
+import eu.dnetlib.openaire.exporter.model.community.CommunityContentprovider;
+import eu.dnetlib.openaire.exporter.model.community.CommunityDetails;
+import eu.dnetlib.openaire.exporter.model.community.CommunityMembershipType;
+import eu.dnetlib.openaire.exporter.model.community.CommunityOrganization;
+import eu.dnetlib.openaire.exporter.model.community.CommunityPlanType;
+import eu.dnetlib.openaire.exporter.model.community.CommunityProject;
+import eu.dnetlib.openaire.exporter.model.community.CommunityStatus;
+import eu.dnetlib.openaire.exporter.model.community.CommunityType;
+import eu.dnetlib.openaire.exporter.model.community.SubCommunity;
+import eu.dnetlib.openaire.exporter.model.community.selectioncriteria.SelectionCriteria;
+import eu.dnetlib.openaire.exporter.model.context.Category;
+import eu.dnetlib.openaire.exporter.model.context.Concept;
+import eu.dnetlib.openaire.exporter.model.context.Context;
+import eu.dnetlib.openaire.exporter.model.context.Param;
+
+@Service
+@ConditionalOnProperty(value = "openaire.exporter.enable.community.import", havingValue = "true")
+public class CommunityImporterService {
+
+	// common
+	public final static String OPENAIRE_ID = "openaireId";
+	public final static String PIPE_SEPARATOR = "||";
+	public final static String ID_SEPARATOR = "::";
+	public final static String CSV_DELIMITER = ",";
+	public final static String CLABEL = "label";
+
+	// id suffixes
+	public final static String PROJECTS_ID_SUFFIX = ID_SEPARATOR + "projects";
+	public final static String CONTENTPROVIDERS_ID_SUFFIX = ID_SEPARATOR + "contentproviders";
+	public final static String ZENODOCOMMUNITY_ID_SUFFIX = ID_SEPARATOR + "zenodocommunities";
+	public final static String ORGANIZATION_ID_SUFFIX = ID_SEPARATOR + "organizations";
+
+	// community summary
+	public final static String CSUMMARY_DESCRIPTION = "description";
+	public final static String CSUMMARY_LOGOURL = "logourl";
+	public final static String CSUMMARY_STATUS = "status";
+	public final static String CSUMMARY_NAME = "name";
+	public final static String CSUMMARY_MANAGER = "manager";
+	public final static String CSUMMARY_ZENODOC = "zenodoCommunity";
+
+	// community profile
+	public final static String CPROFILE_SUBJECT = "subject";
+	public final static String CPROFILE_CREATIONDATE = "creationdate";
+	public final static String CPROFILE_FOS = "fos";
+	public final static String CPROFILE_SDG = "sdg";
+	public final static String CPROFILE_ADVANCED_CONSTRAINT = "advancedConstraints";
+	public final static String CPROFILE_REMOVE_CONSTRAINT = "removeConstraints";
+	public final static String CPROFILE_SUGGESTED_ACKNOWLEDGEMENT = "suggestedAcknowledgement";
+
+	// community project
+	public final static String CPROJECT_FUNDER = "funder";
+	public final static String CPROJECT_NUMBER = "CD_PROJECT_NUMBER";
+	public final static String CPROJECT_FULLNAME = "projectfullname";
+	public final static String CPROJECT_ACRONYM = "acronym";
+
+	// community content provider
+	public final static String CCONTENTPROVIDER_NAME = "name";
+	public final static String CCONTENTPROVIDER_OFFICIALNAME = "officialname";
+	public final static String CCONTENTPROVIDER_ENABLED = "enabled";
+	public final static String CCONTENTPROVIDERENABLED_DEFAULT = "true";
+	public final static String CCONTENTPROVIDER_SELCRITERIA = "selcriteria";
+
+	// community zenodo community
+	public final static String CZENODOCOMMUNITY_ID = "zenodoid";
+
+	// community organization
+	public final static String CORGANIZATION_NAME = "name";
+	public final static String CORGANIZATION_LOGOURL = "logourl";
+	public final static String CORGANIZATION_WEBSITEURL = "websiteurl";
+
+	@Autowired
+	private DbOrganizationRepository dbOrganizationRepository;
+
+	@Autowired
+	private CommunityService service;
+
+	@Autowired
+	private JdbcTemplate jdbcTemplate;
+
+	private static final Log log = LogFactory.getLog(CommunityImporterService.class);
+
+	public List<DbOrganization> importPropagationOrganizationsFromProfile(final String xml, final boolean simulation) throws Exception {
+		final String json = DocumentHelper.parseText(xml)
+				.selectSingleNode("//NODE[@name='setPropagationOrganizationCommunityMap']//PARAM[@name='parameterValue']")
+				.getText();
+
+		final List<DbOrganization> list = new ObjectMapper()
+				.readValue(json, new TypeReference<Map<String, List<String>>>() {})
+				.entrySet()
+				.stream()
+				.flatMap(e -> e.getValue()
+						.stream()
+						.map(community -> {
+							if (e.getKey().contains("|")) { return new DbOrganization(community, StringUtils.substringAfter(e.getKey(), "|")); }
+							return new DbOrganization(community, e.getKey());
+						}))
+				.collect(Collectors.toList());
+
+		if (!simulation) {
+			list.forEach(o -> {
+				try {
+					dbOrganizationRepository.save(o);
+				} catch (final Throwable e) {
+					log.error("ERROR saving org: " + o);
+				}
+			});
+		}
+
+		return list;
+	}
+
+	@Transactional
+	public void importCommunity(final Context context) {
+		try {
+
+			final CommunityDetails community = asCommunityDetails(context);
+
+			final List<CommunityContentprovider> datasources =
+					getCommunityInfo(context, CONTENTPROVIDERS_ID_SUFFIX, c -> asCommunityDataprovider(context.getId(), c))
+							.stream()
+							.map(o -> {
+								if (o.getOpenaireId() == null) {
+									log.warn("Openaire ID is missing, organization: " + o.getOfficialname());
+								} else if (o.getOpenaireId().contains("|")) {
+									o.setOpenaireId(StringUtils.substringAfter(o.getOpenaireId(), "|"));
+								}
+								return o;
+							})
+							.filter(o -> o.getOpenaireId() != null)
+							.collect(Collectors.toList());
+
+			final List<CommunityProject> projects =
+					getCommunityInfo(context, PROJECTS_ID_SUFFIX, c -> asCommunityProject(context.getId(), c))
+							.stream()
+							.map(p -> {
+								if (p.getOpenaireId() == null) {
+									if ("EC".equalsIgnoreCase(p.getFunder())) {
+										final String ns = findNamespaceForECProject(p.getGrantId());
+										if (ns != null) {
+											p.setOpenaireId(ns + "::" + Hashing.md5(p.getGrantId()));
+										} else {
+											log.warn("EC project not in the db: " + p.getGrantId());
+										}
+									} else if ("NSF".equalsIgnoreCase(p.getFunder())) {
+										p.setOpenaireId("nsf_________::" + Hashing.md5(p.getGrantId()));
+									} else if ("NIH".equalsIgnoreCase(p.getFunder())) {
+										p.setOpenaireId("nih_________::" + Hashing.md5(p.getGrantId()));
+									} else {
+										log.warn("Openaire ID is missing, funder: " + p.getFunder());
+									}
+								} else if (p.getOpenaireId().contains("|")) {
+									p.setOpenaireId(StringUtils.substringAfter(p.getOpenaireId(), "|"));
+								}
+								return p;
+							})
+							.filter(p -> p.getOpenaireId() != null)
+							.collect(Collectors.toList());
+
+			final List<CommunityOrganization> orgs =
+					getCommunityInfo(context, ORGANIZATION_ID_SUFFIX, c -> asCommunityOrganization(context.getId(), c));
+
+			final List<String> otherZenodoCommunities =
+					getCommunityInfo(context, ZENODOCOMMUNITY_ID_SUFFIX, CommunityImporterService::asZenodoCommunity);
+
+			community.setOtherZenodoCommunities(otherZenodoCommunities);
+
+			final List<SubCommunity> subs = context.getCategories()
+					.entrySet()
+					.stream()
+					.filter(e -> !(context.getId() + CONTENTPROVIDERS_ID_SUFFIX).equals(e.getKey()))
+					.filter(e -> !(context.getId() + PROJECTS_ID_SUFFIX).equals(e.getKey()))
+					.filter(e -> !(context.getId() + ORGANIZATION_ID_SUFFIX).equals(e.getKey()))
+					.filter(e -> !(context.getId() + ZENODOCOMMUNITY_ID_SUFFIX).equals(e.getKey()))
+					.map(Entry::getValue)
+					.map(cat -> asSubCommunities(context.getId(), null, cat.getLabel(), cat.getConcepts()))
+					.flatMap(List::stream)
+					.collect(Collectors.toList());
+
+			service.saveCommunity(community);
+			service.addCommunityProjects(context.getId(), projects.toArray(new CommunityProject[projects.size()]));
+			service.addCommunityDatasources(context.getId(), datasources.toArray(new CommunityContentprovider[datasources.size()]));
+			service.addCommunityOrganizations(context.getId(), orgs.toArray(new CommunityOrganization[orgs.size()]));
+			service.addSubCommunities(context.getId(), subs.toArray(new SubCommunity[subs.size()]));
+		} catch (
+
+		final Exception e) {
+			throw new RuntimeException("Error importing community: " + context.getId(), e);
+		}
+	}
+
+	private <R> List<R> getCommunityInfo(final Context context, final String idSuffix, final Function<Concept, R> mapping)
+			throws CommunityException {
+		if (context != null) {
+			final Map<String, Category> categories = context.getCategories();
+			final Category category = categories.get(context.getId() + idSuffix);
+			if (category != null) { return category.getConcepts()
+					.stream()
+					.map(mapping)
+					.collect(Collectors.toList()); }
+		}
+		return Lists.newArrayList();
+	}
+
+	private static CommunityDetails asCommunityDetails(final Context c) {
+
+		final CommunityDetails details = new CommunityDetails();
+
+		details.setId(c.getId());
+		details.setShortName(c.getLabel());
+		details.setDisplayShortName(c.getLabel());
+		details.setLastUpdateDate(CommunityMappingUtils.asLocalDateTime(c.getLastUpdateDate()));
+		details.setCreationDate(CommunityMappingUtils.asLocalDateTime(c.getCreationDate()));
+		details.setQueryId(c.getId() + PIPE_SEPARATOR + c.getLabel());
+		details.setType(CommunityType.valueOf(c.getType()));
+		details.setMembership(CommunityMembershipType.open);
+		details.setClaim(CommunityClaimType.all);
+		details.setDescription(asCsv(CSUMMARY_DESCRIPTION, c.getParams()));
+		details.setLogoUrl(asCsv(CSUMMARY_LOGOURL, c.getParams()));
+
+		final String status = firstValue(CSUMMARY_STATUS, c.getParams());
+		if (StringUtils.isNotBlank(status)) {
+			details.setStatus(CommunityStatus.valueOf(status));
+		} else {
+			details.setStatus(CommunityStatus.hidden);
+		}
+
+		details.setName(StringUtils.firstNonBlank(asCsv(CSUMMARY_NAME, c.getParams()), c.getLabel()));
+		details.setDisplayName(StringUtils.firstNonBlank(asCsv(CSUMMARY_NAME, c.getParams()), c.getLabel()));
+
+		details.setZenodoCommunity(asCsv(CSUMMARY_ZENODOC, c.getParams()));
+		details.setSubjects(splitValues(asValues(CPROFILE_SUBJECT, c.getParams()), CSV_DELIMITER));
+		details.setFos(splitValues(asValues(CPROFILE_FOS, c.getParams()), CSV_DELIMITER));
+		details.setSdg(splitValues(asValues(CPROFILE_SDG, c.getParams()), CSV_DELIMITER));
+		// In the map the string is the serialization of the json representing the selection criteria so it is a valid json
+		details.setAdvancedConstraints(SelectionCriteria.fromJson(asCsv(CPROFILE_ADVANCED_CONSTRAINT, c.getParams())));
+		// In the map the string is the serialization of the json representing the selection criteria so it is a valid json
+		details.setRemoveConstraints(SelectionCriteria.fromJson(asCsv(CPROFILE_REMOVE_CONSTRAINT, c.getParams())));
+		details.setSuggestedAcknowledgements(splitValues(asValues(CPROFILE_SUGGESTED_ACKNOWLEDGEMENT, c.getParams()), CSV_DELIMITER));
+		details.setPlan(CommunityPlanType.Default);
+		try {
+			details.setCreationDate(CommunityMappingUtils.asLocalDateTime(asCsv(CPROFILE_CREATIONDATE, c.getParams())));
+		} catch (final Exception e) {
+			log.debug("Exception on date format: " + e.getMessage());
+		}
+
+		return details;
+	}
+
+	private static CommunityProject asCommunityProject(final String communityId, final Concept c) {
+		final List<Param> p = c.getParams();
+		final CommunityProject project = new CommunityProject();
+		project.setCommunityId(communityId);
+		project.setOpenaireId(firstValue(OPENAIRE_ID, p));
+		project.setFunder(firstValue(CPROJECT_FUNDER, p));
+		project.setGrantId(firstValue(CPROJECT_NUMBER, p));
+		project.setName(firstValue(CPROJECT_FULLNAME, p));
+		project.setAcronym(firstValue(CPROJECT_ACRONYM, p));
+		project.setAvailableSince(LocalDate.of(2017, 2, 25)); // Birillo Birth Date
+		return project;
+	}
+
+	private static CommunityContentprovider asCommunityDataprovider(final String communityId, final Concept c) {
+		final List<Param> p = c.getParams();
+		final CommunityContentprovider d = new CommunityContentprovider();
+		d.setCommunityId(communityId);
+		d.setOpenaireId(firstValue(OPENAIRE_ID, p));
+		d.setName(firstValue(CCONTENTPROVIDER_NAME, p));
+		d.setOfficialname(firstValue(CCONTENTPROVIDER_OFFICIALNAME, p));
+		d.setEnabled(BooleanUtils.toBoolean(firstValue(CCONTENTPROVIDER_ENABLED, p)));
+		d.setSelectioncriteria(SelectionCriteria.fromJson(firstValue(CCONTENTPROVIDER_SELCRITERIA, p)));
+		d.setDeposit(false);
+		d.setMessage(null);
+		return d;
+	}
+
+	private static CommunityOrganization asCommunityOrganization(final String id, final Concept c) {
+		final List<Param> p = c.getParams();
+		final CommunityOrganization o = new CommunityOrganization();
+		o.setCommunityId(id);
+		o.setName(firstValue(CORGANIZATION_NAME, p));
+		o.setLogo_url(getDecodedUrl(firstValue(CORGANIZATION_LOGOURL, p)));
+		o.setWebsite_url(getDecodedUrl(firstValue(CORGANIZATION_WEBSITEURL, p)));
+		return o;
+	}
+
+	private static String asZenodoCommunity(final Concept c) {
+		return firstValue(CZENODOCOMMUNITY_ID, c.getParams());
+	}
+
+	private static List<SubCommunity> asSubCommunities(final String communityId, final String parent, final String category, final List<Concept> concepts) {
+		final List<SubCommunity> list = new ArrayList<>();
+		for (final Concept c : concepts) {
+			final SubCommunity sc = new SubCommunity();
+			sc.setSubCommunityId(c.getId());
+			sc.setCommunityId(communityId);
+			sc.setParent(parent);
+			sc.setCategory(category);
+			sc.setLabel(c.getLabel());
+			sc.setParams(c.getParams());
+			sc.setClaim(c.isClaim());
+			sc.setBrowsable(false);
+			list.add(sc);
+			list.addAll(asSubCommunities(communityId, c.getId(), category, c.getConcepts()));
+		}
+		return list;
+	}
+
+	private String findNamespaceForECProject(final String code) {
+		final List<String> list =
+				jdbcTemplate.queryForList("SELECT substr(id, 1, 12) from projects where code = ? and id like 'corda%'", String.class, code);
+		return list.isEmpty() ? null : list.get(0);
+	}
+
+	private static String getDecodedUrl(final String encoded_url) {
+		if (encoded_url == null || encoded_url.startsWith("http")) { return encoded_url; }
+		try {
+			return new String(Base64.getDecoder().decode(encoded_url));
+		} catch (final Exception e) {
+			log.warn("Invalid base64: " + encoded_url);
+			return encoded_url;
+		}
+	}
+
+	private static List<String> splitValues(final Stream<String> stream, final String separator) {
+		return stream.map(s -> s.split(separator))
+				.map(Arrays::asList)
+				.flatMap(List::stream)
+				.filter(StringUtils::isNotBlank)
+				.map(StringUtils::trim)
+				.collect(Collectors.toList());
+	}
+
+	private static String firstValue(final String name, final List<Param> params) {
+		return asValues(name, params).findFirst().orElse(null);
+	}
+
+	private static String asCsv(final String name, final List<Param> params) {
+		return asValues(name, params).collect(Collectors.joining(CSV_DELIMITER));
+	}
+
+	private static Stream<String> asValues(final String name, final List<Param> params) {
+		return params == null ? Stream.empty()
+				: params.stream()
+						.filter(p -> p != null)
+						.filter(p -> StringUtils.isNotBlank(p.getName()))
+						.filter(p -> p.getName().trim().equals(name.trim()))
+						.map(Param::getValue)
+						.map(StringUtils::trim)
+						.distinct();
+	}
+
+	protected DbOrganizationRepository getDbOrganizationRepository() {
+		return dbOrganizationRepository;
+	}
+
+	protected void setDbOrganizationRepository(final DbOrganizationRepository dbOrganizationRepository) {
+		this.dbOrganizationRepository = dbOrganizationRepository;
+	}
+
+	protected CommunityService getService() {
+		return service;
+	}
+
+	protected void setService(final CommunityService service) {
+		this.service = service;
+	}
+
+	protected JdbcTemplate getJdbcTemplate() {
+		return jdbcTemplate;
+	}
+
+	protected void setJdbcTemplate(final JdbcTemplate jdbcTemplate) {
+		this.jdbcTemplate = jdbcTemplate;
+	}
+}

apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/model/DbCommunity.java

@@ -0,0 +1,304 @@
+package eu.dnetlib.openaire.community.model;
+
+import java.io.Serializable;
+import java.time.LocalDateTime;
+
+import javax.persistence.Column;
+import javax.persistence.Convert;
+import javax.persistence.Entity;
+import javax.persistence.EnumType;
+import javax.persistence.Enumerated;
+import javax.persistence.GeneratedValue;
+import javax.persistence.GenerationType;
+import javax.persistence.Id;
+import javax.persistence.Table;
+
+import org.hibernate.annotations.Type;
+import org.hibernate.annotations.TypeDef;
+import org.hibernate.annotations.TypeDefs;
+import org.springframework.data.annotation.CreatedDate;
+import org.springframework.data.annotation.LastModifiedDate;
+
+import com.vladmihalcea.hibernate.type.array.StringArrayType;
+import com.vladmihalcea.hibernate.type.json.JsonBinaryType;
+import com.vladmihalcea.hibernate.type.json.JsonStringType;
+
+import eu.dnetlib.openaire.community.utils.CommunityClaimTypeConverter;
+import eu.dnetlib.openaire.community.utils.CommunityMembershipTypeConverter;
+import eu.dnetlib.openaire.exporter.model.community.CommunityClaimType;
+import eu.dnetlib.openaire.exporter.model.community.CommunityMembershipType;
+import eu.dnetlib.openaire.exporter.model.community.CommunityPlanType;
+import eu.dnetlib.openaire.exporter.model.community.CommunityStatus;
+import eu.dnetlib.openaire.exporter.model.community.CommunityType;
+import eu.dnetlib.openaire.exporter.model.community.selectioncriteria.SelectionCriteria;
+
+@Entity
+@Table(name = "communities")
+@TypeDefs({
+		@TypeDef(name = "string-array", typeClass = StringArrayType.class),
+		@TypeDef(name = "json", typeClass = JsonStringType.class),
+		@TypeDef(name = "jsonb", typeClass = JsonBinaryType.class)
+})
+public class DbCommunity implements Serializable {
+
+	private static final long serialVersionUID = 4315597783109726539L;
+
+	@Id
+	@Column(name = "id")
+	private String id;
+
+	@Column(name = "name")
+	private String name;
+
+	@Column(name = "shortname")
+	private String shortName;
+
+	@Column(name = "displayname")
+	private String displayName;
+
+	@Column(name = "displayshortname")
+	private String displayShortName;
+
+	@Column(name = "description")
+	private String description;
+
+	@Column(name = "status")
+	@Enumerated(EnumType.STRING)
+	private CommunityStatus status = CommunityStatus.hidden;
+
+	@Column(name = "membership")
+	@Convert(converter = CommunityMembershipTypeConverter.class)
+	private CommunityMembershipType membership = CommunityMembershipType.byInvitation;
+
+	@Column(name = "type")
+	@Enumerated(EnumType.STRING)
+	private CommunityType type;
+
+	@Column(name = "claim")
+	@Convert(converter = CommunityClaimTypeConverter.class)
+	private CommunityClaimType claim;
+
+	@Type(type = "string-array")
+	@Column(name = "subjects", columnDefinition = "text[]")
+	private String[] subjects;
+
+	@Type(type = "string-array")
+	@Column(name = "fos", columnDefinition = "text[]")
+	private String[] fos;
+
+	@Type(type = "string-array")
+	@Column(name = "sdg", columnDefinition = "text[]")
+	private String[] sdg;
+
+	@Type(type = "jsonb")
+	@Column(name = "adv_constraints")
+	private SelectionCriteria advancedConstraints;
+
+	@Type(type = "jsonb")
+	@Column(name = "remove_constraints")
+	private SelectionCriteria removeConstraints;
+
+	@Column(name = "main_zenodo_community")
+	private String mainZenodoCommunity;
+
+	@Type(type = "string-array")
+	@Column(name = "other_zenodo_communities", columnDefinition = "text[]")
+	private String[] otherZenodoCommunities;
+
+	@CreatedDate
+	@Column(name = "creation_date")
+	private LocalDateTime creationDate;
+
+	@LastModifiedDate
+	@Column(name = "last_update")
+	@GeneratedValue(strategy = GenerationType.IDENTITY)
+	private LocalDateTime lastUpdateDate;
+
+	@Column(name = "logo_url")
+	private String logoUrl;
+
+	@Type(type = "string-array")
+	@Column(name = "suggested_acknowledgements", columnDefinition = "text[]")
+	private String[] suggestedAcknowledgements;
+
+	@Column(name = "plan")
+	@Enumerated(EnumType.STRING)
+	private CommunityPlanType plan;
+
+	public String getId() {
+		return id;
+	}
+
+	public void setId(final String id) {
+		this.id = id;
+	}
+
+	public String getName() {
+		return name;
+	}
+
+	public void setName(final String name) {
+		this.name = name;
+	}
+
+	public String getShortName() {
+		return shortName;
+	}
+
+	public void setShortName(final String shortName) {
+		this.shortName = shortName;
+	}
+
+	public String getDisplayName() {
+		return displayName;
+	}
+
+	public void setDisplayName(final String displayName) {
+		this.displayName = displayName;
+	}
+
+	public String getDisplayShortName() {
+		return displayShortName;
+	}
+
+	public void setDisplayShortName(final String displayShortName) {
+		this.displayShortName = displayShortName;
+	}
+
+	public String getDescription() {
+		return description;
+	}
+
+	public void setDescription(final String description) {
+		this.description = description;
+	}
+
+	public CommunityStatus getStatus() {
+		return status;
+	}
+
+	public void setStatus(final CommunityStatus status) {
+		this.status = status;
+	}
+
+	public CommunityMembershipType getMembership() {
+		return membership;
+	}
+
+	public void setMembership(final CommunityMembershipType membership) {
+		this.membership = membership;
+	}
+
+	public CommunityType getType() {
+		return type;
+	}
+
+	public void setType(final CommunityType type) {
+		this.type = type;
+	}
+
+	public CommunityClaimType getClaim() {
+		return claim;
+	}
+
+	public void setClaim(final CommunityClaimType claim) {
+		this.claim = claim;
+	}
+
+	public String[] getSubjects() {
+		return subjects;
+	}
+
+	public void setSubjects(final String[] subjects) {
+		this.subjects = subjects;
+	}
+
+	public String[] getFos() {
+		return fos;
+	}
+
+	public void setFos(final String[] fos) {
+		this.fos = fos;
+	}
+
+	public String[] getSdg() {
+		return sdg;
+	}
+
+	public void setSdg(final String[] sdg) {
+		this.sdg = sdg;
+	}
+
+	public SelectionCriteria getAdvancedConstraints() {
+		return advancedConstraints;
+	}
+
+	public void setAdvancedConstraints(final SelectionCriteria advancedConstraints) {
+		this.advancedConstraints = advancedConstraints;
+	}
+
+	public SelectionCriteria getRemoveConstraints() {
+		return removeConstraints;
+	}
+
+	public void setRemoveConstraints(final SelectionCriteria removeConstraints) {
+		this.removeConstraints = removeConstraints;
+	}
+
+	public String getMainZenodoCommunity() {
+		return mainZenodoCommunity;
+	}
+
+	public void setMainZenodoCommunity(final String mainZenodoCommunity) {
+		this.mainZenodoCommunity = mainZenodoCommunity;
+	}
+
+	public String[] getOtherZenodoCommunities() {
+		return otherZenodoCommunities;
+	}
+
+	public void setOtherZenodoCommunities(final String[] otherZenodoCommunities) {
+		this.otherZenodoCommunities = otherZenodoCommunities;
+	}
+
+	public LocalDateTime getCreationDate() {
+		return creationDate;
+	}
+
+	public void setCreationDate(final LocalDateTime creationDate) {
+		this.creationDate = creationDate;
+	}
+
+	public LocalDateTime getLastUpdateDate() {
+		return lastUpdateDate;
+	}
+
+	public void setLastUpdateDate(final LocalDateTime lastUpdateDate) {
+		this.lastUpdateDate = lastUpdateDate;
+	}
+
+	public String getLogoUrl() {
+		return logoUrl;
+	}
+
+	public void setLogoUrl(final String logoUrl) {
+		this.logoUrl = logoUrl;
+	}
+
+	public String[] getSuggestedAcknowledgements() {
+		return suggestedAcknowledgements;
+	}
+
+	public void setSuggestedAcknowledgements(final String[] suggestedAcknowledgements) {
+		this.suggestedAcknowledgements = suggestedAcknowledgements;
+	}
+
+	public CommunityPlanType getPlan() {
+		return plan;
+	}
+
+	public void setPlan(final CommunityPlanType plan) {
+		this.plan = plan;
+	}
+
+}

apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/model/DbDatasource.java

@@ -0,0 +1,123 @@
+package eu.dnetlib.openaire.community.model;
+
+import java.io.Serializable;
+
+import javax.persistence.Column;
+import javax.persistence.Entity;
+import javax.persistence.Id;
+import javax.persistence.IdClass;
+import javax.persistence.Table;
+
+import org.hibernate.annotations.Type;
+
+import eu.dnetlib.openaire.exporter.model.community.selectioncriteria.SelectionCriteria;
+
+@Entity
+@Table(name = "community_datasources")
+@IdClass(DbDatasourcePK.class)
+public class DbDatasource implements Serializable {
+
+	private static final long serialVersionUID = -8782576185861694228L;
+
+	@Id
+	@Column(name = "community")
+	private String community;
+
+	@Id
+	@Column(name = "ds_id")
+	private String dsId;
+
+	@Column(name = "ds_name")
+	private String dsName;
+
+	@Column(name = "ds_officialname")
+	private String dsOfficialName;
+
+	@Column(name = "enabled")
+	private Boolean enabled;
+
+	@Type(type = "jsonb")
+	@Column(name = "constraints")
+	private SelectionCriteria constraints;
+
+	@Column(name = "deposit")
+	private Boolean deposit;
+
+	@Column(name = "message")
+	private String message;
+
+	public DbDatasource() {}
+
+	public DbDatasource(final String community, final String dsId, final String dsName, final String dsOfficialName, final SelectionCriteria constraints) {
+		this.community = community;
+		this.dsId = dsId;
+		this.dsName = dsName;
+		this.dsOfficialName = dsOfficialName;
+		this.constraints = constraints;
+	}
+
+	public String getCommunity() {
+		return community;
+	}
+
+	public void setCommunity(final String community) {
+		this.community = community;
+	}
+
+	public String getDsId() {
+		return dsId;
+	}
+
+	public void setDsId(final String dsId) {
+		this.dsId = dsId;
+	}
+
+	public String getDsName() {
+		return dsName;
+	}
+
+	public void setDsName(final String dsName) {
+		this.dsName = dsName;
+	}
+
+	public String getDsOfficialName() {
+		return dsOfficialName;
+	}
+
+	public void setDsOfficialName(final String dsOfficialName) {
+		this.dsOfficialName = dsOfficialName;
+	}
+
+	public Boolean getEnabled() {
+		return enabled;
+	}
+
+	public void setEnabled(final Boolean enabled) {
+		this.enabled = enabled;
+	}
+
+	public SelectionCriteria getConstraints() {
+		return constraints;
+	}
+
+	public void setConstraints(final SelectionCriteria constraints) {
+		this.constraints = constraints;
+	}
+
+	public Boolean getDeposit() {
+		return deposit;
+	}
+
+	public void setDeposit(final Boolean deposit) {
+		this.deposit = deposit;
+	}
+
+	public String getMessage() {
+		return message;
+	}
+
+	public void setMessage(final String message) {
+		this.message = message;
+	}
+
+}

apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/model/DbDatasourcePK.java

@@ -0,0 +1,55 @@
+package eu.dnetlib.openaire.community.model;
+
+import java.io.Serializable;
+import java.util.Objects;
+
+public class DbDatasourcePK implements Serializable {
+
+	private static final long serialVersionUID = -8073510491611213955L;
+
+	private String community;
+
+	private String dsId;
+
+	public DbDatasourcePK() {}
+
+	public DbDatasourcePK(final String community, final String dsId) {
+		this.community = community;
+		this.dsId = dsId;
+	}
+
+	public String getCommunity() {
+		return community;
+	}
+
+	public void setCommunity(final String community) {
+		this.community = community;
+	}
+
+	public String getDsId() {
+		return dsId;
+	}
+
+	public void setDsId(final String dsId) {
+		this.dsId = dsId;
+	}
+
+	@Override
+	public int hashCode() {
+		return Objects.hash(community, dsId);
+	}
+
+	@Override
+	public boolean equals(final Object obj) {
+		if (this == obj) { return true; }
+		if (!(obj instanceof DbDatasourcePK)) { return false; }
+		final DbDatasourcePK other = (DbDatasourcePK) obj;
+		return Objects.equals(community, other.community) && Objects.equals(dsId, other.dsId);
+	}
+
+	@Override
+	public String toString() {
+		return String.format("CommunityDatasourcePK [community=%s, dsId=%s]", community, dsId);
+	}
+
+}

apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/model/DbOrganization.java

@@ -0,0 +1,54 @@
+package eu.dnetlib.openaire.community.model;
+
+import java.io.Serializable;
+
+import javax.persistence.Column;
+import javax.persistence.Entity;
+import javax.persistence.Id;
+import javax.persistence.IdClass;
+import javax.persistence.Table;
+
+@Entity
+@Table(name = "community_orgs")
+@IdClass(DbOrganizationPK.class)
+public class DbOrganization implements Serializable {
+
+	private static final long serialVersionUID = -602114117980437763L;
+
+	@Id
+	@Column(name = "community")
+	private String community;
+
+	@Id
+	@Column(name = "org_id")
+	private String orgId;
+
+	public DbOrganization() {}
+
+	public DbOrganization(final String community, final String orgId) {
+		this.community = community;
+		this.orgId = orgId;
+	}
+
+	public String getCommunity() {
+		return community;
+	}
+
+	public void setCommunity(final String community) {
+		this.community = community;
+	}
+
+	public String getOrgId() {
+		return orgId;
+	}
+
+	public void setOrgId(final String orgId) {
+		this.orgId = orgId;
+	}
+
+	@Override
+	public String toString() {
+		return String.format("DbOrganization [community=%s, orgId=%s]", community, orgId);
+	}
+
+}

apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/model/DbOrganizationPK.java

@@ -0,0 +1,54 @@
+package eu.dnetlib.openaire.community.model;
+
+import java.io.Serializable;
+import java.util.Objects;
+
+public class DbOrganizationPK implements Serializable {
+
+	private static final long serialVersionUID = -6720182815397534837L;
+
+	private String community;
+
+	private String orgId;
+
+	public DbOrganizationPK() {}
+
+	public DbOrganizationPK(final String community, final String orgId) {
+		this.community = community;
+		this.orgId = orgId;
+	}
+
+	public String getCommunity() {
+		return community;
+	}
+
+	public void setCommunity(final String community) {
+		this.community = community;
+	}
+
+	public String getOrgId() {
+		return orgId;
+	}
+
+	public void setOrgId(final String orgId) {
+		this.orgId = orgId;
+	}
+
+	@Override
+	public int hashCode() {
+		return Objects.hash(community, orgId);
+	}
+
+	@Override
+	public boolean equals(final Object obj) {
+		if (this == obj) { return true; }
+		if (!(obj instanceof DbDatasourcePK)) { return false; }
+		final DbOrganizationPK other = (DbOrganizationPK) obj;
+		return Objects.equals(community, other.community) && Objects.equals(orgId, other.orgId);
+	}
+
+	@Override
+	public String toString() {
+		return String.format("CommunityOrgPK [community=%s, orgId=%s]", community, orgId);
+	}
+}

apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/model/DbProject.java

@@ -0,0 +1,113 @@
+package eu.dnetlib.openaire.community.model;
+
+import java.io.Serializable;
+import java.time.LocalDate;
+
+import javax.persistence.Column;
+import javax.persistence.Entity;
+import javax.persistence.Id;
+import javax.persistence.IdClass;
+import javax.persistence.Table;
+
+import org.springframework.data.annotation.CreatedDate;
+
+@Entity
+@Table(name = "community_projects")
+@IdClass(DbProjectPK.class)
+public class DbProject implements Serializable {
+
+	private static final long serialVersionUID = 1649065971750517925L;
+
+	@Id
+	@Column(name = "community")
+	private String community;
+
+	@Id
+	@Column(name = "project_id")
+	private String projectId;
+
+	@Column(name = "project_code")
+	private String projectCode;
+
+	@Column(name = "project_name")
+	private String projectName;
+
+	@Column(name = "project_acronym")
+	private String projectAcronym;
+
+	@Column(name = "project_funder")
+	private String projectFunder;
+
+	@CreatedDate
+	@Column(name = "available_since")
+	private LocalDate availableSince;
+
+	public DbProject() {}
+
+	public DbProject(final String community, final String projectId, final String projectCode, final String projectName, final String projectAcronym,
+		final String projectFunder, final LocalDate availableSince) {
+		this.community = community;
+		this.projectId = projectId;
+		this.projectCode = projectCode;
+		this.projectName = projectName;
+		this.projectAcronym = projectAcronym;
+		this.projectFunder = projectFunder;
+		this.availableSince = availableSince;
+	}
+
+	public String getCommunity() {
+		return community;
+	}
+
+	public void setCommunity(final String community) {
+		this.community = community;
+	}
+
+	public String getProjectId() {
+		return projectId;
+	}
+
+	public void setProjectId(final String projectId) {
+		this.projectId = projectId;
+	}
+
+	public String getProjectCode() {
+		return projectCode;
+	}
+
+	public void setProjectCode(final String projectCode) {
+		this.projectCode = projectCode;
+	}
+
+	public String getProjectName() {
+		return projectName;
+	}
+
+	public void setProjectName(final String projectName) {
+		this.projectName = projectName;
+	}
+
+	public String getProjectAcronym() {
+		return projectAcronym;
+	}
+
+	public void setProjectAcronym(final String projectAcronym) {
+		this.projectAcronym = projectAcronym;
+	}
+
+	public String getProjectFunder() {
+		return projectFunder;
+	}
+
+	public void setProjectFunder(final String projectFunder) {
+		this.projectFunder = projectFunder;
+	}
+
+	public LocalDate getAvailableSince() {
+		return availableSince;
+	}
+
+	public void setAvailableSince(final LocalDate availableSince) {
+		this.availableSince = availableSince;
+	}
+}

apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/model/DbProjectPK.java

@@ -0,0 +1,54 @@
+package eu.dnetlib.openaire.community.model;
+
+import java.io.Serializable;
+import java.util.Objects;
+
+public class DbProjectPK implements Serializable {
+
+	private static final long serialVersionUID = -4236577148534835803L;
+
+	private String community;
+
+	private String projectId;
+
+	public DbProjectPK() {}
+
+	public DbProjectPK(final String community, final String projectId) {
+		this.community = community;
+		this.projectId = projectId;
+	}
+
+	public String getCommunity() {
+		return community;
+	}
+
+	public void setCommunity(final String community) {
+		this.community = community;
+	}
+
+	public String getProjectId() {
+		return projectId;
+	}
+
+	public void setProjectId(final String projectId) {
+		this.projectId = projectId;
+	}
+
+	@Override
+	public int hashCode() {
+		return Objects.hash(community, projectId);
+	}
+
+	@Override
+	public boolean equals(final Object obj) {
+		if (this == obj) { return true; }
+		if (!(obj instanceof DbProjectPK)) { return false; }
+		final DbProjectPK other = (DbProjectPK) obj;
+		return Objects.equals(community, other.community) && Objects.equals(projectId, other.projectId);
+	}
+
+	@Override
+	public String toString() {
+		return String.format("CommunityProjectPK [community=%s, projectId=%s]", community, projectId);
+	}
+}

apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/model/DbSubCommunity.java

@@ -0,0 +1,122 @@
+package eu.dnetlib.openaire.community.model;
+
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.List;
+
+import javax.persistence.Column;
+import javax.persistence.Entity;
+import javax.persistence.Id;
+import javax.persistence.Table;
+
+import org.hibernate.annotations.Type;
+import org.hibernate.annotations.TypeDef;
+import org.hibernate.annotations.TypeDefs;
+
+import com.vladmihalcea.hibernate.type.array.StringArrayType;
+import com.vladmihalcea.hibernate.type.json.JsonBinaryType;
+import com.vladmihalcea.hibernate.type.json.JsonStringType;
+
+import eu.dnetlib.openaire.exporter.model.context.Param;
+
+@Entity
+@Table(name = "community_subs")
+@TypeDefs({
+	@TypeDef(name = "string-array", typeClass = StringArrayType.class),
+	@TypeDef(name = "json", typeClass = JsonStringType.class),
+	@TypeDef(name = "jsonb", typeClass = JsonBinaryType.class)
+})
+public class DbSubCommunity implements Serializable {
+
+	private static final long serialVersionUID = 7104936574383307358L;
+
+	@Id
+	@Column(name = "sub_id")
+	private String id;
+
+	@Column(name = "community")
+	private String community;
+
+	@Column(name = "label")
+	private String label;
+
+	@Column(name = "category")
+	private String category;
+
+	@Type(type = "jsonb")
+	@Column(name = "params")
+	private List<Param> params = new ArrayList<>();
+
+	@Column(name = "parent")
+	private String parent;
+
+	@Column(name = "claim")
+	private boolean claim = false;
+
+	@Column(name = "browsable")
+	private boolean browsable = false;
+
+	public String getId() {
+		return id;
+	}
+
+	public void setId(final String id) {
+		this.id = id;
+	}
+
+	public String getCommunity() {
+		return community;
+	}
+
+	public void setCommunity(final String community) {
+		this.community = community;
+	}
+
+	public String getLabel() {
+		return label;
+	}
+
+	public void setLabel(final String label) {
+		this.label = label;
+	}
+
+	public String getCategory() {
+		return category;
+	}
+
+	public void setCategory(final String category) {
+		this.category = category;
+	}
+
+	public List<Param> getParams() {
+		return params;
+	}
+
+	public void setParams(final List<Param> params) {
+		this.params = params;
+	}
+
+	public String getParent() {
+		return parent;
+	}
+
+	public void setParent(final String parent) {
+		this.parent = parent;
+	}
+
+	public boolean isClaim() {
+		return claim;
+	}
+
+	public void setClaim(final boolean claim) {
+		this.claim = claim;
+	}
+
+	public boolean isBrowsable() {
+		return browsable;
+	}
+
+	public void setBrowsable(final boolean browsable) {
+		this.browsable = browsable;
+	}
+}

apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/model/DbSupportOrg.java

@@ -0,0 +1,64 @@
+package eu.dnetlib.openaire.community.model;
+
+import java.io.Serializable;
+
+import javax.persistence.Column;
+import javax.persistence.Entity;
+import javax.persistence.Id;
+import javax.persistence.IdClass;
+import javax.persistence.Table;
+
+@Entity
+@Table(name = "community_support_orgs")
+@IdClass(DbSupportOrgPK.class)
+public class DbSupportOrg implements Serializable {
+
+	private static final long serialVersionUID = 1308759097276753411L;
+
+	@Id
+	@Column(name = "community")
+	private String community;
+
+	@Id
+	@Column(name = "org_name")
+	private String orgName;
+
+	@Column(name = "org_url")
+	private String orgUrl;
+
+	@Column(name = "org_logourl")
+	private String orgLogoUrl;
+
+	public String getCommunity() {
+		return community;
+	}
+
+	public void setCommunity(final String community) {
+		this.community = community;
+	}
+
+	public String getOrgName() {
+		return orgName;
+	}
+
+	public void setOrgName(final String orgName) {
+		this.orgName = orgName;
+	}
+
+	public String getOrgUrl() {
+		return orgUrl;
+	}
+
+	public void setOrgUrl(final String orgUrl) {
+		this.orgUrl = orgUrl;
+	}
+
+	public String getOrgLogoUrl() {
+		return orgLogoUrl;
+	}
+
+	public void setOrgLogoUrl(final String orgLogoUrl) {
+		this.orgLogoUrl = orgLogoUrl;
+	}
+
+}

apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/model/DbSupportOrgPK.java

@@ -0,0 +1,55 @@
+package eu.dnetlib.openaire.community.model;
+
+import java.io.Serializable;
+import java.util.Objects;
+
+public class DbSupportOrgPK implements Serializable {
+
+	private static final long serialVersionUID = -4117154543803798310L;
+
+	private String community;
+
+	private String orgName;
+
+	public DbSupportOrgPK() {}
+
+	public DbSupportOrgPK(final String community, final String orgName) {
+		this.community = community;
+		this.orgName = orgName;
+	}
+
+	public String getCommunity() {
+		return community;
+	}
+
+	public void setCommunity(final String community) {
+		this.community = community;
+	}
+
+	public String getOrgName() {
+		return orgName;
+	}
+
+	public void setOrgName(final String orgName) {
+		this.orgName = orgName;
+	}
+
+	@Override
+	public int hashCode() {
+		return Objects.hash(community, orgName);
+	}
+
+	@Override
+	public boolean equals(final Object obj) {
+		if (this == obj) { return true; }
+		if (!(obj instanceof DbSupportOrgPK)) { return false; }
+		final DbSupportOrgPK other = (DbSupportOrgPK) obj;
+		return Objects.equals(community, other.community) && Objects.equals(orgName, other.orgName);
+	}
+
+	@Override
+	public String toString() {
+		return String.format("CommunitySupportOrgPK [community=%s, orgName=%s]", community, orgName);
+	}
+
+}

apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/model/DepositionInfo.java

@@ -0,0 +1,39 @@
+package eu.dnetlib.openaire.community.model;
+
+import java.io.Serializable;
+
+public class DepositionInfo implements Serializable {
+
+	private static final long serialVersionUID = 7538663287451167904L;
+
+	private String openaireId;
+
+	private Boolean deposit;
+
+	private String message;
+
+	public String getOpenaireId() {
+		return openaireId;
+	}
+
+	public void setOpenaireId(final String openaireId) {
+		this.openaireId = openaireId;
+	}
+
+	public Boolean getDeposit() {
+		return deposit;
+	}
+
+	public void setDeposit(final Boolean deposit) {
+		this.deposit = deposit;
+	}
+
+	public String getMessage() {
+		return message;
+	}
+
+	public void setMessage(final String message) {
+		this.message = message;
+	}
+
+}

apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/repository/DbCommunityRepository.java

@@ -0,0 +1,17 @@
+package eu.dnetlib.openaire.community.repository;
+
+import java.util.List;
+
+import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
+import org.springframework.data.jpa.repository.JpaRepository;
+import org.springframework.data.jpa.repository.Query;
+
+import eu.dnetlib.openaire.community.model.DbCommunity;
+
+@ConditionalOnProperty(value = "openaire.exporter.enable.community", havingValue = "true")
+public interface DbCommunityRepository extends JpaRepository<DbCommunity, String> {
+
+	@Query(value = "select id from communities where ?1 = ANY(array_append(other_zenodo_communities, main_zenodo_community))", nativeQuery = true)
+	List<String> findByZenodoId(String zenodoId);
+
+}

apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/repository/DbDatasourceRepository.java

@@ -0,0 +1,20 @@
+package eu.dnetlib.openaire.community.repository;
+
+import java.util.List;
+
+import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
+import org.springframework.data.jpa.repository.JpaRepository;
+
+import eu.dnetlib.openaire.community.model.DbDatasource;
+import eu.dnetlib.openaire.community.model.DbDatasourcePK;
+
+@ConditionalOnProperty(value = "openaire.exporter.enable.community", havingValue = "true")
+public interface DbDatasourceRepository extends JpaRepository<DbDatasource, DbDatasourcePK> {
+
+	List<DbDatasource> findByCommunity(String community);
+
+	List<DbDatasource> findByCommunityAndDeposit(String community, boolean deposit);
+
+	void deleteByCommunity(String id);
+
+}

apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/repository/DbOrganizationRepository.java

@@ -0,0 +1,18 @@
+package eu.dnetlib.openaire.community.repository;
+
+import java.util.List;
+
+import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
+import org.springframework.data.jpa.repository.JpaRepository;
+
+import eu.dnetlib.openaire.community.model.DbOrganization;
+import eu.dnetlib.openaire.community.model.DbOrganizationPK;
+
+@ConditionalOnProperty(value = "openaire.exporter.enable.community", havingValue = "true")
+public interface DbOrganizationRepository extends JpaRepository<DbOrganization, DbOrganizationPK> {
+
+	List<DbOrganization> findByCommunity(String community);
+
+	void deleteByCommunity(String id);
+
+}

apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/repository/DbProjectRepository.java

@@ -0,0 +1,25 @@
+package eu.dnetlib.openaire.community.repository;
+
+import java.util.List;
+
+import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
+import org.springframework.data.domain.Page;
+import org.springframework.data.domain.Pageable;
+import org.springframework.data.jpa.repository.JpaRepository;
+import org.springframework.data.jpa.repository.JpaSpecificationExecutor;
+import org.springframework.data.jpa.repository.Query;
+
+import eu.dnetlib.openaire.community.model.DbProject;
+import eu.dnetlib.openaire.community.model.DbProjectPK;
+
+@ConditionalOnProperty(value = "openaire.exporter.enable.community", havingValue = "true")
+public interface DbProjectRepository extends JpaRepository<DbProject, DbProjectPK>, JpaSpecificationExecutor<DbProject> {
+
+	Page<DbProject> findByCommunity(String community, Pageable page);
+
+	void deleteByCommunity(String id);
+
+	@Query(value = "select distinct project_funder from community_projects where community = ?1 order by project_funder", nativeQuery = true)
+	List<String> findFundersByCommunity(String id);
+
+}

apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/repository/DbSubCommunityRepository.java

@@ -0,0 +1,19 @@
+package eu.dnetlib.openaire.community.repository;
+
+import java.util.List;
+
+import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
+import org.springframework.data.jpa.repository.JpaRepository;
+
+import eu.dnetlib.openaire.community.model.DbSubCommunity;
+
+@ConditionalOnProperty(value = "openaire.exporter.enable.community", havingValue = "true")
+public interface DbSubCommunityRepository extends JpaRepository<DbSubCommunity, String> {
+
+	List<DbSubCommunity> findByCommunity(String community);
+
+	List<DbSubCommunity> findByCommunityAndParent(String community, String parent);
+
+	void deleteByCommunity(String id);
+
+}

apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/repository/DbSupportOrgRepository.java

@@ -0,0 +1,18 @@
+package eu.dnetlib.openaire.community.repository;
+
+import java.util.List;
+
+import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
+import org.springframework.data.jpa.repository.JpaRepository;
+
+import eu.dnetlib.openaire.community.model.DbSupportOrg;
+import eu.dnetlib.openaire.community.model.DbSupportOrgPK;
+
+@ConditionalOnProperty(value = "openaire.exporter.enable.community", havingValue = "true")
+public interface DbSupportOrgRepository extends JpaRepository<DbSupportOrg, DbSupportOrgPK> {
+
+	List<DbSupportOrg> findByCommunity(String community);
+
+	void deleteByCommunity(String id);
+
+}

apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/utils/CommunityClaimTypeConverter.java

@@ -0,0 +1,29 @@
+package eu.dnetlib.openaire.community.utils;
+
+import javax.persistence.AttributeConverter;
+
+import org.apache.commons.lang3.StringUtils;
+
+import eu.dnetlib.openaire.exporter.model.community.CommunityClaimType;
+
+public class CommunityClaimTypeConverter implements AttributeConverter<CommunityClaimType, String> {
+
+	@Override
+	public String convertToDatabaseColumn(final CommunityClaimType attribute) {
+		if (attribute == null) {
+			return null;
+		} else {
+			return attribute.getDescription();
+		}
+	}
+
+	@Override
+	public CommunityClaimType convertToEntityAttribute(final String dbData) {
+		if (StringUtils.isBlank(dbData)) {
+			return null;
+		} else {
+			return CommunityClaimType.fromDescription(dbData);
+		}
+	}
+
+}

apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/utils/CommunityMappingUtils.java

@@ -0,0 +1,315 @@
+package eu.dnetlib.openaire.community.utils;
+
+import java.text.ParseException;
+import java.time.LocalDate;
+import java.time.LocalDateTime;
+import java.time.ZoneId;
+import java.util.Arrays;
+import java.util.Date;
+import java.util.List;
+
+import org.apache.commons.lang3.ObjectUtils;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.lang3.time.DateUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+import com.google.common.collect.Lists;
+
+import eu.dnetlib.openaire.community.importer.CommunityImporterService;
+import eu.dnetlib.openaire.community.model.DbCommunity;
+import eu.dnetlib.openaire.community.model.DbDatasource;
+import eu.dnetlib.openaire.community.model.DbProject;
+import eu.dnetlib.openaire.community.model.DbSubCommunity;
+import eu.dnetlib.openaire.community.model.DbSupportOrg;
+import eu.dnetlib.openaire.exporter.model.community.CommunityContentprovider;
+import eu.dnetlib.openaire.exporter.model.community.CommunityDetails;
+import eu.dnetlib.openaire.exporter.model.community.CommunityOrganization;
+import eu.dnetlib.openaire.exporter.model.community.CommunityProject;
+import eu.dnetlib.openaire.exporter.model.community.CommunitySummary;
+import eu.dnetlib.openaire.exporter.model.community.CommunityWritableProperties;
+import eu.dnetlib.openaire.exporter.model.community.SubCommunity;
+import eu.dnetlib.openaire.exporter.model.context.IISConfigurationEntry;
+
+public class CommunityMappingUtils {
+
+	public final static String PIPE_SEPARATOR = "||";
+
+	private static final List<String> DATE_PATTERN = Lists.newArrayList("yyyy-MM-dd'T'hh:mm:ss", "yyyy-MM-dd'T'hh:mm:ssXXX", "yyyy-MM-dd'T'hh:mm:ss+00:00");
+
+	private static final Log log = LogFactory.getLog(CommunityMappingUtils.class);
+
+	public static CommunitySummary toCommunitySummary(final DbCommunity c) {
+		final CommunitySummary summary = new CommunitySummary();
+		populateSummary(summary, c);
+		return summary;
+	}
+
+	public static DbCommunity toCommunity(final CommunityDetails details) {
+		final DbCommunity c = new DbCommunity();
+		c.setId(details.getId());
+		c.setName(details.getName());
+		c.setShortName(details.getShortName());
+		c.setDisplayName(details.getDisplayName());
+		c.setDisplayShortName(details.getDisplayShortName());
+		c.setDescription(details.getDescription());
+		c.setStatus(details.getStatus());
+		c.setLogoUrl(details.getLogoUrl());
+		c.setMembership(details.getMembership());
+		c.setType(details.getType());
+		c.setClaim(details.getClaim());
+		c.setSubjects(toStringArray(details.getSubjects()));
+		c.setFos(toStringArray(details.getFos()));
+		c.setSdg(toStringArray(details.getSdg()));
+		c.setAdvancedConstraints(details.getAdvancedConstraints());
+		c.setRemoveConstraints(details.getRemoveConstraints());
+		c.setMainZenodoCommunity(details.getZenodoCommunity());
+		c.setOtherZenodoCommunities(toStringArray(details.getOtherZenodoCommunities()));
+		c.setSuggestedAcknowledgements(toStringArray(details.getSuggestedAcknowledgements()));
+		c.setPlan(details.getPlan());
+		c.setCreationDate(ObjectUtils.firstNonNull(details.getCreationDate(), LocalDateTime.now()));
+		c.setLastUpdateDate(LocalDateTime.now());
+		return c;
+	}
+
+	public static void populateCommunity(final DbCommunity c, final CommunityWritableProperties details) {
+		if (StringUtils.isNotBlank(details.getName())) {
+			c.setName(details.getName());
+		}
+		if (StringUtils.isNotBlank(details.getShortName())) {
+			c.setShortName(details.getShortName());
+		}
+		if (StringUtils.isNotBlank(details.getDisplayName())) {
+			c.setDisplayName(details.getDisplayName());
+		}
+		if (StringUtils.isNotBlank(details.getDisplayShortName())) {
+			c.setDisplayShortName(details.getDisplayShortName());
+		}
+		if (StringUtils.isNotBlank(details.getDescription())) {
+			c.setDescription(details.getDescription());
+		}
+		if (details.getStatus() != null) {
+			c.setStatus(details.getStatus());
+		}
+		if (details.getMembership() != null) {
+			c.setMembership(details.getMembership());
+		}
+		if (details.getType() != null) {
+			c.setType(details.getType());
+		}
+		if (details.getClaim() != null) {
+			c.setClaim(details.getClaim());
+		}
+		if (StringUtils.isNotBlank(details.getLogoUrl())) {
+			c.setLogoUrl(details.getLogoUrl());
+		}
+		if (details.getFos() != null) {
+			c.setFos(toStringArray(details.getFos()));
+		}
+		if (details.getSdg() != null) {
+			c.setSdg(toStringArray(details.getSdg()));
+		}
+		if (details.getSubjects() != null) {
+			c.setSubjects(toStringArray(details.getSubjects()));
+		}
+		if (details.getAdvancedConstraints() != null) {
+			c.setAdvancedConstraints(details.getAdvancedConstraints());
+		}
+		if (details.getRemoveConstraints() != null) {
+			c.setRemoveConstraints(details.getRemoveConstraints());
+		}
+		if (StringUtils.isNotBlank(details.getMainZenodoCommunity())) {
+			c.setMainZenodoCommunity(details.getMainZenodoCommunity());
+		}
+		if (details.getOtherZenodoCommunities() != null) {
+			c.setOtherZenodoCommunities(toStringArray(details.getOtherZenodoCommunities()));
+		}
+		if (details.getPlan() != null) {
+			c.setPlan(details.getPlan());
+		}
+
+		c.setLastUpdateDate(LocalDateTime.now());
+	}
+
+	public static CommunityDetails toCommunityDetails(final DbCommunity c) {
+		final CommunityDetails details = new CommunityDetails();
+		populateSummary(details, c);
+		details.setAdvancedConstraints(c.getAdvancedConstraints());
+		details.setRemoveConstraints(c.getRemoveConstraints());
+		details.setFos(Arrays.asList(c.getFos()));
+		details.setSdg(Arrays.asList(c.getSdg()));
+		details.setSubjects(Arrays.asList(c.getSubjects()));
+		details.setOtherZenodoCommunities(Arrays.asList(c.getOtherZenodoCommunities()));
+		details.setSuggestedAcknowledgements(Arrays.asList(c.getSuggestedAcknowledgements()));
+		return details;
+	}
+
+	private static void populateSummary(final CommunitySummary summary, final DbCommunity c) {
+		summary.setId(c.getId());
+		summary.setName(c.getName());
+		summary.setShortName(c.getShortName());
+		summary.setDisplayName(c.getDisplayName());
+		summary.setDisplayShortName(c.getDisplayShortName());
+		summary.setLastUpdateDate(c.getLastUpdateDate());
+		summary.setCreationDate(c.getCreationDate());
+		summary.setQueryId(c.getId() + PIPE_SEPARATOR + c.getShortName());
+		summary.setType(c.getType());
+		summary.setDescription(c.getDescription());
+		summary.setLogoUrl(c.getLogoUrl());
+		summary.setStatus(c.getStatus());
+		summary.setClaim(c.getClaim());
+		summary.setMembership(c.getMembership());
+		summary.setZenodoCommunity(c.getMainZenodoCommunity());
+		summary.setPlan(c.getPlan());
+	}
+
+	public static CommunityProject toCommunityProject(final DbProject dbEntry) {
+		final CommunityProject cp = new CommunityProject();
+		cp.setCommunityId(dbEntry.getCommunity());
+		cp.setOpenaireId(dbEntry.getProjectId());
+		cp.setName(dbEntry.getProjectName());
+		cp.setAcronym(dbEntry.getProjectAcronym());
+		cp.setFunder(dbEntry.getProjectFunder());
+		cp.setGrantId(dbEntry.getProjectCode());
+		cp.setAvailableSince(dbEntry.getAvailableSince());
+		return cp;
+	}
+
+	public static DbProject toDbProject(final String id, final CommunityProject project) {
+		final DbProject p = new DbProject();
+		p.setCommunity(id);
+		p.setProjectId(project.getOpenaireId());
+		p.setProjectName(project.getName());
+		p.setProjectAcronym(project.getAcronym());
+		p.setProjectCode(project.getGrantId());
+		p.setProjectFunder(project.getFunder());
+		if (project.getAvailableSince() != null) {
+			p.setAvailableSince(project.getAvailableSince());
+		} else {
+			p.setAvailableSince(LocalDate.now());
+		}
+		return p;
+	}
+
+	public static CommunityContentprovider toCommunityContentprovider(final DbDatasource dbEntry) {
+		final CommunityContentprovider ccp = new CommunityContentprovider();
+		ccp.setCommunityId(dbEntry.getCommunity());
+		ccp.setOpenaireId(dbEntry.getDsId());
+		ccp.setName(dbEntry.getDsName());
+		ccp.setOfficialname(dbEntry.getDsOfficialName());
+		ccp.setSelectioncriteria(dbEntry.getConstraints());
+		ccp.setEnabled(dbEntry.getEnabled() != null ? dbEntry.getEnabled() : true);
+		ccp.setDeposit(dbEntry.getDeposit() != null ? dbEntry.getDeposit() : false);
+		ccp.setMessage(dbEntry.getMessage());
+		return ccp;
+	}
+
+	public static DbDatasource toDbDatasource(final String id, final CommunityContentprovider provider) {
+		final DbDatasource ds = new DbDatasource();
+		ds.setCommunity(id);
+		ds.setDsId(provider.getOpenaireId());
+		ds.setDsName(provider.getName());
+		ds.setDsOfficialName(provider.getOfficialname());
+		ds.setConstraints(provider.getSelectioncriteria());
+		ds.setEnabled(provider.isEnabled());
+		ds.setDeposit(provider.getDeposit() != null ? provider.getDeposit() : false);
+		ds.setMessage(provider.getMessage());
+		return ds;
+	}
+
+	public static CommunityOrganization toCommunityOrganization(final DbSupportOrg dbEntry) {
+		final CommunityOrganization co = new CommunityOrganization();
+		co.setCommunityId(dbEntry.getCommunity());
+		co.setName(dbEntry.getOrgName());
+		co.setWebsite_url(dbEntry.getOrgUrl());
+		co.setLogo_url(dbEntry.getOrgLogoUrl());
+		return co;
+	}
+
+	public static DbSupportOrg toDbSupportOrg(final String id, final CommunityOrganization org) {
+		final DbSupportOrg dbo = new DbSupportOrg();
+		dbo.setCommunity(id);
+		dbo.setOrgName(org.getName());
+		dbo.setOrgUrl(org.getWebsite_url());
+		dbo.setOrgLogoUrl(org.getLogo_url());
+		return dbo;
+	}
+
+	public static DbSubCommunity toDbSubCommunity(final String id, final SubCommunity sub) {
+		final DbSubCommunity dbsc = new DbSubCommunity();
+		dbsc.setCommunity(id);
+		dbsc.setId(sub.getSubCommunityId());
+		dbsc.setCategory(sub.getCategory());
+		dbsc.setLabel(sub.getLabel());
+		dbsc.setParams(sub.getParams());
+		dbsc.setParent(sub.getParent());
+		dbsc.setClaim(sub.isClaim());
+		dbsc.setBrowsable(sub.isBrowsable());
+		return dbsc;
+	}
+
+	public static SubCommunity toSubCommunity(final DbSubCommunity sub) {
+		final SubCommunity sc = new SubCommunity();
+		sc.setSubCommunityId(sub.getId());
+		sc.setCategory(sub.getCategory());
+		sc.setCommunityId(sub.getCommunity());
+		sc.setLabel(sub.getLabel());
+		sc.setParams(sub.getParams());
+		sc.setParent(sub.getParent());
+		sc.setClaim(sub.isClaim());
+		sc.setBrowsable(sub.isBrowsable());
+		return sc;
+	}
+
+	public static LocalDateTime asLocalDateTime(final String s) {
+		if (StringUtils.isBlank(s)) { return null; }
+
+		for (final String pattern : DATE_PATTERN) {
+			try {
+				final Date res = DateUtils.parseDate(s, pattern);
+				if (res != null) { return asLocalDateTime(res); }
+			} catch (final ParseException e) {}
+		}
+		log.warn("Invalid Date: " + s);
+		return null;
+	}
+
+	public static LocalDateTime asLocalDateTime(final Date date) {
+		return date.toInstant()
+				.atZone(ZoneId.systemDefault())
+				.toLocalDateTime();
+	}
+
+	private static String[] toStringArray(final List<String> list) {
+		return list != null ? list.toArray(new String[list.size()]) : new String[0];
+	}
+
+	public static IISConfigurationEntry asIISConfigurationEntry(final DbCommunity c) {
+		final IISConfigurationEntry conf = new IISConfigurationEntry();
+		conf.setId(c.getId());
+		conf.setLabel(c.getName());
+		conf.addParams(CommunityImporterService.CSUMMARY_DESCRIPTION, c.getDescription());
+		conf.addParams(CommunityImporterService.CSUMMARY_LOGOURL, c.getLogoUrl());
+		conf.addParams(CommunityImporterService.CSUMMARY_STATUS, c.getStatus().toString());
+		conf.addParams(CommunityImporterService.CSUMMARY_NAME, c.getName());
+		conf.addParams(CommunityImporterService.CSUMMARY_ZENODOC, c.getMainZenodoCommunity());
+		conf.addParams(CommunityImporterService.CPROFILE_SUBJECT, c.getSubjects());
+		conf.addParams(CommunityImporterService.CPROFILE_FOS, c.getFos());
+		conf.addParams(CommunityImporterService.CPROFILE_SDG, c.getSdg());
+		conf.addParams(CommunityImporterService.CPROFILE_ADVANCED_CONSTRAINT, c.getAdvancedConstraints() != null ? c.getAdvancedConstraints().toJson() : null);
+		conf.addParams(CommunityImporterService.CPROFILE_REMOVE_CONSTRAINT, c.getRemoveConstraints() != null ? c.getRemoveConstraints().toJson() : null);
+		conf.addParams(CommunityImporterService.CPROFILE_SUGGESTED_ACKNOWLEDGEMENT, c.getSuggestedAcknowledgements());
+		conf.addParams(CommunityImporterService.CPROFILE_CREATIONDATE, c.getCreationDate() != null ? c.getCreationDate().toString() : null);
+		return conf;
+	}
+
+	public static IISConfigurationEntry asIISConfigurationEntry(final DbSubCommunity subc) {
+		final IISConfigurationEntry conf = new IISConfigurationEntry();
+		conf.setId(subc.getId());
+		conf.setLabel(subc.getLabel());
+		if (subc.getParams() != null) {
+			conf.getParams().addAll(subc.getParams());
+		}
+		return conf;
+	}
+}

apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/utils/CommunityMembershipTypeConverter.java

@@ -0,0 +1,29 @@
+package eu.dnetlib.openaire.community.utils;
+
+import javax.persistence.AttributeConverter;
+
+import org.apache.commons.lang3.StringUtils;
+
+import eu.dnetlib.openaire.exporter.model.community.CommunityMembershipType;
+
+public class CommunityMembershipTypeConverter implements AttributeConverter<CommunityMembershipType, String> {
+
+	@Override
+	public String convertToDatabaseColumn(final CommunityMembershipType attribute) {
+		if (attribute == null) {
+			return null;
+		} else {
+			return attribute.getDescription();
+		}
+	}
+
+	@Override
+	public CommunityMembershipType convertToEntityAttribute(final String dbData) {
+		if (StringUtils.isBlank(dbData)) {
+			return null;
+		} else {
+			return CommunityMembershipType.fromDescription(dbData);
+		}
+	}
+
+}

apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/context/ContextApiController.java

@@ -1,8 +1,13 @@
 package eu.dnetlib.openaire.context;
 
 import java.util.List;
-import java.util.Optional;
+import java.util.Objects;
+import java.util.Set;
+import java.util.stream.Collectors;
 
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
 import org.springframework.web.bind.annotation.CrossOrigin;
@@ -12,10 +17,16 @@ import org.springframework.web.bind.annotation.RequestMethod;
 import org.springframework.web.bind.annotation.RequestParam;
 import org.springframework.web.bind.annotation.RestController;
 
-import eu.dnetlib.openaire.exporter.exceptions.ContextException;
+import eu.dnetlib.openaire.common.AbstractExporterController;
+import eu.dnetlib.openaire.community.CommunityService;
+import eu.dnetlib.openaire.exporter.exceptions.CommunityException;
+import eu.dnetlib.openaire.exporter.exceptions.ResourceNotFoundException;
+import eu.dnetlib.openaire.exporter.model.community.CommunityType;
+import eu.dnetlib.openaire.exporter.model.community.SubCommunity;
 import eu.dnetlib.openaire.exporter.model.context.CategorySummary;
 import eu.dnetlib.openaire.exporter.model.context.ConceptSummary;
 import eu.dnetlib.openaire.exporter.model.context.ContextSummary;
+import eu.dnetlib.openaire.exporter.model.context.IISConfigurationEntry;
 import io.swagger.v3.oas.annotations.Operation;
 import io.swagger.v3.oas.annotations.responses.ApiResponse;
 import io.swagger.v3.oas.annotations.responses.ApiResponses;
@@ -25,12 +36,14 @@ import io.swagger.v3.oas.annotations.tags.Tag;
 @CrossOrigin(origins = {
 	"*"
 })
-@ConditionalOnProperty(value = "openaire.exporter.enable.context", havingValue = "true")
+@ConditionalOnProperty(value = "openaire.exporter.enable.community", havingValue = "true")
 @Tag(name = "OpenAIRE Context API", description = "the OpenAIRE Context API")
-public class ContextApiController {
+public class ContextApiController extends AbstractExporterController {
 
 	@Autowired
-	private ContextApiCore contextApiCore;
+	private CommunityService communityService;
+
+	private static final Log log = LogFactory.getLog(ContextApiController.class);
 
 	@RequestMapping(value = "/contexts", produces = {
 		"application/json"
@@ -38,10 +51,28 @@ public class ContextApiController {
 	@Operation(summary = "list brief information about all the context profiles", description = "list brief information about all the context profiles")
 	@ApiResponses(value = {
 		@ApiResponse(responseCode = "200", description = "OK"),
+		@ApiResponse(responseCode = "404", description = "not found"),
 		@ApiResponse(responseCode = "500", description = "unexpected error")
 	})
-	public List<ContextSummary> listContexts(@RequestParam(required = false, defaultValue = "") final List<String> type) throws ContextException {
-		return contextApiCore.listContexts(type);
+	public List<ContextSummary> listContexts(@RequestParam(required = false) final Set<CommunityType> type) throws CommunityException {
+		try {
+			return communityService.listCommunities()
+				.stream()
+				.filter(c -> type == null || type.contains(c.getType()))
+				.map(c -> {
+					final ContextSummary ctx = new ContextSummary();
+					ctx.setId(c.getId());
+					ctx.setLabel(c.getName());
+					ctx.setStatus(c.getStatus().toString());
+					ctx.setType(c.getType().toString());
+					return ctx;
+				})
+				.collect(Collectors.toList());
+		} catch (final ResourceNotFoundException e) {
+			throw e;
+		} catch (final Throwable e) {
+			throw new CommunityException(e);
+		}
 	}
 
 	@RequestMapping(value = "/context/{contextId}", produces = {
@@ -50,14 +81,52 @@ public class ContextApiController {
 	@Operation(summary = "list the categories defined within a context", description = "list the categories defined within a context")
 	@ApiResponses(value = {
 		@ApiResponse(responseCode = "200", description = "OK"),
+		@ApiResponse(responseCode = "404", description = "not found"),
 		@ApiResponse(responseCode = "500", description = "unexpected error")
 	})
 	public List<CategorySummary> listCategories(
 		@PathVariable final String contextId,
-		@RequestParam(required = false, defaultValue = "false") final Boolean all) throws ContextException {
+		@RequestParam(required = false, defaultValue = "false") final boolean all) throws CommunityException {
 
-		final Boolean allFilter = Optional.ofNullable(all).orElse(false);
-		return contextApiCore.listCategories(contextId, allFilter);
+		try {
+			return communityService.getSubCommunities(contextId)
+				.stream()
+				.filter(sc -> all || sc.isClaim())
+				.map(sc -> {
+					final String[] parts = StringUtils.split(sc.getSubCommunityId(), "::");
+					if (parts.length < 3) { throw new RuntimeException("Invalid conceptId (It should have 3 (or more) parts): " + sc.getSubCommunityId()); }
+					final CategorySummary cat = new CategorySummary();
+					cat.setId(parts[0] + "::" + parts[1]);
+					cat.setLabel(sc.getCategory());
+					cat.setHasConcept(true);
+					return cat;
+				})
+				.distinct()
+				.collect(Collectors.toList());
+		} catch (final ResourceNotFoundException e) {
+			throw e;
+		} catch (final Throwable e) {
+			throw new CommunityException(e);
+		}
+	}
+
+	@RequestMapping(value = "/context/iis/conf/{contextId}", produces = {
+		"application/json"
+	}, method = RequestMethod.GET)
+	@Operation(summary = "return a list of entries for IIS", description = "return a list of entries for IIS")
+	@ApiResponses(value = {
+		@ApiResponse(responseCode = "200", description = "OK"),
+		@ApiResponse(responseCode = "404", description = "not found"),
+		@ApiResponse(responseCode = "500", description = "unexpected error")
+	})
+	public List<IISConfigurationEntry> getIISConfiguration(@PathVariable final String contextId) throws CommunityException {
+		try {
+			return communityService.getIISConfiguration(contextId);
+		} catch (final ResourceNotFoundException e) {
+			throw e;
+		} catch (final Throwable e) {
+			throw new CommunityException(e);
+		}
 	}
 
 	@RequestMapping(value = "/context/category/{categoryId}", produces = {
@@ -66,14 +135,30 @@ public class ContextApiController {
 	@Operation(summary = "list the concepts defined within a category", description = "list the concepts defined within a category")
 	@ApiResponses(value = {
 		@ApiResponse(responseCode = "200", description = "OK"),
+		@ApiResponse(responseCode = "404", description = "not found"),
 		@ApiResponse(responseCode = "500", description = "unexpected error")
 	})
 	public List<ConceptSummary> listConcepts(
 		@PathVariable final String categoryId,
-		@RequestParam(required = false, defaultValue = "false") final Boolean all) throws ContextException {
+		@RequestParam(required = false, defaultValue = "false") final boolean all) throws CommunityException {
+		try {
+			final String[] parts = StringUtils.split(categoryId, "::");
+			if (parts.length != 2) {
+				log.error("Invalid category id (it should have 2 parts): " + categoryId);
+				throw new CommunityException("Invalid category id (it should have 2 parts): " + categoryId);
+			}
+
+			final String contextId = parts[0];
+
+			final List<SubCommunity> list = findSubCommunities(categoryId + "::", all, contextId);
+
+			return processSubCommunities(null, list);
+		} catch (final ResourceNotFoundException e) {
+			throw e;
+		} catch (final Throwable e) {
+			throw new CommunityException(e);
+		}
 
-		final Boolean allFilter = Optional.ofNullable(all).orElse(false);
-		return contextApiCore.listConcepts(categoryId, allFilter);
 	}
 
 	@RequestMapping(value = "/context/category/concept/{conceptId}", produces = {
@@ -82,14 +167,53 @@ public class ContextApiController {
 	@Operation(summary = "list the concepts defined within a category", description = "list the concepts defined within a category")
 	@ApiResponses(value = {
 		@ApiResponse(responseCode = "200", description = "OK"),
+		@ApiResponse(responseCode = "404", description = "not found"),
 		@ApiResponse(responseCode = "500", description = "unexpected error")
 	})
 	public List<ConceptSummary> listSubConcepts(
 		@PathVariable final String conceptId,
-		@RequestParam(required = false, defaultValue = "false") final Boolean all) throws ContextException {
+		@RequestParam(required = false, defaultValue = "false") final boolean all) throws CommunityException {
+		try {
+			final String[] parts = StringUtils.split(conceptId, "::");
+			if (parts.length < 3) {
+				log.error("Invalid concept id (it should have 3 (or more) parts): " + conceptId);
+				throw new CommunityException("Invalid concept id (it should have 3 (or more) parts): " + conceptId);
+			}
 
-		final Boolean allFilter = Optional.ofNullable(all).orElse(false);
-		return contextApiCore.listSubConcepts(conceptId, allFilter);
+			final String contextId = parts[0];
+
+			final List<SubCommunity> list = findSubCommunities(conceptId + "::", all, contextId);
+
+			return processSubCommunities(conceptId, list);
+		} catch (final ResourceNotFoundException e) {
+			throw e;
+		} catch (final Throwable e) {
+			throw new CommunityException(e);
+		}
+
+	}
+
+	private List<SubCommunity> findSubCommunities(final String prefix, final boolean all, final String contextId) throws CommunityException {
+		return communityService.getSubCommunities(contextId)
+			.stream()
+			.filter(sc -> all || sc.isClaim())
+			.filter(sc -> sc.getSubCommunityId().startsWith(prefix))
+			.collect(Collectors.toList());
+	}
+
+	private List<ConceptSummary> processSubCommunities(final String parent, final List<SubCommunity> list) {
+		return list.stream()
+			.filter(sc -> Objects.equals(sc.getParent(), parent))
+			.map(sc -> {
+				final List<ConceptSummary> childs = processSubCommunities(sc.getSubCommunityId(), list);
+				final ConceptSummary concept = new ConceptSummary();
+				concept.setId(sc.getSubCommunityId());
+				concept.setLabel(sc.getLabel());
+				concept.setHasSubConcept(!childs.isEmpty());
+				concept.setConcept(childs);
+				return concept;
+			})
+			.collect(Collectors.toList());
 	}
 
 }

apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/context/ContextApiCore.java

@@ -8,14 +8,12 @@ import java.util.stream.Stream;
 
 import org.apache.commons.lang3.StringUtils;
 import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
-import org.springframework.stereotype.Component;
 
 import com.google.common.base.Splitter;
 import com.google.common.collect.Lists;
 
 import eu.dnetlib.openaire.common.ISClient;
-import eu.dnetlib.openaire.exporter.exceptions.ContextException;
+import eu.dnetlib.openaire.exporter.exceptions.CommunityException;
 import eu.dnetlib.openaire.exporter.model.context.Category;
 import eu.dnetlib.openaire.exporter.model.context.CategorySummary;
 import eu.dnetlib.openaire.exporter.model.context.Concept;
@@ -23,8 +21,9 @@ import eu.dnetlib.openaire.exporter.model.context.ConceptSummary;
 import eu.dnetlib.openaire.exporter.model.context.Context;
 import eu.dnetlib.openaire.exporter.model.context.ContextSummary;
 
-@Component
-@ConditionalOnProperty(value = "openaire.exporter.enable.context", havingValue = "true")
+// @Component
+// @ConditionalOnProperty(value = "openaire.exporter.enable.context", havingValue = "true")
+@Deprecated
 public class ContextApiCore {
 
 	private static final String SEPARATOR = "::";
@@ -32,18 +31,24 @@ public class ContextApiCore {
 	@Autowired
 	private ISClient isClient;
 
-	public List<ContextSummary> listContexts(final List<String> type) throws ContextException {
+	public List<ContextSummary> listContexts(final List<String> type) throws CommunityException {
+
 		return getContextMap(type).values()
 			.stream()
 			.map(c -> new ContextSummary()
 				.setId(c.getId())
 				.setType(c.getType())
 				.setLabel(c.getLabel())
-				.setStatus(c.getParams().containsKey("status") ? c.getParams().get("status").get(0).getValue() : ""))
+				.setStatus(c.getParams()
+					.stream()
+					.filter(p -> p.getName().equals("status"))
+					.map(p -> p.getValue())
+					.findFirst()
+					.orElse("")))
 			.collect(Collectors.toList());
 	}
 
-	public List<CategorySummary> listCategories(final String contextId, final Boolean all) throws ContextException {
+	public List<CategorySummary> listCategories(final String contextId, final Boolean all) throws CommunityException {
 		final Stream<Category> categories = getContextMap().get(contextId).getCategories().values().stream();
 		return all ? asCategorySummaries(categories) : asCategorySummaries(categories.filter(Category::isClaim));
 	}
@@ -57,7 +62,7 @@ public class ContextApiCore {
 			.collect(Collectors.toList());
 	}
 
-	public List<ConceptSummary> listConcepts(final String categoryId, final Boolean all) throws ContextException {
+	public List<ConceptSummary> listConcepts(final String categoryId, final Boolean all) throws CommunityException {
 		final String contextId = StringUtils.substringBefore(categoryId, SEPARATOR);
 		final Stream<Concept> concepts = getContextMap().get(contextId)
 			.getCategories()
@@ -77,9 +82,9 @@ public class ContextApiCore {
 			.collect(Collectors.toList());
 	}
 
-	public List<ConceptSummary> listSubConcepts(final String conceptId, final Boolean all) throws ContextException {
+	public List<ConceptSummary> listSubConcepts(final String conceptId, final Boolean all) throws CommunityException {
 		final List<String> ids = Splitter.on(SEPARATOR).splitToList(conceptId);
-		if (ids.size() < 3) { throw new ContextException(""); }
+		if (ids.size() < 3) { throw new CommunityException(""); }
 
 		final String contextId = ids.get(0);
 		final String categoryId = contextId + SEPARATOR + ids.get(1);
@@ -105,15 +110,15 @@ public class ContextApiCore {
 			.collect(Collectors.toList());
 	}
 
-	private Map<String, Context> getContextMap() throws ContextException {
+	private Map<String, Context> getContextMap() throws CommunityException {
 		return getContextMap(Lists.newArrayList());
 	}
 
-	private Map<String, Context> getContextMap(final List<String> type) throws ContextException {
+	private Map<String, Context> getContextMap(final List<String> type) throws CommunityException {
 		try {
 			return isClient.getContextMap(type);
 		} catch (final IOException e) {
-			throw new ContextException(e);
+			throw new CommunityException(e);
 		}
 	}
 

apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/context/ContextMappingUtils.java

@@ -1,7 +1,6 @@
 package eu.dnetlib.openaire.context;
 
 import java.text.ParseException;
-import java.util.ArrayList;
 import java.util.Date;
 import java.util.List;
 import java.util.Map;
@@ -11,6 +10,8 @@ import java.util.stream.Collectors;
 import org.apache.commons.lang3.BooleanUtils;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.commons.lang3.time.DateUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
 import org.dom4j.Document;
 import org.dom4j.DocumentException;
 import org.dom4j.DocumentHelper;
@@ -26,25 +27,31 @@ import eu.dnetlib.openaire.exporter.model.context.Context;
 import eu.dnetlib.openaire.exporter.model.context.Param;
 import eu.dnetlib.openaire.exporter.model.funders.FunderDetails;
 
+@Deprecated
 public class ContextMappingUtils {
 
 	private static final List<String> DATE_PATTERN = Lists.newArrayList("yyyy-MM-dd'T'hh:mm:ss", "yyyy-MM-dd'T'hh:mm:ssXXX", "yyyy-MM-dd'T'hh:mm:ss+00:00");
 
+	private static final Log log = LogFactory.getLog(ContextMappingUtils.class);
+
 	public static Context parseContext(final String s, final Queue<Throwable> errors) {
 		try {
 			final Document doc = DocumentHelper.parseText(s);
 			final Element eContext = (Element) doc.selectSingleNode("/RESOURCE_PROFILE/BODY/CONFIGURATION/context");
 			final String creationDate = eContext.valueOf("./param[./@name='creationdate']/text()");
+			final String otherDate = doc.valueOf("/RESOURCE_PROFILE/HEADER/DATE_OF_CREATION/@value");
 			final Context c = new Context()
 				.setId(eContext.attributeValue("id"))
 				.setLabel(eContext.attributeValue("label"))
 				.setType(eContext.attributeValue("type"))
-				.setLastUpdateDate(asDate(doc.valueOf("/RESOURCE_PROFILE/HEADER/DATE_OF_CREATION/@value")))
+				.setLastUpdateDate(asDate(otherDate))
 				.setParams(parseParams(eContext))
 				.setCategories(parseCategories(eContext));
 			// the creation date will be added in the param elements of the community profile. Funders may not have it, hence the check.
 			if (StringUtils.isNotBlank(creationDate)) {
 				c.setCreationDate(asDate(creationDate));
+			} else {
+				c.setCreationDate(asDate(otherDate));
 			}
 			return c;
 		} catch (final DocumentException e) {
@@ -54,11 +61,15 @@ public class ContextMappingUtils {
 	}
 
 	private static Date asDate(final String s) {
+		if (StringUtils.isBlank(s)) { return null; }
+
 		for (final String pattern : DATE_PATTERN) {
 			try {
-				return DateUtils.parseDate(s, pattern);
+				final Date res = DateUtils.parseDate(s, pattern);
+				if (res != null) { return res; }
 			} catch (final ParseException e) {}
 		}
+		log.warn("Invalid Date: " + s);
 		return null;
 	}
 
@@ -93,18 +104,12 @@ public class ContextMappingUtils {
 		return BooleanUtils.toBooleanObject(StringUtils.isNotBlank(claim) ? claim : "false");
 	}
 
-	private static Map<String, List<Param>> parseParams(final Element e) {
+	private static List<Param> parseParams(final Element e) {
 		final List<Node> params = e.selectNodes("./param");
 		return params.stream()
 			.map(n -> (Element) n)
-			.map(p -> new Param()
-				.setName(p.attributeValue("name"))
-				.setValue(p.getTextTrim()))
-			.collect(Collectors.toMap(Param::getName, Lists::newArrayList, (p1, p2) -> {
-				final List<Param> p = new ArrayList<>(p1);
-				p.addAll(p2);
-				return p;
-			}));
+			.map(p -> new Param().setName(p.attributeValue("name")).setValue(p.getTextTrim()))
+			.collect(Collectors.toList());
 	}
 
 	public static FunderDetails asFunderDetails(final Context c) {

apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/DsmCore.java

@@ -36,7 +36,6 @@ import eu.dnetlib.enabling.datasources.common.DsmException;
 import eu.dnetlib.enabling.datasources.common.DsmForbiddenException;
 import eu.dnetlib.enabling.datasources.common.DsmNotFoundException;
 import eu.dnetlib.openaire.common.ISClient;
-import eu.dnetlib.openaire.community.CommunityClient;
 import eu.dnetlib.openaire.dsm.dao.CountryTermRepository;
 import eu.dnetlib.openaire.dsm.dao.DatasourceDao;
 import eu.dnetlib.openaire.dsm.dao.MongoLoggerClient;
@@ -91,9 +90,6 @@ public class DsmCore {
 	@Autowired
 	private JdbcTemplate jdbcTemplate;
 
-	@Autowired
-	private CommunityClient communityClient;
-
 	public List<Country> listCountries() throws DsmApiException {
 		final List<Country> countries = Lists.newArrayList();
 		final Vocabulary v = vocabularyClient.getCountries();
@@ -321,7 +317,6 @@ public class DsmCore {
 		mongoLoggerClient.dropCache();
 		isClient.dropCache();
 		vocabularyClient.dropCache();
-		communityClient.dropCache();
 	}
 
 	// HELPERS //////////////

apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/MongoLoggerClientImpl.java

@@ -48,11 +48,7 @@ import eu.dnetlib.openaire.exporter.exceptions.DsmApiException;
 import eu.dnetlib.openaire.exporter.model.dsm.AggregationInfo;
 import eu.dnetlib.openaire.exporter.model.dsm.AggregationInfoV1;
 import eu.dnetlib.openaire.exporter.model.dsm.AggregationStage;
-import eu.dnetlib.openaire.exporter.model.dsm.CollectionInfoV1;
-import eu.dnetlib.openaire.exporter.model.dsm.CollectionInfoV2;
 import eu.dnetlib.openaire.exporter.model.dsm.CollectionMode;
-import eu.dnetlib.openaire.exporter.model.dsm.TransformationInfoV1;
-import eu.dnetlib.openaire.exporter.model.dsm.TransformationInfoV2;
 import eu.dnetlib.openaire.info.JdbcInfoDao;
 
 /**
@@ -75,16 +71,16 @@ public class MongoLoggerClientImpl implements MongoLoggerClient {
 
 	private final static String LOADTIME = "loadtime";
 	private final LoadingCache<String, Instant> loadingCache =
-		CacheBuilder.newBuilder().maximumSize(1).expireAfterWrite(60, TimeUnit.MINUTES).build(new CacheLoader<String, Instant>() {
+			CacheBuilder.newBuilder().maximumSize(1).expireAfterWrite(60, TimeUnit.MINUTES).build(new CacheLoader<String, Instant>() {
 
-			// The only cached value is associated to "loadtime"
-			@Override
-			public Instant load(final String key) {
-				final Instant loadTime = getLoadTime();
-				log.debug("found load time: " + loadTime.toString());
-				return loadTime;
-			}
-		});
+				// The only cached value is associated to "loadtime"
+				@Override
+				public Instant load(final String key) {
+					final Instant loadTime = getLoadTime();
+					log.debug("found load time: " + loadTime.toString());
+					return loadTime;
+				}
+			});
 
 	private static final Bson fields = getFields();
 
@@ -94,30 +90,64 @@ public class MongoLoggerClientImpl implements MongoLoggerClient {
 	@Cacheable("dsm-aggregationhistory-cache-v1")
 	@Deprecated
 	public List<AggregationInfoV1> getAggregationHistoryV1(final String dsId) throws DsmApiException {
-		return getAggregationHistory(dsId, queryForAggregationHistoryV1(dsId, "(collect|transform)"), getMapperV1());
+		return getAggregationHistoryV1(dsId, queryForAggregationHistoryV1(dsId, "(collect|transform)"), getMapperV1());
 	}
 
 	@Override
 	@Cacheable("dsm-aggregationhistory-cache-v2")
 	public List<AggregationInfo> getAggregationHistoryV2(final String dsId) throws DsmApiException {
-		return getAggregationHistory(dsId, queryForAggregationHistoryV2(dsId, "(collect|transform)"), getMapperV2());
+		return getAggregationHistoryV2(dsId, queryForAggregationHistoryV2(dsId, "(collect|transform)"), getMapperV2());
 	}
 
-	private <T extends AggregationInfo> List<T> getAggregationHistory(final String dsId,
-		final Bson queryForAggregationHistory,
-		final Function<Document, T> mapper) throws DsmApiException {
+	@Deprecated
+	private List<AggregationInfoV1> getAggregationHistoryV1(final String dsId,
+			final Bson queryForAggregationHistory,
+			final Function<Document, AggregationInfoV1> mapper) throws DsmApiException {
 		log.warn(String.format("getAggregationHistory(dsId = %s): not using cache", dsId));
 		final Datasource conf = config.getDatasource();
 		try {
 			final FindIterable<Document> aggregationDocs = getCollection().find(queryForAggregationHistory)
-				.projection(fields)
-				.limit(conf.getMongoQueryLimit())
-				.sort(dbo("system:startHumanDate", -1));
+					.projection(fields)
+					.limit(conf.getMongoQueryLimit())
+					.sort(dbo("system:startHumanDate", -1));
 
-			final List<T> aggregationInfos = Utils.stream(aggregationDocs.iterator())
-				.map(mapper)
-				.filter(ai -> ai.getNumberOfRecords() >= 0 && StringUtils.isNotBlank(ai.getDate()))
-				.collect(Collectors.toList());
+			final List<AggregationInfoV1> aggregationInfos = Utils.stream(aggregationDocs.iterator())
+					.map(mapper)
+					.filter(ai -> ai.getNumberOfRecords() >= 0 && StringUtils.isNotBlank(ai.getDate()))
+					.collect(Collectors.toList());
+
+			final Instant loadTime = loadingCache.get(LOADTIME);
+
+			if (!Objects.equals(Instant.MIN, loadTime)) {
+				for (final AggregationInfoV1 a : aggregationInfos) {
+					if (asInstant(a).isBefore(loadTime) && AggregationStage.COLLECT.equals(a.getAggregationStage())) {
+						a.setIndexedVersion(true);
+						break;
+					}
+				}
+			}
+
+			return aggregationInfos;
+		} catch (final Throwable e) {
+			throw new DsmApiException(HttpStatus.SC_INTERNAL_SERVER_ERROR, String.format("error reading aggregation history for '%s'", dsId), e);
+		}
+	}
+
+	private List<AggregationInfo> getAggregationHistoryV2(final String dsId,
+			final Bson queryForAggregationHistory,
+			final Function<Document, AggregationInfo> mapper) throws DsmApiException {
+		log.warn(String.format("getAggregationHistory(dsId = %s): not using cache", dsId));
+		final Datasource conf = config.getDatasource();
+		try {
+			final FindIterable<Document> aggregationDocs = getCollection().find(queryForAggregationHistory)
+					.projection(fields)
+					.limit(conf.getMongoQueryLimit())
+					.sort(dbo("system:startHumanDate", -1));
+
+			final List<AggregationInfo> aggregationInfos = Utils.stream(aggregationDocs.iterator())
+					.map(mapper)
+					.filter(ai -> ai.getNumberOfRecords() >= 0 && StringUtils.isNotBlank(ai.getDate()))
+					.collect(Collectors.toList());
 
 			final Instant loadTime = loadingCache.get(LOADTIME);
 
@@ -146,9 +176,14 @@ public class MongoLoggerClientImpl implements MongoLoggerClient {
 		return Instant.parse(a.getDate() + "T00:00:00Z");
 	}
 
+	@Deprecated
+	private Instant asInstant(final AggregationInfoV1 a) {
+		return Instant.parse(a.getDate() + "T00:00:00Z");
+	}
+
 	@Override
 	@CacheEvict(cacheNames = {
-		"dsm-aggregationhistory-cache-v1", "dsm-aggregationhistory-cache-v2", "dsm-firstharvestdate-cache"
+			"dsm-aggregationhistory-cache-v1", "dsm-aggregationhistory-cache-v2", "dsm-firstharvestdate-cache"
 	}, allEntries = true)
 	@Scheduled(fixedDelayString = "${openaire.exporter.cache.ttl}")
 	public void dropCache() {
@@ -166,20 +201,18 @@ public class MongoLoggerClientImpl implements MongoLoggerClient {
 			switch (stage) {
 
 			case COLLECT:
-				final CollectionInfoV1 cInfo = new CollectionInfoV1();
+				final AggregationInfoV1 cInfo = new AggregationInfoV1();
 				cInfo.setAggregationStage(stage);
 				cInfo.setCollectionMode(getCollectionMode(d));
 				cInfo.setNumberOfRecords(success ? getNumberOfRecords(d) : 0);
 				cInfo.setDate(getDate(d));
-				cInfo.setCompletedSuccessfully(success);
 				info = cInfo;
 				break;
 			case TRANSFORM:
-				final TransformationInfoV1 tInfo = new TransformationInfoV1();
+				final AggregationInfoV1 tInfo = new AggregationInfoV1();
 				tInfo.setAggregationStage(stage);
 				tInfo.setNumberOfRecords(success ? getNumberOfRecords(d) : 0);
 				tInfo.setDate(getDate(d));
-				tInfo.setCompletedSuccessfully(success);
 				info = tInfo;
 				break;
 			}
@@ -197,7 +230,7 @@ public class MongoLoggerClientImpl implements MongoLoggerClient {
 			switch (stage) {
 
 			case COLLECT:
-				final CollectionInfoV2 cInfo = new CollectionInfoV2();
+				final AggregationInfo cInfo = new AggregationInfo();
 				cInfo.setAggregationStage(stage);
 				cInfo.setCollectionMode(getCollectionMode(d));
 				cInfo.setNumberOfRecords(success ? getNumberOfRecords(d) : 0);
@@ -206,7 +239,7 @@ public class MongoLoggerClientImpl implements MongoLoggerClient {
 				info = cInfo;
 				break;
 			case TRANSFORM:
-				final TransformationInfoV2 tInfo = new TransformationInfoV2();
+				final AggregationInfo tInfo = new AggregationInfo();
 				tInfo.setAggregationStage(stage);
 				tInfo.setNumberOfRecords(success ? getNumberOfRecords(d) : 0);
 				tInfo.setDate(getDate(d));
@@ -220,23 +253,19 @@ public class MongoLoggerClientImpl implements MongoLoggerClient {
 
 	private CollectionMode getCollectionMode(final Document d) {
 		return Optional.ofNullable(d.getString("system:node:SELECT_MODE:selection"))
-			.map(CollectionMode::valueOf)
-			.orElseGet(() -> Optional.ofNullable(d.getString("collectionMode"))
 				.map(CollectionMode::valueOf)
-				.orElse(null));
+				.orElseGet(() -> Optional.ofNullable(d.getString("collectionMode"))
+						.map(CollectionMode::valueOf)
+						.orElse(null));
 	}
 
 	private Integer getNumberOfRecords(final Document d) {
 		final String sinkSize = d.getString("mainlog:sinkSize");
 		final String total = d.getString("mainlog:total");
 
-		if (StringUtils.isNotBlank(sinkSize)) {
-			return Ints.tryParse(sinkSize);
-		} else if (StringUtils.isNotBlank(total)) {
-			return Ints.tryParse(total);
-		} else {
-			return -1;
-		}
+		if (StringUtils.isNotBlank(sinkSize)) { return Ints.tryParse(sinkSize); }
+		if (StringUtils.isNotBlank(total)) { return Ints.tryParse(total); }
+		return -1;
 	}
 
 	private String getDate(final Document d) {

apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/ConversionUtils.java

@@ -1,91 +0,0 @@
-package eu.dnetlib.openaire.funders;
-
-import java.io.StringReader;
-import java.util.stream.Collectors;
-
-import eu.dnetlib.openaire.exporter.model.funders.ExtendedFunderDetails;
-import eu.dnetlib.openaire.exporter.model.funders.FunderDetails;
-import eu.dnetlib.openaire.exporter.model.funders.FundingStream;
-import eu.dnetlib.openaire.funders.domain.db.FunderDbEntry;
-import eu.dnetlib.openaire.funders.domain.db.FundingPathDbEntry;
-import org.dom4j.Document;
-import org.dom4j.DocumentException;
-import org.dom4j.io.SAXReader;
-
-import static org.apache.commons.lang3.StringUtils.*;
-
-public class ConversionUtils {
-
-	public static final String SEPARATOR = "::";
-
-	public static FunderDetails asFunderDetails(final FunderDbEntry fdb) {
-		final FunderDetails f = new FunderDetails();
-
-		f.setId(fdb.getId());
-		f.setName(fdb.getName());
-		f.setShortname(fdb.getShortname());
-		f.setJurisdiction(fdb.getJurisdiction());
-		f.setRegistrationDate(fdb.getRegistrationdate());
-		f.setLastUpdateDate(fdb.getLastupdatedate());
-
-		return f;
-	}
-
-	public static ExtendedFunderDetails asExtendedFunderDetails(final FunderDbEntry fdb) {
-		final ExtendedFunderDetails f = new ExtendedFunderDetails(asFunderDetails(fdb));
-
-		if (fdb.getFundingpaths() != null) {
-			f.setFundingStreams(
-					fdb.getFundingpaths().stream()
-						.map(ConversionUtils::asFundingStream)
-						.collect(Collectors.toList()));
-		}
-
-		return f;
-	}
-
-	private static FundingStream asFundingStream(final FundingPathDbEntry pathDbEntry) {
-		final FundingStream f = new FundingStream();
-
-		try {
-			final Document xml = new SAXReader().read(new StringReader(pathDbEntry.getPath()));
-
-			for(int i=2;i>=0;i--) {
-				if (hasFundingLevel(i, xml)) {
-					f.setId(getId(i, xml));
-					f.setName(getName(i, xml));
-					break;
-				}
-			}
-
-			if (isBlank(f.getId()) && isNoneBlank(xml.valueOf("//funder/id/text()"))) {
-				f.setId(xml.valueOf("//funder/shortname/text()"));
-				f.setName(xml.valueOf("//funder/name/text()"));
-			}
-
-			if (isBlank(f.getId())) {
-				throw new IllegalStateException("invalid funding path:\n" + xml.asXML());
-			}
-
-			return f;
-
-		} catch (DocumentException e) {
-			throw new IllegalStateException("unable to parse funding path:\n" + pathDbEntry.getPath());
-		}
-	}
-
-	private static String getName(int level, final Document d) {
-		return d.valueOf(String.format("//funding_level_%s/description/text()", level));
-	}
-
-	private static String getId(int level, final Document d) {
-		return substringAfter(
-				d.valueOf(
-					String.format("//funding_level_%s/id/text()", level)
-				), SEPARATOR);
-	}
-
-	private static boolean hasFundingLevel(int level, Document d) {
-		return isNotBlank(getId(level, d));
-	}
-}

apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/FunderDao.java

@@ -1,43 +0,0 @@
-package eu.dnetlib.openaire.funders;
-
-import java.util.List;
-import java.util.stream.Collectors;
-
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
-import org.springframework.data.domain.PageRequest;
-import org.springframework.stereotype.Component;
-
-import eu.dnetlib.openaire.exporter.exceptions.FundersApiException;
-import eu.dnetlib.openaire.exporter.model.funders.ExtendedFunderDetails;
-import eu.dnetlib.openaire.exporter.model.funders.FunderDetails;
-
-@Component
-@ConditionalOnProperty(value = "openaire.exporter.enable.funders", havingValue = "true")
-public class FunderDao {
-
-	@Autowired
-	private FunderRepository funderRepository;
-
-	public ExtendedFunderDetails getExtendedFunderDetails(final String funderId) throws FundersApiException {
-		return ConversionUtils
-			.asExtendedFunderDetails(funderRepository.findById(funderId).orElseThrow(() -> new FundersApiException("Funder not found. ID:  " + funderId)));
-	}
-
-	public List<FunderDetails> listFunderDetails(final int page, final int size) throws FundersApiException {
-		return funderRepository.findAll(PageRequest.of(page, size))
-			.getContent()
-			.stream()
-			.map(ConversionUtils::asFunderDetails)
-			.collect(Collectors.toList());
-	}
-
-	public List<String> listFunderIds(final int page, final int size) throws FundersApiException {
-		return funderRepository.findAll(PageRequest.of(page, size))
-			.getContent()
-			.stream()
-			.map(f -> f.getId())
-			.collect(Collectors.toList());
-	}
-
-}

apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/FunderRepository.java

@@ -1,10 +1,11 @@
 package eu.dnetlib.openaire.funders;
 
-import eu.dnetlib.openaire.funders.domain.db.FunderDbEntry;
-import org.springframework.data.jpa.repository.JpaRepository;
 import org.springframework.stereotype.Repository;
 
+import eu.dnetlib.openaire.common.ReadOnlyRepository;
+import eu.dnetlib.openaire.funders.domain.db.FunderDbEntry;
+
 @Repository
-public interface FunderRepository extends JpaRepository<FunderDbEntry, String> {
+public interface FunderRepository extends ReadOnlyRepository<FunderDbEntry, String> {
 
 }

apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/FunderService.java

@@ -0,0 +1,183 @@
+package eu.dnetlib.openaire.funders;
+
+import java.io.File;
+import java.io.FileWriter;
+import java.io.FilenameFilter;
+import java.time.LocalDate;
+import java.time.format.DateTimeFormatter;
+import java.util.Arrays;
+import java.util.Comparator;
+import java.util.List;
+import java.util.Objects;
+import java.util.stream.Collectors;
+
+import javax.annotation.PostConstruct;
+
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
+import org.springframework.scheduling.annotation.Scheduled;
+import org.springframework.stereotype.Component;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.SerializationFeature;
+import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
+
+import eu.dnetlib.openaire.dsm.dao.MongoLoggerClient;
+import eu.dnetlib.openaire.exporter.exceptions.DsmApiException;
+import eu.dnetlib.openaire.exporter.model.dsm.AggregationInfo;
+import eu.dnetlib.openaire.exporter.model.dsm.AggregationStage;
+import eu.dnetlib.openaire.funders.domain.db.FunderDatasource;
+import eu.dnetlib.openaire.funders.domain.db.FunderDbEntry;
+import eu.dnetlib.openaire.funders.domain.db.FunderPid;
+
+@Component
+@ConditionalOnProperty(value = "openaire.exporter.enable.funders", havingValue = "true")
+public class FunderService {
+
+	private static final String TEMP_FILE_SUFFIX = ".funds.tmp";
+	private static final String SEPARATOR = "@=@";
+
+	@Autowired
+	private FunderRepository funderRepository;
+
+	@Autowired
+	private MongoLoggerClient mongoLoggerClient;
+
+	private File tempDir;
+
+	private File tempFile;
+
+	private final DateTimeFormatter DATEFORMATTER = DateTimeFormatter.ofPattern("yyyy-MM-dd");
+
+	private static final Log log = LogFactory.getLog(FunderService.class);
+
+	@PostConstruct
+	public void init() {
+
+		tempDir = new File(System.getProperty("java.io.tmpdir", "/tmp"));
+
+		for (final File f : tempDir.listFiles((FilenameFilter) (dir, name) -> name.endsWith(TEMP_FILE_SUFFIX))) {
+			deleteFile(f);
+		}
+
+		new Thread(this::updateFunders).start();
+	}
+
+	private void deleteFile(final File f) {
+		if (f != null && f.exists()) {
+			log.info("Deleting file: " + f.getAbsolutePath());
+			f.delete();
+		}
+	}
+
+	@Scheduled(cron = "${openaire.exporter.funders.cron}")
+	public void updateFunders() {
+		try {
+			final ObjectMapper mapper = new ObjectMapper();
+			mapper.registerModule(new JavaTimeModule());
+			mapper.configure(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS, false);
+
+			final File tmp = File.createTempFile("funders-api-", TEMP_FILE_SUFFIX, tempDir);
+
+			log.info("Generating funders file: " + tmp.getAbsolutePath());
+
+			try (final FileWriter writer = new FileWriter(tmp)) {
+				writer.write("[");
+				boolean first = true;
+				for (final FunderDbEntry funder : funderRepository.findAll()) {
+					log.info("  - adding: " + funder.getId());
+
+					// THIS PATCH IS NECESSARY FOR COMPATIBILITY WITH POSTGRES 9.3 (PARTIAL SUPPORT OF THE JSON LIBRARY)
+
+					final List<FunderDatasource> datasources = Arrays.stream(funder.getDatasourcesPostgres())
+							.filter(Objects::nonNull)
+							.map(s -> s.split(SEPARATOR))
+							.filter(arr -> arr.length == 3)
+							.map(arr -> {
+								final FunderDatasource ds = new FunderDatasource();
+								ds.setId(arr[0].trim());
+								ds.setName(arr[1].trim());
+								ds.setType(arr[2].trim());
+								return ds;
+							})
+							.filter(ds -> StringUtils.isNotBlank(ds.getId()))
+							.collect(Collectors.toList());
+
+					funder.setDatasources(datasources);
+
+					final List<FunderPid> pids = Arrays.stream(funder.getPidsPostgres())
+							.filter(Objects::nonNull)
+							.map(s -> s.split(SEPARATOR))
+							.filter(arr -> arr.length == 2)
+							.map(arr -> {
+								final FunderPid pid = new FunderPid();
+								pid.setType(arr[0].trim());
+								pid.setValue(arr[1].trim());
+								return pid;
+							})
+							.filter(pid -> StringUtils.isNotBlank(pid.getValue()))
+							.collect(Collectors.toList());
+
+					funder.setPids(pids);
+
+					// END PATCH
+
+					addAggregationHistory(funder);
+
+					if (first) {
+						first = false;
+					} else {
+						writer.write(",");
+					}
+					writer.write(mapper.writeValueAsString(funder));
+				}
+				writer.write("]");
+				log.info("Publish funders file: " + tmp.getAbsolutePath());
+
+				deleteFile(tempFile);
+				setTempFile(tmp);
+			}
+		} catch (final Throwable e) {
+			log.error("Error generating funders file", e);
+			throw new RuntimeException("Error generating funders file", e);
+		}
+	}
+
+	private void addAggregationHistory(final FunderDbEntry funder) {
+
+		final List<LocalDate> dates = funder.getDatasources()
+				.stream()
+				.map(FunderDatasource::getId)
+				.map(id -> {
+					try {
+						return mongoLoggerClient.getAggregationHistoryV2(id);
+					} catch (final DsmApiException e) {
+						log.error("Error retrieving the aggregation history", e);
+						throw new RuntimeException("Error retrieving the aggregation history", e);
+					}
+				})
+				.flatMap(List::stream)
+				.filter(AggregationInfo::isCompletedSuccessfully)
+				.filter(info -> info.getAggregationStage() == AggregationStage.TRANSFORM)
+				.map(AggregationInfo::getDate)
+				.distinct()
+				.map(s -> LocalDate.parse(s, DATEFORMATTER))
+				.sorted(Comparator.reverseOrder())
+				.limit(10)
+				.collect(Collectors.toList());
+
+		funder.setAggregationDates(dates);
+	}
+
+	public File getTempFile() {
+		return tempFile;
+	}
+
+	public void setTempFile(final File tempFile) {
+		this.tempFile = tempFile;
+	}
+
+}

apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/FundersApiController.java

@@ -1,21 +1,25 @@
 package eu.dnetlib.openaire.funders;
 
-import java.util.List;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.InputStream;
+import java.io.OutputStream;
 
+import javax.servlet.http.HttpServletResponse;
+
+import org.apache.commons.io.IOUtils;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
+import org.springframework.http.MediaType;
 import org.springframework.web.bind.annotation.CrossOrigin;
-import org.springframework.web.bind.annotation.PathVariable;
 import org.springframework.web.bind.annotation.RequestMapping;
 import org.springframework.web.bind.annotation.RequestMethod;
 import org.springframework.web.bind.annotation.RestController;
 
 import eu.dnetlib.openaire.common.AbstractExporterController;
 import eu.dnetlib.openaire.exporter.exceptions.FundersApiException;
-import eu.dnetlib.openaire.exporter.model.funders.ExtendedFunderDetails;
-import eu.dnetlib.openaire.exporter.model.funders.FunderDetails;
 import io.swagger.v3.oas.annotations.Operation;
 import io.swagger.v3.oas.annotations.responses.ApiResponse;
 import io.swagger.v3.oas.annotations.responses.ApiResponses;
@@ -23,59 +27,48 @@ import io.swagger.v3.oas.annotations.tags.Tag;
 
 @RestController
 @CrossOrigin(origins = {
-	"*"
+		"*"
 })
 @ConditionalOnProperty(value = "openaire.exporter.enable.funders", havingValue = "true")
 @Tag(name = "OpenAIRE funders API", description = "the OpenAIRE funders API")
 public class FundersApiController extends AbstractExporterController {
 
+	@Autowired
+	private FunderService service;
+
 	private static final Log log = LogFactory.getLog(FundersApiController.class);
 
-	@Autowired
-	private FunderDao fDao;
-
 	@RequestMapping(value = "/funders", produces = {
-		"application/json"
+			"application/json"
 	}, method = RequestMethod.GET)
-	@Operation(summary = "get basic information about funders", description = "basic information about funders: id, name, shortname, last update date, registration date")
+	@Operation(summary = "get all funders", description = "get all funders")
 	@ApiResponses(value = {
-		@ApiResponse(responseCode = "200", description = "OK"),
-		@ApiResponse(responseCode = "500", description = "unexpected error")
+			@ApiResponse(responseCode = "200", description = "OK"),
+			@ApiResponse(responseCode = "500", description = "unexpected error")
 	})
-	public List<FunderDetails> getFunders(
-		@PathVariable final int page,
-		@PathVariable final int size) throws FundersApiException {
+	public void getFunders(final HttpServletResponse res) throws FundersApiException {
 
-		return fDao.listFunderDetails(page, size);
-	}
+		res.setContentType(MediaType.APPLICATION_JSON_VALUE);
 
-	@RequestMapping(value = "/funder/{id}", produces = {
-		"application/json"
-	}, method = RequestMethod.GET)
-	@Operation(summary = "get the funder details", description = "complete funder information")
-	@ApiResponses(value = {
-		@ApiResponse(responseCode = "200", description = "OK"),
-		@ApiResponse(responseCode = "500", description = "unexpected error")
-	})
-	public ExtendedFunderDetails getFunderDetails(
-		@PathVariable final String id) throws FundersApiException {
+		final File file = service.getTempFile();
 
-		return fDao.getExtendedFunderDetails(id);
-	}
+		if (file == null) {
+			log.error("Missing temp file (NULL)");
+			throw new FundersApiException("Missing temp file (NULL)");
+		}
 
-	@RequestMapping(value = "/funder/ids", produces = {
-		"application/json"
-	}, method = RequestMethod.GET)
-	@Operation(summary = "get the list of funder ids", description = "get the list of funder ids")
-	@ApiResponses(value = {
-		@ApiResponse(responseCode = "200", description = "OK"),
-		@ApiResponse(responseCode = "500", description = "unexpected error")
-	})
-	public List<String> getFunderIds(
-		@PathVariable final int page,
-		@PathVariable final int size) throws FundersApiException {
+		if (!file.exists()) {
+			log.error("Missing temp file " + service.getTempFile());
+			throw new FundersApiException("Missing temp file " + service.getTempFile());
+		}
 
-		return fDao.listFunderIds(page, size);
+		try (final InputStream in = new FileInputStream(file); OutputStream out = res.getOutputStream()) {
+			IOUtils.copy(in, out);
+			return;
+		} catch (final Exception e) {
+			log.error("Error reading file " + service.getTempFile(), e);
+			throw new FundersApiException("Error reading file " + service.getTempFile(), e);
+		}
 	}
 
 }

apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/domain/db/FunderDatasource.java

@@ -0,0 +1,39 @@
+package eu.dnetlib.openaire.funders.domain.db;
+
+import java.io.Serializable;
+
+public class FunderDatasource implements Serializable {
+
+	private static final long serialVersionUID = 2145493560459874509L;
+
+	private String id;
+
+	private String name;
+
+	private String type;
+
+	public String getId() {
+		return id;
+	}
+
+	public void setId(final String id) {
+		this.id = id;
+	}
+
+	public String getName() {
+		return name;
+	}
+
+	public void setName(final String name) {
+		this.name = name;
+	}
+
+	public String getType() {
+		return type;
+	}
+
+	public void setType(final String type) {
+		this.type = type;
+	}
+
+}

apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/domain/db/FunderDbEntry.java

@@ -1,38 +1,75 @@
 package eu.dnetlib.openaire.funders.domain.db;
 
-import java.sql.Date;
-import java.util.Set;
-import javax.persistence.*;
+import java.io.Serializable;
+import java.time.LocalDate;
+import java.util.ArrayList;
+import java.util.List;
 
-import eu.dnetlib.openaire.dsm.domain.db.IdentityDbEntry;
+import javax.persistence.Column;
+import javax.persistence.Entity;
+import javax.persistence.Id;
+import javax.persistence.Table;
+import javax.persistence.Transient;
+
+import org.hibernate.annotations.Type;
+import org.hibernate.annotations.TypeDef;
+import org.hibernate.annotations.TypeDefs;
+
+import com.fasterxml.jackson.annotation.JsonIgnore;
+import com.vladmihalcea.hibernate.type.array.StringArrayType;
 
 @Entity
-@Table(name = "funders")
-public class FunderDbEntry {
+@Table(name = "funders_view")
+@TypeDefs({
+		@TypeDef(name = "string-array", typeClass = StringArrayType.class)
+})
+public class FunderDbEntry implements Serializable {
+
+	private static final long serialVersionUID = 1290088460508203016L;
 
 	@Id
+	@Column(name = "id")
 	private String id;
-	private String name;
-	private String shortname;
-	private String jurisdiction;
-	private String websiteurl;
-	private String policy;
-	private Date registrationdate;
-	private Date lastupdatedate;
 
-	@OneToMany(
-			cascade = { CascadeType.PERSIST, CascadeType.MERGE },
-			fetch = FetchType.LAZY )
-	@JoinTable(
-			name = "funder_identity",
-			joinColumns = @JoinColumn(name = "funder"),
-			inverseJoinColumns = @JoinColumn(name = "pid"))
-	private Set<IdentityDbEntry> pids;
+	@Column(name = "legalshortname")
+	private String legalShortName;
 
-	@OneToMany(mappedBy = "funderid", cascade = CascadeType.ALL, fetch = FetchType.LAZY)
-	private Set<FundingPathDbEntry> fundingpaths;
+	@Column(name = "legalname")
+	private String legalName;
 
-	public FunderDbEntry() {}
+	@Column(name = "websiteurl")
+	private String websiteUrl;
+
+	@Column(name = "logourl")
+	private String logoUrl;
+
+	@Column(name = "country")
+	private String country;
+
+	@Column(name = "registrationdate")
+	private LocalDate registrationDate;
+
+	@Column(name = "registered")
+	private Boolean registered;
+
+	@JsonIgnore
+	@Type(type = "string-array")
+	@Column(name = "pids", columnDefinition = "text[]")
+	private String[] pidsPostgres;
+
+	@Transient
+	private List<FunderPid> pids = new ArrayList<FunderPid>();
+
+	@JsonIgnore
+	@Type(type = "string-array")
+	@Column(name = "datasources", columnDefinition = "text[]")
+	private String[] datasourcesPostgres;
+
+	@Transient
+	private List<FunderDatasource> datasources = new ArrayList<FunderDatasource>();
+
+	@Transient
+	private List<LocalDate> aggregationDates;
 
 	public String getId() {
 		return id;
@@ -42,75 +79,100 @@ public class FunderDbEntry {
 		this.id = id;
 	}
 
-	public String getName() {
-		return name;
+	public String getLegalShortName() {
+		return legalShortName;
 	}
 
-	public void setName(final String name) {
-		this.name = name;
+	public void setLegalShortName(final String legalShortName) {
+		this.legalShortName = legalShortName;
 	}
 
-	public String getShortname() {
-		return shortname;
+	public String getLegalName() {
+		return legalName;
 	}
 
-	public void setShortname(final String shortname) {
-		this.shortname = shortname;
+	public void setLegalName(final String legalName) {
+		this.legalName = legalName;
 	}
 
-	public String getJurisdiction() {
-		return jurisdiction;
+	public String getWebsiteUrl() {
+		return websiteUrl;
 	}
 
-	public void setJurisdiction(final String jurisdiction) {
-		this.jurisdiction = jurisdiction;
+	public void setWebsiteUrl(final String websiteUrl) {
+		this.websiteUrl = websiteUrl;
 	}
 
-	public String getWebsiteurl() {
-		return websiteurl;
+	public String getLogoUrl() {
+		return logoUrl;
 	}
 
-	public void setWebsiteurl(final String websiteurl) {
-		this.websiteurl = websiteurl;
+	public void setLogoUrl(final String logoUrl) {
+		this.logoUrl = logoUrl;
 	}
 
-	public String getPolicy() {
-		return policy;
+	public String getCountry() {
+		return country;
 	}
 
-	public void setPolicy(final String policy) {
-		this.policy = policy;
+	public void setCountry(final String country) {
+		this.country = country;
 	}
 
-	public Date getRegistrationdate() {
-		return registrationdate;
+	public LocalDate getRegistrationDate() {
+		return registrationDate;
 	}
 
-	public void setRegistrationdate(final Date registrationdate) {
-		this.registrationdate = registrationdate;
+	public void setRegistrationDate(final LocalDate registrationDate) {
+		this.registrationDate = registrationDate;
 	}
 
-	public Date getLastupdatedate() {
-		return lastupdatedate;
+	public Boolean getRegistered() {
+		return registered;
 	}
 
-	public void setLastupdatedate(final Date lastupdatedate) {
-		this.lastupdatedate = lastupdatedate;
+	public void setRegistered(final Boolean registered) {
+		this.registered = registered;
 	}
 
-	public Set<IdentityDbEntry> getPids() {
+	public String[] getPidsPostgres() {
+		return pidsPostgres;
+	}
+
+	public void setPidsPostgres(final String[] pidsPostgres) {
+		this.pidsPostgres = pidsPostgres;
+	}
+
+	public List<FunderPid> getPids() {
 		return pids;
 	}
 
-	public void setPids(final Set<IdentityDbEntry> pids) {
+	public void setPids(final List<FunderPid> pids) {
 		this.pids = pids;
 	}
 
-	public Set<FundingPathDbEntry> getFundingpaths() {
-		return fundingpaths;
+	public String[] getDatasourcesPostgres() {
+		return datasourcesPostgres;
 	}
 
-	public void setFundingpaths(final Set<FundingPathDbEntry> fundingpaths) {
-		this.fundingpaths = fundingpaths;
+	public void setDatasourcesPostgres(final String[] datasourcesPostgres) {
+		this.datasourcesPostgres = datasourcesPostgres;
 	}
+
+	public List<FunderDatasource> getDatasources() {
+		return datasources;
+	}
+
+	public void setDatasources(final List<FunderDatasource> datasources) {
+		this.datasources = datasources;
+	}
+
+	public List<LocalDate> getAggregationDates() {
+		return aggregationDates;
+	}
+
+	public void setAggregationDates(final List<LocalDate> aggregationDates) {
+		this.aggregationDates = aggregationDates;
+	}
+
 }

apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/domain/db/FunderPid.java

@@ -0,0 +1,29 @@
+package eu.dnetlib.openaire.funders.domain.db;
+
+import java.io.Serializable;
+
+public class FunderPid implements Serializable {
+
+	private static final long serialVersionUID = 2145493560459874509L;
+
+	private String type;
+
+	private String value;
+
+	public String getType() {
+		return type;
+	}
+
+	public void setType(final String type) {
+		this.type = type;
+	}
+
+	public String getValue() {
+		return value;
+	}
+
+	public void setValue(final String value) {
+		this.value = value;
+	}
+
+}

apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/domain/db/OldFunderDbEntry.java

@@ -0,0 +1,118 @@
+package eu.dnetlib.openaire.funders.domain.db;
+
+import java.sql.Date;
+import java.util.Set;
+
+import javax.persistence.CascadeType;
+import javax.persistence.FetchType;
+import javax.persistence.Id;
+import javax.persistence.JoinColumn;
+import javax.persistence.JoinTable;
+import javax.persistence.OneToMany;
+
+import eu.dnetlib.openaire.dsm.domain.db.IdentityDbEntry;
+
+// @Entity
+// @Table(name = "funders")
+@Deprecated
+public class OldFunderDbEntry {
+
+	@Id
+	private String id;
+	private String name;
+	private String shortname;
+	private String jurisdiction;
+	private String websiteurl;
+	private String policy;
+	private Date registrationdate;
+	private Date lastupdatedate;
+
+	@OneToMany(cascade = { CascadeType.PERSIST, CascadeType.MERGE }, fetch = FetchType.LAZY)
+	@JoinTable(name = "funder_identity", joinColumns = @JoinColumn(name = "funder"), inverseJoinColumns = @JoinColumn(name = "pid"))
+	private Set<IdentityDbEntry> pids;
+
+	@OneToMany(mappedBy = "funderid", cascade = CascadeType.ALL, fetch = FetchType.LAZY)
+	private Set<OldFundingPathDbEntry> fundingpaths;
+
+	public OldFunderDbEntry() {}
+
+	public String getId() {
+		return id;
+	}
+
+	public void setId(final String id) {
+		this.id = id;
+	}
+
+	public String getName() {
+		return name;
+	}
+
+	public void setName(final String name) {
+		this.name = name;
+	}
+
+	public String getShortname() {
+		return shortname;
+	}
+
+	public void setShortname(final String shortname) {
+		this.shortname = shortname;
+	}
+
+	public String getJurisdiction() {
+		return jurisdiction;
+	}
+
+	public void setJurisdiction(final String jurisdiction) {
+		this.jurisdiction = jurisdiction;
+	}
+
+	public String getWebsiteurl() {
+		return websiteurl;
+	}
+
+	public void setWebsiteurl(final String websiteurl) {
+		this.websiteurl = websiteurl;
+	}
+
+	public String getPolicy() {
+		return policy;
+	}
+
+	public void setPolicy(final String policy) {
+		this.policy = policy;
+	}
+
+	public Date getRegistrationdate() {
+		return registrationdate;
+	}
+
+	public void setRegistrationdate(final Date registrationdate) {
+		this.registrationdate = registrationdate;
+	}
+
+	public Date getLastupdatedate() {
+		return lastupdatedate;
+	}
+
+	public void setLastupdatedate(final Date lastupdatedate) {
+		this.lastupdatedate = lastupdatedate;
+	}
+
+	public Set<IdentityDbEntry> getPids() {
+		return pids;
+	}
+
+	public void setPids(final Set<IdentityDbEntry> pids) {
+		this.pids = pids;
+	}
+
+	public Set<OldFundingPathDbEntry> getFundingpaths() {
+		return fundingpaths;
+	}
+
+	public void setFundingpaths(final Set<OldFundingPathDbEntry> fundingpaths) {
+		this.fundingpaths = fundingpaths;
+	}
+}

apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/domain/db/OldFundingPathDbEntry.java

@@ -1,12 +1,15 @@
 package eu.dnetlib.openaire.funders.domain.db;
 
-import javax.persistence.*;
+import javax.persistence.Column;
+import javax.persistence.Id;
+import javax.persistence.Transient;
 
 import com.fasterxml.jackson.annotation.JsonIgnore;
 
-@Entity
-@Table(name = "fundingpaths")
-public class FundingPathDbEntry {
+// @Entity
+// @Table(name = "fundingpaths")
+@Deprecated
+public class OldFundingPathDbEntry {
 
 	@JsonIgnore
 	@Transient
@@ -28,7 +31,7 @@ public class FundingPathDbEntry {
 
 	private String funderid;
 
-	public FundingPathDbEntry() {}
+	public OldFundingPathDbEntry() {}
 
 	public String getDnetresourceidentifier() {
 		return dnetresourceidentifier;

apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/info/JdbcInfoDao.java

@@ -8,6 +8,7 @@ public interface JdbcInfoDao {
 		claim_load_date,
 		oaf_load_date,
 		odf_load_date,
+		db_load_date,
 		inference_date,
 		stats_update_date,
 		crossref_update_date,

apps/dnet-exporter-api/src/main/resources/application.properties

@@ -25,8 +25,9 @@ management.endpoints.web.path-mapping.health        =   health
 # ENABLE / DISABLE CONTROLLERS
 openaire.exporter.enable.dsm            =   true
 openaire.exporter.enable.community      =   true
+openaire.exporter.enable.community.import =   false
 openaire.exporter.enable.context        =   true
-openaire.exporter.enable.funders        =   false
+openaire.exporter.enable.funders        =   true
 openaire.exporter.enable.project        =   true
 openaire.exporter.enable.info           =   true
 
@@ -35,3 +36,4 @@ openaire.exporter.cache.ttl             =   43200000
 
 maven.pom.path = /META-INF/maven/eu.dnetlib.dhp/dnet-exporter-api/effective-pom.xml
 
+openaire.exporter.funders.cron = 0 30 2 * * *

apps/dnet-exporter-api/src/main/resources/global.properties

@@ -1,12 +1,8 @@
-#services.is.host                    =   localhost
-services.is.host                    =   dev-openaire.d4science.org
-#services.is.port                    =   8280
-services.is.port                    =   443
-#services.is.protocol                =   http
-services.is.protocol                =   https
-#services.is.context                 =   app
-services.is.context                 =   is
-#services.is.baseurl                 =   ${services.is.protocol}://${services.is.host}:${services.is.port}/${services.is.context}/services
+services.is.protocol                =   http
+services.is.host                    =   localhost
+services.is.port                    =   8280
+services.is.context                 =   app
+
 services.is.baseurl                 =   ${services.is.protocol}://${services.is.host}:${services.is.port}/${services.is.context}/services
 
 openaire.exporter.isLookupUrl           =   ${services.is.baseurl}/isLookUp
@@ -20,8 +16,8 @@ openaire.exporter.cxfClientConnectTimeout    =   60000
 openaire.exporter.cxfClientReceiveTimeout    =   120000
 
 # JDBC
-#openaire.exporter.jdbc.url          =   jdbc:postgresql://localhost:5432/dnet_openaire
-openaire.exporter.jdbc.url          =   jdbc:postgresql://localhost:5432/dev_openaire_8280
+openaire.exporter.jdbc.url          =   jdbc:postgresql://localhost:5432/dnet_openaireplus
+#openaire.exporter.jdbc.url          =   jdbc:postgresql://localhost:5432/dev_openaire_8280
 openaire.exporter.jdbc.user         =   dnetapi
 openaire.exporter.jdbc.pwd          =   dnetPwd
 openaire.exporter.jdbc.minIdle      =   1
@@ -51,10 +47,9 @@ openaire.exporter.findObjectStore           =   /eu/dnetlib/openaire/xquery/find
 openaire.exporter.findFunderContexts        =   /eu/dnetlib/openaire/xquery/findFunderContexts.xquery
 openaire.exporter.findCommunityContexts     =   /eu/dnetlib/openaire/xquery/findCommunityContexts.xquery
 openaire.exporter.findContextProfiles       =   /eu/dnetlib/openaire/xquery/findContextProfiles.xquery
+openaire.exporter.findContextProfilesByType =   /eu/dnetlib/openaire/xquery/findContextProfilesByType.xquery
 openaire.exporter.getRepoProfile            =   /eu/dnetlib/openaire/xquery/getRepoProfile.xquery
 
-openaire.exporter.contentLoadQuery          =   { "$and" : [ { "system:profileName" : "Graph construction [PROD]" }, { "system:isCompletedSuccessfully" : "true" }, { "reuseContent" : "false" } ] }
-
 # REST API CONFIGURATION
 openaire.exporter.swaggerDsm.apiTitle          =   OpenAIRE aggregator REST API
 openaire.exporter.swaggerDsm.apiDescription    =   The OpenAIRE data provision REST API allows developers to access the metadata information space of OpenAIRE programmatically.
@@ -107,4 +102,4 @@ openaire.exporter.swaggerInfo.apiContactEmail   =   ${openaire.exporter.swaggerD
 # VOCABULARIES
 openaire.exporter.vocabularies.baseUrl                      = http://localhost:8980/provision/mvc/vocabularies
 openaire.exporter.vocabularies.countriesEndpoint            = ${openaire.exporter.vocabularies.baseUrl}/dnet:countries.json
-openaire.exporter.vocabularies.datasourceTypologiesEndpoint = ${openaire.exporter.vocabularies.baseUrl}/dnet:datasource_typologies.json
+openaire.exporter.vocabularies.datasourceTypologiesEndpoint = ${openaire.exporter.vocabularies.baseUrl}/dnet:eosc_datasource_types.json

apps/dnet-exporter-api/src/main/resources/global.properties.locale

@@ -1,7 +1,12 @@
-services.is.host                    =   localhost
-services.is.port                    =   8280
-services.is.protocol                =   http
-services.is.context                 =   app
+#services.is.host                    =   localhost
+services.is.host                    =   dev-openaire.d4science.org
+#services.is.port                    =   8280
+services.is.port                    =   443
+#services.is.protocol                =   http
+services.is.protocol                =   https
+#services.is.context                 =   app
+services.is.context                 =   is
+#services.is.baseurl                 =   ${services.is.protocol}://${services.is.host}:${services.is.port}/${services.is.context}/services
 services.is.baseurl                 =   ${services.is.protocol}://${services.is.host}:${services.is.port}/${services.is.context}/services
 
 openaire.exporter.isLookupUrl           =   ${services.is.baseurl}/isLookUp
@@ -15,7 +20,7 @@ openaire.exporter.cxfClientConnectTimeout    =   60000
 openaire.exporter.cxfClientReceiveTimeout    =   120000
 
 # JDBC
-#openaire.exporter.jdbc.url          =   jdbc:postgresql://localhost:5432/dnet_openaireplus
+#openaire.exporter.jdbc.url          =   jdbc:postgresql://localhost:5432/dnet_openaire
 openaire.exporter.jdbc.url          =   jdbc:postgresql://localhost:5432/dev_openaire_8280
 openaire.exporter.jdbc.user         =   dnetapi
 openaire.exporter.jdbc.pwd          =   dnetPwd
@@ -49,6 +54,8 @@ openaire.exporter.findContextProfiles       =   /eu/dnetlib/openaire/xquery/find
 openaire.exporter.findContextProfilesByType =   /eu/dnetlib/openaire/xquery/findContextProfilesByType.xquery
 openaire.exporter.getRepoProfile            =   /eu/dnetlib/openaire/xquery/getRepoProfile.xquery
 
+openaire.exporter.contentLoadQuery          =   { "$and" : [ { "system:profileName" : "Graph construction [PROD]" }, { "system:isCompletedSuccessfully" : "true" }, { "reuseContent" : "false" } ] }
+
 # REST API CONFIGURATION
 openaire.exporter.swaggerDsm.apiTitle          =   OpenAIRE aggregator REST API
 openaire.exporter.swaggerDsm.apiDescription    =   The OpenAIRE data provision REST API allows developers to access the metadata information space of OpenAIRE programmatically.
@@ -101,4 +108,4 @@ openaire.exporter.swaggerInfo.apiContactEmail   =   ${openaire.exporter.swaggerD
 # VOCABULARIES
 openaire.exporter.vocabularies.baseUrl                      = http://localhost:8980/provision/mvc/vocabularies
 openaire.exporter.vocabularies.countriesEndpoint            = ${openaire.exporter.vocabularies.baseUrl}/dnet:countries.json
-openaire.exporter.vocabularies.datasourceTypologiesEndpoint = ${openaire.exporter.vocabularies.baseUrl}/dnet:eosc_datasource_types.json
+openaire.exporter.vocabularies.datasourceTypologiesEndpoint = ${openaire.exporter.vocabularies.baseUrl}/dnet:datasource_typologies.json

apps/dnet-exporter-api/src/main/resources/sql/community-schema.sql

@@ -0,0 +1,87 @@
+DROP TABLE IF EXISTS community_subs;
+DROP TABLE IF EXISTS community_projects;
+DROP TABLE IF EXISTS community_datasources;
+DROP TABLE IF EXISTS community_support_orgs;
+DROP TABLE IF EXISTS community_orgs;
+DROP TABLE IF EXISTS communities;
+
+CREATE TABLE communities (
+	id                       text PRIMARY KEY,
+	name                     text NOT NULL,
+	shortname                text NOT NULL, -- in the profile is label
+	displayname              text,
+	displayshortname         text,
+	description              text NOT NULL DEFAULT '',
+	status                   text NOT NULL DEFAULT 'hidden', -- all, manager, hidden, members
+	membership               text NOT NULL DEFAULT 'by-invitation', -- open, by-invitation
+	type                     text NOT NULL, -- community, ri
+	claim                    text, -- managers-only, members-only, all
+	subjects                 text[],
+	fos                      text[],
+	sdg                      text[],
+	adv_constraints          jsonb,
+	remove_constraints       jsonb,
+	main_zenodo_community    text,
+	other_zenodo_communities text[],
+	creation_date            timestamp NOT NULL DEFAULT now(),
+	last_update              timestamp NOT NULL DEFAULT now(),
+	logo_url                 text,
+	suggested_acknowledgements text[],
+	plan                     text NOT NULL DEFAULT 'Default'
+);
+
+CREATE TABLE community_projects (
+	community       text NOT NULL REFERENCES communities(id),
+	project_id      text NOT NULL,
+	project_code    text NOT NULL,
+	project_name    text NOT NULL,
+	project_acronym text,
+	project_funder  text NOT NULL,
+	available_since date NOT NULL default now(),
+	PRIMARY KEY (community, project_id)
+);
+
+CREATE TABLE community_datasources (
+	community       text NOT NULL REFERENCES communities(id),
+	ds_id           text NOT NULL,
+	ds_name         text NOT NULL,
+	ds_officialname text NOT NULL,
+	enabled         boolean NOT NULL DEFAULT true
+	constraints     jsonb,
+	deposit         boolean NOT NULL DEFAULT false,
+	message         text,
+	PRIMARY KEY (community, ds_id)
+);
+
+CREATE TABLE community_support_orgs (
+	community     text NOT NULL REFERENCES communities(id),
+	org_name      text NOT NULL,
+	org_url       text NOT NULL,
+	org_logourl   text NOT NULL,
+	PRIMARY KEY (community, org_name)
+);
+
+CREATE TABLE community_orgs (
+	community   text NOT NULL REFERENCES communities(id),
+	org_id      text NOT NULL,
+	PRIMARY KEY (community, org_id)
+);
+
+
+CREATE TABLE community_subs (
+	sub_id      text    NOT NULL PRIMARY KEY,
+	community   text    NOT NULL REFERENCES communities(id),
+	label       text    NOT NULL,
+	category    text    NOT NULL,
+	claim       boolean NOT NULL DEFAULT false,
+	browsable   boolean NOT NULL DEFAULT false,
+	params      jsonb,
+	parent      text REFERENCES community_subs(sub_id) -- NULL for the first level
+);
+
+CREATE INDEX community_projects_community ON community_projects(community);
+CREATE INDEX community_datasources_community ON community_datasources(community);
+CREATE INDEX community_support_orgs_community ON community_support_orgs(community);
+CREATE INDEX community_orgs_community ON community_orgs(community);
+CREATE INDEX community_subs_community ON community_subs(community);
+CREATE INDEX community_subs_parent ON community_subs(parent);

apps/dnet-exporter-api/src/main/resources/sql/export_nwo_projects.sql

@@ -0,0 +1,11 @@
+\copy (
+	SELECT 
+		'netherlands'                         AS community,
+		'nwo_________::'||md5(substr(id, 15)) AS project_id,
+		code                                  AS project_code,
+		title                                 AS project_name,
+		acronym                               AS project_acronym,
+		'NWO'                                 AS project_funder
+	FROM projects
+	WHERE id LIKE 'nwo\_\_\_\_\_\_\_\_\_::%'
+) TO '/tmp/nwo_community_projects.csv' CSV HEADER;

apps/dnet-exporter-api/src/main/resources/sql/funders-schema.sql

@@ -0,0 +1,25 @@
+ALTER TABLE dsm_organizations ADD COLUMN registered_funder boolean;
+
+CREATE VIEW funders_view AS SELECT
+	o.id                AS id, 
+	o.legalshortname    AS legalshortname,
+	o.legalname         AS legalname,
+	o.websiteurl        AS websiteurl, 
+	o.logourl           AS logourl, 
+	o.country           AS country,
+	o.dateofcollection  AS registrationdate,
+	o.registered_funder AS registered,
+	array_agg(DISTINCT s.id||' @=@ '||s.officialname||' @=@ '||s.eosc_datasource_type) AS datasources,
+	array_agg(DISTINCT pids.issuertype||' @=@ '||pids.pid) AS pids
+FROM 
+	dsm_organizations o 
+	JOIN dsm_service_organization so           ON (o.id = so.organization) 
+	JOIN dsm_services s                        ON (so.service = s.id) 
+	JOIN projects p                            ON (p.collectedfrom = s.id)
+	LEFT OUTER JOIN dsm_organizationpids opids ON (o.id = opids.organization)
+	LEFT OUTER JOIN dsm_identities pids        ON (opids.pid = pids.pid)
+GROUP BY o.id;
+
+GRANT ALL ON funders_view TO dnetapi;
+
+

apps/dnet-exporter-api/src/main/resources/sql/import_nwo_projects.sql

@@ -0,0 +1 @@
+\copy community_projects FROM '/tmp/nwo_community_projects.csv' CSV HEADER;

apps/dnet-exporter-api/src/test/java/eu/dnetlib/openaire/community/CommunityApiControllerTest.java

@@ -8,7 +8,7 @@ import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.
 import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status;
 
 import java.nio.charset.Charset;
-import java.util.Date;
+import java.time.LocalDateTime;
 import java.util.List;
 
 import org.junit.jupiter.api.Test;
@@ -26,7 +26,7 @@ import eu.dnetlib.openaire.exporter.model.community.CommunitySummary;
  *
  * @author Alessia Bardi
  */
-@SpringBootTest
+//@SpringBootTest
 @WebMvcTest(CommunityApiController.class)
 public class CommunityApiControllerTest {
 
@@ -46,7 +46,7 @@ public class CommunityApiControllerTest {
 		final CommunitySummary cs = new CommunitySummary();
 		cs.setDescription("the description");
 		cs.setId("id1");
-		cs.setLastUpdateDate(new Date());
+		cs.setLastUpdateDate(LocalDateTime.now());
 		cs.setName("X");
 		cs.setShortName("x");
 		final List<CommunitySummary> csList = singletonList(cs);

apps/dnet-exporter-api/src/test/java/eu/dnetlib/openaire/community/importer/CommunityImporterServiceTest.java

@@ -0,0 +1,161 @@
+package eu.dnetlib.openaire.community.importer;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.nio.charset.StandardCharsets;
+import java.util.Arrays;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Queue;
+
+import org.apache.commons.io.IOUtils;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.mockito.ArgumentCaptor;
+import org.mockito.Mock;
+import org.mockito.Mockito;
+import org.mockito.junit.jupiter.MockitoExtension;
+import org.springframework.jdbc.core.JdbcTemplate;
+
+import eu.dnetlib.openaire.community.CommunityService;
+import eu.dnetlib.openaire.community.model.DbOrganization;
+import eu.dnetlib.openaire.community.repository.DbOrganizationRepository;
+import eu.dnetlib.openaire.context.ContextMappingUtils;
+import eu.dnetlib.openaire.exporter.model.community.CommunityContentprovider;
+import eu.dnetlib.openaire.exporter.model.community.CommunityDetails;
+import eu.dnetlib.openaire.exporter.model.community.CommunityOrganization;
+import eu.dnetlib.openaire.exporter.model.community.CommunityProject;
+import eu.dnetlib.openaire.exporter.model.community.SubCommunity;
+import eu.dnetlib.openaire.exporter.model.context.Context;
+
+@ExtendWith(MockitoExtension.class)
+class CommunityImporterServiceTest {
+
+	// Class under test
+	private CommunityImporterService importer;
+
+	@Mock
+	private DbOrganizationRepository dbOrganizationRepository;
+
+	@Mock
+	private CommunityService service;
+
+	@Mock
+	private JdbcTemplate jdbcTemplate;
+
+	@BeforeEach
+	public void setUp() {
+		importer = new CommunityImporterService();
+		importer.setDbOrganizationRepository(dbOrganizationRepository);
+		importer.setService(service);
+		importer.setJdbcTemplate(jdbcTemplate);
+	}
+
+	@Test
+	public void testImportPropagationOrganizationsFromProfile() throws Exception {
+		final String profile = IOUtils.toString(getClass().getResourceAsStream("old_provision_wf.xml"), StandardCharsets.UTF_8.toString());
+		final List<DbOrganization> list = importer.importPropagationOrganizationsFromProfile(profile, true);
+		// list.forEach(System.out::println);
+
+		assertEquals(245, list.size());
+		assertEquals(1, list.stream().filter(o -> "openorgs____::9dd5545aacd3d8019e00c3f837269746".equals(o.getOrgId())).count());
+		assertEquals(2, list.stream().filter(o -> "openorgs____::d11f981828c485cd23d93f7f24f24db1".equals(o.getOrgId())).count());
+		assertEquals(14, list.stream().filter(o -> "beopen".equals(o.getCommunity())).count());
+	}
+
+	@SuppressWarnings("unchecked")
+	@Test
+	public void testImportCommunity() throws Exception {
+		final String profile = IOUtils.toString(getClass().getResourceAsStream("old_community_profile.xml"), StandardCharsets.UTF_8.toString());
+
+		final Queue<Throwable> errors = new LinkedList<>();
+		final Context context = ContextMappingUtils.parseContext(profile, errors);
+		assertTrue(errors.isEmpty());
+
+		Mockito.when(jdbcTemplate.queryForList(Mockito.anyString(), Mockito.any(Class.class), Mockito.anyString())).thenReturn(Arrays.asList("corda_______"));
+		importer.importCommunity(context);
+
+		final ArgumentCaptor<CommunityDetails> detailsCapture = ArgumentCaptor.forClass(CommunityDetails.class);
+		final ArgumentCaptor<CommunityProject> projectsCapture = ArgumentCaptor.forClass(CommunityProject.class);
+		final ArgumentCaptor<CommunityContentprovider> datasourcesCapture = ArgumentCaptor.forClass(CommunityContentprovider.class);
+		final ArgumentCaptor<CommunityOrganization> orgsCapture = ArgumentCaptor.forClass(CommunityOrganization.class);
+		final ArgumentCaptor<SubCommunity> subCommunitiesCapture = ArgumentCaptor.forClass(SubCommunity.class);
+
+		Mockito.verify(service, Mockito.times(1)).saveCommunity(detailsCapture.capture());
+		Mockito.verify(service, Mockito.times(1)).addCommunityProjects(Mockito.anyString(), projectsCapture.capture());
+		Mockito.verify(service, Mockito.times(1)).addCommunityDatasources(Mockito.anyString(), datasourcesCapture.capture());
+		Mockito.verify(service, Mockito.times(1)).addCommunityOrganizations(Mockito.anyString(), orgsCapture.capture());
+		Mockito.verify(service, Mockito.times(1)).addSubCommunities(Mockito.anyString(), subCommunitiesCapture.capture());
+
+		final CommunityDetails details = detailsCapture.getValue();
+		assertEquals("egi", details.getId());
+		// System.out.println(details);
+
+		final List<CommunityProject> projects = projectsCapture.getAllValues();
+		assertEquals(83, projects.size());
+		// projects.forEach(System.out::println);
+
+		final List<CommunityContentprovider> datasources = datasourcesCapture.getAllValues();
+		assertEquals(1, datasources.size());
+		// datasources.forEach(System.out::println);
+
+		final List<CommunityOrganization> orgs = orgsCapture.getAllValues();
+		assertEquals(1, orgs.size());
+		// orgs.forEach(System.out::println);
+
+		final List<SubCommunity> subs = subCommunitiesCapture.getAllValues();
+		assertEquals(688, subs.size());
+		// subs.forEach(System.out::println);
+
+	}
+
+	@SuppressWarnings("unchecked")
+	@Test
+	public void testImportCommunityFetFp7() throws Exception {
+		final String profile = IOUtils.toString(getClass().getResourceAsStream("old_community_profile_fet-fp7.xml"), StandardCharsets.UTF_8.toString());
+
+		final Queue<Throwable> errors = new LinkedList<>();
+		final Context context = ContextMappingUtils.parseContext(profile, errors);
+		assertTrue(errors.isEmpty());
+
+		// Mockito.when(jdbcTemplate.queryForList(Mockito.anyString(), Mockito.any(Class.class),
+		// Mockito.anyString())).thenReturn(Arrays.asList("corda_______"));
+		importer.importCommunity(context);
+
+		final ArgumentCaptor<CommunityDetails> detailsCapture = ArgumentCaptor.forClass(CommunityDetails.class);
+		final ArgumentCaptor<CommunityProject> projectsCapture = ArgumentCaptor.forClass(CommunityProject.class);
+		final ArgumentCaptor<CommunityContentprovider> datasourcesCapture = ArgumentCaptor.forClass(CommunityContentprovider.class);
+		final ArgumentCaptor<CommunityOrganization> orgsCapture = ArgumentCaptor.forClass(CommunityOrganization.class);
+		final ArgumentCaptor<SubCommunity> subCommunitiesCapture = ArgumentCaptor.forClass(SubCommunity.class);
+
+		Mockito.verify(service, Mockito.times(1)).saveCommunity(detailsCapture.capture());
+		Mockito.verify(service, Mockito.times(1)).addCommunityProjects(Mockito.anyString(), projectsCapture.capture());
+		Mockito.verify(service, Mockito.times(1)).addCommunityDatasources(Mockito.anyString(), datasourcesCapture.capture());
+		Mockito.verify(service, Mockito.times(1)).addCommunityOrganizations(Mockito.anyString(), orgsCapture.capture());
+		Mockito.verify(service, Mockito.times(1)).addSubCommunities(Mockito.anyString(), subCommunitiesCapture.capture());
+
+		final CommunityDetails details = detailsCapture.getValue();
+		assertEquals("fet-fp7", details.getId());
+		// System.out.println(details);
+
+		final List<CommunityProject> projects = projectsCapture.getAllValues();
+		assertEquals(0, projects.size());
+		// projects.forEach(System.out::println);
+
+		final List<CommunityContentprovider> datasources = datasourcesCapture.getAllValues();
+		assertEquals(0, datasources.size());
+		// datasources.forEach(System.out::println);
+
+		final List<CommunityOrganization> orgs = orgsCapture.getAllValues();
+		assertEquals(0, orgs.size());
+		// orgs.forEach(System.out::println);
+
+		final List<SubCommunity> subs = subCommunitiesCapture.getAllValues();
+		assertEquals(151, subs.size());
+		subs.forEach(System.out::println);
+
+	}
+
+}

apps/dnet-exporter-api/src/test/java/eu/dnetlib/openaire/dsm/DatasourceApiControllerTest.java

@@ -25,7 +25,6 @@ import eu.dnetlib.openaire.dsm.domain.db.DatasourceDbEntry;
 import eu.dnetlib.openaire.exporter.model.dsm.DatasourceDetails;
 import eu.dnetlib.openaire.exporter.model.vocabularies.Country;
 
-@SpringBootTest
 @WebMvcTest(DsmApiController.class)
 public class DatasourceApiControllerTest {
 

apps/dnet-exporter-api/src/test/java/eu/dnetlib/openaire/funders/FunderContextClientTest.java

@@ -20,11 +20,11 @@ import eu.dnetlib.openaire.exporter.model.funders.FunderDetails;
 
 public class FunderContextClientTest {
 
-	private FunderDao fDao;
+	private FunderService fDao;
 
 	@BeforeEach
 	public void setUp() {
-		fDao = new FunderDao();
+		fDao = new FunderService();
 	}
 
 	@Test

apps/dnet-exporter-api/src/test/java/eu/dnetlib/openaire/info/InfoControllerTest.java

@@ -7,6 +7,7 @@ import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.
 import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status;
 
 import java.nio.charset.Charset;
+import java.nio.charset.StandardCharsets;
 import java.time.LocalDate;
 import java.time.format.DateTimeFormatter;
 import java.util.Arrays;
@@ -14,22 +15,23 @@ import java.util.stream.Collectors;
 
 import org.assertj.core.util.Maps;
 import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Disabled;
 import org.junit.jupiter.api.Test;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.boot.test.autoconfigure.web.servlet.WebMvcTest;
-import org.springframework.boot.test.context.SpringBootTest;
 import org.springframework.boot.test.mock.mockito.MockBean;
 import org.springframework.http.MediaType;
 import org.springframework.test.web.servlet.MockMvc;
 
-@SpringBootTest
+
+@Disabled
 @WebMvcTest(InfoController.class)
 public class InfoControllerTest {
 
 	public static final MediaType APPLICATION_JSON_UTF8 = new MediaType(
 		MediaType.APPLICATION_JSON.getType(),
 		MediaType.APPLICATION_JSON.getSubtype(),
-		Charset.forName("utf8"));
+			StandardCharsets.UTF_8);
 
 	@Autowired
 	private MockMvc mvc;
@@ -37,18 +39,18 @@ public class InfoControllerTest {
 	@MockBean
 	private InfoController infoController;
 
-	private LocalDate expectedDate;
 	private String formattedDate;
 
 	@BeforeEach
 	public void setup() {
-		expectedDate = LocalDate.now();
+		LocalDate expectedDate = LocalDate.now();
 		formattedDate = expectedDate.format(DateTimeFormatter.ofPattern("yyyy-MM-dd"));
 		given(infoController.getDate(JdbcInfoDao.DATE_INFO.claim_load_date.name())).willReturn(expectedDate);
 		given(infoController.getDate(JdbcInfoDao.DATE_INFO.oaf_load_date.name())).willReturn(expectedDate);
 		given(infoController.getDate(JdbcInfoDao.DATE_INFO.odf_load_date.name())).willReturn(expectedDate);
 		given(infoController.getDate(JdbcInfoDao.DATE_INFO.inference_date.name())).willReturn(expectedDate);
 		given(infoController.getDate(JdbcInfoDao.DATE_INFO.stats_update_date.name())).willReturn(expectedDate);
+		given(infoController.getDate(JdbcInfoDao.DATE_INFO.db_load_date.name())).willReturn(expectedDate);
 		given(infoController.listInfo()).willReturn(Maps.newHashMap(JdbcInfoDao.DATE_INFO.inference_date.name(), LocalDate.now()));
 		given(infoController.listInfoKeys())
 			.willReturn(Arrays.stream(JdbcInfoDao.DATE_INFO.values()).map(JdbcInfoDao.DATE_INFO::name).collect(Collectors.toList()));
@@ -116,4 +118,12 @@ public class InfoControllerTest {
 			.getContentAsString());
 	}
 
+	@Test
+	public void testGetDbLoadDate() throws Exception {
+		mvc.perform(get("/info/db_load_date")
+						.contentType(APPLICATION_JSON_UTF8))
+				.andExpect(status().isOk())
+				.andExpect(jsonPath("$", is(formattedDate)));
+	}
+
 }

apps/dnet-is-application/pom.xml

@@ -1,82 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
-
-	<parent>
-		<groupId>eu.dnetlib.dhp</groupId>
-		<artifactId>apps</artifactId>
-		<version>3.4.3-SNAPSHOT</version>
-		<relativePath>../pom.xml</relativePath>
-	</parent>
-
-	<modelVersion>4.0.0</modelVersion>
-	<artifactId>dnet-is-application</artifactId>
-	<packaging>jar</packaging>
-
-	<dependencies>
-		<!-- Openaire IS common -->
-		<dependency>
-			<groupId>eu.dnetlib.dhp</groupId>
-			<artifactId>dnet-is-services</artifactId>
-			<version>${project.version}</version>
-		</dependency>
-		
-		<dependency>
-			<groupId>eu.dnetlib.dhp</groupId>
-			<artifactId>dnet-wf-service</artifactId>
-			<version>${project.version}</version>
-		</dependency>
-		
-		<dependency>
-			<groupId>eu.dnetlib.dhp</groupId>
-			<artifactId>dnet-data-services</artifactId>
-			<version>${project.version}</version>
-		</dependency>
-		
-		<!-- hot swapping, disable cache for template, enable live reload -->
-		<dependency>
-			<groupId>org.springframework.boot</groupId>
-			<artifactId>spring-boot-devtools</artifactId>
-			<optional>true</optional>
-		</dependency>
-
-		<!-- Tests -->
-		<dependency>
-			<groupId>org.junit.jupiter</groupId>
-			<artifactId>junit-jupiter</artifactId>
-			<scope>test</scope>
-		</dependency>
-
-		<dependency>
-			<groupId>org.mockito</groupId>
-			<artifactId>mockito-core</artifactId>
-			<scope>test</scope>
-		</dependency>
-
-		<dependency>
-			<groupId>org.mockito</groupId>
-			<artifactId>mockito-junit-jupiter</artifactId>
-			<scope>test</scope>
-		</dependency>
-
-		<dependency>
-			<groupId>org.springframework.boot</groupId>
-			<artifactId>spring-boot-starter-test</artifactId>
-			<scope>test</scope>
-		</dependency>
-
-	</dependencies>
-
-	<build>
-		<plugins>
-			<plugin>
-				<groupId>org.apache.maven.plugins</groupId>
-				<artifactId>maven-help-plugin</artifactId>
-			</plugin>
-			<plugin>
-				<groupId>org.springframework.boot</groupId>
-				<artifactId>spring-boot-maven-plugin</artifactId>
-			</plugin>
-		</plugins>
-	</build>
-
-</project>

apps/dnet-is-application/src/main/java/eu/dnetlib/MainAjaxController.java

@@ -1,30 +0,0 @@
-package eu.dnetlib;
-
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.http.MediaType;
-import org.springframework.web.bind.annotation.GetMapping;
-import org.springframework.web.bind.annotation.PathVariable;
-import org.springframework.web.bind.annotation.RestController;
-
-import eu.dnetlib.is.model.resource.ResourceType;
-import eu.dnetlib.is.resource.repository.ResourceTypeRepository;
-
-@RestController
-public class MainAjaxController {
-
-	@Autowired
-	private ResourceTypeRepository resourceTypeRepository;
-
-	@GetMapping("/ajax/resourceTypes")
-	public Iterable<ResourceType> resourceTypes() {
-		return resourceTypeRepository.findAll();
-	}
-
-	@GetMapping("/ajax/resourceTypes/{id}")
-	public ResourceType resourceTypes(@PathVariable final String id) {
-		return resourceTypeRepository
-			.findById(id)
-			.orElse(new ResourceType("not_present", "???", MediaType.TEXT_PLAIN_VALUE, 0));
-	}
-
-}

apps/dnet-is-application/src/main/java/eu/dnetlib/MainApplication.java

@@ -1,36 +0,0 @@
-package eu.dnetlib;
-
-import org.springdoc.core.GroupedOpenApi;
-import org.springframework.boot.SpringApplication;
-import org.springframework.boot.autoconfigure.SpringBootApplication;
-import org.springframework.cache.annotation.EnableCaching;
-import org.springframework.context.annotation.Bean;
-import org.springframework.context.annotation.ComponentScan;
-import org.springframework.scheduling.annotation.EnableScheduling;
-
-import eu.dnetlib.common.app.AbstractDnetApp;
-
-@SpringBootApplication
-@EnableCaching
-@EnableScheduling
-@ComponentScan(basePackages = "eu.dnetlib")
-public class MainApplication extends AbstractDnetApp {
-
-	public static void main(final String[] args) {
-		SpringApplication.run(MainApplication.class, args);
-	}
-
-	@Bean
-	public GroupedOpenApi publicApi() {
-		return GroupedOpenApi.builder()
-			.group("D-Net Information Service API")
-			.pathsToMatch("/api/**")
-			.build();
-	}
-
-	@Override
-	protected String swaggerTitle() {
-		return "D-Net Information Service APIs";
-	}
-
-}

apps/dnet-is-application/src/main/java/eu/dnetlib/MainDBConfig.java

@@ -1,63 +0,0 @@
-package eu.dnetlib;
-
-import javax.persistence.EntityManagerFactory;
-import javax.sql.DataSource;
-
-import org.springframework.beans.factory.annotation.Qualifier;
-import org.springframework.boot.context.properties.ConfigurationProperties;
-import org.springframework.boot.jdbc.DataSourceBuilder;
-import org.springframework.boot.orm.jpa.EntityManagerFactoryBuilder;
-import org.springframework.context.annotation.Bean;
-import org.springframework.context.annotation.Configuration;
-import org.springframework.context.annotation.Primary;
-import org.springframework.data.jpa.repository.config.EnableJpaRepositories;
-import org.springframework.jdbc.core.JdbcTemplate;
-import org.springframework.orm.jpa.JpaTransactionManager;
-import org.springframework.orm.jpa.LocalContainerEntityManagerFactoryBean;
-import org.springframework.transaction.PlatformTransactionManager;
-import org.springframework.transaction.annotation.EnableTransactionManagement;
-
-@Configuration
-@EnableTransactionManagement
-@EnableJpaRepositories(entityManagerFactoryRef = "mainEntityManagerFactory", transactionManagerRef = "mainTransactionManager", basePackages = {
-	"eu.dnetlib.is",
-	"eu.dnetlib.common",
-	"eu.dnetlib.manager.history",
-	"eu.dnetlib.manager.wf",
-	"eu.dnetlib.data.mdstore"
-})
-public class MainDBConfig {
-
-	@Primary
-	@Bean(name = "mainDataSource")
-	@ConfigurationProperties(prefix = "is.datasource")
-	public DataSource dataSource() {
-		return DataSourceBuilder.create().build();
-	}
-
-	@Primary
-	@Bean(name = "mainEntityManagerFactory")
-	public LocalContainerEntityManagerFactoryBean entityManagerFactory(
-		final EntityManagerFactoryBuilder builder,
-		@Qualifier("mainDataSource") final DataSource ds) {
-		return builder
-			.dataSource(ds)
-			.packages("eu.dnetlib.is.model", "eu.dnetlib.manager.history.model", "eu.dnetlib.common.model", "eu.dnetlib.manager.wf.model", "eu.dnetlib.data.mdstore.model")
-			.persistenceUnit("is")
-			.build();
-	}
-
-	@Primary
-	@Bean(name = "mainTransactionManager")
-	public PlatformTransactionManager transactionManager(
-		@Qualifier("mainEntityManagerFactory") final EntityManagerFactory emf) {
-		return new JpaTransactionManager(emf);
-	}
-
-	@Primary
-	@Bean(name = "mainJdbcTemplate")
-	public JdbcTemplate jdbcTemplate1(@Qualifier("mainDataSource") final DataSource ds) {
-		return new JdbcTemplate(ds);
-	}
-
-}

apps/dnet-is-application/src/main/java/eu/dnetlib/OpenaireDBConfig.java

@@ -1,53 +0,0 @@
-package eu.dnetlib;
-
-import javax.persistence.EntityManagerFactory;
-import javax.sql.DataSource;
-
-import org.springframework.beans.factory.annotation.Qualifier;
-import org.springframework.boot.context.properties.ConfigurationProperties;
-import org.springframework.boot.jdbc.DataSourceBuilder;
-import org.springframework.boot.orm.jpa.EntityManagerFactoryBuilder;
-import org.springframework.context.annotation.Bean;
-import org.springframework.context.annotation.Configuration;
-import org.springframework.data.jpa.repository.config.EnableJpaRepositories;
-import org.springframework.jdbc.core.JdbcTemplate;
-import org.springframework.orm.jpa.JpaTransactionManager;
-import org.springframework.orm.jpa.LocalContainerEntityManagerFactoryBean;
-import org.springframework.transaction.PlatformTransactionManager;
-import org.springframework.transaction.annotation.EnableTransactionManagement;
-
-@Configuration
-@EnableTransactionManagement
-@EnableJpaRepositories(entityManagerFactoryRef = "openaireEntityManagerFactory", transactionManagerRef = "openaireTransactionManager", basePackages = {
-	"eu.dnetlib.dsm"
-})
-public class OpenaireDBConfig {
-
-	@Bean(name = "openaireDataSource")
-	@ConfigurationProperties(prefix = "openaire.datasource")
-	public DataSource dataSource() {
-		return DataSourceBuilder.create().build();
-	}
-
-	@Bean(name = "openaireEntityManagerFactory")
-	public LocalContainerEntityManagerFactoryBean openaireEntityManagerFactory(
-		final EntityManagerFactoryBuilder builder,
-		@Qualifier("openaireDataSource") final DataSource ds) {
-		return builder
-			.dataSource(ds)
-			.packages("eu.dnetlib.dsm.model")
-			.persistenceUnit("openaire")
-			.build();
-	}
-
-	@Bean(name = "openaireTransactionManager")
-	public PlatformTransactionManager opeanaireTransactionManager(
-		@Qualifier("openaireEntityManagerFactory") final EntityManagerFactory emf) {
-		return new JpaTransactionManager(emf);
-	}
-
-	@Bean(name = "openaireJdbcTemplate")
-	public JdbcTemplate jdbcTemplate1(@Qualifier("openaireDataSource") final DataSource ds) {
-		return new JdbcTemplate(ds);
-	}
-}

apps/dnet-is-application/src/main/java/eu/dnetlib/data/mapping/MappingAjaxController.java

@@ -1,44 +0,0 @@
-package eu.dnetlib.data.mapping;
-
-import java.nio.charset.StandardCharsets;
-
-import javax.servlet.http.HttpServletRequest;
-import javax.servlet.http.HttpServletResponse;
-
-import org.apache.commons.io.IOUtils;
-import org.apache.commons.lang3.StringUtils;
-import org.apache.http.HttpStatus;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.http.MediaType;
-import org.springframework.web.bind.annotation.PostMapping;
-import org.springframework.web.bind.annotation.RequestMapping;
-import org.springframework.web.bind.annotation.RequestParam;
-import org.springframework.web.bind.annotation.RestController;
-
-import eu.dnetlib.data.mapping.cleaner.CleanerFactory;
-
-@RestController
-@RequestMapping("/ajax/mapping")
-public class MappingAjaxController {
-
-	@Autowired
-	private CleanerFactory cleanerFactory;
-
-	@PostMapping(value = "/clean", consumes = {
-		MediaType.TEXT_PLAIN_VALUE, MediaType.APPLICATION_XML_VALUE,
-	}, produces = MediaType.APPLICATION_XML_VALUE)
-	public void importResource(@RequestParam final String rule, final HttpServletRequest req, final HttpServletResponse res) throws Exception {
-		final String xmlIn = IOUtils.toString(req.getInputStream(), StandardCharsets.UTF_8);
-		final String xmlOut = cleanerFactory.obtainCleaningRule(rule).transform(xmlIn);
-
-		res.setCharacterEncoding(StandardCharsets.UTF_8.name());
-
-		res.setContentType(MediaType.APPLICATION_XML_VALUE);
-
-		if (StringUtils.isNotBlank(xmlOut)) {
-			IOUtils.write(xmlOut, res.getOutputStream(), StandardCharsets.UTF_8.name());
-		} else {
-			res.sendError(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Invalid record");
-		}
-	}
-}

apps/dnet-is-application/src/main/java/eu/dnetlib/data/mdstore/AbstractMDStoreController.java

@@ -1,124 +0,0 @@
-package eu.dnetlib.data.mdstore;
-
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.web.bind.annotation.DeleteMapping;
-import org.springframework.web.bind.annotation.GetMapping;
-import org.springframework.web.bind.annotation.PathVariable;
-import org.springframework.web.bind.annotation.PostMapping;
-import org.springframework.web.bind.annotation.RequestParam;
-
-import eu.dnetlib.common.controller.AbstractDnetController;
-import eu.dnetlib.data.mdstore.model.MDStoreType;
-import eu.dnetlib.data.mdstore.model.MDStoreVersion;
-import eu.dnetlib.data.mdstore.model.MDStoreWithInfo;
-import eu.dnetlib.errors.MDStoreManagerException;
-import io.swagger.v3.oas.annotations.Operation;
-
-public class AbstractMDStoreController extends AbstractDnetController {
-
-	@Autowired
-	protected MDStoreService service;
-
-	@Operation(summary = "Return all the mdstores")
-	@GetMapping("/")
-	public Iterable<MDStoreWithInfo> find() {
-		return service.listMdStores();
-	}
-
-	@Operation(summary = "Return a mdstores by id")
-	@GetMapping("/mdstore/{mdId}")
-	public MDStoreWithInfo getMdStore(@PathVariable final String mdId) throws MDStoreManagerException {
-		return service.findMdStore(mdId);
-	}
-
-	@Operation(summary = "Create a new mdstore")
-	@PostMapping("/new/{format}/{layout}/{interpretation}/{type}")
-	public MDStoreWithInfo createMDStore(
-		@PathVariable final String format,
-		@PathVariable final String layout,
-		@PathVariable final String interpretation,
-		@PathVariable final String type,
-		@RequestParam(required = true) final String dsName,
-		@RequestParam(required = true) final String dsId,
-		@RequestParam(required = true) final String apiId) throws MDStoreManagerException {
-
-		final String id = service.createMDStore(format, layout, interpretation, MDStoreType.valueOf(type), dsName, dsId, apiId);
-
-		return service.findMdStore(id);
-	}
-
-	@Operation(summary = "Delete a mdstore by id")
-	@DeleteMapping("/mdstore/{mdId}")
-	public StatusResponse delete(@PathVariable final String mdId)
-		throws MDStoreManagerException {
-		service.deleteMdStore(mdId);
-
-		return StatusResponse.DELETED;
-	}
-
-	@Operation(summary = "Return all the versions of a mdstore")
-	@GetMapping("/mdstore/{mdId}/versions")
-	public Iterable<MDStoreVersion> listVersions(@PathVariable final String mdId) throws MDStoreManagerException {
-		return service.listVersions(mdId);
-	}
-
-	@Operation(summary = "Create a new preliminary version of a mdstore")
-	@GetMapping("/mdstore/{mdId}/newVersion")
-	public MDStoreVersion prepareNewVersion(
-		@PathVariable final String mdId) throws MDStoreManagerException {
-		return service.prepareMdStoreVersion(mdId);
-	}
-
-	@Operation(summary = "Promote a preliminary version to current")
-	@GetMapping("/version/{versionId}/commit/{size}")
-	public MDStoreVersion commitVersion(
-		@PathVariable final String versionId,
-		@PathVariable final long size) throws MDStoreManagerException {
-		try {
-			return service.commitMdStoreVersion(versionId, size);
-		} finally {
-			service.deleteExpiredVersions();
-		}
-	}
-
-	@Operation(summary = "Abort a preliminary version")
-	@GetMapping("/version/{versionId}/abort")
-	public StatusResponse commitVersion(@PathVariable final String versionId)
-		throws MDStoreManagerException {
-
-		service.deleteMdStoreVersion(versionId, true);
-
-		return StatusResponse.ABORTED;
-	}
-
-	@Operation(summary = "Return an existing mdstore version")
-	@GetMapping("/version/{versionId}")
-	public MDStoreVersion getVersion(@PathVariable final String versionId)
-		throws MDStoreManagerException {
-		return service.findVersion(versionId);
-	}
-
-	@Operation(summary = "Delete a mdstore version")
-	@DeleteMapping("/version/{versionId}")
-	public StatusResponse deleteVersion(@PathVariable final String versionId,
-		@RequestParam(required = false, defaultValue = "false") final boolean force)
-		throws MDStoreManagerException {
-
-		service.deleteMdStoreVersion(versionId, force);
-
-		return StatusResponse.DELETED;
-	}
-
-	@Operation(summary = "Decrease the read count of a mdstore version")
-	@GetMapping("/version/{versionId}/endReading")
-	public MDStoreVersion endReading(@PathVariable final String versionId) throws MDStoreManagerException {
-		return service.endReading(versionId);
-	}
-
-	@Operation(summary = "Reset the read count of a mdstore version")
-	@GetMapping("/version/{versionId}/resetReading")
-	public MDStoreVersion resetReading(@PathVariable final String versionId) throws MDStoreManagerException {
-		return service.resetReading(versionId);
-	}
-
-}

apps/dnet-is-application/src/main/java/eu/dnetlib/data/mdstore/MDStoreAjaxController.java

@@ -1,56 +0,0 @@
-package eu.dnetlib.data.mdstore;
-
-import java.util.LinkedHashMap;
-import java.util.List;
-import java.util.Map;
-
-import org.springframework.beans.factory.annotation.Value;
-import org.springframework.web.bind.annotation.GetMapping;
-import org.springframework.web.bind.annotation.PathVariable;
-import org.springframework.web.bind.annotation.RequestMapping;
-import org.springframework.web.bind.annotation.RestController;
-
-import eu.dnetlib.data.mdstore.model.records.MetadataRecord;
-import eu.dnetlib.errors.MDStoreManagerException;
-import io.swagger.v3.oas.annotations.Operation;
-import io.swagger.v3.oas.annotations.tags.Tag;
-
-@RestController
-@RequestMapping("/ajax/mdstores")
-@Tag(name = "Metadata Stores")
-public class MDStoreAjaxController extends AbstractMDStoreController {
-
-	@Value("${dhp.mdstore-manager.hadoop.cluster}")
-	private String hadoopCluster;
-
-	@Value("${dhp.mdstore-manager.hadoop.user}")
-	private String hadoopUser;
-
-	@Value("${dhp.mdstore-manager.hdfs.base-path}")
-	private String hdfsBasePath;
-
-	@Operation(summary = "Show informations")
-	@GetMapping("/info")
-	public Map<String, Object> info() {
-		final Map<String, Object> info = new LinkedHashMap<>();
-		info.put("number_of_mdstores", service.countMdStores());
-		info.put("hadoop_user", hadoopUser);
-		info.put("hadoop_cluster", hadoopCluster);
-		info.put("hdfs_base_path", hdfsBasePath);
-		info.put("expired_versions", service.listExpiredVersions());
-		return info;
-	}
-
-	@Operation(summary = "read the parquet file of a mdstore version")
-	@GetMapping("/version/{versionId}/content/{limit}")
-	public List<? extends MetadataRecord> listVersionRecords(@PathVariable final String versionId, @PathVariable final long limit)
-		throws MDStoreManagerException {
-		return service.listVersionRecords(versionId, limit);
-	}
-
-	@Operation(summary = "read the parquet file of a mdstore (current version)")
-	@GetMapping("/mdstore/{mdId}/content/{limit}")
-	public List<? extends MetadataRecord> listMdstoreRecords(@PathVariable final String mdId, @PathVariable final long limit) throws MDStoreManagerException {
-		return service.listVersionRecords(service.findMdStore(mdId).getCurrentVersion(), limit);
-	}
-}

apps/dnet-is-application/src/main/java/eu/dnetlib/data/mdstore/MDStoreApiController.java

@@ -1,59 +0,0 @@
-package eu.dnetlib.data.mdstore;
-
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
-import org.springframework.web.bind.annotation.DeleteMapping;
-import org.springframework.web.bind.annotation.GetMapping;
-import org.springframework.web.bind.annotation.PathVariable;
-import org.springframework.web.bind.annotation.RequestMapping;
-import org.springframework.web.bind.annotation.RequestParam;
-import org.springframework.web.bind.annotation.RestController;
-
-import eu.dnetlib.data.mdstore.model.MDStoreType;
-import eu.dnetlib.data.mdstore.model.MDStoreVersion;
-import eu.dnetlib.errors.MDStoreManagerException;
-import io.swagger.v3.oas.annotations.Operation;
-import io.swagger.v3.oas.annotations.tags.Tag;
-
-@RestController
-@RequestMapping("/api/mdstores")
-@Tag(name = "Metadata Stores")
-public class MDStoreApiController extends AbstractMDStoreController {
-
-	@Operation(summary = "Return all the mdstore identifiers")
-	@GetMapping("/ids")
-	public List<String> findIdentifiers() {
-		return service.listMdStoreIDs();
-	}
-
-	@Operation(summary = "Increase the read count of the current mdstore")
-	@GetMapping("/mdstore/{mdId}/startReading")
-	public MDStoreVersion startReading(@PathVariable final String mdId) throws MDStoreManagerException {
-		return service.startReading(mdId);
-	}
-
-	@Operation(summary = "Fix the inconsistencies on HDFS")
-	@GetMapping("/hdfs/inconsistencies")
-	public Map<MDStoreType, Set<String>> fixHdfsInconsistencies(
-		@RequestParam(required = false, defaultValue = "false") final boolean forceDelete)
-		throws MDStoreManagerException {
-
-		return service.fixInconsistencies(forceDelete);
-	}
-
-	@Operation(summary = "Delete expired versions")
-	@DeleteMapping("/versions/expired")
-	public StatusResponse deleteExpiredVersions() {
-		new Thread(service::deleteExpiredVersions).start();
-		return StatusResponse.DELETING;
-	}
-
-	@Operation(summary = "list the file inside the path of a mdstore version")
-	@GetMapping("/version/{versionId}/files")
-	public Set<String> listVersionFiles(@PathVariable final String versionId) throws MDStoreManagerException {
-		return service.listVersionInternalFiles(versionId);
-	}
-
-}

apps/dnet-is-application/src/main/java/eu/dnetlib/data/mdstore/StatusResponse.java

@@ -1,25 +0,0 @@
-package eu.dnetlib.data.mdstore;
-
-public class StatusResponse {
-
-	public static final StatusResponse DELETED = new StatusResponse("DELETED");
-	public static final StatusResponse DELETING = new StatusResponse("DELETING...");
-	public static final StatusResponse ABORTED = new StatusResponse("ABORTED");;
-
-	private String status;
-
-	public StatusResponse() {}
-
-	public StatusResponse(final String status) {
-		this.status = status;
-	}
-
-	public String getStatus() {
-		return status;
-	}
-
-	public void setStatus(final String status) {
-		this.status = status;
-	}
-
-}

apps/dnet-is-application/src/main/java/eu/dnetlib/data/mdstore/ZeppelinAjaxController.java

@@ -1,30 +0,0 @@
-package eu.dnetlib.data.mdstore;
-
-import java.util.List;
-
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.web.bind.annotation.GetMapping;
-import org.springframework.web.bind.annotation.RequestMapping;
-import org.springframework.web.bind.annotation.RestController;
-
-import eu.dnetlib.common.controller.AbstractDnetController;
-import eu.dnetlib.data.mdstore.hadoop.ZeppelinClient;
-import eu.dnetlib.errors.MDStoreManagerException;
-
-@RestController
-@RequestMapping("/zeppelin")
-public class ZeppelinAjaxController extends AbstractDnetController {
-
-	@Autowired
-	private ZeppelinClient zeppelinClient;
-
-	@GetMapping("/templates")
-	public List<String> getTemplates() throws MDStoreManagerException {
-		try {
-			return zeppelinClient.listTemplates();
-		} catch (final Throwable e) {
-			throw new MDStoreManagerException("Zeppelin is unreachable", e);
-		}
-	}
-
-}

apps/dnet-is-application/src/main/java/eu/dnetlib/data/mdstore/ZeppelinController.java

@@ -1,37 +0,0 @@
-package eu.dnetlib.data.mdstore;
-
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.stereotype.Controller;
-import org.springframework.web.bind.annotation.PathVariable;
-import org.springframework.web.bind.annotation.RequestMapping;
-
-import eu.dnetlib.data.mdstore.hadoop.ZeppelinClient;
-import eu.dnetlib.data.mdstore.model.MDStoreType;
-import eu.dnetlib.data.mdstore.model.MDStoreVersion;
-import eu.dnetlib.data.mdstore.model.MDStoreWithInfo;
-import eu.dnetlib.errors.MDStoreManagerException;
-
-@Controller
-public class ZeppelinController {
-
-	@Autowired
-	private ZeppelinClient zeppelinClient;
-
-	@Autowired
-	private MDStoreService service;
-
-	@RequestMapping("/zeppelin/{mdId}/{note}")
-	public String goToZeppelin(@PathVariable final String mdId, final @PathVariable String note) throws MDStoreManagerException {
-		final MDStoreWithInfo mdstore = service.findMdStore(mdId);
-		if (mdstore.getType() == MDStoreType.HDFS) {
-			final String currentVersion = mdstore.getCurrentVersion();
-			final MDStoreVersion version = service.findVersion(currentVersion);
-			if (version.getParams().containsKey("hdfs_path")) {
-				final String path = version.getParams().get("hdfs_path") + "/store";
-				return "redirect:" + zeppelinClient.zeppelinNote(note, mdstore, path);
-			}
-		}
-		throw new MDStoreManagerException();
-	}
-
-}

apps/dnet-is-application/src/main/java/eu/dnetlib/dsm/AbstractDsmController.java

@@ -1,119 +0,0 @@
-package eu.dnetlib.dsm;
-
-import java.util.List;
-import java.util.stream.Collectors;
-
-import org.apache.commons.lang3.StringUtils;
-import org.apache.commons.lang3.exception.ExceptionUtils;
-import org.apache.commons.lang3.time.StopWatch;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.springframework.http.HttpStatus;
-import org.springframework.web.bind.MethodArgumentNotValidException;
-import org.springframework.web.bind.annotation.ExceptionHandler;
-import org.springframework.web.bind.annotation.ResponseBody;
-import org.springframework.web.bind.annotation.ResponseStatus;
-
-import com.fasterxml.jackson.annotation.JsonAutoDetect;
-
-import eu.dnetlib.common.controller.AbstractDnetController;
-import eu.dnetlib.dsm.domain.Response;
-import eu.dnetlib.errors.DsmException;
-import eu.dnetlib.errors.DsmForbiddenException;
-import eu.dnetlib.errors.DsmNotFoundException;
-
-/**
- * Created by claudio on 18/07/2017.
- */
-public abstract class AbstractDsmController extends AbstractDnetController {
-
-	private static final Log log = LogFactory.getLog(DsmException.class); // NOPMD by marko on 11/24/08 5:02 PM
-
-	@ResponseBody
-	@ExceptionHandler({
-		DsmException.class
-	})
-	@ResponseStatus(value = HttpStatus.INTERNAL_SERVER_ERROR)
-	public ErrorMessage handleDSMException(final Exception e) {
-		return _handleError(e);
-	}
-
-	@ResponseBody
-	@ExceptionHandler(DsmForbiddenException.class)
-	@ResponseStatus(value = HttpStatus.FORBIDDEN)
-	public ErrorMessage handleForbiddenException(final Exception e) {
-		return _handleError(e);
-	}
-
-	@ResponseBody
-	@ExceptionHandler({
-		DsmNotFoundException.class
-	})
-	@ResponseStatus(value = HttpStatus.NOT_FOUND)
-	public ErrorMessage handleNotFoundException(final Exception e) {
-		return _handleError(e);
-	}
-
-	@ResponseBody
-	@ExceptionHandler(MethodArgumentNotValidException.class)
-	@ResponseStatus(HttpStatus.BAD_REQUEST)
-	public List<ErrorMessage> processValidationError(final MethodArgumentNotValidException e) {
-		return e.getBindingResult()
-			.getFieldErrors()
-			.stream()
-			.map(fe -> new ErrorMessage(
-				String.format("field '%s'", fe.getField()),
-				String.format("rejected value '%s'", fe.getRejectedValue()),
-				fe.getDefaultMessage()))
-			.collect(Collectors.toList());
-	}
-
-	private ErrorMessage _handleError(final Exception e) {
-		log.error(e);
-		if (StringUtils.containsIgnoreCase(ExceptionUtils.getRootCauseMessage(e), "Broken pipe")) {
-			return null;        // socket is closed, cannot return any response
-		} else {
-			return new ErrorMessage(e);
-		}
-	}
-
-	// HELPERS
-	protected <T extends Response> T prepareResponse(final int page, final int size, final StopWatch stopWatch, final T rsp) {
-		rsp.getHeader()
-			.setTime(stopWatch.getTime())
-			.setPage(page)
-			.setSize(size);
-		return rsp;
-	}
-
-	@JsonAutoDetect
-	public class ErrorMessage {
-
-		private final String message;
-		private final String details;
-		private final String stacktrace;
-
-		public ErrorMessage(final Exception e) {
-			this(e.getMessage(), "", ExceptionUtils.getStackTrace(e));
-		}
-
-		public ErrorMessage(final String message, final String details, final String stacktrace) {
-			this.message = message;
-			this.details = details;
-			this.stacktrace = stacktrace;
-		}
-
-		public String getMessage() {
-			return this.message;
-		}
-
-		public String getStacktrace() {
-			return this.stacktrace;
-		}
-
-		public String getDetails() {
-			return details;
-		}
-	}
-
-}

apps/dnet-is-application/src/main/java/eu/dnetlib/dsm/DsmAjaxController.java

@@ -1,86 +0,0 @@
-package eu.dnetlib.dsm;
-
-import java.util.Arrays;
-import java.util.LinkedHashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.stream.Collectors;
-
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.data.domain.Page;
-import org.springframework.web.bind.annotation.GetMapping;
-import org.springframework.web.bind.annotation.PathVariable;
-import org.springframework.web.bind.annotation.PostMapping;
-import org.springframework.web.bind.annotation.RequestBody;
-import org.springframework.web.bind.annotation.RequestMapping;
-import org.springframework.web.bind.annotation.RequestParam;
-import org.springframework.web.bind.annotation.RestController;
-
-import eu.dnetlib.common.controller.AbstractDnetController;
-import eu.dnetlib.dsm.model.Api;
-import eu.dnetlib.dsm.model.BrowseTerm;
-import eu.dnetlib.dsm.model.readonly.SimpleDsWithApis;
-import eu.dnetlib.dsm.utils.DsmBrowsableFields;
-import eu.dnetlib.errors.DsmException;
-import eu.dnetlib.is.info.KeyValue;
-import eu.dnetlib.is.protocol.ProtocolService;
-import eu.dnetlib.is.vocabulary.repository.VocabularyTermRepository;
-
-@RestController
-@RequestMapping("/ajax/dsm")
-public class DsmAjaxController extends AbstractDnetController {
-
-	@Autowired
-	private DsmService dsmService;
-
-	@Autowired
-	private VocabularyTermRepository vocabularyTermRepository;
-
-	@Autowired
-	private ProtocolService protocolService;
-
-	@GetMapping("/browsableFields")
-	public List<KeyValue<String>> browsableFields() {
-		return Arrays.stream(DsmBrowsableFields.values())
-			.map(f -> new KeyValue<>(f.name(), f.desc))
-			.collect(Collectors.toList());
-	}
-
-	@GetMapping("/conf")
-	public Map<String, Iterable<?>> configuration() {
-		final Map<String, Iterable<?>> map = new LinkedHashMap<>();
-		map.put("protocols", protocolService.listProtocols());
-		map.put("compatibilityLevels", vocabularyTermRepository.findTermsByVocabulary("dnet:compatibilityLevel"));
-		map.put("contentDescTypes", vocabularyTermRepository.findTermsByVocabulary("dnet:content_description_typologies"));
-		return map;
-	}
-
-	@GetMapping("/browse/{field}")
-	public List<BrowseTerm> browse(@PathVariable final String field) {
-		return dsmService.browseTerm(DsmBrowsableFields.valueOf(field));
-	}
-
-	@GetMapping("/searchByField/{field}/{page}/{size}")
-	public Page<SimpleDsWithApis> searchByField(@PathVariable final String field,
-		@PathVariable final int page,
-		@PathVariable final int size,
-		@RequestParam final String value) {
-		return dsmService.searchByField(DsmBrowsableFields.valueOf(field), value, page, size);
-	}
-
-	@GetMapping("/search/{page}/{size}")
-	public Page<SimpleDsWithApis> search(@RequestParam final String value, @PathVariable final int page, @PathVariable final int size) {
-		return dsmService.search(value, page, size);
-	}
-
-	@GetMapping("/api")
-	public Api findApi(@RequestParam final String id) throws DsmException {
-		return dsmService.findApi(id);
-	}
-
-	@PostMapping("/api")
-	public Api saveApi(@RequestBody final Api api) throws DsmException {
-		throw new DsmException("TO BE IMPLEMENTED");
-	}
-
-}

apps/dnet-is-application/src/main/java/eu/dnetlib/dsm/DsmApiControllerV1.java

@@ -1,281 +0,0 @@
-package eu.dnetlib.dsm;
-
-import static eu.dnetlib.dsm.utils.DsmMappingUtils.asDbEntry;
-import static eu.dnetlib.dsm.utils.DsmMappingUtils.copyNonNullProperties;
-import static eu.dnetlib.dsm.utils.DsmMappingUtils.createId;
-
-import java.util.Collection;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-import java.util.function.Function;
-import java.util.stream.Collectors;
-import java.util.stream.Stream;
-
-import javax.validation.Valid;
-
-import org.apache.commons.lang3.StringUtils;
-import org.apache.commons.lang3.time.StopWatch;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
-import org.springframework.data.domain.Page;
-import org.springframework.web.bind.annotation.DeleteMapping;
-import org.springframework.web.bind.annotation.GetMapping;
-import org.springframework.web.bind.annotation.PathVariable;
-import org.springframework.web.bind.annotation.PostMapping;
-import org.springframework.web.bind.annotation.RequestBody;
-import org.springframework.web.bind.annotation.RequestMapping;
-import org.springframework.web.bind.annotation.RequestParam;
-import org.springframework.web.bind.annotation.RestController;
-
-import eu.dnetlib.dsm.domain.AggregationHistoryResponse;
-import eu.dnetlib.dsm.domain.AggregationInfo;
-import eu.dnetlib.dsm.domain.ApiDetails;
-import eu.dnetlib.dsm.domain.ApiDetailsResponse;
-import eu.dnetlib.dsm.domain.Country;
-import eu.dnetlib.dsm.domain.DatasourceDetailResponse;
-import eu.dnetlib.dsm.domain.DatasourceDetails;
-import eu.dnetlib.dsm.domain.DatasourceDetailsUpdate;
-import eu.dnetlib.dsm.domain.DatasourceDetailsWithApis;
-import eu.dnetlib.dsm.domain.DatasourceSnippetResponse;
-import eu.dnetlib.dsm.domain.RegisteredDatasourceInfo;
-import eu.dnetlib.dsm.domain.RequestFilter;
-import eu.dnetlib.dsm.domain.RequestSort;
-import eu.dnetlib.dsm.domain.RequestSortOrder;
-import eu.dnetlib.dsm.domain.SimpleResponse;
-import eu.dnetlib.dsm.model.Api;
-import eu.dnetlib.dsm.model.Datasource;
-import eu.dnetlib.dsm.model.Identity;
-import eu.dnetlib.dsm.utils.DsmMappingUtils;
-import eu.dnetlib.dsm.utils.ResponseUtils;
-import eu.dnetlib.dsm.utils.WfLoggerClient;
-import eu.dnetlib.errors.DsmException;
-import eu.dnetlib.errors.DsmForbiddenException;
-import eu.dnetlib.errors.DsmNotFoundException;
-import io.swagger.v3.oas.annotations.responses.ApiResponse;
-import io.swagger.v3.oas.annotations.responses.ApiResponses;
-import io.swagger.v3.oas.annotations.tags.Tag;
-
-@RestController
-@RequestMapping("/api/dsm/1.0")
-@ConditionalOnProperty(value = "openaire.api.enable.dsm", havingValue = "true")
-@Tag(name = "OpenAIRE DSM API", description = "the OpenAIRE Datasource Manager API")
-public class DsmApiControllerV1 extends AbstractDsmController {
-
-	private static final Log log = LogFactory.getLog(DsmApiControllerV1.class);
-
-	@Autowired
-	private WfLoggerClient wfLoggerClient;
-
-	@Autowired
-	private DsmService dsmService;
-
-	@GetMapping("/ds/countries")
-	public List<Country> listCountries() throws DsmException {
-		return dsmService.listCountries();
-	}
-
-	@PostMapping("/ds/searchdetails/{page}/{size}")
-	public DatasourceDetailResponse searchDsDetails(
-		@RequestParam final RequestSort requestSortBy,
-		@RequestParam final RequestSortOrder order,
-		@RequestBody final RequestFilter requestFilter,
-		@PathVariable final int page,
-		@PathVariable final int size) throws DsmException {
-		final StopWatch stop = StopWatch.createStarted();
-
-		final DatasourceDetailResponse rsp = dsmService.searchDetails(requestSortBy, order, requestFilter, page, size);
-		return prepareResponse(page, size, stop, rsp);
-	}
-
-	@GetMapping("/ds/aggregationhistory/{dsId}")
-	public AggregationHistoryResponse aggregationHistory(@PathVariable final String dsId) throws DsmException {
-		final StopWatch stop = StopWatch.createStarted();
-		final List<AggregationInfo> history = wfLoggerClient.getAggregationHistory(dsId);
-		final AggregationHistoryResponse rsp = new AggregationHistoryResponse(history);
-		rsp.setHeader(ResponseUtils.header(history.size()));
-
-		return prepareResponse(0, rsp.getAggregationInfo().size(), stop, rsp);
-	}
-
-	@PostMapping("/ds/searchsnippet/{page}/{size}")
-	public DatasourceSnippetResponse searchSnippet(
-		@RequestParam final RequestSort requestSortBy,
-		@RequestParam final RequestSortOrder order,
-		@RequestBody final RequestFilter requestFilter,
-		@PathVariable final int page,
-		@PathVariable final int size) throws DsmException {
-
-		final StopWatch stop = StopWatch.createStarted();
-		final DatasourceSnippetResponse rsp = dsmService.searchSnippet(requestSortBy, order, requestFilter, page, size);
-		return prepareResponse(page, size, stop, rsp);
-	}
-
-	@PostMapping("/ds/searchregistered/{page}/{size}")
-	public DatasourceSnippetResponse searchRegistered(
-		@RequestParam final RequestSort requestSortBy,
-		@RequestParam final RequestSortOrder order,
-		@RequestBody final RequestFilter requestFilter,
-		@PathVariable final int page,
-		@PathVariable final int size) throws DsmException {
-		final StopWatch stop = StopWatch.createStarted();
-
-		final Page<Datasource> dsPage = dsmService.searchRegistered(requestSortBy, order, requestFilter, page, size);
-		final DatasourceSnippetResponse rsp =
-			ResponseUtils.snippetResponse(dsPage.map(DsmMappingUtils::asSnippetExtended).getContent(), dsPage.getTotalElements());
-		return prepareResponse(page, size, stop, rsp);
-	}
-
-	@GetMapping("/ds/recentregistered/{size}")
-	public SimpleResponse<RegisteredDatasourceInfo> recentRegistered(@PathVariable final int size) throws Throwable {
-		final StopWatch stop = StopWatch.createStarted();
-		final SimpleResponse<RegisteredDatasourceInfo> rsp = dsmService.searchRecentRegistered(size);
-		return prepareResponse(1, size, stop, rsp);
-	}
-
-	@GetMapping("/ds/countregistered")
-	public Long countRegistered(@RequestParam final String fromDate,
-		@RequestParam(required = false) final String typologyFilter) throws Throwable {
-		return dsmService.countRegisteredAfter(fromDate, typologyFilter);
-	}
-
-	@GetMapping("/ds/api/{dsId}")
-	public ApiDetailsResponse getApi(
-		@PathVariable final String dsId) throws DsmException {
-
-		final StopWatch stop = StopWatch.createStarted();
-
-		final Datasource ds = dsmService.findDs(dsId);
-		final List<Api> apis = dsmService.findApis(dsId);
-		final List<ApiDetails> api = apis.stream()
-			.map(DsmMappingUtils::asDetails)
-			.map(a -> a.setEoscDatasourceType(ds.getEoscDatasourceType()))
-			.map(a -> a.setTypology(ds.getTypology()))
-			.collect(Collectors.toList());
-		final ApiDetailsResponse rsp = ResponseUtils.apiResponse(api, api.size());
-
-		return prepareResponse(0, rsp.getApi().size(), stop, rsp);
-
-	}
-
-	@PostMapping("/api/baseurl/{page}/{size}")
-	public List<String> searchBaseUrls(
-		@RequestBody final RequestFilter requestFilter,
-		@PathVariable final int page,
-		@PathVariable final int size) throws DsmException {
-
-		return dsmService.findApiBaseURLs(requestFilter, page, size);
-	}
-
-	@DeleteMapping("/ds/api/{apiId}")
-	public void deleteApi(@PathVariable final String apiId) throws DsmForbiddenException, DsmNotFoundException {
-		dsmService.deleteApi(null, apiId);
-	}
-
-	@PostMapping("/ds/manage")
-	public void setManaged(
-		@RequestParam final String id,
-		@RequestParam final boolean managed) throws DsmException {
-
-		log.info(String.format("updated ds '%s' managed with '%s'", id, managed));
-		dsmService.setManaged(id, managed);
-	}
-
-	@GetMapping("/ds/managed/{id}")
-	public boolean isManaged(@PathVariable final String id) throws DsmException {
-		return dsmService.isManaged(id);
-	}
-
-	@PostMapping("/ds/add")
-	public void saveDs(@Valid @RequestBody final DatasourceDetails datasource) throws DsmException {
-
-		if (dsmService.existDs(datasource.getId())) { // TODO further check that the DS doesn't have any API
-			throw new DsmException(String.format("cannot register, datasource already defined '%s'", datasource.getId()));
-		}
-		dsmService.saveDs(asDbEntry(datasource));
-		log.info("DS saved, " + datasource.getId());
-	}
-
-	@PostMapping("/ds/addWithApis")
-	public void saveDsWithApis(@Valid @RequestBody final DatasourceDetailsWithApis d) throws DsmException {
-		if (d.getDatasource() == null) { throw new DsmException("Datasource field is null"); }
-		if (dsmService.existDs(d.getDatasource().getId())) { // TODO further check that the DS doesn't have any API
-			throw new DsmException(String.format("cannot register, datasource already defined '%s'", d.getDatasource().getId()));
-		}
-		dsmService.addDsAndApis(asDbEntry(d.getDatasource()), d.getApis());
-	}
-
-	@PostMapping("/ds/update")
-	public void updateDatasource(@RequestBody final DatasourceDetailsUpdate d) throws DsmException, DsmNotFoundException {
-
-		// initialize with current values from DB
-		final Datasource ds = dsmService.findDs(d.getId());
-
-		if (ds == null) { throw new DsmNotFoundException(String.format("ds '%s' does not exist", d.getId())); }
-
-		final Datasource update = asDbEntry(d);
-		if (ds.getIdentities() != null) {
-			final Set<Identity> identities = new HashSet<>(
-				Stream.of(update.getIdentities(), ds.getIdentities())
-					.flatMap(Collection::stream)
-					.collect(Collectors.toMap(i -> i.getIssuertype() + i.getPid(), Function.identity(), (i1, i2) -> i1))
-					.values());
-			copyNonNullProperties(update, ds);
-			ds.setIdentities(identities);
-		} else {
-			copyNonNullProperties(update, ds);
-		}
-
-		dsmService.saveDs(ds);
-	}
-
-	@PostMapping("/ds/api/baseurl")
-	@ApiResponses(value = {
-		@ApiResponse(responseCode = "200", description = "OK"),
-		@ApiResponse(responseCode = "500", description = "unexpected error")
-	})
-	public void updateBaseUrl(
-		@RequestParam final String dsId,
-		@RequestParam final String apiId,
-		@RequestParam final String baseUrl) throws DsmException {
-
-		log.info(String.format("updated api '%s' baseurl with '%s'", apiId, baseUrl));
-
-		dsmService.updateApiBaseUrl(apiId, baseUrl);
-	}
-
-	@PostMapping("/ds/api/compliance")
-	public void updateCompliance(
-		@RequestParam final String dsId,
-		@RequestParam final String apiId,
-		@RequestParam final String compliance,
-		@RequestParam(required = false, defaultValue = "false") final boolean override) throws DsmException {
-
-		log.info(String.format("updated api '%s' compliance with '%s'", apiId, compliance));
-
-		dsmService.updateCompliance(null, apiId, compliance, override);
-	}
-
-	@PostMapping("/ds/api/oaiset")
-	public void updateOaiSetl(
-		@RequestParam final String dsId,
-		@RequestParam final String apiId,
-		@RequestParam final String oaiSet) throws DsmException, DsmNotFoundException {
-
-		dsmService.upsertApiOaiSet(apiId, oaiSet);
-	}
-
-	@PostMapping("/ds/api/add")
-	public void addApi(@RequestBody final ApiDetails api) throws DsmException {
-		if (StringUtils.isBlank(api.getDatasource())) { throw new DsmException("missing datasource id"); }
-		if (StringUtils.isBlank(api.getId())) {
-			api.setId(createId(api));
-			log.info(String.format("missing api id, created '%s'", api.getId()));
-		}
-		dsmService.addApi(asDbEntry(api));
-		log.info("API saved, id: " + api.getId());
-	}
-
-}

apps/dnet-is-application/src/main/java/eu/dnetlib/dsm/DsmApiControllerV2.java

@@ -1,51 +0,0 @@
-package eu.dnetlib.dsm;
-
-import java.util.List;
-
-import org.apache.commons.lang3.time.StopWatch;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
-import org.springframework.web.bind.annotation.GetMapping;
-import org.springframework.web.bind.annotation.PathVariable;
-import org.springframework.web.bind.annotation.RequestMapping;
-import org.springframework.web.bind.annotation.RequestParam;
-import org.springframework.web.bind.annotation.RestController;
-
-import eu.dnetlib.dsm.domain.SimpleDatasourceInfo;
-import eu.dnetlib.dsm.domain.SimpleResponse;
-import eu.dnetlib.dsm.utils.ResponseUtils;
-import io.swagger.v3.oas.annotations.tags.Tag;
-
-@RestController
-@RequestMapping("/api/dsm/2.0")
-@ConditionalOnProperty(value = "openaire.api.enable.dsm", havingValue = "true")
-@Tag(name = "OpenAIRE DSM API (version 2.0)", description = "the OpenAIRE Datasource Manager API 2.0")
-public class DsmApiControllerV2 extends AbstractDsmController {
-
-	@Autowired
-	private DsmService dsmService;
-
-	@GetMapping("/recentregistered/{size}")
-	public SimpleResponse<SimpleDatasourceInfo> recentRegisteredV2(@PathVariable final int size) throws Throwable {
-		final StopWatch stop = StopWatch.createStarted();
-		final SimpleResponse<SimpleDatasourceInfo> rsp = dsmService.searchRecentRegisteredV2(size);
-		return prepareResponse(1, size, stop, rsp);
-	}
-
-	@GetMapping("/countfirstcollect")
-	public Long countFirstCollectAfter(@RequestParam final String fromDate,
-		@RequestParam(required = false) final String typologyFilter) throws Throwable {
-		return dsmService.countFirstCollect(fromDate, typologyFilter);
-	}
-
-	@GetMapping("/firstCollected")
-	public SimpleResponse<SimpleDatasourceInfo> firstCollectedAfter(@RequestParam final String fromDate,
-		@RequestParam(required = false) final String typologyFilter) throws Throwable {
-		final StopWatch stop = StopWatch.createStarted();
-		final List<SimpleDatasourceInfo> list = dsmService.findFirstCollectedAfter(fromDate, typologyFilter);
-		final SimpleResponse<SimpleDatasourceInfo> rsp = ResponseUtils.simpleResponse(list);
-
-		return prepareResponse(1, list.size(), stop, rsp);
-	}
-
-}

apps/dnet-is-application/src/main/java/eu/dnetlib/is/context/AbstractContextController.java

@@ -1,21 +0,0 @@
-package eu.dnetlib.is.context;
-
-import java.util.List;
-
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.web.bind.annotation.GetMapping;
-
-import eu.dnetlib.common.controller.AbstractDnetController;
-import eu.dnetlib.is.model.context.Context;
-
-public class AbstractContextController extends AbstractDnetController {
-
-	@Autowired
-	protected ContextService contextService;
-
-	@GetMapping("/")
-	public List<Context> listContexts() {
-		return contextService.listContexts();
-	}
-
-}

apps/dnet-is-application/src/main/java/eu/dnetlib/is/context/ContextAjaxController.java

@@ -1,33 +0,0 @@
-package eu.dnetlib.is.context;
-
-import org.springframework.web.bind.annotation.GetMapping;
-import org.springframework.web.bind.annotation.PathVariable;
-import org.springframework.web.bind.annotation.RequestMapping;
-import org.springframework.web.bind.annotation.RestController;
-
-import eu.dnetlib.errors.InformationServiceException;
-import eu.dnetlib.is.model.context.Category;
-import eu.dnetlib.is.model.context.Context;
-import eu.dnetlib.is.model.context.CtxChildInfo;
-
-@RestController
-@RequestMapping("/ajax/contexts")
-public class ContextAjaxController extends AbstractContextController {
-
-	@GetMapping("/{ctxId}")
-	public Context getContext(@PathVariable final String ctxId) throws InformationServiceException {
-		return contextService.getContext(ctxId);
-	}
-
-	@GetMapping("/{parent}/categories")
-	public Iterable<Category> listCategories(@PathVariable final String parent) {
-		return contextService.listCategories(parent);
-	}
-
-	@GetMapping("/{level}/{parent}/concepts")
-	public Iterable<? extends CtxChildInfo> listCategories(@PathVariable final int level, @PathVariable final String parent)
-		throws InformationServiceException {
-		return contextService.listConcepts(level, parent);
-	}
-
-}

apps/dnet-is-application/src/main/java/eu/dnetlib/is/context/ContextApiController.java

@@ -1,21 +0,0 @@
-package eu.dnetlib.is.context;
-
-import org.springframework.web.bind.annotation.GetMapping;
-import org.springframework.web.bind.annotation.PathVariable;
-import org.springframework.web.bind.annotation.RequestMapping;
-import org.springframework.web.bind.annotation.RestController;
-
-import com.fasterxml.jackson.databind.node.ObjectNode;
-
-import eu.dnetlib.errors.InformationServiceException;
-
-@RestController
-@RequestMapping("/api/contexts")
-public class ContextApiController extends AbstractContextController {
-
-	@GetMapping("/{ctxId}")
-	public ObjectNode getContextFull(@PathVariable final String ctxId) throws InformationServiceException {
-		return contextService.getContextFull(ctxId);
-	}
-
-}

apps/dnet-is-application/src/main/java/eu/dnetlib/is/email/EmailTemplateController.java

@@ -1,41 +0,0 @@
-package eu.dnetlib.is.email;
-
-import java.util.List;
-
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.web.bind.annotation.DeleteMapping;
-import org.springframework.web.bind.annotation.GetMapping;
-import org.springframework.web.bind.annotation.PathVariable;
-import org.springframework.web.bind.annotation.PostMapping;
-import org.springframework.web.bind.annotation.RequestBody;
-import org.springframework.web.bind.annotation.RequestMapping;
-import org.springframework.web.bind.annotation.RestController;
-
-import eu.dnetlib.common.controller.AbstractDnetController;
-import eu.dnetlib.common.model.EmailTemplate;
-import eu.dnetlib.notifications.mail.EmailService;
-
-@RestController
-@RequestMapping("/ajax/templates/email")
-public class EmailTemplateController extends AbstractDnetController {
-
-	@Autowired
-	protected EmailService emailService;
-
-	@GetMapping("/")
-	public List<EmailTemplate> listEmailTemplates() {
-		return emailService.listEmailTemplates();
-	}
-
-	@PostMapping("/")
-	public List<EmailTemplate> saveEmailTemplate(@RequestBody final EmailTemplate email) {
-		emailService.saveEmailTemplate(email);
-		return emailService.listEmailTemplates();
-	}
-
-	@DeleteMapping("/{id}")
-	public List<EmailTemplate> deleteEmailTemplate(@PathVariable final String id) {
-		emailService.deleteEmailTemplate(id);
-		return emailService.listEmailTemplates();
-	}
-}

apps/dnet-is-application/src/main/java/eu/dnetlib/is/importer/ContextImporter.java

@@ -1,113 +0,0 @@
-package eu.dnetlib.is.importer;
-
-import java.io.IOException;
-
-import org.apache.commons.lang3.BooleanUtils;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.dom4j.DocumentException;
-import org.dom4j.DocumentHelper;
-import org.dom4j.Node;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.stereotype.Service;
-
-import eu.dnetlib.is.context.repository.CategoryRepository;
-import eu.dnetlib.is.context.repository.ConceptLevel0Repository;
-import eu.dnetlib.is.context.repository.ConceptLevel1Repository;
-import eu.dnetlib.is.context.repository.ConceptLevel2Repository;
-import eu.dnetlib.is.context.repository.ContextRepository;
-import eu.dnetlib.is.model.context.Category;
-import eu.dnetlib.is.model.context.ConceptLevel0;
-import eu.dnetlib.is.model.context.ConceptLevel1;
-import eu.dnetlib.is.model.context.ConceptLevel2;
-import eu.dnetlib.is.model.context.Context;
-import eu.dnetlib.is.model.context.CtxChildInfo;
-import eu.dnetlib.is.model.context.CtxInfo;
-import eu.dnetlib.is.model.context.Parameter;
-
-@Service
-public class ContextImporter {
-
-	@Autowired
-	private ContextRepository contextRepository;
-	@Autowired
-	private CategoryRepository categoryRepository;
-	@Autowired
-	private ConceptLevel0Repository conceptLevel0Repository;
-	@Autowired
-	private ConceptLevel1Repository conceptLevel1Repository;
-	@Autowired
-	private ConceptLevel2Repository conceptLevel2Repository;
-
-	private static final Log log = LogFactory.getLog(ContextImporter.class);
-
-	public void loadFromOldProfile(final String xml) throws DocumentException, IOException {
-		DocumentHelper.parseText(xml)
-			.selectNodes("//context")
-			.forEach(this::saveContext);
-	}
-
-	private void saveContext(final Node node) {
-		final Context ctx = new Context();
-		setCommonInfo(ctx, null, node);
-		ctx.setType(node.valueOf("@type"));
-		contextRepository.save(ctx);
-
-		log.info("Saved context: " + ctx.getId());
-
-		node.selectNodes("./category").forEach(n -> saveCategory(n, ctx.getId()));
-	}
-
-	private void saveCategory(final Node node, final String ctxId) {
-		final Category cat = new Category();
-		setCommonInfo(cat, ctxId, node);
-		categoryRepository.save(cat);
-
-		log.info("- Saved category: " + cat.getId());
-
-		node.selectNodes("./concept").forEach(n -> saveConceptLvl0(n, cat.getId()));
-	}
-
-	private void saveConceptLvl0(final Node node, final String parent) {
-		final ConceptLevel0 c0 = new ConceptLevel0();
-		setCommonInfo(c0, parent, node);
-		conceptLevel0Repository.save(c0);
-
-		log.info("-- Saved concept 0: " + c0.getId());
-
-		node.selectNodes("./concept").forEach(n -> saveConceptLvl1(n, c0.getId()));
-	}
-
-	private void saveConceptLvl1(final Node node, final String parent) {
-		final ConceptLevel1 c1 = new ConceptLevel1();
-		setCommonInfo(c1, parent, node);
-		conceptLevel1Repository.save(c1);
-
-		log.info("--- Saved concept 1: " + c1.getId());
-
-		node.selectNodes("./concept").forEach(n -> saveConceptLvl2(n, c1.getId()));
-	}
-
-	private void saveConceptLvl2(final Node node, final String parent) {
-		final ConceptLevel2 c2 = new ConceptLevel2();
-		setCommonInfo(c2, parent, node);
-		conceptLevel2Repository.save(c2);
-
-		log.info("---- Saved concept 2: " + c2.getId());
-	}
-
-	private void setCommonInfo(final CtxInfo o, final String parent, final Node n) {
-		o.setId(n.valueOf("@id"));
-		o.setLabel(n.valueOf("@label"));
-		if (o instanceof CtxChildInfo) {
-			((CtxChildInfo) o).setParent(parent);
-			((CtxChildInfo) o).setClaim(BooleanUtils.toBoolean(n.valueOf("@claim")));
-		}
-		o.setParameters(n.selectNodes("./param")
-			.stream()
-			.map(np -> new Parameter(np.valueOf("@name"), np.getText()))
-			.sorted()
-			.toArray(Parameter[]::new));
-
-	}
-}

apps/dnet-is-application/src/main/java/eu/dnetlib/is/importer/ImporterController.java

@@ -1,80 +0,0 @@
-package eu.dnetlib.is.importer;
-
-import java.io.IOException;
-import java.nio.charset.StandardCharsets;
-import java.util.Arrays;
-import java.util.List;
-
-import javax.servlet.http.HttpServletRequest;
-
-import org.apache.commons.io.IOUtils;
-import org.dom4j.DocumentException;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.http.MediaType;
-import org.springframework.web.bind.annotation.GetMapping;
-import org.springframework.web.bind.annotation.PostMapping;
-import org.springframework.web.bind.annotation.RequestMapping;
-import org.springframework.web.bind.annotation.RequestParam;
-import org.springframework.web.bind.annotation.RestController;
-
-import eu.dnetlib.common.controller.AbstractDnetController;
-import eu.dnetlib.is.model.vocabulary.Vocabulary;
-
-@RestController
-@RequestMapping("/api/import")
-public class ImporterController extends AbstractDnetController {
-
-	// EXAMPLE:
-	// find ./VocabularyDSResourceType/ -name "*.xml" -exec curl -X POST "http://localhost:8280/api/import/vocabulary" -H "accept: */*" -H
-	// "Content-Type: text/plain" --data-binary @{} \;
-	// find ./DedupConfigurationDSResources/ -name "*.xml" -exec curl -X POST "http://localhost:8280/api/import/resource" -H "accept: */*"
-	// -H "Content-Type: text/plain" --data-binary @{} \;
-
-	@Autowired
-	private ContextImporter contextImporter;
-
-	@Autowired
-	private OldProfilesImporter oldProfilesImporter;
-
-	@Autowired
-	private WfHistoryImporter wfHistoryImporter;
-
-	@PostMapping(value = "/context", consumes = {
-		MediaType.TEXT_PLAIN_VALUE, MediaType.APPLICATION_XML_VALUE, MediaType.APPLICATION_JSON_VALUE
-	})
-	public List<String> importContext(final HttpServletRequest request) throws DocumentException, IOException {
-
-		final String xml = IOUtils.toString(request.getInputStream(), StandardCharsets.UTF_8);
-
-		contextImporter.loadFromOldProfile(xml);
-
-		return Arrays.asList("Done.");
-
-	}
-
-	@PostMapping(value = "/resource", consumes = {
-		MediaType.TEXT_PLAIN_VALUE, MediaType.APPLICATION_XML_VALUE, MediaType.APPLICATION_JSON_VALUE
-	})
-	public String importResource(final HttpServletRequest request) throws Exception {
-		final String xml = IOUtils.toString(request.getInputStream(), StandardCharsets.UTF_8);
-
-		return oldProfilesImporter.importSimpleResource(xml);
-	}
-
-	@PostMapping(value = "/vocabulary", consumes = {
-		MediaType.TEXT_PLAIN_VALUE, MediaType.APPLICATION_XML_VALUE, MediaType.APPLICATION_JSON_VALUE
-	})
-	public Vocabulary importVocabulary(final HttpServletRequest request) throws Exception, IOException {
-		final String xml = IOUtils.toString(request.getInputStream(), StandardCharsets.UTF_8);
-		return oldProfilesImporter.importVocabulary(xml);
-	}
-
-	@GetMapping(value = "/wf_logs")
-	public List<String> importWfLogs(@RequestParam final String path) throws Exception {
-		// mongoexport -d dnet_logs -c wf_logs --jsonArray -o /tmp/mongodump.json
-
-		wfHistoryImporter.load(path);
-
-		return Arrays.asList("Done.");
-	}
-}

apps/dnet-is-application/src/main/java/eu/dnetlib/is/importer/OldProfilesImporter.java

@@ -1,156 +0,0 @@
-package eu.dnetlib.is.importer;
-
-import java.time.LocalDateTime;
-
-import javax.transaction.Transactional;
-
-import org.apache.commons.lang3.StringUtils;
-import org.dom4j.Attribute;
-import org.dom4j.Document;
-import org.dom4j.DocumentException;
-import org.dom4j.DocumentHelper;
-import org.dom4j.Element;
-import org.dom4j.Node;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.stereotype.Service;
-
-import eu.dnetlib.errors.InformationServiceException;
-import eu.dnetlib.is.model.resource.SimpleResource;
-import eu.dnetlib.is.model.vocabulary.Synonym;
-import eu.dnetlib.is.model.vocabulary.Vocabulary;
-import eu.dnetlib.is.model.vocabulary.VocabularyTerm;
-import eu.dnetlib.is.resource.ResourceValidator;
-import eu.dnetlib.is.resource.repository.SimpleResourceRepository;
-import eu.dnetlib.is.vocabulary.repository.VocabularyRepository;
-import eu.dnetlib.is.vocabulary.repository.VocabularyTermRepository;
-import eu.dnetlib.utils.XmlIndenter;
-
-@Service
-public class OldProfilesImporter {
-
-	@Autowired
-	private SimpleResourceRepository simpleResourceRepository;
-
-	@Autowired
-	private VocabularyRepository vocabularyRepository;
-
-	@Autowired
-	private VocabularyTermRepository vocabularyTermRepository;
-
-	@Autowired
-	private ResourceValidator resourceValidator;
-
-	@Transactional
-	public String importSimpleResource(final String xml) throws InformationServiceException {
-
-		try {
-			final Document doc = DocumentHelper.parseText(xml);
-
-			final String id = StringUtils.substringBefore(doc.valueOf("//RESOURCE_IDENTIFIER/@value"), "_");
-			final LocalDateTime now = LocalDateTime.now();
-
-			final SimpleResource res = new SimpleResource();
-			res.setId(id);
-			res.setCreationDate(now);
-			res.setModificationDate(now);
-
-			String resContent;
-			switch (doc.valueOf("//RESOURCE_TYPE/@value")) {
-			case "CleanerDSResourceType":
-				res.setType("cleaning_rule");
-				res.setName(doc.valueOf("//CLEANER_NAME"));
-				res.setDescription(doc.valueOf("//CLEANER_DESCRIPTION"));
-
-				for (final Node n : doc.selectNodes("//RULE")) {
-					final Element e = (Element) n;
-					e.addAttribute("vocabulary", e.valueOf("@vocabularies"));
-					for (final Attribute attr : e.attributes()) {
-						if (attr.getName() == "vocabularies" || attr.getName() == "groovy") {
-							attr.detach();
-						}
-					}
-				}
-
-				resContent = XmlIndenter.indent(doc.selectSingleNode("//CLEANER_RULES"));
-				break;
-			case "TransformationRuleDSResourceType":
-				res.setName(doc.valueOf("//SCRIPT/TITLE"));
-				res.setDescription("");
-				if (doc.selectNodes("//*[local-name() = 'stylesheet']").size() > 0) {
-					res.setType("transformation_rule_xslt");
-					resContent = XmlIndenter.indent(doc.selectSingleNode("//*[local-name() = 'stylesheet']"));
-				} else {
-					final String code = doc.valueOf("//SCRIPT/CODE").trim();
-
-					try {
-						final Document xsltDoc = DocumentHelper.parseText(code);
-						res.setType("transformation_rule_xslt");
-						resContent = XmlIndenter.indent(xsltDoc);
-					} catch (final DocumentException e) {
-						res.setType("transformation_rule_legacy");
-						resContent = code;
-					}
-				}
-				break;
-			case "HadoopJobConfigurationDSResourceType":
-				res.setType("hadoop_job_configuration");
-				res.setName(doc.valueOf("//HADOOP_JOB/@name"));
-				res.setDescription(doc.valueOf("//HADOOP_JOB/DESCRIPTION"));
-				resContent = XmlIndenter.indent(doc.selectSingleNode("//HADOOP_JOB"));
-				break;
-			case "DedupConfigurationDSResourceType":
-				res.setType("dedup_configuration");
-				res.setName(doc.valueOf("//DESCRIPTION"));
-				res.setDescription(doc.valueOf("//DESCRIPTION"));
-				resContent = doc.valueOf("//DEDUPLICATION");
-				break;
-			default:
-				throw new InformationServiceException("Invalid resource type: " + doc.valueOf("//RESOURCE_TYPE/@value"));
-			}
-
-			resourceValidator.validate(res.getType(), resContent.trim());
-
-			simpleResourceRepository.save(res);
-			simpleResourceRepository.setContentById(id, resContent.trim());
-
-			return res.getId();
-		} catch (final Exception e) {
-			throw new InformationServiceException("Error parsing file", e);
-		}
-	}
-
-	@Transactional
-	public Vocabulary importVocabulary(final String xml) throws Exception {
-		final Document doc = DocumentHelper.parseText(xml);
-
-		final Vocabulary voc = new Vocabulary();
-		final String vocId = doc.valueOf("//VOCABULARY_NAME/@code");
-		final String vocName = doc.valueOf("//VOCABULARY_NAME");
-		final String vocDesc = doc.valueOf("//VOCABULARY_DESCRIPTION");
-
-		voc.setId(vocId);
-		voc.setName(vocName);
-		voc.setDescription(vocDesc);
-
-		vocabularyRepository.save(voc);
-
-		for (final Node n : doc.selectNodes("//TERM")) {
-			final VocabularyTerm term = new VocabularyTerm();
-			term.setVocabulary(vocId);
-			term.setCode(n.valueOf("@code"));
-			term.setName(n.valueOf("@english_name"));
-			term.setEncoding(n.valueOf("@encoding"));
-			term.setSynonyms(n.selectNodes(".//SYNONYM")
-				.stream()
-				.map(ns -> new Synonym(ns.valueOf("@term"), ns.valueOf("@encoding")))
-				.sorted()
-				.distinct()
-				.toArray(Synonym[]::new));
-
-			vocabularyTermRepository.save(term);
-		}
-
-		return voc;
-	}
-
-}

apps/dnet-is-application/src/main/java/eu/dnetlib/is/importer/WfHistoryImporter.java

@@ -1,85 +0,0 @@
-package eu.dnetlib.is.importer;
-
-import java.io.File;
-import java.time.Instant;
-import java.time.LocalDateTime;
-import java.util.Iterator;
-import java.util.LinkedHashMap;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.TimeZone;
-
-import org.apache.commons.lang3.BooleanUtils;
-import org.apache.commons.lang3.math.NumberUtils;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.stereotype.Service;
-
-import com.fasterxml.jackson.databind.JsonNode;
-import com.fasterxml.jackson.databind.ObjectMapper;
-
-import eu.dnetlib.manager.history.model.WfHistoryEntry;
-import eu.dnetlib.manager.history.repository.WfHistoryEntryRepository;
-
-@Service
-public class WfHistoryImporter {
-
-	private static final Log log = LogFactory.getLog(WfHistoryImporter.class);
-
-	@Autowired
-	private WfHistoryEntryRepository wfHistoryEntryRepository;
-
-	public void load(final String path) throws Exception {
-		final ObjectMapper mapper = new ObjectMapper();
-
-		final JsonNode rootNode = mapper.readTree(new File(path));
-
-		rootNode.forEach(wf -> saveWf(wf));
-	}
-
-	private void saveWf(final JsonNode node) {
-
-		final WfHistoryEntry wf = new WfHistoryEntry();
-		wf.setProcessId(node.get("system:processId").asText());
-		wf.setName(node.get("system:wfName").asText());
-		wf.setFamily(node.get("system:profileFamily").asText());
-
-		if (node.has("dataprovider:id")) {
-			wf.setDsId(node.get("dataprovider:id").asText());
-		}
-		if (node.has("dataprovider:name")) {
-			wf.setDsName(node.get("dataprovider:name").asText());
-		}
-		if (node.has("dataprovider:interface")) {
-			wf.setDsApi(node.get("dataprovider:interface").asText());
-		}
-
-		wf.setStartDate(LocalDateTime
-			.ofInstant(Instant.ofEpochMilli(NumberUtils.toLong(node.get("system:startDate").asText())), TimeZone.getDefault().toZoneId()));
-
-		wf.setEndDate(LocalDateTime.ofInstant(Instant.ofEpochMilli(NumberUtils.toLong(node.get("system:endDate").asText())), TimeZone.getDefault().toZoneId()));
-
-		if (BooleanUtils.toBoolean(node.get("system:isCompletedSuccessfully").asText())) {
-			wf.setStatus("success");
-		} else {
-			wf.setStatus("failure");
-		}
-
-		final Map<String, String> details = new LinkedHashMap<>();
-		final Iterator<Entry<String, JsonNode>> fields = node.fields();
-		while (fields.hasNext()) {
-			final Entry<String, JsonNode> f = fields.next();
-			if (f.getValue().isValueNode()) {
-				details.put(f.getKey(), f.getValue().asText());
-			}
-		}
-
-		wf.setDetails(details);
-
-		wfHistoryEntryRepository.save(wf);
-
-		log.info("Wf saved with id: " + wf.getProcessId());
-
-	}
-}