added a new working table to import the suggestions

Reviewed-on: #20

apps/bioschemas-api/pom.xml

@@ -3,7 +3,7 @@
     <parent>
         <groupId>eu.dnetlib.dhp</groupId>
         <artifactId>apps</artifactId>
-        <version>3.4.5-SNAPSHOT</version>
+        <version>3.5.5-SNAPSHOT</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
 

apps/dhp-broker-application/pom.xml

@@ -4,7 +4,7 @@
 	<parent>
 		<groupId>eu.dnetlib.dhp</groupId>
 		<artifactId>apps</artifactId>
-		<version>3.4.5-SNAPSHOT</version>
+		<version>3.5.5-SNAPSHOT</version>
 		<relativePath>../pom.xml</relativePath>
 	</parent>
 

apps/dhp-broker-public-application/pom.xml

@@ -4,7 +4,7 @@
 	<parent>
 		<groupId>eu.dnetlib.dhp</groupId>
 		<artifactId>apps</artifactId>
-		<version>3.4.5-SNAPSHOT</version>
+		<version>3.5.5-SNAPSHOT</version>
 		<relativePath>../pom.xml</relativePath>
 	</parent>
 

apps/dhp-mdstore-manager/pom.xml

@@ -4,7 +4,7 @@
 	<parent>
 		<groupId>eu.dnetlib.dhp</groupId>
 		<artifactId>apps</artifactId>
-		<version>3.4.5-SNAPSHOT</version>
+		<version>3.5.5-SNAPSHOT</version>
 		<relativePath>../pom.xml</relativePath>
 	</parent>
 

apps/dnet-exporter-api/pom.xml

@@ -3,7 +3,7 @@
 	<parent>
 		<groupId>eu.dnetlib.dhp</groupId>
 		<artifactId>apps</artifactId>
-		<version>3.4.5-SNAPSHOT</version>
+		<version>3.5.5-SNAPSHOT</version>
 		<relativePath>../</relativePath>
 	</parent>
 

apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/importer/CommunityImporterService.java

@@ -38,6 +38,7 @@ import eu.dnetlib.openaire.exporter.model.community.CommunityContentprovider;
 import eu.dnetlib.openaire.exporter.model.community.CommunityDetails;
 import eu.dnetlib.openaire.exporter.model.community.CommunityMembershipType;
 import eu.dnetlib.openaire.exporter.model.community.CommunityOrganization;
+import eu.dnetlib.openaire.exporter.model.community.CommunityPlanType;
 import eu.dnetlib.openaire.exporter.model.community.CommunityProject;
 import eu.dnetlib.openaire.exporter.model.community.CommunityStatus;
 import eu.dnetlib.openaire.exporter.model.community.CommunityType;
@@ -242,6 +243,7 @@ public class CommunityImporterService {
 
 		details.setId(c.getId());
 		details.setShortName(c.getLabel());
+		details.setDisplayShortName(c.getLabel());
 		details.setLastUpdateDate(CommunityMappingUtils.asLocalDateTime(c.getLastUpdateDate()));
 		details.setCreationDate(CommunityMappingUtils.asLocalDateTime(c.getCreationDate()));
 		details.setQueryId(c.getId() + PIPE_SEPARATOR + c.getLabel());
@@ -259,6 +261,8 @@ public class CommunityImporterService {
 		}
 
 		details.setName(StringUtils.firstNonBlank(asCsv(CSUMMARY_NAME, c.getParams()), c.getLabel()));
+		details.setDisplayName(StringUtils.firstNonBlank(asCsv(CSUMMARY_NAME, c.getParams()), c.getLabel()));
+
 		details.setZenodoCommunity(asCsv(CSUMMARY_ZENODOC, c.getParams()));
 		details.setSubjects(splitValues(asValues(CPROFILE_SUBJECT, c.getParams()), CSV_DELIMITER));
 		details.setFos(splitValues(asValues(CPROFILE_FOS, c.getParams()), CSV_DELIMITER));
@@ -268,7 +272,7 @@ public class CommunityImporterService {
 		// In the map the string is the serialization of the json representing the selection criteria so it is a valid json
 		details.setRemoveConstraints(SelectionCriteria.fromJson(asCsv(CPROFILE_REMOVE_CONSTRAINT, c.getParams())));
 		details.setSuggestedAcknowledgements(splitValues(asValues(CPROFILE_SUGGESTED_ACKNOWLEDGEMENT, c.getParams()), CSV_DELIMITER));
-		details.setPlan(null);
+		details.setPlan(CommunityPlanType.Default);
 		try {
 			details.setCreationDate(CommunityMappingUtils.asLocalDateTime(asCsv(CPROFILE_CREATIONDATE, c.getParams())));
 		} catch (final Exception e) {

apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/model/DbCommunity.java

@@ -27,6 +27,7 @@ import eu.dnetlib.openaire.community.utils.CommunityClaimTypeConverter;
 import eu.dnetlib.openaire.community.utils.CommunityMembershipTypeConverter;
 import eu.dnetlib.openaire.exporter.model.community.CommunityClaimType;
 import eu.dnetlib.openaire.exporter.model.community.CommunityMembershipType;
+import eu.dnetlib.openaire.exporter.model.community.CommunityPlanType;
 import eu.dnetlib.openaire.exporter.model.community.CommunityStatus;
 import eu.dnetlib.openaire.exporter.model.community.CommunityType;
 import eu.dnetlib.openaire.exporter.model.community.selectioncriteria.SelectionCriteria;
@@ -34,9 +35,9 @@ import eu.dnetlib.openaire.exporter.model.community.selectioncriteria.SelectionC
 @Entity
 @Table(name = "communities")
 @TypeDefs({
-	@TypeDef(name = "string-array", typeClass = StringArrayType.class),
-	@TypeDef(name = "json", typeClass = JsonStringType.class),
-	@TypeDef(name = "jsonb", typeClass = JsonBinaryType.class)
+		@TypeDef(name = "string-array", typeClass = StringArrayType.class),
+		@TypeDef(name = "json", typeClass = JsonStringType.class),
+		@TypeDef(name = "jsonb", typeClass = JsonBinaryType.class)
 })
 public class DbCommunity implements Serializable {
 
@@ -52,6 +53,12 @@ public class DbCommunity implements Serializable {
 	@Column(name = "shortname")
 	private String shortName;
 
+	@Column(name = "displayname")
+	private String displayName;
+
+	@Column(name = "displayshortname")
+	private String displayShortName;
+
 	@Column(name = "description")
 	private String description;
 
@@ -115,7 +122,8 @@ public class DbCommunity implements Serializable {
 	private String[] suggestedAcknowledgements;
 
 	@Column(name = "plan")
-	private String plan;
+	@Enumerated(EnumType.STRING)
+	private CommunityPlanType plan;
 
 	public String getId() {
 		return id;
@@ -141,6 +149,22 @@ public class DbCommunity implements Serializable {
 		this.shortName = shortName;
 	}
 
+	public String getDisplayName() {
+		return displayName;
+	}
+
+	public void setDisplayName(final String displayName) {
+		this.displayName = displayName;
+	}
+
+	public String getDisplayShortName() {
+		return displayShortName;
+	}
+
+	public void setDisplayShortName(final String displayShortName) {
+		this.displayShortName = displayShortName;
+	}
+
 	public String getDescription() {
 		return description;
 	}
@@ -269,11 +293,11 @@ public class DbCommunity implements Serializable {
 		this.suggestedAcknowledgements = suggestedAcknowledgements;
 	}
 
-	public String getPlan() {
+	public CommunityPlanType getPlan() {
 		return plan;
 	}
 
-	public void setPlan(final String plan) {
+	public void setPlan(final CommunityPlanType plan) {
 		this.plan = plan;
 	}
 

apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/utils/CommunityMappingUtils.java

@@ -50,6 +50,8 @@ public class CommunityMappingUtils {
 		c.setId(details.getId());
 		c.setName(details.getName());
 		c.setShortName(details.getShortName());
+		c.setDisplayName(details.getDisplayName());
+		c.setDisplayShortName(details.getDisplayShortName());
 		c.setDescription(details.getDescription());
 		c.setStatus(details.getStatus());
 		c.setLogoUrl(details.getLogoUrl());
@@ -77,6 +79,12 @@ public class CommunityMappingUtils {
 		if (StringUtils.isNotBlank(details.getShortName())) {
 			c.setShortName(details.getShortName());
 		}
+		if (StringUtils.isNotBlank(details.getDisplayName())) {
+			c.setDisplayName(details.getDisplayName());
+		}
+		if (StringUtils.isNotBlank(details.getDisplayShortName())) {
+			c.setDisplayShortName(details.getDisplayShortName());
+		}
 		if (StringUtils.isNotBlank(details.getDescription())) {
 			c.setDescription(details.getDescription());
 		}
@@ -138,8 +146,10 @@ public class CommunityMappingUtils {
 
 	private static void populateSummary(final CommunitySummary summary, final DbCommunity c) {
 		summary.setId(c.getId());
-		summary.setShortName(c.getShortName());
 		summary.setName(c.getName());
+		summary.setShortName(c.getShortName());
+		summary.setDisplayName(c.getDisplayName());
+		summary.setDisplayShortName(c.getDisplayShortName());
 		summary.setLastUpdateDate(c.getLastUpdateDate());
 		summary.setCreationDate(c.getCreationDate());
 		summary.setQueryId(c.getId() + PIPE_SEPARATOR + c.getShortName());

apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/MongoLoggerClientImpl.java

@@ -48,11 +48,7 @@ import eu.dnetlib.openaire.exporter.exceptions.DsmApiException;
 import eu.dnetlib.openaire.exporter.model.dsm.AggregationInfo;
 import eu.dnetlib.openaire.exporter.model.dsm.AggregationInfoV1;
 import eu.dnetlib.openaire.exporter.model.dsm.AggregationStage;
-import eu.dnetlib.openaire.exporter.model.dsm.CollectionInfoV1;
-import eu.dnetlib.openaire.exporter.model.dsm.CollectionInfoV2;
 import eu.dnetlib.openaire.exporter.model.dsm.CollectionMode;
-import eu.dnetlib.openaire.exporter.model.dsm.TransformationInfoV1;
-import eu.dnetlib.openaire.exporter.model.dsm.TransformationInfoV2;
 import eu.dnetlib.openaire.info.JdbcInfoDao;
 
 /**
@@ -75,16 +71,16 @@ public class MongoLoggerClientImpl implements MongoLoggerClient {
 
 	private final static String LOADTIME = "loadtime";
 	private final LoadingCache<String, Instant> loadingCache =
-		CacheBuilder.newBuilder().maximumSize(1).expireAfterWrite(60, TimeUnit.MINUTES).build(new CacheLoader<String, Instant>() {
+			CacheBuilder.newBuilder().maximumSize(1).expireAfterWrite(60, TimeUnit.MINUTES).build(new CacheLoader<String, Instant>() {
 
-			// The only cached value is associated to "loadtime"
-			@Override
-			public Instant load(final String key) {
-				final Instant loadTime = getLoadTime();
-				log.debug("found load time: " + loadTime.toString());
-				return loadTime;
-			}
-		});
+				// The only cached value is associated to "loadtime"
+				@Override
+				public Instant load(final String key) {
+					final Instant loadTime = getLoadTime();
+					log.debug("found load time: " + loadTime.toString());
+					return loadTime;
+				}
+			});
 
 	private static final Bson fields = getFields();
 
@@ -94,30 +90,64 @@ public class MongoLoggerClientImpl implements MongoLoggerClient {
 	@Cacheable("dsm-aggregationhistory-cache-v1")
 	@Deprecated
 	public List<AggregationInfoV1> getAggregationHistoryV1(final String dsId) throws DsmApiException {
-		return getAggregationHistory(dsId, queryForAggregationHistoryV1(dsId, "(collect|transform)"), getMapperV1());
+		return getAggregationHistoryV1(dsId, queryForAggregationHistoryV1(dsId, "(collect|transform)"), getMapperV1());
 	}
 
 	@Override
 	@Cacheable("dsm-aggregationhistory-cache-v2")
 	public List<AggregationInfo> getAggregationHistoryV2(final String dsId) throws DsmApiException {
-		return getAggregationHistory(dsId, queryForAggregationHistoryV2(dsId, "(collect|transform)"), getMapperV2());
+		return getAggregationHistoryV2(dsId, queryForAggregationHistoryV2(dsId, "(collect|transform)"), getMapperV2());
 	}
 
-	private <T extends AggregationInfo> List<T> getAggregationHistory(final String dsId,
-		final Bson queryForAggregationHistory,
-		final Function<Document, T> mapper) throws DsmApiException {
+	@Deprecated
+	private List<AggregationInfoV1> getAggregationHistoryV1(final String dsId,
+			final Bson queryForAggregationHistory,
+			final Function<Document, AggregationInfoV1> mapper) throws DsmApiException {
 		log.warn(String.format("getAggregationHistory(dsId = %s): not using cache", dsId));
 		final Datasource conf = config.getDatasource();
 		try {
 			final FindIterable<Document> aggregationDocs = getCollection().find(queryForAggregationHistory)
-				.projection(fields)
-				.limit(conf.getMongoQueryLimit())
-				.sort(dbo("system:startHumanDate", -1));
+					.projection(fields)
+					.limit(conf.getMongoQueryLimit())
+					.sort(dbo("system:startHumanDate", -1));
 
-			final List<T> aggregationInfos = Utils.stream(aggregationDocs.iterator())
-				.map(mapper)
-				.filter(ai -> ai.getNumberOfRecords() >= 0 && StringUtils.isNotBlank(ai.getDate()))
-				.collect(Collectors.toList());
+			final List<AggregationInfoV1> aggregationInfos = Utils.stream(aggregationDocs.iterator())
+					.map(mapper)
+					.filter(ai -> ai.getNumberOfRecords() >= 0 && StringUtils.isNotBlank(ai.getDate()))
+					.collect(Collectors.toList());
+
+			final Instant loadTime = loadingCache.get(LOADTIME);
+
+			if (!Objects.equals(Instant.MIN, loadTime)) {
+				for (final AggregationInfoV1 a : aggregationInfos) {
+					if (asInstant(a).isBefore(loadTime) && AggregationStage.COLLECT.equals(a.getAggregationStage())) {
+						a.setIndexedVersion(true);
+						break;
+					}
+				}
+			}
+
+			return aggregationInfos;
+		} catch (final Throwable e) {
+			throw new DsmApiException(HttpStatus.SC_INTERNAL_SERVER_ERROR, String.format("error reading aggregation history for '%s'", dsId), e);
+		}
+	}
+
+	private List<AggregationInfo> getAggregationHistoryV2(final String dsId,
+			final Bson queryForAggregationHistory,
+			final Function<Document, AggregationInfo> mapper) throws DsmApiException {
+		log.warn(String.format("getAggregationHistory(dsId = %s): not using cache", dsId));
+		final Datasource conf = config.getDatasource();
+		try {
+			final FindIterable<Document> aggregationDocs = getCollection().find(queryForAggregationHistory)
+					.projection(fields)
+					.limit(conf.getMongoQueryLimit())
+					.sort(dbo("system:startHumanDate", -1));
+
+			final List<AggregationInfo> aggregationInfos = Utils.stream(aggregationDocs.iterator())
+					.map(mapper)
+					.filter(ai -> ai.getNumberOfRecords() >= 0 && StringUtils.isNotBlank(ai.getDate()))
+					.collect(Collectors.toList());
 
 			final Instant loadTime = loadingCache.get(LOADTIME);
 
@@ -146,9 +176,14 @@ public class MongoLoggerClientImpl implements MongoLoggerClient {
 		return Instant.parse(a.getDate() + "T00:00:00Z");
 	}
 
+	@Deprecated
+	private Instant asInstant(final AggregationInfoV1 a) {
+		return Instant.parse(a.getDate() + "T00:00:00Z");
+	}
+
 	@Override
 	@CacheEvict(cacheNames = {
-		"dsm-aggregationhistory-cache-v1", "dsm-aggregationhistory-cache-v2", "dsm-firstharvestdate-cache"
+			"dsm-aggregationhistory-cache-v1", "dsm-aggregationhistory-cache-v2", "dsm-firstharvestdate-cache"
 	}, allEntries = true)
 	@Scheduled(fixedDelayString = "${openaire.exporter.cache.ttl}")
 	public void dropCache() {
@@ -166,20 +201,18 @@ public class MongoLoggerClientImpl implements MongoLoggerClient {
 			switch (stage) {
 
 			case COLLECT:
-				final CollectionInfoV1 cInfo = new CollectionInfoV1();
+				final AggregationInfoV1 cInfo = new AggregationInfoV1();
 				cInfo.setAggregationStage(stage);
 				cInfo.setCollectionMode(getCollectionMode(d));
 				cInfo.setNumberOfRecords(success ? getNumberOfRecords(d) : 0);
 				cInfo.setDate(getDate(d));
-				cInfo.setCompletedSuccessfully(success);
 				info = cInfo;
 				break;
 			case TRANSFORM:
-				final TransformationInfoV1 tInfo = new TransformationInfoV1();
+				final AggregationInfoV1 tInfo = new AggregationInfoV1();
 				tInfo.setAggregationStage(stage);
 				tInfo.setNumberOfRecords(success ? getNumberOfRecords(d) : 0);
 				tInfo.setDate(getDate(d));
-				tInfo.setCompletedSuccessfully(success);
 				info = tInfo;
 				break;
 			}
@@ -197,7 +230,7 @@ public class MongoLoggerClientImpl implements MongoLoggerClient {
 			switch (stage) {
 
 			case COLLECT:
-				final CollectionInfoV2 cInfo = new CollectionInfoV2();
+				final AggregationInfo cInfo = new AggregationInfo();
 				cInfo.setAggregationStage(stage);
 				cInfo.setCollectionMode(getCollectionMode(d));
 				cInfo.setNumberOfRecords(success ? getNumberOfRecords(d) : 0);
@@ -206,7 +239,7 @@ public class MongoLoggerClientImpl implements MongoLoggerClient {
 				info = cInfo;
 				break;
 			case TRANSFORM:
-				final TransformationInfoV2 tInfo = new TransformationInfoV2();
+				final AggregationInfo tInfo = new AggregationInfo();
 				tInfo.setAggregationStage(stage);
 				tInfo.setNumberOfRecords(success ? getNumberOfRecords(d) : 0);
 				tInfo.setDate(getDate(d));
@@ -220,23 +253,19 @@ public class MongoLoggerClientImpl implements MongoLoggerClient {
 
 	private CollectionMode getCollectionMode(final Document d) {
 		return Optional.ofNullable(d.getString("system:node:SELECT_MODE:selection"))
-			.map(CollectionMode::valueOf)
-			.orElseGet(() -> Optional.ofNullable(d.getString("collectionMode"))
 				.map(CollectionMode::valueOf)
-				.orElse(null));
+				.orElseGet(() -> Optional.ofNullable(d.getString("collectionMode"))
+						.map(CollectionMode::valueOf)
+						.orElse(null));
 	}
 
 	private Integer getNumberOfRecords(final Document d) {
 		final String sinkSize = d.getString("mainlog:sinkSize");
 		final String total = d.getString("mainlog:total");
 
-		if (StringUtils.isNotBlank(sinkSize)) {
-			return Ints.tryParse(sinkSize);
-		} else if (StringUtils.isNotBlank(total)) {
-			return Ints.tryParse(total);
-		} else {
-			return -1;
-		}
+		if (StringUtils.isNotBlank(sinkSize)) { return Ints.tryParse(sinkSize); }
+		if (StringUtils.isNotBlank(total)) { return Ints.tryParse(total); }
+		return -1;
 	}
 
 	private String getDate(final Document d) {

apps/dnet-exporter-api/src/main/resources/sql/community-schema.sql

@@ -9,6 +9,8 @@ CREATE TABLE communities (
 	id                       text PRIMARY KEY,
 	name                     text NOT NULL,
 	shortname                text NOT NULL, -- in the profile is label
+	displayname              text,
+	displayshortname         text,
 	description              text NOT NULL DEFAULT '',
 	status                   text NOT NULL DEFAULT 'hidden', -- all, manager, hidden, members
 	membership               text NOT NULL DEFAULT 'by-invitation', -- open, by-invitation
@@ -25,7 +27,7 @@ CREATE TABLE communities (
 	last_update              timestamp NOT NULL DEFAULT now(),
 	logo_url                 text,
 	suggested_acknowledgements text[],
-	plan                     text
+	plan                     text NOT NULL DEFAULT 'Default'
 );
 
 CREATE TABLE community_projects (

apps/dnet-orgs-database-application/pom.xml

@@ -4,7 +4,7 @@
 	<parent>
 		<groupId>eu.dnetlib.dhp</groupId>
 		<artifactId>apps</artifactId>
-		<version>3.4.5-SNAPSHOT</version>
+		<version>3.5.5-SNAPSHOT</version>
 		<relativePath>../</relativePath>
 	</parent>
 

apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/controller/OrganizationController.java

@@ -86,20 +86,16 @@ public class OrganizationController extends AbstractDnetController {
 	@PostMapping("/save")
 	public List<String> save(@RequestBody final OrganizationView org, final Authentication authentication) {
 
-		if (StringUtils.isBlank(org.getName())) {
-			throw new RuntimeException("Missing field: name");
-		} else if (StringUtils.isBlank(org.getCountry())) {
-			throw new RuntimeException("Missing field: country");
-		} else if (StringUtils.isBlank(org.getType())) {
-			throw new RuntimeException("Missing field: type");
-		} else if (UserInfo.isSuperAdmin(authentication)
-			|| userCountryRepository.verifyAuthorizationForCountry(org.getCountry(), UserInfo.getEmail(authentication))) {
-				final String orgId =
+		if (StringUtils.isBlank(org.getName())) { throw new RuntimeException("Missing field: name"); }
+		if (StringUtils.isBlank(org.getCountry())) { throw new RuntimeException("Missing field: country"); }
+		if (StringUtils.isBlank(org.getType())) { throw new RuntimeException("Missing field: type"); }
+		if (UserInfo.isSuperAdmin(authentication)
+				|| userCountryRepository.verifyAuthorizationForCountry(org.getCountry(), UserInfo.getEmail(authentication))) {
+			final String orgId =
 					databaseUtils.insertOrUpdateOrganization(org, UserInfo.getEmail(authentication), UserInfo.isSimpleUser(authentication));
-				return Arrays.asList(orgId);
-			} else {
-				throw new RuntimeException("User not authorized");
-			}
+			return Arrays.asList(orgId);
+		}
+		throw new RuntimeException("User not authorized");
 	}
 
 	@GetMapping("/info")
@@ -115,11 +111,11 @@ public class OrganizationController extends AbstractDnetController {
 			suggestionInfoViewByCountryRepository.findAll().forEach(info::add);
 		} else if (UserInfo.isSimpleUser(authentication) || UserInfo.isNationalAdmin(authentication)) {
 			userCountryRepository.getCountriesForUser(UserInfo.getEmail(authentication))
-				.stream()
-				.map(suggestionInfoViewByCountryRepository::findById)
-				.filter(Optional::isPresent)
-				.map(Optional::get)
-				.forEach(info::add);
+					.stream()
+					.map(suggestionInfoViewByCountryRepository::findById)
+					.filter(Optional::isPresent)
+					.map(Optional::get)
+					.forEach(info::add);
 		}
 		return info;
 	}
@@ -129,29 +125,26 @@ public class OrganizationController extends AbstractDnetController {
 		final OrganizationView org = organizationViewRepository.findById(id).get();
 
 		if (UserInfo.isSuperAdmin(authentication)
-			|| userCountryRepository.verifyAuthorizationForCountry(org.getCountry(), UserInfo.getEmail(authentication))) {
+				|| userCountryRepository.verifyAuthorizationForCountry(org.getCountry(), UserInfo.getEmail(authentication))) {
 			return org;
-		} else {
-			throw new RuntimeException("User not authorized");
 		}
+		throw new RuntimeException("User not authorized");
 	}
 
 	@GetMapping("/conflicts")
 	public List<OrganizationConflict> conflicts(@RequestParam final String id, final Authentication authentication) {
 		if (UserInfo.isSuperAdmin(authentication) || userCountryRepository.verifyAuthorizationForId(id, UserInfo.getEmail(authentication))) {
 			return databaseUtils.listConflictsForId(id);
-		} else {
-			throw new RuntimeException("User not authorized");
 		}
+		throw new RuntimeException("User not authorized");
 	}
 
 	@GetMapping("/duplicates")
 	public List<OpenaireDuplicateView> duplicates(@RequestParam final String id, final Authentication authentication) {
 		if (UserInfo.isSuperAdmin(authentication) || userCountryRepository.verifyAuthorizationForId(id, UserInfo.getEmail(authentication))) {
 			return listDuplicates(id);
-		} else {
-			throw new RuntimeException("User not authorized");
 		}
+		throw new RuntimeException("User not authorized");
 	}
 
 	private List<OpenaireDuplicateView> listDuplicates(final String id) {
@@ -161,43 +154,42 @@ public class OrganizationController extends AbstractDnetController {
 	@GetMapping("/conflicts/byCountry/{country}")
 	public Collection<Set<OrganizationConflict>> findConflictsByCountry(@PathVariable final String country, final Authentication authentication) {
 
-		if (UserInfo.isSuperAdmin(authentication)) {
-			return groupConflicts(conflictGroupViewRepository.findByCountry1OrCountry2(country, country).stream());
-		} else if (UserInfo.isSimpleUser(authentication) || UserInfo.isNationalAdmin(authentication)) {
+		if (UserInfo.isSuperAdmin(authentication)) { return groupConflicts(conflictGroupViewRepository.findByCountry1OrCountry2(country, country).stream()); }
+		if (UserInfo.isSimpleUser(authentication) || UserInfo.isNationalAdmin(authentication)) {
 			final Stream<ConflictGroupView> list = userCountryRepository.getCountriesForUser(UserInfo.getEmail(authentication))
-				.stream()
-				.filter(country::equalsIgnoreCase)
-				.map(c -> conflictGroupViewRepository.findByCountry1OrCountry2(c, c).stream())
-				.findFirst()
-				.orElse(Stream.empty());
+					.stream()
+					.filter(country::equalsIgnoreCase)
+					.map(c -> conflictGroupViewRepository.findByCountry1OrCountry2(c, c).stream())
+					.findFirst()
+					.orElse(Stream.empty());
 			return groupConflicts(list);
-		} else {
-			throw new RuntimeException("User not authorized");
 		}
+		throw new RuntimeException("User not authorized");
 
 	}
 
 	@GetMapping("/duplicates/byCountry/{country}")
-	public Iterable<DuplicateGroupView> findDuplicatesByCountry(@PathVariable final String country, final Authentication authentication) {
+	public Iterable<DuplicateGroupView> findDuplicatesByCountry(@PathVariable final String country,
+			@RequestParam(required = false, defaultValue = "0") final int page,
+			@RequestParam(required = false, defaultValue = "${openorgs.findDuplicatesByCountry.limit.default}") final int size,
+			final Authentication authentication) {
 
-		if (UserInfo.isSuperAdmin(authentication)) {
-			return duplicateGroupViewRepository.findByCountry(country);
-		} else if (UserInfo.isSimpleUser(authentication) || UserInfo.isNationalAdmin(authentication)) {
+		if (UserInfo.isSuperAdmin(authentication)) { return duplicateGroupViewRepository.findByCountry(country, PageRequest.of(page, size)); }
+		if (UserInfo.isSimpleUser(authentication) || UserInfo.isNationalAdmin(authentication)) {
 			return userCountryRepository.getCountriesForUser(UserInfo.getEmail(authentication))
-				.stream()
-				.filter(country::equalsIgnoreCase)
-				.map(duplicateGroupViewRepository::findByCountry)
-				.findFirst()
-				.orElse(new ArrayList<DuplicateGroupView>());
-		} else {
-			throw new RuntimeException("User not authorized");
+					.stream()
+					.filter(country::equalsIgnoreCase)
+					.map(c -> duplicateGroupViewRepository.findByCountry(c, PageRequest.of(page, size)))
+					.findFirst()
+					.orElse(new ArrayList<DuplicateGroupView>());
 		}
+		throw new RuntimeException("User not authorized");
 	}
 
 	@GetMapping(value = "/duplicates/byCountry/{country}/csv", produces = "text/csv")
 	public void findDuplicatesByCountryCSV(@PathVariable final String country, final HttpServletResponse res, final Authentication authentication)
-		throws IOException {
-		final Iterable<DuplicateGroupView> list = findDuplicatesByCountry(country, authentication);
+			throws IOException {
+		final Iterable<DuplicateGroupView> list = findDuplicatesByCountry(country, 0, Integer.MAX_VALUE, authentication);
 		CSVConverter.writeCSV(res.getOutputStream(), list, DuplicateGroupView.class, "id", "name", "city", "country", "numberOfDuplicates");
 	}
 
@@ -219,148 +211,131 @@ public class OrganizationController extends AbstractDnetController {
 		if (simrels.isEmpty()) { return new ArrayList<>(); }
 
 		final boolean b = UserInfo.isSuperAdmin(authentication)
-			|| simrels.stream()
-				.map(OpenaireDuplicate::getLocalId)
-				.distinct()
-				.allMatch(id -> userCountryRepository.verifyAuthorizationForId(id, UserInfo.getEmail(authentication)));
+				|| simrels.stream()
+						.map(OpenaireDuplicate::getLocalId)
+						.distinct()
+						.allMatch(id -> userCountryRepository.verifyAuthorizationForId(id, UserInfo.getEmail(authentication)));
 
 		if (b) {
 			databaseUtils.saveDuplicates(simrels, UserInfo.getEmail(authentication));
 			return listDuplicates(simrels.get(0).getLocalId());
-		} else {
-			throw new RuntimeException("User not authorized");
 		}
+		throw new RuntimeException("User not authorized");
 	}
 
 	@GetMapping("/search/{page}/{size}")
 	public Page<OrganizationSimpleView> search(@PathVariable final int page,
-		@PathVariable final int size,
-		@RequestParam final String q,
-		@RequestParam(required = false, defaultValue = "") final String status,
-		final Authentication authentication) {
+			@PathVariable final int size,
+			@RequestParam final String q,
+			@RequestParam(required = false, defaultValue = "") final String status,
+			final Authentication authentication) {
 
-		if (status.equals(SPECIAL_STATUS_FOR_CANDIDATE_DUP)) {
+		if (SPECIAL_STATUS_FOR_CANDIDATE_DUP.equals(status)) {
 			return UserInfo.isSuperAdmin(authentication)
-				? organizationSimpleViewRepository.searchCandidateDuplicates(q, PageRequest.of(page, size))
-				: organizationSimpleViewRepository.searchCandidateDuplicatesForUser(q, UserInfo.getEmail(authentication), PageRequest.of(page, size));
+					? organizationSimpleViewRepository.searchCandidateDuplicates(q, PageRequest.of(page, size))
+					: organizationSimpleViewRepository.searchCandidateDuplicatesForUser(q, UserInfo.getEmail(authentication), PageRequest.of(page, size));
+		}
+		final List<String> statuses;
+		if (StringUtils.isNotBlank(status)) {
+			statuses = Arrays.asList(status.split(","));
+		} else if (UserInfo.isSimpleUser(authentication)) {
+			statuses = Arrays.asList(OrganizationStatus.approved.toString());
 		} else {
-			final List<String> statuses;
-			if (StringUtils.isNotBlank(status)) {
-				statuses = Arrays.asList(status.split(","));
-			} else if (UserInfo.isSimpleUser(authentication)) {
-				statuses = Arrays.asList(OrganizationStatus.approved.toString());
-			} else {
-				statuses = Arrays.asList(OrganizationStatus.approved.toString(), OrganizationStatus.suggested.toString());
-			}
+			statuses = Arrays.asList(OrganizationStatus.approved.toString(), OrganizationStatus.suggested.toString());
+		}
 
-			return UserInfo.isSuperAdmin(authentication)
+		return UserInfo.isSuperAdmin(authentication)
 				? organizationSimpleViewRepository.search(q, statuses, PageRequest.of(page, size))
 				: organizationSimpleViewRepository.searchForUser(q, UserInfo.getEmail(authentication), statuses, PageRequest.of(page, size));
-		}
 	}
 
 	@GetMapping("/byCountry/{status}/{code}/{page}/{size}")
 	public Page<OrganizationSimpleView> findByCountry(@PathVariable final String status,
-		@PathVariable final String code,
-		@PathVariable final int page,
-		@PathVariable final int size,
-		final Authentication authentication) {
-		if (UserInfo.isSuperAdmin(authentication)
-			|| userCountryRepository.verifyAuthorizationForCountry(code, UserInfo.getEmail(authentication))) {
-			if (status.equalsIgnoreCase("all")) {
-				return organizationSimpleViewRepository.findByCountryOrderByName(code, PageRequest.of(page, size));
-			} else {
-				return organizationSimpleViewRepository.findByCountryAndStatusOrderByName(code, status, PageRequest.of(page, size));
-			}
-		} else {
+			@PathVariable final String code,
+			@PathVariable final int page,
+			@PathVariable final int size,
+			final Authentication authentication) {
+		if (!UserInfo.isSuperAdmin(authentication) && !userCountryRepository.verifyAuthorizationForCountry(code, UserInfo.getEmail(authentication))) {
 			throw new RuntimeException("User not authorized");
 		}
+		if ("all".equalsIgnoreCase(status)) { return organizationSimpleViewRepository.findByCountryOrderByName(code, PageRequest.of(page, size)); }
+		return organizationSimpleViewRepository.findByCountryAndStatusOrderByName(code, status, PageRequest.of(page, size));
 	}
 
 	@GetMapping("/byCountry/{status}/{code}")
-	public Iterable<OrganizationSimpleView> findOrgsByStatusAndCountry(@PathVariable final String status,
-		@PathVariable final String code,
-		final Authentication authentication) {
-		if (UserInfo.isSuperAdmin(authentication)
-			|| userCountryRepository.verifyAuthorizationForCountry(code, UserInfo.getEmail(authentication))) {
-			if (status.equalsIgnoreCase("all")) {
-				return organizationSimpleViewRepository.findByCountryOrderByName(code);
-			} else {
-				return organizationSimpleViewRepository.findByCountryAndStatusOrderByName(code, status);
-			}
-		} else {
+	public List<OrganizationSimpleView> findOrgsByStatusAndCountry(@PathVariable final String status,
+			@PathVariable final String code,
+			@RequestParam(required = false, defaultValue = "0") final int page,
+			@RequestParam(required = false, defaultValue = "${openorgs.findOrgsByStatusAndCountry.limit.default}") final int size,
+			final Authentication authentication) {
+		if (!UserInfo.isSuperAdmin(authentication) && !userCountryRepository.verifyAuthorizationForCountry(code, UserInfo.getEmail(authentication))) {
 			throw new RuntimeException("User not authorized");
 		}
+		if ("all".equalsIgnoreCase(status)) { return organizationSimpleViewRepository.findByCountryOrderByName(code, PageRequest.of(page, size)).getContent(); }
+		return organizationSimpleViewRepository.findByCountryAndStatusOrderByName(code, status, PageRequest.of(page, size)).getContent();
 	}
 
 	@GetMapping(value = "/byCountry/{status}/{code}/csv", produces = "text/csv")
 	public void findOrgsByStatusAndCountryCSV(@PathVariable final String status,
-		@PathVariable final String code,
-		final HttpServletResponse res,
-		final Authentication authentication) throws IOException {
-		final Iterable<OrganizationSimpleView> list = findOrgsByStatusAndCountry(status, code, authentication);
+			@PathVariable final String code,
+			final HttpServletResponse res,
+			final Authentication authentication) throws IOException {
+		final Iterable<OrganizationSimpleView> list = findOrgsByStatusAndCountry(status, code, 0, Integer.MAX_VALUE, authentication);
 		CSVConverter.writeCSV(res
-			.getOutputStream(), list, OrganizationSimpleView.class, "id", "name", "type", "city", "country", "acronyms", "urls", "status", "nSimilarDups", "nSuggestedDups", "nDifferentDups");
+				.getOutputStream(), list, OrganizationSimpleView.class, "id", "name", "type", "city", "country", "acronyms", "urls", "status", "nSimilarDups", "nSuggestedDups", "nDifferentDups");
 	}
 
 	@GetMapping("/byType/{status}/{type}/{page}/{size}")
 	public Page<OrganizationSimpleView> findByType(@PathVariable final String status,
-		@PathVariable final String type,
-		@PathVariable final int page,
-		@PathVariable final int size,
-		final Authentication authentication) {
+			@PathVariable final String type,
+			@PathVariable final int page,
+			@PathVariable final int size,
+			final Authentication authentication) {
 
 		if (UserInfo.isSuperAdmin(authentication)) {
-			if (status.equalsIgnoreCase("all")) {
-				return organizationSimpleViewRepository.findByTypeOrderByName(type, PageRequest.of(page, size));
-			} else {
-				return organizationSimpleViewRepository.findByTypeAndStatusOrderByName(type, status, PageRequest.of(page, size));
-			}
-		} else {
-			if (status.equalsIgnoreCase("all")) {
-				return organizationSimpleViewRepository.findByTypeForUser(type, UserInfo.getEmail(authentication), PageRequest.of(page, size));
-			} else {
-				return organizationSimpleViewRepository
-					.findByTypeAndStatusForUser(type, status, UserInfo.getEmail(authentication), PageRequest.of(page, size));
-			}
+			if ("all".equalsIgnoreCase(status)) { return organizationSimpleViewRepository.findByTypeOrderByName(type, PageRequest.of(page, size)); }
+			return organizationSimpleViewRepository.findByTypeAndStatusOrderByName(type, status, PageRequest.of(page, size));
 		}
+		if ("all".equalsIgnoreCase(status)) {
+			return organizationSimpleViewRepository.findByTypeForUser(type, UserInfo.getEmail(authentication), PageRequest.of(page, size));
+		}
+		return organizationSimpleViewRepository
+				.findByTypeAndStatusForUser(type, status, UserInfo.getEmail(authentication), PageRequest.of(page, size));
 
 	}
 
 	@GetMapping("/browse/countries")
 	public List<BrowseEntry> browseCountries(final Authentication authentication) {
 		return UserInfo.isSuperAdmin(authentication)
-			? databaseUtils.browseCountries()
-			: databaseUtils.browseCountriesForUser(UserInfo.getEmail(authentication));
+				? databaseUtils.browseCountries()
+				: databaseUtils.browseCountriesForUser(UserInfo.getEmail(authentication));
 	}
 
 	@GetMapping("/browse/types")
 	public List<BrowseEntry> browseOrganizationTypes(final Authentication authentication) {
 		return UserInfo.isSuperAdmin(authentication)
-			? databaseUtils.browseTypes()
-			: databaseUtils.browseTypesForUser(UserInfo.getEmail(authentication));
+				? databaseUtils.browseTypes()
+				: databaseUtils.browseTypesForUser(UserInfo.getEmail(authentication));
 	}
 
 	@PostMapping("/conflicts/fix/similar")
 	public List<String> fixConflictSim(final Authentication authentication, @RequestBody final List<String> ids) {
 		if (ids.size() > 1 && UserInfo.isSuperAdmin(authentication)
-			|| userCountryRepository.verifyAuthorizationForId(ids.get(0), UserInfo.getEmail(authentication))) {
+				|| userCountryRepository.verifyAuthorizationForId(ids.get(0), UserInfo.getEmail(authentication))) {
 			final String newOrgId = databaseUtils.fixConflictSimilars(ids, UserInfo.getEmail(authentication));
 			return Arrays.asList(newOrgId);
-		} else {
-			return new ArrayList<>();
 		}
+		return new ArrayList<>();
 	}
 
 	@PostMapping("/conflicts/fix/different")
 	public List<String> fixConflictDiff(final Authentication authentication, @RequestBody final List<String> ids) {
 		if (ids.size() > 1 && UserInfo.isSuperAdmin(authentication)
-			|| userCountryRepository.verifyAuthorizationForId(ids.get(0), UserInfo.getEmail(authentication))) {
+				|| userCountryRepository.verifyAuthorizationForId(ids.get(0), UserInfo.getEmail(authentication))) {
 			databaseUtils.fixConflictDifferents(ids, UserInfo.getEmail(authentication));
 			return ids;
-		} else {
-			return new ArrayList<>();
 		}
+		return new ArrayList<>();
 	}
 
 	@GetMapping("/note")
@@ -368,11 +343,10 @@ public class OrganizationController extends AbstractDnetController {
 		final OrganizationView org = organizationViewRepository.findById(id).get();
 
 		if (UserInfo.isSuperAdmin(authentication)
-			|| userCountryRepository.verifyAuthorizationForCountry(org.getCountry(), UserInfo.getEmail(authentication))) {
+				|| userCountryRepository.verifyAuthorizationForCountry(org.getCountry(), UserInfo.getEmail(authentication))) {
 			return noteRepository.findById(id).orElse(new Note(id, "", null, null));
-		} else {
-			throw new RuntimeException("User not authorized");
 		}
+		throw new RuntimeException("User not authorized");
 	}
 
 	@PostMapping("/note")
@@ -381,20 +355,17 @@ public class OrganizationController extends AbstractDnetController {
 
 		final OrganizationView org = organizationViewRepository.findById(orgId).get();
 
-		if (UserInfo.isSuperAdmin(authentication)
-			|| userCountryRepository.verifyAuthorizationForCountry(org.getCountry(), UserInfo.getEmail(authentication))) {
-
-			if (StringUtils.isNotBlank(note.getNote())) {
-				note.setModifiedBy(UserInfo.getEmail(authentication));
-				note.setModificationDate(OffsetDateTime.now());
-				return noteRepository.save(note);
-			} else {
-				noteRepository.deleteById(orgId);
-				return new Note(orgId, "", null, null);
-			}
-		} else {
+		if (!UserInfo.isSuperAdmin(authentication)
+				&& !userCountryRepository.verifyAuthorizationForCountry(org.getCountry(), UserInfo.getEmail(authentication))) {
 			throw new RuntimeException("User not authorized");
 		}
+		if (StringUtils.isNotBlank(note.getNote())) {
+			note.setModifiedBy(UserInfo.getEmail(authentication));
+			note.setModificationDate(OffsetDateTime.now());
+			return noteRepository.save(note);
+		}
+		noteRepository.deleteById(orgId);
+		return new Note(orgId, "", null, null);
 	}
 
 	@GetMapping("/journal")
@@ -402,11 +373,10 @@ public class OrganizationController extends AbstractDnetController {
 		final OrganizationView org = organizationViewRepository.findById(id).get();
 
 		if (UserInfo.isSuperAdmin(authentication)
-			|| userCountryRepository.verifyAuthorizationForCountry(org.getCountry(), UserInfo.getEmail(authentication))) {
+				|| userCountryRepository.verifyAuthorizationForCountry(org.getCountry(), UserInfo.getEmail(authentication))) {
 			return journalEntryRepository.findByOrgIdOrderByDateDesc(id);
-		} else {
-			throw new RuntimeException("User not authorized");
 		}
+		throw new RuntimeException("User not authorized");
 	}
 
 }

apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/repository/readonly/DuplicateGroupViewRepository.java

@@ -2,6 +2,7 @@ package eu.dnetlib.organizations.repository.readonly;
 
 import java.util.List;
 
+import org.springframework.data.domain.Pageable;
 import org.springframework.stereotype.Repository;
 
 import eu.dnetlib.organizations.model.view.DuplicateGroupView;
@@ -9,5 +10,5 @@ import eu.dnetlib.organizations.model.view.DuplicateGroupView;
 @Repository
 public interface DuplicateGroupViewRepository extends ReadOnlyRepository<DuplicateGroupView, String> {
 
-	List<DuplicateGroupView> findByCountry(String country);
+	List<DuplicateGroupView> findByCountry(String country, Pageable page);
 }

apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/repository/readonly/OrganizationSimpleViewRepository.java

@@ -15,60 +15,56 @@ public interface OrganizationSimpleViewRepository extends ReadOnlyRepository<Org
 
 	// SEARCH
 	@Query(value = "SELECT \n"
-		+ "    org.id,\n"
-		+ "    org.name,\n"
-		+ "    org.type,\n"
-		+ "    org.city,\n"
-		+ "    org.country,\n"
-		+ "    org.status,\n"
-		+ "    array_remove(array_agg(DISTINCT a.acronym), NULL) AS acronyms,\n"
-		+ "    array_remove(array_agg(DISTINCT u.url), NULL) AS urls,\n"
-		+ "    count(DISTINCT d1.oa_original_id) FILTER (WHERE d1.reltype = 'is_similar') AS n_similar_dups,\n"
-		+ "    count(DISTINCT d1.oa_original_id) FILTER (WHERE d1.reltype = 'suggested') AS n_suggested_dups,\n"
-		+ "    count(DISTINCT d1.oa_original_id) FILTER (WHERE d1.reltype = 'is_different') AS n_different_dups\n"
-		+ "FROM org_index_search idx "
-		+ "     JOIN organizations org ON (idx.id = org.id) \n"
-		+ "     LEFT OUTER JOIN acronyms a ON org.id = a.id\n"
-		+ "     LEFT OUTER JOIN urls u ON org.id = u.id\n"
-		+ "     LEFT OUTER JOIN oa_duplicates d1 ON org.id = d1.local_id\n"
-		+ "WHERE org.status in :statuses AND (org.name ilike '%'||:text||'%' OR idx.txt @@ plainto_tsquery(:text))\n"
-		+ "GROUP BY org.id, org.name, org.type, org.city, org.country, org.status\n"
-		+ "ORDER BY org.name", nativeQuery = true)
+			+ "    org.id,\n"
+			+ "    org.name,\n"
+			+ "    org.type,\n"
+			+ "    org.city,\n"
+			+ "    org.country,\n"
+			+ "    org.status,\n"
+			+ "    array_remove(array_agg(DISTINCT a.acronym), NULL) AS acronyms,\n"
+			+ "    array_remove(array_agg(DISTINCT u.url), NULL) AS urls,\n"
+			+ "    count(DISTINCT d1.oa_original_id) FILTER (WHERE d1.reltype = 'is_similar') AS n_similar_dups,\n"
+			+ "    count(DISTINCT d1.oa_original_id) FILTER (WHERE d1.reltype = 'suggested') AS n_suggested_dups,\n"
+			+ "    count(DISTINCT d1.oa_original_id) FILTER (WHERE d1.reltype = 'is_different') AS n_different_dups\n"
+			+ "FROM org_index_search idx "
+			+ "     JOIN organizations org ON (idx.id = org.id) \n"
+			+ "     LEFT OUTER JOIN acronyms a ON org.id = a.id\n"
+			+ "     LEFT OUTER JOIN urls u ON org.id = u.id\n"
+			+ "     LEFT OUTER JOIN oa_duplicates d1 ON org.id = d1.local_id\n"
+			+ "WHERE org.status in :statuses AND (org.name ilike '%'||:text||'%' OR idx.txt @@ plainto_tsquery(:text))\n"
+			+ "GROUP BY org.id, org.name, org.type, org.city, org.country, org.status\n"
+			+ "ORDER BY org.name", nativeQuery = true)
 	Page<OrganizationSimpleView> search(@Param("text") String text, @Param("statuses") List<String> statuses, Pageable pageable);
 
 	// SEARCH FOR USER
 	@Query(value = "SELECT\n"
-		+ "    org.id,\n"
-		+ "    org.name,\n"
-		+ "    org.type,\n"
-		+ "    org.city,\n"
-		+ "    org.country,\n"
-		+ "    org.status,\n"
-		+ "    array_remove(array_agg(DISTINCT a.acronym), NULL) AS acronyms,\n"
-		+ "    array_remove(array_agg(DISTINCT u.url), NULL) AS urls,\n"
-		+ "    count(DISTINCT d1.oa_original_id) FILTER (WHERE d1.reltype = 'is_similar'  ) AS n_similar_dups,\n"
-		+ "    count(DISTINCT d1.oa_original_id) FILTER (WHERE d1.reltype = 'suggested'   ) AS n_suggested_dups,\n"
-		+ "    count(DISTINCT d1.oa_original_id) FILTER (WHERE d1.reltype = 'is_different') AS n_different_dups\n"
-		+ "FROM org_index_search idx\n"
-		+ "    JOIN organizations org ON (idx.id = org.id)\n"
-		+ "    LEFT OUTER JOIN acronyms a ON (org.id = a.id)\n"
-		+ "    LEFT OUTER JOIN urls u ON (org.id = u.id)\n"
-		+ "    LEFT OUTER JOIN oa_duplicates d1 ON (org.id = d1.local_id)\n"
-		+ "    LEFT OUTER JOIN user_countries uc ON (uc.country = org.country) \n"
-		+ "WHERE uc.email = :email AND org.status IN :statuses AND (org.name ilike '%'||:text||'%' OR idx.txt @@ plainto_tsquery(:text))\n"
-		+ "GROUP BY org.id, org.name, org.type, org.city, org.country, org.status\n"
-		+ "ORDER BY org.name", nativeQuery = true)
+			+ "    org.id,\n"
+			+ "    org.name,\n"
+			+ "    org.type,\n"
+			+ "    org.city,\n"
+			+ "    org.country,\n"
+			+ "    org.status,\n"
+			+ "    array_remove(array_agg(DISTINCT a.acronym), NULL) AS acronyms,\n"
+			+ "    array_remove(array_agg(DISTINCT u.url), NULL) AS urls,\n"
+			+ "    count(DISTINCT d1.oa_original_id) FILTER (WHERE d1.reltype = 'is_similar'  ) AS n_similar_dups,\n"
+			+ "    count(DISTINCT d1.oa_original_id) FILTER (WHERE d1.reltype = 'suggested'   ) AS n_suggested_dups,\n"
+			+ "    count(DISTINCT d1.oa_original_id) FILTER (WHERE d1.reltype = 'is_different') AS n_different_dups\n"
+			+ "FROM org_index_search idx\n"
+			+ "    JOIN organizations org ON (idx.id = org.id)\n"
+			+ "    LEFT OUTER JOIN acronyms a ON (org.id = a.id)\n"
+			+ "    LEFT OUTER JOIN urls u ON (org.id = u.id)\n"
+			+ "    LEFT OUTER JOIN oa_duplicates d1 ON (org.id = d1.local_id)\n"
+			+ "    LEFT OUTER JOIN user_countries uc ON (uc.country = org.country) \n"
+			+ "WHERE uc.email = :email AND org.status IN :statuses AND (org.name ilike '%'||:text||'%' OR idx.txt @@ plainto_tsquery(:text))\n"
+			+ "GROUP BY org.id, org.name, org.type, org.city, org.country, org.status\n"
+			+ "ORDER BY org.name", nativeQuery = true)
 	Page<OrganizationSimpleView> searchForUser(@Param("text") String text,
-		@Param("email") String email,
-		@Param("statuses") List<String> statuses,
-		Pageable pageable);
+			@Param("email") String email,
+			@Param("statuses") List<String> statuses,
+			Pageable pageable);
 
 	Page<OrganizationSimpleView> findByCountryOrderByName(String country, Pageable pageable);
 
-	Iterable<OrganizationSimpleView> findByCountryOrderByName(String code);
-
-	Iterable<OrganizationSimpleView> findByCountryAndStatusOrderByName(String code, String status);
-
 	Page<OrganizationSimpleView> findByCountryAndStatusOrderByName(String code, String status, Pageable pageable);
 
 	Page<OrganizationSimpleView> findByTypeOrderByName(String type, Pageable pageable);
@@ -83,51 +79,51 @@ public interface OrganizationSimpleViewRepository extends ReadOnlyRepository<Org
 
 	// SEARCH FOR VALID DUPLICATE CANDIDATES
 	@Query(value = "SELECT\n"
-		+ "    org.id,\n"
-		+ "    org.name,\n"
-		+ "    org.type,\n"
-		+ "    org.city,\n"
-		+ "    org.country,\n"
-		+ "    org.status,\n"
-		+ "    array_remove(array_agg(DISTINCT a.acronym), NULL) AS acronyms,\n"
-		+ "    array_remove(array_agg(DISTINCT u.url), NULL) AS urls,\n"
-		+ "    NULL AS n_similar_dups,\n"
-		+ "    NULL AS n_suggested_dups,\n"
-		+ "    NULL AS n_different_dups\n"
-		+ "FROM org_index_search idx\n"
-		+ "    JOIN organizations org ON (idx.id = org.id) \n"
-		+ "    LEFT OUTER JOIN acronyms a ON (org.id = a.id)\n"
-		+ "    LEFT OUTER JOIN urls u ON (org.id = u.id)\n"
-		+ "    LEFT OUTER JOIN oa_duplicates d2 ON (org.id = d2.oa_original_id)\n"
-		+ "WHERE org.status = 'raw' AND (org.name ilike '%'||:text||'%' OR idx.txt @@ plainto_tsquery(:text))\n"
-		+ "GROUP BY org.id, org.name, org.type, org.city, org.country, org.status\n"
-		+ "HAVING  not('is_similar' = ANY(array_agg(d2.reltype)))\n"
-		+ "ORDER BY org.name", nativeQuery = true)
+			+ "    org.id,\n"
+			+ "    org.name,\n"
+			+ "    org.type,\n"
+			+ "    org.city,\n"
+			+ "    org.country,\n"
+			+ "    org.status,\n"
+			+ "    array_remove(array_agg(DISTINCT a.acronym), NULL) AS acronyms,\n"
+			+ "    array_remove(array_agg(DISTINCT u.url), NULL) AS urls,\n"
+			+ "    NULL AS n_similar_dups,\n"
+			+ "    NULL AS n_suggested_dups,\n"
+			+ "    NULL AS n_different_dups\n"
+			+ "FROM org_index_search idx\n"
+			+ "    JOIN organizations org ON (idx.id = org.id) \n"
+			+ "    LEFT OUTER JOIN acronyms a ON (org.id = a.id)\n"
+			+ "    LEFT OUTER JOIN urls u ON (org.id = u.id)\n"
+			+ "    LEFT OUTER JOIN oa_duplicates d2 ON (org.id = d2.oa_original_id)\n"
+			+ "WHERE org.status = 'raw' AND (org.name ilike '%'||:text||'%' OR idx.txt @@ plainto_tsquery(:text))\n"
+			+ "GROUP BY org.id, org.name, org.type, org.city, org.country, org.status\n"
+			+ "HAVING  not('is_similar' = ANY(array_agg(d2.reltype)))\n"
+			+ "ORDER BY org.name", nativeQuery = true)
 	Page<OrganizationSimpleView> searchCandidateDuplicates(@Param("text") String text, Pageable pageable);
 
 	// SEARCH FOR VALID DUPLICATE CANDIDATES FOR USER
 	@Query(value = "SELECT\n"
-		+ "    org.id,\n"
-		+ "    org.name,\n"
-		+ "    org.type,\n"
-		+ "    org.city,\n"
-		+ "    org.country,\n"
-		+ "    org.status,\n"
-		+ "    array_remove(array_agg(DISTINCT a.acronym), NULL) AS acronyms,\n"
-		+ "    array_remove(array_agg(DISTINCT u.url), NULL) AS urls,\n"
-		+ "    NULL AS n_similar_dups,\n"
-		+ "    NULL AS n_suggested_dups,\n"
-		+ "    NULL AS n_different_dups\n"
-		+ "FROM org_index_search idx\n"
-		+ "    JOIN organizations org ON (idx.id = org.id) \n"
-		+ "    LEFT OUTER JOIN acronyms a ON (org.id = a.id)\n"
-		+ "    LEFT OUTER JOIN urls u ON (org.id = u.id)\n"
-		+ "    LEFT OUTER JOIN oa_duplicates d2 ON (org.id = d2.oa_original_id)\n"
-		+ "    LEFT OUTER JOIN user_countries uc ON (uc.country = org.country)\n"
-		+ "WHERE org.status = 'raw' AND uc.email = :email AND (org.name ilike '%'||:text||'%' OR idx.txt @@ plainto_tsquery(:text))\n"
-		+ "GROUP BY org.id, org.name, org.type, org.city, org.country, org.status\n"
-		+ "HAVING  not('is_similar' = ANY(array_agg(d2.reltype)))\n"
-		+ "ORDER BY org.name", nativeQuery = true)
+			+ "    org.id,\n"
+			+ "    org.name,\n"
+			+ "    org.type,\n"
+			+ "    org.city,\n"
+			+ "    org.country,\n"
+			+ "    org.status,\n"
+			+ "    array_remove(array_agg(DISTINCT a.acronym), NULL) AS acronyms,\n"
+			+ "    array_remove(array_agg(DISTINCT u.url), NULL) AS urls,\n"
+			+ "    NULL AS n_similar_dups,\n"
+			+ "    NULL AS n_suggested_dups,\n"
+			+ "    NULL AS n_different_dups\n"
+			+ "FROM org_index_search idx\n"
+			+ "    JOIN organizations org ON (idx.id = org.id) \n"
+			+ "    LEFT OUTER JOIN acronyms a ON (org.id = a.id)\n"
+			+ "    LEFT OUTER JOIN urls u ON (org.id = u.id)\n"
+			+ "    LEFT OUTER JOIN oa_duplicates d2 ON (org.id = d2.oa_original_id)\n"
+			+ "    LEFT OUTER JOIN user_countries uc ON (uc.country = org.country)\n"
+			+ "WHERE org.status = 'raw' AND uc.email = :email AND (org.name ilike '%'||:text||'%' OR idx.txt @@ plainto_tsquery(:text))\n"
+			+ "GROUP BY org.id, org.name, org.type, org.city, org.country, org.status\n"
+			+ "HAVING  not('is_similar' = ANY(array_agg(d2.reltype)))\n"
+			+ "ORDER BY org.name", nativeQuery = true)
 	Page<OrganizationSimpleView> searchCandidateDuplicatesForUser(@Param("text") String text, @Param("email") String email, Pageable pageable);
 
 }

apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/utils/MailDispatcher.java

@@ -42,12 +42,12 @@ public class MailDispatcher {
 	}
 
 	public void configure(final SystemConfiguration conf) {
-		this.enabled = conf.getSmtpEnabled();
-		this.fromMail = conf.getSmtpFromMail();
-		this.fromName = conf.getSmtpFromName();
-		this.toMailAdmin = conf.getSmtpToMailAdmin();
-		this.newUserMessage = conf.getSmtpNewUserMessage();
-		this.updateUserMessage = conf.getSmtpUpdateUserMessage();
+		enabled = conf.getSmtpEnabled();
+		fromMail = conf.getSmtpFromMail();
+		fromName = conf.getSmtpFromName();
+		toMailAdmin = conf.getSmtpToMailAdmin();
+		newUserMessage = conf.getSmtpNewUserMessage();
+		updateUserMessage = conf.getSmtpUpdateUserMessage();
 
 		if (StringUtils.isNotBlank(conf.getSmtpHost())) {
 			emailFactory.setSmtpHost(conf.getSmtpHost());
@@ -70,7 +70,7 @@ public class MailDispatcher {
 
 	public void sendRequestRegistrationMail(final UserView user) {
 		if (StringUtils.isNotBlank(newUserMessage)) {
-			final String message = prepareMessage(this.newUserMessage, user);
+			final String message = prepareMessage(newUserMessage, user);
 
 			for (final String to : StringUtils.split(toMailAdmin, ",")) {
 				sendMail("OpenOrgs: new registration request", message, to.trim());
@@ -82,7 +82,7 @@ public class MailDispatcher {
 
 	public void sendUpdatedUserMail(final UserView user) {
 		if (StringUtils.isNotBlank(updateUserMessage)) {
-			final String message = prepareMessage(this.updateUserMessage, user);
+			final String message = prepareMessage(updateUserMessage, user);
 			sendMail("OpenOrgs: user updated", message, user.getEmail());
 		} else {
 			log.warn("Template is empty (updateUserMessage)");
@@ -92,13 +92,13 @@ public class MailDispatcher {
 	private String prepareMessage(final String template, final UserView user) {
 		final String countries = user.getRole().equals(UserRole.ADMIN.toString()) ? "All" : StringUtils.join(user.getCountries(), ", ");
 
-		return template.replaceAll(":email:", user.getEmail())
-			.replaceAll(":fullname:", user.getFullname())
-			.replaceAll(":organization:", user.getOrganization())
-			.replaceAll(":refperson:", user.getReferencePerson())
-			.replaceAll(":reqmessage:", user.getRequestMessage())
-			.replaceAll(":role:", user.getRole())
-			.replaceAll(":countries:", countries);
+		return template.replaceAll(":email:", "" + user.getEmail())
+				.replaceAll(":fullname:", "" + user.getFullname())
+				.replaceAll(":organization:", "" + user.getOrganization())
+				.replaceAll(":refperson:", "" + user.getReferencePerson())
+				.replaceAll(":reqmessage:", "" + user.getRequestMessage())
+				.replaceAll(":role:", "" + user.getRole())
+				.replaceAll(":countries:", countries);
 
 	}
 

apps/dnet-orgs-database-application/src/main/resources/application.properties

@@ -49,6 +49,9 @@ openaire.api.https.proxy = 10.19.65.35
 
 openorgs.support.pages = { "Ask a question": "https://www.openaire.eu/support/helpdesk?view=ticket&layout=open", "FAQ": "https://www.openaire.eu/faqs" }
 
+openorgs.findDuplicatesByCountry.limit.default = 1000
+openorgs.findOrgsByStatusAndCountry.limit.default = 1000
+
 openaire.override.logout.url = 
 
 openaire.explore.organization.baseurl = https://explore.openaire.eu/search/organization?organizationId=%s

apps/dnet-orgs-database-application/src/main/resources/sql/importDedupEvents.sql

@@ -1,6 +1,10 @@
 CREATE OR REPLACE PROCEDURE import_dedup_events() LANGUAGE plpgsql AS $$
 BEGIN
 
+-- MAKE A WORKING COPY OF THE TABLE PREPARED BY THE DEDUP JOB
+DROP TABLE IF EXISTS tmp_dedup_events_work;
+CREATE TABLE tmp_dedup_events_work AS SELECT * FROM tmp_dedup_events;
+
 DELETE FROM oa_conflicts  WHERE created_by = 'dedupWf' and modified_by = 'dedupWf' and reltype = 'suggested';
 DELETE FROM oa_duplicates WHERE created_by = 'dedupWf' and modified_by = 'dedupWf' and reltype = 'suggested';
 DELETE FROM organizations WHERE created_by = 'dedupWf' and modified_by = 'dedupWf' and status = 'suggested'; 
@@ -10,51 +14,51 @@ DELETE FROM organizations WHERE created_by = 'dedupWf' and modified_by = 'dedupW
 UPDATE organizations SET id = 'pending_org_::'||MD5(id) WHERE status = 'suggested' AND id NOT LIKE 'pending_org_::%';
 
 -- FIX IMPORT DATA
-DELETE FROM tmp_dedup_events WHERE oa_original_id = '' OR oa_original_id IS NULL;
-UPDATE tmp_dedup_events SET local_id = oa_original_id  WHERE local_id = '' OR local_id IS NULL;
-UPDATE tmp_dedup_events SET oa_country = 'UNKNOWN'  WHERE oa_country = '' OR oa_country IS NULL;
-UPDATE tmp_dedup_events SET oa_name    = oa_acronym WHERE oa_name    = '' OR oa_name    IS NULL;
-DELETE FROM tmp_dedup_events WHERE oa_name = '' OR oa_name IS NULL;
+DELETE FROM tmp_dedup_events_work WHERE oa_original_id = '' OR oa_original_id IS NULL;
+UPDATE tmp_dedup_events_work SET local_id = oa_original_id  WHERE local_id = '' OR local_id IS NULL;
+UPDATE tmp_dedup_events_work SET oa_country = 'UNKNOWN'  WHERE oa_country = '' OR oa_country IS NULL;
+UPDATE tmp_dedup_events_work SET oa_name    = oa_acronym WHERE oa_name    = '' OR oa_name    IS NULL;
+DELETE FROM tmp_dedup_events_work WHERE oa_name = '' OR oa_name IS NULL;
 
 -- delete invalid relations (a raw org can not be suggested as duplicate and as new org)
-DELETE FROM tmp_dedup_events WHERE oa_original_id IN (
+DELETE FROM tmp_dedup_events_work WHERE oa_original_id IN (
 	SELECT oa_original_id 
-	FROM tmp_dedup_events 
+	FROM tmp_dedup_events_work 
 	GROUP BY oa_original_id HAVING count(oa_original_id) > 1) 
 AND (local_id = '' OR local_id is NULL OR local_id = oa_original_id)
 AND (group_id = '' OR group_id is NULL);
 
 -- delete invalid relations (a raw org can not be suggested to multiple orgs)
-DELETE FROM tmp_dedup_events WHERE oa_original_id IN (
+DELETE FROM tmp_dedup_events_work WHERE oa_original_id IN (
 	SELECT oa_original_id 
-	FROM tmp_dedup_events 
+	FROM tmp_dedup_events_work 
 	GROUP BY oa_original_id HAVING count(oa_original_id) > 1)
 AND local_id NOT LIKE 'openorgs____::%';
 
 -- delete invalid groups (only one row)
-DELETE FROM tmp_dedup_events WHERE group_id IN (
+DELETE FROM tmp_dedup_events_work WHERE group_id IN (
 	SELECT group_id 
-	FROM tmp_dedup_events GROUP BY group_id 
+	FROM tmp_dedup_events_work GROUP BY group_id 
 	HAVING count(*) = 1
 );
 
 -- IMPORT MISSING TERMS
-INSERT INTO id_types(val, name) SELECT distinct arr[2], arr[2] FROM (SELECT string_to_array(unnest(string_to_array(pid_list, '@@@')), '###') AS arr FROM tmp_dedup_events WHERE oa_original_id NOT LIKE 'openorgs\_\_\_\_::%') as c  ON CONFLICT DO NOTHING;
+INSERT INTO id_types(val, name) SELECT distinct arr[2], arr[2] FROM (SELECT string_to_array(unnest(string_to_array(pid_list, '@@@')), '###') AS arr FROM tmp_dedup_events_work WHERE oa_original_id NOT LIKE 'openorgs\_\_\_\_::%') as c  ON CONFLICT DO NOTHING;
 
 -- NEW ORGANIZATIONS (suggested)
 INSERT INTO organizations(id, name, country, status, ec_legalbody, ec_legalperson, ec_nonprofit, ec_researchorganization, ec_highereducation, ec_internationalorganizationeurinterests, ec_internationalorganization, ec_enterprise, ec_smevalidated, ec_nutscode, created_by, modified_by) 
 SELECT 'pending_org_::'||MD5(local_id), oa_name, oa_country, 'suggested', ec_legalbody, ec_legalperson, ec_nonprofit, ec_researchorganization, ec_highereducation, ec_internationalorganizationeurinterests, ec_internationalorganization, ec_enterprise, ec_smevalidated, ec_nutscode, 'dedupWf', 'dedupWf' 
-FROM tmp_dedup_events 
+FROM tmp_dedup_events_work 
 WHERE local_id NOT LIKE 'openorgs\_\_\_\_::%' AND local_id = oa_original_id
 ON CONFLICT DO NOTHING;
 
 INSERT INTO acronyms(id, acronym) 
-SELECT 'pending_org_::'||MD5(local_id), oa_acronym FROM tmp_dedup_events 
+SELECT 'pending_org_::'||MD5(local_id), oa_acronym FROM tmp_dedup_events_work 
 WHERE local_id NOT LIKE 'openorgs\_\_\_\_::%' AND local_id = oa_original_id AND oa_acronym IS NOT NULL AND oa_acronym != ''
 ON CONFLICT DO NOTHING;
 
 INSERT INTO urls(id, url)
-SELECT 'pending_org_::'||MD5(local_id), oa_url FROM tmp_dedup_events 
+SELECT 'pending_org_::'||MD5(local_id), oa_url FROM tmp_dedup_events_work 
 WHERE local_id NOT LIKE 'openorgs\_\_\_\_::%' AND local_id = oa_original_id AND oa_url IS NOT NULL AND oa_url != ''
 ON CONFLICT DO NOTHING;
 
@@ -62,7 +66,7 @@ INSERT INTO other_ids(id, otherid, type)
 SELECT 'pending_org_::'||MD5(local_id), arr[1] AS otherid, arr[2] AS type 
 FROM (
 	SELECT local_id, string_to_array(unnest(string_to_array(pid_list, '@@@')), '###') AS arr 
-	FROM tmp_dedup_events 
+	FROM tmp_dedup_events_work 
 	WHERE local_id NOT LIKE 'openorgs\_\_\_\_::%' AND local_id = oa_original_id
 ) as c 
 ON CONFLICT DO NOTHING;
@@ -70,7 +74,7 @@ ON CONFLICT DO NOTHING;
 -- NEW ORGANIZATIONS (raw)
 INSERT INTO organizations(id, name, country, status, ec_legalbody, ec_legalperson, ec_nonprofit, ec_researchorganization, ec_highereducation, ec_internationalorganizationeurinterests, ec_internationalorganization, ec_enterprise, ec_smevalidated, ec_nutscode, created_by, modified_by) 
 SELECT oa_original_id, oa_name, oa_country, 'raw', ec_legalbody, ec_legalperson, ec_nonprofit, ec_researchorganization, ec_highereducation, ec_internationalorganizationeurinterests, ec_internationalorganization, ec_enterprise, ec_smevalidated, ec_nutscode, 'dedupWf', 'dedupWf' 
-FROM tmp_dedup_events 
+FROM tmp_dedup_events_work 
 WHERE oa_original_id NOT LIKE 'openorgs\_\_\_\_::%'
 ON CONFLICT(id) DO UPDATE SET
 	(name, country, ec_legalbody, ec_legalperson, ec_nonprofit, ec_researchorganization, ec_highereducation, ec_internationalorganizationeurinterests, ec_internationalorganization, ec_enterprise, ec_smevalidated, ec_nutscode, modification_date, modified_by) = 
@@ -78,13 +82,13 @@ ON CONFLICT(id) DO UPDATE SET
 
 INSERT INTO acronyms(id, acronym) 
 SELECT oa_original_id, oa_acronym 
-FROM tmp_dedup_events 
+FROM tmp_dedup_events_work 
 WHERE oa_original_id NOT LIKE 'openorgs\_\_\_\_::%' AND oa_acronym IS NOT NULL AND oa_acronym != ''
 ON CONFLICT DO NOTHING;
 
 INSERT INTO urls(id, url) 
 SELECT oa_original_id, oa_url 
-FROM tmp_dedup_events 
+FROM tmp_dedup_events_work 
 WHERE oa_original_id NOT LIKE 'openorgs\_\_\_\_::%' AND oa_url IS NOT NULL AND oa_url != ''
 ON CONFLICT DO NOTHING;
 
@@ -93,7 +97,7 @@ SELECT oa_original_id, arr[1] AS otherid, arr[2] AS type
 FROM (
 	SELECT oa_original_id, 
 	string_to_array(unnest(string_to_array(pid_list, '@@@')), '###') AS arr 
-	FROM tmp_dedup_events 
+	FROM tmp_dedup_events_work 
 	WHERE oa_original_id NOT LIKE 'openorgs\_\_\_\_::%'
 ) as c  
 ON CONFLICT DO NOTHING;
@@ -101,23 +105,23 @@ ON CONFLICT DO NOTHING;
 -- DUPLICATES (relations to openorgs)
 INSERT INTO oa_duplicates (local_id, oa_original_id, oa_collectedfrom, created_by, modified_by)
 SELECT local_id, oa_original_id, oa_collectedfrom, 'dedupWf', 'dedupWf'
-FROM tmp_dedup_events
+FROM tmp_dedup_events_work
 WHERE local_id LIKE 'openorgs\_\_\_\_::%' AND oa_original_id NOT LIKE 'openorgs\_\_\_\_::%'
 ON CONFLICT DO NOTHING;
 
 -- DUPLICATES (relations to suggested)
 INSERT INTO oa_duplicates (local_id, oa_original_id, oa_collectedfrom, created_by, modified_by)
 SELECT 'pending_org_::'||MD5(local_id), oa_original_id, oa_collectedfrom, 'dedupWf', 'dedupWf'
-FROM tmp_dedup_events
+FROM tmp_dedup_events_work
 WHERE local_id NOT LIKE 'openorgs\_\_\_\_::%' AND oa_original_id NOT LIKE 'openorgs\_\_\_\_::%'
 ON CONFLICT DO NOTHING;
 
 -- CONFLICTS (I generate all the couples)
 CREATE TEMPORARY TABLE tmp_conflict_groups AS 
-	SELECT DISTINCT group_id as gid, local_id oid from tmp_dedup_events
+	SELECT DISTINCT group_id as gid, local_id oid from tmp_dedup_events_work
 	WHERE local_id LIKE 'openorgs\_\_\_\_::%' AND oa_original_id LIKE 'openorgs\_\_\_\_::%' AND local_id != oa_original_id AND group_id IS NOT NULL AND group_id != ''
 UNION 
-	SELECT DISTINCT group_id as gid, oa_original_id oid from tmp_dedup_events
+	SELECT DISTINCT group_id as gid, oa_original_id oid from tmp_dedup_events_work
 	WHERE local_id LIKE 'openorgs\_\_\_\_::%' AND oa_original_id LIKE 'openorgs\_\_\_\_::%' AND local_id != oa_original_id AND group_id IS NOT NULL AND group_id != '';
 
 INSERT INTO oa_conflicts (id1, id2, idgroup, created_by, modified_by) SELECT DISTINCT 

apps/pom.xml

@@ -3,7 +3,7 @@
 	<parent>
 		<groupId>eu.dnetlib.dhp</groupId>
 		<artifactId>dnet-applications</artifactId>
-		<version>3.4.5-SNAPSHOT</version>
+		<version>3.5.5-SNAPSHOT</version>
 		<relativePath>../pom.xml</relativePath>
 	</parent>
 

apps/scholexplorer-api/pom.xml

@@ -3,7 +3,7 @@
     <parent>
         <groupId>eu.dnetlib.dhp</groupId>
         <artifactId>apps</artifactId>
-        <version>3.4.5-SNAPSHOT</version>
+        <version>3.5.5-SNAPSHOT</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
 

cmd-line-apps/dhp-broker-client/pom.xml

@@ -4,7 +4,7 @@
 	<parent>
 		<groupId>eu.dnetlib.dhp</groupId>
 		<artifactId>cmd-line-apps</artifactId>
-		<version>3.4.5-SNAPSHOT</version>
+		<version>3.5.5-SNAPSHOT</version>
 		<relativePath>../</relativePath>
 	</parent>
 

cmd-line-apps/pom.xml

@@ -3,7 +3,7 @@
 	<parent>
 		<groupId>eu.dnetlib.dhp</groupId>
 		<artifactId>dnet-applications</artifactId>
-		<version>3.4.5-SNAPSHOT</version>
+		<version>3.5.5-SNAPSHOT</version>
 		<relativePath>../</relativePath>
 	</parent>
 

libs/dnet-apps-common/pom.xml

@@ -4,7 +4,7 @@
 	<parent>
 		<groupId>eu.dnetlib.dhp</groupId>
 		<artifactId>libs</artifactId>
-		<version>3.4.5-SNAPSHOT</version>
+		<version>3.5.5-SNAPSHOT</version>
 		<relativePath>../</relativePath>
 	</parent>
 

libs/dnet-broker-apps-common/pom.xml

@@ -4,7 +4,7 @@
 	<parent>
 		<groupId>eu.dnetlib.dhp</groupId>
 		<artifactId>libs</artifactId>
-		<version>3.4.5-SNAPSHOT</version>
+		<version>3.5.5-SNAPSHOT</version>
 		<relativePath>../</relativePath>
 	</parent>
 

libs/dnet-exporter-model/pom.xml

@@ -4,7 +4,7 @@
 	<parent>
 		<groupId>eu.dnetlib.dhp</groupId>
 		<artifactId>libs</artifactId>
-		<version>3.4.5-SNAPSHOT</version>
+		<version>3.5.5-SNAPSHOT</version>
 		<relativePath>../</relativePath>
 	</parent>
 

libs/dnet-exporter-model/src/main/java/eu/dnetlib/openaire/exporter/model/community/CommunityDetails.java

@@ -124,44 +124,48 @@ public class CommunityDetails extends CommunitySummary {
 	public String toString() {
 		final StringBuilder builder = new StringBuilder();
 		builder.append("CommunityDetails [\n\tcreationDate = ")
-			.append(creationDate)
-			.append(",\n\tlastUpdateDate = ")
-			.append(lastUpdateDate)
-			.append(",\n\tsubjects = ")
-			.append(subjects)
-			.append(",\n\tfos = ")
-			.append(fos)
-			.append(",\n\tsdg = ")
-			.append(sdg)
-			.append(",\n\tadvancedConstraints = ")
-			.append(advancedConstraints)
-			.append(",\n\tremoveConstraints = ")
-			.append(removeConstraints)
-			.append(",\n\totherZenodoCommunities = ")
-			.append(otherZenodoCommunities)
-			.append(",\n\tid = ")
-			.append(id)
-			.append(",\n\tqueryId = ")
-			.append(queryId)
-			.append(",\n\ttype = ")
-			.append(type)
-			.append(",\n\tname = ")
-			.append(name)
-			.append(",\n\tshortName = ")
-			.append(shortName)
-			.append(",\n\tdescription = ")
-			.append(description)
-			.append(",\n\tlogoUrl = ")
-			.append(logoUrl)
-			.append(",\n\tstatus = ")
-			.append(status)
-			.append(",\n\tzenodoCommunity = ")
-			.append(zenodoCommunity)
-			.append(",\n\tsuggestedAcknowledgements = ")
-			.append(suggestedAcknowledgements)
-			.append(",\n\tplan = ")
-			.append(plan)
-			.append("\n]");
+				.append(creationDate)
+				.append(",\n\tlastUpdateDate = ")
+				.append(lastUpdateDate)
+				.append(",\n\tsubjects = ")
+				.append(subjects)
+				.append(",\n\tfos = ")
+				.append(fos)
+				.append(",\n\tsdg = ")
+				.append(sdg)
+				.append(",\n\tadvancedConstraints = ")
+				.append(advancedConstraints)
+				.append(",\n\tremoveConstraints = ")
+				.append(removeConstraints)
+				.append(",\n\totherZenodoCommunities = ")
+				.append(otherZenodoCommunities)
+				.append(",\n\tid = ")
+				.append(id)
+				.append(",\n\tqueryId = ")
+				.append(queryId)
+				.append(",\n\ttype = ")
+				.append(type)
+				.append(",\n\tname = ")
+				.append(name)
+				.append(",\n\tshortName = ")
+				.append(shortName)
+				.append(",\n\tdisplayName = ")
+				.append(displayName)
+				.append(",\n\tdisplayShortName = ")
+				.append(displayShortName)
+				.append(",\n\tdescription = ")
+				.append(description)
+				.append(",\n\tlogoUrl = ")
+				.append(logoUrl)
+				.append(",\n\tstatus = ")
+				.append(status)
+				.append(",\n\tzenodoCommunity = ")
+				.append(zenodoCommunity)
+				.append(",\n\tsuggestedAcknowledgements = ")
+				.append(suggestedAcknowledgements)
+				.append(",\n\tplan = ")
+				.append(plan)
+				.append("\n]");
 		return builder.toString();
 	}
 

libs/dnet-exporter-model/src/main/java/eu/dnetlib/openaire/exporter/model/community/CommunityPlanType.java

@@ -0,0 +1,5 @@
+package eu.dnetlib.openaire.exporter.model.community;
+
+public enum CommunityPlanType {
+	Standard, Advanced, Premium, National, Default
+}

libs/dnet-exporter-model/src/main/java/eu/dnetlib/openaire/exporter/model/community/CommunitySummary.java

@@ -24,6 +24,12 @@ public class CommunitySummary {
 	@Schema(description = "community short name")
 	protected String shortName;
 
+	@Schema(description = "community name for display")
+	protected String displayName;
+
+	@Schema(description = "community short name for display")
+	protected String displayShortName;
+
 	@Schema(description = "community creation date")
 	protected LocalDateTime creationDate;
 
@@ -49,28 +55,32 @@ public class CommunitySummary {
 	protected String zenodoCommunity;
 
 	@Schema(description = "community plan")
-	protected String plan;
+	protected CommunityPlanType plan;
 
 	public CommunitySummary() {}
 
 	public CommunitySummary(
-		final String id,
-		final String queryId,
-		final CommunityType type,
-		final String name,
-		final String shortName,
-		final LocalDateTime creationDate,
-		final LocalDateTime lastUpdateDate,
-		final String description,
-		final String logoUrl,
-		final CommunityStatus status,
-		final String zenodoCommunity,
-		final String plan) {
+			final String id,
+			final String queryId,
+			final CommunityType type,
+			final String name,
+			final String shortName,
+			final String displayName,
+			final String displayShortName,
+			final LocalDateTime creationDate,
+			final LocalDateTime lastUpdateDate,
+			final String description,
+			final String logoUrl,
+			final CommunityStatus status,
+			final String zenodoCommunity,
+			final CommunityPlanType plan) {
 		this.id = id;
 		this.queryId = queryId;
 		this.type = type;
 		this.name = name;
 		this.shortName = shortName;
+		this.displayName = displayName;
+		this.displayShortName = displayShortName;
 		this.creationDate = creationDate;
 		this.lastUpdateDate = lastUpdateDate;
 		this.description = description;
@@ -82,17 +92,19 @@ public class CommunitySummary {
 
 	public CommunitySummary(final CommunitySummary summary) {
 		this(summary.getId(),
-			summary.getQueryId(),
-			summary.getType(),
-			summary.getName(),
-			summary.getShortName(),
-			summary.getCreationDate(),
-			summary.getLastUpdateDate(),
-			summary.getDescription(),
-			summary.getLogoUrl(),
-			summary.getStatus(),
-			summary.getZenodoCommunity(),
-			summary.getPlan());
+				summary.getQueryId(),
+				summary.getType(),
+				summary.getName(),
+				summary.getShortName(),
+				summary.getDisplayName(),
+				summary.getDisplayShortName(),
+				summary.getCreationDate(),
+				summary.getLastUpdateDate(),
+				summary.getDescription(),
+				summary.getLogoUrl(),
+				summary.getStatus(),
+				summary.getZenodoCommunity(),
+				summary.getPlan());
 	}
 
 	public String getId() {
@@ -135,6 +147,22 @@ public class CommunitySummary {
 		this.shortName = shortName;
 	}
 
+	public String getDisplayName() {
+		return displayName;
+	}
+
+	public void setDisplayName(final String displayName) {
+		this.displayName = displayName;
+	}
+
+	public String getDisplayShortName() {
+		return displayShortName;
+	}
+
+	public void setDisplayShortName(final String displayShortName) {
+		this.displayShortName = displayShortName;
+	}
+
 	public LocalDateTime getCreationDate() {
 		return creationDate;
 	}
@@ -199,11 +227,11 @@ public class CommunitySummary {
 		this.zenodoCommunity = zenodoCommunity;
 	}
 
-	public String getPlan() {
+	public CommunityPlanType getPlan() {
 		return plan;
 	}
 
-	public void setPlan(final String plan) {
+	public void setPlan(final CommunityPlanType plan) {
 		this.plan = plan;
 	}
 

libs/dnet-exporter-model/src/main/java/eu/dnetlib/openaire/exporter/model/community/CommunityWritableProperties.java

@@ -16,6 +16,12 @@ public class CommunityWritableProperties {
 	@Schema(description = "community short name")
 	private String shortName;
 
+	@Schema(description = "community name for display")
+	private String displayName;
+
+	@Schema(description = "community short name for display")
+	private String displayShortName;
+
 	@Schema(description = "community description")
 	private String description;
 
@@ -56,7 +62,7 @@ public class CommunityWritableProperties {
 	private CommunityClaimType claim;
 
 	@Schema(description = "community plan")
-	private String plan;
+	private CommunityPlanType plan;
 
 	public List<String> getFos() {
 		return fos;
@@ -98,6 +104,22 @@ public class CommunityWritableProperties {
 		this.shortName = shortName;
 	}
 
+	public String getDisplayName() {
+		return displayName;
+	}
+
+	public void setDisplayName(final String displayName) {
+		this.displayName = displayName;
+	}
+
+	public String getDisplayShortName() {
+		return displayShortName;
+	}
+
+	public void setDisplayShortName(final String displayShortName) {
+		this.displayShortName = displayShortName;
+	}
+
 	public String getDescription() {
 		return description;
 	}
@@ -178,11 +200,12 @@ public class CommunityWritableProperties {
 		this.otherZenodoCommunities = otherZenodoCommunities;
 	}
 
-	public String getPlan() {
+	public CommunityPlanType getPlan() {
 		return plan;
 	}
 
-	public void setPlan(final String plan) {
+	public void setPlan(final CommunityPlanType plan) {
 		this.plan = plan;
 	}
+
 }

libs/dnet-exporter-model/src/main/java/eu/dnetlib/openaire/exporter/model/context/IISConfigurationEntry.java

@@ -39,12 +39,14 @@ public class IISConfigurationEntry implements Serializable {
 	}
 
 	public void addParams(final String name, final String... values) {
-		if (StringUtils.isNoneBlank(name) && values != null) {
-			for (final String v : values) {
-				if (StringUtils.isNotBlank(v)) {
-					params.add(new Param().setName(name).setValue(v));
-				}
-			}
+		if (StringUtils.isBlank(name)) { return; }
+
+		if (values == null) {
+			params.add(new Param().setName(name).setValue(""));
+		}
+
+		for (final String v : values) {
+			params.add(new Param().setName(name).setValue(v != null ? v : ""));
 		}
 	}
 

libs/dnet-exporter-model/src/main/java/eu/dnetlib/openaire/exporter/model/dsm/AggregationHistoryResponseV2.java

@@ -9,6 +9,10 @@ public class AggregationHistoryResponseV2 extends Response {
 
 	private List<AggregationInfo> aggregationInfo;
 
+	public AggregationHistoryResponseV2() {
+		super();
+	}
+
 	public AggregationHistoryResponseV2(final List<AggregationInfo> aggregationInfo) {
 		super();
 		this.aggregationInfo = aggregationInfo;

libs/dnet-exporter-model/src/main/java/eu/dnetlib/openaire/exporter/model/dsm/AggregationInfo.java

@@ -1,6 +1,16 @@
 package eu.dnetlib.openaire.exporter.model.dsm;
 
-public abstract class AggregationInfo {
+import java.io.Serializable;
+
+import com.fasterxml.jackson.annotation.JsonAutoDetect;
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.annotation.JsonInclude.Include;
+
+@JsonAutoDetect
+@JsonInclude(Include.NON_NULL)
+public class AggregationInfo implements Serializable {
+
+	private static final long serialVersionUID = -4908395195618212510L;
 
 	private int numberOfRecords;
 
@@ -12,6 +22,8 @@ public abstract class AggregationInfo {
 
 	private boolean completedSuccessfully = true;
 
+	private CollectionMode collectionMode;
+
 	public AggregationInfo() {}
 
 	public int getNumberOfRecords() {
@@ -53,4 +65,13 @@ public abstract class AggregationInfo {
 	public void setCompletedSuccessfully(final boolean completedSuccessfully) {
 		this.completedSuccessfully = completedSuccessfully;
 	}
+
+	public CollectionMode getCollectionMode() {
+		return collectionMode;
+	}
+
+	public void setCollectionMode(final CollectionMode collectionMode) {
+		this.collectionMode = collectionMode;
+	}
+
 }

libs/dnet-exporter-model/src/main/java/eu/dnetlib/openaire/exporter/model/dsm/AggregationInfoV1.java

@@ -1,13 +1,67 @@
 package eu.dnetlib.openaire.exporter.model.dsm;
 
-import com.fasterxml.jackson.annotation.JsonIgnore;
+import java.io.Serializable;
 
-public class AggregationInfoV1 extends AggregationInfo {
+import com.fasterxml.jackson.annotation.JsonAutoDetect;
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.annotation.JsonInclude.Include;
 
-	@Override
-	@JsonIgnore
-	public boolean isCompletedSuccessfully() {
-		return super.isCompletedSuccessfully();
+@JsonAutoDetect
+@JsonInclude(Include.NON_NULL)
+public class AggregationInfoV1 implements Serializable {
+
+	private static final long serialVersionUID = 7333873121568523946L;
+
+	private int numberOfRecords;
+
+	private String date;
+
+	private AggregationStage aggregationStage;
+
+	private boolean indexedVersion = false;
+
+	private CollectionMode collectionMode;
+
+	public AggregationInfoV1() {}
+
+	public int getNumberOfRecords() {
+		return numberOfRecords;
+	}
+
+	public void setNumberOfRecords(final int numberOfRecords) {
+		this.numberOfRecords = numberOfRecords;
+	}
+
+	public String getDate() {
+		return date;
+	}
+
+	public void setDate(final String date) {
+		this.date = date;
+	}
+
+	public AggregationStage getAggregationStage() {
+		return aggregationStage;
+	}
+
+	public void setAggregationStage(final AggregationStage aggregationStage) {
+		this.aggregationStage = aggregationStage;
+	}
+
+	public boolean isIndexedVersion() {
+		return indexedVersion;
+	}
+
+	public void setIndexedVersion(final boolean indexedVersion) {
+		this.indexedVersion = indexedVersion;
+	}
+
+	public CollectionMode getCollectionMode() {
+		return collectionMode;
+	}
+
+	public void setCollectionMode(final CollectionMode collectionMode) {
+		this.collectionMode = collectionMode;
 	}
 
 }

libs/dnet-exporter-model/src/main/java/eu/dnetlib/openaire/exporter/model/dsm/CollectionInfoV1.java

@@ -1,22 +0,0 @@
-package eu.dnetlib.openaire.exporter.model.dsm;
-
-import com.fasterxml.jackson.annotation.JsonAutoDetect;
-
-/**
- * Created by claudio on 29/11/2016.
- */
-@JsonAutoDetect
-@Deprecated
-public class CollectionInfoV1 extends AggregationInfoV1 {
-
-	private CollectionMode collectionMode;
-
-	public CollectionMode getCollectionMode() {
-		return collectionMode;
-	}
-
-	public void setCollectionMode(final CollectionMode collectionMode) {
-		this.collectionMode = collectionMode;
-	}
-
-}

libs/dnet-exporter-model/src/main/java/eu/dnetlib/openaire/exporter/model/dsm/CollectionInfoV2.java

@@ -1,21 +0,0 @@
-package eu.dnetlib.openaire.exporter.model.dsm;
-
-import com.fasterxml.jackson.annotation.JsonAutoDetect;
-
-/**
- * Created by claudio on 29/11/2016.
- */
-@JsonAutoDetect
-public class CollectionInfoV2 extends AggregationInfo {
-
-	private CollectionMode collectionMode;
-
-	public CollectionMode getCollectionMode() {
-		return collectionMode;
-	}
-
-	public void setCollectionMode(final CollectionMode collectionMode) {
-		this.collectionMode = collectionMode;
-	}
-
-}

libs/dnet-exporter-model/src/main/java/eu/dnetlib/openaire/exporter/model/dsm/TransformationInfoV1.java

@@ -1,12 +0,0 @@
-package eu.dnetlib.openaire.exporter.model.dsm;
-
-import com.fasterxml.jackson.annotation.JsonAutoDetect;
-
-/**
- * Created by claudio on 29/11/2016.
- */
-@JsonAutoDetect
-@Deprecated
-public class TransformationInfoV1 extends AggregationInfoV1 {
-
-}

libs/dnet-exporter-model/src/main/java/eu/dnetlib/openaire/exporter/model/dsm/TransformationInfoV2.java

@@ -1,11 +0,0 @@
-package eu.dnetlib.openaire.exporter.model.dsm;
-
-import com.fasterxml.jackson.annotation.JsonAutoDetect;
-
-/**
- * Created by claudio on 29/11/2016.
- */
-@JsonAutoDetect
-public class TransformationInfoV2 extends AggregationInfo {
-
-}

libs/dnet-exporter-model/src/test/java/eu/dnetlib/openaire/exporter/model/dsm/DatasourceApiClientTest.java

@@ -0,0 +1,25 @@
+package eu.dnetlib.openaire.exporter.model.dsm;
+
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import org.apache.commons.io.IOUtils;
+import org.junit.jupiter.api.Test;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+
+public class DatasourceApiClientTest {
+
+	@Test
+	public void testaAgregationHistoryV2_local() throws Exception {
+
+		final ObjectMapper mapper = new ObjectMapper();
+
+		final AggregationHistoryResponseV2 res =
+				mapper.readValue(IOUtils.toString(getClass().getResourceAsStream("aggregation-info-v2.json"), "UTF-8"), AggregationHistoryResponseV2.class);
+
+		System.out.println(mapper.writeValueAsString(res));
+
+		assertTrue(res.getAggregationInfo().size() > 0);
+	}
+
+}

libs/dnet-exporter-model/src/test/resources/eu/dnetlib/openaire/exporter/model/dsm/aggregation-info-v2.json

@@ -0,0 +1,27 @@
+{
+  "header": {
+    "total": 100,
+    "page": 0,
+    "size": 100,
+    "time": 0,
+    "statusCode": 0,
+    "errors": []
+  },
+  "aggregationInfo": [
+    {
+      "numberOfRecords": 739,
+      "date": "2024-03-14",
+      "aggregationStage": "TRANSFORM",
+      "indexedVersion": false,
+      "completedSuccessfully": true
+    },
+    {
+      "numberOfRecords": 822,
+      "date": "2024-03-14",
+      "aggregationStage": "COLLECT",
+      "indexedVersion": false,
+      "completedSuccessfully": true,
+      "collectionMode": "REFRESH"
+    }
+  ]
+}

libs/dnet-openaire-broker-common/pom.xml

@@ -3,7 +3,7 @@
 	<parent>
 		<groupId>eu.dnetlib.dhp</groupId>
 		<artifactId>libs</artifactId>
-		<version>3.4.5-SNAPSHOT</version>
+		<version>3.5.5-SNAPSHOT</version>
 		<relativePath>../</relativePath>
 	</parent>
 

libs/pom.xml

@@ -4,7 +4,7 @@
 	<parent>
 		<groupId>eu.dnetlib.dhp</groupId>
 		<artifactId>dnet-applications</artifactId>
-		<version>3.4.5-SNAPSHOT</version>
+		<version>3.5.5-SNAPSHOT</version>
 		<relativePath>../</relativePath>
 	</parent>
 

pom.xml

@@ -10,7 +10,7 @@
 	<modelVersion>4.0.0</modelVersion>
 	<groupId>eu.dnetlib.dhp</groupId>
 	<artifactId>dnet-applications</artifactId>
-	<version>3.4.5-SNAPSHOT</version>
+	<version>3.5.5-SNAPSHOT</version>
 	<packaging>pom</packaging>
 
 	<licenses>