Refactors external url fetching to support more complex APIs and adds requested external url on "Grant". (Issue #179)

dmp-backend/data/src/main/java/eu/eudat/data/dao/entities/GrantDaoImpl.java

@@ -11,6 +11,7 @@ import eu.eudat.types.grant.GrantStateType;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.scheduling.annotation.Async;
 import org.springframework.stereotype.Component;
+import schemasMicrosoftComOfficeOffice.LeftDocument;
 
 import javax.persistence.criteria.JoinType;
 import java.util.Date;
@@ -51,6 +52,8 @@ public class GrantDaoImpl extends DatabaseAccess<Grant> implements GrantDao {
         }
         if (criteria.getFunderId() != null && !criteria.getFunderId().trim().isEmpty())
             query.where((builder, root) -> builder.equal(root.get("funder").get("id"), UUID.fromString(criteria.getFunderId())));
+        if (criteria.getFunderReference() != null && !criteria.getFunderReference().isEmpty())
+            query.where((builder, root) -> builder.equal(root.join("funder", JoinType.LEFT).get("reference"), criteria.getFunderReference()));
         query.where((builder, root) -> builder.notEqual(root.get("status"), Grant.Status.DELETED.getValue()));
         return query;
     }

dmp-backend/web/src/main/java/eu/eudat/controllers/Organisations.java

@@ -2,6 +2,7 @@ package eu.eudat.controllers;
 
 import eu.eudat.data.query.items.table.organisations.OrganisationsTableRequest;
 import eu.eudat.logic.managers.OrganisationsManager;
+import eu.eudat.logic.proxy.config.ExternalUrlCriteria;
 import eu.eudat.logic.proxy.config.exceptions.HugeResultSet;
 import eu.eudat.logic.proxy.config.exceptions.NoURLFound;
 import eu.eudat.logic.services.ApiContext;
@@ -38,7 +39,8 @@ public class Organisations extends BaseController {
     ResponseEntity<ResponseItem<OrganisationsExternalSourcesModel>> listExternalOrganisations(
             @RequestParam(value = "query", required = false) String query, @RequestParam(value = "type", required = false) String type
     ) throws HugeResultSet, NoURLFound {
-        List<Map<String, String>> remoteRepos = this.getApiContext().getOperationsContext().getRemoteFetcher().getOrganisations(query, type);
+        ExternalUrlCriteria externalUrlCriteria = new ExternalUrlCriteria(query);
+        List<Map<String, String>> remoteRepos = this.getApiContext().getOperationsContext().getRemoteFetcher().getOrganisations(externalUrlCriteria, type);
         OrganisationsExternalSourcesModel organisationsExternalSourcesModel = new OrganisationsExternalSourcesModel().fromExternalItem(remoteRepos);
         return ResponseEntity.status(HttpStatus.OK).body(new ResponseItem<OrganisationsExternalSourcesModel>().payload(organisationsExternalSourcesModel).status(ApiMessageCode.NO_MESSAGE));
     }

dmp-backend/web/src/main/java/eu/eudat/controllers/TagController.java

@@ -5,6 +5,7 @@ import com.fasterxml.jackson.databind.ObjectMapper;
 import eu.eudat.elastic.criteria.TagCriteria;
 import eu.eudat.elastic.entities.Dataset;
 import eu.eudat.elastic.repository.Repository;
+import eu.eudat.logic.proxy.config.ExternalUrlCriteria;
 import eu.eudat.logic.proxy.config.exceptions.HugeResultSet;
 import eu.eudat.logic.proxy.config.exceptions.NoURLFound;
 import eu.eudat.logic.services.ApiContext;
@@ -47,7 +48,8 @@ public class TagController extends BaseController {
     public @ResponseBody
     ResponseEntity<ResponseItem<TagExternalSourcesModel>> listExternalTagModel(
             @RequestParam(value = "query", required = false) String query, @RequestParam(value = "type", required = false) String type) throws HugeResultSet, NoURLFound, IOException {
-        List<Map<String, String>> remoteRepos = this.getApiContext().getOperationsContext().getRemoteFetcher().getTags(query, type);
+        ExternalUrlCriteria externalUrlCriteria = new ExternalUrlCriteria(query);
+        List<Map<String, String>> remoteRepos = this.getApiContext().getOperationsContext().getRemoteFetcher().getTags(externalUrlCriteria, type);
         TagExternalSourcesModel researchersExternalSourcesModel = new TagExternalSourcesModel().fromExternalItem(remoteRepos);
 
 //        ObjectMapper mapper = new ObjectMapper();

dmp-backend/web/src/main/java/eu/eudat/logic/managers/DataManagementPlanManager.java

@@ -659,7 +659,6 @@ public class DataManagementPlanManager {
             eu.eudat.data.entities.Funder funderEntity = funderDao.getWithCritetia(criteria).getSingleOrDefault();
             if (funderEntity != null) funder.setId(funderEntity.getId());
             else {
-                // funder.setType(Funder.FunderType.EXTERNAL.getValue());
                 funderDao.createOrUpdate(funder);
             }
         }

dmp-backend/web/src/main/java/eu/eudat/logic/managers/DataRepositoryManager.java

@@ -4,6 +4,7 @@ import com.fasterxml.jackson.databind.DeserializationFeature;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import eu.eudat.data.dao.criteria.DataRepositoryCriteria;
 import eu.eudat.data.entities.DataRepository;
+import eu.eudat.logic.proxy.config.ExternalUrlCriteria;
 import eu.eudat.logic.proxy.config.exceptions.HugeResultSet;
 import eu.eudat.logic.proxy.config.exceptions.NoURLFound;
 import eu.eudat.logic.services.ApiContext;
@@ -35,7 +36,8 @@ public class DataRepositoryManager {
     }
 
     public List<DataRepositoryModel> getDataRepositories(String query, String type, Principal principal) throws HugeResultSet, NoURLFound {
-        List<Map<String, String>> remoteRepos = this.apiContext.getOperationsContext().getRemoteFetcher().getRepositories(query, type);
+        ExternalUrlCriteria externalUrlCriteria = new ExternalUrlCriteria(query);
+        List<Map<String, String>> remoteRepos = this.apiContext.getOperationsContext().getRemoteFetcher().getRepositories(externalUrlCriteria, type);
 
         DataRepositoryCriteria criteria = new DataRepositoryCriteria();
         if (!query.isEmpty()) criteria.setLike(query);

dmp-backend/web/src/main/java/eu/eudat/logic/managers/FunderManager.java

@@ -2,6 +2,7 @@ package eu.eudat.logic.managers;
 
 import eu.eudat.data.query.items.item.funder.FunderCriteriaRequest;
 import eu.eudat.logic.builders.model.models.FunderBuilder;
+import eu.eudat.logic.proxy.config.ExternalUrlCriteria;
 import eu.eudat.logic.proxy.config.exceptions.HugeResultSet;
 import eu.eudat.logic.proxy.config.exceptions.NoURLFound;
 import eu.eudat.logic.proxy.fetching.RemoteFetcher;
@@ -37,7 +38,8 @@ public class FunderManager {
 		QueryableList<eu.eudat.data.entities.Funder> items = apiContext.getOperationsContext().getDatabaseRepository().getFunderDao().getWithCritetia(funderCriteria.getCriteria());
 		QueryableList<eu.eudat.data.entities.Funder> authItems = apiContext.getOperationsContext().getDatabaseRepository().getFunderDao().getAuthenticated(items, userInfo);
 		List<Funder> funders = authItems.select(item -> new eu.eudat.models.data.funder.Funder().fromDataModel(item));
-		List<Map<String, String>> remoteRepos = remoteFetcher.getFunders(funderCriteria.getCriteria().getLike());
+		ExternalUrlCriteria externalUrlCriteria = new ExternalUrlCriteria(funderCriteria.getCriteria().getLike());
+		List<Map<String, String>> remoteRepos = remoteFetcher.getFunders(externalUrlCriteria);
 		FundersExternalSourcesModel fundersExternalSourcesModel = new FundersExternalSourcesModel().fromExternalItem(remoteRepos);
 		for (ExternalSourcesItemModel externalListingItem : fundersExternalSourcesModel) {
 			eu.eudat.models.data.funder.Funder funder = apiContext.getOperationsContext().getBuilderFactory().getBuilder(FunderBuilder.class)

dmp-backend/web/src/main/java/eu/eudat/logic/managers/GrantManager.java

@@ -10,7 +10,9 @@ import eu.eudat.data.dao.entities.GrantDao;
 import eu.eudat.data.entities.Content;
 import eu.eudat.data.entities.DMP;
 import eu.eudat.exceptions.files.TempFileNotFoundException;
+import eu.eudat.logic.proxy.config.ExternalUrlCriteria;
 import eu.eudat.logic.services.operations.DatabaseRepository;
+import eu.eudat.logic.utilities.helpers.ListHelper;
 import eu.eudat.models.data.external.ExternalSourcesItemModel;
 import eu.eudat.models.data.external.GrantsExternalSourcesModel;
 import eu.eudat.models.data.files.ContentFile;
@@ -34,6 +36,7 @@ import java.util.List;
 import java.util.Map;
 import java.util.UUID;
 import java.util.concurrent.CompletableFuture;
+import java.util.stream.Collectors;
 
 @Component
 public class GrantManager {
@@ -122,7 +125,12 @@ public class GrantManager {
         QueryableList<eu.eudat.data.entities.Grant> items = apiContext.getOperationsContext().getDatabaseRepository().getGrantDao().getWithCriteria(grantCriteria.getCriteria());
         QueryableList<eu.eudat.data.entities.Grant> authItems = apiContext.getOperationsContext().getDatabaseRepository().getGrantDao().getAuthenticated(items, userInfo);
         List<eu.eudat.models.data.grant.Grant> grants = authItems.select(item -> new Grant().fromDataModel(item));
-        List<Map<String, String>> remoteRepos = remoteFetcher.getGrants(grantCriteria.getCriteria().getLike());
+
+        ExternalUrlCriteria externalUrlCriteria = new ExternalUrlCriteria(grantCriteria.getCriteria().getLike());
+        if (externalUrlCriteria.getPage() == null) externalUrlCriteria.setPage("0");
+        if (grantCriteria.getCriteria().getFunderReference() != null) externalUrlCriteria.setFunderId(grantCriteria.getCriteria().getFunderReference());
+        List<Map<String, String>> remoteRepos = remoteFetcher.getGrants(externalUrlCriteria);
+
         GrantsExternalSourcesModel grantsExternalSourcesModel = new GrantsExternalSourcesModel().fromExternalItem(remoteRepos);
         for (ExternalSourcesItemModel externalListingItem : grantsExternalSourcesModel) {
             eu.eudat.models.data.grant.Grant grant = apiContext.getOperationsContext().getBuilderFactory().getBuilder(GrantBuilder.class)
@@ -135,6 +143,7 @@ public class GrantManager {
             grants.add(grant);
         }
         grants.sort(Comparator.comparing(Grant::getLabel));
+        grants = grants.stream().filter(ListHelper.distinctByKey(Grant::getLabel)).collect(Collectors.toList());
         return grants;
     }
 

dmp-backend/web/src/main/java/eu/eudat/logic/managers/ProjectManager.java

@@ -1,5 +1,7 @@
 package eu.eudat.logic.managers;
 
+import eu.eudat.logic.proxy.config.ExternalUrlCriteria;
+import eu.eudat.logic.utilities.helpers.ListHelper;
 import eu.eudat.models.data.external.ProjectsExternalSourcesModel;
 import eu.eudat.models.data.project.Project;
 import eu.eudat.data.query.items.item.project.ProjectCriteriaRequest;
@@ -16,6 +18,7 @@ import org.springframework.stereotype.Component;
 import java.util.Comparator;
 import java.util.List;
 import java.util.Map;
+import java.util.stream.Collectors;
 
 @Component
 public class ProjectManager {
@@ -35,7 +38,8 @@ public class ProjectManager {
 		QueryableList<eu.eudat.data.entities.Project> items = apiContext.getOperationsContext().getDatabaseRepository().getProjectDao().getWithCritetia(projectCriteria.getCriteria());
 		QueryableList<eu.eudat.data.entities.Project> authItems = apiContext.getOperationsContext().getDatabaseRepository().getProjectDao().getAuthenticated(items, userInfo);
 		List<Project> projects = authItems.select(item -> new Project().fromDataModel(item));
-		List<Map<String, String>> remoteRepos = remoteFetcher.getProjects(projectCriteria.getCriteria().getLike());
+		ExternalUrlCriteria externalUrlCriteria = new ExternalUrlCriteria(projectCriteria.getCriteria().getLike());
+		List<Map<String, String>> remoteRepos = remoteFetcher.getProjects(externalUrlCriteria);
 		ProjectsExternalSourcesModel projectsExternalSourcesModel = new ProjectsExternalSourcesModel().fromExternalItem(remoteRepos);
 		for (ExternalSourcesItemModel externalListingItem : projectsExternalSourcesModel) {
 			eu.eudat.models.data.project.Project project = apiContext.getOperationsContext().getBuilderFactory().getBuilder(ProjectBuilder.class)
@@ -48,6 +52,7 @@ public class ProjectManager {
 			projects.add(project);
 		}
 		projects.sort(Comparator.comparing(Project::getLabel));
+		projects = projects.stream().filter(ListHelper.distinctByKey(Project::getLabel)).collect(Collectors.toList());
 		return projects;
 	}
 }

dmp-backend/web/src/main/java/eu/eudat/logic/managers/RegistryManager.java

@@ -4,6 +4,7 @@ import com.fasterxml.jackson.databind.DeserializationFeature;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import eu.eudat.data.dao.criteria.RegistryCriteria;
 import eu.eudat.data.entities.Registry;
+import eu.eudat.logic.proxy.config.ExternalUrlCriteria;
 import eu.eudat.logic.proxy.config.exceptions.HugeResultSet;
 import eu.eudat.logic.proxy.config.exceptions.NoURLFound;
 import eu.eudat.logic.services.ApiContext;
@@ -33,7 +34,8 @@ public class RegistryManager {
     }
 
     public List<RegistryModel> getRegistries(String query, String type, Principal principal) throws HugeResultSet, NoURLFound {
-        List<Map<String, String>> remoteRepos = this.apiContext.getOperationsContext().getRemoteFetcher().getRegistries(query, type);
+        ExternalUrlCriteria externalUrlCriteria = new ExternalUrlCriteria(query);
+        List<Map<String, String>> remoteRepos = this.apiContext.getOperationsContext().getRemoteFetcher().getRegistries(externalUrlCriteria, type);
 
         RegistryCriteria criteria = new RegistryCriteria();
         if (!query.isEmpty()) criteria.setLike(query);

dmp-backend/web/src/main/java/eu/eudat/logic/managers/ResearcherManager.java

@@ -2,6 +2,7 @@ package eu.eudat.logic.managers;
 
 import eu.eudat.logic.builders.model.models.ResearcherBuilder;
 import eu.eudat.data.entities.Researcher;
+import eu.eudat.logic.proxy.config.ExternalUrlCriteria;
 import eu.eudat.models.data.external.ExternalSourcesItemModel;
 import eu.eudat.models.data.external.ResearchersExternalSourcesModel;
 import eu.eudat.data.query.items.item.researcher.ResearcherCriteriaRequest;
@@ -45,7 +46,8 @@ public class ResearcherManager {
         QueryableList<eu.eudat.data.entities.Researcher> items = apiContext.getOperationsContext().getDatabaseRepository().getResearcherDao().getWithCriteria(researcherCriteriaRequest.getCriteria());
         items.where((builder, root) -> builder.equal(root.get("creationUser").get("id"), principal.getId()));
         List<eu.eudat.models.data.dmp.Researcher> researchers = items.select(item -> new eu.eudat.models.data.dmp.Researcher().fromDataModel(item));
-        List<Map<String, String>> remoteRepos = remoteFetcher.getResearchers(researcherCriteriaRequest.getCriteria().getName(),null);
+        ExternalUrlCriteria externalUrlCriteria = new ExternalUrlCriteria(researcherCriteriaRequest.getCriteria().getName());
+        List<Map<String, String>> remoteRepos = remoteFetcher.getResearchers(externalUrlCriteria,null);
         ResearchersExternalSourcesModel researchersExternalSourcesModel = new ResearchersExternalSourcesModel().fromExternalItem(remoteRepos);
         for (ExternalSourcesItemModel externalListingItem : researchersExternalSourcesModel) {
             eu.eudat.models.data.dmp.Researcher researcher = apiContext.getOperationsContext().getBuilderFactory().getBuilder(ResearcherBuilder.class)

dmp-backend/web/src/main/java/eu/eudat/logic/managers/ServiceManager.java

@@ -4,6 +4,7 @@ import com.fasterxml.jackson.databind.DeserializationFeature;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import eu.eudat.data.dao.criteria.ServiceCriteria;
 import eu.eudat.data.entities.Service;
+import eu.eudat.logic.proxy.config.ExternalUrlCriteria;
 import eu.eudat.logic.proxy.config.exceptions.HugeResultSet;
 import eu.eudat.logic.proxy.config.exceptions.NoURLFound;
 import eu.eudat.logic.services.ApiContext;
@@ -36,7 +37,8 @@ public class ServiceManager {
     }
 
     public List<ServiceModel> getServices(String query, String type, Principal principal) throws HugeResultSet, NoURLFound {
-        List<Map<String, String>> remoteRepos = this.apiContext.getOperationsContext().getRemoteFetcher().getServices(query, type);
+        ExternalUrlCriteria externalUrlCriteria = new ExternalUrlCriteria(query);
+        List<Map<String, String>> remoteRepos = this.apiContext.getOperationsContext().getRemoteFetcher().getServices(externalUrlCriteria, type);
 
         ServiceCriteria criteria = new ServiceCriteria();
         if (!query.isEmpty()) criteria.setLike(query);

dmp-backend/web/src/main/java/eu/eudat/logic/proxy/config/ExternalUrlCriteria.java

@@ -0,0 +1,43 @@
+package eu.eudat.logic.proxy.config;
+
+public class ExternalUrlCriteria {
+	private String like;
+	private String page;
+	private String pageSize;
+	private String funderId;
+
+	public String getLike() {
+		return like;
+	}
+	public void setLike(String like) {
+		this.like = like;
+	}
+
+	public String getPage() {
+		return page;
+	}
+	public void setPage(String page) {
+		this.page = page;
+	}
+
+	public String getPageSize() {
+		return pageSize;
+	}
+	public void setPageSize(String pageSize) {
+		this.pageSize = pageSize;
+	}
+
+	public String getFunderId() {
+		return funderId;
+	}
+	public void setFunderId(String funderId) {
+		this.funderId = funderId;
+	}
+
+	public ExternalUrlCriteria(String like) {
+		this.like = like;
+	}
+
+	public ExternalUrlCriteria() {
+	}
+}

dmp-backend/web/src/main/java/eu/eudat/logic/proxy/config/UrlConfiguration.java

@@ -8,18 +8,16 @@ public class UrlConfiguration {
     private String key;
     private String label;
     private Integer ordinal;
-    private DataSearchConfiguration search;
-    private DataPageConfiguration page;
     private String url;
     private DataUrlConfiguration data;
     private String type;
     private String paginationPath;
     private String contentType;
+    private String funderQuery;
 
     public String getKey() {
         return key;
     }
-
     @XmlElement(name = "key")
     public void setKey(String key) {
         this.key = key;
@@ -28,7 +26,6 @@ public class UrlConfiguration {
     public String getLabel() {
         return label;
     }
-
     @XmlElement(name = "label")
     public void setLabel(String label) {
         this.label = label;
@@ -37,7 +34,6 @@ public class UrlConfiguration {
     public String getUrl() {
         return url;
     }
-
     @XmlElement(name = "url")
     public void setUrl(String url) {
         this.url = url;
@@ -46,34 +42,14 @@ public class UrlConfiguration {
     public Integer getOrdinal() {
         return ordinal;
     }
-
     @XmlElement(name = "ordinal")
     public void setOrdinal(Integer ordinal) {
         this.ordinal = ordinal;
     }
 
-    public DataSearchConfiguration getSearch() {
-        return search;
-    }
-
-    @XmlElement(name = "search")
-    public void setSearch(DataSearchConfiguration search) {
-        this.search = search;
-    }
-
-    public DataPageConfiguration getPage() {
-        return page;
-    }
-
-    @XmlElement(name = "page")
-    public void setPage(DataPageConfiguration page) {
-        this.page = page;
-    }
-
     public DataUrlConfiguration getData() {
         return data;
     }
-
     @XmlElement(name = "data")
     public void setData(DataUrlConfiguration data) {
         this.data = data;
@@ -82,7 +58,6 @@ public class UrlConfiguration {
     public String getPaginationPath() {
         return paginationPath;
     }
-
     @XmlElement(name = "paginationpath")
     public void setPaginationPath(String paginationPath) {
         this.paginationPath = paginationPath;
@@ -91,7 +66,6 @@ public class UrlConfiguration {
     public String getType() {
         return type;
     }
-
     @XmlElement(name = "type")
     public void setType(String type) {
         this.type = type;
@@ -100,9 +74,16 @@ public class UrlConfiguration {
     public String getContentType() {
         return contentType;
     }
-
     @XmlElement(name = "contenttype")
     public void setContentType(String contentType) {
         this.contentType = contentType;
     }
+
+    public String getFunderQuery() {
+        return funderQuery;
+    }
+    @XmlElement(name = "funderQuery")
+    public void setFunderQuery(String funderQuery) {
+        this.funderQuery = funderQuery;
+    }
 }

dmp-backend/web/src/main/java/eu/eudat/logic/proxy/fetching/RemoteFetcher.java

@@ -15,9 +15,11 @@ import org.springframework.stereotype.Service;
 
 import java.io.File;
 import java.io.IOException;
+import java.io.UnsupportedEncodingException;
 import java.net.HttpURLConnection;
 import java.net.MalformedURLException;
 import java.net.URL;
+import java.net.URLEncoder;
 import java.util.*;
 import java.util.stream.Collectors;
 
@@ -35,127 +37,157 @@ public class RemoteFetcher {
     }
 
     @Cacheable("repositories")
-    public List<Map<String, String>> getRepositories(String query, String key) throws NoURLFound, HugeResultSet {
+    public List<Map<String, String>> getRepositories(ExternalUrlCriteria externalUrlCriteria, String key) throws NoURLFound, HugeResultSet {
         List<UrlConfiguration> urlConfigs =
                 key != null && !key.isEmpty() ? configLoader.getExternalUrls().getRepositories().getUrls().stream().filter(item -> item.getKey().equals(key)).collect(Collectors.toList())
                         : configLoader.getExternalUrls().getRepositories().getUrls();
         FetchStrategy fetchStrategy = configLoader.getExternalUrls().getRepositories().getFetchMode();
-        return getAll(urlConfigs, fetchStrategy, query);
+        return getAll(urlConfigs, fetchStrategy, externalUrlCriteria);
     }
 
     @Cacheable("grants")
-    public List<Map<String, String>> getGrants(String query) throws NoURLFound, HugeResultSet {
+    public List<Map<String, String>> getGrants(ExternalUrlCriteria externalUrlCriteria) throws NoURLFound, HugeResultSet {
         List<UrlConfiguration> urlConfigs = configLoader.getExternalUrls().getGrants().getUrls();
         FetchStrategy fetchStrategy = configLoader.getExternalUrls().getGrants().getFetchMode();
-        return getAll(urlConfigs, fetchStrategy, query);
+        return getAll(urlConfigs, fetchStrategy, externalUrlCriteria);
     }
 
     @Cacheable("projects")
-    public List<Map<String, String>> getProjects(String query) throws NoURLFound, HugeResultSet {
+    public List<Map<String, String>> getProjects(ExternalUrlCriteria externalUrlCriteria) throws NoURLFound, HugeResultSet {
         List<UrlConfiguration> urlConfigs = configLoader.getExternalUrls().getProjects().getUrls();
         FetchStrategy fetchStrategy = configLoader.getExternalUrls().getProjects().getFetchMode();
-        return getAll(urlConfigs, fetchStrategy, query);
+        return getAll(urlConfigs, fetchStrategy, externalUrlCriteria);
     }
 
     @Cacheable("funders")
-    public List<Map<String, String>> getFunders(String query) throws NoURLFound, HugeResultSet {
+    public List<Map<String, String>> getFunders(ExternalUrlCriteria externalUrlCriteria) throws NoURLFound, HugeResultSet {
         List<UrlConfiguration> urlConfigs = configLoader.getExternalUrls().getFunders().getUrls();
         FetchStrategy fetchStrategy = configLoader.getExternalUrls().getFunders().getFetchMode();
-        return getAll(urlConfigs, fetchStrategy, query);
+        return getAll(urlConfigs, fetchStrategy, externalUrlCriteria);
     }
 
     @Cacheable("organisations")
-    public List<Map<String, String>> getOrganisations(String query, String key) throws NoURLFound, HugeResultSet {
+    public List<Map<String, String>> getOrganisations(ExternalUrlCriteria externalUrlCriteria, String key) throws NoURLFound, HugeResultSet {
         List<UrlConfiguration> urlConfigs =
                 key != null && !key.isEmpty() ? configLoader.getExternalUrls().getOrganisations().getUrls().stream().filter(item -> item.getKey().equals(key)).collect(Collectors.toList())
                         : configLoader.getExternalUrls().getOrganisations().getUrls();
         FetchStrategy fetchStrategy = configLoader.getExternalUrls().getOrganisations().getFetchMode();
-        return getAll(urlConfigs, fetchStrategy, query);
+        return getAll(urlConfigs, fetchStrategy, externalUrlCriteria);
     }
 
     @Cacheable("registries")
-    public List<Map<String, String>> getRegistries(String query, String key) throws NoURLFound, HugeResultSet {
+    public List<Map<String, String>> getRegistries(ExternalUrlCriteria externalUrlCriteria, String key) throws NoURLFound, HugeResultSet {
         List<UrlConfiguration> urlConfigs =
                 key != null && !key.isEmpty() ? configLoader.getExternalUrls().getRegistries().getUrls().stream().filter(item -> item.getKey().equals(key)).collect(Collectors.toList())
                         : configLoader.getExternalUrls().getRegistries().getUrls();
         FetchStrategy fetchStrategy = configLoader.getExternalUrls().getRegistries().getFetchMode();
-        return getAll(urlConfigs, fetchStrategy, query);
+        return getAll(urlConfigs, fetchStrategy, externalUrlCriteria);
     }
 
     @Cacheable("services")
-    public List<Map<String, String>> getServices(String query, String key) throws NoURLFound, HugeResultSet {
+    public List<Map<String, String>> getServices(ExternalUrlCriteria externalUrlCriteria, String key) throws NoURLFound, HugeResultSet {
         List<UrlConfiguration> urlConfigs =
                 key != null && !key.isEmpty() ? configLoader.getExternalUrls().getServices().getUrls().stream().filter(item -> item.getKey().equals(key)).collect(Collectors.toList())
                         : configLoader.getExternalUrls().getServices().getUrls();
         FetchStrategy fetchStrategy = configLoader.getExternalUrls().getServices().getFetchMode();
-        return getAll(urlConfigs, fetchStrategy, query);
+        return getAll(urlConfigs, fetchStrategy, externalUrlCriteria);
     }
 
     @Cacheable("researchers")
-    public List<Map<String, String>> getResearchers(String query, String key) throws NoURLFound, HugeResultSet {
+    public List<Map<String, String>> getResearchers(ExternalUrlCriteria externalUrlCriteria, String key) throws NoURLFound, HugeResultSet {
         List<UrlConfiguration> urlConfigs =
                 key != null && !key.isEmpty() ? configLoader.getExternalUrls().getResearchers().getUrls().stream().filter(item -> item.getKey().equals(key)).collect(Collectors.toList())
                         : configLoader.getExternalUrls().getResearchers().getUrls();
         FetchStrategy fetchStrategy = configLoader.getExternalUrls().getResearchers().getFetchMode();
-        return getAll(urlConfigs, fetchStrategy, query);
+        return getAll(urlConfigs, fetchStrategy, externalUrlCriteria);
     }
 
     @Cacheable("tags")
-    public List<Map<String, String>> getTags(String query, String key) throws NoURLFound, HugeResultSet {
+    public List<Map<String, String>> getTags(ExternalUrlCriteria externalUrlCriteria, String key) throws NoURLFound, HugeResultSet {
         List<UrlConfiguration> urlConfigs =
                 key != null && !key.isEmpty() ? configLoader.getExternalUrls().getTags().getUrls().stream().filter(item -> item.getKey().equals(key)).collect(Collectors.toList())
                         : configLoader.getExternalUrls().getTags().getUrls();
         FetchStrategy fetchStrategy = configLoader.getExternalUrls().getTags().getFetchMode();
-        return getAll(urlConfigs, fetchStrategy, query);
+        return getAll(urlConfigs, fetchStrategy, externalUrlCriteria);
     }
 
     @Cacheable("datasets")
-    public List<Map<String, String>> getDatasets(String query, String key) throws NoURLFound, HugeResultSet {
+    public List<Map<String, String>> getDatasets(ExternalUrlCriteria externalUrlCriteria, String key) throws NoURLFound, HugeResultSet {
         List<UrlConfiguration> urlConfigs =
                 key != null && !key.isEmpty() ? configLoader.getExternalUrls().getDatasets().getUrls().stream().filter(item -> item.getKey().equals(key)).collect(Collectors.toList())
                         : configLoader.getExternalUrls().getDatasets().getUrls();
         FetchStrategy fetchStrategy = configLoader.getExternalUrls().getDatasets().getFetchMode();
-        return getAll(urlConfigs, fetchStrategy, query);
+        return getAll(urlConfigs, fetchStrategy, externalUrlCriteria);
     }
 
 
-    private List<Map<String, String>> getAll(List<UrlConfiguration> urlConfigs, FetchStrategy fetchStrategy, String query) throws NoURLFound, HugeResultSet {
+    private List<Map<String, String>> getAll(List<UrlConfiguration> urlConfigs, FetchStrategy fetchStrategy, ExternalUrlCriteria externalUrlCriteria) throws NoURLFound, HugeResultSet {
 
         if (urlConfigs == null || urlConfigs.isEmpty())
             throw new NoURLFound("No Repository urls found in configuration");
 
-        Collections.sort(urlConfigs, Comparator.comparing(UrlConfiguration::getOrdinal));
+        urlConfigs.sort(Comparator.comparing(UrlConfiguration::getOrdinal));
         List<Map<String, String>> results = new LinkedList<>();
         for (UrlConfiguration urlConfig : urlConfigs) {
+            ifFunderQueryExist(urlConfig, externalUrlCriteria);
             if (urlConfig.getType() == null || urlConfig.getType().equals("External")) {
-                results.addAll(getAllResultsFromUrl(urlConfig.getUrl(), fetchStrategy, urlConfig.getData(), urlConfig.getPaginationPath(), query, urlConfig.getLabel(), urlConfig.getSearch(), urlConfig.getPage(), urlConfig.getContentType()));
+                results.addAll(getAllResultsFromUrl(urlConfig.getUrl(), fetchStrategy, urlConfig.getData(), urlConfig.getPaginationPath(), externalUrlCriteria, urlConfig.getLabel(), urlConfig.getContentType()));
             } else if (urlConfig.getType() != null && urlConfig.getType().equals("Internal")) {
-                results.addAll(getAllResultsFromMockUpJson(urlConfig.getUrl(), query));
+                results.addAll(getAllResultsFromMockUpJson(urlConfig.getUrl(), externalUrlCriteria.getLike()));
             }
         }
         return results;
     }
 
+    private void ifFunderQueryExist(UrlConfiguration urlConfiguration, ExternalUrlCriteria externalUrlCriteria) {
+        if (urlConfiguration.getFunderQuery() != null) {
+            if (externalUrlCriteria.getFunderId() != null) {
+                urlConfiguration.setUrl(urlConfiguration.getUrl().replace("{funderQuery}", urlConfiguration.getFunderQuery()));
+            }
+            else {
+                urlConfiguration.setUrl(urlConfiguration.getUrl().replace("{funderQuery}", ""));
+            }
+        }
+    }
 
-    private List<Map<String, String>> getAllResultsFromUrl(String path, FetchStrategy fetchStrategy, final DataUrlConfiguration jsonDataPath, final String jsonPaginationPath, String query, String key, DataSearchConfiguration search, DataPageConfiguration pageConfig, String contentType) throws HugeResultSet {
+    private String replaceCriteriaOnUrl(String path, ExternalUrlCriteria externalUrlCriteria) {
+        String completedPath = path;
+        if (externalUrlCriteria.getLike() != null) {
+            if (path.contains("openaire") && externalUrlCriteria.getLike().equals(""))
+                completedPath = completedPath.replaceAll("\\{like}", "*");
+            else
+                completedPath = completedPath.replaceAll("\\{like}", externalUrlCriteria.getLike());
+        } else {
+            completedPath = completedPath.replace("{like}", "");
+        }
+        if (externalUrlCriteria.getFunderId() != null) {
+            String funderId = externalUrlCriteria.getFunderId();
+            try {
+                funderId = URLEncoder.encode(externalUrlCriteria.getFunderId(), "UTF-8");
+            } catch (UnsupportedEncodingException e) {
+                e.printStackTrace();
+            }
+            completedPath = completedPath.replace("{funderId}", funderId);
+        }
+        if (externalUrlCriteria.getPage() != null) {
+            completedPath = completedPath.replace("{page}", externalUrlCriteria.getPage());
+        } else {
+            completedPath = completedPath.replace("{page}", "1");
+        }
+        if (externalUrlCriteria.getPageSize() != null) {
+            completedPath = completedPath.replace("{pageSize}", externalUrlCriteria.getPageSize());
+        } else {
+            completedPath = completedPath.replace("{pageSize}", "10");
+        }
+        return completedPath;
+    }
+
+    private List<Map<String, String>> getAllResultsFromUrl(String path, FetchStrategy fetchStrategy, final DataUrlConfiguration jsonDataPath, final String jsonPaginationPath, ExternalUrlCriteria externalUrlCriteria, String key, String contentType) throws HugeResultSet {
         Set<Integer> pages = new HashSet<>();
 
-        String searchQuery = "";
+        String replacedPath = replaceCriteriaOnUrl(path, externalUrlCriteria);
-        Results results = null;
-        if (search != null && search.getType() != null && !search.getType().trim().isEmpty() &&
-                search.getQueryParam() != null && !search.getQueryParam().trim().isEmpty() &&
-                query != null && !query.trim().isEmpty()) {
-            if (search.getType().equals("queryParam"))
-                searchQuery = "&" + search.getQueryParam() + "=" + query;
-        }
-
-        if (pageConfig != null) {
-            if (pageConfig.getType().equals("queryParam"))
-                results = getResultsFromUrl(path + "?" + pageConfig.getPageParam() + "=1" + searchQuery, jsonDataPath, jsonPaginationPath, contentType);
-        } else {
-            results = getResultsFromUrl(path, jsonDataPath, jsonPaginationPath, contentType);
-        }
 
+        Results results = getResultsFromUrl(replacedPath, jsonDataPath, jsonPaginationPath, contentType);
         if (fetchStrategy == FetchStrategy.FIRST)
             return results == null ? new LinkedList<>() : results.getResults().stream().peek(x -> x.put("tag", key)).collect(Collectors.toList());
 
@@ -165,17 +197,15 @@ public class RemoteFetcher {
 
         Long maxResults = configLoader.getExternalUrls().getMaxresults();
         if ((maxResults > 0 && results != null) && (results.getPagination().get("count") > maxResults))
-            throw new HugeResultSet("The submitted search query " + query + " is about to return " + results.getPagination().get("count") + " results... Please submit a more detailed search query");
+            throw new HugeResultSet("The submitted search query " + externalUrlCriteria.getLike() + " is about to return " + results.getPagination().get("count") + " results... Please submit a more detailed search query");
 
-        String finalSearchQuery = searchQuery;
         Optional<Results> optionalResults = pages.parallelStream()
-                .map(page -> getResultsFromUrl(path + "?page=" + page + finalSearchQuery, jsonDataPath, jsonPaginationPath, contentType))
+                .map(page -> getResultsFromUrl(path + "&page=" + page, jsonDataPath, jsonPaginationPath, contentType))
                 .reduce((result1, result2) -> {
                     result1.getResults().addAll(result2.getResults());
                     return result1;
                 });
         Results remainingResults = optionalResults.orElseGet(Results::new);
-
         remainingResults.getResults().addAll(results.getResults());
 
         return remainingResults.getResults().stream().peek(x -> x.put("tag", key)).collect(Collectors.toList());
@@ -203,7 +233,7 @@ public class RemoteFetcher {
                             + "," + jsonDataPath.getFieldsUrlConfiguration().getUri() + "," + jsonDataPath.getFieldsUrlConfiguration().getId()
                             + "," + jsonDataPath.getFieldsUrlConfiguration().getSource() + "]"),
                             jsonContext.read(jsonPaginationPath));
-                } else if (jsonDataPath.getFieldsUrlConfiguration().getCount() != null) { // parsing services.openair.eu
+                } else if (jsonDataPath.getFieldsUrlConfiguration().getCount() != null) { // parsing services.openaire.eu
                     results = new Results(jsonContext.read(jsonDataPath.getPath()
                             + "[" + jsonDataPath.getFieldsUrlConfiguration().getName()
                             + "," + jsonDataPath.getFieldsUrlConfiguration().getId() + "]"),
@@ -262,12 +292,12 @@ public class RemoteFetcher {
     }
 
     private String transformKey(DataUrlConfiguration dataUrlConfiguration, String key) {
-        if (dataUrlConfiguration.getFieldsUrlConfiguration().getId() != null &&  key.equals(dataUrlConfiguration.getFieldsUrlConfiguration().getId().replace("'",""))) return "pid";
+        if (dataUrlConfiguration.getFieldsUrlConfiguration().getId() != null && key.equals(dataUrlConfiguration.getFieldsUrlConfiguration().getId().replace("'",""))) return "pid";
         if (dataUrlConfiguration.getFieldsUrlConfiguration().getDescription() != null && key.equals(dataUrlConfiguration.getFieldsUrlConfiguration().getDescription().replace("'",""))) return "description";
-        if (dataUrlConfiguration.getFieldsUrlConfiguration().getUri() != null &&  key.equals(dataUrlConfiguration.getFieldsUrlConfiguration().getUri().replace("'",""))) return "uri";
+        if (dataUrlConfiguration.getFieldsUrlConfiguration().getUri() != null && key.equals(dataUrlConfiguration.getFieldsUrlConfiguration().getUri().replace("'",""))) return "uri";
-        if (dataUrlConfiguration.getFieldsUrlConfiguration().getName() != null &&  key.equals(dataUrlConfiguration.getFieldsUrlConfiguration().getName().replace("'",""))) return "name";
+        if (dataUrlConfiguration.getFieldsUrlConfiguration().getName() != null && key.equals(dataUrlConfiguration.getFieldsUrlConfiguration().getName().replace("'",""))) return "name";
         if (dataUrlConfiguration.getFieldsUrlConfiguration().getSource() != null && key.equals(dataUrlConfiguration.getFieldsUrlConfiguration().getSource().replace("'",""))) return "source";
-        if (dataUrlConfiguration.getFieldsUrlConfiguration().getCount() != null &&  key.equals(dataUrlConfiguration.getFieldsUrlConfiguration().getCount().replace("'",""))) return "count";
+        if (dataUrlConfiguration.getFieldsUrlConfiguration().getCount() != null && key.equals(dataUrlConfiguration.getFieldsUrlConfiguration().getCount().replace("'",""))) return "count";
         return null;
     }
 

dmp-backend/web/src/main/java/eu/eudat/models/data/external/ProjectsExternalSourcesModel.java

@@ -1,19 +1,61 @@
 package eu.eudat.models.data.external;
 
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+
+import java.io.IOException;
 import java.util.List;
 import java.util.Map;
 
-public class ProjectsExternalSourcesModel extends ExternalListingItem<ProjectsExternalSourcesModel>{
+public class ProjectsExternalSourcesModel extends ExternalListingItem<ProjectsExternalSourcesModel> {
+	private static final ObjectMapper mapper = new ObjectMapper();
+
 	@Override
 	public ProjectsExternalSourcesModel fromExternalItem(List<Map<String, String>> values) {
 		for (Map<String, String> item : values) {
 			ExternalSourcesItemModel model = new ExternalSourcesItemModel();
-			model.setRemoteId(item.get("pid"));
+			try {
-			model.setUri(item.get("uri"));
+				JsonNode node = mapper.readTree(mapper.writeValueAsBytes(item));
-			model.setName(item.get("name"));
+				JsonNode name = node.get("name");
-			model.setDescription(item.get("description"));
+				if (name != null && !name.isNull() && name.isObject()) {
-			model.setTag(item.get("tag"));
+					model.setName(node.get("name").get("$").asText());
-			this.add(model);
+				} else {
+					model.setName(item.get("name"));
+				}
+
+				JsonNode pid = node.get("pid");
+				if (pid != null && !pid.isNull() && pid.isObject()) {
+					model.setRemoteId(node.get("pid").get("$").asText());
+				} else {
+					model.setRemoteId(item.get("pid"));
+				}
+
+				JsonNode uri = node.get("uri");
+				if (uri != null && !uri.isNull() && uri.isObject()) {
+					model.setUri(node.get("uri").get("$").asText());
+				} else {
+					model.setUri(item.get("uri"));
+				}
+
+				JsonNode description = node.get("description");
+				if (description != null && !description.isNull() && description.isObject()) {
+					model.setDescription(node.get("description").get("$").asText());
+				} else {
+					model.setDescription(item.get("description"));
+				}
+
+				JsonNode tag = node.get("tag");
+				if (tag != null && !tag.isNull() && tag.isObject()) {
+					model.setTag(node.get("tag").get("$").asText());
+				} else {
+					model.setTag(item.get("tag"));
+				}
+
+				this.add(model);
+
+			} catch (IOException e) {
+				e.printStackTrace();
+			}
 		}
 		return this;
 	}

dmp-backend/web/src/main/resources/ExternalUrls.xml

@@ -128,7 +128,8 @@
 <!--                <key>cristin</key>-->
 <!--                <label>Cristin</label>-->
 <!--                <ordinal>1</ordinal>-->
-<!--                <url>https://eestore.paas2.uninett.no/api/projectrepo/</url>-->
+<!--                <url>https://eestore.paas2.uninett.no/api/projectrepo/?search={like}&amp;page={page}&amp;size={pageSize}</url>-->
+<!--                <contenttype>application/vnd.api+json; charset=utf-8</contenttype>-->
 <!--                <data>-->
 <!--                    <path>$['data'][*]['attributes']</path>-->
 <!--                    <fields>-->
@@ -140,6 +141,25 @@
 <!--                </data>-->
 <!--                <paginationpath>$['meta']['pagination']['page','pages','count']</paginationpath>-->
 <!--            </urlConfig>-->
+
+            <urlConfig>
+                <key>openAIRE</key>
+                <label>OpenAIRE</label>
+                <ordinal>1</ordinal>
+                <url>https://services.openaire.eu/search/v2/api/resources?query=((oaftype%20exact%20project)%20and%20((projectcode_nt%20exact%20%22{like}%22)or({like}))){funderQuery}&amp;page={page}&amp;size={pageSize}&amp;format=json</url>
+                <funderQuery>&amp;fq=(funder%20exact%20%22{funderId}%22)</funderQuery>
+                <contenttype>application/json; charset=utf-8</contenttype>
+                <data>
+                    <path>$['results'][*]['result']['metadata']['oaf:entity']['oaf:project']</path>
+                    <fields>
+                        <id>'originalId'</id>
+                        <name>'title'</name>
+                        <count>'count'</count>
+                    </fields>
+                </data>
+                <paginationpath>$['meta']['pagination']['page','pages','count']</paginationpath>
+            </urlConfig>
+
 <!--            <urlConfig>-->
 <!--                <key>openAire</key>-->
 <!--                <label>OpenAIRE</label>-->
@@ -157,37 +177,12 @@
 <!--                </data>-->
 <!--                <paginationpath>$['meta']['pagination']['page','pages','count']</paginationpath>-->
 <!--            </urlConfig>-->
-            <urlConfig>
-                <key>internal</key>
-                <label>Internal</label>
-                <ordinal>1</ordinal>
-                <url>GrantInternalMockUpData.json</url>
-                <type>Internal</type>
-                <data>
-                    <path>$['data'][*]['attributes']</path>
-                    <fields>
-                        <id>'pid'</id>
-                        <name>'name'</name>
-                        <uri>'uri'</uri>
-                        <description>'description'</description>
-                    </fields>
-                </data>
-                <paginationpath>$['meta']['pagination']['page','pages','count']</paginationpath>
-            </urlConfig>
-
-        </urls>
-
-        <fetchMode>FIRST</fetchMode>  <!-- EITHER 'FIRST' OR 'ALL' -->
-    </grants>
-
-    <projects>
-        <urls>
 <!--            <urlConfig>-->
-<!--                <key>cristin</key>-->
+<!--                <key>internal</key>-->
-<!--                <label>Cristin</label>-->
+<!--                <label>Internal</label>-->
 <!--                <ordinal>1</ordinal>-->
-<!--                <type>External</type>-->
+<!--                <url>GrantInternalMockUpData.json</url>-->
-<!--                <url>https://eestore.paas2.uninett.no/api/projectrepo/</url>-->
+<!--                <type>Internal</type>-->
 <!--                <data>-->
 <!--                    <path>$['data'][*]['attributes']</path>-->
 <!--                    <fields>-->
@@ -199,6 +194,49 @@
 <!--                </data>-->
 <!--                <paginationpath>$['meta']['pagination']['page','pages','count']</paginationpath>-->
 <!--            </urlConfig>-->
+
+        </urls>
+
+        <fetchMode>FIRST</fetchMode>  <!-- EITHER 'FIRST' OR 'ALL' -->
+    </grants>
+
+    <projects>
+        <urls>
+            <urlConfig>
+                <key>cristin</key>
+                <label>Cristin</label>
+                <ordinal>1</ordinal>
+                <type>External</type>
+                <url>https://eestore.paas2.uninett.no/api/projectrepo/</url>
+                <contenttype>application/vnd.api+json; charset=utf-8</contenttype>
+                <data>
+                    <path>$['data'][*]['attributes']</path>
+                    <fields>
+                        <id>'pid'</id>
+                        <name>'name'</name>
+                        <uri>'uri'</uri>
+                        <description>'description'</description>
+                    </fields>
+                </data>
+                <paginationpath>$['meta']['pagination']['page','pages','count']</paginationpath>
+            </urlConfig>
+            <urlConfig>
+                <key>openAIRE</key>
+                <label>OpenAIRE</label>
+                <ordinal>1</ordinal>
+                <type>External</type>
+                <url>https://services.openaire.eu/search/v2/api/resources?query=((oaftype%20exact%20project)%20and%20((projectcode_nt%20exact%20%22*%22)or(*)))&amp;fq=(funder%20exact%20{funderId})&amp;page={page}&amp;size={pageSize}&amp;format=json</url>
+                <contenttype>application/json; charset=utf-8</contenttype>
+                <data>
+                    <path>$['response']['results']['result'][*]['metadata']['oaf:entity']['oaf:project']</path>
+                    <fields>
+                        <id>'originalId'</id>
+                        <name>'title'</name>
+                        <count>'count'</count>
+                    </fields>
+                </data>
+                <paginationpath>$['meta']['pagination']['page','pages','count']</paginationpath>
+            </urlConfig>
 <!--            <urlConfig>-->
 <!--                <key>openAire</key>-->
 <!--                <label>OpenAIRE</label>-->