diff --git a/dmp-backend/web/src/main/java/eu/eudat/logic/builders/model/models/FunderBuilder.java b/dmp-backend/web/src/main/java/eu/eudat/logic/builders/model/models/FunderBuilder.java index 0bbcc235f..ae8414d49 100644 --- a/dmp-backend/web/src/main/java/eu/eudat/logic/builders/model/models/FunderBuilder.java +++ b/dmp-backend/web/src/main/java/eu/eudat/logic/builders/model/models/FunderBuilder.java @@ -17,6 +17,7 @@ public class FunderBuilder extends Builder { private Date modified; private Integer type; private String source; + private String key; public FunderBuilder id(UUID id) { this.id = id; @@ -63,6 +64,11 @@ public class FunderBuilder extends Builder { return this; } + public FunderBuilder key(String key) { + this.key = key; + return this; + } + @Override public Funder build() { Funder funder = new Funder(); @@ -75,6 +81,7 @@ public class FunderBuilder extends Builder { funder.setModified(modified); funder.setType(type); funder.setSource(source); + funder.setKey(key); return funder; } } diff --git a/dmp-backend/web/src/main/java/eu/eudat/logic/builders/model/models/GrantBuilder.java b/dmp-backend/web/src/main/java/eu/eudat/logic/builders/model/models/GrantBuilder.java index 12f87f4d9..cd8d57df2 100644 --- a/dmp-backend/web/src/main/java/eu/eudat/logic/builders/model/models/GrantBuilder.java +++ b/dmp-backend/web/src/main/java/eu/eudat/logic/builders/model/models/GrantBuilder.java @@ -9,9 +9,6 @@ import java.util.Date; import java.util.List; import java.util.UUID; -/** - * Created by ikalyvas on 2/15/2018. - */ public class GrantBuilder extends Builder { private UUID id; @@ -44,6 +41,8 @@ public class GrantBuilder extends Builder { private String source; + private String key; + public GrantBuilder id(UUID id) { this.id = id; return this; @@ -119,6 +118,11 @@ public class GrantBuilder extends Builder { return this; } + public GrantBuilder key(String key) { + this.key = key; + return this; + } + @Override public Grant build() { Grant grant = new Grant(); @@ -137,6 +141,7 @@ public class GrantBuilder extends Builder { grant.setCreationUser(creationUser); grant.setStartDate(startDate); grant.setSource(source); + grant.setKey(key); return grant; } } diff --git a/dmp-backend/web/src/main/java/eu/eudat/logic/builders/model/models/ProjectBuilder.java b/dmp-backend/web/src/main/java/eu/eudat/logic/builders/model/models/ProjectBuilder.java index d0741d631..a0560d5d5 100644 --- a/dmp-backend/web/src/main/java/eu/eudat/logic/builders/model/models/ProjectBuilder.java +++ b/dmp-backend/web/src/main/java/eu/eudat/logic/builders/model/models/ProjectBuilder.java @@ -28,6 +28,7 @@ public class ProjectBuilder extends Builder { private Date modified; private String description; private String source; + private String key; public ProjectBuilder id(UUID id) { this.id = id; @@ -109,6 +110,11 @@ public class ProjectBuilder extends Builder { return this; } + public ProjectBuilder key(String key) { + this.key = key; + return this; + } + @Override public Project build() { Project project = new Project(); @@ -127,6 +133,7 @@ public class ProjectBuilder extends Builder { project.setCreationUser(creationUser); project.setStartDate(startDate); project.setSource(source); + project.setKey(key); return project; } } diff --git a/dmp-backend/web/src/main/java/eu/eudat/logic/managers/FunderManager.java b/dmp-backend/web/src/main/java/eu/eudat/logic/managers/FunderManager.java index 1db66f391..a14a1e8df 100644 --- a/dmp-backend/web/src/main/java/eu/eudat/logic/managers/FunderManager.java +++ b/dmp-backend/web/src/main/java/eu/eudat/logic/managers/FunderManager.java @@ -3,6 +3,7 @@ package eu.eudat.logic.managers; import eu.eudat.data.query.items.item.funder.FunderCriteriaRequest; import eu.eudat.logic.builders.model.models.FunderBuilder; import eu.eudat.logic.proxy.config.ExternalUrlCriteria; +import eu.eudat.logic.proxy.config.configloaders.ConfigLoader; import eu.eudat.logic.proxy.config.exceptions.HugeResultSet; import eu.eudat.logic.proxy.config.exceptions.NoURLFound; import eu.eudat.logic.proxy.fetching.RemoteFetcher; @@ -47,6 +48,8 @@ public class FunderManager { eu.eudat.models.data.funder.Funder funder = apiContext.getOperationsContext().getBuilderFactory().getBuilder(FunderBuilder.class) .reference(externalListingItem.getRemoteId()).label(externalListingItem.getName()) .status(eu.eudat.data.entities.Funder.Status.fromInteger(0)) + .key(externalListingItem.getKey()) + .source(externalListingItem.getTag()) .build(); if (externalListingItem.getSource() != null) { funder.setSource(externalListingItem.getSource()); diff --git a/dmp-backend/web/src/main/java/eu/eudat/logic/managers/GrantManager.java b/dmp-backend/web/src/main/java/eu/eudat/logic/managers/GrantManager.java index e3c0c7360..e69ca22e6 100644 --- a/dmp-backend/web/src/main/java/eu/eudat/logic/managers/GrantManager.java +++ b/dmp-backend/web/src/main/java/eu/eudat/logic/managers/GrantManager.java @@ -134,6 +134,7 @@ public class GrantManager { eu.eudat.models.data.grant.Grant grant = apiContext.getOperationsContext().getBuilderFactory().getBuilder(GrantBuilder.class) .reference(externalListingItem.getRemoteId()).label(externalListingItem.getName()) .description(externalListingItem.getDescription()).uri(externalListingItem.getUri()) + .key(externalListingItem.getKey()) .abbreviation(externalListingItem.getAbbreviation()).status(eu.eudat.data.entities.Grant.Status.fromInteger(0)) .source(externalListingItem.getTag()) .build(); diff --git a/dmp-backend/web/src/main/java/eu/eudat/logic/managers/ProjectManager.java b/dmp-backend/web/src/main/java/eu/eudat/logic/managers/ProjectManager.java index bd9b494b0..6919a5de5 100644 --- a/dmp-backend/web/src/main/java/eu/eudat/logic/managers/ProjectManager.java +++ b/dmp-backend/web/src/main/java/eu/eudat/logic/managers/ProjectManager.java @@ -48,6 +48,7 @@ public class ProjectManager { .reference(externalListingItem.getRemoteId()).label(externalListingItem.getName()) .description(externalListingItem.getDescription()).uri(externalListingItem.getUri()) .abbreviation(externalListingItem.getAbbreviation()).status(eu.eudat.data.entities.Project.Status.fromInteger(0)) + .key(externalListingItem.getKey()) .source(externalListingItem.getTag()) .build(); diff --git a/dmp-backend/web/src/main/java/eu/eudat/logic/proxy/fetching/RemoteFetcher.java b/dmp-backend/web/src/main/java/eu/eudat/logic/proxy/fetching/RemoteFetcher.java index 9c6c90d44..3ffcfb193 100644 --- a/dmp-backend/web/src/main/java/eu/eudat/logic/proxy/fetching/RemoteFetcher.java +++ b/dmp-backend/web/src/main/java/eu/eudat/logic/proxy/fetching/RemoteFetcher.java @@ -129,7 +129,7 @@ public class RemoteFetcher { for (UrlConfiguration urlConfig : urlConfigs) { ifFunderQueryExist(urlConfig, externalUrlCriteria); if (urlConfig.getType() == null || urlConfig.getType().equals("External")) { - results.addAll(getAllResultsFromUrl(urlConfig.getUrl(), fetchStrategy, urlConfig.getData(), urlConfig.getPaginationPath(), externalUrlCriteria, urlConfig.getLabel(), urlConfig.getContentType(), urlConfig.getFirstpage())); + results.addAll(getAllResultsFromUrl(urlConfig.getUrl(), fetchStrategy, urlConfig.getData(), urlConfig.getPaginationPath(), externalUrlCriteria, urlConfig.getLabel(), urlConfig.getKey(), urlConfig.getContentType(), urlConfig.getFirstpage())); } else if (urlConfig.getType() != null && urlConfig.getType().equals("Internal")) { results.addAll(getAllResultsFromMockUpJson(urlConfig.getUrl(), externalUrlCriteria.getLike())); } @@ -184,14 +184,14 @@ public class RemoteFetcher { return completedPath; } - private List> getAllResultsFromUrl(String path, FetchStrategy fetchStrategy, final DataUrlConfiguration jsonDataPath, final String jsonPaginationPath, ExternalUrlCriteria externalUrlCriteria, String key, String contentType, String firstPage) throws HugeResultSet { + private List> getAllResultsFromUrl(String path, FetchStrategy fetchStrategy, final DataUrlConfiguration jsonDataPath, final String jsonPaginationPath, ExternalUrlCriteria externalUrlCriteria, String tag, String key, String contentType, String firstPage) throws HugeResultSet { Set pages = new HashSet<>(); String replacedPath = replaceCriteriaOnUrl(path, externalUrlCriteria, firstPage); Results results = getResultsFromUrl(replacedPath, jsonDataPath, jsonPaginationPath, contentType); if (fetchStrategy == FetchStrategy.FIRST) - return results == null ? new LinkedList<>() : results.getResults().stream().peek(x -> x.put("tag", key)).collect(Collectors.toList()); + return results == null ? new LinkedList<>() : results.getResults().stream().peek(x -> x.put("tag", tag)).peek(x -> x.put("key", key)).collect(Collectors.toList()); if (results != null && results.getPagination() != null && results.getPagination().get("pages") != null) //if has more pages, add them to the pages set for (int i = 2; i <= results.getPagination().get("pages"); i++) @@ -210,7 +210,7 @@ public class RemoteFetcher { Results remainingResults = optionalResults.orElseGet(Results::new); remainingResults.getResults().addAll(results.getResults()); - return remainingResults.getResults().stream().peek(x -> x.put("tag", key)).collect(Collectors.toList()); + return remainingResults.getResults().stream().peek(x -> x.put("tag", tag)).collect(Collectors.toList()); } @@ -271,8 +271,7 @@ public class RemoteFetcher { String filePath = Paths.get(path).toUri().toURL().toString(); ObjectMapper mapper = new ObjectMapper(); internalResults = mapper.readValue(new File(filePath), new TypeReference>>(){}); - searchListMap(internalResults, query); - return internalResults; + return searchListMap(internalResults, query); } catch (Exception e) { e.printStackTrace(); return new LinkedList<>(); diff --git a/dmp-backend/web/src/main/java/eu/eudat/models/data/external/FundersExternalSourcesModel.java b/dmp-backend/web/src/main/java/eu/eudat/models/data/external/FundersExternalSourcesModel.java index 5526a5552..9ef6c2c8d 100644 --- a/dmp-backend/web/src/main/java/eu/eudat/models/data/external/FundersExternalSourcesModel.java +++ b/dmp-backend/web/src/main/java/eu/eudat/models/data/external/FundersExternalSourcesModel.java @@ -15,6 +15,7 @@ public class FundersExternalSourcesModel extends ExternalListingItem private Date modified; private Integer type; private String source; + private String key; public UUID getId() { return id; @@ -80,6 +81,13 @@ public class Funder implements DataModel this.source = source; } + public String getKey() { + return key; + } + public void setKey(String key) { + this.key = key; + } + @Override public Funder fromDataModel(eu.eudat.data.entities.Funder entity) { this.id = entity.getId(); @@ -93,9 +101,9 @@ public class Funder implements DataModel if (entity.getReference() != null) { String source = entity.getReference().substring(0, entity.getReference().indexOf(":")); if (source.equals("dmp")) { - this.source = "Internal"; + this.key = "Internal"; } else { - this.source = source; + this.key = source; } } @@ -105,19 +113,26 @@ public class Funder implements DataModel @Override public eu.eudat.data.entities.Funder toDataModel() { eu.eudat.data.entities.Funder entity = new eu.eudat.data.entities.Funder(); - entity.setId(UUID.randomUUID()); + if (this.getId() != null) { + entity.setId(this.getId()); + } else { + entity.setId(UUID.randomUUID()); + } entity.setLabel(this.label); entity.setType(this.type); - if (this.source != null && this.source.equals("Internal")) this.source = "dmp"; - if (this.source == null && this.reference != null && this.reference.startsWith("dmp:")) { - entity.setReference(this.reference); + // If internal, key has no value, fill it. + if ((this.source != null && this.source.equals("Internal")) || (this.key != null && this.key.equals("Internal"))) this.key = "dmp"; + // Logic for the different "key" cases. + if ((this.key == null || this.key.trim().isEmpty()) && (this.source != null && !this.source.trim().isEmpty())) { + this.key = this.source; } - if (this.reference != null && !this.reference.trim().isEmpty() - && this.source != null && !this.source.trim().isEmpty()) { - if (this.source.equals(this.reference.substring(0, this.source.length()))) { + if (this.reference != null && this.reference.startsWith("dmp:")) { + entity.setReference(this.reference); + } else if (this.reference != null && !this.reference.trim().isEmpty() && this.key != null && !this.key.trim().isEmpty()) { + if (this.key.equals(this.reference.substring(0, this.key.length()))) { entity.setReference(this.reference); } else { - entity.setReference(this.source.toLowerCase() + ":" + this.reference); + entity.setReference(this.key.toLowerCase() + ":" + this.reference); } } entity.setDefinition(this.definition); diff --git a/dmp-backend/web/src/main/java/eu/eudat/models/data/grant/Grant.java b/dmp-backend/web/src/main/java/eu/eudat/models/data/grant/Grant.java index 7d42a3964..798b3a6ab 100644 --- a/dmp-backend/web/src/main/java/eu/eudat/models/data/grant/Grant.java +++ b/dmp-backend/web/src/main/java/eu/eudat/models/data/grant/Grant.java @@ -32,6 +32,7 @@ public class Grant implements DataModel { private List files; private UUID funderId; private String source; + private String key; public UUID getId() { return id; @@ -159,6 +160,13 @@ public class Grant implements DataModel { this.source = source; } + public String getKey() { + return key; + } + public void setKey(String key) { + this.key = key; + } + @Override public Grant fromDataModel(eu.eudat.data.entities.Grant entity) { this.id = entity.getId(); @@ -180,9 +188,9 @@ public class Grant implements DataModel { if (entity.getReference() != null) { String source = entity.getReference().substring(0, entity.getReference().indexOf(":")); if (source.equals("dmp")) { - this.source = "Internal"; + this.key = "Internal"; } else { - this.source = source; + this.key = source; } } @@ -192,20 +200,22 @@ public class Grant implements DataModel { @Override public eu.eudat.data.entities.Grant toDataModel() { eu.eudat.data.entities.Grant entity = new eu.eudat.data.entities.Grant(); - entity.setId(UUID.randomUUID()); + if (this.getId() != null) { + entity.setId(this.getId()); + } else { + entity.setId(UUID.randomUUID()); + } entity.setAbbreviation(this.abbreviation); entity.setLabel(this.label); entity.setType(this.type); - if (this.source != null && this.source.equals("Internal")) this.source = "dmp"; - if (this.source == null && this.reference != null && this.reference.startsWith("dmp:")) { + if ((this.source != null && this.source.equals("Internal")) || (this.key != null && this.key.equals("Internal"))) this.key = "dmp"; + if (this.reference != null && this.reference.startsWith("dmp:")) { entity.setReference(this.reference); - } - if (this.reference != null && !this.reference.trim().isEmpty() - && this.source != null && !this.source.trim().isEmpty()) { - if (this.source.equals(this.reference.substring(0, this.source.length()))) { + } else if (this.reference != null && !this.reference.trim().isEmpty() && this.key != null && !this.key.trim().isEmpty()) { + if (this.key.equals(this.reference.substring(0, this.key.length()))) { entity.setReference(this.reference); } else { - entity.setReference(this.source.toLowerCase() + ":" + this.reference); + entity.setReference(this.key.toLowerCase() + ":" + this.reference); } } entity.setUri(this.uri); diff --git a/dmp-backend/web/src/main/java/eu/eudat/models/data/project/Project.java b/dmp-backend/web/src/main/java/eu/eudat/models/data/project/Project.java index 45be84962..430eb6017 100644 --- a/dmp-backend/web/src/main/java/eu/eudat/models/data/project/Project.java +++ b/dmp-backend/web/src/main/java/eu/eudat/models/data/project/Project.java @@ -29,6 +29,7 @@ public class Project implements DataModel files; private String source; + private String key; public UUID getId() { return id; @@ -149,6 +150,13 @@ public class Project implements DataModelInternal 1 Internal - mockData/RegistriesInternalMockUpData.json + web/src/main/resources/mockData/RegistriesInternalMockUpData.json $['data'][*]['attributes'] @@ -109,7 +109,7 @@ 1 Internal - mockData/TagsInternalMockUpData.json + web/src/main/resources/mockData/TagsInternalMockUpData.json $['data'][*]['attributes'] @@ -146,7 +146,7 @@ $['meta']['pagination']['page','pages','count'] - openAIRE + openaire 1 https://services.openaire.eu/search/v2/api/resources?query=((oaftype%20exact%20project)%20and%20((projectcode_nt%20exact%20%22{like}%22)or({like}))){funderQuery}&page={page}&size={pageSize}&format=json @@ -185,7 +185,7 @@ internal 1 - mockData/GrantInternalMockUpData.json + web/src/main/resources/mockData/GrantInternalMockUpData.json Internal $['data'][*]['attributes'] @@ -224,7 +224,7 @@ $['meta']['pagination']['page','pages','count'] - openAIRE + openaire 1 External @@ -265,7 +265,7 @@ 1 Internal - mockData/ProjectInternalMockUpData.json + web/src/main/resources/mockData/ProjectInternalMockUpData.json $['data'][*]['attributes'] @@ -283,6 +283,74 @@ + + openaire + + 1 + External + https://services.openaire.eu/search/v2/api/publications?&refine=true&fields=relfunder&page={page}&size=0&format=json + 0 + application/json; charset=utf-8 + + $['refineResults']['relfunder'][*] + + 'name' + 'id' + 'count' + + + + + openaire + + 1 + External + https://services.openaire.eu/search/v2/api/datasets?&refine=true&fields=relfunder&page=0&size={page}&format=json + 0 + application/json; charset=utf-8 + + $['refineResults']['relfunder'][*] + + 'name' + 'id' + 'count' + + + + + openaire + + 1 + External + https://services.openaire.eu/search/v2/api/software?&refine=true&fields=relfunder&page={page}&size=0&format=json + 0 + application/json; charset=utf-8 + + $['refineResults']['relfunder'][*] + + 'name' + 'id' + 'count' + + + + + openaire + + 1 + External + https://services.openaire.eu/search/v2/api/other?&refine=true&fields=relfunder&page={page}&size=0&format=json + 0 + application/json; charset=utf-8 + + $['refineResults']['relfunder'][*] + + 'name' + 'id' + 'count' + + + - openAire - + cristin + 1 External https://eestore.paas2.uninett.no/api/projectrepo/?search={like}&page={page}&size={pageSize} @@ -317,11 +385,26 @@ 'name' 'uri' 'description' - 'source' $['meta']['pagination']['page','pages','count'] + + crossref + + 1 + External + https://api.crossref.org/funders?query={like}&rows={pageSize} + application/json; charset=utf-8 + + $['message']['items'][*] + + 'name' + 'id' + 'count' + + + -