MaDgiK-CITE
/
argos
13
0
Fork 0
Code Issues Pull Requests Projects Releases 3 Wiki Activity

Merge remote-tracking branch 'origin/Development' into roadmap

This commit is contained in:
Konstantinos Triantafyllou 2022-05-31 16:17:03 +03:00
parent 78b030b12e 03f7afede8
commit 0d7dbf3e24
7 changed files with 152 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
dmp-backend/web/src/main/java/eu/eudat/controllers/Admin.java
View File

@ -138,19 +138,25 @@ public class Admin extends BaseController {
}
}
@RequestMapping(method = RequestMethod.POST, value = {"/upload"})
@RequestMapping(method = RequestMethod.POST, value = {"/upload", "/upload/{id}"})
public ResponseEntity<Object> setDatasetProfileXml(@RequestParam("file") MultipartFile file,
@ClaimedAuthorities(claims = {ADMIN, DATASET_PROFILE_MANAGER}) Principal principal) throws IllegalAccessException, IOException {
@PathVariable(value = "id") String id,
@ClaimedAuthorities(claims = {ADMIN, DATASET_PROFILE_MANAGER}) Principal principal) throws Exception {
eu.eudat.logic.utilities.documents.xml.datasetProfileXml.datasetProfileModel.DatasetProfile datasetProfileModel = this.datasetProfileManager.createDatasetProfileFromXml(file);
eu.eudat.models.data.admin.composite.DatasetProfile datasetProfileEntity = datasetProfileModel.toAdminCompositeModel(file.getOriginalFilename());
eu.eudat.data.entities.DatasetProfile modelDefinition = AdminManager.generateViewStyleDefinition(datasetProfileEntity, getApiContext());
eu.eudat.data.entities.DatasetProfile datasetProfile = this.getApiContext().getOperationsContext().getDatabaseRepository().getDatasetProfileDao().createOrUpdate(modelDefinition);
UserDatasetProfile userDatasetProfile = new UserDatasetProfile();
userDatasetProfile.setDatasetProfile(datasetProfile);
UserInfo userInfo = getApiContext().getOperationsContext().getDatabaseRepository().getUserInfoDao().find(principal.getId());
userDatasetProfile.setUser(userInfo);
userDatasetProfile.setRole(0);
getApiContext().getOperationsContext().getDatabaseRepository().getUserDatasetProfileDao().createOrUpdate(userDatasetProfile);
eu.eudat.data.entities.DatasetProfile modelDefinition;
if (id == null) {
modelDefinition = AdminManager.generateViewStyleDefinition(datasetProfileEntity, getApiContext());
eu.eudat.data.entities.DatasetProfile datasetProfile = this.getApiContext().getOperationsContext().getDatabaseRepository().getDatasetProfileDao().createOrUpdate(modelDefinition);
UserDatasetProfile userDatasetProfile = new UserDatasetProfile();
userDatasetProfile.setDatasetProfile(datasetProfile);
UserInfo userInfo = getApiContext().getOperationsContext().getDatabaseRepository().getUserInfoDao().find(principal.getId());
userDatasetProfile.setUser(userInfo);
userDatasetProfile.setRole(0);
getApiContext().getOperationsContext().getDatabaseRepository().getUserDatasetProfileDao().createOrUpdate(userDatasetProfile);
} else {
modelDefinition = datasetProfileManager.createNewVersionDatasetProfile(id, datasetProfileEntity);
}
return ResponseEntity.status(HttpStatus.OK).body(new ResponseItem<List<eu.eudat.data.entities.DatasetProfile>>()
.status(ApiMessageCode.SUCCESS_MESSAGE).message(""));
}

4
dmp-backend/web/src/main/java/eu/eudat/logic/managers/DataManagementPlanManager.java
View File

@ -179,7 +179,6 @@ public class DataManagementPlanManager {
DataTableData<DataManagementPlanListingModel> dataTable = new DataTableData<>();
CompletableFuture itemsFuture;
if (fieldsGroup.equals("listing")) {
if (!dataManagementPlanTableRequest.getCriteria().isOnlyPublic()) {
List<DMP> dmps1 = items.withHint(HintedModelFactory.getHint(DataManagementPlanListingModel.class))
@ -249,7 +248,8 @@ public class DataManagementPlanManager {
}
private Set<Dataset> retrieveRelevantDatasets (DatasetCriteria datasetCriteria, UUID principal) {
QueryableList<Dataset> datasetItems = apiContext.getOperationsContext().getDatabaseRepository().getDatasetDao().getWithCriteria(datasetCriteria);
QueryableList<Dataset> datasetItems = apiContext.getOperationsContext().getDatabaseRepository().getDatasetDao().getWithCriteria(datasetCriteria)
.orderBy((builder, root) -> builder.desc(root.get("modified")));
if (principal != null) {
UserInfo userInfo = apiContext.getOperationsContext().getDatabaseRepository().getUserInfoDao().find(principal);
List<Integer> roles = new ArrayList<>();

48
dmp-backend/web/src/main/java/eu/eudat/logic/proxy/config/QueryConfig.java Normal file
View File

@ -0,0 +1,48 @@
package eu.eudat.logic.proxy.config;
import javax.xml.bind.annotation.XmlElement;
public class QueryConfig {
private String condition;
private String separator;
private String value;
private Integer ordinal;
public String getCondition() {
return condition;
}
@XmlElement(name = "condition")
public void setCondition(String condition) {
this.condition = condition;
}
public String getSeparator() {
return separator;
}
@XmlElement(name = "separator")
public void setSeparator(String separator) {
this.separator = separator;
}
public String getValue() {
return value;
}
@XmlElement(name = "value")
public void setValue(String value) {
this.value = value;
}
public Integer getOrdinal() {
return ordinal;
}
@XmlElement(name = "ordinal")
public void setOrdinal(Integer ordinal) {
this.ordinal = ordinal;
}
}

16
dmp-backend/web/src/main/java/eu/eudat/logic/proxy/config/UrlConfiguration.java
View File

@ -2,6 +2,8 @@ package eu.eudat.logic.proxy.config;
import javax.xml.bind.annotation.XmlElement;
import javax.xml.bind.annotation.XmlElementWrapper;
import java.util.List;
public class UrlConfiguration {
@ -17,7 +19,9 @@ public class UrlConfiguration {
private String firstpage;
private String requestType = "GET";
private String requestBody = "";
private String filterType = "local";
private String filterType = "remote";
private List<QueryConfig> queries;
public String getKey() {
return key;
@ -120,4 +124,14 @@ public class UrlConfiguration {
public void setFilterType(String filterType) {
this.filterType = filterType;
}
public List<QueryConfig> getQueries() {
return queries;
}
@XmlElementWrapper
@XmlElement(name = "query")
public void setQueries(List<QueryConfig> queries) {
this.queries = queries;
}
}

41
dmp-backend/web/src/main/java/eu/eudat/logic/proxy/fetching/RemoteFetcher.java
View File

@ -5,10 +5,7 @@ import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.jayway.jsonpath.DocumentContext;
import com.jayway.jsonpath.JsonPath;
import eu.eudat.logic.proxy.config.DataUrlConfiguration;
import eu.eudat.logic.proxy.config.ExternalUrlCriteria;
import eu.eudat.logic.proxy.config.FetchStrategy;
import eu.eudat.logic.proxy.config.UrlConfiguration;
import eu.eudat.logic.proxy.config.*;
import eu.eudat.logic.proxy.config.configloaders.ConfigLoader;
import eu.eudat.logic.proxy.config.entities.GenericUrls;
import eu.eudat.logic.proxy.config.exceptions.HugeResultSet;
@ -197,7 +194,7 @@ public class RemoteFetcher {
ifFunderQueryExist(urlConfiguration, externalUrlCriteria);
if (urlConfiguration.getType() == null || urlConfiguration.getType().equals("External")) {
try {
results.addAll(getAllResultsFromUrl(urlConfiguration.getUrl(), fetchStrategy, urlConfiguration.getData(), urlConfiguration.getPaginationPath(), externalUrlCriteria, urlConfiguration.getLabel(), urlConfiguration.getKey(), urlConfiguration.getContentType(), urlConfiguration.getFirstpage(), urlConfiguration.getRequestBody(), urlConfiguration.getRequestType(), urlConfiguration.getFilterType()));
results.addAll(getAllResultsFromUrl(urlConfiguration.getUrl(), fetchStrategy, urlConfiguration.getData(), urlConfiguration.getPaginationPath(), externalUrlCriteria, urlConfiguration.getLabel(), urlConfiguration.getKey(), urlConfiguration.getContentType(), urlConfiguration.getFirstpage(), urlConfiguration.getRequestBody(), urlConfiguration.getRequestType(), urlConfiguration.getFilterType(), urlConfiguration.getQueries()));
} catch (Exception e) {
logger.error(e.getLocalizedMessage(), e);
}
@ -227,13 +224,37 @@ public class RemoteFetcher {
}
}
protected String replaceCriteriaOnUrl(String path, ExternalUrlCriteria externalUrlCriteria, String firstPage) {
private String calculateQuery(ExternalUrlCriteria externalUrlCriteria, List<QueryConfig> queryConfigs) {
String finalQuery = "";
QueryConfig queryConfig = queryConfigs.stream().filter(queryConfigl -> externalUrlCriteria.getLike().matches(queryConfigl.getCondition()))
.min((Comparator.comparing(QueryConfig::getOrdinal))).orElse(null);
if (queryConfig != null) {
if (queryConfig.getSeparator() != null) {
String[] likes = externalUrlCriteria.getLike().split(queryConfig.getSeparator());
finalQuery = queryConfig.getValue();
for (int i = 0; i < likes.length; i++) {
finalQuery = finalQuery.replaceAll("\\{like" + (i+1) + "}", likes[i]);
}
} else {
finalQuery = queryConfig.getValue().replaceAll("\\{like}", externalUrlCriteria.getLike());
}
}
return finalQuery;
}
protected String replaceCriteriaOnUrl(String path, ExternalUrlCriteria externalUrlCriteria, String firstPage, List<QueryConfig> queries) {
String completedPath = path;
if (externalUrlCriteria.getLike() != null) {
if ((path.contains("openaire") || path.contains("orcid") || path.contains("ror")) && externalUrlCriteria.getLike().equals("")) {
completedPath = completedPath.replaceAll("\\{like}", "*");
completedPath = completedPath.replaceAll("\\{query}", "*");
} else {
completedPath = completedPath.replaceAll("\\{like}", externalUrlCriteria.getLike());
if (completedPath.contains("{query}")) {
completedPath = completedPath.replaceAll("\\{query}", this.calculateQuery(externalUrlCriteria, queries));
} else {
completedPath = completedPath.replaceAll("\\{like}", externalUrlCriteria.getLike());
}
}
} else {
completedPath = completedPath.replace("{like}", "");
@ -277,11 +298,11 @@ public class RemoteFetcher {
return completedPath;
}
private List<Map<String, String>> getAllResultsFromUrl(String path, FetchStrategy fetchStrategy, final DataUrlConfiguration jsonDataPath, final String jsonPaginationPath, ExternalUrlCriteria externalUrlCriteria, String tag, String key, String contentType, String firstPage, String requestBody, String requestType, String filterType) throws Exception {
private List<Map<String, String>> getAllResultsFromUrl(String path, FetchStrategy fetchStrategy, final DataUrlConfiguration jsonDataPath, final String jsonPaginationPath, ExternalUrlCriteria externalUrlCriteria, String tag, String key, String contentType, String firstPage, String requestBody, String requestType, String filterType, List<QueryConfig> queries) throws Exception {
Set<Integer> pages = new HashSet<>();
String replacedPath = replaceCriteriaOnUrl(path, externalUrlCriteria, firstPage);
String replacedBody = replaceCriteriaOnUrl(requestBody, externalUrlCriteria, firstPage);
String replacedPath = replaceCriteriaOnUrl(path, externalUrlCriteria, firstPage, queries);
String replacedBody = replaceCriteriaOnUrl(requestBody, externalUrlCriteria, firstPage, queries);
Results results = getResultsFromUrl(replacedPath, jsonDataPath, jsonPaginationPath, contentType, replacedBody, requestType);
if(filterType != null && filterType.equals("local") && (externalUrlCriteria.getLike() != null && !externalUrlCriteria.getLike().isEmpty())){

2
dmp-backend/web/src/main/java/eu/eudat/logic/proxy/fetching/RemoteFetcherUtils.java
View File

@ -31,7 +31,7 @@ public class RemoteFetcherUtils {
ExternalUrlCriteria externalUrlCriteria = new ExternalUrlCriteria();
externalUrlCriteria.setPath(result.get("path"));
externalUrlCriteria.setHost(result.get("host"));
String replacedPath = remoteFetcher.replaceCriteriaOnUrl(jsonDataPath.getUrlConfiguration().getUrl(), externalUrlCriteria, jsonDataPath.getUrlConfiguration().getFirstpage());
String replacedPath = remoteFetcher.replaceCriteriaOnUrl(jsonDataPath.getUrlConfiguration().getUrl(), externalUrlCriteria, jsonDataPath.getUrlConfiguration().getFirstpage(), jsonDataPath.getUrlConfiguration().getQueries());
return remoteFetcher.getResultsFromUrl(replacedPath, jsonDataPath.getUrlConfiguration().getData(), jsonDataPath.getUrlConfiguration().getData().getPath(), jsonDataPath.getUrlConfiguration().getContentType(), requestBody, requestType);
}).filter(Objects::nonNull).map(results1 -> results1.getResults().get(0)).collect(Collectors.toList());
return new Results(multiResults, new HashMap<>(1, 1));

42
dmp-backend/web/src/main/resources/externalUrls/ExternalUrls.xml
View File

@ -748,9 +748,26 @@ but not
<label>OpenAIRE Publications</label>
<ordinal>1</ordinal>
<type>External</type>
<url>https://services.openaire.eu/search/v2/api/resources?query=oaftype exact result and {like}&amp;page={page}&amp;size={pageSize}&amp;format=json</url>
<url>https://services.openaire.eu/search/v2/api/resources?query=oaftype exact result and {query}&amp;page={page}&amp;size={pageSize}&amp;format=json</url>
<firstPage>0</firstPage>
<contenttype>application/json;charset=UTF-8</contenttype>
<queries>
<query>
<ordinal>0</ordinal>
<condition>(10[.][0-9]{4,}(?:[.][0-9]+)*\/(?:(?![&quot;&amp;\&#39;&lt;&gt;])[[:graph:]])+)</condition>
<value>(pidclassid exact "doi" and pid="{like}")</value>
</query>
<query>
<ordinal>1</ordinal>
<condition>(10[.][0-9]{4,}(?:[.][0-9]+)*\/(?:(?![&quot;&amp;\&#39;&lt;&gt;])\S)+)</condition>
<value>(pidclassid exact "doi" and pid="{like}")</value>
</query>
<query>
<ordinal>2</ordinal>
<condition>.+</condition>
<value>{like}</value>
</query>
</queries>
<data>
<path>$['results'][*]['result']['metadata']['oaf:entity']['oaf:result']</path>
<fields>
@ -873,9 +890,27 @@ but not
<label>ORCID</label>
<ordinal>2</ordinal>
<type>External</type>
<url>https://pub.sandbox.orcid.org/v3.0/expanded-search/?q={like}&amp;start={page}&amp;rows={pageSize}</url>
<url>https://pub.sandbox.orcid.org/v3.0/expanded-search/?q={query}&amp;start={page}&amp;rows={pageSize}</url>
<firstPage>0</firstPage>
<contenttype>application/json; charset=UTF-8</contenttype>
<queries>
<query>
<ordinal>0</ordinal>
<condition>^[A-Za-z0-9]+ [A-Za-z0-9]+$</condition>
<separator> </separator>
<value>((given-names:{like1}+AND+family-name:{like2})+OR+(given-names:{like2}+AND+family-name:{like1}))</value>
</query>
<query>
<ordinal>1</ordinal>
<condition>\d{4}-\d{4}-\d{4}-\d{4}</condition>
<value>orcid:{like}</value>
</query>
<query>
<ordinal>2</ordinal>
<condition>.+</condition>
<value>given-names:{like}+OR+family-name:{like}</value>
</query>
</queries>
<data>
<path>$['expanded-result'][*]</path>
<fields>
@ -885,6 +920,7 @@ but not
<name>'name'</name>
</fields>
</data>
<filterType>api</filterType>
<paginationpath>$['num-found']</paginationpath>
</urlConfig>
<!-- <urlConfig>
@ -1164,7 +1200,7 @@ but not
<label>Zenodo</label>
<ordinal>1</ordinal>
<type>External</type>
<url>https://zenodo.org/api/records/?page={page}&amp;size={pageSize}&amp;q="{like}"</url>
<url>https://zenodo.org/api/records/?page={page}&amp;size={pageSize}&amp;q=title:"{like}" doi:"{like}" conceptdoi:"{like}"</url>
<firstPage>1</firstPage>
<contenttype>application/json</contenttype>
<data>