argos/dmp-backend/web/src/main/java/eu/eudat/logic/services/references/ReferenceService.java

459 lines
24 KiB
Java

package eu.eudat.logic.services.references;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.jayway.jsonpath.DocumentContext;
import com.jayway.jsonpath.JsonPath;
import eu.eudat.authorization.AuthorizationFlags;
import eu.eudat.authorization.Permission;
import eu.eudat.commons.XmlHandlingService;
import eu.eudat.commons.enums.IsActive;
import eu.eudat.commons.enums.ReferenceTypeExternalApiHTTPMethodType;
import eu.eudat.commons.enums.ReferenceTypeSourceType;
import eu.eudat.commons.scope.user.UserScope;
import eu.eudat.commons.types.reference.DefinitionEntity;
import eu.eudat.commons.types.reference.FieldEntity;
import eu.eudat.commons.types.referencetype.*;
import eu.eudat.configurations.referencetype.ReferenceTypeProperties;
import eu.eudat.convention.ConventionService;
import eu.eudat.data.ReferenceEntity;
import eu.eudat.data.ReferenceTypeEntity;
import eu.eudat.logic.proxy.config.ExternalUrlCriteria;
import eu.eudat.logic.proxy.config.FetchStrategy;
import eu.eudat.logic.proxy.config.exceptions.HugeResultSet;
import eu.eudat.logic.proxy.config.exceptions.NoURLFound;
import eu.eudat.logic.proxy.fetching.RemoteFetcher;
import eu.eudat.logic.proxy.fetching.entities.Results;
import eu.eudat.logic.services.ApiContext;
import eu.eudat.model.Reference;
import eu.eudat.model.builder.ReferenceBuilder;
import eu.eudat.model.builder.referencesearch.ReferenceSearchBuilder;
import eu.eudat.model.deleter.ReferenceDeleter;
import eu.eudat.model.persist.ReferencePersist;
import eu.eudat.model.persist.referencedefinition.DefinitionPersist;
import eu.eudat.model.persist.referencedefinition.FieldPersist;
import eu.eudat.query.ReferenceQuery;
import eu.eudat.query.ReferenceTypeQuery;
import eu.eudat.query.lookup.ReferenceDefinitionSearchLookup;
import eu.eudat.query.lookup.ReferenceSearchLookup;
import gr.cite.commons.web.authz.service.AuthorizationService;
import gr.cite.tools.data.builder.BuilderFactory;
import gr.cite.tools.data.deleter.DeleterFactory;
import gr.cite.tools.data.query.QueryFactory;
import gr.cite.tools.exception.MyApplicationException;
import gr.cite.tools.exception.MyForbiddenException;
import gr.cite.tools.exception.MyNotFoundException;
import gr.cite.tools.exception.MyValidationException;
import gr.cite.tools.fieldset.BaseFieldSet;
import gr.cite.tools.fieldset.FieldSet;
import gr.cite.tools.logging.LoggerService;
import gr.cite.tools.logging.MapLogEntry;
import jakarta.persistence.EntityManager;
import jakarta.xml.bind.JAXBException;
import org.jetbrains.annotations.NotNull;
import org.slf4j.LoggerFactory;
import org.springframework.context.MessageSource;
import org.springframework.context.i18n.LocaleContextHolder;
import org.springframework.core.ParameterizedTypeReference;
import org.springframework.http.HttpMethod;
import org.springframework.http.HttpStatus;
import org.springframework.http.MediaType;
import org.springframework.http.ResponseEntity;
import org.springframework.http.client.reactive.ReactorClientHttpConnector;
import org.springframework.http.codec.json.Jackson2JsonDecoder;
import org.springframework.web.reactive.function.client.WebClient;
import reactor.netty.http.client.HttpClient;
import javax.management.InvalidApplicationException;
import javax.xml.parsers.ParserConfigurationException;
import javax.xml.transform.TransformerException;
import java.time.Instant;
import java.util.*;
import java.util.stream.Collectors;
@org.springframework.stereotype.Service
public class ReferenceService {
private static final LoggerService logger = new LoggerService(LoggerFactory.getLogger(ReferenceService.class));
private final ApiContext apiContext;
private final UserScope userScope;
private final RemoteFetcher remoteFetcher;
private final EntityManager entityManager;
private final AuthorizationService authorizationService;
private final DeleterFactory deleterFactory;
private final BuilderFactory builderFactory;
private final ConventionService conventionService;
private final MessageSource messageSource;
private final QueryFactory queryFactory;
private final XmlHandlingService xmlHandlingService;
private final ReferenceTypeProperties referenceTypeProperties;
private final WebClient client;
public ReferenceService(ApiContext apiContext,
UserScope userScope,
RemoteFetcher remoteFetcher,
EntityManager entityManager,
AuthorizationService authorizationService,
DeleterFactory deleterFactory,
BuilderFactory builderFactory,
ConventionService conventionService,
MessageSource messageSource,
QueryFactory queryFactory,
XmlHandlingService xmlHandlingService,
ReferenceTypeProperties referenceTypeProperties) {
this.apiContext = apiContext;
this.userScope = userScope;
this.remoteFetcher = remoteFetcher;
this.entityManager = entityManager;
this.authorizationService = authorizationService;
this.deleterFactory = deleterFactory;
this.builderFactory = builderFactory;
this.conventionService = conventionService;
this.messageSource = messageSource;
this.queryFactory = queryFactory;
this.xmlHandlingService = xmlHandlingService;
this.referenceTypeProperties = referenceTypeProperties;
this.client = WebClient.builder().codecs(clientCodecConfigurer -> {
clientCodecConfigurer.defaultCodecs().jackson2JsonDecoder(new Jackson2JsonDecoder(new ObjectMapper(), MediaType.APPLICATION_JSON));
clientCodecConfigurer.defaultCodecs().maxInMemorySize(2 * ((int) Math.pow(1024, 3))); //GK: Why here???
}
).clientConnector(new ReactorClientHttpConnector(HttpClient.create().followRedirect(true))).build();
}
public Reference persist(ReferencePersist model, FieldSet fields) throws MyForbiddenException, MyValidationException, MyApplicationException, MyNotFoundException, InvalidApplicationException, JAXBException, JsonProcessingException, TransformerException, ParserConfigurationException {
logger.debug(new MapLogEntry("persisting data").And("model", model).And("fields", fields));
this.authorizationService.authorizeForce(Permission.EditDmpBlueprint);
Boolean isUpdate = this.conventionService.isValidGuid(model.getId());
ReferenceEntity data;
if (isUpdate) {
data = this.entityManager.find(ReferenceEntity.class, model.getId());
if (data == null)
throw new MyNotFoundException(messageSource.getMessage("General_ItemNotFound", new Object[]{model.getId(), Reference.class.getSimpleName()}, LocaleContextHolder.getLocale()));
} else {
data = new ReferenceEntity();
data.setId(UUID.randomUUID());
data.setIsActive(IsActive.Active);
data.setCreatedAt(Instant.now());
}
data.setLabel(model.getLabel());
data.setType(model.getType());
data.setDescription(model.getDescription());
data.setDefinition(this.xmlHandlingService.toXmlSafe(this.buildDefinitionEntity(model.getDefinition())));
data.setUpdatedAt(Instant.now());
data.setReference(model.getReference());
data.setAbbreviation(model.getAbbreviation());
data.setSource(model.getSource());
data.setSourceType(model.getSourceType());
if (isUpdate) this.entityManager.merge(data);
else this.entityManager.persist(data);
this.entityManager.flush();
return this.builderFactory.builder(ReferenceBuilder.class).authorize(AuthorizationFlags.OwnerOrPermissionOrMemberOrPublic).build(BaseFieldSet.build(fields, Reference._id), data);
}
private @NotNull DefinitionEntity buildDefinitionEntity(DefinitionPersist persist){
DefinitionEntity data = new DefinitionEntity();
if (persist == null) return data;
if (!this.conventionService.isListNullOrEmpty(persist.getFields())){
data.setFields(new ArrayList<>());
for (FieldPersist fieldPersist: persist.getFields()) {
data.getFields().add(this.buildFieldEntity(fieldPersist));
}
}
return data;
}
private @NotNull FieldEntity buildFieldEntity(FieldPersist persist){
FieldEntity data = new FieldEntity();
if (persist == null) return data;
data.setCode(persist.getCode());
data.setDataType(persist.getDataType());
data.setValue(persist.getValue());
return data;
}
public void deleteAndSave(UUID id) throws MyForbiddenException, InvalidApplicationException {
logger.debug("deleting : {}", id);
this.authorizationService.authorizeForce(Permission.DeleteReference);
this.deleterFactory.deleter(ReferenceDeleter.class).deleteAndSaveByIds(List.of(id));
}
// public List<FetcherReference> searchReference(ReferenceType externalType, String query, String type) throws HugeResultSet, NoURLFound, InvalidApplicationException {
// ExternalUrlCriteria externalUrlCriteria = new ExternalUrlCriteria(query);
//
// List<Map<String, String>> remoteRepos = this.apiContext.getOperationsContext().getRemoteFetcher().get(externalType, externalUrlCriteria, type);
//
// List<FetcherReference> list = this.fetchFromDb(externalType, query, type, remoteRepos);
//
// list.addAll(remoteRepos.stream().map(FetcherReference::fromRemoteModel).toList());
// list = list.stream().filter(x -> x.getName().toLowerCase().contains(query.toLowerCase())).collect(Collectors.toList());
// list.sort(Comparator.comparing(FetcherReference::getName));
// return list;
// }
public List<Reference> searchReference(ReferenceSearchLookup lookup) throws HugeResultSet, NoURLFound, InvalidApplicationException {
ExternalUrlCriteria externalUrlCriteria = new ExternalUrlCriteria(lookup.getLike());
List<Map<String, String>> remoteRepos = this.apiContext.getOperationsContext().getRemoteFetcher().get(lookup.getType(), externalUrlCriteria, lookup.getKey());
List<Reference> externalModels = this.builderFactory.builder(ReferenceSearchBuilder.class).authorize(AuthorizationFlags.OwnerOrPermissionOrMemberOrPublic).build(lookup.getProject(), remoteRepos);
List<Reference> models = this.fetchFromDb(lookup);
models.addAll(externalModels);
models = models.stream().filter(x -> x.getLabel().toLowerCase().contains(lookup.getLike().toLowerCase())).collect(Collectors.toList());
models.sort(Comparator.comparing(Reference::getLabel));
return models;
}
private List<Reference> fetchFromDb(ReferenceSearchLookup lookup){
List<String> fields = Arrays.asList(Reference._id, Reference._label, Reference._description, Reference._abbreviation, Reference._source,
Reference._isActive, Reference._createdAt, Reference._updatedAt, Reference._type, Reference._definition, Reference._reference, Reference._sourceType );
lookup.setProject(new BaseFieldSet(fields));
ReferenceQuery query = lookup.enrich(this.queryFactory).authorize(AuthorizationFlags.OwnerOrPermissionOrMemberOrPublic);
List<ReferenceEntity> data = query.collectAs(lookup.getProject());
return this.builderFactory.builder(ReferenceBuilder.class).authorize(AuthorizationFlags.OwnerOrPermissionOrMemberOrPublic).build(lookup.getProject(), data);
}
// private List<FetcherReference> fetchFromDb(ReferenceType externalType, String query, String type, List<Map<String, String>> remoteRepos) throws InvalidApplicationException {
// List<FetcherReference> list = new LinkedList<>();
// switch (externalType) {
// case DataRepositories:
// case PubRepositories:
// case Journals: {
// DataRepositoryCriteria criteria = new DataRepositoryCriteria();
// if (!query.isEmpty()) criteria.setLike(query);
// criteria.setCreationUserId(this.userScope.getUserId());
// if (type.equals("")) {
// List<DataRepository> dataRepositoryList = (this.apiContext.getOperationsContext().getDatabaseRepository().getDataRepositoryDao().getWithCriteria(criteria)).toList();
// list = dataRepositoryList.stream().map(item -> new FetcherReference().fromDataRepository(item)).collect(Collectors.toList());
// }
// }
// case Registries: {
// RegistryCriteria criteria = new RegistryCriteria();
// if (!query.isEmpty()) criteria.setLike(query);
// criteria.setCreationUserId(this.userScope.getUserId());
// if (type.equals("")) {
// List<Registry> registryList = (this.apiContext.getOperationsContext().getDatabaseRepository().getRegistryDao().getWithCriteria(criteria)).toList();
// list = registryList.stream().map(item -> new FetcherReference().fromRegistry(item)).collect(Collectors.toList());
// }
// }
// case Services:
// {
// ServiceCriteria criteria = new ServiceCriteria();
//
// if (!query.isEmpty()) criteria.setLike(query);
// criteria.setCreationUserId(this.userScope.getUserId());
//
// if (type.equals("")) {
// List<Service> serviceList = (this.apiContext.getOperationsContext().getDatabaseRepository().getServiceDao().getWithCriteria(criteria)).toList();
// list = serviceList.stream().map(item -> new FetcherReference().fromService(item)).collect(Collectors.toList());
// }
// }
// case Datasets:{
//
// ExternalDatasetCriteria criteria = apiContext.getOperationsContext().getBuilderFactory().getBuilder(ExternalDatasetCriteriaBuilder.class).like(query).build();
//
// criteria.setCreationUserId(this.userScope.getUserId());
// QueryableList<ExternalDataset> items = apiContext.getOperationsContext().getDatabaseRepository().getExternalDatasetDao().getWithCriteria(criteria);
//
// list = items.select(item -> new FetcherReference().fromDataset(item));
// }
// case Taxonomies:
// case Publications:
// case Licenses:
// break;
// }
//
// return list;
// }
public List<Reference> searchReferenceWithDefinition(ReferenceDefinitionSearchLookup lookup) throws HugeResultSet, NoURLFound, InvalidApplicationException {
ReferenceTypeQuery query = this.queryFactory.query(ReferenceTypeQuery.class).authorize(AuthorizationFlags.OwnerOrPermissionOrMemberOrPublic).ids(lookup.getReferenceTypeId());
List<ReferenceTypeEntity> datas = query.collectAs(lookup.getProject());
if (datas.size() != 1 ){
return null;
}
ReferenceTypeDefinitionEntity definition = this.xmlHandlingService.fromXmlSafe(ReferenceTypeDefinitionEntity.class, datas.get(0).getDefinition());
List<Map<String, String>> remoteRepos = this.getAll(definition.getSources());
return null;
}
private List<Map<String, String>> getAll ( List<ReferenceTypeSourceBaseConfigurationEntity> sources){
List<Map<String, String>> results = new LinkedList<>();
if (sources == null || sources.isEmpty()) {
return results;
}
sources.sort(Comparator.comparing(ReferenceTypeSourceBaseConfigurationEntity::getOrdinal));
List<ReferenceTypeSourceExternalApiConfigurationEntity> apiSources = sources.stream().filter(x-> ReferenceTypeSourceType.API.equals(x.getType())).map(x-> (ReferenceTypeSourceExternalApiConfigurationEntity)x).toList();
apiSources.forEach(source -> {
try {
String auth = null;
if (source.getAuth()!= null) {
//auth = this.getAuthentication(source.getAuth());
}
results.addAll(getAllResultsFromUrl(source.getUrl(), null, source.getResults(), source.getPaginationPath(), null, source.getLabel(), source.getKey(), source.getContentType(), source.getFirstPage(), source.getRequestBody(), source.getHttpMethod(), source.getFilterType(), source.getQueries(), auth));
} catch (Exception e) {
logger.error(e.getLocalizedMessage(), e);
}
});
return results;
}
private String getAuthentication(AuthenticationConfigurationEntity authenticationConfiguration) {
HttpMethod method = HttpMethod.valueOf(authenticationConfiguration.getAuthMethod().name());
Map<String, Object> response = this.client.method(method).uri(authenticationConfiguration.getAuthUrl())
.contentType(MediaType.APPLICATION_JSON)
.bodyValue(this.parseBodyString(authenticationConfiguration.getAuthRequestBody()))
.exchangeToMono(mono -> mono.bodyToMono(new ParameterizedTypeReference<Map<String, Object>>() {
})).block();
return authenticationConfiguration.getType() + " " + response.get(authenticationConfiguration.getAuthTokenPath());
}
private String parseBodyString(String bodyString) {
String finalBodyString = bodyString;
if (bodyString.contains("{env:")) {
int index = bodyString.indexOf("{env: ");
while (index >= 0) {
int endIndex = bodyString.indexOf("}", index + 6);
String envName = bodyString.substring(index + 6, endIndex);
finalBodyString = finalBodyString.replace("{env: " + envName + "}", System.getenv(envName));
index = bodyString.indexOf("{env: ", index + 6);
}
}
return finalBodyString;
}
private List<Map<String, String>> getAllResultsFromUrl(String urlPath, FetchStrategy fetchStrategy, final ResultsConfigurationEntity jsonResultsPath, final String jsonPaginationPath, ExternalUrlCriteria externalUrlCriteria, String tag, String key, String contentType, String firstPage, String requestBody, ReferenceTypeExternalApiHTTPMethodType requestType, String filterType, List<QueryConfigEntity> queries, String auth) throws Exception {
Set<Integer> pages = new HashSet<>();
//String replacedPath = replaceCriteriaOnUrl(urlPath, externalUrlCriteria, firstPage, queries);
String replacedUrlPath = urlPath;
//String replacedBody = replaceCriteriaOnUrl(requestBody, externalUrlCriteria, firstPage, queries);
String replacedUrlBody = requestBody;
Results results = getResultsFromUrl(replacedUrlPath, jsonResultsPath, jsonPaginationPath, contentType, replacedUrlBody, requestType, auth);
if(results != null) {
if (filterType != null && filterType.equals("local") && (externalUrlCriteria.getLike() != null && !externalUrlCriteria.getLike().isEmpty())) {
results.setResults(results.getResults().stream()
.filter(r -> r.get("name").toLowerCase().contains(externalUrlCriteria.getLike().toLowerCase()))
.collect(Collectors.toList()));
}
if (fetchStrategy == FetchStrategy.FIRST)
return results.getResults().stream().peek(x -> x.put("tag", tag)).peek(x -> x.put("key", key)).collect(Collectors.toList());
if (results.getPagination() != null && results.getPagination().get("pages") != null) //if has more pages, add them to the pages set
for (int i = 2; i <= results.getPagination().get("pages"); i++)
pages.add(i);
//Long maxResults = configLoader.getExternalUrls().getMaxresults();
Long maxResults = Long.valueOf(1000);
if ((maxResults > 0) && (results.getPagination().get("count") > maxResults))
throw new HugeResultSet("The submitted search query " + externalUrlCriteria.getLike() + " is about to return " + results.getPagination().get("count") + " results... Please submit a more detailed search query");
Optional<Results> optionalResults = pages.parallelStream()
.map(page -> getResultsFromUrl(urlPath + "&page=" + page, jsonResultsPath, jsonPaginationPath, contentType, replacedUrlBody, requestType, auth))
.filter(Objects::nonNull)
.reduce((result1, result2) -> {
result1.getResults().addAll(result2.getResults());
return result1;
});
Results remainingResults = optionalResults.orElseGet(Results::new);
remainingResults.getResults().addAll(results.getResults());
return remainingResults.getResults().stream().peek(x -> x.put("tag", tag)).peek(x -> x.put("key", key)).collect(Collectors.toList());
}
else {
return new LinkedList<>();
}
}
protected Results getResultsFromUrl(String urlString, ResultsConfigurationEntity jsonResultsPath, String jsonPaginationPath, String contentType, String requestBody, ReferenceTypeExternalApiHTTPMethodType httpMethod, String auth) {
try {
ResponseEntity<String> response;
JsonNode jsonBody = new ObjectMapper().readTree(requestBody);
response = this.client.method(HttpMethod.valueOf(httpMethod.name())).uri(urlString).headers(httpHeaders -> {
if (contentType != null && !contentType.isEmpty()) {
httpHeaders.setAccept(Collections.singletonList(MediaType.valueOf(contentType)));
httpHeaders.setContentType(MediaType.valueOf(contentType));
}
if (auth != null) {
httpHeaders.set("Authorization", auth);
}
}).bodyValue(jsonBody).retrieve().toEntity(String.class).block();
if (response.getStatusCode() == HttpStatus.OK) { // success
Results results = new Results();
if (response.getHeaders().get("Content-Type").get(0).contains("json")) {
DocumentContext jsonContext = JsonPath.parse(response.getBody());
results = this.getFromJson(jsonContext, jsonResultsPath);
}
if (results.getPagination().size() == 0) {
results.getPagination().put("pages", 1);
results.getPagination().put("count", results.getResults().size());
}
return results;
}
} catch (Exception exception) {
logger.error(exception.getMessage(), exception);
}
return null;
}
public static Results getFromJson(DocumentContext jsonContext, ResultsConfigurationEntity jsonResultsPath) {
return new Results(parseData(jsonContext, jsonResultsPath),
new HashMap<>(1, 1));
}
private static List<Map<String, String>> parseData (DocumentContext jsonContext, ResultsConfigurationEntity jsonResultsPath) {
List <Map<String, String>> rawData = jsonContext.read(jsonResultsPath.getResultsArrayPath());
List<Map<String, String>> parsedData = new ArrayList<>();
rawData.forEach(stringObjectMap -> {
Map<String, String> map = new HashMap<>();
jsonResultsPath.getFieldsMapping().forEach(field ->{
String pathValue = field.getResponsePath();
if (stringObjectMap.containsKey(pathValue)){
map.put(field.getCode(), stringObjectMap.get(pathValue));
parsedData.add(map);
}
});
});
return parsedData;
}
}