2023-10-26 11:07:04 +02:00
package eu.eudat.logic.services.references ;
2023-10-25 11:42:34 +02:00
2023-10-27 16:35:35 +02:00
import com.fasterxml.jackson.core.JsonProcessingException ;
2023-11-15 15:09:24 +01:00
import com.fasterxml.jackson.databind.JsonNode ;
import com.fasterxml.jackson.databind.ObjectMapper ;
import com.jayway.jsonpath.DocumentContext ;
import com.jayway.jsonpath.JsonPath ;
2023-10-27 16:35:35 +02:00
import eu.eudat.authorization.AuthorizationFlags ;
2023-10-25 11:42:34 +02:00
import eu.eudat.authorization.Permission ;
2023-10-27 16:35:35 +02:00
import eu.eudat.commons.XmlHandlingService ;
import eu.eudat.commons.enums.IsActive ;
2023-11-15 15:09:24 +01:00
import eu.eudat.commons.enums.ReferenceTypeExternalApiHTTPMethodType ;
import eu.eudat.commons.enums.ReferenceTypeSourceType ;
2023-10-25 11:42:34 +02:00
import eu.eudat.commons.scope.user.UserScope ;
2023-10-27 16:35:35 +02:00
import eu.eudat.commons.types.reference.DefinitionEntity ;
import eu.eudat.commons.types.reference.FieldEntity ;
2023-11-15 15:09:24 +01:00
import eu.eudat.commons.types.referencetype.* ;
2023-10-27 16:35:35 +02:00
import eu.eudat.configurations.referencetype.ReferenceTypeProperties ;
2023-10-25 11:42:34 +02:00
import eu.eudat.convention.ConventionService ;
2023-10-27 16:35:35 +02:00
import eu.eudat.data.ReferenceEntity ;
2023-11-15 15:09:24 +01:00
import eu.eudat.data.ReferenceTypeEntity ;
2023-10-25 11:42:34 +02:00
import eu.eudat.logic.proxy.config.ExternalUrlCriteria ;
2023-11-15 15:09:24 +01:00
import eu.eudat.logic.proxy.config.FetchStrategy ;
2023-10-25 11:42:34 +02:00
import eu.eudat.logic.proxy.config.exceptions.HugeResultSet ;
import eu.eudat.logic.proxy.config.exceptions.NoURLFound ;
import eu.eudat.logic.proxy.fetching.RemoteFetcher ;
2023-11-15 15:09:24 +01:00
import eu.eudat.logic.proxy.fetching.entities.Results ;
2023-10-25 11:42:34 +02:00
import eu.eudat.logic.services.ApiContext ;
2023-10-27 16:35:35 +02:00
import eu.eudat.model.Reference ;
import eu.eudat.model.builder.ReferenceBuilder ;
2023-10-30 16:15:40 +01:00
import eu.eudat.model.builder.referencesearch.ReferenceSearchBuilder ;
2023-10-26 11:07:04 +02:00
import eu.eudat.model.deleter.ReferenceDeleter ;
2023-10-27 16:35:35 +02:00
import eu.eudat.model.persist.ReferencePersist ;
import eu.eudat.model.persist.referencedefinition.DefinitionPersist ;
import eu.eudat.model.persist.referencedefinition.FieldPersist ;
2023-10-31 16:57:41 +01:00
import eu.eudat.query.ReferenceQuery ;
2023-11-15 15:09:24 +01:00
import eu.eudat.query.ReferenceTypeQuery ;
import eu.eudat.query.lookup.ReferenceDefinitionSearchLookup ;
2023-10-30 16:15:40 +01:00
import eu.eudat.query.lookup.ReferenceSearchLookup ;
2023-10-25 11:42:34 +02:00
import gr.cite.commons.web.authz.service.AuthorizationService ;
import gr.cite.tools.data.builder.BuilderFactory ;
import gr.cite.tools.data.deleter.DeleterFactory ;
import gr.cite.tools.data.query.QueryFactory ;
2023-10-27 16:35:35 +02:00
import gr.cite.tools.exception.MyApplicationException ;
2023-10-25 11:42:34 +02:00
import gr.cite.tools.exception.MyForbiddenException ;
2023-10-27 16:35:35 +02:00
import gr.cite.tools.exception.MyNotFoundException ;
import gr.cite.tools.exception.MyValidationException ;
import gr.cite.tools.fieldset.BaseFieldSet ;
import gr.cite.tools.fieldset.FieldSet ;
2023-10-25 11:42:34 +02:00
import gr.cite.tools.logging.LoggerService ;
2023-10-27 16:35:35 +02:00
import gr.cite.tools.logging.MapLogEntry ;
2023-10-25 11:42:34 +02:00
import jakarta.persistence.EntityManager ;
2023-10-27 16:35:35 +02:00
import jakarta.xml.bind.JAXBException ;
2023-11-21 11:51:26 +01:00
import net.minidev.json.JSONArray ;
2023-10-27 16:35:35 +02:00
import org.jetbrains.annotations.NotNull ;
2023-10-25 11:42:34 +02:00
import org.slf4j.LoggerFactory ;
import org.springframework.context.MessageSource ;
2023-10-27 16:35:35 +02:00
import org.springframework.context.i18n.LocaleContextHolder ;
2023-11-15 15:09:24 +01:00
import org.springframework.core.ParameterizedTypeReference ;
import org.springframework.http.HttpMethod ;
import org.springframework.http.HttpStatus ;
import org.springframework.http.MediaType ;
import org.springframework.http.ResponseEntity ;
import org.springframework.http.client.reactive.ReactorClientHttpConnector ;
import org.springframework.http.codec.json.Jackson2JsonDecoder ;
import org.springframework.web.reactive.function.client.WebClient ;
import reactor.netty.http.client.HttpClient ;
2023-10-25 11:42:34 +02:00
import javax.management.InvalidApplicationException ;
2023-10-27 16:35:35 +02:00
import javax.xml.parsers.ParserConfigurationException ;
import javax.xml.transform.TransformerException ;
import java.time.Instant ;
2023-10-25 11:42:34 +02:00
import java.util.* ;
import java.util.stream.Collectors ;
@org.springframework.stereotype.Service
2023-10-26 11:07:04 +02:00
public class ReferenceService {
2023-10-25 11:42:34 +02:00
2023-10-26 11:07:04 +02:00
private static final LoggerService logger = new LoggerService ( LoggerFactory . getLogger ( ReferenceService . class ) ) ;
2023-10-25 11:42:34 +02:00
private final ApiContext apiContext ;
private final UserScope userScope ;
private final RemoteFetcher remoteFetcher ;
private final EntityManager entityManager ;
private final AuthorizationService authorizationService ;
private final DeleterFactory deleterFactory ;
private final BuilderFactory builderFactory ;
private final ConventionService conventionService ;
private final MessageSource messageSource ;
private final QueryFactory queryFactory ;
2023-10-27 16:35:35 +02:00
private final XmlHandlingService xmlHandlingService ;
private final ReferenceTypeProperties referenceTypeProperties ;
2023-10-25 11:42:34 +02:00
2023-11-15 15:09:24 +01:00
private final WebClient client ;
2023-10-26 11:07:04 +02:00
public ReferenceService ( ApiContext apiContext ,
UserScope userScope ,
RemoteFetcher remoteFetcher ,
EntityManager entityManager ,
AuthorizationService authorizationService ,
DeleterFactory deleterFactory ,
BuilderFactory builderFactory ,
ConventionService conventionService ,
MessageSource messageSource ,
2023-10-27 16:35:35 +02:00
QueryFactory queryFactory ,
XmlHandlingService xmlHandlingService ,
ReferenceTypeProperties referenceTypeProperties ) {
2023-10-25 11:42:34 +02:00
this . apiContext = apiContext ;
this . userScope = userScope ;
this . remoteFetcher = remoteFetcher ;
this . entityManager = entityManager ;
this . authorizationService = authorizationService ;
this . deleterFactory = deleterFactory ;
this . builderFactory = builderFactory ;
this . conventionService = conventionService ;
this . messageSource = messageSource ;
this . queryFactory = queryFactory ;
2023-10-27 16:35:35 +02:00
this . xmlHandlingService = xmlHandlingService ;
this . referenceTypeProperties = referenceTypeProperties ;
2023-11-15 15:09:24 +01:00
this . client = WebClient . builder ( ) . codecs ( clientCodecConfigurer - > {
clientCodecConfigurer . defaultCodecs ( ) . jackson2JsonDecoder ( new Jackson2JsonDecoder ( new ObjectMapper ( ) , MediaType . APPLICATION_JSON ) ) ;
clientCodecConfigurer . defaultCodecs ( ) . maxInMemorySize ( 2 * ( ( int ) Math . pow ( 1024 , 3 ) ) ) ; //GK: Why here???
}
) . clientConnector ( new ReactorClientHttpConnector ( HttpClient . create ( ) . followRedirect ( true ) ) ) . build ( ) ;
2023-10-27 16:35:35 +02:00
}
public Reference persist ( ReferencePersist model , FieldSet fields ) throws MyForbiddenException , MyValidationException , MyApplicationException , MyNotFoundException , InvalidApplicationException , JAXBException , JsonProcessingException , TransformerException , ParserConfigurationException {
logger . debug ( new MapLogEntry ( " persisting data " ) . And ( " model " , model ) . And ( " fields " , fields ) ) ;
this . authorizationService . authorizeForce ( Permission . EditDmpBlueprint ) ;
Boolean isUpdate = this . conventionService . isValidGuid ( model . getId ( ) ) ;
ReferenceEntity data ;
if ( isUpdate ) {
data = this . entityManager . find ( ReferenceEntity . class , model . getId ( ) ) ;
if ( data = = null )
throw new MyNotFoundException ( messageSource . getMessage ( " General_ItemNotFound " , new Object [ ] { model . getId ( ) , Reference . class . getSimpleName ( ) } , LocaleContextHolder . getLocale ( ) ) ) ;
} else {
data = new ReferenceEntity ( ) ;
data . setId ( UUID . randomUUID ( ) ) ;
data . setIsActive ( IsActive . Active ) ;
data . setCreatedAt ( Instant . now ( ) ) ;
}
data . setLabel ( model . getLabel ( ) ) ;
data . setType ( model . getType ( ) ) ;
data . setDescription ( model . getDescription ( ) ) ;
data . setDefinition ( this . xmlHandlingService . toXmlSafe ( this . buildDefinitionEntity ( model . getDefinition ( ) ) ) ) ;
data . setUpdatedAt ( Instant . now ( ) ) ;
data . setReference ( model . getReference ( ) ) ;
data . setAbbreviation ( model . getAbbreviation ( ) ) ;
data . setSource ( model . getSource ( ) ) ;
2023-10-30 09:43:50 +01:00
data . setSourceType ( model . getSourceType ( ) ) ;
2023-10-27 16:35:35 +02:00
if ( isUpdate ) this . entityManager . merge ( data ) ;
else this . entityManager . persist ( data ) ;
this . entityManager . flush ( ) ;
2023-11-17 18:01:44 +01:00
return this . builderFactory . builder ( ReferenceBuilder . class ) . authorize ( AuthorizationFlags . OwnerOrDmpAssociatedOrPermissionOrPublic ) . build ( BaseFieldSet . build ( fields , Reference . _id ) , data ) ;
2023-10-27 16:35:35 +02:00
}
private @NotNull DefinitionEntity buildDefinitionEntity ( DefinitionPersist persist ) {
DefinitionEntity data = new DefinitionEntity ( ) ;
if ( persist = = null ) return data ;
if ( ! this . conventionService . isListNullOrEmpty ( persist . getFields ( ) ) ) {
data . setFields ( new ArrayList < > ( ) ) ;
for ( FieldPersist fieldPersist : persist . getFields ( ) ) {
data . getFields ( ) . add ( this . buildFieldEntity ( fieldPersist ) ) ;
}
}
return data ;
}
private @NotNull FieldEntity buildFieldEntity ( FieldPersist persist ) {
FieldEntity data = new FieldEntity ( ) ;
if ( persist = = null ) return data ;
data . setCode ( persist . getCode ( ) ) ;
data . setDataType ( persist . getDataType ( ) ) ;
2023-11-07 12:15:25 +01:00
data . setValue ( persist . getValue ( ) ) ;
2023-10-27 16:35:35 +02:00
return data ;
2023-10-25 11:42:34 +02:00
}
public void deleteAndSave ( UUID id ) throws MyForbiddenException , InvalidApplicationException {
logger . debug ( " deleting : {} " , id ) ;
2023-10-26 13:38:18 +02:00
this . authorizationService . authorizeForce ( Permission . DeleteReference ) ;
2023-10-25 11:42:34 +02:00
2023-10-26 11:07:04 +02:00
this . deleterFactory . deleter ( ReferenceDeleter . class ) . deleteAndSaveByIds ( List . of ( id ) ) ;
2023-10-25 11:42:34 +02:00
}
2023-10-30 16:15:40 +01:00
// public List<FetcherReference> searchReference(ReferenceType externalType, String query, String type) throws HugeResultSet, NoURLFound, InvalidApplicationException {
// ExternalUrlCriteria externalUrlCriteria = new ExternalUrlCriteria(query);
//
// List<Map<String, String>> remoteRepos = this.apiContext.getOperationsContext().getRemoteFetcher().get(externalType, externalUrlCriteria, type);
//
// List<FetcherReference> list = this.fetchFromDb(externalType, query, type, remoteRepos);
//
// list.addAll(remoteRepos.stream().map(FetcherReference::fromRemoteModel).toList());
// list = list.stream().filter(x -> x.getName().toLowerCase().contains(query.toLowerCase())).collect(Collectors.toList());
// list.sort(Comparator.comparing(FetcherReference::getName));
// return list;
// }
public List < Reference > searchReference ( ReferenceSearchLookup lookup ) throws HugeResultSet , NoURLFound , InvalidApplicationException {
ExternalUrlCriteria externalUrlCriteria = new ExternalUrlCriteria ( lookup . getLike ( ) ) ;
List < Map < String , String > > remoteRepos = this . apiContext . getOperationsContext ( ) . getRemoteFetcher ( ) . get ( lookup . getType ( ) , externalUrlCriteria , lookup . getKey ( ) ) ;
2023-10-25 11:42:34 +02:00
2023-11-17 18:01:44 +01:00
List < Reference > externalModels = this . builderFactory . builder ( ReferenceSearchBuilder . class ) . authorize ( AuthorizationFlags . OwnerOrDmpAssociatedOrPermissionOrPublic ) . build ( lookup . getProject ( ) , remoteRepos ) ;
2023-10-31 16:57:41 +01:00
List < Reference > models = this . fetchFromDb ( lookup ) ;
models . addAll ( externalModels ) ;
2023-10-25 11:42:34 +02:00
2023-10-30 16:15:40 +01:00
models = models . stream ( ) . filter ( x - > x . getLabel ( ) . toLowerCase ( ) . contains ( lookup . getLike ( ) . toLowerCase ( ) ) ) . collect ( Collectors . toList ( ) ) ;
models . sort ( Comparator . comparing ( Reference : : getLabel ) ) ;
2023-10-27 16:35:35 +02:00
2023-10-30 16:15:40 +01:00
return models ;
2023-10-25 11:42:34 +02:00
}
2023-10-31 16:57:41 +01:00
private List < Reference > fetchFromDb ( ReferenceSearchLookup lookup ) {
2023-10-27 16:35:35 +02:00
2023-10-31 16:57:41 +01:00
List < String > fields = Arrays . asList ( Reference . _id , Reference . _label , Reference . _description , Reference . _abbreviation , Reference . _source ,
Reference . _isActive , Reference . _createdAt , Reference . _updatedAt , Reference . _type , Reference . _definition , Reference . _reference , Reference . _sourceType ) ;
lookup . setProject ( new BaseFieldSet ( fields ) ) ;
2023-10-27 16:35:35 +02:00
2023-11-17 18:01:44 +01:00
ReferenceQuery query = lookup . enrich ( this . queryFactory ) . authorize ( AuthorizationFlags . OwnerOrDmpAssociatedOrPermissionOrPublic ) ;
2023-11-13 08:39:31 +01:00
List < ReferenceEntity > data = query . collectAs ( lookup . getProject ( ) ) ;
2023-11-17 18:01:44 +01:00
return this . builderFactory . builder ( ReferenceBuilder . class ) . authorize ( AuthorizationFlags . OwnerOrDmpAssociatedOrPermissionOrPublic ) . build ( lookup . getProject ( ) , data ) ;
2023-10-25 11:42:34 +02:00
}
2023-10-31 16:57:41 +01:00
// private List<FetcherReference> fetchFromDb(ReferenceType externalType, String query, String type, List<Map<String, String>> remoteRepos) throws InvalidApplicationException {
// List<FetcherReference> list = new LinkedList<>();
// switch (externalType) {
// case DataRepositories:
// case PubRepositories:
// case Journals: {
// DataRepositoryCriteria criteria = new DataRepositoryCriteria();
// if (!query.isEmpty()) criteria.setLike(query);
// criteria.setCreationUserId(this.userScope.getUserId());
// if (type.equals("")) {
// List<DataRepository> dataRepositoryList = (this.apiContext.getOperationsContext().getDatabaseRepository().getDataRepositoryDao().getWithCriteria(criteria)).toList();
// list = dataRepositoryList.stream().map(item -> new FetcherReference().fromDataRepository(item)).collect(Collectors.toList());
// }
// }
// case Registries: {
// RegistryCriteria criteria = new RegistryCriteria();
// if (!query.isEmpty()) criteria.setLike(query);
// criteria.setCreationUserId(this.userScope.getUserId());
// if (type.equals("")) {
// List<Registry> registryList = (this.apiContext.getOperationsContext().getDatabaseRepository().getRegistryDao().getWithCriteria(criteria)).toList();
// list = registryList.stream().map(item -> new FetcherReference().fromRegistry(item)).collect(Collectors.toList());
// }
// }
// case Services:
// {
// ServiceCriteria criteria = new ServiceCriteria();
//
// if (!query.isEmpty()) criteria.setLike(query);
// criteria.setCreationUserId(this.userScope.getUserId());
//
// if (type.equals("")) {
// List<Service> serviceList = (this.apiContext.getOperationsContext().getDatabaseRepository().getServiceDao().getWithCriteria(criteria)).toList();
// list = serviceList.stream().map(item -> new FetcherReference().fromService(item)).collect(Collectors.toList());
// }
// }
// case Datasets:{
//
// ExternalDatasetCriteria criteria = apiContext.getOperationsContext().getBuilderFactory().getBuilder(ExternalDatasetCriteriaBuilder.class).like(query).build();
//
// criteria.setCreationUserId(this.userScope.getUserId());
// QueryableList<ExternalDataset> items = apiContext.getOperationsContext().getDatabaseRepository().getExternalDatasetDao().getWithCriteria(criteria);
//
// list = items.select(item -> new FetcherReference().fromDataset(item));
// }
// case Taxonomies:
// case Publications:
// case Licenses:
// break;
// }
//
// return list;
// }
2023-11-15 15:09:24 +01:00
public List < Reference > searchReferenceWithDefinition ( ReferenceDefinitionSearchLookup lookup ) throws HugeResultSet , NoURLFound , InvalidApplicationException {
2023-11-17 18:01:44 +01:00
ReferenceTypeQuery query = this . queryFactory . query ( ReferenceTypeQuery . class ) . authorize ( AuthorizationFlags . OwnerOrDmpAssociatedOrPermissionOrPublic ) . ids ( lookup . getReferenceTypeId ( ) ) ;
2023-11-15 15:09:24 +01:00
List < ReferenceTypeEntity > datas = query . collectAs ( lookup . getProject ( ) ) ;
if ( datas . size ( ) ! = 1 ) {
return null ;
}
ReferenceTypeDefinitionEntity definition = this . xmlHandlingService . fromXmlSafe ( ReferenceTypeDefinitionEntity . class , datas . get ( 0 ) . getDefinition ( ) ) ;
2023-11-16 14:40:25 +01:00
List < Map < String , String > > remoteRepos = this . getAll ( definition . getSources ( ) , lookup ) ;
2023-11-15 15:09:24 +01:00
return null ;
}
2023-11-16 14:40:25 +01:00
private List < Map < String , String > > getAll ( List < ReferenceTypeSourceBaseConfigurationEntity > sources , ReferenceDefinitionSearchLookup lookup ) {
2023-11-15 15:09:24 +01:00
List < Map < String , String > > results = new LinkedList < > ( ) ;
if ( sources = = null | | sources . isEmpty ( ) ) {
return results ;
}
sources . sort ( Comparator . comparing ( ReferenceTypeSourceBaseConfigurationEntity : : getOrdinal ) ) ;
2023-11-16 14:40:25 +01:00
List < ReferenceTypeSourceExternalApiConfigurationEntity > apiSources = sources . stream ( ) . filter ( x - > ReferenceTypeSourceType . API . equals ( x . getType ( ) ) ) . map ( x - > ( ReferenceTypeSourceExternalApiConfigurationEntity ) x ) . toList ( ) ;
2023-11-15 15:09:24 +01:00
apiSources . forEach ( source - > {
try {
String auth = null ;
if ( source . getAuth ( ) ! = null ) {
//auth = this.getAuthentication(source.getAuth());
}
2023-11-16 14:40:25 +01:00
results . addAll ( getAllApiResultsFromUrl ( source . getUrl ( ) , null , source . getResults ( ) , source . getPaginationPath ( ) , lookup , source . getLabel ( ) , source . getKey ( ) , source . getContentType ( ) , source . getFirstPage ( ) , source . getRequestBody ( ) , source . getHttpMethod ( ) , source . getFilterType ( ) , source . getQueries ( ) , auth ) ) ;
2023-11-15 15:09:24 +01:00
} catch ( Exception e ) {
logger . error ( e . getLocalizedMessage ( ) , e ) ;
}
} ) ;
2023-11-16 14:40:25 +01:00
List < ReferenceTypeSourceStaticOptionConfigurationEntity > staticSources = sources . stream ( ) . filter ( x - > ReferenceTypeSourceType . STATIC . equals ( x . getType ( ) ) ) . map ( x - > ( ReferenceTypeSourceStaticOptionConfigurationEntity ) x ) . toList ( ) ;
staticSources . forEach ( source - > {
Map < String , String > map = new HashMap < > ( ) ;
source . getOptions ( ) . forEach ( option - > {
map . put ( option . getCode ( ) , option . getValue ( ) ) ;
2023-11-21 11:51:26 +01:00
map . put ( " tag " , source . getLabel ( ) ) ;
2023-11-16 14:40:25 +01:00
map . put ( " key " , source . getKey ( ) ) ;
} ) ;
results . add ( map ) ;
} ) ;
2023-11-15 15:09:24 +01:00
return results ;
}
private String getAuthentication ( AuthenticationConfigurationEntity authenticationConfiguration ) {
HttpMethod method = HttpMethod . valueOf ( authenticationConfiguration . getAuthMethod ( ) . name ( ) ) ;
Map < String , Object > response = this . client . method ( method ) . uri ( authenticationConfiguration . getAuthUrl ( ) )
. contentType ( MediaType . APPLICATION_JSON )
. bodyValue ( this . parseBodyString ( authenticationConfiguration . getAuthRequestBody ( ) ) )
. exchangeToMono ( mono - > mono . bodyToMono ( new ParameterizedTypeReference < Map < String , Object > > ( ) {
} ) ) . block ( ) ;
return authenticationConfiguration . getType ( ) + " " + response . get ( authenticationConfiguration . getAuthTokenPath ( ) ) ;
}
private String parseBodyString ( String bodyString ) {
String finalBodyString = bodyString ;
if ( bodyString . contains ( " {env: " ) ) {
int index = bodyString . indexOf ( " {env: " ) ;
while ( index > = 0 ) {
int endIndex = bodyString . indexOf ( " } " , index + 6 ) ;
String envName = bodyString . substring ( index + 6 , endIndex ) ;
finalBodyString = finalBodyString . replace ( " {env: " + envName + " } " , System . getenv ( envName ) ) ;
index = bodyString . indexOf ( " {env: " , index + 6 ) ;
}
}
return finalBodyString ;
}
2023-11-16 14:40:25 +01:00
private List < Map < String , String > > getAllApiResultsFromUrl ( String urlPath , FetchStrategy fetchStrategy , final ResultsConfigurationEntity resultsEntity , final String jsonPaginationPath , ReferenceDefinitionSearchLookup lookup , String label , String key , String contentType , String firstPage , String requestBody , ReferenceTypeExternalApiHTTPMethodType requestType , String filterType , List < QueryConfigEntity > queries , String auth ) throws Exception {
2023-11-15 15:09:24 +01:00
Set < Integer > pages = new HashSet < > ( ) ;
2023-11-16 14:40:25 +01:00
String replacedUrlPath = replaceLookupFields ( urlPath , lookup , firstPage , queries ) ;
String replacedUrlBody = replaceLookupFields ( requestBody , lookup , firstPage , queries ) ;
2023-11-15 15:09:24 +01:00
2023-11-16 14:40:25 +01:00
Results results = getResultsFromUrl ( replacedUrlPath , resultsEntity , jsonPaginationPath , contentType , replacedUrlBody , requestType , auth ) ;
2023-11-15 15:09:24 +01:00
if ( results ! = null ) {
2023-11-16 14:40:25 +01:00
if ( filterType ! = null & & filterType . equals ( " local " ) & & ( lookup . getLike ( ) ! = null & & ! lookup . getLike ( ) . isEmpty ( ) ) ) {
2023-11-15 15:09:24 +01:00
results . setResults ( results . getResults ( ) . stream ( )
2023-11-16 14:40:25 +01:00
. filter ( r - > r . get ( " name " ) . toLowerCase ( ) . contains ( lookup . getLike ( ) . toLowerCase ( ) ) )
2023-11-15 15:09:24 +01:00
. collect ( Collectors . toList ( ) ) ) ;
}
if ( fetchStrategy = = FetchStrategy . FIRST )
2023-11-21 11:51:26 +01:00
return results . getResults ( ) . stream ( ) . peek ( x - > x . put ( " tag " , label ) ) . peek ( x - > x . put ( " key " , key ) ) . collect ( Collectors . toList ( ) ) ;
2023-11-15 15:09:24 +01:00
if ( results . getPagination ( ) ! = null & & results . getPagination ( ) . get ( " pages " ) ! = null ) //if has more pages, add them to the pages set
for ( int i = 2 ; i < = results . getPagination ( ) . get ( " pages " ) ; i + + )
pages . add ( i ) ;
//Long maxResults = configLoader.getExternalUrls().getMaxresults();
Long maxResults = Long . valueOf ( 1000 ) ;
if ( ( maxResults > 0 ) & & ( results . getPagination ( ) . get ( " count " ) > maxResults ) )
2023-11-16 14:40:25 +01:00
throw new HugeResultSet ( " The submitted search query " + lookup . getLike ( ) + " is about to return " + results . getPagination ( ) . get ( " count " ) + " results... Please submit a more detailed search query " ) ;
2023-11-15 15:09:24 +01:00
Optional < Results > optionalResults = pages . parallelStream ( )
2023-11-16 14:40:25 +01:00
. map ( page - > getResultsFromUrl ( urlPath + " &page= " + page , resultsEntity , jsonPaginationPath , contentType , replacedUrlBody , requestType , auth ) )
2023-11-15 15:09:24 +01:00
. filter ( Objects : : nonNull )
. reduce ( ( result1 , result2 ) - > {
result1 . getResults ( ) . addAll ( result2 . getResults ( ) ) ;
return result1 ;
} ) ;
Results remainingResults = optionalResults . orElseGet ( Results : : new ) ;
remainingResults . getResults ( ) . addAll ( results . getResults ( ) ) ;
2023-11-21 11:51:26 +01:00
return remainingResults . getResults ( ) . stream ( ) . peek ( x - > x . put ( " tag " , label ) ) . peek ( x - > x . put ( " key " , key ) ) . collect ( Collectors . toList ( ) ) ;
2023-11-15 15:09:24 +01:00
}
else {
return new LinkedList < > ( ) ;
}
}
2023-11-16 14:40:25 +01:00
private String replaceLookupFields ( String urlPath , ReferenceDefinitionSearchLookup lookup , String firstPage , List < QueryConfigEntity > queries ) {
String completedPath = urlPath ;
2023-11-21 11:51:26 +01:00
if ( urlPath . contains ( " openaire " ) | | urlPath . contains ( " orcid " ) ) {
2023-11-16 14:40:25 +01:00
if ( lookup . getLike ( ) ! = null ) {
2023-11-21 11:51:26 +01:00
completedPath = completedPath . replace ( " {query} " , lookup . getLike ( ) ) ;
2023-11-16 14:40:25 +01:00
completedPath = completedPath . replace ( " {like} " , lookup . getLike ( ) ) ;
} else {
2023-11-21 11:51:26 +01:00
completedPath = completedPath . replace ( " {query} " , " * " ) ;
2023-11-16 14:40:25 +01:00
completedPath = completedPath . replace ( " {like} " , " * " ) ;
}
}
2023-11-21 11:51:26 +01:00
if ( urlPath . contains ( " {like} " ) ) {
if ( lookup . getLike ( ) ! = null ) {
completedPath = completedPath . replace ( " {like} " , lookup . getLike ( ) ) ;
} else {
completedPath = completedPath . replace ( " {like} " , " " ) ;
}
}
if ( urlPath . contains ( " {page} " ) ) {
2023-11-16 14:40:25 +01:00
if ( lookup . getPage ( ) ! = null & & lookup . getPage ( ) . getOffset ( ) > 0 ) {
completedPath = completedPath . replace ( " {page} " , String . valueOf ( lookup . getPage ( ) . getOffset ( ) ) ) ;
2023-11-21 11:51:26 +01:00
} else if ( firstPage ! = null ) {
completedPath = completedPath . replace ( " {page} " , firstPage ) ;
2023-11-16 14:40:25 +01:00
} else {
completedPath = completedPath . replace ( " {page} " , " 1 " ) ;
}
}
if ( urlPath . contains ( " {pageSize} " ) ) {
if ( lookup . getPage ( ) ! = null & & lookup . getPage ( ) . getSize ( ) > 0 ) {
completedPath = completedPath . replace ( " {pageSize} " , String . valueOf ( lookup . getPage ( ) . getSize ( ) ) ) ;
} else {
completedPath = completedPath . replace ( " {pageSize} " , " 100 " ) ;
}
}
return completedPath ;
}
protected Results getResultsFromUrl ( String urlString , ResultsConfigurationEntity resultsEntity , String jsonPaginationPath , String contentType , String requestBody , ReferenceTypeExternalApiHTTPMethodType httpMethod , String auth ) {
2023-11-15 15:09:24 +01:00
try {
ResponseEntity < String > response ;
JsonNode jsonBody = new ObjectMapper ( ) . readTree ( requestBody ) ;
response = this . client . method ( HttpMethod . valueOf ( httpMethod . name ( ) ) ) . uri ( urlString ) . headers ( httpHeaders - > {
if ( contentType ! = null & & ! contentType . isEmpty ( ) ) {
httpHeaders . setAccept ( Collections . singletonList ( MediaType . valueOf ( contentType ) ) ) ;
httpHeaders . setContentType ( MediaType . valueOf ( contentType ) ) ;
}
if ( auth ! = null ) {
httpHeaders . set ( " Authorization " , auth ) ;
}
} ) . bodyValue ( jsonBody ) . retrieve ( ) . toEntity ( String . class ) . block ( ) ;
if ( response . getStatusCode ( ) = = HttpStatus . OK ) { // success
Results results = new Results ( ) ;
if ( response . getHeaders ( ) . get ( " Content-Type " ) . get ( 0 ) . contains ( " json " ) ) {
DocumentContext jsonContext = JsonPath . parse ( response . getBody ( ) ) ;
2023-11-16 14:40:25 +01:00
results = this . getFromJson ( jsonContext , resultsEntity ) ;
2023-11-15 15:09:24 +01:00
}
if ( results . getPagination ( ) . size ( ) = = 0 ) {
results . getPagination ( ) . put ( " pages " , 1 ) ;
results . getPagination ( ) . put ( " count " , results . getResults ( ) . size ( ) ) ;
}
return results ;
}
} catch ( Exception exception ) {
logger . error ( exception . getMessage ( ) , exception ) ;
}
return null ;
}
2023-11-16 14:40:25 +01:00
public static Results getFromJson ( DocumentContext jsonContext , ResultsConfigurationEntity resultsEntity ) {
return new Results ( parseData ( jsonContext , resultsEntity ) ,
2023-11-15 15:09:24 +01:00
new HashMap < > ( 1 , 1 ) ) ;
}
2023-11-16 14:40:25 +01:00
private static List < Map < String , String > > parseData ( DocumentContext jsonContext , ResultsConfigurationEntity resultsEntity ) {
List < Map < String , String > > rawData = jsonContext . read ( resultsEntity . getResultsArrayPath ( ) ) ;
2023-11-15 15:09:24 +01:00
List < Map < String , String > > parsedData = new ArrayList < > ( ) ;
2023-11-21 11:51:26 +01:00
for ( Map < String , String > stringObjectMap : rawData ) {
2023-11-15 15:09:24 +01:00
Map < String , String > map = new HashMap < > ( ) ;
2023-11-21 11:51:26 +01:00
for ( ResultFieldsMappingConfigurationEntity field : resultsEntity . getFieldsMapping ( ) ) {
2023-11-15 15:09:24 +01:00
String pathValue = field . getResponsePath ( ) ;
2023-11-21 11:51:26 +01:00
if ( ! pathValue . contains ( " . " ) ) {
if ( stringObjectMap . containsKey ( pathValue ) ) {
//map.put(field.getCode(), stringObjectMap.get(pathValue));
map . put ( field . getCode ( ) , normalizeValue ( stringObjectMap . get ( pathValue ) ) ) ;
}
} else {
if ( stringObjectMap . containsKey ( pathValue . split ( " \\ . " ) [ 0 ] ) ) {
String value = null ;
Object fieldObj = stringObjectMap . get ( pathValue . split ( " \\ . " ) [ 0 ] ) ;
if ( fieldObj ! = null ) {
if ( fieldObj instanceof Map ) {
Object o = ( ( Map < String , Object > ) fieldObj ) . get ( pathValue . split ( " \\ . " ) [ 1 ] ) ;
if ( o instanceof String ) {
value = ( String ) o ;
}
else if ( o instanceof Integer ) {
value = String . valueOf ( o ) ;
}
} else if ( fieldObj instanceof List ) {
Object o = ( ( List < Map < String , ? > > ) fieldObj ) . get ( 0 ) . get ( pathValue . split ( " \\ . " ) [ 1 ] ) ;
if ( o instanceof String ) {
value = ( String ) o ;
}
else if ( o instanceof Integer ) {
value = String . valueOf ( o ) ;
}
}
}
if ( value ! = null ) {
map . put ( field . getCode ( ) , value ) ;
}
}
2023-11-15 15:09:24 +01:00
}
2023-11-21 11:51:26 +01:00
}
2023-11-16 14:40:25 +01:00
parsedData . add ( map ) ;
2023-11-21 11:51:26 +01:00
}
2023-11-15 15:09:24 +01:00
return parsedData ;
}
2023-11-21 11:51:26 +01:00
private static String normalizeValue ( Object value ) {
if ( value instanceof JSONArray ) {
JSONArray jarr = ( JSONArray ) value ;
if ( jarr . get ( 0 ) instanceof String ) {
return jarr . get ( 0 ) . toString ( ) ;
} else {
for ( Object o : jarr ) {
if ( ( o instanceof Map ) & & ( ( Map ) o ) . containsKey ( " content " ) ) {
try {
return ( ( Map < String , String > ) o ) . get ( " content " ) ;
}
catch ( ClassCastException e ) {
if ( ( ( Map < ? , ? > ) o ) . get ( " content " ) instanceof Integer ) {
return String . valueOf ( ( ( Map < ? , ? > ) o ) . get ( " content " ) ) ;
}
return null ;
}
}
}
}
} else if ( value instanceof Map ) {
String key = ( ( Map < String , String > ) value ) . containsKey ( " $ " ) ? " $ " : " content " ;
return ( ( Map < String , String > ) value ) . get ( key ) ;
}
return value ! = null ? value . toString ( ) : null ;
}
2023-10-25 11:42:34 +02:00
}