2022-02-04 10:12:15 +01:00
package eu.dnetlib.openaire.common ;
2022-02-07 09:03:36 +01:00
import static eu.dnetlib.openaire.common.Utils.escape ;
2022-02-04 10:12:15 +01:00
import java.io.IOException ;
import java.nio.charset.Charset ;
import java.util.List ;
import java.util.Map ;
import java.util.Objects ;
import java.util.Queue ;
import java.util.concurrent.LinkedBlockingQueue ;
import java.util.function.Function ;
import java.util.stream.Collectors ;
import org.apache.commons.io.IOUtils ;
import org.apache.commons.lang3.StringUtils ;
import org.apache.commons.logging.Log ;
import org.apache.commons.logging.LogFactory ;
import org.springframework.beans.factory.annotation.Autowired ;
import org.springframework.cache.annotation.CacheEvict ;
import org.springframework.cache.annotation.Cacheable ;
import org.springframework.core.io.ClassPathResource ;
import org.springframework.scheduling.annotation.Scheduled ;
import org.springframework.stereotype.Component ;
2022-02-07 09:03:36 +01:00
import com.google.common.collect.Lists ;
import com.google.common.escape.Escaper ;
import com.google.common.xml.XmlEscapers ;
2022-02-07 10:09:18 +01:00
import eu.dnetlib.DnetOpenaireExporterProperties ;
2022-02-07 09:03:36 +01:00
import eu.dnetlib.enabling.datasources.common.DsmException ;
import eu.dnetlib.enabling.datasources.common.DsmRuntimeException ;
import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpException ;
import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpService ;
import eu.dnetlib.openaire.context.Context ;
import eu.dnetlib.openaire.context.ContextMappingUtils ;
import eu.dnetlib.openaire.dsm.dao.utils.IndexDsInfo ;
2022-02-04 10:12:15 +01:00
/ * *
* Created by claudio on 20 / 10 / 2016 .
* /
@Component
public class ISClientImpl implements ISClient {
2022-02-07 09:03:36 +01:00
private static final Log log = LogFactory . getLog ( ISClientImpl . class ) ;
2022-02-04 10:12:15 +01:00
@Autowired
2022-02-07 10:09:18 +01:00
private DnetOpenaireExporterProperties config ;
2022-02-04 10:12:15 +01:00
@Autowired
private ISLookUpService isLookUpService ;
@Override
@Cacheable ( " indexdsinfo-cache " )
public IndexDsInfo calculateCurrentIndexDsInfo ( ) throws DsmException {
log . warn ( " calculateCurrentIndexDsInfo(): not using cache " ) ;
final String [ ] arr ;
try {
arr = _isLookUp ( _getQuery ( config . getFindIndexDsInfo ( ) ) ) . split ( " @@@ " ) ;
return new IndexDsInfo (
_isLookUp ( _getQuery ( config . getFindSolrIndexUrl ( ) ) ) ,
arr [ 0 ] . trim ( ) , arr [ 1 ] . trim ( ) , arr [ 2 ] . trim ( ) ) ;
} catch ( IOException | ISLookUpException e ) {
throw new DsmException ( " unable fetch index DS information from IS " ) ;
}
}
@Override
@Cacheable ( " objectstoreid-cache " )
public String getObjectStoreId ( final String dsId ) throws DsmException {
log . warn ( String . format ( " getObjectStoreId(%s): not using cache " , dsId ) ) ;
try {
final String xqueryTemplate = _getQuery ( config . getFindObjectStore ( ) ) ;
return _isLookUp ( String . format ( xqueryTemplate , dsId ) ) ;
} catch ( IOException | ISLookUpException e ) {
throw new DsmException ( " unble to find objectstore for ds " + dsId ) ;
}
}
@Override
@Cacheable ( " context-cache-funder " )
public Map < String , Context > getFunderContextMap ( ) throws IOException {
return _processContext ( _getQuery ( config . getFindFunderContexts ( ) ) ) ;
}
@Override
@Cacheable ( " context-cache-community " )
public Map < String , Context > getCommunityContextMap ( ) throws IOException {
return _processContext ( _getQuery ( config . getFindCommunityContexts ( ) ) ) ;
}
@Override
@Cacheable ( " context-cache " )
public Map < String , Context > getContextMap ( final List < String > type ) throws IOException {
if ( Objects . isNull ( type ) | | type . isEmpty ( ) ) {
return _processContext ( _getQuery ( config . getFindContextProfiles ( ) ) ) ;
} else {
final String xqueryTemplate = _getQuery ( config . getFindContextProfilesByType ( ) ) ;
final String xquery = String . format ( xqueryTemplate , type . stream ( )
2022-02-07 09:03:36 +01:00
. map ( t - > String . format ( " ./RESOURCE_PROFILE/BODY/CONFIGURATION/context/@type = '%s' " , t ) )
. collect ( Collectors . joining ( " or " ) ) ) ;
2022-02-04 10:12:15 +01:00
return _processContext ( xquery ) ;
}
}
2022-09-27 16:06:31 +02:00
2022-02-04 10:12:15 +01:00
@Override
2022-02-07 09:03:36 +01:00
@CacheEvict ( value = {
" context-cache " , " context-cache-funder "
} , allEntries = true )
2022-09-27 16:06:31 +02:00
public void updateContextParam ( final String id , final String name , final String value , boolean toEscape ) {
if ( getSize ( id , name ) > 0 ) {
try {
_quickSeachProfile ( getXQuery ( id , name , value , toEscape ) ) ;
} catch ( final ISLookUpException e ) {
throw new DsmRuntimeException ( String . format ( " unable to update context param [id: %s, name: %s, value: %s] " , id , name , value ) , e ) ;
}
2022-02-04 10:12:15 +01:00
}
2022-09-27 16:06:31 +02:00
else {
try {
_quickSeachProfile ( getInsertXQuery ( id , name , value , toEscape ) ) ;
} catch ( final ISLookUpException e ) {
throw new DsmRuntimeException ( String . format ( " unable to insert context param [id: %s, name: %s, value: %s] " , id , name , value ) , e ) ;
}
}
2022-02-04 10:12:15 +01:00
}
2022-09-27 16:06:31 +02:00
private int getSize ( String id , String name ) {
int size = 0 ;
2022-09-26 19:06:51 +02:00
try {
2022-09-27 16:06:31 +02:00
size = _quickSeachProfile ( String . format ( " for $x in collection('/db/DRIVER/ContextDSResources/ContextDSResourceType') where $x//context[@id='%s']/param[@name='%s'] return $x " , id , name ) ) . size ( ) ;
} catch ( ISLookUpException e ) {
throw new DsmRuntimeException ( " unable to execute search query " , e ) ;
2022-09-26 19:06:51 +02:00
}
2022-09-27 16:06:31 +02:00
return size ;
2022-09-26 19:06:51 +02:00
}
2022-02-04 10:12:15 +01:00
@Override
2022-02-07 09:03:36 +01:00
@CacheEvict ( value = {
" context-cache " , " context-cache-funder "
} , allEntries = true )
2022-02-04 10:12:15 +01:00
public void updateContextAttribute ( final String id , final String name , final String value ) {
final Escaper esc = XmlEscapers . xmlAttributeEscaper ( ) ;
try {
2022-02-07 09:03:36 +01:00
_quickSeachProfile ( String . format ( " update value collection('/db/DRIVER/ContextDSResources/ContextDSResourceType') " +
" /RESOURCE_PROFILE/BODY/CONFIGURATION/context[./@id = '%s']/@%s with '%s' " , id , name , escape ( esc , value ) ) ) ;
} catch ( final ISLookUpException e ) {
2022-02-04 10:12:15 +01:00
throw new DsmRuntimeException ( String . format ( " unable update context attribute [id: %s, name: %s, data: %s] " , id , name , value ) , e ) ;
}
}
@Override
2022-02-07 09:03:36 +01:00
@CacheEvict ( value = {
" context-cache " , " context-cache-funder "
} , allEntries = true )
2022-02-04 10:12:15 +01:00
public void addConcept ( final String id , final String categoryId , final String data ) {
try {
2022-02-07 09:03:36 +01:00
_quickSeachProfile ( String . format ( " update insert %s into collection('/db/DRIVER/ContextDSResources/ContextDSResourceType') " +
" /RESOURCE_PROFILE/BODY/CONFIGURATION/context[./@id = '%s']/category[./@id = '%s'] " , data , id , categoryId ) ) ;
} catch ( final ISLookUpException e ) {
2022-02-04 10:12:15 +01:00
throw new DsmRuntimeException ( String . format ( " unable add concept [id: %s, categoryId: %s, data: %s] " , id , categoryId , data ) , e ) ;
}
}
@Override
2022-02-07 09:03:36 +01:00
@CacheEvict ( value = {
" context-cache " , " context-cache-funder "
} , allEntries = true )
2022-02-04 10:12:15 +01:00
public void removeConcept ( final String id , final String categoryId , final String conceptId ) {
try {
2022-02-07 09:03:36 +01:00
_quickSeachProfile ( String . format ( " for $concept in collection('/db/DRIVER/ContextDSResources/ContextDSResourceType') " +
" /RESOURCE_PROFILE/BODY/CONFIGURATION/context[./@id = '%s'] " +
" /category[./@id = '%s']/concept[./@id = '%s'] " +
" return update delete $concept " , id , categoryId , conceptId ) ) ;
} catch ( final ISLookUpException e ) {
2022-02-04 10:12:15 +01:00
throw new DsmRuntimeException ( String . format ( " unable remove concept [id: %s, categoryId: %s, conceptId: %s] " , id , categoryId , conceptId ) , e ) ;
}
}
@Override
2022-02-07 09:03:36 +01:00
@CacheEvict ( value = {
" context-cache " , " context-cache-community " , " context-cache-funder "
} , allEntries = true )
public void updateConceptAttribute ( final String id , final String name , final String value ) {
2022-02-04 10:12:15 +01:00
final Escaper esc = XmlEscapers . xmlAttributeEscaper ( ) ;
try {
2022-02-07 09:03:36 +01:00
_quickSeachProfile ( String . format ( " update value collection('/db/DRIVER/ContextDSResources/ContextDSResourceType') " +
" /RESOURCE_PROFILE/BODY/CONFIGURATION/context/category/concept[./@id = '%s']/@%s with '%s' " , id , name , escape ( esc , value ) ) ) ;
} catch ( final ISLookUpException e ) {
2022-02-04 10:12:15 +01:00
throw new DsmRuntimeException ( String . format ( " unable update concept attribute [id: %s, name: %s, value: %s] " , id , name , value ) , e ) ;
}
}
@Override
2022-02-07 09:03:36 +01:00
@CacheEvict ( value = {
" context-cache " , " context-cache-funder "
} , allEntries = true )
public void updateConceptParam ( final String id , final String name , final String value ) {
2022-02-04 10:12:15 +01:00
try {
_quickSeachProfile ( getConceptXQuery ( id , name , value ) ) ;
2022-02-07 09:03:36 +01:00
} catch ( final ISLookUpException e ) {
2022-02-04 10:12:15 +01:00
throw new DsmRuntimeException ( String . format ( " unable update concept param [id: %s, name: %s, value: %s] " , id , name , value ) , e ) ;
}
}
@Override
2022-02-07 09:03:36 +01:00
@CacheEvict ( value = {
" context-cache " , " context-cache-funder "
} , allEntries = true )
public void updateConceptParamNoEscape ( final String id , final String name , final String value ) {
2022-02-04 10:12:15 +01:00
try {
_quickSeachProfile ( getConceptXQueryNoEscape ( id , name , value ) ) ;
2022-02-07 09:03:36 +01:00
} catch ( final ISLookUpException e ) {
2022-02-04 10:12:15 +01:00
throw new DsmRuntimeException ( String . format ( " unable update concept param [id: %s, name: %s, value: %s] " , id , name , value ) , e ) ;
}
}
/// HELPERS
2022-09-27 16:06:31 +02:00
private String getInsertXQuery ( final String id , final String paramName , final String paramValue , final boolean toEscape ) {
String value ;
if ( toEscape )
value = escape ( XmlEscapers . xmlContentEscaper ( ) , paramValue ) ;
else
value = paramValue ;
2022-02-04 10:12:15 +01:00
if ( StringUtils . isNotBlank ( value ) ) {
2022-09-27 16:06:31 +02:00
return String . format ( " update insert <param name='%s'>%s</param> into collection('/db/DRIVER/ContextDSResources/ContextDSResourceType') " +
" /RESOURCE_PROFILE/BODY/CONFIGURATION/context[./@id = '%s'] " , paramName , value , id ) ;
2022-02-04 10:12:15 +01:00
} else {
2022-09-27 16:06:31 +02:00
return String . format ( " update insert <param name='%s'/> into collection('/db/DRIVER/ContextDSResources/ContextDSResourceType') " +
" /RESOURCE_PROFILE/BODY/CONFIGURATION/context[./@id = '%s'] " , paramName , id ) ;
2022-02-04 10:12:15 +01:00
}
}
2022-09-27 16:06:31 +02:00
private String getXQuery ( final String id , final String name , final String paramValue , final boolean toEscape ) {
String value = paramValue ;
if ( toEscape )
value = escape ( XmlEscapers . xmlContentEscaper ( ) , paramValue ) ;
2022-09-26 19:06:51 +02:00
if ( StringUtils . isNotBlank ( value ) ) {
return String . format ( " update replace collection('/db/DRIVER/ContextDSResources/ContextDSResourceType') " +
2022-09-27 16:06:31 +02:00
" /RESOURCE_PROFILE/BODY/CONFIGURATION/context[./@id = '%s']/param[./@name = '%s'] with <param name='%s'>%s</param> " , id , name , name , value ) ;
2022-09-26 19:06:51 +02:00
} else {
return String . format ( " update replace collection('/db/DRIVER/ContextDSResources/ContextDSResourceType') " +
2022-09-27 16:06:31 +02:00
" /RESOURCE_PROFILE/BODY/CONFIGURATION/context[./@id = '%s']/param[./@name = '%s'] with <param name='%s'/> " , id , name , name ) ;
2022-09-26 19:06:51 +02:00
}
}
2022-09-27 16:06:31 +02:00
// private String getXQueryNoEscape(final String id, final String name, final String value) {
// if (StringUtils.isNotBlank(value)) {
// return String.format("update replace collection('/db/DRIVER/ContextDSResources/ContextDSResourceType')" +
// "/RESOURCE_PROFILE/BODY/CONFIGURATION/context[./@id = '%s']/param[./@name = '%s'] with <param name='%s'>%s</param>", id, name, name, value);
// } else {
// return String.format("update replace collection('/db/DRIVER/ContextDSResources/ContextDSResourceType')" +
// "/RESOURCE_PROFILE/BODY/CONFIGURATION/context[./@id = '%s']/param[./@name = '%s'] with <param name='%s'/>", id, name, name);
// }
// }
2022-02-04 10:12:15 +01:00
private String getConceptXQuery ( final String id , final String name , final String value ) {
final Escaper esc = XmlEscapers . xmlContentEscaper ( ) ;
if ( StringUtils . isNotBlank ( value ) ) {
2022-02-07 09:03:36 +01:00
return String . format ( " update replace collection('/db/DRIVER/ContextDSResources/ContextDSResourceType')// " +
" concept[./@id = '%s']/param[./@name = '%s'] with <param name='%s'>%s</param> " , id , name , name , escape ( esc , value ) ) ;
2022-02-04 10:12:15 +01:00
} else {
2022-02-07 09:03:36 +01:00
return String
. format ( " update replace collection('/db/DRIVER/ContextDSResources/ContextDSResourceType')//concept[./@id = '%s']/param[./@name = '%s'] with <param name='%s'/> " , id , name , name ) ;
2022-02-04 10:12:15 +01:00
}
}
private String getConceptXQueryNoEscape ( final String id , final String name , final String value ) {
if ( StringUtils . isNotBlank ( value ) ) {
2022-02-07 09:03:36 +01:00
return String . format ( " update replace collection('/db/DRIVER/ContextDSResources/ContextDSResourceType')// " +
" concept[./@id = '%s']/param[./@name = '%s'] with <param name='%s'>%s</param> " , id , name , name , value ) ;
2022-02-04 10:12:15 +01:00
} else {
2022-02-07 09:03:36 +01:00
return String
. format ( " update replace collection('/db/DRIVER/ContextDSResources/ContextDSResourceType')//concept[./@id = '%s']/param[./@name = '%s'] with <param name='%s'/> " , id , name , name ) ;
2022-02-04 10:12:15 +01:00
}
}
private Map < String , Context > _processContext ( final String xquery ) throws IOException {
return _processContext ( new LinkedBlockingQueue < > ( ) , xquery ) ;
}
private Map < String , Context > _processContext ( final Queue < Throwable > errors , final String xquery ) throws IOException {
try {
return getContextProfiles ( errors , xquery ) . stream ( )
2022-02-07 09:03:36 +01:00
. filter ( StringUtils : : isNotBlank )
. map ( s - > ContextMappingUtils . parseContext ( s , errors ) )
. collect ( Collectors . toMap ( Context : : getId , Function . identity ( ) , ( c1 , c2 ) - > {
log . warn ( String . format ( " found duplicate context profile '%s' " , c1 . getId ( ) ) ) ;
return c1 ;
} ) ) ;
2022-02-04 10:12:15 +01:00
} finally {
if ( ! errors . isEmpty ( ) ) {
log . error ( errors ) ;
errors . forEach ( Throwable : : printStackTrace ) ;
}
}
}
private List < String > getContextProfiles ( final Queue < Throwable > errors , final String xquery ) throws IOException {
log . warn ( " getContextProfiles(): not using cache " ) ;
try {
return _quickSeachProfile ( xquery ) ;
2022-02-07 09:03:36 +01:00
} catch ( final ISLookUpException e ) {
2022-02-04 10:12:15 +01:00
throw new DsmRuntimeException ( " unable to get context profiles " , e ) ;
}
}
private String _getQuery ( final ClassPathResource resource ) throws IOException {
return IOUtils . toString ( resource . getInputStream ( ) , Charset . defaultCharset ( ) ) ;
}
private String _isLookUp ( final String xquery ) throws ISLookUpException {
log . debug ( String . format ( " running xquery: \ n%s " , xquery ) ) ;
2022-02-07 09:03:36 +01:00
// log.debug(String.format("query result: %s", res));
2022-02-04 10:12:15 +01:00
return isLookUpService . getResourceProfileByQuery ( xquery ) ;
}
private List < String > _quickSeachProfile ( final String xquery ) throws ISLookUpException {
final List < String > res = Lists . newArrayList ( ) ;
2022-02-07 09:03:36 +01:00
log . debug ( String . format ( " running xquery: \ n%s " , xquery ) ) ;
final List < String > list = isLookUpService . quickSearchProfile ( xquery ) ;
if ( list ! = null ) {
res . addAll ( list ) ;
}
log . debug ( String . format ( " query result size: %s " , res . size ( ) ) ) ;
return res ;
2022-02-04 10:12:15 +01:00
}
2022-02-07 09:03:36 +01:00
@Override
@CacheEvict ( cacheNames = {
" context-cache " , " indexdsinfo-cache " , " objectstoreid-cache "
} , allEntries = true )
2022-02-04 10:12:15 +01:00
@Scheduled ( fixedDelayString = " ${openaire.exporter.cache.ttl} " )
public void dropCache ( ) {
log . debug ( " dropped dsManager IS cache " ) ;
}
}