package eu.dnetlib.openaire.common; import static eu.dnetlib.openaire.common.Utils.escape; import java.io.IOException; import java.nio.charset.Charset; import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Queue; import java.util.concurrent.LinkedBlockingQueue; import java.util.function.Function; import java.util.stream.Collectors; import org.apache.commons.io.IOUtils; import org.apache.commons.lang3.StringUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.cache.annotation.CacheEvict; import org.springframework.cache.annotation.Cacheable; import org.springframework.core.io.ClassPathResource; import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Component; import com.google.common.collect.Lists; import com.google.common.escape.Escaper; import com.google.common.xml.XmlEscapers; import eu.dnetlib.DnetOpenaireExporterProperties; import eu.dnetlib.enabling.datasources.common.DsmException; import eu.dnetlib.enabling.datasources.common.DsmRuntimeException; import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpException; import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpService; import eu.dnetlib.openaire.context.Context; import eu.dnetlib.openaire.context.ContextMappingUtils; import eu.dnetlib.openaire.dsm.dao.utils.IndexDsInfo; /** * Created by claudio on 20/10/2016. */ @Component public class ISClientImpl implements ISClient { private static final Log log = LogFactory.getLog(ISClientImpl.class); @Autowired private DnetOpenaireExporterProperties config; @Autowired private ISLookUpService isLookUpService; @Override @Cacheable("indexdsinfo-cache") public IndexDsInfo calculateCurrentIndexDsInfo() throws DsmException { log.warn("calculateCurrentIndexDsInfo(): not using cache"); final String[] arr; try { arr = _isLookUp(_getQuery(config.getFindIndexDsInfo())).split("@@@"); return new IndexDsInfo( _isLookUp(_getQuery(config.getFindSolrIndexUrl())), arr[0].trim(), arr[1].trim(), arr[2].trim()); } catch (IOException | ISLookUpException e) { throw new DsmException("unable fetch index DS information from IS"); } } @Override @Cacheable("objectstoreid-cache") public String getObjectStoreId(final String dsId) throws DsmException { log.warn(String.format("getObjectStoreId(%s): not using cache", dsId)); try { final String xqueryTemplate = _getQuery(config.getFindObjectStore()); return _isLookUp(String.format(xqueryTemplate, dsId)); } catch (IOException | ISLookUpException e) { throw new DsmException("unble to find objectstore for ds " + dsId); } } @Override @Cacheable("context-cache-funder") public Map getFunderContextMap() throws IOException { return _processContext(_getQuery(config.getFindFunderContexts())); } @Override @Cacheable("context-cache-community") public Map getCommunityContextMap() throws IOException { return _processContext(_getQuery(config.getFindCommunityContexts())); } @Override @Cacheable("context-cache") public Map getContextMap(final List type) throws IOException { if (Objects.isNull(type) || type.isEmpty()) { return _processContext(_getQuery(config.getFindContextProfiles())); } else { final String xqueryTemplate = _getQuery(config.getFindContextProfilesByType()); final String xquery = String.format(xqueryTemplate, type.stream() .map(t -> String.format("./RESOURCE_PROFILE/BODY/CONFIGURATION/context/@type = '%s'", t)) .collect(Collectors.joining(" or "))); return _processContext(xquery); } } @Override @CacheEvict(value = { "context-cache", "context-cache-funder" }, allEntries = true) public void updateContextParam(final String id, final String name, final String value) { try { _quickSeachProfile(getXQuery(id, name, value)); } catch (final ISLookUpException e) { throw new DsmRuntimeException(String.format("unable update context param [id: %s, name: %s, value: %s]", id, name, value), e); } } @Override @CacheEvict(value = { "context-cache", "context-cache-funder" }, allEntries = true) public void updateContextAttribute(final String id, final String name, final String value) { final Escaper esc = XmlEscapers.xmlAttributeEscaper(); try { _quickSeachProfile(String.format("update value collection('/db/DRIVER/ContextDSResources/ContextDSResourceType')" + "/RESOURCE_PROFILE/BODY/CONFIGURATION/context[./@id = '%s']/@%s with '%s'", id, name, escape(esc, value))); } catch (final ISLookUpException e) { throw new DsmRuntimeException(String.format("unable update context attribute [id: %s, name: %s, data: %s]", id, name, value), e); } } @Override @CacheEvict(value = { "context-cache", "context-cache-funder" }, allEntries = true) public void addConcept(final String id, final String categoryId, final String data) { try { _quickSeachProfile(String.format("update insert %s into collection('/db/DRIVER/ContextDSResources/ContextDSResourceType')" + "/RESOURCE_PROFILE/BODY/CONFIGURATION/context[./@id = '%s']/category[./@id = '%s']", data, id, categoryId)); } catch (final ISLookUpException e) { throw new DsmRuntimeException(String.format("unable add concept [id: %s, categoryId: %s, data: %s]", id, categoryId, data), e); } } @Override @CacheEvict(value = { "context-cache", "context-cache-funder" }, allEntries = true) public void removeConcept(final String id, final String categoryId, final String conceptId) { try { _quickSeachProfile(String.format("for $concept in collection('/db/DRIVER/ContextDSResources/ContextDSResourceType')" + "/RESOURCE_PROFILE/BODY/CONFIGURATION/context[./@id = '%s']" + "/category[./@id = '%s']/concept[./@id = '%s'] " + "return update delete $concept", id, categoryId, conceptId)); } catch (final ISLookUpException e) { throw new DsmRuntimeException(String.format("unable remove concept [id: %s, categoryId: %s, conceptId: %s]", id, categoryId, conceptId), e); } } @Override @CacheEvict(value = { "context-cache", "context-cache-community", "context-cache-funder" }, allEntries = true) public void updateConceptAttribute(final String id, final String name, final String value) { final Escaper esc = XmlEscapers.xmlAttributeEscaper(); try { _quickSeachProfile(String.format("update value collection('/db/DRIVER/ContextDSResources/ContextDSResourceType')" + "/RESOURCE_PROFILE/BODY/CONFIGURATION/context/category/concept[./@id = '%s']/@%s with '%s'", id, name, escape(esc, value))); } catch (final ISLookUpException e) { throw new DsmRuntimeException(String.format("unable update concept attribute [id: %s, name: %s, value: %s]", id, name, value), e); } } @Override @CacheEvict(value = { "context-cache", "context-cache-funder" }, allEntries = true) public void updateConceptParam(final String id, final String name, final String value) { try { _quickSeachProfile(getConceptXQuery(id, name, value)); } catch (final ISLookUpException e) { throw new DsmRuntimeException(String.format("unable update concept param [id: %s, name: %s, value: %s]", id, name, value), e); } } @Override @CacheEvict(value = { "context-cache", "context-cache-funder" }, allEntries = true) public void updateConceptParamNoEscape(final String id, final String name, final String value) { try { _quickSeachProfile(getConceptXQueryNoEscape(id, name, value)); } catch (final ISLookUpException e) { throw new DsmRuntimeException(String.format("unable update concept param [id: %s, name: %s, value: %s]", id, name, value), e); } } /// HELPERS private String getXQuery(final String id, final String name, final String value) { final Escaper esc = XmlEscapers.xmlContentEscaper(); if (StringUtils.isNotBlank(value)) { return String.format("update replace collection('/db/DRIVER/ContextDSResources/ContextDSResourceType')" + "/RESOURCE_PROFILE/BODY/CONFIGURATION/context[./@id = '%s']/param[./@name = '%s'] with %s", id, name, name, escape(esc, value)); } else { return String.format("update replace collection('/db/DRIVER/ContextDSResources/ContextDSResourceType')" + "/RESOURCE_PROFILE/BODY/CONFIGURATION/context[./@id = '%s']/param[./@name = '%s'] with ", id, name, name); } } private String getConceptXQuery(final String id, final String name, final String value) { final Escaper esc = XmlEscapers.xmlContentEscaper(); if (StringUtils.isNotBlank(value)) { return String.format("update replace collection('/db/DRIVER/ContextDSResources/ContextDSResourceType')//" + "concept[./@id = '%s']/param[./@name = '%s'] with %s", id, name, name, escape(esc, value)); } else { return String .format("update replace collection('/db/DRIVER/ContextDSResources/ContextDSResourceType')//concept[./@id = '%s']/param[./@name = '%s'] with ", id, name, name); } } private String getConceptXQueryNoEscape(final String id, final String name, final String value) { if (StringUtils.isNotBlank(value)) { return String.format("update replace collection('/db/DRIVER/ContextDSResources/ContextDSResourceType')//" + "concept[./@id = '%s']/param[./@name = '%s'] with %s", id, name, name, value); } else { return String .format("update replace collection('/db/DRIVER/ContextDSResources/ContextDSResourceType')//concept[./@id = '%s']/param[./@name = '%s'] with ", id, name, name); } } private Map _processContext(final String xquery) throws IOException { return _processContext(new LinkedBlockingQueue<>(), xquery); } private Map _processContext(final Queue errors, final String xquery) throws IOException { try { return getContextProfiles(errors, xquery).stream() .filter(StringUtils::isNotBlank) .map(s -> ContextMappingUtils.parseContext(s, errors)) .collect(Collectors.toMap(Context::getId, Function.identity(), (c1, c2) -> { log.warn(String.format("found duplicate context profile '%s'", c1.getId())); return c1; })); } finally { if (!errors.isEmpty()) { log.error(errors); errors.forEach(Throwable::printStackTrace); } } } private List getContextProfiles(final Queue errors, final String xquery) throws IOException { log.warn("getContextProfiles(): not using cache"); try { return _quickSeachProfile(xquery); } catch (final ISLookUpException e) { throw new DsmRuntimeException("unable to get context profiles", e); } } private String _getQuery(final ClassPathResource resource) throws IOException { return IOUtils.toString(resource.getInputStream(), Charset.defaultCharset()); } private String _isLookUp(final String xquery) throws ISLookUpException { log.debug(String.format("running xquery:\n%s", xquery)); // log.debug(String.format("query result: %s", res)); return isLookUpService.getResourceProfileByQuery(xquery); } private List _quickSeachProfile(final String xquery) throws ISLookUpException { final List res = Lists.newArrayList(); log.debug(String.format("running xquery:\n%s", xquery)); final List list = isLookUpService.quickSearchProfile(xquery); if (list != null) { res.addAll(list); } log.debug(String.format("query result size: %s", res.size())); return res; } @Override @CacheEvict(cacheNames = { "context-cache", "indexdsinfo-cache", "objectstoreid-cache" }, allEntries = true) @Scheduled(fixedDelayString = "${openaire.exporter.cache.ttl}") public void dropCache() { log.debug("dropped dsManager IS cache"); } }