package eu.dnetlib.data.datasource; import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.common.collect.Sets; import eu.dnetlib.enabling.datasources.common.*; import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpException; import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpService; import eu.dnetlib.enabling.is.registry.rmi.ISRegistryException; import eu.dnetlib.enabling.is.registry.rmi.ISRegistryService; import eu.dnetlib.enabling.locators.UniqueServiceLocator; import org.antlr.stringtemplate.StringTemplate; import org.apache.commons.io.IOUtils; import org.apache.commons.lang3.StringEscapeUtils; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.math.NumberUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.http.HttpStatus; import org.dom4j.Document; import org.dom4j.Node; import org.dom4j.io.SAXReader; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Required; import org.springframework.core.io.ClassPathResource; import java.io.StringReader; import java.util.*; import java.util.stream.Collectors; public class LocalSimpleDatasourceManagerImpl implements LocalSimpleDatasourceManager { private static final Log log = LogFactory.getLog(LocalSimpleDatasourceManagerImpl.class); private static final String REPOSITORY_RESOURCE_TYPE = "RepositoryServiceResourceType"; private ClassPathResource findReposQueryTmpl = new ClassPathResource("/eu/dnetlib/enabling/datasources/templates/findRepos.xquery.st"); private ClassPathResource browseRepoApisQueryTmpl = new ClassPathResource("/eu/dnetlib/enabling/datasources/templates/browseRepoApis.xquery.st"); private ClassPathResource findRepoApisQueryTmpl = new ClassPathResource("/eu/dnetlib/enabling/datasources/templates/findRepoApis.xquery.st"); private ClassPathResource findReposMapQuery = new ClassPathResource("/eu/dnetlib/enabling/datasources/templates/findReposMap.xquery"); private ClassPathResource simpleFindReposQueryTmpl = new ClassPathResource("/eu/dnetlib/enabling/datasources/templates/simpleFindRepos.xquery.st"); private List browsableFields; @Autowired private UniqueServiceLocator serviceLocator; @Override public Set listManagedDatasourceIds() throws DsmRuntimeException { throw new DsmRuntimeException("method 'listManagedDatasourceIds' not implemented"); } @Override public List searchDatasourcesByType(String type) throws DsmException { try { final List list = new ArrayList<>(); final StringTemplate st = new StringTemplate(IOUtils.toString(simpleFindReposQueryTmpl.getInputStream())); st.setAttribute("type", type); for (String s : serviceLocator.getService(ISLookUpService.class).quickSearchProfile(st.toString())) { final SimpleDatasource r = new SimpleDatasource(); final String[] arr = s.split("@=@"); r.setId(arr[0].trim()); r.setName(arr[1].trim()); r.setOrigId(arr[2].trim()); r.setApis(Sets.newHashSet(arr[3].replaceAll("\\s", "").split(","))); r.setValid("true".equals(arr[4].trim())); r.setTypology(type); list.add(r); } Collections.sort(list); return list; } catch (Exception e) { log.error("Error listing repos", e); } return Lists.newArrayList(); } @Override public List searchApis(String field, Object value) throws DsmException { try { final StringTemplate st = new StringTemplate(); st.setTemplate(IOUtils.toString(findRepoApisQueryTmpl.getInputStream())); final String val = value.toString(); if (field.equalsIgnoreCase("__search__")) { st.setAttribute("cond", "contains(../..//(*|@*)/lower-case(.), '" + StringEscapeUtils.escapeXml(val.toLowerCase()) + "')"); } else { final XmlBrowsableField f = findBrowseField(field); if (f != null) { st.setAttribute("cond", f.getXpath() + "='" + StringEscapeUtils.escapeXml(val) + "'"); } else { return Lists.newArrayList(); } } final String query = st.toString(); final SAXReader reader = new SAXReader(); final List list = serviceLocator.getService(ISLookUpService.class).quickSearchProfile(query); return list.stream() .map(s -> { final SearchApisEntry iface = new SearchApisEntry(); try { final Document doc = reader.read(new StringReader(s)); final String country = doc.valueOf("//REPO/@country"); iface.setRepoId(doc.valueOf("//REPO/@id")); iface.setRepoCountry(StringUtils.isEmpty(country) ? "-" : country.toUpperCase()); iface.setRepoName(doc.valueOf("//REPO/@name")); iface.setRepoPrefix(doc.valueOf("//REPO/@prefix")); final Node ifcNode = doc.selectSingleNode("//INTERFACE"); iface.setId(ifcNode.valueOf("./@id")); iface.setActive(Boolean.valueOf(ifcNode.valueOf("./@active"))); iface.setProtocol(ifcNode.valueOf("./ACCESS_PROTOCOL/text()")); final String overCompliance = ifcNode.valueOf("./INTERFACE_EXTRA_FIELD[@name='overriding_compliance']"); if (StringUtils.isEmpty(overCompliance)) { iface.setCompliance(ifcNode.valueOf("@compliance")); } else { iface.setCompliance(overCompliance); } final String lastAggregationDate = ifcNode.valueOf("./INTERFACE_EXTRA_FIELD[@name='last_aggregation_date']"); if (!StringUtils.isEmpty(lastAggregationDate)) { iface.setAggrDate(lastAggregationDate); } else { final String lastDownloadDate = ifcNode.valueOf("./INTERFACE_EXTRA_FIELD[@name='last_download_date']"); if (!StringUtils.isEmpty(lastDownloadDate)) { iface.setAggrDate(lastDownloadDate); } } final String lastAggregationTotal = ifcNode.valueOf("./INTERFACE_EXTRA_FIELD[@name='last_aggregation_total']"); if (StringUtils.isEmpty(lastAggregationTotal)) { final String lastDownloadTotal = ifcNode.valueOf("./INTERFACE_EXTRA_FIELD[@name='last_download_total']"); if (StringUtils.isEmpty(lastDownloadTotal)) { iface.setAggrTotal(0); } else { iface.setAggrTotal(NumberUtils.toInt(lastDownloadTotal, 0)); } } else { iface.setAggrTotal(NumberUtils.toInt(lastAggregationTotal, 0)); } } catch (Exception e) { log.error(e); } return iface; }).collect(Collectors.toList()); } catch (Exception e) { log.error("Error searching field " + field + " - value: " + value, e); } return Lists.newArrayList(); } @Override public List listBrowsableFields() throws DsmException { return getBrowsableFields(); } public List getBrowsableFields() { return browsableFields; } @Required public void setBrowsableFields(List browsableFields) { this.browsableFields = browsableFields; } @Override public List browseField(final String f) throws DsmException { final List list = Lists.newArrayList(); try { final XmlBrowsableField field = findBrowseField(f); if (field != null) { final StringTemplate st = new StringTemplate(IOUtils.toString(browseRepoApisQueryTmpl.getInputStream())); st.setAttribute("xpath", field.getXpath()); final Map terms = fetchVocabularyTerms(field.getVocabulary()); for (String s : serviceLocator.getService(ISLookUpService.class).quickSearchProfile(st.toString())) { final String[] arr = s.split("@-@-@"); final String id = arr[0].trim(); final int count = NumberUtils.toInt(arr[1].trim(), 0); list.add(new BrowseTermImpl(id, findLabel(id, terms), count)); } } } catch (Exception e) { log.error("Error browsing field " + f, e); } return list; } @Override public void setActive(String dsId, String apiId, boolean active) throws DsmException { throw new DsmRuntimeException("method 'setActive' not implemented"); } @Override public boolean isActive(String dsId, String apiId) throws DsmException { throw new DsmRuntimeException("method 'isActive' not implemented"); } @Override public void setLastCollectionInfo(String dsId, String apiId, String mdId, Integer size, Date date) throws DsmException { Map updates = Maps.newHashMap(); updates.put("last_collection_date", date.toString()); updates.put("last_collection_total", size.toString()); updateInterfaceExtraInfo(dsId, apiId, updates); } @Override public void setLastAggregationInfo(String dsId, String apiId, String mdId, Integer size, Date date) throws DsmException { Map updates = Maps.newHashMap(); updates.put("last_aggregation_date", date.toString()); updates.put("last_aggregation_total", size.toString()); updateInterfaceExtraInfo(dsId, apiId, updates); } @Override public void setLastDownloadInfo(String dsId, String apiId, String objId, Integer size, Date date) throws DsmException { Map updates = Maps.newHashMap(); updates.put("last_download_date", date.toString()); updates.put("last_download_total", size.toString()); updateInterfaceExtraInfo(dsId, apiId, updates); } @Override public void setLastValidationJob(String dsId, String apiId, String jobId) throws DsmException { Map updates = Maps.newHashMap(); updates.put("last_validation_job", jobId); updateInterfaceExtraInfo(dsId, apiId, updates); } @Override public void updateApiDetails(String dsId, String apiId, String metadataIdentifierPath, String baseUrl, Map params) throws DsmException { // oaiset, baseurl, compatibility /* oai http://pub.uni-bielefeld.de/oai //*[local-name()='header']/*[local-name()='identifier'] */ } @Override public boolean isRemovable(String dsId, String apiId) throws DsmException { return false; } @Override public void regenerateProfiles() throws DsmException { } @Override public Datasource, Identity> getDs(String id) throws DsmException { return null; } @Override public List> getApis(String dsId) throws DsmException { return null; } @Override public void deleteDs(String dsId) throws DsmException { } @Override public void deleteApi(String dsId, String apiId) throws DsmException { } @Override public void addApi(Api api) throws DsmException { } @Override public void setManaged(String id, boolean managed) throws DsmException { } @Override public boolean isManaged(String id) throws DsmException { return false; } @Override public void saveDs(Datasource, Identity> datasource) throws DsmException { } @Override public void updateCompliance(String dsId, String apiId, String compliance, boolean override) throws DsmException { } // HELPERS private void updateInterfaceExtraInfo(String dsId, String apiId, Map updates) throws DsmException { final String xpath = "//INTERFACE[@id = '%s']/INTERFACE_EXTRA_FIELD[@name='%s']"; final ISRegistryService isRegistry = serviceLocator.getService(ISRegistryService.class); updates.forEach((k, v) -> { try { isRegistry.updateProfileNode(dsId, String.format(xpath, apiId, k), v); } catch (ISRegistryException e) { log.error("Error updating profile: " + dsId, e); throw new DsmRuntimeException("Error updating profile: " + dsId, e); } }); } private void updateInterfaceExtraInfo(String dsId, String apiId, Integer size, Date date, String last_date, String last_total) throws DsmException { final String xpath = "//INTERFACE[@id = '%s']/INTERFACE_EXTRA_FIELD[@name='%s']"; final ISRegistryService isRegistry = serviceLocator.getService(ISRegistryService.class); try { isRegistry.updateProfileNode(dsId, String.format(xpath, apiId, last_date), date.toString()); isRegistry.updateProfileNode(dsId, String.format(xpath, apiId, last_total), size.toString()); } catch (ISRegistryException e) { log.error("Error updating profile: " + dsId, e); throw new DsmException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Error updating profile: " + dsId, e); } } private XmlBrowsableField findBrowseField(final String id) throws DsmException { for (XmlBrowsableField f : listBrowsableFields()) { if (f.getId().equals(id)) { return f; } } return null; } public Map fetchVocabularyTerms(final String voc) throws ISLookUpException { final String xquery = "for $x in collection('/db/DRIVER/VocabularyDSResources/VocabularyDSResourceType')[.//VOCABULARY_NAME/@code = '" + voc.trim() + "']//TERM return concat($x/@code, ' @@@ ', $x/@english_name)"; final Map map = Maps.newHashMap(); for (String s : serviceLocator.getService(ISLookUpService.class).quickSearchProfile(xquery)) { final String[] arr = s.split("@@@"); map.put(arr[0].trim(), arr[1].trim()); } return map; } private String findLabel(final String code, final Map terms) { return terms.containsKey(code) ? terms.get(code) : code; } }