You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
365 lines
15 KiB
Java
365 lines
15 KiB
Java
package eu.dnetlib.data.datasource;
|
|
|
|
import com.google.common.collect.Lists;
|
|
import com.google.common.collect.Maps;
|
|
import com.google.common.collect.Sets;
|
|
import eu.dnetlib.enabling.datasources.common.*;
|
|
import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpException;
|
|
import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpService;
|
|
import eu.dnetlib.enabling.is.registry.rmi.ISRegistryException;
|
|
import eu.dnetlib.enabling.is.registry.rmi.ISRegistryService;
|
|
import eu.dnetlib.enabling.locators.UniqueServiceLocator;
|
|
import org.antlr.stringtemplate.StringTemplate;
|
|
import org.apache.commons.io.IOUtils;
|
|
import org.apache.commons.lang3.StringEscapeUtils;
|
|
import org.apache.commons.lang3.StringUtils;
|
|
import org.apache.commons.lang3.math.NumberUtils;
|
|
import org.apache.commons.logging.Log;
|
|
import org.apache.commons.logging.LogFactory;
|
|
import org.apache.http.HttpStatus;
|
|
import org.dom4j.Document;
|
|
import org.dom4j.Node;
|
|
import org.dom4j.io.SAXReader;
|
|
import org.springframework.beans.factory.annotation.Autowired;
|
|
import org.springframework.beans.factory.annotation.Required;
|
|
import org.springframework.core.io.ClassPathResource;
|
|
|
|
import java.io.StringReader;
|
|
import java.util.*;
|
|
import java.util.stream.Collectors;
|
|
|
|
public class LocalSimpleDatasourceManagerImpl implements LocalSimpleDatasourceManager {
|
|
|
|
private static final Log log = LogFactory.getLog(LocalSimpleDatasourceManagerImpl.class);
|
|
|
|
private static final String REPOSITORY_RESOURCE_TYPE = "RepositoryServiceResourceType";
|
|
|
|
private ClassPathResource findReposQueryTmpl = new ClassPathResource("/eu/dnetlib/enabling/datasources/templates/findRepos.xquery.st");
|
|
|
|
private ClassPathResource browseRepoApisQueryTmpl = new ClassPathResource("/eu/dnetlib/enabling/datasources/templates/browseRepoApis.xquery.st");
|
|
private ClassPathResource findRepoApisQueryTmpl = new ClassPathResource("/eu/dnetlib/enabling/datasources/templates/findRepoApis.xquery.st");
|
|
private ClassPathResource findReposMapQuery = new ClassPathResource("/eu/dnetlib/enabling/datasources/templates/findReposMap.xquery");
|
|
private ClassPathResource simpleFindReposQueryTmpl = new ClassPathResource("/eu/dnetlib/enabling/datasources/templates/simpleFindRepos.xquery.st");
|
|
|
|
private List<XmlBrowsableField> browsableFields;
|
|
|
|
@Autowired
|
|
private UniqueServiceLocator serviceLocator;
|
|
|
|
@Override
|
|
public Set<String> listManagedDatasourceIds() throws DsmRuntimeException {
|
|
throw new DsmRuntimeException("method 'listManagedDatasourceIds' not implemented");
|
|
}
|
|
|
|
@Override
|
|
public List<SimpleDatasource> searchDatasourcesByType(String type) throws DsmException {
|
|
try {
|
|
final List<SimpleDatasource> list = new ArrayList<>();
|
|
|
|
final StringTemplate st = new StringTemplate(IOUtils.toString(simpleFindReposQueryTmpl.getInputStream()));
|
|
st.setAttribute("type", type);
|
|
|
|
for (String s : serviceLocator.getService(ISLookUpService.class).quickSearchProfile(st.toString())) {
|
|
final SimpleDatasource r = new SimpleDatasource();
|
|
final String[] arr = s.split("@=@");
|
|
r.setId(arr[0].trim());
|
|
r.setName(arr[1].trim());
|
|
r.setOrigId(arr[2].trim());
|
|
r.setApis(Sets.newHashSet(arr[3].replaceAll("\\s", "").split(",")));
|
|
r.setValid("true".equals(arr[4].trim()));
|
|
r.setTypology(type);
|
|
list.add(r);
|
|
}
|
|
|
|
Collections.sort(list);
|
|
|
|
return list;
|
|
} catch (Exception e) {
|
|
log.error("Error listing repos", e);
|
|
}
|
|
return Lists.newArrayList();
|
|
}
|
|
|
|
@Override
|
|
public List<? extends SearchApisEntry> searchApis(String field, Object value) throws DsmException {
|
|
try {
|
|
final StringTemplate st = new StringTemplate();
|
|
st.setTemplate(IOUtils.toString(findRepoApisQueryTmpl.getInputStream()));
|
|
|
|
final String val = value.toString();
|
|
if (field.equalsIgnoreCase("__search__")) {
|
|
st.setAttribute("cond", "contains(../..//(*|@*)/lower-case(.), '" + StringEscapeUtils.escapeXml(val.toLowerCase()) + "')");
|
|
} else {
|
|
final XmlBrowsableField f = findBrowseField(field);
|
|
if (f != null) {
|
|
st.setAttribute("cond", f.getXpath() + "='" + StringEscapeUtils.escapeXml(val) + "'");
|
|
} else {
|
|
return Lists.newArrayList();
|
|
}
|
|
}
|
|
|
|
final String query = st.toString();
|
|
final SAXReader reader = new SAXReader();
|
|
final List<String> list = serviceLocator.getService(ISLookUpService.class).quickSearchProfile(query);
|
|
|
|
return list.stream()
|
|
.map(s -> {
|
|
final SearchApisEntry iface = new SearchApisEntry();
|
|
try {
|
|
final Document doc = reader.read(new StringReader(s));
|
|
final String country = doc.valueOf("//REPO/@country");
|
|
|
|
iface.setRepoId(doc.valueOf("//REPO/@id"));
|
|
iface.setRepoCountry(StringUtils.isEmpty(country) ? "-" : country.toUpperCase());
|
|
iface.setRepoName(doc.valueOf("//REPO/@name"));
|
|
iface.setRepoPrefix(doc.valueOf("//REPO/@prefix"));
|
|
|
|
final Node ifcNode = doc.selectSingleNode("//INTERFACE");
|
|
|
|
iface.setId(ifcNode.valueOf("./@id"));
|
|
iface.setActive(Boolean.valueOf(ifcNode.valueOf("./@active")));
|
|
iface.setProtocol(ifcNode.valueOf("./ACCESS_PROTOCOL/text()"));
|
|
|
|
final String overCompliance = ifcNode.valueOf("./INTERFACE_EXTRA_FIELD[@name='overriding_compliance']");
|
|
if (StringUtils.isEmpty(overCompliance)) {
|
|
iface.setCompliance(ifcNode.valueOf("@compliance"));
|
|
} else {
|
|
iface.setCompliance(overCompliance);
|
|
}
|
|
|
|
final String lastAggregationDate = ifcNode.valueOf("./INTERFACE_EXTRA_FIELD[@name='last_aggregation_date']");
|
|
if (!StringUtils.isEmpty(lastAggregationDate)) {
|
|
iface.setAggrDate(lastAggregationDate);
|
|
} else {
|
|
final String lastDownloadDate = ifcNode.valueOf("./INTERFACE_EXTRA_FIELD[@name='last_download_date']");
|
|
if (!StringUtils.isEmpty(lastDownloadDate)) {
|
|
iface.setAggrDate(lastDownloadDate);
|
|
}
|
|
}
|
|
final String lastAggregationTotal = ifcNode.valueOf("./INTERFACE_EXTRA_FIELD[@name='last_aggregation_total']");
|
|
if (StringUtils.isEmpty(lastAggregationTotal)) {
|
|
final String lastDownloadTotal = ifcNode.valueOf("./INTERFACE_EXTRA_FIELD[@name='last_download_total']");
|
|
if (StringUtils.isEmpty(lastDownloadTotal)) {
|
|
iface.setAggrTotal(0);
|
|
} else {
|
|
iface.setAggrTotal(NumberUtils.toInt(lastDownloadTotal, 0));
|
|
}
|
|
} else {
|
|
iface.setAggrTotal(NumberUtils.toInt(lastAggregationTotal, 0));
|
|
}
|
|
} catch (Exception e) {
|
|
log.error(e);
|
|
}
|
|
return iface;
|
|
}).collect(Collectors.toList());
|
|
} catch (Exception e) {
|
|
log.error("Error searching field " + field + " - value: " + value, e);
|
|
}
|
|
return Lists.newArrayList();
|
|
}
|
|
|
|
@Override
|
|
public List<XmlBrowsableField> listBrowsableFields() throws DsmException {
|
|
return getBrowsableFields();
|
|
}
|
|
|
|
public List<XmlBrowsableField> getBrowsableFields() {
|
|
return browsableFields;
|
|
}
|
|
|
|
@Required
|
|
public void setBrowsableFields(List<XmlBrowsableField> browsableFields) {
|
|
this.browsableFields = browsableFields;
|
|
}
|
|
|
|
@Override
|
|
public List<BrowseTerm> browseField(final String f) throws DsmException {
|
|
final List<BrowseTerm> list = Lists.newArrayList();
|
|
try {
|
|
final XmlBrowsableField field = findBrowseField(f);
|
|
if (field != null) {
|
|
final StringTemplate st = new StringTemplate(IOUtils.toString(browseRepoApisQueryTmpl.getInputStream()));
|
|
st.setAttribute("xpath", field.getXpath());
|
|
|
|
final Map<String, String> terms = fetchVocabularyTerms(field.getVocabulary());
|
|
|
|
for (String s : serviceLocator.getService(ISLookUpService.class).quickSearchProfile(st.toString())) {
|
|
final String[] arr = s.split("@-@-@");
|
|
final String id = arr[0].trim();
|
|
final int count = NumberUtils.toInt(arr[1].trim(), 0);
|
|
list.add(new BrowseTermImpl(id, findLabel(id, terms), count));
|
|
}
|
|
}
|
|
} catch (Exception e) {
|
|
log.error("Error browsing field " + f, e);
|
|
}
|
|
|
|
return list;
|
|
}
|
|
|
|
@Override
|
|
public void setActive(String dsId, String apiId, boolean active) throws DsmException {
|
|
throw new DsmRuntimeException("method 'setActive' not implemented");
|
|
}
|
|
|
|
@Override
|
|
public boolean isActive(String dsId, String apiId) throws DsmException {
|
|
throw new DsmRuntimeException("method 'isActive' not implemented");
|
|
}
|
|
|
|
@Override
|
|
public void setLastCollectionInfo(String dsId, String apiId, String mdId, Integer size, Date date) throws DsmException {
|
|
Map<String, String> updates = Maps.newHashMap();
|
|
updates.put("last_collection_date", date.toString());
|
|
updates.put("last_collection_total", size.toString());
|
|
updateInterfaceExtraInfo(dsId, apiId, updates);
|
|
}
|
|
|
|
@Override
|
|
public void setLastAggregationInfo(String dsId, String apiId, String mdId, Integer size, Date date) throws DsmException {
|
|
Map<String, String> updates = Maps.newHashMap();
|
|
updates.put("last_aggregation_date", date.toString());
|
|
updates.put("last_aggregation_total", size.toString());
|
|
updateInterfaceExtraInfo(dsId, apiId, updates);
|
|
}
|
|
|
|
@Override
|
|
public void setLastDownloadInfo(String dsId, String apiId, String objId, Integer size, Date date) throws DsmException {
|
|
Map<String, String> updates = Maps.newHashMap();
|
|
updates.put("last_download_date", date.toString());
|
|
updates.put("last_download_total", size.toString());
|
|
updateInterfaceExtraInfo(dsId, apiId, updates);
|
|
}
|
|
|
|
@Override
|
|
public void setLastValidationJob(String dsId, String apiId, String jobId) throws DsmException {
|
|
Map<String, String> updates = Maps.newHashMap();
|
|
updates.put("last_validation_job", jobId);
|
|
updateInterfaceExtraInfo(dsId, apiId, updates);
|
|
}
|
|
|
|
@Override
|
|
public void updateApiDetails(String dsId, String apiId, String metadataIdentifierPath, String baseUrl, Map<String, String> params) throws DsmException {
|
|
|
|
// oaiset, baseurl, compatibility
|
|
/*
|
|
<INTERFACE active="true" compliance="openaire3.0" contentDescription="metadata" id="api_________::opendoar____::2294::0" label="pubsrepository::institutional (openaire3.0)" removable="true" typology="pubsrepository::institutional">
|
|
<ACCESS_PROTOCOL format="oai_dc" set="openaire">oai</ACCESS_PROTOCOL>
|
|
<BASE_URL>http://pub.uni-bielefeld.de/oai</BASE_URL>
|
|
<INTERFACE_EXTRA_FIELD name="metadata_identifier_path">//*[local-name()='header']/*[local-name()='identifier']</INTERFACE_EXTRA_FIELD>
|
|
*/
|
|
|
|
|
|
}
|
|
|
|
@Override
|
|
public boolean isRemovable(String dsId, String apiId) throws DsmException {
|
|
return false;
|
|
}
|
|
|
|
@Override
|
|
public void regenerateProfiles() throws DsmException {
|
|
|
|
}
|
|
|
|
@Override
|
|
public Datasource<Organization<?>, Identity> getDs(String id) throws DsmException {
|
|
return null;
|
|
}
|
|
|
|
@Override
|
|
public List<? extends Api<ApiParam>> getApis(String dsId) throws DsmException {
|
|
return null;
|
|
}
|
|
|
|
@Override
|
|
public void deleteDs(String dsId) throws DsmException {
|
|
|
|
}
|
|
|
|
@Override
|
|
public void deleteApi(String dsId, String apiId) throws DsmException {
|
|
|
|
}
|
|
|
|
@Override
|
|
public void addApi(Api<ApiParam> api) throws DsmException {
|
|
|
|
}
|
|
|
|
@Override
|
|
public void setManaged(String id, boolean managed) throws DsmException {
|
|
|
|
}
|
|
|
|
@Override
|
|
public boolean isManaged(String id) throws DsmException {
|
|
return false;
|
|
}
|
|
|
|
@Override
|
|
public void saveDs(Datasource<Organization<?>, Identity> datasource) throws DsmException {
|
|
|
|
}
|
|
|
|
@Override
|
|
public void updateCompliance(String dsId, String apiId, String compliance, boolean override) throws DsmException {
|
|
|
|
}
|
|
|
|
// HELPERS
|
|
|
|
private void updateInterfaceExtraInfo(String dsId, String apiId, Map<String, String> updates) throws DsmException {
|
|
final String xpath = "//INTERFACE[@id = '%s']/INTERFACE_EXTRA_FIELD[@name='%s']";
|
|
final ISRegistryService isRegistry = serviceLocator.getService(ISRegistryService.class);
|
|
|
|
updates.forEach((k, v) -> {
|
|
try {
|
|
isRegistry.updateProfileNode(dsId, String.format(xpath, apiId, k), v);
|
|
} catch (ISRegistryException e) {
|
|
log.error("Error updating profile: " + dsId, e);
|
|
throw new DsmRuntimeException("Error updating profile: " + dsId, e);
|
|
}
|
|
});
|
|
}
|
|
|
|
private void updateInterfaceExtraInfo(String dsId, String apiId, Integer size, Date date, String last_date, String last_total) throws DsmException {
|
|
final String xpath = "//INTERFACE[@id = '%s']/INTERFACE_EXTRA_FIELD[@name='%s']";
|
|
final ISRegistryService isRegistry = serviceLocator.getService(ISRegistryService.class);
|
|
try {
|
|
isRegistry.updateProfileNode(dsId, String.format(xpath, apiId, last_date), date.toString());
|
|
isRegistry.updateProfileNode(dsId, String.format(xpath, apiId, last_total), size.toString());
|
|
|
|
} catch (ISRegistryException e) {
|
|
log.error("Error updating profile: " + dsId, e);
|
|
throw new DsmException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Error updating profile: " + dsId, e);
|
|
}
|
|
}
|
|
|
|
private XmlBrowsableField findBrowseField(final String id) throws DsmException {
|
|
for (XmlBrowsableField f : listBrowsableFields()) {
|
|
if (f.getId().equals(id)) {
|
|
return f;
|
|
}
|
|
}
|
|
return null;
|
|
}
|
|
|
|
public Map<String, String> fetchVocabularyTerms(final String voc) throws ISLookUpException {
|
|
final String xquery = "for $x in collection('/db/DRIVER/VocabularyDSResources/VocabularyDSResourceType')[.//VOCABULARY_NAME/@code = '"
|
|
+ voc.trim() + "']//TERM return concat($x/@code, ' @@@ ', $x/@english_name)";
|
|
|
|
final Map<String, String> map = Maps.newHashMap();
|
|
for (String s : serviceLocator.getService(ISLookUpService.class).quickSearchProfile(xquery)) {
|
|
final String[] arr = s.split("@@@");
|
|
map.put(arr[0].trim(), arr[1].trim());
|
|
}
|
|
return map;
|
|
}
|
|
|
|
private String findLabel(final String code, final Map<String, String> terms) {
|
|
return terms.containsKey(code) ? terms.get(code) : code;
|
|
}
|
|
|
|
}
|