working on the DSM

master
Claudio Atzori 5 years ago
parent 77292eb853
commit cfbdebaf4d

@ -2,6 +2,10 @@ package eu.dnetlib.enabling.datasources.common;
public interface BrowseTerm {
String getCode();
void setCode(String code);
String getTerm();
void setTerm(String term);

@ -2,28 +2,44 @@ package eu.dnetlib.enabling.datasources.common;
public class BrowseTermImpl implements BrowseTerm {
private String code;
private String term;
private long total;
public BrowseTermImpl() {}
public BrowseTermImpl(final String term, final int total) {
public BrowseTermImpl(final String code, final String term, final int total) {
this.code = code;
this.term = term;
this.total = total;
}
@Override
public String getCode() {
return code;
}
@Override
public void setCode(final String code) {
this.code = code;
}
@Override
public String getTerm() {
return term;
}
@Override
public void setTerm(final String term) {
this.term = term;
}
@Override
public long getTotal() {
return total;
}
@Override
public void setTotal(final long total) {
this.total = total;
}

@ -15,7 +15,7 @@ public interface LocalDatasourceManager<DS extends Datasource<?, ?>, API extends
List<? extends BrowsableField> listBrowsableFields() throws DsmException;
List<? extends BrowseTerm> browseField(String field) throws DsmException;
List<BrowseTerm> browseField(String field) throws DsmException;
void setActive(String dsId, String apiId, boolean active) throws DsmException;

@ -2,7 +2,6 @@ package eu.dnetlib.enabling.datasources.common;
import javax.xml.bind.annotation.XmlRootElement;
@XmlRootElement
public class SearchApisEntry implements Comparable<SearchApisEntry> {
private String id;

@ -1,20 +1,51 @@
package eu.dnetlib.data.datasource;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
import eu.dnetlib.enabling.datasources.common.*;
import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpException;
import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpService;
import eu.dnetlib.enabling.is.registry.rmi.ISRegistryException;
import eu.dnetlib.enabling.is.registry.rmi.ISRegistryService;
import eu.dnetlib.enabling.locators.UniqueServiceLocator;
import org.antlr.stringtemplate.StringTemplate;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.StringEscapeUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.math.NumberUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.http.HttpStatus;
import org.dom4j.Document;
import org.dom4j.Node;
import org.dom4j.io.SAXReader;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Required;
import org.springframework.core.io.ClassPathResource;
import java.util.Date;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.io.StringReader;
import java.util.*;
import java.util.stream.Collectors;
public class LocalSimpleDatasourceManagerImpl implements LocalSimpleDatasourceManager {
private static final Log log = LogFactory.getLog(LocalSimpleDatasourceManagerImpl.class);
private static final String REPOSITORY_RESOURCE_TYPE = "RepositoryServiceResourceType";
private ClassPathResource findReposQueryTmpl = new ClassPathResource("/eu/dnetlib/enabling/datasources/templates/findRepos.xquery.st");
private ClassPathResource browseRepoApisQueryTmpl = new ClassPathResource("/eu/dnetlib/enabling/datasources/templates/browseRepoApis.xquery.st");
private ClassPathResource findRepoApisQueryTmpl = new ClassPathResource("/eu/dnetlib/enabling/datasources/templates/findRepoApis.xquery.st");
private ClassPathResource findReposMapQuery = new ClassPathResource("/eu/dnetlib/enabling/datasources/templates/findReposMap.xquery");
private ClassPathResource simpleFindReposQueryTmpl = new ClassPathResource("/eu/dnetlib/enabling/datasources/templates/simpleFindRepos.xquery.st");
private List<XmlBrowsableField> browsableFields;
@Autowired
private UniqueServiceLocator serviceLocator;
@Override
public Set<String> listManagedDatasourceIds() throws DsmRuntimeException {
throw new DsmRuntimeException("method 'listManagedDatasourceIds' not implemented");
@ -22,58 +53,203 @@ public class LocalSimpleDatasourceManagerImpl implements LocalSimpleDatasourceMa
@Override
public List<SimpleDatasource> searchDatasourcesByType(String type) throws DsmException {
return null;
try {
final List<SimpleDatasource> list = new ArrayList<>();
final StringTemplate st = new StringTemplate(IOUtils.toString(simpleFindReposQueryTmpl.getInputStream()));
st.setAttribute("type", type);
for (String s : serviceLocator.getService(ISLookUpService.class).quickSearchProfile(st.toString())) {
final SimpleDatasource r = new SimpleDatasource();
final String[] arr = s.split("@=@");
r.setId(arr[0].trim());
r.setName(arr[1].trim());
r.setOrigId(arr[2].trim());
r.setApis(Sets.newHashSet(arr[3].replaceAll("\\s", "").split(",")));
r.setValid("true".equals(arr[4].trim()));
r.setTypology(type);
list.add(r);
}
Collections.sort(list);
return list;
} catch (Exception e) {
log.error("Error listing repos", e);
}
return Lists.newArrayList();
}
@Override
public List<? extends SearchApisEntry> searchApis(String field, Object value) throws DsmException {
return null;
try {
final StringTemplate st = new StringTemplate();
st.setTemplate(IOUtils.toString(findRepoApisQueryTmpl.getInputStream()));
final String val = value.toString();
if (field.equalsIgnoreCase("__search__")) {
st.setAttribute("cond", "contains(../..//(*|@*)/lower-case(.), '" + StringEscapeUtils.escapeXml(val.toLowerCase()) + "')");
} else {
final XmlBrowsableField f = findBrowseField(field);
if (f != null) {
st.setAttribute("cond", f.getXpath() + "='" + StringEscapeUtils.escapeXml(val) + "'");
} else {
return Lists.newArrayList();
}
}
final String query = st.toString();
final SAXReader reader = new SAXReader();
final List<String> list = serviceLocator.getService(ISLookUpService.class).quickSearchProfile(query);
return list.stream()
.map(s -> {
final SearchApisEntry iface = new SearchApisEntry();
try {
final Document doc = reader.read(new StringReader(s));
final String country = doc.valueOf("//REPO/@country");
iface.setRepoId(doc.valueOf("//REPO/@id"));
iface.setRepoCountry(StringUtils.isEmpty(country) ? "-" : country.toUpperCase());
iface.setRepoName(doc.valueOf("//REPO/@name"));
iface.setRepoPrefix(doc.valueOf("//REPO/@prefix"));
final Node ifcNode = doc.selectSingleNode("//INTERFACE");
iface.setId(ifcNode.valueOf("./@id"));
iface.setActive(Boolean.valueOf(ifcNode.valueOf("./@active")));
iface.setProtocol(ifcNode.valueOf("./ACCESS_PROTOCOL/text()"));
final String overCompliance = ifcNode.valueOf("./INTERFACE_EXTRA_FIELD[@name='overriding_compliance']");
if (StringUtils.isEmpty(overCompliance)) {
iface.setCompliance(ifcNode.valueOf("@compliance"));
} else {
iface.setCompliance(overCompliance);
}
final String lastAggregationDate = ifcNode.valueOf("./INTERFACE_EXTRA_FIELD[@name='last_aggregation_date']");
if (!StringUtils.isEmpty(lastAggregationDate)) {
iface.setAggrDate(lastAggregationDate);
} else {
final String lastDownloadDate = ifcNode.valueOf("./INTERFACE_EXTRA_FIELD[@name='last_download_date']");
if (!StringUtils.isEmpty(lastDownloadDate)) {
iface.setAggrDate(lastDownloadDate);
}
}
final String lastAggregationTotal = ifcNode.valueOf("./INTERFACE_EXTRA_FIELD[@name='last_aggregation_total']");
if (StringUtils.isEmpty(lastAggregationTotal)) {
final String lastDownloadTotal = ifcNode.valueOf("./INTERFACE_EXTRA_FIELD[@name='last_download_total']");
if (StringUtils.isEmpty(lastDownloadTotal)) {
iface.setAggrTotal(0);
} else {
iface.setAggrTotal(NumberUtils.toInt(lastDownloadTotal, 0));
}
} else {
iface.setAggrTotal(NumberUtils.toInt(lastAggregationTotal, 0));
}
} catch (Exception e) {
log.error(e);
}
return iface;
}).collect(Collectors.toList());
} catch (Exception e) {
log.error("Error searching field " + field + " - value: " + value, e);
}
return Lists.newArrayList();
}
@Override
public List<? extends BrowsableField> listBrowsableFields() throws DsmException {
return null;
public List<XmlBrowsableField> listBrowsableFields() throws DsmException {
return getBrowsableFields();
}
public List<XmlBrowsableField> getBrowsableFields() {
return browsableFields;
}
@Required
public void setBrowsableFields(List<XmlBrowsableField> browsableFields) {
this.browsableFields = browsableFields;
}
@Override
public List<? extends BrowseTerm> browseField(String field) throws DsmException {
return null;
public List<BrowseTerm> browseField(final String f) throws DsmException {
final List<BrowseTerm> list = Lists.newArrayList();
try {
final XmlBrowsableField field = findBrowseField(f);
if (field != null) {
final StringTemplate st = new StringTemplate(IOUtils.toString(browseRepoApisQueryTmpl.getInputStream()));
st.setAttribute("xpath", field.getXpath());
final Map<String, String> terms = fetchVocabularyTerms(field.getVocabulary());
for (String s : serviceLocator.getService(ISLookUpService.class).quickSearchProfile(st.toString())) {
final String[] arr = s.split("@-@-@");
final String id = arr[0].trim();
final int count = NumberUtils.toInt(arr[1].trim(), 0);
list.add(new BrowseTermImpl(id, findLabel(id, terms), count));
}
}
} catch (Exception e) {
log.error("Error browsing field " + f, e);
}
return list;
}
@Override
public void setActive(String dsId, String apiId, boolean active) throws DsmException {
throw new DsmRuntimeException("method 'setActive' not implemented");
}
@Override
public boolean isActive(String dsId, String apiId) throws DsmException {
return false;
throw new DsmRuntimeException("method 'isActive' not implemented");
}
@Override
public void setLastCollectionInfo(String dsId, String apiId, String mdId, Integer size, Date date) throws DsmException {
Map<String, String> updates = Maps.newHashMap();
updates.put("last_collection_date", date.toString());
updates.put("last_collection_total", size.toString());
updateInterfaceExtraInfo(dsId, apiId, updates);
}
@Override
public void setLastAggregationInfo(String dsId, String apiId, String mdId, Integer size, Date date) throws DsmException {
Map<String, String> updates = Maps.newHashMap();
updates.put("last_aggregation_date", date.toString());
updates.put("last_aggregation_total", size.toString());
updateInterfaceExtraInfo(dsId, apiId, updates);
}
@Override
public void setLastDownloadInfo(String dsId, String apiId, String objId, Integer size, Date date) throws DsmException {
Map<String, String> updates = Maps.newHashMap();
updates.put("last_download_date", date.toString());
updates.put("last_download_total", size.toString());
updateInterfaceExtraInfo(dsId, apiId, updates);
}
@Override
public void setLastValidationJob(String dsId, String apiId, String jobId) throws DsmException {
Map<String, String> updates = Maps.newHashMap();
updates.put("last_validation_job", jobId);
updateInterfaceExtraInfo(dsId, apiId, updates);
}
@Override
public void updateApiDetails(String dsId, String apiId, String metadataIdentifierPath, String baseUrl, Map<String, String> params) throws DsmException {
// oaiset, baseurl, compatibility
/*
<INTERFACE active="true" compliance="openaire3.0" contentDescription="metadata" id="api_________::opendoar____::2294::0" label="pubsrepository::institutional (openaire3.0)" removable="true" typology="pubsrepository::institutional">
<ACCESS_PROTOCOL format="oai_dc" set="openaire">oai</ACCESS_PROTOCOL>
<BASE_URL>http://pub.uni-bielefeld.de/oai</BASE_URL>
<INTERFACE_EXTRA_FIELD name="metadata_identifier_path">//*[local-name()='header']/*[local-name()='identifier']</INTERFACE_EXTRA_FIELD>
*/
}
@Override
@ -130,4 +306,59 @@ public class LocalSimpleDatasourceManagerImpl implements LocalSimpleDatasourceMa
public void updateCompliance(String dsId, String apiId, String compliance, boolean override) throws DsmException {
}
// HELPERS
private void updateInterfaceExtraInfo(String dsId, String apiId, Map<String, String> updates) throws DsmException {
final String xpath = "//INTERFACE[@id = '%s']/INTERFACE_EXTRA_FIELD[@name='%s']";
final ISRegistryService isRegistry = serviceLocator.getService(ISRegistryService.class);
updates.forEach((k, v) -> {
try {
isRegistry.updateProfileNode(dsId, String.format(xpath, apiId, k), v);
} catch (ISRegistryException e) {
log.error("Error updating profile: " + dsId, e);
throw new DsmRuntimeException("Error updating profile: " + dsId, e);
}
});
}
private void updateInterfaceExtraInfo(String dsId, String apiId, Integer size, Date date, String last_date, String last_total) throws DsmException {
final String xpath = "//INTERFACE[@id = '%s']/INTERFACE_EXTRA_FIELD[@name='%s']";
final ISRegistryService isRegistry = serviceLocator.getService(ISRegistryService.class);
try {
isRegistry.updateProfileNode(dsId, String.format(xpath, apiId, last_date), date.toString());
isRegistry.updateProfileNode(dsId, String.format(xpath, apiId, last_total), size.toString());
} catch (ISRegistryException e) {
log.error("Error updating profile: " + dsId, e);
throw new DsmException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Error updating profile: " + dsId, e);
}
}
private XmlBrowsableField findBrowseField(final String id) throws DsmException {
for (XmlBrowsableField f : listBrowsableFields()) {
if (f.getId().equals(id)) {
return f;
}
}
return null;
}
public Map<String, String> fetchVocabularyTerms(final String voc) throws ISLookUpException {
final String xquery = "for $x in collection('/db/DRIVER/VocabularyDSResources/VocabularyDSResourceType')[.//VOCABULARY_NAME/@code = '"
+ voc.trim() + "']//TERM return concat($x/@code, ' @@@ ', $x/@english_name)";
final Map<String, String> map = Maps.newHashMap();
for (String s : serviceLocator.getService(ISLookUpService.class).quickSearchProfile(xquery)) {
final String[] arr = s.split("@@@");
map.put(arr[0].trim(), arr[1].trim());
}
return map;
}
private String findLabel(final String code, final Map<String, String> terms) {
return terms.containsKey(code) ? terms.get(code) : code;
}
}

@ -0,0 +1,40 @@
package eu.dnetlib.data.datasource;
import eu.dnetlib.enabling.datasources.common.BrowsableField;
public class XmlBrowsableField extends BrowsableField {
private String xpath;
private String vocabulary;
public XmlBrowsableField() {
super();
}
public XmlBrowsableField(final String id, final String label) {
super(id, label);
}
public XmlBrowsableField(final String id, final String label, final String xpath, final String vocabulary) {
super(id, label);
this.xpath = xpath;
this.vocabulary = vocabulary;
}
public String getXpath() {
return xpath;
}
public void setXpath(final String xpath) {
this.xpath = xpath;
}
public String getVocabulary() {
return vocabulary;
}
public void setVocabulary(final String vocabulary) {
this.vocabulary = vocabulary;
}
}

@ -10,8 +10,24 @@
http://cxf.apache.org/jaxws http://cxf.apache.org/schemas/jaxws.xsd
http://dnetlib.eu/springbeans/template http://dnetlib.eu/springbeans/template.xsd">
<bean id="datasourceManager" class="eu.dnetlib.data.datasource.LocalSimpleDatasourceManagerImpl" />
<bean id="datasourceManager" class="eu.dnetlib.data.datasource.LocalSimpleDatasourceManagerImpl">
<property name="browsableFields">
<list>
<bean class="eu.dnetlib.data.datasource.XmlBrowsableField"
p:id="country" p:label="Datasource countries" p:xpath="../../COUNTRY/text()"
p:vocabulary="dnet:countries" />
<bean class="eu.dnetlib.data.datasource.XmlBrowsableField"
p:id="type" p:label="API typologies" p:xpath="@typology/string()"
p:vocabulary="dnet:datasource_typologies" />
<bean class="eu.dnetlib.data.datasource.XmlBrowsableField"
p:id="protocol" p:label="API protocols" p:xpath="ACCESS_PROTOCOL/text()"
p:vocabulary="dnet:protocols" />
<bean class="eu.dnetlib.data.datasource.XmlBrowsableField"
p:id="compliance" p:label="API compatibility levels"
p:xpath="concat(substring(./INTERFACE_EXTRA_FIELD[@name='overriding_compliance'], 1, number(boolean(./INTERFACE_EXTRA_FIELD[@name='overriding_compliance'])) * string-length(./INTERFACE_EXTRA_FIELD[@name='overriding_compliance'])), substring(@compliance, 1, number(not(boolean(./INTERFACE_EXTRA_FIELD[@name='overriding_compliance']))) * string-length(@compliance)))"
p:vocabulary="dnet:compatibilityLevel" />
</list>
</property>
</bean>
</beans>

@ -0,0 +1,5 @@
let \$list := collection('/db/DRIVER/RepositoryServiceResources/RepositoryServiceResourceType')//INTERFACE/$xpath$
for \$x in distinct-values(\$list)
where string-length(\$x) > 0
return concat(\$x, ' @-@-@ ', count(\$list[.=\$x]))

@ -0,0 +1,11 @@
for \$x in
collection('/db/DRIVER/RepositoryServiceResources/RepositoryServiceResourceType')//INTERFACE[$cond$]
return
<ROW>
<REPO
id="{\$x/../../../../HEADER/RESOURCE_IDENTIFIER/@value/string()}"
name="{\$x/../../OFFICIAL_NAME/text()}"
country="{\$x/../../COUNTRY/text()}"
prefix="{\$x/../..//EXTRA_FIELDS/FIELD[./key='NamespacePrefix']/value}" />
{\$x}
</ROW>

@ -0,0 +1,11 @@
for \$x in collection('/db/DRIVER/RepositoryServiceResources/RepositoryServiceResourceType')
$if(conds)$
where $conds.keys:{xpath| \$x$xpath$ = '$conds.(xpath)$'}; separator=" and "$
$endif$
return \$x
,
for \$x in collection('/db/DRIVER/PendingRepositoryResources/RepositoryServiceResourceType')
$if(conds)$
where $conds.keys:{xpath| \$x$xpath$ = '$conds.(xpath)$'}; separator=" and "$
$endif$
return \$x

@ -0,0 +1,11 @@
for $x in
collection('/db/DRIVER/RepositoryServiceResources/RepositoryServiceResourceType')
where
$x//NUMBER_OF_OBJECTS != 0 and not($x//LATITUDE = 0 and $x//LONGITUDE = 0)
return
<ds>
<dsId>{$x//RESOURCE_IDENTIFIER/@value/string()}</dsId>
<name>{$x//OFFICIAL_NAME/text()}</name>
<lat>{$x//LATITUDE/text()}</lat>
<lng>{$x//LONGITUDE/text()}</lng>
</ds>

@ -0,0 +1,12 @@
for \$x in
collection('/db/DRIVER/RepositoryServiceResources/RepositoryServiceResourceType')
where
\$x//DATASOURCE_TYPE = '$type$'
return
concat(\$x//RESOURCE_IDENTIFIER/@value, ' @=@ ', \$x//OFFICIAL_NAME, ' @=@ ', \$x//DATASOURCE_ORIGINAL_ID, ' @=@ ', string-join(\$x//INTERFACE/@id, ','), ' @=@ true'),
for \$x in
collection('/db/DRIVER/PendingRepositoryResources/RepositoryServiceResourceType')
where
\$x//DATASOURCE_TYPE = '$type$'
return
concat(\$x//RESOURCE_IDENTIFIER/@value, ' @=@ ', \$x//OFFICIAL_NAME, ' @=@ ', \$x//DATASOURCE_ORIGINAL_ID, ' @=@ ', string-join(\$x//INTERFACE/@id, ','), ' @=@ false')

@ -33,7 +33,7 @@
<version>${project.version}</version>
</dependency>
<!--
<!-- candidate for removal
<dependency>
<groupId>org.codehaus.jackson</groupId>
<artifactId>jackson-mapper-asl</artifactId>

@ -76,7 +76,7 @@ public class RepoInternalController {
private static final Log log = LogFactory.getLog(RepoInternalController.class);
@RequestMapping(value = "/ui/browseRepoField.do")
public @ResponseBody List<? extends BrowseTerm> browseRepoField(@RequestParam(value = "field", required = true) final String field) throws Exception {
public @ResponseBody List<BrowseTerm> browseRepoField(@RequestParam(value = "field", required = true) final String field) throws Exception {
return dsManager.browseField(field);
}

@ -58,7 +58,7 @@
</thead>
<tbody>
<tr ng-repeat="row in repoBrowseData.data | orderBy:sortType:sortReverse | filter:filterBrowseData">
<td><a href="javascript:void(0)" ng-click="browseApis(repoBrowseData.id,row.term)">{{row.term}}</a></td>
<td><a href="javascript:void(0)" ng-click="browseApis(repoBrowseData.id,row.code)">{{row.term}}</a></td>
<td class="text-right">{{row.total}}</td>
</tr>
</tbody>

Loading…
Cancel
Save