D-Net
/
dnet-hadoop
14
5
Fork 11
Code Issues 14 Pull Requests 12 Projects 2 Releases 2 Wiki Activity

fixed a xpath

This commit is contained in:
Michele Artini 2024-03-08 08:23:07 +01:00
parent 6648d710a3
commit 3b5163d8e3
1 changed files with 17 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

31
dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/collection/plugin/base/BaseCollectorPlugin.java
View File

@ -45,9 +45,9 @@ public class BaseCollectorPlugin implements CollectorPlugin {
public Stream<String> collect(final ApiDescriptor api, final AggregatorReport report) throws CollectorException { public Stream<String> collect(final ApiDescriptor api, final AggregatorReport report) throws CollectorException {
// get path to file // get path to file
final Path filePath = Optional final Path filePath = Optional
.ofNullable(api.getBaseUrl()) .ofNullable(api.getBaseUrl())
.map(Path::new) .map(Path::new)
.orElseThrow(() -> new CollectorException("missing baseUrl")); .orElseThrow(() -> new CollectorException("missing baseUrl"));
final String dbUrl = api.getParams().get("dbUrl"); final String dbUrl = api.getParams().get("dbUrl");
final String dbUser = api.getParams().get("dbUser"); final String dbUser = api.getParams().get("dbUser");
@ -59,9 +59,7 @@ public class BaseCollectorPlugin implements CollectorPlugin {
log.info("dbPassword: {}", "***"); log.info("dbPassword: {}", "***");
try { try {
if (!this.fs.exists(filePath)) { if (!this.fs.exists(filePath)) { throw new CollectorException("path does not exist: " + filePath); }
throw new CollectorException("path does not exist: " + filePath);
}
} catch (final Throwable e) { } catch (final Throwable e) {
throw new CollectorException(e); throw new CollectorException(e);
} }
@ -71,39 +69,44 @@ public class BaseCollectorPlugin implements CollectorPlugin {
final Iterator<String> iterator = new BaseCollectorIterator(this.fs, filePath, report); final Iterator<String> iterator = new BaseCollectorIterator(this.fs, filePath, report);
final Spliterator<String> spliterator = Spliterators.spliteratorUnknownSize(iterator, Spliterator.ORDERED); final Spliterator<String> spliterator = Spliterators.spliteratorUnknownSize(iterator, Spliterator.ORDERED);
return StreamSupport return StreamSupport
.stream(spliterator, false) .stream(spliterator, false)
.filter(doc -> filterXml(doc, acceptedOpendoarIds, report)); .filter(doc -> filterXml(doc, acceptedOpendoarIds, report));
} }
private Set<String> findAcceptedOpendoarIds(final String dbUrl, final String dbUser, final String dbPassword) private Set<String> findAcceptedOpendoarIds(final String dbUrl, final String dbUser, final String dbPassword)
throws CollectorException { throws CollectorException {
final Set<String> accepted = new HashSet<>(); final Set<String> accepted = new HashSet<>();
try (final DbClient dbClient = new DbClient(dbUrl, dbUser, dbPassword)) { try (final DbClient dbClient = new DbClient(dbUrl, dbUser, dbPassword)) {
final String sql = IOUtils final String sql = IOUtils
.toString( .toString(BaseAnalyzerJob.class
BaseAnalyzerJob.class .getResourceAsStream("/eu/dnetlib/dhp/collection/plugin/base/sql/opendoar-accepted.sql"));
.getResourceAsStream("/eu/dnetlib/dhp/collection/plugin/base/sql/opendoar-accepted.sql"));
dbClient.processResults(sql, row -> { dbClient.processResults(sql, row -> {
try { try {
accepted.add(row.getString("id")); final String dsId = row.getString("id");
log.info("Accepted Datasource: " + dsId);
accepted.add(dsId);
} catch (final SQLException e) { } catch (final SQLException e) {
log.error("Error in SQL", e); log.error("Error in SQL", e);
throw new RuntimeException("Error in SQL", e); throw new RuntimeException("Error in SQL", e);
} }
}); });
} catch (final IOException e) { } catch (final IOException e) {
log.error("Error accessong SQL", e); log.error("Error accessong SQL", e);
throw new CollectorException("Error accessong SQL", e); throw new CollectorException("Error accessong SQL", e);
} }
log.info("Accepted Datasources (TOTAL): " + accepted.size());
return accepted; return accepted;
} }
private boolean filterXml(final String xml, final Set<String> acceptedOpendoarIds, final AggregatorReport report) { private boolean filterXml(final String xml, final Set<String> acceptedOpendoarIds, final AggregatorReport report) {
try { try {
final String id = DocumentHelper.parseText(xml).valueOf("//*[local-name='collection']/@opendoar_id").trim(); final String id = DocumentHelper.parseText(xml).valueOf("//*[local-name()='collection']/@opendoar_id").trim();
return (StringUtils.isNotBlank(id) && acceptedOpendoarIds.contains("opendoar____::" + id.trim())); return (StringUtils.isNotBlank(id) && acceptedOpendoarIds.contains("opendoar____::" + id.trim()));
} catch (final DocumentException e) { } catch (final DocumentException e) {
log.error("Error parsing document", e); log.error("Error parsing document", e);