D-Net
/
dnet-hadoop
14
5
Fork 11
Code Issues 14 Pull Requests 12 Projects 2 Releases 2 Wiki Activity

changed sql to select accepted datasources

This commit is contained in:
Michele Artini 2024-03-04 08:25:40 +01:00
parent c2b6841eb0
commit 9506d80ddc
3 changed files with 16 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/collection/plugin/base/BaseCollectorPlugin.java
View File

@ -64,27 +64,27 @@ public class BaseCollectorPlugin implements CollectorPlugin {
throw new CollectorException(e); throw new CollectorException(e);
} }
final Set<String> excludedOpendoarIds = findExcludedOpendoarIds(dbUrl, dbUser, dbPassword); final Set<String> acceptedOpendoarIds = findAcceptedOpendoarIds(dbUrl, dbUser, dbPassword);
final Iterator<String> iterator = new BaseCollectorIterator(this.fs, filePath, report); final Iterator<String> iterator = new BaseCollectorIterator(this.fs, filePath, report);
final Spliterator<String> spliterator = Spliterators.spliteratorUnknownSize(iterator, Spliterator.ORDERED); final Spliterator<String> spliterator = Spliterators.spliteratorUnknownSize(iterator, Spliterator.ORDERED);
return StreamSupport return StreamSupport
.stream(spliterator, false) .stream(spliterator, false)
.filter(doc -> filterXml(doc, excludedOpendoarIds, report)); .filter(doc -> filterXml(doc, acceptedOpendoarIds, report));
} }
private Set<String> findExcludedOpendoarIds(final String dbUrl, final String dbUser, final String dbPassword) throws CollectorException { private Set<String> findAcceptedOpendoarIds(final String dbUrl, final String dbUser, final String dbPassword) throws CollectorException {
final Set<String> excluded = new HashSet<>(); final Set<String> accepted = new HashSet<>();
try (final DbClient dbClient = new DbClient(dbUrl, dbUser, dbPassword)) { try (final DbClient dbClient = new DbClient(dbUrl, dbUser, dbPassword)) {
final String sql = IOUtils final String sql = IOUtils
.toString(BaseAnalyzerJob.class .toString(BaseAnalyzerJob.class
.getResourceAsStream("/eu/dnetlib/dhp/collection/plugin/base/sql/opendoar-ds-exclusion.sql")); .getResourceAsStream("/eu/dnetlib/dhp/collection/plugin/base/sql/opendoar-accepted.sql"));
dbClient.processResults(sql, row -> { dbClient.processResults(sql, row -> {
try { try {
excluded.add(row.getString("id")); accepted.add(row.getString("id"));
} catch (final SQLException e) { } catch (final SQLException e) {
log.error("Error in SQL", e); log.error("Error in SQL", e);
throw new RuntimeException("Error in SQL", e); throw new RuntimeException("Error in SQL", e);
@ -94,13 +94,13 @@ public class BaseCollectorPlugin implements CollectorPlugin {
log.error("Error accessong SQL", e); log.error("Error accessong SQL", e);
throw new CollectorException("Error accessong SQL", e); throw new CollectorException("Error accessong SQL", e);
} }
return excluded; return accepted;
} }
private boolean filterXml(final String xml, final Set<String> excludedOpendoarIds, final AggregatorReport report) { private boolean filterXml(final String xml, final Set<String> acceptedOpendoarIds, final AggregatorReport report) {
try { try {
final String id = DocumentHelper.parseText(xml).valueOf("//*[local-name='collection']/@opendoar_id").trim(); final String id = DocumentHelper.parseText(xml).valueOf("//*[local-name='collection']/@opendoar_id").trim();
return (StringUtils.isNotBlank(id) && !excludedOpendoarIds.contains("opendoar____::" + id.trim())); return (StringUtils.isNotBlank(id) && acceptedOpendoarIds.contains("opendoar____::" + id.trim()));
} catch (final DocumentException e) { } catch (final DocumentException e) {
log.error("Error parsing document", e); log.error("Error parsing document", e);
throw new RuntimeException("Error parsing document", e); throw new RuntimeException("Error parsing document", e);

7
dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/collection/plugin/base/sql/opendoar-accepted.sql Normal file
View File

@ -0,0 +1,7 @@
select s.id as id
from dsm_services s
where collectedfrom = 'openaire____::opendoar'
and jurisdiction = 'Institutional'
and s.id not in (
select service from dsm_api where coalesce(compatibility_override, compatibility) like '%openaire%' or last_collection_total > 0
);

8
dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/collection/plugin/base/sql/opendoar-ds-exclusion.sql
View File

@ -1,8 +0,0 @@
select distinct
s.id as id
from
dsm_services s
join dsm_api a on (s.id = a.service)
where
collectedfrom = 'openaire____::opendoar'
and (coalesce(a.compatibility_override, a.compatibility) like '%openaire%' or a.last_collection_total > 0);