michele.artini
/
dnet-docker
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

collector nodes

This commit is contained in:
Michele Artini 2024-01-18 12:06:09 +01:00
parent d7f5a60e2e
commit 2520e51c3b
8 changed files with 26 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
dnet-app/apps/dnet-wf-aggregation-postgres/src/main/java/eu/dnetlib/wfs/nodes/MdCleanerJobNode.java
View File

@ -59,17 +59,17 @@ public class MdCleanerJobNode extends ProcessNode {
@Transactional
protected void execute() throws Exception {
final Predicate<Document> filter = XpathFilterFactory.createFilter(filterXpath);
final Predicate<Document> filter = XpathFilterFactory.createFilter(this.filterXpath);
final Cleaner cleaner = cleanerFactory.newCleaner(ruleId);
final Cleaner cleaner = this.cleanerFactory.newCleaner(this.ruleId);
final MDStoreManagerClient mdstoreManager = clientFactory.getClient(MDStoreManagerClient.class);
final MDStoreManagerClient mdstoreManager = this.clientFactory.getClient(MDStoreManagerClient.class);
final MDStoreVersion inputVersion = mdstoreManager.startReading(inputMdId);
final MDStoreVersion outputVersion = mdstoreManager.newVersion(outputMdId);
final MDStoreVersion inputVersion = mdstoreManager.startReading(this.inputMdId);
final MDStoreVersion outputVersion = mdstoreManager.newVersion(this.outputMdId);
try {
final Stream<MetadataRecord> stream = mdStoreSqlBackend.streamEntries(inputVersion)
final Stream<MetadataRecord> stream = this.mdStoreSqlBackend.streamEntries(inputVersion)
.filter(record -> {
try {
final Document doc = DocumentHelper.parseText(record.getBody());
@ -86,13 +86,13 @@ public class MdCleanerJobNode extends ProcessNode {
return output;
});
mdStoreSqlBackend.saveRecords(outputVersion, stream);
this.mdStoreSqlBackend.saveRecords(outputVersion, stream);
final long size = mdStoreSqlBackend.countRecords(outputVersion.getId());
outputVersion.setSize(size);
mdstoreManager.commitVersion(outputVersion);
final long size = this.mdStoreSqlBackend.countRecords(outputVersion.getId());
clientFactory.getClient(DsmClient.class).updateApiAggregationInfo(api.getId(), outputMdId, size);
mdstoreManager.commitVersion(outputVersion.getId(), size);
this.clientFactory.getClient(DsmClient.class).updateApiAggregationInfo(this.api.getId(), this.outputMdId, size);
} catch (final Throwable e) {
mdstoreManager.abortVersion(outputVersion);
throw e;

4
dnet-app/apps/dnet-wf-aggregation-postgres/src/main/java/eu/dnetlib/wfs/nodes/MdCollectIncrementalJobNode.java
View File

@ -85,8 +85,8 @@ public class MdCollectIncrementalJobNode extends ProcessNode {
this.mdStoreSqlBackend.saveRecords(outputVersion, stream);
final long size = this.mdStoreSqlBackend.countRecords(outputVersion.getId());
outputVersion.setSize(size);
mdstoreManager.commitVersion(outputVersion);
mdstoreManager.commitVersion(outputVersion.getId(), size);
this.clientFactory.getClient(DsmClient.class).updateApiCollectionInfo(this.api.getId(), this.mdId, size);
} catch (final Throwable e) {

5
dnet-app/apps/dnet-wf-aggregation-postgres/src/main/java/eu/dnetlib/wfs/nodes/MdCollectRefreshJobNode.java
View File

@ -74,11 +74,12 @@ public class MdCollectRefreshJobNode extends ProcessNode {
this.mdStoreSqlBackend.saveRecords(outputVersion, stream);
final long size = this.mdStoreSqlBackend.countRecords(outputVersion.getId());
outputVersion.setSize(size);
mdstoreManager.commitVersion(outputVersion);
mdstoreManager.commitVersion(outputVersion.getId(), size);
this.clientFactory.getClient(DsmClient.class).updateApiCollectionInfo(this.api.getId(), this.mdId, size);
} catch (final Throwable e) {
System.err.println("****************** ABORTED");
mdstoreManager.abortVersion(outputVersion);
throw e;
}

4
dnet-app/apps/dnet-wf-aggregation-postgres/src/main/java/eu/dnetlib/wfs/nodes/MdTransformJobNode.java
View File

@ -92,8 +92,8 @@ public class MdTransformJobNode extends ProcessNode {
this.mdStoreSqlBackend.saveRecords(outputVersion, stream);
final long size = this.mdStoreSqlBackend.countRecords(outputVersion.getId());
outputVersion.setSize(size);
mdstoreManager.commitVersion(outputVersion);
mdstoreManager.commitVersion(outputVersion.getId(), size);
this.clientFactory.getClient(DsmClient.class).updateApiAggregationInfo(this.api.getId(), this.outputMdId, size);
} catch (final Throwable e) {

4
dnet-app/libs/dnet-app-common/src/main/java/eu/dnetlib/common/clients/MDStoreManagerClient.java
View File

@ -38,8 +38,8 @@ public class MDStoreManagerClient extends DnetServiceClient {
return httpGet("/api/mdstores/mdstore/{mdId}/newVersion", MDStoreVersion.class, Map.of("mdId", mdId));
}
public void commitVersion(final MDStoreVersion version) {
httpGet("/api/mdstores/version/{versionId}/commit/" + version.getSize(), Void.class, Map.of("versionId", version.getId()));
public void commitVersion(final String versionId, final long size) {
httpGet("/api/mdstores/version/{versionId}/commit/{size}", Void.class, Map.of("versionId", versionId, "size", size));
}
public void abortVersion(final MDStoreVersion version) {

4
dnet-app/libs/dnet-mdstore-postgres/src/main/java/eu/dnetlib/common/mdstores/backends/sql/MDStoreSqlBackend.java
View File

@ -54,12 +54,12 @@ public class MDStoreSqlBackend implements MDStoreBackend {
final List<String> toDelete =
this.jdbcTemplate.queryForList("select tablename FROM pg_tables WHERE tablename = ? OR tablename like ?", String.class, table, vCond);
toDelete.forEach(t -> this.jdbcTemplate.execute("drop table " + t));
toDelete.forEach(t -> this.jdbcTemplate.execute("drop table if exists " + t));
}
@Override
public void delete(final MDStoreVersion version) throws MDStoreManagerException {
this.jdbcTemplate.execute("drop table " + asTableName(version.getId()));
this.jdbcTemplate.execute("drop table if exists " + asTableName(version.getId()));
}
@Override

4
dnet-app/libs/dnet-wf-executor-common/src/main/java/eu/dnetlib/wfs/collector/filesystem/FilesystemCollectorPlugin.java
View File

@ -29,9 +29,9 @@ public class FilesystemCollectorPlugin implements DnetCollectorPlugin {
final URL basePath = new URL(baseUrl);
final File baseDir = new File(basePath.getPath());
if (!baseDir.exists()) { throw new DnetException(String.format("The base ULR %s, does not exist", basePath.getPath())); }
if (!baseDir.exists()) { throw new DnetException(String.format("The base URL %s, does not exist", basePath.getPath())); }
final FileSystemIterator fsi = new FileSystemIterator(baseDir.getAbsolutePath(), apiParams.get("extension"));
final FileSystemIterator fsi = new FileSystemIterator(baseDir.getAbsolutePath(), apiParams.getOrDefault("extensions", "xml"));
return DnetStreamSupport.stream(fsi).map(this::loadFile);
}

2
docker-compose.dev.yml
View File

@ -162,6 +162,8 @@ services:
- OAI_DATABASE_URL=jdbc:postgresql://db-oai:${PG_PORT}/${PG_OAI_DB}
- OAI_DATABASE_USERNAME=${PG_USER}
- OAI_DATABASE_PASSWORD=${PG_PASSWORD}
volumes:
- /Users/michele/Develop/istiOpenPortal/puma_import/data:/var/lib/dnet/puma_import
profiles:
- wfs