diff --git a/dnet-app/apps/dnet-wf-aggregation-postgres/src/main/java/eu/dnetlib/wfs/nodes/MdCleanerJobNode.java b/dnet-app/apps/dnet-wf-aggregation-postgres/src/main/java/eu/dnetlib/wfs/nodes/MdCleanerJobNode.java index 4fdd399..03abc8b 100644 --- a/dnet-app/apps/dnet-wf-aggregation-postgres/src/main/java/eu/dnetlib/wfs/nodes/MdCleanerJobNode.java +++ b/dnet-app/apps/dnet-wf-aggregation-postgres/src/main/java/eu/dnetlib/wfs/nodes/MdCleanerJobNode.java @@ -59,17 +59,17 @@ public class MdCleanerJobNode extends ProcessNode { @Transactional protected void execute() throws Exception { - final Predicate filter = XpathFilterFactory.createFilter(filterXpath); + final Predicate filter = XpathFilterFactory.createFilter(this.filterXpath); - final Cleaner cleaner = cleanerFactory.newCleaner(ruleId); + final Cleaner cleaner = this.cleanerFactory.newCleaner(this.ruleId); - final MDStoreManagerClient mdstoreManager = clientFactory.getClient(MDStoreManagerClient.class); + final MDStoreManagerClient mdstoreManager = this.clientFactory.getClient(MDStoreManagerClient.class); - final MDStoreVersion inputVersion = mdstoreManager.startReading(inputMdId); - final MDStoreVersion outputVersion = mdstoreManager.newVersion(outputMdId); + final MDStoreVersion inputVersion = mdstoreManager.startReading(this.inputMdId); + final MDStoreVersion outputVersion = mdstoreManager.newVersion(this.outputMdId); try { - final Stream stream = mdStoreSqlBackend.streamEntries(inputVersion) + final Stream stream = this.mdStoreSqlBackend.streamEntries(inputVersion) .filter(record -> { try { final Document doc = DocumentHelper.parseText(record.getBody()); @@ -86,13 +86,13 @@ public class MdCleanerJobNode extends ProcessNode { return output; }); - mdStoreSqlBackend.saveRecords(outputVersion, stream); + this.mdStoreSqlBackend.saveRecords(outputVersion, stream); - final long size = mdStoreSqlBackend.countRecords(outputVersion.getId()); - outputVersion.setSize(size); - mdstoreManager.commitVersion(outputVersion); + final long size = this.mdStoreSqlBackend.countRecords(outputVersion.getId()); - clientFactory.getClient(DsmClient.class).updateApiAggregationInfo(api.getId(), outputMdId, size); + mdstoreManager.commitVersion(outputVersion.getId(), size); + + this.clientFactory.getClient(DsmClient.class).updateApiAggregationInfo(this.api.getId(), this.outputMdId, size); } catch (final Throwable e) { mdstoreManager.abortVersion(outputVersion); throw e; diff --git a/dnet-app/apps/dnet-wf-aggregation-postgres/src/main/java/eu/dnetlib/wfs/nodes/MdCollectIncrementalJobNode.java b/dnet-app/apps/dnet-wf-aggregation-postgres/src/main/java/eu/dnetlib/wfs/nodes/MdCollectIncrementalJobNode.java index 70119fa..f89a1e9 100644 --- a/dnet-app/apps/dnet-wf-aggregation-postgres/src/main/java/eu/dnetlib/wfs/nodes/MdCollectIncrementalJobNode.java +++ b/dnet-app/apps/dnet-wf-aggregation-postgres/src/main/java/eu/dnetlib/wfs/nodes/MdCollectIncrementalJobNode.java @@ -85,8 +85,8 @@ public class MdCollectIncrementalJobNode extends ProcessNode { this.mdStoreSqlBackend.saveRecords(outputVersion, stream); final long size = this.mdStoreSqlBackend.countRecords(outputVersion.getId()); - outputVersion.setSize(size); - mdstoreManager.commitVersion(outputVersion); + + mdstoreManager.commitVersion(outputVersion.getId(), size); this.clientFactory.getClient(DsmClient.class).updateApiCollectionInfo(this.api.getId(), this.mdId, size); } catch (final Throwable e) { diff --git a/dnet-app/apps/dnet-wf-aggregation-postgres/src/main/java/eu/dnetlib/wfs/nodes/MdCollectRefreshJobNode.java b/dnet-app/apps/dnet-wf-aggregation-postgres/src/main/java/eu/dnetlib/wfs/nodes/MdCollectRefreshJobNode.java index fe7e2f9..1fff12c 100644 --- a/dnet-app/apps/dnet-wf-aggregation-postgres/src/main/java/eu/dnetlib/wfs/nodes/MdCollectRefreshJobNode.java +++ b/dnet-app/apps/dnet-wf-aggregation-postgres/src/main/java/eu/dnetlib/wfs/nodes/MdCollectRefreshJobNode.java @@ -74,11 +74,12 @@ public class MdCollectRefreshJobNode extends ProcessNode { this.mdStoreSqlBackend.saveRecords(outputVersion, stream); final long size = this.mdStoreSqlBackend.countRecords(outputVersion.getId()); - outputVersion.setSize(size); - mdstoreManager.commitVersion(outputVersion); + + mdstoreManager.commitVersion(outputVersion.getId(), size); this.clientFactory.getClient(DsmClient.class).updateApiCollectionInfo(this.api.getId(), this.mdId, size); } catch (final Throwable e) { + System.err.println("****************** ABORTED"); mdstoreManager.abortVersion(outputVersion); throw e; } diff --git a/dnet-app/apps/dnet-wf-aggregation-postgres/src/main/java/eu/dnetlib/wfs/nodes/MdTransformJobNode.java b/dnet-app/apps/dnet-wf-aggregation-postgres/src/main/java/eu/dnetlib/wfs/nodes/MdTransformJobNode.java index b515b90..8f9b943 100644 --- a/dnet-app/apps/dnet-wf-aggregation-postgres/src/main/java/eu/dnetlib/wfs/nodes/MdTransformJobNode.java +++ b/dnet-app/apps/dnet-wf-aggregation-postgres/src/main/java/eu/dnetlib/wfs/nodes/MdTransformJobNode.java @@ -92,8 +92,8 @@ public class MdTransformJobNode extends ProcessNode { this.mdStoreSqlBackend.saveRecords(outputVersion, stream); final long size = this.mdStoreSqlBackend.countRecords(outputVersion.getId()); - outputVersion.setSize(size); - mdstoreManager.commitVersion(outputVersion); + + mdstoreManager.commitVersion(outputVersion.getId(), size); this.clientFactory.getClient(DsmClient.class).updateApiAggregationInfo(this.api.getId(), this.outputMdId, size); } catch (final Throwable e) { diff --git a/dnet-app/libs/dnet-app-common/src/main/java/eu/dnetlib/common/clients/MDStoreManagerClient.java b/dnet-app/libs/dnet-app-common/src/main/java/eu/dnetlib/common/clients/MDStoreManagerClient.java index 320d2ca..1db69d0 100644 --- a/dnet-app/libs/dnet-app-common/src/main/java/eu/dnetlib/common/clients/MDStoreManagerClient.java +++ b/dnet-app/libs/dnet-app-common/src/main/java/eu/dnetlib/common/clients/MDStoreManagerClient.java @@ -38,8 +38,8 @@ public class MDStoreManagerClient extends DnetServiceClient { return httpGet("/api/mdstores/mdstore/{mdId}/newVersion", MDStoreVersion.class, Map.of("mdId", mdId)); } - public void commitVersion(final MDStoreVersion version) { - httpGet("/api/mdstores/version/{versionId}/commit/" + version.getSize(), Void.class, Map.of("versionId", version.getId())); + public void commitVersion(final String versionId, final long size) { + httpGet("/api/mdstores/version/{versionId}/commit/{size}", Void.class, Map.of("versionId", versionId, "size", size)); } public void abortVersion(final MDStoreVersion version) { diff --git a/dnet-app/libs/dnet-mdstore-postgres/src/main/java/eu/dnetlib/common/mdstores/backends/sql/MDStoreSqlBackend.java b/dnet-app/libs/dnet-mdstore-postgres/src/main/java/eu/dnetlib/common/mdstores/backends/sql/MDStoreSqlBackend.java index 5431e57..9c4372c 100644 --- a/dnet-app/libs/dnet-mdstore-postgres/src/main/java/eu/dnetlib/common/mdstores/backends/sql/MDStoreSqlBackend.java +++ b/dnet-app/libs/dnet-mdstore-postgres/src/main/java/eu/dnetlib/common/mdstores/backends/sql/MDStoreSqlBackend.java @@ -54,12 +54,12 @@ public class MDStoreSqlBackend implements MDStoreBackend { final List toDelete = this.jdbcTemplate.queryForList("select tablename FROM pg_tables WHERE tablename = ? OR tablename like ?", String.class, table, vCond); - toDelete.forEach(t -> this.jdbcTemplate.execute("drop table " + t)); + toDelete.forEach(t -> this.jdbcTemplate.execute("drop table if exists " + t)); } @Override public void delete(final MDStoreVersion version) throws MDStoreManagerException { - this.jdbcTemplate.execute("drop table " + asTableName(version.getId())); + this.jdbcTemplate.execute("drop table if exists " + asTableName(version.getId())); } @Override diff --git a/dnet-app/libs/dnet-wf-executor-common/src/main/java/eu/dnetlib/wfs/collector/filesystem/FilesystemCollectorPlugin.java b/dnet-app/libs/dnet-wf-executor-common/src/main/java/eu/dnetlib/wfs/collector/filesystem/FilesystemCollectorPlugin.java index 725f888..6f1b849 100644 --- a/dnet-app/libs/dnet-wf-executor-common/src/main/java/eu/dnetlib/wfs/collector/filesystem/FilesystemCollectorPlugin.java +++ b/dnet-app/libs/dnet-wf-executor-common/src/main/java/eu/dnetlib/wfs/collector/filesystem/FilesystemCollectorPlugin.java @@ -29,9 +29,9 @@ public class FilesystemCollectorPlugin implements DnetCollectorPlugin { final URL basePath = new URL(baseUrl); final File baseDir = new File(basePath.getPath()); - if (!baseDir.exists()) { throw new DnetException(String.format("The base ULR %s, does not exist", basePath.getPath())); } + if (!baseDir.exists()) { throw new DnetException(String.format("The base URL %s, does not exist", basePath.getPath())); } - final FileSystemIterator fsi = new FileSystemIterator(baseDir.getAbsolutePath(), apiParams.get("extension")); + final FileSystemIterator fsi = new FileSystemIterator(baseDir.getAbsolutePath(), apiParams.getOrDefault("extensions", "xml")); return DnetStreamSupport.stream(fsi).map(this::loadFile); } diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index 8ec7efa..e77fe60 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -162,6 +162,8 @@ services: - OAI_DATABASE_URL=jdbc:postgresql://db-oai:${PG_PORT}/${PG_OAI_DB} - OAI_DATABASE_USERNAME=${PG_USER} - OAI_DATABASE_PASSWORD=${PG_PASSWORD} + volumes: + - /Users/michele/Develop/istiOpenPortal/puma_import/data:/var/lib/dnet/puma_import profiles: - wfs