From 2d9f71b2b36835190102529dba554061ab9b2f83 Mon Sep 17 00:00:00 2001
From: "michele.artini" <michele.artini@isti.cnr.it>
Date: Tue, 16 Jan 2024 14:07:15 +0100
Subject: [PATCH] wf profiles

---
 .../eu/dnetlib/wfs/nodes/MdTransformJobNode.java   | 14 +++++++-------
 .../src/main/resources/wf_templates/collect.json   | 10 +++++-----
 .../src/main/resources/wf_templates/transform.json |  7 ++++++-
 3 files changed, 18 insertions(+), 13 deletions(-)

diff --git a/dnet-app/apps/dnet-wf-aggregation-postgres/src/main/java/eu/dnetlib/wfs/nodes/MdTransformJobNode.java b/dnet-app/apps/dnet-wf-aggregation-postgres/src/main/java/eu/dnetlib/wfs/nodes/MdTransformJobNode.java
index c8f8ff9..42827b4 100644
--- a/dnet-app/apps/dnet-wf-aggregation-postgres/src/main/java/eu/dnetlib/wfs/nodes/MdTransformJobNode.java
+++ b/dnet-app/apps/dnet-wf-aggregation-postgres/src/main/java/eu/dnetlib/wfs/nodes/MdTransformJobNode.java
@@ -73,6 +73,13 @@ public class MdTransformJobNode extends AbstractJobNode {
 
 		try {
 			final Stream<MetadataRecord> stream = this.mdStoreSqlBackend.streamEntries(inputVersion)
+					.map(input -> {
+						final MetadataRecord output = new MetadataRecord();
+						BeanUtils.copyProperties(input, output);
+						output.setBody(xslt.transform(input.getBody()));
+						output.setDateOfTransformation(LocalDateTime.now());
+						return output;
+					})
 					.filter(record -> {
 						try {
 							final Document doc = DocumentHelper.parseText(record.getBody());
@@ -80,13 +87,6 @@ public class MdTransformJobNode extends AbstractJobNode {
 						} catch (final DocumentException e) {
 							throw new RuntimeException("Invalid record: " + record.getBody());
 						}
-					})
-					.map(input -> {
-						final MetadataRecord output = new MetadataRecord();
-						BeanUtils.copyProperties(input, output);
-						output.setBody(xslt.transform(input.getBody()));
-						output.setDateOfTransformation(LocalDateTime.now());
-						return output;
 					});
 
 			this.mdStoreSqlBackend.saveRecords(outputVersion, stream);
diff --git a/dnet-app/apps/dnet-wf-manager/src/main/resources/wf_templates/collect.json b/dnet-app/apps/dnet-wf-manager/src/main/resources/wf_templates/collect.json
index a89d655..2f928e2 100644
--- a/dnet-app/apps/dnet-wf-manager/src/main/resources/wf_templates/collect.json
+++ b/dnet-app/apps/dnet-wf-manager/src/main/resources/wf_templates/collect.json
@@ -17,7 +17,7 @@
 			"required":"false"
 		},
 		{
-			"name":"overrideFromDate",
+			"name":"overrideUntilDate",
 			"description":"Override the default untilDate (now)",
 			"type":"DATE",
 			"required":"false"
@@ -101,19 +101,19 @@
 				},
 				{
 					"name":"mdId",
-					"env":"nativeMdStoreId"
+					"ref":"nativeMdStoreId"
 				},
 				{
 					"name":"filterXpath",
-					"env":"nativeMdStoreFilterXpath"
+					"ref":"nativeMdStoreFilterXpath"
 				},
 				{
 					"name":"overrideFromDate",
-					"env":"overrideFromDate"
+					"ref":"overrideFromDate"
 				},
 				{
 					"name":"overrideUntilDate",
-					"env":"overrideUntilDate"
+					"ref":"overrideUntilDate"
 				}
 			],
 			"output":[],
diff --git a/dnet-app/apps/dnet-wf-manager/src/main/resources/wf_templates/transform.json b/dnet-app/apps/dnet-wf-manager/src/main/resources/wf_templates/transform.json
index a7e1c97..75e7956 100644
--- a/dnet-app/apps/dnet-wf-manager/src/main/resources/wf_templates/transform.json
+++ b/dnet-app/apps/dnet-wf-manager/src/main/resources/wf_templates/transform.json
@@ -11,6 +11,11 @@
 		{
 			"name":"xsltRuleId",
 			"description":"Transformation Rule ID"
+		},
+		{
+			"name":"cleanedMdStoreFilterXpath",
+			"description":"Filter the cleaned records",
+			"required":"false"
 		}
 	],
 	"graph":[
@@ -61,7 +66,7 @@
 				},
 				{
 					"name":"filterXpath",
-					"ref":"nativeMdStoreFilterXpath"
+					"ref":"cleanedMdStoreFilterXpath"
 				}
 			],
 			"output":[],