Implemented cleaning date

dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/transformation/xslt/DateCleaner.java

@@ -0,0 +1,100 @@
+package eu.dnetlib.dhp.transformation.xslt;
+
+import net.sf.saxon.s9api.*;
+import scala.Serializable;
+import java.time.LocalDate;
+import java.time.format.DateTimeFormatter;
+import java.util.*;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+
+public class DateCleaner implements ExtensionFunction, Serializable {
+
+    private final static List<Pattern> dateRegex = Arrays.asList(
+            //Y-M-D
+            Pattern.compile("(18|19|20)\\d\\d([- /.])(0[1-9]|1[012])\\2(0[1-9]|[12][0-9]|3[01])", Pattern.MULTILINE),
+            //M-D-Y
+            Pattern.compile("((0[1-9]|1[012])|([1-9]))([- /.])(0[1-9]|[12][0-9]|3[01])([- /.])(18|19|20)?\\d\\d", Pattern.MULTILINE),
+            //D-M-Y
+            Pattern.compile("(?:(?:31(/|-|\\.)(?:0?[13578]|1[02]|(?:Jan|Mar|May|Jul|Aug|Oct|Dec)))\\1|(?:(?:29|30)(/|-|\\.)(?:0?[1,3-9]|1[0-2]|(?:Jan|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec))\\2))(?:(?:1[6-9]|[2-9]\\d)?\\d{2})|(?:29(/|-|\\.)(?:0?2|(?:Feb))\\3(?:(?:(?:1[6-9]|[2-9]\\d)?(?:0[48]|[2468][048]|[13579][26])|(?:(?:16|[2468][048]|[3579][26])00))))|(?:0?[1-9]|1\\d|2[0-8])(/|-|\\.)(?:(?:0?[1-9]|(?:Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep))|(?:1[0-2]|(?:Oct|Nov|Dec)))\\4(?:(?:1[6-9]|[2-9]\\d)?\\d{2})", Pattern.MULTILINE),
+            //Y
+            Pattern.compile("(19|20)\\d\\d", Pattern.MULTILINE)
+    );
+
+    private final static Pattern incompleteDateRegex = Pattern.compile("^((18|19|20)\\d\\d){1}([- \\\\ \\/](0?[1-9]|1[012]))?", Pattern.MULTILINE);
+
+    private final static List<DateTimeFormatter> dformats = Arrays.asList(
+            DateTimeFormatter.ofPattern("[MM-dd-yyyy][MM/dd/yyyy][dd-MM-yy][dd-MMM-yyyy][dd/MMM/yyyy][dd-MMM-yy][dd/MMM/yy][dd-MM-yy][dd/MM/yy][dd-MM-yyyy][dd/MM/yyyy][yyyy-MM-dd][yyyy/MM/dd]", Locale.ENGLISH),
+            DateTimeFormatter.ofPattern("[dd-MM-yyyy][dd/MM/yyyy]", Locale.ITALIAN)
+    );
+
+    public String clean(final String inputDate) {
+
+        Optional<String> cleanedDate = dateRegex.stream().map(
+                p -> {
+                    final Matcher matcher = p.matcher(inputDate);
+                    if (matcher.find())
+                        return matcher.group(0);
+                    else
+                        return null;
+                }
+        ).filter(Objects::nonNull)
+                .map(m -> {
+                    Optional<String> cleanDate = dformats.stream()
+                            .map(f -> {
+                                        try {
+                                            LocalDate parsedDate = LocalDate.parse(m, f);
+                                            if (parsedDate != null)
+                                                return parsedDate.toString();
+                                            else
+                                                return null;
+                                        } catch (Throwable e) {
+                                            return null;
+                                        }
+                                    }
+
+                            ).filter(Objects::nonNull).findAny();
+
+                    return cleanDate.orElse(null);
+                }).filter(Objects::nonNull).findAny();
+
+        if (cleanedDate.isPresent())
+            return cleanedDate.get();
+
+        final Matcher matcher = incompleteDateRegex.matcher(inputDate);
+        if (matcher.find()){
+            final Integer year = Integer.parseInt(matcher.group(1));
+            final Integer month = Integer.parseInt(matcher.group(4) == null ? "01":matcher.group(4));
+            return String.format("%d-%02d-01",year, month);
+        }
+        return null;
+    }
+
+    @Override
+    public QName getName() {
+        return new QName("http://eu/dnetlib/trasform/dates", "dateISO");
+    }
+
+    @Override
+    public SequenceType getResultType() {
+        return SequenceType.makeSequenceType(ItemType.STRING, OccurrenceIndicator.ZERO_OR_ONE);
+    }
+
+    @Override
+    public SequenceType[] getArgumentTypes() {
+        return new SequenceType[] {
+                SequenceType.makeSequenceType(ItemType.STRING, OccurrenceIndicator.ZERO_OR_ONE)
+        };
+    }
+
+    @Override
+    public XdmValue call(XdmValue[] xdmValues) throws SaxonApiException {
+        XdmValue r = xdmValues[0];
+        if (r.size() == 0) {
+            return new XdmAtomicValue("");
+        }
+        final String currentValue = xdmValues[0].itemAt(0).getStringValue();
+        return new XdmAtomicValue(clean(currentValue));
+    }
+}

dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/transformation/xslt/XSLTTransformationFunction.java

@@ -41,6 +41,7 @@ public class XSLTTransformationFunction implements MapFunction<MetadataRecord, M
 		try {
 			Processor processor = new Processor(false);
 			processor.registerExtensionFunction(cleanFunction);
+			processor.registerExtensionFunction(new DateCleaner());
 			final XsltCompiler comp = processor.newXsltCompiler();
 			XsltExecutable xslt = comp
 				.compile(new StreamSource(new ByteArrayInputStream(transformationRule.getBytes())));

dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/transformation/TransformationJobTest.java

@@ -8,9 +8,12 @@ import java.io.IOException;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.util.Map;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
+import eu.dnetlib.dhp.transformation.xslt.DateCleaner;
 import org.apache.commons.io.IOUtils;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.spark.SparkConf;
@@ -39,10 +42,10 @@ public class TransformationJobTest extends AbstractVocabularyTest {
 
 	@BeforeAll
 	public static void beforeAll() throws IOException, ISLookUpException {
-		SparkConf conf = new SparkConf();
+//		SparkConf conf = new SparkConf();
-		conf.setAppName(CollectionJobTest.class.getSimpleName());
+//		conf.setAppName(CollectionJobTest.class.getSimpleName());
-		conf.setMaster("local");
+//		conf.setMaster("local");
-		spark = SparkSession.builder().config(conf).getOrCreate();
+//		spark = SparkSession.builder().config(conf).getOrCreate();
 	}
 
 	@BeforeEach
@@ -52,9 +55,22 @@ public class TransformationJobTest extends AbstractVocabularyTest {
 
 	@AfterAll
 	public static void afterAll() {
-		spark.stop();
+//		spark.stop();
 	}
 
+
+	@Test
+	@DisplayName("Test Date cleaner")
+	public void testDateCleaner() throws Exception {
+		DateCleaner dc = new DateCleaner();
+		assertEquals(dc.clean("20/09/1982"),"1982-09-20");
+		assertEquals(dc.clean("20-09-2002"),"2002-09-20");
+		assertEquals(dc.clean("2002-09-20"),"2002-09-20");
+		assertEquals(dc.clean("2002-9"),"2002-09-01");
+		assertEquals(dc.clean("2021"),"2021-01-01");
+	}
+
+
 	@Test
 	@DisplayName("Test Transform Single XML using XSLTTransformator")
 	public void testTransformSaxonHE() throws Exception {
@@ -66,12 +82,8 @@ public class TransformationJobTest extends AbstractVocabularyTest {
 		// We Load the XSLT transformation Rule from the classpath
 		XSLTTransformationFunction tr = loadTransformationRule("/eu/dnetlib/dhp/transform/zenodo_tr.xslt");
 
-
 		MetadataRecord result = tr.call(mr);
 
-
-
-
 		// Print the record
 		System.out.println(result.getBody());
 		// TODO Create significant Assert

dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/transform/input_zenodo.xml

@@ -56,7 +56,7 @@
                         <subject>ADCP</subject>
                     </subjects>
                     <dates>
-                        <date dateType="Issued">2019-05-29</date>
+                        <date dateType="Available">2019</date>
                     </dates>
                     <resourceType resourceTypeGeneral="Dataset"/>
                     <relatedIdentifiers>

dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/transform/zenodo_tr.xslt

@@ -4,8 +4,9 @@
         xmlns:oai="http://www.openarchives.org/OAI/2.0/"
         xmlns:oaf="http://namespace.openaire.eu/oaf"
         xmlns:vocabulary="http://eu/dnetlib/trasform/extension"
+        xmlns:dateCleaner="http://eu/dnetlib/trasform/dates"
         xmlns:dr="http://www.driver-repository.eu/namespace/dr"
-        exclude-result-prefixes="xsl vocabulary">
+        exclude-result-prefixes="xsl vocabulary dateCleaner">
     <xsl:param name="varOfficialName"/>
     <xsl:param name="varDsType"/>
     <xsl:param name="varDataSourceId"/>
@@ -53,7 +54,7 @@
 
                 <xsl:if test="//*[local-name()='date']/@dateType='Available'">
                     <xsl:variable name='varEmbargoEndDate'
-                                  select="vocabulary:clean(normalize-space(//*[local-name()='date'][@dateType='Available']), 'DateISO8601')"/>
+                                  select="dateCleaner:dateISO(normalize-space(//*[local-name()='date'][@dateType='Available']))"/>
                     <xsl:choose>
                         <xsl:when test="string-length($varEmbargoEndDate) > 0">
                             <oaf:embargoenddate>
@@ -112,7 +113,7 @@
 
                 <oaf:dateAccepted>
                     <xsl:value-of
-                            select="normalize-space(//*[local-name()='publicationYear'])"/>
+                            select="dateCleaner:dateISO(normalize-space(//*[local-name()='publicationYear']))"/>
                 </oaf:dateAccepted>
                 <xsl:choose>