From 263c1beb7e4cf6666a377cc5f0c3517fbae2fd2f Mon Sep 17 00:00:00 2001 From: Enrico Ottonello Date: Tue, 13 Oct 2020 00:31:17 +0200 Subject: [PATCH] fix date parsing on es record creation --- .../dnetlib/ariadneplus/elasticsearch/BulkUpload.java | 2 ++ .../elasticsearch/model/AriadneCatalogEntry.java | 9 +++++++++ .../eu/dnetlib/ariadneplus/reader/ResourceManager.java | 1 + .../ariadneplus/reader/RunSPARQLQueryService.java | 1 + .../eu/dnetlib/ariadneplus/reader/utils/ESUtils.java | 10 +++++++--- .../ariadneplus/GraphDbReaderAndESIndexTest.java | 2 +- 6 files changed, 21 insertions(+), 4 deletions(-) diff --git a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/BulkUpload.java b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/BulkUpload.java index e6012bc..e5a1be1 100644 --- a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/BulkUpload.java +++ b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/BulkUpload.java @@ -57,6 +57,7 @@ public class BulkUpload { try { Object next = manager.next(); AriadneCatalogEntry ace = ((AriadneCatalogEntry) next); +// log.warn("ready to index: "+ace.toJson()); if (isCollection) { ace.setResourceType("collection"); if (ace.getSpatial()==null) { @@ -138,6 +139,7 @@ public class BulkUpload { esResponseCode = -3; } } catch (Exception e) { + e.printStackTrace(); log.error("Indexing "+e.getMessage()); return -1; } diff --git a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/AriadneCatalogEntry.java b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/AriadneCatalogEntry.java index b0b9337..2d802c4 100644 --- a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/AriadneCatalogEntry.java +++ b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/elasticsearch/model/AriadneCatalogEntry.java @@ -32,6 +32,7 @@ public class AriadneCatalogEntry { private String originalId; private List owner; private String resourceType; + private String rdfType; private List scientificResponsible; private List spatial; // private List spatialRegion; @@ -381,4 +382,12 @@ public class AriadneCatalogEntry { public void setAudience(String audience) { this.audience = audience; } + + public String getRdfType() { + return rdfType; + } + + public void setRdfType(String rdfType) { + this.rdfType = rdfType; + } } \ No newline at end of file diff --git a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/reader/ResourceManager.java b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/reader/ResourceManager.java index c55e30b..9c343a7 100644 --- a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/reader/ResourceManager.java +++ b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/reader/ResourceManager.java @@ -91,6 +91,7 @@ public class ResourceManager { if (class_name.equals("Record") || class_name.equals("Collection")) { class_name = "AriadneCatalogEntry"; } + } } if (entry == null) { diff --git a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/reader/RunSPARQLQueryService.java b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/reader/RunSPARQLQueryService.java index 2ce15dd..25767d5 100644 --- a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/reader/RunSPARQLQueryService.java +++ b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/reader/RunSPARQLQueryService.java @@ -153,6 +153,7 @@ public class RunSPARQLQueryService { Rio.write(resultsModel, rdfRecordWriter); parser.setCollection(isCollection); String bufferedRecord = recordWriter.toString(); +// log.debug(bufferedRecord); int size = parser.parse(bufferedRecord); log.debug("json elements: "+size); if (size==-1) { diff --git a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/reader/utils/ESUtils.java b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/reader/utils/ESUtils.java index 61aca6e..bd1a140 100644 --- a/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/reader/utils/ESUtils.java +++ b/dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/reader/utils/ESUtils.java @@ -14,7 +14,7 @@ public class ESUtils { .parseDefaulting(ChronoField.DAY_OF_MONTH, 1) .toFormatter(); - public static String getESFormatDate(String originalDate) { + public static String getESFormatDate(String originalDate) { try{ LocalDate parsedDate = LocalDate.parse(originalDate, elasticSearchDateFormatter); return parsedDate.format(elasticSearchDateFormatter); @@ -23,8 +23,12 @@ public class ESUtils { LocalDate parsedDate = LocalDate.parse(originalDate, originalRecordDateFormatter); return parsedDate.format(elasticSearchDateFormatter); } catch (Exception e1) { - LocalDate parsedDate = LocalDate.parse(originalDate, yearOnlyDateFormatter); - return parsedDate.format(yearOnlyDateFormatter); + try { + LocalDate parsedDate = LocalDate.parse(originalDate.substring(0, 10), elasticSearchDateFormatter); + return parsedDate.format(elasticSearchDateFormatter); + } catch (Exception e2) { + return "0000"; + } } } diff --git a/dnet-ariadneplus-graphdb-publisher/test/java/eu/dnetlib/ariadneplus/GraphDbReaderAndESIndexTest.java b/dnet-ariadneplus-graphdb-publisher/test/java/eu/dnetlib/ariadneplus/GraphDbReaderAndESIndexTest.java index d282bc5..2c24ff5 100644 --- a/dnet-ariadneplus-graphdb-publisher/test/java/eu/dnetlib/ariadneplus/GraphDbReaderAndESIndexTest.java +++ b/dnet-ariadneplus-graphdb-publisher/test/java/eu/dnetlib/ariadneplus/GraphDbReaderAndESIndexTest.java @@ -56,7 +56,7 @@ public class GraphDbReaderAndESIndexTest { runSPQRLQuery.setBulkUpload(bulkUpload); String recordId; final ClassPathResource queryTemplateResource; - boolean testRecord = false; + boolean testRecord = true; if (testRecord) { recordId = "https://ariadne-infrastructure.eu/aocat/Resource/3037F979-F94F-380B-A6A6-3972ED10E61C"; queryTemplateResource = new ClassPathResource("eu/dnetlib/ariadneplus/sparql/read_record_data_template.sparql");