From ce259b0dac2400b44693197ec8662ee956cc8164 Mon Sep 17 00:00:00 2001 From: Luca Frosini Date: Fri, 5 Apr 2024 17:03:53 +0200 Subject: [PATCH] Improved tests --- .../org/gcube/grsf/publisher/ContextTest.java | 1 - .../grsf/publisher/record/RecordTest.java | 105 +++++++++++------- 2 files changed, 62 insertions(+), 44 deletions(-) diff --git a/src/test/java/org/gcube/grsf/publisher/ContextTest.java b/src/test/java/org/gcube/grsf/publisher/ContextTest.java index 79665ed..c19dc73 100644 --- a/src/test/java/org/gcube/grsf/publisher/ContextTest.java +++ b/src/test/java/org/gcube/grsf/publisher/ContextTest.java @@ -43,7 +43,6 @@ public class ContextTest { private static final String ROOT_PROD; - protected static final Properties properties; public static final String TYPE_PROPERTY_KEY = "type"; diff --git a/src/test/java/org/gcube/grsf/publisher/record/RecordTest.java b/src/test/java/org/gcube/grsf/publisher/record/RecordTest.java index 881b482..48086d7 100644 --- a/src/test/java/org/gcube/grsf/publisher/record/RecordTest.java +++ b/src/test/java/org/gcube/grsf/publisher/record/RecordTest.java @@ -182,13 +182,18 @@ public class RecordTest extends ContextTest { public void testFisheryRecords() throws Exception { List types = new ArrayList<>(); types.add(FISHERY); + int maxTestRecords = Integer.MAX_VALUE; +// maxTestRecords = 1; + int maxTestRecordsPerSource = Integer.MAX_VALUE; + maxTestRecordsPerSource = 1; boolean restart = false; String restartFromSource = null; // String restartFromSource = "firms"; // String restartFromSource = "fishsource"; -// String restartFromSource = "grsf"; +// String restartFromSource = "grsf"; +// String restartFromSource = "sdg"; String restartFromFile = null; - testRecords(types, restart, restartFromSource, restartFromFile); + testRecords(types, restart, restartFromSource, restartFromFile, maxTestRecords, maxTestRecordsPerSource); } @Ignore @@ -196,15 +201,20 @@ public class RecordTest extends ContextTest { public void testStockRecords() throws Exception { List types = new ArrayList<>(); types.add(STOCK); + int maxTestRecords = Integer.MAX_VALUE; + maxTestRecords = 1; + int maxTestRecordsPerSource = Integer.MAX_VALUE; + maxTestRecordsPerSource = 1; boolean restart = false; String restartFromSource = null; // String restartFromSource = "firms"; // String restartFromSource = "fishsource"; // String restartFromSource = "grsf"; // String restartFromSource = "ram"; -// String restartFromSource = "sdg_14_4_1"; +// String restartFromSource = "sdg"; + restart = restartFromSource!=null ? true : false; String restartFromFile = null; - testRecords(types, restart, restartFromSource, restartFromFile); + testRecords(types, restart, restartFromSource, restartFromFile, maxTestRecords, maxTestRecordsPerSource); } @Ignore @@ -212,40 +222,44 @@ public class RecordTest extends ContextTest { public void testTraceabilityUnitRecords() throws Exception { List types = new ArrayList<>(); types.add(TRACEABILITY_UNIT); + int maxTestRecords = Integer.MAX_VALUE; + maxTestRecords = 1; + int maxTestRecordsPerSource = Integer.MAX_VALUE; + maxTestRecordsPerSource = 1; boolean restart = false; String restartFromSource = null; -// String restartFromSource = "grsf"; +// String restartFromSource = "grsf"; + restart = restartFromSource!=null ? true : false; String restartFromFile = null; - testRecords(types, restart, restartFromSource, restartFromFile); + testRecords(types, restart, restartFromSource, restartFromFile, maxTestRecords, maxTestRecordsPerSource); } @Ignore @Test public void testAll() throws Exception { + ContextTest.setContextByName(VRE_GRSF_PRE); List types = new ArrayList<>(); types.add(FISHERY); types.add(STOCK); types.add(TRACEABILITY_UNIT); + int maxTestRecords = Integer.MAX_VALUE; + maxTestRecords = 1; + int maxTestRecordsPerSource = Integer.MAX_VALUE; + maxTestRecordsPerSource = 1; boolean restart = false; String restartFromSource = null; String restartFromFile = null; - testRecords(types, restart, restartFromSource, restartFromFile); + testRecords(types, restart, restartFromSource, restartFromFile, maxTestRecords, maxTestRecordsPerSource); } - private void testRecords(List types, boolean restart, String restartFromSource, String restartFromFile) throws Exception { - - int maxTestRecords = Integer.MAX_VALUE; -// maxTestRecords = 1; - - int maxTestRecordsPerSource = Integer.MAX_VALUE; - maxTestRecordsPerSource = 5; - + private void testRecords(List types, boolean restart, String restartFromSource, String restartFromFile, int maxTestRecords, int maxTestRecordsPerSource) throws Exception { + /* * Tell if we want test the whole creation process; * * - when true the record is created in Ckan; - * - when false the process of applying Freemakrer template and upload resources - * in the workspace. All the files uploaded in the will be deleted at the end the test of the record. + * - when false the process of applying Freemarker template and upload resources + * in the workspace. All the files uploaded in the workspace will be deleted at the end the test of the record. * */ boolean create = true; @@ -268,7 +282,8 @@ public class RecordTest extends ContextTest { Calendar start = Calendar.getInstance(); - int countRecord = 0; + int testedRecords = 0; + int analysedRecords = 0; File outputDir = new File(getResourcesDirectory(), "output"); outputDir.mkdir(); @@ -289,8 +304,8 @@ public class RecordTest extends ContextTest { }; for (String type : types) { - if(countRecord >= maxTestRecords) { - logger.info("The test has already elaborated {} records which is the max allowed (i.e. allowed {} records)", countRecord, maxTestRecords); + if(testedRecords >= maxTestRecords) { + logger.info("The test has already elaborated {} records which is the max allowed (i.e. allowed {} records)", testedRecords, maxTestRecords); return; } @@ -307,7 +322,7 @@ public class RecordTest extends ContextTest { // sourceDirectory.add(new File(typeDir, "fishsource")); // sourceDirectory.add(new File(typeDir, "grsf")); // sourceDirectory.add(new File(typeDir, "ram")); -// sourceDirectory.add(new File(typeDir, "sdg_14_4_1")); +// sourceDirectory.add(new File(typeDir, "sdg")); for (File source : sourceDirectory) { Calendar startSource = Calendar.getInstance(); @@ -326,8 +341,8 @@ public class RecordTest extends ContextTest { } } - if(countRecord >= maxTestRecords) { - logger.info("The test has already elaborated {} records which is the max allowed (i.e. allowed {} records)", countRecord, maxTestRecords); + if(testedRecords >= maxTestRecords) { + logger.info("The test has already elaborated {} records which is the max allowed (i.e. allowed {} records)", testedRecords, maxTestRecords); return; } @@ -341,18 +356,20 @@ public class RecordTest extends ContextTest { List jsonFiles = Arrays.asList(jfs); - if(maxTestRecords==Integer.MAX_VALUE && maxTestRecordsPerSource==Integer.MAX_VALUE) { + if(restart) { Collections.sort(jsonFiles); }else{ Collections.shuffle(jsonFiles); } - int countRecordPerSource = 0; + int testedRecordsPerSource = 0; + int analysedRecordsPerSource = 0; + int numberOfRecordsPerSource = jsonFiles.size(); - if(maxTestRecordsPerSource!=Integer.MAX_VALUE) { - numberOfRecordsPerSource = numberOfRecordsPerSource < maxTestRecordsPerSource ? numberOfRecordsPerSource : maxTestRecordsPerSource; - } +// if(maxTestRecordsPerSource!=Integer.MAX_VALUE) { +// numberOfRecordsPerSource = numberOfRecordsPerSource < maxTestRecordsPerSource ? numberOfRecordsPerSource : maxTestRecordsPerSource; +// } for (File jsonFile : jsonFiles) { String expectedGRSFUUID = jsonFile.getName().replace(".json", ""); @@ -363,26 +380,26 @@ public class RecordTest extends ContextTest { logger.info("Found the record to restart i.e. {} {} {}", sourceString, type, jsonFile.getName()); restartedFromFile = true; }else { - logger.info("[{} of {}] {} {} with GRSF UUID {} has been already elaborated", countRecordPerSource + 1, numberOfRecordsPerSource, sourceString, type, expectedGRSFUUID); - ++countRecord; - ++countRecordPerSource; + logger.info("[{} of {}] {} {} with GRSF UUID {} has been already elaborated", analysedRecordsPerSource + 1, numberOfRecordsPerSource, sourceString, type, expectedGRSFUUID); + ++analysedRecords; + ++analysedRecordsPerSource; continue; } } } - if(countRecord >= maxTestRecords) { - logger.info("The test has already elaborated {} records which is the max allowed (i.e. allowed {} records)", countRecord, maxTestRecords); + if(testedRecords >= maxTestRecords) { + logger.info("The test has already elaborated {} records which is the max allowed (i.e. allowed {} records)", testedRecords, maxTestRecords); return; } - if(countRecordPerSource >= maxTestRecordsPerSource) { - logger.info("The test has already elaborated {} records for the source '{}' which is the max allowed (i.e. allowed {} records per source)", countRecordPerSource, sourceString, maxTestRecordsPerSource); + if(testedRecordsPerSource >= maxTestRecordsPerSource) { + logger.info("The test has already elaborated {} records for the source '{}' which is the max allowed (i.e. allowed {} records per source)", testedRecordsPerSource, sourceString, maxTestRecordsPerSource); break; } - logger.trace("[{} of {}] Going to elaborate {} {} with GRSF UUID {} from file {}", countRecordPerSource + 1, numberOfRecordsPerSource, sourceString, type, expectedGRSFUUID, jsonFile.getAbsolutePath()); - logger.info("[{} of {}] Going to elaborate {} {} with GRSF UUID {}", countRecordPerSource + 1, numberOfRecordsPerSource, sourceString, type, expectedGRSFUUID); + logger.trace("[{} of {}] Going to elaborate {} {} with GRSF UUID {} from file {}", analysedRecordsPerSource + 1, numberOfRecordsPerSource, sourceString, type, expectedGRSFUUID, jsonFile.getAbsolutePath()); + logger.info("[{} of {}] Going to elaborate {} {} with GRSF UUID {}", analysedRecordsPerSource + 1, numberOfRecordsPerSource, sourceString, type, expectedGRSFUUID); ObjectMapper mapper = new ObjectMapper(); @@ -407,12 +424,14 @@ public class RecordTest extends ContextTest { ret = record.simulateCreation(mapper.writeValueAsString(input)); } - ++countRecord; - ++countRecordPerSource; + ++testedRecords; + ++testedRecordsPerSource; + ++analysedRecords; + ++analysedRecordsPerSource; - logger.trace("[{} of {}] {} {} with GRSF UUID {} from file {} {} successfully\n{}", countRecordPerSource, numberOfRecordsPerSource, sourceString, type, grsfUUID, create ? "created" : "elaborated", jsonFile.getAbsolutePath(), ret); + logger.trace("[{} of {}] {} {} with GRSF UUID {} from file {} {} successfully\n{}", analysedRecordsPerSource, numberOfRecordsPerSource, sourceString, type, grsfUUID, create ? "created" : "elaborated", jsonFile.getAbsolutePath(), ret); - logger.info("[{} of {}] {} {} with GRSF UUID {} {} successfully", countRecordPerSource, numberOfRecordsPerSource, sourceString, type, grsfUUID, create ? "created" : "elaborated"); + logger.info("[{} of {}] {} {} with GRSF UUID {} {} successfully", analysedRecordsPerSource, numberOfRecordsPerSource, sourceString, type, grsfUUID, create ? "created" : "elaborated"); if(create) { long sleepTime = TimeUnit.MILLISECONDS.toMillis(200); @@ -427,7 +446,7 @@ public class RecordTest extends ContextTest { long diff = endSource.getTimeInMillis() - startSource.getTimeInMillis(); logger.info("{} {} {} records in {} milliseconds (~{} seconds)", create?"Created in Ckan " : "Simulated Creation of ", source, type, - countRecordPerSource, diff, TimeUnit.MILLISECONDS.toSeconds(diff)); + testedRecordsPerSource, diff, TimeUnit.MILLISECONDS.toSeconds(diff)); } } @@ -435,7 +454,7 @@ public class RecordTest extends ContextTest { Calendar end = Calendar.getInstance(); long diff = end.getTimeInMillis() - start.getTimeInMillis(); logger.info("Created in Ckan {} records in {} milliseconds (~{} seconds)", - countRecord, diff, TimeUnit.MILLISECONDS.toSeconds(diff)); + testedRecords, diff, TimeUnit.MILLISECONDS.toSeconds(diff)); } // @Ignore