gCubeSystem
/
grsf-publisher
16
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Improved tests

This commit is contained in:
Luca Frosini 2024-04-05 17:03:53 +02:00
parent 85571c6986
commit ce259b0dac
2 changed files with 62 additions and 44 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
src/test/java/org/gcube/grsf/publisher/ContextTest.java
View File

@ -43,7 +43,6 @@ public class ContextTest {
private static final String ROOT_PROD; private static final String ROOT_PROD;
protected static final Properties properties; protected static final Properties properties;
public static final String TYPE_PROPERTY_KEY = "type"; public static final String TYPE_PROPERTY_KEY = "type";

105
src/test/java/org/gcube/grsf/publisher/record/RecordTest.java
View File

@ -182,13 +182,18 @@ public class RecordTest extends ContextTest {
public void testFisheryRecords() throws Exception { public void testFisheryRecords() throws Exception {
List<String> types = new ArrayList<>(); List<String> types = new ArrayList<>();
types.add(FISHERY); types.add(FISHERY);
int maxTestRecords = Integer.MAX_VALUE;
// maxTestRecords = 1;
int maxTestRecordsPerSource = Integer.MAX_VALUE;
maxTestRecordsPerSource = 1;
boolean restart = false; boolean restart = false;
String restartFromSource = null; String restartFromSource = null;
// String restartFromSource = "firms"; // String restartFromSource = "firms";
// String restartFromSource = "fishsource"; // String restartFromSource = "fishsource";
// String restartFromSource = "grsf"; // String restartFromSource = "grsf";
// String restartFromSource = "sdg";
String restartFromFile = null; String restartFromFile = null;
testRecords(types, restart, restartFromSource, restartFromFile); testRecords(types, restart, restartFromSource, restartFromFile, maxTestRecords, maxTestRecordsPerSource);
} }
@Ignore @Ignore
@ -196,15 +201,20 @@ public class RecordTest extends ContextTest {
public void testStockRecords() throws Exception { public void testStockRecords() throws Exception {
List<String> types = new ArrayList<>(); List<String> types = new ArrayList<>();
types.add(STOCK); types.add(STOCK);
int maxTestRecords = Integer.MAX_VALUE;
maxTestRecords = 1;
int maxTestRecordsPerSource = Integer.MAX_VALUE;
maxTestRecordsPerSource = 1;
boolean restart = false; boolean restart = false;
String restartFromSource = null; String restartFromSource = null;
// String restartFromSource = "firms"; // String restartFromSource = "firms";
// String restartFromSource = "fishsource"; // String restartFromSource = "fishsource";
// String restartFromSource = "grsf"; // String restartFromSource = "grsf";
// String restartFromSource = "ram"; // String restartFromSource = "ram";
// String restartFromSource = "sdg_14_4_1"; // String restartFromSource = "sdg";
restart = restartFromSource!=null ? true : false;
String restartFromFile = null; String restartFromFile = null;
testRecords(types, restart, restartFromSource, restartFromFile); testRecords(types, restart, restartFromSource, restartFromFile, maxTestRecords, maxTestRecordsPerSource);
} }
@Ignore @Ignore
@ -212,40 +222,44 @@ public class RecordTest extends ContextTest {
public void testTraceabilityUnitRecords() throws Exception { public void testTraceabilityUnitRecords() throws Exception {
List<String> types = new ArrayList<>(); List<String> types = new ArrayList<>();
types.add(TRACEABILITY_UNIT); types.add(TRACEABILITY_UNIT);
int maxTestRecords = Integer.MAX_VALUE;
maxTestRecords = 1;
int maxTestRecordsPerSource = Integer.MAX_VALUE;
maxTestRecordsPerSource = 1;
boolean restart = false; boolean restart = false;
String restartFromSource = null; String restartFromSource = null;
// String restartFromSource = "grsf"; // String restartFromSource = "grsf";
restart = restartFromSource!=null ? true : false;
String restartFromFile = null; String restartFromFile = null;
testRecords(types, restart, restartFromSource, restartFromFile); testRecords(types, restart, restartFromSource, restartFromFile, maxTestRecords, maxTestRecordsPerSource);
} }
@Ignore @Ignore
@Test @Test
public void testAll() throws Exception { public void testAll() throws Exception {
ContextTest.setContextByName(VRE_GRSF_PRE);
List<String> types = new ArrayList<>(); List<String> types = new ArrayList<>();
types.add(FISHERY); types.add(FISHERY);
types.add(STOCK); types.add(STOCK);
types.add(TRACEABILITY_UNIT); types.add(TRACEABILITY_UNIT);
int maxTestRecords = Integer.MAX_VALUE;
maxTestRecords = 1;
int maxTestRecordsPerSource = Integer.MAX_VALUE;
maxTestRecordsPerSource = 1;
boolean restart = false; boolean restart = false;
String restartFromSource = null; String restartFromSource = null;
String restartFromFile = null; String restartFromFile = null;
testRecords(types, restart, restartFromSource, restartFromFile); testRecords(types, restart, restartFromSource, restartFromFile, maxTestRecords, maxTestRecordsPerSource);
} }
private void testRecords(List<String> types, boolean restart, String restartFromSource, String restartFromFile) throws Exception { private void testRecords(List<String> types, boolean restart, String restartFromSource, String restartFromFile, int maxTestRecords, int maxTestRecordsPerSource) throws Exception {
int maxTestRecords = Integer.MAX_VALUE;
// maxTestRecords = 1;
int maxTestRecordsPerSource = Integer.MAX_VALUE;
maxTestRecordsPerSource = 5;
/* /*
* Tell if we want test the whole creation process; * Tell if we want test the whole creation process;
* *
* - when true the record is created in Ckan; * - when true the record is created in Ckan;
* - when false the process of applying Freemakrer template and upload resources * - when false the process of applying Freemarker template and upload resources
* in the workspace. All the files uploaded in the will be deleted at the end the test of the record. * in the workspace. All the files uploaded in the workspace will be deleted at the end the test of the record.
* *
*/ */
boolean create = true; boolean create = true;
@ -268,7 +282,8 @@ public class RecordTest extends ContextTest {
Calendar start = Calendar.getInstance(); Calendar start = Calendar.getInstance();
int countRecord = 0; int testedRecords = 0;
int analysedRecords = 0;
File outputDir = new File(getResourcesDirectory(), "output"); File outputDir = new File(getResourcesDirectory(), "output");
outputDir.mkdir(); outputDir.mkdir();
@ -289,8 +304,8 @@ public class RecordTest extends ContextTest {
}; };
for (String type : types) { for (String type : types) {
if(countRecord >= maxTestRecords) { if(testedRecords >= maxTestRecords) {
logger.info("The test has already elaborated {} records which is the max allowed (i.e. allowed {} records)", countRecord, maxTestRecords); logger.info("The test has already elaborated {} records which is the max allowed (i.e. allowed {} records)", testedRecords, maxTestRecords);
return; return;
} }
@ -307,7 +322,7 @@ public class RecordTest extends ContextTest {
// sourceDirectory.add(new File(typeDir, "fishsource")); // sourceDirectory.add(new File(typeDir, "fishsource"));
// sourceDirectory.add(new File(typeDir, "grsf")); // sourceDirectory.add(new File(typeDir, "grsf"));
// sourceDirectory.add(new File(typeDir, "ram")); // sourceDirectory.add(new File(typeDir, "ram"));
// sourceDirectory.add(new File(typeDir, "sdg_14_4_1")); // sourceDirectory.add(new File(typeDir, "sdg"));
for (File source : sourceDirectory) { for (File source : sourceDirectory) {
Calendar startSource = Calendar.getInstance(); Calendar startSource = Calendar.getInstance();
@ -326,8 +341,8 @@ public class RecordTest extends ContextTest {
} }
} }
if(countRecord >= maxTestRecords) { if(testedRecords >= maxTestRecords) {
logger.info("The test has already elaborated {} records which is the max allowed (i.e. allowed {} records)", countRecord, maxTestRecords); logger.info("The test has already elaborated {} records which is the max allowed (i.e. allowed {} records)", testedRecords, maxTestRecords);
return; return;
} }
@ -341,18 +356,20 @@ public class RecordTest extends ContextTest {
List<File> jsonFiles = Arrays.asList(jfs); List<File> jsonFiles = Arrays.asList(jfs);
if(maxTestRecords==Integer.MAX_VALUE && maxTestRecordsPerSource==Integer.MAX_VALUE) { if(restart) {
Collections.sort(jsonFiles); Collections.sort(jsonFiles);
}else{ }else{
Collections.shuffle(jsonFiles); Collections.shuffle(jsonFiles);
} }
int countRecordPerSource = 0; int testedRecordsPerSource = 0;
int analysedRecordsPerSource = 0;
int numberOfRecordsPerSource = jsonFiles.size(); int numberOfRecordsPerSource = jsonFiles.size();
if(maxTestRecordsPerSource!=Integer.MAX_VALUE) { // if(maxTestRecordsPerSource!=Integer.MAX_VALUE) {
numberOfRecordsPerSource = numberOfRecordsPerSource < maxTestRecordsPerSource ? numberOfRecordsPerSource : maxTestRecordsPerSource; // numberOfRecordsPerSource = numberOfRecordsPerSource < maxTestRecordsPerSource ? numberOfRecordsPerSource : maxTestRecordsPerSource;
} // }
for (File jsonFile : jsonFiles) { for (File jsonFile : jsonFiles) {
String expectedGRSFUUID = jsonFile.getName().replace(".json", ""); String expectedGRSFUUID = jsonFile.getName().replace(".json", "");
@ -363,26 +380,26 @@ public class RecordTest extends ContextTest {
logger.info("Found the record to restart i.e. {} {} {}", sourceString, type, jsonFile.getName()); logger.info("Found the record to restart i.e. {} {} {}", sourceString, type, jsonFile.getName());
restartedFromFile = true; restartedFromFile = true;
}else { }else {
logger.info("[{} of {}] {} {} with GRSF UUID {} has been already elaborated", countRecordPerSource + 1, numberOfRecordsPerSource, sourceString, type, expectedGRSFUUID); logger.info("[{} of {}] {} {} with GRSF UUID {} has been already elaborated", analysedRecordsPerSource + 1, numberOfRecordsPerSource, sourceString, type, expectedGRSFUUID);
++countRecord; ++analysedRecords;
++countRecordPerSource; ++analysedRecordsPerSource;
continue; continue;
} }
} }
} }
if(countRecord >= maxTestRecords) { if(testedRecords >= maxTestRecords) {
logger.info("The test has already elaborated {} records which is the max allowed (i.e. allowed {} records)", countRecord, maxTestRecords); logger.info("The test has already elaborated {} records which is the max allowed (i.e. allowed {} records)", testedRecords, maxTestRecords);
return; return;
} }
if(countRecordPerSource >= maxTestRecordsPerSource) { if(testedRecordsPerSource >= maxTestRecordsPerSource) {
logger.info("The test has already elaborated {} records for the source '{}' which is the max allowed (i.e. allowed {} records per source)", countRecordPerSource, sourceString, maxTestRecordsPerSource); logger.info("The test has already elaborated {} records for the source '{}' which is the max allowed (i.e. allowed {} records per source)", testedRecordsPerSource, sourceString, maxTestRecordsPerSource);
break; break;
} }
logger.trace("[{} of {}] Going to elaborate {} {} with GRSF UUID {} from file {}", countRecordPerSource + 1, numberOfRecordsPerSource, sourceString, type, expectedGRSFUUID, jsonFile.getAbsolutePath()); logger.trace("[{} of {}] Going to elaborate {} {} with GRSF UUID {} from file {}", analysedRecordsPerSource + 1, numberOfRecordsPerSource, sourceString, type, expectedGRSFUUID, jsonFile.getAbsolutePath());
logger.info("[{} of {}] Going to elaborate {} {} with GRSF UUID {}", countRecordPerSource + 1, numberOfRecordsPerSource, sourceString, type, expectedGRSFUUID); logger.info("[{} of {}] Going to elaborate {} {} with GRSF UUID {}", analysedRecordsPerSource + 1, numberOfRecordsPerSource, sourceString, type, expectedGRSFUUID);
ObjectMapper mapper = new ObjectMapper(); ObjectMapper mapper = new ObjectMapper();
@ -407,12 +424,14 @@ public class RecordTest extends ContextTest {
ret = record.simulateCreation(mapper.writeValueAsString(input)); ret = record.simulateCreation(mapper.writeValueAsString(input));
} }
++countRecord; ++testedRecords;
++countRecordPerSource; ++testedRecordsPerSource;
++analysedRecords;
++analysedRecordsPerSource;
logger.trace("[{} of {}] {} {} with GRSF UUID {} from file {} {} successfully\n{}", countRecordPerSource, numberOfRecordsPerSource, sourceString, type, grsfUUID, create ? "created" : "elaborated", jsonFile.getAbsolutePath(), ret); logger.trace("[{} of {}] {} {} with GRSF UUID {} from file {} {} successfully\n{}", analysedRecordsPerSource, numberOfRecordsPerSource, sourceString, type, grsfUUID, create ? "created" : "elaborated", jsonFile.getAbsolutePath(), ret);
logger.info("[{} of {}] {} {} with GRSF UUID {} {} successfully", countRecordPerSource, numberOfRecordsPerSource, sourceString, type, grsfUUID, create ? "created" : "elaborated"); logger.info("[{} of {}] {} {} with GRSF UUID {} {} successfully", analysedRecordsPerSource, numberOfRecordsPerSource, sourceString, type, grsfUUID, create ? "created" : "elaborated");
if(create) { if(create) {
long sleepTime = TimeUnit.MILLISECONDS.toMillis(200); long sleepTime = TimeUnit.MILLISECONDS.toMillis(200);
@ -427,7 +446,7 @@ public class RecordTest extends ContextTest {
long diff = endSource.getTimeInMillis() - startSource.getTimeInMillis(); long diff = endSource.getTimeInMillis() - startSource.getTimeInMillis();
logger.info("{} {} {} records in {} milliseconds (~{} seconds)", logger.info("{} {} {} records in {} milliseconds (~{} seconds)",
create?"Created in Ckan " : "Simulated Creation of ", source, type, create?"Created in Ckan " : "Simulated Creation of ", source, type,
countRecordPerSource, diff, TimeUnit.MILLISECONDS.toSeconds(diff)); testedRecordsPerSource, diff, TimeUnit.MILLISECONDS.toSeconds(diff));
} }
} }
@ -435,7 +454,7 @@ public class RecordTest extends ContextTest {
Calendar end = Calendar.getInstance(); Calendar end = Calendar.getInstance();
long diff = end.getTimeInMillis() - start.getTimeInMillis(); long diff = end.getTimeInMillis() - start.getTimeInMillis();
logger.info("Created in Ckan {} records in {} milliseconds (~{} seconds)", logger.info("Created in Ckan {} records in {} milliseconds (~{} seconds)",
countRecord, diff, TimeUnit.MILLISECONDS.toSeconds(diff)); testedRecords, diff, TimeUnit.MILLISECONDS.toSeconds(diff));
} }
// @Ignore // @Ignore