gCubeSystem
/
grsf-publisher
16
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Improved tests

This commit is contained in:
Luca Frosini 2024-04-05 17:03:53 +02:00
parent 85571c6986
commit ce259b0dac
2 changed files with 62 additions and 44 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
src/test/java/org/gcube/grsf/publisher/ContextTest.java
View File

@ -43,7 +43,6 @@ public class ContextTest {
private static final String ROOT_PROD;
protected static final Properties properties;
public static final String TYPE_PROPERTY_KEY = "type";

105
src/test/java/org/gcube/grsf/publisher/record/RecordTest.java
View File

@ -182,13 +182,18 @@ public class RecordTest extends ContextTest {
public void testFisheryRecords() throws Exception {
List<String> types = new ArrayList<>();
types.add(FISHERY);
int maxTestRecords = Integer.MAX_VALUE;
// maxTestRecords = 1;
int maxTestRecordsPerSource = Integer.MAX_VALUE;
maxTestRecordsPerSource = 1;
boolean restart = false;
String restartFromSource = null;
// String restartFromSource = "firms";
// String restartFromSource = "fishsource";
// String restartFromSource = "grsf";
// String restartFromSource = "grsf";
// String restartFromSource = "sdg";
String restartFromFile = null;
testRecords(types, restart, restartFromSource, restartFromFile);
testRecords(types, restart, restartFromSource, restartFromFile, maxTestRecords, maxTestRecordsPerSource);
}
@Ignore
@ -196,15 +201,20 @@ public class RecordTest extends ContextTest {
public void testStockRecords() throws Exception {
List<String> types = new ArrayList<>();
types.add(STOCK);
int maxTestRecords = Integer.MAX_VALUE;
maxTestRecords = 1;
int maxTestRecordsPerSource = Integer.MAX_VALUE;
maxTestRecordsPerSource = 1;
boolean restart = false;
String restartFromSource = null;
// String restartFromSource = "firms";
// String restartFromSource = "fishsource";
// String restartFromSource = "grsf";
// String restartFromSource = "ram";
// String restartFromSource = "sdg_14_4_1";
// String restartFromSource = "sdg";
restart = restartFromSource!=null ? true : false;
String restartFromFile = null;
testRecords(types, restart, restartFromSource, restartFromFile);
testRecords(types, restart, restartFromSource, restartFromFile, maxTestRecords, maxTestRecordsPerSource);
}
@Ignore
@ -212,40 +222,44 @@ public class RecordTest extends ContextTest {
public void testTraceabilityUnitRecords() throws Exception {
List<String> types = new ArrayList<>();
types.add(TRACEABILITY_UNIT);
int maxTestRecords = Integer.MAX_VALUE;
maxTestRecords = 1;
int maxTestRecordsPerSource = Integer.MAX_VALUE;
maxTestRecordsPerSource = 1;
boolean restart = false;
String restartFromSource = null;
// String restartFromSource = "grsf";
// String restartFromSource = "grsf";
restart = restartFromSource!=null ? true : false;
String restartFromFile = null;
testRecords(types, restart, restartFromSource, restartFromFile);
testRecords(types, restart, restartFromSource, restartFromFile, maxTestRecords, maxTestRecordsPerSource);
}
@Ignore
@Test
public void testAll() throws Exception {
ContextTest.setContextByName(VRE_GRSF_PRE);
List<String> types = new ArrayList<>();
types.add(FISHERY);
types.add(STOCK);
types.add(TRACEABILITY_UNIT);
int maxTestRecords = Integer.MAX_VALUE;
maxTestRecords = 1;
int maxTestRecordsPerSource = Integer.MAX_VALUE;
maxTestRecordsPerSource = 1;
boolean restart = false;
String restartFromSource = null;
String restartFromFile = null;
testRecords(types, restart, restartFromSource, restartFromFile);
testRecords(types, restart, restartFromSource, restartFromFile, maxTestRecords, maxTestRecordsPerSource);
}
private void testRecords(List<String> types, boolean restart, String restartFromSource, String restartFromFile) throws Exception {
int maxTestRecords = Integer.MAX_VALUE;
// maxTestRecords = 1;
int maxTestRecordsPerSource = Integer.MAX_VALUE;
maxTestRecordsPerSource = 5;
private void testRecords(List<String> types, boolean restart, String restartFromSource, String restartFromFile, int maxTestRecords, int maxTestRecordsPerSource) throws Exception {
/*
* Tell if we want test the whole creation process;
*
* - when true the record is created in Ckan;
* - when false the process of applying Freemakrer template and upload resources
* in the workspace. All the files uploaded in the will be deleted at the end the test of the record.
* - when false the process of applying Freemarker template and upload resources
* in the workspace. All the files uploaded in the workspace will be deleted at the end the test of the record.
*
*/
boolean create = true;
@ -268,7 +282,8 @@ public class RecordTest extends ContextTest {
Calendar start = Calendar.getInstance();
int countRecord = 0;
int testedRecords = 0;
int analysedRecords = 0;
File outputDir = new File(getResourcesDirectory(), "output");
outputDir.mkdir();
@ -289,8 +304,8 @@ public class RecordTest extends ContextTest {
};
for (String type : types) {
if(countRecord >= maxTestRecords) {
logger.info("The test has already elaborated {} records which is the max allowed (i.e. allowed {} records)", countRecord, maxTestRecords);
if(testedRecords >= maxTestRecords) {
logger.info("The test has already elaborated {} records which is the max allowed (i.e. allowed {} records)", testedRecords, maxTestRecords);
return;
}
@ -307,7 +322,7 @@ public class RecordTest extends ContextTest {
// sourceDirectory.add(new File(typeDir, "fishsource"));
// sourceDirectory.add(new File(typeDir, "grsf"));
// sourceDirectory.add(new File(typeDir, "ram"));
// sourceDirectory.add(new File(typeDir, "sdg_14_4_1"));
// sourceDirectory.add(new File(typeDir, "sdg"));
for (File source : sourceDirectory) {
Calendar startSource = Calendar.getInstance();
@ -326,8 +341,8 @@ public class RecordTest extends ContextTest {
}
}
if(countRecord >= maxTestRecords) {
logger.info("The test has already elaborated {} records which is the max allowed (i.e. allowed {} records)", countRecord, maxTestRecords);
if(testedRecords >= maxTestRecords) {
logger.info("The test has already elaborated {} records which is the max allowed (i.e. allowed {} records)", testedRecords, maxTestRecords);
return;
}
@ -341,18 +356,20 @@ public class RecordTest extends ContextTest {
List<File> jsonFiles = Arrays.asList(jfs);
if(maxTestRecords==Integer.MAX_VALUE && maxTestRecordsPerSource==Integer.MAX_VALUE) {
if(restart) {
Collections.sort(jsonFiles);
}else{
Collections.shuffle(jsonFiles);
}
int countRecordPerSource = 0;
int testedRecordsPerSource = 0;
int analysedRecordsPerSource = 0;
int numberOfRecordsPerSource = jsonFiles.size();
if(maxTestRecordsPerSource!=Integer.MAX_VALUE) {
numberOfRecordsPerSource = numberOfRecordsPerSource < maxTestRecordsPerSource ? numberOfRecordsPerSource : maxTestRecordsPerSource;
}
// if(maxTestRecordsPerSource!=Integer.MAX_VALUE) {
// numberOfRecordsPerSource = numberOfRecordsPerSource < maxTestRecordsPerSource ? numberOfRecordsPerSource : maxTestRecordsPerSource;
// }
for (File jsonFile : jsonFiles) {
String expectedGRSFUUID = jsonFile.getName().replace(".json", "");
@ -363,26 +380,26 @@ public class RecordTest extends ContextTest {
logger.info("Found the record to restart i.e. {} {} {}", sourceString, type, jsonFile.getName());
restartedFromFile = true;
}else {
logger.info("[{} of {}] {} {} with GRSF UUID {} has been already elaborated", countRecordPerSource + 1, numberOfRecordsPerSource, sourceString, type, expectedGRSFUUID);
++countRecord;
++countRecordPerSource;
logger.info("[{} of {}] {} {} with GRSF UUID {} has been already elaborated", analysedRecordsPerSource + 1, numberOfRecordsPerSource, sourceString, type, expectedGRSFUUID);
++analysedRecords;
++analysedRecordsPerSource;
continue;
}
}
}
if(countRecord >= maxTestRecords) {
logger.info("The test has already elaborated {} records which is the max allowed (i.e. allowed {} records)", countRecord, maxTestRecords);
if(testedRecords >= maxTestRecords) {
logger.info("The test has already elaborated {} records which is the max allowed (i.e. allowed {} records)", testedRecords, maxTestRecords);
return;
}
if(countRecordPerSource >= maxTestRecordsPerSource) {
logger.info("The test has already elaborated {} records for the source '{}' which is the max allowed (i.e. allowed {} records per source)", countRecordPerSource, sourceString, maxTestRecordsPerSource);
if(testedRecordsPerSource >= maxTestRecordsPerSource) {
logger.info("The test has already elaborated {} records for the source '{}' which is the max allowed (i.e. allowed {} records per source)", testedRecordsPerSource, sourceString, maxTestRecordsPerSource);
break;
}
logger.trace("[{} of {}] Going to elaborate {} {} with GRSF UUID {} from file {}", countRecordPerSource + 1, numberOfRecordsPerSource, sourceString, type, expectedGRSFUUID, jsonFile.getAbsolutePath());
logger.info("[{} of {}] Going to elaborate {} {} with GRSF UUID {}", countRecordPerSource + 1, numberOfRecordsPerSource, sourceString, type, expectedGRSFUUID);
logger.trace("[{} of {}] Going to elaborate {} {} with GRSF UUID {} from file {}", analysedRecordsPerSource + 1, numberOfRecordsPerSource, sourceString, type, expectedGRSFUUID, jsonFile.getAbsolutePath());
logger.info("[{} of {}] Going to elaborate {} {} with GRSF UUID {}", analysedRecordsPerSource + 1, numberOfRecordsPerSource, sourceString, type, expectedGRSFUUID);
ObjectMapper mapper = new ObjectMapper();
@ -407,12 +424,14 @@ public class RecordTest extends ContextTest {
ret = record.simulateCreation(mapper.writeValueAsString(input));
}
++countRecord;
++countRecordPerSource;
++testedRecords;
++testedRecordsPerSource;
++analysedRecords;
++analysedRecordsPerSource;
logger.trace("[{} of {}] {} {} with GRSF UUID {} from file {} {} successfully\n{}", countRecordPerSource, numberOfRecordsPerSource, sourceString, type, grsfUUID, create ? "created" : "elaborated", jsonFile.getAbsolutePath(), ret);
logger.trace("[{} of {}] {} {} with GRSF UUID {} from file {} {} successfully\n{}", analysedRecordsPerSource, numberOfRecordsPerSource, sourceString, type, grsfUUID, create ? "created" : "elaborated", jsonFile.getAbsolutePath(), ret);
logger.info("[{} of {}] {} {} with GRSF UUID {} {} successfully", countRecordPerSource, numberOfRecordsPerSource, sourceString, type, grsfUUID, create ? "created" : "elaborated");
logger.info("[{} of {}] {} {} with GRSF UUID {} {} successfully", analysedRecordsPerSource, numberOfRecordsPerSource, sourceString, type, grsfUUID, create ? "created" : "elaborated");
if(create) {
long sleepTime = TimeUnit.MILLISECONDS.toMillis(200);
@ -427,7 +446,7 @@ public class RecordTest extends ContextTest {
long diff = endSource.getTimeInMillis() - startSource.getTimeInMillis();
logger.info("{} {} {} records in {} milliseconds (~{} seconds)",
create?"Created in Ckan " : "Simulated Creation of ", source, type,
countRecordPerSource, diff, TimeUnit.MILLISECONDS.toSeconds(diff));
testedRecordsPerSource, diff, TimeUnit.MILLISECONDS.toSeconds(diff));
}
}
@ -435,7 +454,7 @@ public class RecordTest extends ContextTest {
Calendar end = Calendar.getInstance();
long diff = end.getTimeInMillis() - start.getTimeInMillis();
logger.info("Created in Ckan {} records in {} milliseconds (~{} seconds)",
countRecord, diff, TimeUnit.MILLISECONDS.toSeconds(diff));
testedRecords, diff, TimeUnit.MILLISECONDS.toSeconds(diff));
}
// @Ignore