Improved tests
This commit is contained in:
parent
85571c6986
commit
ce259b0dac
|
@ -43,7 +43,6 @@ public class ContextTest {
|
|||
|
||||
private static final String ROOT_PROD;
|
||||
|
||||
|
||||
protected static final Properties properties;
|
||||
|
||||
public static final String TYPE_PROPERTY_KEY = "type";
|
||||
|
|
|
@ -182,13 +182,18 @@ public class RecordTest extends ContextTest {
|
|||
public void testFisheryRecords() throws Exception {
|
||||
List<String> types = new ArrayList<>();
|
||||
types.add(FISHERY);
|
||||
int maxTestRecords = Integer.MAX_VALUE;
|
||||
// maxTestRecords = 1;
|
||||
int maxTestRecordsPerSource = Integer.MAX_VALUE;
|
||||
maxTestRecordsPerSource = 1;
|
||||
boolean restart = false;
|
||||
String restartFromSource = null;
|
||||
// String restartFromSource = "firms";
|
||||
// String restartFromSource = "fishsource";
|
||||
// String restartFromSource = "grsf";
|
||||
// String restartFromSource = "grsf";
|
||||
// String restartFromSource = "sdg";
|
||||
String restartFromFile = null;
|
||||
testRecords(types, restart, restartFromSource, restartFromFile);
|
||||
testRecords(types, restart, restartFromSource, restartFromFile, maxTestRecords, maxTestRecordsPerSource);
|
||||
}
|
||||
|
||||
@Ignore
|
||||
|
@ -196,15 +201,20 @@ public class RecordTest extends ContextTest {
|
|||
public void testStockRecords() throws Exception {
|
||||
List<String> types = new ArrayList<>();
|
||||
types.add(STOCK);
|
||||
int maxTestRecords = Integer.MAX_VALUE;
|
||||
maxTestRecords = 1;
|
||||
int maxTestRecordsPerSource = Integer.MAX_VALUE;
|
||||
maxTestRecordsPerSource = 1;
|
||||
boolean restart = false;
|
||||
String restartFromSource = null;
|
||||
// String restartFromSource = "firms";
|
||||
// String restartFromSource = "fishsource";
|
||||
// String restartFromSource = "grsf";
|
||||
// String restartFromSource = "ram";
|
||||
// String restartFromSource = "sdg_14_4_1";
|
||||
// String restartFromSource = "sdg";
|
||||
restart = restartFromSource!=null ? true : false;
|
||||
String restartFromFile = null;
|
||||
testRecords(types, restart, restartFromSource, restartFromFile);
|
||||
testRecords(types, restart, restartFromSource, restartFromFile, maxTestRecords, maxTestRecordsPerSource);
|
||||
}
|
||||
|
||||
@Ignore
|
||||
|
@ -212,40 +222,44 @@ public class RecordTest extends ContextTest {
|
|||
public void testTraceabilityUnitRecords() throws Exception {
|
||||
List<String> types = new ArrayList<>();
|
||||
types.add(TRACEABILITY_UNIT);
|
||||
int maxTestRecords = Integer.MAX_VALUE;
|
||||
maxTestRecords = 1;
|
||||
int maxTestRecordsPerSource = Integer.MAX_VALUE;
|
||||
maxTestRecordsPerSource = 1;
|
||||
boolean restart = false;
|
||||
String restartFromSource = null;
|
||||
// String restartFromSource = "grsf";
|
||||
// String restartFromSource = "grsf";
|
||||
restart = restartFromSource!=null ? true : false;
|
||||
String restartFromFile = null;
|
||||
testRecords(types, restart, restartFromSource, restartFromFile);
|
||||
testRecords(types, restart, restartFromSource, restartFromFile, maxTestRecords, maxTestRecordsPerSource);
|
||||
}
|
||||
|
||||
@Ignore
|
||||
@Test
|
||||
public void testAll() throws Exception {
|
||||
ContextTest.setContextByName(VRE_GRSF_PRE);
|
||||
List<String> types = new ArrayList<>();
|
||||
types.add(FISHERY);
|
||||
types.add(STOCK);
|
||||
types.add(TRACEABILITY_UNIT);
|
||||
int maxTestRecords = Integer.MAX_VALUE;
|
||||
maxTestRecords = 1;
|
||||
int maxTestRecordsPerSource = Integer.MAX_VALUE;
|
||||
maxTestRecordsPerSource = 1;
|
||||
boolean restart = false;
|
||||
String restartFromSource = null;
|
||||
String restartFromFile = null;
|
||||
testRecords(types, restart, restartFromSource, restartFromFile);
|
||||
testRecords(types, restart, restartFromSource, restartFromFile, maxTestRecords, maxTestRecordsPerSource);
|
||||
}
|
||||
|
||||
private void testRecords(List<String> types, boolean restart, String restartFromSource, String restartFromFile) throws Exception {
|
||||
|
||||
int maxTestRecords = Integer.MAX_VALUE;
|
||||
// maxTestRecords = 1;
|
||||
|
||||
int maxTestRecordsPerSource = Integer.MAX_VALUE;
|
||||
maxTestRecordsPerSource = 5;
|
||||
|
||||
private void testRecords(List<String> types, boolean restart, String restartFromSource, String restartFromFile, int maxTestRecords, int maxTestRecordsPerSource) throws Exception {
|
||||
|
||||
/*
|
||||
* Tell if we want test the whole creation process;
|
||||
*
|
||||
* - when true the record is created in Ckan;
|
||||
* - when false the process of applying Freemakrer template and upload resources
|
||||
* in the workspace. All the files uploaded in the will be deleted at the end the test of the record.
|
||||
* - when false the process of applying Freemarker template and upload resources
|
||||
* in the workspace. All the files uploaded in the workspace will be deleted at the end the test of the record.
|
||||
*
|
||||
*/
|
||||
boolean create = true;
|
||||
|
@ -268,7 +282,8 @@ public class RecordTest extends ContextTest {
|
|||
|
||||
Calendar start = Calendar.getInstance();
|
||||
|
||||
int countRecord = 0;
|
||||
int testedRecords = 0;
|
||||
int analysedRecords = 0;
|
||||
|
||||
File outputDir = new File(getResourcesDirectory(), "output");
|
||||
outputDir.mkdir();
|
||||
|
@ -289,8 +304,8 @@ public class RecordTest extends ContextTest {
|
|||
};
|
||||
|
||||
for (String type : types) {
|
||||
if(countRecord >= maxTestRecords) {
|
||||
logger.info("The test has already elaborated {} records which is the max allowed (i.e. allowed {} records)", countRecord, maxTestRecords);
|
||||
if(testedRecords >= maxTestRecords) {
|
||||
logger.info("The test has already elaborated {} records which is the max allowed (i.e. allowed {} records)", testedRecords, maxTestRecords);
|
||||
return;
|
||||
}
|
||||
|
||||
|
@ -307,7 +322,7 @@ public class RecordTest extends ContextTest {
|
|||
// sourceDirectory.add(new File(typeDir, "fishsource"));
|
||||
// sourceDirectory.add(new File(typeDir, "grsf"));
|
||||
// sourceDirectory.add(new File(typeDir, "ram"));
|
||||
// sourceDirectory.add(new File(typeDir, "sdg_14_4_1"));
|
||||
// sourceDirectory.add(new File(typeDir, "sdg"));
|
||||
|
||||
for (File source : sourceDirectory) {
|
||||
Calendar startSource = Calendar.getInstance();
|
||||
|
@ -326,8 +341,8 @@ public class RecordTest extends ContextTest {
|
|||
}
|
||||
}
|
||||
|
||||
if(countRecord >= maxTestRecords) {
|
||||
logger.info("The test has already elaborated {} records which is the max allowed (i.e. allowed {} records)", countRecord, maxTestRecords);
|
||||
if(testedRecords >= maxTestRecords) {
|
||||
logger.info("The test has already elaborated {} records which is the max allowed (i.e. allowed {} records)", testedRecords, maxTestRecords);
|
||||
return;
|
||||
}
|
||||
|
||||
|
@ -341,18 +356,20 @@ public class RecordTest extends ContextTest {
|
|||
|
||||
List<File> jsonFiles = Arrays.asList(jfs);
|
||||
|
||||
if(maxTestRecords==Integer.MAX_VALUE && maxTestRecordsPerSource==Integer.MAX_VALUE) {
|
||||
if(restart) {
|
||||
Collections.sort(jsonFiles);
|
||||
}else{
|
||||
Collections.shuffle(jsonFiles);
|
||||
}
|
||||
|
||||
int countRecordPerSource = 0;
|
||||
int testedRecordsPerSource = 0;
|
||||
int analysedRecordsPerSource = 0;
|
||||
|
||||
int numberOfRecordsPerSource = jsonFiles.size();
|
||||
|
||||
if(maxTestRecordsPerSource!=Integer.MAX_VALUE) {
|
||||
numberOfRecordsPerSource = numberOfRecordsPerSource < maxTestRecordsPerSource ? numberOfRecordsPerSource : maxTestRecordsPerSource;
|
||||
}
|
||||
// if(maxTestRecordsPerSource!=Integer.MAX_VALUE) {
|
||||
// numberOfRecordsPerSource = numberOfRecordsPerSource < maxTestRecordsPerSource ? numberOfRecordsPerSource : maxTestRecordsPerSource;
|
||||
// }
|
||||
|
||||
for (File jsonFile : jsonFiles) {
|
||||
String expectedGRSFUUID = jsonFile.getName().replace(".json", "");
|
||||
|
@ -363,26 +380,26 @@ public class RecordTest extends ContextTest {
|
|||
logger.info("Found the record to restart i.e. {} {} {}", sourceString, type, jsonFile.getName());
|
||||
restartedFromFile = true;
|
||||
}else {
|
||||
logger.info("[{} of {}] {} {} with GRSF UUID {} has been already elaborated", countRecordPerSource + 1, numberOfRecordsPerSource, sourceString, type, expectedGRSFUUID);
|
||||
++countRecord;
|
||||
++countRecordPerSource;
|
||||
logger.info("[{} of {}] {} {} with GRSF UUID {} has been already elaborated", analysedRecordsPerSource + 1, numberOfRecordsPerSource, sourceString, type, expectedGRSFUUID);
|
||||
++analysedRecords;
|
||||
++analysedRecordsPerSource;
|
||||
continue;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if(countRecord >= maxTestRecords) {
|
||||
logger.info("The test has already elaborated {} records which is the max allowed (i.e. allowed {} records)", countRecord, maxTestRecords);
|
||||
if(testedRecords >= maxTestRecords) {
|
||||
logger.info("The test has already elaborated {} records which is the max allowed (i.e. allowed {} records)", testedRecords, maxTestRecords);
|
||||
return;
|
||||
}
|
||||
|
||||
if(countRecordPerSource >= maxTestRecordsPerSource) {
|
||||
logger.info("The test has already elaborated {} records for the source '{}' which is the max allowed (i.e. allowed {} records per source)", countRecordPerSource, sourceString, maxTestRecordsPerSource);
|
||||
if(testedRecordsPerSource >= maxTestRecordsPerSource) {
|
||||
logger.info("The test has already elaborated {} records for the source '{}' which is the max allowed (i.e. allowed {} records per source)", testedRecordsPerSource, sourceString, maxTestRecordsPerSource);
|
||||
break;
|
||||
}
|
||||
|
||||
logger.trace("[{} of {}] Going to elaborate {} {} with GRSF UUID {} from file {}", countRecordPerSource + 1, numberOfRecordsPerSource, sourceString, type, expectedGRSFUUID, jsonFile.getAbsolutePath());
|
||||
logger.info("[{} of {}] Going to elaborate {} {} with GRSF UUID {}", countRecordPerSource + 1, numberOfRecordsPerSource, sourceString, type, expectedGRSFUUID);
|
||||
logger.trace("[{} of {}] Going to elaborate {} {} with GRSF UUID {} from file {}", analysedRecordsPerSource + 1, numberOfRecordsPerSource, sourceString, type, expectedGRSFUUID, jsonFile.getAbsolutePath());
|
||||
logger.info("[{} of {}] Going to elaborate {} {} with GRSF UUID {}", analysedRecordsPerSource + 1, numberOfRecordsPerSource, sourceString, type, expectedGRSFUUID);
|
||||
|
||||
|
||||
ObjectMapper mapper = new ObjectMapper();
|
||||
|
@ -407,12 +424,14 @@ public class RecordTest extends ContextTest {
|
|||
ret = record.simulateCreation(mapper.writeValueAsString(input));
|
||||
}
|
||||
|
||||
++countRecord;
|
||||
++countRecordPerSource;
|
||||
++testedRecords;
|
||||
++testedRecordsPerSource;
|
||||
++analysedRecords;
|
||||
++analysedRecordsPerSource;
|
||||
|
||||
logger.trace("[{} of {}] {} {} with GRSF UUID {} from file {} {} successfully\n{}", countRecordPerSource, numberOfRecordsPerSource, sourceString, type, grsfUUID, create ? "created" : "elaborated", jsonFile.getAbsolutePath(), ret);
|
||||
logger.trace("[{} of {}] {} {} with GRSF UUID {} from file {} {} successfully\n{}", analysedRecordsPerSource, numberOfRecordsPerSource, sourceString, type, grsfUUID, create ? "created" : "elaborated", jsonFile.getAbsolutePath(), ret);
|
||||
|
||||
logger.info("[{} of {}] {} {} with GRSF UUID {} {} successfully", countRecordPerSource, numberOfRecordsPerSource, sourceString, type, grsfUUID, create ? "created" : "elaborated");
|
||||
logger.info("[{} of {}] {} {} with GRSF UUID {} {} successfully", analysedRecordsPerSource, numberOfRecordsPerSource, sourceString, type, grsfUUID, create ? "created" : "elaborated");
|
||||
|
||||
if(create) {
|
||||
long sleepTime = TimeUnit.MILLISECONDS.toMillis(200);
|
||||
|
@ -427,7 +446,7 @@ public class RecordTest extends ContextTest {
|
|||
long diff = endSource.getTimeInMillis() - startSource.getTimeInMillis();
|
||||
logger.info("{} {} {} records in {} milliseconds (~{} seconds)",
|
||||
create?"Created in Ckan " : "Simulated Creation of ", source, type,
|
||||
countRecordPerSource, diff, TimeUnit.MILLISECONDS.toSeconds(diff));
|
||||
testedRecordsPerSource, diff, TimeUnit.MILLISECONDS.toSeconds(diff));
|
||||
|
||||
}
|
||||
}
|
||||
|
@ -435,7 +454,7 @@ public class RecordTest extends ContextTest {
|
|||
Calendar end = Calendar.getInstance();
|
||||
long diff = end.getTimeInMillis() - start.getTimeInMillis();
|
||||
logger.info("Created in Ckan {} records in {} milliseconds (~{} seconds)",
|
||||
countRecord, diff, TimeUnit.MILLISECONDS.toSeconds(diff));
|
||||
testedRecords, diff, TimeUnit.MILLISECONDS.toSeconds(diff));
|
||||
}
|
||||
|
||||
// @Ignore
|
||||
|
|
Loading…
Reference in New Issue