526 lines
16 KiB
Java
526 lines
16 KiB
Java
package org.gcube.grsf.publisher.record;
|
||
|
||
import java.io.File;
|
||
import java.io.FilenameFilter;
|
||
import java.net.URI;
|
||
import java.net.URL;
|
||
import java.util.ArrayList;
|
||
import java.util.Arrays;
|
||
import java.util.Calendar;
|
||
import java.util.Collections;
|
||
import java.util.List;
|
||
import java.util.concurrent.TimeUnit;
|
||
|
||
import javax.ws.rs.core.MultivaluedHashMap;
|
||
import javax.ws.rs.core.MultivaluedMap;
|
||
import javax.ws.rs.core.PathSegment;
|
||
import javax.ws.rs.core.UriBuilder;
|
||
import javax.ws.rs.core.UriInfo;
|
||
|
||
import org.gcube.com.fasterxml.jackson.databind.ObjectMapper;
|
||
import org.gcube.com.fasterxml.jackson.databind.node.ArrayNode;
|
||
import org.gcube.com.fasterxml.jackson.databind.node.ObjectNode;
|
||
import org.gcube.grsf.publisher.ContextTest;
|
||
import org.gcube.grsf.publisher.ckan.record.FisheryRecord;
|
||
import org.gcube.grsf.publisher.ckan.record.Record;
|
||
import org.gcube.grsf.publisher.ckan.record.StockRecord;
|
||
import org.gcube.grsf.publisher.ckan.record.TraceabilityUnitRecord;
|
||
import org.gcube.grsf.publisher.freemarker.FreeMarker;
|
||
import org.gcube.grsf.publisher.freemarker.FreeMarkerTest;
|
||
import org.junit.Assert;
|
||
import org.junit.Ignore;
|
||
import org.junit.Test;
|
||
import org.slf4j.Logger;
|
||
import org.slf4j.LoggerFactory;
|
||
|
||
public class RecordTest extends ContextTest {
|
||
|
||
private static Logger logger = LoggerFactory.getLogger(RecordTest.class);
|
||
|
||
public static final String FISHERY = "Fishery";
|
||
public static final String STOCK = "Stock";
|
||
public static final String TRACEABILITY_UNIT = "Traceability Unit";
|
||
|
||
public File getExampleDirectory(String type) throws Exception {
|
||
File resourcesDirectory = getResourcesDirectory();
|
||
return new File(resourcesDirectory, "examples/" + type);
|
||
}
|
||
|
||
public static UriInfo getUriInfo(MultivaluedMap<String, String> queryParameters) {
|
||
UriInfo uriInfo = new UriInfo() {
|
||
|
||
@Override
|
||
public URI resolve(URI uri) {
|
||
return null;
|
||
}
|
||
|
||
@Override
|
||
public URI relativize(URI uri) {
|
||
return null;
|
||
}
|
||
|
||
@Override
|
||
public UriBuilder getRequestUriBuilder() {
|
||
return null;
|
||
}
|
||
|
||
@Override
|
||
public URI getRequestUri() {
|
||
return null;
|
||
}
|
||
|
||
@Override
|
||
public MultivaluedMap<String, String> getQueryParameters(boolean decode) {
|
||
return null;
|
||
}
|
||
|
||
@Override
|
||
public MultivaluedMap<String, String> getQueryParameters() {
|
||
return queryParameters;
|
||
}
|
||
|
||
@Override
|
||
public List<PathSegment> getPathSegments(boolean decode) {
|
||
return null;
|
||
}
|
||
|
||
@Override
|
||
public List<PathSegment> getPathSegments() {
|
||
return null;
|
||
}
|
||
|
||
@Override
|
||
public MultivaluedMap<String, String> getPathParameters(boolean decode) {
|
||
return null;
|
||
}
|
||
|
||
@Override
|
||
public MultivaluedMap<String, String> getPathParameters() {
|
||
return null;
|
||
}
|
||
|
||
@Override
|
||
public String getPath(boolean decode) {
|
||
return null;
|
||
}
|
||
|
||
@Override
|
||
public String getPath() {
|
||
return null;
|
||
}
|
||
|
||
@Override
|
||
public List<String> getMatchedURIs(boolean decode) {
|
||
return null;
|
||
}
|
||
|
||
@Override
|
||
public List<String> getMatchedURIs() {
|
||
return null;
|
||
}
|
||
|
||
@Override
|
||
public List<Object> getMatchedResources() {
|
||
return null;
|
||
}
|
||
|
||
@Override
|
||
public UriBuilder getBaseUriBuilder() {
|
||
return null;
|
||
}
|
||
|
||
@Override
|
||
public URI getBaseUri() {
|
||
return null;
|
||
}
|
||
|
||
@Override
|
||
public UriBuilder getAbsolutePathBuilder() {
|
||
return null;
|
||
}
|
||
|
||
@Override
|
||
public URI getAbsolutePath() {
|
||
return null;
|
||
}
|
||
};
|
||
|
||
return uriInfo;
|
||
}
|
||
|
||
protected File getResourcesDirectory() throws Exception {
|
||
URL fileURL = FreeMarker.class.getClassLoader().getResource("config.properties");
|
||
File logbackFile = new File(fileURL.toURI());
|
||
File resourcesDirectory = logbackFile.getParentFile();
|
||
return resourcesDirectory;
|
||
}
|
||
|
||
protected Record getRecordByType(String type) {
|
||
Record record;
|
||
switch (type) {
|
||
case FISHERY:
|
||
record = new FisheryRecord();
|
||
break;
|
||
|
||
case STOCK:
|
||
record = new StockRecord();
|
||
break;
|
||
|
||
case TRACEABILITY_UNIT:
|
||
record = new TraceabilityUnitRecord();
|
||
break;
|
||
|
||
default:
|
||
throw new RuntimeException(type + " type not supported");
|
||
}
|
||
|
||
return record;
|
||
}
|
||
|
||
// @Ignore
|
||
@Test
|
||
public void testFisheryRecords() throws Exception {
|
||
List<String> types = new ArrayList<>();
|
||
types.add(FISHERY);
|
||
int maxTestRecords = Integer.MAX_VALUE;
|
||
// maxTestRecords = 1;
|
||
int maxTestRecordsPerSource = Integer.MAX_VALUE;
|
||
maxTestRecordsPerSource = 3;
|
||
boolean restart = false;
|
||
String restartFromSource = null;
|
||
// String restartFromSource = "firms";
|
||
// String restartFromSource = "fishsource";
|
||
// String restartFromSource = "grsf";
|
||
// String restartFromSource = "sdg";
|
||
String restartFromFile = null;
|
||
testRecords(types, restart, restartFromSource, restartFromFile, maxTestRecords, maxTestRecordsPerSource);
|
||
}
|
||
|
||
@Ignore
|
||
@Test
|
||
public void testStockRecords() throws Exception {
|
||
List<String> types = new ArrayList<>();
|
||
types.add(STOCK);
|
||
int maxTestRecords = Integer.MAX_VALUE;
|
||
maxTestRecords = 1;
|
||
int maxTestRecordsPerSource = Integer.MAX_VALUE;
|
||
maxTestRecordsPerSource = 1;
|
||
boolean restart = false;
|
||
String restartFromSource = null;
|
||
// String restartFromSource = "firms";
|
||
// String restartFromSource = "fishsource";
|
||
// String restartFromSource = "grsf";
|
||
// String restartFromSource = "ram";
|
||
// String restartFromSource = "sdg";
|
||
restart = restartFromSource!=null ? true : false;
|
||
String restartFromFile = null;
|
||
testRecords(types, restart, restartFromSource, restartFromFile, maxTestRecords, maxTestRecordsPerSource);
|
||
}
|
||
|
||
// @Ignore
|
||
@Test
|
||
public void testTraceabilityUnitRecords() throws Exception {
|
||
List<String> types = new ArrayList<>();
|
||
types.add(TRACEABILITY_UNIT);
|
||
int maxTestRecords = Integer.MAX_VALUE;
|
||
maxTestRecords = 1;
|
||
int maxTestRecordsPerSource = Integer.MAX_VALUE;
|
||
maxTestRecordsPerSource = 1;
|
||
boolean restart = false;
|
||
String restartFromSource = null;
|
||
// String restartFromSource = "grsf";
|
||
restart = restartFromSource!=null ? true : false;
|
||
String restartFromFile = null;
|
||
testRecords(types, restart, restartFromSource, restartFromFile, maxTestRecords, maxTestRecordsPerSource);
|
||
}
|
||
|
||
// @Ignore
|
||
@Test
|
||
public void testAll() throws Exception {
|
||
ContextTest.setContextByName(VRE_GRSF_PRE);
|
||
List<String> types = new ArrayList<>();
|
||
types.add(FISHERY);
|
||
types.add(STOCK);
|
||
types.add(TRACEABILITY_UNIT);
|
||
int maxTestRecords = Integer.MAX_VALUE;
|
||
// maxTestRecords = 1;
|
||
int maxTestRecordsPerSource = Integer.MAX_VALUE;
|
||
maxTestRecordsPerSource = 25;
|
||
boolean restart = false;
|
||
String restartFromSource = null;
|
||
String restartFromFile = null;
|
||
testRecords(types, restart, restartFromSource, restartFromFile, maxTestRecords, maxTestRecordsPerSource);
|
||
}
|
||
|
||
private void testRecords(List<String> types, boolean restart, String restartFromSource, String restartFromFile, int maxTestRecords, int maxTestRecordsPerSource) throws Exception {
|
||
|
||
/*
|
||
* Tell if we want test the whole creation process;
|
||
*
|
||
* - when true the record is created in Ckan;
|
||
* - when false the process of applying Freemarker template and upload resources
|
||
* in the workspace. All the files uploaded in the workspace will be deleted at the end the test of the record.
|
||
*
|
||
*/
|
||
boolean create = true;
|
||
|
||
// The restart is valid only
|
||
// and only if we selected one Type
|
||
if(types.size()>1) {
|
||
restart = false;
|
||
}
|
||
|
||
boolean restartedFromSource = true;
|
||
if(restart && restartFromSource!=null) {
|
||
restartedFromSource = false;
|
||
}
|
||
|
||
boolean restartedFromFile = true;
|
||
if(restart && restartFromFile!=null) {
|
||
restartedFromFile = false;
|
||
}
|
||
|
||
Calendar start = Calendar.getInstance();
|
||
|
||
int testedRecords = 0;
|
||
int analysedRecords = 0;
|
||
|
||
File outputDir = new File(getResourcesDirectory(), "output");
|
||
outputDir.mkdir();
|
||
|
||
FilenameFilter dirnameFilter = new FilenameFilter() {
|
||
@Override
|
||
public boolean accept(File dir, String name) {
|
||
File f = new File(dir, name);
|
||
return f.isDirectory();
|
||
}
|
||
};
|
||
|
||
FilenameFilter filenameFilter = new FilenameFilter() {
|
||
@Override
|
||
public boolean accept(File dir, String name) {
|
||
return name.toLowerCase().endsWith(".json");
|
||
}
|
||
};
|
||
|
||
for (String type : types) {
|
||
if(testedRecords >= maxTestRecords) {
|
||
logger.info("The test has already elaborated {} records which is the max allowed (i.e. allowed {} records)", testedRecords, maxTestRecords);
|
||
return;
|
||
}
|
||
|
||
File typeDir = getExampleDirectory(type);
|
||
|
||
File outputTypeDir = new File(outputDir, type);
|
||
outputTypeDir.mkdir();
|
||
|
||
File[] sourceDirectory = typeDir.listFiles(dirnameFilter);
|
||
Arrays.sort(sourceDirectory);
|
||
|
||
// List<File> sourceDirectory = new ArrayList<>();
|
||
// sourceDirectory.add(new File(typeDir, "firms"));
|
||
// sourceDirectory.add(new File(typeDir, "fishsource"));
|
||
// sourceDirectory.add(new File(typeDir, "grsf"));
|
||
// sourceDirectory.add(new File(typeDir, "ram"));
|
||
// sourceDirectory.add(new File(typeDir, "sdg"));
|
||
|
||
for (File source : sourceDirectory) {
|
||
Calendar startSource = Calendar.getInstance();
|
||
|
||
String sourceString = source.getName();
|
||
|
||
if(restart && restartedFromSource==false) {
|
||
if(restartFromSource!=null) {
|
||
if(source.getName().startsWith(restartFromSource)) {
|
||
logger.info("Found the source where restarting i.e. {}", sourceString);
|
||
restartedFromSource = true;
|
||
}else {
|
||
logger.info("The source {} has been already elaborated. Skipping...", sourceString);
|
||
continue;
|
||
}
|
||
}
|
||
}
|
||
|
||
if(testedRecords >= maxTestRecords) {
|
||
logger.info("The test has already elaborated {} records which is the max allowed (i.e. allowed {} records)", testedRecords, maxTestRecords);
|
||
return;
|
||
}
|
||
|
||
|
||
File outputSourceDir = new File(outputTypeDir, sourceString);
|
||
outputSourceDir.mkdir();
|
||
|
||
File[] jfs = source.listFiles(filenameFilter);
|
||
// File[] jfs = new File[] {new File(source,"88818c3f-7120-322b-9637-7c7d2e9fc1e5.json")};
|
||
// File[] jfs = new File[] {new File(source,"00702023-0e2d-345d-8b20-60580c107acd.json")};
|
||
|
||
List<File> jsonFiles = Arrays.asList(jfs);
|
||
|
||
if(restart) {
|
||
Collections.sort(jsonFiles);
|
||
}else{
|
||
Collections.shuffle(jsonFiles);
|
||
}
|
||
|
||
int testedRecordsPerSource = 0;
|
||
int analysedRecordsPerSource = 0;
|
||
|
||
int numberOfRecordsPerSource = jsonFiles.size();
|
||
|
||
// if(maxTestRecordsPerSource!=Integer.MAX_VALUE) {
|
||
// numberOfRecordsPerSource = numberOfRecordsPerSource < maxTestRecordsPerSource ? numberOfRecordsPerSource : maxTestRecordsPerSource;
|
||
// }
|
||
|
||
for (File jsonFile : jsonFiles) {
|
||
String expectedGRSFUUID = jsonFile.getName().replace(".json", "");
|
||
|
||
if(restart && restartedFromFile==false) {
|
||
if(restartFromFile!=null) {
|
||
if(jsonFile.getName().startsWith(restartFromFile)) {
|
||
logger.info("Found the record to restart i.e. {} {} {}", sourceString, type, jsonFile.getName());
|
||
restartedFromFile = true;
|
||
}else {
|
||
logger.info("[{} of {}] {} {} with GRSF UUID {} has been already elaborated", analysedRecordsPerSource + 1, numberOfRecordsPerSource, sourceString, type, expectedGRSFUUID);
|
||
++analysedRecords;
|
||
++analysedRecordsPerSource;
|
||
continue;
|
||
}
|
||
}
|
||
}
|
||
|
||
if(testedRecords >= maxTestRecords) {
|
||
logger.info("The test has already elaborated {} records which is the max allowed (i.e. allowed {} records)", testedRecords, maxTestRecords);
|
||
return;
|
||
}
|
||
|
||
if(testedRecordsPerSource >= maxTestRecordsPerSource) {
|
||
logger.info("The test has already elaborated {} records for the source '{}' which is the max allowed (i.e. allowed {} records per source)", testedRecordsPerSource, sourceString, maxTestRecordsPerSource);
|
||
break;
|
||
}
|
||
|
||
logger.trace("[{} of {}] Going to elaborate {} {} with GRSF UUID {} from file {}", analysedRecordsPerSource + 1, numberOfRecordsPerSource, sourceString, type, expectedGRSFUUID, jsonFile.getAbsolutePath());
|
||
logger.info("[{} of {}] Going to elaborate {} {} with GRSF UUID {}", analysedRecordsPerSource + 1, numberOfRecordsPerSource, sourceString, type, expectedGRSFUUID);
|
||
|
||
|
||
ObjectMapper mapper = new ObjectMapper();
|
||
|
||
ObjectNode input = (ObjectNode) mapper.readTree(jsonFile);
|
||
|
||
if(type.compareTo(TRACEABILITY_UNIT)==0) {
|
||
input.put(FreeMarkerTest.SOURCE_PROPERTY, sourceString);
|
||
}
|
||
|
||
Record record = getRecordByType(type);
|
||
|
||
String grsfUUID = input.get(Record.GRSF_UUID_PROPERTY).asText();
|
||
Assert.assertTrue(expectedGRSFUUID.compareTo(grsfUUID)==0);
|
||
|
||
record.setName(grsfUUID);
|
||
|
||
String ret = "";
|
||
if(create) {
|
||
ret = record.create(mapper.writeValueAsString(input));
|
||
}else {
|
||
ret = record.simulateCreation(mapper.writeValueAsString(input));
|
||
}
|
||
|
||
++testedRecords;
|
||
++testedRecordsPerSource;
|
||
++analysedRecords;
|
||
++analysedRecordsPerSource;
|
||
|
||
logger.trace("[{} of {}] {} {} with GRSF UUID {} from file {} {} successfully\n{}", analysedRecordsPerSource, numberOfRecordsPerSource, sourceString, type, grsfUUID, create ? "created" : "elaborated", jsonFile.getAbsolutePath(), ret);
|
||
|
||
logger.info("[{} of {}] {} {} with GRSF UUID {} {} successfully", analysedRecordsPerSource, numberOfRecordsPerSource, sourceString, type, grsfUUID, create ? "created" : "elaborated");
|
||
|
||
if(create) {
|
||
long sleepTime = TimeUnit.MILLISECONDS.toMillis(200);
|
||
logger.trace("Going to sleep for {} millis", sleepTime);
|
||
Thread.sleep(sleepTime);
|
||
}
|
||
|
||
|
||
}
|
||
|
||
Calendar endSource = Calendar.getInstance();
|
||
long diff = endSource.getTimeInMillis() - startSource.getTimeInMillis();
|
||
logger.info("{} {} {} records in {} milliseconds (~{} seconds)",
|
||
create?"Created in Ckan " : "Simulated Creation of ", source, type,
|
||
testedRecordsPerSource, diff, TimeUnit.MILLISECONDS.toSeconds(diff));
|
||
|
||
}
|
||
}
|
||
|
||
Calendar end = Calendar.getInstance();
|
||
long diff = end.getTimeInMillis() - start.getTimeInMillis();
|
||
logger.info("Created in Ckan {} records in {} milliseconds (~{} seconds)",
|
||
testedRecords, diff, TimeUnit.MILLISECONDS.toSeconds(diff));
|
||
}
|
||
|
||
@Ignore
|
||
@Test
|
||
public void testList() {
|
||
MultivaluedMap<String, String> mvm = new MultivaluedHashMap<String,String>();
|
||
UriInfo uriInfo = getUriInfo(mvm);
|
||
|
||
String[] types = new String[] { FISHERY, STOCK, TRACEABILITY_UNIT };
|
||
for (String type : types) {
|
||
Record record = getRecordByType(type);
|
||
record.setUriInfo(uriInfo);
|
||
|
||
String ret = record.list(30, 0);
|
||
logger.info("Listed {} records are {}", record.getType(), ret);
|
||
}
|
||
}
|
||
|
||
// @Ignore
|
||
@Test
|
||
public void testDeleteAll() throws Exception {
|
||
// ContextTest.setContextByName(DEVVRE);
|
||
|
||
MultivaluedMap<String, String> mvm = new MultivaluedHashMap<String,String>();
|
||
UriInfo uriInfo = getUriInfo(mvm);
|
||
|
||
ObjectMapper mapper = new ObjectMapper();
|
||
|
||
int maxPerPage = 10;
|
||
|
||
String[] types = new String[] { FISHERY, STOCK, TRACEABILITY_UNIT };
|
||
|
||
long sleepTime = TimeUnit.MILLISECONDS.toMillis(500);
|
||
|
||
for (String type : types) {
|
||
boolean go = true;
|
||
while(go) {
|
||
Record record = getRecordByType(type);
|
||
record.setUriInfo(uriInfo);
|
||
|
||
String ret = record.list(maxPerPage, 0);
|
||
logger.info("Listed {} records are {}", record.getType(), ret);
|
||
|
||
ArrayNode arrayNode = (ArrayNode) mapper.readTree(ret);
|
||
int size = arrayNode.size();
|
||
|
||
if(size>0) {
|
||
for(int i=0; i<size; i++) {
|
||
record = getRecordByType(type);
|
||
record.setUriInfo(uriInfo);
|
||
|
||
String name = arrayNode.get(i).asText();
|
||
record.setName(name);
|
||
|
||
logger.info("Going to delete {} with GRSF UUID {}", record.getType(), name);
|
||
record.purge();
|
||
|
||
logger.trace("Going to sleep for {} millis", sleepTime);
|
||
Thread.sleep(sleepTime);
|
||
}
|
||
}else {
|
||
go = false;
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
}
|