gCubeSystem
/
grsf-publisher
16
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Fixing service

This commit is contained in:
luca.frosini 2023-07-20 15:14:56 +02:00
parent bce7332e3f
commit 039347f6fa
9 changed files with 102 additions and 53 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
src/main/java/org/gcube/grsf/publisher/ckan/others/GRSFResource.java
View File

@ -13,17 +13,11 @@ import org.slf4j.LoggerFactory;
public class GRSFResource extends CKANResource {
private static final Logger logger = LoggerFactory.getLogger(GRSFResource.class);
protected String grsfUUID;
public GRSFResource(String itemID) {
super(itemID);
this.storageHubManagement = new GRSFStorageHubManagement();
}
public void setGRSFUUID(String grsfUUID) {
this.grsfUUID = grsfUUID;
}
@Override
public void deleteFile() {
@ -44,8 +38,8 @@ public class GRSFResource extends CKANResource {
persistedURL = url;
if(isStorageFile(persistedURL)) {
try {
String storageFilename = Record.getFilename(grsfUUID, filename);
((GRSFStorageHubManagement) storageHubManagement).setGrsfUUID(grsfUUID);
String storageFilename = Record.getFilename(name, filename);
((GRSFStorageHubManagement) storageHubManagement).setGrsfUUID(name);
storageHubManagement.deleteResourcePersistence(itemID, storageFilename, mimetype);
} catch(Exception e) {
throw new InternalServerErrorException(e);

40
src/main/java/org/gcube/grsf/publisher/ckan/record/Record.java
View File

@ -44,6 +44,7 @@ import org.gcube.grsf.publisher.ckan.others.GRSFResource;
import org.gcube.grsf.publisher.configuration.GRSFCatalogueConfiguration;
import org.gcube.grsf.publisher.freemarker.FreeMarker;
import org.gcube.grsf.publisher.utils.TypeUtils;
import org.gcube.grsf.publisher.workspace.GRSFStorageHubManagement;
import org.gcube.storagehub.StorageHubManagement;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -57,6 +58,10 @@ public abstract class Record extends CKANPackage {
private final Logger logger = LoggerFactory.getLogger(this.getClass());
public static final String RECORD_URL_TEMPLATE_PROPERTY_KEY = "record_url";
public static final String INCLUDE_SENSITIVE_TEMPLATE_PROPERTY_KEY = "include_sensitive";
public static final String IS_PATCH_TEMPLATE_PROPERTY_KEY = "is_patch";
public static final String GRSF_UUID_PROPERTY = "grsf_uuid";
public static final String TAGS_PROPERTY = "tags";
public static final String GROUPS_PROPERTY = "groups";
@ -82,10 +87,13 @@ public abstract class Record extends CKANPackage {
protected ObjectMapper objectMapper;
protected JsonNode jsonNode;
protected boolean patch;
protected Set<FileContainer> wsUploadedFiles;
public Record() {
this.objectMapper = new ObjectMapper();
this.patch = false;
}
public abstract String getType();
@ -224,17 +232,19 @@ public abstract class Record extends CKANPackage {
logger.debug("File {} has been persisted in StorageHub with ID:{}", file.getName(), fileContainer.getId());
return fileContainer;
}
protected Map<String, Object> getMapFromSourceJson(JsonNode jsonNode) throws Exception {
Map<String, Object> map = objectMapper.convertValue(jsonNode, new TypeReference<Map<String, Object>>(){});
grsfUUID = map.get(Record.GRSF_UUID_PROPERTY).toString();
URIResolver uriResolver = URIResolver.getInstance();
String recordURL = uriResolver.getCatalogueItemURL(grsfUUID);
map.put("record_url", recordURL);
map.put(RECORD_URL_TEMPLATE_PROPERTY_KEY, recordURL);
GRSFCatalogueConfiguration grsfCC = (GRSFCatalogueConfiguration) CatalogueConfigurationFactory.getInstance();
map.put("include_sensitive", grsfCC.isIncludeSensitive());
map.put(INCLUDE_SENSITIVE_TEMPLATE_PROPERTY_KEY, grsfCC.isIncludeSensitive());
map.put(IS_PATCH_TEMPLATE_PROPERTY_KEY, patch);
return map;
}
@ -262,7 +272,11 @@ public abstract class Record extends CKANPackage {
ArrayNode timeseries = (ArrayNode) jsonNode.get(Record.TIMESERIES_PROPERTY);
List<File> timeseriesFiles = new ArrayList<>();
Secret secret = Constants.getCatalogueSecret();
SecretManager secretManager = SecretManagerProvider.instance.get();
try {
secretManager.startSession(secret);
for(JsonNode n : timeseries) {
String key = n.get(Record.TIMESERIES_ELEMENT_PROPERTY_PROPERTY).asText();
try {
@ -295,13 +309,13 @@ public abstract class Record extends CKANPackage {
*/
map.remove(Record.TIMESERIES_PROPERTY);
Secret secret = Constants.getCatalogueSecret();
SecretManager secretManager = SecretManagerProvider.instance.get();
secretManager.startSession(secret);
FileContainer fc = persistFile(tsOut);
this.wsUploadedFiles.add(fc);
URL url = fc.getPublicLink();
secretManager.endSession();
GRSFStorageHubManagement grsfSHM = new GRSFStorageHubManagement();
grsfSHM.setGrsfUUID(grsfUUID);
grsfSHM.persistFile(tsOut);
this.wsUploadedFiles.add(grsfSHM.getFileContainer());
URL url = grsfSHM.getPublicLink();
ObjectNode resourceNode = objectMapper.createObjectNode();
resourceNode.put(NAME_PROPERTY, fileName);
@ -320,6 +334,7 @@ public abstract class Record extends CKANPackage {
((ObjectNode)jsonNode).remove(Record.TIMESERIES_PROPERTY);
} finally {
secretManager.endSession();
/*
* Remove the local files of timeseries
* if an error occurs to keep the environment clean
@ -383,7 +398,7 @@ public abstract class Record extends CKANPackage {
ArrayNode originalResourcesarrayNode = (ArrayNode) result.get(RESOURCES_KEY);
for(JsonNode resourceNode : originalResourcesarrayNode) {
GRSFResource grsfResource = new GRSFResource(itemID);
grsfResource.setGRSFUUID(name);
grsfResource.setName(name);
grsfResource.setPreviousRepresentation(resourceNode);
grsfResource.deleteFile();
}
@ -406,6 +421,7 @@ public abstract class Record extends CKANPackage {
@Override
public String patch(String json) {
try {
this.patch = true;
JsonNode jsonNode = objectMapper.readTree(json);
elaborate(jsonNode);
// TODO
@ -431,7 +447,7 @@ public abstract class Record extends CKANPackage {
ArrayNode arrayNode = (ArrayNode) result.get(RESOURCES_KEY);
for(JsonNode jsonNode : arrayNode) {
GRSFResource grsfResource = new GRSFResource(itemID);
grsfResource.setGRSFUUID(name);
grsfResource.setName(name);
grsfResource.setPreviousRepresentation(jsonNode);
grsfResource.deleteFile(); // Only delete file is required because the item will be purged at the end
}

35
src/main/java/org/gcube/grsf/publisher/workspace/GRSFStorageHubManagement.java
View File

@ -1,12 +1,25 @@
package org.gcube.grsf.publisher.workspace;
import java.io.File;
import java.io.FileInputStream;
import java.net.URL;
import org.gcube.common.storagehub.client.dsl.FileContainer;
import org.gcube.gcat.workspace.GcatStorageHubManagement;
import org.gcube.grsf.publisher.ckan.record.Record;
import org.gcube.storagehub.MetadataMatcher;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class GRSFStorageHubManagement extends GcatStorageHubManagement {
private final Logger logger = LoggerFactory.getLogger(GRSFStorageHubManagement.class);
protected String grsfUUID;
protected URL url;
protected FileContainer fileContainer;
public GRSFStorageHubManagement() {
super();
}
@ -15,8 +28,30 @@ public class GRSFStorageHubManagement extends GcatStorageHubManagement {
this.grsfUUID = grsfUUID;
}
public URL getPublicLink() {
return url;
}
public FileContainer getFileContainer() {
return fileContainer;
}
protected MetadataMatcher getMetadataMatcher() {
MetadataMatcher metadataMatcher = new GRSFMetadataMatcher(grsfUUID);
return metadataMatcher;
}
/**
* Save the file in the workspace using storagehub-application-persistence
* @param file the file to persist in the workspace
* @return
* @throws Exception
*/
public void persistFile(File file) throws Exception{
FileInputStream fis = new FileInputStream(file);
storageHubManagement.setMetadataMatcher(getMetadataMatcher());
url = storageHubManagement.persistFile(fis, file.getName(), Record.TIMESERIES_MIMETYPE);
fileContainer = storageHubManagement.getPersistedFile();
logger.debug("File {} has been persisted in StorageHub with ID:{}. Public Link : {}", file.getName(), fileContainer.getId(), url);
}
}

6
src/main/resources/freemarker/Fishery.ftl
View File

@ -3,13 +3,13 @@
<#assign data_namespace="fishery_data" >
{
"name" : "${grsf_uuid?json_string}",
<#if (!is_patch?? || !is_patch) && !fishery_name?has_content>
<#if !is_patch && fishery_name?has_content>
"title" : "${fishery_name?json_string}",
</#if>
<#if (!is_patch?? || !is_patch) && !license_id?has_content>
<#if !is_patch && license_id?has_content>
"license_id": "${license_id}",
</#if>
<#if (!is_patch?? || !is_patch) && !description?has_content>
<#if !is_patch && description?has_content>
"notes": "${description?json_string}",
</#if>
"extras": [

6
src/main/resources/freemarker/Stock.ftl
View File

@ -3,13 +3,13 @@
<#assign data_namespace="stock_data" >
{
"name" : "${grsf_uuid?json_string}",
<#if (!is_patch?? || !is_patch) && !stock_name?has_content>
<#if !is_patch && stock_name?has_content>
"title" : "${stock_name?json_string}",
</#if>
<#if (!is_patch?? || !is_patch) && !license_id?has_content>
<#if !is_patch && license_id?has_content>
"license_id": "${license_id}",
</#if>
<#if (!is_patch?? || !is_patch) && !description?has_content>
<#if !is_patch && description?has_content>
"notes": "${description?json_string}",
</#if>
"extras": [

10
src/main/resources/freemarker/Traceability Unit.ftl
View File

@ -3,13 +3,13 @@
<#assign data_namespace="traceability_unit_data" >
{
"name" : "${grsf_uuid?json_string}",
<#if (!is_patch?? || !is_patch) && !traceability_unit_name?has_content>
<#if !is_patch && traceability_unit_name?has_content>
"title" : "${traceability_unit_name?json_string}",
</#if>
<#if (!is_patch?? || !is_patch) && !license_id?has_content>
<#if !is_patch && license_id?has_content>
"license_id": "${license_id}",
</#if>
<#if (!is_patch?? || !is_patch) && !description?has_content>
<#if !is_patch && description?has_content>
"notes": "${description?json_string}",
</#if>
"extras": [
@ -53,10 +53,10 @@
<@tag tagname="Traceability Unit" sep=""/>
],
"resources": [
<#if (!is_patch?? || !is_patch) && !referring_stock_record?has_content>
<#if !is_patch && referring_stock_record?has_content>
<@resource name="GRSF Stock" url=referring_stock_record.url description=referring_stock_record.semantic_id />
</#if>
<#if (!is_patch?? || !is_patch) && !referring_fishery_record?has_content>
<#if !is_patch && referring_fishery_record?has_content>
<@resource name="GRSF Fishery" url=referring_fishery_record.url description=referring_fishery_record.semantic_id />
</#if>
<@resource name="Traceability Unit URI" url=traceability_record_uri sep=""/>

2
src/main/resources/freemarker/macros.ftl
View File

@ -4,7 +4,7 @@
<#assign group_list = [] >
<#assign tag_list = [] >
<#if !is_patch?? || !is_patch>
<#if !is_patch>
<#assign description = "Short Name: ${short_name}\n" >
<#if grsf_semantic_identifier??>
<#assign description += "GRSF Semantic Identifier: ${grsf_semantic_identifier}\n" >

30
src/test/java/org/gcube/grsf/publisher/freemarker/FreeMarkerTest.java
View File

@ -31,6 +31,9 @@ public class FreeMarkerTest {
private static Logger logger = LoggerFactory.getLogger(FreeMarkerTest.class);
public static final String GRSF_TYPE_PROPERTY = "grsf_type";
public static final String SOURCE_PROPERTY = "source";
protected ObjectMapper mapper;
protected GRSFCatalogueConfiguration grsfCC;
protected URIResolver uriResolver;
@ -74,14 +77,14 @@ public class FreeMarkerTest {
}
public void testAll(boolean patch) throws Exception {
int maxTestRecords = 35;
int maxTestRecordsPerSource = 5;
int maxTestRecords = 1;
int maxTestRecordsPerSource = 1;
FreeMarker freeMarker = new FreeMarker();
Template tsTemplate = freeMarker.getTemplate("timeseries.ftl");
String[] types = new String[] {"Stock", "Fishery", "Traceability Unit"};
String[] types = new String[] { "Stock", "Fishery", "Traceability Unit" };
// String[] types = new String[] {"Stock"};
// String[] types = new String[] {"Fishery"};
// String[] types = new String[] {"Traceability Unit"};
@ -179,7 +182,7 @@ public class FreeMarkerTest {
for(String key : keys) {
switch (key) {
case Record.GRSF_UUID_PROPERTY:
case "grsf_type":
case GRSF_TYPE_PROPERTY:
continue;
default:
@ -189,10 +192,11 @@ public class FreeMarkerTest {
Map<String, Object> mapForPatch = new HashMap<>();
mapForPatch.put(key, map.get(key));
mapForPatch.put(Record.GRSF_UUID_PROPERTY, grsfUUID);
mapForPatch.put("grsf_type", map.get("grsf_type"));
mapForPatch.put("include_sensitive", grsfCC.isIncludeSensitive());
mapForPatch.put("source", sourceString);
mapForPatch.put("is_patch", true);
mapForPatch.put(Record.INCLUDE_SENSITIVE_TEMPLATE_PROPERTY_KEY, grsfCC.isIncludeSensitive());
mapForPatch.put(Record.IS_PATCH_TEMPLATE_PROPERTY_KEY, patch);
mapForPatch.put(GRSF_TYPE_PROPERTY, map.get(GRSF_TYPE_PROPERTY));
mapForPatch.put(SOURCE_PROPERTY, sourceString);
logger.info("Map created for key {} is {}", key, mapForPatch);
@ -212,10 +216,10 @@ public class FreeMarkerTest {
// This allow to test template if we are offline
recordURL = "http://data.d4science.org/ctlg/GRSF/" + grsfUUID;
}
map.put("record_url", recordURL);
map.put("include_sensitive", grsfCC.isIncludeSensitive());
map.put("source", sourceString);
map.put(Record.RECORD_URL_TEMPLATE_PROPERTY_KEY, recordURL);
map.put(Record.INCLUDE_SENSITIVE_TEMPLATE_PROPERTY_KEY, grsfCC.isIncludeSensitive());
map.put(Record.IS_PATCH_TEMPLATE_PROPERTY_KEY, patch);
map.put(SOURCE_PROPERTY, sourceString);
logger.trace("Elaborating {} {} from file {}", sourceString, type, jsonFile.getName());
@ -264,7 +268,7 @@ public class FreeMarkerTest {
++countTimeSeries;
map.remove("timeseries");
map.remove(Record.TIMESERIES_PROPERTY);
}
return countTimeSeries;

14
src/test/java/org/gcube/grsf/publisher/record/RecordTest.java
View File

@ -23,6 +23,7 @@ import org.gcube.grsf.publisher.ckan.record.Record;
import org.gcube.grsf.publisher.ckan.record.Stock;
import org.gcube.grsf.publisher.ckan.record.TraceabilityUnit;
import org.gcube.grsf.publisher.freemarker.FreeMarker;
import org.gcube.grsf.publisher.freemarker.FreeMarkerTest;
import org.junit.Ignore;
import org.junit.Test;
import org.slf4j.Logger;
@ -172,12 +173,12 @@ public class RecordTest extends ContextTest {
@Test
public void testCreateRecords() throws Exception {
int maxTestRecords = 1000000000;
int maxTestRecordsPerSource = 3;
int maxTestRecords = 1;
int maxTestRecordsPerSource = 1;
String[] types = new String[] { "Fishery", "Stock", "Traceability Unit" };
// String[] types = new String[] {"Fishery"};
String[] types = new String[] { "Stock", "Fishery", "Traceability Unit" };
// String[] types = new String[] {"Stock"};
// String[] types = new String[] {"Fishery"};
// String[] types = new String[] {"Traceability Unit"};
Calendar start = Calendar.getInstance();
@ -231,7 +232,6 @@ public class RecordTest extends ContextTest {
File[] jsonFiles = source.listFiles(filenameFilter);
// File[] jsonFiles = new File[] {new File(source,"88818c3f-7120-322b-9637-7c7d2e9fc1e5.json")};
// File[] jsonFiles = new File[] {new File(source,"00702023-0e2d-345d-8b20-60580c107acd.json")};
// File[] jsonFiles = new File[] {new File(source,"88818c3f-7120-322b-9637-7c7d2e9fc1e5.json")};
int countRecordPerSource = 0;
@ -252,11 +252,11 @@ public class RecordTest extends ContextTest {
ObjectMapper mapper = new ObjectMapper();
ObjectNode input = (ObjectNode) mapper.readTree(jsonFile);
input.put("source", sourceString);
input.put(FreeMarkerTest.SOURCE_PROPERTY, sourceString);
Record record = getRecordByType(type);
String grsfUUID = input.get("grsf_uuid").asText();
String grsfUUID = input.get(Record.GRSF_UUID_PROPERTY).asText();
record.setName(grsfUUID);
String ret = record.create(mapper.writeValueAsString(input));
logger.debug("{} {} with GRSF UUID {} created successfully\n{}", sourceString, type, grsfUUID, ret);