From 039347f6fa36cf72e34245a5cff543a9f8f14507 Mon Sep 17 00:00:00 2001 From: "luca.frosini" Date: Thu, 20 Jul 2023 15:14:56 +0200 Subject: [PATCH] Fixing service --- .../publisher/ckan/others/GRSFResource.java | 12 ++---- .../grsf/publisher/ckan/record/Record.java | 40 +++++++++++++------ .../workspace/GRSFStorageHubManagement.java | 35 ++++++++++++++++ src/main/resources/freemarker/Fishery.ftl | 6 +-- src/main/resources/freemarker/Stock.ftl | 6 +-- .../freemarker/Traceability Unit.ftl | 10 ++--- src/main/resources/freemarker/macros.ftl | 2 +- .../publisher/freemarker/FreeMarkerTest.java | 30 ++++++++------ .../grsf/publisher/record/RecordTest.java | 14 +++---- 9 files changed, 102 insertions(+), 53 deletions(-) diff --git a/src/main/java/org/gcube/grsf/publisher/ckan/others/GRSFResource.java b/src/main/java/org/gcube/grsf/publisher/ckan/others/GRSFResource.java index 653329b..dcbed70 100644 --- a/src/main/java/org/gcube/grsf/publisher/ckan/others/GRSFResource.java +++ b/src/main/java/org/gcube/grsf/publisher/ckan/others/GRSFResource.java @@ -13,17 +13,11 @@ import org.slf4j.LoggerFactory; public class GRSFResource extends CKANResource { private static final Logger logger = LoggerFactory.getLogger(GRSFResource.class); - - protected String grsfUUID; - + public GRSFResource(String itemID) { super(itemID); this.storageHubManagement = new GRSFStorageHubManagement(); } - - public void setGRSFUUID(String grsfUUID) { - this.grsfUUID = grsfUUID; - } @Override public void deleteFile() { @@ -44,8 +38,8 @@ public class GRSFResource extends CKANResource { persistedURL = url; if(isStorageFile(persistedURL)) { try { - String storageFilename = Record.getFilename(grsfUUID, filename); - ((GRSFStorageHubManagement) storageHubManagement).setGrsfUUID(grsfUUID); + String storageFilename = Record.getFilename(name, filename); + ((GRSFStorageHubManagement) storageHubManagement).setGrsfUUID(name); storageHubManagement.deleteResourcePersistence(itemID, storageFilename, mimetype); } catch(Exception e) { throw new InternalServerErrorException(e); diff --git a/src/main/java/org/gcube/grsf/publisher/ckan/record/Record.java b/src/main/java/org/gcube/grsf/publisher/ckan/record/Record.java index 0d656fa..cd932dc 100644 --- a/src/main/java/org/gcube/grsf/publisher/ckan/record/Record.java +++ b/src/main/java/org/gcube/grsf/publisher/ckan/record/Record.java @@ -44,6 +44,7 @@ import org.gcube.grsf.publisher.ckan.others.GRSFResource; import org.gcube.grsf.publisher.configuration.GRSFCatalogueConfiguration; import org.gcube.grsf.publisher.freemarker.FreeMarker; import org.gcube.grsf.publisher.utils.TypeUtils; +import org.gcube.grsf.publisher.workspace.GRSFStorageHubManagement; import org.gcube.storagehub.StorageHubManagement; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -57,6 +58,10 @@ public abstract class Record extends CKANPackage { private final Logger logger = LoggerFactory.getLogger(this.getClass()); + public static final String RECORD_URL_TEMPLATE_PROPERTY_KEY = "record_url"; + public static final String INCLUDE_SENSITIVE_TEMPLATE_PROPERTY_KEY = "include_sensitive"; + public static final String IS_PATCH_TEMPLATE_PROPERTY_KEY = "is_patch"; + public static final String GRSF_UUID_PROPERTY = "grsf_uuid"; public static final String TAGS_PROPERTY = "tags"; public static final String GROUPS_PROPERTY = "groups"; @@ -82,10 +87,13 @@ public abstract class Record extends CKANPackage { protected ObjectMapper objectMapper; protected JsonNode jsonNode; + protected boolean patch; + protected Set wsUploadedFiles; public Record() { this.objectMapper = new ObjectMapper(); + this.patch = false; } public abstract String getType(); @@ -224,17 +232,19 @@ public abstract class Record extends CKANPackage { logger.debug("File {} has been persisted in StorageHub with ID:{}", file.getName(), fileContainer.getId()); return fileContainer; } - + protected Map getMapFromSourceJson(JsonNode jsonNode) throws Exception { Map map = objectMapper.convertValue(jsonNode, new TypeReference>(){}); grsfUUID = map.get(Record.GRSF_UUID_PROPERTY).toString(); URIResolver uriResolver = URIResolver.getInstance(); String recordURL = uriResolver.getCatalogueItemURL(grsfUUID); - map.put("record_url", recordURL); + map.put(RECORD_URL_TEMPLATE_PROPERTY_KEY, recordURL); GRSFCatalogueConfiguration grsfCC = (GRSFCatalogueConfiguration) CatalogueConfigurationFactory.getInstance(); - map.put("include_sensitive", grsfCC.isIncludeSensitive()); + map.put(INCLUDE_SENSITIVE_TEMPLATE_PROPERTY_KEY, grsfCC.isIncludeSensitive()); + + map.put(IS_PATCH_TEMPLATE_PROPERTY_KEY, patch); return map; } @@ -262,7 +272,11 @@ public abstract class Record extends CKANPackage { ArrayNode timeseries = (ArrayNode) jsonNode.get(Record.TIMESERIES_PROPERTY); List timeseriesFiles = new ArrayList<>(); + Secret secret = Constants.getCatalogueSecret(); + SecretManager secretManager = SecretManagerProvider.instance.get(); + try { + secretManager.startSession(secret); for(JsonNode n : timeseries) { String key = n.get(Record.TIMESERIES_ELEMENT_PROPERTY_PROPERTY).asText(); try { @@ -295,13 +309,13 @@ public abstract class Record extends CKANPackage { */ map.remove(Record.TIMESERIES_PROPERTY); - Secret secret = Constants.getCatalogueSecret(); - SecretManager secretManager = SecretManagerProvider.instance.get(); - secretManager.startSession(secret); - FileContainer fc = persistFile(tsOut); - this.wsUploadedFiles.add(fc); - URL url = fc.getPublicLink(); - secretManager.endSession(); + + GRSFStorageHubManagement grsfSHM = new GRSFStorageHubManagement(); + grsfSHM.setGrsfUUID(grsfUUID); + grsfSHM.persistFile(tsOut); + this.wsUploadedFiles.add(grsfSHM.getFileContainer()); + URL url = grsfSHM.getPublicLink(); + ObjectNode resourceNode = objectMapper.createObjectNode(); resourceNode.put(NAME_PROPERTY, fileName); @@ -320,6 +334,7 @@ public abstract class Record extends CKANPackage { ((ObjectNode)jsonNode).remove(Record.TIMESERIES_PROPERTY); } finally { + secretManager.endSession(); /* * Remove the local files of timeseries * if an error occurs to keep the environment clean @@ -383,7 +398,7 @@ public abstract class Record extends CKANPackage { ArrayNode originalResourcesarrayNode = (ArrayNode) result.get(RESOURCES_KEY); for(JsonNode resourceNode : originalResourcesarrayNode) { GRSFResource grsfResource = new GRSFResource(itemID); - grsfResource.setGRSFUUID(name); + grsfResource.setName(name); grsfResource.setPreviousRepresentation(resourceNode); grsfResource.deleteFile(); } @@ -406,6 +421,7 @@ public abstract class Record extends CKANPackage { @Override public String patch(String json) { try { + this.patch = true; JsonNode jsonNode = objectMapper.readTree(json); elaborate(jsonNode); // TODO @@ -431,7 +447,7 @@ public abstract class Record extends CKANPackage { ArrayNode arrayNode = (ArrayNode) result.get(RESOURCES_KEY); for(JsonNode jsonNode : arrayNode) { GRSFResource grsfResource = new GRSFResource(itemID); - grsfResource.setGRSFUUID(name); + grsfResource.setName(name); grsfResource.setPreviousRepresentation(jsonNode); grsfResource.deleteFile(); // Only delete file is required because the item will be purged at the end } diff --git a/src/main/java/org/gcube/grsf/publisher/workspace/GRSFStorageHubManagement.java b/src/main/java/org/gcube/grsf/publisher/workspace/GRSFStorageHubManagement.java index f96adc5..68f5125 100644 --- a/src/main/java/org/gcube/grsf/publisher/workspace/GRSFStorageHubManagement.java +++ b/src/main/java/org/gcube/grsf/publisher/workspace/GRSFStorageHubManagement.java @@ -1,12 +1,25 @@ package org.gcube.grsf.publisher.workspace; +import java.io.File; +import java.io.FileInputStream; +import java.net.URL; + +import org.gcube.common.storagehub.client.dsl.FileContainer; import org.gcube.gcat.workspace.GcatStorageHubManagement; +import org.gcube.grsf.publisher.ckan.record.Record; import org.gcube.storagehub.MetadataMatcher; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class GRSFStorageHubManagement extends GcatStorageHubManagement { + private final Logger logger = LoggerFactory.getLogger(GRSFStorageHubManagement.class); + protected String grsfUUID; + protected URL url; + protected FileContainer fileContainer; + public GRSFStorageHubManagement() { super(); } @@ -15,8 +28,30 @@ public class GRSFStorageHubManagement extends GcatStorageHubManagement { this.grsfUUID = grsfUUID; } + public URL getPublicLink() { + return url; + } + + public FileContainer getFileContainer() { + return fileContainer; + } + protected MetadataMatcher getMetadataMatcher() { MetadataMatcher metadataMatcher = new GRSFMetadataMatcher(grsfUUID); return metadataMatcher; } + + /** + * Save the file in the workspace using storagehub-application-persistence + * @param file the file to persist in the workspace + * @return + * @throws Exception + */ + public void persistFile(File file) throws Exception{ + FileInputStream fis = new FileInputStream(file); + storageHubManagement.setMetadataMatcher(getMetadataMatcher()); + url = storageHubManagement.persistFile(fis, file.getName(), Record.TIMESERIES_MIMETYPE); + fileContainer = storageHubManagement.getPersistedFile(); + logger.debug("File {} has been persisted in StorageHub with ID:{}. Public Link : {}", file.getName(), fileContainer.getId(), url); + } } diff --git a/src/main/resources/freemarker/Fishery.ftl b/src/main/resources/freemarker/Fishery.ftl index e07c40f..71947e1 100644 --- a/src/main/resources/freemarker/Fishery.ftl +++ b/src/main/resources/freemarker/Fishery.ftl @@ -3,13 +3,13 @@ <#assign data_namespace="fishery_data" > { "name" : "${grsf_uuid?json_string}", - <#if (!is_patch?? || !is_patch) && !fishery_name?has_content> + <#if !is_patch && fishery_name?has_content> "title" : "${fishery_name?json_string}", - <#if (!is_patch?? || !is_patch) && !license_id?has_content> + <#if !is_patch && license_id?has_content> "license_id": "${license_id}", - <#if (!is_patch?? || !is_patch) && !description?has_content> + <#if !is_patch && description?has_content> "notes": "${description?json_string}", "extras": [ diff --git a/src/main/resources/freemarker/Stock.ftl b/src/main/resources/freemarker/Stock.ftl index 2921719..2f057a9 100644 --- a/src/main/resources/freemarker/Stock.ftl +++ b/src/main/resources/freemarker/Stock.ftl @@ -3,13 +3,13 @@ <#assign data_namespace="stock_data" > { "name" : "${grsf_uuid?json_string}", - <#if (!is_patch?? || !is_patch) && !stock_name?has_content> + <#if !is_patch && stock_name?has_content> "title" : "${stock_name?json_string}", - <#if (!is_patch?? || !is_patch) && !license_id?has_content> + <#if !is_patch && license_id?has_content> "license_id": "${license_id}", - <#if (!is_patch?? || !is_patch) && !description?has_content> + <#if !is_patch && description?has_content> "notes": "${description?json_string}", "extras": [ diff --git a/src/main/resources/freemarker/Traceability Unit.ftl b/src/main/resources/freemarker/Traceability Unit.ftl index 46ab9a1..466f1dc 100644 --- a/src/main/resources/freemarker/Traceability Unit.ftl +++ b/src/main/resources/freemarker/Traceability Unit.ftl @@ -3,13 +3,13 @@ <#assign data_namespace="traceability_unit_data" > { "name" : "${grsf_uuid?json_string}", - <#if (!is_patch?? || !is_patch) && !traceability_unit_name?has_content> + <#if !is_patch && traceability_unit_name?has_content> "title" : "${traceability_unit_name?json_string}", - <#if (!is_patch?? || !is_patch) && !license_id?has_content> + <#if !is_patch && license_id?has_content> "license_id": "${license_id}", - <#if (!is_patch?? || !is_patch) && !description?has_content> + <#if !is_patch && description?has_content> "notes": "${description?json_string}", "extras": [ @@ -53,10 +53,10 @@ <@tag tagname="Traceability Unit" sep=""/> ], "resources": [ - <#if (!is_patch?? || !is_patch) && !referring_stock_record?has_content> + <#if !is_patch && referring_stock_record?has_content> <@resource name="GRSF Stock" url=referring_stock_record.url description=referring_stock_record.semantic_id /> - <#if (!is_patch?? || !is_patch) && !referring_fishery_record?has_content> + <#if !is_patch && referring_fishery_record?has_content> <@resource name="GRSF Fishery" url=referring_fishery_record.url description=referring_fishery_record.semantic_id /> <@resource name="Traceability Unit URI" url=traceability_record_uri sep=""/> diff --git a/src/main/resources/freemarker/macros.ftl b/src/main/resources/freemarker/macros.ftl index 1084e13..cb0d871 100644 --- a/src/main/resources/freemarker/macros.ftl +++ b/src/main/resources/freemarker/macros.ftl @@ -4,7 +4,7 @@ <#assign group_list = [] > <#assign tag_list = [] > -<#if !is_patch?? || !is_patch> +<#if !is_patch> <#assign description = "Short Name: ${short_name}\n" > <#if grsf_semantic_identifier??> <#assign description += "GRSF Semantic Identifier: ${grsf_semantic_identifier}\n" > diff --git a/src/test/java/org/gcube/grsf/publisher/freemarker/FreeMarkerTest.java b/src/test/java/org/gcube/grsf/publisher/freemarker/FreeMarkerTest.java index 4c0b092..19135a5 100644 --- a/src/test/java/org/gcube/grsf/publisher/freemarker/FreeMarkerTest.java +++ b/src/test/java/org/gcube/grsf/publisher/freemarker/FreeMarkerTest.java @@ -31,6 +31,9 @@ public class FreeMarkerTest { private static Logger logger = LoggerFactory.getLogger(FreeMarkerTest.class); + public static final String GRSF_TYPE_PROPERTY = "grsf_type"; + public static final String SOURCE_PROPERTY = "source"; + protected ObjectMapper mapper; protected GRSFCatalogueConfiguration grsfCC; protected URIResolver uriResolver; @@ -74,14 +77,14 @@ public class FreeMarkerTest { } public void testAll(boolean patch) throws Exception { - int maxTestRecords = 35; - int maxTestRecordsPerSource = 5; + int maxTestRecords = 1; + int maxTestRecordsPerSource = 1; FreeMarker freeMarker = new FreeMarker(); Template tsTemplate = freeMarker.getTemplate("timeseries.ftl"); - String[] types = new String[] {"Stock", "Fishery", "Traceability Unit"}; + String[] types = new String[] { "Stock", "Fishery", "Traceability Unit" }; // String[] types = new String[] {"Stock"}; // String[] types = new String[] {"Fishery"}; // String[] types = new String[] {"Traceability Unit"}; @@ -179,7 +182,7 @@ public class FreeMarkerTest { for(String key : keys) { switch (key) { case Record.GRSF_UUID_PROPERTY: - case "grsf_type": + case GRSF_TYPE_PROPERTY: continue; default: @@ -189,10 +192,11 @@ public class FreeMarkerTest { Map mapForPatch = new HashMap<>(); mapForPatch.put(key, map.get(key)); mapForPatch.put(Record.GRSF_UUID_PROPERTY, grsfUUID); - mapForPatch.put("grsf_type", map.get("grsf_type")); - mapForPatch.put("include_sensitive", grsfCC.isIncludeSensitive()); - mapForPatch.put("source", sourceString); - mapForPatch.put("is_patch", true); + mapForPatch.put(Record.INCLUDE_SENSITIVE_TEMPLATE_PROPERTY_KEY, grsfCC.isIncludeSensitive()); + mapForPatch.put(Record.IS_PATCH_TEMPLATE_PROPERTY_KEY, patch); + mapForPatch.put(GRSF_TYPE_PROPERTY, map.get(GRSF_TYPE_PROPERTY)); + mapForPatch.put(SOURCE_PROPERTY, sourceString); + logger.info("Map created for key {} is {}", key, mapForPatch); @@ -212,10 +216,10 @@ public class FreeMarkerTest { // This allow to test template if we are offline recordURL = "http://data.d4science.org/ctlg/GRSF/" + grsfUUID; } - map.put("record_url", recordURL); - - map.put("include_sensitive", grsfCC.isIncludeSensitive()); - map.put("source", sourceString); + map.put(Record.RECORD_URL_TEMPLATE_PROPERTY_KEY, recordURL); + map.put(Record.INCLUDE_SENSITIVE_TEMPLATE_PROPERTY_KEY, grsfCC.isIncludeSensitive()); + map.put(Record.IS_PATCH_TEMPLATE_PROPERTY_KEY, patch); + map.put(SOURCE_PROPERTY, sourceString); logger.trace("Elaborating {} {} from file {}", sourceString, type, jsonFile.getName()); @@ -264,7 +268,7 @@ public class FreeMarkerTest { ++countTimeSeries; - map.remove("timeseries"); + map.remove(Record.TIMESERIES_PROPERTY); } return countTimeSeries; diff --git a/src/test/java/org/gcube/grsf/publisher/record/RecordTest.java b/src/test/java/org/gcube/grsf/publisher/record/RecordTest.java index 447ba06..99b2008 100644 --- a/src/test/java/org/gcube/grsf/publisher/record/RecordTest.java +++ b/src/test/java/org/gcube/grsf/publisher/record/RecordTest.java @@ -23,6 +23,7 @@ import org.gcube.grsf.publisher.ckan.record.Record; import org.gcube.grsf.publisher.ckan.record.Stock; import org.gcube.grsf.publisher.ckan.record.TraceabilityUnit; import org.gcube.grsf.publisher.freemarker.FreeMarker; +import org.gcube.grsf.publisher.freemarker.FreeMarkerTest; import org.junit.Ignore; import org.junit.Test; import org.slf4j.Logger; @@ -172,12 +173,12 @@ public class RecordTest extends ContextTest { @Test public void testCreateRecords() throws Exception { - int maxTestRecords = 1000000000; - int maxTestRecordsPerSource = 3; + int maxTestRecords = 1; + int maxTestRecordsPerSource = 1; - String[] types = new String[] { "Fishery", "Stock", "Traceability Unit" }; - // String[] types = new String[] {"Fishery"}; + String[] types = new String[] { "Stock", "Fishery", "Traceability Unit" }; // String[] types = new String[] {"Stock"}; + // String[] types = new String[] {"Fishery"}; // String[] types = new String[] {"Traceability Unit"}; Calendar start = Calendar.getInstance(); @@ -231,7 +232,6 @@ public class RecordTest extends ContextTest { File[] jsonFiles = source.listFiles(filenameFilter); // File[] jsonFiles = new File[] {new File(source,"88818c3f-7120-322b-9637-7c7d2e9fc1e5.json")}; // File[] jsonFiles = new File[] {new File(source,"00702023-0e2d-345d-8b20-60580c107acd.json")}; - // File[] jsonFiles = new File[] {new File(source,"88818c3f-7120-322b-9637-7c7d2e9fc1e5.json")}; int countRecordPerSource = 0; @@ -252,11 +252,11 @@ public class RecordTest extends ContextTest { ObjectMapper mapper = new ObjectMapper(); ObjectNode input = (ObjectNode) mapper.readTree(jsonFile); - input.put("source", sourceString); + input.put(FreeMarkerTest.SOURCE_PROPERTY, sourceString); Record record = getRecordByType(type); - String grsfUUID = input.get("grsf_uuid").asText(); + String grsfUUID = input.get(Record.GRSF_UUID_PROPERTY).asText(); record.setName(grsfUUID); String ret = record.create(mapper.writeValueAsString(input)); logger.debug("{} {} with GRSF UUID {} created successfully\n{}", sourceString, type, grsfUUID, ret);