diff --git a/src/test/java/org/gcube/grsf/publisher/freemarker/FreeMarkerTest.java b/src/test/java/org/gcube/grsf/publisher/freemarker/FreeMarkerTest.java index 4917aea..ead20ab 100644 --- a/src/test/java/org/gcube/grsf/publisher/freemarker/FreeMarkerTest.java +++ b/src/test/java/org/gcube/grsf/publisher/freemarker/FreeMarkerTest.java @@ -42,32 +42,39 @@ public class FreeMarkerTest { int countRecord = 0; int countTimeSeries = 0; + File outputDir = new File(FreeMarker.getResourcesDirectory(), "output"); + outputDir.mkdir(); + + FilenameFilter dirnameFilter = new FilenameFilter() { + @Override + public boolean accept(File dir, String name) { + File f = new File(dir, name); + return f.isDirectory(); + } + }; + + FilenameFilter filenameFilter = new FilenameFilter() { + @Override + public boolean accept(File dir, String name) { + return name.toLowerCase().endsWith(".json"); + } + }; + for(String type : types) { Template template = freeMarker.getTemplate(type + ".ftl"); File examplesDir = getExampleDirectory(type); - File outDir = new File(FreeMarker.getResourcesDirectory(), type + "-out"); - outDir.mkdir(); - - FilenameFilter dirnameFilter = new FilenameFilter() { - @Override - public boolean accept(File dir, String name) { - return dir.isDirectory(); - } - }; - - FilenameFilter filenameFilter = new FilenameFilter() { - @Override - public boolean accept(File dir, String name) { - return name.toLowerCase().endsWith(".json"); - } - }; + File outputTypeDir = new File(outputDir, type); + outputTypeDir.mkdir(); for(File source : examplesDir.listFiles(dirnameFilter)){ String sourceString = source.getName(); + File outputSourceDir = new File(outputTypeDir, sourceString); + outputSourceDir.mkdir(); + for(File jsonFile : source.listFiles(filenameFilter)) { ObjectMapper mapper = new ObjectMapper(); Map map = mapper.readValue(jsonFile, Map.class); @@ -83,7 +90,7 @@ public class FreeMarkerTest { * <#assign uri_resolver_base_url="https://data.d4science.org/ctlg/GRSF_Admin"> * <#assign include_sensitive=true> */ - File out = new File(outDir, jsonFile.getName()); + File out = new File(outputSourceDir, jsonFile.getName()); out.delete(); Writer writer = new FileWriter(out); @@ -103,7 +110,7 @@ public class FreeMarkerTest { map.put("timeseries", map.get(key)); - File tsOut = new File(outDir, jsonFile.getName().replace(".json","")+"-"+fileName+".csv"); + File tsOut = new File(outputSourceDir, jsonFile.getName().replace(".json","")+"-"+fileName+".csv"); tsOut.delete(); Writer tsWriter = new FileWriter(tsOut); @@ -121,8 +128,8 @@ public class FreeMarkerTest { } Calendar end = Calendar.getInstance(); - - logger.info("Generated {} records and {} CSV files from TimeSeries in {} seconds", countRecord, countTimeSeries, TimeUnit.MILLISECONDS.toSeconds(end.getTimeInMillis() - start.getTimeInMillis())); + long diff= end.getTimeInMillis() - start.getTimeInMillis(); + logger.info("Generated {} records and {} CSV files from TimeSeries in {} milliseconds (~{} seconds)", countRecord, countTimeSeries, diff, TimeUnit.MILLISECONDS.toSeconds(diff)); } } diff --git a/src/test/resources/freemarker/Fishery.ftl b/src/test/resources/freemarker/Fishery.ftl index fda5636..3ee37fc 100644 --- a/src/test/resources/freemarker/Fishery.ftl +++ b/src/test/resources/freemarker/Fishery.ftl @@ -1,11 +1,10 @@ <#include "macros.ftl"> <#assign identity_namespace="fishery_identity" > <#assign data_namespace="fishery_data" > -<#assign record_url="${uri_resolver_base_url}/${grsf_uuid}" /> { "title" : "${fishery_name}", "license_id": "${license_id}", - "description": "Short Name: ${short_name}\n<#if grsf_semantic_identifier??>GRSF Semantic Identifier: ${grsf_semantic_identifier}\nRecord URL: ${record_url}", + "description": "${description?json_string}", "extras": [ <#-- START of Common Metadata --> <@metadata key="GRSF Type" namespace=identity_namespace var=grsf_type?capitalize /> @@ -14,6 +13,14 @@ <@metadatalist key="Database Source" namespace=identity_namespace list=database_sources?map(database_source -> database_source.name) /> <#-- database_sources will change from list of objects to list of string. Use the line below in place of the line above--> <#-- <@metadatalist key="Database Source" namespace=identity_namespace list=database_sources /> --> + <#if include_sensitive> + <#-- We are in Administrative VRE e.g. GRSF_Admin --> + <#list database_sources as database_source> + <#assign group_list += [{"name":"${database_source.name?lower_case?json_string}-group", "title":"${database_source_id_to_name[database_source.name?lower_case]?json_string}"}] > + <#-- database_sources will change from list of objects to list of string. Use the line below in place of the line above--> + <#-- <#assign group_list += [{"name":"${database_source_name_to_id[database_source?json_string]}-group", "title":"${database_source?json_string]}"}] > --> + + <@metadatalist key="Species" namespace=identity_namespace list=species tagValues=true/> <@metadatalist key="Similar GRSF Record" namespace=identity_namespace list=similar_grsf_record /> @@ -27,7 +34,7 @@ <#if sdg_flag??> <@metadata key="SDG Flag" var=sdg_flag?c /> <#if sdg_flag> - <#assign groupList += [{"name":"grsf-sdg-flag-group", "title":"GRSF SDG Flag"}] > + <#assign group_list += [{"name":"grsf-sdg-flag-group", "title":"GRSF SDG Flag"}] > @@ -45,7 +52,7 @@ <#if traceability_flag??> <@metadata key="Traceability Flag" var=traceability_flag?c /> <#if traceability_flag> - <#assign groupList += [{"name":"grsf-traceability-flag-group", "title":"GRSF Traceability Flag"}] > + <#assign group_list += [{"name":"grsf-traceability-flag-group", "title":"GRSF Traceability Flag"}] > <@metadatalist key="Fishing Area" namespace=identity_namespace list=fishing_area tagValues=true /> @@ -60,8 +67,9 @@ <#-- TODO Set Legacy if source is not GRSF --> <@metadata key="system:type" var=grsf_type?capitalize sep=""/> ], + "organization": <@source_macro /> "groups": [ - <#list groupList as group> + <#list group_list as group> { "name": "${group.name}", "title": "${group.title}" @@ -72,7 +80,7 @@ <@group name="fishery-group" title="Fishery" var="OK" sep=""/> ], "tags": [ - <#list tagList as tag> + <#list tag_list as tag> { "name": "${tag}" }, @@ -111,7 +119,7 @@ --> ], "timeseries": [ - <#list timeseriesList as elem> + <#list timeseries_list as elem> { <#list elem as key, value> "${key}":"${value}"<#sep>, diff --git a/src/test/resources/freemarker/Stock.ftl b/src/test/resources/freemarker/Stock.ftl index 042a40b..dfb7b67 100644 --- a/src/test/resources/freemarker/Stock.ftl +++ b/src/test/resources/freemarker/Stock.ftl @@ -1,7 +1,6 @@ <#include "macros.ftl"> <#assign identity_namespace="stock_identity" > <#assign data_namespace="stock_data" > -<#assign record_url="${uri_resolver_base_url}/${grsf_uuid}" /> { "title" : "${stock_name}", "license_id": "${license_id}", @@ -27,7 +26,7 @@ <#if sdg_flag??> <@metadata key="SDG Flag" var=sdg_flag?c /> <#if sdg_flag> - <#assign groupList += [{"name":"grsf-sdg-flag-group", "title":"GRSF SDG Flag"}] > + <#assign group_list += [{"name":"grsf-sdg-flag-group", "title":"GRSF SDG Flag"}] > <@metadata key="Status of the Record" var=status_grsf_record /> @@ -83,8 +82,9 @@ <#-- TODO Set Legacy if source is not GRSF --> <@metadata key="system:type" var=grsf_type?capitalize sep=""/> ], + "organization": <@source_macro /> "groups": [ - <#list groupList as group> + <#list group_list as group> { "name": "${group.name}", "title": "${group.title}" @@ -96,7 +96,7 @@ <@group name="stock-group" title="Stock" var="OK" sep=""/> ], "tags": [ - <#list tagList as tag> + <#list tag_list as tag> { "name": "${tag}" }, @@ -135,7 +135,7 @@ --> ], "timeseries": [ - <#list timeseriesList as elem> + <#list timeseries_list as elem> { <#list elem as key, value> "${key}":"${value}"<#sep>, diff --git a/src/test/resources/freemarker/TraceabilityUnit.ftl b/src/test/resources/freemarker/TraceabilityUnit.ftl index 8b09cd5..11f593a 100644 --- a/src/test/resources/freemarker/TraceabilityUnit.ftl +++ b/src/test/resources/freemarker/TraceabilityUnit.ftl @@ -1,7 +1,6 @@ <#include "macros.ftl"> <#assign identity_namespace="traceability_unit_identity" > <#assign data_namespace="traceability_unit_data" > -<#assign record_url="${uri_resolver_base_url}/${grsf_uuid}" /> { "title" : "${traceability_unit_name}", "license_id": "${license_id}", @@ -30,8 +29,9 @@ <@metadata key="Record URL" var=record_url /> <@metadata key="system:type" var="Traceability Unit" sep=""/> ], + "organization": <@source_macro /> "groups": [ - <#list groupList as group> + <#list group_list as group> { "name": "${group.name}", "title": "${group.title}" @@ -40,7 +40,7 @@ <@group name="traceability-unit-group" title="Traceability Unit" var="OK" sep=""/> ], "tags": [ - <#list tagList as tag> + <#list tag_list as tag> { "name": "${tag}" }, @@ -53,7 +53,7 @@ <@resource name="Traceability Unit URI" url=traceability_record_uri sep=""/> ], "timeseries": [ - <#list timeseriesList as elem> + <#list timeseries_list as elem> { <#list elem as key, value> "${key}":"${value}"<#sep>, diff --git a/src/test/resources/freemarker/macros.ftl b/src/test/resources/freemarker/macros.ftl index 410bbe3..8277eff 100644 --- a/src/test/resources/freemarker/macros.ftl +++ b/src/test/resources/freemarker/macros.ftl @@ -1,6 +1,21 @@ -<#assign timeseriesList = [] > -<#assign groupList = [] > -<#assign tagList = [] > +<#assign timeseries_list = [] > +<#assign group_list = [] > +<#assign tag_list = [] > + +<#assign database_source_id_to_name = { "grsf":"Global Record of Stocks and Fisheries (GRSF)", "ram":"RAM Legacy Stock Assessment Database", "firms":"Fisheries and Resources Monitoring System (FIRMS)", "fishsource":"FishSource", "sdg":"FAO SDG 14.4.1 Questionnaire"} > + +<#assign database_source_name_to_id = {} > +<#list database_source_id_to_name as key, value> + <#assign database_source_name_to_id += { "${value?json_string}" : "${key?json_string}" } > + + +<#assign record_url="${uri_resolver_base_url}/${grsf_uuid}" /> + +<#assign description = "Short Name: ${short_name}\n" > +<#if grsf_semantic_identifier??> + <#assign description += "GRSF Semantic Identifier: ${grsf_semantic_identifier}\n" > + +<#assign description += "Record URL: ${record_url}" > <#macro group name title var="" include=true sep=","> <#if var?has_content && include> @@ -55,7 +70,7 @@ <#if list?has_content && include> <#list list as elem> <#if tagValues> - <#assign tagList += ["${elem?json_string}"]> + <#assign tag_list += ["${elem?json_string}"]> { "key": "<#if namespace?has_content>${namespace}:${key}", @@ -78,9 +93,9 @@ <#macro timeseries key namespace="" timeseries="" groupname="" include=include_sensitive sep=","> <#if timeseries?has_content && .data_model[timeseries]??> - <#assign timeseriesList += [{"property":"${timeseries}","filename":"${key}"}]> + <#assign timeseries_list += [{"property":"${timeseries}","filename":"${key}"}]> <#if groupname?has_content> - <#assign groupList += [{"name":"${groupname}", "title":"${key}"}] > + <#assign group_list += [{"name":"${groupname}", "title":"${key}"}] > <#if include> <#local list = .data_model[timeseries] > @@ -116,3 +131,17 @@ }${sep} + + +<#macro source_macro sep=","> + <#local source_title = "${database_source_id_to_name[source]}" > + <#if source = "grsf"> + <#assign group_list += [{"name":"${source}", "title":"${source_title}"}] > + <#else> + <#assign group_list += [{"name":"legacy-group", "title":"Legacy"}] > + + { + "name": "${source}", + "title": "${source_title}" + }${sep} + \ No newline at end of file