Implementing FreeMarker template

This commit is contained in:
Luca Frosini 2023-04-12 18:06:34 +02:00
parent 6fc93597f4
commit 83f9a45143
5 changed files with 259 additions and 188 deletions

View File

@ -3,8 +3,10 @@ package org.gcube.grsf.publisher.freemarker;
import java.io.File;
import java.io.FileWriter;
import java.io.Writer;
import java.util.HashMap;
import java.util.Map;
import org.gcube.com.fasterxml.jackson.databind.JsonNode;
import org.gcube.com.fasterxml.jackson.databind.ObjectMapper;
import org.junit.Test;
@ -52,4 +54,40 @@ public class FreeMarkerTest {
}
@Test
public void testTimeSeries() throws Exception {
FreeMarker freeMarker = new FreeMarker();
Template template = freeMarker.getTemplate("timeseries.ftl");
File assessmentUnitExamplesDir = getAssessmentUnitExampleDirectory();
String key = "catches";
for(File json : assessmentUnitExamplesDir.listFiles()) {
ObjectMapper mapper = new ObjectMapper();
Map map = mapper.readValue(json, Map.class);
if(!map.containsKey(key)) {
continue;
}
map.put("timeseries", map.get(key));
/**
* We can also use this in template
* <#assign uri_resolver_base_url="https://data.d4science.org/ctlg/GRSF_Admin">
* <#assign include_sensitive=true>
*/
File out = new File(FreeMarker.getResourcesDirectory(), json.getName()+"-"+key+".csv");
out.delete();
Writer writer = new FileWriter(out);
template.process(map, writer);
writer.flush();
writer.close();
}
}
}

View File

@ -1,199 +1,192 @@
<#macro group groupname var="" include=true sep=",">
<#if var?has_content && include>
{
"name": "${groupname}"
}${sep}
</#if>
</#macro>
<#macro tag tagname="" include=true sep=",">
<#if tagname?has_content && include>
{
"name": "${tagname}"
}${sep}
</#if>
</#macro>
<#macro tags list include=true sep=",">
<#if list?? && include>
<#list list as elem>
{
"name": "${elem}"
}${sep}
</#list>
</#if>
</#macro>
<#macro metadata key namespace="" var="" include=true sep=",">
<#if var?has_content && include>
{
"key": "<#if namespace?has_content>${namespace}:</#if>${key}",
"value": "${var}"
}${sep}
</#if>
</#macro>
<#macro metadatalist key namespace="" list="" include=true sep=",">
<#if list?has_content && include>
<#list list as elem>
{
"key": "<#if namespace?has_content>${namespace}:</#if>${key}",
"value": "${elem}"
}${sep}
</#list>
</#if>
</#macro>
<#macro timeseries key namespace="" timeseries="" include=include_sensitive sep=",">
<#if timeseries?has_content && include>
<#list timeseries[0..*5] as ts>
{
"key": "<#if namespace?has_content>${namespace}:</#if>${key}",
"value" "${ts.value} [<#if ts.unit?has_content>Unit: ${ts.unit} - </#if><#if ts.reporting_year_or_assessment_id?has_content>Rep. Year or Assessment ID: ${ts.reporting_year_or_assessment_id} - </#if><#if ts.reference_year?has_content>Ref. Year: ${ts.reference_year} - </#if><#if ts.data_owner?has_content>Data Owner: ${ts.data_owner} - </#if><#if ts.db_source?has_content>DB Source: ${ts.db_source}</#if>]"
}${sep}
</#list>
</#if>
</#macro>
<#macro resources var="" name="name" url="url" include=true sep=",">
<#if var?has_content>
<#list var as v>
{
"name": "${v[name]}",
"url": "${v[url]}"
}<#sep>${sep}</#sep>
</#list>
</#if>
</#macro>
<#assign record_url="${uri_resolver_base_url}/${grsf_uuid}" />
{
"title" : "${stock_name}",
"license_id": "${license_id}",
"description": "Short Name: ${short_name}\nGRSF Semantic Identifier: ${grsf_semantic_identifier}\nRecord URL: ${uri_resolver_base_url}/${grsf_uuid}",
"description": "Short Name: ${short_name}\nGRSF Semantic Identifier: ${grsf_semantic_identifier}\nRecord URL: ${record_url}",
"groups": [
<#if abundance_level??>
{
"name": "abundance-level-group"
},
</#if>
<#if firms_standard_abundance_level??>
{
"name": "abundance-level-firms-standard-group"
},
</#if>
<#if assessment_methods??>
{
"name": "assessment-method-group"
},
</#if>
<#if biomass??>
{
"name": "biomass-group"
},
</#if>
<#if catches??>
{
"name": "catch-group"
},
</#if>
{
"name": "stock-group"
}
<@group groupname="abundance-level-group" var=abundance_level />
<@group groupname="abundance-level-firms-standard-group" var=firms_standard_abundance_level />
<@group groupname="assessment-method-group" var=assessment_methods />
<@group groupname="biomass-group" var=biomass />
<@group groupname="catch-group" var=catches />
<@group groupname="fao-stock-status-category-group" var=fao_categories />
<@group groupname="fishing-pressure-group" var=fishing_pressure />
<@group groupname="fishing-pressure-firms-standard-group" var=firms_standard_fishing_pressure />
<@group groupname="landing-group" var=landings />
<@group groupname="scientific-advice-group" var=scientific_advice />
<@group groupname="state-and-trend-group" var=state_and_trend_of_marine_resources />
<@group groupname="grsf-sdg-flag-group" var=sdg_flag?c include=sdg_flag />
<@group groupname="stock-group" var="OK" sep=""/>
],
"tags" : [
<#if assessment_area??>
<#list assessment_area as aa>
{
"name": "${aa}"
},
</#list>
</#if>
{
"name": "${grsf_type?capitalize}"
},
<#if species??>
<#list species as s>
{
"name": "${s}"
},
</#list>
</#if>
{
"name": "${similarities_indicator}"
}
<@tags list=assessment_area />
<@tag tagname=grsf_type?capitalize />
<@tags list=species />
<@tag tagname=connections_indicator />
<@tag tagname=similarities_indicator sep=""/>
],
"resources": [
<#--
"refers_to": [
{
"id": "eb7ddeb0-ecc3-393b-b98a-fdb90be2c706",
"url": "https://data.d4science.org/ctlg/GRSF_Admin/eb7ddeb0-ecc3-393b-b98a-fdb90be2c706"
}
]
It generate the Resource with name FishSource which point to https://data.d4science.org/ctlg/GRSF_Admin/eb7ddeb0-ecc3-393b-b98a-fdb90be2c706
https://data.d4science.org/ctlg/GRSF_Admin/00a1f849-1b43-3fd1-9255-1d1134c969bb
According to ticket https://support.d4science.org/issues/24816 this field will be modified as following
"refers_to": [
{
"name": "FishSource",
"url": "https://data.d4science.org/ctlg/GRSF_Admin/eb7ddeb0-ecc3-393b-b98a-fdb90be2c706"
}
]
When changed use the following macro in place of the one using id
which is also wrong but it avoid code on server side for the moment
<@resources var=refers_to /><#if source_of_information??>,</#if>
-->
<@resources var=refers_to name="id" /><#if source_of_information??>,</#if>
<@resources var=source_of_information />
],
"extras": [
<#-- Stock Identity Section -->
<@metadata key="GRSF Stock Name" namespace="stock_identity" var=stock_name />
<@metadata key="Short Name" namespace="stock_identity" var=short_name />
<@metadata key="GRSF Semantic Identifier" namespace="stock_identity" var=grsf_semantic_identifier />
<@metadatalist key="Assessment Area" namespace="stock_identity" list=assessment_area />
<@metadatalist key="Connected Record" namespace="stock_identity" list=connected />
<@metadatalist key="Database Source" namespace="stock_identity" list=database_sources?map(database_source -> database_source.name) />
<#-- database_sources will change from list of objects to list of string. Use the line below in place of the line above-->
<#-- <@metadatalist key="Database Source" namespace="stock_identity" list=database_sources /> -->
<@metadata key="GRSF Type" namespace="stock_identity" var=grsf_type?capitalize />
<@metadatalist key="Similar GRSF Record" namespace="stock_identity" list=similar_grsf_record />
<@metadatalist key="Species" namespace="stock_identity" list=species />
<#-- END of Stock Identity Section -->
<#-- Stock Data Section -->
<@timeseries key="Abundance Level" namespace="stock_data" timeseries=abundance_level />
<@timeseries key="Abundance Level (FIRMS Standard)" namespace="stock_data" timeseries=firms_standard_abundance_level />
<@metadatalist key="Assessment Method" namespace="stock_data" list=assessment_methods include=include_sensitive />
<#--
assessment_methods will change from list of string to timeseries.
Add the following in the configuration
{
"key": "stock_identity:GRSF Stock Name",
"value": "${stock_name}"
},
{
"key": "stock_identity:Short Name",
"value": "${short_name}"
},
{
"key": "stock_identity:GRSF Semantic Identifier",
"value": "${grsf_semantic_identifier}"
},
{
"key": "stock_identity:GRSF Type",
"value": "${grsf_type?capitalize}"
"property": "assessment_methods",
"filename": "Assessment Method"
},
<#if assessment_area??>
<#list assessment_area as aa>
{
"key": "stock_identity:Assessment Area",
"value": "${aa}"
},
</#list>
</#if>
<#if connected??>
<#list connected as connected_record>
{
"key": "stock_identity:Connected Record",
"value": "${connected_record}"
},
</#list>
</#if>
<#if database_sources??>
<#list database_sources as database_source>
{
"key": "stock_identity:Database Source",
"value": "${database_source.name}"
},
</#list>
</#if>
<#if similar_grsf_record??>
<#list similar_grsf_record as sgr>
{
"key": "stock_identity:Similar GRSF Record",
"value": "${sgr}"
},
</#list>
</#if>
<#if species??>
<#list species as s>
{
"key": "stock_identity:stock_identity:Species",
"value": "${s}"
},
</#list>
</#if>
<#if abundance_level?? && include_sensitive>
<#list abundance_level[0..*5] as ts>
{
"key": "stock_data:Abundance Level",
"value" "${ts.value} [<#if ts.unit?has_content>Unit: ${ts.unit} - </#if><#if ts.reporting_year_or_assessment_id?has_content>Rep. Year or Assessment Id: ${ts.reporting_year_or_assessment_id} - </#if><#if ts.reference_year?has_content>Ref. Year: ${ts.reference_year} - </#if><#if ts.data_owner?has_content>Data Owner: ${ts.data_owner} - </#if><#if ts.db_source?has_content>DB Source: ${ts.db_source}</#if>]"
},
</#list>
</#if>
<#if firms_standard_abundance_level?? && include_sensitive>
<#list firms_standard_abundance_level[0..*5] as ts>
{
"key": "stock_data:Abundance Level (FIRMS Standard)",
"value" "${ts.value} [<#if ts.unit?has_content>Unit: ${ts.unit} - </#if><#if ts.reporting_year_or_assessment_id?has_content>Rep. Year or Assessment Id: ${ts.reporting_year_or_assessment_id} - </#if><#if ts.reference_year?has_content>Ref. Year: ${ts.reference_year} - </#if><#if ts.data_owner?has_content>Data Owner: ${ts.data_owner} - </#if><#if ts.db_source?has_content>DB Source: ${ts.db_source}</#if>]"
},
</#list>
</#if>
<#if assessment_methods?? && include_sensitive>
<#list assessment_methods as am>
{
"key": "stock_data:Assessment Method",
"value" "${am}"
},
</#list>
</#if>
<#if assessor?? && include_sensitive>
{
"key": "stock_data:Assessor",
"value" "${assessor}"
},
</#if>
Use the line below in place of the line above
-->
<#-- <@timeseries key="Assessment Method" namespace="stock_data" timeseries=assessment_methods /> -->
<@metadata key="Assessor" namespace="stock_data" var=assessor include=include_sensitive />
<@timeseries key="Biomass" namespace="stock_data" timeseries=biomass />
<@timeseries key="Catch" namespace="stock_data" timeseries=catches />
<@metadatalist key="Data Owner" namespace="stock_data" list=data_owner include=include_sensitive />
<@timeseries key="FAO Stock Status Category" namespace="stock_data" timeseries=fao_categories />
<@timeseries key="Fishing Pressure" namespace="stock_data" timeseries=fishing_pressure />
<@timeseries key="Fishing Pressure (FIRMS Standard)" namespace="stock_data" timeseries=firms_standard_fishing_pressure />
<@timeseries key="Landing" namespace="stock_data" timeseries=landings />
<@metadatalist key="Scientific Advice" namespace="stock_data" list=scientific_advice include=include_sensitive />
<#--
scientific_advice will change from list of string to timeseries.
<#if biomass?? && include_sensitive>
<#list biomass[0..*5] as ts>
Add the following in the configuration
{
"key": "stock_data:Biomass",
"value" "${ts.value} [<#if ts.unit?has_content>Unit: ${ts.unit} - </#if><#if ts.reporting_year_or_assessment_id?has_content>Rep. Year or Assessment Id: ${ts.reporting_year_or_assessment_id} - </#if><#if ts.reference_year?has_content>Ref. Year: ${ts.reference_year} - </#if><#if ts.data_owner?has_content>Data Owner: ${ts.data_owner} - </#if><#if ts.db_source?has_content>DB Source: ${ts.db_source}</#if>]"
"property": "scientific_advice",
"filename": "Scientific Advice"
},
</#list>
</#if>
<#if catches?? && include_sensitive>
<#list catches[0..*5] as ts>
{
"key": "stock_data:Catch",
"value" "${ts.value} [<#if ts.unit?has_content>Unit: ${ts.unit} - </#if><#if ts.reporting_year_or_assessment_id?has_content>Rep. Year or Assessment Id: ${ts.reporting_year_or_assessment_id} - </#if><#if ts.reference_year?has_content>Ref. Year: ${ts.reference_year} - </#if><#if ts.data_owner?has_content>Data Owner: ${ts.data_owner} - </#if><#if ts.db_source?has_content>DB Source: ${ts.db_source}</#if>]"
},
</#list>
</#if>
Use the line below in place of the line above
-->
<#-- <@timeseries key="Scientific Advice" namespace="stock_data" timeseries=scientific_advice /> -->
<@timeseries key="State and Trend" namespace="stock_data" timeseries=state_and_trend_of_marine_resources />
<#-- END of Stock Data Section-->
<#if data_owner?? && include_sensitive>
<#list data_owner as do>
{
"key": "stock_data:Data Owner",
"value" "${do}"
},
</#list>
</#if>
{
"key": "Record URL",
"value": "${uri_resolver_base_url}/${grsf_uuid}"
},
{
"key": "system:type",
"value": "${grsf_type?capitalize}"
}
<#-- Record Info Section -->
<@metadata key="Annotation" var=annotations include=include_sensitive />
<@metadata key="Citation" var=citation />
<@metadata key="Domain" var="Stock" />
<@metadata key="spatial" var=spatial?json_string />
<@metadata key="GRSF UUID" var=grsf_uuid />
<@metadata key="Record URL" var=record_url />
<@metadata key="SDG Flag" var=sdg_flag?c />
<@metadata key="Status of the Record" var=status_grsf_record />
<@metadata key="system:type" var=grsf_type?capitalize sep=""/>
<#-- END of Record Info Section -->
]
}

View File

@ -0,0 +1,5 @@
<#assign keys = timeseries[0]?keys>
${keys?join(",")}
<#list timeseries as item>
<#list keys as key>${item[key]?c}<#sep>,</#sep></#list>
</#list>

View File

@ -11,7 +11,42 @@
}
]
},
"timeseries": ["abundance_level", "firms_standard_abundance_level", "biomass"],
"resources": ["assessment_methods"]
"timeseries": [
{
"property": "abundance_level",
"filename": "Abundance Level"
},
{
"property": "firms_standard_abundance_level",
"filename": "Abundance Level (FIRMS Standard)"
},
{
"property": "biomass",
"filename": "Biomass"
},
{
"property": "catches",
"filename": "Catch"
},
{
"property": "fao_categories",
"filename": "FAO Stock Status Category"
},
{
"property": "fishing_pressure",
"filename": "Fishing Pressure"
},
{
"property": "firms_standard_fishing_pressure",
"filename": "Fishing Pressure (FIRMS Standard)"
},
{
"property": "landings",
"filename": "Landing"
},
{
"property": "state_and_trend_of_marine_resources",
"filename": "State and Trend"
}
]
}

View File

@ -982,7 +982,7 @@
</dstTypes>
<comments>
It generates
- a Resource with name 'Abundance Level' containing a CSV where
- a Resource with name 'Catch' containing a CSV where
- the first row is the CSV header which is generated by taking the key of the element
of the JsonArray.
- the subsequent rows are generated with the value of each element of the JSONArray.
@ -1270,7 +1270,7 @@
</metadatafield>
<metadatafield categoryref="stock_data">
<fieldName>Scientific Advice </fieldName>
<fieldName>Scientific Advice</fieldName>
<mandatory>false</mandatory>
<dataType>String</dataType>
<groups>
@ -1675,7 +1675,7 @@
<type>String</type>
</field>
<field>
<id>name</id>
<id>description</id>
<type>String</type>
</field>
<field>
@ -1692,7 +1692,7 @@
<comments>
A list of elements of the type {"name": "a name", "description": "a description", "url": "http://...."}.
Name and url are mandatory.
It generates a Resource with as name the value of name of the element of the array and as content the value of the urlof the element of the array.
It generates a Resource with as name the value of name of the element of the array and as content the value of the url of the element of the array.
E.g. Resource Name: "a name" - Resource Content: "http://...."
No metadata field is created.