gCubeSystem
/
grsf-publisher
15
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Improved solution

This commit is contained in:
Luca Frosini 2023-04-18 12:07:29 +02:00
parent 399d968062
commit 3e0778e3f7
5 changed files with 86 additions and 42 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

47
src/test/java/org/gcube/grsf/publisher/freemarker/FreeMarkerTest.java
View File

@ -42,32 +42,39 @@ public class FreeMarkerTest {
int countRecord = 0;
int countTimeSeries = 0;
File outputDir = new File(FreeMarker.getResourcesDirectory(), "output");
outputDir.mkdir();
FilenameFilter dirnameFilter = new FilenameFilter() {
@Override
public boolean accept(File dir, String name) {
File f = new File(dir, name);
return f.isDirectory();
}
};
FilenameFilter filenameFilter = new FilenameFilter() {
@Override
public boolean accept(File dir, String name) {
return name.toLowerCase().endsWith(".json");
}
};
for(String type : types) {
Template template = freeMarker.getTemplate(type + ".ftl");
File examplesDir = getExampleDirectory(type);
File outDir = new File(FreeMarker.getResourcesDirectory(), type + "-out");
outDir.mkdir();
FilenameFilter dirnameFilter = new FilenameFilter() {
@Override
public boolean accept(File dir, String name) {
return dir.isDirectory();
}
};
FilenameFilter filenameFilter = new FilenameFilter() {
@Override
public boolean accept(File dir, String name) {
return name.toLowerCase().endsWith(".json");
}
};
File outputTypeDir = new File(outputDir, type);
outputTypeDir.mkdir();
for(File source : examplesDir.listFiles(dirnameFilter)){
String sourceString = source.getName();
File outputSourceDir = new File(outputTypeDir, sourceString);
outputSourceDir.mkdir();
for(File jsonFile : source.listFiles(filenameFilter)) {
ObjectMapper mapper = new ObjectMapper();
Map map = mapper.readValue(jsonFile, Map.class);
@ -83,7 +90,7 @@ public class FreeMarkerTest {
* <#assign uri_resolver_base_url="https://data.d4science.org/ctlg/GRSF_Admin">
* <#assign include_sensitive=true>
*/
File out = new File(outDir, jsonFile.getName());
File out = new File(outputSourceDir, jsonFile.getName());
out.delete();
Writer writer = new FileWriter(out);
@ -103,7 +110,7 @@ public class FreeMarkerTest {
map.put("timeseries", map.get(key));
File tsOut = new File(outDir, jsonFile.getName().replace(".json","")+"-"+fileName+".csv");
File tsOut = new File(outputSourceDir, jsonFile.getName().replace(".json","")+"-"+fileName+".csv");
tsOut.delete();
Writer tsWriter = new FileWriter(tsOut);
@ -121,8 +128,8 @@ public class FreeMarkerTest {
}
Calendar end = Calendar.getInstance();
logger.info("Generated {} records and {} CSV files from TimeSeries in {} seconds", countRecord, countTimeSeries, TimeUnit.MILLISECONDS.toSeconds(end.getTimeInMillis() - start.getTimeInMillis()));
long diff= end.getTimeInMillis() - start.getTimeInMillis();
logger.info("Generated {} records and {} CSV files from TimeSeries in {} milliseconds (~{} seconds)", countRecord, countTimeSeries, diff, TimeUnit.MILLISECONDS.toSeconds(diff));
}
}

22
src/test/resources/freemarker/Fishery.ftl
View File

@ -1,11 +1,10 @@
<#include "macros.ftl">
<#assign identity_namespace="fishery_identity" >
<#assign data_namespace="fishery_data" >
<#assign record_url="${uri_resolver_base_url}/${grsf_uuid}" />
{
"title" : "${fishery_name}",
"license_id": "${license_id}",
"description": "Short Name: ${short_name}\n<#if grsf_semantic_identifier??>GRSF Semantic Identifier: ${grsf_semantic_identifier}\n</#if>Record URL: ${record_url}",
"description": "${description?json_string}",
"extras": [
<#-- START of Common Metadata -->
<@metadata key="GRSF Type" namespace=identity_namespace var=grsf_type?capitalize />
@ -14,6 +13,14 @@
<@metadatalist key="Database Source" namespace=identity_namespace list=database_sources?map(database_source -> database_source.name) />
<#-- database_sources will change from list of objects to list of string. Use the line below in place of the line above-->
<#-- <@metadatalist key="Database Source" namespace=identity_namespace list=database_sources /> -->
<#if include_sensitive>
<#-- We are in Administrative VRE e.g. GRSF_Admin -->
<#list database_sources as database_source>
<#assign group_list += [{"name":"${database_source.name?lower_case?json_string}-group", "title":"${database_source_id_to_name[database_source.name?lower_case]?json_string}"}] >
<#-- database_sources will change from list of objects to list of string. Use the line below in place of the line above-->
<#-- <#assign group_list += [{"name":"${database_source_name_to_id[database_source?json_string]}-group", "title":"${database_source?json_string]}"}] > -->
</#list>
</#if>
</#if>
<@metadatalist key="Species" namespace=identity_namespace list=species tagValues=true/>
<@metadatalist key="Similar GRSF Record" namespace=identity_namespace list=similar_grsf_record />
@ -27,7 +34,7 @@
<#if sdg_flag??>
<@metadata key="SDG Flag" var=sdg_flag?c />
<#if sdg_flag>
<#assign groupList += [{"name":"grsf-sdg-flag-group", "title":"GRSF SDG Flag"}] >
<#assign group_list += [{"name":"grsf-sdg-flag-group", "title":"GRSF SDG Flag"}] >
</#if>
</#if>
@ -45,7 +52,7 @@
<#if traceability_flag??>
<@metadata key="Traceability Flag" var=traceability_flag?c />
<#if traceability_flag>
<#assign groupList += [{"name":"grsf-traceability-flag-group", "title":"GRSF Traceability Flag"}] >
<#assign group_list += [{"name":"grsf-traceability-flag-group", "title":"GRSF Traceability Flag"}] >
</#if>
</#if>
<@metadatalist key="Fishing Area" namespace=identity_namespace list=fishing_area tagValues=true />
@ -60,8 +67,9 @@
<#-- TODO Set Legacy if source is not GRSF -->
<@metadata key="system:type" var=grsf_type?capitalize sep=""/>
],
"organization": <@source_macro />
"groups": [
<#list groupList as group>
<#list group_list as group>
{
"name": "${group.name}",
"title": "${group.title}"
@ -72,7 +80,7 @@
<@group name="fishery-group" title="Fishery" var="OK" sep=""/>
],
"tags": [
<#list tagList as tag>
<#list tag_list as tag>
{
"name": "${tag}"
},
@ -111,7 +119,7 @@
-->
],
"timeseries": [
<#list timeseriesList as elem>
<#list timeseries_list as elem>
{
<#list elem as key, value>
"${key}":"${value}"<#sep>,</#sep>

10
src/test/resources/freemarker/Stock.ftl
View File

@ -1,7 +1,6 @@
<#include "macros.ftl">
<#assign identity_namespace="stock_identity" >
<#assign data_namespace="stock_data" >
<#assign record_url="${uri_resolver_base_url}/${grsf_uuid}" />
{
"title" : "${stock_name}",
"license_id": "${license_id}",
@ -27,7 +26,7 @@
<#if sdg_flag??>
<@metadata key="SDG Flag" var=sdg_flag?c />
<#if sdg_flag>
<#assign groupList += [{"name":"grsf-sdg-flag-group", "title":"GRSF SDG Flag"}] >
<#assign group_list += [{"name":"grsf-sdg-flag-group", "title":"GRSF SDG Flag"}] >
</#if>
</#if>
<@metadata key="Status of the Record" var=status_grsf_record />
@ -83,8 +82,9 @@
<#-- TODO Set Legacy if source is not GRSF -->
<@metadata key="system:type" var=grsf_type?capitalize sep=""/>
],
"organization": <@source_macro />
"groups": [
<#list groupList as group>
<#list group_list as group>
{
"name": "${group.name}",
"title": "${group.title}"
@ -96,7 +96,7 @@
<@group name="stock-group" title="Stock" var="OK" sep=""/>
],
"tags": [
<#list tagList as tag>
<#list tag_list as tag>
{
"name": "${tag}"
},
@ -135,7 +135,7 @@
-->
],
"timeseries": [
<#list timeseriesList as elem>
<#list timeseries_list as elem>
{
<#list elem as key, value>
"${key}":"${value}"<#sep>,</#sep>

8
src/test/resources/freemarker/TraceabilityUnit.ftl
View File

@ -1,7 +1,6 @@
<#include "macros.ftl">
<#assign identity_namespace="traceability_unit_identity" >
<#assign data_namespace="traceability_unit_data" >
<#assign record_url="${uri_resolver_base_url}/${grsf_uuid}" />
{
"title" : "${traceability_unit_name}",
"license_id": "${license_id}",
@ -30,8 +29,9 @@
<@metadata key="Record URL" var=record_url />
<@metadata key="system:type" var="Traceability Unit" sep=""/>
],
"organization": <@source_macro />
"groups": [
<#list groupList as group>
<#list group_list as group>
{
"name": "${group.name}",
"title": "${group.title}"
@ -40,7 +40,7 @@
<@group name="traceability-unit-group" title="Traceability Unit" var="OK" sep=""/>
],
"tags": [
<#list tagList as tag>
<#list tag_list as tag>
{
"name": "${tag}"
},
@ -53,7 +53,7 @@
<@resource name="Traceability Unit URI" url=traceability_record_uri sep=""/>
],
"timeseries": [
<#list timeseriesList as elem>
<#list timeseries_list as elem>
{
<#list elem as key, value>
"${key}":"${value}"<#sep>,</#sep>

41
src/test/resources/freemarker/macros.ftl
View File

@ -1,6 +1,21 @@
<#assign timeseriesList = [] >
<#assign groupList = [] >
<#assign tagList = [] >
<#assign timeseries_list = [] >
<#assign group_list = [] >
<#assign tag_list = [] >
<#assign database_source_id_to_name = { "grsf":"Global Record of Stocks and Fisheries (GRSF)", "ram":"RAM Legacy Stock Assessment Database", "firms":"Fisheries and Resources Monitoring System (FIRMS)", "fishsource":"FishSource", "sdg":"FAO SDG 14.4.1 Questionnaire"} >
<#assign database_source_name_to_id = {} >
<#list database_source_id_to_name as key, value>
<#assign database_source_name_to_id += { "${value?json_string}" : "${key?json_string}" } >
</#list>
<#assign record_url="${uri_resolver_base_url}/${grsf_uuid}" />
<#assign description = "Short Name: ${short_name}\n" >
<#if grsf_semantic_identifier??>
<#assign description += "GRSF Semantic Identifier: ${grsf_semantic_identifier}\n" >
</#if>
<#assign description += "Record URL: ${record_url}" >
<#macro group name title var="" include=true sep=",">
<#if var?has_content && include>
@ -55,7 +70,7 @@
<#if list?has_content && include>
<#list list as elem>
<#if tagValues>
<#assign tagList += ["${elem?json_string}"]>
<#assign tag_list += ["${elem?json_string}"]>
</#if>
{
"key": "<#if namespace?has_content>${namespace}:</#if>${key}",
@ -78,9 +93,9 @@
<#macro timeseries key namespace="" timeseries="" groupname="" include=include_sensitive sep=",">
<#if timeseries?has_content && .data_model[timeseries]??>
<#assign timeseriesList += [{"property":"${timeseries}","filename":"${key}"}]>
<#assign timeseries_list += [{"property":"${timeseries}","filename":"${key}"}]>
<#if groupname?has_content>
<#assign groupList += [{"name":"${groupname}", "title":"${key}"}] >
<#assign group_list += [{"name":"${groupname}", "title":"${key}"}] >
</#if>
<#if include>
<#local list = .data_model[timeseries] >
@ -116,3 +131,17 @@
}${sep}
</#if>
</#macro>
<#macro source_macro sep=",">
<#local source_title = "${database_source_id_to_name[source]}" >
<#if source = "grsf">
<#assign group_list += [{"name":"${source}", "title":"${source_title}"}] >
<#else>
<#assign group_list += [{"name":"legacy-group", "title":"Legacy"}] >
</#if>
{
"name": "${source}",
"title": "${source_title}"
}${sep}
</#macro>