gCubeSystem
/
grsf-publisher
15
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Improved solution

This commit is contained in:
Luca Frosini 2023-04-18 12:07:29 +02:00
parent 399d968062
commit 3e0778e3f7
5 changed files with 86 additions and 42 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

31
src/test/java/org/gcube/grsf/publisher/freemarker/FreeMarkerTest.java
View File

@ -42,18 +42,14 @@ public class FreeMarkerTest {
int countRecord = 0; int countRecord = 0;
int countTimeSeries = 0; int countTimeSeries = 0;
for(String type : types) { File outputDir = new File(FreeMarker.getResourcesDirectory(), "output");
Template template = freeMarker.getTemplate(type + ".ftl"); outputDir.mkdir();
File examplesDir = getExampleDirectory(type);
File outDir = new File(FreeMarker.getResourcesDirectory(), type + "-out");
outDir.mkdir();
FilenameFilter dirnameFilter = new FilenameFilter() { FilenameFilter dirnameFilter = new FilenameFilter() {
@Override @Override
public boolean accept(File dir, String name) { public boolean accept(File dir, String name) {
return dir.isDirectory(); File f = new File(dir, name);
return f.isDirectory();
} }
}; };
@ -64,10 +60,21 @@ public class FreeMarkerTest {
} }
}; };
for(String type : types) {
Template template = freeMarker.getTemplate(type + ".ftl");
File examplesDir = getExampleDirectory(type);
File outputTypeDir = new File(outputDir, type);
outputTypeDir.mkdir();
for(File source : examplesDir.listFiles(dirnameFilter)){ for(File source : examplesDir.listFiles(dirnameFilter)){
String sourceString = source.getName(); String sourceString = source.getName();
File outputSourceDir = new File(outputTypeDir, sourceString);
outputSourceDir.mkdir();
for(File jsonFile : source.listFiles(filenameFilter)) { for(File jsonFile : source.listFiles(filenameFilter)) {
ObjectMapper mapper = new ObjectMapper(); ObjectMapper mapper = new ObjectMapper();
Map map = mapper.readValue(jsonFile, Map.class); Map map = mapper.readValue(jsonFile, Map.class);
@ -83,7 +90,7 @@ public class FreeMarkerTest {
* <#assign uri_resolver_base_url="https://data.d4science.org/ctlg/GRSF_Admin"> * <#assign uri_resolver_base_url="https://data.d4science.org/ctlg/GRSF_Admin">
* <#assign include_sensitive=true> * <#assign include_sensitive=true>
*/ */
File out = new File(outDir, jsonFile.getName()); File out = new File(outputSourceDir, jsonFile.getName());
out.delete(); out.delete();
Writer writer = new FileWriter(out); Writer writer = new FileWriter(out);
@ -103,7 +110,7 @@ public class FreeMarkerTest {
map.put("timeseries", map.get(key)); map.put("timeseries", map.get(key));
File tsOut = new File(outDir, jsonFile.getName().replace(".json","")+"-"+fileName+".csv"); File tsOut = new File(outputSourceDir, jsonFile.getName().replace(".json","")+"-"+fileName+".csv");
tsOut.delete(); tsOut.delete();
Writer tsWriter = new FileWriter(tsOut); Writer tsWriter = new FileWriter(tsOut);
@ -121,8 +128,8 @@ public class FreeMarkerTest {
} }
Calendar end = Calendar.getInstance(); Calendar end = Calendar.getInstance();
long diff= end.getTimeInMillis() - start.getTimeInMillis();
logger.info("Generated {} records and {} CSV files from TimeSeries in {} seconds", countRecord, countTimeSeries, TimeUnit.MILLISECONDS.toSeconds(end.getTimeInMillis() - start.getTimeInMillis())); logger.info("Generated {} records and {} CSV files from TimeSeries in {} milliseconds (~{} seconds)", countRecord, countTimeSeries, diff, TimeUnit.MILLISECONDS.toSeconds(diff));
} }
} }

22
src/test/resources/freemarker/Fishery.ftl
View File

@ -1,11 +1,10 @@
<#include "macros.ftl"> <#include "macros.ftl">
<#assign identity_namespace="fishery_identity" > <#assign identity_namespace="fishery_identity" >
<#assign data_namespace="fishery_data" > <#assign data_namespace="fishery_data" >
<#assign record_url="${uri_resolver_base_url}/${grsf_uuid}" />
{ {
"title" : "${fishery_name}", "title" : "${fishery_name}",
"license_id": "${license_id}", "license_id": "${license_id}",
"description": "Short Name: ${short_name}\n<#if grsf_semantic_identifier??>GRSF Semantic Identifier: ${grsf_semantic_identifier}\n</#if>Record URL: ${record_url}", "description": "${description?json_string}",
"extras": [ "extras": [
<#-- START of Common Metadata --> <#-- START of Common Metadata -->
<@metadata key="GRSF Type" namespace=identity_namespace var=grsf_type?capitalize /> <@metadata key="GRSF Type" namespace=identity_namespace var=grsf_type?capitalize />
@ -14,6 +13,14 @@
<@metadatalist key="Database Source" namespace=identity_namespace list=database_sources?map(database_source -> database_source.name) /> <@metadatalist key="Database Source" namespace=identity_namespace list=database_sources?map(database_source -> database_source.name) />
<#-- database_sources will change from list of objects to list of string. Use the line below in place of the line above--> <#-- database_sources will change from list of objects to list of string. Use the line below in place of the line above-->
<#-- <@metadatalist key="Database Source" namespace=identity_namespace list=database_sources /> --> <#-- <@metadatalist key="Database Source" namespace=identity_namespace list=database_sources /> -->
<#if include_sensitive>
<#-- We are in Administrative VRE e.g. GRSF_Admin -->
<#list database_sources as database_source>
<#assign group_list += [{"name":"${database_source.name?lower_case?json_string}-group", "title":"${database_source_id_to_name[database_source.name?lower_case]?json_string}"}] >
<#-- database_sources will change from list of objects to list of string. Use the line below in place of the line above-->
<#-- <#assign group_list += [{"name":"${database_source_name_to_id[database_source?json_string]}-group", "title":"${database_source?json_string]}"}] > -->
</#list>
</#if>
</#if> </#if>
<@metadatalist key="Species" namespace=identity_namespace list=species tagValues=true/> <@metadatalist key="Species" namespace=identity_namespace list=species tagValues=true/>
<@metadatalist key="Similar GRSF Record" namespace=identity_namespace list=similar_grsf_record /> <@metadatalist key="Similar GRSF Record" namespace=identity_namespace list=similar_grsf_record />
@ -27,7 +34,7 @@
<#if sdg_flag??> <#if sdg_flag??>
<@metadata key="SDG Flag" var=sdg_flag?c /> <@metadata key="SDG Flag" var=sdg_flag?c />
<#if sdg_flag> <#if sdg_flag>
<#assign groupList += [{"name":"grsf-sdg-flag-group", "title":"GRSF SDG Flag"}] > <#assign group_list += [{"name":"grsf-sdg-flag-group", "title":"GRSF SDG Flag"}] >
</#if> </#if>
</#if> </#if>
@ -45,7 +52,7 @@
<#if traceability_flag??> <#if traceability_flag??>
<@metadata key="Traceability Flag" var=traceability_flag?c /> <@metadata key="Traceability Flag" var=traceability_flag?c />
<#if traceability_flag> <#if traceability_flag>
<#assign groupList += [{"name":"grsf-traceability-flag-group", "title":"GRSF Traceability Flag"}] > <#assign group_list += [{"name":"grsf-traceability-flag-group", "title":"GRSF Traceability Flag"}] >
</#if> </#if>
</#if> </#if>
<@metadatalist key="Fishing Area" namespace=identity_namespace list=fishing_area tagValues=true /> <@metadatalist key="Fishing Area" namespace=identity_namespace list=fishing_area tagValues=true />
@ -60,8 +67,9 @@
<#-- TODO Set Legacy if source is not GRSF --> <#-- TODO Set Legacy if source is not GRSF -->
<@metadata key="system:type" var=grsf_type?capitalize sep=""/> <@metadata key="system:type" var=grsf_type?capitalize sep=""/>
], ],
"organization": <@source_macro />
"groups": [ "groups": [
<#list groupList as group> <#list group_list as group>
{ {
"name": "${group.name}", "name": "${group.name}",
"title": "${group.title}" "title": "${group.title}"
@ -72,7 +80,7 @@
<@group name="fishery-group" title="Fishery" var="OK" sep=""/> <@group name="fishery-group" title="Fishery" var="OK" sep=""/>
], ],
"tags": [ "tags": [
<#list tagList as tag> <#list tag_list as tag>
{ {
"name": "${tag}" "name": "${tag}"
}, },
@ -111,7 +119,7 @@
--> -->
], ],
"timeseries": [ "timeseries": [
<#list timeseriesList as elem> <#list timeseries_list as elem>
{ {
<#list elem as key, value> <#list elem as key, value>
"${key}":"${value}"<#sep>,</#sep> "${key}":"${value}"<#sep>,</#sep>

10
src/test/resources/freemarker/Stock.ftl
View File

@ -1,7 +1,6 @@
<#include "macros.ftl"> <#include "macros.ftl">
<#assign identity_namespace="stock_identity" > <#assign identity_namespace="stock_identity" >
<#assign data_namespace="stock_data" > <#assign data_namespace="stock_data" >
<#assign record_url="${uri_resolver_base_url}/${grsf_uuid}" />
{ {
"title" : "${stock_name}", "title" : "${stock_name}",
"license_id": "${license_id}", "license_id": "${license_id}",
@ -27,7 +26,7 @@
<#if sdg_flag??> <#if sdg_flag??>
<@metadata key="SDG Flag" var=sdg_flag?c /> <@metadata key="SDG Flag" var=sdg_flag?c />
<#if sdg_flag> <#if sdg_flag>
<#assign groupList += [{"name":"grsf-sdg-flag-group", "title":"GRSF SDG Flag"}] > <#assign group_list += [{"name":"grsf-sdg-flag-group", "title":"GRSF SDG Flag"}] >
</#if> </#if>
</#if> </#if>
<@metadata key="Status of the Record" var=status_grsf_record /> <@metadata key="Status of the Record" var=status_grsf_record />
@ -83,8 +82,9 @@
<#-- TODO Set Legacy if source is not GRSF --> <#-- TODO Set Legacy if source is not GRSF -->
<@metadata key="system:type" var=grsf_type?capitalize sep=""/> <@metadata key="system:type" var=grsf_type?capitalize sep=""/>
], ],
"organization": <@source_macro />
"groups": [ "groups": [
<#list groupList as group> <#list group_list as group>
{ {
"name": "${group.name}", "name": "${group.name}",
"title": "${group.title}" "title": "${group.title}"
@ -96,7 +96,7 @@
<@group name="stock-group" title="Stock" var="OK" sep=""/> <@group name="stock-group" title="Stock" var="OK" sep=""/>
], ],
"tags": [ "tags": [
<#list tagList as tag> <#list tag_list as tag>
{ {
"name": "${tag}" "name": "${tag}"
}, },
@ -135,7 +135,7 @@
--> -->
], ],
"timeseries": [ "timeseries": [
<#list timeseriesList as elem> <#list timeseries_list as elem>
{ {
<#list elem as key, value> <#list elem as key, value>
"${key}":"${value}"<#sep>,</#sep> "${key}":"${value}"<#sep>,</#sep>

8
src/test/resources/freemarker/TraceabilityUnit.ftl
View File

@ -1,7 +1,6 @@
<#include "macros.ftl"> <#include "macros.ftl">
<#assign identity_namespace="traceability_unit_identity" > <#assign identity_namespace="traceability_unit_identity" >
<#assign data_namespace="traceability_unit_data" > <#assign data_namespace="traceability_unit_data" >
<#assign record_url="${uri_resolver_base_url}/${grsf_uuid}" />
{ {
"title" : "${traceability_unit_name}", "title" : "${traceability_unit_name}",
"license_id": "${license_id}", "license_id": "${license_id}",
@ -30,8 +29,9 @@
<@metadata key="Record URL" var=record_url /> <@metadata key="Record URL" var=record_url />
<@metadata key="system:type" var="Traceability Unit" sep=""/> <@metadata key="system:type" var="Traceability Unit" sep=""/>
], ],
"organization": <@source_macro />
"groups": [ "groups": [
<#list groupList as group> <#list group_list as group>
{ {
"name": "${group.name}", "name": "${group.name}",
"title": "${group.title}" "title": "${group.title}"
@ -40,7 +40,7 @@
<@group name="traceability-unit-group" title="Traceability Unit" var="OK" sep=""/> <@group name="traceability-unit-group" title="Traceability Unit" var="OK" sep=""/>
], ],
"tags": [ "tags": [
<#list tagList as tag> <#list tag_list as tag>
{ {
"name": "${tag}" "name": "${tag}"
}, },
@ -53,7 +53,7 @@
<@resource name="Traceability Unit URI" url=traceability_record_uri sep=""/> <@resource name="Traceability Unit URI" url=traceability_record_uri sep=""/>
], ],
"timeseries": [ "timeseries": [
<#list timeseriesList as elem> <#list timeseries_list as elem>
{ {
<#list elem as key, value> <#list elem as key, value>
"${key}":"${value}"<#sep>,</#sep> "${key}":"${value}"<#sep>,</#sep>

41
src/test/resources/freemarker/macros.ftl
View File

@ -1,6 +1,21 @@
<#assign timeseriesList = [] > <#assign timeseries_list = [] >
<#assign groupList = [] > <#assign group_list = [] >
<#assign tagList = [] > <#assign tag_list = [] >
<#assign database_source_id_to_name = { "grsf":"Global Record of Stocks and Fisheries (GRSF)", "ram":"RAM Legacy Stock Assessment Database", "firms":"Fisheries and Resources Monitoring System (FIRMS)", "fishsource":"FishSource", "sdg":"FAO SDG 14.4.1 Questionnaire"} >
<#assign database_source_name_to_id = {} >
<#list database_source_id_to_name as key, value>
<#assign database_source_name_to_id += { "${value?json_string}" : "${key?json_string}" } >
</#list>
<#assign record_url="${uri_resolver_base_url}/${grsf_uuid}" />
<#assign description = "Short Name: ${short_name}\n" >
<#if grsf_semantic_identifier??>
<#assign description += "GRSF Semantic Identifier: ${grsf_semantic_identifier}\n" >
</#if>
<#assign description += "Record URL: ${record_url}" >
<#macro group name title var="" include=true sep=","> <#macro group name title var="" include=true sep=",">
<#if var?has_content && include> <#if var?has_content && include>
@ -55,7 +70,7 @@
<#if list?has_content && include> <#if list?has_content && include>
<#list list as elem> <#list list as elem>
<#if tagValues> <#if tagValues>
<#assign tagList += ["${elem?json_string}"]> <#assign tag_list += ["${elem?json_string}"]>
</#if> </#if>
{ {
"key": "<#if namespace?has_content>${namespace}:</#if>${key}", "key": "<#if namespace?has_content>${namespace}:</#if>${key}",
@ -78,9 +93,9 @@
<#macro timeseries key namespace="" timeseries="" groupname="" include=include_sensitive sep=","> <#macro timeseries key namespace="" timeseries="" groupname="" include=include_sensitive sep=",">
<#if timeseries?has_content && .data_model[timeseries]??> <#if timeseries?has_content && .data_model[timeseries]??>
<#assign timeseriesList += [{"property":"${timeseries}","filename":"${key}"}]> <#assign timeseries_list += [{"property":"${timeseries}","filename":"${key}"}]>
<#if groupname?has_content> <#if groupname?has_content>
<#assign groupList += [{"name":"${groupname}", "title":"${key}"}] > <#assign group_list += [{"name":"${groupname}", "title":"${key}"}] >
</#if> </#if>
<#if include> <#if include>
<#local list = .data_model[timeseries] > <#local list = .data_model[timeseries] >
@ -116,3 +131,17 @@
}${sep} }${sep}
</#if> </#if>
</#macro> </#macro>
<#macro source_macro sep=",">
<#local source_title = "${database_source_id_to_name[source]}" >
<#if source = "grsf">
<#assign group_list += [{"name":"${source}", "title":"${source_title}"}] >
<#else>
<#assign group_list += [{"name":"legacy-group", "title":"Legacy"}] >
</#if>
{
"name": "${source}",
"title": "${source_title}"
}${sep}
</#macro>