gCubeSystem
/
grsf-publisher
15
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Fixing service

This commit is contained in:
luca.frosini 2023-06-26 16:26:31 +02:00
parent b7285c92bc
commit ad42814e7e
6 changed files with 59 additions and 30 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
src/test/java/org/gcube/grsf/publisher/freemarker/FreeMarkerTest.java
View File

@ -73,7 +73,7 @@ public class FreeMarkerTest {
// File[] sourceFiles = examplesDir.listFiles(dirnameFilter);
// File[] sourceFiles = new File[] {new File(examplesDir,"sdg")};
File[] sourceFiles = new File[] { new File(examplesDir, "grsf") };
File[] sourceFiles = new File[] { new File(examplesDir, "firms") };
for(File source : sourceFiles) {
@ -82,9 +82,9 @@ public class FreeMarkerTest {
File outputSourceDir = new File(outputTypeDir, sourceString);
outputSourceDir.mkdir();
// File[] jsonFiles = source.listFiles(filenameFilter);
File[] jsonFiles = source.listFiles(filenameFilter);
// File[] jsonFiles = new File[] {new File(source,"31601d3d-6b0f-4b06-9afb-19d4cc6f7e12.json")};
File[] jsonFiles = new File[] {new File(source,"88818c3f-7120-322b-9637-7c7d2e9fc1e5.json")};
// File[] jsonFiles = new File[] {new File(source,"c75bf19c-37d4-3977-87f2-a8c4a3ecfc85.json")};
for(File jsonFile : jsonFiles) {
ObjectMapper mapper = new ObjectMapper();

40
src/test/java/org/gcube/grsf/publisher/record/RecordTest.java
View File

@ -24,8 +24,13 @@ public class RecordTest extends ContextTest {
@Test
public void testRecords() throws Exception {
// -1 mean unlimited
int maxTestRecords = -1;
int maxTestRecordsPerSource = 3;
// String[] types = new String[] {"Stock", "Fishery", "TraceabilityUnit"};
String[] types = new String[] { "Stock" };
String[] types = new String[] {"Stock"};
// String[] types = new String[] {"Fishery"};
// String[] types = new String[] {"TraceabilityUnit"};
@ -60,8 +65,8 @@ public class RecordTest extends ContextTest {
// File[] sourceFiles = examplesDir.listFiles(dirnameFilter);
// File[] sourceFiles = new File[] { new File(examplesDir, "grsf") };
File[] sourceFiles = new File[] { new File(examplesDir, "ram") };
File[] sourceFiles = new File[] { new File(examplesDir, "firms") };
for (File source : sourceFiles) {
String sourceString = source.getName();
@ -69,12 +74,26 @@ public class RecordTest extends ContextTest {
File outputSourceDir = new File(outputTypeDir, sourceString);
outputSourceDir.mkdir();
// File[] jsonFiles = source.listFiles(filenameFilter);
File[] jsonFiles = source.listFiles(filenameFilter);
// File[] jsonFiles = new File[] {new File(source,"88818c3f-7120-322b-9637-7c7d2e9fc1e5.json")};
File[] jsonFiles = new File[] {new File(source,"00702023-0e2d-345d-8b20-60580c107acd.json")};
// File[] jsonFiles = new File[] {new File(source,"00702023-0e2d-345d-8b20-60580c107acd.json")};
// File[] jsonFiles = new File[] {new File(source,"88818c3f-7120-322b-9637-7c7d2e9fc1e5.json")};
int countRecordPerSource = 0;
for (File jsonFile : jsonFiles) {
if(maxTestRecords>0 && countRecord >= maxTestRecords) {
logger.info("The test has already elaborated {} records which is the max allowed", countRecord);
return;
}
if(maxTestRecordsPerSource>0 && countRecordPerSource >= maxTestRecordsPerSource) {
logger.info("The test has already elaborated {} records for the source {} which is the max allowed for the source", countRecordPerSource, sourceString);
break;
}
logger.trace("Going to elaborate {} {} from file {}", sourceString, type, jsonFile.getAbsolutePath());
ObjectMapper mapper = new ObjectMapper();
ObjectNode input = (ObjectNode) mapper.readTree(jsonFile);
@ -101,7 +120,16 @@ public class RecordTest extends ContextTest {
record.setName(input.get("grsf_uuid").asText());
String ret = record.create(mapper.writeValueAsString(input));
record.purge();
// record.purge();
logger.trace("{} {} from file {} elaborated successfully", sourceString, type, jsonFile.getAbsolutePath());
++countRecord;
++countRecordPerSource;
long sleepTime = TimeUnit.MILLISECONDS.toMillis(200);
logger.trace("Going to sleep for {} millis", sleepTime);
// Thread.sleep(sleepTime);
}
}
}

6
src/test/resources/freemarker/Fishery.ftl
View File

@ -5,7 +5,7 @@
"name" : "${grsf_uuid?json_string}",
"title" : "${fishery_name?json_string}",
"license_id": "${license_id}",
"description": "${description?json_string}",
"notes": "${description?json_string}",
"extras": [
<#-- START of Common Metadata -->
<@metadata key="GRSF Type" namespace=identity_namespace var=grsf_type?capitalize />
@ -82,9 +82,7 @@
],
"tags": [
<#list tag_list as tag>
{
"name": "${tag}"
},
<@tagbyvalue value=tag include=true />
</#list>
<@tag tagname=grsf_type?capitalize />
<@tag tagname=connections_indicator />

11
src/test/resources/freemarker/Stock.ftl
View File

@ -5,7 +5,7 @@
"name" : "${grsf_uuid?json_string}",
"title" : "${stock_name?json_string}",
"license_id": "${license_id}",
"description": "Short Name: ${short_name}\n<#if grsf_semantic_identifier??>GRSF Semantic Identifier: ${grsf_semantic_identifier}\n</#if>Record URL: ${record_url}",
"notes": "${description?json_string}",
"extras": [
<#-- START of Common Metadata -->
<@metadata key="GRSF Type" namespace=identity_namespace var=grsf_type?capitalize />
@ -63,14 +63,13 @@
<@timeseries key="Fishing Pressure" namespace=data_namespace timeseries="fishing_pressure" groupname="fishing-pressure-group" />
<@timeseries key="State and Trend" namespace=data_namespace timeseries="state_and_trend_of_marine_resources" groupname="state-and-trend-group" />
<@timeseries key="FAO Stock Status Category" namespace=data_namespace timeseries="fao_categories" groupname="fao-stock-status-category-group" />
<#--
<@metadatalist key="Scientific Advice" namespace=data_namespace list=scientific_advice include=include_sensitive />
<#--
scientific_advice will change from list of string to timeseries.
Use the line below in place of the line above
-->
<@timeseries key="Scientific Advice" namespace=data_namespace timeseries="scientific_advice" groupname="scientific-advice-group" />
-->
<@metadata key="Assessor" namespace=data_namespace var=assessor include=include_sensitive />
<#-- END of Stock Metadata -->
@ -93,9 +92,7 @@
],
"tags": [
<#list tag_list as tag>
{
"name": "${tag}"
},
<@tagbyvalue value=tag include=true />
</#list>
<@tag tagname=grsf_type?capitalize />
<@tag tagname=connections_indicator />

6
src/test/resources/freemarker/TraceabilityUnit.ftl
View File

@ -5,7 +5,7 @@
"name" : "${grsf_uuid?json_string}",
"title" : "${traceability_unit_name?json_string}",
"license_id": "${license_id}",
"description": "Short Name: ${short_name}\n<#if grsf_semantic_identifier??>GRSF Semantic Identifier: ${grsf_semantic_identifier}\n</#if>Record URL: ${record_url}",
"notes": "${description?json_string}",
"extras": [
<#if database_sources??>
<@metadatalist key="Database Source" namespace=identity_namespace list=database_sources />
@ -42,9 +42,7 @@
],
"tags": [
<#list tag_list as tag>
{
"name": "${tag}"
},
<@tagbyvalue value=tag include=true />
</#list>
<@tag tagname="Traceability Unit" sep=""/>
],

20
src/test/resources/freemarker/macros.ftl
View File

@ -41,22 +41,30 @@
<#macro tag tagname="" include=true sep=",">
<#if tagname?has_content && include>
{
"name": "${tagname?json_string}"
}${sep}
<@tagbyvalue value=tagname include=include sep=sep />
</#if>
</#macro>
<#macro tags list include=true sep=",">
<#if list?? && include>
<#list list as tagname>
{
"name": "${tagname?json_string}"
}${sep}
<@tagbyvalue value=tagname include=include sep=sep />
</#list>
</#if>
</#macro>
<#macro tagbyvalue value include=true sep=",">
<#assign tagvalue=value?json_string?replace("[,:]\\s{0,1}", " ", "r") />
{
<#if tagvalue?length gt 100 >
"name": "${tagvalue[0..<99]}"
<#else>
"name": "${tagvalue}"
</#if>
}${sep}
</#macro>
<#macro metadata key namespace="" var="" include=true sep=",">
<#if var?has_content && include>
{