Compare commits
28 Commits
Author | SHA1 | Date |
---|---|---|
luca.frosini | ce3c4af8eb | |
Luca Frosini | 8427224976 | |
Luca Frosini | 8eb0e30897 | |
Luca Frosini | 54b20686aa | |
Luca Frosini | 10b4c7dbf6 | |
Luca Frosini | d1ff03affc | |
Luca Frosini | f985363878 | |
Luca Frosini | abb21c3988 | |
Luca Frosini | 9e3bdeff3b | |
Luca Frosini | 84781a9a8d | |
Luca Frosini | 9f8e8a5f5e | |
Luca Frosini | 98eda28a8e | |
Luca Frosini | f00306cbb6 | |
Luca Frosini | 0fd88f657c | |
Luca Frosini | 9f182ededd | |
Luca Frosini | dd8874178d | |
Luca Frosini | 40ea1235d9 | |
Luca Frosini | ca752576bc | |
Luca Frosini | 587c0971ad | |
Luca Frosini | 3f29a00f05 | |
Luca Frosini | 13e9f7309e | |
Luca Frosini | 490619efe7 | |
Luca Frosini | 044d2edd05 | |
Luca Frosini | d6d373a94c | |
Luca Frosini | c28f0dcd65 | |
Luca Frosini | dbad160e36 | |
Luca Frosini | 6d37e4b939 | |
Luca Frosini | 75b55cc8bb |
|
@ -1,3 +1,4 @@
|
||||||
target
|
target
|
||||||
.classpath
|
.classpath
|
||||||
.project
|
.project
|
||||||
|
/.DS_Store
|
||||||
|
|
|
@ -1,4 +1,6 @@
|
||||||
eclipse.preferences.version=1
|
eclipse.preferences.version=1
|
||||||
encoding//src/main/java=UTF-8
|
encoding//src/main/java=UTF-8
|
||||||
|
encoding//src/main/resources=UTF-8
|
||||||
encoding//src/test/java=UTF-8
|
encoding//src/test/java=UTF-8
|
||||||
|
encoding//src/test/resources=UTF-8
|
||||||
encoding/<project>=UTF-8
|
encoding/<project>=UTF-8
|
||||||
|
|
12
CHANGELOG.md
12
CHANGELOG.md
|
@ -3,6 +3,17 @@
|
||||||
All notable changes to this project will be documented in this file.
|
All notable changes to this project will be documented in this file.
|
||||||
This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
||||||
|
|
||||||
|
## [v1.13.3]
|
||||||
|
|
||||||
|
- Adding support for "FAO SDG 14.4.1 Questionnaire" source [#23670]
|
||||||
|
|
||||||
|
|
||||||
|
## [v1.13.2]
|
||||||
|
|
||||||
|
- Migrated request to social-service-client [#23679]
|
||||||
|
- Added "Assessment Methods" as Group [#23409]
|
||||||
|
|
||||||
|
|
||||||
## [v1.13.1]
|
## [v1.13.1]
|
||||||
|
|
||||||
- Aligned code and wiki to the new requirements [#23167]
|
- Aligned code and wiki to the new requirements [#23167]
|
||||||
|
@ -10,6 +21,7 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm
|
||||||
- Tag are added also to legacy records [#23216]
|
- Tag are added also to legacy records [#23216]
|
||||||
- Fixed code which generated groups id from name [#23215]
|
- Fixed code which generated groups id from name [#23215]
|
||||||
|
|
||||||
|
|
||||||
## [v1.13.0]
|
## [v1.13.0]
|
||||||
|
|
||||||
### Added
|
### Added
|
||||||
|
|
20
pom.xml
20
pom.xml
|
@ -11,7 +11,7 @@
|
||||||
|
|
||||||
<groupId>org.gcube.data-catalogue</groupId>
|
<groupId>org.gcube.data-catalogue</groupId>
|
||||||
<artifactId>grsf-publisher-ws</artifactId>
|
<artifactId>grsf-publisher-ws</artifactId>
|
||||||
<version>1.13.1</version>
|
<version>1.13.3</version>
|
||||||
<packaging>war</packaging>
|
<packaging>war</packaging>
|
||||||
<name>grsf-publisher-ws</name>
|
<name>grsf-publisher-ws</name>
|
||||||
<description>Utility library to publish GRSF products on GRSF catalogue.</description>
|
<description>Utility library to publish GRSF products on GRSF catalogue.</description>
|
||||||
|
@ -26,6 +26,7 @@
|
||||||
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
|
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
|
||||||
<jsoup.version>1.10.1</jsoup.version>
|
<jsoup.version>1.10.1</jsoup.version>
|
||||||
<serviceClass>DataPublishing</serviceClass>
|
<serviceClass>DataPublishing</serviceClass>
|
||||||
|
<maven.compiler.release>8</maven.compiler.release>
|
||||||
</properties>
|
</properties>
|
||||||
|
|
||||||
<scm>
|
<scm>
|
||||||
|
@ -39,7 +40,7 @@
|
||||||
<dependency>
|
<dependency>
|
||||||
<groupId>org.gcube.distribution</groupId>
|
<groupId>org.gcube.distribution</groupId>
|
||||||
<artifactId>gcube-smartgears-bom</artifactId>
|
<artifactId>gcube-smartgears-bom</artifactId>
|
||||||
<version>2.1.1</version>
|
<version>2.2.0</version>
|
||||||
<type>pom</type>
|
<type>pom</type>
|
||||||
<scope>import</scope>
|
<scope>import</scope>
|
||||||
</dependency>
|
</dependency>
|
||||||
|
@ -50,12 +51,17 @@
|
||||||
<dependency>
|
<dependency>
|
||||||
<groupId>org.gcube.data-catalogue</groupId>
|
<groupId>org.gcube.data-catalogue</groupId>
|
||||||
<artifactId>grsf-common-library</artifactId>
|
<artifactId>grsf-common-library</artifactId>
|
||||||
<version>[1-0-0, 2.0.0-SNAPSHOT)</version>
|
<version>[2.0.0, 3.0.0-SNAPSHOT)</version>
|
||||||
</dependency>
|
</dependency>
|
||||||
<dependency>
|
<dependency>
|
||||||
<groupId>org.gcube.common</groupId>
|
<groupId>org.gcube.common</groupId>
|
||||||
<artifactId>storagehub-client-library</artifactId>
|
<artifactId>storagehub-client-library</artifactId>
|
||||||
</dependency>
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.gcube.social-networking</groupId>
|
||||||
|
<artifactId>social-service-client</artifactId>
|
||||||
|
<version>[1.0.0, 2.0.0-SNAPSHOT)</version>
|
||||||
|
</dependency>
|
||||||
<!-- jsoup HTML parser library @ http://jsoup.org/ -->
|
<!-- jsoup HTML parser library @ http://jsoup.org/ -->
|
||||||
<dependency>
|
<dependency>
|
||||||
<groupId>org.jsoup</groupId>
|
<groupId>org.jsoup</groupId>
|
||||||
|
@ -162,6 +168,14 @@
|
||||||
<artifactId>jersey-test-framework-provider-jetty</artifactId>
|
<artifactId>jersey-test-framework-provider-jetty</artifactId>
|
||||||
<scope>test</scope>
|
<scope>test</scope>
|
||||||
</dependency>
|
</dependency>
|
||||||
|
|
||||||
|
<!-- Added to support Java 11 JDK -->
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.projectlombok</groupId>
|
||||||
|
<artifactId>lombok</artifactId>
|
||||||
|
<scope>provided</scope>
|
||||||
|
</dependency>
|
||||||
|
<!-- END Added to support Java 11 JDK -->
|
||||||
</dependencies>
|
</dependencies>
|
||||||
|
|
||||||
<build>
|
<build>
|
||||||
|
|
|
@ -46,6 +46,7 @@ public class StockRecord extends Common{
|
||||||
|
|
||||||
@JsonProperty(Constants.ASSESSMENT_METHODS_JSON_KEY)
|
@JsonProperty(Constants.ASSESSMENT_METHODS_JSON_KEY)
|
||||||
@CustomField(key=Constants.ASSESSMENT_METHODS_CUSTOM_KEY)
|
@CustomField(key=Constants.ASSESSMENT_METHODS_CUSTOM_KEY)
|
||||||
|
@Group(groupNameOverValue=Constants.ASSESSMENT_METHODS_CUSTOM_KEY, prependSourceToGroupName=false)
|
||||||
private List<String> assessmentMethods;
|
private List<String> assessmentMethods;
|
||||||
|
|
||||||
@JsonProperty(Constants.FIRMS_ABUNDANCE_LEVEL_JSON_KEY)
|
@JsonProperty(Constants.FIRMS_ABUNDANCE_LEVEL_JSON_KEY)
|
||||||
|
|
|
@ -51,7 +51,7 @@ import eu.trentorise.opendata.jackan.model.CkanDataset;
|
||||||
* @author Costantino Perciante (ISTI - CNR)
|
* @author Costantino Perciante (ISTI - CNR)
|
||||||
* @author Luca Frosini (ISTI - CNR)
|
* @author Luca Frosini (ISTI - CNR)
|
||||||
*/
|
*/
|
||||||
@Path("{source:firms|FIRMS|grsf|GRSF|FishSource|fishsource}/fishery/")
|
@Path("{source:firms|FIRMS|grsf|GRSF|FishSource|fishsource|sdg|SDG}/fishery/")
|
||||||
public class GrsfPublisherFisheryService {
|
public class GrsfPublisherFisheryService {
|
||||||
|
|
||||||
// the context
|
// the context
|
||||||
|
|
|
@ -51,7 +51,7 @@ import eu.trentorise.opendata.jackan.model.CkanDataset;
|
||||||
* @author Costantino Perciante (ISTI - CNR)
|
* @author Costantino Perciante (ISTI - CNR)
|
||||||
* @author Luca Frosini (ISTI - CNR)
|
* @author Luca Frosini (ISTI - CNR)
|
||||||
*/
|
*/
|
||||||
@Path("{source:firms|FIRMS|ram|RAM|grsf|GRSF|FishSource|fishsource}/stock/")
|
@Path("{source:firms|FIRMS|ram|RAM|grsf|GRSF|FishSource|fishsource|sdg|SDG}/stock/")
|
||||||
public class GrsfPublisherStockService {
|
public class GrsfPublisherStockService {
|
||||||
|
|
||||||
// the context
|
// the context
|
||||||
|
|
|
@ -4,7 +4,6 @@ import java.beans.PropertyDescriptor;
|
||||||
import java.lang.reflect.Field;
|
import java.lang.reflect.Field;
|
||||||
import java.util.ArrayList;
|
import java.util.ArrayList;
|
||||||
import java.util.Arrays;
|
import java.util.Arrays;
|
||||||
import java.util.Collections;
|
|
||||||
import java.util.HashMap;
|
import java.util.HashMap;
|
||||||
import java.util.HashSet;
|
import java.util.HashSet;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
|
@ -266,7 +265,7 @@ public class CommonServiceUtils {
|
||||||
if(match) {
|
if(match) {
|
||||||
StringBuffer stringBuffer = new StringBuffer();
|
StringBuffer stringBuffer = new StringBuffer();
|
||||||
if(prependSource) {
|
if(prependSource) {
|
||||||
stringBuffer.append(source.toString());
|
stringBuffer.append(source.getURLPath());
|
||||||
stringBuffer.append(" ");
|
stringBuffer.append(" ");
|
||||||
}
|
}
|
||||||
if(groupNameOverValue.isEmpty()) {
|
if(groupNameOverValue.isEmpty()) {
|
||||||
|
@ -289,7 +288,7 @@ public class CommonServiceUtils {
|
||||||
*/
|
*/
|
||||||
private static void addRecordToGroups(Set<String> groups, Set<String> sourcesList, Product_Type productType, Sources sourceInPath) {
|
private static void addRecordToGroups(Set<String> groups, Set<String> sourcesList, Product_Type productType, Sources sourceInPath) {
|
||||||
if(sourceInPath == Sources.GRSF) {
|
if(sourceInPath == Sources.GRSF) {
|
||||||
groups.add(getGroupId(Sources.GRSF.getOrigName())); // i.e. grsf_group
|
groups.add(getGroupId(Sources.GRSF.getURLPath())); // i.e. grsf_group
|
||||||
}else {
|
}else {
|
||||||
groups.add(getGroupId(Constants.SYSTEM_TYPE_LEGACY_RECORD)); // i.e. legacy_group
|
groups.add(getGroupId(Constants.SYSTEM_TYPE_LEGACY_RECORD)); // i.e. legacy_group
|
||||||
}
|
}
|
||||||
|
@ -543,7 +542,7 @@ public class CommonServiceUtils {
|
||||||
if(databaseSources!=null) {
|
if(databaseSources!=null) {
|
||||||
for(Resource<Sources> source : databaseSources) {
|
for(Resource<Sources> source : databaseSources) {
|
||||||
Sources sourceName = source.getName();
|
Sources sourceName = source.getName();
|
||||||
sourcesList.add(sourceName.getOrigName().toLowerCase());
|
sourcesList.add(sourceName.getURLPath());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -719,6 +718,6 @@ public class CommonServiceUtils {
|
||||||
return Constants.GRSF_PRE_ORGANIZATION_NAME;
|
return Constants.GRSF_PRE_ORGANIZATION_NAME;
|
||||||
}
|
}
|
||||||
}else
|
}else
|
||||||
return sourceInPath.getOrigName().toLowerCase();
|
return sourceInPath.getURLPath();
|
||||||
}
|
}
|
||||||
}
|
}
|
|
@ -31,6 +31,7 @@ import org.gcube.datacatalogue.common.caches.CacheInterface;
|
||||||
import org.gcube.resources.discovery.client.api.DiscoveryClient;
|
import org.gcube.resources.discovery.client.api.DiscoveryClient;
|
||||||
import org.gcube.resources.discovery.client.queries.api.Query;
|
import org.gcube.resources.discovery.client.queries.api.Query;
|
||||||
import org.gcube.resources.discovery.client.queries.impl.QueryBox;
|
import org.gcube.resources.discovery.client.queries.impl.QueryBox;
|
||||||
|
import org.gcube.social_networking.social_networking_client_library.UserClient;
|
||||||
import org.jsoup.Jsoup;
|
import org.jsoup.Jsoup;
|
||||||
import org.jsoup.safety.Whitelist;
|
import org.jsoup.safety.Whitelist;
|
||||||
import org.slf4j.LoggerFactory;
|
import org.slf4j.LoggerFactory;
|
||||||
|
@ -134,21 +135,18 @@ public abstract class HelperMethods {
|
||||||
* @return
|
* @return
|
||||||
* @throws Exception
|
* @throws Exception
|
||||||
*/
|
*/
|
||||||
public static String getUserEmail(String context, String token){
|
public static String getUserEmail(String context, String token) throws Exception{
|
||||||
|
|
||||||
// check in cache
|
// check in cache
|
||||||
String result = null;
|
String email = null;
|
||||||
if((result = (String) userEmailCache.get(token)) != null){
|
if((email = (String) userEmailCache.get(token)) != null){
|
||||||
return result;
|
return email;
|
||||||
}else{
|
}else{
|
||||||
String baseUrl = new GcoreEndPointReaderSocial(context).getBasePath();
|
UserClient userClient = new UserClient();
|
||||||
String url = baseUrl.endsWith("/") ? baseUrl + "users/getUserEmail?gcube-token=" + token :
|
email = userClient.getEmail();
|
||||||
baseUrl + "/users/getUserEmail?gcube-token=" + token;
|
userEmailCache.insert(token, email);
|
||||||
logger.debug("Request url is " + url);
|
|
||||||
result = executGETHttpRequest(url, 200);
|
|
||||||
userEmailCache.insert(token, result);
|
|
||||||
}
|
}
|
||||||
return result;
|
return email;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
@ -158,21 +156,18 @@ public abstract class HelperMethods {
|
||||||
* @return
|
* @return
|
||||||
* @throws Exception
|
* @throws Exception
|
||||||
*/
|
*/
|
||||||
public static String getUserFullname(String context, String token){
|
public static String getUserFullname(String context, String token) throws Exception{
|
||||||
|
|
||||||
// check in cache
|
// check in cache
|
||||||
String result = null;
|
String fullName = null;
|
||||||
if((result = (String) userFullnameCache.get(token)) != null){
|
if((fullName = (String) userFullnameCache.get(token)) != null){
|
||||||
return result;
|
return fullName;
|
||||||
}else{
|
}else{
|
||||||
String baseUrl = new GcoreEndPointReaderSocial(context).getBasePath();
|
UserClient userClient = new UserClient();
|
||||||
String url = baseUrl.endsWith("/") ? baseUrl + "users/getUserFullname?gcube-token=" + token :
|
fullName = userClient.getFullName();
|
||||||
baseUrl + "/users/getUserFullname?gcube-token=" + token;
|
userFullnameCache.insert(token, fullName);
|
||||||
logger.debug("Request url is " + url);
|
|
||||||
result = executGETHttpRequest(url, 200);
|
|
||||||
userFullnameCache.insert(token, result);
|
|
||||||
}
|
}
|
||||||
return result;
|
return fullName;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|
|
@ -70,7 +70,7 @@
|
||||||
</field>
|
</field>
|
||||||
<field>
|
<field>
|
||||||
<originalKey>Assessment Method</originalKey>
|
<originalKey>Assessment Method</originalKey>
|
||||||
<modifiedKey>stock_data:Assessment Method</modifiedKey>
|
<modifiedKey>none:none</modifiedKey>
|
||||||
</field>
|
</field>
|
||||||
<field>
|
<field>
|
||||||
<originalKey>Abundance Level (FIRMS Standard)</originalKey>
|
<originalKey>Abundance Level (FIRMS Standard)</originalKey>
|
||||||
|
|
|
@ -270,7 +270,7 @@ public class JTests {
|
||||||
}
|
}
|
||||||
|
|
||||||
//@Test
|
//@Test
|
||||||
public void testCaches(){
|
public void testCaches() throws Exception{
|
||||||
|
|
||||||
String context = "/gcube/devNext/NextNext";
|
String context = "/gcube/devNext/NextNext";
|
||||||
String token = "";
|
String token = "";
|
||||||
|
|
|
@ -0,0 +1,38 @@
|
||||||
|
package org.gcube.data_catalogue.grsf_publish_ws;
|
||||||
|
|
||||||
|
import java.io.File;
|
||||||
|
import java.net.URL;
|
||||||
|
|
||||||
|
import org.gcube.data_catalogue.grsf_publish_ws.json.input.record.StockRecord;
|
||||||
|
import org.junit.Test;
|
||||||
|
import org.slf4j.Logger;
|
||||||
|
import org.slf4j.LoggerFactory;
|
||||||
|
|
||||||
|
import com.fasterxml.jackson.databind.JsonNode;
|
||||||
|
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @author Luca Frosini (ISTI - CNR)
|
||||||
|
*/
|
||||||
|
public class TestJson {
|
||||||
|
|
||||||
|
private static final Logger logger = LoggerFactory.getLogger(Test.class);
|
||||||
|
|
||||||
|
public File getResourcesDirectory() throws Exception {
|
||||||
|
URL logbackFileURL = TestJson.class.getClassLoader().getResource("logback-test.xml");
|
||||||
|
File logbackFile = new File(logbackFileURL.toURI());
|
||||||
|
File resourcesDirectory = logbackFile.getParentFile();
|
||||||
|
return resourcesDirectory;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testJsonDeserialization() throws Exception {
|
||||||
|
File jsonQueryFile = new File(getResourcesDirectory(), "70ae6895-7d3d-4f4a-86f9-bcb17d41bff6.json");
|
||||||
|
ObjectMapper objectMapper = new ObjectMapper();
|
||||||
|
JsonNode jsonNode = objectMapper.readTree(jsonQueryFile);
|
||||||
|
logger.debug("{}", jsonNode);
|
||||||
|
StockRecord sr = objectMapper.readValue(jsonQueryFile, StockRecord.class);
|
||||||
|
logger.debug("{}", sr);
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
|
@ -0,0 +1,34 @@
|
||||||
|
{
|
||||||
|
"stock_name" : "European hake - Southern Adriatic",
|
||||||
|
"license_id" : "CC-BY-SA-4.0",
|
||||||
|
"version" : 1.0,
|
||||||
|
"database_sources" : [ {
|
||||||
|
"name" : "FAO SDG 14.4.1 questionnaire",
|
||||||
|
"description" : "FAO SDG 14.4.1 questionnaire",
|
||||||
|
"url" : "https://www.fao.org/sustainable-development-goals/indicators/14.4.1/en/"
|
||||||
|
} ],
|
||||||
|
"stock_uri" : "https://github.com/grsf/resource/sdg_14_4_1/stock/70ae6895-7d3d-4f4a-86f9-bcb17d41bff6",
|
||||||
|
"grsf_uuid" : "70ae6895-7d3d-4f4a-86f9-bcb17d41bff6",
|
||||||
|
"short_name" : "European hake - Southern Adriatic",
|
||||||
|
"description" : "European hake - Southern Adriatic",
|
||||||
|
"grsf_type" : "assessment unit",
|
||||||
|
"species" : [ "Code: HKE, Classification System: ASFIS, Scientific Name: Merluccius merluccius" ],
|
||||||
|
"assessment_area" : [ "Code: 18, System: gfcm, Name: Southern Adriatic " ],
|
||||||
|
"source_of_information" : [ {
|
||||||
|
"name" : "https://www.fao.org/sustainable-development-goals/indicators/14.4.1/en/sdg-alb-1",
|
||||||
|
"description" : "",
|
||||||
|
"url" : "https://www.fao.org/sustainable-development-goals/indicators/14.4.1/en/sdg-alb-1"
|
||||||
|
} ],
|
||||||
|
"data_owner" : [ "Albania" ],
|
||||||
|
"assessment_methods" : [ "The official stock assessment concludes 'Overfished' with respect to abundance reference points. [Rep. Year or Assessment ID: 2019, Ref. Year: 2018]" ],
|
||||||
|
"connections_indicator" : "not connected",
|
||||||
|
"similarities_indicator" : "without similar records",
|
||||||
|
"landings" : [ {
|
||||||
|
"value" : "872",
|
||||||
|
"unit" : "Tonnes",
|
||||||
|
"reference_year" : 2018,
|
||||||
|
"reporting_year_or_assessment_id" : "2020",
|
||||||
|
"data_owner" : "Albania"
|
||||||
|
} ],
|
||||||
|
"citation" : "citation TBD"
|
||||||
|
}
|
|
@ -0,0 +1,19 @@
|
||||||
|
<?xml version="1.0" encoding="UTF-8"?>
|
||||||
|
<!DOCTYPE xml>
|
||||||
|
<configuration>
|
||||||
|
|
||||||
|
<appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
|
||||||
|
<encoder>
|
||||||
|
<pattern>%d{HH:mm:ss.SSS} [%thread] %-5level %logger{0}: %msg%n</pattern>
|
||||||
|
</encoder>
|
||||||
|
</appender>
|
||||||
|
|
||||||
|
|
||||||
|
<logger name="org.gcube" level="INFO" />
|
||||||
|
<logger name="org.gcube.data_catalogue.grsf_publish_ws" level="TRACE" />
|
||||||
|
|
||||||
|
<root level="WARN">
|
||||||
|
<appender-ref ref="STDOUT" />
|
||||||
|
</root>
|
||||||
|
|
||||||
|
</configuration>
|
Loading…
Reference in New Issue