updated model and managed mapping to namespaces (#8719)

git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/data-catalogue/grsf-publisher-ws@149007 82a268e6-3cf1-43bd-a215-b396298e98cf
This commit is contained in:
Costantino Perciante 2017-05-24 14:53:31 +00:00
parent 80f856af5e
commit 3b7969997c
8 changed files with 246 additions and 162 deletions

View File

@ -8,6 +8,7 @@ import org.gcube.data_catalogue.grsf_publish_ws.custom_annotations.CkanResource;
import org.gcube.data_catalogue.grsf_publish_ws.custom_annotations.CustomField;
import org.gcube.data_catalogue.grsf_publish_ws.custom_annotations.Group;
import org.gcube.data_catalogue.grsf_publish_ws.custom_annotations.Tag;
import org.gcube.data_catalogue.grsf_publish_ws.custom_annotations.TimeSeries;
import org.gcube.data_catalogue.grsf_publish_ws.utils.groups.Sources;
import org.gcube.data_catalogue.grsf_publish_ws.utils.groups.Status;
@ -23,7 +24,7 @@ public class Common extends Base{
public static final String GRSF_TYPE_KEY = "GRSF type"; // stock, fishery
public static final String SOURCE_KEY = "Source"; // in case it is a RAM/FIRMS/FishSource record it is not added
// it is added in case of GRSF record
// it is added in case of GRSF record
@JsonProperty("data_owner")
@CustomField(key="Data owner")
@Tag
@ -74,16 +75,48 @@ public class Common extends Base{
@CustomField(key=SOURCE_KEY)
private String sourceType;
@JsonProperty("catches_or_landings")
@CustomField(key="Catches or landings")
@TimeSeries
@Valid
private List<TimeSeriesBean<String, String>> catchesOrLandings;
@JsonProperty("species_name")
@CustomField(key="Species Scientific Name")
@Tag
private List<String> speciesName;
@JsonProperty("species_code")
@CustomField(key="Species code")
@Tag
private List<String> speciesCode;
public Common() {
super();
}
/**
* @param dataOwner
* @param databaseSources
* @param sourceOfInformation
* @param refersTo
* @param shortTitle
* @param traceabilityFlag
* @param status
* @param reportingYear
* @param grsfType
* @param sourceType
* @param catchesOrLandings
* @param speciesName
* @param speciesCode
*/
public Common(String dataOwner, List<Resource<Sources>> databaseSources,
List<Resource<String>> sourceOfInformation,
List<RefersToBean> refersTo, String shortTitle,
Boolean traceabilityFlag, Status status, Long reportingYear,
String grsfType, String sourceType) {
String grsfType, String sourceType,
List<TimeSeriesBean<String, String>> catchesOrLandings,
List<String> speciesName, List<String> speciesCode) {
super();
this.dataOwner = dataOwner;
this.databaseSources = databaseSources;
@ -95,6 +128,9 @@ public class Common extends Base{
this.reportingYear = reportingYear;
this.grsfType = grsfType;
this.sourceType = sourceType;
this.catchesOrLandings = catchesOrLandings;
this.speciesName = speciesName;
this.speciesCode = speciesCode;
}
public String getGrsfType() {
@ -186,6 +222,31 @@ public class Common extends Base{
}
public List<TimeSeriesBean<String, String>> getCatchesOrLandings() {
return catchesOrLandings;
}
public void setCatchesOrLandings(
List<TimeSeriesBean<String, String>> catchesOrLandings) {
this.catchesOrLandings = catchesOrLandings;
}
public List<String> getSpeciesName() {
return speciesName;
}
public void setSpeciesName(List<String> speciesName) {
this.speciesName = speciesName;
}
public List<String> getSpeciesCode() {
return speciesCode;
}
public void setSpeciesCode(List<String> speciesCode) {
this.speciesCode = speciesCode;
}
@Override
public String toString() {
return "Common [dataOwner=" + dataOwner + ", databaseSources="
@ -194,6 +255,9 @@ public class Common extends Base{
+ ", shortTitle=" + shortTitle + ", traceabilityFlag="
+ traceabilityFlag + ", status=" + status + ", reportingYear="
+ reportingYear + ", grsfType=" + grsfType + ", sourceType="
+ sourceType + "]";
+ sourceType + ", catchesOrLandings=" + catchesOrLandings
+ ", speciesName=" + speciesName + ", speciesCode="
+ speciesCode + "]";
}
}

View File

@ -2,14 +2,12 @@ package org.gcube.data_catalogue.grsf_publish_ws.json.input;
import java.util.List;
import javax.validation.Valid;
import javax.validation.constraints.NotNull;
import javax.validation.constraints.Size;
import org.gcube.data_catalogue.grsf_publish_ws.custom_annotations.CustomField;
import org.gcube.data_catalogue.grsf_publish_ws.custom_annotations.Group;
import org.gcube.data_catalogue.grsf_publish_ws.custom_annotations.Tag;
import org.gcube.data_catalogue.grsf_publish_ws.custom_annotations.TimeSeries;
import org.gcube.data_catalogue.grsf_publish_ws.utils.groups.Fishery_Type;
import org.gcube.data_catalogue.grsf_publish_ws.utils.groups.Production_System_Type;
@ -77,12 +75,12 @@ public class FisheryRecord extends Common{
@CustomField(key="Environment")
private String environment;
@JsonProperty("catches_or_landings")
@CustomField(key="Catches or landings")
//@Tag
@TimeSeries
@Valid
private List<TimeSeriesBean<String, String>> catchesOrLandings;
// @JsonProperty("catches_or_landings")
// @CustomField(key="Catches or landings")
// //@Tag
// @TimeSeries
// @Valid
// private List<TimeSeriesBean<String, String>> catchesOrLandings;
@JsonProperty("type")
@CustomField(key="Type")
@ -117,7 +115,7 @@ public class FisheryRecord extends Common{
List<Production_System_Type> productionSystemType,
List<String> flagState, List<String> fishingGear,
String environment,
List<TimeSeriesBean<String, String>> catchesOrLandings,
// List<TimeSeriesBean<String, String>> catchesOrLandings,
Fishery_Type type) {
super();
this.fisheryName = fisheryName;
@ -131,10 +129,10 @@ public class FisheryRecord extends Common{
this.flagState = flagState;
this.fishingGear = fishingGear;
this.environment = environment;
this.catchesOrLandings = catchesOrLandings;
// this.catchesOrLandings = catchesOrLandings;
this.type = type;
}
public String getFisheryName() {
return fisheryName;
}
@ -175,13 +173,13 @@ public class FisheryRecord extends Common{
this.environment = environment;
}
public List<TimeSeriesBean<String, String>> getCatchesOrLandings() {
return catchesOrLandings;
}
public void setCatchesOrLandings(List<TimeSeriesBean<String, String>> catchesOrLandings) {
this.catchesOrLandings = catchesOrLandings;
}
// public List<TimeSeriesBean<String, String>> getCatchesOrLandings() {
// return catchesOrLandings;
// }
//
// public void setCatchesOrLandings(List<TimeSeriesBean<String, String>> catchesOrLandings) {
// this.catchesOrLandings = catchesOrLandings;
// }
public Fishery_Type getType() {
return type;
@ -249,8 +247,7 @@ public class FisheryRecord extends Common{
+ ", jurisdictionArea=" + jurisdictionArea
+ ", productionSystemType=" + productionSystemType
+ ", flagState=" + flagState + ", fishingGear=" + fishingGear
+ ", environment=" + environment + ", catchesOrLandings="
+ catchesOrLandings + ", type=" + type + "]";
+ ", environment=" + environment + ", type=" + type + "]";
}
}

View File

@ -34,10 +34,10 @@ public class StockRecord extends Common{
@CustomField(key="Stock id")
private String stockId;
@JsonProperty("species")
@CustomField(key="Species")
@Tag
private List<String> species;
// @JsonProperty("species")
// @CustomField(key="Species")
// @Tag
// private List<String> species;
@JsonProperty("assessment_distribution_area")
@CustomField(key="Assessment distribution area")
@ -87,6 +87,18 @@ public class StockRecord extends Common{
@Valid
private List<TimeSeriesBean<String, String>> abundanceLevel;
@JsonProperty("standard_fishing_pressure")
@CustomField(key="Standard Fishing Pressure")
@TimeSeries
@Valid
private List<TimeSeriesBean<Abundance_Level, Void>> fishingPressureStandard;
@JsonProperty("fishing_pressure")
@CustomField(key="Fishing Pressure")
@TimeSeries
@Valid
private List<TimeSeriesBean<String, String>> fishingPressure;
@JsonProperty("narrative_state_and_trend")
@CustomField(key="Narrative state and trend")
private String narrativeStateAndTrend;
@ -144,7 +156,7 @@ public class StockRecord extends Common{
public StockRecord(
String stockName,
String stockId,
List<String> species,
//List<String> species,
List<String> area,
List<String> exploitingFishery,
String managementEntity,
@ -160,7 +172,7 @@ public class StockRecord extends Common{
super();
this.stockName = stockName;
this.stockId = stockId;
this.species = species;
//this.species = species;
this.area = area;
this.exploitingFishery = exploitingFishery;
this.managementEntity = managementEntity;
@ -218,18 +230,36 @@ public class StockRecord extends Common{
this.stockId = stockId;
}
public List<String> getSpecies() {
return species;
}
public void setSpecies(List<String> species) {
this.species = species;
}
// public List<String> getSpecies() {
// return species;
// }
//
// public void setSpecies(List<String> species) {
// this.species = species;
// }
public List<String> getArea() {
return area;
}
public List<TimeSeriesBean<Abundance_Level, Void>> getFishingPressureStandard() {
return fishingPressureStandard;
}
public void setFishingPressureStandard(
List<TimeSeriesBean<Abundance_Level, Void>> fishingPressureStandard) {
this.fishingPressureStandard = fishingPressureStandard;
}
public List<TimeSeriesBean<String, String>> getFishingPressure() {
return fishingPressure;
}
public void setFishingPressure(
List<TimeSeriesBean<String, String>> fishingPressure) {
this.fishingPressure = fishingPressure;
}
public void setArea(List<String> area) {
this.area = area;
}
@ -325,12 +355,11 @@ public class StockRecord extends Common{
List<TimeSeriesBean<String, String>> exploitationRate) {
this.exploitationRate = exploitationRate;
}
@Override
public String toString() {
return "StockRecord [stockName=" + stockName + ", stockId=" + stockId
+ ", species=" + species + ", area=" + area
+ ", exploitingFishery=" + exploitingFishery
+ ", area=" + area + ", exploitingFishery=" + exploitingFishery
+ ", managementEntity=" + managementEntity
+ ", assessmentMethods=" + assessmentMethods
+ ", stateOfMarineResource=" + stateOfMarineResource
@ -338,11 +367,12 @@ public class StockRecord extends Common{
+ ", exploitationRate=" + exploitationRate
+ ", abundanceLevelStandard=" + abundanceLevelStandard
+ ", abundanceLevel=" + abundanceLevel
+ ", fishingPressureStandard=" + fishingPressureStandard
+ ", fishingPressure=" + fishingPressure
+ ", narrativeStateAndTrend=" + narrativeStateAndTrend
+ ", scientificAdvice=" + scientificAdvice
+ ", assessor=" + assessor + ", stockUri="
+ stockUri + ", waterArea=" + waterArea + ", type=" + type
+ "]";
+ ", scientificAdvice=" + scientificAdvice + ", assessor="
+ assessor + ", stockUri=" + stockUri + ", waterArea="
+ waterArea + ", type=" + type + "]";
}
}

View File

@ -84,10 +84,10 @@ public class CommonServiceUtils {
if(record.getClass().equals(StockRecord.class)){
StockRecord stock = (StockRecord) record;
List<String> species = stock.getSpecies();
if(species == null || species.isEmpty())
throw new Exception("species cannot be null/empty in a GRSF record");
List<String> speciesNames = stock.getSpeciesName();
List<String> speciesCodes = stock.getSpeciesCode();
if(speciesNames == null || speciesNames.isEmpty() || speciesCodes == null || speciesCodes.isEmpty())
throw new Exception("species name/code cannot be null/empty in a GRSF record");
}
// check if it is a stock and perform related checks

View File

@ -232,6 +232,14 @@ public class GrsfPublisherFisheryService {
// add the SYSTEM_TYPE
customFields.put(CommonServiceUtils.SYSTEM_TYPE, Arrays.asList(sourceInPath.getOrigName()));
// convert extras' keys to keys with namespace
Map<String, String> namespaces = HelperMethods.getFieldToFieldNameSpaceMapping(HelperMethods.GENERIC_RESOURCE_NAME_MAP_KEY_NAMESPACES_FISHERY);
if(namespaces == null)
throw new Exception("Failed to retrieve the namespaces for the key fields!");
customFields = HelperMethods.replaceFieldsKey(customFields, namespaces);
logger.info("Invoking creation method..");

View File

@ -227,6 +227,14 @@ public class GrsfPublisherStockService {
customFields.put(CommonServiceUtils.SYSTEM_TYPE, Arrays.asList(sourceInPath.getOrigName()));
logger.info("Invoking creation method..");
// convert extras' keys to keys with namespace
Map<String, String> namespaces = HelperMethods.getFieldToFieldNameSpaceMapping(HelperMethods.GENERIC_RESOURCE_NAME_MAP_KEY_NAMESPACES_STOCK);
if(namespaces == null)
throw new Exception("Failed to retrieve the namespaces for the key fields!");
customFields = HelperMethods.replaceFieldsKey(customFields, namespaces);
// create the product
String id = catalogue.createCKanDatasetMultipleCustomFields(

View File

@ -1,13 +1,21 @@
package org.gcube.data_catalogue.grsf_publish_ws.utils;
import static org.gcube.resources.discovery.icclient.ICFactory.client;
import java.io.BufferedReader;
import java.io.File;
import java.io.InputStreamReader;
import java.io.StringReader;
import java.io.UnsupportedEncodingException;
import java.sql.SQLException;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
import org.gcube.common.authorization.library.provider.SecurityTokenProvider;
import org.gcube.common.homelibrary.home.exceptions.InternalErrorException;
@ -16,15 +24,22 @@ import org.gcube.common.homelibrary.home.workspace.WorkspaceItem;
import org.gcube.common.homelibrary.home.workspace.exceptions.InsufficientPrivilegesException;
import org.gcube.common.homelibrary.home.workspace.exceptions.ItemAlreadyExistException;
import org.gcube.common.homelibrary.home.workspace.folder.items.ExternalFile;
import org.gcube.common.resources.gcore.utils.XPathHelper;
import org.gcube.common.scope.api.ScopeProvider;
import org.gcube.data_catalogue.grsf_publish_ws.utils.cache.CacheImpl;
import org.gcube.data_catalogue.grsf_publish_ws.utils.cache.CacheInterface;
import org.gcube.datacatalogue.ckanutillibrary.server.DataCatalogue;
import org.gcube.datacatalogue.ckanutillibrary.server.DataCatalogueFactory;
import org.gcube.datacatalogue.ckanutillibrary.server.DataCatalogueImpl;
import org.gcube.resources.discovery.client.api.DiscoveryClient;
import org.gcube.resources.discovery.client.queries.api.Query;
import org.gcube.resources.discovery.client.queries.impl.QueryBox;
import org.jsoup.Jsoup;
import org.jsoup.safety.Whitelist;
import org.slf4j.LoggerFactory;
import org.w3c.dom.Node;
import org.w3c.dom.NodeList;
import org.xml.sax.InputSource;
import com.fasterxml.jackson.databind.ObjectMapper;
@ -53,6 +68,9 @@ public abstract class HelperMethods {
private static final String SOCIAL_SERVICE_WRITE_APPLICATION_POST = "/2/posts/write-post-app/";
private static final String MEDIATYPE_JSON = "application/json";
public static final String GENERIC_RESOURCE_NAME_MAP_KEY_NAMESPACES_STOCK = "GRSF Stock";
public static final String GENERIC_RESOURCE_NAME_MAP_KEY_NAMESPACES_FISHERY = "GRSF Fishery";
// to be retrieved from the web.xml
public static final String MANAGE_CONTEX_KEY = "ManageVRE";
public static final String PUBLIC_CONTEX_KEY = "PublicVRE";
@ -141,10 +159,7 @@ public abstract class HelperMethods {
if((result = (String) userEmailCache.get(token)) != null){
return result;
}else{
String baseUrl = new ServiceEndPointReaderSocial(context).getBasePath();
// fallback: check for gcore endpoint
if(baseUrl == null || baseUrl.isEmpty())
baseUrl = new GcoreEndPointReaderSocial(context).getBasePath();
String baseUrl = new GcoreEndPointReaderSocial(context).getBasePath();
String url = baseUrl.endsWith("/") ? baseUrl + "users/getUserEmail?gcube-token=" + token :
baseUrl + "/users/getUserEmail?gcube-token=" + token;
logger.debug("Request url is " + url);
@ -168,10 +183,7 @@ public abstract class HelperMethods {
if((result = (String) userFullnameCache.get(token)) != null){
return result;
}else{
String baseUrl = new ServiceEndPointReaderSocial(context).getBasePath();
// fallback: check for gcore endpoint
if(baseUrl == null || baseUrl.isEmpty())
baseUrl = new GcoreEndPointReaderSocial(context).getBasePath();
String baseUrl = new GcoreEndPointReaderSocial(context).getBasePath();
String url = baseUrl.endsWith("/") ? baseUrl + "users/getUserFullname?gcube-token=" + token :
baseUrl + "/users/getUserFullname?gcube-token=" + token;
logger.debug("Request url is " + url);
@ -365,10 +377,7 @@ public abstract class HelperMethods {
String tokenUser = SecurityTokenProvider.instance.get();
logger.info("Current scope for writeProductPost is " + currentScope + " and token is " + tokenUser.substring(0, 10) + "***************");
String basePath = new ServiceEndPointReaderSocial(currentScope).getBasePath();
// fallback: check for gcore endpoint
if(basePath == null || basePath.isEmpty())
basePath = new GcoreEndPointReaderSocial(currentScope).getBasePath();
String basePath = new GcoreEndPointReaderSocial(currentScope).getBasePath();
if(basePath == null){
@ -479,4 +488,74 @@ public abstract class HelperMethods {
return catalogue.getProductsIdsInGroupOrOrg(groupName, false, 0, Integer.MAX_VALUE);
}
/**
* Return a map for converting a key to a namespace:key format by reading a generic resource.
* @return a map
*/
public static Map<String, String> getFieldToFieldNameSpaceMapping(String resourceName){
Map<String, String> toReturn = new HashMap<String, String>();
try {
Query q = new QueryBox("for $profile in collection('/db/Profiles/GenericResource')//Resource " +
"where $profile/Profile/SecondaryType/string() eq '"+ "ApplicationProfile" + "' and $profile/Profile/Name/string() " +
" eq '" + resourceName + "'" +
"return $profile");
DiscoveryClient<String> client = client();
List<String> appProfile = client.submit(q);
if (appProfile == null || appProfile.size() == 0)
throw new Exception("Your applicationProfile is not registered in the infrastructure");
else {
String elem = appProfile.get(0);
DocumentBuilder docBuilder = DocumentBuilderFactory.newInstance().newDocumentBuilder();
Node node = docBuilder.parse(new InputSource(new StringReader(elem))).getDocumentElement();
XPathHelper helper = new XPathHelper(node);
NodeList nodeListKeys = helper.evaluateForNodes("//originalKey");
NodeList nodeListModifiedKeys = helper.evaluateForNodes("//modifiedKey");
int sizeKeys = nodeListKeys != null ? nodeListKeys.getLength() : 0;
int sizeKeysModifed = nodeListModifiedKeys != null ? nodeListModifiedKeys.getLength() : 0;
if(sizeKeys != sizeKeysModifed)
throw new Exception("Malformed XML");
logger.debug("Size is " + sizeKeys);
for (int i = 0; i < sizeKeys; i++) {
toReturn.put(nodeListKeys.item(i).getTextContent(), nodeListModifiedKeys.item(i).getTextContent());
}
}
} catch (Exception e) {
logger.error("Error while trying to fetch applicationProfile profile from the infrastructure", e);
return null;
}
logger.debug("Map is " + toReturn);
return toReturn;
}
/**
* Replace the extras' keys if needed
* @param customFields
* @param namespaces
* @return
*/
public static Map<String, List<String>> replaceFieldsKey(Map<String, List<String>> customFields,
Map<String, String> namespaces) {
Map<String, List<String>> toReturn = new HashMap<String, List<String>>();
Iterator<Entry<String, List<String>>> iterator = customFields.entrySet().iterator();
while (iterator.hasNext()) {
Map.Entry<java.lang.String, java.util.List<java.lang.String>> entry = (Map.Entry<java.lang.String, java.util.List<java.lang.String>>) iterator
.next();
if(namespaces.containsKey(entry.getKey()))
toReturn.put(namespaces.get(entry.getKey()), entry.getValue());
else
toReturn.put(entry.getKey(), entry.getValue());
}
return toReturn;
}
}

View File

@ -1,102 +0,0 @@
package org.gcube.data_catalogue.grsf_publish_ws.utils;
import static org.gcube.resources.discovery.icclient.ICFactory.clientFor;
import static org.gcube.resources.discovery.icclient.ICFactory.queryFor;
import java.util.Iterator;
import java.util.List;
import org.gcube.common.resources.gcore.ServiceEndpoint;
import org.gcube.common.resources.gcore.ServiceEndpoint.AccessPoint;
import org.gcube.common.scope.api.ScopeProvider;
import org.gcube.resources.discovery.client.api.DiscoveryClient;
import org.gcube.resources.discovery.client.queries.api.SimpleQuery;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* Retrieves the base url of the social-networking service in the scope provided
* @author Costantino Perciante at ISTI-CNR
* (costantino.perciante@isti.cnr.it)
*/
public class ServiceEndPointReaderSocial {
private String basePath = null;
private static Logger logger = LoggerFactory.getLogger(ServiceEndPointReaderSocial.class);
private final static String RUNTIME_RESOURCE_NAME = "SocialNetworking";
private final static String CATEGORY = "Portal";
public ServiceEndPointReaderSocial(String context){
if(context == null || context.isEmpty())
throw new IllegalArgumentException("A valid context is needed to discover the service");
String oldContext = ScopeProvider.instance.get();
ScopeProvider.instance.set(context);
try{
List<ServiceEndpoint> resources = getConfigurationFromIS();
if (resources.size() == 0){
logger.error("There is no Runtime Resource having name " + RUNTIME_RESOURCE_NAME +" and Category " + CATEGORY + " in this scope.");
throw new Exception("There is no Runtime Resource having name " + RUNTIME_RESOURCE_NAME +" and Category " + CATEGORY + " in this scope.");
}
else {
for (ServiceEndpoint res : resources) {
Iterator<AccessPoint> accessPointIterator = res.profile().accessPoints().iterator();
while (accessPointIterator.hasNext()) {
ServiceEndpoint.AccessPoint accessPoint = (ServiceEndpoint.AccessPoint) accessPointIterator
.next();
// get base path
basePath = accessPoint.address();
// break
break;
}
}
}
}catch(Exception e){
logger.error("Unable to retrieve such service endpoint information!", e);
}finally{
if(oldContext != null && !oldContext.equals(context))
ScopeProvider.instance.set(oldContext);
}
logger.info("Found base path " + basePath + " for the service");
}
/**
* Retrieve endpoints information from IS for the Service endpoint
* @return list of endpoints
* @throws Exception
*/
private List<ServiceEndpoint> getConfigurationFromIS() throws Exception{
SimpleQuery query = queryFor(ServiceEndpoint.class);
query.addCondition("$resource/Profile/Name/text() eq '"+ RUNTIME_RESOURCE_NAME +"'");
query.addCondition("$resource/Profile/Category/text() eq '"+ CATEGORY +"'");
DiscoveryClient<ServiceEndpoint> client = clientFor(ServiceEndpoint.class);
List<ServiceEndpoint> toReturn = client.submit(query);
return toReturn;
}
/**
* Get the base path of the social networking service
* @return
*/
public String getBasePath() {
return basePath;
}
}