gCubeSystem
/
grsf-publisher-ws
16
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Merged changes made in branch 

git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/data-catalogue/grsf-publisher-ws@171271 82a268e6-3cf1-43bd-a215-b396298e98cf
This commit is contained in:
Luca Frosini 2018-09-03 09:07:06 +00:00
parent 163710176a
commit 9774bcd2ef
7 changed files with 316 additions and 246 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
distro/changelog.xml
View File

@ -9,6 +9,10 @@
<Change>Added 'Connected'-'Not Connected' tag to GRSF Records #11766</Change>
<Change>Added group for SDG flag #11767</Change>
<Change>Added citation field #11811</Change>
<Change>Added sub-groups support for available time series related to GRSF Type "Assessment Unit" #11832</Change>
<Change>Added Biomass group #11967</Change>
<Change>Changed 'State and trend of Marine Resource' to 'State and Trend' #11968</Change>
<Change>Changed 'Scientific advice' to 'Scientific Advice' #11969</Change>
</Changeset>
<Changeset component="org.gcube.data-catalogue.grsf-publisher-ws.1-5-0"
date="2017-01-10">

2
pom.xml
View File

@ -11,7 +11,7 @@
<groupId>org.gcube.data-catalogue</groupId>
<artifactId>grsf-publisher-ws</artifactId>
<version>1.6.0-SNAPSHOT</version>
<version>1.7.0-SNAPSHOT</version>
<packaging>war</packaging>
<name>grsf-publisher-ws</name>

12
src/main/java/org/gcube/data_catalogue/grsf_publish_ws/custom_annotations/Group.java
View File

@ -7,12 +7,13 @@ import java.lang.annotation.Target;
/**
* Group annotation
* @author Costantino Perciante at ISTI-CNR (costantino.perciante@isti.cnr.it)
* @author Costantino Perciante (ISTI - CNR)
* @author Luca Frosini (ISTI - CNR)
*/
@Retention(RetentionPolicy.RUNTIME)
@Target(ElementType.FIELD)
public @interface Group {
/**
* Define a REGEX condition to be checked before the record
* is actually added to the group.
@ -26,4 +27,11 @@ public @interface Group {
*/
String groupNameOverValue() default "";
/**
* When the group is created the source is prepended to the group name.
* Set to false to avoid source prepending
* @return
*/
boolean prependSourceToGroupName() default true;
}

3
src/main/java/org/gcube/data_catalogue/grsf_publish_ws/json/input/record/Common.java
View File

@ -26,6 +26,7 @@ import com.fasterxml.jackson.annotation.JsonProperty;
* Information that both Stock and Fishery records must contain.
* @author Costantino Perciante (ISTI - CNR)
* @author Luca Frosini (ISTI - CNR)
*
*/
public abstract class Common extends Base{
@ -72,12 +73,14 @@ public abstract class Common extends Base{
@JsonProperty(Constants.CATCHES_JSON_KEY)
@CustomField(key=Constants.CATCHES_CUSTOM_KEY)
@TimeSeries
@Group(groupNameOverValue=Constants.CATCHES_CUSTOM_KEY, prependSourceToGroupName=false)
@Valid
private List<TimeSeriesBean<String, String>> catches;
@JsonProperty(Constants.LANDINGS_JSON_KEY)
@CustomField(key=Constants.LANDINGS_CUSTOM_KEY)
@TimeSeries
@Group(groupNameOverValue=Constants.LANDINGS_CUSTOM_KEY, prependSourceToGroupName=false)
@Valid
private List<TimeSeriesBean<String, String>> landings;

6
src/main/java/org/gcube/data_catalogue/grsf_publish_ws/json/input/record/FisheryRecord.java
View File

@ -16,10 +16,12 @@ import com.fasterxml.jackson.annotation.JsonProperty;
/**
* A fishery record bean.
* @author Costantino Perciante at ISTI-CNR (costantino.perciante@isti.cnr.it)
* @author Costantino Perciante (ISTI - CNR)
* @author Luca Frosini (ISTI - CNR)
*/
public class FisheryRecord extends Common{
public class FisheryRecord extends Common {
@JsonProperty(Constants.FISHERY_NAME_JSON_KEY)
@NotNull(message="fishery_name cannot be null")
@Size(min=1, message="fishery_name cannot be empty")

11
src/main/java/org/gcube/data_catalogue/grsf_publish_ws/json/input/record/StockRecord.java
View File

@ -20,7 +20,8 @@ import com.fasterxml.jackson.annotation.JsonProperty;
/**
* A stock record bean.
* @author Costantino Perciante at ISTI-CNR (costantino.perciante@isti.cnr.it)
* @author Costantino Perciante (ISTI - CNR)
* @author Luca Frosini (ISTI - CNR)
*/
public class StockRecord extends Common{
@ -50,47 +51,55 @@ public class StockRecord extends Common{
@JsonProperty(Constants.FIRMS_ABUNDANCE_LEVEL_JSON_KEY)
@CustomField(key=Constants.FIRMS_ABUNDANCE_LEVEL_CUSTOM_KEY)
@TimeSeries
@Group(groupNameOverValue=Constants.FIRMS_ABUNDANCE_LEVEL_CUSTOM_KEY, prependSourceToGroupName=false)
@Valid
private List<TimeSeriesBean<Abundance_Level, Void>> abundanceLevelStandard;
@JsonProperty(Constants.ABUNDANCE_LEVEL_JSON_KEY)
@CustomField(key=Constants.ABUNDANCE_LEVEL_CUSTOM_KEY)
@TimeSeries
@Group(groupNameOverValue=Constants.ABUNDANCE_LEVEL_CUSTOM_KEY, prependSourceToGroupName=false)
@Valid
private List<TimeSeriesBean<String, String>> abundanceLevel;
@JsonProperty(Constants.BIOMASS_JSON_KEY)
@CustomField(key=Constants.BIOMASS_CUSTOM_KEY)
@TimeSeries
@Group(groupNameOverValue=Constants.BIOMASS_CUSTOM_KEY, prependSourceToGroupName=false)
@Valid
private List<TimeSeriesBean<String, String>> biomass;
@JsonProperty(Constants.FISHING_PRESSURE_FIRMS_JSON_KEY)
@CustomField(key=Constants.FISHING_PRESSURE_FIRMS_CUSTOM_KEY)
@TimeSeries
@Group(groupNameOverValue=Constants.FISHING_PRESSURE_FIRMS_CUSTOM_KEY, prependSourceToGroupName=false)
@Valid
private List<TimeSeriesBean<Fishing_Pressure, Void>> fishingPressureStandard;
@JsonProperty(Constants.FISHING_PRESSURE_JSON_KEY)
@CustomField(key=Constants.FISHING_PRESSURE_CUSTOM_KEY)
@TimeSeries
@Group(groupNameOverValue=Constants.FISHING_PRESSURE_CUSTOM_KEY, prependSourceToGroupName=false)
@Valid
private List<TimeSeriesBean<String, String>> fishingPressure;
@JsonProperty(Constants.STATE_AND_TREND_MARINE_RESOURCE_JSON_KEY)
@CustomField(key=Constants.STATE_AND_TREND_MARINE_RESOURCE_CUSTOM_KEY)
@TimeSeries
@Group(groupNameOverValue=Constants.STATE_AND_TREND_MARINE_RESOURCE_CUSTOM_KEY, prependSourceToGroupName=false)
@Valid
private List<TimeSeriesBean<String, Void>> narrativeState;
@JsonProperty(Constants.FAO_CATEGORIES_JSON_KEY)
@CustomField(key=Constants.FAO_CATEGORIES_CUSTOM_KEY)
@TimeSeries
@Group(groupNameOverValue=Constants.FAO_CATEGORIES_CUSTOM_KEY, prependSourceToGroupName=false)
@Valid
private List<TimeSeriesBean<String, Void>> faoState;
@JsonProperty(Constants.SCIENTIFIC_ADVICE_JSON_KEY)
@CustomField(key=Constants.SCIENTIFIC_ADVICE_CUSTOM_KEY)
@Group(groupNameOverValue=Constants.SCIENTIFIC_ADVICE_CUSTOM_KEY, prependSourceToGroupName=false)
private List<String> scientificAdvice;
@JsonProperty(Constants.ASSESSOR_JSON_KEY)

524
src/main/java/org/gcube/data_catalogue/grsf_publish_ws/utils/CommonServiceUtils.java
View File

@ -36,6 +36,7 @@ import org.gcube.datacatalogue.common.Constants;
import org.gcube.datacatalogue.common.enums.Product_Type;
import org.gcube.datacatalogue.common.enums.Sources;
import org.gcube.datacatalogue.common.enums.Status;
import org.gcube.datacatalogue.common.enums.Stock_Type;
import org.json.simple.JSONObject;
import org.slf4j.LoggerFactory;
@ -44,75 +45,77 @@ import eu.trentorise.opendata.jackan.model.CkanLicense;
/**
* Services common utils.
* @author Costantino Perciante at ISTI-CNR
* @author Costantino Perciante (ISTI - CNR)
* @author Luca Frosini (ISTI - CNR)
*/
@SuppressWarnings({"rawtypes", "unchecked"})
public class CommonServiceUtils {
private static final org.slf4j.Logger logger = LoggerFactory.getLogger(CommonServiceUtils.class);
private static final int TAG_MAX_SIZE = 100;
private static Map<String, Boolean> extensionsCheck = new ConcurrentHashMap<>();
private static Map<String,Boolean> extensionsCheck = new ConcurrentHashMap<>();
/**
* Retrieve the list of licenses for stocks and fisheries
* @return
*/
public static Map<String, String> getLicenses(DataCatalogue catalogue){
public static Map<String,String> getLicenses(DataCatalogue catalogue) {
logger.info("Requested licenses...");
Map<String, String> toReturn = new HashMap<String, String>();
Map<String,String> toReturn = new HashMap<String,String>();
List<CkanLicense> licenses = catalogue.getLicenses();
for (CkanLicense ckanLicense : licenses) {
for(CkanLicense ckanLicense : licenses) {
toReturn.put(ckanLicense.getId(), ckanLicense.getTitle());
}
return toReturn;
}
/**
* Validate an aggregated GRSF record. TODO use @Valid tags
* @throws Exception
*/
public static void validateAggregatedRecord(Common record) throws Exception {
List<RefersToBean> refersToList = record.getRefersTo();
String shortTitle = record.getShortName();
Boolean traceabilityFlag = record.isTraceabilityFlag();
Status status = record.getStatus();
if(refersToList == null || refersToList.isEmpty())
throw new Exception("refers_to cannot be null/empty");
if(traceabilityFlag == null)
throw new Exception("traceability_flag cannot be null");
if(shortTitle == null || shortTitle.isEmpty())
throw new Exception("short_title cannot be null/empty");
if(status == null)
throw new Exception("status cannot be null/empty");
// check if it is a stock and perform related checks
if(record.getClass().equals(StockRecord.class)){
if(record.getClass().equals(StockRecord.class)) {
StockRecord stock = (StockRecord) record;
List<String> species = stock.getSpecies();
if(species == null || species.isEmpty())
throw new Exception("species cannot be null/empty in a GRSF record");
}
// check if it is a stock and perform related checks
if(record.getClass().equals(FisheryRecord.class)){
if(record.getClass().equals(FisheryRecord.class)) {
FisheryRecord fishery = (FisheryRecord) record;
List<String> fishingArea = fishery.getFishingArea();
List<String> jurisdictionArea = fishery.getJurisdictionArea();
if((fishingArea == null || fishingArea.isEmpty()) && (jurisdictionArea == null || jurisdictionArea.isEmpty()))
if((fishingArea == null || fishingArea.isEmpty())
&& (jurisdictionArea == null || jurisdictionArea.isEmpty()))
throw new Exception("fishing_area and jurisdiction_area cannot be null/empty at the same time!");
}
}
/**
* Parse the record to look up tags, groups and resources
* @param tags
@ -126,217 +129,248 @@ public class CommonServiceUtils {
* @param username
* @param source
*/
public static void getTagsGroupsResourcesExtrasByRecord(
Set<String> tags,
boolean skipTags,
Set<String> groups,
boolean skipGroups,
List<ResourceBean> resources,
boolean skipResources,
Map<String, List<String>> extras,
Base record,
String username,
Sources source // it comes from the source type e.g., "grsf-", "ram-" ..
){
public static void getTagsGroupsResourcesExtrasByRecord(Set<String> tags, boolean skipTags, Set<String> groups,
boolean skipGroups, List<ResourceBean> resources, boolean skipResources, Map<String,List<String>> extras,
Base record, String username, Sources source // it comes from the source type e.g., "grsf-", "ram-" ..
) {
Class<?> current = record.getClass();
do{
do {
Field[] fields = current.getDeclaredFields();
for (Field field : fields) {
for(Field field : fields) {
if(!skipTags)
getTagsByField(field, current, record, tags);
if(!skipGroups)
getGroupsByField(field, current, record, groups, source);
getExtrasByField(field, current, record, extras, source);
if(!skipResources)
getResourcesByField(field, current, record, username, resources);
}
}
while((current = current.getSuperclass())!=null); // start from the inherited class up to the Object.class
} while((current = current.getSuperclass()) != null); // start from the inherited class up to the Object.class
logger.debug("Tags are " + tags);
logger.debug("Groups are " + groups);
logger.debug("Extras are " + extras);
logger.debug("Resources without timeseries are " + resources);
}
/**
* Retrieve the list of tags for this object
*/
private static void getTagsByField(Field field, Class<?> current, Base record, Set<String> tags){
if(field.isAnnotationPresent(Tag.class)){
try{
private static void getTagsByField(Field field, Class<?> current, Base record, Set<String> tags) {
if(field.isAnnotationPresent(Tag.class)) {
try {
Object f = new PropertyDescriptor(field.getName(), current).getReadMethod().invoke(record);
if(f != null){
if(f instanceof List<?>){
if(f != null) {
if(f instanceof List<?>) {
List asList = ((List) f);
if(!asList.isEmpty()){
if(!asList.isEmpty()) {
logger.debug("The object annotated with @Tag is a list. Adding ... ");
int elementsToConsider = asList.size();
// check if it is a time series, in this take the last X elements
if(asList.get(0).getClass().equals(TimeSeriesBean.class)){
elementsToConsider = Math.min(elementsToConsider, Constants.TIME_SERIES_TAKE_LAST_VALUES);
for (int i = 0; i < elementsToConsider; i++) {
String finalTag = asList.get(i).toString().trim().replaceAll(Constants.REGEX_TAGS, "");
if(asList.get(0).getClass().equals(TimeSeriesBean.class)) {
elementsToConsider = Math.min(elementsToConsider,
Constants.TIME_SERIES_TAKE_LAST_VALUES);
for(int i = 0; i < elementsToConsider; i++) {
String finalTag = asList.get(i).toString().trim().replaceAll(Constants.REGEX_TAGS,
"");
if(finalTag.length() <= TAG_MAX_SIZE)
tags.add(finalTag);
tags.add(finalTag);
}
}else{
} else {
// else add all the available elements
for (int i = 0; i < elementsToConsider; i++) {
String finalTag = asList.get(i).toString().trim().replaceAll(Constants.REGEX_TAGS, "");
for(int i = 0; i < elementsToConsider; i++) {
String finalTag = asList.get(i).toString().trim().replaceAll(Constants.REGEX_TAGS,
"");
if(finalTag.length() <= TAG_MAX_SIZE)
tags.add(finalTag);
tags.add(finalTag);
}
}
}
}else{
} else {
logger.debug("The object annotated with @Tag is a simple one. Adding ... ");
String finalTag = f.toString().trim().replaceAll(Constants.REGEX_TAGS, "");
logger.debug(finalTag);
if(finalTag.length() <= TAG_MAX_SIZE)
tags.add(finalTag);
tags.add(finalTag);
}
}
}catch(Exception e){
} catch(Exception e) {
logger.error("Failed to read value for field " + field.getName() + " skipping", e);
}
}
}
/**
* Retrieve the list of groups' names for this object
*/
private static void getGroupsByField(Field field, Class<?> current, Base record, Set<String> groups, Sources source){
if(field.isAnnotationPresent(Group.class)){
String conditionToCheck = field.getAnnotation(Group.class).condition();
String groupNameOverValue = field.getAnnotation(Group.class).groupNameOverValue();
try{
private static void getGroupsByField(Field field, Class<?> current, Base record, Set<String> groups,
Sources source) {
if(field.isAnnotationPresent(Group.class)) {
Group group = field.getAnnotation(Group.class);
String conditionToCheck = group.condition();
String groupNameOverValue = group.groupNameOverValue();
// See https://support.d4science.org/issues/11832
boolean assessmentUnit = false;
boolean prependSource = group.prependSourceToGroupName();
if(record instanceof StockRecord) {
StockRecord stockRecord = (StockRecord) record;
Stock_Type stock_Type = stockRecord.getType();
if(stock_Type != Stock_Type.Assessment_Unit) {
prependSource = false;
}else {
assessmentUnit = true;
}
}
// end patch for https://support.d4science.org/issues/11832
try {
Object f = new PropertyDescriptor(field.getName(), current).getReadMethod().invoke(record);
if(f != null){
if(f instanceof List<?>){
if(f != null) {
if(f instanceof List<?>) {
List asList = ((List) f);
if(!asList.isEmpty()){
if(!asList.isEmpty()) {
logger.debug("The object annotated with @Group is a list. Adding ... ");
// else add all the available elements
for (int i = 0; i < asList.size(); i++) {
boolean match = conditionToCheck.isEmpty() ? true : asList.get(i).toString().trim().matches(conditionToCheck);
if(match){
String groupName = groupNameOverValue.isEmpty() ?
HelperMethods.getGroupNameOnCkan(source.toString().toLowerCase() + "-" + asList.get(i).toString().trim()) :
source.toString().toLowerCase() + "-" + groupNameOverValue;
groups.add(groupName);
for(int i = 0; i < asList.size(); i++) {
boolean match = conditionToCheck.isEmpty() ? true
: asList.get(i).toString().trim().matches(conditionToCheck);
if(match) {
String groupName = groupNameOverValue.isEmpty()
? HelperMethods.getGroupNameOnCkan(source.toString().toLowerCase() + "-"
+ asList.get(i).toString().trim())
: source.toString().toLowerCase() + "-" + groupNameOverValue;
if(assessmentUnit && !prependSource) {
groups.add(groupNameOverValue);
}else {
groups.add(groupName);
}
}
}
}
}else{
} else {
// also convert to the group name that should be on ckan
boolean match = conditionToCheck.isEmpty() ? true : f.toString().trim().matches(conditionToCheck);
if(match){
String groupName = groupNameOverValue.isEmpty() ?
HelperMethods.getGroupNameOnCkan(source.toString().toLowerCase() + "-" + f.toString().trim()) :
source.toString().toLowerCase() + "-" + groupNameOverValue;
groups.add(groupName);
boolean match = conditionToCheck.isEmpty() ? true
: f.toString().trim().matches(conditionToCheck);
if(match) {
String groupName = groupNameOverValue.isEmpty()
? HelperMethods.getGroupNameOnCkan(
source.toString().toLowerCase() + "-" + f.toString().trim())
: source.toString().toLowerCase() + "-" + groupNameOverValue;
if(assessmentUnit && !prependSource) {
groups.add(groupNameOverValue);
}else {
groups.add(groupName);
}
}
}
}
}catch(Exception e){
} catch(Exception e) {
logger.error("Failed to read value for field " + field.getName() + " skipping", e);
}
}
}
/**
* Retrieve the list of extras for this object
* @param source
*/
private static void getExtrasByField(Field field, Class<?> current, Base record, Map<String, List<String>> extras, Sources source){
if(field.isAnnotationPresent(CustomField.class)){
try{
private static void getExtrasByField(Field field, Class<?> current, Base record, Map<String,List<String>> extras,
Sources source) {
if(field.isAnnotationPresent(CustomField.class)) {
try {
Object f = new PropertyDescriptor(field.getName(), current).getReadMethod().invoke(record);
String keyField = field.getAnnotation(CustomField.class).key();
// manage no connections nor similar grsf records here for GRSF records only
if(source.equals(Sources.GRSF) && keyField.equals(Constants.SIMILAR_GRSF_RECORDS_CUSTOM_KEY)){
List asList = (List)f;
if(asList == null || asList.isEmpty()){
if(source.equals(Sources.GRSF) && keyField.equals(Constants.SIMILAR_GRSF_RECORDS_CUSTOM_KEY)) {
List asList = (List) f;
if(asList == null || asList.isEmpty()) {
extras.put(keyField, Arrays.asList(Constants.NO_SIMILAR_GRSF_RECORDS));
return;
}
}
if(source.equals(Sources.GRSF) && keyField.equals(Constants.CONNECTED_CUSTOM_KEY)){
List asList = (List)f;
if(asList == null || asList.isEmpty()){
if(source.equals(Sources.GRSF) && keyField.equals(Constants.CONNECTED_CUSTOM_KEY)) {
List asList = (List) f;
if(asList == null || asList.isEmpty()) {
extras.put(keyField, Arrays.asList(Constants.NO_CONNECTED_RECORDS));
return;
}
}
if(f != null){
if(f != null) {
Set<String> valuesForKey = null;
// check if the map already contains this key
if(extras.containsKey(keyField))
valuesForKey = new HashSet(extras.get(keyField));
else
valuesForKey = new HashSet<String>();
if(f instanceof List<?>){
logger.debug("The object " + field.getName() + " is a list and is annotated with @CustomField. Adding ...");
List asList = (List)f;
if(!asList.isEmpty()){
if(f instanceof List<?>) {
logger.debug("The object " + field.getName()
+ " is a list and is annotated with @CustomField. Adding ...");
List asList = (List) f;
if(!asList.isEmpty()) {
int elementsToConsider = asList.size();
// check if it is a time series, in this case take the last X elements
if(asList.get(0).getClass().equals(TimeSeriesBean.class)){
elementsToConsider = Math.min(elementsToConsider, Constants.TIME_SERIES_TAKE_LAST_VALUES);
for (int i = 0; i < elementsToConsider; i++) {
if(asList.get(0).getClass().equals(TimeSeriesBean.class)) {
elementsToConsider = Math.min(elementsToConsider,
Constants.TIME_SERIES_TAKE_LAST_VALUES);
for(int i = 0; i < elementsToConsider; i++) {
// trim and remove html
String clean = HelperMethods.removeHTML(asList.get(i).toString().trim());
String clean = HelperMethods.removeHTML(asList.get(i).toString().trim());
valuesForKey.add(clean);
}
}
else
for (int i = 0; i < elementsToConsider; i++) {
String clean = HelperMethods.removeHTML(asList.get(i).toString().trim());
} else
for(int i = 0; i < elementsToConsider; i++) {
String clean = HelperMethods.removeHTML(asList.get(i).toString().trim());
valuesForKey.add(clean);
}
}
}else{
String clean = HelperMethods.removeHTML(f.toString().trim());
} else {
String clean = HelperMethods.removeHTML(f.toString().trim());
valuesForKey.add(clean);
}
// add to the map
extras.put(keyField, new ArrayList<String>(valuesForKey));
}
}catch(Exception e){
} catch(Exception e) {
logger.error("Failed to read value for field " + field.getName() + " skipping", e);
}
}
}
/**
* Retrieve the ResourceBean given the record (extract resources from Database Sources and Source of Information and others)
* @param record
@ -345,47 +379,52 @@ public class CommonServiceUtils {
* @param resources
* @return
*/
private static void getResourcesByField(Field field, Class<?> current, Base record, String username, List<ResourceBean> resources){
if(field.isAnnotationPresent(CkanResource.class)){
try{
private static void getResourcesByField(Field field, Class<?> current, Base record, String username,
List<ResourceBean> resources) {
if(field.isAnnotationPresent(CkanResource.class)) {
try {
Object f = new PropertyDescriptor(field.getName(), current).getReadMethod().invoke(record);
if(f != null){
if(f instanceof List<?>){
List<Resource> listOfResources = (List<Resource>)f;
for (Resource resource : listOfResources) {
resources.add(new ResourceBean(resource.getUrl(), resource.getName().toString(), resource.getDescription(), null, username, null, null));
if(f != null) {
if(f instanceof List<?>) {
List<Resource> listOfResources = (List<Resource>) f;
for(Resource resource : listOfResources) {
resources.add(new ResourceBean(resource.getUrl(), resource.getName().toString(),
resource.getDescription(), null, username, null, null));
}
}else{
Resource res = (Resource)f;
resources.add(new ResourceBean(res.getUrl(), res.getName().toString(), res.getDescription(), null, username, null, null));
} else {
Resource res = (Resource) f;
resources.add(new ResourceBean(res.getUrl(), res.getName().toString(), res.getDescription(),
null, username, null, null));
}
}
}catch(Exception e){
} catch(Exception e) {
logger.error("Failed to read value for field " + field.getName() + " skipping", e);
}
}
}
/**
* Evaluate if the user has the admin role
* Throws exception if he/she doesn't
*/
public static void hasAdminRole(String username, DataCatalogue catalogue, String apiKey, String organization) throws Exception{
public static void hasAdminRole(String username, DataCatalogue catalogue, String apiKey, String organization)
throws Exception {
String role = catalogue.getRoleOfUserInOrganization(username, organization, apiKey);
logger.info("Role of the user " + username + " is " + role + " in " + organization);
if(role == null || role.isEmpty() || !role.equalsIgnoreCase(RolesCkanGroupOrOrg.ADMIN.toString()))
throw new Exception("You are not authorized to create a product. Please check you have the Catalogue-Administrator role!");
throw new Exception(
"You are not authorized to create a product. Please check you have the Catalogue-Administrator role!");
}
/**
* Check this record's name
* @param futureName
@ -393,22 +432,23 @@ public class CommonServiceUtils {
* @throws Exception on name check
*/
public static void checkName(String futureName, DataCatalogue catalogue) throws Exception {
if(!HelperMethods.isNameValid(futureName)){
throw new Exception("The 'uuid_knowledge_base' must contain only alphanumeric characters, and symbols like '.' or '_', '-'");
}else{
if(!HelperMethods.isNameValid(futureName)) {
throw new Exception(
"The 'uuid_knowledge_base' must contain only alphanumeric characters, and symbols like '.' or '_', '-'");
} else {
logger.debug("Checking if such name [" + futureName + "] doesn't exist ...");
boolean alreadyExists = catalogue.existProductWithNameOrId(futureName);
if(alreadyExists){
if(alreadyExists) {
logger.debug("A product with 'uuid_knowledge_base' " + futureName + " already exists");
throw new Exception("A product with 'uuid_knowledge_base' " + futureName + " already exists");
}
}
}
/**
* Validate and check sources
* @param apiKey
@ -425,57 +465,62 @@ public class CommonServiceUtils {
* @throws Exception
*/
public static void validateRecordAndMapFields(String apiKey, String context, ServletContext contextServlet,
Sources sourceInPath, Common record, Product_Type productType, Set<String> tags, Map<String, List<String>> customFields,
Set<String> groups, List<ResourceBean> resources, String username, String futureTitle) throws Exception {
Sources sourceInPath, Common record, Product_Type productType, Set<String> tags,
Map<String,List<String>> customFields, Set<String> groups, List<ResourceBean> resources, String username,
String futureTitle) throws Exception {
// validate the record if it is a GRSF one and set the record type and in manage context
// Status field is needed only in the Manage context for GRSF records
if(context.equals((String)contextServlet.getInitParameter(HelperMethods.MANAGE_CONTEX_KEY))){
if(sourceInPath.equals(Sources.GRSF)){
if(context.equals((String) contextServlet.getInitParameter(HelperMethods.MANAGE_CONTEX_KEY))) {
if(sourceInPath.equals(Sources.GRSF)) {
List<RefersToBean> refersTo = record.getRefersTo();
if(refersTo == null || refersTo.isEmpty())
throw new Exception("refers_to is empty for a GRSF record");
Set<String> sourcesList = new HashSet<String>();
String databaseSource = "";
// we have the id within the catalog of this record. This means that we can retrieve the record and its system:type
for (RefersToBean refersToBean : refersTo) {
for(RefersToBean refersToBean : refersTo) {
String sourceOrganization = getRecordOrganization(refersToBean.getId(), apiKey, context);
resources.add(new ResourceBean(refersToBean.getUrl(), sourceOrganization , "", null, username, null, null));
resources.add(new ResourceBean(refersToBean.getUrl(), sourceOrganization, "", null, username, null,
null));
sourcesList.add(sourceOrganization.toLowerCase());
databaseSource += sourceOrganization + " ";
}
// create the Database Source information
customFields.put(Constants.GRSF_DATABASE_SOURCE, Arrays.asList(databaseSource.trim()));
// append to groups: we need to add this record to the correspondent group of the sources
addRecordToGroupSources(groups, new ArrayList(sourcesList), productType, sourceInPath);
// validate
CommonServiceUtils.validateAggregatedRecord(record);
}
}
// set the domain
record.setDomain(productType.getOrigName());
// set system type (it is equal to the GRSF Type for GRSF records, "Legacy" for source records)
record.setSystemType(sourceInPath.equals(Sources.GRSF) ?
productType.equals(Product_Type.FISHERY) ? ((FisheryRecord)record).getType().getOrigName() : ((StockRecord)record).getType().getOrigName()
record.setSystemType(
sourceInPath.equals(Sources.GRSF)
? productType.equals(Product_Type.FISHERY) ? ((FisheryRecord) record).getType().getOrigName()
: ((StockRecord) record).getType().getOrigName()
: Constants.SYSTEM_TYPE_FOR_SOURCES_VALUE);
logger.debug("Domain is " + productType.getOrigName() + " and system type " + record.getSystemType());
// evaluate the custom fields/tags, resources and groups
groups.add(sourceInPath.getOrigName().toLowerCase() + "-" + productType.getOrigName().toLowerCase()); //e.g. grsf-fishery
boolean skipTags = !sourceInPath.equals(Sources.GRSF); // no tags for the Original records
CommonServiceUtils.getTagsGroupsResourcesExtrasByRecord(tags, skipTags, groups, false, resources, false, customFields, record, username, sourceInPath);
CommonServiceUtils.getTagsGroupsResourcesExtrasByRecord(tags, skipTags, groups, false, resources, false,
customFields, record, username, sourceInPath);
}
/**
* Add the record to the group of sources
* @param groups
@ -483,18 +528,18 @@ public class CommonServiceUtils {
* @param productType
* @param sourceInPath
*/
private static void addRecordToGroupSources(Set<String> groups,
List<String> sourcesList, Product_Type productType, Sources sourceInPath) {
private static void addRecordToGroupSources(Set<String> groups, List<String> sourcesList, Product_Type productType,
Sources sourceInPath) {
Collections.sort(sourcesList); // be sure the name are sorted because the groups have been generated this way
String groupName = sourceInPath.getOrigName().toLowerCase() + "-" + productType.getOrigName().toLowerCase();
for (String source : sourcesList) {
for(String source : sourcesList) {
groupName += "-" + source;
}
groups.add(groupName);
}
/**
* Fetch the system:type property from a record
* @param itemIdOrName
@ -502,8 +547,8 @@ public class CommonServiceUtils {
* @return null on error
* @throws Exception
*/
public static String getSystemTypeValue(String itemIdOrName, String apiKey, String context) throws Exception{
public static String getSystemTypeValue(String itemIdOrName, String apiKey, String context) throws Exception {
DataCatalogue catalog = HelperMethods.getDataCatalogueRunningInstance(context);
CkanDataset dataset = catalog.getDataset(itemIdOrName, apiKey);
if(dataset == null)
@ -513,10 +558,10 @@ public class CommonServiceUtils {
throw new Exception(Constants.SYSTEM_TYPE_CUSTOM_KEY + " property not set in record " + itemIdOrName);
else
return systemTypeValue;
}
public static String getRecordOrganization(String itemIdOrName, String apiKey, String context) throws Exception{
public static String getRecordOrganization(String itemIdOrName, String apiKey, String context) throws Exception {
DataCatalogue catalog = HelperMethods.getDataCatalogueRunningInstance(context);
CkanDataset dataset = catalog.getDataset(itemIdOrName, apiKey);
if(dataset == null)
@ -524,7 +569,7 @@ public class CommonServiceUtils {
else
return dataset.getOrganization().getTitle();
}
/**
* Actions to execute once the dataset has been updated or created.
* @param responseBean
@ -539,71 +584,69 @@ public class CommonServiceUtils {
* @param partialDescription
* @throws InterruptedException
*/
public static void actionsPostCreateOrUpdate(
final String datasetId, final String futureName, final Common record, final String apiKey, final String username, final String organization, String itemUrl,
ResponseCreationBean responseBean, final DataCatalogue catalogue,
Map<String, String> namespaces, final Set<String> groups, final String context,
final String token, final String futureTitle, final String authorFullname, final ServletContext contextServlet, final boolean isUpdated,
public static void actionsPostCreateOrUpdate(final String datasetId, final String futureName, final Common record,
final String apiKey, final String username, final String organization, String itemUrl,
ResponseCreationBean responseBean, final DataCatalogue catalogue, Map<String,String> namespaces,
final Set<String> groups, final String context, final String token, final String futureTitle,
final String authorFullname, final ServletContext contextServlet, final boolean isUpdated,
String description) throws InterruptedException {
// on create, we need to add the item url... the description can be set on create and update instead
if(!isUpdated){
if(!isUpdated) {
itemUrl = catalogue.getUnencryptedUrlFromDatasetIdOrName(futureName);
Map<String, List<String>> addField = new HashMap<String, List<String>>();
String modifiedUUIDKey = namespaces.containsKey(Constants.ITEM_URL_FIELD) ? namespaces.get(Constants.ITEM_URL_FIELD) : Constants.ITEM_URL_FIELD;
Map<String,List<String>> addField = new HashMap<String,List<String>>();
String modifiedUUIDKey = namespaces.containsKey(Constants.ITEM_URL_FIELD)
? namespaces.get(Constants.ITEM_URL_FIELD)
: Constants.ITEM_URL_FIELD;
addField.put(modifiedUUIDKey, Arrays.asList(itemUrl));
catalogue.patchProductCustomFields(datasetId, apiKey, addField, false);
}
// update description anyway
description += "Record URL: " + itemUrl;
JSONObject obj = new JSONObject();
obj.put("notes", description);
catalogue.patchProductWithJSON(datasetId, obj, apiKey);
// set info in the response bean
responseBean.setId(datasetId);
responseBean.setItemUrl(itemUrl);
responseBean.setKbUuid(record.getUuid());
// it is needed...
final String itemUrlForThread = itemUrl;
new Thread(new Runnable() {
@Override
public void run() {
try {
// manage groups (wait thread to die: ckan doesn't support too much concurrency on same record ...)
if(!groups.isEmpty()){
if(!groups.isEmpty()) {
logger.info("Launching thread for association to the list of groups " + groups);
AssociationToGroupThread threadGroups = new AssociationToGroupThread(new ArrayList<String>(groups), datasetId, organization, username, catalogue, apiKey);
AssociationToGroupThread threadGroups = new AssociationToGroupThread(
new ArrayList<String>(groups), datasetId, organization, username, catalogue, apiKey);
threadGroups.start();
threadGroups.join();
}
// manage time series as resources
logger.info("Launching thread for time series handling");
new ManageTimeSeriesThread(record, futureName, username, catalogue, context, token).start();
// write a post if the product has been published in grsf context
if(!isUpdated && context.equals((String)contextServlet.getInitParameter(HelperMethods.PUBLIC_CONTEX_KEY))){
new WritePostCatalogueManagerThread(
context,
token,
futureTitle,
itemUrlForThread,
true,
new ArrayList<String>(),
authorFullname).start();
if(!isUpdated && context
.equals((String) contextServlet.getInitParameter(HelperMethods.PUBLIC_CONTEX_KEY))) {
new WritePostCatalogueManagerThread(context, token, futureTitle, itemUrlForThread, true,
new ArrayList<String>(), authorFullname).start();
logger.info("Thread to write a post about the new product has been launched");
}
}catch (InterruptedException e) {
} catch(InterruptedException e) {
logger.error("Error", e);
}
}
}).start();
}
/**
* Extend roles to other organization
* @param username
@ -611,18 +654,19 @@ public class CommonServiceUtils {
* @param organization
* @param admin
*/
public static void extendRoleToOtherOrganizations(String username, DataCatalogue catalogue, String organization, RolesCkanGroupOrOrg admin) {
public static void extendRoleToOtherOrganizations(String username, DataCatalogue catalogue, String organization,
RolesCkanGroupOrOrg admin) {
logger.debug("Checking if role extension is needed here");
if(extensionsCheck.containsKey(username) && extensionsCheck.get(username))
return;
else{
else {
catalogue.assignRolesOtherOrganization(username, organization, admin);
extensionsCheck.put(username, true);
}
}
/**
* Evaluate in which organization a record has to be published. The only exception is when grsf_admin is involved.
* @param organization