gCubeSystem
/
catalogue-util-library
16
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

just some updates

This commit is contained in:
Francesco Mangiacrapa 2021-02-08 17:44:09 +01:00
parent c1185a77b9
commit ba56fba7d3
7 changed files with 264 additions and 202 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
CHANGELOG.md
View File

@ -7,4 +7,4 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm
## [v0.1.0-SNAPSHOT] - 2020-08-27
First Release
[#19378] First Release

7
pom.xml
View File

@ -74,13 +74,6 @@
<scope>compile</scope>
</dependency>
<!-- <dependency> -->
<!-- <groupId>org.apache.httpcomponents</groupId> -->
<!-- <artifactId>fluent-hc</artifactId> -->
<!-- <version>4.5.3</version> -->
<!-- <scope>provided</scope> -->
<!-- </dependency> -->
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>

3
src/main/java/org/gcube/datacatalogue/ckanutillibrary/ckan/ExtendCkanClient.java
View File

@ -8,7 +8,6 @@ import java.net.URLEncoder;
import java.util.Arrays;
import java.util.HashMap;
import java.util.Map;
import java.util.logging.Level;
import javax.annotation.Nullable;
@ -28,8 +27,6 @@ import eu.trentorise.opendata.jackan.model.CkanGroup;
import eu.trentorise.opendata.jackan.model.CkanOrganization;
import eu.trentorise.opendata.jackan.model.CkanResponse;
// TODO: Auto-generated Javadoc
/**
* The Class SimpleExtendCkanClient.
*

19
src/main/java/org/gcube/datacatalogue/ckanutillibrary/ckan/MarshUnmarshCkanObject.java
View File

@ -2,7 +2,6 @@ package org.gcube.datacatalogue.ckanutillibrary.ckan;
import java.io.IOException;
import com.fasterxml.jackson.annotation.JsonInclude.Include;
import com.fasterxml.jackson.core.JsonParseException;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.JsonMappingException;
@ -12,23 +11,25 @@ import eu.trentorise.opendata.jackan.model.CkanDataset;
import eu.trentorise.opendata.jackan.model.CkanGroup;
import eu.trentorise.opendata.jackan.model.CkanResource;
// TODO: Auto-generated Javadoc
/**
* The Class MarshUnmarshCkanObject.
*
* @author Francesco Mangiacrapa at ISTI-CNR Pisa (Italy)
* Jun 3, 2020
* @author Francesco Mangiacrapa at ISTI-CNR (francesco.mangiacrapa@isti.cnr.it)
*
* Feb 8, 2021
*/
public class MarshUnmarshCkanObject {
/**
* The Enum METHOD.
*
* @author Francesco Mangiacrapa at ISTI-CNR Pisa (Italy)
* Sep 2, 2020
* @author Francesco Mangiacrapa at ISTI-CNR (francesco.mangiacrapa@isti.cnr.it)
*
* Feb 8, 2021
*/
public static enum METHOD {TO_READ, TO_CREATE}
public static enum METHOD {TO_READ, TO_CREATE, TO_UPDATE}
/**
@ -40,7 +41,7 @@ public class MarshUnmarshCkanObject {
*/
private static ObjectMapper getObjectMapper(METHOD method, Class clazz) {
ObjectMapper mapper = ExtendCkanClient.getObjectMapper();
if(method!=null && method.equals(METHOD.TO_CREATE))
if(method!=null && (method.equals(METHOD.TO_CREATE) || method.equals(METHOD.TO_UPDATE)))
return ExtendCkanClient.getObjectMapperForPosting(clazz);
return mapper;

224
src/main/java/org/gcube/datacatalogue/ckanutillibrary/server/DataCatalogueImpl.java
View File

@ -6,10 +6,8 @@ import static com.google.common.base.Preconditions.checkNotNull;
import java.net.URLEncoder;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.concurrent.ConcurrentHashMap;
import org.gcube.common.authorization.library.provider.SecurityTokenProvider;
@ -21,6 +19,7 @@ import org.gcube.datacatalogue.ckanutillibrary.ckan.MarshUnmarshCkanObject.METHO
import org.gcube.datacatalogue.ckanutillibrary.db.DBCaller;
import org.gcube.datacatalogue.ckanutillibrary.gcat.GCatCaller;
import org.gcube.datacatalogue.ckanutillibrary.server.DataCatalogueRunningCluster.ACCESS_LEVEL_TO_CATALOGUE_PORTLET;
import org.gcube.datacatalogue.ckanutillibrary.server.utils.CKANConveter;
import org.gcube.datacatalogue.ckanutillibrary.server.utils.CatalogueUtilMethods;
import org.gcube.datacatalogue.ckanutillibrary.server.utils.url.EntityContext;
import org.gcube.datacatalogue.ckanutillibrary.shared.LandingPages;
@ -43,13 +42,8 @@ import eu.trentorise.opendata.jackan.model.CkanDataset;
import eu.trentorise.opendata.jackan.model.CkanGroup;
import eu.trentorise.opendata.jackan.model.CkanLicense;
import eu.trentorise.opendata.jackan.model.CkanOrganization;
import eu.trentorise.opendata.jackan.model.CkanPair;
import eu.trentorise.opendata.jackan.model.CkanResource;
import eu.trentorise.opendata.jackan.model.CkanTag;
import eu.trentorise.opendata.jackan.model.CkanUser;
import net.htmlparser.jericho.Renderer;
import net.htmlparser.jericho.Segment;
import net.htmlparser.jericho.Source;
/**
* This is the Ckan Utils implementation class.
@ -699,7 +693,7 @@ public class DataCatalogueImpl implements DataCatalogue {
String author, String authorMail, String maintainer, String maintainerMail, long version,
String description, String licenseId, List<String> tags, Map<String, List<String>> customFieldsMultiple,
List<ResourceBean> resources, boolean setPublic, boolean setSearchable, boolean socialPost) throws Exception {
LOG.info("Called createCKanDatasetMultipleCustomFields");
LOG.info("Called createCkanDatasetMultipleCustomFields");
// checks (minimum)
checkNotNull(username);
@ -707,185 +701,51 @@ public class DataCatalogueImpl implements DataCatalogue {
checkArgument(!organizationNameOrId.isEmpty());
checkArgument(!(title == null && name == null || title.isEmpty() && name.isEmpty()), "Name and Title cannot be empty/null at the same time!");
// String ckanUsername = getUserFromApiKey(apiKey).getName();
CkanDataset dataset = convertToCkanDataset(username, title, name, organizationNameOrId, author, authorMail,
maintainer, maintainerMail, version, description, licenseId, tags, null, customFieldsMultiple,
resources, setPublic, setSearchable);
// trying to create by gCat
String jsonValueDataset = MarshUnmarshCkanObject.toJsonValueDataset(dataset,METHOD.TO_CREATE);
LOG.info("Serialized dataset is: " + jsonValueDataset);
jsonValueDataset = gCatCaller.createDataset(jsonValueDataset,socialPost);
LOG.debug("Created dataset is: " + jsonValueDataset);
if(jsonValueDataset != null){
CkanDataset toCkanDataset = MarshUnmarshCkanObject.toCkanDataset(jsonValueDataset,METHOD.TO_READ);
LOG.info("Dataset with name " + toCkanDataset.getName() + " has been created correctly");
//TODO FOLLOWING OPERATIONS NOT SHOULD LONGER NEEDED ON DATASET CREATION, MUST BE CHECKED with @LUCA
try {
// String ckanUsername = getUserFromApiKey(apiKey).getName();
CkanDataset dataset = CKANConveter.toCkanDataset(ckanCaller, username, title, name, organizationNameOrId, author, authorMail,
maintainer, maintainerMail, version, description, licenseId, tags, null, customFieldsMultiple,
resources, setPublic, setSearchable);
// // set visibility
// boolean visibilitySet = setDatasetPrivate(
// !setPublic, // swap to private
// res.getOrganization().getId(),
// res.getId(),
// CKAN_TOKEN_SYS); // use sysadmin api key to be sure it will be set
//
// logger.info("Was visibility set to " + (setPublic ? "public" : "private") + "? " + visibilitySet);
//
// // set searchable to true if dataset visibility is private
// if(!setPublic){ // (swap to private)
// boolean searchableSet = setSearchableField(res.getId(), true);
// logger.info("Was searchable set to True? " + searchableSet);
// }
// return res.getId();
// trying to create by gCat
String jsonValueDataset = MarshUnmarshCkanObject.toJsonValueDataset(dataset,METHOD.TO_CREATE);
LOG.info("Serialized dataset is: " + jsonValueDataset);
jsonValueDataset = gCatCaller.createDataset(jsonValueDataset,socialPost);
LOG.debug("Created dataset is: " + jsonValueDataset);
return toCkanDataset.getId();
if(jsonValueDataset != null){
CkanDataset toCkanDataset = MarshUnmarshCkanObject.toCkanDataset(jsonValueDataset,METHOD.TO_READ);
LOG.info("Dataset with name " + toCkanDataset.getName() + " has been created correctly");
//TODO FOLLOWING OPERATIONS NOT SHOULD LONGER NEEDED ON DATASET CREATION, MUST BE CHECKED with @LUCA
// // set visibility
// boolean visibilitySet = setDatasetPrivate(
// !setPublic, // swap to private
// res.getOrganization().getId(),
// res.getId(),
// CKAN_TOKEN_SYS); // use sysadmin api key to be sure it will be set
//
// logger.info("Was visibility set to " + (setPublic ? "public" : "private") + "? " + visibilitySet);
//
// // set searchable to true if dataset visibility is private
// if(!setPublic){ // (swap to private)
// boolean searchableSet = setSearchableField(res.getId(), true);
// logger.info("Was searchable set to True? " + searchableSet);
// }
// return res.getId();
return toCkanDataset.getId();
}
}catch (Exception e) {
LOG.error("Error on creating the dataset: ", e);
}
return null;
}
/**
* Convert to ckan dataset. This method creates the body of the dataset
*
* @param username the username
* @param title the title
* @param name the name
* @param organizationNameOrId the organization name or id
* @param author the author
* @param authorMail the author mail
* @param maintainer the maintainer
* @param maintainerMail the maintainer mail
* @param version the version
* @param description the description
* @param licenseId the license id
* @param tags the tags
* @param customFields the custom fields
* @param customFieldsMultipleValues the custom fields multiple values
* @param resources the resources
* @param setPublic the set public
* @param setSearchable the set searchable
* @return the ckan dataset
*/
private CkanDataset convertToCkanDataset(String username, String title, String name, String organizationNameOrId, String author,
String authorMail, String maintainer, String maintainerMail, long version, String description,
String licenseId, List<String> tags, Map<String, String> customFields,
Map<String, List<String>> customFieldsMultipleValues, List<ResourceBean> resources, boolean setPublic, boolean setSearchable) {
LOG.info("Called convert data to CkanDataset");
//ExtendCkanClient client = new ExtendCkanClient(CKAN_CATALOGUE_URL, apiKey);
//String ckanUsername = getUserFromApiKey(apiKey).getName();
CkanDataset dataset = new CkanDataset();
// in order to avoid errors, the username is always converted
String ckanUsername = CatalogueUtilMethods.fromUsernameToCKanUsername(username);
String nameToUse = name;
if(nameToUse == null)
nameToUse = CatalogueUtilMethods.fromProductTitleToName(title);
LOG.debug("Name of the dataset is going to be " + nameToUse + ". Title is going to be " + title);
dataset.setName(nameToUse);
dataset.setTitle(title);
CkanOrganization orgOwner = ckanCaller.getOrganization(organizationNameOrId);
dataset.setOwnerOrg(orgOwner.getId());
dataset.setAuthor(author);
dataset.setAuthorEmail(authorMail);
dataset.setMaintainer(maintainer);
dataset.setMaintainerEmail(maintainerMail);
dataset.setVersion(String.valueOf(version));
//set is private
dataset.setPriv(!setPublic);
// description must be escaped
if(description != null && !description.isEmpty()){
Source descriptionEscaped = new Source(description);
Segment htmlSeg = new Segment(descriptionEscaped, 0, descriptionEscaped.length());
Renderer htmlRend = new Renderer(htmlSeg);
dataset.setNotes(htmlRend.toString());
LOG.debug("Description escaped " + htmlRend.toString());
}
dataset.setLicenseId(licenseId);
// set the tags, if any
if(tags != null && !tags.isEmpty()){
List<CkanTag> ckanTags = new ArrayList<CkanTag>(tags.size());
for (String stringTag : tags) {
ckanTags.add(new CkanTag(stringTag));
}
dataset.setTags(ckanTags);
dataset.setNumTags(ckanTags.size());
}
// set the custom fields, if any
List<CkanPair> extras = new ArrayList<CkanPair>();
if(customFields != null && !customFields.isEmpty()){
Iterator<Entry<String, String>> iterator = customFields.entrySet().iterator();
while (iterator.hasNext()) {
Map.Entry<String, String> entry = iterator.next();
extras.add(new CkanPair(entry.getKey(), entry.getValue()));
}
}else if(customFieldsMultipleValues != null && !customFieldsMultipleValues.isEmpty()){
Iterator<Entry<String, List<String>>> iterator = customFieldsMultipleValues.entrySet().iterator();
while (iterator.hasNext()) {
Map.Entry<String, List<String>> entry = iterator.next();
List<String> valuesForEntry = entry.getValue();
for (String value : valuesForEntry) {
extras.add(new CkanPair(entry.getKey(), value));
}
}
}
dataset.setExtras(extras);
// check if we need to add the resources
if(resources != null && !resources.isEmpty()){
LOG.debug("We need to add resources to the dataset");
try{
List<CkanResource> resourcesCkan = new ArrayList<CkanResource>();
for(ResourceBean resource: resources){
LOG.debug("Going to add resource described by " + resource);
CkanResource newResource = new CkanResource();
newResource.setDescription(resource.getDescription());
newResource.setId(resource.getId());
newResource.setUrl(resource.getUrl());
newResource.setName(resource.getName());
newResource.setMimetype(resource.getMimeType());
newResource.setFormat(resource.getMimeType());
newResource.setOwner(ckanUsername);
resourcesCkan.add(newResource);
}
// add to the dataset
dataset.setResources(resourcesCkan);
dataset.setNumResources(resourcesCkan.size());
}catch(Exception e){
LOG.error("Unable to add those resources to the dataset", e);
}
}
return dataset;
}
@Override
public boolean patchFieldsForDataset(String datasetId, Map<String, String> mapFields) throws Exception {
LOG.info("Called patch the fields "+mapFields+" for dataset id: "+datasetId);
@ -942,7 +802,7 @@ public class DataCatalogueImpl implements DataCatalogue {
if(CatalogueUtilMethods.resourceExists(resourceBean.getUrl())){
// in order to avoid errors, the username is always converted
/*// in order to avoid errors, the username is always converted
String ckanUsername = CatalogueUtilMethods.fromUsernameToCKanUsername(resourceBean.getOwner());
CkanResource resource = new CkanResource(CKAN_CATALOGUE_URL, resourceBean.getDatasetId());
@ -955,7 +815,9 @@ public class DataCatalogueImpl implements DataCatalogue {
resource.setDescription(htmlRend.toString());
resource.setUrl(resourceBean.getUrl());
resource.setOwner(ckanUsername);
resource.setOwner(ckanUsername);*/
CkanResource resource = CKANConveter.toCkanResource(CKAN_CATALOGUE_URL, resourceBean);
String jsonValueResource = MarshUnmarshCkanObject.toJsonValueResource(resource);
LOG.trace("Serialized resource is: " + jsonValueResource);

209
src/main/java/org/gcube/datacatalogue/ckanutillibrary/server/utils/CKANConveter.java Normal file
View File

@ -0,0 +1,209 @@
package org.gcube.datacatalogue.ckanutillibrary.server.utils;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import org.gcube.datacatalogue.ckanutillibrary.ckan.ExtendCkanClient;
import org.gcube.datacatalogue.ckanutillibrary.shared.ResourceBean;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import eu.trentorise.opendata.jackan.model.CkanDataset;
import eu.trentorise.opendata.jackan.model.CkanOrganization;
import eu.trentorise.opendata.jackan.model.CkanPair;
import eu.trentorise.opendata.jackan.model.CkanResource;
import eu.trentorise.opendata.jackan.model.CkanTag;
import net.htmlparser.jericho.Renderer;
import net.htmlparser.jericho.Segment;
import net.htmlparser.jericho.Source;
/**
* The Class CKANConveter.
*
* @author Francesco Mangiacrapa at ISTI-CNR (francesco.mangiacrapa@isti.cnr.it)
*
* Feb 8, 2021
*/
public class CKANConveter {
private static final Logger LOG = LoggerFactory.getLogger(CKANConveter.class);
/**
* Convert to ckan dataset. This method creates the body of the dataset
*
* @param ckanCaller the ckan caller
* @param username the username
* @param title the title
* @param name the name
* @param organizationNameOrId the organization name or id
* @param author the author
* @param authorMail the author mail
* @param maintainer the maintainer
* @param maintainerMail the maintainer mail
* @param version the version
* @param description the description
* @param licenseId the license id
* @param tags the tags
* @param customFields the custom fields
* @param customFieldsMultipleValues the custom fields multiple values
* @param resources the resources
* @param setPublic the set public
* @param setSearchable the set searchable
* @return the ckan dataset
*/
public static CkanDataset toCkanDataset(ExtendCkanClient ckanCaller, String username, String title, String name, String organizationNameOrId, String author,
String authorMail, String maintainer, String maintainerMail, long version, String description,
String licenseId, List<String> tags, Map<String, String> customFields,
Map<String, List<String>> customFieldsMultipleValues, List<ResourceBean> resources, boolean setPublic, boolean setSearchable) {
LOG.info("Called convert data to CkanDataset");
//ExtendCkanClient client = new ExtendCkanClient(CKAN_CATALOGUE_URL, apiKey);
//String ckanUsername = getUserFromApiKey(apiKey).getName();
CkanDataset dataset = new CkanDataset();
// in order to avoid errors, the username is always converted
String ckanUsername = CatalogueUtilMethods.fromUsernameToCKanUsername(username);
String nameToUse = name;
if(nameToUse == null)
nameToUse = CatalogueUtilMethods.fromProductTitleToName(title);
LOG.debug("Name of the dataset is going to be " + nameToUse + ". Title is going to be " + title);
dataset.setName(nameToUse);
dataset.setTitle(title);
//TODO SHOULD BE REVISITED by gCAT?
CkanOrganization orgOwner = ckanCaller.getOrganization(organizationNameOrId);
dataset.setOwnerOrg(orgOwner.getId());
dataset.setAuthor(author);
dataset.setAuthorEmail(authorMail);
dataset.setMaintainer(maintainer);
dataset.setMaintainerEmail(maintainerMail);
dataset.setVersion(String.valueOf(version));
//set is private
dataset.setPriv(!setPublic);
// description must be escaped
if(description != null && !description.isEmpty()){
Source descriptionEscaped = new Source(description);
Segment htmlSeg = new Segment(descriptionEscaped, 0, descriptionEscaped.length());
Renderer htmlRend = new Renderer(htmlSeg);
dataset.setNotes(htmlRend.toString());
LOG.debug("Description escaped " + htmlRend.toString());
}
dataset.setLicenseId(licenseId);
// set the tags, if any
if(tags != null && !tags.isEmpty()){
List<CkanTag> ckanTags = new ArrayList<CkanTag>(tags.size());
for (String stringTag : tags) {
ckanTags.add(new CkanTag(stringTag));
}
dataset.setTags(ckanTags);
dataset.setNumTags(ckanTags.size());
}
// set the custom fields, if any
List<CkanPair> extras = new ArrayList<CkanPair>();
if(customFields != null && !customFields.isEmpty()){
Iterator<Entry<String, String>> iterator = customFields.entrySet().iterator();
while (iterator.hasNext()) {
Map.Entry<String, String> entry = iterator.next();
extras.add(new CkanPair(entry.getKey(), entry.getValue()));
}
}else if(customFieldsMultipleValues != null && !customFieldsMultipleValues.isEmpty()){
Iterator<Entry<String, List<String>>> iterator = customFieldsMultipleValues.entrySet().iterator();
while (iterator.hasNext()) {
Map.Entry<String, List<String>> entry = iterator.next();
List<String> valuesForEntry = entry.getValue();
for (String value : valuesForEntry) {
extras.add(new CkanPair(entry.getKey(), value));
}
}
}
dataset.setExtras(extras);
// check if we need to add the resources
if(resources != null && !resources.isEmpty()){
LOG.debug("We need to add resources to the dataset");
try{
List<CkanResource> resourcesCkan = new ArrayList<CkanResource>();
for(ResourceBean resource: resources){
LOG.debug("Going to add resource described by " + resource);
CkanResource newResource = new CkanResource();
newResource.setDescription(resource.getDescription());
newResource.setId(resource.getId());
newResource.setUrl(resource.getUrl());
newResource.setName(resource.getName());
newResource.setMimetype(resource.getMimeType());
newResource.setFormat(resource.getMimeType());
newResource.setOwner(ckanUsername);
resourcesCkan.add(newResource);
}
// add to the dataset
dataset.setResources(resourcesCkan);
dataset.setNumResources(resourcesCkan.size());
}catch(Exception e){
LOG.error("Unable to add those resources to the dataset", e);
}
}
return dataset;
}
/**
* To ckan resource.
*
* @param CKAN_CATALOGUE_URL the ckan catalogue url
* @param resourceBean the resource bean
* @return the ckan resource
*/
public static CkanResource toCkanResource(String CKAN_CATALOGUE_URL, ResourceBean resourceBean) {
// in order to avoid errors, the username is always converted
String ckanUsername = CatalogueUtilMethods.fromUsernameToCKanUsername(resourceBean.getOwner());
CkanResource resource = new CkanResource(CKAN_CATALOGUE_URL, resourceBean.getDatasetId());
resource.setName(resourceBean.getName());
// escape description
Source description = new Source(resourceBean.getDescription());
Segment htmlSeg = new Segment(description, 0, description.length());
Renderer htmlRend = new Renderer(htmlSeg);
resource.setDescription(htmlRend.toString());
resource.setUrl(resourceBean.getUrl());
//TODO SHOULD BE REVISITED by gCAT?
resource.setOwner(ckanUsername);
return resource;
}
}

2
src/test/java/org/gcube/datacatalogue/ckanutillibrary/test/TestDataCatalogueLib.java
View File

@ -38,7 +38,7 @@ public class TestDataCatalogueLib {
//private String scope = "/d4science.research-infrastructures.eu/SoBigData/Catalogue-TerritoriAperti";
//private String scope = "/d4science.research-infrastructures.eu/gCubeApps/BiodiversityLab";
//private String testUser = "costantino_perciante";
private String testUser = "francescomangiacrapa";
private String testUser = "francesco.mangiacrapa";
private String authorizationToken = "";
String subjectId = "aa_father4";
String objectId = "bb_son4";