gCubeSystem
/
catalogue-util-library
16
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

on going on migration to gCat

This commit is contained in:
Francesco Mangiacrapa 2020-06-03 14:39:41 +02:00
parent 319564e993
commit 95da82b9b6
24 changed files with 941 additions and 697 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

140
src/main/java/org/gcube/datacatalogue/ckanutillibrary/ckan/DirectCkanCaller.java Normal file
View File

@ -0,0 +1,140 @@
package org.gcube.datacatalogue.ckanutillibrary.ckan;
import static com.google.common.base.Preconditions.checkArgument;
import static com.google.common.base.Preconditions.checkNotNull;
import org.json.simple.JSONArray;
import org.json.simple.JSONObject;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import eu.trentorise.opendata.jackan.internal.org.apache.http.HttpResponse;
import eu.trentorise.opendata.jackan.internal.org.apache.http.HttpStatus;
import eu.trentorise.opendata.jackan.internal.org.apache.http.client.methods.HttpPost;
import eu.trentorise.opendata.jackan.internal.org.apache.http.entity.StringEntity;
import eu.trentorise.opendata.jackan.internal.org.apache.http.impl.client.CloseableHttpClient;
import eu.trentorise.opendata.jackan.internal.org.apache.http.impl.client.HttpClientBuilder;
// TODO: Auto-generated Javadoc
/**
* The Class DirectCkanCaller.
*
* @author Francesco Mangiacrapa at ISTI-CNR Pisa (Italy)
* Jun 3, 2020
*/
public class DirectCkanCaller {
private static final String PATH_PACKAGE_PATCH = "/api/3/action/package_patch";
public final static String PATH_SET_PRIVATE_DATASET = "/api/3/action/bulk_update_private";
public final static String PATH_SET_PUBLIC_DATASET = "/api/3/action/bulk_update_public";
private String catalogueURL;
private static final Logger LOG = LoggerFactory.getLogger(DirectCkanCaller.class);
/**
* Instantiates a new direct ckan caller.
*
* @param catalogueURL the catalogue URL
*/
public DirectCkanCaller(String catalogueURL){
this.catalogueURL = catalogueURL;
}
/**
* Set dataset private/public.
*
* @param priv the priv
* @param organizationId (NOTE: The ID, not the name!)
* @param datasetId (NOTE: The ID, not the name!)
* @param apiKey the user's api key
* @return true on success, false otherwise
*/
public boolean setDatasetPrivate(boolean priv, String organizationId,
String datasetId, String apiKey) {
// checks
checkNotNull(organizationId);
checkNotNull(apiKey);
checkNotNull(datasetId);
checkArgument(!apiKey.isEmpty());
checkArgument(!datasetId.isEmpty());
checkArgument(!organizationId.isEmpty());
JSONObject obj = new JSONObject();
obj.put("org_id", organizationId);
JSONArray array = new JSONArray();
array.add(datasetId);
obj.put("datasets", array);
try(CloseableHttpClient httpClient = HttpClientBuilder.create().build();) {
HttpPost request;
if(priv)
request = new HttpPost(catalogueURL + PATH_SET_PRIVATE_DATASET);
else
request = new HttpPost(catalogueURL + PATH_SET_PUBLIC_DATASET);
LOG.info("Excuting request for making dataset with id " + datasetId + " " + (priv? "private" : "public"));
request.addHeader("Authorization", apiKey);
StringEntity params = new StringEntity(obj.toJSONString());
request.setEntity(params);
HttpResponse response = httpClient.execute(request);
LOG.info("Response code is " + response.getStatusLine().getStatusCode() + " and response message is " + response.getStatusLine().getReasonPhrase());
if(response.getStatusLine().getStatusCode() == HttpStatus.SC_OK)
return true;
}catch (Exception ex) {
LOG.error("Error while trying to set private the dataset ", ex);
}
return false;
}
/**
* Sets the searchable field.
*
* @param datasetId the dataset id
* @param searchable the searchable
* @return true, if successful
*/
public boolean setSearchableField(String datasetId, boolean searchable, String apiKey) {
// checks
checkNotNull(datasetId);
checkArgument(!datasetId.isEmpty());
String searchableAsString = searchable ? "True" : "False";
// Patch package path
String patchPackage = catalogueURL + PATH_PACKAGE_PATCH;
JSONObject obj = new JSONObject();
obj.put("id", datasetId);
obj.put("searchable", searchableAsString);
try(CloseableHttpClient httpClient = HttpClientBuilder.create().build();) {
HttpPost request = new HttpPost(patchPackage);
request.addHeader("Authorization", apiKey);
StringEntity params = new StringEntity(obj.toJSONString());
request.setEntity(params);
HttpResponse response = httpClient.execute(request);
LOG.debug("Response code is " + response.getStatusLine().getStatusCode() + " and response message is " + response.getStatusLine().getReasonPhrase());
if(response.getStatusLine().getStatusCode() == HttpStatus.SC_OK)
return true;
}catch (Exception ex) {
LOG.error("Error while trying to set searchable the dataset ", ex);
}
return false;
}
}

115
src/main/java/org/gcube/datacatalogue/ckanutillibrary/ckan/ExtendCkanClient.java
View File

@ -1,6 +1,3 @@
/**
*
*/
package org.gcube.datacatalogue.ckanutillibrary.ckan;
import static com.google.common.base.Preconditions.checkNotNull;
@ -19,7 +16,7 @@ import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.base.Charsets;
import com.google.common.io.CharStreams;
import eu.trentorise.opendata.jackan.CheckedCkanClient;
import eu.trentorise.opendata.jackan.CkanClient;
import eu.trentorise.opendata.jackan.exceptions.CkanException;
import eu.trentorise.opendata.jackan.exceptions.JackanException;
import eu.trentorise.opendata.jackan.internal.org.apache.http.client.fluent.Request;
@ -29,36 +26,34 @@ import eu.trentorise.opendata.jackan.model.CkanOrganization;
import eu.trentorise.opendata.jackan.model.CkanResponse;
// TODO: Auto-generated Javadoc
/**
* The Class ExtendCkanClient.
* The Class SimpleExtendCkanClient.
*
* @author Francesco Mangiacrapa at ISTI-CNR (francesco.mangiacrapa@isti.cnr.it)
*
* Jun 14, 2019
* Jun 7, 2019
*
*/
public class ExtendCkanClient extends CheckedCkanClient implements PatchedCkan{
private static final Logger logger = LoggerFactory.getLogger(ExtendCkanClient.class);
public class ExtendCkanClient extends CkanClient implements PatchedCkan{
@Nullable
private static ObjectMapper objectMapper;
private String catalogueURL;
private String ckanToken;
private int extendTimeout = 120000; //in milliseconds
@Nullable
private static ObjectMapper objectMapper;
private static final Logger logger = LoggerFactory.getLogger(ExtendCkanClient.class);
/**
* Instantiates a new extend ckan client.
* Instantiates a new simple extend ckan client.
*
* @param catalogueURL the catalogue url
* @param catalogueURL the catalogue URL
*/
public ExtendCkanClient(String catalogueURL) {
super(catalogueURL);
this.catalogueURL = catalogueURL;
}
/**
* Instantiates a new extend ckan client.
*
@ -84,7 +79,7 @@ public class ExtendCkanClient extends CheckedCkanClient implements PatchedCkan{
this.ckanToken = ckanToken;
this.extendTimeout = timeout;
}
/**
* Configures the request. Should work both for GETs and POSTs.
@ -104,38 +99,13 @@ public class ExtendCkanClient extends CheckedCkanClient implements PatchedCkan{
return request;
}
/**
* Gets the catalogue url.
*
* @return the catalogueURL
*/
public String getCatalogueURL() {
return catalogueURL;
}
/**
* Gets the ckan token.
*
* @return the ckanToken
*/
@Override
public String getCkanToken() {
return ckanToken;
}
/**
* Gets the timeout.
*
* @return the timeout
*/
public int getTimeout() {
return extendTimeout;
}
/* (non-Javadoc)
/**
* Gets the organization.
*
* @param idOrName the id or name
* @return the organization
*/
/* (non-Javadoc)
* @see eu.trentorise.opendata.jackan.CkanClient#getOrganization(java.lang.String)
*/
public synchronized CkanOrganization getOrganization(String idOrName) {
@ -147,6 +117,12 @@ public class ExtendCkanClient extends CheckedCkanClient implements PatchedCkan{
}
/**
* Gets the group.
*
* @param idOrName the id or name
* @return the group
*/
/* (non-Javadoc)
* @see eu.trentorise.opendata.jackan.CkanClient#getGroup(java.lang.String)
*/
@ -164,7 +140,7 @@ public class ExtendCkanClient extends CheckedCkanClient implements PatchedCkan{
*
* @return the object mapper
*/
static ObjectMapper getObjectMapper() {
public static ObjectMapper getObjectMapper() {
if (objectMapper == null) {
objectMapper = new ObjectMapper();
configureObjectMapper(objectMapper);
@ -172,6 +148,15 @@ public class ExtendCkanClient extends CheckedCkanClient implements PatchedCkan{
return objectMapper;
}
/**
* Gets the http.
*
* @param <T> the generic type
* @param responseType the response type
* @param path the path
* @param params the params
* @return the http
*/
/* (non-Javadoc)
* @see org.gcube.datacatalogue.ckanutillibrary.server.patch.PatchedCkan#getHttp(java.lang.Class, java.lang.String, java.lang.Object[])
*/
@ -215,6 +200,13 @@ public class ExtendCkanClient extends CheckedCkanClient implements PatchedCkan{
}
/**
* Calc full url.
*
* @param path the path
* @param params the params
* @return the string
*/
/* (non-Javadoc)
* @see org.gcube.datacatalogue.ckanutillibrary.server.patch.PatchedCkan#calcFullUrl(java.lang.String, java.lang.Object[])
*/
@ -236,5 +228,26 @@ public class ExtendCkanClient extends CheckedCkanClient implements PatchedCkan{
+ Arrays.toString(params), ex);
}
}
/**
* Gets the catalogue URL.
*
* @return the catalogue URL
*/
public String getCatalogueURL() {
return catalogueURL;
}
/**
* Gets the ckan token.
*
* @return the ckan token
*/
@Override
public String getCkanToken() {
return ckanToken;
}
}

38
src/main/java/org/gcube/datacatalogue/ckanutillibrary/ckan/MarshUnmarshCkanObject.java Normal file
View File

@ -0,0 +1,38 @@
package org.gcube.datacatalogue.ckanutillibrary.ckan;
import java.io.IOException;
import com.fasterxml.jackson.core.JsonParseException;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.JsonMappingException;
import eu.trentorise.opendata.jackan.model.CkanDataset;
public class MarshUnmarshCkanObject {
/**
* To ckan dataset.
*
* @param jsonDataset the json dataset
* @return the ckan dataset
* @throws IOException
* @throws JsonMappingException
* @throws JsonParseException
*/
public static CkanDataset toCkanDataset(String jsonDataset) throws JsonParseException, JsonMappingException, IOException {
return ExtendCkanClient.getObjectMapper().readValue(jsonDataset, CkanDataset.class);
}
/**
* To json value dataset.
*
* @param dataset the dataset
* @return the string
* @throws JsonProcessingException
*/
public static String toJsonValueDataset(CkanDataset dataset) throws JsonProcessingException {
return ExtendCkanClient.getObjectMapper().writeValueAsString(dataset);
}
}

168
src/main/java/org/gcube/datacatalogue/ckanutillibrary/ckan/SimpleExtendCkanClient.java
View File

@ -1,168 +0,0 @@
package org.gcube.datacatalogue.ckanutillibrary.ckan;
import static com.google.common.base.Preconditions.checkNotNull;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.net.URLEncoder;
import java.util.Arrays;
import javax.annotation.Nullable;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.base.Charsets;
import com.google.common.io.CharStreams;
import eu.trentorise.opendata.jackan.CkanClient;
import eu.trentorise.opendata.jackan.exceptions.CkanException;
import eu.trentorise.opendata.jackan.exceptions.JackanException;
import eu.trentorise.opendata.jackan.internal.org.apache.http.client.fluent.Request;
import eu.trentorise.opendata.jackan.internal.org.apache.http.client.fluent.Response;
import eu.trentorise.opendata.jackan.model.CkanGroup;
import eu.trentorise.opendata.jackan.model.CkanOrganization;
import eu.trentorise.opendata.jackan.model.CkanResponse;
/**
* The Class SimpleExtendCkanClient.
*
* @author Francesco Mangiacrapa at ISTI-CNR (francesco.mangiacrapa@isti.cnr.it)
*
* Jun 7, 2019
*/
public class SimpleExtendCkanClient extends CkanClient implements PatchedCkan{
private String catalogueURL;
@Nullable
private static ObjectMapper objectMapper;
private static final Logger logger = LoggerFactory.getLogger(SimpleExtendCkanClient.class);
/**
* Instantiates a new simple extend ckan client.
*
* @param catalogueURL the catalogue URL
*/
public SimpleExtendCkanClient(String catalogueURL) {
super(catalogueURL);
this.catalogueURL = catalogueURL;
}
/* (non-Javadoc)
* @see eu.trentorise.opendata.jackan.CkanClient#getOrganization(java.lang.String)
*/
public synchronized CkanOrganization getOrganization(String idOrName) {
checkNotNull(idOrName, "Need a valid id or name!");
logger.info("Patched read organization for id/name: {}", idOrName);
return getHttp(OrganizationResponse.class, "/api/3/action/organization_show", "id", idOrName,
"include_datasets", "false", "include_users", "true").result;
}
/* (non-Javadoc)
* @see eu.trentorise.opendata.jackan.CkanClient#getGroup(java.lang.String)
*/
public synchronized CkanGroup getGroup(String idOrName) {
checkNotNull(idOrName, "Need a valid id or name!");
logger.info("Patched read group for id/name: {}", idOrName);
return getHttp(GroupResponse.class, "/api/3/action/group_show", "id", idOrName, "include_datasets",
"false", "include_users", "true").result;
}
/**
* Retrieves the Jackson object mapper for reading operations. Internally,
* Object mapper is initialized at first call.
*
* @return the object mapper
*/
public static ObjectMapper getObjectMapper() {
if (objectMapper == null) {
objectMapper = new ObjectMapper();
configureObjectMapper(objectMapper);
}
return objectMapper;
}
/* (non-Javadoc)
* @see org.gcube.datacatalogue.ckanutillibrary.server.patch.PatchedCkan#getHttp(java.lang.Class, java.lang.String, java.lang.Object[])
*/
public <T extends CkanResponse> T getHttp(Class<T> responseType, String path, Object... params) {
checkNotNull(responseType);
checkNotNull(path);
String fullUrl = calcFullUrl(path, params);
T ckanResponse;
String returnedText;
try {
logger.debug("getting {}", fullUrl);
Request request = Request.Get(fullUrl);
configureRequest(request);
Response response = request.execute();
InputStream stream = response.returnResponse()
.getEntity()
.getContent();
try (InputStreamReader reader = new InputStreamReader(stream, Charsets.UTF_8)) {
returnedText = CharStreams.toString(reader);
}
logger.trace("returnedText {}", returnedText);
} catch (Exception ex) {
throw new CkanException("Error while performing GET. Request url was: " + fullUrl, this, ex);
}
try {
ckanResponse = getObjectMapper().readValue(returnedText, responseType);
} catch (Exception ex) {
throw new CkanException(
"Couldn't interpret json returned by the server! Returned text was: " + returnedText, this, ex);
}
if (!ckanResponse.isSuccess()) {
throwCkanException("Error while performing GET. Request url was: " + fullUrl, ckanResponse);
}
return ckanResponse;
}
/* (non-Javadoc)
* @see org.gcube.datacatalogue.ckanutillibrary.server.patch.PatchedCkan#calcFullUrl(java.lang.String, java.lang.Object[])
*/
public String calcFullUrl(String path, Object[] params) {
checkNotNull(path);
try {
StringBuilder sb = new StringBuilder().append(catalogueURL)
.append(path);
for (int i = 0; i < params.length; i += 2) {
sb.append(i == 0 ? "?" : "&")
.append(URLEncoder.encode(params[i].toString(), "UTF-8"))
.append("=")
.append(URLEncoder.encode(params[i + 1].toString(), "UTF-8"));
}
return sb.toString();
} catch (Exception ex) {
throw new JackanException("Error while building url to perform GET! \n path: " + path + " \n params: "
+ Arrays.toString(params), ex);
}
}
/**
* Gets the catalogue url.
*
* @return the catalogueURL
*/
public String getCatalogueURL() {
return catalogueURL;
}
}

10
src/main/java/org/gcube/datacatalogue/ckanutillibrary/db/DBCaller.java
View File

@ -95,7 +95,7 @@ public class DBCaller {
*/
public Map<String, RolesCkanGroupOrOrg> getGroupsByUserFromDB(String username){
checkNotNull(username);
LOG.debug("Get groups by user called");
LOG.debug("Get groups by user called for username: "+username);
//couples (groups id, capacity) of the user in the group
@ -129,6 +129,8 @@ public class DBCaller {
}finally{
closeConnection(connection);
}
LOG.debug("returning map: "+toReturn);
return toReturn;
}
@ -140,7 +142,7 @@ public class DBCaller {
* @return the user id for username
*/
private String getUserIdForUsername(String username) {
LOG.debug("Get user id for username called");
LOG.debug("Get user id for username called for username: "+username);
Connection connection = null;
String userId = null;
@ -150,7 +152,7 @@ public class DBCaller {
String selQuery = "SELECT \"id\" FROM \"public\".\"user\" WHERE \"name\"=?;";
LOG.info("Performing query: "+selQuery);
LOG.debug("Performing query: "+selQuery);
PreparedStatement preparedStatement = connection.prepareStatement(selQuery);
preparedStatement.setString(1, username);
@ -213,6 +215,8 @@ public class DBCaller {
}finally{
closeConnection(connection);
}
LOG.debug("returning map: "+toReturn);
return toReturn;
}

45
src/main/java/org/gcube/datacatalogue/ckanutillibrary/gcat/GCatCaller.java
View File

@ -2,19 +2,11 @@ package org.gcube.datacatalogue.ckanutillibrary.gcat;
import java.io.IOException;
import java.net.MalformedURLException;
import java.util.List;
import org.gcube.datacatalogue.ckanutillibrary.ckan.SimpleExtendCkanClient;
import org.gcube.gcat.client.Item;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.fasterxml.jackson.core.JsonParseException;
import com.fasterxml.jackson.databind.JsonMappingException;
import eu.trentorise.opendata.jackan.model.CkanDataset;
import eu.trentorise.opendata.jackan.model.CkanLicense;
// TODO: Auto-generated Javadoc
/**
* The Class GCatCaller.
@ -24,18 +16,17 @@ import eu.trentorise.opendata.jackan.model.CkanLicense;
*/
public class GCatCaller {
private SimpleExtendCkanClient jackanCkanClient;
private String catalogueURL;
private static final Logger LOG = LoggerFactory.getLogger(GCatCaller.class);
/**
* Instantiates a new g cat caller.
*
* @param catalogueURL the catalogue URL
*/
public GCatCaller(String catalogueURL) {
this.catalogueURL = catalogueURL;
this.jackanCkanClient = new SimpleExtendCkanClient(catalogueURL);
}
@ -43,27 +34,27 @@ public class GCatCaller {
* Gets the dataset for name.
*
* @param datasetIdOrName the dataset id or name
* @return the dataset for name
* @throws JsonParseException the json parse exception
* @throws JsonMappingException the json mapping exception
* @return the jsonValue
* @throws MalformedURLException
* @throws IOException Signals that an I/O exception has occurred.
*/
public CkanDataset getDatasetForName(String datasetIdOrName) throws JsonParseException, JsonMappingException, IOException {
public String getDatasetForName(String datasetIdOrName) throws MalformedURLException {
LOG.debug("Get dataset for name "+datasetIdOrName+ "called");
String json = new Item().read(datasetIdOrName);
return SimpleExtendCkanClient.getObjectMapper().readValue(json, CkanDataset.class);
return new Item().read(datasetIdOrName);
}
/**
* TODO gCAT missing
* @return
* @throws MalformedURLException
*/
public List<CkanLicense> getLicenseList() {
LOG.debug("Get linces list called");
return jackanCkanClient.getLicenseList();
/**
* Creates the dataset.
*
* @param jsonDataset the json dataset
* @param socialPost if true sends the social post
* @return the jsonValue
* @throws MalformedURLException the malformed URL exception
*/
public String createDataset(String jsonDataset, boolean socialPost) throws MalformedURLException {
LOG.debug("Create dataset called");
return new Item().create(jsonDataset,socialPost);
}
}

143
src/main/java/org/gcube/datacatalogue/ckanutillibrary/server/DataCatalogue.java
View File

@ -39,67 +39,6 @@ public interface DataCatalogue {
*/
String getUnencryptedUrlFromDatasetIdOrName(String datasetIdOrName);
/**
* Create a dataset with those information. The method allows to have multiple
* values for the same custom field key. NOTE: unfortunately java doesn't
* support overload in java interface methods (that's way I cannot use the same
* name for the method)
*
* @param title the title
* @param name (unique identifier)
* @param organizationNameOrId the organization name or id
* @param author the author
* @param authorMail the author mail
* @param maintainer the maintainer
* @param maintainerMail the maintainer mail
* @param version the version
* @param description the description
* @param licenseId the license id
* @param tags the tags
* @param customFields the custom fields
* @param resources the resources
* @param setPublic (manage visibility: Admin role is needed)
* @return the id of the dataset on success, null otherwise
* @throws Exception the exception
*/
String createCKanDatasetMultipleCustomFields(String title, String name, String organizationNameOrId, String author,
String authorMail, String maintainer, String maintainerMail, long version, String description,
String licenseId, List<String> tags, Map<String, List<String>> customFields, List<ResourceBean> resources,
boolean setPublic) throws Exception;
/**
* Patch a product with product id productId by using the couples in
* customFieldsToChange. NOTE: only the specified custom fields will be changed.
* If a custom field with a given key already exists, and removeOld is set to
* false, the new values are added at the end of the list. Otherwise they are
* lost.
*
* @param productId the product id
* @param customFieldsToChange the custom fields to change
* @param removeOld the remove old
* @return true, if successful
*/
boolean patchProductCustomFields(String productId, Map<String, List<String>> customFieldsToChange,
boolean removeOld);
/**
* Add a resource described by the bean to the dataset id into
* resource.datasetId
*
* @param resource the resource
* @return String the id of the resource on success, null otherwise
* @throws Exception the exception
*/
String addResourceToDataset(ResourceBean resource) throws Exception;
/**
* Remove the resource with id resourceId from the dataset in which it is.
*
* @param resourceId the resource id
* @return true on success, false otherwise.
*/
boolean deleteResourceFromDataset(String resourceId);
/**
* Create a dataset with those information.
*
@ -201,4 +140,86 @@ public interface DataCatalogue {
*/
Map<String, Map<CkanOrganization, RolesCkanGroupOrOrg>> getUserRoleByOrganization(String username);
//WRITE OPERATIONS
/**
* Set searchable field
* The field, if set to true, allows organization's users to view private products also in groups
* @param datasetId the id of the dataset to update
* @param searchable the value to assign to the field
* @return true if the field is set, false otherwise
*/
boolean setSearchableField(String datasetId, boolean searchable);
/**
* Create a dataset with those information. The method allows to have multiple
* values for the same custom field key. NOTE: unfortunately java doesn't
* support overload in java interface methods (that's way I cannot use the same
* name for the method)
*
* @param the username that is submitting the publishing request
* @param title the title
* @param name (unique identifier)
* @param organizationNameOrId the organization name or id
* @param author the author
* @param authorMail the author mail
* @param maintainer the maintainer
* @param maintainerMail the maintainer mail
* @param version the version
* @param description the description
* @param licenseId the license id
* @param tags the tags
* @param customFields the custom fields
* @param resources the resources
* @param setPublic (manage visibility: Admin role is needed)
* @return the id of the dataset on success, null otherwise
* @throws Exception the exception
*/
String createCKanDatasetMultipleCustomFields(String username, String title, String name, String organizationNameOrId, String author,
String authorMail, String maintainer, String maintainerMail, long version, String description,
String licenseId, List<String> tags, Map<String, List<String>> customFields, List<ResourceBean> resources,
boolean setPublic) throws Exception;
/**
* Patch a product with product id productId by using the couples in
* customFieldsToChange. NOTE: only the specified custom fields will be changed.
* If a custom field with a given key already exists, and removeOld is set to
* false, the new values are added at the end of the list. Otherwise they are
* lost.
*
* @param productId the product id
* @param customFieldsToChange the custom fields to change
* @param removeOld the remove old
* @return true, if successful
*/
boolean patchProductCustomFields(String productId, Map<String, List<String>> customFieldsToChange,
boolean removeOld);
/**
* Add a resource described by the bean to the dataset id into
* resource.datasetId
*
* @param resource the resource
* @return String the id of the resource on success, null otherwise
* @throws Exception the exception
*/
String addResourceToDataset(ResourceBean resource) throws Exception;
/**
* Remove the resource with id resourceId from the dataset in which it is.
*
* @param resourceId the resource id
* @return true on success, false otherwise.
*/
boolean deleteResourceFromDataset(String resourceId);
}

235
src/main/java/org/gcube/datacatalogue/ckanutillibrary/server/DataCatalogueImpl.java
View File

@ -6,15 +6,18 @@ import static com.google.common.base.Preconditions.checkNotNull;
import java.net.URLEncoder;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.concurrent.ConcurrentHashMap;
import org.gcube.common.authorization.library.provider.SecurityTokenProvider;
import org.gcube.common.scope.impl.ScopeBean;
import org.gcube.common.scope.impl.ScopeBean.Type;
import org.gcube.datacatalogue.ckanutillibrary.ckan.DirectCkanCaller;
import org.gcube.datacatalogue.ckanutillibrary.ckan.ExtendCkanClient;
import org.gcube.datacatalogue.ckanutillibrary.ckan.SimpleExtendCkanClient;
import org.gcube.datacatalogue.ckanutillibrary.ckan.MarshUnmarshCkanObject;
import org.gcube.datacatalogue.ckanutillibrary.db.DBCaller;
import org.gcube.datacatalogue.ckanutillibrary.gcat.GCatCaller;
import org.gcube.datacatalogue.ckanutillibrary.server.DataCatalogueRunningCluster.ACCESS_LEVEL_TO_CATALOGUE_PORTLET;
@ -38,7 +41,13 @@ import eu.trentorise.opendata.jackan.model.CkanDataset;
import eu.trentorise.opendata.jackan.model.CkanGroup;
import eu.trentorise.opendata.jackan.model.CkanLicense;
import eu.trentorise.opendata.jackan.model.CkanOrganization;
import eu.trentorise.opendata.jackan.model.CkanPair;
import eu.trentorise.opendata.jackan.model.CkanResource;
import eu.trentorise.opendata.jackan.model.CkanTag;
import eu.trentorise.opendata.jackan.model.CkanUser;
import net.htmlparser.jericho.Renderer;
import net.htmlparser.jericho.Segment;
import net.htmlparser.jericho.Source;
public class DataCatalogueImpl implements DataCatalogue {
@ -63,6 +72,7 @@ public class DataCatalogueImpl implements DataCatalogue {
public Map<ACCESS_LEVEL_TO_CATALOGUE_PORTLET, String> mapAccessURLToCatalogue;
private static final String CATALOGUE_TAB_ENDING_URL = "/catalogue";
// gCat client
@ -72,10 +82,13 @@ public class DataCatalogueImpl implements DataCatalogue {
private DBCaller dbCaller;
// ckan client
private SimpleExtendCkanClient ckanCaller;
private ExtendCkanClient ckanCaller;
// hashmap for ckan api keys
private ConcurrentHashMap<String, CKANTokenBean> apiKeysMap;
//http ckan caller
private DirectCkanCaller directCkanCaller;
// apikey bean expires after X minutes in the above map
private static final int EXPIRE_KEY_TIME = 60 * 60 * 1000;
@ -97,7 +110,7 @@ public class DataCatalogueImpl implements DataCatalogue {
CKAN_DB_USER = runningInstance.getDataBaseUser().trim();
CKAN_DB_PASSWORD = runningInstance.getDataBasePassword().trim();
//CKAN_TOKEN_SYS = runningInstance.getSysAdminToken().trim();
CKAN_TOKEN_SYS = runningInstance.getSysAdminToken().trim();
CKAN_EMAIL = runningInstance.getEmailCatalogue().trim();
CKAN_CATALOGUE_URL = runningInstance.getDataCatalogueUrl().get(0).trim();
@ -113,17 +126,13 @@ public class DataCatalogueImpl implements DataCatalogue {
// build the clients
gCatCaller = new GCatCaller(CKAN_CATALOGUE_URL);
dbCaller = new DBCaller(CKAN_DB_URL, CKAN_DB_PORT, CKAN_DB_NAME, CKAN_DB_USER, CKAN_DB_PASSWORD);
ckanCaller = new SimpleExtendCkanClient(CKAN_CATALOGUE_URL);
ckanCaller = new ExtendCkanClient(CKAN_CATALOGUE_URL);
directCkanCaller = new DirectCkanCaller(CKAN_CATALOGUE_URL);
// init map
apiKeysMap = new ConcurrentHashMap<String, CKANTokenBean>();
// save the context
CONTEXT = scope;
// extended roles
extendRoleInOrganization = runningInstance.getExtendRoleInOrganization();
}
@ -228,7 +237,8 @@ public class DataCatalogueImpl implements DataCatalogue {
String authzToken = SecurityTokenProvider.instance.get();
if(authzToken!=null && !authzToken.isEmpty()) {
LOG.info("gcube-token found. Calling the gCat client");
return gCatCaller.getDatasetForName(datasetId);
String jsonDataset = gCatCaller.getDatasetForName(datasetId);
return MarshUnmarshCkanObject.toCkanDataset(jsonDataset);
}
LOG.info("No api-key or gcube-token found. Calling Ckan Client without API-KEY");
@ -272,13 +282,7 @@ public class DataCatalogueImpl implements DataCatalogue {
return url;
}
public String createCKanDatasetMultipleCustomFields(String title, String name, String organizationNameOrId,
String author, String authorMail, String maintainer, String maintainerMail, long version,
String description, String licenseId, List<String> tags, Map<String, List<String>> customFields,
List<ResourceBean> resources, boolean setPublic) throws Exception {
return null;
}
@Override
public Map<String, Map<CkanGroup, RolesCkanGroupOrOrg>> getUserRoleByGroup(String username) {
LOG.info("Get user role by group called. The username is: "+username);
@ -328,6 +332,7 @@ public class DataCatalogueImpl implements DataCatalogue {
HashMap<CkanOrganization, RolesCkanGroupOrOrg> subMap = new HashMap<CkanOrganization, RolesCkanGroupOrOrg>();
subMap.put(org, partialResult.get(orgID));
toReturn.put(orgID, subMap);
LOG.debug("For organisation: " + org.getName() + ", the user "+username+" has role: "+subMap);
}
LOG.debug("Returning map " + toReturn);
@ -399,10 +404,8 @@ public class DataCatalogueImpl implements DataCatalogue {
// checks
checkNotNull(username);
// in order to avoid errors, the username is always converted
String ckanUsername = CatalogueUtilMethods.fromUsernameToCKanUsername(username);
// list to return
List<CkanOrganization> toReturn = new ArrayList<CkanOrganization>();
@ -413,12 +416,11 @@ public class DataCatalogueImpl implements DataCatalogue {
// iterate over them
for (CkanOrganization ckanOrganization : organizations) {
// get the list of users in it (if you try ckanOrganization.getUsers() it returns null.. maybe a bug TODO)
List<CkanUser> users = ckanCaller.getOrganization(ckanOrganization.getName()).getUsers();
// check if the current user is among them
for (CkanUser ckanUser : users) {
if(ckanUser.getName().equals(ckanUsername)){
LOG.debug("User " + ckanUsername + " is into " + ckanOrganization.getName());
@ -440,10 +442,8 @@ public class DataCatalogueImpl implements DataCatalogue {
// checks
checkNotNull(username);
// in order to avoid errors, the username is always converted
String ckanUsername = CatalogueUtilMethods.fromUsernameToCKanUsername(username);
// list to return
List<CkanGroup> toReturn = new ArrayList<CkanGroup>();
@ -459,6 +459,7 @@ public class DataCatalogueImpl implements DataCatalogue {
// check if the current user is among them
for (CkanUser ckanUser : users) {
if(ckanUser.getName().equals(ckanUsername)){
LOG.debug("User " + ckanUsername + " is into " + ckanGroup.getName());
@ -513,15 +514,22 @@ public class DataCatalogueImpl implements DataCatalogue {
return null;
}
@Override
public boolean existProductWithNameOrId(String nameOrId) {
checkNotNull(nameOrId);
checkArgument(!nameOrId.isEmpty());
try{
ExtendCkanClient client = new ExtendCkanClient(CKAN_CATALOGUE_URL, CKAN_TOKEN_SYS);
CkanDataset product = client.getDataset(nameOrId);
return product != null;
}catch(Exception e){
LOG.debug("A dataset with name " + nameOrId + " doesn't exist");
return false;
}
}
/*
*
@ -546,6 +554,163 @@ public class DataCatalogueImpl implements DataCatalogue {
*
*/
@Override
public String createCKanDatasetMultipleCustomFields(String username, String title, String name, String organizationNameOrId,
String author, String authorMail, String maintainer, String maintainerMail, long version,
String description, String licenseId, List<String> tags, Map<String, List<String>> customFieldsMultiple,
List<ResourceBean> resources, boolean setPublic) throws Exception {
// delegate the private method
return createCkanDatasetBody(username, title, name, organizationNameOrId, author, authorMail, maintainer, maintainerMail,
version, description, licenseId, tags, null, customFieldsMultiple, resources, setPublic);
}
// the body of the actual dataset creation methods
private String createCkanDatasetBody(String username, String title, String name, String organizationNameOrId, String author,
String authorMail, String maintainer, String maintainerMail, long version, String description,
String licenseId, List<String> tags, Map<String, String> customFields,
Map<String, List<String>> customFieldsMultipleValues, List<ResourceBean> resources, boolean setPublic)
throws Exception {
// checks (minimum)
checkNotNull(username);
checkNotNull(organizationNameOrId);
checkArgument(!organizationNameOrId.isEmpty());
checkArgument(!(title == null && name == null || title.isEmpty() && name.isEmpty()), "Name and Title cannot be empty/null at the same time!");
LOG.debug("Request for dataset creation");
//ExtendCkanClient client = new ExtendCkanClient(CKAN_CATALOGUE_URL, apiKey);
//String ckanUsername = getUserFromApiKey(apiKey).getName();
CkanDataset dataset = new CkanDataset();
// in order to avoid errors, the username is always converted
String ckanUsername = CatalogueUtilMethods.fromUsernameToCKanUsername(username);
String nameToUse = name;
if(nameToUse == null)
nameToUse = CatalogueUtilMethods.fromProductTitleToName(title);
LOG.debug("Name of the dataset is going to be " + nameToUse + ". Title is going to be " + title);
dataset.setName(nameToUse);
dataset.setTitle(title);
// CkanOrganization orgOwner = client.getOrganization(organizationNameOrId);
// dataset.setOwnerOrg(orgOwner.getId());
dataset.setAuthor(author);
dataset.setAuthorEmail(authorMail);
dataset.setMaintainer(maintainer);
dataset.setMaintainerEmail(maintainerMail);
dataset.setVersion(String.valueOf(version));
// description must be escaped
if(description != null && !description.isEmpty()){
Source descriptionEscaped = new Source(description);
Segment htmlSeg = new Segment(descriptionEscaped, 0, descriptionEscaped.length());
Renderer htmlRend = new Renderer(htmlSeg);
dataset.setNotes(htmlRend.toString());
LOG.debug("Description escaped " + htmlRend.toString());
}
dataset.setLicenseId(licenseId);
// set the tags, if any
if(tags != null && !tags.isEmpty()){
List<CkanTag> ckanTags = new ArrayList<CkanTag>(tags.size());
for (String stringTag : tags) {
ckanTags.add(new CkanTag(stringTag));
}
dataset.setTags(ckanTags);
}
// set the custom fields, if any
List<CkanPair> extras = new ArrayList<CkanPair>();
if(customFields != null && !customFields.isEmpty()){
Iterator<Entry<String, String>> iterator = customFields.entrySet().iterator();
while (iterator.hasNext()) {
Map.Entry<String, String> entry = iterator.next();
extras.add(new CkanPair(entry.getKey(), entry.getValue()));
}
}else if(customFieldsMultipleValues != null && !customFieldsMultipleValues.isEmpty()){
Iterator<Entry<String, List<String>>> iterator = customFieldsMultipleValues.entrySet().iterator();
while (iterator.hasNext()) {
Map.Entry<String, List<String>> entry = iterator.next();
List<String> valuesForEntry = entry.getValue();
for (String value : valuesForEntry) {
extras.add(new CkanPair(entry.getKey(), value));
}
}
}
dataset.setExtras(extras);
// check if we need to add the resources
if(resources != null && !resources.isEmpty()){
LOG.debug("We need to add resources to the dataset");
try{
List<CkanResource> resourcesCkan = new ArrayList<CkanResource>();
for(ResourceBean resource: resources){
LOG.debug("Going to add resource described by " + resource);
CkanResource newResource = new CkanResource();
newResource.setDescription(resource.getDescription());
newResource.setId(resource.getId());
newResource.setUrl(resource.getUrl());
newResource.setName(resource.getName());
newResource.setMimetype(resource.getMimeType());
newResource.setFormat(resource.getMimeType());
newResource.setOwner(ckanUsername);
resourcesCkan.add(newResource);
}
// add to the dataset
dataset.setResources(resourcesCkan);
}catch(Exception e){
LOG.error("Unable to add those resources to the dataset", e);
}
}
// try to create
String jsonValueDataset = MarshUnmarshCkanObject.toJsonValueDataset(dataset);
LOG.debug("Marshalling dataset is: " + jsonValueDataset);
jsonValueDataset = gCatCaller.createDataset(jsonValueDataset,true);
LOG.debug("Created dataset is: " + jsonValueDataset);
if(jsonValueDataset != null){
CkanDataset toCkanDataset = MarshUnmarshCkanObject.toCkanDataset(jsonValueDataset);
LOG.debug("Dataset with name " + toCkanDataset.getName() + " has been created. Setting visibility");
// set visibility
boolean visibilitySet = directCkanCaller.setDatasetPrivate(
!setPublic, // swap to private
toCkanDataset.getOrganization().getId(),
toCkanDataset.getId(),
CKAN_TOKEN_SYS); // use sysadmin api key to be sure it will be set
LOG.info("Was visibility set to " + (setPublic ? "public" : "private") + "? " + visibilitySet);
// set searchable to true if dataset visibility is private
if(!setPublic){ // (swap to private)
boolean searchableSet = setSearchableField(toCkanDataset.getId(), true);
LOG.info("Was searchable set to True? " + searchableSet);
}
return toCkanDataset.getId();
}
return null;
}
public boolean patchProductCustomFields(String productId, Map<String, List<String>> customFieldsToChange,
boolean removeOld) {
@ -560,9 +725,11 @@ public class DataCatalogueImpl implements DataCatalogue {
return false;
}
public boolean existProductWithNameOrId(String nameOrId) {
return false;
@Override
public boolean setSearchableField(String datasetId, boolean searchable) {
return directCkanCaller.setSearchableField(datasetId, searchable, CKAN_TOKEN_SYS);
}
}

15
src/test/java/TestDataCatalogueLib.java
View File

@ -5,7 +5,6 @@ import org.gcube.datacatalogue.ckanutillibrary.server.ApplicationProfileScopePer
import org.gcube.datacatalogue.ckanutillibrary.server.DataCatalogueFactory;
import org.gcube.datacatalogue.ckanutillibrary.server.DataCatalogueImpl;
import org.junit.Before;
import org.junit.Test;
import org.slf4j.LoggerFactory;
/**
@ -19,7 +18,9 @@ public class TestDataCatalogueLib {
private static final org.slf4j.Logger logger = LoggerFactory.getLogger(TestDataCatalogueLib.class);
private DataCatalogueFactory factory;
private String scope = "/gcube/devsec/devVRE";
//private String scope = "/gcube/devsec/devVRE";
private String scope = "/d4science.research-infrastructures.eu/SoBigData/Catalogue-TerritoriAperti";
//private String scope = "/d4science.research-infrastructures.eu/gCubeApps/BiodiversityLab";
//private String testUser = "costantino_perciante";
private String testUser = "francesco_mangiacrapa";
String subjectId = "aa_father4";
@ -46,15 +47,15 @@ public class TestDataCatalogueLib {
DataCatalogueFactory factory = DataCatalogueFactory.getFactory();
while(true){
factory.getUtilsPerScope("/gcube");
factory.getUtilsPerScope(scope);
Thread.sleep(60* 1000 * 3);
factory.getUtilsPerScope("/gcube");
factory.getUtilsPerScope(scope);
break;
}
for (int i = 0; i < 5; i++) {
Thread.sleep(1000);
factory.getUtilsPerScope("/gcube");
factory.getUtilsPerScope(scope);
}
}
@ -68,7 +69,7 @@ public class TestDataCatalogueLib {
//@Test
public void getScopePerUrl(){
ScopeProvider.instance.set("/gcube");
ScopeProvider.instance.set(scope);
String url = "https://dev4.d4science.org/group/devvre/ckan";
String scopeToUse = ApplicationProfileScopePerUrlReader.getScopePerUrl(url);
logger.debug("Retrieved scope is " + scopeToUse);
@ -86,7 +87,7 @@ public class TestDataCatalogueLib {
* @return the user role by group
* @throws Exception the exception
*/
@Test
//@Test
public void getUserRoleByGroup() throws Exception{
DataCatalogueImpl instance = factory.getUtilsPerScope(scope);
String username = testUser;

9
src/test/resources/.gitignore vendored
View File

@ -5,3 +5,12 @@
/log4j.properties
/pred4s.gcubekey
/preprod.gcubekey
/D4OS.gcubekey
/D4Research.gcubekey
/FARM.gcubekey
/OpenAIRE.gcubekey
/ParthenosVO.gcubekey
/SmartArea.gcubekey
/SoBigData.gcubekey
/d4science.research-infrastructures.eu.gcubekey
/gCubeApps.gcubekey

2
target/classes/META-INF/MANIFEST.MF
View File

@ -6,7 +6,7 @@ Specification-Version: 0.1.0-SNAPSHOT
Implementation-Title: CKan utility library
Implementation-Version: 0.1.0-SNAPSHOT
Implementation-Vendor-Id: org.gcube.datacatalogue
Build-Time: 20200601-145428
Build-Time: 20200603-095004
Created-By: Maven Integration for Eclipse
SCM-Branch:
SCM-Revision:

2
target/classes/META-INF/maven/org.gcube.datacatalogue/catalogue-util-library/pom.properties
View File

@ -1,5 +1,5 @@
#Generated by Maven Integration for Eclipse
#Mon Jun 01 16:54:29 CEST 2020
#Wed Jun 03 11:50:06 CEST 2020
version=0.1.0-SNAPSHOT
groupId=org.gcube.datacatalogue
m2e.projectName=catalogue-util-library

BIN
target/classes/org/gcube/datacatalogue/ckanutillibrary/ckan/ExtendCkanClient.class
View File

Binary file not shown.

115
target/classes/org/gcube/datacatalogue/ckanutillibrary/ckan/ExtendCkanClient.java
View File

@ -1,6 +1,3 @@
/**
*
*/
package org.gcube.datacatalogue.ckanutillibrary.ckan;
import static com.google.common.base.Preconditions.checkNotNull;
@ -19,7 +16,7 @@ import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.base.Charsets;
import com.google.common.io.CharStreams;
import eu.trentorise.opendata.jackan.CheckedCkanClient;
import eu.trentorise.opendata.jackan.CkanClient;
import eu.trentorise.opendata.jackan.exceptions.CkanException;
import eu.trentorise.opendata.jackan.exceptions.JackanException;
import eu.trentorise.opendata.jackan.internal.org.apache.http.client.fluent.Request;
@ -29,36 +26,34 @@ import eu.trentorise.opendata.jackan.model.CkanOrganization;
import eu.trentorise.opendata.jackan.model.CkanResponse;
// TODO: Auto-generated Javadoc
/**
* The Class ExtendCkanClient.
* The Class SimpleExtendCkanClient.
*
* @author Francesco Mangiacrapa at ISTI-CNR (francesco.mangiacrapa@isti.cnr.it)
*
* Jun 14, 2019
* Jun 7, 2019
*
*/
public class ExtendCkanClient extends CheckedCkanClient implements PatchedCkan{
private static final Logger logger = LoggerFactory.getLogger(ExtendCkanClient.class);
public class ExtendCkanClient extends CkanClient implements PatchedCkan{
@Nullable
private static ObjectMapper objectMapper;
private String catalogueURL;
private String ckanToken;
private int extendTimeout = 120000; //in milliseconds
@Nullable
private static ObjectMapper objectMapper;
private static final Logger logger = LoggerFactory.getLogger(ExtendCkanClient.class);
/**
* Instantiates a new extend ckan client.
* Instantiates a new simple extend ckan client.
*
* @param catalogueURL the catalogue url
* @param catalogueURL the catalogue URL
*/
public ExtendCkanClient(String catalogueURL) {
super(catalogueURL);
this.catalogueURL = catalogueURL;
}
/**
* Instantiates a new extend ckan client.
*
@ -84,7 +79,7 @@ public class ExtendCkanClient extends CheckedCkanClient implements PatchedCkan{
this.ckanToken = ckanToken;
this.extendTimeout = timeout;
}
/**
* Configures the request. Should work both for GETs and POSTs.
@ -104,38 +99,13 @@ public class ExtendCkanClient extends CheckedCkanClient implements PatchedCkan{
return request;
}
/**
* Gets the catalogue url.
*
* @return the catalogueURL
*/
public String getCatalogueURL() {
return catalogueURL;
}
/**
* Gets the ckan token.
*
* @return the ckanToken
*/
@Override
public String getCkanToken() {
return ckanToken;
}
/**
* Gets the timeout.
*
* @return the timeout
*/
public int getTimeout() {
return extendTimeout;
}
/* (non-Javadoc)
/**
* Gets the organization.
*
* @param idOrName the id or name
* @return the organization
*/
/* (non-Javadoc)
* @see eu.trentorise.opendata.jackan.CkanClient#getOrganization(java.lang.String)
*/
public synchronized CkanOrganization getOrganization(String idOrName) {
@ -147,6 +117,12 @@ public class ExtendCkanClient extends CheckedCkanClient implements PatchedCkan{
}
/**
* Gets the group.
*
* @param idOrName the id or name
* @return the group
*/
/* (non-Javadoc)
* @see eu.trentorise.opendata.jackan.CkanClient#getGroup(java.lang.String)
*/
@ -164,7 +140,7 @@ public class ExtendCkanClient extends CheckedCkanClient implements PatchedCkan{
*
* @return the object mapper
*/
static ObjectMapper getObjectMapper() {
public static ObjectMapper getObjectMapper() {
if (objectMapper == null) {
objectMapper = new ObjectMapper();
configureObjectMapper(objectMapper);
@ -172,6 +148,15 @@ public class ExtendCkanClient extends CheckedCkanClient implements PatchedCkan{
return objectMapper;
}
/**
* Gets the http.
*
* @param <T> the generic type
* @param responseType the response type
* @param path the path
* @param params the params
* @return the http
*/
/* (non-Javadoc)
* @see org.gcube.datacatalogue.ckanutillibrary.server.patch.PatchedCkan#getHttp(java.lang.Class, java.lang.String, java.lang.Object[])
*/
@ -215,6 +200,13 @@ public class ExtendCkanClient extends CheckedCkanClient implements PatchedCkan{
}
/**
* Calc full url.
*
* @param path the path
* @param params the params
* @return the string
*/
/* (non-Javadoc)
* @see org.gcube.datacatalogue.ckanutillibrary.server.patch.PatchedCkan#calcFullUrl(java.lang.String, java.lang.Object[])
*/
@ -236,5 +228,26 @@ public class ExtendCkanClient extends CheckedCkanClient implements PatchedCkan{
+ Arrays.toString(params), ex);
}
}
/**
* Gets the catalogue URL.
*
* @return the catalogue URL
*/
public String getCatalogueURL() {
return catalogueURL;
}
/**
* Gets the ckan token.
*
* @return the ckan token
*/
@Override
public String getCkanToken() {
return ckanToken;
}
}

BIN
target/classes/org/gcube/datacatalogue/ckanutillibrary/ckan/SimpleExtendCkanClient.class
View File

Binary file not shown.

168
target/classes/org/gcube/datacatalogue/ckanutillibrary/ckan/SimpleExtendCkanClient.java
View File

@ -1,168 +0,0 @@
package org.gcube.datacatalogue.ckanutillibrary.ckan;
import static com.google.common.base.Preconditions.checkNotNull;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.net.URLEncoder;
import java.util.Arrays;
import javax.annotation.Nullable;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.base.Charsets;
import com.google.common.io.CharStreams;
import eu.trentorise.opendata.jackan.CkanClient;
import eu.trentorise.opendata.jackan.exceptions.CkanException;
import eu.trentorise.opendata.jackan.exceptions.JackanException;
import eu.trentorise.opendata.jackan.internal.org.apache.http.client.fluent.Request;
import eu.trentorise.opendata.jackan.internal.org.apache.http.client.fluent.Response;
import eu.trentorise.opendata.jackan.model.CkanGroup;
import eu.trentorise.opendata.jackan.model.CkanOrganization;
import eu.trentorise.opendata.jackan.model.CkanResponse;
/**
* The Class SimpleExtendCkanClient.
*
* @author Francesco Mangiacrapa at ISTI-CNR (francesco.mangiacrapa@isti.cnr.it)
*
* Jun 7, 2019
*/
public class SimpleExtendCkanClient extends CkanClient implements PatchedCkan{
private String catalogueURL;
@Nullable
private static ObjectMapper objectMapper;
private static final Logger logger = LoggerFactory.getLogger(SimpleExtendCkanClient.class);
/**
* Instantiates a new simple extend ckan client.
*
* @param catalogueURL the catalogue URL
*/
public SimpleExtendCkanClient(String catalogueURL) {
super(catalogueURL);
this.catalogueURL = catalogueURL;
}
/* (non-Javadoc)
* @see eu.trentorise.opendata.jackan.CkanClient#getOrganization(java.lang.String)
*/
public synchronized CkanOrganization getOrganization(String idOrName) {
checkNotNull(idOrName, "Need a valid id or name!");
logger.info("Patched read organization for id/name: {}", idOrName);
return getHttp(OrganizationResponse.class, "/api/3/action/organization_show", "id", idOrName,
"include_datasets", "false", "include_users", "true").result;
}
/* (non-Javadoc)
* @see eu.trentorise.opendata.jackan.CkanClient#getGroup(java.lang.String)
*/
public synchronized CkanGroup getGroup(String idOrName) {
checkNotNull(idOrName, "Need a valid id or name!");
logger.info("Patched read group for id/name: {}", idOrName);
return getHttp(GroupResponse.class, "/api/3/action/group_show", "id", idOrName, "include_datasets",
"false", "include_users", "true").result;
}
/**
* Retrieves the Jackson object mapper for reading operations. Internally,
* Object mapper is initialized at first call.
*
* @return the object mapper
*/
public static ObjectMapper getObjectMapper() {
if (objectMapper == null) {
objectMapper = new ObjectMapper();
configureObjectMapper(objectMapper);
}
return objectMapper;
}
/* (non-Javadoc)
* @see org.gcube.datacatalogue.ckanutillibrary.server.patch.PatchedCkan#getHttp(java.lang.Class, java.lang.String, java.lang.Object[])
*/
public <T extends CkanResponse> T getHttp(Class<T> responseType, String path, Object... params) {
checkNotNull(responseType);
checkNotNull(path);
String fullUrl = calcFullUrl(path, params);
T ckanResponse;
String returnedText;
try {
logger.debug("getting {}", fullUrl);
Request request = Request.Get(fullUrl);
configureRequest(request);
Response response = request.execute();
InputStream stream = response.returnResponse()
.getEntity()
.getContent();
try (InputStreamReader reader = new InputStreamReader(stream, Charsets.UTF_8)) {
returnedText = CharStreams.toString(reader);
}
logger.trace("returnedText {}", returnedText);
} catch (Exception ex) {
throw new CkanException("Error while performing GET. Request url was: " + fullUrl, this, ex);
}
try {
ckanResponse = getObjectMapper().readValue(returnedText, responseType);
} catch (Exception ex) {
throw new CkanException(
"Couldn't interpret json returned by the server! Returned text was: " + returnedText, this, ex);
}
if (!ckanResponse.isSuccess()) {
throwCkanException("Error while performing GET. Request url was: " + fullUrl, ckanResponse);
}
return ckanResponse;
}
/* (non-Javadoc)
* @see org.gcube.datacatalogue.ckanutillibrary.server.patch.PatchedCkan#calcFullUrl(java.lang.String, java.lang.Object[])
*/
public String calcFullUrl(String path, Object[] params) {
checkNotNull(path);
try {
StringBuilder sb = new StringBuilder().append(catalogueURL)
.append(path);
for (int i = 0; i < params.length; i += 2) {
sb.append(i == 0 ? "?" : "&")
.append(URLEncoder.encode(params[i].toString(), "UTF-8"))
.append("=")
.append(URLEncoder.encode(params[i + 1].toString(), "UTF-8"));
}
return sb.toString();
} catch (Exception ex) {
throw new JackanException("Error while building url to perform GET! \n path: " + path + " \n params: "
+ Arrays.toString(params), ex);
}
}
/**
* Gets the catalogue url.
*
* @return the catalogueURL
*/
public String getCatalogueURL() {
return catalogueURL;
}
}

BIN
target/classes/org/gcube/datacatalogue/ckanutillibrary/db/DBCaller.class
View File

Binary file not shown.

10
target/classes/org/gcube/datacatalogue/ckanutillibrary/db/DBCaller.java
View File

@ -95,7 +95,7 @@ public class DBCaller {
*/
public Map<String, RolesCkanGroupOrOrg> getGroupsByUserFromDB(String username){
checkNotNull(username);
LOG.debug("Get groups by user called");
LOG.debug("Get groups by user called for username: "+username);
//couples (groups id, capacity) of the user in the group
@ -129,6 +129,8 @@ public class DBCaller {
}finally{
closeConnection(connection);
}
LOG.debug("returning map: "+toReturn);
return toReturn;
}
@ -140,7 +142,7 @@ public class DBCaller {
* @return the user id for username
*/
private String getUserIdForUsername(String username) {
LOG.debug("Get user id for username called");
LOG.debug("Get user id for username called for username: "+username);
Connection connection = null;
String userId = null;
@ -150,7 +152,7 @@ public class DBCaller {
String selQuery = "SELECT \"id\" FROM \"public\".\"user\" WHERE \"name\"=?;";
LOG.info("Performing query: "+selQuery);
LOG.debug("Performing query: "+selQuery);
PreparedStatement preparedStatement = connection.prepareStatement(selQuery);
preparedStatement.setString(1, username);
@ -213,6 +215,8 @@ public class DBCaller {
}finally{
closeConnection(connection);
}
LOG.debug("returning map: "+toReturn);
return toReturn;
}

BIN
target/classes/org/gcube/datacatalogue/ckanutillibrary/gcat/GCatCaller.class
View File

Binary file not shown.

45
target/classes/org/gcube/datacatalogue/ckanutillibrary/gcat/GCatCaller.java
View File

@ -2,19 +2,11 @@ package org.gcube.datacatalogue.ckanutillibrary.gcat;
import java.io.IOException;
import java.net.MalformedURLException;
import java.util.List;
import org.gcube.datacatalogue.ckanutillibrary.ckan.SimpleExtendCkanClient;
import org.gcube.gcat.client.Item;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.fasterxml.jackson.core.JsonParseException;
import com.fasterxml.jackson.databind.JsonMappingException;
import eu.trentorise.opendata.jackan.model.CkanDataset;
import eu.trentorise.opendata.jackan.model.CkanLicense;
// TODO: Auto-generated Javadoc
/**
* The Class GCatCaller.
@ -24,18 +16,17 @@ import eu.trentorise.opendata.jackan.model.CkanLicense;
*/
public class GCatCaller {
private SimpleExtendCkanClient jackanCkanClient;
private String catalogueURL;
private static final Logger LOG = LoggerFactory.getLogger(GCatCaller.class);
/**
* Instantiates a new g cat caller.
*
* @param catalogueURL the catalogue URL
*/
public GCatCaller(String catalogueURL) {
this.catalogueURL = catalogueURL;
this.jackanCkanClient = new SimpleExtendCkanClient(catalogueURL);
}
@ -43,27 +34,27 @@ public class GCatCaller {
* Gets the dataset for name.
*
* @param datasetIdOrName the dataset id or name
* @return the dataset for name
* @throws JsonParseException the json parse exception
* @throws JsonMappingException the json mapping exception
* @return the jsonValue
* @throws MalformedURLException
* @throws IOException Signals that an I/O exception has occurred.
*/
public CkanDataset getDatasetForName(String datasetIdOrName) throws JsonParseException, JsonMappingException, IOException {
public String getDatasetForName(String datasetIdOrName) throws MalformedURLException {
LOG.debug("Get dataset for name "+datasetIdOrName+ "called");
String json = new Item().read(datasetIdOrName);
return SimpleExtendCkanClient.getObjectMapper().readValue(json, CkanDataset.class);
return new Item().read(datasetIdOrName);
}
/**
* TODO gCAT missing
* @return
* @throws MalformedURLException
*/
public List<CkanLicense> getLicenseList() {
LOG.debug("Get linces list called");
return jackanCkanClient.getLicenseList();
/**
* Creates the dataset.
*
* @param jsonDataset the json dataset
* @param socialPost if true sends the social post
* @return the jsonValue
* @throws MalformedURLException the malformed URL exception
*/
public String createDataset(String jsonDataset, boolean socialPost) throws MalformedURLException {
LOG.debug("Create dataset called");
return new Item().create(jsonDataset,socialPost);
}
}

BIN
target/classes/org/gcube/datacatalogue/ckanutillibrary/server/DataCatalogue.class
View File

Binary file not shown.

143
target/classes/org/gcube/datacatalogue/ckanutillibrary/server/DataCatalogue.java
View File

@ -39,67 +39,6 @@ public interface DataCatalogue {
*/
String getUnencryptedUrlFromDatasetIdOrName(String datasetIdOrName);
/**
* Create a dataset with those information. The method allows to have multiple
* values for the same custom field key. NOTE: unfortunately java doesn't
* support overload in java interface methods (that's way I cannot use the same
* name for the method)
*
* @param title the title
* @param name (unique identifier)
* @param organizationNameOrId the organization name or id
* @param author the author
* @param authorMail the author mail
* @param maintainer the maintainer
* @param maintainerMail the maintainer mail
* @param version the version
* @param description the description
* @param licenseId the license id
* @param tags the tags
* @param customFields the custom fields
* @param resources the resources
* @param setPublic (manage visibility: Admin role is needed)
* @return the id of the dataset on success, null otherwise
* @throws Exception the exception
*/
String createCKanDatasetMultipleCustomFields(String title, String name, String organizationNameOrId, String author,
String authorMail, String maintainer, String maintainerMail, long version, String description,
String licenseId, List<String> tags, Map<String, List<String>> customFields, List<ResourceBean> resources,
boolean setPublic) throws Exception;
/**
* Patch a product with product id productId by using the couples in
* customFieldsToChange. NOTE: only the specified custom fields will be changed.
* If a custom field with a given key already exists, and removeOld is set to
* false, the new values are added at the end of the list. Otherwise they are
* lost.
*
* @param productId the product id
* @param customFieldsToChange the custom fields to change
* @param removeOld the remove old
* @return true, if successful
*/
boolean patchProductCustomFields(String productId, Map<String, List<String>> customFieldsToChange,
boolean removeOld);
/**
* Add a resource described by the bean to the dataset id into
* resource.datasetId
*
* @param resource the resource
* @return String the id of the resource on success, null otherwise
* @throws Exception the exception
*/
String addResourceToDataset(ResourceBean resource) throws Exception;
/**
* Remove the resource with id resourceId from the dataset in which it is.
*
* @param resourceId the resource id
* @return true on success, false otherwise.
*/
boolean deleteResourceFromDataset(String resourceId);
/**
* Create a dataset with those information.
*
@ -201,4 +140,86 @@ public interface DataCatalogue {
*/
Map<String, Map<CkanOrganization, RolesCkanGroupOrOrg>> getUserRoleByOrganization(String username);
//WRITE OPERATIONS
/**
* Set searchable field
* The field, if set to true, allows organization's users to view private products also in groups
* @param datasetId the id of the dataset to update
* @param searchable the value to assign to the field
* @return true if the field is set, false otherwise
*/
boolean setSearchableField(String datasetId, boolean searchable);
/**
* Create a dataset with those information. The method allows to have multiple
* values for the same custom field key. NOTE: unfortunately java doesn't
* support overload in java interface methods (that's way I cannot use the same
* name for the method)
*
* @param the username that is submitting the publishing request
* @param title the title
* @param name (unique identifier)
* @param organizationNameOrId the organization name or id
* @param author the author
* @param authorMail the author mail
* @param maintainer the maintainer
* @param maintainerMail the maintainer mail
* @param version the version
* @param description the description
* @param licenseId the license id
* @param tags the tags
* @param customFields the custom fields
* @param resources the resources
* @param setPublic (manage visibility: Admin role is needed)
* @return the id of the dataset on success, null otherwise
* @throws Exception the exception
*/
String createCKanDatasetMultipleCustomFields(String username, String title, String name, String organizationNameOrId, String author,
String authorMail, String maintainer, String maintainerMail, long version, String description,
String licenseId, List<String> tags, Map<String, List<String>> customFields, List<ResourceBean> resources,
boolean setPublic) throws Exception;
/**
* Patch a product with product id productId by using the couples in
* customFieldsToChange. NOTE: only the specified custom fields will be changed.
* If a custom field with a given key already exists, and removeOld is set to
* false, the new values are added at the end of the list. Otherwise they are
* lost.
*
* @param productId the product id
* @param customFieldsToChange the custom fields to change
* @param removeOld the remove old
* @return true, if successful
*/
boolean patchProductCustomFields(String productId, Map<String, List<String>> customFieldsToChange,
boolean removeOld);
/**
* Add a resource described by the bean to the dataset id into
* resource.datasetId
*
* @param resource the resource
* @return String the id of the resource on success, null otherwise
* @throws Exception the exception
*/
String addResourceToDataset(ResourceBean resource) throws Exception;
/**
* Remove the resource with id resourceId from the dataset in which it is.
*
* @param resourceId the resource id
* @return true on success, false otherwise.
*/
boolean deleteResourceFromDataset(String resourceId);
}

BIN
target/classes/org/gcube/datacatalogue/ckanutillibrary/server/DataCatalogueImpl.class
View File

Binary file not shown.

235
target/classes/org/gcube/datacatalogue/ckanutillibrary/server/DataCatalogueImpl.java
View File

@ -6,15 +6,18 @@ import static com.google.common.base.Preconditions.checkNotNull;
import java.net.URLEncoder;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.concurrent.ConcurrentHashMap;
import org.gcube.common.authorization.library.provider.SecurityTokenProvider;
import org.gcube.common.scope.impl.ScopeBean;
import org.gcube.common.scope.impl.ScopeBean.Type;
import org.gcube.datacatalogue.ckanutillibrary.ckan.DirectCkanCaller;
import org.gcube.datacatalogue.ckanutillibrary.ckan.ExtendCkanClient;
import org.gcube.datacatalogue.ckanutillibrary.ckan.SimpleExtendCkanClient;
import org.gcube.datacatalogue.ckanutillibrary.ckan.MarshUnmarshCkanObject;
import org.gcube.datacatalogue.ckanutillibrary.db.DBCaller;
import org.gcube.datacatalogue.ckanutillibrary.gcat.GCatCaller;
import org.gcube.datacatalogue.ckanutillibrary.server.DataCatalogueRunningCluster.ACCESS_LEVEL_TO_CATALOGUE_PORTLET;
@ -38,7 +41,13 @@ import eu.trentorise.opendata.jackan.model.CkanDataset;
import eu.trentorise.opendata.jackan.model.CkanGroup;
import eu.trentorise.opendata.jackan.model.CkanLicense;
import eu.trentorise.opendata.jackan.model.CkanOrganization;
import eu.trentorise.opendata.jackan.model.CkanPair;
import eu.trentorise.opendata.jackan.model.CkanResource;
import eu.trentorise.opendata.jackan.model.CkanTag;
import eu.trentorise.opendata.jackan.model.CkanUser;
import net.htmlparser.jericho.Renderer;
import net.htmlparser.jericho.Segment;
import net.htmlparser.jericho.Source;
public class DataCatalogueImpl implements DataCatalogue {
@ -63,6 +72,7 @@ public class DataCatalogueImpl implements DataCatalogue {
public Map<ACCESS_LEVEL_TO_CATALOGUE_PORTLET, String> mapAccessURLToCatalogue;
private static final String CATALOGUE_TAB_ENDING_URL = "/catalogue";
// gCat client
@ -72,10 +82,13 @@ public class DataCatalogueImpl implements DataCatalogue {
private DBCaller dbCaller;
// ckan client
private SimpleExtendCkanClient ckanCaller;
private ExtendCkanClient ckanCaller;
// hashmap for ckan api keys
private ConcurrentHashMap<String, CKANTokenBean> apiKeysMap;
//http ckan caller
private DirectCkanCaller directCkanCaller;
// apikey bean expires after X minutes in the above map
private static final int EXPIRE_KEY_TIME = 60 * 60 * 1000;
@ -97,7 +110,7 @@ public class DataCatalogueImpl implements DataCatalogue {
CKAN_DB_USER = runningInstance.getDataBaseUser().trim();
CKAN_DB_PASSWORD = runningInstance.getDataBasePassword().trim();
//CKAN_TOKEN_SYS = runningInstance.getSysAdminToken().trim();
CKAN_TOKEN_SYS = runningInstance.getSysAdminToken().trim();
CKAN_EMAIL = runningInstance.getEmailCatalogue().trim();
CKAN_CATALOGUE_URL = runningInstance.getDataCatalogueUrl().get(0).trim();
@ -113,17 +126,13 @@ public class DataCatalogueImpl implements DataCatalogue {
// build the clients
gCatCaller = new GCatCaller(CKAN_CATALOGUE_URL);
dbCaller = new DBCaller(CKAN_DB_URL, CKAN_DB_PORT, CKAN_DB_NAME, CKAN_DB_USER, CKAN_DB_PASSWORD);
ckanCaller = new SimpleExtendCkanClient(CKAN_CATALOGUE_URL);
ckanCaller = new ExtendCkanClient(CKAN_CATALOGUE_URL);
directCkanCaller = new DirectCkanCaller(CKAN_CATALOGUE_URL);
// init map
apiKeysMap = new ConcurrentHashMap<String, CKANTokenBean>();
// save the context
CONTEXT = scope;
// extended roles
extendRoleInOrganization = runningInstance.getExtendRoleInOrganization();
}
@ -228,7 +237,8 @@ public class DataCatalogueImpl implements DataCatalogue {
String authzToken = SecurityTokenProvider.instance.get();
if(authzToken!=null && !authzToken.isEmpty()) {
LOG.info("gcube-token found. Calling the gCat client");
return gCatCaller.getDatasetForName(datasetId);
String jsonDataset = gCatCaller.getDatasetForName(datasetId);
return MarshUnmarshCkanObject.toCkanDataset(jsonDataset);
}
LOG.info("No api-key or gcube-token found. Calling Ckan Client without API-KEY");
@ -272,13 +282,7 @@ public class DataCatalogueImpl implements DataCatalogue {
return url;
}
public String createCKanDatasetMultipleCustomFields(String title, String name, String organizationNameOrId,
String author, String authorMail, String maintainer, String maintainerMail, long version,
String description, String licenseId, List<String> tags, Map<String, List<String>> customFields,
List<ResourceBean> resources, boolean setPublic) throws Exception {
return null;
}
@Override
public Map<String, Map<CkanGroup, RolesCkanGroupOrOrg>> getUserRoleByGroup(String username) {
LOG.info("Get user role by group called. The username is: "+username);
@ -328,6 +332,7 @@ public class DataCatalogueImpl implements DataCatalogue {
HashMap<CkanOrganization, RolesCkanGroupOrOrg> subMap = new HashMap<CkanOrganization, RolesCkanGroupOrOrg>();
subMap.put(org, partialResult.get(orgID));
toReturn.put(orgID, subMap);
LOG.debug("For organisation: " + org.getName() + ", the user "+username+" has role: "+subMap);
}
LOG.debug("Returning map " + toReturn);
@ -399,10 +404,8 @@ public class DataCatalogueImpl implements DataCatalogue {
// checks
checkNotNull(username);
// in order to avoid errors, the username is always converted
String ckanUsername = CatalogueUtilMethods.fromUsernameToCKanUsername(username);
// list to return
List<CkanOrganization> toReturn = new ArrayList<CkanOrganization>();
@ -413,12 +416,11 @@ public class DataCatalogueImpl implements DataCatalogue {
// iterate over them
for (CkanOrganization ckanOrganization : organizations) {
// get the list of users in it (if you try ckanOrganization.getUsers() it returns null.. maybe a bug TODO)
List<CkanUser> users = ckanCaller.getOrganization(ckanOrganization.getName()).getUsers();
// check if the current user is among them
for (CkanUser ckanUser : users) {
if(ckanUser.getName().equals(ckanUsername)){
LOG.debug("User " + ckanUsername + " is into " + ckanOrganization.getName());
@ -440,10 +442,8 @@ public class DataCatalogueImpl implements DataCatalogue {
// checks
checkNotNull(username);
// in order to avoid errors, the username is always converted
String ckanUsername = CatalogueUtilMethods.fromUsernameToCKanUsername(username);
// list to return
List<CkanGroup> toReturn = new ArrayList<CkanGroup>();
@ -459,6 +459,7 @@ public class DataCatalogueImpl implements DataCatalogue {
// check if the current user is among them
for (CkanUser ckanUser : users) {
if(ckanUser.getName().equals(ckanUsername)){
LOG.debug("User " + ckanUsername + " is into " + ckanGroup.getName());
@ -513,15 +514,22 @@ public class DataCatalogueImpl implements DataCatalogue {
return null;
}
@Override
public boolean existProductWithNameOrId(String nameOrId) {
checkNotNull(nameOrId);
checkArgument(!nameOrId.isEmpty());
try{
ExtendCkanClient client = new ExtendCkanClient(CKAN_CATALOGUE_URL, CKAN_TOKEN_SYS);
CkanDataset product = client.getDataset(nameOrId);
return product != null;
}catch(Exception e){
LOG.debug("A dataset with name " + nameOrId + " doesn't exist");
return false;
}
}
/*
*
@ -546,6 +554,163 @@ public class DataCatalogueImpl implements DataCatalogue {
*
*/
@Override
public String createCKanDatasetMultipleCustomFields(String username, String title, String name, String organizationNameOrId,
String author, String authorMail, String maintainer, String maintainerMail, long version,
String description, String licenseId, List<String> tags, Map<String, List<String>> customFieldsMultiple,
List<ResourceBean> resources, boolean setPublic) throws Exception {
// delegate the private method
return createCkanDatasetBody(username, title, name, organizationNameOrId, author, authorMail, maintainer, maintainerMail,
version, description, licenseId, tags, null, customFieldsMultiple, resources, setPublic);
}
// the body of the actual dataset creation methods
private String createCkanDatasetBody(String username, String title, String name, String organizationNameOrId, String author,
String authorMail, String maintainer, String maintainerMail, long version, String description,
String licenseId, List<String> tags, Map<String, String> customFields,
Map<String, List<String>> customFieldsMultipleValues, List<ResourceBean> resources, boolean setPublic)
throws Exception {
// checks (minimum)
checkNotNull(username);
checkNotNull(organizationNameOrId);
checkArgument(!organizationNameOrId.isEmpty());
checkArgument(!(title == null && name == null || title.isEmpty() && name.isEmpty()), "Name and Title cannot be empty/null at the same time!");
LOG.debug("Request for dataset creation");
//ExtendCkanClient client = new ExtendCkanClient(CKAN_CATALOGUE_URL, apiKey);
//String ckanUsername = getUserFromApiKey(apiKey).getName();
CkanDataset dataset = new CkanDataset();
// in order to avoid errors, the username is always converted
String ckanUsername = CatalogueUtilMethods.fromUsernameToCKanUsername(username);
String nameToUse = name;
if(nameToUse == null)
nameToUse = CatalogueUtilMethods.fromProductTitleToName(title);
LOG.debug("Name of the dataset is going to be " + nameToUse + ". Title is going to be " + title);
dataset.setName(nameToUse);
dataset.setTitle(title);
// CkanOrganization orgOwner = client.getOrganization(organizationNameOrId);
// dataset.setOwnerOrg(orgOwner.getId());
dataset.setAuthor(author);
dataset.setAuthorEmail(authorMail);
dataset.setMaintainer(maintainer);
dataset.setMaintainerEmail(maintainerMail);
dataset.setVersion(String.valueOf(version));
// description must be escaped
if(description != null && !description.isEmpty()){
Source descriptionEscaped = new Source(description);
Segment htmlSeg = new Segment(descriptionEscaped, 0, descriptionEscaped.length());
Renderer htmlRend = new Renderer(htmlSeg);
dataset.setNotes(htmlRend.toString());
LOG.debug("Description escaped " + htmlRend.toString());
}
dataset.setLicenseId(licenseId);
// set the tags, if any
if(tags != null && !tags.isEmpty()){
List<CkanTag> ckanTags = new ArrayList<CkanTag>(tags.size());
for (String stringTag : tags) {
ckanTags.add(new CkanTag(stringTag));
}
dataset.setTags(ckanTags);
}
// set the custom fields, if any
List<CkanPair> extras = new ArrayList<CkanPair>();
if(customFields != null && !customFields.isEmpty()){
Iterator<Entry<String, String>> iterator = customFields.entrySet().iterator();
while (iterator.hasNext()) {
Map.Entry<String, String> entry = iterator.next();
extras.add(new CkanPair(entry.getKey(), entry.getValue()));
}
}else if(customFieldsMultipleValues != null && !customFieldsMultipleValues.isEmpty()){
Iterator<Entry<String, List<String>>> iterator = customFieldsMultipleValues.entrySet().iterator();
while (iterator.hasNext()) {
Map.Entry<String, List<String>> entry = iterator.next();
List<String> valuesForEntry = entry.getValue();
for (String value : valuesForEntry) {
extras.add(new CkanPair(entry.getKey(), value));
}
}
}
dataset.setExtras(extras);
// check if we need to add the resources
if(resources != null && !resources.isEmpty()){
LOG.debug("We need to add resources to the dataset");
try{
List<CkanResource> resourcesCkan = new ArrayList<CkanResource>();
for(ResourceBean resource: resources){
LOG.debug("Going to add resource described by " + resource);
CkanResource newResource = new CkanResource();
newResource.setDescription(resource.getDescription());
newResource.setId(resource.getId());
newResource.setUrl(resource.getUrl());
newResource.setName(resource.getName());
newResource.setMimetype(resource.getMimeType());
newResource.setFormat(resource.getMimeType());
newResource.setOwner(ckanUsername);
resourcesCkan.add(newResource);
}
// add to the dataset
dataset.setResources(resourcesCkan);
}catch(Exception e){
LOG.error("Unable to add those resources to the dataset", e);
}
}
// try to create
String jsonValueDataset = MarshUnmarshCkanObject.toJsonValueDataset(dataset);
LOG.debug("Marshalling dataset is: " + jsonValueDataset);
jsonValueDataset = gCatCaller.createDataset(jsonValueDataset,true);
LOG.debug("Created dataset is: " + jsonValueDataset);
if(jsonValueDataset != null){
CkanDataset toCkanDataset = MarshUnmarshCkanObject.toCkanDataset(jsonValueDataset);
LOG.debug("Dataset with name " + toCkanDataset.getName() + " has been created. Setting visibility");
// set visibility
boolean visibilitySet = directCkanCaller.setDatasetPrivate(
!setPublic, // swap to private
toCkanDataset.getOrganization().getId(),
toCkanDataset.getId(),
CKAN_TOKEN_SYS); // use sysadmin api key to be sure it will be set
LOG.info("Was visibility set to " + (setPublic ? "public" : "private") + "? " + visibilitySet);
// set searchable to true if dataset visibility is private
if(!setPublic){ // (swap to private)
boolean searchableSet = setSearchableField(toCkanDataset.getId(), true);
LOG.info("Was searchable set to True? " + searchableSet);
}
return toCkanDataset.getId();
}
return null;
}
public boolean patchProductCustomFields(String productId, Map<String, List<String>> customFieldsToChange,
boolean removeOld) {
@ -560,9 +725,11 @@ public class DataCatalogueImpl implements DataCatalogue {
return false;
}
public boolean existProductWithNameOrId(String nameOrId) {
return false;
@Override
public boolean setSearchableField(String datasetId, boolean searchable) {
return directCkanCaller.setSearchableField(datasetId, searchable, CKAN_TOKEN_SYS);
}
}