diff --git a/src/main/java/org/gcube/datacatalogue/ckanutillibrary/ckan/DirectCkanCaller.java b/src/main/java/org/gcube/datacatalogue/ckanutillibrary/ckan/DirectCkanCaller.java new file mode 100644 index 0000000..f01a465 --- /dev/null +++ b/src/main/java/org/gcube/datacatalogue/ckanutillibrary/ckan/DirectCkanCaller.java @@ -0,0 +1,140 @@ +package org.gcube.datacatalogue.ckanutillibrary.ckan; + +import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.base.Preconditions.checkNotNull; + +import org.json.simple.JSONArray; +import org.json.simple.JSONObject; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import eu.trentorise.opendata.jackan.internal.org.apache.http.HttpResponse; +import eu.trentorise.opendata.jackan.internal.org.apache.http.HttpStatus; +import eu.trentorise.opendata.jackan.internal.org.apache.http.client.methods.HttpPost; +import eu.trentorise.opendata.jackan.internal.org.apache.http.entity.StringEntity; +import eu.trentorise.opendata.jackan.internal.org.apache.http.impl.client.CloseableHttpClient; +import eu.trentorise.opendata.jackan.internal.org.apache.http.impl.client.HttpClientBuilder; + + +// TODO: Auto-generated Javadoc +/** + * The Class DirectCkanCaller. + * + * @author Francesco Mangiacrapa at ISTI-CNR Pisa (Italy) + * Jun 3, 2020 + */ +public class DirectCkanCaller { + + private static final String PATH_PACKAGE_PATCH = "/api/3/action/package_patch"; + public final static String PATH_SET_PRIVATE_DATASET = "/api/3/action/bulk_update_private"; + public final static String PATH_SET_PUBLIC_DATASET = "/api/3/action/bulk_update_public"; + private String catalogueURL; + + private static final Logger LOG = LoggerFactory.getLogger(DirectCkanCaller.class); + + + /** + * Instantiates a new direct ckan caller. + * + * @param catalogueURL the catalogue URL + */ + public DirectCkanCaller(String catalogueURL){ + this.catalogueURL = catalogueURL; + } + + /** + * Set dataset private/public. + * + * @param priv the priv + * @param organizationId (NOTE: The ID, not the name!) + * @param datasetId (NOTE: The ID, not the name!) + * @param apiKey the user's api key + * @return true on success, false otherwise + */ + public boolean setDatasetPrivate(boolean priv, String organizationId, + String datasetId, String apiKey) { + + // checks + checkNotNull(organizationId); + checkNotNull(apiKey); + checkNotNull(datasetId); + checkArgument(!apiKey.isEmpty()); + checkArgument(!datasetId.isEmpty()); + checkArgument(!organizationId.isEmpty()); + + JSONObject obj = new JSONObject(); + obj.put("org_id", organizationId); + + JSONArray array = new JSONArray(); + array.add(datasetId); + obj.put("datasets", array); + + try(CloseableHttpClient httpClient = HttpClientBuilder.create().build();) { + HttpPost request; + + if(priv) + request = new HttpPost(catalogueURL + PATH_SET_PRIVATE_DATASET); + else + request = new HttpPost(catalogueURL + PATH_SET_PUBLIC_DATASET); + + LOG.info("Excuting request for making dataset with id " + datasetId + " " + (priv? "private" : "public")); + + request.addHeader("Authorization", apiKey); + StringEntity params = new StringEntity(obj.toJSONString()); + request.setEntity(params); + HttpResponse response = httpClient.execute(request); + + LOG.info("Response code is " + response.getStatusLine().getStatusCode() + " and response message is " + response.getStatusLine().getReasonPhrase()); + + if(response.getStatusLine().getStatusCode() == HttpStatus.SC_OK) + return true; + + }catch (Exception ex) { + LOG.error("Error while trying to set private the dataset ", ex); + } + + + return false; + } + + /** + * Sets the searchable field. + * + * @param datasetId the dataset id + * @param searchable the searchable + * @return true, if successful + */ + public boolean setSearchableField(String datasetId, boolean searchable, String apiKey) { + + // checks + checkNotNull(datasetId); + checkArgument(!datasetId.isEmpty()); + String searchableAsString = searchable ? "True" : "False"; + + // Patch package path + String patchPackage = catalogueURL + PATH_PACKAGE_PATCH; + + JSONObject obj = new JSONObject(); + obj.put("id", datasetId); + obj.put("searchable", searchableAsString); + + try(CloseableHttpClient httpClient = HttpClientBuilder.create().build();) { + HttpPost request = new HttpPost(patchPackage); + request.addHeader("Authorization", apiKey); + StringEntity params = new StringEntity(obj.toJSONString()); + request.setEntity(params); + HttpResponse response = httpClient.execute(request); + LOG.debug("Response code is " + response.getStatusLine().getStatusCode() + " and response message is " + response.getStatusLine().getReasonPhrase()); + + if(response.getStatusLine().getStatusCode() == HttpStatus.SC_OK) + return true; + + }catch (Exception ex) { + LOG.error("Error while trying to set searchable the dataset ", ex); + } + + return false; + } + + +} diff --git a/src/main/java/org/gcube/datacatalogue/ckanutillibrary/ckan/ExtendCkanClient.java b/src/main/java/org/gcube/datacatalogue/ckanutillibrary/ckan/ExtendCkanClient.java index 43a2fce..59fd7b4 100644 --- a/src/main/java/org/gcube/datacatalogue/ckanutillibrary/ckan/ExtendCkanClient.java +++ b/src/main/java/org/gcube/datacatalogue/ckanutillibrary/ckan/ExtendCkanClient.java @@ -1,6 +1,3 @@ -/** - * - */ package org.gcube.datacatalogue.ckanutillibrary.ckan; import static com.google.common.base.Preconditions.checkNotNull; @@ -19,7 +16,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.base.Charsets; import com.google.common.io.CharStreams; -import eu.trentorise.opendata.jackan.CheckedCkanClient; +import eu.trentorise.opendata.jackan.CkanClient; import eu.trentorise.opendata.jackan.exceptions.CkanException; import eu.trentorise.opendata.jackan.exceptions.JackanException; import eu.trentorise.opendata.jackan.internal.org.apache.http.client.fluent.Request; @@ -29,36 +26,34 @@ import eu.trentorise.opendata.jackan.model.CkanOrganization; import eu.trentorise.opendata.jackan.model.CkanResponse; +// TODO: Auto-generated Javadoc /** - * The Class ExtendCkanClient. + * The Class SimpleExtendCkanClient. * * @author Francesco Mangiacrapa at ISTI-CNR (francesco.mangiacrapa@isti.cnr.it) * - * Jun 14, 2019 + * Jun 7, 2019 + * */ -public class ExtendCkanClient extends CheckedCkanClient implements PatchedCkan{ - - private static final Logger logger = LoggerFactory.getLogger(ExtendCkanClient.class); +public class ExtendCkanClient extends CkanClient implements PatchedCkan{ - @Nullable - private static ObjectMapper objectMapper; - private String catalogueURL; private String ckanToken; private int extendTimeout = 120000; //in milliseconds - + @Nullable + private static ObjectMapper objectMapper; + private static final Logger logger = LoggerFactory.getLogger(ExtendCkanClient.class); /** - * Instantiates a new extend ckan client. + * Instantiates a new simple extend ckan client. * - * @param catalogueURL the catalogue url + * @param catalogueURL the catalogue URL */ public ExtendCkanClient(String catalogueURL) { super(catalogueURL); this.catalogueURL = catalogueURL; - } - + /** * Instantiates a new extend ckan client. * @@ -84,7 +79,7 @@ public class ExtendCkanClient extends CheckedCkanClient implements PatchedCkan{ this.ckanToken = ckanToken; this.extendTimeout = timeout; } - + /** * Configures the request. Should work both for GETs and POSTs. @@ -104,38 +99,13 @@ public class ExtendCkanClient extends CheckedCkanClient implements PatchedCkan{ return request; } - /** - * Gets the catalogue url. - * - * @return the catalogueURL - */ - public String getCatalogueURL() { - - return catalogueURL; - } - - - /** - * Gets the ckan token. - * - * @return the ckanToken - */ - @Override - public String getCkanToken() { - - return ckanToken; - } - - /** - * Gets the timeout. - * - * @return the timeout - */ - public int getTimeout() { - return extendTimeout; - } - - /* (non-Javadoc) + /** + * Gets the organization. + * + * @param idOrName the id or name + * @return the organization + */ + /* (non-Javadoc) * @see eu.trentorise.opendata.jackan.CkanClient#getOrganization(java.lang.String) */ public synchronized CkanOrganization getOrganization(String idOrName) { @@ -147,6 +117,12 @@ public class ExtendCkanClient extends CheckedCkanClient implements PatchedCkan{ } + /** + * Gets the group. + * + * @param idOrName the id or name + * @return the group + */ /* (non-Javadoc) * @see eu.trentorise.opendata.jackan.CkanClient#getGroup(java.lang.String) */ @@ -164,7 +140,7 @@ public class ExtendCkanClient extends CheckedCkanClient implements PatchedCkan{ * * @return the object mapper */ - static ObjectMapper getObjectMapper() { + public static ObjectMapper getObjectMapper() { if (objectMapper == null) { objectMapper = new ObjectMapper(); configureObjectMapper(objectMapper); @@ -172,6 +148,15 @@ public class ExtendCkanClient extends CheckedCkanClient implements PatchedCkan{ return objectMapper; } + /** + * Gets the http. + * + * @param the generic type + * @param responseType the response type + * @param path the path + * @param params the params + * @return the http + */ /* (non-Javadoc) * @see org.gcube.datacatalogue.ckanutillibrary.server.patch.PatchedCkan#getHttp(java.lang.Class, java.lang.String, java.lang.Object[]) */ @@ -215,6 +200,13 @@ public class ExtendCkanClient extends CheckedCkanClient implements PatchedCkan{ } + /** + * Calc full url. + * + * @param path the path + * @param params the params + * @return the string + */ /* (non-Javadoc) * @see org.gcube.datacatalogue.ckanutillibrary.server.patch.PatchedCkan#calcFullUrl(java.lang.String, java.lang.Object[]) */ @@ -236,5 +228,26 @@ public class ExtendCkanClient extends CheckedCkanClient implements PatchedCkan{ + Arrays.toString(params), ex); } } + + /** + * Gets the catalogue URL. + * + * @return the catalogue URL + */ + public String getCatalogueURL() { + + return catalogueURL; + } + + /** + * Gets the ckan token. + * + * @return the ckan token + */ + @Override + public String getCkanToken() { + return ckanToken; + } + } diff --git a/src/main/java/org/gcube/datacatalogue/ckanutillibrary/ckan/MarshUnmarshCkanObject.java b/src/main/java/org/gcube/datacatalogue/ckanutillibrary/ckan/MarshUnmarshCkanObject.java new file mode 100644 index 0000000..0767857 --- /dev/null +++ b/src/main/java/org/gcube/datacatalogue/ckanutillibrary/ckan/MarshUnmarshCkanObject.java @@ -0,0 +1,38 @@ +package org.gcube.datacatalogue.ckanutillibrary.ckan; + +import java.io.IOException; + +import com.fasterxml.jackson.core.JsonParseException; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonMappingException; + +import eu.trentorise.opendata.jackan.model.CkanDataset; + +public class MarshUnmarshCkanObject { + + /** + * To ckan dataset. + * + * @param jsonDataset the json dataset + * @return the ckan dataset + * @throws IOException + * @throws JsonMappingException + * @throws JsonParseException + */ + public static CkanDataset toCkanDataset(String jsonDataset) throws JsonParseException, JsonMappingException, IOException { + return ExtendCkanClient.getObjectMapper().readValue(jsonDataset, CkanDataset.class); + } + + + /** + * To json value dataset. + * + * @param dataset the dataset + * @return the string + * @throws JsonProcessingException + */ + public static String toJsonValueDataset(CkanDataset dataset) throws JsonProcessingException { + return ExtendCkanClient.getObjectMapper().writeValueAsString(dataset); + } + +} diff --git a/src/main/java/org/gcube/datacatalogue/ckanutillibrary/ckan/SimpleExtendCkanClient.java b/src/main/java/org/gcube/datacatalogue/ckanutillibrary/ckan/SimpleExtendCkanClient.java deleted file mode 100644 index 93c59cd..0000000 --- a/src/main/java/org/gcube/datacatalogue/ckanutillibrary/ckan/SimpleExtendCkanClient.java +++ /dev/null @@ -1,168 +0,0 @@ -package org.gcube.datacatalogue.ckanutillibrary.ckan; - -import static com.google.common.base.Preconditions.checkNotNull; - -import java.io.InputStream; -import java.io.InputStreamReader; -import java.net.URLEncoder; -import java.util.Arrays; - -import javax.annotation.Nullable; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import com.fasterxml.jackson.databind.ObjectMapper; -import com.google.common.base.Charsets; -import com.google.common.io.CharStreams; - -import eu.trentorise.opendata.jackan.CkanClient; -import eu.trentorise.opendata.jackan.exceptions.CkanException; -import eu.trentorise.opendata.jackan.exceptions.JackanException; -import eu.trentorise.opendata.jackan.internal.org.apache.http.client.fluent.Request; -import eu.trentorise.opendata.jackan.internal.org.apache.http.client.fluent.Response; -import eu.trentorise.opendata.jackan.model.CkanGroup; -import eu.trentorise.opendata.jackan.model.CkanOrganization; -import eu.trentorise.opendata.jackan.model.CkanResponse; - - -/** - * The Class SimpleExtendCkanClient. - * - * @author Francesco Mangiacrapa at ISTI-CNR (francesco.mangiacrapa@isti.cnr.it) - * - * Jun 7, 2019 - */ -public class SimpleExtendCkanClient extends CkanClient implements PatchedCkan{ - - - private String catalogueURL; - - @Nullable - private static ObjectMapper objectMapper; - - private static final Logger logger = LoggerFactory.getLogger(SimpleExtendCkanClient.class); - - /** - * Instantiates a new simple extend ckan client. - * - * @param catalogueURL the catalogue URL - */ - public SimpleExtendCkanClient(String catalogueURL) { - super(catalogueURL); - this.catalogueURL = catalogueURL; - } - - /* (non-Javadoc) - * @see eu.trentorise.opendata.jackan.CkanClient#getOrganization(java.lang.String) - */ - public synchronized CkanOrganization getOrganization(String idOrName) { - checkNotNull(idOrName, "Need a valid id or name!"); - logger.info("Patched read organization for id/name: {}", idOrName); - return getHttp(OrganizationResponse.class, "/api/3/action/organization_show", "id", idOrName, - "include_datasets", "false", "include_users", "true").result; - - } - - - /* (non-Javadoc) - * @see eu.trentorise.opendata.jackan.CkanClient#getGroup(java.lang.String) - */ - public synchronized CkanGroup getGroup(String idOrName) { - checkNotNull(idOrName, "Need a valid id or name!"); - logger.info("Patched read group for id/name: {}", idOrName); - return getHttp(GroupResponse.class, "/api/3/action/group_show", "id", idOrName, "include_datasets", - "false", "include_users", "true").result; - } - - - /** - * Retrieves the Jackson object mapper for reading operations. Internally, - * Object mapper is initialized at first call. - * - * @return the object mapper - */ - public static ObjectMapper getObjectMapper() { - if (objectMapper == null) { - objectMapper = new ObjectMapper(); - configureObjectMapper(objectMapper); - } - return objectMapper; - } - - /* (non-Javadoc) - * @see org.gcube.datacatalogue.ckanutillibrary.server.patch.PatchedCkan#getHttp(java.lang.Class, java.lang.String, java.lang.Object[]) - */ - public T getHttp(Class responseType, String path, Object... params) { - - checkNotNull(responseType); - checkNotNull(path); - String fullUrl = calcFullUrl(path, params); - T ckanResponse; - String returnedText; - - try { - - logger.debug("getting {}", fullUrl); - Request request = Request.Get(fullUrl); - configureRequest(request); - Response response = request.execute(); - InputStream stream = response.returnResponse() - .getEntity() - .getContent(); - - try (InputStreamReader reader = new InputStreamReader(stream, Charsets.UTF_8)) { - returnedText = CharStreams.toString(reader); - } - - logger.trace("returnedText {}", returnedText); - } catch (Exception ex) { - throw new CkanException("Error while performing GET. Request url was: " + fullUrl, this, ex); - } - try { - ckanResponse = getObjectMapper().readValue(returnedText, responseType); - } catch (Exception ex) { - throw new CkanException( - "Couldn't interpret json returned by the server! Returned text was: " + returnedText, this, ex); - } - - if (!ckanResponse.isSuccess()) { - throwCkanException("Error while performing GET. Request url was: " + fullUrl, ckanResponse); - } - return ckanResponse; - } - - - /* (non-Javadoc) - * @see org.gcube.datacatalogue.ckanutillibrary.server.patch.PatchedCkan#calcFullUrl(java.lang.String, java.lang.Object[]) - */ - public String calcFullUrl(String path, Object[] params) { - checkNotNull(path); - - try { - StringBuilder sb = new StringBuilder().append(catalogueURL) - .append(path); - for (int i = 0; i < params.length; i += 2) { - sb.append(i == 0 ? "?" : "&") - .append(URLEncoder.encode(params[i].toString(), "UTF-8")) - .append("=") - .append(URLEncoder.encode(params[i + 1].toString(), "UTF-8")); - } - return sb.toString(); - } catch (Exception ex) { - throw new JackanException("Error while building url to perform GET! \n path: " + path + " \n params: " - + Arrays.toString(params), ex); - } - } - - /** - * Gets the catalogue url. - * - * @return the catalogueURL - */ - public String getCatalogueURL() { - - return catalogueURL; - } - -} diff --git a/src/main/java/org/gcube/datacatalogue/ckanutillibrary/db/DBCaller.java b/src/main/java/org/gcube/datacatalogue/ckanutillibrary/db/DBCaller.java index f989172..1e1532d 100644 --- a/src/main/java/org/gcube/datacatalogue/ckanutillibrary/db/DBCaller.java +++ b/src/main/java/org/gcube/datacatalogue/ckanutillibrary/db/DBCaller.java @@ -95,7 +95,7 @@ public class DBCaller { */ public Map getGroupsByUserFromDB(String username){ checkNotNull(username); - LOG.debug("Get groups by user called"); + LOG.debug("Get groups by user called for username: "+username); //couples (groups id, capacity) of the user in the group @@ -129,6 +129,8 @@ public class DBCaller { }finally{ closeConnection(connection); } + + LOG.debug("returning map: "+toReturn); return toReturn; } @@ -140,7 +142,7 @@ public class DBCaller { * @return the user id for username */ private String getUserIdForUsername(String username) { - LOG.debug("Get user id for username called"); + LOG.debug("Get user id for username called for username: "+username); Connection connection = null; String userId = null; @@ -150,7 +152,7 @@ public class DBCaller { String selQuery = "SELECT \"id\" FROM \"public\".\"user\" WHERE \"name\"=?;"; - LOG.info("Performing query: "+selQuery); + LOG.debug("Performing query: "+selQuery); PreparedStatement preparedStatement = connection.prepareStatement(selQuery); preparedStatement.setString(1, username); @@ -213,6 +215,8 @@ public class DBCaller { }finally{ closeConnection(connection); } + + LOG.debug("returning map: "+toReturn); return toReturn; } diff --git a/src/main/java/org/gcube/datacatalogue/ckanutillibrary/gcat/GCatCaller.java b/src/main/java/org/gcube/datacatalogue/ckanutillibrary/gcat/GCatCaller.java index c14eca7..82cf4c6 100644 --- a/src/main/java/org/gcube/datacatalogue/ckanutillibrary/gcat/GCatCaller.java +++ b/src/main/java/org/gcube/datacatalogue/ckanutillibrary/gcat/GCatCaller.java @@ -2,19 +2,11 @@ package org.gcube.datacatalogue.ckanutillibrary.gcat; import java.io.IOException; import java.net.MalformedURLException; -import java.util.List; -import org.gcube.datacatalogue.ckanutillibrary.ckan.SimpleExtendCkanClient; import org.gcube.gcat.client.Item; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.fasterxml.jackson.core.JsonParseException; -import com.fasterxml.jackson.databind.JsonMappingException; - -import eu.trentorise.opendata.jackan.model.CkanDataset; -import eu.trentorise.opendata.jackan.model.CkanLicense; - // TODO: Auto-generated Javadoc /** * The Class GCatCaller. @@ -24,18 +16,17 @@ import eu.trentorise.opendata.jackan.model.CkanLicense; */ public class GCatCaller { - private SimpleExtendCkanClient jackanCkanClient; - private String catalogueURL; private static final Logger LOG = LoggerFactory.getLogger(GCatCaller.class); /** * Instantiates a new g cat caller. + * + * @param catalogueURL the catalogue URL */ public GCatCaller(String catalogueURL) { this.catalogueURL = catalogueURL; - this.jackanCkanClient = new SimpleExtendCkanClient(catalogueURL); } @@ -43,27 +34,27 @@ public class GCatCaller { * Gets the dataset for name. * * @param datasetIdOrName the dataset id or name - * @return the dataset for name - * @throws JsonParseException the json parse exception - * @throws JsonMappingException the json mapping exception + * @return the jsonValue + * @throws MalformedURLException * @throws IOException Signals that an I/O exception has occurred. */ - public CkanDataset getDatasetForName(String datasetIdOrName) throws JsonParseException, JsonMappingException, IOException { + public String getDatasetForName(String datasetIdOrName) throws MalformedURLException { LOG.debug("Get dataset for name "+datasetIdOrName+ "called"); - String json = new Item().read(datasetIdOrName); - return SimpleExtendCkanClient.getObjectMapper().readValue(json, CkanDataset.class); + return new Item().read(datasetIdOrName); } - - - /** - * TODO gCAT missing - * @return - * @throws MalformedURLException - */ - public List getLicenseList() { - LOG.debug("Get linces list called"); - return jackanCkanClient.getLicenseList(); + + /** + * Creates the dataset. + * + * @param jsonDataset the json dataset + * @param socialPost if true sends the social post + * @return the jsonValue + * @throws MalformedURLException the malformed URL exception + */ + public String createDataset(String jsonDataset, boolean socialPost) throws MalformedURLException { + LOG.debug("Create dataset called"); + return new Item().create(jsonDataset,socialPost); } } diff --git a/src/main/java/org/gcube/datacatalogue/ckanutillibrary/server/DataCatalogue.java b/src/main/java/org/gcube/datacatalogue/ckanutillibrary/server/DataCatalogue.java index e8b5494..78cece6 100644 --- a/src/main/java/org/gcube/datacatalogue/ckanutillibrary/server/DataCatalogue.java +++ b/src/main/java/org/gcube/datacatalogue/ckanutillibrary/server/DataCatalogue.java @@ -39,67 +39,6 @@ public interface DataCatalogue { */ String getUnencryptedUrlFromDatasetIdOrName(String datasetIdOrName); - /** - * Create a dataset with those information. The method allows to have multiple - * values for the same custom field key. NOTE: unfortunately java doesn't - * support overload in java interface methods (that's way I cannot use the same - * name for the method) - * - * @param title the title - * @param name (unique identifier) - * @param organizationNameOrId the organization name or id - * @param author the author - * @param authorMail the author mail - * @param maintainer the maintainer - * @param maintainerMail the maintainer mail - * @param version the version - * @param description the description - * @param licenseId the license id - * @param tags the tags - * @param customFields the custom fields - * @param resources the resources - * @param setPublic (manage visibility: Admin role is needed) - * @return the id of the dataset on success, null otherwise - * @throws Exception the exception - */ - String createCKanDatasetMultipleCustomFields(String title, String name, String organizationNameOrId, String author, - String authorMail, String maintainer, String maintainerMail, long version, String description, - String licenseId, List tags, Map> customFields, List resources, - boolean setPublic) throws Exception; - - /** - * Patch a product with product id productId by using the couples in - * customFieldsToChange. NOTE: only the specified custom fields will be changed. - * If a custom field with a given key already exists, and removeOld is set to - * false, the new values are added at the end of the list. Otherwise they are - * lost. - * - * @param productId the product id - * @param customFieldsToChange the custom fields to change - * @param removeOld the remove old - * @return true, if successful - */ - boolean patchProductCustomFields(String productId, Map> customFieldsToChange, - boolean removeOld); - - /** - * Add a resource described by the bean to the dataset id into - * resource.datasetId - * - * @param resource the resource - * @return String the id of the resource on success, null otherwise - * @throws Exception the exception - */ - String addResourceToDataset(ResourceBean resource) throws Exception; - - /** - * Remove the resource with id resourceId from the dataset in which it is. - * - * @param resourceId the resource id - * @return true on success, false otherwise. - */ - boolean deleteResourceFromDataset(String resourceId); - /** * Create a dataset with those information. * @@ -201,4 +140,86 @@ public interface DataCatalogue { */ Map> getUserRoleByOrganization(String username); + + + + + //WRITE OPERATIONS + + + + + + /** + * Set searchable field + * The field, if set to true, allows organization's users to view private products also in groups + * @param datasetId the id of the dataset to update + * @param searchable the value to assign to the field + * @return true if the field is set, false otherwise + */ + boolean setSearchableField(String datasetId, boolean searchable); + + + /** + * Create a dataset with those information. The method allows to have multiple + * values for the same custom field key. NOTE: unfortunately java doesn't + * support overload in java interface methods (that's way I cannot use the same + * name for the method) + * + * @param the username that is submitting the publishing request + * @param title the title + * @param name (unique identifier) + * @param organizationNameOrId the organization name or id + * @param author the author + * @param authorMail the author mail + * @param maintainer the maintainer + * @param maintainerMail the maintainer mail + * @param version the version + * @param description the description + * @param licenseId the license id + * @param tags the tags + * @param customFields the custom fields + * @param resources the resources + * @param setPublic (manage visibility: Admin role is needed) + * @return the id of the dataset on success, null otherwise + * @throws Exception the exception + */ + String createCKanDatasetMultipleCustomFields(String username, String title, String name, String organizationNameOrId, String author, + String authorMail, String maintainer, String maintainerMail, long version, String description, + String licenseId, List tags, Map> customFields, List resources, + boolean setPublic) throws Exception; + + /** + * Patch a product with product id productId by using the couples in + * customFieldsToChange. NOTE: only the specified custom fields will be changed. + * If a custom field with a given key already exists, and removeOld is set to + * false, the new values are added at the end of the list. Otherwise they are + * lost. + * + * @param productId the product id + * @param customFieldsToChange the custom fields to change + * @param removeOld the remove old + * @return true, if successful + */ + boolean patchProductCustomFields(String productId, Map> customFieldsToChange, + boolean removeOld); + + /** + * Add a resource described by the bean to the dataset id into + * resource.datasetId + * + * @param resource the resource + * @return String the id of the resource on success, null otherwise + * @throws Exception the exception + */ + String addResourceToDataset(ResourceBean resource) throws Exception; + + /** + * Remove the resource with id resourceId from the dataset in which it is. + * + * @param resourceId the resource id + * @return true on success, false otherwise. + */ + boolean deleteResourceFromDataset(String resourceId); + } diff --git a/src/main/java/org/gcube/datacatalogue/ckanutillibrary/server/DataCatalogueImpl.java b/src/main/java/org/gcube/datacatalogue/ckanutillibrary/server/DataCatalogueImpl.java index 099f1e9..f635940 100644 --- a/src/main/java/org/gcube/datacatalogue/ckanutillibrary/server/DataCatalogueImpl.java +++ b/src/main/java/org/gcube/datacatalogue/ckanutillibrary/server/DataCatalogueImpl.java @@ -6,15 +6,18 @@ import static com.google.common.base.Preconditions.checkNotNull; import java.net.URLEncoder; import java.util.ArrayList; import java.util.HashMap; +import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.Map.Entry; import java.util.concurrent.ConcurrentHashMap; import org.gcube.common.authorization.library.provider.SecurityTokenProvider; import org.gcube.common.scope.impl.ScopeBean; import org.gcube.common.scope.impl.ScopeBean.Type; +import org.gcube.datacatalogue.ckanutillibrary.ckan.DirectCkanCaller; import org.gcube.datacatalogue.ckanutillibrary.ckan.ExtendCkanClient; -import org.gcube.datacatalogue.ckanutillibrary.ckan.SimpleExtendCkanClient; +import org.gcube.datacatalogue.ckanutillibrary.ckan.MarshUnmarshCkanObject; import org.gcube.datacatalogue.ckanutillibrary.db.DBCaller; import org.gcube.datacatalogue.ckanutillibrary.gcat.GCatCaller; import org.gcube.datacatalogue.ckanutillibrary.server.DataCatalogueRunningCluster.ACCESS_LEVEL_TO_CATALOGUE_PORTLET; @@ -38,7 +41,13 @@ import eu.trentorise.opendata.jackan.model.CkanDataset; import eu.trentorise.opendata.jackan.model.CkanGroup; import eu.trentorise.opendata.jackan.model.CkanLicense; import eu.trentorise.opendata.jackan.model.CkanOrganization; +import eu.trentorise.opendata.jackan.model.CkanPair; +import eu.trentorise.opendata.jackan.model.CkanResource; +import eu.trentorise.opendata.jackan.model.CkanTag; import eu.trentorise.opendata.jackan.model.CkanUser; +import net.htmlparser.jericho.Renderer; +import net.htmlparser.jericho.Segment; +import net.htmlparser.jericho.Source; public class DataCatalogueImpl implements DataCatalogue { @@ -63,6 +72,7 @@ public class DataCatalogueImpl implements DataCatalogue { public Map mapAccessURLToCatalogue; + private static final String CATALOGUE_TAB_ENDING_URL = "/catalogue"; // gCat client @@ -72,10 +82,13 @@ public class DataCatalogueImpl implements DataCatalogue { private DBCaller dbCaller; // ckan client - private SimpleExtendCkanClient ckanCaller; + private ExtendCkanClient ckanCaller; // hashmap for ckan api keys private ConcurrentHashMap apiKeysMap; + + //http ckan caller + private DirectCkanCaller directCkanCaller; // apikey bean expires after X minutes in the above map private static final int EXPIRE_KEY_TIME = 60 * 60 * 1000; @@ -97,7 +110,7 @@ public class DataCatalogueImpl implements DataCatalogue { CKAN_DB_USER = runningInstance.getDataBaseUser().trim(); CKAN_DB_PASSWORD = runningInstance.getDataBasePassword().trim(); - //CKAN_TOKEN_SYS = runningInstance.getSysAdminToken().trim(); + CKAN_TOKEN_SYS = runningInstance.getSysAdminToken().trim(); CKAN_EMAIL = runningInstance.getEmailCatalogue().trim(); CKAN_CATALOGUE_URL = runningInstance.getDataCatalogueUrl().get(0).trim(); @@ -113,17 +126,13 @@ public class DataCatalogueImpl implements DataCatalogue { // build the clients gCatCaller = new GCatCaller(CKAN_CATALOGUE_URL); - dbCaller = new DBCaller(CKAN_DB_URL, CKAN_DB_PORT, CKAN_DB_NAME, CKAN_DB_USER, CKAN_DB_PASSWORD); - - ckanCaller = new SimpleExtendCkanClient(CKAN_CATALOGUE_URL); - + ckanCaller = new ExtendCkanClient(CKAN_CATALOGUE_URL); + directCkanCaller = new DirectCkanCaller(CKAN_CATALOGUE_URL); // init map apiKeysMap = new ConcurrentHashMap(); - // save the context CONTEXT = scope; - // extended roles extendRoleInOrganization = runningInstance.getExtendRoleInOrganization(); } @@ -228,7 +237,8 @@ public class DataCatalogueImpl implements DataCatalogue { String authzToken = SecurityTokenProvider.instance.get(); if(authzToken!=null && !authzToken.isEmpty()) { LOG.info("gcube-token found. Calling the gCat client"); - return gCatCaller.getDatasetForName(datasetId); + String jsonDataset = gCatCaller.getDatasetForName(datasetId); + return MarshUnmarshCkanObject.toCkanDataset(jsonDataset); } LOG.info("No api-key or gcube-token found. Calling Ckan Client without API-KEY"); @@ -272,13 +282,7 @@ public class DataCatalogueImpl implements DataCatalogue { return url; } - public String createCKanDatasetMultipleCustomFields(String title, String name, String organizationNameOrId, - String author, String authorMail, String maintainer, String maintainerMail, long version, - String description, String licenseId, List tags, Map> customFields, - List resources, boolean setPublic) throws Exception { - return null; - } - + @Override public Map> getUserRoleByGroup(String username) { LOG.info("Get user role by group called. The username is: "+username); @@ -328,6 +332,7 @@ public class DataCatalogueImpl implements DataCatalogue { HashMap subMap = new HashMap(); subMap.put(org, partialResult.get(orgID)); toReturn.put(orgID, subMap); + LOG.debug("For organisation: " + org.getName() + ", the user "+username+" has role: "+subMap); } LOG.debug("Returning map " + toReturn); @@ -399,10 +404,8 @@ public class DataCatalogueImpl implements DataCatalogue { // checks checkNotNull(username); - // in order to avoid errors, the username is always converted String ckanUsername = CatalogueUtilMethods.fromUsernameToCKanUsername(username); - // list to return List toReturn = new ArrayList(); @@ -413,12 +416,11 @@ public class DataCatalogueImpl implements DataCatalogue { // iterate over them for (CkanOrganization ckanOrganization : organizations) { - // get the list of users in it (if you try ckanOrganization.getUsers() it returns null.. maybe a bug TODO) List users = ckanCaller.getOrganization(ckanOrganization.getName()).getUsers(); - // check if the current user is among them for (CkanUser ckanUser : users) { + if(ckanUser.getName().equals(ckanUsername)){ LOG.debug("User " + ckanUsername + " is into " + ckanOrganization.getName()); @@ -440,10 +442,8 @@ public class DataCatalogueImpl implements DataCatalogue { // checks checkNotNull(username); - // in order to avoid errors, the username is always converted String ckanUsername = CatalogueUtilMethods.fromUsernameToCKanUsername(username); - // list to return List toReturn = new ArrayList(); @@ -459,6 +459,7 @@ public class DataCatalogueImpl implements DataCatalogue { // check if the current user is among them for (CkanUser ckanUser : users) { + if(ckanUser.getName().equals(ckanUsername)){ LOG.debug("User " + ckanUsername + " is into " + ckanGroup.getName()); @@ -513,15 +514,22 @@ public class DataCatalogueImpl implements DataCatalogue { return null; } - - - - - - - - - + @Override + public boolean existProductWithNameOrId(String nameOrId) { + + checkNotNull(nameOrId); + checkArgument(!nameOrId.isEmpty()); + + try{ + ExtendCkanClient client = new ExtendCkanClient(CKAN_CATALOGUE_URL, CKAN_TOKEN_SYS); + CkanDataset product = client.getDataset(nameOrId); + return product != null; + }catch(Exception e){ + LOG.debug("A dataset with name " + nameOrId + " doesn't exist"); + return false; + } + } + /* * @@ -546,6 +554,163 @@ public class DataCatalogueImpl implements DataCatalogue { * */ + @Override + public String createCKanDatasetMultipleCustomFields(String username, String title, String name, String organizationNameOrId, + String author, String authorMail, String maintainer, String maintainerMail, long version, + String description, String licenseId, List tags, Map> customFieldsMultiple, + List resources, boolean setPublic) throws Exception { + + // delegate the private method + return createCkanDatasetBody(username, title, name, organizationNameOrId, author, authorMail, maintainer, maintainerMail, + version, description, licenseId, tags, null, customFieldsMultiple, resources, setPublic); + } + + // the body of the actual dataset creation methods + private String createCkanDatasetBody(String username, String title, String name, String organizationNameOrId, String author, + String authorMail, String maintainer, String maintainerMail, long version, String description, + String licenseId, List tags, Map customFields, + Map> customFieldsMultipleValues, List resources, boolean setPublic) + throws Exception { + + // checks (minimum) + checkNotNull(username); + checkNotNull(organizationNameOrId); + checkArgument(!organizationNameOrId.isEmpty()); + checkArgument(!(title == null && name == null || title.isEmpty() && name.isEmpty()), "Name and Title cannot be empty/null at the same time!"); + + LOG.debug("Request for dataset creation"); + + //ExtendCkanClient client = new ExtendCkanClient(CKAN_CATALOGUE_URL, apiKey); + + //String ckanUsername = getUserFromApiKey(apiKey).getName(); + CkanDataset dataset = new CkanDataset(); + + // in order to avoid errors, the username is always converted + String ckanUsername = CatalogueUtilMethods.fromUsernameToCKanUsername(username); + + String nameToUse = name; + if(nameToUse == null) + nameToUse = CatalogueUtilMethods.fromProductTitleToName(title); + + LOG.debug("Name of the dataset is going to be " + nameToUse + ". Title is going to be " + title); + + dataset.setName(nameToUse); + dataset.setTitle(title); + +// CkanOrganization orgOwner = client.getOrganization(organizationNameOrId); +// dataset.setOwnerOrg(orgOwner.getId()); + dataset.setAuthor(author); + dataset.setAuthorEmail(authorMail); + dataset.setMaintainer(maintainer); + dataset.setMaintainerEmail(maintainerMail); + dataset.setVersion(String.valueOf(version)); + + // description must be escaped + if(description != null && !description.isEmpty()){ + Source descriptionEscaped = new Source(description); + Segment htmlSeg = new Segment(descriptionEscaped, 0, descriptionEscaped.length()); + Renderer htmlRend = new Renderer(htmlSeg); + dataset.setNotes(htmlRend.toString()); + + LOG.debug("Description escaped " + htmlRend.toString()); + } + + dataset.setLicenseId(licenseId); + + // set the tags, if any + if(tags != null && !tags.isEmpty()){ + List ckanTags = new ArrayList(tags.size()); + for (String stringTag : tags) { + ckanTags.add(new CkanTag(stringTag)); + } + dataset.setTags(ckanTags); + } + + // set the custom fields, if any + List extras = new ArrayList(); + + if(customFields != null && !customFields.isEmpty()){ + + Iterator> iterator = customFields.entrySet().iterator(); + + while (iterator.hasNext()) { + Map.Entry entry = iterator.next(); + extras.add(new CkanPair(entry.getKey(), entry.getValue())); + } + + }else if(customFieldsMultipleValues != null && !customFieldsMultipleValues.isEmpty()){ + + Iterator>> iterator = customFieldsMultipleValues.entrySet().iterator(); + while (iterator.hasNext()) { + Map.Entry> entry = iterator.next(); + List valuesForEntry = entry.getValue(); + for (String value : valuesForEntry) { + extras.add(new CkanPair(entry.getKey(), value)); + } + } + } + + dataset.setExtras(extras); + + // check if we need to add the resources + if(resources != null && !resources.isEmpty()){ + + LOG.debug("We need to add resources to the dataset"); + + try{ + List resourcesCkan = new ArrayList(); + for(ResourceBean resource: resources){ + + LOG.debug("Going to add resource described by " + resource); + CkanResource newResource = new CkanResource(); + newResource.setDescription(resource.getDescription()); + newResource.setId(resource.getId()); + newResource.setUrl(resource.getUrl()); + newResource.setName(resource.getName()); + newResource.setMimetype(resource.getMimeType()); + newResource.setFormat(resource.getMimeType()); + newResource.setOwner(ckanUsername); + resourcesCkan.add(newResource); + } + + // add to the dataset + dataset.setResources(resourcesCkan); + + }catch(Exception e){ + LOG.error("Unable to add those resources to the dataset", e); + } + + } + + // try to create + String jsonValueDataset = MarshUnmarshCkanObject.toJsonValueDataset(dataset); + LOG.debug("Marshalling dataset is: " + jsonValueDataset); + jsonValueDataset = gCatCaller.createDataset(jsonValueDataset,true); + LOG.debug("Created dataset is: " + jsonValueDataset); + + if(jsonValueDataset != null){ + CkanDataset toCkanDataset = MarshUnmarshCkanObject.toCkanDataset(jsonValueDataset); + LOG.debug("Dataset with name " + toCkanDataset.getName() + " has been created. Setting visibility"); + + // set visibility + boolean visibilitySet = directCkanCaller.setDatasetPrivate( + !setPublic, // swap to private + toCkanDataset.getOrganization().getId(), + toCkanDataset.getId(), + CKAN_TOKEN_SYS); // use sysadmin api key to be sure it will be set + + LOG.info("Was visibility set to " + (setPublic ? "public" : "private") + "? " + visibilitySet); + + // set searchable to true if dataset visibility is private + if(!setPublic){ // (swap to private) + boolean searchableSet = setSearchableField(toCkanDataset.getId(), true); + LOG.info("Was searchable set to True? " + searchableSet); + } + return toCkanDataset.getId(); + } + + return null; + } public boolean patchProductCustomFields(String productId, Map> customFieldsToChange, boolean removeOld) { @@ -560,9 +725,11 @@ public class DataCatalogueImpl implements DataCatalogue { return false; } - public boolean existProductWithNameOrId(String nameOrId) { - return false; + @Override + public boolean setSearchableField(String datasetId, boolean searchable) { + return directCkanCaller.setSearchableField(datasetId, searchable, CKAN_TOKEN_SYS); } + } diff --git a/src/test/java/TestDataCatalogueLib.java b/src/test/java/TestDataCatalogueLib.java index cb00cbe..8c58717 100644 --- a/src/test/java/TestDataCatalogueLib.java +++ b/src/test/java/TestDataCatalogueLib.java @@ -5,7 +5,6 @@ import org.gcube.datacatalogue.ckanutillibrary.server.ApplicationProfileScopePer import org.gcube.datacatalogue.ckanutillibrary.server.DataCatalogueFactory; import org.gcube.datacatalogue.ckanutillibrary.server.DataCatalogueImpl; import org.junit.Before; -import org.junit.Test; import org.slf4j.LoggerFactory; /** @@ -19,7 +18,9 @@ public class TestDataCatalogueLib { private static final org.slf4j.Logger logger = LoggerFactory.getLogger(TestDataCatalogueLib.class); private DataCatalogueFactory factory; - private String scope = "/gcube/devsec/devVRE"; + //private String scope = "/gcube/devsec/devVRE"; + private String scope = "/d4science.research-infrastructures.eu/SoBigData/Catalogue-TerritoriAperti"; + //private String scope = "/d4science.research-infrastructures.eu/gCubeApps/BiodiversityLab"; //private String testUser = "costantino_perciante"; private String testUser = "francesco_mangiacrapa"; String subjectId = "aa_father4"; @@ -46,15 +47,15 @@ public class TestDataCatalogueLib { DataCatalogueFactory factory = DataCatalogueFactory.getFactory(); while(true){ - factory.getUtilsPerScope("/gcube"); + factory.getUtilsPerScope(scope); Thread.sleep(60* 1000 * 3); - factory.getUtilsPerScope("/gcube"); + factory.getUtilsPerScope(scope); break; } for (int i = 0; i < 5; i++) { Thread.sleep(1000); - factory.getUtilsPerScope("/gcube"); + factory.getUtilsPerScope(scope); } } @@ -68,7 +69,7 @@ public class TestDataCatalogueLib { //@Test public void getScopePerUrl(){ - ScopeProvider.instance.set("/gcube"); + ScopeProvider.instance.set(scope); String url = "https://dev4.d4science.org/group/devvre/ckan"; String scopeToUse = ApplicationProfileScopePerUrlReader.getScopePerUrl(url); logger.debug("Retrieved scope is " + scopeToUse); @@ -86,7 +87,7 @@ public class TestDataCatalogueLib { * @return the user role by group * @throws Exception the exception */ - @Test + //@Test public void getUserRoleByGroup() throws Exception{ DataCatalogueImpl instance = factory.getUtilsPerScope(scope); String username = testUser; diff --git a/src/test/resources/.gitignore b/src/test/resources/.gitignore index 8f08e73..602005e 100644 --- a/src/test/resources/.gitignore +++ b/src/test/resources/.gitignore @@ -5,3 +5,12 @@ /log4j.properties /pred4s.gcubekey /preprod.gcubekey +/D4OS.gcubekey +/D4Research.gcubekey +/FARM.gcubekey +/OpenAIRE.gcubekey +/ParthenosVO.gcubekey +/SmartArea.gcubekey +/SoBigData.gcubekey +/d4science.research-infrastructures.eu.gcubekey +/gCubeApps.gcubekey diff --git a/target/classes/META-INF/MANIFEST.MF b/target/classes/META-INF/MANIFEST.MF index e5cf844..6eaf68d 100644 --- a/target/classes/META-INF/MANIFEST.MF +++ b/target/classes/META-INF/MANIFEST.MF @@ -6,7 +6,7 @@ Specification-Version: 0.1.0-SNAPSHOT Implementation-Title: CKan utility library Implementation-Version: 0.1.0-SNAPSHOT Implementation-Vendor-Id: org.gcube.datacatalogue -Build-Time: 20200601-145428 +Build-Time: 20200603-095004 Created-By: Maven Integration for Eclipse SCM-Branch: SCM-Revision: diff --git a/target/classes/META-INF/maven/org.gcube.datacatalogue/catalogue-util-library/pom.properties b/target/classes/META-INF/maven/org.gcube.datacatalogue/catalogue-util-library/pom.properties index bff563c..1f12d60 100644 --- a/target/classes/META-INF/maven/org.gcube.datacatalogue/catalogue-util-library/pom.properties +++ b/target/classes/META-INF/maven/org.gcube.datacatalogue/catalogue-util-library/pom.properties @@ -1,5 +1,5 @@ #Generated by Maven Integration for Eclipse -#Mon Jun 01 16:54:29 CEST 2020 +#Wed Jun 03 11:50:06 CEST 2020 version=0.1.0-SNAPSHOT groupId=org.gcube.datacatalogue m2e.projectName=catalogue-util-library diff --git a/target/classes/org/gcube/datacatalogue/ckanutillibrary/ckan/ExtendCkanClient.class b/target/classes/org/gcube/datacatalogue/ckanutillibrary/ckan/ExtendCkanClient.class index 91f6d88..9de95db 100644 Binary files a/target/classes/org/gcube/datacatalogue/ckanutillibrary/ckan/ExtendCkanClient.class and b/target/classes/org/gcube/datacatalogue/ckanutillibrary/ckan/ExtendCkanClient.class differ diff --git a/target/classes/org/gcube/datacatalogue/ckanutillibrary/ckan/ExtendCkanClient.java b/target/classes/org/gcube/datacatalogue/ckanutillibrary/ckan/ExtendCkanClient.java index 43a2fce..59fd7b4 100644 --- a/target/classes/org/gcube/datacatalogue/ckanutillibrary/ckan/ExtendCkanClient.java +++ b/target/classes/org/gcube/datacatalogue/ckanutillibrary/ckan/ExtendCkanClient.java @@ -1,6 +1,3 @@ -/** - * - */ package org.gcube.datacatalogue.ckanutillibrary.ckan; import static com.google.common.base.Preconditions.checkNotNull; @@ -19,7 +16,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.base.Charsets; import com.google.common.io.CharStreams; -import eu.trentorise.opendata.jackan.CheckedCkanClient; +import eu.trentorise.opendata.jackan.CkanClient; import eu.trentorise.opendata.jackan.exceptions.CkanException; import eu.trentorise.opendata.jackan.exceptions.JackanException; import eu.trentorise.opendata.jackan.internal.org.apache.http.client.fluent.Request; @@ -29,36 +26,34 @@ import eu.trentorise.opendata.jackan.model.CkanOrganization; import eu.trentorise.opendata.jackan.model.CkanResponse; +// TODO: Auto-generated Javadoc /** - * The Class ExtendCkanClient. + * The Class SimpleExtendCkanClient. * * @author Francesco Mangiacrapa at ISTI-CNR (francesco.mangiacrapa@isti.cnr.it) * - * Jun 14, 2019 + * Jun 7, 2019 + * */ -public class ExtendCkanClient extends CheckedCkanClient implements PatchedCkan{ - - private static final Logger logger = LoggerFactory.getLogger(ExtendCkanClient.class); +public class ExtendCkanClient extends CkanClient implements PatchedCkan{ - @Nullable - private static ObjectMapper objectMapper; - private String catalogueURL; private String ckanToken; private int extendTimeout = 120000; //in milliseconds - + @Nullable + private static ObjectMapper objectMapper; + private static final Logger logger = LoggerFactory.getLogger(ExtendCkanClient.class); /** - * Instantiates a new extend ckan client. + * Instantiates a new simple extend ckan client. * - * @param catalogueURL the catalogue url + * @param catalogueURL the catalogue URL */ public ExtendCkanClient(String catalogueURL) { super(catalogueURL); this.catalogueURL = catalogueURL; - } - + /** * Instantiates a new extend ckan client. * @@ -84,7 +79,7 @@ public class ExtendCkanClient extends CheckedCkanClient implements PatchedCkan{ this.ckanToken = ckanToken; this.extendTimeout = timeout; } - + /** * Configures the request. Should work both for GETs and POSTs. @@ -104,38 +99,13 @@ public class ExtendCkanClient extends CheckedCkanClient implements PatchedCkan{ return request; } - /** - * Gets the catalogue url. - * - * @return the catalogueURL - */ - public String getCatalogueURL() { - - return catalogueURL; - } - - - /** - * Gets the ckan token. - * - * @return the ckanToken - */ - @Override - public String getCkanToken() { - - return ckanToken; - } - - /** - * Gets the timeout. - * - * @return the timeout - */ - public int getTimeout() { - return extendTimeout; - } - - /* (non-Javadoc) + /** + * Gets the organization. + * + * @param idOrName the id or name + * @return the organization + */ + /* (non-Javadoc) * @see eu.trentorise.opendata.jackan.CkanClient#getOrganization(java.lang.String) */ public synchronized CkanOrganization getOrganization(String idOrName) { @@ -147,6 +117,12 @@ public class ExtendCkanClient extends CheckedCkanClient implements PatchedCkan{ } + /** + * Gets the group. + * + * @param idOrName the id or name + * @return the group + */ /* (non-Javadoc) * @see eu.trentorise.opendata.jackan.CkanClient#getGroup(java.lang.String) */ @@ -164,7 +140,7 @@ public class ExtendCkanClient extends CheckedCkanClient implements PatchedCkan{ * * @return the object mapper */ - static ObjectMapper getObjectMapper() { + public static ObjectMapper getObjectMapper() { if (objectMapper == null) { objectMapper = new ObjectMapper(); configureObjectMapper(objectMapper); @@ -172,6 +148,15 @@ public class ExtendCkanClient extends CheckedCkanClient implements PatchedCkan{ return objectMapper; } + /** + * Gets the http. + * + * @param the generic type + * @param responseType the response type + * @param path the path + * @param params the params + * @return the http + */ /* (non-Javadoc) * @see org.gcube.datacatalogue.ckanutillibrary.server.patch.PatchedCkan#getHttp(java.lang.Class, java.lang.String, java.lang.Object[]) */ @@ -215,6 +200,13 @@ public class ExtendCkanClient extends CheckedCkanClient implements PatchedCkan{ } + /** + * Calc full url. + * + * @param path the path + * @param params the params + * @return the string + */ /* (non-Javadoc) * @see org.gcube.datacatalogue.ckanutillibrary.server.patch.PatchedCkan#calcFullUrl(java.lang.String, java.lang.Object[]) */ @@ -236,5 +228,26 @@ public class ExtendCkanClient extends CheckedCkanClient implements PatchedCkan{ + Arrays.toString(params), ex); } } + + /** + * Gets the catalogue URL. + * + * @return the catalogue URL + */ + public String getCatalogueURL() { + + return catalogueURL; + } + + /** + * Gets the ckan token. + * + * @return the ckan token + */ + @Override + public String getCkanToken() { + return ckanToken; + } + } diff --git a/target/classes/org/gcube/datacatalogue/ckanutillibrary/ckan/SimpleExtendCkanClient.class b/target/classes/org/gcube/datacatalogue/ckanutillibrary/ckan/SimpleExtendCkanClient.class deleted file mode 100644 index bd71c39..0000000 Binary files a/target/classes/org/gcube/datacatalogue/ckanutillibrary/ckan/SimpleExtendCkanClient.class and /dev/null differ diff --git a/target/classes/org/gcube/datacatalogue/ckanutillibrary/ckan/SimpleExtendCkanClient.java b/target/classes/org/gcube/datacatalogue/ckanutillibrary/ckan/SimpleExtendCkanClient.java deleted file mode 100644 index 93c59cd..0000000 --- a/target/classes/org/gcube/datacatalogue/ckanutillibrary/ckan/SimpleExtendCkanClient.java +++ /dev/null @@ -1,168 +0,0 @@ -package org.gcube.datacatalogue.ckanutillibrary.ckan; - -import static com.google.common.base.Preconditions.checkNotNull; - -import java.io.InputStream; -import java.io.InputStreamReader; -import java.net.URLEncoder; -import java.util.Arrays; - -import javax.annotation.Nullable; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import com.fasterxml.jackson.databind.ObjectMapper; -import com.google.common.base.Charsets; -import com.google.common.io.CharStreams; - -import eu.trentorise.opendata.jackan.CkanClient; -import eu.trentorise.opendata.jackan.exceptions.CkanException; -import eu.trentorise.opendata.jackan.exceptions.JackanException; -import eu.trentorise.opendata.jackan.internal.org.apache.http.client.fluent.Request; -import eu.trentorise.opendata.jackan.internal.org.apache.http.client.fluent.Response; -import eu.trentorise.opendata.jackan.model.CkanGroup; -import eu.trentorise.opendata.jackan.model.CkanOrganization; -import eu.trentorise.opendata.jackan.model.CkanResponse; - - -/** - * The Class SimpleExtendCkanClient. - * - * @author Francesco Mangiacrapa at ISTI-CNR (francesco.mangiacrapa@isti.cnr.it) - * - * Jun 7, 2019 - */ -public class SimpleExtendCkanClient extends CkanClient implements PatchedCkan{ - - - private String catalogueURL; - - @Nullable - private static ObjectMapper objectMapper; - - private static final Logger logger = LoggerFactory.getLogger(SimpleExtendCkanClient.class); - - /** - * Instantiates a new simple extend ckan client. - * - * @param catalogueURL the catalogue URL - */ - public SimpleExtendCkanClient(String catalogueURL) { - super(catalogueURL); - this.catalogueURL = catalogueURL; - } - - /* (non-Javadoc) - * @see eu.trentorise.opendata.jackan.CkanClient#getOrganization(java.lang.String) - */ - public synchronized CkanOrganization getOrganization(String idOrName) { - checkNotNull(idOrName, "Need a valid id or name!"); - logger.info("Patched read organization for id/name: {}", idOrName); - return getHttp(OrganizationResponse.class, "/api/3/action/organization_show", "id", idOrName, - "include_datasets", "false", "include_users", "true").result; - - } - - - /* (non-Javadoc) - * @see eu.trentorise.opendata.jackan.CkanClient#getGroup(java.lang.String) - */ - public synchronized CkanGroup getGroup(String idOrName) { - checkNotNull(idOrName, "Need a valid id or name!"); - logger.info("Patched read group for id/name: {}", idOrName); - return getHttp(GroupResponse.class, "/api/3/action/group_show", "id", idOrName, "include_datasets", - "false", "include_users", "true").result; - } - - - /** - * Retrieves the Jackson object mapper for reading operations. Internally, - * Object mapper is initialized at first call. - * - * @return the object mapper - */ - public static ObjectMapper getObjectMapper() { - if (objectMapper == null) { - objectMapper = new ObjectMapper(); - configureObjectMapper(objectMapper); - } - return objectMapper; - } - - /* (non-Javadoc) - * @see org.gcube.datacatalogue.ckanutillibrary.server.patch.PatchedCkan#getHttp(java.lang.Class, java.lang.String, java.lang.Object[]) - */ - public T getHttp(Class responseType, String path, Object... params) { - - checkNotNull(responseType); - checkNotNull(path); - String fullUrl = calcFullUrl(path, params); - T ckanResponse; - String returnedText; - - try { - - logger.debug("getting {}", fullUrl); - Request request = Request.Get(fullUrl); - configureRequest(request); - Response response = request.execute(); - InputStream stream = response.returnResponse() - .getEntity() - .getContent(); - - try (InputStreamReader reader = new InputStreamReader(stream, Charsets.UTF_8)) { - returnedText = CharStreams.toString(reader); - } - - logger.trace("returnedText {}", returnedText); - } catch (Exception ex) { - throw new CkanException("Error while performing GET. Request url was: " + fullUrl, this, ex); - } - try { - ckanResponse = getObjectMapper().readValue(returnedText, responseType); - } catch (Exception ex) { - throw new CkanException( - "Couldn't interpret json returned by the server! Returned text was: " + returnedText, this, ex); - } - - if (!ckanResponse.isSuccess()) { - throwCkanException("Error while performing GET. Request url was: " + fullUrl, ckanResponse); - } - return ckanResponse; - } - - - /* (non-Javadoc) - * @see org.gcube.datacatalogue.ckanutillibrary.server.patch.PatchedCkan#calcFullUrl(java.lang.String, java.lang.Object[]) - */ - public String calcFullUrl(String path, Object[] params) { - checkNotNull(path); - - try { - StringBuilder sb = new StringBuilder().append(catalogueURL) - .append(path); - for (int i = 0; i < params.length; i += 2) { - sb.append(i == 0 ? "?" : "&") - .append(URLEncoder.encode(params[i].toString(), "UTF-8")) - .append("=") - .append(URLEncoder.encode(params[i + 1].toString(), "UTF-8")); - } - return sb.toString(); - } catch (Exception ex) { - throw new JackanException("Error while building url to perform GET! \n path: " + path + " \n params: " - + Arrays.toString(params), ex); - } - } - - /** - * Gets the catalogue url. - * - * @return the catalogueURL - */ - public String getCatalogueURL() { - - return catalogueURL; - } - -} diff --git a/target/classes/org/gcube/datacatalogue/ckanutillibrary/db/DBCaller.class b/target/classes/org/gcube/datacatalogue/ckanutillibrary/db/DBCaller.class index 6147ada..5cb155f 100644 Binary files a/target/classes/org/gcube/datacatalogue/ckanutillibrary/db/DBCaller.class and b/target/classes/org/gcube/datacatalogue/ckanutillibrary/db/DBCaller.class differ diff --git a/target/classes/org/gcube/datacatalogue/ckanutillibrary/db/DBCaller.java b/target/classes/org/gcube/datacatalogue/ckanutillibrary/db/DBCaller.java index f989172..1e1532d 100644 --- a/target/classes/org/gcube/datacatalogue/ckanutillibrary/db/DBCaller.java +++ b/target/classes/org/gcube/datacatalogue/ckanutillibrary/db/DBCaller.java @@ -95,7 +95,7 @@ public class DBCaller { */ public Map getGroupsByUserFromDB(String username){ checkNotNull(username); - LOG.debug("Get groups by user called"); + LOG.debug("Get groups by user called for username: "+username); //couples (groups id, capacity) of the user in the group @@ -129,6 +129,8 @@ public class DBCaller { }finally{ closeConnection(connection); } + + LOG.debug("returning map: "+toReturn); return toReturn; } @@ -140,7 +142,7 @@ public class DBCaller { * @return the user id for username */ private String getUserIdForUsername(String username) { - LOG.debug("Get user id for username called"); + LOG.debug("Get user id for username called for username: "+username); Connection connection = null; String userId = null; @@ -150,7 +152,7 @@ public class DBCaller { String selQuery = "SELECT \"id\" FROM \"public\".\"user\" WHERE \"name\"=?;"; - LOG.info("Performing query: "+selQuery); + LOG.debug("Performing query: "+selQuery); PreparedStatement preparedStatement = connection.prepareStatement(selQuery); preparedStatement.setString(1, username); @@ -213,6 +215,8 @@ public class DBCaller { }finally{ closeConnection(connection); } + + LOG.debug("returning map: "+toReturn); return toReturn; } diff --git a/target/classes/org/gcube/datacatalogue/ckanutillibrary/gcat/GCatCaller.class b/target/classes/org/gcube/datacatalogue/ckanutillibrary/gcat/GCatCaller.class index 19c51a3..1cda543 100644 Binary files a/target/classes/org/gcube/datacatalogue/ckanutillibrary/gcat/GCatCaller.class and b/target/classes/org/gcube/datacatalogue/ckanutillibrary/gcat/GCatCaller.class differ diff --git a/target/classes/org/gcube/datacatalogue/ckanutillibrary/gcat/GCatCaller.java b/target/classes/org/gcube/datacatalogue/ckanutillibrary/gcat/GCatCaller.java index c14eca7..82cf4c6 100644 --- a/target/classes/org/gcube/datacatalogue/ckanutillibrary/gcat/GCatCaller.java +++ b/target/classes/org/gcube/datacatalogue/ckanutillibrary/gcat/GCatCaller.java @@ -2,19 +2,11 @@ package org.gcube.datacatalogue.ckanutillibrary.gcat; import java.io.IOException; import java.net.MalformedURLException; -import java.util.List; -import org.gcube.datacatalogue.ckanutillibrary.ckan.SimpleExtendCkanClient; import org.gcube.gcat.client.Item; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.fasterxml.jackson.core.JsonParseException; -import com.fasterxml.jackson.databind.JsonMappingException; - -import eu.trentorise.opendata.jackan.model.CkanDataset; -import eu.trentorise.opendata.jackan.model.CkanLicense; - // TODO: Auto-generated Javadoc /** * The Class GCatCaller. @@ -24,18 +16,17 @@ import eu.trentorise.opendata.jackan.model.CkanLicense; */ public class GCatCaller { - private SimpleExtendCkanClient jackanCkanClient; - private String catalogueURL; private static final Logger LOG = LoggerFactory.getLogger(GCatCaller.class); /** * Instantiates a new g cat caller. + * + * @param catalogueURL the catalogue URL */ public GCatCaller(String catalogueURL) { this.catalogueURL = catalogueURL; - this.jackanCkanClient = new SimpleExtendCkanClient(catalogueURL); } @@ -43,27 +34,27 @@ public class GCatCaller { * Gets the dataset for name. * * @param datasetIdOrName the dataset id or name - * @return the dataset for name - * @throws JsonParseException the json parse exception - * @throws JsonMappingException the json mapping exception + * @return the jsonValue + * @throws MalformedURLException * @throws IOException Signals that an I/O exception has occurred. */ - public CkanDataset getDatasetForName(String datasetIdOrName) throws JsonParseException, JsonMappingException, IOException { + public String getDatasetForName(String datasetIdOrName) throws MalformedURLException { LOG.debug("Get dataset for name "+datasetIdOrName+ "called"); - String json = new Item().read(datasetIdOrName); - return SimpleExtendCkanClient.getObjectMapper().readValue(json, CkanDataset.class); + return new Item().read(datasetIdOrName); } - - - /** - * TODO gCAT missing - * @return - * @throws MalformedURLException - */ - public List getLicenseList() { - LOG.debug("Get linces list called"); - return jackanCkanClient.getLicenseList(); + + /** + * Creates the dataset. + * + * @param jsonDataset the json dataset + * @param socialPost if true sends the social post + * @return the jsonValue + * @throws MalformedURLException the malformed URL exception + */ + public String createDataset(String jsonDataset, boolean socialPost) throws MalformedURLException { + LOG.debug("Create dataset called"); + return new Item().create(jsonDataset,socialPost); } } diff --git a/target/classes/org/gcube/datacatalogue/ckanutillibrary/server/DataCatalogue.class b/target/classes/org/gcube/datacatalogue/ckanutillibrary/server/DataCatalogue.class index 8b83f35..02fc08f 100644 Binary files a/target/classes/org/gcube/datacatalogue/ckanutillibrary/server/DataCatalogue.class and b/target/classes/org/gcube/datacatalogue/ckanutillibrary/server/DataCatalogue.class differ diff --git a/target/classes/org/gcube/datacatalogue/ckanutillibrary/server/DataCatalogue.java b/target/classes/org/gcube/datacatalogue/ckanutillibrary/server/DataCatalogue.java index e8b5494..78cece6 100644 --- a/target/classes/org/gcube/datacatalogue/ckanutillibrary/server/DataCatalogue.java +++ b/target/classes/org/gcube/datacatalogue/ckanutillibrary/server/DataCatalogue.java @@ -39,67 +39,6 @@ public interface DataCatalogue { */ String getUnencryptedUrlFromDatasetIdOrName(String datasetIdOrName); - /** - * Create a dataset with those information. The method allows to have multiple - * values for the same custom field key. NOTE: unfortunately java doesn't - * support overload in java interface methods (that's way I cannot use the same - * name for the method) - * - * @param title the title - * @param name (unique identifier) - * @param organizationNameOrId the organization name or id - * @param author the author - * @param authorMail the author mail - * @param maintainer the maintainer - * @param maintainerMail the maintainer mail - * @param version the version - * @param description the description - * @param licenseId the license id - * @param tags the tags - * @param customFields the custom fields - * @param resources the resources - * @param setPublic (manage visibility: Admin role is needed) - * @return the id of the dataset on success, null otherwise - * @throws Exception the exception - */ - String createCKanDatasetMultipleCustomFields(String title, String name, String organizationNameOrId, String author, - String authorMail, String maintainer, String maintainerMail, long version, String description, - String licenseId, List tags, Map> customFields, List resources, - boolean setPublic) throws Exception; - - /** - * Patch a product with product id productId by using the couples in - * customFieldsToChange. NOTE: only the specified custom fields will be changed. - * If a custom field with a given key already exists, and removeOld is set to - * false, the new values are added at the end of the list. Otherwise they are - * lost. - * - * @param productId the product id - * @param customFieldsToChange the custom fields to change - * @param removeOld the remove old - * @return true, if successful - */ - boolean patchProductCustomFields(String productId, Map> customFieldsToChange, - boolean removeOld); - - /** - * Add a resource described by the bean to the dataset id into - * resource.datasetId - * - * @param resource the resource - * @return String the id of the resource on success, null otherwise - * @throws Exception the exception - */ - String addResourceToDataset(ResourceBean resource) throws Exception; - - /** - * Remove the resource with id resourceId from the dataset in which it is. - * - * @param resourceId the resource id - * @return true on success, false otherwise. - */ - boolean deleteResourceFromDataset(String resourceId); - /** * Create a dataset with those information. * @@ -201,4 +140,86 @@ public interface DataCatalogue { */ Map> getUserRoleByOrganization(String username); + + + + + //WRITE OPERATIONS + + + + + + /** + * Set searchable field + * The field, if set to true, allows organization's users to view private products also in groups + * @param datasetId the id of the dataset to update + * @param searchable the value to assign to the field + * @return true if the field is set, false otherwise + */ + boolean setSearchableField(String datasetId, boolean searchable); + + + /** + * Create a dataset with those information. The method allows to have multiple + * values for the same custom field key. NOTE: unfortunately java doesn't + * support overload in java interface methods (that's way I cannot use the same + * name for the method) + * + * @param the username that is submitting the publishing request + * @param title the title + * @param name (unique identifier) + * @param organizationNameOrId the organization name or id + * @param author the author + * @param authorMail the author mail + * @param maintainer the maintainer + * @param maintainerMail the maintainer mail + * @param version the version + * @param description the description + * @param licenseId the license id + * @param tags the tags + * @param customFields the custom fields + * @param resources the resources + * @param setPublic (manage visibility: Admin role is needed) + * @return the id of the dataset on success, null otherwise + * @throws Exception the exception + */ + String createCKanDatasetMultipleCustomFields(String username, String title, String name, String organizationNameOrId, String author, + String authorMail, String maintainer, String maintainerMail, long version, String description, + String licenseId, List tags, Map> customFields, List resources, + boolean setPublic) throws Exception; + + /** + * Patch a product with product id productId by using the couples in + * customFieldsToChange. NOTE: only the specified custom fields will be changed. + * If a custom field with a given key already exists, and removeOld is set to + * false, the new values are added at the end of the list. Otherwise they are + * lost. + * + * @param productId the product id + * @param customFieldsToChange the custom fields to change + * @param removeOld the remove old + * @return true, if successful + */ + boolean patchProductCustomFields(String productId, Map> customFieldsToChange, + boolean removeOld); + + /** + * Add a resource described by the bean to the dataset id into + * resource.datasetId + * + * @param resource the resource + * @return String the id of the resource on success, null otherwise + * @throws Exception the exception + */ + String addResourceToDataset(ResourceBean resource) throws Exception; + + /** + * Remove the resource with id resourceId from the dataset in which it is. + * + * @param resourceId the resource id + * @return true on success, false otherwise. + */ + boolean deleteResourceFromDataset(String resourceId); + } diff --git a/target/classes/org/gcube/datacatalogue/ckanutillibrary/server/DataCatalogueImpl.class b/target/classes/org/gcube/datacatalogue/ckanutillibrary/server/DataCatalogueImpl.class index 8f19b4d..66e6004 100644 Binary files a/target/classes/org/gcube/datacatalogue/ckanutillibrary/server/DataCatalogueImpl.class and b/target/classes/org/gcube/datacatalogue/ckanutillibrary/server/DataCatalogueImpl.class differ diff --git a/target/classes/org/gcube/datacatalogue/ckanutillibrary/server/DataCatalogueImpl.java b/target/classes/org/gcube/datacatalogue/ckanutillibrary/server/DataCatalogueImpl.java index 099f1e9..f635940 100644 --- a/target/classes/org/gcube/datacatalogue/ckanutillibrary/server/DataCatalogueImpl.java +++ b/target/classes/org/gcube/datacatalogue/ckanutillibrary/server/DataCatalogueImpl.java @@ -6,15 +6,18 @@ import static com.google.common.base.Preconditions.checkNotNull; import java.net.URLEncoder; import java.util.ArrayList; import java.util.HashMap; +import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.Map.Entry; import java.util.concurrent.ConcurrentHashMap; import org.gcube.common.authorization.library.provider.SecurityTokenProvider; import org.gcube.common.scope.impl.ScopeBean; import org.gcube.common.scope.impl.ScopeBean.Type; +import org.gcube.datacatalogue.ckanutillibrary.ckan.DirectCkanCaller; import org.gcube.datacatalogue.ckanutillibrary.ckan.ExtendCkanClient; -import org.gcube.datacatalogue.ckanutillibrary.ckan.SimpleExtendCkanClient; +import org.gcube.datacatalogue.ckanutillibrary.ckan.MarshUnmarshCkanObject; import org.gcube.datacatalogue.ckanutillibrary.db.DBCaller; import org.gcube.datacatalogue.ckanutillibrary.gcat.GCatCaller; import org.gcube.datacatalogue.ckanutillibrary.server.DataCatalogueRunningCluster.ACCESS_LEVEL_TO_CATALOGUE_PORTLET; @@ -38,7 +41,13 @@ import eu.trentorise.opendata.jackan.model.CkanDataset; import eu.trentorise.opendata.jackan.model.CkanGroup; import eu.trentorise.opendata.jackan.model.CkanLicense; import eu.trentorise.opendata.jackan.model.CkanOrganization; +import eu.trentorise.opendata.jackan.model.CkanPair; +import eu.trentorise.opendata.jackan.model.CkanResource; +import eu.trentorise.opendata.jackan.model.CkanTag; import eu.trentorise.opendata.jackan.model.CkanUser; +import net.htmlparser.jericho.Renderer; +import net.htmlparser.jericho.Segment; +import net.htmlparser.jericho.Source; public class DataCatalogueImpl implements DataCatalogue { @@ -63,6 +72,7 @@ public class DataCatalogueImpl implements DataCatalogue { public Map mapAccessURLToCatalogue; + private static final String CATALOGUE_TAB_ENDING_URL = "/catalogue"; // gCat client @@ -72,10 +82,13 @@ public class DataCatalogueImpl implements DataCatalogue { private DBCaller dbCaller; // ckan client - private SimpleExtendCkanClient ckanCaller; + private ExtendCkanClient ckanCaller; // hashmap for ckan api keys private ConcurrentHashMap apiKeysMap; + + //http ckan caller + private DirectCkanCaller directCkanCaller; // apikey bean expires after X minutes in the above map private static final int EXPIRE_KEY_TIME = 60 * 60 * 1000; @@ -97,7 +110,7 @@ public class DataCatalogueImpl implements DataCatalogue { CKAN_DB_USER = runningInstance.getDataBaseUser().trim(); CKAN_DB_PASSWORD = runningInstance.getDataBasePassword().trim(); - //CKAN_TOKEN_SYS = runningInstance.getSysAdminToken().trim(); + CKAN_TOKEN_SYS = runningInstance.getSysAdminToken().trim(); CKAN_EMAIL = runningInstance.getEmailCatalogue().trim(); CKAN_CATALOGUE_URL = runningInstance.getDataCatalogueUrl().get(0).trim(); @@ -113,17 +126,13 @@ public class DataCatalogueImpl implements DataCatalogue { // build the clients gCatCaller = new GCatCaller(CKAN_CATALOGUE_URL); - dbCaller = new DBCaller(CKAN_DB_URL, CKAN_DB_PORT, CKAN_DB_NAME, CKAN_DB_USER, CKAN_DB_PASSWORD); - - ckanCaller = new SimpleExtendCkanClient(CKAN_CATALOGUE_URL); - + ckanCaller = new ExtendCkanClient(CKAN_CATALOGUE_URL); + directCkanCaller = new DirectCkanCaller(CKAN_CATALOGUE_URL); // init map apiKeysMap = new ConcurrentHashMap(); - // save the context CONTEXT = scope; - // extended roles extendRoleInOrganization = runningInstance.getExtendRoleInOrganization(); } @@ -228,7 +237,8 @@ public class DataCatalogueImpl implements DataCatalogue { String authzToken = SecurityTokenProvider.instance.get(); if(authzToken!=null && !authzToken.isEmpty()) { LOG.info("gcube-token found. Calling the gCat client"); - return gCatCaller.getDatasetForName(datasetId); + String jsonDataset = gCatCaller.getDatasetForName(datasetId); + return MarshUnmarshCkanObject.toCkanDataset(jsonDataset); } LOG.info("No api-key or gcube-token found. Calling Ckan Client without API-KEY"); @@ -272,13 +282,7 @@ public class DataCatalogueImpl implements DataCatalogue { return url; } - public String createCKanDatasetMultipleCustomFields(String title, String name, String organizationNameOrId, - String author, String authorMail, String maintainer, String maintainerMail, long version, - String description, String licenseId, List tags, Map> customFields, - List resources, boolean setPublic) throws Exception { - return null; - } - + @Override public Map> getUserRoleByGroup(String username) { LOG.info("Get user role by group called. The username is: "+username); @@ -328,6 +332,7 @@ public class DataCatalogueImpl implements DataCatalogue { HashMap subMap = new HashMap(); subMap.put(org, partialResult.get(orgID)); toReturn.put(orgID, subMap); + LOG.debug("For organisation: " + org.getName() + ", the user "+username+" has role: "+subMap); } LOG.debug("Returning map " + toReturn); @@ -399,10 +404,8 @@ public class DataCatalogueImpl implements DataCatalogue { // checks checkNotNull(username); - // in order to avoid errors, the username is always converted String ckanUsername = CatalogueUtilMethods.fromUsernameToCKanUsername(username); - // list to return List toReturn = new ArrayList(); @@ -413,12 +416,11 @@ public class DataCatalogueImpl implements DataCatalogue { // iterate over them for (CkanOrganization ckanOrganization : organizations) { - // get the list of users in it (if you try ckanOrganization.getUsers() it returns null.. maybe a bug TODO) List users = ckanCaller.getOrganization(ckanOrganization.getName()).getUsers(); - // check if the current user is among them for (CkanUser ckanUser : users) { + if(ckanUser.getName().equals(ckanUsername)){ LOG.debug("User " + ckanUsername + " is into " + ckanOrganization.getName()); @@ -440,10 +442,8 @@ public class DataCatalogueImpl implements DataCatalogue { // checks checkNotNull(username); - // in order to avoid errors, the username is always converted String ckanUsername = CatalogueUtilMethods.fromUsernameToCKanUsername(username); - // list to return List toReturn = new ArrayList(); @@ -459,6 +459,7 @@ public class DataCatalogueImpl implements DataCatalogue { // check if the current user is among them for (CkanUser ckanUser : users) { + if(ckanUser.getName().equals(ckanUsername)){ LOG.debug("User " + ckanUsername + " is into " + ckanGroup.getName()); @@ -513,15 +514,22 @@ public class DataCatalogueImpl implements DataCatalogue { return null; } - - - - - - - - - + @Override + public boolean existProductWithNameOrId(String nameOrId) { + + checkNotNull(nameOrId); + checkArgument(!nameOrId.isEmpty()); + + try{ + ExtendCkanClient client = new ExtendCkanClient(CKAN_CATALOGUE_URL, CKAN_TOKEN_SYS); + CkanDataset product = client.getDataset(nameOrId); + return product != null; + }catch(Exception e){ + LOG.debug("A dataset with name " + nameOrId + " doesn't exist"); + return false; + } + } + /* * @@ -546,6 +554,163 @@ public class DataCatalogueImpl implements DataCatalogue { * */ + @Override + public String createCKanDatasetMultipleCustomFields(String username, String title, String name, String organizationNameOrId, + String author, String authorMail, String maintainer, String maintainerMail, long version, + String description, String licenseId, List tags, Map> customFieldsMultiple, + List resources, boolean setPublic) throws Exception { + + // delegate the private method + return createCkanDatasetBody(username, title, name, organizationNameOrId, author, authorMail, maintainer, maintainerMail, + version, description, licenseId, tags, null, customFieldsMultiple, resources, setPublic); + } + + // the body of the actual dataset creation methods + private String createCkanDatasetBody(String username, String title, String name, String organizationNameOrId, String author, + String authorMail, String maintainer, String maintainerMail, long version, String description, + String licenseId, List tags, Map customFields, + Map> customFieldsMultipleValues, List resources, boolean setPublic) + throws Exception { + + // checks (minimum) + checkNotNull(username); + checkNotNull(organizationNameOrId); + checkArgument(!organizationNameOrId.isEmpty()); + checkArgument(!(title == null && name == null || title.isEmpty() && name.isEmpty()), "Name and Title cannot be empty/null at the same time!"); + + LOG.debug("Request for dataset creation"); + + //ExtendCkanClient client = new ExtendCkanClient(CKAN_CATALOGUE_URL, apiKey); + + //String ckanUsername = getUserFromApiKey(apiKey).getName(); + CkanDataset dataset = new CkanDataset(); + + // in order to avoid errors, the username is always converted + String ckanUsername = CatalogueUtilMethods.fromUsernameToCKanUsername(username); + + String nameToUse = name; + if(nameToUse == null) + nameToUse = CatalogueUtilMethods.fromProductTitleToName(title); + + LOG.debug("Name of the dataset is going to be " + nameToUse + ". Title is going to be " + title); + + dataset.setName(nameToUse); + dataset.setTitle(title); + +// CkanOrganization orgOwner = client.getOrganization(organizationNameOrId); +// dataset.setOwnerOrg(orgOwner.getId()); + dataset.setAuthor(author); + dataset.setAuthorEmail(authorMail); + dataset.setMaintainer(maintainer); + dataset.setMaintainerEmail(maintainerMail); + dataset.setVersion(String.valueOf(version)); + + // description must be escaped + if(description != null && !description.isEmpty()){ + Source descriptionEscaped = new Source(description); + Segment htmlSeg = new Segment(descriptionEscaped, 0, descriptionEscaped.length()); + Renderer htmlRend = new Renderer(htmlSeg); + dataset.setNotes(htmlRend.toString()); + + LOG.debug("Description escaped " + htmlRend.toString()); + } + + dataset.setLicenseId(licenseId); + + // set the tags, if any + if(tags != null && !tags.isEmpty()){ + List ckanTags = new ArrayList(tags.size()); + for (String stringTag : tags) { + ckanTags.add(new CkanTag(stringTag)); + } + dataset.setTags(ckanTags); + } + + // set the custom fields, if any + List extras = new ArrayList(); + + if(customFields != null && !customFields.isEmpty()){ + + Iterator> iterator = customFields.entrySet().iterator(); + + while (iterator.hasNext()) { + Map.Entry entry = iterator.next(); + extras.add(new CkanPair(entry.getKey(), entry.getValue())); + } + + }else if(customFieldsMultipleValues != null && !customFieldsMultipleValues.isEmpty()){ + + Iterator>> iterator = customFieldsMultipleValues.entrySet().iterator(); + while (iterator.hasNext()) { + Map.Entry> entry = iterator.next(); + List valuesForEntry = entry.getValue(); + for (String value : valuesForEntry) { + extras.add(new CkanPair(entry.getKey(), value)); + } + } + } + + dataset.setExtras(extras); + + // check if we need to add the resources + if(resources != null && !resources.isEmpty()){ + + LOG.debug("We need to add resources to the dataset"); + + try{ + List resourcesCkan = new ArrayList(); + for(ResourceBean resource: resources){ + + LOG.debug("Going to add resource described by " + resource); + CkanResource newResource = new CkanResource(); + newResource.setDescription(resource.getDescription()); + newResource.setId(resource.getId()); + newResource.setUrl(resource.getUrl()); + newResource.setName(resource.getName()); + newResource.setMimetype(resource.getMimeType()); + newResource.setFormat(resource.getMimeType()); + newResource.setOwner(ckanUsername); + resourcesCkan.add(newResource); + } + + // add to the dataset + dataset.setResources(resourcesCkan); + + }catch(Exception e){ + LOG.error("Unable to add those resources to the dataset", e); + } + + } + + // try to create + String jsonValueDataset = MarshUnmarshCkanObject.toJsonValueDataset(dataset); + LOG.debug("Marshalling dataset is: " + jsonValueDataset); + jsonValueDataset = gCatCaller.createDataset(jsonValueDataset,true); + LOG.debug("Created dataset is: " + jsonValueDataset); + + if(jsonValueDataset != null){ + CkanDataset toCkanDataset = MarshUnmarshCkanObject.toCkanDataset(jsonValueDataset); + LOG.debug("Dataset with name " + toCkanDataset.getName() + " has been created. Setting visibility"); + + // set visibility + boolean visibilitySet = directCkanCaller.setDatasetPrivate( + !setPublic, // swap to private + toCkanDataset.getOrganization().getId(), + toCkanDataset.getId(), + CKAN_TOKEN_SYS); // use sysadmin api key to be sure it will be set + + LOG.info("Was visibility set to " + (setPublic ? "public" : "private") + "? " + visibilitySet); + + // set searchable to true if dataset visibility is private + if(!setPublic){ // (swap to private) + boolean searchableSet = setSearchableField(toCkanDataset.getId(), true); + LOG.info("Was searchable set to True? " + searchableSet); + } + return toCkanDataset.getId(); + } + + return null; + } public boolean patchProductCustomFields(String productId, Map> customFieldsToChange, boolean removeOld) { @@ -560,9 +725,11 @@ public class DataCatalogueImpl implements DataCatalogue { return false; } - public boolean existProductWithNameOrId(String nameOrId) { - return false; + @Override + public boolean setSearchableField(String datasetId, boolean searchable) { + return directCkanCaller.setSearchableField(datasetId, searchable, CKAN_TOKEN_SYS); } + }