on going on migration to gCat

src/main/java/org/gcube/datacatalogue/ckanutillibrary/ckan/DirectCkanCaller.java

@@ -0,0 +1,140 @@
+package org.gcube.datacatalogue.ckanutillibrary.ckan;
+
+import static com.google.common.base.Preconditions.checkArgument;
+import static com.google.common.base.Preconditions.checkNotNull;
+
+import org.json.simple.JSONArray;
+import org.json.simple.JSONObject;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import eu.trentorise.opendata.jackan.internal.org.apache.http.HttpResponse;
+import eu.trentorise.opendata.jackan.internal.org.apache.http.HttpStatus;
+import eu.trentorise.opendata.jackan.internal.org.apache.http.client.methods.HttpPost;
+import eu.trentorise.opendata.jackan.internal.org.apache.http.entity.StringEntity;
+import eu.trentorise.opendata.jackan.internal.org.apache.http.impl.client.CloseableHttpClient;
+import eu.trentorise.opendata.jackan.internal.org.apache.http.impl.client.HttpClientBuilder;
+
+
+// TODO: Auto-generated Javadoc
+/**
+ * The Class DirectCkanCaller.
+ *
+ * @author Francesco Mangiacrapa at ISTI-CNR Pisa (Italy)
+ * Jun 3, 2020
+ */
+public class DirectCkanCaller {
+	
+	private static final String PATH_PACKAGE_PATCH = "/api/3/action/package_patch";
+	public final static String PATH_SET_PRIVATE_DATASET = "/api/3/action/bulk_update_private";
+	public final static String PATH_SET_PUBLIC_DATASET = "/api/3/action/bulk_update_public";
+	private String catalogueURL;
+	
+	private static final Logger LOG = LoggerFactory.getLogger(DirectCkanCaller.class);
+	
+	
+	/**
+	 * Instantiates a new direct ckan caller.
+	 *
+	 * @param catalogueURL the catalogue URL
+	 */
+	public DirectCkanCaller(String catalogueURL){
+		this.catalogueURL = catalogueURL;
+	}
+	
+	/**
+	 * Set dataset private/public.
+	 *
+	 * @param priv the priv
+	 * @param organizationId (NOTE: The ID, not the name!)
+	 * @param datasetId (NOTE: The ID, not the name!)
+	 * @param apiKey the user's api key
+	 * @return true on success, false otherwise
+	 */
+	public boolean setDatasetPrivate(boolean priv, String organizationId,
+			String datasetId, String apiKey) {
+
+		// checks
+		checkNotNull(organizationId);
+		checkNotNull(apiKey);
+		checkNotNull(datasetId);
+		checkArgument(!apiKey.isEmpty());
+		checkArgument(!datasetId.isEmpty());
+		checkArgument(!organizationId.isEmpty());
+
+		JSONObject obj = new JSONObject();
+		obj.put("org_id", organizationId);
+
+		JSONArray array = new JSONArray();
+		array.add(datasetId);
+		obj.put("datasets", array);
+
+		try(CloseableHttpClient httpClient = HttpClientBuilder.create().build();) {
+			HttpPost request;
+
+			if(priv)
+				request = new HttpPost(catalogueURL + PATH_SET_PRIVATE_DATASET);
+			else
+				request = new HttpPost(catalogueURL + PATH_SET_PUBLIC_DATASET);
+
+			LOG.info("Excuting request for making dataset with id " + datasetId + " " + (priv? "private" : "public"));
+
+			request.addHeader("Authorization", apiKey);
+			StringEntity params = new StringEntity(obj.toJSONString());
+			request.setEntity(params);
+			HttpResponse response = httpClient.execute(request);
+
+			LOG.info("Response code is " + response.getStatusLine().getStatusCode() + " and response message is " + response.getStatusLine().getReasonPhrase());
+
+			if(response.getStatusLine().getStatusCode() == HttpStatus.SC_OK)
+				return true;
+
+		}catch (Exception ex) {
+			LOG.error("Error while trying to set private the dataset ", ex);
+		}
+
+
+		return false;
+	}
+	
+	/**
+	 * Sets the searchable field.
+	 *
+	 * @param datasetId the dataset id
+	 * @param searchable the searchable
+	 * @return true, if successful
+	 */
+	public boolean setSearchableField(String datasetId, boolean searchable, String apiKey) {
+
+		// checks
+		checkNotNull(datasetId);
+		checkArgument(!datasetId.isEmpty());
+		String searchableAsString = searchable ? "True" : "False";
+
+		// Patch package path
+		String patchPackage = catalogueURL + PATH_PACKAGE_PATCH;
+
+		JSONObject obj = new JSONObject();
+		obj.put("id", datasetId);
+		obj.put("searchable", searchableAsString);
+
+		try(CloseableHttpClient httpClient = HttpClientBuilder.create().build();) {
+			HttpPost request = new HttpPost(patchPackage);
+			request.addHeader("Authorization", apiKey);
+			StringEntity params = new StringEntity(obj.toJSONString());
+			request.setEntity(params);
+			HttpResponse response = httpClient.execute(request);
+			LOG.debug("Response code is " + response.getStatusLine().getStatusCode() + " and response message is " + response.getStatusLine().getReasonPhrase());
+
+			if(response.getStatusLine().getStatusCode() == HttpStatus.SC_OK)
+				return true;
+
+		}catch (Exception ex) {
+			LOG.error("Error while trying to set searchable the dataset ", ex);
+		}
+
+		return false;
+	}
+	
+
+}

src/main/java/org/gcube/datacatalogue/ckanutillibrary/ckan/ExtendCkanClient.java

@@ -1,6 +1,3 @@
-/**
- *
- */
 package org.gcube.datacatalogue.ckanutillibrary.ckan;
 
 import static com.google.common.base.Preconditions.checkNotNull;
@@ -19,7 +16,7 @@ import com.fasterxml.jackson.databind.ObjectMapper;
 import com.google.common.base.Charsets;
 import com.google.common.io.CharStreams;
 
-import eu.trentorise.opendata.jackan.CheckedCkanClient;
+import eu.trentorise.opendata.jackan.CkanClient;
 import eu.trentorise.opendata.jackan.exceptions.CkanException;
 import eu.trentorise.opendata.jackan.exceptions.JackanException;
 import eu.trentorise.opendata.jackan.internal.org.apache.http.client.fluent.Request;
@@ -29,36 +26,34 @@ import eu.trentorise.opendata.jackan.model.CkanOrganization;
 import eu.trentorise.opendata.jackan.model.CkanResponse;
 
 
+// TODO: Auto-generated Javadoc
 /**
- * The Class ExtendCkanClient.
+ * The Class SimpleExtendCkanClient.
  *
  * @author Francesco Mangiacrapa at ISTI-CNR (francesco.mangiacrapa@isti.cnr.it)
  * 
- * Jun 14, 2019
+ * Jun 7, 2019
+ * 
  */
-public class ExtendCkanClient extends CheckedCkanClient  implements PatchedCkan{
+public class ExtendCkanClient extends CkanClient implements PatchedCkan{
-
-	private static final Logger logger = LoggerFactory.getLogger(ExtendCkanClient.class);
 	
-	@Nullable
-	private static ObjectMapper objectMapper;
-
 	private String catalogueURL;
 	private String ckanToken;
     private int extendTimeout = 120000; //in milliseconds
-
+	@Nullable
+	private static ObjectMapper objectMapper;
+	private static final Logger logger = LoggerFactory.getLogger(ExtendCkanClient.class);
 
 	/**
-	 * Instantiates a new extend ckan client.
+	 * Instantiates a new simple extend ckan client.
 	 *
-	 * @param catalogueURL the catalogue url
+	 * @param catalogueURL the catalogue URL
 	 */
 	public ExtendCkanClient(String catalogueURL) {
 		super(catalogueURL);
 		this.catalogueURL = catalogueURL;
-		
 	}
-
+	
     /**
      * Instantiates a new extend ckan client.
      *
@@ -84,7 +79,7 @@ public class ExtendCkanClient extends CheckedCkanClient  implements PatchedCkan{
         this.ckanToken = ckanToken;
         this.extendTimeout = timeout;
     }
-
+    
 
     /**
      * Configures the request. Should work both for GETs and POSTs.
@@ -104,38 +99,13 @@ public class ExtendCkanClient extends CheckedCkanClient  implements PatchedCkan{
         return request;
     }
 
-	/**
+    /**
-	 * Gets the catalogue url.
+     * Gets the organization.
-	 *
+     *
-	 * @return the catalogueURL
+     * @param idOrName the id or name
-	 */
+     * @return the organization
-	public String getCatalogueURL() {
+     */
-
+    /* (non-Javadoc)
-		return catalogueURL;
-	}
-
-
-	/**
-	 * Gets the ckan token.
-	 *
-	 * @return the ckanToken
-	 */
-	@Override
-	public String getCkanToken() {
-
-		return ckanToken;
-	}
-
-	/**
-	 * Gets the timeout.
-	 *
-	 * @return the timeout
-	 */
-	public int getTimeout() {
-		return extendTimeout;
-	}
-
-	  /* (non-Javadoc)
      * @see eu.trentorise.opendata.jackan.CkanClient#getOrganization(java.lang.String)
      */
     public synchronized CkanOrganization getOrganization(String idOrName) {
@@ -147,6 +117,12 @@ public class ExtendCkanClient extends CheckedCkanClient  implements PatchedCkan{
     }
     
 
+    /**
+     * Gets the group.
+     *
+     * @param idOrName the id or name
+     * @return the group
+     */
     /* (non-Javadoc)
      * @see eu.trentorise.opendata.jackan.CkanClient#getGroup(java.lang.String)
      */
@@ -164,7 +140,7 @@ public class ExtendCkanClient extends CheckedCkanClient  implements PatchedCkan{
      *
      * @return the object mapper
      */
-    static ObjectMapper getObjectMapper() {
+    public static ObjectMapper getObjectMapper() {
         if (objectMapper == null) {
             objectMapper = new ObjectMapper();
             configureObjectMapper(objectMapper);
@@ -172,6 +148,15 @@ public class ExtendCkanClient extends CheckedCkanClient  implements PatchedCkan{
         return objectMapper;
     }
     
+    /**
+     * Gets the http.
+     *
+     * @param <T> the generic type
+     * @param responseType the response type
+     * @param path the path
+     * @param params the params
+     * @return the http
+     */
     /* (non-Javadoc)
      * @see org.gcube.datacatalogue.ckanutillibrary.server.patch.PatchedCkan#getHttp(java.lang.Class, java.lang.String, java.lang.Object[])
      */
@@ -215,6 +200,13 @@ public class ExtendCkanClient extends CheckedCkanClient  implements PatchedCkan{
     }
 
 
+    /**
+     * Calc full url.
+     *
+     * @param path the path
+     * @param params the params
+     * @return the string
+     */
     /* (non-Javadoc)
      * @see org.gcube.datacatalogue.ckanutillibrary.server.patch.PatchedCkan#calcFullUrl(java.lang.String, java.lang.Object[])
      */
@@ -236,5 +228,26 @@ public class ExtendCkanClient extends CheckedCkanClient  implements PatchedCkan{
                     + Arrays.toString(params), ex);
         }
     }
+    
 
+	/**
+	 * Gets the catalogue URL.
+	 *
+	 * @return the catalogue URL
+	 */
+	public String getCatalogueURL() {
+
+		return catalogueURL;
+	}
+	
+	/**
+	 * Gets the ckan token.
+	 *
+	 * @return the ckan token
+	 */
+	@Override
+	public String getCkanToken() {
+		return ckanToken;
+	}
+	
 }

src/main/java/org/gcube/datacatalogue/ckanutillibrary/ckan/MarshUnmarshCkanObject.java

@@ -0,0 +1,38 @@
+package org.gcube.datacatalogue.ckanutillibrary.ckan;
+
+import java.io.IOException;
+
+import com.fasterxml.jackson.core.JsonParseException;
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.JsonMappingException;
+
+import eu.trentorise.opendata.jackan.model.CkanDataset;
+
+public class MarshUnmarshCkanObject {
+	
+	/**
+	 * To ckan dataset.
+	 *
+	 * @param jsonDataset the json dataset
+	 * @return the ckan dataset
+	 * @throws IOException 
+	 * @throws JsonMappingException 
+	 * @throws JsonParseException 
+	 */
+	public static CkanDataset toCkanDataset(String jsonDataset) throws JsonParseException, JsonMappingException, IOException {
+		return ExtendCkanClient.getObjectMapper().readValue(jsonDataset, CkanDataset.class);	
+	}
+
+	
+	/**
+	 * To json value dataset.
+	 *
+	 * @param dataset the dataset
+	 * @return the string
+	 * @throws JsonProcessingException 
+	 */
+	public static String toJsonValueDataset(CkanDataset dataset) throws JsonProcessingException {
+		return ExtendCkanClient.getObjectMapper().writeValueAsString(dataset);
+	}
+
+}

src/main/java/org/gcube/datacatalogue/ckanutillibrary/ckan/SimpleExtendCkanClient.java

@@ -1,168 +0,0 @@
-package org.gcube.datacatalogue.ckanutillibrary.ckan;
-
-import static com.google.common.base.Preconditions.checkNotNull;
-
-import java.io.InputStream;
-import java.io.InputStreamReader;
-import java.net.URLEncoder;
-import java.util.Arrays;
-
-import javax.annotation.Nullable;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.google.common.base.Charsets;
-import com.google.common.io.CharStreams;
-
-import eu.trentorise.opendata.jackan.CkanClient;
-import eu.trentorise.opendata.jackan.exceptions.CkanException;
-import eu.trentorise.opendata.jackan.exceptions.JackanException;
-import eu.trentorise.opendata.jackan.internal.org.apache.http.client.fluent.Request;
-import eu.trentorise.opendata.jackan.internal.org.apache.http.client.fluent.Response;
-import eu.trentorise.opendata.jackan.model.CkanGroup;
-import eu.trentorise.opendata.jackan.model.CkanOrganization;
-import eu.trentorise.opendata.jackan.model.CkanResponse;
-
-
-/**
- * The Class SimpleExtendCkanClient.
- *
- * @author Francesco Mangiacrapa at ISTI-CNR (francesco.mangiacrapa@isti.cnr.it)
- * 
- * Jun 7, 2019
- */
-public class SimpleExtendCkanClient extends CkanClient implements PatchedCkan{
-	
-	
-	private String catalogueURL;
-	
-	@Nullable
-	private static ObjectMapper objectMapper;
-
-	private static final Logger logger = LoggerFactory.getLogger(SimpleExtendCkanClient.class);
-
-	/**
-	 * Instantiates a new simple extend ckan client.
-	 *
-	 * @param catalogueURL the catalogue URL
-	 */
-	public SimpleExtendCkanClient(String catalogueURL) {
-		super(catalogueURL);
-		this.catalogueURL = catalogueURL;
-	}
-
-    /* (non-Javadoc)
-     * @see eu.trentorise.opendata.jackan.CkanClient#getOrganization(java.lang.String)
-     */
-    public synchronized CkanOrganization getOrganization(String idOrName) {
-       checkNotNull(idOrName, "Need a valid id or name!");
-       logger.info("Patched read organization for id/name: {}", idOrName);
-       return  getHttp(OrganizationResponse.class, "/api/3/action/organization_show", "id", idOrName,
-                "include_datasets", "false", "include_users", "true").result;
-
-    }
-    
-
-    /* (non-Javadoc)
-     * @see eu.trentorise.opendata.jackan.CkanClient#getGroup(java.lang.String)
-     */
-    public synchronized CkanGroup getGroup(String idOrName) {
-        checkNotNull(idOrName, "Need a valid id or name!");
-        logger.info("Patched read group for id/name: {}", idOrName);
-        return getHttp(GroupResponse.class, "/api/3/action/group_show", "id", idOrName, "include_datasets",
-                "false", "include_users", "true").result; 
-    }
-    
-    
-    /**
-     * Retrieves the Jackson object mapper for reading operations. Internally,
-     * Object mapper is initialized at first call.
-     *
-     * @return the object mapper
-     */
-    public static ObjectMapper getObjectMapper() {
-        if (objectMapper == null) {
-            objectMapper = new ObjectMapper();
-            configureObjectMapper(objectMapper);
-        }
-        return objectMapper;
-    }
-    
-    /* (non-Javadoc)
-     * @see org.gcube.datacatalogue.ckanutillibrary.server.patch.PatchedCkan#getHttp(java.lang.Class, java.lang.String, java.lang.Object[])
-     */
-    public <T extends CkanResponse> T getHttp(Class<T> responseType, String path, Object... params) {
-
-        checkNotNull(responseType);
-        checkNotNull(path);
-        String fullUrl = calcFullUrl(path, params);
-        T ckanResponse;
-        String returnedText;
-
-        try {
-        	
-        	logger.debug("getting {}", fullUrl);
-            Request request = Request.Get(fullUrl);
-            configureRequest(request);
-            Response response = request.execute();
-            InputStream stream = response.returnResponse()
-                                         .getEntity()
-                                         .getContent();
-
-            try (InputStreamReader reader = new InputStreamReader(stream, Charsets.UTF_8)) {
-                returnedText = CharStreams.toString(reader);
-            }
-            
-        	logger.trace("returnedText {}", returnedText);
-        } catch (Exception ex) {
-            throw new CkanException("Error while performing GET. Request url was: " + fullUrl, this, ex);
-        }
-        try {
-            ckanResponse = getObjectMapper().readValue(returnedText, responseType);
-        } catch (Exception ex) {
-            throw new CkanException(
-                    "Couldn't interpret json returned by the server! Returned text was: " + returnedText, this, ex);
-        }
-
-        if (!ckanResponse.isSuccess()) {
-            throwCkanException("Error while performing GET. Request url was: " + fullUrl, ckanResponse);
-        }
-        return ckanResponse;
-    }
-
-
-    /* (non-Javadoc)
-     * @see org.gcube.datacatalogue.ckanutillibrary.server.patch.PatchedCkan#calcFullUrl(java.lang.String, java.lang.Object[])
-     */
-    public String calcFullUrl(String path, Object[] params) {
-        checkNotNull(path);
-
-        try {
-            StringBuilder sb = new StringBuilder().append(catalogueURL)
-                                                  .append(path);
-            for (int i = 0; i < params.length; i += 2) {
-                sb.append(i == 0 ? "?" : "&")
-                  .append(URLEncoder.encode(params[i].toString(), "UTF-8"))
-                  .append("=")
-                  .append(URLEncoder.encode(params[i + 1].toString(), "UTF-8"));
-            }
-            return sb.toString();
-        } catch (Exception ex) {
-            throw new JackanException("Error while building url to perform GET! \n path: " + path + " \n params: "
-                    + Arrays.toString(params), ex);
-        }
-    }
-    
-	/**
-	 * Gets the catalogue url.
-	 *
-	 * @return the catalogueURL
-	 */
-	public String getCatalogueURL() {
-
-		return catalogueURL;
-	}
-	
-}

src/main/java/org/gcube/datacatalogue/ckanutillibrary/db/DBCaller.java

@@ -95,7 +95,7 @@ public class DBCaller {
 	 */
 	public Map<String, RolesCkanGroupOrOrg> getGroupsByUserFromDB(String username){
 		checkNotNull(username);
-		LOG.debug("Get groups by user called");
+		LOG.debug("Get groups by user called for username: "+username);
 
 		//couples (groups id, capacity) of the user in the group
 		
@@ -129,6 +129,8 @@ public class DBCaller {
 		}finally{
 			closeConnection(connection);
 		}
+		
+		LOG.debug("returning map: "+toReturn);
 		return toReturn;
 	}
 	
@@ -140,7 +142,7 @@ public class DBCaller {
 	 * @return the user id for username
 	 */
 	private String getUserIdForUsername(String username) {
-		LOG.debug("Get user id for username called");
+		LOG.debug("Get user id for username called for username: "+username);
 		
 		Connection connection = null;
 		String userId = null;
@@ -150,7 +152,7 @@ public class DBCaller {
 
 			String selQuery = "SELECT \"id\" FROM \"public\".\"user\" WHERE \"name\"=?;";
 			
-			LOG.info("Performing query: "+selQuery);
+			LOG.debug("Performing query: "+selQuery);
 
 			PreparedStatement preparedStatement = connection.prepareStatement(selQuery);
 			preparedStatement.setString(1, username);
@@ -213,6 +215,8 @@ public class DBCaller {
 		}finally{
 			closeConnection(connection);
 		}
+		
+		LOG.debug("returning map: "+toReturn);
 		return toReturn;
 	}
 	

src/main/java/org/gcube/datacatalogue/ckanutillibrary/gcat/GCatCaller.java

@@ -2,19 +2,11 @@ package org.gcube.datacatalogue.ckanutillibrary.gcat;
 
 import java.io.IOException;
 import java.net.MalformedURLException;
-import java.util.List;
 
-import org.gcube.datacatalogue.ckanutillibrary.ckan.SimpleExtendCkanClient;
 import org.gcube.gcat.client.Item;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import com.fasterxml.jackson.core.JsonParseException;
-import com.fasterxml.jackson.databind.JsonMappingException;
-
-import eu.trentorise.opendata.jackan.model.CkanDataset;
-import eu.trentorise.opendata.jackan.model.CkanLicense;
-
 // TODO: Auto-generated Javadoc
 /**
  * The Class GCatCaller.
@@ -24,18 +16,17 @@ import eu.trentorise.opendata.jackan.model.CkanLicense;
  */
 public class GCatCaller {
 	
-	private SimpleExtendCkanClient jackanCkanClient;
-
 	private String catalogueURL;
 	
 	private static final Logger LOG = LoggerFactory.getLogger(GCatCaller.class);
 	
 	/**
 	 * Instantiates a new g cat caller.
+	 *
+	 * @param catalogueURL the catalogue URL
 	 */
 	public GCatCaller(String catalogueURL) {
 		this.catalogueURL = catalogueURL;
-		this.jackanCkanClient = new SimpleExtendCkanClient(catalogueURL);
 	}
 	
 	
@@ -43,27 +34,27 @@ public class GCatCaller {
 	 * Gets the dataset for name.
 	 *
 	 * @param datasetIdOrName the dataset id or name
-	 * @return the dataset for name
+	 * @return the jsonValue
-	 * @throws JsonParseException the json parse exception
+	 * @throws MalformedURLException 
-	 * @throws JsonMappingException the json mapping exception
 	 * @throws IOException Signals that an I/O exception has occurred.
 	 */
-	public CkanDataset getDatasetForName(String datasetIdOrName) throws JsonParseException, JsonMappingException, IOException {
+	public String getDatasetForName(String datasetIdOrName) throws MalformedURLException {
 		LOG.debug("Get dataset for name "+datasetIdOrName+ "called");
-		String json = new Item().read(datasetIdOrName);
+		return new Item().read(datasetIdOrName);
-		return SimpleExtendCkanClient.getObjectMapper().readValue(json, CkanDataset.class);		
 	}
-
-
-	/**
-	 * TODO gCAT missing
-	 * @return
-	 * @throws MalformedURLException
-	 */
 	
-	public List<CkanLicense> getLicenseList() {
+	
-		LOG.debug("Get linces list called");
+	/**
-		return jackanCkanClient.getLicenseList();
+	 * Creates the dataset.
+	 *
+	 * @param jsonDataset the json dataset
+	 * @param socialPost if true sends the social post
+	 * @return the jsonValue
+	 * @throws MalformedURLException the malformed URL exception
+	 */
+	public String createDataset(String jsonDataset, boolean socialPost) throws MalformedURLException {
+		LOG.debug("Create dataset called");
+		return new Item().create(jsonDataset,socialPost);
 	}
 
 }

src/main/java/org/gcube/datacatalogue/ckanutillibrary/server/DataCatalogue.java

@@ -39,67 +39,6 @@ public interface DataCatalogue {
 	 */
 	String getUnencryptedUrlFromDatasetIdOrName(String datasetIdOrName);
 
-	/**
-	 * Create a dataset with those information. The method allows to have multiple
-	 * values for the same custom field key. NOTE: unfortunately java doesn't
-	 * support overload in java interface methods (that's way I cannot use the same
-	 * name for the method)
-	 *
-	 * @param title the title
-	 * @param name                 (unique identifier)
-	 * @param organizationNameOrId the organization name or id
-	 * @param author the author
-	 * @param authorMail the author mail
-	 * @param maintainer the maintainer
-	 * @param maintainerMail the maintainer mail
-	 * @param version the version
-	 * @param description the description
-	 * @param licenseId the license id
-	 * @param tags the tags
-	 * @param customFields the custom fields
-	 * @param resources the resources
-	 * @param setPublic            (manage visibility: Admin role is needed)
-	 * @return the id of the dataset on success, null otherwise
-	 * @throws Exception the exception
-	 */
-	String createCKanDatasetMultipleCustomFields(String title, String name, String organizationNameOrId, String author,
-			String authorMail, String maintainer, String maintainerMail, long version, String description,
-			String licenseId, List<String> tags, Map<String, List<String>> customFields, List<ResourceBean> resources,
-			boolean setPublic) throws Exception;
-
-	/**
-	 * Patch a product with product id productId by using the couples in
-	 * customFieldsToChange. NOTE: only the specified custom fields will be changed.
-	 * If a custom field with a given key already exists, and removeOld is set to
-	 * false, the new values are added at the end of the list. Otherwise they are
-	 * lost.
-	 *
-	 * @param productId the product id
-	 * @param customFieldsToChange the custom fields to change
-	 * @param removeOld the remove old
-	 * @return true, if successful
-	 */
-	boolean patchProductCustomFields(String productId, Map<String, List<String>> customFieldsToChange,
-			boolean removeOld);
-
-	/**
-	 * Add a resource described by the bean to the dataset id into
-	 * resource.datasetId
-	 *
-	 * @param resource the resource
-	 * @return String the id of the resource on success, null otherwise
-	 * @throws Exception the exception
-	 */
-	String addResourceToDataset(ResourceBean resource) throws Exception;
-
-	/**
-	 * Remove the resource with id resourceId from the dataset in which it is.
-	 *
-	 * @param resourceId the resource id
-	 * @return true on success, false otherwise.
-	 */
-	boolean deleteResourceFromDataset(String resourceId);
-
 	/**
 	 * Create a dataset with those information.
 	 *
@@ -201,4 +140,86 @@ public interface DataCatalogue {
 	 */
 	Map<String, Map<CkanOrganization, RolesCkanGroupOrOrg>> getUserRoleByOrganization(String username);
 
+	
+	
+	
+	
+	//WRITE OPERATIONS
+	
+	
+	
+	
+	
+	/**
+	 * Set searchable field
+	 * The field, if set to true, allows organization's users to view private products also in groups
+	 * @param datasetId the id of the dataset to update
+	 * @param searchable the value to assign to the field
+	 * @return true if the field is set, false otherwise
+	 */
+	boolean setSearchableField(String datasetId, boolean searchable);
+	
+	
+	/**
+	 * Create a dataset with those information. The method allows to have multiple
+	 * values for the same custom field key. NOTE: unfortunately java doesn't
+	 * support overload in java interface methods (that's way I cannot use the same
+	 * name for the method)
+	 * 
+	 * @param the username that is submitting the publishing request
+	 * @param title the title
+	 * @param name                 (unique identifier)
+	 * @param organizationNameOrId the organization name or id
+	 * @param author the author
+	 * @param authorMail the author mail
+	 * @param maintainer the maintainer
+	 * @param maintainerMail the maintainer mail
+	 * @param version the version
+	 * @param description the description
+	 * @param licenseId the license id
+	 * @param tags the tags
+	 * @param customFields the custom fields
+	 * @param resources the resources
+	 * @param setPublic            (manage visibility: Admin role is needed)
+	 * @return the id of the dataset on success, null otherwise
+	 * @throws Exception the exception
+	 */
+	String createCKanDatasetMultipleCustomFields(String username, String title, String name, String organizationNameOrId, String author,
+			String authorMail, String maintainer, String maintainerMail, long version, String description,
+			String licenseId, List<String> tags, Map<String, List<String>> customFields, List<ResourceBean> resources,
+			boolean setPublic) throws Exception;
+
+	/**
+	 * Patch a product with product id productId by using the couples in
+	 * customFieldsToChange. NOTE: only the specified custom fields will be changed.
+	 * If a custom field with a given key already exists, and removeOld is set to
+	 * false, the new values are added at the end of the list. Otherwise they are
+	 * lost.
+	 *
+	 * @param productId the product id
+	 * @param customFieldsToChange the custom fields to change
+	 * @param removeOld the remove old
+	 * @return true, if successful
+	 */
+	boolean patchProductCustomFields(String productId, Map<String, List<String>> customFieldsToChange,
+			boolean removeOld);
+
+	/**
+	 * Add a resource described by the bean to the dataset id into
+	 * resource.datasetId
+	 *
+	 * @param resource the resource
+	 * @return String the id of the resource on success, null otherwise
+	 * @throws Exception the exception
+	 */
+	String addResourceToDataset(ResourceBean resource) throws Exception;
+
+	/**
+	 * Remove the resource with id resourceId from the dataset in which it is.
+	 *
+	 * @param resourceId the resource id
+	 * @return true on success, false otherwise.
+	 */
+	boolean deleteResourceFromDataset(String resourceId);
+
 }

src/main/java/org/gcube/datacatalogue/ckanutillibrary/server/DataCatalogueImpl.java

@@ -6,15 +6,18 @@ import static com.google.common.base.Preconditions.checkNotNull;
 import java.net.URLEncoder;
 import java.util.ArrayList;
 import java.util.HashMap;
+import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
+import java.util.Map.Entry;
 import java.util.concurrent.ConcurrentHashMap;
 
 import org.gcube.common.authorization.library.provider.SecurityTokenProvider;
 import org.gcube.common.scope.impl.ScopeBean;
 import org.gcube.common.scope.impl.ScopeBean.Type;
+import org.gcube.datacatalogue.ckanutillibrary.ckan.DirectCkanCaller;
 import org.gcube.datacatalogue.ckanutillibrary.ckan.ExtendCkanClient;
-import org.gcube.datacatalogue.ckanutillibrary.ckan.SimpleExtendCkanClient;
+import org.gcube.datacatalogue.ckanutillibrary.ckan.MarshUnmarshCkanObject;
 import org.gcube.datacatalogue.ckanutillibrary.db.DBCaller;
 import org.gcube.datacatalogue.ckanutillibrary.gcat.GCatCaller;
 import org.gcube.datacatalogue.ckanutillibrary.server.DataCatalogueRunningCluster.ACCESS_LEVEL_TO_CATALOGUE_PORTLET;
@@ -38,7 +41,13 @@ import eu.trentorise.opendata.jackan.model.CkanDataset;
 import eu.trentorise.opendata.jackan.model.CkanGroup;
 import eu.trentorise.opendata.jackan.model.CkanLicense;
 import eu.trentorise.opendata.jackan.model.CkanOrganization;
+import eu.trentorise.opendata.jackan.model.CkanPair;
+import eu.trentorise.opendata.jackan.model.CkanResource;
+import eu.trentorise.opendata.jackan.model.CkanTag;
 import eu.trentorise.opendata.jackan.model.CkanUser;
+import net.htmlparser.jericho.Renderer;
+import net.htmlparser.jericho.Segment;
+import net.htmlparser.jericho.Source;
 
 public class DataCatalogueImpl implements DataCatalogue {
 
@@ -63,6 +72,7 @@ public class DataCatalogueImpl implements DataCatalogue {
 
 	public Map<ACCESS_LEVEL_TO_CATALOGUE_PORTLET, String> mapAccessURLToCatalogue;
 	
+
 	private static final String CATALOGUE_TAB_ENDING_URL = "/catalogue";
 	
 	// gCat client
@@ -72,10 +82,13 @@ public class DataCatalogueImpl implements DataCatalogue {
 	private DBCaller dbCaller;
 	
 	// ckan client
-	private SimpleExtendCkanClient ckanCaller;
+	private ExtendCkanClient ckanCaller;
 	
 	// hashmap for ckan api keys
 	private ConcurrentHashMap<String, CKANTokenBean> apiKeysMap;
+
+	//http ckan caller
+	private DirectCkanCaller directCkanCaller;
 	
 	// apikey bean expires after X minutes in the above map
 	private static final int EXPIRE_KEY_TIME = 60 * 60 * 1000;
@@ -97,7 +110,7 @@ public class DataCatalogueImpl implements DataCatalogue {
 		CKAN_DB_USER = runningInstance.getDataBaseUser().trim();
 		CKAN_DB_PASSWORD = runningInstance.getDataBasePassword().trim();
 
-		//CKAN_TOKEN_SYS = runningInstance.getSysAdminToken().trim();
+		CKAN_TOKEN_SYS = runningInstance.getSysAdminToken().trim();
 		CKAN_EMAIL = runningInstance.getEmailCatalogue().trim();
 
 		CKAN_CATALOGUE_URL = runningInstance.getDataCatalogueUrl().get(0).trim();
@@ -113,17 +126,13 @@ public class DataCatalogueImpl implements DataCatalogue {
 		
 		// build the clients
 		gCatCaller = new GCatCaller(CKAN_CATALOGUE_URL);
-		
 		dbCaller = new DBCaller(CKAN_DB_URL, CKAN_DB_PORT, CKAN_DB_NAME, CKAN_DB_USER, CKAN_DB_PASSWORD);
-		
+		ckanCaller = new ExtendCkanClient(CKAN_CATALOGUE_URL);
-		ckanCaller = new SimpleExtendCkanClient(CKAN_CATALOGUE_URL);
+		directCkanCaller = new DirectCkanCaller(CKAN_CATALOGUE_URL);
-
 		// init map
 		apiKeysMap = new ConcurrentHashMap<String, CKANTokenBean>();
-
 		// save the context
 		CONTEXT = scope;
-
 		// extended roles
 		extendRoleInOrganization = runningInstance.getExtendRoleInOrganization();
 	}
@@ -228,7 +237,8 @@ public class DataCatalogueImpl implements DataCatalogue {
 			String authzToken = SecurityTokenProvider.instance.get();
 			if(authzToken!=null && !authzToken.isEmpty()) {
 				LOG.info("gcube-token found. Calling the gCat client");
-				return gCatCaller.getDatasetForName(datasetId);
+				String jsonDataset = gCatCaller.getDatasetForName(datasetId);
+				return MarshUnmarshCkanObject.toCkanDataset(jsonDataset);
 			}
 			
 			LOG.info("No api-key or gcube-token found. Calling Ckan Client without API-KEY");
@@ -272,13 +282,7 @@ public class DataCatalogueImpl implements DataCatalogue {
 		return url;
 	}
 
-	public String createCKanDatasetMultipleCustomFields(String title, String name, String organizationNameOrId,
+	
-			String author, String authorMail, String maintainer, String maintainerMail, long version,
-			String description, String licenseId, List<String> tags, Map<String, List<String>> customFields,
-			List<ResourceBean> resources, boolean setPublic) throws Exception {
-		return null;
-	}
-
 	@Override
 	public Map<String, Map<CkanGroup, RolesCkanGroupOrOrg>> getUserRoleByGroup(String username) {
 		LOG.info("Get user role by group called. The username is: "+username);
@@ -328,6 +332,7 @@ public class DataCatalogueImpl implements DataCatalogue {
 				HashMap<CkanOrganization, RolesCkanGroupOrOrg> subMap = new HashMap<CkanOrganization, RolesCkanGroupOrOrg>();
 				subMap.put(org, partialResult.get(orgID));
 				toReturn.put(orgID, subMap);
+				LOG.debug("For organisation: " + org.getName() + ", the user "+username+" has role: "+subMap);
 			}
 
 			LOG.debug("Returning map " + toReturn);
@@ -399,10 +404,8 @@ public class DataCatalogueImpl implements DataCatalogue {
 
 		// checks
 		checkNotNull(username);
-
 		// in order to avoid errors, the username is always converted
 		String ckanUsername = CatalogueUtilMethods.fromUsernameToCKanUsername(username);
-
 		// list to return
 		List<CkanOrganization> toReturn = new ArrayList<CkanOrganization>();
 
@@ -413,12 +416,11 @@ public class DataCatalogueImpl implements DataCatalogue {
 
 			// iterate over them
 			for (CkanOrganization ckanOrganization : organizations) {
-
 				// get the list of users in it (if you try ckanOrganization.getUsers() it returns null.. maybe a bug TODO)
 				List<CkanUser> users = ckanCaller.getOrganization(ckanOrganization.getName()).getUsers();
-
 				// check if the current user is among them
 				for (CkanUser ckanUser : users) {
+					
 					if(ckanUser.getName().equals(ckanUsername)){
 
 						LOG.debug("User " + ckanUsername + " is into " + ckanOrganization.getName());
@@ -440,10 +442,8 @@ public class DataCatalogueImpl implements DataCatalogue {
 
 		// checks
 		checkNotNull(username);
-
 		// in order to avoid errors, the username is always converted
 		String ckanUsername = CatalogueUtilMethods.fromUsernameToCKanUsername(username);
-
 		// list to return
 		List<CkanGroup> toReturn = new ArrayList<CkanGroup>();
 
@@ -459,6 +459,7 @@ public class DataCatalogueImpl implements DataCatalogue {
 
 				// check if the current user is among them
 				for (CkanUser ckanUser : users) {
+					
 					if(ckanUser.getName().equals(ckanUsername)){
 
 						LOG.debug("User " + ckanUsername + " is into " + ckanGroup.getName());
@@ -513,15 +514,22 @@ public class DataCatalogueImpl implements DataCatalogue {
 		return null;
 	}
 	
-	
+	@Override
-	
+	public boolean existProductWithNameOrId(String nameOrId) {
-	
+		
-	
+		checkNotNull(nameOrId);
-	
+		checkArgument(!nameOrId.isEmpty());
-	
+
-	
+		try{
-	
+			ExtendCkanClient client = new ExtendCkanClient(CKAN_CATALOGUE_URL, CKAN_TOKEN_SYS);
-	
+			CkanDataset product = client.getDataset(nameOrId);
+			return product != null;
+		}catch(Exception e){
+			LOG.debug("A dataset with name " + nameOrId + " doesn't exist");
+			return false;
+		}
+	}
+
 	
 	/*
 	 * 
@@ -546,6 +554,163 @@ public class DataCatalogueImpl implements DataCatalogue {
 	 * 
 	 */
 	
+	@Override
+	public String createCKanDatasetMultipleCustomFields(String username, String title, String name, String organizationNameOrId,
+			String author, String authorMail, String maintainer, String maintainerMail, long version,
+			String description, String licenseId, List<String> tags, Map<String, List<String>> customFieldsMultiple,
+			List<ResourceBean> resources, boolean setPublic) throws Exception {
+
+		// delegate the private method
+		return createCkanDatasetBody(username, title, name, organizationNameOrId, author, authorMail, maintainer, maintainerMail,
+				version, description, licenseId, tags, null, customFieldsMultiple, resources, setPublic);
+	}
+
+	// the body of the actual dataset creation methods
+	private String createCkanDatasetBody(String username, String title, String name, String organizationNameOrId, String author,
+			String authorMail, String maintainer, String maintainerMail, long version, String description,
+			String licenseId, List<String> tags, Map<String, String> customFields,
+			Map<String, List<String>> customFieldsMultipleValues, List<ResourceBean> resources, boolean setPublic)
+			throws Exception {
+
+		// checks (minimum)
+		checkNotNull(username);
+		checkNotNull(organizationNameOrId);
+		checkArgument(!organizationNameOrId.isEmpty());
+		checkArgument(!(title == null && name == null || title.isEmpty() && name.isEmpty()), "Name and Title cannot be empty/null at the same time!");
+
+		LOG.debug("Request for dataset creation");
+
+		//ExtendCkanClient client = new ExtendCkanClient(CKAN_CATALOGUE_URL, apiKey);
+
+		//String ckanUsername = getUserFromApiKey(apiKey).getName();
+		CkanDataset dataset = new CkanDataset();
+		
+		// in order to avoid errors, the username is always converted
+		String ckanUsername = CatalogueUtilMethods.fromUsernameToCKanUsername(username);
+
+		String nameToUse = name;
+		if(nameToUse == null)
+			nameToUse = CatalogueUtilMethods.fromProductTitleToName(title);
+
+		LOG.debug("Name of the dataset is going to be " + nameToUse + ". Title is going to be " + title);
+
+		dataset.setName(nameToUse);
+		dataset.setTitle(title);
+
+//		CkanOrganization orgOwner = client.getOrganization(organizationNameOrId);
+//		dataset.setOwnerOrg(orgOwner.getId());
+		dataset.setAuthor(author);
+		dataset.setAuthorEmail(authorMail);
+		dataset.setMaintainer(maintainer);
+		dataset.setMaintainerEmail(maintainerMail);
+		dataset.setVersion(String.valueOf(version));
+
+		// description must be escaped
+		if(description != null && !description.isEmpty()){
+			Source descriptionEscaped = new Source(description);
+			Segment htmlSeg = new Segment(descriptionEscaped, 0, descriptionEscaped.length());
+			Renderer htmlRend = new Renderer(htmlSeg);
+			dataset.setNotes(htmlRend.toString());
+
+			LOG.debug("Description escaped " + htmlRend.toString());
+		}
+
+		dataset.setLicenseId(licenseId);
+
+		// set the tags, if any
+		if(tags != null && !tags.isEmpty()){
+			List<CkanTag> ckanTags = new ArrayList<CkanTag>(tags.size());
+			for (String stringTag : tags) {
+				ckanTags.add(new CkanTag(stringTag));
+			}
+			dataset.setTags(ckanTags);
+		}
+
+		// set the custom fields, if any
+		List<CkanPair> extras = new ArrayList<CkanPair>();
+
+		if(customFields != null && !customFields.isEmpty()){
+
+			Iterator<Entry<String, String>> iterator = customFields.entrySet().iterator();
+
+			while (iterator.hasNext()) {
+				Map.Entry<String, String> entry = iterator.next();
+				extras.add(new CkanPair(entry.getKey(), entry.getValue()));
+			}
+
+		}else if(customFieldsMultipleValues != null && !customFieldsMultipleValues.isEmpty()){
+
+			Iterator<Entry<String, List<String>>> iterator = customFieldsMultipleValues.entrySet().iterator();
+			while (iterator.hasNext()) {
+				Map.Entry<String, List<String>> entry = iterator.next();
+				List<String> valuesForEntry = entry.getValue();
+				for (String value : valuesForEntry) {
+					extras.add(new CkanPair(entry.getKey(), value));
+				}
+			}
+		}
+
+		dataset.setExtras(extras);
+
+		// check if we need to add the resources
+		if(resources != null && !resources.isEmpty()){
+
+			LOG.debug("We need to add resources to the dataset");
+
+			try{
+				List<CkanResource> resourcesCkan = new ArrayList<CkanResource>();
+				for(ResourceBean resource: resources){
+
+					LOG.debug("Going to add resource described by " + resource);
+					CkanResource newResource = new CkanResource();
+					newResource.setDescription(resource.getDescription());
+					newResource.setId(resource.getId());
+					newResource.setUrl(resource.getUrl());
+					newResource.setName(resource.getName());
+					newResource.setMimetype(resource.getMimeType());
+					newResource.setFormat(resource.getMimeType());
+					newResource.setOwner(ckanUsername);
+					resourcesCkan.add(newResource);
+				}
+
+				// add to the dataset
+				dataset.setResources(resourcesCkan);
+
+			}catch(Exception e){
+				LOG.error("Unable to add those resources to the dataset", e);
+			}
+
+		}
+
+		// try to create
+		String jsonValueDataset = MarshUnmarshCkanObject.toJsonValueDataset(dataset);
+		LOG.debug("Marshalling dataset is: " + jsonValueDataset);
+		jsonValueDataset = gCatCaller.createDataset(jsonValueDataset,true);
+		LOG.debug("Created dataset is: " + jsonValueDataset);
+
+		if(jsonValueDataset != null){
+			CkanDataset toCkanDataset = MarshUnmarshCkanObject.toCkanDataset(jsonValueDataset);
+			LOG.debug("Dataset with name " + toCkanDataset.getName() + " has been created. Setting visibility");
+
+			// set visibility
+			boolean visibilitySet = directCkanCaller.setDatasetPrivate(
+					!setPublic, // swap to private
+					toCkanDataset.getOrganization().getId(),
+					toCkanDataset.getId(),
+					CKAN_TOKEN_SYS); // use sysadmin api key to be sure it will be set
+
+			LOG.info("Was visibility set to " + (setPublic ? "public" : "private") + "? " + visibilitySet);
+
+			// set searchable to true if dataset visibility is private
+			if(!setPublic){ // (swap to private)
+				boolean searchableSet = setSearchableField(toCkanDataset.getId(), true);
+				LOG.info("Was searchable set to True? " + searchableSet);
+			}
+			return toCkanDataset.getId();
+		}
+
+		return null;
+	}
 	
 	public boolean patchProductCustomFields(String productId, Map<String, List<String>> customFieldsToChange,
 			boolean removeOld) {
@@ -560,9 +725,11 @@ public class DataCatalogueImpl implements DataCatalogue {
 		return false;
 	}
 
-	public boolean existProductWithNameOrId(String nameOrId) {
+	@Override
-		return false;
+	public boolean setSearchableField(String datasetId, boolean searchable) {
+		return directCkanCaller.setSearchableField(datasetId, searchable, CKAN_TOKEN_SYS);
 	}
 
 
+
 }

src/test/java/TestDataCatalogueLib.java

@@ -5,7 +5,6 @@ import org.gcube.datacatalogue.ckanutillibrary.server.ApplicationProfileScopePer
 import org.gcube.datacatalogue.ckanutillibrary.server.DataCatalogueFactory;
 import org.gcube.datacatalogue.ckanutillibrary.server.DataCatalogueImpl;
 import org.junit.Before;
-import org.junit.Test;
 import org.slf4j.LoggerFactory;
 
 /**
@@ -19,7 +18,9 @@ public class TestDataCatalogueLib {
 	private static final org.slf4j.Logger logger = LoggerFactory.getLogger(TestDataCatalogueLib.class);
 
 	private DataCatalogueFactory factory;
-	private String scope = "/gcube/devsec/devVRE";
+	//private String scope = "/gcube/devsec/devVRE";
+	private String scope = "/d4science.research-infrastructures.eu/SoBigData/Catalogue-TerritoriAperti";
+	//private String scope = "/d4science.research-infrastructures.eu/gCubeApps/BiodiversityLab";
 	//private String testUser = "costantino_perciante";
 	private String testUser = "francesco_mangiacrapa";
 	String subjectId = "aa_father4";
@@ -46,15 +47,15 @@ public class TestDataCatalogueLib {
 		DataCatalogueFactory factory = DataCatalogueFactory.getFactory();
 
 		while(true){
-			factory.getUtilsPerScope("/gcube");
+			factory.getUtilsPerScope(scope);
 			Thread.sleep(60* 1000 * 3);
-			factory.getUtilsPerScope("/gcube");
+			factory.getUtilsPerScope(scope);
 			break;
 		}
 
 		for (int i = 0; i < 5; i++) {
 			Thread.sleep(1000);
-			factory.getUtilsPerScope("/gcube");
+			factory.getUtilsPerScope(scope);
 		}
 
 	}
@@ -68,7 +69,7 @@ public class TestDataCatalogueLib {
 	//@Test
 	public void getScopePerUrl(){
 
-		ScopeProvider.instance.set("/gcube");
+		ScopeProvider.instance.set(scope);
 		String url = "https://dev4.d4science.org/group/devvre/ckan";
 		String scopeToUse = ApplicationProfileScopePerUrlReader.getScopePerUrl(url);
 		logger.debug("Retrieved scope is " + scopeToUse);
@@ -86,7 +87,7 @@ public class TestDataCatalogueLib {
 	 * @return the user role by group
 	 * @throws Exception the exception
 	 */
-	@Test
+	//@Test
 	public void getUserRoleByGroup() throws Exception{
 		DataCatalogueImpl instance = factory.getUtilsPerScope(scope);
 		String username = testUser;

src/test/resources/.gitignore

@@ -5,3 +5,12 @@
 /log4j.properties
 /pred4s.gcubekey
 /preprod.gcubekey
+/D4OS.gcubekey
+/D4Research.gcubekey
+/FARM.gcubekey
+/OpenAIRE.gcubekey
+/ParthenosVO.gcubekey
+/SmartArea.gcubekey
+/SoBigData.gcubekey
+/d4science.research-infrastructures.eu.gcubekey
+/gCubeApps.gcubekey

target/classes/META-INF/MANIFEST.MF

@@ -6,7 +6,7 @@ Specification-Version: 0.1.0-SNAPSHOT
 Implementation-Title: CKan utility library
 Implementation-Version: 0.1.0-SNAPSHOT
 Implementation-Vendor-Id: org.gcube.datacatalogue
-Build-Time: 20200601-145428
+Build-Time: 20200603-095004
 Created-By: Maven Integration for Eclipse
 SCM-Branch: 
 SCM-Revision: 

target/classes/META-INF/maven/org.gcube.datacatalogue/catalogue-util-library/pom.properties

@@ -1,5 +1,5 @@
 #Generated by Maven Integration for Eclipse
-#Mon Jun 01 16:54:29 CEST 2020
+#Wed Jun 03 11:50:06 CEST 2020
 version=0.1.0-SNAPSHOT
 groupId=org.gcube.datacatalogue
 m2e.projectName=catalogue-util-library

target/classes/org/gcube/datacatalogue/ckanutillibrary/ckan/ExtendCkanClient.java

@@ -1,6 +1,3 @@
-/**
- *
- */
 package org.gcube.datacatalogue.ckanutillibrary.ckan;
 
 import static com.google.common.base.Preconditions.checkNotNull;
@@ -19,7 +16,7 @@ import com.fasterxml.jackson.databind.ObjectMapper;
 import com.google.common.base.Charsets;
 import com.google.common.io.CharStreams;
 
-import eu.trentorise.opendata.jackan.CheckedCkanClient;
+import eu.trentorise.opendata.jackan.CkanClient;
 import eu.trentorise.opendata.jackan.exceptions.CkanException;
 import eu.trentorise.opendata.jackan.exceptions.JackanException;
 import eu.trentorise.opendata.jackan.internal.org.apache.http.client.fluent.Request;
@@ -29,36 +26,34 @@ import eu.trentorise.opendata.jackan.model.CkanOrganization;
 import eu.trentorise.opendata.jackan.model.CkanResponse;
 
 
+// TODO: Auto-generated Javadoc
 /**
- * The Class ExtendCkanClient.
+ * The Class SimpleExtendCkanClient.
  *
  * @author Francesco Mangiacrapa at ISTI-CNR (francesco.mangiacrapa@isti.cnr.it)
  * 
- * Jun 14, 2019
+ * Jun 7, 2019
+ * 
  */
-public class ExtendCkanClient extends CheckedCkanClient  implements PatchedCkan{
+public class ExtendCkanClient extends CkanClient implements PatchedCkan{
-
-	private static final Logger logger = LoggerFactory.getLogger(ExtendCkanClient.class);
 	
-	@Nullable
-	private static ObjectMapper objectMapper;
-
 	private String catalogueURL;
 	private String ckanToken;
     private int extendTimeout = 120000; //in milliseconds
-
+	@Nullable
+	private static ObjectMapper objectMapper;
+	private static final Logger logger = LoggerFactory.getLogger(ExtendCkanClient.class);
 
 	/**
-	 * Instantiates a new extend ckan client.
+	 * Instantiates a new simple extend ckan client.
 	 *
-	 * @param catalogueURL the catalogue url
+	 * @param catalogueURL the catalogue URL
 	 */
 	public ExtendCkanClient(String catalogueURL) {
 		super(catalogueURL);
 		this.catalogueURL = catalogueURL;
-		
 	}
-
+	
     /**
      * Instantiates a new extend ckan client.
      *
@@ -84,7 +79,7 @@ public class ExtendCkanClient extends CheckedCkanClient  implements PatchedCkan{
         this.ckanToken = ckanToken;
         this.extendTimeout = timeout;
     }
-
+    
 
     /**
      * Configures the request. Should work both for GETs and POSTs.
@@ -104,38 +99,13 @@ public class ExtendCkanClient extends CheckedCkanClient  implements PatchedCkan{
         return request;
     }
 
-	/**
+    /**
-	 * Gets the catalogue url.
+     * Gets the organization.
-	 *
+     *
-	 * @return the catalogueURL
+     * @param idOrName the id or name
-	 */
+     * @return the organization
-	public String getCatalogueURL() {
+     */
-
+    /* (non-Javadoc)
-		return catalogueURL;
-	}
-
-
-	/**
-	 * Gets the ckan token.
-	 *
-	 * @return the ckanToken
-	 */
-	@Override
-	public String getCkanToken() {
-
-		return ckanToken;
-	}
-
-	/**
-	 * Gets the timeout.
-	 *
-	 * @return the timeout
-	 */
-	public int getTimeout() {
-		return extendTimeout;
-	}
-
-	  /* (non-Javadoc)
      * @see eu.trentorise.opendata.jackan.CkanClient#getOrganization(java.lang.String)
      */
     public synchronized CkanOrganization getOrganization(String idOrName) {
@@ -147,6 +117,12 @@ public class ExtendCkanClient extends CheckedCkanClient  implements PatchedCkan{
     }
     
 
+    /**
+     * Gets the group.
+     *
+     * @param idOrName the id or name
+     * @return the group
+     */
     /* (non-Javadoc)
      * @see eu.trentorise.opendata.jackan.CkanClient#getGroup(java.lang.String)
      */
@@ -164,7 +140,7 @@ public class ExtendCkanClient extends CheckedCkanClient  implements PatchedCkan{
      *
      * @return the object mapper
      */
-    static ObjectMapper getObjectMapper() {
+    public static ObjectMapper getObjectMapper() {
         if (objectMapper == null) {
             objectMapper = new ObjectMapper();
             configureObjectMapper(objectMapper);
@@ -172,6 +148,15 @@ public class ExtendCkanClient extends CheckedCkanClient  implements PatchedCkan{
         return objectMapper;
     }
     
+    /**
+     * Gets the http.
+     *
+     * @param <T> the generic type
+     * @param responseType the response type
+     * @param path the path
+     * @param params the params
+     * @return the http
+     */
     /* (non-Javadoc)
      * @see org.gcube.datacatalogue.ckanutillibrary.server.patch.PatchedCkan#getHttp(java.lang.Class, java.lang.String, java.lang.Object[])
      */
@@ -215,6 +200,13 @@ public class ExtendCkanClient extends CheckedCkanClient  implements PatchedCkan{
     }
 
 
+    /**
+     * Calc full url.
+     *
+     * @param path the path
+     * @param params the params
+     * @return the string
+     */
     /* (non-Javadoc)
      * @see org.gcube.datacatalogue.ckanutillibrary.server.patch.PatchedCkan#calcFullUrl(java.lang.String, java.lang.Object[])
      */
@@ -236,5 +228,26 @@ public class ExtendCkanClient extends CheckedCkanClient  implements PatchedCkan{
                     + Arrays.toString(params), ex);
         }
     }
+    
 
+	/**
+	 * Gets the catalogue URL.
+	 *
+	 * @return the catalogue URL
+	 */
+	public String getCatalogueURL() {
+
+		return catalogueURL;
+	}
+	
+	/**
+	 * Gets the ckan token.
+	 *
+	 * @return the ckan token
+	 */
+	@Override
+	public String getCkanToken() {
+		return ckanToken;
+	}
+	
 }

target/classes/org/gcube/datacatalogue/ckanutillibrary/ckan/SimpleExtendCkanClient.java

@@ -1,168 +0,0 @@
-package org.gcube.datacatalogue.ckanutillibrary.ckan;
-
-import static com.google.common.base.Preconditions.checkNotNull;
-
-import java.io.InputStream;
-import java.io.InputStreamReader;
-import java.net.URLEncoder;
-import java.util.Arrays;
-
-import javax.annotation.Nullable;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.google.common.base.Charsets;
-import com.google.common.io.CharStreams;
-
-import eu.trentorise.opendata.jackan.CkanClient;
-import eu.trentorise.opendata.jackan.exceptions.CkanException;
-import eu.trentorise.opendata.jackan.exceptions.JackanException;
-import eu.trentorise.opendata.jackan.internal.org.apache.http.client.fluent.Request;
-import eu.trentorise.opendata.jackan.internal.org.apache.http.client.fluent.Response;
-import eu.trentorise.opendata.jackan.model.CkanGroup;
-import eu.trentorise.opendata.jackan.model.CkanOrganization;
-import eu.trentorise.opendata.jackan.model.CkanResponse;
-
-
-/**
- * The Class SimpleExtendCkanClient.
- *
- * @author Francesco Mangiacrapa at ISTI-CNR (francesco.mangiacrapa@isti.cnr.it)
- * 
- * Jun 7, 2019
- */
-public class SimpleExtendCkanClient extends CkanClient implements PatchedCkan{
-	
-	
-	private String catalogueURL;
-	
-	@Nullable
-	private static ObjectMapper objectMapper;
-
-	private static final Logger logger = LoggerFactory.getLogger(SimpleExtendCkanClient.class);
-
-	/**
-	 * Instantiates a new simple extend ckan client.
-	 *
-	 * @param catalogueURL the catalogue URL
-	 */
-	public SimpleExtendCkanClient(String catalogueURL) {
-		super(catalogueURL);
-		this.catalogueURL = catalogueURL;
-	}
-
-    /* (non-Javadoc)
-     * @see eu.trentorise.opendata.jackan.CkanClient#getOrganization(java.lang.String)
-     */
-    public synchronized CkanOrganization getOrganization(String idOrName) {
-       checkNotNull(idOrName, "Need a valid id or name!");
-       logger.info("Patched read organization for id/name: {}", idOrName);
-       return  getHttp(OrganizationResponse.class, "/api/3/action/organization_show", "id", idOrName,
-                "include_datasets", "false", "include_users", "true").result;
-
-    }
-    
-
-    /* (non-Javadoc)
-     * @see eu.trentorise.opendata.jackan.CkanClient#getGroup(java.lang.String)
-     */
-    public synchronized CkanGroup getGroup(String idOrName) {
-        checkNotNull(idOrName, "Need a valid id or name!");
-        logger.info("Patched read group for id/name: {}", idOrName);
-        return getHttp(GroupResponse.class, "/api/3/action/group_show", "id", idOrName, "include_datasets",
-                "false", "include_users", "true").result; 
-    }
-    
-    
-    /**
-     * Retrieves the Jackson object mapper for reading operations. Internally,
-     * Object mapper is initialized at first call.
-     *
-     * @return the object mapper
-     */
-    public static ObjectMapper getObjectMapper() {
-        if (objectMapper == null) {
-            objectMapper = new ObjectMapper();
-            configureObjectMapper(objectMapper);
-        }
-        return objectMapper;
-    }
-    
-    /* (non-Javadoc)
-     * @see org.gcube.datacatalogue.ckanutillibrary.server.patch.PatchedCkan#getHttp(java.lang.Class, java.lang.String, java.lang.Object[])
-     */
-    public <T extends CkanResponse> T getHttp(Class<T> responseType, String path, Object... params) {
-
-        checkNotNull(responseType);
-        checkNotNull(path);
-        String fullUrl = calcFullUrl(path, params);
-        T ckanResponse;
-        String returnedText;
-
-        try {
-        	
-        	logger.debug("getting {}", fullUrl);
-            Request request = Request.Get(fullUrl);
-            configureRequest(request);
-            Response response = request.execute();
-            InputStream stream = response.returnResponse()
-                                         .getEntity()
-                                         .getContent();
-
-            try (InputStreamReader reader = new InputStreamReader(stream, Charsets.UTF_8)) {
-                returnedText = CharStreams.toString(reader);
-            }
-            
-        	logger.trace("returnedText {}", returnedText);
-        } catch (Exception ex) {
-            throw new CkanException("Error while performing GET. Request url was: " + fullUrl, this, ex);
-        }
-        try {
-            ckanResponse = getObjectMapper().readValue(returnedText, responseType);
-        } catch (Exception ex) {
-            throw new CkanException(
-                    "Couldn't interpret json returned by the server! Returned text was: " + returnedText, this, ex);
-        }
-
-        if (!ckanResponse.isSuccess()) {
-            throwCkanException("Error while performing GET. Request url was: " + fullUrl, ckanResponse);
-        }
-        return ckanResponse;
-    }
-
-
-    /* (non-Javadoc)
-     * @see org.gcube.datacatalogue.ckanutillibrary.server.patch.PatchedCkan#calcFullUrl(java.lang.String, java.lang.Object[])
-     */
-    public String calcFullUrl(String path, Object[] params) {
-        checkNotNull(path);
-
-        try {
-            StringBuilder sb = new StringBuilder().append(catalogueURL)
-                                                  .append(path);
-            for (int i = 0; i < params.length; i += 2) {
-                sb.append(i == 0 ? "?" : "&")
-                  .append(URLEncoder.encode(params[i].toString(), "UTF-8"))
-                  .append("=")
-                  .append(URLEncoder.encode(params[i + 1].toString(), "UTF-8"));
-            }
-            return sb.toString();
-        } catch (Exception ex) {
-            throw new JackanException("Error while building url to perform GET! \n path: " + path + " \n params: "
-                    + Arrays.toString(params), ex);
-        }
-    }
-    
-	/**
-	 * Gets the catalogue url.
-	 *
-	 * @return the catalogueURL
-	 */
-	public String getCatalogueURL() {
-
-		return catalogueURL;
-	}
-	
-}

target/classes/org/gcube/datacatalogue/ckanutillibrary/db/DBCaller.java

@@ -95,7 +95,7 @@ public class DBCaller {
 	 */
 	public Map<String, RolesCkanGroupOrOrg> getGroupsByUserFromDB(String username){
 		checkNotNull(username);
-		LOG.debug("Get groups by user called");
+		LOG.debug("Get groups by user called for username: "+username);
 
 		//couples (groups id, capacity) of the user in the group
 		
@@ -129,6 +129,8 @@ public class DBCaller {
 		}finally{
 			closeConnection(connection);
 		}
+		
+		LOG.debug("returning map: "+toReturn);
 		return toReturn;
 	}
 	
@@ -140,7 +142,7 @@ public class DBCaller {
 	 * @return the user id for username
 	 */
 	private String getUserIdForUsername(String username) {
-		LOG.debug("Get user id for username called");
+		LOG.debug("Get user id for username called for username: "+username);
 		
 		Connection connection = null;
 		String userId = null;
@@ -150,7 +152,7 @@ public class DBCaller {
 
 			String selQuery = "SELECT \"id\" FROM \"public\".\"user\" WHERE \"name\"=?;";
 			
-			LOG.info("Performing query: "+selQuery);
+			LOG.debug("Performing query: "+selQuery);
 
 			PreparedStatement preparedStatement = connection.prepareStatement(selQuery);
 			preparedStatement.setString(1, username);
@@ -213,6 +215,8 @@ public class DBCaller {
 		}finally{
 			closeConnection(connection);
 		}
+		
+		LOG.debug("returning map: "+toReturn);
 		return toReturn;
 	}
 	

target/classes/org/gcube/datacatalogue/ckanutillibrary/gcat/GCatCaller.java

@@ -2,19 +2,11 @@ package org.gcube.datacatalogue.ckanutillibrary.gcat;
 
 import java.io.IOException;
 import java.net.MalformedURLException;
-import java.util.List;
 
-import org.gcube.datacatalogue.ckanutillibrary.ckan.SimpleExtendCkanClient;
 import org.gcube.gcat.client.Item;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import com.fasterxml.jackson.core.JsonParseException;
-import com.fasterxml.jackson.databind.JsonMappingException;
-
-import eu.trentorise.opendata.jackan.model.CkanDataset;
-import eu.trentorise.opendata.jackan.model.CkanLicense;
-
 // TODO: Auto-generated Javadoc
 /**
  * The Class GCatCaller.
@@ -24,18 +16,17 @@ import eu.trentorise.opendata.jackan.model.CkanLicense;
  */
 public class GCatCaller {
 	
-	private SimpleExtendCkanClient jackanCkanClient;
-
 	private String catalogueURL;
 	
 	private static final Logger LOG = LoggerFactory.getLogger(GCatCaller.class);
 	
 	/**
 	 * Instantiates a new g cat caller.
+	 *
+	 * @param catalogueURL the catalogue URL
 	 */
 	public GCatCaller(String catalogueURL) {
 		this.catalogueURL = catalogueURL;
-		this.jackanCkanClient = new SimpleExtendCkanClient(catalogueURL);
 	}
 	
 	
@@ -43,27 +34,27 @@ public class GCatCaller {
 	 * Gets the dataset for name.
 	 *
 	 * @param datasetIdOrName the dataset id or name
-	 * @return the dataset for name
+	 * @return the jsonValue
-	 * @throws JsonParseException the json parse exception
+	 * @throws MalformedURLException 
-	 * @throws JsonMappingException the json mapping exception
 	 * @throws IOException Signals that an I/O exception has occurred.
 	 */
-	public CkanDataset getDatasetForName(String datasetIdOrName) throws JsonParseException, JsonMappingException, IOException {
+	public String getDatasetForName(String datasetIdOrName) throws MalformedURLException {
 		LOG.debug("Get dataset for name "+datasetIdOrName+ "called");
-		String json = new Item().read(datasetIdOrName);
+		return new Item().read(datasetIdOrName);
-		return SimpleExtendCkanClient.getObjectMapper().readValue(json, CkanDataset.class);		
 	}
-
-
-	/**
-	 * TODO gCAT missing
-	 * @return
-	 * @throws MalformedURLException
-	 */
 	
-	public List<CkanLicense> getLicenseList() {
+	
-		LOG.debug("Get linces list called");
+	/**
-		return jackanCkanClient.getLicenseList();
+	 * Creates the dataset.
+	 *
+	 * @param jsonDataset the json dataset
+	 * @param socialPost if true sends the social post
+	 * @return the jsonValue
+	 * @throws MalformedURLException the malformed URL exception
+	 */
+	public String createDataset(String jsonDataset, boolean socialPost) throws MalformedURLException {
+		LOG.debug("Create dataset called");
+		return new Item().create(jsonDataset,socialPost);
 	}
 
 }

target/classes/org/gcube/datacatalogue/ckanutillibrary/server/DataCatalogue.java

@@ -39,67 +39,6 @@ public interface DataCatalogue {
 	 */
 	String getUnencryptedUrlFromDatasetIdOrName(String datasetIdOrName);
 
-	/**
-	 * Create a dataset with those information. The method allows to have multiple
-	 * values for the same custom field key. NOTE: unfortunately java doesn't
-	 * support overload in java interface methods (that's way I cannot use the same
-	 * name for the method)
-	 *
-	 * @param title the title
-	 * @param name                 (unique identifier)
-	 * @param organizationNameOrId the organization name or id
-	 * @param author the author
-	 * @param authorMail the author mail
-	 * @param maintainer the maintainer
-	 * @param maintainerMail the maintainer mail
-	 * @param version the version
-	 * @param description the description
-	 * @param licenseId the license id
-	 * @param tags the tags
-	 * @param customFields the custom fields
-	 * @param resources the resources
-	 * @param setPublic            (manage visibility: Admin role is needed)
-	 * @return the id of the dataset on success, null otherwise
-	 * @throws Exception the exception
-	 */
-	String createCKanDatasetMultipleCustomFields(String title, String name, String organizationNameOrId, String author,
-			String authorMail, String maintainer, String maintainerMail, long version, String description,
-			String licenseId, List<String> tags, Map<String, List<String>> customFields, List<ResourceBean> resources,
-			boolean setPublic) throws Exception;
-
-	/**
-	 * Patch a product with product id productId by using the couples in
-	 * customFieldsToChange. NOTE: only the specified custom fields will be changed.
-	 * If a custom field with a given key already exists, and removeOld is set to
-	 * false, the new values are added at the end of the list. Otherwise they are
-	 * lost.
-	 *
-	 * @param productId the product id
-	 * @param customFieldsToChange the custom fields to change
-	 * @param removeOld the remove old
-	 * @return true, if successful
-	 */
-	boolean patchProductCustomFields(String productId, Map<String, List<String>> customFieldsToChange,
-			boolean removeOld);
-
-	/**
-	 * Add a resource described by the bean to the dataset id into
-	 * resource.datasetId
-	 *
-	 * @param resource the resource
-	 * @return String the id of the resource on success, null otherwise
-	 * @throws Exception the exception
-	 */
-	String addResourceToDataset(ResourceBean resource) throws Exception;
-
-	/**
-	 * Remove the resource with id resourceId from the dataset in which it is.
-	 *
-	 * @param resourceId the resource id
-	 * @return true on success, false otherwise.
-	 */
-	boolean deleteResourceFromDataset(String resourceId);
-
 	/**
 	 * Create a dataset with those information.
 	 *
@@ -201,4 +140,86 @@ public interface DataCatalogue {
 	 */
 	Map<String, Map<CkanOrganization, RolesCkanGroupOrOrg>> getUserRoleByOrganization(String username);
 
+	
+	
+	
+	
+	//WRITE OPERATIONS
+	
+	
+	
+	
+	
+	/**
+	 * Set searchable field
+	 * The field, if set to true, allows organization's users to view private products also in groups
+	 * @param datasetId the id of the dataset to update
+	 * @param searchable the value to assign to the field
+	 * @return true if the field is set, false otherwise
+	 */
+	boolean setSearchableField(String datasetId, boolean searchable);
+	
+	
+	/**
+	 * Create a dataset with those information. The method allows to have multiple
+	 * values for the same custom field key. NOTE: unfortunately java doesn't
+	 * support overload in java interface methods (that's way I cannot use the same
+	 * name for the method)
+	 * 
+	 * @param the username that is submitting the publishing request
+	 * @param title the title
+	 * @param name                 (unique identifier)
+	 * @param organizationNameOrId the organization name or id
+	 * @param author the author
+	 * @param authorMail the author mail
+	 * @param maintainer the maintainer
+	 * @param maintainerMail the maintainer mail
+	 * @param version the version
+	 * @param description the description
+	 * @param licenseId the license id
+	 * @param tags the tags
+	 * @param customFields the custom fields
+	 * @param resources the resources
+	 * @param setPublic            (manage visibility: Admin role is needed)
+	 * @return the id of the dataset on success, null otherwise
+	 * @throws Exception the exception
+	 */
+	String createCKanDatasetMultipleCustomFields(String username, String title, String name, String organizationNameOrId, String author,
+			String authorMail, String maintainer, String maintainerMail, long version, String description,
+			String licenseId, List<String> tags, Map<String, List<String>> customFields, List<ResourceBean> resources,
+			boolean setPublic) throws Exception;
+
+	/**
+	 * Patch a product with product id productId by using the couples in
+	 * customFieldsToChange. NOTE: only the specified custom fields will be changed.
+	 * If a custom field with a given key already exists, and removeOld is set to
+	 * false, the new values are added at the end of the list. Otherwise they are
+	 * lost.
+	 *
+	 * @param productId the product id
+	 * @param customFieldsToChange the custom fields to change
+	 * @param removeOld the remove old
+	 * @return true, if successful
+	 */
+	boolean patchProductCustomFields(String productId, Map<String, List<String>> customFieldsToChange,
+			boolean removeOld);
+
+	/**
+	 * Add a resource described by the bean to the dataset id into
+	 * resource.datasetId
+	 *
+	 * @param resource the resource
+	 * @return String the id of the resource on success, null otherwise
+	 * @throws Exception the exception
+	 */
+	String addResourceToDataset(ResourceBean resource) throws Exception;
+
+	/**
+	 * Remove the resource with id resourceId from the dataset in which it is.
+	 *
+	 * @param resourceId the resource id
+	 * @return true on success, false otherwise.
+	 */
+	boolean deleteResourceFromDataset(String resourceId);
+
 }

target/classes/org/gcube/datacatalogue/ckanutillibrary/server/DataCatalogueImpl.java

@@ -6,15 +6,18 @@ import static com.google.common.base.Preconditions.checkNotNull;
 import java.net.URLEncoder;
 import java.util.ArrayList;
 import java.util.HashMap;
+import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
+import java.util.Map.Entry;
 import java.util.concurrent.ConcurrentHashMap;
 
 import org.gcube.common.authorization.library.provider.SecurityTokenProvider;
 import org.gcube.common.scope.impl.ScopeBean;
 import org.gcube.common.scope.impl.ScopeBean.Type;
+import org.gcube.datacatalogue.ckanutillibrary.ckan.DirectCkanCaller;
 import org.gcube.datacatalogue.ckanutillibrary.ckan.ExtendCkanClient;
-import org.gcube.datacatalogue.ckanutillibrary.ckan.SimpleExtendCkanClient;
+import org.gcube.datacatalogue.ckanutillibrary.ckan.MarshUnmarshCkanObject;
 import org.gcube.datacatalogue.ckanutillibrary.db.DBCaller;
 import org.gcube.datacatalogue.ckanutillibrary.gcat.GCatCaller;
 import org.gcube.datacatalogue.ckanutillibrary.server.DataCatalogueRunningCluster.ACCESS_LEVEL_TO_CATALOGUE_PORTLET;
@@ -38,7 +41,13 @@ import eu.trentorise.opendata.jackan.model.CkanDataset;
 import eu.trentorise.opendata.jackan.model.CkanGroup;
 import eu.trentorise.opendata.jackan.model.CkanLicense;
 import eu.trentorise.opendata.jackan.model.CkanOrganization;
+import eu.trentorise.opendata.jackan.model.CkanPair;
+import eu.trentorise.opendata.jackan.model.CkanResource;
+import eu.trentorise.opendata.jackan.model.CkanTag;
 import eu.trentorise.opendata.jackan.model.CkanUser;
+import net.htmlparser.jericho.Renderer;
+import net.htmlparser.jericho.Segment;
+import net.htmlparser.jericho.Source;
 
 public class DataCatalogueImpl implements DataCatalogue {
 
@@ -63,6 +72,7 @@ public class DataCatalogueImpl implements DataCatalogue {
 
 	public Map<ACCESS_LEVEL_TO_CATALOGUE_PORTLET, String> mapAccessURLToCatalogue;
 	
+
 	private static final String CATALOGUE_TAB_ENDING_URL = "/catalogue";
 	
 	// gCat client
@@ -72,10 +82,13 @@ public class DataCatalogueImpl implements DataCatalogue {
 	private DBCaller dbCaller;
 	
 	// ckan client
-	private SimpleExtendCkanClient ckanCaller;
+	private ExtendCkanClient ckanCaller;
 	
 	// hashmap for ckan api keys
 	private ConcurrentHashMap<String, CKANTokenBean> apiKeysMap;
+
+	//http ckan caller
+	private DirectCkanCaller directCkanCaller;
 	
 	// apikey bean expires after X minutes in the above map
 	private static final int EXPIRE_KEY_TIME = 60 * 60 * 1000;
@@ -97,7 +110,7 @@ public class DataCatalogueImpl implements DataCatalogue {
 		CKAN_DB_USER = runningInstance.getDataBaseUser().trim();
 		CKAN_DB_PASSWORD = runningInstance.getDataBasePassword().trim();
 
-		//CKAN_TOKEN_SYS = runningInstance.getSysAdminToken().trim();
+		CKAN_TOKEN_SYS = runningInstance.getSysAdminToken().trim();
 		CKAN_EMAIL = runningInstance.getEmailCatalogue().trim();
 
 		CKAN_CATALOGUE_URL = runningInstance.getDataCatalogueUrl().get(0).trim();
@@ -113,17 +126,13 @@ public class DataCatalogueImpl implements DataCatalogue {
 		
 		// build the clients
 		gCatCaller = new GCatCaller(CKAN_CATALOGUE_URL);
-		
 		dbCaller = new DBCaller(CKAN_DB_URL, CKAN_DB_PORT, CKAN_DB_NAME, CKAN_DB_USER, CKAN_DB_PASSWORD);
-		
+		ckanCaller = new ExtendCkanClient(CKAN_CATALOGUE_URL);
-		ckanCaller = new SimpleExtendCkanClient(CKAN_CATALOGUE_URL);
+		directCkanCaller = new DirectCkanCaller(CKAN_CATALOGUE_URL);
-
 		// init map
 		apiKeysMap = new ConcurrentHashMap<String, CKANTokenBean>();
-
 		// save the context
 		CONTEXT = scope;
-
 		// extended roles
 		extendRoleInOrganization = runningInstance.getExtendRoleInOrganization();
 	}
@@ -228,7 +237,8 @@ public class DataCatalogueImpl implements DataCatalogue {
 			String authzToken = SecurityTokenProvider.instance.get();
 			if(authzToken!=null && !authzToken.isEmpty()) {
 				LOG.info("gcube-token found. Calling the gCat client");
-				return gCatCaller.getDatasetForName(datasetId);
+				String jsonDataset = gCatCaller.getDatasetForName(datasetId);
+				return MarshUnmarshCkanObject.toCkanDataset(jsonDataset);
 			}
 			
 			LOG.info("No api-key or gcube-token found. Calling Ckan Client without API-KEY");
@@ -272,13 +282,7 @@ public class DataCatalogueImpl implements DataCatalogue {
 		return url;
 	}
 
-	public String createCKanDatasetMultipleCustomFields(String title, String name, String organizationNameOrId,
+	
-			String author, String authorMail, String maintainer, String maintainerMail, long version,
-			String description, String licenseId, List<String> tags, Map<String, List<String>> customFields,
-			List<ResourceBean> resources, boolean setPublic) throws Exception {
-		return null;
-	}
-
 	@Override
 	public Map<String, Map<CkanGroup, RolesCkanGroupOrOrg>> getUserRoleByGroup(String username) {
 		LOG.info("Get user role by group called. The username is: "+username);
@@ -328,6 +332,7 @@ public class DataCatalogueImpl implements DataCatalogue {
 				HashMap<CkanOrganization, RolesCkanGroupOrOrg> subMap = new HashMap<CkanOrganization, RolesCkanGroupOrOrg>();
 				subMap.put(org, partialResult.get(orgID));
 				toReturn.put(orgID, subMap);
+				LOG.debug("For organisation: " + org.getName() + ", the user "+username+" has role: "+subMap);
 			}
 
 			LOG.debug("Returning map " + toReturn);
@@ -399,10 +404,8 @@ public class DataCatalogueImpl implements DataCatalogue {
 
 		// checks
 		checkNotNull(username);
-
 		// in order to avoid errors, the username is always converted
 		String ckanUsername = CatalogueUtilMethods.fromUsernameToCKanUsername(username);
-
 		// list to return
 		List<CkanOrganization> toReturn = new ArrayList<CkanOrganization>();
 
@@ -413,12 +416,11 @@ public class DataCatalogueImpl implements DataCatalogue {
 
 			// iterate over them
 			for (CkanOrganization ckanOrganization : organizations) {
-
 				// get the list of users in it (if you try ckanOrganization.getUsers() it returns null.. maybe a bug TODO)
 				List<CkanUser> users = ckanCaller.getOrganization(ckanOrganization.getName()).getUsers();
-
 				// check if the current user is among them
 				for (CkanUser ckanUser : users) {
+					
 					if(ckanUser.getName().equals(ckanUsername)){
 
 						LOG.debug("User " + ckanUsername + " is into " + ckanOrganization.getName());
@@ -440,10 +442,8 @@ public class DataCatalogueImpl implements DataCatalogue {
 
 		// checks
 		checkNotNull(username);
-
 		// in order to avoid errors, the username is always converted
 		String ckanUsername = CatalogueUtilMethods.fromUsernameToCKanUsername(username);
-
 		// list to return
 		List<CkanGroup> toReturn = new ArrayList<CkanGroup>();
 
@@ -459,6 +459,7 @@ public class DataCatalogueImpl implements DataCatalogue {
 
 				// check if the current user is among them
 				for (CkanUser ckanUser : users) {
+					
 					if(ckanUser.getName().equals(ckanUsername)){
 
 						LOG.debug("User " + ckanUsername + " is into " + ckanGroup.getName());
@@ -513,15 +514,22 @@ public class DataCatalogueImpl implements DataCatalogue {
 		return null;
 	}
 	
-	
+	@Override
-	
+	public boolean existProductWithNameOrId(String nameOrId) {
-	
+		
-	
+		checkNotNull(nameOrId);
-	
+		checkArgument(!nameOrId.isEmpty());
-	
+
-	
+		try{
-	
+			ExtendCkanClient client = new ExtendCkanClient(CKAN_CATALOGUE_URL, CKAN_TOKEN_SYS);
-	
+			CkanDataset product = client.getDataset(nameOrId);
+			return product != null;
+		}catch(Exception e){
+			LOG.debug("A dataset with name " + nameOrId + " doesn't exist");
+			return false;
+		}
+	}
+
 	
 	/*
 	 * 
@@ -546,6 +554,163 @@ public class DataCatalogueImpl implements DataCatalogue {
 	 * 
 	 */
 	
+	@Override
+	public String createCKanDatasetMultipleCustomFields(String username, String title, String name, String organizationNameOrId,
+			String author, String authorMail, String maintainer, String maintainerMail, long version,
+			String description, String licenseId, List<String> tags, Map<String, List<String>> customFieldsMultiple,
+			List<ResourceBean> resources, boolean setPublic) throws Exception {
+
+		// delegate the private method
+		return createCkanDatasetBody(username, title, name, organizationNameOrId, author, authorMail, maintainer, maintainerMail,
+				version, description, licenseId, tags, null, customFieldsMultiple, resources, setPublic);
+	}
+
+	// the body of the actual dataset creation methods
+	private String createCkanDatasetBody(String username, String title, String name, String organizationNameOrId, String author,
+			String authorMail, String maintainer, String maintainerMail, long version, String description,
+			String licenseId, List<String> tags, Map<String, String> customFields,
+			Map<String, List<String>> customFieldsMultipleValues, List<ResourceBean> resources, boolean setPublic)
+			throws Exception {
+
+		// checks (minimum)
+		checkNotNull(username);
+		checkNotNull(organizationNameOrId);
+		checkArgument(!organizationNameOrId.isEmpty());
+		checkArgument(!(title == null && name == null || title.isEmpty() && name.isEmpty()), "Name and Title cannot be empty/null at the same time!");
+
+		LOG.debug("Request for dataset creation");
+
+		//ExtendCkanClient client = new ExtendCkanClient(CKAN_CATALOGUE_URL, apiKey);
+
+		//String ckanUsername = getUserFromApiKey(apiKey).getName();
+		CkanDataset dataset = new CkanDataset();
+		
+		// in order to avoid errors, the username is always converted
+		String ckanUsername = CatalogueUtilMethods.fromUsernameToCKanUsername(username);
+
+		String nameToUse = name;
+		if(nameToUse == null)
+			nameToUse = CatalogueUtilMethods.fromProductTitleToName(title);
+
+		LOG.debug("Name of the dataset is going to be " + nameToUse + ". Title is going to be " + title);
+
+		dataset.setName(nameToUse);
+		dataset.setTitle(title);
+
+//		CkanOrganization orgOwner = client.getOrganization(organizationNameOrId);
+//		dataset.setOwnerOrg(orgOwner.getId());
+		dataset.setAuthor(author);
+		dataset.setAuthorEmail(authorMail);
+		dataset.setMaintainer(maintainer);
+		dataset.setMaintainerEmail(maintainerMail);
+		dataset.setVersion(String.valueOf(version));
+
+		// description must be escaped
+		if(description != null && !description.isEmpty()){
+			Source descriptionEscaped = new Source(description);
+			Segment htmlSeg = new Segment(descriptionEscaped, 0, descriptionEscaped.length());
+			Renderer htmlRend = new Renderer(htmlSeg);
+			dataset.setNotes(htmlRend.toString());
+
+			LOG.debug("Description escaped " + htmlRend.toString());
+		}
+
+		dataset.setLicenseId(licenseId);
+
+		// set the tags, if any
+		if(tags != null && !tags.isEmpty()){
+			List<CkanTag> ckanTags = new ArrayList<CkanTag>(tags.size());
+			for (String stringTag : tags) {
+				ckanTags.add(new CkanTag(stringTag));
+			}
+			dataset.setTags(ckanTags);
+		}
+
+		// set the custom fields, if any
+		List<CkanPair> extras = new ArrayList<CkanPair>();
+
+		if(customFields != null && !customFields.isEmpty()){
+
+			Iterator<Entry<String, String>> iterator = customFields.entrySet().iterator();
+
+			while (iterator.hasNext()) {
+				Map.Entry<String, String> entry = iterator.next();
+				extras.add(new CkanPair(entry.getKey(), entry.getValue()));
+			}
+
+		}else if(customFieldsMultipleValues != null && !customFieldsMultipleValues.isEmpty()){
+
+			Iterator<Entry<String, List<String>>> iterator = customFieldsMultipleValues.entrySet().iterator();
+			while (iterator.hasNext()) {
+				Map.Entry<String, List<String>> entry = iterator.next();
+				List<String> valuesForEntry = entry.getValue();
+				for (String value : valuesForEntry) {
+					extras.add(new CkanPair(entry.getKey(), value));
+				}
+			}
+		}
+
+		dataset.setExtras(extras);
+
+		// check if we need to add the resources
+		if(resources != null && !resources.isEmpty()){
+
+			LOG.debug("We need to add resources to the dataset");
+
+			try{
+				List<CkanResource> resourcesCkan = new ArrayList<CkanResource>();
+				for(ResourceBean resource: resources){
+
+					LOG.debug("Going to add resource described by " + resource);
+					CkanResource newResource = new CkanResource();
+					newResource.setDescription(resource.getDescription());
+					newResource.setId(resource.getId());
+					newResource.setUrl(resource.getUrl());
+					newResource.setName(resource.getName());
+					newResource.setMimetype(resource.getMimeType());
+					newResource.setFormat(resource.getMimeType());
+					newResource.setOwner(ckanUsername);
+					resourcesCkan.add(newResource);
+				}
+
+				// add to the dataset
+				dataset.setResources(resourcesCkan);
+
+			}catch(Exception e){
+				LOG.error("Unable to add those resources to the dataset", e);
+			}
+
+		}
+
+		// try to create
+		String jsonValueDataset = MarshUnmarshCkanObject.toJsonValueDataset(dataset);
+		LOG.debug("Marshalling dataset is: " + jsonValueDataset);
+		jsonValueDataset = gCatCaller.createDataset(jsonValueDataset,true);
+		LOG.debug("Created dataset is: " + jsonValueDataset);
+
+		if(jsonValueDataset != null){
+			CkanDataset toCkanDataset = MarshUnmarshCkanObject.toCkanDataset(jsonValueDataset);
+			LOG.debug("Dataset with name " + toCkanDataset.getName() + " has been created. Setting visibility");
+
+			// set visibility
+			boolean visibilitySet = directCkanCaller.setDatasetPrivate(
+					!setPublic, // swap to private
+					toCkanDataset.getOrganization().getId(),
+					toCkanDataset.getId(),
+					CKAN_TOKEN_SYS); // use sysadmin api key to be sure it will be set
+
+			LOG.info("Was visibility set to " + (setPublic ? "public" : "private") + "? " + visibilitySet);
+
+			// set searchable to true if dataset visibility is private
+			if(!setPublic){ // (swap to private)
+				boolean searchableSet = setSearchableField(toCkanDataset.getId(), true);
+				LOG.info("Was searchable set to True? " + searchableSet);
+			}
+			return toCkanDataset.getId();
+		}
+
+		return null;
+	}
 	
 	public boolean patchProductCustomFields(String productId, Map<String, List<String>> customFieldsToChange,
 			boolean removeOld) {
@@ -560,9 +725,11 @@ public class DataCatalogueImpl implements DataCatalogue {
 		return false;
 	}
 
-	public boolean existProductWithNameOrId(String nameOrId) {
+	@Override
-		return false;
+	public boolean setSearchableField(String datasetId, boolean searchable) {
+		return directCkanCaller.setSearchableField(datasetId, searchable, CKAN_TOKEN_SYS);
 	}
 
 
+
 }