From 3e7c11ad4f55a1335df496b898713f9f2df9b966 Mon Sep 17 00:00:00 2001 From: Francesco Mangiacrapa Date: Wed, 27 Jun 2018 10:04:03 +0000 Subject: [PATCH] Fix: https://support.d4science.org/issues/12040#note-9 Increasing http timeout releasing ExtendCkanClient git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/data-catalogue/ckan-util-library@169475 82a268e6-3cf1-43bd-a215-b396298e98cf --- distro/changelog.xml | 4 + pom.xml | 8 +- .../server/DataCatalogueImpl.java | 157 +++++++++--------- .../server/ExtendCkanClient.java | 118 +++++++++++++ 4 files changed, 207 insertions(+), 80 deletions(-) create mode 100644 src/main/java/org/gcube/datacatalogue/ckanutillibrary/server/ExtendCkanClient.java diff --git a/distro/changelog.xml b/distro/changelog.xml index a845a38..52681d8 100644 --- a/distro/changelog.xml +++ b/distro/changelog.xml @@ -1,6 +1,10 @@ + + [Task 12040] Increasing http timeout releasing ExtendCkanClient + diff --git a/pom.xml b/pom.xml index a2adf65..1f1b5c0 100644 --- a/pom.xml +++ b/pom.xml @@ -10,7 +10,7 @@ org.gcube.data-catalogue ckan-util-library - 2.5.0-SNAPSHOT + 2.6.0-SNAPSHOT jar CKan utility library @@ -73,6 +73,12 @@ ${jackanVersion} compile + + org.apache.httpcomponents + fluent-hc + 4.5.3 + provided + junit junit diff --git a/src/main/java/org/gcube/datacatalogue/ckanutillibrary/server/DataCatalogueImpl.java b/src/main/java/org/gcube/datacatalogue/ckanutillibrary/server/DataCatalogueImpl.java index 8652a62..4a9f4f0 100644 --- a/src/main/java/org/gcube/datacatalogue/ckanutillibrary/server/DataCatalogueImpl.java +++ b/src/main/java/org/gcube/datacatalogue/ckanutillibrary/server/DataCatalogueImpl.java @@ -51,7 +51,6 @@ import org.json.simple.parser.JSONParser; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import eu.trentorise.opendata.jackan.CheckedCkanClient; import eu.trentorise.opendata.jackan.CkanClient; import eu.trentorise.opendata.jackan.CkanQuery; import eu.trentorise.opendata.jackan.exceptions.JackanException; @@ -151,7 +150,7 @@ public class DataCatalogueImpl implements DataCatalogue{ logger.debug("Plain db password first 3 chars are " + CKAN_DB_PASSWORD.substring(0, 3)); // build the client - client = new CkanClient(CKAN_CATALOGUE_URL); + //client = new CkanClient(CKAN_CATALOGUE_URL); // init map apiKeysMap = new ConcurrentHashMap(); @@ -194,8 +193,8 @@ public class DataCatalogueImpl implements DataCatalogue{ /** * Retrieve connection from the pool * @return a connection available within the pool - * @throws SQLException - * @throws ClassNotFoundException + * @throws SQLException + * @throws ClassNotFoundException */ private Connection getConnection() throws SQLException, ClassNotFoundException{ @@ -239,7 +238,7 @@ public class DataCatalogueImpl implements DataCatalogue{ // check in the hashmap first if(apiKeysMap.containsKey(ckanUsername)){ CKANTokenBean bean = apiKeysMap.get(ckanUsername); - if((bean.timestamp + EXPIRE_KEY_TIME) > System.currentTimeMillis()){ // it's still ok + if(bean.timestamp + EXPIRE_KEY_TIME > System.currentTimeMillis()){ // it's still ok return bean.apiKey; } } @@ -451,7 +450,7 @@ public class DataCatalogueImpl implements DataCatalogue{ if(rolesToMatch.contains(RolesCkanGroupOrOrg.convertFromCapacity(capacity))){ RolesCkanGroupOrOrg enumRole = RolesCkanGroupOrOrg.convertFromCapacity(capacity); rolesIntoOrg.add(enumRole); - logger.debug("User " + ckanUsername + " has role " + enumRole + + logger.debug("User " + ckanUsername + " has role " + enumRole + " into organization with name " + ckanOrganization.getName()); } @@ -657,7 +656,7 @@ public class DataCatalogueImpl implements DataCatalogue{ resource.setOwner(ckanUsername); // Checked client - CheckedCkanClient client = new CheckedCkanClient(CKAN_CATALOGUE_URL, apiKey); + ExtendCkanClient client = new ExtendCkanClient(CKAN_CATALOGUE_URL, apiKey); CkanResource createdRes = client.createResource(resource); if(createdRes != null){ @@ -686,7 +685,7 @@ public class DataCatalogueImpl implements DataCatalogue{ try{ - CheckedCkanClient client = new CheckedCkanClient(CKAN_CATALOGUE_URL, apiKey); + ExtendCkanClient client = new ExtendCkanClient(CKAN_CATALOGUE_URL, apiKey); client.deleteResource(resourceId); return true; @@ -746,11 +745,11 @@ public class DataCatalogueImpl implements DataCatalogue{ checkNotNull(organizationNameOrId); checkArgument(!apiKey.isEmpty()); checkArgument(!organizationNameOrId.isEmpty()); - checkArgument(!((title == null && name == null) || (title.isEmpty() && name.isEmpty())), "Name and Title cannot be empty/null at the same time!"); + checkArgument(!(title == null && name == null || title.isEmpty() && name.isEmpty()), "Name and Title cannot be empty/null at the same time!"); logger.debug("Request for dataset creation"); - CheckedCkanClient client = new CheckedCkanClient(CKAN_CATALOGUE_URL, apiKey); + ExtendCkanClient client = new ExtendCkanClient(CKAN_CATALOGUE_URL, apiKey); String ckanUsername = getUserFromApiKey(apiKey).getName(); CkanDataset dataset = new CkanDataset(); @@ -801,7 +800,7 @@ public class DataCatalogueImpl implements DataCatalogue{ Iterator> iterator = customFields.entrySet().iterator(); while (iterator.hasNext()) { - Map.Entry entry = (Map.Entry) iterator.next(); + Map.Entry entry = iterator.next(); extras.add(new CkanPair(entry.getKey(), entry.getValue())); } @@ -809,7 +808,7 @@ public class DataCatalogueImpl implements DataCatalogue{ Iterator>> iterator = customFieldsMultipleValues.entrySet().iterator(); while (iterator.hasNext()) { - Map.Entry> entry = (Map.Entry>) iterator.next(); + Map.Entry> entry = iterator.next(); List valuesForEntry = entry.getValue(); for (String value : valuesForEntry) { extras.add(new CkanPair(entry.getKey(), value)); @@ -858,9 +857,9 @@ public class DataCatalogueImpl implements DataCatalogue{ // set visibility boolean visibilitySet = setDatasetPrivate( - !setPublic, // swap to private - res.getOrganization().getId(), - res.getId(), + !setPublic, // swap to private + res.getOrganization().getId(), + res.getId(), CKAN_TOKEN_SYS); // use sysadmin api key to be sure it will be set logger.info("Was visibility set to " + (setPublic ? "public" : "private") + "? " + visibilitySet); @@ -892,11 +891,11 @@ public class DataCatalogueImpl implements DataCatalogue{ checkArgument(!apiKey.isEmpty()); checkArgument(!organizationNameOrId.isEmpty()); checkArgument(!id.isEmpty()); - checkArgument(!((title == null && name == null) || (title.isEmpty() && name.isEmpty())), "Name and Title cannot be empty/null at the same time!"); + checkArgument(!(title == null && name == null || title.isEmpty() && name.isEmpty()), "Name and Title cannot be empty/null at the same time!"); logger.debug("Request for dataset update"); - CheckedCkanClient client = new CheckedCkanClient(CKAN_CATALOGUE_URL, apiKey); + ExtendCkanClient client = new ExtendCkanClient(CKAN_CATALOGUE_URL, apiKey); String ckanUsername = getUserFromApiKey(apiKey).getName(); @@ -947,7 +946,7 @@ public class DataCatalogueImpl implements DataCatalogue{ Iterator>> iterator = customFields.entrySet().iterator(); while (iterator.hasNext()) { - Map.Entry> entry = (Map.Entry>) iterator.next(); + Map.Entry> entry = iterator.next(); List valuesForEntry = entry.getValue(); for (String value : valuesForEntry) { extras.add(new CkanPair(entry.getKey(), value)); @@ -1005,14 +1004,14 @@ public class DataCatalogueImpl implements DataCatalogue{ // set visibility boolean visibilitySet = setDatasetPrivate( - !setPublic, // swap to private - updated.getOrganization().getId(), - updated.getId(), + !setPublic, // swap to private + updated.getOrganization().getId(), + updated.getId(), CKAN_TOKEN_SYS); // use sysadmin api key to be sure it will be set logger.info("Was visibility set to " + (setPublic ? "public" : "private") + "? " + visibilitySet); - if(!setPublic){ + if(!setPublic){ boolean searchableSet = setSearchableField(updated.getId(), true); logger.info("Was searchable set to True? " + searchableSet); } @@ -1035,7 +1034,7 @@ public class DataCatalogueImpl implements DataCatalogue{ try{ // get the dataset from name - CheckedCkanClient client = new CheckedCkanClient(CKAN_CATALOGUE_URL, CKAN_TOKEN_SYS); + ExtendCkanClient client = new ExtendCkanClient(CKAN_CATALOGUE_URL, CKAN_TOKEN_SYS); CkanDataset dataset = client.getDataset(datasetIdOrName); String name = dataset.getName(); @@ -1065,7 +1064,7 @@ public class DataCatalogueImpl implements DataCatalogue{ try{ // get the dataset from name - CheckedCkanClient client = new CheckedCkanClient(CKAN_CATALOGUE_URL, CKAN_TOKEN_SYS); + ExtendCkanClient client = new ExtendCkanClient(CKAN_CATALOGUE_URL, CKAN_TOKEN_SYS); CkanDataset dataset = client.getDataset(datasetIdOrName); String name = dataset.getName(); @@ -1167,7 +1166,7 @@ public class DataCatalogueImpl implements DataCatalogue{ HttpResponse response = httpClient.execute(request); logger.debug("Response code is " + response.getStatusLine().getStatusCode() + " and response message is " + response.getStatusLine().getReasonPhrase()); - return (response.getStatusLine().getStatusCode() == HttpStatus.SC_OK); + return response.getStatusLine().getStatusCode() == HttpStatus.SC_OK; }catch (Exception ex) { logger.error("Error while trying to change the role for this user ", ex); @@ -1250,7 +1249,7 @@ public class DataCatalogueImpl implements DataCatalogue{ HttpResponse response = httpClient.execute(request); logger.debug("Response code is " + response.getStatusLine().getStatusCode() + " and response message is " + response.getStatusLine().getReasonPhrase()); - return (response.getStatusLine().getStatusCode() == HttpStatus.SC_OK); + return response.getStatusLine().getStatusCode() == HttpStatus.SC_OK; }catch (Exception ex) { logger.error("Error while trying to change the role for this user ", ex); @@ -1315,10 +1314,10 @@ public class DataCatalogueImpl implements DataCatalogue{ HttpResponse response = httpClient.execute(request); logger.debug("Response code is " + response.getStatusLine().getStatusCode() + " and response message is " + response.getStatusLine().getReasonPhrase()); - return (response.getStatusLine().getStatusCode() == HttpStatus.SC_OK); + return response.getStatusLine().getStatusCode() == HttpStatus.SC_OK; }catch(Exception e){ - logger.error("Failed to create the relationship between dataset subject = " + datasetIdSubject + logger.error("Failed to create the relationship between dataset subject = " + datasetIdSubject + " and " + " dataset subject " + datasetIdObject, e); } return false; @@ -1356,9 +1355,9 @@ public class DataCatalogueImpl implements DataCatalogue{ HttpResponse response = httpClient.execute(request); logger.debug("Response code is " + response.getStatusLine().getStatusCode() + " and response message is " + response.getStatusLine().getReasonPhrase()); - return (response.getStatusLine().getStatusCode() == HttpStatus.SC_OK); + return response.getStatusLine().getStatusCode() == HttpStatus.SC_OK; }catch(Exception e){ - logger.error("Failed to delete the relationship between dataset subject = " + datasetIdSubject + logger.error("Failed to delete the relationship between dataset subject = " + datasetIdSubject + " and " + " dataset subject " + datasetIdObject, e); } return false; @@ -1408,7 +1407,7 @@ public class DataCatalogueImpl implements DataCatalogue{ Iterator it = resultJson.iterator(); while (it.hasNext()) { - JSONObject object = (JSONObject) it.next(); + JSONObject object = it.next(); try{ CkanDatasetRelationship relation = new CkanDatasetRelationship(object); toReturn.add(relation); @@ -1421,7 +1420,7 @@ public class DataCatalogueImpl implements DataCatalogue{ return toReturn; }catch(Exception e){ - logger.error("Failed to retrieve the relationship between dataset subject = " + datasetIdSubject + logger.error("Failed to retrieve the relationship between dataset subject = " + datasetIdSubject + " and " + " dataset object " + datasetIdObject, e); } return null; @@ -1435,7 +1434,7 @@ public class DataCatalogueImpl implements DataCatalogue{ checkArgument(!nameOrId.isEmpty()); try{ - CheckedCkanClient client = new CheckedCkanClient(CKAN_CATALOGUE_URL, CKAN_TOKEN_SYS); + ExtendCkanClient client = new ExtendCkanClient(CKAN_CATALOGUE_URL, CKAN_TOKEN_SYS); CkanDataset product = client.getDataset(nameOrId); return product != null; }catch(Exception e){ @@ -1453,7 +1452,7 @@ public class DataCatalogueImpl implements DataCatalogue{ // check if it exists CkanGroup toCreate = null; - CheckedCkanClient client = new CheckedCkanClient(CKAN_CATALOGUE_URL, CKAN_TOKEN_SYS); + ExtendCkanClient client = new ExtendCkanClient(CKAN_CATALOGUE_URL, CKAN_TOKEN_SYS); logger.debug("Request for creating group with name " + nameOrId + " title " + title + " and description " + description); @@ -1483,7 +1482,7 @@ public class DataCatalogueImpl implements DataCatalogue{ * @param client * @return */ - private CkanGroup groupExists(String nameOrId, CheckedCkanClient client){ + private CkanGroup groupExists(String nameOrId, ExtendCkanClient client){ CkanGroup toReturn = null; @@ -1519,11 +1518,11 @@ public class DataCatalogueImpl implements DataCatalogue{ * @param user's api key */ private void findHierarchyGroups( - List groupsTitles, + List groupsTitles, String apiKey) { ListIterator iterator = groupsTitles.listIterator(); while (iterator.hasNext()) { - String group = (String) iterator.next(); + String group = iterator.next(); List parents = getParentGroups(group, apiKey); @@ -1534,7 +1533,7 @@ public class DataCatalogueImpl implements DataCatalogue{ List parentsList = new ArrayList(Arrays.asList(ckanGroup.getName())); findHierarchyGroups(parentsList, apiKey); for (String parent : parentsList) { - iterator.add(parent); + iterator.add(parent); } } } @@ -1563,7 +1562,7 @@ public class DataCatalogueImpl implements DataCatalogue{ try(CloseableHttpClient httpClient = HttpClientBuilder.create().build();){ - CheckedCkanClient client = new CheckedCkanClient(CKAN_CATALOGUE_URL, CKAN_TOKEN_SYS); + ExtendCkanClient client = new ExtendCkanClient(CKAN_CATALOGUE_URL, CKAN_TOKEN_SYS); // check the group exists CkanGroup group = client.getGroup(groupNameToCheck); @@ -1595,7 +1594,7 @@ public class DataCatalogueImpl implements DataCatalogue{ Iterator it = groupsJson.iterator(); while (it.hasNext()) { - JSONObject object = (JSONObject) it.next(); + JSONObject object = it.next(); try{ if(object.containsKey("name")) groupNames.add((String)object.get("name")); @@ -1618,7 +1617,7 @@ public class DataCatalogueImpl implements DataCatalogue{ JSONArray groups = new JSONArray(); Iterator iteratorNameSet = groupNamesSet.iterator(); while (iteratorNameSet.hasNext()) { - String groupName = (String) iteratorNameSet.next(); + String groupName = iteratorNameSet.next(); JSONObject groupJSON = new JSONObject(); groupJSON.put("name", groupName); groups.add(groupJSON); @@ -1663,11 +1662,11 @@ public class DataCatalogueImpl implements DataCatalogue{ HttpPost httpPostRequest = new HttpPost(apiRequestUrl); httpPostRequest.setHeader("Authorization", CKAN_TOKEN_SYS); - // Request parameters to be replaced + // Request parameters to be replaced JSONObject jsonRequest = new JSONObject(); JSONArray groupsAsJson = new JSONArray(); - CheckedCkanClient client = new CheckedCkanClient(CKAN_CATALOGUE_URL, CKAN_TOKEN_SYS); + ExtendCkanClient client = new ExtendCkanClient(CKAN_CATALOGUE_URL, CKAN_TOKEN_SYS); CkanDataset dataset = client.getDataset(datasetNameOrId); List currentGroups = dataset.getGroups(); @@ -1781,7 +1780,7 @@ public class DataCatalogueImpl implements DataCatalogue{ try{ - CheckedCkanClient client = new CheckedCkanClient(CKAN_CATALOGUE_URL, apiKey); + ExtendCkanClient client = new ExtendCkanClient(CKAN_CATALOGUE_URL, apiKey); List users = client.getOrganization(orgName).getUsers(); for (CkanUser ckanUser : users) { if(ckanUser.getName().equals(usernameCkan)){ @@ -1806,7 +1805,7 @@ public class DataCatalogueImpl implements DataCatalogue{ try{ - CheckedCkanClient client = new CheckedCkanClient(CKAN_CATALOGUE_URL, apiKey); + ExtendCkanClient client = new ExtendCkanClient(CKAN_CATALOGUE_URL, apiKey); List users = client.getGroup(groupName).getUsers(); for (CkanUser ckanUser : users) { if(ckanUser.getName().equals(usernameCkan)){ @@ -1836,7 +1835,7 @@ public class DataCatalogueImpl implements DataCatalogue{ try{ - CheckedCkanClient client = new CheckedCkanClient(CKAN_CATALOGUE_URL, apiKey); + ExtendCkanClient client = new ExtendCkanClient(CKAN_CATALOGUE_URL, apiKey); client.deleteDataset(datasetId); logger.info("Dataset with id " + datasetId + " deleted!"); @@ -1884,7 +1883,7 @@ public class DataCatalogueImpl implements DataCatalogue{ checkArgument(!apiKey.isEmpty()); try{ - CheckedCkanClient client = new CheckedCkanClient(CKAN_CATALOGUE_URL, apiKey); + ExtendCkanClient client = new ExtendCkanClient(CKAN_CATALOGUE_URL, apiKey); return client.getResource(id); }catch(Exception e){ logger.error("Unable to retrieve such resource, returning null ...", e); @@ -1905,7 +1904,7 @@ public class DataCatalogueImpl implements DataCatalogue{ checkArgument(!apiKey.isEmpty()); try{ - CheckedCkanClient client = new CheckedCkanClient(CKAN_CATALOGUE_URL, apiKey); + ExtendCkanClient client = new ExtendCkanClient(CKAN_CATALOGUE_URL, apiKey); return client.getDataset(datasetId); }catch(Exception e){ logger.error("Unable to retrieve such dataset, returning null ...", e); @@ -1989,7 +1988,7 @@ public class DataCatalogueImpl implements DataCatalogue{ String apiRequestUrl = CKAN_CATALOGUE_URL + "/api/3/action/resource_create"; HttpPost httpPostRequest = new HttpPost(apiRequestUrl); httpPostRequest.setHeader("Authorization", token); - HttpEntity mpEntity = + HttpEntity mpEntity = MultipartEntityBuilder.create() .addTextBody("package_id", packageId, ContentType.TEXT_PLAIN) .addTextBody("url", "upload", ContentType.TEXT_PLAIN) @@ -1997,10 +1996,10 @@ public class DataCatalogueImpl implements DataCatalogue{ .addTextBody("name", name, ContentType.TEXT_PLAIN) .addTextBody("mimetype", mimeType) .addTextBody("format", format) - .addBinaryBody("upload", file, + .addBinaryBody("upload", file, ContentType.create( "application/octet-stream", - Charset.forName("UTF-8")), + Charset.forName("UTF-8")), name) .build(); @@ -2020,7 +2019,7 @@ public class DataCatalogueImpl implements DataCatalogue{ JSONObject result = (JSONObject)finalResult.get("result"); logger.debug("Returned json is " + result.get("id")); returnedId = (String) result.get("id"); - return new CheckedCkanClient(CKAN_CATALOGUE_URL, token).getResource(returnedId); + return new ExtendCkanClient(CKAN_CATALOGUE_URL, token).getResource(returnedId); } catch (Exception e) { logger.error("Error while uploading file"); return null; @@ -2045,7 +2044,7 @@ public class DataCatalogueImpl implements DataCatalogue{ HttpPost httpPostRequest = new HttpPost(apiRequestUrl); httpPostRequest.setHeader("Authorization", apiKey); - // Request parameters to be replaced + // Request parameters to be replaced JSONObject jsonRequest = new JSONObject(); Map requestMap = new HashMap(); @@ -2097,7 +2096,7 @@ public class DataCatalogueImpl implements DataCatalogue{ // Get already available custom fields Map> fromCKANCustomFields = new HashMap>(); - CheckedCkanClient client = new CheckedCkanClient(CKAN_CATALOGUE_URL, apiKey); + ExtendCkanClient client = new ExtendCkanClient(CKAN_CATALOGUE_URL, apiKey); List extras = client.getDataset(productId).getExtras(); if(extras == null) @@ -2120,7 +2119,7 @@ public class DataCatalogueImpl implements DataCatalogue{ // merge them with the new values Iterator>> iteratorUserMap = customFieldsToChange.entrySet().iterator(); while (iteratorUserMap.hasNext()) { - Map.Entry> entry = (Map.Entry>) iteratorUserMap + Map.Entry> entry = iteratorUserMap .next(); String key = entry.getKey(); @@ -2144,7 +2143,7 @@ public class DataCatalogueImpl implements DataCatalogue{ HttpPost httpPostRequest = new HttpPost(apiRequestUrl); httpPostRequest.setHeader("Authorization", apiKey); - // Request parameters to be replaced + // Request parameters to be replaced JSONObject jsonRequest = new JSONObject(); // build the json array for the "extras" field.. each object looks like {"key": ..., "value": ...} @@ -2152,7 +2151,7 @@ public class DataCatalogueImpl implements DataCatalogue{ Iterator>> iteratorNewFields = fromCKANCustomFields.entrySet().iterator(); while (iteratorNewFields.hasNext()) { - Map.Entry> entry = (Map.Entry>) iteratorNewFields + Map.Entry> entry = iteratorNewFields .next(); String key = entry.getKey(); @@ -2207,12 +2206,12 @@ public class DataCatalogueImpl implements DataCatalogue{ logger.info("The couple key/value to remove from custom fields is [" + key + "," + value +"]"); - CheckedCkanClient client = new CheckedCkanClient(CKAN_CATALOGUE_URL, apiKey); + ExtendCkanClient client = new ExtendCkanClient(CKAN_CATALOGUE_URL, apiKey); List extras = client.getDataset(productId).getExtras(); Iterator iterator = extras.iterator(); while (iterator.hasNext()) { - CkanPair ckanPair = (CkanPair) iterator.next(); + CkanPair ckanPair = iterator.next(); if(ckanPair.getKey().equals(key) && ckanPair.getValue().equals(value)){ logger.info("Removed from the ckan pairs list"); @@ -2227,7 +2226,7 @@ public class DataCatalogueImpl implements DataCatalogue{ HttpPost httpPostRequest = new HttpPost(apiRequestUrl); httpPostRequest.setHeader("Authorization", apiKey); - // Request parameters to be replaced + // Request parameters to be replaced JSONObject jsonRequest = new JSONObject(); // build the json array for the "extras" field.. each object looks like {"key": ..., "value": ...} @@ -2279,15 +2278,15 @@ public class DataCatalogueImpl implements DataCatalogue{ HttpPost httpPostRequest = new HttpPost(apiRequestUrl); httpPostRequest.setHeader("Authorization", apiKey); - // Request parameters to be replaced + // Request parameters to be replaced JSONObject jsonRequest = new JSONObject(); JSONArray tagsAsJson = new JSONArray(); - CheckedCkanClient client = new CheckedCkanClient(CKAN_CATALOGUE_URL, apiKey); + ExtendCkanClient client = new ExtendCkanClient(CKAN_CATALOGUE_URL, apiKey); CkanDataset dataset = client.getDataset(productId); List currentTags = dataset.getTags(); - // build the json array for the "tags" field.. each object looks like + // build the json array for the "tags" field.. each object looks like // { // // "vocabulary_id": null, @@ -2345,7 +2344,7 @@ public class DataCatalogueImpl implements DataCatalogue{ try{ - CheckedCkanClient client = new CheckedCkanClient(CKAN_CATALOGUE_URL, apiKey); + ExtendCkanClient client = new ExtendCkanClient(CKAN_CATALOGUE_URL, apiKey); CkanDataset dataset = client.getDataset(productId); List currentTags = dataset.getTags(); @@ -2355,7 +2354,7 @@ public class DataCatalogueImpl implements DataCatalogue{ // check if it is already there ... while (tagsIterator.hasNext()) { - CkanTag ckanTag = (CkanTag) tagsIterator.next(); + CkanTag ckanTag = tagsIterator.next(); if(ckanTag.getName().equals(tagToAdd)){ added = false; break; @@ -2384,7 +2383,7 @@ public class DataCatalogueImpl implements DataCatalogue{ checkNotNull(apiKey); try{ - CheckedCkanClient client = new CheckedCkanClient(CKAN_CATALOGUE_URL, apiKey); + ExtendCkanClient client = new ExtendCkanClient(CKAN_CATALOGUE_URL, apiKey); return client.getGroup(groupName).getGroups(); }catch(Exception e){ logger.error("Something went wrong, returning null", e); @@ -2406,7 +2405,7 @@ public class DataCatalogueImpl implements DataCatalogue{ HttpPost httpPostRequest = new HttpPost(apiRequestUrl); httpPostRequest.setHeader("Authorization", CKAN_TOKEN_SYS); - CheckedCkanClient client = new CheckedCkanClient(CKAN_CATALOGUE_URL, CKAN_TOKEN_SYS); + ExtendCkanClient client = new ExtendCkanClient(CKAN_CATALOGUE_URL, CKAN_TOKEN_SYS); List alreadyAvailableParents = client.getGroup(groupName).getGroups(); Set parentsNames = new HashSet(); @@ -2418,7 +2417,7 @@ public class DataCatalogueImpl implements DataCatalogue{ logger.info("Setting as parents of group " + groupName + " :" + parentsNames); - // Request parameters to be replaced + // Request parameters to be replaced JSONObject jsonRequest = new JSONObject(); JSONArray parentsAsJson = new JSONArray(); @@ -2460,7 +2459,7 @@ public class DataCatalogueImpl implements DataCatalogue{ checkNotNull(datasetId); checkNotNull(groupName); - CheckedCkanClient client = new CheckedCkanClient(CKAN_CATALOGUE_URL, CKAN_TOKEN_SYS); + ExtendCkanClient client = new ExtendCkanClient(CKAN_CATALOGUE_URL, CKAN_TOKEN_SYS); List groups = client.getDataset(datasetId).getGroups(); for (CkanGroup ckanGroup : groups) { @@ -2480,7 +2479,7 @@ public class DataCatalogueImpl implements DataCatalogue{ try (CloseableHttpClient httpClient = HttpClientBuilder.create().build()) { - CheckedCkanClient client = new CheckedCkanClient(CKAN_CATALOGUE_URL, CKAN_TOKEN_SYS); + ExtendCkanClient client = new ExtendCkanClient(CKAN_CATALOGUE_URL, CKAN_TOKEN_SYS); String apiRequestUrl = CKAN_CATALOGUE_URL + "/api/3/action/group_show" + "?id=" + groupName + "&include_datasets=true"; HttpGet httpGetRequest = new HttpGet(apiRequestUrl); httpGetRequest.setHeader("Authorization", CKAN_TOKEN_SYS); @@ -2735,8 +2734,8 @@ public class DataCatalogueImpl implements DataCatalogue{ * Retrieve the list of groups(plus capacities) the user belongs by querying directly the database. * @param username * @return - * @throws SQLException - * @throws ClassNotFoundException + * @throws SQLException + * @throws ClassNotFoundException */ private Map getGroupsByUserFromDB(String username){ checkNotNull(username); @@ -2748,7 +2747,7 @@ public class DataCatalogueImpl implements DataCatalogue{ connection = getConnection(); ResultSet rs; - CheckedCkanClient client = new CheckedCkanClient(CKAN_CATALOGUE_URL, CKAN_TOKEN_SYS); + ExtendCkanClient client = new ExtendCkanClient(CKAN_CATALOGUE_URL, CKAN_TOKEN_SYS); String userId = client.getUser(username).getId(); String joinQuery = "SELECT \"group_id\",\"capacity\" FROM \"public\".\"member\" " + "JOIN \"public\".\"group\" ON \"member\".\"group_id\" = \"group\".\"id\" where \"table_id\"=?" @@ -2778,8 +2777,8 @@ public class DataCatalogueImpl implements DataCatalogue{ * Retrieve the list of organizations(plus capacities) the user belongs by querying directly the database. * @param username * @return - * @throws SQLException - * @throws ClassNotFoundException + * @throws SQLException + * @throws ClassNotFoundException */ private Map getOrganizationsByUserFromDB(String username){ checkNotNull(username); @@ -2791,7 +2790,7 @@ public class DataCatalogueImpl implements DataCatalogue{ connection = getConnection(); ResultSet rs; - CheckedCkanClient client = new CheckedCkanClient(CKAN_CATALOGUE_URL, CKAN_TOKEN_SYS); + ExtendCkanClient client = new ExtendCkanClient(CKAN_CATALOGUE_URL, CKAN_TOKEN_SYS); String userId = client.getUser(username).getId(); String joinQuery = "SELECT \"group_id\",\"capacity\" FROM \"public\".\"member\" " + "JOIN \"public\".\"group\" ON \"member\".\"group_id\" = \"group\".\"id\" where \"table_id\"=?" @@ -2846,7 +2845,7 @@ public class DataCatalogueImpl implements DataCatalogue{ Iterator> iterator = extendRoleInOrganization.entrySet().iterator(); logger.debug("List of entries to check is " + extendRoleInOrganization); while (iterator.hasNext()) { - Map.Entry entry = (Map.Entry) iterator + Map.Entry entry = iterator .next(); String sourceOrg = entry.getKey(); @@ -2885,7 +2884,7 @@ public class DataCatalogueImpl implements DataCatalogue{ checkArgument(start >= 0); checkArgument(offset >= 0); - CheckedCkanClient client = new CheckedCkanClient(CKAN_CATALOGUE_URL, apiKey); + ExtendCkanClient client = new ExtendCkanClient(CKAN_CATALOGUE_URL, apiKey); try{ @@ -2906,7 +2905,7 @@ public class DataCatalogueImpl implements DataCatalogue{ checkArgument(start >= 0); checkArgument(offset >= 0); - CheckedCkanClient client = new CheckedCkanClient(CKAN_CATALOGUE_URL, apiKey); + ExtendCkanClient client = new ExtendCkanClient(CKAN_CATALOGUE_URL, apiKey); try{ diff --git a/src/main/java/org/gcube/datacatalogue/ckanutillibrary/server/ExtendCkanClient.java b/src/main/java/org/gcube/datacatalogue/ckanutillibrary/server/ExtendCkanClient.java new file mode 100644 index 0000000..e35a52a --- /dev/null +++ b/src/main/java/org/gcube/datacatalogue/ckanutillibrary/server/ExtendCkanClient.java @@ -0,0 +1,118 @@ +/** + * + */ +package org.gcube.datacatalogue.ckanutillibrary.server; + +import javax.annotation.Nullable; + +import org.apache.http.HttpHost; +import org.apache.http.client.fluent.Request; + +import eu.trentorise.opendata.jackan.CheckedCkanClient; + + +/** + * The Class ExtendCkanClient. + * + * @author Francesco Mangiacrapa at ISTI-CNR (francesco.mangiacrapa@isti.cnr.it) + * Jun 27, 2018 + */ +public class ExtendCkanClient extends CheckedCkanClient{ + + + private String catalogueURL; + private String ckanToken; + private HttpHost proxy = null; //never used + private int timeout = 120000; //in milliseconds + + + /** + * Instantiates a new extend ckan client. + * + * @param catalogueURL the catalogue url + */ + public ExtendCkanClient(String catalogueURL) { + super(catalogueURL); + this.catalogueURL = catalogueURL; + } + + /** + * Instantiates a new extend ckan client. + * + * @param catalogueURL the catalogue url + * @param ckanToken the ckan token + */ + public ExtendCkanClient(String catalogueURL, @Nullable String ckanToken) { + super(catalogueURL, ckanToken); + this.catalogueURL = catalogueURL; + this.ckanToken = ckanToken; + } + + /** + * Instantiates a new extend ckan client. + * + * @param catalogueURL the catalogue url + * @param ckanToken the ckan token + * @param timeout the timeout + */ + public ExtendCkanClient(String catalogueURL, @Nullable String ckanToken, int timeout) { + super(catalogueURL, ckanToken); + this.catalogueURL = catalogueURL; + this.ckanToken = ckanToken; + this.timeout = timeout; + } + + + /** + * Configures the request. Should work both for GETs and POSTs. + * + * @param request the request + * @return the request + */ + protected Request configureRequest(Request request) { + if (ckanToken != null) { + request.addHeader("Authorization", ckanToken); + } + if (proxy != null) { + request.viaProxy(proxy); + } + request.socketTimeout(this.timeout) + .connectTimeout(this.timeout); + + return request; + } + + + /** + * Gets the catalogue url. + * + * @return the catalogueURL + */ + public String getCatalogueURL() { + + return catalogueURL; + } + + + /** + * Gets the ckan token. + * + * @return the ckanToken + */ + public String getCkanToken() { + + return ckanToken; + } + + /** + * Gets the timeout. + * + * @return the timeout + */ + public int getTimeout() { + + return timeout; + } + + +}