package org.gcube.datacatalogue.ckanutillibrary.server; import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkNotNull; import java.net.URLEncoder; import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import org.gcube.common.authorization.library.provider.SecurityTokenProvider; import org.gcube.common.scope.impl.ScopeBean; import org.gcube.common.scope.impl.ScopeBean.Type; import org.gcube.datacatalogue.ckanutillibrary.ckan.ExtendCkanClient; import org.gcube.datacatalogue.ckanutillibrary.ckan.SimpleExtendCkanClient; import org.gcube.datacatalogue.ckanutillibrary.db.DBCaller; import org.gcube.datacatalogue.ckanutillibrary.gcat.GCatCaller; import org.gcube.datacatalogue.ckanutillibrary.server.DataCatalogueRunningCluster.ACCESS_LEVEL_TO_CATALOGUE_PORTLET; import org.gcube.datacatalogue.ckanutillibrary.server.utils.CatalogueUtilMethods; import org.gcube.datacatalogue.ckanutillibrary.server.utils.url.EntityContext; import org.gcube.datacatalogue.ckanutillibrary.shared.ResourceBean; import org.gcube.datacatalogue.ckanutillibrary.shared.RolesCkanGroupOrOrg; import org.gcube.datacatalogue.ckanutillibrary.shared.State; import org.json.simple.JSONObject; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import eu.trentorise.opendata.jackan.internal.org.apache.http.HttpResponse; import eu.trentorise.opendata.jackan.internal.org.apache.http.client.methods.HttpPost; import eu.trentorise.opendata.jackan.internal.org.apache.http.entity.ContentType; import eu.trentorise.opendata.jackan.internal.org.apache.http.entity.StringEntity; import eu.trentorise.opendata.jackan.internal.org.apache.http.impl.client.CloseableHttpClient; import eu.trentorise.opendata.jackan.internal.org.apache.http.impl.client.HttpClientBuilder; import eu.trentorise.opendata.jackan.internal.org.apache.http.util.EntityUtils; import eu.trentorise.opendata.jackan.model.CkanDataset; import eu.trentorise.opendata.jackan.model.CkanGroup; import eu.trentorise.opendata.jackan.model.CkanLicense; import eu.trentorise.opendata.jackan.model.CkanOrganization; import eu.trentorise.opendata.jackan.model.CkanUser; public class DataCatalogueImpl implements DataCatalogue { private static final Logger LOG = LoggerFactory.getLogger(DataCatalogueImpl.class); private String CKAN_CATALOGUE_URL; private String CKAN_DB_NAME; private String CKAN_DB_USER; private String CKAN_DB_PASSWORD; private String CKAN_DB_URL; private Integer CKAN_DB_PORT; private String PORTLET_URL_FOR_SCOPE; private String SOLR_URL; private String CKAN_TOKEN_SYS; private String CKAN_EMAIL; private String URI_RESOLVER_URL; private boolean MANAGE_PRODUCT_BUTTON; private boolean SOCIAL_POST; private boolean ALERT_USERS_ON_POST_CREATION; private String CONTEXT; private Map extendRoleInOrganization; public Map mapAccessURLToCatalogue; private static final String CATALOGUE_TAB_ENDING_URL = "/catalogue"; // gCat client private GCatCaller gCatCaller; // db client private DBCaller dbCaller; // ckan client private SimpleExtendCkanClient ckanCaller; // hashmap for ckan api keys private ConcurrentHashMap apiKeysMap; // apikey bean expires after X minutes in the above map private static final int EXPIRE_KEY_TIME = 60 * 60 * 1000; /** * The ckan catalogue url and database will be discovered in this scope * @param scope * @throws Exception if unable to find datacatalogue info */ public DataCatalogueImpl(String scope) throws Exception{ DataCatalogueRunningCluster runningInstance = new DataCatalogueRunningCluster(scope); // save information CKAN_DB_URL = runningInstance.getDatabaseHosts().get(0).trim(); CKAN_DB_PORT = runningInstance.getDatabasePorts().get(0); CKAN_DB_NAME = runningInstance.getDataBaseName().trim(); CKAN_DB_USER = runningInstance.getDataBaseUser().trim(); CKAN_DB_PASSWORD = runningInstance.getDataBasePassword().trim(); //CKAN_TOKEN_SYS = runningInstance.getSysAdminToken().trim(); CKAN_EMAIL = runningInstance.getEmailCatalogue().trim(); CKAN_CATALOGUE_URL = runningInstance.getDataCatalogueUrl().get(0).trim(); PORTLET_URL_FOR_SCOPE = runningInstance.getPortletUrl().trim(); mapAccessURLToCatalogue = runningInstance.getMapAccessURLToCatalogue(); MANAGE_PRODUCT_BUTTON = runningInstance.isManageProductEnabled(); URI_RESOLVER_URL = runningInstance.getUrlResolver(); SOCIAL_POST = runningInstance.isSocialPostEnabled(); ALERT_USERS_ON_POST_CREATION = runningInstance.isAlertEnabled(); SOLR_URL = runningInstance.getUrlSolr(); LOG.info("In the scope: "+scope+", I read the catalogue URL: " + CKAN_CATALOGUE_URL); // build the clients gCatCaller = new GCatCaller(CKAN_CATALOGUE_URL); dbCaller = new DBCaller(CKAN_DB_URL, CKAN_DB_PORT, CKAN_DB_NAME, CKAN_DB_USER, CKAN_DB_PASSWORD); ckanCaller = new SimpleExtendCkanClient(CKAN_CATALOGUE_URL); // init map apiKeysMap = new ConcurrentHashMap(); // save the context CONTEXT = scope; // extended roles extendRoleInOrganization = runningInstance.getExtendRoleInOrganization(); } @Override public String getCatalogueUrl() { return CKAN_CATALOGUE_URL; } @Override public String getPortletUrl() { //PATCHED By Francesco ScopeBean context = new ScopeBean(CONTEXT); if(context.is(Type.INFRASTRUCTURE)) { LOG.info("Working with the {} scope returning the path read from GR 'Ckan-Porltet': {}", Type.INFRASTRUCTURE.toString(), PORTLET_URL_FOR_SCOPE); return PORTLET_URL_FOR_SCOPE; } String vreNameLower = context.name().toLowerCase(); //CHECKING IF THE PORTLET URL CONTAINS THE VRE NAME INTO URL if(PORTLET_URL_FOR_SCOPE.toLowerCase().contains(vreNameLower)){ //THE PORLTET URL READ FROM GENERIC RESOUCE 'CkanPortlet' SHOULD BE ALREADY VALID, POITING TO CKAN PORTLET return PORTLET_URL_FOR_SCOPE; }else{ //ADDING VRE getApiKeyFromUsernameNAME AND THE SUFFIX 'CATALOGUE_TAB_ENDING_URL' TO URL String buildedUrl = PORTLET_URL_FOR_SCOPE.endsWith("/") ? PORTLET_URL_FOR_SCOPE : PORTLET_URL_FOR_SCOPE + "/"; String defaultSuffix = vreNameLower + CATALOGUE_TAB_ENDING_URL; buildedUrl+= defaultSuffix; LOG.warn("The Portlet URL read from Generic Resource 'Ckan-Porltet' does not contain the portlet suffix, so I added the default: "+defaultSuffix); return buildedUrl; } } @Override public String findLicenseIdByLicenseTitle(String chosenLicense) { LOG.debug("Requested license id"); // checks checkNotNull(chosenLicense); //retrieve the list of available licenses List licenses = ckanCaller.getLicenseList(); for (CkanLicense ckanLicense : licenses) { if(ckanLicense.getTitle().equals(chosenLicense)) return ckanLicense.getId(); } return null; } @Override public List getLicenseTitles() { LOG.debug("Request for CKAN licenses"); // get the url and the api key of the user List result = new ArrayList(); //retrieve the list of available licenses List licenses = ckanCaller.getLicenseList(); for (CkanLicense ckanLicense : licenses) { result.add(ckanLicense.getTitle()); LOG.debug("License is " + ckanLicense.getTitle() + " and id " + ckanLicense.getId()); } return result; } @Override public List getLicenses() { LOG.debug("Request for CKAN licenses (original jackan objects are going to be retrieved)"); //retrieve the list of available licenses return ckanCaller.getLicenseList(); } @Override public CkanDataset getDataset(String datasetId, String apiKey) { LOG.info("Request ckan dataset with id " + datasetId); // checks checkNotNull(datasetId); checkArgument(!datasetId.isEmpty()); try{ if(apiKey!=null && !apiKey.isEmpty()) { LOG.info("API-KEY found. Calling the "+ExtendCkanClient.class.getSimpleName()); ExtendCkanClient client = new ExtendCkanClient(CKAN_CATALOGUE_URL, apiKey); return client.getDataset(datasetId); } String authzToken = SecurityTokenProvider.instance.get(); if(authzToken!=null && !authzToken.isEmpty()) { LOG.info("gcube-token found. Calling the gCat client"); return gCatCaller.getDatasetForName(datasetId); } LOG.info("No api-key or gcube-token found. Calling Ckan Client without API-KEY"); return ckanCaller.getDataset(datasetId); }catch(Exception e){ LOG.error("Unable to retrieve such dataset, returning null ...", e); } return null; } @Override public String getUnencryptedUrlFromDatasetIdOrName(String datasetIdOrName) { LOG.debug("Request coming for getting dataset url (not encrypted) of dataset with name/id " + datasetIdOrName); // checks checkNotNull(datasetIdOrName); checkArgument(!datasetIdOrName.isEmpty()); String url = null; try{ // get the dataset from name ExtendCkanClient client = new ExtendCkanClient(CKAN_CATALOGUE_URL, CKAN_TOKEN_SYS); CkanDataset dataset = client.getDataset(datasetIdOrName); String name = dataset.getName(); if(dataset != null){ if(getUriResolverUrl() != null) url = getUrlForProduct(CONTEXT, EntityContext.DATASET, name); if(url == null || url.isEmpty()) url = getPortletUrl() + "?" + URLEncoder.encode("path=/dataset/" + name, "UTF-8"); } }catch(Exception e){ LOG.error("Error while retrieving dataset with id/name=" + datasetIdOrName, e); } //requestEntity.put("clear_url", Boolean.toString(unencrypted)); return url; } public String createCKanDatasetMultipleCustomFields(String title, String name, String organizationNameOrId, String author, String authorMail, String maintainer, String maintainerMail, long version, String description, String licenseId, List tags, Map> customFields, List resources, boolean setPublic) throws Exception { return null; } @Override public Map> getUserRoleByGroup(String username) { LOG.info("Get user role by group called. The username is: "+username); checkNotNull(username); Map> toReturn = new HashMap>(); try{ String ckanUsername = CatalogueUtilMethods.fromUsernameToCKanUsername(username); Map partialResult = dbCaller.getGroupsByUserFromDB(ckanUsername); for (String groupID : partialResult.keySet()) { CkanGroup group = ckanCaller.getGroup(groupID); HashMap subMap = new HashMap(); subMap.put(group, partialResult.get(groupID)); toReturn.put(groupID, subMap); } LOG.debug("Returning map " + toReturn); }catch(Exception e){ LOG.error("Failed to retrieve roles of user in his/her own groups",e); } return toReturn; } @Override public Map> getUserRoleByOrganization( String username) { LOG.info("Get user role by organization called. The username is: "+username); checkNotNull(username); Map> toReturn = new HashMap>(); try{ String ckanUsername = CatalogueUtilMethods.fromUsernameToCKanUsername(username); Map partialResult = dbCaller.getOrganizationsByUserFromDB(ckanUsername); for (String orgID : partialResult.keySet()) { CkanOrganization org = ckanCaller.getOrganization(orgID); HashMap subMap = new HashMap(); subMap.put(org, partialResult.get(orgID)); toReturn.put(orgID, subMap); } LOG.debug("Returning map " + toReturn); }catch(Exception e){ LOG.error("Failed to retrieve roles of user in his/her own groups",e); } return toReturn; } /** * Retrieve an url for the tuple scope, entity, entity name * @param context * @param entityContext * @param entityName */ private String getUrlForProduct(String context, EntityContext entityContext, String entityName){ String toReturn = null; try (CloseableHttpClient httpClient = HttpClientBuilder.create().build()) { HttpPost httpPostRequest = new HttpPost(getUriResolverUrl()); JSONObject requestEntity = new JSONObject(); requestEntity.put("gcube_scope", context); requestEntity.put("entity_context", entityContext.toString()); requestEntity.put("entity_name", entityName); StringEntity params = new StringEntity(requestEntity.toJSONString(), ContentType.APPLICATION_JSON); httpPostRequest.setEntity(params); HttpResponse response = httpClient.execute(httpPostRequest); if(response.getStatusLine().getStatusCode() != 200) throw new Exception("There was an error while creating an url " + response.getStatusLine()); toReturn = EntityUtils.toString(response.getEntity()); LOG.debug("Result is " + toReturn); }catch(Exception e){ LOG.error("Failed to get an url for this product", e); } return toReturn; } @Override public String getUriResolverUrl() { return URI_RESOLVER_URL; } /** * Check if the manage product is enabled * @return */ @Override public boolean isManageProductEnabled() { return MANAGE_PRODUCT_BUTTON; } @Override public List getOrganizationsByUser(String username) { LOG.debug("Requested organizations for user " + username); // checks checkNotNull(username); // in order to avoid errors, the username is always converted String ckanUsername = CatalogueUtilMethods.fromUsernameToCKanUsername(username); // list to return List toReturn = new ArrayList(); try{ // get the list of all organizations List organizations = ckanCaller.getOrganizationList(); // iterate over them for (CkanOrganization ckanOrganization : organizations) { // get the list of users in it (if you try ckanOrganization.getUsers() it returns null.. maybe a bug TODO) List users = ckanCaller.getOrganization(ckanOrganization.getName()).getUsers(); // check if the current user is among them for (CkanUser ckanUser : users) { if(ckanUser.getName().equals(ckanUsername)){ LOG.debug("User " + ckanUsername + " is into " + ckanOrganization.getName()); toReturn.add(ckanOrganization); break; } } } }catch(Exception e){ LOG.error("Unable to get user's organizations", e); } return toReturn; } @Override public List getGroupsByUser(String username) { LOG.debug("Requested groups for user " + username); // checks checkNotNull(username); // in order to avoid errors, the username is always converted String ckanUsername = CatalogueUtilMethods.fromUsernameToCKanUsername(username); // list to return List toReturn = new ArrayList(); try{ // get the list of all organizations List groups = ckanCaller.getGroupList(); // iterate over them for (CkanGroup ckanGroup : groups) { List users = ckanCaller.getGroup(ckanGroup.getName()).getUsers(); // check if the current user is among them for (CkanUser ckanUser : users) { if(ckanUser.getName().equals(ckanUsername)){ LOG.debug("User " + ckanUsername + " is into " + ckanGroup.getName()); toReturn.add(ckanGroup); break; } } } }catch(Exception e){ LOG.error("Unable to get user's groups", e); } return toReturn; } //@Override private String getApiKeyFromUsername(String username) { LOG.debug("Request api key for user = " + username); // checks checkNotNull(username); checkArgument(!username.isEmpty()); // in order to avoid errors, the username is always converted String ckanUsername = CatalogueUtilMethods.fromUsernameToCKanUsername(username); // check in the hashmap first if(apiKeysMap.containsKey(ckanUsername)){ CKANTokenBean bean = apiKeysMap.get(ckanUsername); if(bean.timestamp + EXPIRE_KEY_TIME > System.currentTimeMillis()){ // it's still ok return bean.apiKey; } } LOG.debug("Api key was not in cache or it expired"); try{ String apiToReturn = dbCaller.getApiKeyFromUsername(username, State.ACTIVE.name().toLowerCase()); // save into the hashmap if(apiToReturn != null) apiKeysMap.put(ckanUsername, new CKANTokenBean(apiToReturn, System.currentTimeMillis())); return apiToReturn; }catch(Exception e){ LOG.error("Unable to retrieve key for user " + ckanUsername, e); } return null; } /* * * * * * * * * * * WRITE OPERATIONS * * * * * * * * * * */ public boolean patchProductCustomFields(String productId, Map> customFieldsToChange, boolean removeOld) { return false; } public String addResourceToDataset(ResourceBean resource) throws Exception { return null; } public boolean deleteResourceFromDataset(String resourceId) { return false; } public boolean existProductWithNameOrId(String nameOrId) { return false; } }