package org.gcube.gcat.configuration; import static org.gcube.resources.discovery.icclient.ICFactory.clientFor; import static org.gcube.resources.discovery.icclient.ICFactory.queryFor; import java.io.IOException; import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Set; import java.util.UUID; import javax.ws.rs.InternalServerErrorException; import javax.ws.rs.NotFoundException; import javax.ws.rs.WebApplicationException; import org.gcube.com.fasterxml.jackson.core.JsonProcessingException; import org.gcube.com.fasterxml.jackson.databind.JsonNode; import org.gcube.com.fasterxml.jackson.databind.ObjectMapper; import org.gcube.com.fasterxml.jackson.databind.node.ArrayNode; import org.gcube.com.fasterxml.jackson.databind.node.ObjectNode; import org.gcube.common.resources.gcore.GenericResource; import org.gcube.common.resources.gcore.ServiceEndpoint; import org.gcube.common.resources.gcore.ServiceEndpoint.AccessPoint; import org.gcube.common.resources.gcore.ServiceEndpoint.Profile; import org.gcube.common.resources.gcore.ServiceEndpoint.Property; import org.gcube.common.resources.gcore.ServiceEndpoint.Runtime; import org.gcube.common.resources.gcore.common.Platform; import org.gcube.common.resources.gcore.utils.Group; import org.gcube.gcat.api.GCatConstants; import org.gcube.gcat.api.configuration.CatalogueConfiguration; import org.gcube.informationsystem.publisher.RegistryPublisher; import org.gcube.informationsystem.publisher.RegistryPublisherFactory; import org.gcube.resources.discovery.client.api.DiscoveryClient; import org.gcube.resources.discovery.client.queries.api.SimpleQuery; import org.gcube.resources.discovery.icclient.ICFactory; import org.gcube.smartgears.ContextProvider; import org.gcube.smartgears.configuration.application.ApplicationConfiguration; import org.gcube.smartgears.configuration.container.ContainerConfiguration; import org.gcube.smartgears.context.application.ApplicationContext; import org.gcube.smartgears.context.container.ContainerContext; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** * @author Luca Frosini (ISTI - CNR) */ public class GCoreISConfigurationProxy { private static final Logger logger = LoggerFactory.getLogger(GCoreISConfigurationProxy.class); // property to retrieve the master service endpoint into the /root scope public final static String IS_ROOT_MASTER_PROPERTY_KEY = "IS_ROOT_MASTER"; // true, false.. missing means false as public final static String DEFAULT_ORGANIZATION_PROPERTY_KEY = "DEFAULT_ORGANIZATION"; public final static String SUPPORTED_ORGANIZATION_PROPERTY_KEY = "SUPPORTED_ORGANIZATION"; public final static String API_KEY_PROPERTY_KEY = "API_KEY"; public final static String SOLR_INDEX_ADDRESS_PROPERTY_KEY = "SOLR_INDEX_ADDRESS"; public final static String SOCIAL_POST_PROPERTY_KEY = "SOCIAL_POST"; public final static String ALERT_USERS_ON_POST_CREATION_PROPERTY_KEY = "ALERT_USERS_ON_POST_CREATION"; public final static String MODERATION_ENABLED_KEY_PROPERTY_KEY = "MODERATION_ENABLED"; public static final Map gCoreToConfigurationMapping; public static final Map configurationToGCoreMapping; static { gCoreToConfigurationMapping = new HashMap<>(); configurationToGCoreMapping = new HashMap<>(); gCoreToConfigurationMapping.put(API_KEY_PROPERTY_KEY, CatalogueConfiguration.SYS_ADMIN_TOKEN_KEY); gCoreToConfigurationMapping.put(SOLR_INDEX_ADDRESS_PROPERTY_KEY, CatalogueConfiguration.SOLR_URL_KEY); gCoreToConfigurationMapping.put(SOCIAL_POST_PROPERTY_KEY, CatalogueConfiguration.SOCIAL_POST_ENABLED_KEY); gCoreToConfigurationMapping.put(ALERT_USERS_ON_POST_CREATION_PROPERTY_KEY, CatalogueConfiguration.NOTIFICATION_TO_USER_ENABLED_KEY); gCoreToConfigurationMapping.put(MODERATION_ENABLED_KEY_PROPERTY_KEY, CatalogueConfiguration.MODERATION_ENABLED_KEY); for(String key : gCoreToConfigurationMapping.keySet()) { configurationToGCoreMapping.put(gCoreToConfigurationMapping.get(key), key); } } // CKAN Instance info private final static String OLD_CATEGORY = "Application"; private final static String OLD_NAME = "CKanDataCatalogue"; protected final String context; protected ObjectMapper mapper; protected ServiceCatalogueConfiguration catalogueConfiguration; public GCoreISConfigurationProxy(String context) { this.context = context; this.mapper = new ObjectMapper(); } public GCoreISConfigurationProxy(String context, ServiceCatalogueConfiguration catalogueConfiguration) { this(context); this.catalogueConfiguration = catalogueConfiguration; this.mapper = new ObjectMapper(); } public ServiceCatalogueConfiguration getCatalogueConfiguration() { if (catalogueConfiguration == null) { catalogueConfiguration = getOLDCatalogueConfigurationFromIS(); } return catalogueConfiguration; } protected AccessPoint getAccessPoint(Profile profile) { Group accessPoints = profile.accessPoints(); Iterator accessPointIterator = accessPoints.iterator(); AccessPoint accessPoint = accessPointIterator.next(); return accessPoint; } protected String getDefaultSolrURL(String ckanURL) { StringBuffer stringBuffer = new StringBuffer(); stringBuffer.append(ckanURL); stringBuffer.append(ckanURL.endsWith("/")?"":"/"); stringBuffer.append("solr/"); return stringBuffer.toString(); } protected ObjectNode setValue(ObjectNode node, String key, String value) throws IOException { if(value.toLowerCase().compareTo("true")==0 || value.toLowerCase().compareTo("false")==0) { node.put(key, Boolean.parseBoolean(value)); return node; } if(value.startsWith("{") || value.startsWith("[")){ JsonNode n = mapper.readTree(value); node.set(key, n); return node; } node.put(key, value); return node; } protected ServiceCatalogueConfiguration getConfiguration(ServiceEndpoint serviceEndpoint) throws IOException { Profile profile = serviceEndpoint.profile(); AccessPoint accessPoint = getAccessPoint(profile); Map propertyMap = accessPoint.propertyMap(); ObjectNode node = mapper.createObjectNode(); node.put(CatalogueConfiguration.ID_KEY, serviceEndpoint.id()); for(String key : propertyMap.keySet()) { String value = propertyMap.get(key).value().trim(); setValue(node, key, value); } return mapper.treeToValue(node, ServiceCatalogueConfiguration.class); } /** * Retrieve endpoints information from IS for DataCatalogue URL * * @return list of endpoints for ckan data catalogue * @throws Exception */ private List getServiceEndpoints(String category, String name) { SimpleQuery query = queryFor(ServiceEndpoint.class); query.addCondition("$resource/Profile/Category/text() eq '" + category + "'"); query.addCondition("$resource/Profile/Name/text() eq '" + name + "'"); DiscoveryClient client = clientFor(ServiceEndpoint.class); List serviceEndpoints = client.submit(query); return serviceEndpoints; } protected ServiceEndpoint getServiceEndpoint() { List serviceEndpoints = getServiceEndpoints(GCatConstants.CONFIGURATION_CATEGORY, GCatConstants.CONFIGURATION_NAME); if (serviceEndpoints==null || serviceEndpoints.size() == 0) { logger.error("There is no {} having Category {} and Name {} in this context.", ServiceEndpoint.class.getSimpleName(), GCatConstants.CONFIGURATION_CATEGORY, GCatConstants.CONFIGURATION_NAME); return null; } ServiceEndpoint serviceEndpoint = serviceEndpoints.get(0); return serviceEndpoint; } protected ServiceCatalogueConfiguration getCatalogueConfigurationFromIS() throws IOException { ServiceEndpoint serviceEndpoint = getServiceEndpoint(); if(serviceEndpoint==null) { return getOLDCatalogueConfigurationFromIS(); } return getConfiguration(serviceEndpoint); } @Deprecated private ServiceEndpoint getOldServiceEndpoint() { List serviceEndpoints = getServiceEndpoints(OLD_CATEGORY, OLD_NAME); if (serviceEndpoints.size() == 0) { logger.error("There is no {} having Category {} and Name {} in this context.", ServiceEndpoint.class.getSimpleName(), OLD_CATEGORY, OLD_NAME); return null; } ServiceEndpoint serviceEndpoint = null; if (serviceEndpoints.size() > 1) { logger.info("Too many {} having Category {} and Name {} in this context. Looking for the one that has the property {}", ServiceEndpoint.class.getSimpleName(), OLD_CATEGORY, OLD_NAME, IS_ROOT_MASTER_PROPERTY_KEY); for (ServiceEndpoint se : serviceEndpoints) { Iterator accessPointIterator = se.profile().accessPoints().iterator(); while (accessPointIterator.hasNext()) { ServiceEndpoint.AccessPoint accessPoint = accessPointIterator.next(); // get the is master property Property entry = accessPoint.propertyMap().get(IS_ROOT_MASTER_PROPERTY_KEY); String isMaster = entry != null ? entry.value() : null; if (isMaster == null || !isMaster.equals("true")) { continue; } // set this variable serviceEndpoint = se; return serviceEndpoint; } } // if none of them was master, throw an exception if (serviceEndpoint == null) { throw new InternalServerErrorException( "Too many catalogue configuration on IS and no one with MASTER property"); } } else { serviceEndpoint = serviceEndpoints.get(0); } return serviceEndpoint; } @Deprecated protected ServiceCatalogueConfiguration getOLDCatalogueConfigurationFromIS() { ServiceCatalogueConfiguration catalogueConfiguration = new ServiceCatalogueConfiguration(context); try { // boolean mustBeUpdated = false; ServiceEndpoint serviceEndpoint = getOldServiceEndpoint(); if (serviceEndpoint == null) { throw new NotFoundException("No configuration found in this context"); } catalogueConfiguration.setID(serviceEndpoint.id()); Profile profile = serviceEndpoint.profile(); AccessPoint accessPoint = getAccessPoint(profile); // add this host String ckanURL = accessPoint.address(); catalogueConfiguration.setCkanURL(ckanURL); Map propertyMap = accessPoint.propertyMap(); // retrieve sys admin token String encryptedSysAdminToken = propertyMap.get(API_KEY_PROPERTY_KEY).value(); catalogueConfiguration.setEncryptedSysAdminToken(encryptedSysAdminToken); String defaultOrganization = CatalogueConfiguration.getOrganizationName(context); String solrURL = null; if (propertyMap.containsKey(SOLR_INDEX_ADDRESS_PROPERTY_KEY)) { solrURL = propertyMap.get(SOLR_INDEX_ADDRESS_PROPERTY_KEY).value(); }else { solrURL = getDefaultSolrURL(ckanURL); } catalogueConfiguration.setSolrURL(solrURL); // retrieve option to check if the social post has to be made Boolean socialPostEnabled = true; if (propertyMap.containsKey(SOCIAL_POST_PROPERTY_KEY)) { if (propertyMap.get(SOCIAL_POST_PROPERTY_KEY).value().trim().equalsIgnoreCase("false")) { socialPostEnabled = false; } } catalogueConfiguration.setSocialPostEnabled(socialPostEnabled); // retrieve option for user alert boolean notificationToUsersEnabled = false; // default is false if (propertyMap.containsKey(ALERT_USERS_ON_POST_CREATION_PROPERTY_KEY)) { if (propertyMap.get(ALERT_USERS_ON_POST_CREATION_PROPERTY_KEY).value().trim() .equalsIgnoreCase("true")) { notificationToUsersEnabled = true; } } catalogueConfiguration.setNotificationToUsersEnabled(notificationToUsersEnabled); boolean moderationEnabled = false; // default is false if (propertyMap.containsKey(MODERATION_ENABLED_KEY_PROPERTY_KEY)) { if (propertyMap.get(MODERATION_ENABLED_KEY_PROPERTY_KEY).value().trim().equalsIgnoreCase("true")) { moderationEnabled = true; } } catalogueConfiguration.setModerationEnabled(moderationEnabled); Set supportedOrganizations = getSupportedOrganizationsFromGenericResource(); if (supportedOrganizations != null) { catalogueConfiguration.setSupportedOrganizations(supportedOrganizations); if(defaultOrganization==null) { defaultOrganization = supportedOrganizations.toArray(new String[supportedOrganizations.size()])[0]; catalogueConfiguration.setDefaultOrganization(defaultOrganization); } } ServiceCKANDB ckanDB = getCKANDBFromIS(); catalogueConfiguration.setCkanDB(ckanDB); } catch (WebApplicationException e) { throw e; } catch (Exception e) { throw new InternalServerErrorException("Error while getting configuration on IS", e); } return catalogueConfiguration; } // CKAN Instance info @Deprecated private final static String CKAN_DB_SERVICE_ENDPOINT_CATEGORY= "Database"; @Deprecated private final static String CKAN_DB_SERVICE_ENDPOINT_NAME = "CKanDatabase"; @Deprecated protected ServiceCKANDB getCKANDBFromIS() { try { List serviceEndpoints = getServiceEndpoints(CKAN_DB_SERVICE_ENDPOINT_CATEGORY, CKAN_DB_SERVICE_ENDPOINT_NAME); if(serviceEndpoints.size() == 0) { String error = String.format("There is no %s having category '%s' and name '%s' in this context.", ServiceEndpoint.class.getSimpleName(), CKAN_DB_SERVICE_ENDPOINT_CATEGORY, CKAN_DB_SERVICE_ENDPOINT_NAME); logger.error(error); throw new InternalServerErrorException(error); } ServiceEndpoint serviceEndpoint = null; if(serviceEndpoints.size() > 1) { logger.info("Too many {} having category {} and name {} in this context. Looking for the one that has the property {}", ServiceEndpoint.class.getSimpleName(), CKAN_DB_SERVICE_ENDPOINT_CATEGORY, CKAN_DB_SERVICE_ENDPOINT_NAME); for(ServiceEndpoint se : serviceEndpoints) { Iterator accessPointIterator = se.profile().accessPoints().iterator(); while(accessPointIterator.hasNext()) { ServiceEndpoint.AccessPoint accessPoint = accessPointIterator.next(); // get the is master property Property entry = accessPoint.propertyMap().get(IS_ROOT_MASTER_PROPERTY_KEY); String isMaster = entry != null ? entry.value() : null; if(isMaster == null || !isMaster.equals("true")) { continue; } // set this variable serviceEndpoint = se; break; } } // if none of them was master, throw an exception if(serviceEndpoint == null) { throw new InternalServerErrorException( "Too many CKAN configuration on IS and no one with MASTER property"); } } else { serviceEndpoint = serviceEndpoints.get(0); } Iterator accessPointIterator = serviceEndpoint.profile().accessPoints().iterator(); while(accessPointIterator.hasNext()) { AccessPoint accessPoint = accessPointIterator.next(); String host = accessPoint.address(); String db = accessPoint.name(); ServiceCKANDB ckanDB = new ServiceCKANDB(); String url = String.format("jdbc:postgresql://%s/%s", host, db); ckanDB.setUrl(url); ckanDB.setUsername(accessPoint.username()); ckanDB.setEncryptedPassword(accessPoint.password()); return ckanDB; } return null; } catch(WebApplicationException e) { throw e; } catch(Exception e) { throw new InternalServerErrorException("Error while getting configuration on IS", e); } } @Deprecated public static final String GENERIC_RESOURCE_SECONDARY_TYPE_FOR_ORGANIZATIONS = "ApplicationProfile"; @Deprecated public static final String GENERIC_RESOURCE_NAME_FOR_ORGANIZATIONS = "Supported CKAN Organizations"; @Deprecated public static final String GENERIC_RESOURCE_CKAN_ORGANIZATIONS = "CKANOrganizations"; @Deprecated private List getGenericResources() { SimpleQuery query = ICFactory.queryFor(GenericResource.class); query.addCondition(String.format("$resource/Profile/SecondaryType/text() eq '%s'", GENERIC_RESOURCE_SECONDARY_TYPE_FOR_ORGANIZATIONS)); query.addCondition( String.format("$resource/Profile/Name/text() eq '%s'", GENERIC_RESOURCE_NAME_FOR_ORGANIZATIONS)); DiscoveryClient client = ICFactory.clientFor(GenericResource.class); List genericResources = client.submit(query); return genericResources; } protected String marshallSupportedOrganizations() throws JsonProcessingException { Set supportedOrganizations = catalogueConfiguration.getSupportedOrganizations(); return marshallSupportedOrganizations(supportedOrganizations); } protected String marshallSupportedOrganizations(Set supportedOrganizations) throws JsonProcessingException { ObjectMapper objectMapper = new ObjectMapper(); ArrayNode arrayNode = objectMapper.createArrayNode(); for(String org : supportedOrganizations) { arrayNode.add(org); } return objectMapper.writeValueAsString(arrayNode); } @Deprecated protected Set unmarshallSupportedOrganizations(String supportedOrganizationsJsonArray){ try { ObjectMapper objectMapper = new ObjectMapper(); JsonNode jsonNode = objectMapper.readTree(supportedOrganizationsJsonArray); ArrayNode array = (ArrayNode) jsonNode.get(GENERIC_RESOURCE_CKAN_ORGANIZATIONS); Set supportedOrganizations = new HashSet<>(array.size()); for (int i = 0; i < array.size(); i++) { String o = array.get(i).asText(); supportedOrganizations.add(o); } logger.debug("Supported CKAN Organization for current Context ({}) are {}", context, supportedOrganizations); return supportedOrganizations; } catch (Exception e) { return null; } } @Deprecated protected Set getSupportedOrganizationsFromGenericResource() { List genericResources = getGenericResources(); if (genericResources == null || genericResources.size() == 0) { logger.trace( "{} with SecondaryType {} and Name %s not found. Item will be only be created in {} CKAN organization", GenericResource.class.getSimpleName(), GENERIC_RESOURCE_SECONDARY_TYPE_FOR_ORGANIZATIONS, GENERIC_RESOURCE_NAME_FOR_ORGANIZATIONS, ServiceCatalogueConfiguration.getOrganizationName(context)); return null; } GenericResource genericResource = genericResources.get(0); String supportedOrganizationsJsonArray = genericResource.profile().body().getTextContent(); Set supportedOrganizatins = unmarshallSupportedOrganizations(supportedOrganizationsJsonArray); return supportedOrganizatins; } @Deprecated public void deleteOldConfiguration() { RegistryPublisher registryPublisher = RegistryPublisherFactory.create(); deleteOldConfiguration(registryPublisher); } @Deprecated protected void deleteOldConfiguration(RegistryPublisher registryPublisher) { ServiceEndpoint serviceEndpoint = getOldServiceEndpoint(); if(serviceEndpoint!=null) { registryPublisher.remove(serviceEndpoint); } List genericResources = getGenericResources(); if(genericResources!=null) { for(GenericResource genericResource : genericResources) { registryPublisher.remove(genericResource); } } } public void delete() { RegistryPublisher registryPublisher = RegistryPublisherFactory.create(); ServiceEndpoint serviceEndpoint = getServiceEndpoint(); if(serviceEndpoint!=null) { registryPublisher.remove(serviceEndpoint); } deleteOldConfiguration(); } protected Property addProperty(Group properties, String name, String value) { return addProperty(properties, name, value, false); } protected Property addProperty(Group properties, String name, String value, boolean encrypted) { Property property = new Property(); property.nameAndValue(name, value); if(encrypted) { property.encrypted(encrypted); } properties.add(property); return property; } protected Group setAccessPointProperties(AccessPoint accessPoint, String address, boolean update) throws JsonProcessingException { accessPoint.description(String.format("Access Point %s by gcat %s", update ? "updated" : "created", getGcatVersion().toString())); accessPoint.address(address); accessPoint.name(GCatConstants.CONFIGURATION_NAME); Group properties = accessPoint.properties(); JsonNode jsonNode = mapper.valueToTree(catalogueConfiguration); Iterator iterator = jsonNode.fieldNames(); while (iterator.hasNext()) { String key = iterator.next(); JsonNode valueJsonNode = jsonNode.get(key); String value = valueJsonNode.toString(); addProperty(properties, key, value); } return properties; } protected Version getGcatVersion() { try { ApplicationContext applicationContext = ContextProvider.get(); ApplicationConfiguration applicationConfiguration = applicationContext.configuration(); Version version = new Version(applicationConfiguration.version()); return version; }catch (Exception e) { return new Version("2.2.0"); } } /** * Set the version of gcat so that in future implementation * we can understand if the configuration must be updated. * @param platform * @return */ protected Platform setVersion(Platform platform) { Version version = getGcatVersion(); platform.version((short) version.getMajor()); platform.minorVersion((short) version.getMinor()); platform.revisionVersion((short) version.getRevision()); platform.buildVersion((short) 0); return platform; } protected Platform setPlatformProperty(Platform platform) { /* * * gcat * * 2 * 2 * 0 * 0 * */ platform.name(GCatConstants.SERVICE_NAME); platform = setVersion(platform); return platform; } private String getRunningOn(ContainerConfiguration containerConfiguration) { return String.format("%s:%s", containerConfiguration.hostname(), containerConfiguration.port()); } protected Runtime setRuntimeProperties(Runtime runtime) { try { ApplicationContext applicationContext = ContextProvider.get(); ContainerContext containerContext = applicationContext.container(); ContainerConfiguration containerConfiguration = containerContext.configuration(); String runningOn = getRunningOn(containerConfiguration); runtime.hostedOn(runningOn); runtime.ghnId(containerContext.id()); runtime.status(applicationContext.configuration().mode().toString()); }catch (Exception e) { runtime.hostedOn("localhost"); runtime.ghnId(""); runtime.status("READY"); } return runtime; } protected Profile setProfileProperties(Profile profile, boolean update) { /* * * Application * CKanDataCatalogue * gCat Configuration created/updated by the service via REST */ profile.category(GCatConstants.CONFIGURATION_CATEGORY); profile.name(GCatConstants.CONFIGURATION_NAME); profile.description(String.format("gCat configuration %s by the service via REST", update ? "updated" : "created")); return profile; } // @Deprecated // protected boolean isRootMaster(ServiceEndpoint serviceEndpoint) { // Profile profile = serviceEndpoint.profile(); // AccessPoint accessPoint = getAccessPoint(profile); // Map propertyMap = accessPoint.propertyMap(); // if (propertyMap.containsKey(IS_ROOT_MASTER_PROPERTY_KEY)) { // if (propertyMap.get(IS_ROOT_MASTER_PROPERTY_KEY).value().trim().equalsIgnoreCase("true")) { // return true; // } // } // return false; // } protected ServiceEndpoint createServiceEndpoint(ServiceEndpoint serviceEndpoint) throws Exception { boolean update = serviceEndpoint != null; serviceEndpoint = new ServiceEndpoint(); serviceEndpoint.setId(catalogueConfiguration.getID()); Profile profile = serviceEndpoint.newProfile(); profile = setProfileProperties(profile, update); Platform platform = profile.newPlatform(); setPlatformProperty(platform); Runtime runtime = profile.newRuntime(); runtime = setRuntimeProperties(runtime); Group accessPoints = profile.accessPoints(); AccessPoint accessPoint = accessPoints.add(); setAccessPointProperties(accessPoint, runtime.hostedOn(), update); return serviceEndpoint; } public ServiceCatalogueConfiguration createOnIS(ServiceEndpoint serviceEndpoint) throws Exception { RegistryPublisher registryPublisher = RegistryPublisherFactory.create(); String id = catalogueConfiguration.getID(); if(id==null || id.compareTo("")==0) { id = UUID.randomUUID().toString(); catalogueConfiguration.setID(id); } serviceEndpoint = createServiceEndpoint(serviceEndpoint); registryPublisher.create(serviceEndpoint); return catalogueConfiguration; } public ServiceCatalogueConfiguration updateOnIS(ServiceEndpoint serviceEndpoint) throws Exception { RegistryPublisher registryPublisher = RegistryPublisherFactory.create(); String id = serviceEndpoint.id(); if(catalogueConfiguration.getID().compareTo(id)!=0) { catalogueConfiguration.setID(id); } serviceEndpoint = createServiceEndpoint(serviceEndpoint); registryPublisher.update(serviceEndpoint); return catalogueConfiguration; } public ServiceCatalogueConfiguration createOrUpdateOnIS() throws Exception { ServiceEndpoint serviceEndpoint = getOldServiceEndpoint(); if(serviceEndpoint!=null) { // It's an update updateOnIS(serviceEndpoint); }else { // It's a create createOnIS(serviceEndpoint); } return catalogueConfiguration; } }