diff --git a/distro/changelog.xml b/distro/changelog.xml index a81e60e..660faed 100644 --- a/distro/changelog.xml +++ b/distro/changelog.xml @@ -1,5 +1,12 @@ - - First Release + + Added check for HomeLibraryWebApp GCoreEP + + + First Release \ No newline at end of file diff --git a/pom.xml b/pom.xml index bf2e7f7..914f603 100644 --- a/pom.xml +++ b/pom.xml @@ -9,7 +9,7 @@ org.gcube.information-system resource-checker-se-plugin - 1.0.0-SNAPSHOT + 1.1.0-SNAPSHOT jar Resource Checker Smart Executor Plugin Resource Checker Smart Executor Plugin diff --git a/src/main/java/org/gcube/informationsystem/resource_checker/ResourceCheckerPlugin.java b/src/main/java/org/gcube/informationsystem/resource_checker/ResourceCheckerPlugin.java index 718c308..3963c4a 100644 --- a/src/main/java/org/gcube/informationsystem/resource_checker/ResourceCheckerPlugin.java +++ b/src/main/java/org/gcube/informationsystem/resource_checker/ResourceCheckerPlugin.java @@ -1,6 +1,3 @@ -/** - * - */ package org.gcube.informationsystem.resource_checker; import java.io.File; import java.io.FileWriter; @@ -10,8 +7,10 @@ import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; +import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.Map.Entry; import java.util.Properties; import java.util.Set; @@ -26,10 +25,13 @@ import org.gcube.common.resources.gcore.ServiceEndpoint; import org.gcube.common.resources.gcore.Software; import org.gcube.common.resources.gcore.utils.XPathHelper; import org.gcube.common.scope.api.ScopeProvider; +import org.gcube.common.scope.impl.ScopeBean; import org.gcube.informationsystem.publisher.RegistryPublisher; import org.gcube.informationsystem.publisher.RegistryPublisherFactory; +import org.gcube.informationsystem.resource_checker.beans.BasicFunctionalityBean; +import org.gcube.informationsystem.resource_checker.beans.ContextLevel; +import org.gcube.informationsystem.resource_checker.beans.OperationLevel; import org.gcube.informationsystem.resource_checker.utils.BasicFunctionalitiesMandatoryReader; -import org.gcube.informationsystem.resource_checker.utils.BasicFunctionalityBean; import org.gcube.informationsystem.resource_checker.utils.RetrieveContextsList; import org.gcube.informationsystem.resource_checker.utils.SendNotification; import org.gcube.resources.discovery.client.api.DiscoveryClient; @@ -53,12 +55,11 @@ public class ResourceCheckerPlugin extends Plugin> missingResourcesPerContext = new HashMap>(contexts.size()); String otherFailures = INITIAL_ERRORS_STATEMENT; - DiscoveryClient client = ICFactory.client(); DocumentBuilder docBuilder = DocumentBuilderFactory.newInstance().newDocumentBuilder(); - Map resourceToIdentifierMap = new HashMap(mandatoryFunctionalities.size()); - Set missingResourcesKey = new HashSet(mandatoryFunctionalities.size()); + // write nagios report + Set missingResourceIds = new HashSet(mandatoryFunctionalities.size()); + Map idToResource = new HashMap(mandatoryFunctionalities.size()); + boolean missingToReadArePresent = false; + + // loop over contexts and resources for (String context : contexts) { - // switch context ScopeProvider.instance.set(context); for (BasicFunctionalityBean service : mandatoryFunctionalities) { + + // check if this resource should be checked in this scope + ScopeBean sb = new ScopeBean(context); + ContextLevel contextDepth = service.getCtxLevel(); + + if(contextDepth.equals(ContextLevel.VO) && sb.type().equals(ScopeBean.Type.VRE)){ + logger.info("Resource " + service + " doesn't need to be checked in a VRE context (" + context + ")"); + continue; + } try{ - String resourceId = null; - String resourceKey = service.getName() + ":" + service.getCategory(); - if((resourceId = isServicePresent(service, client, docBuilder)) == null){ - - List missingServices = null; - if(missingResourcesPerContext.containsKey(context)) - missingServices = missingResourcesPerContext.get(context); - else - missingServices = new ArrayList(); - - // add among missing resources - missingResourcesKey.add(resourceKey); - - missingServices.add(service); - missingResourcesPerContext.put(context, missingServices); - + if((resourceId = isServicePresent(service, client, docBuilder)) != null){ + service.setResourceId(resourceId); + idToResource.put(resourceId, service); }else{ - resourceToIdentifierMap.put(resourceKey, resourceId); + List missingServices = missingResourcesPerContext.containsKey(context) ? + missingResourcesPerContext.get(context) : new ArrayList(); + missingServices.add(service); + missingResourcesPerContext.put(context, missingServices); + + if(service.getOpLevel().equals(OperationLevel.ALERT_READD)) + missingToReadArePresent = true; } - Thread.sleep(1000 * SECONDS2WAIT); - }catch(Exception e){ if(!(e instanceof InterruptedException)){ logger.warn("An error arose when checking for resource " + service + " into context " + context); otherFailures += "\nAn error arose while checking for resource with name/category " - + service.getName() + "/" + service.getCategory() + " into context " + context + "(stack trace is : " + e.getMessage() + ")"; + + service.getName() + "/" + service.getCategory() + " and type " + service.getType().getCanonicalName() + " into context " + context + "(stack trace is : " + e.getMessage() + ")"; } } } @@ -129,24 +132,30 @@ public class ResourceCheckerPlugin extends Plugin identifiers = new ArrayList(mandatoryFunctionalities.size()); - for (String key : missingResourcesKey) { - identifiers.add(resourceToIdentifierMap.get(key)); - } + // evaluate missing resources from the map + evaluateMissingIds(missingResourceIds, missingResourcesPerContext); - writeReport4Nagios(identifiers); + // update the file used by nagios + writeReport4Nagios(missingResourceIds); - if(!identifiers.isEmpty()){ - try{ - logger.info("Going to execute code to re-add them"); - List vosContexts = new ArrayList(); - RetrieveContextsList.loadVOs(vosContexts); - accessPointQuery(vosContexts, identifiers , ServiceEndpoint.class); - otherFailures = "\nThe above resources have been readded in the scopes they were missing!."; - }catch(Exception e){ - otherFailures = "\n\nMoreover, while trying to re-add the following identifiers " + identifiers + " there was this error " + e.getMessage(); - logger.error("While readding the resources this error arose", e); + if(!missingResourceIds.isEmpty() && missingToReadArePresent){ + logger.info("Going to perform add and alert or just alert of missing resources"); + otherFailures = "Re-add operation on some resources has been executed. This is the result:"; + for(String resId: missingResourceIds){ + + // check if it must be readded or just alert is needed + if(idToResource.get(resId).getOpLevel().equals(OperationLevel.ALERT)){ + logger.info("Resource " + idToResource.get(resId) + " doesn't need to be readded into the scopes it is missing"); + continue; + } + + try{ + readdResource(contexts, resId, idToResource.get(resId)); + otherFailures += "\n- resource with identifier " + resId + " has been readded in the scopes it was missing;"; + }catch(Exception e){ + otherFailures += "\n- while trying to re-add the resource with the following identifier " + resId + " there was this error: " + e.getMessage() +";"; + logger.error("While readding a resource this error arose", e); + } } } @@ -154,7 +163,8 @@ public class ResourceCheckerPlugin extends Plugin missingResourceIds, + Map> missingResourcesPerContext) { + + Iterator>> iterator = missingResourcesPerContext.entrySet().iterator(); + while (iterator.hasNext()) { + Map.Entry> entry = (Map.Entry>) iterator + .next(); + List missingResources = entry.getValue(); + for (BasicFunctionalityBean basicFunctionalityBean : missingResources) { + missingResourceIds.add(basicFunctionalityBean.getResourceId()); + } + } + + } + /** * Check if a given resource is present or is missing. * @param service @@ -185,9 +215,9 @@ public class ResourceCheckerPlugin extends Plugin idsMissingResources) throws Exception{ + private void writeReport4Nagios(Set idsMissingResources) throws Exception{ try{ Properties prop = new Properties(); @@ -251,7 +282,7 @@ public class ResourceCheckerPlugin extends Plugin void accessPointQuery(List voScopes, List resourceIds, Class resourceType){ + @SuppressWarnings("unchecked") + private void readdResource(List allScopes, String resourceId, BasicFunctionalityBean service){ - String currentScope = voScopes.get(0); - HashSet scopeSet = new HashSet(); + logger.info("Adding back resource with id " + resourceId + "("+ service + ") to scopes " + allScopes); - SimpleQuery queryVRE = ICFactory.queryFor(GenericResource.class); - queryVRE.addCondition("$resource/Profile/SecondaryType/text() = 'VRE'"); - queryVRE.setResult("$resource/Profile/Body/Scope/string()"); - DiscoveryClient vREScopeClient = ICFactory.client(); + String currentScope = allScopes.get(0); // infrastructure root + ScopeProvider.instance.set(currentScope); - scopeSet.addAll(voScopes); + SimpleQuery query = ICFactory.queryFor(service.getType()); + DiscoveryClient client = ICFactory.clientFor(service.getType()); + query.addCondition("$resource/ID/text() = '"+resourceId+"'"); + T resource = client.submit(query).get(0); - for (String scope: voScopes){ - ScopeProvider.instance.set(scope); - List vresscope = vREScopeClient.submit(queryVRE); - System.out.println("found "+vresscope.size()+ " vres in "+scope); - scopeSet.addAll(vresscope); - } + resource.scopes().asCollection().retainAll(Collections.emptyList()); // remove all scopes + resource.scopes().asCollection().addAll(allScopes); // add them all again - for (String resourceId : resourceIds){ + RegistryPublisher pub = RegistryPublisherFactory.create(); + pub.vosUpdate(resource); - ScopeProvider.instance.set(currentScope); - SimpleQuery query = ICFactory.queryFor(resourceType); - DiscoveryClient client = ICFactory.clientFor(resourceType); - query.addCondition("$resource/ID/text() = '"+resourceId+"'"); - T resource = client.submit(query).get(0); - - resource.scopes().asCollection().retainAll(Collections.emptyList()); - resource.scopes().asCollection().addAll(scopeSet); - - RegistryPublisher pub = RegistryPublisherFactory.create(); - pub.vosUpdate(resource); - } } /**{@inheritDoc}*/ @Override protected void onStop() throws Exception { - logger.debug("onStop() invoked"); Thread.currentThread().interrupt(); + logger.debug("onStop() invoked"); } } diff --git a/src/main/java/org/gcube/informationsystem/resource_checker/ResourceCheckerPluginDeclaration.java b/src/main/java/org/gcube/informationsystem/resource_checker/ResourceCheckerPluginDeclaration.java index 04dfe2e..c86da6c 100644 --- a/src/main/java/org/gcube/informationsystem/resource_checker/ResourceCheckerPluginDeclaration.java +++ b/src/main/java/org/gcube/informationsystem/resource_checker/ResourceCheckerPluginDeclaration.java @@ -28,7 +28,7 @@ public class ResourceCheckerPluginDeclaration implements PluginDeclaration { */ public static final String NAME = "resource-checker-se-plugin"; public static final String DESCRIPTION = "The resource-checker-plugin has the role to check the existence of some resources in all Infrastructure's contexts."; - public static final String VERSION = "1.0.0"; + public static final String VERSION = "1.1.0"; /**{@inheritDoc}*/ @Override diff --git a/src/main/java/org/gcube/informationsystem/resource_checker/utils/BasicFunctionalityBean.java b/src/main/java/org/gcube/informationsystem/resource_checker/beans/BasicFunctionalityBean.java similarity index 58% rename from src/main/java/org/gcube/informationsystem/resource_checker/utils/BasicFunctionalityBean.java rename to src/main/java/org/gcube/informationsystem/resource_checker/beans/BasicFunctionalityBean.java index d77f910..613a80d 100644 --- a/src/main/java/org/gcube/informationsystem/resource_checker/utils/BasicFunctionalityBean.java +++ b/src/main/java/org/gcube/informationsystem/resource_checker/beans/BasicFunctionalityBean.java @@ -1,4 +1,4 @@ -package org.gcube.informationsystem.resource_checker.utils; +package org.gcube.informationsystem.resource_checker.beans; import org.gcube.common.resources.gcore.GCoreEndpoint; import org.gcube.common.resources.gcore.GenericResource; @@ -8,7 +8,9 @@ import org.gcube.common.resources.gcore.Resource.Type; import org.gcube.common.resources.gcore.Software; /** - * A basic functionality bean with name, category and type + * A basic functionality bean with name, category and type. + * Resource Identifier will be discovered at run time. It is supposed that the same resource + * is published in all contexts (so the identifier is the same). * @author Costantino Perciante at ISTI-CNR (costantino.perciante@isti.cnr.it) */ @SuppressWarnings("rawtypes") @@ -16,14 +18,17 @@ public class BasicFunctionalityBean { private String category; private String name; - private Class type; // service endpoint only for now + private Class type; + private String resourceId; + private OperationLevel opLevel = OperationLevel.ALERT_READD; + private ContextLevel ctxLevel = ContextLevel.ALL; /** * @param category * @param name * @param type */ - public BasicFunctionalityBean(String category, String name, String type) { + public BasicFunctionalityBean(String category, String name, String type, String opLevel, String ctxLevel) { super(); this.category = category; this.name = name; @@ -34,9 +39,6 @@ public class BasicFunctionalityBean { Type extractedType = Type.valueOf(type); switch(extractedType){ - case ENDPOINT: - this.type = ServiceEndpoint.class; - break; case GCOREENDPOINT: this.type = GCoreEndpoint.class; break; @@ -49,10 +51,18 @@ public class BasicFunctionalityBean { case SOFTWARE: this.type = Software.class; break; + case ENDPOINT: default: this.type = ServiceEndpoint.class; } } + + if(opLevel != null) + this.opLevel = OperationLevel.valueOf(opLevel); + + if(ctxLevel != null) + this.ctxLevel = ContextLevel.valueOf(ctxLevel); + } public String getCategory() { return category; @@ -72,10 +82,29 @@ public class BasicFunctionalityBean { public void Class(Class type) { this.type = type; } + public String getResourceId() { + return resourceId; + } + public void setResourceId(String resourceId) { + this.resourceId = resourceId; + } + public OperationLevel getOpLevel() { + return opLevel; + } + public void setOpLevel(OperationLevel opLevel) { + this.opLevel = opLevel; + } + public ContextLevel getCtxLevel() { + return ctxLevel; + } + public void setCtxLevel(ContextLevel ctxLevel) { + this.ctxLevel = ctxLevel; + } @Override public String toString() { return "BasicFunctionalityBean [category=" + category + ", name=" - + name + ", type=" + type + "]"; + + name + ", type=" + type + ", resourceId=" + resourceId + + ", opLevel=" + opLevel + ", ctxLevel=" + ctxLevel + "]"; } } diff --git a/src/main/java/org/gcube/informationsystem/resource_checker/beans/ContextLevel.java b/src/main/java/org/gcube/informationsystem/resource_checker/beans/ContextLevel.java new file mode 100644 index 0000000..474c726 --- /dev/null +++ b/src/main/java/org/gcube/informationsystem/resource_checker/beans/ContextLevel.java @@ -0,0 +1,12 @@ +package org.gcube.informationsystem.resource_checker.beans; + +/** + * Evaluate at which level the resource must be checked. + * @author Costantino Perciante at ISTI-CNR (costantino.perciante@isti.cnr.it) + */ +public enum ContextLevel { + + VO, // check only at VO level (and root vo) + ALL // check at all levels + +} diff --git a/src/main/java/org/gcube/informationsystem/resource_checker/beans/OperationLevel.java b/src/main/java/org/gcube/informationsystem/resource_checker/beans/OperationLevel.java new file mode 100644 index 0000000..75de51e --- /dev/null +++ b/src/main/java/org/gcube/informationsystem/resource_checker/beans/OperationLevel.java @@ -0,0 +1,12 @@ +package org.gcube.informationsystem.resource_checker.beans; + +/** + * Enumerators for the actions to be taken wrt missing resources: ALERT only or ALERT and READD + * @author Costantino Perciante at ISTI-CNR (costantino.perciante@isti.cnr.it) + */ +public enum OperationLevel { + + ALERT_READD, // alert and readd the resource + ALERT // just alert it is missing + +} diff --git a/src/main/java/org/gcube/informationsystem/resource_checker/utils/BasicFunctionalitiesMandatoryReader.java b/src/main/java/org/gcube/informationsystem/resource_checker/utils/BasicFunctionalitiesMandatoryReader.java index 6eddb01..a549ba0 100644 --- a/src/main/java/org/gcube/informationsystem/resource_checker/utils/BasicFunctionalitiesMandatoryReader.java +++ b/src/main/java/org/gcube/informationsystem/resource_checker/utils/BasicFunctionalitiesMandatoryReader.java @@ -5,6 +5,7 @@ import java.util.ArrayList; import java.util.List; import java.util.Properties; +import org.gcube.informationsystem.resource_checker.beans.BasicFunctionalityBean; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -23,6 +24,8 @@ public class BasicFunctionalitiesMandatoryReader { private static final String SERVICE_NAME_KEY = "ServiceNames"; private static final String CATEGORY_NAME_KEY = "CategoryNames"; private static final String TYPES = "Types"; + private static final String CONTEXTS = "Contexts"; + private static final String OPERATIONS = "Operations"; private static final String SEPARATOR = ","; public BasicFunctionalitiesMandatoryReader() throws Exception{ @@ -42,17 +45,27 @@ public class BasicFunctionalitiesMandatoryReader { String serviceNames = prop.getProperty(SERVICE_NAME_KEY); String serviceCategories = prop.getProperty(CATEGORY_NAME_KEY); String types = prop.getProperty(TYPES); + String contexts = prop.getProperty(CONTEXTS); + String operations = prop.getProperty(OPERATIONS); if(serviceNames == null || serviceCategories == null) throw new Exception("Service names or categories are missing in file " + FILE_PROPRETIES_LOCATION_PATH); String[] serviceNamesSplitted = serviceNames.split(SEPARATOR); String[] serviceCategoriesSplitted = serviceCategories.split(SEPARATOR); - String[] typesSplitted = null; + String[] contextsSplitted = null; + String[] operationsSplitted = null; + if(types != null) typesSplitted = types.split(SEPARATOR); + if(contexts != null) + contextsSplitted = contexts.split(SEPARATOR); + + if(operations != null) + operationsSplitted = operations.split(SEPARATOR); + // build the java objects if(serviceNamesSplitted.length != serviceCategoriesSplitted.length) throw new Exception("The file at " + FILE_PROPRETIES_LOCATION_PATH + " seems to be malformed (service names and categories number do not match)!"); @@ -60,10 +73,21 @@ public class BasicFunctionalitiesMandatoryReader { if(typesSplitted != null && serviceNamesSplitted.length != typesSplitted.length) throw new Exception("The file at " + FILE_PROPRETIES_LOCATION_PATH + " seems to be malformed (types lenght doesn't match the other properties)!"); + if(contextsSplitted != null && serviceNamesSplitted.length != contextsSplitted.length) + throw new Exception("The file at " + FILE_PROPRETIES_LOCATION_PATH + " seems to be malformed (contexts lenght doesn't match the other properties)!"); + + if(operationsSplitted != null && serviceNamesSplitted.length != operationsSplitted.length) + throw new Exception("The file at " + FILE_PROPRETIES_LOCATION_PATH + " seems to be malformed (operations lenght doesn't match the other properties)!"); + // Build the objects mandatoryFunctionalities = new ArrayList(); for (int i = 0; i < serviceCategoriesSplitted.length; i++) { - mandatoryFunctionalities.add(new BasicFunctionalityBean(serviceCategoriesSplitted[i], serviceNamesSplitted[i], typesSplitted != null ? typesSplitted[i] : null)); + mandatoryFunctionalities.add(new BasicFunctionalityBean( + serviceCategoriesSplitted[i], + serviceNamesSplitted[i], + typesSplitted != null ? typesSplitted[i] : null, + operationsSplitted != null ? operationsSplitted[i] : null, + contextsSplitted != null ? contextsSplitted[i] : null )); } logger.info("Built list is " + mandatoryFunctionalities); diff --git a/src/main/java/org/gcube/informationsystem/resource_checker/utils/RetrieveContextsList.java b/src/main/java/org/gcube/informationsystem/resource_checker/utils/RetrieveContextsList.java index 537dc93..f8683f6 100644 --- a/src/main/java/org/gcube/informationsystem/resource_checker/utils/RetrieveContextsList.java +++ b/src/main/java/org/gcube/informationsystem/resource_checker/utils/RetrieveContextsList.java @@ -48,7 +48,7 @@ public class RetrieveContextsList { SimpleQuery queryVRE = ICFactory.queryFor(GenericResource.class); queryVRE.addCondition("$resource/Profile/SecondaryType/text() = 'VRE'"); queryVRE.setResult("$resource/Profile/Body/Scope/string()"); - + String currentScope = ScopeProvider.instance.get(); DiscoveryClient vREScopeClient = ICFactory.client(); List vres = new ArrayList(); @@ -61,6 +61,7 @@ public class RetrieveContextsList { } voContexts.addAll(vres); + ScopeProvider.instance.set(currentScope); logger.info("List of all contexts is " + voContexts); diff --git a/src/main/java/org/gcube/informationsystem/resource_checker/utils/SendNotification.java b/src/main/java/org/gcube/informationsystem/resource_checker/utils/SendNotification.java index 87fb3e2..5d7138a 100644 --- a/src/main/java/org/gcube/informationsystem/resource_checker/utils/SendNotification.java +++ b/src/main/java/org/gcube/informationsystem/resource_checker/utils/SendNotification.java @@ -26,6 +26,7 @@ import org.gcube.common.authorization.library.provider.SecurityTokenProvider; import org.gcube.common.resources.gcore.GCoreEndpoint; import org.gcube.common.scope.api.ScopeProvider; import org.gcube.informationsystem.resource_checker.ResourceCheckerPluginDeclaration; +import org.gcube.informationsystem.resource_checker.beans.BasicFunctionalityBean; import org.gcube.resources.discovery.client.api.DiscoveryClient; import org.gcube.resources.discovery.client.queries.api.SimpleQuery; import org.gcube.vremanagement.executor.plugin.PluginStateEvolution; @@ -205,11 +206,13 @@ public class SendNotification extends PluginStateNotification { sb.append("none."); while (iterator.hasNext()) { - Map.Entry> entry = (Map.Entry>) iterator + Map.Entry> entry = (Map.Entry>) iterator .next(); List list = entry.getValue(); for (BasicFunctionalityBean basicFunctionalityBean : list) { - sb.append("- resource's name "); + sb.append("- resource's id is "); + sb.append(basicFunctionalityBean.getResourceId()); + sb.append(", resource's name "); sb.append(basicFunctionalityBean.getName()); sb.append(" and resource's class/category "); sb.append(basicFunctionalityBean.getCategory()); diff --git a/src/main/resources/META-INF/plugin_resources/resources_to_fetch.properties b/src/main/resources/META-INF/plugin_resources/resources_to_fetch.properties index 0925bd2..3857be8 100644 --- a/src/main/resources/META-INF/plugin_resources/resources_to_fetch.properties +++ b/src/main/resources/META-INF/plugin_resources/resources_to_fetch.properties @@ -1,5 +1,11 @@ -#Resources to fetch are reported here. It can be read as follows +# Resources to fetch are reported here. It can be read as follows # check for resource having (ServiceName = ServiceNames[i], CategoryName = CategoryNames[i]) # where i stands for the i-th position -ServiceNames=HTTP-URI-Resolver,Persistence,StorageManager -CategoryNames=Service,Accounting,DataStorage \ No newline at end of file +# For Types look at org.gcube.informationsystem.resource_checker.beans.BasicFunctionalityBean +# For Contexts: ALL means it has to check a resource in all contexts, VO just in ROOT VO and VOs +# For Operations: ALERT_READD means the service is readded and alert is sent, or only alert is sent (ALERT) +ServiceNames=HTTP-URI-Resolver,Persistence,StorageManager,HomeLibraryWebapp +CategoryNames=Service,Accounting,DataStorage,DataAccess +Types=ENDPOINT,ENDPOINT,ENDPOINT,GCOREENDPOINT +Contexts=ALL,ALL,ALL,VO +Operations=ALERT_READD,ALERT_READD,ALERT_READD,ALERT \ No newline at end of file