From 40df7bb161b366633b2a61f0ef0102ef208c23c2 Mon Sep 17 00:00:00 2001 From: Luca Frosini Date: Mon, 8 Feb 2016 10:12:24 +0000 Subject: [PATCH] refs #2032: Create IS Sweeper as SmartExecutor Plugin https://support.d4science.org/issues/2032 git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/information-system/is-sweeper-se-plugin@122892 82a268e6-3cf1-43bd-a215-b396298e98cf --- pom.xml | 35 ++-- .../sweeper/ISSweeperPlugin.java | 93 ++++------ .../informationsystem/sweeper/Sweeper.java | 163 ++++++++++++++++++ .../sweeper/ISSweeperPluginTest.java | 4 +- src/test/resources/logback-test.xml | 3 +- 5 files changed, 216 insertions(+), 82 deletions(-) create mode 100644 src/main/java/org/gcube/informationsystem/sweeper/Sweeper.java diff --git a/pom.xml b/pom.xml index f1575a1..5512540 100644 --- a/pom.xml +++ b/pom.xml @@ -39,41 +39,32 @@ + + org.gcube.vremanagement + smart-executor-api + [1.2.0-SNAPSHOT, 2.0.0-SNAPSHOT) + + org.gcube.resources registry-publisher provided - org.gcube.vremanagement - smart-executor-api - [1.2.0-SNAPSHOT, 2.0.0-SNAPSHOT) - - - org.slf4j - slf4j-api + org.gcube.resources + common-gcore-resources provided - - org.gcube.portlets.admin - rmp-common-library - [2.6.0-SNAPSHOT,3.0.0-SNAPSHOT) - - - com.google.gwt - gwt-user - - - com.sencha.gxt - gxt2.2.5-gwt2.X - - - org.gcube.core common-scope provided + + org.slf4j + slf4j-api + provided + diff --git a/src/main/java/org/gcube/informationsystem/sweeper/ISSweeperPlugin.java b/src/main/java/org/gcube/informationsystem/sweeper/ISSweeperPlugin.java index e4f49ec..5f645f3 100644 --- a/src/main/java/org/gcube/informationsystem/sweeper/ISSweeperPlugin.java +++ b/src/main/java/org/gcube/informationsystem/sweeper/ISSweeperPlugin.java @@ -1,15 +1,12 @@ package org.gcube.informationsystem.sweeper; -import java.util.List; +import java.util.Calendar; import java.util.Map; -import org.gcube.common.resources.gcore.HostingNode; +import org.gcube.common.authorization.client.Constants; +import org.gcube.common.authorization.library.AuthorizationEntry; +import org.gcube.common.authorization.library.provider.SecurityTokenProvider; import org.gcube.common.scope.api.ScopeProvider; -import org.gcube.common.scope.impl.ScopeBean; -import org.gcube.resourcemanagement.support.server.managers.resources.GHNManager; -import org.gcube.resourcemanagement.support.server.managers.resources.RunningInstanceManager; -import org.gcube.resourcemanagement.support.server.sweeper.Sweeper; -import org.gcube.resourcemanagement.support.shared.util.SweeperActions; import org.gcube.vremanagement.executor.plugin.Plugin; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -29,70 +26,52 @@ public class ISSweeperPlugin extends Plugin { logger.debug("contructor"); } + /* public static String getTagValue(String xml, String tagName){ return xml.split("<"+tagName+">")[1].split("")[0]; } - - @SuppressWarnings("deprecation") - public void applySweep(ScopeBean scopeBean, List elements) { - for (String element : elements) { - try { - String elementID = getTagValue(element, "ID"); - SweeperActions action = SweeperActions.valueOf(getTagValue(element, "Actions")); - logger.info("Cleaning up {} {}", elementID, action); + */ + - switch(action) { - case APPLY_GHN_DELETE: - GHNManager manager = new GHNManager(elementID); - manager.forceDelete(scopeBean); - break; - case APPLY_GHN_MOVE_TO_UNREACHABLE: - GHNManager ghnManager = new GHNManager(elementID); - HostingNode res = (HostingNode) ghnManager.getResource(scopeBean); - logger.trace("Setting {} (ID : {}) status to unreachable", - HostingNode.class.getSimpleName(), elementID); - res.profile().description().status("unreachable"); - HostingNode hn = ghnManager.getRegistryPublisher().update(res); - logger.trace("{} (ID : {}) get after update {}", - HostingNode.class.getSimpleName(), elementID, hn); - break; - case APPLY_RI_DELETE: - RunningInstanceManager riManager = new RunningInstanceManager(elementID); - logger.trace("RunningInstance with ID {} will be deleted"); - riManager.forceDelete(scopeBean); - break; - default: - } - } catch (Exception e) { - logger.error("Error cleaning {}", element, e); - } - } - } - - /**{@inheritDoc}*/ @Override public void launch(Map inputs) throws Exception { logger.debug("Launching {} execution", ISSweeperPluginDeclaration.NAME); // No inputs needed - String scope = ScopeProvider.instance.get(); - ScopeBean scopeBean = new ScopeBean(scope); + String token = SecurityTokenProvider.instance.get(); + AuthorizationEntry authorizationEntry = Constants.authorizationService().get(token); + String scope = authorizationEntry.getContext(); + ScopeProvider.instance.set(scope); Sweeper sweeper = new Sweeper(); - List expiredGHNs = sweeper.getExpiredGHNs(scopeBean); - logger.trace("Expired GHNs : {}", expiredGHNs); - applySweep(scopeBean, expiredGHNs); - - List deadGHNs = sweeper.getDeadGHNs(scopeBean); - logger.trace("Dead GHNs : {}", deadGHNs); - applySweep(scopeBean, deadGHNs); + try { + sweeper.sweepDeadGHNs(Calendar.DAY_OF_YEAR, -15); + } catch(Exception e){ + logger.error("Error removing Dead HostingNodes", e); + } + logger.trace("---------------------------------\n\n"); + + try { + sweeper.sweepExpiredGHNs(Calendar.MINUTE, -30); + logger.trace("---------------------------------\n\n"); + } catch(Exception e){ + logger.error("Error sweeping Expired HostingNodes", e); + } + logger.trace("---------------------------------\n\n"); + + Thread.sleep(1000*90); // Waiting 90 sec + + try { + sweeper.sweepOrphanRI(); + logger.trace("---------------------------------\n\n"); + } catch(Exception e){ + logger.error("Error sweeping Orphan RunningInstances", e); + } + + - List orphanRIs = sweeper.getOrphanRI(scopeBean); - logger.trace("Orphan Running Instances : {}", orphanRIs); - applySweep(scopeBean, orphanRIs); - logger.debug("{} execution finished", ISSweeperPluginDeclaration.NAME); } diff --git a/src/main/java/org/gcube/informationsystem/sweeper/Sweeper.java b/src/main/java/org/gcube/informationsystem/sweeper/Sweeper.java new file mode 100644 index 0000000..286ca8f --- /dev/null +++ b/src/main/java/org/gcube/informationsystem/sweeper/Sweeper.java @@ -0,0 +1,163 @@ +package org.gcube.informationsystem.sweeper; + +import java.util.Calendar; +import java.util.List; + +import org.gcube.common.resources.gcore.GCoreEndpoint; +import org.gcube.common.resources.gcore.HostingNode; +import org.gcube.common.resources.gcore.utils.DateFormatterAdapter; +import org.gcube.informationsystem.publisher.RegistryPublisher; +import org.gcube.informationsystem.publisher.RegistryPublisherFactory; +import org.gcube.resources.discovery.client.api.DiscoveryClient; +import org.gcube.resources.discovery.client.queries.api.SimpleQuery; +import org.gcube.resources.discovery.icclient.ICFactory; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * @author Luca Frosini (ISTI - CNR) http://www.lucafrosini.com/ + */ +public class Sweeper { + + private static final Logger logger = LoggerFactory.getLogger(Sweeper.class); + + protected RegistryPublisher registryPublisher; + protected DateFormatterAdapter dateFormatterAdapter; + + public Sweeper(){ + registryPublisher = RegistryPublisherFactory.create(); + dateFormatterAdapter = new DateFormatterAdapter(); + } + + protected final static String UNREACHABLE = "unreachable"; + protected final static String CERTIFIED = "certified"; + + protected String getHostingNodeMinimalInfo(HostingNode hostingNode) throws Exception{ + Calendar lastUpdate = hostingNode.profile().description().lastUpdate(); + return String.format("%s (ID : %s - Name : %s - Status : %s - LastUpdate : %s)", + HostingNode.class.getSimpleName(), hostingNode.id(), + hostingNode.profile().description().name(), + hostingNode.profile().description().status(), + dateFormatterAdapter.marshal(lastUpdate)); + } + + protected String getGCoreEndpointMinimalInfo(GCoreEndpoint gCoreEndpoint, HostingNode hostingNode) throws Exception{ + return String.format("%s (ID : %s - ServiceClass : %s - ServiceName : %s. Was running on %s)", + GCoreEndpoint.class.getSimpleName(), gCoreEndpoint.id(), + gCoreEndpoint.profile().serviceClass(), + gCoreEndpoint.profile().serviceName(), + getHostingNodeMinimalInfo(hostingNode)); + } + + public void sweepExpiredGHNs(int expiringField, int expiringQuantity) throws Exception { + Calendar expiryCalendar = Calendar.getInstance(); + expiryCalendar.add(expiringField, expiringQuantity); + + String formattedDate = dateFormatterAdapter.marshal(expiryCalendar); + + String condition = String.format( + "xs:dateTime($resource/Profile/GHNDescription/LastUpdate/text()) lt xs:dateTime('%s')", + formattedDate); + + SimpleQuery query = ICFactory.queryFor(HostingNode.class) + .addCondition(String.format("$resource/Profile/GHNDescription/Status/text() eq '%s'", CERTIFIED)) + .addCondition(condition) + .setResult("$resource"); + + DiscoveryClient client = ICFactory.clientFor(HostingNode.class); + List hostingNodes = client.submit(query); + + for (HostingNode hostingNode : hostingNodes) { + try { + + logger.debug("Setting {} status to {}", + getHostingNodeMinimalInfo(hostingNode), + UNREACHABLE); + + hostingNode.profile().description().status(UNREACHABLE); + registryPublisher.update(hostingNode); + + logger.debug("Request to set status to {} for {} successfully sent\n", + UNREACHABLE, getHostingNodeMinimalInfo(hostingNode)); + + }catch(Exception e){ + logger.error("Unable to set status to {} for {}", UNREACHABLE, + getHostingNodeMinimalInfo(hostingNode), e); + } + } + } + + public void sweepDeadGHNs(int deadField, int deadQuantity) throws Exception { + Calendar deadCalendar = Calendar.getInstance(); + deadCalendar.add(deadField, deadQuantity); + + String formattedDate = dateFormatterAdapter.marshal(deadCalendar); + + String condition = String.format( + "xs:dateTime($resource/Profile/GHNDescription/LastUpdate/text()) lt xs:dateTime('%s')", + formattedDate); + + SimpleQuery query = ICFactory.queryFor(HostingNode.class) + .addCondition(condition) + .setResult("$resource"); + + DiscoveryClient client = ICFactory.clientFor(HostingNode.class); + List hostingNodes = client.submit(query); + + for (HostingNode hostingNode : hostingNodes) { + try { + + logger.debug("Going to remove dead {}", + getHostingNodeMinimalInfo(hostingNode)); + + registryPublisher.remove(hostingNode); + + logger.debug("Request to remove {} successfully sent\n", + getHostingNodeMinimalInfo(hostingNode)); + }catch(Exception e){ + logger.error("Unable to remove {}", + getHostingNodeMinimalInfo(hostingNode), e); + } + } + + } + + public void sweepOrphanRI() throws Exception { + + SimpleQuery hostingNodeQuery = ICFactory.queryFor(HostingNode.class) + .addCondition(String.format("$resource/Profile/GHNDescription/Status/text() ne '%s'", CERTIFIED)) + .setResult("$resource"); + DiscoveryClient hostingNodeClient = ICFactory.clientFor(HostingNode.class); + List hostingNodes = hostingNodeClient.submit(hostingNodeQuery); + + for(HostingNode hostingNode : hostingNodes){ + String condition = String.format("$resource/Profile/GHN/@UniqueID/string() eq '%s'", hostingNode.id()); + + SimpleQuery query = ICFactory.queryFor(GCoreEndpoint.class) + .addCondition(condition) + .setResult("$resource"); + + DiscoveryClient client = ICFactory.clientFor(GCoreEndpoint.class); + List gCoreEndpoints = client.submit(query); + + for (GCoreEndpoint gCoreEndpoint : gCoreEndpoints) { + try { + + logger.debug("Going to remove orphan {}", + getGCoreEndpointMinimalInfo(gCoreEndpoint, hostingNode)); + + registryPublisher.remove(hostingNode); + + logger.debug("Request to remove {} successfully sent\n", + getGCoreEndpointMinimalInfo(gCoreEndpoint, hostingNode)); + }catch(Exception e){ + logger.error("Unable to remove {}", + getGCoreEndpointMinimalInfo(gCoreEndpoint, hostingNode), e); + } + } + + } + + } + +} diff --git a/src/test/java/org/gcube/informationsystem/sweeper/ISSweeperPluginTest.java b/src/test/java/org/gcube/informationsystem/sweeper/ISSweeperPluginTest.java index dd6775d..90d384b 100644 --- a/src/test/java/org/gcube/informationsystem/sweeper/ISSweeperPluginTest.java +++ b/src/test/java/org/gcube/informationsystem/sweeper/ISSweeperPluginTest.java @@ -3,7 +3,7 @@ package org.gcube.informationsystem.sweeper; import java.util.HashMap; import java.util.Map; -import org.gcube.common.scope.api.ScopeProvider; +import org.gcube.common.authorization.library.provider.SecurityTokenProvider; import org.junit.Before; import org.junit.Test; import org.slf4j.Logger; @@ -21,7 +21,7 @@ public class ISSweeperPluginTest { @Before public void beforeTest(){ - ScopeProvider.instance.set("/gcube/devsec"); + SecurityTokenProvider.instance.set("59074832-f986-46ce-9e46-5f2211300f9c"); } @Test diff --git a/src/test/resources/logback-test.xml b/src/test/resources/logback-test.xml index 4f36cc8..4e0f8be 100644 --- a/src/test/resources/logback-test.xml +++ b/src/test/resources/logback-test.xml @@ -7,7 +7,8 @@ - + +