gCubeSystem
/
is-sweeper-se-plugin
16
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

refs #2032: Create IS Sweeper as SmartExecutor Plugin 

https://support.d4science.org/issues/2032

git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/information-system/is-sweeper-se-plugin@122892 82a268e6-3cf1-43bd-a215-b396298e98cf
This commit is contained in:
Luca Frosini 2016-02-08 10:12:24 +00:00
parent aa1acd0e82
commit 40df7bb161
5 changed files with 216 additions and 82 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

35
pom.xml
View File

@ -39,41 +39,32 @@
</properties>
<dependencies>
<dependency>
<groupId>org.gcube.vremanagement</groupId>
<artifactId>smart-executor-api</artifactId>
<version>[1.2.0-SNAPSHOT, 2.0.0-SNAPSHOT)</version>
</dependency>
<dependency>
<groupId>org.gcube.resources</groupId>
<artifactId>registry-publisher</artifactId>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.gcube.vremanagement</groupId>
<artifactId>smart-executor-api</artifactId>
<version>[1.2.0-SNAPSHOT, 2.0.0-SNAPSHOT)</version>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
<groupId>org.gcube.resources</groupId>
<artifactId>common-gcore-resources</artifactId>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.gcube.portlets.admin</groupId>
<artifactId>rmp-common-library</artifactId>
<version>[2.6.0-SNAPSHOT,3.0.0-SNAPSHOT)</version>
<exclusions>
<exclusion>
<groupId>com.google.gwt</groupId>
<artifactId>gwt-user</artifactId>
</exclusion>
<exclusion>
<groupId>com.sencha.gxt</groupId>
<artifactId>gxt2.2.5-gwt2.X</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.gcube.core</groupId>
<artifactId>common-scope</artifactId>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
<scope>provided</scope>
</dependency>
<!-- Test dependencies -->
<dependency>

93
src/main/java/org/gcube/informationsystem/sweeper/ISSweeperPlugin.java
View File

@ -1,15 +1,12 @@
package org.gcube.informationsystem.sweeper;
import java.util.List;
import java.util.Calendar;
import java.util.Map;
import org.gcube.common.resources.gcore.HostingNode;
import org.gcube.common.authorization.client.Constants;
import org.gcube.common.authorization.library.AuthorizationEntry;
import org.gcube.common.authorization.library.provider.SecurityTokenProvider;
import org.gcube.common.scope.api.ScopeProvider;
import org.gcube.common.scope.impl.ScopeBean;
import org.gcube.resourcemanagement.support.server.managers.resources.GHNManager;
import org.gcube.resourcemanagement.support.server.managers.resources.RunningInstanceManager;
import org.gcube.resourcemanagement.support.server.sweeper.Sweeper;
import org.gcube.resourcemanagement.support.shared.util.SweeperActions;
import org.gcube.vremanagement.executor.plugin.Plugin;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -29,70 +26,52 @@ public class ISSweeperPlugin extends Plugin<ISSweeperPluginDeclaration> {
logger.debug("contructor");
}
/*
public static String getTagValue(String xml, String tagName){
return xml.split("<"+tagName+">")[1].split("</"+tagName+">")[0];
}
@SuppressWarnings("deprecation")
public void applySweep(ScopeBean scopeBean, List<String> elements) {
for (String element : elements) {
try {
String elementID = getTagValue(element, "ID");
SweeperActions action = SweeperActions.valueOf(getTagValue(element, "Actions"));
logger.info("Cleaning up {} {}", elementID, action);
*/
switch(action) {
case APPLY_GHN_DELETE:
GHNManager manager = new GHNManager(elementID);
manager.forceDelete(scopeBean);
break;
case APPLY_GHN_MOVE_TO_UNREACHABLE:
GHNManager ghnManager = new GHNManager(elementID);
HostingNode res = (HostingNode) ghnManager.getResource(scopeBean);
logger.trace("Setting {} (ID : {}) status to unreachable",
HostingNode.class.getSimpleName(), elementID);
res.profile().description().status("unreachable");
HostingNode hn = ghnManager.getRegistryPublisher().update(res);
logger.trace("{} (ID : {}) get after update {}",
HostingNode.class.getSimpleName(), elementID, hn);
break;
case APPLY_RI_DELETE:
RunningInstanceManager riManager = new RunningInstanceManager(elementID);
logger.trace("RunningInstance with ID {} will be deleted");
riManager.forceDelete(scopeBean);
break;
default:
}
} catch (Exception e) {
logger.error("Error cleaning {}", element, e);
}
}
}
/**{@inheritDoc}*/
@Override
public void launch(Map<String, Object> inputs) throws Exception {
logger.debug("Launching {} execution", ISSweeperPluginDeclaration.NAME);
// No inputs needed
String scope = ScopeProvider.instance.get();
ScopeBean scopeBean = new ScopeBean(scope);
String token = SecurityTokenProvider.instance.get();
AuthorizationEntry authorizationEntry = Constants.authorizationService().get(token);
String scope = authorizationEntry.getContext();
ScopeProvider.instance.set(scope);
Sweeper sweeper = new Sweeper();
List<String> expiredGHNs = sweeper.getExpiredGHNs(scopeBean);
logger.trace("Expired GHNs : {}", expiredGHNs);
applySweep(scopeBean, expiredGHNs);
List<String> deadGHNs = sweeper.getDeadGHNs(scopeBean);
logger.trace("Dead GHNs : {}", deadGHNs);
applySweep(scopeBean, deadGHNs);
try {
sweeper.sweepDeadGHNs(Calendar.DAY_OF_YEAR, -15);
} catch(Exception e){
logger.error("Error removing Dead HostingNodes", e);
}
logger.trace("---------------------------------\n\n");
try {
sweeper.sweepExpiredGHNs(Calendar.MINUTE, -30);
logger.trace("---------------------------------\n\n");
} catch(Exception e){
logger.error("Error sweeping Expired HostingNodes", e);
}
logger.trace("---------------------------------\n\n");
Thread.sleep(1000*90); // Waiting 90 sec
try {
sweeper.sweepOrphanRI();
logger.trace("---------------------------------\n\n");
} catch(Exception e){
logger.error("Error sweeping Orphan RunningInstances", e);
}
List<String> orphanRIs = sweeper.getOrphanRI(scopeBean);
logger.trace("Orphan Running Instances : {}", orphanRIs);
applySweep(scopeBean, orphanRIs);
logger.debug("{} execution finished", ISSweeperPluginDeclaration.NAME);
}

163
src/main/java/org/gcube/informationsystem/sweeper/Sweeper.java Normal file
View File

@ -0,0 +1,163 @@
package org.gcube.informationsystem.sweeper;
import java.util.Calendar;
import java.util.List;
import org.gcube.common.resources.gcore.GCoreEndpoint;
import org.gcube.common.resources.gcore.HostingNode;
import org.gcube.common.resources.gcore.utils.DateFormatterAdapter;
import org.gcube.informationsystem.publisher.RegistryPublisher;
import org.gcube.informationsystem.publisher.RegistryPublisherFactory;
import org.gcube.resources.discovery.client.api.DiscoveryClient;
import org.gcube.resources.discovery.client.queries.api.SimpleQuery;
import org.gcube.resources.discovery.icclient.ICFactory;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* @author Luca Frosini (ISTI - CNR) http://www.lucafrosini.com/
*/
public class Sweeper {
private static final Logger logger = LoggerFactory.getLogger(Sweeper.class);
protected RegistryPublisher registryPublisher;
protected DateFormatterAdapter dateFormatterAdapter;
public Sweeper(){
registryPublisher = RegistryPublisherFactory.create();
dateFormatterAdapter = new DateFormatterAdapter();
}
protected final static String UNREACHABLE = "unreachable";
protected final static String CERTIFIED = "certified";
protected String getHostingNodeMinimalInfo(HostingNode hostingNode) throws Exception{
Calendar lastUpdate = hostingNode.profile().description().lastUpdate();
return String.format("%s (ID : %s - Name : %s - Status : %s - LastUpdate : %s)",
HostingNode.class.getSimpleName(), hostingNode.id(),
hostingNode.profile().description().name(),
hostingNode.profile().description().status(),
dateFormatterAdapter.marshal(lastUpdate));
}
protected String getGCoreEndpointMinimalInfo(GCoreEndpoint gCoreEndpoint, HostingNode hostingNode) throws Exception{
return String.format("%s (ID : %s - ServiceClass : %s - ServiceName : %s. Was running on %s)",
GCoreEndpoint.class.getSimpleName(), gCoreEndpoint.id(),
gCoreEndpoint.profile().serviceClass(),
gCoreEndpoint.profile().serviceName(),
getHostingNodeMinimalInfo(hostingNode));
}
public void sweepExpiredGHNs(int expiringField, int expiringQuantity) throws Exception {
Calendar expiryCalendar = Calendar.getInstance();
expiryCalendar.add(expiringField, expiringQuantity);
String formattedDate = dateFormatterAdapter.marshal(expiryCalendar);
String condition = String.format(
"xs:dateTime($resource/Profile/GHNDescription/LastUpdate/text()) lt xs:dateTime('%s')",
formattedDate);
SimpleQuery query = ICFactory.queryFor(HostingNode.class)
.addCondition(String.format("$resource/Profile/GHNDescription/Status/text() eq '%s'", CERTIFIED))
.addCondition(condition)
.setResult("$resource");
DiscoveryClient<HostingNode> client = ICFactory.clientFor(HostingNode.class);
List<HostingNode> hostingNodes = client.submit(query);
for (HostingNode hostingNode : hostingNodes) {
try {
logger.debug("Setting {} status to {}",
getHostingNodeMinimalInfo(hostingNode),
UNREACHABLE);
hostingNode.profile().description().status(UNREACHABLE);
registryPublisher.update(hostingNode);
logger.debug("Request to set status to {} for {} successfully sent\n",
UNREACHABLE, getHostingNodeMinimalInfo(hostingNode));
}catch(Exception e){
logger.error("Unable to set status to {} for {}", UNREACHABLE,
getHostingNodeMinimalInfo(hostingNode), e);
}
}
}
public void sweepDeadGHNs(int deadField, int deadQuantity) throws Exception {
Calendar deadCalendar = Calendar.getInstance();
deadCalendar.add(deadField, deadQuantity);
String formattedDate = dateFormatterAdapter.marshal(deadCalendar);
String condition = String.format(
"xs:dateTime($resource/Profile/GHNDescription/LastUpdate/text()) lt xs:dateTime('%s')",
formattedDate);
SimpleQuery query = ICFactory.queryFor(HostingNode.class)
.addCondition(condition)
.setResult("$resource");
DiscoveryClient<HostingNode> client = ICFactory.clientFor(HostingNode.class);
List<HostingNode> hostingNodes = client.submit(query);
for (HostingNode hostingNode : hostingNodes) {
try {
logger.debug("Going to remove dead {}",
getHostingNodeMinimalInfo(hostingNode));
registryPublisher.remove(hostingNode);
logger.debug("Request to remove {} successfully sent\n",
getHostingNodeMinimalInfo(hostingNode));
}catch(Exception e){
logger.error("Unable to remove {}",
getHostingNodeMinimalInfo(hostingNode), e);
}
}
}
public void sweepOrphanRI() throws Exception {
SimpleQuery hostingNodeQuery = ICFactory.queryFor(HostingNode.class)
.addCondition(String.format("$resource/Profile/GHNDescription/Status/text() ne '%s'", CERTIFIED))
.setResult("$resource");
DiscoveryClient<HostingNode> hostingNodeClient = ICFactory.clientFor(HostingNode.class);
List<HostingNode> hostingNodes = hostingNodeClient.submit(hostingNodeQuery);
for(HostingNode hostingNode : hostingNodes){
String condition = String.format("$resource/Profile/GHN/@UniqueID/string() eq '%s'", hostingNode.id());
SimpleQuery query = ICFactory.queryFor(GCoreEndpoint.class)
.addCondition(condition)
.setResult("$resource");
DiscoveryClient<GCoreEndpoint> client = ICFactory.clientFor(GCoreEndpoint.class);
List<GCoreEndpoint> gCoreEndpoints = client.submit(query);
for (GCoreEndpoint gCoreEndpoint : gCoreEndpoints) {
try {
logger.debug("Going to remove orphan {}",
getGCoreEndpointMinimalInfo(gCoreEndpoint, hostingNode));
registryPublisher.remove(hostingNode);
logger.debug("Request to remove {} successfully sent\n",
getGCoreEndpointMinimalInfo(gCoreEndpoint, hostingNode));
}catch(Exception e){
logger.error("Unable to remove {}",
getGCoreEndpointMinimalInfo(gCoreEndpoint, hostingNode), e);
}
}
}
}
}

4
src/test/java/org/gcube/informationsystem/sweeper/ISSweeperPluginTest.java
View File

@ -3,7 +3,7 @@ package org.gcube.informationsystem.sweeper;
import java.util.HashMap;
import java.util.Map;
import org.gcube.common.scope.api.ScopeProvider;
import org.gcube.common.authorization.library.provider.SecurityTokenProvider;
import org.junit.Before;
import org.junit.Test;
import org.slf4j.Logger;
@ -21,7 +21,7 @@ public class ISSweeperPluginTest {
@Before
public void beforeTest(){
ScopeProvider.instance.set("/gcube/devsec");
SecurityTokenProvider.instance.set("59074832-f986-46ce-9e46-5f2211300f9c");
}
@Test

3
src/test/resources/logback-test.xml
View File

@ -7,7 +7,8 @@
</appender>
<logger name="org.gcube" level="TRACE" />
<logger name="org.gcube" level="WARN" />
<logger name="org.gcube.informationsystem.sweeper" level="TRACE" />
<root level="WARN">
<appender-ref ref="STDOUT" />