gCubeSystem
/
accounting-dashboard-harvester-se-plugin
16
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/accounting/accounting-dashboard-harvester-se-plugin@167643 82a268e6-3cf1-43bd-a215-b396298e98cf

This commit is contained in:
Luca Frosini 2018-05-21 13:37:57 +00:00
parent 9aea29c6e1
commit 95284de142
2 changed files with 101 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

72
src/main/java/org/gcube/dataharvest/harvester/sobigdata/SoBigDataHarvester.java
View File

@ -11,11 +11,19 @@ import org.gcube.common.resources.gcore.GenericResource;
import org.gcube.dataharvest.harvester.BasicHarvester;
import org.gcube.resources.discovery.client.queries.api.SimpleQuery;
import org.gcube.resources.discovery.icclient.ICFactory;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.w3c.dom.Element;
import org.w3c.dom.Node;
import org.w3c.dom.NodeList;
public abstract class SoBigDataHarvester extends BasicHarvester {
private static Logger logger = LoggerFactory.getLogger(SoBigDataHarvester.class);
public static final String SO_BIG_DATA_CONTEXT = "/d4science.research-infrastructures.eu/SoBigData";
protected SortedSet<String> excludedContexts;
protected SortedSet<String> contexts;
public SoBigDataHarvester(Date start, Date end) throws ParseException {
@ -25,10 +33,12 @@ public abstract class SoBigDataHarvester extends BasicHarvester {
public SortedSet<String> getContexts() {
return contexts;
}
public void setContexts(SortedSet<String> contexts) {
this.excludedContexts = getExcludedContexts();
// Adding trailing slash to SO_BIG_DATA_CONTEXT to avoid to get VO
this.contexts = getSoBigDataContexts(contexts, SO_BIG_DATA_CONTEXT + "/");
logger.trace("Valid contexts are {}", contexts);
}
public static String SECONDARY_TYPE_FORMAT = "$resource/Profile/SecondaryType/text() eq '%1s'";
@ -43,21 +53,69 @@ public abstract class SoBigDataHarvester extends BasicHarvester {
.addCondition(String.format(NAME_FORMAT, NAME));
}
protected void getExcludedContexts(){
protected GenericResource getGenericResource() {
SimpleQuery simpleQuery = getFilteringGenericResource();
List<String> res = ICFactory.client().submit(simpleQuery);
List<GenericResource> res = ICFactory.clientFor(GenericResource.class).submit(simpleQuery);
if(res.size()==0) {
// At time of writing it should be an error but it can change in the future
logger.info("No {} for filtering contexts.", GenericResource.class.getSimpleName());
return null;
}
return res.get(0);
}
public SortedSet<String> getExcludedContexts() {
SortedSet<String> excludedContexts = new TreeSet<>();
GenericResource genericResource = getGenericResource();
if(genericResource==null) {
return excludedContexts;
}
Element body = genericResource.profile().body();
/*
* The following code parse an XML formatted as this
*
* <ResourceCatalogueHarvester>
* <vres>
* <vre>/d4science.research-infrastructures.eu/SoBigData/ResourceCatalogue</vre>
* <vre>/d4science.research-infrastructures.eu/SoBigData/TagMe</vre>
* </vres>
* </ResourceCatalogueHarvester>
*
*/
NodeList nodeList = body.getElementsByTagName(this.getClass().getSimpleName());
if(nodeList.getLength()==0) {
// At time of writing it should be an error but it can change in the future
logger.info("The body of the {} does not contains any information to filter contexts.", GenericResource.class.getSimpleName());
}
Node node = nodeList.item(0).getChildNodes().item(1);
NodeList contexts = node.getChildNodes();
for(int i=1; i<contexts.getLength()-1; i++){
Node context = contexts.item(i);
String contextToExclude = context.getChildNodes().item(0).getNodeValue();
excludedContexts.add(contextToExclude);
}
return excludedContexts;
}
protected boolean filterContext(String context) {
if(excludedContexts.contains(context)) {
return true;
}
return false;
}
protected SortedSet<String> getSoBigDataContexts(Set<String> contexts, String base){
protected SortedSet<String> getSoBigDataContexts(Set<String> contexts, String base) {
SortedSet<String> filteredContext = new TreeSet<>();
for(String context : contexts) {
if(context.startsWith(SO_BIG_DATA_CONTEXT)){
if(context.startsWith(SO_BIG_DATA_CONTEXT)) {
if(!filterContext(context)) {
filteredContext.add(context);
}

49
src/test/java/org/gcube/dataharvest/AccountingDataHarvesterPluginTest.java
View File

@ -4,11 +4,13 @@ import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.SortedSet;
import org.gcube.dataharvest.datamodel.HarvestedData;
import org.gcube.dataharvest.harvester.MethodInvocationHarvester;
import org.gcube.dataharvest.harvester.sobigdata.ResourceCatalogueHarvester;
import org.gcube.dataharvest.harvester.sobigdata.TagMeMethodInvocationHarvester;
import org.gcube.dataharvest.utils.ContextAuthorization;
import org.gcube.dataharvest.utils.ContextTest;
import org.gcube.dataharvest.utils.DateUtils;
import org.gcube.dataharvest.utils.MeasureType;
@ -27,11 +29,11 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
org.gcube.dataharvest.utils.Utils.setContext(ROOT);
DataHarvestPluginDeclaration dataHarvestPluginDeclaration = new DataHarvestPluginDeclaration();
AccountingDataHarvesterPlugin accountingDataHarvesterPlugin = new AccountingDataHarvesterPlugin(dataHarvestPluginDeclaration);
AccountingDataHarvesterPlugin accountingDataHarvesterPlugin = new AccountingDataHarvesterPlugin(
dataHarvestPluginDeclaration);
Map<String,Object> inputs = new HashMap<>();
MeasureType measureType = MeasureType.MONTHLY;
inputs.put(AccountingDataHarvesterPlugin.MEASURE_TYPE_INPUT_PARAMETER, measureType.name());
@ -46,10 +48,10 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
*/
accountingDataHarvesterPlugin.launch(inputs);
logger.info("End.");
} catch (Exception e) {
} catch(Exception e) {
logger.error("", e);
}
}
@ -70,12 +72,11 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
logger.debug("{}", harvestedData);
}catch (Exception e) {
} catch(Exception e) {
logger.error("", e);
}
}
@Test
public void testTagMeMethodInvocation() {
try {
@ -92,13 +93,11 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
logger.debug("{}", harvestedData);
}catch (Exception e) {
} catch(Exception e) {
logger.error("", e);
}
}
@Test
public void testResourceCatalogueHarvester() {
try {
@ -115,10 +114,34 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
logger.debug("{}", harvestedData);
}catch (Exception e) {
} catch(Exception e) {
logger.error("", e);
}
}
@Test
public void testFilteringGenericResource() {
try {
org.gcube.dataharvest.utils.Utils.setContext(RESOURCE_CATALOGUE);
MeasureType measureType = MeasureType.MONTHLY;
Date start = DateUtils.getPreviousPeriod(measureType).getTime();
Date end = DateUtils.getEndDateFromStartDate(measureType, start, 1);
ContextAuthorization contextAuthorization = new ContextAuthorization();
SortedSet<String> contexts = contextAuthorization.getContexts();
ResourceCatalogueHarvester resourceCatalogueHarvester = new ResourceCatalogueHarvester(start, end);
resourceCatalogueHarvester.setContexts(contexts);
SortedSet<String> validContexts = resourceCatalogueHarvester.getContexts();
logger.info("{}", validContexts);
} catch(Exception e) {
logger.error("", e);
}
}
}