git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/accounting/accounting-dashboard-harvester-se-plugin@167643 82a268e6-3cf1-43bd-a215-b396298e98cf
This commit is contained in:
parent
9aea29c6e1
commit
95284de142
|
@ -11,11 +11,19 @@ import org.gcube.common.resources.gcore.GenericResource;
|
||||||
import org.gcube.dataharvest.harvester.BasicHarvester;
|
import org.gcube.dataharvest.harvester.BasicHarvester;
|
||||||
import org.gcube.resources.discovery.client.queries.api.SimpleQuery;
|
import org.gcube.resources.discovery.client.queries.api.SimpleQuery;
|
||||||
import org.gcube.resources.discovery.icclient.ICFactory;
|
import org.gcube.resources.discovery.icclient.ICFactory;
|
||||||
|
import org.slf4j.Logger;
|
||||||
|
import org.slf4j.LoggerFactory;
|
||||||
|
import org.w3c.dom.Element;
|
||||||
|
import org.w3c.dom.Node;
|
||||||
|
import org.w3c.dom.NodeList;
|
||||||
|
|
||||||
public abstract class SoBigDataHarvester extends BasicHarvester {
|
public abstract class SoBigDataHarvester extends BasicHarvester {
|
||||||
|
|
||||||
|
private static Logger logger = LoggerFactory.getLogger(SoBigDataHarvester.class);
|
||||||
|
|
||||||
public static final String SO_BIG_DATA_CONTEXT = "/d4science.research-infrastructures.eu/SoBigData";
|
public static final String SO_BIG_DATA_CONTEXT = "/d4science.research-infrastructures.eu/SoBigData";
|
||||||
|
|
||||||
|
protected SortedSet<String> excludedContexts;
|
||||||
protected SortedSet<String> contexts;
|
protected SortedSet<String> contexts;
|
||||||
|
|
||||||
public SoBigDataHarvester(Date start, Date end) throws ParseException {
|
public SoBigDataHarvester(Date start, Date end) throws ParseException {
|
||||||
|
@ -25,10 +33,12 @@ public abstract class SoBigDataHarvester extends BasicHarvester {
|
||||||
public SortedSet<String> getContexts() {
|
public SortedSet<String> getContexts() {
|
||||||
return contexts;
|
return contexts;
|
||||||
}
|
}
|
||||||
|
|
||||||
public void setContexts(SortedSet<String> contexts) {
|
public void setContexts(SortedSet<String> contexts) {
|
||||||
|
this.excludedContexts = getExcludedContexts();
|
||||||
// Adding trailing slash to SO_BIG_DATA_CONTEXT to avoid to get VO
|
// Adding trailing slash to SO_BIG_DATA_CONTEXT to avoid to get VO
|
||||||
this.contexts = getSoBigDataContexts(contexts, SO_BIG_DATA_CONTEXT + "/");
|
this.contexts = getSoBigDataContexts(contexts, SO_BIG_DATA_CONTEXT + "/");
|
||||||
|
logger.trace("Valid contexts are {}", contexts);
|
||||||
}
|
}
|
||||||
|
|
||||||
public static String SECONDARY_TYPE_FORMAT = "$resource/Profile/SecondaryType/text() eq '%1s'";
|
public static String SECONDARY_TYPE_FORMAT = "$resource/Profile/SecondaryType/text() eq '%1s'";
|
||||||
|
@ -43,21 +53,69 @@ public abstract class SoBigDataHarvester extends BasicHarvester {
|
||||||
.addCondition(String.format(NAME_FORMAT, NAME));
|
.addCondition(String.format(NAME_FORMAT, NAME));
|
||||||
}
|
}
|
||||||
|
|
||||||
protected void getExcludedContexts(){
|
protected GenericResource getGenericResource() {
|
||||||
SimpleQuery simpleQuery = getFilteringGenericResource();
|
SimpleQuery simpleQuery = getFilteringGenericResource();
|
||||||
List<String> res = ICFactory.client().submit(simpleQuery);
|
List<GenericResource> res = ICFactory.clientFor(GenericResource.class).submit(simpleQuery);
|
||||||
|
if(res.size()==0) {
|
||||||
|
// At time of writing it should be an error but it can change in the future
|
||||||
|
logger.info("No {} for filtering contexts.", GenericResource.class.getSimpleName());
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
return res.get(0);
|
||||||
|
}
|
||||||
|
|
||||||
|
public SortedSet<String> getExcludedContexts() {
|
||||||
|
SortedSet<String> excludedContexts = new TreeSet<>();
|
||||||
|
|
||||||
|
GenericResource genericResource = getGenericResource();
|
||||||
|
if(genericResource==null) {
|
||||||
|
return excludedContexts;
|
||||||
|
}
|
||||||
|
|
||||||
|
Element body = genericResource.profile().body();
|
||||||
|
|
||||||
|
/*
|
||||||
|
* The following code parse an XML formatted as this
|
||||||
|
*
|
||||||
|
* <ResourceCatalogueHarvester>
|
||||||
|
* <vres>
|
||||||
|
* <vre>/d4science.research-infrastructures.eu/SoBigData/ResourceCatalogue</vre>
|
||||||
|
* <vre>/d4science.research-infrastructures.eu/SoBigData/TagMe</vre>
|
||||||
|
* </vres>
|
||||||
|
* </ResourceCatalogueHarvester>
|
||||||
|
*
|
||||||
|
*/
|
||||||
|
|
||||||
|
NodeList nodeList = body.getElementsByTagName(this.getClass().getSimpleName());
|
||||||
|
if(nodeList.getLength()==0) {
|
||||||
|
// At time of writing it should be an error but it can change in the future
|
||||||
|
logger.info("The body of the {} does not contains any information to filter contexts.", GenericResource.class.getSimpleName());
|
||||||
|
}
|
||||||
|
|
||||||
|
Node node = nodeList.item(0).getChildNodes().item(1);
|
||||||
|
NodeList contexts = node.getChildNodes();
|
||||||
|
|
||||||
|
for(int i=1; i<contexts.getLength()-1; i++){
|
||||||
|
Node context = contexts.item(i);
|
||||||
|
String contextToExclude = context.getChildNodes().item(0).getNodeValue();
|
||||||
|
excludedContexts.add(contextToExclude);
|
||||||
|
}
|
||||||
|
|
||||||
|
return excludedContexts;
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
protected boolean filterContext(String context) {
|
protected boolean filterContext(String context) {
|
||||||
|
if(excludedContexts.contains(context)) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
protected SortedSet<String> getSoBigDataContexts(Set<String> contexts, String base) {
|
||||||
protected SortedSet<String> getSoBigDataContexts(Set<String> contexts, String base){
|
|
||||||
SortedSet<String> filteredContext = new TreeSet<>();
|
SortedSet<String> filteredContext = new TreeSet<>();
|
||||||
for(String context : contexts) {
|
for(String context : contexts) {
|
||||||
if(context.startsWith(SO_BIG_DATA_CONTEXT)){
|
if(context.startsWith(SO_BIG_DATA_CONTEXT)) {
|
||||||
if(!filterContext(context)) {
|
if(!filterContext(context)) {
|
||||||
filteredContext.add(context);
|
filteredContext.add(context);
|
||||||
}
|
}
|
||||||
|
|
|
@ -4,11 +4,13 @@ import java.util.Date;
|
||||||
import java.util.HashMap;
|
import java.util.HashMap;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import java.util.Map;
|
import java.util.Map;
|
||||||
|
import java.util.SortedSet;
|
||||||
|
|
||||||
import org.gcube.dataharvest.datamodel.HarvestedData;
|
import org.gcube.dataharvest.datamodel.HarvestedData;
|
||||||
import org.gcube.dataharvest.harvester.MethodInvocationHarvester;
|
import org.gcube.dataharvest.harvester.MethodInvocationHarvester;
|
||||||
import org.gcube.dataharvest.harvester.sobigdata.ResourceCatalogueHarvester;
|
import org.gcube.dataharvest.harvester.sobigdata.ResourceCatalogueHarvester;
|
||||||
import org.gcube.dataharvest.harvester.sobigdata.TagMeMethodInvocationHarvester;
|
import org.gcube.dataharvest.harvester.sobigdata.TagMeMethodInvocationHarvester;
|
||||||
|
import org.gcube.dataharvest.utils.ContextAuthorization;
|
||||||
import org.gcube.dataharvest.utils.ContextTest;
|
import org.gcube.dataharvest.utils.ContextTest;
|
||||||
import org.gcube.dataharvest.utils.DateUtils;
|
import org.gcube.dataharvest.utils.DateUtils;
|
||||||
import org.gcube.dataharvest.utils.MeasureType;
|
import org.gcube.dataharvest.utils.MeasureType;
|
||||||
|
@ -27,11 +29,11 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
|
||||||
org.gcube.dataharvest.utils.Utils.setContext(ROOT);
|
org.gcube.dataharvest.utils.Utils.setContext(ROOT);
|
||||||
|
|
||||||
DataHarvestPluginDeclaration dataHarvestPluginDeclaration = new DataHarvestPluginDeclaration();
|
DataHarvestPluginDeclaration dataHarvestPluginDeclaration = new DataHarvestPluginDeclaration();
|
||||||
AccountingDataHarvesterPlugin accountingDataHarvesterPlugin = new AccountingDataHarvesterPlugin(dataHarvestPluginDeclaration);
|
AccountingDataHarvesterPlugin accountingDataHarvesterPlugin = new AccountingDataHarvesterPlugin(
|
||||||
|
dataHarvestPluginDeclaration);
|
||||||
|
|
||||||
Map<String,Object> inputs = new HashMap<>();
|
Map<String,Object> inputs = new HashMap<>();
|
||||||
|
|
||||||
|
|
||||||
MeasureType measureType = MeasureType.MONTHLY;
|
MeasureType measureType = MeasureType.MONTHLY;
|
||||||
|
|
||||||
inputs.put(AccountingDataHarvesterPlugin.MEASURE_TYPE_INPUT_PARAMETER, measureType.name());
|
inputs.put(AccountingDataHarvesterPlugin.MEASURE_TYPE_INPUT_PARAMETER, measureType.name());
|
||||||
|
@ -46,10 +48,10 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
|
||||||
*/
|
*/
|
||||||
|
|
||||||
accountingDataHarvesterPlugin.launch(inputs);
|
accountingDataHarvesterPlugin.launch(inputs);
|
||||||
|
|
||||||
logger.info("End.");
|
logger.info("End.");
|
||||||
|
|
||||||
} catch (Exception e) {
|
} catch(Exception e) {
|
||||||
logger.error("", e);
|
logger.error("", e);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -70,12 +72,11 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
|
||||||
|
|
||||||
logger.debug("{}", harvestedData);
|
logger.debug("{}", harvestedData);
|
||||||
|
|
||||||
|
} catch(Exception e) {
|
||||||
}catch (Exception e) {
|
|
||||||
logger.error("", e);
|
logger.error("", e);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void testTagMeMethodInvocation() {
|
public void testTagMeMethodInvocation() {
|
||||||
try {
|
try {
|
||||||
|
@ -92,13 +93,11 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
|
||||||
|
|
||||||
logger.debug("{}", harvestedData);
|
logger.debug("{}", harvestedData);
|
||||||
|
|
||||||
|
} catch(Exception e) {
|
||||||
}catch (Exception e) {
|
|
||||||
logger.error("", e);
|
logger.error("", e);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void testResourceCatalogueHarvester() {
|
public void testResourceCatalogueHarvester() {
|
||||||
try {
|
try {
|
||||||
|
@ -115,10 +114,34 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
|
||||||
|
|
||||||
logger.debug("{}", harvestedData);
|
logger.debug("{}", harvestedData);
|
||||||
|
|
||||||
|
} catch(Exception e) {
|
||||||
}catch (Exception e) {
|
|
||||||
logger.error("", e);
|
logger.error("", e);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testFilteringGenericResource() {
|
||||||
|
try {
|
||||||
|
org.gcube.dataharvest.utils.Utils.setContext(RESOURCE_CATALOGUE);
|
||||||
|
|
||||||
|
MeasureType measureType = MeasureType.MONTHLY;
|
||||||
|
|
||||||
|
Date start = DateUtils.getPreviousPeriod(measureType).getTime();
|
||||||
|
Date end = DateUtils.getEndDateFromStartDate(measureType, start, 1);
|
||||||
|
|
||||||
|
|
||||||
|
ContextAuthorization contextAuthorization = new ContextAuthorization();
|
||||||
|
SortedSet<String> contexts = contextAuthorization.getContexts();
|
||||||
|
|
||||||
|
ResourceCatalogueHarvester resourceCatalogueHarvester = new ResourceCatalogueHarvester(start, end);
|
||||||
|
resourceCatalogueHarvester.setContexts(contexts);
|
||||||
|
SortedSet<String> validContexts = resourceCatalogueHarvester.getContexts();
|
||||||
|
|
||||||
|
logger.info("{}", validContexts);
|
||||||
|
|
||||||
|
} catch(Exception e) {
|
||||||
|
logger.error("", e);
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in New Issue