gCubeSystem
/
accounting-dashboard-harvester-se-plugin
16
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Code redesign 

Refs #11756: Refactor DataHArvesterPlugin to support scheduled execution from smart-executor 

Task-Url: https://support.d4science.org/issues/11756

git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/accounting/accounting-dashboard-harvester-se-plugin@167642 82a268e6-3cf1-43bd-a215-b396298e98cf
This commit is contained in:
Luca Frosini 2018-05-21 12:49:01 +00:00
parent 0de8fd20a6
commit 9aea29c6e1
2 changed files with 46 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
src/main/java/org/gcube/dataharvest/AccountingDataHarvesterPlugin.java
View File

@ -7,9 +7,7 @@ import java.util.Date;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.Set;
import java.util.SortedSet;
import java.util.TreeSet;
import org.gcube.dataharvest.dao.DatabaseManager;
import org.gcube.dataharvest.datamodel.HarvestedData;
@ -17,8 +15,8 @@ import org.gcube.dataharvest.harvester.MethodInvocationHarvester;
import org.gcube.dataharvest.harvester.SocialInteractionsHarvester;
import org.gcube.dataharvest.harvester.VREUsersHarvester;
import org.gcube.dataharvest.harvester.sobigdata.DataMethodDownloadHarvester;
import org.gcube.dataharvest.harvester.sobigdata.TagMeMethodInvocationHarvester;
import org.gcube.dataharvest.harvester.sobigdata.ResourceCatalogueHarvester;
import org.gcube.dataharvest.harvester.sobigdata.TagMeMethodInvocationHarvester;
import org.gcube.dataharvest.utils.ContextAuthorization;
import org.gcube.dataharvest.utils.DateUtils;
import org.gcube.dataharvest.utils.MeasureType;
@ -42,7 +40,6 @@ public class AccountingDataHarvesterPlugin extends Plugin<DataHarvestPluginDecla
public static final String RERUN_INPUT_PARAMETER = "reRun";
public static final String DRY_RUN_INPUT_PARAMETER = "dryRun";
public static final String SO_BIG_DATA_CONTEXT = "/d4science.research-infrastructures.eu/SoBigData";
public static final String RESOURCE_CATALOGUE_CONTEXT = "/d4science.research-infrastructures.eu/SoBigData/ResourceCatalogue";
public static final String TAGME_CONTEXT = "/d4science.research-infrastructures.eu/SoBigData/TagMe";
@ -161,9 +158,6 @@ public class AccountingDataHarvesterPlugin extends Plugin<DataHarvestPluginDecla
if(context.startsWith(RESOURCE_CATALOGUE_CONTEXT)) {
// Adding trailing slash to SO_BIG_DATA_CONTEXT to avoid to get VO
SortedSet<String> soBigDataContexts = getSoBigDataContexts(contexts, SO_BIG_DATA_CONTEXT + "/");
try {
// Collecting info on Resource Catalogue (Dataset, Application, Deliverables, Methods)
ResourceCatalogueHarvester resourceCatalogueHarvester = new ResourceCatalogueHarvester(start, end);
@ -214,17 +208,6 @@ public class AccountingDataHarvesterPlugin extends Plugin<DataHarvestPluginDecla
}
protected SortedSet<String> getSoBigDataContexts(Set<String> contexts, String base){
SortedSet<String> filteredContext = new TreeSet<>();
for(String context : contexts) {
if(context.startsWith(SO_BIG_DATA_CONTEXT)){
filteredContext.add(context);
}
}
return filteredContext;
}
/** {@inheritDoc} */
@Override
protected void onStop() throws Exception {

46
src/main/java/org/gcube/dataharvest/harvester/sobigdata/SoBigDataHarvester.java
View File

@ -2,12 +2,20 @@ package org.gcube.dataharvest.harvester.sobigdata;
import java.text.ParseException;
import java.util.Date;
import java.util.List;
import java.util.Set;
import java.util.SortedSet;
import java.util.TreeSet;
import org.gcube.common.resources.gcore.GenericResource;
import org.gcube.dataharvest.harvester.BasicHarvester;
import org.gcube.resources.discovery.client.queries.api.SimpleQuery;
import org.gcube.resources.discovery.icclient.ICFactory;
public abstract class SoBigDataHarvester extends BasicHarvester {
public static final String SO_BIG_DATA_CONTEXT = "/d4science.research-infrastructures.eu/SoBigData";
protected SortedSet<String> contexts;
public SoBigDataHarvester(Date start, Date end) throws ParseException {
@ -19,7 +27,43 @@ public abstract class SoBigDataHarvester extends BasicHarvester {
}
public void setContexts(SortedSet<String> contexts) {
this.contexts = contexts;
// Adding trailing slash to SO_BIG_DATA_CONTEXT to avoid to get VO
this.contexts = getSoBigDataContexts(contexts, SO_BIG_DATA_CONTEXT + "/");
}
public static String SECONDARY_TYPE_FORMAT = "$resource/Profile/SecondaryType/text() eq '%1s'";
public static String NAME_FORMAT = "$resource/Profile/Name/text() eq '%1s'";
public static String SECONDARY_TYPE = "ExcludingVREs";
public static String NAME = "AccountingHarvesters";
protected SimpleQuery getFilteringGenericResource() {
return ICFactory.queryFor(GenericResource.class)
.addCondition(String.format(SECONDARY_TYPE_FORMAT, SECONDARY_TYPE))
.addCondition(String.format(NAME_FORMAT, NAME));
}
protected void getExcludedContexts(){
SimpleQuery simpleQuery = getFilteringGenericResource();
List<String> res = ICFactory.client().submit(simpleQuery);
}
protected boolean filterContext(String context) {
return false;
}
protected SortedSet<String> getSoBigDataContexts(Set<String> contexts, String base){
SortedSet<String> filteredContext = new TreeSet<>();
for(String context : contexts) {
if(context.startsWith(SO_BIG_DATA_CONTEXT)){
if(!filterContext(context)) {
filteredContext.add(context);
}
}
}
return filteredContext;
}
}