Code redesign
Refs #11756: Refactor DataHArvesterPlugin to support scheduled execution from smart-executor Task-Url: https://support.d4science.org/issues/11756 git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/accounting/accounting-dashboard-harvester-se-plugin@167642 82a268e6-3cf1-43bd-a215-b396298e98cf
This commit is contained in:
parent
0de8fd20a6
commit
9aea29c6e1
|
@ -7,9 +7,7 @@ import java.util.Date;
|
|||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Properties;
|
||||
import java.util.Set;
|
||||
import java.util.SortedSet;
|
||||
import java.util.TreeSet;
|
||||
|
||||
import org.gcube.dataharvest.dao.DatabaseManager;
|
||||
import org.gcube.dataharvest.datamodel.HarvestedData;
|
||||
|
@ -17,8 +15,8 @@ import org.gcube.dataharvest.harvester.MethodInvocationHarvester;
|
|||
import org.gcube.dataharvest.harvester.SocialInteractionsHarvester;
|
||||
import org.gcube.dataharvest.harvester.VREUsersHarvester;
|
||||
import org.gcube.dataharvest.harvester.sobigdata.DataMethodDownloadHarvester;
|
||||
import org.gcube.dataharvest.harvester.sobigdata.TagMeMethodInvocationHarvester;
|
||||
import org.gcube.dataharvest.harvester.sobigdata.ResourceCatalogueHarvester;
|
||||
import org.gcube.dataharvest.harvester.sobigdata.TagMeMethodInvocationHarvester;
|
||||
import org.gcube.dataharvest.utils.ContextAuthorization;
|
||||
import org.gcube.dataharvest.utils.DateUtils;
|
||||
import org.gcube.dataharvest.utils.MeasureType;
|
||||
|
@ -42,7 +40,6 @@ public class AccountingDataHarvesterPlugin extends Plugin<DataHarvestPluginDecla
|
|||
public static final String RERUN_INPUT_PARAMETER = "reRun";
|
||||
public static final String DRY_RUN_INPUT_PARAMETER = "dryRun";
|
||||
|
||||
public static final String SO_BIG_DATA_CONTEXT = "/d4science.research-infrastructures.eu/SoBigData";
|
||||
public static final String RESOURCE_CATALOGUE_CONTEXT = "/d4science.research-infrastructures.eu/SoBigData/ResourceCatalogue";
|
||||
public static final String TAGME_CONTEXT = "/d4science.research-infrastructures.eu/SoBigData/TagMe";
|
||||
|
||||
|
@ -161,9 +158,6 @@ public class AccountingDataHarvesterPlugin extends Plugin<DataHarvestPluginDecla
|
|||
|
||||
if(context.startsWith(RESOURCE_CATALOGUE_CONTEXT)) {
|
||||
|
||||
// Adding trailing slash to SO_BIG_DATA_CONTEXT to avoid to get VO
|
||||
SortedSet<String> soBigDataContexts = getSoBigDataContexts(contexts, SO_BIG_DATA_CONTEXT + "/");
|
||||
|
||||
try {
|
||||
// Collecting info on Resource Catalogue (Dataset, Application, Deliverables, Methods)
|
||||
ResourceCatalogueHarvester resourceCatalogueHarvester = new ResourceCatalogueHarvester(start, end);
|
||||
|
@ -214,17 +208,6 @@ public class AccountingDataHarvesterPlugin extends Plugin<DataHarvestPluginDecla
|
|||
|
||||
}
|
||||
|
||||
|
||||
protected SortedSet<String> getSoBigDataContexts(Set<String> contexts, String base){
|
||||
SortedSet<String> filteredContext = new TreeSet<>();
|
||||
for(String context : contexts) {
|
||||
if(context.startsWith(SO_BIG_DATA_CONTEXT)){
|
||||
filteredContext.add(context);
|
||||
}
|
||||
}
|
||||
return filteredContext;
|
||||
}
|
||||
|
||||
/** {@inheritDoc} */
|
||||
@Override
|
||||
protected void onStop() throws Exception {
|
||||
|
|
|
@ -2,12 +2,20 @@ package org.gcube.dataharvest.harvester.sobigdata;
|
|||
|
||||
import java.text.ParseException;
|
||||
import java.util.Date;
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
import java.util.SortedSet;
|
||||
import java.util.TreeSet;
|
||||
|
||||
import org.gcube.common.resources.gcore.GenericResource;
|
||||
import org.gcube.dataharvest.harvester.BasicHarvester;
|
||||
import org.gcube.resources.discovery.client.queries.api.SimpleQuery;
|
||||
import org.gcube.resources.discovery.icclient.ICFactory;
|
||||
|
||||
public abstract class SoBigDataHarvester extends BasicHarvester {
|
||||
|
||||
public static final String SO_BIG_DATA_CONTEXT = "/d4science.research-infrastructures.eu/SoBigData";
|
||||
|
||||
protected SortedSet<String> contexts;
|
||||
|
||||
public SoBigDataHarvester(Date start, Date end) throws ParseException {
|
||||
|
@ -19,7 +27,43 @@ public abstract class SoBigDataHarvester extends BasicHarvester {
|
|||
}
|
||||
|
||||
public void setContexts(SortedSet<String> contexts) {
|
||||
this.contexts = contexts;
|
||||
// Adding trailing slash to SO_BIG_DATA_CONTEXT to avoid to get VO
|
||||
this.contexts = getSoBigDataContexts(contexts, SO_BIG_DATA_CONTEXT + "/");
|
||||
}
|
||||
|
||||
public static String SECONDARY_TYPE_FORMAT = "$resource/Profile/SecondaryType/text() eq '%1s'";
|
||||
public static String NAME_FORMAT = "$resource/Profile/Name/text() eq '%1s'";
|
||||
|
||||
public static String SECONDARY_TYPE = "ExcludingVREs";
|
||||
public static String NAME = "AccountingHarvesters";
|
||||
|
||||
protected SimpleQuery getFilteringGenericResource() {
|
||||
return ICFactory.queryFor(GenericResource.class)
|
||||
.addCondition(String.format(SECONDARY_TYPE_FORMAT, SECONDARY_TYPE))
|
||||
.addCondition(String.format(NAME_FORMAT, NAME));
|
||||
}
|
||||
|
||||
protected void getExcludedContexts(){
|
||||
SimpleQuery simpleQuery = getFilteringGenericResource();
|
||||
List<String> res = ICFactory.client().submit(simpleQuery);
|
||||
|
||||
}
|
||||
|
||||
protected boolean filterContext(String context) {
|
||||
return false;
|
||||
}
|
||||
|
||||
|
||||
protected SortedSet<String> getSoBigDataContexts(Set<String> contexts, String base){
|
||||
SortedSet<String> filteredContext = new TreeSet<>();
|
||||
for(String context : contexts) {
|
||||
if(context.startsWith(SO_BIG_DATA_CONTEXT)){
|
||||
if(!filterContext(context)) {
|
||||
filteredContext.add(context);
|
||||
}
|
||||
}
|
||||
}
|
||||
return filteredContext;
|
||||
}
|
||||
|
||||
}
|
||||
|
|
Loading…
Reference in New Issue