gCubeSystem
/
accounting-dashboard-harvester-se-plugin
16
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Code redesign 

Refs #11756: Refactor DataHArvesterPlugin to support scheduled execution from smart-executor 

Task-Url: https://support.d4science.org/issues/11756

git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/accounting/accounting-dashboard-harvester-se-plugin@167636 82a268e6-3cf1-43bd-a215-b396298e98cf
This commit is contained in:
Luca Frosini 2018-05-21 10:51:17 +00:00
parent 8edab20ef0
commit 1ef7cee987
3 changed files with 40 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
src/main/java/org/gcube/dataharvest/harvester/sobigdata/ResourceCatalogueHarvester.java
View File

@ -30,10 +30,17 @@ public class ResourceCatalogueHarvester extends BasicHarvester {
private int societalDebatesCounter = 0;
private int wellBeingAndEconomyCounter = 0;
protected String solrBaseUrl;
public ResourceCatalogueHarvester(Date start, Date end) throws ParseException {
super(start, end);
}
protected String getSolrBaseUrl() {
return null;
}
@Override
public List<HarvestedData> getData() throws Exception {
@ -117,6 +124,7 @@ public class ResourceCatalogueHarvester extends BasicHarvester {
}
}
h = new HarvestedData(dataType, "/d4science.research-infrastructures.eu/SoBigData/CityOfCitizens",
cityOfCitizensCounter);
logger.debug(h.toString());
@ -140,7 +148,9 @@ public class ResourceCatalogueHarvester extends BasicHarvester {
}
private String executeQuery(String fqSubString) throws Exception {
String query = "https://ckan-solr-d4s.d4science.org/solr/sobigdata/select?";
String q = UrlEncoderUtil.encodeQuery("metadata_created:[" + DateUtils.dateToStringWithTZ(startDate) + " TO "
+ DateUtils.dateToStringWithTZ(endDate) + "]");
query += "q=" + q;

24
src/test/java/org/gcube/dataharvest/AccountingDataHarvesterPluginTest.java
View File

@ -7,6 +7,7 @@ import java.util.Map;
import org.gcube.dataharvest.datamodel.HarvestedData;
import org.gcube.dataharvest.harvester.MethodInvocationHarvester;
import org.gcube.dataharvest.harvester.sobigdata.ResourceCatalogueHarvester;
import org.gcube.dataharvest.harvester.sobigdata.TagMeMethodInvocationHarvester;
import org.gcube.dataharvest.utils.ContextTest;
import org.gcube.dataharvest.utils.DateUtils;
@ -97,4 +98,27 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
}
}
@Test
public void testResourceCatalogueHarvester() {
try {
org.gcube.dataharvest.utils.Utils.setContext(RESOURCE_CATALOGUE);
MeasureType measureType = MeasureType.MONTHLY;
Date start = DateUtils.getPreviousPeriod(measureType).getTime();
Date end = DateUtils.getEndDateFromStartDate(measureType, start, 1);
ResourceCatalogueHarvester resourceCatalogueHarvester = new ResourceCatalogueHarvester(start, end);
List<HarvestedData> harvestedData = resourceCatalogueHarvester.getData();
logger.debug("{}", harvestedData);
}catch (Exception e) {
logger.error("", e);
}
}
}

15
src/test/java/org/gcube/dataharvest/utils/ContextTest.java
View File

@ -29,12 +29,6 @@ public class ContextTest {
private static final String GCUBE_DEVNEXT_NEXTNEXT_VARNAME = "GCUBE_DEVNEXT_NEXTNEXT";
public static final String GCUBE_DEVNEXT_NEXTNEXT;
public static final String GCUBE_DEVSEC_VARNAME = "GCUBE_DEVSEC";
public static final String GCUBE_DEVSEC;
public static final String GCUBE_DEVSEC_DEVVRE_VARNAME = "GCUBE_DEVSEC_DEVVRE";
public static final String GCUBE_DEVSEC_DEVVRE;
public static final String GCUBE_VARNAME = "GCUBE";
public static final String GCUBE;
@ -49,6 +43,10 @@ public class ContextTest {
public static final String StockAssessment_VARNAME = "StockAssessment";
public static final String StockAssessment;
public static final String RESOURCE_CATALOGUE_VARNAME = "RESOURCE_CATALOGUE";
public static final String RESOURCE_CATALOGUE;
static {
@ -69,13 +67,12 @@ public class ContextTest {
GCUBE_DEVNEXT = properties.getProperty(GCUBE_DEVNEXT_VARNAME);
GCUBE_DEVNEXT_NEXTNEXT = properties.getProperty(GCUBE_DEVNEXT_NEXTNEXT_VARNAME);
GCUBE_DEVSEC = properties.getProperty(GCUBE_DEVSEC_VARNAME);
GCUBE_DEVSEC_DEVVRE = properties.getProperty(GCUBE_DEVSEC_DEVVRE_VARNAME);
ROOT = properties.getProperty(ROOT_VARNAME);
TAGME = properties.getProperty(TAGME_VARNAME);
RESOURCE_CATALOGUE = properties.getProperty(RESOURCE_CATALOGUE_VARNAME);
StockAssessment = properties.getProperty(StockAssessment_VARNAME);
DEFAULT_TEST_SCOPE = GCUBE;