Added logs and fixed tests
git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/accounting/accounting-dashboard-harvester-se-plugin@169689 82a268e6-3cf1-43bd-a215-b396298e98cf
This commit is contained in:
parent
c6e7ad3077
commit
456f8b31b7
|
|
@ -139,7 +139,8 @@ public class AccountingDataHarvesterPlugin extends Plugin<DataHarvestPluginDecla
|
|||
|
||||
end = DateUtils.getEndDateFromStartDate(aggregationType, start, 1);
|
||||
|
||||
logger.debug("Harvesting from {} to {}", DateUtils.format(start), DateUtils.format(end));
|
||||
logger.debug("Harvesting from {} to {} (ReRun:{} - GetVREUsers:{} - DryRun:{})",
|
||||
DateUtils.format(start), DateUtils.format(end), reRun, getVREUsers, dryRun);
|
||||
|
||||
Properties properties = getConfigParameters();
|
||||
getProperties().set(properties);
|
||||
|
|
@ -195,14 +196,16 @@ public class AccountingDataHarvesterPlugin extends Plugin<DataHarvestPluginDecla
|
|||
|
||||
try {
|
||||
// Collecting Google Analytics Data for VREs Accesses
|
||||
logger.info("Going to harvest VRE Accesses for {}", context);
|
||||
List<HarvestedData> harvested = vreAccessesHarvester.getData();
|
||||
data.addAll(harvested);
|
||||
} catch(Exception e) {
|
||||
logger.error("Error harvesting Social Interactions for {}", context, e);
|
||||
logger.error("Error harvesting VRE Accesses for {}", context, e);
|
||||
}
|
||||
|
||||
try {
|
||||
// Collecting info on social (posts, replies and likes)
|
||||
logger.info("Going to harvest Social Interactions for {}", context);
|
||||
SocialInteractionsHarvester socialHarvester = new SocialInteractionsHarvester(start, end);
|
||||
List<HarvestedData> harvested = socialHarvester.getData();
|
||||
data.addAll(harvested);
|
||||
|
|
@ -216,6 +219,7 @@ public class AccountingDataHarvesterPlugin extends Plugin<DataHarvestPluginDecla
|
|||
// Harvesting Users only for VREs (not for VO and ROOT which is the sum of the children contexts)
|
||||
// The VREUsers can be only Harvested for the last month
|
||||
if(scopeBean.is(Type.VRE) && start.equals(DateUtils.getPreviousPeriod(aggregationType).getTime())) {
|
||||
logger.info("Going to harvest Context Users for {}", context);
|
||||
VREUsersHarvester vreUsersHarvester = new VREUsersHarvester(start, end);
|
||||
List<HarvestedData> harvested = vreUsersHarvester.getData();
|
||||
data.addAll(harvested);
|
||||
|
|
@ -229,6 +233,7 @@ public class AccountingDataHarvesterPlugin extends Plugin<DataHarvestPluginDecla
|
|||
|
||||
try {
|
||||
// Collecting info on Resource Catalogue (Dataset, Application, Deliverables, Methods)
|
||||
logger.info("Going to harvest Resource Catalogue Information for {}", context);
|
||||
ResourceCatalogueHarvester resourceCatalogueHarvester = new ResourceCatalogueHarvester(start, end,
|
||||
contexts);
|
||||
List<HarvestedData> harvested = resourceCatalogueHarvester.getData();
|
||||
|
|
@ -239,6 +244,7 @@ public class AccountingDataHarvesterPlugin extends Plugin<DataHarvestPluginDecla
|
|||
|
||||
try {
|
||||
// Collecting info on Data/Method download
|
||||
logger.info("Going to harvest Data Method Download for {}", context);
|
||||
DataMethodDownloadHarvester dataMethodDownloadHarvester = new DataMethodDownloadHarvester(start,
|
||||
end, contexts);
|
||||
List<HarvestedData> harvested = dataMethodDownloadHarvester.getData();
|
||||
|
|
@ -252,6 +258,7 @@ public class AccountingDataHarvesterPlugin extends Plugin<DataHarvestPluginDecla
|
|||
if(context.startsWith(TAGME_CONTEXT)) {
|
||||
try {
|
||||
// Collecting info on method invocation
|
||||
logger.info("Going to harvest Method Invocations for {}", context);
|
||||
TagMeMethodInvocationHarvester tagMeMethodInvocationHarvester = new TagMeMethodInvocationHarvester(
|
||||
start, end);
|
||||
List<HarvestedData> harvested = tagMeMethodInvocationHarvester.getData();
|
||||
|
|
@ -262,6 +269,7 @@ public class AccountingDataHarvesterPlugin extends Plugin<DataHarvestPluginDecla
|
|||
} else {
|
||||
try {
|
||||
// Collecting info on method invocation
|
||||
logger.info("Going to harvest Method Invocations for {}", context);
|
||||
MethodInvocationHarvester methodInvocationHarvester = new MethodInvocationHarvester(start, end);
|
||||
List<HarvestedData> harvested = methodInvocationHarvester.getData();
|
||||
data.addAll(harvested);
|
||||
|
|
|
|||
|
|
@ -17,6 +17,7 @@ import org.gcube.common.scope.impl.ScopeBean.Type;
|
|||
import org.gcube.dataharvest.dao.DatabaseManager;
|
||||
import org.gcube.dataharvest.datamodel.HarvestedData;
|
||||
import org.gcube.dataharvest.harvester.MethodInvocationHarvester;
|
||||
import org.gcube.dataharvest.harvester.SocialInteractionsHarvester;
|
||||
import org.gcube.dataharvest.harvester.VREAccessesHarvester;
|
||||
import org.gcube.dataharvest.harvester.VREUsersHarvester;
|
||||
import org.gcube.dataharvest.harvester.sobigdata.DataMethodDownloadHarvester;
|
||||
|
|
@ -68,9 +69,9 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
|
|||
AggregationType aggregationType = AggregationType.MONTHLY;
|
||||
|
||||
inputs.put(AccountingDataHarvesterPlugin.MEASURE_TYPE_INPUT_PARAMETER, aggregationType.name());
|
||||
inputs.put(AccountingDataHarvesterPlugin.GET_VRE_USERS_INPUT_PARAMETER, false);
|
||||
inputs.put(AccountingDataHarvesterPlugin.GET_VRE_USERS_INPUT_PARAMETER, true);
|
||||
inputs.put(AccountingDataHarvesterPlugin.RERUN_INPUT_PARAMETER, true);
|
||||
inputs.put(AccountingDataHarvesterPlugin.DRY_RUN_INPUT_PARAMETER, true);
|
||||
inputs.put(AccountingDataHarvesterPlugin.DRY_RUN_INPUT_PARAMETER, false);
|
||||
|
||||
/*
|
||||
Calendar from = DateUtils.getStartCalendar(2016, Calendar.SEPTEMBER, 1);
|
||||
|
|
@ -88,7 +89,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
|
|||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
// @Test
|
||||
public void launchOldData() {
|
||||
try {
|
||||
|
||||
|
|
@ -272,7 +273,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
|
|||
|
||||
}
|
||||
|
||||
@Test
|
||||
// @Test
|
||||
public void testVREAccessesHarvester() {
|
||||
try {
|
||||
|
||||
|
|
@ -345,8 +346,72 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
|
|||
}
|
||||
}
|
||||
|
||||
// @Test
|
||||
public void testSocialInteraction() {
|
||||
try {
|
||||
|
||||
Utils.setContext(ROOT);
|
||||
DatabaseManager dbaseManager = new DatabaseManager();
|
||||
|
||||
DataHarvestPluginDeclaration dataHarvestPluginDeclaration = new DataHarvestPluginDeclaration();
|
||||
|
||||
AccountingDataHarvesterPlugin accountingDataHarvesterPlugin = new AccountingDataHarvesterPlugin(
|
||||
dataHarvestPluginDeclaration);
|
||||
Properties properties = accountingDataHarvesterPlugin.getConfigParameters();
|
||||
AccountingDataHarvesterPlugin.getProperties().set(properties);
|
||||
|
||||
ContextAuthorization contextAuthorization = new ContextAuthorization();
|
||||
|
||||
SortedSet<String> contexts = new TreeSet<>();
|
||||
contexts.add("/d4science.research-infrastructures.eu/D4Research");
|
||||
contexts.add("/d4science.research-infrastructures.eu/FARM/WECAFC-FIRMS");
|
||||
contexts.add("/d4science.research-infrastructures.eu/gCubeApps/BlueBridgeProject");
|
||||
contexts.add("/d4science.research-infrastructures.eu/gCubeApps/Parthenos");
|
||||
contexts.add("/d4science.research-infrastructures.eu/gCubeApps/ScalableDataMining");
|
||||
contexts.add("/d4science.research-infrastructures.eu/gCubeApps/gCube");
|
||||
|
||||
AggregationType aggregationType = AggregationType.MONTHLY;
|
||||
|
||||
Calendar from = DateUtils.getStartCalendar(2018, Calendar.JUNE, 1);
|
||||
Date start = from.getTime();
|
||||
Date end = DateUtils.getEndDateFromStartDate(aggregationType, start, 1);
|
||||
|
||||
logger.debug("Harvesting Social Interaction from {} to {}", DateUtils.format(start), DateUtils.format(end));
|
||||
|
||||
ArrayList<HarvestedData> data = new ArrayList<HarvestedData>();
|
||||
|
||||
|
||||
for(String context : contexts) {
|
||||
// Setting the token for the context
|
||||
Utils.setContext(contextAuthorization.getTokenForContext(context));
|
||||
try {
|
||||
// Collecting info on social (posts, replies and likes)
|
||||
logger.info("Going to harvest Social Interactions for {}", context);
|
||||
SocialInteractionsHarvester socialHarvester = new SocialInteractionsHarvester(start, end);
|
||||
List<HarvestedData> harvested = socialHarvester.getData();
|
||||
data.addAll(harvested);
|
||||
} catch(Exception e) {
|
||||
logger.error("Error harvesting Social Interactions for {}", context, e);
|
||||
}
|
||||
}
|
||||
|
||||
logger.debug("Harvest Measures from {} to {} are {}", DateUtils.format(start), DateUtils.format(end), data);
|
||||
Utils.setContext(ROOT);
|
||||
dbaseManager.insertMonthlyData(start, end, data, true);
|
||||
|
||||
|
||||
|
||||
} catch(Exception e) {
|
||||
logger.error("", e);
|
||||
}
|
||||
|
||||
logger.info("End.");
|
||||
|
||||
}
|
||||
|
||||
@Test
|
||||
|
||||
|
||||
// @Test
|
||||
public void testMethodInvocation() {
|
||||
try {
|
||||
|
||||
|
|
@ -367,7 +432,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
|
|||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
// @Test
|
||||
public void testTagMeMethodInvocation() {
|
||||
try {
|
||||
|
||||
|
|
@ -424,7 +489,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
|
|||
}
|
||||
|
||||
|
||||
@Test
|
||||
// @Test
|
||||
public void testFilteringGenericResource() {
|
||||
try {
|
||||
Utils.setContext(RESOURCE_CATALOGUE);
|
||||
|
|
@ -449,7 +514,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
|
|||
|
||||
}
|
||||
|
||||
@Test
|
||||
// @Test
|
||||
public void testResourceCatalogueHarvester() {
|
||||
try {
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue