Added logs and fixed tests
git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/accounting/accounting-dashboard-harvester-se-plugin@169689 82a268e6-3cf1-43bd-a215-b396298e98cf
This commit is contained in:
parent
c6e7ad3077
commit
456f8b31b7
|
@ -139,7 +139,8 @@ public class AccountingDataHarvesterPlugin extends Plugin<DataHarvestPluginDecla
|
||||||
|
|
||||||
end = DateUtils.getEndDateFromStartDate(aggregationType, start, 1);
|
end = DateUtils.getEndDateFromStartDate(aggregationType, start, 1);
|
||||||
|
|
||||||
logger.debug("Harvesting from {} to {}", DateUtils.format(start), DateUtils.format(end));
|
logger.debug("Harvesting from {} to {} (ReRun:{} - GetVREUsers:{} - DryRun:{})",
|
||||||
|
DateUtils.format(start), DateUtils.format(end), reRun, getVREUsers, dryRun);
|
||||||
|
|
||||||
Properties properties = getConfigParameters();
|
Properties properties = getConfigParameters();
|
||||||
getProperties().set(properties);
|
getProperties().set(properties);
|
||||||
|
@ -195,14 +196,16 @@ public class AccountingDataHarvesterPlugin extends Plugin<DataHarvestPluginDecla
|
||||||
|
|
||||||
try {
|
try {
|
||||||
// Collecting Google Analytics Data for VREs Accesses
|
// Collecting Google Analytics Data for VREs Accesses
|
||||||
|
logger.info("Going to harvest VRE Accesses for {}", context);
|
||||||
List<HarvestedData> harvested = vreAccessesHarvester.getData();
|
List<HarvestedData> harvested = vreAccessesHarvester.getData();
|
||||||
data.addAll(harvested);
|
data.addAll(harvested);
|
||||||
} catch(Exception e) {
|
} catch(Exception e) {
|
||||||
logger.error("Error harvesting Social Interactions for {}", context, e);
|
logger.error("Error harvesting VRE Accesses for {}", context, e);
|
||||||
}
|
}
|
||||||
|
|
||||||
try {
|
try {
|
||||||
// Collecting info on social (posts, replies and likes)
|
// Collecting info on social (posts, replies and likes)
|
||||||
|
logger.info("Going to harvest Social Interactions for {}", context);
|
||||||
SocialInteractionsHarvester socialHarvester = new SocialInteractionsHarvester(start, end);
|
SocialInteractionsHarvester socialHarvester = new SocialInteractionsHarvester(start, end);
|
||||||
List<HarvestedData> harvested = socialHarvester.getData();
|
List<HarvestedData> harvested = socialHarvester.getData();
|
||||||
data.addAll(harvested);
|
data.addAll(harvested);
|
||||||
|
@ -216,6 +219,7 @@ public class AccountingDataHarvesterPlugin extends Plugin<DataHarvestPluginDecla
|
||||||
// Harvesting Users only for VREs (not for VO and ROOT which is the sum of the children contexts)
|
// Harvesting Users only for VREs (not for VO and ROOT which is the sum of the children contexts)
|
||||||
// The VREUsers can be only Harvested for the last month
|
// The VREUsers can be only Harvested for the last month
|
||||||
if(scopeBean.is(Type.VRE) && start.equals(DateUtils.getPreviousPeriod(aggregationType).getTime())) {
|
if(scopeBean.is(Type.VRE) && start.equals(DateUtils.getPreviousPeriod(aggregationType).getTime())) {
|
||||||
|
logger.info("Going to harvest Context Users for {}", context);
|
||||||
VREUsersHarvester vreUsersHarvester = new VREUsersHarvester(start, end);
|
VREUsersHarvester vreUsersHarvester = new VREUsersHarvester(start, end);
|
||||||
List<HarvestedData> harvested = vreUsersHarvester.getData();
|
List<HarvestedData> harvested = vreUsersHarvester.getData();
|
||||||
data.addAll(harvested);
|
data.addAll(harvested);
|
||||||
|
@ -229,6 +233,7 @@ public class AccountingDataHarvesterPlugin extends Plugin<DataHarvestPluginDecla
|
||||||
|
|
||||||
try {
|
try {
|
||||||
// Collecting info on Resource Catalogue (Dataset, Application, Deliverables, Methods)
|
// Collecting info on Resource Catalogue (Dataset, Application, Deliverables, Methods)
|
||||||
|
logger.info("Going to harvest Resource Catalogue Information for {}", context);
|
||||||
ResourceCatalogueHarvester resourceCatalogueHarvester = new ResourceCatalogueHarvester(start, end,
|
ResourceCatalogueHarvester resourceCatalogueHarvester = new ResourceCatalogueHarvester(start, end,
|
||||||
contexts);
|
contexts);
|
||||||
List<HarvestedData> harvested = resourceCatalogueHarvester.getData();
|
List<HarvestedData> harvested = resourceCatalogueHarvester.getData();
|
||||||
|
@ -239,6 +244,7 @@ public class AccountingDataHarvesterPlugin extends Plugin<DataHarvestPluginDecla
|
||||||
|
|
||||||
try {
|
try {
|
||||||
// Collecting info on Data/Method download
|
// Collecting info on Data/Method download
|
||||||
|
logger.info("Going to harvest Data Method Download for {}", context);
|
||||||
DataMethodDownloadHarvester dataMethodDownloadHarvester = new DataMethodDownloadHarvester(start,
|
DataMethodDownloadHarvester dataMethodDownloadHarvester = new DataMethodDownloadHarvester(start,
|
||||||
end, contexts);
|
end, contexts);
|
||||||
List<HarvestedData> harvested = dataMethodDownloadHarvester.getData();
|
List<HarvestedData> harvested = dataMethodDownloadHarvester.getData();
|
||||||
|
@ -252,6 +258,7 @@ public class AccountingDataHarvesterPlugin extends Plugin<DataHarvestPluginDecla
|
||||||
if(context.startsWith(TAGME_CONTEXT)) {
|
if(context.startsWith(TAGME_CONTEXT)) {
|
||||||
try {
|
try {
|
||||||
// Collecting info on method invocation
|
// Collecting info on method invocation
|
||||||
|
logger.info("Going to harvest Method Invocations for {}", context);
|
||||||
TagMeMethodInvocationHarvester tagMeMethodInvocationHarvester = new TagMeMethodInvocationHarvester(
|
TagMeMethodInvocationHarvester tagMeMethodInvocationHarvester = new TagMeMethodInvocationHarvester(
|
||||||
start, end);
|
start, end);
|
||||||
List<HarvestedData> harvested = tagMeMethodInvocationHarvester.getData();
|
List<HarvestedData> harvested = tagMeMethodInvocationHarvester.getData();
|
||||||
|
@ -262,6 +269,7 @@ public class AccountingDataHarvesterPlugin extends Plugin<DataHarvestPluginDecla
|
||||||
} else {
|
} else {
|
||||||
try {
|
try {
|
||||||
// Collecting info on method invocation
|
// Collecting info on method invocation
|
||||||
|
logger.info("Going to harvest Method Invocations for {}", context);
|
||||||
MethodInvocationHarvester methodInvocationHarvester = new MethodInvocationHarvester(start, end);
|
MethodInvocationHarvester methodInvocationHarvester = new MethodInvocationHarvester(start, end);
|
||||||
List<HarvestedData> harvested = methodInvocationHarvester.getData();
|
List<HarvestedData> harvested = methodInvocationHarvester.getData();
|
||||||
data.addAll(harvested);
|
data.addAll(harvested);
|
||||||
|
|
|
@ -17,6 +17,7 @@ import org.gcube.common.scope.impl.ScopeBean.Type;
|
||||||
import org.gcube.dataharvest.dao.DatabaseManager;
|
import org.gcube.dataharvest.dao.DatabaseManager;
|
||||||
import org.gcube.dataharvest.datamodel.HarvestedData;
|
import org.gcube.dataharvest.datamodel.HarvestedData;
|
||||||
import org.gcube.dataharvest.harvester.MethodInvocationHarvester;
|
import org.gcube.dataharvest.harvester.MethodInvocationHarvester;
|
||||||
|
import org.gcube.dataharvest.harvester.SocialInteractionsHarvester;
|
||||||
import org.gcube.dataharvest.harvester.VREAccessesHarvester;
|
import org.gcube.dataharvest.harvester.VREAccessesHarvester;
|
||||||
import org.gcube.dataharvest.harvester.VREUsersHarvester;
|
import org.gcube.dataharvest.harvester.VREUsersHarvester;
|
||||||
import org.gcube.dataharvest.harvester.sobigdata.DataMethodDownloadHarvester;
|
import org.gcube.dataharvest.harvester.sobigdata.DataMethodDownloadHarvester;
|
||||||
|
@ -68,9 +69,9 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
|
||||||
AggregationType aggregationType = AggregationType.MONTHLY;
|
AggregationType aggregationType = AggregationType.MONTHLY;
|
||||||
|
|
||||||
inputs.put(AccountingDataHarvesterPlugin.MEASURE_TYPE_INPUT_PARAMETER, aggregationType.name());
|
inputs.put(AccountingDataHarvesterPlugin.MEASURE_TYPE_INPUT_PARAMETER, aggregationType.name());
|
||||||
inputs.put(AccountingDataHarvesterPlugin.GET_VRE_USERS_INPUT_PARAMETER, false);
|
inputs.put(AccountingDataHarvesterPlugin.GET_VRE_USERS_INPUT_PARAMETER, true);
|
||||||
inputs.put(AccountingDataHarvesterPlugin.RERUN_INPUT_PARAMETER, true);
|
inputs.put(AccountingDataHarvesterPlugin.RERUN_INPUT_PARAMETER, true);
|
||||||
inputs.put(AccountingDataHarvesterPlugin.DRY_RUN_INPUT_PARAMETER, true);
|
inputs.put(AccountingDataHarvesterPlugin.DRY_RUN_INPUT_PARAMETER, false);
|
||||||
|
|
||||||
/*
|
/*
|
||||||
Calendar from = DateUtils.getStartCalendar(2016, Calendar.SEPTEMBER, 1);
|
Calendar from = DateUtils.getStartCalendar(2016, Calendar.SEPTEMBER, 1);
|
||||||
|
@ -88,7 +89,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
// @Test
|
||||||
public void launchOldData() {
|
public void launchOldData() {
|
||||||
try {
|
try {
|
||||||
|
|
||||||
|
@ -272,7 +273,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
// @Test
|
||||||
public void testVREAccessesHarvester() {
|
public void testVREAccessesHarvester() {
|
||||||
try {
|
try {
|
||||||
|
|
||||||
|
@ -345,8 +346,72 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// @Test
|
||||||
|
public void testSocialInteraction() {
|
||||||
|
try {
|
||||||
|
|
||||||
|
Utils.setContext(ROOT);
|
||||||
|
DatabaseManager dbaseManager = new DatabaseManager();
|
||||||
|
|
||||||
|
DataHarvestPluginDeclaration dataHarvestPluginDeclaration = new DataHarvestPluginDeclaration();
|
||||||
|
|
||||||
|
AccountingDataHarvesterPlugin accountingDataHarvesterPlugin = new AccountingDataHarvesterPlugin(
|
||||||
|
dataHarvestPluginDeclaration);
|
||||||
|
Properties properties = accountingDataHarvesterPlugin.getConfigParameters();
|
||||||
|
AccountingDataHarvesterPlugin.getProperties().set(properties);
|
||||||
|
|
||||||
|
ContextAuthorization contextAuthorization = new ContextAuthorization();
|
||||||
|
|
||||||
|
SortedSet<String> contexts = new TreeSet<>();
|
||||||
|
contexts.add("/d4science.research-infrastructures.eu/D4Research");
|
||||||
|
contexts.add("/d4science.research-infrastructures.eu/FARM/WECAFC-FIRMS");
|
||||||
|
contexts.add("/d4science.research-infrastructures.eu/gCubeApps/BlueBridgeProject");
|
||||||
|
contexts.add("/d4science.research-infrastructures.eu/gCubeApps/Parthenos");
|
||||||
|
contexts.add("/d4science.research-infrastructures.eu/gCubeApps/ScalableDataMining");
|
||||||
|
contexts.add("/d4science.research-infrastructures.eu/gCubeApps/gCube");
|
||||||
|
|
||||||
|
AggregationType aggregationType = AggregationType.MONTHLY;
|
||||||
|
|
||||||
|
Calendar from = DateUtils.getStartCalendar(2018, Calendar.JUNE, 1);
|
||||||
|
Date start = from.getTime();
|
||||||
|
Date end = DateUtils.getEndDateFromStartDate(aggregationType, start, 1);
|
||||||
|
|
||||||
|
logger.debug("Harvesting Social Interaction from {} to {}", DateUtils.format(start), DateUtils.format(end));
|
||||||
|
|
||||||
|
ArrayList<HarvestedData> data = new ArrayList<HarvestedData>();
|
||||||
|
|
||||||
|
|
||||||
|
for(String context : contexts) {
|
||||||
|
// Setting the token for the context
|
||||||
|
Utils.setContext(contextAuthorization.getTokenForContext(context));
|
||||||
|
try {
|
||||||
|
// Collecting info on social (posts, replies and likes)
|
||||||
|
logger.info("Going to harvest Social Interactions for {}", context);
|
||||||
|
SocialInteractionsHarvester socialHarvester = new SocialInteractionsHarvester(start, end);
|
||||||
|
List<HarvestedData> harvested = socialHarvester.getData();
|
||||||
|
data.addAll(harvested);
|
||||||
|
} catch(Exception e) {
|
||||||
|
logger.error("Error harvesting Social Interactions for {}", context, e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
logger.debug("Harvest Measures from {} to {} are {}", DateUtils.format(start), DateUtils.format(end), data);
|
||||||
|
Utils.setContext(ROOT);
|
||||||
|
dbaseManager.insertMonthlyData(start, end, data, true);
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
} catch(Exception e) {
|
||||||
|
logger.error("", e);
|
||||||
|
}
|
||||||
|
|
||||||
|
logger.info("End.");
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
@Test
|
|
||||||
|
|
||||||
|
// @Test
|
||||||
public void testMethodInvocation() {
|
public void testMethodInvocation() {
|
||||||
try {
|
try {
|
||||||
|
|
||||||
|
@ -367,7 +432,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
// @Test
|
||||||
public void testTagMeMethodInvocation() {
|
public void testTagMeMethodInvocation() {
|
||||||
try {
|
try {
|
||||||
|
|
||||||
|
@ -424,7 +489,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
@Test
|
// @Test
|
||||||
public void testFilteringGenericResource() {
|
public void testFilteringGenericResource() {
|
||||||
try {
|
try {
|
||||||
Utils.setContext(RESOURCE_CATALOGUE);
|
Utils.setContext(RESOURCE_CATALOGUE);
|
||||||
|
@ -449,7 +514,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
// @Test
|
||||||
public void testResourceCatalogueHarvester() {
|
public void testResourceCatalogueHarvester() {
|
||||||
try {
|
try {
|
||||||
|
|
||||||
|
|
Loading…
Reference in New Issue