gCubeSystem
/
accounting-dashboard-harvester-se-plugin
15
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Commented tests 

Refs #11760: Refactor accounting-dashboard-harvester-se-plugin to support new DB schema 

Task-Url: https://support.d4science.org/issues/11760

git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/accounting/accounting-dashboard-harvester-se-plugin@171296 82a268e6-3cf1-43bd-a215-b396298e98cf
This commit is contained in:
Luca Frosini 2018-09-04 09:20:11 +00:00
parent 95bf87332f
commit 03aa4b0c42
1 changed files with 117 additions and 161 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

134
src/test/java/org/gcube/dataharvest/AccountingDataHarvesterPluginTest.java
View File

@ -41,21 +41,21 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
public static final String SO_BIG_DATA_CONTEXT = "/d4science.research-infrastructures.eu/SoBigData"; public static final String SO_BIG_DATA_CONTEXT = "/d4science.research-infrastructures.eu/SoBigData";
public static SortedSet<String> getContexts() throws Exception{ public static SortedSet<String> getContexts() throws Exception {
SortedSet<String> contexts = new TreeSet<>(); SortedSet<String> contexts = new TreeSet<>();
LinkedHashMap<String,ScopeBean> map = ContextManager.readContexts(); LinkedHashMap<String,ScopeBean> map = ContextManager.readContexts();
for(String scope : map.keySet()) { for(String scope : map.keySet()) {
try { try {
String context = map.get(scope).toString(); String context = map.get(scope).toString();
contexts.add(context); contexts.add(context);
}catch (Exception e) { } catch(Exception e) {
throw e; throw e;
} }
} }
return contexts; return contexts;
} }
@Test // @Test
public void getDimensions() { public void getDimensions() {
try { try {
@ -68,7 +68,6 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
logger.debug("{} - {} - {} - {}", d.getId(), d.getGroup(), d.getAggregatedMeasure(), d.getLabel()); logger.debug("{} - {} - {} - {}", d.getId(), d.getGroup(), d.getAggregatedMeasure(), d.getLabel());
} }
logger.info("End."); logger.info("End.");
} catch(Exception e) { } catch(Exception e) {
@ -76,7 +75,6 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
} }
} }
// @Test // @Test
public void launch() { public void launch() {
try { try {
@ -135,7 +133,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
Calendar from = DateUtils.getStartCalendar(2016, Calendar.SEPTEMBER, 1); Calendar from = DateUtils.getStartCalendar(2016, Calendar.SEPTEMBER, 1);
Calendar runbeforeDate = DateUtils.getStartCalendar(2018, Calendar.JUNE, 1); Calendar runbeforeDate = DateUtils.getStartCalendar(2018, Calendar.JUNE, 1);
while(from.before(runbeforeDate)) { while(from.before(runbeforeDate)) {
String fromDate = DateUtils.LAUNCH_DATE_FORMAT.format(from.getTime()); String fromDate = DateUtils.LAUNCH_DATE_FORMAT.format(from.getTime());
@ -152,14 +150,12 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
} }
} }
// @Test // @Test
public void launchOldDataVREAccessesHarvester() { public void launchOldDataVREAccessesHarvester() {
try { try {
Utils.setContext(ROOT); Utils.setContext(ROOT);
//DatabaseManager dbaseManager = new DatabaseManager(); // AccountingDao dao = AccountingDao.get();
AccountingDao dao = AccountingDao.get();
DataHarvestPluginDeclaration dataHarvestPluginDeclaration = new DataHarvestPluginDeclaration(); DataHarvestPluginDeclaration dataHarvestPluginDeclaration = new DataHarvestPluginDeclaration();
@ -175,7 +171,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
Calendar from = DateUtils.getStartCalendar(2018, Calendar.APRIL, 1); Calendar from = DateUtils.getStartCalendar(2018, Calendar.APRIL, 1);
Calendar runbeforeDate = DateUtils.getStartCalendar(2018, Calendar.JUNE, 1); Calendar runbeforeDate = DateUtils.getStartCalendar(2018, Calendar.JUNE, 1);
while(from.before(runbeforeDate)) { while(from.before(runbeforeDate)) {
Date start = from.getTime(); Date start = from.getTime();
@ -183,7 +179,6 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
logger.debug("Harvesting from {} to {}", DateUtils.format(start), DateUtils.format(end)); logger.debug("Harvesting from {} to {}", DateUtils.format(start), DateUtils.format(end));
// ArrayList<HarvestedData> data = new ArrayList<HarvestedData>();
ArrayList<AccountingRecord> accountingRecords = new ArrayList<>(); ArrayList<AccountingRecord> accountingRecords = new ArrayList<>();
VREAccessesHarvester vreAccessesHarvester = null; VREAccessesHarvester vreAccessesHarvester = null;
@ -198,7 +193,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
if(scopeBean.is(Type.INFRASTRUCTURE)) { if(scopeBean.is(Type.INFRASTRUCTURE)) {
vreAccessesHarvester = new VREAccessesHarvester(start, end); vreAccessesHarvester = new VREAccessesHarvester(start, end);
}else { } else {
// This code should be never used because the scopes are sorted by fullname // This code should be never used because the scopes are sorted by fullname
ScopeBean parent = scopeBean.enclosingScope(); ScopeBean parent = scopeBean.enclosingScope();
@ -218,28 +213,26 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
} }
try { try {
if(context.startsWith(AccountingDataHarvesterPlugin.SO_BIG_DATA_VO) && start.before(DateUtils.getStartCalendar(2018, Calendar.APRIL, 1).getTime())) { if(context.startsWith(AccountingDataHarvesterPlugin.SO_BIG_DATA_VO)
logger.info("Not Harvesting VREs Accesses for {} from {} to {}", context, DateUtils.format(start), DateUtils.format(end)); && start.before(DateUtils.getStartCalendar(2018, Calendar.APRIL, 1).getTime())) {
logger.info("Not Harvesting VREs Accesses for {} from {} to {}", context,
DateUtils.format(start), DateUtils.format(end));
} else { } else {
// Collecting Google Analytics Data for VREs Accesses // Collecting Google Analytics Data for VREs Accesses
List<AccountingRecord> harvested = vreAccessesHarvester.getAccountingRecords(); List<AccountingRecord> harvested = vreAccessesHarvester.getAccountingRecords();
accountingRecords.addAll(harvested); accountingRecords.addAll(harvested);
/*
List<AccountingRecord> harvested = vreAccessesHarvester.getData();
data.addAll(harvested);
*/
} }
} catch(Exception e) { } catch(Exception e) {
logger.error("Error harvesting Social Interactions for {}", context, e); logger.error("Error harvesting Social Interactions for {}", context, e);
} }
} }
// logger.debug("Harvest Measures from {} to {} are {}", DateUtils.format(start), DateUtils.format(end), data); logger.debug("Harvest Measures from {} to {} are {}", DateUtils.format(start), DateUtils.format(end),
logger.debug("Harvest Measures from {} to {} are {}", DateUtils.format(start), DateUtils.format(end), accountingRecords); accountingRecords);
Utils.setContext(ROOT); Utils.setContext(ROOT);
dao.insertRecords(accountingRecords.toArray(new AccountingRecord[1])); // dao.insertRecords(accountingRecords.toArray(new AccountingRecord[1]));
// dbaseManager.insertMonthlyData(start, end, data, true);
Thread.sleep(TimeUnit.SECONDS.toMillis(10)); Thread.sleep(TimeUnit.SECONDS.toMillis(10));
@ -249,8 +242,6 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
Utils.setContext(ROOT); Utils.setContext(ROOT);
} catch(Exception e) { } catch(Exception e) {
logger.error("", e); logger.error("", e);
} }
@ -278,21 +269,20 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
try { try {
if(scopeBean.is(Type.VRE) && start.equals(DateUtils.getPreviousPeriod(aggregationType).getTime())) { if(scopeBean.is(Type.VRE) && start.equals(DateUtils.getPreviousPeriod(aggregationType).getTime())) {
logger.info("Harvesting (VRE Users) for {} from {} to {}", context, DateUtils.format(start), logger.info("Harvesting (VRE Users) for {} from {} to {}", context, DateUtils.format(start),
DateUtils.format(end)); DateUtils.format(end));
}else { } else {
logger.info("--- Not Harvesting (VRE Users) for {} from {} to {}", context, DateUtils.format(start), logger.info("--- Not Harvesting (VRE Users) for {} from {} to {}", context, DateUtils.format(start),
DateUtils.format(end)); DateUtils.format(end));
} }
if((context.startsWith(AccountingDataHarvesterPlugin.SO_BIG_DATA_VO) || if((context.startsWith(AccountingDataHarvesterPlugin.SO_BIG_DATA_VO)
context.startsWith(AccountingDataHarvesterPlugin.SO_BIG_DATA_EU_VRE) || context.startsWith(AccountingDataHarvesterPlugin.SO_BIG_DATA_EU_VRE)
|| context.startsWith(AccountingDataHarvesterPlugin.SO_BIG_DATA_IT_VRE)) || context.startsWith(AccountingDataHarvesterPlugin.SO_BIG_DATA_IT_VRE))
&& start.before(DateUtils.getStartCalendar(2018, Calendar.APRIL, 1).getTime())) { && start.before(DateUtils.getStartCalendar(2018, Calendar.APRIL, 1).getTime())) {
logger.info("--- Not Harvesting (SoBigData Check) for {} from {} to {}", context, DateUtils.format(start), logger.info("--- Not Harvesting (SoBigData Check) for {} from {} to {}", context,
DateUtils.format(end)); DateUtils.format(start), DateUtils.format(end));
} else { } else {
logger.info("Harvesting (SoBigData Check) for {} from {} to {}", context, DateUtils.format(start), logger.info("Harvesting (SoBigData Check) for {} from {} to {}", context, DateUtils.format(start),
DateUtils.format(end)); DateUtils.format(end));
@ -328,7 +318,6 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
VREAccessesHarvester vreAccessesHarvester = null; VREAccessesHarvester vreAccessesHarvester = null;
// ArrayList<HarvestedData> data = new ArrayList<HarvestedData>();
ArrayList<AccountingRecord> accountingRecords = new ArrayList<>(); ArrayList<AccountingRecord> accountingRecords = new ArrayList<>();
for(String context : contexts) { for(String context : contexts) {
@ -341,7 +330,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
if(scopeBean.is(Type.INFRASTRUCTURE)) { if(scopeBean.is(Type.INFRASTRUCTURE)) {
vreAccessesHarvester = new VREAccessesHarvester(start, end); vreAccessesHarvester = new VREAccessesHarvester(start, end);
}else { } else {
// This code should be never used because the scopes are sorted by fullname // This code should be never used because the scopes are sorted by fullname
ScopeBean parent = scopeBean.enclosingScope(); ScopeBean parent = scopeBean.enclosingScope();
@ -361,18 +350,14 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
} }
try { try {
if(context.startsWith(AccountingDataHarvesterPlugin.SO_BIG_DATA_VO) && start.before(DateUtils.getStartCalendar(2018, Calendar.APRIL, 1).getTime())) { if(context.startsWith(AccountingDataHarvesterPlugin.SO_BIG_DATA_VO)
logger.info("Not Harvesting VREs Accesses for {} from {} to {}", context, DateUtils.format(start), DateUtils.format(end)); && start.before(DateUtils.getStartCalendar(2018, Calendar.APRIL, 1).getTime())) {
logger.info("Not Harvesting VREs Accesses for {} from {} to {}", context,
DateUtils.format(start), DateUtils.format(end));
} else { } else {
// Collecting Google Analytics Data for VREs Accesses // Collecting Google Analytics Data for VREs Accesses
List<AccountingRecord> harvested = vreAccessesHarvester.getAccountingRecords(); List<AccountingRecord> harvested = vreAccessesHarvester.getAccountingRecords();
accountingRecords.addAll(harvested); accountingRecords.addAll(harvested);
/*
List<HarvestedData> harvested = vreAccessesHarvester.getData();
data.addAll(harvested);
*/
} }
} catch(Exception e) { } catch(Exception e) {
logger.error("Error harvesting Social Interactions for {}", context, e); logger.error("Error harvesting Social Interactions for {}", context, e);
@ -391,8 +376,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
try { try {
Utils.setContext(ROOT); Utils.setContext(ROOT);
//DatabaseManager dbaseManager = new DatabaseManager(); // AccountingDao dao = AccountingDao.get();
AccountingDao dao = AccountingDao.get();
DataHarvestPluginDeclaration dataHarvestPluginDeclaration = new DataHarvestPluginDeclaration(); DataHarvestPluginDeclaration dataHarvestPluginDeclaration = new DataHarvestPluginDeclaration();
@ -419,7 +403,6 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
logger.debug("Harvesting Social Interaction from {} to {}", DateUtils.format(start), DateUtils.format(end)); logger.debug("Harvesting Social Interaction from {} to {}", DateUtils.format(start), DateUtils.format(end));
// ArrayList<HarvestedData> data = new ArrayList<HarvestedData>();
ArrayList<AccountingRecord> accountingRecords = new ArrayList<>(); ArrayList<AccountingRecord> accountingRecords = new ArrayList<>();
for(String context : contexts) { for(String context : contexts) {
@ -431,23 +414,15 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
SocialInteractionsHarvester socialHarvester = new SocialInteractionsHarvester(start, end); SocialInteractionsHarvester socialHarvester = new SocialInteractionsHarvester(start, end);
List<AccountingRecord> harvested = socialHarvester.getAccountingRecords(); List<AccountingRecord> harvested = socialHarvester.getAccountingRecords();
accountingRecords.addAll(harvested); accountingRecords.addAll(harvested);
/*
List<HarvestedData> harvested = socialHarvester.getData();
data.addAll(harvested);
*/
} catch(Exception e) { } catch(Exception e) {
logger.error("Error harvesting Social Interactions for {}", context, e); logger.error("Error harvesting Social Interactions for {}", context, e);
} }
} }
//logger.debug("Harvest Measures from {} to {} are {}", DateUtils.format(start), DateUtils.format(end), data); logger.debug("Harvest Measures from {} to {} are {}", DateUtils.format(start), DateUtils.format(end),
logger.debug("Harvest Measures from {} to {} are {}", DateUtils.format(start), DateUtils.format(end), accountingRecords); accountingRecords);
Utils.setContext(ROOT); Utils.setContext(ROOT);
dao.insertRecords(accountingRecords.toArray(new AccountingRecord[1])); // dao.insertRecords(accountingRecords.toArray(new AccountingRecord[1]));
// dbaseManager.insertMonthlyData(start, end, data, true);
} catch(Exception e) { } catch(Exception e) {
logger.error("", e); logger.error("", e);
@ -457,8 +432,6 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
} }
// @Test // @Test
public void testMethodInvocation() { public void testMethodInvocation() {
try { try {
@ -471,13 +444,9 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
Date end = DateUtils.getEndDateFromStartDate(measureType, start, 1); Date end = DateUtils.getEndDateFromStartDate(measureType, start, 1);
MethodInvocationHarvester methodInvocationHarvester = new MethodInvocationHarvester(start, end); MethodInvocationHarvester methodInvocationHarvester = new MethodInvocationHarvester(start, end);
List<AccountingRecord> harvestedData = methodInvocationHarvester.getAccountingRecords(); List<AccountingRecord> accountingRecords = methodInvocationHarvester.getAccountingRecords();
/*
List<HarvestedData> harvested = methodInvocationHarvester.getData();
*/
logger.debug("{}", accountingRecords);
logger.debug("{}", harvestedData);
} catch(Exception e) { } catch(Exception e) {
logger.error("", e); logger.error("", e);
@ -496,12 +465,9 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
Date end = DateUtils.getEndDateFromStartDate(measureType, start, 1); Date end = DateUtils.getEndDateFromStartDate(measureType, start, 1);
TagMeMethodInvocationHarvester methodInvocationHarvester = new TagMeMethodInvocationHarvester(start, end); TagMeMethodInvocationHarvester methodInvocationHarvester = new TagMeMethodInvocationHarvester(start, end);
List<AccountingRecord> harvestedData = methodInvocationHarvester.getAccountingRecords(); List<AccountingRecord> accountingRecords = methodInvocationHarvester.getAccountingRecords();
/*
List<HarvestedData> harvested = methodInvocationHarvester.getData();
*/
logger.debug("{}", harvestedData); logger.debug("{}", accountingRecords);
} catch(Exception e) { } catch(Exception e) {
logger.error("", e); logger.error("", e);
@ -519,10 +485,11 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
Properties properties = accountingDataHarvesterPlugin.getConfigParameters(); Properties properties = accountingDataHarvesterPlugin.getConfigParameters();
AccountingDataHarvesterPlugin.getProperties().set(properties); AccountingDataHarvesterPlugin.getProperties().set(properties);
// DatabaseManager dbaseManager = new DatabaseManager(); // AccountingDao dao = AccountingDao.get();
ContextAuthorization contextAuthorization = new ContextAuthorization(); ContextAuthorization contextAuthorization = new ContextAuthorization();
Utils.setContext(contextAuthorization.getTokenForContext("/d4science.research-infrastructures.eu/SoBigData/SportsDataScience")); Utils.setContext(contextAuthorization
.getTokenForContext("/d4science.research-infrastructures.eu/SoBigData/SportsDataScience"));
AggregationType measureType = AggregationType.MONTHLY; AggregationType measureType = AggregationType.MONTHLY;
@ -531,15 +498,11 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
VREUsersHarvester vreUsersHarvester = new VREUsersHarvester(start, end); VREUsersHarvester vreUsersHarvester = new VREUsersHarvester(start, end);
List<AccountingRecord> harvested = vreUsersHarvester.getAccountingRecords(); List<AccountingRecord> harvested = vreUsersHarvester.getAccountingRecords();
/*
List<HarvestedData> harvested = vreUsersHarvester.getData();
*/
logger.info("Harvested Data from {} to {} : {}", DateUtils.format(start), DateUtils.format(end), harvested); logger.info("Harvested Data from {} to {} : {}", DateUtils.format(start), DateUtils.format(end), harvested);
org.gcube.dataharvest.utils.Utils.setContext(ROOT); org.gcube.dataharvest.utils.Utils.setContext(ROOT);
// dbaseManager.insertMonthlyData(start, end, harvested, true); // dao.insertRecords(accountingRecords.toArray(new AccountingRecord[1]));
} catch(Exception e) { } catch(Exception e) {
logger.error("", e); logger.error("", e);
@ -547,7 +510,6 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
} }
// @Test // @Test
public void testFilteringGenericResource() { public void testFilteringGenericResource() {
try { try {
@ -563,8 +525,10 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
AccountingDataHarvesterPlugin accountingDataHarvesterPlugin = new AccountingDataHarvesterPlugin(null); AccountingDataHarvesterPlugin accountingDataHarvesterPlugin = new AccountingDataHarvesterPlugin(null);
accountingDataHarvesterPlugin.getConfigParameters(); accountingDataHarvesterPlugin.getConfigParameters();
ResourceCatalogueHarvester resourceCatalogueHarvester = new ResourceCatalogueHarvester(start, end, contexts); ResourceCatalogueHarvester resourceCatalogueHarvester = new ResourceCatalogueHarvester(start, end,
SortedSet<String> validContexts = resourceCatalogueHarvester.getValidContexts(contexts, SO_BIG_DATA_CONTEXT + "/"); contexts);
SortedSet<String> validContexts = resourceCatalogueHarvester.getValidContexts(contexts,
SO_BIG_DATA_CONTEXT + "/");
logger.info("Valid Contexts {}", validContexts); logger.info("Valid Contexts {}", validContexts);
} catch(Exception e) { } catch(Exception e) {
@ -592,11 +556,9 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
SortedSet<String> contexts = getContexts(); SortedSet<String> contexts = getContexts();
ResourceCatalogueHarvester resourceCatalogueHarvester = new ResourceCatalogueHarvester(start, end, contexts); ResourceCatalogueHarvester resourceCatalogueHarvester = new ResourceCatalogueHarvester(start, end,
contexts);
List<AccountingRecord> data = resourceCatalogueHarvester.getAccountingRecords(); List<AccountingRecord> data = resourceCatalogueHarvester.getAccountingRecords();
/*
List<HarvestedData> data = resourceCatalogueHarvester.getData();
*/
logger.debug("{}", data); logger.debug("{}", data);
@ -605,7 +567,6 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
} }
} }
@Test @Test
public void testDataMethodDownloadHarvester() { public void testDataMethodDownloadHarvester() {
try { try {
@ -625,11 +586,9 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
SortedSet<String> contexts = getContexts(); SortedSet<String> contexts = getContexts();
DataMethodDownloadHarvester dataMethodDownloadHarvester = new DataMethodDownloadHarvester(start, end, contexts); DataMethodDownloadHarvester dataMethodDownloadHarvester = new DataMethodDownloadHarvester(start, end,
contexts);
List<AccountingRecord> data = dataMethodDownloadHarvester.getAccountingRecords(); List<AccountingRecord> data = dataMethodDownloadHarvester.getAccountingRecords();
/*
List<HarvestedData> data = dataMethodDownloadHarvester.getData();
*/
logger.debug("{}", data); logger.debug("{}", data);
@ -638,7 +597,4 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
} }
} }
} }