gCubeSystem
/
accounting-dashboard-harvester-se-plugin
15
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Commented tests 

Refs #11760: Refactor accounting-dashboard-harvester-se-plugin to support new DB schema 

Task-Url: https://support.d4science.org/issues/11760

git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/accounting/accounting-dashboard-harvester-se-plugin@171296 82a268e6-3cf1-43bd-a215-b396298e98cf
This commit is contained in:
Luca Frosini 2018-09-04 09:20:11 +00:00
parent 95bf87332f
commit 03aa4b0c42
1 changed files with 117 additions and 161 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

130
src/test/java/org/gcube/dataharvest/AccountingDataHarvesterPluginTest.java
View File

@ -41,21 +41,21 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
public static final String SO_BIG_DATA_CONTEXT = "/d4science.research-infrastructures.eu/SoBigData";
public static SortedSet<String> getContexts() throws Exception{
public static SortedSet<String> getContexts() throws Exception {
SortedSet<String> contexts = new TreeSet<>();
LinkedHashMap<String,ScopeBean> map = ContextManager.readContexts();
for(String scope : map.keySet()) {
try {
String context = map.get(scope).toString();
contexts.add(context);
}catch (Exception e) {
} catch(Exception e) {
throw e;
}
}
return contexts;
}
@Test
// @Test
public void getDimensions() {
try {
@ -68,7 +68,6 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
logger.debug("{} - {} - {} - {}", d.getId(), d.getGroup(), d.getAggregatedMeasure(), d.getLabel());
}
logger.info("End.");
} catch(Exception e) {
@ -76,7 +75,6 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
}
}
// @Test
public void launch() {
try {
@ -152,14 +150,12 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
}
}
// @Test
public void launchOldDataVREAccessesHarvester() {
try {
Utils.setContext(ROOT);
//DatabaseManager dbaseManager = new DatabaseManager();
AccountingDao dao = AccountingDao.get();
// AccountingDao dao = AccountingDao.get();
DataHarvestPluginDeclaration dataHarvestPluginDeclaration = new DataHarvestPluginDeclaration();
@ -183,7 +179,6 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
logger.debug("Harvesting from {} to {}", DateUtils.format(start), DateUtils.format(end));
// ArrayList<HarvestedData> data = new ArrayList<HarvestedData>();
ArrayList<AccountingRecord> accountingRecords = new ArrayList<>();
VREAccessesHarvester vreAccessesHarvester = null;
@ -198,7 +193,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
if(scopeBean.is(Type.INFRASTRUCTURE)) {
vreAccessesHarvester = new VREAccessesHarvester(start, end);
}else {
} else {
// This code should be never used because the scopes are sorted by fullname
ScopeBean parent = scopeBean.enclosingScope();
@ -218,28 +213,26 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
}
try {
if(context.startsWith(AccountingDataHarvesterPlugin.SO_BIG_DATA_VO) && start.before(DateUtils.getStartCalendar(2018, Calendar.APRIL, 1).getTime())) {
logger.info("Not Harvesting VREs Accesses for {} from {} to {}", context, DateUtils.format(start), DateUtils.format(end));
if(context.startsWith(AccountingDataHarvesterPlugin.SO_BIG_DATA_VO)
&& start.before(DateUtils.getStartCalendar(2018, Calendar.APRIL, 1).getTime())) {
logger.info("Not Harvesting VREs Accesses for {} from {} to {}", context,
DateUtils.format(start), DateUtils.format(end));
} else {
// Collecting Google Analytics Data for VREs Accesses
List<AccountingRecord> harvested = vreAccessesHarvester.getAccountingRecords();
accountingRecords.addAll(harvested);
/*
List<AccountingRecord> harvested = vreAccessesHarvester.getData();
data.addAll(harvested);
*/
}
} catch(Exception e) {
logger.error("Error harvesting Social Interactions for {}", context, e);
}
}
// logger.debug("Harvest Measures from {} to {} are {}", DateUtils.format(start), DateUtils.format(end), data);
logger.debug("Harvest Measures from {} to {} are {}", DateUtils.format(start), DateUtils.format(end), accountingRecords);
logger.debug("Harvest Measures from {} to {} are {}", DateUtils.format(start), DateUtils.format(end),
accountingRecords);
Utils.setContext(ROOT);
dao.insertRecords(accountingRecords.toArray(new AccountingRecord[1]));
// dbaseManager.insertMonthlyData(start, end, data, true);
// dao.insertRecords(accountingRecords.toArray(new AccountingRecord[1]));
Thread.sleep(TimeUnit.SECONDS.toMillis(10));
@ -249,8 +242,6 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
Utils.setContext(ROOT);
} catch(Exception e) {
logger.error("", e);
}
@ -278,21 +269,20 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
try {
if(scopeBean.is(Type.VRE) && start.equals(DateUtils.getPreviousPeriod(aggregationType).getTime())) {
logger.info("Harvesting (VRE Users) for {} from {} to {}", context, DateUtils.format(start),
DateUtils.format(end));
}else {
} else {
logger.info("--- Not Harvesting (VRE Users) for {} from {} to {}", context, DateUtils.format(start),
DateUtils.format(end));
}
if((context.startsWith(AccountingDataHarvesterPlugin.SO_BIG_DATA_VO) ||
context.startsWith(AccountingDataHarvesterPlugin.SO_BIG_DATA_EU_VRE)
if((context.startsWith(AccountingDataHarvesterPlugin.SO_BIG_DATA_VO)
|| context.startsWith(AccountingDataHarvesterPlugin.SO_BIG_DATA_EU_VRE)
|| context.startsWith(AccountingDataHarvesterPlugin.SO_BIG_DATA_IT_VRE))
&& start.before(DateUtils.getStartCalendar(2018, Calendar.APRIL, 1).getTime())) {
logger.info("--- Not Harvesting (SoBigData Check) for {} from {} to {}", context, DateUtils.format(start),
DateUtils.format(end));
logger.info("--- Not Harvesting (SoBigData Check) for {} from {} to {}", context,
DateUtils.format(start), DateUtils.format(end));
} else {
logger.info("Harvesting (SoBigData Check) for {} from {} to {}", context, DateUtils.format(start),
DateUtils.format(end));
@ -328,7 +318,6 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
VREAccessesHarvester vreAccessesHarvester = null;
// ArrayList<HarvestedData> data = new ArrayList<HarvestedData>();
ArrayList<AccountingRecord> accountingRecords = new ArrayList<>();
for(String context : contexts) {
@ -341,7 +330,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
if(scopeBean.is(Type.INFRASTRUCTURE)) {
vreAccessesHarvester = new VREAccessesHarvester(start, end);
}else {
} else {
// This code should be never used because the scopes are sorted by fullname
ScopeBean parent = scopeBean.enclosingScope();
@ -361,18 +350,14 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
}
try {
if(context.startsWith(AccountingDataHarvesterPlugin.SO_BIG_DATA_VO) && start.before(DateUtils.getStartCalendar(2018, Calendar.APRIL, 1).getTime())) {
logger.info("Not Harvesting VREs Accesses for {} from {} to {}", context, DateUtils.format(start), DateUtils.format(end));
if(context.startsWith(AccountingDataHarvesterPlugin.SO_BIG_DATA_VO)
&& start.before(DateUtils.getStartCalendar(2018, Calendar.APRIL, 1).getTime())) {
logger.info("Not Harvesting VREs Accesses for {} from {} to {}", context,
DateUtils.format(start), DateUtils.format(end));
} else {
// Collecting Google Analytics Data for VREs Accesses
List<AccountingRecord> harvested = vreAccessesHarvester.getAccountingRecords();
accountingRecords.addAll(harvested);
/*
List<HarvestedData> harvested = vreAccessesHarvester.getData();
data.addAll(harvested);
*/
}
} catch(Exception e) {
logger.error("Error harvesting Social Interactions for {}", context, e);
@ -391,8 +376,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
try {
Utils.setContext(ROOT);
//DatabaseManager dbaseManager = new DatabaseManager();
AccountingDao dao = AccountingDao.get();
// AccountingDao dao = AccountingDao.get();
DataHarvestPluginDeclaration dataHarvestPluginDeclaration = new DataHarvestPluginDeclaration();
@ -419,7 +403,6 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
logger.debug("Harvesting Social Interaction from {} to {}", DateUtils.format(start), DateUtils.format(end));
// ArrayList<HarvestedData> data = new ArrayList<HarvestedData>();
ArrayList<AccountingRecord> accountingRecords = new ArrayList<>();
for(String context : contexts) {
@ -431,23 +414,15 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
SocialInteractionsHarvester socialHarvester = new SocialInteractionsHarvester(start, end);
List<AccountingRecord> harvested = socialHarvester.getAccountingRecords();
accountingRecords.addAll(harvested);
/*
List<HarvestedData> harvested = socialHarvester.getData();
data.addAll(harvested);
*/
} catch(Exception e) {
logger.error("Error harvesting Social Interactions for {}", context, e);
}
}
//logger.debug("Harvest Measures from {} to {} are {}", DateUtils.format(start), DateUtils.format(end), data);
logger.debug("Harvest Measures from {} to {} are {}", DateUtils.format(start), DateUtils.format(end), accountingRecords);
logger.debug("Harvest Measures from {} to {} are {}", DateUtils.format(start), DateUtils.format(end),
accountingRecords);
Utils.setContext(ROOT);
dao.insertRecords(accountingRecords.toArray(new AccountingRecord[1]));
// dbaseManager.insertMonthlyData(start, end, data, true);
// dao.insertRecords(accountingRecords.toArray(new AccountingRecord[1]));
} catch(Exception e) {
logger.error("", e);
@ -457,8 +432,6 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
}
// @Test
public void testMethodInvocation() {
try {
@ -471,13 +444,9 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
Date end = DateUtils.getEndDateFromStartDate(measureType, start, 1);
MethodInvocationHarvester methodInvocationHarvester = new MethodInvocationHarvester(start, end);
List<AccountingRecord> harvestedData = methodInvocationHarvester.getAccountingRecords();
/*
List<HarvestedData> harvested = methodInvocationHarvester.getData();
*/
List<AccountingRecord> accountingRecords = methodInvocationHarvester.getAccountingRecords();
logger.debug("{}", harvestedData);
logger.debug("{}", accountingRecords);
} catch(Exception e) {
logger.error("", e);
@ -496,12 +465,9 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
Date end = DateUtils.getEndDateFromStartDate(measureType, start, 1);
TagMeMethodInvocationHarvester methodInvocationHarvester = new TagMeMethodInvocationHarvester(start, end);
List<AccountingRecord> harvestedData = methodInvocationHarvester.getAccountingRecords();
/*
List<HarvestedData> harvested = methodInvocationHarvester.getData();
*/
List<AccountingRecord> accountingRecords = methodInvocationHarvester.getAccountingRecords();
logger.debug("{}", harvestedData);
logger.debug("{}", accountingRecords);
} catch(Exception e) {
logger.error("", e);
@ -519,10 +485,11 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
Properties properties = accountingDataHarvesterPlugin.getConfigParameters();
AccountingDataHarvesterPlugin.getProperties().set(properties);
// DatabaseManager dbaseManager = new DatabaseManager();
// AccountingDao dao = AccountingDao.get();
ContextAuthorization contextAuthorization = new ContextAuthorization();
Utils.setContext(contextAuthorization.getTokenForContext("/d4science.research-infrastructures.eu/SoBigData/SportsDataScience"));
Utils.setContext(contextAuthorization
.getTokenForContext("/d4science.research-infrastructures.eu/SoBigData/SportsDataScience"));
AggregationType measureType = AggregationType.MONTHLY;
@ -531,15 +498,11 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
VREUsersHarvester vreUsersHarvester = new VREUsersHarvester(start, end);
List<AccountingRecord> harvested = vreUsersHarvester.getAccountingRecords();
/*
List<HarvestedData> harvested = vreUsersHarvester.getData();
*/
logger.info("Harvested Data from {} to {} : {}", DateUtils.format(start), DateUtils.format(end), harvested);
org.gcube.dataharvest.utils.Utils.setContext(ROOT);
// dbaseManager.insertMonthlyData(start, end, harvested, true);
// dao.insertRecords(accountingRecords.toArray(new AccountingRecord[1]));
} catch(Exception e) {
logger.error("", e);
@ -547,7 +510,6 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
}
// @Test
public void testFilteringGenericResource() {
try {
@ -563,8 +525,10 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
AccountingDataHarvesterPlugin accountingDataHarvesterPlugin = new AccountingDataHarvesterPlugin(null);
accountingDataHarvesterPlugin.getConfigParameters();
ResourceCatalogueHarvester resourceCatalogueHarvester = new ResourceCatalogueHarvester(start, end, contexts);
SortedSet<String> validContexts = resourceCatalogueHarvester.getValidContexts(contexts, SO_BIG_DATA_CONTEXT + "/");
ResourceCatalogueHarvester resourceCatalogueHarvester = new ResourceCatalogueHarvester(start, end,
contexts);
SortedSet<String> validContexts = resourceCatalogueHarvester.getValidContexts(contexts,
SO_BIG_DATA_CONTEXT + "/");
logger.info("Valid Contexts {}", validContexts);
} catch(Exception e) {
@ -592,11 +556,9 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
SortedSet<String> contexts = getContexts();
ResourceCatalogueHarvester resourceCatalogueHarvester = new ResourceCatalogueHarvester(start, end, contexts);
ResourceCatalogueHarvester resourceCatalogueHarvester = new ResourceCatalogueHarvester(start, end,
contexts);
List<AccountingRecord> data = resourceCatalogueHarvester.getAccountingRecords();
/*
List<HarvestedData> data = resourceCatalogueHarvester.getData();
*/
logger.debug("{}", data);
@ -605,7 +567,6 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
}
}
@Test
public void testDataMethodDownloadHarvester() {
try {
@ -625,11 +586,9 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
SortedSet<String> contexts = getContexts();
DataMethodDownloadHarvester dataMethodDownloadHarvester = new DataMethodDownloadHarvester(start, end, contexts);
DataMethodDownloadHarvester dataMethodDownloadHarvester = new DataMethodDownloadHarvester(start, end,
contexts);
List<AccountingRecord> data = dataMethodDownloadHarvester.getAccountingRecords();
/*
List<HarvestedData> data = dataMethodDownloadHarvester.getData();
*/
logger.debug("{}", data);
@ -638,7 +597,4 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
}
}
}