gCubeSystem
/
accounting-dashboard-harvester-se-plugin
16
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

ref 21031: Add support to Jupyter 

Updated Jupyter Accesses Harvester
This commit is contained in:
Giancarlo Panichi 2021-03-26 16:04:07 +01:00
parent 38ec08e0a3
commit 24f2409df7
5 changed files with 42 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
src/main/java/org/gcube/dataharvest/harvester/JupyterAccessesHarvester.java
View File

@ -80,7 +80,7 @@ public class JupyterAccessesHarvester extends BasicHarvester {
public JupyterAccessesHarvester(Date start, Date end) throws Exception { public JupyterAccessesHarvester(Date start, Date end) throws Exception {
super(start, end); super(start, end);
logger.debug("JupyerAccessHArvester: {}, {}",start,end); logger.debug("JupyerAccessHArvester: {}, {}", start, end);
vreAccesses = getAllAccesses(start, end); vreAccesses = getAllAccesses(start, end);
} }
@ -95,21 +95,21 @@ public class JupyterAccessesHarvester extends BasicHarvester {
ScopeBean scopeBean = new ScopeBean(context); ScopeBean scopeBean = new ScopeBean(context);
String lowerCasedContext = scopeBean.name().toLowerCase(); String lowerCasedContext = scopeBean.name().toLowerCase();
logger.debug("JupyerAccessHArvester lowerCasedContext: {}",lowerCasedContext); logger.debug("JupyerAccessHArvester lowerCasedContext: {}", lowerCasedContext);
for (VREAccessesReportRow row : vreAccesses) { for (VREAccessesReportRow row : vreAccesses) {
String pagePath = row.getPagePath().toLowerCase(); String pagePath = row.getPagePath().toLowerCase();
//logger.debug("JupyerAccessHArvester pagePath: {}",lowerCasedContext);
if (pagePath != null && !pagePath.isEmpty()) { if (pagePath != null && !pagePath.isEmpty()) {
if (pagePath.contains(lowerCasedContext)) { if (pagePath.contains(lowerCasedContext)) {
if (pagePath.contains("jupyter") || pagePath.contains("jupiter")) { if (!pagePath.contains("catalogue")) {
logger.trace("Matched jupyter or jupiter ({}) : {}", lowerCasedContext, pagePath); if (pagePath.contains("jupyter") || pagePath.contains("jupiter")) {
measure += row.getVisitNumber(); logger.trace("Matched jupyter or jupiter ({}) : {}", lowerCasedContext, pagePath);
measure += row.getVisitNumber();
}
} }
} }
} }
} }
ScopeDescriptor scopeDescriptor = AccountingDashboardHarvesterPlugin.getScopeDescriptor(); ScopeDescriptor scopeDescriptor = AccountingDashboardHarvesterPlugin.getScopeDescriptor();
AccountingRecord ar = new AccountingRecord(scopeDescriptor, instant, AccountingRecord ar = new AccountingRecord(scopeDescriptor, instant,

35
src/test/java/org/gcube/dataharvest/AccountingDataHarvesterPluginTest.java
View File

@ -24,7 +24,6 @@ import org.gcube.common.scope.impl.ScopeBean.Type;
import org.gcube.dataharvest.datamodel.HarvestedDataKey; import org.gcube.dataharvest.datamodel.HarvestedDataKey;
import org.gcube.dataharvest.harvester.CatalogueAccessesHarvester; import org.gcube.dataharvest.harvester.CatalogueAccessesHarvester;
import org.gcube.dataharvest.harvester.CoreServicesAccessesHarvester; import org.gcube.dataharvest.harvester.CoreServicesAccessesHarvester;
import org.gcube.dataharvest.harvester.JupyterAccessesHarvester;
import org.gcube.dataharvest.harvester.MethodInvocationHarvester; import org.gcube.dataharvest.harvester.MethodInvocationHarvester;
import org.gcube.dataharvest.harvester.SocialInteractionsHarvester; import org.gcube.dataharvest.harvester.SocialInteractionsHarvester;
import org.gcube.dataharvest.harvester.VREAccessesHarvester; import org.gcube.dataharvest.harvester.VREAccessesHarvester;
@ -42,6 +41,7 @@ import org.gcube.vremanagement.executor.api.types.Scheduling;
import org.gcube.vremanagement.executor.client.SmartExecutorClient; import org.gcube.vremanagement.executor.client.SmartExecutorClient;
import org.gcube.vremanagement.executor.client.SmartExecutorClientFactory; import org.gcube.vremanagement.executor.client.SmartExecutorClientFactory;
import org.junit.Assert; import org.junit.Assert;
import org.junit.Ignore;
import org.junit.Test; import org.junit.Test;
import org.quartz.CronExpression; import org.quartz.CronExpression;
import org.slf4j.Logger; import org.slf4j.Logger;
@ -71,7 +71,8 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
return contexts; return contexts;
} }
// @Test @Ignore
@Test
public void getDimensions() { public void getDimensions() {
try { try {
@ -90,7 +91,8 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
logger.error("", e); logger.error("", e);
} }
} }
@Ignore
@Test @Test
public void launch() { public void launch() {
try { try {
@ -128,7 +130,8 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
} }
} }
// @Test @Ignore
@Test
public void launchPluginOnSmartExecutor() { public void launchPluginOnSmartExecutor() {
try { try {
@ -181,7 +184,8 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
} }
} }
// @Test @Ignore
//@Test
public void launchOldData() { public void launchOldData() {
try { try {
@ -217,6 +221,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
} }
} }
@Ignore
// @Test // @Test
public void launchOldDataVREAccessesHarvester() { public void launchOldDataVREAccessesHarvester() {
try { try {
@ -317,6 +322,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
} }
@Ignore
// @Test // @Test
public void testScopeBean() throws Exception { public void testScopeBean() throws Exception {
ContextTest.setContextByName(ROOT); ContextTest.setContextByName(ROOT);
@ -365,7 +371,8 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
} }
// @Test @Ignore
@Test
public void testVREAccessesHarvester() throws Exception { public void testVREAccessesHarvester() throws Exception {
try { try {
// AccountingDao dao = getAccountingDao(); // AccountingDao dao = getAccountingDao();
@ -416,6 +423,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
} }
} }
@Ignore
// @Test // @Test
public void testVREAccessesHarvesterAll() { public void testVREAccessesHarvesterAll() {
try { try {
@ -494,7 +502,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
} }
} }
@Ignore
// @Test // @Test
public void testSocialInteraction() { public void testSocialInteraction() {
try { try {
@ -554,6 +562,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
} }
@Ignore
// @Test // @Test
public void testMethodInvocation() { public void testMethodInvocation() {
try { try {
@ -616,6 +625,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
AccountingDashboardHarvesterPlugin.scopeDescriptor.set(actualScopeDescriptor); AccountingDashboardHarvesterPlugin.scopeDescriptor.set(actualScopeDescriptor);
} }
@Ignore
// @Test // @Test
public void testTagMeMethodInvocation() throws Exception { public void testTagMeMethodInvocation() throws Exception {
try { try {
@ -654,6 +664,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
} }
} }
@Ignore
// @Test // @Test
public void testGetVREUsersForSpecificVRE() { public void testGetVREUsersForSpecificVRE() {
try { try {
@ -689,6 +700,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
} }
@Ignore
// @Test // @Test
public void testFilteringGenericResource() { public void testFilteringGenericResource() {
try { try {
@ -716,6 +728,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
} }
@Ignore
// @Test // @Test
public void testResourceCatalogueHarvester() { public void testResourceCatalogueHarvester() {
try { try {
@ -748,7 +761,8 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
logger.error("", e); logger.error("", e);
} }
} }
@Ignore
@Test @Test
public void testCoreServicesHarvester() { public void testCoreServicesHarvester() {
try { try {
@ -783,7 +797,8 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
logger.error("", e); logger.error("", e);
} }
} }
@Ignore
@Test @Test
public void testCatalogueHarvester() { public void testCatalogueHarvester() {
try { try {
@ -819,6 +834,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
} }
} }
@Ignore
// @Test // @Test
public void testDataMethodDownloadHarvester() { public void testDataMethodDownloadHarvester() {
try { try {
@ -868,6 +884,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
public static final String E_LEARNING_AREA_VRE = "/d4science.research-infrastructures.eu/SoBigData/E-Learning_Area"; public static final String E_LEARNING_AREA_VRE = "/d4science.research-infrastructures.eu/SoBigData/E-Learning_Area";
@Ignore
// @Test // @Test
public void addMissingVREAccesses() { public void addMissingVREAccesses() {
try { try {

2
src/test/java/org/gcube/dataharvest/harvester/sobigdata/SoBigDataHarvesterTest.java
View File

@ -3,6 +3,7 @@ package org.gcube.dataharvest.harvester.sobigdata;
import java.util.List; import java.util.List;
import org.gcube.dataharvest.utils.ContextTest; import org.gcube.dataharvest.utils.ContextTest;
import org.junit.Ignore;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
@ -10,6 +11,7 @@ public class SoBigDataHarvesterTest extends ContextTest {
private static Logger logger = LoggerFactory.getLogger(SoBigDataHarvesterTest.class); private static Logger logger = LoggerFactory.getLogger(SoBigDataHarvesterTest.class);
@Ignore
// @Test // @Test
public void testGroupList() throws Exception { public void testGroupList() throws Exception {
// ContextTest.setContextByName("/d4science.research-infrastructures.eu/D4Research/AGINFRAplusDev"); // ContextTest.setContextByName("/d4science.research-infrastructures.eu/D4Research/AGINFRAplusDev");

8
src/test/java/org/gcube/dataharvest/jupyter/AccountingDataHarvesterJupyterTest.java
View File

@ -10,6 +10,7 @@ import org.gcube.dataharvest.harvester.JupyterAccessesHarvester;
import org.gcube.dataharvest.utils.AggregationType; import org.gcube.dataharvest.utils.AggregationType;
import org.gcube.dataharvest.utils.ContextTest; import org.gcube.dataharvest.utils.ContextTest;
import org.gcube.dataharvest.utils.DateUtils; import org.gcube.dataharvest.utils.DateUtils;
import org.junit.Ignore;
import org.junit.Test; import org.junit.Test;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
@ -19,15 +20,16 @@ public class AccountingDataHarvesterJupyterTest extends ContextTest {
private static Logger logger = LoggerFactory.getLogger(AccountingDataHarvesterJupyterTest.class); private static Logger logger = LoggerFactory.getLogger(AccountingDataHarvesterJupyterTest.class);
public static final String ROOT = "/d4science.research-infrastructures.eu"; public static final String ROOT = "/d4science.research-infrastructures.eu";
private static final String SCOPE = "/d4science.research-infrastructures.eu/D4OS/Blue-CloudLab"; private static final String SCOPE = "/d4science.research-infrastructures.eu/D4OS/Blue-CloudLab";
@Ignore
@Test @Test
public void testJupyterccessesHarvester() throws Exception { public void testJupyterccessesHarvester() throws Exception {
try { try {
// AccountingDao dao = getAccountingDao(); // AccountingDao dao = getAccountingDao();
List<Date> starts = new ArrayList<>(); List<Date> starts = new ArrayList<>();
starts.add(DateUtils.getStartCalendar(2021, Calendar.JANUARY, 1).getTime()); //starts.add(DateUtils.getStartCalendar(2021, Calendar.JANUARY, 1).getTime());
starts.add(DateUtils.getStartCalendar(2021, Calendar.FEBRUARY, 1).getTime()); //starts.add(DateUtils.getStartCalendar(2021, Calendar.FEBRUARY, 1).getTime());
starts.add(DateUtils.getStartCalendar(2021, Calendar.MARCH, 1).getTime()); starts.add(DateUtils.getStartCalendar(2021, Calendar.MARCH, 1).getTime());
AggregationType measureType = AggregationType.MONTHLY; AggregationType measureType = AggregationType.MONTHLY;

2
src/test/java/org/gcube/dataharvest/utils/ContextAuthorizationTest.java
View File

@ -5,6 +5,7 @@ import java.io.InputStream;
import java.util.Properties; import java.util.Properties;
import org.gcube.dataharvest.AccountingDashboardHarvesterPlugin; import org.gcube.dataharvest.AccountingDashboardHarvesterPlugin;
import org.junit.Ignore;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
@ -26,6 +27,7 @@ public class ContextAuthorizationTest extends ContextTest {
AccountingDashboardHarvesterPlugin.getProperties().set(properties); AccountingDashboardHarvesterPlugin.getProperties().set(properties);
} }
@Ignore
// @Test // @Test
public void testRetrieveContextsAndTokens() throws Exception { public void testRetrieveContextsAndTokens() throws Exception {
try { try {