Added logic to exclude harvesting in some cases

git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/accounting/accounting-dashboard-harvester-se-plugin@169212 82a268e6-3cf1-43bd-a215-b396298e98cf

src/main/java/org/gcube/dataharvest/AccountingDataHarvesterPlugin.java

@@ -3,6 +3,7 @@ package org.gcube.dataharvest;
 import java.io.IOException;
 import java.io.InputStream;
 import java.util.ArrayList;
+import java.util.Calendar;
 import java.util.Date;
 import java.util.List;
 import java.util.Map;
@@ -45,6 +46,7 @@ public class AccountingDataHarvesterPlugin extends Plugin<DataHarvestPluginDecla
 	public static final String GET_VRE_USERS_INPUT_PARAMETER = "getVREUsers";
 	public static final String DRY_RUN_INPUT_PARAMETER = "dryRun";
 
+	public static final String SO_BIG_DATA_VO = "/d4science.research-infrastructures.eu/SoBigData";
 	public static final String SO_BIG_DATA_CATALOGUE_CONTEXT = "/d4science.research-infrastructures.eu/SoBigData/ResourceCatalogue";
 	public static final String TAGME_CONTEXT = "/d4science.research-infrastructures.eu/SoBigData/TagMe";
 
@@ -184,9 +186,13 @@ public class AccountingDataHarvesterPlugin extends Plugin<DataHarvestPluginDecla
 			
 			
 			try {
-				// Collecting Google Analytics Data for VREs Accesses
+				if(context.startsWith(SO_BIG_DATA_VO) && start.before(DateUtils.getStartCalendar(2018, Calendar.APRIL, 1).getTime())) {
-				List<HarvestedData> harvested = vreAccessesHarvester.getData();
+					logger.info("Not Harvesting VREs Accesses for {} from {} to {}", context, DateUtils.format(start), DateUtils.format(end));
-				data.addAll(harvested);
+				} else {
+					// Collecting Google Analytics Data for VREs Accesses
+					List<HarvestedData> harvested = vreAccessesHarvester.getData();
+					data.addAll(harvested);
+				}
 			} catch(Exception e) {
 				logger.error("Error harvesting Social Interactions for {}", context, e);
 			}
@@ -205,7 +211,8 @@ public class AccountingDataHarvesterPlugin extends Plugin<DataHarvestPluginDecla
 				// Collecting info on VRE users
 				if(getVREUsers) {
 					// Harvesting Users only for VREs (not for VO and ROOT which is the sum of the children contexts)
-					if(scopeBean.is(Type.VRE)) {
+					// The VREUsers can be only Harvested for the lst month
+					if(scopeBean.is(Type.VRE) && start.equals(DateUtils.getPreviousPeriod(measureType).getTime())) {
 						VREUsersHarvester vreUsersHarvester = new VREUsersHarvester(start, end);
 						List<HarvestedData> harvested = vreUsersHarvester.getData();
 						data.addAll(harvested);

src/main/java/org/gcube/dataharvest/harvester/VREAccessesHarvester.java

@@ -93,17 +93,16 @@ public class VREAccessesHarvester extends BasicHarvester {
 			
 			ScopeBean scopeBean = new ScopeBean(context);
 			String lowerCasedContext = scopeBean.name().toLowerCase();
-			String case1=lowerCasedContext+"/";
+			String case1 = lowerCasedContext + "/";
-			String case2=lowerCasedContext+"?";
+			String case2 = lowerCasedContext + "?";
-			for (VREAccessesReportRow row : vreAccesses) {
+			for(VREAccessesReportRow row : vreAccesses) {
 				String pagePath = row.getPagePath();
-				if (!pagePath.contains("_redirect=/group")) {
+				if(!pagePath.contains("_redirect=/group")) {
-					if (pagePath.endsWith(lowerCasedContext)) {
+					if(pagePath.endsWith(lowerCasedContext)) {
-						System.out.println("match end->"+pagePath);
+						logger.trace("Matched endsWith({}) : {}", lowerCasedContext, pagePath);
 						measure++;
-					} 
+					} else if(pagePath.contains(case1) || pagePath.contains(case2)) {
-					else if (pagePath.contains(case1) || pagePath.contains(case2) ) {
+						logger.trace("Matched contains({}) || contains({}) : {}", case1, case2, pagePath);
-						System.out.println("match compare->"+pagePath);
 						measure++;
 					}
 				}
@@ -127,23 +126,23 @@ public class VREAccessesHarvester extends BasicHarvester {
 	 */
 	private static List<VREAccessesReportRow> getAllAccesses(Date start, Date end) throws Exception {
 		DateRange dateRange = getDateRangeForAnalytics(start, end);
-		System.out.println("getting accesses in this time range: " + dateRange.toPrettyString());
+		logger.trace("Getting accesses in this time range {}", dateRange.toPrettyString());
 		
 		AnalyticsReportCredentials credentialsFromD4S = getAuthorisedApplicationInfoFromIs();
 		AnalyticsReporting service = initializeAnalyticsReporting(credentialsFromD4S);
-		HashMap<String, GetReportsResponse> responses = getReportResponses(service, credentialsFromD4S.getViewIds(), dateRange);
+		HashMap<String,GetReportsResponse> responses = getReportResponses(service, credentialsFromD4S.getViewIds(),
+				dateRange);
 		List<VREAccessesReportRow> totalAccesses = new ArrayList<>();
 		
-		for (String view : responses.keySet()) {
+		for(String view : responses.keySet()) {
 			List<VREAccessesReportRow> viewReport = parseResponse(view, responses.get(view));
-			System.out.println("got " + viewReport.size() + " entries from view id= "+view);
+			logger.trace("Got {} entries from view id={}", viewReport.size(), view);
 			totalAccesses.addAll(viewReport);
 		}
-		System.out.println("Merged in " + totalAccesses.size() + " toal entries from all views");
+		logger.trace("Merged in {} total entries from all views", totalAccesses.size());
 		return totalAccesses;
 	}
 	
-
 	/**
 	 * Initializes an Analytics Reporting API V4 service object.
 	 *
@@ -151,7 +150,8 @@ public class VREAccessesHarvester extends BasicHarvester {
 	 * @throws IOException
 	 * @throws GeneralSecurityException
 	 */
-	private static AnalyticsReporting initializeAnalyticsReporting(AnalyticsReportCredentials cred) throws GeneralSecurityException, IOException {
+	private static AnalyticsReporting initializeAnalyticsReporting(AnalyticsReportCredentials cred)
+			throws GeneralSecurityException, IOException {
 		HttpTransport httpTransport = GoogleNetHttpTransport.newTrustedTransport();
 		GoogleCredential credential = fromD4SServiceEndpoint(cred).createScoped(AnalyticsReportingScopes.all());
 		
@@ -167,31 +167,26 @@ public class VREAccessesHarvester extends BasicHarvester {
 	 * @return GetReportResponse The Analytics Reporting API V4 response.
 	 * @throws IOException
 	 */
-	private static HashMap<String, GetReportsResponse> getReportResponses(AnalyticsReporting service, List<String> viewIDs, DateRange dateRange) throws IOException {
+	private static HashMap<String,GetReportsResponse> getReportResponses(AnalyticsReporting service,
+			List<String> viewIDs, DateRange dateRange) throws IOException {
 		
-		HashMap<String, GetReportsResponse> reports = new HashMap<>();
+		HashMap<String,GetReportsResponse> reports = new HashMap<>();
 		
 		// Create the Metrics object.
-		Metric sessions = new Metric()
+		Metric sessions = new Metric().setExpression("ga:pageviews").setAlias("pages");
-				.setExpression("ga:pageviews")
-				.setAlias("pages");
 		Dimension pageTitle = new Dimension().setName("ga:pagePath");
 		
-		for (String view : viewIDs) {
+		for(String view : viewIDs) {
-			logger.info("Getting data from Google Analytics for viewid: "+ view);
+			logger.info("Getting data from Google Analytics for viewid: " + view);
 			// Create the ReportRequest object.
-			ReportRequest request = new ReportRequest()
+			ReportRequest request = new ReportRequest().setViewId(view).setDateRanges(Arrays.asList(dateRange))
-					.setViewId(view)
+					.setMetrics(Arrays.asList(sessions)).setDimensions(Arrays.asList(pageTitle));
-					.setDateRanges(Arrays.asList(dateRange))
-					.setMetrics(Arrays.asList(sessions))
-					.setDimensions(Arrays.asList(pageTitle));
 			
 			ArrayList<ReportRequest> requests = new ArrayList<ReportRequest>();
 			requests.add(request);
 			
 			// Create the GetReportsRequest object.
-			GetReportsRequest getReport = new GetReportsRequest()
+			GetReportsRequest getReport = new GetReportsRequest().setReportRequests(requests);
-					.setReportRequests(requests);
 			
 			// Call the batchGet method.
 			GetReportsResponse response = service.reports().batchGet(getReport).execute();
@@ -207,38 +202,37 @@ public class VREAccessesHarvester extends BasicHarvester {
 	 * @param response An Analytics Reporting API V4 response.
 	 */
 	private static List<VREAccessesReportRow> parseResponse(String viewId, GetReportsResponse response) {
-		System.out.println("\n*** parsing Response for " + viewId);
+		logger.trace("\n*** parsing Response for {}", viewId);
 		
 		List<VREAccessesReportRow> toReturn = new ArrayList<>();
 		
-		for (Report report: response.getReports()) {
+		for(Report report : response.getReports()) {
 			ColumnHeader header = report.getColumnHeader();
 			List<String> dimensionHeaders = header.getDimensions();
 			List<MetricHeaderEntry> metricHeaders = header.getMetricHeader().getMetricHeaderEntries();
 			List<ReportRow> rows = report.getData().getRows();
 			
-			if (rows == null) {
+			if(rows == null) {
 				logger.warn("No data found for " + viewId);
-			}
+			} else
-			else
+				for(ReportRow row : rows) {
-				for (ReportRow row: rows) {
 					List<String> dimensions = row.getDimensions();
 					List<DateRangeValues> metrics = row.getMetrics();
 					
 					VREAccessesReportRow var = new VREAccessesReportRow();
 					boolean validEntry = false;
-					for (int i = 0; i < dimensionHeaders.size() && i < dimensions.size(); i++) {
+					for(int i = 0; i < dimensionHeaders.size() && i < dimensions.size(); i++) {
-						//System.out.println(dimensionHeaders.get(i) + ": " + dimensions.get(i));
+						//logger.trace("{} : {}", dimensionHeaders.get(i), dimensions.get(i));
 						String pagePath = dimensions.get(i);
-						if (pagePath.startsWith("/group") || pagePath.startsWith("/web")) {
+						if(pagePath.startsWith("/group") || pagePath.startsWith("/web")) {
 							var.setPagePath(dimensions.get(i));
 							validEntry = true;
 						}
 					}
-					if (validEntry) {
+					if(validEntry) {
-						for (int j = 0; j < metrics.size(); j++) {
+						for(int j = 0; j < metrics.size(); j++) {
 							DateRangeValues values = metrics.get(j);
-							for (int k = 0; k < values.getValues().size() && k < metricHeaders.size(); k++) {
+							for(int k = 0; k < values.getValues().size() && k < metricHeaders.size(); k++) {
 								var.setVisitNumber(Integer.parseInt(values.getValues().get(k)));
 							}
 						}
@@ -249,6 +243,7 @@ public class VREAccessesHarvester extends BasicHarvester {
 		}
 		return toReturn;
 	}
+	
 	private static GoogleCredential fromD4SServiceEndpoint(AnalyticsReportCredentials cred) throws IOException {
 		
 		String clientId = cred.getClientId();
@@ -258,8 +253,7 @@ public class VREAccessesHarvester extends BasicHarvester {
 		String tokenUri = cred.getTokenUri();
 		String projectId = cred.getProjectId();
 		
-		if (clientId == null || clientEmail == null || privateKeyPem == null
+		if(clientId == null || clientEmail == null || privateKeyPem == null || privateKeyId == null) {
-				|| privateKeyId == null) {
 			throw new IOException("Error reading service account credential from stream, "
 					+ "expecting  'client_id', 'client_email', 'private_key' and 'private_key_id'.");
 		}
@@ -268,19 +262,16 @@ public class VREAccessesHarvester extends BasicHarvester {
 		
 		Collection<String> emptyScopes = Collections.emptyList();
 		
-		Builder credentialBuilder = new GoogleCredential.Builder()
+		Builder credentialBuilder = new GoogleCredential.Builder().setTransport(Utils.getDefaultTransport())
-				.setTransport( Utils.getDefaultTransport())
+				.setJsonFactory(Utils.getDefaultJsonFactory()).setServiceAccountId(clientEmail)
-				.setJsonFactory(Utils.getDefaultJsonFactory())
+				.setServiceAccountScopes(emptyScopes).setServiceAccountPrivateKey(privateKey)
-				.setServiceAccountId(clientEmail)
-				.setServiceAccountScopes(emptyScopes)
-				.setServiceAccountPrivateKey(privateKey)
 				.setServiceAccountPrivateKeyId(privateKeyId);
 		
-		if (tokenUri != null) {
+		if(tokenUri != null) {
 			credentialBuilder.setTokenServerEncodedUrl(tokenUri);
 		}
 		
-		if (projectId != null) {
+		if(projectId != null) {
 			credentialBuilder.setServiceAccountProjectId(projectId);
 		}
 		
@@ -291,7 +282,7 @@ public class VREAccessesHarvester extends BasicHarvester {
 	private static PrivateKey privateKeyFromPkcs8(String privateKeyPem) throws IOException {
 		Reader reader = new StringReader(privateKeyPem);
 		Section section = PemReader.readFirstSectionAndClose(reader, "PRIVATE KEY");
-		if (section == null) {
+		if(section == null) {
 			throw new IOException("Invalid PKCS8 data.");
 		}
 		byte[] bytes = section.getBase64DecodedBytes();
@@ -301,26 +292,28 @@ public class VREAccessesHarvester extends BasicHarvester {
 			KeyFactory keyFactory = SecurityUtils.getRsaKeyFactory();
 			PrivateKey privateKey = keyFactory.generatePrivate(keySpec);
 			return privateKey;
-		} catch (NoSuchAlgorithmException exception) {
+		} catch(NoSuchAlgorithmException exception) {
 			unexpectedException = exception;
-		} catch (InvalidKeySpecException exception) {
+		} catch(InvalidKeySpecException exception) {
 			unexpectedException = exception;
 		}
 		throw new IOException("Unexpected exception reading PKCS data", unexpectedException);
 	}
 	
-	private static List<ServiceEndpoint> getAnalyticsReportingConfigurationFromIS(String infrastructureScope) throws Exception  {
+	private static List<ServiceEndpoint> getAnalyticsReportingConfigurationFromIS(String infrastructureScope)
+			throws Exception {
 		String scope = infrastructureScope;
-		String currScope = 	ScopeProvider.instance.get();
+		String currScope = ScopeProvider.instance.get();
 		ScopeProvider.instance.set(scope);
 		SimpleQuery query = queryFor(ServiceEndpoint.class);
-		query.addCondition("$resource/Profile/Category/text() eq '"+ SERVICE_ENDPOINT_CATEGORY +"'");
+		query.addCondition("$resource/Profile/Category/text() eq '" + SERVICE_ENDPOINT_CATEGORY + "'");
-		query.addCondition("$resource/Profile/Name/text() eq '"+ SERVICE_ENDPOINT_NAME +"'");
+		query.addCondition("$resource/Profile/Name/text() eq '" + SERVICE_ENDPOINT_NAME + "'");
 		DiscoveryClient<ServiceEndpoint> client = clientFor(ServiceEndpoint.class);
 		List<ServiceEndpoint> toReturn = client.submit(query);
 		ScopeProvider.instance.set(currScope);
 		return toReturn;
 	}
+	
 	/**
 	 * l
 	 * @throws Exception 
@@ -331,48 +324,50 @@ public class VREAccessesHarvester extends BasicHarvester {
 		String context = org.gcube.dataharvest.utils.Utils.getCurrentContext();
 		try {
 			List<ServiceEndpoint> list = getAnalyticsReportingConfigurationFromIS(context);
-			if (list.size() > 1) {
+			if(list.size() > 1) {
-				logger.error("Too many Service Endpoints having name " + SERVICE_ENDPOINT_NAME +" in this scope having Category " + SERVICE_ENDPOINT_CATEGORY);
+				logger.error("Too many Service Endpoints having name " + SERVICE_ENDPOINT_NAME
-			}
+						+ " in this scope having Category " + SERVICE_ENDPOINT_CATEGORY);
-			else if (list.size() == 0){
+			} else if(list.size() == 0) {
-				logger.warn("There is no Service Endpoint having name " + SERVICE_ENDPOINT_NAME +" and Category " + SERVICE_ENDPOINT_CATEGORY + " in this context: " + context);
+				logger.warn("There is no Service Endpoint having name " + SERVICE_ENDPOINT_NAME + " and Category "
-			}
+						+ SERVICE_ENDPOINT_CATEGORY + " in this context: " + context);
-			else {
+			} else {
 				
-				for (ServiceEndpoint res : list) {
+				for(ServiceEndpoint res : list) {
 					reportCredentials.setTokenUri(res.profile().runtime().hostedOn());
-					Group<AccessPoint> apGroup =  res.profile().accessPoints();
+					Group<AccessPoint> apGroup = res.profile().accessPoints();
 					AccessPoint[] accessPoints = (AccessPoint[]) apGroup.toArray(new AccessPoint[apGroup.size()]);
 					AccessPoint found = accessPoints[0];
 					reportCredentials.setClientEmail(found.address());
 					reportCredentials.setProjectId(found.username());
 					reportCredentials.setPrivateKeyPem(StringEncrypter.getEncrypter().decrypt(found.password()));
-					for (Property prop : found.properties()) {
+					for(Property prop : found.properties()) {
-						if (prop.name().compareTo(AP_VIEWS_PROPERTY) == 0) {
+						if(prop.name().compareTo(AP_VIEWS_PROPERTY) == 0) {
 							String decryptedValue = StringEncrypter.getEncrypter().decrypt(prop.value());
 							String[] views = decryptedValue.split(";");
 							reportCredentials.setViewIds(Arrays.asList(views));
 						}
-						if (prop.name().compareTo(AP_CLIENT_PROPERTY) == 0) {
+						if(prop.name().compareTo(AP_CLIENT_PROPERTY) == 0) {
 							String decryptedValue = StringEncrypter.getEncrypter().decrypt(prop.value());
 							reportCredentials.setClientId(decryptedValue);
 						}
-						if (prop.name().compareTo(AP_PRIVATEKEY_PROPERTY) == 0) {
+						if(prop.name().compareTo(AP_PRIVATEKEY_PROPERTY) == 0) {
 							String decryptedValue = StringEncrypter.getEncrypter().decrypt(prop.value());
 							reportCredentials.setPrivateKeyId(decryptedValue);
 						}
 					}
 				}
 			}
-		} catch (Exception e) {
+		} catch(Exception e) {
 			e.printStackTrace();
 			return null;
 		}
 		return reportCredentials;
 	}
+	
 	private static LocalDate asLocalDate(Date date) {
 		return Instant.ofEpochMilli(date.getTime()).atZone(ZoneId.systemDefault()).toLocalDate();
 	}
+	
 	private static DateRange getDateRangeForAnalytics(Date start, Date end) {
 		DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd"); //required by Analytics
 		String startDate = asLocalDate(start).format(formatter);

src/test/java/org/gcube/dataharvest/AccountingDataHarvesterPluginTest.java

@@ -1,6 +1,7 @@
 package org.gcube.dataharvest;
 
 import java.util.ArrayList;
+import java.util.Calendar;
 import java.util.Date;
 import java.util.HashMap;
 import java.util.LinkedHashMap;
@@ -85,10 +86,11 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
 	
 	@Test
 	public void testScopeBean() throws Exception {
+		org.gcube.dataharvest.utils.Utils.setContext(ROOT);
 		SortedSet<String> contexts = getContexts();
 		for(String context : contexts) {
 			ScopeBean scopeBean = new ScopeBean(context);
-			logger.debug("Name {}, FullName {}", scopeBean.name(), scopeBean.toString());
+			logger.debug("FullName {} - Name {}", scopeBean.toString(), scopeBean.name());
 		}
 		
 	}
@@ -147,9 +149,13 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
 				}
 				
 				try {
-					// Collecting Google Analytics Data for VREs Accesses
+					if(context.startsWith(AccountingDataHarvesterPlugin.SO_BIG_DATA_VO) && start.before(DateUtils.getStartCalendar(2018, Calendar.APRIL, 1).getTime())) {
-					List<HarvestedData> harvested = vreAccessesHarvester.getData();
+						logger.info("Not Harvesting VREs Accesses for {} from {} to {}", context, DateUtils.format(start), DateUtils.format(end));
-					data.addAll(harvested);
+					} else {
+						// Collecting Google Analytics Data for VREs Accesses
+						List<HarvestedData> harvested = vreAccessesHarvester.getData();
+						data.addAll(harvested);
+					}
 				} catch(Exception e) {
 					logger.error("Error harvesting Social Interactions for {}", context, e);
 				}