Added logic to exclude harvesting in some cases

git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/accounting/accounting-dashboard-harvester-se-plugin@169212 82a268e6-3cf1-43bd-a215-b396298e98cf

src/main/java/org/gcube/dataharvest/AccountingDataHarvesterPlugin.java

@@ -3,6 +3,7 @@ package org.gcube.dataharvest;
 import java.io.IOException;
 import java.io.InputStream;
 import java.util.ArrayList;
+import java.util.Calendar;
 import java.util.Date;
 import java.util.List;
 import java.util.Map;
@@ -45,6 +46,7 @@ public class AccountingDataHarvesterPlugin extends Plugin<DataHarvestPluginDecla
 	public static final String GET_VRE_USERS_INPUT_PARAMETER = "getVREUsers";
 	public static final String DRY_RUN_INPUT_PARAMETER = "dryRun";
 
+	public static final String SO_BIG_DATA_VO = "/d4science.research-infrastructures.eu/SoBigData";
 	public static final String SO_BIG_DATA_CATALOGUE_CONTEXT = "/d4science.research-infrastructures.eu/SoBigData/ResourceCatalogue";
 	public static final String TAGME_CONTEXT = "/d4science.research-infrastructures.eu/SoBigData/TagMe";
 
@@ -184,9 +186,13 @@ public class AccountingDataHarvesterPlugin extends Plugin<DataHarvestPluginDecla
 			
 			
 			try {
+				if(context.startsWith(SO_BIG_DATA_VO) && start.before(DateUtils.getStartCalendar(2018, Calendar.APRIL, 1).getTime())) {
+					logger.info("Not Harvesting VREs Accesses for {} from {} to {}", context, DateUtils.format(start), DateUtils.format(end));
+				} else {
 					// Collecting Google Analytics Data for VREs Accesses
 					List<HarvestedData> harvested = vreAccessesHarvester.getData();
 					data.addAll(harvested);
+				}
 			} catch(Exception e) {
 				logger.error("Error harvesting Social Interactions for {}", context, e);
 			}
@@ -205,7 +211,8 @@ public class AccountingDataHarvesterPlugin extends Plugin<DataHarvestPluginDecla
 				// Collecting info on VRE users
 				if(getVREUsers) {
 					// Harvesting Users only for VREs (not for VO and ROOT which is the sum of the children contexts)
-					if(scopeBean.is(Type.VRE)) {
+					// The VREUsers can be only Harvested for the lst month
+					if(scopeBean.is(Type.VRE) && start.equals(DateUtils.getPreviousPeriod(measureType).getTime())) {
 						VREUsersHarvester vreUsersHarvester = new VREUsersHarvester(start, end);
 						List<HarvestedData> harvested = vreUsersHarvester.getData();
 						data.addAll(harvested);

src/main/java/org/gcube/dataharvest/harvester/VREAccessesHarvester.java

@@ -99,11 +99,10 @@ public class VREAccessesHarvester extends BasicHarvester {
 				String pagePath = row.getPagePath();
 				if(!pagePath.contains("_redirect=/group")) {
 					if(pagePath.endsWith(lowerCasedContext)) {
-						System.out.println("match end->"+pagePath);
+						logger.trace("Matched endsWith({}) : {}", lowerCasedContext, pagePath);
 						measure++;
-					} 
+					} else if(pagePath.contains(case1) || pagePath.contains(case2)) {
-					else if (pagePath.contains(case1) || pagePath.contains(case2) ) {
+						logger.trace("Matched contains({}) || contains({}) : {}", case1, case2, pagePath);
-						System.out.println("match compare->"+pagePath);
 						measure++;
 					}
 				}
@@ -127,23 +126,23 @@ public class VREAccessesHarvester extends BasicHarvester {
 	 */
 	private static List<VREAccessesReportRow> getAllAccesses(Date start, Date end) throws Exception {
 		DateRange dateRange = getDateRangeForAnalytics(start, end);
-		System.out.println("getting accesses in this time range: " + dateRange.toPrettyString());
+		logger.trace("Getting accesses in this time range {}", dateRange.toPrettyString());
 		
 		AnalyticsReportCredentials credentialsFromD4S = getAuthorisedApplicationInfoFromIs();
 		AnalyticsReporting service = initializeAnalyticsReporting(credentialsFromD4S);
-		HashMap<String, GetReportsResponse> responses = getReportResponses(service, credentialsFromD4S.getViewIds(), dateRange);
+		HashMap<String,GetReportsResponse> responses = getReportResponses(service, credentialsFromD4S.getViewIds(),
+				dateRange);
 		List<VREAccessesReportRow> totalAccesses = new ArrayList<>();
 		
 		for(String view : responses.keySet()) {
 			List<VREAccessesReportRow> viewReport = parseResponse(view, responses.get(view));
-			System.out.println("got " + viewReport.size() + " entries from view id= "+view);
+			logger.trace("Got {} entries from view id={}", viewReport.size(), view);
 			totalAccesses.addAll(viewReport);
 		}
-		System.out.println("Merged in " + totalAccesses.size() + " toal entries from all views");
+		logger.trace("Merged in {} total entries from all views", totalAccesses.size());
 		return totalAccesses;
 	}
 	
-
 	/**
 	 * Initializes an Analytics Reporting API V4 service object.
 	 *
@@ -151,7 +150,8 @@ public class VREAccessesHarvester extends BasicHarvester {
 	 * @throws IOException
 	 * @throws GeneralSecurityException
 	 */
-	private static AnalyticsReporting initializeAnalyticsReporting(AnalyticsReportCredentials cred) throws GeneralSecurityException, IOException {
+	private static AnalyticsReporting initializeAnalyticsReporting(AnalyticsReportCredentials cred)
+			throws GeneralSecurityException, IOException {
 		HttpTransport httpTransport = GoogleNetHttpTransport.newTrustedTransport();
 		GoogleCredential credential = fromD4SServiceEndpoint(cred).createScoped(AnalyticsReportingScopes.all());
 		
@@ -167,31 +167,26 @@ public class VREAccessesHarvester extends BasicHarvester {
 	 * @return GetReportResponse The Analytics Reporting API V4 response.
 	 * @throws IOException
 	 */
-	private static HashMap<String, GetReportsResponse> getReportResponses(AnalyticsReporting service, List<String> viewIDs, DateRange dateRange) throws IOException {
+	private static HashMap<String,GetReportsResponse> getReportResponses(AnalyticsReporting service,
+			List<String> viewIDs, DateRange dateRange) throws IOException {
 		
 		HashMap<String,GetReportsResponse> reports = new HashMap<>();
 		
 		// Create the Metrics object.
-		Metric sessions = new Metric()
+		Metric sessions = new Metric().setExpression("ga:pageviews").setAlias("pages");
-				.setExpression("ga:pageviews")
-				.setAlias("pages");
 		Dimension pageTitle = new Dimension().setName("ga:pagePath");
 		
 		for(String view : viewIDs) {
 			logger.info("Getting data from Google Analytics for viewid: " + view);
 			// Create the ReportRequest object.
-			ReportRequest request = new ReportRequest()
+			ReportRequest request = new ReportRequest().setViewId(view).setDateRanges(Arrays.asList(dateRange))
-					.setViewId(view)
+					.setMetrics(Arrays.asList(sessions)).setDimensions(Arrays.asList(pageTitle));
-					.setDateRanges(Arrays.asList(dateRange))
-					.setMetrics(Arrays.asList(sessions))
-					.setDimensions(Arrays.asList(pageTitle));
 			
 			ArrayList<ReportRequest> requests = new ArrayList<ReportRequest>();
 			requests.add(request);
 			
 			// Create the GetReportsRequest object.
-			GetReportsRequest getReport = new GetReportsRequest()
+			GetReportsRequest getReport = new GetReportsRequest().setReportRequests(requests);
-					.setReportRequests(requests);
 			
 			// Call the batchGet method.
 			GetReportsResponse response = service.reports().batchGet(getReport).execute();
@@ -207,7 +202,7 @@ public class VREAccessesHarvester extends BasicHarvester {
 	 * @param response An Analytics Reporting API V4 response.
 	 */
 	private static List<VREAccessesReportRow> parseResponse(String viewId, GetReportsResponse response) {
-		System.out.println("\n*** parsing Response for " + viewId);
+		logger.trace("\n*** parsing Response for {}", viewId);
 		
 		List<VREAccessesReportRow> toReturn = new ArrayList<>();
 		
@@ -219,8 +214,7 @@ public class VREAccessesHarvester extends BasicHarvester {
 			
 			if(rows == null) {
 				logger.warn("No data found for " + viewId);
-			}
+			} else
-			else
 				for(ReportRow row : rows) {
 					List<String> dimensions = row.getDimensions();
 					List<DateRangeValues> metrics = row.getMetrics();
@@ -228,7 +222,7 @@ public class VREAccessesHarvester extends BasicHarvester {
 					VREAccessesReportRow var = new VREAccessesReportRow();
 					boolean validEntry = false;
 					for(int i = 0; i < dimensionHeaders.size() && i < dimensions.size(); i++) {
-						//System.out.println(dimensionHeaders.get(i) + ": " + dimensions.get(i));
+						//logger.trace("{} : {}", dimensionHeaders.get(i), dimensions.get(i));
 						String pagePath = dimensions.get(i);
 						if(pagePath.startsWith("/group") || pagePath.startsWith("/web")) {
 							var.setPagePath(dimensions.get(i));
@@ -249,6 +243,7 @@ public class VREAccessesHarvester extends BasicHarvester {
 		}
 		return toReturn;
 	}
+	
 	private static GoogleCredential fromD4SServiceEndpoint(AnalyticsReportCredentials cred) throws IOException {
 		
 		String clientId = cred.getClientId();
@@ -258,8 +253,7 @@ public class VREAccessesHarvester extends BasicHarvester {
 		String tokenUri = cred.getTokenUri();
 		String projectId = cred.getProjectId();
 		
-		if (clientId == null || clientEmail == null || privateKeyPem == null
+		if(clientId == null || clientEmail == null || privateKeyPem == null || privateKeyId == null) {
-				|| privateKeyId == null) {
 			throw new IOException("Error reading service account credential from stream, "
 					+ "expecting  'client_id', 'client_email', 'private_key' and 'private_key_id'.");
 		}
@@ -268,12 +262,9 @@ public class VREAccessesHarvester extends BasicHarvester {
 		
 		Collection<String> emptyScopes = Collections.emptyList();
 		
-		Builder credentialBuilder = new GoogleCredential.Builder()
+		Builder credentialBuilder = new GoogleCredential.Builder().setTransport(Utils.getDefaultTransport())
-				.setTransport( Utils.getDefaultTransport())
+				.setJsonFactory(Utils.getDefaultJsonFactory()).setServiceAccountId(clientEmail)
-				.setJsonFactory(Utils.getDefaultJsonFactory())
+				.setServiceAccountScopes(emptyScopes).setServiceAccountPrivateKey(privateKey)
-				.setServiceAccountId(clientEmail)
-				.setServiceAccountScopes(emptyScopes)
-				.setServiceAccountPrivateKey(privateKey)
 				.setServiceAccountPrivateKeyId(privateKeyId);
 		
 		if(tokenUri != null) {
@@ -309,7 +300,8 @@ public class VREAccessesHarvester extends BasicHarvester {
 		throw new IOException("Unexpected exception reading PKCS data", unexpectedException);
 	}
 	
-	private static List<ServiceEndpoint> getAnalyticsReportingConfigurationFromIS(String infrastructureScope) throws Exception  {
+	private static List<ServiceEndpoint> getAnalyticsReportingConfigurationFromIS(String infrastructureScope)
+			throws Exception {
 		String scope = infrastructureScope;
 		String currScope = ScopeProvider.instance.get();
 		ScopeProvider.instance.set(scope);
@@ -321,6 +313,7 @@ public class VREAccessesHarvester extends BasicHarvester {
 		ScopeProvider.instance.set(currScope);
 		return toReturn;
 	}
+	
 	/**
 	 * l
 	 * @throws Exception 
@@ -332,12 +325,12 @@ public class VREAccessesHarvester extends BasicHarvester {
 		try {
 			List<ServiceEndpoint> list = getAnalyticsReportingConfigurationFromIS(context);
 			if(list.size() > 1) {
-				logger.error("Too many Service Endpoints having name " + SERVICE_ENDPOINT_NAME +" in this scope having Category " + SERVICE_ENDPOINT_CATEGORY);
+				logger.error("Too many Service Endpoints having name " + SERVICE_ENDPOINT_NAME
-			}
+						+ " in this scope having Category " + SERVICE_ENDPOINT_CATEGORY);
-			else if (list.size() == 0){
+			} else if(list.size() == 0) {
-				logger.warn("There is no Service Endpoint having name " + SERVICE_ENDPOINT_NAME +" and Category " + SERVICE_ENDPOINT_CATEGORY + " in this context: " + context);
+				logger.warn("There is no Service Endpoint having name " + SERVICE_ENDPOINT_NAME + " and Category "
-			}
+						+ SERVICE_ENDPOINT_CATEGORY + " in this context: " + context);
-			else {
+			} else {
 				
 				for(ServiceEndpoint res : list) {
 					reportCredentials.setTokenUri(res.profile().runtime().hostedOn());
@@ -370,9 +363,11 @@ public class VREAccessesHarvester extends BasicHarvester {
 		}
 		return reportCredentials;
 	}
+	
 	private static LocalDate asLocalDate(Date date) {
 		return Instant.ofEpochMilli(date.getTime()).atZone(ZoneId.systemDefault()).toLocalDate();
 	}
+	
 	private static DateRange getDateRangeForAnalytics(Date start, Date end) {
 		DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd"); //required by Analytics
 		String startDate = asLocalDate(start).format(formatter);

src/test/java/org/gcube/dataharvest/AccountingDataHarvesterPluginTest.java

@@ -1,6 +1,7 @@
 package org.gcube.dataharvest;
 
 import java.util.ArrayList;
+import java.util.Calendar;
 import java.util.Date;
 import java.util.HashMap;
 import java.util.LinkedHashMap;
@@ -85,10 +86,11 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
 	
 	@Test
 	public void testScopeBean() throws Exception {
+		org.gcube.dataharvest.utils.Utils.setContext(ROOT);
 		SortedSet<String> contexts = getContexts();
 		for(String context : contexts) {
 			ScopeBean scopeBean = new ScopeBean(context);
-			logger.debug("Name {}, FullName {}", scopeBean.name(), scopeBean.toString());
+			logger.debug("FullName {} - Name {}", scopeBean.toString(), scopeBean.name());
 		}
 		
 	}
@@ -147,9 +149,13 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
 				}
 				
 				try {
+					if(context.startsWith(AccountingDataHarvesterPlugin.SO_BIG_DATA_VO) && start.before(DateUtils.getStartCalendar(2018, Calendar.APRIL, 1).getTime())) {
+						logger.info("Not Harvesting VREs Accesses for {} from {} to {}", context, DateUtils.format(start), DateUtils.format(end));
+					} else {
 						// Collecting Google Analytics Data for VREs Accesses
 						List<HarvestedData> harvested = vreAccessesHarvester.getData();
 						data.addAll(harvested);
+					}
 				} catch(Exception e) {
 					logger.error("Error harvesting Social Interactions for {}", context, e);
 				}