package org.gcube.dataharvest; import java.io.IOException; import java.io.InputStream; import java.util.ArrayList; import java.util.Date; import java.util.List; import java.util.Map; import java.util.Properties; import java.util.SortedSet; import org.gcube.dataharvest.dao.DatabaseManager; import org.gcube.dataharvest.datamodel.HarvestedData; import org.gcube.dataharvest.harvester.SocialHarvester; import org.gcube.dataharvest.harvester.VREUsersHarvester; import org.gcube.dataharvest.harvester.sobigdata.DataMethodDownloadHarvester; import org.gcube.dataharvest.harvester.sobigdata.MethodInvocationHarvester; import org.gcube.dataharvest.harvester.sobigdata.ResourceCatalogueHarvester; import org.gcube.dataharvest.utils.ContextAuthorization; import org.gcube.dataharvest.utils.DateUtils; import org.gcube.dataharvest.utils.MeasureType; import org.gcube.dataharvest.utils.Utils; import org.gcube.vremanagement.executor.plugin.Plugin; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** * @author Eric Perrone (ISTI - CNR) * @author Luca Frosini (ISTI - CNR) */ public class AccountingDataHarvesterPlugin extends Plugin { private static Logger logger = LoggerFactory.getLogger(AccountingDataHarvesterPlugin.class); private static final String PROPERTY_FILENAME = "config.properties"; public static final String START_DATE_INPUT_PARAMETER = "startDate"; public static final String MEASURE_TYPE_INPUT_PARAMETER = "measureType"; public static final String RERUN_INPUT_PARAMETER = "reRun"; public static final String DRY_RUN_INPUT_PARAMETER = "dryRun"; public static final String SOBIGDATA_CONTEXT = "/d4science.research-infrastructures.eu/SoBigData"; public static final String SOBIGDATA_EU_CONTEXT = "/d4science.research-infrastructures.eu/gCubeApps/SoBigData.eu"; public static final String TAGME_CONTEXT = "/d4science.research-infrastructures.eu/SoBigData/TagMe"; protected Date start; protected Date end; public AccountingDataHarvesterPlugin(DataHarvestPluginDeclaration pluginDeclaration) { super(pluginDeclaration); } private static final InheritableThreadLocal properties = new InheritableThreadLocal() { @Override protected Properties initialValue() { return new Properties(); } }; public static InheritableThreadLocal getProperties() { return properties; } private void getConfigParameters() throws IOException { try { Properties properties = new Properties(); InputStream input = AccountingDataHarvesterPlugin.class.getClassLoader() .getResourceAsStream(PROPERTY_FILENAME); properties.load(input); getProperties().set(properties); } catch(Exception e) { logger.warn( "Unable to load {} file containing configuration properties. AccountingDataHarvesterPlugin will use defaults", PROPERTY_FILENAME); } } /** {@inheritDoc} */ @Override public void launch(Map inputs) throws Exception { logger.debug("{} is starting", this.getClass().getSimpleName()); if(inputs == null || inputs.isEmpty()) { throw new IllegalArgumentException("The can only be launched providing valid input parameters"); } if(!inputs.containsKey(MEASURE_TYPE_INPUT_PARAMETER)) { throw new IllegalArgumentException("Please set required parameter '" + MEASURE_TYPE_INPUT_PARAMETER + "'"); } MeasureType measureType = MeasureType.valueOf((String) inputs.get(MEASURE_TYPE_INPUT_PARAMETER)); boolean reRun = true; if(inputs.containsKey(RERUN_INPUT_PARAMETER)) { try { reRun = (boolean) inputs.get(RERUN_INPUT_PARAMETER); }catch (Exception e) { throw new IllegalArgumentException("'" + RERUN_INPUT_PARAMETER + "' must be a boolean"); } } boolean dryRun = true; if(inputs.containsKey(DRY_RUN_INPUT_PARAMETER)) { try { dryRun = (boolean) inputs.get(DRY_RUN_INPUT_PARAMETER); }catch (Exception e) { throw new IllegalArgumentException("'" + DRY_RUN_INPUT_PARAMETER + "' must be a boolean"); } } if(inputs.containsKey(START_DATE_INPUT_PARAMETER)) { String startDateString = (String) inputs.get(START_DATE_INPUT_PARAMETER); start = DateUtils.UTC_DATE_FORMAT.parse(startDateString + " " + DateUtils.UTC); } else { start = DateUtils.getPreviousPeriod(measureType).getTime(); } end = DateUtils.getEndDateFromStartDate(measureType, start, 1); logger.debug("Harvesting from {} to {}", DateUtils.LAUNCH_DATE_FORMAT.format(start), DateUtils.LAUNCH_DATE_FORMAT.format(end)); getConfigParameters(); ContextAuthorization contextAuthorization = new ContextAuthorization(); DatabaseManager dbaseManager = new DatabaseManager(); SortedSet contexts = contextAuthorization.getContexts(); ArrayList data = new ArrayList(); for(String context : contexts) { // Setting the token for the context Utils.setContext(contextAuthorization.getTokenForContext(context)); try { // Collecting info on social (posts, replies and likes) SocialHarvester socialHarvester = new SocialHarvester(start, end); List harvested = socialHarvester.getData(); data.addAll(harvested); } catch(Exception e) { logger.error("Error harvesting Scoial Interactions for {}", context, e); } try { // Collecting info on VRE users VREUsersHarvester vreUsersHarvester = new VREUsersHarvester(start, end); List harvested = vreUsersHarvester.getData(); data.addAll(harvested); } catch(Exception e) { logger.error("Error harvesting Context Users for {}", context, e); } try { // Collecting info on Resource Catalogue (Dataset, Application, Deliverables, Methods) ResourceCatalogueHarvester resourceCatalogueHarvester = new ResourceCatalogueHarvester(start, end); List harvested = resourceCatalogueHarvester.getData(); data.addAll(harvested); } catch(Exception e) { logger.error("Error harvesting Resource Catalogue Information for {}", context, e); } if(context.startsWith(SOBIGDATA_CONTEXT) || context.startsWith(SOBIGDATA_EU_CONTEXT)) { try { // Collecting info on Data/Method download DataMethodDownloadHarvester dataMethodDownloadHarvester = new DataMethodDownloadHarvester(start, end); List harvested = dataMethodDownloadHarvester.getData(); data.addAll(harvested); } catch(Exception e) { logger.error("Error harvesting Data Method Download for {}", context, e); } if(context.startsWith(TAGME_CONTEXT)) { try { // Collecting info on method invocation MethodInvocationHarvester methodInvocationHarvester = new MethodInvocationHarvester(start, end); List harvested = methodInvocationHarvester.getData(); data.addAll(harvested); } catch(Exception e) { logger.error("Error harvesting Method Invocations for {}", context, e); } } } } if(!dryRun) { dbaseManager.insertMonthlyData(start, end, data, reRun); } } /** {@inheritDoc} */ @Override protected void onStop() throws Exception { logger.debug("{} is stopping", this.getClass().getSimpleName()); } }