gCubeSystem
/
accounting-dashboard-harvester-se-plugin
16
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Code redesign 

Refs #11756: Refactor DataHArvesterPlugin to support scheduled execution from smart-executor 

Task-Url: https://support.d4science.org/issues/11756

git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/accounting/accounting-dashboard-harvester-se-plugin@167644 82a268e6-3cf1-43bd-a215-b396298e98cf
This commit is contained in:
Luca Frosini 2018-05-21 13:52:19 +00:00
parent 95284de142
commit b15ba30b1d
6 changed files with 20 additions and 251 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
src/main/java/org/gcube/dataharvest/AccountingDataHarvesterPlugin.java
View File

@ -160,8 +160,7 @@ public class AccountingDataHarvesterPlugin extends Plugin<DataHarvestPluginDecla
try {
// Collecting info on Resource Catalogue (Dataset, Application, Deliverables, Methods)
ResourceCatalogueHarvester resourceCatalogueHarvester = new ResourceCatalogueHarvester(start, end);
resourceCatalogueHarvester.setContexts(contexts);
ResourceCatalogueHarvester resourceCatalogueHarvester = new ResourceCatalogueHarvester(start, end,contexts);
List<HarvestedData> harvested = resourceCatalogueHarvester.getData();
data.addAll(harvested);
} catch(Exception e) {
@ -170,8 +169,7 @@ public class AccountingDataHarvesterPlugin extends Plugin<DataHarvestPluginDecla
try {
// Collecting info on Data/Method download
DataMethodDownloadHarvester dataMethodDownloadHarvester = new DataMethodDownloadHarvester(start, end);
dataMethodDownloadHarvester.setContexts(contexts);
DataMethodDownloadHarvester dataMethodDownloadHarvester = new DataMethodDownloadHarvester(start, end,contexts);;
List<HarvestedData> harvested = dataMethodDownloadHarvester.getData();
data.addAll(harvested);
} catch(Exception e) {

5
src/main/java/org/gcube/dataharvest/harvester/sobigdata/DataMethodDownloadHarvester.java
View File

@ -5,6 +5,7 @@ import java.util.ArrayList;
import java.util.Calendar;
import java.util.Date;
import java.util.List;
import java.util.SortedSet;
import org.apache.commons.lang.Validate;
import org.gcube.common.homelibrary.home.Home;
@ -32,8 +33,8 @@ public class DataMethodDownloadHarvester extends SoBigDataHarvester {
private int count = 0;
public DataMethodDownloadHarvester(Date start, Date end) throws ParseException {
super(start, end);
public DataMethodDownloadHarvester(Date start, Date end, SortedSet<String> contexts) throws ParseException {
super(start, end, contexts);
}
@Override

5
src/main/java/org/gcube/dataharvest/harvester/sobigdata/ResourceCatalogueHarvester.java
View File

@ -5,6 +5,7 @@ import java.util.ArrayList;
import java.util.Date;
import java.util.Iterator;
import java.util.List;
import java.util.SortedSet;
import org.gcube.dataharvest.datamodel.HarvestedData;
import org.gcube.dataharvest.utils.DateUtils;
@ -31,8 +32,8 @@ public class ResourceCatalogueHarvester extends SoBigDataHarvester {
protected String solrBaseUrl;
public ResourceCatalogueHarvester(Date start, Date end) throws ParseException {
super(start, end);
public ResourceCatalogueHarvester(Date start, Date end, SortedSet<String> contexts) throws ParseException {
super(start, end, contexts);
}
protected String getSolrBaseUrl() {

15
src/main/java/org/gcube/dataharvest/harvester/sobigdata/SoBigDataHarvester.java
View File

@ -26,21 +26,18 @@ public abstract class SoBigDataHarvester extends BasicHarvester {
protected SortedSet<String> excludedContexts;
protected SortedSet<String> contexts;
public SoBigDataHarvester(Date start, Date end) throws ParseException {
public SoBigDataHarvester(Date start, Date end, SortedSet<String> contexts) throws ParseException {
super(start, end);
}
public SortedSet<String> getContexts() {
return contexts;
}
public void setContexts(SortedSet<String> contexts) {
this.excludedContexts = getExcludedContexts();
// Adding trailing slash to SO_BIG_DATA_CONTEXT to avoid to get VO
this.contexts = getSoBigDataContexts(contexts, SO_BIG_DATA_CONTEXT + "/");
logger.trace("Valid contexts are {}", contexts);
}
public SortedSet<String> getFilteredContexts() {
return contexts;
}
public static String SECONDARY_TYPE_FORMAT = "$resource/Profile/SecondaryType/text() eq '%1s'";
public static String NAME_FORMAT = "$resource/Profile/Name/text() eq '%1s'";
@ -64,7 +61,7 @@ public abstract class SoBigDataHarvester extends BasicHarvester {
return res.get(0);
}
public SortedSet<String> getExcludedContexts() {
protected SortedSet<String> getExcludedContexts() {
SortedSet<String> excludedContexts = new TreeSet<>();
GenericResource genericResource = getGenericResource();

10
src/test/java/org/gcube/dataharvest/AccountingDataHarvesterPluginTest.java
View File

@ -109,7 +109,10 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
Date start = DateUtils.getPreviousPeriod(measureType).getTime();
Date end = DateUtils.getEndDateFromStartDate(measureType, start, 1);
ResourceCatalogueHarvester resourceCatalogueHarvester = new ResourceCatalogueHarvester(start, end);
ContextAuthorization contextAuthorization = new ContextAuthorization();
SortedSet<String> contexts = contextAuthorization.getContexts();
ResourceCatalogueHarvester resourceCatalogueHarvester = new ResourceCatalogueHarvester(start, end, contexts);
List<HarvestedData> harvestedData = resourceCatalogueHarvester.getData();
logger.debug("{}", harvestedData);
@ -133,9 +136,8 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
ContextAuthorization contextAuthorization = new ContextAuthorization();
SortedSet<String> contexts = contextAuthorization.getContexts();
ResourceCatalogueHarvester resourceCatalogueHarvester = new ResourceCatalogueHarvester(start, end);
resourceCatalogueHarvester.setContexts(contexts);
SortedSet<String> validContexts = resourceCatalogueHarvester.getContexts();
ResourceCatalogueHarvester resourceCatalogueHarvester = new ResourceCatalogueHarvester(start, end, contexts);
SortedSet<String> validContexts = resourceCatalogueHarvester.getFilteredContexts();
logger.info("{}", validContexts);

230
src/test/java/org/gcube/dataharvest/Harvester.java
View File

@ -1,230 +0,0 @@
package org.gcube.dataharvest;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import org.gcube.dataharvest.dao.Dao;
import org.gcube.dataharvest.dao.DaoException;
import org.gcube.dataharvest.dao.DatabaseConnectionData;
import org.gcube.dataharvest.dao.DatabaseParameterRetriever;
import org.gcube.dataharvest.datamodel.HarvestedData;
import org.gcube.dataharvest.harvester.BasicHarvester;
import org.gcube.dataharvest.harvester.SocialInteractionsHarvester;
import org.gcube.dataharvest.harvester.VREUsersHarvester;
import org.gcube.dataharvest.harvester.sobigdata.DataMethodDownloadHarvester;
import org.gcube.dataharvest.harvester.sobigdata.TagMeMethodInvocationHarvester;
import org.gcube.dataharvest.harvester.sobigdata.ResourceCatalogueHarvester;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* @author Eric Perrone (ISTI - CNR)
* @author Luca Frosini (ISTI - CNR)
*/
public class Harvester {
private static Logger logger = LoggerFactory.getLogger(Harvester.class);
public static final String PARAMETER_FROM = "from";
public static final String PARAMETER_TO = "to";
public static final String TEST = "test";
private boolean testMode = false;
private Date dateFrom, dateTo;
public static void main(String[] args) {
Harvester harvester = new Harvester();
try {
harvester.processParameterArray(args);
SocialInteractionsHarvester socialHarvester = new SocialInteractionsHarvester(harvester.getDateFrom(), harvester.getDateTo());
harvester.runOne(socialHarvester);
// harvester.run();
//ArrayList<Integer> list = harvester.getSubTree(17);
//harvester.createSocialReports(list);
System.out.println("End.");
} catch(Exception e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
public Harvester() {
testMode = false;
}
public void setDateFrom(Date dateFrom) {
this.dateFrom = dateFrom;
}
public void setDateTo(Date dateTo) {
this.dateTo = dateTo;
}
public void setTestMode(boolean testMode) {
this.testMode = testMode;
}
public Date getDateFrom() {
return dateFrom;
}
public Date getDateTo() {
return dateTo;
}
public void runOne(BasicHarvester harvester) {
try {
List<HarvestedData> data = harvester.getData();
if(data != null) {
insertMonthlyData((Date) dateFrom, (Date) dateTo, data);
}
} catch(Exception x) {
logger.error(x.getLocalizedMessage());
}
}
public void runAll() {
try {
// collecting info on VRE users
VREUsersHarvester vreUsersHarvester = new VREUsersHarvester(dateFrom, dateTo);
List<HarvestedData> users = vreUsersHarvester.getData();
insertMonthlyData((Date) dateFrom, (Date) dateTo, users);
} catch(Exception x) {
logger.error(x.getLocalizedMessage());
}
try {
// collecting info on Res. Catalogue (Dataset, Application,
// Deliverables, Methods)
ResourceCatalogueHarvester resourceCatalogueHarvester = new ResourceCatalogueHarvester(dateFrom, dateTo);
List<HarvestedData> res = resourceCatalogueHarvester.getData();
insertMonthlyData((Date) dateFrom, (Date) dateTo, res);
} catch(Exception x) {
logger.error(x.getLocalizedMessage());
}
try {
// collecting info on Data/Method download
DataMethodDownloadHarvester dataMethodDownloadHarvester = new DataMethodDownloadHarvester(dateFrom, dateTo);
List<HarvestedData> res = dataMethodDownloadHarvester.getData();
insertMonthlyData((Date) dateFrom, (Date) dateTo, res);
} catch(Exception x) {
logger.error(x.getLocalizedMessage());
}
try {
// collecting info on social (posts, replies and likes)
SocialInteractionsHarvester socialHarvester = new SocialInteractionsHarvester(dateFrom, dateTo);
List<HarvestedData> res = socialHarvester.getData();
insertMonthlyData((Date) dateFrom, (Date) dateTo, res);
} catch(Exception x) {
logger.error(x.getLocalizedMessage());
}
try {
// collecting info on method invocation
TagMeMethodInvocationHarvester methodInvocationHarvester = new TagMeMethodInvocationHarvester(dateFrom, dateTo);
List<HarvestedData> res = methodInvocationHarvester.getData();
logger.debug("{}", res);
// insertMonthlyData((Date) dateFrom, (Date) dateTo, res);
} catch(Exception x) {
logger.error(x.getLocalizedMessage());
}
}
private void insertMonthlyData(Date from, Date to, List<HarvestedData> data) {
Dao dao = null;
try {
dao = dbConnect();
dao.insertMonthlyMeasure(data, from, to, false);
} catch(Exception e) {
logger.error(e.getLocalizedMessage());
} finally {
if(dao != null) {
try {
dao.disconnect();
} catch(DaoException e) {
logger.error(e.getLocalizedMessage());
}
}
}
}
private Dao dbConnect() throws DaoException {
DatabaseParameterRetriever dde = new DatabaseParameterRetriever();
//dde.setTestMode(testMode);
DatabaseConnectionData dcd = dde.retrieveDatabaseInfo();
Dao dao = new Dao();
dao.init();
dao.connect(dcd.getURI(), dcd.getUser(), dcd.getPassword());
return dao;
}
private void processParameterArray(String[] args) throws Exception {
SimpleDateFormat formatter = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss");
for(int i = 0; i < args.length; i++) {
if(args[i].startsWith(PARAMETER_FROM)) {
String from = getArgValue(args[i]);
dateFrom = formatter.parse(from);
System.out.println(dateFrom.toString());
} else if(args[i].startsWith(PARAMETER_TO)) {
String to = getArgValue(args[i]);
dateTo = formatter.parse(to);
System.out.println(dateTo.toString());
} else if(args[i].toLowerCase().startsWith(TEST)) {
testMode = true;
System.out.println("Test: " + testMode);
}
}
}
private String getArgValue(String arg) throws Exception {
String[] tokens = arg.split("=");
if(tokens.length != 2)
throw new Exception("Argument must be in the format 'name=value'.");
return tokens[1].trim();
}
protected ArrayList<Integer> getSubTree(int root) {
Dao dao = null;
try {
dao = dbConnect();
ArrayList<Integer> subTree = dao.getSubTree(root);
return subTree;
} catch(Exception e) {
logger.error(e.getLocalizedMessage());
return null;
} finally {
if(dao != null) {
try {
dao.disconnect();
} catch(DaoException e) {
logger.error(e.getLocalizedMessage());
}
}
}
}
protected void createSocialReports(ArrayList<Integer> ids) {
Dao dao = null;
try {
dao = dbConnect();
for(Integer contextId : ids)
dao.createSocialReport(contextId, 2018);
} catch(Exception e) {
logger.error(e.getLocalizedMessage());
} finally {
if(dao != null) {
try {
dao.disconnect();
} catch(DaoException e) {
logger.error(e.getLocalizedMessage());
}
}
}
}
}