package org.gcube.vremanagement; import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.Calendar; import java.util.HashMap; import java.util.List; import java.util.Map; import java.io.File; import java.io.FileOutputStream; import java.io.FileReader; import java.io.FileWriter; import java.io.IOException; import java.io.InputStream; import java.net.URL; import java.net.URLConnection; import com.google.common.collect.ArrayListMultimap; import com.opencsv.CSVParser; import com.opencsv.CSVParserBuilder; import com.opencsv.CSVReader; import com.opencsv.CSVReaderBuilder; import com.opencsv.CSVWriter; import static org.gcube.vremanagement.vremodel.cl.plugin.AbstractPlugin.factory; import static org.gcube.vremanagement.vremodel.cl.plugin.AbstractPlugin.manager; import javax.xml.ws.wsaddressing.W3CEndpointReference; import org.gcube.vremanagement.vremodel.cl.stubs.types.VREDescription; import org.apache.commons.collections.bag.SynchronizedSortedBag; import org.asynchttpclient.AsyncCompletionHandler; import org.asynchttpclient.AsyncHttpClient; import org.asynchttpclient.Dsl; import org.asynchttpclient.HttpResponseBodyPart; import org.asynchttpclient.Response; import org.gcube.common.scope.api.ScopeProvider; import org.gcube.vremanagement.vremodel.cl.stubs.types.Report; import org.gcube.vremanagement.vremodel.cl.stubs.types.VREDescription; import org.gcube.vremanagement.vremodeler.utils.reports.DeployReport; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** * Class that implement the methods to parse the CSV file produced by redmine, * query the remote client to get additional information about the VREs contained in CSV * like start time, end time and description and, finally, store the information in a new CSV file. */ public class VREQueryService implements AutoCloseable{ private File fileCSV; private File outFileCSV; private File missFileCSV; private CSVParser csvParser; private CSVReader csvReader; private CSVWriter csvWriter; private CSVWriter csvMissWriter; private SimpleDateFormat dateFormat; private String OutFilePath; private String MissFilePath; private Map>> mapVres; private List allRows = null; private String[] rowHeader = {"Ticket #", "Status", "Subject","VRE Names", "VO", "Gateways", "Ticket Closed", "Start Time" , "End Time", "Managers", "Description"}; private static final Logger logger = LoggerFactory.getLogger(VREQueryService.class); private static VREQueryService VQSInstance; /** * Private Class constructor that takes as input the path and the name of the CSV file to read, * and the starting rows to skip inside the CSV file * * @param _filePath path of the CSV file produced by redmine to read * @param _InCsvPath name of the CSV file produced by redmine to read * @param _skipLines initial rows to skip in the CSV file */ private VREQueryService (String _filePath,String _InCsvPath, int _skipLines) { try { String InFilePath = _filePath+File.separator+ _InCsvPath; this.OutFilePath = _filePath+File.separator+"updated_"+_InCsvPath; this.MissFilePath =_filePath+File.separator+"missing_"+_InCsvPath; this.fileCSV = new File(InFilePath); if(!this.fileCSV.exists()) { System.err.println("Input CSV File "+_InCsvPath+" does not exists in the path "+_filePath+"!"); System.exit(0); } logger.debug("file url path:" + fileCSV.getPath()); this.csvParser = new CSVParserBuilder().withSeparator(',').withIgnoreQuotations(true).build(); this.csvReader = new CSVReaderBuilder(new FileReader(fileCSV)).withSkipLines(_skipLines).withCSVParser(csvParser).build(); this.allRows = csvReader.readAll(); this.dateFormat = new SimpleDateFormat("MMM dd yyyy hh:mm a"); this.mapVres = new HashMap<>(); } catch(Exception e) { e.printStackTrace();}; } /** * Class builder */ public static class VQSBuilder{ private String filePath; private String csvName; private int toSkip; public VQSBuilder setCSVFiles(String _filePath, String _csvName, int _toSkip) { this.filePath = _filePath; this.csvName= _csvName; this.toSkip = _toSkip; return this; } public VREQueryService build() { return new VREQueryService(filePath,csvName,toSkip); } } /** * Empty class constructor */ public VREQueryService() {} /** * Singleton of the service * @return */ public static synchronized VREQueryService getInstance() { if(VQSInstance !=null) return VQSInstance; return new VREQueryService();}; /** * Close metodo of the AutoCloseable interface */ public void close() throws IOException { if(csvReader !=null) { csvReader.close();} if(csvWriter !=null) { csvWriter.close();} if(csvMissWriter !=null) {csvMissWriter.close();} } /** * Return all the VREs of a specific VO * @param _scopeVO the VO to query * @return list of all the VREs of a specific VO */ public List getReportVREs(String _scopeVO) { try { ScopeProvider.instance.set(_scopeVO); logger.debug("Start getting vre info of VO: "+_scopeVO); List reports = factory().build().getAllVREs(); return reports; }catch(Exception e) { System.err.println("a map for " +_scopeVO+" is undefined"); e.printStackTrace(); return null; } } /** * Method that try to reduce the query number to the remote service. * It save all the VREs of the specified VO in a hash map data structure * @param _scopeList list of VOs * @return hash map with key=VO, value = List of Vres of the VO */ public Map>> fillMapAllVREs(String[] _scopeList) { Map>> vreAllMap = new HashMap<>(); for(String _voItem : _scopeList) { try { if(!vreAllMap.containsKey(_voItem)) { vreAllMap.put(_voItem, new ArrayList<>()); } logger.debug("Adding "+_voItem+" in Map of VREs"); vreAllMap.get(_voItem).add(this.getReportVREs(_voItem)); }catch(IllegalStateException ise) { System.err.println("a map for " +_voItem+" is undefined"); } } return vreAllMap; } /** * Method that find a specific VRE name in a VO * @param _scope * @param _vo * @param _vreName * @return information of the VRE found or null */ public VREDescription findVREDescr(String _scope, String _vo, String _vreName) { String scopeVO = _scope.concat("/").concat(_vo); List reportVREs = this.getReportVREs(scopeVO); if (reportVREs !=null) { for (Report report: reportVREs) { W3CEndpointReference epr = factory().build().getEPRbyId(report.id()); VREDescription vre= manager().at(epr).build().getDescription(); if (report.state().compareTo("Disposed") ==0) { if (vre.name().compareToIgnoreCase(_vreName) == 0) { logger.debug("VRE "+_vreName+ " Found!"); return vre; } } } } logger.debug("VRE " + _vreName+ " Not Found!"); return null; } /** * Method that checks if a VRE is in the CSV file * @param _vreName VRE name to find * @return 0= not fount, 1=found */ public int isVREinCSVbyName(String _vreName) { for (String[] row: allRows) { if(!row[3].isEmpty()) { if(row[3].toString().trim().compareToIgnoreCase(_vreName) == 0) { logger.debug("VRE Found in the CSV...skip"); return 1; } } } return 0; } /* * CSV ROW FORMAT * ROW[0] = ticket * ROW[1] = Status * ROW[2] = Subject * ROW[3] = VREName * ROW[4] = VO * ROW[5] = Gateway * ROW[6] = TicketClosed * ROW[7] = Start Date * ROW[8] = End Date * ROW[9] = Managers * ROM[10] = Description */ /** * Method used to integrate the VRE information contained in the redmine report * with the information taken from the client (start time, end time, managers, description) * It store the new information in a new CSV file. * @param _scope */ public void updateCSVformCSVList(String _scope) { try { outFileCSV = new File (OutFilePath); csvWriter = new CSVWriter (new FileWriter(outFileCSV)); csvWriter.writeNext(rowHeader); for (String[] row: allRows) { if(!(row[3].isEmpty() || row[4].isEmpty())) { VREDescription vreDescr = this.findVREDescr(_scope, row[4], row[3]); if(vreDescr != null) { row[6] = row[6].replace(",", ";")+row[7]; row[7] = dateFormat.format(vreDescr.startTime().getTime()); row[8] = dateFormat.format(vreDescr.endTime().getTime()); row[9] = vreDescr.manager(); row[10] =vreDescr.description(); csvWriter.writeNext(row); } else { row[6] = row[6].replace(",", ";")+row[7]; csvWriter.writeNext(row); } } } csvWriter.close(); } catch (IOException e) { // TODO Auto-generated catch block e.printStackTrace(); } } /* * CSV ROW FORMAT * ROW[0] = ticket * ROW[1] = Status * ROW[2] = Subject * ROW[3] = VREName * ROW[4] = VO * ROW[5] = Gateway * ROW[6] = TicketClosed * ROW[7] = Start Date * ROW[8] = End Date * ROW[9] = Managers * ROW[10] = Description */ /** * Method used to find the missing VRE Disposed in the redmine report across the specified VOs. * It creates a new CSV file. * @param _scope * @param _voList */ public void missingVREinCSV(String _scope, String[] _voList) { try { missFileCSV = new File (MissFilePath); csvMissWriter = new CSVWriter (new FileWriter(missFileCSV)); csvMissWriter.writeNext(rowHeader); for (String voItem : _voList) { String scopeVO = _scope.concat("/").concat(voItem); List reportVREs = this.getReportVREs(scopeVO); for (Report report: reportVREs) { W3CEndpointReference epr = factory().build().getEPRbyId(report.id()); VREDescription vre= manager().at(epr).build().getDescription(); if (report.state().compareTo("Disposed") ==0) { if(this.isVREinCSVbyName(vre.name()) < 1) { logger.debug("VRE "+vre.name()+" not found in the CSV...adding in the missing file"); String[] newRecord = {" ", report.state(), voItem+" / "+vre.name() + " Removal", vre.name(), voItem, " ", " ", dateFormat.format(vre.startTime().getTime()), dateFormat.format(vre.endTime().getTime()), vre.manager(), vre.description() }; csvMissWriter.writeNext(newRecord); } } } } csvMissWriter.close(); } catch (IOException e) { // TODO Auto-generated catch block e.printStackTrace(); } } /** * Method used to retrieve information about a specific VRE of a specific VO * In prints the main information of the VREs if found. * @param _scope * @param _vo VO that contains the VRE to find * @param _vreName VRE name to find * @return 0=not found, 1 found */ public int getVREInfoFromClient(String _scope, String _vo, String _vreName) { try { VREDescription vreFound = this.findVREDescr(_scope, _vo, _vreName); if(vreFound !=null) { logger.debug("VRE Found: " + "name: " + vreFound.name()+"\n" + "vo: "+ _vo +"\n" + "manager: "+ vreFound.manager()+"\n" + "start time:"+ vreFound.startTime().getTime()+"\n" + "end time:"+ vreFound.endTime().getTime()+"\n" ); return 1; } }catch(Exception ise) { System.err.println("a map for " +_scope+ "/"+_vo+" is undefined"); ise.getMessage(); return 0; } logger.debug("VRE Not Found!"); return 0; } /** * Method used to update the end time of a removed VO on the remote service * @param _scope * @param _vo VO that contains the VRE to update * @param _vreName VRE name to update * @param _year end_time year * @param _month end_time month * @param _day end_time day */ public void updateEndTimeVRE(String _scope, String _vo, String _vreName, int _year, int _month, int _day ) { try { Calendar cal = Calendar.getInstance(); cal.set(_year,_month, _day); String scopeVO = _scope.concat("/").concat(_vo); List reportVREs = this.getReportVREs(scopeVO); for (Report report: reportVREs) { W3CEndpointReference epr = factory().build().getEPRbyId(report.id()); VREDescription vre= manager().at(epr).build().getDescription(); if (vre.name().compareToIgnoreCase(_vreName) == 0) { logger.debug("VRE "+vre.name()+ " Found!"); logger.debug("Actual endTime: " +vre.endTime().getTime()); logger.debug("Updating time..."); manager().at(epr).build().setDescription(vre.name(), vre.description(), vre.designer(), vre.manager(), vre.startTime(), cal); break; } } this.getVREInfoFromClient(_scope, _vo, _vreName); }catch(Exception e) { System.err.println("Error in updateTimeVRE "); e.printStackTrace(); } } }