package org.gcube.data.spd.executor.jobs.csv; import static org.gcube.data.streams.dsl.Streams.convert; import java.io.File; import java.io.FileWriter; import java.util.Calendar; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.UUID; import net.sf.csv4j.CSVWriter; import org.gcube.contentmanagement.blobstorage.service.IClient; import org.gcube.contentmanager.storageclient.wrapper.AccessType; import org.gcube.contentmanager.storageclient.wrapper.StorageClient; import org.gcube.data.spd.Constants; import org.gcube.data.spd.executor.jobs.URLJob; import org.gcube.data.spd.model.products.OccurrencePoint; import org.gcube.data.spd.model.products.TaxonomyItem; import org.gcube.data.spd.model.service.types.CompleteJobStatus; import org.gcube.data.spd.model.service.types.JobStatus; import org.gcube.data.spd.plugin.fwk.AbstractPlugin; import org.gcube.data.spd.plugin.fwk.readers.LocalReader; import org.gcube.data.spd.plugin.fwk.writers.Writer; import org.gcube.data.spd.plugin.fwk.writers.rswrapper.LocalWrapper; import org.gcube.data.spd.utils.DynamicMap; import org.gcube.data.streams.Stream; import org.slf4j.Logger; import org.slf4j.LoggerFactory; abstract class CSVJob implements URLJob{ /** * */ private static final long serialVersionUID = 1L; private static Logger logger = LoggerFactory.getLogger(CSVJob.class); private int completedEntries = 0; private String resultURL = null; private Calendar endDate, startDate; private JobStatus status; private String id; private Map mapSubJobs; private Map plugins; public CSVJob(Map plugins) { this.mapSubJobs = new HashMap(); this.id = UUID.randomUUID().toString(); this.status = JobStatus.PENDING; this.plugins = plugins; } @Override public void run() { File csvFile = null; try{ this.startDate = Calendar.getInstance(); this.status = JobStatus.RUNNING; csvFile = File.createTempFile(this.id.replace("-", ""), ".csv"); logger.trace("outputfile "+csvFile.getAbsolutePath()); LocalWrapper localWrapper = new LocalWrapper(1000); localWrapper.forceOpen(); final LocalWrapper errorWrapper = new LocalWrapper(2000); errorWrapper.forceOpen(); Writer errorWriter = new Writer(errorWrapper); errorWriter.register(); Stream ids =convert(DynamicMap.get(this.id)); OccurrenceReaderByKey occurrenceReader =new OccurrenceReaderByKey(localWrapper, ids, plugins); new Thread(occurrenceReader).start(); FileWriter fileWriter = new FileWriter(csvFile); CSVWriter csvWriter = new CSVWriter(fileWriter); csvWriter.writeLine(getHeader()); LocalReader ocReader= new LocalReader(localWrapper); Converter> csvConverter = getConverter(); logger.debug("starting to read from localReader"); while (ocReader.hasNext()){ OccurrencePoint op = ocReader.next(); csvWriter.writeLine(csvConverter.convert(op)); completedEntries++; } if (completedEntries==0) throw new Exception("no record waswritten"); logger.debug("closing file, writing it to the storage"); fileWriter.close(); csvWriter.close(); IClient client = new StorageClient(Constants.SERVICE_CLASS, Constants.SERVICE_NAME, "CSV", AccessType.SHARED).getClient(); String filePath = "/csv/"+this.id.replace("-", "")+".csv"; client.put(true).LFile(csvFile.getAbsolutePath()).RFile(filePath); this.resultURL=client.getUrl().RFile(filePath); logger.debug("job completed"); this.status = JobStatus.COMPLETED; }catch (Exception e) { logger.error("error executing CSVJob",e); this.status = JobStatus.FAILED; return; }finally{ if (csvFile!=null) csvFile.delete(); this.endDate = Calendar.getInstance(); DynamicMap.remove(this.id); } } public JobStatus getStatus() { return status; } public void setStatus(JobStatus status) { this.status = status; } public String getId() { return id; } public Map getMapSubJobs() { return mapSubJobs; } public String getResultURL() { return resultURL; } @Override public String getErrorURL() { // TODO Auto-generated method stub return null; } public abstract Converter> getConverter(); public abstract List getHeader(); @Override public boolean validateInput(String input) { return true; } @Override public int getCompletedEntries() { return completedEntries; } public Calendar getEndDate() { return endDate; } public Calendar getStartDate() { return startDate; } }