git-svn-id: http://svn.research-infrastructures.eu/public/d4science/gcube/trunk/application/perform-service@176634 82a268e6-3cf1-43bd-a215-b396298e98cf
This commit is contained in:
parent
a02c0b5846
commit
754a15318e
|
@ -22,7 +22,7 @@ import org.slf4j.LoggerFactory;
|
||||||
public class ImporterMonitor implements DMMonitorListener {
|
public class ImporterMonitor implements DMMonitorListener {
|
||||||
|
|
||||||
private static final Logger log= LoggerFactory.getLogger(ImporterMonitor.class);
|
private static final Logger log= LoggerFactory.getLogger(ImporterMonitor.class);
|
||||||
|
|
||||||
public ImporterMonitor(ImportRoutineDescriptor routine, ISQueryDescriptor isQuery) {
|
public ImporterMonitor(ImportRoutineDescriptor routine, ISQueryDescriptor isQuery) {
|
||||||
super();
|
super();
|
||||||
this.routine = routine;
|
this.routine = routine;
|
||||||
|
@ -32,7 +32,7 @@ public class ImporterMonitor implements DMMonitorListener {
|
||||||
|
|
||||||
private ImportRoutineDescriptor routine;
|
private ImportRoutineDescriptor routine;
|
||||||
private ISQueryDescriptor isQuery;
|
private ISQueryDescriptor isQuery;
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void accepted() {
|
public void accepted() {
|
||||||
updateStatus(ImportStatus.ACCEPTED,routine,isQuery);
|
updateStatus(ImportStatus.ACCEPTED,routine,isQuery);
|
||||||
|
@ -59,41 +59,45 @@ public class ImporterMonitor implements DMMonitorListener {
|
||||||
updateStatus(ImportStatus.RUNNING,routine,isQuery);
|
updateStatus(ImportStatus.RUNNING,routine,isQuery);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
private static final void updateStatus(ImportStatus status,ImportRoutineDescriptor routine,ISQueryDescriptor is) {
|
private static final void updateStatus(ImportStatus status,ImportRoutineDescriptor routine,ISQueryDescriptor is) {
|
||||||
try{
|
try{
|
||||||
log.debug("Updateing status {} for {} ",status,routine);
|
log.debug("Updateing status {} for {} ",status,routine);
|
||||||
DataBaseManager db=DataBaseManager.get(is);
|
DataBaseManager db=DataBaseManager.get(is);
|
||||||
Connection conn=db.getConnection();
|
Connection conn=db.getConnection();
|
||||||
conn.setAutoCommit(true);
|
try {
|
||||||
|
conn.setAutoCommit(true);
|
||||||
Instant endTime=null;
|
|
||||||
switch(status) {
|
Instant endTime=null;
|
||||||
case CANCELLED:
|
switch(status) {
|
||||||
case COMPLETE :
|
case CANCELLED:
|
||||||
case FAILED : endTime=Instant.now();
|
case COMPLETE :
|
||||||
}
|
case FAILED : endTime=Instant.now();
|
||||||
|
}
|
||||||
DBQueryDescriptor queryValues=new DBQueryDescriptor().
|
|
||||||
add(DBField.ImportRoutine.fields.get(ImportRoutine.ID), routine.getId()).
|
DBQueryDescriptor queryValues=new DBQueryDescriptor().
|
||||||
add(DBField.ImportRoutine.fields.get(ImportRoutine.STATUS), status.toString()).
|
add(DBField.ImportRoutine.fields.get(ImportRoutine.ID), routine.getId()).
|
||||||
add(DBField.ImportRoutine.fields.get(ImportRoutine.END), endTime);
|
add(DBField.ImportRoutine.fields.get(ImportRoutine.STATUS), status.toString()).
|
||||||
|
add(DBField.ImportRoutine.fields.get(ImportRoutine.END), endTime);
|
||||||
PreparedStatement psUpdate=Queries.UPDATE_IMPORT_STATUS.get(conn, queryValues);
|
|
||||||
psUpdate.executeUpdate();
|
PreparedStatement psUpdate=Queries.UPDATE_IMPORT_STATUS.get(conn, queryValues);
|
||||||
|
psUpdate.executeUpdate();
|
||||||
|
}finally {
|
||||||
|
conn.close();
|
||||||
|
}
|
||||||
}catch(Throwable t) {
|
}catch(Throwable t) {
|
||||||
log.warn("Unable to update status on database");
|
log.warn("Unable to update status on database");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
private static final void loadOutputData(ImportRoutineDescriptor routine, ISQueryDescriptor is) {
|
private static final void loadOutputData(ImportRoutineDescriptor routine, ISQueryDescriptor is) {
|
||||||
log.debug("Loading output data for {} ",routine);
|
log.debug("Loading output data for {} ",routine);
|
||||||
ComputationId id=DMUtils.getComputation(routine);
|
ComputationId id=DMUtils.getComputation(routine);
|
||||||
Map<String,String> outputFiles=DMUtils.getOutputFiles(id);
|
Map<String,String> outputFiles=DMUtils.getOutputFiles(id);
|
||||||
|
|
||||||
// if(outputFiles.containsKey(Local))
|
// if(outputFiles.containsKey(Local))
|
||||||
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -68,9 +68,9 @@ public class DataBaseManagerImpl implements DataBaseManager{
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public Connection getConnection() throws SQLException, InternalException {
|
public Connection getConnection() throws SQLException, InternalException {
|
||||||
DataSource ds=getDataSource();
|
DataSource ds=getDataSource();
|
||||||
Connection conn=ds.getConnection();
|
Connection conn=ds.getConnection();
|
||||||
conn.setAutoCommit(false);
|
conn.setAutoCommit(false);
|
||||||
return conn;
|
return conn;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -78,8 +78,7 @@ public class DataBaseManagerImpl implements DataBaseManager{
|
||||||
private synchronized DataSource getDataSource() throws InternalException {
|
private synchronized DataSource getDataSource() throws InternalException {
|
||||||
DatabaseConnectionDescriptor dbDescriptor=getDB();
|
DatabaseConnectionDescriptor dbDescriptor=getDB();
|
||||||
|
|
||||||
if(!datasources.containsKey(dbDescriptor.getUrl())) {
|
if(!datasources.containsKey(dbDescriptor.getUrl())) {
|
||||||
|
|
||||||
datasources.put(dbDescriptor.getUrl(), setupDataSource(dbDescriptor));
|
datasources.put(dbDescriptor.getUrl(), setupDataSource(dbDescriptor));
|
||||||
}
|
}
|
||||||
return datasources.get(dbDescriptor.getUrl());
|
return datasources.get(dbDescriptor.getUrl());
|
||||||
|
|
|
@ -0,0 +1,68 @@
|
||||||
|
package org.gcube.application.perform.service.engine.impl;
|
||||||
|
|
||||||
|
import java.io.FileReader;
|
||||||
|
import java.io.Reader;
|
||||||
|
import java.sql.PreparedStatement;
|
||||||
|
import java.util.ArrayList;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.Map.Entry;
|
||||||
|
|
||||||
|
import org.apache.commons.csv.CSVFormat;
|
||||||
|
import org.apache.commons.csv.CSVParser;
|
||||||
|
import org.apache.commons.csv.CSVRecord;
|
||||||
|
import org.gcube.application.perform.service.engine.model.DBQueryDescriptor;
|
||||||
|
import org.gcube.application.perform.service.engine.model.importer.ImportRoutineDescriptor;
|
||||||
|
import org.gcube.application.perform.service.engine.model.importer.ImportedTable;
|
||||||
|
import org.slf4j.Logger;
|
||||||
|
import org.slf4j.LoggerFactory;
|
||||||
|
|
||||||
|
public class ImportedTableManager {
|
||||||
|
|
||||||
|
private static final Logger log= LoggerFactory.getLogger(ImportedTableManager.class);
|
||||||
|
|
||||||
|
private static List<ImportedTable> tables;
|
||||||
|
|
||||||
|
|
||||||
|
public void loadImportedData(ImportRoutineDescriptor desc) {
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
private static final long parse(String path, String description, ImportRoutineDescriptor routine) {
|
||||||
|
Reader in = new FileReader(path);
|
||||||
|
CSVParser parser= CSVFormat.DEFAULT.withFirstRecordAsHeader().parse(in);
|
||||||
|
try {
|
||||||
|
log.debug("Parsing file {} : {} ",description,path);
|
||||||
|
// Extract CSV Schema
|
||||||
|
ArrayList<String> csvSchema=new ArrayList<String>();
|
||||||
|
for(Entry<String,Integer> entry : parser.getHeaderMap().entrySet()) {
|
||||||
|
csvSchema.add(entry.getValue(), entry.getKey());
|
||||||
|
}
|
||||||
|
|
||||||
|
log.debug("CSV Schema is {} ",csvSchema);
|
||||||
|
|
||||||
|
long counter=0l;
|
||||||
|
//Get the right table
|
||||||
|
for(ImportedTable table:tables) {
|
||||||
|
if(table.matchesSchema(csvSchema)) {
|
||||||
|
log.debug("Mathing table is {} ",table.getTableName());
|
||||||
|
Query query=table.getQuery();
|
||||||
|
PreparedStatement psInsert=query.prepare(conn);
|
||||||
|
log.debug("Reading csvLines");
|
||||||
|
for(CSVRecord record:parser) {
|
||||||
|
DBQueryDescriptor desc=table.getSetRow(record.toMap(), routine.getId(), routine.getFarmId());
|
||||||
|
query.fill(psInsert, desc);
|
||||||
|
counter+=psInsert.executeUpdate();
|
||||||
|
}
|
||||||
|
log.debug("Inserted {} lines into {} for routine {} [FARM ID {}]",counter,table.getTableName(),routine.getId(),routine.getFarmId());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}finally {
|
||||||
|
parser.close();
|
||||||
|
in.close();
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
}
|
|
@ -39,12 +39,12 @@ import org.slf4j.LoggerFactory;
|
||||||
public class ImporterImpl implements Importer {
|
public class ImporterImpl implements Importer {
|
||||||
|
|
||||||
private static final Logger log= LoggerFactory.getLogger(ImporterImpl.class);
|
private static final Logger log= LoggerFactory.getLogger(ImporterImpl.class);
|
||||||
|
|
||||||
|
|
||||||
private static ISQueryDescriptor isQueryDescriptor=null;
|
private static ISQueryDescriptor isQueryDescriptor=null;
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
private static synchronized ISQueryDescriptor getISQueryDescriptor() {
|
private static synchronized ISQueryDescriptor getISQueryDescriptor() {
|
||||||
if(isQueryDescriptor==null) {
|
if(isQueryDescriptor==null) {
|
||||||
isQueryDescriptor=
|
isQueryDescriptor=
|
||||||
|
@ -54,67 +54,70 @@ public class ImporterImpl implements Importer {
|
||||||
}
|
}
|
||||||
return isQueryDescriptor;
|
return isQueryDescriptor;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
private static final String getHostname() {
|
private static final String getHostname() {
|
||||||
try{
|
try{
|
||||||
ApplicationContext context=ContextProvider.get();
|
ApplicationContext context=ContextProvider.get();
|
||||||
ContainerConfiguration configuration=context.container().configuration();
|
ContainerConfiguration configuration=context.container().configuration();
|
||||||
|
|
||||||
return configuration.hostname();
|
return configuration.hostname();
|
||||||
}catch(Throwable t) {
|
}catch(Throwable t) {
|
||||||
log.warn("UNABLE TO GET HOSTNAME. This should happen only in debug mode.");
|
log.warn("UNABLE TO GET HOSTNAME. This should happen only in debug mode.");
|
||||||
return "localhost";
|
return "localhost";
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
public void init() throws InternalException{
|
public void init() throws InternalException{
|
||||||
try {
|
try {
|
||||||
log.info("Initializing IMPORTER");
|
log.info("Initializing IMPORTER");
|
||||||
DataBaseManager db=DataBaseManager.get(getISQueryDescriptor());
|
DataBaseManager db=DataBaseManager.get(getISQueryDescriptor());
|
||||||
Connection conn=db.getConnection();
|
Connection conn=db.getConnection();
|
||||||
conn.setAutoCommit(true);
|
|
||||||
|
|
||||||
|
|
||||||
PreparedStatement psOrphans=Queries.ORPHAN_IMPORTS.prepare(conn);
|
|
||||||
PreparedStatement psAcquire=Queries.ACQUIRE_IMPORT_ROUTINE.prepare(conn);
|
|
||||||
// set ps
|
|
||||||
ResultSet rsOrphans=psOrphans.executeQuery();
|
|
||||||
long monitoredCount=0l;
|
|
||||||
while(rsOrphans.next()) {
|
|
||||||
Long id=rsOrphans.getLong(ImportRoutine.ID);
|
|
||||||
try {
|
try {
|
||||||
ImportRoutineDescriptor desc=Queries.rowToDescriptor(rsOrphans);
|
conn.setAutoCommit(true);
|
||||||
String hostname=getHostname();
|
PreparedStatement psOrphans=Queries.ORPHAN_IMPORTS.prepare(conn);
|
||||||
DBQueryDescriptor acquireDesc=new DBQueryDescriptor().
|
PreparedStatement psAcquire=Queries.ACQUIRE_IMPORT_ROUTINE.prepare(conn);
|
||||||
add(ImportRoutine.fields.get(ImportRoutine.LOCK), hostname).
|
// set ps
|
||||||
add(ImportRoutine.fields.get(ImportRoutine.ID), id);
|
ResultSet rsOrphans=psOrphans.executeQuery();
|
||||||
|
long monitoredCount=0l;
|
||||||
Queries.ACQUIRE_IMPORT_ROUTINE.fill(psAcquire, acquireDesc);
|
while(rsOrphans.next()) {
|
||||||
|
Long id=rsOrphans.getLong(ImportRoutine.ID);
|
||||||
if(psAcquire.executeUpdate()>0) {
|
try {
|
||||||
log.debug("Acquired {} ",id);
|
ImportRoutineDescriptor desc=Queries.rowToDescriptor(rsOrphans);
|
||||||
// Stored caller token
|
String hostname=getHostname();
|
||||||
log.debug("Setting stored token.. ");
|
DBQueryDescriptor acquireDesc=new DBQueryDescriptor().
|
||||||
SecurityTokenProvider.instance.set(CommonUtils.decryptString(desc.getCaller()));
|
add(ImportRoutine.fields.get(ImportRoutine.LOCK), hostname).
|
||||||
|
add(ImportRoutine.fields.get(ImportRoutine.ID), id);
|
||||||
monitor(desc);
|
|
||||||
monitoredCount++;
|
Queries.ACQUIRE_IMPORT_ROUTINE.fill(psAcquire, acquireDesc);
|
||||||
|
|
||||||
|
if(psAcquire.executeUpdate()>0) {
|
||||||
|
log.debug("Acquired {} ",id);
|
||||||
|
// Stored caller token
|
||||||
|
log.debug("Setting stored token.. ");
|
||||||
|
SecurityTokenProvider.instance.set(CommonUtils.decryptString(desc.getCaller()));
|
||||||
|
|
||||||
|
monitor(desc);
|
||||||
|
monitoredCount++;
|
||||||
|
}
|
||||||
|
}catch(Throwable t) {
|
||||||
|
log.warn("Unable to monitor orphan with ID {} ",id,t);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
log.info("Acquired {} import executions for monitoring",monitoredCount);
|
||||||
|
}finally {
|
||||||
|
conn.close();
|
||||||
}
|
}
|
||||||
}catch(Throwable t) {
|
|
||||||
log.warn("Unable to monitor orphan with ID {} ",id,t);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
log.info("Acquired {} import executions for monitoring",monitoredCount);
|
|
||||||
}catch(Throwable t) {
|
}catch(Throwable t) {
|
||||||
log.warn("Unexpected Error while trying to check orphan import routines");
|
log.warn("Unexpected Error while trying to check orphan import routines");
|
||||||
throw new InternalException(t);
|
throw new InternalException(t);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public ImportRoutineDescriptor importExcel(ImportRequest request) throws DMException, SQLException, InternalException {
|
public ImportRoutineDescriptor importExcel(ImportRequest request) throws DMException, SQLException, InternalException {
|
||||||
log.debug("Submitting {} ",request);
|
log.debug("Submitting {} ",request);
|
||||||
|
@ -125,17 +128,17 @@ public class ImporterImpl implements Importer {
|
||||||
monitor(desc);
|
monitor(desc);
|
||||||
return getDescriptorById(desc.getId());
|
return getDescriptorById(desc.getId());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
private void monitor(ImportRoutineDescriptor desc) throws DMException {
|
private void monitor(ImportRoutineDescriptor desc) throws DMException {
|
||||||
log.debug("Monitoring {} ",desc);
|
log.debug("Monitoring {} ",desc);
|
||||||
DMUtils.monitor(DMUtils.getComputation(desc), new ImporterMonitor(desc,getISQueryDescriptor()));
|
DMUtils.monitor(DMUtils.getComputation(desc), new ImporterMonitor(desc,getISQueryDescriptor()));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
private ComputationId submit(ImportRequest request) throws DMException {
|
private ComputationId submit(ImportRequest request) throws DMException {
|
||||||
/**
|
/**
|
||||||
* dataminer-prototypes.d4science.org/wps/WebProcessingService?
|
* dataminer-prototypes.d4science.org/wps/WebProcessingService?
|
||||||
|
@ -147,17 +150,17 @@ public class ImporterImpl implements Importer {
|
||||||
* BatchType=GROW_OUT_AGGREGATED;
|
* BatchType=GROW_OUT_AGGREGATED;
|
||||||
* FarmID=ID
|
* FarmID=ID
|
||||||
*/
|
*/
|
||||||
|
|
||||||
Map<String,String> parameters=new HashMap<>();
|
Map<String,String> parameters=new HashMap<>();
|
||||||
parameters.put("InputData", request.getSource());
|
parameters.put("InputData", request.getSource());
|
||||||
parameters.put("BatchType", request.getBatchType());
|
parameters.put("BatchType", request.getBatchType());
|
||||||
parameters.put("FarmID", request.getFarmId().toString());
|
parameters.put("FarmID", request.getFarmId().toString());
|
||||||
|
|
||||||
return DMUtils.submitJob(LocalConfiguration.getProperty(LocalConfiguration.IMPORTER_COMPUTATION_ID), parameters);
|
return DMUtils.submitJob(LocalConfiguration.getProperty(LocalConfiguration.IMPORTER_COMPUTATION_ID), parameters);
|
||||||
}
|
}
|
||||||
|
|
||||||
private ImportRoutineDescriptor register(ComputationId computationId,ImportRequest request) throws SQLException, InternalException {
|
private ImportRoutineDescriptor register(ComputationId computationId,ImportRequest request) throws SQLException, InternalException {
|
||||||
|
|
||||||
DBQueryDescriptor insertionRow=new DBQueryDescriptor().
|
DBQueryDescriptor insertionRow=new DBQueryDescriptor().
|
||||||
add(ImportRoutine.fields.get(ImportRoutine.BATCH_TYPE), request.getBatchType()).
|
add(ImportRoutine.fields.get(ImportRoutine.BATCH_TYPE), request.getBatchType()).
|
||||||
add(ImportRoutine.fields.get(ImportRoutine.CALLER), CommonUtils.encryptString(ScopeUtils.getCaller())).
|
add(ImportRoutine.fields.get(ImportRoutine.CALLER), CommonUtils.encryptString(ScopeUtils.getCaller())).
|
||||||
|
@ -172,37 +175,42 @@ public class ImporterImpl implements Importer {
|
||||||
add(ImportRoutine.fields.get(ImportRoutine.SOURCE_VERSION), request.getVersion()).
|
add(ImportRoutine.fields.get(ImportRoutine.SOURCE_VERSION), request.getVersion()).
|
||||||
add(ImportRoutine.fields.get(ImportRoutine.START), java.sql.Timestamp.from(Instant.now())).
|
add(ImportRoutine.fields.get(ImportRoutine.START), java.sql.Timestamp.from(Instant.now())).
|
||||||
add(ImportRoutine.fields.get(ImportRoutine.STATUS),ImportStatus.ACCEPTED.toString());
|
add(ImportRoutine.fields.get(ImportRoutine.STATUS),ImportStatus.ACCEPTED.toString());
|
||||||
|
|
||||||
|
|
||||||
DataBaseManager db=DataBaseManager.get(getISQueryDescriptor());
|
DataBaseManager db=DataBaseManager.get(getISQueryDescriptor());
|
||||||
Connection conn=db.getConnection();
|
Connection conn=db.getConnection();
|
||||||
conn.setAutoCommit(true);
|
conn.setAutoCommit(true);
|
||||||
|
try {
|
||||||
PreparedStatement ps=Queries.INSERT_ROUTINE.prepare(conn,Statement.RETURN_GENERATED_KEYS);
|
PreparedStatement ps=Queries.INSERT_ROUTINE.prepare(conn,Statement.RETURN_GENERATED_KEYS);
|
||||||
Queries.INSERT_ROUTINE.fill(ps, insertionRow);
|
Queries.INSERT_ROUTINE.fill(ps, insertionRow);
|
||||||
ps.executeUpdate();
|
ps.executeUpdate();
|
||||||
|
|
||||||
ResultSet rs=ps.getGeneratedKeys();
|
ResultSet rs=ps.getGeneratedKeys();
|
||||||
rs.next();
|
rs.next();
|
||||||
|
|
||||||
PreparedStatement psGet=Queries.GET_IMPORT_ROUTINE_BY_ID.get(conn,
|
PreparedStatement psGet=Queries.GET_IMPORT_ROUTINE_BY_ID.get(conn,
|
||||||
new DBQueryDescriptor().add(ImportRoutine.fields.get(ImportRoutine.ID), rs.getLong(ImportRoutine.ID)));
|
new DBQueryDescriptor().add(ImportRoutine.fields.get(ImportRoutine.ID), rs.getLong(ImportRoutine.ID)));
|
||||||
ResultSet rsGet=psGet.executeQuery();
|
ResultSet rsGet=psGet.executeQuery();
|
||||||
rsGet.next();
|
rsGet.next();
|
||||||
return Queries.rowToDescriptor(rsGet);
|
return Queries.rowToDescriptor(rsGet);
|
||||||
|
}finally {
|
||||||
|
conn.close();
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private ImportRoutineDescriptor getDescriptorById(Long id) throws SQLException, InternalException {
|
private ImportRoutineDescriptor getDescriptorById(Long id) throws SQLException, InternalException {
|
||||||
DataBaseManager db=DataBaseManager.get(getISQueryDescriptor());
|
DataBaseManager db=DataBaseManager.get(getISQueryDescriptor());
|
||||||
Connection conn=db.getConnection();
|
Connection conn=db.getConnection();
|
||||||
|
try {
|
||||||
PreparedStatement ps=Queries.GET_IMPORT_ROUTINE_BY_ID.get(conn,
|
PreparedStatement ps=Queries.GET_IMPORT_ROUTINE_BY_ID.get(conn,
|
||||||
new DBQueryDescriptor().add(ImportRoutine.fields.get(ImportRoutine.ID), id));
|
new DBQueryDescriptor().add(ImportRoutine.fields.get(ImportRoutine.ID), id));
|
||||||
ps.setLong(1, id);
|
ps.setLong(1, id);
|
||||||
ResultSet rs=ps.executeQuery();
|
ResultSet rs=ps.executeQuery();
|
||||||
if(rs.next()) return Queries.rowToDescriptor(rs);
|
if(rs.next()) return Queries.rowToDescriptor(rs);
|
||||||
else throw new BeanNotFound("Unable to find Routine with ID "+id);
|
else throw new BeanNotFound("Unable to find Routine with ID "+id);
|
||||||
|
}finally {
|
||||||
|
conn.close();
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@ -210,21 +218,24 @@ public class ImporterImpl implements Importer {
|
||||||
public List<ImportRoutineDescriptor> getDescriptors(DBQueryDescriptor desc) throws SQLException, InternalException {
|
public List<ImportRoutineDescriptor> getDescriptors(DBQueryDescriptor desc) throws SQLException, InternalException {
|
||||||
DataBaseManager db=DataBaseManager.get(getISQueryDescriptor());
|
DataBaseManager db=DataBaseManager.get(getISQueryDescriptor());
|
||||||
Connection conn=db.getConnection();
|
Connection conn=db.getConnection();
|
||||||
|
try {
|
||||||
PreparedStatement ps=Queries.FILTER_IMPORTS.get(conn, desc);
|
PreparedStatement ps=Queries.FILTER_IMPORTS.get(conn, desc);
|
||||||
ResultSet rs=ps.executeQuery();
|
ResultSet rs=ps.executeQuery();
|
||||||
ArrayList<ImportRoutineDescriptor> toReturn=new ArrayList<>();
|
ArrayList<ImportRoutineDescriptor> toReturn=new ArrayList<>();
|
||||||
while (rs.next())
|
while (rs.next())
|
||||||
toReturn.add(Queries.rowToDescriptor(rs));
|
toReturn.add(Queries.rowToDescriptor(rs));
|
||||||
|
|
||||||
return toReturn;
|
return toReturn;
|
||||||
}
|
}finally {
|
||||||
|
conn.close();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
|
@ -26,7 +26,7 @@ public class MappingManagerImpl implements MappingManager {
|
||||||
|
|
||||||
private static ISQueryDescriptor isQueryDescriptor=null;
|
private static ISQueryDescriptor isQueryDescriptor=null;
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
private static synchronized ISQueryDescriptor getISQueryDescriptor() {
|
private static synchronized ISQueryDescriptor getISQueryDescriptor() {
|
||||||
if(isQueryDescriptor==null) {
|
if(isQueryDescriptor==null) {
|
||||||
|
@ -43,27 +43,31 @@ public class MappingManagerImpl implements MappingManager {
|
||||||
public Batch getBatch(DBQueryDescriptor desc) throws SQLException, InternalException{
|
public Batch getBatch(DBQueryDescriptor desc) throws SQLException, InternalException{
|
||||||
DataBaseManager db=DataBaseManager.get(getISQueryDescriptor());
|
DataBaseManager db=DataBaseManager.get(getISQueryDescriptor());
|
||||||
Connection conn=db.getConnection();
|
Connection conn=db.getConnection();
|
||||||
conn.setAutoCommit(true);
|
try{
|
||||||
Query getQuery=Queries.GET_BATCH_BY_DESCRIPTIVE_KEY;
|
conn.setAutoCommit(true);
|
||||||
PreparedStatement psSearch=getQuery.get(conn, desc);
|
Query getQuery=Queries.GET_BATCH_BY_DESCRIPTIVE_KEY;
|
||||||
|
PreparedStatement psSearch=getQuery.get(conn, desc);
|
||||||
|
|
||||||
ResultSet rs=psSearch.executeQuery();
|
ResultSet rs=psSearch.executeQuery();
|
||||||
|
|
||||||
if(rs.next())
|
if(rs.next())
|
||||||
return Queries.rowToBatch(rs);
|
return Queries.rowToBatch(rs);
|
||||||
|
|
||||||
// ID NOT FOUND, TRY TO REGISTER IT
|
// ID NOT FOUND, TRY TO REGISTER IT
|
||||||
|
|
||||||
log.trace("Registering new Batch from condition {}",desc);
|
log.trace("Registering new Batch from condition {}",desc);
|
||||||
desc.add(DBField.Batch.fields.get(DBField.Batch.UUID), UUID.randomUUID());
|
desc.add(DBField.Batch.fields.get(DBField.Batch.UUID), UUID.randomUUID());
|
||||||
PreparedStatement psInsert=Queries.INSERT_BATCH.get(conn, desc);
|
PreparedStatement psInsert=Queries.INSERT_BATCH.get(conn, desc);
|
||||||
|
|
||||||
psInsert.executeUpdate();
|
psInsert.executeUpdate();
|
||||||
rs=psSearch.executeQuery();
|
rs=psSearch.executeQuery();
|
||||||
|
|
||||||
if(rs.next())
|
if(rs.next())
|
||||||
return Queries.rowToBatch(rs);
|
return Queries.rowToBatch(rs);
|
||||||
else throw new BeanNotFound(String.format("Unable to find Bean with ",desc));
|
else throw new BeanNotFound(String.format("Unable to find Bean with ",desc));
|
||||||
|
}finally {
|
||||||
|
conn.close();
|
||||||
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -73,7 +77,8 @@ public class MappingManagerImpl implements MappingManager {
|
||||||
|
|
||||||
|
|
||||||
Connection conn=db.getConnection();
|
Connection conn=db.getConnection();
|
||||||
PreparedStatement psGet=null;
|
try{
|
||||||
|
PreparedStatement psGet=null;
|
||||||
DBField IDField=DBField.Farm.fields.get(DBField.Farm.FARM_ID);
|
DBField IDField=DBField.Farm.fields.get(DBField.Farm.FARM_ID);
|
||||||
if(desc.getCondition().containsKey(IDField)) {
|
if(desc.getCondition().containsKey(IDField)) {
|
||||||
psGet=Queries.GET_FARM_BY_ID.get(conn, desc);
|
psGet=Queries.GET_FARM_BY_ID.get(conn, desc);
|
||||||
|
@ -84,6 +89,9 @@ public class MappingManagerImpl implements MappingManager {
|
||||||
throw new BeanNotFound("Farm not found. Condition was "+desc);
|
throw new BeanNotFound("Farm not found. Condition was "+desc);
|
||||||
|
|
||||||
return Queries.rowToFarm(rs);
|
return Queries.rowToFarm(rs);
|
||||||
|
}finally {
|
||||||
|
conn.close();
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
|
@ -0,0 +1,38 @@
|
||||||
|
package org.gcube.application.perform.service.engine.impl;
|
||||||
|
|
||||||
|
import java.util.Map;
|
||||||
|
|
||||||
|
import org.gcube.application.perform.service.engine.PerformanceManager;
|
||||||
|
import org.gcube.application.perform.service.engine.dm.DMUtils;
|
||||||
|
import org.gcube.application.perform.service.engine.model.CSVExportRequest;
|
||||||
|
import org.gcube.application.perform.service.engine.model.importer.ImportRoutineDescriptor;
|
||||||
|
import org.gcube.data.analysis.dataminermanagercl.shared.data.computations.ComputationId;
|
||||||
|
|
||||||
|
public class PerformanceManagerImpl implements PerformanceManager{
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public String generateCSV(CSVExportRequest request) {
|
||||||
|
throw new RuntimeException("Not yet implemented");
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void loadOutputData(ImportRoutineDescriptor desc) {
|
||||||
|
// TODO Auto-generated method stub
|
||||||
|
ComputationId id= DMUtils.getComputation(desc);
|
||||||
|
Map<String,String> outputFiles=DMUtils.getOutputFiles(id);
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
}
|
|
@ -0,0 +1,98 @@
|
||||||
|
package org.gcube.application.perform.service.engine.model.importer;
|
||||||
|
|
||||||
|
import java.io.File;
|
||||||
|
import java.sql.Types;
|
||||||
|
import java.util.ArrayList;
|
||||||
|
import java.util.Map;
|
||||||
|
import java.util.Map.Entry;
|
||||||
|
|
||||||
|
import org.gcube.application.perform.service.engine.impl.Query;
|
||||||
|
import org.gcube.application.perform.service.engine.model.CSVExportRequest;
|
||||||
|
import org.gcube.application.perform.service.engine.model.DBField;
|
||||||
|
import org.gcube.application.perform.service.engine.model.DBQueryDescriptor;
|
||||||
|
|
||||||
|
public class ImportedTable {
|
||||||
|
|
||||||
|
/**
|
||||||
|
* CSV FILE labels -> DBField
|
||||||
|
*/
|
||||||
|
|
||||||
|
private Map<String,DBField> mappings;
|
||||||
|
|
||||||
|
private ArrayList<String> csvFields;
|
||||||
|
|
||||||
|
private String tablename;
|
||||||
|
private DBField routineIdField;
|
||||||
|
|
||||||
|
private Query query;
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Checks if passed set of labels is
|
||||||
|
*
|
||||||
|
* @param toMatchSchema
|
||||||
|
* @return
|
||||||
|
*/
|
||||||
|
public boolean matchesSchema(ArrayList<String> toMatchSchema) {
|
||||||
|
return csvFields.equals(toMatchSchema);
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
public DBQueryDescriptor getSetRow(Map<String,String> csvRow, Long routineId) {
|
||||||
|
DBQueryDescriptor desc=new DBQueryDescriptor();
|
||||||
|
|
||||||
|
for(Entry<String,String> csvField:csvRow.entrySet()) {
|
||||||
|
DBField toSetField=mappings.get(csvField.getKey());
|
||||||
|
|
||||||
|
Object value=csvField.getValue();
|
||||||
|
|
||||||
|
switch(toSetField.getType()) {
|
||||||
|
case Types.BIGINT : value=Long.parseLong((String) value);
|
||||||
|
break;
|
||||||
|
case Types.REAL : value=Double.parseDouble((String) value);
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
desc.add(toSetField, value);
|
||||||
|
}
|
||||||
|
|
||||||
|
desc.add(routineIdField, routineId);
|
||||||
|
return desc;
|
||||||
|
}
|
||||||
|
|
||||||
|
public Query getQuery() {
|
||||||
|
return query;
|
||||||
|
}
|
||||||
|
public String getTableName() {
|
||||||
|
return tablename;
|
||||||
|
}
|
||||||
|
|
||||||
|
public File exportCSV(CSVExportRequest request) {
|
||||||
|
// USE CASE to replace values in
|
||||||
|
/**
|
||||||
|
* SELECT
|
||||||
|
CASE status
|
||||||
|
WHEN 'VS' THEN 'validated by subsidiary'
|
||||||
|
WHEN 'NA' THEN 'not acceptable'
|
||||||
|
WHEN 'D' THEN 'delisted'
|
||||||
|
ELSE 'validated'
|
||||||
|
END AS STATUS
|
||||||
|
FROM SUPP_STATUS
|
||||||
|
*/
|
||||||
|
|
||||||
|
// get deanonimized labels
|
||||||
|
// Create query
|
||||||
|
|
||||||
|
|
||||||
|
// Pass the result set to the CSV specifying labels
|
||||||
|
// final Appendable out = ...;
|
||||||
|
// final CSVPrinter printer = CSVFormat.DEFAULT.withHeader("H1", "H2").print(out)
|
||||||
|
|
||||||
|
// return file
|
||||||
|
throw new RuntimeException("Not implemented");
|
||||||
|
|
||||||
|
}
|
||||||
|
}
|
|
@ -1,41 +1,82 @@
|
||||||
package org.gcube.application.perform.service;
|
package org.gcube.application.perform.service;
|
||||||
|
|
||||||
|
import java.util.concurrent.Semaphore;
|
||||||
|
import java.util.concurrent.atomic.AtomicLong;
|
||||||
|
|
||||||
import javax.ws.rs.client.WebTarget;
|
import javax.ws.rs.client.WebTarget;
|
||||||
import javax.ws.rs.core.Response;
|
import javax.ws.rs.core.Response;
|
||||||
|
|
||||||
import org.junit.Test;
|
import org.junit.Test;
|
||||||
|
|
||||||
public class AnagraphicTests extends CommonTest{
|
public class AnagraphicTests extends CommonTest{
|
||||||
|
|
||||||
|
private static int CALLS=100;
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void getBatch() {
|
public void getBatch() {
|
||||||
WebTarget target=
|
WebTarget target=
|
||||||
// target(ServiceConstants.SERVICE_NAME).
|
// target(ServiceConstants.SERVICE_NAME).
|
||||||
// path(ServiceConstants.APPLICATION_PATH).
|
// path(ServiceConstants.APPLICATION_PATH).
|
||||||
target(ServiceConstants.Mappings.PATH).
|
target(ServiceConstants.Mappings.PATH).
|
||||||
path(ServiceConstants.Mappings.BATCHES_METHOD).
|
path(ServiceConstants.Mappings.BATCHES_METHOD).
|
||||||
queryParam(ServiceConstants.Mappings.BATCH_NAME_PARAMETER, "gino").
|
queryParam(ServiceConstants.Mappings.BATCH_NAME_PARAMETER, "gino").
|
||||||
queryParam(ServiceConstants.Mappings.BATCH_TYPE_PARAMETER, "pino").
|
queryParam(ServiceConstants.Mappings.BATCH_TYPE_PARAMETER, "pino").
|
||||||
queryParam(ServiceConstants.Mappings.FARM_ID_PARAMETER, 12682549);
|
queryParam(ServiceConstants.Mappings.FARM_ID_PARAMETER, 12682549);
|
||||||
|
|
||||||
System.out.println(target.getUri());
|
System.out.println(target.getUri());
|
||||||
Response resp=target.request().get();
|
Response resp=target.request().get();
|
||||||
System.out.println(resp.getStatus() + " : "+ resp.readEntity(String.class));
|
System.out.println(resp.getStatus() + " : "+ resp.readEntity(String.class));
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void getFarm() {
|
public void getFarm() {
|
||||||
WebTarget target=
|
WebTarget target=
|
||||||
// target(ServiceConstants.SERVICE_NAME).
|
// target(ServiceConstants.SERVICE_NAME).
|
||||||
// path(ServiceConstants.APPLICATION_PATH).
|
// path(ServiceConstants.APPLICATION_PATH).
|
||||||
target(ServiceConstants.Mappings.PATH).
|
target(ServiceConstants.Mappings.PATH).
|
||||||
path(ServiceConstants.Mappings.FARM_METHOD).
|
path(ServiceConstants.Mappings.FARM_METHOD).
|
||||||
queryParam(ServiceConstants.Mappings.FARM_ID_PARAMETER, 126825).
|
queryParam(ServiceConstants.Mappings.FARM_ID_PARAMETER, 126825).
|
||||||
queryParam(ServiceConstants.Mappings.FARM_UUID_PARAMETER, "pino");
|
queryParam(ServiceConstants.Mappings.FARM_UUID_PARAMETER, "pino");
|
||||||
|
|
||||||
System.out.println(target.getUri());
|
System.out.println(target.getUri());
|
||||||
|
|
||||||
Response resp=target.request().get();
|
Response resp=target.request().get();
|
||||||
System.out.println(resp.getStatus() + " : "+ resp.readEntity(String.class));
|
System.out.println(resp.getStatus() + " : "+ resp.readEntity(String.class));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void parallelRequests() {
|
||||||
|
final AtomicLong currentExecution=new AtomicLong(0);
|
||||||
|
final Semaphore sem=new Semaphore((CALLS-1)*(-1));
|
||||||
|
final WebTarget farmTarget=target(ServiceConstants.Mappings.PATH).
|
||||||
|
path(ServiceConstants.Mappings.FARM_METHOD).
|
||||||
|
queryParam(ServiceConstants.Mappings.FARM_ID_PARAMETER, 126825).
|
||||||
|
queryParam(ServiceConstants.Mappings.FARM_UUID_PARAMETER, "pino");
|
||||||
|
final WebTarget batchesTarget=target(ServiceConstants.Mappings.PATH).
|
||||||
|
path(ServiceConstants.Mappings.BATCHES_METHOD).
|
||||||
|
queryParam(ServiceConstants.Mappings.BATCH_NAME_PARAMETER, "gino").
|
||||||
|
queryParam(ServiceConstants.Mappings.BATCH_TYPE_PARAMETER, "pino").
|
||||||
|
queryParam(ServiceConstants.Mappings.FARM_ID_PARAMETER, 12682549);
|
||||||
|
for(int i=0;i<CALLS;i++) {
|
||||||
|
Thread t=new Thread() {
|
||||||
|
@Override
|
||||||
|
public void run() {
|
||||||
|
farmTarget.request().get();
|
||||||
|
batchesTarget.request().get();
|
||||||
|
System.out.println("Performed "+currentExecution.incrementAndGet());
|
||||||
|
sem.release();
|
||||||
|
}
|
||||||
|
};
|
||||||
|
t.setDaemon(true);
|
||||||
|
t.start();
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
sem.acquire();
|
||||||
|
} catch (InterruptedException e) {
|
||||||
|
System.out.println("COMPLETED");
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in New Issue