diff --git a/.project b/.project index 4a1ea22..5a69db8 100644 --- a/.project +++ b/.project @@ -16,12 +16,12 @@ - org.eclipse.m2e.core.maven2Builder + org.eclipse.wst.validation.validationbuilder - org.eclipse.wst.validation.validationbuilder + org.eclipse.m2e.core.maven2Builder diff --git a/.settings/org.eclipse.wst.common.project.facet.core.xml b/.settings/org.eclipse.wst.common.project.facet.core.xml index 7a104c1..db18f9c 100644 --- a/.settings/org.eclipse.wst.common.project.facet.core.xml +++ b/.settings/org.eclipse.wst.common.project.facet.core.xml @@ -2,7 +2,7 @@ - + diff --git a/pom.xml b/pom.xml index 8e5952e..a656fe9 100644 --- a/pom.xml +++ b/pom.xml @@ -16,9 +16,7 @@ ${project.basedir}/src/main/webapp/WEB-INF ${project.basedir}/distro - 2.14 - 2.14 - 2.2.4.Final + @@ -43,6 +41,11 @@ common-smartgears + + org.gcube.core + common-encryption + + org.apache.commons commons-dbcp2 @@ -61,36 +64,38 @@ javax.ws.rs-api 2.0 - + org.projectlombok lombok 1.14.8 - - - + + + + + javax.enterprise cdi-api - 1.2 + 2.0 - - org.jboss.weld.servlet - weld-servlet - ${weld-version} - - - + + org.gcube.data.analysis data-miner-manager-cl - [1.0.0-SNAPSHOT,2.0.0-SNAPSHOT) + [1.0.0-SNAPSHOT,2.0.0-SNAPSHOT) - + + + \ No newline at end of file diff --git a/src/main/java/org/gcube/application/perform/service/LocalConfiguration.java b/src/main/java/org/gcube/application/perform/service/LocalConfiguration.java index b968c9d..a893a99 100644 --- a/src/main/java/org/gcube/application/perform/service/LocalConfiguration.java +++ b/src/main/java/org/gcube/application/perform/service/LocalConfiguration.java @@ -23,6 +23,8 @@ public class LocalConfiguration { public static final String PERFORMANCE_DB_ENDPOINT_CATEGORY="performance-db.ep.name"; + public static final String IMPORTER_COMPUTATION_ID="dm.importer.computationid"; + static LocalConfiguration instance=null; diff --git a/src/main/java/org/gcube/application/perform/service/PerformService.java b/src/main/java/org/gcube/application/perform/service/PerformService.java new file mode 100644 index 0000000..fe8cd0b --- /dev/null +++ b/src/main/java/org/gcube/application/perform/service/PerformService.java @@ -0,0 +1,14 @@ +package org.gcube.application.perform.service; + +import javax.ws.rs.ApplicationPath; + + +@ApplicationPath(ServiceConstants.APPLICATION_PATH) +public class PerformService { + + public PerformService() { + super(); + + } + +} diff --git a/src/main/java/org/gcube/application/perform/service/ServiceConstants.java b/src/main/java/org/gcube/application/perform/service/ServiceConstants.java index b19a91a..e81e7ff 100644 --- a/src/main/java/org/gcube/application/perform/service/ServiceConstants.java +++ b/src/main/java/org/gcube/application/perform/service/ServiceConstants.java @@ -2,6 +2,8 @@ package org.gcube.application.perform.service; public interface ServiceConstants { + public static final String APPLICATION_PATH="/gcube/service/"; + public static interface Mappings{ public static final String PATH="mappings"; public static final String BATCHES_METHOD="batch"; diff --git a/src/main/java/org/gcube/application/perform/service/engine/DataBaseManager.java b/src/main/java/org/gcube/application/perform/service/engine/DataBaseManager.java index 510353c..4c16fc0 100644 --- a/src/main/java/org/gcube/application/perform/service/engine/DataBaseManager.java +++ b/src/main/java/org/gcube/application/perform/service/engine/DataBaseManager.java @@ -4,6 +4,7 @@ import java.sql.Connection; import java.sql.SQLException; import org.gcube.application.perform.service.engine.model.ISQueryDescriptor; +import org.gcube.application.perform.service.engine.model.InternalException; public interface DataBaseManager { @@ -13,7 +14,7 @@ public interface DataBaseManager { } - public Connection getConnection() throws SQLException; + public Connection getConnection() throws SQLException, InternalException; } diff --git a/src/main/java/org/gcube/application/perform/service/engine/DataBaseManagerImpl.java b/src/main/java/org/gcube/application/perform/service/engine/DataBaseManagerImpl.java index 8a6ccb9..0229ce5 100644 --- a/src/main/java/org/gcube/application/perform/service/engine/DataBaseManagerImpl.java +++ b/src/main/java/org/gcube/application/perform/service/engine/DataBaseManagerImpl.java @@ -17,6 +17,7 @@ import org.apache.commons.pool2.impl.GenericObjectPoolConfig; import org.gcube.application.perform.service.LocalConfiguration; import org.gcube.application.perform.service.engine.model.DatabaseConnectionDescriptor; import org.gcube.application.perform.service.engine.model.ISQueryDescriptor; +import org.gcube.application.perform.service.engine.model.InternalException; import org.gcube.application.perform.service.engine.utils.ISUtils; import lombok.Synchronized; @@ -58,7 +59,7 @@ public class DataBaseManagerImpl implements DataBaseManager{ } - private synchronized DatabaseConnectionDescriptor getDB() { + private synchronized DatabaseConnectionDescriptor getDB() throws InternalException { if(dbDescriptor==null) { dbDescriptor=ISUtils.queryForDatabase(queryDescriptor); } @@ -66,13 +67,13 @@ public class DataBaseManagerImpl implements DataBaseManager{ } @Override - public Connection getConnection() throws SQLException { + public Connection getConnection() throws SQLException, InternalException { DataSource ds=getDataSource(); return ds.getConnection(); } @Synchronized - private DataSource getDataSource() { + private DataSource getDataSource() throws InternalException { DatabaseConnectionDescriptor dbDescriptor=getDB(); if(!datasources.containsKey(dbDescriptor.getUrl())) { diff --git a/src/main/java/org/gcube/application/perform/service/engine/Importer.java b/src/main/java/org/gcube/application/perform/service/engine/Importer.java index e52a4bf..82a82aa 100644 --- a/src/main/java/org/gcube/application/perform/service/engine/Importer.java +++ b/src/main/java/org/gcube/application/perform/service/engine/Importer.java @@ -6,12 +6,13 @@ import java.util.List; import org.gcube.application.perform.service.engine.dm.DMException; import org.gcube.application.perform.service.engine.model.BeanNotFound; import org.gcube.application.perform.service.engine.model.DBQueryDescriptor; +import org.gcube.application.perform.service.engine.model.InternalException; import org.gcube.application.perform.service.engine.model.importer.ImportRequest; import org.gcube.application.perform.service.engine.model.importer.ImportRoutineDescriptor; public interface Importer { - public ImportRoutineDescriptor importExcel(ImportRequest request) throws DMException, BeanNotFound, SQLException; - public List getDescriptors(DBQueryDescriptor query); + public ImportRoutineDescriptor importExcel(ImportRequest request) throws DMException, BeanNotFound, SQLException, InternalException; + public List getDescriptors(DBQueryDescriptor query) throws SQLException, InternalException; } diff --git a/src/main/java/org/gcube/application/perform/service/engine/ImporterImpl.java b/src/main/java/org/gcube/application/perform/service/engine/ImporterImpl.java index cfc4d02..6f3c359 100644 --- a/src/main/java/org/gcube/application/perform/service/engine/ImporterImpl.java +++ b/src/main/java/org/gcube/application/perform/service/engine/ImporterImpl.java @@ -4,7 +4,12 @@ import java.sql.Connection; import java.sql.PreparedStatement; import java.sql.ResultSet; import java.sql.SQLException; +import java.sql.Statement; +import java.time.Instant; +import java.util.ArrayList; +import java.util.HashMap; import java.util.List; +import java.util.Map; import org.gcube.application.perform.service.LocalConfiguration; import org.gcube.application.perform.service.engine.dm.DMException; @@ -17,7 +22,13 @@ import org.gcube.application.perform.service.engine.model.ISQueryDescriptor; import org.gcube.application.perform.service.engine.model.InternalException; import org.gcube.application.perform.service.engine.model.importer.ImportRequest; import org.gcube.application.perform.service.engine.model.importer.ImportRoutineDescriptor; +import org.gcube.application.perform.service.engine.model.importer.ImportStatus; +import org.gcube.application.perform.service.engine.utils.CommonUtils; +import org.gcube.application.perform.service.engine.utils.ScopeUtils; import org.gcube.data.analysis.dataminermanagercl.shared.data.computations.ComputationId; +import org.gcube.smartgears.ContextProvider; +import org.gcube.smartgears.configuration.container.ContainerConfiguration; +import org.gcube.smartgears.context.application.ApplicationContext; import lombok.Synchronized; import lombok.extern.slf4j.Slf4j; @@ -25,10 +36,7 @@ import lombok.extern.slf4j.Slf4j; @Slf4j public class ImporterImpl implements Importer { - private static final String ORPHAN_IMPORTS=""; - private static final String ACQUIRE_PS=""; - private static final String GET_BY_ID=""; - private static final String INSERT_ROUTINE=""; + private static ISQueryDescriptor isQueryDescriptor=null; @@ -47,31 +55,37 @@ public class ImporterImpl implements Importer { private static final String getHostname() { - throw new RuntimeException("IMPLEMENT THIS"); + ApplicationContext context=ContextProvider.get(); + ContainerConfiguration configuration=context.container().configuration(); + + return configuration.hostname(); } + + public void init() throws InternalException{ try { log.info("Initializing IMPORTER"); DataBaseManager db=DataBaseManager.get(getISQueryDescriptor()); Connection conn=db.getConnection(); conn.setAutoCommit(true); - PreparedStatement psOrphans=conn.prepareStatement(ORPHAN_IMPORTS); - PreparedStatement psAcquire=conn.prepareStatement(ACQUIRE_PS); + + + PreparedStatement psOrphans=Queries.ORPHAN_IMPORTS.prepare(conn); + PreparedStatement psAcquire=Queries.ACQUIRE_PS.prepare(conn); // set ps ResultSet rsOrphans=psOrphans.executeQuery(); long monitoredCount=0l; while(rsOrphans.next()) { Long id=rsOrphans.getLong(ImportRoutine.ID); try { - ImportRoutineDescriptor desc=rowToDescriptor(rsOrphans); + ImportRoutineDescriptor desc=Queries.rowToDescriptor(rsOrphans); String hostname=getHostname(); + DBQueryDescriptor acquireDesc=new DBQueryDescriptor(). + add(ImportRoutine.fields.get(ImportRoutine.LOCK), hostname). + add(ImportRoutine.fields.get(ImportRoutine.ID), id); - // "acquire" - // set lock = hostname where ID =? and LOCK is null - // Acquired = updated rows == 1 - psAcquire.setString(1, hostname); - psAcquire.setLong(2, id); + Queries.ACQUIRE_PS.fill(psAcquire, acquireDesc); if(psAcquire.executeUpdate()>0) { log.debug("Acquired {} ",id); @@ -91,7 +105,7 @@ public class ImporterImpl implements Importer { @Override - public ImportRoutineDescriptor importExcel(ImportRequest request) throws DMException, BeanNotFound, SQLException { + public ImportRoutineDescriptor importExcel(ImportRequest request) throws DMException, SQLException, InternalException { log.debug("Submitting {} ",request); ComputationId id=submit(request); log.debug("Registering {} computationID {} ",request,id); @@ -108,42 +122,91 @@ public class ImporterImpl implements Importer { DMUtils.monitor(DMUtils.getComputation(desc), new ImporterMonitor(desc,getISQueryDescriptor())); } - private ComputationId submit(ImportRequest request) { + + + + private ComputationId submit(ImportRequest request) throws DMException { + /** + * dataminer-prototypes.d4science.org/wps/WebProcessingService? + * request=Execute&service=WPS&Version=1.0.0&gcube-token=***REMOVED***&lang=en-US& + * Identifier=org.gcube.dataanalysis.wps.statisticalmanager.synchserver.mappedclasses.transducerers.PERFORMFISH_DATA_EXTRACTOR + * & + * DataInputs= + * InputData=https%3A%2F%2Fdata.d4science.org%2Fshub%2F9689bbe2-148f-4406-ab69-6e0f6ab892ca; + * BatchType=GROW_OUT_AGGREGATED; + * FarmID=ID + */ - throw new RuntimeException("IMPLEMENT THIS SHIT"); + Map parameters=new HashMap<>(); + parameters.put("InputData", request.getSource()); + parameters.put("BatchType", request.getBatchType()); + parameters.put("FarmID", request.getFarmId().toString()); + + return DMUtils.submitJob(LocalConfiguration.getProperty(LocalConfiguration.IMPORTER_COMPUTATION_ID), parameters); } - private ImportRoutineDescriptor register(ComputationId computationId,ImportRequest request) throws SQLException { - - ImportRoutineDescriptor toReturn=new ImportRoutineDescriptor(); - // SET FIELDS into OBJECT - - + private ImportRoutineDescriptor register(ComputationId computationId,ImportRequest request) throws SQLException, InternalException { + + DBQueryDescriptor insertionRow=new DBQueryDescriptor(). + add(ImportRoutine.fields.get(ImportRoutine.BATCH_TYPE), request.getBatchType()). + add(ImportRoutine.fields.get(ImportRoutine.CALLER), CommonUtils.encryptString(ScopeUtils.getCaller())). + add(ImportRoutine.fields.get(ImportRoutine.COMPUTATION_ID), computationId.getId()). + add(ImportRoutine.fields.get(ImportRoutine.COMPUTATION_OPID), computationId.getOperatorId()). + add(ImportRoutine.fields.get(ImportRoutine.COMPUTATION_OPNAME), computationId.getOperatorName()). + add(ImportRoutine.fields.get(ImportRoutine.COMPUTATION_REQ), computationId.getEquivalentRequest()). + add(ImportRoutine.fields.get(ImportRoutine.COMPUTATION_URL), computationId.getUrlId()). + add(ImportRoutine.fields.get(ImportRoutine.FARM_ID), request.getFarmId()). + add(ImportRoutine.fields.get(ImportRoutine.LOCK), getHostname()). + add(ImportRoutine.fields.get(ImportRoutine.SOURCE_URL), request.getSource()). + add(ImportRoutine.fields.get(ImportRoutine.SOURCE_VERSION), request.getVersion()). + add(ImportRoutine.fields.get(ImportRoutine.START), Instant.now()). + add(ImportRoutine.fields.get(ImportRoutine.STATUS),ImportStatus.ACCEPTED.toString()); + DataBaseManager db=DataBaseManager.get(getISQueryDescriptor()); Connection conn=db.getConnection(); - PreparedStatement ps=conn.prepareStatement(INSERT_ROUTINE); - // set fields + conn.setAutoCommit(true); - throw new RuntimeException("IMPLEMENT THIS SHIT"); + PreparedStatement ps=Queries.INSERT_ROUTINE.prepare(conn,Statement.RETURN_GENERATED_KEYS); + Queries.INSERT_ROUTINE.fill(ps, insertionRow); + ps.executeUpdate(); + + ResultSet rs=ps.getGeneratedKeys(); + rs.next(); + + PreparedStatement psGet=Queries.GET_BY_ID.get(conn, + new DBQueryDescriptor().add(ImportRoutine.fields.get(ImportRoutine.ID), rs.getLong(ImportRoutine.ID))); + ResultSet rsGet=psGet.executeQuery(); + rsGet.next(); + return Queries.rowToDescriptor(rsGet); } - private ImportRoutineDescriptor getDescriptorById(Long id) throws BeanNotFound, SQLException { + private ImportRoutineDescriptor getDescriptorById(Long id) throws SQLException, InternalException { DataBaseManager db=DataBaseManager.get(getISQueryDescriptor()); - Connection conn=db.getConnection(); - PreparedStatement ps=conn.prepareStatement(GET_BY_ID); + Connection conn=db.getConnection(); + + PreparedStatement ps=Queries.GET_BY_ID.get(conn, + new DBQueryDescriptor().add(ImportRoutine.fields.get(ImportRoutine.ID), id)); ps.setLong(1, id); ResultSet rs=ps.executeQuery(); - if(rs.next()) return rowToDescriptor(rs); + if(rs.next()) return Queries.rowToDescriptor(rs); else throw new BeanNotFound("Unable to find Routine with ID "+id); } @Override - public List getDescriptors(DBQueryDescriptor desc) { - // TODO Auto-generated method stub - return null; + public List getDescriptors(DBQueryDescriptor desc) throws SQLException, InternalException { + DataBaseManager db=DataBaseManager.get(getISQueryDescriptor()); + Connection conn=db.getConnection(); + + PreparedStatement ps=Queries.FILTER_IMPORTS.get(conn, desc); + ResultSet rs=ps.executeQuery(); + ArrayList toReturn=new ArrayList<>(); + while (rs.next()) + toReturn.add(Queries.rowToDescriptor(rs)); + + return toReturn; } @@ -151,8 +214,6 @@ public class ImporterImpl implements Importer { - private static ImportRoutineDescriptor rowToDescriptor(ResultSet rs) { - throw new RuntimeException("IMPLEMENT THIS SHIT"); - } - } + + diff --git a/src/main/java/org/gcube/application/perform/service/engine/MappingManager.java b/src/main/java/org/gcube/application/perform/service/engine/MappingManager.java index 787c582..bcd9552 100644 --- a/src/main/java/org/gcube/application/perform/service/engine/MappingManager.java +++ b/src/main/java/org/gcube/application/perform/service/engine/MappingManager.java @@ -4,12 +4,14 @@ import java.sql.SQLException; import org.gcube.application.perform.service.engine.model.BeanNotFound; import org.gcube.application.perform.service.engine.model.DBQueryDescriptor; +import org.gcube.application.perform.service.engine.model.InternalException; +import org.gcube.application.perform.service.engine.model.InvalidRequestException; import org.gcube.application.perform.service.engine.model.anagraphic.Batch; import org.gcube.application.perform.service.engine.model.anagraphic.Farm; public interface MappingManager { - public Batch getBatch(DBQueryDescriptor desc) throws BeanNotFound, SQLException; - public Farm getFarm(DBQueryDescriptor desc)throws BeanNotFound, SQLException; + public Batch getBatch(DBQueryDescriptor desc) throws BeanNotFound, SQLException, InvalidRequestException, InternalException; + public Farm getFarm(DBQueryDescriptor desc)throws BeanNotFound, SQLException,InvalidRequestException, InternalException; } diff --git a/src/main/java/org/gcube/application/perform/service/engine/MappingManagerImpl.java b/src/main/java/org/gcube/application/perform/service/engine/MappingManagerImpl.java index 68f9612..89668e5 100644 --- a/src/main/java/org/gcube/application/perform/service/engine/MappingManagerImpl.java +++ b/src/main/java/org/gcube/application/perform/service/engine/MappingManagerImpl.java @@ -11,6 +11,8 @@ import org.gcube.application.perform.service.engine.model.BeanNotFound; import org.gcube.application.perform.service.engine.model.DBField; import org.gcube.application.perform.service.engine.model.DBQueryDescriptor; import org.gcube.application.perform.service.engine.model.ISQueryDescriptor; +import org.gcube.application.perform.service.engine.model.InternalException; +import org.gcube.application.perform.service.engine.model.InvalidRequestException; import org.gcube.application.perform.service.engine.model.anagraphic.Batch; import org.gcube.application.perform.service.engine.model.anagraphic.Farm; @@ -19,17 +21,6 @@ import lombok.extern.slf4j.Slf4j; @Slf4j public class MappingManagerImpl implements MappingManager { - - - private static final String INSERT_BATCH="INSERT INTO batches (id,uuid,farmid,type,name) VALUES (SELECT (max(id)+1 from batches),?,?,?,?) ON CONFLICT DO NOTHING"; - private static final String GET_BATCH_BY_ID="Select * from batches where id = ?"; - private static final String GET_BATCH_BY_DESCRIPTIVE_KEY="Select * from batches where farmid=? AND type=? AND name= ?"; - - - private static final String GET_FARM_BY_ID=String.format("SELECT f.farmid as $1%s, f.uuid as %2$s, c.companyid as %3$s, c.uuid as %4$s, a.associationid as %5$s, a.uuid as %6$s FROM " - + "farms as f INNER JOIN companies as c ON f.companyid=c.companyid INNER JOIN associations as a ON c.associationid=a.associationid WHERE f.farmid = ?", - DBField.Farm.FARM_ID,DBField.Farm.UUID,DBField.Farm.COMPANY_ID,DBField.Farm.COMPANY_UUID,DBField.Farm.ASSOCIATION_ID,DBField.Farm.ASSOCIATION_UUID); - private static ISQueryDescriptor isQueryDescriptor=null; @@ -49,32 +40,22 @@ public class MappingManagerImpl implements MappingManager { @Override - public Batch getBatch(DBQueryDescriptor desc) throws BeanNotFound,SQLException{ + public Batch getBatch(DBQueryDescriptor desc) throws SQLException, InternalException{ DataBaseManager db=DataBaseManager.get(getISQueryDescriptor()); - - String type=desc.getCondition().get(DBField.Batch.fields.get(DBField.Batch.BATCH_TYPE)); - String name=desc.getCondition().get(DBField.Batch.fields.get(DBField.Batch.BATCH_NAME)); - Long farmid=Long.parseLong(desc.getCondition().get(DBField.Batch.fields.get(DBField.Batch.FARM_ID))); - Connection conn=db.getConnection(); - PreparedStatement psSearch=conn.prepareStatement(GET_BATCH_BY_DESCRIPTIVE_KEY); - psSearch.setLong(1, farmid); - psSearch.setString(2, type); - psSearch.setString(3, name); + Query getQuery=Queries.GET_BATCH_BY_DESCRIPTIVE_KEY; + PreparedStatement psSearch=getQuery.get(conn, desc); + ResultSet rs=psSearch.executeQuery(); if(rs.next()) - return rowToBatch(rs); + return Queries.rowToBatch(rs); // ID NOT FOUND, TRY TO REGISTER IT - log.trace("Registering new Batch [name {} , type {}, farmid {} ] ",name,type,farmid); - PreparedStatement psInsert=conn.prepareStatement(INSERT_BATCH); - psInsert.setString(1, UUID.randomUUID().toString()); - psInsert.setLong(2,farmid); - psInsert.setString(3, type); - psInsert.setString(4, name); - + log.trace("Registering new Batch from condition {}",desc); + PreparedStatement psInsert=Queries.INSERT_BATCH.get(conn, desc); + boolean inserted=psInsert.executeUpdate()==1; if(inserted) { conn.commit(); @@ -83,52 +64,31 @@ public class MappingManagerImpl implements MappingManager { rs=psSearch.executeQuery(); if(rs.next()) - return rowToBatch(rs); - else throw new BeanNotFound(String.format("Unable to find Bean [farmid=%1$d, type=%2$s,name=%3$s]",farmid,type,name)); + return Queries.rowToBatch(rs); + else throw new BeanNotFound(String.format("Unable to find Bean with ",desc)); } @Override - public Farm getFarm(DBQueryDescriptor desc) throws BeanNotFound,SQLException{ + public Farm getFarm(DBQueryDescriptor desc) throws SQLException, InternalException{ DataBaseManager db=DataBaseManager.get(getISQueryDescriptor()); Connection conn=db.getConnection(); PreparedStatement psGet=null; DBField IDField=DBField.Farm.fields.get(DBField.Farm.FARM_ID); - if(desc.getCondition().containsKey(IDField)) { - // get By ID - psGet=conn.prepareStatement(GET_FARM_BY_ID); - psGet.setLong(1,Long.parseLong(desc.getCondition().get(IDField))); + if(desc.getCondition().containsKey(IDField)) { + psGet=Queries.GET_FARM_BY_ID.get(conn, desc); } ResultSet rs=psGet.executeQuery(); if(!rs.next()) throw new BeanNotFound("Farm not found. Condition was "+desc); - return rowToFarm(rs); + return Queries.rowToFarm(rs); } - private static Batch rowToBatch(ResultSet rs) throws SQLException { - Batch toReturn=new Batch(); - toReturn.setFarmId(rs.getLong(DBField.Batch.FARM_ID)); - toReturn.setId(rs.getLong(DBField.Batch.BATCH_ID)); - toReturn.setName(rs.getString(DBField.Batch.BATCH_NAME)); - toReturn.setType(rs.getString(DBField.Batch.BATCH_TYPE)); - return toReturn; - } - - private static Farm rowToFarm(ResultSet rs) throws SQLException { - Farm toReturn=new Farm(); - toReturn.setAssociationId(rs.getLong(DBField.Farm.ASSOCIATION_ID)); - toReturn.setAssociationUUID(rs.getString(DBField.Farm.ASSOCIATION_UUID)); - toReturn.setCompanyId(rs.getLong(DBField.Farm.COMPANY_ID)); - toReturn.setCompanyUUID(rs.getString(DBField.Farm.COMPANY_UUID)); - toReturn.setId(rs.getLong(DBField.Farm.FARM_ID)); - toReturn.setUUID(rs.getString(DBField.Farm.UUID)); - return toReturn; - } } diff --git a/src/main/java/org/gcube/application/perform/service/engine/Queries.java b/src/main/java/org/gcube/application/perform/service/engine/Queries.java new file mode 100644 index 0000000..8c16be5 --- /dev/null +++ b/src/main/java/org/gcube/application/perform/service/engine/Queries.java @@ -0,0 +1,68 @@ +package org.gcube.application.perform.service.engine; + +import java.sql.ResultSet; +import java.sql.SQLException; + +import org.gcube.application.perform.service.engine.model.DBField; +import org.gcube.application.perform.service.engine.model.anagraphic.Batch; +import org.gcube.application.perform.service.engine.model.anagraphic.Farm; +import org.gcube.application.perform.service.engine.model.importer.ImportRoutineDescriptor; + +public class Queries { + + public static final Query GET_BATCH_BY_DESCRIPTIVE_KEY= new Query("Select * from batches where farmid=? AND type=? AND name= ?", + new DBField[] {}); + + public static final Query INSERT_BATCH=new Query("INSERT INTO batches (id,uuid,farmid,type,name) VALUES (SELECT (max(id)+1 from batches),?,?,?,?) ON CONFLICT DO NOTHING", + new DBField[] {}); + + public static final Query GET_FARM_BY_ID=new Query(String.format("SELECT f.farmid as $1%s, f.uuid as %2$s, c.companyid as %3$s, c.uuid as %4$s, a.associationid as %5$s, a.uuid as %6$s FROM " + + "farms as f INNER JOIN companies as c ON f.companyid=c.companyid INNER JOIN associations as a ON c.associationid=a.associationid WHERE f.farmid = ?", + DBField.Farm.FARM_ID,DBField.Farm.UUID,DBField.Farm.COMPANY_ID,DBField.Farm.COMPANY_UUID,DBField.Farm.ASSOCIATION_ID,DBField.Farm.ASSOCIATION_UUID), + new DBField[] {}); + + public static final Query GET_BATCH_BY_ID=new Query("Select * from batches where id = ?", + new DBField[] {}); + + + // "acquire" + // set lock = hostname where ID =? and LOCK is null + // Acquired = updated rows == 1 + public static final Query ORPHAN_IMPORTS=new Query("",new DBField[]{}); + public static final Query ACQUIRE_PS=new Query("",new DBField[]{}); + public static final Query GET_BY_ID=new Query("",new DBField[]{}); + public static final Query INSERT_ROUTINE=new Query("",new DBField[]{}); + + public static final Query FILTER_IMPORTS=new Query("",new DBField[]{}); + + + + // LOADERS + + public static Batch rowToBatch(ResultSet rs) throws SQLException { + Batch toReturn=new Batch(); + toReturn.setFarmId(rs.getLong(DBField.Batch.FARM_ID)); + toReturn.setId(rs.getLong(DBField.Batch.BATCH_ID)); + toReturn.setName(rs.getString(DBField.Batch.BATCH_NAME)); + toReturn.setType(rs.getString(DBField.Batch.BATCH_TYPE)); + return toReturn; + } + + + public static Farm rowToFarm(ResultSet rs) throws SQLException { + Farm toReturn=new Farm(); + toReturn.setAssociationId(rs.getLong(DBField.Farm.ASSOCIATION_ID)); + toReturn.setAssociationUUID(rs.getString(DBField.Farm.ASSOCIATION_UUID)); + toReturn.setCompanyId(rs.getLong(DBField.Farm.COMPANY_ID)); + toReturn.setCompanyUUID(rs.getString(DBField.Farm.COMPANY_UUID)); + toReturn.setId(rs.getLong(DBField.Farm.FARM_ID)); + toReturn.setUUID(rs.getString(DBField.Farm.UUID)); + return toReturn; + } + + + public static ImportRoutineDescriptor rowToDescriptor(ResultSet rs) { + throw new RuntimeException("IMPLEMENT THIS SHIT"); + } + +} diff --git a/src/main/java/org/gcube/application/perform/service/engine/Query.java b/src/main/java/org/gcube/application/perform/service/engine/Query.java new file mode 100644 index 0000000..7f1bc3d --- /dev/null +++ b/src/main/java/org/gcube/application/perform/service/engine/Query.java @@ -0,0 +1,58 @@ +package org.gcube.application.perform.service.engine; + +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.Arrays; + +import org.gcube.application.perform.service.engine.model.DBField; +import org.gcube.application.perform.service.engine.model.DBQueryDescriptor; +import org.gcube.application.perform.service.engine.model.InvalidRequestException; + + +public class Query { + + private final String query; + private final ArrayList psFields; + + public Query(String query,DBField[] fields) { + this.query=query; + this.psFields=new ArrayList<>(Arrays.asList(fields)); + } + + + public PreparedStatement prepare(Connection conn, int statementOption) throws SQLException { + return conn.prepareStatement(getQuery(),statementOption); + } + public PreparedStatement prepare(Connection conn) throws SQLException { + return conn.prepareStatement(getQuery()); + } + + + public PreparedStatement get(Connection conn, DBQueryDescriptor desc) throws SQLException, InvalidRequestException { + PreparedStatement ps=prepare(conn); + return fill(ps,desc); + } + + public PreparedStatement fill(PreparedStatement ps,DBQueryDescriptor desc) throws SQLException, InvalidRequestException{ + ArrayList fields=getPSFields(); + for(int i=0;i getPSFields(){ + return psFields; + } + + +} diff --git a/src/main/java/org/gcube/application/perform/service/engine/dm/DMUtils.java b/src/main/java/org/gcube/application/perform/service/engine/dm/DMUtils.java index 58cb833..ccb0180 100644 --- a/src/main/java/org/gcube/application/perform/service/engine/dm/DMUtils.java +++ b/src/main/java/org/gcube/application/perform/service/engine/dm/DMUtils.java @@ -41,6 +41,10 @@ public class DMUtils { monitor.start(); } + public static ComputationId submitJob(String operatorId, Map parameters) throws DMException { + return submitJob(getClient(),operatorId,parameters); + } + public static ComputationId submitJob(SClient client, String operatorId, Map parameters) throws DMException { try { log.debug("Looking for operator by Id {} ",operatorId); diff --git a/src/main/java/org/gcube/application/perform/service/engine/model/DBQueryDescriptor.java b/src/main/java/org/gcube/application/perform/service/engine/model/DBQueryDescriptor.java index d28b530..1031d67 100644 --- a/src/main/java/org/gcube/application/perform/service/engine/model/DBQueryDescriptor.java +++ b/src/main/java/org/gcube/application/perform/service/engine/model/DBQueryDescriptor.java @@ -5,19 +5,26 @@ import java.util.Map.Entry; import lombok.AllArgsConstructor; import lombok.Getter; +import lombok.NoArgsConstructor; @Getter @AllArgsConstructor +@NoArgsConstructor public class DBQueryDescriptor { - private Map condition; + private Map condition; public String toString() { StringBuilder builder=new StringBuilder(); - for(Entry entry : condition.entrySet()) { + for(Entry entry : condition.entrySet()) { builder.append(String.format("%1$s = %2$s AND ", entry.getKey().getFieldName(),entry.getValue())); } return builder.substring(0,builder.lastIndexOf(" AND ")).toString(); } + + public DBQueryDescriptor add(DBField field,Object obj) { + condition.put(field, obj); + return this; + } } diff --git a/src/main/java/org/gcube/application/perform/service/engine/model/InvalidRequestException.java b/src/main/java/org/gcube/application/perform/service/engine/model/InvalidRequestException.java new file mode 100644 index 0000000..da83e3d --- /dev/null +++ b/src/main/java/org/gcube/application/perform/service/engine/model/InvalidRequestException.java @@ -0,0 +1,38 @@ +package org.gcube.application.perform.service.engine.model; + +public class InvalidRequestException extends InternalException { + + /** + * + */ + private static final long serialVersionUID = 1L; + + public InvalidRequestException() { + super(); + // TODO Auto-generated constructor stub + } + + public InvalidRequestException(String message, Throwable cause, boolean enableSuppression, + boolean writableStackTrace) { + super(message, cause, enableSuppression, writableStackTrace); + // TODO Auto-generated constructor stub + } + + public InvalidRequestException(String message, Throwable cause) { + super(message, cause); + // TODO Auto-generated constructor stub + } + + public InvalidRequestException(String message) { + super(message); + // TODO Auto-generated constructor stub + } + + public InvalidRequestException(Throwable cause) { + super(cause); + // TODO Auto-generated constructor stub + } + + + +} diff --git a/src/main/java/org/gcube/application/perform/service/engine/model/importer/ImportRequest.java b/src/main/java/org/gcube/application/perform/service/engine/model/importer/ImportRequest.java index 3211a4e..674925c 100644 --- a/src/main/java/org/gcube/application/perform/service/engine/model/importer/ImportRequest.java +++ b/src/main/java/org/gcube/application/perform/service/engine/model/importer/ImportRequest.java @@ -1,5 +1,18 @@ package org.gcube.application.perform.service.engine.model.importer; +import lombok.AllArgsConstructor; +import lombok.Getter; +import lombok.ToString; + +@AllArgsConstructor +@Getter +@ToString public class ImportRequest { + private String source; + private String version; + private String batchType; + private Long farmId; + + } diff --git a/src/main/java/org/gcube/application/perform/service/engine/utils/CommonUtils.java b/src/main/java/org/gcube/application/perform/service/engine/utils/CommonUtils.java new file mode 100644 index 0000000..a3b3ef4 --- /dev/null +++ b/src/main/java/org/gcube/application/perform/service/engine/utils/CommonUtils.java @@ -0,0 +1,22 @@ +package org.gcube.application.perform.service.engine.utils; + +import org.gcube.common.encryption.StringEncrypter; + +public class CommonUtils { + + public static String decryptString(String toDecrypt){ + try{ + return StringEncrypter.getEncrypter().decrypt(toDecrypt); + }catch(Exception e) { + throw new RuntimeException("Unable to decrypt : "+toDecrypt,e); + } + } + + public static String encryptString(String toEncrypt){ + try{ + return StringEncrypter.getEncrypter().encrypt(toEncrypt); + }catch(Exception e) { + throw new RuntimeException("Unable to encrypt : "+toEncrypt,e); + } + } +} diff --git a/src/main/java/org/gcube/application/perform/service/engine/utils/ISUtils.java b/src/main/java/org/gcube/application/perform/service/engine/utils/ISUtils.java index a7ab45b..e5cd37d 100644 --- a/src/main/java/org/gcube/application/perform/service/engine/utils/ISUtils.java +++ b/src/main/java/org/gcube/application/perform/service/engine/utils/ISUtils.java @@ -1,13 +1,54 @@ package org.gcube.application.perform.service.engine.utils; +import static org.gcube.resources.discovery.icclient.ICFactory.clientFor; +import static org.gcube.resources.discovery.icclient.ICFactory.queryFor; + +import java.util.List; + import org.gcube.application.perform.service.engine.model.DatabaseConnectionDescriptor; import org.gcube.application.perform.service.engine.model.ISQueryDescriptor; +import org.gcube.application.perform.service.engine.model.InternalException; +import org.gcube.common.resources.gcore.ServiceEndpoint; +import org.gcube.common.resources.gcore.ServiceEndpoint.AccessPoint; +import org.gcube.resources.discovery.client.api.DiscoveryClient; +import org.gcube.resources.discovery.client.queries.api.SimpleQuery; +import lombok.extern.slf4j.Slf4j; + +@Slf4j public class ISUtils { - public static DatabaseConnectionDescriptor queryForDatabase(ISQueryDescriptor desc) { - throw new RuntimeException("Implment me, asshole!"); + public static DatabaseConnectionDescriptor queryForDatabase(ISQueryDescriptor desc) throws InternalException { + + log.debug("Querying for Service Endpoints {} ",desc,ScopeUtils.getCurrentScope()); + + SimpleQuery query = queryFor(ServiceEndpoint.class); + + if(desc.getCategory()!=null) + query.addCondition("$resource/Profile/Category/text() eq '"+desc.getCategory()+"'"); + + if(desc.getPlatformName()!=null) + query.addCondition("$resource/Profile/Platform/Name/text() eq '"+desc.getPlatformName()+"'"); + + if(desc.getResourceName()!=null) + query.addCondition("$resource/Profile/Name/text() eq '"+desc.getCategory()+"'"); + + + DiscoveryClient client = clientFor(ServiceEndpoint.class); + + List found=client.submit(query); + + if(found.size()==0) throw new InternalException("Unable to find Service Endpoint "+desc+" in "+ScopeUtils.getCurrentScope()); + if(found.size()>1) log.warn("Multiple Endpoints "+desc+" found in "+ScopeUtils.getCurrentScope()); + AccessPoint point= found.get(0).profile().accessPoints().iterator().next(); + + String url="jdbc:postgresql://"+point.address()+"/"+point.name(); + + DatabaseConnectionDescriptor toReturn= new DatabaseConnectionDescriptor(point.username(), url, CommonUtils.decryptString(point.password())); + log.debug("Going to use DB : "+toReturn); + + return toReturn; } diff --git a/src/main/java/org/gcube/application/perform/service/engine/utils/ScopeUtils.java b/src/main/java/org/gcube/application/perform/service/engine/utils/ScopeUtils.java index bcf5f89..e430ddd 100644 --- a/src/main/java/org/gcube/application/perform/service/engine/utils/ScopeUtils.java +++ b/src/main/java/org/gcube/application/perform/service/engine/utils/ScopeUtils.java @@ -42,5 +42,10 @@ public class ScopeUtils { } } } + + + public static String getCaller() { + return SecurityTokenProvider.instance.get(); + } } diff --git a/src/main/java/org/gcube/application/perform/service/rest/Import.java b/src/main/java/org/gcube/application/perform/service/rest/Import.java index e26810f..071864d 100644 --- a/src/main/java/org/gcube/application/perform/service/rest/Import.java +++ b/src/main/java/org/gcube/application/perform/service/rest/Import.java @@ -2,6 +2,7 @@ package org.gcube.application.perform.service.rest; import java.util.List; +import javax.inject.Inject; import javax.ws.rs.GET; import javax.ws.rs.POST; import javax.ws.rs.Path; @@ -11,6 +12,7 @@ import javax.ws.rs.core.Response; import org.gcube.application.perform.service.PerformServiceManager; import org.gcube.application.perform.service.ServiceConstants; +import org.gcube.application.perform.service.engine.Importer; import org.gcube.application.perform.service.engine.model.importer.ImportRoutineDescriptor; import org.gcube.smartgears.annotations.ManagedBy; @@ -20,6 +22,10 @@ import org.gcube.smartgears.annotations.ManagedBy; public class Import { + @Inject + private Importer importer; + + /** * Launches DM Import * diff --git a/src/main/java/org/gcube/application/perform/service/rest/Mappings.java b/src/main/java/org/gcube/application/perform/service/rest/Mappings.java index 817cb64..be5a7c5 100644 --- a/src/main/java/org/gcube/application/perform/service/rest/Mappings.java +++ b/src/main/java/org/gcube/application/perform/service/rest/Mappings.java @@ -19,6 +19,8 @@ import org.gcube.application.perform.service.engine.MappingManager; import org.gcube.application.perform.service.engine.model.BeanNotFound; import org.gcube.application.perform.service.engine.model.DBField; import org.gcube.application.perform.service.engine.model.DBQueryDescriptor; +import org.gcube.application.perform.service.engine.model.InternalException; +import org.gcube.application.perform.service.engine.model.InvalidRequestException; import org.gcube.application.perform.service.engine.model.anagraphic.Batch; import org.gcube.application.perform.service.engine.model.anagraphic.Farm; import org.gcube.smartgears.annotations.ManagedBy; @@ -54,20 +56,24 @@ public class Mappings { InterfaceCommons.checkMandatory(type, ServiceConstants.Mappings.BATCH_TYPE_PARAMETER); InterfaceCommons.checkMandatory(farmid, ServiceConstants.Mappings.FARM_ID_PARAMETER); - HashMap condition=new HashMap(); + HashMap condition=new HashMap(); condition.put(DBField.Batch.fields.get(DBField.Batch.BATCH_NAME), name); condition.put(DBField.Batch.fields.get(DBField.Batch.BATCH_TYPE), type); - condition.put(DBField.Batch.fields.get(DBField.Batch.FARM_ID), farmid.toString()); + condition.put(DBField.Batch.fields.get(DBField.Batch.FARM_ID), farmid); DBQueryDescriptor desc=new DBQueryDescriptor(condition); try{ return mappings.getBatch(desc); }catch(BeanNotFound e) { - throw new WebApplicationException("Unable to find Farm with condition "+desc,Response.Status.BAD_REQUEST); + throw new WebApplicationException("Unable to find Batch with condition "+desc,Response.Status.BAD_REQUEST); }catch(SQLException e) { throw new WebApplicationException("Unexpected Exception occurred while dealing with database.", e,Response.Status.INTERNAL_SERVER_ERROR); + } catch (InvalidRequestException e) { + throw new WebApplicationException("Unable to search for Batch. ",e,Response.Status.BAD_REQUEST); + } catch (InternalException e) { + throw new WebApplicationException("Unexpected Exception.", e,Response.Status.INTERNAL_SERVER_ERROR); } } @@ -89,7 +95,7 @@ public class Mappings { try { InterfaceCommons.checkMandatory(farmid, ServiceConstants.Mappings.FARM_ID_PARAMETER); - desc=new DBQueryDescriptor(Collections.singletonMap(DBField.Farm.fields.get(DBField.Farm.FARM_ID), farmid.toString())); + desc=new DBQueryDescriptor(Collections.singletonMap(DBField.Farm.fields.get(DBField.Farm.FARM_ID), farmid)); }catch(WebApplicationException e) { try { InterfaceCommons.checkMandatory(farmuuid, ServiceConstants.Mappings.FARM_UUID_PARAMETER); @@ -105,6 +111,10 @@ public class Mappings { throw new WebApplicationException("Unable to find Farm with condition "+desc,Response.Status.BAD_REQUEST); }catch(SQLException e) { throw new WebApplicationException("Unexpected Exception occurred while dealing with database.", e,Response.Status.INTERNAL_SERVER_ERROR); + } catch (InvalidRequestException e) { + throw new WebApplicationException("Unable to search for Farm. ",e,Response.Status.BAD_REQUEST); + } catch (InternalException e) { + throw new WebApplicationException("Unexpected Exception.", e,Response.Status.INTERNAL_SERVER_ERROR); } } } diff --git a/src/main/webapp/WEB-INF/web.xml b/src/main/webapp/WEB-INF/web.xml new file mode 100644 index 0000000..f1d7ed4 --- /dev/null +++ b/src/main/webapp/WEB-INF/web.xml @@ -0,0 +1,7 @@ + + + + org.gcube.application.perform.service.PerformService + /gcube/service/* + + \ No newline at end of file