diff --git a/catalogue-plugin-framework/src/main/java/org/gcube/data/publishing/gCatFeeder/catalogues/CatalogueController.java b/catalogue-plugin-framework/src/main/java/org/gcube/data/publishing/gCatFeeder/catalogues/CatalogueController.java index aad25a8..967eaab 100644 --- a/catalogue-plugin-framework/src/main/java/org/gcube/data/publishing/gCatFeeder/catalogues/CatalogueController.java +++ b/catalogue-plugin-framework/src/main/java/org/gcube/data/publishing/gCatFeeder/catalogues/CatalogueController.java @@ -1,11 +1,13 @@ package org.gcube.data.publishing.gCatFeeder.catalogues; +import java.io.Serializable; + import org.gcube.data.publishing.gCatFeeder.catalogues.model.PublishReport; import org.gcube.data.publishing.gCatFeeder.catalogues.model.faults.WrongObjectFormatException; public interface CatalogueController { - public PublishReport publishItem(String serializedItem) throws WrongObjectFormatException; + public PublishReport publishItem(Serializable toPublish) throws WrongObjectFormatException; } diff --git a/catalogue-plugin-framework/src/main/java/org/gcube/data/publishing/gCatFeeder/catalogues/model/PublishReport.java b/catalogue-plugin-framework/src/main/java/org/gcube/data/publishing/gCatFeeder/catalogues/model/PublishReport.java index 5186aec..988e60f 100644 --- a/catalogue-plugin-framework/src/main/java/org/gcube/data/publishing/gCatFeeder/catalogues/model/PublishReport.java +++ b/catalogue-plugin-framework/src/main/java/org/gcube/data/publishing/gCatFeeder/catalogues/model/PublishReport.java @@ -1,6 +1,7 @@ package org.gcube.data.publishing.gCatFeeder.catalogues.model; import lombok.Getter; +import lombok.NonNull; import lombok.RequiredArgsConstructor; import lombok.Setter; @@ -8,9 +9,12 @@ import lombok.Setter; @RequiredArgsConstructor @Setter public class PublishReport { - - private boolean successful=false; + + @NonNull + private Boolean successful; + + @NonNull private String publishedId; } diff --git a/collectors-plugin-framework/src/main/java/org/gcube/data/publishing/gCatfeeder/collectors/CatalogueRetriever.java b/collectors-plugin-framework/src/main/java/org/gcube/data/publishing/gCatfeeder/collectors/CatalogueRetriever.java index 3a93614..a1fceb6 100644 --- a/collectors-plugin-framework/src/main/java/org/gcube/data/publishing/gCatfeeder/collectors/CatalogueRetriever.java +++ b/collectors-plugin-framework/src/main/java/org/gcube/data/publishing/gCatfeeder/collectors/CatalogueRetriever.java @@ -1,9 +1,10 @@ package org.gcube.data.publishing.gCatfeeder.collectors; import org.gcube.data.publishing.gCatFeeder.model.CatalogueInstanceDescriptor; +import org.gcube.data.publishing.gCatfeeder.collectors.model.faults.CatalogueInstanceNotFound; public interface CatalogueRetriever { - public CatalogueInstanceDescriptor getInstance(); + public CatalogueInstanceDescriptor getInstance() throws CatalogueInstanceNotFound; } diff --git a/collectors-plugin-framework/src/main/java/org/gcube/data/publishing/gCatfeeder/collectors/CollectorPlugin.java b/collectors-plugin-framework/src/main/java/org/gcube/data/publishing/gCatfeeder/collectors/CollectorPlugin.java index 2763583..d1a4434 100644 --- a/collectors-plugin-framework/src/main/java/org/gcube/data/publishing/gCatfeeder/collectors/CollectorPlugin.java +++ b/collectors-plugin-framework/src/main/java/org/gcube/data/publishing/gCatfeeder/collectors/CollectorPlugin.java @@ -1,20 +1,21 @@ package org.gcube.data.publishing.gCatfeeder.collectors; +import java.io.Serializable; import java.util.Set; -import org.gcube.data.publishing.gCatFeeder.model.FormatData; import org.gcube.data.publishing.gCatfeeder.collectors.model.CustomData; import org.gcube.data.publishing.gCatfeeder.collectors.model.PluginDescriptor; -import org.gcube.data.publishing.gCatfeeder.collectors.model.PublisherFormatData; import org.gcube.data.publishing.gCatfeeder.collectors.model.faults.CatalogueNotSupportedException; public interface CollectorPlugin{ public PluginDescriptor getDescriptor(); - public CatalogueRetriever getRetrieverByCatalogueType(FormatData catalogueType) throws CatalogueNotSupportedException; + public CatalogueRetriever getRetrieverByCatalogueType(String catalogueType) throws CatalogueNotSupportedException; - public Set> getImplementedTransformers(); + public Set getSupportedCatalogueTypes(); + + public DataTransformer getTransformerByCatalogueType(String catalogueType); public DataCollector getCollector(); } diff --git a/collectors-plugin-framework/src/main/java/org/gcube/data/publishing/gCatfeeder/collectors/DataTransformer.java b/collectors-plugin-framework/src/main/java/org/gcube/data/publishing/gCatfeeder/collectors/DataTransformer.java index 9da6a2e..177a855 100644 --- a/collectors-plugin-framework/src/main/java/org/gcube/data/publishing/gCatfeeder/collectors/DataTransformer.java +++ b/collectors-plugin-framework/src/main/java/org/gcube/data/publishing/gCatfeeder/collectors/DataTransformer.java @@ -1,15 +1,13 @@ package org.gcube.data.publishing.gCatfeeder.collectors; +import java.io.Serializable; import java.util.Collection; import java.util.Set; -import org.gcube.data.publishing.gCatFeeder.model.FormatData; import org.gcube.data.publishing.gCatfeeder.collectors.model.CustomData; -import org.gcube.data.publishing.gCatfeeder.collectors.model.PublisherFormatData; -public interface DataTransformer { +public interface DataTransformer { - public FormatData getTargetFormat(); - + public Set transform(Collection collectedData); } diff --git a/collectors-plugin-framework/src/main/java/org/gcube/data/publishing/gCatfeeder/collectors/model/PublisherFormatData.java b/collectors-plugin-framework/src/main/java/org/gcube/data/publishing/gCatfeeder/collectors/model/PublisherFormatData.java deleted file mode 100644 index b36abe1..0000000 --- a/collectors-plugin-framework/src/main/java/org/gcube/data/publishing/gCatfeeder/collectors/model/PublisherFormatData.java +++ /dev/null @@ -1,12 +0,0 @@ -package org.gcube.data.publishing.gCatfeeder.collectors.model; - -import org.gcube.data.publishing.gCatFeeder.model.FormatData; - -public interface PublisherFormatData { - - - public FormatData getFormat(); - - public String serialize(); - -} diff --git a/collectors-plugin-framework/src/main/java/org/gcube/data/publishing/gCatfeeder/collectors/model/faults/CatalogueNotFound.java b/collectors-plugin-framework/src/main/java/org/gcube/data/publishing/gCatfeeder/collectors/model/faults/CatalogueInstanceNotFound.java similarity index 51% rename from collectors-plugin-framework/src/main/java/org/gcube/data/publishing/gCatfeeder/collectors/model/faults/CatalogueNotFound.java rename to collectors-plugin-framework/src/main/java/org/gcube/data/publishing/gCatfeeder/collectors/model/faults/CatalogueInstanceNotFound.java index 1cd1d2e..3901673 100644 --- a/collectors-plugin-framework/src/main/java/org/gcube/data/publishing/gCatfeeder/collectors/model/faults/CatalogueNotFound.java +++ b/collectors-plugin-framework/src/main/java/org/gcube/data/publishing/gCatfeeder/collectors/model/faults/CatalogueInstanceNotFound.java @@ -1,27 +1,27 @@ package org.gcube.data.publishing.gCatfeeder.collectors.model.faults; -public class CatalogueNotFound extends CrawlerException { +public class CatalogueInstanceNotFound extends CrawlerException { - public CatalogueNotFound() { + public CatalogueInstanceNotFound() { // TODO Auto-generated constructor stub } - public CatalogueNotFound(String message) { + public CatalogueInstanceNotFound(String message) { super(message); // TODO Auto-generated constructor stub } - public CatalogueNotFound(Throwable cause) { + public CatalogueInstanceNotFound(Throwable cause) { super(cause); // TODO Auto-generated constructor stub } - public CatalogueNotFound(String message, Throwable cause) { + public CatalogueInstanceNotFound(String message, Throwable cause) { super(message, cause); // TODO Auto-generated constructor stub } - public CatalogueNotFound(String message, Throwable cause, boolean enableSuppression, boolean writableStackTrace) { + public CatalogueInstanceNotFound(String message, Throwable cause, boolean enableSuppression, boolean writableStackTrace) { super(message, cause, enableSuppression, writableStackTrace); // TODO Auto-generated constructor stub } diff --git a/commons/pom.xml b/commons/pom.xml index c3d6ed7..e22ede1 100644 --- a/commons/pom.xml +++ b/commons/pom.xml @@ -28,6 +28,11 @@ + + org.gcube.core + common-encryption + + \ No newline at end of file diff --git a/commons/src/main/java/org/gcube/data/publishing/gCatFeeder/model/FormatData.java b/commons/src/main/java/org/gcube/data/publishing/gCatFeeder/model/FormatData.java deleted file mode 100644 index 5c34767..0000000 --- a/commons/src/main/java/org/gcube/data/publishing/gCatFeeder/model/FormatData.java +++ /dev/null @@ -1,5 +0,0 @@ -package org.gcube.data.publishing.gCatFeeder.model; - -public enum FormatData { - CKAN -} diff --git a/commons/src/main/java/org/gcube/data/publishing/gCatFeeder/utils/CommonUtils.java b/commons/src/main/java/org/gcube/data/publishing/gCatFeeder/utils/CommonUtils.java new file mode 100644 index 0000000..d22756e --- /dev/null +++ b/commons/src/main/java/org/gcube/data/publishing/gCatFeeder/utils/CommonUtils.java @@ -0,0 +1,22 @@ +package org.gcube.data.publishing.gCatFeeder.utils; + +import org.gcube.common.encryption.encrypter.StringEncrypter; + +public class CommonUtils { + + public static String decryptString(String toDecrypt){ + try{ + return StringEncrypter.getEncrypter().decrypt(toDecrypt); + }catch(Exception e) { + throw new RuntimeException("Unable to decrypt : "+toDecrypt,e); + } + } + + public static String encryptString(String toEncrypt){ + try{ + return StringEncrypter.getEncrypter().encrypt(toEncrypt); + }catch(Exception e) { + throw new RuntimeException("Unable to encrypt : "+toEncrypt,e); + } + } +} diff --git a/gCat-Feeder/pom.xml b/gCat-Feeder/pom.xml index 581f821..c4c1685 100644 --- a/gCat-Feeder/pom.xml +++ b/gCat-Feeder/pom.xml @@ -39,6 +39,12 @@ + + org.gcube.data-publishing.gCat-Feeder + catalogue-plugin-framework + [1.0.0-SNAPSHOT,2.0.0-SNAPSHOT) + + org.gcube.core diff --git a/gCat-Feeder/src/main/java/org/gcube/data/publishing/gCatFeeder/service/GCatFeederManager.java b/gCat-Feeder/src/main/java/org/gcube/data/publishing/gCatFeeder/service/GCatFeederManager.java index 2f2a8bf..77e57d7 100644 --- a/gCat-Feeder/src/main/java/org/gcube/data/publishing/gCatFeeder/service/GCatFeederManager.java +++ b/gCat-Feeder/src/main/java/org/gcube/data/publishing/gCatFeeder/service/GCatFeederManager.java @@ -1,19 +1,25 @@ package org.gcube.data.publishing.gCatFeeder.service; +import javax.inject.Inject; + +import org.gcube.data.publishing.gCatFeeder.service.engine.ExecutionManager; import org.gcube.smartgears.ApplicationManager; public class GCatFeederManager implements ApplicationManager{ + @Inject + ExecutionManager executions; + + @Override public void onInit() { - + executions.load(); } @Override public void onShutdown() { - // TODO Auto-generated method stub - + executions.stop(); } diff --git a/gCat-Feeder/src/main/java/org/gcube/data/publishing/gCatFeeder/service/engine/CatalogueControllersManager.java b/gCat-Feeder/src/main/java/org/gcube/data/publishing/gCatFeeder/service/engine/CatalogueControllersManager.java index 39e904b..fe6f68a 100644 --- a/gCat-Feeder/src/main/java/org/gcube/data/publishing/gCatFeeder/service/engine/CatalogueControllersManager.java +++ b/gCat-Feeder/src/main/java/org/gcube/data/publishing/gCatFeeder/service/engine/CatalogueControllersManager.java @@ -1,5 +1,17 @@ package org.gcube.data.publishing.gCatFeeder.service.engine; +import java.util.Set; + +import org.gcube.data.publishing.gCatFeeder.catalogues.CataloguePlugin; +import org.gcube.data.publishing.gCatFeeder.service.model.fault.CataloguePluginNotFound; +import org.gcube.data.publishing.gCatFeeder.service.model.fault.InternalError; + public interface CatalogueControllersManager { + + public Set getAvailableControllers(); + public CataloguePlugin getPluginById(String collectorId) throws CataloguePluginNotFound; + + public void init() throws InternalError; + } diff --git a/gCat-Feeder/src/main/java/org/gcube/data/publishing/gCatFeeder/service/engine/CollectorsManager.java b/gCat-Feeder/src/main/java/org/gcube/data/publishing/gCatFeeder/service/engine/CollectorsManager.java index c581f72..1e7c6a9 100644 --- a/gCat-Feeder/src/main/java/org/gcube/data/publishing/gCatFeeder/service/engine/CollectorsManager.java +++ b/gCat-Feeder/src/main/java/org/gcube/data/publishing/gCatFeeder/service/engine/CollectorsManager.java @@ -9,7 +9,7 @@ import org.gcube.data.publishing.gCatfeeder.collectors.CollectorPlugin; public interface CollectorsManager { public Set getAvailableCollectors(); - public CollectorPlugin getPluginById() throws CollectorNotFound; + public CollectorPlugin getPluginById(String collectorId) throws CollectorNotFound; public void init() throws InternalError; } diff --git a/gCat-Feeder/src/main/java/org/gcube/data/publishing/gCatFeeder/service/engine/ExecutionManager.java b/gCat-Feeder/src/main/java/org/gcube/data/publishing/gCatFeeder/service/engine/ExecutionManager.java index b67f732..f07c31d 100644 --- a/gCat-Feeder/src/main/java/org/gcube/data/publishing/gCatFeeder/service/engine/ExecutionManager.java +++ b/gCat-Feeder/src/main/java/org/gcube/data/publishing/gCatFeeder/service/engine/ExecutionManager.java @@ -1,5 +1,6 @@ package org.gcube.data.publishing.gCatFeeder.service.engine; +import org.gcube.data.publishing.gCatFeeder.service.engine.impl.ExecutionManagerConfiguration; import org.gcube.data.publishing.gCatFeeder.service.model.ExecutionDescriptor; public interface ExecutionManager { @@ -7,4 +8,8 @@ public interface ExecutionManager { public void submit(ExecutionDescriptor desc); public void stop(); + + public void load(); + + public void init(ExecutionManagerConfiguration config); } diff --git a/gCat-Feeder/src/main/java/org/gcube/data/publishing/gCatFeeder/service/engine/PersistenceManager.java b/gCat-Feeder/src/main/java/org/gcube/data/publishing/gCatFeeder/service/engine/PersistenceManager.java index 03a24a9..b193cff 100644 --- a/gCat-Feeder/src/main/java/org/gcube/data/publishing/gCatFeeder/service/engine/PersistenceManager.java +++ b/gCat-Feeder/src/main/java/org/gcube/data/publishing/gCatFeeder/service/engine/PersistenceManager.java @@ -15,5 +15,18 @@ public interface PersistenceManager { public ExecutionDescriptor getById(String id)throws PersistenceError,ElementNotFound,InvalidRequest; public Collection get(ExecutionDescriptorFilter filter)throws PersistenceError,InvalidRequest; + + // DIRECT QUERIES + public boolean update(ExecutionDescriptor toUpdate)throws PersistenceError,ElementNotFound; + + /** + * Updates status only if current status value is PENDING + * + * @param id + * @return + * @throws PersistenceError + * @throws ElementNotFound + */ + public boolean acquire(String id)throws PersistenceError,ElementNotFound; } diff --git a/gCat-Feeder/src/main/java/org/gcube/data/publishing/gCatFeeder/service/engine/impl/ExecutionManagerConfiguration.java b/gCat-Feeder/src/main/java/org/gcube/data/publishing/gCatFeeder/service/engine/impl/ExecutionManagerConfiguration.java new file mode 100644 index 0000000..dad9b35 --- /dev/null +++ b/gCat-Feeder/src/main/java/org/gcube/data/publishing/gCatFeeder/service/engine/impl/ExecutionManagerConfiguration.java @@ -0,0 +1,8 @@ +package org.gcube.data.publishing.gCatFeeder.service.engine.impl; + +public class ExecutionManagerConfiguration { + + private int threadPoolSize; + + +} diff --git a/gCat-Feeder/src/main/java/org/gcube/data/publishing/gCatFeeder/service/engine/impl/ExecutionManagerImpl.java b/gCat-Feeder/src/main/java/org/gcube/data/publishing/gCatFeeder/service/engine/impl/ExecutionManagerImpl.java new file mode 100644 index 0000000..1a6f40f --- /dev/null +++ b/gCat-Feeder/src/main/java/org/gcube/data/publishing/gCatFeeder/service/engine/impl/ExecutionManagerImpl.java @@ -0,0 +1,51 @@ +package org.gcube.data.publishing.gCatFeeder.service.engine.impl; + +import java.util.concurrent.ThreadPoolExecutor; + +import javax.inject.Singleton; + +import org.gcube.data.publishing.gCatFeeder.service.engine.ExecutionManager; +import org.gcube.data.publishing.gCatFeeder.service.model.ExecutionDescriptor; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +@Singleton +public class ExecutionManagerImpl implements ExecutionManager { + + private ThreadPoolExecutor executor=null; + private static final Logger log= LoggerFactory.getLogger(ExecutionManagerImpl.class); + + + @Override + public synchronized void submit(ExecutionDescriptor desc) { + log.debug("Checking if {} is already in queue"); + ExecutionTask toSubmit=new ExecutionTask(desc); + if(!executor.getQueue().contains(toSubmit)) { + log.trace("Inserting execution in queue {} "); + executor.execute(toSubmit); + } + } + + @Override + public void stop() { + // Clear queue + // Stop all + } + + @Override + public void load() { + // connect to persistence + // load all pending + throw new RuntimeException("NOT YET IMPLEMENTED"); + } + + @Override + public synchronized void init(ExecutionManagerConfiguration config) { + // NEED TO BE IDEMPOTENT + if(executor==null) { + throw new RuntimeException("NOT YET IMPLEMENTED"); +// executor=new ThreadPoolExecutor(corePoolSize, maximumPoolSize, keepAliveTime, unit, new ConcurrentLinkedQueue()); + } + } + +} diff --git a/gCat-Feeder/src/main/java/org/gcube/data/publishing/gCatFeeder/service/engine/impl/ExecutionTask.java b/gCat-Feeder/src/main/java/org/gcube/data/publishing/gCatFeeder/service/engine/impl/ExecutionTask.java new file mode 100644 index 0000000..c0364db --- /dev/null +++ b/gCat-Feeder/src/main/java/org/gcube/data/publishing/gCatFeeder/service/engine/impl/ExecutionTask.java @@ -0,0 +1,265 @@ +package org.gcube.data.publishing.gCatFeeder.service.engine.impl; + +import java.io.File; +import java.io.Serializable; +import java.time.Instant; +import java.util.Set; + +import javax.inject.Inject; + +import org.gcube.common.authorization.library.provider.SecurityTokenProvider; +import org.gcube.data.publishing.gCatFeeder.catalogues.CatalogueController; +import org.gcube.data.publishing.gCatFeeder.catalogues.CataloguePlugin; +import org.gcube.data.publishing.gCatFeeder.catalogues.model.PublishReport; +import org.gcube.data.publishing.gCatFeeder.catalogues.model.faults.ControllerInstantiationFault; +import org.gcube.data.publishing.gCatFeeder.catalogues.model.faults.WrongObjectFormatException; +import org.gcube.data.publishing.gCatFeeder.model.CatalogueInstanceDescriptor; +import org.gcube.data.publishing.gCatFeeder.service.engine.CatalogueControllersManager; +import org.gcube.data.publishing.gCatFeeder.service.engine.CollectorsManager; +import org.gcube.data.publishing.gCatFeeder.service.engine.PersistenceManager; +import org.gcube.data.publishing.gCatFeeder.service.model.ExecutionDescriptor; +import org.gcube.data.publishing.gCatFeeder.service.model.ExecutionStatus; +import org.gcube.data.publishing.gCatFeeder.service.model.fault.CataloguePluginNotFound; +import org.gcube.data.publishing.gCatFeeder.service.model.fault.CollectorNotFound; +import org.gcube.data.publishing.gCatFeeder.service.model.fault.InvalidRequest; +import org.gcube.data.publishing.gCatFeeder.service.model.fault.PersistenceError; +import org.gcube.data.publishing.gCatFeeder.service.model.reports.CatalogueReport; +import org.gcube.data.publishing.gCatFeeder.service.model.reports.CollectorReport; +import org.gcube.data.publishing.gCatFeeder.service.model.reports.ExecutionReport; +import org.gcube.data.publishing.gCatFeeder.utils.CommonUtils; +import org.gcube.data.publishing.gCatFeeder.utils.ContextUtils; +import org.gcube.data.publishing.gCatfeeder.collectors.CatalogueRetriever; +import org.gcube.data.publishing.gCatfeeder.collectors.CollectorPlugin; +import org.gcube.data.publishing.gCatfeeder.collectors.DataCollector; +import org.gcube.data.publishing.gCatfeeder.collectors.DataTransformer; +import org.gcube.data.publishing.gCatfeeder.collectors.model.faults.CatalogueInstanceNotFound; +import org.gcube.data.publishing.gCatfeeder.collectors.model.faults.CatalogueNotSupportedException; +import org.gcube.data.publishing.gCatfeeder.collectors.model.faults.CollectorFault; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class ExecutionTask implements Runnable { + + private static final Logger log= LoggerFactory.getLogger(ExecutionTask.class); + + private ExecutionDescriptor request; + private PersistenceManager persistence; + private CollectorsManager collectors; + private CatalogueControllersManager catalogues; + + public ExecutionTask(ExecutionDescriptor desc) { + super(); + this.request=desc; + } + + + + + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + ((request == null) ? 0 : request.hashCode()); + return result; + } + + + + + @Override + public boolean equals(Object obj) { + if (this == obj) + return true; + if (obj == null) + return false; + if (getClass() != obj.getClass()) + return false; + ExecutionTask other = (ExecutionTask) obj; + if (request == null) { + if (other.request != null) + return false; + } else if (!request.equals(other.request)) + return false; + return true; + } + + + @Inject + public void setPersistence(PersistenceManager p) { + this.persistence=p; + } + + @Inject + public void setCollectorPluginManager(CollectorsManager c) { + this.collectors=c; + } + + @Inject + public void setCataloguesPluginManager(CatalogueControllersManager c) { + this.catalogues=c; + } + + @Override + public void run() { + try { + log.debug("Starting to handle {} ",request); + + log.debug("Setting caller token.."); + String actualToken=CommonUtils.decryptString(request.getCallerEncryptedToken()); + SecurityTokenProvider.instance.set(actualToken); + + // try to lock request (FINISH ON FAIL) + if(persistence.acquire(request.getId())) { + try { + + + log.info("Acquired : {} ",request); + + ExecutionReport report=new ExecutionReport(); + report.getGenericInformations().setStartTime(Instant.now()); + report.setStartingScope(ContextUtils.getCurrentScopeName()); + + // -- ON SUCCESS reload request + request =persistence.getById(request.getId()); + + // For ALL COLLECTORS IN REQUEST + for(String collectorId : request.getCollectors()) { + CollectorReport collectorReport=handleCollector(collectorId); + report.getCollectorReports().add(collectorReport); + + } + + // FINALIZE + + report.getGenericInformations().setEndTime(Instant.now()); + + String reportUrl=storeReport(report); + log.info("Stored report at {} ",reportUrl); + request.setReportUrl(reportUrl); + request.setStatus(ExecutionStatus.SUCCESS); + persistence.update(request); + }catch(PersistenceError | InvalidRequest e) { + log.error("Unexpected exception while dealing with persistence ",e); + }catch(Throwable t) { + log.error("Unexpected generic exception.",t); + request.setStatus(ExecutionStatus.FAILED); + persistence.update(request); + } + + + + }else { + log.debug("Request [{}] is already being managed.",request); + } + }catch(Throwable t) { + log.error("THREAD CANNOT HANDLE REQUESTS!!!. ",t); + } + } + + + + private CatalogueReport handleCatalogueController(String cataloguePluginId,CollectorPlugin collectorPlugin,Set collectedData) { + CatalogueReport catalogueReport=new CatalogueReport(); + catalogueReport.getGenericInformations().setStartTime(Instant.now()); + try { + log.debug("Checking catalogue {} support ",cataloguePluginId); + if(collectorPlugin.getSupportedCatalogueTypes().contains(cataloguePluginId)) { + CataloguePlugin cataloguePlugin=catalogues.getPluginById(cataloguePluginId); + // ** INSTANTIATE CATALOGUE CONTROLLER + log.debug("Looking for catalogue instance .."); + CatalogueRetriever retriever=collectorPlugin.getRetrieverByCatalogueType(cataloguePluginId); + CatalogueInstanceDescriptor instanceDescriptor=retriever.getInstance(); + CatalogueController controller=cataloguePlugin.instantiateController(instanceDescriptor); + + // ** TRANSFORM + log.debug("Transforming Collected Data"); + DataTransformer transformer=collectorPlugin.getTransformerByCatalogueType(cataloguePluginId); + Set transformed=transformer.transform(collectedData); + log.trace("Going to publish {} items to {} ",transformed.size(),instanceDescriptor.getUrl()); + + // ** PUBLISH VIA CONTROLLER + for(Serializable item : transformed) { + try { + PublishReport itemReport=controller.publishItem(item); + }catch(WrongObjectFormatException e) { + catalogueReport.getPublishedRecords().add(new PublishReport(false,"Wrong format : "+e.getMessage())); + } + } + + catalogueReport.getGenericInformations().setSuccess(true); + catalogueReport.getGenericInformations().setGenericMessage("Data published. See detailed log."); + + }else { + catalogueReport.getGenericInformations().setSuccess(false); + catalogueReport.getGenericInformations().setGenericMessage("Catalogue not supported by Collector Plugin."); + } + }catch(CataloguePluginNotFound e) { + String msg="Supported catalogue implementation not found. Catalogue id : "+cataloguePluginId; + log.warn(msg,e); + catalogueReport.getGenericInformations().setSuccess(false); + catalogueReport.getGenericInformations().setGenericMessage(msg); + } catch (CatalogueNotSupportedException e) { + String msg="Catalogue not supported by Collector Plugin."; + log.warn(msg,e); + catalogueReport.getGenericInformations().setSuccess(false); + catalogueReport.getGenericInformations().setGenericMessage(msg); + } catch (ControllerInstantiationFault e) { + String msg="Unable to contact Catalogue instance."; + log.warn(msg,e); + catalogueReport.getGenericInformations().setSuccess(false); + catalogueReport.getGenericInformations().setGenericMessage(msg); + } catch (CatalogueInstanceNotFound e) { + String msg="Unable to find Catalogue instance."; + log.warn(msg,e); + catalogueReport.getGenericInformations().setSuccess(false); + catalogueReport.getGenericInformations().setGenericMessage(msg); + } + catalogueReport.getGenericInformations().setEndTime(Instant.now()); + return catalogueReport; + } + + + private CollectorReport handleCollector(String collectorId) { + CollectorReport collectorReport=new CollectorReport(); + collectorReport.getGenericInformations().setStartTime(Instant.now()); + collectorReport.setSource(collectorId); + + try { + // * COLLECT DATA + log.info("Starting collector {} ",collectorId); + CollectorPlugin collectorPlugin=collectors.getPluginById(collectorId); + DataCollector collector=collectorPlugin.getCollector(); + log.info("Collecting data.."); + Set collectedData=collector.collect(); + log.debug("Collected {} items. Going to transform..",collectedData.size()); + collectorReport.setCollectedItems(collectedData.size()); + + // * FOR ALL CATALOGUES IN REQUEST + for(String cataloguePluginId:request.getCatalogues()) { + CatalogueReport catalogueReport=handleCatalogueController(cataloguePluginId,collectorPlugin,collectedData); + + collectorReport.getPublisherReports().add(catalogueReport); + } + + } catch (CollectorNotFound e) { + String msg="Requested collector implementation not found."; + log.warn(msg,e); + collectorReport.getGenericInformations().setSuccess(false); + collectorReport.getGenericInformations().setGenericMessage(msg); + } catch (CollectorFault e) { + String msg="Collector Failed. "+e.getMessage(); + log.warn(msg,e); + collectorReport.getGenericInformations().setSuccess(false); + collectorReport.getGenericInformations().setGenericMessage(msg); + } + + collectorReport.getGenericInformations().setEndTime(Instant.now()); + return collectorReport; + } + + + private static final String storeReport(ExecutionReport report) { + throw new RuntimeException("Implement me"); + } + +} diff --git a/gCat-Feeder/src/main/java/org/gcube/data/publishing/gCatFeeder/service/model/ExecutionDescriptor.java b/gCat-Feeder/src/main/java/org/gcube/data/publishing/gCatFeeder/service/model/ExecutionDescriptor.java index aff44d0..9b0136f 100644 --- a/gCat-Feeder/src/main/java/org/gcube/data/publishing/gCatFeeder/service/model/ExecutionDescriptor.java +++ b/gCat-Feeder/src/main/java/org/gcube/data/publishing/gCatFeeder/service/model/ExecutionDescriptor.java @@ -1,5 +1,95 @@ package org.gcube.data.publishing.gCatFeeder.service.model; +import java.util.Set; + public class ExecutionDescriptor { + private String id; + private Set collectors; + private Set catalogues; + + + private String callerEncryptedToken; + private String callerIdentity; + private String callerContext; + + private ExecutionStatus status; + private String reportUrl; + + + public String getId() { + return id; + } + public void setId(String id) { + this.id = id; + } + public Set getCollectors() { + return collectors; + } + public void setCollectors(Set collectors) { + this.collectors = collectors; + } + public Set getCatalogues() { + return catalogues; + } + public void setCatalogues(Set catalogues) { + this.catalogues = catalogues; + } + + public String getCallerEncryptedToken() { + return callerEncryptedToken; + } + public void setCallerEncryptedToken(String callerEncryptedToken) { + this.callerEncryptedToken = callerEncryptedToken; + } + public String getCallerIdentity() { + return callerIdentity; + } + public void setCallerIdentity(String callerIdentity) { + this.callerIdentity = callerIdentity; + } + public String getCallerContext() { + return callerContext; + } + public void setCallerContext(String callerContext) { + this.callerContext = callerContext; + } + public ExecutionStatus getStatus() { + return status; + } + public void setStatus(ExecutionStatus status) { + this.status = status; + } + public String getReportUrl() { + return reportUrl; + } + public void setReportUrl(String reportUrl) { + this.reportUrl = reportUrl; + } + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + ((id == null) ? 0 : id.hashCode()); + return result; + } + @Override + public boolean equals(Object obj) { + if (this == obj) + return true; + if (obj == null) + return false; + if (getClass() != obj.getClass()) + return false; + ExecutionDescriptor other = (ExecutionDescriptor) obj; + if (id == null) { + if (other.id != null) + return false; + } else if (!id.equals(other.id)) + return false; + return true; + } + + + } diff --git a/gCat-Feeder/src/main/java/org/gcube/data/publishing/gCatFeeder/service/model/ExecutionStatus.java b/gCat-Feeder/src/main/java/org/gcube/data/publishing/gCatFeeder/service/model/ExecutionStatus.java new file mode 100644 index 0000000..07d38aa --- /dev/null +++ b/gCat-Feeder/src/main/java/org/gcube/data/publishing/gCatFeeder/service/model/ExecutionStatus.java @@ -0,0 +1,11 @@ +package org.gcube.data.publishing.gCatFeeder.service.model; + +public enum ExecutionStatus { + + PENDING, + RUNNING, + STOPPED, + FAILED, + SUCCESS + +} diff --git a/gCat-Feeder/src/main/java/org/gcube/data/publishing/gCatFeeder/service/model/fault/CataloguePluginNotFound.java b/gCat-Feeder/src/main/java/org/gcube/data/publishing/gCatFeeder/service/model/fault/CataloguePluginNotFound.java new file mode 100644 index 0000000..27a5ef2 --- /dev/null +++ b/gCat-Feeder/src/main/java/org/gcube/data/publishing/gCatFeeder/service/model/fault/CataloguePluginNotFound.java @@ -0,0 +1,37 @@ +package org.gcube.data.publishing.gCatFeeder.service.model.fault; + +public class CataloguePluginNotFound extends Exception { + + /** + * + */ + private static final long serialVersionUID = -2510068055460236480L; + + public CataloguePluginNotFound() { + super(); + // TODO Auto-generated constructor stub + } + + public CataloguePluginNotFound(String message, Throwable cause, boolean enableSuppression, boolean writableStackTrace) { + super(message, cause, enableSuppression, writableStackTrace); + // TODO Auto-generated constructor stub + } + + public CataloguePluginNotFound(String message, Throwable cause) { + super(message, cause); + // TODO Auto-generated constructor stub + } + + public CataloguePluginNotFound(String message) { + super(message); + // TODO Auto-generated constructor stub + } + + public CataloguePluginNotFound(Throwable cause) { + super(cause); + // TODO Auto-generated constructor stub + } + + + +} diff --git a/gCat-Feeder/src/main/java/org/gcube/data/publishing/gCatFeeder/service/model/reports/CatalogueReport.java b/gCat-Feeder/src/main/java/org/gcube/data/publishing/gCatFeeder/service/model/reports/CatalogueReport.java new file mode 100644 index 0000000..3cf42e0 --- /dev/null +++ b/gCat-Feeder/src/main/java/org/gcube/data/publishing/gCatFeeder/service/model/reports/CatalogueReport.java @@ -0,0 +1,26 @@ +package org.gcube.data.publishing.gCatFeeder.service.model.reports; + +import java.util.ArrayList; + +import org.gcube.data.publishing.gCatFeeder.catalogues.model.PublishReport; + +public class CatalogueReport{ + private GenericInfos genericInformations=new GenericInfos(); + private ArrayList publishedRecords=new ArrayList<>(); + + public GenericInfos getGenericInformations() { + return genericInformations; + } + public void setGenericInformations(GenericInfos genericInformations) { + this.genericInformations = genericInformations; + } + public ArrayList getPublishedRecords() { + return publishedRecords; + } + public void setPublishedRecords(ArrayList publishedRecords) { + this.publishedRecords = publishedRecords; + } + + + +} \ No newline at end of file diff --git a/gCat-Feeder/src/main/java/org/gcube/data/publishing/gCatFeeder/service/model/reports/CollectorReport.java b/gCat-Feeder/src/main/java/org/gcube/data/publishing/gCatFeeder/service/model/reports/CollectorReport.java new file mode 100644 index 0000000..0156f07 --- /dev/null +++ b/gCat-Feeder/src/main/java/org/gcube/data/publishing/gCatFeeder/service/model/reports/CollectorReport.java @@ -0,0 +1,47 @@ +package org.gcube.data.publishing.gCatFeeder.service.model.reports; + +import java.util.ArrayList; + +public class CollectorReport{ + private GenericInfos genericInformations=new GenericInfos(); + + private String source; + private long collectedItems; + + private ArrayList publisherReports=new ArrayList<>(); + + public GenericInfos getGenericInformations() { + return genericInformations; + } + + public void setGenericInformations(GenericInfos genericInformations) { + this.genericInformations = genericInformations; + } + + public String getSource() { + return source; + } + + public void setSource(String source) { + this.source = source; + } + + public long getCollectedItems() { + return collectedItems; + } + + public void setCollectedItems(long collectedItems) { + this.collectedItems = collectedItems; + } + + public ArrayList getPublisherReports() { + return publisherReports; + } + + public void setPublisherReports(ArrayList publisherReports) { + this.publisherReports = publisherReports; + } + + + +} \ No newline at end of file diff --git a/gCat-Feeder/src/main/java/org/gcube/data/publishing/gCatFeeder/service/model/reports/ExecutionReport.java b/gCat-Feeder/src/main/java/org/gcube/data/publishing/gCatFeeder/service/model/reports/ExecutionReport.java new file mode 100644 index 0000000..555daf9 --- /dev/null +++ b/gCat-Feeder/src/main/java/org/gcube/data/publishing/gCatFeeder/service/model/reports/ExecutionReport.java @@ -0,0 +1,38 @@ +package org.gcube.data.publishing.gCatFeeder.service.model.reports; + +import java.util.ArrayList; + +public class ExecutionReport { + + private GenericInfos genericInformations=new GenericInfos(); + + private String startingScope; + + private ArrayList collectorReports=new ArrayList<>(); + + public GenericInfos getGenericInformations() { + return genericInformations; + } + + public void setGenericInformations(GenericInfos genericInformations) { + this.genericInformations = genericInformations; + } + + public String getStartingScope() { + return startingScope; + } + + public void setStartingScope(String startingScope) { + this.startingScope = startingScope; + } + + public ArrayList getCollectorReports() { + return collectorReports; + } + + public void setCollectorReports(ArrayList collectorReports) { + this.collectorReports = collectorReports; + } + + +} diff --git a/gCat-Feeder/src/main/java/org/gcube/data/publishing/gCatFeeder/service/model/reports/GenericInfos.java b/gCat-Feeder/src/main/java/org/gcube/data/publishing/gCatFeeder/service/model/reports/GenericInfos.java new file mode 100644 index 0000000..aaccea6 --- /dev/null +++ b/gCat-Feeder/src/main/java/org/gcube/data/publishing/gCatFeeder/service/model/reports/GenericInfos.java @@ -0,0 +1,40 @@ +package org.gcube.data.publishing.gCatFeeder.service.model.reports; + +import java.time.Instant; + +public class GenericInfos{ + + private Instant startTime; + private Instant endTime; + private Boolean success=false; + private String genericMessage; + + + public Instant getStartTime() { + return startTime; + } + public void setStartTime(Instant startTime) { + this.startTime = startTime; + } + public Instant getEndTime() { + return endTime; + } + public void setEndTime(Instant endTime) { + this.endTime = endTime; + } + public Boolean getSuccess() { + return success; + } + public void setSuccess(Boolean success) { + this.success = success; + } + public String getGenericMessage() { + return genericMessage; + } + public void setGenericMessage(String genericMessage) { + this.genericMessage = genericMessage; + } + + + +} \ No newline at end of file