This commit is contained in:
Fabio Sinibaldi 2019-03-21 16:25:40 +00:00
parent 6f2014cfeb
commit a17e6756fe
21 changed files with 483 additions and 20 deletions

View File

@ -62,7 +62,6 @@
<dependency>
<groupId>javax.ws.rs</groupId>
<artifactId>javax.ws.rs-api</artifactId>
<version>2.0</version>
</dependency>
@ -72,6 +71,20 @@
<artifactId>jersey-container-servlet</artifactId>
</dependency>
<!-- test -->
<dependency>
<groupId>org.glassfish.jersey.test-framework.providers</groupId>
<artifactId>jersey-test-framework-provider-simple</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>ch.qos.logback</groupId>
<artifactId>logback-classic</artifactId>
<version>1.0.13</version>
<scope>test</scope>
</dependency>
</dependencies>
</project>

View File

@ -2,8 +2,15 @@ package org.gcube.data.publishing.gCatFeeder.service;
import javax.ws.rs.ApplicationPath;
import org.gcube.data.publishing.gCatFeeder.service.engine.CatalogueControllersManager;
import org.gcube.data.publishing.gCatFeeder.service.engine.CollectorsManager;
import org.gcube.data.publishing.gCatFeeder.service.engine.ExecutionManager;
import org.gcube.data.publishing.gCatFeeder.service.engine.FeederEngine;
import org.gcube.data.publishing.gCatFeeder.service.engine.impl.CatalogueControllersManagerImpl;
import org.gcube.data.publishing.gCatFeeder.service.engine.impl.CollectorsManagerImpl;
import org.gcube.data.publishing.gCatFeeder.service.engine.impl.ExecutionManagerImpl;
import org.gcube.data.publishing.gCatFeeder.service.engine.impl.FeederEngineImpl;
import org.gcube.data.publishing.gCatFeeder.service.rest.Capabilities;
import org.gcube.data.publishing.gCatFeeder.service.rest.Executions;
import org.glassfish.hk2.utilities.binding.AbstractBinder;
import org.glassfish.jersey.server.ResourceConfig;
@ -18,13 +25,14 @@ public class GCatFeeder extends ResourceConfig{
@Override
protected void configure() {
bind(FeederEngineImpl.class).to(FeederEngine.class);
// bind(ImporterImpl.class).to(Importer.class);
// bind(PerformanceManagerImpl.class).to(PerformanceManager.class);
bind(CatalogueControllersManagerImpl.class).to(CatalogueControllersManager.class);
bind(CollectorsManagerImpl.class).to(CollectorsManager.class);
bind(ExecutionManagerImpl.class).to(ExecutionManager.class);
}
};
register(binder);
registerClasses(Executions.class);
registerClasses(Capabilities.class);
}
}

View File

@ -1,7 +1,12 @@
package org.gcube.data.publishing.gCatFeeder.service;
import javax.inject.Inject;
import javax.xml.bind.annotation.XmlRootElement;
import org.gcube.data.publishing.gCatFeeder.service.engine.CatalogueControllersManager;
import org.gcube.data.publishing.gCatFeeder.service.engine.CollectorsManager;
import org.gcube.data.publishing.gCatFeeder.service.engine.ExecutionManager;
import org.gcube.data.publishing.gCatFeeder.service.model.fault.InternalError;
import org.gcube.smartgears.handlers.application.ApplicationLifecycleHandler;
import org.gcube.smartgears.handlers.application.ApplicationLifecycleEvent.Start;
import org.gcube.smartgears.handlers.application.ApplicationLifecycleEvent.Stop;
@ -9,17 +14,38 @@ import org.gcube.smartgears.handlers.application.ApplicationLifecycleEvent.Stop;
@XmlRootElement(name = "gcatFeeder-lifecycle")
public class GCatFeederLyfeCycleManager extends ApplicationLifecycleHandler{
@Inject
ExecutionManager executions;
@Inject
CollectorsManager collectors;
@Inject
CatalogueControllersManager controllers;
@Override
public void onStart(Start e) {
// TODO Auto-generated method stub
super.onStart(e);
try {
collectors.initInScope();
controllers.initInScope();
} catch (InternalError ex) {
throw new RuntimeException("Initialization Error",ex);
}
executions.load();
}
@Override
public void onStop(Stop e) {
// TODO Auto-generated method stub
public void onStop(Stop e) {
super.onStop(e);
}
}

View File

@ -2,18 +2,32 @@ package org.gcube.data.publishing.gCatFeeder.service;
import javax.inject.Inject;
import org.gcube.data.publishing.gCatFeeder.service.engine.CatalogueControllersManager;
import org.gcube.data.publishing.gCatFeeder.service.engine.CollectorsManager;
import org.gcube.data.publishing.gCatFeeder.service.engine.ExecutionManager;
import org.gcube.data.publishing.gCatFeeder.service.model.fault.InternalError;
import org.gcube.smartgears.ApplicationManager;
public class GCatFeederManager implements ApplicationManager{
@Inject
ExecutionManager executions;
@Inject
CollectorsManager collectors;
@Inject
CatalogueControllersManager controllers;
@Override
public void onInit() {
executions.load();
try {
collectors.init();
controllers.init();
} catch (InternalError e) {
throw new RuntimeException("Initialization Error",e);
}
}

View File

@ -10,6 +10,10 @@ public class ServiceConstants {
public static final String PATH="execution";
public static final String EXECUTION_ID_PARAMETER="executionId";
public static final String COLLECTOR_ID_PARAMETER="collector";
public static final String CATALOGUE_ID_PARAMETER="collector";
}
public static interface Capabilities{

View File

@ -23,7 +23,7 @@ public class CatalogueControllersManagerImpl implements CatalogueControllersMana
private ServiceLoader<CataloguePlugin> cataloguePluginsLoader = null;
private ConcurrentHashMap<String,CataloguePlugin> availablePlugins=null;
private ConcurrentHashMap<String,CataloguePlugin> availablePlugins=new ConcurrentHashMap<String, CataloguePlugin>();
public CatalogueControllersManagerImpl() {

View File

@ -26,7 +26,7 @@ public class CollectorsManagerImpl implements CollectorsManager {
private ServiceLoader<CollectorPlugin> collectorPluginsLoader = null;
private ConcurrentHashMap<String,CollectorPlugin> availablePlugins=null;
private ConcurrentHashMap<String,CollectorPlugin> availablePlugins=new ConcurrentHashMap<>();
public CollectorsManagerImpl() {

View File

@ -1,6 +1,7 @@
package org.gcube.data.publishing.gCatFeeder.service.engine.impl;
import java.util.Collection;
import java.util.Set;
import javax.inject.Inject;
@ -24,7 +25,7 @@ public class FeederEngineImpl implements FeederEngine {
@Inject
private ExecutionManager executions;
@Inject
private CollectorsManager crawlers;
private CollectorsManager collectors;
@Inject
private CatalogueControllersManager catalogues;
@Inject
@ -54,8 +55,24 @@ public class FeederEngineImpl implements FeederEngine {
private void verifyRequest(ExecutionRequest request) throws InternalError,CollectorNotFound,DescriptorNotFound{
// for each requested source check if available
// for each requested target check if available
Set<String> availableControllers=catalogues.getAvailableControllers();
if(request.getToInvokeControllers().size()==1&&request.getToInvokeControllers().contains("ALL")) {
request.setToInvokeControllers(availableControllers);
}
for(String requestedCatalogue:request.getToInvokeControllers())
if(!availableControllers.contains(requestedCatalogue))
throw new DescriptorNotFound("Requested catalogue controller "+requestedCatalogue+" not found.");
Set<String> availableCollectors=collectors.getAvailableCollectors();
if(request.getToInvokeCollectors().size()==1&&request.getToInvokeCollectors().contains("ALL")) {
request.setToInvokeCollectors(availableControllers);
}
for(String requestedCollector:request.getToInvokeCollectors())
if(!availableCollectors.contains(requestedCollector))
throw new DescriptorNotFound("Requested collector "+requestedCollector+" not found.");
}

View File

@ -0,0 +1,46 @@
package org.gcube.data.publishing.gCatFeeder.service.engine.impl;
import java.util.Collection;
import org.gcube.data.publishing.gCatFeeder.service.engine.PersistenceManager;
import org.gcube.data.publishing.gCatFeeder.service.model.ExecutionDescriptor;
import org.gcube.data.publishing.gCatFeeder.service.model.ExecutionDescriptorFilter;
import org.gcube.data.publishing.gCatFeeder.service.model.ExecutionRequest;
import org.gcube.data.publishing.gCatFeeder.service.model.fault.ElementNotFound;
import org.gcube.data.publishing.gCatFeeder.service.model.fault.InvalidRequest;
import org.gcube.data.publishing.gCatFeeder.service.model.fault.PersistenceError;
public class PersistenceManagerImpl implements PersistenceManager {
@Override
public ExecutionDescriptor create(ExecutionRequest request) throws PersistenceError, InvalidRequest {
// TODO Auto-generated method stub
return null;
}
@Override
public ExecutionDescriptor getById(String id) throws PersistenceError, ElementNotFound, InvalidRequest {
// TODO Auto-generated method stub
return null;
}
@Override
public Collection<ExecutionDescriptor> get(ExecutionDescriptorFilter filter)
throws PersistenceError, InvalidRequest {
// TODO Auto-generated method stub
return null;
}
@Override
public boolean update(ExecutionDescriptor toUpdate) throws PersistenceError, ElementNotFound {
// TODO Auto-generated method stub
return false;
}
@Override
public boolean acquire(String id) throws PersistenceError, ElementNotFound {
// TODO Auto-generated method stub
return false;
}
}

View File

@ -1,5 +1,39 @@
package org.gcube.data.publishing.gCatFeeder.service.model;
import java.util.HashSet;
import java.util.Set;
public class ExecutionRequest {
private Set<String> toInvokeCollectors=new HashSet<>();
private Set<String> toInvokeControllers=new HashSet<>();
public ExecutionRequest addCollectorId(String id) {
this.toInvokeCollectors.add(id);
return this;
}
public ExecutionRequest addControllerId(String id) {
this.toInvokeControllers.add(id);
return this;
}
public Set<String> getToInvokeCollectors() {
return toInvokeCollectors;
}
public Set<String> getToInvokeControllers() {
return toInvokeControllers;
}
public void setToInvokeCollectors(Set<String> toInvokeCollectors) {
this.toInvokeCollectors = toInvokeCollectors;
}
public void setToInvokeControllers(Set<String> toInvokeControllers) {
this.toInvokeControllers = toInvokeControllers;
}
}

View File

@ -23,7 +23,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ManagedBy(GCatFeederManager.class)
@Path(ServiceConstants.Executions.PATH)
@Path(ServiceConstants.Capabilities.PATH)
public class Capabilities {
private static final Logger log= LoggerFactory.getLogger(Capabilities.class);

View File

@ -9,9 +9,11 @@ import javax.ws.rs.Path;
import javax.ws.rs.PathParam;
import javax.ws.rs.Produces;
import javax.ws.rs.WebApplicationException;
import javax.ws.rs.core.Context;
import javax.ws.rs.core.GenericEntity;
import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.Response;
import javax.ws.rs.core.UriInfo;
import org.gcube.data.publishing.gCatFeeder.service.GCatFeederManager;
import org.gcube.data.publishing.gCatFeeder.service.ServiceConstants;
@ -39,11 +41,19 @@ public class Executions {
@POST
@Produces(MediaType.APPLICATION_JSON)
public ExecutionDescriptor submit() {
public ExecutionDescriptor submit(@Context UriInfo info) {
try {
ExecutionRequest request=new ExecutionRequest();
return engine.submit(request);
for(String collector:info.getQueryParameters().get(ServiceConstants.Executions.COLLECTOR_ID_PARAMETER))
request.addCollectorId(collector);
for(String catalogue:info.getQueryParameters().get(ServiceConstants.Executions.CATALOGUE_ID_PARAMETER))
request.addCollectorId(catalogue);
log.trace("Submitting request {} ",request);
return engine.submit(request);
} catch (PersistenceError e) {
log.warn("Unexpected Exception while talking to persistnce",e);
throw new WebApplicationException("Invalid Request.", e,Response.Status.INTERNAL_SERVER_ERROR);
@ -53,8 +63,7 @@ public class Executions {
}catch(Throwable t) {
log.warn("Unexpected Exception ",t);
throw new WebApplicationException("Unexpected Exception.", t,Response.Status.INTERNAL_SERVER_ERROR);
}
}
}

View File

@ -0,0 +1,26 @@
package org.gcube.data.publishing.gCatFeeder.service;
import java.io.IOException;
import java.sql.SQLException;
import javax.ws.rs.core.Application;
import org.glassfish.jersey.test.JerseyTest;
import org.junit.BeforeClass;
public class BaseTest extends JerseyTest{
@BeforeClass
public static void init() throws IOException, SQLException{
}
@Override
protected Application configure() {
return new GCatFeeder();
}
}

View File

@ -0,0 +1,33 @@
package org.gcube.data.publishing.gCatFeeder.service;
import javax.ws.rs.client.WebTarget;
import javax.ws.rs.core.Response;
import org.junit.Test;
public class CapabilitiesTest extends BaseTest {
@Test
public void getCollectors() {
WebTarget target=
target(ServiceConstants.Capabilities.PATH).path(ServiceConstants.Capabilities.COLLECTORS_PATH);
System.out.println(target.getUri());
Response resp=target.request().get();
System.out.println(resp.getStatus() + " : "+ resp.readEntity(String.class));
if(resp.getStatus()!=200) throw new RuntimeException("Capabilities interface error should never happen");
}
@Test
public void getControllers() {
WebTarget target=
target(ServiceConstants.Capabilities.PATH).path(ServiceConstants.Capabilities.CATALOGUES_PATH);
System.out.println(target.getUri());
Response resp=target.request().get();
System.out.println(resp.getStatus() + " : "+ resp.readEntity(String.class));
if(resp.getStatus()!=200) throw new RuntimeException("Capabilities interface error should never happen");
}
}

View File

@ -0,0 +1,10 @@
package org.gcube.data.publishing.gCatFeeder.service;
public class TestCommon {
public static final String FAKE_CATALOGUE_ID="FAKE_CATALOGUE";
public static final String FAKE_COLLECTOR_NAME="FAKE_COLLECTOR";
public static final String UNAVAILABLE_CATALOGUE="UNAVAILABLE_CATALOGUE";
}

View File

@ -0,0 +1,71 @@
package org.gcube.data.publishing.gCatFeeder.service.mockups;
import org.gcube.data.publishing.gCatFeeder.catalogues.CatalogueController;
import org.gcube.data.publishing.gCatFeeder.catalogues.CataloguePlugin;
import org.gcube.data.publishing.gCatFeeder.catalogues.model.CataloguePluginDescriptor;
import org.gcube.data.publishing.gCatFeeder.catalogues.model.PublishReport;
import org.gcube.data.publishing.gCatFeeder.catalogues.model.faults.CatalogueInteractionException;
import org.gcube.data.publishing.gCatFeeder.catalogues.model.faults.ControllerInstantiationFault;
import org.gcube.data.publishing.gCatFeeder.catalogues.model.faults.PublicationException;
import org.gcube.data.publishing.gCatFeeder.catalogues.model.faults.WrongObjectFormatException;
import org.gcube.data.publishing.gCatFeeder.model.CatalogueFormatData;
import org.gcube.data.publishing.gCatFeeder.model.CatalogueInstanceDescriptor;
import org.gcube.data.publishing.gCatFeeder.model.ControllerConfiguration;
import org.gcube.data.publishing.gCatFeeder.service.TestCommon;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class FakeCataloguePlugin implements CataloguePlugin {
private static final Logger log= LoggerFactory.getLogger(FakeCataloguePlugin.class);
private class FakeController implements CatalogueController{
@Override
public PublishReport publishItem(CatalogueFormatData toPublish)
throws WrongObjectFormatException, CatalogueInteractionException, PublicationException {
try {
log.debug("Simulating publish wait..");
Thread.sleep(400);
} catch (InterruptedException e) {
}
return new PublishReport(true, "FAKE CONTROLLER DOES NOT PUBLISH DATA");
}
@Override
public void configure(ControllerConfiguration config) {
}
}
@Override
public CataloguePluginDescriptor getDescriptor() {
return new CataloguePluginDescriptor(TestCommon.FAKE_CATALOGUE_ID);
}
@Override
public CatalogueController instantiateController(CatalogueInstanceDescriptor desc)
throws ControllerInstantiationFault {
return new FakeController();
}
@Override
public void init() throws Exception {
log.debug("Simulating init...");
try{
Thread.sleep(400);
}catch(InterruptedException e) {}
}
@Override
public void initInScope() throws Exception {
log.debug("Simulating init in scope...");
try{
Thread.sleep(400);
}catch(InterruptedException e) {}
}
}

View File

@ -0,0 +1,112 @@
package org.gcube.data.publishing.gCatFeeder.service.mockups;
import java.util.Arrays;
import java.util.Collection;
import java.util.HashSet;
import java.util.Set;
import org.gcube.data.publishing.gCatFeeder.model.CatalogueFormatData;
import org.gcube.data.publishing.gCatFeeder.model.CatalogueInstanceDescriptor;
import org.gcube.data.publishing.gCatFeeder.model.ControllerConfiguration;
import org.gcube.data.publishing.gCatFeeder.service.TestCommon;
import org.gcube.data.publishing.gCatfeeder.collectors.CatalogueRetriever;
import org.gcube.data.publishing.gCatfeeder.collectors.CollectorPlugin;
import org.gcube.data.publishing.gCatfeeder.collectors.DataCollector;
import org.gcube.data.publishing.gCatfeeder.collectors.DataTransformer;
import org.gcube.data.publishing.gCatfeeder.collectors.model.PluginDescriptor;
import org.gcube.data.publishing.gCatfeeder.collectors.model.faults.CatalogueInstanceNotFound;
import org.gcube.data.publishing.gCatfeeder.collectors.model.faults.CatalogueNotSupportedException;
import org.gcube.data.publishing.gCatfeeder.collectors.model.faults.CollectorFault;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class FakeCollectorPlugin implements CollectorPlugin<FakeCustomData> {
private static final Logger log= LoggerFactory.getLogger(FakeCollectorPlugin.class);
@Override
public PluginDescriptor getDescriptor() {
return new PluginDescriptor(TestCommon.FAKE_COLLECTOR_NAME);
}
@Override
public CatalogueRetriever getRetrieverByCatalogueType(String catalogueType) throws CatalogueNotSupportedException {
switch(catalogueType) {
case TestCommon.UNAVAILABLE_CATALOGUE :
case TestCommon.FAKE_CATALOGUE_ID : return new CatalogueRetriever() {
@Override
public CatalogueInstanceDescriptor getInstance() throws CatalogueInstanceNotFound {
// TODO Auto-generated method stub
return null;
}
};
default : throw new CatalogueNotSupportedException(catalogueType+" not supported by "+TestCommon.FAKE_COLLECTOR_NAME);
}
}
@Override
public Set<String> getSupportedCatalogueTypes() {
return new HashSet<String> (Arrays.asList(new String[] {
TestCommon.FAKE_CATALOGUE_ID,TestCommon.UNAVAILABLE_CATALOGUE}));
}
@Override
public DataTransformer<? extends CatalogueFormatData, FakeCustomData> getTransformerByCatalogueType(
String catalogueType) throws CatalogueNotSupportedException {
switch(catalogueType) {
case TestCommon.UNAVAILABLE_CATALOGUE :
case TestCommon.FAKE_CATALOGUE_ID : return new DataTransformer<FakeCustomData,FakeCustomData>() {
@Override
public Set<FakeCustomData> transform(Collection<FakeCustomData> collectedData) {
return new HashSet<>(collectedData);
}
};
default : throw new CatalogueNotSupportedException(catalogueType+" not supported by "+TestCommon.FAKE_COLLECTOR_NAME);
}
}
@Override
public DataCollector<FakeCustomData> getCollector() {
return new DataCollector<FakeCustomData>() {
@Override
public Set<FakeCustomData> collect() throws CollectorFault {
HashSet<FakeCustomData> toReturn=new HashSet<>();
for(int i=0;i<1000;i++)
toReturn.add(new FakeCustomData());
return toReturn;
}
};
}
@Override
public ControllerConfiguration getPublisherControllerConfiguration(String catalogueType)
throws CatalogueNotSupportedException {
switch(catalogueType) {
case TestCommon.UNAVAILABLE_CATALOGUE :
case TestCommon.FAKE_CATALOGUE_ID : return new ControllerConfiguration();
default : throw new CatalogueNotSupportedException(catalogueType+" not supported by "+TestCommon.FAKE_COLLECTOR_NAME);
}
}
@Override
public void init() throws Exception {
log.debug("Simulating init...");
try{
Thread.sleep(400);
}catch(InterruptedException e) {}
}
@Override
public void initInScope() throws Exception {
log.debug("Simulating init in scope...");
try{
Thread.sleep(400);
}catch(InterruptedException e) {}
}
}

View File

@ -0,0 +1,22 @@
package org.gcube.data.publishing.gCatFeeder.service.mockups;
import java.util.concurrent.atomic.AtomicLong;
import org.gcube.data.publishing.gCatFeeder.model.CatalogueFormatData;
import org.gcube.data.publishing.gCatfeeder.collectors.model.CustomData;
public class FakeCustomData implements CustomData, CatalogueFormatData{
private static final AtomicLong counter=new AtomicLong(0);
private long number=counter.incrementAndGet();
@Override
public String toCatalogueFormat() {
return "DATA "+number;
}
}

View File

@ -0,0 +1 @@
org.gcube.data.publishing.gCatFeeder.service.mockups.FakeCataloguePlugin

View File

@ -0,0 +1 @@
org.gcube.data.publishing.gCatFeeder.service.mockups.FakeCollectorPlugin

View File

@ -0,0 +1,16 @@
<configuration debug="true">
<appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
<!-- encoders are assigned the type
ch.qos.logback.classic.encoder.PatternLayoutEncoder by default -->
<encoder>
<pattern>%d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n</pattern>
</encoder>
</appender>
<logger name="org.gcube" level="DEBUG"/>
<root level="WARN">
<appender-ref ref="STDOUT" />
</root>
</configuration>