This commit is contained in:
Fabio Sinibaldi 2019-03-11 17:40:39 +00:00
parent 881282f9cf
commit 24209aa014
24 changed files with 385 additions and 37 deletions

View File

@ -1,8 +1,6 @@
package org.gcube.data.publishing.gCatFeeder.collectors.dm;
import org.gcube.common.resources.gcore.GCoreEndpoint;
import org.gcube.data.publishing.gCatFeeder.model.CatalogueInstanceDescriptor;
import org.gcube.data.publishing.gCatFeeder.utils.ISUtils;
import org.gcube.data.publishing.gCatfeeder.collectors.CatalogueRetriever;
public class CKANRetriever implements CatalogueRetriever {

View File

@ -7,14 +7,14 @@ import org.gcube.data.publishing.gCatFeeder.collectors.dm.model.CkanModel;
import org.gcube.data.publishing.gCatFeeder.collectors.dm.model.InternalAlgorithmDescriptor;
import org.gcube.data.publishing.gCatFeeder.model.FormatData;
import org.gcube.data.publishing.gCatfeeder.collectors.CatalogueRetriever;
import org.gcube.data.publishing.gCatfeeder.collectors.CrawlerPlugin;
import org.gcube.data.publishing.gCatfeeder.collectors.CollectorPlugin;
import org.gcube.data.publishing.gCatfeeder.collectors.DataCollector;
import org.gcube.data.publishing.gCatfeeder.collectors.DataTransformer;
import org.gcube.data.publishing.gCatfeeder.collectors.model.CustomData;
import org.gcube.data.publishing.gCatfeeder.collectors.model.PluginDescriptor;
import org.gcube.data.publishing.gCatfeeder.collectors.model.faults.CatalogueNotSupportedException;
public class DataMinerCrawler implements CrawlerPlugin<InternalAlgorithmDescriptor> {
public class DataMinerPlugin implements CollectorPlugin<InternalAlgorithmDescriptor> {
@Override
public PluginDescriptor getDescriptor() {

View File

@ -0,0 +1,36 @@
<?xml version="1.0" encoding="UTF-8"?>
<classpath>
<classpathentry kind="src" output="target/classes" path="src/main/java">
<attributes>
<attribute name="optional" value="true"/>
<attribute name="maven.pomderived" value="true"/>
</attributes>
</classpathentry>
<classpathentry excluding="**" kind="src" output="target/classes" path="src/main/resources">
<attributes>
<attribute name="maven.pomderived" value="true"/>
</attributes>
</classpathentry>
<classpathentry kind="src" output="target/test-classes" path="src/test/java">
<attributes>
<attribute name="optional" value="true"/>
<attribute name="maven.pomderived" value="true"/>
</attributes>
</classpathentry>
<classpathentry excluding="**" kind="src" output="target/test-classes" path="src/test/resources">
<attributes>
<attribute name="maven.pomderived" value="true"/>
</attributes>
</classpathentry>
<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-1.8">
<attributes>
<attribute name="maven.pomderived" value="true"/>
</attributes>
</classpathentry>
<classpathentry kind="con" path="org.eclipse.m2e.MAVEN2_CLASSPATH_CONTAINER">
<attributes>
<attribute name="maven.pomderived" value="true"/>
</attributes>
</classpathentry>
<classpathentry kind="output" path="target/classes"/>
</classpath>

View File

@ -0,0 +1,23 @@
<?xml version="1.0" encoding="UTF-8"?>
<projectDescription>
<name>catalogue-plugin-framework</name>
<comment></comment>
<projects>
</projects>
<buildSpec>
<buildCommand>
<name>org.eclipse.jdt.core.javabuilder</name>
<arguments>
</arguments>
</buildCommand>
<buildCommand>
<name>org.eclipse.m2e.core.maven2Builder</name>
<arguments>
</arguments>
</buildCommand>
</buildSpec>
<natures>
<nature>org.eclipse.jdt.core.javanature</nature>
<nature>org.eclipse.m2e.core.maven2Nature</nature>
</natures>
</projectDescription>

View File

@ -0,0 +1,6 @@
eclipse.preferences.version=1
encoding//src/main/java=UTF-8
encoding//src/main/resources=UTF-8
encoding//src/test/java=UTF-8
encoding//src/test/resources=UTF-8
encoding/<project>=UTF-8

View File

@ -0,0 +1,5 @@
eclipse.preferences.version=1
org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.8
org.eclipse.jdt.core.compiler.compliance=1.8
org.eclipse.jdt.core.compiler.problem.forbiddenReference=warning
org.eclipse.jdt.core.compiler.source=1.8

View File

@ -0,0 +1,4 @@
activeProfiles=
eclipse.preferences.version=1
resolveWorkspaceProjects=true
version=1

View File

@ -0,0 +1,23 @@
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.gcube.data-publishing.gCat-Feeder</groupId>
<artifactId>gCat-Feeder-Suite</artifactId>
<version>1.0.0-SNAPSHOT</version>
</parent>
<artifactId>catalogue-plugin-framework</artifactId>
<dependencies>
<dependency>
<groupId>org.projectlombok</groupId>
<artifactId>lombok</artifactId>
<version>1.14.8</version>
</dependency>
<dependency>
<groupId>org.gcube.data-publishing.gCat-Feeder</groupId>
<artifactId>commons</artifactId>
<version>1.0.0-SNAPSHOT</version>
</dependency>
</dependencies>
</project>

View File

@ -0,0 +1,11 @@
package org.gcube.data.publishing.gCatFeeder.catalogues;
import org.gcube.data.publishing.gCatFeeder.catalogues.model.PublishReport;
import org.gcube.data.publishing.gCatFeeder.catalogues.model.faults.WrongObjectFormatException;
public interface CatalogueController {
public PublishReport publishItem(String serializedItem) throws WrongObjectFormatException;
}

View File

@ -0,0 +1,13 @@
package org.gcube.data.publishing.gCatFeeder.catalogues;
import org.gcube.data.publishing.gCatFeeder.catalogues.model.CataloguePluginDescriptor;
import org.gcube.data.publishing.gCatFeeder.catalogues.model.faults.ControllerInstantiationFault;
import org.gcube.data.publishing.gCatFeeder.model.CatalogueInstanceDescriptor;
public interface CataloguePlugin {
public CataloguePluginDescriptor getDescriptor();
public CatalogueController instantiateController(CatalogueInstanceDescriptor desc) throws ControllerInstantiationFault;
}

View File

@ -0,0 +1,9 @@
package org.gcube.data.publishing.gCatFeeder.catalogues.model;
public class CataloguePluginDescriptor {
public CataloguePluginDescriptor() {
// TODO Auto-generated constructor stub
}
}

View File

@ -0,0 +1,16 @@
package org.gcube.data.publishing.gCatFeeder.catalogues.model;
import lombok.Getter;
import lombok.RequiredArgsConstructor;
import lombok.Setter;
@Getter
@RequiredArgsConstructor
@Setter
public class PublishReport {
private boolean successful=false;
private String publishedId;
}

View File

@ -0,0 +1,30 @@
package org.gcube.data.publishing.gCatFeeder.catalogues.model.faults;
public class ControllerInstantiationFault extends Exception {
public ControllerInstantiationFault() {
// TODO Auto-generated constructor stub
}
public ControllerInstantiationFault(String message) {
super(message);
// TODO Auto-generated constructor stub
}
public ControllerInstantiationFault(Throwable cause) {
super(cause);
// TODO Auto-generated constructor stub
}
public ControllerInstantiationFault(String message, Throwable cause) {
super(message, cause);
// TODO Auto-generated constructor stub
}
public ControllerInstantiationFault(String message, Throwable cause, boolean enableSuppression,
boolean writableStackTrace) {
super(message, cause, enableSuppression, writableStackTrace);
// TODO Auto-generated constructor stub
}
}

View File

@ -0,0 +1,30 @@
package org.gcube.data.publishing.gCatFeeder.catalogues.model.faults;
public class WrongObjectFormatException extends Exception {
public WrongObjectFormatException() {
// TODO Auto-generated constructor stub
}
public WrongObjectFormatException(String message) {
super(message);
// TODO Auto-generated constructor stub
}
public WrongObjectFormatException(Throwable cause) {
super(cause);
// TODO Auto-generated constructor stub
}
public WrongObjectFormatException(String message, Throwable cause) {
super(message, cause);
// TODO Auto-generated constructor stub
}
public WrongObjectFormatException(String message, Throwable cause, boolean enableSuppression,
boolean writableStackTrace) {
super(message, cause, enableSuppression, writableStackTrace);
// TODO Auto-generated constructor stub
}
}

View File

@ -8,7 +8,7 @@ import org.gcube.data.publishing.gCatfeeder.collectors.model.PluginDescriptor;
import org.gcube.data.publishing.gCatfeeder.collectors.model.PublisherFormatData;
import org.gcube.data.publishing.gCatfeeder.collectors.model.faults.CatalogueNotSupportedException;
public interface CrawlerPlugin<E extends CustomData>{
public interface CollectorPlugin<E extends CustomData>{
public PluginDescriptor getDescriptor();

View File

@ -31,6 +31,15 @@
<dependencies>
<dependency>
<groupId>org.gcube.data-publishing.gCat-Feeder</groupId>
<artifactId>collectors-plugin-framework</artifactId>
<version>[1.0.0-SNAPSHOT,2.0.0-SNAPSHOT)</version>
</dependency>
<dependency>
<groupId>org.gcube.core</groupId>
<artifactId>common-smartgears-app</artifactId>

View File

@ -1,9 +1,15 @@
package org.gcube.data.publishing.gCatFeeder.service.engine;
import java.util.Set;
import org.gcube.data.publishing.gCatFeeder.service.model.fault.CollectorNotFound;
import org.gcube.data.publishing.gCatFeeder.service.model.fault.InternalError;
import org.gcube.data.publishing.gCatfeeder.collectors.CollectorPlugin;
public interface CollectorsManager {
public Set<String> getAvailableCollectors();
public CollectorPlugin<?> getPluginById() throws CollectorNotFound;
public void init() throws InternalError;
}

View File

@ -5,12 +5,17 @@ import java.util.Collection;
import org.gcube.data.publishing.gCatFeeder.service.model.ExecutionDescriptor;
import org.gcube.data.publishing.gCatFeeder.service.model.ExecutionDescriptorFilter;
import org.gcube.data.publishing.gCatFeeder.service.model.ExecutionRequest;
import org.gcube.data.publishing.gCatFeeder.service.model.fault.CollectorNotFound;
import org.gcube.data.publishing.gCatFeeder.service.model.fault.DescriptorNotFound;
import org.gcube.data.publishing.gCatFeeder.service.model.fault.ElementNotFound;
import org.gcube.data.publishing.gCatFeeder.service.model.fault.InvalidRequest;
import org.gcube.data.publishing.gCatFeeder.service.model.fault.PersistenceError;
public interface FeederEngine {
public Collection<ExecutionDescriptor> get(ExecutionDescriptorFilter filter);
public ExecutionDescriptor getById(String id);
public ExecutionDescriptor submit(ExecutionRequest req);
public Collection<ExecutionDescriptor> get(ExecutionDescriptorFilter filter) throws PersistenceError, InvalidRequest;
public ExecutionDescriptor getById(String id) throws PersistenceError, ElementNotFound, InvalidRequest;
public ExecutionDescriptor submit(ExecutionRequest req) throws InternalError, CollectorNotFound, DescriptorNotFound, PersistenceError, InvalidRequest;
}

View File

@ -5,12 +5,15 @@ import java.util.Collection;
import org.gcube.data.publishing.gCatFeeder.service.model.ExecutionDescriptor;
import org.gcube.data.publishing.gCatFeeder.service.model.ExecutionDescriptorFilter;
import org.gcube.data.publishing.gCatFeeder.service.model.ExecutionRequest;
import org.gcube.data.publishing.gCatFeeder.service.model.fault.ElementNotFound;
import org.gcube.data.publishing.gCatFeeder.service.model.fault.InvalidRequest;
import org.gcube.data.publishing.gCatFeeder.service.model.fault.PersistenceError;
public interface PersistenceManager {
public ExecutionDescriptor create(ExecutionRequest request);
public ExecutionDescriptor getById(String id);
public Collection<ExecutionDescriptor> get(ExecutionDescriptorFilter filter);
public ExecutionDescriptor create(ExecutionRequest request) throws PersistenceError,InvalidRequest;
public ExecutionDescriptor getById(String id)throws PersistenceError,ElementNotFound,InvalidRequest;
public Collection<ExecutionDescriptor> get(ExecutionDescriptorFilter filter)throws PersistenceError,InvalidRequest;
}

View File

@ -12,6 +12,11 @@ import org.gcube.data.publishing.gCatFeeder.service.engine.PersistenceManager;
import org.gcube.data.publishing.gCatFeeder.service.model.ExecutionDescriptor;
import org.gcube.data.publishing.gCatFeeder.service.model.ExecutionDescriptorFilter;
import org.gcube.data.publishing.gCatFeeder.service.model.ExecutionRequest;
import org.gcube.data.publishing.gCatFeeder.service.model.fault.CollectorNotFound;
import org.gcube.data.publishing.gCatFeeder.service.model.fault.DescriptorNotFound;
import org.gcube.data.publishing.gCatFeeder.service.model.fault.ElementNotFound;
import org.gcube.data.publishing.gCatFeeder.service.model.fault.InvalidRequest;
import org.gcube.data.publishing.gCatFeeder.service.model.fault.PersistenceError;
public class FeederEngineImpl implements FeederEngine {
@ -26,29 +31,32 @@ public class FeederEngineImpl implements FeederEngine {
private PersistenceManager persistence;
@Override
public ExecutionDescriptor submit(ExecutionRequest req) {
// verify request correctness
// store in persistence : returns descriptor
// pass it to executor
// return the descriptor
throw new RuntimeException("Not YET Implemented");
public ExecutionDescriptor submit(ExecutionRequest req) throws InternalError, PersistenceError, InvalidRequest {
try{
verifyRequest(req);
ExecutionDescriptor related=persistence.create(req);
executions.submit(related);
return related;
}catch(CollectorNotFound | DescriptorNotFound e) {
throw new InvalidRequest(e);
}
}
@Override
public Collection<ExecutionDescriptor> get(ExecutionDescriptorFilter filter) {
public Collection<ExecutionDescriptor> get(ExecutionDescriptorFilter filter) throws PersistenceError, InvalidRequest {
return persistence.get(filter);
}
@Override
public ExecutionDescriptor getById(String id) {
public ExecutionDescriptor getById(String id) throws PersistenceError, ElementNotFound, InvalidRequest {
return persistence.getById(id);
}
private void verifyRequest(ExecutionRequest request) {
private void verifyRequest(ExecutionRequest request) throws InternalError,CollectorNotFound,DescriptorNotFound{
// for each requested source check if available
// for each requested target check if available
//
}
}

View File

@ -0,0 +1,29 @@
package org.gcube.data.publishing.gCatFeeder.service.model.fault;
public class ElementNotFound extends InvalidRequest {
public ElementNotFound() {
// TODO Auto-generated constructor stub
}
public ElementNotFound(String message) {
super(message);
// TODO Auto-generated constructor stub
}
public ElementNotFound(Throwable cause) {
super(cause);
// TODO Auto-generated constructor stub
}
public ElementNotFound(String message, Throwable cause) {
super(message, cause);
// TODO Auto-generated constructor stub
}
public ElementNotFound(String message, Throwable cause, boolean enableSuppression, boolean writableStackTrace) {
super(message, cause, enableSuppression, writableStackTrace);
// TODO Auto-generated constructor stub
}
}

View File

@ -0,0 +1,29 @@
package org.gcube.data.publishing.gCatFeeder.service.model.fault;
public class InvalidRequest extends Exception {
public InvalidRequest() {
// TODO Auto-generated constructor stub
}
public InvalidRequest(String message) {
super(message);
// TODO Auto-generated constructor stub
}
public InvalidRequest(Throwable cause) {
super(cause);
// TODO Auto-generated constructor stub
}
public InvalidRequest(String message, Throwable cause) {
super(message, cause);
// TODO Auto-generated constructor stub
}
public InvalidRequest(String message, Throwable cause, boolean enableSuppression, boolean writableStackTrace) {
super(message, cause, enableSuppression, writableStackTrace);
// TODO Auto-generated constructor stub
}
}

View File

@ -1,46 +1,100 @@
package org.gcube.data.publishing.gCatFeeder.service.rest;
import java.util.Collection;
import javax.inject.Inject;
import javax.ws.rs.GET;
import javax.ws.rs.POST;
import javax.ws.rs.Path;
import javax.ws.rs.PathParam;
import javax.ws.rs.Produces;
import javax.ws.rs.WebApplicationException;
import javax.ws.rs.core.GenericEntity;
import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.Response;
import org.gcube.data.publishing.gCatFeeder.service.ServiceConstants;
import org.gcube.data.publishing.gCatFeeder.service.engine.FeederEngine;
import org.gcube.data.publishing.gCatFeeder.service.model.ExecutionDescriptor;
import org.gcube.data.publishing.gCatFeeder.service.model.ExecutionDescriptorFilter;
import org.gcube.data.publishing.gCatFeeder.service.model.ExecutionRequest;
import org.gcube.data.publishing.gCatFeeder.service.model.fault.ElementNotFound;
import org.gcube.data.publishing.gCatFeeder.service.model.fault.InvalidRequest;
import org.gcube.data.publishing.gCatFeeder.service.model.fault.PersistenceError;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@Path(ServiceConstants.Executions.PATH)
public class Executions {
private static final Logger log= LoggerFactory.getLogger(Executions.class);
@Inject
private FeederEngine engine;
@POST
@Produces(MediaType.APPLICATION_JSON)
public ExecutionDescriptor submit() {
throw new WebApplicationException("Method Unavailable.", Response.Status.NOT_IMPLEMENTED);
try {
ExecutionRequest request=new ExecutionRequest();
return engine.submit(request);
} catch (PersistenceError e) {
log.warn("Unexpected Exception while talking to persistnce",e);
throw new WebApplicationException("Invalid Request.", e,Response.Status.INTERNAL_SERVER_ERROR);
} catch (InvalidRequest e) {
log.warn("Unexpected Exception ",e);
throw new WebApplicationException("Invalid Request.", e,Response.Status.BAD_REQUEST);
}catch(Throwable t) {
log.warn("Unexpected Exception ",t);
throw new WebApplicationException("Unexpected Exception.", t,Response.Status.INTERNAL_SERVER_ERROR);
}
}
@GET
@Produces(MediaType.APPLICATION_JSON)
public Response getAll() {
throw new WebApplicationException("Method Unavailable.", Response.Status.NOT_IMPLEMENTED);
try {
ExecutionDescriptorFilter filter=new ExecutionDescriptorFilter();
Collection<ExecutionDescriptor> toReturn=engine.get(filter);
GenericEntity<Collection<ExecutionDescriptor>> entity=new GenericEntity<Collection<ExecutionDescriptor>>(toReturn) {};
return Response.ok(entity).build();
} catch (PersistenceError e) {
log.warn("Unexpected Exception while talking to persistnce",e);
throw new WebApplicationException("Invalid Request.", e,Response.Status.INTERNAL_SERVER_ERROR);
} catch (InvalidRequest e) {
log.warn("Unexpected Exception ",e);
throw new WebApplicationException("Invalid Request.", e,Response.Status.BAD_REQUEST);
}catch(Throwable t) {
log.warn("Unexpected Exception ",t);
throw new WebApplicationException("Unexpected Exception.", t,Response.Status.INTERNAL_SERVER_ERROR);
}
}
@GET
@Produces(MediaType.APPLICATION_JSON)
@Path("{"+ServiceConstants.Executions.EXECUTION_ID_PARAMETER+"}")
public ExecutionDescriptor get() {
throw new WebApplicationException("Method Unavailable.", Response.Status.NOT_IMPLEMENTED);
public ExecutionDescriptor get(@PathParam(ServiceConstants.Executions.EXECUTION_ID_PARAMETER) String executionId) {
try {
return engine.getById(executionId);
} catch (PersistenceError e) {
log.warn("Unexpected Exception while talking to persistnce",e);
throw new WebApplicationException("Invalid Request.", e,Response.Status.INTERNAL_SERVER_ERROR);
} catch (ElementNotFound e) {
log.warn("Unexpected Exception ",e);
throw new WebApplicationException("Descriptor not found for "+executionId, e,Response.Status.NOT_FOUND);
} catch (InvalidRequest e) {
log.warn("Unexpected Exception ",e);
throw new WebApplicationException("Invalid Request.", e,Response.Status.BAD_REQUEST);
}catch(Throwable t) {
log.warn("Unexpected Exception ",t);
throw new WebApplicationException("Unexpected Exception.", t,Response.Status.INTERNAL_SERVER_ERROR);
}
}
}

View File

@ -19,6 +19,7 @@
<module>collectors-plugin-framework</module>
<module>DataMinerAlgorithmsCrawler</module>
<module>commons</module>
<module>catalogue-plugin-framework</module>
</modules>
<dependencyManagement>