git-svn-id: http://svn.research-infrastructures.eu/public/d4science/gcube/trunk/data-publishing/gCat-Feeder-Suite@178777 82a268e6-3cf1-43bd-a215-b396298e98cf
This commit is contained in:
parent
c1b2e7d353
commit
76417a6212
|
|
@ -26,6 +26,14 @@
|
|||
</dependency>
|
||||
|
||||
|
||||
<!-- TEST -->
|
||||
<dependency>
|
||||
<groupId>org.gcube.data-publishing.gCat-Feeder</groupId>
|
||||
<artifactId>test-commons</artifactId>
|
||||
<version>[1.0.0-SNAPSHOT,2.0.0-SNAPSHOT)</version>
|
||||
</dependency>
|
||||
|
||||
|
||||
<dependency>
|
||||
<groupId>org.codehaus.jackson</groupId>
|
||||
<artifactId>jackson-mapper-asl</artifactId>
|
||||
|
|
|
|||
|
|
@ -2,6 +2,6 @@ package org.gcube.data.publishing.gCatFeeder.collectors.dm;
|
|||
|
||||
public class Constants {
|
||||
|
||||
public static final String CKAN_TYPE="CKAN";
|
||||
public static final String GCAT_TYPE="GCAT";
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@ import java.util.Set;
|
|||
|
||||
import org.gcube.data.publishing.gCatFeeder.collectors.dm.model.InternalAlgorithmDescriptor;
|
||||
import org.gcube.data.publishing.gCatFeeder.model.CatalogueFormatData;
|
||||
import org.gcube.data.publishing.gCatFeeder.model.ControllerConfiguration;
|
||||
import org.gcube.data.publishing.gCatfeeder.collectors.CatalogueRetriever;
|
||||
import org.gcube.data.publishing.gCatfeeder.collectors.CollectorPlugin;
|
||||
import org.gcube.data.publishing.gCatfeeder.collectors.DataCollector;
|
||||
|
|
@ -16,21 +17,20 @@ public class DataMinerPlugin implements CollectorPlugin<InternalAlgorithmDescrip
|
|||
|
||||
@Override
|
||||
public PluginDescriptor getDescriptor() {
|
||||
// TODO Auto-generated method stub
|
||||
return null;
|
||||
return new PluginDescriptor("DATA_MINER_ALGORITHMS");
|
||||
}
|
||||
|
||||
@Override
|
||||
public CatalogueRetriever getRetrieverByCatalogueType(String catalogueType) throws CatalogueNotSupportedException {
|
||||
switch(catalogueType) {
|
||||
case Constants.CKAN_TYPE : return CKANRetriever.get();
|
||||
case Constants.GCAT_TYPE : return GCATRetriever.get();
|
||||
default : throw new CatalogueNotSupportedException("No support for "+catalogueType);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public Set<String> getSupportedCatalogueTypes() {
|
||||
return Collections.singleton(Constants.CKAN_TYPE);
|
||||
return Collections.singleton(Constants.GCAT_TYPE);
|
||||
}
|
||||
|
||||
@Override
|
||||
|
|
@ -42,9 +42,27 @@ public class DataMinerPlugin implements CollectorPlugin<InternalAlgorithmDescrip
|
|||
public DataTransformer<? extends CatalogueFormatData, InternalAlgorithmDescriptor> getTransformerByCatalogueType(
|
||||
String catalogueType) throws CatalogueNotSupportedException {
|
||||
switch(catalogueType) {
|
||||
case Constants.CKAN_TYPE : return new CKANTransformer();
|
||||
case Constants.GCAT_TYPE : return new GCATTransformer();
|
||||
default : throw new CatalogueNotSupportedException("No support for "+catalogueType);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void init() throws Exception {
|
||||
// TODO Auto-generated method stub
|
||||
|
||||
}
|
||||
|
||||
@Override
|
||||
public void initInScope() throws Exception {
|
||||
// TODO Auto-generated method stub
|
||||
|
||||
}
|
||||
|
||||
@Override
|
||||
public ControllerConfiguration getPublisherControllerConfiguration(String catalogueType)
|
||||
throws CatalogueNotSupportedException {
|
||||
return new ControllerConfiguration();
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -3,12 +3,12 @@ package org.gcube.data.publishing.gCatFeeder.collectors.dm;
|
|||
import org.gcube.data.publishing.gCatFeeder.model.CatalogueInstanceDescriptor;
|
||||
import org.gcube.data.publishing.gCatfeeder.collectors.CatalogueRetriever;
|
||||
|
||||
public class CKANRetriever implements CatalogueRetriever {
|
||||
public class GCATRetriever implements CatalogueRetriever {
|
||||
|
||||
private static CKANRetriever instance=null;
|
||||
private static GCATRetriever instance=null;
|
||||
|
||||
static synchronized CKANRetriever get() {
|
||||
if(instance==null) instance =new CKANRetriever();
|
||||
static synchronized GCATRetriever get() {
|
||||
if(instance==null) instance =new GCATRetriever();
|
||||
return instance;
|
||||
}
|
||||
|
||||
|
|
@ -8,7 +8,7 @@ import org.gcube.data.publishing.gCatFeeder.collectors.dm.model.CkanModel;
|
|||
import org.gcube.data.publishing.gCatFeeder.collectors.dm.model.InternalAlgorithmDescriptor;
|
||||
import org.gcube.data.publishing.gCatfeeder.collectors.DataTransformer;
|
||||
|
||||
public class CKANTransformer implements DataTransformer<CkanModel,InternalAlgorithmDescriptor>{
|
||||
public class GCATTransformer implements DataTransformer<CkanModel,InternalAlgorithmDescriptor>{
|
||||
|
||||
@Override
|
||||
public Set<CkanModel> transform(Collection<InternalAlgorithmDescriptor> collectedData) {
|
||||
|
|
@ -0,0 +1 @@
|
|||
org.gcube.data.publishing.gCatFeeder.collectors.dm.DataMinerPlugin
|
||||
|
|
@ -7,49 +7,37 @@ import java.util.Set;
|
|||
import org.codehaus.jackson.JsonGenerationException;
|
||||
import org.codehaus.jackson.map.JsonMappingException;
|
||||
import org.codehaus.jackson.map.ObjectMapper;
|
||||
import org.gcube.common.authorization.library.provider.SecurityTokenProvider;
|
||||
import org.gcube.data.publishing.gCatFeeder.collectors.dm.model.InternalAlgorithmDescriptor;
|
||||
import org.gcube.data.publishing.gCatFeeder.model.CatalogueFormatData;
|
||||
import org.gcube.data.publishing.gCatFeeder.tests.BaseCollectorTest;
|
||||
import org.gcube.data.publishing.gCatfeeder.collectors.CollectorPlugin;
|
||||
import org.gcube.data.publishing.gCatfeeder.collectors.DataCollector;
|
||||
import org.gcube.data.publishing.gCatfeeder.collectors.DataTransformer;
|
||||
import org.gcube.data.publishing.gCatfeeder.collectors.model.faults.CatalogueNotSupportedException;
|
||||
import org.gcube.data.publishing.gCatfeeder.collectors.model.faults.CollectorFault;
|
||||
import org.junit.Before;
|
||||
import org.junit.BeforeClass;
|
||||
import org.junit.Test;
|
||||
|
||||
public class TranslationTest {
|
||||
public class TranslationTest extends BaseCollectorTest{
|
||||
|
||||
|
||||
@BeforeClass
|
||||
public static void checkTestEnabled() {
|
||||
|
||||
}
|
||||
|
||||
@Before
|
||||
public void setToken() {
|
||||
// disable if maven property
|
||||
// set token
|
||||
SecurityTokenProvider.instance.set("***REMOVED***");
|
||||
}
|
||||
|
||||
|
||||
@Test
|
||||
public void testTranslation() throws CollectorFault, CatalogueNotSupportedException, JsonGenerationException, JsonMappingException, IOException {
|
||||
|
||||
ObjectMapper mapper = new ObjectMapper();
|
||||
|
||||
CollectorPlugin plugin=new DataMinerPlugin();
|
||||
DataCollector collector=plugin.getCollector();
|
||||
Collection collected=collector.collect();
|
||||
System.out.println("Found "+collected.size()+" elements");
|
||||
for(Object obj:collected)
|
||||
System.out.println(mapper.writeValueAsString(obj)+"\n");
|
||||
|
||||
for(String destinationcatalogue : (Set<String>)plugin.getSupportedCatalogueTypes()) {
|
||||
DataTransformer<? extends CatalogueFormatData, InternalAlgorithmDescriptor> transformer=plugin.getTransformerByCatalogueType(destinationcatalogue);
|
||||
transformer.transform(collected);
|
||||
if(isTestInfrastructureEnabled()) {
|
||||
System.out.println("Entering Infrastructure enabled tests..");
|
||||
ObjectMapper mapper = new ObjectMapper();
|
||||
|
||||
CollectorPlugin plugin=new DataMinerPlugin();
|
||||
DataCollector collector=plugin.getCollector();
|
||||
Collection collected=collector.collect();
|
||||
System.out.println("Found "+collected.size()+" elements");
|
||||
for(Object obj:collected)
|
||||
System.out.println(mapper.writeValueAsString(obj)+"\n");
|
||||
|
||||
for(String destinationcatalogue : (Set<String>)plugin.getSupportedCatalogueTypes()) {
|
||||
DataTransformer<? extends CatalogueFormatData, InternalAlgorithmDescriptor> transformer=plugin.getTransformerByCatalogueType(destinationcatalogue);
|
||||
transformer.transform(collected);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,36 +0,0 @@
|
|||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<classpath>
|
||||
<classpathentry kind="src" output="target/classes" path="src/main/java">
|
||||
<attributes>
|
||||
<attribute name="optional" value="true"/>
|
||||
<attribute name="maven.pomderived" value="true"/>
|
||||
</attributes>
|
||||
</classpathentry>
|
||||
<classpathentry excluding="**" kind="src" output="target/classes" path="src/main/resources">
|
||||
<attributes>
|
||||
<attribute name="maven.pomderived" value="true"/>
|
||||
</attributes>
|
||||
</classpathentry>
|
||||
<classpathentry kind="src" output="target/test-classes" path="src/test/java">
|
||||
<attributes>
|
||||
<attribute name="optional" value="true"/>
|
||||
<attribute name="maven.pomderived" value="true"/>
|
||||
</attributes>
|
||||
</classpathentry>
|
||||
<classpathentry excluding="**" kind="src" output="target/test-classes" path="src/test/resources">
|
||||
<attributes>
|
||||
<attribute name="maven.pomderived" value="true"/>
|
||||
</attributes>
|
||||
</classpathentry>
|
||||
<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-1.8">
|
||||
<attributes>
|
||||
<attribute name="maven.pomderived" value="true"/>
|
||||
</attributes>
|
||||
</classpathentry>
|
||||
<classpathentry kind="con" path="org.eclipse.m2e.MAVEN2_CLASSPATH_CONTAINER">
|
||||
<attributes>
|
||||
<attribute name="maven.pomderived" value="true"/>
|
||||
</attributes>
|
||||
</classpathentry>
|
||||
<classpathentry kind="output" path="target/classes"/>
|
||||
</classpath>
|
||||
|
|
@ -1,23 +0,0 @@
|
|||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<projectDescription>
|
||||
<name>ckan-controller-plugin</name>
|
||||
<comment></comment>
|
||||
<projects>
|
||||
</projects>
|
||||
<buildSpec>
|
||||
<buildCommand>
|
||||
<name>org.eclipse.jdt.core.javabuilder</name>
|
||||
<arguments>
|
||||
</arguments>
|
||||
</buildCommand>
|
||||
<buildCommand>
|
||||
<name>org.eclipse.m2e.core.maven2Builder</name>
|
||||
<arguments>
|
||||
</arguments>
|
||||
</buildCommand>
|
||||
</buildSpec>
|
||||
<natures>
|
||||
<nature>org.eclipse.jdt.core.javanature</nature>
|
||||
<nature>org.eclipse.m2e.core.maven2Nature</nature>
|
||||
</natures>
|
||||
</projectDescription>
|
||||
|
|
@ -1,6 +0,0 @@
|
|||
eclipse.preferences.version=1
|
||||
encoding//src/main/java=UTF-8
|
||||
encoding//src/main/resources=UTF-8
|
||||
encoding//src/test/java=UTF-8
|
||||
encoding//src/test/resources=UTF-8
|
||||
encoding/<project>=UTF-8
|
||||
|
|
@ -1,5 +0,0 @@
|
|||
eclipse.preferences.version=1
|
||||
org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.8
|
||||
org.eclipse.jdt.core.compiler.compliance=1.8
|
||||
org.eclipse.jdt.core.compiler.problem.forbiddenReference=warning
|
||||
org.eclipse.jdt.core.compiler.source=1.8
|
||||
|
|
@ -1,4 +0,0 @@
|
|||
activeProfiles=
|
||||
eclipse.preferences.version=1
|
||||
resolveWorkspaceProjects=true
|
||||
version=1
|
||||
|
|
@ -1,26 +0,0 @@
|
|||
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
<parent>
|
||||
<groupId>org.gcube.data-publishing.gCat-Feeder</groupId>
|
||||
<artifactId>gCat-Feeder-Suite</artifactId>
|
||||
<version>1.0.0-SNAPSHOT</version>
|
||||
</parent>
|
||||
<artifactId>ckan-controller-plugin</artifactId>
|
||||
<name>ckan-controller-plugin</name>
|
||||
<description>controller plugin for gCat-Feeder aimed to publish metadata to CKAN catalogue</description>
|
||||
|
||||
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>org.gcube.data-publishing.gCat-Feeder</groupId>
|
||||
<artifactId>catalogue-plugin-framework</artifactId>
|
||||
<version>[1.0.0-SNAPSHOT,2.0.0-SNAPSHOT)</version>
|
||||
</dependency>
|
||||
|
||||
|
||||
|
||||
|
||||
</dependencies>
|
||||
|
||||
|
||||
</project>
|
||||
|
|
@ -1,203 +0,0 @@
|
|||
package org.gcube.data.publishing.gCatFeeder.catalogues.ckan;
|
||||
|
||||
import java.net.URL;
|
||||
import java.util.Iterator;
|
||||
|
||||
import javax.ws.rs.client.Client;
|
||||
import javax.ws.rs.client.ClientBuilder;
|
||||
import javax.ws.rs.client.Entity;
|
||||
import javax.ws.rs.core.Response;
|
||||
|
||||
import org.gcube.data.publishing.gCatFeeder.catalogues.CatalogueController;
|
||||
import org.gcube.data.publishing.gCatFeeder.catalogues.model.PublishReport;
|
||||
import org.gcube.data.publishing.gCatFeeder.catalogues.model.faults.ControllerInstantiationFault;
|
||||
import org.gcube.data.publishing.gCatFeeder.catalogues.model.faults.PublicationException;
|
||||
import org.gcube.data.publishing.gCatFeeder.catalogues.model.faults.WrongObjectFormatException;
|
||||
import org.gcube.data.publishing.gCatFeeder.model.CatalogueFormatData;
|
||||
import org.gcube.data.publishing.gCatFeeder.model.CatalogueInstanceDescriptor;
|
||||
import org.gcube.data.publishing.gCatFeeder.utils.GCubeAuthorizationFilter;
|
||||
import org.gcube.data.publishing.gCatFeeder.utils.TokenUtils;
|
||||
import org.glassfish.jersey.client.ClientConfig;
|
||||
import org.glassfish.jersey.client.ClientProperties;
|
||||
|
||||
import com.fasterxml.jackson.databind.JsonNode;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.fasterxml.jackson.databind.node.ObjectNode;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
@Slf4j
|
||||
public class CKANController implements CatalogueController {
|
||||
|
||||
private static ObjectMapper mapper=new ObjectMapper();
|
||||
|
||||
|
||||
|
||||
private static final String PUBLISH_ITEM="catalogue-ws/rest/api/items/create";
|
||||
private static final String PUBLSIH_RESOURCE="catalogue-ws/rest/api/resources/create";
|
||||
|
||||
private String hostname=null;
|
||||
private String customToken=null;
|
||||
private final int maxAttempts=3;
|
||||
|
||||
public CKANController(CatalogueInstanceDescriptor instance) throws ControllerInstantiationFault {
|
||||
try{
|
||||
log.trace("Instantiating controller towards {} ",instance);
|
||||
URL url=new URL(instance.getUrl());
|
||||
this.hostname=url.getHost();
|
||||
this.customToken=instance.getCustomToken();
|
||||
}catch(Throwable t) {
|
||||
throw new ControllerInstantiationFault("Unable to instantiate CKAN controller. ",t);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Expected structure
|
||||
*
|
||||
* { "item" : "<serialized_item>",
|
||||
* "resources" : [
|
||||
* "<serialized_resource>",
|
||||
* "<serialized_resource>",
|
||||
* ....]
|
||||
* }
|
||||
*
|
||||
* NB serialized resources are updated with "package_id" set as the published item id
|
||||
*
|
||||
*
|
||||
*/
|
||||
@Override
|
||||
public PublishReport publishItem(CatalogueFormatData toPublish) throws WrongObjectFormatException {
|
||||
//Get serialized format
|
||||
log.debug("Publishing {} ",toPublish);
|
||||
String serialized=toPublish.toCatalogueFormat();
|
||||
String toResetToken=TokenUtils.getCurrentToken();
|
||||
|
||||
if(customToken!=null) {
|
||||
log.debug("Custom token found.. Setting it..");
|
||||
TokenUtils.setToken(customToken);
|
||||
}
|
||||
|
||||
|
||||
try {
|
||||
log.debug("Checking serialized structure.. ");
|
||||
JsonNode node=mapper.readTree(serialized);
|
||||
String item=node.path("item").asText();
|
||||
String itemResp=publishItem(item);
|
||||
String itemUrl=getPublishedUrl(itemResp);
|
||||
log.info("Published item {} ",itemUrl);
|
||||
if(node.has("resources")) {
|
||||
log.debug("Publishing resources..");
|
||||
JsonNode resourcesNode=node.path("resources");
|
||||
String itemId=getId(itemResp);
|
||||
log.debug("Setting package id {} ",itemId);
|
||||
Iterator<JsonNode> it=resourcesNode.iterator();
|
||||
while(it.hasNext()) {
|
||||
JsonNode res=it.next();
|
||||
((ObjectNode)res).put("package_id",itemId);
|
||||
String resResp=publishResource(res.asText());
|
||||
String resUrl=getPublishedUrl(resResp);
|
||||
log.info("Published resource {} ",resUrl);
|
||||
}
|
||||
}
|
||||
|
||||
return new PublishReport(true, itemUrl);
|
||||
|
||||
}catch(PublicationException e) {
|
||||
log.error("Unable to Publish ",e);
|
||||
return new PublishReport(false, e.getMessage());
|
||||
}catch(Throwable t) {
|
||||
throw new WrongObjectFormatException("Unable to parse Serialized object.",t);
|
||||
}finally {
|
||||
TokenUtils.setToken(toResetToken);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
private String publishItem(String jsonItem) throws PublicationException {
|
||||
String urlString="http://"+hostname+":80/"+PUBLISH_ITEM;
|
||||
String respString=null;
|
||||
int attemptCounter=0;
|
||||
while(respString==null&&attemptCounter<maxAttempts)
|
||||
try {
|
||||
Response resp=getWebClient().target(urlString).request("application/json")
|
||||
// .header("Authorization", ckanToken)
|
||||
.post(Entity.json(jsonItem));
|
||||
respString=check(resp);
|
||||
}catch(PublicationException e) {
|
||||
attemptCounter++;
|
||||
if(attemptCounter==maxAttempts) throw new PublicationException("Unable to publish item "+jsonItem, e);
|
||||
}
|
||||
return respString;
|
||||
}
|
||||
|
||||
private String publishResource(String jsonResource) throws PublicationException {
|
||||
String urlString="http://"+hostname+":80/"+PUBLSIH_RESOURCE;
|
||||
int attemptCounter=0;
|
||||
boolean done=false;
|
||||
String respString=null;
|
||||
while(attemptCounter<maxAttempts&&(!done))
|
||||
try {
|
||||
Response resp=getWebClient().target(urlString).request("application/json")
|
||||
// .header("Authorization", ckanToken)
|
||||
.post(Entity.json(jsonResource));
|
||||
respString=check(resp);
|
||||
done=true;
|
||||
}catch(PublicationException e) {
|
||||
attemptCounter++;
|
||||
if(attemptCounter==maxAttempts) throw new PublicationException("Unable to publish resource "+jsonResource, e);
|
||||
}
|
||||
return respString;
|
||||
}
|
||||
|
||||
|
||||
|
||||
private static String check(Response resp) throws PublicationException {
|
||||
if(resp.getStatus()<200||resp.getStatus()>=300) {
|
||||
throw new PublicationException("RESP STATUS IS "+resp.getStatus()+". Message : "+resp.readEntity(String.class));
|
||||
}else {
|
||||
try {
|
||||
String respString=resp.readEntity(String.class);
|
||||
if(!mapper.readTree(respString).path("success").asBoolean())
|
||||
throw new PublicationException("Error : response is "+respString);
|
||||
return respString;
|
||||
}catch(PublicationException e) {
|
||||
throw e;
|
||||
}catch(Throwable t) {
|
||||
throw new PublicationException("Unable to check response ",t);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static String getId(String publishResponse) {
|
||||
try {
|
||||
return mapper.readTree(publishResponse).path("result").path("id").asText();
|
||||
} catch (Throwable t) {
|
||||
t.printStackTrace();
|
||||
throw new RuntimeException("FAILED Parsing of "+publishResponse);
|
||||
}
|
||||
}
|
||||
|
||||
private static String getPublishedUrl(String publishResponse) {
|
||||
try {
|
||||
Iterator<JsonNode> iterator=mapper.readTree(publishResponse).path("result").path("extras").elements();
|
||||
while(iterator.hasNext()) {
|
||||
JsonNode node=iterator.next();
|
||||
if(node.path("key").asText().equals("Item URL"))
|
||||
return node.path("value").asText();
|
||||
}
|
||||
return "N/A";
|
||||
} catch (Throwable t) {
|
||||
t.printStackTrace();
|
||||
throw new RuntimeException("FAILED Parsing of "+publishResponse);
|
||||
}
|
||||
}
|
||||
|
||||
private static Client getWebClient() {
|
||||
return ClientBuilder.newClient(new ClientConfig().register(GCubeAuthorizationFilter.class))
|
||||
.property(ClientProperties.SUPPRESS_HTTP_COMPLIANCE_VALIDATION, true);
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
|
@ -1,23 +0,0 @@
|
|||
package org.gcube.data.publishing.gCatFeeder.catalogues.ckan;
|
||||
|
||||
import org.gcube.data.publishing.gCatFeeder.catalogues.CataloguePlugin;
|
||||
import org.gcube.data.publishing.gCatFeeder.catalogues.model.CataloguePluginDescriptor;
|
||||
import org.gcube.data.publishing.gCatFeeder.catalogues.model.faults.ControllerInstantiationFault;
|
||||
import org.gcube.data.publishing.gCatFeeder.model.CatalogueInstanceDescriptor;
|
||||
|
||||
public class CkanPlugin implements CataloguePlugin{
|
||||
|
||||
@Override
|
||||
public CataloguePluginDescriptor getDescriptor() {
|
||||
return new CataloguePluginDescriptor("CKAN");
|
||||
}
|
||||
|
||||
@Override
|
||||
public CKANController instantiateController(CatalogueInstanceDescriptor desc)
|
||||
throws ControllerInstantiationFault {
|
||||
return new CKANController(desc);
|
||||
}
|
||||
|
||||
|
||||
|
||||
}
|
||||
|
|
@ -1,5 +0,0 @@
|
|||
Manifest-Version: 1.0
|
||||
Built-By: fabio
|
||||
Build-Jdk: 1.8.0_201
|
||||
Created-By: Maven Integration for Eclipse
|
||||
|
||||
|
|
@ -1,7 +0,0 @@
|
|||
#Generated by Maven Integration for Eclipse
|
||||
#Mon Mar 25 12:23:14 CET 2019
|
||||
version=1.0.0-SNAPSHOT
|
||||
groupId=org.gcube.data-publishing.gCat-Feeder
|
||||
m2e.projectName=ckan-controller-plugin
|
||||
m2e.projectLocation=/home/fabio/workspaces/DEV/gCat-Feeder-Suite/ckan-controller-plugin
|
||||
artifactId=ckan-controller-plugin
|
||||
|
|
@ -1,26 +0,0 @@
|
|||
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
<parent>
|
||||
<groupId>org.gcube.data-publishing.gCat-Feeder</groupId>
|
||||
<artifactId>gCat-Feeder-Suite</artifactId>
|
||||
<version>1.0.0-SNAPSHOT</version>
|
||||
</parent>
|
||||
<artifactId>ckan-controller-plugin</artifactId>
|
||||
<name>ckan-controller-plugin</name>
|
||||
<description>controller plugin for gCat-Feeder aimed to publish metadata to CKAN catalogue</description>
|
||||
|
||||
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>org.gcube.data-publishing.gCat-Feeder</groupId>
|
||||
<artifactId>catalogue-plugin-framework</artifactId>
|
||||
<version>[1.0.0-SNAPSHOT,2.0.0-SNAPSHOT)</version>
|
||||
</dependency>
|
||||
|
||||
|
||||
|
||||
|
||||
</dependencies>
|
||||
|
||||
|
||||
</project>
|
||||
Binary file not shown.
Binary file not shown.
|
|
@ -1,5 +1,5 @@
|
|||
#Generated by Maven Integration for Eclipse
|
||||
#Mon Mar 25 12:23:24 CET 2019
|
||||
#Thu Mar 28 15:36:44 CET 2019
|
||||
version=1.0.0-SNAPSHOT
|
||||
groupId=org.gcube.data-publishing.gCat-Feeder
|
||||
m2e.projectName=gCat-Controller
|
||||
|
|
|
|||
Binary file not shown.
Binary file not shown.
Binary file not shown.
|
|
@ -186,7 +186,8 @@ public class ExecutionTask implements Runnable {
|
|||
CatalogueRetriever retriever=collectorPlugin.getRetrieverByCatalogueType(cataloguePluginId);
|
||||
CatalogueInstanceDescriptor instanceDescriptor=retriever.getInstance();
|
||||
CatalogueController controller=cataloguePlugin.instantiateController(instanceDescriptor);
|
||||
|
||||
|
||||
controller.configure(collectorPlugin.getPublisherControllerConfiguration(cataloguePluginId));
|
||||
// ** TRANSFORM
|
||||
log.debug("Transforming Collected Data");
|
||||
DataTransformer transformer=collectorPlugin.getTransformerByCatalogueType(cataloguePluginId);
|
||||
|
|
|
|||
1
pom.xml
1
pom.xml
|
|
@ -20,7 +20,6 @@
|
|||
<module>DataMinerAlgorithmsCrawler</module>
|
||||
<module>commons</module>
|
||||
<module>catalogue-plugin-framework</module>
|
||||
<module>ckan-controller-plugin</module>
|
||||
<module>gCat-Controller</module>
|
||||
<module>test-commons</module>
|
||||
</modules>
|
||||
|
|
|
|||
|
|
@ -8,4 +8,20 @@
|
|||
<artifactId>test-commons</artifactId>
|
||||
<name>test commons</name>
|
||||
<description>Utility classes for test purposes</description>
|
||||
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>org.gcube.data-publishing.gCat-Feeder</groupId>
|
||||
<artifactId>collectors-plugin-framework</artifactId>
|
||||
<version>[1.0.0-SNAPSHOT,2.0.0-SNAPSHOT)</version>
|
||||
</dependency>
|
||||
|
||||
|
||||
<dependency>
|
||||
<groupId>org.gcube.data-publishing.gCat-Feeder</groupId>
|
||||
<artifactId>catalogue-plugin-framework</artifactId>
|
||||
<version>[1.0.0-SNAPSHOT,2.0.0-SNAPSHOT)</version>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
|
||||
</project>
|
||||
|
|
@ -0,0 +1,45 @@
|
|||
package org.gcube.data.publishing.gCatFeeder.tests;
|
||||
|
||||
import java.util.ServiceLoader;
|
||||
|
||||
import org.gcube.data.publishing.gCatFeeder.catalogues.CataloguePlugin;
|
||||
import org.gcube.data.publishing.gCatFeeder.catalogues.model.CataloguePluginDescriptor;
|
||||
import org.junit.Assert;
|
||||
import org.junit.BeforeClass;
|
||||
import org.junit.Test;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
public class BaseCataloguePluginTest {
|
||||
|
||||
private static final Logger log= LoggerFactory.getLogger(BaseCataloguePluginTest.class);
|
||||
|
||||
private static ServiceLoader<CataloguePlugin> cataloguePluginsLoader = ServiceLoader.load(CataloguePlugin.class);
|
||||
|
||||
|
||||
|
||||
@BeforeClass
|
||||
public void checkPluginRegistration() {
|
||||
int pluginsCounter=0;
|
||||
for(CataloguePlugin pl:cataloguePluginsLoader) {
|
||||
pluginsCounter++;
|
||||
}
|
||||
|
||||
Assert.assertFalse(pluginsCounter==0);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void checkImplementations() throws Exception{
|
||||
for(CataloguePlugin plugin:cataloguePluginsLoader) {
|
||||
CataloguePluginDescriptor desc=plugin.getDescriptor();
|
||||
Assert.assertNotNull(plugin.getClass()+" No Descriptor exposed", desc);
|
||||
try{
|
||||
plugin.init();
|
||||
}catch(Throwable t) {
|
||||
throw new Exception("Unable to init plugin "+desc.getId());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,86 @@
|
|||
package org.gcube.data.publishing.gCatFeeder.tests;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.Properties;
|
||||
import java.util.ServiceLoader;
|
||||
|
||||
import org.gcube.common.authorization.library.provider.SecurityTokenProvider;
|
||||
import org.gcube.common.scope.api.ScopeProvider;
|
||||
import org.gcube.data.publishing.gCatfeeder.collectors.CollectorPlugin;
|
||||
import org.gcube.data.publishing.gCatfeeder.collectors.model.CustomData;
|
||||
import org.gcube.data.publishing.gCatfeeder.collectors.model.PluginDescriptor;
|
||||
import org.gcube.data.publishing.gCatfeeder.collectors.model.faults.CatalogueNotSupportedException;
|
||||
import org.junit.Assert;
|
||||
import org.junit.BeforeClass;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
public class BaseCollectorTest {
|
||||
|
||||
private static String testContext=null;
|
||||
|
||||
private static final Logger log= LoggerFactory.getLogger(BaseCollectorTest.class);
|
||||
|
||||
private static ServiceLoader<CollectorPlugin> collectorPluginsLoader = ServiceLoader.load(CollectorPlugin.class);
|
||||
|
||||
|
||||
|
||||
@BeforeClass
|
||||
public static void checkPluginRegistration() {
|
||||
int pluginsCounter=0;
|
||||
for(CollectorPlugin pl:collectorPluginsLoader) {
|
||||
pluginsCounter++;
|
||||
}
|
||||
|
||||
Assert.assertFalse("No plugin is actually registered ",pluginsCounter==0);
|
||||
|
||||
for(CollectorPlugin<? extends CustomData> plugin:collectorPluginsLoader) {
|
||||
PluginDescriptor desc=plugin.getDescriptor();
|
||||
Assert.assertNotNull(plugin.getClass()+" No Descriptor exposed", desc);
|
||||
try{
|
||||
plugin.init();
|
||||
}catch(Throwable t) {
|
||||
Assert.fail("Unable to init plugin "+desc.getName());
|
||||
}
|
||||
|
||||
Assert.assertTrue(desc.getName()+": No catalogues supported",plugin.getSupportedCatalogueTypes().size()>0);
|
||||
|
||||
for(String supportedCatalogue:plugin.getSupportedCatalogueTypes()) {
|
||||
try{
|
||||
Assert.assertNotNull(desc.getName()+": Null configuration for exposed "+supportedCatalogue,plugin.getPublisherControllerConfiguration(supportedCatalogue));
|
||||
Assert.assertNotNull(desc.getName()+": Null retriever for exposed "+supportedCatalogue,plugin.getRetrieverByCatalogueType(supportedCatalogue));
|
||||
Assert.assertNotNull(desc.getName()+": Null transformer for exposed "+supportedCatalogue,plugin.getTransformerByCatalogueType(supportedCatalogue));
|
||||
}catch(CatalogueNotSupportedException e) {
|
||||
Assert.fail("Exposed supported catalogue actually not covered."+e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
Assert.assertNotNull(desc.getName()+": No actual collector ",plugin.getCollector());
|
||||
}
|
||||
|
||||
|
||||
// Check if instrastructure is enabled
|
||||
testContext=System.getProperty("testContext");
|
||||
System.out.println("TEST CONTEXT = "+testContext);
|
||||
|
||||
|
||||
if(isTestInfrastructureEnabled()) {
|
||||
Properties props=new Properties();
|
||||
try{
|
||||
props.load(BaseCollectorTest.class.getResourceAsStream("/tokens.properties"));
|
||||
}catch(IOException e) {throw new RuntimeException(e);}
|
||||
if(!props.containsKey(testContext)) throw new RuntimeException("No token found for scope : "+testContext);
|
||||
SecurityTokenProvider.instance.set(props.getProperty(testContext));
|
||||
ScopeProvider.instance.set(testContext);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
protected static boolean isTestInfrastructureEnabled() {
|
||||
return testContext!=null;
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -1,5 +1,5 @@
|
|||
#Generated by Maven Integration for Eclipse
|
||||
#Mon Mar 25 12:23:24 CET 2019
|
||||
#Thu Mar 28 15:36:44 CET 2019
|
||||
version=1.0.0-SNAPSHOT
|
||||
groupId=org.gcube.data-publishing.gCat-Feeder
|
||||
m2e.projectName=test-commons
|
||||
|
|
|
|||
|
|
@ -8,4 +8,20 @@
|
|||
<artifactId>test-commons</artifactId>
|
||||
<name>test commons</name>
|
||||
<description>Utility classes for test purposes</description>
|
||||
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>org.gcube.data-publishing.gCat-Feeder</groupId>
|
||||
<artifactId>collectors-plugin-framework</artifactId>
|
||||
<version>[1.0.0-SNAPSHOT,2.0.0-SNAPSHOT)</version>
|
||||
</dependency>
|
||||
|
||||
|
||||
<dependency>
|
||||
<groupId>org.gcube.data-publishing.gCat-Feeder</groupId>
|
||||
<artifactId>catalogue-plugin-framework</artifactId>
|
||||
<version>[1.0.0-SNAPSHOT,2.0.0-SNAPSHOT)</version>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
|
||||
</project>
|
||||
Binary file not shown.
Binary file not shown.
Loading…
Reference in New Issue