diff --git a/DataMinerAlgorithmsCrawler/pom.xml b/DataMinerAlgorithmsCrawler/pom.xml index 944f864..a7b13f9 100644 --- a/DataMinerAlgorithmsCrawler/pom.xml +++ b/DataMinerAlgorithmsCrawler/pom.xml @@ -26,6 +26,14 @@ + + + org.gcube.data-publishing.gCat-Feeder + test-commons + [1.0.0-SNAPSHOT,2.0.0-SNAPSHOT) + + + org.codehaus.jackson jackson-mapper-asl diff --git a/DataMinerAlgorithmsCrawler/src/main/java/org/gcube/data/publishing/gCatFeeder/collectors/dm/Constants.java b/DataMinerAlgorithmsCrawler/src/main/java/org/gcube/data/publishing/gCatFeeder/collectors/dm/Constants.java index 3daf1ec..79b68ac 100644 --- a/DataMinerAlgorithmsCrawler/src/main/java/org/gcube/data/publishing/gCatFeeder/collectors/dm/Constants.java +++ b/DataMinerAlgorithmsCrawler/src/main/java/org/gcube/data/publishing/gCatFeeder/collectors/dm/Constants.java @@ -2,6 +2,6 @@ package org.gcube.data.publishing.gCatFeeder.collectors.dm; public class Constants { - public static final String CKAN_TYPE="CKAN"; + public static final String GCAT_TYPE="GCAT"; } diff --git a/DataMinerAlgorithmsCrawler/src/main/java/org/gcube/data/publishing/gCatFeeder/collectors/dm/DataMinerPlugin.java b/DataMinerAlgorithmsCrawler/src/main/java/org/gcube/data/publishing/gCatFeeder/collectors/dm/DataMinerPlugin.java index f661e55..61e8cdf 100644 --- a/DataMinerAlgorithmsCrawler/src/main/java/org/gcube/data/publishing/gCatFeeder/collectors/dm/DataMinerPlugin.java +++ b/DataMinerAlgorithmsCrawler/src/main/java/org/gcube/data/publishing/gCatFeeder/collectors/dm/DataMinerPlugin.java @@ -5,6 +5,7 @@ import java.util.Set; import org.gcube.data.publishing.gCatFeeder.collectors.dm.model.InternalAlgorithmDescriptor; import org.gcube.data.publishing.gCatFeeder.model.CatalogueFormatData; +import org.gcube.data.publishing.gCatFeeder.model.ControllerConfiguration; import org.gcube.data.publishing.gCatfeeder.collectors.CatalogueRetriever; import org.gcube.data.publishing.gCatfeeder.collectors.CollectorPlugin; import org.gcube.data.publishing.gCatfeeder.collectors.DataCollector; @@ -16,21 +17,20 @@ public class DataMinerPlugin implements CollectorPlugin getSupportedCatalogueTypes() { - return Collections.singleton(Constants.CKAN_TYPE); + return Collections.singleton(Constants.GCAT_TYPE); } @Override @@ -42,9 +42,27 @@ public class DataMinerPlugin implements CollectorPlugin getTransformerByCatalogueType( String catalogueType) throws CatalogueNotSupportedException { switch(catalogueType) { - case Constants.CKAN_TYPE : return new CKANTransformer(); + case Constants.GCAT_TYPE : return new GCATTransformer(); default : throw new CatalogueNotSupportedException("No support for "+catalogueType); } } + @Override + public void init() throws Exception { + // TODO Auto-generated method stub + + } + + @Override + public void initInScope() throws Exception { + // TODO Auto-generated method stub + + } + + @Override + public ControllerConfiguration getPublisherControllerConfiguration(String catalogueType) + throws CatalogueNotSupportedException { + return new ControllerConfiguration(); + } + } diff --git a/DataMinerAlgorithmsCrawler/src/main/java/org/gcube/data/publishing/gCatFeeder/collectors/dm/CKANRetriever.java b/DataMinerAlgorithmsCrawler/src/main/java/org/gcube/data/publishing/gCatFeeder/collectors/dm/GCATRetriever.java similarity index 69% rename from DataMinerAlgorithmsCrawler/src/main/java/org/gcube/data/publishing/gCatFeeder/collectors/dm/CKANRetriever.java rename to DataMinerAlgorithmsCrawler/src/main/java/org/gcube/data/publishing/gCatFeeder/collectors/dm/GCATRetriever.java index 1b6a221..857d394 100644 --- a/DataMinerAlgorithmsCrawler/src/main/java/org/gcube/data/publishing/gCatFeeder/collectors/dm/CKANRetriever.java +++ b/DataMinerAlgorithmsCrawler/src/main/java/org/gcube/data/publishing/gCatFeeder/collectors/dm/GCATRetriever.java @@ -3,12 +3,12 @@ package org.gcube.data.publishing.gCatFeeder.collectors.dm; import org.gcube.data.publishing.gCatFeeder.model.CatalogueInstanceDescriptor; import org.gcube.data.publishing.gCatfeeder.collectors.CatalogueRetriever; -public class CKANRetriever implements CatalogueRetriever { +public class GCATRetriever implements CatalogueRetriever { - private static CKANRetriever instance=null; + private static GCATRetriever instance=null; - static synchronized CKANRetriever get() { - if(instance==null) instance =new CKANRetriever(); + static synchronized GCATRetriever get() { + if(instance==null) instance =new GCATRetriever(); return instance; } diff --git a/DataMinerAlgorithmsCrawler/src/main/java/org/gcube/data/publishing/gCatFeeder/collectors/dm/CKANTransformer.java b/DataMinerAlgorithmsCrawler/src/main/java/org/gcube/data/publishing/gCatFeeder/collectors/dm/GCATTransformer.java similarity index 91% rename from DataMinerAlgorithmsCrawler/src/main/java/org/gcube/data/publishing/gCatFeeder/collectors/dm/CKANTransformer.java rename to DataMinerAlgorithmsCrawler/src/main/java/org/gcube/data/publishing/gCatFeeder/collectors/dm/GCATTransformer.java index 497a866..170538a 100644 --- a/DataMinerAlgorithmsCrawler/src/main/java/org/gcube/data/publishing/gCatFeeder/collectors/dm/CKANTransformer.java +++ b/DataMinerAlgorithmsCrawler/src/main/java/org/gcube/data/publishing/gCatFeeder/collectors/dm/GCATTransformer.java @@ -8,7 +8,7 @@ import org.gcube.data.publishing.gCatFeeder.collectors.dm.model.CkanModel; import org.gcube.data.publishing.gCatFeeder.collectors.dm.model.InternalAlgorithmDescriptor; import org.gcube.data.publishing.gCatfeeder.collectors.DataTransformer; -public class CKANTransformer implements DataTransformer{ +public class GCATTransformer implements DataTransformer{ @Override public Set transform(Collection collectedData) { diff --git a/DataMinerAlgorithmsCrawler/src/main/resources/META-INF/services/org.gcube.data.publishing.gCatfeeder.collectors.CollectorPlugin b/DataMinerAlgorithmsCrawler/src/main/resources/META-INF/services/org.gcube.data.publishing.gCatfeeder.collectors.CollectorPlugin new file mode 100644 index 0000000..9c0d63d --- /dev/null +++ b/DataMinerAlgorithmsCrawler/src/main/resources/META-INF/services/org.gcube.data.publishing.gCatfeeder.collectors.CollectorPlugin @@ -0,0 +1 @@ +org.gcube.data.publishing.gCatFeeder.collectors.dm.DataMinerPlugin \ No newline at end of file diff --git a/DataMinerAlgorithmsCrawler/src/test/java/org/gcube/data/publishing/gCatFeeder/collectors/dm/TranslationTest.java b/DataMinerAlgorithmsCrawler/src/test/java/org/gcube/data/publishing/gCatFeeder/collectors/dm/TranslationTest.java index 81ca9ab..7f7d8dc 100644 --- a/DataMinerAlgorithmsCrawler/src/test/java/org/gcube/data/publishing/gCatFeeder/collectors/dm/TranslationTest.java +++ b/DataMinerAlgorithmsCrawler/src/test/java/org/gcube/data/publishing/gCatFeeder/collectors/dm/TranslationTest.java @@ -7,49 +7,37 @@ import java.util.Set; import org.codehaus.jackson.JsonGenerationException; import org.codehaus.jackson.map.JsonMappingException; import org.codehaus.jackson.map.ObjectMapper; -import org.gcube.common.authorization.library.provider.SecurityTokenProvider; import org.gcube.data.publishing.gCatFeeder.collectors.dm.model.InternalAlgorithmDescriptor; import org.gcube.data.publishing.gCatFeeder.model.CatalogueFormatData; +import org.gcube.data.publishing.gCatFeeder.tests.BaseCollectorTest; import org.gcube.data.publishing.gCatfeeder.collectors.CollectorPlugin; import org.gcube.data.publishing.gCatfeeder.collectors.DataCollector; import org.gcube.data.publishing.gCatfeeder.collectors.DataTransformer; import org.gcube.data.publishing.gCatfeeder.collectors.model.faults.CatalogueNotSupportedException; import org.gcube.data.publishing.gCatfeeder.collectors.model.faults.CollectorFault; -import org.junit.Before; -import org.junit.BeforeClass; import org.junit.Test; -public class TranslationTest { +public class TranslationTest extends BaseCollectorTest{ + - @BeforeClass - public static void checkTestEnabled() { - - } - - @Before - public void setToken() { - // disable if maven property - // set token - SecurityTokenProvider.instance.set("***REMOVED***"); - } - - @Test public void testTranslation() throws CollectorFault, CatalogueNotSupportedException, JsonGenerationException, JsonMappingException, IOException { - - ObjectMapper mapper = new ObjectMapper(); - - CollectorPlugin plugin=new DataMinerPlugin(); - DataCollector collector=plugin.getCollector(); - Collection collected=collector.collect(); - System.out.println("Found "+collected.size()+" elements"); - for(Object obj:collected) - System.out.println(mapper.writeValueAsString(obj)+"\n"); - - for(String destinationcatalogue : (Set)plugin.getSupportedCatalogueTypes()) { - DataTransformer transformer=plugin.getTransformerByCatalogueType(destinationcatalogue); - transformer.transform(collected); + if(isTestInfrastructureEnabled()) { + System.out.println("Entering Infrastructure enabled tests.."); + ObjectMapper mapper = new ObjectMapper(); + + CollectorPlugin plugin=new DataMinerPlugin(); + DataCollector collector=plugin.getCollector(); + Collection collected=collector.collect(); + System.out.println("Found "+collected.size()+" elements"); + for(Object obj:collected) + System.out.println(mapper.writeValueAsString(obj)+"\n"); + + for(String destinationcatalogue : (Set)plugin.getSupportedCatalogueTypes()) { + DataTransformer transformer=plugin.getTransformerByCatalogueType(destinationcatalogue); + transformer.transform(collected); + } } } - + } diff --git a/ckan-controller-plugin/.classpath b/ckan-controller-plugin/.classpath deleted file mode 100644 index fae1a2b..0000000 --- a/ckan-controller-plugin/.classpath +++ /dev/null @@ -1,36 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/ckan-controller-plugin/.project b/ckan-controller-plugin/.project deleted file mode 100644 index 29e74b7..0000000 --- a/ckan-controller-plugin/.project +++ /dev/null @@ -1,23 +0,0 @@ - - - ckan-controller-plugin - - - - - - org.eclipse.jdt.core.javabuilder - - - - - org.eclipse.m2e.core.maven2Builder - - - - - - org.eclipse.jdt.core.javanature - org.eclipse.m2e.core.maven2Nature - - diff --git a/ckan-controller-plugin/.settings/org.eclipse.core.resources.prefs b/ckan-controller-plugin/.settings/org.eclipse.core.resources.prefs deleted file mode 100644 index 29abf99..0000000 --- a/ckan-controller-plugin/.settings/org.eclipse.core.resources.prefs +++ /dev/null @@ -1,6 +0,0 @@ -eclipse.preferences.version=1 -encoding//src/main/java=UTF-8 -encoding//src/main/resources=UTF-8 -encoding//src/test/java=UTF-8 -encoding//src/test/resources=UTF-8 -encoding/=UTF-8 diff --git a/ckan-controller-plugin/.settings/org.eclipse.jdt.core.prefs b/ckan-controller-plugin/.settings/org.eclipse.jdt.core.prefs deleted file mode 100644 index 714351a..0000000 --- a/ckan-controller-plugin/.settings/org.eclipse.jdt.core.prefs +++ /dev/null @@ -1,5 +0,0 @@ -eclipse.preferences.version=1 -org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.8 -org.eclipse.jdt.core.compiler.compliance=1.8 -org.eclipse.jdt.core.compiler.problem.forbiddenReference=warning -org.eclipse.jdt.core.compiler.source=1.8 diff --git a/ckan-controller-plugin/.settings/org.eclipse.m2e.core.prefs b/ckan-controller-plugin/.settings/org.eclipse.m2e.core.prefs deleted file mode 100644 index f897a7f..0000000 --- a/ckan-controller-plugin/.settings/org.eclipse.m2e.core.prefs +++ /dev/null @@ -1,4 +0,0 @@ -activeProfiles= -eclipse.preferences.version=1 -resolveWorkspaceProjects=true -version=1 diff --git a/ckan-controller-plugin/pom.xml b/ckan-controller-plugin/pom.xml deleted file mode 100644 index 623869d..0000000 --- a/ckan-controller-plugin/pom.xml +++ /dev/null @@ -1,26 +0,0 @@ - - 4.0.0 - - org.gcube.data-publishing.gCat-Feeder - gCat-Feeder-Suite - 1.0.0-SNAPSHOT - - ckan-controller-plugin - ckan-controller-plugin - controller plugin for gCat-Feeder aimed to publish metadata to CKAN catalogue - - - - - org.gcube.data-publishing.gCat-Feeder - catalogue-plugin-framework - [1.0.0-SNAPSHOT,2.0.0-SNAPSHOT) - - - - - - - - - \ No newline at end of file diff --git a/ckan-controller-plugin/src/main/java/org/gcube/data/publishing/gCatFeeder/catalogues/ckan/CKANController.java b/ckan-controller-plugin/src/main/java/org/gcube/data/publishing/gCatFeeder/catalogues/ckan/CKANController.java deleted file mode 100644 index 0585358..0000000 --- a/ckan-controller-plugin/src/main/java/org/gcube/data/publishing/gCatFeeder/catalogues/ckan/CKANController.java +++ /dev/null @@ -1,203 +0,0 @@ -package org.gcube.data.publishing.gCatFeeder.catalogues.ckan; - -import java.net.URL; -import java.util.Iterator; - -import javax.ws.rs.client.Client; -import javax.ws.rs.client.ClientBuilder; -import javax.ws.rs.client.Entity; -import javax.ws.rs.core.Response; - -import org.gcube.data.publishing.gCatFeeder.catalogues.CatalogueController; -import org.gcube.data.publishing.gCatFeeder.catalogues.model.PublishReport; -import org.gcube.data.publishing.gCatFeeder.catalogues.model.faults.ControllerInstantiationFault; -import org.gcube.data.publishing.gCatFeeder.catalogues.model.faults.PublicationException; -import org.gcube.data.publishing.gCatFeeder.catalogues.model.faults.WrongObjectFormatException; -import org.gcube.data.publishing.gCatFeeder.model.CatalogueFormatData; -import org.gcube.data.publishing.gCatFeeder.model.CatalogueInstanceDescriptor; -import org.gcube.data.publishing.gCatFeeder.utils.GCubeAuthorizationFilter; -import org.gcube.data.publishing.gCatFeeder.utils.TokenUtils; -import org.glassfish.jersey.client.ClientConfig; -import org.glassfish.jersey.client.ClientProperties; - -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.node.ObjectNode; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class CKANController implements CatalogueController { - - private static ObjectMapper mapper=new ObjectMapper(); - - - - private static final String PUBLISH_ITEM="catalogue-ws/rest/api/items/create"; - private static final String PUBLSIH_RESOURCE="catalogue-ws/rest/api/resources/create"; - - private String hostname=null; - private String customToken=null; - private final int maxAttempts=3; - - public CKANController(CatalogueInstanceDescriptor instance) throws ControllerInstantiationFault { - try{ - log.trace("Instantiating controller towards {} ",instance); - URL url=new URL(instance.getUrl()); - this.hostname=url.getHost(); - this.customToken=instance.getCustomToken(); - }catch(Throwable t) { - throw new ControllerInstantiationFault("Unable to instantiate CKAN controller. ",t); - } - } - - - /** - * Expected structure - * - * { "item" : "", - * "resources" : [ - * "", - * "", - * ....] - * } - * - * NB serialized resources are updated with "package_id" set as the published item id - * - * - */ - @Override - public PublishReport publishItem(CatalogueFormatData toPublish) throws WrongObjectFormatException { - //Get serialized format - log.debug("Publishing {} ",toPublish); - String serialized=toPublish.toCatalogueFormat(); - String toResetToken=TokenUtils.getCurrentToken(); - - if(customToken!=null) { - log.debug("Custom token found.. Setting it.."); - TokenUtils.setToken(customToken); - } - - - try { - log.debug("Checking serialized structure.. "); - JsonNode node=mapper.readTree(serialized); - String item=node.path("item").asText(); - String itemResp=publishItem(item); - String itemUrl=getPublishedUrl(itemResp); - log.info("Published item {} ",itemUrl); - if(node.has("resources")) { - log.debug("Publishing resources.."); - JsonNode resourcesNode=node.path("resources"); - String itemId=getId(itemResp); - log.debug("Setting package id {} ",itemId); - Iterator it=resourcesNode.iterator(); - while(it.hasNext()) { - JsonNode res=it.next(); - ((ObjectNode)res).put("package_id",itemId); - String resResp=publishResource(res.asText()); - String resUrl=getPublishedUrl(resResp); - log.info("Published resource {} ",resUrl); - } - } - - return new PublishReport(true, itemUrl); - - }catch(PublicationException e) { - log.error("Unable to Publish ",e); - return new PublishReport(false, e.getMessage()); - }catch(Throwable t) { - throw new WrongObjectFormatException("Unable to parse Serialized object.",t); - }finally { - TokenUtils.setToken(toResetToken); - } - } - - - - private String publishItem(String jsonItem) throws PublicationException { - String urlString="http://"+hostname+":80/"+PUBLISH_ITEM; - String respString=null; - int attemptCounter=0; - while(respString==null&&attemptCounter=300) { - throw new PublicationException("RESP STATUS IS "+resp.getStatus()+". Message : "+resp.readEntity(String.class)); - }else { - try { - String respString=resp.readEntity(String.class); - if(!mapper.readTree(respString).path("success").asBoolean()) - throw new PublicationException("Error : response is "+respString); - return respString; - }catch(PublicationException e) { - throw e; - }catch(Throwable t) { - throw new PublicationException("Unable to check response ",t); - } - } - } - - private static String getId(String publishResponse) { - try { - return mapper.readTree(publishResponse).path("result").path("id").asText(); - } catch (Throwable t) { - t.printStackTrace(); - throw new RuntimeException("FAILED Parsing of "+publishResponse); - } - } - - private static String getPublishedUrl(String publishResponse) { - try { - Iterator iterator=mapper.readTree(publishResponse).path("result").path("extras").elements(); - while(iterator.hasNext()) { - JsonNode node=iterator.next(); - if(node.path("key").asText().equals("Item URL")) - return node.path("value").asText(); - } - return "N/A"; - } catch (Throwable t) { - t.printStackTrace(); - throw new RuntimeException("FAILED Parsing of "+publishResponse); - } - } - - private static Client getWebClient() { - return ClientBuilder.newClient(new ClientConfig().register(GCubeAuthorizationFilter.class)) - .property(ClientProperties.SUPPRESS_HTTP_COMPLIANCE_VALIDATION, true); - } - - -} diff --git a/ckan-controller-plugin/src/main/java/org/gcube/data/publishing/gCatFeeder/catalogues/ckan/CkanPlugin.java b/ckan-controller-plugin/src/main/java/org/gcube/data/publishing/gCatFeeder/catalogues/ckan/CkanPlugin.java deleted file mode 100644 index bb1b958..0000000 --- a/ckan-controller-plugin/src/main/java/org/gcube/data/publishing/gCatFeeder/catalogues/ckan/CkanPlugin.java +++ /dev/null @@ -1,23 +0,0 @@ -package org.gcube.data.publishing.gCatFeeder.catalogues.ckan; - -import org.gcube.data.publishing.gCatFeeder.catalogues.CataloguePlugin; -import org.gcube.data.publishing.gCatFeeder.catalogues.model.CataloguePluginDescriptor; -import org.gcube.data.publishing.gCatFeeder.catalogues.model.faults.ControllerInstantiationFault; -import org.gcube.data.publishing.gCatFeeder.model.CatalogueInstanceDescriptor; - -public class CkanPlugin implements CataloguePlugin{ - - @Override - public CataloguePluginDescriptor getDescriptor() { - return new CataloguePluginDescriptor("CKAN"); - } - - @Override - public CKANController instantiateController(CatalogueInstanceDescriptor desc) - throws ControllerInstantiationFault { - return new CKANController(desc); - } - - - -} diff --git a/ckan-controller-plugin/target/classes/META-INF/MANIFEST.MF b/ckan-controller-plugin/target/classes/META-INF/MANIFEST.MF deleted file mode 100644 index e2601c5..0000000 --- a/ckan-controller-plugin/target/classes/META-INF/MANIFEST.MF +++ /dev/null @@ -1,5 +0,0 @@ -Manifest-Version: 1.0 -Built-By: fabio -Build-Jdk: 1.8.0_201 -Created-By: Maven Integration for Eclipse - diff --git a/ckan-controller-plugin/target/classes/META-INF/maven/org.gcube.data-publishing.gCat-Feeder/ckan-controller-plugin/pom.properties b/ckan-controller-plugin/target/classes/META-INF/maven/org.gcube.data-publishing.gCat-Feeder/ckan-controller-plugin/pom.properties deleted file mode 100644 index da3f57b..0000000 --- a/ckan-controller-plugin/target/classes/META-INF/maven/org.gcube.data-publishing.gCat-Feeder/ckan-controller-plugin/pom.properties +++ /dev/null @@ -1,7 +0,0 @@ -#Generated by Maven Integration for Eclipse -#Mon Mar 25 12:23:14 CET 2019 -version=1.0.0-SNAPSHOT -groupId=org.gcube.data-publishing.gCat-Feeder -m2e.projectName=ckan-controller-plugin -m2e.projectLocation=/home/fabio/workspaces/DEV/gCat-Feeder-Suite/ckan-controller-plugin -artifactId=ckan-controller-plugin diff --git a/ckan-controller-plugin/target/classes/META-INF/maven/org.gcube.data-publishing.gCat-Feeder/ckan-controller-plugin/pom.xml b/ckan-controller-plugin/target/classes/META-INF/maven/org.gcube.data-publishing.gCat-Feeder/ckan-controller-plugin/pom.xml deleted file mode 100644 index 623869d..0000000 --- a/ckan-controller-plugin/target/classes/META-INF/maven/org.gcube.data-publishing.gCat-Feeder/ckan-controller-plugin/pom.xml +++ /dev/null @@ -1,26 +0,0 @@ - - 4.0.0 - - org.gcube.data-publishing.gCat-Feeder - gCat-Feeder-Suite - 1.0.0-SNAPSHOT - - ckan-controller-plugin - ckan-controller-plugin - controller plugin for gCat-Feeder aimed to publish metadata to CKAN catalogue - - - - - org.gcube.data-publishing.gCat-Feeder - catalogue-plugin-framework - [1.0.0-SNAPSHOT,2.0.0-SNAPSHOT) - - - - - - - - - \ No newline at end of file diff --git a/ckan-controller-plugin/target/classes/org/gcube/data/publishing/gCatFeeder/catalogues/ckan/CKANController.class b/ckan-controller-plugin/target/classes/org/gcube/data/publishing/gCatFeeder/catalogues/ckan/CKANController.class deleted file mode 100644 index d9229fb..0000000 Binary files a/ckan-controller-plugin/target/classes/org/gcube/data/publishing/gCatFeeder/catalogues/ckan/CKANController.class and /dev/null differ diff --git a/ckan-controller-plugin/target/classes/org/gcube/data/publishing/gCatFeeder/catalogues/ckan/CkanPlugin.class b/ckan-controller-plugin/target/classes/org/gcube/data/publishing/gCatFeeder/catalogues/ckan/CkanPlugin.class deleted file mode 100644 index df3ef6f..0000000 Binary files a/ckan-controller-plugin/target/classes/org/gcube/data/publishing/gCatFeeder/catalogues/ckan/CkanPlugin.class and /dev/null differ diff --git a/gCat-Controller/target/classes/META-INF/maven/org.gcube.data-publishing.gCat-Feeder/gCat-Controller/pom.properties b/gCat-Controller/target/classes/META-INF/maven/org.gcube.data-publishing.gCat-Feeder/gCat-Controller/pom.properties index 4171c98..d5fb37e 100644 --- a/gCat-Controller/target/classes/META-INF/maven/org.gcube.data-publishing.gCat-Feeder/gCat-Controller/pom.properties +++ b/gCat-Controller/target/classes/META-INF/maven/org.gcube.data-publishing.gCat-Feeder/gCat-Controller/pom.properties @@ -1,5 +1,5 @@ #Generated by Maven Integration for Eclipse -#Mon Mar 25 12:23:24 CET 2019 +#Thu Mar 28 15:36:44 CET 2019 version=1.0.0-SNAPSHOT groupId=org.gcube.data-publishing.gCat-Feeder m2e.projectName=gCat-Controller diff --git a/gCat-Controller/target/classes/org/gcube/data/publishing/gCataFeeder/catalogues/gCat/GCatController.class b/gCat-Controller/target/classes/org/gcube/data/publishing/gCataFeeder/catalogues/gCat/GCatController.class index 449854b..1825a95 100644 Binary files a/gCat-Controller/target/classes/org/gcube/data/publishing/gCataFeeder/catalogues/gCat/GCatController.class and b/gCat-Controller/target/classes/org/gcube/data/publishing/gCataFeeder/catalogues/gCat/GCatController.class differ diff --git a/gCat-Controller/target/classes/org/gcube/data/publishing/gCataFeeder/catalogues/gCat/GCatPlugin.class b/gCat-Controller/target/classes/org/gcube/data/publishing/gCataFeeder/catalogues/gCat/GCatPlugin.class index 81317e1..9c4ec64 100644 Binary files a/gCat-Controller/target/classes/org/gcube/data/publishing/gCataFeeder/catalogues/gCat/GCatPlugin.class and b/gCat-Controller/target/classes/org/gcube/data/publishing/gCataFeeder/catalogues/gCat/GCatPlugin.class differ diff --git a/gCat-Controller/target/test-classes/org/gcube/data/publishing/gCatFeeder/catalogues/gCat/Interactions.class b/gCat-Controller/target/test-classes/org/gcube/data/publishing/gCatFeeder/catalogues/gCat/Interactions.class index 3dad7a9..912f015 100644 Binary files a/gCat-Controller/target/test-classes/org/gcube/data/publishing/gCatFeeder/catalogues/gCat/Interactions.class and b/gCat-Controller/target/test-classes/org/gcube/data/publishing/gCatFeeder/catalogues/gCat/Interactions.class differ diff --git a/gCat-Feeder/src/main/java/org/gcube/data/publishing/gCatFeeder/service/engine/impl/ExecutionTask.java b/gCat-Feeder/src/main/java/org/gcube/data/publishing/gCatFeeder/service/engine/impl/ExecutionTask.java index 55796d0..5f696d0 100644 --- a/gCat-Feeder/src/main/java/org/gcube/data/publishing/gCatFeeder/service/engine/impl/ExecutionTask.java +++ b/gCat-Feeder/src/main/java/org/gcube/data/publishing/gCatFeeder/service/engine/impl/ExecutionTask.java @@ -186,7 +186,8 @@ public class ExecutionTask implements Runnable { CatalogueRetriever retriever=collectorPlugin.getRetrieverByCatalogueType(cataloguePluginId); CatalogueInstanceDescriptor instanceDescriptor=retriever.getInstance(); CatalogueController controller=cataloguePlugin.instantiateController(instanceDescriptor); - + + controller.configure(collectorPlugin.getPublisherControllerConfiguration(cataloguePluginId)); // ** TRANSFORM log.debug("Transforming Collected Data"); DataTransformer transformer=collectorPlugin.getTransformerByCatalogueType(cataloguePluginId); diff --git a/pom.xml b/pom.xml index dbc38e8..a9b556f 100644 --- a/pom.xml +++ b/pom.xml @@ -20,7 +20,6 @@ DataMinerAlgorithmsCrawler commons catalogue-plugin-framework - ckan-controller-plugin gCat-Controller test-commons diff --git a/test-commons/pom.xml b/test-commons/pom.xml index f69b76e..d079e9d 100644 --- a/test-commons/pom.xml +++ b/test-commons/pom.xml @@ -8,4 +8,20 @@ test-commons test commons Utility classes for test purposes + + + + org.gcube.data-publishing.gCat-Feeder + collectors-plugin-framework + [1.0.0-SNAPSHOT,2.0.0-SNAPSHOT) + + + + + org.gcube.data-publishing.gCat-Feeder + catalogue-plugin-framework + [1.0.0-SNAPSHOT,2.0.0-SNAPSHOT) + + + \ No newline at end of file diff --git a/test-commons/src/main/java/org/gcube/data/publishing/gCatFeeder/tests/BaseCataloguePluginTest.java b/test-commons/src/main/java/org/gcube/data/publishing/gCatFeeder/tests/BaseCataloguePluginTest.java new file mode 100644 index 0000000..1a0ab3f --- /dev/null +++ b/test-commons/src/main/java/org/gcube/data/publishing/gCatFeeder/tests/BaseCataloguePluginTest.java @@ -0,0 +1,45 @@ +package org.gcube.data.publishing.gCatFeeder.tests; + +import java.util.ServiceLoader; + +import org.gcube.data.publishing.gCatFeeder.catalogues.CataloguePlugin; +import org.gcube.data.publishing.gCatFeeder.catalogues.model.CataloguePluginDescriptor; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class BaseCataloguePluginTest { + + private static final Logger log= LoggerFactory.getLogger(BaseCataloguePluginTest.class); + + private static ServiceLoader cataloguePluginsLoader = ServiceLoader.load(CataloguePlugin.class); + + + + @BeforeClass + public void checkPluginRegistration() { + int pluginsCounter=0; + for(CataloguePlugin pl:cataloguePluginsLoader) { + pluginsCounter++; + } + + Assert.assertFalse(pluginsCounter==0); + } + + @Test + public void checkImplementations() throws Exception{ + for(CataloguePlugin plugin:cataloguePluginsLoader) { + CataloguePluginDescriptor desc=plugin.getDescriptor(); + Assert.assertNotNull(plugin.getClass()+" No Descriptor exposed", desc); + try{ + plugin.init(); + }catch(Throwable t) { + throw new Exception("Unable to init plugin "+desc.getId()); + } + } + } + + +} diff --git a/test-commons/src/main/java/org/gcube/data/publishing/gCatFeeder/tests/BaseCollectorTest.java b/test-commons/src/main/java/org/gcube/data/publishing/gCatFeeder/tests/BaseCollectorTest.java new file mode 100644 index 0000000..4ad5cc7 --- /dev/null +++ b/test-commons/src/main/java/org/gcube/data/publishing/gCatFeeder/tests/BaseCollectorTest.java @@ -0,0 +1,86 @@ +package org.gcube.data.publishing.gCatFeeder.tests; + +import java.io.IOException; +import java.util.Properties; +import java.util.ServiceLoader; + +import org.gcube.common.authorization.library.provider.SecurityTokenProvider; +import org.gcube.common.scope.api.ScopeProvider; +import org.gcube.data.publishing.gCatfeeder.collectors.CollectorPlugin; +import org.gcube.data.publishing.gCatfeeder.collectors.model.CustomData; +import org.gcube.data.publishing.gCatfeeder.collectors.model.PluginDescriptor; +import org.gcube.data.publishing.gCatfeeder.collectors.model.faults.CatalogueNotSupportedException; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class BaseCollectorTest { + + private static String testContext=null; + + private static final Logger log= LoggerFactory.getLogger(BaseCollectorTest.class); + + private static ServiceLoader collectorPluginsLoader = ServiceLoader.load(CollectorPlugin.class); + + + + @BeforeClass + public static void checkPluginRegistration() { + int pluginsCounter=0; + for(CollectorPlugin pl:collectorPluginsLoader) { + pluginsCounter++; + } + + Assert.assertFalse("No plugin is actually registered ",pluginsCounter==0); + + for(CollectorPlugin plugin:collectorPluginsLoader) { + PluginDescriptor desc=plugin.getDescriptor(); + Assert.assertNotNull(plugin.getClass()+" No Descriptor exposed", desc); + try{ + plugin.init(); + }catch(Throwable t) { + Assert.fail("Unable to init plugin "+desc.getName()); + } + + Assert.assertTrue(desc.getName()+": No catalogues supported",plugin.getSupportedCatalogueTypes().size()>0); + + for(String supportedCatalogue:plugin.getSupportedCatalogueTypes()) { + try{ + Assert.assertNotNull(desc.getName()+": Null configuration for exposed "+supportedCatalogue,plugin.getPublisherControllerConfiguration(supportedCatalogue)); + Assert.assertNotNull(desc.getName()+": Null retriever for exposed "+supportedCatalogue,plugin.getRetrieverByCatalogueType(supportedCatalogue)); + Assert.assertNotNull(desc.getName()+": Null transformer for exposed "+supportedCatalogue,plugin.getTransformerByCatalogueType(supportedCatalogue)); + }catch(CatalogueNotSupportedException e) { + Assert.fail("Exposed supported catalogue actually not covered."+e.getMessage()); + } + } + + Assert.assertNotNull(desc.getName()+": No actual collector ",plugin.getCollector()); + } + + + // Check if instrastructure is enabled + testContext=System.getProperty("testContext"); + System.out.println("TEST CONTEXT = "+testContext); + + + if(isTestInfrastructureEnabled()) { + Properties props=new Properties(); + try{ + props.load(BaseCollectorTest.class.getResourceAsStream("/tokens.properties")); + }catch(IOException e) {throw new RuntimeException(e);} + if(!props.containsKey(testContext)) throw new RuntimeException("No token found for scope : "+testContext); + SecurityTokenProvider.instance.set(props.getProperty(testContext)); + ScopeProvider.instance.set(testContext); + } + } + + + + + +protected static boolean isTestInfrastructureEnabled() { + return testContext!=null; +} + +} diff --git a/test-commons/target/classes/META-INF/maven/org.gcube.data-publishing.gCat-Feeder/test-commons/pom.properties b/test-commons/target/classes/META-INF/maven/org.gcube.data-publishing.gCat-Feeder/test-commons/pom.properties index 8227f35..f860194 100644 --- a/test-commons/target/classes/META-INF/maven/org.gcube.data-publishing.gCat-Feeder/test-commons/pom.properties +++ b/test-commons/target/classes/META-INF/maven/org.gcube.data-publishing.gCat-Feeder/test-commons/pom.properties @@ -1,5 +1,5 @@ #Generated by Maven Integration for Eclipse -#Mon Mar 25 12:23:24 CET 2019 +#Thu Mar 28 15:36:44 CET 2019 version=1.0.0-SNAPSHOT groupId=org.gcube.data-publishing.gCat-Feeder m2e.projectName=test-commons diff --git a/test-commons/target/classes/META-INF/maven/org.gcube.data-publishing.gCat-Feeder/test-commons/pom.xml b/test-commons/target/classes/META-INF/maven/org.gcube.data-publishing.gCat-Feeder/test-commons/pom.xml index f69b76e..d079e9d 100644 --- a/test-commons/target/classes/META-INF/maven/org.gcube.data-publishing.gCat-Feeder/test-commons/pom.xml +++ b/test-commons/target/classes/META-INF/maven/org.gcube.data-publishing.gCat-Feeder/test-commons/pom.xml @@ -8,4 +8,20 @@ test-commons test commons Utility classes for test purposes + + + + org.gcube.data-publishing.gCat-Feeder + collectors-plugin-framework + [1.0.0-SNAPSHOT,2.0.0-SNAPSHOT) + + + + + org.gcube.data-publishing.gCat-Feeder + catalogue-plugin-framework + [1.0.0-SNAPSHOT,2.0.0-SNAPSHOT) + + + \ No newline at end of file diff --git a/test-commons/target/classes/org/gcube/data/publishing/gCatFeeder/tests/BaseCataloguePluginTest.class b/test-commons/target/classes/org/gcube/data/publishing/gCatFeeder/tests/BaseCataloguePluginTest.class new file mode 100644 index 0000000..47f5b5b Binary files /dev/null and b/test-commons/target/classes/org/gcube/data/publishing/gCatFeeder/tests/BaseCataloguePluginTest.class differ diff --git a/test-commons/target/classes/org/gcube/data/publishing/gCatFeeder/tests/BaseCollectorTest.class b/test-commons/target/classes/org/gcube/data/publishing/gCatFeeder/tests/BaseCollectorTest.class new file mode 100644 index 0000000..df7ad41 Binary files /dev/null and b/test-commons/target/classes/org/gcube/data/publishing/gCatFeeder/tests/BaseCollectorTest.class differ