From 20a57b80dc748094472e67f298f888eb6c0bf09f Mon Sep 17 00:00:00 2001 From: "fabio.sinibaldi" Date: Thu, 14 Mar 2019 15:58:03 +0000 Subject: [PATCH] git-svn-id: http://svn.research-infrastructures.eu/public/d4science/gcube/trunk/data-publishing/gCat-Feeder-Suite@178587 82a268e6-3cf1-43bd-a215-b396298e98cf --- DataMinerAlgorithmsCrawler/pom.xml | 10 ++ .../collectors/dm/CKANTransformer.java | 9 +- .../gCatFeeder/collectors/dm/Constants.java | 7 ++ .../dm/DMAlgorithmsInfoCollector.java | 109 ++++++++++++------ .../collectors/dm/DataMinerPlugin.java | 21 ++-- .../collectors/dm/model/CkanModel.java | 23 ++-- .../dm/model/InternalAlgorithmDescriptor.java | 63 +++++++++- .../collectors/dm/TranslationTest.java | 55 +++++++++ .../catalogues/CatalogueController.java | 5 +- .../collectors/CollectorPlugin.java | 4 +- .../collectors/DataTransformer.java | 4 +- .../gCatFeeder/model/CatalogueFormatData.java | 8 ++ .../service/engine/impl/ExecutionTask.java | 5 +- 13 files changed, 245 insertions(+), 78 deletions(-) create mode 100644 DataMinerAlgorithmsCrawler/src/main/java/org/gcube/data/publishing/gCatFeeder/collectors/dm/Constants.java create mode 100644 DataMinerAlgorithmsCrawler/src/test/java/org/gcube/data/publishing/gCatFeeder/collectors/dm/TranslationTest.java create mode 100644 commons/src/main/java/org/gcube/data/publishing/gCatFeeder/model/CatalogueFormatData.java diff --git a/DataMinerAlgorithmsCrawler/pom.xml b/DataMinerAlgorithmsCrawler/pom.xml index 2e15a2e..944f864 100644 --- a/DataMinerAlgorithmsCrawler/pom.xml +++ b/DataMinerAlgorithmsCrawler/pom.xml @@ -24,5 +24,15 @@ data-miner-manager-cl [1.6.0-SNAPSHOT,2.0.0-SNAPSHOT) + + + + org.codehaus.jackson + jackson-mapper-asl + 1.9.13 + test + + + \ No newline at end of file diff --git a/DataMinerAlgorithmsCrawler/src/main/java/org/gcube/data/publishing/gCatFeeder/collectors/dm/CKANTransformer.java b/DataMinerAlgorithmsCrawler/src/main/java/org/gcube/data/publishing/gCatFeeder/collectors/dm/CKANTransformer.java index 5d8e902..497a866 100644 --- a/DataMinerAlgorithmsCrawler/src/main/java/org/gcube/data/publishing/gCatFeeder/collectors/dm/CKANTransformer.java +++ b/DataMinerAlgorithmsCrawler/src/main/java/org/gcube/data/publishing/gCatFeeder/collectors/dm/CKANTransformer.java @@ -6,17 +6,10 @@ import java.util.Set; import org.gcube.data.publishing.gCatFeeder.collectors.dm.model.CkanModel; import org.gcube.data.publishing.gCatFeeder.collectors.dm.model.InternalAlgorithmDescriptor; -import org.gcube.data.publishing.gCatFeeder.model.FormatData; import org.gcube.data.publishing.gCatfeeder.collectors.DataTransformer; -public class CKANTransformer implements DataTransformer{ +public class CKANTransformer implements DataTransformer{ - - @Override - public FormatData getTargetFormat() { - return FormatData.CKAN; - } - @Override public Set transform(Collection collectedData) { HashSet toReturn=new HashSet<>(); diff --git a/DataMinerAlgorithmsCrawler/src/main/java/org/gcube/data/publishing/gCatFeeder/collectors/dm/Constants.java b/DataMinerAlgorithmsCrawler/src/main/java/org/gcube/data/publishing/gCatFeeder/collectors/dm/Constants.java new file mode 100644 index 0000000..3daf1ec --- /dev/null +++ b/DataMinerAlgorithmsCrawler/src/main/java/org/gcube/data/publishing/gCatFeeder/collectors/dm/Constants.java @@ -0,0 +1,7 @@ +package org.gcube.data.publishing.gCatFeeder.collectors.dm; + +public class Constants { + + public static final String CKAN_TYPE="CKAN"; + +} diff --git a/DataMinerAlgorithmsCrawler/src/main/java/org/gcube/data/publishing/gCatFeeder/collectors/dm/DMAlgorithmsInfoCollector.java b/DataMinerAlgorithmsCrawler/src/main/java/org/gcube/data/publishing/gCatFeeder/collectors/dm/DMAlgorithmsInfoCollector.java index 09153cc..9ad9863 100644 --- a/DataMinerAlgorithmsCrawler/src/main/java/org/gcube/data/publishing/gCatFeeder/collectors/dm/DMAlgorithmsInfoCollector.java +++ b/DataMinerAlgorithmsCrawler/src/main/java/org/gcube/data/publishing/gCatFeeder/collectors/dm/DMAlgorithmsInfoCollector.java @@ -1,6 +1,7 @@ package org.gcube.data.publishing.gCatFeeder.collectors.dm; import java.util.HashSet; +import java.util.List; import java.util.Set; import org.gcube.data.analysis.dataminermanagercl.server.DataMinerService; @@ -13,56 +14,88 @@ import org.gcube.data.publishing.gCatFeeder.collectors.dm.model.InternalAlgorith import org.gcube.data.publishing.gCatfeeder.collectors.DataCollector; import org.gcube.data.publishing.gCatfeeder.collectors.model.faults.CollectorFault; +import lombok.extern.slf4j.Slf4j; +@Slf4j public class DMAlgorithmsInfoCollector implements DataCollector { - + @Override public Set collect() throws CollectorFault { try { - SClient client=new DataMinerService().getClient(); - for(OperatorsClassification opClass: client.getOperatorsClassifications()) { - // Load info - String opClassName =opClass.getName(); + log.trace("Collecting information for Dataminer algorithms"); + HashSet toReturn=new HashSet(); + SClient client=new DataMinerService().getClient(); + List opClassifications=client.getOperatorsClassifications(); + log.debug("Found {} classifications.",opClassifications.size()); - for(Operator op : opClass.getOperators()) { - // OperatorCategory Info - OperatorCategory cat = op.getCategory(); - String categoryBriefDescription = cat.getBriefDescription(); - String categoryDescription= cat.getDescription(); - String categoryID=cat.getId(); - String categoryName=cat.getName(); + for(OperatorsClassification opClass: opClassifications) { + // Load info + String opClassName =opClass.getName(); + - // Operator info - - String opBriefDescription=op.getBriefDescription(); - String opDescription=op.getDescription(); - String opID=op.getId(); - String operatorName=op.getName(); - - - // Parameters info - for(Parameter param:op.getOperatorParameters()) { - String paramDescription=param.getDescription(); - String paramName=param.getName(); - String paramType=param.getTypology().toString(); - String paramValue=param.getValue(); + List ops=opClass.getOperators(); + log.debug("Found {} operators under classification {} ",ops.size(),opClassName); + for(Operator op : ops) { + + InternalAlgorithmDescriptor desc=new InternalAlgorithmDescriptor(); + desc.setClassName(opClassName); + + // OperatorCategory Info + OperatorCategory cat = op.getCategory(); + String categoryBriefDescription = cat.getBriefDescription(); + String categoryDescription= cat.getDescription(); + String categoryID=cat.getId(); + String categoryName=cat.getName(); + + + desc.setCategoryBriefDescription(categoryBriefDescription); + desc.setCategoryID(categoryID); + desc.setCategoryName(categoryName); + desc.setCategoryDescription(categoryDescription); + + + + // Operator info + + String opBriefDescription=op.getBriefDescription(); + String opDescription=op.getDescription(); + String opID=op.getId(); + String operatorName=op.getName(); + + + desc.setBriefDescription(opBriefDescription); + desc.setDescription(opDescription); + desc.setId(opID); + desc.setName(operatorName); + + + + // Parameters info + for(Parameter param:op.getOperatorParameters()) { + String paramDescription=param.getDescription(); + String paramName=param.getName(); + String paramType=param.getTypology().toString(); + String paramValue=param.getValue(); + } + + toReturn.add(desc); } + + + // Create bean + + } - - - // Create bean - } - - - - HashSet toReturn=new HashSet(); - - - return toReturn; + + + + + + return toReturn; }catch(Exception e) { throw new CollectorFault("Unable to retrieve information",e); } } - + } diff --git a/DataMinerAlgorithmsCrawler/src/main/java/org/gcube/data/publishing/gCatFeeder/collectors/dm/DataMinerPlugin.java b/DataMinerAlgorithmsCrawler/src/main/java/org/gcube/data/publishing/gCatFeeder/collectors/dm/DataMinerPlugin.java index ac51f5f..f661e55 100644 --- a/DataMinerAlgorithmsCrawler/src/main/java/org/gcube/data/publishing/gCatFeeder/collectors/dm/DataMinerPlugin.java +++ b/DataMinerAlgorithmsCrawler/src/main/java/org/gcube/data/publishing/gCatFeeder/collectors/dm/DataMinerPlugin.java @@ -3,14 +3,12 @@ package org.gcube.data.publishing.gCatFeeder.collectors.dm; import java.util.Collections; import java.util.Set; -import org.gcube.data.publishing.gCatFeeder.collectors.dm.model.CkanModel; import org.gcube.data.publishing.gCatFeeder.collectors.dm.model.InternalAlgorithmDescriptor; -import org.gcube.data.publishing.gCatFeeder.model.FormatData; +import org.gcube.data.publishing.gCatFeeder.model.CatalogueFormatData; import org.gcube.data.publishing.gCatfeeder.collectors.CatalogueRetriever; import org.gcube.data.publishing.gCatfeeder.collectors.CollectorPlugin; import org.gcube.data.publishing.gCatfeeder.collectors.DataCollector; import org.gcube.data.publishing.gCatfeeder.collectors.DataTransformer; -import org.gcube.data.publishing.gCatfeeder.collectors.model.CustomData; import org.gcube.data.publishing.gCatfeeder.collectors.model.PluginDescriptor; import org.gcube.data.publishing.gCatfeeder.collectors.model.faults.CatalogueNotSupportedException; @@ -23,16 +21,16 @@ public class DataMinerPlugin implements CollectorPlugin> getImplementedTransformers() { - return Collections.singleton((DataTransformer) new CKANTransformer()); + public Set getSupportedCatalogueTypes() { + return Collections.singleton(Constants.CKAN_TYPE); } @Override @@ -40,4 +38,13 @@ public class DataMinerPlugin implements CollectorPlugin getTransformerByCatalogueType( + String catalogueType) throws CatalogueNotSupportedException { + switch(catalogueType) { + case Constants.CKAN_TYPE : return new CKANTransformer(); + default : throw new CatalogueNotSupportedException("No support for "+catalogueType); + } + } + } diff --git a/DataMinerAlgorithmsCrawler/src/main/java/org/gcube/data/publishing/gCatFeeder/collectors/dm/model/CkanModel.java b/DataMinerAlgorithmsCrawler/src/main/java/org/gcube/data/publishing/gCatFeeder/collectors/dm/model/CkanModel.java index 91323a5..c27f16a 100644 --- a/DataMinerAlgorithmsCrawler/src/main/java/org/gcube/data/publishing/gCatFeeder/collectors/dm/model/CkanModel.java +++ b/DataMinerAlgorithmsCrawler/src/main/java/org/gcube/data/publishing/gCatFeeder/collectors/dm/model/CkanModel.java @@ -1,9 +1,14 @@ package org.gcube.data.publishing.gCatFeeder.collectors.dm.model; -import org.gcube.data.publishing.gCatFeeder.model.FormatData; -import org.gcube.data.publishing.gCatfeeder.collectors.model.PublisherFormatData; +import org.gcube.data.publishing.gCatFeeder.model.CatalogueFormatData; -public class CkanModel implements PublisherFormatData { +public class CkanModel implements CatalogueFormatData { + + @Override + public String toCatalogueFormat() { + // TODO Auto-generated method stub + return null; + } /** @@ -25,15 +30,5 @@ public class CkanModel implements PublisherFormatData { */ - @Override - public FormatData getFormat() { - return FormatData.CKAN; - } - - @Override - public String serialize() { - // TODO Auto-generated method stub - return null; - } - + } diff --git a/DataMinerAlgorithmsCrawler/src/main/java/org/gcube/data/publishing/gCatFeeder/collectors/dm/model/InternalAlgorithmDescriptor.java b/DataMinerAlgorithmsCrawler/src/main/java/org/gcube/data/publishing/gCatFeeder/collectors/dm/model/InternalAlgorithmDescriptor.java index d0b6d47..8805bdd 100644 --- a/DataMinerAlgorithmsCrawler/src/main/java/org/gcube/data/publishing/gCatFeeder/collectors/dm/model/InternalAlgorithmDescriptor.java +++ b/DataMinerAlgorithmsCrawler/src/main/java/org/gcube/data/publishing/gCatFeeder/collectors/dm/model/InternalAlgorithmDescriptor.java @@ -1,20 +1,79 @@ package org.gcube.data.publishing.gCatFeeder.collectors.dm.model; +import java.util.HashSet; import java.util.List; +import java.util.Set; import org.gcube.data.publishing.gCatfeeder.collectors.model.CustomData; +import lombok.AllArgsConstructor; +import lombok.Getter; +import lombok.Setter; + +@Getter +@Setter public class InternalAlgorithmDescriptor implements CustomData { + @AllArgsConstructor + public static class Parameter{ + private String name; + private String type; + private String description; + private String value; + + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + ((name == null) ? 0 : name.hashCode()); + return result; + } + @Override + public boolean equals(Object obj) { + if (this == obj) + return true; + if (obj == null) + return false; + if (getClass() != obj.getClass()) + return false; + Parameter other = (Parameter) obj; + if (name == null) { + if (other.name != null) + return false; + } else if (!name.equals(other.name)) + return false; + return true; + } + + + } + + private String className; + + //operator info private String name; - private String Description; - private List tags; + private String description; + private String briefDescription; private String id; + + //category info + + private String categoryBriefDescription; + private String categoryDescription; + private String categoryID; + private String categoryName; + private Set parameters=new HashSet<>(); + + + + private List tags; public CkanModel asCKANModel() { throw new RuntimeException("Implement This"); } + + } diff --git a/DataMinerAlgorithmsCrawler/src/test/java/org/gcube/data/publishing/gCatFeeder/collectors/dm/TranslationTest.java b/DataMinerAlgorithmsCrawler/src/test/java/org/gcube/data/publishing/gCatFeeder/collectors/dm/TranslationTest.java new file mode 100644 index 0000000..81ca9ab --- /dev/null +++ b/DataMinerAlgorithmsCrawler/src/test/java/org/gcube/data/publishing/gCatFeeder/collectors/dm/TranslationTest.java @@ -0,0 +1,55 @@ +package org.gcube.data.publishing.gCatFeeder.collectors.dm; + +import java.io.IOException; +import java.util.Collection; +import java.util.Set; + +import org.codehaus.jackson.JsonGenerationException; +import org.codehaus.jackson.map.JsonMappingException; +import org.codehaus.jackson.map.ObjectMapper; +import org.gcube.common.authorization.library.provider.SecurityTokenProvider; +import org.gcube.data.publishing.gCatFeeder.collectors.dm.model.InternalAlgorithmDescriptor; +import org.gcube.data.publishing.gCatFeeder.model.CatalogueFormatData; +import org.gcube.data.publishing.gCatfeeder.collectors.CollectorPlugin; +import org.gcube.data.publishing.gCatfeeder.collectors.DataCollector; +import org.gcube.data.publishing.gCatfeeder.collectors.DataTransformer; +import org.gcube.data.publishing.gCatfeeder.collectors.model.faults.CatalogueNotSupportedException; +import org.gcube.data.publishing.gCatfeeder.collectors.model.faults.CollectorFault; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; + +public class TranslationTest { + + @BeforeClass + public static void checkTestEnabled() { + + } + + @Before + public void setToken() { + // disable if maven property + // set token + SecurityTokenProvider.instance.set("***REMOVED***"); + } + + + @Test + public void testTranslation() throws CollectorFault, CatalogueNotSupportedException, JsonGenerationException, JsonMappingException, IOException { + + ObjectMapper mapper = new ObjectMapper(); + + CollectorPlugin plugin=new DataMinerPlugin(); + DataCollector collector=plugin.getCollector(); + Collection collected=collector.collect(); + System.out.println("Found "+collected.size()+" elements"); + for(Object obj:collected) + System.out.println(mapper.writeValueAsString(obj)+"\n"); + + for(String destinationcatalogue : (Set)plugin.getSupportedCatalogueTypes()) { + DataTransformer transformer=plugin.getTransformerByCatalogueType(destinationcatalogue); + transformer.transform(collected); + } + } + +} diff --git a/catalogue-plugin-framework/src/main/java/org/gcube/data/publishing/gCatFeeder/catalogues/CatalogueController.java b/catalogue-plugin-framework/src/main/java/org/gcube/data/publishing/gCatFeeder/catalogues/CatalogueController.java index 967eaab..4a0abe6 100644 --- a/catalogue-plugin-framework/src/main/java/org/gcube/data/publishing/gCatFeeder/catalogues/CatalogueController.java +++ b/catalogue-plugin-framework/src/main/java/org/gcube/data/publishing/gCatFeeder/catalogues/CatalogueController.java @@ -1,13 +1,12 @@ package org.gcube.data.publishing.gCatFeeder.catalogues; -import java.io.Serializable; - import org.gcube.data.publishing.gCatFeeder.catalogues.model.PublishReport; import org.gcube.data.publishing.gCatFeeder.catalogues.model.faults.WrongObjectFormatException; +import org.gcube.data.publishing.gCatFeeder.model.CatalogueFormatData; public interface CatalogueController { - public PublishReport publishItem(Serializable toPublish) throws WrongObjectFormatException; + public PublishReport publishItem(CatalogueFormatData toPublish) throws WrongObjectFormatException; } diff --git a/collectors-plugin-framework/src/main/java/org/gcube/data/publishing/gCatfeeder/collectors/CollectorPlugin.java b/collectors-plugin-framework/src/main/java/org/gcube/data/publishing/gCatfeeder/collectors/CollectorPlugin.java index d1a4434..bddc249 100644 --- a/collectors-plugin-framework/src/main/java/org/gcube/data/publishing/gCatfeeder/collectors/CollectorPlugin.java +++ b/collectors-plugin-framework/src/main/java/org/gcube/data/publishing/gCatfeeder/collectors/CollectorPlugin.java @@ -1,8 +1,8 @@ package org.gcube.data.publishing.gCatfeeder.collectors; -import java.io.Serializable; import java.util.Set; +import org.gcube.data.publishing.gCatFeeder.model.CatalogueFormatData; import org.gcube.data.publishing.gCatfeeder.collectors.model.CustomData; import org.gcube.data.publishing.gCatfeeder.collectors.model.PluginDescriptor; import org.gcube.data.publishing.gCatfeeder.collectors.model.faults.CatalogueNotSupportedException; @@ -15,7 +15,7 @@ public interface CollectorPlugin{ public Set getSupportedCatalogueTypes(); - public DataTransformer getTransformerByCatalogueType(String catalogueType); + public DataTransformer getTransformerByCatalogueType(String catalogueType)throws CatalogueNotSupportedException; public DataCollector getCollector(); } diff --git a/collectors-plugin-framework/src/main/java/org/gcube/data/publishing/gCatfeeder/collectors/DataTransformer.java b/collectors-plugin-framework/src/main/java/org/gcube/data/publishing/gCatfeeder/collectors/DataTransformer.java index 177a855..57b1274 100644 --- a/collectors-plugin-framework/src/main/java/org/gcube/data/publishing/gCatfeeder/collectors/DataTransformer.java +++ b/collectors-plugin-framework/src/main/java/org/gcube/data/publishing/gCatfeeder/collectors/DataTransformer.java @@ -1,12 +1,12 @@ package org.gcube.data.publishing.gCatfeeder.collectors; -import java.io.Serializable; import java.util.Collection; import java.util.Set; +import org.gcube.data.publishing.gCatFeeder.model.CatalogueFormatData; import org.gcube.data.publishing.gCatfeeder.collectors.model.CustomData; -public interface DataTransformer { +public interface DataTransformer { public Set transform(Collection collectedData); diff --git a/commons/src/main/java/org/gcube/data/publishing/gCatFeeder/model/CatalogueFormatData.java b/commons/src/main/java/org/gcube/data/publishing/gCatFeeder/model/CatalogueFormatData.java new file mode 100644 index 0000000..c019d75 --- /dev/null +++ b/commons/src/main/java/org/gcube/data/publishing/gCatFeeder/model/CatalogueFormatData.java @@ -0,0 +1,8 @@ +package org.gcube.data.publishing.gCatFeeder.model; + +public interface CatalogueFormatData { + + + public String toCatalogueFormat(); + +} diff --git a/gCat-Feeder/src/main/java/org/gcube/data/publishing/gCatFeeder/service/engine/impl/ExecutionTask.java b/gCat-Feeder/src/main/java/org/gcube/data/publishing/gCatFeeder/service/engine/impl/ExecutionTask.java index c0364db..9abdc21 100644 --- a/gCat-Feeder/src/main/java/org/gcube/data/publishing/gCatFeeder/service/engine/impl/ExecutionTask.java +++ b/gCat-Feeder/src/main/java/org/gcube/data/publishing/gCatFeeder/service/engine/impl/ExecutionTask.java @@ -13,6 +13,7 @@ import org.gcube.data.publishing.gCatFeeder.catalogues.CataloguePlugin; import org.gcube.data.publishing.gCatFeeder.catalogues.model.PublishReport; import org.gcube.data.publishing.gCatFeeder.catalogues.model.faults.ControllerInstantiationFault; import org.gcube.data.publishing.gCatFeeder.catalogues.model.faults.WrongObjectFormatException; +import org.gcube.data.publishing.gCatFeeder.model.CatalogueFormatData; import org.gcube.data.publishing.gCatFeeder.model.CatalogueInstanceDescriptor; import org.gcube.data.publishing.gCatFeeder.service.engine.CatalogueControllersManager; import org.gcube.data.publishing.gCatFeeder.service.engine.CollectorsManager; @@ -174,11 +175,11 @@ public class ExecutionTask implements Runnable { // ** TRANSFORM log.debug("Transforming Collected Data"); DataTransformer transformer=collectorPlugin.getTransformerByCatalogueType(cataloguePluginId); - Set transformed=transformer.transform(collectedData); + Set transformed=transformer.transform(collectedData); log.trace("Going to publish {} items to {} ",transformed.size(),instanceDescriptor.getUrl()); // ** PUBLISH VIA CONTROLLER - for(Serializable item : transformed) { + for(CatalogueFormatData item : transformed) { try { PublishReport itemReport=controller.publishItem(item); }catch(WrongObjectFormatException e) {