diff --git a/DataMinerAlgorithmsCrawler/pom.xml b/DataMinerAlgorithmsCrawler/pom.xml
index 2e15a2e..944f864 100644
--- a/DataMinerAlgorithmsCrawler/pom.xml
+++ b/DataMinerAlgorithmsCrawler/pom.xml
@@ -24,5 +24,15 @@
data-miner-manager-cl
[1.6.0-SNAPSHOT,2.0.0-SNAPSHOT)
+
+
+
+ org.codehaus.jackson
+ jackson-mapper-asl
+ 1.9.13
+ test
+
+
+
\ No newline at end of file
diff --git a/DataMinerAlgorithmsCrawler/src/main/java/org/gcube/data/publishing/gCatFeeder/collectors/dm/CKANTransformer.java b/DataMinerAlgorithmsCrawler/src/main/java/org/gcube/data/publishing/gCatFeeder/collectors/dm/CKANTransformer.java
index 5d8e902..497a866 100644
--- a/DataMinerAlgorithmsCrawler/src/main/java/org/gcube/data/publishing/gCatFeeder/collectors/dm/CKANTransformer.java
+++ b/DataMinerAlgorithmsCrawler/src/main/java/org/gcube/data/publishing/gCatFeeder/collectors/dm/CKANTransformer.java
@@ -6,17 +6,10 @@ import java.util.Set;
import org.gcube.data.publishing.gCatFeeder.collectors.dm.model.CkanModel;
import org.gcube.data.publishing.gCatFeeder.collectors.dm.model.InternalAlgorithmDescriptor;
-import org.gcube.data.publishing.gCatFeeder.model.FormatData;
import org.gcube.data.publishing.gCatfeeder.collectors.DataTransformer;
-public class CKANTransformer implements DataTransformer{
+public class CKANTransformer implements DataTransformer{
-
- @Override
- public FormatData getTargetFormat() {
- return FormatData.CKAN;
- }
-
@Override
public Set transform(Collection collectedData) {
HashSet toReturn=new HashSet<>();
diff --git a/DataMinerAlgorithmsCrawler/src/main/java/org/gcube/data/publishing/gCatFeeder/collectors/dm/Constants.java b/DataMinerAlgorithmsCrawler/src/main/java/org/gcube/data/publishing/gCatFeeder/collectors/dm/Constants.java
new file mode 100644
index 0000000..3daf1ec
--- /dev/null
+++ b/DataMinerAlgorithmsCrawler/src/main/java/org/gcube/data/publishing/gCatFeeder/collectors/dm/Constants.java
@@ -0,0 +1,7 @@
+package org.gcube.data.publishing.gCatFeeder.collectors.dm;
+
+public class Constants {
+
+ public static final String CKAN_TYPE="CKAN";
+
+}
diff --git a/DataMinerAlgorithmsCrawler/src/main/java/org/gcube/data/publishing/gCatFeeder/collectors/dm/DMAlgorithmsInfoCollector.java b/DataMinerAlgorithmsCrawler/src/main/java/org/gcube/data/publishing/gCatFeeder/collectors/dm/DMAlgorithmsInfoCollector.java
index 09153cc..9ad9863 100644
--- a/DataMinerAlgorithmsCrawler/src/main/java/org/gcube/data/publishing/gCatFeeder/collectors/dm/DMAlgorithmsInfoCollector.java
+++ b/DataMinerAlgorithmsCrawler/src/main/java/org/gcube/data/publishing/gCatFeeder/collectors/dm/DMAlgorithmsInfoCollector.java
@@ -1,6 +1,7 @@
package org.gcube.data.publishing.gCatFeeder.collectors.dm;
import java.util.HashSet;
+import java.util.List;
import java.util.Set;
import org.gcube.data.analysis.dataminermanagercl.server.DataMinerService;
@@ -13,56 +14,88 @@ import org.gcube.data.publishing.gCatFeeder.collectors.dm.model.InternalAlgorith
import org.gcube.data.publishing.gCatfeeder.collectors.DataCollector;
import org.gcube.data.publishing.gCatfeeder.collectors.model.faults.CollectorFault;
+import lombok.extern.slf4j.Slf4j;
+@Slf4j
public class DMAlgorithmsInfoCollector implements DataCollector {
-
+
@Override
public Set collect() throws CollectorFault {
try {
- SClient client=new DataMinerService().getClient();
- for(OperatorsClassification opClass: client.getOperatorsClassifications()) {
- // Load info
- String opClassName =opClass.getName();
+ log.trace("Collecting information for Dataminer algorithms");
+ HashSet toReturn=new HashSet();
+ SClient client=new DataMinerService().getClient();
+ List opClassifications=client.getOperatorsClassifications();
+ log.debug("Found {} classifications.",opClassifications.size());
- for(Operator op : opClass.getOperators()) {
- // OperatorCategory Info
- OperatorCategory cat = op.getCategory();
- String categoryBriefDescription = cat.getBriefDescription();
- String categoryDescription= cat.getDescription();
- String categoryID=cat.getId();
- String categoryName=cat.getName();
+ for(OperatorsClassification opClass: opClassifications) {
+ // Load info
+ String opClassName =opClass.getName();
+
- // Operator info
-
- String opBriefDescription=op.getBriefDescription();
- String opDescription=op.getDescription();
- String opID=op.getId();
- String operatorName=op.getName();
-
-
- // Parameters info
- for(Parameter param:op.getOperatorParameters()) {
- String paramDescription=param.getDescription();
- String paramName=param.getName();
- String paramType=param.getTypology().toString();
- String paramValue=param.getValue();
+ List ops=opClass.getOperators();
+ log.debug("Found {} operators under classification {} ",ops.size(),opClassName);
+ for(Operator op : ops) {
+
+ InternalAlgorithmDescriptor desc=new InternalAlgorithmDescriptor();
+ desc.setClassName(opClassName);
+
+ // OperatorCategory Info
+ OperatorCategory cat = op.getCategory();
+ String categoryBriefDescription = cat.getBriefDescription();
+ String categoryDescription= cat.getDescription();
+ String categoryID=cat.getId();
+ String categoryName=cat.getName();
+
+
+ desc.setCategoryBriefDescription(categoryBriefDescription);
+ desc.setCategoryID(categoryID);
+ desc.setCategoryName(categoryName);
+ desc.setCategoryDescription(categoryDescription);
+
+
+
+ // Operator info
+
+ String opBriefDescription=op.getBriefDescription();
+ String opDescription=op.getDescription();
+ String opID=op.getId();
+ String operatorName=op.getName();
+
+
+ desc.setBriefDescription(opBriefDescription);
+ desc.setDescription(opDescription);
+ desc.setId(opID);
+ desc.setName(operatorName);
+
+
+
+ // Parameters info
+ for(Parameter param:op.getOperatorParameters()) {
+ String paramDescription=param.getDescription();
+ String paramName=param.getName();
+ String paramType=param.getTypology().toString();
+ String paramValue=param.getValue();
+ }
+
+ toReturn.add(desc);
}
+
+
+ // Create bean
+
+
}
-
-
- // Create bean
- }
-
-
-
- HashSet toReturn=new HashSet();
-
-
- return toReturn;
+
+
+
+
+
+ return toReturn;
}catch(Exception e) {
throw new CollectorFault("Unable to retrieve information",e);
}
}
-
+
}
diff --git a/DataMinerAlgorithmsCrawler/src/main/java/org/gcube/data/publishing/gCatFeeder/collectors/dm/DataMinerPlugin.java b/DataMinerAlgorithmsCrawler/src/main/java/org/gcube/data/publishing/gCatFeeder/collectors/dm/DataMinerPlugin.java
index ac51f5f..f661e55 100644
--- a/DataMinerAlgorithmsCrawler/src/main/java/org/gcube/data/publishing/gCatFeeder/collectors/dm/DataMinerPlugin.java
+++ b/DataMinerAlgorithmsCrawler/src/main/java/org/gcube/data/publishing/gCatFeeder/collectors/dm/DataMinerPlugin.java
@@ -3,14 +3,12 @@ package org.gcube.data.publishing.gCatFeeder.collectors.dm;
import java.util.Collections;
import java.util.Set;
-import org.gcube.data.publishing.gCatFeeder.collectors.dm.model.CkanModel;
import org.gcube.data.publishing.gCatFeeder.collectors.dm.model.InternalAlgorithmDescriptor;
-import org.gcube.data.publishing.gCatFeeder.model.FormatData;
+import org.gcube.data.publishing.gCatFeeder.model.CatalogueFormatData;
import org.gcube.data.publishing.gCatfeeder.collectors.CatalogueRetriever;
import org.gcube.data.publishing.gCatfeeder.collectors.CollectorPlugin;
import org.gcube.data.publishing.gCatfeeder.collectors.DataCollector;
import org.gcube.data.publishing.gCatfeeder.collectors.DataTransformer;
-import org.gcube.data.publishing.gCatfeeder.collectors.model.CustomData;
import org.gcube.data.publishing.gCatfeeder.collectors.model.PluginDescriptor;
import org.gcube.data.publishing.gCatfeeder.collectors.model.faults.CatalogueNotSupportedException;
@@ -23,16 +21,16 @@ public class DataMinerPlugin implements CollectorPlugin> getImplementedTransformers() {
- return Collections.singleton((DataTransformer) new CKANTransformer());
+ public Set getSupportedCatalogueTypes() {
+ return Collections.singleton(Constants.CKAN_TYPE);
}
@Override
@@ -40,4 +38,13 @@ public class DataMinerPlugin implements CollectorPlugin getTransformerByCatalogueType(
+ String catalogueType) throws CatalogueNotSupportedException {
+ switch(catalogueType) {
+ case Constants.CKAN_TYPE : return new CKANTransformer();
+ default : throw new CatalogueNotSupportedException("No support for "+catalogueType);
+ }
+ }
+
}
diff --git a/DataMinerAlgorithmsCrawler/src/main/java/org/gcube/data/publishing/gCatFeeder/collectors/dm/model/CkanModel.java b/DataMinerAlgorithmsCrawler/src/main/java/org/gcube/data/publishing/gCatFeeder/collectors/dm/model/CkanModel.java
index 91323a5..c27f16a 100644
--- a/DataMinerAlgorithmsCrawler/src/main/java/org/gcube/data/publishing/gCatFeeder/collectors/dm/model/CkanModel.java
+++ b/DataMinerAlgorithmsCrawler/src/main/java/org/gcube/data/publishing/gCatFeeder/collectors/dm/model/CkanModel.java
@@ -1,9 +1,14 @@
package org.gcube.data.publishing.gCatFeeder.collectors.dm.model;
-import org.gcube.data.publishing.gCatFeeder.model.FormatData;
-import org.gcube.data.publishing.gCatfeeder.collectors.model.PublisherFormatData;
+import org.gcube.data.publishing.gCatFeeder.model.CatalogueFormatData;
-public class CkanModel implements PublisherFormatData {
+public class CkanModel implements CatalogueFormatData {
+
+ @Override
+ public String toCatalogueFormat() {
+ // TODO Auto-generated method stub
+ return null;
+ }
/**
@@ -25,15 +30,5 @@ public class CkanModel implements PublisherFormatData {
*/
- @Override
- public FormatData getFormat() {
- return FormatData.CKAN;
- }
-
- @Override
- public String serialize() {
- // TODO Auto-generated method stub
- return null;
- }
-
+
}
diff --git a/DataMinerAlgorithmsCrawler/src/main/java/org/gcube/data/publishing/gCatFeeder/collectors/dm/model/InternalAlgorithmDescriptor.java b/DataMinerAlgorithmsCrawler/src/main/java/org/gcube/data/publishing/gCatFeeder/collectors/dm/model/InternalAlgorithmDescriptor.java
index d0b6d47..8805bdd 100644
--- a/DataMinerAlgorithmsCrawler/src/main/java/org/gcube/data/publishing/gCatFeeder/collectors/dm/model/InternalAlgorithmDescriptor.java
+++ b/DataMinerAlgorithmsCrawler/src/main/java/org/gcube/data/publishing/gCatFeeder/collectors/dm/model/InternalAlgorithmDescriptor.java
@@ -1,20 +1,79 @@
package org.gcube.data.publishing.gCatFeeder.collectors.dm.model;
+import java.util.HashSet;
import java.util.List;
+import java.util.Set;
import org.gcube.data.publishing.gCatfeeder.collectors.model.CustomData;
+import lombok.AllArgsConstructor;
+import lombok.Getter;
+import lombok.Setter;
+
+@Getter
+@Setter
public class InternalAlgorithmDescriptor implements CustomData {
+ @AllArgsConstructor
+ public static class Parameter{
+ private String name;
+ private String type;
+ private String description;
+ private String value;
+
+ @Override
+ public int hashCode() {
+ final int prime = 31;
+ int result = 1;
+ result = prime * result + ((name == null) ? 0 : name.hashCode());
+ return result;
+ }
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj)
+ return true;
+ if (obj == null)
+ return false;
+ if (getClass() != obj.getClass())
+ return false;
+ Parameter other = (Parameter) obj;
+ if (name == null) {
+ if (other.name != null)
+ return false;
+ } else if (!name.equals(other.name))
+ return false;
+ return true;
+ }
+
+
+ }
+
+ private String className;
+
+ //operator info
private String name;
- private String Description;
- private List tags;
+ private String description;
+ private String briefDescription;
private String id;
+
+ //category info
+
+ private String categoryBriefDescription;
+ private String categoryDescription;
+ private String categoryID;
+ private String categoryName;
+ private Set parameters=new HashSet<>();
+
+
+
+ private List tags;
public CkanModel asCKANModel() {
throw new RuntimeException("Implement This");
}
+
+
}
diff --git a/DataMinerAlgorithmsCrawler/src/test/java/org/gcube/data/publishing/gCatFeeder/collectors/dm/TranslationTest.java b/DataMinerAlgorithmsCrawler/src/test/java/org/gcube/data/publishing/gCatFeeder/collectors/dm/TranslationTest.java
new file mode 100644
index 0000000..81ca9ab
--- /dev/null
+++ b/DataMinerAlgorithmsCrawler/src/test/java/org/gcube/data/publishing/gCatFeeder/collectors/dm/TranslationTest.java
@@ -0,0 +1,55 @@
+package org.gcube.data.publishing.gCatFeeder.collectors.dm;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.Set;
+
+import org.codehaus.jackson.JsonGenerationException;
+import org.codehaus.jackson.map.JsonMappingException;
+import org.codehaus.jackson.map.ObjectMapper;
+import org.gcube.common.authorization.library.provider.SecurityTokenProvider;
+import org.gcube.data.publishing.gCatFeeder.collectors.dm.model.InternalAlgorithmDescriptor;
+import org.gcube.data.publishing.gCatFeeder.model.CatalogueFormatData;
+import org.gcube.data.publishing.gCatfeeder.collectors.CollectorPlugin;
+import org.gcube.data.publishing.gCatfeeder.collectors.DataCollector;
+import org.gcube.data.publishing.gCatfeeder.collectors.DataTransformer;
+import org.gcube.data.publishing.gCatfeeder.collectors.model.faults.CatalogueNotSupportedException;
+import org.gcube.data.publishing.gCatfeeder.collectors.model.faults.CollectorFault;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+public class TranslationTest {
+
+ @BeforeClass
+ public static void checkTestEnabled() {
+
+ }
+
+ @Before
+ public void setToken() {
+ // disable if maven property
+ // set token
+ SecurityTokenProvider.instance.set("***REMOVED***");
+ }
+
+
+ @Test
+ public void testTranslation() throws CollectorFault, CatalogueNotSupportedException, JsonGenerationException, JsonMappingException, IOException {
+
+ ObjectMapper mapper = new ObjectMapper();
+
+ CollectorPlugin plugin=new DataMinerPlugin();
+ DataCollector collector=plugin.getCollector();
+ Collection collected=collector.collect();
+ System.out.println("Found "+collected.size()+" elements");
+ for(Object obj:collected)
+ System.out.println(mapper.writeValueAsString(obj)+"\n");
+
+ for(String destinationcatalogue : (Set)plugin.getSupportedCatalogueTypes()) {
+ DataTransformer extends CatalogueFormatData, InternalAlgorithmDescriptor> transformer=plugin.getTransformerByCatalogueType(destinationcatalogue);
+ transformer.transform(collected);
+ }
+ }
+
+}
diff --git a/catalogue-plugin-framework/src/main/java/org/gcube/data/publishing/gCatFeeder/catalogues/CatalogueController.java b/catalogue-plugin-framework/src/main/java/org/gcube/data/publishing/gCatFeeder/catalogues/CatalogueController.java
index 967eaab..4a0abe6 100644
--- a/catalogue-plugin-framework/src/main/java/org/gcube/data/publishing/gCatFeeder/catalogues/CatalogueController.java
+++ b/catalogue-plugin-framework/src/main/java/org/gcube/data/publishing/gCatFeeder/catalogues/CatalogueController.java
@@ -1,13 +1,12 @@
package org.gcube.data.publishing.gCatFeeder.catalogues;
-import java.io.Serializable;
-
import org.gcube.data.publishing.gCatFeeder.catalogues.model.PublishReport;
import org.gcube.data.publishing.gCatFeeder.catalogues.model.faults.WrongObjectFormatException;
+import org.gcube.data.publishing.gCatFeeder.model.CatalogueFormatData;
public interface CatalogueController {
- public PublishReport publishItem(Serializable toPublish) throws WrongObjectFormatException;
+ public PublishReport publishItem(CatalogueFormatData toPublish) throws WrongObjectFormatException;
}
diff --git a/collectors-plugin-framework/src/main/java/org/gcube/data/publishing/gCatfeeder/collectors/CollectorPlugin.java b/collectors-plugin-framework/src/main/java/org/gcube/data/publishing/gCatfeeder/collectors/CollectorPlugin.java
index d1a4434..bddc249 100644
--- a/collectors-plugin-framework/src/main/java/org/gcube/data/publishing/gCatfeeder/collectors/CollectorPlugin.java
+++ b/collectors-plugin-framework/src/main/java/org/gcube/data/publishing/gCatfeeder/collectors/CollectorPlugin.java
@@ -1,8 +1,8 @@
package org.gcube.data.publishing.gCatfeeder.collectors;
-import java.io.Serializable;
import java.util.Set;
+import org.gcube.data.publishing.gCatFeeder.model.CatalogueFormatData;
import org.gcube.data.publishing.gCatfeeder.collectors.model.CustomData;
import org.gcube.data.publishing.gCatfeeder.collectors.model.PluginDescriptor;
import org.gcube.data.publishing.gCatfeeder.collectors.model.faults.CatalogueNotSupportedException;
@@ -15,7 +15,7 @@ public interface CollectorPlugin{
public Set getSupportedCatalogueTypes();
- public DataTransformer getTransformerByCatalogueType(String catalogueType);
+ public DataTransformer extends CatalogueFormatData,E> getTransformerByCatalogueType(String catalogueType)throws CatalogueNotSupportedException;
public DataCollector getCollector();
}
diff --git a/collectors-plugin-framework/src/main/java/org/gcube/data/publishing/gCatfeeder/collectors/DataTransformer.java b/collectors-plugin-framework/src/main/java/org/gcube/data/publishing/gCatfeeder/collectors/DataTransformer.java
index 177a855..57b1274 100644
--- a/collectors-plugin-framework/src/main/java/org/gcube/data/publishing/gCatfeeder/collectors/DataTransformer.java
+++ b/collectors-plugin-framework/src/main/java/org/gcube/data/publishing/gCatfeeder/collectors/DataTransformer.java
@@ -1,12 +1,12 @@
package org.gcube.data.publishing.gCatfeeder.collectors;
-import java.io.Serializable;
import java.util.Collection;
import java.util.Set;
+import org.gcube.data.publishing.gCatFeeder.model.CatalogueFormatData;
import org.gcube.data.publishing.gCatfeeder.collectors.model.CustomData;
-public interface DataTransformer {
+public interface DataTransformer {
public Set transform(Collection collectedData);
diff --git a/commons/src/main/java/org/gcube/data/publishing/gCatFeeder/model/CatalogueFormatData.java b/commons/src/main/java/org/gcube/data/publishing/gCatFeeder/model/CatalogueFormatData.java
new file mode 100644
index 0000000..c019d75
--- /dev/null
+++ b/commons/src/main/java/org/gcube/data/publishing/gCatFeeder/model/CatalogueFormatData.java
@@ -0,0 +1,8 @@
+package org.gcube.data.publishing.gCatFeeder.model;
+
+public interface CatalogueFormatData {
+
+
+ public String toCatalogueFormat();
+
+}
diff --git a/gCat-Feeder/src/main/java/org/gcube/data/publishing/gCatFeeder/service/engine/impl/ExecutionTask.java b/gCat-Feeder/src/main/java/org/gcube/data/publishing/gCatFeeder/service/engine/impl/ExecutionTask.java
index c0364db..9abdc21 100644
--- a/gCat-Feeder/src/main/java/org/gcube/data/publishing/gCatFeeder/service/engine/impl/ExecutionTask.java
+++ b/gCat-Feeder/src/main/java/org/gcube/data/publishing/gCatFeeder/service/engine/impl/ExecutionTask.java
@@ -13,6 +13,7 @@ import org.gcube.data.publishing.gCatFeeder.catalogues.CataloguePlugin;
import org.gcube.data.publishing.gCatFeeder.catalogues.model.PublishReport;
import org.gcube.data.publishing.gCatFeeder.catalogues.model.faults.ControllerInstantiationFault;
import org.gcube.data.publishing.gCatFeeder.catalogues.model.faults.WrongObjectFormatException;
+import org.gcube.data.publishing.gCatFeeder.model.CatalogueFormatData;
import org.gcube.data.publishing.gCatFeeder.model.CatalogueInstanceDescriptor;
import org.gcube.data.publishing.gCatFeeder.service.engine.CatalogueControllersManager;
import org.gcube.data.publishing.gCatFeeder.service.engine.CollectorsManager;
@@ -174,11 +175,11 @@ public class ExecutionTask implements Runnable {
// ** TRANSFORM
log.debug("Transforming Collected Data");
DataTransformer transformer=collectorPlugin.getTransformerByCatalogueType(cataloguePluginId);
- Set transformed=transformer.transform(collectedData);
+ Set transformed=transformer.transform(collectedData);
log.trace("Going to publish {} items to {} ",transformed.size(),instanceDescriptor.getUrl());
// ** PUBLISH VIA CONTROLLER
- for(Serializable item : transformed) {
+ for(CatalogueFormatData item : transformed) {
try {
PublishReport itemReport=controller.publishItem(item);
}catch(WrongObjectFormatException e) {