gCubeSystem
/
gFeed
15
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

git-svn-id: http://svn.research-infrastructures.eu/public/d4science/gcube/trunk/data-publishing/gCat-Feeder-Suite@178587 82a268e6-3cf1-43bd-a215-b396298e98cf

This commit is contained in:
Fabio Sinibaldi 2019-03-14 15:58:03 +00:00
parent 5dd63116b6
commit 20a57b80dc
13 changed files with 245 additions and 78 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
DataMinerAlgorithmsCrawler/pom.xml
View File

@ -24,5 +24,15 @@
<artifactId>data-miner-manager-cl</artifactId>
<version>[1.6.0-SNAPSHOT,2.0.0-SNAPSHOT)</version>
</dependency>
<dependency>
<groupId>org.codehaus.jackson</groupId>
<artifactId>jackson-mapper-asl</artifactId>
<version>1.9.13</version>
<scope>test</scope>
</dependency>
</dependencies>
</project>

9
DataMinerAlgorithmsCrawler/src/main/java/org/gcube/data/publishing/gCatFeeder/collectors/dm/CKANTransformer.java
View File

@ -6,17 +6,10 @@ import java.util.Set;
import org.gcube.data.publishing.gCatFeeder.collectors.dm.model.CkanModel;
import org.gcube.data.publishing.gCatFeeder.collectors.dm.model.InternalAlgorithmDescriptor;
import org.gcube.data.publishing.gCatFeeder.model.FormatData;
import org.gcube.data.publishing.gCatfeeder.collectors.DataTransformer;
public class CKANTransformer implements DataTransformer<CkanModel, InternalAlgorithmDescriptor>{
public class CKANTransformer implements DataTransformer<CkanModel,InternalAlgorithmDescriptor>{
@Override
public FormatData getTargetFormat() {
return FormatData.CKAN;
}
@Override
public Set<CkanModel> transform(Collection<InternalAlgorithmDescriptor> collectedData) {
HashSet<CkanModel> toReturn=new HashSet<>();

7
DataMinerAlgorithmsCrawler/src/main/java/org/gcube/data/publishing/gCatFeeder/collectors/dm/Constants.java Normal file
View File

@ -0,0 +1,7 @@
package org.gcube.data.publishing.gCatFeeder.collectors.dm;
public class Constants {
public static final String CKAN_TYPE="CKAN";
}

109
DataMinerAlgorithmsCrawler/src/main/java/org/gcube/data/publishing/gCatFeeder/collectors/dm/DMAlgorithmsInfoCollector.java
View File

@ -1,6 +1,7 @@
package org.gcube.data.publishing.gCatFeeder.collectors.dm;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import org.gcube.data.analysis.dataminermanagercl.server.DataMinerService;
@ -13,56 +14,88 @@ import org.gcube.data.publishing.gCatFeeder.collectors.dm.model.InternalAlgorith
import org.gcube.data.publishing.gCatfeeder.collectors.DataCollector;
import org.gcube.data.publishing.gCatfeeder.collectors.model.faults.CollectorFault;
import lombok.extern.slf4j.Slf4j;
@Slf4j
public class DMAlgorithmsInfoCollector implements DataCollector<InternalAlgorithmDescriptor> {
@Override
public Set<InternalAlgorithmDescriptor> collect() throws CollectorFault {
try {
SClient client=new DataMinerService().getClient();
for(OperatorsClassification opClass: client.getOperatorsClassifications()) {
// Load info
String opClassName =opClass.getName();
log.trace("Collecting information for Dataminer algorithms");
HashSet<InternalAlgorithmDescriptor> toReturn=new HashSet();
SClient client=new DataMinerService().getClient();
List<OperatorsClassification> opClassifications=client.getOperatorsClassifications();
log.debug("Found {} classifications.",opClassifications.size());
for(Operator op : opClass.getOperators()) {
// OperatorCategory Info
OperatorCategory cat = op.getCategory();
String categoryBriefDescription = cat.getBriefDescription();
String categoryDescription= cat.getDescription();
String categoryID=cat.getId();
String categoryName=cat.getName();
for(OperatorsClassification opClass: opClassifications) {
// Load info
String opClassName =opClass.getName();
// Operator info
String opBriefDescription=op.getBriefDescription();
String opDescription=op.getDescription();
String opID=op.getId();
String operatorName=op.getName();
// Parameters info
for(Parameter param:op.getOperatorParameters()) {
String paramDescription=param.getDescription();
String paramName=param.getName();
String paramType=param.getTypology().toString();
String paramValue=param.getValue();
List<Operator> ops=opClass.getOperators();
log.debug("Found {} operators under classification {} ",ops.size(),opClassName);
for(Operator op : ops) {
InternalAlgorithmDescriptor desc=new InternalAlgorithmDescriptor();
desc.setClassName(opClassName);
// OperatorCategory Info
OperatorCategory cat = op.getCategory();
String categoryBriefDescription = cat.getBriefDescription();
String categoryDescription= cat.getDescription();
String categoryID=cat.getId();
String categoryName=cat.getName();
desc.setCategoryBriefDescription(categoryBriefDescription);
desc.setCategoryID(categoryID);
desc.setCategoryName(categoryName);
desc.setCategoryDescription(categoryDescription);
// Operator info
String opBriefDescription=op.getBriefDescription();
String opDescription=op.getDescription();
String opID=op.getId();
String operatorName=op.getName();
desc.setBriefDescription(opBriefDescription);
desc.setDescription(opDescription);
desc.setId(opID);
desc.setName(operatorName);
// Parameters info
for(Parameter param:op.getOperatorParameters()) {
String paramDescription=param.getDescription();
String paramName=param.getName();
String paramType=param.getTypology().toString();
String paramValue=param.getValue();
}
toReturn.add(desc);
}
// Create bean
}
// Create bean
}
HashSet<InternalAlgorithmDescriptor> toReturn=new HashSet();
return toReturn;
return toReturn;
}catch(Exception e) {
throw new CollectorFault("Unable to retrieve information",e);
}
}
}

21
DataMinerAlgorithmsCrawler/src/main/java/org/gcube/data/publishing/gCatFeeder/collectors/dm/DataMinerPlugin.java
View File

@ -3,14 +3,12 @@ package org.gcube.data.publishing.gCatFeeder.collectors.dm;
import java.util.Collections;
import java.util.Set;
import org.gcube.data.publishing.gCatFeeder.collectors.dm.model.CkanModel;
import org.gcube.data.publishing.gCatFeeder.collectors.dm.model.InternalAlgorithmDescriptor;
import org.gcube.data.publishing.gCatFeeder.model.FormatData;
import org.gcube.data.publishing.gCatFeeder.model.CatalogueFormatData;
import org.gcube.data.publishing.gCatfeeder.collectors.CatalogueRetriever;
import org.gcube.data.publishing.gCatfeeder.collectors.CollectorPlugin;
import org.gcube.data.publishing.gCatfeeder.collectors.DataCollector;
import org.gcube.data.publishing.gCatfeeder.collectors.DataTransformer;
import org.gcube.data.publishing.gCatfeeder.collectors.model.CustomData;
import org.gcube.data.publishing.gCatfeeder.collectors.model.PluginDescriptor;
import org.gcube.data.publishing.gCatfeeder.collectors.model.faults.CatalogueNotSupportedException;
@ -23,16 +21,16 @@ public class DataMinerPlugin implements CollectorPlugin<InternalAlgorithmDescrip
}
@Override
public CatalogueRetriever getRetrieverByCatalogueType(FormatData catalogueType) throws CatalogueNotSupportedException {
public CatalogueRetriever getRetrieverByCatalogueType(String catalogueType) throws CatalogueNotSupportedException {
switch(catalogueType) {
case CKAN : return CKANRetriever.get();
case Constants.CKAN_TYPE : return CKANRetriever.get();
default : throw new CatalogueNotSupportedException("No support for "+catalogueType);
}
}
@Override
public Set<DataTransformer<?,InternalAlgorithmDescriptor>> getImplementedTransformers() {
return Collections.singleton((DataTransformer<CkanModel,InternalAlgorithmDescriptor>) new CKANTransformer());
public Set<String> getSupportedCatalogueTypes() {
return Collections.singleton(Constants.CKAN_TYPE);
}
@Override
@ -40,4 +38,13 @@ public class DataMinerPlugin implements CollectorPlugin<InternalAlgorithmDescrip
return new DMAlgorithmsInfoCollector();
}
@Override
public DataTransformer<? extends CatalogueFormatData, InternalAlgorithmDescriptor> getTransformerByCatalogueType(
String catalogueType) throws CatalogueNotSupportedException {
switch(catalogueType) {
case Constants.CKAN_TYPE : return new CKANTransformer();
default : throw new CatalogueNotSupportedException("No support for "+catalogueType);
}
}
}

23
DataMinerAlgorithmsCrawler/src/main/java/org/gcube/data/publishing/gCatFeeder/collectors/dm/model/CkanModel.java
View File

@ -1,9 +1,14 @@
package org.gcube.data.publishing.gCatFeeder.collectors.dm.model;
import org.gcube.data.publishing.gCatFeeder.model.FormatData;
import org.gcube.data.publishing.gCatfeeder.collectors.model.PublisherFormatData;
import org.gcube.data.publishing.gCatFeeder.model.CatalogueFormatData;
public class CkanModel implements PublisherFormatData {
public class CkanModel implements CatalogueFormatData {
@Override
public String toCatalogueFormat() {
// TODO Auto-generated method stub
return null;
}
/**
@ -25,15 +30,5 @@ public class CkanModel implements PublisherFormatData {
*/
@Override
public FormatData getFormat() {
return FormatData.CKAN;
}
@Override
public String serialize() {
// TODO Auto-generated method stub
return null;
}
}

63
DataMinerAlgorithmsCrawler/src/main/java/org/gcube/data/publishing/gCatFeeder/collectors/dm/model/InternalAlgorithmDescriptor.java
View File

@ -1,20 +1,79 @@
package org.gcube.data.publishing.gCatFeeder.collectors.dm.model;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import org.gcube.data.publishing.gCatfeeder.collectors.model.CustomData;
import lombok.AllArgsConstructor;
import lombok.Getter;
import lombok.Setter;
@Getter
@Setter
public class InternalAlgorithmDescriptor implements CustomData {
@AllArgsConstructor
public static class Parameter{
private String name;
private String type;
private String description;
private String value;
@Override
public int hashCode() {
final int prime = 31;
int result = 1;
result = prime * result + ((name == null) ? 0 : name.hashCode());
return result;
}
@Override
public boolean equals(Object obj) {
if (this == obj)
return true;
if (obj == null)
return false;
if (getClass() != obj.getClass())
return false;
Parameter other = (Parameter) obj;
if (name == null) {
if (other.name != null)
return false;
} else if (!name.equals(other.name))
return false;
return true;
}
}
private String className;
//operator info
private String name;
private String Description;
private List<String> tags;
private String description;
private String briefDescription;
private String id;
//category info
private String categoryBriefDescription;
private String categoryDescription;
private String categoryID;
private String categoryName;
private Set<Parameter> parameters=new HashSet<>();
private List<String> tags;
public CkanModel asCKANModel() {
throw new RuntimeException("Implement This");
}
}

55
DataMinerAlgorithmsCrawler/src/test/java/org/gcube/data/publishing/gCatFeeder/collectors/dm/TranslationTest.java Normal file
View File

@ -0,0 +1,55 @@
package org.gcube.data.publishing.gCatFeeder.collectors.dm;
import java.io.IOException;
import java.util.Collection;
import java.util.Set;
import org.codehaus.jackson.JsonGenerationException;
import org.codehaus.jackson.map.JsonMappingException;
import org.codehaus.jackson.map.ObjectMapper;
import org.gcube.common.authorization.library.provider.SecurityTokenProvider;
import org.gcube.data.publishing.gCatFeeder.collectors.dm.model.InternalAlgorithmDescriptor;
import org.gcube.data.publishing.gCatFeeder.model.CatalogueFormatData;
import org.gcube.data.publishing.gCatfeeder.collectors.CollectorPlugin;
import org.gcube.data.publishing.gCatfeeder.collectors.DataCollector;
import org.gcube.data.publishing.gCatfeeder.collectors.DataTransformer;
import org.gcube.data.publishing.gCatfeeder.collectors.model.faults.CatalogueNotSupportedException;
import org.gcube.data.publishing.gCatfeeder.collectors.model.faults.CollectorFault;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;
public class TranslationTest {
@BeforeClass
public static void checkTestEnabled() {
}
@Before
public void setToken() {
// disable if maven property
// set token
SecurityTokenProvider.instance.set("***REMOVED***");
}
@Test
public void testTranslation() throws CollectorFault, CatalogueNotSupportedException, JsonGenerationException, JsonMappingException, IOException {
ObjectMapper mapper = new ObjectMapper();
CollectorPlugin plugin=new DataMinerPlugin();
DataCollector collector=plugin.getCollector();
Collection collected=collector.collect();
System.out.println("Found "+collected.size()+" elements");
for(Object obj:collected)
System.out.println(mapper.writeValueAsString(obj)+"\n");
for(String destinationcatalogue : (Set<String>)plugin.getSupportedCatalogueTypes()) {
DataTransformer<? extends CatalogueFormatData, InternalAlgorithmDescriptor> transformer=plugin.getTransformerByCatalogueType(destinationcatalogue);
transformer.transform(collected);
}
}
}

5
catalogue-plugin-framework/src/main/java/org/gcube/data/publishing/gCatFeeder/catalogues/CatalogueController.java
View File

@ -1,13 +1,12 @@
package org.gcube.data.publishing.gCatFeeder.catalogues;
import java.io.Serializable;
import org.gcube.data.publishing.gCatFeeder.catalogues.model.PublishReport;
import org.gcube.data.publishing.gCatFeeder.catalogues.model.faults.WrongObjectFormatException;
import org.gcube.data.publishing.gCatFeeder.model.CatalogueFormatData;
public interface CatalogueController {
public PublishReport publishItem(Serializable toPublish) throws WrongObjectFormatException;
public PublishReport publishItem(CatalogueFormatData toPublish) throws WrongObjectFormatException;
}

4
collectors-plugin-framework/src/main/java/org/gcube/data/publishing/gCatfeeder/collectors/CollectorPlugin.java
View File

@ -1,8 +1,8 @@
package org.gcube.data.publishing.gCatfeeder.collectors;
import java.io.Serializable;
import java.util.Set;
import org.gcube.data.publishing.gCatFeeder.model.CatalogueFormatData;
import org.gcube.data.publishing.gCatfeeder.collectors.model.CustomData;
import org.gcube.data.publishing.gCatfeeder.collectors.model.PluginDescriptor;
import org.gcube.data.publishing.gCatfeeder.collectors.model.faults.CatalogueNotSupportedException;
@ -15,7 +15,7 @@ public interface CollectorPlugin<E extends CustomData>{
public Set<String> getSupportedCatalogueTypes();
public DataTransformer<Serializable,E> getTransformerByCatalogueType(String catalogueType);
public DataTransformer<? extends CatalogueFormatData,E> getTransformerByCatalogueType(String catalogueType)throws CatalogueNotSupportedException;
public DataCollector<E> getCollector();
}

4
collectors-plugin-framework/src/main/java/org/gcube/data/publishing/gCatfeeder/collectors/DataTransformer.java
View File

@ -1,12 +1,12 @@
package org.gcube.data.publishing.gCatfeeder.collectors;
import java.io.Serializable;
import java.util.Collection;
import java.util.Set;
import org.gcube.data.publishing.gCatFeeder.model.CatalogueFormatData;
import org.gcube.data.publishing.gCatfeeder.collectors.model.CustomData;
public interface DataTransformer<T extends Serializable,E extends CustomData> {
public interface DataTransformer<T extends CatalogueFormatData,E extends CustomData> {
public Set<T> transform(Collection<E> collectedData);

8
commons/src/main/java/org/gcube/data/publishing/gCatFeeder/model/CatalogueFormatData.java Normal file
View File

@ -0,0 +1,8 @@
package org.gcube.data.publishing.gCatFeeder.model;
public interface CatalogueFormatData {
public String toCatalogueFormat();
}

5
gCat-Feeder/src/main/java/org/gcube/data/publishing/gCatFeeder/service/engine/impl/ExecutionTask.java
View File

@ -13,6 +13,7 @@ import org.gcube.data.publishing.gCatFeeder.catalogues.CataloguePlugin;
import org.gcube.data.publishing.gCatFeeder.catalogues.model.PublishReport;
import org.gcube.data.publishing.gCatFeeder.catalogues.model.faults.ControllerInstantiationFault;
import org.gcube.data.publishing.gCatFeeder.catalogues.model.faults.WrongObjectFormatException;
import org.gcube.data.publishing.gCatFeeder.model.CatalogueFormatData;
import org.gcube.data.publishing.gCatFeeder.model.CatalogueInstanceDescriptor;
import org.gcube.data.publishing.gCatFeeder.service.engine.CatalogueControllersManager;
import org.gcube.data.publishing.gCatFeeder.service.engine.CollectorsManager;
@ -174,11 +175,11 @@ public class ExecutionTask implements Runnable {
// ** TRANSFORM
log.debug("Transforming Collected Data");
DataTransformer transformer=collectorPlugin.getTransformerByCatalogueType(cataloguePluginId);
Set<Serializable> transformed=transformer.transform(collectedData);
Set<CatalogueFormatData> transformed=transformer.transform(collectedData);
log.trace("Going to publish {} items to {} ",transformed.size(),instanceDescriptor.getUrl());
// ** PUBLISH VIA CONTROLLER
for(Serializable item : transformed) {
for(CatalogueFormatData item : transformed) {
try {
PublishReport itemReport=controller.publishItem(item);
}catch(WrongObjectFormatException e) {