This commit is contained in:
Fabio Sinibaldi 2019-04-01 13:30:16 +00:00
parent 76417a6212
commit 4e512ef3d1
26 changed files with 305 additions and 54 deletions

View File

@ -26,6 +26,7 @@ public class DMAlgorithmsInfoCollector implements DataCollector<InternalAlgorith
log.trace("Collecting information for Dataminer algorithms");
HashSet<InternalAlgorithmDescriptor> toReturn=new HashSet();
SClient client=new DataMinerService().getClient();
// client.
List<OperatorsClassification> opClassifications=client.getOperatorsClassifications();
log.debug("Found {} classifications.",opClassifications.size());
@ -79,6 +80,9 @@ public class DMAlgorithmsInfoCollector implements DataCollector<InternalAlgorith
String paramValue=param.getValue();
}
// op.get
toReturn.add(desc);
}

View File

@ -1,9 +1,13 @@
package org.gcube.data.publishing.gCatFeeder.collectors.dm;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.Collections;
import java.util.Set;
import org.gcube.data.publishing.gCatFeeder.collectors.dm.model.InternalAlgorithmDescriptor;
import org.gcube.data.publishing.gCatFeeder.collectors.dm.model.ckan.GCatModel;
import org.gcube.data.publishing.gCatFeeder.model.CatalogueFormatData;
import org.gcube.data.publishing.gCatFeeder.model.ControllerConfiguration;
import org.gcube.data.publishing.gCatfeeder.collectors.CatalogueRetriever;
@ -13,6 +17,9 @@ import org.gcube.data.publishing.gCatfeeder.collectors.DataTransformer;
import org.gcube.data.publishing.gCatfeeder.collectors.model.PluginDescriptor;
import org.gcube.data.publishing.gCatfeeder.collectors.model.faults.CatalogueNotSupportedException;
import lombok.extern.slf4j.Slf4j;
@Slf4j
public class DataMinerPlugin implements CollectorPlugin<InternalAlgorithmDescriptor> {
@Override
@ -49,8 +56,11 @@ public class DataMinerPlugin implements CollectorPlugin<InternalAlgorithmDescrip
@Override
public void init() throws Exception {
// TODO Auto-generated method stub
log.debug("Initializing..");
Path p=Paths.get(this.getClass().getResource( "profile.xml").toURI());
log.debug("Path is : {} ",p.toUri());
GCatModel.setProfile(
new String(Files.readAllBytes(p)));
}
@Override

View File

@ -4,15 +4,15 @@ import java.util.Collection;
import java.util.HashSet;
import java.util.Set;
import org.gcube.data.publishing.gCatFeeder.collectors.dm.model.CkanModel;
import org.gcube.data.publishing.gCatFeeder.collectors.dm.model.InternalAlgorithmDescriptor;
import org.gcube.data.publishing.gCatFeeder.collectors.dm.model.ckan.GCatModel;
import org.gcube.data.publishing.gCatfeeder.collectors.DataTransformer;
public class GCATTransformer implements DataTransformer<CkanModel,InternalAlgorithmDescriptor>{
public class GCATTransformer implements DataTransformer<GCatModel,InternalAlgorithmDescriptor>{
@Override
public Set<CkanModel> transform(Collection<InternalAlgorithmDescriptor> collectedData) {
HashSet<CkanModel> toReturn=new HashSet<>();
public Set<GCatModel> transform(Collection<InternalAlgorithmDescriptor> collectedData) {
HashSet<GCatModel> toReturn=new HashSet<>();
for(InternalAlgorithmDescriptor desc:collectedData) {
toReturn.add(desc.asCKANModel());
}

View File

@ -1,34 +0,0 @@
package org.gcube.data.publishing.gCatFeeder.collectors.dm.model;
import org.gcube.data.publishing.gCatFeeder.model.CatalogueFormatData;
public class CkanModel implements CatalogueFormatData {
@Override
public String toCatalogueFormat() {
// TODO Auto-generated method stub
return null;
}
/**
* (Common) Title
* (Common) Description
* (Common) Tags: free list of keywords
* (Common) License
* (Common) Visibility: either public or private
* (Common) Version
* (Common) Author: the creator of metadata. Only one occurrence is supported;
* (Common) Maintainer:
* (Method specific) Creator: the author of the method (with email and ORCID). Repeatable field;
* (Method specific) Creation date: when the method has been released;
* (Method specific) Input: Repeatable field;
* (Method specific) Output: Repeatable field;
* (Method specific) RelatedPaper: a reference to an associated paper;
* (Method specific) Restrictions On Use: an optional text
* (Method specific) Attribution requirements: the text to use to acknowledge method usage;
*/
}

View File

@ -1,9 +1,11 @@
package org.gcube.data.publishing.gCatFeeder.collectors.dm.model;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import org.gcube.data.publishing.gCatFeeder.collectors.dm.model.ckan.GCatModel;
import org.gcube.data.publishing.gCatfeeder.collectors.model.CustomData;
import lombok.AllArgsConstructor;
@ -68,10 +70,10 @@ public class InternalAlgorithmDescriptor implements CustomData {
private List<String> tags;
private List<String> tags=new ArrayList<>();
public CkanModel asCKANModel() {
throw new RuntimeException("Implement This");
public GCatModel asCKANModel() {
return new GCatModel(this);
}

View File

@ -0,0 +1,16 @@
package org.gcube.data.publishing.gCatFeeder.collectors.dm.model.ckan;
import lombok.AllArgsConstructor;
import lombok.Getter;
import lombok.NoArgsConstructor;
import lombok.Setter;
@Getter
@Setter
@NoArgsConstructor
@AllArgsConstructor
public class CKanExtraField {
private String key;
private String value;
}

View File

@ -0,0 +1,27 @@
package org.gcube.data.publishing.gCatFeeder.collectors.dm.model.ckan;
import java.util.ArrayList;
import lombok.Getter;
import lombok.Setter;
@Getter
@Setter
public class CkanItem {
public CkanItem() {
// TODO Auto-generated constructor stub
}
private String name;
private String title;
private String description;
private String version;
private String visibility;
private String license;
private String author;
private String maintainer;
private ArrayList<String> tags=new ArrayList<String>();
private ArrayList<CKanExtraField> extras=new ArrayList<>();
}

View File

@ -0,0 +1,7 @@
package org.gcube.data.publishing.gCatFeeder.collectors.dm.model.ckan;
public class CkanResource {
}

View File

@ -0,0 +1,92 @@
package org.gcube.data.publishing.gCatFeeder.collectors.dm.model.ckan;
import java.io.ByteArrayOutputStream;
import java.util.ArrayList;
import org.gcube.data.publishing.gCatFeeder.collectors.dm.model.InternalAlgorithmDescriptor;
import org.gcube.data.publishing.gCatFeeder.model.CatalogueFormatData;
import org.gcube.data.publishing.gCatFeeder.model.InternalConversionException;
import org.gcube.data.publishing.gCatFeeder.utils.ContextUtils;
import com.fasterxml.jackson.databind.ObjectMapper;
import lombok.Getter;
import lombok.NoArgsConstructor;
import lombok.Setter;
@NoArgsConstructor
@Getter
@Setter
public class GCatModel implements CatalogueFormatData {
private static ObjectMapper mapper=new ObjectMapper();
private static String profileXML=null;
public static void setProfile(String toSet) {
profileXML=toSet;
}
public GCatModel(InternalAlgorithmDescriptor desc) {
item=new CkanItem();
// item.setAuthor(desc.getAuthor());
item.setDescription(desc.getDescription());
item.setTitle(desc.getName()+" in "+ContextUtils.getCurrentScopeName());
item.setLicense("CC-BY-NC-SA-4.0");
// item.setMaintainer(desc.getAuthor());
item.setName(item.getTitle().toLowerCase().toLowerCase().replaceAll(" ", "_"));
for(String tag:desc.getTags()) {
item.getTags().add(fixTag(tag));
}
item.getExtras().add(new CKanExtraField("system", "Algorithm"));
}
private String profile=profileXML;
private CkanItem item=null;
private ArrayList<CkanResource> resources=new ArrayList<>();
@Override
public String toCatalogueFormat() throws InternalConversionException {
try{
ByteArrayOutputStream baos=new ByteArrayOutputStream();
mapper.writeValue(baos, this);
return baos.toString();
}catch(Throwable t) {
throw new InternalConversionException("Unable to convert",t);
}
}
/**
* (Common) Title
* (Common) Description
* (Common) Tags: free list of keywords
* (Common) License
* (Common) Visibility: either public or private
* (Common) Version
* (Common) Author: the creator of metadata. Only one occurrence is supported;
* (Common) Maintainer:
* (Method specific) Creator: the author of the method (with email and ORCID). Repeatable field;
* (Method specific) Creation date: when the method has been released;
* (Method specific) Input: Repeatable field;
* (Method specific) Output: Repeatable field;
* (Method specific) RelatedPaper: a reference to an associated paper;
* (Method specific) Restrictions On Use: an optional text
* (Method specific) Attribution requirements: the text to use to acknowledge method usage;
*/
static final String fixTag(String toFix) {
String fixedTag=toFix.replaceAll(":", " ").
replaceAll("[\\(\\)\\\\/]", "-").replaceAll("[']","_");
if(fixedTag.length()>100)
fixedTag=fixedTag.substring(0,96)+"...";
return fixedTag.trim();
}
}

View File

@ -0,0 +1,8 @@
<metadataformat type="Algorithm">
<metadatafield>
<fieldName>Input Parameter</fieldName>
<dataType>String</dataType>
<mandatory>false</mandatory>
<note>Input parameter expected for the execution of the algorithm</note>
</metadatafield>
</metadataformat>

View File

@ -0,0 +1,12 @@
package org.gcube.data.publishing.gCatFeeder.collectors.dm;
import org.junit.Test;
public class LocalTest {
@Test
public void initTest() throws Exception {
new DataMinerPlugin().init();
}
}

View File

@ -6,10 +6,11 @@ import org.gcube.data.publishing.gCatFeeder.catalogues.model.faults.PublicationE
import org.gcube.data.publishing.gCatFeeder.catalogues.model.faults.WrongObjectFormatException;
import org.gcube.data.publishing.gCatFeeder.model.CatalogueFormatData;
import org.gcube.data.publishing.gCatFeeder.model.ControllerConfiguration;
import org.gcube.data.publishing.gCatFeeder.model.InternalConversionException;
public interface CatalogueController {
public PublishReport publishItem(CatalogueFormatData toPublish) throws WrongObjectFormatException,CatalogueInteractionException,PublicationException;
public PublishReport publishItem(CatalogueFormatData toPublish) throws WrongObjectFormatException,CatalogueInteractionException,PublicationException, InternalConversionException;
public void configure (ControllerConfiguration config);
}

View File

@ -3,6 +3,6 @@ package org.gcube.data.publishing.gCatFeeder.model;
public interface CatalogueFormatData {
public String toCatalogueFormat();
public String toCatalogueFormat() throws InternalConversionException;
}

View File

@ -0,0 +1,35 @@
package org.gcube.data.publishing.gCatFeeder.model;
public class InternalConversionException extends Exception {
/**
*
*/
private static final long serialVersionUID = -5731594279138149384L;
public InternalConversionException() {
// TODO Auto-generated constructor stub
}
public InternalConversionException(String message) {
super(message);
// TODO Auto-generated constructor stub
}
public InternalConversionException(Throwable cause) {
super(cause);
// TODO Auto-generated constructor stub
}
public InternalConversionException(String message, Throwable cause) {
super(message, cause);
// TODO Auto-generated constructor stub
}
public InternalConversionException(String message, Throwable cause, boolean enableSuppression,
boolean writableStackTrace) {
super(message, cause, enableSuppression, writableStackTrace);
// TODO Auto-generated constructor stub
}
}

View File

@ -26,6 +26,7 @@ import org.gcube.data.publishing.gCatFeeder.model.CatalogueFormatData;
import org.gcube.data.publishing.gCatFeeder.model.CatalogueInstanceDescriptor;
import org.gcube.data.publishing.gCatFeeder.model.ControllerConfiguration;
import org.gcube.data.publishing.gCatFeeder.model.ControllerConfiguration.PublishingPolicy;
import org.gcube.data.publishing.gCatFeeder.model.InternalConversionException;
import org.gcube.gcat.client.Item;
import org.gcube.gcat.client.Profile;
import org.gcube.gcat.client.Resource;
@ -115,11 +116,12 @@ public class GCatController implements CatalogueController{
* }
*
* NB serialized resources are updated with "package_id" set as the published item id
* @throws InternalConversionException
*
*
*/
@Override
public PublishReport publishItem(CatalogueFormatData arg0) throws WrongObjectFormatException,CatalogueInteractionException,PublicationException{
public PublishReport publishItem(CatalogueFormatData arg0) throws WrongObjectFormatException,CatalogueInteractionException,PublicationException, InternalConversionException{
log.debug("Publishing {} ",arg0);
String serialized=arg0.toCatalogueFormat();
try {

View File

@ -11,6 +11,7 @@ import org.gcube.data.publishing.gCatFeeder.catalogues.model.faults.PublicationE
import org.gcube.data.publishing.gCatFeeder.catalogues.model.faults.WrongObjectFormatException;
import org.gcube.data.publishing.gCatFeeder.model.CatalogueFormatData;
import org.gcube.data.publishing.gCatFeeder.model.CatalogueInstanceDescriptor;
import org.gcube.data.publishing.gCatFeeder.model.InternalConversionException;
import org.gcube.data.publishing.gCatFeeder.tests.TokenSetter;
import org.gcube.data.publishing.gCataFeeder.catalogues.gCat.GCatPlugin;
import org.junit.Assert;
@ -73,19 +74,19 @@ public class Interactions {
@Test(expected=WrongObjectFormatException.class)
public void empties() throws CatalogueInteractionException, WrongObjectFormatException, PublicationException, ControllerInstantiationFault {
public void empties() throws CatalogueInteractionException, WrongObjectFormatException, PublicationException, ControllerInstantiationFault, InternalConversionException {
publish(getController(),"empties.json");
}
@Test(expected=PublicationException.class)
public void missingProfile() throws CatalogueInteractionException, WrongObjectFormatException, PublicationException, ControllerInstantiationFault {
public void missingProfile() throws CatalogueInteractionException, WrongObjectFormatException, PublicationException, ControllerInstantiationFault, InternalConversionException {
publish(getController(),"missingProfile.json");
}
@Test(expected=WrongObjectFormatException.class)
public void noitem() throws CatalogueInteractionException, WrongObjectFormatException, PublicationException, ControllerInstantiationFault {
public void noitem() throws CatalogueInteractionException, WrongObjectFormatException, PublicationException, ControllerInstantiationFault, InternalConversionException {
publish(getController(),"noItem.json");
}
@ -93,13 +94,13 @@ public class Interactions {
public void item() {
try {
publish(getController(),"onlyItem.json");
} catch (WrongObjectFormatException | PublicationException | ControllerInstantiationFault e) {
} catch (WrongObjectFormatException | PublicationException | ControllerInstantiationFault | InternalConversionException e) {
Assert.fail(e.getMessage());
}
}
private static PublishReport publish(CatalogueController controller, String filename) throws CatalogueInteractionException, WrongObjectFormatException, PublicationException {
private static PublishReport publish(CatalogueController controller, String filename) throws CatalogueInteractionException, WrongObjectFormatException, PublicationException, InternalConversionException {
return controller.publishItem(new CustomTargetFormat(filename));
}
}

View File

@ -1,5 +1,5 @@
#Generated by Maven Integration for Eclipse
#Thu Mar 28 15:36:44 CET 2019
#Mon Apr 01 15:14:16 CEST 2019
version=1.0.0-SNAPSHOT
groupId=org.gcube.data-publishing.gCat-Feeder
m2e.projectName=gCat-Controller

View File

@ -13,6 +13,7 @@ import org.gcube.data.publishing.gCatFeeder.catalogues.model.faults.PublicationE
import org.gcube.data.publishing.gCatFeeder.catalogues.model.faults.WrongObjectFormatException;
import org.gcube.data.publishing.gCatFeeder.model.CatalogueFormatData;
import org.gcube.data.publishing.gCatFeeder.model.CatalogueInstanceDescriptor;
import org.gcube.data.publishing.gCatFeeder.model.InternalConversionException;
import org.gcube.data.publishing.gCatFeeder.service.engine.CatalogueControllersManager;
import org.gcube.data.publishing.gCatFeeder.service.engine.CollectorsManager;
import org.gcube.data.publishing.gCatFeeder.service.engine.Infrastructure;
@ -204,6 +205,8 @@ public class ExecutionTask implements Runnable {
catalogueReport.getPublishedRecords().add(new PublishReport(false,"Error while communicating with catalogue : "+e.getMessage()));
} catch (PublicationException e) {
catalogueReport.getPublishedRecords().add(new PublishReport(false,"Publication error : "+e.getMessage()));
} catch (InternalConversionException e) {
catalogueReport.getPublishedRecords().add(new PublishReport(false,"Conversion error : "+e.getMessage()));
}
}

View File

@ -1,15 +1,20 @@
package org.gcube.data.publishing.gCatFeeder.tests;
import java.io.IOException;
import java.util.Collection;
import java.util.Properties;
import java.util.ServiceLoader;
import java.util.Set;
import org.gcube.common.authorization.library.provider.SecurityTokenProvider;
import org.gcube.common.scope.api.ScopeProvider;
import org.gcube.data.publishing.gCatFeeder.model.CatalogueFormatData;
import org.gcube.data.publishing.gCatFeeder.model.InternalConversionException;
import org.gcube.data.publishing.gCatfeeder.collectors.CollectorPlugin;
import org.gcube.data.publishing.gCatfeeder.collectors.model.CustomData;
import org.gcube.data.publishing.gCatfeeder.collectors.model.PluginDescriptor;
import org.gcube.data.publishing.gCatfeeder.collectors.model.faults.CatalogueNotSupportedException;
import org.gcube.data.publishing.gCatfeeder.collectors.model.faults.CollectorFault;
import org.junit.Assert;
import org.junit.BeforeClass;
import org.slf4j.Logger;
@ -40,6 +45,7 @@ public class BaseCollectorTest {
try{
plugin.init();
}catch(Throwable t) {
log.error("",t);
Assert.fail("Unable to init plugin "+desc.getName());
}
@ -51,6 +57,7 @@ public class BaseCollectorTest {
Assert.assertNotNull(desc.getName()+": Null retriever for exposed "+supportedCatalogue,plugin.getRetrieverByCatalogueType(supportedCatalogue));
Assert.assertNotNull(desc.getName()+": Null transformer for exposed "+supportedCatalogue,plugin.getTransformerByCatalogueType(supportedCatalogue));
}catch(CatalogueNotSupportedException e) {
log.error("",e);
Assert.fail("Exposed supported catalogue actually not covered."+e.getMessage());
}
}
@ -72,12 +79,38 @@ public class BaseCollectorTest {
if(!props.containsKey(testContext)) throw new RuntimeException("No token found for scope : "+testContext);
SecurityTokenProvider.instance.set(props.getProperty(testContext));
ScopeProvider.instance.set(testContext);
for(CollectorPlugin<? extends CustomData> plugin:collectorPluginsLoader) {
try {
Collection collected=plugin.getCollector().collect();
for(String catalogue:plugin.getSupportedCatalogueTypes()) {
log.debug("Simulating publication towards {} ",catalogue);
Set<CatalogueFormatData> transformed=plugin.getTransformerByCatalogueType(catalogue).transform(collected);
log.debug("Transformed size = {} ");
for(CatalogueFormatData f:transformed)
log.debug(f.toCatalogueFormat());
}
} catch (CollectorFault e) {
log.error("",e);
Assert.fail("Infrastructure Environment not suitable for testing"+e.getMessage());
} catch (CatalogueNotSupportedException e) {
log.error("",e);
Assert.fail("Exception while getting transformer in scope : "+e.getMessage());
} catch (InternalConversionException e) {
log.error("",e);
Assert.fail("Failed internatl conversion : "+e.getMessage());
}
}
}
}
protected static boolean isTestInfrastructureEnabled() {
return testContext!=null;

View File

@ -0,0 +1,16 @@
<configuration debug="true">
<appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
<!-- encoders are assigned the type
ch.qos.logback.classic.encoder.PatternLayoutEncoder by default -->
<encoder>
<pattern>%d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n</pattern>
</encoder>
</appender>
<logger name="org.gcube" level="DEBUG"/>
<root level="WARN">
<appender-ref ref="STDOUT" />
</root>
</configuration>

View File

@ -1,5 +1,5 @@
#Generated by Maven Integration for Eclipse
#Thu Mar 28 15:36:44 CET 2019
#Mon Apr 01 15:14:17 CEST 2019
version=1.0.0-SNAPSHOT
groupId=org.gcube.data-publishing.gCat-Feeder
m2e.projectName=test-commons

View File

@ -0,0 +1,16 @@
<configuration debug="true">
<appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
<!-- encoders are assigned the type
ch.qos.logback.classic.encoder.PatternLayoutEncoder by default -->
<encoder>
<pattern>%d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n</pattern>
</encoder>
</appender>
<logger name="org.gcube" level="DEBUG"/>
<root level="WARN">
<appender-ref ref="STDOUT" />
</root>
</configuration>