git-svn-id: http://svn.research-infrastructures.eu/public/d4science/gcube/trunk/data-publishing/gCat-Feeder-Suite@178484 82a268e6-3cf1-43bd-a215-b396298e98cf
This commit is contained in:
parent
2702dd8fbc
commit
e2cd4abd8d
|
@ -0,0 +1,17 @@
|
|||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<projectDescription>
|
||||
<name>gCat-Feeder-Suite</name>
|
||||
<comment></comment>
|
||||
<projects>
|
||||
</projects>
|
||||
<buildSpec>
|
||||
<buildCommand>
|
||||
<name>org.eclipse.m2e.core.maven2Builder</name>
|
||||
<arguments>
|
||||
</arguments>
|
||||
</buildCommand>
|
||||
</buildSpec>
|
||||
<natures>
|
||||
<nature>org.eclipse.m2e.core.maven2Nature</nature>
|
||||
</natures>
|
||||
</projectDescription>
|
|
@ -0,0 +1,2 @@
|
|||
eclipse.preferences.version=1
|
||||
encoding/<project>=UTF-8
|
|
@ -0,0 +1,4 @@
|
|||
activeProfiles=
|
||||
eclipse.preferences.version=1
|
||||
resolveWorkspaceProjects=true
|
||||
version=1
|
|
@ -0,0 +1,36 @@
|
|||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<classpath>
|
||||
<classpathentry kind="src" output="target/classes" path="src/main/java">
|
||||
<attributes>
|
||||
<attribute name="optional" value="true"/>
|
||||
<attribute name="maven.pomderived" value="true"/>
|
||||
</attributes>
|
||||
</classpathentry>
|
||||
<classpathentry excluding="**" kind="src" output="target/classes" path="src/main/resources">
|
||||
<attributes>
|
||||
<attribute name="maven.pomderived" value="true"/>
|
||||
</attributes>
|
||||
</classpathentry>
|
||||
<classpathentry kind="src" output="target/test-classes" path="src/test/java">
|
||||
<attributes>
|
||||
<attribute name="optional" value="true"/>
|
||||
<attribute name="maven.pomderived" value="true"/>
|
||||
</attributes>
|
||||
</classpathentry>
|
||||
<classpathentry excluding="**" kind="src" output="target/test-classes" path="src/test/resources">
|
||||
<attributes>
|
||||
<attribute name="maven.pomderived" value="true"/>
|
||||
</attributes>
|
||||
</classpathentry>
|
||||
<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-1.8">
|
||||
<attributes>
|
||||
<attribute name="maven.pomderived" value="true"/>
|
||||
</attributes>
|
||||
</classpathentry>
|
||||
<classpathentry kind="con" path="org.eclipse.m2e.MAVEN2_CLASSPATH_CONTAINER">
|
||||
<attributes>
|
||||
<attribute name="maven.pomderived" value="true"/>
|
||||
</attributes>
|
||||
</classpathentry>
|
||||
<classpathentry kind="output" path="target/classes"/>
|
||||
</classpath>
|
|
@ -0,0 +1,23 @@
|
|||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<projectDescription>
|
||||
<name>DataMinerAlgorithmsCrawler</name>
|
||||
<comment></comment>
|
||||
<projects>
|
||||
</projects>
|
||||
<buildSpec>
|
||||
<buildCommand>
|
||||
<name>org.eclipse.jdt.core.javabuilder</name>
|
||||
<arguments>
|
||||
</arguments>
|
||||
</buildCommand>
|
||||
<buildCommand>
|
||||
<name>org.eclipse.m2e.core.maven2Builder</name>
|
||||
<arguments>
|
||||
</arguments>
|
||||
</buildCommand>
|
||||
</buildSpec>
|
||||
<natures>
|
||||
<nature>org.eclipse.jdt.core.javanature</nature>
|
||||
<nature>org.eclipse.m2e.core.maven2Nature</nature>
|
||||
</natures>
|
||||
</projectDescription>
|
|
@ -0,0 +1,6 @@
|
|||
eclipse.preferences.version=1
|
||||
encoding//src/main/java=UTF-8
|
||||
encoding//src/main/resources=UTF-8
|
||||
encoding//src/test/java=UTF-8
|
||||
encoding//src/test/resources=UTF-8
|
||||
encoding/<project>=UTF-8
|
|
@ -0,0 +1,5 @@
|
|||
eclipse.preferences.version=1
|
||||
org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.8
|
||||
org.eclipse.jdt.core.compiler.compliance=1.8
|
||||
org.eclipse.jdt.core.compiler.problem.forbiddenReference=warning
|
||||
org.eclipse.jdt.core.compiler.source=1.8
|
|
@ -0,0 +1,4 @@
|
|||
activeProfiles=
|
||||
eclipse.preferences.version=1
|
||||
resolveWorkspaceProjects=true
|
||||
version=1
|
|
@ -0,0 +1,28 @@
|
|||
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
<parent>
|
||||
<groupId>org.gcube.data-publishing.gCat-Feeder</groupId>
|
||||
<artifactId>gCat-Feeder-Suite</artifactId>
|
||||
<version>1.0.0-SNAPSHOT</version>
|
||||
</parent>
|
||||
<artifactId>DataMinerAlgorithmsCrawler</artifactId>
|
||||
<name>DataMinerAlgorithmsCrawler</name>
|
||||
<description>Plugin for gCat-Feeder for DataMiner Algorithms publishing</description>
|
||||
|
||||
|
||||
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>org.gcube.data-publishing.gCat-Feeder</groupId>
|
||||
<artifactId>crawler-plugin-framework</artifactId>
|
||||
<version>[1.0.0-SNAPSHOT,2.0.0-SNAPSHOT)</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.gcube.data.analysis</groupId>
|
||||
<artifactId>data-miner-manager-cl</artifactId>
|
||||
<version>[1.6.0-SNAPSHOT,2.0.0-SNAPSHOT)</version>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
</project>
|
|
@ -0,0 +1,26 @@
|
|||
package org.gcube.data.publishing.gCatFeeder.crawlers.dm;
|
||||
|
||||
import org.gcube.common.resources.gcore.GCoreEndpoint;
|
||||
import org.gcube.data.publishing.gCatFeeder.model.CatalogueInstanceDescriptor;
|
||||
import org.gcube.data.publishing.gCatFeeder.utils.ISUtils;
|
||||
import org.gcube.data.publishing.gCatfeeder.crawlers.CatalogueRetriever;
|
||||
|
||||
public class CKANRetriever implements CatalogueRetriever {
|
||||
|
||||
private static CKANRetriever instance=null;
|
||||
|
||||
static synchronized CKANRetriever get() {
|
||||
if(instance==null) instance =new CKANRetriever();
|
||||
return instance;
|
||||
}
|
||||
|
||||
|
||||
|
||||
@Override
|
||||
public CatalogueInstanceDescriptor getInstance() {
|
||||
throw new RuntimeException("Implement this");
|
||||
// GCoreEndpoint ep=ISUtils.queryForGCoreEndpoint("NO", "NO");
|
||||
// return null;
|
||||
}
|
||||
|
||||
}
|
|
@ -0,0 +1,29 @@
|
|||
package org.gcube.data.publishing.gCatFeeder.crawlers.dm;
|
||||
|
||||
import java.util.Collection;
|
||||
import java.util.HashSet;
|
||||
import java.util.Set;
|
||||
|
||||
import org.gcube.data.publishing.gCatFeeder.crawlers.dm.model.CkanModel;
|
||||
import org.gcube.data.publishing.gCatFeeder.crawlers.dm.model.InternalAlgorithmDescriptor;
|
||||
import org.gcube.data.publishing.gCatFeeder.model.FormatData;
|
||||
import org.gcube.data.publishing.gCatfeeder.crawlers.DataTransformer;
|
||||
|
||||
public class CKANTransformer implements DataTransformer<CkanModel, InternalAlgorithmDescriptor>{
|
||||
|
||||
|
||||
@Override
|
||||
public FormatData getTargetFormat() {
|
||||
return FormatData.CKAN;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Set<CkanModel> transform(Collection<InternalAlgorithmDescriptor> collectedData) {
|
||||
HashSet<CkanModel> toReturn=new HashSet<>();
|
||||
for(InternalAlgorithmDescriptor desc:collectedData) {
|
||||
toReturn.add(desc.asCKANModel());
|
||||
}
|
||||
return toReturn;
|
||||
}
|
||||
|
||||
}
|
|
@ -0,0 +1,68 @@
|
|||
package org.gcube.data.publishing.gCatFeeder.crawlers.dm;
|
||||
|
||||
import java.util.HashSet;
|
||||
import java.util.Set;
|
||||
|
||||
import org.gcube.data.analysis.dataminermanagercl.server.DataMinerService;
|
||||
import org.gcube.data.analysis.dataminermanagercl.server.dmservice.SClient;
|
||||
import org.gcube.data.analysis.dataminermanagercl.shared.parameters.Parameter;
|
||||
import org.gcube.data.analysis.dataminermanagercl.shared.process.Operator;
|
||||
import org.gcube.data.analysis.dataminermanagercl.shared.process.OperatorCategory;
|
||||
import org.gcube.data.analysis.dataminermanagercl.shared.process.OperatorsClassification;
|
||||
import org.gcube.data.publishing.gCatFeeder.crawlers.dm.model.InternalAlgorithmDescriptor;
|
||||
import org.gcube.data.publishing.gCatfeeder.crawlers.DataCollector;
|
||||
import org.gcube.data.publishing.gCatfeeder.crawlers.model.faults.CollectorFault;
|
||||
|
||||
|
||||
public class DMAlgorithmsInfoCollector implements DataCollector<InternalAlgorithmDescriptor> {
|
||||
|
||||
|
||||
@Override
|
||||
public Set<InternalAlgorithmDescriptor> collect() throws CollectorFault {
|
||||
try {
|
||||
SClient client=new DataMinerService().getClient();
|
||||
for(OperatorsClassification opClass: client.getOperatorsClassifications()) {
|
||||
// Load info
|
||||
String opClassName =opClass.getName();
|
||||
|
||||
for(Operator op : opClass.getOperators()) {
|
||||
// OperatorCategory Info
|
||||
OperatorCategory cat = op.getCategory();
|
||||
String categoryBriefDescription = cat.getBriefDescription();
|
||||
String categoryDescription= cat.getDescription();
|
||||
String categoryID=cat.getId();
|
||||
String categoryName=cat.getName();
|
||||
|
||||
// Operator info
|
||||
|
||||
String opBriefDescription=op.getBriefDescription();
|
||||
String opDescription=op.getDescription();
|
||||
String opID=op.getId();
|
||||
String operatorName=op.getName();
|
||||
|
||||
|
||||
// Parameters info
|
||||
for(Parameter param:op.getOperatorParameters()) {
|
||||
String paramDescription=param.getDescription();
|
||||
String paramName=param.getName();
|
||||
String paramType=param.getTypology().toString();
|
||||
String paramValue=param.getValue();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
// Create bean
|
||||
}
|
||||
|
||||
|
||||
|
||||
HashSet<InternalAlgorithmDescriptor> toReturn=new HashSet();
|
||||
|
||||
|
||||
return toReturn;
|
||||
}catch(Exception e) {
|
||||
throw new CollectorFault("Unable to retrieve information",e);
|
||||
}
|
||||
}
|
||||
|
||||
}
|
|
@ -0,0 +1,43 @@
|
|||
package org.gcube.data.publishing.gCatFeeder.crawlers.dm;
|
||||
|
||||
import java.util.Collections;
|
||||
import java.util.Set;
|
||||
|
||||
import org.gcube.data.publishing.gCatFeeder.crawlers.dm.model.CkanModel;
|
||||
import org.gcube.data.publishing.gCatFeeder.crawlers.dm.model.InternalAlgorithmDescriptor;
|
||||
import org.gcube.data.publishing.gCatFeeder.model.FormatData;
|
||||
import org.gcube.data.publishing.gCatfeeder.crawlers.CatalogueRetriever;
|
||||
import org.gcube.data.publishing.gCatfeeder.crawlers.CrawlerPlugin;
|
||||
import org.gcube.data.publishing.gCatfeeder.crawlers.DataCollector;
|
||||
import org.gcube.data.publishing.gCatfeeder.crawlers.DataTransformer;
|
||||
import org.gcube.data.publishing.gCatfeeder.crawlers.model.CustomData;
|
||||
import org.gcube.data.publishing.gCatfeeder.crawlers.model.PluginDescriptor;
|
||||
import org.gcube.data.publishing.gCatfeeder.crawlers.model.faults.CatalogueNotSupportedException;
|
||||
|
||||
public class DataMinerCrawler implements CrawlerPlugin<InternalAlgorithmDescriptor> {
|
||||
|
||||
@Override
|
||||
public PluginDescriptor getDescriptor() {
|
||||
// TODO Auto-generated method stub
|
||||
return null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public CatalogueRetriever getRetrieverByCatalogueType(FormatData catalogueType) throws CatalogueNotSupportedException {
|
||||
switch(catalogueType) {
|
||||
case CKAN : return CKANRetriever.get();
|
||||
default : throw new CatalogueNotSupportedException("No support for "+catalogueType);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public Set<DataTransformer<?,InternalAlgorithmDescriptor>> getImplementedTransformers() {
|
||||
return Collections.singleton((DataTransformer<CkanModel,InternalAlgorithmDescriptor>) new CKANTransformer());
|
||||
}
|
||||
|
||||
@Override
|
||||
public DataCollector<InternalAlgorithmDescriptor> getCollector() {
|
||||
return new DMAlgorithmsInfoCollector();
|
||||
}
|
||||
|
||||
}
|
|
@ -0,0 +1,39 @@
|
|||
package org.gcube.data.publishing.gCatFeeder.crawlers.dm.model;
|
||||
|
||||
import org.gcube.data.publishing.gCatFeeder.model.FormatData;
|
||||
import org.gcube.data.publishing.gCatfeeder.crawlers.model.PublisherFormatData;
|
||||
|
||||
public class CkanModel implements PublisherFormatData {
|
||||
|
||||
|
||||
/**
|
||||
* (Common) Title
|
||||
* (Common) Description
|
||||
* (Common) Tags: free list of keywords
|
||||
* (Common) License
|
||||
* (Common) Visibility: either public or private
|
||||
* (Common) Version
|
||||
* (Common) Author: the creator of metadata. Only one occurrence is supported;
|
||||
* (Common) Maintainer:
|
||||
* (Method specific) Creator: the author of the method (with email and ORCID). Repeatable field;
|
||||
* (Method specific) Creation date: when the method has been released;
|
||||
* (Method specific) Input: Repeatable field;
|
||||
* (Method specific) Output: Repeatable field;
|
||||
* (Method specific) RelatedPaper: a reference to an associated paper;
|
||||
* (Method specific) Restrictions On Use: an optional text
|
||||
* (Method specific) Attribution requirements: the text to use to acknowledge method usage;
|
||||
*/
|
||||
|
||||
|
||||
@Override
|
||||
public FormatData getFormat() {
|
||||
return FormatData.CKAN;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String serialize() {
|
||||
// TODO Auto-generated method stub
|
||||
return null;
|
||||
}
|
||||
|
||||
}
|
|
@ -0,0 +1,20 @@
|
|||
package org.gcube.data.publishing.gCatFeeder.crawlers.dm.model;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
import org.gcube.data.publishing.gCatfeeder.crawlers.model.CustomData;
|
||||
|
||||
public class InternalAlgorithmDescriptor implements CustomData {
|
||||
|
||||
private String name;
|
||||
private String Description;
|
||||
private List<String> tags;
|
||||
private String id;
|
||||
|
||||
|
||||
|
||||
public CkanModel asCKANModel() {
|
||||
throw new RuntimeException("Implement This");
|
||||
}
|
||||
|
||||
}
|
|
@ -0,0 +1,5 @@
|
|||
Manifest-Version: 1.0
|
||||
Built-By: fabio
|
||||
Build-Jdk: 1.8.0_201
|
||||
Created-By: Maven Integration for Eclipse
|
||||
|
|
@ -0,0 +1,7 @@
|
|||
#Generated by Maven Integration for Eclipse
|
||||
#Fri Mar 01 18:17:42 CET 2019
|
||||
version=1.0.0-SNAPSHOT
|
||||
groupId=org.gcube.data-publishing.gCat-Feeder
|
||||
m2e.projectName=DataMinerAlgorithmsCrawler
|
||||
m2e.projectLocation=/home/fabio/workspaces/DEV/gCat-Feeder-Suite/DataMinerAlgorithmsCrawler
|
||||
artifactId=DataMinerAlgorithmsCrawler
|
|
@ -0,0 +1,28 @@
|
|||
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
<parent>
|
||||
<groupId>org.gcube.data-publishing.gCat-Feeder</groupId>
|
||||
<artifactId>gCat-Feeder-Suite</artifactId>
|
||||
<version>1.0.0-SNAPSHOT</version>
|
||||
</parent>
|
||||
<artifactId>DataMinerAlgorithmsCrawler</artifactId>
|
||||
<name>DataMinerAlgorithmsCrawler</name>
|
||||
<description>Plugin for gCat-Feeder for DataMiner Algorithms publishing</description>
|
||||
|
||||
|
||||
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>org.gcube.data-publishing.gCat-Feeder</groupId>
|
||||
<artifactId>crawler-plugin-framework</artifactId>
|
||||
<version>[1.0.0-SNAPSHOT,2.0.0-SNAPSHOT)</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.gcube.data.analysis</groupId>
|
||||
<artifactId>data-miner-manager-cl</artifactId>
|
||||
<version>[1.6.0-SNAPSHOT,2.0.0-SNAPSHOT)</version>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
</project>
|
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
|
@ -0,0 +1,36 @@
|
|||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<classpath>
|
||||
<classpathentry kind="src" output="target/classes" path="src/main/java">
|
||||
<attributes>
|
||||
<attribute name="optional" value="true"/>
|
||||
<attribute name="maven.pomderived" value="true"/>
|
||||
</attributes>
|
||||
</classpathentry>
|
||||
<classpathentry excluding="**" kind="src" output="target/classes" path="src/main/resources">
|
||||
<attributes>
|
||||
<attribute name="maven.pomderived" value="true"/>
|
||||
</attributes>
|
||||
</classpathentry>
|
||||
<classpathentry kind="src" output="target/test-classes" path="src/test/java">
|
||||
<attributes>
|
||||
<attribute name="optional" value="true"/>
|
||||
<attribute name="maven.pomderived" value="true"/>
|
||||
</attributes>
|
||||
</classpathentry>
|
||||
<classpathentry excluding="**" kind="src" output="target/test-classes" path="src/test/resources">
|
||||
<attributes>
|
||||
<attribute name="maven.pomderived" value="true"/>
|
||||
</attributes>
|
||||
</classpathentry>
|
||||
<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-1.8">
|
||||
<attributes>
|
||||
<attribute name="maven.pomderived" value="true"/>
|
||||
</attributes>
|
||||
</classpathentry>
|
||||
<classpathentry kind="con" path="org.eclipse.m2e.MAVEN2_CLASSPATH_CONTAINER">
|
||||
<attributes>
|
||||
<attribute name="maven.pomderived" value="true"/>
|
||||
</attributes>
|
||||
</classpathentry>
|
||||
<classpathentry kind="output" path="target/classes"/>
|
||||
</classpath>
|
|
@ -0,0 +1,23 @@
|
|||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<projectDescription>
|
||||
<name>commons</name>
|
||||
<comment></comment>
|
||||
<projects>
|
||||
</projects>
|
||||
<buildSpec>
|
||||
<buildCommand>
|
||||
<name>org.eclipse.jdt.core.javabuilder</name>
|
||||
<arguments>
|
||||
</arguments>
|
||||
</buildCommand>
|
||||
<buildCommand>
|
||||
<name>org.eclipse.m2e.core.maven2Builder</name>
|
||||
<arguments>
|
||||
</arguments>
|
||||
</buildCommand>
|
||||
</buildSpec>
|
||||
<natures>
|
||||
<nature>org.eclipse.jdt.core.javanature</nature>
|
||||
<nature>org.eclipse.m2e.core.maven2Nature</nature>
|
||||
</natures>
|
||||
</projectDescription>
|
|
@ -0,0 +1,6 @@
|
|||
eclipse.preferences.version=1
|
||||
encoding//src/main/java=UTF-8
|
||||
encoding//src/main/resources=UTF-8
|
||||
encoding//src/test/java=UTF-8
|
||||
encoding//src/test/resources=UTF-8
|
||||
encoding/<project>=UTF-8
|
|
@ -0,0 +1,5 @@
|
|||
eclipse.preferences.version=1
|
||||
org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.8
|
||||
org.eclipse.jdt.core.compiler.compliance=1.8
|
||||
org.eclipse.jdt.core.compiler.problem.forbiddenReference=warning
|
||||
org.eclipse.jdt.core.compiler.source=1.8
|
File diff suppressed because one or more lines are too long
|
@ -0,0 +1,4 @@
|
|||
activeProfiles=
|
||||
eclipse.preferences.version=1
|
||||
resolveWorkspaceProjects=true
|
||||
version=1
|
|
@ -0,0 +1,33 @@
|
|||
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
<parent>
|
||||
<groupId>org.gcube.data-publishing.gCat-Feeder</groupId>
|
||||
<artifactId>gCat-Feeder-Suite</artifactId>
|
||||
<version>1.0.0-SNAPSHOT</version>
|
||||
</parent>
|
||||
<artifactId>commons</artifactId>
|
||||
<name>commons</name>
|
||||
<description>Common utilities for gCatFeeder logic</description>
|
||||
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>org.gcube.resourcemanagement</groupId>
|
||||
<artifactId>resourcemanager-client</artifactId>
|
||||
<version>[1.0.0-SNAPSHOT,2.0.0-SNAPSHOT)</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.slf4j</groupId>
|
||||
<artifactId>slf4j-api</artifactId>
|
||||
</dependency>
|
||||
|
||||
|
||||
<dependency>
|
||||
<groupId>org.gcube.common</groupId>
|
||||
<artifactId>authorization-client</artifactId>
|
||||
</dependency>
|
||||
|
||||
|
||||
</dependencies>
|
||||
|
||||
</project>
|
|
@ -0,0 +1,60 @@
|
|||
package org.gcube.data.publishing.gCatFeeder.model;
|
||||
|
||||
import java.util.HashMap;
|
||||
|
||||
public class CatalogueInstanceDescriptor {
|
||||
|
||||
private String url=null;
|
||||
private String user=null;
|
||||
private String password=null;
|
||||
|
||||
private String customToken=null;
|
||||
|
||||
private HashMap<String,String> headersParameters=new HashMap<>();
|
||||
|
||||
public CatalogueInstanceDescriptor(String url) {
|
||||
super();
|
||||
this.url = url;
|
||||
}
|
||||
|
||||
public String getUrl() {
|
||||
return url;
|
||||
}
|
||||
|
||||
public String getUser() {
|
||||
return user;
|
||||
}
|
||||
|
||||
public String getPassword() {
|
||||
return password;
|
||||
}
|
||||
|
||||
public String getCustomToken() {
|
||||
return customToken;
|
||||
}
|
||||
|
||||
public HashMap<String, String> getHeadersParameters() {
|
||||
return headersParameters;
|
||||
}
|
||||
|
||||
public CatalogueInstanceDescriptor setUser(String user) {
|
||||
this.user = user;
|
||||
return this;
|
||||
}
|
||||
|
||||
public CatalogueInstanceDescriptor setPassword(String password) {
|
||||
this.password = password;
|
||||
return this;
|
||||
}
|
||||
|
||||
public CatalogueInstanceDescriptor setCustomToken(String customToken) {
|
||||
this.customToken = customToken;
|
||||
return this;
|
||||
}
|
||||
|
||||
public CatalogueInstanceDescriptor addHeaderParameter(String name,String value) {
|
||||
this.headersParameters.put(name, value);
|
||||
return this;
|
||||
}
|
||||
|
||||
}
|
|
@ -0,0 +1,5 @@
|
|||
package org.gcube.data.publishing.gCatFeeder.model;
|
||||
|
||||
public enum FormatData {
|
||||
CKAN
|
||||
}
|
|
@ -0,0 +1,88 @@
|
|||
package org.gcube.data.publishing.gCatFeeder.utils;
|
||||
|
||||
import static org.gcube.common.authorization.client.Constants.authorizationService;
|
||||
|
||||
import java.util.ArrayList;
|
||||
|
||||
import org.gcube.common.authorization.library.AuthorizationEntry;
|
||||
import org.gcube.common.authorization.library.provider.SecurityTokenProvider;
|
||||
import org.gcube.common.scope.api.ScopeProvider;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
public class ContextUtils {
|
||||
|
||||
private static final Logger log= LoggerFactory.getLogger(ContextUtils.class);
|
||||
|
||||
public static String getCurrentCaller(){
|
||||
try{
|
||||
String token=SecurityTokenProvider.instance.get();
|
||||
log.debug("Token is : "+token);
|
||||
if(token==null) throw new Exception("Security Token is null");
|
||||
AuthorizationEntry entry = authorizationService().get(token);
|
||||
return entry.getClientInfo().getId();
|
||||
}catch(Exception e ){
|
||||
log.debug("Unable to resolve token, checking scope provider..",e);
|
||||
return "Unidentified data-transfer user";
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
public static String getCurrentScope(){
|
||||
|
||||
String scope=ScopeProvider.instance.get();
|
||||
if(scope!=null) {
|
||||
log.debug("Found scope provider {}, skipping token",scope);
|
||||
return scope;
|
||||
}else{
|
||||
try{
|
||||
log.debug("Scope provider not set, reverting to token");
|
||||
String token=SecurityTokenProvider.instance.get();
|
||||
log.debug("Token is : "+token);
|
||||
if(token==null) throw new Exception("Security Token is null");
|
||||
AuthorizationEntry entry = authorizationService().get(token);
|
||||
return entry.getContext();
|
||||
}catch(Exception e){
|
||||
throw new RuntimeException("Unable to evaluate scope ",e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
public static String getCurrentScopeName(){
|
||||
return getScopeName(getCurrentScope());
|
||||
}
|
||||
|
||||
public static String getScopeName(String scope) {
|
||||
return scope.substring(scope.lastIndexOf('/')+1);
|
||||
}
|
||||
|
||||
|
||||
public static ArrayList<String> getParentScopes(){
|
||||
String currentScope=getCurrentScope();
|
||||
String[] splitted=currentScope.substring(1).split("/");
|
||||
ArrayList<String> toReturn=new ArrayList<String>();
|
||||
for(int i=0;i<splitted.length-1;i++){
|
||||
toReturn.add(splitted[i]);
|
||||
}
|
||||
return toReturn;
|
||||
}
|
||||
|
||||
public static String getParentScope(String scope) {
|
||||
if(scope.lastIndexOf("/")==0) return null; // NO PARENT
|
||||
else return scope.substring(0, scope.lastIndexOf("/"));
|
||||
}
|
||||
|
||||
public static ArrayList<String> getParentScopes(String scope){
|
||||
String[] splitted=scope.substring(1).split("/");
|
||||
ArrayList<String> toReturn=new ArrayList<String>();
|
||||
for(int i=0;i<splitted.length-1;i++){
|
||||
toReturn.add(splitted[i]);
|
||||
}
|
||||
return toReturn;
|
||||
}
|
||||
|
||||
|
||||
}
|
|
@ -0,0 +1,48 @@
|
|||
package org.gcube.data.publishing.gCatFeeder.utils;
|
||||
|
||||
import static org.gcube.resources.discovery.icclient.ICFactory.clientFor;
|
||||
import static org.gcube.resources.discovery.icclient.ICFactory.queryFor;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
import org.gcube.common.resources.gcore.GCoreEndpoint;
|
||||
import org.gcube.common.resources.gcore.ServiceEndpoint;
|
||||
import org.gcube.resources.discovery.client.api.DiscoveryClient;
|
||||
import org.gcube.resources.discovery.client.queries.api.SimpleQuery;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
|
||||
public class ISUtils {
|
||||
|
||||
private static final Logger log= LoggerFactory.getLogger(ISUtils.class);
|
||||
|
||||
public static List<ServiceEndpoint> queryForServiceEndpoints(String category, String platformName){
|
||||
log.debug("Querying for Service Endpoints [category : {} , platformName : {}, currentScope : {} ]",category,platformName,ContextUtils.getCurrentScope());
|
||||
|
||||
SimpleQuery query = queryFor(ServiceEndpoint.class);
|
||||
|
||||
query.addCondition("$resource/Profile/Category/text() eq '"+category+"'")
|
||||
.addCondition("$resource/Profile/Platform/Name/text() eq '"+platformName+"'");
|
||||
// .setResult("$resource/Profile/AccessPoint");
|
||||
|
||||
DiscoveryClient<ServiceEndpoint> client = clientFor(ServiceEndpoint.class);
|
||||
|
||||
return client.submit(query);
|
||||
}
|
||||
|
||||
public static List<GCoreEndpoint> queryForGCoreEndpoint(String serviceClass,String serviceName){
|
||||
log.debug("Querying for GCore Endpoints [ServiceClass : {} , ServiceName : {}, currentScope : {} ]",serviceClass,serviceName,ContextUtils.getCurrentScope());
|
||||
|
||||
|
||||
SimpleQuery query =queryFor(GCoreEndpoint.class);
|
||||
query.addCondition("$resource/Profile/ServiceClass/text() eq '"+serviceClass+"'")
|
||||
.addCondition("$resource/Profile/ServiceName/text() eq '"+serviceName+"'");
|
||||
// .setResult("$resource/Profile/AccessPoint");
|
||||
|
||||
DiscoveryClient<GCoreEndpoint> client = clientFor(GCoreEndpoint.class);
|
||||
|
||||
return client.submit(query);
|
||||
}
|
||||
|
||||
}
|
|
@ -0,0 +1,5 @@
|
|||
package org.gcube.data.publishing.gCatFeeder.utils;
|
||||
|
||||
public class TokenUtils {
|
||||
|
||||
}
|
|
@ -0,0 +1,5 @@
|
|||
Manifest-Version: 1.0
|
||||
Built-By: fabio
|
||||
Build-Jdk: 1.8.0_201
|
||||
Created-By: Maven Integration for Eclipse
|
||||
|
|
@ -0,0 +1,7 @@
|
|||
#Generated by Maven Integration for Eclipse
|
||||
#Wed Mar 06 11:12:02 CET 2019
|
||||
version=1.0.0-SNAPSHOT
|
||||
groupId=org.gcube.data-publishing.gCat-Feeder
|
||||
m2e.projectName=commons
|
||||
m2e.projectLocation=/home/fabio/workspaces/DEV/gCat-Feeder-Suite/commons
|
||||
artifactId=commons
|
|
@ -0,0 +1,33 @@
|
|||
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
<parent>
|
||||
<groupId>org.gcube.data-publishing.gCat-Feeder</groupId>
|
||||
<artifactId>gCat-Feeder-Suite</artifactId>
|
||||
<version>1.0.0-SNAPSHOT</version>
|
||||
</parent>
|
||||
<artifactId>commons</artifactId>
|
||||
<name>commons</name>
|
||||
<description>Common utilities for gCatFeeder logic</description>
|
||||
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>org.gcube.resourcemanagement</groupId>
|
||||
<artifactId>resourcemanager-client</artifactId>
|
||||
<version>[1.0.0-SNAPSHOT,2.0.0-SNAPSHOT)</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.slf4j</groupId>
|
||||
<artifactId>slf4j-api</artifactId>
|
||||
</dependency>
|
||||
|
||||
|
||||
<dependency>
|
||||
<groupId>org.gcube.common</groupId>
|
||||
<artifactId>authorization-client</artifactId>
|
||||
</dependency>
|
||||
|
||||
|
||||
</dependencies>
|
||||
|
||||
</project>
|
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
|
@ -0,0 +1,36 @@
|
|||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<classpath>
|
||||
<classpathentry kind="src" output="target/classes" path="src/main/java">
|
||||
<attributes>
|
||||
<attribute name="optional" value="true"/>
|
||||
<attribute name="maven.pomderived" value="true"/>
|
||||
</attributes>
|
||||
</classpathentry>
|
||||
<classpathentry excluding="**" kind="src" output="target/classes" path="src/main/resources">
|
||||
<attributes>
|
||||
<attribute name="maven.pomderived" value="true"/>
|
||||
</attributes>
|
||||
</classpathentry>
|
||||
<classpathentry kind="src" output="target/test-classes" path="src/test/java">
|
||||
<attributes>
|
||||
<attribute name="optional" value="true"/>
|
||||
<attribute name="maven.pomderived" value="true"/>
|
||||
</attributes>
|
||||
</classpathentry>
|
||||
<classpathentry excluding="**" kind="src" output="target/test-classes" path="src/test/resources">
|
||||
<attributes>
|
||||
<attribute name="maven.pomderived" value="true"/>
|
||||
</attributes>
|
||||
</classpathentry>
|
||||
<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-1.8">
|
||||
<attributes>
|
||||
<attribute name="maven.pomderived" value="true"/>
|
||||
</attributes>
|
||||
</classpathentry>
|
||||
<classpathentry kind="con" path="org.eclipse.m2e.MAVEN2_CLASSPATH_CONTAINER">
|
||||
<attributes>
|
||||
<attribute name="maven.pomderived" value="true"/>
|
||||
</attributes>
|
||||
</classpathentry>
|
||||
<classpathentry kind="output" path="target/classes"/>
|
||||
</classpath>
|
|
@ -0,0 +1,23 @@
|
|||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<projectDescription>
|
||||
<name>crawler-plugin-framework</name>
|
||||
<comment></comment>
|
||||
<projects>
|
||||
</projects>
|
||||
<buildSpec>
|
||||
<buildCommand>
|
||||
<name>org.eclipse.jdt.core.javabuilder</name>
|
||||
<arguments>
|
||||
</arguments>
|
||||
</buildCommand>
|
||||
<buildCommand>
|
||||
<name>org.eclipse.m2e.core.maven2Builder</name>
|
||||
<arguments>
|
||||
</arguments>
|
||||
</buildCommand>
|
||||
</buildSpec>
|
||||
<natures>
|
||||
<nature>org.eclipse.jdt.core.javanature</nature>
|
||||
<nature>org.eclipse.m2e.core.maven2Nature</nature>
|
||||
</natures>
|
||||
</projectDescription>
|
|
@ -0,0 +1,6 @@
|
|||
eclipse.preferences.version=1
|
||||
encoding//src/main/java=UTF-8
|
||||
encoding//src/main/resources=UTF-8
|
||||
encoding//src/test/java=UTF-8
|
||||
encoding//src/test/resources=UTF-8
|
||||
encoding/<project>=UTF-8
|
|
@ -0,0 +1,5 @@
|
|||
eclipse.preferences.version=1
|
||||
org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.8
|
||||
org.eclipse.jdt.core.compiler.compliance=1.8
|
||||
org.eclipse.jdt.core.compiler.problem.forbiddenReference=warning
|
||||
org.eclipse.jdt.core.compiler.source=1.8
|
|
@ -0,0 +1,4 @@
|
|||
activeProfiles=
|
||||
eclipse.preferences.version=1
|
||||
resolveWorkspaceProjects=true
|
||||
version=1
|
|
@ -0,0 +1,26 @@
|
|||
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
<parent>
|
||||
<groupId>org.gcube.data-publishing.gCat-Feeder</groupId>
|
||||
<artifactId>gCat-Feeder-Suite</artifactId>
|
||||
<version>1.0.0-SNAPSHOT</version>
|
||||
</parent>
|
||||
<artifactId>crawler-plugin-framework</artifactId>
|
||||
<name>crawler-plugin-framework</name>
|
||||
<description>Framework for crawler implementations</description>
|
||||
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>org.projectlombok</groupId>
|
||||
<artifactId>lombok</artifactId>
|
||||
<version>1.14.8</version>
|
||||
</dependency>
|
||||
|
||||
|
||||
<dependency>
|
||||
<groupId>org.gcube.data-publishing.gCat-Feeder</groupId>
|
||||
<artifactId>commons</artifactId>
|
||||
<version>1.0.0-SNAPSHOT</version>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
</project>
|
|
@ -0,0 +1,9 @@
|
|||
package org.gcube.data.publishing.gCatfeeder.crawlers;
|
||||
|
||||
import org.gcube.data.publishing.gCatFeeder.model.CatalogueInstanceDescriptor;
|
||||
|
||||
public interface CatalogueRetriever {
|
||||
|
||||
public CatalogueInstanceDescriptor getInstance();
|
||||
|
||||
}
|
|
@ -0,0 +1,20 @@
|
|||
package org.gcube.data.publishing.gCatfeeder.crawlers;
|
||||
|
||||
import java.util.Set;
|
||||
|
||||
import org.gcube.data.publishing.gCatFeeder.model.FormatData;
|
||||
import org.gcube.data.publishing.gCatfeeder.crawlers.model.CustomData;
|
||||
import org.gcube.data.publishing.gCatfeeder.crawlers.model.PluginDescriptor;
|
||||
import org.gcube.data.publishing.gCatfeeder.crawlers.model.PublisherFormatData;
|
||||
import org.gcube.data.publishing.gCatfeeder.crawlers.model.faults.CatalogueNotSupportedException;
|
||||
|
||||
public interface CrawlerPlugin<E extends CustomData>{
|
||||
|
||||
public PluginDescriptor getDescriptor();
|
||||
|
||||
public CatalogueRetriever getRetrieverByCatalogueType(FormatData catalogueType) throws CatalogueNotSupportedException;
|
||||
|
||||
public Set<DataTransformer<? extends PublisherFormatData,E>> getImplementedTransformers();
|
||||
|
||||
public DataCollector<E> getCollector();
|
||||
}
|
|
@ -0,0 +1,15 @@
|
|||
package org.gcube.data.publishing.gCatfeeder.crawlers;
|
||||
|
||||
import java.util.Set;
|
||||
|
||||
import org.gcube.data.publishing.gCatfeeder.crawlers.model.CustomData;
|
||||
import org.gcube.data.publishing.gCatfeeder.crawlers.model.faults.CollectorFault;
|
||||
|
||||
public interface DataCollector<T extends CustomData> {
|
||||
|
||||
|
||||
public Set<T> collect() throws CollectorFault;
|
||||
|
||||
|
||||
|
||||
}
|
|
@ -0,0 +1,15 @@
|
|||
package org.gcube.data.publishing.gCatfeeder.crawlers;
|
||||
|
||||
import java.util.Collection;
|
||||
import java.util.Set;
|
||||
|
||||
import org.gcube.data.publishing.gCatFeeder.model.FormatData;
|
||||
import org.gcube.data.publishing.gCatfeeder.crawlers.model.CustomData;
|
||||
import org.gcube.data.publishing.gCatfeeder.crawlers.model.PublisherFormatData;
|
||||
|
||||
public interface DataTransformer<T extends PublisherFormatData,E extends CustomData> {
|
||||
|
||||
public FormatData getTargetFormat();
|
||||
|
||||
public Set<T> transform(Collection<E> collectedData);
|
||||
}
|
|
@ -0,0 +1,5 @@
|
|||
package org.gcube.data.publishing.gCatfeeder.crawlers.model;
|
||||
|
||||
public interface CustomData {
|
||||
|
||||
}
|
|
@ -0,0 +1,22 @@
|
|||
package org.gcube.data.publishing.gCatfeeder.crawlers.model;
|
||||
|
||||
import lombok.Getter;
|
||||
import lombok.NonNull;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.Setter;
|
||||
|
||||
@Getter
|
||||
@Setter
|
||||
@RequiredArgsConstructor
|
||||
public class PluginDescriptor {
|
||||
|
||||
// Plugin info
|
||||
|
||||
@NonNull
|
||||
private String name;
|
||||
|
||||
|
||||
// Run Interval
|
||||
|
||||
|
||||
}
|
|
@ -0,0 +1,12 @@
|
|||
package org.gcube.data.publishing.gCatfeeder.crawlers.model;
|
||||
|
||||
import org.gcube.data.publishing.gCatFeeder.model.FormatData;
|
||||
|
||||
public interface PublisherFormatData {
|
||||
|
||||
|
||||
public FormatData getFormat();
|
||||
|
||||
public String serialize();
|
||||
|
||||
}
|
|
@ -0,0 +1,29 @@
|
|||
package org.gcube.data.publishing.gCatfeeder.crawlers.model.faults;
|
||||
|
||||
public class CatalogueNotFound extends CrawlerException {
|
||||
|
||||
public CatalogueNotFound() {
|
||||
// TODO Auto-generated constructor stub
|
||||
}
|
||||
|
||||
public CatalogueNotFound(String message) {
|
||||
super(message);
|
||||
// TODO Auto-generated constructor stub
|
||||
}
|
||||
|
||||
public CatalogueNotFound(Throwable cause) {
|
||||
super(cause);
|
||||
// TODO Auto-generated constructor stub
|
||||
}
|
||||
|
||||
public CatalogueNotFound(String message, Throwable cause) {
|
||||
super(message, cause);
|
||||
// TODO Auto-generated constructor stub
|
||||
}
|
||||
|
||||
public CatalogueNotFound(String message, Throwable cause, boolean enableSuppression, boolean writableStackTrace) {
|
||||
super(message, cause, enableSuppression, writableStackTrace);
|
||||
// TODO Auto-generated constructor stub
|
||||
}
|
||||
|
||||
}
|
|
@ -0,0 +1,30 @@
|
|||
package org.gcube.data.publishing.gCatfeeder.crawlers.model.faults;
|
||||
|
||||
public class CatalogueNotSupportedException extends CrawlerException {
|
||||
|
||||
public CatalogueNotSupportedException() {
|
||||
// TODO Auto-generated constructor stub
|
||||
}
|
||||
|
||||
public CatalogueNotSupportedException(String message) {
|
||||
super(message);
|
||||
// TODO Auto-generated constructor stub
|
||||
}
|
||||
|
||||
public CatalogueNotSupportedException(Throwable cause) {
|
||||
super(cause);
|
||||
// TODO Auto-generated constructor stub
|
||||
}
|
||||
|
||||
public CatalogueNotSupportedException(String message, Throwable cause) {
|
||||
super(message, cause);
|
||||
// TODO Auto-generated constructor stub
|
||||
}
|
||||
|
||||
public CatalogueNotSupportedException(String message, Throwable cause, boolean enableSuppression,
|
||||
boolean writableStackTrace) {
|
||||
super(message, cause, enableSuppression, writableStackTrace);
|
||||
// TODO Auto-generated constructor stub
|
||||
}
|
||||
|
||||
}
|
|
@ -0,0 +1,29 @@
|
|||
package org.gcube.data.publishing.gCatfeeder.crawlers.model.faults;
|
||||
|
||||
public class CollectorFault extends CrawlerException {
|
||||
|
||||
public CollectorFault() {
|
||||
// TODO Auto-generated constructor stub
|
||||
}
|
||||
|
||||
public CollectorFault(String message) {
|
||||
super(message);
|
||||
// TODO Auto-generated constructor stub
|
||||
}
|
||||
|
||||
public CollectorFault(Throwable cause) {
|
||||
super(cause);
|
||||
// TODO Auto-generated constructor stub
|
||||
}
|
||||
|
||||
public CollectorFault(String message, Throwable cause) {
|
||||
super(message, cause);
|
||||
// TODO Auto-generated constructor stub
|
||||
}
|
||||
|
||||
public CollectorFault(String message, Throwable cause, boolean enableSuppression, boolean writableStackTrace) {
|
||||
super(message, cause, enableSuppression, writableStackTrace);
|
||||
// TODO Auto-generated constructor stub
|
||||
}
|
||||
|
||||
}
|
|
@ -0,0 +1,29 @@
|
|||
package org.gcube.data.publishing.gCatfeeder.crawlers.model.faults;
|
||||
|
||||
public class CrawlerException extends Exception {
|
||||
|
||||
public CrawlerException() {
|
||||
// TODO Auto-generated constructor stub
|
||||
}
|
||||
|
||||
public CrawlerException(String message) {
|
||||
super(message);
|
||||
// TODO Auto-generated constructor stub
|
||||
}
|
||||
|
||||
public CrawlerException(Throwable cause) {
|
||||
super(cause);
|
||||
// TODO Auto-generated constructor stub
|
||||
}
|
||||
|
||||
public CrawlerException(String message, Throwable cause) {
|
||||
super(message, cause);
|
||||
// TODO Auto-generated constructor stub
|
||||
}
|
||||
|
||||
public CrawlerException(String message, Throwable cause, boolean enableSuppression, boolean writableStackTrace) {
|
||||
super(message, cause, enableSuppression, writableStackTrace);
|
||||
// TODO Auto-generated constructor stub
|
||||
}
|
||||
|
||||
}
|
|
@ -0,0 +1,30 @@
|
|||
package org.gcube.data.publishing.gCatfeeder.crawlers.model.faults;
|
||||
|
||||
public class InvalidCustomData extends CrawlerException {
|
||||
|
||||
public InvalidCustomData() {
|
||||
// TODO Auto-generated constructor stub
|
||||
}
|
||||
|
||||
public InvalidCustomData(String message) {
|
||||
super(message);
|
||||
// TODO Auto-generated constructor stub
|
||||
}
|
||||
|
||||
public InvalidCustomData(Throwable cause) {
|
||||
super(cause);
|
||||
// TODO Auto-generated constructor stub
|
||||
}
|
||||
|
||||
public InvalidCustomData(String message, Throwable cause) {
|
||||
super(message, cause);
|
||||
// TODO Auto-generated constructor stub
|
||||
}
|
||||
|
||||
public InvalidCustomData(String message, Throwable cause, boolean enableSuppression,
|
||||
boolean writableStackTrace) {
|
||||
super(message, cause, enableSuppression, writableStackTrace);
|
||||
// TODO Auto-generated constructor stub
|
||||
}
|
||||
|
||||
}
|
|
@ -0,0 +1,29 @@
|
|||
package org.gcube.data.publishing.gCatfeeder.crawlers.model.faults;
|
||||
|
||||
public class TransformException extends CrawlerException {
|
||||
|
||||
public TransformException() {
|
||||
// TODO Auto-generated constructor stub
|
||||
}
|
||||
|
||||
public TransformException(String message) {
|
||||
super(message);
|
||||
// TODO Auto-generated constructor stub
|
||||
}
|
||||
|
||||
public TransformException(Throwable cause) {
|
||||
super(cause);
|
||||
// TODO Auto-generated constructor stub
|
||||
}
|
||||
|
||||
public TransformException(String message, Throwable cause) {
|
||||
super(message, cause);
|
||||
// TODO Auto-generated constructor stub
|
||||
}
|
||||
|
||||
public TransformException(String message, Throwable cause, boolean enableSuppression, boolean writableStackTrace) {
|
||||
super(message, cause, enableSuppression, writableStackTrace);
|
||||
// TODO Auto-generated constructor stub
|
||||
}
|
||||
|
||||
}
|
|
@ -0,0 +1,11 @@
|
|||
package org.gcube.data.publishing.gCatFeeder.crawlers.model;
|
||||
|
||||
import org.junit.Test;
|
||||
|
||||
public class ModelTests {
|
||||
|
||||
@Test
|
||||
public void testConversion() {
|
||||
|
||||
}
|
||||
}
|
|
@ -0,0 +1,5 @@
|
|||
Manifest-Version: 1.0
|
||||
Built-By: fabio
|
||||
Build-Jdk: 1.8.0_201
|
||||
Created-By: Maven Integration for Eclipse
|
||||
|
|
@ -0,0 +1,7 @@
|
|||
#Generated by Maven Integration for Eclipse
|
||||
#Wed Mar 06 11:12:02 CET 2019
|
||||
version=1.0.0-SNAPSHOT
|
||||
groupId=org.gcube.data-publishing.gCat-Feeder
|
||||
m2e.projectName=crawler-plugin-framework
|
||||
m2e.projectLocation=/home/fabio/workspaces/DEV/gCat-Feeder-Suite/crawler-plugin-framework
|
||||
artifactId=crawler-plugin-framework
|
|
@ -0,0 +1,26 @@
|
|||
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
<parent>
|
||||
<groupId>org.gcube.data-publishing.gCat-Feeder</groupId>
|
||||
<artifactId>gCat-Feeder-Suite</artifactId>
|
||||
<version>1.0.0-SNAPSHOT</version>
|
||||
</parent>
|
||||
<artifactId>crawler-plugin-framework</artifactId>
|
||||
<name>crawler-plugin-framework</name>
|
||||
<description>Framework for crawler implementations</description>
|
||||
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>org.projectlombok</groupId>
|
||||
<artifactId>lombok</artifactId>
|
||||
<version>1.14.8</version>
|
||||
</dependency>
|
||||
|
||||
|
||||
<dependency>
|
||||
<groupId>org.gcube.data-publishing.gCat-Feeder</groupId>
|
||||
<artifactId>commons</artifactId>
|
||||
<version>1.0.0-SNAPSHOT</version>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
</project>
|
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
|
@ -0,0 +1,36 @@
|
|||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<classpath>
|
||||
<classpathentry kind="src" output="target/classes" path="src/main/java">
|
||||
<attributes>
|
||||
<attribute name="optional" value="true"/>
|
||||
<attribute name="maven.pomderived" value="true"/>
|
||||
</attributes>
|
||||
</classpathentry>
|
||||
<classpathentry excluding="**" kind="src" output="target/classes" path="src/main/resources">
|
||||
<attributes>
|
||||
<attribute name="maven.pomderived" value="true"/>
|
||||
</attributes>
|
||||
</classpathentry>
|
||||
<classpathentry kind="src" output="target/test-classes" path="src/test/java">
|
||||
<attributes>
|
||||
<attribute name="optional" value="true"/>
|
||||
<attribute name="maven.pomderived" value="true"/>
|
||||
</attributes>
|
||||
</classpathentry>
|
||||
<classpathentry excluding="**" kind="src" output="target/test-classes" path="src/test/resources">
|
||||
<attributes>
|
||||
<attribute name="maven.pomderived" value="true"/>
|
||||
</attributes>
|
||||
</classpathentry>
|
||||
<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-1.8">
|
||||
<attributes>
|
||||
<attribute name="maven.pomderived" value="true"/>
|
||||
</attributes>
|
||||
</classpathentry>
|
||||
<classpathentry kind="con" path="org.eclipse.m2e.MAVEN2_CLASSPATH_CONTAINER">
|
||||
<attributes>
|
||||
<attribute name="maven.pomderived" value="true"/>
|
||||
</attributes>
|
||||
</classpathentry>
|
||||
<classpathentry kind="output" path="target/classes"/>
|
||||
</classpath>
|
|
@ -0,0 +1,23 @@
|
|||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<projectDescription>
|
||||
<name>gCat-Feeder</name>
|
||||
<comment></comment>
|
||||
<projects>
|
||||
</projects>
|
||||
<buildSpec>
|
||||
<buildCommand>
|
||||
<name>org.eclipse.jdt.core.javabuilder</name>
|
||||
<arguments>
|
||||
</arguments>
|
||||
</buildCommand>
|
||||
<buildCommand>
|
||||
<name>org.eclipse.m2e.core.maven2Builder</name>
|
||||
<arguments>
|
||||
</arguments>
|
||||
</buildCommand>
|
||||
</buildSpec>
|
||||
<natures>
|
||||
<nature>org.eclipse.jdt.core.javanature</nature>
|
||||
<nature>org.eclipse.m2e.core.maven2Nature</nature>
|
||||
</natures>
|
||||
</projectDescription>
|
|
@ -0,0 +1,6 @@
|
|||
eclipse.preferences.version=1
|
||||
encoding//src/main/java=UTF-8
|
||||
encoding//src/main/resources=UTF-8
|
||||
encoding//src/test/java=UTF-8
|
||||
encoding//src/test/resources=UTF-8
|
||||
encoding/<project>=UTF-8
|
|
@ -0,0 +1,5 @@
|
|||
eclipse.preferences.version=1
|
||||
org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.8
|
||||
org.eclipse.jdt.core.compiler.compliance=1.8
|
||||
org.eclipse.jdt.core.compiler.problem.forbiddenReference=warning
|
||||
org.eclipse.jdt.core.compiler.source=1.8
|
|
@ -0,0 +1,4 @@
|
|||
activeProfiles=
|
||||
eclipse.preferences.version=1
|
||||
resolveWorkspaceProjects=true
|
||||
version=1
|
|
@ -0,0 +1,33 @@
|
|||
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
<parent>
|
||||
<groupId>org.gcube.data-publishing.gCat-Feeder</groupId>
|
||||
<artifactId>gCat-Feeder-Suite</artifactId>
|
||||
<version>1.0.0-SNAPSHOT</version>
|
||||
</parent>
|
||||
<artifactId>gCat-Feeder</artifactId>
|
||||
<name>gCat-Feeder</name>
|
||||
<description>Service implementation</description>
|
||||
|
||||
<dependencyManagement>
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>org.gcube.distribution</groupId>
|
||||
<artifactId>maven-smartgears-bom</artifactId>
|
||||
<version>LATEST</version>
|
||||
<type>pom</type>
|
||||
<scope>import</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId> org.glassfish.jersey </groupId>
|
||||
<artifactId>jersey-bom</artifactId>
|
||||
<version>2.25.1</version>
|
||||
<type>pom</type>
|
||||
<scope>import</scope>
|
||||
</dependency>
|
||||
|
||||
</dependencies>
|
||||
</dependencyManagement>
|
||||
|
||||
|
||||
</project>
|
|
@ -0,0 +1,5 @@
|
|||
Manifest-Version: 1.0
|
||||
Built-By: fabio
|
||||
Build-Jdk: 1.8.0_201
|
||||
Created-By: Maven Integration for Eclipse
|
||||
|
|
@ -0,0 +1,7 @@
|
|||
#Generated by Maven Integration for Eclipse
|
||||
#Thu Feb 28 15:58:44 CET 2019
|
||||
version=1.0.0-SNAPSHOT
|
||||
groupId=org.gcube.data-publishing.gCat-Feeder
|
||||
m2e.projectName=gCat-Feeder
|
||||
m2e.projectLocation=/home/fabio/workspaces/DEV/gCat-Feeder-Suite/gCat-Feeder
|
||||
artifactId=gCat-Feeder
|
|
@ -0,0 +1,33 @@
|
|||
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
<parent>
|
||||
<groupId>org.gcube.data-publishing.gCat-Feeder</groupId>
|
||||
<artifactId>gCat-Feeder-Suite</artifactId>
|
||||
<version>1.0.0-SNAPSHOT</version>
|
||||
</parent>
|
||||
<artifactId>gCat-Feeder</artifactId>
|
||||
<name>gCat-Feeder</name>
|
||||
<description>Service implementation</description>
|
||||
|
||||
<dependencyManagement>
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>org.gcube.distribution</groupId>
|
||||
<artifactId>maven-smartgears-bom</artifactId>
|
||||
<version>LATEST</version>
|
||||
<type>pom</type>
|
||||
<scope>import</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId> org.glassfish.jersey </groupId>
|
||||
<artifactId>jersey-bom</artifactId>
|
||||
<version>2.25.1</version>
|
||||
<type>pom</type>
|
||||
<scope>import</scope>
|
||||
</dependency>
|
||||
|
||||
</dependencies>
|
||||
</dependencyManagement>
|
||||
|
||||
|
||||
</project>
|
|
@ -0,0 +1,55 @@
|
|||
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
<parent>
|
||||
<groupId>org.gcube.tools</groupId>
|
||||
<artifactId>maven-parent</artifactId>
|
||||
<version>LATEST</version>
|
||||
</parent>
|
||||
<groupId>org.gcube.data-publishing.gCat-Feeder</groupId>
|
||||
<artifactId>gCat-Feeder-Suite</artifactId>
|
||||
<version>1.0.0-SNAPSHOT</version>
|
||||
<packaging>pom</packaging>
|
||||
<name>gCat-Feeder Suite</name>
|
||||
<description>gCat-Feeder Suite of components : service, plugin framework, plugins</description>
|
||||
|
||||
|
||||
<modules>
|
||||
<module>gCat-Feeder</module>
|
||||
<module>crawler-plugin-framework</module>
|
||||
<module>DataMinerAlgorithmsCrawler</module>
|
||||
<module>commons</module>
|
||||
</modules>
|
||||
|
||||
<dependencyManagement>
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>org.gcube.distribution</groupId>
|
||||
<artifactId>gcube-bom</artifactId>
|
||||
<version>1.3.0-SNAPSHOT</version>
|
||||
<type>pom</type>
|
||||
<scope>import</scope>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
</dependencyManagement>
|
||||
|
||||
|
||||
<dependencies>
|
||||
|
||||
<dependency>
|
||||
<groupId>ch.qos.logback</groupId>
|
||||
<artifactId>logback-classic</artifactId>
|
||||
<version>1.0.13</version>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>junit</groupId>
|
||||
<artifactId>junit</artifactId>
|
||||
<version>4.12</version>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
|
||||
</dependencies>
|
||||
|
||||
</project>
|
Loading…
Reference in New Issue