diff --git a/.project b/.project new file mode 100644 index 0000000..3464a36 --- /dev/null +++ b/.project @@ -0,0 +1,17 @@ + + + gCat-Feeder-Suite + + + + + + org.eclipse.m2e.core.maven2Builder + + + + + + org.eclipse.m2e.core.maven2Nature + + diff --git a/.settings/org.eclipse.core.resources.prefs b/.settings/org.eclipse.core.resources.prefs new file mode 100644 index 0000000..99f26c0 --- /dev/null +++ b/.settings/org.eclipse.core.resources.prefs @@ -0,0 +1,2 @@ +eclipse.preferences.version=1 +encoding/=UTF-8 diff --git a/.settings/org.eclipse.m2e.core.prefs b/.settings/org.eclipse.m2e.core.prefs new file mode 100644 index 0000000..f897a7f --- /dev/null +++ b/.settings/org.eclipse.m2e.core.prefs @@ -0,0 +1,4 @@ +activeProfiles= +eclipse.preferences.version=1 +resolveWorkspaceProjects=true +version=1 diff --git a/DataMinerAlgorithmsCrawler/.classpath b/DataMinerAlgorithmsCrawler/.classpath new file mode 100644 index 0000000..fae1a2b --- /dev/null +++ b/DataMinerAlgorithmsCrawler/.classpath @@ -0,0 +1,36 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/DataMinerAlgorithmsCrawler/.project b/DataMinerAlgorithmsCrawler/.project new file mode 100644 index 0000000..82042c8 --- /dev/null +++ b/DataMinerAlgorithmsCrawler/.project @@ -0,0 +1,23 @@ + + + DataMinerAlgorithmsCrawler + + + + + + org.eclipse.jdt.core.javabuilder + + + + + org.eclipse.m2e.core.maven2Builder + + + + + + org.eclipse.jdt.core.javanature + org.eclipse.m2e.core.maven2Nature + + diff --git a/DataMinerAlgorithmsCrawler/.settings/org.eclipse.core.resources.prefs b/DataMinerAlgorithmsCrawler/.settings/org.eclipse.core.resources.prefs new file mode 100644 index 0000000..29abf99 --- /dev/null +++ b/DataMinerAlgorithmsCrawler/.settings/org.eclipse.core.resources.prefs @@ -0,0 +1,6 @@ +eclipse.preferences.version=1 +encoding//src/main/java=UTF-8 +encoding//src/main/resources=UTF-8 +encoding//src/test/java=UTF-8 +encoding//src/test/resources=UTF-8 +encoding/=UTF-8 diff --git a/DataMinerAlgorithmsCrawler/.settings/org.eclipse.jdt.core.prefs b/DataMinerAlgorithmsCrawler/.settings/org.eclipse.jdt.core.prefs new file mode 100644 index 0000000..714351a --- /dev/null +++ b/DataMinerAlgorithmsCrawler/.settings/org.eclipse.jdt.core.prefs @@ -0,0 +1,5 @@ +eclipse.preferences.version=1 +org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.8 +org.eclipse.jdt.core.compiler.compliance=1.8 +org.eclipse.jdt.core.compiler.problem.forbiddenReference=warning +org.eclipse.jdt.core.compiler.source=1.8 diff --git a/DataMinerAlgorithmsCrawler/.settings/org.eclipse.m2e.core.prefs b/DataMinerAlgorithmsCrawler/.settings/org.eclipse.m2e.core.prefs new file mode 100644 index 0000000..f897a7f --- /dev/null +++ b/DataMinerAlgorithmsCrawler/.settings/org.eclipse.m2e.core.prefs @@ -0,0 +1,4 @@ +activeProfiles= +eclipse.preferences.version=1 +resolveWorkspaceProjects=true +version=1 diff --git a/DataMinerAlgorithmsCrawler/pom.xml b/DataMinerAlgorithmsCrawler/pom.xml new file mode 100644 index 0000000..10e0cfb --- /dev/null +++ b/DataMinerAlgorithmsCrawler/pom.xml @@ -0,0 +1,28 @@ + + 4.0.0 + + org.gcube.data-publishing.gCat-Feeder + gCat-Feeder-Suite + 1.0.0-SNAPSHOT + + DataMinerAlgorithmsCrawler + DataMinerAlgorithmsCrawler + Plugin for gCat-Feeder for DataMiner Algorithms publishing + + + + + + org.gcube.data-publishing.gCat-Feeder + crawler-plugin-framework + [1.0.0-SNAPSHOT,2.0.0-SNAPSHOT) + + + + org.gcube.data.analysis + data-miner-manager-cl + [1.6.0-SNAPSHOT,2.0.0-SNAPSHOT) + + + \ No newline at end of file diff --git a/DataMinerAlgorithmsCrawler/src/main/java/org/gcube/data/publishing/gCatFeeder/crawlers/dm/CKANRetriever.java b/DataMinerAlgorithmsCrawler/src/main/java/org/gcube/data/publishing/gCatFeeder/crawlers/dm/CKANRetriever.java new file mode 100644 index 0000000..5d778ef --- /dev/null +++ b/DataMinerAlgorithmsCrawler/src/main/java/org/gcube/data/publishing/gCatFeeder/crawlers/dm/CKANRetriever.java @@ -0,0 +1,26 @@ +package org.gcube.data.publishing.gCatFeeder.crawlers.dm; + +import org.gcube.common.resources.gcore.GCoreEndpoint; +import org.gcube.data.publishing.gCatFeeder.model.CatalogueInstanceDescriptor; +import org.gcube.data.publishing.gCatFeeder.utils.ISUtils; +import org.gcube.data.publishing.gCatfeeder.crawlers.CatalogueRetriever; + +public class CKANRetriever implements CatalogueRetriever { + + private static CKANRetriever instance=null; + + static synchronized CKANRetriever get() { + if(instance==null) instance =new CKANRetriever(); + return instance; + } + + + + @Override + public CatalogueInstanceDescriptor getInstance() { + throw new RuntimeException("Implement this"); +// GCoreEndpoint ep=ISUtils.queryForGCoreEndpoint("NO", "NO"); +// return null; + } + +} diff --git a/DataMinerAlgorithmsCrawler/src/main/java/org/gcube/data/publishing/gCatFeeder/crawlers/dm/CKANTransformer.java b/DataMinerAlgorithmsCrawler/src/main/java/org/gcube/data/publishing/gCatFeeder/crawlers/dm/CKANTransformer.java new file mode 100644 index 0000000..5e71103 --- /dev/null +++ b/DataMinerAlgorithmsCrawler/src/main/java/org/gcube/data/publishing/gCatFeeder/crawlers/dm/CKANTransformer.java @@ -0,0 +1,29 @@ +package org.gcube.data.publishing.gCatFeeder.crawlers.dm; + +import java.util.Collection; +import java.util.HashSet; +import java.util.Set; + +import org.gcube.data.publishing.gCatFeeder.crawlers.dm.model.CkanModel; +import org.gcube.data.publishing.gCatFeeder.crawlers.dm.model.InternalAlgorithmDescriptor; +import org.gcube.data.publishing.gCatFeeder.model.FormatData; +import org.gcube.data.publishing.gCatfeeder.crawlers.DataTransformer; + +public class CKANTransformer implements DataTransformer{ + + + @Override + public FormatData getTargetFormat() { + return FormatData.CKAN; + } + + @Override + public Set transform(Collection collectedData) { + HashSet toReturn=new HashSet<>(); + for(InternalAlgorithmDescriptor desc:collectedData) { + toReturn.add(desc.asCKANModel()); + } + return toReturn; + } + +} diff --git a/DataMinerAlgorithmsCrawler/src/main/java/org/gcube/data/publishing/gCatFeeder/crawlers/dm/DMAlgorithmsInfoCollector.java b/DataMinerAlgorithmsCrawler/src/main/java/org/gcube/data/publishing/gCatFeeder/crawlers/dm/DMAlgorithmsInfoCollector.java new file mode 100644 index 0000000..5806af9 --- /dev/null +++ b/DataMinerAlgorithmsCrawler/src/main/java/org/gcube/data/publishing/gCatFeeder/crawlers/dm/DMAlgorithmsInfoCollector.java @@ -0,0 +1,68 @@ +package org.gcube.data.publishing.gCatFeeder.crawlers.dm; + +import java.util.HashSet; +import java.util.Set; + +import org.gcube.data.analysis.dataminermanagercl.server.DataMinerService; +import org.gcube.data.analysis.dataminermanagercl.server.dmservice.SClient; +import org.gcube.data.analysis.dataminermanagercl.shared.parameters.Parameter; +import org.gcube.data.analysis.dataminermanagercl.shared.process.Operator; +import org.gcube.data.analysis.dataminermanagercl.shared.process.OperatorCategory; +import org.gcube.data.analysis.dataminermanagercl.shared.process.OperatorsClassification; +import org.gcube.data.publishing.gCatFeeder.crawlers.dm.model.InternalAlgorithmDescriptor; +import org.gcube.data.publishing.gCatfeeder.crawlers.DataCollector; +import org.gcube.data.publishing.gCatfeeder.crawlers.model.faults.CollectorFault; + + +public class DMAlgorithmsInfoCollector implements DataCollector { + + + @Override + public Set collect() throws CollectorFault { + try { + SClient client=new DataMinerService().getClient(); + for(OperatorsClassification opClass: client.getOperatorsClassifications()) { + // Load info + String opClassName =opClass.getName(); + + for(Operator op : opClass.getOperators()) { + // OperatorCategory Info + OperatorCategory cat = op.getCategory(); + String categoryBriefDescription = cat.getBriefDescription(); + String categoryDescription= cat.getDescription(); + String categoryID=cat.getId(); + String categoryName=cat.getName(); + + // Operator info + + String opBriefDescription=op.getBriefDescription(); + String opDescription=op.getDescription(); + String opID=op.getId(); + String operatorName=op.getName(); + + + // Parameters info + for(Parameter param:op.getOperatorParameters()) { + String paramDescription=param.getDescription(); + String paramName=param.getName(); + String paramType=param.getTypology().toString(); + String paramValue=param.getValue(); + } + } + + + // Create bean + } + + + + HashSet toReturn=new HashSet(); + + + return toReturn; + }catch(Exception e) { + throw new CollectorFault("Unable to retrieve information",e); + } + } + +} diff --git a/DataMinerAlgorithmsCrawler/src/main/java/org/gcube/data/publishing/gCatFeeder/crawlers/dm/DataMinerCrawler.java b/DataMinerAlgorithmsCrawler/src/main/java/org/gcube/data/publishing/gCatFeeder/crawlers/dm/DataMinerCrawler.java new file mode 100644 index 0000000..97fe0b5 --- /dev/null +++ b/DataMinerAlgorithmsCrawler/src/main/java/org/gcube/data/publishing/gCatFeeder/crawlers/dm/DataMinerCrawler.java @@ -0,0 +1,43 @@ +package org.gcube.data.publishing.gCatFeeder.crawlers.dm; + +import java.util.Collections; +import java.util.Set; + +import org.gcube.data.publishing.gCatFeeder.crawlers.dm.model.CkanModel; +import org.gcube.data.publishing.gCatFeeder.crawlers.dm.model.InternalAlgorithmDescriptor; +import org.gcube.data.publishing.gCatFeeder.model.FormatData; +import org.gcube.data.publishing.gCatfeeder.crawlers.CatalogueRetriever; +import org.gcube.data.publishing.gCatfeeder.crawlers.CrawlerPlugin; +import org.gcube.data.publishing.gCatfeeder.crawlers.DataCollector; +import org.gcube.data.publishing.gCatfeeder.crawlers.DataTransformer; +import org.gcube.data.publishing.gCatfeeder.crawlers.model.CustomData; +import org.gcube.data.publishing.gCatfeeder.crawlers.model.PluginDescriptor; +import org.gcube.data.publishing.gCatfeeder.crawlers.model.faults.CatalogueNotSupportedException; + +public class DataMinerCrawler implements CrawlerPlugin { + + @Override + public PluginDescriptor getDescriptor() { + // TODO Auto-generated method stub + return null; + } + + @Override + public CatalogueRetriever getRetrieverByCatalogueType(FormatData catalogueType) throws CatalogueNotSupportedException { + switch(catalogueType) { + case CKAN : return CKANRetriever.get(); + default : throw new CatalogueNotSupportedException("No support for "+catalogueType); + } + } + + @Override + public Set> getImplementedTransformers() { + return Collections.singleton((DataTransformer) new CKANTransformer()); + } + + @Override + public DataCollector getCollector() { + return new DMAlgorithmsInfoCollector(); + } + +} diff --git a/DataMinerAlgorithmsCrawler/src/main/java/org/gcube/data/publishing/gCatFeeder/crawlers/dm/model/CkanModel.java b/DataMinerAlgorithmsCrawler/src/main/java/org/gcube/data/publishing/gCatFeeder/crawlers/dm/model/CkanModel.java new file mode 100644 index 0000000..dfccad9 --- /dev/null +++ b/DataMinerAlgorithmsCrawler/src/main/java/org/gcube/data/publishing/gCatFeeder/crawlers/dm/model/CkanModel.java @@ -0,0 +1,39 @@ +package org.gcube.data.publishing.gCatFeeder.crawlers.dm.model; + +import org.gcube.data.publishing.gCatFeeder.model.FormatData; +import org.gcube.data.publishing.gCatfeeder.crawlers.model.PublisherFormatData; + +public class CkanModel implements PublisherFormatData { + + + /** + * (Common) Title + * (Common) Description + * (Common) Tags: free list of keywords + * (Common) License + * (Common) Visibility: either public or private + * (Common) Version + * (Common) Author: the creator of metadata. Only one occurrence is supported; + * (Common) Maintainer: + * (Method specific) Creator: the author of the method (with email and ORCID). Repeatable field; + * (Method specific) Creation date: when the method has been released; + * (Method specific) Input: Repeatable field; + * (Method specific) Output: Repeatable field; + * (Method specific) RelatedPaper: a reference to an associated paper; + * (Method specific) Restrictions On Use: an optional text + * (Method specific) Attribution requirements: the text to use to acknowledge method usage; + */ + + + @Override + public FormatData getFormat() { + return FormatData.CKAN; + } + + @Override + public String serialize() { + // TODO Auto-generated method stub + return null; + } + +} diff --git a/DataMinerAlgorithmsCrawler/src/main/java/org/gcube/data/publishing/gCatFeeder/crawlers/dm/model/InternalAlgorithmDescriptor.java b/DataMinerAlgorithmsCrawler/src/main/java/org/gcube/data/publishing/gCatFeeder/crawlers/dm/model/InternalAlgorithmDescriptor.java new file mode 100644 index 0000000..0cdb224 --- /dev/null +++ b/DataMinerAlgorithmsCrawler/src/main/java/org/gcube/data/publishing/gCatFeeder/crawlers/dm/model/InternalAlgorithmDescriptor.java @@ -0,0 +1,20 @@ +package org.gcube.data.publishing.gCatFeeder.crawlers.dm.model; + +import java.util.List; + +import org.gcube.data.publishing.gCatfeeder.crawlers.model.CustomData; + +public class InternalAlgorithmDescriptor implements CustomData { + + private String name; + private String Description; + private List tags; + private String id; + + + + public CkanModel asCKANModel() { + throw new RuntimeException("Implement This"); + } + +} diff --git a/DataMinerAlgorithmsCrawler/target/classes/META-INF/MANIFEST.MF b/DataMinerAlgorithmsCrawler/target/classes/META-INF/MANIFEST.MF new file mode 100644 index 0000000..e2601c5 --- /dev/null +++ b/DataMinerAlgorithmsCrawler/target/classes/META-INF/MANIFEST.MF @@ -0,0 +1,5 @@ +Manifest-Version: 1.0 +Built-By: fabio +Build-Jdk: 1.8.0_201 +Created-By: Maven Integration for Eclipse + diff --git a/DataMinerAlgorithmsCrawler/target/classes/META-INF/maven/org.gcube.data-publishing.gCat-Feeder/DataMinerAlgorithmsCrawler/pom.properties b/DataMinerAlgorithmsCrawler/target/classes/META-INF/maven/org.gcube.data-publishing.gCat-Feeder/DataMinerAlgorithmsCrawler/pom.properties new file mode 100644 index 0000000..c3b3e6f --- /dev/null +++ b/DataMinerAlgorithmsCrawler/target/classes/META-INF/maven/org.gcube.data-publishing.gCat-Feeder/DataMinerAlgorithmsCrawler/pom.properties @@ -0,0 +1,7 @@ +#Generated by Maven Integration for Eclipse +#Fri Mar 01 18:17:42 CET 2019 +version=1.0.0-SNAPSHOT +groupId=org.gcube.data-publishing.gCat-Feeder +m2e.projectName=DataMinerAlgorithmsCrawler +m2e.projectLocation=/home/fabio/workspaces/DEV/gCat-Feeder-Suite/DataMinerAlgorithmsCrawler +artifactId=DataMinerAlgorithmsCrawler diff --git a/DataMinerAlgorithmsCrawler/target/classes/META-INF/maven/org.gcube.data-publishing.gCat-Feeder/DataMinerAlgorithmsCrawler/pom.xml b/DataMinerAlgorithmsCrawler/target/classes/META-INF/maven/org.gcube.data-publishing.gCat-Feeder/DataMinerAlgorithmsCrawler/pom.xml new file mode 100644 index 0000000..10e0cfb --- /dev/null +++ b/DataMinerAlgorithmsCrawler/target/classes/META-INF/maven/org.gcube.data-publishing.gCat-Feeder/DataMinerAlgorithmsCrawler/pom.xml @@ -0,0 +1,28 @@ + + 4.0.0 + + org.gcube.data-publishing.gCat-Feeder + gCat-Feeder-Suite + 1.0.0-SNAPSHOT + + DataMinerAlgorithmsCrawler + DataMinerAlgorithmsCrawler + Plugin for gCat-Feeder for DataMiner Algorithms publishing + + + + + + org.gcube.data-publishing.gCat-Feeder + crawler-plugin-framework + [1.0.0-SNAPSHOT,2.0.0-SNAPSHOT) + + + + org.gcube.data.analysis + data-miner-manager-cl + [1.6.0-SNAPSHOT,2.0.0-SNAPSHOT) + + + \ No newline at end of file diff --git a/DataMinerAlgorithmsCrawler/target/classes/org/gcube/data/publishing/gCatFeeder/crawlers/dm/CKANRetriever.class b/DataMinerAlgorithmsCrawler/target/classes/org/gcube/data/publishing/gCatFeeder/crawlers/dm/CKANRetriever.class new file mode 100644 index 0000000..e4a3546 Binary files /dev/null and b/DataMinerAlgorithmsCrawler/target/classes/org/gcube/data/publishing/gCatFeeder/crawlers/dm/CKANRetriever.class differ diff --git a/DataMinerAlgorithmsCrawler/target/classes/org/gcube/data/publishing/gCatFeeder/crawlers/dm/CKANTransformer.class b/DataMinerAlgorithmsCrawler/target/classes/org/gcube/data/publishing/gCatFeeder/crawlers/dm/CKANTransformer.class new file mode 100644 index 0000000..e9b9e6c Binary files /dev/null and b/DataMinerAlgorithmsCrawler/target/classes/org/gcube/data/publishing/gCatFeeder/crawlers/dm/CKANTransformer.class differ diff --git a/DataMinerAlgorithmsCrawler/target/classes/org/gcube/data/publishing/gCatFeeder/crawlers/dm/DMAlgorithmsInfoCollector.class b/DataMinerAlgorithmsCrawler/target/classes/org/gcube/data/publishing/gCatFeeder/crawlers/dm/DMAlgorithmsInfoCollector.class new file mode 100644 index 0000000..2995ead Binary files /dev/null and b/DataMinerAlgorithmsCrawler/target/classes/org/gcube/data/publishing/gCatFeeder/crawlers/dm/DMAlgorithmsInfoCollector.class differ diff --git a/DataMinerAlgorithmsCrawler/target/classes/org/gcube/data/publishing/gCatFeeder/crawlers/dm/DataMinerCrawler.class b/DataMinerAlgorithmsCrawler/target/classes/org/gcube/data/publishing/gCatFeeder/crawlers/dm/DataMinerCrawler.class new file mode 100644 index 0000000..797a738 Binary files /dev/null and b/DataMinerAlgorithmsCrawler/target/classes/org/gcube/data/publishing/gCatFeeder/crawlers/dm/DataMinerCrawler.class differ diff --git a/DataMinerAlgorithmsCrawler/target/classes/org/gcube/data/publishing/gCatFeeder/crawlers/dm/model/CkanModel.class b/DataMinerAlgorithmsCrawler/target/classes/org/gcube/data/publishing/gCatFeeder/crawlers/dm/model/CkanModel.class new file mode 100644 index 0000000..dfe784b Binary files /dev/null and b/DataMinerAlgorithmsCrawler/target/classes/org/gcube/data/publishing/gCatFeeder/crawlers/dm/model/CkanModel.class differ diff --git a/DataMinerAlgorithmsCrawler/target/classes/org/gcube/data/publishing/gCatFeeder/crawlers/dm/model/InternalAlgorithmDescriptor.class b/DataMinerAlgorithmsCrawler/target/classes/org/gcube/data/publishing/gCatFeeder/crawlers/dm/model/InternalAlgorithmDescriptor.class new file mode 100644 index 0000000..83d537d Binary files /dev/null and b/DataMinerAlgorithmsCrawler/target/classes/org/gcube/data/publishing/gCatFeeder/crawlers/dm/model/InternalAlgorithmDescriptor.class differ diff --git a/commons/.classpath b/commons/.classpath new file mode 100644 index 0000000..fae1a2b --- /dev/null +++ b/commons/.classpath @@ -0,0 +1,36 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/commons/.project b/commons/.project new file mode 100644 index 0000000..242e05b --- /dev/null +++ b/commons/.project @@ -0,0 +1,23 @@ + + + commons + + + + + + org.eclipse.jdt.core.javabuilder + + + + + org.eclipse.m2e.core.maven2Builder + + + + + + org.eclipse.jdt.core.javanature + org.eclipse.m2e.core.maven2Nature + + diff --git a/commons/.settings/org.eclipse.core.resources.prefs b/commons/.settings/org.eclipse.core.resources.prefs new file mode 100644 index 0000000..29abf99 --- /dev/null +++ b/commons/.settings/org.eclipse.core.resources.prefs @@ -0,0 +1,6 @@ +eclipse.preferences.version=1 +encoding//src/main/java=UTF-8 +encoding//src/main/resources=UTF-8 +encoding//src/test/java=UTF-8 +encoding//src/test/resources=UTF-8 +encoding/=UTF-8 diff --git a/commons/.settings/org.eclipse.jdt.core.prefs b/commons/.settings/org.eclipse.jdt.core.prefs new file mode 100644 index 0000000..714351a --- /dev/null +++ b/commons/.settings/org.eclipse.jdt.core.prefs @@ -0,0 +1,5 @@ +eclipse.preferences.version=1 +org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.8 +org.eclipse.jdt.core.compiler.compliance=1.8 +org.eclipse.jdt.core.compiler.problem.forbiddenReference=warning +org.eclipse.jdt.core.compiler.source=1.8 diff --git a/commons/.settings/org.eclipse.jdt.ui.prefs b/commons/.settings/org.eclipse.jdt.ui.prefs new file mode 100644 index 0000000..ced03a5 --- /dev/null +++ b/commons/.settings/org.eclipse.jdt.ui.prefs @@ -0,0 +1,3 @@ +eclipse.preferences.version=1 +org.eclipse.jdt.ui.javadoc=false +org.eclipse.jdt.ui.text.custom_code_templates= diff --git a/commons/.settings/org.eclipse.m2e.core.prefs b/commons/.settings/org.eclipse.m2e.core.prefs new file mode 100644 index 0000000..f897a7f --- /dev/null +++ b/commons/.settings/org.eclipse.m2e.core.prefs @@ -0,0 +1,4 @@ +activeProfiles= +eclipse.preferences.version=1 +resolveWorkspaceProjects=true +version=1 diff --git a/commons/pom.xml b/commons/pom.xml new file mode 100644 index 0000000..c3d6ed7 --- /dev/null +++ b/commons/pom.xml @@ -0,0 +1,33 @@ + + 4.0.0 + + org.gcube.data-publishing.gCat-Feeder + gCat-Feeder-Suite + 1.0.0-SNAPSHOT + + commons + commons + Common utilities for gCatFeeder logic + + + + org.gcube.resourcemanagement + resourcemanager-client + [1.0.0-SNAPSHOT,2.0.0-SNAPSHOT) + + + + org.slf4j + slf4j-api + + + + + org.gcube.common + authorization-client + + + + + + \ No newline at end of file diff --git a/commons/src/main/java/org/gcube/data/publishing/gCatFeeder/model/CatalogueInstanceDescriptor.java b/commons/src/main/java/org/gcube/data/publishing/gCatFeeder/model/CatalogueInstanceDescriptor.java new file mode 100644 index 0000000..c08f111 --- /dev/null +++ b/commons/src/main/java/org/gcube/data/publishing/gCatFeeder/model/CatalogueInstanceDescriptor.java @@ -0,0 +1,60 @@ +package org.gcube.data.publishing.gCatFeeder.model; + +import java.util.HashMap; + +public class CatalogueInstanceDescriptor { + + private String url=null; + private String user=null; + private String password=null; + + private String customToken=null; + + private HashMap headersParameters=new HashMap<>(); + + public CatalogueInstanceDescriptor(String url) { + super(); + this.url = url; + } + + public String getUrl() { + return url; + } + + public String getUser() { + return user; + } + + public String getPassword() { + return password; + } + + public String getCustomToken() { + return customToken; + } + + public HashMap getHeadersParameters() { + return headersParameters; + } + + public CatalogueInstanceDescriptor setUser(String user) { + this.user = user; + return this; + } + + public CatalogueInstanceDescriptor setPassword(String password) { + this.password = password; + return this; + } + + public CatalogueInstanceDescriptor setCustomToken(String customToken) { + this.customToken = customToken; + return this; + } + + public CatalogueInstanceDescriptor addHeaderParameter(String name,String value) { + this.headersParameters.put(name, value); + return this; + } + +} diff --git a/commons/src/main/java/org/gcube/data/publishing/gCatFeeder/model/FormatData.java b/commons/src/main/java/org/gcube/data/publishing/gCatFeeder/model/FormatData.java new file mode 100644 index 0000000..5c34767 --- /dev/null +++ b/commons/src/main/java/org/gcube/data/publishing/gCatFeeder/model/FormatData.java @@ -0,0 +1,5 @@ +package org.gcube.data.publishing.gCatFeeder.model; + +public enum FormatData { + CKAN +} diff --git a/commons/src/main/java/org/gcube/data/publishing/gCatFeeder/utils/ContextUtils.java b/commons/src/main/java/org/gcube/data/publishing/gCatFeeder/utils/ContextUtils.java new file mode 100644 index 0000000..55a46a6 --- /dev/null +++ b/commons/src/main/java/org/gcube/data/publishing/gCatFeeder/utils/ContextUtils.java @@ -0,0 +1,88 @@ +package org.gcube.data.publishing.gCatFeeder.utils; + +import static org.gcube.common.authorization.client.Constants.authorizationService; + +import java.util.ArrayList; + +import org.gcube.common.authorization.library.AuthorizationEntry; +import org.gcube.common.authorization.library.provider.SecurityTokenProvider; +import org.gcube.common.scope.api.ScopeProvider; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class ContextUtils { + + private static final Logger log= LoggerFactory.getLogger(ContextUtils.class); + + public static String getCurrentCaller(){ + try{ + String token=SecurityTokenProvider.instance.get(); + log.debug("Token is : "+token); + if(token==null) throw new Exception("Security Token is null"); + AuthorizationEntry entry = authorizationService().get(token); + return entry.getClientInfo().getId(); + }catch(Exception e ){ + log.debug("Unable to resolve token, checking scope provider..",e); + return "Unidentified data-transfer user"; + } + } + + + public static String getCurrentScope(){ + + String scope=ScopeProvider.instance.get(); + if(scope!=null) { + log.debug("Found scope provider {}, skipping token",scope); + return scope; + }else{ + try{ + log.debug("Scope provider not set, reverting to token"); + String token=SecurityTokenProvider.instance.get(); + log.debug("Token is : "+token); + if(token==null) throw new Exception("Security Token is null"); + AuthorizationEntry entry = authorizationService().get(token); + return entry.getContext(); + }catch(Exception e){ + throw new RuntimeException("Unable to evaluate scope ",e); + } + } + } + + + + + public static String getCurrentScopeName(){ + return getScopeName(getCurrentScope()); + } + + public static String getScopeName(String scope) { + return scope.substring(scope.lastIndexOf('/')+1); + } + + + public static ArrayList getParentScopes(){ + String currentScope=getCurrentScope(); + String[] splitted=currentScope.substring(1).split("/"); + ArrayList toReturn=new ArrayList(); + for(int i=0;i getParentScopes(String scope){ + String[] splitted=scope.substring(1).split("/"); + ArrayList toReturn=new ArrayList(); + for(int i=0;i queryForServiceEndpoints(String category, String platformName){ + log.debug("Querying for Service Endpoints [category : {} , platformName : {}, currentScope : {} ]",category,platformName,ContextUtils.getCurrentScope()); + + SimpleQuery query = queryFor(ServiceEndpoint.class); + + query.addCondition("$resource/Profile/Category/text() eq '"+category+"'") + .addCondition("$resource/Profile/Platform/Name/text() eq '"+platformName+"'"); + // .setResult("$resource/Profile/AccessPoint"); + + DiscoveryClient client = clientFor(ServiceEndpoint.class); + + return client.submit(query); + } + + public static List queryForGCoreEndpoint(String serviceClass,String serviceName){ + log.debug("Querying for GCore Endpoints [ServiceClass : {} , ServiceName : {}, currentScope : {} ]",serviceClass,serviceName,ContextUtils.getCurrentScope()); + + + SimpleQuery query =queryFor(GCoreEndpoint.class); + query.addCondition("$resource/Profile/ServiceClass/text() eq '"+serviceClass+"'") + .addCondition("$resource/Profile/ServiceName/text() eq '"+serviceName+"'"); + // .setResult("$resource/Profile/AccessPoint"); + + DiscoveryClient client = clientFor(GCoreEndpoint.class); + + return client.submit(query); + } + +} diff --git a/commons/src/main/java/org/gcube/data/publishing/gCatFeeder/utils/TokenUtils.java b/commons/src/main/java/org/gcube/data/publishing/gCatFeeder/utils/TokenUtils.java new file mode 100644 index 0000000..fef0c82 --- /dev/null +++ b/commons/src/main/java/org/gcube/data/publishing/gCatFeeder/utils/TokenUtils.java @@ -0,0 +1,5 @@ +package org.gcube.data.publishing.gCatFeeder.utils; + +public class TokenUtils { + +} diff --git a/commons/target/classes/META-INF/MANIFEST.MF b/commons/target/classes/META-INF/MANIFEST.MF new file mode 100644 index 0000000..e2601c5 --- /dev/null +++ b/commons/target/classes/META-INF/MANIFEST.MF @@ -0,0 +1,5 @@ +Manifest-Version: 1.0 +Built-By: fabio +Build-Jdk: 1.8.0_201 +Created-By: Maven Integration for Eclipse + diff --git a/commons/target/classes/META-INF/maven/org.gcube.data-publishing.gCat-Feeder/commons/pom.properties b/commons/target/classes/META-INF/maven/org.gcube.data-publishing.gCat-Feeder/commons/pom.properties new file mode 100644 index 0000000..5c70b25 --- /dev/null +++ b/commons/target/classes/META-INF/maven/org.gcube.data-publishing.gCat-Feeder/commons/pom.properties @@ -0,0 +1,7 @@ +#Generated by Maven Integration for Eclipse +#Wed Mar 06 11:12:02 CET 2019 +version=1.0.0-SNAPSHOT +groupId=org.gcube.data-publishing.gCat-Feeder +m2e.projectName=commons +m2e.projectLocation=/home/fabio/workspaces/DEV/gCat-Feeder-Suite/commons +artifactId=commons diff --git a/commons/target/classes/META-INF/maven/org.gcube.data-publishing.gCat-Feeder/commons/pom.xml b/commons/target/classes/META-INF/maven/org.gcube.data-publishing.gCat-Feeder/commons/pom.xml new file mode 100644 index 0000000..c3d6ed7 --- /dev/null +++ b/commons/target/classes/META-INF/maven/org.gcube.data-publishing.gCat-Feeder/commons/pom.xml @@ -0,0 +1,33 @@ + + 4.0.0 + + org.gcube.data-publishing.gCat-Feeder + gCat-Feeder-Suite + 1.0.0-SNAPSHOT + + commons + commons + Common utilities for gCatFeeder logic + + + + org.gcube.resourcemanagement + resourcemanager-client + [1.0.0-SNAPSHOT,2.0.0-SNAPSHOT) + + + + org.slf4j + slf4j-api + + + + + org.gcube.common + authorization-client + + + + + + \ No newline at end of file diff --git a/commons/target/classes/org/gcube/data/publishing/gCatFeeder/model/CatalogueInstanceDescriptor.class b/commons/target/classes/org/gcube/data/publishing/gCatFeeder/model/CatalogueInstanceDescriptor.class new file mode 100644 index 0000000..b89d20b Binary files /dev/null and b/commons/target/classes/org/gcube/data/publishing/gCatFeeder/model/CatalogueInstanceDescriptor.class differ diff --git a/commons/target/classes/org/gcube/data/publishing/gCatFeeder/model/FormatData.class b/commons/target/classes/org/gcube/data/publishing/gCatFeeder/model/FormatData.class new file mode 100644 index 0000000..169ea73 Binary files /dev/null and b/commons/target/classes/org/gcube/data/publishing/gCatFeeder/model/FormatData.class differ diff --git a/commons/target/classes/org/gcube/data/publishing/gCatFeeder/utils/ContextUtils.class b/commons/target/classes/org/gcube/data/publishing/gCatFeeder/utils/ContextUtils.class new file mode 100644 index 0000000..cfdf33a Binary files /dev/null and b/commons/target/classes/org/gcube/data/publishing/gCatFeeder/utils/ContextUtils.class differ diff --git a/commons/target/classes/org/gcube/data/publishing/gCatFeeder/utils/ISUtils.class b/commons/target/classes/org/gcube/data/publishing/gCatFeeder/utils/ISUtils.class new file mode 100644 index 0000000..f9846d8 Binary files /dev/null and b/commons/target/classes/org/gcube/data/publishing/gCatFeeder/utils/ISUtils.class differ diff --git a/commons/target/classes/org/gcube/data/publishing/gCatFeeder/utils/TokenUtils.class b/commons/target/classes/org/gcube/data/publishing/gCatFeeder/utils/TokenUtils.class new file mode 100644 index 0000000..a4cf5e8 Binary files /dev/null and b/commons/target/classes/org/gcube/data/publishing/gCatFeeder/utils/TokenUtils.class differ diff --git a/crawler-plugin-framework/.classpath b/crawler-plugin-framework/.classpath new file mode 100644 index 0000000..fae1a2b --- /dev/null +++ b/crawler-plugin-framework/.classpath @@ -0,0 +1,36 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/crawler-plugin-framework/.project b/crawler-plugin-framework/.project new file mode 100644 index 0000000..ec2ebcf --- /dev/null +++ b/crawler-plugin-framework/.project @@ -0,0 +1,23 @@ + + + crawler-plugin-framework + + + + + + org.eclipse.jdt.core.javabuilder + + + + + org.eclipse.m2e.core.maven2Builder + + + + + + org.eclipse.jdt.core.javanature + org.eclipse.m2e.core.maven2Nature + + diff --git a/crawler-plugin-framework/.settings/org.eclipse.core.resources.prefs b/crawler-plugin-framework/.settings/org.eclipse.core.resources.prefs new file mode 100644 index 0000000..29abf99 --- /dev/null +++ b/crawler-plugin-framework/.settings/org.eclipse.core.resources.prefs @@ -0,0 +1,6 @@ +eclipse.preferences.version=1 +encoding//src/main/java=UTF-8 +encoding//src/main/resources=UTF-8 +encoding//src/test/java=UTF-8 +encoding//src/test/resources=UTF-8 +encoding/=UTF-8 diff --git a/crawler-plugin-framework/.settings/org.eclipse.jdt.core.prefs b/crawler-plugin-framework/.settings/org.eclipse.jdt.core.prefs new file mode 100644 index 0000000..714351a --- /dev/null +++ b/crawler-plugin-framework/.settings/org.eclipse.jdt.core.prefs @@ -0,0 +1,5 @@ +eclipse.preferences.version=1 +org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.8 +org.eclipse.jdt.core.compiler.compliance=1.8 +org.eclipse.jdt.core.compiler.problem.forbiddenReference=warning +org.eclipse.jdt.core.compiler.source=1.8 diff --git a/crawler-plugin-framework/.settings/org.eclipse.m2e.core.prefs b/crawler-plugin-framework/.settings/org.eclipse.m2e.core.prefs new file mode 100644 index 0000000..f897a7f --- /dev/null +++ b/crawler-plugin-framework/.settings/org.eclipse.m2e.core.prefs @@ -0,0 +1,4 @@ +activeProfiles= +eclipse.preferences.version=1 +resolveWorkspaceProjects=true +version=1 diff --git a/crawler-plugin-framework/pom.xml b/crawler-plugin-framework/pom.xml new file mode 100644 index 0000000..3138b2c --- /dev/null +++ b/crawler-plugin-framework/pom.xml @@ -0,0 +1,26 @@ + + 4.0.0 + + org.gcube.data-publishing.gCat-Feeder + gCat-Feeder-Suite + 1.0.0-SNAPSHOT + + crawler-plugin-framework + crawler-plugin-framework + Framework for crawler implementations + + + + org.projectlombok + lombok + 1.14.8 + + + + + org.gcube.data-publishing.gCat-Feeder + commons + 1.0.0-SNAPSHOT + + + \ No newline at end of file diff --git a/crawler-plugin-framework/src/main/java/org/gcube/data/publishing/gCatfeeder/crawlers/CatalogueRetriever.java b/crawler-plugin-framework/src/main/java/org/gcube/data/publishing/gCatfeeder/crawlers/CatalogueRetriever.java new file mode 100644 index 0000000..582de5b --- /dev/null +++ b/crawler-plugin-framework/src/main/java/org/gcube/data/publishing/gCatfeeder/crawlers/CatalogueRetriever.java @@ -0,0 +1,9 @@ +package org.gcube.data.publishing.gCatfeeder.crawlers; + +import org.gcube.data.publishing.gCatFeeder.model.CatalogueInstanceDescriptor; + +public interface CatalogueRetriever { + + public CatalogueInstanceDescriptor getInstance(); + +} diff --git a/crawler-plugin-framework/src/main/java/org/gcube/data/publishing/gCatfeeder/crawlers/CrawlerPlugin.java b/crawler-plugin-framework/src/main/java/org/gcube/data/publishing/gCatfeeder/crawlers/CrawlerPlugin.java new file mode 100644 index 0000000..443535f --- /dev/null +++ b/crawler-plugin-framework/src/main/java/org/gcube/data/publishing/gCatfeeder/crawlers/CrawlerPlugin.java @@ -0,0 +1,20 @@ +package org.gcube.data.publishing.gCatfeeder.crawlers; + +import java.util.Set; + +import org.gcube.data.publishing.gCatFeeder.model.FormatData; +import org.gcube.data.publishing.gCatfeeder.crawlers.model.CustomData; +import org.gcube.data.publishing.gCatfeeder.crawlers.model.PluginDescriptor; +import org.gcube.data.publishing.gCatfeeder.crawlers.model.PublisherFormatData; +import org.gcube.data.publishing.gCatfeeder.crawlers.model.faults.CatalogueNotSupportedException; + +public interface CrawlerPlugin{ + + public PluginDescriptor getDescriptor(); + + public CatalogueRetriever getRetrieverByCatalogueType(FormatData catalogueType) throws CatalogueNotSupportedException; + + public Set> getImplementedTransformers(); + + public DataCollector getCollector(); +} diff --git a/crawler-plugin-framework/src/main/java/org/gcube/data/publishing/gCatfeeder/crawlers/DataCollector.java b/crawler-plugin-framework/src/main/java/org/gcube/data/publishing/gCatfeeder/crawlers/DataCollector.java new file mode 100644 index 0000000..2b71a91 --- /dev/null +++ b/crawler-plugin-framework/src/main/java/org/gcube/data/publishing/gCatfeeder/crawlers/DataCollector.java @@ -0,0 +1,15 @@ +package org.gcube.data.publishing.gCatfeeder.crawlers; + +import java.util.Set; + +import org.gcube.data.publishing.gCatfeeder.crawlers.model.CustomData; +import org.gcube.data.publishing.gCatfeeder.crawlers.model.faults.CollectorFault; + +public interface DataCollector { + + + public Set collect() throws CollectorFault; + + + +} diff --git a/crawler-plugin-framework/src/main/java/org/gcube/data/publishing/gCatfeeder/crawlers/DataTransformer.java b/crawler-plugin-framework/src/main/java/org/gcube/data/publishing/gCatfeeder/crawlers/DataTransformer.java new file mode 100644 index 0000000..fcbafe4 --- /dev/null +++ b/crawler-plugin-framework/src/main/java/org/gcube/data/publishing/gCatfeeder/crawlers/DataTransformer.java @@ -0,0 +1,15 @@ +package org.gcube.data.publishing.gCatfeeder.crawlers; + +import java.util.Collection; +import java.util.Set; + +import org.gcube.data.publishing.gCatFeeder.model.FormatData; +import org.gcube.data.publishing.gCatfeeder.crawlers.model.CustomData; +import org.gcube.data.publishing.gCatfeeder.crawlers.model.PublisherFormatData; + +public interface DataTransformer { + + public FormatData getTargetFormat(); + + public Set transform(Collection collectedData); +} diff --git a/crawler-plugin-framework/src/main/java/org/gcube/data/publishing/gCatfeeder/crawlers/model/CustomData.java b/crawler-plugin-framework/src/main/java/org/gcube/data/publishing/gCatfeeder/crawlers/model/CustomData.java new file mode 100644 index 0000000..efb4bcf --- /dev/null +++ b/crawler-plugin-framework/src/main/java/org/gcube/data/publishing/gCatfeeder/crawlers/model/CustomData.java @@ -0,0 +1,5 @@ +package org.gcube.data.publishing.gCatfeeder.crawlers.model; + +public interface CustomData { + +} diff --git a/crawler-plugin-framework/src/main/java/org/gcube/data/publishing/gCatfeeder/crawlers/model/PluginDescriptor.java b/crawler-plugin-framework/src/main/java/org/gcube/data/publishing/gCatfeeder/crawlers/model/PluginDescriptor.java new file mode 100644 index 0000000..2ad1c4b --- /dev/null +++ b/crawler-plugin-framework/src/main/java/org/gcube/data/publishing/gCatfeeder/crawlers/model/PluginDescriptor.java @@ -0,0 +1,22 @@ +package org.gcube.data.publishing.gCatfeeder.crawlers.model; + +import lombok.Getter; +import lombok.NonNull; +import lombok.RequiredArgsConstructor; +import lombok.Setter; + +@Getter +@Setter +@RequiredArgsConstructor +public class PluginDescriptor { + + // Plugin info + + @NonNull + private String name; + + + // Run Interval + + +} diff --git a/crawler-plugin-framework/src/main/java/org/gcube/data/publishing/gCatfeeder/crawlers/model/PublisherFormatData.java b/crawler-plugin-framework/src/main/java/org/gcube/data/publishing/gCatfeeder/crawlers/model/PublisherFormatData.java new file mode 100644 index 0000000..8837862 --- /dev/null +++ b/crawler-plugin-framework/src/main/java/org/gcube/data/publishing/gCatfeeder/crawlers/model/PublisherFormatData.java @@ -0,0 +1,12 @@ +package org.gcube.data.publishing.gCatfeeder.crawlers.model; + +import org.gcube.data.publishing.gCatFeeder.model.FormatData; + +public interface PublisherFormatData { + + + public FormatData getFormat(); + + public String serialize(); + +} diff --git a/crawler-plugin-framework/src/main/java/org/gcube/data/publishing/gCatfeeder/crawlers/model/faults/CatalogueNotFound.java b/crawler-plugin-framework/src/main/java/org/gcube/data/publishing/gCatfeeder/crawlers/model/faults/CatalogueNotFound.java new file mode 100644 index 0000000..076a6db --- /dev/null +++ b/crawler-plugin-framework/src/main/java/org/gcube/data/publishing/gCatfeeder/crawlers/model/faults/CatalogueNotFound.java @@ -0,0 +1,29 @@ +package org.gcube.data.publishing.gCatfeeder.crawlers.model.faults; + +public class CatalogueNotFound extends CrawlerException { + + public CatalogueNotFound() { + // TODO Auto-generated constructor stub + } + + public CatalogueNotFound(String message) { + super(message); + // TODO Auto-generated constructor stub + } + + public CatalogueNotFound(Throwable cause) { + super(cause); + // TODO Auto-generated constructor stub + } + + public CatalogueNotFound(String message, Throwable cause) { + super(message, cause); + // TODO Auto-generated constructor stub + } + + public CatalogueNotFound(String message, Throwable cause, boolean enableSuppression, boolean writableStackTrace) { + super(message, cause, enableSuppression, writableStackTrace); + // TODO Auto-generated constructor stub + } + +} diff --git a/crawler-plugin-framework/src/main/java/org/gcube/data/publishing/gCatfeeder/crawlers/model/faults/CatalogueNotSupportedException.java b/crawler-plugin-framework/src/main/java/org/gcube/data/publishing/gCatfeeder/crawlers/model/faults/CatalogueNotSupportedException.java new file mode 100644 index 0000000..dd3b317 --- /dev/null +++ b/crawler-plugin-framework/src/main/java/org/gcube/data/publishing/gCatfeeder/crawlers/model/faults/CatalogueNotSupportedException.java @@ -0,0 +1,30 @@ +package org.gcube.data.publishing.gCatfeeder.crawlers.model.faults; + +public class CatalogueNotSupportedException extends CrawlerException { + + public CatalogueNotSupportedException() { + // TODO Auto-generated constructor stub + } + + public CatalogueNotSupportedException(String message) { + super(message); + // TODO Auto-generated constructor stub + } + + public CatalogueNotSupportedException(Throwable cause) { + super(cause); + // TODO Auto-generated constructor stub + } + + public CatalogueNotSupportedException(String message, Throwable cause) { + super(message, cause); + // TODO Auto-generated constructor stub + } + + public CatalogueNotSupportedException(String message, Throwable cause, boolean enableSuppression, + boolean writableStackTrace) { + super(message, cause, enableSuppression, writableStackTrace); + // TODO Auto-generated constructor stub + } + +} diff --git a/crawler-plugin-framework/src/main/java/org/gcube/data/publishing/gCatfeeder/crawlers/model/faults/CollectorFault.java b/crawler-plugin-framework/src/main/java/org/gcube/data/publishing/gCatfeeder/crawlers/model/faults/CollectorFault.java new file mode 100644 index 0000000..47f7da5 --- /dev/null +++ b/crawler-plugin-framework/src/main/java/org/gcube/data/publishing/gCatfeeder/crawlers/model/faults/CollectorFault.java @@ -0,0 +1,29 @@ +package org.gcube.data.publishing.gCatfeeder.crawlers.model.faults; + +public class CollectorFault extends CrawlerException { + + public CollectorFault() { + // TODO Auto-generated constructor stub + } + + public CollectorFault(String message) { + super(message); + // TODO Auto-generated constructor stub + } + + public CollectorFault(Throwable cause) { + super(cause); + // TODO Auto-generated constructor stub + } + + public CollectorFault(String message, Throwable cause) { + super(message, cause); + // TODO Auto-generated constructor stub + } + + public CollectorFault(String message, Throwable cause, boolean enableSuppression, boolean writableStackTrace) { + super(message, cause, enableSuppression, writableStackTrace); + // TODO Auto-generated constructor stub + } + +} diff --git a/crawler-plugin-framework/src/main/java/org/gcube/data/publishing/gCatfeeder/crawlers/model/faults/CrawlerException.java b/crawler-plugin-framework/src/main/java/org/gcube/data/publishing/gCatfeeder/crawlers/model/faults/CrawlerException.java new file mode 100644 index 0000000..4d6520d --- /dev/null +++ b/crawler-plugin-framework/src/main/java/org/gcube/data/publishing/gCatfeeder/crawlers/model/faults/CrawlerException.java @@ -0,0 +1,29 @@ +package org.gcube.data.publishing.gCatfeeder.crawlers.model.faults; + +public class CrawlerException extends Exception { + + public CrawlerException() { + // TODO Auto-generated constructor stub + } + + public CrawlerException(String message) { + super(message); + // TODO Auto-generated constructor stub + } + + public CrawlerException(Throwable cause) { + super(cause); + // TODO Auto-generated constructor stub + } + + public CrawlerException(String message, Throwable cause) { + super(message, cause); + // TODO Auto-generated constructor stub + } + + public CrawlerException(String message, Throwable cause, boolean enableSuppression, boolean writableStackTrace) { + super(message, cause, enableSuppression, writableStackTrace); + // TODO Auto-generated constructor stub + } + +} diff --git a/crawler-plugin-framework/src/main/java/org/gcube/data/publishing/gCatfeeder/crawlers/model/faults/InvalidCustomData.java b/crawler-plugin-framework/src/main/java/org/gcube/data/publishing/gCatfeeder/crawlers/model/faults/InvalidCustomData.java new file mode 100644 index 0000000..ed6d972 --- /dev/null +++ b/crawler-plugin-framework/src/main/java/org/gcube/data/publishing/gCatfeeder/crawlers/model/faults/InvalidCustomData.java @@ -0,0 +1,30 @@ +package org.gcube.data.publishing.gCatfeeder.crawlers.model.faults; + +public class InvalidCustomData extends CrawlerException { + + public InvalidCustomData() { + // TODO Auto-generated constructor stub + } + + public InvalidCustomData(String message) { + super(message); + // TODO Auto-generated constructor stub + } + + public InvalidCustomData(Throwable cause) { + super(cause); + // TODO Auto-generated constructor stub + } + + public InvalidCustomData(String message, Throwable cause) { + super(message, cause); + // TODO Auto-generated constructor stub + } + + public InvalidCustomData(String message, Throwable cause, boolean enableSuppression, + boolean writableStackTrace) { + super(message, cause, enableSuppression, writableStackTrace); + // TODO Auto-generated constructor stub + } + +} diff --git a/crawler-plugin-framework/src/main/java/org/gcube/data/publishing/gCatfeeder/crawlers/model/faults/TransformException.java b/crawler-plugin-framework/src/main/java/org/gcube/data/publishing/gCatfeeder/crawlers/model/faults/TransformException.java new file mode 100644 index 0000000..bd38526 --- /dev/null +++ b/crawler-plugin-framework/src/main/java/org/gcube/data/publishing/gCatfeeder/crawlers/model/faults/TransformException.java @@ -0,0 +1,29 @@ +package org.gcube.data.publishing.gCatfeeder.crawlers.model.faults; + +public class TransformException extends CrawlerException { + + public TransformException() { + // TODO Auto-generated constructor stub + } + + public TransformException(String message) { + super(message); + // TODO Auto-generated constructor stub + } + + public TransformException(Throwable cause) { + super(cause); + // TODO Auto-generated constructor stub + } + + public TransformException(String message, Throwable cause) { + super(message, cause); + // TODO Auto-generated constructor stub + } + + public TransformException(String message, Throwable cause, boolean enableSuppression, boolean writableStackTrace) { + super(message, cause, enableSuppression, writableStackTrace); + // TODO Auto-generated constructor stub + } + +} diff --git a/crawler-plugin-framework/src/test/java/org/gcube/data/publishing/gCatFeeder/crawlers/model/ModelTests.java b/crawler-plugin-framework/src/test/java/org/gcube/data/publishing/gCatFeeder/crawlers/model/ModelTests.java new file mode 100644 index 0000000..c7c53b1 --- /dev/null +++ b/crawler-plugin-framework/src/test/java/org/gcube/data/publishing/gCatFeeder/crawlers/model/ModelTests.java @@ -0,0 +1,11 @@ +package org.gcube.data.publishing.gCatFeeder.crawlers.model; + +import org.junit.Test; + +public class ModelTests { + + @Test + public void testConversion() { + + } +} diff --git a/crawler-plugin-framework/target/classes/META-INF/MANIFEST.MF b/crawler-plugin-framework/target/classes/META-INF/MANIFEST.MF new file mode 100644 index 0000000..e2601c5 --- /dev/null +++ b/crawler-plugin-framework/target/classes/META-INF/MANIFEST.MF @@ -0,0 +1,5 @@ +Manifest-Version: 1.0 +Built-By: fabio +Build-Jdk: 1.8.0_201 +Created-By: Maven Integration for Eclipse + diff --git a/crawler-plugin-framework/target/classes/META-INF/maven/org.gcube.data-publishing.gCat-Feeder/crawler-plugin-framework/pom.properties b/crawler-plugin-framework/target/classes/META-INF/maven/org.gcube.data-publishing.gCat-Feeder/crawler-plugin-framework/pom.properties new file mode 100644 index 0000000..ac7a47d --- /dev/null +++ b/crawler-plugin-framework/target/classes/META-INF/maven/org.gcube.data-publishing.gCat-Feeder/crawler-plugin-framework/pom.properties @@ -0,0 +1,7 @@ +#Generated by Maven Integration for Eclipse +#Wed Mar 06 11:12:02 CET 2019 +version=1.0.0-SNAPSHOT +groupId=org.gcube.data-publishing.gCat-Feeder +m2e.projectName=crawler-plugin-framework +m2e.projectLocation=/home/fabio/workspaces/DEV/gCat-Feeder-Suite/crawler-plugin-framework +artifactId=crawler-plugin-framework diff --git a/crawler-plugin-framework/target/classes/META-INF/maven/org.gcube.data-publishing.gCat-Feeder/crawler-plugin-framework/pom.xml b/crawler-plugin-framework/target/classes/META-INF/maven/org.gcube.data-publishing.gCat-Feeder/crawler-plugin-framework/pom.xml new file mode 100644 index 0000000..3138b2c --- /dev/null +++ b/crawler-plugin-framework/target/classes/META-INF/maven/org.gcube.data-publishing.gCat-Feeder/crawler-plugin-framework/pom.xml @@ -0,0 +1,26 @@ + + 4.0.0 + + org.gcube.data-publishing.gCat-Feeder + gCat-Feeder-Suite + 1.0.0-SNAPSHOT + + crawler-plugin-framework + crawler-plugin-framework + Framework for crawler implementations + + + + org.projectlombok + lombok + 1.14.8 + + + + + org.gcube.data-publishing.gCat-Feeder + commons + 1.0.0-SNAPSHOT + + + \ No newline at end of file diff --git a/crawler-plugin-framework/target/classes/org/gcube/data/publishing/gCatfeeder/crawlers/CatalogueRetriever.class b/crawler-plugin-framework/target/classes/org/gcube/data/publishing/gCatfeeder/crawlers/CatalogueRetriever.class new file mode 100644 index 0000000..bf927dd Binary files /dev/null and b/crawler-plugin-framework/target/classes/org/gcube/data/publishing/gCatfeeder/crawlers/CatalogueRetriever.class differ diff --git a/crawler-plugin-framework/target/classes/org/gcube/data/publishing/gCatfeeder/crawlers/CrawlerPlugin.class b/crawler-plugin-framework/target/classes/org/gcube/data/publishing/gCatfeeder/crawlers/CrawlerPlugin.class new file mode 100644 index 0000000..2691306 Binary files /dev/null and b/crawler-plugin-framework/target/classes/org/gcube/data/publishing/gCatfeeder/crawlers/CrawlerPlugin.class differ diff --git a/crawler-plugin-framework/target/classes/org/gcube/data/publishing/gCatfeeder/crawlers/DataCollector.class b/crawler-plugin-framework/target/classes/org/gcube/data/publishing/gCatfeeder/crawlers/DataCollector.class new file mode 100644 index 0000000..e1f6411 Binary files /dev/null and b/crawler-plugin-framework/target/classes/org/gcube/data/publishing/gCatfeeder/crawlers/DataCollector.class differ diff --git a/crawler-plugin-framework/target/classes/org/gcube/data/publishing/gCatfeeder/crawlers/DataTransformer.class b/crawler-plugin-framework/target/classes/org/gcube/data/publishing/gCatfeeder/crawlers/DataTransformer.class new file mode 100644 index 0000000..51eadce Binary files /dev/null and b/crawler-plugin-framework/target/classes/org/gcube/data/publishing/gCatfeeder/crawlers/DataTransformer.class differ diff --git a/crawler-plugin-framework/target/classes/org/gcube/data/publishing/gCatfeeder/crawlers/model/CustomData.class b/crawler-plugin-framework/target/classes/org/gcube/data/publishing/gCatfeeder/crawlers/model/CustomData.class new file mode 100644 index 0000000..615ad38 Binary files /dev/null and b/crawler-plugin-framework/target/classes/org/gcube/data/publishing/gCatfeeder/crawlers/model/CustomData.class differ diff --git a/crawler-plugin-framework/target/classes/org/gcube/data/publishing/gCatfeeder/crawlers/model/PluginDescriptor.class b/crawler-plugin-framework/target/classes/org/gcube/data/publishing/gCatfeeder/crawlers/model/PluginDescriptor.class new file mode 100644 index 0000000..299c183 Binary files /dev/null and b/crawler-plugin-framework/target/classes/org/gcube/data/publishing/gCatfeeder/crawlers/model/PluginDescriptor.class differ diff --git a/crawler-plugin-framework/target/classes/org/gcube/data/publishing/gCatfeeder/crawlers/model/PublisherFormatData.class b/crawler-plugin-framework/target/classes/org/gcube/data/publishing/gCatfeeder/crawlers/model/PublisherFormatData.class new file mode 100644 index 0000000..f01d2ab Binary files /dev/null and b/crawler-plugin-framework/target/classes/org/gcube/data/publishing/gCatfeeder/crawlers/model/PublisherFormatData.class differ diff --git a/crawler-plugin-framework/target/classes/org/gcube/data/publishing/gCatfeeder/crawlers/model/faults/CatalogueNotFound.class b/crawler-plugin-framework/target/classes/org/gcube/data/publishing/gCatfeeder/crawlers/model/faults/CatalogueNotFound.class new file mode 100644 index 0000000..581d493 Binary files /dev/null and b/crawler-plugin-framework/target/classes/org/gcube/data/publishing/gCatfeeder/crawlers/model/faults/CatalogueNotFound.class differ diff --git a/crawler-plugin-framework/target/classes/org/gcube/data/publishing/gCatfeeder/crawlers/model/faults/CatalogueNotSupportedException.class b/crawler-plugin-framework/target/classes/org/gcube/data/publishing/gCatfeeder/crawlers/model/faults/CatalogueNotSupportedException.class new file mode 100644 index 0000000..e29f071 Binary files /dev/null and b/crawler-plugin-framework/target/classes/org/gcube/data/publishing/gCatfeeder/crawlers/model/faults/CatalogueNotSupportedException.class differ diff --git a/crawler-plugin-framework/target/classes/org/gcube/data/publishing/gCatfeeder/crawlers/model/faults/CollectorFault.class b/crawler-plugin-framework/target/classes/org/gcube/data/publishing/gCatfeeder/crawlers/model/faults/CollectorFault.class new file mode 100644 index 0000000..13c62c8 Binary files /dev/null and b/crawler-plugin-framework/target/classes/org/gcube/data/publishing/gCatfeeder/crawlers/model/faults/CollectorFault.class differ diff --git a/crawler-plugin-framework/target/classes/org/gcube/data/publishing/gCatfeeder/crawlers/model/faults/CrawlerException.class b/crawler-plugin-framework/target/classes/org/gcube/data/publishing/gCatfeeder/crawlers/model/faults/CrawlerException.class new file mode 100644 index 0000000..b1a7e50 Binary files /dev/null and b/crawler-plugin-framework/target/classes/org/gcube/data/publishing/gCatfeeder/crawlers/model/faults/CrawlerException.class differ diff --git a/crawler-plugin-framework/target/classes/org/gcube/data/publishing/gCatfeeder/crawlers/model/faults/InvalidCustomData.class b/crawler-plugin-framework/target/classes/org/gcube/data/publishing/gCatfeeder/crawlers/model/faults/InvalidCustomData.class new file mode 100644 index 0000000..73e6194 Binary files /dev/null and b/crawler-plugin-framework/target/classes/org/gcube/data/publishing/gCatfeeder/crawlers/model/faults/InvalidCustomData.class differ diff --git a/crawler-plugin-framework/target/classes/org/gcube/data/publishing/gCatfeeder/crawlers/model/faults/TransformException.class b/crawler-plugin-framework/target/classes/org/gcube/data/publishing/gCatfeeder/crawlers/model/faults/TransformException.class new file mode 100644 index 0000000..10ae320 Binary files /dev/null and b/crawler-plugin-framework/target/classes/org/gcube/data/publishing/gCatfeeder/crawlers/model/faults/TransformException.class differ diff --git a/crawler-plugin-framework/target/test-classes/org/gcube/data/publishing/gCatFeeder/crawlers/model/ModelTests.class b/crawler-plugin-framework/target/test-classes/org/gcube/data/publishing/gCatFeeder/crawlers/model/ModelTests.class new file mode 100644 index 0000000..bfcfbfd Binary files /dev/null and b/crawler-plugin-framework/target/test-classes/org/gcube/data/publishing/gCatFeeder/crawlers/model/ModelTests.class differ diff --git a/gCat-Feeder/.classpath b/gCat-Feeder/.classpath new file mode 100644 index 0000000..fae1a2b --- /dev/null +++ b/gCat-Feeder/.classpath @@ -0,0 +1,36 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/gCat-Feeder/.project b/gCat-Feeder/.project new file mode 100644 index 0000000..c84310d --- /dev/null +++ b/gCat-Feeder/.project @@ -0,0 +1,23 @@ + + + gCat-Feeder + + + + + + org.eclipse.jdt.core.javabuilder + + + + + org.eclipse.m2e.core.maven2Builder + + + + + + org.eclipse.jdt.core.javanature + org.eclipse.m2e.core.maven2Nature + + diff --git a/gCat-Feeder/.settings/org.eclipse.core.resources.prefs b/gCat-Feeder/.settings/org.eclipse.core.resources.prefs new file mode 100644 index 0000000..29abf99 --- /dev/null +++ b/gCat-Feeder/.settings/org.eclipse.core.resources.prefs @@ -0,0 +1,6 @@ +eclipse.preferences.version=1 +encoding//src/main/java=UTF-8 +encoding//src/main/resources=UTF-8 +encoding//src/test/java=UTF-8 +encoding//src/test/resources=UTF-8 +encoding/=UTF-8 diff --git a/gCat-Feeder/.settings/org.eclipse.jdt.core.prefs b/gCat-Feeder/.settings/org.eclipse.jdt.core.prefs new file mode 100644 index 0000000..714351a --- /dev/null +++ b/gCat-Feeder/.settings/org.eclipse.jdt.core.prefs @@ -0,0 +1,5 @@ +eclipse.preferences.version=1 +org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.8 +org.eclipse.jdt.core.compiler.compliance=1.8 +org.eclipse.jdt.core.compiler.problem.forbiddenReference=warning +org.eclipse.jdt.core.compiler.source=1.8 diff --git a/gCat-Feeder/.settings/org.eclipse.m2e.core.prefs b/gCat-Feeder/.settings/org.eclipse.m2e.core.prefs new file mode 100644 index 0000000..f897a7f --- /dev/null +++ b/gCat-Feeder/.settings/org.eclipse.m2e.core.prefs @@ -0,0 +1,4 @@ +activeProfiles= +eclipse.preferences.version=1 +resolveWorkspaceProjects=true +version=1 diff --git a/gCat-Feeder/pom.xml b/gCat-Feeder/pom.xml new file mode 100644 index 0000000..900843c --- /dev/null +++ b/gCat-Feeder/pom.xml @@ -0,0 +1,33 @@ + + 4.0.0 + + org.gcube.data-publishing.gCat-Feeder + gCat-Feeder-Suite + 1.0.0-SNAPSHOT + + gCat-Feeder + gCat-Feeder + Service implementation + + + + + org.gcube.distribution + maven-smartgears-bom + LATEST + pom + import + + + org.glassfish.jersey + jersey-bom + 2.25.1 + pom + import + + + + + + + \ No newline at end of file diff --git a/gCat-Feeder/target/classes/META-INF/MANIFEST.MF b/gCat-Feeder/target/classes/META-INF/MANIFEST.MF new file mode 100644 index 0000000..e2601c5 --- /dev/null +++ b/gCat-Feeder/target/classes/META-INF/MANIFEST.MF @@ -0,0 +1,5 @@ +Manifest-Version: 1.0 +Built-By: fabio +Build-Jdk: 1.8.0_201 +Created-By: Maven Integration for Eclipse + diff --git a/gCat-Feeder/target/classes/META-INF/maven/org.gcube.data-publishing.gCat-Feeder/gCat-Feeder/pom.properties b/gCat-Feeder/target/classes/META-INF/maven/org.gcube.data-publishing.gCat-Feeder/gCat-Feeder/pom.properties new file mode 100644 index 0000000..0a18c95 --- /dev/null +++ b/gCat-Feeder/target/classes/META-INF/maven/org.gcube.data-publishing.gCat-Feeder/gCat-Feeder/pom.properties @@ -0,0 +1,7 @@ +#Generated by Maven Integration for Eclipse +#Thu Feb 28 15:58:44 CET 2019 +version=1.0.0-SNAPSHOT +groupId=org.gcube.data-publishing.gCat-Feeder +m2e.projectName=gCat-Feeder +m2e.projectLocation=/home/fabio/workspaces/DEV/gCat-Feeder-Suite/gCat-Feeder +artifactId=gCat-Feeder diff --git a/gCat-Feeder/target/classes/META-INF/maven/org.gcube.data-publishing.gCat-Feeder/gCat-Feeder/pom.xml b/gCat-Feeder/target/classes/META-INF/maven/org.gcube.data-publishing.gCat-Feeder/gCat-Feeder/pom.xml new file mode 100644 index 0000000..900843c --- /dev/null +++ b/gCat-Feeder/target/classes/META-INF/maven/org.gcube.data-publishing.gCat-Feeder/gCat-Feeder/pom.xml @@ -0,0 +1,33 @@ + + 4.0.0 + + org.gcube.data-publishing.gCat-Feeder + gCat-Feeder-Suite + 1.0.0-SNAPSHOT + + gCat-Feeder + gCat-Feeder + Service implementation + + + + + org.gcube.distribution + maven-smartgears-bom + LATEST + pom + import + + + org.glassfish.jersey + jersey-bom + 2.25.1 + pom + import + + + + + + + \ No newline at end of file diff --git a/pom.xml b/pom.xml new file mode 100644 index 0000000..684a479 --- /dev/null +++ b/pom.xml @@ -0,0 +1,55 @@ + + 4.0.0 + + org.gcube.tools + maven-parent + LATEST + + org.gcube.data-publishing.gCat-Feeder + gCat-Feeder-Suite + 1.0.0-SNAPSHOT + pom + gCat-Feeder Suite + gCat-Feeder Suite of components : service, plugin framework, plugins + + + + gCat-Feeder + crawler-plugin-framework + DataMinerAlgorithmsCrawler + commons + + + + + + org.gcube.distribution + gcube-bom + 1.3.0-SNAPSHOT + pom + import + + + + + + + + + ch.qos.logback + logback-classic + 1.0.13 + test + + + + junit + junit + 4.12 + test + + + + + \ No newline at end of file