gCubeSystem
/
gFeed
14
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

1.0.6

This commit is contained in:
Fabio Sinibaldi 2022-11-04 13:54:37 +01:00
parent 2d4eb75d5f
commit 3e4fe33171
21 changed files with 119 additions and 35 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
CHANGELOG.md
View File

@ -2,6 +2,9 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm
# Changelog for gFeed
## [v1.0.6-SNAPSHOT]
- Pom updates
## [v1.0.5]
- Pom updates

2
DataMinerAlgorithmsCrawler/CHANGELOG.md
View File

@ -2,6 +2,8 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm
# Changelog for org.gcube.data-publishing.gFeed.DataMinerAlgorithmsCrawler
## [v1.0.7-SNAPSHOT]
- Lombok version
## [v1.0.6]
- Pom updates

35
DataMinerAlgorithmsCrawler/pom.xml
View File

@ -4,11 +4,11 @@
<parent>
<groupId>org.gcube.data-publishing.gFeed</groupId>
<artifactId>gFeed-Suite</artifactId>
<version>1.0.5</version>
<version>1.0.6-SNAPSHOT</version>
</parent>
<artifactId>DataMinerAlgorithmsCrawler</artifactId>
<name>DataMinerAlgorithmsCrawler</name>
<version>1.0.6</version>
<version>1.0.7-SNAPSHOT</version>
<description>Plugin for gCat-Feeder for DataMiner Algorithms publishing</description>
@ -45,7 +45,7 @@
<dependency>
<groupId>org.gcube.data.analysis</groupId>
<artifactId>data-miner-manager-cl</artifactId>
<version>[1.8.0,2.0.0)</version>
<version>[1.9.0,1.10.0-SNAPSHOT)</version>
</dependency>
@ -57,7 +57,7 @@
<dependency>
<groupId>org.gcube.data-catalogue</groupId>
<artifactId>gcat-client</artifactId>
<version>[2.0.0,3.0.0)</version>
<version>[2.0.0,3.0.0-SNAPSHOT)</version>
</dependency>
<dependency>
@ -69,7 +69,6 @@
<dependency>
<groupId>org.projectlombok</groupId>
<artifactId>lombok</artifactId>
<version>1.18.20</version>
<scope>provided</scope>
</dependency>
@ -116,5 +115,29 @@
</plugins>
</build>
<repositories>
<!-- copied from DM cl pom -->
<repository>
<id>n52-releases</id>
<name>52n Releases</name>
<url>https://52north.org/maven/repo/releases/</url>
<releases>
<enabled>true</enabled>
</releases>
<snapshots>
<enabled>false</enabled>
</snapshots>
</repository>
<repository>
<id>geotools</id>
<name>Geotools</name>
<url>https://repo.osgeo.org/repository/release/</url>
<releases>
<enabled>true</enabled>
</releases>
<snapshots>
<enabled>false</enabled>
</snapshots>
</repository>
</repositories>
</project>

3
catalogue-plugin-framework/CHANGELOG.md
View File

@ -2,6 +2,9 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm
# Changelog for org.gcube.data-publishing.gFeed.catalogue-plugin-framework
## [v1.0.6-SNAPSHOT]
- Lombok dependency upgrade
## [v1.0.5]
- Pom updates

5
catalogue-plugin-framework/pom.xml
View File

@ -5,10 +5,10 @@
<parent>
<groupId>org.gcube.data-publishing.gFeed</groupId>
<artifactId>gFeed-Suite</artifactId>
<version>1.0.5</version>
<version>1.0.6-SNAPSHOT</version>
</parent>
<artifactId>catalogue-plugin-framework</artifactId>
<version>1.0.5</version>
<version>1.0.6-SNAPSHOT</version>
<scm>
@ -20,7 +20,6 @@
<dependency>
<groupId>org.projectlombok</groupId>
<artifactId>lombok</artifactId>
<version>1.14.8</version>
</dependency>

3
collectors-plugin-framework/CHANGELOG.md
View File

@ -2,6 +2,9 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm
# Changelog for org.gcube.data-publishing.gFeed.collectors-plugin-framework
## [v1.0.6-SNAPSHOT]
- Pom updates
## [v1.0.5]
- Pom updates

5
collectors-plugin-framework/pom.xml
View File

@ -5,12 +5,12 @@
<parent>
<groupId>org.gcube.data-publishing.gFeed</groupId>
<artifactId>gFeed-Suite</artifactId>
<version>1.0.5</version>
<version>1.0.6-SNAPSHOT</version>
</parent>
<artifactId>collectors-plugin-framework</artifactId>
<name>collectors-plugin-framework</name>
<description>Framework for collectors implementations</description>
<version>1.0.5</version>
<version>1.0.6-SNAPSHOT</version>
@ -26,7 +26,6 @@
<dependency>
<groupId>org.projectlombok</groupId>
<artifactId>lombok</artifactId>
<version>1.14.8</version>
</dependency>

3
commons/CHANGELOG.md
View File

@ -2,6 +2,9 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm
# Changelog for org.gcube.data-publishing.gFeed.test-commons
## [v1.0.6-SNAPSHOT]
- Pom updates
## [v1.0.5]
- Pom updates

4
commons/pom.xml
View File

@ -5,11 +5,11 @@
<parent>
<groupId>org.gcube.data-publishing.gFeed</groupId>
<artifactId>gFeed-Suite</artifactId>
<version>1.0.5</version>
<version>1.0.6-SNAPSHOT</version>
</parent>
<artifactId>commons</artifactId>
<name>commons</name>
<version>1.0.5</version>
<version>1.0.6-SNAPSHOT</version>
<description>Common utilities for gCatFeeder logic</description>

3
gCat-Controller/CHANGELOG.md
View File

@ -2,6 +2,9 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm
# Changelog for org.gcube.data-publishing.gFeed.gCat-controller
## [v1.0.8-SNAPSHOT]
- Pom updates
## [v1.0.7]
- Pom updates

4
gCat-Controller/pom.xml
View File

@ -3,11 +3,11 @@
<parent>
<groupId>org.gcube.data-publishing.gFeed</groupId>
<artifactId>gFeed-Suite</artifactId>
<version>1.0.5</version>
<version>1.0.6-SNAPSHOT</version>
</parent>
<artifactId>gCat-Controller</artifactId>
<name>gCat-Controller</name>
<version>1.0.7</version>
<version>1.0.8-SNAPSHOT</version>
<description>Controller implementation for GCat Service</description>
<scm>

3
gCat-Feeder/CHANGELOG.md
View File

@ -2,6 +2,9 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm
# Changelog for org.gcube.data-publishing.gFeed.gFeed-service
## [v1.0.6-SNAPSHOT]
- Pom updates
## [v1.0.5]
- Pom updates

4
gCat-Feeder/pom.xml
View File

@ -4,11 +4,11 @@
<parent>
<groupId>org.gcube.data-publishing.gFeed</groupId>
<artifactId>gFeed-Suite</artifactId>
<version>1.0.5</version>
<version>1.0.6-SNAPSHOT</version>
</parent>
<artifactId>gCat-Feeder</artifactId>
<name>gFeed Service</name>
<version>1.0.5</version>
<version>1.0.6-SNAPSHOT</version>
<description>Service implementation</description>
<properties>

3
oai-harvester/CHANGELOG.md
View File

@ -2,6 +2,9 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm
# Changelog for org.gcube.data-publishing.gFeed.oai-pmh
## [v1.0.7-SNAPSHOT]
- Pom updates
## [v1.0.6]
- Pom updates

4
oai-harvester/pom.xml
View File

@ -5,11 +5,11 @@
<parent>
<groupId>org.gcube.data-publishing.gFeed</groupId>
<artifactId>gFeed-Suite</artifactId>
<version>1.0.5</version>
<version>1.0.6-SNAPSHOT</version>
</parent>
<artifactId>oai-harvester</artifactId>
<name>oai-harvester</name>
<version>1.0.6</version>
<version>1.0.7-SNAPSHOT</version>
<scm>
<connection>scm:git:${gitBaseUrl}/gFeed</connection>

2
oai-harvester/src/main/java/org/gcube/data/publishing/gFeed/collectors/oai/OAIClient.java
View File

@ -83,7 +83,7 @@ public class OAIClient {
for(String set : specifiedSets) {
log.info("Loading "+metadataPrefix+" SET : "+set+" from "+baseUrl);
target.queryParam("set", set);
toReturn.addAll(call(target,metadataPrefix));
toReturn.addAll(call(target.queryParam("set", set),metadataPrefix));
}
else {
log.info("Loading "+metadataPrefix+" from "+baseUrl);

15
oai-harvester/src/test/java/org/gcube/application/gfeed/oai/TranslationTest.java
View File

@ -5,6 +5,7 @@ import java.util.Collection;
import java.util.Set;
import org.gcube.data.publishing.gCatFeeder.model.CatalogueFormatData;
import org.gcube.data.publishing.gCatFeeder.model.EnvironmentConfiguration;
import org.gcube.data.publishing.gCatFeeder.model.InternalConversionException;
import org.gcube.data.publishing.gCatFeeder.tests.BaseCollectorTest;
import org.gcube.data.publishing.gCatfeeder.collectors.CollectorPlugin;
@ -31,18 +32,20 @@ public class TranslationTest extends BaseCollectorTest{
ObjectMapper mapper = new ObjectMapper();
CollectorPlugin plugin=new OAIHarvester();
plugin.setEnvironmentConfiguration(getEnvironmentConfiguration());
EnvironmentConfiguration env = getEnvironmentConfiguration();
System.out.println("Setting ENV : "+env);
plugin.setEnvironmentConfiguration(env);
DataCollector collector=plugin.getCollector();
Collection collected=collector.collect();
System.out.println("Found "+collected.size()+" elements");
for(Object obj:collected)
System.out.println(mapper.writeValueAsString(obj)+"\n");
for(String destinationcatalogue : (Set<String>)plugin.getSupportedCatalogueTypes()) {
DataTransformer<? extends CatalogueFormatData, OAIRecord> transformer=plugin.getTransformerByCatalogueType(destinationcatalogue);
for(Object data:transformer.transform(collected))
System.out.println(((CatalogueFormatData)data).toCatalogueFormat());
}
// for(String destinationcatalogue : (Set<String>)plugin.getSupportedCatalogueTypes()) {
// DataTransformer<? extends CatalogueFormatData, OAIRecord> transformer=plugin.getTransformerByCatalogueType(destinationcatalogue);
// for(Object data:transformer.transform(collected))
// System.out.println(((CatalogueFormatData)data).toCatalogueFormat());
// }
}

16
pom.xml
View File

@ -9,7 +9,7 @@
</parent>
<groupId>org.gcube.data-publishing.gFeed</groupId>
<artifactId>gFeed-Suite</artifactId>
<version>1.0.5</version>
<version>1.0.6-SNAPSHOT</version>
<packaging>pom</packaging>
<name>gFeed Suite</name>
<description>gCat-Feeder Suite of components : service, plugin framework, plugins</description>
@ -18,6 +18,14 @@
<properties>
<distroDirectory>distro</distroDirectory>
<gitBaseUrl>https://code-repo.d4science.org/gCubeSystem</gitBaseUrl>
<!-- PROD -->
<!-- <smartgears-bom>2.1.1</smartgears-bom>-->
<!-- <gcube-bom>2.0.2</gcube-bom>-->
<!-- DEV -->
<smartgears-bom>2.2.0-SNAPSHOT</smartgears-bom>
<gcube-bom>2.1.0-SNAPSHOT</gcube-bom>
</properties>
<scm>
@ -43,19 +51,21 @@
<module>oai-harvester</module>
</modules>
<dependencyManagement>
<dependencies>
<dependency>
<groupId>org.gcube.distribution</groupId>
<artifactId>gcube-smartgears-bom</artifactId>
<version>2.1.1</version>
<version>${smartgears-bom}</version>
<type>pom</type>
<scope>import</scope>
</dependency>
<dependency>
<groupId>org.gcube.distribution</groupId>
<artifactId>gcube-bom</artifactId>
<version>2.0.2</version>
<version>${gcube-bom}</version>
<type>pom</type>
<scope>import</scope>
</dependency>

3
test-commons/CHANGELOG.md
View File

@ -2,6 +2,9 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm
# Changelog for org.gcube.data-publishing.gFeed.test-commons
## [v1.0.6-SNAPSHOT]
- Test support to jwt token
## [v1.0.5]
- Pom updates

11
test-commons/pom.xml
View File

@ -3,11 +3,11 @@
<parent>
<groupId>org.gcube.data-publishing.gFeed</groupId>
<artifactId>gFeed-Suite</artifactId>
<version>1.0.5</version>
<version>1.0.6-SNAPSHOT</version>
</parent>
<artifactId>test-commons</artifactId>
<name>test commons</name>
<version>1.0.5</version>
<version>1.0.6-SNAPSHOT</version>
<description>Utility classes for test purposes</description>
@ -26,7 +26,12 @@
<groupId>org.gcube.data-publishing.gFeed</groupId>
<artifactId>commons</artifactId>
</dependency>
<dependency>
<groupId>org.gcube.common</groupId>
<artifactId>authorization-utils</artifactId>
<version>[2.0.0, 3.0.0-SNAPSHOT)</version>
</dependency>
<dependency>

23
test-commons/src/main/java/org/gcube/data/publishing/gCatFeeder/tests/InfrastructureTests.java
View File

@ -6,6 +6,11 @@ import java.util.Map;
import java.util.Properties;
import org.gcube.common.authorization.library.provider.SecurityTokenProvider;
import org.gcube.common.authorization.utils.manager.SecretManager;
import org.gcube.common.authorization.utils.manager.SecretManagerProvider;
import org.gcube.common.authorization.utils.secret.GCubeSecret;
import org.gcube.common.authorization.utils.secret.JWTSecret;
import org.gcube.common.authorization.utils.secret.Secret;
import org.gcube.common.scope.api.ScopeProvider;
import org.gcube.data.publishing.gCatFeeder.model.EnvironmentConfiguration;
import org.gcube.data.publishing.gCatFeeder.utils.ISUtils;
@ -28,13 +33,27 @@ public class InfrastructureTests {
public static void setTestContext() {
if(isTestInfrastructureEnabled()) {
SecretManagerProvider.instance.set(new SecretManager());
Properties props=new Properties();
try{
props.load(BaseCollectorTest.class.getResourceAsStream("/tokens.properties"));
}catch(IOException e) {throw new RuntimeException(e);}
if(!props.containsKey(testContext)) throw new RuntimeException("No token found for scope : "+testContext);
SecurityTokenProvider.instance.set(props.getProperty(testContext));
ScopeProvider.instance.set(testContext);
String toSet = props.getProperty(testContext);
Secret secret = null;
if(toSet.length()>50)
secret = new JWTSecret(toSet); // se nuovo token
else
secret = new GCubeSecret(toSet); // se vecchio token
SecretManagerProvider.instance.get().addSecret(secret);
try{
SecretManagerProvider.instance.get().set();
}catch(Exception e ){throw new RuntimeException("Unable to set secret for context "+testContext,e);}
}
}