Compare commits
19 Commits
Author | SHA1 | Date |
---|---|---|
Fabio Sinibaldi | e1e23c4d4d | |
Fabio Sinibaldi | de73defa53 | |
Fabio Sinibaldi | 794898ba99 | |
Fabio Sinibaldi | 87703c97c5 | |
Fabio Sinibaldi | d70fe5c9db | |
Fabio Sinibaldi | 519fdda147 | |
Fabio Sinibaldi | c6fe3b3815 | |
Fabio Sinibaldi | c791dd4b95 | |
Fabio Sinibaldi | 2b33bbe835 | |
Fabio Sinibaldi | 15eb85e856 | |
Fabio Sinibaldi | 15f0f5af5e | |
Fabio Sinibaldi | bb053681c8 | |
Fabio Sinibaldi | 10705e3b6d | |
Fabio Sinibaldi | de7eba0614 | |
Fabio Sinibaldi | 03e4d86b99 | |
Fabio Sinibaldi | 3e4fe33171 | |
Fabio Sinibaldi | 8d54d1622b | |
Fabio Sinibaldi | 2d4eb75d5f | |
Fabio Sinibaldi | d037515cb8 |
|
@ -2,6 +2,12 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm
|
||||||
|
|
||||||
# Changelog for gFeed
|
# Changelog for gFeed
|
||||||
|
|
||||||
|
## [v1.0.6]
|
||||||
|
- Pom updates
|
||||||
|
|
||||||
|
## [v1.0.5]
|
||||||
|
- Pom updates
|
||||||
|
|
||||||
## [v1.0.4] - 2020-12-15
|
## [v1.0.4] - 2020-12-15
|
||||||
- Dependency management
|
- Dependency management
|
||||||
- Fixed naming
|
- Fixed naming
|
||||||
|
|
|
@ -2,6 +2,12 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm
|
||||||
|
|
||||||
# Changelog for org.gcube.data-publishing.gFeed.DataMinerAlgorithmsCrawler
|
# Changelog for org.gcube.data-publishing.gFeed.DataMinerAlgorithmsCrawler
|
||||||
|
|
||||||
|
## [v1.0.7]
|
||||||
|
- Lombok version
|
||||||
|
|
||||||
|
## [v1.0.6]
|
||||||
|
- Pom updates
|
||||||
|
|
||||||
## [v1.0.5] - 2020-12-15
|
## [v1.0.5] - 2020-12-15
|
||||||
- Fixes [#22344](https://support.d4science.org/issues/22344#change-128440) : publish DM algorithms as Methods
|
- Fixes [#22344](https://support.d4science.org/issues/22344#change-128440) : publish DM algorithms as Methods
|
||||||
|
|
||||||
|
|
|
@ -4,16 +4,14 @@
|
||||||
<parent>
|
<parent>
|
||||||
<groupId>org.gcube.data-publishing.gFeed</groupId>
|
<groupId>org.gcube.data-publishing.gFeed</groupId>
|
||||||
<artifactId>gFeed-Suite</artifactId>
|
<artifactId>gFeed-Suite</artifactId>
|
||||||
<version>1.0.4</version>
|
<version>1.0.6</version>
|
||||||
</parent>
|
</parent>
|
||||||
<artifactId>DataMinerAlgorithmsCrawler</artifactId>
|
<artifactId>DataMinerAlgorithmsCrawler</artifactId>
|
||||||
<name>DataMinerAlgorithmsCrawler</name>
|
<name>DataMinerAlgorithmsCrawler</name>
|
||||||
<version>1.0.5</version>
|
<version>1.0.7</version>
|
||||||
<description>Plugin for gCat-Feeder for DataMiner Algorithms publishing</description>
|
<description>Plugin for gCat-Feeder for DataMiner Algorithms publishing</description>
|
||||||
|
|
||||||
<!-- <properties> -->
|
|
||||||
<!-- <gitBaseUrl>https://code-repo.d4science.org/gCubeSystem</gitBaseUrl> -->
|
|
||||||
<!-- </properties> -->
|
|
||||||
|
|
||||||
<scm>
|
<scm>
|
||||||
<connection>scm:git:${gitBaseUrl}/gFeed</connection>
|
<connection>scm:git:${gitBaseUrl}/gFeed</connection>
|
||||||
|
@ -47,7 +45,7 @@
|
||||||
<dependency>
|
<dependency>
|
||||||
<groupId>org.gcube.data.analysis</groupId>
|
<groupId>org.gcube.data.analysis</groupId>
|
||||||
<artifactId>data-miner-manager-cl</artifactId>
|
<artifactId>data-miner-manager-cl</artifactId>
|
||||||
<version>[1.8.0,2.0.0)</version>
|
<version>[1.9.0,1.10.0-SNAPSHOT)</version>
|
||||||
</dependency>
|
</dependency>
|
||||||
|
|
||||||
|
|
||||||
|
@ -59,7 +57,7 @@
|
||||||
<dependency>
|
<dependency>
|
||||||
<groupId>org.gcube.data-catalogue</groupId>
|
<groupId>org.gcube.data-catalogue</groupId>
|
||||||
<artifactId>gcat-client</artifactId>
|
<artifactId>gcat-client</artifactId>
|
||||||
<version>[2.0.0,3.0.0)</version>
|
<version>[2.0.0,3.0.0-SNAPSHOT)</version>
|
||||||
</dependency>
|
</dependency>
|
||||||
|
|
||||||
<dependency>
|
<dependency>
|
||||||
|
@ -71,7 +69,6 @@
|
||||||
<dependency>
|
<dependency>
|
||||||
<groupId>org.projectlombok</groupId>
|
<groupId>org.projectlombok</groupId>
|
||||||
<artifactId>lombok</artifactId>
|
<artifactId>lombok</artifactId>
|
||||||
<version>1.18.20</version>
|
|
||||||
<scope>provided</scope>
|
<scope>provided</scope>
|
||||||
</dependency>
|
</dependency>
|
||||||
|
|
||||||
|
@ -118,5 +115,29 @@
|
||||||
</plugins>
|
</plugins>
|
||||||
</build>
|
</build>
|
||||||
|
|
||||||
|
<repositories>
|
||||||
|
<!-- copied from DM cl pom -->
|
||||||
|
<repository>
|
||||||
|
<id>n52-releases</id>
|
||||||
|
<name>52n Releases</name>
|
||||||
|
<url>https://52north.org/maven/repo/releases/</url>
|
||||||
|
<releases>
|
||||||
|
<enabled>true</enabled>
|
||||||
|
</releases>
|
||||||
|
<snapshots>
|
||||||
|
<enabled>false</enabled>
|
||||||
|
</snapshots>
|
||||||
|
</repository>
|
||||||
|
<repository>
|
||||||
|
<id>geotools</id>
|
||||||
|
<name>Geotools</name>
|
||||||
|
<url>https://repo.osgeo.org/repository/release/</url>
|
||||||
|
<releases>
|
||||||
|
<enabled>true</enabled>
|
||||||
|
</releases>
|
||||||
|
<snapshots>
|
||||||
|
<enabled>false</enabled>
|
||||||
|
</snapshots>
|
||||||
|
</repository>
|
||||||
|
</repositories>
|
||||||
</project>
|
</project>
|
|
@ -11,8 +11,7 @@ public class ItemUrlTests extends BaseCollectorTest {
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void getItemURL() throws UriResolverMapException, IllegalArgumentException {
|
public void getItemURL() throws UriResolverMapException, IllegalArgumentException {
|
||||||
//SecurityTokenProvider.instance.set("***REMOVED***");
|
TokenSetter.set("/pred4s/preprod/preVRE");
|
||||||
TokenSetter.set("/gcube/devsec/devVRE");
|
|
||||||
String name = "fake";
|
String name = "fake";
|
||||||
System.out.println(new URIResolver().getCatalogueItemURL(name));
|
System.out.println(new URIResolver().getCatalogueItemURL(name));
|
||||||
}
|
}
|
||||||
|
|
|
@ -2,6 +2,14 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm
|
||||||
|
|
||||||
# Changelog for org.gcube.data-publishing.gFeed.catalogue-plugin-framework
|
# Changelog for org.gcube.data-publishing.gFeed.catalogue-plugin-framework
|
||||||
|
|
||||||
|
## [v1.0.6]
|
||||||
|
- Lombok dependency upgrade
|
||||||
|
|
||||||
|
|
||||||
|
## [v1.0.5]
|
||||||
|
- Pom updates
|
||||||
|
|
||||||
|
|
||||||
## [v1.0.4] - 2020-12-15
|
## [v1.0.4] - 2020-12-15
|
||||||
- Dependency management
|
- Dependency management
|
||||||
- Naming Convention
|
- Naming Convention
|
|
@ -5,10 +5,10 @@
|
||||||
<parent>
|
<parent>
|
||||||
<groupId>org.gcube.data-publishing.gFeed</groupId>
|
<groupId>org.gcube.data-publishing.gFeed</groupId>
|
||||||
<artifactId>gFeed-Suite</artifactId>
|
<artifactId>gFeed-Suite</artifactId>
|
||||||
<version>1.0.4</version>
|
<version>1.0.6</version>
|
||||||
</parent>
|
</parent>
|
||||||
<artifactId>catalogue-plugin-framework</artifactId>
|
<artifactId>catalogue-plugin-framework</artifactId>
|
||||||
<version>1.0.4</version>
|
<version>1.0.6</version>
|
||||||
|
|
||||||
|
|
||||||
<scm>
|
<scm>
|
||||||
|
@ -20,7 +20,6 @@
|
||||||
<dependency>
|
<dependency>
|
||||||
<groupId>org.projectlombok</groupId>
|
<groupId>org.projectlombok</groupId>
|
||||||
<artifactId>lombok</artifactId>
|
<artifactId>lombok</artifactId>
|
||||||
<version>1.14.8</version>
|
|
||||||
</dependency>
|
</dependency>
|
||||||
|
|
||||||
|
|
||||||
|
|
|
@ -2,6 +2,12 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm
|
||||||
|
|
||||||
# Changelog for org.gcube.data-publishing.gFeed.collectors-plugin-framework
|
# Changelog for org.gcube.data-publishing.gFeed.collectors-plugin-framework
|
||||||
|
|
||||||
|
## [v1.0.6]
|
||||||
|
- Pom updates
|
||||||
|
|
||||||
|
## [v1.0.5]
|
||||||
|
- Pom updates
|
||||||
|
|
||||||
## [v1.0.4] - 2020-12-15
|
## [v1.0.4] - 2020-12-15
|
||||||
- Dependency management
|
- Dependency management
|
||||||
- Naming Convention
|
- Naming Convention
|
|
@ -5,12 +5,12 @@
|
||||||
<parent>
|
<parent>
|
||||||
<groupId>org.gcube.data-publishing.gFeed</groupId>
|
<groupId>org.gcube.data-publishing.gFeed</groupId>
|
||||||
<artifactId>gFeed-Suite</artifactId>
|
<artifactId>gFeed-Suite</artifactId>
|
||||||
<version>1.0.4</version>
|
<version>1.0.6</version>
|
||||||
</parent>
|
</parent>
|
||||||
<artifactId>collectors-plugin-framework</artifactId>
|
<artifactId>collectors-plugin-framework</artifactId>
|
||||||
<name>collectors-plugin-framework</name>
|
<name>collectors-plugin-framework</name>
|
||||||
<description>Framework for collectors implementations</description>
|
<description>Framework for collectors implementations</description>
|
||||||
<version>1.0.4</version>
|
<version>1.0.6</version>
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
@ -26,7 +26,6 @@
|
||||||
<dependency>
|
<dependency>
|
||||||
<groupId>org.projectlombok</groupId>
|
<groupId>org.projectlombok</groupId>
|
||||||
<artifactId>lombok</artifactId>
|
<artifactId>lombok</artifactId>
|
||||||
<version>1.14.8</version>
|
|
||||||
</dependency>
|
</dependency>
|
||||||
|
|
||||||
|
|
||||||
|
|
|
@ -2,6 +2,12 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm
|
||||||
|
|
||||||
# Changelog for org.gcube.data-publishing.gFeed.test-commons
|
# Changelog for org.gcube.data-publishing.gFeed.test-commons
|
||||||
|
|
||||||
|
## [v1.0.6]
|
||||||
|
- Pom updates
|
||||||
|
|
||||||
|
## [v1.0.5]
|
||||||
|
- Pom updates
|
||||||
|
|
||||||
## [v1.0.4] - 2020-12-15
|
## [v1.0.4] - 2020-12-15
|
||||||
- Dependency management
|
- Dependency management
|
||||||
- Naming Convention
|
- Naming Convention
|
|
@ -5,11 +5,11 @@
|
||||||
<parent>
|
<parent>
|
||||||
<groupId>org.gcube.data-publishing.gFeed</groupId>
|
<groupId>org.gcube.data-publishing.gFeed</groupId>
|
||||||
<artifactId>gFeed-Suite</artifactId>
|
<artifactId>gFeed-Suite</artifactId>
|
||||||
<version>1.0.4</version>
|
<version>1.0.6</version>
|
||||||
</parent>
|
</parent>
|
||||||
<artifactId>commons</artifactId>
|
<artifactId>commons</artifactId>
|
||||||
<name>commons</name>
|
<name>commons</name>
|
||||||
<version>1.0.4</version>
|
<version>1.0.6</version>
|
||||||
<description>Common utilities for gCatFeeder logic</description>
|
<description>Common utilities for gCatFeeder logic</description>
|
||||||
|
|
||||||
|
|
||||||
|
|
|
@ -2,6 +2,12 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm
|
||||||
|
|
||||||
# Changelog for org.gcube.data-publishing.gFeed.gCat-controller
|
# Changelog for org.gcube.data-publishing.gFeed.gCat-controller
|
||||||
|
|
||||||
|
## [v1.0.8]
|
||||||
|
- Pom updates
|
||||||
|
|
||||||
|
## [v1.0.7]
|
||||||
|
- Pom updates
|
||||||
|
|
||||||
## [v1.0.6] - 2021-07-01
|
## [v1.0.6] - 2021-07-01
|
||||||
- gCat client coordinates
|
- gCat client coordinates
|
||||||
|
|
||||||
|
|
|
@ -3,15 +3,12 @@
|
||||||
<parent>
|
<parent>
|
||||||
<groupId>org.gcube.data-publishing.gFeed</groupId>
|
<groupId>org.gcube.data-publishing.gFeed</groupId>
|
||||||
<artifactId>gFeed-Suite</artifactId>
|
<artifactId>gFeed-Suite</artifactId>
|
||||||
<version>1.0.4</version>
|
<version>1.0.6</version>
|
||||||
</parent>
|
</parent>
|
||||||
<artifactId>gCat-Controller</artifactId>
|
<artifactId>gCat-Controller</artifactId>
|
||||||
<name>gCat-Controller</name>
|
<name>gCat-Controller</name>
|
||||||
<version>1.0.6</version>
|
<version>1.0.8</version>
|
||||||
<description>Controller implementation for GCat Service</description>
|
<description>Controller implementation for GCat Service</description>
|
||||||
<!-- <properties> -->
|
|
||||||
<!-- <gitBaseUrl>https://code-repo.d4science.org/gCubeSystem</gitBaseUrl> -->
|
|
||||||
<!-- </properties> -->
|
|
||||||
|
|
||||||
<scm>
|
<scm>
|
||||||
<connection>scm:git:${gitBaseUrl}/gFeed</connection>
|
<connection>scm:git:${gitBaseUrl}/gFeed</connection>
|
||||||
|
@ -49,7 +46,7 @@
|
||||||
<dependency>
|
<dependency>
|
||||||
<groupId>org.gcube.data-catalogue</groupId>
|
<groupId>org.gcube.data-catalogue</groupId>
|
||||||
<artifactId>gcat-client</artifactId>
|
<artifactId>gcat-client</artifactId>
|
||||||
<version>[2.0.0,3.0.0)</version>
|
<version>[2.2.0,3.0.0)</version>
|
||||||
</dependency>
|
</dependency>
|
||||||
|
|
||||||
|
|
||||||
|
|
|
@ -2,6 +2,12 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm
|
||||||
|
|
||||||
# Changelog for org.gcube.data-publishing.gFeed.gFeed-service
|
# Changelog for org.gcube.data-publishing.gFeed.gFeed-service
|
||||||
|
|
||||||
|
## [v1.0.6]
|
||||||
|
- Pom updates
|
||||||
|
|
||||||
|
## [v1.0.5]
|
||||||
|
- Pom updates
|
||||||
|
|
||||||
## [v1.0.4] - 2020-12-15
|
## [v1.0.4] - 2020-12-15
|
||||||
- Dependency management
|
- Dependency management
|
||||||
- Naming Convention
|
- Naming Convention
|
|
@ -1,20 +1,14 @@
|
||||||
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||||
<modelVersion>4.0.0</modelVersion>
|
<modelVersion>4.0.0</modelVersion>
|
||||||
<!-- <parent> -->
|
|
||||||
<!-- <groupId>org.gcube.tools</groupId> -->
|
|
||||||
<!-- <artifactId>maven-parent</artifactId> -->
|
|
||||||
<!-- <version>1.1.0</version> -->
|
|
||||||
<!-- </parent> -->
|
|
||||||
|
|
||||||
<parent>
|
<parent>
|
||||||
<groupId>org.gcube.data-publishing.gFeed</groupId>
|
<groupId>org.gcube.data-publishing.gFeed</groupId>
|
||||||
<artifactId>gFeed-Suite</artifactId>
|
<artifactId>gFeed-Suite</artifactId>
|
||||||
<version>1.0.4</version>
|
<version>1.0.6</version>
|
||||||
</parent>
|
</parent>
|
||||||
<artifactId>gCat-Feeder</artifactId>
|
<artifactId>gCat-Feeder</artifactId>
|
||||||
<name>gFeed Service</name>
|
<name>gFeed Service</name>
|
||||||
<version>1.0.4</version>
|
<version>1.0.6</version>
|
||||||
<description>Service implementation</description>
|
<description>Service implementation</description>
|
||||||
|
|
||||||
<properties>
|
<properties>
|
||||||
|
@ -74,6 +68,10 @@
|
||||||
<artifactId>javax.ws.rs-api</artifactId>
|
<artifactId>javax.ws.rs-api</artifactId>
|
||||||
</dependency>
|
</dependency>
|
||||||
|
|
||||||
|
<dependency>
|
||||||
|
<groupId>javax.xml.bind</groupId>
|
||||||
|
<artifactId>jaxb-api</artifactId>
|
||||||
|
</dependency>
|
||||||
|
|
||||||
|
|
||||||
<dependency>
|
<dependency>
|
||||||
|
|
|
@ -2,6 +2,16 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm
|
||||||
|
|
||||||
# Changelog for org.gcube.data-publishing.gFeed.oai-pmh
|
# Changelog for org.gcube.data-publishing.gFeed.oai-pmh
|
||||||
|
|
||||||
|
## [v1.0.8]
|
||||||
|
Harvested Object profile update
|
||||||
|
|
||||||
|
|
||||||
|
## [v1.0.7]
|
||||||
|
- Pom updates
|
||||||
|
|
||||||
|
## [v1.0.6]
|
||||||
|
- Pom updates
|
||||||
|
|
||||||
## [v1.0.5] - 2020-12-15
|
## [v1.0.5] - 2020-12-15
|
||||||
Do not stop on single repository error
|
Do not stop on single repository error
|
||||||
|
|
||||||
|
|
|
@ -5,11 +5,11 @@
|
||||||
<parent>
|
<parent>
|
||||||
<groupId>org.gcube.data-publishing.gFeed</groupId>
|
<groupId>org.gcube.data-publishing.gFeed</groupId>
|
||||||
<artifactId>gFeed-Suite</artifactId>
|
<artifactId>gFeed-Suite</artifactId>
|
||||||
<version>1.0.4</version>
|
<version>1.0.6</version>
|
||||||
</parent>
|
</parent>
|
||||||
<artifactId>oai-harvester</artifactId>
|
<artifactId>oai-harvester</artifactId>
|
||||||
<name>oai-harvester</name>
|
<name>oai-harvester</name>
|
||||||
<version>1.0.5</version>
|
<version>1.0.8</version>
|
||||||
|
|
||||||
<scm>
|
<scm>
|
||||||
<connection>scm:git:${gitBaseUrl}/gFeed</connection>
|
<connection>scm:git:${gitBaseUrl}/gFeed</connection>
|
||||||
|
@ -40,6 +40,11 @@
|
||||||
</dependency>
|
</dependency>
|
||||||
|
|
||||||
|
|
||||||
|
<dependency>
|
||||||
|
<groupId>javax.xml.bind</groupId>
|
||||||
|
<artifactId>jaxb-api</artifactId>
|
||||||
|
</dependency>
|
||||||
|
|
||||||
<dependency>
|
<dependency>
|
||||||
<groupId>org.glassfish.jersey.media</groupId>
|
<groupId>org.glassfish.jersey.media</groupId>
|
||||||
<artifactId>jersey-media-json-jackson</artifactId>
|
<artifactId>jersey-media-json-jackson</artifactId>
|
||||||
|
|
|
@ -83,7 +83,7 @@ public class OAIClient {
|
||||||
for(String set : specifiedSets) {
|
for(String set : specifiedSets) {
|
||||||
log.info("Loading "+metadataPrefix+" SET : "+set+" from "+baseUrl);
|
log.info("Loading "+metadataPrefix+" SET : "+set+" from "+baseUrl);
|
||||||
target.queryParam("set", set);
|
target.queryParam("set", set);
|
||||||
toReturn.addAll(call(target,metadataPrefix));
|
toReturn.addAll(call(target.queryParam("set", set),metadataPrefix));
|
||||||
}
|
}
|
||||||
else {
|
else {
|
||||||
log.info("Loading "+metadataPrefix+" from "+baseUrl);
|
log.info("Loading "+metadataPrefix+" from "+baseUrl);
|
||||||
|
@ -98,6 +98,8 @@ public class OAIClient {
|
||||||
private List<OAIRecord> call(WebTarget target,String metadataPrefix){
|
private List<OAIRecord> call(WebTarget target,String metadataPrefix){
|
||||||
ArrayList<OAIRecord> toReturn=new ArrayList<OAIRecord>();
|
ArrayList<OAIRecord> toReturn=new ArrayList<OAIRecord>();
|
||||||
|
|
||||||
|
log.info("Harvesting from resulting url {} ",target.getUri());
|
||||||
|
|
||||||
String resumptionToken=null;
|
String resumptionToken=null;
|
||||||
|
|
||||||
// call & iterate
|
// call & iterate
|
||||||
|
@ -106,12 +108,16 @@ public class OAIClient {
|
||||||
while(!isComplete) {
|
while(!isComplete) {
|
||||||
try {
|
try {
|
||||||
|
|
||||||
if(resumptionToken==null)
|
if(resumptionToken==null) {
|
||||||
target=target.queryParam("metadataPrefix",metadataPrefix);
|
target=target.queryParam("metadataPrefix",metadataPrefix);
|
||||||
else
|
}
|
||||||
|
else {
|
||||||
target=target.queryParam("resumptionToken", resumptionToken);
|
target=target.queryParam("resumptionToken", resumptionToken);
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
log.trace("Calling {} ",target.getUri());
|
||||||
|
|
||||||
Response resp=target.request("application/xml").get();
|
Response resp=target.request("application/xml").get();
|
||||||
|
|
||||||
|
|
||||||
|
@ -123,7 +129,9 @@ public class OAIClient {
|
||||||
//No errors, thus reset attempt counter
|
//No errors, thus reset attempt counter
|
||||||
currentAttempt=1;
|
currentAttempt=1;
|
||||||
|
|
||||||
|
if(msg.getResponseRecords().getRecords()!=null)
|
||||||
toReturn.addAll(msg.getResponseRecords().getRecords());
|
toReturn.addAll(msg.getResponseRecords().getRecords());
|
||||||
|
else log.info("NB {} didn't returned any record",msg.getRequest().getPath());
|
||||||
log.debug("Parsed "+toReturn.size()+" records so far.");
|
log.debug("Parsed "+toReturn.size()+" records so far.");
|
||||||
|
|
||||||
|
|
||||||
|
@ -140,7 +148,7 @@ public class OAIClient {
|
||||||
isComplete=true;
|
isComplete=true;
|
||||||
}
|
}
|
||||||
}catch(Throwable t) {
|
}catch(Throwable t) {
|
||||||
log.warn("Unexpected ERROR "+t.getMessage());
|
log.warn("Unexpected ERROR ",t);
|
||||||
log.debug("Current attempt number = "+currentAttempt," max attempt Number = "+MAX_ATTEMPTS+", attempts delay factor = ");
|
log.debug("Current attempt number = "+currentAttempt," max attempt Number = "+MAX_ATTEMPTS+", attempts delay factor = ");
|
||||||
isComplete=currentAttempt>MAX_ATTEMPTS;
|
isComplete=currentAttempt>MAX_ATTEMPTS;
|
||||||
try {
|
try {
|
||||||
|
|
|
@ -56,7 +56,7 @@
|
||||||
<mandatory>false</mandatory>
|
<mandatory>false</mandatory>
|
||||||
<dataType>String</dataType>
|
<dataType>String</dataType>
|
||||||
<maxOccurs>*</maxOccurs>
|
<maxOccurs>*</maxOccurs>
|
||||||
<tagging create="true" separator=" ">onValue</tagging>
|
<!-- <tagging create="true" separator=" ">onValue</tagging> -->
|
||||||
</metadatafield>
|
</metadatafield>
|
||||||
<metadatafield categoryref="Harvested Object">
|
<metadatafield categoryref="Harvested Object">
|
||||||
<fieldName>relation</fieldName>
|
<fieldName>relation</fieldName>
|
||||||
|
@ -81,7 +81,7 @@
|
||||||
<mandatory>false</mandatory>
|
<mandatory>false</mandatory>
|
||||||
<dataType>String</dataType>
|
<dataType>String</dataType>
|
||||||
<maxOccurs>*</maxOccurs>
|
<maxOccurs>*</maxOccurs>
|
||||||
<tagging create="true" separator=" ">onValue</tagging>
|
<!-- <tagging create="true" separator=" ">onValue</tagging> -->
|
||||||
</metadatafield>
|
</metadatafield>
|
||||||
<metadatafield categoryref="Harvested Object">
|
<metadatafield categoryref="Harvested Object">
|
||||||
<fieldName>title</fieldName>
|
<fieldName>title</fieldName>
|
||||||
|
@ -94,6 +94,6 @@
|
||||||
<mandatory>false</mandatory>
|
<mandatory>false</mandatory>
|
||||||
<dataType>String</dataType>
|
<dataType>String</dataType>
|
||||||
<maxOccurs>*</maxOccurs>
|
<maxOccurs>*</maxOccurs>
|
||||||
<tagging create="true" separator=" ">onValue</tagging>
|
<!-- <tagging create="true" separator=" ">onValue</tagging> -->
|
||||||
</metadatafield>
|
</metadatafield>
|
||||||
</metadataformat>
|
</metadataformat>
|
|
@ -5,6 +5,7 @@ import java.util.Collection;
|
||||||
import java.util.Set;
|
import java.util.Set;
|
||||||
|
|
||||||
import org.gcube.data.publishing.gCatFeeder.model.CatalogueFormatData;
|
import org.gcube.data.publishing.gCatFeeder.model.CatalogueFormatData;
|
||||||
|
import org.gcube.data.publishing.gCatFeeder.model.EnvironmentConfiguration;
|
||||||
import org.gcube.data.publishing.gCatFeeder.model.InternalConversionException;
|
import org.gcube.data.publishing.gCatFeeder.model.InternalConversionException;
|
||||||
import org.gcube.data.publishing.gCatFeeder.tests.BaseCollectorTest;
|
import org.gcube.data.publishing.gCatFeeder.tests.BaseCollectorTest;
|
||||||
import org.gcube.data.publishing.gCatfeeder.collectors.CollectorPlugin;
|
import org.gcube.data.publishing.gCatfeeder.collectors.CollectorPlugin;
|
||||||
|
@ -31,18 +32,20 @@ public class TranslationTest extends BaseCollectorTest{
|
||||||
ObjectMapper mapper = new ObjectMapper();
|
ObjectMapper mapper = new ObjectMapper();
|
||||||
|
|
||||||
CollectorPlugin plugin=new OAIHarvester();
|
CollectorPlugin plugin=new OAIHarvester();
|
||||||
plugin.setEnvironmentConfiguration(getEnvironmentConfiguration());
|
EnvironmentConfiguration env = getEnvironmentConfiguration();
|
||||||
|
System.out.println("Setting ENV : "+env);
|
||||||
|
plugin.setEnvironmentConfiguration(env);
|
||||||
DataCollector collector=plugin.getCollector();
|
DataCollector collector=plugin.getCollector();
|
||||||
Collection collected=collector.collect();
|
Collection collected=collector.collect();
|
||||||
System.out.println("Found "+collected.size()+" elements");
|
System.out.println("Found "+collected.size()+" elements");
|
||||||
for(Object obj:collected)
|
for(Object obj:collected)
|
||||||
System.out.println(mapper.writeValueAsString(obj)+"\n");
|
System.out.println(mapper.writeValueAsString(obj)+"\n");
|
||||||
|
|
||||||
for(String destinationcatalogue : (Set<String>)plugin.getSupportedCatalogueTypes()) {
|
// for(String destinationcatalogue : (Set<String>)plugin.getSupportedCatalogueTypes()) {
|
||||||
DataTransformer<? extends CatalogueFormatData, OAIRecord> transformer=plugin.getTransformerByCatalogueType(destinationcatalogue);
|
// DataTransformer<? extends CatalogueFormatData, OAIRecord> transformer=plugin.getTransformerByCatalogueType(destinationcatalogue);
|
||||||
for(Object data:transformer.transform(collected))
|
// for(Object data:transformer.transform(collected))
|
||||||
System.out.println(((CatalogueFormatData)data).toCatalogueFormat());
|
// System.out.println(((CatalogueFormatData)data).toCatalogueFormat());
|
||||||
}
|
// }
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
23
pom.xml
23
pom.xml
|
@ -9,7 +9,7 @@
|
||||||
</parent>
|
</parent>
|
||||||
<groupId>org.gcube.data-publishing.gFeed</groupId>
|
<groupId>org.gcube.data-publishing.gFeed</groupId>
|
||||||
<artifactId>gFeed-Suite</artifactId>
|
<artifactId>gFeed-Suite</artifactId>
|
||||||
<version>1.0.4</version>
|
<version>1.0.6</version>
|
||||||
<packaging>pom</packaging>
|
<packaging>pom</packaging>
|
||||||
<name>gFeed Suite</name>
|
<name>gFeed Suite</name>
|
||||||
<description>gCat-Feeder Suite of components : service, plugin framework, plugins</description>
|
<description>gCat-Feeder Suite of components : service, plugin framework, plugins</description>
|
||||||
|
@ -18,6 +18,15 @@
|
||||||
<properties>
|
<properties>
|
||||||
<distroDirectory>distro</distroDirectory>
|
<distroDirectory>distro</distroDirectory>
|
||||||
<gitBaseUrl>https://code-repo.d4science.org/gCubeSystem</gitBaseUrl>
|
<gitBaseUrl>https://code-repo.d4science.org/gCubeSystem</gitBaseUrl>
|
||||||
|
|
||||||
|
<!-- PROD -->
|
||||||
|
<smartgears-bom>2.2.0</smartgears-bom>
|
||||||
|
<gcube-bom>2.1.0</gcube-bom>
|
||||||
|
<lombok.version>1.18.4</lombok.version>
|
||||||
|
<!-- DEV -->
|
||||||
|
<!-- <smartgears-bom>2.2.0-SNAPSHOT</smartgears-bom>-->
|
||||||
|
<!-- <gcube-bom>2.1.0-SNAPSHOT</gcube-bom>-->
|
||||||
|
|
||||||
</properties>
|
</properties>
|
||||||
|
|
||||||
<scm>
|
<scm>
|
||||||
|
@ -43,24 +52,32 @@
|
||||||
<module>oai-harvester</module>
|
<module>oai-harvester</module>
|
||||||
</modules>
|
</modules>
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
<dependencyManagement>
|
<dependencyManagement>
|
||||||
<dependencies>
|
<dependencies>
|
||||||
<dependency>
|
<dependency>
|
||||||
<groupId>org.gcube.distribution</groupId>
|
<groupId>org.gcube.distribution</groupId>
|
||||||
<artifactId>gcube-smartgears-bom</artifactId>
|
<artifactId>gcube-smartgears-bom</artifactId>
|
||||||
<version>2.0.0</version>
|
<version>${smartgears-bom}</version>
|
||||||
<type>pom</type>
|
<type>pom</type>
|
||||||
<scope>import</scope>
|
<scope>import</scope>
|
||||||
</dependency>
|
</dependency>
|
||||||
<dependency>
|
<dependency>
|
||||||
<groupId>org.gcube.distribution</groupId>
|
<groupId>org.gcube.distribution</groupId>
|
||||||
<artifactId>gcube-bom</artifactId>
|
<artifactId>gcube-bom</artifactId>
|
||||||
<version>2.0.0</version>
|
<version>${gcube-bom}</version>
|
||||||
<type>pom</type>
|
<type>pom</type>
|
||||||
<scope>import</scope>
|
<scope>import</scope>
|
||||||
</dependency>
|
</dependency>
|
||||||
|
|
||||||
|
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.projectlombok</groupId>
|
||||||
|
<artifactId>lombok</artifactId>
|
||||||
|
<version>${lombok.version}</version>
|
||||||
|
</dependency>
|
||||||
|
|
||||||
<dependency>
|
<dependency>
|
||||||
<groupId>org.gcube.data-publishing.gFeed</groupId>
|
<groupId>org.gcube.data-publishing.gFeed</groupId>
|
||||||
<artifactId>collectors-plugin-framework</artifactId>
|
<artifactId>collectors-plugin-framework</artifactId>
|
||||||
|
|
|
@ -2,6 +2,12 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm
|
||||||
|
|
||||||
# Changelog for org.gcube.data-publishing.gFeed.test-commons
|
# Changelog for org.gcube.data-publishing.gFeed.test-commons
|
||||||
|
|
||||||
|
## [v1.0.6]
|
||||||
|
- Test support to jwt token
|
||||||
|
|
||||||
|
## [v1.0.5]
|
||||||
|
- Pom updates
|
||||||
|
|
||||||
## [v1.0.4] - 2020-12-15
|
## [v1.0.4] - 2020-12-15
|
||||||
- Dependency management
|
- Dependency management
|
||||||
- Naming Convention
|
- Naming Convention
|
|
@ -3,11 +3,11 @@
|
||||||
<parent>
|
<parent>
|
||||||
<groupId>org.gcube.data-publishing.gFeed</groupId>
|
<groupId>org.gcube.data-publishing.gFeed</groupId>
|
||||||
<artifactId>gFeed-Suite</artifactId>
|
<artifactId>gFeed-Suite</artifactId>
|
||||||
<version>1.0.4</version>
|
<version>1.0.6</version>
|
||||||
</parent>
|
</parent>
|
||||||
<artifactId>test-commons</artifactId>
|
<artifactId>test-commons</artifactId>
|
||||||
<name>test commons</name>
|
<name>test commons</name>
|
||||||
<version>1.0.4</version>
|
<version>1.0.6</version>
|
||||||
<description>Utility classes for test purposes</description>
|
<description>Utility classes for test purposes</description>
|
||||||
|
|
||||||
|
|
||||||
|
@ -27,6 +27,11 @@
|
||||||
<artifactId>commons</artifactId>
|
<artifactId>commons</artifactId>
|
||||||
</dependency>
|
</dependency>
|
||||||
|
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.gcube.common</groupId>
|
||||||
|
<artifactId>authorization-utils</artifactId>
|
||||||
|
<version>[2.0.0, 3.0.0-SNAPSHOT)</version>
|
||||||
|
</dependency>
|
||||||
|
|
||||||
|
|
||||||
<dependency>
|
<dependency>
|
||||||
|
|
|
@ -6,6 +6,11 @@ import java.util.Map;
|
||||||
import java.util.Properties;
|
import java.util.Properties;
|
||||||
|
|
||||||
import org.gcube.common.authorization.library.provider.SecurityTokenProvider;
|
import org.gcube.common.authorization.library.provider.SecurityTokenProvider;
|
||||||
|
import org.gcube.common.authorization.utils.manager.SecretManager;
|
||||||
|
import org.gcube.common.authorization.utils.manager.SecretManagerProvider;
|
||||||
|
import org.gcube.common.authorization.utils.secret.GCubeSecret;
|
||||||
|
import org.gcube.common.authorization.utils.secret.JWTSecret;
|
||||||
|
import org.gcube.common.authorization.utils.secret.Secret;
|
||||||
import org.gcube.common.scope.api.ScopeProvider;
|
import org.gcube.common.scope.api.ScopeProvider;
|
||||||
import org.gcube.data.publishing.gCatFeeder.model.EnvironmentConfiguration;
|
import org.gcube.data.publishing.gCatFeeder.model.EnvironmentConfiguration;
|
||||||
import org.gcube.data.publishing.gCatFeeder.utils.ISUtils;
|
import org.gcube.data.publishing.gCatFeeder.utils.ISUtils;
|
||||||
|
@ -28,13 +33,27 @@ public class InfrastructureTests {
|
||||||
public static void setTestContext() {
|
public static void setTestContext() {
|
||||||
|
|
||||||
if(isTestInfrastructureEnabled()) {
|
if(isTestInfrastructureEnabled()) {
|
||||||
|
SecretManagerProvider.instance.set(new SecretManager());
|
||||||
Properties props=new Properties();
|
Properties props=new Properties();
|
||||||
try{
|
try{
|
||||||
props.load(BaseCollectorTest.class.getResourceAsStream("/tokens.properties"));
|
props.load(BaseCollectorTest.class.getResourceAsStream("/tokens.properties"));
|
||||||
}catch(IOException e) {throw new RuntimeException(e);}
|
}catch(IOException e) {throw new RuntimeException(e);}
|
||||||
if(!props.containsKey(testContext)) throw new RuntimeException("No token found for scope : "+testContext);
|
if(!props.containsKey(testContext)) throw new RuntimeException("No token found for scope : "+testContext);
|
||||||
SecurityTokenProvider.instance.set(props.getProperty(testContext));
|
|
||||||
ScopeProvider.instance.set(testContext);
|
String toSet = props.getProperty(testContext);
|
||||||
|
|
||||||
|
Secret secret = null;
|
||||||
|
if(toSet.length()>50)
|
||||||
|
secret = new JWTSecret(toSet); // se nuovo token
|
||||||
|
else
|
||||||
|
secret = new GCubeSecret(toSet); // se vecchio token
|
||||||
|
|
||||||
|
|
||||||
|
SecretManagerProvider.instance.get().addSecret(secret);
|
||||||
|
try{
|
||||||
|
SecretManagerProvider.instance.get().set();
|
||||||
|
}catch(Exception e ){throw new RuntimeException("Unable to set secret for context "+testContext,e);}
|
||||||
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
Loading…
Reference in New Issue