Update 'DataMinerAlgorithmsCrawler/src/test/java/org/gcube/data/publishing/gCatFeeder/collectors/dm/ItemUrlTests.java'

Removed commented explicit token

CHANGELOG.md

@@ -2,6 +2,9 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm
 
 # Changelog for gFeed
 
+## [v1.0.6]
+- Pom updates
+
 ## [v1.0.5]
 - Pom updates
 

DataMinerAlgorithmsCrawler/CHANGELOG.md

@@ -2,6 +2,8 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm
 
 # Changelog for org.gcube.data-publishing.gFeed.DataMinerAlgorithmsCrawler
 
+## [v1.0.7]
+- Lombok version
 
 ## [v1.0.6]
 - Pom updates

DataMinerAlgorithmsCrawler/pom.xml

@@ -4,11 +4,11 @@
 	<parent>
 		<groupId>org.gcube.data-publishing.gFeed</groupId>
     	<artifactId>gFeed-Suite</artifactId>
-		<version>1.0.5</version>
+		<version>1.0.6</version>
 	</parent>
 	<artifactId>DataMinerAlgorithmsCrawler</artifactId>
 	<name>DataMinerAlgorithmsCrawler</name>
-	<version>1.0.6</version>
+	<version>1.0.7</version>
 	<description>Plugin for gCat-Feeder for DataMiner Algorithms publishing</description>
   
 
@@ -45,7 +45,7 @@
 		<dependency>
 			<groupId>org.gcube.data.analysis</groupId>
 			<artifactId>data-miner-manager-cl</artifactId>
-			<version>[1.8.0,2.0.0)</version>
+			<version>[1.9.0,1.10.0-SNAPSHOT)</version>
 		</dependency>
 
 		
@@ -57,7 +57,7 @@
 		<dependency>
 			<groupId>org.gcube.data-catalogue</groupId>
 			<artifactId>gcat-client</artifactId>
-			<version>[2.0.0,3.0.0)</version>
+			<version>[2.0.0,3.0.0-SNAPSHOT)</version>
 		</dependency>
 
 		<dependency>
@@ -69,7 +69,6 @@
 		<dependency>
 			<groupId>org.projectlombok</groupId>
 			<artifactId>lombok</artifactId>
-			<version>1.18.20</version>
 			<scope>provided</scope>
 		</dependency>
 
@@ -116,5 +115,29 @@
 		</plugins>
 	</build>
 
-
+	<repositories>
+		<!-- copied from DM cl pom -->
+		<repository>
+			<id>n52-releases</id>
+			<name>52n Releases</name>
+			<url>https://52north.org/maven/repo/releases/</url>
+			<releases>
+				<enabled>true</enabled>
+			</releases>
+			<snapshots>
+				<enabled>false</enabled>
+			</snapshots>
+		</repository>
+		<repository>
+			<id>geotools</id>
+			<name>Geotools</name>
+			<url>https://repo.osgeo.org/repository/release/</url>
+			<releases>
+				<enabled>true</enabled>
+			</releases>
+			<snapshots>
+				<enabled>false</enabled>
+			</snapshots>
+		</repository>
+	</repositories>
 </project>

DataMinerAlgorithmsCrawler/src/test/java/org/gcube/data/publishing/gCatFeeder/collectors/dm/ItemUrlTests.java

@@ -11,7 +11,6 @@ public class ItemUrlTests extends BaseCollectorTest {
 
     @Test
     public void getItemURL() throws UriResolverMapException, IllegalArgumentException {
-        //SecurityTokenProvider.instance.set("***REMOVED***");
         TokenSetter.set("/pred4s/preprod/preVRE");
         String name = "fake";
         System.out.println(new URIResolver().getCatalogueItemURL(name));

catalogue-plugin-framework/CHANGELOG.md

@@ -2,6 +2,9 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm
 
 # Changelog for org.gcube.data-publishing.gFeed.catalogue-plugin-framework
 
+## [v1.0.6]
+- Lombok dependency upgrade
+
 
 ## [v1.0.5]
 - Pom updates

catalogue-plugin-framework/pom.xml

@@ -5,10 +5,10 @@
 	<parent>
 		<groupId>org.gcube.data-publishing.gFeed</groupId>
 		<artifactId>gFeed-Suite</artifactId>
-		<version>1.0.5</version>
+		<version>1.0.6</version>
 	</parent>
 	<artifactId>catalogue-plugin-framework</artifactId>
-	<version>1.0.5</version>
+	<version>1.0.6</version>
 
 
 	<scm>
@@ -20,7 +20,6 @@
 		<dependency>
 			<groupId>org.projectlombok</groupId>
 			<artifactId>lombok</artifactId>
-			<version>1.14.8</version>
 		</dependency>
 
 

collectors-plugin-framework/CHANGELOG.md

@@ -2,6 +2,9 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm
 
 # Changelog for org.gcube.data-publishing.gFeed.collectors-plugin-framework
 
+## [v1.0.6]
+- Pom updates
+
 ## [v1.0.5]
 - Pom updates
 

collectors-plugin-framework/pom.xml

@@ -5,12 +5,12 @@
 	<parent>
 		<groupId>org.gcube.data-publishing.gFeed</groupId>
 		<artifactId>gFeed-Suite</artifactId>
-		<version>1.0.5</version>
+		<version>1.0.6</version>
 	</parent>
 	<artifactId>collectors-plugin-framework</artifactId>
 	<name>collectors-plugin-framework</name>
 	<description>Framework for collectors implementations</description>
-	<version>1.0.5</version>
+	<version>1.0.6</version>
 
 
 
@@ -26,7 +26,6 @@
 		<dependency>
 			<groupId>org.projectlombok</groupId>
 			<artifactId>lombok</artifactId>
-			<version>1.14.8</version>
 		</dependency>
 
 

commons/CHANGELOG.md

@@ -2,6 +2,9 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm
 
 # Changelog for org.gcube.data-publishing.gFeed.test-commons
 
+## [v1.0.6]
+- Pom updates
+
 ## [v1.0.5]
 - Pom updates
 

commons/pom.xml

@@ -5,11 +5,11 @@
 	<parent>
 		<groupId>org.gcube.data-publishing.gFeed</groupId>
 		<artifactId>gFeed-Suite</artifactId>
-		<version>1.0.5</version>
+		<version>1.0.6</version>
 	</parent>
 	<artifactId>commons</artifactId>
 	<name>commons</name>
-	<version>1.0.5</version>
+	<version>1.0.6</version>
 	<description>Common utilities for gCatFeeder logic</description>
 
 

gCat-Controller/CHANGELOG.md

@@ -2,6 +2,9 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm
 
 # Changelog for org.gcube.data-publishing.gFeed.gCat-controller
 
+## [v1.0.8]
+- Pom updates
+
 ## [v1.0.7]
 - Pom updates
 

gCat-Controller/pom.xml

@@ -3,11 +3,11 @@
   <parent>
    <groupId>org.gcube.data-publishing.gFeed</groupId>
     <artifactId>gFeed-Suite</artifactId>
-    <version>1.0.5</version>
+    <version>1.0.6</version>
   </parent>
   <artifactId>gCat-Controller</artifactId>
   <name>gCat-Controller</name>
-  <version>1.0.7</version>
+  <version>1.0.8</version>
   <description>Controller implementation for GCat Service</description>
 
 	<scm>

gCat-Feeder/CHANGELOG.md

@@ -2,6 +2,9 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm
 
 # Changelog for org.gcube.data-publishing.gFeed.gFeed-service
 
+## [v1.0.6]
+- Pom updates
+
 ## [v1.0.5]
 - Pom updates
 

gCat-Feeder/pom.xml

@@ -4,11 +4,11 @@
 	 <parent>
     <groupId>org.gcube.data-publishing.gFeed</groupId>
     <artifactId>gFeed-Suite</artifactId>
-    <version>1.0.5</version>
+    <version>1.0.6</version>
   </parent>
 	<artifactId>gCat-Feeder</artifactId>
 	<name>gFeed Service</name>
-	<version>1.0.5</version>
+	<version>1.0.6</version>
 	<description>Service implementation</description>
 
 	<properties>
@@ -68,6 +68,10 @@
 			<artifactId>javax.ws.rs-api</artifactId>
 		</dependency>
 
+		<dependency>
+			<groupId>javax.xml.bind</groupId>
+			<artifactId>jaxb-api</artifactId>
+		</dependency>
 
 
 		<dependency>

oai-harvester/CHANGELOG.md

@@ -2,6 +2,13 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm
 
 # Changelog for org.gcube.data-publishing.gFeed.oai-pmh
 
+## [v1.0.8]
+Harvested Object profile update
+
+
+## [v1.0.7]
+- Pom updates
+
 ## [v1.0.6]
 - Pom updates
 

oai-harvester/pom.xml

@@ -5,11 +5,11 @@
 	<parent>
 		<groupId>org.gcube.data-publishing.gFeed</groupId>
 		<artifactId>gFeed-Suite</artifactId>
-		<version>1.0.5</version>
+		<version>1.0.6</version>
 	</parent>
 	<artifactId>oai-harvester</artifactId>
 	<name>oai-harvester</name>
-	<version>1.0.6</version>
+	<version>1.0.8</version>
 
 	<scm>
 		<connection>scm:git:${gitBaseUrl}/gFeed</connection>
@@ -40,6 +40,11 @@
 		</dependency>
 
 
+		<dependency>
+			<groupId>javax.xml.bind</groupId>
+			<artifactId>jaxb-api</artifactId>
+		</dependency>
+
 		<dependency>
 			<groupId>org.glassfish.jersey.media</groupId>
 			<artifactId>jersey-media-json-jackson</artifactId>

oai-harvester/src/main/java/org/gcube/data/publishing/gFeed/collectors/oai/OAIClient.java

@@ -83,7 +83,7 @@ public class OAIClient {
 			for(String set : specifiedSets) {
 				log.info("Loading "+metadataPrefix+" SET : "+set+" from "+baseUrl);
 				target.queryParam("set", set);
-				toReturn.addAll(call(target,metadataPrefix));
+				toReturn.addAll(call(target.queryParam("set", set),metadataPrefix));
 			}		
 		else {
 			log.info("Loading "+metadataPrefix+" from "+baseUrl);
@@ -98,6 +98,8 @@ public class OAIClient {
 	private List<OAIRecord> call(WebTarget target,String metadataPrefix){
 		ArrayList<OAIRecord> toReturn=new ArrayList<OAIRecord>();
 
+		log.info("Harvesting from resulting url {} ",target.getUri());
+
 		String resumptionToken=null;
 
 		// call & iterate
@@ -105,14 +107,18 @@ public class OAIClient {
 		int currentAttempt=1;
 		while(!isComplete) {
 			try {
-				
-				if(resumptionToken==null)
+
+				if(resumptionToken==null) {
 					target=target.queryParam("metadataPrefix",metadataPrefix);
-				else
+				}
+				else {
 					target=target.queryParam("resumptionToken", resumptionToken);
+				}
 
 
-			Response resp=target.request("application/xml").get();
+				log.trace("Calling {} ",target.getUri());
+
+				Response resp=target.request("application/xml").get();
 
 
 
@@ -123,11 +129,13 @@ public class OAIClient {
 			//No errors, thus reset attempt counter
 			currentAttempt=1;
 
-			toReturn.addAll(msg.getResponseRecords().getRecords());
+			if(msg.getResponseRecords().getRecords()!=null)
+				toReturn.addAll(msg.getResponseRecords().getRecords());
+			else log.info("NB {} didn't returned any record",msg.getRequest().getPath());
 			log.debug("Parsed "+toReturn.size()+" records so far.");
 
-				
-				Token t=msg.getResponseRecords().getResumptionToken();
+
+			Token t=msg.getResponseRecords().getResumptionToken();
 				log.debug("Obtained token : "+t);
 
 				if(t!=null && t.getId()!=null && !t.getId().isEmpty()) {
@@ -140,7 +148,7 @@ public class OAIClient {
 					isComplete=true;
 				}
 			}catch(Throwable t) {
-				log.warn("Unexpected ERROR	 "+t.getMessage());
+				log.warn("Unexpected ERROR	 ",t);
 				log.debug("Current attempt number = "+currentAttempt," max attempt Number = "+MAX_ATTEMPTS+", attempts delay factor =  ");
 				isComplete=currentAttempt>MAX_ATTEMPTS;
 				try {

oai-harvester/src/main/resources/profiles/HarvestedObject.xml

@@ -56,7 +56,7 @@
 		<mandatory>false</mandatory>
 		<dataType>String</dataType>
 		<maxOccurs>*</maxOccurs>
-		<tagging create="true" separator=" ">onValue</tagging>
+		<!-- <tagging create="true" separator=" ">onValue</tagging> -->
 	</metadatafield>
 	<metadatafield categoryref="Harvested Object">
 		<fieldName>relation</fieldName>
@@ -81,7 +81,7 @@
 		<mandatory>false</mandatory>
 		<dataType>String</dataType>
 		<maxOccurs>*</maxOccurs>
-		<tagging create="true" separator=" ">onValue</tagging>
+		<!-- <tagging create="true" separator=" ">onValue</tagging> -->
 	</metadatafield>
 	<metadatafield categoryref="Harvested Object">
 		<fieldName>title</fieldName>
@@ -94,6 +94,6 @@
 		<mandatory>false</mandatory>
 		<dataType>String</dataType>
 		<maxOccurs>*</maxOccurs>
-		<tagging create="true" separator=" ">onValue</tagging>
+		<!-- <tagging create="true" separator=" ">onValue</tagging> -->
 	</metadatafield>
 </metadataformat>

oai-harvester/src/test/java/org/gcube/application/gfeed/oai/TranslationTest.java

@@ -5,6 +5,7 @@ import java.util.Collection;
 import java.util.Set;
 
 import org.gcube.data.publishing.gCatFeeder.model.CatalogueFormatData;
+import org.gcube.data.publishing.gCatFeeder.model.EnvironmentConfiguration;
 import org.gcube.data.publishing.gCatFeeder.model.InternalConversionException;
 import org.gcube.data.publishing.gCatFeeder.tests.BaseCollectorTest;
 import org.gcube.data.publishing.gCatfeeder.collectors.CollectorPlugin;
@@ -31,18 +32,20 @@ public class TranslationTest extends BaseCollectorTest{
 		ObjectMapper mapper = new ObjectMapper();
 
 		CollectorPlugin plugin=new OAIHarvester();
-		plugin.setEnvironmentConfiguration(getEnvironmentConfiguration());
+		EnvironmentConfiguration env = getEnvironmentConfiguration();
+		System.out.println("Setting ENV : "+env);
+		plugin.setEnvironmentConfiguration(env);
 		DataCollector collector=plugin.getCollector();
 		Collection collected=collector.collect();
 		System.out.println("Found "+collected.size()+" elements");
 		for(Object obj:collected)
 			System.out.println(mapper.writeValueAsString(obj)+"\n");
 
-		for(String destinationcatalogue : (Set<String>)plugin.getSupportedCatalogueTypes()) {
-			DataTransformer<? extends CatalogueFormatData, OAIRecord> transformer=plugin.getTransformerByCatalogueType(destinationcatalogue);
-			for(Object data:transformer.transform(collected))
-				System.out.println(((CatalogueFormatData)data).toCatalogueFormat());
-		}
+//		for(String destinationcatalogue : (Set<String>)plugin.getSupportedCatalogueTypes()) {
+//			DataTransformer<? extends CatalogueFormatData, OAIRecord> transformer=plugin.getTransformerByCatalogueType(destinationcatalogue);
+//			for(Object data:transformer.transform(collected))
+//				System.out.println(((CatalogueFormatData)data).toCatalogueFormat());
+//		}
 	}
 	
 	

pom.xml

@@ -9,7 +9,7 @@
 	</parent>
 	<groupId>org.gcube.data-publishing.gFeed</groupId>
 	<artifactId>gFeed-Suite</artifactId>
-	<version>1.0.5</version>
+	<version>1.0.6</version>
 	<packaging>pom</packaging>
 	<name>gFeed Suite</name>
 	<description>gCat-Feeder Suite of components : service, plugin framework, plugins</description>
@@ -18,6 +18,15 @@
 	<properties>
 		<distroDirectory>distro</distroDirectory>
 		<gitBaseUrl>https://code-repo.d4science.org/gCubeSystem</gitBaseUrl>
+
+		<!-- PROD -->
+		<smartgears-bom>2.2.0</smartgears-bom>
+		<gcube-bom>2.1.0</gcube-bom>
+		<lombok.version>1.18.4</lombok.version>
+		<!-- DEV -->
+<!--		<smartgears-bom>2.2.0-SNAPSHOT</smartgears-bom>-->
+<!--		<gcube-bom>2.1.0-SNAPSHOT</gcube-bom>-->
+
 	</properties>
 
 	<scm>
@@ -43,24 +52,32 @@
 		<module>oai-harvester</module>
 	</modules>
 
+
+
 	<dependencyManagement>
 		<dependencies>
 			<dependency>
 				<groupId>org.gcube.distribution</groupId>
 				<artifactId>gcube-smartgears-bom</artifactId>
-				<version>2.1.1</version>
+				<version>${smartgears-bom}</version>
 				<type>pom</type>
 				<scope>import</scope>
 			</dependency>
 			<dependency>
 				<groupId>org.gcube.distribution</groupId>
 				<artifactId>gcube-bom</artifactId>
-				<version>2.0.2</version>
+				<version>${gcube-bom}</version>
 				<type>pom</type>
 				<scope>import</scope>
 			</dependency>
-			
-			
+
+
+			<dependency>
+				<groupId>org.projectlombok</groupId>
+				<artifactId>lombok</artifactId>
+				<version>${lombok.version}</version>
+			</dependency>
+
 			<dependency>
 				<groupId>org.gcube.data-publishing.gFeed</groupId>
 				<artifactId>collectors-plugin-framework</artifactId>

test-commons/CHANGELOG.md

@@ -2,6 +2,9 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm
 
 # Changelog for org.gcube.data-publishing.gFeed.test-commons
 
+## [v1.0.6]
+- Test support to jwt token
+
 ## [v1.0.5]
 - Pom updates
 

test-commons/pom.xml

@@ -3,11 +3,11 @@
   <parent>
     <groupId>org.gcube.data-publishing.gFeed</groupId>
     <artifactId>gFeed-Suite</artifactId>
-    <version>1.0.5</version>
+    <version>1.0.6</version>
   </parent>
   <artifactId>test-commons</artifactId>
   <name>test commons</name>
-  <version>1.0.5</version>
+  <version>1.0.6</version>
   <description>Utility classes for test purposes</description>
 
 
@@ -26,7 +26,12 @@
 			<groupId>org.gcube.data-publishing.gFeed</groupId>
 			<artifactId>commons</artifactId>
 		</dependency>
-		
+
+		 <dependency>
+			 <groupId>org.gcube.common</groupId>
+			 <artifactId>authorization-utils</artifactId>
+			 <version>[2.0.0, 3.0.0-SNAPSHOT)</version>
+		 </dependency>
 		
 
 		<dependency>

test-commons/src/main/java/org/gcube/data/publishing/gCatFeeder/tests/InfrastructureTests.java

@@ -6,6 +6,11 @@ import java.util.Map;
 import java.util.Properties;
 
 import org.gcube.common.authorization.library.provider.SecurityTokenProvider;
+import org.gcube.common.authorization.utils.manager.SecretManager;
+import org.gcube.common.authorization.utils.manager.SecretManagerProvider;
+import org.gcube.common.authorization.utils.secret.GCubeSecret;
+import org.gcube.common.authorization.utils.secret.JWTSecret;
+import org.gcube.common.authorization.utils.secret.Secret;
 import org.gcube.common.scope.api.ScopeProvider;
 import org.gcube.data.publishing.gCatFeeder.model.EnvironmentConfiguration;
 import org.gcube.data.publishing.gCatFeeder.utils.ISUtils;
@@ -28,13 +33,27 @@ public class InfrastructureTests {
 	public static void setTestContext() {
 
 		if(isTestInfrastructureEnabled()) {
+			SecretManagerProvider.instance.set(new SecretManager());
 			Properties props=new Properties();
 			try{
 				props.load(BaseCollectorTest.class.getResourceAsStream("/tokens.properties"));
 			}catch(IOException e) {throw new RuntimeException(e);}
 			if(!props.containsKey(testContext)) throw new RuntimeException("No token found for scope : "+testContext);
-			SecurityTokenProvider.instance.set(props.getProperty(testContext));
-			ScopeProvider.instance.set(testContext);
+
+			String toSet = props.getProperty(testContext);
+
+			Secret secret = null;
+			if(toSet.length()>50)
+				secret = new JWTSecret(toSet); // se nuovo token
+			else
+				secret = new GCubeSecret(toSet); // se vecchio token
+
+
+			SecretManagerProvider.instance.get().addSecret(secret);
+			try{
+				SecretManagerProvider.instance.get().set();
+			}catch(Exception e ){throw new RuntimeException("Unable to set secret for context "+testContext,e);}
+
 		}
 	}