Update 'DataMinerAlgorithmsCrawler/src/test/java/org/gcube/data/publishing/gCatFeeder/collectors/dm/ItemUrlTests.java'

Removed commented explicit token

CHANGELOG.md

@@ -2,6 +2,12 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm
 
 # Changelog for gFeed
 
+## [v1.0.6]
+- Pom updates
+
+## [v1.0.5]
+- Pom updates
+
 ## [v1.0.4] - 2020-12-15
 - Dependency management
 - Fixed naming

DataMinerAlgorithmsCrawler/CHANGELOG.md

@@ -2,6 +2,15 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm
 
 # Changelog for org.gcube.data-publishing.gFeed.DataMinerAlgorithmsCrawler
 
+## [v1.0.7]
+- Lombok version
+
+## [v1.0.6]
+- Pom updates
+
+## [v1.0.5] - 2020-12-15
+- Fixes [#22344](https://support.d4science.org/issues/22344#change-128440) : publish DM algorithms as Methods
+
 ## [v1.0.4] - 2020-12-15
 - Dependency management
 - Naming Convention

DataMinerAlgorithmsCrawler/pom.xml

@@ -4,16 +4,14 @@
 	<parent>
 		<groupId>org.gcube.data-publishing.gFeed</groupId>
     	<artifactId>gFeed-Suite</artifactId>
-		<version>1.0.4</version>
+		<version>1.0.6</version>
 	</parent>
 	<artifactId>DataMinerAlgorithmsCrawler</artifactId>
 	<name>DataMinerAlgorithmsCrawler</name>
-	<version>1.0.4</version>
+	<version>1.0.7</version>
 	<description>Plugin for gCat-Feeder for DataMiner Algorithms publishing</description>
   
-<!--    	<properties>		 -->
-<!-- 		<gitBaseUrl>https://code-repo.d4science.org/gCubeSystem</gitBaseUrl> -->
-<!-- 	</properties> -->
+
 
 	<scm>
 		<connection>scm:git:${gitBaseUrl}/gFeed</connection>
@@ -47,7 +45,7 @@
 		<dependency>
 			<groupId>org.gcube.data.analysis</groupId>
 			<artifactId>data-miner-manager-cl</artifactId>
-			<version>[1.6.0,2.0.0)</version>
+			<version>[1.9.0,1.10.0-SNAPSHOT)</version>
 		</dependency>
 
 		
@@ -56,7 +54,23 @@
 			<artifactId>jersey-media-json-jackson</artifactId>
 		</dependency>
 
+		<dependency>
+			<groupId>org.gcube.data-catalogue</groupId>
+			<artifactId>gcat-client</artifactId>
+			<version>[2.0.0,3.0.0-SNAPSHOT)</version>
+		</dependency>
 
+		<dependency>
+			<groupId>org.gcube.portlets.user</groupId>
+			<artifactId>uri-resolver-manager</artifactId>
+			<version>[1.5.0-SNAPSHOT,2.0.0)</version>
+		</dependency>
+
+		<dependency>
+			<groupId>org.projectlombok</groupId>
+			<artifactId>lombok</artifactId>
+			<scope>provided</scope>
+		</dependency>
 
 		<dependency>
 			<groupId>org.slf4j</groupId>
@@ -69,6 +83,11 @@
 			<scope>test</scope>
 		</dependency>
 
+		<dependency>
+			<groupId>org.gcube.common</groupId>
+			<artifactId>authorization-client</artifactId>
+			<scope>compile</scope>
+		</dependency>
 
 
 
@@ -96,5 +115,29 @@
 		</plugins>
 	</build>
 
-
+	<repositories>
+		<!-- copied from DM cl pom -->
+		<repository>
+			<id>n52-releases</id>
+			<name>52n Releases</name>
+			<url>https://52north.org/maven/repo/releases/</url>
+			<releases>
+				<enabled>true</enabled>
+			</releases>
+			<snapshots>
+				<enabled>false</enabled>
+			</snapshots>
+		</repository>
+		<repository>
+			<id>geotools</id>
+			<name>Geotools</name>
+			<url>https://repo.osgeo.org/repository/release/</url>
+			<releases>
+				<enabled>true</enabled>
+			</releases>
+			<snapshots>
+				<enabled>false</enabled>
+			</snapshots>
+		</repository>
+	</repositories>
 </project>

DataMinerAlgorithmsCrawler/src/main/java/org/gcube/data/publishing/gCatFeeder/collectors/dm/Constants.java

@@ -11,7 +11,8 @@ public class Constants {
 	
 	// ENVIRONMENT EXPECTED PARAMETERS
 	public static final String GUI_BASE_URL=ENVIRONMENT_PROPERTIES_BASE+"GUI_BASE_URL";
-	
+	public static final String GATEWAY_NAME=ENVIRONMENT_PROPERTIES_BASE+"GATEWAY_NAME";
+
 	public static final String DEFAULT_AUTHOR=ENVIRONMENT_PROPERTIES_BASE+"DEFAULT_AUTHOR";
 	
 	public static final String DEFAULT_MAINTAINER=ENVIRONMENT_PROPERTIES_BASE+"DEFAULT_MAINTAINER";

DataMinerAlgorithmsCrawler/src/main/java/org/gcube/data/publishing/gCatFeeder/collectors/dm/DMAlgorithmsInfoCollector.java

@@ -1,14 +1,18 @@
 package org.gcube.data.publishing.gCatFeeder.collectors.dm;
 
-import java.util.Collections;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
+import java.time.LocalDateTime;
+import java.time.Month;
+import java.time.ZoneId;
+import java.time.chrono.IsoChronology;
+import java.time.format.DateTimeFormatter;
+import java.time.format.DateTimeFormatterBuilder;
+import java.time.format.ResolverStyle;
+import java.time.temporal.ChronoField;
+import java.util.*;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
-import org.gcube.common.resources.gcore.ServiceEndpoint;
+
 import org.gcube.data.analysis.dataminermanagercl.server.DataMinerService;
 import org.gcube.data.analysis.dataminermanagercl.server.dmservice.SClient;
 import org.gcube.data.analysis.dataminermanagercl.shared.process.Operator;
@@ -23,16 +27,49 @@ import org.gcube.data.publishing.gCatfeeder.collectors.DataCollector;
 import org.gcube.data.publishing.gCatfeeder.collectors.model.faults.CollectorFault;
 
 import lombok.extern.slf4j.Slf4j;
+import org.gcube.portlets.user.uriresolvermanager.exception.IllegalArgumentException;
+import org.gcube.portlets.user.uriresolvermanager.exception.UriResolverMapException;
+
 
 @Slf4j
 public class DMAlgorithmsInfoCollector implements DataCollector<InternalAlgorithmDescriptor> {
 
-	private static final Pattern p = Pattern.compile("\\{Published by (.*)\\((.*(\\.)?.*\\)).*\\}$");
+	/*
+	 * i.e. "Basic statistic max min average {Published by Giancarlo Panichi (giancarlo.panichi) on 2018/07/20 10:24 GMT}"
+	 */
+
+	private static final Pattern descriptionPattern = Pattern.compile("\\{Published by (.*)\\((.*(\\.)?.*\\)) on (.*)\\}$");
+
+
+	static final DateTimeFormatter versionDateParser= DateTimeFormatter.ofPattern("uuuu/MM/dd HH:mm 'GMT'[Z]")
+			.withResolverStyle(ResolverStyle.STRICT)
+			.withLocale(Locale.getDefault())
+			.withZone(ZoneId.systemDefault());
+
+/*	static final DateTimeFormatter versionDateParser=new DateTimeFormatterBuilder()
+			.parseCaseInsensitive().parseLenient()
+			.appendValue(ChronoField.YEAR,4)
+			.appendLiteral('/')
+			.appendValue(ChronoField.MONTH_OF_YEAR,2)
+			.appendLiteral('/')
+			.appendValue(ChronoField.DAY_OF_MONTH,2)
+			.appendLiteral(' ')
+			.appendValue(ChronoField.HOUR_OF_DAY,2)
+			.appendLiteral(':')
+			.appendValue(ChronoField.MINUTE_OF_HOUR,2)
+			.appendLiteral(' ')
+			.appendOffset("+HHMM", "GMT")
+			.toFormatter().withChronology(IsoChronology.INSTANCE)
+			.withResolverStyle(ResolverStyle.SMART); */
+
+
 
-	
 	private Map<String,String> env=null;
-	
-	
+
+
+
+
+
 	public void setEnvironmentConfiguration(EnvironmentConfiguration envConfig) {
 		if(envConfig!=null) {
 			log.debug("Current Environment Configuration is : "+envConfig.getCurrentConfiguration());
@@ -74,7 +111,7 @@ public class DMAlgorithmsInfoCollector implements DataCollector<InternalAlgorith
 					String categoryID=cat.getId();
 					String categoryName=cat.getName();
 
-					
+
 					desc.setCategoryBriefDescription(categoryBriefDescription);
 					desc.setCategoryID(categoryID);
 					desc.setCategoryName(categoryName);
@@ -83,7 +120,7 @@ public class DMAlgorithmsInfoCollector implements DataCollector<InternalAlgorith
 					
 					
 					// Operator info
-
+					
 					String opBriefDescription=op.getBriefDescription();
 					String opDescription=op.getDescription();
 					String opID=op.getId();
@@ -99,7 +136,20 @@ public class DMAlgorithmsInfoCollector implements DataCollector<InternalAlgorith
 					desc.setAuthor(parseUser(getAuthor(opDescription)));
 					desc.setMaintainer(parseUser(getAuthor(opDescription)));
 					
-					
+
+					desc.setVersion(parseDescriptionForDate(opDescription));
+
+					// Try to use version as creation time
+					LocalDateTime toSetCreationDate=null;
+					try{
+						toSetCreationDate=LocalDateTime.parse(desc.getVersion(),versionDateParser);
+					}catch(Throwable t){
+						log.debug("Version {} is not a date. Using 1900-01-01 as Creation date..",desc.getVersion());
+						toSetCreationDate = LocalDateTime.of(1900, Month.JANUARY,1,0,0);
+					}
+
+					desc.setCreationDate(toSetCreationDate);
+
 					
 					// Parameters info
 					for(org.gcube.data.analysis.dataminermanagercl.shared.parameters.Parameter param:client.getInputParameters(op)) {
@@ -132,11 +182,17 @@ public class DMAlgorithmsInfoCollector implements DataCollector<InternalAlgorith
 					if(guiBasePath!=null)
 						desc.setGuiLink(getGUIBasePath()+"?"+
 							DataMinerCollectorProperties.getProperty(DataMinerCollectorProperties.GUI_PARAM_NAME)+"="+opID);
-					
+
+
+					desc.setGatewayName(env.get(Constants.GATEWAY_NAME));
 					
 					if(wpsbaseUrl!=null) {
 						desc.setWpsLink(wpsbaseUrl+"?Request=DescribeProcess&Version=1.0.0&Service=WPS"+"&Identifier="+opID);
 					}
+
+
+
+
 					desc.setPrivateFlag(Boolean.parseBoolean(env.get(Constants.PRIVATE)));
 					
 					toReturn.add(desc);
@@ -164,7 +220,7 @@ public class DMAlgorithmsInfoCollector implements DataCollector<InternalAlgorith
 	}
 	
 	private String getAuthor(String algorithmDescription) {
-		String toReturn=parseDescription(algorithmDescription);
+		String toReturn=parseDescriptionForUser(algorithmDescription);
 		if(toReturn==null)
 			toReturn=env.get(Constants.DEFAULT_AUTHOR);
 		if(toReturn==null)
@@ -173,35 +229,48 @@ public class DMAlgorithmsInfoCollector implements DataCollector<InternalAlgorith
 	}
 	
 	
-	private static final UserIdentity parseUser(String userString) {
-		String splitter=null;
-		if(userString.contains(" "))
-			splitter=" ";
-		else if (userString.contains(".")) splitter="\\.";
-		
-		String[] splitted=userString.split(splitter);
-		return new UserIdentity(splitted[0], splitted[1], null, null); 
+	static final UserIdentity parseUser(String userString) {
+		try{
+			String splitter=null;
+			if(userString.contains(" "))
+				splitter=" ";
+			else if (userString.contains(".")) splitter="\\.";
+			String[] splitted=userString.split(splitter);
+			return new UserIdentity(splitted[0], splitted[1], null, null);
+		}catch(NullPointerException e){
+			System.err.println("Error with userString "+userString);
+			e.printStackTrace(System.err);
+			throw e;
+		}
 	}
 	
 	/*
-	 * i.e. "Basic statistic max min average {Published by Giancarlo Panichi (giancarlo.panichi) on 2018/07/20 10:24 GMT}"
+		 * e.g. "Basic statistic max min average {Published by Giancarlo Panichi (giancarlo.panichi) on 2018/07/20 10:24 GMT}"
 	 */
-	private static final String parseDescription(String description) {
-		Matcher m=p.matcher(description);
+	static final String parseDescriptionForUser(String description) {
+		Matcher m=descriptionPattern.matcher(description);
 		if(m.find())
 			return m.group(1); // group 0 == {...}, group 1 == Giancarlo Panichi, group 2 == giancarlo.panichi
 		else return null;		
 	}
+
+	static final String parseDescriptionForDate(String description) {
+		Matcher m=descriptionPattern.matcher(description);
+		if(m.find())
+			return m.group(4); // group 0 == {...}, group 1 == Giancarlo Panichi, group 2 == giancarlo.panichi, group 4 == 2018/07/20 10:24 GMT
+		else return "n.d.";
+	}
 	
-	
-	private final String getWPSBasePath() {
+	static final String getWPSBasePath() {
 		try{
-			ServiceEndpoint se=ISUtils.queryForServiceEndpoints("DataAnalysis", "DataMiner").get(0);
-			return se.profile().accessPoints().iterator().next().address();
+			return ISUtils.queryForServiceEndpointsByName("DataAnalysis", "DataMiner").get(0).profile().
+					accessPoints().iterator().next().address();
 		}catch(Throwable t) {
 			log.warn("Unable to find DM proxy. No WPS URL will be provided",t);
 			return null;
 		}
 	}
-	
+
+
+
 }

DataMinerAlgorithmsCrawler/src/main/java/org/gcube/data/publishing/gCatFeeder/collectors/dm/DataMinerCollectorProperties.java

@@ -8,7 +8,9 @@ public class DataMinerCollectorProperties {
 	public static final String DEFAULT_AUTHOR="default_author";
 	public static final String GUI_PARAM_NAME="gui_param_name";
 	public static final String CKAN_RESOURCE_TYPE="ckan_resource_type";
-	
+
+
+
 	
 	
 	

DataMinerAlgorithmsCrawler/src/main/java/org/gcube/data/publishing/gCatFeeder/collectors/dm/GCATTransformer.java

@@ -13,7 +13,7 @@ public class GCATTransformer implements DataTransformer<GCatModel,InternalAlgori
 	@Override
 	public Set<GCatModel> transform(Collection<InternalAlgorithmDescriptor> collectedData) {
 		HashSet<GCatModel> toReturn=new HashSet<>();
-		boolean specifyProfile=true;
+		boolean specifyProfile=false; //never
 		for(InternalAlgorithmDescriptor desc:collectedData) {
 			toReturn.add(desc.asCKANModel(specifyProfile));
 			specifyProfile=false; //only once

DataMinerAlgorithmsCrawler/src/main/java/org/gcube/data/publishing/gCatFeeder/collectors/dm/URIResolver.java

@@ -0,0 +1,123 @@
+package org.gcube.data.publishing.gCatFeeder.collectors.dm;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.util.HashMap;
+import java.util.Map;
+
+
+import org.gcube.com.fasterxml.jackson.databind.ObjectMapper;
+import org.gcube.data.publishing.gCatFeeder.utils.ContextUtils;
+import org.gcube.portlets.user.uriresolvermanager.UriResolverManager;
+import org.gcube.portlets.user.uriresolvermanager.exception.IllegalArgumentException;
+import org.gcube.portlets.user.uriresolvermanager.exception.UriResolverMapException;
+
+public class URIResolver {
+
+
+    private static final String CTLG_RESOLVER_NAME="CTLG";
+
+   // private static final String CATALOGUE_CONTEXT = "gcube_scope";
+    private static final String ENTITY_CONTEXT = "entity_context";
+    private static final String ENTITY_NAME = "entity_name";
+
+    private static final String DATASET = "dataset";
+
+    protected ObjectMapper mapper;
+
+    public URIResolver() {
+        this.mapper = new ObjectMapper();
+    }
+
+    protected StringBuilder getStringBuilder(InputStream inputStream) throws IOException {
+        StringBuilder result = new StringBuilder();
+        try(BufferedReader reader = new BufferedReader(new InputStreamReader(inputStream))) {
+            String line;
+            while((line = reader.readLine()) != null) {
+                result.append(line);
+            }
+        }
+        return result;
+    }
+
+
+    /*
+        {
+        "gcube_scope" : "/gcube/devsec/devVRE",
+        "entity_context" : "dataset",
+        "entity_name" : "sarda-sarda"
+        }
+     */
+    public String getCatalogueItemURL(String name) throws UriResolverMapException, IllegalArgumentException {
+        UriResolverManager resolver = new UriResolverManager(CTLG_RESOLVER_NAME);
+        Map<String, String> params = new HashMap<String, String>();
+        params.put(ENTITY_NAME, name);
+        params.put(ENTITY_CONTEXT,DATASET);
+        params.put("gcube_scope", ContextUtils.getCurrentScope());
+        String shortLink = resolver.getLink(params, false);
+        return shortLink;
+    }
+
+    /*public String getCatalogueItemURL(String name) {
+        try {
+            String uriResolverURL = getConfigurationFromIS();
+
+            ObjectNode requestContent = mapper.createObjectNode();
+            requestContent.put(CATALOGUE_CONTEXT, ContextUtils.getCurrentScope());
+
+            requestContent.put(ENTITY_TYPE, DATASET);
+            requestContent.put(ENTITY_NAME, name);
+
+            GXHTTPStringRequest gxhttpStringRequest = GXHTTPStringRequest.newRequest(uriResolverURL);
+            gxhttpStringRequest.from(CTLG_RESOLVER_NAME);
+            gxhttpStringRequest.header(HttpHeaders.CONTENT_TYPE, MediaType.APPLICATION_JSON);
+            gxhttpStringRequest.isExternalCall(true);
+            String body = mapper.writeValueAsString(requestContent);
+            HttpURLConnection httpURLConnection = gxhttpStringRequest.post(body);
+
+            if(httpURLConnection.getResponseCode() != 200) {
+                try{
+                    IOUtils.copy(httpURLConnection.getInputStream(),System.out);
+                }catch(Throwable t){
+                    System.out.println("No message");
+                }
+                throw new InternalServerErrorException("Unable to get Item URL via URI Resolver. Code : "+
+                        httpURLConnection.getResponseCode());
+            }
+
+            String url = getStringBuilder(httpURLConnection.getInputStream()).toString();
+
+            return url;
+        } catch(WebApplicationException e) {
+            throw e;
+        } catch(Exception e) {
+            throw new WebApplicationException(e);
+        }
+    }*/
+
+
+    /*
+    static String getConfigurationFromIS() {
+        try {
+
+            StringBuilder toReturn=new StringBuilder();
+            ServiceEndpoint serviceEndpoint = ISUtils.queryForServiceEndpointsByName("Service","HTTP-URI-Resolver").get(0);
+            serviceEndpoint.profile().accessPoints().
+                    forEach(a->{
+                        if(a.name().equals(CTLG_RESOLVER_NAME))
+                          toReturn.append(a.address());
+                    });
+            if(toReturn.length()>0) return toReturn.toString();
+            else throw new Exception("Access point for "+CTLG_RESOLVER_NAME+" not found ");
+        } catch(WebApplicationException e) {
+            throw e;
+        } catch(Exception e) {
+            throw new InternalServerErrorException("Error while getting configuration on IS", e);
+        }
+
+    }
+
+     */
+}

DataMinerAlgorithmsCrawler/src/main/java/org/gcube/data/publishing/gCatFeeder/collectors/dm/model/InternalAlgorithmDescriptor.java

@@ -1,5 +1,6 @@
 package org.gcube.data.publishing.gCatFeeder.collectors.dm.model;
 
+import java.time.LocalDateTime;
 import java.util.ArrayList;
 import java.util.HashSet;
 import java.util.List;
@@ -25,7 +26,10 @@ public class InternalAlgorithmDescriptor implements CustomData {
 
 	private UserIdentity author;
 	private UserIdentity maintainer;
-	
+
+	private String version;
+	private LocalDateTime creationDate;
+
 	//category info
 	
 	private String categoryBriefDescription;
@@ -38,8 +42,10 @@ public class InternalAlgorithmDescriptor implements CustomData {
 	private Set<Parameter> outputParameters=new HashSet<>();
 	
 	private String guiLink;
+	private String gatewayName;
 	private String wpsLink;
-	
+
+
 	private Boolean privateFlag;
 	
 	

DataMinerAlgorithmsCrawler/src/main/java/org/gcube/data/publishing/gCatFeeder/collectors/dm/model/ckan/GCatModel.java

@@ -2,9 +2,14 @@ package org.gcube.data.publishing.gCatFeeder.collectors.dm.model.ckan;
 
 import java.io.ByteArrayOutputStream;
 import java.net.URL;
+import java.text.SimpleDateFormat;
+import java.time.LocalDateTime;
+import java.time.format.DateTimeFormatter;
+import java.time.format.DateTimeFormatterBuilder;
 import java.util.ArrayList;
 
 import org.gcube.data.publishing.gCatFeeder.collectors.dm.DataMinerCollectorProperties;
+import org.gcube.data.publishing.gCatFeeder.collectors.dm.URIResolver;
 import org.gcube.data.publishing.gCatFeeder.collectors.dm.model.InternalAlgorithmDescriptor;
 import org.gcube.data.publishing.gCatFeeder.collectors.dm.model.Parameter;
 import org.gcube.data.publishing.gCatFeeder.collectors.dm.model.UserIdentity;
@@ -25,6 +30,9 @@ import lombok.extern.slf4j.Slf4j;
 @Slf4j
 public class GCatModel implements CatalogueFormatData {
 
+	private static final DateTimeFormatter dateFormatter=DateTimeFormatter.ISO_DATE;
+
+
 	private static ObjectMapper mapper=new ObjectMapper();
 
 	private static String profileXML=null;
@@ -35,13 +43,29 @@ public class GCatModel implements CatalogueFormatData {
 		profileXML=toSet;
 	}
 
+
+
+	public static String getItemUrl(String name) {
+		try{
+			URIResolver uriResolver = new URIResolver();
+			log.debug("Evaluating item url for {}",name);
+			String catalogueItemURL = uriResolver.getCatalogueItemURL(name);
+			log.info("Item URL for {} is {}",name,catalogueItemURL);
+			return catalogueItemURL;
+		}catch(Exception e){
+			log.warn("Unable to evaluate item URL for "+name,e);
+			return null;
+		}
+	}
+
 	public GCatModel(InternalAlgorithmDescriptor desc) {
 		item=new CkanItem();
-		//		item.setAuthor(desc.getAuthor());
+		item.setAuthor(desc.getAuthor().getFirstName()+" "+desc.getAuthor().getLastName());
 
 		item.setTitle(desc.getName()+" in "+ContextUtils.getCurrentScopeName());
 		item.setLicense_id("CC-BY-NC-SA-4.0");
-		//		item.setMaintainer(desc.getMaintainer());
+		item.setMaintainer(desc.getMaintainer().getFirstName()+" "+desc.getMaintainer().getLastName());
+
 		item.setName(item.getTitle().toLowerCase().toLowerCase().replaceAll(" ", "_"));
 		for(String tag:desc.getTags()) {
 			item.getTags().add(new CkanItem.Tag(fixTag(tag)));
@@ -54,21 +78,53 @@ public class GCatModel implements CatalogueFormatData {
 		item.setPrivateFlag(desc.getPrivateFlag());
 
 		for(Parameter param: desc.getInputParameters())
-			item.getExtras().add(new CKanExtraField(profileID+":Input Parameter", 
-					String.format("%1$s [%2$s] %3$s : %4$s", 
+			item.getExtras().add(new CKanExtraField("TechnicalDetails:input",
+					String.format("%1$s [%2$s] %3$s : %4$s",
 							param.getName(),param.getType(),
 							((param.getValue()!=null&&!param.getValue().isEmpty())?"default : "+param.getValue():""),
 							param.getDescription())));
 
 
 		for(Parameter param: desc.getOutputParameters())
-			item.getExtras().add(new CKanExtraField(profileID+":Output Parameter", 
-					String.format("%1$s [%2$s] %3$s : %4$s", 
+			item.getExtras().add(new CKanExtraField("TechnicalDetails:output",
+					String.format("%1$s [%2$s] %3$s : %4$s",
 							param.getName(),param.getType(),
 							((param.getValue()!=null&&!param.getValue().isEmpty())?"default : "+param.getValue():""),
 							param.getDescription())));
 
 
+		item.getExtras().add(new CKanExtraField("Identity:Creator", desc.getAuthor().asStringValue()));
+
+
+		item.getExtras().add(new CKanExtraField("Identity:CreationDate", dateFormatter.format(desc.getCreationDate())));
+
+		item.getExtras().add(new CKanExtraField("AccessMode:UsageMode", "as-a-Service via Blue-Cloud Infrastructure"));
+		item.getExtras().add(new CKanExtraField("AccessMode:Availability", "On-Line"));
+
+		item.getExtras().add(new CKanExtraField("TechnicalDetails:Hosting Environment", "gCube SmartGear"));
+		item.getExtras().add(new CKanExtraField("TechnicalDetails:Dependencies on Other SW", "gCube DataMiner"));
+
+		item.getExtras().add(new CKanExtraField("Rights:Field/Scope of use", "Any use"));
+		item.getExtras().add(new CKanExtraField("Rights:Basic rights", "Communication"));
+		item.getExtras().add(new CKanExtraField("Rights:Basic rights", "Making available to the public"));
+		item.getExtras().add(new CKanExtraField("Rights:Basic rights", "Distribution"));
+
+		item.getExtras().add(new CKanExtraField("Attribution:Attribution requirements",
+				String.format("Cite as: %1$s (%2$d): %3$s. %4$s. %5$s. %6$s. %7$s. Retrieved from the %8$s (%9$s) operated by D4Science.org www.d4science.org",
+						desc.getAuthor().asStringValue(),
+						LocalDateTime.now().getYear(),
+						desc.getName(),
+						desc.getVersion(),
+						"Blue-Cloud",
+						"DataMiner Process",
+						getItemUrl(item.getName()),
+						desc.getGatewayName(),
+						desc.getGuiLink())));
+
+
+
+
+
 		//Algorithm Description
 		//		item.getExtras().add(new CKanExtraField(profileID+":Process Description", desc.getDescription()));
 
@@ -77,8 +133,8 @@ public class GCatModel implements CatalogueFormatData {
 
 		// Algorithm Users
 
-		item.getExtras().add(new CKanExtraField(profileID+":Process Author",desc.getAuthor().asStringValue()));
-		item.getExtras().add(new CKanExtraField(profileID+":Process Maintainer",desc.getAuthor().asStringValue()));
+		//item.getExtras().add(new CKanExtraField(profileID+":Process Author",desc.getAuthor().asStringValue()));
+		//item.getExtras().add(new CKanExtraField(profileID+":Process Maintainer",desc.getAuthor().asStringValue()));
 
 		if(desc.getGuiLink()!=null) {	
 			try {
@@ -112,23 +168,6 @@ public class GCatModel implements CatalogueFormatData {
 	}
 
 
-	/**
-	 * (Common) Title
-	 * (Common) Description
-	 * (Common) Tags: free list of keywords
-	 * (Common) License 
-	 * (Common) Visibility: either public or private
-	 * (Common) Version
-	 * (Common) Author: the creator of metadata. Only one occurrence is supported; 
-	 * (Common) Maintainer: 
-	 * (Method specific) Creator: the author of the method (with email and ORCID). Repeatable field; 
-	 * (Method specific) Creation date: when the method has been released; 
-	 * (Method specific) Input: Repeatable field;
-	 * (Method specific) Output: Repeatable field; 
-	 * (Method specific) RelatedPaper: a reference to an associated paper;
-	 * (Method specific) Restrictions On Use: an optional text 
-	 * (Method specific) Attribution requirements: the text to use to acknowledge method usage; 
-	 */
 
 
 	static final String fixTag(String toFix) {

DataMinerAlgorithmsCrawler/src/main/resources/org/gcube/data/publishing/gCatFeeder/collectors/dm/config.properties

@@ -1,3 +1,3 @@
-default_author=Gianpaolo Coro
+default_author=Gianpaolo.Coro
 gui_param_name=OperatorId
-ckan_resource_type=DataMiner Process
+ckan_resource_type=Method

DataMinerAlgorithmsCrawler/src/main/resources/org/gcube/data/publishing/gCatFeeder/collectors/dm/profile.xml

@@ -1,41 +1,196 @@
-<metadataformat type="DataMiner Process">
-	<metadatafield categoryref="DataMiner Process">
-		<fieldName>Input Parameter</fieldName>
-		<mandatory>false</mandatory>
-		<dataType>String</dataType>
-		<note>Input parameter expected for the execution of the process</note>
-	</metadatafield>	
-	<metadatafield categoryref="DataMiner Process">
-		<fieldName>Output Parameter</fieldName>
-		<mandatory>false</mandatory>
-		<dataType>String</dataType>
-		<note>Output parameter expected from the execution of the process
-		</note>
-	</metadatafield>
-	<metadatafield categoryref="DataMiner Process">
-		<fieldName>Process Author</fieldName>
+<metadataformat type="Method">
+	<metadatafield categoryref="Identity">
+		<fieldName>External Identifier</fieldName>
 		<mandatory>false</mandatory>
 		<dataType>String</dataType>
 		<maxOccurs>*</maxOccurs>
-		<defaultValue />
-		<note>The name of the author, with email and ORCID. The format should
-			be: family, given[, email][, ORCID]. Example: Smith, John,
-			js@acme.org, orcid.org/0000-0002-1825-0097</note>
+		<note>This applies only to methods that have been already published. Insert here a DOI, an handle, and any other Identifier assigned when publishing the dataset alsewhere.</note>
+	</metadatafield>
+	<metadatafield categoryref="Identity">
+		<fieldName>Creator</fieldName>
+		<mandatory>true</mandatory>
+		<dataType>String</dataType>
+		<maxOccurs>*</maxOccurs>
+		<note>The name of the creator, with email and ORCID. The format should be: family, given[, email][, ORCID]. Example: Smith, John, js@acme.org, orcid.org/0000-0002-1825-0097</note>
 		<validator>
-			<regularExpression>^[a-zA-ZÀ-ÿ .'-]+, [a-zA-ZÀ-ÿ .'-]+[,]*([a-zA-Z0-9_!#$%’*+=?`{|}~^.-]+@[a-zA-Z0-9À-ÿ.-]+)?[,]*(orcid.org\/0000-000(1-[5-9]|2-[0-9]|3-[0-4])\d\d\d-\d\d\d[\dX])?$</regularExpression>
+			<regularExpression>^[a-zA-ZÀ-ÿ .'-]+, [a-zA-ZÀ-ÿ .'-]+[, ]*([a-zA-Z0-9_!#$%’*+=?`{|}~^.-]+@[a-zA-Z0-9À-ÿ.-]+)?[, ]*(orcid.org\/0000-000(1-[5-9]|2-[0-9]|3-[0-4])\d\d\d-\d\d\d[\dX])?$</regularExpression>
 		</validator>
 	</metadatafield>
-	<metadatafield categoryref="DataMiner Process">
-		<fieldName>Process Maintainer</fieldName>
+	<metadatafield categoryref="Identity">
+		<fieldName>CreationDate</fieldName>
+		<mandatory>true</mandatory>
+		<dataType>Time</dataType>
+		<maxOccurs>1</maxOccurs>
+		<note>The date of creation of the dataset (different from the date of registration of the dataset automatically added by the system). Use ISO 8601 Date Format: YYYY-MM-DD[ HH:MM] Ex. 1998-11-10 or 2015-05-29 11:55</note>
+		<validator>
+			<regularExpression>^(\d{4}\-(0?[1-9]|1[012])\-(0?[1-9]|[12][0-9]|3[01]))+([ ]+(\d{2}(:?\d{2})?)?)?$</regularExpression>
+		</validator>
+	</metadatafield>
+	<metadatafield categoryref="Identity">
+		<fieldName>Owner</fieldName>
 		<mandatory>false</mandatory>
 		<dataType>String</dataType>
 		<maxOccurs>*</maxOccurs>
-		<defaultValue />
-		<note>The name of the author, with email and ORCID. The format should
-			be: family, given[, email][, ORCID]. Example: Smith, John,
-			js@acme.org, orcid.org/0000-0002-1825-0097</note>
+		<note>The owner of the method (free text).</note>
+	</metadatafield>
+	<metadatafield categoryref="Identity">
+		<fieldName>RelatedPaper</fieldName>
+		<mandatory>false</mandatory>
+		<dataType>String</dataType>
+		<maxOccurs>1</maxOccurs>
+		<note>Insert a complete reference to an associated work.</note>
+	</metadatafield>
+	<metadatafield categoryref="Coverage">
+		<fieldName>Semantic Coverage</fieldName>
+		<mandatory>false</mandatory>
+		<dataType>String</dataType>
+		<maxOccurs>*</maxOccurs>
+		<note>Tagging e.g. Functional analysis, Environment analysis and visualisation, ...</note>
+	</metadatafield>
+	<metadatafield categoryref="AccessMode">
+		<fieldName>UsageMode</fieldName>
+		<mandatory>true</mandatory>
+		<dataType>String</dataType>
+		<maxOccurs>1</maxOccurs>
+		<note>How the method is expected to be exploited.</note>
+		<vocabulary isMultiSelection="true">
+			<vocabularyField>Download</vocabularyField>
+			<vocabularyField>as-a-Application via Blue-Cloud Infrastructure</vocabularyField>
+			<vocabularyField>as-a-Application via third-party Infrastructure</vocabularyField>
+			<vocabularyField>as-a-Service via Blue-Cloud Infrastructure</vocabularyField>
+			<vocabularyField>as-a-Service via third-party Infrastructure</vocabularyField>
+		</vocabulary>
+	</metadatafield>
+	<metadatafield categoryref="AccessMode">
+		<fieldName>Availability</fieldName>
+		<mandatory>false</mandatory>
+		<dataType>String</dataType>
+		<maxOccurs>1</maxOccurs>
+		<note>How the availability to the resource is offered. On-line means that the method can be executed through the Virtual Laboratory Gateway. On-site means that the method can only be executed by visiting the hosting provider.</note>
+		<vocabulary>
+			<vocabularyField>On-Line</vocabularyField>
+			<vocabularyField>On-Site</vocabularyField>
+		</vocabulary>
+	</metadatafield>
+	<metadatafield categoryref="TechnicalDetails">
+		<fieldName>Hosting Environment</fieldName>
+		<mandatory>false</mandatory>
+		<dataType>String</dataType>
+		<maxOccurs>1</maxOccurs>
+		<note>E.g. Linux, Microsoft Azure, Amazon EC2</note>
+	</metadatafield>
+	<metadatafield categoryref="TechnicalDetails">
+		<fieldName>ProgrammingLanguage</fieldName>
+		<mandatory>false</mandatory>
+		<dataType>String</dataType>
+		<maxOccurs>1</maxOccurs>
+		<note>The primary language used to implement the method. </note>
+	</metadatafield>
+	<metadatafield categoryref="TechnicalDetails">
+		<fieldName>Dependencies on Other SW</fieldName>
+		<mandatory>false</mandatory>
+		<dataType>String</dataType>
+		<maxOccurs>*</maxOccurs>
+		<note>E.g. this sowftware requires an Hadoop cluster to run</note>
+	</metadatafield>
+	<metadatafield categoryref="TechnicalDetails">
+		<fieldName>input</fieldName>
+		<mandatory>false</mandatory>
+		<dataType>String</dataType>
+		<maxOccurs>*</maxOccurs>
+		<note>inputParametersType. See WPS specifications</note>
+	</metadatafield>
+	<metadatafield categoryref="TechnicalDetails">
+		<fieldName>output</fieldName>
+		<mandatory>false</mandatory>
+		<dataType>String</dataType>
+		<maxOccurs>*</maxOccurs>
+		<note>outputType. See WPS specifications </note>
+	</metadatafield>
+	<metadatafield categoryref="Rights">
+		<fieldName>IP/Copyrights</fieldName>
+		<mandatory>false</mandatory>
+		<dataType>String</dataType>
+		<maxOccurs>1</maxOccurs>
+		<note>Whether software is covered by any rights: copyright, related rights, know how, proprietary, etc.</note>
+	</metadatafield>
+	<metadatafield categoryref="Rights">
+		<fieldName>Field/Scope of use</fieldName>
+		<mandatory>true</mandatory>
+		<dataType>String</dataType>
+		<maxOccurs>1</maxOccurs>
+		<vocabulary isMultiSelection="true">
+			<vocabularyField>Any use</vocabularyField>
+			<vocabularyField>Non-commercial only</vocabularyField>
+			<vocabularyField>Research only</vocabularyField>
+			<vocabularyField>Non-commercial research only</vocabularyField>
+			<vocabularyField>Private use</vocabularyField>
+			<vocabularyField>Use for developing and providing a service</vocabularyField>
+		</vocabulary>
+	</metadatafield>
+	<metadatafield categoryref="Rights">
+		<fieldName>Basic rights</fieldName>
+		<mandatory>true</mandatory>
+		<dataType>String</dataType>
+		<maxOccurs>1</maxOccurs>
+		<vocabulary isMultiSelection="true">
+			<vocabularyField>Temporary download of a single copy only</vocabularyField>
+			<vocabularyField>Download</vocabularyField>
+			<vocabularyField>Copying</vocabularyField>
+			<vocabularyField>Distribution</vocabularyField>
+			<vocabularyField>Modification</vocabularyField>
+			<vocabularyField>Communication</vocabularyField>
+			<vocabularyField>Making available to the public</vocabularyField>
+			<vocabularyField>Other rights</vocabularyField>
+		</vocabulary>
+	</metadatafield>
+	<metadatafield categoryref="Rights">
+		<fieldName>Restrictions on use</fieldName>
+		<mandatory>false</mandatory>
+		<dataType>String</dataType>
+		<maxOccurs>1</maxOccurs>
+		<note>Any restrictions on how where the dataset may be used </note>
+	</metadatafield>
+	<metadatafield categoryref="Rights">
+		<fieldName>Sublicense rights</fieldName>
+		<mandatory>false</mandatory>
+		<dataType>String</dataType>
+		<maxOccurs>1</maxOccurs>
+		<note>Any restrictions on how where the dataset may be used</note>
+		<vocabulary>
+			<vocabularyField>No</vocabularyField>
+			<vocabularyField>Yes</vocabularyField>
+		</vocabulary>
+	</metadatafield>
+	<metadatafield categoryref="Rights">
+		<fieldName>Requirement of non-disclosure (confidentiality mark)</fieldName>
+		<mandatory>false</mandatory>
+		<dataType>String</dataType>
+		<maxOccurs>1</maxOccurs>
+		<note>Requirement of non-disclosure (confidentiality mark). Whether the dataset bears confidentiality mark/may be used and shared subject to the obligation of non-disclosure</note>
+	</metadatafield>
+	<metadatafield categoryref="Rights">
+		<fieldName>Embargo period</fieldName>
+		<mandatory>false</mandatory>
+		<dataType>Time_Interval</dataType>
+		<maxOccurs>1</maxOccurs>
+		<note>Period of time during which the resource may be used. Use ISO 8601 Date Format: YYYY-MM-DD[ HH:MM] Ex. 2016-07-31 or 2015-05-10 12:00</note>
 		<validator>
-			<regularExpression>^[a-zA-ZÀ-ÿ .'-]+, [a-zA-ZÀ-ÿ .'-]+[,]*([a-zA-Z0-9_!#$%’*+=?`{|}~^.-]+@[a-zA-Z0-9À-ÿ.-]+)?[,]*(orcid.org\/0000-000(1-[5-9]|2-[0-9]|3-[0-4])\d\d\d-\d\d\d[\dX])?$</regularExpression>
+			<regularExpression>^(\d{4}\-(0?[1-9]|1[012])\-(0?[1-9]|[12][0-9]|3[01]))+([ ]+(\d{2}(:?\d{2})?)?)?$</regularExpression>
 		</validator>
 	</metadatafield>
+	<metadatafield categoryref="Attribution">
+		<fieldName>Attribution requirements</fieldName>
+		<mandatory>false</mandatory>
+		<dataType>String</dataType>
+		<maxOccurs>1</maxOccurs>
+		<note>The text to acknowledge the resource when using it</note>
+	</metadatafield>
+	<metadatafield categoryref="Attribution">
+		<fieldName>Distribution requirements</fieldName>
+		<mandatory>false</mandatory>
+		<dataType>String</dataType>
+		<maxOccurs>1</maxOccurs>
+		<note>The text to acknowledge the resource when distributing it</note>
+	</metadatafield>
 </metadataformat>

DataMinerAlgorithmsCrawler/src/test/java/org/gcube/data/publishing/gCatFeeder/collectors/dm/ItemUrlTests.java

@@ -0,0 +1,18 @@
+package org.gcube.data.publishing.gCatFeeder.collectors.dm;
+
+import org.gcube.common.authorization.library.provider.SecurityTokenProvider;
+import org.gcube.data.publishing.gCatFeeder.tests.BaseCollectorTest;
+import org.gcube.data.publishing.gCatFeeder.tests.TokenSetter;
+import org.gcube.portlets.user.uriresolvermanager.exception.IllegalArgumentException;
+import org.gcube.portlets.user.uriresolvermanager.exception.UriResolverMapException;
+import org.junit.Test;
+
+public class ItemUrlTests extends BaseCollectorTest {
+
+    @Test
+    public void getItemURL() throws UriResolverMapException, IllegalArgumentException {
+        TokenSetter.set("/pred4s/preprod/preVRE");
+        String name = "fake";
+        System.out.println(new URIResolver().getCatalogueItemURL(name));
+    }
+}

DataMinerAlgorithmsCrawler/src/test/java/org/gcube/data/publishing/gCatFeeder/collectors/dm/TranslationTest.java

@@ -1,13 +1,18 @@
 package org.gcube.data.publishing.gCatFeeder.collectors.dm;
 
 import java.io.IOException;
+import java.time.LocalDateTime;
+import java.time.ZonedDateTime;
 import java.util.Collection;
 import java.util.Set;
 
+import org.gcube.common.authorization.library.provider.SecurityTokenProvider;
 import org.gcube.data.publishing.gCatFeeder.collectors.dm.model.InternalAlgorithmDescriptor;
+import org.gcube.data.publishing.gCatFeeder.collectors.dm.model.ckan.GCatModel;
 import org.gcube.data.publishing.gCatFeeder.model.CatalogueFormatData;
 import org.gcube.data.publishing.gCatFeeder.model.InternalConversionException;
 import org.gcube.data.publishing.gCatFeeder.tests.BaseCollectorTest;
+import org.gcube.data.publishing.gCatFeeder.tests.TokenSetter;
 import org.gcube.data.publishing.gCatfeeder.collectors.CollectorPlugin;
 import org.gcube.data.publishing.gCatfeeder.collectors.DataCollector;
 import org.gcube.data.publishing.gCatfeeder.collectors.DataTransformer;
@@ -20,6 +25,8 @@ import com.fasterxml.jackson.core.JsonGenerationException;
 import com.fasterxml.jackson.databind.JsonMappingException;
 import com.fasterxml.jackson.databind.ObjectMapper;
 
+import static org.junit.Assert.*;
+
 public class TranslationTest extends BaseCollectorTest{
 
 
@@ -38,12 +45,38 @@ public class TranslationTest extends BaseCollectorTest{
 				System.out.println(mapper.writeValueAsString(obj)+"\n");
 
 			for(String destinationcatalogue : (Set<String>)plugin.getSupportedCatalogueTypes()) {
-				DataTransformer<? extends CatalogueFormatData, InternalAlgorithmDescriptor> transformer=plugin.getTransformerByCatalogueType(destinationcatalogue);
+				DataTransformer<? extends CatalogueFormatData, InternalAlgorithmDescriptor> transformer=
+						plugin.getTransformerByCatalogueType(destinationcatalogue);
+				System.out.println("Starting Transformation "+transformer.toString());
+
 				for(Object data:transformer.transform(collected))
 					System.out.println(((CatalogueFormatData)data).toCatalogueFormat());
 			
 		}
 	}
 
-	
+
+
+	@Test
+	public void testParseDescription(){
+		String s="balablabalba";
+		assertEquals("n/a",DMAlgorithmsInfoCollector.parseDescriptionForDate(s));
+		assertEquals(null, DMAlgorithmsInfoCollector.parseDescriptionForUser(s));
+		s="Basic statistic max min average {Published by Giancarlo Panichi (giancarlo.panichi) on 2018/07/20 10:24 GMT}";
+		assertEquals("2018/07/20 10:24 GMT",DMAlgorithmsInfoCollector.parseDescriptionForDate(s));
+		assertEquals("Giancarlo Panichi ",DMAlgorithmsInfoCollector.parseDescriptionForUser(s));
+
+
+
+		System.out.println(DMAlgorithmsInfoCollector.versionDateParser.format(ZonedDateTime.now()));
+		System.out.println(DMAlgorithmsInfoCollector.versionDateParser.parse(DMAlgorithmsInfoCollector.parseDescriptionForDate(s)));
+	}
+
+	@Test
+	public void testEnvironment(){
+		Assume.assumeTrue(isTestInfrastructureEnabled());
+		//assertNotNull(DMAlgorithmsInfoCollector.getWPSBasePath());
+		//assertNotNull(GCatModel.getItemUrl("fake"));
+		System.out.println(getEnvironmentConfiguration().getCurrentConfiguration());
+	}
 }

catalogue-plugin-framework/CHANGELOG.md

@@ -2,6 +2,14 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm
 
 # Changelog for org.gcube.data-publishing.gFeed.catalogue-plugin-framework
 
+## [v1.0.6]
+- Lombok dependency upgrade
+
+
+## [v1.0.5]
+- Pom updates
+
+
 ## [v1.0.4] - 2020-12-15
 - Dependency management
 - Naming Convention

catalogue-plugin-framework/pom.xml

@@ -5,10 +5,10 @@
 	<parent>
 		<groupId>org.gcube.data-publishing.gFeed</groupId>
 		<artifactId>gFeed-Suite</artifactId>
-		<version>1.0.4</version>
+		<version>1.0.6</version>
 	</parent>
 	<artifactId>catalogue-plugin-framework</artifactId>
-	<version>1.0.4</version>
+	<version>1.0.6</version>
 
 
 	<scm>
@@ -20,7 +20,6 @@
 		<dependency>
 			<groupId>org.projectlombok</groupId>
 			<artifactId>lombok</artifactId>
-			<version>1.14.8</version>
 		</dependency>
 
 

collectors-plugin-framework/CHANGELOG.md

@@ -2,6 +2,12 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm
 
 # Changelog for org.gcube.data-publishing.gFeed.collectors-plugin-framework
 
+## [v1.0.6]
+- Pom updates
+
+## [v1.0.5]
+- Pom updates
+
 ## [v1.0.4] - 2020-12-15
 - Dependency management
 - Naming Convention

collectors-plugin-framework/pom.xml

@@ -5,12 +5,12 @@
 	<parent>
 		<groupId>org.gcube.data-publishing.gFeed</groupId>
 		<artifactId>gFeed-Suite</artifactId>
-		<version>1.0.4</version>
+		<version>1.0.6</version>
 	</parent>
 	<artifactId>collectors-plugin-framework</artifactId>
 	<name>collectors-plugin-framework</name>
 	<description>Framework for collectors implementations</description>
-	<version>1.0.4</version>
+	<version>1.0.6</version>
 
 
 
@@ -26,7 +26,6 @@
 		<dependency>
 			<groupId>org.projectlombok</groupId>
 			<artifactId>lombok</artifactId>
-			<version>1.14.8</version>
 		</dependency>
 
 

commons/CHANGELOG.md

@@ -2,6 +2,12 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm
 
 # Changelog for org.gcube.data-publishing.gFeed.test-commons
 
+## [v1.0.6]
+- Pom updates
+
+## [v1.0.5]
+- Pom updates
+
 ## [v1.0.4] - 2020-12-15
 - Dependency management
 - Naming Convention

commons/pom.xml

@@ -5,11 +5,11 @@
 	<parent>
 		<groupId>org.gcube.data-publishing.gFeed</groupId>
 		<artifactId>gFeed-Suite</artifactId>
-		<version>1.0.4</version>
+		<version>1.0.6</version>
 	</parent>
 	<artifactId>commons</artifactId>
 	<name>commons</name>
-	<version>1.0.4</version>
+	<version>1.0.6</version>
 	<description>Common utilities for gCatFeeder logic</description>
 
 

gCat-Controller/CHANGELOG.md

@@ -2,6 +2,15 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm
 
 # Changelog for org.gcube.data-publishing.gFeed.gCat-controller
 
+## [v1.0.8]
+- Pom updates
+
+## [v1.0.7]
+- Pom updates
+
+## [v1.0.6] - 2021-07-01
+- gCat client coordinates 
+
 ## [v1.0.5] - 2020-12-15
 - gCat Client upgrade
 

gCat-Controller/pom.xml

@@ -3,15 +3,12 @@
   <parent>
    <groupId>org.gcube.data-publishing.gFeed</groupId>
     <artifactId>gFeed-Suite</artifactId>
-    <version>1.0.4</version>
+    <version>1.0.6</version>
   </parent>
   <artifactId>gCat-Controller</artifactId>
   <name>gCat-Controller</name>
-  <version>1.0.5</version>
+  <version>1.0.8</version>
   <description>Controller implementation for GCat Service</description>
-<!--     <properties>		 -->
-<!-- 		<gitBaseUrl>https://code-repo.d4science.org/gCubeSystem</gitBaseUrl> -->
-<!-- 	</properties> -->
 
 	<scm>
 		<connection>scm:git:${gitBaseUrl}/gFeed</connection>
@@ -47,9 +44,9 @@
   		
   		
   		<dependency>
-  			<groupId>org.gcube.data-publishing</groupId>
-    		<artifactId>gcat-client</artifactId>
-    		<version>[1.0.0,2.0.0)</version>
+			<groupId>org.gcube.data-catalogue</groupId>
+			<artifactId>gcat-client</artifactId>
+    		<version>[2.2.0,3.0.0)</version>
   		</dependency>
   		
   		

gCat-Controller/src/test/java/org/gcube/data/publishing/gCatFeeder/catalogues/gCat/InfrastructureTest.java

@@ -0,0 +1,24 @@
+package org.gcube.data.publishing.gCatFeeder.catalogues.gCat;
+
+import com.fasterxml.jackson.core.JsonGenerationException;
+import com.fasterxml.jackson.databind.JsonMappingException;
+import org.gcube.data.publishing.gCatFeeder.model.InternalConversionException;
+import org.gcube.data.publishing.gCatFeeder.tests.InfrastructureTests;
+import org.gcube.gcat.client.Profile;
+import org.junit.Assume;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.net.MalformedURLException;
+
+public class InfrastructureTest extends InfrastructureTests {
+
+    @Test
+    public void testIsServicePresent() throws MalformedURLException {
+        Assume.assumeTrue(isTestInfrastructureEnabled());
+        System.out.println("START HERE");
+        new Profile();
+
+    }
+
+}

gCat-Controller/src/test/java/org/gcube/data/publishing/gCatFeeder/catalogues/gCat/Interactions.java

@@ -10,6 +10,7 @@ import org.gcube.data.publishing.gCatFeeder.model.CatalogueFormatData;
 import org.gcube.data.publishing.gCatFeeder.model.CatalogueInstanceDescriptor;
 import org.gcube.data.publishing.gCatFeeder.model.InternalConversionException;
 import org.gcube.data.publishing.gCatFeeder.tests.BaseCataloguePluginTest;
+import org.gcube.data.publishing.gCatFeeder.tests.TokenSetter;
 import org.gcube.data.publishing.gCataFeeder.catalogues.gCat.GCatPlugin;
 import org.junit.Assert;
 import org.junit.Assume;

gCat-Feeder/CHANGELOG.md

@@ -2,6 +2,12 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm
 
 # Changelog for org.gcube.data-publishing.gFeed.gFeed-service
 
+## [v1.0.6]
+- Pom updates
+
+## [v1.0.5]
+- Pom updates
+
 ## [v1.0.4] - 2020-12-15
 - Dependency management
 - Naming Convention

gCat-Feeder/pom.xml

@@ -1,20 +1,14 @@
 <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
 	xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
 	<modelVersion>4.0.0</modelVersion>
-<!-- 	<parent> -->
-<!-- 		<groupId>org.gcube.tools</groupId> -->
-<!-- 		<artifactId>maven-parent</artifactId> -->
-<!-- 		<version>1.1.0</version> -->
-<!-- 	</parent> -->
-
 	 <parent>
     <groupId>org.gcube.data-publishing.gFeed</groupId>
     <artifactId>gFeed-Suite</artifactId>
-    <version>1.0.4</version>
+    <version>1.0.6</version>
   </parent>
 	<artifactId>gCat-Feeder</artifactId>
 	<name>gFeed Service</name>
-	<version>1.0.4</version>
+	<version>1.0.6</version>
 	<description>Service implementation</description>
 
 	<properties>
@@ -74,6 +68,10 @@
 			<artifactId>javax.ws.rs-api</artifactId>
 		</dependency>
 
+		<dependency>
+			<groupId>javax.xml.bind</groupId>
+			<artifactId>jaxb-api</artifactId>
+		</dependency>
 
 
 		<dependency>

gCat-Feeder/src/test/resources/logback.xml

@@ -8,7 +8,7 @@
     </encoder>
   </appender>
 
-	<logger name="org.gcube" level="DEBUG"/>
+	<logger name="org.gcube.data" level="DEBUG"/>
 
   <root level="WARN">
     <appender-ref ref="STDOUT" />

oai-harvester/CHANGELOG.md

@@ -2,6 +2,16 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm
 
 # Changelog for org.gcube.data-publishing.gFeed.oai-pmh
 
+## [v1.0.8]
+Harvested Object profile update
+
+
+## [v1.0.7]
+- Pom updates
+
+## [v1.0.6]
+- Pom updates
+
 ## [v1.0.5] - 2020-12-15
 Do not stop on single repository error
 

oai-harvester/pom.xml

@@ -5,11 +5,11 @@
 	<parent>
 		<groupId>org.gcube.data-publishing.gFeed</groupId>
 		<artifactId>gFeed-Suite</artifactId>
-		<version>1.0.4</version>
+		<version>1.0.6</version>
 	</parent>
 	<artifactId>oai-harvester</artifactId>
 	<name>oai-harvester</name>
-	<version>1.0.5</version>
+	<version>1.0.8</version>
 
 	<scm>
 		<connection>scm:git:${gitBaseUrl}/gFeed</connection>
@@ -40,6 +40,11 @@
 		</dependency>
 
 
+		<dependency>
+			<groupId>javax.xml.bind</groupId>
+			<artifactId>jaxb-api</artifactId>
+		</dependency>
+
 		<dependency>
 			<groupId>org.glassfish.jersey.media</groupId>
 			<artifactId>jersey-media-json-jackson</artifactId>

oai-harvester/src/main/java/org/gcube/data/publishing/gFeed/collectors/oai/OAIClient.java

@@ -83,7 +83,7 @@ public class OAIClient {
 			for(String set : specifiedSets) {
 				log.info("Loading "+metadataPrefix+" SET : "+set+" from "+baseUrl);
 				target.queryParam("set", set);
-				toReturn.addAll(call(target,metadataPrefix));
+				toReturn.addAll(call(target.queryParam("set", set),metadataPrefix));
 			}		
 		else {
 			log.info("Loading "+metadataPrefix+" from "+baseUrl);
@@ -98,6 +98,8 @@ public class OAIClient {
 	private List<OAIRecord> call(WebTarget target,String metadataPrefix){
 		ArrayList<OAIRecord> toReturn=new ArrayList<OAIRecord>();
 
+		log.info("Harvesting from resulting url {} ",target.getUri());
+
 		String resumptionToken=null;
 
 		// call & iterate
@@ -105,14 +107,18 @@ public class OAIClient {
 		int currentAttempt=1;
 		while(!isComplete) {
 			try {
-				
-				if(resumptionToken==null)
+
+				if(resumptionToken==null) {
 					target=target.queryParam("metadataPrefix",metadataPrefix);
-				else
+				}
+				else {
 					target=target.queryParam("resumptionToken", resumptionToken);
+				}
 
 
-			Response resp=target.request("application/xml").get();
+				log.trace("Calling {} ",target.getUri());
+
+				Response resp=target.request("application/xml").get();
 
 
 
@@ -123,11 +129,13 @@ public class OAIClient {
 			//No errors, thus reset attempt counter
 			currentAttempt=1;
 
-			toReturn.addAll(msg.getResponseRecords().getRecords());
+			if(msg.getResponseRecords().getRecords()!=null)
+				toReturn.addAll(msg.getResponseRecords().getRecords());
+			else log.info("NB {} didn't returned any record",msg.getRequest().getPath());
 			log.debug("Parsed "+toReturn.size()+" records so far.");
 
-				
-				Token t=msg.getResponseRecords().getResumptionToken();
+
+			Token t=msg.getResponseRecords().getResumptionToken();
 				log.debug("Obtained token : "+t);
 
 				if(t!=null && t.getId()!=null && !t.getId().isEmpty()) {
@@ -140,7 +148,7 @@ public class OAIClient {
 					isComplete=true;
 				}
 			}catch(Throwable t) {
-				log.warn("Unexpected ERROR	 "+t.getMessage());
+				log.warn("Unexpected ERROR	 ",t);
 				log.debug("Current attempt number = "+currentAttempt," max attempt Number = "+MAX_ATTEMPTS+", attempts delay factor =  ");
 				isComplete=currentAttempt>MAX_ATTEMPTS;
 				try {

oai-harvester/src/main/resources/profiles/HarvestedObject.xml

@@ -56,7 +56,7 @@
 		<mandatory>false</mandatory>
 		<dataType>String</dataType>
 		<maxOccurs>*</maxOccurs>
-		<tagging create="true" separator=" ">onValue</tagging>
+		<!-- <tagging create="true" separator=" ">onValue</tagging> -->
 	</metadatafield>
 	<metadatafield categoryref="Harvested Object">
 		<fieldName>relation</fieldName>
@@ -81,7 +81,7 @@
 		<mandatory>false</mandatory>
 		<dataType>String</dataType>
 		<maxOccurs>*</maxOccurs>
-		<tagging create="true" separator=" ">onValue</tagging>
+		<!-- <tagging create="true" separator=" ">onValue</tagging> -->
 	</metadatafield>
 	<metadatafield categoryref="Harvested Object">
 		<fieldName>title</fieldName>
@@ -94,6 +94,6 @@
 		<mandatory>false</mandatory>
 		<dataType>String</dataType>
 		<maxOccurs>*</maxOccurs>
-		<tagging create="true" separator=" ">onValue</tagging>
+		<!-- <tagging create="true" separator=" ">onValue</tagging> -->
 	</metadatafield>
 </metadataformat>

oai-harvester/src/test/java/org/gcube/application/gfeed/oai/TranslationTest.java

@@ -5,6 +5,7 @@ import java.util.Collection;
 import java.util.Set;
 
 import org.gcube.data.publishing.gCatFeeder.model.CatalogueFormatData;
+import org.gcube.data.publishing.gCatFeeder.model.EnvironmentConfiguration;
 import org.gcube.data.publishing.gCatFeeder.model.InternalConversionException;
 import org.gcube.data.publishing.gCatFeeder.tests.BaseCollectorTest;
 import org.gcube.data.publishing.gCatfeeder.collectors.CollectorPlugin;
@@ -31,18 +32,20 @@ public class TranslationTest extends BaseCollectorTest{
 		ObjectMapper mapper = new ObjectMapper();
 
 		CollectorPlugin plugin=new OAIHarvester();
-		plugin.setEnvironmentConfiguration(getEnvironmentConfiguration());
+		EnvironmentConfiguration env = getEnvironmentConfiguration();
+		System.out.println("Setting ENV : "+env);
+		plugin.setEnvironmentConfiguration(env);
 		DataCollector collector=plugin.getCollector();
 		Collection collected=collector.collect();
 		System.out.println("Found "+collected.size()+" elements");
 		for(Object obj:collected)
 			System.out.println(mapper.writeValueAsString(obj)+"\n");
 
-		for(String destinationcatalogue : (Set<String>)plugin.getSupportedCatalogueTypes()) {
-			DataTransformer<? extends CatalogueFormatData, OAIRecord> transformer=plugin.getTransformerByCatalogueType(destinationcatalogue);
-			for(Object data:transformer.transform(collected))
-				System.out.println(((CatalogueFormatData)data).toCatalogueFormat());
-		}
+//		for(String destinationcatalogue : (Set<String>)plugin.getSupportedCatalogueTypes()) {
+//			DataTransformer<? extends CatalogueFormatData, OAIRecord> transformer=plugin.getTransformerByCatalogueType(destinationcatalogue);
+//			for(Object data:transformer.transform(collected))
+//				System.out.println(((CatalogueFormatData)data).toCatalogueFormat());
+//		}
 	}
 	
 	

pom.xml

@@ -9,7 +9,7 @@
 	</parent>
 	<groupId>org.gcube.data-publishing.gFeed</groupId>
 	<artifactId>gFeed-Suite</artifactId>
-	<version>1.0.4</version>
+	<version>1.0.6</version>
 	<packaging>pom</packaging>
 	<name>gFeed Suite</name>
 	<description>gCat-Feeder Suite of components : service, plugin framework, plugins</description>
@@ -18,6 +18,15 @@
 	<properties>
 		<distroDirectory>distro</distroDirectory>
 		<gitBaseUrl>https://code-repo.d4science.org/gCubeSystem</gitBaseUrl>
+
+		<!-- PROD -->
+		<smartgears-bom>2.2.0</smartgears-bom>
+		<gcube-bom>2.1.0</gcube-bom>
+		<lombok.version>1.18.4</lombok.version>
+		<!-- DEV -->
+<!--		<smartgears-bom>2.2.0-SNAPSHOT</smartgears-bom>-->
+<!--		<gcube-bom>2.1.0-SNAPSHOT</gcube-bom>-->
+
 	</properties>
 
 	<scm>
@@ -43,24 +52,32 @@
 		<module>oai-harvester</module>
 	</modules>
 
+
+
 	<dependencyManagement>
 		<dependencies>
 			<dependency>
 				<groupId>org.gcube.distribution</groupId>
 				<artifactId>gcube-smartgears-bom</artifactId>
-				<version>2.0.0</version>
+				<version>${smartgears-bom}</version>
 				<type>pom</type>
 				<scope>import</scope>
 			</dependency>
 			<dependency>
 				<groupId>org.gcube.distribution</groupId>
 				<artifactId>gcube-bom</artifactId>
-				<version>2.0.0</version>
+				<version>${gcube-bom}</version>
 				<type>pom</type>
 				<scope>import</scope>
 			</dependency>
-			
-			
+
+
+			<dependency>
+				<groupId>org.projectlombok</groupId>
+				<artifactId>lombok</artifactId>
+				<version>${lombok.version}</version>
+			</dependency>
+
 			<dependency>
 				<groupId>org.gcube.data-publishing.gFeed</groupId>
 				<artifactId>collectors-plugin-framework</artifactId>

test-commons/CHANGELOG.md

@@ -2,6 +2,12 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm
 
 # Changelog for org.gcube.data-publishing.gFeed.test-commons
 
+## [v1.0.6]
+- Test support to jwt token
+
+## [v1.0.5]
+- Pom updates
+
 ## [v1.0.4] - 2020-12-15
 - Dependency management
 - Naming Convention

test-commons/pom.xml

@@ -3,11 +3,11 @@
   <parent>
     <groupId>org.gcube.data-publishing.gFeed</groupId>
     <artifactId>gFeed-Suite</artifactId>
-    <version>1.0.4</version>
+    <version>1.0.6</version>
   </parent>
   <artifactId>test-commons</artifactId>
   <name>test commons</name>
-  <version>1.0.4</version>
+  <version>1.0.6</version>
   <description>Utility classes for test purposes</description>
 
 
@@ -26,7 +26,12 @@
 			<groupId>org.gcube.data-publishing.gFeed</groupId>
 			<artifactId>commons</artifactId>
 		</dependency>
-		
+
+		 <dependency>
+			 <groupId>org.gcube.common</groupId>
+			 <artifactId>authorization-utils</artifactId>
+			 <version>[2.0.0, 3.0.0-SNAPSHOT)</version>
+		 </dependency>
 		
 
 		<dependency>

test-commons/src/main/java/org/gcube/data/publishing/gCatFeeder/tests/InfrastructureTests.java

@@ -6,6 +6,11 @@ import java.util.Map;
 import java.util.Properties;
 
 import org.gcube.common.authorization.library.provider.SecurityTokenProvider;
+import org.gcube.common.authorization.utils.manager.SecretManager;
+import org.gcube.common.authorization.utils.manager.SecretManagerProvider;
+import org.gcube.common.authorization.utils.secret.GCubeSecret;
+import org.gcube.common.authorization.utils.secret.JWTSecret;
+import org.gcube.common.authorization.utils.secret.Secret;
 import org.gcube.common.scope.api.ScopeProvider;
 import org.gcube.data.publishing.gCatFeeder.model.EnvironmentConfiguration;
 import org.gcube.data.publishing.gCatFeeder.utils.ISUtils;
@@ -28,13 +33,27 @@ public class InfrastructureTests {
 	public static void setTestContext() {
 
 		if(isTestInfrastructureEnabled()) {
+			SecretManagerProvider.instance.set(new SecretManager());
 			Properties props=new Properties();
 			try{
 				props.load(BaseCollectorTest.class.getResourceAsStream("/tokens.properties"));
 			}catch(IOException e) {throw new RuntimeException(e);}
 			if(!props.containsKey(testContext)) throw new RuntimeException("No token found for scope : "+testContext);
-			SecurityTokenProvider.instance.set(props.getProperty(testContext));
-			ScopeProvider.instance.set(testContext);
+
+			String toSet = props.getProperty(testContext);
+
+			Secret secret = null;
+			if(toSet.length()>50)
+				secret = new JWTSecret(toSet); // se nuovo token
+			else
+				secret = new GCubeSecret(toSet); // se vecchio token
+
+
+			SecretManagerProvider.instance.get().addSecret(secret);
+			try{
+				SecretManagerProvider.instance.get().set();
+			}catch(Exception e ){throw new RuntimeException("Unable to set secret for context "+testContext,e);}
+
 		}
 	}