in progress on creating template #26454

catalogue-binding-plugin/src/main/java/org/gcube/application/cms/cataloguebinding/doaction/GCatCaller.java

@@ -48,6 +48,7 @@ public class GCatCaller {
 				else
 					log.warn("Dataset not created!");
 			} else {
+				log.info("calling update dataset...");
 				String itemCreated = new Item().update(datasetName, datasetJSON);
 				if (itemCreated != null)
 					log.info("Dataset updated with success!");
@@ -91,4 +92,17 @@ public class GCatCaller {
 			log.error("Error occurred on deleting the dataset " + datasetName + " on the catalogue", e);
 		}
 	}
+
+	public String list(int limit, int offset) {
+		log.info("list called");
+
+		try {
+			return new Item().list(limit, offset);
+		} catch (WebApplicationException | MalformedURLException e) {
+			log.error("Error occurred on listing datasets", e);
+		}
+		
+		return null;
+
+	}
 }

catalogue-binding-plugin/src/main/java/org/gcube/application/cms/cataloguebinding/freemarker/MappingToCatalogue.java

@@ -75,8 +75,8 @@ public class MappingToCatalogue {
 			log.info("template applied with success..");
 
 			String toCatalogueDatasetJSON = stringWriter.toString();
-			if (log.isDebugEnabled()) {
+			if (log.isTraceEnabled()) {
-				log.debug("Template applyed with result: " + toCatalogueDatasetJSON);
+				log.trace("Template applyed with result: " + toCatalogueDatasetJSON);
 			}
 
 			// Parsing the Catalogue Dataset json as Document Context
@@ -114,8 +114,8 @@ public class MappingToCatalogue {
 
 			String catalogueDatasetJSON = prettyPrintJsonUtil(theDoc.jsonString());
 
-			if (log.isDebugEnabled()) {
+			if (log.isTraceEnabled()) {
-				log.debug("To pretty print JSON: \n" + catalogueDatasetJSON);
+				log.trace("To pretty print JSON: \n" + catalogueDatasetJSON);
 			}
 
 			log.info("returning catalogue dataset, json lenght is: " + catalogueDatasetJSON.length());

catalogue-binding-plugin/src/main/java/org/gcube/application/cms/cataloguebinding/gis/GisClient.java

@@ -25,7 +25,9 @@ public class GisClient {
 			builder.scope(scope);
 			builder.resolverAs(resolveAs);
 
-			builder.targetApp(targetApp);
+			if(targetApp!=null)
+				builder.targetApp(targetApp);
+			
 			Map<String, String> params = builder.buildQueryParameters();
 			theLink = uriResolverManager.getLink(params, shortLink);
 			log.info("Geoportal GisViewer link is {} ", theLink);

catalogue-binding-plugin/src/test/java/geoportal_to_catalogue/GeoportalToCatalogue.java

@@ -4,20 +4,31 @@ import java.io.File;
 import java.io.FileWriter;
 import java.io.IOException;
 import java.io.StringWriter;
+import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.HashMap;
+import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 
 import org.bson.BsonDocument;
 import org.bson.BsonString;
 import org.bson.Document;
+import org.gcube.application.cms.cataloguebinding.doaction.BindingAction;
+import org.gcube.application.cms.cataloguebinding.doaction.GCatCaller;
+import org.gcube.application.cms.cataloguebinding.freemarker.FreemarkerConfig;
+import org.gcube.application.cms.cataloguebinding.freemarker.MappingToCatalogue;
 import org.gcube.application.cms.cataloguebinding.util.SerializationUtil;
 import org.gcube.application.geoportal.common.model.document.Project;
 import org.gcube.application.geoportal.common.model.useCaseDescriptor.UseCaseDescriptor;
 import org.gcube.application.geoportalcommon.geoportal.GeoportalClientCaller;
 import org.gcube.application.geoportalcommon.geoportal.ProjectsCaller;
+import org.gcube.application.geoportalcommon.shared.SearchingFilter;
+import org.gcube.application.geoportalcommon.shared.SearchingFilter.ORDER;
+import org.gcube.application.geoportalcommon.shared.geoportal.config.ItemFieldDV;
 import org.gcube.common.authorization.library.provider.SecurityTokenProvider;
 import org.gcube.common.scope.api.ScopeProvider;
+import org.json.JSONArray;
 import org.json.JSONException;
 import org.json.JSONObject;
 
@@ -42,100 +53,161 @@ import test.TestContextConfig;
 public class GeoportalToCatalogue {
 
 	public final static String profileID = "profiledConcessioni";
-	public final static String projectID = "66697afda08b3011b506ddb6";
+	public final static String projectID = "666afb4ea08b3011b506de90";
+	//666afb4ea08b3011b506de90 
+	//66697afda08b3011b506ddb6
 
-//	public static void main(String[] args) {
+	/*
-//
+	 * public static void main(String[] args) {
-//		try {
+	 * 
-//
+	 * try {
-//			TestContextConfig.readContextSettings();
+	 * 
-//
+	 * TestContextConfig.readContextSettings();
-//			ScopeProvider.instance.set(TestContextConfig.CONTEXT);
+	 * 
-//			SecurityTokenProvider.instance.set(TestContextConfig.TOKEN);
+	 * ScopeProvider.instance.set(TestContextConfig.CONTEXT);
-//
+	 * SecurityTokenProvider.instance.set(TestContextConfig.TOKEN);
-//			ProjectsCaller clientProjects = GeoportalClientCaller.projects();
+	 * 
-//			Project theProject = clientProjects.getProjectByID(profileID, projectID);
+	 * ProjectsCaller clientProjects = GeoportalClientCaller.projects(); Project
-//			Document asDocument = SerializationUtil.asDocument(theProject);
+	 * theProject = clientProjects.getProjectByID(profileID, projectID); Document
-//
+	 * asDocument = SerializationUtil.asDocument(theProject);
-//			
+	 * 
-//			UseCaseDescriptor ucd = GeoportalClientCaller.useCaseDescriptors().getUCDForId(profileID);
+	 * UseCaseDescriptor ucd =
-//			
+	 * GeoportalClientCaller.useCaseDescriptors().getUCDForId(profileID);
-//			
+	 * 
-//			System.out.println("Source JSON:");
+	 * System.out.println("Source JSON:");
-//			System.out.println(prettyPrintUsingGson(asDocument.toJson()));
+	 * System.out.println(prettyPrintUsingGson(asDocument.toJson()));
-//			applyMappingToCatalogue(theProject, ucd);
+	 * applyMappingToCatalogue(theProject, ucd);
-//
+	 * 
-//		} catch (Exception e) {
+	 * } catch (Exception e) { // TODO: handle exception }
-//			// TODO: handle exception
+	 * 
-//		}
+	 * }
-//
+	 */
-//	}
-	
-	public static void main(String[] args) throws JsonProcessingException, JSONException {
-		
-		StringWriter stringWriter = new StringWriter();
-		stringWriter.append("{\"extras\":[]}");
-		String toCatalogueJSON = stringWriter.toString();
-		System.out.println("\n\ntoCatalogueString:" + toCatalogueJSON);
 
-		com.jayway.jsonpath.Configuration configuration = com.jayway.jsonpath.Configuration.builder()
+	public static void main(String[] args) {
-				.jsonProvider(new JsonOrgJsonProvider()).build();
 		
-		// GIS LINK
+		TestContextConfig.readContextSettings();
-		JSONObject theGisLinkObj = new JSONObject();
+		String sourceScope = TestContextConfig.CONTEXT;
-		theGisLinkObj.put("key", "Gis Link");
+		String sourceToken = TestContextConfig.TOKEN;
-		theGisLinkObj.put("value","https://data.dev.d4science.org/test/my_gis_link");
-		DocumentContext theDoc = JsonPath.parse(toCatalogueJSON, configuration);
-		
-		System.out.println("json: "+theGisLinkObj.toString());
-		
-		theDoc.add("$.extras", theGisLinkObj);
 
-		// system:type
+		String targetScope = "/gcube/devsec/devVRE";
-		JSONObject theSystemType = new JSONObject();
+		String targetToken = "";
-		theSystemType.put("key", "system:type");
-		theSystemType.put("value", "ucd name");
-		
-		System.out.println("json: "+theSystemType);
-		
-		theDoc.add("$.extras", theSystemType);
-		
-		
-		System.out.println(theDoc.jsonString());
 
+		publishProjectsOnCatalogue(10, sourceScope, sourceToken, targetScope, targetToken);
+		
+		//purgeDatasets(10, 0);
 	}
 
-	public void publishProjectsOnCatalogue(int limit) {
+	/**
+	 * Harvesting from scope and copy to another scope
+	 * 
+	 * @param limit
+	 */
+	public static void publishProjectsOnCatalogue(int limit, String sourceScope, String sourceToken, String targetScope,
+			String targetToken) {
 
-		
+		TestContextConfig.readContextSettings();
-		List<Project> list;
+
+		List<Project> list = new ArrayList<Project>();
 		try {
-			list = getListProejcts();
+			ScopeProvider.instance.set(sourceScope);
-			UseCaseDescriptor ucd = GeoportalClientCaller.useCaseDescriptors().getUCDForId(profileID);
+			SecurityTokenProvider.instance.set(sourceToken);
 			
-			limit = limit < list.size() ? limit : list.size();
+			//Single Project
+//			ProjectsCaller clientProjects = GeoportalClientCaller.projects(); 
+//			Project project = clientProjects.getProjectByID(profileID, projectID); 
+//			list.add(project);
+			
+			list = getListProjects(limit, 0);
+
+			UseCaseDescriptor ucd = GeoportalClientCaller.useCaseDescriptors().getUCDForId(profileID);
 
 			for (int i = 0; i < limit; i++) {
 				Project theProject = list.get(i);
-				applyMappingToCatalogue(theProject, ucd);
+				System.out.println("\n\n######Publishing the PROJECT N. "+i);
+
+				Template fmTemplate = getLocalFreemarkerTemplate();
+
+				ScopeProvider.instance.set(sourceScope);
+				SecurityTokenProvider.instance.set(sourceToken);
+				String toCatalogueJSON = MappingToCatalogue.apply(fmTemplate, theProject, ucd, sourceScope);
+
+				ScopeProvider.instance.set(targetScope);
+				SecurityTokenProvider.instance.set(targetToken);
+				new GCatCaller().createOrUpdateTheDatasetOnCatalogue(theProject.getId(), toCatalogueJSON);
+				
+				System.out.println("\n###### END Publishing the PROJECT N. "+i);
+				
+				System.out.println("\n###### sleeping "+i);
+				Thread.sleep(5000);
 			}
+			
+			System.out.println("\ndone!!!!");
 
 		} catch (Exception e) {
 			// TODO Auto-generated catch block
 			e.printStackTrace();
 		}
 	}
+	
+	/**
+	 * Harvesting from scope and copy to another scope
+	 * 
+	 * @param limit
+	 */
+	/*public static void purgeDatasets(int limit, int offset) {
 
-	public List<Project> getListProejcts() throws Exception {
 		TestContextConfig.readContextSettings();
+		
+		List<Project> list;
+		try {
+			ScopeProvider.instance.set(TestContextConfig.CONTEXT);
+			SecurityTokenProvider.instance.set(TestContextConfig.TOKEN);
+			String items = new GCatCaller().list(limit, offset);
+			System.out.println("items: "+items);
+			
+			JSONArray array = new JSONArray(items);
+			for (int i=0; i<array.length(); i++) {
+				String datasetName = array.getString(i);
+				System.out.println(i+") name : "+datasetName);
+				ScopeProvider.instance.set(TestContextConfig.CONTEXT);
+				SecurityTokenProvider.instance.set(TestContextConfig.TOKEN);
+				new GCatCaller().deleteDatasetOnCatalogue(datasetName);
+				
+				Thread.sleep(5000);
+			}
+			
+			System.out.println("done!");
 
-		ScopeProvider.instance.set(TestContextConfig.CONTEXT);
+		} catch (Exception e) {
-		SecurityTokenProvider.instance.set(TestContextConfig.TOKEN);
+			// TODO Auto-generated catch block
+			e.printStackTrace();
+		}
+	}*/
 
+	public static List<Project> getListProjects() throws Exception {
 		ProjectsCaller clientProjects = GeoportalClientCaller.projects();
 		return clientProjects.getListForProfileID(profileID);
 	}
 
+	public static List<Project> getListProjects(int limit, int offset) throws Exception {
+		ProjectsCaller clientProjects = GeoportalClientCaller.projects();
+		SearchingFilter filter = new SearchingFilter();
+		filter.setOrder(ORDER.DESC);
+		List<ItemFieldDV> orederByFields = new ArrayList<ItemFieldDV>();
+		ItemFieldDV field = new ItemFieldDV("Updated", Arrays.asList("_info._lastEditInfo._instant"), "$or", true, true, true);
+		orederByFields.add(field);
+		filter.setOrderByFields(orederByFields);
+		Iterator<Project> iterator = clientProjects.queryOnMongo(profileID, 300, offset, limit, filter);
+		List<Project> listP = new ArrayList<Project>();
+		while (iterator.hasNext()) {
+
+			listP.add(iterator.next());
+		}
+
+		return listP;
+
+	}
+
 	public static Map<String, Object> fmtJsonUtil(String json)
 			throws JsonParseException, JsonMappingException, IOException {
 		ObjectMapper OBJECT_MAPPER = new ObjectMapper();
@@ -144,36 +216,34 @@ public class GeoportalToCatalogue {
 
 	}
 
-	public static void applyMappingToCatalogue(Project theProject, UseCaseDescriptor ucd) throws JsonProcessingException {
+	public static Template getLocalFreemarkerTemplate() throws IOException {
-		
-		
-		Document asDocument = SerializationUtil.asDocument(theProject);
-		
-		String json = asDocument.toJson();
 
 		// Create your Configuration instance, and specify if up to what FreeMarker
 		// version (here 2.3.32) do you want to apply the fixes that are not 100%
 		// backward-compatible. See the Configuration JavaDoc for details.
 		Configuration cfg = new Configuration(Configuration.VERSION_2_3_32);
 		System.out.println("Working Directory = " + System.getProperty("user.dir"));
+		// Specify the source where the template files come from. Here I set a
+		// plain directory for it, but non-file-system sources are possible too:
+		cfg.setDirectoryForTemplateLoading(
+				new File(System.getProperty("user.dir") + "/src/test/java/geoportal_to_catalogue"));
+		// Set the preferred charset template files are stored in. UTF-8 is
+		// a good choice in most applications:
+		cfg.setDefaultEncoding("UTF-8");
+
+		Template template = cfg.getTemplate("d4gna_to_catalogue_template.ftl");
+		return template;
+	}
+
+	public static void applyMappingToCatalogue(Project theProject, UseCaseDescriptor ucd)
+			throws JsonProcessingException {
+
+		Document asDocument = SerializationUtil.asDocument(theProject);
+		String json = asDocument.toJson();
 
-		Template template;
 		try {
 
-			// Specify the source where the template files come from. Here I set a
+			Template template = getLocalFreemarkerTemplate();
-			// plain directory for it, but non-file-system sources are possible too:
-			cfg.setDirectoryForTemplateLoading(
-					new File(System.getProperty("user.dir") + "/src/test/java/geoportal_to_catalogue"));
-			
-
-			// From here we will set the settings recommended for new projects. These
-			// aren't the defaults for backward compatibilty.
-
-			// Set the preferred charset template files are stored in. UTF-8 is
-			// a good choice in most applications:
-			cfg.setDefaultEncoding("UTF-8");
-
-			template = cfg.getTemplate("d4gna_to_catalogue_template.ftl");
 
 			Map toJsonMap = fmtJsonUtil(json);
 			File tempFile = writeTempFile(json);
@@ -191,14 +261,14 @@ public class GeoportalToCatalogue {
 
 			com.jayway.jsonpath.Configuration configuration = com.jayway.jsonpath.Configuration.builder()
 					.jsonProvider(new JsonOrgJsonProvider()).build();
-			
+
 			DocumentContext theDoc = JsonPath.parse(toCatalogueJSON, configuration);
 			// GIS LINK
 			JSONObject theGisLinkObj = JSONObjectOrdered.instance();
 			theGisLinkObj.put("key", "Gis Link");
 			theGisLinkObj.put("value", "https://data.dev.d4science.org/test/my_gis_link");
 			theDoc.add("$.extras", theGisLinkObj);
-			
+
 			// system:type
 			JSONObject theSystemType = JSONObjectOrdered.instance();
 			theSystemType.put("key", "system:type");

catalogue-binding-plugin/src/test/java/geoportal_to_catalogue/d4gna_to_catalogue_template.ftl

@@ -13,15 +13,22 @@
 <#return "">  
 </#function>
 
+<#function onlyAlphanumericString(value)>
+<#if value??>
+  <#return value?trim?replace('\n','')?replace('\"','')?replace("[^\\w]+", "-", "r")>
+</#if>
+<#return "">  
+</#function>
+
 <#-- Macro to assign Tags -->
 <#macro assignTag in_tags>
 <#if in_tags??>
 	<#if in_tags?is_sequence>
 	  <#list in_tags as my_tag>
-		<#assign tags = tags +  [my_tag] />
+		<#assign tags = tags +  [onlyAlphanumericString(my_tag)] />
 	  </#list>
 	 <#else>
-	  	<#assign tags = tags +  [in_tags] />
+	  	<#assign tags = tags +  [onlyAlphanumericString(in_tags)] />
 	 </#if>
 	</#if>
 </#macro>