From 53a60b110e9ba76bf4c7232e8f75abca85ecde0c Mon Sep 17 00:00:00 2001 From: "francesco.mangiacrapa" Date: Fri, 21 Jun 2024 18:23:16 +0200 Subject: [PATCH] in progress on creating template #26454 --- .../cataloguebinding/doaction/GCatCaller.java | 14 + .../freemarker/MappingToCatalogue.java | 8 +- .../cms/cataloguebinding/gis/GisClient.java | 4 +- .../GeoportalToCatalogue.java | 248 +++++++++++------- .../d4gna_to_catalogue_template.ftl | 11 +- 5 files changed, 189 insertions(+), 96 deletions(-) diff --git a/catalogue-binding-plugin/src/main/java/org/gcube/application/cms/cataloguebinding/doaction/GCatCaller.java b/catalogue-binding-plugin/src/main/java/org/gcube/application/cms/cataloguebinding/doaction/GCatCaller.java index 7058859..3d087c9 100644 --- a/catalogue-binding-plugin/src/main/java/org/gcube/application/cms/cataloguebinding/doaction/GCatCaller.java +++ b/catalogue-binding-plugin/src/main/java/org/gcube/application/cms/cataloguebinding/doaction/GCatCaller.java @@ -48,6 +48,7 @@ public class GCatCaller { else log.warn("Dataset not created!"); } else { + log.info("calling update dataset..."); String itemCreated = new Item().update(datasetName, datasetJSON); if (itemCreated != null) log.info("Dataset updated with success!"); @@ -91,4 +92,17 @@ public class GCatCaller { log.error("Error occurred on deleting the dataset " + datasetName + " on the catalogue", e); } } + + public String list(int limit, int offset) { + log.info("list called"); + + try { + return new Item().list(limit, offset); + } catch (WebApplicationException | MalformedURLException e) { + log.error("Error occurred on listing datasets", e); + } + + return null; + + } } diff --git a/catalogue-binding-plugin/src/main/java/org/gcube/application/cms/cataloguebinding/freemarker/MappingToCatalogue.java b/catalogue-binding-plugin/src/main/java/org/gcube/application/cms/cataloguebinding/freemarker/MappingToCatalogue.java index 555d979..97480bb 100644 --- a/catalogue-binding-plugin/src/main/java/org/gcube/application/cms/cataloguebinding/freemarker/MappingToCatalogue.java +++ b/catalogue-binding-plugin/src/main/java/org/gcube/application/cms/cataloguebinding/freemarker/MappingToCatalogue.java @@ -75,8 +75,8 @@ public class MappingToCatalogue { log.info("template applied with success.."); String toCatalogueDatasetJSON = stringWriter.toString(); - if (log.isDebugEnabled()) { - log.debug("Template applyed with result: " + toCatalogueDatasetJSON); + if (log.isTraceEnabled()) { + log.trace("Template applyed with result: " + toCatalogueDatasetJSON); } // Parsing the Catalogue Dataset json as Document Context @@ -114,8 +114,8 @@ public class MappingToCatalogue { String catalogueDatasetJSON = prettyPrintJsonUtil(theDoc.jsonString()); - if (log.isDebugEnabled()) { - log.debug("To pretty print JSON: \n" + catalogueDatasetJSON); + if (log.isTraceEnabled()) { + log.trace("To pretty print JSON: \n" + catalogueDatasetJSON); } log.info("returning catalogue dataset, json lenght is: " + catalogueDatasetJSON.length()); diff --git a/catalogue-binding-plugin/src/main/java/org/gcube/application/cms/cataloguebinding/gis/GisClient.java b/catalogue-binding-plugin/src/main/java/org/gcube/application/cms/cataloguebinding/gis/GisClient.java index 9a24ed5..af92ba0 100644 --- a/catalogue-binding-plugin/src/main/java/org/gcube/application/cms/cataloguebinding/gis/GisClient.java +++ b/catalogue-binding-plugin/src/main/java/org/gcube/application/cms/cataloguebinding/gis/GisClient.java @@ -25,7 +25,9 @@ public class GisClient { builder.scope(scope); builder.resolverAs(resolveAs); - builder.targetApp(targetApp); + if(targetApp!=null) + builder.targetApp(targetApp); + Map params = builder.buildQueryParameters(); theLink = uriResolverManager.getLink(params, shortLink); log.info("Geoportal GisViewer link is {} ", theLink); diff --git a/catalogue-binding-plugin/src/test/java/geoportal_to_catalogue/GeoportalToCatalogue.java b/catalogue-binding-plugin/src/test/java/geoportal_to_catalogue/GeoportalToCatalogue.java index 61e8b3e..98a9c1d 100644 --- a/catalogue-binding-plugin/src/test/java/geoportal_to_catalogue/GeoportalToCatalogue.java +++ b/catalogue-binding-plugin/src/test/java/geoportal_to_catalogue/GeoportalToCatalogue.java @@ -4,20 +4,31 @@ import java.io.File; import java.io.FileWriter; import java.io.IOException; import java.io.StringWriter; +import java.util.ArrayList; +import java.util.Arrays; import java.util.HashMap; +import java.util.Iterator; import java.util.List; import java.util.Map; import org.bson.BsonDocument; import org.bson.BsonString; import org.bson.Document; +import org.gcube.application.cms.cataloguebinding.doaction.BindingAction; +import org.gcube.application.cms.cataloguebinding.doaction.GCatCaller; +import org.gcube.application.cms.cataloguebinding.freemarker.FreemarkerConfig; +import org.gcube.application.cms.cataloguebinding.freemarker.MappingToCatalogue; import org.gcube.application.cms.cataloguebinding.util.SerializationUtil; import org.gcube.application.geoportal.common.model.document.Project; import org.gcube.application.geoportal.common.model.useCaseDescriptor.UseCaseDescriptor; import org.gcube.application.geoportalcommon.geoportal.GeoportalClientCaller; import org.gcube.application.geoportalcommon.geoportal.ProjectsCaller; +import org.gcube.application.geoportalcommon.shared.SearchingFilter; +import org.gcube.application.geoportalcommon.shared.SearchingFilter.ORDER; +import org.gcube.application.geoportalcommon.shared.geoportal.config.ItemFieldDV; import org.gcube.common.authorization.library.provider.SecurityTokenProvider; import org.gcube.common.scope.api.ScopeProvider; +import org.json.JSONArray; import org.json.JSONException; import org.json.JSONObject; @@ -42,100 +53,161 @@ import test.TestContextConfig; public class GeoportalToCatalogue { public final static String profileID = "profiledConcessioni"; - public final static String projectID = "66697afda08b3011b506ddb6"; + public final static String projectID = "666afb4ea08b3011b506de90"; + //666afb4ea08b3011b506de90 + //66697afda08b3011b506ddb6 -// public static void main(String[] args) { -// -// try { -// -// TestContextConfig.readContextSettings(); -// -// ScopeProvider.instance.set(TestContextConfig.CONTEXT); -// SecurityTokenProvider.instance.set(TestContextConfig.TOKEN); -// -// ProjectsCaller clientProjects = GeoportalClientCaller.projects(); -// Project theProject = clientProjects.getProjectByID(profileID, projectID); -// Document asDocument = SerializationUtil.asDocument(theProject); -// -// -// UseCaseDescriptor ucd = GeoportalClientCaller.useCaseDescriptors().getUCDForId(profileID); -// -// -// System.out.println("Source JSON:"); -// System.out.println(prettyPrintUsingGson(asDocument.toJson())); -// applyMappingToCatalogue(theProject, ucd); -// -// } catch (Exception e) { -// // TODO: handle exception -// } -// -// } - - public static void main(String[] args) throws JsonProcessingException, JSONException { - - StringWriter stringWriter = new StringWriter(); - stringWriter.append("{\"extras\":[]}"); - String toCatalogueJSON = stringWriter.toString(); - System.out.println("\n\ntoCatalogueString:" + toCatalogueJSON); + /* + * public static void main(String[] args) { + * + * try { + * + * TestContextConfig.readContextSettings(); + * + * ScopeProvider.instance.set(TestContextConfig.CONTEXT); + * SecurityTokenProvider.instance.set(TestContextConfig.TOKEN); + * + * ProjectsCaller clientProjects = GeoportalClientCaller.projects(); Project + * theProject = clientProjects.getProjectByID(profileID, projectID); Document + * asDocument = SerializationUtil.asDocument(theProject); + * + * UseCaseDescriptor ucd = + * GeoportalClientCaller.useCaseDescriptors().getUCDForId(profileID); + * + * System.out.println("Source JSON:"); + * System.out.println(prettyPrintUsingGson(asDocument.toJson())); + * applyMappingToCatalogue(theProject, ucd); + * + * } catch (Exception e) { // TODO: handle exception } + * + * } + */ - com.jayway.jsonpath.Configuration configuration = com.jayway.jsonpath.Configuration.builder() - .jsonProvider(new JsonOrgJsonProvider()).build(); + public static void main(String[] args) { - // GIS LINK - JSONObject theGisLinkObj = new JSONObject(); - theGisLinkObj.put("key", "Gis Link"); - theGisLinkObj.put("value","https://data.dev.d4science.org/test/my_gis_link"); - DocumentContext theDoc = JsonPath.parse(toCatalogueJSON, configuration); - - System.out.println("json: "+theGisLinkObj.toString()); - - theDoc.add("$.extras", theGisLinkObj); + TestContextConfig.readContextSettings(); + String sourceScope = TestContextConfig.CONTEXT; + String sourceToken = TestContextConfig.TOKEN; - // system:type - JSONObject theSystemType = new JSONObject(); - theSystemType.put("key", "system:type"); - theSystemType.put("value", "ucd name"); - - System.out.println("json: "+theSystemType); - - theDoc.add("$.extras", theSystemType); - - - System.out.println(theDoc.jsonString()); + String targetScope = "/gcube/devsec/devVRE"; + String targetToken = ""; + publishProjectsOnCatalogue(10, sourceScope, sourceToken, targetScope, targetToken); + + //purgeDatasets(10, 0); } - public void publishProjectsOnCatalogue(int limit) { + /** + * Harvesting from scope and copy to another scope + * + * @param limit + */ + public static void publishProjectsOnCatalogue(int limit, String sourceScope, String sourceToken, String targetScope, + String targetToken) { - - List list; + TestContextConfig.readContextSettings(); + + List list = new ArrayList(); try { - list = getListProejcts(); - UseCaseDescriptor ucd = GeoportalClientCaller.useCaseDescriptors().getUCDForId(profileID); + ScopeProvider.instance.set(sourceScope); + SecurityTokenProvider.instance.set(sourceToken); - limit = limit < list.size() ? limit : list.size(); + //Single Project +// ProjectsCaller clientProjects = GeoportalClientCaller.projects(); +// Project project = clientProjects.getProjectByID(profileID, projectID); +// list.add(project); + + list = getListProjects(limit, 0); + + UseCaseDescriptor ucd = GeoportalClientCaller.useCaseDescriptors().getUCDForId(profileID); for (int i = 0; i < limit; i++) { Project theProject = list.get(i); - applyMappingToCatalogue(theProject, ucd); + System.out.println("\n\n######Publishing the PROJECT N. "+i); + + Template fmTemplate = getLocalFreemarkerTemplate(); + + ScopeProvider.instance.set(sourceScope); + SecurityTokenProvider.instance.set(sourceToken); + String toCatalogueJSON = MappingToCatalogue.apply(fmTemplate, theProject, ucd, sourceScope); + + ScopeProvider.instance.set(targetScope); + SecurityTokenProvider.instance.set(targetToken); + new GCatCaller().createOrUpdateTheDatasetOnCatalogue(theProject.getId(), toCatalogueJSON); + + System.out.println("\n###### END Publishing the PROJECT N. "+i); + + System.out.println("\n###### sleeping "+i); + Thread.sleep(5000); } + + System.out.println("\ndone!!!!"); } catch (Exception e) { // TODO Auto-generated catch block e.printStackTrace(); } } + + /** + * Harvesting from scope and copy to another scope + * + * @param limit + */ + /*public static void purgeDatasets(int limit, int offset) { - public List getListProejcts() throws Exception { TestContextConfig.readContextSettings(); + + List list; + try { + ScopeProvider.instance.set(TestContextConfig.CONTEXT); + SecurityTokenProvider.instance.set(TestContextConfig.TOKEN); + String items = new GCatCaller().list(limit, offset); + System.out.println("items: "+items); + + JSONArray array = new JSONArray(items); + for (int i=0; i getListProjects() throws Exception { ProjectsCaller clientProjects = GeoportalClientCaller.projects(); return clientProjects.getListForProfileID(profileID); } + public static List getListProjects(int limit, int offset) throws Exception { + ProjectsCaller clientProjects = GeoportalClientCaller.projects(); + SearchingFilter filter = new SearchingFilter(); + filter.setOrder(ORDER.DESC); + List orederByFields = new ArrayList(); + ItemFieldDV field = new ItemFieldDV("Updated", Arrays.asList("_info._lastEditInfo._instant"), "$or", true, true, true); + orederByFields.add(field); + filter.setOrderByFields(orederByFields); + Iterator iterator = clientProjects.queryOnMongo(profileID, 300, offset, limit, filter); + List listP = new ArrayList(); + while (iterator.hasNext()) { + + listP.add(iterator.next()); + } + + return listP; + + } + public static Map fmtJsonUtil(String json) throws JsonParseException, JsonMappingException, IOException { ObjectMapper OBJECT_MAPPER = new ObjectMapper(); @@ -144,36 +216,34 @@ public class GeoportalToCatalogue { } - public static void applyMappingToCatalogue(Project theProject, UseCaseDescriptor ucd) throws JsonProcessingException { - - - Document asDocument = SerializationUtil.asDocument(theProject); - - String json = asDocument.toJson(); + public static Template getLocalFreemarkerTemplate() throws IOException { // Create your Configuration instance, and specify if up to what FreeMarker // version (here 2.3.32) do you want to apply the fixes that are not 100% // backward-compatible. See the Configuration JavaDoc for details. Configuration cfg = new Configuration(Configuration.VERSION_2_3_32); System.out.println("Working Directory = " + System.getProperty("user.dir")); + // Specify the source where the template files come from. Here I set a + // plain directory for it, but non-file-system sources are possible too: + cfg.setDirectoryForTemplateLoading( + new File(System.getProperty("user.dir") + "/src/test/java/geoportal_to_catalogue")); + // Set the preferred charset template files are stored in. UTF-8 is + // a good choice in most applications: + cfg.setDefaultEncoding("UTF-8"); + + Template template = cfg.getTemplate("d4gna_to_catalogue_template.ftl"); + return template; + } + + public static void applyMappingToCatalogue(Project theProject, UseCaseDescriptor ucd) + throws JsonProcessingException { + + Document asDocument = SerializationUtil.asDocument(theProject); + String json = asDocument.toJson(); - Template template; try { - // Specify the source where the template files come from. Here I set a - // plain directory for it, but non-file-system sources are possible too: - cfg.setDirectoryForTemplateLoading( - new File(System.getProperty("user.dir") + "/src/test/java/geoportal_to_catalogue")); - - - // From here we will set the settings recommended for new projects. These - // aren't the defaults for backward compatibilty. - - // Set the preferred charset template files are stored in. UTF-8 is - // a good choice in most applications: - cfg.setDefaultEncoding("UTF-8"); - - template = cfg.getTemplate("d4gna_to_catalogue_template.ftl"); + Template template = getLocalFreemarkerTemplate(); Map toJsonMap = fmtJsonUtil(json); File tempFile = writeTempFile(json); @@ -191,14 +261,14 @@ public class GeoportalToCatalogue { com.jayway.jsonpath.Configuration configuration = com.jayway.jsonpath.Configuration.builder() .jsonProvider(new JsonOrgJsonProvider()).build(); - + DocumentContext theDoc = JsonPath.parse(toCatalogueJSON, configuration); // GIS LINK JSONObject theGisLinkObj = JSONObjectOrdered.instance(); theGisLinkObj.put("key", "Gis Link"); theGisLinkObj.put("value", "https://data.dev.d4science.org/test/my_gis_link"); theDoc.add("$.extras", theGisLinkObj); - + // system:type JSONObject theSystemType = JSONObjectOrdered.instance(); theSystemType.put("key", "system:type"); diff --git a/catalogue-binding-plugin/src/test/java/geoportal_to_catalogue/d4gna_to_catalogue_template.ftl b/catalogue-binding-plugin/src/test/java/geoportal_to_catalogue/d4gna_to_catalogue_template.ftl index 11508a9..be32e32 100644 --- a/catalogue-binding-plugin/src/test/java/geoportal_to_catalogue/d4gna_to_catalogue_template.ftl +++ b/catalogue-binding-plugin/src/test/java/geoportal_to_catalogue/d4gna_to_catalogue_template.ftl @@ -13,15 +13,22 @@ <#return ""> +<#function onlyAlphanumericString(value)> +<#if value??> + <#return value?trim?replace('\n','')?replace('\"','')?replace("[^\\w]+", "-", "r")> + +<#return ""> + + <#-- Macro to assign Tags --> <#macro assignTag in_tags> <#if in_tags??> <#if in_tags?is_sequence> <#list in_tags as my_tag> - <#assign tags = tags + [my_tag] /> + <#assign tags = tags + [onlyAlphanumericString(my_tag)] /> <#else> - <#assign tags = tags + [in_tags] /> + <#assign tags = tags + [onlyAlphanumericString(in_tags)] />