gCubeSystem
/
gcube-cms-suite
14
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

First release, implemented the plugin [#26454]

This commit is contained in:
Francesco Mangiacrapa 2024-06-26 15:17:54 +02:00
parent 987dc4f378
commit daf6f65f48
10 changed files with 775 additions and 452 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
catalogue-binding-plugin/src/main/java/org/gcube/application/cms/cataloguebinding/CatalogueBindingPlugin.java
View File

@ -51,6 +51,7 @@ public class CatalogueBindingPlugin extends AbstractPlugin implements EventListe
public static final PluginDescriptor DESCRIPTOR = new PluginDescriptor(PLUGIN_ID, PLUGIN_TYPE);
private static final String FREEMARKER_TEMPLATE_PATH = "freemarker_template_path";
private static final String FREEMARKER_TEMPLATE_HOST = "freemarker_template_host";
private static final String CONFIG_ENABLED = "enabled";
static {
DESCRIPTOR.setVersion(new Semver("1.0.0"));
@ -233,6 +234,11 @@ public class CatalogueBindingPlugin extends AbstractPlugin implements EventListe
Document profileConfiguration = getConfigurationFromProfile(useCaseDescriptor).getConfiguration();
log.debug("UseCaseDescriptor Configuration is {} ", profileConfiguration);
boolean configEnabled = profileConfiguration.getBoolean(CONFIG_ENABLED, false);
log.info("Is Config enabled? {} ", CONFIG_ENABLED, configEnabled);
if(configEnabled) {
for (Object fsConfigObj : profileConfiguration.get(SUBSCRIBE_EVENTS_CONFIG, List.class)) {
log.debug("Managing {} ", fsConfigObj);
SubscribeEventsConfig fsConfig = SerializationUtil.convert(fsConfigObj,
@ -256,6 +262,7 @@ public class CatalogueBindingPlugin extends AbstractPlugin implements EventListe
eventsSubscrInTheUCD.add(eventSub);
}
}
String freemarkerTPath = profileConfiguration.get(FREEMARKER_TEMPLATE_PATH, String.class);
String freemakerTHost = profileConfiguration.get(FREEMARKER_TEMPLATE_HOST, String.class);

4
catalogue-binding-plugin/src/main/java/org/gcube/application/cms/cataloguebinding/config/CatalogueBindingPluginConfigModel.java
View File

@ -9,11 +9,13 @@ import lombok.Data;
@Data
public class CatalogueBindingPluginConfigModel {
String context; //the scope
String context; // the scope
List<EventSubscribed> listEventSubscribed;
String freemarker_template_path;
String freemarker_template_host;
Boolean enabled = false;
}

29
catalogue-binding-plugin/src/main/java/org/gcube/application/cms/cataloguebinding/doaction/BindingAction.java
View File

@ -1,7 +1,6 @@
package org.gcube.application.cms.cataloguebinding.doaction;
import java.io.IOException;
import java.net.MalformedURLException;
import java.util.List;
import org.gcube.application.cms.cataloguebinding.EventSubscribed;
@ -12,11 +11,8 @@ import org.gcube.application.cms.plugins.events.ItemObserved;
import org.gcube.application.geoportal.common.model.document.Project;
import org.gcube.application.geoportal.common.model.document.lifecycle.LifecycleInformation;
import freemarker.core.ParseException;
import freemarker.template.MalformedTemplateNameException;
import freemarker.template.Template;
import freemarker.template.TemplateException;
import freemarker.template.TemplateNotFoundException;
import lombok.extern.slf4j.Slf4j;
@Slf4j
@ -43,11 +39,14 @@ public class BindingAction {
// The project id is the dataset name
String datasetName = itemObserved.getProjectId();
try {
switch (eventSubscribed.getEvent()) {
case PROJECT_CREATED: {
log.info("Going to create item with id {} on the catalogue...", datasetName);
// Create or Update the item on the catalogue
String catalogueItemJSON = toCatalogueItem();
if (catalogueItemJSON != null)
new GCatCaller().createOrUpdateTheDatasetOnCatalogue(datasetName, catalogueItemJSON);
break;
}
@ -60,6 +59,7 @@ public class BindingAction {
log.info("Going to update item with id {} on the catalogue...", datasetName);
// Create or Update the item on the catalogue
String catalogueItemJSON = toCatalogueItem();
if (catalogueItemJSON != null)
new GCatCaller().createOrUpdateTheDatasetOnCatalogue(datasetName, catalogueItemJSON);
break;
}
@ -79,7 +79,8 @@ public class BindingAction {
continue;
}
if (bindingWhen.getLast_invoked_step() == null || bindingWhen.getLast_invoked_step().length() == 0) {
if (bindingWhen.getLast_invoked_step() == null
|| bindingWhen.getLast_invoked_step().length() == 0) {
log.warn("No binding on last invoked step, so returning!!");
continue;
}
@ -100,8 +101,10 @@ public class BindingAction {
if (itemPhase.equalsIgnoreCase(TARGET_PHASE_PUBLISHED)) {
String catalogueItemJSON = toCatalogueItem();
if (catalogueItemJSON != null) {
log.info("Going to create item with name {} on the catalogue...", datasetName);
new GCatCaller().createOrUpdateTheDatasetOnCatalogue(datasetName, catalogueItemJSON);
}
} else if (itemPhase.equalsIgnoreCase(TARGET_PHASE_DRAFT)) {
// Delete the item on the catalogue
@ -122,6 +125,10 @@ public class BindingAction {
default:
break;
}
} catch (Exception e) {
log.error("Error on applying binding action: ", e);
}
}
public String toCatalogueItem() {
@ -135,17 +142,7 @@ public class BindingAction {
toCatalogueJSON = MappingToCatalogue.apply(fmTemplate, itemObserved.getProject(),
itemObserved.getUseCaseDescriptor(), itemObserved.getContext().getId());
} catch (MalformedURLException e) {
log.error("Error: ", e);
} catch (TemplateException e) {
log.error("Error: ", e);
} catch (TemplateNotFoundException e) {
log.error("Error: ", e);
} catch (MalformedTemplateNameException e) {
log.error("Error: ", e);
} catch (ParseException e) {
log.error("Error: ", e);
} catch (IOException e) {
} catch (TemplateException | IOException e) {
log.error("Error: ", e);
}

4
catalogue-binding-plugin/src/main/java/org/gcube/application/cms/cataloguebinding/doaction/GCatCaller.java
View File

@ -26,8 +26,9 @@ public class GCatCaller {
*
* @param datasetName the dataset name
* @param datasetJSON the dataset JSON
* @throws Exception
*/
public void createOrUpdateTheDatasetOnCatalogue(String datasetName, String datasetJSON) {
public void createOrUpdateTheDatasetOnCatalogue(String datasetName, String datasetJSON) throws Exception {
log.info("createOrUpdateTheDatasetOnCatalogue with name {} called", datasetName);
try {
@ -58,6 +59,7 @@ public class GCatCaller {
} catch (WebApplicationException | MalformedURLException e) {
log.error("Error occurred on creating the dataset " + datasetName + " on the catalogue", e);
throw e;
}
}

42
catalogue-binding-plugin/src/main/java/org/gcube/application/cms/cataloguebinding/freemarker/MappingToCatalogue.java
View File

@ -17,9 +17,9 @@ import org.json.JSONException;
import org.json.JSONObject;
import com.fasterxml.jackson.core.JsonParseException;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.JsonMappingException;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.SerializationFeature;
import com.jayway.jsonpath.DocumentContext;
@ -51,10 +51,11 @@ public class MappingToCatalogue {
* @param ucd the ucd
* @param scope the scope
* @return the string
* @throws JsonProcessingException the json processing exception
* @throws TemplateException
* @throws IOException
*/
public static String apply(Template fmTemplate, Project theProject, UseCaseDescriptor ucd, String scope)
throws JsonProcessingException {
throws TemplateException, IOException {
log.info("Apply template {} to project with id '{}' and ucd id '{}' called", fmTemplate.getName(),
theProject.getId(), ucd.getId());
@ -86,6 +87,8 @@ public class MappingToCatalogue {
log.info("Adding service fields..");
// Adding GIS LINK to doc
// TODO ACTIVATE THIS FOR GENERATING GIS LINK
JSONObject theGisLinkObj = JSONObjectOrdered.instance();
try {
String link = GisClient.gisLink(RESOLVE_AS.PUBLIC, ucd.getId(), theProject.getId(), scope,
@ -96,8 +99,7 @@ public class MappingToCatalogue {
theDoc.add("$.extras", theGisLinkObj);
} catch (JSONException e) {
String error = "Error occurrend on adding " + GIS_LINK;
log.warn(error);
log.debug(error, e);
log.warn(error, e);
}
// Adding system:type to doc
@ -108,26 +110,22 @@ public class MappingToCatalogue {
theDoc.add("$.extras", theSystemType);
} catch (JSONException e) {
String error = "Error occurrend on adding " + SYSTEM_TYPE;
log.warn(error);
log.debug(error, e);
log.warn(error, e);
}
String catalogueDatasetJSON = prettyPrintJsonUtil(theDoc.jsonString());
if (log.isTraceEnabled()) {
log.trace("To pretty print JSON: \n" + catalogueDatasetJSON);
}
String catalogueDatasetJSON = prettyPrintJsonUtil(theDoc.jsonString(), false);
log.info("To catalogue JSON: \n" + catalogueDatasetJSON);
log.info("returning catalogue dataset, json lenght is: " + catalogueDatasetJSON.length());
return catalogueDatasetJSON;
} catch (IOException e) {
log.error("Error on apply mapping to catalogue: ", e);
log.error(IOException.class.getName() + " on apply mapping to catalogue: ", e);
throw e;
} catch (TemplateException e) {
log.error("Error on apply mapping to catalogue: ", e);
log.error(TemplateException.class.getName() + " on apply mapping to catalogue: ", e);
throw e;
}
return null;
}
/**
@ -153,12 +151,20 @@ public class MappingToCatalogue {
* @param uglyJsonString the ugly json string
* @return the string
*/
public static String prettyPrintJsonUtil(String uglyJsonString) {
ObjectMapper mapper = new ObjectMapper().enable(SerializationFeature.INDENT_OUTPUT);
public static String prettyPrintJsonUtil(String uglyJsonString, boolean indent) {
ObjectMapper mapper = new ObjectMapper();
if (indent)
mapper.enable(SerializationFeature.INDENT_OUTPUT);
Object jsonObject;
String prettyJson = "";
try {
if (indent)
jsonObject = mapper.readValue(uglyJsonString, Object.class);
else
jsonObject = mapper.readValue(uglyJsonString, JsonNode.class);
prettyJson = mapper.writeValueAsString(jsonObject);
} catch (IOException e) {
return uglyJsonString;

107
catalogue-binding-plugin/src/test/java/geoportal_to_catalogue/CataloguePurgeDatasets.java Normal file
View File

@ -0,0 +1,107 @@
package geoportal_to_catalogue;
import org.gcube.application.cms.cataloguebinding.doaction.GCatCaller;
import org.gcube.common.authorization.library.provider.SecurityTokenProvider;
import org.gcube.common.scope.api.ScopeProvider;
import org.json.JSONArray;
import org.json.JSONException;
import test.TestContextConfig;
public class CataloguePurgeDatasets {
public static void main(String[] args) {
TestContextConfig.readContextSettings();
System.out.println("CONTEXT: " + TestContextConfig.CONTEXT);
System.out.println("TOKEN: " + TestContextConfig.TOKEN);
int limit = 10;
int offset = 0;
boolean resultExists = true;
while (resultExists) {
System.out.println("offest: " + offset + ", limit: " + limit);
ScopeProvider.instance.set(TestContextConfig.CONTEXT);
SecurityTokenProvider.instance.set(TestContextConfig.TOKEN);
String items = new GCatCaller().list(limit, offset);
System.out.println("items: " + items);
if (items != null) {
resultExists = true;
JSONArray array;
try {
array = new JSONArray(items);
if (array.length() > 0) {
purgeDatasets(array);
} else
resultExists = false;
} catch (JSONException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
offset = limit + offset;
} else
resultExists = false;
}
System.out.println("done!");
}
public static void purgeDatasets(JSONArray array) {
try {
int index = 0;
int sleepingTime = 4000;
for (int i = 0; i < array.length(); i++) {
index++;
String datasetName = array.getString(i);
System.out.println(index + ") Deleting dataset name : " + datasetName);
ScopeProvider.instance.set(TestContextConfig.CONTEXT);
SecurityTokenProvider.instance.set(TestContextConfig.TOKEN);
new GCatCaller().deleteDatasetOnCatalogue(datasetName);
System.out.println("sleeping..." + sleepingTime);
Thread.sleep(sleepingTime);
}
System.out.println("done!");
} catch (Exception e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
public static void purgeDatasets(int limit, int offset) {
TestContextConfig.readContextSettings();
try {
ScopeProvider.instance.set(TestContextConfig.CONTEXT);
SecurityTokenProvider.instance.set(TestContextConfig.TOKEN);
String items = new GCatCaller().list(limit, offset);
System.out.println("items: " + items);
JSONArray array = new JSONArray(items);
for (int i = 0; i < array.length(); i++) {
String datasetName = array.getString(i);
System.out.println(i + ") name : " + datasetName);
ScopeProvider.instance.set(TestContextConfig.CONTEXT);
SecurityTokenProvider.instance.set(TestContextConfig.TOKEN);
new GCatCaller().deleteDatasetOnCatalogue(datasetName);
Thread.sleep(5000);
}
System.out.println("done!");
} catch (Exception e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
}

310
catalogue-binding-plugin/src/test/java/geoportal_to_catalogue/GeoportalToCatalogue.java
View File

@ -1,310 +0,0 @@
package geoportal_to_catalogue;
import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.io.StringWriter;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import org.bson.BsonDocument;
import org.bson.BsonString;
import org.bson.Document;
import org.gcube.application.cms.cataloguebinding.doaction.BindingAction;
import org.gcube.application.cms.cataloguebinding.doaction.GCatCaller;
import org.gcube.application.cms.cataloguebinding.freemarker.FreemarkerConfig;
import org.gcube.application.cms.cataloguebinding.freemarker.MappingToCatalogue;
import org.gcube.application.cms.cataloguebinding.util.SerializationUtil;
import org.gcube.application.geoportal.common.model.document.Project;
import org.gcube.application.geoportal.common.model.useCaseDescriptor.UseCaseDescriptor;
import org.gcube.application.geoportalcommon.geoportal.GeoportalClientCaller;
import org.gcube.application.geoportalcommon.geoportal.ProjectsCaller;
import org.gcube.application.geoportalcommon.shared.SearchingFilter;
import org.gcube.application.geoportalcommon.shared.SearchingFilter.ORDER;
import org.gcube.application.geoportalcommon.shared.geoportal.config.ItemFieldDV;
import org.gcube.common.authorization.library.provider.SecurityTokenProvider;
import org.gcube.common.scope.api.ScopeProvider;
import org.json.JSONArray;
import org.json.JSONException;
import org.json.JSONObject;
import com.fasterxml.jackson.core.JsonParseException;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.JsonMappingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
import com.google.gson.JsonElement;
import com.google.gson.JsonParser;
import com.jayway.jsonpath.DocumentContext;
import com.jayway.jsonpath.JsonPath;
import com.jayway.jsonpath.spi.json.JsonOrgJsonProvider;
import freemarker.template.Configuration;
import freemarker.template.Template;
import freemarker.template.TemplateException;
import test.TestContextConfig;
public class GeoportalToCatalogue {
public final static String profileID = "profiledConcessioni";
public final static String projectID = "666afb4ea08b3011b506de90";
//666afb4ea08b3011b506de90
//66697afda08b3011b506ddb6
/*
* public static void main(String[] args) {
*
* try {
*
* TestContextConfig.readContextSettings();
*
* ScopeProvider.instance.set(TestContextConfig.CONTEXT);
* SecurityTokenProvider.instance.set(TestContextConfig.TOKEN);
*
* ProjectsCaller clientProjects = GeoportalClientCaller.projects(); Project
* theProject = clientProjects.getProjectByID(profileID, projectID); Document
* asDocument = SerializationUtil.asDocument(theProject);
*
* UseCaseDescriptor ucd =
* GeoportalClientCaller.useCaseDescriptors().getUCDForId(profileID);
*
* System.out.println("Source JSON:");
* System.out.println(prettyPrintUsingGson(asDocument.toJson()));
* applyMappingToCatalogue(theProject, ucd);
*
* } catch (Exception e) { // TODO: handle exception }
*
* }
*/
public static void main(String[] args) {
TestContextConfig.readContextSettings();
String sourceScope = TestContextConfig.CONTEXT;
String sourceToken = TestContextConfig.TOKEN;
String targetScope = "/gcube/devsec/devVRE";
String targetToken = "";
publishProjectsOnCatalogue(10, sourceScope, sourceToken, targetScope, targetToken);
//purgeDatasets(10, 0);
}
/**
* Harvesting from scope and copy to another scope
*
* @param limit
*/
public static void publishProjectsOnCatalogue(int limit, String sourceScope, String sourceToken, String targetScope,
String targetToken) {
TestContextConfig.readContextSettings();
List<Project> list = new ArrayList<Project>();
try {
ScopeProvider.instance.set(sourceScope);
SecurityTokenProvider.instance.set(sourceToken);
//Single Project
// ProjectsCaller clientProjects = GeoportalClientCaller.projects();
// Project project = clientProjects.getProjectByID(profileID, projectID);
// list.add(project);
list = getListProjects(limit, 0);
UseCaseDescriptor ucd = GeoportalClientCaller.useCaseDescriptors().getUCDForId(profileID);
for (int i = 0; i < limit; i++) {
Project theProject = list.get(i);
System.out.println("\n\n######Publishing the PROJECT N. "+i);
Template fmTemplate = getLocalFreemarkerTemplate();
ScopeProvider.instance.set(sourceScope);
SecurityTokenProvider.instance.set(sourceToken);
String toCatalogueJSON = MappingToCatalogue.apply(fmTemplate, theProject, ucd, sourceScope);
ScopeProvider.instance.set(targetScope);
SecurityTokenProvider.instance.set(targetToken);
new GCatCaller().createOrUpdateTheDatasetOnCatalogue(theProject.getId(), toCatalogueJSON);
System.out.println("\n###### END Publishing the PROJECT N. "+i);
System.out.println("\n###### sleeping "+i);
Thread.sleep(5000);
}
System.out.println("\ndone!!!!");
} catch (Exception e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
/**
* Harvesting from scope and copy to another scope
*
* @param limit
*/
/*public static void purgeDatasets(int limit, int offset) {
TestContextConfig.readContextSettings();
List<Project> list;
try {
ScopeProvider.instance.set(TestContextConfig.CONTEXT);
SecurityTokenProvider.instance.set(TestContextConfig.TOKEN);
String items = new GCatCaller().list(limit, offset);
System.out.println("items: "+items);
JSONArray array = new JSONArray(items);
for (int i=0; i<array.length(); i++) {
String datasetName = array.getString(i);
System.out.println(i+") name : "+datasetName);
ScopeProvider.instance.set(TestContextConfig.CONTEXT);
SecurityTokenProvider.instance.set(TestContextConfig.TOKEN);
new GCatCaller().deleteDatasetOnCatalogue(datasetName);
Thread.sleep(5000);
}
System.out.println("done!");
} catch (Exception e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}*/
public static List<Project> getListProjects() throws Exception {
ProjectsCaller clientProjects = GeoportalClientCaller.projects();
return clientProjects.getListForProfileID(profileID);
}
public static List<Project> getListProjects(int limit, int offset) throws Exception {
ProjectsCaller clientProjects = GeoportalClientCaller.projects();
SearchingFilter filter = new SearchingFilter();
filter.setOrder(ORDER.DESC);
List<ItemFieldDV> orederByFields = new ArrayList<ItemFieldDV>();
ItemFieldDV field = new ItemFieldDV("Updated", Arrays.asList("_info._lastEditInfo._instant"), "$or", true, true, true);
orederByFields.add(field);
filter.setOrderByFields(orederByFields);
Iterator<Project> iterator = clientProjects.queryOnMongo(profileID, 300, offset, limit, filter);
List<Project> listP = new ArrayList<Project>();
while (iterator.hasNext()) {
listP.add(iterator.next());
}
return listP;
}
public static Map<String, Object> fmtJsonUtil(String json)
throws JsonParseException, JsonMappingException, IOException {
ObjectMapper OBJECT_MAPPER = new ObjectMapper();
return OBJECT_MAPPER.readValue(json, new TypeReference<HashMap<String, Object>>() {
});
}
public static Template getLocalFreemarkerTemplate() throws IOException {
// Create your Configuration instance, and specify if up to what FreeMarker
// version (here 2.3.32) do you want to apply the fixes that are not 100%
// backward-compatible. See the Configuration JavaDoc for details.
Configuration cfg = new Configuration(Configuration.VERSION_2_3_32);
System.out.println("Working Directory = " + System.getProperty("user.dir"));
// Specify the source where the template files come from. Here I set a
// plain directory for it, but non-file-system sources are possible too:
cfg.setDirectoryForTemplateLoading(
new File(System.getProperty("user.dir") + "/src/test/java/geoportal_to_catalogue"));
// Set the preferred charset template files are stored in. UTF-8 is
// a good choice in most applications:
cfg.setDefaultEncoding("UTF-8");
Template template = cfg.getTemplate("d4gna_to_catalogue_template.ftl");
return template;
}
public static void applyMappingToCatalogue(Project theProject, UseCaseDescriptor ucd)
throws JsonProcessingException {
Document asDocument = SerializationUtil.asDocument(theProject);
String json = asDocument.toJson();
try {
Template template = getLocalFreemarkerTemplate();
Map toJsonMap = fmtJsonUtil(json);
File tempFile = writeTempFile(json);
StringWriter stringWriter = new StringWriter();
Map root = new HashMap();
root.put("doc", toJsonMap);
template.process(root, stringWriter);
String toCatalogueJSON = stringWriter.toString();
System.out.println("\n\ntoCatalogueString:" + toCatalogueJSON);
System.out.println("\nPretty printing 1");
System.out.println(prettyPrintUsingGson(toCatalogueJSON));
com.jayway.jsonpath.Configuration configuration = com.jayway.jsonpath.Configuration.builder()
.jsonProvider(new JsonOrgJsonProvider()).build();
DocumentContext theDoc = JsonPath.parse(toCatalogueJSON, configuration);
// GIS LINK
JSONObject theGisLinkObj = JSONObjectOrdered.instance();
theGisLinkObj.put("key", "Gis Link");
theGisLinkObj.put("value", "https://data.dev.d4science.org/test/my_gis_link");
theDoc.add("$.extras", theGisLinkObj);
// system:type
JSONObject theSystemType = JSONObjectOrdered.instance();
theSystemType.put("key", "system:type");
theSystemType.put("value", ucd.getName());
theDoc.add("$.extras", theSystemType);
System.out.println("\n\nTo pretty print JSON:");
// System.out.println("gis link: "+theDoc.jsonString());
System.out.println(prettyPrintUsingGson(theDoc.jsonString()));
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
} catch (TemplateException e) {
// TODO Auto-generated catch block
e.printStackTrace();
} catch (JSONException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
public static String prettyPrintUsingGson(String uglyJson) {
Gson gson = new GsonBuilder().setPrettyPrinting().create();
JsonElement jsonElement = new JsonParser().parse(uglyJson);
String prettyJsonString = gson.toJson(jsonElement);
return prettyJsonString;
}
public static File writeTempFile(String toWrite) throws IOException {
File tmpFile = File.createTempFile("test", ".tmp");
FileWriter writer = new FileWriter(tmpFile);
writer.write(toWrite);
writer.close();
return tmpFile;
}
}

487
catalogue-binding-plugin/src/test/java/geoportal_to_catalogue/GeoportalToCatalogueBatchPublisher.java Normal file
View File

@ -0,0 +1,487 @@
package geoportal_to_catalogue;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.io.PrintWriter;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.rmi.RemoteException;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Date;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.concurrent.TimeUnit;
import org.gcube.application.cms.cataloguebinding.doaction.GCatCaller;
import org.gcube.application.cms.cataloguebinding.freemarker.MappingToCatalogue;
import org.gcube.application.geoportal.common.model.configuration.Archive;
import org.gcube.application.geoportal.common.model.document.Project;
import org.gcube.application.geoportal.common.model.useCaseDescriptor.UseCaseDescriptor;
import org.gcube.application.geoportal.common.rest.Projects;
import org.gcube.application.geoportalcommon.geoportal.GeoportalClientCaller;
import org.gcube.application.geoportalcommon.geoportal.ProjectsCaller;
import org.gcube.application.geoportalcommon.shared.SearchingFilter;
import org.gcube.application.geoportalcommon.shared.SearchingFilter.LOGICAL_OP;
import org.gcube.application.geoportalcommon.shared.SearchingFilter.ORDER;
import org.gcube.application.geoportalcommon.shared.WhereClause;
import org.gcube.application.geoportalcommon.shared.geoportal.config.ItemFieldDV;
import org.gcube.common.authorization.library.provider.SecurityTokenProvider;
import org.gcube.common.scope.api.ScopeProvider;
import com.fasterxml.jackson.core.JsonParseException;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.JsonMappingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
import com.google.gson.JsonElement;
import com.google.gson.JsonParser;
import freemarker.template.Configuration;
import freemarker.template.Template;
import freemarker.template.TemplateException;
import lombok.extern.slf4j.Slf4j;
import test.TestContextConfig;
/**
* The Class GeoportalToCatalogueBatchPublisher.
*
* @author Francesco Mangiacrapa at ISTI-CNR francesco.mangiacrapa@isti.cnr.it
*
* Jun 25, 2024
*/
@Slf4j
public class GeoportalToCatalogueBatchPublisher {
public final static String profileID = "profiledConcessioni";
public final static Integer MAX_ITEMS = 100;
static PrintWriter reportPrintWriter;
static PrintWriter errorPrintWriter;
private static long startTime;
private static final String CSV_DELIMITER = ";";
/**
* The main method.
*
* @param args the arguments
*/
public static void main(String[] args) {
//procedureToPublishProjectsOnCatalogue();
String projectId = " 6663016a312dc236d217be5c";
checkMappingToJSONForProjectByID(projectId);
}
public static void checkMappingToJSONForProjectByID(String projectID) {
TestContextConfig.readContextSettings();
String sourceScope = TestContextConfig.CONTEXT;
String sourceToken = TestContextConfig.TOKEN;
ScopeProvider.instance.set(sourceScope);
SecurityTokenProvider.instance.set(sourceToken);
ProjectsCaller clientProjects = GeoportalClientCaller.projects();
Projects<Project> client = (Projects<Project>) clientProjects.getClient(profileID);
try {
UseCaseDescriptor ucd = GeoportalClientCaller.useCaseDescriptors().getUCDForId(profileID);
Project theProject = client.getById(projectID);
log.info("\n\nConverting the project {}", theProject.getId());
String toCatalogueJSON = mapToCatalogue(theProject, ucd, sourceScope, sourceToken);
String jsonPretty = prettyPrintUsingGson(toCatalogueJSON);
log.info(jsonPretty);
} catch (RemoteException e) {
// TODO Auto-generated catch block
e.printStackTrace();
} catch (Exception e1) {
// TODO Auto-generated catch block
e1.printStackTrace();
}
}
private static void procedureToPublishProjectsOnCatalogue() {
TestContextConfig.readContextSettings();
String sourceScope = TestContextConfig.CONTEXT;
String sourceToken = TestContextConfig.TOKEN;
String reportFile = TestContextConfig.CONTEXT.replaceAll("/", "_") + "_report_.csv";
String errorFile = TestContextConfig.CONTEXT.replaceAll("/", "_") + "_error_.csv";
try {
Files.deleteIfExists(Paths.get(reportFile));
Files.deleteIfExists(Paths.get(errorFile));
} catch (IOException e1) {
// TODO Auto-generated catch block
e1.printStackTrace();
}
try {
FileWriter reportWriter = new FileWriter(reportFile, true);
FileWriter errorWriter = new FileWriter(errorFile, true);
BufferedWriter reportBW = new BufferedWriter(reportWriter);
BufferedWriter errorBW = new BufferedWriter(errorWriter);
reportPrintWriter = new PrintWriter(reportBW);
reportPrintWriter.println("NB." + CSV_DELIMITER + " PROJECT_ID" + CSV_DELIMITER + " MAPPING_TO_CATALOGUE"
+ CSV_DELIMITER + "");
errorPrintWriter = new PrintWriter(errorBW);
errorPrintWriter.println("NB." + CSV_DELIMITER + " PROJECT_ID" + CSV_DELIMITER + " MAPPING_TO_CATALOGUE"
+ CSV_DELIMITER + " ERROR" + CSV_DELIMITER + "");
} catch (Exception e) {
e.printStackTrace();
}
System.out.println("Procedure starting at: "+getFormattedDateTime());
startTime = System.currentTimeMillis();
Integer totalProjects = null;
try {
ScopeProvider.instance.set(sourceScope);
SecurityTokenProvider.instance.set(sourceToken);
totalProjects = getTotalProjects(profileID);
} catch (Exception e) {
e.printStackTrace();
}
if (totalProjects == null) {
log.error("No document found!!!");
return;
}
log.info("Total projects are: " + totalProjects);
boolean documentFound = true;
boolean maxItemsReached = false;
int limit = 10;
int offset = 0;
int counter = 0;
try {
while (documentFound && !maxItemsReached) {
try {
/**
*
* ##### NB!!!! USING SOURCE SCOPE
*
*/
ScopeProvider.instance.set(sourceScope);
SecurityTokenProvider.instance.set(sourceToken);
List<Project> list = getListProjects(limit, offset);
log.info("limit {}, offset {}", limit, offset);
if (list.size() > 0) {
UseCaseDescriptor ucd = GeoportalClientCaller.useCaseDescriptors().getUCDForId(profileID);
for (int i = 0; i < list.size(); i++) {
counter++;
log.info("\n\nConuter is: " + counter);
if (MAX_ITEMS != null && counter > MAX_ITEMS) {
maxItemsReached = true;
log.info("\n###### MAX_ITEMS reached: " + maxItemsReached);
break;
}
Project theProject = list.get(i);
log.info("\n\nConverting the project {}", theProject.getId());
String toCatalogueJSON = "";
try {
toCatalogueJSON = mapToCatalogue(theProject, ucd, sourceScope, sourceToken);
/**
*
* ##### NB!!!! USING TARGET SCOPE
*
*/
String targetScope = "";
String targetToken = "";
if(targetScope==null || targetToken==null) {
throw new Exception("Check targetScope and/or targetToken!!!");
}
ScopeProvider.instance.set(targetScope);
SecurityTokenProvider.instance.set(targetToken);
log.info("\n publishOnCatalogue the PROJECT N. " + counter + " with id: "
+ theProject.getId());
publishOnCatalogue(theProject, toCatalogueJSON);
log.info("\n end publishOnCatalogue the PROJECT N. " + counter);
writeReport(counter + CSV_DELIMITER + " " + theProject.getId() + CSV_DELIMITER + "'"
+ toCatalogueJSON + "'" + CSV_DELIMITER);
log.info("\n###### sleeping " + counter);
Thread.sleep(3000);
} catch (Exception e) {
log.error("ERROR on publishing project with id: " + theProject.getId());
writeError(counter + CSV_DELIMITER + " " + theProject.getId() + CSV_DELIMITER + " '"
+ toCatalogueJSON + "'" + CSV_DELIMITER + " '" + e.getMessage() + "'"
+ CSV_DELIMITER);
}
}
} else {
documentFound = false;
}
offset = offset + limit;
} catch (Exception e) {
// TODO Auto-generated catch block
e.printStackTrace();
documentFound = false;
}
}
} catch (Exception e) {
log.error("External error");
e.printStackTrace();
} finally {
if (reportPrintWriter != null)
reportPrintWriter.close();
if (errorPrintWriter != null)
errorPrintWriter.close();
System.out.println("\n\nFINISHED!!!");
System.out.println("Procedure terimated at: "+getFormattedDateTime());
long millis = System.currentTimeMillis()-startTime;
long minutes = TimeUnit.MILLISECONDS.toMinutes(millis);
System.out.println("Done! In ms: "+(millis) + ", minutes: "+minutes);
}
}
/**
* Publish on catalogue.
*
* @param theProject the the project
* @param toCatalogueJSON the to catalogue JSON
* @throws Exception the exception
*/
public static void publishOnCatalogue(Project theProject, String toCatalogueJSON) throws Exception {
new GCatCaller().createOrUpdateTheDatasetOnCatalogue(theProject.getId(), toCatalogueJSON);
}
/**
* Map to catalogue.
*
* @param theProject the the project
* @param ucd the ucd
* @param sourceScope the source scope
* @param sourceToken the source token
* @return the string
* @throws TemplateException the template exception
* @throws IOException Signals that an I/O exception has occurred.
*/
public static String mapToCatalogue(Project theProject, UseCaseDescriptor ucd, String sourceScope,
String sourceToken) throws TemplateException, IOException {
log.info("\n\nMapping the PROJECT with id: " + theProject.getId());
ScopeProvider.instance.set(sourceScope);
SecurityTokenProvider.instance.set(sourceToken);
Template fmTemplate = getLocalFreemarkerTemplate();
String toCatalogueJSON = MappingToCatalogue.apply(fmTemplate, theProject, ucd, sourceScope);
log.trace("mapping is: " + toCatalogueJSON);
return toCatalogueJSON;
}
/**
* Gets the list projects.
*
* @return the list projects
* @throws Exception the exception
*/
public static List<Project> getListProjects() throws Exception {
ProjectsCaller clientProjects = GeoportalClientCaller.projects();
return clientProjects.getListForProfileID(profileID);
}
/**
* Gets the list projects.
*
* @param limit the limit
* @param offset the offset
* @return the list projects
* @throws Exception the exception
*/
public static List<Project> getListProjects(int limit, int offset) throws Exception {
ProjectsCaller clientProjects = GeoportalClientCaller.projects();
SearchingFilter filter = new SearchingFilter();
filter.setOrder(ORDER.DESC);
List<ItemFieldDV> orederByFields = new ArrayList<ItemFieldDV>();
// Order By
ItemFieldDV fieldUpdated = new ItemFieldDV("Updated", Arrays.asList("_info._lastEditInfo._instant"), "$or",
true, true, true);
orederByFields.add(fieldUpdated);
// Where Conditions
List<WhereClause> conditions = new ArrayList<WhereClause>();
Map<String, Object> searchInto = new HashMap<String, Object>();
searchInto.put("_lifecycleInformation._phase", "Published");
conditions.add(new WhereClause(LOGICAL_OP.OR, searchInto));
filter.setConditions(conditions);
filter.setOrderByFields(orederByFields);
Iterator<Project> iterator = clientProjects.queryOnMongo(profileID, 300, offset, limit, filter);
List<Project> listP = new ArrayList<Project>();
while (iterator.hasNext()) {
listP.add(iterator.next());
}
return listP;
}
/**
* Gets the total projects.
*
* @param profileID the profile ID
* @return the total projects
* @throws Exception the exception
*/
public static Integer getTotalProjects(String profileID) throws Exception {
System.out.println("getTotalDocument called for profileID: " + profileID);
ProjectsCaller clientProjects = GeoportalClientCaller.projects();
Projects<Project> client = (Projects<Project>) clientProjects.getClient(profileID);
org.gcube.application.geoportal.common.model.configuration.Configuration config = client.getConfiguration();
List<Archive> listArchives = config.getArchives();
for (Archive archive : listArchives) {
String theType = archive.getString("_type");
if (theType.equalsIgnoreCase(ProjectsCaller.DOCUMENT_STORE_COLLECTION)) {
String totalDocumentAre = archive.get("count").toString();
int total = Integer.parseInt(totalDocumentAre);
log.info("total docs for profileID: {}, are: {}", profileID, total);
return total;
}
}
return null;
}
/**
* Fmt json util.
*
* @param json the json
* @return the map
* @throws JsonParseException the json parse exception
* @throws JsonMappingException the json mapping exception
* @throws IOException Signals that an I/O exception has occurred.
*/
public static Map<String, Object> fmtJsonUtil(String json)
throws JsonParseException, JsonMappingException, IOException {
ObjectMapper OBJECT_MAPPER = new ObjectMapper();
return OBJECT_MAPPER.readValue(json, new TypeReference<HashMap<String, Object>>() {
});
}
/**
* Gets the local freemarker template.
*
* @return the local freemarker template
* @throws IOException Signals that an I/O exception has occurred.
*/
public static Template getLocalFreemarkerTemplate() throws IOException {
// Create your Configuration instance, and specify if up to what FreeMarker
// version (here 2.3.32) do you want to apply the fixes that are not 100%
// backward-compatible. See the Configuration JavaDoc for details.
Configuration cfg = new Configuration(Configuration.VERSION_2_3_32);
System.out.println("Working Directory = " + System.getProperty("user.dir"));
// Specify the source where the template files come from. Here I set a
// plain directory for it, but non-file-system sources are possible too:
cfg.setDirectoryForTemplateLoading(
new File(System.getProperty("user.dir") + "/src/test/java/geoportal_to_catalogue"));
// Set the preferred charset template files are stored in. UTF-8 is
// a good choice in most applications:
cfg.setDefaultEncoding("UTF-8");
Template template = cfg.getTemplate("d4gna_to_catalogue_template.ftl");
return template;
}
/**
* Pretty print using gson.
*
* @param uglyJson the ugly json
* @return the string
*/
public static String prettyPrintUsingGson(String uglyJson) {
Gson gson = new GsonBuilder().setPrettyPrinting().create();
JsonElement jsonElement = new JsonParser().parse(uglyJson);
String prettyJsonString = gson.toJson(jsonElement);
return prettyJsonString;
}
/**
* Write report.
*
* @param newline the newline
*/
private static synchronized void writeReport(String newline) {
reportPrintWriter.println(newline);
reportPrintWriter.flush();
}
/**
* Write error.
*
* @param newline the newline
*/
private static synchronized void writeError(String newline) {
errorPrintWriter.println(newline);
errorPrintWriter.flush();
}
/**
* Gets the formatted date time.
*
* @return the formatted date time
*/
public static String getFormattedDateTime(){
long yourmilliseconds = System.currentTimeMillis();
SimpleDateFormat sdf = new SimpleDateFormat("MMM dd,yyyy HH:mm:ss");
Date resultdate = new Date(yourmilliseconds);
return sdf.format(resultdate);
}
/*
* public static void main(String[] args) {
*
* try {
*
* TestContextConfig.readContextSettings();
*
* ScopeProvider.instance.set(TestContextConfig.CONTEXT);
* SecurityTokenProvider.instance.set(TestContextConfig.TOKEN);
*
* ProjectsCaller clientProjects = GeoportalClientCaller.projects(); Project
* theProject = clientProjects.getProjectByID(profileID, projectID); Document
* asDocument = SerializationUtil.asDocument(theProject);
*
* UseCaseDescriptor ucd =
* GeoportalClientCaller.useCaseDescriptors().getUCDForId(profileID);
*
* System.out.println("Source JSON:");
* System.out.println(prettyPrintUsingGson(asDocument.toJson()));
* applyMappingToCatalogue(theProject, ucd);
*
* } catch (Exception e) { // TODO: handle exception }
*
* }
*/
}

45
catalogue-binding-plugin/src/test/java/geoportal_to_catalogue/d4gna_to_catalogue_template.ftl
View File

@ -140,23 +140,29 @@
<#--
Starting document mapping to Catalogue
############################################################
#
#
# Starting document mapping to Catalogue
#
#
############################################################
-->
<#compress>
{
"name": "${jsonProj._id}",
"title": "${theDocument.nome}",
"title": "${sanitizeString(theDocument.nome)}",
"license_id": "CC-BY-SA-4.0",
"private": false,
<#assign sanitizedNotes = sanitizeString(theDocument.introduzione) />
"notes": "${sanitizedNotes}",
<#--
#######################################
######################
RESOURCES
#######################################
######################
-->
<#-- Mapping "Relazione Scavo" as resource -->
@ -197,11 +203,11 @@ Starting document mapping to Catalogue
<#--
#######################################
######################
TAGS
#######################################
######################
-->
<#-- Mapping "paroleChiaveLibere" as tag -->
@ -214,11 +220,11 @@ Starting document mapping to Catalogue
<#--
#######################################
######################
GROUPS
#######################################
######################
-->
<#-- Mapping the year of the "dataInizioProgetto" as group -->
@ -230,14 +236,19 @@ Starting document mapping to Catalogue
<#--
#######################################
######################
EXTRAS
#######################################
######################
-->
<#-- Mapping extras fields -->
<#if theDocument.contributore??>
<@assignExtraField key="Contributore" value=theDocument.contributore asObject=false></@assignExtraField>
</#if>
<#if theDocument.responsabile??>
<@assignExtraField key="Responsabile dei contenuti" value=theDocument.responsabile asObject=false></@assignExtraField>
</#if>
@ -250,6 +261,10 @@ Starting document mapping to Catalogue
<@assignExtraField key="Ufficio MiC competente per territorio" value=theDocument.ufficioMic asObject=false></@assignExtraField>
</#if>
<#if theDocument.funzionarioResponsabile??>
<@assignExtraField key="Funzionario Responsabile" value=theDocument.funzionarioResponsabile asObject=false></@assignExtraField>
</#if>
<#if theDocument.fontiFinanziamento??>
<#if theDocument.fontiFinanziamento?is_sequence>
<#list theDocument.fontiFinanziamento as my_extra>
@ -260,6 +275,16 @@ Starting document mapping to Catalogue
</#if>
</#if>
<#if theDocument.risorseCorrelate??>
<#if theDocument.risorseCorrelate?is_sequence>
<#list theDocument.risorseCorrelate as my_extra>
<@assignExtraField key="Risorse Correlate" value=my_extra asObject=false></@assignExtraField>
</#list>
<#else>
<@assignExtraField key="Risorse Correlate" value=theDocument.risorseCorrelate asObject=false></@assignExtraField>
</#if>
</#if>
<#if theDocument.dataInizioProgetto??>
<@assignExtraField key="Data inizio Campagna" value=theDocument.dataInizioProgetto?trim asObject=false></@assignExtraField>
</#if>

4
catalogue-binding-plugin/src/test/java/junit/CatalogueBindingPluginTest.java
View File

@ -61,7 +61,7 @@ public class CatalogueBindingPluginTest extends BasicPluginTest {
/**
* Check plugin config.
*/
// @Test
//@Test
public void checkPluginConfig() {
org.junit.Assume.assumeTrue(GCubeTest.isTestInfrastructureEnabled());
CatalogueBindingPlugin plugin = (CatalogueBindingPlugin) plugins.get(CatalogueBindingPlugin.DESCRIPTOR.getId());
@ -219,7 +219,7 @@ public class CatalogueBindingPluginTest extends BasicPluginTest {
/**
* Check notify PROJEC T DELETE D on catalogue.
*/
@Test
//@Test
public void checkNotify_PROJECT_DELETED_onCatalogue() {
org.junit.Assume.assumeTrue(GCubeTest.isTestInfrastructureEnabled());
CatalogueBindingPlugin plugin = (CatalogueBindingPlugin) plugins.get(CatalogueBindingPlugin.DESCRIPTOR.getId());