gCubeSystem
/
is-exporter-se-plugin
16
0
Fork 0
Code Issues Pull Requests Releases Activity

Merged from Branch 

git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/information-system/is-exporter-se-plugin@152669 82a268e6-3cf1-43bd-a215-b396298e98cf
This commit is contained in:
Luca Frosini 2017-09-05 14:14:57 +00:00
parent 20eb34ed77
commit 2c221bd88f
13 changed files with 338 additions and 282 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.classpath
View File

@ -22,7 +22,7 @@
<attribute name="maven.pomderived" value="true"/> <attribute name="maven.pomderived" value="true"/>
</attributes> </attributes>
</classpathentry> </classpathentry>
<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-1.7"> <classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-1.8">
<attributes> <attributes>
<attribute name="maven.pomderived" value="true"/> <attribute name="maven.pomderived" value="true"/>
</attributes> </attributes>

6
.settings/org.eclipse.jdt.core.prefs
View File

@ -1,5 +1,5 @@
eclipse.preferences.version=1 eclipse.preferences.version=1
org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.7 org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.8
org.eclipse.jdt.core.compiler.compliance=1.7 org.eclipse.jdt.core.compiler.compliance=1.8
org.eclipse.jdt.core.compiler.problem.forbiddenReference=warning org.eclipse.jdt.core.compiler.problem.forbiddenReference=warning
org.eclipse.jdt.core.compiler.source=1.7 org.eclipse.jdt.core.compiler.source=1.8

5
distro/changelog.xml
View File

@ -1,7 +1,10 @@
<?xml version="1.0" encoding="UTF-8"?> <?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE xml> <!DOCTYPE xml>
<ReleaseNotes> <ReleaseNotes>
<Changeset component="org.gcube.informationsystem.is-exporter-se-plugin.1.0.0" date="${buildDate}"> <Changeset component="org.gcube.informationsystem.is-exporter-se-plugin.1.1.0" date="${buildDate}">
<Change>Removed List and Set usage as workaround fo OrientDB bug #9021</Change>
</Changeset>
<Changeset component="org.gcube.informationsystem.is-exporter-se-plugin.1.0.0" date="2017-06-07">
<Change>First Release</Change> <Change>First Release</Change>
</Changeset> </Changeset>
</ReleaseNotes> </ReleaseNotes>

68
pom.xml
View File

@ -7,7 +7,7 @@
</parent> </parent>
<groupId>org.gcube.information-system</groupId> <groupId>org.gcube.information-system</groupId>
<artifactId>is-exporter-se-plugin</artifactId> <artifactId>is-exporter-se-plugin</artifactId>
<version>1.0.0-SNAPSHOT</version> <version>1.1.0-SNAPSHOT</version>
<name>IS Exporter</name> <name>IS Exporter</name>
<description>Exports GenericReosurce and ServiceEndpoint from gCoreIS to Resource Registry</description> <description>Exports GenericReosurce and ServiceEndpoint from gCoreIS to Resource Registry</description>
<properties> <properties>
@ -55,12 +55,6 @@
<version>[1.5.0-SNAPSHOT, 2.0.0-SNAPSHOT)</version> <version>[1.5.0-SNAPSHOT, 2.0.0-SNAPSHOT)</version>
<scope>provided</scope> <scope>provided</scope>
</dependency> </dependency>
<dependency>
<groupId>org.gcube.core</groupId>
<artifactId>common-scope</artifactId>
<scope>provided</scope>
</dependency>
<dependency> <dependency>
<groupId>org.gcube.resources</groupId> <groupId>org.gcube.resources</groupId>
<artifactId>common-gcore-resources</artifactId> <artifactId>common-gcore-resources</artifactId>
@ -76,7 +70,21 @@
<dependency> <dependency>
<groupId>org.gcube.common</groupId> <groupId>org.gcube.common</groupId>
<artifactId>authorization-client</artifactId> <artifactId>authorization-client</artifactId>
<scope>provided</scope>
</dependency> </dependency>
<dependency>
<groupId>org.gcube.core</groupId>
<artifactId>common-encryption</artifactId>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.gcube.core</groupId>
<artifactId>common-scope-maps</artifactId>
<scope>provided</scope>
</dependency>
<dependency> <dependency>
<groupId>org.gcube.information-system</groupId> <groupId>org.gcube.information-system</groupId>
<artifactId>information-system-model</artifactId> <artifactId>information-system-model</artifactId>
@ -96,6 +104,27 @@
<artifactId>resource-registry-publisher</artifactId> <artifactId>resource-registry-publisher</artifactId>
</dependency> </dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-annotations</artifactId>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-core</artifactId>
<scope>provided</scope>
</dependency>
<!--
<dependency> <dependency>
<groupId>org.gcube.common</groupId> <groupId>org.gcube.common</groupId>
<artifactId>home-library-jcr</artifactId> <artifactId>home-library-jcr</artifactId>
@ -113,6 +142,26 @@
<artifactId>home-library</artifactId> <artifactId>home-library</artifactId>
<version>[2.0.0-SNAPSHOT,3.0.0-SNAPSHOT)</version> <version>[2.0.0-SNAPSHOT,3.0.0-SNAPSHOT)</version>
</dependency> </dependency>
-->
<!-- dependency>
<groupId>org.gcube.core</groupId>
<artifactId>common-scope</artifactId>
<scope>provided</scope>
</dependency -->
<!-- dependency>
<groupId>org.gcube.common</groupId>
<artifactId>common-authorization</artifactId>
<scope>provided</scope>
</dependency -->
<!-- dependency>
<groupId>org.gcube.core</groupId>
<artifactId>common-gcore-stubs</artifactId>
<scope>provided</scope>
</dependency -->
<dependency> <dependency>
@ -129,6 +178,7 @@
</dependency> </dependency>
<!-- Test dependencies --> <!-- Test dependencies -->
<dependency> <dependency>
<groupId>junit</groupId> <groupId>junit</groupId>
<artifactId>junit</artifactId> <artifactId>junit</artifactId>
@ -142,11 +192,7 @@
<scope>test</scope> <scope>test</scope>
</dependency> </dependency>
</dependencies> </dependencies>
<build> <build>
<plugins> <plugins>
<plugin> <plugin>

21
src/main/java/org/gcube/informationsystem/exporter/ISExporterPlugin.java
View File

@ -21,6 +21,9 @@ public class ISExporterPlugin extends Plugin<ISExporterPluginDeclaration> {
*/ */
private static Logger logger = LoggerFactory.getLogger(ISExporterPlugin.class); private static Logger logger = LoggerFactory.getLogger(ISExporterPlugin.class);
public static final String FILTERED_REPORT = "FILTERED_REPORT";
public ISExporterPlugin(ISExporterPluginDeclaration pluginDeclaration) { public ISExporterPlugin(ISExporterPluginDeclaration pluginDeclaration) {
super(pluginDeclaration); super(pluginDeclaration);
logger.debug("contructor"); logger.debug("contructor");
@ -32,16 +35,26 @@ public class ISExporterPlugin extends Plugin<ISExporterPluginDeclaration> {
String token = SecurityTokenProvider.instance.get(); String token = SecurityTokenProvider.instance.get();
AuthorizationEntry authorizationEntry = Constants.authorizationService().get(token); AuthorizationEntry authorizationEntry = Constants.authorizationService().get(token);
String scope = authorizationEntry.getContext(); String scope = authorizationEntry.getContext();
logger.debug("Launching {} execution on scope {}", logger.info("Launching {} execution on scope {}",
ISExporterPluginDeclaration.NAME, scope); ISExporterPluginDeclaration.NAME, scope);
GenericResourceExporter genericResourceExporter = new GenericResourceExporter(); boolean filteredReport = false;
try {
filteredReport = (boolean) inputs.getOrDefault(FILTERED_REPORT, false);
}catch (Exception e) {
filteredReport = false;
}
logger.debug("Failure Report are filtered (e.g. Failing UUID are not shown to avoid to produce to much uneeded reports)");
GenericResourceExporter genericResourceExporter = new GenericResourceExporter(filteredReport);
genericResourceExporter.export(); genericResourceExporter.export();
ServiceEndpointExporter serviceEndpointExporter = new ServiceEndpointExporter(); ServiceEndpointExporter serviceEndpointExporter = new ServiceEndpointExporter(filteredReport);
serviceEndpointExporter.export(); serviceEndpointExporter.export();
logger.debug("{} execution finished", ISExporterPluginDeclaration.NAME); logger.info("{} execution finished", ISExporterPluginDeclaration.NAME);
} }
/**{@inheritDoc}*/ /**{@inheritDoc}*/

227
src/main/java/org/gcube/informationsystem/exporter/mapper/GCoreResourceMapper.java
View File

@ -2,7 +2,6 @@ package org.gcube.informationsystem.exporter.mapper;
import java.io.BufferedWriter; import java.io.BufferedWriter;
import java.io.File; import java.io.File;
import java.io.FileInputStream;
import java.io.FileWriter; import java.io.FileWriter;
import java.io.IOException; import java.io.IOException;
import java.io.PrintWriter; import java.io.PrintWriter;
@ -17,16 +16,7 @@ import java.util.UUID;
import org.gcube.common.authorization.client.Constants; import org.gcube.common.authorization.client.Constants;
import org.gcube.common.authorization.library.AuthorizationEntry; import org.gcube.common.authorization.library.AuthorizationEntry;
import org.gcube.common.authorization.library.provider.SecurityTokenProvider; import org.gcube.common.authorization.library.provider.SecurityTokenProvider;
import org.gcube.common.homelibrary.home.Home;
import org.gcube.common.homelibrary.home.HomeLibrary;
import org.gcube.common.homelibrary.home.HomeManager;
import org.gcube.common.homelibrary.home.HomeManagerFactory;
import org.gcube.common.homelibrary.home.workspace.Workspace;
import org.gcube.common.homelibrary.home.workspace.WorkspaceFolder;
import org.gcube.common.homelibrary.home.workspace.folder.FolderItem;
import org.gcube.common.homelibrary.util.WorkspaceUtil;
import org.gcube.common.scope.api.ScopeProvider; import org.gcube.common.scope.api.ScopeProvider;
import org.gcube.informationsystem.exporter.ISExporterPluginDeclaration;
import org.gcube.informationsystem.exporter.mapper.exception.CreateException; import org.gcube.informationsystem.exporter.mapper.exception.CreateException;
import org.gcube.informationsystem.exporter.mapper.exception.UpdateException; import org.gcube.informationsystem.exporter.mapper.exception.UpdateException;
import org.gcube.informationsystem.impl.utils.ISMapper; import org.gcube.informationsystem.impl.utils.ISMapper;
@ -72,15 +62,16 @@ public abstract class GCoreResourceMapper<GR extends org.gcube.common.resources.
public static final String CREATE = "create"; public static final String CREATE = "create";
public static final String UPDATE = "update"; public static final String UPDATE = "update";
protected Class<GR> grClass; protected final Class<GR> grClass;
protected Class<R> rClass; protected final Class<R> rClass;
protected final boolean filteredReport;
public static final String UTF8 = "UTF-8"; public static final String UTF8 = "UTF-8";
protected ResourceRegistryPublisher resourceRegistryPublisher; protected ResourceRegistryPublisher resourceRegistryPublisher;
protected ResourceRegistryClient resourceRegistryClient; protected ResourceRegistryClient resourceRegistryClient;
public static String getCurrentContextName(){ public static String getCurrentContextName() {
String token = SecurityTokenProvider.instance.get(); String token = SecurityTokenProvider.instance.get();
AuthorizationEntry authorizationEntry = null; AuthorizationEntry authorizationEntry = null;
try { try {
@ -91,21 +82,23 @@ public abstract class GCoreResourceMapper<GR extends org.gcube.common.resources.
return authorizationEntry.getContext(); return authorizationEntry.getContext();
} }
public static String getDateString(Calendar calendar){ public static String getDateString(Calendar calendar) {
Date date = calendar.getTime(); Date date = calendar.getTime();
SimpleDateFormat format = new SimpleDateFormat(ISConstants.DATETIME_PATTERN); SimpleDateFormat format = new SimpleDateFormat(ISConstants.DATETIME_PATTERN);
return format.format(date); return format.format(date);
} }
public File getFile(Class<?> grClass, String contextFullName, String dateString){ public File getFile(Class<?> grClass, String contextFullName, String dateString) {
return new File(contextFullName.replace("/", "_") + "-" + grClass.getSimpleName() + "-" + dateString + "-exporter.json"); return new File(contextFullName.replace("/", "_") + "-" + grClass.getSimpleName() + "-" + dateString
+ "-exporter.json");
} }
protected GCoreResourceMapper(Class<GR> grClass, Class<R> rClass){ protected GCoreResourceMapper(Class<GR> grClass, Class<R> rClass, boolean filteredReport) {
this.grClass = grClass; this.grClass = grClass;
this.rClass = rClass; this.rClass = rClass;
this.resourceRegistryPublisher = ResourceRegistryPublisherFactory.create(); this.resourceRegistryPublisher = ResourceRegistryPublisherFactory.create();
this.resourceRegistryClient = ResourceRegistryClientFactory.create(); this.resourceRegistryClient = ResourceRegistryClientFactory.create();
this.filteredReport = filteredReport;
} }
protected List<GR> getAll() { protected List<GR> getAll() {
@ -114,7 +107,7 @@ public abstract class GCoreResourceMapper<GR extends org.gcube.common.resources.
return client.submit(query); return client.submit(query);
} }
protected String getStringAsUTF8(String s) throws UnsupportedEncodingException{ protected String getStringAsUTF8(String s) throws UnsupportedEncodingException {
byte bytes[] = s.getBytes("ISO-8859-1"); byte bytes[] = s.getBytes("ISO-8859-1");
return new String(bytes, UTF8); return new String(bytes, UTF8);
} }
@ -139,36 +132,38 @@ public abstract class GCoreResourceMapper<GR extends org.gcube.common.resources.
try { try {
resourceRegistryClient.exists(rClass, uuid); resourceRegistryClient.exists(rClass, uuid);
update = true; update = true;
}catch (ResourceNotFoundException e) { } catch (ResourceNotFoundException e) {
update = false; update = false;
} catch (ResourceAvailableInAnotherContextException e) { } catch (ResourceAvailableInAnotherContextException e) {
// This code should never be reached because this should be fixed in map function // This code should never be reached because this should be fixed in
// map function
resourceRegistryPublisher.addResourceToContext(uuid); resourceRegistryPublisher.addResourceToContext(uuid);
try{ try {
Thread.sleep(100); Thread.sleep(100);
}catch (Exception ee) {} } catch (Exception ee) {
update= true; }
update = true;
} }
if(update){ if (update) {
logger.debug("Resource with UUID {} exist. It will be updated", uuid); logger.debug("Resource with UUID {} exist. It will be updated", uuid);
try{ try {
return update(r); return update(r);
}catch (ResourceRegistryException e) { } catch (ResourceRegistryException e) {
throw new UpdateException(e); throw new UpdateException(e);
} }
}else{ } else {
logger.debug("Resource with UUID {} does not exist. It will be created", uuid); logger.debug("Resource with UUID {} does not exist. It will be created", uuid);
try{ try {
return create(r); return create(r);
}catch (ResourceRegistryException e) { } catch (ResourceRegistryException e) {
throw new CreateException(e); throw new CreateException(e);
} }
} }
} }
private ObjectNode addNodeToArray(ArrayNode arrayNode, ObjectMapper objectMapper, GR gr, Exception e){ private ObjectNode addNodeToArray(ArrayNode arrayNode, ObjectMapper objectMapper, GR gr, Exception e) {
ObjectNode objectNode = objectMapper.createObjectNode(); ObjectNode objectNode = objectMapper.createObjectNode();
objectNode.put(TYPE, grClass.getSimpleName()); objectNode.put(TYPE, grClass.getSimpleName());
objectNode.put(ID, gr.id()); objectNode.put(ID, gr.id());
@ -182,64 +177,59 @@ public abstract class GCoreResourceMapper<GR extends org.gcube.common.resources.
return Constants.authorizationService().get(token).getClientInfo().getId(); return Constants.authorizationService().get(token).getClientInfo().getId();
} }
@SuppressWarnings("deprecation") /*
protected Home getHome(HomeManager manager) throws Exception{ *
Home home = null; * @SuppressWarnings("deprecation") protected Home getHome(HomeManager
String scope = ScopeProvider.instance.get(); * manager) throws Exception{ Home home = null; String scope =
if(scope!=null){ * ScopeProvider.instance.get(); if(scope!=null){ home =
home = manager.getHome(getUsername()); * manager.getHome(getUsername()); }else{ home = manager.getHome(); } return
}else{ * home; }
home = manager.getHome(); *
} * protected WorkspaceFolder getExporterFolder(Workspace ws) throws
return home; * Exception { WorkspaceFolder root = ws.getRoot();
} *
* WorkspaceFolder exporterFolder = null;
protected WorkspaceFolder getExporterFolder(Workspace ws) throws Exception { * if(!ws.exists(ISExporterPluginDeclaration.NAME, root.getId())){ String
WorkspaceFolder root = ws.getRoot(); * folderDescription = String.
* format("The folder is used by %s plugin to store informations regarding failures exporting old GCore Resource to new Resource Registry"
WorkspaceFolder exporterFolder = null; * , ISExporterPluginDeclaration.NAME); exporterFolder =
if(!ws.exists(ISExporterPluginDeclaration.NAME, root.getId())){ * ws.createFolder(ISExporterPluginDeclaration.NAME, folderDescription,
String folderDescription = String.format("The folder is used by %s plugin to store informations regarding failures exporting old GCore Resource to new Resource Registry", ISExporterPluginDeclaration.NAME); * root.getId()); }else{ exporterFolder = (WorkspaceFolder)
exporterFolder = ws.createFolder(ISExporterPluginDeclaration.NAME, folderDescription, root.getId()); * ws.find(ISExporterPluginDeclaration.NAME, root.getId()); } return
}else{ * exporterFolder; }
exporterFolder = (WorkspaceFolder) ws.find(ISExporterPluginDeclaration.NAME, root.getId()); *
} * public static final String APPLICATION_JSON_MIMETYPE =
return exporterFolder; * "application/json";
} *
* private static final String FOLDER_DESCRIPTION =
public static final String APPLICATION_JSON_MIMETYPE = "application/json"; * "Failures Report Folder for " + ISExporterPluginDeclaration.NAME;
*
private static final String FOLDER_DESCRIPTION = "Failures Report Folder for " + ISExporterPluginDeclaration.NAME; * protected FolderItem publishFileToWorkspace(File file) throws Exception {
* HomeManagerFactory factory = HomeLibrary.getHomeManagerFactory();
protected FolderItem publishFileToWorkspace(File file) throws Exception { * HomeManager manager = factory.getHomeManager(); Home home =
HomeManagerFactory factory = HomeLibrary.getHomeManagerFactory(); * getHome(manager); Workspace ws = home.getWorkspace(); WorkspaceFolder
HomeManager manager = factory.getHomeManager(); * exporterFolder = getExporterFolder(ws); FileInputStream fileInputStream =
Home home = getHome(manager); * new FileInputStream(file);
Workspace ws = home.getWorkspace(); *
WorkspaceFolder exporterFolder = getExporterFolder(ws); * FolderItem folderItem = WorkspaceUtil.createExternalFile( exporterFolder,
FileInputStream fileInputStream = new FileInputStream(file); * file.getName(), FOLDER_DESCRIPTION, APPLICATION_JSON_MIMETYPE,
* fileInputStream); return folderItem; }
FolderItem folderItem = WorkspaceUtil.createExternalFile( */
exporterFolder, file.getName(),
FOLDER_DESCRIPTION,
APPLICATION_JSON_MIMETYPE,
fileInputStream);
return folderItem;
}
protected abstract R map(GR gr) throws Exception; protected abstract R map(GR gr) throws Exception;
protected void notifyFailures(int allSize, List<GR> failed){ protected void notifyFailures(int allSize, List<GR> failed) {
String contextName = getCurrentContextName(); String contextName = getCurrentContextName();
if(failed.size()==0){ if (failed.size() == 0) {
logger.debug("No needs to create an empty report besause there were no exporting failures on {}", contextName); logger.debug("No needs to create an empty report besause there were no exporting failures on {}",
contextName);
return; return;
} }
logger.warn("-------------------------------------------------------"); logger.warn("-------------------------------------------------------");
logger.warn("{} : {} of {} ({} failures) {}s were exported as {}s", contextName, logger.warn("{} : {} of {} ({} failures) {}s were exported as {}s", contextName, allSize - failed.size(),
allSize-failed.size(), allSize, failed.size(), grClass.getSimpleName(), rClass.getSimpleName()); allSize, failed.size(), grClass.getSimpleName(), rClass.getSimpleName());
logger.warn("-------------------------------------------------------\n\n"); logger.warn("-------------------------------------------------------\n\n");
ObjectMapper objectMapper = new ObjectMapper(); ObjectMapper objectMapper = new ObjectMapper();
@ -253,40 +243,55 @@ public abstract class GCoreResourceMapper<GR extends org.gcube.common.resources.
ArrayNode mappingArrayNode = context.putArray(MAPPING_ERROR); ArrayNode mappingArrayNode = context.putArray(MAPPING_ERROR);
ArrayNode publishingArrayNode = context.putArray(PUBLISHING_ERROR); ArrayNode publishingArrayNode = context.putArray(PUBLISHING_ERROR);
for(GR gr : failed){ for (GR gr : failed) {
logger.trace("-------------------------------------------------------"); logger.trace("-------------------------------------------------------");
if (filteredReport) {
try {
UUID.fromString(gr.id());
} catch (Exception e) {
logger.debug(
"{} with has id {} which is not a valid UUID. The reports are filtered and such an error is not reported.",
gr.getClass().getSimpleName(), gr.id());
continue;
}
}
R r = null; R r = null;
try { try {
r = map(gr); r = map(gr);
} catch (Exception e) { } catch (Exception e) {
addNodeToArray(mappingArrayNode, objectMapper, gr, e); addNodeToArray(mappingArrayNode, objectMapper, gr, e);
logger.trace("Error exporting {}. The problem was on mapping {} with UUID {}", gr, grClass.getSimpleName(), gr.id()); logger.trace("Error exporting {}. The problem was on mapping {} with UUID {}", gr,
grClass.getSimpleName(), gr.id());
logger.trace("-------------------------------------------------------\n"); logger.trace("-------------------------------------------------------\n");
continue; continue;
} }
try { try {
createOrUpdate(r); createOrUpdate(r);
}catch (Exception e) { } catch (Exception e) {
if(e.getCause() instanceof NullPointerException){ if (e.getCause() instanceof NullPointerException) {
logger.error("This MUST BE A BUG. Please Investigate"); logger.error("This MUST BE A BUG. Please Investigate");
} }
if(e.getMessage().contains("com.orientechnologies.orient.server.distributed.task.ODistributedOperationException")){ if (e.getMessage().contains(
"com.orientechnologies.orient.server.distributed.task.ODistributedOperationException")) {
logger.error("This is an OrientDB distributed Issue"); logger.error("This is an OrientDB distributed Issue");
} }
ObjectNode node = addNodeToArray(publishingArrayNode, objectMapper, gr, e); ObjectNode node = addNodeToArray(publishingArrayNode, objectMapper, gr, e);
if(e instanceof CreateException) { if (e instanceof CreateException) {
node.put(EXCEPTION_TYPE, CREATE); node.put(EXCEPTION_TYPE, CREATE);
} }
if(e instanceof UpdateException){ if (e instanceof UpdateException) {
node.put(EXCEPTION_TYPE, UPDATE); node.put(EXCEPTION_TYPE, UPDATE);
} }
try { try {
logger.trace("Error exporting {}. The problem was on publishing {} as {}", gr, rClass.getSimpleName(), ISMapper.marshal(r)); logger.trace("Error exporting {}. The problem was on publishing {} as {}", gr,
rClass.getSimpleName(), ISMapper.marshal(r));
} catch (JsonProcessingException e1) { } catch (JsonProcessingException e1) {
logger.trace("", e1); logger.trace("", e1);
} }
@ -294,28 +299,34 @@ public abstract class GCoreResourceMapper<GR extends org.gcube.common.resources.
logger.trace("-------------------------------------------------------\n"); logger.trace("-------------------------------------------------------\n");
} }
if(filteredReport){
if(mappingArrayNode.size()==0 && publishingArrayNode.size()==0){
logger.debug("No need to produce JSON reports because alla the errors where filtered as requested by the input parameters.");
return;
}
}
try { try {
String json = objectMapper.writeValueAsString(objectNode); String json = objectMapper.writeValueAsString(objectNode);
File file = getFile(rClass, contextName, dateString); File file = getFile(rClass, contextName, dateString);
synchronized (file) { synchronized (file) {
try(FileWriter fw = new FileWriter(file, true); try (FileWriter fw = new FileWriter(file, true);
BufferedWriter bw = new BufferedWriter(fw); BufferedWriter bw = new BufferedWriter(fw);
PrintWriter out = new PrintWriter(bw)){ PrintWriter out = new PrintWriter(bw)) {
out.println(json); out.println(json);
out.flush(); out.flush();
} catch( IOException e ){ } catch (IOException e) {
throw e; throw e;
} }
} }
try { /*
publishFileToWorkspace(file); * try { publishFileToWorkspace(file); file.delete(); }catch
file.delete(); * (Exception e) {
}catch (Exception e) { * logger.error("Error while saving file {} on Workspace",
logger.error("Error while saving file {} on Workspace", file.getName(), e); * file.getName(), e); // TODO Use the Social Notification }
// TODO Use the Social Notification */
}
} catch (Exception e) { } catch (Exception e) {
logger.error("Error exporting JSON error result", e); logger.error("Error exporting JSON error result", e);
@ -328,7 +339,7 @@ public abstract class GCoreResourceMapper<GR extends org.gcube.common.resources.
R r = map(gr); R r = map(gr);
List<? extends Facet> facets = r.getIdentificationFacets(); List<? extends Facet> facets = r.getIdentificationFacets();
for(Facet f : facets){ for (Facet f : facets) {
f.setAdditionalProperty(EXPORTED, EXPORTED_FROM_OLD_GCORE_IS); f.setAdditionalProperty(EXPORTED, EXPORTED_FROM_OLD_GCORE_IS);
} }
@ -340,18 +351,16 @@ public abstract class GCoreResourceMapper<GR extends org.gcube.common.resources.
List<GR> all = getAll(); List<GR> all = getAll();
logger.debug("-------------------------------------------------------"); logger.debug("-------------------------------------------------------");
logger.debug("Going to export {} {}s as {}s", logger.debug("Going to export {} {}s as {}s", all.size(), grClass.getSimpleName(), rClass.getSimpleName());
all.size(), grClass.getSimpleName(), rClass.getSimpleName());
logger.debug("-------------------------------------------------------"); logger.debug("-------------------------------------------------------");
List<GR> failed = new ArrayList<>(); List<GR> failed = new ArrayList<>();
for(GR gr : all){ for (GR gr : all) {
try { try {
Thread.sleep(300); Thread.sleep(300);
mapAndPublish(gr); mapAndPublish(gr);
}catch (Exception e) { } catch (Exception e) {
failed.add(gr); failed.add(gr);
} }
} }
@ -360,8 +369,4 @@ public abstract class GCoreResourceMapper<GR extends org.gcube.common.resources.
} }
} }

4
src/main/java/org/gcube/informationsystem/exporter/mapper/GenericResourceExporter.java
View File

@ -29,8 +29,8 @@ public class GenericResourceExporter extends GCoreResourceMapper<GenericResource
public static final String FIXED_VERSION = "1.0.0"; public static final String FIXED_VERSION = "1.0.0";
public static final String FULL_BODY = "FULL_BODY"; public static final String FULL_BODY = "FULL_BODY";
public GenericResourceExporter(){ public GenericResourceExporter(Boolean filteredReport){
super(GenericResource.class, Configuration.class); super(GenericResource.class, Configuration.class, filteredReport);
} }
@Override @Override

50
src/main/java/org/gcube/informationsystem/exporter/mapper/ServiceEndpointExporter.java
View File

@ -1,7 +1,6 @@
package org.gcube.informationsystem.exporter.mapper; package org.gcube.informationsystem.exporter.mapper;
import java.net.URI; import java.net.URI;
import java.util.ArrayList;
import java.util.List; import java.util.List;
import java.util.UUID; import java.util.UUID;
@ -21,6 +20,7 @@ import org.gcube.informationsystem.impl.entity.facet.SoftwareFacetImpl;
import org.gcube.informationsystem.impl.entity.resource.EServiceImpl; import org.gcube.informationsystem.impl.entity.resource.EServiceImpl;
import org.gcube.informationsystem.impl.relation.ConsistsOfImpl; import org.gcube.informationsystem.impl.relation.ConsistsOfImpl;
import org.gcube.informationsystem.impl.relation.IsIdentifiedByImpl; import org.gcube.informationsystem.impl.relation.IsIdentifiedByImpl;
import org.gcube.informationsystem.impl.utils.ISMapper;
import org.gcube.informationsystem.model.embedded.Header; import org.gcube.informationsystem.model.embedded.Header;
import org.gcube.informationsystem.model.embedded.ValueSchema; import org.gcube.informationsystem.model.embedded.ValueSchema;
import org.gcube.informationsystem.model.entity.Facet; import org.gcube.informationsystem.model.entity.Facet;
@ -37,6 +37,9 @@ import org.gcube.informationsystem.resourceregistry.api.exceptions.entity.resour
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import com.fasterxml.jackson.databind.node.ArrayNode;
import com.fasterxml.jackson.databind.node.ObjectNode;
public class ServiceEndpointExporter extends GCoreResourceMapper<ServiceEndpoint, EService> { public class ServiceEndpointExporter extends GCoreResourceMapper<ServiceEndpoint, EService> {
private static Logger logger = LoggerFactory.getLogger(ServiceEndpointExporter.class); private static Logger logger = LoggerFactory.getLogger(ServiceEndpointExporter.class);
@ -51,25 +54,23 @@ public class ServiceEndpointExporter extends GCoreResourceMapper<ServiceEndpoint
public static final String USERNAME = "USERNAME"; public static final String USERNAME = "USERNAME";
public static final String PASSWORD = "PASSWORD"; public static final String PASSWORD = "PASSWORD";
public static final String USERNAME_PASSWORD_SCHEMA_STRING; public static final String USERNAME_PASSWORD_SCHEMA_STRING;
//public static final URI USERNAME_PASSWORD_SCHEMA;
public static final String NAME = "NAME"; public static final String NAME = "name";
public static final String ENCRYPTED = "ENCRYPTED"; public static final String VALUE = "value";
public static final String VALUE = "VALUE"; public static final String ENCRYPTED = "encrypted";
public static final String PROPERTY_SCHEMA_STRING;
public static final URI PROPERTY_SCHEMA; public static final String ARRAY = "ARRAY";
public static final String PROPERTIES = "PROPERTIES";
public static final String NOT_URI_ENDPOINT = "NOT_URI_ENDPOINT"; public static final String NOT_URI_ENDPOINT = "NOT_URI_ENDPOINT";
static { static {
USERNAME_PASSWORD_SCHEMA_STRING = String.format("%s:%s", USERNAME, PASSWORD); USERNAME_PASSWORD_SCHEMA_STRING = String.format("%s:%s", USERNAME, PASSWORD);
//USERNAME_PASSWORD_SCHEMA = URI.create(USERNAME_PASSWORD_SCHEMA_STRING);
PROPERTY_SCHEMA_STRING = String.format("%s:%s:%s", NAME, ENCRYPTED, VALUE);
PROPERTY_SCHEMA = URI.create(PROPERTY_SCHEMA_STRING);
} }
public ServiceEndpointExporter(){ public ServiceEndpointExporter(boolean filteredReport){
super(ServiceEndpoint.class, EService.class); super(ServiceEndpoint.class, EService.class, filteredReport);
} }
@Override @Override
@ -274,23 +275,22 @@ public class ServiceEndpointExporter extends GCoreResourceMapper<ServiceEndpoint
} }
/* ----- */ /* ----- */
Group<Property> properties = accessPoint.properties();
if(properties!=null && properties.size()>0){
ObjectNode propertiesObjectNode = ISMapper.getObjectMapper().createObjectNode();
ArrayNode arrayNode = propertiesObjectNode.putArray(ARRAY);
List<ValueSchema> properties = new ArrayList<>(); for(Property property : properties){
ObjectNode objectNode = ISMapper.getObjectMapper().createObjectNode();
objectNode.put(NAME, property.name());
objectNode.put(VALUE, property.value());
objectNode.put(ENCRYPTED, Boolean.toString(property.isEncrypted()));
for(Property property : accessPoint.properties()){ arrayNode.add(objectNode);
ValueSchema valueSchema = new ValueSchemaImpl(); }
String propertyValue = PROPERTY_SCHEMA.toString();
propertyValue = propertyValue.replace(NAME, property.name());
propertyValue = propertyValue.replace(ENCRYPTED, Boolean.toString(property.isEncrypted()));
propertyValue = propertyValue.replace(VALUE, property.value());
valueSchema.setValue(propertyValue);
valueSchema.setSchema(PROPERTY_SCHEMA);
properties.add(valueSchema);
accessPointFacets[i].setAdditionalProperty(PROPERTIES, propertiesObjectNode);
} }
//accessPointFacets[i].setProperties(properties);
i++; i++;
} }

1
src/main/resources/META-INF/services/org.gcube.vremanagement.executor.plugin.PluginDeclaration Normal file
View File

@ -0,0 +1 @@
org.gcube.informationsystem.exporter.ISExporterPluginDeclaration

13
src/test/java/org/gcube/informationsystem/exporter/ISExporterPluginTest.java
View File

@ -1,5 +1,8 @@
package org.gcube.informationsystem.exporter; package org.gcube.informationsystem.exporter;
import java.util.HashMap;
import java.util.Map;
import org.gcube.common.authorization.client.exceptions.ObjectNotFound; import org.gcube.common.authorization.client.exceptions.ObjectNotFound;
import org.gcube.informationsystem.exporter.mapper.GenericResourceExporterTest; import org.gcube.informationsystem.exporter.mapper.GenericResourceExporterTest;
import org.gcube.informationsystem.exporter.mapper.ServiceEndpointExporterTest; import org.gcube.informationsystem.exporter.mapper.ServiceEndpointExporterTest;
@ -25,13 +28,15 @@ public class ISExporterPluginTest {
logger.info("\n\n\n-------------------------------------------------------------------------"); logger.info("\n\n\n-------------------------------------------------------------------------");
ScopedTest.setContext(token); ScopedTest.setContext(token);
ISExporterPlugin isExporterPlugin = new ISExporterPlugin(new ISExporterPluginDeclaration()); ISExporterPlugin isExporterPlugin = new ISExporterPlugin(new ISExporterPluginDeclaration());
isExporterPlugin.launch(null); Map<String, Object> inputs = new HashMap<String, Object>();
inputs.put(ISExporterPlugin.FILTERED_REPORT, true);
isExporterPlugin.launch(inputs);
logger.info("\n\n\n"); logger.info("\n\n\n");
} }
} }
//@Test //@Test
public void delAllExported() throws ObjectNotFound, Exception{ public void removeExported() throws ObjectNotFound, Exception{
String[] tokens = { String[] tokens = {
ScopedTest.GCUBE, ScopedTest.GCUBE,
ScopedTest.GCUBE_DEVSEC, ScopedTest.GCUBE_DEVSEC,
@ -44,10 +49,10 @@ public class ISExporterPluginTest {
ScopedTest.setContext(token); ScopedTest.setContext(token);
GenericResourceExporterTest genericResourceExporterTest = new GenericResourceExporterTest(); GenericResourceExporterTest genericResourceExporterTest = new GenericResourceExporterTest();
genericResourceExporterTest.del(); genericResourceExporterTest.removeExported();
ServiceEndpointExporterTest serviceEndpointExporterTest = new ServiceEndpointExporterTest(); ServiceEndpointExporterTest serviceEndpointExporterTest = new ServiceEndpointExporterTest();
serviceEndpointExporterTest.del(); serviceEndpointExporterTest.removeExported();
} }
} }
} }

5
src/test/java/org/gcube/informationsystem/exporter/mapper/GCoreResourceMapperTest.java
View File

@ -47,11 +47,14 @@ public class GCoreResourceMapperTest extends ScopedTest {
for(String token : tokens){ for(String token : tokens){
logger.info("\n\n\n-------------------------------------------------------------------------"); logger.info("\n\n\n-------------------------------------------------------------------------");
ScopedTest.setContext(token); ScopedTest.setContext(token);
GenericResourceExporter gre = new GenericResourceExporter(); GenericResourceExporter gre = new GenericResourceExporter(false);
File file = getFile(gre); File file = getFile(gre);
logger.info("\n\n\n {}", file);
/*
gre.publishFileToWorkspace(file); gre.publishFileToWorkspace(file);
logger.info("\n\n\n"); logger.info("\n\n\n");
file.delete(); file.delete();
*/
} }
} }

57
src/test/java/org/gcube/informationsystem/exporter/mapper/GenericResourceExporterTest.java
View File

@ -4,11 +4,11 @@ import java.util.ArrayList;
import java.util.List; import java.util.List;
import java.util.UUID; import java.util.UUID;
import org.gcube.common.authorization.client.exceptions.ObjectNotFound;
import org.gcube.common.resources.gcore.GenericResource; import org.gcube.common.resources.gcore.GenericResource;
import org.gcube.informationsystem.exporter.ScopedTest; import org.gcube.informationsystem.exporter.ScopedTest;
import org.gcube.informationsystem.model.entity.Facet; import org.gcube.informationsystem.model.entity.Facet;
import org.gcube.informationsystem.model.entity.resource.Configuration; import org.gcube.informationsystem.model.entity.resource.Configuration;
import org.gcube.informationsystem.resourceregistry.api.exceptions.ResourceRegistryException;
import org.gcube.informationsystem.resourceregistry.client.ResourceRegistryClient; import org.gcube.informationsystem.resourceregistry.client.ResourceRegistryClient;
import org.gcube.informationsystem.resourceregistry.client.ResourceRegistryClientFactory; import org.gcube.informationsystem.resourceregistry.client.ResourceRegistryClientFactory;
import org.gcube.informationsystem.resourceregistry.publisher.ResourceRegistryPublisher; import org.gcube.informationsystem.resourceregistry.publisher.ResourceRegistryPublisher;
@ -23,18 +23,17 @@ import org.slf4j.LoggerFactory;
/** /**
* @author Luca Frosini (ISTI - CNR) * @author Luca Frosini (ISTI - CNR)
*/ */
public class GenericResourceExporterTest extends ScopedTest{ public class GenericResourceExporterTest extends ScopedTest {
private static Logger logger = LoggerFactory.getLogger(GenericResourceExporterTest.class); private static Logger logger = LoggerFactory.getLogger(GenericResourceExporterTest.class);
@Test @Test
public void export(){ public void export() {
GenericResourceExporter gre = new GenericResourceExporter(); GenericResourceExporter gre = new GenericResourceExporter(false);
gre.export(); gre.export();
} }
public void removeExported() throws ObjectNotFound, Exception {
public void del() throws ResourceRegistryException{
ResourceRegistryClient client = ResourceRegistryClientFactory.create(); ResourceRegistryClient client = ResourceRegistryClientFactory.create();
ResourceRegistryPublisher publisher = ResourceRegistryPublisherFactory.create(); ResourceRegistryPublisher publisher = ResourceRegistryPublisherFactory.create();
@ -42,70 +41,50 @@ public class GenericResourceExporterTest extends ScopedTest{
List<Configuration> configurations = (List<Configuration>) client.getInstances(Configuration.NAME, false); List<Configuration> configurations = (List<Configuration>) client.getInstances(Configuration.NAME, false);
List<Configuration> failed = new ArrayList<>(); List<Configuration> failed = new ArrayList<>();
logger.debug("Going to delete {} {}s", logger.debug("Going to delete {} {}s", configurations.size(), Configuration.NAME);
configurations.size(), Configuration.NAME);
int excluded = 0; int excluded = 0;
for(Configuration configuration : configurations){ for (Configuration configuration : configurations) {
try { try {
Facet facet = configuration.getIdentificationFacets().get(0); Facet facet = configuration.getIdentificationFacets().get(0);
String string = (String) facet.getAdditionalProperty(GCoreResourceMapper.EXPORTED); String string = (String) facet.getAdditionalProperty(GCoreResourceMapper.EXPORTED);
publisher.deleteResource(configuration); publisher.deleteResource(configuration);
if(string!=null && string.compareTo(GCoreResourceMapper.EXPORTED_FROM_OLD_GCORE_IS)==0){ if (string != null && string.compareTo(GCoreResourceMapper.EXPORTED_FROM_OLD_GCORE_IS) == 0) {
publisher.deleteResource(configuration); publisher.deleteResource(configuration);
}else{ } else {
excluded++; excluded++;
} }
}catch (Exception e) { } catch (Exception e) {
failed.add(configuration); failed.add(configuration);
} }
} }
logger.debug("{} of {} ({} failures) {}s were deleted. {} {}s were excluded because there was not exported from gCore IS.", logger.debug(
configurations.size()-failed.size()-excluded, configurations.size(), failed.size(), Configuration.NAME, excluded, Configuration.NAME); "{} of {} ({} failures) {}s were deleted. {} {}s were excluded because there was not exported from gCore IS.",
configurations.size() - failed.size() - excluded, configurations.size(), failed.size(),
Configuration.NAME, excluded, Configuration.NAME);
} }
//@Test // @Test
public void investigateSingleResource() throws Exception { public void investigateSingleResource() throws Exception {
ScopedTest.setContext(ScopedTest.GCUBE); //ScopedTest.setContext(ScopedTest.GCUBE_DEVSEC);
UUID uuid = UUID.fromString(""); UUID uuid = UUID.fromString("");
//ResourceRegistryPublisher publisher = ResourceRegistryPublisherFactory.create(); //ResourceRegistryPublisher publisher = ResourceRegistryPublisherFactory.create();
//publisher.deleteResource(uuid); //publisher.deleteResource(uuid);
//ResourceRegistryClient resourceRegistryClient = ResourceRegistryClientFactory.create(); // ResourceRegistryClient resourceRegistryClient = ResourceRegistryClientFactory.create();
SimpleQuery query = ICFactory.queryFor(GenericResource.class) SimpleQuery query = ICFactory.queryFor(GenericResource.class)
.addCondition(String.format("$resource/ID/text() eq '%1s'", uuid.toString())); .addCondition(String.format("$resource/ID/text() eq '%1s'", uuid.toString()));
DiscoveryClient<GenericResource> client = ICFactory.clientFor(GenericResource.class); DiscoveryClient<GenericResource> client = ICFactory.clientFor(GenericResource.class);
List<GenericResource> seList = client.submit(query); List<GenericResource> seList = client.submit(query);
GenericResourceExporter see = new GenericResourceExporter(); GenericResourceExporter see = new GenericResourceExporter(false);
see.mapAndPublish(seList.get(0)); see.mapAndPublish(seList.get(0));
} }
//@Test
public void addToAllScopes() throws Exception {
UUID uuid = UUID.fromString("");
ResourceRegistryPublisher publisher = ResourceRegistryPublisherFactory.create();
String[] tokens = {
ScopedTest.GCUBE,
ScopedTest.GCUBE_DEVSEC,
ScopedTest.GCUBE_DEVSEC_DEVVRE,
ScopedTest.GCUBE_DEVNEXT,
ScopedTest.GCUBE_DEVNEXT_NEXTNEXT
};
for(String token : tokens){
ScopedTest.setContext(token);
publisher.addResourceToContext(uuid);
}
}
} }

19
src/test/java/org/gcube/informationsystem/exporter/mapper/ServiceEndpointExporterTest.java
View File

@ -4,11 +4,11 @@ import java.util.ArrayList;
import java.util.List; import java.util.List;
import java.util.UUID; import java.util.UUID;
import org.gcube.common.authorization.client.exceptions.ObjectNotFound;
import org.gcube.common.resources.gcore.ServiceEndpoint; import org.gcube.common.resources.gcore.ServiceEndpoint;
import org.gcube.informationsystem.exporter.ScopedTest; import org.gcube.informationsystem.exporter.ScopedTest;
import org.gcube.informationsystem.model.entity.Facet; import org.gcube.informationsystem.model.entity.Facet;
import org.gcube.informationsystem.model.entity.resource.EService; import org.gcube.informationsystem.model.entity.resource.EService;
import org.gcube.informationsystem.resourceregistry.api.exceptions.ResourceRegistryException;
import org.gcube.informationsystem.resourceregistry.client.ResourceRegistryClient; import org.gcube.informationsystem.resourceregistry.client.ResourceRegistryClient;
import org.gcube.informationsystem.resourceregistry.client.ResourceRegistryClientFactory; import org.gcube.informationsystem.resourceregistry.client.ResourceRegistryClientFactory;
import org.gcube.informationsystem.resourceregistry.publisher.ResourceRegistryPublisher; import org.gcube.informationsystem.resourceregistry.publisher.ResourceRegistryPublisher;
@ -29,15 +29,14 @@ public class ServiceEndpointExporterTest extends ScopedTest{
@Test @Test
public void export(){ public void export(){
ServiceEndpointExporter see = new ServiceEndpointExporter(); ServiceEndpointExporter see = new ServiceEndpointExporter(false);
see.export(); see.export();
} }
public void del() throws ResourceRegistryException{ public void removeExported() throws ObjectNotFound, Exception{
ResourceRegistryClient client = ResourceRegistryClientFactory.create(); ResourceRegistryClient client = ResourceRegistryClientFactory.create();
ResourceRegistryPublisher publisher = ResourceRegistryPublisherFactory.create(); ResourceRegistryPublisher publisher = ResourceRegistryPublisherFactory.create();
@SuppressWarnings("unchecked") @SuppressWarnings("unchecked")
List<EService> eServices = (List<EService>) client.getInstances(EService.NAME, false); List<EService> eServices = (List<EService>) client.getInstances(EService.NAME, false);
List<EService> failed = new ArrayList<>(); List<EService> failed = new ArrayList<>();
@ -65,11 +64,13 @@ public class ServiceEndpointExporterTest extends ScopedTest{
eServices.size()-failed.size()-excluded, eServices.size(), failed.size(), EService.NAME, excluded, EService.NAME); eServices.size()-failed.size()-excluded, eServices.size(), failed.size(), EService.NAME, excluded, EService.NAME);
} }
@Test // @Test
public void investigateSingleResource() throws Exception { public void investigateSingleResource() throws Exception {
UUID uuid = UUID.fromString("66a674a2-8319-43b4-86c1-bd092bc7c490"); // ScopedTest.setContext(GCUBE);
ResourceRegistryPublisher publisher = ResourceRegistryPublisherFactory.create();
publisher.addResourceToContext(uuid); UUID uuid = UUID.fromString("");
//ResourceRegistryPublisher publisher = ResourceRegistryPublisherFactory.create();
//publisher.deleteResource(uuid);
//ResourceRegistryClient resourceRegistryClient = ResourceRegistryClientFactory.create(); //ResourceRegistryClient resourceRegistryClient = ResourceRegistryClientFactory.create();
@ -78,7 +79,7 @@ public class ServiceEndpointExporterTest extends ScopedTest{
DiscoveryClient<ServiceEndpoint> client = ICFactory.clientFor(ServiceEndpoint.class); DiscoveryClient<ServiceEndpoint> client = ICFactory.clientFor(ServiceEndpoint.class);
List<ServiceEndpoint> seList = client.submit(query); List<ServiceEndpoint> seList = client.submit(query);
ServiceEndpointExporter see = new ServiceEndpointExporter(); ServiceEndpointExporter see = new ServiceEndpointExporter(false);
see.notifyFailures(seList.size(), seList); see.notifyFailures(seList.size(), seList);
} }