From f50cb2d9da30bf03791124fa6beede269de34b0a Mon Sep 17 00:00:00 2001 From: Luca Frosini Date: Tue, 26 Feb 2019 08:44:11 +0000 Subject: [PATCH] Added profiles sizes statistics csv generation git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/information-system/is-exporter-se-plugin@177267 82a268e6-3cf1-43bd-a215-b396298e98cf --- .../exporter/ISExporterPlugin.java | 13 +- .../exporter/mapper/GCoreResourceMapper.java | 112 ++++++++++++++---- .../mapper/GenericResourceExporter.java | 6 +- .../mapper/ServiceEndpointExporter.java | 4 +- .../exporter/ISExporterPluginTest.java | 6 +- .../mapper/GCoreResourceMapperTest.java | 9 +- .../mapper/GenericResourceExporterTest.java | 9 +- .../mapper/ServiceEndpointExporterTest.java | 10 +- 8 files changed, 123 insertions(+), 46 deletions(-) diff --git a/src/main/java/org/gcube/informationsystem/exporter/ISExporterPlugin.java b/src/main/java/org/gcube/informationsystem/exporter/ISExporterPlugin.java index 13abe76..07d4885 100644 --- a/src/main/java/org/gcube/informationsystem/exporter/ISExporterPlugin.java +++ b/src/main/java/org/gcube/informationsystem/exporter/ISExporterPlugin.java @@ -22,7 +22,7 @@ public class ISExporterPlugin extends Plugin { private static Logger logger = LoggerFactory.getLogger(ISExporterPlugin.class); public static final String FILTERED_REPORT = "FILTERED_REPORT"; - + public static final String STATISTICS = "STATISTICS"; public ISExporterPlugin(ISExporterPluginDeclaration pluginDeclaration) { super(pluginDeclaration); @@ -46,12 +46,19 @@ public class ISExporterPlugin extends Plugin { filteredReport = false; } + boolean statistics = false; + + try { + statistics = (boolean) inputs.getOrDefault(STATISTICS, false); + }catch (Exception e) { + statistics = false; + } logger.debug("Failure Report are filtered (e.g. Failing UUID are not shown to avoid to produce to much uneeded reports)"); - GenericResourceExporter genericResourceExporter = new GenericResourceExporter(filteredReport); + GenericResourceExporter genericResourceExporter = new GenericResourceExporter(filteredReport, statistics); genericResourceExporter.export(); - ServiceEndpointExporter serviceEndpointExporter = new ServiceEndpointExporter(filteredReport); + ServiceEndpointExporter serviceEndpointExporter = new ServiceEndpointExporter(filteredReport, statistics); serviceEndpointExporter.export(); logger.info("{} execution finished", ISExporterPluginDeclaration.NAME); diff --git a/src/main/java/org/gcube/informationsystem/exporter/mapper/GCoreResourceMapper.java b/src/main/java/org/gcube/informationsystem/exporter/mapper/GCoreResourceMapper.java index 9b0ea8d..ba88d52 100644 --- a/src/main/java/org/gcube/informationsystem/exporter/mapper/GCoreResourceMapper.java +++ b/src/main/java/org/gcube/informationsystem/exporter/mapper/GCoreResourceMapper.java @@ -5,6 +5,7 @@ import java.io.File; import java.io.FileWriter; import java.io.IOException; import java.io.PrintWriter; +import java.io.StringWriter; import java.io.UnsupportedEncodingException; import java.text.SimpleDateFormat; import java.util.ArrayList; @@ -16,6 +17,7 @@ import java.util.UUID; import org.gcube.common.authorization.client.Constants; import org.gcube.common.authorization.library.AuthorizationEntry; import org.gcube.common.authorization.library.provider.SecurityTokenProvider; +import org.gcube.common.resources.gcore.Resources; import org.gcube.common.scope.api.ScopeProvider; import org.gcube.informationsystem.exporter.mapper.exception.CreateException; import org.gcube.informationsystem.exporter.mapper.exception.UpdateException; @@ -70,6 +72,7 @@ public abstract class GCoreResourceMapper grClass; protected final Class rClass; protected final boolean filteredReport; + protected final boolean statistics; public static final String UTF8 = "UTF-8"; @@ -93,17 +96,33 @@ public abstract class GCoreResourceMapper grClass, String contextFullName, String dateString) { - return new File(contextFullName.replace("/", "_") + "-" + grClass.getSimpleName() + "-" + dateString - + "-exporter.json"); + private String getFileName(Class rClass, String contextFullName, Calendar calendar, String suffix) { + String dateString = getDateString(calendar); + StringBuffer stringBuffer = new StringBuffer(); + stringBuffer.append(contextFullName.replace("/", "_")); + stringBuffer.append("-"); + stringBuffer.append(rClass.getSimpleName()); + stringBuffer.append("-"); + stringBuffer.append(dateString); + stringBuffer.append(suffix); + return stringBuffer.toString(); + } + + public File getReportFile(Class rClass, String contextFullName, Calendar calendar) { + return new File(getFileName(rClass, contextFullName, calendar, ".json")); } - protected GCoreResourceMapper(Class grClass, Class rClass, boolean filteredReport) { + public File getStatisticFile(Class rClass, String contextFullName, Calendar calendar) { + return new File(getFileName(rClass, contextFullName, calendar, ".csv")); + } + + protected GCoreResourceMapper(Class grClass, Class rClass, boolean filteredReport, Boolean statistics) { this.grClass = grClass; this.rClass = rClass; this.resourceRegistryPublisher = ResourceRegistryPublisherFactory.create(); this.resourceRegistryClient = ResourceRegistryClientFactory.create(); this.filteredReport = filteredReport; + this.statistics = statistics; } protected List getAll() { @@ -222,7 +241,7 @@ public abstract class GCoreResourceMapper failed) { + protected void notifyFailures(int allSize, List failed, Calendar start, File reportFile) { String contextName = getCurrentContextName(); if (failed.size() == 0) { @@ -240,8 +259,8 @@ public abstract class GCoreResourceMapper all = getAll(); logger.debug("-------------------------------------------------------"); @@ -359,18 +369,70 @@ public abstract class GCoreResourceMapper failed = new ArrayList<>(); - + boolean first = true; + for (GR gr : all) { + R r = null; try { Thread.sleep(300); - mapAndPublish(gr); + r = mapAndPublish(gr); } catch (Exception e) { failed.add(gr); } + + if(statistics && r!=null) { + try { + addToCSV(gr, r, statisticsFile, first); + first = false; + }catch (Throwable t) { + logger.trace("Unable to report the statistic in file {} for resource with UUID {}", + statisticsFile.getAbsolutePath(), r.getHeader().getUUID(), t); + } + } } - notifyFailures(all.size(), failed); + notifyFailures(all.size(), failed, start, reportFile); } + private void printLineToFile(String line, File file) throws IOException { + synchronized (file) { + try (FileWriter fw = new FileWriter(file, true); + BufferedWriter bw = new BufferedWriter(fw); + PrintWriter out = new PrintWriter(bw)) { + out.println(line); + out.flush(); + } catch (IOException e) { + throw e; + } + } + } + + protected String getUnmarshalledResource(GR gr) { + StringWriter stringWriter = new StringWriter(); + Resources.marshal(gr, stringWriter); + return stringWriter.toString(); + } + + private void addToCSV(GR gr, R r, File statisticsFile, boolean addHeader) throws IOException { + StringBuffer stringBuffer = new StringBuffer(); + if(addHeader) { + stringBuffer.append(gr.getClass().getSimpleName()); + stringBuffer.append("(byte size)"); + stringBuffer.append(","); + stringBuffer.append(r.getClass().getSimpleName()); + stringBuffer.append("(byte size)"); + printLineToFile(stringBuffer.toString(), statisticsFile); + stringBuffer = new StringBuffer(); + } + String unmarshalledGR = getUnmarshalledResource(gr); + final byte[] grUTF8Bytes = unmarshalledGR.getBytes("UTF-8"); + stringBuffer.append(grUTF8Bytes.length); + stringBuffer.append(","); + String unmarshalledR = ISMapper.marshal(r); + final byte[] rUTF8Bytes = unmarshalledR.getBytes("UTF-8"); + stringBuffer.append(rUTF8Bytes.length); + printLineToFile(stringBuffer.toString(), statisticsFile); + } + } diff --git a/src/main/java/org/gcube/informationsystem/exporter/mapper/GenericResourceExporter.java b/src/main/java/org/gcube/informationsystem/exporter/mapper/GenericResourceExporter.java index 028e433..7f39c5e 100644 --- a/src/main/java/org/gcube/informationsystem/exporter/mapper/GenericResourceExporter.java +++ b/src/main/java/org/gcube/informationsystem/exporter/mapper/GenericResourceExporter.java @@ -32,8 +32,8 @@ public class GenericResourceExporter extends GCoreResourceMapper grm) throws Exception { + protected File getReportFile(GCoreResourceMapper grm) throws Exception { String contextName = GCoreResourceMapper.getCurrentContextName(); - String dateString = GCoreResourceMapper.getDateString(Calendar.getInstance()); - File file = grm.getFile(GenericResourceExporterTest.class, contextName, dateString); + File file = grm.getReportFile(GenericResourceExporterTest.class, contextName, Calendar.getInstance()); String json = "{}"; synchronized (file) { try(FileWriter fw = new FileWriter(file, true); @@ -50,8 +49,8 @@ public class GCoreResourceMapperTest extends ContextTest { for(String token : tokens){ logger.info("\n\n\n-------------------------------------------------------------------------"); ContextTest.setContext(token); - GenericResourceExporter gre = new GenericResourceExporter(false); - File file = getFile(gre); + GenericResourceExporter gre = new GenericResourceExporter(false, false); + File file = getReportFile(gre); logger.info("\n\n\n {}", file); /* gre.publishFileToWorkspace(file); diff --git a/src/test/java/org/gcube/informationsystem/exporter/mapper/GenericResourceExporterTest.java b/src/test/java/org/gcube/informationsystem/exporter/mapper/GenericResourceExporterTest.java index 7b0cad8..4e41524 100644 --- a/src/test/java/org/gcube/informationsystem/exporter/mapper/GenericResourceExporterTest.java +++ b/src/test/java/org/gcube/informationsystem/exporter/mapper/GenericResourceExporterTest.java @@ -1,6 +1,8 @@ package org.gcube.informationsystem.exporter.mapper; +import java.io.File; import java.util.ArrayList; +import java.util.Calendar; import java.util.List; import java.util.UUID; @@ -29,7 +31,7 @@ public class GenericResourceExporterTest extends ContextTest { @Test public void export() { - GenericResourceExporter gre = new GenericResourceExporter(false); + GenericResourceExporter gre = new GenericResourceExporter(false, false); gre.export(); } @@ -81,7 +83,10 @@ public class GenericResourceExporterTest extends ContextTest { DiscoveryClient client = ICFactory.clientFor(GenericResource.class); List seList = client.submit(query); - GenericResourceExporter see = new GenericResourceExporter(false); + GenericResourceExporter see = new GenericResourceExporter(false, false); + Calendar start = Calendar.getInstance(); + File reportFile = see.getReportFile(Configuration.class, GCoreResourceMapper.getCurrentContextName(), start); + see.notifyFailures(seList.size(), seList, start, reportFile); see.mapAndPublish(seList.get(0)); } diff --git a/src/test/java/org/gcube/informationsystem/exporter/mapper/ServiceEndpointExporterTest.java b/src/test/java/org/gcube/informationsystem/exporter/mapper/ServiceEndpointExporterTest.java index 625cd08..bcc955e 100644 --- a/src/test/java/org/gcube/informationsystem/exporter/mapper/ServiceEndpointExporterTest.java +++ b/src/test/java/org/gcube/informationsystem/exporter/mapper/ServiceEndpointExporterTest.java @@ -1,6 +1,8 @@ package org.gcube.informationsystem.exporter.mapper; +import java.io.File; import java.util.ArrayList; +import java.util.Calendar; import java.util.List; import java.util.UUID; @@ -29,7 +31,7 @@ public class ServiceEndpointExporterTest extends ContextTest{ @Test public void export(){ - ServiceEndpointExporter see = new ServiceEndpointExporter(false); + ServiceEndpointExporter see = new ServiceEndpointExporter(false, false); see.export(); } @@ -77,8 +79,10 @@ public class ServiceEndpointExporterTest extends ContextTest{ DiscoveryClient client = ICFactory.clientFor(ServiceEndpoint.class); List seList = client.submit(query); - ServiceEndpointExporter see = new ServiceEndpointExporter(false); - see.notifyFailures(seList.size(), seList); + ServiceEndpointExporter see = new ServiceEndpointExporter(false, false); + Calendar start = Calendar.getInstance(); + File reportFile = see.getReportFile(EService.class, GCoreResourceMapper.getCurrentContextName(), start); + see.notifyFailures(seList.size(), seList, start, reportFile); }