Added profiles sizes statistics csv generation
git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/information-system/is-exporter-se-plugin@177267 82a268e6-3cf1-43bd-a215-b396298e98cf
This commit is contained in:
parent
8494877b2e
commit
f50cb2d9da
|
@ -22,7 +22,7 @@ public class ISExporterPlugin extends Plugin<ISExporterPluginDeclaration> {
|
||||||
private static Logger logger = LoggerFactory.getLogger(ISExporterPlugin.class);
|
private static Logger logger = LoggerFactory.getLogger(ISExporterPlugin.class);
|
||||||
|
|
||||||
public static final String FILTERED_REPORT = "FILTERED_REPORT";
|
public static final String FILTERED_REPORT = "FILTERED_REPORT";
|
||||||
|
public static final String STATISTICS = "STATISTICS";
|
||||||
|
|
||||||
public ISExporterPlugin(ISExporterPluginDeclaration pluginDeclaration) {
|
public ISExporterPlugin(ISExporterPluginDeclaration pluginDeclaration) {
|
||||||
super(pluginDeclaration);
|
super(pluginDeclaration);
|
||||||
|
@ -46,12 +46,19 @@ public class ISExporterPlugin extends Plugin<ISExporterPluginDeclaration> {
|
||||||
filteredReport = false;
|
filteredReport = false;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
boolean statistics = false;
|
||||||
|
|
||||||
|
try {
|
||||||
|
statistics = (boolean) inputs.getOrDefault(STATISTICS, false);
|
||||||
|
}catch (Exception e) {
|
||||||
|
statistics = false;
|
||||||
|
}
|
||||||
logger.debug("Failure Report are filtered (e.g. Failing UUID are not shown to avoid to produce to much uneeded reports)");
|
logger.debug("Failure Report are filtered (e.g. Failing UUID are not shown to avoid to produce to much uneeded reports)");
|
||||||
|
|
||||||
GenericResourceExporter genericResourceExporter = new GenericResourceExporter(filteredReport);
|
GenericResourceExporter genericResourceExporter = new GenericResourceExporter(filteredReport, statistics);
|
||||||
genericResourceExporter.export();
|
genericResourceExporter.export();
|
||||||
|
|
||||||
ServiceEndpointExporter serviceEndpointExporter = new ServiceEndpointExporter(filteredReport);
|
ServiceEndpointExporter serviceEndpointExporter = new ServiceEndpointExporter(filteredReport, statistics);
|
||||||
serviceEndpointExporter.export();
|
serviceEndpointExporter.export();
|
||||||
|
|
||||||
logger.info("{} execution finished", ISExporterPluginDeclaration.NAME);
|
logger.info("{} execution finished", ISExporterPluginDeclaration.NAME);
|
||||||
|
|
|
@ -5,6 +5,7 @@ import java.io.File;
|
||||||
import java.io.FileWriter;
|
import java.io.FileWriter;
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
import java.io.PrintWriter;
|
import java.io.PrintWriter;
|
||||||
|
import java.io.StringWriter;
|
||||||
import java.io.UnsupportedEncodingException;
|
import java.io.UnsupportedEncodingException;
|
||||||
import java.text.SimpleDateFormat;
|
import java.text.SimpleDateFormat;
|
||||||
import java.util.ArrayList;
|
import java.util.ArrayList;
|
||||||
|
@ -16,6 +17,7 @@ import java.util.UUID;
|
||||||
import org.gcube.common.authorization.client.Constants;
|
import org.gcube.common.authorization.client.Constants;
|
||||||
import org.gcube.common.authorization.library.AuthorizationEntry;
|
import org.gcube.common.authorization.library.AuthorizationEntry;
|
||||||
import org.gcube.common.authorization.library.provider.SecurityTokenProvider;
|
import org.gcube.common.authorization.library.provider.SecurityTokenProvider;
|
||||||
|
import org.gcube.common.resources.gcore.Resources;
|
||||||
import org.gcube.common.scope.api.ScopeProvider;
|
import org.gcube.common.scope.api.ScopeProvider;
|
||||||
import org.gcube.informationsystem.exporter.mapper.exception.CreateException;
|
import org.gcube.informationsystem.exporter.mapper.exception.CreateException;
|
||||||
import org.gcube.informationsystem.exporter.mapper.exception.UpdateException;
|
import org.gcube.informationsystem.exporter.mapper.exception.UpdateException;
|
||||||
|
@ -70,6 +72,7 @@ public abstract class GCoreResourceMapper<GR extends org.gcube.common.resources.
|
||||||
protected final Class<GR> grClass;
|
protected final Class<GR> grClass;
|
||||||
protected final Class<R> rClass;
|
protected final Class<R> rClass;
|
||||||
protected final boolean filteredReport;
|
protected final boolean filteredReport;
|
||||||
|
protected final boolean statistics;
|
||||||
|
|
||||||
public static final String UTF8 = "UTF-8";
|
public static final String UTF8 = "UTF-8";
|
||||||
|
|
||||||
|
@ -93,17 +96,33 @@ public abstract class GCoreResourceMapper<GR extends org.gcube.common.resources.
|
||||||
return format.format(date);
|
return format.format(date);
|
||||||
}
|
}
|
||||||
|
|
||||||
public File getFile(Class<?> grClass, String contextFullName, String dateString) {
|
private String getFileName(Class<?> rClass, String contextFullName, Calendar calendar, String suffix) {
|
||||||
return new File(contextFullName.replace("/", "_") + "-" + grClass.getSimpleName() + "-" + dateString
|
String dateString = getDateString(calendar);
|
||||||
+ "-exporter.json");
|
StringBuffer stringBuffer = new StringBuffer();
|
||||||
|
stringBuffer.append(contextFullName.replace("/", "_"));
|
||||||
|
stringBuffer.append("-");
|
||||||
|
stringBuffer.append(rClass.getSimpleName());
|
||||||
|
stringBuffer.append("-");
|
||||||
|
stringBuffer.append(dateString);
|
||||||
|
stringBuffer.append(suffix);
|
||||||
|
return stringBuffer.toString();
|
||||||
|
}
|
||||||
|
|
||||||
|
public File getReportFile(Class<?> rClass, String contextFullName, Calendar calendar) {
|
||||||
|
return new File(getFileName(rClass, contextFullName, calendar, ".json"));
|
||||||
}
|
}
|
||||||
|
|
||||||
protected GCoreResourceMapper(Class<GR> grClass, Class<R> rClass, boolean filteredReport) {
|
public File getStatisticFile(Class<?> rClass, String contextFullName, Calendar calendar) {
|
||||||
|
return new File(getFileName(rClass, contextFullName, calendar, ".csv"));
|
||||||
|
}
|
||||||
|
|
||||||
|
protected GCoreResourceMapper(Class<GR> grClass, Class<R> rClass, boolean filteredReport, Boolean statistics) {
|
||||||
this.grClass = grClass;
|
this.grClass = grClass;
|
||||||
this.rClass = rClass;
|
this.rClass = rClass;
|
||||||
this.resourceRegistryPublisher = ResourceRegistryPublisherFactory.create();
|
this.resourceRegistryPublisher = ResourceRegistryPublisherFactory.create();
|
||||||
this.resourceRegistryClient = ResourceRegistryClientFactory.create();
|
this.resourceRegistryClient = ResourceRegistryClientFactory.create();
|
||||||
this.filteredReport = filteredReport;
|
this.filteredReport = filteredReport;
|
||||||
|
this.statistics = statistics;
|
||||||
}
|
}
|
||||||
|
|
||||||
protected List<GR> getAll() {
|
protected List<GR> getAll() {
|
||||||
|
@ -222,7 +241,7 @@ public abstract class GCoreResourceMapper<GR extends org.gcube.common.resources.
|
||||||
|
|
||||||
protected abstract R map(GR gr) throws Exception;
|
protected abstract R map(GR gr) throws Exception;
|
||||||
|
|
||||||
protected void notifyFailures(int allSize, List<GR> failed) {
|
protected void notifyFailures(int allSize, List<GR> failed, Calendar start, File reportFile) {
|
||||||
String contextName = getCurrentContextName();
|
String contextName = getCurrentContextName();
|
||||||
|
|
||||||
if (failed.size() == 0) {
|
if (failed.size() == 0) {
|
||||||
|
@ -240,8 +259,8 @@ public abstract class GCoreResourceMapper<GR extends org.gcube.common.resources.
|
||||||
|
|
||||||
ObjectNode objectNode = objectMapper.createObjectNode();
|
ObjectNode objectNode = objectMapper.createObjectNode();
|
||||||
|
|
||||||
Calendar calendar = Calendar.getInstance();
|
String dateString = getDateString(start);
|
||||||
String dateString = getDateString(calendar);
|
|
||||||
objectNode.put(DATE, dateString);
|
objectNode.put(DATE, dateString);
|
||||||
ObjectNode context = objectNode.putObject(contextName);
|
ObjectNode context = objectNode.putObject(contextName);
|
||||||
ArrayNode mappingArrayNode = context.putArray(MAPPING_ERROR);
|
ArrayNode mappingArrayNode = context.putArray(MAPPING_ERROR);
|
||||||
|
@ -312,26 +331,13 @@ public abstract class GCoreResourceMapper<GR extends org.gcube.common.resources.
|
||||||
|
|
||||||
try {
|
try {
|
||||||
String json = objectMapper.writeValueAsString(objectNode);
|
String json = objectMapper.writeValueAsString(objectNode);
|
||||||
|
printLineToFile(json, reportFile);
|
||||||
File file = getFile(rClass, contextName, dateString);
|
|
||||||
synchronized (file) {
|
|
||||||
try (FileWriter fw = new FileWriter(file, true);
|
|
||||||
BufferedWriter bw = new BufferedWriter(fw);
|
|
||||||
PrintWriter out = new PrintWriter(bw)) {
|
|
||||||
out.println(json);
|
|
||||||
out.flush();
|
|
||||||
} catch (IOException e) {
|
|
||||||
throw e;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* try { publishFileToWorkspace(file); file.delete(); }catch
|
* try { publishFileToWorkspace(file); file.delete(); }catch
|
||||||
* (Exception e) {
|
* (Exception e) {
|
||||||
* logger.error("Error while saving file {} on Workspace",
|
* logger.error("Error while saving file {} on Workspace",
|
||||||
* file.getName(), e); // TODO Use the Social Notification }
|
* file.getName(), e); // TODO Use the Social Notification }
|
||||||
*/
|
*/
|
||||||
|
|
||||||
} catch (Exception e) {
|
} catch (Exception e) {
|
||||||
logger.error("Error exporting JSON error result", e);
|
logger.error("Error exporting JSON error result", e);
|
||||||
}
|
}
|
||||||
|
@ -351,7 +357,11 @@ public abstract class GCoreResourceMapper<GR extends org.gcube.common.resources.
|
||||||
}
|
}
|
||||||
|
|
||||||
public void export() {
|
public void export() {
|
||||||
|
String contextName = getCurrentContextName();
|
||||||
|
Calendar start = Calendar.getInstance();
|
||||||
|
File statisticsFile = getStatisticFile(rClass, contextName, start);
|
||||||
|
File reportFile = getReportFile(rClass, contextName, start);
|
||||||
|
|
||||||
List<GR> all = getAll();
|
List<GR> all = getAll();
|
||||||
|
|
||||||
logger.debug("-------------------------------------------------------");
|
logger.debug("-------------------------------------------------------");
|
||||||
|
@ -359,18 +369,70 @@ public abstract class GCoreResourceMapper<GR extends org.gcube.common.resources.
|
||||||
logger.debug("-------------------------------------------------------");
|
logger.debug("-------------------------------------------------------");
|
||||||
|
|
||||||
List<GR> failed = new ArrayList<>();
|
List<GR> failed = new ArrayList<>();
|
||||||
|
boolean first = true;
|
||||||
|
|
||||||
for (GR gr : all) {
|
for (GR gr : all) {
|
||||||
|
R r = null;
|
||||||
try {
|
try {
|
||||||
Thread.sleep(300);
|
Thread.sleep(300);
|
||||||
mapAndPublish(gr);
|
r = mapAndPublish(gr);
|
||||||
} catch (Exception e) {
|
} catch (Exception e) {
|
||||||
failed.add(gr);
|
failed.add(gr);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if(statistics && r!=null) {
|
||||||
|
try {
|
||||||
|
addToCSV(gr, r, statisticsFile, first);
|
||||||
|
first = false;
|
||||||
|
}catch (Throwable t) {
|
||||||
|
logger.trace("Unable to report the statistic in file {} for resource with UUID {}",
|
||||||
|
statisticsFile.getAbsolutePath(), r.getHeader().getUUID(), t);
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
notifyFailures(all.size(), failed);
|
notifyFailures(all.size(), failed, start, reportFile);
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private void printLineToFile(String line, File file) throws IOException {
|
||||||
|
synchronized (file) {
|
||||||
|
try (FileWriter fw = new FileWriter(file, true);
|
||||||
|
BufferedWriter bw = new BufferedWriter(fw);
|
||||||
|
PrintWriter out = new PrintWriter(bw)) {
|
||||||
|
out.println(line);
|
||||||
|
out.flush();
|
||||||
|
} catch (IOException e) {
|
||||||
|
throw e;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
protected String getUnmarshalledResource(GR gr) {
|
||||||
|
StringWriter stringWriter = new StringWriter();
|
||||||
|
Resources.marshal(gr, stringWriter);
|
||||||
|
return stringWriter.toString();
|
||||||
|
}
|
||||||
|
|
||||||
|
private void addToCSV(GR gr, R r, File statisticsFile, boolean addHeader) throws IOException {
|
||||||
|
StringBuffer stringBuffer = new StringBuffer();
|
||||||
|
if(addHeader) {
|
||||||
|
stringBuffer.append(gr.getClass().getSimpleName());
|
||||||
|
stringBuffer.append("(byte size)");
|
||||||
|
stringBuffer.append(",");
|
||||||
|
stringBuffer.append(r.getClass().getSimpleName());
|
||||||
|
stringBuffer.append("(byte size)");
|
||||||
|
printLineToFile(stringBuffer.toString(), statisticsFile);
|
||||||
|
stringBuffer = new StringBuffer();
|
||||||
|
}
|
||||||
|
String unmarshalledGR = getUnmarshalledResource(gr);
|
||||||
|
final byte[] grUTF8Bytes = unmarshalledGR.getBytes("UTF-8");
|
||||||
|
stringBuffer.append(grUTF8Bytes.length);
|
||||||
|
stringBuffer.append(",");
|
||||||
|
String unmarshalledR = ISMapper.marshal(r);
|
||||||
|
final byte[] rUTF8Bytes = unmarshalledR.getBytes("UTF-8");
|
||||||
|
stringBuffer.append(rUTF8Bytes.length);
|
||||||
|
printLineToFile(stringBuffer.toString(), statisticsFile);
|
||||||
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
|
@ -32,8 +32,8 @@ public class GenericResourceExporter extends GCoreResourceMapper<GenericResource
|
||||||
public static final String FIXED_VERSION = "1.0.0";
|
public static final String FIXED_VERSION = "1.0.0";
|
||||||
public static final String FULL_BODY = "FULL_BODY";
|
public static final String FULL_BODY = "FULL_BODY";
|
||||||
|
|
||||||
public GenericResourceExporter(Boolean filteredReport) {
|
public GenericResourceExporter(Boolean filteredReport, Boolean statistics) {
|
||||||
super(GenericResource.class, Configuration.class, filteredReport);
|
super(GenericResource.class, Configuration.class, filteredReport, statistics);
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
|
@ -130,5 +130,5 @@ public class GenericResourceExporter extends GCoreResourceMapper<GenericResource
|
||||||
|
|
||||||
return configuration;
|
return configuration;
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
|
@ -61,8 +61,8 @@ public class ServiceEndpointExporter extends GCoreResourceMapper<ServiceEndpoint
|
||||||
|
|
||||||
public static final String NOT_URI_ENDPOINT = "NOT_URI_ENDPOINT";
|
public static final String NOT_URI_ENDPOINT = "NOT_URI_ENDPOINT";
|
||||||
|
|
||||||
public ServiceEndpointExporter(boolean filteredReport) {
|
public ServiceEndpointExporter(boolean filteredReport, Boolean statistics) {
|
||||||
super(ServiceEndpoint.class, EService.class, filteredReport);
|
super(ServiceEndpoint.class, EService.class, filteredReport, statistics);
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
|
|
|
@ -14,14 +14,14 @@ public class ISExporterPluginTest {
|
||||||
|
|
||||||
private static Logger logger = LoggerFactory.getLogger(ISExporterPluginTest.class);
|
private static Logger logger = LoggerFactory.getLogger(ISExporterPluginTest.class);
|
||||||
|
|
||||||
// @Test
|
@Test
|
||||||
public void testLaunch() throws ObjectNotFound, Exception {
|
public void testLaunch() throws ObjectNotFound, Exception {
|
||||||
String[] tokens = {
|
String[] tokens = {
|
||||||
ContextTest.GCUBE,
|
ContextTest.GCUBE,
|
||||||
ContextTest.GCUBE_DEVSEC,
|
ContextTest.GCUBE_DEVSEC,
|
||||||
ContextTest.GCUBE_DEVSEC_DEVVRE,
|
ContextTest.GCUBE_DEVSEC_DEVVRE,
|
||||||
// ContextTest.GCUBE_DEVNEXT,
|
ContextTest.GCUBE_DEVNEXT,
|
||||||
// ContextTest.GCUBE_DEVNEXT_NEXTNEXT
|
ContextTest.GCUBE_DEVNEXT_NEXTNEXT
|
||||||
};
|
};
|
||||||
|
|
||||||
for(String token : tokens) {
|
for(String token : tokens) {
|
||||||
|
|
|
@ -18,10 +18,9 @@ public class GCoreResourceMapperTest extends ContextTest {
|
||||||
|
|
||||||
private static Logger logger = LoggerFactory.getLogger(GenericResourceExporterTest.class);
|
private static Logger logger = LoggerFactory.getLogger(GenericResourceExporterTest.class);
|
||||||
|
|
||||||
protected File getFile(GCoreResourceMapper<?,?> grm) throws Exception {
|
protected File getReportFile(GCoreResourceMapper<?,?> grm) throws Exception {
|
||||||
String contextName = GCoreResourceMapper.getCurrentContextName();
|
String contextName = GCoreResourceMapper.getCurrentContextName();
|
||||||
String dateString = GCoreResourceMapper.getDateString(Calendar.getInstance());
|
File file = grm.getReportFile(GenericResourceExporterTest.class, contextName, Calendar.getInstance());
|
||||||
File file = grm.getFile(GenericResourceExporterTest.class, contextName, dateString);
|
|
||||||
String json = "{}";
|
String json = "{}";
|
||||||
synchronized (file) {
|
synchronized (file) {
|
||||||
try(FileWriter fw = new FileWriter(file, true);
|
try(FileWriter fw = new FileWriter(file, true);
|
||||||
|
@ -50,8 +49,8 @@ public class GCoreResourceMapperTest extends ContextTest {
|
||||||
for(String token : tokens){
|
for(String token : tokens){
|
||||||
logger.info("\n\n\n-------------------------------------------------------------------------");
|
logger.info("\n\n\n-------------------------------------------------------------------------");
|
||||||
ContextTest.setContext(token);
|
ContextTest.setContext(token);
|
||||||
GenericResourceExporter gre = new GenericResourceExporter(false);
|
GenericResourceExporter gre = new GenericResourceExporter(false, false);
|
||||||
File file = getFile(gre);
|
File file = getReportFile(gre);
|
||||||
logger.info("\n\n\n {}", file);
|
logger.info("\n\n\n {}", file);
|
||||||
/*
|
/*
|
||||||
gre.publishFileToWorkspace(file);
|
gre.publishFileToWorkspace(file);
|
||||||
|
|
|
@ -1,6 +1,8 @@
|
||||||
package org.gcube.informationsystem.exporter.mapper;
|
package org.gcube.informationsystem.exporter.mapper;
|
||||||
|
|
||||||
|
import java.io.File;
|
||||||
import java.util.ArrayList;
|
import java.util.ArrayList;
|
||||||
|
import java.util.Calendar;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import java.util.UUID;
|
import java.util.UUID;
|
||||||
|
|
||||||
|
@ -29,7 +31,7 @@ public class GenericResourceExporterTest extends ContextTest {
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void export() {
|
public void export() {
|
||||||
GenericResourceExporter gre = new GenericResourceExporter(false);
|
GenericResourceExporter gre = new GenericResourceExporter(false, false);
|
||||||
gre.export();
|
gre.export();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -81,7 +83,10 @@ public class GenericResourceExporterTest extends ContextTest {
|
||||||
DiscoveryClient<GenericResource> client = ICFactory.clientFor(GenericResource.class);
|
DiscoveryClient<GenericResource> client = ICFactory.clientFor(GenericResource.class);
|
||||||
List<GenericResource> seList = client.submit(query);
|
List<GenericResource> seList = client.submit(query);
|
||||||
|
|
||||||
GenericResourceExporter see = new GenericResourceExporter(false);
|
GenericResourceExporter see = new GenericResourceExporter(false, false);
|
||||||
|
Calendar start = Calendar.getInstance();
|
||||||
|
File reportFile = see.getReportFile(Configuration.class, GCoreResourceMapper.getCurrentContextName(), start);
|
||||||
|
see.notifyFailures(seList.size(), seList, start, reportFile);
|
||||||
see.mapAndPublish(seList.get(0));
|
see.mapAndPublish(seList.get(0));
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
|
@ -1,6 +1,8 @@
|
||||||
package org.gcube.informationsystem.exporter.mapper;
|
package org.gcube.informationsystem.exporter.mapper;
|
||||||
|
|
||||||
|
import java.io.File;
|
||||||
import java.util.ArrayList;
|
import java.util.ArrayList;
|
||||||
|
import java.util.Calendar;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import java.util.UUID;
|
import java.util.UUID;
|
||||||
|
|
||||||
|
@ -29,7 +31,7 @@ public class ServiceEndpointExporterTest extends ContextTest{
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void export(){
|
public void export(){
|
||||||
ServiceEndpointExporter see = new ServiceEndpointExporter(false);
|
ServiceEndpointExporter see = new ServiceEndpointExporter(false, false);
|
||||||
see.export();
|
see.export();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -77,8 +79,10 @@ public class ServiceEndpointExporterTest extends ContextTest{
|
||||||
DiscoveryClient<ServiceEndpoint> client = ICFactory.clientFor(ServiceEndpoint.class);
|
DiscoveryClient<ServiceEndpoint> client = ICFactory.clientFor(ServiceEndpoint.class);
|
||||||
List<ServiceEndpoint> seList = client.submit(query);
|
List<ServiceEndpoint> seList = client.submit(query);
|
||||||
|
|
||||||
ServiceEndpointExporter see = new ServiceEndpointExporter(false);
|
ServiceEndpointExporter see = new ServiceEndpointExporter(false, false);
|
||||||
see.notifyFailures(seList.size(), seList);
|
Calendar start = Calendar.getInstance();
|
||||||
|
File reportFile = see.getReportFile(EService.class, GCoreResourceMapper.getCurrentContextName(), start);
|
||||||
|
see.notifyFailures(seList.size(), seList, start, reportFile);
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
Loading…
Reference in New Issue