Added profiles sizes statistics csv generation
git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/information-system/is-exporter-se-plugin@177267 82a268e6-3cf1-43bd-a215-b396298e98cf
This commit is contained in:
parent
8494877b2e
commit
f50cb2d9da
|
@ -22,7 +22,7 @@ public class ISExporterPlugin extends Plugin<ISExporterPluginDeclaration> {
|
|||
private static Logger logger = LoggerFactory.getLogger(ISExporterPlugin.class);
|
||||
|
||||
public static final String FILTERED_REPORT = "FILTERED_REPORT";
|
||||
|
||||
public static final String STATISTICS = "STATISTICS";
|
||||
|
||||
public ISExporterPlugin(ISExporterPluginDeclaration pluginDeclaration) {
|
||||
super(pluginDeclaration);
|
||||
|
@ -46,12 +46,19 @@ public class ISExporterPlugin extends Plugin<ISExporterPluginDeclaration> {
|
|||
filteredReport = false;
|
||||
}
|
||||
|
||||
boolean statistics = false;
|
||||
|
||||
try {
|
||||
statistics = (boolean) inputs.getOrDefault(STATISTICS, false);
|
||||
}catch (Exception e) {
|
||||
statistics = false;
|
||||
}
|
||||
logger.debug("Failure Report are filtered (e.g. Failing UUID are not shown to avoid to produce to much uneeded reports)");
|
||||
|
||||
GenericResourceExporter genericResourceExporter = new GenericResourceExporter(filteredReport);
|
||||
GenericResourceExporter genericResourceExporter = new GenericResourceExporter(filteredReport, statistics);
|
||||
genericResourceExporter.export();
|
||||
|
||||
ServiceEndpointExporter serviceEndpointExporter = new ServiceEndpointExporter(filteredReport);
|
||||
ServiceEndpointExporter serviceEndpointExporter = new ServiceEndpointExporter(filteredReport, statistics);
|
||||
serviceEndpointExporter.export();
|
||||
|
||||
logger.info("{} execution finished", ISExporterPluginDeclaration.NAME);
|
||||
|
|
|
@ -5,6 +5,7 @@ import java.io.File;
|
|||
import java.io.FileWriter;
|
||||
import java.io.IOException;
|
||||
import java.io.PrintWriter;
|
||||
import java.io.StringWriter;
|
||||
import java.io.UnsupportedEncodingException;
|
||||
import java.text.SimpleDateFormat;
|
||||
import java.util.ArrayList;
|
||||
|
@ -16,6 +17,7 @@ import java.util.UUID;
|
|||
import org.gcube.common.authorization.client.Constants;
|
||||
import org.gcube.common.authorization.library.AuthorizationEntry;
|
||||
import org.gcube.common.authorization.library.provider.SecurityTokenProvider;
|
||||
import org.gcube.common.resources.gcore.Resources;
|
||||
import org.gcube.common.scope.api.ScopeProvider;
|
||||
import org.gcube.informationsystem.exporter.mapper.exception.CreateException;
|
||||
import org.gcube.informationsystem.exporter.mapper.exception.UpdateException;
|
||||
|
@ -70,6 +72,7 @@ public abstract class GCoreResourceMapper<GR extends org.gcube.common.resources.
|
|||
protected final Class<GR> grClass;
|
||||
protected final Class<R> rClass;
|
||||
protected final boolean filteredReport;
|
||||
protected final boolean statistics;
|
||||
|
||||
public static final String UTF8 = "UTF-8";
|
||||
|
||||
|
@ -93,17 +96,33 @@ public abstract class GCoreResourceMapper<GR extends org.gcube.common.resources.
|
|||
return format.format(date);
|
||||
}
|
||||
|
||||
public File getFile(Class<?> grClass, String contextFullName, String dateString) {
|
||||
return new File(contextFullName.replace("/", "_") + "-" + grClass.getSimpleName() + "-" + dateString
|
||||
+ "-exporter.json");
|
||||
private String getFileName(Class<?> rClass, String contextFullName, Calendar calendar, String suffix) {
|
||||
String dateString = getDateString(calendar);
|
||||
StringBuffer stringBuffer = new StringBuffer();
|
||||
stringBuffer.append(contextFullName.replace("/", "_"));
|
||||
stringBuffer.append("-");
|
||||
stringBuffer.append(rClass.getSimpleName());
|
||||
stringBuffer.append("-");
|
||||
stringBuffer.append(dateString);
|
||||
stringBuffer.append(suffix);
|
||||
return stringBuffer.toString();
|
||||
}
|
||||
|
||||
public File getReportFile(Class<?> rClass, String contextFullName, Calendar calendar) {
|
||||
return new File(getFileName(rClass, contextFullName, calendar, ".json"));
|
||||
}
|
||||
|
||||
protected GCoreResourceMapper(Class<GR> grClass, Class<R> rClass, boolean filteredReport) {
|
||||
public File getStatisticFile(Class<?> rClass, String contextFullName, Calendar calendar) {
|
||||
return new File(getFileName(rClass, contextFullName, calendar, ".csv"));
|
||||
}
|
||||
|
||||
protected GCoreResourceMapper(Class<GR> grClass, Class<R> rClass, boolean filteredReport, Boolean statistics) {
|
||||
this.grClass = grClass;
|
||||
this.rClass = rClass;
|
||||
this.resourceRegistryPublisher = ResourceRegistryPublisherFactory.create();
|
||||
this.resourceRegistryClient = ResourceRegistryClientFactory.create();
|
||||
this.filteredReport = filteredReport;
|
||||
this.statistics = statistics;
|
||||
}
|
||||
|
||||
protected List<GR> getAll() {
|
||||
|
@ -222,7 +241,7 @@ public abstract class GCoreResourceMapper<GR extends org.gcube.common.resources.
|
|||
|
||||
protected abstract R map(GR gr) throws Exception;
|
||||
|
||||
protected void notifyFailures(int allSize, List<GR> failed) {
|
||||
protected void notifyFailures(int allSize, List<GR> failed, Calendar start, File reportFile) {
|
||||
String contextName = getCurrentContextName();
|
||||
|
||||
if (failed.size() == 0) {
|
||||
|
@ -240,8 +259,8 @@ public abstract class GCoreResourceMapper<GR extends org.gcube.common.resources.
|
|||
|
||||
ObjectNode objectNode = objectMapper.createObjectNode();
|
||||
|
||||
Calendar calendar = Calendar.getInstance();
|
||||
String dateString = getDateString(calendar);
|
||||
String dateString = getDateString(start);
|
||||
|
||||
objectNode.put(DATE, dateString);
|
||||
ObjectNode context = objectNode.putObject(contextName);
|
||||
ArrayNode mappingArrayNode = context.putArray(MAPPING_ERROR);
|
||||
|
@ -312,26 +331,13 @@ public abstract class GCoreResourceMapper<GR extends org.gcube.common.resources.
|
|||
|
||||
try {
|
||||
String json = objectMapper.writeValueAsString(objectNode);
|
||||
|
||||
File file = getFile(rClass, contextName, dateString);
|
||||
synchronized (file) {
|
||||
try (FileWriter fw = new FileWriter(file, true);
|
||||
BufferedWriter bw = new BufferedWriter(fw);
|
||||
PrintWriter out = new PrintWriter(bw)) {
|
||||
out.println(json);
|
||||
out.flush();
|
||||
} catch (IOException e) {
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
|
||||
printLineToFile(json, reportFile);
|
||||
/*
|
||||
* try { publishFileToWorkspace(file); file.delete(); }catch
|
||||
* (Exception e) {
|
||||
* logger.error("Error while saving file {} on Workspace",
|
||||
* file.getName(), e); // TODO Use the Social Notification }
|
||||
*/
|
||||
|
||||
} catch (Exception e) {
|
||||
logger.error("Error exporting JSON error result", e);
|
||||
}
|
||||
|
@ -351,7 +357,11 @@ public abstract class GCoreResourceMapper<GR extends org.gcube.common.resources.
|
|||
}
|
||||
|
||||
public void export() {
|
||||
|
||||
String contextName = getCurrentContextName();
|
||||
Calendar start = Calendar.getInstance();
|
||||
File statisticsFile = getStatisticFile(rClass, contextName, start);
|
||||
File reportFile = getReportFile(rClass, contextName, start);
|
||||
|
||||
List<GR> all = getAll();
|
||||
|
||||
logger.debug("-------------------------------------------------------");
|
||||
|
@ -359,18 +369,70 @@ public abstract class GCoreResourceMapper<GR extends org.gcube.common.resources.
|
|||
logger.debug("-------------------------------------------------------");
|
||||
|
||||
List<GR> failed = new ArrayList<>();
|
||||
|
||||
boolean first = true;
|
||||
|
||||
for (GR gr : all) {
|
||||
R r = null;
|
||||
try {
|
||||
Thread.sleep(300);
|
||||
mapAndPublish(gr);
|
||||
r = mapAndPublish(gr);
|
||||
} catch (Exception e) {
|
||||
failed.add(gr);
|
||||
}
|
||||
|
||||
if(statistics && r!=null) {
|
||||
try {
|
||||
addToCSV(gr, r, statisticsFile, first);
|
||||
first = false;
|
||||
}catch (Throwable t) {
|
||||
logger.trace("Unable to report the statistic in file {} for resource with UUID {}",
|
||||
statisticsFile.getAbsolutePath(), r.getHeader().getUUID(), t);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
notifyFailures(all.size(), failed);
|
||||
notifyFailures(all.size(), failed, start, reportFile);
|
||||
|
||||
}
|
||||
|
||||
private void printLineToFile(String line, File file) throws IOException {
|
||||
synchronized (file) {
|
||||
try (FileWriter fw = new FileWriter(file, true);
|
||||
BufferedWriter bw = new BufferedWriter(fw);
|
||||
PrintWriter out = new PrintWriter(bw)) {
|
||||
out.println(line);
|
||||
out.flush();
|
||||
} catch (IOException e) {
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
protected String getUnmarshalledResource(GR gr) {
|
||||
StringWriter stringWriter = new StringWriter();
|
||||
Resources.marshal(gr, stringWriter);
|
||||
return stringWriter.toString();
|
||||
}
|
||||
|
||||
private void addToCSV(GR gr, R r, File statisticsFile, boolean addHeader) throws IOException {
|
||||
StringBuffer stringBuffer = new StringBuffer();
|
||||
if(addHeader) {
|
||||
stringBuffer.append(gr.getClass().getSimpleName());
|
||||
stringBuffer.append("(byte size)");
|
||||
stringBuffer.append(",");
|
||||
stringBuffer.append(r.getClass().getSimpleName());
|
||||
stringBuffer.append("(byte size)");
|
||||
printLineToFile(stringBuffer.toString(), statisticsFile);
|
||||
stringBuffer = new StringBuffer();
|
||||
}
|
||||
String unmarshalledGR = getUnmarshalledResource(gr);
|
||||
final byte[] grUTF8Bytes = unmarshalledGR.getBytes("UTF-8");
|
||||
stringBuffer.append(grUTF8Bytes.length);
|
||||
stringBuffer.append(",");
|
||||
String unmarshalledR = ISMapper.marshal(r);
|
||||
final byte[] rUTF8Bytes = unmarshalledR.getBytes("UTF-8");
|
||||
stringBuffer.append(rUTF8Bytes.length);
|
||||
printLineToFile(stringBuffer.toString(), statisticsFile);
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
@ -32,8 +32,8 @@ public class GenericResourceExporter extends GCoreResourceMapper<GenericResource
|
|||
public static final String FIXED_VERSION = "1.0.0";
|
||||
public static final String FULL_BODY = "FULL_BODY";
|
||||
|
||||
public GenericResourceExporter(Boolean filteredReport) {
|
||||
super(GenericResource.class, Configuration.class, filteredReport);
|
||||
public GenericResourceExporter(Boolean filteredReport, Boolean statistics) {
|
||||
super(GenericResource.class, Configuration.class, filteredReport, statistics);
|
||||
}
|
||||
|
||||
@Override
|
||||
|
@ -130,5 +130,5 @@ public class GenericResourceExporter extends GCoreResourceMapper<GenericResource
|
|||
|
||||
return configuration;
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
|
|
@ -61,8 +61,8 @@ public class ServiceEndpointExporter extends GCoreResourceMapper<ServiceEndpoint
|
|||
|
||||
public static final String NOT_URI_ENDPOINT = "NOT_URI_ENDPOINT";
|
||||
|
||||
public ServiceEndpointExporter(boolean filteredReport) {
|
||||
super(ServiceEndpoint.class, EService.class, filteredReport);
|
||||
public ServiceEndpointExporter(boolean filteredReport, Boolean statistics) {
|
||||
super(ServiceEndpoint.class, EService.class, filteredReport, statistics);
|
||||
}
|
||||
|
||||
@Override
|
||||
|
|
|
@ -14,14 +14,14 @@ public class ISExporterPluginTest {
|
|||
|
||||
private static Logger logger = LoggerFactory.getLogger(ISExporterPluginTest.class);
|
||||
|
||||
// @Test
|
||||
@Test
|
||||
public void testLaunch() throws ObjectNotFound, Exception {
|
||||
String[] tokens = {
|
||||
ContextTest.GCUBE,
|
||||
ContextTest.GCUBE_DEVSEC,
|
||||
ContextTest.GCUBE_DEVSEC_DEVVRE,
|
||||
// ContextTest.GCUBE_DEVNEXT,
|
||||
// ContextTest.GCUBE_DEVNEXT_NEXTNEXT
|
||||
ContextTest.GCUBE_DEVNEXT,
|
||||
ContextTest.GCUBE_DEVNEXT_NEXTNEXT
|
||||
};
|
||||
|
||||
for(String token : tokens) {
|
||||
|
|
|
@ -18,10 +18,9 @@ public class GCoreResourceMapperTest extends ContextTest {
|
|||
|
||||
private static Logger logger = LoggerFactory.getLogger(GenericResourceExporterTest.class);
|
||||
|
||||
protected File getFile(GCoreResourceMapper<?,?> grm) throws Exception {
|
||||
protected File getReportFile(GCoreResourceMapper<?,?> grm) throws Exception {
|
||||
String contextName = GCoreResourceMapper.getCurrentContextName();
|
||||
String dateString = GCoreResourceMapper.getDateString(Calendar.getInstance());
|
||||
File file = grm.getFile(GenericResourceExporterTest.class, contextName, dateString);
|
||||
File file = grm.getReportFile(GenericResourceExporterTest.class, contextName, Calendar.getInstance());
|
||||
String json = "{}";
|
||||
synchronized (file) {
|
||||
try(FileWriter fw = new FileWriter(file, true);
|
||||
|
@ -50,8 +49,8 @@ public class GCoreResourceMapperTest extends ContextTest {
|
|||
for(String token : tokens){
|
||||
logger.info("\n\n\n-------------------------------------------------------------------------");
|
||||
ContextTest.setContext(token);
|
||||
GenericResourceExporter gre = new GenericResourceExporter(false);
|
||||
File file = getFile(gre);
|
||||
GenericResourceExporter gre = new GenericResourceExporter(false, false);
|
||||
File file = getReportFile(gre);
|
||||
logger.info("\n\n\n {}", file);
|
||||
/*
|
||||
gre.publishFileToWorkspace(file);
|
||||
|
|
|
@ -1,6 +1,8 @@
|
|||
package org.gcube.informationsystem.exporter.mapper;
|
||||
|
||||
import java.io.File;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Calendar;
|
||||
import java.util.List;
|
||||
import java.util.UUID;
|
||||
|
||||
|
@ -29,7 +31,7 @@ public class GenericResourceExporterTest extends ContextTest {
|
|||
|
||||
@Test
|
||||
public void export() {
|
||||
GenericResourceExporter gre = new GenericResourceExporter(false);
|
||||
GenericResourceExporter gre = new GenericResourceExporter(false, false);
|
||||
gre.export();
|
||||
}
|
||||
|
||||
|
@ -81,7 +83,10 @@ public class GenericResourceExporterTest extends ContextTest {
|
|||
DiscoveryClient<GenericResource> client = ICFactory.clientFor(GenericResource.class);
|
||||
List<GenericResource> seList = client.submit(query);
|
||||
|
||||
GenericResourceExporter see = new GenericResourceExporter(false);
|
||||
GenericResourceExporter see = new GenericResourceExporter(false, false);
|
||||
Calendar start = Calendar.getInstance();
|
||||
File reportFile = see.getReportFile(Configuration.class, GCoreResourceMapper.getCurrentContextName(), start);
|
||||
see.notifyFailures(seList.size(), seList, start, reportFile);
|
||||
see.mapAndPublish(seList.get(0));
|
||||
|
||||
}
|
||||
|
|
|
@ -1,6 +1,8 @@
|
|||
package org.gcube.informationsystem.exporter.mapper;
|
||||
|
||||
import java.io.File;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Calendar;
|
||||
import java.util.List;
|
||||
import java.util.UUID;
|
||||
|
||||
|
@ -29,7 +31,7 @@ public class ServiceEndpointExporterTest extends ContextTest{
|
|||
|
||||
@Test
|
||||
public void export(){
|
||||
ServiceEndpointExporter see = new ServiceEndpointExporter(false);
|
||||
ServiceEndpointExporter see = new ServiceEndpointExporter(false, false);
|
||||
see.export();
|
||||
}
|
||||
|
||||
|
@ -77,8 +79,10 @@ public class ServiceEndpointExporterTest extends ContextTest{
|
|||
DiscoveryClient<ServiceEndpoint> client = ICFactory.clientFor(ServiceEndpoint.class);
|
||||
List<ServiceEndpoint> seList = client.submit(query);
|
||||
|
||||
ServiceEndpointExporter see = new ServiceEndpointExporter(false);
|
||||
see.notifyFailures(seList.size(), seList);
|
||||
ServiceEndpointExporter see = new ServiceEndpointExporter(false, false);
|
||||
Calendar start = Calendar.getInstance();
|
||||
File reportFile = see.getReportFile(EService.class, GCoreResourceMapper.getCurrentContextName(), start);
|
||||
see.notifyFailures(seList.size(), seList, start, reportFile);
|
||||
|
||||
}
|
||||
|
||||
|
|
Loading…
Reference in New Issue