gCubeSystem
/
is-exporter-se-plugin
16
0
Fork 0
Code Issues Pull Requests Releases Activity

Added profiles sizes statistics csv generation

Browse Source 
git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/information-system/is-exporter-se-plugin@177267 82a268e6-3cf1-43bd-a215-b396298e98cf
model-reorganization
Luca Frosini 5 years ago
parent 8494877b2e
commit f50cb2d9da
8 changed files with 123 additions and 46 deletions
Show Stats Download Patch File Download Diff File

13
src/main/java/org/gcube/informationsystem/exporter/ISExporterPlugin.java
Unescape Escape View File

@ -22,7 +22,7 @@ public class ISExporterPlugin extends Plugin<ISExporterPluginDeclaration> {
private static Logger logger = LoggerFactory.getLogger(ISExporterPlugin.class);
public static final String FILTERED_REPORT = "FILTERED_REPORT";
public static final String STATISTICS = "STATISTICS";
public ISExporterPlugin(ISExporterPluginDeclaration pluginDeclaration) {
super(pluginDeclaration);
@ -46,12 +46,19 @@ public class ISExporterPlugin extends Plugin<ISExporterPluginDeclaration> {
filteredReport = false;
}
boolean statistics = false;
try {
statistics = (boolean) inputs.getOrDefault(STATISTICS, false);
}catch (Exception e) {
statistics = false;
}
logger.debug("Failure Report are filtered (e.g. Failing UUID are not shown to avoid to produce to much uneeded reports)");
GenericResourceExporter genericResourceExporter = new GenericResourceExporter(filteredReport);
GenericResourceExporter genericResourceExporter = new GenericResourceExporter(filteredReport, statistics);
genericResourceExporter.export();
ServiceEndpointExporter serviceEndpointExporter = new ServiceEndpointExporter(filteredReport);
ServiceEndpointExporter serviceEndpointExporter = new ServiceEndpointExporter(filteredReport, statistics);
serviceEndpointExporter.export();
logger.info("{} execution finished", ISExporterPluginDeclaration.NAME);

112
src/main/java/org/gcube/informationsystem/exporter/mapper/GCoreResourceMapper.java
Unescape Escape View File

@ -5,6 +5,7 @@ import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.io.PrintWriter;
import java.io.StringWriter;
import java.io.UnsupportedEncodingException;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
@ -16,6 +17,7 @@ import java.util.UUID;
import org.gcube.common.authorization.client.Constants;
import org.gcube.common.authorization.library.AuthorizationEntry;
import org.gcube.common.authorization.library.provider.SecurityTokenProvider;
import org.gcube.common.resources.gcore.Resources;
import org.gcube.common.scope.api.ScopeProvider;
import org.gcube.informationsystem.exporter.mapper.exception.CreateException;
import org.gcube.informationsystem.exporter.mapper.exception.UpdateException;
@ -70,6 +72,7 @@ public abstract class GCoreResourceMapper<GR extends org.gcube.common.resources.
protected final Class<GR> grClass;
protected final Class<R> rClass;
protected final boolean filteredReport;
protected final boolean statistics;
public static final String UTF8 = "UTF-8";
@ -93,17 +96,33 @@ public abstract class GCoreResourceMapper<GR extends org.gcube.common.resources.
return format.format(date);
}
public File getFile(Class<?> grClass, String contextFullName, String dateString) {
return new File(contextFullName.replace("/", "_") + "-" + grClass.getSimpleName() + "-" + dateString
+ "-exporter.json");
private String getFileName(Class<?> rClass, String contextFullName, Calendar calendar, String suffix) {
String dateString = getDateString(calendar);
StringBuffer stringBuffer = new StringBuffer();
stringBuffer.append(contextFullName.replace("/", "_"));
stringBuffer.append("-");
stringBuffer.append(rClass.getSimpleName());
stringBuffer.append("-");
stringBuffer.append(dateString);
stringBuffer.append(suffix);
return stringBuffer.toString();
}
public File getReportFile(Class<?> rClass, String contextFullName, Calendar calendar) {
return new File(getFileName(rClass, contextFullName, calendar, ".json"));
}
protected GCoreResourceMapper(Class<GR> grClass, Class<R> rClass, boolean filteredReport) {
public File getStatisticFile(Class<?> rClass, String contextFullName, Calendar calendar) {
return new File(getFileName(rClass, contextFullName, calendar, ".csv"));
}
protected GCoreResourceMapper(Class<GR> grClass, Class<R> rClass, boolean filteredReport, Boolean statistics) {
this.grClass = grClass;
this.rClass = rClass;
this.resourceRegistryPublisher = ResourceRegistryPublisherFactory.create();
this.resourceRegistryClient = ResourceRegistryClientFactory.create();
this.filteredReport = filteredReport;
this.statistics = statistics;
}
protected List<GR> getAll() {
@ -222,7 +241,7 @@ public abstract class GCoreResourceMapper<GR extends org.gcube.common.resources.
protected abstract R map(GR gr) throws Exception;
protected void notifyFailures(int allSize, List<GR> failed) {
protected void notifyFailures(int allSize, List<GR> failed, Calendar start, File reportFile) {
String contextName = getCurrentContextName();
if (failed.size() == 0) {
@ -240,8 +259,8 @@ public abstract class GCoreResourceMapper<GR extends org.gcube.common.resources.
ObjectNode objectNode = objectMapper.createObjectNode();
Calendar calendar = Calendar.getInstance();
String dateString = getDateString(calendar);
String dateString = getDateString(start);
objectNode.put(DATE, dateString);
ObjectNode context = objectNode.putObject(contextName);
ArrayNode mappingArrayNode = context.putArray(MAPPING_ERROR);
@ -312,26 +331,13 @@ public abstract class GCoreResourceMapper<GR extends org.gcube.common.resources.
try {
String json = objectMapper.writeValueAsString(objectNode);
File file = getFile(rClass, contextName, dateString);
synchronized (file) {
try (FileWriter fw = new FileWriter(file, true);
BufferedWriter bw = new BufferedWriter(fw);
PrintWriter out = new PrintWriter(bw)) {
out.println(json);
out.flush();
} catch (IOException e) {
throw e;
}
}
printLineToFile(json, reportFile);
/*
* try { publishFileToWorkspace(file); file.delete(); }catch
* (Exception e) {
* logger.error("Error while saving file {} on Workspace",
* file.getName(), e); // TODO Use the Social Notification }
*/
} catch (Exception e) {
logger.error("Error exporting JSON error result", e);
}
@ -351,7 +357,11 @@ public abstract class GCoreResourceMapper<GR extends org.gcube.common.resources.
}
public void export() {
String contextName = getCurrentContextName();
Calendar start = Calendar.getInstance();
File statisticsFile = getStatisticFile(rClass, contextName, start);
File reportFile = getReportFile(rClass, contextName, start);
List<GR> all = getAll();
logger.debug("-------------------------------------------------------");
@ -359,18 +369,70 @@ public abstract class GCoreResourceMapper<GR extends org.gcube.common.resources.
logger.debug("-------------------------------------------------------");
List<GR> failed = new ArrayList<>();
boolean first = true;
for (GR gr : all) {
R r = null;
try {
Thread.sleep(300);
mapAndPublish(gr);
r = mapAndPublish(gr);
} catch (Exception e) {
failed.add(gr);
}
if(statistics && r!=null) {
try {
addToCSV(gr, r, statisticsFile, first);
first = false;
}catch (Throwable t) {
logger.trace("Unable to report the statistic in file {} for resource with UUID {}",
statisticsFile.getAbsolutePath(), r.getHeader().getUUID(), t);
}
}
}
notifyFailures(all.size(), failed);
notifyFailures(all.size(), failed, start, reportFile);
}
private void printLineToFile(String line, File file) throws IOException {
synchronized (file) {
try (FileWriter fw = new FileWriter(file, true);
BufferedWriter bw = new BufferedWriter(fw);
PrintWriter out = new PrintWriter(bw)) {
out.println(line);
out.flush();
} catch (IOException e) {
throw e;
}
}
}
protected String getUnmarshalledResource(GR gr) {
StringWriter stringWriter = new StringWriter();
Resources.marshal(gr, stringWriter);
return stringWriter.toString();
}
private void addToCSV(GR gr, R r, File statisticsFile, boolean addHeader) throws IOException {
StringBuffer stringBuffer = new StringBuffer();
if(addHeader) {
stringBuffer.append(gr.getClass().getSimpleName());
stringBuffer.append("(byte size)");
stringBuffer.append(",");
stringBuffer.append(r.getClass().getSimpleName());
stringBuffer.append("(byte size)");
printLineToFile(stringBuffer.toString(), statisticsFile);
stringBuffer = new StringBuffer();
}
String unmarshalledGR = getUnmarshalledResource(gr);
final byte[] grUTF8Bytes = unmarshalledGR.getBytes("UTF-8");
stringBuffer.append(grUTF8Bytes.length);
stringBuffer.append(",");
String unmarshalledR = ISMapper.marshal(r);
final byte[] rUTF8Bytes = unmarshalledR.getBytes("UTF-8");
stringBuffer.append(rUTF8Bytes.length);
printLineToFile(stringBuffer.toString(), statisticsFile);
}
}

6
src/main/java/org/gcube/informationsystem/exporter/mapper/GenericResourceExporter.java
Unescape Escape View File

@ -32,8 +32,8 @@ public class GenericResourceExporter extends GCoreResourceMapper<GenericResource
public static final String FIXED_VERSION = "1.0.0";
public static final String FULL_BODY = "FULL_BODY";
public GenericResourceExporter(Boolean filteredReport) {
super(GenericResource.class, Configuration.class, filteredReport);
public GenericResourceExporter(Boolean filteredReport, Boolean statistics) {
super(GenericResource.class, Configuration.class, filteredReport, statistics);
}
@Override
@ -130,5 +130,5 @@ public class GenericResourceExporter extends GCoreResourceMapper<GenericResource
return configuration;
}
}

4
src/main/java/org/gcube/informationsystem/exporter/mapper/ServiceEndpointExporter.java
Unescape Escape View File

@ -61,8 +61,8 @@ public class ServiceEndpointExporter extends GCoreResourceMapper<ServiceEndpoint
public static final String NOT_URI_ENDPOINT = "NOT_URI_ENDPOINT";
public ServiceEndpointExporter(boolean filteredReport) {
super(ServiceEndpoint.class, EService.class, filteredReport);
public ServiceEndpointExporter(boolean filteredReport, Boolean statistics) {
super(ServiceEndpoint.class, EService.class, filteredReport, statistics);
}
@Override

6
src/test/java/org/gcube/informationsystem/exporter/ISExporterPluginTest.java
Unescape Escape View File

@ -14,14 +14,14 @@ public class ISExporterPluginTest {
private static Logger logger = LoggerFactory.getLogger(ISExporterPluginTest.class);
// @Test
@Test
public void testLaunch() throws ObjectNotFound, Exception {
String[] tokens = {
ContextTest.GCUBE,
ContextTest.GCUBE_DEVSEC,
ContextTest.GCUBE_DEVSEC_DEVVRE,
// ContextTest.GCUBE_DEVNEXT,
// ContextTest.GCUBE_DEVNEXT_NEXTNEXT
ContextTest.GCUBE_DEVNEXT,
ContextTest.GCUBE_DEVNEXT_NEXTNEXT
};
for(String token : tokens) {

9
src/test/java/org/gcube/informationsystem/exporter/mapper/GCoreResourceMapperTest.java
Unescape Escape View File

@ -18,10 +18,9 @@ public class GCoreResourceMapperTest extends ContextTest {
private static Logger logger = LoggerFactory.getLogger(GenericResourceExporterTest.class);
protected File getFile(GCoreResourceMapper<?,?> grm) throws Exception {
protected File getReportFile(GCoreResourceMapper<?,?> grm) throws Exception {
String contextName = GCoreResourceMapper.getCurrentContextName();
String dateString = GCoreResourceMapper.getDateString(Calendar.getInstance());
File file = grm.getFile(GenericResourceExporterTest.class, contextName, dateString);
File file = grm.getReportFile(GenericResourceExporterTest.class, contextName, Calendar.getInstance());
String json = "{}";
synchronized (file) {
try(FileWriter fw = new FileWriter(file, true);
@ -50,8 +49,8 @@ public class GCoreResourceMapperTest extends ContextTest {
for(String token : tokens){
logger.info("\n\n\n-------------------------------------------------------------------------");
ContextTest.setContext(token);
GenericResourceExporter gre = new GenericResourceExporter(false);
File file = getFile(gre);
GenericResourceExporter gre = new GenericResourceExporter(false, false);
File file = getReportFile(gre);
logger.info("\n\n\n {}", file);
/*
gre.publishFileToWorkspace(file);

9
src/test/java/org/gcube/informationsystem/exporter/mapper/GenericResourceExporterTest.java
Unescape Escape View File

@ -1,6 +1,8 @@
package org.gcube.informationsystem.exporter.mapper;
import java.io.File;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.List;
import java.util.UUID;
@ -29,7 +31,7 @@ public class GenericResourceExporterTest extends ContextTest {
@Test
public void export() {
GenericResourceExporter gre = new GenericResourceExporter(false);
GenericResourceExporter gre = new GenericResourceExporter(false, false);
gre.export();
}
@ -81,7 +83,10 @@ public class GenericResourceExporterTest extends ContextTest {
DiscoveryClient<GenericResource> client = ICFactory.clientFor(GenericResource.class);
List<GenericResource> seList = client.submit(query);
GenericResourceExporter see = new GenericResourceExporter(false);
GenericResourceExporter see = new GenericResourceExporter(false, false);
Calendar start = Calendar.getInstance();
File reportFile = see.getReportFile(Configuration.class, GCoreResourceMapper.getCurrentContextName(), start);
see.notifyFailures(seList.size(), seList, start, reportFile);
see.mapAndPublish(seList.get(0));
}

10
src/test/java/org/gcube/informationsystem/exporter/mapper/ServiceEndpointExporterTest.java
Unescape Escape View File

@ -1,6 +1,8 @@
package org.gcube.informationsystem.exporter.mapper;
import java.io.File;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.List;
import java.util.UUID;
@ -29,7 +31,7 @@ public class ServiceEndpointExporterTest extends ContextTest{
@Test
public void export(){
ServiceEndpointExporter see = new ServiceEndpointExporter(false);
ServiceEndpointExporter see = new ServiceEndpointExporter(false, false);
see.export();
}
@ -77,8 +79,10 @@ public class ServiceEndpointExporterTest extends ContextTest{
DiscoveryClient<ServiceEndpoint> client = ICFactory.clientFor(ServiceEndpoint.class);
List<ServiceEndpoint> seList = client.submit(query);
ServiceEndpointExporter see = new ServiceEndpointExporter(false);
see.notifyFailures(seList.size(), seList);
ServiceEndpointExporter see = new ServiceEndpointExporter(false, false);
Calendar start = Calendar.getInstance();
File reportFile = see.getReportFile(EService.class, GCoreResourceMapper.getCurrentContextName(), start);
see.notifyFailures(seList.size(), seList, start, reportFile);
}

Write Preview
Loading…
Cancel
Save