package org.gcube.informationsystem.exporter.mapper; import java.io.BufferedWriter; import java.io.File; import java.io.FileWriter; import java.io.IOException; import java.io.PrintWriter; import java.io.StringWriter; import java.io.UnsupportedEncodingException; import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.Calendar; import java.util.Date; import java.util.List; import java.util.UUID; import org.gcube.common.authorization.client.Constants; import org.gcube.common.authorization.library.AuthorizationEntry; import org.gcube.common.authorization.library.provider.SecurityTokenProvider; import org.gcube.common.resources.gcore.Resources; import org.gcube.common.scope.api.ScopeProvider; import org.gcube.informationsystem.base.reference.Element; import org.gcube.informationsystem.exporter.mapper.exception.CreateException; import org.gcube.informationsystem.exporter.mapper.exception.UpdateException; import org.gcube.informationsystem.model.reference.entities.Facet; import org.gcube.informationsystem.model.reference.entities.Resource; import org.gcube.informationsystem.resourceregistry.api.exceptions.AvailableInAnotherContextException; import org.gcube.informationsystem.resourceregistry.api.exceptions.NotFoundException; import org.gcube.informationsystem.resourceregistry.api.exceptions.ResourceRegistryException; import org.gcube.informationsystem.resourceregistry.client.ResourceRegistryClient; import org.gcube.informationsystem.resourceregistry.client.ResourceRegistryClientFactory; import org.gcube.informationsystem.resourceregistry.publisher.ResourceRegistryPublisher; import org.gcube.informationsystem.resourceregistry.publisher.ResourceRegistryPublisherFactory; import org.gcube.informationsystem.utils.ElementMapper; import org.gcube.resourcemanagement.model.reference.entities.resources.GCubeResource; import org.gcube.resources.discovery.client.api.DiscoveryClient; import org.gcube.resources.discovery.client.queries.api.SimpleQuery; import org.gcube.resources.discovery.icclient.ICFactory; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ArrayNode; import com.fasterxml.jackson.databind.node.ObjectNode; /** * @author Luca Frosini (ISTI - CNR) * @param the gCore Resource class to map on gCure Model Resource * @param the gCube Model Resource which map the gCore Resource class */ public abstract class GCoreResourceMapper { /** * Logger */ private static Logger logger = LoggerFactory.getLogger(GCoreResourceMapper.class); public static final String EXPORTED = "EXPORTED"; public static final String EXPORTED_FROM_OLD_GCORE_IS = "EXPORTED_FROM_OLD_GCORE_IS"; public static final String MAPPING_ERROR = "mapping"; public static final String PUBLISHING_ERROR = "publishing"; public static final String TYPE = "type"; public static final String ID = "id"; public static final String ERROR = "error"; public static final String DATE = "date"; public static final String EXCEPTION_TYPE = "exceptionType"; public static final String CREATE = "create"; public static final String UPDATE = "update"; protected final Class grClass; protected final Class rClass; protected final boolean filteredReport; protected final boolean statistics; public static final String UTF8 = "UTF-8"; protected ResourceRegistryPublisher resourceRegistryPublisher; protected ResourceRegistryClient resourceRegistryClient; public static String getCurrentContextName() { String token = SecurityTokenProvider.instance.get(); AuthorizationEntry authorizationEntry = null; try { authorizationEntry = Constants.authorizationService().get(token); } catch (Exception e) { return ScopeProvider.instance.get(); } return authorizationEntry.getContext(); } public static String getDateString(Calendar calendar) { Date date = calendar.getTime(); SimpleDateFormat format = new SimpleDateFormat(Element.DATETIME_PATTERN); return format.format(date); } private String getFileName(Class rClass, String contextFullName, Calendar calendar, String suffix) { String dateString = getDateString(calendar); StringBuffer stringBuffer = new StringBuffer(); stringBuffer.append(contextFullName.replace("/", "_")); stringBuffer.append("-"); stringBuffer.append(rClass.getSimpleName()); stringBuffer.append("-"); stringBuffer.append(dateString); stringBuffer.append(suffix); return stringBuffer.toString(); } public File getReportFile(Class rClass, String contextFullName, Calendar calendar) { return new File(getFileName(rClass, contextFullName, calendar, ".json")); } public File getStatisticFile(Class rClass, String contextFullName, Calendar calendar) { return new File(getFileName(rClass, contextFullName, calendar, ".csv")); } protected GCoreResourceMapper(Class grClass, Class rClass, boolean filteredReport, Boolean statistics) { this.grClass = grClass; this.rClass = rClass; this.resourceRegistryPublisher = ResourceRegistryPublisherFactory.create(); this.resourceRegistryClient = ResourceRegistryClientFactory.create(); this.filteredReport = filteredReport; this.statistics = statistics; } protected List getAll() { SimpleQuery query = ICFactory.queryFor(grClass); DiscoveryClient client = ICFactory.clientFor(grClass); return client.submit(query); } protected String getStringAsUTF8(String s) throws UnsupportedEncodingException { byte bytes[] = s.getBytes("ISO-8859-1"); return new String(bytes, UTF8); } protected R create(R r) throws ResourceRegistryException { return resourceRegistryPublisher.createResource(r); } protected R update(R r) throws ResourceRegistryException { return resourceRegistryPublisher.updateResource(r); } protected R read(UUID uuid) throws ResourceRegistryException { return resourceRegistryClient.getInstance(rClass, uuid); } protected R createOrUpdate(R r) throws ResourceRegistryException { UUID uuid = r.getHeader().getUUID(); boolean update = false; try { update = resourceRegistryClient.exists(rClass, uuid); } catch (NotFoundException e) { update = false; } catch (AvailableInAnotherContextException e) { // This code should never be reached because this should be fixed in // map function resourceRegistryPublisher.addResourceToCurrentContext(r); try { Thread.sleep(100); } catch (Exception ee) { } update = true; } if (update) { logger.debug("Resource with UUID {} exist. It will be updated", uuid); try { return update(r); } catch (ResourceRegistryException e) { throw new UpdateException(e); } } else { logger.debug("Resource with UUID {} does not exist. It will be created", uuid); try { return create(r); } catch (ResourceRegistryException e) { throw new CreateException(e); } } } private ObjectNode addNodeToArray(ArrayNode arrayNode, ObjectMapper objectMapper, GR gr, Exception e) { ObjectNode objectNode = objectMapper.createObjectNode(); objectNode.put(TYPE, grClass.getSimpleName()); objectNode.put(ID, gr.id()); objectNode.put(ERROR, e.getMessage()); arrayNode.add(objectNode); return objectNode; } protected String getUsername() throws Exception { String token = SecurityTokenProvider.instance.get(); return Constants.authorizationService().get(token).getClientInfo().getId(); } /* * * @SuppressWarnings("deprecation") protected Home getHome(HomeManager * manager) throws Exception{ Home home = null; String scope = * ScopeProvider.instance.get(); if(scope!=null){ home = * manager.getHome(getUsername()); }else{ home = manager.getHome(); } return * home; } * * protected WorkspaceFolder getExporterFolder(Workspace ws) throws * Exception { WorkspaceFolder root = ws.getRoot(); * * WorkspaceFolder exporterFolder = null; * if(!ws.exists(ISExporterPluginDeclaration.NAME, root.getId())){ String * folderDescription = String. * format("The folder is used by %s plugin to store informations regarding failures exporting old GCore Resource to new Resource Registry" * , ISExporterPluginDeclaration.NAME); exporterFolder = * ws.createFolder(ISExporterPluginDeclaration.NAME, folderDescription, * root.getId()); }else{ exporterFolder = (WorkspaceFolder) * ws.find(ISExporterPluginDeclaration.NAME, root.getId()); } return * exporterFolder; } * * public static final String APPLICATION_JSON_MIMETYPE = * "application/json"; * * private static final String FOLDER_DESCRIPTION = * "Failures Report Folder for " + ISExporterPluginDeclaration.NAME; * * protected FolderItem publishFileToWorkspace(File file) throws Exception { * HomeManagerFactory factory = HomeLibrary.getHomeManagerFactory(); * HomeManager manager = factory.getHomeManager(); Home home = * getHome(manager); Workspace ws = home.getWorkspace(); WorkspaceFolder * exporterFolder = getExporterFolder(ws); FileInputStream fileInputStream = * new FileInputStream(file); * * FolderItem folderItem = WorkspaceUtil.createExternalFile( exporterFolder, * file.getName(), FOLDER_DESCRIPTION, APPLICATION_JSON_MIMETYPE, * fileInputStream); return folderItem; } */ protected abstract R map(GR gr) throws Exception; protected void notifyFailures(int allSize, List failed, Calendar start, File reportFile) { String contextName = getCurrentContextName(); if (failed.size() == 0) { logger.debug("No needs to create an empty report besause there were no exporting failures on {}", contextName); return; } logger.warn("-------------------------------------------------------"); logger.warn("{} : {} of {} ({} failures) {}s were exported as {}s", contextName, allSize - failed.size(), allSize, failed.size(), grClass.getSimpleName(), rClass.getSimpleName()); logger.warn("-------------------------------------------------------\n\n"); ObjectMapper objectMapper = new ObjectMapper(); ObjectNode objectNode = objectMapper.createObjectNode(); String dateString = getDateString(start); objectNode.put(DATE, dateString); ObjectNode context = objectNode.putObject(contextName); ArrayNode mappingArrayNode = context.putArray(MAPPING_ERROR); ArrayNode publishingArrayNode = context.putArray(PUBLISHING_ERROR); for (GR gr : failed) { logger.trace("-------------------------------------------------------"); if (filteredReport) { try { UUID.fromString(gr.id()); } catch (Exception e) { logger.debug( "{} with has id {} which is not a valid UUID. The reports are filtered and such an error is not reported.", gr.getClass().getSimpleName(), gr.id()); continue; } } R r = null; try { r = map(gr); } catch (Exception e) { addNodeToArray(mappingArrayNode, objectMapper, gr, e); logger.trace("Error exporting {}. The problem was on mapping {} with UUID {}", gr, grClass.getSimpleName(), gr.id()); logger.trace("-------------------------------------------------------\n"); continue; } try { createOrUpdate(r); } catch (Exception e) { if (e.getCause() instanceof NullPointerException) { logger.error("This MUST BE A BUG. Please Investigate"); } if (e.getMessage().contains( "com.orientechnologies.orient.server.distributed.task.ODistributedOperationException")) { logger.error("This is an OrientDB distributed Issue"); } ObjectNode node = addNodeToArray(publishingArrayNode, objectMapper, gr, e); if (e instanceof CreateException) { node.put(EXCEPTION_TYPE, CREATE); } if (e instanceof UpdateException) { node.put(EXCEPTION_TYPE, UPDATE); } try { logger.trace("Error exporting {}. The problem was on publishing {} as {}", gr, rClass.getSimpleName(), ElementMapper.marshal(r)); } catch (JsonProcessingException e1) { logger.trace("", e1); } } logger.trace("-------------------------------------------------------\n"); } if(filteredReport){ if(mappingArrayNode.size()==0 && publishingArrayNode.size()==0){ logger.debug("No need to produce JSON reports because alla the errors where filtered as requested by the input parameters."); return; } } try { String json = objectMapper.writeValueAsString(objectNode); printLineToFile(json, reportFile); /* * try { publishFileToWorkspace(file); file.delete(); }catch * (Exception e) { * logger.error("Error while saving file {} on Workspace", * file.getName(), e); // TODO Use the Social Notification } */ } catch (Exception e) { logger.error("Error exporting JSON error result", e); } logger.trace("-------------------------------------------------------\n\n\n\n\n"); } protected R mapAndPublish(GR gr) throws Exception { R r = map(gr); List facets = ((GCubeResource)r).getIdentificationFacets(); for (Facet f : facets) { f.setAdditionalProperty(EXPORTED, EXPORTED_FROM_OLD_GCORE_IS); } return createOrUpdate(r); } public void export() { String contextName = getCurrentContextName(); Calendar start = Calendar.getInstance(); File statisticsFile = getStatisticFile(rClass, contextName, start); File reportFile = getReportFile(rClass, contextName, start); List all = getAll(); logger.debug("-------------------------------------------------------"); logger.debug("Going to export {} {}s as {}s", all.size(), grClass.getSimpleName(), rClass.getSimpleName()); logger.debug("-------------------------------------------------------"); List failed = new ArrayList<>(); boolean first = true; for (GR gr : all) { R r = null; try { Thread.sleep(300); r = mapAndPublish(gr); } catch (Exception e) { failed.add(gr); } if(statistics && r!=null) { try { addToCSV(gr, r, statisticsFile, first); first = false; }catch (Throwable t) { logger.trace("Unable to report the statistic in file {} for resource with UUID {}", statisticsFile.getAbsolutePath(), r.getHeader().getUUID(), t); } } } notifyFailures(all.size(), failed, start, reportFile); } private void printLineToFile(String line, File file) throws IOException { synchronized (file) { try (FileWriter fw = new FileWriter(file, true); BufferedWriter bw = new BufferedWriter(fw); PrintWriter out = new PrintWriter(bw)) { out.println(line); out.flush(); } catch (IOException e) { throw e; } } } protected String getUnmarshalledResource(GR gr) { StringWriter stringWriter = new StringWriter(); Resources.marshal(gr, stringWriter); return stringWriter.toString(); } private void addToCSV(GR gr, R r, File statisticsFile, boolean addHeader) throws IOException { StringBuffer stringBuffer = new StringBuffer(); if(addHeader) { stringBuffer.append(gr.getClass().getSimpleName()); stringBuffer.append("(byte size)"); stringBuffer.append(","); stringBuffer.append(r.getClass().getSimpleName()); stringBuffer.append("(byte size)"); printLineToFile(stringBuffer.toString(), statisticsFile); stringBuffer = new StringBuffer(); } String unmarshalledGR = getUnmarshalledResource(gr); final byte[] grUTF8Bytes = unmarshalledGR.getBytes("UTF-8"); stringBuffer.append(grUTF8Bytes.length); stringBuffer.append(","); String unmarshalledR = ElementMapper.marshal(r); final byte[] rUTF8Bytes = unmarshalledR.getBytes("UTF-8"); stringBuffer.append(rUTF8Bytes.length); printLineToFile(stringBuffer.toString(), statisticsFile); } }