is-exporter-se-plugin/src/main/java/org/gcube/informationsystem/exporter/mapper/GCoreResourceMapper.java

package org.gcube.informationsystem.exporter.mapper;

import java.io.BufferedWriter;
import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.io.PrintWriter;
import java.io.StringWriter;
import java.io.UnsupportedEncodingException;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.Date;
import java.util.List;
import java.util.UUID;

import org.gcube.common.authorization.client.Constants;
import org.gcube.common.authorization.library.AuthorizationEntry;
import org.gcube.common.authorization.library.provider.SecurityTokenProvider;
import org.gcube.common.resources.gcore.Resources;
import org.gcube.common.scope.api.ScopeProvider;
import org.gcube.informationsystem.base.reference.Element;
import org.gcube.informationsystem.exporter.mapper.exception.CreateException;
import org.gcube.informationsystem.exporter.mapper.exception.UpdateException;
import org.gcube.informationsystem.model.reference.entities.Facet;
import org.gcube.informationsystem.model.reference.entities.Resource;
import org.gcube.informationsystem.resourceregistry.api.exceptions.AvailableInAnotherContextException;
import org.gcube.informationsystem.resourceregistry.api.exceptions.NotFoundException;
import org.gcube.informationsystem.resourceregistry.api.exceptions.ResourceRegistryException;
import org.gcube.informationsystem.resourceregistry.client.ResourceRegistryClient;
import org.gcube.informationsystem.resourceregistry.client.ResourceRegistryClientFactory;
import org.gcube.informationsystem.resourceregistry.publisher.ResourceRegistryPublisher;
import org.gcube.informationsystem.resourceregistry.publisher.ResourceRegistryPublisherFactory;
import org.gcube.informationsystem.utils.ElementMapper;
import org.gcube.resourcemanagement.model.reference.entities.resources.GCubeResource;
import org.gcube.resources.discovery.client.api.DiscoveryClient;
import org.gcube.resources.discovery.client.queries.api.SimpleQuery;
import org.gcube.resources.discovery.icclient.ICFactory;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.node.ArrayNode;
import com.fasterxml.jackson.databind.node.ObjectNode;

/**
 * @author Luca Frosini (ISTI - CNR)
 * @param <GR> the gCore Resource class to map on gCure Model Resource 
 * @param <R> the gCube Model Resource which map the gCore Resource class
 */
public abstract class GCoreResourceMapper<GR extends org.gcube.common.resources.gcore.Resource, R extends Resource> {

	/**
	 * Logger
	 */
	private static Logger logger = LoggerFactory.getLogger(GCoreResourceMapper.class);

	public static final String EXPORTED = "EXPORTED";
	public static final String EXPORTED_FROM_OLD_GCORE_IS = "EXPORTED_FROM_OLD_GCORE_IS";

	public static final String MAPPING_ERROR = "mapping";
	public static final String PUBLISHING_ERROR = "publishing";
	public static final String TYPE = "type";
	public static final String ID = "id";
	public static final String ERROR = "error";
	public static final String DATE = "date";

	public static final String EXCEPTION_TYPE = "exceptionType";
	public static final String CREATE = "create";
	public static final String UPDATE = "update";

	protected final Class<GR> grClass;
	protected final Class<R> rClass;
	protected final boolean filteredReport;
	protected final boolean statistics;

	public static final String UTF8 = "UTF-8";

	protected ResourceRegistryPublisher resourceRegistryPublisher;
	protected ResourceRegistryClient resourceRegistryClient;

	public static String getCurrentContextName() {
		String token = SecurityTokenProvider.instance.get();
		AuthorizationEntry authorizationEntry = null;
		try {
			authorizationEntry = Constants.authorizationService().get(token);
		} catch (Exception e) {
			return ScopeProvider.instance.get();
		}
		return authorizationEntry.getContext();
	}

	public static String getDateString(Calendar calendar) {
		Date date = calendar.getTime();
		SimpleDateFormat format = new SimpleDateFormat(Element.DATETIME_PATTERN);
		return format.format(date);
	}

	private String getFileName(Class<?> rClass, String contextFullName, Calendar calendar, String suffix) {
		String dateString = getDateString(calendar);
		StringBuffer stringBuffer = new StringBuffer();
		stringBuffer.append(contextFullName.replace("/", "_"));
		stringBuffer.append("-");
		stringBuffer.append(rClass.getSimpleName());
		stringBuffer.append("-");
		stringBuffer.append(dateString);
		stringBuffer.append(suffix);
		return  stringBuffer.toString();
	}
	
	public File getReportFile(Class<?> rClass, String contextFullName, Calendar calendar) {
		return new File(getFileName(rClass, contextFullName, calendar, ".json"));
	}

	public File getStatisticFile(Class<?> rClass, String contextFullName, Calendar calendar) {
		return new File(getFileName(rClass, contextFullName, calendar, ".csv"));
	}
	
	protected GCoreResourceMapper(Class<GR> grClass, Class<R> rClass, boolean filteredReport, Boolean statistics) {
		this.grClass = grClass;
		this.rClass = rClass;
		this.resourceRegistryPublisher = ResourceRegistryPublisherFactory.create();
		this.resourceRegistryClient = ResourceRegistryClientFactory.create();
		this.filteredReport = filteredReport;
		this.statistics = statistics;
	}

	protected List<GR> getAll() {
		SimpleQuery query = ICFactory.queryFor(grClass);
		DiscoveryClient<GR> client = ICFactory.clientFor(grClass);
		return client.submit(query);
	}

	protected String getStringAsUTF8(String s) throws UnsupportedEncodingException {
		byte bytes[] = s.getBytes("ISO-8859-1");
		return new String(bytes, UTF8);
	}

	protected R create(R r) throws ResourceRegistryException {
		return resourceRegistryPublisher.createResource(r);
	}

	protected R update(R r) throws ResourceRegistryException {
		return resourceRegistryPublisher.updateResource(r);
	}

	protected R read(UUID uuid) throws ResourceRegistryException {
		return resourceRegistryClient.getInstance(rClass, uuid);
	}

	protected R createOrUpdate(R r) throws ResourceRegistryException {

		UUID uuid = r.getHeader().getUUID();
		boolean update = false;

		try {
			update = resourceRegistryClient.exists(rClass, uuid);
		} catch (NotFoundException e) {
			update = false;
		} catch (AvailableInAnotherContextException e) {
			// This code should never be reached because this should be fixed in
			// map function
			resourceRegistryPublisher.addResourceToCurrentContext(r);
			try {
				Thread.sleep(100);
			} catch (Exception ee) {
			}
			update = true;
		}

		if (update) {
			logger.debug("Resource with UUID {} exist. It will be updated", uuid);
			try {
				return update(r);
			} catch (ResourceRegistryException e) {
				throw new UpdateException(e);
			}
		} else {
			logger.debug("Resource with UUID {} does not exist. It will be created", uuid);
			try {
				return create(r);
			} catch (ResourceRegistryException e) {
				throw new CreateException(e);
			}
		}

	}

	private ObjectNode addNodeToArray(ArrayNode arrayNode, ObjectMapper objectMapper, GR gr, Exception e) {
		ObjectNode objectNode = objectMapper.createObjectNode();
		objectNode.put(TYPE, grClass.getSimpleName());
		objectNode.put(ID, gr.id());
		objectNode.put(ERROR, e.getMessage());
		arrayNode.add(objectNode);
		return objectNode;
	}

	protected String getUsername() throws Exception {
		String token = SecurityTokenProvider.instance.get();
		return Constants.authorizationService().get(token).getClientInfo().getId();
	}

	/*
	 * 
	 * @SuppressWarnings("deprecation") protected Home getHome(HomeManager
	 * manager) throws Exception{ Home home = null; String scope =
	 * ScopeProvider.instance.get(); if(scope!=null){ home =
	 * manager.getHome(getUsername()); }else{ home = manager.getHome(); } return
	 * home; }
	 * 
	 * protected WorkspaceFolder getExporterFolder(Workspace ws) throws
	 * Exception { WorkspaceFolder root = ws.getRoot();
	 * 
	 * WorkspaceFolder exporterFolder = null;
	 * if(!ws.exists(ISExporterPluginDeclaration.NAME, root.getId())){ String
	 * folderDescription = String.
	 * format("The folder is used by %s plugin to store informations regarding failures exporting old GCore Resource to new Resource Registry"
	 * , ISExporterPluginDeclaration.NAME); exporterFolder =
	 * ws.createFolder(ISExporterPluginDeclaration.NAME, folderDescription,
	 * root.getId()); }else{ exporterFolder = (WorkspaceFolder)
	 * ws.find(ISExporterPluginDeclaration.NAME, root.getId()); } return
	 * exporterFolder; }
	 * 
	 * public static final String APPLICATION_JSON_MIMETYPE =
	 * "application/json";
	 * 
	 * private static final String FOLDER_DESCRIPTION =
	 * "Failures Report Folder for " + ISExporterPluginDeclaration.NAME;
	 * 
	 * protected FolderItem publishFileToWorkspace(File file) throws Exception {
	 * HomeManagerFactory factory = HomeLibrary.getHomeManagerFactory();
	 * HomeManager manager = factory.getHomeManager(); Home home =
	 * getHome(manager); Workspace ws = home.getWorkspace(); WorkspaceFolder
	 * exporterFolder = getExporterFolder(ws); FileInputStream fileInputStream =
	 * new FileInputStream(file);
	 * 
	 * FolderItem folderItem = WorkspaceUtil.createExternalFile( exporterFolder,
	 * file.getName(), FOLDER_DESCRIPTION, APPLICATION_JSON_MIMETYPE,
	 * fileInputStream); return folderItem; }
	 */

	protected abstract R map(GR gr) throws Exception;

	protected void notifyFailures(int allSize, List<GR> failed, Calendar start, File reportFile) {
		String contextName = getCurrentContextName();

		if (failed.size() == 0) {
			logger.debug("No needs to create an empty report besause there were no exporting failures on {}",
					contextName);
			return;
		}

		logger.warn("-------------------------------------------------------");
		logger.warn("{} : {} of {} ({} failures) {}s were exported as {}s", contextName, allSize - failed.size(),
				allSize, failed.size(), grClass.getSimpleName(), rClass.getSimpleName());
		logger.warn("-------------------------------------------------------\n\n");

		ObjectMapper objectMapper = new ObjectMapper();

		ObjectNode objectNode = objectMapper.createObjectNode();

		String dateString = getDateString(start);
		
		objectNode.put(DATE, dateString);
		ObjectNode context = objectNode.putObject(contextName);
		ArrayNode mappingArrayNode = context.putArray(MAPPING_ERROR);
		ArrayNode publishingArrayNode = context.putArray(PUBLISHING_ERROR);

		for (GR gr : failed) {
			logger.trace("-------------------------------------------------------");

			if (filteredReport) {
				try {
					UUID.fromString(gr.id());
				} catch (Exception e) {
					logger.debug(
							"{} with has id {} which is not a valid UUID. The reports are filtered and such an error is not reported.",
							gr.getClass().getSimpleName(), gr.id());
					continue;
				}
			}

			R r = null;
			try {
				r = map(gr);
			} catch (Exception e) {
				addNodeToArray(mappingArrayNode, objectMapper, gr, e);
				logger.trace("Error exporting {}. The problem was on mapping {} with UUID {}", gr,
						grClass.getSimpleName(), gr.id());
				logger.trace("-------------------------------------------------------\n");
				continue;
			}

			try {
				createOrUpdate(r);
			} catch (Exception e) {
				if (e.getCause() instanceof NullPointerException) {
					logger.error("This MUST BE A BUG. Please Investigate");
				}

				if (e.getMessage().contains(
						"com.orientechnologies.orient.server.distributed.task.ODistributedOperationException")) {
					logger.error("This is an OrientDB distributed Issue");
				}

				ObjectNode node = addNodeToArray(publishingArrayNode, objectMapper, gr, e);
				if (e instanceof CreateException) {
					node.put(EXCEPTION_TYPE, CREATE);
				}

				if (e instanceof UpdateException) {
					node.put(EXCEPTION_TYPE, UPDATE);
				}

				try {
					logger.trace("Error exporting {}. The problem was on publishing {} as {}", gr,
							rClass.getSimpleName(), ElementMapper.marshal(r));
				} catch (JsonProcessingException e1) {
					logger.trace("", e1);
				}
			}
			logger.trace("-------------------------------------------------------\n");
		}

		if(filteredReport){
			if(mappingArrayNode.size()==0 && publishingArrayNode.size()==0){
				logger.debug("No need to produce JSON reports because alla the errors where filtered as requested by the input parameters.");
				return;
			}
		}
		
		try {
			String json = objectMapper.writeValueAsString(objectNode);
			printLineToFile(json, reportFile);
			/*
			 * try { publishFileToWorkspace(file); file.delete(); }catch
			 * (Exception e) {
			 * logger.error("Error while saving file {} on Workspace",
			 * file.getName(), e); // TODO Use the Social Notification }
			 */
		} catch (Exception e) {
			logger.error("Error exporting JSON error result", e);
		}

		logger.trace("-------------------------------------------------------\n\n\n\n\n");
	}

	protected R mapAndPublish(GR gr) throws Exception {
		R r = map(gr);

		List<? extends Facet> facets = ((GCubeResource)r).getIdentificationFacets();
		for (Facet f : facets) {
			f.setAdditionalProperty(EXPORTED, EXPORTED_FROM_OLD_GCORE_IS);
		}

		return createOrUpdate(r);
	}

	public void export() {
		String contextName = getCurrentContextName();
		Calendar start = Calendar.getInstance();
		File statisticsFile = getStatisticFile(rClass, contextName, start);
		File reportFile = getReportFile(rClass, contextName, start);
		
		List<GR> all = getAll();

		logger.debug("-------------------------------------------------------");
		logger.debug("Going to export {} {}s as {}s", all.size(), grClass.getSimpleName(), rClass.getSimpleName());
		logger.debug("-------------------------------------------------------");

		List<GR> failed = new ArrayList<>();
		boolean first = true;
		
		for (GR gr : all) {
			R r = null;
			try {
				Thread.sleep(300);
				r = mapAndPublish(gr);
			} catch (Exception e) {
				failed.add(gr);
			}
			
			if(statistics && r!=null) {
				try {
					addToCSV(gr, r, statisticsFile, first);
					first = false;
				}catch (Throwable t) {
					logger.trace("Unable to report the statistic in file {} for resource with UUID {}", 
							statisticsFile.getAbsolutePath(), r.getHeader().getUUID(), t);
				}
			}
		}

		notifyFailures(all.size(), failed, start, reportFile);

	}

	private void printLineToFile(String line, File file) throws IOException {
		synchronized (file) {
			try (FileWriter fw = new FileWriter(file, true);
					BufferedWriter bw = new BufferedWriter(fw);
					PrintWriter out = new PrintWriter(bw)) {
				out.println(line);
				out.flush();
			} catch (IOException e) {
				throw e;
			}
		}
	}
	
	protected String getUnmarshalledResource(GR gr) {
		StringWriter stringWriter = new StringWriter();
		Resources.marshal(gr, stringWriter);
		return stringWriter.toString();
	}
	
	private void addToCSV(GR gr, R r, File statisticsFile, boolean addHeader) throws IOException {
		StringBuffer stringBuffer = new StringBuffer();
		if(addHeader) {
			stringBuffer.append(gr.getClass().getSimpleName());
			stringBuffer.append("(byte size)");
			stringBuffer.append(",");
			stringBuffer.append(r.getClass().getSimpleName());
			stringBuffer.append("(byte size)");
			printLineToFile(stringBuffer.toString(), statisticsFile);
			stringBuffer = new StringBuffer();
		}
		String unmarshalledGR = getUnmarshalledResource(gr);
		final byte[] grUTF8Bytes = unmarshalledGR.getBytes("UTF-8");
		stringBuffer.append(grUTF8Bytes.length);
		stringBuffer.append(",");
		String unmarshalledR = ElementMapper.marshal(r);
		final byte[] rUTF8Bytes = unmarshalledR.getBytes("UTF-8");
		stringBuffer.append(rUTF8Bytes.length);
		printLineToFile(stringBuffer.toString(), statisticsFile);
	}

}