software-versions-processor-lib/src/main/java/org/gcube/common/software/export/zenodo/ZenodoExporter.java

package org.gcube.common.software.export.zenodo;

import java.io.BufferedReader;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.net.HttpURLConnection;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.TimeUnit;

import javax.ws.rs.client.Client;
import javax.ws.rs.client.ClientBuilder;
import javax.ws.rs.client.Entity;
import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.Response;

import org.gcube.com.fasterxml.jackson.databind.JsonNode;
import org.gcube.com.fasterxml.jackson.databind.ObjectMapper;
import org.gcube.com.fasterxml.jackson.databind.node.ArrayNode;
import org.gcube.com.fasterxml.jackson.databind.node.JsonNodeType;
import org.gcube.com.fasterxml.jackson.databind.node.ObjectNode;
import org.gcube.common.gxhttp.request.GXHTTPStringRequest;
import org.gcube.common.software.analyser.AnalyserFactory;
import org.gcube.common.software.config.Config;
import org.gcube.common.software.export.SoftwareArtifactExporter;
import org.gcube.common.software.model.ElaborationType;
import org.gcube.common.software.model.SoftwareArtifactConfig;
import org.gcube.common.software.model.SoftwareArtifactFile;
import org.gcube.common.software.utils.Utils;
import org.glassfish.jersey.client.ClientProperties;
import org.glassfish.jersey.media.multipart.FormDataMultiPart;
import org.glassfish.jersey.media.multipart.MultiPartFeature;
import org.glassfish.jersey.media.multipart.file.FileDataBodyPart;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

/**
 * @author Luca Frosini (ISTI - CNR)
 */
public class ZenodoExporter extends SoftwareArtifactExporter {
	
	private static final Logger logger = LoggerFactory.getLogger(ZenodoExporter.class);
	
	public static final String EXPORT_FILENAME_EXTENSION = ".json";
	
	public static final String GUCBE_ZENODO_SOFTWARE_DEPOSIT = "gCubeSoftwareDeposit";
	
	public static final String HTML_DESCRIPTION_CONFIG_FIELD_NAME = "html_description";
	public static final String ADDITIONAL_HTML_DESCRIPTION_CONFIG_FIELD_NAME = "additional_html_description";
	public static final String SKIP_GRANTS_CONFIG_FIELD_NAME = "skip_grants";
	
	public static final String METADATA_FIELD_NAME = "metadata";
	public static final String COMMUNITIES_FIELD_NAME = "communities";
	
	public static final String DEPOSITIONS_COLLECTION_PATH = "/api/deposit/depositions";
	public static final String DEPOSITION_PATH = DEPOSITIONS_COLLECTION_PATH + "/:id";
	
	public static final String RECORD_PATH = "/api/records/:id";
	
	public static final String DEPOSTION_FILES_PATH = 		DEPOSITION_PATH + "/files";
	
	public static final String DEPOSTION_NEW_VERSION_PATH = DEPOSITION_PATH + "/actions/newversion";
	public static final String DEPOSTION_EDIT_PATH = 		DEPOSITION_PATH + "/actions/edit";
	public static final String DEPOSTION_PUBLISH_PATH = 	DEPOSITION_PATH + "/actions/publish";
	
	protected URL zenodoBaseURL;
	protected String accessToken;
	
	protected String zenodoID;
	protected JsonNode response;
	protected String doiBaseURL;
	
	protected String getZenodoIDFromDOIURL(URL doiURL) {
		return getZenodoIDFromDOIURL(doiURL.toString());
	}
	
	protected String getZenodoIDFromDOIURL(String doiURL) {
		return doiURL.replace(doiBaseURL, "");
	}
	
	protected Map<String, String> getAccessTokenQueryParamters() {
		Map<String, String> map = new HashMap<>();
		map.put("access_token", accessToken);
		return map;
	}
	
	public ZenodoExporter() {
		super(ZenodoExporter.EXPORT_FILENAME_EXTENSION);
	}
	
	protected void addFilesToDeposition(List<File> files ) throws Exception {
		String depositID = getZenodoIDFromDOIURL(softwareVersionConfig.getVersionDOIURL());
		String newFilePath = DEPOSTION_FILES_PATH.replace(":id", depositID);
		URL url = new URL(zenodoBaseURL, newFilePath);
		
		for(File file : files) {
			Client client = ClientBuilder.newClient()
					.property(ClientProperties.SUPPRESS_HTTP_COMPLIANCE_VALIDATION, true);;
			client.register(MultiPartFeature.class);

			FormDataMultiPart multi=new FormDataMultiPart();
			FileDataBodyPart fileDataBodyPart = new FileDataBodyPart("file", file, MediaType.APPLICATION_OCTET_STREAM_TYPE);
			multi.field("name", file.getName());
			multi.bodyPart(fileDataBodyPart);

			Response response = client.target(url.toURI().toString())
					.queryParam("access_token", accessToken)
					.request("application/json")
					.post(Entity.entity(multi,multi.getMediaType()));

			int statusCode = response.getStatus();
			if(statusCode>400) {
				throw new RuntimeException("Error while uploading file " + file.getAbsolutePath());
			}
			
		}
	}
	
	protected void updateMetadata() throws Exception {
		GXHTTPStringRequest gxHTTPStringRequest = GXHTTPStringRequest.newRequest(zenodoBaseURL.toString());
		gxHTTPStringRequest.isExternalCall(true);
		gxHTTPStringRequest.from(GUCBE_ZENODO_SOFTWARE_DEPOSIT);
		gxHTTPStringRequest.queryParams(getAccessTokenQueryParamters());
		gxHTTPStringRequest.header("Content-Type", "application/json");
		gxHTTPStringRequest.header("Accept", "application/json");
		
		String id = getZenodoIDFromDOIURL(softwareVersionConfig.getVersionDOIURL());
		gxHTTPStringRequest.path(DEPOSITION_PATH.replace(":id", id));
		
		ObjectNode metadata = generateMetadata();
		
		HttpURLConnection httpURLConnection = gxHTTPStringRequest.put(Utils.getObjectMapper().writeValueAsString(metadata));
		getResponse(httpURLConnection);
	}
	
	protected void publishToZenodo() throws Exception {
		GXHTTPStringRequest gxHTTPStringRequest = GXHTTPStringRequest.newRequest(zenodoBaseURL.toString());
		gxHTTPStringRequest.isExternalCall(true);
		gxHTTPStringRequest.from(GUCBE_ZENODO_SOFTWARE_DEPOSIT);
		gxHTTPStringRequest.queryParams(getAccessTokenQueryParamters());
		gxHTTPStringRequest.header("Content-Type", "application/json");
		gxHTTPStringRequest.header("Accept", "application/json");
		
		String id = getZenodoIDFromDOIURL(softwareVersionConfig.getVersionDOIURL());
		gxHTTPStringRequest.path(DEPOSTION_PUBLISH_PATH.replace(":id", id));

		HttpURLConnection httpURLConnection = gxHTTPStringRequest.post();
		getResponse(httpURLConnection);
	}
	
	protected void finalize() throws Exception {
		List<File> files = new ArrayList<>(); 
		for(SoftwareArtifactFile svf : softwareVersionConfig.getFiles()) {
			File file = svf.downloadFile();
			files.add(file);
			Thread.sleep(TimeUnit.SECONDS.toMillis(1));
		}
		
		try {
			//Add depositionFiles
			addFilesToDeposition(files);
			
			//Update deposit metadata
			updateMetadata();
			
			// Publish the version
			publishToZenodo();
		}finally {
			for(File file : files) {
				logger.trace("Going to delete file {}", file.getAbsolutePath());
				if(!file.exists()) {
		        	throw new RuntimeException(file.getAbsolutePath() + " does not exist");
		        }
				try {
					int i = 0;
					while(!file.delete() && i<10) {
						int millis = 100;
						logger.warn("File {} not deleted at the attemp {}. Retrying in {} milliseconds.", file.getAbsolutePath(), i+1, millis);
						++i;
						Thread.sleep(millis);
					}
					
					if(i==10) {
						logger.warn("After {} attemps the file {} was not deleted. Trying using deleteOnExit().", i, file.getAbsolutePath());
						file.deleteOnExit();
					}
				}catch (Exception e) {
					logger.error("Unable to delete file {}", file.getAbsolutePath());
				}
			}
		}
	}
	
	protected StringBuilder getStringBuilder(InputStream inputStream) throws IOException {
		StringBuilder result = new StringBuilder();
		try (BufferedReader reader = new BufferedReader(new InputStreamReader(inputStream))) {
			String line;
			while ((line = reader.readLine()) != null) {
				result.append(line);
			}
		}

		return result;
	}
	
	public JsonNode getResponse(HttpURLConnection connection) throws Exception {
		
		try {
			
			int responseCode = connection.getResponseCode();
			String responseMessage = connection.getResponseMessage();
			
			logger.trace("Response {} {}", responseCode, responseMessage);
			
			if(responseCode == HttpURLConnection.HTTP_NO_CONTENT) {
				return null;
			}
			
			if(responseCode == HttpURLConnection.HTTP_NOT_FOUND) {
				throw new RuntimeException(responseCode + " " + responseMessage);
			}
			
			if(responseCode == HttpURLConnection.HTTP_FORBIDDEN) {
				throw new RuntimeException(responseCode + " " + responseMessage);
			}
			
			if(responseCode >= HttpURLConnection.HTTP_BAD_REQUEST) {
				
				InputStream inputStream = connection.getErrorStream();
				StringBuilder result = getStringBuilder(inputStream);
				
				String res = result.toString();
				
				throw new RuntimeException(res);
				
			}
			
			StringBuilder result = getStringBuilder(connection.getInputStream());
			
			String res = result.toString();
			logger.trace("Server returned content : {}", res);
			
			return Utils.getObjectMapper().readTree(res);
		} finally {
			connection.disconnect();
		}
	}
	
	protected String createZenodoDOIURLFromID(String id) throws MalformedURLException {
		return doiBaseURL + id;
	}
	
	public void create() throws Exception {
		GXHTTPStringRequest gxHTTPStringRequest = GXHTTPStringRequest.newRequest(zenodoBaseURL.toString());
		gxHTTPStringRequest.isExternalCall(true);
		gxHTTPStringRequest.from(GUCBE_ZENODO_SOFTWARE_DEPOSIT);
		gxHTTPStringRequest.queryParams(getAccessTokenQueryParamters());
		gxHTTPStringRequest.header("Content-Type", "application/json");
		gxHTTPStringRequest.header("Accept", "application/json");
		gxHTTPStringRequest.path(DEPOSITIONS_COLLECTION_PATH);
		
		ObjectNode metadata = generateMetadata();
		
		HttpURLConnection httpURLConnection = gxHTTPStringRequest.post(Utils.getObjectMapper().writeValueAsString(metadata));
		response = getResponse(httpURLConnection);
		
		String conceptDOIURL = createZenodoDOIURLFromID(response.get("conceptrecid").asText());
		softwareVersionConfig.setConceptDOIURL(conceptDOIURL);
		String versionDOIURL = createZenodoDOIURLFromID(response.get("id").asText());
		softwareVersionConfig.setVersionDOIURL(versionDOIURL);
		finalize();
	}
	
	private ArrayNode getAuthors(){
		ArrayNode authors = softwareVersionConfig.getAuthors().deepCopy();
		return authors;
	}
	
	private String getDescription() {
		StringBuffer stringBuffer = new StringBuffer();
		stringBuffer.append(softwareVersionConfig.getAdditionalProperty(HTML_DESCRIPTION_CONFIG_FIELD_NAME).asText());
		
		if(exporterConfig.getProperty(ADDITIONAL_HTML_DESCRIPTION_CONFIG_FIELD_NAME)!=null) {
			String additionalHTMLDescription = exporterConfig.getProperty(ADDITIONAL_HTML_DESCRIPTION_CONFIG_FIELD_NAME).asText();
			stringBuffer.append(additionalHTMLDescription);
		}

		return stringBuffer.toString();
	}
	
	private ArrayNode getGrants(){
		ObjectMapper objectMapper = Utils.getObjectMapper();
		ArrayNode grants = objectMapper.createArrayNode();
		ArrayNode arrayNode = (ArrayNode) exporterConfig.getProperty(SKIP_GRANTS_CONFIG_FIELD_NAME);
		Set<String> idToSkip = new HashSet<>();
		for(JsonNode idNode : arrayNode) {
			idToSkip.add(idNode.asText());
		}
		for(JsonNode g : softwareVersionConfig.getGrants()) {
			String id = g.get("id").asText();
			if(idToSkip.contains(id)) {
				continue;
			}
			ObjectNode grant = objectMapper.createObjectNode();
			grant.put("id", id);
			grants.add(grant);
		}
		return grants;
	}
	
	private ArrayNode getKeywords(){
		Set<String> keywords = softwareVersionConfig.getKeywords();
		ObjectMapper objectMapper = Utils.getObjectMapper();
		ArrayNode keywordsArrayNode = objectMapper.createArrayNode();
		for(String keyword : keywords) {
			keywordsArrayNode.add(keyword);
		}
		return keywordsArrayNode;
	}
	
	private ArrayNode getCommunities() {
		return (ArrayNode) softwareVersionConfig.getAdditionalProperty(COMMUNITIES_FIELD_NAME);
	}

	private String getLicense() {
		return softwareVersionConfig.getLicense().get("id").asText();
	}
	
	private String getDate() {
		return Utils.getDateAsString(softwareVersionConfig.getDate());
	}
	
	private ObjectNode generateMetadata() {
		ObjectMapper objectMapper = Utils.getObjectMapper();
		ObjectNode metadatWrapper = objectMapper.createObjectNode();
		
		ObjectNode metadata = objectMapper.createObjectNode();
		metadata.put("access_right", "open");
		metadata.put("upload_type", "software");
		metadata.replace("creators", getAuthors());
		metadata.put("description", getDescription());
		metadata.replace("communities", getCommunities());
		metadata.replace("grants", getGrants());
		metadata.replace("keywords", getKeywords());
		metadata.put("license", getLicense());
		metadata.put("publication_date", getDate());
		metadata.put("title", softwareVersionConfig.getTitle());
		metadata.put("version", softwareVersionConfig.getVersion());
		
		metadatWrapper.set(METADATA_FIELD_NAME, metadata);
		return metadatWrapper;
	}

	public void update() throws Exception {
		// Enable deposit edit 
		GXHTTPStringRequest gxHTTPStringRequest = GXHTTPStringRequest.newRequest(zenodoBaseURL.toString());
		gxHTTPStringRequest.isExternalCall(true);
		gxHTTPStringRequest.from(GUCBE_ZENODO_SOFTWARE_DEPOSIT);
		gxHTTPStringRequest.queryParams(getAccessTokenQueryParamters());
		gxHTTPStringRequest.header("Accept", "application/json");
		String id = getZenodoIDFromDOIURL(softwareVersionConfig.getVersionDOIURL());
		gxHTTPStringRequest.path(DEPOSTION_EDIT_PATH.replace(":id", id));
		HttpURLConnection httpURLConnection = gxHTTPStringRequest.post();
		getResponse(httpURLConnection);
		
		//Update deposit metadata
		updateMetadata();
		
		// Publish the version
		publishToZenodo();
	}
	
	/**
	 * Remove previous depositionFiles
	 * @throws Exception
	 */
	protected void deletePreviousFiles() throws Exception {
		ArrayNode files = (ArrayNode) response.get("files");
		for(int i=0; i<files.size(); i++) {
			ObjectNode file = (ObjectNode) files.get(i);
			String fileURLString = file.get("links").get("self").asText();
			GXHTTPStringRequest gxHTTPStringRequest = GXHTTPStringRequest.newRequest(fileURLString);
			gxHTTPStringRequest.isExternalCall(true);
			gxHTTPStringRequest.from(GUCBE_ZENODO_SOFTWARE_DEPOSIT);
			gxHTTPStringRequest.queryParams(getAccessTokenQueryParamters());
			HttpURLConnection httpURLConnection = gxHTTPStringRequest.delete();
			getResponse(httpURLConnection);
		}
	}
	
	public void newVersion() throws Exception {
		
		// Reading Record using conceptID to get the latest published version
		GXHTTPStringRequest gxHTTPStringRequest = GXHTTPStringRequest.newRequest(zenodoBaseURL.toString()); 
		gxHTTPStringRequest.isExternalCall(true);
		gxHTTPStringRequest.from(GUCBE_ZENODO_SOFTWARE_DEPOSIT);
		gxHTTPStringRequest.queryParams(getAccessTokenQueryParamters());
		gxHTTPStringRequest.header("Content-Type", "application/json");
		gxHTTPStringRequest.header("Accept", "application/json");
		
		String conceptDOIURL = softwareVersionConfig.getConceptDOIURL();
		String conceptID = getZenodoIDFromDOIURL(conceptDOIURL);
		gxHTTPStringRequest.path(RECORD_PATH.replace(":id", conceptID));
		HttpURLConnection httpURLConnection = gxHTTPStringRequest.get();
		JsonNode jsonNode = getResponse(httpURLConnection);
		
		
		/*
		 * Comparing obtained latestDOI and its declared version with the previuos version DOI and its declared version.
		 * If they differs the configuration is not up to date and must be fixed 
		 * this should avoid errors on softwareConcept.
		 */
		String latestVersionDOI = jsonNode.get("links").get("doi").asText();
		String previousVersionDOI = softwareVersionConfig.getPrevious().getVersionDOIURL().toString(); 
		if(previousVersionDOI.compareTo(latestVersionDOI)!=0) {
			logger.error("Zenodo obtained latest DOI {} != {} DOI from previous version", latestVersionDOI, previousVersionDOI);
			throw new RuntimeException("It seems that your json is not up to date with Zenodo.");
		}
		String latestVersionVersion = jsonNode.get("metadata").get("version").asText();
		String previousVersionVersion = softwareVersionConfig.getPrevious().getVersion().toString();
		if(latestVersionVersion.compareTo(previousVersionVersion)!=0) {
			logger.error("Zenodo obtained latest Version {} != {} Version from previous version", latestVersionVersion, previousVersionVersion);
			throw new RuntimeException("It seems that your json is not up to date with Zenodo.");
		}
		
		
		// Creating new version from latest deposited version
		gxHTTPStringRequest = GXHTTPStringRequest.newRequest(zenodoBaseURL.toString()); 
		gxHTTPStringRequest.isExternalCall(true);
		gxHTTPStringRequest.from(GUCBE_ZENODO_SOFTWARE_DEPOSIT);
		gxHTTPStringRequest.queryParams(getAccessTokenQueryParamters());
		gxHTTPStringRequest.header("Content-Type", "application/json");
		gxHTTPStringRequest.header("Accept", "application/json");
		String latestID = getZenodoIDFromDOIURL(latestVersionDOI);
		gxHTTPStringRequest.path(DEPOSTION_NEW_VERSION_PATH.replace(":id", latestID));
		httpURLConnection = gxHTTPStringRequest.post();
		jsonNode = getResponse(httpURLConnection);
		
		
		// Getting draft new Version ID
		String draftURL = jsonNode.get("links").get("latest_draft").asText();
		String draftID = draftURL.replace(zenodoBaseURL.toString() + DEPOSITIONS_COLLECTION_PATH + "/", "");
		
		
		gxHTTPStringRequest = GXHTTPStringRequest.newRequest(zenodoBaseURL.toString());
		gxHTTPStringRequest.isExternalCall(true);
		gxHTTPStringRequest.from(GUCBE_ZENODO_SOFTWARE_DEPOSIT);
		gxHTTPStringRequest.queryParams(getAccessTokenQueryParamters());
		gxHTTPStringRequest.header("Accept", "application/json");
		gxHTTPStringRequest.path(DEPOSITION_PATH.replace(":id", draftID));
		httpURLConnection = gxHTTPStringRequest.get();
		response = getResponse(httpURLConnection);
		
		// The reserved DOI of this created new version will be
		String newVersionDOIURL = response.get("doi_url").asText();
		softwareVersionConfig.setVersionDOIURL(newVersionDOIURL);
		
		// Remove previous depositionFiles
		deletePreviousFiles();
		
		finalize();
	}
	
	protected String getConfig(String propertyName) throws Exception {
		String conf = null;
		JsonNode node = exporterConfig.getProperty(propertyName);
		if(node == null || node.getNodeType()==JsonNodeType.NULL) {
			conf = Config.getProperties().getProperty(propertyName);
		}
		if(conf==null) {
			throw new Exception("No configuration for '" + propertyName + "' property found.");
		}
		return conf;
	}
	
	protected void getZenodoConnectionConfig() throws Exception {
		this.zenodoBaseURL = new URL(getConfig("zenodo_base_url"));
		this.accessToken = getConfig("zenodo_access_token");
		this.doiBaseURL = getConfig("doi_base_url");
	}
	
	@Override
	public void export() throws Exception {
		if(first) {
			File exportFile = super.getOutputFile();
			if(exportFile.exists()) {
				exportFile.delete();
			}
			exportFile.createNewFile();
		}
		
		getZenodoConnectionConfig();
		
		String title = softwareVersionConfig.getTitle();
		
		ElaborationType publish = exporterConfig.getElaboration();
		
		if(publish==ElaborationType.NONE) {
			logger.info("Zenodo Deposit is disabled for {}.",title);
			return;
		}
		
		if(softwareVersionConfig.getVersionDOIURL()!=null) {
			
			softwareVersionConfig.setNewDeposition(false);
			
			if(publish==ElaborationType.ALL || 
					publish==ElaborationType.UPDATE_ONLY) {
				logger.info("Going to update {}.",title);
				update();
			}else {
				logger.info("{} has been already deposited.", title);
			}
			
		}else {
			if(publish==ElaborationType.ALL || 
					publish==ElaborationType.NEW) {
				logger.info("Going to deposit {}", title);
				
				softwareVersionConfig.setNewDeposition(true);
				
				if(softwareVersionConfig.getConceptDOIURL()==null) {
					create();
				}else {
					newVersion();				
				}
			}
		}
		
	}

	protected ObjectNode getObjectNode() throws Exception {
		ObjectMapper objectMapper = Utils.getObjectMapper();
		ObjectNode toBeExported = objectMapper.createObjectNode();
		toBeExported.replace(AnalyserFactory.CONFIGURATION_PROPERTY_NAME, globalConfig.getOriginalJson().deepCopy());
		ArrayNode array = objectMapper.createArrayNode();
		SoftwareArtifactConfig previous = softwareVersionConfig;
		boolean firstNode = true;
		while(previous!=null){
			ObjectNode node = previous.getOriginalJson().deepCopy();
			node.put(SoftwareArtifactConfig.CONCEPT_DOI_URL_PROPERTY_NAME, previous.getConceptDOIURL());
			if(firstNode) {
				toBeExported.put(SoftwareArtifactConfig.CONCEPT_DOI_URL_PROPERTY_NAME, previous.getConceptDOIURL());
				firstNode = false;
			}
			node.put(SoftwareArtifactConfig.VERSION_DOI_URL_PROPERTY_NAME, previous.getVersionDOIURL());
			array.insert(0, node);
			previous = previous.getPrevious();
		}
		toBeExported.replace(AnalyserFactory.ARTIFACTS_PROPERTY_NAME, array);
		return toBeExported;
	}
	
	
	protected void writeObjectNodeToFile(ObjectNode toBeExported, File file) throws Exception {
		ObjectMapper objectMapper = Utils.getObjectMapper();
		objectMapper.writeValue(file, toBeExported);
	}
	
	@Override
	public File getOutputFile() throws Exception {
		File exportFile = super.getOutputFile();
		if(last) {
			ObjectNode toBeExported = getObjectNode();
			writeObjectNodeToFile(toBeExported, exportFile);
		}
		return exportFile;
	}

}