package eu.dnetlib.data.collector.plugins.ftp;

import java.io.IOException;
import java.io.OutputStream;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.*;

import eu.dnetlib.data.collector.rmi.CollectorServiceRuntimeException;
import org.apache.commons.io.output.ByteArrayOutputStream;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.commons.net.ftp.FTPClient;
import org.apache.commons.net.ftp.FTPFile;
import org.apache.commons.net.ftp.FTPReply;
import org.joda.time.DateTime;
import org.joda.time.format.DateTimeFormat;
import org.joda.time.format.DateTimeFormatter;

/**
 *
 * @author Author: Andrea Mannocci
 *
 */
public class FtpIterator implements Iterator<String> {

	private static final Log log = LogFactory.getLog(FtpIterator.class);

	private static final int MAX_RETRIES = 5;
	private static final int DEFAULT_TIMEOUT = 30000;
	private static final long BACKOFF_MILLIS = 10000;

	private FTPClient ftpClient;
	private String ftpServerAddress;
	private String remoteFtpBasePath;
	private String username;
	private String password;
	private boolean isRecursive;
	private Set<String> extensionsSet;
	private boolean incremental;
	private DateTime fromDate = null;
	private DateTimeFormatter simpleDateTimeFormatter = DateTimeFormat.forPattern("yyyy-MM-dd");

	private Queue<String> queue;

	public FtpIterator(final String baseUrl, final String username, final String password, final boolean isRecursive,
			final Set<String> extensionsSet, String fromDate) {
		this.username = username;
		this.password = password;
		this.isRecursive = isRecursive;
		this.extensionsSet = extensionsSet;
		this.incremental = StringUtils.isNotBlank(fromDate);
		if (incremental) {
			//I expect fromDate in the format 'yyyy-MM-dd'. See class eu.dnetlib.msro.workflows.nodes.collect.FindDateRangeForIncrementalHarvestingJobNode .
			this.fromDate = DateTime.parse(fromDate, simpleDateTimeFormatter);
			log.debug("fromDate string: " + fromDate + " -- parsed: " + this.fromDate.toString());
		}
		try {
			URL server = new URL(baseUrl);
			this.ftpServerAddress = server.getHost();
			this.remoteFtpBasePath = server.getPath();
		} catch (MalformedURLException e1) {
			throw new CollectorServiceRuntimeException("Malformed URL exception " + baseUrl);
		}

		connectToFtpServer();
		initializeQueue();
	}

	private void connectToFtpServer() {
		ftpClient = new FTPClient();
		ftpClient.setDefaultTimeout(DEFAULT_TIMEOUT);
		ftpClient.setDataTimeout(DEFAULT_TIMEOUT);
		ftpClient.setConnectTimeout(DEFAULT_TIMEOUT);
		try {
			ftpClient.connect(ftpServerAddress);

			// try to login
			if (!ftpClient.login(username, password)) {
				ftpClient.logout();
				throw new CollectorServiceRuntimeException("Unable to login to FTP server " + ftpServerAddress);
			}
			int reply = ftpClient.getReplyCode();
			if (!FTPReply.isPositiveCompletion(reply)) {
				ftpClient.disconnect();
				throw new CollectorServiceRuntimeException("Unable to connect to FTP server " + ftpServerAddress);
			}

			ftpClient.enterLocalPassiveMode();
			log.info("Connected to FTP server " + ftpServerAddress);
			log.info(String.format("FTP collecting from %s with recursion = %s", remoteFtpBasePath, isRecursive));
		} catch (IOException e) {
			throw new CollectorServiceRuntimeException("Unable to connect to FTP server " + ftpServerAddress);
		}
	}

	private void disconnectFromFtpServer() {
		try {
			if (ftpClient.isConnected()) {
				ftpClient.logout();
				ftpClient.disconnect();
			}
		} catch (IOException e) {
			log.error("Failed to logout & disconnect from the FTP server", e);
		}
	}

	private void initializeQueue() {
		queue = new LinkedList<String>();
		listDirectoryRecursive(remoteFtpBasePath, "");
	}

	private void listDirectoryRecursive(final String parentDir, final String currentDir) {
		String dirToList = parentDir;
		if (!currentDir.equals("")) {
			dirToList += "/" + currentDir;
		}
		FTPFile[] subFiles;
		try {
			subFiles = ftpClient.listFiles(dirToList);
			if ((subFiles != null) && (subFiles.length > 0)) {
				for (FTPFile aFile : subFiles) {
					String currentFileName = aFile.getName();

					if (currentFileName.equals(".") || currentFileName.equals("..")) {
						// skip parent directory and directory itself
						continue;
					}
					if (aFile.isDirectory()) {
						if (isRecursive) {
							listDirectoryRecursive(dirToList, currentFileName);
						}
					} else {
						// test the file for extensions compliance and, just in case, add it to the list.
						for (String ext : extensionsSet) {
							if (currentFileName.endsWith(ext)) {
								//incremental mode: let's check the last update date
								if(incremental){
									Calendar timestamp = aFile.getTimestamp();
									DateTime lastModificationDate = new DateTime(timestamp);
									if(lastModificationDate.isAfter(fromDate)){
										queue.add(dirToList + "/" + currentFileName);
										log.debug(currentFileName + " has changed and must be re-collected");
									} else {
										if (log.isDebugEnabled()) {
											log.debug(currentFileName + " has not changed since last collection");
										}
									}
								}
								else {
									//not incremental: just add it to the queue
									queue.add(dirToList + "/" + currentFileName);
								}
							}
						}
					}
				}
			}
		} catch (IOException e) {
			throw new CollectorServiceRuntimeException("Unable to list FTP remote folder", e);
		}
	}

	@Override
	public boolean hasNext() {
		if (queue.isEmpty()) {
			disconnectFromFtpServer();
			return false;
		} else {
			return true;
		}
	}

	@Override
	public String next() {
		String nextRemotePath = queue.remove();
		int nRepeat = 0;
		while (nRepeat < MAX_RETRIES) {
			try {
				OutputStream baos = new ByteArrayOutputStream();
				if (!ftpClient.isConnected()) {
					connectToFtpServer();
				}
				ftpClient.retrieveFile(nextRemotePath, baos);

				log.debug(String.format("Collected file from FTP: %s%s", ftpServerAddress, nextRemotePath));
				return baos.toString();
			} catch (IOException e) {
				nRepeat++;
				log.warn(String.format("An error occurred [%s] for %s%s, retrying.. [retried %s time(s)]", e.getMessage(), ftpServerAddress, nextRemotePath,
						nRepeat));
				disconnectFromFtpServer();
				try {
					Thread.sleep(BACKOFF_MILLIS);
				} catch (InterruptedException e1) {
					log.error(e1);
				}
			}
		}
		throw new CollectorServiceRuntimeException(String.format("Impossible to retrieve FTP file %s after %s retries. Aborting FTP collection.", nextRemotePath, nRepeat));
	}

	@Override
	public void remove() {
		throw new UnsupportedOperationException();
	}
}
