package eu.dnetlib.data.collector.plugins.schemaorg.sitemapindex;

import eu.dnetlib.data.collector.plugins.schemaorg.Utils;
import org.apache.commons.io.IOUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;

import java.net.URL;
import java.nio.charset.Charset;
import java.util.*;

public class SitemapIndexIterator implements Iterator<String> {
	private static final Log log = LogFactory.getLog(SitemapIndexIterator.class);

	public static class Options {
		private URL indexUrl;
		private Charset charset;

		public Options(){}

		public Options(URL indexUrl, Charset charset){
			this.indexUrl = indexUrl;
			this.charset = charset;
		}

		public URL getIndexUrl() {
			return indexUrl;
		}

		public void setIndexUrl(URL indexUrl) {
			this.indexUrl = indexUrl;
		}

		public Charset getCharset() {
			return charset;
		}

		public void setCharset(Charset charset) {
			this.charset = charset;
		}
	}

	private Options options;
	private Queue<String> sitemapFiles;

	public SitemapIndexIterator(Options options) {
		this.options = options;
	}

	public void bootstrap() {
		List<String> files = null;
		try {
			log.debug("bootstrapping sitemapindex index access");
			String sitemapIndexPayload = Utils.RemoteAccessWithRetry(3, 5000, this.options.getIndexUrl(), this.options.getCharset());
			log.debug(String.format("sitemapindex payload is: %s", sitemapIndexPayload));
			files = Utils.collectAsStrings(sitemapIndexPayload, "/sitemapindex/sitemap/loc/text()");
			log.debug(String.format("extracted %d sitemapindex files", files.size()));
		}catch(Exception ex){
			log.error("problem bootstrapping sitemapindex index access. returning 0 files", ex);
			files = new ArrayList<>();
		}
		this.sitemapFiles = new PriorityQueue<String>(files);
	}

	@Override
	public boolean hasNext() {
		return !this.sitemapFiles.isEmpty();
	}

	@Override
	public String next() {
		return this.sitemapFiles.poll();
	}
}
