new functionalities (zip, limits, emails, exposed urls)

.project

@@ -1,6 +1,6 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <projectDescription>
-	<name>oai2ftp</name>
+	<name>simpleOaiCollectorService</name>
 	<comment></comment>
 	<projects>
 	</projects>

src/main/java/eu/dnetlib/apps/Oai2ftp/utils/StorageClient.java

@@ -1,13 +0,0 @@
-package eu.dnetlib.apps.oai2ftp.utils;
-
-public interface StorageClient {
-
-	void login(String user, String password);
-
-	void disconnect();
-
-	boolean changeDir(String dir);
-
-	void saveFile(String filename, String body);
-
-}

src/main/java/eu/dnetlib/apps/oai/MainApplication.java

@@ -1,4 +1,4 @@
-package eu.dnetlib.apps.oai2ftp;
+package eu.dnetlib.apps.oai;
 
 import java.util.ArrayList;
 import java.util.List;
@@ -19,7 +19,7 @@ import io.swagger.v3.oas.models.tags.Tag;
 
 @SpringBootApplication
 @EnableScheduling
-public class Oai2ftpApplication {
+public class MainApplication {
 
 	@Value("${swagger.public_url}")
 	private String swaggerPublicUrl;
@@ -40,7 +40,7 @@ public class Oai2ftpApplication {
 		new License().name("GNU Affero General Public License v3.0 or later").url("https://www.gnu.org/licenses/agpl-3.0.txt");
 
 	public static void main(final String[] args) {
-		SpringApplication.run(Oai2ftpApplication.class, args);
+		SpringApplication.run(MainApplication.class, args);
 	}
 
 	@Bean

src/main/java/eu/dnetlib/apps/oai/controller/ApiController.java

@@ -1,14 +1,15 @@
-package eu.dnetlib.apps.oai2ftp.controller;
+package eu.dnetlib.apps.oai.controller;
 
 import java.time.LocalDateTime;
-import java.util.Arrays;
-import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
 
 import org.apache.commons.lang3.exception.ExceptionUtils;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.http.HttpStatus;
+import org.springframework.web.bind.annotation.DeleteMapping;
 import org.springframework.web.bind.annotation.ExceptionHandler;
 import org.springframework.web.bind.annotation.GetMapping;
 import org.springframework.web.bind.annotation.PathVariable;
@@ -18,36 +19,47 @@ import org.springframework.web.bind.annotation.ResponseBody;
 import org.springframework.web.bind.annotation.ResponseStatus;
 import org.springframework.web.bind.annotation.RestController;
 
-import eu.dnetlib.apps.oai2ftp.model.CollectionInfo;
-import eu.dnetlib.apps.oai2ftp.service.Oai2FtpService;
+import eu.dnetlib.apps.oai.model.CollectionInfo;
+import eu.dnetlib.apps.oai.model.ExecutionStatus;
+import eu.dnetlib.apps.oai.service.CollectorService;
 
 @RestController
 @RequestMapping("/api")
-public class Oai2FtpController {
+public class ApiController {
 
-	private static final Log log = LogFactory.getLog(Oai2FtpController.class);
+	private static final Log log = LogFactory.getLog(ApiController.class);
 
 	@Autowired
-	private Oai2FtpService service;
+	private CollectorService service;
 
 	@GetMapping("/collect")
 	public CollectionInfo startCollection(@RequestParam final String oaiBaseUrl,
 		@RequestParam(required = false, defaultValue = "oai_dc") final String oaiFormat,
 		@RequestParam(required = false) final String oaiSet,
 		@RequestParam(required = false) final LocalDateTime oaiFrom,
-		@RequestParam(required = false) final LocalDateTime oaiUntil) {
-		return service.startCollection(oaiBaseUrl, oaiFormat, oaiSet, oaiFrom, oaiUntil);
+		@RequestParam(required = false) final LocalDateTime oaiUntil,
+		@RequestParam(required = false) final Long max,
+		@RequestParam(required = false) final String notificationEmail) {
+		return service.startCollection(oaiBaseUrl, oaiFormat, oaiSet, oaiFrom, oaiUntil, max, notificationEmail);
 	}
 
-	@GetMapping("/status/{id}")
+	@GetMapping("/history/{id}")
 	public CollectionInfo getExecutionStatus(@PathVariable final String id) {
 		return service.getStatus(id);
 	}
 
-	@GetMapping("/history/clean")
-	public List<String> cleanHistory() {
+	@GetMapping("/history")
+	public Map<String, ExecutionStatus> history() {
+		return service.history()
+			.entrySet()
+			.stream()
+			.collect(Collectors.toMap(e -> e.getKey(), e -> e.getValue().getExecutionStatus()));
+	}
+
+	@DeleteMapping("/history")
+	public Map<String, ExecutionStatus> cleanHistory() {
 		service.cleanHistory();
-		return Arrays.asList("DONE.");
+		return history();
 	}
 
 	@ExceptionHandler(Exception.class)

src/main/java/eu/dnetlib/apps/oai/controller/SwaggerController.java

@@ -1,4 +1,4 @@
-package eu.dnetlib.apps.oai2ftp.controller;
+package eu.dnetlib.apps.oai.controller;
 
 import org.springframework.stereotype.Controller;
 import org.springframework.web.bind.annotation.GetMapping;

src/main/java/eu/dnetlib/apps/oai/model/CollectionCall.java

@@ -1,4 +1,4 @@
-package eu.dnetlib.apps.oai2ftp.model;
+package eu.dnetlib.apps.oai.model;
 
 import java.io.Serializable;
 import java.util.Objects;

src/main/java/eu/dnetlib/apps/oai/model/CollectionInfo.java

@@ -1,4 +1,4 @@
-package eu.dnetlib.apps.oai2ftp.model;
+package eu.dnetlib.apps.oai.model;
 
 import java.io.Serializable;
 import java.time.LocalDateTime;
@@ -40,18 +40,27 @@ public class CollectionInfo implements Serializable {
 	@Column(name = "storage_url", length = 1024)
 	private String storageUrl;
 
+	@Column(name = "public_url", length = 1024)
+	private String publicUrl;
+
 	@Column(name = "start_date")
 	private LocalDateTime start;
 
 	@Column(name = "end_date")
 	private LocalDateTime end;
 
+	@Column(name = "max")
+	private long max = Long.MAX_VALUE;
+
 	@Column(name = "status")
 	private ExecutionStatus executionStatus;
 
 	@Column(name = "total")
 	private long total = 0;
 
+	@Column(name = "notification_email")
+	private String notificationEmail;
+
 	@Lob
 	@Column(name = "message")
 	private String message = "";
@@ -115,6 +124,14 @@ public class CollectionInfo implements Serializable {
 		this.storageUrl = storageUrl;
 	}
 
+	public String getPublicUrl() {
+		return publicUrl;
+	}
+
+	public void setPublicUrl(final String publicUrl) {
+		this.publicUrl = publicUrl;
+	}
+
 	public LocalDateTime getStart() {
 		return start;
 	}
@@ -131,6 +148,14 @@ public class CollectionInfo implements Serializable {
 		this.end = end;
 	}
 
+	public long getMax() {
+		return max;
+	}
+
+	public void setMax(final long max) {
+		this.max = max;
+	}
+
 	public ExecutionStatus getExecutionStatus() {
 		return executionStatus;
 	}
@@ -147,6 +172,14 @@ public class CollectionInfo implements Serializable {
 		this.total = total;
 	}
 
+	public String getNotificationEmail() {
+		return notificationEmail;
+	}
+
+	public void setNotificationEmail(final String notificationEmail) {
+		this.notificationEmail = notificationEmail;
+	}
+
 	public String getMessage() {
 		return message;
 	}

src/main/java/eu/dnetlib/apps/oai/model/ExecutionStatus.java

@@ -1,4 +1,4 @@
-package eu.dnetlib.apps.oai2ftp.model;
+package eu.dnetlib.apps.oai.model;
 
 public enum ExecutionStatus {
 	READY,

src/main/java/eu/dnetlib/apps/oai/repository/CollectionInfoRepository.java

@@ -1,8 +1,8 @@
-package eu.dnetlib.apps.oai2ftp.repository;
+package eu.dnetlib.apps.oai.repository;
 
 import org.springframework.data.jpa.repository.JpaRepository;
 
-import eu.dnetlib.apps.oai2ftp.model.CollectionInfo;
+import eu.dnetlib.apps.oai.model.CollectionInfo;
 
 public interface CollectionInfoRepository extends JpaRepository<CollectionInfo, String> {
 

src/main/java/eu/dnetlib/apps/oai/service/CollectorService.java

@@ -1,4 +1,4 @@
-package eu.dnetlib.apps.oai2ftp.service;
+package eu.dnetlib.apps.oai.service;
 
 import java.time.Duration;
 import java.time.LocalDateTime;
@@ -23,19 +23,21 @@ import org.springframework.beans.factory.annotation.Value;
 import org.springframework.scheduling.annotation.Scheduled;
 import org.springframework.stereotype.Service;
 
-import eu.dnetlib.apps.oai2ftp.model.CollectionCall;
-import eu.dnetlib.apps.oai2ftp.model.CollectionInfo;
-import eu.dnetlib.apps.oai2ftp.model.ExecutionStatus;
-import eu.dnetlib.apps.oai2ftp.repository.CollectionInfoRepository;
-import eu.dnetlib.apps.oai2ftp.utils.HttpFetcher;
-import eu.dnetlib.apps.oai2ftp.utils.SimpleUtils;
-import eu.dnetlib.apps.oai2ftp.utils.StorageClient;
-import eu.dnetlib.apps.oai2ftp.utils.StorageClientFactory;
+import eu.dnetlib.apps.oai.model.CollectionCall;
+import eu.dnetlib.apps.oai.model.CollectionInfo;
+import eu.dnetlib.apps.oai.model.ExecutionStatus;
+import eu.dnetlib.apps.oai.repository.CollectionInfoRepository;
+import eu.dnetlib.apps.oai.storage.StorageClient;
+import eu.dnetlib.apps.oai.storage.StorageClientFactory;
+import eu.dnetlib.apps.oai.storage.ZipStorage;
+import eu.dnetlib.apps.oai.utils.EmailSender;
+import eu.dnetlib.apps.oai.utils.HttpFetcher;
+import eu.dnetlib.apps.oai.utils.SimpleUtils;
 
 @Service
-public class Oai2FtpService {
+public class CollectorService {
 
-	private static final Log log = LogFactory.getLog(Oai2FtpService.class);
+	private static final Log log = LogFactory.getLog(CollectorService.class);
 
 	private final ExecutorService jobExecutor = Executors.newFixedThreadPool(100);
 
@@ -44,22 +46,35 @@ public class Oai2FtpService {
 	@Autowired
 	private StorageClientFactory storageClientFactory;
 
-	@Value("${oai2ftp.conf.execution.expirationTime}")
+	@Value("${oai.conf.execution.expirationTime}")
 	private long fullInfoExpirationTime; // in hours
 
+	@Value("${oai.conf.maxRecords}")
+	private long defaultMaxRecords;
+
 	@Autowired
 	private CollectionInfoRepository collectionInfoRepository;
 
+	@Value("${oai.conf.storage.basePath}")
+	private String storageBasePath;
+
+	@Value("${oai.conf.public.basePath}")
+	private String publicBasePath;
+
+	@Autowired
+	private EmailSender emailSender;
+
 	public CollectionInfo startCollection(final String baseUrl,
 		final String format,
 		final String setSpec,
 		final LocalDateTime from,
-		final LocalDateTime until) {
+		final LocalDateTime until,
+		final Long max,
+		final String notificationEmail) {
 
 		final String jobId = SimpleUtils.generateNewJobId();
 
-		final StorageClient sc = storageClientFactory.newClient();
-		sc.changeDir(jobId);
+		final StorageClient sc = storageClientFactory.newClient(jobId);
 
 		final CollectionInfo info = new CollectionInfo();
 		info.setId(jobId);
@@ -68,23 +83,49 @@ public class Oai2FtpService {
 		info.setOaiSet(setSpec);
 		info.setOaiFrom(from);
 		info.setOaiUntil(until);
-		info.setStorageUrl(storageClientFactory.getStorageUrlAsString());
+		if (sc instanceof ZipStorage) {
+			info.setStorageUrl(storageBasePath + "/" + jobId + ".zip");
+		} else {
+			info.setStorageUrl(storageBasePath + "/" + jobId);
+		}
+		info.setPublicUrl(null);
 		info.setStart(LocalDateTime.now());
 		info.setExecutionStatus(ExecutionStatus.READY);
 
+		if (StringUtils.isNotBlank(notificationEmail)) {
+			info.setNotificationEmail(notificationEmail);
+		}
+		if (max != null && max > 0) {
+			info.setMax(max);
+		} else {
+			info.setMax(defaultMaxRecords);
+		}
+
 		infoMap.put(jobId, info);
 
 		jobExecutor.execute(() -> {
 			try {
 				info.setExecutionStatus(ExecutionStatus.RUNNING);
-				oaiCollect(baseUrl, format, setSpec, from, until, sc, info);
+				oaiCollect(sc, info);
 				info.setExecutionStatus(ExecutionStatus.COMPLETED);
+				info.setEnd(LocalDateTime.now());
+				emailSender.notifySuccess(info);
 			} catch (final Throwable e) {
 				info.setExecutionStatus(ExecutionStatus.FAILED);
 				info.setMessage(e.getMessage() + ": " + ExceptionUtils.getStackTrace(e));
-			} finally {
 				info.setEnd(LocalDateTime.now());
-				sc.disconnect();
+				emailSender.notifyFailure(info);
+			} finally {
+				sc.complete();
+
+				if (StringUtils.isNotBlank(publicBasePath)) {
+					if (sc instanceof ZipStorage) {
+						info.setPublicUrl(publicBasePath + "/" + jobId + ".zip");
+					} else {
+						info.setPublicUrl(publicBasePath + "/" + jobId);
+					}
+				}
+
 				collectionInfoRepository.save(info);
 			}
 		});
@@ -92,18 +133,13 @@ public class Oai2FtpService {
 		return info;
 	}
 
-	public void oaiCollect(final String baseUrl,
-		final String format,
-		final String setSpec,
-		final LocalDateTime from,
-		final LocalDateTime until,
-		final StorageClient sc,
-		final CollectionInfo info)
-		throws Exception {
+	public void oaiCollect(final StorageClient sc, final CollectionInfo info) throws Exception {
 
-		String url = SimpleUtils.oaiFirstUrl(baseUrl, format, setSpec, from, until);
+		final String baseUrl = info.getOaiBaseUrl();
 
-		int count = 0;
+		String url = SimpleUtils.oaiFirstUrl(baseUrl, info.getOaiFormat(), info.getOaiSet(), info.getOaiFrom(), info.getOaiUntil());
+
+		long count = 0;
 		while (StringUtils.isNotBlank(url)) {
 			final CollectionCall call = new CollectionCall();
 			call.setUrl(url);
@@ -115,17 +151,21 @@ public class Oai2FtpService {
 			final List<Node> records = doc.selectNodes("//*[local-name()='ListRecords']/*[local-name()='record']");
 			call.setNumberOfRecords(records.size());
 
-			sc.changeDir(Integer.toString(count++));
+			sc.prepareCurrentPage(count++);
 			for (final Node n : records) {
-				final String id = n.valueOf(".//*[local-name()='header']/*[local-name()='identifier']");
-				sc.saveFile(SimpleUtils.oaiIdToFilename(id), n.asXML());
-				info.setTotal(info.getTotal() + 1);
+				if (info.getTotal() < info.getMax()) {
+					final String id = n.valueOf(".//*[local-name()='header']/*[local-name()='identifier']");
+					sc.saveFile(SimpleUtils.oaiIdToFilename(id), n.asXML());
+					info.setTotal(info.getTotal() + 1);
+				}
 			}
-			sc.changeDir("..");
 
-			final String rtoken = doc.valueOf("//*[local-name()='resumptionToken']").trim();
-
-			url = SimpleUtils.oaiNextUrl(baseUrl, rtoken);
+			if (info.getTotal() < info.getMax()) {
+				final String rtoken = doc.valueOf("//*[local-name()='resumptionToken']").trim();
+				url = SimpleUtils.oaiNextUrl(baseUrl, rtoken);
+			} else {
+				url = "";
+			}
 		}
 	}
 
@@ -168,4 +208,8 @@ public class Oai2FtpService {
 		toDelete.forEach(infoMap::remove);
 	}
 
+	public Map<String, CollectionInfo> history() {
+		return infoMap;
+	}
+
 }

src/main/java/eu/dnetlib/apps/oai/storage/FtpStorage.java

@@ -1,4 +1,4 @@
-package eu.dnetlib.apps.oai2ftp.utils;
+package eu.dnetlib.apps.oai.storage;
 
 import java.io.ByteArrayInputStream;
 import java.io.IOException;
@@ -15,6 +15,8 @@ public class FtpStorage implements StorageClient {
 
 	private final FTPClient ftp;
 
+	private long currentPage = -1;
+
 	public FtpStorage(final String server, final int port, final boolean secure) {
 		this.ftp = ftpConnect(server, port, secure);
 	}
@@ -41,7 +43,7 @@ public class FtpStorage implements StorageClient {
 	}
 
 	@Override
-	public void disconnect() {
+	public void complete() {
 		if (ftp != null && ftp.isConnected()) {
 			try {
 				ftp.disconnect();
@@ -63,13 +65,27 @@ public class FtpStorage implements StorageClient {
 			log.info("Ftp logged");
 		} catch (final IOException e) {
 			log.error("Ftp Login Failed", e);
-			disconnect();
+			complete();
 			throw new RuntimeException(e);
 		}
 	}
 
 	@Override
-	public boolean changeDir(final String dir) {
+	public void init(final String baseDir, final String jobId) {
+		changeDir(baseDir);
+		changeDir(jobId);
+	}
+
+	@Override
+	public void prepareCurrentPage(final long page) {
+		if (page != this.currentPage) {
+			this.currentPage = page;
+			changeDir("..");
+			changeDir(Long.toString(page));
+		}
+	}
+
+	private boolean changeDir(final String dir) {
 		try {
 			if (!ftp.changeWorkingDirectory(dir)) {
 				ftp.makeDirectory(dir);
@@ -78,7 +94,7 @@ public class FtpStorage implements StorageClient {
 			return true;
 		} catch (final IOException e) {
 			log.error("Error changing or create dir: " + dir);
-			disconnect();
+			complete();
 			throw new RuntimeException("Error changing or create dir: " + dir, e);
 		}
 	}

src/main/java/eu/dnetlib/apps/oai/storage/LocalStorage.java

@@ -1,4 +1,4 @@
-package eu.dnetlib.apps.oai2ftp.utils;
+package eu.dnetlib.apps.oai.storage;
 
 import java.io.File;
 import java.io.FileWriter;
@@ -13,29 +13,41 @@ public class LocalStorage implements StorageClient {
 
 	private static final Log log = LogFactory.getLog(LocalStorage.class);
 
+	private long currentPage = -1;
+
+	private String rootDir;
+
 	@Override
 	public void login(final String user, final String password) {}
 
 	@Override
-	public void disconnect() {}
-
-	public String currDir = "/tmp";
+	public void complete() {}
 
 	@Override
-	public boolean changeDir(final String dir) {
+	public void init(final String baseDir, final String jobId) {
 		try {
-			final File d = new File(dir.startsWith("/") ? dir : currDir + "/" + dir);
+			final File d = new File(baseDir + "/" + jobId);
 			FileUtils.forceMkdir(d);
-			currDir = d.getAbsolutePath();
-			return true;
+			this.rootDir = d.getAbsolutePath();
 		} catch (final IOException e) {
-			return false;
+			throw new RuntimeException("Errore creating root dir", e);
+		}
+	}
+
+	@Override
+	public void prepareCurrentPage(final long page) {
+		try {
+			final File d = new File(rootDir + "/" + page);
+			FileUtils.forceMkdir(d);
+			this.currentPage = page;
+		} catch (final IOException e) {
+			throw new RuntimeException("Errore creating page", e);
 		}
 	}
 
 	@Override
 	public void saveFile(final String filename, final String body) {
-		try (FileWriter fw = new FileWriter(currDir + "/" + filename)) {
+		try (FileWriter fw = new FileWriter(this.rootDir + "/" + this.currentPage + "/" + filename)) {
 			IOUtils.write(body, fw);
 		} catch (final IOException e) {
 			log.error("Error saving info file");

src/main/java/eu/dnetlib/apps/oai/storage/StorageClient.java

@@ -0,0 +1,15 @@
+package eu.dnetlib.apps.oai.storage;
+
+public interface StorageClient {
+
+	void login(final String user, final String password);
+
+	void complete();
+
+	void init(String baseDir, String jobId);
+
+	void prepareCurrentPage(long page);
+
+	void saveFile(String filename, String body);
+
+}

src/main/java/eu/dnetlib/apps/oai/storage/StorageClientFactory.java

@@ -1,4 +1,4 @@
-package eu.dnetlib.apps.oai2ftp.utils;
+package eu.dnetlib.apps.oai.storage;
 
 import java.net.URI;
 
@@ -8,21 +8,21 @@ import org.springframework.stereotype.Component;
 @Component
 public class StorageClientFactory {
 
-	@Value("${oai2ftp.conf.storage.url}")
-	private URI storageUrl;
+	@Value("${oai.conf.storage.basePath}")
+	private URI storageBasePath;
 
-	@Value("${oai2ftp.conf.storage.user}")
+	@Value("${oai.conf.storage.user}")
 	private String storageUser;
 
-	@Value("${oai2ftp.conf.storage.password}")
+	@Value("${oai.conf.storage.password}")
 	private String storagePassword;
 
-	public StorageClient newClient() {
+	public StorageClient newClient(final String jobId) {
 
-		final String protocol = storageUrl.getScheme();
-		final String host = storageUrl.getHost();
-		final int port = storageUrl.getPort();
-		final String path = storageUrl.getPath();
+		final String protocol = storageBasePath.getScheme();
+		final String host = storageBasePath.getHost();
+		final int port = storageBasePath.getPort();
+		final String path = storageBasePath.getPath();
 
 		StorageClient client;
 		if (protocol.equalsIgnoreCase("ftp")) {
@@ -31,17 +31,15 @@ public class StorageClientFactory {
 			client = new FtpStorage(host, port, true);
 		} else if (protocol.equalsIgnoreCase("file")) {
 			client = new LocalStorage();
+		} else if (protocol.equalsIgnoreCase("zip")) {
+			client = new ZipStorage();
 		} else {
 			throw new RuntimeException("Invalid storage protocol: " + protocol + " (valid protocol are: file, ftp and ftps)");
 		}
 		client.login(storageUser, storagePassword);
-		client.changeDir(path);
+		client.init(path, jobId);
 
 		return client;
 	}
 
-	public String getStorageUrlAsString() {
-		return storageUrl.toString();
-	}
-
 }

src/main/java/eu/dnetlib/apps/oai/storage/ZipStorage.java

@@ -0,0 +1,74 @@
+package eu.dnetlib.apps.oai.storage;
+
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipOutputStream;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+public class ZipStorage implements StorageClient {
+
+	private static final Log log = LogFactory.getLog(ZipStorage.class);
+
+	private FileOutputStream fos;
+	private ZipOutputStream zipOut;
+	private long currPage = -1;
+
+	@Override
+	public void login(final String user, final String password) {}
+
+	@Override
+	public void complete() {
+		try {
+			if (zipOut != null) {
+				zipOut.close();
+			}
+			if (fos != null) {
+				fos.close();
+			}
+		} catch (final IOException e) {
+			log.error("Ftp Disconnection Failed");
+			throw new RuntimeException(e);
+		}
+	}
+
+	@Override
+	public void init(final String baseDir, final String jobId) {
+		try {
+			final File rootDir = new File(baseDir);
+			FileUtils.forceMkdir(rootDir);
+
+			this.fos = new FileOutputStream(rootDir.getAbsolutePath() + "/" + jobId + ".zip");
+			this.zipOut = new ZipOutputStream(fos);
+		} catch (final IOException e) {
+			throw new RuntimeException("Error preparing zip", e);
+		}
+	}
+
+	@Override
+	public void prepareCurrentPage(final long page) {
+		try {
+			zipOut.putNextEntry(new ZipEntry(page + "/"));
+			zipOut.closeEntry();
+			this.currPage = page;
+		} catch (final IOException e) {
+			throw new RuntimeException("Error adding a directory to zip", e);
+		}
+
+	}
+
+	@Override
+	public void saveFile(final String filename, final String body) {
+		try {
+			zipOut.putNextEntry(new ZipEntry(new ZipEntry(this.currPage + "/" + filename)));
+			zipOut.write(body.getBytes());
+		} catch (final IOException e) {
+			throw new RuntimeException("Error adding a file to zip", e);
+		}
+	}
+
+}

src/main/java/eu/dnetlib/apps/oai/utils/EmailSender.java

@@ -0,0 +1,37 @@
+package eu.dnetlib.apps.oai.utils;
+
+import org.apache.commons.lang3.StringUtils;
+import org.springframework.stereotype.Service;
+
+import eu.dnetlib.apps.oai.model.CollectionInfo;
+
+@Service
+public class EmailSender {
+
+	public void notifySuccess(final CollectionInfo info) {
+		if (StringUtils.isNotBlank(info.getNotificationEmail())) {
+			sendMail(info.getNotificationEmail(), "OAI Harvesting completed", prepareSuccessMessage(info));
+		}
+	}
+
+	public void notifyFailure(final CollectionInfo info) {
+		if (StringUtils.isNotBlank(info.getNotificationEmail())) {
+			sendMail(info.getNotificationEmail(), "OAI Harvesting completed", prepareFailureMessage(info));
+		}
+	}
+
+	private String prepareSuccessMessage(final CollectionInfo info) {
+		// TODO Auto-generated method stub
+		return null;
+	}
+
+	private String prepareFailureMessage(final CollectionInfo info) {
+		// TODO Auto-generated method stub
+		return null;
+	}
+
+	private void sendMail(final String to, final String subject, final String message) {
+		// TODO Auto-generated method stub
+
+	}
+}

src/main/java/eu/dnetlib/apps/oai/utils/HttpFetcher.java

@@ -1,4 +1,4 @@
-package eu.dnetlib.apps.oai2ftp.utils;
+package eu.dnetlib.apps.oai.utils;
 
 import java.io.IOException;
 
@@ -7,8 +7,8 @@ import org.apache.hc.client5.http.impl.classic.CloseableHttpClient;
 import org.apache.hc.client5.http.impl.classic.HttpClientBuilder;
 import org.apache.hc.core5.http.io.entity.EntityUtils;
 
-import eu.dnetlib.apps.oai2ftp.model.CollectionCall;
-import eu.dnetlib.apps.oai2ftp.model.ExecutionStatus;
+import eu.dnetlib.apps.oai.model.CollectionCall;
+import eu.dnetlib.apps.oai.model.ExecutionStatus;
 
 public class HttpFetcher {
 

src/main/java/eu/dnetlib/apps/oai/utils/SimpleUtils.java

@@ -1,4 +1,4 @@
-package eu.dnetlib.apps.oai2ftp.utils;
+package eu.dnetlib.apps.oai.utils;
 
 import java.io.UnsupportedEncodingException;
 import java.net.URLEncoder;

src/main/resources/application.properties

@@ -7,15 +7,20 @@ swagger.api_version = 0.0.1
 spring.datasource.url=jdbc:h2:mem:
 spring.datasource.username=
 spring.datasource.password=
-spring.jpa.show-sql=true
+spring.jpa.show-sql=false
 spring.jpa.hibernate.ddl-auto=update
 spring.jpa.properties.hibernate.format_sql=true
 spring.jpa.properties.hibernate.dialect=org.hibernate.dialect.H2Dialect
 
-# supported protocols: file, ftp and ftps
-# oai2ftp.conf.storage.url      = file:///tmp/test_oai 
-oai2ftp.conf.storage.url      = ftp://localhost/oai_dumps
-oai2ftp.conf.storage.user     = test
-oai2ftp.conf.storage.password = testPwd
+# supported protocols: file, zip, ftp and ftps
+#oai.conf.storage.basePath = file:///tmp/test_oai
+oai.conf.storage.basePath = zip:///tmp/test_oai
+#oai.conf.storage.basePath = ftp://localhost/oai_dumps
+oai.conf.storage.user     = test
+oai.conf.storage.password = testPwd
+
+oai.conf.public.basePath  = http://localhost/oai_dumps
+
+oai.conf.execution.expirationTime = 12
+oai.conf.maxRecords = 1000
 
-oai2ftp.conf.execution.expirationTime = 12

src/test/java/eu/dnetlib/apps/oai/OaiApplicationTests.java

@@ -1,10 +1,10 @@
-package eu.dnetlib.apps.oai2ftp;
+package eu.dnetlib.apps.oai;
 
 import org.junit.jupiter.api.Test;
 import org.springframework.boot.test.context.SpringBootTest;
 
 @SpringBootTest
-class Oai2ftpApplicationTests {
+class OaiApplicationTests {
 
 	@Test
 	void contextLoads() {