local storage

src/main/java/eu/dnetlib/apps/Oai2ftp/service/Oai2FtpService.java

@@ -27,10 +27,10 @@ import eu.dnetlib.apps.oai2ftp.model.CollectionCall;
 import eu.dnetlib.apps.oai2ftp.model.CollectionInfo;
 import eu.dnetlib.apps.oai2ftp.model.ExecutionStatus;
 import eu.dnetlib.apps.oai2ftp.repository.CollectionLogEntryRepository;
-import eu.dnetlib.apps.oai2ftp.utils.FtpClientFactory;
-import eu.dnetlib.apps.oai2ftp.utils.FtpClientWrapper;
 import eu.dnetlib.apps.oai2ftp.utils.HttpFetcher;
 import eu.dnetlib.apps.oai2ftp.utils.SimpleUtils;
+import eu.dnetlib.apps.oai2ftp.utils.StorageClient;
+import eu.dnetlib.apps.oai2ftp.utils.StorageClientFactory;
 
 @Service
 public class Oai2FtpService {
@@ -42,7 +42,7 @@ public class Oai2FtpService {
 	private final Map<String, CollectionInfo> infoMap = new LinkedHashMap<>();
 
 	@Autowired
-	private FtpClientFactory ftpClientFactory;
+	private StorageClientFactory storageClientFactory;
 
 	@Value("${oai2ftp.conf.execution.expirationTime}")
 	private long fullInfoExpirationTime; // in hours
@@ -57,7 +57,7 @@ public class Oai2FtpService {
 		final LocalDateTime until) {
 		final String jobId = SimpleUtils.generateNewJobId();
 
-		final FtpClientWrapper ftp = ftpClientFactory.newClientForJob(jobId);
+		final StorageClient sc = storageClientFactory.newClientForJob(jobId);
 
 		final CollectionInfo info = new CollectionInfo();
 		info.setId(jobId);
@@ -80,14 +80,14 @@ public class Oai2FtpService {
 		jobExecutor.execute(() -> {
 			try {
 				info.setExecutionStatus(ExecutionStatus.RUNNING);
-				oaiCollect(baseUrl, format, setSpec, from, until, ftp, info);
+				oaiCollect(baseUrl, format, setSpec, from, until, sc, info);
 				info.setExecutionStatus(ExecutionStatus.COMPLETED);
 			} catch (final Throwable e) {
 				info.setExecutionStatus(ExecutionStatus.FAILED);
 				info.setMessage(e.getMessage() + ": " + ExceptionUtils.getStackTrace(e));
 			} finally {
 				info.setEnd(LocalDateTime.now());
-				ftp.disconnect();
+				sc.disconnect();
 				collectionLogEntryRepository.save(SimpleUtils.infoToLog(info));
 			}
 		});
@@ -100,7 +100,7 @@ public class Oai2FtpService {
 		final String setSpec,
 		final LocalDateTime from,
 		final LocalDateTime until,
-		final FtpClientWrapper ftp,
+		final StorageClient sc,
 		final CollectionInfo info)
 		throws Exception {
 
@@ -118,8 +118,9 @@ public class Oai2FtpService {
 			call.setNumberOfRecords(records.size());
 
 			for (final Node n : records) {
-				final String id = n.valueOf("/*[local-name()='record']/*[local-name()='header']/*[local-name()='identifier']");
+				final String id = n.valueOf(".//*[local-name()='header']/*[local-name()='identifier']");
-				ftp.saveFile(SimpleUtils.oaiIdToFilename(id), n.asXML());
+
+				sc.saveFile(SimpleUtils.oaiIdToFilename(id), n.asXML());
 				info.setTotal(info.getTotal() + 1);
 			}
 

src/main/java/eu/dnetlib/apps/Oai2ftp/utils/FtpClientFactory.java

@@ -1,32 +0,0 @@
-package eu.dnetlib.apps.oai2ftp.utils;
-
-import org.springframework.beans.factory.annotation.Value;
-import org.springframework.stereotype.Component;
-
-@Component
-public class FtpClientFactory {
-
-	@Value("${oai2ftp.conf.ftp.server}")
-	private String ftpServer;
-
-	@Value("${oai2ftp.conf.ftp.user}")
-	private String ftpUser;
-
-	@Value("${oai2ftp.conf.ftp.password}")
-	private String ftpPassword;
-
-	@Value("${oai2ftp.conf.ftp.basedir}")
-	private String ftpBaseDir;
-
-	@Value("${oai2ftp.conf.ftp.secure}")
-	private boolean ftpSecure;
-
-	public FtpClientWrapper newClientForJob(final String jobId) {
-		final FtpClientWrapper ftp = new FtpClientWrapper(ftpServer, ftpSecure);
-		ftp.login(ftpUser, ftpPassword);
-		ftp.changeDir(ftpBaseDir);
-		ftp.changeDir(jobId);
-		return ftp;
-	}
-
-}

src/main/java/eu/dnetlib/apps/Oai2ftp/utils/FtpStorage.java

@@ -4,29 +4,26 @@ import java.io.ByteArrayInputStream;
 import java.io.IOException;
 import java.io.InputStream;
 
-import org.apache.commons.lang3.StringUtils;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.commons.net.ftp.FTPClient;
 import org.apache.commons.net.ftp.FTPSClient;
 
-public class FtpClientWrapper {
+public class FtpStorage implements StorageClient {
 
-	private static final Log log = LogFactory.getLog(FtpClientWrapper.class);
+	private static final Log log = LogFactory.getLog(FtpStorage.class);
 
 	private final FTPClient ftp;
 
-	public FtpClientWrapper(final String server, final boolean secure) {
+	public FtpStorage(final String server, final int port, final boolean secure) {
-		this.ftp = ftpConnect(server, secure);
+		this.ftp = ftpConnect(server, port, secure);
 	}
 
-	private FTPClient ftpConnect(final String server, final boolean secure) {
+	private FTPClient ftpConnect(final String server, final int port, final boolean secure) {
 		try {
-			if (StringUtils.isBlank(server)) {
+			if (secure) {
-				return new MockFtpClient();
-			} else if (secure) {
 				final FTPSClient ftp = new FTPSClient();
-				ftp.connect(server);
+				ftp.connect(server, port > 0 ? port : FTPSClient.DEFAULT_FTPS_PORT);
 				// Set protection buffer size
 				ftp.execPBSZ(0);
 				// Set data channel protection to private
@@ -34,7 +31,7 @@ public class FtpClientWrapper {
 				return ftp;
 			} else {
 				final FTPClient ftp = new FTPClient();
-				ftp.connect(server);
+				ftp.connect(server, port > 0 ? port : FTPClient.DEFAULT_PORT);
 				return ftp;
 			}
 		} catch (final IOException e) {
@@ -43,6 +40,7 @@ public class FtpClientWrapper {
 		}
 	}
 
+	@Override
 	public void disconnect() {
 		if (ftp != null && ftp.isConnected()) {
 			try {
@@ -55,6 +53,7 @@ public class FtpClientWrapper {
 		}
 	}
 
+	@Override
 	public void login(final String user, final String password) {
 		try {
 			if (!ftp.login(user, password)) { throw new RuntimeException("FTP login failed"); }
@@ -69,6 +68,7 @@ public class FtpClientWrapper {
 		}
 	}
 
+	@Override
 	public boolean changeDir(final String dir) {
 		try {
 			if (!ftp.changeWorkingDirectory(dir)) {
@@ -83,6 +83,7 @@ public class FtpClientWrapper {
 		}
 	}
 
+	@Override
 	public void saveFile(final String filename, final String content) {
 		try (InputStream is = new ByteArrayInputStream(content.getBytes())) {
 			if (log.isDebugEnabled()) {

src/main/java/eu/dnetlib/apps/Oai2ftp/utils/LocalStorage.java

@@ -0,0 +1,46 @@
+package eu.dnetlib.apps.oai2ftp.utils;
+
+import java.io.File;
+import java.io.FileWriter;
+import java.io.IOException;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.io.IOUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+public class LocalStorage implements StorageClient {
+
+	private static final Log log = LogFactory.getLog(LocalStorage.class);
+
+	@Override
+	public void login(final String user, final String password) {}
+
+	@Override
+	public void disconnect() {}
+
+	public String currDir = "/tmp";
+
+	@Override
+	public boolean changeDir(final String dir) {
+		try {
+			final File d = new File(dir.startsWith("/") ? dir : currDir + "/" + dir);
+			FileUtils.forceMkdir(d);
+			currDir = d.getAbsolutePath();
+			return true;
+		} catch (final IOException e) {
+			return false;
+		}
+	}
+
+	@Override
+	public void saveFile(final String filename, final String body) {
+		try {
+			IOUtils.write(body, new FileWriter(currDir + "/" + filename));
+		} catch (final IOException e) {
+			log.error("Error saving info file");
+			throw new RuntimeException("Error saving info file", e);
+		}
+	}
+
+}

src/main/java/eu/dnetlib/apps/Oai2ftp/utils/MockFtpClient.java

@@ -1,81 +0,0 @@
-package eu.dnetlib.apps.oai2ftp.utils;
-
-import java.io.IOException;
-import java.io.InputStream;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.commons.net.ftp.FTPClient;
-
-public class MockFtpClient extends FTPClient {
-
-	private static final Log log = LogFactory.getLog(MockFtpClient.class);
-
-	public MockFtpClient() {
-		log.debug("MOCK FTP CLIENT - Constructor");
-	}
-
-	@Override
-	public boolean login(final String username, final String password) throws IOException {
-		log.debug("MOCK FTP CLIENT - login: " + username + ", " + password);
-		return true;
-	}
-
-	@Override
-	public boolean isConnected() {
-		log.debug("MOCK FTP CLIENT - isConnected");
-		return true;
-	}
-
-	@Override
-	public void disconnect() throws IOException {
-		log.debug("MOCK FTP CLIENT - disconnect");
-	}
-
-	@Override
-	public boolean setFileType(final int fileType) throws IOException {
-		log.debug("MOCK FTP CLIENT - setFileType");
-		return true;
-	}
-
-	@Override
-	public void setBufferSize(final int bufSize) {
-		log.debug("MOCK FTP CLIENT - setBufferSize");
-	}
-
-	@Override
-	public void enterLocalPassiveMode() {
-		log.debug("MOCK FTP CLIENT - enterLocalPassiveMode");
-	}
-
-	@Override
-	public boolean changeWorkingDirectory(final String pathname) throws IOException {
-		log.debug("MOCK FTP CLIENT - changeWorkingDirectory: " + pathname);
-		return true;
-	}
-
-	@Override
-	public boolean makeDirectory(final String pathname) throws IOException {
-		log.debug("MOCK FTP CLIENT - makeDirectory: " + pathname);
-		return true;
-	}
-
-	@Override
-	public boolean storeFile(final String remote, final InputStream local) throws IOException {
-		log.debug("MOCK FTP CLIENT - storeFile: " + remote);
-		return true;
-	}
-
-	@Override
-	public int getReplyCode() {
-		log.debug("MOCK FTP CLIENT - getReplyCode");
-		return -1;
-	}
-
-	@Override
-	public String getReplyString() {
-		log.debug("MOCK FTP CLIENT - getReplyString");
-		return "MOCK";
-	}
-
-}

src/main/java/eu/dnetlib/apps/Oai2ftp/utils/StorageClient.java

@@ -0,0 +1,13 @@
+package eu.dnetlib.apps.oai2ftp.utils;
+
+public interface StorageClient {
+
+	void login(String user, String password);
+
+	void disconnect();
+
+	boolean changeDir(String dir);
+
+	void saveFile(String filename, String body);
+
+}

src/main/java/eu/dnetlib/apps/Oai2ftp/utils/StorageClientFactory.java

@@ -0,0 +1,44 @@
+package eu.dnetlib.apps.oai2ftp.utils;
+
+import java.net.URL;
+
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.stereotype.Component;
+
+@Component
+public class StorageClientFactory {
+
+	@Value("${oai2ftp.conf.storage.url}")
+	private URL storageUrl;
+
+	@Value("${oai2ftp.conf.storage.user}")
+	private String storageUser;
+
+	@Value("${oai2ftp.conf.storage.password}")
+	private String storagePassword;
+
+	public StorageClient newClientForJob(final String jobId) {
+
+		final String protocol = storageUrl.getProtocol();
+		final String host = storageUrl.getHost();
+		final int port = storageUrl.getPort();
+		final String path = storageUrl.getPath();
+
+		StorageClient client;
+		if (protocol.equalsIgnoreCase("ftp")) {
+			client = new FtpStorage(host, port, false);
+		} else if (protocol.equalsIgnoreCase("ftps")) {
+			client = new FtpStorage(host, port, true);
+		} else if (protocol.equalsIgnoreCase("file")) {
+			client = new LocalStorage();
+		} else {
+			throw new RuntimeException("Invalid storage protocol: " + protocol);
+		}
+		client.login(storageUser, storagePassword);
+		client.changeDir(path);
+		client.changeDir(jobId);
+
+		return client;
+	}
+
+}

src/main/resources/application.properties

@@ -12,10 +12,8 @@ spring.jpa.hibernate.ddl-auto=update
 spring.jpa.properties.hibernate.format_sql=true
 spring.jpa.properties.hibernate.dialect=org.hibernate.dialect.H2Dialect
 
-oai2ftp.conf.ftp.server   = 
+oai2ftp.conf.storage.url      = file:///tmp/test_oai 
-oai2ftp.conf.ftp.user     = test
+oai2ftp.conf.storage.user     = 
-oai2ftp.conf.ftp.password = testPwd
+oai2ftp.conf.storage.password = 
-oai2ftp.conf.ftp.basedir  = /oai
-oai2ftp.conf.ftp.secure   = true
 
 oai2ftp.conf.execution.expirationTime = 12