diff --git a/src/main/java/eu/dnetlib/apps/Oai2ftp/service/Oai2FtpService.java b/src/main/java/eu/dnetlib/apps/Oai2ftp/service/Oai2FtpService.java index 9f537d3..d8aee01 100644 --- a/src/main/java/eu/dnetlib/apps/Oai2ftp/service/Oai2FtpService.java +++ b/src/main/java/eu/dnetlib/apps/Oai2ftp/service/Oai2FtpService.java @@ -27,10 +27,10 @@ import eu.dnetlib.apps.oai2ftp.model.CollectionCall; import eu.dnetlib.apps.oai2ftp.model.CollectionInfo; import eu.dnetlib.apps.oai2ftp.model.ExecutionStatus; import eu.dnetlib.apps.oai2ftp.repository.CollectionLogEntryRepository; -import eu.dnetlib.apps.oai2ftp.utils.FtpClientFactory; -import eu.dnetlib.apps.oai2ftp.utils.FtpClientWrapper; import eu.dnetlib.apps.oai2ftp.utils.HttpFetcher; import eu.dnetlib.apps.oai2ftp.utils.SimpleUtils; +import eu.dnetlib.apps.oai2ftp.utils.StorageClient; +import eu.dnetlib.apps.oai2ftp.utils.StorageClientFactory; @Service public class Oai2FtpService { @@ -42,7 +42,7 @@ public class Oai2FtpService { private final Map infoMap = new LinkedHashMap<>(); @Autowired - private FtpClientFactory ftpClientFactory; + private StorageClientFactory storageClientFactory; @Value("${oai2ftp.conf.execution.expirationTime}") private long fullInfoExpirationTime; // in hours @@ -57,7 +57,7 @@ public class Oai2FtpService { final LocalDateTime until) { final String jobId = SimpleUtils.generateNewJobId(); - final FtpClientWrapper ftp = ftpClientFactory.newClientForJob(jobId); + final StorageClient sc = storageClientFactory.newClientForJob(jobId); final CollectionInfo info = new CollectionInfo(); info.setId(jobId); @@ -80,14 +80,14 @@ public class Oai2FtpService { jobExecutor.execute(() -> { try { info.setExecutionStatus(ExecutionStatus.RUNNING); - oaiCollect(baseUrl, format, setSpec, from, until, ftp, info); + oaiCollect(baseUrl, format, setSpec, from, until, sc, info); info.setExecutionStatus(ExecutionStatus.COMPLETED); } catch (final Throwable e) { info.setExecutionStatus(ExecutionStatus.FAILED); info.setMessage(e.getMessage() + ": " + ExceptionUtils.getStackTrace(e)); } finally { info.setEnd(LocalDateTime.now()); - ftp.disconnect(); + sc.disconnect(); collectionLogEntryRepository.save(SimpleUtils.infoToLog(info)); } }); @@ -100,7 +100,7 @@ public class Oai2FtpService { final String setSpec, final LocalDateTime from, final LocalDateTime until, - final FtpClientWrapper ftp, + final StorageClient sc, final CollectionInfo info) throws Exception { @@ -118,8 +118,9 @@ public class Oai2FtpService { call.setNumberOfRecords(records.size()); for (final Node n : records) { - final String id = n.valueOf("/*[local-name()='record']/*[local-name()='header']/*[local-name()='identifier']"); - ftp.saveFile(SimpleUtils.oaiIdToFilename(id), n.asXML()); + final String id = n.valueOf(".//*[local-name()='header']/*[local-name()='identifier']"); + + sc.saveFile(SimpleUtils.oaiIdToFilename(id), n.asXML()); info.setTotal(info.getTotal() + 1); } diff --git a/src/main/java/eu/dnetlib/apps/Oai2ftp/utils/FtpClientFactory.java b/src/main/java/eu/dnetlib/apps/Oai2ftp/utils/FtpClientFactory.java deleted file mode 100644 index 02b8440..0000000 --- a/src/main/java/eu/dnetlib/apps/Oai2ftp/utils/FtpClientFactory.java +++ /dev/null @@ -1,32 +0,0 @@ -package eu.dnetlib.apps.oai2ftp.utils; - -import org.springframework.beans.factory.annotation.Value; -import org.springframework.stereotype.Component; - -@Component -public class FtpClientFactory { - - @Value("${oai2ftp.conf.ftp.server}") - private String ftpServer; - - @Value("${oai2ftp.conf.ftp.user}") - private String ftpUser; - - @Value("${oai2ftp.conf.ftp.password}") - private String ftpPassword; - - @Value("${oai2ftp.conf.ftp.basedir}") - private String ftpBaseDir; - - @Value("${oai2ftp.conf.ftp.secure}") - private boolean ftpSecure; - - public FtpClientWrapper newClientForJob(final String jobId) { - final FtpClientWrapper ftp = new FtpClientWrapper(ftpServer, ftpSecure); - ftp.login(ftpUser, ftpPassword); - ftp.changeDir(ftpBaseDir); - ftp.changeDir(jobId); - return ftp; - } - -} diff --git a/src/main/java/eu/dnetlib/apps/Oai2ftp/utils/FtpClientWrapper.java b/src/main/java/eu/dnetlib/apps/Oai2ftp/utils/FtpStorage.java similarity index 81% rename from src/main/java/eu/dnetlib/apps/Oai2ftp/utils/FtpClientWrapper.java rename to src/main/java/eu/dnetlib/apps/Oai2ftp/utils/FtpStorage.java index 8ee07fd..c71452b 100644 --- a/src/main/java/eu/dnetlib/apps/Oai2ftp/utils/FtpClientWrapper.java +++ b/src/main/java/eu/dnetlib/apps/Oai2ftp/utils/FtpStorage.java @@ -4,29 +4,26 @@ import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.InputStream; -import org.apache.commons.lang3.StringUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.commons.net.ftp.FTPClient; import org.apache.commons.net.ftp.FTPSClient; -public class FtpClientWrapper { +public class FtpStorage implements StorageClient { - private static final Log log = LogFactory.getLog(FtpClientWrapper.class); + private static final Log log = LogFactory.getLog(FtpStorage.class); private final FTPClient ftp; - public FtpClientWrapper(final String server, final boolean secure) { - this.ftp = ftpConnect(server, secure); + public FtpStorage(final String server, final int port, final boolean secure) { + this.ftp = ftpConnect(server, port, secure); } - private FTPClient ftpConnect(final String server, final boolean secure) { + private FTPClient ftpConnect(final String server, final int port, final boolean secure) { try { - if (StringUtils.isBlank(server)) { - return new MockFtpClient(); - } else if (secure) { + if (secure) { final FTPSClient ftp = new FTPSClient(); - ftp.connect(server); + ftp.connect(server, port > 0 ? port : FTPSClient.DEFAULT_FTPS_PORT); // Set protection buffer size ftp.execPBSZ(0); // Set data channel protection to private @@ -34,7 +31,7 @@ public class FtpClientWrapper { return ftp; } else { final FTPClient ftp = new FTPClient(); - ftp.connect(server); + ftp.connect(server, port > 0 ? port : FTPClient.DEFAULT_PORT); return ftp; } } catch (final IOException e) { @@ -43,6 +40,7 @@ public class FtpClientWrapper { } } + @Override public void disconnect() { if (ftp != null && ftp.isConnected()) { try { @@ -55,6 +53,7 @@ public class FtpClientWrapper { } } + @Override public void login(final String user, final String password) { try { if (!ftp.login(user, password)) { throw new RuntimeException("FTP login failed"); } @@ -69,6 +68,7 @@ public class FtpClientWrapper { } } + @Override public boolean changeDir(final String dir) { try { if (!ftp.changeWorkingDirectory(dir)) { @@ -83,6 +83,7 @@ public class FtpClientWrapper { } } + @Override public void saveFile(final String filename, final String content) { try (InputStream is = new ByteArrayInputStream(content.getBytes())) { if (log.isDebugEnabled()) { diff --git a/src/main/java/eu/dnetlib/apps/Oai2ftp/utils/LocalStorage.java b/src/main/java/eu/dnetlib/apps/Oai2ftp/utils/LocalStorage.java new file mode 100644 index 0000000..99fe946 --- /dev/null +++ b/src/main/java/eu/dnetlib/apps/Oai2ftp/utils/LocalStorage.java @@ -0,0 +1,46 @@ +package eu.dnetlib.apps.oai2ftp.utils; + +import java.io.File; +import java.io.FileWriter; +import java.io.IOException; + +import org.apache.commons.io.FileUtils; +import org.apache.commons.io.IOUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; + +public class LocalStorage implements StorageClient { + + private static final Log log = LogFactory.getLog(LocalStorage.class); + + @Override + public void login(final String user, final String password) {} + + @Override + public void disconnect() {} + + public String currDir = "/tmp"; + + @Override + public boolean changeDir(final String dir) { + try { + final File d = new File(dir.startsWith("/") ? dir : currDir + "/" + dir); + FileUtils.forceMkdir(d); + currDir = d.getAbsolutePath(); + return true; + } catch (final IOException e) { + return false; + } + } + + @Override + public void saveFile(final String filename, final String body) { + try { + IOUtils.write(body, new FileWriter(currDir + "/" + filename)); + } catch (final IOException e) { + log.error("Error saving info file"); + throw new RuntimeException("Error saving info file", e); + } + } + +} diff --git a/src/main/java/eu/dnetlib/apps/Oai2ftp/utils/MockFtpClient.java b/src/main/java/eu/dnetlib/apps/Oai2ftp/utils/MockFtpClient.java deleted file mode 100644 index 00d6464..0000000 --- a/src/main/java/eu/dnetlib/apps/Oai2ftp/utils/MockFtpClient.java +++ /dev/null @@ -1,81 +0,0 @@ -package eu.dnetlib.apps.oai2ftp.utils; - -import java.io.IOException; -import java.io.InputStream; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.commons.net.ftp.FTPClient; - -public class MockFtpClient extends FTPClient { - - private static final Log log = LogFactory.getLog(MockFtpClient.class); - - public MockFtpClient() { - log.debug("MOCK FTP CLIENT - Constructor"); - } - - @Override - public boolean login(final String username, final String password) throws IOException { - log.debug("MOCK FTP CLIENT - login: " + username + ", " + password); - return true; - } - - @Override - public boolean isConnected() { - log.debug("MOCK FTP CLIENT - isConnected"); - return true; - } - - @Override - public void disconnect() throws IOException { - log.debug("MOCK FTP CLIENT - disconnect"); - } - - @Override - public boolean setFileType(final int fileType) throws IOException { - log.debug("MOCK FTP CLIENT - setFileType"); - return true; - } - - @Override - public void setBufferSize(final int bufSize) { - log.debug("MOCK FTP CLIENT - setBufferSize"); - } - - @Override - public void enterLocalPassiveMode() { - log.debug("MOCK FTP CLIENT - enterLocalPassiveMode"); - } - - @Override - public boolean changeWorkingDirectory(final String pathname) throws IOException { - log.debug("MOCK FTP CLIENT - changeWorkingDirectory: " + pathname); - return true; - } - - @Override - public boolean makeDirectory(final String pathname) throws IOException { - log.debug("MOCK FTP CLIENT - makeDirectory: " + pathname); - return true; - } - - @Override - public boolean storeFile(final String remote, final InputStream local) throws IOException { - log.debug("MOCK FTP CLIENT - storeFile: " + remote); - return true; - } - - @Override - public int getReplyCode() { - log.debug("MOCK FTP CLIENT - getReplyCode"); - return -1; - } - - @Override - public String getReplyString() { - log.debug("MOCK FTP CLIENT - getReplyString"); - return "MOCK"; - } - -} diff --git a/src/main/java/eu/dnetlib/apps/Oai2ftp/utils/StorageClient.java b/src/main/java/eu/dnetlib/apps/Oai2ftp/utils/StorageClient.java new file mode 100644 index 0000000..0505eef --- /dev/null +++ b/src/main/java/eu/dnetlib/apps/Oai2ftp/utils/StorageClient.java @@ -0,0 +1,13 @@ +package eu.dnetlib.apps.oai2ftp.utils; + +public interface StorageClient { + + void login(String user, String password); + + void disconnect(); + + boolean changeDir(String dir); + + void saveFile(String filename, String body); + +} diff --git a/src/main/java/eu/dnetlib/apps/Oai2ftp/utils/StorageClientFactory.java b/src/main/java/eu/dnetlib/apps/Oai2ftp/utils/StorageClientFactory.java new file mode 100644 index 0000000..5637e63 --- /dev/null +++ b/src/main/java/eu/dnetlib/apps/Oai2ftp/utils/StorageClientFactory.java @@ -0,0 +1,44 @@ +package eu.dnetlib.apps.oai2ftp.utils; + +import java.net.URL; + +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; + +@Component +public class StorageClientFactory { + + @Value("${oai2ftp.conf.storage.url}") + private URL storageUrl; + + @Value("${oai2ftp.conf.storage.user}") + private String storageUser; + + @Value("${oai2ftp.conf.storage.password}") + private String storagePassword; + + public StorageClient newClientForJob(final String jobId) { + + final String protocol = storageUrl.getProtocol(); + final String host = storageUrl.getHost(); + final int port = storageUrl.getPort(); + final String path = storageUrl.getPath(); + + StorageClient client; + if (protocol.equalsIgnoreCase("ftp")) { + client = new FtpStorage(host, port, false); + } else if (protocol.equalsIgnoreCase("ftps")) { + client = new FtpStorage(host, port, true); + } else if (protocol.equalsIgnoreCase("file")) { + client = new LocalStorage(); + } else { + throw new RuntimeException("Invalid storage protocol: " + protocol); + } + client.login(storageUser, storagePassword); + client.changeDir(path); + client.changeDir(jobId); + + return client; + } + +} diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties index 177e3e4..c06e76b 100644 --- a/src/main/resources/application.properties +++ b/src/main/resources/application.properties @@ -12,10 +12,8 @@ spring.jpa.hibernate.ddl-auto=update spring.jpa.properties.hibernate.format_sql=true spring.jpa.properties.hibernate.dialect=org.hibernate.dialect.H2Dialect -oai2ftp.conf.ftp.server = -oai2ftp.conf.ftp.user = test -oai2ftp.conf.ftp.password = testPwd -oai2ftp.conf.ftp.basedir = /oai -oai2ftp.conf.ftp.secure = true +oai2ftp.conf.storage.url = file:///tmp/test_oai +oai2ftp.conf.storage.user = +oai2ftp.conf.storage.password = oai2ftp.conf.execution.expirationTime = 12