local storage
This commit is contained in:
parent
f91e19e8f6
commit
71a72d0bb2
|
@ -27,10 +27,10 @@ import eu.dnetlib.apps.oai2ftp.model.CollectionCall;
|
|||
import eu.dnetlib.apps.oai2ftp.model.CollectionInfo;
|
||||
import eu.dnetlib.apps.oai2ftp.model.ExecutionStatus;
|
||||
import eu.dnetlib.apps.oai2ftp.repository.CollectionLogEntryRepository;
|
||||
import eu.dnetlib.apps.oai2ftp.utils.FtpClientFactory;
|
||||
import eu.dnetlib.apps.oai2ftp.utils.FtpClientWrapper;
|
||||
import eu.dnetlib.apps.oai2ftp.utils.HttpFetcher;
|
||||
import eu.dnetlib.apps.oai2ftp.utils.SimpleUtils;
|
||||
import eu.dnetlib.apps.oai2ftp.utils.StorageClient;
|
||||
import eu.dnetlib.apps.oai2ftp.utils.StorageClientFactory;
|
||||
|
||||
@Service
|
||||
public class Oai2FtpService {
|
||||
|
@ -42,7 +42,7 @@ public class Oai2FtpService {
|
|||
private final Map<String, CollectionInfo> infoMap = new LinkedHashMap<>();
|
||||
|
||||
@Autowired
|
||||
private FtpClientFactory ftpClientFactory;
|
||||
private StorageClientFactory storageClientFactory;
|
||||
|
||||
@Value("${oai2ftp.conf.execution.expirationTime}")
|
||||
private long fullInfoExpirationTime; // in hours
|
||||
|
@ -57,7 +57,7 @@ public class Oai2FtpService {
|
|||
final LocalDateTime until) {
|
||||
final String jobId = SimpleUtils.generateNewJobId();
|
||||
|
||||
final FtpClientWrapper ftp = ftpClientFactory.newClientForJob(jobId);
|
||||
final StorageClient sc = storageClientFactory.newClientForJob(jobId);
|
||||
|
||||
final CollectionInfo info = new CollectionInfo();
|
||||
info.setId(jobId);
|
||||
|
@ -80,14 +80,14 @@ public class Oai2FtpService {
|
|||
jobExecutor.execute(() -> {
|
||||
try {
|
||||
info.setExecutionStatus(ExecutionStatus.RUNNING);
|
||||
oaiCollect(baseUrl, format, setSpec, from, until, ftp, info);
|
||||
oaiCollect(baseUrl, format, setSpec, from, until, sc, info);
|
||||
info.setExecutionStatus(ExecutionStatus.COMPLETED);
|
||||
} catch (final Throwable e) {
|
||||
info.setExecutionStatus(ExecutionStatus.FAILED);
|
||||
info.setMessage(e.getMessage() + ": " + ExceptionUtils.getStackTrace(e));
|
||||
} finally {
|
||||
info.setEnd(LocalDateTime.now());
|
||||
ftp.disconnect();
|
||||
sc.disconnect();
|
||||
collectionLogEntryRepository.save(SimpleUtils.infoToLog(info));
|
||||
}
|
||||
});
|
||||
|
@ -100,7 +100,7 @@ public class Oai2FtpService {
|
|||
final String setSpec,
|
||||
final LocalDateTime from,
|
||||
final LocalDateTime until,
|
||||
final FtpClientWrapper ftp,
|
||||
final StorageClient sc,
|
||||
final CollectionInfo info)
|
||||
throws Exception {
|
||||
|
||||
|
@ -118,8 +118,9 @@ public class Oai2FtpService {
|
|||
call.setNumberOfRecords(records.size());
|
||||
|
||||
for (final Node n : records) {
|
||||
final String id = n.valueOf("/*[local-name()='record']/*[local-name()='header']/*[local-name()='identifier']");
|
||||
ftp.saveFile(SimpleUtils.oaiIdToFilename(id), n.asXML());
|
||||
final String id = n.valueOf(".//*[local-name()='header']/*[local-name()='identifier']");
|
||||
|
||||
sc.saveFile(SimpleUtils.oaiIdToFilename(id), n.asXML());
|
||||
info.setTotal(info.getTotal() + 1);
|
||||
}
|
||||
|
||||
|
|
|
@ -1,32 +0,0 @@
|
|||
package eu.dnetlib.apps.oai2ftp.utils;
|
||||
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
@Component
|
||||
public class FtpClientFactory {
|
||||
|
||||
@Value("${oai2ftp.conf.ftp.server}")
|
||||
private String ftpServer;
|
||||
|
||||
@Value("${oai2ftp.conf.ftp.user}")
|
||||
private String ftpUser;
|
||||
|
||||
@Value("${oai2ftp.conf.ftp.password}")
|
||||
private String ftpPassword;
|
||||
|
||||
@Value("${oai2ftp.conf.ftp.basedir}")
|
||||
private String ftpBaseDir;
|
||||
|
||||
@Value("${oai2ftp.conf.ftp.secure}")
|
||||
private boolean ftpSecure;
|
||||
|
||||
public FtpClientWrapper newClientForJob(final String jobId) {
|
||||
final FtpClientWrapper ftp = new FtpClientWrapper(ftpServer, ftpSecure);
|
||||
ftp.login(ftpUser, ftpPassword);
|
||||
ftp.changeDir(ftpBaseDir);
|
||||
ftp.changeDir(jobId);
|
||||
return ftp;
|
||||
}
|
||||
|
||||
}
|
|
@ -4,29 +4,26 @@ import java.io.ByteArrayInputStream;
|
|||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.apache.commons.logging.Log;
|
||||
import org.apache.commons.logging.LogFactory;
|
||||
import org.apache.commons.net.ftp.FTPClient;
|
||||
import org.apache.commons.net.ftp.FTPSClient;
|
||||
|
||||
public class FtpClientWrapper {
|
||||
public class FtpStorage implements StorageClient {
|
||||
|
||||
private static final Log log = LogFactory.getLog(FtpClientWrapper.class);
|
||||
private static final Log log = LogFactory.getLog(FtpStorage.class);
|
||||
|
||||
private final FTPClient ftp;
|
||||
|
||||
public FtpClientWrapper(final String server, final boolean secure) {
|
||||
this.ftp = ftpConnect(server, secure);
|
||||
public FtpStorage(final String server, final int port, final boolean secure) {
|
||||
this.ftp = ftpConnect(server, port, secure);
|
||||
}
|
||||
|
||||
private FTPClient ftpConnect(final String server, final boolean secure) {
|
||||
private FTPClient ftpConnect(final String server, final int port, final boolean secure) {
|
||||
try {
|
||||
if (StringUtils.isBlank(server)) {
|
||||
return new MockFtpClient();
|
||||
} else if (secure) {
|
||||
if (secure) {
|
||||
final FTPSClient ftp = new FTPSClient();
|
||||
ftp.connect(server);
|
||||
ftp.connect(server, port > 0 ? port : FTPSClient.DEFAULT_FTPS_PORT);
|
||||
// Set protection buffer size
|
||||
ftp.execPBSZ(0);
|
||||
// Set data channel protection to private
|
||||
|
@ -34,7 +31,7 @@ public class FtpClientWrapper {
|
|||
return ftp;
|
||||
} else {
|
||||
final FTPClient ftp = new FTPClient();
|
||||
ftp.connect(server);
|
||||
ftp.connect(server, port > 0 ? port : FTPClient.DEFAULT_PORT);
|
||||
return ftp;
|
||||
}
|
||||
} catch (final IOException e) {
|
||||
|
@ -43,6 +40,7 @@ public class FtpClientWrapper {
|
|||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void disconnect() {
|
||||
if (ftp != null && ftp.isConnected()) {
|
||||
try {
|
||||
|
@ -55,6 +53,7 @@ public class FtpClientWrapper {
|
|||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void login(final String user, final String password) {
|
||||
try {
|
||||
if (!ftp.login(user, password)) { throw new RuntimeException("FTP login failed"); }
|
||||
|
@ -69,6 +68,7 @@ public class FtpClientWrapper {
|
|||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean changeDir(final String dir) {
|
||||
try {
|
||||
if (!ftp.changeWorkingDirectory(dir)) {
|
||||
|
@ -83,6 +83,7 @@ public class FtpClientWrapper {
|
|||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void saveFile(final String filename, final String content) {
|
||||
try (InputStream is = new ByteArrayInputStream(content.getBytes())) {
|
||||
if (log.isDebugEnabled()) {
|
|
@ -0,0 +1,46 @@
|
|||
package eu.dnetlib.apps.oai2ftp.utils;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.FileWriter;
|
||||
import java.io.IOException;
|
||||
|
||||
import org.apache.commons.io.FileUtils;
|
||||
import org.apache.commons.io.IOUtils;
|
||||
import org.apache.commons.logging.Log;
|
||||
import org.apache.commons.logging.LogFactory;
|
||||
|
||||
public class LocalStorage implements StorageClient {
|
||||
|
||||
private static final Log log = LogFactory.getLog(LocalStorage.class);
|
||||
|
||||
@Override
|
||||
public void login(final String user, final String password) {}
|
||||
|
||||
@Override
|
||||
public void disconnect() {}
|
||||
|
||||
public String currDir = "/tmp";
|
||||
|
||||
@Override
|
||||
public boolean changeDir(final String dir) {
|
||||
try {
|
||||
final File d = new File(dir.startsWith("/") ? dir : currDir + "/" + dir);
|
||||
FileUtils.forceMkdir(d);
|
||||
currDir = d.getAbsolutePath();
|
||||
return true;
|
||||
} catch (final IOException e) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void saveFile(final String filename, final String body) {
|
||||
try {
|
||||
IOUtils.write(body, new FileWriter(currDir + "/" + filename));
|
||||
} catch (final IOException e) {
|
||||
log.error("Error saving info file");
|
||||
throw new RuntimeException("Error saving info file", e);
|
||||
}
|
||||
}
|
||||
|
||||
}
|
|
@ -1,81 +0,0 @@
|
|||
package eu.dnetlib.apps.oai2ftp.utils;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
|
||||
import org.apache.commons.logging.Log;
|
||||
import org.apache.commons.logging.LogFactory;
|
||||
import org.apache.commons.net.ftp.FTPClient;
|
||||
|
||||
public class MockFtpClient extends FTPClient {
|
||||
|
||||
private static final Log log = LogFactory.getLog(MockFtpClient.class);
|
||||
|
||||
public MockFtpClient() {
|
||||
log.debug("MOCK FTP CLIENT - Constructor");
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean login(final String username, final String password) throws IOException {
|
||||
log.debug("MOCK FTP CLIENT - login: " + username + ", " + password);
|
||||
return true;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean isConnected() {
|
||||
log.debug("MOCK FTP CLIENT - isConnected");
|
||||
return true;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void disconnect() throws IOException {
|
||||
log.debug("MOCK FTP CLIENT - disconnect");
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean setFileType(final int fileType) throws IOException {
|
||||
log.debug("MOCK FTP CLIENT - setFileType");
|
||||
return true;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void setBufferSize(final int bufSize) {
|
||||
log.debug("MOCK FTP CLIENT - setBufferSize");
|
||||
}
|
||||
|
||||
@Override
|
||||
public void enterLocalPassiveMode() {
|
||||
log.debug("MOCK FTP CLIENT - enterLocalPassiveMode");
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean changeWorkingDirectory(final String pathname) throws IOException {
|
||||
log.debug("MOCK FTP CLIENT - changeWorkingDirectory: " + pathname);
|
||||
return true;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean makeDirectory(final String pathname) throws IOException {
|
||||
log.debug("MOCK FTP CLIENT - makeDirectory: " + pathname);
|
||||
return true;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean storeFile(final String remote, final InputStream local) throws IOException {
|
||||
log.debug("MOCK FTP CLIENT - storeFile: " + remote);
|
||||
return true;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int getReplyCode() {
|
||||
log.debug("MOCK FTP CLIENT - getReplyCode");
|
||||
return -1;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getReplyString() {
|
||||
log.debug("MOCK FTP CLIENT - getReplyString");
|
||||
return "MOCK";
|
||||
}
|
||||
|
||||
}
|
|
@ -0,0 +1,13 @@
|
|||
package eu.dnetlib.apps.oai2ftp.utils;
|
||||
|
||||
public interface StorageClient {
|
||||
|
||||
void login(String user, String password);
|
||||
|
||||
void disconnect();
|
||||
|
||||
boolean changeDir(String dir);
|
||||
|
||||
void saveFile(String filename, String body);
|
||||
|
||||
}
|
|
@ -0,0 +1,44 @@
|
|||
package eu.dnetlib.apps.oai2ftp.utils;
|
||||
|
||||
import java.net.URL;
|
||||
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
@Component
|
||||
public class StorageClientFactory {
|
||||
|
||||
@Value("${oai2ftp.conf.storage.url}")
|
||||
private URL storageUrl;
|
||||
|
||||
@Value("${oai2ftp.conf.storage.user}")
|
||||
private String storageUser;
|
||||
|
||||
@Value("${oai2ftp.conf.storage.password}")
|
||||
private String storagePassword;
|
||||
|
||||
public StorageClient newClientForJob(final String jobId) {
|
||||
|
||||
final String protocol = storageUrl.getProtocol();
|
||||
final String host = storageUrl.getHost();
|
||||
final int port = storageUrl.getPort();
|
||||
final String path = storageUrl.getPath();
|
||||
|
||||
StorageClient client;
|
||||
if (protocol.equalsIgnoreCase("ftp")) {
|
||||
client = new FtpStorage(host, port, false);
|
||||
} else if (protocol.equalsIgnoreCase("ftps")) {
|
||||
client = new FtpStorage(host, port, true);
|
||||
} else if (protocol.equalsIgnoreCase("file")) {
|
||||
client = new LocalStorage();
|
||||
} else {
|
||||
throw new RuntimeException("Invalid storage protocol: " + protocol);
|
||||
}
|
||||
client.login(storageUser, storagePassword);
|
||||
client.changeDir(path);
|
||||
client.changeDir(jobId);
|
||||
|
||||
return client;
|
||||
}
|
||||
|
||||
}
|
|
@ -12,10 +12,8 @@ spring.jpa.hibernate.ddl-auto=update
|
|||
spring.jpa.properties.hibernate.format_sql=true
|
||||
spring.jpa.properties.hibernate.dialect=org.hibernate.dialect.H2Dialect
|
||||
|
||||
oai2ftp.conf.ftp.server =
|
||||
oai2ftp.conf.ftp.user = test
|
||||
oai2ftp.conf.ftp.password = testPwd
|
||||
oai2ftp.conf.ftp.basedir = /oai
|
||||
oai2ftp.conf.ftp.secure = true
|
||||
oai2ftp.conf.storage.url = file:///tmp/test_oai
|
||||
oai2ftp.conf.storage.user =
|
||||
oai2ftp.conf.storage.password =
|
||||
|
||||
oai2ftp.conf.execution.expirationTime = 12
|
||||
|
|
Loading…
Reference in New Issue