michele.artini
/
simpleOaiCollectorService
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

local storage

This commit is contained in:
Michele Artini 2023-05-26 13:52:31 +02:00
parent f91e19e8f6
commit 71a72d0bb2
8 changed files with 128 additions and 138 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
src/main/java/eu/dnetlib/apps/Oai2ftp/service/Oai2FtpService.java
View File

@ -27,10 +27,10 @@ import eu.dnetlib.apps.oai2ftp.model.CollectionCall;
import eu.dnetlib.apps.oai2ftp.model.CollectionInfo;
import eu.dnetlib.apps.oai2ftp.model.ExecutionStatus;
import eu.dnetlib.apps.oai2ftp.repository.CollectionLogEntryRepository;
import eu.dnetlib.apps.oai2ftp.utils.FtpClientFactory;
import eu.dnetlib.apps.oai2ftp.utils.FtpClientWrapper;
import eu.dnetlib.apps.oai2ftp.utils.HttpFetcher;
import eu.dnetlib.apps.oai2ftp.utils.SimpleUtils;
import eu.dnetlib.apps.oai2ftp.utils.StorageClient;
import eu.dnetlib.apps.oai2ftp.utils.StorageClientFactory;
@Service
public class Oai2FtpService {
@ -42,7 +42,7 @@ public class Oai2FtpService {
private final Map<String, CollectionInfo> infoMap = new LinkedHashMap<>();
@Autowired
private FtpClientFactory ftpClientFactory;
private StorageClientFactory storageClientFactory;
@Value("${oai2ftp.conf.execution.expirationTime}")
private long fullInfoExpirationTime; // in hours
@ -57,7 +57,7 @@ public class Oai2FtpService {
final LocalDateTime until) {
final String jobId = SimpleUtils.generateNewJobId();
final FtpClientWrapper ftp = ftpClientFactory.newClientForJob(jobId);
final StorageClient sc = storageClientFactory.newClientForJob(jobId);
final CollectionInfo info = new CollectionInfo();
info.setId(jobId);
@ -80,14 +80,14 @@ public class Oai2FtpService {
jobExecutor.execute(() -> {
try {
info.setExecutionStatus(ExecutionStatus.RUNNING);
oaiCollect(baseUrl, format, setSpec, from, until, ftp, info);
oaiCollect(baseUrl, format, setSpec, from, until, sc, info);
info.setExecutionStatus(ExecutionStatus.COMPLETED);
} catch (final Throwable e) {
info.setExecutionStatus(ExecutionStatus.FAILED);
info.setMessage(e.getMessage() + ": " + ExceptionUtils.getStackTrace(e));
} finally {
info.setEnd(LocalDateTime.now());
ftp.disconnect();
sc.disconnect();
collectionLogEntryRepository.save(SimpleUtils.infoToLog(info));
}
});
@ -100,7 +100,7 @@ public class Oai2FtpService {
final String setSpec,
final LocalDateTime from,
final LocalDateTime until,
final FtpClientWrapper ftp,
final StorageClient sc,
final CollectionInfo info)
throws Exception {
@ -118,8 +118,9 @@ public class Oai2FtpService {
call.setNumberOfRecords(records.size());
for (final Node n : records) {
final String id = n.valueOf("/*[local-name()='record']/*[local-name()='header']/*[local-name()='identifier']");
ftp.saveFile(SimpleUtils.oaiIdToFilename(id), n.asXML());
final String id = n.valueOf(".//*[local-name()='header']/*[local-name()='identifier']");
sc.saveFile(SimpleUtils.oaiIdToFilename(id), n.asXML());
info.setTotal(info.getTotal() + 1);
}

32
src/main/java/eu/dnetlib/apps/Oai2ftp/utils/FtpClientFactory.java
View File

@ -1,32 +0,0 @@
package eu.dnetlib.apps.oai2ftp.utils;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
@Component
public class FtpClientFactory {
@Value("${oai2ftp.conf.ftp.server}")
private String ftpServer;
@Value("${oai2ftp.conf.ftp.user}")
private String ftpUser;
@Value("${oai2ftp.conf.ftp.password}")
private String ftpPassword;
@Value("${oai2ftp.conf.ftp.basedir}")
private String ftpBaseDir;
@Value("${oai2ftp.conf.ftp.secure}")
private boolean ftpSecure;
public FtpClientWrapper newClientForJob(final String jobId) {
final FtpClientWrapper ftp = new FtpClientWrapper(ftpServer, ftpSecure);
ftp.login(ftpUser, ftpPassword);
ftp.changeDir(ftpBaseDir);
ftp.changeDir(jobId);
return ftp;
}
}

23
src/main/java/eu/dnetlib/apps/Oai2ftp/utils/FtpClientWrapper.java → src/main/java/eu/dnetlib/apps/Oai2ftp/utils/FtpStorage.java
View File

@ -4,29 +4,26 @@ import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.commons.net.ftp.FTPClient;
import org.apache.commons.net.ftp.FTPSClient;
public class FtpClientWrapper {
public class FtpStorage implements StorageClient {
private static final Log log = LogFactory.getLog(FtpClientWrapper.class);
private static final Log log = LogFactory.getLog(FtpStorage.class);
private final FTPClient ftp;
public FtpClientWrapper(final String server, final boolean secure) {
this.ftp = ftpConnect(server, secure);
public FtpStorage(final String server, final int port, final boolean secure) {
this.ftp = ftpConnect(server, port, secure);
}
private FTPClient ftpConnect(final String server, final boolean secure) {
private FTPClient ftpConnect(final String server, final int port, final boolean secure) {
try {
if (StringUtils.isBlank(server)) {
return new MockFtpClient();
} else if (secure) {
if (secure) {
final FTPSClient ftp = new FTPSClient();
ftp.connect(server);
ftp.connect(server, port > 0 ? port : FTPSClient.DEFAULT_FTPS_PORT);
// Set protection buffer size
ftp.execPBSZ(0);
// Set data channel protection to private
@ -34,7 +31,7 @@ public class FtpClientWrapper {
return ftp;
} else {
final FTPClient ftp = new FTPClient();
ftp.connect(server);
ftp.connect(server, port > 0 ? port : FTPClient.DEFAULT_PORT);
return ftp;
}
} catch (final IOException e) {
@ -43,6 +40,7 @@ public class FtpClientWrapper {
}
}
@Override
public void disconnect() {
if (ftp != null && ftp.isConnected()) {
try {
@ -55,6 +53,7 @@ public class FtpClientWrapper {
}
}
@Override
public void login(final String user, final String password) {
try {
if (!ftp.login(user, password)) { throw new RuntimeException("FTP login failed"); }
@ -69,6 +68,7 @@ public class FtpClientWrapper {
}
}
@Override
public boolean changeDir(final String dir) {
try {
if (!ftp.changeWorkingDirectory(dir)) {
@ -83,6 +83,7 @@ public class FtpClientWrapper {
}
}
@Override
public void saveFile(final String filename, final String content) {
try (InputStream is = new ByteArrayInputStream(content.getBytes())) {
if (log.isDebugEnabled()) {

46
src/main/java/eu/dnetlib/apps/Oai2ftp/utils/LocalStorage.java Normal file
View File

@ -0,0 +1,46 @@
package eu.dnetlib.apps.oai2ftp.utils;
import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.IOUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
public class LocalStorage implements StorageClient {
private static final Log log = LogFactory.getLog(LocalStorage.class);
@Override
public void login(final String user, final String password) {}
@Override
public void disconnect() {}
public String currDir = "/tmp";
@Override
public boolean changeDir(final String dir) {
try {
final File d = new File(dir.startsWith("/") ? dir : currDir + "/" + dir);
FileUtils.forceMkdir(d);
currDir = d.getAbsolutePath();
return true;
} catch (final IOException e) {
return false;
}
}
@Override
public void saveFile(final String filename, final String body) {
try {
IOUtils.write(body, new FileWriter(currDir + "/" + filename));
} catch (final IOException e) {
log.error("Error saving info file");
throw new RuntimeException("Error saving info file", e);
}
}
}

81
src/main/java/eu/dnetlib/apps/Oai2ftp/utils/MockFtpClient.java
View File

@ -1,81 +0,0 @@
package eu.dnetlib.apps.oai2ftp.utils;
import java.io.IOException;
import java.io.InputStream;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.commons.net.ftp.FTPClient;
public class MockFtpClient extends FTPClient {
private static final Log log = LogFactory.getLog(MockFtpClient.class);
public MockFtpClient() {
log.debug("MOCK FTP CLIENT - Constructor");
}
@Override
public boolean login(final String username, final String password) throws IOException {
log.debug("MOCK FTP CLIENT - login: " + username + ", " + password);
return true;
}
@Override
public boolean isConnected() {
log.debug("MOCK FTP CLIENT - isConnected");
return true;
}
@Override
public void disconnect() throws IOException {
log.debug("MOCK FTP CLIENT - disconnect");
}
@Override
public boolean setFileType(final int fileType) throws IOException {
log.debug("MOCK FTP CLIENT - setFileType");
return true;
}
@Override
public void setBufferSize(final int bufSize) {
log.debug("MOCK FTP CLIENT - setBufferSize");
}
@Override
public void enterLocalPassiveMode() {
log.debug("MOCK FTP CLIENT - enterLocalPassiveMode");
}
@Override
public boolean changeWorkingDirectory(final String pathname) throws IOException {
log.debug("MOCK FTP CLIENT - changeWorkingDirectory: " + pathname);
return true;
}
@Override
public boolean makeDirectory(final String pathname) throws IOException {
log.debug("MOCK FTP CLIENT - makeDirectory: " + pathname);
return true;
}
@Override
public boolean storeFile(final String remote, final InputStream local) throws IOException {
log.debug("MOCK FTP CLIENT - storeFile: " + remote);
return true;
}
@Override
public int getReplyCode() {
log.debug("MOCK FTP CLIENT - getReplyCode");
return -1;
}
@Override
public String getReplyString() {
log.debug("MOCK FTP CLIENT - getReplyString");
return "MOCK";
}
}

13
src/main/java/eu/dnetlib/apps/Oai2ftp/utils/StorageClient.java Normal file
View File

@ -0,0 +1,13 @@
package eu.dnetlib.apps.oai2ftp.utils;
public interface StorageClient {
void login(String user, String password);
void disconnect();
boolean changeDir(String dir);
void saveFile(String filename, String body);
}

44
src/main/java/eu/dnetlib/apps/Oai2ftp/utils/StorageClientFactory.java Normal file
View File

@ -0,0 +1,44 @@
package eu.dnetlib.apps.oai2ftp.utils;
import java.net.URL;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
@Component
public class StorageClientFactory {
@Value("${oai2ftp.conf.storage.url}")
private URL storageUrl;
@Value("${oai2ftp.conf.storage.user}")
private String storageUser;
@Value("${oai2ftp.conf.storage.password}")
private String storagePassword;
public StorageClient newClientForJob(final String jobId) {
final String protocol = storageUrl.getProtocol();
final String host = storageUrl.getHost();
final int port = storageUrl.getPort();
final String path = storageUrl.getPath();
StorageClient client;
if (protocol.equalsIgnoreCase("ftp")) {
client = new FtpStorage(host, port, false);
} else if (protocol.equalsIgnoreCase("ftps")) {
client = new FtpStorage(host, port, true);
} else if (protocol.equalsIgnoreCase("file")) {
client = new LocalStorage();
} else {
throw new RuntimeException("Invalid storage protocol: " + protocol);
}
client.login(storageUser, storagePassword);
client.changeDir(path);
client.changeDir(jobId);
return client;
}
}

8
src/main/resources/application.properties
View File

@ -12,10 +12,8 @@ spring.jpa.hibernate.ddl-auto=update
spring.jpa.properties.hibernate.format_sql=true
spring.jpa.properties.hibernate.dialect=org.hibernate.dialect.H2Dialect
oai2ftp.conf.ftp.server =
oai2ftp.conf.ftp.user = test
oai2ftp.conf.ftp.password = testPwd
oai2ftp.conf.ftp.basedir = /oai
oai2ftp.conf.ftp.secure = true
oai2ftp.conf.storage.url = file:///tmp/test_oai
oai2ftp.conf.storage.user =
oai2ftp.conf.storage.password =
oai2ftp.conf.execution.expirationTime = 12