88 lines
2.4 KiB
Java
88 lines
2.4 KiB
Java
package eu.dnetlib.apps.oai2ftp.service;
|
|
|
|
import java.util.LinkedHashMap;
|
|
import java.util.Map;
|
|
import java.util.UUID;
|
|
import java.util.concurrent.ExecutorService;
|
|
import java.util.concurrent.Executors;
|
|
|
|
import org.apache.commons.logging.Log;
|
|
import org.apache.commons.logging.LogFactory;
|
|
import org.apache.commons.net.ftp.FTPClient;
|
|
import org.springframework.beans.factory.annotation.Autowired;
|
|
import org.springframework.beans.factory.annotation.Value;
|
|
import org.springframework.stereotype.Service;
|
|
|
|
import eu.dnetlib.apps.oai2ftp.model.CollectionStatus;
|
|
import eu.dnetlib.apps.oai2ftp.repository.CollectionLogEntryRepository;
|
|
import eu.dnetlib.apps.oai2ftp.utils.ConvertUtils;
|
|
import eu.dnetlib.apps.oai2ftp.utils.FtpUtils;
|
|
|
|
@Service
|
|
public class Oai2FtpService {
|
|
|
|
private static final Log log = LogFactory.getLog(Oai2FtpService.class);
|
|
|
|
private final ExecutorService jobExecutor = Executors.newFixedThreadPool(100);
|
|
|
|
private final Map<String, CollectionJob> runningJobs = new LinkedHashMap<>();
|
|
|
|
@Value("oai2ftp.conf.ftp.server")
|
|
private String ftpServer;
|
|
|
|
@Value("oai2ftp.conf.ftp.user")
|
|
private String ftpUser;
|
|
|
|
@Value("oai2ftp.conf.ftp.password")
|
|
private String ftpPassword;
|
|
|
|
@Value("oai2ftp.conf.ftp.basedir")
|
|
private String ftpBaseDir;
|
|
|
|
private final boolean ftpSecure = false;
|
|
|
|
@Autowired
|
|
private CollectionLogEntryRepository collectionLogEntryRepository;
|
|
|
|
public CollectionStatus startCollection(final String baseUrl, final String format, final String setSpec) {
|
|
final String jobId = generateNewJobId();
|
|
|
|
final FTPClient ftp = FtpUtils.ftpConnect(ftpServer, ftpSecure);
|
|
FtpUtils.ftpLogin(ftp, ftpUser, ftpPassword);
|
|
FtpUtils.changeDir(ftp, ftpBaseDir);
|
|
FtpUtils.changeDir(ftp, jobId);
|
|
|
|
final CollectionJob job = new CollectionJob(jobId,
|
|
baseUrl,
|
|
format,
|
|
setSpec,
|
|
(id, body) -> FtpUtils.saveRecord(id, body),
|
|
(status) -> {
|
|
FtpUtils.ftpDisconnect(ftp);
|
|
collectionLogEntryRepository.save(ConvertUtils.statusToLog(status));
|
|
});
|
|
|
|
runningJobs.put(jobId, job);
|
|
|
|
jobExecutor.execute(() -> job.oaiCollect());
|
|
|
|
return job.getStatus();
|
|
};
|
|
|
|
private String generateNewJobId() {
|
|
return "job-" + UUID.randomUUID();
|
|
}
|
|
|
|
public CollectionStatus getStatus(final String jobId) {
|
|
final CollectionJob job = runningJobs.get(jobId);
|
|
if (job != null) {
|
|
return job.getStatus();
|
|
} else {
|
|
return collectionLogEntryRepository.findById(jobId)
|
|
.map(ConvertUtils::logToStatus)
|
|
.orElse(null);
|
|
}
|
|
}
|
|
|
|
}
|