diff --git a/src/main/java/eu/dnetlib/apps/Oai2ftp/model/CollectionInfo.java b/src/main/java/eu/dnetlib/apps/Oai2ftp/model/CollectionInfo.java index 8e5ac29..5d8e406 100644 --- a/src/main/java/eu/dnetlib/apps/Oai2ftp/model/CollectionInfo.java +++ b/src/main/java/eu/dnetlib/apps/Oai2ftp/model/CollectionInfo.java @@ -15,14 +15,13 @@ public class CollectionInfo implements Serializable { private String oaiSet; private LocalDateTime oaiFrom; private LocalDateTime oaiUntil; - private String ftpServer; - private String ftpDir; + private String storageUrl; private LocalDateTime start; private LocalDateTime end; private ExecutionStatus executionStatus; - private long total; + private long total = 0; private final List calls = new ArrayList<>(); - private String message; + private String message = ""; public String getId() { return id; @@ -72,20 +71,12 @@ public class CollectionInfo implements Serializable { this.oaiUntil = oaiUntil; } - public String getFtpServer() { - return ftpServer; + public String getStorageUrl() { + return storageUrl; } - public void setFtpServer(final String ftpServer) { - this.ftpServer = ftpServer; - } - - public String getFtpDir() { - return ftpDir; - } - - public void setFtpDir(final String ftpDir) { - this.ftpDir = ftpDir; + public void setStorageUrl(final String storageUrl) { + this.storageUrl = storageUrl; } public LocalDateTime getStart() { diff --git a/src/main/java/eu/dnetlib/apps/Oai2ftp/model/CollectionLogEntry.java b/src/main/java/eu/dnetlib/apps/Oai2ftp/model/CollectionLogEntry.java index 9393851..a4a984b 100644 --- a/src/main/java/eu/dnetlib/apps/Oai2ftp/model/CollectionLogEntry.java +++ b/src/main/java/eu/dnetlib/apps/Oai2ftp/model/CollectionLogEntry.java @@ -33,11 +33,8 @@ public class CollectionLogEntry implements Serializable { @Column(name = "oai_until") private LocalDateTime oaiUntil; - @Column(name = "ftp_server") - private String ftpServer; - - @Column(name = "ftp_dir") - private String ftpDir; + @Column(name = "storage_url") + private String storageUrl; @Column(name = "start_date") private LocalDateTime start; @@ -105,20 +102,12 @@ public class CollectionLogEntry implements Serializable { this.oaiUntil = oaiUntil; } - public String getFtpServer() { - return ftpServer; + public String getStorageUrl() { + return storageUrl; } - public void setFtpServer(final String ftpServer) { - this.ftpServer = ftpServer; - } - - public String getFtpDir() { - return ftpDir; - } - - public void setFtpDir(final String ftpDir) { - this.ftpDir = ftpDir; + public void setStorageUrl(final String storageUrl) { + this.storageUrl = storageUrl; } public LocalDateTime getStart() { diff --git a/src/main/java/eu/dnetlib/apps/Oai2ftp/service/Oai2FtpService.java b/src/main/java/eu/dnetlib/apps/Oai2ftp/service/Oai2FtpService.java index d8aee01..b0a6558 100644 --- a/src/main/java/eu/dnetlib/apps/Oai2ftp/service/Oai2FtpService.java +++ b/src/main/java/eu/dnetlib/apps/Oai2ftp/service/Oai2FtpService.java @@ -55,25 +55,22 @@ public class Oai2FtpService { final String setSpec, final LocalDateTime from, final LocalDateTime until) { + final String jobId = SimpleUtils.generateNewJobId(); - final StorageClient sc = storageClientFactory.newClientForJob(jobId); + final StorageClient sc = storageClientFactory.newClient(); + sc.changeDir(jobId); final CollectionInfo info = new CollectionInfo(); info.setId(jobId); - info.setOaiBaseUrl(baseUrl); info.setOaiFormat(format); info.setOaiSet(setSpec); info.setOaiFrom(from); info.setOaiUntil(until); - + info.setStorageUrl(storageClientFactory.getStorageUrlAsString()); info.setStart(LocalDateTime.now()); - info.setEnd(null); - info.setExecutionStatus(ExecutionStatus.READY); - info.setTotal(0); - info.setMessage(""); infoMap.put(jobId, info); @@ -106,6 +103,7 @@ public class Oai2FtpService { String url = SimpleUtils.oaiFirstUrl(baseUrl, format, setSpec, from, until); + int count = 0; while (StringUtils.isNotBlank(url)) { final CollectionCall call = new CollectionCall(); call.setUrl(url); @@ -117,12 +115,13 @@ public class Oai2FtpService { final List records = doc.selectNodes("//*[local-name()='ListRecords']/*[local-name()='record']"); call.setNumberOfRecords(records.size()); + sc.changeDir(Integer.toString(count++)); for (final Node n : records) { final String id = n.valueOf(".//*[local-name()='header']/*[local-name()='identifier']"); - sc.saveFile(SimpleUtils.oaiIdToFilename(id), n.asXML()); info.setTotal(info.getTotal() + 1); } + sc.changeDir(".."); final String rtoken = doc.valueOf("//*[local-name()='resumptionToken']").trim(); diff --git a/src/main/java/eu/dnetlib/apps/Oai2ftp/utils/LocalStorage.java b/src/main/java/eu/dnetlib/apps/Oai2ftp/utils/LocalStorage.java index 99fe946..c10d823 100644 --- a/src/main/java/eu/dnetlib/apps/Oai2ftp/utils/LocalStorage.java +++ b/src/main/java/eu/dnetlib/apps/Oai2ftp/utils/LocalStorage.java @@ -35,8 +35,8 @@ public class LocalStorage implements StorageClient { @Override public void saveFile(final String filename, final String body) { - try { - IOUtils.write(body, new FileWriter(currDir + "/" + filename)); + try (FileWriter fw = new FileWriter(currDir + "/" + filename)) { + IOUtils.write(body, fw); } catch (final IOException e) { log.error("Error saving info file"); throw new RuntimeException("Error saving info file", e); diff --git a/src/main/java/eu/dnetlib/apps/Oai2ftp/utils/SimpleUtils.java b/src/main/java/eu/dnetlib/apps/Oai2ftp/utils/SimpleUtils.java index d71122c..f7f327a 100644 --- a/src/main/java/eu/dnetlib/apps/Oai2ftp/utils/SimpleUtils.java +++ b/src/main/java/eu/dnetlib/apps/Oai2ftp/utils/SimpleUtils.java @@ -63,8 +63,7 @@ public class SimpleUtils { info.setOaiSet(log.getOaiSet()); info.setOaiFrom(log.getOaiFrom()); info.setOaiUntil(log.getOaiUntil()); - info.setFtpServer(log.getFtpServer()); - info.setFtpDir(log.getFtpDir()); + info.setStorageUrl(log.getStorageUrl()); info.setStart(log.getStart()); info.setEnd(log.getEnd()); info.setExecutionStatus(log.isSuccess() ? ExecutionStatus.COMPLETED : ExecutionStatus.FAILED); @@ -81,8 +80,7 @@ public class SimpleUtils { log.setOaiSet(info.getOaiSet()); log.setOaiFrom(info.getOaiFrom()); log.setOaiUntil(info.getOaiUntil()); - log.setFtpServer(info.getFtpServer()); - log.setFtpDir(info.getFtpDir()); + log.setStorageUrl(info.getStorageUrl()); log.setStart(info.getStart()); log.setEnd(info.getEnd()); log.setSuccess(info.getExecutionStatus() == ExecutionStatus.COMPLETED); diff --git a/src/main/java/eu/dnetlib/apps/Oai2ftp/utils/StorageClientFactory.java b/src/main/java/eu/dnetlib/apps/Oai2ftp/utils/StorageClientFactory.java index 5637e63..f213384 100644 --- a/src/main/java/eu/dnetlib/apps/Oai2ftp/utils/StorageClientFactory.java +++ b/src/main/java/eu/dnetlib/apps/Oai2ftp/utils/StorageClientFactory.java @@ -17,7 +17,7 @@ public class StorageClientFactory { @Value("${oai2ftp.conf.storage.password}") private String storagePassword; - public StorageClient newClientForJob(final String jobId) { + public StorageClient newClient() { final String protocol = storageUrl.getProtocol(); final String host = storageUrl.getHost(); @@ -36,9 +36,12 @@ public class StorageClientFactory { } client.login(storageUser, storagePassword); client.changeDir(path); - client.changeDir(jobId); return client; } + public String getStorageUrlAsString() { + return storageUrl.toString(); + } + }