From d4f36ca37f8ffcabc6a1600f10a010afa5402ddd Mon Sep 17 00:00:00 2001 From: "michele.artini" Date: Wed, 20 Sep 2023 16:33:14 +0200 Subject: [PATCH] new functionalities (zip, limits, emails, exposed urls) --- .project | 2 +- .../apps/Oai2ftp/utils/StorageClient.java | 13 -- .../MainApplication.java} | 6 +- .../controller/ApiController.java} | 40 +++--- .../controller/SwaggerController.java | 2 +- .../model/CollectionCall.java | 2 +- .../model/CollectionInfo.java | 35 +++++- .../model/ExecutionStatus.java | 2 +- .../repository/CollectionInfoRepository.java | 4 +- .../service/CollectorService.java} | 118 ++++++++++++------ .../utils => oai/storage}/FtpStorage.java | 26 +++- .../utils => oai/storage}/LocalStorage.java | 32 +++-- .../apps/oai/storage/StorageClient.java | 15 +++ .../storage}/StorageClientFactory.java | 28 ++--- .../dnetlib/apps/oai/storage/ZipStorage.java | 74 +++++++++++ .../dnetlib/apps/oai/utils/EmailSender.java | 37 ++++++ .../{Oai2ftp => oai}/utils/HttpFetcher.java | 6 +- .../{Oai2ftp => oai}/utils/SimpleUtils.java | 2 +- src/main/resources/application.properties | 19 +-- .../OaiApplicationTests.java} | 4 +- 20 files changed, 350 insertions(+), 117 deletions(-) delete mode 100644 src/main/java/eu/dnetlib/apps/Oai2ftp/utils/StorageClient.java rename src/main/java/eu/dnetlib/apps/{Oai2ftp/Oai2ftpApplication.java => oai/MainApplication.java} (94%) rename src/main/java/eu/dnetlib/apps/{Oai2ftp/controller/Oai2FtpController.java => oai/controller/ApiController.java} (66%) rename src/main/java/eu/dnetlib/apps/{Oai2ftp => oai}/controller/SwaggerController.java (87%) rename src/main/java/eu/dnetlib/apps/{Oai2ftp => oai}/model/CollectionCall.java (96%) rename src/main/java/eu/dnetlib/apps/{Oai2ftp => oai}/model/CollectionInfo.java (81%) rename src/main/java/eu/dnetlib/apps/{Oai2ftp => oai}/model/ExecutionStatus.java (64%) rename src/main/java/eu/dnetlib/apps/{Oai2ftp => oai}/repository/CollectionInfoRepository.java (61%) rename src/main/java/eu/dnetlib/apps/{Oai2ftp/service/Oai2FtpService.java => oai/service/CollectorService.java} (58%) rename src/main/java/eu/dnetlib/apps/{Oai2ftp/utils => oai/storage}/FtpStorage.java (85%) rename src/main/java/eu/dnetlib/apps/{Oai2ftp/utils => oai/storage}/LocalStorage.java (52%) create mode 100644 src/main/java/eu/dnetlib/apps/oai/storage/StorageClient.java rename src/main/java/eu/dnetlib/apps/{Oai2ftp/utils => oai/storage}/StorageClientFactory.java (57%) create mode 100644 src/main/java/eu/dnetlib/apps/oai/storage/ZipStorage.java create mode 100644 src/main/java/eu/dnetlib/apps/oai/utils/EmailSender.java rename src/main/java/eu/dnetlib/apps/{Oai2ftp => oai}/utils/HttpFetcher.java (88%) rename src/main/java/eu/dnetlib/apps/{Oai2ftp => oai}/utils/SimpleUtils.java (97%) rename src/test/java/eu/dnetlib/apps/{oai2ftp/Oai2ftpApplicationTests.java => oai/OaiApplicationTests.java} (69%) diff --git a/.project b/.project index 5b7f2cd..7b7f352 100644 --- a/.project +++ b/.project @@ -1,6 +1,6 @@ - oai2ftp + simpleOaiCollectorService diff --git a/src/main/java/eu/dnetlib/apps/Oai2ftp/utils/StorageClient.java b/src/main/java/eu/dnetlib/apps/Oai2ftp/utils/StorageClient.java deleted file mode 100644 index 0505eef..0000000 --- a/src/main/java/eu/dnetlib/apps/Oai2ftp/utils/StorageClient.java +++ /dev/null @@ -1,13 +0,0 @@ -package eu.dnetlib.apps.oai2ftp.utils; - -public interface StorageClient { - - void login(String user, String password); - - void disconnect(); - - boolean changeDir(String dir); - - void saveFile(String filename, String body); - -} diff --git a/src/main/java/eu/dnetlib/apps/Oai2ftp/Oai2ftpApplication.java b/src/main/java/eu/dnetlib/apps/oai/MainApplication.java similarity index 94% rename from src/main/java/eu/dnetlib/apps/Oai2ftp/Oai2ftpApplication.java rename to src/main/java/eu/dnetlib/apps/oai/MainApplication.java index 483c323..931f1f9 100644 --- a/src/main/java/eu/dnetlib/apps/Oai2ftp/Oai2ftpApplication.java +++ b/src/main/java/eu/dnetlib/apps/oai/MainApplication.java @@ -1,4 +1,4 @@ -package eu.dnetlib.apps.oai2ftp; +package eu.dnetlib.apps.oai; import java.util.ArrayList; import java.util.List; @@ -19,7 +19,7 @@ import io.swagger.v3.oas.models.tags.Tag; @SpringBootApplication @EnableScheduling -public class Oai2ftpApplication { +public class MainApplication { @Value("${swagger.public_url}") private String swaggerPublicUrl; @@ -40,7 +40,7 @@ public class Oai2ftpApplication { new License().name("GNU Affero General Public License v3.0 or later").url("https://www.gnu.org/licenses/agpl-3.0.txt"); public static void main(final String[] args) { - SpringApplication.run(Oai2ftpApplication.class, args); + SpringApplication.run(MainApplication.class, args); } @Bean diff --git a/src/main/java/eu/dnetlib/apps/Oai2ftp/controller/Oai2FtpController.java b/src/main/java/eu/dnetlib/apps/oai/controller/ApiController.java similarity index 66% rename from src/main/java/eu/dnetlib/apps/Oai2ftp/controller/Oai2FtpController.java rename to src/main/java/eu/dnetlib/apps/oai/controller/ApiController.java index 41c02d1..d047e97 100644 --- a/src/main/java/eu/dnetlib/apps/Oai2ftp/controller/Oai2FtpController.java +++ b/src/main/java/eu/dnetlib/apps/oai/controller/ApiController.java @@ -1,14 +1,15 @@ -package eu.dnetlib.apps.oai2ftp.controller; +package eu.dnetlib.apps.oai.controller; import java.time.LocalDateTime; -import java.util.Arrays; -import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; import org.apache.commons.lang3.exception.ExceptionUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.http.HttpStatus; +import org.springframework.web.bind.annotation.DeleteMapping; import org.springframework.web.bind.annotation.ExceptionHandler; import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.PathVariable; @@ -18,36 +19,47 @@ import org.springframework.web.bind.annotation.ResponseBody; import org.springframework.web.bind.annotation.ResponseStatus; import org.springframework.web.bind.annotation.RestController; -import eu.dnetlib.apps.oai2ftp.model.CollectionInfo; -import eu.dnetlib.apps.oai2ftp.service.Oai2FtpService; +import eu.dnetlib.apps.oai.model.CollectionInfo; +import eu.dnetlib.apps.oai.model.ExecutionStatus; +import eu.dnetlib.apps.oai.service.CollectorService; @RestController @RequestMapping("/api") -public class Oai2FtpController { +public class ApiController { - private static final Log log = LogFactory.getLog(Oai2FtpController.class); + private static final Log log = LogFactory.getLog(ApiController.class); @Autowired - private Oai2FtpService service; + private CollectorService service; @GetMapping("/collect") public CollectionInfo startCollection(@RequestParam final String oaiBaseUrl, @RequestParam(required = false, defaultValue = "oai_dc") final String oaiFormat, @RequestParam(required = false) final String oaiSet, @RequestParam(required = false) final LocalDateTime oaiFrom, - @RequestParam(required = false) final LocalDateTime oaiUntil) { - return service.startCollection(oaiBaseUrl, oaiFormat, oaiSet, oaiFrom, oaiUntil); + @RequestParam(required = false) final LocalDateTime oaiUntil, + @RequestParam(required = false) final Long max, + @RequestParam(required = false) final String notificationEmail) { + return service.startCollection(oaiBaseUrl, oaiFormat, oaiSet, oaiFrom, oaiUntil, max, notificationEmail); } - @GetMapping("/status/{id}") + @GetMapping("/history/{id}") public CollectionInfo getExecutionStatus(@PathVariable final String id) { return service.getStatus(id); } - @GetMapping("/history/clean") - public List cleanHistory() { + @GetMapping("/history") + public Map history() { + return service.history() + .entrySet() + .stream() + .collect(Collectors.toMap(e -> e.getKey(), e -> e.getValue().getExecutionStatus())); + } + + @DeleteMapping("/history") + public Map cleanHistory() { service.cleanHistory(); - return Arrays.asList("DONE."); + return history(); } @ExceptionHandler(Exception.class) diff --git a/src/main/java/eu/dnetlib/apps/Oai2ftp/controller/SwaggerController.java b/src/main/java/eu/dnetlib/apps/oai/controller/SwaggerController.java similarity index 87% rename from src/main/java/eu/dnetlib/apps/Oai2ftp/controller/SwaggerController.java rename to src/main/java/eu/dnetlib/apps/oai/controller/SwaggerController.java index cd0668c..bc3cb3c 100644 --- a/src/main/java/eu/dnetlib/apps/Oai2ftp/controller/SwaggerController.java +++ b/src/main/java/eu/dnetlib/apps/oai/controller/SwaggerController.java @@ -1,4 +1,4 @@ -package eu.dnetlib.apps.oai2ftp.controller; +package eu.dnetlib.apps.oai.controller; import org.springframework.stereotype.Controller; import org.springframework.web.bind.annotation.GetMapping; diff --git a/src/main/java/eu/dnetlib/apps/Oai2ftp/model/CollectionCall.java b/src/main/java/eu/dnetlib/apps/oai/model/CollectionCall.java similarity index 96% rename from src/main/java/eu/dnetlib/apps/Oai2ftp/model/CollectionCall.java rename to src/main/java/eu/dnetlib/apps/oai/model/CollectionCall.java index 28fbb4d..713cbdd 100644 --- a/src/main/java/eu/dnetlib/apps/Oai2ftp/model/CollectionCall.java +++ b/src/main/java/eu/dnetlib/apps/oai/model/CollectionCall.java @@ -1,4 +1,4 @@ -package eu.dnetlib.apps.oai2ftp.model; +package eu.dnetlib.apps.oai.model; import java.io.Serializable; import java.util.Objects; diff --git a/src/main/java/eu/dnetlib/apps/Oai2ftp/model/CollectionInfo.java b/src/main/java/eu/dnetlib/apps/oai/model/CollectionInfo.java similarity index 81% rename from src/main/java/eu/dnetlib/apps/Oai2ftp/model/CollectionInfo.java rename to src/main/java/eu/dnetlib/apps/oai/model/CollectionInfo.java index cfe4edb..a93cc0b 100644 --- a/src/main/java/eu/dnetlib/apps/Oai2ftp/model/CollectionInfo.java +++ b/src/main/java/eu/dnetlib/apps/oai/model/CollectionInfo.java @@ -1,4 +1,4 @@ -package eu.dnetlib.apps.oai2ftp.model; +package eu.dnetlib.apps.oai.model; import java.io.Serializable; import java.time.LocalDateTime; @@ -40,18 +40,27 @@ public class CollectionInfo implements Serializable { @Column(name = "storage_url", length = 1024) private String storageUrl; + @Column(name = "public_url", length = 1024) + private String publicUrl; + @Column(name = "start_date") private LocalDateTime start; @Column(name = "end_date") private LocalDateTime end; + @Column(name = "max") + private long max = Long.MAX_VALUE; + @Column(name = "status") private ExecutionStatus executionStatus; @Column(name = "total") private long total = 0; + @Column(name = "notification_email") + private String notificationEmail; + @Lob @Column(name = "message") private String message = ""; @@ -115,6 +124,14 @@ public class CollectionInfo implements Serializable { this.storageUrl = storageUrl; } + public String getPublicUrl() { + return publicUrl; + } + + public void setPublicUrl(final String publicUrl) { + this.publicUrl = publicUrl; + } + public LocalDateTime getStart() { return start; } @@ -131,6 +148,14 @@ public class CollectionInfo implements Serializable { this.end = end; } + public long getMax() { + return max; + } + + public void setMax(final long max) { + this.max = max; + } + public ExecutionStatus getExecutionStatus() { return executionStatus; } @@ -147,6 +172,14 @@ public class CollectionInfo implements Serializable { this.total = total; } + public String getNotificationEmail() { + return notificationEmail; + } + + public void setNotificationEmail(final String notificationEmail) { + this.notificationEmail = notificationEmail; + } + public String getMessage() { return message; } diff --git a/src/main/java/eu/dnetlib/apps/Oai2ftp/model/ExecutionStatus.java b/src/main/java/eu/dnetlib/apps/oai/model/ExecutionStatus.java similarity index 64% rename from src/main/java/eu/dnetlib/apps/Oai2ftp/model/ExecutionStatus.java rename to src/main/java/eu/dnetlib/apps/oai/model/ExecutionStatus.java index f58a3a3..10b331e 100644 --- a/src/main/java/eu/dnetlib/apps/Oai2ftp/model/ExecutionStatus.java +++ b/src/main/java/eu/dnetlib/apps/oai/model/ExecutionStatus.java @@ -1,4 +1,4 @@ -package eu.dnetlib.apps.oai2ftp.model; +package eu.dnetlib.apps.oai.model; public enum ExecutionStatus { READY, diff --git a/src/main/java/eu/dnetlib/apps/Oai2ftp/repository/CollectionInfoRepository.java b/src/main/java/eu/dnetlib/apps/oai/repository/CollectionInfoRepository.java similarity index 61% rename from src/main/java/eu/dnetlib/apps/Oai2ftp/repository/CollectionInfoRepository.java rename to src/main/java/eu/dnetlib/apps/oai/repository/CollectionInfoRepository.java index 3f48760..96ade73 100644 --- a/src/main/java/eu/dnetlib/apps/Oai2ftp/repository/CollectionInfoRepository.java +++ b/src/main/java/eu/dnetlib/apps/oai/repository/CollectionInfoRepository.java @@ -1,8 +1,8 @@ -package eu.dnetlib.apps.oai2ftp.repository; +package eu.dnetlib.apps.oai.repository; import org.springframework.data.jpa.repository.JpaRepository; -import eu.dnetlib.apps.oai2ftp.model.CollectionInfo; +import eu.dnetlib.apps.oai.model.CollectionInfo; public interface CollectionInfoRepository extends JpaRepository { diff --git a/src/main/java/eu/dnetlib/apps/Oai2ftp/service/Oai2FtpService.java b/src/main/java/eu/dnetlib/apps/oai/service/CollectorService.java similarity index 58% rename from src/main/java/eu/dnetlib/apps/Oai2ftp/service/Oai2FtpService.java rename to src/main/java/eu/dnetlib/apps/oai/service/CollectorService.java index 8bfae2e..0d59be7 100644 --- a/src/main/java/eu/dnetlib/apps/Oai2ftp/service/Oai2FtpService.java +++ b/src/main/java/eu/dnetlib/apps/oai/service/CollectorService.java @@ -1,4 +1,4 @@ -package eu.dnetlib.apps.oai2ftp.service; +package eu.dnetlib.apps.oai.service; import java.time.Duration; import java.time.LocalDateTime; @@ -23,19 +23,21 @@ import org.springframework.beans.factory.annotation.Value; import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Service; -import eu.dnetlib.apps.oai2ftp.model.CollectionCall; -import eu.dnetlib.apps.oai2ftp.model.CollectionInfo; -import eu.dnetlib.apps.oai2ftp.model.ExecutionStatus; -import eu.dnetlib.apps.oai2ftp.repository.CollectionInfoRepository; -import eu.dnetlib.apps.oai2ftp.utils.HttpFetcher; -import eu.dnetlib.apps.oai2ftp.utils.SimpleUtils; -import eu.dnetlib.apps.oai2ftp.utils.StorageClient; -import eu.dnetlib.apps.oai2ftp.utils.StorageClientFactory; +import eu.dnetlib.apps.oai.model.CollectionCall; +import eu.dnetlib.apps.oai.model.CollectionInfo; +import eu.dnetlib.apps.oai.model.ExecutionStatus; +import eu.dnetlib.apps.oai.repository.CollectionInfoRepository; +import eu.dnetlib.apps.oai.storage.StorageClient; +import eu.dnetlib.apps.oai.storage.StorageClientFactory; +import eu.dnetlib.apps.oai.storage.ZipStorage; +import eu.dnetlib.apps.oai.utils.EmailSender; +import eu.dnetlib.apps.oai.utils.HttpFetcher; +import eu.dnetlib.apps.oai.utils.SimpleUtils; @Service -public class Oai2FtpService { +public class CollectorService { - private static final Log log = LogFactory.getLog(Oai2FtpService.class); + private static final Log log = LogFactory.getLog(CollectorService.class); private final ExecutorService jobExecutor = Executors.newFixedThreadPool(100); @@ -44,22 +46,35 @@ public class Oai2FtpService { @Autowired private StorageClientFactory storageClientFactory; - @Value("${oai2ftp.conf.execution.expirationTime}") + @Value("${oai.conf.execution.expirationTime}") private long fullInfoExpirationTime; // in hours + @Value("${oai.conf.maxRecords}") + private long defaultMaxRecords; + @Autowired private CollectionInfoRepository collectionInfoRepository; + @Value("${oai.conf.storage.basePath}") + private String storageBasePath; + + @Value("${oai.conf.public.basePath}") + private String publicBasePath; + + @Autowired + private EmailSender emailSender; + public CollectionInfo startCollection(final String baseUrl, final String format, final String setSpec, final LocalDateTime from, - final LocalDateTime until) { + final LocalDateTime until, + final Long max, + final String notificationEmail) { final String jobId = SimpleUtils.generateNewJobId(); - final StorageClient sc = storageClientFactory.newClient(); - sc.changeDir(jobId); + final StorageClient sc = storageClientFactory.newClient(jobId); final CollectionInfo info = new CollectionInfo(); info.setId(jobId); @@ -68,23 +83,49 @@ public class Oai2FtpService { info.setOaiSet(setSpec); info.setOaiFrom(from); info.setOaiUntil(until); - info.setStorageUrl(storageClientFactory.getStorageUrlAsString()); + if (sc instanceof ZipStorage) { + info.setStorageUrl(storageBasePath + "/" + jobId + ".zip"); + } else { + info.setStorageUrl(storageBasePath + "/" + jobId); + } + info.setPublicUrl(null); info.setStart(LocalDateTime.now()); info.setExecutionStatus(ExecutionStatus.READY); + if (StringUtils.isNotBlank(notificationEmail)) { + info.setNotificationEmail(notificationEmail); + } + if (max != null && max > 0) { + info.setMax(max); + } else { + info.setMax(defaultMaxRecords); + } + infoMap.put(jobId, info); jobExecutor.execute(() -> { try { info.setExecutionStatus(ExecutionStatus.RUNNING); - oaiCollect(baseUrl, format, setSpec, from, until, sc, info); + oaiCollect(sc, info); info.setExecutionStatus(ExecutionStatus.COMPLETED); + info.setEnd(LocalDateTime.now()); + emailSender.notifySuccess(info); } catch (final Throwable e) { info.setExecutionStatus(ExecutionStatus.FAILED); info.setMessage(e.getMessage() + ": " + ExceptionUtils.getStackTrace(e)); - } finally { info.setEnd(LocalDateTime.now()); - sc.disconnect(); + emailSender.notifyFailure(info); + } finally { + sc.complete(); + + if (StringUtils.isNotBlank(publicBasePath)) { + if (sc instanceof ZipStorage) { + info.setPublicUrl(publicBasePath + "/" + jobId + ".zip"); + } else { + info.setPublicUrl(publicBasePath + "/" + jobId); + } + } + collectionInfoRepository.save(info); } }); @@ -92,18 +133,13 @@ public class Oai2FtpService { return info; } - public void oaiCollect(final String baseUrl, - final String format, - final String setSpec, - final LocalDateTime from, - final LocalDateTime until, - final StorageClient sc, - final CollectionInfo info) - throws Exception { + public void oaiCollect(final StorageClient sc, final CollectionInfo info) throws Exception { - String url = SimpleUtils.oaiFirstUrl(baseUrl, format, setSpec, from, until); + final String baseUrl = info.getOaiBaseUrl(); - int count = 0; + String url = SimpleUtils.oaiFirstUrl(baseUrl, info.getOaiFormat(), info.getOaiSet(), info.getOaiFrom(), info.getOaiUntil()); + + long count = 0; while (StringUtils.isNotBlank(url)) { final CollectionCall call = new CollectionCall(); call.setUrl(url); @@ -115,17 +151,21 @@ public class Oai2FtpService { final List records = doc.selectNodes("//*[local-name()='ListRecords']/*[local-name()='record']"); call.setNumberOfRecords(records.size()); - sc.changeDir(Integer.toString(count++)); + sc.prepareCurrentPage(count++); for (final Node n : records) { - final String id = n.valueOf(".//*[local-name()='header']/*[local-name()='identifier']"); - sc.saveFile(SimpleUtils.oaiIdToFilename(id), n.asXML()); - info.setTotal(info.getTotal() + 1); + if (info.getTotal() < info.getMax()) { + final String id = n.valueOf(".//*[local-name()='header']/*[local-name()='identifier']"); + sc.saveFile(SimpleUtils.oaiIdToFilename(id), n.asXML()); + info.setTotal(info.getTotal() + 1); + } } - sc.changeDir(".."); - final String rtoken = doc.valueOf("//*[local-name()='resumptionToken']").trim(); - - url = SimpleUtils.oaiNextUrl(baseUrl, rtoken); + if (info.getTotal() < info.getMax()) { + final String rtoken = doc.valueOf("//*[local-name()='resumptionToken']").trim(); + url = SimpleUtils.oaiNextUrl(baseUrl, rtoken); + } else { + url = ""; + } } } @@ -168,4 +208,8 @@ public class Oai2FtpService { toDelete.forEach(infoMap::remove); } + public Map history() { + return infoMap; + } + } diff --git a/src/main/java/eu/dnetlib/apps/Oai2ftp/utils/FtpStorage.java b/src/main/java/eu/dnetlib/apps/oai/storage/FtpStorage.java similarity index 85% rename from src/main/java/eu/dnetlib/apps/Oai2ftp/utils/FtpStorage.java rename to src/main/java/eu/dnetlib/apps/oai/storage/FtpStorage.java index c71452b..3668e42 100644 --- a/src/main/java/eu/dnetlib/apps/Oai2ftp/utils/FtpStorage.java +++ b/src/main/java/eu/dnetlib/apps/oai/storage/FtpStorage.java @@ -1,4 +1,4 @@ -package eu.dnetlib.apps.oai2ftp.utils; +package eu.dnetlib.apps.oai.storage; import java.io.ByteArrayInputStream; import java.io.IOException; @@ -15,6 +15,8 @@ public class FtpStorage implements StorageClient { private final FTPClient ftp; + private long currentPage = -1; + public FtpStorage(final String server, final int port, final boolean secure) { this.ftp = ftpConnect(server, port, secure); } @@ -41,7 +43,7 @@ public class FtpStorage implements StorageClient { } @Override - public void disconnect() { + public void complete() { if (ftp != null && ftp.isConnected()) { try { ftp.disconnect(); @@ -63,13 +65,27 @@ public class FtpStorage implements StorageClient { log.info("Ftp logged"); } catch (final IOException e) { log.error("Ftp Login Failed", e); - disconnect(); + complete(); throw new RuntimeException(e); } } @Override - public boolean changeDir(final String dir) { + public void init(final String baseDir, final String jobId) { + changeDir(baseDir); + changeDir(jobId); + } + + @Override + public void prepareCurrentPage(final long page) { + if (page != this.currentPage) { + this.currentPage = page; + changeDir(".."); + changeDir(Long.toString(page)); + } + } + + private boolean changeDir(final String dir) { try { if (!ftp.changeWorkingDirectory(dir)) { ftp.makeDirectory(dir); @@ -78,7 +94,7 @@ public class FtpStorage implements StorageClient { return true; } catch (final IOException e) { log.error("Error changing or create dir: " + dir); - disconnect(); + complete(); throw new RuntimeException("Error changing or create dir: " + dir, e); } } diff --git a/src/main/java/eu/dnetlib/apps/Oai2ftp/utils/LocalStorage.java b/src/main/java/eu/dnetlib/apps/oai/storage/LocalStorage.java similarity index 52% rename from src/main/java/eu/dnetlib/apps/Oai2ftp/utils/LocalStorage.java rename to src/main/java/eu/dnetlib/apps/oai/storage/LocalStorage.java index c10d823..88b3769 100644 --- a/src/main/java/eu/dnetlib/apps/Oai2ftp/utils/LocalStorage.java +++ b/src/main/java/eu/dnetlib/apps/oai/storage/LocalStorage.java @@ -1,4 +1,4 @@ -package eu.dnetlib.apps.oai2ftp.utils; +package eu.dnetlib.apps.oai.storage; import java.io.File; import java.io.FileWriter; @@ -13,29 +13,41 @@ public class LocalStorage implements StorageClient { private static final Log log = LogFactory.getLog(LocalStorage.class); + private long currentPage = -1; + + private String rootDir; + @Override public void login(final String user, final String password) {} @Override - public void disconnect() {} - - public String currDir = "/tmp"; + public void complete() {} @Override - public boolean changeDir(final String dir) { + public void init(final String baseDir, final String jobId) { try { - final File d = new File(dir.startsWith("/") ? dir : currDir + "/" + dir); + final File d = new File(baseDir + "/" + jobId); FileUtils.forceMkdir(d); - currDir = d.getAbsolutePath(); - return true; + this.rootDir = d.getAbsolutePath(); } catch (final IOException e) { - return false; + throw new RuntimeException("Errore creating root dir", e); + } + } + + @Override + public void prepareCurrentPage(final long page) { + try { + final File d = new File(rootDir + "/" + page); + FileUtils.forceMkdir(d); + this.currentPage = page; + } catch (final IOException e) { + throw new RuntimeException("Errore creating page", e); } } @Override public void saveFile(final String filename, final String body) { - try (FileWriter fw = new FileWriter(currDir + "/" + filename)) { + try (FileWriter fw = new FileWriter(this.rootDir + "/" + this.currentPage + "/" + filename)) { IOUtils.write(body, fw); } catch (final IOException e) { log.error("Error saving info file"); diff --git a/src/main/java/eu/dnetlib/apps/oai/storage/StorageClient.java b/src/main/java/eu/dnetlib/apps/oai/storage/StorageClient.java new file mode 100644 index 0000000..eb78d39 --- /dev/null +++ b/src/main/java/eu/dnetlib/apps/oai/storage/StorageClient.java @@ -0,0 +1,15 @@ +package eu.dnetlib.apps.oai.storage; + +public interface StorageClient { + + void login(final String user, final String password); + + void complete(); + + void init(String baseDir, String jobId); + + void prepareCurrentPage(long page); + + void saveFile(String filename, String body); + +} diff --git a/src/main/java/eu/dnetlib/apps/Oai2ftp/utils/StorageClientFactory.java b/src/main/java/eu/dnetlib/apps/oai/storage/StorageClientFactory.java similarity index 57% rename from src/main/java/eu/dnetlib/apps/Oai2ftp/utils/StorageClientFactory.java rename to src/main/java/eu/dnetlib/apps/oai/storage/StorageClientFactory.java index 2475bb4..115f6cf 100644 --- a/src/main/java/eu/dnetlib/apps/Oai2ftp/utils/StorageClientFactory.java +++ b/src/main/java/eu/dnetlib/apps/oai/storage/StorageClientFactory.java @@ -1,4 +1,4 @@ -package eu.dnetlib.apps.oai2ftp.utils; +package eu.dnetlib.apps.oai.storage; import java.net.URI; @@ -8,21 +8,21 @@ import org.springframework.stereotype.Component; @Component public class StorageClientFactory { - @Value("${oai2ftp.conf.storage.url}") - private URI storageUrl; + @Value("${oai.conf.storage.basePath}") + private URI storageBasePath; - @Value("${oai2ftp.conf.storage.user}") + @Value("${oai.conf.storage.user}") private String storageUser; - @Value("${oai2ftp.conf.storage.password}") + @Value("${oai.conf.storage.password}") private String storagePassword; - public StorageClient newClient() { + public StorageClient newClient(final String jobId) { - final String protocol = storageUrl.getScheme(); - final String host = storageUrl.getHost(); - final int port = storageUrl.getPort(); - final String path = storageUrl.getPath(); + final String protocol = storageBasePath.getScheme(); + final String host = storageBasePath.getHost(); + final int port = storageBasePath.getPort(); + final String path = storageBasePath.getPath(); StorageClient client; if (protocol.equalsIgnoreCase("ftp")) { @@ -31,17 +31,15 @@ public class StorageClientFactory { client = new FtpStorage(host, port, true); } else if (protocol.equalsIgnoreCase("file")) { client = new LocalStorage(); + } else if (protocol.equalsIgnoreCase("zip")) { + client = new ZipStorage(); } else { throw new RuntimeException("Invalid storage protocol: " + protocol + " (valid protocol are: file, ftp and ftps)"); } client.login(storageUser, storagePassword); - client.changeDir(path); + client.init(path, jobId); return client; } - public String getStorageUrlAsString() { - return storageUrl.toString(); - } - } diff --git a/src/main/java/eu/dnetlib/apps/oai/storage/ZipStorage.java b/src/main/java/eu/dnetlib/apps/oai/storage/ZipStorage.java new file mode 100644 index 0000000..af7dc15 --- /dev/null +++ b/src/main/java/eu/dnetlib/apps/oai/storage/ZipStorage.java @@ -0,0 +1,74 @@ +package eu.dnetlib.apps.oai.storage; + +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.util.zip.ZipEntry; +import java.util.zip.ZipOutputStream; + +import org.apache.commons.io.FileUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; + +public class ZipStorage implements StorageClient { + + private static final Log log = LogFactory.getLog(ZipStorage.class); + + private FileOutputStream fos; + private ZipOutputStream zipOut; + private long currPage = -1; + + @Override + public void login(final String user, final String password) {} + + @Override + public void complete() { + try { + if (zipOut != null) { + zipOut.close(); + } + if (fos != null) { + fos.close(); + } + } catch (final IOException e) { + log.error("Ftp Disconnection Failed"); + throw new RuntimeException(e); + } + } + + @Override + public void init(final String baseDir, final String jobId) { + try { + final File rootDir = new File(baseDir); + FileUtils.forceMkdir(rootDir); + + this.fos = new FileOutputStream(rootDir.getAbsolutePath() + "/" + jobId + ".zip"); + this.zipOut = new ZipOutputStream(fos); + } catch (final IOException e) { + throw new RuntimeException("Error preparing zip", e); + } + } + + @Override + public void prepareCurrentPage(final long page) { + try { + zipOut.putNextEntry(new ZipEntry(page + "/")); + zipOut.closeEntry(); + this.currPage = page; + } catch (final IOException e) { + throw new RuntimeException("Error adding a directory to zip", e); + } + + } + + @Override + public void saveFile(final String filename, final String body) { + try { + zipOut.putNextEntry(new ZipEntry(new ZipEntry(this.currPage + "/" + filename))); + zipOut.write(body.getBytes()); + } catch (final IOException e) { + throw new RuntimeException("Error adding a file to zip", e); + } + } + +} diff --git a/src/main/java/eu/dnetlib/apps/oai/utils/EmailSender.java b/src/main/java/eu/dnetlib/apps/oai/utils/EmailSender.java new file mode 100644 index 0000000..9299743 --- /dev/null +++ b/src/main/java/eu/dnetlib/apps/oai/utils/EmailSender.java @@ -0,0 +1,37 @@ +package eu.dnetlib.apps.oai.utils; + +import org.apache.commons.lang3.StringUtils; +import org.springframework.stereotype.Service; + +import eu.dnetlib.apps.oai.model.CollectionInfo; + +@Service +public class EmailSender { + + public void notifySuccess(final CollectionInfo info) { + if (StringUtils.isNotBlank(info.getNotificationEmail())) { + sendMail(info.getNotificationEmail(), "OAI Harvesting completed", prepareSuccessMessage(info)); + } + } + + public void notifyFailure(final CollectionInfo info) { + if (StringUtils.isNotBlank(info.getNotificationEmail())) { + sendMail(info.getNotificationEmail(), "OAI Harvesting completed", prepareFailureMessage(info)); + } + } + + private String prepareSuccessMessage(final CollectionInfo info) { + // TODO Auto-generated method stub + return null; + } + + private String prepareFailureMessage(final CollectionInfo info) { + // TODO Auto-generated method stub + return null; + } + + private void sendMail(final String to, final String subject, final String message) { + // TODO Auto-generated method stub + + } +} diff --git a/src/main/java/eu/dnetlib/apps/Oai2ftp/utils/HttpFetcher.java b/src/main/java/eu/dnetlib/apps/oai/utils/HttpFetcher.java similarity index 88% rename from src/main/java/eu/dnetlib/apps/Oai2ftp/utils/HttpFetcher.java rename to src/main/java/eu/dnetlib/apps/oai/utils/HttpFetcher.java index 38188e3..5d994fb 100644 --- a/src/main/java/eu/dnetlib/apps/Oai2ftp/utils/HttpFetcher.java +++ b/src/main/java/eu/dnetlib/apps/oai/utils/HttpFetcher.java @@ -1,4 +1,4 @@ -package eu.dnetlib.apps.oai2ftp.utils; +package eu.dnetlib.apps.oai.utils; import java.io.IOException; @@ -7,8 +7,8 @@ import org.apache.hc.client5.http.impl.classic.CloseableHttpClient; import org.apache.hc.client5.http.impl.classic.HttpClientBuilder; import org.apache.hc.core5.http.io.entity.EntityUtils; -import eu.dnetlib.apps.oai2ftp.model.CollectionCall; -import eu.dnetlib.apps.oai2ftp.model.ExecutionStatus; +import eu.dnetlib.apps.oai.model.CollectionCall; +import eu.dnetlib.apps.oai.model.ExecutionStatus; public class HttpFetcher { diff --git a/src/main/java/eu/dnetlib/apps/Oai2ftp/utils/SimpleUtils.java b/src/main/java/eu/dnetlib/apps/oai/utils/SimpleUtils.java similarity index 97% rename from src/main/java/eu/dnetlib/apps/Oai2ftp/utils/SimpleUtils.java rename to src/main/java/eu/dnetlib/apps/oai/utils/SimpleUtils.java index 056d678..c6337a1 100644 --- a/src/main/java/eu/dnetlib/apps/Oai2ftp/utils/SimpleUtils.java +++ b/src/main/java/eu/dnetlib/apps/oai/utils/SimpleUtils.java @@ -1,4 +1,4 @@ -package eu.dnetlib.apps.oai2ftp.utils; +package eu.dnetlib.apps.oai.utils; import java.io.UnsupportedEncodingException; import java.net.URLEncoder; diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties index 17870a5..e004af8 100644 --- a/src/main/resources/application.properties +++ b/src/main/resources/application.properties @@ -7,15 +7,20 @@ swagger.api_version = 0.0.1 spring.datasource.url=jdbc:h2:mem: spring.datasource.username= spring.datasource.password= -spring.jpa.show-sql=true +spring.jpa.show-sql=false spring.jpa.hibernate.ddl-auto=update spring.jpa.properties.hibernate.format_sql=true spring.jpa.properties.hibernate.dialect=org.hibernate.dialect.H2Dialect -# supported protocols: file, ftp and ftps -# oai2ftp.conf.storage.url = file:///tmp/test_oai -oai2ftp.conf.storage.url = ftp://localhost/oai_dumps -oai2ftp.conf.storage.user = test -oai2ftp.conf.storage.password = testPwd +# supported protocols: file, zip, ftp and ftps +#oai.conf.storage.basePath = file:///tmp/test_oai +oai.conf.storage.basePath = zip:///tmp/test_oai +#oai.conf.storage.basePath = ftp://localhost/oai_dumps +oai.conf.storage.user = test +oai.conf.storage.password = testPwd + +oai.conf.public.basePath = http://localhost/oai_dumps + +oai.conf.execution.expirationTime = 12 +oai.conf.maxRecords = 1000 -oai2ftp.conf.execution.expirationTime = 12 diff --git a/src/test/java/eu/dnetlib/apps/oai2ftp/Oai2ftpApplicationTests.java b/src/test/java/eu/dnetlib/apps/oai/OaiApplicationTests.java similarity index 69% rename from src/test/java/eu/dnetlib/apps/oai2ftp/Oai2ftpApplicationTests.java rename to src/test/java/eu/dnetlib/apps/oai/OaiApplicationTests.java index ed5afed..86ff549 100644 --- a/src/test/java/eu/dnetlib/apps/oai2ftp/Oai2ftpApplicationTests.java +++ b/src/test/java/eu/dnetlib/apps/oai/OaiApplicationTests.java @@ -1,10 +1,10 @@ -package eu.dnetlib.apps.oai2ftp; +package eu.dnetlib.apps.oai; import org.junit.jupiter.api.Test; import org.springframework.boot.test.context.SpringBootTest; @SpringBootTest -class Oai2ftpApplicationTests { +class OaiApplicationTests { @Test void contextLoads() {