diff --git a/.classpath b/.classpath
new file mode 100644
index 0000000..228da75
--- /dev/null
+++ b/.classpath
@@ -0,0 +1,32 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/.project b/.project
new file mode 100644
index 0000000..5b7f2cd
--- /dev/null
+++ b/.project
@@ -0,0 +1,23 @@
+
+
+ oai2ftp
+
+
+
+
+
+ org.eclipse.jdt.core.javabuilder
+
+
+
+
+ org.eclipse.m2e.core.maven2Builder
+
+
+
+
+
+ org.eclipse.jdt.core.javanature
+ org.eclipse.m2e.core.maven2Nature
+
+
diff --git a/.settings/org.eclipse.core.resources.prefs b/.settings/org.eclipse.core.resources.prefs
new file mode 100644
index 0000000..839d647
--- /dev/null
+++ b/.settings/org.eclipse.core.resources.prefs
@@ -0,0 +1,5 @@
+eclipse.preferences.version=1
+encoding//src/main/java=UTF-8
+encoding//src/main/resources=UTF-8
+encoding//src/test/java=UTF-8
+encoding/=UTF-8
diff --git a/.settings/org.eclipse.jdt.core.prefs b/.settings/org.eclipse.jdt.core.prefs
new file mode 100644
index 0000000..5e4ec05
--- /dev/null
+++ b/.settings/org.eclipse.jdt.core.prefs
@@ -0,0 +1,9 @@
+eclipse.preferences.version=1
+org.eclipse.jdt.core.compiler.codegen.methodParameters=generate
+org.eclipse.jdt.core.compiler.codegen.targetPlatform=17
+org.eclipse.jdt.core.compiler.compliance=17
+org.eclipse.jdt.core.compiler.problem.enablePreviewFeatures=disabled
+org.eclipse.jdt.core.compiler.problem.forbiddenReference=warning
+org.eclipse.jdt.core.compiler.problem.reportPreviewFeatures=ignore
+org.eclipse.jdt.core.compiler.release=enabled
+org.eclipse.jdt.core.compiler.source=17
diff --git a/.settings/org.eclipse.m2e.core.prefs b/.settings/org.eclipse.m2e.core.prefs
new file mode 100644
index 0000000..f897a7f
--- /dev/null
+++ b/.settings/org.eclipse.m2e.core.prefs
@@ -0,0 +1,4 @@
+activeProfiles=
+eclipse.preferences.version=1
+resolveWorkspaceProjects=true
+version=1
diff --git a/pom.xml b/pom.xml
index 2faab14..c26aae8 100644
--- a/pom.xml
+++ b/pom.xml
@@ -31,6 +31,18 @@
h2
runtime
+
+
+ commons-codec
+ commons-codec
+
+
+
+ commons-net
+ commons-net
+ 3.9.0
+
+
org.springframework.boot
spring-boot-starter-test
diff --git a/src/main/java/eu/dnetlib/apps/Oai2ftp/Oai2FtpController.java b/src/main/java/eu/dnetlib/apps/Oai2ftp/Oai2FtpController.java
new file mode 100644
index 0000000..d9aff7a
--- /dev/null
+++ b/src/main/java/eu/dnetlib/apps/Oai2ftp/Oai2FtpController.java
@@ -0,0 +1,29 @@
+package eu.dnetlib.apps.oai2ftp;
+
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.web.bind.annotation.GetMapping;
+import org.springframework.web.bind.annotation.PathVariable;
+import org.springframework.web.bind.annotation.RequestParam;
+import org.springframework.web.bind.annotation.RestController;
+
+import eu.dnetlib.apps.oai2ftp.model.CollectionStatus;
+import eu.dnetlib.apps.oai2ftp.service.Oai2FtpService;
+
+@RestController
+public class Oai2FtpController {
+
+ @Autowired
+ private Oai2FtpService service;
+
+ @GetMapping("/collect")
+ public CollectionStatus startCollection(@RequestParam final String baseUrl,
+ @RequestParam(required = false, defaultValue = "oai_dc") final String format,
+ @RequestParam(required = false) final String setSpec) {
+ return service.startCollection(baseUrl, format, setSpec);
+ }
+
+ @GetMapping("/status/{id}")
+ public CollectionStatus getExecutionStatus(@PathVariable final String id) {
+ return service.getStatus(id);
+ }
+}
diff --git a/src/main/java/eu/dnetlib/apps/Oai2ftp/Oai2ftpApplication.java b/src/main/java/eu/dnetlib/apps/Oai2ftp/Oai2ftpApplication.java
index af86229..75089ae 100644
--- a/src/main/java/eu/dnetlib/apps/Oai2ftp/Oai2ftpApplication.java
+++ b/src/main/java/eu/dnetlib/apps/Oai2ftp/Oai2ftpApplication.java
@@ -1,4 +1,4 @@
-package eu.dnetlib.apps.Oai2ftp;
+package eu.dnetlib.apps.oai2ftp;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
diff --git a/src/main/java/eu/dnetlib/apps/Oai2ftp/model/CollectionCall.java b/src/main/java/eu/dnetlib/apps/Oai2ftp/model/CollectionCall.java
new file mode 100644
index 0000000..37132d7
--- /dev/null
+++ b/src/main/java/eu/dnetlib/apps/Oai2ftp/model/CollectionCall.java
@@ -0,0 +1,60 @@
+package eu.dnetlib.apps.oai2ftp.model;
+
+import java.io.Serializable;
+import java.util.Objects;
+
+public class CollectionCall implements Serializable {
+
+ private static final long serialVersionUID = 4915954425467830605L;
+
+ private String url;
+ private ExecutionStatus status;
+ private int responseCode;
+ private long savedRecords;
+
+ public String getUrl() {
+ return url;
+ }
+
+ public void setUrl(final String url) {
+ this.url = url;
+ }
+
+ public ExecutionStatus getStatus() {
+ return status;
+ }
+
+ public void setStatus(final ExecutionStatus status) {
+ this.status = status;
+ }
+
+ public int getResponseCode() {
+ return responseCode;
+ }
+
+ public void setResponseCode(final int responseCode) {
+ this.responseCode = responseCode;
+ }
+
+ public long getSavedRecords() {
+ return savedRecords;
+ }
+
+ public void setSavedRecords(final long savedRecords) {
+ this.savedRecords = savedRecords;
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(url);
+ }
+
+ @Override
+ public boolean equals(final Object obj) {
+ if (this == obj) { return true; }
+ if (obj == null) { return false; }
+ if (getClass() != obj.getClass()) { return false; }
+ final CollectionCall other = (CollectionCall) obj;
+ return Objects.equals(url, other.url);
+ }
+}
diff --git a/src/main/java/eu/dnetlib/apps/Oai2ftp/model/CollectionLogEntry.java b/src/main/java/eu/dnetlib/apps/Oai2ftp/model/CollectionLogEntry.java
new file mode 100644
index 0000000..2281f11
--- /dev/null
+++ b/src/main/java/eu/dnetlib/apps/Oai2ftp/model/CollectionLogEntry.java
@@ -0,0 +1,128 @@
+package eu.dnetlib.apps.oai2ftp.model;
+
+import java.io.Serializable;
+import java.time.LocalDateTime;
+
+import jakarta.persistence.Column;
+import jakarta.persistence.Entity;
+import jakarta.persistence.Id;
+import jakarta.persistence.Table;
+
+@Entity
+@Table(name = "collection_history")
+public class CollectionLogEntry implements Serializable {
+
+ private static final long serialVersionUID = 5843197167336338295L;
+
+ @Id
+ @Column(name = "id")
+ private String id;
+
+ @Column(name = "base_url")
+ private String baseUrl;
+
+ @Column(name = "format")
+ private String format;
+
+ @Column(name = "set_spec")
+ private String setSpec;
+
+ @Column(name = "start_date")
+ private LocalDateTime start;
+
+ @Column(name = "end_date")
+ private LocalDateTime end;
+
+ @Column(name = "success")
+ private boolean success;
+
+ @Column(name = "total")
+ private long total;
+
+ @Column(name = "n_calls")
+ private long numberOfCalls;
+
+ @Column(name = "message")
+ private String message;
+
+ public String getId() {
+ return id;
+ }
+
+ public void setId(final String id) {
+ this.id = id;
+ }
+
+ public String getBaseUrl() {
+ return baseUrl;
+ }
+
+ public void setBaseUrl(final String baseUrl) {
+ this.baseUrl = baseUrl;
+ }
+
+ public String getFormat() {
+ return format;
+ }
+
+ public void setFormat(final String format) {
+ this.format = format;
+ }
+
+ public String getSetSpec() {
+ return setSpec;
+ }
+
+ public void setSetSpec(final String setSpec) {
+ this.setSpec = setSpec;
+ }
+
+ public LocalDateTime getStart() {
+ return start;
+ }
+
+ public void setStart(final LocalDateTime start) {
+ this.start = start;
+ }
+
+ public LocalDateTime getEnd() {
+ return end;
+ }
+
+ public void setEnd(final LocalDateTime end) {
+ this.end = end;
+ }
+
+ public boolean isSuccess() {
+ return success;
+ }
+
+ public void setSuccess(final boolean success) {
+ this.success = success;
+ }
+
+ public long getTotal() {
+ return total;
+ }
+
+ public void setTotal(final long total) {
+ this.total = total;
+ }
+
+ public long getNumberOfCalls() {
+ return numberOfCalls;
+ }
+
+ public void setNumberOfCalls(final long numberOfCalls) {
+ this.numberOfCalls = numberOfCalls;
+ }
+
+ public String getMessage() {
+ return message;
+ }
+
+ public void setMessage(final String message) {
+ this.message = message;
+ }
+
+}
diff --git a/src/main/java/eu/dnetlib/apps/Oai2ftp/model/CollectionStatus.java b/src/main/java/eu/dnetlib/apps/Oai2ftp/model/CollectionStatus.java
new file mode 100644
index 0000000..152f97f
--- /dev/null
+++ b/src/main/java/eu/dnetlib/apps/Oai2ftp/model/CollectionStatus.java
@@ -0,0 +1,98 @@
+package eu.dnetlib.apps.oai2ftp.model;
+
+import java.io.Serializable;
+import java.time.LocalDateTime;
+import java.util.LinkedHashSet;
+
+public class CollectionStatus implements Serializable {
+
+ private static final long serialVersionUID = -8467778040892221645L;
+
+ private String id;
+ private String baseUrl;
+ private String format;
+ private String setSpec;
+ private LocalDateTime start;
+ private LocalDateTime end;
+ private ExecutionStatus status;
+ private long total;
+ private final LinkedHashSet calls = new LinkedHashSet<>();
+ private String message;
+
+ public String getId() {
+ return id;
+ }
+
+ public void setId(final String id) {
+ this.id = id;
+ }
+
+ public String getBaseUrl() {
+ return baseUrl;
+ }
+
+ public void setBaseUrl(final String baseUrl) {
+ this.baseUrl = baseUrl;
+ }
+
+ public String getFormat() {
+ return format;
+ }
+
+ public void setFormat(final String format) {
+ this.format = format;
+ }
+
+ public String getSetSpec() {
+ return setSpec;
+ }
+
+ public void setSetSpec(final String setSpec) {
+ this.setSpec = setSpec;
+ }
+
+ public LocalDateTime getStart() {
+ return start;
+ }
+
+ public void setStart(final LocalDateTime start) {
+ this.start = start;
+ }
+
+ public LocalDateTime getEnd() {
+ return end;
+ }
+
+ public void setEnd(final LocalDateTime end) {
+ this.end = end;
+ }
+
+ public ExecutionStatus getStatus() {
+ return status;
+ }
+
+ public void setStatus(final ExecutionStatus status) {
+ this.status = status;
+ }
+
+ public long getTotal() {
+ return total;
+ }
+
+ public void setTotal(final long total) {
+ this.total = total;
+ }
+
+ public LinkedHashSet getCalls() {
+ return calls;
+ }
+
+ public String getMessage() {
+ return message;
+ }
+
+ public void setMessage(final String message) {
+ this.message = message;
+ }
+
+}
diff --git a/src/main/java/eu/dnetlib/apps/Oai2ftp/model/ExecutionStatus.java b/src/main/java/eu/dnetlib/apps/Oai2ftp/model/ExecutionStatus.java
new file mode 100644
index 0000000..f58a3a3
--- /dev/null
+++ b/src/main/java/eu/dnetlib/apps/Oai2ftp/model/ExecutionStatus.java
@@ -0,0 +1,8 @@
+package eu.dnetlib.apps.oai2ftp.model;
+
+public enum ExecutionStatus {
+ READY,
+ RUNNING,
+ COMPLETED,
+ FAILED
+}
diff --git a/src/main/java/eu/dnetlib/apps/Oai2ftp/repository/CollectionLogEntryRepository.java b/src/main/java/eu/dnetlib/apps/Oai2ftp/repository/CollectionLogEntryRepository.java
new file mode 100644
index 0000000..9ffc1e0
--- /dev/null
+++ b/src/main/java/eu/dnetlib/apps/Oai2ftp/repository/CollectionLogEntryRepository.java
@@ -0,0 +1,10 @@
+package eu.dnetlib.apps.oai2ftp.repository;
+
+import org.springframework.data.jpa.repository.JpaRepository;
+
+import eu.dnetlib.apps.oai2ftp.model.CollectionLogEntry;
+
+public interface CollectionLogEntryRepository extends JpaRepository {
+
+
+}
diff --git a/src/main/java/eu/dnetlib/apps/Oai2ftp/service/CollectionJob.java b/src/main/java/eu/dnetlib/apps/Oai2ftp/service/CollectionJob.java
new file mode 100644
index 0000000..5dd623b
--- /dev/null
+++ b/src/main/java/eu/dnetlib/apps/Oai2ftp/service/CollectionJob.java
@@ -0,0 +1,46 @@
+package eu.dnetlib.apps.oai2ftp.service;
+
+import java.time.LocalDateTime;
+import java.util.function.BiConsumer;
+import java.util.function.Consumer;
+
+import eu.dnetlib.apps.oai2ftp.model.CollectionStatus;
+import eu.dnetlib.apps.oai2ftp.model.ExecutionStatus;
+
+public class CollectionJob {
+
+ private final CollectionStatus status;
+ private final BiConsumer saveRecord;
+ private final Consumer onEnd;
+
+ public CollectionJob(final String id, final String baseUrl, final String format, final String setSpec, final BiConsumer saveRecord,
+ final Consumer onEnd) {
+
+ this.status = new CollectionStatus();
+ status.setId(id);
+
+ status.setBaseUrl(baseUrl);
+ status.setFormat(format);
+ status.setSetSpec(setSpec);
+
+ status.setStart(LocalDateTime.now());
+ status.setEnd(null);
+
+ status.setStatus(ExecutionStatus.READY);
+ status.setTotal(0);
+
+ status.setMessage("");
+
+ this.saveRecord = saveRecord;
+ this.onEnd = onEnd;
+ }
+
+ public void oaiCollect() {
+
+ }
+
+ public CollectionStatus getStatus() {
+ return status;
+ }
+
+}
diff --git a/src/main/java/eu/dnetlib/apps/Oai2ftp/service/Oai2FtpService.java b/src/main/java/eu/dnetlib/apps/Oai2ftp/service/Oai2FtpService.java
new file mode 100644
index 0000000..41445e4
--- /dev/null
+++ b/src/main/java/eu/dnetlib/apps/Oai2ftp/service/Oai2FtpService.java
@@ -0,0 +1,87 @@
+package eu.dnetlib.apps.oai2ftp.service;
+
+import java.util.LinkedHashMap;
+import java.util.Map;
+import java.util.UUID;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.commons.net.ftp.FTPClient;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.stereotype.Service;
+
+import eu.dnetlib.apps.oai2ftp.model.CollectionStatus;
+import eu.dnetlib.apps.oai2ftp.repository.CollectionLogEntryRepository;
+import eu.dnetlib.apps.oai2ftp.utils.ConvertUtils;
+import eu.dnetlib.apps.oai2ftp.utils.FtpUtils;
+
+@Service
+public class Oai2FtpService {
+
+ private static final Log log = LogFactory.getLog(Oai2FtpService.class);
+
+ private final ExecutorService jobExecutor = Executors.newFixedThreadPool(100);
+
+ private final Map runningJobs = new LinkedHashMap<>();
+
+ @Value("oai2ftp.conf.ftp.server")
+ private String ftpServer;
+
+ @Value("oai2ftp.conf.ftp.user")
+ private String ftpUser;
+
+ @Value("oai2ftp.conf.ftp.password")
+ private String ftpPassword;
+
+ @Value("oai2ftp.conf.ftp.basedir")
+ private String ftpBaseDir;
+
+ private final boolean ftpSecure = false;
+
+ @Autowired
+ private CollectionLogEntryRepository collectionLogEntryRepository;
+
+ public CollectionStatus startCollection(final String baseUrl, final String format, final String setSpec) {
+ final String jobId = generateNewJobId();
+
+ final FTPClient ftp = FtpUtils.ftpConnect(ftpServer, ftpSecure);
+ FtpUtils.ftpLogin(ftp, ftpUser, ftpPassword);
+ FtpUtils.changeDir(ftp, ftpBaseDir);
+ FtpUtils.changeDir(ftp, jobId);
+
+ final CollectionJob job = new CollectionJob(jobId,
+ baseUrl,
+ format,
+ setSpec,
+ (id, body) -> FtpUtils.saveRecord(id, body),
+ (status) -> {
+ FtpUtils.ftpDisconnect(ftp);
+ collectionLogEntryRepository.save(ConvertUtils.statusToLog(status));
+ });
+
+ runningJobs.put(jobId, job);
+
+ jobExecutor.execute(() -> job.oaiCollect());
+
+ return job.getStatus();
+ };
+
+ private String generateNewJobId() {
+ return "job-" + UUID.randomUUID();
+ }
+
+ public CollectionStatus getStatus(final String jobId) {
+ final CollectionJob job = runningJobs.get(jobId);
+ if (job != null) {
+ return job.getStatus();
+ } else {
+ return collectionLogEntryRepository.findById(jobId)
+ .map(ConvertUtils::logToStatus)
+ .orElse(null);
+ }
+ }
+
+}
diff --git a/src/main/java/eu/dnetlib/apps/Oai2ftp/utils/ConvertUtils.java b/src/main/java/eu/dnetlib/apps/Oai2ftp/utils/ConvertUtils.java
new file mode 100644
index 0000000..c155e9d
--- /dev/null
+++ b/src/main/java/eu/dnetlib/apps/Oai2ftp/utils/ConvertUtils.java
@@ -0,0 +1,23 @@
+package eu.dnetlib.apps.oai2ftp.utils;
+
+import org.apache.commons.codec.digest.DigestUtils;
+
+import eu.dnetlib.apps.oai2ftp.model.CollectionLogEntry;
+import eu.dnetlib.apps.oai2ftp.model.CollectionStatus;
+
+public class ConvertUtils {
+
+ public static CollectionStatus logToStatus(final CollectionLogEntry log) {
+ // TODO
+ return null;
+ }
+
+ public static CollectionLogEntry statusToLog(final CollectionStatus status) {
+ // TODO
+ return null;
+ }
+
+ public static String oaiIdToFilename(final String id) {
+ return DigestUtils.md5Hex(id) + ".xml";
+ }
+}
diff --git a/src/main/java/eu/dnetlib/apps/Oai2ftp/utils/FtpUtils.java b/src/main/java/eu/dnetlib/apps/Oai2ftp/utils/FtpUtils.java
new file mode 100644
index 0000000..f1ba303
--- /dev/null
+++ b/src/main/java/eu/dnetlib/apps/Oai2ftp/utils/FtpUtils.java
@@ -0,0 +1,82 @@
+package eu.dnetlib.apps.oai2ftp.utils;
+
+import java.io.IOException;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.commons.net.ftp.FTPClient;
+import org.apache.commons.net.ftp.FTPSClient;
+
+public class FtpUtils {
+
+ private static final Log log = LogFactory.getLog(FtpUtils.class);
+
+ public static FTPClient ftpConnect(final String server, final boolean secure) {
+ try {
+ if (secure) {
+ final FTPSClient ftp = new FTPSClient();
+ ftp.connect(server);
+ // Set protection buffer size
+ ftp.execPBSZ(0);
+ // Set data channel protection to private
+ ftp.execPROT("P");
+ return ftp;
+ } else {
+ final FTPClient ftp = new FTPClient();
+ ftp.connect(server);
+ return ftp;
+ }
+ } catch (final IOException e) {
+ log.error("Ftp Connection Failed", e);
+ throw new RuntimeException(e);
+ }
+ }
+
+ public static void ftpLogin(final FTPClient ftp, final String user, final String password) {
+ try {
+ if (!ftp.login(user, password)) { throw new RuntimeException("FTP login failed"); }
+
+ ftp.setFileType(FTPClient.BINARY_FILE_TYPE);
+ ftp.enterLocalPassiveMode();
+ ftp.setBufferSize(1024);
+
+ log.info("Ftp logged");
+ } catch (final IOException e) {
+ log.error("Ftp Login Failed", e);
+ ftpDisconnect(ftp);
+ throw new RuntimeException(e);
+ }
+ }
+
+ public static void ftpDisconnect(final FTPClient ftp) {
+ if (ftp != null && ftp.isConnected()) {
+ try {
+ ftp.disconnect();
+ log.info("Ftp Disconnected");
+ } catch (final IOException e) {
+ log.error("Ftp Disconnection Failed");
+ throw new RuntimeException(e);
+ }
+ }
+ }
+
+ public static boolean changeDir(final FTPClient ftp, final String dir) {
+ try {
+ if (!ftp.changeWorkingDirectory(dir)) {
+ ftp.makeDirectory(dir);
+ return ftp.changeWorkingDirectory(dir);
+ }
+ return true;
+ } catch (final IOException e) {
+ log.error("Error changing or create dir: " + dir);
+ ftpDisconnect(ftp);
+ throw new RuntimeException("Error changing or create dir: " + dir, e);
+ }
+ }
+
+ public static void saveRecord(final String id, final String body) {
+ // TODO Auto-generated method stub
+
+ }
+
+}
diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties
index 8b13789..ce33a3b 100644
--- a/src/main/resources/application.properties
+++ b/src/main/resources/application.properties
@@ -1 +1,7 @@
-
+spring.datasource.url=jdbc:h2:mem:
+spring.datasource.username=
+spring.datasource.password=
+spring.jpa.show-sql=true
+spring.jpa.hibernate.ddl-auto=update
+spring.jpa.properties.hibernate.format_sql=true
+spring.jpa.properties.hibernate.dialect=org.hibernate.dialect.H2Dialect
diff --git a/src/test/java/eu/dnetlib/apps/Oai2ftp/Oai2ftpApplicationTests.java b/src/test/java/eu/dnetlib/apps/oai2ftp/Oai2ftpApplicationTests.java
similarity index 84%
rename from src/test/java/eu/dnetlib/apps/Oai2ftp/Oai2ftpApplicationTests.java
rename to src/test/java/eu/dnetlib/apps/oai2ftp/Oai2ftpApplicationTests.java
index 54dd2be..ed5afed 100644
--- a/src/test/java/eu/dnetlib/apps/Oai2ftp/Oai2ftpApplicationTests.java
+++ b/src/test/java/eu/dnetlib/apps/oai2ftp/Oai2ftpApplicationTests.java
@@ -1,4 +1,4 @@
-package eu.dnetlib.apps.Oai2ftp;
+package eu.dnetlib.apps.oai2ftp;
import org.junit.jupiter.api.Test;
import org.springframework.boot.test.context.SpringBootTest;