partial implementation (controller, jpa, ftp, ...)

.classpath

@@ -0,0 +1,32 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<classpath>
+	<classpathentry kind="src" output="target/classes" path="src/main/java">
+		<attributes>
+			<attribute name="optional" value="true"/>
+			<attribute name="maven.pomderived" value="true"/>
+		</attributes>
+	</classpathentry>
+	<classpathentry excluding="**" kind="src" output="target/classes" path="src/main/resources">
+		<attributes>
+			<attribute name="maven.pomderived" value="true"/>
+		</attributes>
+	</classpathentry>
+	<classpathentry kind="src" output="target/test-classes" path="src/test/java">
+		<attributes>
+			<attribute name="optional" value="true"/>
+			<attribute name="maven.pomderived" value="true"/>
+			<attribute name="test" value="true"/>
+		</attributes>
+	</classpathentry>
+	<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER">
+		<attributes>
+			<attribute name="maven.pomderived" value="true"/>
+		</attributes>
+	</classpathentry>
+	<classpathentry kind="con" path="org.eclipse.m2e.MAVEN2_CLASSPATH_CONTAINER">
+		<attributes>
+			<attribute name="maven.pomderived" value="true"/>
+		</attributes>
+	</classpathentry>
+	<classpathentry kind="output" path="target/classes"/>
+</classpath>

.project

@@ -0,0 +1,23 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<projectDescription>
+	<name>oai2ftp</name>
+	<comment></comment>
+	<projects>
+	</projects>
+	<buildSpec>
+		<buildCommand>
+			<name>org.eclipse.jdt.core.javabuilder</name>
+			<arguments>
+			</arguments>
+		</buildCommand>
+		<buildCommand>
+			<name>org.eclipse.m2e.core.maven2Builder</name>
+			<arguments>
+			</arguments>
+		</buildCommand>
+	</buildSpec>
+	<natures>
+		<nature>org.eclipse.jdt.core.javanature</nature>
+		<nature>org.eclipse.m2e.core.maven2Nature</nature>
+	</natures>
+</projectDescription>

.settings/org.eclipse.core.resources.prefs

@@ -0,0 +1,5 @@
+eclipse.preferences.version=1
+encoding//src/main/java=UTF-8
+encoding//src/main/resources=UTF-8
+encoding//src/test/java=UTF-8
+encoding/<project>=UTF-8

.settings/org.eclipse.jdt.core.prefs

@@ -0,0 +1,9 @@
+eclipse.preferences.version=1
+org.eclipse.jdt.core.compiler.codegen.methodParameters=generate
+org.eclipse.jdt.core.compiler.codegen.targetPlatform=17
+org.eclipse.jdt.core.compiler.compliance=17
+org.eclipse.jdt.core.compiler.problem.enablePreviewFeatures=disabled
+org.eclipse.jdt.core.compiler.problem.forbiddenReference=warning
+org.eclipse.jdt.core.compiler.problem.reportPreviewFeatures=ignore
+org.eclipse.jdt.core.compiler.release=enabled
+org.eclipse.jdt.core.compiler.source=17

.settings/org.eclipse.m2e.core.prefs

@@ -0,0 +1,4 @@
+activeProfiles=
+eclipse.preferences.version=1
+resolveWorkspaceProjects=true
+version=1

pom.xml

@@ -31,6 +31,18 @@
 			<artifactId>h2</artifactId>
 			<scope>runtime</scope>
 		</dependency>
+		
+		<dependency>
+			<groupId>commons-codec</groupId>
+			<artifactId>commons-codec</artifactId>
+		</dependency>
+		
+		<dependency>
+			<groupId>commons-net</groupId>
+			<artifactId>commons-net</artifactId>
+			<version>3.9.0</version>
+		</dependency>
+		
 		<dependency>
 			<groupId>org.springframework.boot</groupId>
 			<artifactId>spring-boot-starter-test</artifactId>

src/main/java/eu/dnetlib/apps/Oai2ftp/Oai2FtpController.java

@@ -0,0 +1,29 @@
+package eu.dnetlib.apps.oai2ftp;
+
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.web.bind.annotation.GetMapping;
+import org.springframework.web.bind.annotation.PathVariable;
+import org.springframework.web.bind.annotation.RequestParam;
+import org.springframework.web.bind.annotation.RestController;
+
+import eu.dnetlib.apps.oai2ftp.model.CollectionStatus;
+import eu.dnetlib.apps.oai2ftp.service.Oai2FtpService;
+
+@RestController
+public class Oai2FtpController {
+
+	@Autowired
+	private Oai2FtpService service;
+
+	@GetMapping("/collect")
+	public CollectionStatus startCollection(@RequestParam final String baseUrl,
+		@RequestParam(required = false, defaultValue = "oai_dc") final String format,
+		@RequestParam(required = false) final String setSpec) {
+		return service.startCollection(baseUrl, format, setSpec);
+	}
+
+	@GetMapping("/status/{id}")
+	public CollectionStatus getExecutionStatus(@PathVariable final String id) {
+		return service.getStatus(id);
+	}
+}

src/main/java/eu/dnetlib/apps/Oai2ftp/Oai2ftpApplication.java

@@ -1,4 +1,4 @@
-package eu.dnetlib.apps.Oai2ftp;
+package eu.dnetlib.apps.oai2ftp;
 
 import org.springframework.boot.SpringApplication;
 import org.springframework.boot.autoconfigure.SpringBootApplication;

src/main/java/eu/dnetlib/apps/Oai2ftp/model/CollectionCall.java

@@ -0,0 +1,60 @@
+package eu.dnetlib.apps.oai2ftp.model;
+
+import java.io.Serializable;
+import java.util.Objects;
+
+public class CollectionCall implements Serializable {
+
+	private static final long serialVersionUID = 4915954425467830605L;
+
+	private String url;
+	private ExecutionStatus status;
+	private int responseCode;
+	private long savedRecords;
+
+	public String getUrl() {
+		return url;
+	}
+
+	public void setUrl(final String url) {
+		this.url = url;
+	}
+
+	public ExecutionStatus getStatus() {
+		return status;
+	}
+
+	public void setStatus(final ExecutionStatus status) {
+		this.status = status;
+	}
+
+	public int getResponseCode() {
+		return responseCode;
+	}
+
+	public void setResponseCode(final int responseCode) {
+		this.responseCode = responseCode;
+	}
+
+	public long getSavedRecords() {
+		return savedRecords;
+	}
+
+	public void setSavedRecords(final long savedRecords) {
+		this.savedRecords = savedRecords;
+	}
+
+	@Override
+	public int hashCode() {
+		return Objects.hash(url);
+	}
+
+	@Override
+	public boolean equals(final Object obj) {
+		if (this == obj) { return true; }
+		if (obj == null) { return false; }
+		if (getClass() != obj.getClass()) { return false; }
+		final CollectionCall other = (CollectionCall) obj;
+		return Objects.equals(url, other.url);
+	}
+}

src/main/java/eu/dnetlib/apps/Oai2ftp/model/CollectionLogEntry.java

@@ -0,0 +1,128 @@
+package eu.dnetlib.apps.oai2ftp.model;
+
+import java.io.Serializable;
+import java.time.LocalDateTime;
+
+import jakarta.persistence.Column;
+import jakarta.persistence.Entity;
+import jakarta.persistence.Id;
+import jakarta.persistence.Table;
+
+@Entity
+@Table(name = "collection_history")
+public class CollectionLogEntry implements Serializable {
+
+	private static final long serialVersionUID = 5843197167336338295L;
+
+	@Id
+	@Column(name = "id")
+	private String id;
+
+	@Column(name = "base_url")
+	private String baseUrl;
+
+	@Column(name = "format")
+	private String format;
+
+	@Column(name = "set_spec")
+	private String setSpec;
+
+	@Column(name = "start_date")
+	private LocalDateTime start;
+
+	@Column(name = "end_date")
+	private LocalDateTime end;
+
+	@Column(name = "success")
+	private boolean success;
+
+	@Column(name = "total")
+	private long total;
+
+	@Column(name = "n_calls")
+	private long numberOfCalls;
+
+	@Column(name = "message")
+	private String message;
+
+	public String getId() {
+		return id;
+	}
+
+	public void setId(final String id) {
+		this.id = id;
+	}
+
+	public String getBaseUrl() {
+		return baseUrl;
+	}
+
+	public void setBaseUrl(final String baseUrl) {
+		this.baseUrl = baseUrl;
+	}
+
+	public String getFormat() {
+		return format;
+	}
+
+	public void setFormat(final String format) {
+		this.format = format;
+	}
+
+	public String getSetSpec() {
+		return setSpec;
+	}
+
+	public void setSetSpec(final String setSpec) {
+		this.setSpec = setSpec;
+	}
+
+	public LocalDateTime getStart() {
+		return start;
+	}
+
+	public void setStart(final LocalDateTime start) {
+		this.start = start;
+	}
+
+	public LocalDateTime getEnd() {
+		return end;
+	}
+
+	public void setEnd(final LocalDateTime end) {
+		this.end = end;
+	}
+
+	public boolean isSuccess() {
+		return success;
+	}
+
+	public void setSuccess(final boolean success) {
+		this.success = success;
+	}
+
+	public long getTotal() {
+		return total;
+	}
+
+	public void setTotal(final long total) {
+		this.total = total;
+	}
+
+	public long getNumberOfCalls() {
+		return numberOfCalls;
+	}
+
+	public void setNumberOfCalls(final long numberOfCalls) {
+		this.numberOfCalls = numberOfCalls;
+	}
+
+	public String getMessage() {
+		return message;
+	}
+
+	public void setMessage(final String message) {
+		this.message = message;
+	}
+
+}

src/main/java/eu/dnetlib/apps/Oai2ftp/model/CollectionStatus.java

@@ -0,0 +1,98 @@
+package eu.dnetlib.apps.oai2ftp.model;
+
+import java.io.Serializable;
+import java.time.LocalDateTime;
+import java.util.LinkedHashSet;
+
+public class CollectionStatus implements Serializable {
+
+	private static final long serialVersionUID = -8467778040892221645L;
+
+	private String id;
+	private String baseUrl;
+	private String format;
+	private String setSpec;
+	private LocalDateTime start;
+	private LocalDateTime end;
+	private ExecutionStatus status;
+	private long total;
+	private final LinkedHashSet<CollectionCall> calls = new LinkedHashSet<>();
+	private String message;
+
+	public String getId() {
+		return id;
+	}
+
+	public void setId(final String id) {
+		this.id = id;
+	}
+
+	public String getBaseUrl() {
+		return baseUrl;
+	}
+
+	public void setBaseUrl(final String baseUrl) {
+		this.baseUrl = baseUrl;
+	}
+
+	public String getFormat() {
+		return format;
+	}
+
+	public void setFormat(final String format) {
+		this.format = format;
+	}
+
+	public String getSetSpec() {
+		return setSpec;
+	}
+
+	public void setSetSpec(final String setSpec) {
+		this.setSpec = setSpec;
+	}
+
+	public LocalDateTime getStart() {
+		return start;
+	}
+
+	public void setStart(final LocalDateTime start) {
+		this.start = start;
+	}
+
+	public LocalDateTime getEnd() {
+		return end;
+	}
+
+	public void setEnd(final LocalDateTime end) {
+		this.end = end;
+	}
+
+	public ExecutionStatus getStatus() {
+		return status;
+	}
+
+	public void setStatus(final ExecutionStatus status) {
+		this.status = status;
+	}
+
+	public long getTotal() {
+		return total;
+	}
+
+	public void setTotal(final long total) {
+		this.total = total;
+	}
+
+	public LinkedHashSet<CollectionCall> getCalls() {
+		return calls;
+	}
+
+	public String getMessage() {
+		return message;
+	}
+
+	public void setMessage(final String message) {
+		this.message = message;
+	}
+
+}

src/main/java/eu/dnetlib/apps/Oai2ftp/model/ExecutionStatus.java

@@ -0,0 +1,8 @@
+package eu.dnetlib.apps.oai2ftp.model;
+
+public enum ExecutionStatus {
+	READY,
+	RUNNING,
+	COMPLETED,
+	FAILED
+}

src/main/java/eu/dnetlib/apps/Oai2ftp/repository/CollectionLogEntryRepository.java

@@ -0,0 +1,10 @@
+package eu.dnetlib.apps.oai2ftp.repository;
+
+import org.springframework.data.jpa.repository.JpaRepository;
+
+import eu.dnetlib.apps.oai2ftp.model.CollectionLogEntry;
+
+public interface CollectionLogEntryRepository extends JpaRepository<CollectionLogEntry, String> {
+
+
+}

src/main/java/eu/dnetlib/apps/Oai2ftp/service/CollectionJob.java

@@ -0,0 +1,46 @@
+package eu.dnetlib.apps.oai2ftp.service;
+
+import java.time.LocalDateTime;
+import java.util.function.BiConsumer;
+import java.util.function.Consumer;
+
+import eu.dnetlib.apps.oai2ftp.model.CollectionStatus;
+import eu.dnetlib.apps.oai2ftp.model.ExecutionStatus;
+
+public class CollectionJob {
+
+	private final CollectionStatus status;
+	private final BiConsumer<String, String> saveRecord;
+	private final Consumer<CollectionStatus> onEnd;
+
+	public CollectionJob(final String id, final String baseUrl, final String format, final String setSpec, final BiConsumer<String, String> saveRecord,
+		final Consumer<CollectionStatus> onEnd) {
+
+		this.status = new CollectionStatus();
+		status.setId(id);
+
+		status.setBaseUrl(baseUrl);
+		status.setFormat(format);
+		status.setSetSpec(setSpec);
+
+		status.setStart(LocalDateTime.now());
+		status.setEnd(null);
+
+		status.setStatus(ExecutionStatus.READY);
+		status.setTotal(0);
+
+		status.setMessage("");
+
+		this.saveRecord = saveRecord;
+		this.onEnd = onEnd;
+	}
+
+	public void oaiCollect() {
+
+	}
+
+	public CollectionStatus getStatus() {
+		return status;
+	}
+
+}

src/main/java/eu/dnetlib/apps/Oai2ftp/service/Oai2FtpService.java

@@ -0,0 +1,87 @@
+package eu.dnetlib.apps.oai2ftp.service;
+
+import java.util.LinkedHashMap;
+import java.util.Map;
+import java.util.UUID;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.commons.net.ftp.FTPClient;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.stereotype.Service;
+
+import eu.dnetlib.apps.oai2ftp.model.CollectionStatus;
+import eu.dnetlib.apps.oai2ftp.repository.CollectionLogEntryRepository;
+import eu.dnetlib.apps.oai2ftp.utils.ConvertUtils;
+import eu.dnetlib.apps.oai2ftp.utils.FtpUtils;
+
+@Service
+public class Oai2FtpService {
+
+	private static final Log log = LogFactory.getLog(Oai2FtpService.class);
+
+	private final ExecutorService jobExecutor = Executors.newFixedThreadPool(100);
+
+	private final Map<String, CollectionJob> runningJobs = new LinkedHashMap<>();
+
+	@Value("oai2ftp.conf.ftp.server")
+	private String ftpServer;
+
+	@Value("oai2ftp.conf.ftp.user")
+	private String ftpUser;
+
+	@Value("oai2ftp.conf.ftp.password")
+	private String ftpPassword;
+
+	@Value("oai2ftp.conf.ftp.basedir")
+	private String ftpBaseDir;
+
+	private final boolean ftpSecure = false;
+
+	@Autowired
+	private CollectionLogEntryRepository collectionLogEntryRepository;
+
+	public CollectionStatus startCollection(final String baseUrl, final String format, final String setSpec) {
+		final String jobId = generateNewJobId();
+
+		final FTPClient ftp = FtpUtils.ftpConnect(ftpServer, ftpSecure);
+		FtpUtils.ftpLogin(ftp, ftpUser, ftpPassword);
+		FtpUtils.changeDir(ftp, ftpBaseDir);
+		FtpUtils.changeDir(ftp, jobId);
+
+		final CollectionJob job = new CollectionJob(jobId,
+			baseUrl,
+			format,
+			setSpec,
+			(id, body) -> FtpUtils.saveRecord(id, body),
+			(status) -> {
+				FtpUtils.ftpDisconnect(ftp);
+				collectionLogEntryRepository.save(ConvertUtils.statusToLog(status));
+			});
+
+		runningJobs.put(jobId, job);
+
+		jobExecutor.execute(() -> job.oaiCollect());
+
+		return job.getStatus();
+	};
+
+	private String generateNewJobId() {
+		return "job-" + UUID.randomUUID();
+	}
+
+	public CollectionStatus getStatus(final String jobId) {
+		final CollectionJob job = runningJobs.get(jobId);
+		if (job != null) {
+			return job.getStatus();
+		} else {
+			return collectionLogEntryRepository.findById(jobId)
+				.map(ConvertUtils::logToStatus)
+				.orElse(null);
+		}
+	}
+
+}

src/main/java/eu/dnetlib/apps/Oai2ftp/utils/ConvertUtils.java

@@ -0,0 +1,23 @@
+package eu.dnetlib.apps.oai2ftp.utils;
+
+import org.apache.commons.codec.digest.DigestUtils;
+
+import eu.dnetlib.apps.oai2ftp.model.CollectionLogEntry;
+import eu.dnetlib.apps.oai2ftp.model.CollectionStatus;
+
+public class ConvertUtils {
+
+	public static CollectionStatus logToStatus(final CollectionLogEntry log) {
+		// TODO
+		return null;
+	}
+
+	public static CollectionLogEntry statusToLog(final CollectionStatus status) {
+		// TODO
+		return null;
+	}
+
+	public static String oaiIdToFilename(final String id) {
+		return DigestUtils.md5Hex(id) + ".xml";
+	}
+}

src/main/java/eu/dnetlib/apps/Oai2ftp/utils/FtpUtils.java

@@ -0,0 +1,82 @@
+package eu.dnetlib.apps.oai2ftp.utils;
+
+import java.io.IOException;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.commons.net.ftp.FTPClient;
+import org.apache.commons.net.ftp.FTPSClient;
+
+public class FtpUtils {
+
+	private static final Log log = LogFactory.getLog(FtpUtils.class);
+
+	public static FTPClient ftpConnect(final String server, final boolean secure) {
+		try {
+			if (secure) {
+				final FTPSClient ftp = new FTPSClient();
+				ftp.connect(server);
+				// Set protection buffer size
+				ftp.execPBSZ(0);
+				// Set data channel protection to private
+				ftp.execPROT("P");
+				return ftp;
+			} else {
+				final FTPClient ftp = new FTPClient();
+				ftp.connect(server);
+				return ftp;
+			}
+		} catch (final IOException e) {
+			log.error("Ftp Connection Failed", e);
+			throw new RuntimeException(e);
+		}
+	}
+
+	public static void ftpLogin(final FTPClient ftp, final String user, final String password) {
+		try {
+			if (!ftp.login(user, password)) { throw new RuntimeException("FTP login failed"); }
+
+			ftp.setFileType(FTPClient.BINARY_FILE_TYPE);
+			ftp.enterLocalPassiveMode();
+			ftp.setBufferSize(1024);
+
+			log.info("Ftp logged");
+		} catch (final IOException e) {
+			log.error("Ftp Login Failed", e);
+			ftpDisconnect(ftp);
+			throw new RuntimeException(e);
+		}
+	}
+
+	public static void ftpDisconnect(final FTPClient ftp) {
+		if (ftp != null && ftp.isConnected()) {
+			try {
+				ftp.disconnect();
+				log.info("Ftp Disconnected");
+			} catch (final IOException e) {
+				log.error("Ftp Disconnection Failed");
+				throw new RuntimeException(e);
+			}
+		}
+	}
+
+	public static boolean changeDir(final FTPClient ftp, final String dir) {
+		try {
+			if (!ftp.changeWorkingDirectory(dir)) {
+				ftp.makeDirectory(dir);
+				return ftp.changeWorkingDirectory(dir);
+			}
+			return true;
+		} catch (final IOException e) {
+			log.error("Error changing or create dir: " + dir);
+			ftpDisconnect(ftp);
+			throw new RuntimeException("Error changing or create dir: " + dir, e);
+		}
+	}
+
+	public static void saveRecord(final String id, final String body) {
+		// TODO Auto-generated method stub
+
+	}
+
+}

src/main/resources/application.properties

@@ -1 +1,7 @@
-
+spring.datasource.url=jdbc:h2:mem:
+spring.datasource.username=
+spring.datasource.password=
+spring.jpa.show-sql=true
+spring.jpa.hibernate.ddl-auto=update
+spring.jpa.properties.hibernate.format_sql=true
+spring.jpa.properties.hibernate.dialect=org.hibernate.dialect.H2Dialect

src/test/java/eu/dnetlib/apps/oai2ftp/Oai2ftpApplicationTests.java

@@ -1,4 +1,4 @@
-package eu.dnetlib.apps.Oai2ftp;
+package eu.dnetlib.apps.oai2ftp;
 
 import org.junit.jupiter.api.Test;
 import org.springframework.boot.test.context.SpringBootTest;