michele.artini
/
simpleOaiCollectorService
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

storage pagination

This commit is contained in:
Michele Artini 2023-05-26 15:53:42 +02:00
parent 71a72d0bb2
commit 2fb9299e2d
6 changed files with 29 additions and 49 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
src/main/java/eu/dnetlib/apps/Oai2ftp/model/CollectionInfo.java
View File

@ -15,14 +15,13 @@ public class CollectionInfo implements Serializable {
private String oaiSet;
private LocalDateTime oaiFrom;
private LocalDateTime oaiUntil;
private String ftpServer;
private String ftpDir;
private String storageUrl;
private LocalDateTime start;
private LocalDateTime end;
private ExecutionStatus executionStatus;
private long total;
private long total = 0;
private final List<CollectionCall> calls = new ArrayList<>();
private String message;
private String message = "";
public String getId() {
return id;
@ -72,20 +71,12 @@ public class CollectionInfo implements Serializable {
this.oaiUntil = oaiUntil;
}
public String getFtpServer() {
return ftpServer;
public String getStorageUrl() {
return storageUrl;
}
public void setFtpServer(final String ftpServer) {
this.ftpServer = ftpServer;
}
public String getFtpDir() {
return ftpDir;
}
public void setFtpDir(final String ftpDir) {
this.ftpDir = ftpDir;
public void setStorageUrl(final String storageUrl) {
this.storageUrl = storageUrl;
}
public LocalDateTime getStart() {

23
src/main/java/eu/dnetlib/apps/Oai2ftp/model/CollectionLogEntry.java
View File

@ -33,11 +33,8 @@ public class CollectionLogEntry implements Serializable {
@Column(name = "oai_until")
private LocalDateTime oaiUntil;
@Column(name = "ftp_server")
private String ftpServer;
@Column(name = "ftp_dir")
private String ftpDir;
@Column(name = "storage_url")
private String storageUrl;
@Column(name = "start_date")
private LocalDateTime start;
@ -105,20 +102,12 @@ public class CollectionLogEntry implements Serializable {
this.oaiUntil = oaiUntil;
}
public String getFtpServer() {
return ftpServer;
public String getStorageUrl() {
return storageUrl;
}
public void setFtpServer(final String ftpServer) {
this.ftpServer = ftpServer;
}
public String getFtpDir() {
return ftpDir;
}
public void setFtpDir(final String ftpDir) {
this.ftpDir = ftpDir;
public void setStorageUrl(final String storageUrl) {
this.storageUrl = storageUrl;
}
public LocalDateTime getStart() {

15
src/main/java/eu/dnetlib/apps/Oai2ftp/service/Oai2FtpService.java
View File

@ -55,25 +55,22 @@ public class Oai2FtpService {
final String setSpec,
final LocalDateTime from,
final LocalDateTime until) {
final String jobId = SimpleUtils.generateNewJobId();
final StorageClient sc = storageClientFactory.newClientForJob(jobId);
final StorageClient sc = storageClientFactory.newClient();
sc.changeDir(jobId);
final CollectionInfo info = new CollectionInfo();
info.setId(jobId);
info.setOaiBaseUrl(baseUrl);
info.setOaiFormat(format);
info.setOaiSet(setSpec);
info.setOaiFrom(from);
info.setOaiUntil(until);
info.setStorageUrl(storageClientFactory.getStorageUrlAsString());
info.setStart(LocalDateTime.now());
info.setEnd(null);
info.setExecutionStatus(ExecutionStatus.READY);
info.setTotal(0);
info.setMessage("");
infoMap.put(jobId, info);
@ -106,6 +103,7 @@ public class Oai2FtpService {
String url = SimpleUtils.oaiFirstUrl(baseUrl, format, setSpec, from, until);
int count = 0;
while (StringUtils.isNotBlank(url)) {
final CollectionCall call = new CollectionCall();
call.setUrl(url);
@ -117,12 +115,13 @@ public class Oai2FtpService {
final List<Node> records = doc.selectNodes("//*[local-name()='ListRecords']/*[local-name()='record']");
call.setNumberOfRecords(records.size());
sc.changeDir(Integer.toString(count++));
for (final Node n : records) {
final String id = n.valueOf(".//*[local-name()='header']/*[local-name()='identifier']");
sc.saveFile(SimpleUtils.oaiIdToFilename(id), n.asXML());
info.setTotal(info.getTotal() + 1);
}
sc.changeDir("..");
final String rtoken = doc.valueOf("//*[local-name()='resumptionToken']").trim();

4
src/main/java/eu/dnetlib/apps/Oai2ftp/utils/LocalStorage.java
View File

@ -35,8 +35,8 @@ public class LocalStorage implements StorageClient {
@Override
public void saveFile(final String filename, final String body) {
try {
IOUtils.write(body, new FileWriter(currDir + "/" + filename));
try (FileWriter fw = new FileWriter(currDir + "/" + filename)) {
IOUtils.write(body, fw);
} catch (final IOException e) {
log.error("Error saving info file");
throw new RuntimeException("Error saving info file", e);

6
src/main/java/eu/dnetlib/apps/Oai2ftp/utils/SimpleUtils.java
View File

@ -63,8 +63,7 @@ public class SimpleUtils {
info.setOaiSet(log.getOaiSet());
info.setOaiFrom(log.getOaiFrom());
info.setOaiUntil(log.getOaiUntil());
info.setFtpServer(log.getFtpServer());
info.setFtpDir(log.getFtpDir());
info.setStorageUrl(log.getStorageUrl());
info.setStart(log.getStart());
info.setEnd(log.getEnd());
info.setExecutionStatus(log.isSuccess() ? ExecutionStatus.COMPLETED : ExecutionStatus.FAILED);
@ -81,8 +80,7 @@ public class SimpleUtils {
log.setOaiSet(info.getOaiSet());
log.setOaiFrom(info.getOaiFrom());
log.setOaiUntil(info.getOaiUntil());
log.setFtpServer(info.getFtpServer());
log.setFtpDir(info.getFtpDir());
log.setStorageUrl(info.getStorageUrl());
log.setStart(info.getStart());
log.setEnd(info.getEnd());
log.setSuccess(info.getExecutionStatus() == ExecutionStatus.COMPLETED);

7
src/main/java/eu/dnetlib/apps/Oai2ftp/utils/StorageClientFactory.java
View File

@ -17,7 +17,7 @@ public class StorageClientFactory {
@Value("${oai2ftp.conf.storage.password}")
private String storagePassword;
public StorageClient newClientForJob(final String jobId) {
public StorageClient newClient() {
final String protocol = storageUrl.getProtocol();
final String host = storageUrl.getHost();
@ -36,9 +36,12 @@ public class StorageClientFactory {
}
client.login(storageUser, storagePassword);
client.changeDir(path);
client.changeDir(jobId);
return client;
}
public String getStorageUrlAsString() {
return storageUrl.toString();
}
}