storage pagination
This commit is contained in:
parent
71a72d0bb2
commit
2fb9299e2d
|
@ -15,14 +15,13 @@ public class CollectionInfo implements Serializable {
|
|||
private String oaiSet;
|
||||
private LocalDateTime oaiFrom;
|
||||
private LocalDateTime oaiUntil;
|
||||
private String ftpServer;
|
||||
private String ftpDir;
|
||||
private String storageUrl;
|
||||
private LocalDateTime start;
|
||||
private LocalDateTime end;
|
||||
private ExecutionStatus executionStatus;
|
||||
private long total;
|
||||
private long total = 0;
|
||||
private final List<CollectionCall> calls = new ArrayList<>();
|
||||
private String message;
|
||||
private String message = "";
|
||||
|
||||
public String getId() {
|
||||
return id;
|
||||
|
@ -72,20 +71,12 @@ public class CollectionInfo implements Serializable {
|
|||
this.oaiUntil = oaiUntil;
|
||||
}
|
||||
|
||||
public String getFtpServer() {
|
||||
return ftpServer;
|
||||
public String getStorageUrl() {
|
||||
return storageUrl;
|
||||
}
|
||||
|
||||
public void setFtpServer(final String ftpServer) {
|
||||
this.ftpServer = ftpServer;
|
||||
}
|
||||
|
||||
public String getFtpDir() {
|
||||
return ftpDir;
|
||||
}
|
||||
|
||||
public void setFtpDir(final String ftpDir) {
|
||||
this.ftpDir = ftpDir;
|
||||
public void setStorageUrl(final String storageUrl) {
|
||||
this.storageUrl = storageUrl;
|
||||
}
|
||||
|
||||
public LocalDateTime getStart() {
|
||||
|
|
|
@ -33,11 +33,8 @@ public class CollectionLogEntry implements Serializable {
|
|||
@Column(name = "oai_until")
|
||||
private LocalDateTime oaiUntil;
|
||||
|
||||
@Column(name = "ftp_server")
|
||||
private String ftpServer;
|
||||
|
||||
@Column(name = "ftp_dir")
|
||||
private String ftpDir;
|
||||
@Column(name = "storage_url")
|
||||
private String storageUrl;
|
||||
|
||||
@Column(name = "start_date")
|
||||
private LocalDateTime start;
|
||||
|
@ -105,20 +102,12 @@ public class CollectionLogEntry implements Serializable {
|
|||
this.oaiUntil = oaiUntil;
|
||||
}
|
||||
|
||||
public String getFtpServer() {
|
||||
return ftpServer;
|
||||
public String getStorageUrl() {
|
||||
return storageUrl;
|
||||
}
|
||||
|
||||
public void setFtpServer(final String ftpServer) {
|
||||
this.ftpServer = ftpServer;
|
||||
}
|
||||
|
||||
public String getFtpDir() {
|
||||
return ftpDir;
|
||||
}
|
||||
|
||||
public void setFtpDir(final String ftpDir) {
|
||||
this.ftpDir = ftpDir;
|
||||
public void setStorageUrl(final String storageUrl) {
|
||||
this.storageUrl = storageUrl;
|
||||
}
|
||||
|
||||
public LocalDateTime getStart() {
|
||||
|
|
|
@ -55,25 +55,22 @@ public class Oai2FtpService {
|
|||
final String setSpec,
|
||||
final LocalDateTime from,
|
||||
final LocalDateTime until) {
|
||||
|
||||
final String jobId = SimpleUtils.generateNewJobId();
|
||||
|
||||
final StorageClient sc = storageClientFactory.newClientForJob(jobId);
|
||||
final StorageClient sc = storageClientFactory.newClient();
|
||||
sc.changeDir(jobId);
|
||||
|
||||
final CollectionInfo info = new CollectionInfo();
|
||||
info.setId(jobId);
|
||||
|
||||
info.setOaiBaseUrl(baseUrl);
|
||||
info.setOaiFormat(format);
|
||||
info.setOaiSet(setSpec);
|
||||
info.setOaiFrom(from);
|
||||
info.setOaiUntil(until);
|
||||
|
||||
info.setStorageUrl(storageClientFactory.getStorageUrlAsString());
|
||||
info.setStart(LocalDateTime.now());
|
||||
info.setEnd(null);
|
||||
|
||||
info.setExecutionStatus(ExecutionStatus.READY);
|
||||
info.setTotal(0);
|
||||
info.setMessage("");
|
||||
|
||||
infoMap.put(jobId, info);
|
||||
|
||||
|
@ -106,6 +103,7 @@ public class Oai2FtpService {
|
|||
|
||||
String url = SimpleUtils.oaiFirstUrl(baseUrl, format, setSpec, from, until);
|
||||
|
||||
int count = 0;
|
||||
while (StringUtils.isNotBlank(url)) {
|
||||
final CollectionCall call = new CollectionCall();
|
||||
call.setUrl(url);
|
||||
|
@ -117,12 +115,13 @@ public class Oai2FtpService {
|
|||
final List<Node> records = doc.selectNodes("//*[local-name()='ListRecords']/*[local-name()='record']");
|
||||
call.setNumberOfRecords(records.size());
|
||||
|
||||
sc.changeDir(Integer.toString(count++));
|
||||
for (final Node n : records) {
|
||||
final String id = n.valueOf(".//*[local-name()='header']/*[local-name()='identifier']");
|
||||
|
||||
sc.saveFile(SimpleUtils.oaiIdToFilename(id), n.asXML());
|
||||
info.setTotal(info.getTotal() + 1);
|
||||
}
|
||||
sc.changeDir("..");
|
||||
|
||||
final String rtoken = doc.valueOf("//*[local-name()='resumptionToken']").trim();
|
||||
|
||||
|
|
|
@ -35,8 +35,8 @@ public class LocalStorage implements StorageClient {
|
|||
|
||||
@Override
|
||||
public void saveFile(final String filename, final String body) {
|
||||
try {
|
||||
IOUtils.write(body, new FileWriter(currDir + "/" + filename));
|
||||
try (FileWriter fw = new FileWriter(currDir + "/" + filename)) {
|
||||
IOUtils.write(body, fw);
|
||||
} catch (final IOException e) {
|
||||
log.error("Error saving info file");
|
||||
throw new RuntimeException("Error saving info file", e);
|
||||
|
|
|
@ -63,8 +63,7 @@ public class SimpleUtils {
|
|||
info.setOaiSet(log.getOaiSet());
|
||||
info.setOaiFrom(log.getOaiFrom());
|
||||
info.setOaiUntil(log.getOaiUntil());
|
||||
info.setFtpServer(log.getFtpServer());
|
||||
info.setFtpDir(log.getFtpDir());
|
||||
info.setStorageUrl(log.getStorageUrl());
|
||||
info.setStart(log.getStart());
|
||||
info.setEnd(log.getEnd());
|
||||
info.setExecutionStatus(log.isSuccess() ? ExecutionStatus.COMPLETED : ExecutionStatus.FAILED);
|
||||
|
@ -81,8 +80,7 @@ public class SimpleUtils {
|
|||
log.setOaiSet(info.getOaiSet());
|
||||
log.setOaiFrom(info.getOaiFrom());
|
||||
log.setOaiUntil(info.getOaiUntil());
|
||||
log.setFtpServer(info.getFtpServer());
|
||||
log.setFtpDir(info.getFtpDir());
|
||||
log.setStorageUrl(info.getStorageUrl());
|
||||
log.setStart(info.getStart());
|
||||
log.setEnd(info.getEnd());
|
||||
log.setSuccess(info.getExecutionStatus() == ExecutionStatus.COMPLETED);
|
||||
|
|
|
@ -17,7 +17,7 @@ public class StorageClientFactory {
|
|||
@Value("${oai2ftp.conf.storage.password}")
|
||||
private String storagePassword;
|
||||
|
||||
public StorageClient newClientForJob(final String jobId) {
|
||||
public StorageClient newClient() {
|
||||
|
||||
final String protocol = storageUrl.getProtocol();
|
||||
final String host = storageUrl.getHost();
|
||||
|
@ -36,9 +36,12 @@ public class StorageClientFactory {
|
|||
}
|
||||
client.login(storageUser, storagePassword);
|
||||
client.changeDir(path);
|
||||
client.changeDir(jobId);
|
||||
|
||||
return client;
|
||||
}
|
||||
|
||||
public String getStorageUrlAsString() {
|
||||
return storageUrl.toString();
|
||||
}
|
||||
|
||||
}
|
||||
|
|
Loading…
Reference in New Issue