storage pagination
This commit is contained in:
parent
71a72d0bb2
commit
2fb9299e2d
|
@ -15,14 +15,13 @@ public class CollectionInfo implements Serializable {
|
||||||
private String oaiSet;
|
private String oaiSet;
|
||||||
private LocalDateTime oaiFrom;
|
private LocalDateTime oaiFrom;
|
||||||
private LocalDateTime oaiUntil;
|
private LocalDateTime oaiUntil;
|
||||||
private String ftpServer;
|
private String storageUrl;
|
||||||
private String ftpDir;
|
|
||||||
private LocalDateTime start;
|
private LocalDateTime start;
|
||||||
private LocalDateTime end;
|
private LocalDateTime end;
|
||||||
private ExecutionStatus executionStatus;
|
private ExecutionStatus executionStatus;
|
||||||
private long total;
|
private long total = 0;
|
||||||
private final List<CollectionCall> calls = new ArrayList<>();
|
private final List<CollectionCall> calls = new ArrayList<>();
|
||||||
private String message;
|
private String message = "";
|
||||||
|
|
||||||
public String getId() {
|
public String getId() {
|
||||||
return id;
|
return id;
|
||||||
|
@ -72,20 +71,12 @@ public class CollectionInfo implements Serializable {
|
||||||
this.oaiUntil = oaiUntil;
|
this.oaiUntil = oaiUntil;
|
||||||
}
|
}
|
||||||
|
|
||||||
public String getFtpServer() {
|
public String getStorageUrl() {
|
||||||
return ftpServer;
|
return storageUrl;
|
||||||
}
|
}
|
||||||
|
|
||||||
public void setFtpServer(final String ftpServer) {
|
public void setStorageUrl(final String storageUrl) {
|
||||||
this.ftpServer = ftpServer;
|
this.storageUrl = storageUrl;
|
||||||
}
|
|
||||||
|
|
||||||
public String getFtpDir() {
|
|
||||||
return ftpDir;
|
|
||||||
}
|
|
||||||
|
|
||||||
public void setFtpDir(final String ftpDir) {
|
|
||||||
this.ftpDir = ftpDir;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
public LocalDateTime getStart() {
|
public LocalDateTime getStart() {
|
||||||
|
|
|
@ -33,11 +33,8 @@ public class CollectionLogEntry implements Serializable {
|
||||||
@Column(name = "oai_until")
|
@Column(name = "oai_until")
|
||||||
private LocalDateTime oaiUntil;
|
private LocalDateTime oaiUntil;
|
||||||
|
|
||||||
@Column(name = "ftp_server")
|
@Column(name = "storage_url")
|
||||||
private String ftpServer;
|
private String storageUrl;
|
||||||
|
|
||||||
@Column(name = "ftp_dir")
|
|
||||||
private String ftpDir;
|
|
||||||
|
|
||||||
@Column(name = "start_date")
|
@Column(name = "start_date")
|
||||||
private LocalDateTime start;
|
private LocalDateTime start;
|
||||||
|
@ -105,20 +102,12 @@ public class CollectionLogEntry implements Serializable {
|
||||||
this.oaiUntil = oaiUntil;
|
this.oaiUntil = oaiUntil;
|
||||||
}
|
}
|
||||||
|
|
||||||
public String getFtpServer() {
|
public String getStorageUrl() {
|
||||||
return ftpServer;
|
return storageUrl;
|
||||||
}
|
}
|
||||||
|
|
||||||
public void setFtpServer(final String ftpServer) {
|
public void setStorageUrl(final String storageUrl) {
|
||||||
this.ftpServer = ftpServer;
|
this.storageUrl = storageUrl;
|
||||||
}
|
|
||||||
|
|
||||||
public String getFtpDir() {
|
|
||||||
return ftpDir;
|
|
||||||
}
|
|
||||||
|
|
||||||
public void setFtpDir(final String ftpDir) {
|
|
||||||
this.ftpDir = ftpDir;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
public LocalDateTime getStart() {
|
public LocalDateTime getStart() {
|
||||||
|
|
|
@ -55,25 +55,22 @@ public class Oai2FtpService {
|
||||||
final String setSpec,
|
final String setSpec,
|
||||||
final LocalDateTime from,
|
final LocalDateTime from,
|
||||||
final LocalDateTime until) {
|
final LocalDateTime until) {
|
||||||
|
|
||||||
final String jobId = SimpleUtils.generateNewJobId();
|
final String jobId = SimpleUtils.generateNewJobId();
|
||||||
|
|
||||||
final StorageClient sc = storageClientFactory.newClientForJob(jobId);
|
final StorageClient sc = storageClientFactory.newClient();
|
||||||
|
sc.changeDir(jobId);
|
||||||
|
|
||||||
final CollectionInfo info = new CollectionInfo();
|
final CollectionInfo info = new CollectionInfo();
|
||||||
info.setId(jobId);
|
info.setId(jobId);
|
||||||
|
|
||||||
info.setOaiBaseUrl(baseUrl);
|
info.setOaiBaseUrl(baseUrl);
|
||||||
info.setOaiFormat(format);
|
info.setOaiFormat(format);
|
||||||
info.setOaiSet(setSpec);
|
info.setOaiSet(setSpec);
|
||||||
info.setOaiFrom(from);
|
info.setOaiFrom(from);
|
||||||
info.setOaiUntil(until);
|
info.setOaiUntil(until);
|
||||||
|
info.setStorageUrl(storageClientFactory.getStorageUrlAsString());
|
||||||
info.setStart(LocalDateTime.now());
|
info.setStart(LocalDateTime.now());
|
||||||
info.setEnd(null);
|
|
||||||
|
|
||||||
info.setExecutionStatus(ExecutionStatus.READY);
|
info.setExecutionStatus(ExecutionStatus.READY);
|
||||||
info.setTotal(0);
|
|
||||||
info.setMessage("");
|
|
||||||
|
|
||||||
infoMap.put(jobId, info);
|
infoMap.put(jobId, info);
|
||||||
|
|
||||||
|
@ -106,6 +103,7 @@ public class Oai2FtpService {
|
||||||
|
|
||||||
String url = SimpleUtils.oaiFirstUrl(baseUrl, format, setSpec, from, until);
|
String url = SimpleUtils.oaiFirstUrl(baseUrl, format, setSpec, from, until);
|
||||||
|
|
||||||
|
int count = 0;
|
||||||
while (StringUtils.isNotBlank(url)) {
|
while (StringUtils.isNotBlank(url)) {
|
||||||
final CollectionCall call = new CollectionCall();
|
final CollectionCall call = new CollectionCall();
|
||||||
call.setUrl(url);
|
call.setUrl(url);
|
||||||
|
@ -117,12 +115,13 @@ public class Oai2FtpService {
|
||||||
final List<Node> records = doc.selectNodes("//*[local-name()='ListRecords']/*[local-name()='record']");
|
final List<Node> records = doc.selectNodes("//*[local-name()='ListRecords']/*[local-name()='record']");
|
||||||
call.setNumberOfRecords(records.size());
|
call.setNumberOfRecords(records.size());
|
||||||
|
|
||||||
|
sc.changeDir(Integer.toString(count++));
|
||||||
for (final Node n : records) {
|
for (final Node n : records) {
|
||||||
final String id = n.valueOf(".//*[local-name()='header']/*[local-name()='identifier']");
|
final String id = n.valueOf(".//*[local-name()='header']/*[local-name()='identifier']");
|
||||||
|
|
||||||
sc.saveFile(SimpleUtils.oaiIdToFilename(id), n.asXML());
|
sc.saveFile(SimpleUtils.oaiIdToFilename(id), n.asXML());
|
||||||
info.setTotal(info.getTotal() + 1);
|
info.setTotal(info.getTotal() + 1);
|
||||||
}
|
}
|
||||||
|
sc.changeDir("..");
|
||||||
|
|
||||||
final String rtoken = doc.valueOf("//*[local-name()='resumptionToken']").trim();
|
final String rtoken = doc.valueOf("//*[local-name()='resumptionToken']").trim();
|
||||||
|
|
||||||
|
|
|
@ -35,8 +35,8 @@ public class LocalStorage implements StorageClient {
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void saveFile(final String filename, final String body) {
|
public void saveFile(final String filename, final String body) {
|
||||||
try {
|
try (FileWriter fw = new FileWriter(currDir + "/" + filename)) {
|
||||||
IOUtils.write(body, new FileWriter(currDir + "/" + filename));
|
IOUtils.write(body, fw);
|
||||||
} catch (final IOException e) {
|
} catch (final IOException e) {
|
||||||
log.error("Error saving info file");
|
log.error("Error saving info file");
|
||||||
throw new RuntimeException("Error saving info file", e);
|
throw new RuntimeException("Error saving info file", e);
|
||||||
|
|
|
@ -63,8 +63,7 @@ public class SimpleUtils {
|
||||||
info.setOaiSet(log.getOaiSet());
|
info.setOaiSet(log.getOaiSet());
|
||||||
info.setOaiFrom(log.getOaiFrom());
|
info.setOaiFrom(log.getOaiFrom());
|
||||||
info.setOaiUntil(log.getOaiUntil());
|
info.setOaiUntil(log.getOaiUntil());
|
||||||
info.setFtpServer(log.getFtpServer());
|
info.setStorageUrl(log.getStorageUrl());
|
||||||
info.setFtpDir(log.getFtpDir());
|
|
||||||
info.setStart(log.getStart());
|
info.setStart(log.getStart());
|
||||||
info.setEnd(log.getEnd());
|
info.setEnd(log.getEnd());
|
||||||
info.setExecutionStatus(log.isSuccess() ? ExecutionStatus.COMPLETED : ExecutionStatus.FAILED);
|
info.setExecutionStatus(log.isSuccess() ? ExecutionStatus.COMPLETED : ExecutionStatus.FAILED);
|
||||||
|
@ -81,8 +80,7 @@ public class SimpleUtils {
|
||||||
log.setOaiSet(info.getOaiSet());
|
log.setOaiSet(info.getOaiSet());
|
||||||
log.setOaiFrom(info.getOaiFrom());
|
log.setOaiFrom(info.getOaiFrom());
|
||||||
log.setOaiUntil(info.getOaiUntil());
|
log.setOaiUntil(info.getOaiUntil());
|
||||||
log.setFtpServer(info.getFtpServer());
|
log.setStorageUrl(info.getStorageUrl());
|
||||||
log.setFtpDir(info.getFtpDir());
|
|
||||||
log.setStart(info.getStart());
|
log.setStart(info.getStart());
|
||||||
log.setEnd(info.getEnd());
|
log.setEnd(info.getEnd());
|
||||||
log.setSuccess(info.getExecutionStatus() == ExecutionStatus.COMPLETED);
|
log.setSuccess(info.getExecutionStatus() == ExecutionStatus.COMPLETED);
|
||||||
|
|
|
@ -17,7 +17,7 @@ public class StorageClientFactory {
|
||||||
@Value("${oai2ftp.conf.storage.password}")
|
@Value("${oai2ftp.conf.storage.password}")
|
||||||
private String storagePassword;
|
private String storagePassword;
|
||||||
|
|
||||||
public StorageClient newClientForJob(final String jobId) {
|
public StorageClient newClient() {
|
||||||
|
|
||||||
final String protocol = storageUrl.getProtocol();
|
final String protocol = storageUrl.getProtocol();
|
||||||
final String host = storageUrl.getHost();
|
final String host = storageUrl.getHost();
|
||||||
|
@ -36,9 +36,12 @@ public class StorageClientFactory {
|
||||||
}
|
}
|
||||||
client.login(storageUser, storagePassword);
|
client.login(storageUser, storagePassword);
|
||||||
client.changeDir(path);
|
client.changeDir(path);
|
||||||
client.changeDir(jobId);
|
|
||||||
|
|
||||||
return client;
|
return client;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public String getStorageUrlAsString() {
|
||||||
|
return storageUrl.toString();
|
||||||
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in New Issue