michele.artini
/
dnet-docker
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

oai client

This commit is contained in:
Michele Artini 2024-01-30 12:03:07 +01:00
parent f627f16b0f
commit 25703c9263
13 changed files with 415 additions and 129 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
dnet-app/libs/dnet-domain/src/main/java/eu/dnetlib/domain/oai/ExportedOaiRecord.java
View File

@ -11,7 +11,7 @@ import jakarta.persistence.Table;
@Entity
@Table(name = "oai_records")
public class ExportedOaiRecord implements OaiRecord {
public class ExportedOaiRecord implements OaiRecord<LocalDateTime, String> {
private static final long serialVersionUID = 9178403183427337500L;

5
dnet-app/libs/dnet-domain/src/main/java/eu/dnetlib/domain/oai/OaiIdentifier.java
View File

@ -1,14 +1,13 @@
package eu.dnetlib.domain.oai;
import java.io.Serializable;
import java.time.LocalDateTime;
import java.util.List;
public interface OaiIdentifier extends Serializable {
public interface OaiIdentifier<T> extends Serializable {
String getId();
LocalDateTime getDate();
T getDate();
List<String> getSets();

21
dnet-app/libs/dnet-domain/src/main/java/eu/dnetlib/domain/oai/OaiInfo.java
View File

@ -1,7 +1,26 @@
package eu.dnetlib.domain.oai;
import java.io.Serializable;
import java.util.List;
public interface OaiInfo extends Serializable {
public interface OaiInfo<T> extends Serializable {
String getRepositoryName();
String getBaseURL();
String getProtocolVersion();
T getEarliestDatestamp();
String getDeletedRecord();
String getGranularity();
List<String> getAdminEmails();
List<String> getCompressions();
List<String> getDescriptions();
}

4
dnet-app/libs/dnet-domain/src/main/java/eu/dnetlib/domain/oai/OaiRecord.java
View File

@ -1,7 +1,7 @@
package eu.dnetlib.domain.oai;
public interface OaiRecord extends OaiIdentifier {
public interface OaiRecord<T, K> extends OaiIdentifier<T> {
String getBody();
K getBody();
}

159
dnet-app/libs/dnet-oai-common/src/main/java/eu/dnetlib/common/oai/OaiClient.java
View File

@ -2,10 +2,10 @@ package eu.dnetlib.common.oai;
import java.util.List;
import java.util.Map;
import java.util.function.Function;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.math.NumberUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.dom4j.Document;
import org.dom4j.DocumentHelper;
import org.dom4j.Element;
@ -26,92 +26,114 @@ import eu.dnetlib.domain.oai.OaiSet;
public class OaiClient {
private static final Log log = LogFactory.getLog(OaiClient.class);
public OaiResponse<List<OaiSet>> listSets(final String baseUrl, final String resumptionToken) throws Exception {
public List<OaiSet> listSets(final String baseUrl) throws Exception {
final OaiRequest req = StringUtils.isNotBlank(resumptionToken)
? OaiRequest.newListSetsWithToken(baseUrl, resumptionToken)
: OaiRequest.newListSets(baseUrl);
final OaiRequest req = new OaiRequest();
req.setBaseUrl(baseUrl);
req.setVerb("ListSets");
final OaiResponse<List<OaiSet>> res = new OaiResponse<>();
return DocumentHelper.parseText(callOaiVerb(req).getBody())
.selectNodes("//oai:set")
callOaiVerb(req, res, doc -> doc
.selectNodes("//*[local-name()='ListSets']/*[local-name()='set']")
.stream()
.map(node -> (Element) node)
.map(OaiSetImpl::new)
.map(set -> (OaiSet) set)
.toList();
.toList());
return res;
}
public List<OaiMetadataFormat> listMetadataFormat(final String baseUrl) throws Exception {
final OaiRequest req = new OaiRequest();
req.setBaseUrl(baseUrl);
req.setVerb("ListMetadataFormats");
public OaiResponse<List<OaiMetadataFormat>> listMetadataFormats(final String baseUrl) throws Exception {
final OaiRequest req = OaiRequest.newListMetadataFormats(baseUrl);
final OaiResponse<List<OaiMetadataFormat>> res = new OaiResponse<>();
return DocumentHelper.parseText(callOaiVerb(req).getBody())
.selectNodes("//oai:metadataFormat")
callOaiVerb(req, res, doc -> doc
.selectNodes("//*[local-name()='ListMetadataFormats']/*[local-name()='metadataFormat']")
.stream()
.map(node -> (Element) node)
.map(OaiMetadataFormatImpl::new)
.map(set -> (OaiMetadataFormat) set)
.toList();
.toList());
return res;
}
public List<OaiRecord> listRecords(final String baseUrl, final String mdPrefix, final String set) {
final OaiRequest req = new OaiRequest();
req.setBaseUrl(baseUrl);
req.setVerb("ListRecords");
req.setMdf(mdPrefix);
req.setSet(set);
public OaiResponse<List<OaiRecord<String, String>>> listRecords(final String baseUrl, final String mdPrefix, final String set, final String resumptionToken)
throws Exception {
final OaiRequest req = StringUtils.isNotBlank(resumptionToken)
? OaiRequest.newListRecordsWithToken(baseUrl, resumptionToken)
: OaiRequest.newListRecords(baseUrl, mdPrefix, set);
final OaiResponse<List<OaiRecord<String, String>>> res = new OaiResponse<>();
callOaiVerb(req, res, doc -> doc
.selectNodes("//*[local-name()='ListRecords']/*[local-name()='record']")
.stream()
.map(node -> (Element) node)
.map(OaiRecordImpl::new)
.map(r -> (OaiRecord<String, String>) r)
.toList());
return res;
// TODO
return null;
}
public List<OaiRecord> listRecords(final String baseUrl, final String resumptionToken) {
final OaiRequest req = new OaiRequest();
req.setBaseUrl(baseUrl);
req.setVerb("ListRecords");
req.setToken(resumptionToken);
public OaiResponse<List<OaiIdentifier<String>>> listIdentifiers(final String baseUrl, final String mdPrefix, final String set, final String resumptionToken)
throws Exception {
final OaiRequest req = StringUtils.isNotBlank(resumptionToken)
? OaiRequest.newListIdentifiersWithToken(baseUrl, resumptionToken)
: OaiRequest.newListIdentifiers(baseUrl, mdPrefix, set);
// TODO
return null;
final OaiResponse<List<OaiIdentifier<String>>> res = new OaiResponse<>();
callOaiVerb(req, res, doc -> doc
.selectNodes("//*[local-name()='ListIdentifiers']/*[local-name()='header']")
.stream()
.map(node -> (Element) node)
.map(OaiIdentifierImpl::new)
.map(r -> (OaiIdentifier<String>) r)
.toList());
return res;
}
public List<OaiIdentifier> listIdentifier(final String baseUrl, final String mdPrefix, final String set) {
final OaiRequest req = new OaiRequest();
req.setBaseUrl(baseUrl);
req.setVerb("ListIdentifier");
req.setMdf(mdPrefix);
req.setSet(set);
public OaiResponse<OaiRecord<String, String>> getRecord(final String baseUrl, final String mdPrefix, final String id) throws Exception {
final OaiRequest req = OaiRequest.newGetRecord(baseUrl, mdPrefix, id);
// TODO
return null;
final OaiResponse<OaiRecord<String, String>> res = new OaiResponse<>();
callOaiVerb(req, res, doc -> doc
.selectNodes("//*[local-name()='GetRecord']/*[local-name()='record']")
.stream()
.map(node -> (Element) node)
.map(OaiRecordImpl::new)
.map(r -> (OaiRecord<String, String>) r)
.findFirst()
.orElseThrow());
return res;
}
public List<OaiIdentifier> listIdentifier(final String baseUrl, final String resumptionToken) {
final OaiRequest req = new OaiRequest();
req.setBaseUrl(baseUrl);
req.setVerb("ListIdentifier");
req.setToken(resumptionToken);
public OaiResponse<OaiInfo<String>> info(final String baseUrl) throws Exception {
final OaiRequest req = OaiRequest.newIdentify(baseUrl);
// TODO
return null;
final OaiResponse<OaiInfo<String>> res = new OaiResponse<>();
callOaiVerb(req, res, doc -> doc
.selectNodes("//*[local-name()='Identify']")
.stream()
.map(node -> (Element) node)
.map(OaiInfoImpl::new)
.map(r -> (OaiInfo<String>) r)
.findFirst()
.orElseThrow());
return res;
}
public OaiInfo info(final String baseUrl) {
final OaiRequest req = new OaiRequest();
req.setBaseUrl(baseUrl);
req.setVerb("Identify");
// TODO
return null;
}
private OaiResponse callOaiVerb(final OaiRequest req) {
final OaiResponse res = new OaiResponse();
private <T> void callOaiVerb(final OaiRequest req, final OaiResponse<T> res, final Function<Document, T> processBody) {
res.setVerb(req.getVerb());
final long start = System.currentTimeMillis();
@ -142,15 +164,17 @@ public class OaiClient {
res.setCursor(NumberUtils.toInt(node.valueOf("@cursor"), -1));
res.setTotal(NumberUtils.toInt(node.valueOf("@completeListSize"), -1));
final OaiRequest nextCall = new OaiRequest();
nextCall.setBaseUrl(req.getBaseUrl());
nextCall.setVerb(req.getVerb());
nextCall.setToken(node.getText());
res.setNextCall(nextCall);
}
res.setBody(doc.asXML());
res.setValid(true);
if (OaiRequest.LISTSETS.equals(req.getVerb())) {
res.setNextCall(OaiRequest.newListSetsWithToken(req.getBaseUrl(), node.getText()));
} else if (OaiRequest.LISTIDENTIFIERS.equals(req.getVerb())) {
res.setNextCall(OaiRequest.newListIdentifiersWithToken(req.getBaseUrl(), node.getText()));
} else if (OaiRequest.LISTRECORDS.equals(req.getVerb())) {
res.setNextCall(OaiRequest.newListRecordsWithToken(req.getBaseUrl(), node.getText()));
}
}
res.setBody(processBody.apply(doc));
res.setValid(true);
}
} catch (final Throwable e) {
res.setValid(false);
@ -159,7 +183,6 @@ public class OaiClient {
res.setTime(System.currentTimeMillis() - start);
}
return res;
}
}

17
dnet-app/libs/dnet-oai-common/src/main/java/eu/dnetlib/common/oai/OaiIdentifierImpl.java
View File

@ -1,21 +1,28 @@
package eu.dnetlib.common.oai;
import java.time.LocalDateTime;
import java.util.ArrayList;
import java.util.List;
import org.dom4j.Element;
import eu.dnetlib.domain.oai.OaiIdentifier;
public class OaiIdentifierImpl implements OaiIdentifier {
public class OaiIdentifierImpl implements OaiIdentifier<String> {
private static final long serialVersionUID = -2097987180475959419L;
private String id;
private LocalDateTime date;
private String date;
private List<String> sets = new ArrayList<>();
public OaiIdentifierImpl(final Element node) {
this.id = node.valueOf("./*[local-name() = 'identifier']");
this.date = node.valueOf("./*[local-name() = 'datestamp']");
node.selectNodes("./*[local-name() = 'setSpec']").forEach(s -> this.sets.add(s.getText()));
}
@Override
public String getId() {
return this.id;
@ -26,11 +33,11 @@ public class OaiIdentifierImpl implements OaiIdentifier {
}
@Override
public LocalDateTime getDate() {
public String getDate() {
return this.date;
}
public void setDate(final LocalDateTime date) {
public void setDate(final String date) {
this.date = date;
}

125
dnet-app/libs/dnet-oai-common/src/main/java/eu/dnetlib/common/oai/OaiInfoImpl.java Normal file
View File

@ -0,0 +1,125 @@
package eu.dnetlib.common.oai;
import java.util.ArrayList;
import java.util.List;
import org.dom4j.Element;
import eu.dnetlib.domain.oai.OaiInfo;
public class OaiInfoImpl implements OaiInfo<String> {
private static final long serialVersionUID = -1062314589986610721L;
private String repositoryName;
private String baseURL;
private String protocolVersion;
private String earliestDatestamp;
private String deletedRecord;
private String granularity;
private List<String> adminEmails = new ArrayList<>();
private List<String> compressions = new ArrayList<>();
private List<String> descriptions = new ArrayList<>();
public OaiInfoImpl(final Element node) {
this.repositoryName = node.valueOf("./*[local-name() = 'repositoryName']");
this.baseURL = node.valueOf("./*[local-name() = 'baseURL']");
this.protocolVersion = node.valueOf("./*[local-name() = 'protocolVersion']");
this.earliestDatestamp = node.valueOf("./*[local-name() = 'earliestDatestamp']");
this.deletedRecord = node.valueOf("./*[local-name() = 'deletedRecord']");
this.granularity = node.valueOf("./*[local-name() = 'granularity']");
node.selectNodes("./*[local-name() = 'adminEmail']").forEach(n -> this.adminEmails.add(n.getText()));
node.selectNodes("./*[local-name() = 'compression']").forEach(n -> this.compressions.add(n.getText()));
node.selectNodes("./*[local-name() = 'description']").forEach(n -> this.descriptions.add(n.getText()));
}
@Override
public String getRepositoryName() {
return this.repositoryName;
}
public void setRepositoryName(final String repositoryName) {
this.repositoryName = repositoryName;
}
@Override
public String getBaseURL() {
return this.baseURL;
}
public void setBaseURL(final String baseURL) {
this.baseURL = baseURL;
}
@Override
public String getProtocolVersion() {
return this.protocolVersion;
}
public void setProtocolVersion(final String protocolVersion) {
this.protocolVersion = protocolVersion;
}
@Override
public String getEarliestDatestamp() {
return this.earliestDatestamp;
}
public void setEarliestDatestamp(final String earliestDatestamp) {
this.earliestDatestamp = earliestDatestamp;
}
@Override
public String getDeletedRecord() {
return this.deletedRecord;
}
public void setDeletedRecord(final String deletedRecord) {
this.deletedRecord = deletedRecord;
}
@Override
public String getGranularity() {
return this.granularity;
}
public void setGranularity(final String granularity) {
this.granularity = granularity;
}
@Override
public List<String> getAdminEmails() {
return this.adminEmails;
}
public void setAdminEmails(final List<String> adminEmails) {
this.adminEmails = adminEmails;
}
@Override
public List<String> getCompressions() {
return this.compressions;
}
public void setCompressions(final List<String> compressions) {
this.compressions = compressions;
}
@Override
public List<String> getDescriptions() {
return this.descriptions;
}
public void setDescriptions(final List<String> descriptions) {
this.descriptions = descriptions;
}
}

7
dnet-app/libs/dnet-oai-common/src/main/java/eu/dnetlib/common/oai/OaiMetadataFormatImpl.java
View File

@ -15,10 +15,9 @@ public class OaiMetadataFormatImpl implements OaiMetadataFormat {
private String metadataNamespace;
public OaiMetadataFormatImpl(final Element node) {
this.metadataPrefix = node.valueOf("./oai:metadataPrefix");
this.metadataSchema = node.valueOf("./oai:schema");
this.metadataNamespace = node.valueOf("./oai:metadataNamespace");
this.metadataPrefix = node.valueOf("./*[local-name()='metadataPrefix']");
this.metadataSchema = node.valueOf("./*[local-name()='schema']");
this.metadataNamespace = node.valueOf("./*[local-name()='metadataNamespace']");
}
@Override

18
dnet-app/libs/dnet-oai-common/src/main/java/eu/dnetlib/common/oai/OaiRecordImpl.java
View File

@ -1,12 +1,13 @@
package eu.dnetlib.common.oai;
import java.time.LocalDateTime;
import java.util.ArrayList;
import java.util.List;
import org.dom4j.Element;
import eu.dnetlib.domain.oai.OaiRecord;
public class OaiRecordImpl implements OaiRecord {
public class OaiRecordImpl implements OaiRecord<String, String> {
private static final long serialVersionUID = -2097987180475959419L;
@ -14,10 +15,17 @@ public class OaiRecordImpl implements OaiRecord {
private String body;
private LocalDateTime date;
private String date;
private List<String> sets = new ArrayList<>();
public OaiRecordImpl(final Element node) {
this.id = node.valueOf("./*[local-name() = 'header']/*[local-name() = 'identifier']");
this.date = node.valueOf("./*[local-name() = 'header']/*[local-name() = 'datestamp']");
this.body = node.selectSingleNode("./*[local-name() = 'metadata']/*").asXML();
node.selectNodes("./*[local-name() = 'header']/*[local-name() = 'setSpec']").forEach(s -> this.sets.add(s.getText()));
}
@Override
public String getId() {
return this.id;
@ -37,11 +45,11 @@ public class OaiRecordImpl implements OaiRecord {
}
@Override
public LocalDateTime getDate() {
public String getDate() {
return this.date;
}
public void setDate(final LocalDateTime date) {
public void setDate(final String date) {
this.date = date;
}

97
dnet-app/libs/dnet-oai-common/src/main/java/eu/dnetlib/common/oai/OaiRequest.java
View File

@ -1,70 +1,100 @@
package eu.dnetlib.common.oai;
import java.io.Serializable;
import java.util.HashMap;
import java.util.Map;
import org.apache.commons.lang3.StringUtils;
public class OaiRequest implements Serializable {
public class OaiRequest {
private static final long serialVersionUID = -1663833829304179507L;
public static final String GETRECORD = "GetRecord";
private String baseUrl;
private String verb;
private String mdf;
private String set;
private String id;
private String token;
public static final String LISTIDENTIFIERS = "ListIdentifiers";
public static final String LISTRECORDS = "ListRecords";
public static final String LISTSETS = "ListSets";
public static final String LISTMETADATAFORMATS = "ListMetadataFormats";
public static final String IDENTIFY = "Identify";
private final String baseUrl;
private final String verb;
private final String mdf;
private final String set;
private final String id;
private final String token;
public static OaiRequest newIdentify(final String baseUrl) {
return new OaiRequest(baseUrl, IDENTIFY, null, null, null, null);
}
public static OaiRequest newListMetadataFormats(final String baseUrl) {
return new OaiRequest(baseUrl, LISTMETADATAFORMATS, null, null, null, null);
}
public static OaiRequest newListSets(final String baseUrl) {
return new OaiRequest(baseUrl, LISTSETS, null, null, null, null);
}
public static OaiRequest newListSetsWithToken(final String baseUrl, final String reusumptionToken) {
return new OaiRequest(baseUrl, LISTSETS, null, null, null, reusumptionToken);
}
public static OaiRequest newListRecords(final String baseUrl, final String mdPrefix, final String set) {
return new OaiRequest(baseUrl, LISTRECORDS, mdPrefix, set, null, null);
}
public static OaiRequest newListRecordsWithToken(final String baseUrl, final String reusumptionToken) {
return new OaiRequest(baseUrl, LISTRECORDS, null, null, null, reusumptionToken);
}
public static OaiRequest newListIdentifiers(final String baseUrl, final String mdPrefix, final String set) {
return new OaiRequest(baseUrl, LISTIDENTIFIERS, mdPrefix, set, null, null);
}
public static OaiRequest newListIdentifiersWithToken(final String baseUrl, final String reusumptionToken) {
return new OaiRequest(baseUrl, LISTIDENTIFIERS, null, null, null, reusumptionToken);
}
public static OaiRequest newGetRecord(final String baseUrl, final String mdPrefix, final String id) {
return new OaiRequest(baseUrl, GETRECORD, mdPrefix, null, id, null);
}
private OaiRequest(final String baseUrl, final String verb, final String mdf, final String set, final String id, final String token) {
this.baseUrl = baseUrl;
this.verb = verb;
this.mdf = mdf;
this.set = set;
this.id = id;
this.token = token;
}
public String getBaseUrl() {
return this.baseUrl;
}
public void setBaseUrl(final String baseUrl) {
this.baseUrl = baseUrl;
}
public String getVerb() {
return this.verb;
}
public void setVerb(final String verb) {
this.verb = verb;
}
public String getMdf() {
return this.mdf;
}
public void setMdf(final String mdf) {
this.mdf = mdf;
}
public String getSet() {
return this.set;
}
public void setSet(final String set) {
this.set = set;
}
public String getId() {
return this.id;
}
public void setId(final String id) {
this.id = id;
}
public String getToken() {
return this.token;
}
public void setToken(final String token) {
this.token = token;
}
public Map<String, Object> toQueryParams() {
final Map<String, Object> params = new HashMap<>();
@ -86,4 +116,5 @@ public class OaiRequest implements Serializable {
return params;
}
}

8
dnet-app/libs/dnet-oai-common/src/main/java/eu/dnetlib/common/oai/OaiResponse.java
View File

@ -2,7 +2,7 @@ package eu.dnetlib.common.oai;
import java.io.Serializable;
public class OaiResponse implements Serializable {
public class OaiResponse<T> implements Serializable {
private static final long serialVersionUID = -7841519266909952558L;
private String verb;
@ -13,7 +13,7 @@ public class OaiResponse implements Serializable {
private int size;
private int cursor;
private int total;
private String body;
private T body;
private OaiRequest nextCall;
public String getVerb() {
@ -80,11 +80,11 @@ public class OaiResponse implements Serializable {
this.total = total;
}
public String getBody() {
public T getBody() {
return this.body;
}
public void setBody(final String body) {
public void setBody(final T body) {
this.body = body;
}

6
dnet-app/libs/dnet-oai-common/src/main/java/eu/dnetlib/common/oai/OaiSetImpl.java
View File

@ -13,9 +13,9 @@ public class OaiSetImpl implements OaiSet {
private String description;
public OaiSetImpl(final Element node) {
this.setSpec = node.valueOf("./oai:setSpec");
this.setName = node.valueOf("./oai:setName");
this.description = node.valueOf("./oai:description");
this.setSpec = node.valueOf("./*[local-name()='setSpec']");
this.setName = node.valueOf("./*[local-name()='setName']");
this.description = node.valueOf("./*[local-name()='description']");
}
@Override

75
dnet-app/libs/dnet-oai-common/src/test/java/eu/dnetlib/common/oai/OaiClientTest.java Normal file
View File

@ -0,0 +1,75 @@
package eu.dnetlib.common.oai;
import java.util.List;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test;
import com.fasterxml.jackson.databind.ObjectMapper;
import eu.dnetlib.domain.oai.OaiIdentifier;
import eu.dnetlib.domain.oai.OaiInfo;
import eu.dnetlib.domain.oai.OaiMetadataFormat;
import eu.dnetlib.domain.oai.OaiRecord;
import eu.dnetlib.domain.oai.OaiSet;
@Disabled
public class OaiClientTest {
private OaiClient oai;
private final ObjectMapper mapper = new ObjectMapper();
private final String BASE_URL = "https://openportal.isti.cnr.it/oai";
@BeforeEach
void setUp() throws Exception {
this.oai = new OaiClient();
}
@Test
void testListSets() throws Exception {
final OaiResponse<List<OaiSet>> res = this.oai.listSets(this.BASE_URL, null);
System.err.println(this.mapper.writeValueAsString(res));
}
@Test
void testListMetadataFormat() throws Exception {
final OaiResponse<List<OaiMetadataFormat>> res = this.oai.listMetadataFormats(this.BASE_URL);
System.err.println(this.mapper.writeValueAsString(res));
}
@Test
void testListRecords() throws Exception {
final OaiResponse<List<OaiRecord<String, String>>> res = this.oai.listRecords(this.BASE_URL, "oai_dc", "ISTI", null);
System.err.println(this.mapper.writeValueAsString(res));
}
@Test
void testListIdentifiers() throws Exception {
final OaiResponse<List<OaiIdentifier<String>>> res = this.oai.listIdentifiers(this.BASE_URL, "oai_dc", "ISTI", null);
System.err.println(this.mapper.writeValueAsString(res));
}
@Test
void testGetRecord() throws Exception {
final OaiResponse<OaiRecord<String, String>> res =
this.oai.getRecord(this.BASE_URL, "oai_dc", "oai:dnet:people______::1ba6ca3a2416b67b4c2206f2d6b865da");
System.err.println(this.mapper.writeValueAsString(res));
}
@Test
void testInfo() throws Exception {
final OaiResponse<OaiInfo<String>> res = this.oai.info(this.BASE_URL);
System.err.println(this.mapper.writeValueAsString(res));
}
}