partial implementation of oai client

dnet-app/libs/dnet-domain/src/main/java/eu/dnetlib/domain/oai/ExportedOaiMetadataFormat.java

@@ -29,7 +29,6 @@ public class ExportedOaiMetadataFormat implements OaiMetadataFormat {
 		return this.metadataPrefix;
 	}
 
-	@Override
 	public void setMetadataPrefix(final String metadataPrefix) {
 		this.metadataPrefix = metadataPrefix;
 	}
@@ -39,7 +38,6 @@ public class ExportedOaiMetadataFormat implements OaiMetadataFormat {
 		return this.metadataSchema;
 	}
 
-	@Override
 	public void setMetadataSchema(final String metadataSchema) {
 		this.metadataSchema = metadataSchema;
 	}
@@ -49,7 +47,6 @@ public class ExportedOaiMetadataFormat implements OaiMetadataFormat {
 		return this.metadataNamespace;
 	}
 
-	@Override
 	public void setMetadataNamespace(final String metadataNamespace) {
 		this.metadataNamespace = metadataNamespace;
 	}

dnet-app/libs/dnet-domain/src/main/java/eu/dnetlib/domain/oai/ExportedOaiRecord.java

@@ -1,6 +1,8 @@
 package eu.dnetlib.domain.oai;
 
 import java.time.LocalDateTime;
+import java.util.Arrays;
+import java.util.List;
 
 import jakarta.persistence.Column;
 import jakarta.persistence.Entity;
@@ -31,7 +33,6 @@ public class ExportedOaiRecord implements OaiRecord {
 		return this.id;
 	}
 
-	@Override
 	public void setId(final String id) {
 		this.id = id;
 	}
@@ -41,7 +42,6 @@ public class ExportedOaiRecord implements OaiRecord {
 		return this.body;
 	}
 
-	@Override
 	public void setBody(final String body) {
 		this.body = body;
 	}
@@ -51,7 +51,6 @@ public class ExportedOaiRecord implements OaiRecord {
 		return this.date;
 	}
 
-	@Override
 	public void setDate(final LocalDateTime date) {
 		this.date = date;
 	}
@@ -63,4 +62,10 @@ public class ExportedOaiRecord implements OaiRecord {
 	public void setOaiSet(final String oaiSet) {
 		this.oaiSet = oaiSet;
 	}
+
+	@Override
+	public List<String> getSets() {
+		return Arrays.asList(this.oaiSet);
+	}
+
 }

dnet-app/libs/dnet-domain/src/main/java/eu/dnetlib/domain/oai/ExportedOaiSet.java

@@ -24,11 +24,11 @@ public class ExportedOaiSet implements OaiSet {
 	@Column(name = "ds_id")
 	private String dsId;
 
+	@Override
 	public String getSetSpec() {
 		return this.setSpec;
 	}
 
-	@Override
 	public void setSetSpec(final String setSpec) {
 		this.setSpec = setSpec;
 	}
@@ -38,7 +38,6 @@ public class ExportedOaiSet implements OaiSet {
 		return this.setName;
 	}
 
-	@Override
 	public void setSetName(final String setName) {
 		this.setName = setName;
 	}
@@ -48,7 +47,6 @@ public class ExportedOaiSet implements OaiSet {
 		return this.description;
 	}
 
-	@Override
 	public void setDescription(final String description) {
 		this.description = description;
 	}

dnet-app/libs/dnet-domain/src/main/java/eu/dnetlib/domain/oai/OaiIdentifier.java

@@ -1,7 +1,15 @@
 package eu.dnetlib.domain.oai;
 
 import java.io.Serializable;
+import java.time.LocalDateTime;
+import java.util.List;
 
 public interface OaiIdentifier extends Serializable {
 
+	String getId();
+
+	LocalDateTime getDate();
+
+	List<String> getSets();
+
 }

dnet-app/libs/dnet-domain/src/main/java/eu/dnetlib/domain/oai/OaiMetadataFormat.java

@@ -6,14 +6,8 @@ public interface OaiMetadataFormat extends Serializable {
 
 	String getMetadataPrefix();
 
-	void setMetadataPrefix(final String metadataPrefix);
-
 	String getMetadataSchema();
 
-	void setMetadataSchema(final String metadataSchema);
-
 	String getMetadataNamespace();
 
-	void setMetadataNamespace(String metadataNamespace);
-
 }

dnet-app/libs/dnet-domain/src/main/java/eu/dnetlib/domain/oai/OaiRecord.java

@@ -1,19 +1,7 @@
 package eu.dnetlib.domain.oai;
 
-import java.io.Serializable;
-import java.time.LocalDateTime;
-
-public interface OaiRecord extends Serializable {
-
-	String getId();
-
-	void setId(String id);
+public interface OaiRecord extends OaiIdentifier {
 
 	String getBody();
 
-	void setBody(String body);
-
-	LocalDateTime getDate();
-
-	void setDate(LocalDateTime date);
 }

dnet-app/libs/dnet-domain/src/main/java/eu/dnetlib/domain/oai/OaiSet.java

@@ -4,13 +4,10 @@ import java.io.Serializable;
 
 public interface OaiSet extends Serializable {
 
-	void setSetSpec(String setSpec);
+	String getSetSpec();
 
 	String getSetName();
 
-	void setSetName(String setName);
-
 	String getDescription();
 
-	void setDescription(String description);
 }

dnet-app/libs/dnet-oai-common/src/main/java/eu/dnetlib/common/oai/OaiClient.java

@@ -0,0 +1,165 @@
+package eu.dnetlib.common.oai;
+
+import java.util.List;
+import java.util.Map;
+
+import org.apache.commons.lang3.math.NumberUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.dom4j.Document;
+import org.dom4j.DocumentHelper;
+import org.dom4j.Element;
+import org.dom4j.Node;
+import org.springframework.http.HttpEntity;
+import org.springframework.http.HttpHeaders;
+import org.springframework.http.HttpMethod;
+import org.springframework.http.MediaType;
+import org.springframework.http.ResponseEntity;
+import org.springframework.web.client.RestTemplate;
+import org.springframework.web.util.UriComponentsBuilder;
+
+import eu.dnetlib.domain.oai.OaiIdentifier;
+import eu.dnetlib.domain.oai.OaiInfo;
+import eu.dnetlib.domain.oai.OaiMetadataFormat;
+import eu.dnetlib.domain.oai.OaiRecord;
+import eu.dnetlib.domain.oai.OaiSet;
+
+public class OaiClient {
+
+	private static final Log log = LogFactory.getLog(OaiClient.class);
+
+	public List<OaiSet> listSets(final String baseUrl) throws Exception {
+
+		final OaiRequest req = new OaiRequest();
+		req.setBaseUrl(baseUrl);
+		req.setVerb("ListSets");
+
+		return DocumentHelper.parseText(callOaiVerb(req).getBody())
+				.selectNodes("//oai:set")
+				.stream()
+				.map(node -> (Element) node)
+				.map(OaiSetImpl::new)
+				.map(set -> (OaiSet) set)
+				.toList();
+
+	}
+
+	public List<OaiMetadataFormat> listMetadataFormat(final String baseUrl) throws Exception {
+		final OaiRequest req = new OaiRequest();
+		req.setBaseUrl(baseUrl);
+		req.setVerb("ListMetadataFormats");
+
+		return DocumentHelper.parseText(callOaiVerb(req).getBody())
+				.selectNodes("//oai:metadataFormat")
+				.stream()
+				.map(node -> (Element) node)
+				.map(OaiMetadataFormatImpl::new)
+				.map(set -> (OaiMetadataFormat) set)
+				.toList();
+	}
+
+	public List<OaiRecord> listRecords(final String baseUrl, final String mdPrefix, final String set) {
+		final OaiRequest req = new OaiRequest();
+		req.setBaseUrl(baseUrl);
+		req.setVerb("ListRecords");
+		req.setMdf(mdPrefix);
+		req.setSet(set);
+
+		// TODO
+		return null;
+	}
+
+	public List<OaiRecord> listRecords(final String baseUrl, final String resumptionToken) {
+		final OaiRequest req = new OaiRequest();
+		req.setBaseUrl(baseUrl);
+		req.setVerb("ListRecords");
+		req.setToken(resumptionToken);
+
+		// TODO
+		return null;
+	}
+
+	public List<OaiIdentifier> listIdentifier(final String baseUrl, final String mdPrefix, final String set) {
+		final OaiRequest req = new OaiRequest();
+		req.setBaseUrl(baseUrl);
+		req.setVerb("ListIdentifier");
+		req.setMdf(mdPrefix);
+		req.setSet(set);
+
+		// TODO
+		return null;
+	}
+
+	public List<OaiIdentifier> listIdentifier(final String baseUrl, final String resumptionToken) {
+		final OaiRequest req = new OaiRequest();
+		req.setBaseUrl(baseUrl);
+		req.setVerb("ListIdentifier");
+		req.setToken(resumptionToken);
+
+		// TODO
+		return null;
+	}
+
+	public OaiInfo info(final String baseUrl) {
+		final OaiRequest req = new OaiRequest();
+		req.setBaseUrl(baseUrl);
+		req.setVerb("Identify");
+
+		// TODO
+		return null;
+	}
+
+	private OaiResponse callOaiVerb(final OaiRequest req) {
+
+		final OaiResponse res = new OaiResponse();
+
+		res.setVerb(req.getVerb());
+		final long start = System.currentTimeMillis();
+
+		try {
+			final HttpHeaders headers = new HttpHeaders();
+			headers.set(HttpHeaders.ACCEPT, MediaType.APPLICATION_XML_VALUE);
+
+			final Map<String, Object> params = req.toQueryParams();
+
+			final UriComponentsBuilder urlBuilder = UriComponentsBuilder.fromHttpUrl(req.getBaseUrl());
+			params.keySet().forEach(k -> urlBuilder.queryParam(k, "{" + k + "}"));
+
+			final ResponseEntity<String> httpResponse =
+					new RestTemplate().exchange(urlBuilder.encode().toUriString(), HttpMethod.GET, new HttpEntity<>(headers), String.class, params);
+
+			res.setHttpCode(httpResponse.getStatusCode().value());
+
+			if (httpResponse.getStatusCode().isError()) {
+				res.setValid(false);
+				res.setError("ERROR: " + httpResponse.getStatusCode().value());
+			} else {
+				final Document doc = DocumentHelper.parseText(httpResponse.getBody());
+				final Node node = doc.selectSingleNode("//*[local-name() = 'resumptionToken']");
+
+				if (node != null) {
+					res.setSize(doc.selectNodes("//*[local-name()='" + req.getVerb() + "']/*[local-name() != 'resumptionToken']").size());
+					res.setCursor(NumberUtils.toInt(node.valueOf("@cursor"), -1));
+					res.setTotal(NumberUtils.toInt(node.valueOf("@completeListSize"), -1));
+
+					final OaiRequest nextCall = new OaiRequest();
+					nextCall.setBaseUrl(req.getBaseUrl());
+					nextCall.setVerb(req.getVerb());
+					nextCall.setToken(node.getText());
+					res.setNextCall(nextCall);
+				}
+				res.setBody(doc.asXML());
+				res.setValid(true);
+
+			}
+		} catch (final Throwable e) {
+			res.setValid(false);
+			res.setError(e.getMessage());
+		} finally {
+			res.setTime(System.currentTimeMillis() - start);
+		}
+
+		return res;
+	}
+
+}

dnet-app/libs/dnet-oai-common/src/main/java/eu/dnetlib/common/oai/OaiIdentifierImpl.java

@@ -0,0 +1,45 @@
+package eu.dnetlib.common.oai;
+
+import java.time.LocalDateTime;
+import java.util.ArrayList;
+import java.util.List;
+
+import eu.dnetlib.domain.oai.OaiIdentifier;
+
+public class OaiIdentifierImpl implements OaiIdentifier {
+
+	private static final long serialVersionUID = -2097987180475959419L;
+
+	private String id;
+
+	private LocalDateTime date;
+
+	private List<String> sets = new ArrayList<>();
+
+	@Override
+	public String getId() {
+		return this.id;
+	}
+
+	public void setId(final String id) {
+		this.id = id;
+	}
+
+	@Override
+	public LocalDateTime getDate() {
+		return this.date;
+	}
+
+	public void setDate(final LocalDateTime date) {
+		this.date = date;
+	}
+
+	@Override
+	public List<String> getSets() {
+		return this.sets;
+	}
+
+	public void setSets(final List<String> sets) {
+		this.sets = sets;
+	}
+}

dnet-app/libs/dnet-oai-common/src/main/java/eu/dnetlib/common/oai/OaiIterator.java

@@ -1,4 +1,4 @@
-package eu.dnetlib.common.oai.domain;
+package eu.dnetlib.common.oai;
 
 import java.io.StringReader;
 import java.net.URLEncoder;
@@ -23,6 +23,8 @@ import eu.dnetlib.utils.XmlCleaner;
 
 public class OaiIterator implements Iterator<String> {
 
+	private static final Log log = LogFactory.getLog(OaiIterator.class);
+
 	private static final DateTimeFormatter oaiDateFormatter = DateTimeFormatter.ofPattern("yyyy-MM-dd");
 
 	private final Queue<String> queue = new PriorityBlockingQueue<>();
@@ -36,8 +38,6 @@ public class OaiIterator implements Iterator<String> {
 	private String token;
 	private boolean started;
 
-	private static final Log log = LogFactory.getLog(OaiIterator.class);
-
 	public OaiIterator(final String baseUrl,
 			final String mdFormat,
 			final String set,

dnet-app/libs/dnet-oai-common/src/main/java/eu/dnetlib/common/oai/OaiMetadataFormatImpl.java

@@ -0,0 +1,51 @@
+package eu.dnetlib.common.oai;
+
+import org.dom4j.Element;
+
+import eu.dnetlib.domain.oai.OaiMetadataFormat;
+
+public class OaiMetadataFormatImpl implements OaiMetadataFormat {
+
+	private static final long serialVersionUID = 1695382415748936665L;
+
+	private String metadataPrefix;
+
+	private String metadataSchema;
+
+	private String metadataNamespace;
+
+	public OaiMetadataFormatImpl(final Element node) {
+		this.metadataPrefix = node.valueOf("./oai:metadataPrefix");
+		this.metadataSchema = node.valueOf("./oai:schema");
+		this.metadataNamespace = node.valueOf("./oai:metadataNamespace");
+
+	}
+
+	@Override
+	public String getMetadataPrefix() {
+		return this.metadataPrefix;
+	}
+
+	public void setMetadataPrefix(final String metadataPrefix) {
+		this.metadataPrefix = metadataPrefix;
+	}
+
+	@Override
+	public String getMetadataSchema() {
+		return this.metadataSchema;
+	}
+
+	public void setMetadataSchema(final String metadataSchema) {
+		this.metadataSchema = metadataSchema;
+	}
+
+	@Override
+	public String getMetadataNamespace() {
+		return this.metadataNamespace;
+	}
+
+	public void setMetadataNamespace(final String metadataNamespace) {
+		this.metadataNamespace = metadataNamespace;
+	}
+
+}

dnet-app/libs/dnet-oai-common/src/main/java/eu/dnetlib/common/oai/OaiRecordImpl.java

@@ -0,0 +1,56 @@
+package eu.dnetlib.common.oai;
+
+import java.time.LocalDateTime;
+import java.util.ArrayList;
+import java.util.List;
+
+import eu.dnetlib.domain.oai.OaiRecord;
+
+public class OaiRecordImpl implements OaiRecord {
+
+	private static final long serialVersionUID = -2097987180475959419L;
+
+	private String id;
+
+	private String body;
+
+	private LocalDateTime date;
+
+	private List<String> sets = new ArrayList<>();
+
+	@Override
+	public String getId() {
+		return this.id;
+	}
+
+	public void setId(final String id) {
+		this.id = id;
+	}
+
+	@Override
+	public String getBody() {
+		return this.body;
+	}
+
+	public void setBody(final String body) {
+		this.body = body;
+	}
+
+	@Override
+	public LocalDateTime getDate() {
+		return this.date;
+	}
+
+	public void setDate(final LocalDateTime date) {
+		this.date = date;
+	}
+
+	@Override
+	public List<String> getSets() {
+		return this.sets;
+	}
+
+	public void setSets(final List<String> sets) {
+		this.sets = sets;
+	}
+}

dnet-app/libs/dnet-oai-common/src/main/java/eu/dnetlib/common/oai/OaiRequest.java

@@ -0,0 +1,89 @@
+package eu.dnetlib.common.oai;
+
+import java.io.Serializable;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.commons.lang3.StringUtils;
+
+public class OaiRequest implements Serializable {
+
+	private static final long serialVersionUID = -1663833829304179507L;
+
+	private String baseUrl;
+	private String verb;
+	private String mdf;
+	private String set;
+	private String id;
+	private String token;
+
+	public String getBaseUrl() {
+		return this.baseUrl;
+	}
+
+	public void setBaseUrl(final String baseUrl) {
+		this.baseUrl = baseUrl;
+	}
+
+	public String getVerb() {
+		return this.verb;
+	}
+
+	public void setVerb(final String verb) {
+		this.verb = verb;
+	}
+
+	public String getMdf() {
+		return this.mdf;
+	}
+
+	public void setMdf(final String mdf) {
+		this.mdf = mdf;
+	}
+
+	public String getSet() {
+		return this.set;
+	}
+
+	public void setSet(final String set) {
+		this.set = set;
+	}
+
+	public String getId() {
+		return this.id;
+	}
+
+	public void setId(final String id) {
+		this.id = id;
+	}
+
+	public String getToken() {
+		return this.token;
+	}
+
+	public void setToken(final String token) {
+		this.token = token;
+	}
+
+	public Map<String, Object> toQueryParams() {
+		final Map<String, Object> params = new HashMap<>();
+
+		if (StringUtils.isNotBlank(this.verb)) {
+			params.put("verb", this.verb);
+		}
+		if (StringUtils.isNotBlank(this.mdf)) {
+			params.put("metadataPrefix", this.mdf);
+		}
+		if (StringUtils.isNotBlank(this.set)) {
+			params.put("set", this.set);
+		}
+		if (StringUtils.isNotBlank(this.id)) {
+			params.put("identifier", this.id);
+		}
+		if (StringUtils.isNotBlank(this.token)) {
+			params.put("resumptionToken", this.token);
+		}
+
+		return params;
+	}
+}

dnet-app/libs/dnet-oai-common/src/main/java/eu/dnetlib/common/oai/OaiResponse.java

@@ -0,0 +1,99 @@
+package eu.dnetlib.common.oai;
+
+import java.io.Serializable;
+
+public class OaiResponse implements Serializable {
+
+	private static final long serialVersionUID = -7841519266909952558L;
+	private String verb;
+	private long time;
+	private int httpCode;
+	private boolean valid;
+	private String error;
+	private int size;
+	private int cursor;
+	private int total;
+	private String body;
+	private OaiRequest nextCall;
+
+	public String getVerb() {
+		return this.verb;
+	}
+
+	public void setVerb(final String verb) {
+		this.verb = verb;
+	}
+
+	public long getTime() {
+		return this.time;
+	}
+
+	public void setTime(final long time) {
+		this.time = time;
+	}
+
+	public int getHttpCode() {
+		return this.httpCode;
+	}
+
+	public void setHttpCode(final int httpCode) {
+		this.httpCode = httpCode;
+	}
+
+	public boolean isValid() {
+		return this.valid;
+	}
+
+	public void setValid(final boolean valid) {
+		this.valid = valid;
+	}
+
+	public String getError() {
+		return this.error;
+	}
+
+	public void setError(final String error) {
+		this.error = error;
+	}
+
+	public int getSize() {
+		return this.size;
+	}
+
+	public void setSize(final int size) {
+		this.size = size;
+	}
+
+	public int getCursor() {
+		return this.cursor;
+	}
+
+	public void setCursor(final int cursor) {
+		this.cursor = cursor;
+	}
+
+	public int getTotal() {
+		return this.total;
+	}
+
+	public void setTotal(final int total) {
+		this.total = total;
+	}
+
+	public String getBody() {
+		return this.body;
+	}
+
+	public void setBody(final String body) {
+		this.body = body;
+	}
+
+	public OaiRequest getNextCall() {
+		return this.nextCall;
+	}
+
+	public void setNextCall(final OaiRequest nextCall) {
+		this.nextCall = nextCall;
+	}
+
+}

dnet-app/libs/dnet-oai-common/src/main/java/eu/dnetlib/common/oai/OaiSetImpl.java

@@ -0,0 +1,48 @@
+package eu.dnetlib.common.oai;
+
+import org.dom4j.Element;
+
+import eu.dnetlib.domain.oai.OaiSet;
+
+public class OaiSetImpl implements OaiSet {
+
+	private static final long serialVersionUID = -8488295530179957767L;
+
+	private String setSpec;
+	private String setName;
+	private String description;
+
+	public OaiSetImpl(final Element node) {
+		this.setSpec = node.valueOf("./oai:setSpec");
+		this.setName = node.valueOf("./oai:setName");
+		this.description = node.valueOf("./oai:description");
+	}
+
+	@Override
+	public String getSetSpec() {
+		return this.setSpec;
+	}
+
+	public void setSetSpec(final String setSpec) {
+		this.setSpec = setSpec;
+	}
+
+	@Override
+	public String getSetName() {
+		return this.setName;
+	}
+
+	public void setSetName(final String setName) {
+		this.setName = setName;
+	}
+
+	@Override
+	public String getDescription() {
+		return this.description;
+	}
+
+	public void setDescription(final String description) {
+		this.description = description;
+	}
+
+}

dnet-app/libs/dnet-oai-common/src/main/java/eu/dnetlib/common/oai/domain/OaiClient.java

@@ -1,43 +0,0 @@
-package eu.dnetlib.common.oai.domain;
-
-import java.util.List;
-
-import eu.dnetlib.domain.oai.OaiIdentifier;
-import eu.dnetlib.domain.oai.OaiInfo;
-import eu.dnetlib.domain.oai.OaiMetadataFormat;
-import eu.dnetlib.domain.oai.OaiRecord;
-import eu.dnetlib.domain.oai.OaiSet;
-
-public class OaiClient {
-
-	public List<OaiSet> listSets(final String baseUrl) {
-
-		// TODO
-		return null;
-	}
-
-	public List<OaiMetadataFormat> listMetadataFormat(final String baseUrl) {
-
-		// TODO
-		return null;
-	}
-
-	public List<OaiRecord> listRecords(final String baseUrl) {
-
-		// TODO
-		return null;
-	}
-
-	public List<OaiIdentifier> listIdentifier(final String baseUrl) {
-
-		// TODO
-		return null;
-	}
-
-	public OaiInfo info(final String baseUrl) {
-
-		// TODO
-		return null;
-	}
-
-}

dnet-app/libs/dnet-wf-executor-common/src/main/java/eu/dnetlib/wfs/collector/oai/OaiCollectorPlugin.java

@@ -6,7 +6,7 @@ import java.util.Iterator;
 import java.util.Map;
 import java.util.stream.Stream;
 
-import eu.dnetlib.common.oai.domain.OaiIterator;
+import eu.dnetlib.common.oai.OaiIterator;
 import eu.dnetlib.utils.DnetStreamSupport;
 import eu.dnetlib.wfs.annotations.CollectorPlugin;
 import eu.dnetlib.wfs.collector.DnetCollectorPlugin;