From bd8af338509797d27c9a56f812f4829f9b085985 Mon Sep 17 00:00:00 2001 From: Giambattista Bloisi Date: Mon, 21 Oct 2024 16:42:19 +0200 Subject: [PATCH 1/7] Give a base path to FileSystem.get to resolve s3 file system --- .../main/java/eu/dnetlib/dhp/oa/dedup/SparkUpdateEntity.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dhp-workflows/dhp-dedup-openaire/src/main/java/eu/dnetlib/dhp/oa/dedup/SparkUpdateEntity.java b/dhp-workflows/dhp-dedup-openaire/src/main/java/eu/dnetlib/dhp/oa/dedup/SparkUpdateEntity.java index 49021ab58b..91795eb470 100644 --- a/dhp-workflows/dhp-dedup-openaire/src/main/java/eu/dnetlib/dhp/oa/dedup/SparkUpdateEntity.java +++ b/dhp-workflows/dhp-dedup-openaire/src/main/java/eu/dnetlib/dhp/oa/dedup/SparkUpdateEntity.java @@ -2,6 +2,7 @@ package eu.dnetlib.dhp.oa.dedup; import java.io.IOException; +import java.net.URI; import java.util.Map; import org.apache.commons.io.IOUtils; @@ -132,7 +133,7 @@ public class SparkUpdateEntity extends AbstractSparkAction { boolean result = false; - FileSystem fileSystem = FileSystem.get(new Configuration()); + FileSystem fileSystem = FileSystem.get(URI.create(basePath), new Configuration()); FileStatus[] fileStatuses = fileSystem.listStatus(new Path(basePath)); for (FileStatus fs : fileStatuses) { From 9a6d44c5b49aa7af3fc1bac69d3da40751b470a5 Mon Sep 17 00:00:00 2001 From: Giambattista Bloisi Date: Mon, 21 Oct 2024 21:38:51 +0200 Subject: [PATCH 2/7] Support for empty value of master option --- .../eu/dnetlib/dhp/application/SparkScalaApplication.scala | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dhp-common/src/main/scala/eu/dnetlib/dhp/application/SparkScalaApplication.scala b/dhp-common/src/main/scala/eu/dnetlib/dhp/application/SparkScalaApplication.scala index 526bbd2953..09a1887b19 100644 --- a/dhp-common/src/main/scala/eu/dnetlib/dhp/application/SparkScalaApplication.scala +++ b/dhp-common/src/main/scala/eu/dnetlib/dhp/application/SparkScalaApplication.scala @@ -2,6 +2,7 @@ package eu.dnetlib.dhp.application import eu.dnetlib.dhp.common.Constants import eu.dnetlib.dhp.utils.DHPUtils.writeHdfsFile +import org.apache.commons.lang3.StringUtils import scala.io.Source @@ -69,7 +70,7 @@ abstract class AbstractScalaApplication( .builder() .config(conf) .appName(getClass.getSimpleName) - if (master != null) + if (StringUtils.isNotBlank(master)) b.master(master) b.getOrCreate() } From 35c287c766de128085712d49322a6a03014ec284 Mon Sep 17 00:00:00 2001 From: Giambattista Bloisi Date: Wed, 23 Oct 2024 13:55:28 +0200 Subject: [PATCH 3/7] Include graph-mapper in shaded package --- dhp-shade-package/pom.xml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/dhp-shade-package/pom.xml b/dhp-shade-package/pom.xml index c4f9b262e9..42735f9d7d 100644 --- a/dhp-shade-package/pom.xml +++ b/dhp-shade-package/pom.xml @@ -56,11 +56,11 @@ - - - - - + + eu.dnetlib.dhp + dhp-graph-mapper + ${project.version} + From 7dfb3116612d70ca939efd9ff3f423238277e8c8 Mon Sep 17 00:00:00 2001 From: Giambattista Bloisi Date: Wed, 30 Oct 2024 11:22:04 +0100 Subject: [PATCH 4/7] New CopyEntitiesSparkJob as replacement to distcp for copying intermediate graph data --- .../dhp/oa/merge/CopyEntitiesSparkJob.java | 108 ++++++++++++++++++ .../merge/copy_graph_entities_parameters.json | 32 ++++++ 2 files changed, 140 insertions(+) create mode 100644 dhp-common/src/main/java/eu/dnetlib/dhp/oa/merge/CopyEntitiesSparkJob.java create mode 100644 dhp-common/src/main/resources/eu/dnetlib/dhp/oa/merge/copy_graph_entities_parameters.json diff --git a/dhp-common/src/main/java/eu/dnetlib/dhp/oa/merge/CopyEntitiesSparkJob.java b/dhp-common/src/main/java/eu/dnetlib/dhp/oa/merge/CopyEntitiesSparkJob.java new file mode 100644 index 0000000000..ba378c7ea4 --- /dev/null +++ b/dhp-common/src/main/java/eu/dnetlib/dhp/oa/merge/CopyEntitiesSparkJob.java @@ -0,0 +1,108 @@ + +package eu.dnetlib.dhp.oa.merge; + +import static eu.dnetlib.dhp.common.SparkSessionSupport.runWithSparkSession; + +import java.util.Arrays; +import java.util.Optional; + +import org.apache.commons.io.IOUtils; +import org.apache.spark.SparkConf; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import eu.dnetlib.dhp.application.ArgumentApplicationParser; +import eu.dnetlib.dhp.schema.common.ModelSupport; +import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpException; + +/** + * Copy specified entities from a graph snapshot to another + */ +public class CopyEntitiesSparkJob { + private static final Logger log = LoggerFactory.getLogger(CopyEntitiesSparkJob.class); + + private ArgumentApplicationParser parser; + + public CopyEntitiesSparkJob(ArgumentApplicationParser parser) { + this.parser = parser; + } + + public static void main(String[] args) throws Exception { + + String jsonConfiguration = IOUtils + .toString( + CopyEntitiesSparkJob.class + .getResourceAsStream( + "/eu/dnetlib/dhp/oa/merge/copy_graph_entities_parameters.json")); + final ArgumentApplicationParser parser = new ArgumentApplicationParser(jsonConfiguration); + parser.parseArgument(args); + + Boolean isSparkSessionManaged = Optional + .ofNullable(parser.get("isSparkSessionManaged")) + .map(Boolean::valueOf) + .orElse(Boolean.TRUE); + log.info("isSparkSessionManaged: {}", isSparkSessionManaged); + + new CopyEntitiesSparkJob(parser).run(isSparkSessionManaged); + } + + public void run(Boolean isSparkSessionManaged) + throws ISLookUpException { + + String graphInputPath = parser.get("graphInputPath"); + log.info("graphInputPath: {}", graphInputPath); + + String outputPath = parser.get("outputPath"); + log.info("outputPath: {}", outputPath); + + String entities = parser.get("entities"); + log.info("entities: {}", entities); + + String format = parser.get("format"); + log.info("format: {}", format); + + SparkConf conf = new SparkConf(); + + runWithSparkSession( + conf, + isSparkSessionManaged, + spark -> { + Arrays + .stream(entities.split(",")) + .map(x -> x.trim().toLowerCase()) + .filter(ModelSupport.oafTypes::containsKey) + .forEachOrdered( + entity -> { + switch (format.toLowerCase()) { + case "text": + spark + .read() + .text(graphInputPath + "/" + entity) + .write() + .option("compression", "gzip") + .mode("overwrite") + .text(outputPath + "/" + entity); + break; + case "json": + spark + .read() + .json(graphInputPath + "/" + entity) + .write() + .option("compression", "gzip") + .mode("overwrite") + .json(outputPath + "/" + entity); + break; + case "parquet": + spark + .read() + .parquet(graphInputPath + "/" + entity) + .write() + .option("compression", "gzip") + .mode("overwrite") + .parquet(outputPath + "/" + entity); + break; + } + }); + }); + } +} diff --git a/dhp-common/src/main/resources/eu/dnetlib/dhp/oa/merge/copy_graph_entities_parameters.json b/dhp-common/src/main/resources/eu/dnetlib/dhp/oa/merge/copy_graph_entities_parameters.json new file mode 100644 index 0000000000..0617228d1f --- /dev/null +++ b/dhp-common/src/main/resources/eu/dnetlib/dhp/oa/merge/copy_graph_entities_parameters.json @@ -0,0 +1,32 @@ +[ + { + "paramName": "issm", + "paramLongName": "isSparkSessionManaged", + "paramDescription": "when true will stop SparkSession after job execution", + "paramRequired": false + }, + { + "paramName": "gin", + "paramLongName": "graphInputPath", + "paramDescription": "the input graph root path", + "paramRequired": true + }, + { + "paramName": "out", + "paramLongName": "outputPath", + "paramDescription": "the output graph root path", + "paramRequired": true + }, + { + "paramName": "ent", + "paramLongName": "entities", + "paramDescription": "the output graph root path", + "paramRequired": true + }, + { + "paramName": "fmt", + "paramLongName": "format", + "paramDescription": "the output graph root path", + "paramRequired": true + } +] \ No newline at end of file From 1a3fc87599a8d908f2caf2089715e1db6b64263c Mon Sep 17 00:00:00 2001 From: Giambattista Bloisi Date: Mon, 21 Oct 2024 09:03:20 +0200 Subject: [PATCH 5/7] Import cnr-rmi-api project, rename it dnet-rmi-api and use jakarta and metro in place of javax for portability to java 17 --- .gitignore | 3 +- dhp-common/pom.xml | 37 +---- dhp-rmi-api/pom.xml | 34 +++++ .../common/rmi/APIDeprecatedException.java | 11 ++ .../eu/dnetlib/common/rmi/BaseService.java | 35 +++++ .../common/rmi/DNetRestDocumentation.java | 19 +++ .../eu/dnetlib/common/rmi/RMIException.java | 27 ++++ .../common/rmi/UnimplementedException.java | 11 ++ .../rmi/CollectionService.java | 31 ++++ .../rmi/CollectionServiceException.java | 27 ++++ .../publisher/rmi/PublisherService.java | 57 ++++++++ .../mdstore/DocumentNotFoundException.java | 30 ++++ .../dnetlib/data/mdstore/MDStoreService.java | 118 +++++++++++++++ .../data/mdstore/MDStoreServiceException.java | 33 +++++ .../rmi/ObjectPackagingException.java | 28 ++++ .../rmi/ObjectPackagingService.java | 37 +++++ .../dnetlib/enabling/dlm/rmi/DlmService.java | 17 +++ .../hcm/rmi/HostingContextManagerService.java | 19 +++ .../hnm/rmi/HostingNodeManagerService.java | 15 ++ .../ISLookUpDocumentNotFoundException.java | 47 ++++++ .../is/lookup/rmi/ISLookUpException.java | 27 ++++ .../is/lookup/rmi/ISLookUpService.java | 57 ++++++++ .../ISRegistryDocumentNotFoundException.java | 28 ++++ .../is/registry/rmi/ISRegistryException.java | 25 ++++ .../is/registry/rmi/ISRegistryService.java | 72 ++++++++++ .../enabling/is/sn/rmi/ISSNException.java | 25 ++++ .../enabling/is/sn/rmi/ISSNService.java | 121 ++++++++++++++++ .../SubscriptionRequestRejectedException.java | 21 +++ .../is/store/rmi/ISStoreException.java | 25 ++++ .../enabling/is/store/rmi/ISStoreService.java | 47 ++++++ .../resultset/rmi/ResultSetException.java | 22 +++ .../resultset/rmi/ResultSetService.java | 135 ++++++++++++++++++ dhp-shade-package/dependency-reduced-pom.xml | 113 --------------- dhp-shade-package/pom.xml | 1 - dhp-workflows/pom.xml | 7 +- pom.xml | 17 +-- 36 files changed, 1224 insertions(+), 155 deletions(-) create mode 100644 dhp-rmi-api/pom.xml create mode 100644 dhp-rmi-api/src/main/java/eu/dnetlib/common/rmi/APIDeprecatedException.java create mode 100644 dhp-rmi-api/src/main/java/eu/dnetlib/common/rmi/BaseService.java create mode 100644 dhp-rmi-api/src/main/java/eu/dnetlib/common/rmi/DNetRestDocumentation.java create mode 100644 dhp-rmi-api/src/main/java/eu/dnetlib/common/rmi/RMIException.java create mode 100644 dhp-rmi-api/src/main/java/eu/dnetlib/common/rmi/UnimplementedException.java create mode 100644 dhp-rmi-api/src/main/java/eu/dnetlib/data/information/collectionservice/rmi/CollectionService.java create mode 100644 dhp-rmi-api/src/main/java/eu/dnetlib/data/information/collectionservice/rmi/CollectionServiceException.java create mode 100644 dhp-rmi-api/src/main/java/eu/dnetlib/data/information/publisher/rmi/PublisherService.java create mode 100644 dhp-rmi-api/src/main/java/eu/dnetlib/data/mdstore/DocumentNotFoundException.java create mode 100644 dhp-rmi-api/src/main/java/eu/dnetlib/data/mdstore/MDStoreService.java create mode 100644 dhp-rmi-api/src/main/java/eu/dnetlib/data/mdstore/MDStoreServiceException.java create mode 100644 dhp-rmi-api/src/main/java/eu/dnetlib/data/utility/objectpackaging/rmi/ObjectPackagingException.java create mode 100644 dhp-rmi-api/src/main/java/eu/dnetlib/data/utility/objectpackaging/rmi/ObjectPackagingService.java create mode 100644 dhp-rmi-api/src/main/java/eu/dnetlib/enabling/dlm/rmi/DlmService.java create mode 100644 dhp-rmi-api/src/main/java/eu/dnetlib/enabling/hcm/rmi/HostingContextManagerService.java create mode 100644 dhp-rmi-api/src/main/java/eu/dnetlib/enabling/hnm/rmi/HostingNodeManagerService.java create mode 100644 dhp-rmi-api/src/main/java/eu/dnetlib/enabling/is/lookup/rmi/ISLookUpDocumentNotFoundException.java create mode 100644 dhp-rmi-api/src/main/java/eu/dnetlib/enabling/is/lookup/rmi/ISLookUpException.java create mode 100644 dhp-rmi-api/src/main/java/eu/dnetlib/enabling/is/lookup/rmi/ISLookUpService.java create mode 100644 dhp-rmi-api/src/main/java/eu/dnetlib/enabling/is/registry/ISRegistryDocumentNotFoundException.java create mode 100644 dhp-rmi-api/src/main/java/eu/dnetlib/enabling/is/registry/rmi/ISRegistryException.java create mode 100644 dhp-rmi-api/src/main/java/eu/dnetlib/enabling/is/registry/rmi/ISRegistryService.java create mode 100644 dhp-rmi-api/src/main/java/eu/dnetlib/enabling/is/sn/rmi/ISSNException.java create mode 100644 dhp-rmi-api/src/main/java/eu/dnetlib/enabling/is/sn/rmi/ISSNService.java create mode 100644 dhp-rmi-api/src/main/java/eu/dnetlib/enabling/is/sn/rmi/SubscriptionRequestRejectedException.java create mode 100644 dhp-rmi-api/src/main/java/eu/dnetlib/enabling/is/store/rmi/ISStoreException.java create mode 100644 dhp-rmi-api/src/main/java/eu/dnetlib/enabling/is/store/rmi/ISStoreService.java create mode 100644 dhp-rmi-api/src/main/java/eu/dnetlib/enabling/resultset/rmi/ResultSetException.java create mode 100644 dhp-rmi-api/src/main/java/eu/dnetlib/enabling/resultset/rmi/ResultSetService.java delete mode 100644 dhp-shade-package/dependency-reduced-pom.xml diff --git a/.gitignore b/.gitignore index ef9144ae33..caeab2b81e 100644 --- a/.gitignore +++ b/.gitignore @@ -27,5 +27,6 @@ spark-warehouse /**/.factorypath /**/.scalafmt.conf /.java-version -/dhp-shade-package/dependency-reduced-pom.xml +/**/dependency-reduced-pom.xml /**/job.properties + diff --git a/dhp-common/pom.xml b/dhp-common/pom.xml index 9ce93ff275..0c7822bf4b 100644 --- a/dhp-common/pom.xml +++ b/dhp-common/pom.xml @@ -62,6 +62,12 @@ + + eu.dnetlib + dhp-rmi-api + ${project.version} + + edu.cmu secondstring @@ -125,17 +131,6 @@ org.apache.cxf cxf-rt-transports-http - - eu.dnetlib - cnr-rmi-api - - - log4j - log4j - - - - com.ximpleware vtd-xml @@ -174,24 +169,4 @@ opencsv - - - - - spark-34 - - - javax.xml.bind - jaxb-api - 2.2.11 - - - com.sun.xml.ws - jaxws-ri - 2.3.3 - pom - - - - diff --git a/dhp-rmi-api/pom.xml b/dhp-rmi-api/pom.xml new file mode 100644 index 0000000000..2073bd8baf --- /dev/null +++ b/dhp-rmi-api/pom.xml @@ -0,0 +1,34 @@ + + + + eu.dnetlib.dhp + dhp + 1.2.5-SNAPSHOT + ../pom.xml + + 4.0.0 + eu.dnetlib + dhp-rmi-api + jar + 1.2.5-SNAPSHOT + + + org.apache.cxf + cxf-core + ${cxf.version} + + + + org.apache.cxf + cxf-rt-frontend-jaxws + ${cxf.version} + + + + org.glassfish.metro + webservices-rt + ${metro.version} + + + diff --git a/dhp-rmi-api/src/main/java/eu/dnetlib/common/rmi/APIDeprecatedException.java b/dhp-rmi-api/src/main/java/eu/dnetlib/common/rmi/APIDeprecatedException.java new file mode 100644 index 0000000000..409bf71c56 --- /dev/null +++ b/dhp-rmi-api/src/main/java/eu/dnetlib/common/rmi/APIDeprecatedException.java @@ -0,0 +1,11 @@ + +package eu.dnetlib.common.rmi; + +public class APIDeprecatedException extends RuntimeException { + + /** + * + */ + private static final long serialVersionUID = -5606373588445519515L; + +} diff --git a/dhp-rmi-api/src/main/java/eu/dnetlib/common/rmi/BaseService.java b/dhp-rmi-api/src/main/java/eu/dnetlib/common/rmi/BaseService.java new file mode 100644 index 0000000000..0d521796f8 --- /dev/null +++ b/dhp-rmi-api/src/main/java/eu/dnetlib/common/rmi/BaseService.java @@ -0,0 +1,35 @@ + +package eu.dnetlib.common.rmi; + +import jakarta.jws.WebMethod; +import jakarta.jws.WebParam; +import jakarta.jws.WebService; + +@WebService(targetNamespace = "http://services.dnetlib.eu/") +public interface BaseService { + + /** + * All DRIVER services must implement method notify() in order to communicate with the IS_SN + * + * @param subsrciptionId + * @param topic + * @param isId + * @param message + */ + @WebMethod(operationName = "notify") + void notify(@WebParam(name = "subscrId") String subscriptionId, + @WebParam(name = "topic") String topic, + @WebParam(name = "is_id") String isId, + @WebParam(name = "message") String message); + + /** + * Identifies the service's version. Version syntax: ${NAME}-${MAJOR}.${MINOR}.${MICRO}[-${LABEL}] + * + * @return the service's version + */ + @WebMethod(operationName = "identify") + String identify(); + + void start(); + +} diff --git a/dhp-rmi-api/src/main/java/eu/dnetlib/common/rmi/DNetRestDocumentation.java b/dhp-rmi-api/src/main/java/eu/dnetlib/common/rmi/DNetRestDocumentation.java new file mode 100644 index 0000000000..4b9bd6b4fd --- /dev/null +++ b/dhp-rmi-api/src/main/java/eu/dnetlib/common/rmi/DNetRestDocumentation.java @@ -0,0 +1,19 @@ + +package eu.dnetlib.common.rmi; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +/** + * Created by claudio on 30/11/2016. + * to be used in REST controllers, and autodiscovered to build and publish their documentation + */ +@Target({ + ElementType.TYPE +}) +@Retention(RetentionPolicy.RUNTIME) +public @interface DNetRestDocumentation { + +} diff --git a/dhp-rmi-api/src/main/java/eu/dnetlib/common/rmi/RMIException.java b/dhp-rmi-api/src/main/java/eu/dnetlib/common/rmi/RMIException.java new file mode 100644 index 0000000000..bf62b7da33 --- /dev/null +++ b/dhp-rmi-api/src/main/java/eu/dnetlib/common/rmi/RMIException.java @@ -0,0 +1,27 @@ + +package eu.dnetlib.common.rmi; + +/** + * All RMI exception thrown from the service remote method invocation interfaces inherit this class + * + * @author marko + */ +abstract public class RMIException extends Exception { // NOPMD + + /** + * + */ + private static final long serialVersionUID = 428841258652765265L; + + public RMIException(final Throwable exception) { + super(exception); + } + + public RMIException(final String string) { + super(string); + } + + public RMIException(final String string, final Throwable exception) { + super(string, exception); + } +} diff --git a/dhp-rmi-api/src/main/java/eu/dnetlib/common/rmi/UnimplementedException.java b/dhp-rmi-api/src/main/java/eu/dnetlib/common/rmi/UnimplementedException.java new file mode 100644 index 0000000000..d3b6611dfb --- /dev/null +++ b/dhp-rmi-api/src/main/java/eu/dnetlib/common/rmi/UnimplementedException.java @@ -0,0 +1,11 @@ + +package eu.dnetlib.common.rmi; + +public class UnimplementedException extends RuntimeException { + + /** + * + */ + private static final long serialVersionUID = 6040968020696349497L; + +} diff --git a/dhp-rmi-api/src/main/java/eu/dnetlib/data/information/collectionservice/rmi/CollectionService.java b/dhp-rmi-api/src/main/java/eu/dnetlib/data/information/collectionservice/rmi/CollectionService.java new file mode 100644 index 0000000000..6827dcfbec --- /dev/null +++ b/dhp-rmi-api/src/main/java/eu/dnetlib/data/information/collectionservice/rmi/CollectionService.java @@ -0,0 +1,31 @@ + +package eu.dnetlib.data.information.collectionservice.rmi; + +import java.util.List; + +import eu.dnetlib.common.rmi.BaseService; +import jakarta.jws.WebParam; +import jakarta.jws.WebService; + +/** + * The Collection Service is used to ... + */ + +@WebService(targetNamespace = "http://services.dnetlib.eu/") +public interface CollectionService extends BaseService { + + String getCollection(@WebParam(name = "collId") + final String collId) throws CollectionServiceException; + + List getCollections(@WebParam(name = "collIds") + final List collIds) throws CollectionServiceException; + + void updateCollection(@WebParam(name = "coll") + final String coll) throws CollectionServiceException; + + void deleteCollection(@WebParam(name = "collId") + final String collId) throws CollectionServiceException; + + String createCollection(@WebParam(name = "coll") + final String coll) throws CollectionServiceException; +} diff --git a/dhp-rmi-api/src/main/java/eu/dnetlib/data/information/collectionservice/rmi/CollectionServiceException.java b/dhp-rmi-api/src/main/java/eu/dnetlib/data/information/collectionservice/rmi/CollectionServiceException.java new file mode 100644 index 0000000000..1d0fd0bd15 --- /dev/null +++ b/dhp-rmi-api/src/main/java/eu/dnetlib/data/information/collectionservice/rmi/CollectionServiceException.java @@ -0,0 +1,27 @@ + +package eu.dnetlib.data.information.collectionservice.rmi; + +import eu.dnetlib.common.rmi.RMIException; +import jakarta.xml.ws.WebFault; + +@WebFault +public class CollectionServiceException extends RMIException { + + /** + * + */ + private static final long serialVersionUID = 8094008463553904905L; + + public CollectionServiceException(Throwable e) { + super(e); + } + + public CollectionServiceException(String message, Throwable e) { + super(message, e); + } + + public CollectionServiceException(String message) { + super(message); + } + +} diff --git a/dhp-rmi-api/src/main/java/eu/dnetlib/data/information/publisher/rmi/PublisherService.java b/dhp-rmi-api/src/main/java/eu/dnetlib/data/information/publisher/rmi/PublisherService.java new file mode 100644 index 0000000000..f9865c7bca --- /dev/null +++ b/dhp-rmi-api/src/main/java/eu/dnetlib/data/information/publisher/rmi/PublisherService.java @@ -0,0 +1,57 @@ + +package eu.dnetlib.data.information.publisher.rmi; + +import java.util.List; + +import eu.dnetlib.common.rmi.BaseService; +import jakarta.jws.WebMethod; +import jakarta.jws.WebParam; +import jakarta.jws.WebService; +import jakarta.xml.ws.wsaddressing.W3CEndpointReference; + +/** + * Publisher service. Provides access to metadata records and objects. + * + * @author marko + */ +@WebService(targetNamespace = "http://services.dnetlib.eu/") +public interface PublisherService extends BaseService { + + /** + * Get a (metadata) resource by ID. + * + * @param id + * @param format + * @param layout + * @param interpretation + * @return + */ + @WebMethod + String getResourceById(@WebParam(name = "id") + final String id, + @WebParam(name = "format") + final String format, + @WebParam(name = "layout") + final String layout, + @WebParam(name = "interpretation") + final String interpretation); + + /** + * Get (metadata) resources by IDs. + * + * @param ids + * @param format + * @param layout + * @param interpretation + * @return + */ + @WebMethod + W3CEndpointReference getResourcesByIds(@WebParam(name = "ids") + final List ids, + @WebParam(name = "format") + final String format, + @WebParam(name = "layout") + final String layout, + @WebParam(name = "interpretation") + final String interpretation); +} diff --git a/dhp-rmi-api/src/main/java/eu/dnetlib/data/mdstore/DocumentNotFoundException.java b/dhp-rmi-api/src/main/java/eu/dnetlib/data/mdstore/DocumentNotFoundException.java new file mode 100644 index 0000000000..cffb113f20 --- /dev/null +++ b/dhp-rmi-api/src/main/java/eu/dnetlib/data/mdstore/DocumentNotFoundException.java @@ -0,0 +1,30 @@ + +package eu.dnetlib.data.mdstore; + +/** + * Signals that a metadata record cannot be found in a given MDStore. + */ +public class DocumentNotFoundException extends MDStoreServiceException { + + /** + * + */ + private static final long serialVersionUID = 5188036989114250548L; + + public DocumentNotFoundException(final String s, final Throwable e) { + super(s, e); + } + + public DocumentNotFoundException(final String s) { + super(s); + } + + public DocumentNotFoundException(final Throwable e) { + super(e); + } + + public DocumentNotFoundException() { + super(); + } + +} diff --git a/dhp-rmi-api/src/main/java/eu/dnetlib/data/mdstore/MDStoreService.java b/dhp-rmi-api/src/main/java/eu/dnetlib/data/mdstore/MDStoreService.java new file mode 100644 index 0000000000..e59c4e9b7c --- /dev/null +++ b/dhp-rmi-api/src/main/java/eu/dnetlib/data/mdstore/MDStoreService.java @@ -0,0 +1,118 @@ + +package eu.dnetlib.data.mdstore; + +import java.util.List; + +import eu.dnetlib.common.rmi.BaseService; +import jakarta.jws.WebMethod; +import jakarta.jws.WebParam; +import jakarta.jws.WebService; +import jakarta.xml.ws.wsaddressing.W3CEndpointReference; + +@WebService(targetNamespace = "http://services.dnetlib.eu/") +public interface MDStoreService extends BaseService { + + /** + * Identifies service and version. + * + * @return + */ + @Override + String identify(); + + /** + * Returns ResultSet EPR for delivered mdstore records. + * + * @param mdId + * @param from + * @param until + * @param recordFilter REGEX on the metadata record + * @return ResultSet EPR + * @throws MDStoreServiceException + */ + W3CEndpointReference deliverMDRecords(@WebParam(name = "mdId") + final String mdId, + @WebParam(name = "from") + final String from, + @WebParam(name = "until") + final String until, + @WebParam(name = "recordsFilter") + final String recordFilter) throws MDStoreServiceException; + + /** + * Deliver single record from selected mdstore. + * + * @param mdId + * @param recordId + * @return record + * @throws MDStoreServiceException + */ + String deliverRecord(@WebParam(name = "mdId") + final String mdId, @WebParam(name = "recordId") + final String recordId) throws MDStoreServiceException; + + /** + * Returns list of all stored indices. + * + * @return list of all stored indices + */ + List getListOfMDStores() throws MDStoreServiceException; + + List listMDStores(@WebParam(name = "format") + final String format, + @WebParam(name = "layout") + final String layout, + @WebParam(name = "interpretation") + final String interpretation) throws MDStoreServiceException; + + W3CEndpointReference bulkDeliverMDRecords(@WebParam(name = "format") + final String format, + @WebParam(name = "layout") + final String layout, + @WebParam(name = "interpretation") + final String interpretation) throws MDStoreServiceException; + + /** + * Store md records from a result set + * + * @param mdId + * @param rsId + * @param storingType + * @return returns true immediately. + * @throws MDStoreServiceException + */ + @Deprecated + boolean storeMDRecordsFromRS(@WebParam(name = "mdId") + final String mdId, + @WebParam(name = "rsId") + final String rsId, + @WebParam(name = "storingType") + final String storingType) throws MDStoreServiceException; + + /** + * Gets the size of the mdstore with the given identifier. + * + * @param mdId identifier of an mdstore + * @return the number of records in the store + */ + @WebMethod(operationName = "size") + int size(@WebParam(name = "mdId") + final String mdId) throws MDStoreServiceException; + + /** + * Gets the sum of records stored in all mdstore with the given format, layout , interpretation + * + * @param format format + * @param layout layout + * @param interpretation interpretation + * @return the total number of records in the mdstores of the given type + */ + @WebMethod(operationName = "sizeByFormat") + int size(@WebParam(name = "format") + final String format, + @WebParam(name = "layout") + final String layout, + @WebParam(name = "interpretation") + final String interpretation) throws MDStoreServiceException; + +} diff --git a/dhp-rmi-api/src/main/java/eu/dnetlib/data/mdstore/MDStoreServiceException.java b/dhp-rmi-api/src/main/java/eu/dnetlib/data/mdstore/MDStoreServiceException.java new file mode 100644 index 0000000000..471b2a90a5 --- /dev/null +++ b/dhp-rmi-api/src/main/java/eu/dnetlib/data/mdstore/MDStoreServiceException.java @@ -0,0 +1,33 @@ + +package eu.dnetlib.data.mdstore; + +/** + * General mdstore service exception. + * + * @author claudio atzori + * @version 1.0.0 + */ +public class MDStoreServiceException extends Exception { + + /** + * + */ + private static final long serialVersionUID = -6772977735282310658L; + + public MDStoreServiceException(String s, Throwable e) { + super(s, e); + } + + public MDStoreServiceException(String s) { + super(s); + } + + public MDStoreServiceException(Throwable e) { + super(e); + } + + public MDStoreServiceException() { + super(); + } + +} diff --git a/dhp-rmi-api/src/main/java/eu/dnetlib/data/utility/objectpackaging/rmi/ObjectPackagingException.java b/dhp-rmi-api/src/main/java/eu/dnetlib/data/utility/objectpackaging/rmi/ObjectPackagingException.java new file mode 100644 index 0000000000..ade6d9ffc5 --- /dev/null +++ b/dhp-rmi-api/src/main/java/eu/dnetlib/data/utility/objectpackaging/rmi/ObjectPackagingException.java @@ -0,0 +1,28 @@ + +package eu.dnetlib.data.utility.objectpackaging.rmi; + +import eu.dnetlib.common.rmi.RMIException; +import jakarta.xml.ws.WebFault; + +@WebFault +public class ObjectPackagingException extends RMIException { + + private static final long serialVersionUID = 3468254939586031822L; + + /** + * + */ + + public ObjectPackagingException(Throwable e) { + super(e); + } + + public ObjectPackagingException(String message, Throwable e) { + super(message, e); + } + + public ObjectPackagingException(String message) { + super(message); + } + +} diff --git a/dhp-rmi-api/src/main/java/eu/dnetlib/data/utility/objectpackaging/rmi/ObjectPackagingService.java b/dhp-rmi-api/src/main/java/eu/dnetlib/data/utility/objectpackaging/rmi/ObjectPackagingService.java new file mode 100644 index 0000000000..6453cfe1dc --- /dev/null +++ b/dhp-rmi-api/src/main/java/eu/dnetlib/data/utility/objectpackaging/rmi/ObjectPackagingService.java @@ -0,0 +1,37 @@ + +package eu.dnetlib.data.utility.objectpackaging.rmi; + +import java.util.List; + +import eu.dnetlib.common.rmi.BaseService; +import jakarta.jws.WebParam; +import jakarta.jws.WebService; +import jakarta.xml.ws.wsaddressing.W3CEndpointReference; + +/** + * The Object Packaging Service is used to combine the records spread + * into one information package, namely an Object Record. + */ + +@WebService(targetNamespace = "http://services.dnetlib.eu/") +public interface ObjectPackagingService extends BaseService { + /** + * Return the EPR of the resultSet containing the generated packages + * + * @param eprs A list of EPRs used to access the input resultSets. ResultSets MUST be ordered using an order key identified by xpath_ID + * @param xpath_ID A valid xpath, used to access the ordered ID of the elements of the input resultSets. + * @return EPR of the generated resultset + */ + W3CEndpointReference generatePackages(@WebParam(name = "eprs") List eprs, + @WebParam(name = "xpath_ID") String xpath_ID) throws ObjectPackagingException; + + /** + * Return the EPR of the resultSet containing the unpackaged element + * + * @param epr The epr used to access the resultset that contains input packages, packages are xml record in this format: REC1REC2REC3 + * @return EPR of the generated resultset + */ + W3CEndpointReference splitPackages(@WebParam(name = "epr") W3CEndpointReference epr) + throws ObjectPackagingException; + +} diff --git a/dhp-rmi-api/src/main/java/eu/dnetlib/enabling/dlm/rmi/DlmService.java b/dhp-rmi-api/src/main/java/eu/dnetlib/enabling/dlm/rmi/DlmService.java new file mode 100644 index 0000000000..046b78a44a --- /dev/null +++ b/dhp-rmi-api/src/main/java/eu/dnetlib/enabling/dlm/rmi/DlmService.java @@ -0,0 +1,17 @@ + +package eu.dnetlib.enabling.dlm.rmi; + +import eu.dnetlib.common.rmi.BaseService; +import jakarta.jws.WebService; + +/** + * Distributed lock manager. Currently is used mostly to start the underlying lock manager (e.g. zookeeper) and let + * client interface directly with it. + * + *

The DLM service profile contains the entry point of the underlying locking service.

+ * + * @author marko + */ +@WebService(targetNamespace = "http://services.dnetlib.eu/") +public interface DlmService extends BaseService { +} diff --git a/dhp-rmi-api/src/main/java/eu/dnetlib/enabling/hcm/rmi/HostingContextManagerService.java b/dhp-rmi-api/src/main/java/eu/dnetlib/enabling/hcm/rmi/HostingContextManagerService.java new file mode 100644 index 0000000000..3d11ca129b --- /dev/null +++ b/dhp-rmi-api/src/main/java/eu/dnetlib/enabling/hcm/rmi/HostingContextManagerService.java @@ -0,0 +1,19 @@ + +package eu.dnetlib.enabling.hcm.rmi; + +import eu.dnetlib.common.rmi.BaseService; +import jakarta.jws.WebService; + +/** + * Like a HostingNodeManager, but any webapp (web context) can have its own. + *

+ * useful for dispatching notifications shared by all the services local to a single context. + *

+ * + * @author marko + * @author antonis + */ +@WebService(targetNamespace = "http://services.dnetlib.eu/") +public interface HostingContextManagerService extends BaseService { + +} diff --git a/dhp-rmi-api/src/main/java/eu/dnetlib/enabling/hnm/rmi/HostingNodeManagerService.java b/dhp-rmi-api/src/main/java/eu/dnetlib/enabling/hnm/rmi/HostingNodeManagerService.java new file mode 100644 index 0000000000..bd84318180 --- /dev/null +++ b/dhp-rmi-api/src/main/java/eu/dnetlib/enabling/hnm/rmi/HostingNodeManagerService.java @@ -0,0 +1,15 @@ + +package eu.dnetlib.enabling.hnm.rmi; + +import eu.dnetlib.common.rmi.BaseService; +import jakarta.jws.WebParam; +import jakarta.jws.WebService; + +/** + * The HostingNodeManager Service is used to ... + */ + +@WebService(targetNamespace = "http://services.dnetlib.eu/") +public interface HostingNodeManagerService extends BaseService { + String echo(@WebParam(name = "s") String s); +} diff --git a/dhp-rmi-api/src/main/java/eu/dnetlib/enabling/is/lookup/rmi/ISLookUpDocumentNotFoundException.java b/dhp-rmi-api/src/main/java/eu/dnetlib/enabling/is/lookup/rmi/ISLookUpDocumentNotFoundException.java new file mode 100644 index 0000000000..82c3a17505 --- /dev/null +++ b/dhp-rmi-api/src/main/java/eu/dnetlib/enabling/is/lookup/rmi/ISLookUpDocumentNotFoundException.java @@ -0,0 +1,47 @@ + +package eu.dnetlib.enabling.is.lookup.rmi; + +import jakarta.xml.ws.WebFault; + +/** + * Thrown when a given document is not found. + * + * @author marko + */ +@WebFault +public class ISLookUpDocumentNotFoundException extends ISLookUpException { + + /** + * exception chain + message. + * + * @param message message + * @param e + */ + public ISLookUpDocumentNotFoundException(String message, Throwable e) { + super(message, e); + } + + /** + * exception chain constructor. + * + * @param e + */ + public ISLookUpDocumentNotFoundException(Throwable e) { + super(e); + } + + /** + * exception message. + * + * @param message + */ + public ISLookUpDocumentNotFoundException(String message) { + super(message); + } + + /** + * + */ + private static final long serialVersionUID = 2295995755165801937L; + +} diff --git a/dhp-rmi-api/src/main/java/eu/dnetlib/enabling/is/lookup/rmi/ISLookUpException.java b/dhp-rmi-api/src/main/java/eu/dnetlib/enabling/is/lookup/rmi/ISLookUpException.java new file mode 100644 index 0000000000..bd5c26ce7e --- /dev/null +++ b/dhp-rmi-api/src/main/java/eu/dnetlib/enabling/is/lookup/rmi/ISLookUpException.java @@ -0,0 +1,27 @@ + +package eu.dnetlib.enabling.is.lookup.rmi; + +import eu.dnetlib.common.rmi.RMIException; +import jakarta.xml.ws.WebFault; + +@WebFault +public class ISLookUpException extends RMIException { + + /** + * + */ + private static final long serialVersionUID = -5626136963653382533L; + + public ISLookUpException(Throwable e) { + super(e); + } + + public ISLookUpException(String message, Throwable e) { + super(message, e); + } + + public ISLookUpException(String message) { + super(message); + } + +} diff --git a/dhp-rmi-api/src/main/java/eu/dnetlib/enabling/is/lookup/rmi/ISLookUpService.java b/dhp-rmi-api/src/main/java/eu/dnetlib/enabling/is/lookup/rmi/ISLookUpService.java new file mode 100644 index 0000000000..a63ea36a61 --- /dev/null +++ b/dhp-rmi-api/src/main/java/eu/dnetlib/enabling/is/lookup/rmi/ISLookUpService.java @@ -0,0 +1,57 @@ + +package eu.dnetlib.enabling.is.lookup.rmi; + +import java.util.List; + +import eu.dnetlib.common.rmi.BaseService; +import jakarta.jws.WebParam; +import jakarta.jws.WebService; + +@WebService(targetNamespace = "http://services.dnetlib.eu/") +public interface ISLookUpService extends BaseService { + + Boolean flushCachedResultSets(); + + @Deprecated + String getCollection(@WebParam(name = "profId") String profId, @WebParam(name = "format") String format) + throws ISLookUpException; + + String retrieveCollection(@WebParam(name = "profId") String profId) throws ISLookUpException; + + String getResourceProfile(@WebParam(name = "profId") String profId) + throws ISLookUpException; + + String getResourceProfileByQuery(@WebParam(name = "XQuery") String XQuery) + throws ISLookUpException; + + String getResourceQoSParams(@WebParam(name = "id") String id) throws ISLookUpException; + + String getResourceTypeSchema(@WebParam(name = "resourceType") String resourceType) + throws ISLookUpException; + + List listCollections( + @WebParam(name = "format") String format, + @WebParam(name = "idfather") String idfather, + @WebParam(name = "owner") String owner) throws ISLookUpException; + + @Deprecated + List listDHNIDs() throws ISLookUpException; + + List listResourceTypes() throws ISLookUpException; + + @Deprecated + List listServiceIDs(@WebParam(name = "serviceType") String serviceType) throws ISLookUpException; + + @Deprecated + List listServiceTypes() throws ISLookUpException; + + /** + * Like searchProfile(), but bypassing the resultset. Useful for short xquery results. + * + * @param xquery xquery to be executed + * @return list of strings (never null) + * @throws ISLookUpException could happen + */ + List quickSearchProfile(@WebParam(name = "XQuery") String xquery) throws ISLookUpException; + +} diff --git a/dhp-rmi-api/src/main/java/eu/dnetlib/enabling/is/registry/ISRegistryDocumentNotFoundException.java b/dhp-rmi-api/src/main/java/eu/dnetlib/enabling/is/registry/ISRegistryDocumentNotFoundException.java new file mode 100644 index 0000000000..62de1aaabd --- /dev/null +++ b/dhp-rmi-api/src/main/java/eu/dnetlib/enabling/is/registry/ISRegistryDocumentNotFoundException.java @@ -0,0 +1,28 @@ + +package eu.dnetlib.enabling.is.registry; + +import eu.dnetlib.enabling.is.registry.rmi.ISRegistryException; + +public class ISRegistryDocumentNotFoundException extends ISRegistryException { + + /** + * + */ + private static final long serialVersionUID = -1304948213334188538L; + + public ISRegistryDocumentNotFoundException(String string, Throwable e) { + super(string, e); + // TODO Auto-generated constructor stub + } + + public ISRegistryDocumentNotFoundException(String string) { + super(string); + // TODO Auto-generated constructor stub + } + + public ISRegistryDocumentNotFoundException(Throwable e) { + super(e); + // TODO Auto-generated constructor stub + } + +} diff --git a/dhp-rmi-api/src/main/java/eu/dnetlib/enabling/is/registry/rmi/ISRegistryException.java b/dhp-rmi-api/src/main/java/eu/dnetlib/enabling/is/registry/rmi/ISRegistryException.java new file mode 100644 index 0000000000..e7795ab1b7 --- /dev/null +++ b/dhp-rmi-api/src/main/java/eu/dnetlib/enabling/is/registry/rmi/ISRegistryException.java @@ -0,0 +1,25 @@ + +package eu.dnetlib.enabling.is.registry.rmi; + +import eu.dnetlib.common.rmi.RMIException; + +public class ISRegistryException extends RMIException { + + /** + * + */ + private static final long serialVersionUID = -3347405941287624771L; + + public ISRegistryException(Throwable e) { + super(e); + } + + public ISRegistryException(String string) { + super(string); + } + + public ISRegistryException(String string, Throwable e) { + super(string, e); + } + +} diff --git a/dhp-rmi-api/src/main/java/eu/dnetlib/enabling/is/registry/rmi/ISRegistryService.java b/dhp-rmi-api/src/main/java/eu/dnetlib/enabling/is/registry/rmi/ISRegistryService.java new file mode 100644 index 0000000000..3fd10bcbf0 --- /dev/null +++ b/dhp-rmi-api/src/main/java/eu/dnetlib/enabling/is/registry/rmi/ISRegistryService.java @@ -0,0 +1,72 @@ + +package eu.dnetlib.enabling.is.registry.rmi; + +import java.util.List; + +import eu.dnetlib.common.rmi.BaseService; +import jakarta.jws.WebService; + +@WebService(targetNamespace = "http://services.dnetlib.eu/") +public interface ISRegistryService extends BaseService { + + boolean addOrUpdateResourceType(String resourceType, String resourceSchema) throws ISRegistryException; + + boolean addResourceType(String resourceType, String resourceSchema) throws ISRegistryException; + + boolean deleteProfile(String profId) throws ISRegistryException; + + @Deprecated + boolean deleteProfiles(List arrayprofId) throws ISRegistryException; + + /** + * @param resourceType + * @param hierarchical remove subscription topics + * @return + * @throws ISRegistryException + */ + boolean deleteResourceType(String resourceType, Boolean hierarchical) throws ISRegistryException; + + boolean executeXUpdate(String XQuery) throws ISRegistryException; + + String insertProfileForValidation(String resourceType, String resourceProfile) throws ISRegistryException; + + String invalidateProfile(String profId) throws ISRegistryException; + + boolean refreshProfile(String profId, String resourceType) throws ISRegistryException; + + /** + * register a XML Profile. + * + * @param resourceProfile xml profile + * @return profile id + * @throws ISRegistryException + */ + String registerProfile(String resourceProfile) throws ISRegistryException; + + String registerSecureProfile(String resourceProfId, String secureProfId) throws ISRegistryException; + + boolean updateProfile(String profId, String resourceProfile, String resourceType) throws ISRegistryException; + + @Deprecated + String updateProfileDHN(String resourceProfile) throws ISRegistryException; + + boolean addProfileNode(String profId, String xpath, String node) throws ISRegistryException; + + boolean updateProfileNode(String profId, String xpath, String node) throws ISRegistryException; + + boolean removeProfileNode(String profId, String nodeId) throws ISRegistryException; + + @Deprecated + boolean updateRegionDescription(String profId, String resourceProfile) throws ISRegistryException; + + String validateProfile(String profId) throws ISRegistryException; + + @Deprecated + List validateProfiles(List profIds) throws ISRegistryException; + + void addBlackBoardMessage(String profId, String messageId, String message) throws ISRegistryException; + + void replyBlackBoardMessage(String profId, String message) throws ISRegistryException; + + void deleteBlackBoardMessage(String profId, String messageId) throws ISRegistryException; +} diff --git a/dhp-rmi-api/src/main/java/eu/dnetlib/enabling/is/sn/rmi/ISSNException.java b/dhp-rmi-api/src/main/java/eu/dnetlib/enabling/is/sn/rmi/ISSNException.java new file mode 100644 index 0000000000..c81470911f --- /dev/null +++ b/dhp-rmi-api/src/main/java/eu/dnetlib/enabling/is/sn/rmi/ISSNException.java @@ -0,0 +1,25 @@ + +package eu.dnetlib.enabling.is.sn.rmi; + +import eu.dnetlib.common.rmi.RMIException; + +public class ISSNException extends RMIException { + + /** + * + */ + private static final long serialVersionUID = -7384073901457430004L; + + public ISSNException(final Throwable e) { + super(e); + } + + public ISSNException(final String message) { + super(message); + } + + public ISSNException(final String message, final Throwable e) { + super(message, e); + } + +} diff --git a/dhp-rmi-api/src/main/java/eu/dnetlib/enabling/is/sn/rmi/ISSNService.java b/dhp-rmi-api/src/main/java/eu/dnetlib/enabling/is/sn/rmi/ISSNService.java new file mode 100644 index 0000000000..c53c4eda17 --- /dev/null +++ b/dhp-rmi-api/src/main/java/eu/dnetlib/enabling/is/sn/rmi/ISSNService.java @@ -0,0 +1,121 @@ + +package eu.dnetlib.enabling.is.sn.rmi; + +import java.util.List; + +import eu.dnetlib.common.rmi.BaseService; +import jakarta.jws.WebParam; +import jakarta.jws.WebService; +import jakarta.xml.ws.wsaddressing.W3CEndpointReference; + +@WebService(targetNamespace = "http://services.dnetlib.eu/") +public interface ISSNService extends BaseService { + + /** + * fossil. + * + * @param topic + * @return + * @throws ISSNException + */ + String getCurrentMessage(@WebParam(name = "topic") String topic) throws ISSNException; + + /** + * puts a subcription in a paused state. paused subscription are not notified even when triggered. + * + * @param subscrId subscription identifier + * @return returns false if the subscription is already paused. + * @throws ISSNException may happen + */ + boolean pauseSubscription(@WebParam(name = "subscrId") String subscrId) throws ISSNException; + + /** + * Used to renew the subscription before it expires. + * + *

+ * In practice it resets the ttl to another value, so it can be used to reset a infinte ttl subscription to a finite + * value. + *

+ * + * @param subscrId subscription id + * @param terminationTime new ttl (from now), or 0 (infinite) + * @return true if successful + * @throws ISSNException may happen + */ + boolean renew(@WebParam(name = "subscrId") String subscrId, @WebParam(name = "terminationTime") int terminationTime) + throws ISSNException; + + /** + * resumes a paused subscription. + * + * @param subscrId subscription id + * @return true if resumed. false if it was not paused. + * @throws ISSNException may happen + */ + boolean resumeSubscription(@WebParam(name = "subscrId") String subscrId) throws ISSNException; + + /** + * @param consumerReference epr to be called when the notification is triggered + * @param topicExpression topic expression to register + * @param initialTerminationTime ttl in seconds (0 = infinite) + * @return subscription id + * @throws ISSNException may happen + */ + String subscribe( + @WebParam(name = "consumerReference") W3CEndpointReference consumerReference, + @WebParam(name = "topicExpression") String topicExpression, + @WebParam(name = "initialTerminationTime") int initialTerminationTime) + throws ISSNException; + + boolean unsubscribe(@WebParam(name = "subscrId") String subscrId) throws ISSNException; + + /** + * fossil. + * + * @param resourceType + * @param profileId + * @param profile + * @return + * @throws ISSNException + */ + boolean actionCreatePerformed( + @WebParam(name = "resourceType") String resourceType, + @WebParam(name = "profileId") String profileId, + @WebParam(name = "profile") String profile) throws ISSNException; + + /** + * fossil. + * + * @param resourceType + * @param profileId + * @param profileBefore + * @param profileAfter + * @return + * @throws ISSNException + */ + boolean actionUpdatePerformed( + @WebParam(name = "resourceType") String resourceType, + @WebParam(name = "profileId") String profileId, + @WebParam(name = "profileBefore") String profileBefore, + @WebParam(name = "profileAfter") String profileAfter) throws ISSNException; + + /** + * fossil. + * + * @param resourceType + * @param profileId + * @return + * @throws ISSNException + */ + boolean actionDeletePerformed(@WebParam(name = "resourceType") String resourceType, + @WebParam(name = "profileId") String profileId) + throws ISSNException; + + /** + * list all subscriptions. Mostly for debug reasons. + * + * @return list of subscription ids. + */ + List listSubscriptions(); + +} diff --git a/dhp-rmi-api/src/main/java/eu/dnetlib/enabling/is/sn/rmi/SubscriptionRequestRejectedException.java b/dhp-rmi-api/src/main/java/eu/dnetlib/enabling/is/sn/rmi/SubscriptionRequestRejectedException.java new file mode 100644 index 0000000000..75928092ee --- /dev/null +++ b/dhp-rmi-api/src/main/java/eu/dnetlib/enabling/is/sn/rmi/SubscriptionRequestRejectedException.java @@ -0,0 +1,21 @@ + +package eu.dnetlib.enabling.is.sn.rmi; + +/** + * Thrown when a subscription request is rejected. + * + * @author claudio + */ +public class SubscriptionRequestRejectedException extends ISSNException { + + /** + * + */ + private static final long serialVersionUID = 263095606953662098L; + + public SubscriptionRequestRejectedException(String message) { + super(message); + // TODO Auto-generated constructor stub + } + +} diff --git a/dhp-rmi-api/src/main/java/eu/dnetlib/enabling/is/store/rmi/ISStoreException.java b/dhp-rmi-api/src/main/java/eu/dnetlib/enabling/is/store/rmi/ISStoreException.java new file mode 100644 index 0000000000..4f9530dfa8 --- /dev/null +++ b/dhp-rmi-api/src/main/java/eu/dnetlib/enabling/is/store/rmi/ISStoreException.java @@ -0,0 +1,25 @@ + +package eu.dnetlib.enabling.is.store.rmi; + +import eu.dnetlib.common.rmi.RMIException; + +public class ISStoreException extends RMIException { + + /** + * + */ + private static final long serialVersionUID = 8683126829156096420L; + + public ISStoreException(Throwable e) { + super(e); + } + + public ISStoreException(String message, Throwable e) { + super(message, e); + } + + public ISStoreException(String message) { + super(message); + } + +} diff --git a/dhp-rmi-api/src/main/java/eu/dnetlib/enabling/is/store/rmi/ISStoreService.java b/dhp-rmi-api/src/main/java/eu/dnetlib/enabling/is/store/rmi/ISStoreService.java new file mode 100644 index 0000000000..4b92b69c91 --- /dev/null +++ b/dhp-rmi-api/src/main/java/eu/dnetlib/enabling/is/store/rmi/ISStoreService.java @@ -0,0 +1,47 @@ + +package eu.dnetlib.enabling.is.store.rmi; + +import java.util.List; + +import eu.dnetlib.common.rmi.BaseService; +import jakarta.jws.WebParam; +import jakarta.jws.WebService; + +@WebService(targetNamespace = "http://services.dnetlib.eu/") +public interface ISStoreService extends BaseService { + + boolean createFileColl(@WebParam(name = "fileColl") String fileColl) throws ISStoreException; + + boolean deleteFileColl(@WebParam(name = "fileColl") String fileColl) throws ISStoreException; + + boolean deleteXML(@WebParam(name = "fileName") String fileName, @WebParam(name = "fileColl") String fileColl) + throws ISStoreException; + + boolean executeXUpdate(@WebParam(name = "query") String query) throws ISStoreException; + + List getFileColls() throws ISStoreException; + + List getFileNames(@WebParam(name = "fileColl") String fileColl) throws ISStoreException; + + String getXML(@WebParam(name = "fileName") String fileName, @WebParam(name = "fileColl") String fileColl) + throws ISStoreException; + + String getXMLbyQuery(@WebParam(name = "query") String query) throws ISStoreException; + + boolean insertXML(@WebParam(name = "fileName") String fileName, @WebParam(name = "fileColl") String fileColl, + @WebParam(name = "file") String file) + throws ISStoreException; + + boolean reindex(); + + List quickSearchXML(@WebParam(name = "query") String query) throws ISStoreException; + + boolean sync(); + + boolean updateXML(@WebParam(name = "fileName") String fileName, @WebParam(name = "fileColl") String fileColl, + @WebParam(name = "file") String file) + throws ISStoreException; + + String backup() throws ISStoreException; + +} diff --git a/dhp-rmi-api/src/main/java/eu/dnetlib/enabling/resultset/rmi/ResultSetException.java b/dhp-rmi-api/src/main/java/eu/dnetlib/enabling/resultset/rmi/ResultSetException.java new file mode 100644 index 0000000000..f376a46256 --- /dev/null +++ b/dhp-rmi-api/src/main/java/eu/dnetlib/enabling/resultset/rmi/ResultSetException.java @@ -0,0 +1,22 @@ + +package eu.dnetlib.enabling.resultset.rmi; + +import eu.dnetlib.common.rmi.RMIException; + +public class ResultSetException extends RMIException { + + /** + * + */ + private static final long serialVersionUID = -7130554407601059627L; + + public ResultSetException(Throwable e) { + super(e); + // TODO Auto-generated constructor stub + } + + public ResultSetException(String string) { + super(string); + } + +} diff --git a/dhp-rmi-api/src/main/java/eu/dnetlib/enabling/resultset/rmi/ResultSetService.java b/dhp-rmi-api/src/main/java/eu/dnetlib/enabling/resultset/rmi/ResultSetService.java new file mode 100644 index 0000000000..10079c9072 --- /dev/null +++ b/dhp-rmi-api/src/main/java/eu/dnetlib/enabling/resultset/rmi/ResultSetService.java @@ -0,0 +1,135 @@ + +package eu.dnetlib.enabling.resultset.rmi; + +import java.util.List; + +import eu.dnetlib.common.rmi.BaseService; +import jakarta.jws.WebMethod; +import jakarta.jws.WebParam; +import jakarta.jws.WebService; +import jakarta.xml.ws.wsaddressing.W3CEndpointReference; + +/** + * ResultSet service interface. + *

+ * TODO: implement other compatibility methods as needed. + * + * @author marko + */ +@WebService(targetNamespace = "http://services.dnetlib.eu/") +public interface ResultSetService extends BaseService { + /** + * create a new pull rs. + * + * @param bdId bulk data identifier + * @param initialPageSize page size for the polling on the server side. + * @param expiryTime RS expiry time + * @return + */ + W3CEndpointReference createPullRSEPR( + @WebParam(name = "dataProviderServiceAddress") W3CEndpointReference dataProviderEPR, + @WebParam(name = "bdId") String bdId, + @WebParam(name = "initialPageSize") int initialPageSize, + @WebParam(name = "expiryTime") int expiryTime, + @WebParam(name = "styleSheet") String styleSheet, + @WebParam(name = "keepAliveTime") Integer keepAliveTime, + @WebParam(name = "total") Integer total); + + /** + * create a new pull rs. + *

+ * compatibility version + * + * @param bdId bulk data identifier + * @param initialPageSize page size for the polling on the server side. + * @param expiryTime RS expiry time + * @return + */ + W3CEndpointReference createPullRS( + @WebParam(name = "dataProviderServiceAddress") String dataProviderServiceAddress, + @WebParam(name = "bdId") String bdId, + @WebParam(name = "initialPageSize") int initialPageSize, + @WebParam(name = "expiryTime") int expiryTime, + @WebParam(name = "styleSheet") String styleSheet, + @WebParam(name = "keepAliveTime") Integer keepAliveTime, + @WebParam(name = "total") Integer total); + + /** + * close a result set. A closed resultset is guaranteed not to grow. + * + * @param rsId + */ + void closeRS(@WebParam(name = "rsId") String rsId); + + /** + * get one 'page' of results. + *

+ * TODO: define how results are returned when the range is not present in the result set. + * + * @param fromPosition counting from 1 + * @param toPosition included + * @param requestMode + * @return a page of data + */ + List getResult( + @WebParam(name = "rsId") String rsId, + @WebParam(name = "fromPosition") int fromPosition, + @WebParam(name = "toPosition") int toPosition, + @WebParam(name = "requestMode") String requestMode) throws ResultSetException; + + /** + * get the number of result elements present in the resultset. + * + * @param rsId result set identifier + * @return number of results available in the resultset + * @throws ResultSetException + */ + int getNumberOfElements(@WebParam(name = "rsId") String rsId) throws ResultSetException; + + /** + * create a new push resultset. + * + * @param expiryTime RS expiry time + * @param keepAliveTime keep alive time + * @return epr of new resultset + * @throws ResultSetException + */ + W3CEndpointReference createPushRS(@WebParam(name = "expiryTime") int expiryTime, + @WebParam(name = "keepAliveTime") int keepAliveTime) + throws ResultSetException; + + /** + * add new data to a push resultset. + * + * @param rsId resultset id + * @param elements list of elements to be addded + * @return dummy value + * @throws ResultSetException + */ + String populateRS(@WebParam(name = "rsId") String rsId, @WebParam(name = "elements") List elements) + throws ResultSetException; + + /** + * return current status of a resultset. + * + * @param rsId resultset id + * @return status + * @throws ResultSetException + */ + String getRSStatus(@WebParam(name = "rsId") String rsId) throws ResultSetException; + + /** + * read a resultset property. + * + * @param rsId resultset id + * @param name property value + * @return property value + * @throws ResultSetException + */ + String getProperty(@WebParam(name = "rsId") String rsId, @WebParam(name = "name") String name) + throws ResultSetException; + + @WebMethod(operationName = "identify") + String identify(); + +} diff --git a/dhp-shade-package/dependency-reduced-pom.xml b/dhp-shade-package/dependency-reduced-pom.xml deleted file mode 100644 index 04843072f7..0000000000 --- a/dhp-shade-package/dependency-reduced-pom.xml +++ /dev/null @@ -1,113 +0,0 @@ - - - - dhp - eu.dnetlib.dhp - 1.2.5-SNAPSHOT - - 4.0.0 - dhp-shade-package - This module create a jar of all module dependencies - - - - maven-shade-plugin - - - package - - shade - - - - - eu.dnetlib.dhp.oa.dedup.SparkCreateSimRels - - - - META-INF/cxf/bus-extensions.txt - - - - - *:* - - META-INF/maven/** - META-INF/*.SF - META-INF/*.DSA - META-INF/*.RSA - - - - - - com - repackaged.com.google.common - - com.google.common.** - - - - - - - - - - - - org.projectlombok - lombok - 1.18.28 - provided - - - org.junit.jupiter - junit-jupiter - 5.6.1 - test - - - junit-jupiter-api - org.junit.jupiter - - - junit-jupiter-params - org.junit.jupiter - - - junit-jupiter-engine - org.junit.jupiter - - - - - org.mockito - mockito-core - 3.3.3 - test - - - byte-buddy - net.bytebuddy - - - byte-buddy-agent - net.bytebuddy - - - - - org.mockito - mockito-junit-jupiter - 3.3.3 - test - - - - - DHPSite - ${dhp.site.stage.path}/dhp-common - - - diff --git a/dhp-shade-package/pom.xml b/dhp-shade-package/pom.xml index 42735f9d7d..fe3b3c0d21 100644 --- a/dhp-shade-package/pom.xml +++ b/dhp-shade-package/pom.xml @@ -25,7 +25,6 @@ - diff --git a/dhp-workflows/pom.xml b/dhp-workflows/pom.xml index 4c3f7509e6..682e62c921 100644 --- a/dhp-workflows/pom.xml +++ b/dhp-workflows/pom.xml @@ -40,7 +40,6 @@ dhp-usage-stats-build dhp-usage-raw-data-update dhp-broker-events - dhp-doiboost dhp-impact-indicators dhp-swh dhp-incremental-graph @@ -95,6 +94,12 @@ + + spark-24 + + dhp-doiboost + + attach-test-resources diff --git a/pom.xml b/pom.xml index 033d88b0be..9d0cac0e53 100644 --- a/pom.xml +++ b/pom.xml @@ -19,6 +19,7 @@ + dhp-rmi-api dhp-build dhp-pace-core dhp-common @@ -440,12 +441,6 @@ provided - - eu.dnetlib - cnr-rmi-api - ${cnr-rmi-api.version} - - eu.dnetlib.dhp dnet-openaire-broker-common @@ -926,7 +921,6 @@ 4.1.2 - [2.6.1] 1.20 1.8 1.8 @@ -936,6 +930,7 @@ 2.4 1.1.3 1.7 + 3.5.9 1.0.7 [10.0.0] cdh5.9.2 @@ -953,6 +948,7 @@ 3.5.3 4.13.0 5.6.1 + 3.0.3 3.3.3 3.4.2 4.7.2 @@ -1036,6 +1032,7 @@ 4.8.1 + 3.3.4 1.23.0 1.8 1.10.0 @@ -1049,7 +1046,7 @@ 14.0.1 8.11.0 4.0.4 - 3.5.1.openaire-SNAPSHOT + 3.5.3 2.15.2 3.12.0 2.20.0 @@ -1067,6 +1064,10 @@ [11 + + 4.0.5 + 4.0.4 + From 79e4728afdf119822661b63d8c606794234f35e9 Mon Sep 17 00:00:00 2001 From: Giambattista Bloisi Date: Fri, 20 Dec 2024 15:24:35 +0100 Subject: [PATCH 6/7] Use json loading in place of text loading + jackson mapper --- dhp-shade-package/pom.xml | 10 +++++----- .../dhp/oa/dedup/AbstractSparkAction.java | 9 ++++----- .../dhp/oa/dedup/SparkCopyOpenorgsMergeRels.java | 16 +++++++--------- .../dhp/oa/dedup/SparkCopyOpenorgsSimRels.java | 4 +++- .../oa/dedup/SparkCopyRelationsNoOpenorgs.java | 4 +++- .../dhp/oa/dedup/SparkPrepareNewOrgs.java | 4 +++- .../dhp/oa/dedup/SparkPrepareOrgRels.java | 4 +++- 7 files changed, 28 insertions(+), 23 deletions(-) diff --git a/dhp-shade-package/pom.xml b/dhp-shade-package/pom.xml index fe3b3c0d21..be0563ef13 100644 --- a/dhp-shade-package/pom.xml +++ b/dhp-shade-package/pom.xml @@ -45,11 +45,11 @@ - - - - - + + eu.dnetlib.dhp + dhp-dedup-openaire + ${project.version} + diff --git a/dhp-workflows/dhp-dedup-openaire/src/main/java/eu/dnetlib/dhp/oa/dedup/AbstractSparkAction.java b/dhp-workflows/dhp-dedup-openaire/src/main/java/eu/dnetlib/dhp/oa/dedup/AbstractSparkAction.java index 0af7bb6d01..7211c46f5d 100644 --- a/dhp-workflows/dhp-dedup-openaire/src/main/java/eu/dnetlib/dhp/oa/dedup/AbstractSparkAction.java +++ b/dhp-workflows/dhp-dedup-openaire/src/main/java/eu/dnetlib/dhp/oa/dedup/AbstractSparkAction.java @@ -128,13 +128,12 @@ abstract class AbstractSparkAction implements Serializable { .collect(Collectors.joining(SP_SEPARATOR)); } - protected static MapFunction patchRelFn() { + protected static MapFunction patchRelFn() { return value -> { - final Relation rel = OBJECT_MAPPER.readValue(value, Relation.class); - if (rel.getDataInfo() == null) { - rel.setDataInfo(new DataInfo()); + if (value.getDataInfo() == null) { + value.setDataInfo(new DataInfo()); } - return rel; + return value; }; } diff --git a/dhp-workflows/dhp-dedup-openaire/src/main/java/eu/dnetlib/dhp/oa/dedup/SparkCopyOpenorgsMergeRels.java b/dhp-workflows/dhp-dedup-openaire/src/main/java/eu/dnetlib/dhp/oa/dedup/SparkCopyOpenorgsMergeRels.java index eca2193af2..8eb4de5f82 100644 --- a/dhp-workflows/dhp-dedup-openaire/src/main/java/eu/dnetlib/dhp/oa/dedup/SparkCopyOpenorgsMergeRels.java +++ b/dhp-workflows/dhp-dedup-openaire/src/main/java/eu/dnetlib/dhp/oa/dedup/SparkCopyOpenorgsMergeRels.java @@ -68,22 +68,20 @@ public class SparkCopyOpenorgsMergeRels extends AbstractSparkAction { final String relationPath = DedupUtility.createEntityPath(graphBasePath, "relation"); // collect organization merge relations from openorgs database - JavaRDD mergeRelsRDD = spark + Dataset relations = spark .read() - .textFile(relationPath) + .schema(Encoders.bean(Relation.class).schema()) + .json(relationPath) + .as(Encoders.bean(Relation.class)) .map(patchRelFn(), Encoders.bean(Relation.class)) - .toJavaRDD() .filter(this::isOpenorgs) // take only openorgs relations .filter(this::isMergeRel); // take merges and isMergedIn relations - log.info("Number of Openorgs Merge Relations collected: {}", mergeRelsRDD.count()); - - final Dataset relations = spark - .createDataset( - mergeRelsRDD.rdd(), - Encoders.bean(Relation.class)); + relations.cache(); + log.info("Number of Openorgs Merge Relations collected: {}", relations.count()); saveParquet(relations, outputPath, SaveMode.Append); + relations.unpersist(); } private boolean isMergeRel(Relation rel) { diff --git a/dhp-workflows/dhp-dedup-openaire/src/main/java/eu/dnetlib/dhp/oa/dedup/SparkCopyOpenorgsSimRels.java b/dhp-workflows/dhp-dedup-openaire/src/main/java/eu/dnetlib/dhp/oa/dedup/SparkCopyOpenorgsSimRels.java index 93027e99ab..a166549f28 100644 --- a/dhp-workflows/dhp-dedup-openaire/src/main/java/eu/dnetlib/dhp/oa/dedup/SparkCopyOpenorgsSimRels.java +++ b/dhp-workflows/dhp-dedup-openaire/src/main/java/eu/dnetlib/dhp/oa/dedup/SparkCopyOpenorgsSimRels.java @@ -69,7 +69,9 @@ public class SparkCopyOpenorgsSimRels extends AbstractSparkAction { Dataset rawRels = spark .read() - .textFile(relationPath) + .schema(Encoders.bean(Relation.class).schema()) + .json(relationPath) + .as(Encoders.bean(Relation.class)) .map(patchRelFn(), Encoders.bean(Relation.class)) .filter(this::filterOpenorgsRels); diff --git a/dhp-workflows/dhp-dedup-openaire/src/main/java/eu/dnetlib/dhp/oa/dedup/SparkCopyRelationsNoOpenorgs.java b/dhp-workflows/dhp-dedup-openaire/src/main/java/eu/dnetlib/dhp/oa/dedup/SparkCopyRelationsNoOpenorgs.java index e10f41c823..308e397e14 100644 --- a/dhp-workflows/dhp-dedup-openaire/src/main/java/eu/dnetlib/dhp/oa/dedup/SparkCopyRelationsNoOpenorgs.java +++ b/dhp-workflows/dhp-dedup-openaire/src/main/java/eu/dnetlib/dhp/oa/dedup/SparkCopyRelationsNoOpenorgs.java @@ -58,7 +58,9 @@ public class SparkCopyRelationsNoOpenorgs extends AbstractSparkAction { JavaRDD simRels = spark .read() - .textFile(relationPath) + .schema(Encoders.bean(Relation.class).schema()) + .json(relationPath) + .as(Encoders.bean(Relation.class)) .map(patchRelFn(), Encoders.bean(Relation.class)) .toJavaRDD() .filter(x -> !isOpenorgsDedupRel(x)); diff --git a/dhp-workflows/dhp-dedup-openaire/src/main/java/eu/dnetlib/dhp/oa/dedup/SparkPrepareNewOrgs.java b/dhp-workflows/dhp-dedup-openaire/src/main/java/eu/dnetlib/dhp/oa/dedup/SparkPrepareNewOrgs.java index 0507b7b9af..39f86713da 100644 --- a/dhp-workflows/dhp-dedup-openaire/src/main/java/eu/dnetlib/dhp/oa/dedup/SparkPrepareNewOrgs.java +++ b/dhp-workflows/dhp-dedup-openaire/src/main/java/eu/dnetlib/dhp/oa/dedup/SparkPrepareNewOrgs.java @@ -111,7 +111,9 @@ public class SparkPrepareNewOrgs extends AbstractSparkAction { // collect diffrels from the raw graph relations: JavaPairRDD diffRels = spark .read() - .textFile(relationPath) + .schema(Encoders.bean(Relation.class).schema()) + .json(relationPath) + .as(Encoders.bean(Relation.class)) .map(patchRelFn(), Encoders.bean(Relation.class)) .toJavaRDD() .filter(r -> filterRels(r, ModelSupport.getMainType(EntityType.organization))) diff --git a/dhp-workflows/dhp-dedup-openaire/src/main/java/eu/dnetlib/dhp/oa/dedup/SparkPrepareOrgRels.java b/dhp-workflows/dhp-dedup-openaire/src/main/java/eu/dnetlib/dhp/oa/dedup/SparkPrepareOrgRels.java index 83ec7e5222..1ab7f739fa 100644 --- a/dhp-workflows/dhp-dedup-openaire/src/main/java/eu/dnetlib/dhp/oa/dedup/SparkPrepareOrgRels.java +++ b/dhp-workflows/dhp-dedup-openaire/src/main/java/eu/dnetlib/dhp/oa/dedup/SparkPrepareOrgRels.java @@ -133,7 +133,9 @@ public class SparkPrepareOrgRels extends AbstractSparkAction { // collect diffrels from the raw graph relations: <, "diffRel"> JavaRDD, String>> diffRels = spark .read() - .textFile(relationPath) + .schema(Encoders.bean(Relation.class).schema()) + .json(relationPath) + .as(Encoders.bean(Relation.class)) .map(patchRelFn(), Encoders.bean(Relation.class)) .toJavaRDD() .filter(r -> filterRels(r, "organization")) From 357ec7414cc11e83df241ddf067d1046ceeeb988 Mon Sep 17 00:00:00 2001 From: Giambattista Bloisi Date: Fri, 20 Dec 2024 15:25:36 +0100 Subject: [PATCH 7/7] Remove global kryo serializer --- .../java/eu/dnetlib/dhp/oa/merge/GroupEntitiesSparkJob.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/dhp-common/src/main/java/eu/dnetlib/dhp/oa/merge/GroupEntitiesSparkJob.java b/dhp-common/src/main/java/eu/dnetlib/dhp/oa/merge/GroupEntitiesSparkJob.java index 98ec09277d..5ce2032413 100644 --- a/dhp-common/src/main/java/eu/dnetlib/dhp/oa/merge/GroupEntitiesSparkJob.java +++ b/dhp-common/src/main/java/eu/dnetlib/dhp/oa/merge/GroupEntitiesSparkJob.java @@ -85,8 +85,6 @@ public class GroupEntitiesSparkJob { log.info("filterInvisible: {}", filterInvisible); SparkConf conf = new SparkConf(); - conf.set("spark.serializer", "org.apache.spark.serializer.KryoSerializer"); - conf.registerKryoClasses(ModelSupport.getOafModelClasses()); final VocabularyGroup vocs = VocabularyGroup.loadVocsFromIS(isLookUpService);