diff --git a/dhp-common/src/main/java/eu/dnetlib/data/mdstore/manager/common/model/MDStore.java b/dhp-common/src/main/java/eu/dnetlib/data/mdstore/manager/common/model/MDStore.java deleted file mode 100644 index 59fe941ed..000000000 --- a/dhp-common/src/main/java/eu/dnetlib/data/mdstore/manager/common/model/MDStore.java +++ /dev/null @@ -1,182 +0,0 @@ - -package eu.dnetlib.data.mdstore.manager.common.model; - -import java.io.Serializable; -import java.util.Date; -import java.util.Objects; -import java.util.UUID; - -import javax.persistence.Column; -import javax.persistence.Entity; -import javax.persistence.Id; -import javax.persistence.Table; -import javax.persistence.Temporal; -import javax.persistence.TemporalType; - -@Entity -@Table(name = "mdstores") -public class MDStore implements Serializable { - - /** */ - private static final long serialVersionUID = 3160530489149700055L; - - @Id - @Column(name = "id") - private String id; - - @Column(name = "format") - private String format; - - @Column(name = "layout") - private String layout; - - @Column(name = "interpretation") - private String interpretation; - - @Column(name = "datasource_name") - private String datasourceName; - - @Column(name = "datasource_id") - private String datasourceId; - - @Column(name = "api_id") - private String apiId; - - @Column(name = "hdfs_path") - private String hdfsPath; - - @Column(name = "creation_date") - @Temporal(TemporalType.TIMESTAMP) - private Date creationDate; - - public String getId() { - return id; - } - - public void setId(final String id) { - this.id = id; - } - - public String getFormat() { - return format; - } - - public void setFormat(final String format) { - this.format = format; - } - - public String getLayout() { - return layout; - } - - public void setLayout(final String layout) { - this.layout = layout; - } - - public String getInterpretation() { - return interpretation; - } - - public void setInterpretation(final String interpretation) { - this.interpretation = interpretation; - } - - public String getDatasourceName() { - return datasourceName; - } - - public void setDatasourceName(final String datasourceName) { - this.datasourceName = datasourceName; - } - - public String getDatasourceId() { - return datasourceId; - } - - public void setDatasourceId(final String datasourceId) { - this.datasourceId = datasourceId; - } - - public String getApiId() { - return apiId; - } - - public void setApiId(final String apiId) { - this.apiId = apiId; - } - - public String getHdfsPath() { - return hdfsPath; - } - - public void setHdfsPath(final String hdfsPath) { - this.hdfsPath = hdfsPath; - } - - public Date getCreationDate() { - return creationDate; - } - - public void setCreationDate(final Date creationDate) { - this.creationDate = creationDate; - } - - public static MDStore newInstance( - final String format, - final String layout, - final String interpretation, - final String hdfsBasePath) { - return newInstance(format, layout, interpretation, null, null, null, hdfsBasePath); - } - - public static MDStore newInstance( - final String format, - final String layout, - final String interpretation, - final String dsName, - final String dsId, - final String apiId, - final String hdfsBasePath) { - - final String mdId = "md-" + UUID.randomUUID(); - - final MDStore md = new MDStore(); - md.setId(mdId); - md.setFormat(format); - md.setLayout(layout); - md.setInterpretation(interpretation); - md.setCreationDate(new Date()); - md.setDatasourceName(dsName); - md.setDatasourceId(dsId); - md.setApiId(apiId); - md.setHdfsPath(String.format("%s/%s", hdfsBasePath, mdId)); - - return md; - } - - @Override - public String toString() { - return String - .format( - "MDStore [id=%s, format=%s, layout=%s, interpretation=%s, datasourceName=%s, datasourceId=%s, apiId=%s, hdfsPath=%s, creationDate=%s]", - id, format, layout, interpretation, datasourceName, datasourceId, apiId, hdfsPath, creationDate); - } - - @Override - public int hashCode() { - return Objects.hash(id); - } - - @Override - public boolean equals(final Object obj) { - if (this == obj) { - return true; - } - if (!(obj instanceof MDStore)) { - return false; - } - final MDStore other = (MDStore) obj; - return Objects.equals(id, other.id); - } - -} diff --git a/dhp-common/src/main/java/eu/dnetlib/data/mdstore/manager/common/model/MDStoreCurrentVersion.java b/dhp-common/src/main/java/eu/dnetlib/data/mdstore/manager/common/model/MDStoreCurrentVersion.java deleted file mode 100644 index d808e2de7..000000000 --- a/dhp-common/src/main/java/eu/dnetlib/data/mdstore/manager/common/model/MDStoreCurrentVersion.java +++ /dev/null @@ -1,74 +0,0 @@ - -package eu.dnetlib.data.mdstore.manager.common.model; - -import java.io.Serializable; -import java.util.Objects; - -import javax.persistence.Column; -import javax.persistence.Entity; -import javax.persistence.Id; -import javax.persistence.Table; - -@Entity -@Table(name = "mdstore_current_versions") -public class MDStoreCurrentVersion implements Serializable { - - /** */ - private static final long serialVersionUID = -4757725888593745773L; - - @Id - @Column(name = "mdstore") - private String mdstore; - - @Column(name = "current_version") - private String currentVersion; - - public String getMdstore() { - return mdstore; - } - - public void setMdstore(final String mdstore) { - this.mdstore = mdstore; - } - - public String getCurrentVersion() { - return currentVersion; - } - - public void setCurrentVersion(final String currentVersion) { - this.currentVersion = currentVersion; - } - - public static MDStoreCurrentVersion newInstance(final String mdId, final String versionId) { - final MDStoreCurrentVersion cv = new MDStoreCurrentVersion(); - cv.setMdstore(mdId); - cv.setCurrentVersion(versionId); - return cv; - } - - public static MDStoreCurrentVersion newInstance(final MDStoreVersion v) { - return newInstance(v.getMdstore(), v.getId()); - } - - @Override - public String toString() { - return String.format("MDStoreCurrentVersion [mdstore=%s, currentVersion=%s]", mdstore, currentVersion); - } - - @Override - public int hashCode() { - return Objects.hash(currentVersion, mdstore); - } - - @Override - public boolean equals(final Object obj) { - if (this == obj) { - return true; - } - if (!(obj instanceof MDStoreCurrentVersion)) { - return false; - } - final MDStoreCurrentVersion other = (MDStoreCurrentVersion) obj; - return Objects.equals(currentVersion, other.currentVersion) && Objects.equals(mdstore, other.mdstore); - } -} diff --git a/dhp-common/src/main/java/eu/dnetlib/data/mdstore/manager/common/model/MDStoreVersion.java b/dhp-common/src/main/java/eu/dnetlib/data/mdstore/manager/common/model/MDStoreVersion.java deleted file mode 100644 index 38f8f275e..000000000 --- a/dhp-common/src/main/java/eu/dnetlib/data/mdstore/manager/common/model/MDStoreVersion.java +++ /dev/null @@ -1,140 +0,0 @@ - -package eu.dnetlib.data.mdstore.manager.common.model; - -import java.io.Serializable; -import java.util.Date; -import java.util.Objects; - -import javax.persistence.Column; -import javax.persistence.Entity; -import javax.persistence.Id; -import javax.persistence.Table; -import javax.persistence.Temporal; -import javax.persistence.TemporalType; - -@Entity -@Table(name = "mdstore_versions") -public class MDStoreVersion implements Serializable { - - /** */ - private static final long serialVersionUID = -4763494442274298339L; - - @Id - @Column(name = "id") - private String id; - - @Column(name = "mdstore") - private String mdstore; - - @Column(name = "writing") - private boolean writing; - - @Column(name = "readcount") - private int readCount = 0; - - @Column(name = "lastupdate") - @Temporal(TemporalType.TIMESTAMP) - private Date lastUpdate; - - @Column(name = "size") - private long size = 0; - - @Column(name = "hdfs_path") - private String hdfsPath; - - public static MDStoreVersion newInstance(final String mdId, final boolean writing, final String hdfsBasePath) { - final MDStoreVersion v = new MDStoreVersion(); - - final String versionId = mdId + "-" + new Date().getTime(); - v.setId(versionId); - v.setMdstore(mdId); - v.setLastUpdate(null); - v.setWriting(writing); - v.setReadCount(0); - v.setSize(0); - v.setHdfsPath(String.format("%s/%s/%s", hdfsBasePath, mdId, versionId)); - - return v; - } - - public String getId() { - return id; - } - - public void setId(final String id) { - this.id = id; - } - - public String getMdstore() { - return mdstore; - } - - public void setMdstore(final String mdstore) { - this.mdstore = mdstore; - } - - public boolean isWriting() { - return writing; - } - - public void setWriting(final boolean writing) { - this.writing = writing; - } - - public int getReadCount() { - return readCount; - } - - public void setReadCount(final int readCount) { - this.readCount = readCount; - } - - public Date getLastUpdate() { - return lastUpdate; - } - - public void setLastUpdate(final Date lastUpdate) { - this.lastUpdate = lastUpdate; - } - - public long getSize() { - return size; - } - - public void setSize(final long size) { - this.size = size; - } - - public String getHdfsPath() { - return hdfsPath; - } - - public void setHdfsPath(final String hdfsPath) { - this.hdfsPath = hdfsPath; - } - - @Override - public String toString() { - return String - .format( - "MDStoreVersion [id=%s, mdstore=%s, writing=%s, readCount=%s, lastUpdate=%s, size=%s, hdfsPath=%s]", id, - mdstore, writing, readCount, lastUpdate, size, hdfsPath); - } - - @Override - public int hashCode() { - return Objects.hash(id); - } - - @Override - public boolean equals(final Object obj) { - if (this == obj) { - return true; - } - if (!(obj instanceof MDStoreVersion)) { - return false; - } - final MDStoreVersion other = (MDStoreVersion) obj; - return Objects.equals(id, other.id); - } -} diff --git a/dhp-common/src/main/java/eu/dnetlib/data/mdstore/manager/common/model/MDStoreWithInfo.java b/dhp-common/src/main/java/eu/dnetlib/data/mdstore/manager/common/model/MDStoreWithInfo.java deleted file mode 100644 index 510c65092..000000000 --- a/dhp-common/src/main/java/eu/dnetlib/data/mdstore/manager/common/model/MDStoreWithInfo.java +++ /dev/null @@ -1,194 +0,0 @@ - -package eu.dnetlib.data.mdstore.manager.common.model; - -import java.io.Serializable; -import java.util.Date; -import java.util.Objects; - -import javax.persistence.Column; -import javax.persistence.Entity; -import javax.persistence.Id; -import javax.persistence.Table; -import javax.persistence.Temporal; -import javax.persistence.TemporalType; - -@Entity -@Table(name = "mdstores_with_info") -public class MDStoreWithInfo implements Serializable { - - /** */ - private static final long serialVersionUID = -8445784770687571492L; - - @Id - @Column(name = "id") - private String id; - - @Column(name = "format") - private String format; - - @Column(name = "layout") - private String layout; - - @Column(name = "interpretation") - private String interpretation; - - @Column(name = "datasource_name") - private String datasourceName; - - @Column(name = "datasource_id") - private String datasourceId; - - @Column(name = "api_id") - private String apiId; - - @Column(name = "current_version") - private String currentVersion; - - @Column(name = "creation_date") - @Temporal(TemporalType.TIMESTAMP) - private Date creationDate; - - @Column(name = "lastupdate") - @Temporal(TemporalType.TIMESTAMP) - private Date lastUpdate; - - @Column(name = "size") - private long size = 0; - - @Column(name = "n_versions") - private long numberOfVersions = 0; - - @Column(name = "hdfs_path") - private String hdfsPath; - - public String getId() { - return id; - } - - public void setId(final String id) { - this.id = id; - } - - public String getFormat() { - return format; - } - - public void setFormat(final String format) { - this.format = format; - } - - public String getLayout() { - return layout; - } - - public void setLayout(final String layout) { - this.layout = layout; - } - - public String getInterpretation() { - return interpretation; - } - - public void setInterpretation(final String interpretation) { - this.interpretation = interpretation; - } - - public String getDatasourceName() { - return datasourceName; - } - - public void setDatasourceName(final String datasourceName) { - this.datasourceName = datasourceName; - } - - public String getDatasourceId() { - return datasourceId; - } - - public void setDatasourceId(final String datasourceId) { - this.datasourceId = datasourceId; - } - - public String getApiId() { - return apiId; - } - - public void setApiId(final String apiId) { - this.apiId = apiId; - } - - public String getCurrentVersion() { - return currentVersion; - } - - public void setCurrentVersion(final String currentVersion) { - this.currentVersion = currentVersion; - } - - public Date getCreationDate() { - return creationDate; - } - - public void setCreationDate(final Date creationDate) { - this.creationDate = creationDate; - } - - public Date getLastUpdate() { - return lastUpdate; - } - - public void setLastUpdate(final Date lastUpdate) { - this.lastUpdate = lastUpdate; - } - - public long getSize() { - return size; - } - - public void setSize(final long size) { - this.size = size; - } - - public long getNumberOfVersions() { - return numberOfVersions; - } - - public void setNumberOfVersions(final long numberOfVersions) { - this.numberOfVersions = numberOfVersions; - } - - public String getHdfsPath() { - return hdfsPath; - } - - public void setHdfsPath(final String hdfsPath) { - this.hdfsPath = hdfsPath; - } - - @Override - public String toString() { - return String - .format( - "MDStoreWithInfo [id=%s, format=%s, layout=%s, interpretation=%s, datasourceName=%s, datasourceId=%s, apiId=%s, currentVersion=%s, creationDate=%s, lastUpdate=%s, size=%s, numberOfVersions=%s, hdfsPath=%s]", - id, format, layout, interpretation, datasourceName, datasourceId, apiId, currentVersion, creationDate, - lastUpdate, size, numberOfVersions, hdfsPath); - } - - @Override - public int hashCode() { - return Objects.hash(id); - } - - @Override - public boolean equals(final Object obj) { - if (this == obj) { - return true; - } - if (!(obj instanceof MDStoreWithInfo)) { - return false; - } - final MDStoreWithInfo other = (MDStoreWithInfo) obj; - return Objects.equals(id, other.id); - } - -} diff --git a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/aggregation/mdstore/MDStoreActionNode.java b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/aggregation/mdstore/MDStoreActionNode.java index 09f3ffd63..65e7805d8 100644 --- a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/aggregation/mdstore/MDStoreActionNode.java +++ b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/aggregation/mdstore/MDStoreActionNode.java @@ -14,9 +14,9 @@ import org.apache.hadoop.fs.Path; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import eu.dnetlib.data.mdstore.manager.common.model.MDStoreVersion; import eu.dnetlib.dhp.application.ArgumentApplicationParser; import eu.dnetlib.dhp.common.rest.DNetRestClient; +import eu.dnetlib.dhp.schema.mdstore.MDStoreVersion; public class MDStoreActionNode { private static final Logger log = LoggerFactory.getLogger(MDStoreActionNode.class); diff --git a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/collection/CollectorWorker.java b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/collection/CollectorWorker.java index f9d7d7dae..23ee3e2c6 100644 --- a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/collection/CollectorWorker.java +++ b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/collection/CollectorWorker.java @@ -16,7 +16,6 @@ import org.apache.hadoop.io.compress.DeflateCodec; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import eu.dnetlib.data.mdstore.manager.common.model.MDStoreVersion; import eu.dnetlib.dhp.aggregation.common.AggregatorReport; import eu.dnetlib.dhp.aggregation.common.ReporterCallback; import eu.dnetlib.dhp.aggregation.common.ReportingJob; @@ -25,6 +24,7 @@ import eu.dnetlib.dhp.collection.plugin.mongodb.MDStoreCollectorPlugin; import eu.dnetlib.dhp.collection.plugin.mongodb.MongoDbDumpCollectorPlugin; import eu.dnetlib.dhp.collection.plugin.oai.OaiCollectorPlugin; import eu.dnetlib.dhp.collection.plugin.rest.RestCollectorPlugin; +import eu.dnetlib.dhp.schema.mdstore.MDStoreVersion; public class CollectorWorker extends ReportingJob { diff --git a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/collection/CollectorWorkerApplication.java b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/collection/CollectorWorkerApplication.java index 2c5640499..704e7bb93 100644 --- a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/collection/CollectorWorkerApplication.java +++ b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/collection/CollectorWorkerApplication.java @@ -13,10 +13,10 @@ import org.apache.hadoop.fs.FileSystem; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import eu.dnetlib.data.mdstore.manager.common.model.MDStoreVersion; import eu.dnetlib.dhp.aggregation.common.AggregatorReport; import eu.dnetlib.dhp.application.ArgumentApplicationParser; import eu.dnetlib.dhp.message.MessageSender; +import eu.dnetlib.dhp.schema.mdstore.MDStoreVersion; /** * CollectorWorkerApplication is the main class responsible to start the metadata collection process, storing the outcomes diff --git a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/collection/GenerateNativeStoreSparkJob.java b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/collection/GenerateNativeStoreSparkJob.java index 043da31f9..f6fdc266e 100644 --- a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/collection/GenerateNativeStoreSparkJob.java +++ b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/collection/GenerateNativeStoreSparkJob.java @@ -28,8 +28,8 @@ import org.dom4j.io.SAXReader; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import eu.dnetlib.data.mdstore.manager.common.model.MDStoreVersion; import eu.dnetlib.dhp.application.ArgumentApplicationParser; +import eu.dnetlib.dhp.schema.mdstore.MDStoreVersion; import eu.dnetlib.dhp.schema.mdstore.MetadataRecord; import eu.dnetlib.dhp.schema.mdstore.Provenance; import scala.Tuple2; diff --git a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/transformation/TransformSparkJobNode.java b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/transformation/TransformSparkJobNode.java index 5ef26b7d5..4673a2394 100644 --- a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/transformation/TransformSparkJobNode.java +++ b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/transformation/TransformSparkJobNode.java @@ -22,14 +22,13 @@ import org.apache.spark.util.LongAccumulator; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import eu.dnetlib.data.mdstore.manager.common.model.MDStoreVersion; import eu.dnetlib.dhp.aggregation.common.AggregationCounter; import eu.dnetlib.dhp.aggregation.common.AggregatorReport; import eu.dnetlib.dhp.application.ArgumentApplicationParser; import eu.dnetlib.dhp.common.vocabulary.VocabularyGroup; import eu.dnetlib.dhp.message.MessageSender; +import eu.dnetlib.dhp.schema.mdstore.MDStoreVersion; import eu.dnetlib.dhp.schema.mdstore.MetadataRecord; -import eu.dnetlib.dhp.transformation.xslt.XSLTTransformationFunction; import eu.dnetlib.dhp.utils.ISLookupClientFactory; import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpService; diff --git a/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/collection/CollectionWorkflowTest.java b/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/collection/CollectionWorkflowTest.java index cd6275d7f..05dd6b1d3 100644 --- a/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/collection/CollectionWorkflowTest.java +++ b/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/collection/CollectionWorkflowTest.java @@ -18,7 +18,7 @@ import org.slf4j.LoggerFactory; import com.fasterxml.jackson.databind.ObjectMapper; -import eu.dnetlib.data.mdstore.manager.common.model.MDStoreVersion; +import eu.dnetlib.dhp.schema.mdstore.MDStoreVersion; @TestMethodOrder(MethodOrderer.OrderAnnotation.class) @ExtendWith(MockitoExtension.class) diff --git a/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/collection/GenerateNativeStoreSparkJobTest.java b/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/collection/GenerateNativeStoreSparkJobTest.java index 7016d39fa..afb6ae6a1 100644 --- a/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/collection/GenerateNativeStoreSparkJobTest.java +++ b/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/collection/GenerateNativeStoreSparkJobTest.java @@ -36,8 +36,8 @@ import org.slf4j.LoggerFactory; import com.fasterxml.jackson.databind.ObjectMapper; -import eu.dnetlib.data.mdstore.manager.common.model.MDStoreVersion; import eu.dnetlib.dhp.aggregation.AbstractVocabularyTest; +import eu.dnetlib.dhp.schema.mdstore.MDStoreVersion; import eu.dnetlib.dhp.schema.mdstore.MetadataRecord; import eu.dnetlib.dhp.schema.mdstore.Provenance; import eu.dnetlib.dhp.transformation.TransformSparkJobNode; diff --git a/pom.xml b/pom.xml index e846b2256..09e304eea 100644 --- a/pom.xml +++ b/pom.xml @@ -730,7 +730,7 @@ 3.3.3 3.4.2 [2.12,3.0) - [2.2.5] + [2.3.6] [4.0.3] [6.0.5] [3.1.6]