diff --git a/apps/dnet-is-application/src/main/java/eu/dnetlib/data/mdstore/MDStoreAjaxController.java b/apps/dnet-is-application/src/main/java/eu/dnetlib/data/mdstore/MDStoreAjaxController.java index ee3ce18c..70f2a058 100644 --- a/apps/dnet-is-application/src/main/java/eu/dnetlib/data/mdstore/MDStoreAjaxController.java +++ b/apps/dnet-is-application/src/main/java/eu/dnetlib/data/mdstore/MDStoreAjaxController.java @@ -10,6 +10,7 @@ import org.springframework.web.bind.annotation.PathVariable; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RestController; +import eu.dnetlib.data.mdstore.model.MetadataRecord; import eu.dnetlib.errors.MDStoreManagerException; import io.swagger.v3.oas.annotations.Operation; import io.swagger.v3.oas.annotations.tags.Tag; @@ -42,13 +43,13 @@ public class MDStoreAjaxController extends AbstractMDStoreController { @Operation(summary = "read the parquet file of a mdstore version") @GetMapping("/version/{versionId}/content/{limit}") - public List> listVersionRecords(@PathVariable final String versionId, @PathVariable final long limit) throws MDStoreManagerException { + public List listVersionRecords(@PathVariable final String versionId, @PathVariable final long limit) throws MDStoreManagerException { return service.listVersionRecords(versionId, limit); } @Operation(summary = "read the parquet file of a mdstore (current version)") @GetMapping("/mdstore/{mdId}/content/{limit}") - public List> listMdstoreRecords(@PathVariable final String mdId, @PathVariable final long limit) throws MDStoreManagerException { + public List listMdstoreRecords(@PathVariable final String mdId, @PathVariable final long limit) throws MDStoreManagerException { return service.listVersionRecords(service.findMdStore(mdId).getCurrentVersion(), limit); } } diff --git a/frontends/dnet-is-application/src/app/common/is.service.ts b/frontends/dnet-is-application/src/app/common/is.service.ts index 01b721ed..7fba48a7 100644 --- a/frontends/dnet-is-application/src/app/common/is.service.ts +++ b/frontends/dnet-is-application/src/app/common/is.service.ts @@ -313,7 +313,7 @@ export class ISService { } loadMDStoreVersionRecords(versionId: string, limit: number, onSuccess: Function): void { - this.client.get('./ajax/mdstores/version/' + encodeURIComponent(versionId) + '/parquet/content/' + limit).subscribe({ + this.client.get('./ajax/mdstores/version/' + encodeURIComponent(versionId) + '/content/' + limit).subscribe({ next: data => onSuccess(data), error: error => this.showError(error) }); diff --git a/frontends/dnet-is-application/src/app/mdstores/mdstore-inspector.component.html b/frontends/dnet-is-application/src/app/mdstores/mdstore-inspector.component.html index 9a382660..b5a9cc2c 100644 --- a/frontends/dnet-is-application/src/app/mdstores/mdstore-inspector.component.html +++ b/frontends/dnet-is-application/src/app/mdstores/mdstore-inspector.component.html @@ -19,7 +19,7 @@ {{mdstore?.datasourceName}} - Version + Version ID current @@ -45,16 +45,17 @@ -
-

The display is limited to the first {{limit}} records

-
+

The display is limited to the + first {{limit}} + records

- + the record is unreadable {{rec.id}} +

Invalid record format

diff --git a/libs/dnet-data-services/src/main/java/eu/dnetlib/data/mdstore/MDStoreService.java b/libs/dnet-data-services/src/main/java/eu/dnetlib/data/mdstore/MDStoreService.java index 16a742c1..db7f48b1 100644 --- a/libs/dnet-data-services/src/main/java/eu/dnetlib/data/mdstore/MDStoreService.java +++ b/libs/dnet-data-services/src/main/java/eu/dnetlib/data/mdstore/MDStoreService.java @@ -19,12 +19,16 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.jdbc.core.JdbcTemplate; import org.springframework.stereotype.Service; -import eu.dnetlib.data.mdstore.hadoop.HdfsBackend; +import eu.dnetlib.data.mdstore.backends.DefaultBackend; +import eu.dnetlib.data.mdstore.backends.HdfsBackend; +import eu.dnetlib.data.mdstore.backends.MDStoreBackend; +import eu.dnetlib.data.mdstore.backends.MockBackend; import eu.dnetlib.data.mdstore.model.MDStore; import eu.dnetlib.data.mdstore.model.MDStoreCurrentVersion; import eu.dnetlib.data.mdstore.model.MDStoreType; import eu.dnetlib.data.mdstore.model.MDStoreVersion; import eu.dnetlib.data.mdstore.model.MDStoreWithInfo; +import eu.dnetlib.data.mdstore.model.MetadataRecord; import eu.dnetlib.data.mdstore.repository.MDStoreCurrentVersionRepository; import eu.dnetlib.data.mdstore.repository.MDStoreRepository; import eu.dnetlib.data.mdstore.repository.MDStoreVersionRepository; @@ -48,6 +52,8 @@ public class MDStoreService { @Autowired private HdfsBackend hdfsBackend; @Autowired + private MockBackend mockBackend; + @Autowired private DefaultBackend defaultBackend; private static final Logger log = LoggerFactory.getLogger(MDStoreService.class); @@ -228,7 +234,7 @@ public class MDStoreService { return selectBackend(md.getType()).listInternalFiles(v); } - public List> listVersionRecords(final String versionId, final long limit) throws MDStoreManagerException { + public List listVersionRecords(final String versionId, final long limit) throws MDStoreManagerException { final MDStoreVersion v = mdstoreVersionRepository.findById(versionId).orElseThrow(() -> new MDStoreManagerException("Version not found")); final MDStore md = mdstoreRepository.findById(v.getMdstore()).orElseThrow(() -> new MDStoreManagerException("MDStore not found")); return selectBackend(md.getType()).listEntries(v, limit); @@ -265,6 +271,7 @@ public class MDStoreService { public Map> fixInconsistencies(final boolean delete) throws MDStoreManagerException { final Map> res = new LinkedHashMap<>(); res.put(MDStoreType.HDFS, hdfsBackend.fixInconsistencies(delete)); + res.put(MDStoreType.MOCK, mockBackend.fixInconsistencies(delete)); // TODO: ADD HERE THE INVOCATION FOR OTHER MDSTORE TYPE return res; @@ -274,6 +281,8 @@ public class MDStoreService { switch (type) { case HDFS: return hdfsBackend; + case MOCK: + return mockBackend; default: return defaultBackend; } diff --git a/libs/dnet-data-services/src/main/java/eu/dnetlib/data/mdstore/DefaultBackend.java b/libs/dnet-data-services/src/main/java/eu/dnetlib/data/mdstore/backends/DefaultBackend.java similarity index 83% rename from libs/dnet-data-services/src/main/java/eu/dnetlib/data/mdstore/DefaultBackend.java rename to libs/dnet-data-services/src/main/java/eu/dnetlib/data/mdstore/backends/DefaultBackend.java index 03917506..1ce1122b 100644 --- a/libs/dnet-data-services/src/main/java/eu/dnetlib/data/mdstore/DefaultBackend.java +++ b/libs/dnet-data-services/src/main/java/eu/dnetlib/data/mdstore/backends/DefaultBackend.java @@ -1,16 +1,16 @@ -package eu.dnetlib.data.mdstore; +package eu.dnetlib.data.mdstore.backends; import java.util.ArrayList; import java.util.HashSet; import java.util.LinkedHashSet; import java.util.List; -import java.util.Map; import java.util.Set; import org.springframework.stereotype.Service; import eu.dnetlib.data.mdstore.model.MDStore; import eu.dnetlib.data.mdstore.model.MDStoreVersion; +import eu.dnetlib.data.mdstore.model.MetadataRecord; import eu.dnetlib.errors.MDStoreManagerException; @Service @@ -29,7 +29,7 @@ public class DefaultBackend implements MDStoreBackend { public void delete(final MDStoreVersion version) throws MDStoreManagerException {} @Override - public List> listEntries(final MDStoreVersion version, final long limit) throws MDStoreManagerException { + public List listEntries(final MDStoreVersion version, final long limit) throws MDStoreManagerException { return new ArrayList<>(); } diff --git a/libs/dnet-data-services/src/main/java/eu/dnetlib/data/mdstore/hadoop/HdfsBackend.java b/libs/dnet-data-services/src/main/java/eu/dnetlib/data/mdstore/backends/HdfsBackend.java similarity index 89% rename from libs/dnet-data-services/src/main/java/eu/dnetlib/data/mdstore/hadoop/HdfsBackend.java rename to libs/dnet-data-services/src/main/java/eu/dnetlib/data/mdstore/backends/HdfsBackend.java index b843c2c2..58fb1b0d 100644 --- a/libs/dnet-data-services/src/main/java/eu/dnetlib/data/mdstore/hadoop/HdfsBackend.java +++ b/libs/dnet-data-services/src/main/java/eu/dnetlib/data/mdstore/backends/HdfsBackend.java @@ -1,8 +1,7 @@ -package eu.dnetlib.data.mdstore.hadoop; +package eu.dnetlib.data.mdstore.backends; import java.util.HashSet; import java.util.List; -import java.util.Map; import java.util.Set; import org.apache.commons.lang3.StringUtils; @@ -15,9 +14,10 @@ import org.springframework.stereotype.Service; import com.google.common.collect.Sets; -import eu.dnetlib.data.mdstore.MDStoreBackend; +import eu.dnetlib.data.mdstore.hadoop.HdfsClient; import eu.dnetlib.data.mdstore.model.MDStore; import eu.dnetlib.data.mdstore.model.MDStoreVersion; +import eu.dnetlib.data.mdstore.model.MetadataRecord; import eu.dnetlib.errors.MDStoreManagerException; @Service @@ -90,10 +90,10 @@ public class HdfsBackend implements MDStoreBackend { } @Override - public List> listEntries(final MDStoreVersion version, final long limit) throws MDStoreManagerException { + public List listEntries(final MDStoreVersion version, final long limit) throws MDStoreManagerException { final String path = version.getParams().getOrDefault("hdfs_path", "").toString(); if (StringUtils.isNotBlank(path)) { - return hdfsClient.readParquetFiles(path + "/store", limit); + return hdfsClient.readParquetFiles(path + "/store", limit, MetadataRecord.class); } else { throw new MDStoreManagerException("hdfs path is missing"); } diff --git a/libs/dnet-data-services/src/main/java/eu/dnetlib/data/mdstore/MDStoreBackend.java b/libs/dnet-data-services/src/main/java/eu/dnetlib/data/mdstore/backends/MDStoreBackend.java similarity index 76% rename from libs/dnet-data-services/src/main/java/eu/dnetlib/data/mdstore/MDStoreBackend.java rename to libs/dnet-data-services/src/main/java/eu/dnetlib/data/mdstore/backends/MDStoreBackend.java index 9c164b59..4293a698 100644 --- a/libs/dnet-data-services/src/main/java/eu/dnetlib/data/mdstore/MDStoreBackend.java +++ b/libs/dnet-data-services/src/main/java/eu/dnetlib/data/mdstore/backends/MDStoreBackend.java @@ -1,11 +1,11 @@ -package eu.dnetlib.data.mdstore; +package eu.dnetlib.data.mdstore.backends; import java.util.List; -import java.util.Map; import java.util.Set; import eu.dnetlib.data.mdstore.model.MDStore; import eu.dnetlib.data.mdstore.model.MDStoreVersion; +import eu.dnetlib.data.mdstore.model.MetadataRecord; import eu.dnetlib.errors.MDStoreManagerException; public interface MDStoreBackend { @@ -18,7 +18,7 @@ public interface MDStoreBackend { void delete(MDStoreVersion version) throws MDStoreManagerException; - List> listEntries(MDStoreVersion version, long limit) throws MDStoreManagerException; + List listEntries(MDStoreVersion version, long limit) throws MDStoreManagerException; Set listInternalFiles(MDStoreVersion version) throws MDStoreManagerException; diff --git a/libs/dnet-data-services/src/main/java/eu/dnetlib/data/mdstore/backends/MockBackend.java b/libs/dnet-data-services/src/main/java/eu/dnetlib/data/mdstore/backends/MockBackend.java new file mode 100644 index 00000000..66ba910b --- /dev/null +++ b/libs/dnet-data-services/src/main/java/eu/dnetlib/data/mdstore/backends/MockBackend.java @@ -0,0 +1,72 @@ +package eu.dnetlib.data.mdstore.backends; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Date; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Set; + +import org.springframework.stereotype.Service; + +import eu.dnetlib.data.mdstore.model.MDStore; +import eu.dnetlib.data.mdstore.model.MDStoreVersion; +import eu.dnetlib.data.mdstore.model.MetadataRecord; +import eu.dnetlib.data.mdstore.model.Provenance; +import eu.dnetlib.errors.MDStoreManagerException; + +@Service +public class MockBackend implements MDStoreBackend { + + private static final Provenance MOCK_PROVENANCE = new Provenance("mock________::MOCK_DS", "Mock Datasource", "mock________"); + + @Override + public void completeNewMDStore(final MDStore mdstore) { + mdstore.getParams().put("mockParam1", "test"); + mdstore.getParams().put("mockParam2", "abc"); + mdstore.getParams().put("mockParam3", 1234); + } + + @Override + public void completeNewMDStoreVersion(final MDStoreVersion version) { + version.getParams().put("mockParam1", "v_test"); + version.getParams().put("mockParam2", "v_abc"); + version.getParams().put("mockParam3", 1234); + } + + @Override + public void delete(final MDStore mdstore) throws MDStoreManagerException {} + + @Override + public void delete(final MDStoreVersion version) throws MDStoreManagerException {} + + @Override + public List listEntries(final MDStoreVersion version, final long limit) throws MDStoreManagerException { + final List list = new ArrayList<>(); + + for (int i = 0; i < limit; i++) { + final MetadataRecord rec = new MetadataRecord(); + rec.setOriginalId("mck-" + i); + rec.setId("mock________::mck-" + i); + rec.setBody("" + i + ""); + rec.setDateOfCollection(new Date().getTime()); + rec.setDateOfTransformation(new Date().getTime()); + rec.setEncoding("XML"); + rec.setProvenance(MOCK_PROVENANCE); + list.add(rec); + } + + return list; + } + + @Override + public Set listInternalFiles(final MDStoreVersion version) throws MDStoreManagerException { + return new LinkedHashSet<>(Arrays.asList("file1", "file2", "file3", "file4")); + } + + @Override + public Set fixInconsistencies(final boolean delete) throws MDStoreManagerException { + return new LinkedHashSet<>(Arrays.asList("1", "2", "3", "4")); + } + +} diff --git a/libs/dnet-data-services/src/main/java/eu/dnetlib/data/mdstore/hadoop/HdfsClient.java b/libs/dnet-data-services/src/main/java/eu/dnetlib/data/mdstore/hadoop/HdfsClient.java index 6df47735..fc3acf9b 100644 --- a/libs/dnet-data-services/src/main/java/eu/dnetlib/data/mdstore/hadoop/HdfsClient.java +++ b/libs/dnet-data-services/src/main/java/eu/dnetlib/data/mdstore/hadoop/HdfsClient.java @@ -20,6 +20,8 @@ import org.apache.hadoop.fs.Path; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Component; +import com.fasterxml.jackson.databind.ObjectMapper; + import eu.dnetlib.errors.MDStoreManagerException; import parquet.avro.AvroParquetReader; import parquet.hadoop.ParquetReader; @@ -96,9 +98,9 @@ public class HdfsClient { } @SuppressWarnings("unchecked") - public List> readParquetFiles(final String path, final long limit) throws MDStoreManagerException { + public List readParquetFiles(final String path, final long limit, final Class clazz) throws MDStoreManagerException { - final List> list = new ArrayList<>(); + final List list = new ArrayList<>(); final Configuration conf = conf(); @@ -113,6 +115,8 @@ public class HdfsClient { try (final ParquetReader reader = AvroParquetReader. builder(new Path(f)).withConf(conf).build()) { log.debug("File parquet OPENED"); + final ObjectMapper mapper = new ObjectMapper(); + GenericRecord rec = null; while (i++ < limit && (rec = reader.read()) != null) { if (fields.isEmpty()) { @@ -124,7 +128,9 @@ public class HdfsClient { final Object v = rec.get(field); map.put(field, v != null ? v.toString() : ""); } - list.add(map); + + list.add(mapper.convertValue(map, clazz)); + log.debug("added record"); } } catch (final FileNotFoundException e) {