D-Net
/
dnet-applications
16
1
Fork 0
Code Issues Pull Requests Releases Wiki Activity

mock mdstore

This commit is contained in:
Michele Artini 2023-02-10 14:54:37 +01:00
parent 438a8a46bc
commit 9e2be69d98
9 changed files with 113 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
apps/dnet-is-application/src/main/java/eu/dnetlib/data/mdstore/MDStoreAjaxController.java
View File

@ -10,6 +10,7 @@ import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
import eu.dnetlib.data.mdstore.model.MetadataRecord;
import eu.dnetlib.errors.MDStoreManagerException;
import io.swagger.v3.oas.annotations.Operation;
import io.swagger.v3.oas.annotations.tags.Tag;
@ -42,13 +43,13 @@ public class MDStoreAjaxController extends AbstractMDStoreController {
@Operation(summary = "read the parquet file of a mdstore version")
@GetMapping("/version/{versionId}/content/{limit}")
public List<Map<String, String>> listVersionRecords(@PathVariable final String versionId, @PathVariable final long limit) throws MDStoreManagerException {
public List<MetadataRecord> listVersionRecords(@PathVariable final String versionId, @PathVariable final long limit) throws MDStoreManagerException {
return service.listVersionRecords(versionId, limit);
}
@Operation(summary = "read the parquet file of a mdstore (current version)")
@GetMapping("/mdstore/{mdId}/content/{limit}")
public List<Map<String, String>> listMdstoreRecords(@PathVariable final String mdId, @PathVariable final long limit) throws MDStoreManagerException {
public List<MetadataRecord> listMdstoreRecords(@PathVariable final String mdId, @PathVariable final long limit) throws MDStoreManagerException {
return service.listVersionRecords(service.findMdStore(mdId).getCurrentVersion(), limit);
}
}

2
frontends/dnet-is-application/src/app/common/is.service.ts
View File

@ -313,7 +313,7 @@ export class ISService {
}
loadMDStoreVersionRecords(versionId: string, limit: number, onSuccess: Function): void {
this.client.get<MDStoreRecord[]>('./ajax/mdstores/version/' + encodeURIComponent(versionId) + '/parquet/content/' + limit).subscribe({
this.client.get<MDStoreRecord[]>('./ajax/mdstores/version/' + encodeURIComponent(versionId) + '/content/' + limit).subscribe({
next: data => onSuccess(data),
error: error => this.showError(error)
});

11
frontends/dnet-is-application/src/app/mdstores/mdstore-inspector.component.html
View File

@ -19,7 +19,7 @@
<td>{{mdstore?.datasourceName}}</td>
</tr>
<tr>
<th rowspan="4">Version</th>
<th [attr.rowspan]="(version?.params | keyvalue).length + 3">Version</th>
<th>ID</th>
<td>
<span *ngIf="version?.id == mdstore?.currentVersion" class="badge-label badge-success">current</span>
@ -45,16 +45,17 @@
</tr>
</table>
<br />
<h3 *ngIf="records.length > 0">The display is limited to the first {{limit}} records</h3>
<br />
<h3 *ngIf="records.length > 0" class="muted" style="margin-top: 2em; text-align: center;">The display is limited to the
first {{limit}}
records</h3>
<mat-card *ngFor="let rec of records" style="margin-top: 10px;">
<mat-card *ngFor="let rec of records" style="margin-top: 1em;">
<mat-card-header>
<mat-card-title *ngIf="!rec.id">the record is unreadable</mat-card-title>
<mat-card-title *ngIf="rec.id">{{rec.id}}</mat-card-title>
</mat-card-header>
<mat-card-content>
<br />
<p *ngIf="!rec.id">Invalid record format</p>
<table class="mdstore-table" *ngIf="rec.id">
<tr>

13
libs/dnet-data-services/src/main/java/eu/dnetlib/data/mdstore/MDStoreService.java
View File

@ -19,12 +19,16 @@ import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.stereotype.Service;
import eu.dnetlib.data.mdstore.hadoop.HdfsBackend;
import eu.dnetlib.data.mdstore.backends.DefaultBackend;
import eu.dnetlib.data.mdstore.backends.HdfsBackend;
import eu.dnetlib.data.mdstore.backends.MDStoreBackend;
import eu.dnetlib.data.mdstore.backends.MockBackend;
import eu.dnetlib.data.mdstore.model.MDStore;
import eu.dnetlib.data.mdstore.model.MDStoreCurrentVersion;
import eu.dnetlib.data.mdstore.model.MDStoreType;
import eu.dnetlib.data.mdstore.model.MDStoreVersion;
import eu.dnetlib.data.mdstore.model.MDStoreWithInfo;
import eu.dnetlib.data.mdstore.model.MetadataRecord;
import eu.dnetlib.data.mdstore.repository.MDStoreCurrentVersionRepository;
import eu.dnetlib.data.mdstore.repository.MDStoreRepository;
import eu.dnetlib.data.mdstore.repository.MDStoreVersionRepository;
@ -48,6 +52,8 @@ public class MDStoreService {
@Autowired
private HdfsBackend hdfsBackend;
@Autowired
private MockBackend mockBackend;
@Autowired
private DefaultBackend defaultBackend;
private static final Logger log = LoggerFactory.getLogger(MDStoreService.class);
@ -228,7 +234,7 @@ public class MDStoreService {
return selectBackend(md.getType()).listInternalFiles(v);
}
public List<Map<String, String>> listVersionRecords(final String versionId, final long limit) throws MDStoreManagerException {
public List<MetadataRecord> listVersionRecords(final String versionId, final long limit) throws MDStoreManagerException {
final MDStoreVersion v = mdstoreVersionRepository.findById(versionId).orElseThrow(() -> new MDStoreManagerException("Version not found"));
final MDStore md = mdstoreRepository.findById(v.getMdstore()).orElseThrow(() -> new MDStoreManagerException("MDStore not found"));
return selectBackend(md.getType()).listEntries(v, limit);
@ -265,6 +271,7 @@ public class MDStoreService {
public Map<MDStoreType, Set<String>> fixInconsistencies(final boolean delete) throws MDStoreManagerException {
final Map<MDStoreType, Set<String>> res = new LinkedHashMap<>();
res.put(MDStoreType.HDFS, hdfsBackend.fixInconsistencies(delete));
res.put(MDStoreType.MOCK, mockBackend.fixInconsistencies(delete));
// TODO: ADD HERE THE INVOCATION FOR OTHER MDSTORE TYPE
return res;
@ -274,6 +281,8 @@ public class MDStoreService {
switch (type) {
case HDFS:
return hdfsBackend;
case MOCK:
return mockBackend;
default:
return defaultBackend;
}

6
libs/dnet-data-services/src/main/java/eu/dnetlib/data/mdstore/DefaultBackend.java → libs/dnet-data-services/src/main/java/eu/dnetlib/data/mdstore/backends/DefaultBackend.java
View File

@ -1,16 +1,16 @@
package eu.dnetlib.data.mdstore;
package eu.dnetlib.data.mdstore.backends;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import org.springframework.stereotype.Service;
import eu.dnetlib.data.mdstore.model.MDStore;
import eu.dnetlib.data.mdstore.model.MDStoreVersion;
import eu.dnetlib.data.mdstore.model.MetadataRecord;
import eu.dnetlib.errors.MDStoreManagerException;
@Service
@ -29,7 +29,7 @@ public class DefaultBackend implements MDStoreBackend {
public void delete(final MDStoreVersion version) throws MDStoreManagerException {}
@Override
public List<Map<String, String>> listEntries(final MDStoreVersion version, final long limit) throws MDStoreManagerException {
public List<MetadataRecord> listEntries(final MDStoreVersion version, final long limit) throws MDStoreManagerException {
return new ArrayList<>();
}

10
libs/dnet-data-services/src/main/java/eu/dnetlib/data/mdstore/hadoop/HdfsBackend.java → libs/dnet-data-services/src/main/java/eu/dnetlib/data/mdstore/backends/HdfsBackend.java
View File

@ -1,8 +1,7 @@
package eu.dnetlib.data.mdstore.hadoop;
package eu.dnetlib.data.mdstore.backends;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import org.apache.commons.lang3.StringUtils;
@ -15,9 +14,10 @@ import org.springframework.stereotype.Service;
import com.google.common.collect.Sets;
import eu.dnetlib.data.mdstore.MDStoreBackend;
import eu.dnetlib.data.mdstore.hadoop.HdfsClient;
import eu.dnetlib.data.mdstore.model.MDStore;
import eu.dnetlib.data.mdstore.model.MDStoreVersion;
import eu.dnetlib.data.mdstore.model.MetadataRecord;
import eu.dnetlib.errors.MDStoreManagerException;
@Service
@ -90,10 +90,10 @@ public class HdfsBackend implements MDStoreBackend {
}
@Override
public List<Map<String, String>> listEntries(final MDStoreVersion version, final long limit) throws MDStoreManagerException {
public List<MetadataRecord> listEntries(final MDStoreVersion version, final long limit) throws MDStoreManagerException {
final String path = version.getParams().getOrDefault("hdfs_path", "").toString();
if (StringUtils.isNotBlank(path)) {
return hdfsClient.readParquetFiles(path + "/store", limit);
return hdfsClient.readParquetFiles(path + "/store", limit, MetadataRecord.class);
} else {
throw new MDStoreManagerException("hdfs path is missing");
}

6
libs/dnet-data-services/src/main/java/eu/dnetlib/data/mdstore/MDStoreBackend.java → libs/dnet-data-services/src/main/java/eu/dnetlib/data/mdstore/backends/MDStoreBackend.java
View File

@ -1,11 +1,11 @@
package eu.dnetlib.data.mdstore;
package eu.dnetlib.data.mdstore.backends;
import java.util.List;
import java.util.Map;
import java.util.Set;
import eu.dnetlib.data.mdstore.model.MDStore;
import eu.dnetlib.data.mdstore.model.MDStoreVersion;
import eu.dnetlib.data.mdstore.model.MetadataRecord;
import eu.dnetlib.errors.MDStoreManagerException;
public interface MDStoreBackend {
@ -18,7 +18,7 @@ public interface MDStoreBackend {
void delete(MDStoreVersion version) throws MDStoreManagerException;
List<Map<String, String>> listEntries(MDStoreVersion version, long limit) throws MDStoreManagerException;
List<MetadataRecord> listEntries(MDStoreVersion version, long limit) throws MDStoreManagerException;
Set<String> listInternalFiles(MDStoreVersion version) throws MDStoreManagerException;

72
libs/dnet-data-services/src/main/java/eu/dnetlib/data/mdstore/backends/MockBackend.java Normal file
View File

@ -0,0 +1,72 @@
package eu.dnetlib.data.mdstore.backends;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Date;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Set;
import org.springframework.stereotype.Service;
import eu.dnetlib.data.mdstore.model.MDStore;
import eu.dnetlib.data.mdstore.model.MDStoreVersion;
import eu.dnetlib.data.mdstore.model.MetadataRecord;
import eu.dnetlib.data.mdstore.model.Provenance;
import eu.dnetlib.errors.MDStoreManagerException;
@Service
public class MockBackend implements MDStoreBackend {
private static final Provenance MOCK_PROVENANCE = new Provenance("mock________::MOCK_DS", "Mock Datasource", "mock________");
@Override
public void completeNewMDStore(final MDStore mdstore) {
mdstore.getParams().put("mockParam1", "test");
mdstore.getParams().put("mockParam2", "abc");
mdstore.getParams().put("mockParam3", 1234);
}
@Override
public void completeNewMDStoreVersion(final MDStoreVersion version) {
version.getParams().put("mockParam1", "v_test");
version.getParams().put("mockParam2", "v_abc");
version.getParams().put("mockParam3", 1234);
}
@Override
public void delete(final MDStore mdstore) throws MDStoreManagerException {}
@Override
public void delete(final MDStoreVersion version) throws MDStoreManagerException {}
@Override
public List<MetadataRecord> listEntries(final MDStoreVersion version, final long limit) throws MDStoreManagerException {
final List<MetadataRecord> list = new ArrayList<>();
for (int i = 0; i < limit; i++) {
final MetadataRecord rec = new MetadataRecord();
rec.setOriginalId("mck-" + i);
rec.setId("mock________::mck-" + i);
rec.setBody("<RECORD>" + i + "</RECORD>");
rec.setDateOfCollection(new Date().getTime());
rec.setDateOfTransformation(new Date().getTime());
rec.setEncoding("XML");
rec.setProvenance(MOCK_PROVENANCE);
list.add(rec);
}
return list;
}
@Override
public Set<String> listInternalFiles(final MDStoreVersion version) throws MDStoreManagerException {
return new LinkedHashSet<>(Arrays.asList("file1", "file2", "file3", "file4"));
}
@Override
public Set<String> fixInconsistencies(final boolean delete) throws MDStoreManagerException {
return new LinkedHashSet<>(Arrays.asList("1", "2", "3", "4"));
}
}

12
libs/dnet-data-services/src/main/java/eu/dnetlib/data/mdstore/hadoop/HdfsClient.java
View File

@ -20,6 +20,8 @@ import org.apache.hadoop.fs.Path;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
import com.fasterxml.jackson.databind.ObjectMapper;
import eu.dnetlib.errors.MDStoreManagerException;
import parquet.avro.AvroParquetReader;
import parquet.hadoop.ParquetReader;
@ -96,9 +98,9 @@ public class HdfsClient {
}
@SuppressWarnings("unchecked")
public List<Map<String, String>> readParquetFiles(final String path, final long limit) throws MDStoreManagerException {
public <T> List<T> readParquetFiles(final String path, final long limit, final Class<T> clazz) throws MDStoreManagerException {
final List<Map<String, String>> list = new ArrayList<>();
final List<T> list = new ArrayList<>();
final Configuration conf = conf();
@ -113,6 +115,8 @@ public class HdfsClient {
try (final ParquetReader<GenericRecord> reader = AvroParquetReader.<GenericRecord> builder(new Path(f)).withConf(conf).build()) {
log.debug("File parquet OPENED");
final ObjectMapper mapper = new ObjectMapper();
GenericRecord rec = null;
while (i++ < limit && (rec = reader.read()) != null) {
if (fields.isEmpty()) {
@ -124,7 +128,9 @@ public class HdfsClient {
final Object v = rec.get(field);
map.put(field, v != null ? v.toString() : "");
}
list.add(map);
list.add(mapper.convertValue(map, clazz));
log.debug("added record");
}
} catch (final FileNotFoundException e) {