mock mdstore
This commit is contained in:
parent
438a8a46bc
commit
9e2be69d98
|
@ -10,6 +10,7 @@ import org.springframework.web.bind.annotation.PathVariable;
|
||||||
import org.springframework.web.bind.annotation.RequestMapping;
|
import org.springframework.web.bind.annotation.RequestMapping;
|
||||||
import org.springframework.web.bind.annotation.RestController;
|
import org.springframework.web.bind.annotation.RestController;
|
||||||
|
|
||||||
|
import eu.dnetlib.data.mdstore.model.MetadataRecord;
|
||||||
import eu.dnetlib.errors.MDStoreManagerException;
|
import eu.dnetlib.errors.MDStoreManagerException;
|
||||||
import io.swagger.v3.oas.annotations.Operation;
|
import io.swagger.v3.oas.annotations.Operation;
|
||||||
import io.swagger.v3.oas.annotations.tags.Tag;
|
import io.swagger.v3.oas.annotations.tags.Tag;
|
||||||
|
@ -42,13 +43,13 @@ public class MDStoreAjaxController extends AbstractMDStoreController {
|
||||||
|
|
||||||
@Operation(summary = "read the parquet file of a mdstore version")
|
@Operation(summary = "read the parquet file of a mdstore version")
|
||||||
@GetMapping("/version/{versionId}/content/{limit}")
|
@GetMapping("/version/{versionId}/content/{limit}")
|
||||||
public List<Map<String, String>> listVersionRecords(@PathVariable final String versionId, @PathVariable final long limit) throws MDStoreManagerException {
|
public List<MetadataRecord> listVersionRecords(@PathVariable final String versionId, @PathVariable final long limit) throws MDStoreManagerException {
|
||||||
return service.listVersionRecords(versionId, limit);
|
return service.listVersionRecords(versionId, limit);
|
||||||
}
|
}
|
||||||
|
|
||||||
@Operation(summary = "read the parquet file of a mdstore (current version)")
|
@Operation(summary = "read the parquet file of a mdstore (current version)")
|
||||||
@GetMapping("/mdstore/{mdId}/content/{limit}")
|
@GetMapping("/mdstore/{mdId}/content/{limit}")
|
||||||
public List<Map<String, String>> listMdstoreRecords(@PathVariable final String mdId, @PathVariable final long limit) throws MDStoreManagerException {
|
public List<MetadataRecord> listMdstoreRecords(@PathVariable final String mdId, @PathVariable final long limit) throws MDStoreManagerException {
|
||||||
return service.listVersionRecords(service.findMdStore(mdId).getCurrentVersion(), limit);
|
return service.listVersionRecords(service.findMdStore(mdId).getCurrentVersion(), limit);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -313,7 +313,7 @@ export class ISService {
|
||||||
}
|
}
|
||||||
|
|
||||||
loadMDStoreVersionRecords(versionId: string, limit: number, onSuccess: Function): void {
|
loadMDStoreVersionRecords(versionId: string, limit: number, onSuccess: Function): void {
|
||||||
this.client.get<MDStoreRecord[]>('./ajax/mdstores/version/' + encodeURIComponent(versionId) + '/parquet/content/' + limit).subscribe({
|
this.client.get<MDStoreRecord[]>('./ajax/mdstores/version/' + encodeURIComponent(versionId) + '/content/' + limit).subscribe({
|
||||||
next: data => onSuccess(data),
|
next: data => onSuccess(data),
|
||||||
error: error => this.showError(error)
|
error: error => this.showError(error)
|
||||||
});
|
});
|
||||||
|
|
|
@ -19,7 +19,7 @@
|
||||||
<td>{{mdstore?.datasourceName}}</td>
|
<td>{{mdstore?.datasourceName}}</td>
|
||||||
</tr>
|
</tr>
|
||||||
<tr>
|
<tr>
|
||||||
<th rowspan="4">Version</th>
|
<th [attr.rowspan]="(version?.params | keyvalue).length + 3">Version</th>
|
||||||
<th>ID</th>
|
<th>ID</th>
|
||||||
<td>
|
<td>
|
||||||
<span *ngIf="version?.id == mdstore?.currentVersion" class="badge-label badge-success">current</span>
|
<span *ngIf="version?.id == mdstore?.currentVersion" class="badge-label badge-success">current</span>
|
||||||
|
@ -45,16 +45,17 @@
|
||||||
</tr>
|
</tr>
|
||||||
</table>
|
</table>
|
||||||
|
|
||||||
<br />
|
<h3 *ngIf="records.length > 0" class="muted" style="margin-top: 2em; text-align: center;">The display is limited to the
|
||||||
<h3 *ngIf="records.length > 0">The display is limited to the first {{limit}} records</h3>
|
first {{limit}}
|
||||||
<br />
|
records</h3>
|
||||||
|
|
||||||
<mat-card *ngFor="let rec of records" style="margin-top: 10px;">
|
<mat-card *ngFor="let rec of records" style="margin-top: 1em;">
|
||||||
<mat-card-header>
|
<mat-card-header>
|
||||||
<mat-card-title *ngIf="!rec.id">the record is unreadable</mat-card-title>
|
<mat-card-title *ngIf="!rec.id">the record is unreadable</mat-card-title>
|
||||||
<mat-card-title *ngIf="rec.id">{{rec.id}}</mat-card-title>
|
<mat-card-title *ngIf="rec.id">{{rec.id}}</mat-card-title>
|
||||||
</mat-card-header>
|
</mat-card-header>
|
||||||
<mat-card-content>
|
<mat-card-content>
|
||||||
|
<br />
|
||||||
<p *ngIf="!rec.id">Invalid record format</p>
|
<p *ngIf="!rec.id">Invalid record format</p>
|
||||||
<table class="mdstore-table" *ngIf="rec.id">
|
<table class="mdstore-table" *ngIf="rec.id">
|
||||||
<tr>
|
<tr>
|
||||||
|
|
|
@ -19,12 +19,16 @@ import org.springframework.beans.factory.annotation.Autowired;
|
||||||
import org.springframework.jdbc.core.JdbcTemplate;
|
import org.springframework.jdbc.core.JdbcTemplate;
|
||||||
import org.springframework.stereotype.Service;
|
import org.springframework.stereotype.Service;
|
||||||
|
|
||||||
import eu.dnetlib.data.mdstore.hadoop.HdfsBackend;
|
import eu.dnetlib.data.mdstore.backends.DefaultBackend;
|
||||||
|
import eu.dnetlib.data.mdstore.backends.HdfsBackend;
|
||||||
|
import eu.dnetlib.data.mdstore.backends.MDStoreBackend;
|
||||||
|
import eu.dnetlib.data.mdstore.backends.MockBackend;
|
||||||
import eu.dnetlib.data.mdstore.model.MDStore;
|
import eu.dnetlib.data.mdstore.model.MDStore;
|
||||||
import eu.dnetlib.data.mdstore.model.MDStoreCurrentVersion;
|
import eu.dnetlib.data.mdstore.model.MDStoreCurrentVersion;
|
||||||
import eu.dnetlib.data.mdstore.model.MDStoreType;
|
import eu.dnetlib.data.mdstore.model.MDStoreType;
|
||||||
import eu.dnetlib.data.mdstore.model.MDStoreVersion;
|
import eu.dnetlib.data.mdstore.model.MDStoreVersion;
|
||||||
import eu.dnetlib.data.mdstore.model.MDStoreWithInfo;
|
import eu.dnetlib.data.mdstore.model.MDStoreWithInfo;
|
||||||
|
import eu.dnetlib.data.mdstore.model.MetadataRecord;
|
||||||
import eu.dnetlib.data.mdstore.repository.MDStoreCurrentVersionRepository;
|
import eu.dnetlib.data.mdstore.repository.MDStoreCurrentVersionRepository;
|
||||||
import eu.dnetlib.data.mdstore.repository.MDStoreRepository;
|
import eu.dnetlib.data.mdstore.repository.MDStoreRepository;
|
||||||
import eu.dnetlib.data.mdstore.repository.MDStoreVersionRepository;
|
import eu.dnetlib.data.mdstore.repository.MDStoreVersionRepository;
|
||||||
|
@ -48,6 +52,8 @@ public class MDStoreService {
|
||||||
@Autowired
|
@Autowired
|
||||||
private HdfsBackend hdfsBackend;
|
private HdfsBackend hdfsBackend;
|
||||||
@Autowired
|
@Autowired
|
||||||
|
private MockBackend mockBackend;
|
||||||
|
@Autowired
|
||||||
private DefaultBackend defaultBackend;
|
private DefaultBackend defaultBackend;
|
||||||
|
|
||||||
private static final Logger log = LoggerFactory.getLogger(MDStoreService.class);
|
private static final Logger log = LoggerFactory.getLogger(MDStoreService.class);
|
||||||
|
@ -228,7 +234,7 @@ public class MDStoreService {
|
||||||
return selectBackend(md.getType()).listInternalFiles(v);
|
return selectBackend(md.getType()).listInternalFiles(v);
|
||||||
}
|
}
|
||||||
|
|
||||||
public List<Map<String, String>> listVersionRecords(final String versionId, final long limit) throws MDStoreManagerException {
|
public List<MetadataRecord> listVersionRecords(final String versionId, final long limit) throws MDStoreManagerException {
|
||||||
final MDStoreVersion v = mdstoreVersionRepository.findById(versionId).orElseThrow(() -> new MDStoreManagerException("Version not found"));
|
final MDStoreVersion v = mdstoreVersionRepository.findById(versionId).orElseThrow(() -> new MDStoreManagerException("Version not found"));
|
||||||
final MDStore md = mdstoreRepository.findById(v.getMdstore()).orElseThrow(() -> new MDStoreManagerException("MDStore not found"));
|
final MDStore md = mdstoreRepository.findById(v.getMdstore()).orElseThrow(() -> new MDStoreManagerException("MDStore not found"));
|
||||||
return selectBackend(md.getType()).listEntries(v, limit);
|
return selectBackend(md.getType()).listEntries(v, limit);
|
||||||
|
@ -265,6 +271,7 @@ public class MDStoreService {
|
||||||
public Map<MDStoreType, Set<String>> fixInconsistencies(final boolean delete) throws MDStoreManagerException {
|
public Map<MDStoreType, Set<String>> fixInconsistencies(final boolean delete) throws MDStoreManagerException {
|
||||||
final Map<MDStoreType, Set<String>> res = new LinkedHashMap<>();
|
final Map<MDStoreType, Set<String>> res = new LinkedHashMap<>();
|
||||||
res.put(MDStoreType.HDFS, hdfsBackend.fixInconsistencies(delete));
|
res.put(MDStoreType.HDFS, hdfsBackend.fixInconsistencies(delete));
|
||||||
|
res.put(MDStoreType.MOCK, mockBackend.fixInconsistencies(delete));
|
||||||
// TODO: ADD HERE THE INVOCATION FOR OTHER MDSTORE TYPE
|
// TODO: ADD HERE THE INVOCATION FOR OTHER MDSTORE TYPE
|
||||||
|
|
||||||
return res;
|
return res;
|
||||||
|
@ -274,6 +281,8 @@ public class MDStoreService {
|
||||||
switch (type) {
|
switch (type) {
|
||||||
case HDFS:
|
case HDFS:
|
||||||
return hdfsBackend;
|
return hdfsBackend;
|
||||||
|
case MOCK:
|
||||||
|
return mockBackend;
|
||||||
default:
|
default:
|
||||||
return defaultBackend;
|
return defaultBackend;
|
||||||
}
|
}
|
||||||
|
|
|
@ -1,16 +1,16 @@
|
||||||
package eu.dnetlib.data.mdstore;
|
package eu.dnetlib.data.mdstore.backends;
|
||||||
|
|
||||||
import java.util.ArrayList;
|
import java.util.ArrayList;
|
||||||
import java.util.HashSet;
|
import java.util.HashSet;
|
||||||
import java.util.LinkedHashSet;
|
import java.util.LinkedHashSet;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import java.util.Map;
|
|
||||||
import java.util.Set;
|
import java.util.Set;
|
||||||
|
|
||||||
import org.springframework.stereotype.Service;
|
import org.springframework.stereotype.Service;
|
||||||
|
|
||||||
import eu.dnetlib.data.mdstore.model.MDStore;
|
import eu.dnetlib.data.mdstore.model.MDStore;
|
||||||
import eu.dnetlib.data.mdstore.model.MDStoreVersion;
|
import eu.dnetlib.data.mdstore.model.MDStoreVersion;
|
||||||
|
import eu.dnetlib.data.mdstore.model.MetadataRecord;
|
||||||
import eu.dnetlib.errors.MDStoreManagerException;
|
import eu.dnetlib.errors.MDStoreManagerException;
|
||||||
|
|
||||||
@Service
|
@Service
|
||||||
|
@ -29,7 +29,7 @@ public class DefaultBackend implements MDStoreBackend {
|
||||||
public void delete(final MDStoreVersion version) throws MDStoreManagerException {}
|
public void delete(final MDStoreVersion version) throws MDStoreManagerException {}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public List<Map<String, String>> listEntries(final MDStoreVersion version, final long limit) throws MDStoreManagerException {
|
public List<MetadataRecord> listEntries(final MDStoreVersion version, final long limit) throws MDStoreManagerException {
|
||||||
return new ArrayList<>();
|
return new ArrayList<>();
|
||||||
}
|
}
|
||||||
|
|
|
@ -1,8 +1,7 @@
|
||||||
package eu.dnetlib.data.mdstore.hadoop;
|
package eu.dnetlib.data.mdstore.backends;
|
||||||
|
|
||||||
import java.util.HashSet;
|
import java.util.HashSet;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import java.util.Map;
|
|
||||||
import java.util.Set;
|
import java.util.Set;
|
||||||
|
|
||||||
import org.apache.commons.lang3.StringUtils;
|
import org.apache.commons.lang3.StringUtils;
|
||||||
|
@ -15,9 +14,10 @@ import org.springframework.stereotype.Service;
|
||||||
|
|
||||||
import com.google.common.collect.Sets;
|
import com.google.common.collect.Sets;
|
||||||
|
|
||||||
import eu.dnetlib.data.mdstore.MDStoreBackend;
|
import eu.dnetlib.data.mdstore.hadoop.HdfsClient;
|
||||||
import eu.dnetlib.data.mdstore.model.MDStore;
|
import eu.dnetlib.data.mdstore.model.MDStore;
|
||||||
import eu.dnetlib.data.mdstore.model.MDStoreVersion;
|
import eu.dnetlib.data.mdstore.model.MDStoreVersion;
|
||||||
|
import eu.dnetlib.data.mdstore.model.MetadataRecord;
|
||||||
import eu.dnetlib.errors.MDStoreManagerException;
|
import eu.dnetlib.errors.MDStoreManagerException;
|
||||||
|
|
||||||
@Service
|
@Service
|
||||||
|
@ -90,10 +90,10 @@ public class HdfsBackend implements MDStoreBackend {
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public List<Map<String, String>> listEntries(final MDStoreVersion version, final long limit) throws MDStoreManagerException {
|
public List<MetadataRecord> listEntries(final MDStoreVersion version, final long limit) throws MDStoreManagerException {
|
||||||
final String path = version.getParams().getOrDefault("hdfs_path", "").toString();
|
final String path = version.getParams().getOrDefault("hdfs_path", "").toString();
|
||||||
if (StringUtils.isNotBlank(path)) {
|
if (StringUtils.isNotBlank(path)) {
|
||||||
return hdfsClient.readParquetFiles(path + "/store", limit);
|
return hdfsClient.readParquetFiles(path + "/store", limit, MetadataRecord.class);
|
||||||
} else {
|
} else {
|
||||||
throw new MDStoreManagerException("hdfs path is missing");
|
throw new MDStoreManagerException("hdfs path is missing");
|
||||||
}
|
}
|
|
@ -1,11 +1,11 @@
|
||||||
package eu.dnetlib.data.mdstore;
|
package eu.dnetlib.data.mdstore.backends;
|
||||||
|
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import java.util.Map;
|
|
||||||
import java.util.Set;
|
import java.util.Set;
|
||||||
|
|
||||||
import eu.dnetlib.data.mdstore.model.MDStore;
|
import eu.dnetlib.data.mdstore.model.MDStore;
|
||||||
import eu.dnetlib.data.mdstore.model.MDStoreVersion;
|
import eu.dnetlib.data.mdstore.model.MDStoreVersion;
|
||||||
|
import eu.dnetlib.data.mdstore.model.MetadataRecord;
|
||||||
import eu.dnetlib.errors.MDStoreManagerException;
|
import eu.dnetlib.errors.MDStoreManagerException;
|
||||||
|
|
||||||
public interface MDStoreBackend {
|
public interface MDStoreBackend {
|
||||||
|
@ -18,7 +18,7 @@ public interface MDStoreBackend {
|
||||||
|
|
||||||
void delete(MDStoreVersion version) throws MDStoreManagerException;
|
void delete(MDStoreVersion version) throws MDStoreManagerException;
|
||||||
|
|
||||||
List<Map<String, String>> listEntries(MDStoreVersion version, long limit) throws MDStoreManagerException;
|
List<MetadataRecord> listEntries(MDStoreVersion version, long limit) throws MDStoreManagerException;
|
||||||
|
|
||||||
Set<String> listInternalFiles(MDStoreVersion version) throws MDStoreManagerException;
|
Set<String> listInternalFiles(MDStoreVersion version) throws MDStoreManagerException;
|
||||||
|
|
|
@ -0,0 +1,72 @@
|
||||||
|
package eu.dnetlib.data.mdstore.backends;
|
||||||
|
|
||||||
|
import java.util.ArrayList;
|
||||||
|
import java.util.Arrays;
|
||||||
|
import java.util.Date;
|
||||||
|
import java.util.LinkedHashSet;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.Set;
|
||||||
|
|
||||||
|
import org.springframework.stereotype.Service;
|
||||||
|
|
||||||
|
import eu.dnetlib.data.mdstore.model.MDStore;
|
||||||
|
import eu.dnetlib.data.mdstore.model.MDStoreVersion;
|
||||||
|
import eu.dnetlib.data.mdstore.model.MetadataRecord;
|
||||||
|
import eu.dnetlib.data.mdstore.model.Provenance;
|
||||||
|
import eu.dnetlib.errors.MDStoreManagerException;
|
||||||
|
|
||||||
|
@Service
|
||||||
|
public class MockBackend implements MDStoreBackend {
|
||||||
|
|
||||||
|
private static final Provenance MOCK_PROVENANCE = new Provenance("mock________::MOCK_DS", "Mock Datasource", "mock________");
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void completeNewMDStore(final MDStore mdstore) {
|
||||||
|
mdstore.getParams().put("mockParam1", "test");
|
||||||
|
mdstore.getParams().put("mockParam2", "abc");
|
||||||
|
mdstore.getParams().put("mockParam3", 1234);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void completeNewMDStoreVersion(final MDStoreVersion version) {
|
||||||
|
version.getParams().put("mockParam1", "v_test");
|
||||||
|
version.getParams().put("mockParam2", "v_abc");
|
||||||
|
version.getParams().put("mockParam3", 1234);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void delete(final MDStore mdstore) throws MDStoreManagerException {}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void delete(final MDStoreVersion version) throws MDStoreManagerException {}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public List<MetadataRecord> listEntries(final MDStoreVersion version, final long limit) throws MDStoreManagerException {
|
||||||
|
final List<MetadataRecord> list = new ArrayList<>();
|
||||||
|
|
||||||
|
for (int i = 0; i < limit; i++) {
|
||||||
|
final MetadataRecord rec = new MetadataRecord();
|
||||||
|
rec.setOriginalId("mck-" + i);
|
||||||
|
rec.setId("mock________::mck-" + i);
|
||||||
|
rec.setBody("<RECORD>" + i + "</RECORD>");
|
||||||
|
rec.setDateOfCollection(new Date().getTime());
|
||||||
|
rec.setDateOfTransformation(new Date().getTime());
|
||||||
|
rec.setEncoding("XML");
|
||||||
|
rec.setProvenance(MOCK_PROVENANCE);
|
||||||
|
list.add(rec);
|
||||||
|
}
|
||||||
|
|
||||||
|
return list;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public Set<String> listInternalFiles(final MDStoreVersion version) throws MDStoreManagerException {
|
||||||
|
return new LinkedHashSet<>(Arrays.asList("file1", "file2", "file3", "file4"));
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public Set<String> fixInconsistencies(final boolean delete) throws MDStoreManagerException {
|
||||||
|
return new LinkedHashSet<>(Arrays.asList("1", "2", "3", "4"));
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
|
@ -20,6 +20,8 @@ import org.apache.hadoop.fs.Path;
|
||||||
import org.springframework.beans.factory.annotation.Value;
|
import org.springframework.beans.factory.annotation.Value;
|
||||||
import org.springframework.stereotype.Component;
|
import org.springframework.stereotype.Component;
|
||||||
|
|
||||||
|
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||||
|
|
||||||
import eu.dnetlib.errors.MDStoreManagerException;
|
import eu.dnetlib.errors.MDStoreManagerException;
|
||||||
import parquet.avro.AvroParquetReader;
|
import parquet.avro.AvroParquetReader;
|
||||||
import parquet.hadoop.ParquetReader;
|
import parquet.hadoop.ParquetReader;
|
||||||
|
@ -96,9 +98,9 @@ public class HdfsClient {
|
||||||
}
|
}
|
||||||
|
|
||||||
@SuppressWarnings("unchecked")
|
@SuppressWarnings("unchecked")
|
||||||
public List<Map<String, String>> readParquetFiles(final String path, final long limit) throws MDStoreManagerException {
|
public <T> List<T> readParquetFiles(final String path, final long limit, final Class<T> clazz) throws MDStoreManagerException {
|
||||||
|
|
||||||
final List<Map<String, String>> list = new ArrayList<>();
|
final List<T> list = new ArrayList<>();
|
||||||
|
|
||||||
final Configuration conf = conf();
|
final Configuration conf = conf();
|
||||||
|
|
||||||
|
@ -113,6 +115,8 @@ public class HdfsClient {
|
||||||
try (final ParquetReader<GenericRecord> reader = AvroParquetReader.<GenericRecord> builder(new Path(f)).withConf(conf).build()) {
|
try (final ParquetReader<GenericRecord> reader = AvroParquetReader.<GenericRecord> builder(new Path(f)).withConf(conf).build()) {
|
||||||
log.debug("File parquet OPENED");
|
log.debug("File parquet OPENED");
|
||||||
|
|
||||||
|
final ObjectMapper mapper = new ObjectMapper();
|
||||||
|
|
||||||
GenericRecord rec = null;
|
GenericRecord rec = null;
|
||||||
while (i++ < limit && (rec = reader.read()) != null) {
|
while (i++ < limit && (rec = reader.read()) != null) {
|
||||||
if (fields.isEmpty()) {
|
if (fields.isEmpty()) {
|
||||||
|
@ -124,7 +128,9 @@ public class HdfsClient {
|
||||||
final Object v = rec.get(field);
|
final Object v = rec.get(field);
|
||||||
map.put(field, v != null ? v.toString() : "");
|
map.put(field, v != null ? v.toString() : "");
|
||||||
}
|
}
|
||||||
list.add(map);
|
|
||||||
|
list.add(mapper.convertValue(map, clazz));
|
||||||
|
|
||||||
log.debug("added record");
|
log.debug("added record");
|
||||||
}
|
}
|
||||||
} catch (final FileNotFoundException e) {
|
} catch (final FileNotFoundException e) {
|
||||||
|
|
Loading…
Reference in New Issue