diff --git a/apps/dnet-is-application/src/main/java/eu/dnetlib/data/mdstore/AbstractMDStoreController.java b/apps/dnet-is-application/src/main/java/eu/dnetlib/data/mdstore/AbstractMDStoreController.java new file mode 100644 index 00000000..d5d629cd --- /dev/null +++ b/apps/dnet-is-application/src/main/java/eu/dnetlib/data/mdstore/AbstractMDStoreController.java @@ -0,0 +1,129 @@ +package eu.dnetlib.data.mdstore; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.web.bind.annotation.DeleteMapping; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestParam; + +import eu.dnetlib.common.controller.AbstractDnetController; +import eu.dnetlib.dhp.schema.mdstore.MDStoreVersion; +import eu.dnetlib.dhp.schema.mdstore.MDStoreWithInfo; +import eu.dnetlib.errors.MDStoreManagerException; +import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.Parameter; + +public class AbstractMDStoreController extends AbstractDnetController { + + @Autowired + protected MDStoreService service; + + @Operation(summary = "Return all the mdstores") + @GetMapping("/") + public Iterable find() { + return service.listMdStores(); + } + + @Operation(summary = "Return a mdstores by id") + @GetMapping("/mdstore/{mdId}") + public MDStoreWithInfo getMdStore(@Parameter(name = "the mdstore identifier") @PathVariable final String mdId) throws MDStoreManagerException { + return service.findMdStore(mdId); + } + + @Operation(summary = "Increase the read count of the current mdstore") + @GetMapping("/mdstore/{mdId}/startReading") + public MDStoreVersion startReading(@Parameter(name = "the mdstore identifier") @PathVariable final String mdId) throws MDStoreManagerException { + return service.startReading(mdId); + } + + @Operation(summary = "Create a new mdstore") + @PostMapping("/new/{format}/{layout}/{interpretation}") + public MDStoreWithInfo createMDStore( + @Parameter(name = "mdstore format") @PathVariable final String format, + @Parameter(name = "mdstore layout") @PathVariable final String layout, + @Parameter(name = "mdstore interpretation") @PathVariable final String interpretation, + @Parameter(name = "datasource name") @RequestParam(required = true) final String dsName, + @Parameter(name = "datasource id") @RequestParam(required = true) final String dsId, + @Parameter(name = "api id") @RequestParam(required = true) final String apiId) throws MDStoreManagerException { + final String id = service.createMDStore(format, layout, interpretation, dsName, dsId, apiId); + return service.findMdStore(id); + } + + @Operation(summary = "Delete a mdstore by id") + @DeleteMapping("/mdstore/{mdId}") + public StatusResponse delete(@Parameter(name = "the id of the mdstore that will be deleted") @PathVariable final String mdId) + throws MDStoreManagerException { + service.deleteMdStore(mdId); + + return StatusResponse.DELETED; + } + + @Operation(summary = "Return all the versions of a mdstore") + @GetMapping("/mdstore/{mdId}/versions") + public Iterable listVersions(@PathVariable final String mdId) throws MDStoreManagerException { + return service.listVersions(mdId); + } + + @Operation(summary = "Create a new preliminary version of a mdstore") + @GetMapping("/mdstore/{mdId}/newVersion") + public MDStoreVersion prepareNewVersion( + @Parameter(name = "the id of the mdstore for which will be created a new version") @PathVariable final String mdId) { + return service.prepareMdStoreVersion(mdId); + } + + @Operation(summary = "Promote a preliminary version to current") + @GetMapping("/version/{versionId}/commit/{size}") + public MDStoreVersion commitVersion( + @Parameter(name = "the id of the version that will be promoted to the current version") @PathVariable final String versionId, + @Parameter(name = "the size of the new current mdstore") @PathVariable final long size) throws MDStoreManagerException { + try { + return service.commitMdStoreVersion(versionId, size); + } finally { + service.deleteExpiredVersions(); + } + } + + @Operation(summary = "Abort a preliminary version") + @GetMapping("/version/{versionId}/abort") + public StatusResponse commitVersion(@Parameter(name = "the id of the version to abort") @PathVariable final String versionId) + throws MDStoreManagerException { + + service.deleteMdStoreVersion(versionId, true); + + return StatusResponse.ABORTED; + } + + @Operation(summary = "Return an existing mdstore version") + @GetMapping("/version/{versionId}") + public MDStoreVersion getVersion(@Parameter(name = "the id of the version that has to be deleted") @PathVariable final String versionId) + throws MDStoreManagerException { + return service.findVersion(versionId); + } + + @Operation(summary = "Delete a mdstore version") + @DeleteMapping("/version/{versionId}") + public StatusResponse deleteVersion(@Parameter(name = "the id of the version that has to be deleted") @PathVariable final String versionId, + @Parameter(name = "if true, the controls on writing and readcount values will be skipped") @RequestParam(required = false, defaultValue = "false") final boolean force) + throws MDStoreManagerException { + + service.deleteMdStoreVersion(versionId, force); + + return StatusResponse.DELETED; + } + + @Operation(summary = "Decrease the read count of a mdstore version") + @GetMapping("/version/{versionId}/endReading") + public MDStoreVersion endReading(@Parameter(name = "the id of the version that has been completely read") @PathVariable final String versionId) + throws MDStoreManagerException { + return service.endReading(versionId); + } + + @Operation(summary = "Reset the read count of a mdstore version") + @GetMapping("/version/{versionId}/resetReading") + public MDStoreVersion resetReading(@Parameter(name = "the id of the version") @PathVariable final String versionId) + throws MDStoreManagerException { + return service.resetReading(versionId); + } + +} diff --git a/apps/dnet-is-application/src/main/java/eu/dnetlib/data/mdstore/MDStoreAjaxController.java b/apps/dnet-is-application/src/main/java/eu/dnetlib/data/mdstore/MDStoreAjaxController.java new file mode 100644 index 00000000..43f7a300 --- /dev/null +++ b/apps/dnet-is-application/src/main/java/eu/dnetlib/data/mdstore/MDStoreAjaxController.java @@ -0,0 +1,54 @@ +package eu.dnetlib.data.mdstore; + +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +import org.springframework.beans.factory.annotation.Value; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RestController; + +import eu.dnetlib.errors.MDStoreManagerException; +import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.tags.Tag; + +@RestController +@RequestMapping("/ajax/mdstores") +@Tag(name = "Metadata Stores") +public class MDStoreAjaxController extends AbstractMDStoreController { + + @Value("${dhp.mdstore-manager.hadoop.cluster}") + private String hadoopCluster; + + @Value("${dhp.mdstore-manager.hadoop.user}") + private String hadoopUser; + + @Value("${dhp.mdstore-manager.hdfs.base-path}") + private String hdfsBasePath; + + @Operation(summary = "Show informations") + @GetMapping("/info") + public Map info() { + final Map info = new LinkedHashMap<>(); + info.put("number_of_mdstores", service.countMdStores()); + info.put("hadoop_user", hadoopUser); + info.put("hadoop_cluster", hadoopCluster); + info.put("hdfs_base_path", hdfsBasePath); + info.put("expired_versions", service.listExpiredVersions()); + return info; + } + + @Operation(summary = "read the parquet file of a mdstore version") + @GetMapping("/version/{versionId}/parquet/content/{limit}") + public List> listVersionParquet(@PathVariable final String versionId, @PathVariable final long limit) throws MDStoreManagerException { + return service.listVersionParquet(versionId, limit); + } + + @Operation(summary = "read the parquet file of a mdstore (current version)") + @GetMapping("/mdstore/{mdId}/parquet/content/{limit}") + public List> listMdstoreParquet(@PathVariable final String mdId, @PathVariable final long limit) throws MDStoreManagerException { + return service.listMdstoreParquet(mdId, limit); + } +} diff --git a/apps/dnet-is-application/src/main/java/eu/dnetlib/data/mdstore/MDStoreApiController.java b/apps/dnet-is-application/src/main/java/eu/dnetlib/data/mdstore/MDStoreApiController.java new file mode 100644 index 00000000..cdfffc1b --- /dev/null +++ b/apps/dnet-is-application/src/main/java/eu/dnetlib/data/mdstore/MDStoreApiController.java @@ -0,0 +1,50 @@ +package eu.dnetlib.data.mdstore; + +import java.util.List; +import java.util.Set; + +import org.springframework.web.bind.annotation.DeleteMapping; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.RestController; + +import eu.dnetlib.errors.MDStoreManagerException; +import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.Parameter; +import io.swagger.v3.oas.annotations.tags.Tag; + +@RestController +@RequestMapping("/api/mdstores") +@Tag(name = "Metadata Stores") +public class MDStoreApiController extends AbstractMDStoreController { + + @Operation(summary = "Return all the mdstore identifiers") + @GetMapping("/ids") + public List findIdentifiers() { + return service.listMdStoreIDs(); + } + + @Operation(summary = "Fix the inconsistencies on HDFS") + @GetMapping("/hdfs/inconsistencies") + public Set fixHdfsInconsistencies( + @Parameter(name = "force the deletion of hdfs paths") @RequestParam(required = false, defaultValue = "false") final boolean delete) + throws MDStoreManagerException { + + return service.fixHdfsInconsistencies(delete); + } + + @Operation(summary = "Delete expired versions") + @DeleteMapping("/versions/expired") + public StatusResponse deleteExpiredVersions() { + return service.deleteExpiredVersions(); + } + + @Operation(summary = "list the file inside the path of a mdstore version") + @GetMapping("/version/{versionId}/parquet/files") + public Set listVersionFiles(@PathVariable final String versionId) throws MDStoreManagerException { + return service.listVersionFiles(versionId); + } + +} diff --git a/apps/dnet-is-application/src/main/java/eu/dnetlib/data/mdstore/MDStoreController.java b/apps/dnet-is-application/src/main/java/eu/dnetlib/data/mdstore/MDStoreController.java deleted file mode 100644 index fc3141bd..00000000 --- a/apps/dnet-is-application/src/main/java/eu/dnetlib/data/mdstore/MDStoreController.java +++ /dev/null @@ -1,231 +0,0 @@ -package eu.dnetlib.data.mdstore; - -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.Set; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.web.bind.annotation.DeleteMapping; -import org.springframework.web.bind.annotation.GetMapping; -import org.springframework.web.bind.annotation.PathVariable; -import org.springframework.web.bind.annotation.RequestMapping; -import org.springframework.web.bind.annotation.RequestParam; -import org.springframework.web.bind.annotation.RestController; - -import com.google.common.collect.Sets; - -import eu.dnetlib.common.controller.AbstractDnetController; -import eu.dnetlib.data.mdstore.hadoop.HdfsClient; -import eu.dnetlib.dhp.schema.mdstore.MDStoreVersion; -import eu.dnetlib.dhp.schema.mdstore.MDStoreWithInfo; -import eu.dnetlib.errors.MDStoreManagerException; -import io.swagger.v3.oas.annotations.Operation; -import io.swagger.v3.oas.annotations.Parameter; -import io.swagger.v3.oas.annotations.tags.Tag; - -@RestController -@RequestMapping("/mdstores") -@Tag(name = "Metadata Stores") -public class MDStoreController extends AbstractDnetController { - - @Autowired - private MDStoreService databaseUtils; - - @Autowired - private HdfsClient hdfsClient; - - private static final Logger log = LoggerFactory.getLogger(MDStoreService.class); - - @Operation(summary = "Return all the mdstores") - @GetMapping("/") - public Iterable find() { - return databaseUtils.listMdStores(); - } - - @Operation(summary = "Return all the mdstore identifiers") - @GetMapping("/ids") - public List findIdentifiers() { - return databaseUtils.listMdStoreIDs(); - } - - @Operation(summary = "Return a mdstores by id") - @GetMapping("/mdstore/{mdId}") - public MDStoreWithInfo getMdStore(@Parameter(name = "the mdstore identifier") @PathVariable final String mdId) throws MDStoreManagerException { - return databaseUtils.findMdStore(mdId); - } - - @Operation(summary = "Increase the read count of the current mdstore") - @GetMapping("/mdstore/{mdId}/startReading") - public MDStoreVersion startReading(@Parameter(name = "the mdstore identifier") @PathVariable final String mdId) throws MDStoreManagerException { - return databaseUtils.startReading(mdId); - } - - @Operation(summary = "Create a new mdstore") - @GetMapping("/new/{format}/{layout}/{interpretation}") - public MDStoreWithInfo createMDStore( - @Parameter(name = "mdstore format") @PathVariable final String format, - @Parameter(name = "mdstore layout") @PathVariable final String layout, - @Parameter(name = "mdstore interpretation") @PathVariable final String interpretation, - @Parameter(name = "datasource name") @RequestParam(required = true) final String dsName, - @Parameter(name = "datasource id") @RequestParam(required = true) final String dsId, - @Parameter(name = "api id") @RequestParam(required = true) final String apiId) throws MDStoreManagerException { - final String id = databaseUtils.createMDStore(format, layout, interpretation, dsName, dsId, apiId); - return databaseUtils.findMdStore(id); - } - - @Operation(summary = "Delete a mdstore by id") - @DeleteMapping("/mdstore/{mdId}") - public StatusResponse delete(@Parameter(name = "the id of the mdstore that will be deleted") @PathVariable final String mdId) throws MDStoreManagerException { - final String hdfsPath = databaseUtils.deleteMdStore(mdId); - hdfsClient.deletePath(hdfsPath); - return StatusResponse.DELETED; - } - - @Operation(summary = "Return all the versions of a mdstore") - @GetMapping("/mdstore/{mdId}/versions") - public Iterable listVersions(@PathVariable final String mdId) throws MDStoreManagerException { - return databaseUtils.listVersions(mdId); - } - - @Operation(summary = "Create a new preliminary version of a mdstore") - @GetMapping("/mdstore/{mdId}/newVersion") - public MDStoreVersion prepareNewVersion(@Parameter(name = "the id of the mdstore for which will be created a new version") @PathVariable final String mdId) { - return databaseUtils.prepareMdStoreVersion(mdId); - } - - @Operation(summary = "Promote a preliminary version to current") - @GetMapping("/version/{versionId}/commit/{size}") - public MDStoreVersion commitVersion(@Parameter(name = "the id of the version that will be promoted to the current version") @PathVariable final String versionId, - @Parameter(name = "the size of the new current mdstore") @PathVariable final long size) throws MDStoreManagerException { - try { - return databaseUtils.commitMdStoreVersion(versionId, size); - } finally { - deleteExpiredVersions(); - } - } - - @Operation(summary = "Abort a preliminary version") - @GetMapping("/version/{versionId}/abort") - public StatusResponse commitVersion(@Parameter(name = "the id of the version to abort") @PathVariable final String versionId) throws MDStoreManagerException { - final String hdfsPath = databaseUtils.deleteMdStoreVersion(versionId, true); - hdfsClient.deletePath(hdfsPath); - return StatusResponse.ABORTED; - } - - @Operation(summary = "Return an existing mdstore version") - @GetMapping("/version/{versionId}") - public MDStoreVersion getVersion(@Parameter(name = "the id of the version that has to be deleted") @PathVariable final String versionId) - throws MDStoreManagerException { - return databaseUtils.findVersion(versionId); - } - - @Operation(summary = "Delete a mdstore version") - @DeleteMapping("/version/{versionId}") - public StatusResponse deleteVersion(@Parameter(name = "the id of the version that has to be deleted") @PathVariable final String versionId, - @Parameter(name = "if true, the controls on writing and readcount values will be skipped") @RequestParam(required = false, defaultValue = "false") final boolean force) - throws MDStoreManagerException { - final String hdfsPath = databaseUtils.deleteMdStoreVersion(versionId, force); - hdfsClient.deletePath(hdfsPath); - return StatusResponse.DELETED; - } - - @Operation(summary = "Decrease the read count of a mdstore version") - @GetMapping("/version/{versionId}/endReading") - public MDStoreVersion endReading(@Parameter(name = "the id of the version that has been completely read") @PathVariable final String versionId) - throws MDStoreManagerException { - return databaseUtils.endReading(versionId); - } - - @Operation(summary = "Reset the read count of a mdstore version") - @GetMapping("/version/{versionId}/resetReading") - public MDStoreVersion resetReading(@Parameter(name = "the id of the version") @PathVariable final String versionId) - throws MDStoreManagerException { - return databaseUtils.resetReading(versionId); - } - - @Operation(summary = "Delete expired versions") - @DeleteMapping("/versions/expired") - public StatusResponse deleteExpiredVersions() { - new Thread(this::performDeleteOfExpiredVersions).start(); - return StatusResponse.DELETING; - } - - private synchronized void performDeleteOfExpiredVersions() { - log.info("Deleting expired version..."); - for (final String versionId : databaseUtils.listExpiredVersions()) { - try { - final String hdfsPath = databaseUtils.deleteMdStoreVersion(versionId, true); - hdfsClient.deletePath(hdfsPath); - } catch (final MDStoreManagerException e) { - log.warn("Error deleteting version " + versionId, e); - } - } - log.info("Done."); - } - - @Operation(summary = "Fix the inconsistencies on HDFS") - @GetMapping("/hdfs/inconsistencies") - public Set fixHdfsInconsistencies( - @Parameter(name = "force the deletion of hdfs paths") @RequestParam(required = false, defaultValue = "false") final boolean delete) - throws MDStoreManagerException { - - final Set hdfsDirs = hdfsClient.listHadoopDirs(); - final Set validDirs = databaseUtils.listValidHdfsPaths(); - - final Set toDelete = Sets.difference(hdfsDirs, validDirs); - log.info("Found " + toDelete.size() + " hdfs paths to remove"); - - if (delete) { - for (final String p : toDelete) { - hdfsClient.deletePath(p); - } - } - return toDelete; - } - - @Operation(summary = "Show informations") - @GetMapping("/info") - public Map info() { - final Map info = new LinkedHashMap<>(); - info.put("number_of_mdstores", databaseUtils.countMdStores()); - info.put("hadoop_user", hdfsClient.getHadoopUser()); - info.put("hadoop_cluster", hdfsClient.getHadoopCluster()); - info.put("hdfs_base_path", databaseUtils.getHdfsBasePath()); - info.put("expired_versions", databaseUtils.listExpiredVersions()); - return info; - } - - @Operation(summary = "list the file inside the path of a mdstore version") - @GetMapping("/version/{versionId}/parquet/files") - public Set listVersionFiles(@PathVariable final String versionId) throws MDStoreManagerException { - final String path = databaseUtils.findVersion(versionId).getHdfsPath(); - return hdfsClient.listContent(path + "/store", HdfsClient::isParquetFile); - } - - @Operation(summary = "read the parquet file of a mdstore version") - @GetMapping("/version/{versionId}/parquet/content/{limit}") - public List> listVersionParquet(@PathVariable final String versionId, @PathVariable final long limit) throws MDStoreManagerException { - final String path = databaseUtils.findVersion(versionId).getHdfsPath(); - return hdfsClient.readParquetFiles(path + "/store", limit); - } - - @Operation(summary = "read the parquet file of a mdstore (current version)") - @GetMapping("/mdstore/{mdId}/parquet/content/{limit}") - public List> listMdstoreParquet(@PathVariable final String mdId, @PathVariable final long limit) throws MDStoreManagerException { - final String versionId = databaseUtils.findMdStore(mdId).getCurrentVersion(); - final String path = databaseUtils.findVersion(versionId).getHdfsPath(); - return hdfsClient.readParquetFiles(path + "/store", limit); - } - - protected void setDatabaseUtils(final MDStoreService databaseUtils) { - this.databaseUtils = databaseUtils; - } - - protected void setHdfsClient(final HdfsClient hdfsClient) { - this.hdfsClient = hdfsClient; - } - -} diff --git a/apps/dnet-is-application/src/main/java/eu/dnetlib/data/mdstore/MDStoreService.java b/apps/dnet-is-application/src/main/java/eu/dnetlib/data/mdstore/MDStoreService.java index 1233a3ee..d20281a5 100644 --- a/apps/dnet-is-application/src/main/java/eu/dnetlib/data/mdstore/MDStoreService.java +++ b/apps/dnet-is-application/src/main/java/eu/dnetlib/data/mdstore/MDStoreService.java @@ -3,6 +3,7 @@ package eu.dnetlib.data.mdstore; import java.util.Date; import java.util.HashSet; import java.util.List; +import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; @@ -16,6 +17,9 @@ import org.springframework.beans.factory.annotation.Value; import org.springframework.jdbc.core.JdbcTemplate; import org.springframework.stereotype.Service; +import com.google.common.collect.Sets; + +import eu.dnetlib.data.mdstore.hadoop.HdfsClient; import eu.dnetlib.data.mdstore.repository.MDStoreCurrentVersionRepository; import eu.dnetlib.data.mdstore.repository.MDStoreRepository; import eu.dnetlib.data.mdstore.repository.MDStoreVersionRepository; @@ -39,6 +43,8 @@ public class MDStoreService { private MDStoreWithInfoRepository mdstoreWithInfoRepository; @Autowired private JdbcTemplate jdbcTemplate; + @Autowired + protected HdfsClient hdfsClient; @Value("${dhp.mdstore-manager.hdfs.base-path}") private String hdfsBasePath; @@ -94,7 +100,7 @@ public class MDStoreService { } @Transactional - public String deleteMdStore(final String mdId) throws MDStoreManagerException { + public void deleteMdStore(final String mdId) throws MDStoreManagerException { final Optional md = mdstoreRepository.findById(mdId); @@ -117,7 +123,7 @@ public class MDStoreService { mdstoreVersionRepository.deleteByMdstore(mdId); mdstoreRepository.deleteById(mdId); - return md.get().getHdfsPath(); + hdfsClient.deletePath(md.get().getHdfsPath()); } @Transactional @@ -164,8 +170,25 @@ public class MDStoreService { return v; } + public StatusResponse deleteExpiredVersions() { + new Thread(this::performDeleteOfExpiredVersions).start(); + return StatusResponse.DELETING; + } + + private synchronized void performDeleteOfExpiredVersions() { + log.info("Deleting expired version..."); + for (final String versionId : listExpiredVersions()) { + try { + deleteMdStoreVersion(versionId, true); + } catch (final MDStoreManagerException e) { + log.warn("Error deleteting version " + versionId, e); + } + } + log.info("Done."); + } + @Transactional - public String deleteMdStoreVersion(final String versionId, final boolean force) throws MDStoreManagerException { + public void deleteMdStoreVersion(final String versionId, final boolean force) throws MDStoreManagerException { final MDStoreVersion v = mdstoreVersionRepository.findById(versionId).orElseThrow(() -> new MDStoreManagerException("Version not found")); @@ -181,20 +204,37 @@ public class MDStoreService { mdstoreVersionRepository.delete(v); - return v.getHdfsPath(); + hdfsClient.deletePath(v.getHdfsPath()); } - public Set listValidHdfsPaths() { - return new HashSet<>(jdbcTemplate - .queryForList(" select hdfs_path from mdstores union all select hdfs_path from mdstore_versions", String.class)); + public Set fixHdfsInconsistencies(final boolean delete) throws MDStoreManagerException { + final Set hdfsDirs = hdfsClient.listHadoopDirs(); + final Set validDirs = new HashSet<>(jdbcTemplate + .queryForList("select hdfs_path from mdstores union all select hdfs_path from mdstore_versions", String.class)); + + final Set toDelete = Sets.difference(hdfsDirs, validDirs); + log.info("Found " + toDelete.size() + " hdfs paths to remove"); + + if (delete) { + for (final String p : toDelete) { + hdfsClient.deletePath(p); + } + } + return toDelete; } - public String getHdfsBasePath() { - return hdfsBasePath; + public List> listMdstoreParquet(final String mdId, final long limit) throws MDStoreManagerException { + return listVersionParquet(findMdStore(mdId).getCurrentVersion(), limit); } - public void setHdfsBasePath(final String hdfsBasePath) { - this.hdfsBasePath = hdfsBasePath; + public List> listVersionParquet(final String versionId, final long limit) throws MDStoreManagerException { + final String path = findVersion(versionId).getHdfsPath(); + return hdfsClient.readParquetFiles(path + "/store", limit); + } + + public Set listVersionFiles(final String versionId) throws MDStoreManagerException { + final String path = findVersion(versionId).getHdfsPath(); + return hdfsClient.listContent(path + "/store", HdfsClient::isParquetFile); } } diff --git a/frontends/dnet-is-application/src/app/app.module.ts b/frontends/dnet-is-application/src/app/app.module.ts index bf5eba88..bb67000f 100644 --- a/frontends/dnet-is-application/src/app/app.module.ts +++ b/frontends/dnet-is-application/src/app/app.module.ts @@ -34,7 +34,7 @@ import { DsmSearchComponent, DsmResultsComponent, DsmApiComponent, DsmAddApiDial import { MatPaginatorModule } from '@angular/material/paginator'; import { MatProgressSpinnerModule } from '@angular/material/progress-spinner'; import { SpinnerHttpInterceptor } from './common/spinner.service'; -import { MdstoresComponent, MdstoreInspectorComponent } from './mdstores/mdstores.component'; +import { MdstoresComponent, MdstoreInspectorComponent, MDStoreVersionsDialog, AddMDStoreDialog } from './mdstores/mdstores.component'; @NgModule({ declarations: [ @@ -62,7 +62,9 @@ import { MdstoresComponent, MdstoreInspectorComponent } from './mdstores/mdstore DsmAddApiDialog, DsmBrowseDialog, MdstoresComponent, - MdstoreInspectorComponent + MdstoreInspectorComponent, + MDStoreVersionsDialog, + AddMDStoreDialog ], imports: [ BrowserModule, diff --git a/frontends/dnet-is-application/src/app/common/is.model.ts b/frontends/dnet-is-application/src/app/common/is.model.ts index 9158ee97..24b7f8a8 100644 --- a/frontends/dnet-is-application/src/app/common/is.model.ts +++ b/frontends/dnet-is-application/src/app/common/is.model.ts @@ -162,3 +162,29 @@ export interface DsmConf { contentDescTypes: string[], protocols: Protocol[] } + +export interface MDStore { + id: string, + format: string, + layout: string, + interpretation: string, + datasourceName: string, + datasourceId: string, + apiId: string, + currentVersion: string, + creationDate: string, + lastUpdate: string, + size: number, + numberOfVersions: number, + hdfsPath: string +} + +export interface MDStoreVersion { + id: string, + mdstore: string, + writing: boolean, + readCount:number, + lastUpdate:string, + siz: number, + hdfsPath: string; +} diff --git a/frontends/dnet-is-application/src/app/common/is.service.ts b/frontends/dnet-is-application/src/app/common/is.service.ts index f110c17e..d3c692d2 100644 --- a/frontends/dnet-is-application/src/app/common/is.service.ts +++ b/frontends/dnet-is-application/src/app/common/is.service.ts @@ -1,6 +1,6 @@ import { Injectable } from '@angular/core'; import { HttpClient, HttpHeaders, HttpParams } from '@angular/common/http'; -import { Page, DsmConf, ResourceType, Protocol, WfHistoryEntry, SimpleResource, Context, ContextNode, Vocabulary, VocabularyTerm, KeyValue, BrowseTerm, Datasource } from './is.model'; +import { Page, DsmConf, ResourceType, Protocol, WfHistoryEntry, SimpleResource, Context, ContextNode, Vocabulary, VocabularyTerm, KeyValue, BrowseTerm, Datasource, MDStore, MDStoreVersion } from './is.model'; import { FormGroup } from '@angular/forms'; import { MatSnackBar } from '@angular/material/snack-bar'; @@ -111,21 +111,21 @@ export class ISService { }); } - loadContext(ctxId:string, onSuccess: Function): void { + loadContext(ctxId: string, onSuccess: Function): void { this.client.get('./ajax/contexts/' + encodeURIComponent(ctxId)).subscribe({ next: data => onSuccess(data), error: error => this.showError(error) }); } - loadContextCategories(ctxId:string, onSuccess: Function): void { + loadContextCategories(ctxId: string, onSuccess: Function): void { this.client.get('./ajax/contexts/' + encodeURIComponent(ctxId) + '/categories').subscribe({ next: data => onSuccess(data), error: error => this.showError(error) }); } - loadContextConcepts(level:number, nodeId:string, onSuccess: Function): void { + loadContextConcepts(level: number, nodeId: string, onSuccess: Function): void { this.client.get('./ajax/contexts/' + encodeURIComponent(level) + '/' + encodeURIComponent(nodeId) + '/concepts').subscribe({ next: data => onSuccess(data), error: error => this.showError(error) @@ -139,45 +139,45 @@ export class ISService { }); } - loadVocabulary(vocId:string, onSuccess: Function): void { + loadVocabulary(vocId: string, onSuccess: Function): void { this.client.get('./ajax/vocs/' + encodeURIComponent(vocId)).subscribe({ next: data => onSuccess(data), error: error => this.showError(error) }); } - loadVocabularyTerms(vocId:string, onSuccess: Function): void { + loadVocabularyTerms(vocId: string, onSuccess: Function): void { this.client.get('./ajax/vocs/' + encodeURIComponent(vocId) + '/terms').subscribe({ next: data => onSuccess(data), error: error => this.showError(error) }); } - saveVocabulary(voc:Vocabulary, onSuccess: Function, relatedForm?: FormGroup): void { + saveVocabulary(voc: Vocabulary, onSuccess: Function, relatedForm?: FormGroup): void { this.client.post('./ajax/vocs/', voc).subscribe({ next: data => onSuccess(data), error: error => this.showError(error, relatedForm) }); } - saveVocabularyTerm(vocId:string, term:VocabularyTerm, onSuccess: Function, relatedForm?: FormGroup): void { + saveVocabularyTerm(vocId: string, term: VocabularyTerm, onSuccess: Function, relatedForm?: FormGroup): void { this.client.post('./ajax/vocs/' + encodeURIComponent(vocId) + '/terms', term).subscribe({ next: data => onSuccess(data), error: error => this.showError(error, relatedForm) }); } - deleteVocabulary(vocId:string, onSuccess: Function): void { + deleteVocabulary(vocId: string, onSuccess: Function): void { this.client.delete('./ajax/vocs/' + encodeURIComponent(vocId)).subscribe({ next: data => onSuccess(data), error: error => this.showError(error) }); } - deleteVocabularyTerm(vocId:string, termCode:string, onSuccess: Function): void { - this.client.delete('./ajax/vocs/' - + encodeURIComponent(vocId) - + '/terms/' + deleteVocabularyTerm(vocId: string, termCode: string, onSuccess: Function): void { + this.client.delete('./ajax/vocs/' + + encodeURIComponent(vocId) + + '/terms/' + encodeURIComponent(termCode) ).subscribe({ next: data => onSuccess(data), @@ -199,27 +199,68 @@ export class ISService { }); } - dsmBrowse(field:string, onSuccess: Function) { + dsmBrowse(field: string, onSuccess: Function) { this.client.get('./ajax/dsm/browse/' + encodeURIComponent(field)).subscribe({ next: data => onSuccess(data), error: error => this.showError(error) }); } - dsmSearchByField(field:string, value:string, page:number, pageSize:number, onSuccess: Function) { + dsmSearchByField(field: string, value: string, page: number, pageSize: number, onSuccess: Function) { this.client.get>('./ajax/dsm/searchByField/' + encodeURIComponent(field) + '/' + page + '/' + pageSize + '?value=' + encodeURIComponent(value)).subscribe({ next: data => onSuccess(data), error: error => this.showError(error) }); } - dsmSearch(value:string, page:number, pageSize:number, onSuccess: Function) { + dsmSearch(value: string, page: number, pageSize: number, onSuccess: Function) { this.client.get>('./ajax/dsm/search/' + page + '/' + pageSize + '?value=' + encodeURIComponent(value)).subscribe({ next: data => onSuccess(data), error: error => this.showError(error) }); } + loadMDStores(onSuccess: Function): void { + this.client.get("/ajax/mdstores/").subscribe({ + next: data => onSuccess(data), + error: error => this.showError(error) + }); + } + + addMDStore(format: string, layout: string, interpretation: string, dsName: string, dsId: string, apiId: string, onSuccess: Function, relatedForm?: FormGroup) { + const headers = new HttpHeaders().set('Content-Type', 'application/x-www-form-urlencoded') + + let body = new HttpParams() + .set('dsName', dsName) + .set('dsId', dsId) + .set('apiId', apiId); + + this.client.post('/ajax/mdstores/new/' + + encodeURIComponent(format) + + '/' + + encodeURIComponent(layout) + + '/' + + encodeURIComponent(interpretation), + body, { headers: headers }).subscribe({ + next: data => onSuccess(data), + error: error => this.showError(error, relatedForm) + }); + } + + deleteMDStore(mdId: string, onSuccess: Function): void { + this.client.delete('./ajax/mdstores/mdstore/' + encodeURIComponent(mdId)).subscribe({ + next: data => onSuccess(data), + error: error => this.showError(error) + }); + } + + prepareNewMDStoreVersion(mdId: string, onSuccess: Function): void { + this.client.get('./ajax/mdstores/mdstore/' + encodeURIComponent(mdId) + '/newVersion').subscribe({ + next: data => onSuccess(data), + error: error => this.showError(error) + }); + } + private showError(error: any, form?: FormGroup) { const msg = this.errorMessage(error); if (form) { diff --git a/frontends/dnet-is-application/src/app/mdstores/add-mdstore-dialog.html b/frontends/dnet-is-application/src/app/mdstores/add-mdstore-dialog.html new file mode 100644 index 00000000..d1e8af63 --- /dev/null +++ b/frontends/dnet-is-application/src/app/mdstores/add-mdstore-dialog.html @@ -0,0 +1,50 @@ +
+

New MDStore

+ +
+ + + Format + + This field is required + + + + Layout + + This field is required + + + + Interpretation + + This field is required + + + + Datasource Name + + + + + Datasource ID + + + + + API ID + + + +
+ +
+ + + + {{ newMdstoreForm.errors?.['serverError'] }} + +
+ +
+ \ No newline at end of file diff --git a/frontends/dnet-is-application/src/app/mdstores/mdstores-versions-dialog.html b/frontends/dnet-is-application/src/app/mdstores/mdstores-versions-dialog.html new file mode 100644 index 00000000..a6e70c9f --- /dev/null +++ b/frontends/dnet-is-application/src/app/mdstores/mdstores-versions-dialog.html @@ -0,0 +1 @@ +VERSIONS \ No newline at end of file diff --git a/frontends/dnet-is-application/src/app/mdstores/mdstores.component.css b/frontends/dnet-is-application/src/app/mdstores/mdstores.component.css index e69de29b..b03a1413 100644 --- a/frontends/dnet-is-application/src/app/mdstores/mdstores.component.css +++ b/frontends/dnet-is-application/src/app/mdstores/mdstores.component.css @@ -0,0 +1,25 @@ +.mdstore-info-table { + margin-top: 1em; + margin-bottom: 1em; + border-collapse: collapse; +} + +.mdstore-info-table tr:not(:last-child) { + border-bottom: 1pt solid lightgrey; +} + +.mdstore-info-table th { + width: 30%; +} + +.mdstore-info-table th, .mdstore-info-table td{ + text-align: left; + font-size: 0.9em; + vertical-align: top; +} + +.mdstore-info-table td button { + font-size: 0.8em !important; + padding: 0 !important; + height: 2.5em !important; +} diff --git a/frontends/dnet-is-application/src/app/mdstores/mdstores.component.html b/frontends/dnet-is-application/src/app/mdstores/mdstores.component.html index 60c736b3..4445342e 100644 --- a/frontends/dnet-is-application/src/app/mdstores/mdstores.component.html +++ b/frontends/dnet-is-application/src/app/mdstores/mdstores.component.html @@ -1 +1,60 @@ -

Metadata Stores

\ No newline at end of file +

Metadata Stores

+ + + + + Filter (Total: {{(mdstores | searchFilter: searchText).length}}) + + + + + + {{md.id}} + {{md.datasourceName}} + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Format / Layout / Interpretation{{md.format}} / {{md.layout}} / {{md.interpretation}}
Datasource + Name: {{md.datasourceName}}
+ ID: {{md.datasourceId}}
+ API: {{md.apiId}} +
Creation Date{{md.creationDate}}
Last Update{{md.lastUpdate}}
Size{{md.size}}
HDFS Path{{md.hdfsPath}}
Versions + {{md.numberOfVersions}} version(s) + / + prepare new version +
+ + + inspect + + + + \ No newline at end of file diff --git a/frontends/dnet-is-application/src/app/mdstores/mdstores.component.ts b/frontends/dnet-is-application/src/app/mdstores/mdstores.component.ts index 6daf9b3b..55533938 100644 --- a/frontends/dnet-is-application/src/app/mdstores/mdstores.component.ts +++ b/frontends/dnet-is-application/src/app/mdstores/mdstores.component.ts @@ -1,12 +1,59 @@ -import { Component } from '@angular/core'; +import { Component, Inject, OnInit } from '@angular/core'; +import { ISService } from '../common/is.service'; +import { ActivatedRoute } from '@angular/router'; +import { MatDialog, MAT_DIALOG_DATA, MatDialogRef } from '@angular/material/dialog'; +import { MDStore, MDStoreVersion } from '../common/is.model'; +import { FormControl, FormGroup, Validators } from '@angular/forms'; @Component({ selector: 'app-mdstores', templateUrl: './mdstores.component.html', styleUrls: ['./mdstores.component.css'] }) -export class MdstoresComponent { +export class MdstoresComponent implements OnInit { + mdstores:MDStore[] = []; + searchText:string = ''; + + constructor(public service: ISService, public route: ActivatedRoute, public dialog: MatDialog) {} + + ngOnInit() { this.reload() } + reload() { this.service.loadMDStores((data: MDStore[]) => this.mdstores = data); } + + openVersionsDialog(md:MDStore): void { + const dialogRef = this.dialog.open(MDStoreVersionsDialog, { + data: md, + width: '80%' + }); + + dialogRef.afterClosed().subscribe(result => { + if (result) this.reload(); + }); + } + + openAddMdstoreDialog(): void { + const dialogRef = this.dialog.open(AddMDStoreDialog, { + data: {}, + width: '80%' + }); + + dialogRef.afterClosed().subscribe(result => { + if (result) this.reload(); + }); + } + + createNewVersion(md:MDStore): void { + this.service.prepareNewMDStoreVersion(md.id, (data:MDStoreVersion) => { + md.numberOfVersions = md.numberOfVersions + 1; + this.openVersionsDialog(md); + }); + } + + deleteMdstore(md:MDStore) { + if (confirm('Are you sure?')) { + this.service.deleteMDStore(md.id, (data: void) => this.reload()); + } + } } @Component({ @@ -17,3 +64,55 @@ export class MdstoresComponent { export class MdstoreInspectorComponent { } + +@Component({ + selector: 'mdstores-versions-dialog', + templateUrl: './mdstores-versions-dialog.html', + styleUrls: ['./mdstores.component.css'] +}) +export class MDStoreVersionsDialog { + + constructor(public dialogRef: MatDialogRef, @Inject(MAT_DIALOG_DATA) public data: any, public service: ISService) { + + } + + onNoClick(): void { + this.dialogRef.close(); + } +} + +@Component({ + selector: 'add-mdstore-dialog', + templateUrl: './add-mdstore-dialog.html', + styleUrls: ['./mdstores.component.css'] +}) +export class AddMDStoreDialog { + + newMdstoreForm = new FormGroup({ + format: new FormControl('', [Validators.required]), + layout : new FormControl('', [Validators.required]), + interpretation : new FormControl('', [Validators.required]), + dsName : new FormControl(''), + dsId : new FormControl(''), + apiId : new FormControl(''), + }); + + constructor(public dialogRef: MatDialogRef, @Inject(MAT_DIALOG_DATA) public data: any, public service: ISService) { + + } + + onSubmit():void { + let format:string = this.newMdstoreForm.get('format')?.value!; + let layout:string = this.newMdstoreForm.get('layout')?.value!; + let interpretation:string = this.newMdstoreForm.get('interpretation')?.value!; + let dsName:string = this.newMdstoreForm.get('dsName')?.value!; + let dsId:string = this.newMdstoreForm.get('dsId')?.value!; + let apiId:string = this.newMdstoreForm.get('apiId')?.value!; + + this.service.addMDStore(format, layout, interpretation, dsName, dsId, apiId, (data: void) => this.dialogRef.close(1), this.newMdstoreForm); + } + + onNoClick(): void { + this.dialogRef.close(); + } +} \ No newline at end of file diff --git a/frontends/dnet-is-application/src/styles.css b/frontends/dnet-is-application/src/styles.css index 5076dbb1..f0100ad8 100644 --- a/frontends/dnet-is-application/src/styles.css +++ b/frontends/dnet-is-application/src/styles.css @@ -6,8 +6,25 @@ body { margin: 0; font-family: Roboto, "Helvetica Neue", sans-serif; } .small { font-size: 0.9em !important; } .muted { color: darkgray; } -table { - width: 100%; +table { + width: 100%; + table-layout: fixed !important; +} + +tr { + height: auto !important; +} + +th.mat-sort-header-sorted { color: black !important; } + +th, td { + white-space: nowrap !important; + overflow: hidden !important; + text-overflow: ellipsis !important; + font-size: 0.9em !important; + padding-top: 0.5em !important; + padding-bottom: 0.5em !important; + text-align: left; } .table-buttons { text-align: right !important; } @@ -39,25 +56,6 @@ a:hover, a:not([href]):hover { text-decoration: underline; } -table { - table-layout: fixed !important; -} - -tr { - height: auto !important; -} - -th.mat-sort-header-sorted { color: black !important; } - -th, td { - white-space: nowrap !important; - overflow: hidden !important; - text-overflow: ellipsis !important; - font-size: 0.9em !important; - padding-top: 0.5em !important; - padding-bottom: 0.5em !important; -} - .badge-label { padding-top: 0.3em; padding-bottom: 0.3em;