diff --git a/apps/dhp-mdstore-manager/pom.xml b/apps/dhp-mdstore-manager/pom.xml index 396b63d6..c977a59c 100644 --- a/apps/dhp-mdstore-manager/pom.xml +++ b/apps/dhp-mdstore-manager/pom.xml @@ -24,6 +24,10 @@ org.springframework.boot spring-boot-starter-json + + org.springframework.boot + spring-boot-starter-thymeleaf + org.postgresql postgresql diff --git a/apps/dhp-mdstore-manager/src/main/java/eu/dnetlib/data/mdstore/manager/controller/MDInspectorController.java b/apps/dhp-mdstore-manager/src/main/java/eu/dnetlib/data/mdstore/manager/controller/MDInspectorController.java new file mode 100644 index 00000000..51f03b0b --- /dev/null +++ b/apps/dhp-mdstore-manager/src/main/java/eu/dnetlib/data/mdstore/manager/controller/MDInspectorController.java @@ -0,0 +1,87 @@ +package eu.dnetlib.data.mdstore.manager.controller; + +import org.apache.commons.lang3.exception.ExceptionUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.http.HttpStatus; +import org.springframework.stereotype.Controller; +import org.springframework.ui.ModelMap; +import org.springframework.web.bind.annotation.ExceptionHandler; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.ResponseStatus; +import org.springframework.web.servlet.ModelAndView; + +import eu.dnetlib.data.mdstore.manager.common.model.MDStoreVersion; +import eu.dnetlib.data.mdstore.manager.common.model.MDStoreWithInfo; +import eu.dnetlib.data.mdstore.manager.exceptions.MDStoreManagerException; +import eu.dnetlib.data.mdstore.manager.utils.DatabaseUtils; + +@Controller +public class MDInspectorController { + + @Autowired + private DatabaseUtils databaseUtils; + + private static final Logger log = LoggerFactory.getLogger(MDInspectorController.class); + + @RequestMapping("/mdrecords/{id}/{limit}") + public String mdstoreInspector(final ModelMap map, @PathVariable final String id, @PathVariable final long limit) throws MDStoreManagerException { + + final MDStoreWithInfo md; + final MDStoreVersion ver; + + if (isMdstoreId(id)) { + log.debug("MDSTORE: " + id); + md = databaseUtils.findMdStore(id); + ver = databaseUtils.findVersion(md.getCurrentVersion()); + } else { + log.debug("VERSION: " + id); + ver = databaseUtils.findVersion(id); + md = databaseUtils.findMdStore(ver.getMdstore()); + } + + map.addAttribute("mdId", md.getId()); + map.addAttribute("versionId", ver.getId()); + + map.addAttribute("dsId", md.getDatasourceId()); + map.addAttribute("dsName", md.getDatasourceName()); + map.addAttribute("apiId", md.getApiId()); + + map.addAttribute("format", md.getFormat()); + map.addAttribute("layout", md.getLayout()); + map.addAttribute("interpretation", md.getInterpretation()); + + map.addAttribute("path", ver.getHdfsPath()); + map.addAttribute("lastUpdate", ver.getLastUpdate()); + map.addAttribute("size", ver.getSize()); + + map.addAttribute("limit", limit); + + if (md.getCurrentVersion().equals(ver.getId())) { + map.addAttribute("status", "current"); + } else if (ver.isWriting()) { + map.addAttribute("status", "writing"); + } else { + map.addAttribute("status", "expired"); + } + + return "inspector"; + } + + @ExceptionHandler(Exception.class) + @ResponseStatus(value = HttpStatus.INTERNAL_SERVER_ERROR) + public ModelAndView handleException(final Exception e) { + log.debug(e.getMessage(), e); + final ModelAndView mv = new ModelAndView(); + mv.setViewName("error"); + mv.addObject("error", e.getMessage()); + mv.addObject("stacktrace", ExceptionUtils.getStackTrace(e)); + return mv; + } + + private boolean isMdstoreId(final String id) { + return id.length() < 40; + } +} diff --git a/apps/dhp-mdstore-manager/src/main/java/eu/dnetlib/data/mdstore/manager/controller/MDStoreController.java b/apps/dhp-mdstore-manager/src/main/java/eu/dnetlib/data/mdstore/manager/controller/MDStoreController.java index 290783e1..6b9b6f90 100644 --- a/apps/dhp-mdstore-manager/src/main/java/eu/dnetlib/data/mdstore/manager/controller/MDStoreController.java +++ b/apps/dhp-mdstore-manager/src/main/java/eu/dnetlib/data/mdstore/manager/controller/MDStoreController.java @@ -210,14 +210,14 @@ public class MDStoreController extends AbstractDnetController { @ApiOperation("read the parquet file of a mdstore version") @GetMapping("/version/{versionId}/parquet/content/{limit}") - public List> listVersionParquet(@PathVariable final String versionId, @PathVariable final long limit) throws MDStoreManagerException { + public List> listVersionParquet(@PathVariable final String versionId, @PathVariable final long limit) throws MDStoreManagerException { final String path = databaseUtils.findVersion(versionId).getHdfsPath(); return hdfsClient.readParquetFiles(path + "/store", limit); } @ApiOperation("read the parquet file of a mdstore (current version)") @GetMapping("/mdstore/{mdId}/parquet/content/{limit}") - public List> listMdstoreParquet(@PathVariable final String mdId, @PathVariable final long limit) throws MDStoreManagerException { + public List> listMdstoreParquet(@PathVariable final String mdId, @PathVariable final long limit) throws MDStoreManagerException { final String versionId = databaseUtils.findMdStore(mdId).getCurrentVersion(); final String path = databaseUtils.findVersion(versionId).getHdfsPath(); return hdfsClient.readParquetFiles(path + "/store", limit); diff --git a/apps/dhp-mdstore-manager/src/main/java/eu/dnetlib/data/mdstore/manager/utils/HdfsClient.java b/apps/dhp-mdstore-manager/src/main/java/eu/dnetlib/data/mdstore/manager/utils/HdfsClient.java index 683e63a8..a761697a 100644 --- a/apps/dhp-mdstore-manager/src/main/java/eu/dnetlib/data/mdstore/manager/utils/HdfsClient.java +++ b/apps/dhp-mdstore-manager/src/main/java/eu/dnetlib/data/mdstore/manager/utils/HdfsClient.java @@ -70,6 +70,11 @@ public class HdfsClient { } public Set listContent(final String path, final Predicate condition) { + + // TODO: remove the following line (it is only for tests) + // if (1 != 2) { return + // Sets.newHashSet("file:///Users/michele/Desktop/part-00000-e3675dc3-69fb-422e-a159-78e34cfe14d2-c000.snappy.parquet"); } + final Set res = new LinkedHashSet<>(); try (final FileSystem fs = FileSystem.get(conf())) { for (final FileStatus f : fs.listStatus(new Path(path))) { @@ -84,9 +89,9 @@ public class HdfsClient { } @SuppressWarnings("unchecked") - public List> readParquetFiles(final String path, final long limit) throws MDStoreManagerException { + public List> readParquetFiles(final String path, final long limit) throws MDStoreManagerException { - final List> list = new ArrayList<>(); + final List> list = new ArrayList<>(); final Configuration conf = conf(); @@ -107,9 +112,10 @@ public class HdfsClient { rec.getSchema().getFields().forEach(field -> fields.add(field.name())); log.debug("Found schema: " + fields); } - final Map map = new LinkedHashMap<>(); + final Map map = new LinkedHashMap<>(); for (final String field : fields) { - map.put(field, rec.get(field)); + final Object v = rec.get(field); + map.put(field, v != null ? v.toString() : ""); } list.add(map); log.debug("added record"); @@ -171,6 +177,8 @@ public class HdfsClient { } else if (hadoopCluster.equalsIgnoreCase("GARR")) { conf.addResource(getClass().getResourceAsStream("/hadoop/GARR/core-site.xml")); conf.addResource(getClass().getResourceAsStream("/hadoop/GARR/garr-hadoop-conf.xml")); + } else if (hadoopCluster.equalsIgnoreCase("MOCK")) { + // NOTHING } else { log.error("Invalid Haddop Cluster: " + hadoopCluster); throw new MDStoreManagerException("Invalid Haddop Cluster: " + hadoopCluster); diff --git a/apps/dhp-mdstore-manager/src/main/resources/application.properties b/apps/dhp-mdstore-manager/src/main/resources/application.properties index f8c34bb8..d0ccecbe 100644 --- a/apps/dhp-mdstore-manager/src/main/resources/application.properties +++ b/apps/dhp-mdstore-manager/src/main/resources/application.properties @@ -1,6 +1,8 @@ spring.main.banner-mode = console logging.level.root = INFO +spring.thymeleaf.cache=false + management.endpoints.web.exposure.include = prometheus,health management.endpoints.web.base-path = / management.endpoints.web.path-mapping.prometheus = metrics @@ -22,6 +24,6 @@ spring.jpa.open-in-view=true logging.level.io.swagger.models.parameters.AbstractSerializableParameter = error # Hadoop -dhp.mdstore-manager.hadoop.cluster = GARR +dhp.mdstore-manager.hadoop.cluster = MOCK dhp.mdstore-manager.hdfs.base-path = /data/dnet.dev/mdstore dhp.mdstore-manager.hadoop.user = dnet.dev diff --git a/apps/dhp-mdstore-manager/src/main/resources/static/index.html b/apps/dhp-mdstore-manager/src/main/resources/static/index.html index 22a4a10a..d8a44b92 100644 --- a/apps/dhp-mdstore-manager/src/main/resources/static/index.html +++ b/apps/dhp-mdstore-manager/src/main/resources/static/index.html @@ -73,6 +73,7 @@ @@ -154,6 +155,7 @@ {{v.id}}
Path: {{v.hdfsPath}}
+ inspect diff --git a/apps/dhp-mdstore-manager/src/main/resources/static/js/mdinspector.js b/apps/dhp-mdstore-manager/src/main/resources/static/js/mdinspector.js new file mode 100644 index 00000000..3a5c4662 --- /dev/null +++ b/apps/dhp-mdstore-manager/src/main/resources/static/js/mdinspector.js @@ -0,0 +1,48 @@ +// Spinner show/hide methods ~ Andrea Mannocci +var spinnerOpts = { + lines: 15, + length: 16, + width: 5, + radius: 25, + color: '#eeeeee', + className: 'spinner', + top: '40%' +}; + +var spinnerTarget = document.getElementById('spinnerdiv'); + +var spinner; + +function showSpinner() { + spinner = new Spinner(spinnerOpts).spin(spinnerTarget); + spinnerTarget.style.visibility = 'visible'; +} + +function hideSpinner() { + spinnerTarget.style.visibility = 'hidden'; + spinner.stop(); +} + + +var app = angular.module('mdInspectorApp', []); + +app.controller('mdInspectorController', function($scope, $http) { + $scope.records = []; + $scope.versionId = versionId(); + $scope.limit = limit(); + + $scope.reload = function() { + showSpinner(); + $http.get('../../mdstores/version/' + $scope.versionId + '/parquet/content/' + $scope.limit + '/?' + $.now()).success(function(data) { + hideSpinner(); + $scope.records = data; + }).error(function(err) { + hideSpinner(); + alert('ERROR: ' + err.message); + }); + }; + + $scope.reload(); + +}); + diff --git a/apps/dhp-mdstore-manager/src/main/resources/static/js/mdstoremanager.js b/apps/dhp-mdstore-manager/src/main/resources/static/js/mdstoremanager.js deleted file mode 100644 index 797a8ddc..00000000 --- a/apps/dhp-mdstore-manager/src/main/resources/static/js/mdstoremanager.js +++ /dev/null @@ -1,117 +0,0 @@ -var app = angular.module('mdstoreManagerApp', []); - -app.controller('mdstoreManagerController', function($scope, $http) { - $scope.mdstores = []; - $scope.versions = []; - $scope.openMdstore = ''; - $scope.openCurrentVersion = '' - - $scope.forceVersionDelete = false; - - $scope.reload = function() { - $http.get('./mdstores/?' + $.now()).success(function(data) { - $scope.mdstores = data; - }).error(function(err) { - alert('ERROR: ' + err.message); - }); - }; - - $scope.newMdstore = function(format, layout, interpretation, dsName, dsId, apiId) { - var url = './mdstores/new/' + encodeURIComponent(format) + '/' + encodeURIComponent(layout) + '/' + encodeURIComponent(interpretation); - if (dsName || dsId || apiId) { - url += '?dsName=' + encodeURIComponent(dsName) + '&dsId=' + encodeURIComponent(dsId) + '&apiId=' + encodeURIComponent(apiId); - } - $http.get(url).success(function(data) { - $scope.reload(); - }).error(function(err) { - alert('ERROR: ' + err.message); - }); - }; - - $scope.deleteMdstore = function(mdId) { - if (confirm("Are you sure ?")) { - $http.delete('./mdstores/mdstore/' + mdId).success(function(data) { - $scope.reload(); - }).error(function(err) { - alert('ERROR: ' + err.message); - }); - } - }; - - $scope.prepareVersion = function(mdId, currentVersion) { - $scope.versions = []; - $http.get('./mdstores/mdstore/' + mdId + '/newVersion?' + $.now()).success(function(data) { - $scope.reload(); - $scope.listVersions(mdId, currentVersion); - }).error(function(err) { - alert('ERROR: ' + err.message); - }); - }; - - $scope.commitVersion = function(versionId) { - var size = parseInt(prompt("New Size", "0")); - if (size >= 0) { - $http.get("./mdstores/version/" + versionId + "/commit/" + size + '?' + $.now()).success(function(data) { - $scope.reload(); - $scope.openCurrentVersion = versionId; - $scope.refreshVersions(); - }).error(function(err) { - alert('ERROR: ' + err.message); - }); - } - }; - - $scope.abortVersion = function(versionId) { - $http.get("./mdstores/version/" + versionId + "/abort?" + $.now()).success(function(data) { - $scope.reload(); - $scope.refreshVersions(); - }).error(function(err) { - alert('ERROR: ' + err.message); - }); - }; - - $scope.resetReading = function(versionId) { - $http.get("./mdstores/version/" + versionId + "/resetReading" + '?' + $.now()).success(function(data) { - $scope.reload(); - $scope.refreshVersions(); - }).error(function(err) { - alert('ERROR: ' + err.message); - }); - }; - - $scope.listVersions = function(mdId, current) { - $scope.openMdstore = mdId; - $scope.openCurrentVersion = current; - $scope.versions = []; - $scope.refreshVersions(); - }; - - $scope.refreshVersions = function() { - $http.get('./mdstores/mdstore/' + $scope.openMdstore + '/versions?' + $.now()).success(function(data) { - angular.forEach(data, function(value, key) { - value.current = (value.id == $scope.openCurrentVersion); - }); - $scope.versions = data; - }).error(function(err) { - alert('ERROR: ' + err.message); - }); - }; - - $scope.deleteVersion = function(versionId, force) { - if (confirm("Are you sure ?")) { - var url = './mdstores/version/' + versionId; - if (force) { url += '?force=true'; } - - $http.delete(url).success(function(data) { - $scope.reload(); - $scope.refreshVersions(); - }).error(function(err) { - alert('ERROR: ' + err.message); - }); - } - }; - - $scope.reload(); - -}); - diff --git a/apps/dhp-mdstore-manager/src/main/resources/static/js/spin.js b/apps/dhp-mdstore-manager/src/main/resources/static/js/spin.js new file mode 100644 index 00000000..0127c85d --- /dev/null +++ b/apps/dhp-mdstore-manager/src/main/resources/static/js/spin.js @@ -0,0 +1,348 @@ +/** + * Copyright (c) 2011-2014 Felix Gnass + * Licensed under the MIT license + */ +(function(root, factory) { + + /* CommonJS */ + if (typeof exports == 'object') module.exports = factory() + + /* AMD module */ + else if (typeof define == 'function' && define.amd) define(factory) + + /* Browser global */ + else root.Spinner = factory() +} +(this, function() { + "use strict"; + + var prefixes = ['webkit', 'Moz', 'ms', 'O'] /* Vendor prefixes */ + , animations = {} /* Animation rules keyed by their name */ + , useCssAnimations /* Whether to use CSS animations or setTimeout */ + + /** + * Utility function to create elements. If no tag name is given, + * a DIV is created. Optionally properties can be passed. + */ + function createEl(tag, prop) { + var el = document.createElement(tag || 'div') + , n + + for(n in prop) el[n] = prop[n] + return el + } + + /** + * Appends children and returns the parent. + */ + function ins(parent /* child1, child2, ...*/) { + for (var i=1, n=arguments.length; i>1) + 'px' + }) + } + + for (; i < o.lines; i++) { + seg = css(createEl(), { + position: 'absolute', + top: 1+~(o.width/2) + 'px', + transform: o.hwaccel ? 'translate3d(0,0,0)' : '', + opacity: o.opacity, + animation: useCssAnimations && addAnimation(o.opacity, o.trail, start + i * o.direction, o.lines) + ' ' + 1/o.speed + 's linear infinite' + }) + + if (o.shadow) ins(seg, css(fill('#000', '0 0 4px ' + '#000'), {top: 2+'px'})) + ins(el, ins(seg, fill(getColor(o.color, i), '0 0 1px rgba(0,0,0,.1)'))) + } + return el + }, + + /** + * Internal method that adjusts the opacity of a single line. + * Will be overwritten in VML fallback mode below. + */ + opacity: function(el, i, val) { + if (i < el.childNodes.length) el.childNodes[i].style.opacity = val + } + + }) + + + function initVML() { + + /* Utility function to create a VML tag */ + function vml(tag, attr) { + return createEl('<' + tag + ' xmlns="urn:schemas-microsoft.com:vml" class="spin-vml">', attr) + } + + // No CSS transforms but VML support, add a CSS rule for VML elements: + sheet.addRule('.spin-vml', 'behavior:url(#default#VML)') + + Spinner.prototype.lines = function(el, o) { + var r = o.length+o.width + , s = 2*r + + function grp() { + return css( + vml('group', { + coordsize: s + ' ' + s, + coordorigin: -r + ' ' + -r + }), + { width: s, height: s } + ) + } + + var margin = -(o.width+o.length)*2 + 'px' + , g = css(grp(), {position: 'absolute', top: margin, left: margin}) + , i + + function seg(i, dx, filter) { + ins(g, + ins(css(grp(), {rotation: 360 / o.lines * i + 'deg', left: ~~dx}), + ins(css(vml('roundrect', {arcsize: o.corners}), { + width: r, + height: o.width, + left: o.radius, + top: -o.width>>1, + filter: filter + }), + vml('fill', {color: getColor(o.color, i), opacity: o.opacity}), + vml('stroke', {opacity: 0}) // transparent stroke to fix color bleeding upon opacity change + ) + ) + ) + } + + if (o.shadow) + for (i = 1; i <= o.lines; i++) + seg(i, -2, 'progid:DXImageTransform.Microsoft.Blur(pixelradius=2,makeshadow=1,shadowopacity=.3)') + + for (i = 1; i <= o.lines; i++) seg(i) + return ins(el, g) + } + + Spinner.prototype.opacity = function(el, i, val, o) { + var c = el.firstChild + o = o.shadow && o.lines || 0 + if (c && i+o < c.childNodes.length) { + c = c.childNodes[i+o]; c = c && c.firstChild; c = c && c.firstChild + if (c) c.opacity = val + } + } + } + + var probe = css(createEl('group'), {behavior: 'url(#default#VML)'}) + + if (!vendor(probe, 'transform') && probe.adj) initVML() + else useCssAnimations = vendor(probe, 'animation') + + return Spinner + +})); diff --git a/apps/dhp-mdstore-manager/src/main/resources/templates/error.html b/apps/dhp-mdstore-manager/src/main/resources/templates/error.html new file mode 100644 index 00000000..aa99d367 --- /dev/null +++ b/apps/dhp-mdstore-manager/src/main/resources/templates/error.html @@ -0,0 +1,27 @@ + + + + + Metadata Inspector - ERROR + + + + + + + + + +
+

Metadata Inspector - ERROR

+
+

+
+
+		

+ + diff --git a/apps/dhp-mdstore-manager/src/main/resources/templates/inspector.html b/apps/dhp-mdstore-manager/src/main/resources/templates/inspector.html new file mode 100644 index 00000000..abfdb1bb --- /dev/null +++ b/apps/dhp-mdstore-manager/src/main/resources/templates/inspector.html @@ -0,0 +1,199 @@ + + + + + Metadata Inspector + + + + + + + + + + + + + + +
+ +
+ +
+
+
+

Metadata Inspector

+ +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
MdStoreID
Format / Layout / Interpretation
Related Datasource
VersionID + current + writing + expired + +
Hdfs Path
Last Update
Size
+ +
+

+ The display is limited to the first {{limit}} records +

+ +
+ +
+
{{rec.id}}
+ + + + + + + + + + + + + + + + + +
Original Id{{rec.originalId}}
Collected on{{rec.dateOfCollection | date:'medium'}}
Transformed on{{rec.dateOfTransformation | date:'medium'}}
Provenance{{rec.provenance}}
+
+ {{rec.encoding}} +
+
{{rec.body}}
+
+
+
+
+
+ + + + + + diff --git a/apps/dhp-mdstore-manager/src/test/java/eu/dnetlib/data/mdstore/manager/utils/HdfsClientTest.java b/apps/dhp-mdstore-manager/src/test/java/eu/dnetlib/data/mdstore/manager/utils/HdfsClientTest.java index 756d7831..7848d17e 100644 --- a/apps/dhp-mdstore-manager/src/test/java/eu/dnetlib/data/mdstore/manager/utils/HdfsClientTest.java +++ b/apps/dhp-mdstore-manager/src/test/java/eu/dnetlib/data/mdstore/manager/utils/HdfsClientTest.java @@ -36,7 +36,11 @@ class HdfsClientTest { GenericRecord rec = null; final Set fields = new LinkedHashSet<>(); + + int i = 0; + while ((rec = reader.read()) != null) { + if (fields.isEmpty()) { rec.getSchema().getFields().forEach(f -> fields.add(f.name())); } @@ -46,8 +50,12 @@ class HdfsClientTest { map.put(f, rec.get(f)); } - System.out.println(map); + // System.out.println(map); + + i++; } + + System.out.println("Total: " + i); } }