md records inspector
This commit is contained in:
parent
e5f7b88ea5
commit
1004273907
|
|
@ -24,6 +24,10 @@
|
|||
<groupId>org.springframework.boot</groupId>
|
||||
<artifactId>spring-boot-starter-json</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.springframework.boot</groupId>
|
||||
<artifactId>spring-boot-starter-thymeleaf</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.postgresql</groupId>
|
||||
<artifactId>postgresql</artifactId>
|
||||
|
|
|
|||
|
|
@ -0,0 +1,87 @@
|
|||
package eu.dnetlib.data.mdstore.manager.controller;
|
||||
|
||||
import org.apache.commons.lang3.exception.ExceptionUtils;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.http.HttpStatus;
|
||||
import org.springframework.stereotype.Controller;
|
||||
import org.springframework.ui.ModelMap;
|
||||
import org.springframework.web.bind.annotation.ExceptionHandler;
|
||||
import org.springframework.web.bind.annotation.PathVariable;
|
||||
import org.springframework.web.bind.annotation.RequestMapping;
|
||||
import org.springframework.web.bind.annotation.ResponseStatus;
|
||||
import org.springframework.web.servlet.ModelAndView;
|
||||
|
||||
import eu.dnetlib.data.mdstore.manager.common.model.MDStoreVersion;
|
||||
import eu.dnetlib.data.mdstore.manager.common.model.MDStoreWithInfo;
|
||||
import eu.dnetlib.data.mdstore.manager.exceptions.MDStoreManagerException;
|
||||
import eu.dnetlib.data.mdstore.manager.utils.DatabaseUtils;
|
||||
|
||||
@Controller
|
||||
public class MDInspectorController {
|
||||
|
||||
@Autowired
|
||||
private DatabaseUtils databaseUtils;
|
||||
|
||||
private static final Logger log = LoggerFactory.getLogger(MDInspectorController.class);
|
||||
|
||||
@RequestMapping("/mdrecords/{id}/{limit}")
|
||||
public String mdstoreInspector(final ModelMap map, @PathVariable final String id, @PathVariable final long limit) throws MDStoreManagerException {
|
||||
|
||||
final MDStoreWithInfo md;
|
||||
final MDStoreVersion ver;
|
||||
|
||||
if (isMdstoreId(id)) {
|
||||
log.debug("MDSTORE: " + id);
|
||||
md = databaseUtils.findMdStore(id);
|
||||
ver = databaseUtils.findVersion(md.getCurrentVersion());
|
||||
} else {
|
||||
log.debug("VERSION: " + id);
|
||||
ver = databaseUtils.findVersion(id);
|
||||
md = databaseUtils.findMdStore(ver.getMdstore());
|
||||
}
|
||||
|
||||
map.addAttribute("mdId", md.getId());
|
||||
map.addAttribute("versionId", ver.getId());
|
||||
|
||||
map.addAttribute("dsId", md.getDatasourceId());
|
||||
map.addAttribute("dsName", md.getDatasourceName());
|
||||
map.addAttribute("apiId", md.getApiId());
|
||||
|
||||
map.addAttribute("format", md.getFormat());
|
||||
map.addAttribute("layout", md.getLayout());
|
||||
map.addAttribute("interpretation", md.getInterpretation());
|
||||
|
||||
map.addAttribute("path", ver.getHdfsPath());
|
||||
map.addAttribute("lastUpdate", ver.getLastUpdate());
|
||||
map.addAttribute("size", ver.getSize());
|
||||
|
||||
map.addAttribute("limit", limit);
|
||||
|
||||
if (md.getCurrentVersion().equals(ver.getId())) {
|
||||
map.addAttribute("status", "current");
|
||||
} else if (ver.isWriting()) {
|
||||
map.addAttribute("status", "writing");
|
||||
} else {
|
||||
map.addAttribute("status", "expired");
|
||||
}
|
||||
|
||||
return "inspector";
|
||||
}
|
||||
|
||||
@ExceptionHandler(Exception.class)
|
||||
@ResponseStatus(value = HttpStatus.INTERNAL_SERVER_ERROR)
|
||||
public ModelAndView handleException(final Exception e) {
|
||||
log.debug(e.getMessage(), e);
|
||||
final ModelAndView mv = new ModelAndView();
|
||||
mv.setViewName("error");
|
||||
mv.addObject("error", e.getMessage());
|
||||
mv.addObject("stacktrace", ExceptionUtils.getStackTrace(e));
|
||||
return mv;
|
||||
}
|
||||
|
||||
private boolean isMdstoreId(final String id) {
|
||||
return id.length() < 40;
|
||||
}
|
||||
}
|
||||
|
|
@ -210,14 +210,14 @@ public class MDStoreController extends AbstractDnetController {
|
|||
|
||||
@ApiOperation("read the parquet file of a mdstore version")
|
||||
@GetMapping("/version/{versionId}/parquet/content/{limit}")
|
||||
public List<Map<String, Object>> listVersionParquet(@PathVariable final String versionId, @PathVariable final long limit) throws MDStoreManagerException {
|
||||
public List<Map<String, String>> listVersionParquet(@PathVariable final String versionId, @PathVariable final long limit) throws MDStoreManagerException {
|
||||
final String path = databaseUtils.findVersion(versionId).getHdfsPath();
|
||||
return hdfsClient.readParquetFiles(path + "/store", limit);
|
||||
}
|
||||
|
||||
@ApiOperation("read the parquet file of a mdstore (current version)")
|
||||
@GetMapping("/mdstore/{mdId}/parquet/content/{limit}")
|
||||
public List<Map<String, Object>> listMdstoreParquet(@PathVariable final String mdId, @PathVariable final long limit) throws MDStoreManagerException {
|
||||
public List<Map<String, String>> listMdstoreParquet(@PathVariable final String mdId, @PathVariable final long limit) throws MDStoreManagerException {
|
||||
final String versionId = databaseUtils.findMdStore(mdId).getCurrentVersion();
|
||||
final String path = databaseUtils.findVersion(versionId).getHdfsPath();
|
||||
return hdfsClient.readParquetFiles(path + "/store", limit);
|
||||
|
|
|
|||
|
|
@ -70,6 +70,11 @@ public class HdfsClient {
|
|||
}
|
||||
|
||||
public Set<String> listContent(final String path, final Predicate<FileStatus> condition) {
|
||||
|
||||
// TODO: remove the following line (it is only for tests)
|
||||
// if (1 != 2) { return
|
||||
// Sets.newHashSet("file:///Users/michele/Desktop/part-00000-e3675dc3-69fb-422e-a159-78e34cfe14d2-c000.snappy.parquet"); }
|
||||
|
||||
final Set<String> res = new LinkedHashSet<>();
|
||||
try (final FileSystem fs = FileSystem.get(conf())) {
|
||||
for (final FileStatus f : fs.listStatus(new Path(path))) {
|
||||
|
|
@ -84,9 +89,9 @@ public class HdfsClient {
|
|||
}
|
||||
|
||||
@SuppressWarnings("unchecked")
|
||||
public List<Map<String, Object>> readParquetFiles(final String path, final long limit) throws MDStoreManagerException {
|
||||
public List<Map<String, String>> readParquetFiles(final String path, final long limit) throws MDStoreManagerException {
|
||||
|
||||
final List<Map<String, Object>> list = new ArrayList<>();
|
||||
final List<Map<String, String>> list = new ArrayList<>();
|
||||
|
||||
final Configuration conf = conf();
|
||||
|
||||
|
|
@ -107,9 +112,10 @@ public class HdfsClient {
|
|||
rec.getSchema().getFields().forEach(field -> fields.add(field.name()));
|
||||
log.debug("Found schema: " + fields);
|
||||
}
|
||||
final Map<String, Object> map = new LinkedHashMap<>();
|
||||
final Map<String, String> map = new LinkedHashMap<>();
|
||||
for (final String field : fields) {
|
||||
map.put(field, rec.get(field));
|
||||
final Object v = rec.get(field);
|
||||
map.put(field, v != null ? v.toString() : "");
|
||||
}
|
||||
list.add(map);
|
||||
log.debug("added record");
|
||||
|
|
@ -171,6 +177,8 @@ public class HdfsClient {
|
|||
} else if (hadoopCluster.equalsIgnoreCase("GARR")) {
|
||||
conf.addResource(getClass().getResourceAsStream("/hadoop/GARR/core-site.xml"));
|
||||
conf.addResource(getClass().getResourceAsStream("/hadoop/GARR/garr-hadoop-conf.xml"));
|
||||
} else if (hadoopCluster.equalsIgnoreCase("MOCK")) {
|
||||
// NOTHING
|
||||
} else {
|
||||
log.error("Invalid Haddop Cluster: " + hadoopCluster);
|
||||
throw new MDStoreManagerException("Invalid Haddop Cluster: " + hadoopCluster);
|
||||
|
|
|
|||
|
|
@ -1,6 +1,8 @@
|
|||
spring.main.banner-mode = console
|
||||
logging.level.root = INFO
|
||||
|
||||
spring.thymeleaf.cache=false
|
||||
|
||||
management.endpoints.web.exposure.include = prometheus,health
|
||||
management.endpoints.web.base-path = /
|
||||
management.endpoints.web.path-mapping.prometheus = metrics
|
||||
|
|
@ -22,6 +24,6 @@ spring.jpa.open-in-view=true
|
|||
logging.level.io.swagger.models.parameters.AbstractSerializableParameter = error
|
||||
|
||||
# Hadoop
|
||||
dhp.mdstore-manager.hadoop.cluster = GARR
|
||||
dhp.mdstore-manager.hadoop.cluster = MOCK
|
||||
dhp.mdstore-manager.hdfs.base-path = /data/dnet.dev/mdstore
|
||||
dhp.mdstore-manager.hadoop.user = dnet.dev
|
||||
|
|
|
|||
|
|
@ -73,6 +73,7 @@
|
|||
</table>
|
||||
<div class="panel-footer">
|
||||
<button class="btn btn-sm btn-danger" ng-click="deleteMdstore(md.id)">delete</button>
|
||||
<a href="./mdrecords/{{md.id}}/50" class="btn btn-sm btn-primary pull-right">inspect</a>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
|
|
@ -154,6 +155,7 @@
|
|||
<td>
|
||||
<span class="glyphicon glyphicon-pencil" ng-if="v.writing" title="writing..."></span> <span ng-class="{'text-success': v.current}">{{v.id}}</span><br />
|
||||
<span class="small"><b>Path:</b> {{v.hdfsPath}}</span><br/>
|
||||
<a class="btn btn-xs btn-info" href="./mdrecords/{{v.id}}/50">inspect</a>
|
||||
<button class="btn btn-xs btn-primary" ng-show="v.writing" ng-click="commitVersion(v.id)">commit</button>
|
||||
<button class="btn btn-xs btn-warning" ng-show="v.writing" ng-click="abortVersion(v.id)">abort</button>
|
||||
<button class="btn btn-xs btn-danger" ng-disabled="v.current" ng-click="deleteVersion(v.id, forceVersionDelete)">delete</button>
|
||||
|
|
|
|||
|
|
@ -0,0 +1,48 @@
|
|||
// Spinner show/hide methods ~ Andrea Mannocci
|
||||
var spinnerOpts = {
|
||||
lines: 15,
|
||||
length: 16,
|
||||
width: 5,
|
||||
radius: 25,
|
||||
color: '#eeeeee',
|
||||
className: 'spinner',
|
||||
top: '40%'
|
||||
};
|
||||
|
||||
var spinnerTarget = document.getElementById('spinnerdiv');
|
||||
|
||||
var spinner;
|
||||
|
||||
function showSpinner() {
|
||||
spinner = new Spinner(spinnerOpts).spin(spinnerTarget);
|
||||
spinnerTarget.style.visibility = 'visible';
|
||||
}
|
||||
|
||||
function hideSpinner() {
|
||||
spinnerTarget.style.visibility = 'hidden';
|
||||
spinner.stop();
|
||||
}
|
||||
|
||||
|
||||
var app = angular.module('mdInspectorApp', []);
|
||||
|
||||
app.controller('mdInspectorController', function($scope, $http) {
|
||||
$scope.records = [];
|
||||
$scope.versionId = versionId();
|
||||
$scope.limit = limit();
|
||||
|
||||
$scope.reload = function() {
|
||||
showSpinner();
|
||||
$http.get('../../mdstores/version/' + $scope.versionId + '/parquet/content/' + $scope.limit + '/?' + $.now()).success(function(data) {
|
||||
hideSpinner();
|
||||
$scope.records = data;
|
||||
}).error(function(err) {
|
||||
hideSpinner();
|
||||
alert('ERROR: ' + err.message);
|
||||
});
|
||||
};
|
||||
|
||||
$scope.reload();
|
||||
|
||||
});
|
||||
|
||||
|
|
@ -1,117 +0,0 @@
|
|||
var app = angular.module('mdstoreManagerApp', []);
|
||||
|
||||
app.controller('mdstoreManagerController', function($scope, $http) {
|
||||
$scope.mdstores = [];
|
||||
$scope.versions = [];
|
||||
$scope.openMdstore = '';
|
||||
$scope.openCurrentVersion = ''
|
||||
|
||||
$scope.forceVersionDelete = false;
|
||||
|
||||
$scope.reload = function() {
|
||||
$http.get('./mdstores/?' + $.now()).success(function(data) {
|
||||
$scope.mdstores = data;
|
||||
}).error(function(err) {
|
||||
alert('ERROR: ' + err.message);
|
||||
});
|
||||
};
|
||||
|
||||
$scope.newMdstore = function(format, layout, interpretation, dsName, dsId, apiId) {
|
||||
var url = './mdstores/new/' + encodeURIComponent(format) + '/' + encodeURIComponent(layout) + '/' + encodeURIComponent(interpretation);
|
||||
if (dsName || dsId || apiId) {
|
||||
url += '?dsName=' + encodeURIComponent(dsName) + '&dsId=' + encodeURIComponent(dsId) + '&apiId=' + encodeURIComponent(apiId);
|
||||
}
|
||||
$http.get(url).success(function(data) {
|
||||
$scope.reload();
|
||||
}).error(function(err) {
|
||||
alert('ERROR: ' + err.message);
|
||||
});
|
||||
};
|
||||
|
||||
$scope.deleteMdstore = function(mdId) {
|
||||
if (confirm("Are you sure ?")) {
|
||||
$http.delete('./mdstores/mdstore/' + mdId).success(function(data) {
|
||||
$scope.reload();
|
||||
}).error(function(err) {
|
||||
alert('ERROR: ' + err.message);
|
||||
});
|
||||
}
|
||||
};
|
||||
|
||||
$scope.prepareVersion = function(mdId, currentVersion) {
|
||||
$scope.versions = [];
|
||||
$http.get('./mdstores/mdstore/' + mdId + '/newVersion?' + $.now()).success(function(data) {
|
||||
$scope.reload();
|
||||
$scope.listVersions(mdId, currentVersion);
|
||||
}).error(function(err) {
|
||||
alert('ERROR: ' + err.message);
|
||||
});
|
||||
};
|
||||
|
||||
$scope.commitVersion = function(versionId) {
|
||||
var size = parseInt(prompt("New Size", "0"));
|
||||
if (size >= 0) {
|
||||
$http.get("./mdstores/version/" + versionId + "/commit/" + size + '?' + $.now()).success(function(data) {
|
||||
$scope.reload();
|
||||
$scope.openCurrentVersion = versionId;
|
||||
$scope.refreshVersions();
|
||||
}).error(function(err) {
|
||||
alert('ERROR: ' + err.message);
|
||||
});
|
||||
}
|
||||
};
|
||||
|
||||
$scope.abortVersion = function(versionId) {
|
||||
$http.get("./mdstores/version/" + versionId + "/abort?" + $.now()).success(function(data) {
|
||||
$scope.reload();
|
||||
$scope.refreshVersions();
|
||||
}).error(function(err) {
|
||||
alert('ERROR: ' + err.message);
|
||||
});
|
||||
};
|
||||
|
||||
$scope.resetReading = function(versionId) {
|
||||
$http.get("./mdstores/version/" + versionId + "/resetReading" + '?' + $.now()).success(function(data) {
|
||||
$scope.reload();
|
||||
$scope.refreshVersions();
|
||||
}).error(function(err) {
|
||||
alert('ERROR: ' + err.message);
|
||||
});
|
||||
};
|
||||
|
||||
$scope.listVersions = function(mdId, current) {
|
||||
$scope.openMdstore = mdId;
|
||||
$scope.openCurrentVersion = current;
|
||||
$scope.versions = [];
|
||||
$scope.refreshVersions();
|
||||
};
|
||||
|
||||
$scope.refreshVersions = function() {
|
||||
$http.get('./mdstores/mdstore/' + $scope.openMdstore + '/versions?' + $.now()).success(function(data) {
|
||||
angular.forEach(data, function(value, key) {
|
||||
value.current = (value.id == $scope.openCurrentVersion);
|
||||
});
|
||||
$scope.versions = data;
|
||||
}).error(function(err) {
|
||||
alert('ERROR: ' + err.message);
|
||||
});
|
||||
};
|
||||
|
||||
$scope.deleteVersion = function(versionId, force) {
|
||||
if (confirm("Are you sure ?")) {
|
||||
var url = './mdstores/version/' + versionId;
|
||||
if (force) { url += '?force=true'; }
|
||||
|
||||
$http.delete(url).success(function(data) {
|
||||
$scope.reload();
|
||||
$scope.refreshVersions();
|
||||
}).error(function(err) {
|
||||
alert('ERROR: ' + err.message);
|
||||
});
|
||||
}
|
||||
};
|
||||
|
||||
$scope.reload();
|
||||
|
||||
});
|
||||
|
||||
|
|
@ -0,0 +1,348 @@
|
|||
/**
|
||||
* Copyright (c) 2011-2014 Felix Gnass
|
||||
* Licensed under the MIT license
|
||||
*/
|
||||
(function(root, factory) {
|
||||
|
||||
/* CommonJS */
|
||||
if (typeof exports == 'object') module.exports = factory()
|
||||
|
||||
/* AMD module */
|
||||
else if (typeof define == 'function' && define.amd) define(factory)
|
||||
|
||||
/* Browser global */
|
||||
else root.Spinner = factory()
|
||||
}
|
||||
(this, function() {
|
||||
"use strict";
|
||||
|
||||
var prefixes = ['webkit', 'Moz', 'ms', 'O'] /* Vendor prefixes */
|
||||
, animations = {} /* Animation rules keyed by their name */
|
||||
, useCssAnimations /* Whether to use CSS animations or setTimeout */
|
||||
|
||||
/**
|
||||
* Utility function to create elements. If no tag name is given,
|
||||
* a DIV is created. Optionally properties can be passed.
|
||||
*/
|
||||
function createEl(tag, prop) {
|
||||
var el = document.createElement(tag || 'div')
|
||||
, n
|
||||
|
||||
for(n in prop) el[n] = prop[n]
|
||||
return el
|
||||
}
|
||||
|
||||
/**
|
||||
* Appends children and returns the parent.
|
||||
*/
|
||||
function ins(parent /* child1, child2, ...*/) {
|
||||
for (var i=1, n=arguments.length; i<n; i++)
|
||||
parent.appendChild(arguments[i])
|
||||
|
||||
return parent
|
||||
}
|
||||
|
||||
/**
|
||||
* Insert a new stylesheet to hold the @keyframe or VML rules.
|
||||
*/
|
||||
var sheet = (function() {
|
||||
var el = createEl('style', {type : 'text/css'})
|
||||
ins(document.getElementsByTagName('head')[0], el)
|
||||
return el.sheet || el.styleSheet
|
||||
}())
|
||||
|
||||
/**
|
||||
* Creates an opacity keyframe animation rule and returns its name.
|
||||
* Since most mobile Webkits have timing issues with animation-delay,
|
||||
* we create separate rules for each line/segment.
|
||||
*/
|
||||
function addAnimation(alpha, trail, i, lines) {
|
||||
var name = ['opacity', trail, ~~(alpha*100), i, lines].join('-')
|
||||
, start = 0.01 + i/lines * 100
|
||||
, z = Math.max(1 - (1-alpha) / trail * (100-start), alpha)
|
||||
, prefix = useCssAnimations.substring(0, useCssAnimations.indexOf('Animation')).toLowerCase()
|
||||
, pre = prefix && '-' + prefix + '-' || ''
|
||||
|
||||
if (!animations[name]) {
|
||||
sheet.insertRule(
|
||||
'@' + pre + 'keyframes ' + name + '{' +
|
||||
'0%{opacity:' + z + '}' +
|
||||
start + '%{opacity:' + alpha + '}' +
|
||||
(start+0.01) + '%{opacity:1}' +
|
||||
(start+trail) % 100 + '%{opacity:' + alpha + '}' +
|
||||
'100%{opacity:' + z + '}' +
|
||||
'}', sheet.cssRules.length)
|
||||
|
||||
animations[name] = 1
|
||||
}
|
||||
|
||||
return name
|
||||
}
|
||||
|
||||
/**
|
||||
* Tries various vendor prefixes and returns the first supported property.
|
||||
*/
|
||||
function vendor(el, prop) {
|
||||
var s = el.style
|
||||
, pp
|
||||
, i
|
||||
|
||||
prop = prop.charAt(0).toUpperCase() + prop.slice(1)
|
||||
for(i=0; i<prefixes.length; i++) {
|
||||
pp = prefixes[i]+prop
|
||||
if(s[pp] !== undefined) return pp
|
||||
}
|
||||
if(s[prop] !== undefined) return prop
|
||||
}
|
||||
|
||||
/**
|
||||
* Sets multiple style properties at once.
|
||||
*/
|
||||
function css(el, prop) {
|
||||
for (var n in prop)
|
||||
el.style[vendor(el, n)||n] = prop[n]
|
||||
|
||||
return el
|
||||
}
|
||||
|
||||
/**
|
||||
* Fills in default values.
|
||||
*/
|
||||
function merge(obj) {
|
||||
for (var i=1; i < arguments.length; i++) {
|
||||
var def = arguments[i]
|
||||
for (var n in def)
|
||||
if (obj[n] === undefined) obj[n] = def[n]
|
||||
}
|
||||
return obj
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the absolute page-offset of the given element.
|
||||
*/
|
||||
function pos(el) {
|
||||
var o = { x:el.offsetLeft, y:el.offsetTop }
|
||||
while((el = el.offsetParent))
|
||||
o.x+=el.offsetLeft, o.y+=el.offsetTop
|
||||
|
||||
return o
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the line color from the given string or array.
|
||||
*/
|
||||
function getColor(color, idx) {
|
||||
return typeof color == 'string' ? color : color[idx % color.length]
|
||||
}
|
||||
|
||||
// Built-in defaults
|
||||
|
||||
var defaults = {
|
||||
lines: 12, // The number of lines to draw
|
||||
length: 7, // The length of each line
|
||||
width: 5, // The line thickness
|
||||
radius: 10, // The radius of the inner circle
|
||||
rotate: 0, // Rotation offset
|
||||
corners: 1, // Roundness (0..1)
|
||||
color: '#000', // #rgb or #rrggbb
|
||||
direction: 1, // 1: clockwise, -1: counterclockwise
|
||||
speed: 1, // Rounds per second
|
||||
trail: 100, // Afterglow percentage
|
||||
opacity: 1/4, // Opacity of the lines
|
||||
fps: 20, // Frames per second when using setTimeout()
|
||||
zIndex: 2e9, // Use a high z-index by default
|
||||
className: 'spinner', // CSS class to assign to the element
|
||||
top: '50%', // center vertically
|
||||
left: '50%', // center horizontally
|
||||
position: 'absolute' // element position
|
||||
}
|
||||
|
||||
/** The constructor */
|
||||
function Spinner(o) {
|
||||
this.opts = merge(o || {}, Spinner.defaults, defaults)
|
||||
}
|
||||
|
||||
// Global defaults that override the built-ins:
|
||||
Spinner.defaults = {}
|
||||
|
||||
merge(Spinner.prototype, {
|
||||
|
||||
/**
|
||||
* Adds the spinner to the given target element. If this instance is already
|
||||
* spinning, it is automatically removed from its previous target b calling
|
||||
* stop() internally.
|
||||
*/
|
||||
spin: function(target) {
|
||||
this.stop()
|
||||
|
||||
var self = this
|
||||
, o = self.opts
|
||||
, el = self.el = css(createEl(0, {className: o.className}), {position: o.position, width: 0, zIndex: o.zIndex})
|
||||
, mid = o.radius+o.length+o.width
|
||||
|
||||
if (target) {
|
||||
target.insertBefore(el, target.firstChild||null)
|
||||
css(el, {
|
||||
left: o.left,
|
||||
top: o.top
|
||||
})
|
||||
}
|
||||
|
||||
el.setAttribute('role', 'progressbar')
|
||||
self.lines(el, self.opts)
|
||||
|
||||
if (!useCssAnimations) {
|
||||
// No CSS animation support, use setTimeout() instead
|
||||
var i = 0
|
||||
, start = (o.lines - 1) * (1 - o.direction) / 2
|
||||
, alpha
|
||||
, fps = o.fps
|
||||
, f = fps/o.speed
|
||||
, ostep = (1-o.opacity) / (f*o.trail / 100)
|
||||
, astep = f/o.lines
|
||||
|
||||
;(function anim() {
|
||||
i++;
|
||||
for (var j = 0; j < o.lines; j++) {
|
||||
alpha = Math.max(1 - (i + (o.lines - j) * astep) % f * ostep, o.opacity)
|
||||
|
||||
self.opacity(el, j * o.direction + start, alpha, o)
|
||||
}
|
||||
self.timeout = self.el && setTimeout(anim, ~~(1000/fps))
|
||||
})()
|
||||
}
|
||||
return self
|
||||
},
|
||||
|
||||
/**
|
||||
* Stops and removes the Spinner.
|
||||
*/
|
||||
stop: function() {
|
||||
var el = this.el
|
||||
if (el) {
|
||||
clearTimeout(this.timeout)
|
||||
if (el.parentNode) el.parentNode.removeChild(el)
|
||||
this.el = undefined
|
||||
}
|
||||
return this
|
||||
},
|
||||
|
||||
/**
|
||||
* Internal method that draws the individual lines. Will be overwritten
|
||||
* in VML fallback mode below.
|
||||
*/
|
||||
lines: function(el, o) {
|
||||
var i = 0
|
||||
, start = (o.lines - 1) * (1 - o.direction) / 2
|
||||
, seg
|
||||
|
||||
function fill(color, shadow) {
|
||||
return css(createEl(), {
|
||||
position: 'absolute',
|
||||
width: (o.length+o.width) + 'px',
|
||||
height: o.width + 'px',
|
||||
background: color,
|
||||
boxShadow: shadow,
|
||||
transformOrigin: 'left',
|
||||
transform: 'rotate(' + ~~(360/o.lines*i+o.rotate) + 'deg) translate(' + o.radius+'px' +',0)',
|
||||
borderRadius: (o.corners * o.width>>1) + 'px'
|
||||
})
|
||||
}
|
||||
|
||||
for (; i < o.lines; i++) {
|
||||
seg = css(createEl(), {
|
||||
position: 'absolute',
|
||||
top: 1+~(o.width/2) + 'px',
|
||||
transform: o.hwaccel ? 'translate3d(0,0,0)' : '',
|
||||
opacity: o.opacity,
|
||||
animation: useCssAnimations && addAnimation(o.opacity, o.trail, start + i * o.direction, o.lines) + ' ' + 1/o.speed + 's linear infinite'
|
||||
})
|
||||
|
||||
if (o.shadow) ins(seg, css(fill('#000', '0 0 4px ' + '#000'), {top: 2+'px'}))
|
||||
ins(el, ins(seg, fill(getColor(o.color, i), '0 0 1px rgba(0,0,0,.1)')))
|
||||
}
|
||||
return el
|
||||
},
|
||||
|
||||
/**
|
||||
* Internal method that adjusts the opacity of a single line.
|
||||
* Will be overwritten in VML fallback mode below.
|
||||
*/
|
||||
opacity: function(el, i, val) {
|
||||
if (i < el.childNodes.length) el.childNodes[i].style.opacity = val
|
||||
}
|
||||
|
||||
})
|
||||
|
||||
|
||||
function initVML() {
|
||||
|
||||
/* Utility function to create a VML tag */
|
||||
function vml(tag, attr) {
|
||||
return createEl('<' + tag + ' xmlns="urn:schemas-microsoft.com:vml" class="spin-vml">', attr)
|
||||
}
|
||||
|
||||
// No CSS transforms but VML support, add a CSS rule for VML elements:
|
||||
sheet.addRule('.spin-vml', 'behavior:url(#default#VML)')
|
||||
|
||||
Spinner.prototype.lines = function(el, o) {
|
||||
var r = o.length+o.width
|
||||
, s = 2*r
|
||||
|
||||
function grp() {
|
||||
return css(
|
||||
vml('group', {
|
||||
coordsize: s + ' ' + s,
|
||||
coordorigin: -r + ' ' + -r
|
||||
}),
|
||||
{ width: s, height: s }
|
||||
)
|
||||
}
|
||||
|
||||
var margin = -(o.width+o.length)*2 + 'px'
|
||||
, g = css(grp(), {position: 'absolute', top: margin, left: margin})
|
||||
, i
|
||||
|
||||
function seg(i, dx, filter) {
|
||||
ins(g,
|
||||
ins(css(grp(), {rotation: 360 / o.lines * i + 'deg', left: ~~dx}),
|
||||
ins(css(vml('roundrect', {arcsize: o.corners}), {
|
||||
width: r,
|
||||
height: o.width,
|
||||
left: o.radius,
|
||||
top: -o.width>>1,
|
||||
filter: filter
|
||||
}),
|
||||
vml('fill', {color: getColor(o.color, i), opacity: o.opacity}),
|
||||
vml('stroke', {opacity: 0}) // transparent stroke to fix color bleeding upon opacity change
|
||||
)
|
||||
)
|
||||
)
|
||||
}
|
||||
|
||||
if (o.shadow)
|
||||
for (i = 1; i <= o.lines; i++)
|
||||
seg(i, -2, 'progid:DXImageTransform.Microsoft.Blur(pixelradius=2,makeshadow=1,shadowopacity=.3)')
|
||||
|
||||
for (i = 1; i <= o.lines; i++) seg(i)
|
||||
return ins(el, g)
|
||||
}
|
||||
|
||||
Spinner.prototype.opacity = function(el, i, val, o) {
|
||||
var c = el.firstChild
|
||||
o = o.shadow && o.lines || 0
|
||||
if (c && i+o < c.childNodes.length) {
|
||||
c = c.childNodes[i+o]; c = c && c.firstChild; c = c && c.firstChild
|
||||
if (c) c.opacity = val
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
var probe = css(createEl('group'), {behavior: 'url(#default#VML)'})
|
||||
|
||||
if (!vendor(probe, 'transform') && probe.adj) initVML()
|
||||
else useCssAnimations = vendor(probe, 'animation')
|
||||
|
||||
return Spinner
|
||||
|
||||
}));
|
||||
|
|
@ -0,0 +1,27 @@
|
|||
<!DOCTYPE html>
|
||||
<html>
|
||||
|
||||
<head>
|
||||
<title>Metadata Inspector - ERROR</title>
|
||||
<link rel="stylesheet" href="./css/bootstrap.min.css" />
|
||||
<link rel="stylesheet" href="./css/bootstrap-theme.min.css" />
|
||||
<script src="./js/jquery-1.12.3.min.js"></script>
|
||||
<script src="./js/bootstrap.min.js"></script>
|
||||
</head>
|
||||
|
||||
<style>
|
||||
td {
|
||||
vertical-align: middle !important;
|
||||
}
|
||||
</style>
|
||||
|
||||
<body>
|
||||
<div class="container-fluid">
|
||||
<h1>Metadata Inspector - ERROR</h1>
|
||||
<hr />
|
||||
<h4 class="text-danger" th:text="${error}" />
|
||||
<hr />
|
||||
<pre class="small" th:text="${stacktrace}" />
|
||||
</div>
|
||||
</body>
|
||||
</html>
|
||||
|
|
@ -0,0 +1,199 @@
|
|||
<!DOCTYPE html>
|
||||
<html>
|
||||
|
||||
<head>
|
||||
<title>Metadata Inspector</title>
|
||||
<link rel="stylesheet" href="../../css/bootstrap.min.css" />
|
||||
<link rel="stylesheet" href="../../css/bootstrap-theme.min.css" />
|
||||
<script src="../../js/jquery-1.12.3.min.js"></script>
|
||||
<script src="../../js/bootstrap.min.js"></script>
|
||||
<script src="../../js/angular.min.js"></script>
|
||||
<script src="../../js/spin.js"></script>
|
||||
</head>
|
||||
|
||||
<style>
|
||||
td {
|
||||
vertical-align: middle !important;
|
||||
}
|
||||
|
||||
pre {
|
||||
border: 0 !important;
|
||||
background-color: transparent !important;
|
||||
}
|
||||
|
||||
.overlaydiv {
|
||||
position: fixed;
|
||||
width: 100%;
|
||||
height: 100%;
|
||||
z-index: 10000;
|
||||
visibility: hidden;
|
||||
}
|
||||
|
||||
.grayRectangle {
|
||||
position: absolute;
|
||||
background-color: black;
|
||||
opacity:0.6;
|
||||
top: 30%;
|
||||
left: 40%;
|
||||
width: 20%;
|
||||
height: 20%;
|
||||
z-index: 100;
|
||||
border-radius: 15px;
|
||||
}
|
||||
</style>
|
||||
|
||||
<script th:inline="javascript">
|
||||
/*<![CDATA[*/
|
||||
|
||||
function versionId() {
|
||||
return /*[[${versionId}]]*/ '';
|
||||
}
|
||||
|
||||
function limit() {
|
||||
return /*[[${limit}]]*/ 100;
|
||||
}
|
||||
|
||||
/*]]>*/
|
||||
</script>
|
||||
|
||||
<body ng-app="mdInspectorApp" ng-controller="mdInspectorController">
|
||||
|
||||
<div id="spinnerdiv" class="overlaydiv">
|
||||
<span class="grayRectangle"><!--The spinner is added on loading here--></span>
|
||||
</div>
|
||||
|
||||
<div class="container-fluid">
|
||||
<div class="row">
|
||||
<div class="col-xs-12">
|
||||
<h1>Metadata Inspector</h1>
|
||||
|
||||
<br />
|
||||
|
||||
<table class="table table-condensed small">
|
||||
<tr>
|
||||
<th rowspan="3" class="col-xs-1">MdStore</th>
|
||||
<th class="col-xs-2">ID</th>
|
||||
<td class="col-xs-9"><span th:text="${mdId}" /></td>
|
||||
</tr>
|
||||
|
||||
<tr>
|
||||
<th>Format / Layout / Interpretation</th>
|
||||
<td><span th:text="${format} + ' / ' + ${layout} + ' / ' + ${interpretation}" /></td>
|
||||
</tr>
|
||||
<tr>
|
||||
<th>Related Datasource</th>
|
||||
<td><span th:text="${dsName}" /></td>
|
||||
</tr>
|
||||
<tr>
|
||||
<th rowspan="4">Version</th>
|
||||
<th>ID</th>
|
||||
<td>
|
||||
<span th:if="${status} == 'current'" class="label label-success">current</span>
|
||||
<span th:if="${status} == 'writing'" class="label label-warning">writing</span>
|
||||
<span th:if="${status} == 'expired'" class="label label-danger">expired</span>
|
||||
<span th:text="${versionId}" />
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<th>Hdfs Path</th>
|
||||
<td><span th:text="${path}" /></td>
|
||||
</tr>
|
||||
<tr>
|
||||
<th>Last Update</th>
|
||||
<td><span th:text="${lastUpdate}" /></td>
|
||||
</tr>
|
||||
<tr>
|
||||
<th>Size</th>
|
||||
<td><span th:text="${size}" /></td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
<hr />
|
||||
<p ng-show="records.length > 0" class="text-primary text-center">
|
||||
The display is limited to the first {{limit}} records
|
||||
</p>
|
||||
|
||||
<br />
|
||||
|
||||
<div class="panel panel-primary" ng-repeat="rec in records | filter:recordsFilter">
|
||||
<div class="panel-heading small">{{rec.id}}</div>
|
||||
<table class="table table-condensed table-striped small">
|
||||
<tr>
|
||||
<th class="col-xs-4">Original Id</th>
|
||||
<td class="col-xs-8">{{rec.originalId}}</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<th>Collected on</th>
|
||||
<td>{{rec.dateOfCollection | date:'medium'}}</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<th>Transformed on</th>
|
||||
<td>{{rec.dateOfTransformation | date:'medium'}}</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<th>Provenance</th>
|
||||
<td>{{rec.provenance}}</td>
|
||||
</tr>
|
||||
</table>
|
||||
<div class="panel-body">
|
||||
<span class="label label-success pull-right">{{rec.encoding}}</span>
|
||||
<br />
|
||||
<pre class="small">{{rec.body}}</pre>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
</body>
|
||||
|
||||
<script lang="javascript">
|
||||
// Spinner show/hide methods ~ Andrea Mannocci
|
||||
var spinnerOpts = {
|
||||
lines: 15,
|
||||
length: 16,
|
||||
width: 5,
|
||||
radius: 25,
|
||||
color: '#eeeeee',
|
||||
className: 'spinner',
|
||||
top: '40%'
|
||||
};
|
||||
|
||||
var spinnerTarget = document.getElementById('spinnerdiv');
|
||||
|
||||
var spinner;
|
||||
|
||||
function showSpinner() {
|
||||
spinner = new Spinner(spinnerOpts).spin(spinnerTarget);
|
||||
spinnerTarget.style.visibility = 'visible';
|
||||
}
|
||||
|
||||
function hideSpinner() {
|
||||
spinnerTarget.style.visibility = 'hidden';
|
||||
spinner.stop();
|
||||
}
|
||||
|
||||
var app = angular.module('mdInspectorApp', []);
|
||||
|
||||
app.controller('mdInspectorController', function($scope, $http) {
|
||||
$scope.records = [];
|
||||
$scope.versionId = versionId();
|
||||
$scope.limit = limit();
|
||||
|
||||
$scope.reload = function() {
|
||||
showSpinner();
|
||||
$http.get('../../mdstores/version/' + $scope.versionId + '/parquet/content/' + $scope.limit + '/?' + $.now()).success(function(data) {
|
||||
hideSpinner();
|
||||
$scope.records = data;
|
||||
}).error(function(err) {
|
||||
hideSpinner();
|
||||
alert('ERROR: ' + err.message);
|
||||
});
|
||||
};
|
||||
|
||||
$scope.reload();
|
||||
|
||||
});
|
||||
</script>
|
||||
|
||||
</html>
|
||||
|
|
@ -36,7 +36,11 @@ class HdfsClientTest {
|
|||
|
||||
GenericRecord rec = null;
|
||||
final Set<String> fields = new LinkedHashSet<>();
|
||||
|
||||
int i = 0;
|
||||
|
||||
while ((rec = reader.read()) != null) {
|
||||
|
||||
if (fields.isEmpty()) {
|
||||
rec.getSchema().getFields().forEach(f -> fields.add(f.name()));
|
||||
}
|
||||
|
|
@ -46,8 +50,12 @@ class HdfsClientTest {
|
|||
map.put(f, rec.get(f));
|
||||
}
|
||||
|
||||
System.out.println(map);
|
||||
// System.out.println(map);
|
||||
|
||||
i++;
|
||||
}
|
||||
|
||||
System.out.println("Total: " + i);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue