D-Net
/
dnet-applications
16
1
Fork 0
Code Issues Pull Requests Releases Wiki Activity

zeppelin calls + list templates

This commit is contained in:
Michele Artini 2021-10-29 16:23:18 +02:00
parent b10bd1f385
commit 7c75386488
9 changed files with 221 additions and 160 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
apps/dhp-mdstore-manager/src/main/java/eu/dnetlib/data/mdstore/manager/controller/ZeppelinController.java
View File

@ -1,11 +1,15 @@
package eu.dnetlib.data.mdstore.manager.controller;
import java.util.List;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.http.HttpStatus;
import org.springframework.stereotype.Controller;
import org.springframework.web.bind.annotation.ExceptionHandler;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.ResponseBody;
import org.springframework.web.bind.annotation.ResponseStatus;
import org.springframework.web.servlet.ModelAndView;
@ -33,6 +37,12 @@ public class ZeppelinController {
return "redirect:" + zeppelinClient.zeppelinNote(note, mdstore, path);
}
@GetMapping("/templates")
@ResponseBody
public List<String> getTemplates() {
return zeppelinClient.listTemplates();
}
@ExceptionHandler(Exception.class)
@ResponseStatus(value = HttpStatus.INTERNAL_SERVER_ERROR)
public ModelAndView handleException(final Exception e) {

314
apps/dhp-mdstore-manager/src/main/java/eu/dnetlib/data/mdstore/manager/utils/ZeppelinClient.java
View File

@ -9,6 +9,10 @@ import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
import java.util.Set;
import java.util.stream.Collectors;
import javax.annotation.PostConstruct;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang.StringEscapeUtils;
@ -18,6 +22,7 @@ import org.apache.commons.logging.LogFactory;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.http.HttpEntity;
import org.springframework.http.HttpHeaders;
import org.springframework.http.HttpMethod;
import org.springframework.http.HttpStatus;
import org.springframework.http.MediaType;
import org.springframework.http.ResponseEntity;
@ -27,9 +32,11 @@ import org.springframework.util.MultiValueMap;
import org.springframework.web.client.RestTemplate;
import eu.dnetlib.data.mdstore.manager.exceptions.MDStoreManagerException;
import eu.dnetlib.data.mdstore.manager.utils.zeppelin.HasStatus;
import eu.dnetlib.data.mdstore.manager.utils.zeppelin.ListResponse;
import eu.dnetlib.data.mdstore.manager.utils.zeppelin.Note;
import eu.dnetlib.data.mdstore.manager.utils.zeppelin.Paragraph;
import eu.dnetlib.data.mdstore.manager.utils.zeppelin.SimpleResponse;
import eu.dnetlib.data.mdstore.manager.utils.zeppelin.StringResponse;
import eu.dnetlib.dhp.schema.mdstore.MDStoreWithInfo;
@ -50,16 +57,27 @@ public class ZeppelinClient {
private static final Log log = LogFactory.getLog(ZeppelinClient.class);
private static final Map<String, List<String>> DEFAULT_RIGHTS = new LinkedHashMap<>();
@PostConstruct
public void init() {
DEFAULT_RIGHTS.put("owners", Arrays.asList(zeppelinLogin));
DEFAULT_RIGHTS.put("readers", new ArrayList<>()); // ALL
DEFAULT_RIGHTS.put("runners", new ArrayList<>()); // ALL
DEFAULT_RIGHTS.put("writers", new ArrayList<>()); // ALL
}
private String jsessionid;
public String zeppelinNote(final String note, final MDStoreWithInfo mdstore, final String currentVersionPath) throws MDStoreManagerException {
final String jsessionid = obtainJsessionID();
final String newName =
StringUtils.join(Arrays.asList(zeppelinNamePrefix, "notes", mdstore.getDatasourceName().replaceAll("/", "-"), mdstore.getApiId()
.replaceAll("/", "-"), note.replaceAll("/", "-"), mdstore.getCurrentVersion().replaceAll("/", "-")), "/");
final Optional<String> oldNoteId = listNotes(jsessionid).stream()
final List<Map<String, String>> notes = listNotes();
final Optional<String> oldNoteId = notes.stream()
.filter(Objects::nonNull)
.filter(map -> newName.equals(map.get("name")))
.map(map -> map.get("id"))
@ -70,140 +88,43 @@ public class ZeppelinClient {
return zeppelinBaseUrl + "/#/notebook/" + oldNoteId.get();
}
final String templateNoteId = findTemplateNoteId(note, jsessionid);
final String templateName = zeppelinNamePrefix + "/templates/" + note;
final String templateNoteId = notes.stream()
.filter(map -> map.get("name").equals(templateName))
.map(map -> map.get("id"))
.findFirst()
.orElseThrow(() -> new MDStoreManagerException("Template Note not found: " + templateName));
final String newId = cloneNote(templateNoteId, newName, jsessionid);
log.info("New note created, id: " + newId + ", name: " + newName);
addParagraph(newId, confParagraph(mdstore, currentVersionPath), jsessionid);
reassignRights(newId, jsessionid);
final String newId = cloneNote(templateNoteId, newName, mdstore, currentVersionPath);
return zeppelinBaseUrl + "/#/notebook/" + newId;
}
// TODO: prepare the cron job
public void cleanExpiredNotes() {
try {
final String jsessionid = obtainJsessionID();
for (final Map<String, String> n : listNotes(jsessionid)) {
final String id = n.get("id");
if (n.get("name").startsWith(zeppelinNamePrefix + "/notes/") && isExpired(id, jsessionid)) {
deleteNote(id, jsessionid);
}
}
} catch (final Exception e) {
log.error("Error cleaning expired notes", e);
}
public List<String> listTemplates() {
final String prefix = zeppelinNamePrefix + "/templates/";
return listNotes().stream()
.map(map -> map.get("name"))
.filter(s -> s.startsWith(prefix))
.map(s -> StringUtils.substringAfter(s, prefix))
.sorted()
.collect(Collectors.toList());
}
private String obtainJsessionID() throws MDStoreManagerException {
final HttpHeaders headers = new HttpHeaders();
headers.setContentType(MediaType.APPLICATION_FORM_URLENCODED);
final MultiValueMap<String, String> map = new LinkedMultiValueMap<>();
map.add("userName", zeppelinLogin);
map.add("password", zeppelinPassword);
final HttpEntity<MultiValueMap<String, String>> request = new HttpEntity<>(map, headers);
final String url = zeppelinBaseUrl + "/api/login";
final ResponseEntity<?> res = new RestTemplate().postForEntity(url, request, Object.class);
if (res.getStatusCode() != HttpStatus.OK) {
log.error("Zeppelin API: login failed with HTTP error: " + res);
throw new MDStoreManagerException("Zeppelin API: login failed with HTTP error: " + res);
} else if (!res.getHeaders().containsKey(HttpHeaders.SET_COOKIE)) {
log.error("Zeppelin API: login failed (missing SET_COOKIE header)");
throw new MDStoreManagerException("Zeppelin API: login failed (missing SET_COOKIE header)");
} else {
return res.getHeaders()
.get(HttpHeaders.SET_COOKIE)
.stream()
.map(s -> s.split(";"))
.flatMap(Arrays::stream)
.map(String::trim)
.filter(s -> s.startsWith("JSESSIONID="))
.map(s -> StringUtils.removeStart(s, "JSESSIONID="))
.filter(s -> !s.equalsIgnoreCase("deleteMe"))
.distinct()
.filter(this::testConnection)
.findFirst()
.orElseThrow(() -> new MDStoreManagerException("Zeppelin API: login failed (invalid jsessionid)"));
}
private List<Map<String, String>> listNotes() {
return callApi(HttpMethod.GET, "notebook", ListResponse.class, null).getBody();
}
private boolean testConnection(final String jsessionid) {
private String cloneNote(final String noteId, final String newName, final MDStoreWithInfo mdstore, final String currentVersionPath)
throws MDStoreManagerException {
final String newId = callApi(HttpMethod.POST, "notebook/" + noteId, StringResponse.class, new Note(newName)).getBody();
callApi(HttpMethod.POST, "notebook/" + noteId + "/paragraph", StringResponse.class, confParagraph(mdstore, currentVersionPath)).getBody();
callApi(HttpMethod.PUT, "notebook/" + noteId + "/permissions", SimpleResponse.class, DEFAULT_RIGHTS);
final String url = zeppelinBaseUrl + "/api/notebook;JSESSIONID=" + jsessionid;
log.info("Performing GET: " + url);
log.info("New note created, id: " + newId + ", name: " + newName);
final ResponseEntity<ListResponse> res = new RestTemplate().getForEntity(url, ListResponse.class);
return newId;
if (res.getStatusCode() != HttpStatus.OK) {
return false;
} else if (res.getBody() == null) {
return false;
} else if (!res.getBody().getStatus().equals("OK")) {
return false;
} else {
log.info("Connected to zeppelin: " + res.getBody());
log.info("Found JSESSIONID: " + jsessionid);
return true;
}
}
private List<Map<String, String>> listNotes(final String jsessionid) throws MDStoreManagerException {
final String url = zeppelinBaseUrl + "/api/notebook;JSESSIONID=" + jsessionid;
log.info("Performing GET: " + url);
final ResponseEntity<ListResponse> res = new RestTemplate().getForEntity(url, ListResponse.class);
if (res.getStatusCode() != HttpStatus.OK) {
log.error("Zeppelin API failed with HTTP error: " + res);
throw new MDStoreManagerException("Zeppelin API failed with HTTP error: " + res);
} else if (res.getBody() == null) {
log.error("Zeppelin API returned a null response");
throw new MDStoreManagerException("Zeppelin API returned a null response");
} else if (!res.getBody().getStatus().equals("OK")) {
log.error("Registration of zeppelin note failed: " + res.getBody());
throw new MDStoreManagerException("Registration of zeppelin note failed: " + res.getBody());
} else {
return res.getBody().getBody();
}
}
private String findTemplateNoteId(final String noteTemplate, final String jsessionid) throws MDStoreManagerException {
final String templateName = zeppelinNamePrefix + "/templates/" + noteTemplate;
return listNotes(jsessionid).stream()
.filter(map -> map.get("name").equals(templateName))
.map(map -> map.get("id"))
.findFirst()
.orElseThrow(() -> new MDStoreManagerException("Template Note not found: " + templateName));
}
private String cloneNote(final String noteId, final String newName, final String jsessionid) throws MDStoreManagerException {
final String url = zeppelinBaseUrl + "/api/notebook/" + noteId + ";JSESSIONID=" + jsessionid;
log.debug("Performing POST: " + url);
final ResponseEntity<StringResponse> res = new RestTemplate().postForEntity(url, new Note(newName), StringResponse.class);
if (res.getStatusCode() != HttpStatus.OK) {
log.error("Zeppelin API failed with HTTP error: " + res);
throw new MDStoreManagerException("Zeppelin API failed with HTTP error: " + res);
} else if (res.getBody() == null) {
log.error("Zeppelin API returned a null response");
throw new MDStoreManagerException("Zeppelin API returned a null response");
} else if (!res.getBody().getStatus().equals("OK")) {
log.error("Registration of zeppelin note failed: " + res.getBody());
throw new MDStoreManagerException("Registration of zeppelin note failed: " + res.getBody());
} else {
return res.getBody().getBody();
}
}
private Paragraph confParagraph(final MDStoreWithInfo mdstore, final String currentVersionPath) throws MDStoreManagerException {
@ -222,48 +143,143 @@ public class ZeppelinClient {
}
}
private String addParagraph(final String noteId, final Paragraph paragraph, final String jsessionid) throws MDStoreManagerException {
final String url = zeppelinBaseUrl + "/api/notebook/" + noteId + "/paragraph;JSESSIONID=" + jsessionid;
log.debug("Performing POST: " + url);
// TODO: prepare the cron job
public void cleanExpiredNotes() {
try {
for (final Map<String, String> n : listNotes()) {
final String id = n.get("id");
if (n.get("name").startsWith(zeppelinNamePrefix + "/notes/") && isExpired(id)) {
deleteNote(id);
}
}
} catch (final Exception e) {
log.error("Error cleaning expired notes", e);
}
}
final ResponseEntity<StringResponse> res = new RestTemplate().postForEntity(url, paragraph, StringResponse.class);
private void deleteNote(final String noteId) {
callApi(HttpMethod.DELETE, "notebook/" + noteId, SimpleResponse.class, null);
}
if (res.getStatusCode() != HttpStatus.OK) {
private boolean isExpired(final String id) {
// TODO Auto-generated method stub
return false;
}
private <T extends HasStatus> T callApi(final HttpMethod method, final String api, final Class<T> resClazz, final Object objRequest) {
if (jsessionid == null) {
final T res = findNewJsessionId(method, api, resClazz, objRequest);
if (res != null) { return res; }
} else {
try {
return callApi(method, api, resClazz, objRequest, jsessionid);
} catch (final MDStoreManagerException e) {
final T res = findNewJsessionId(method, api, resClazz, objRequest);
if (res != null) { return res; }
}
}
throw new RuntimeException("All attempted calls are failed");
}
@SuppressWarnings("unchecked")
private <T extends HasStatus> T callApi(final HttpMethod method,
final String api,
final Class<T> resClazz,
final Object objRequest,
final String jsessionid)
throws MDStoreManagerException {
final String url = String.format("%s/api/%s;JSESSIONID=%s", zeppelinBaseUrl, api, jsessionid);
final RestTemplate restTemplate = new RestTemplate();
ResponseEntity<T> res = null;
switch (method) {
case GET:
log.info("Performing GET: " + url);
res = restTemplate.getForEntity(url, resClazz);
break;
case POST:
log.info("Performing POST: " + url);
res = restTemplate.postForEntity(url, objRequest, resClazz);
break;
case PUT:
log.info("Performing PUT: " + url);
restTemplate.put(url, objRequest);
break;
case DELETE:
log.info("Performing DELETE: " + url);
restTemplate.delete(url);
break;
default:
throw new RuntimeException("Unsupported method: " + method);
}
if (method == HttpMethod.PUT || method == HttpMethod.DELETE) {
return (T) new SimpleResponse("OK");
} else if (res == null) {
log.error("NULL response from the API");
throw new MDStoreManagerException("NULL response from the API");
} else if (res.getStatusCode() != HttpStatus.OK) {
log.error("Zeppelin API failed with HTTP error: " + res);
throw new MDStoreManagerException("Zeppelin API failed with HTTP error: " + res);
} else if (res.getBody() == null) {
log.error("Zeppelin API returned a null response");
throw new MDStoreManagerException("Zeppelin API returned a null response");
} else if (!res.getBody().getStatus().equals("OK")) {
log.error("Registration of zeppelin note failed: " + res.getBody());
log.error("Zeppelin API Operation failed: " + res.getBody());
throw new MDStoreManagerException("Registration of zeppelin note failed: " + res.getBody());
} else {
return res.getBody().getBody();
return res.getBody();
}
}
private void reassignRights(final String noteId, final String jsessionid) {
final String url = zeppelinBaseUrl + "/api/notebook/" + noteId + "/permissions;JSESSIONID=" + jsessionid;
log.info("Performing PUT: " + url);
final Map<String, List<String>> rights = new LinkedHashMap<>();
rights.put("owners", Arrays.asList(zeppelinLogin));
rights.put("readers", new ArrayList<>()); // ALL
rights.put("runners", new ArrayList<>()); // ALL
rights.put("writers", new ArrayList<>()); // ALL
new RestTemplate().put(url, rights);
private <T extends HasStatus> T findNewJsessionId(final HttpMethod method, final String api, final Class<T> resClazz, final Object objRequest) {
for (final String id : obtainJsessionIDs()) {
try {
final T res = callApi(method, api, resClazz, objRequest, id);
setJsessionid(id);
return res;
} catch (final MDStoreManagerException e) {
log.warn("Skipping invalid jsessionid: " + id);
}
}
return null;
}
private void deleteNote(final String id, final String jsessionid) {
final String url = zeppelinBaseUrl + "/api/notebook/" + id + ";JSESSIONID=" + jsessionid;
log.debug("Performing DELETE: " + url);
new RestTemplate().delete(url);
}
private Set<String> obtainJsessionIDs() {
private boolean isExpired(final String id, final String jsessionid) {
// TODO Auto-generated method stub
return false;
final HttpHeaders headers = new HttpHeaders();
headers.setContentType(MediaType.APPLICATION_FORM_URLENCODED);
final MultiValueMap<String, String> map = new LinkedMultiValueMap<>();
map.add("userName", zeppelinLogin);
map.add("password", zeppelinPassword);
final HttpEntity<MultiValueMap<String, String>> request = new HttpEntity<>(map, headers);
final String url = zeppelinBaseUrl + "/api/login";
final ResponseEntity<?> res = new RestTemplate().postForEntity(url, request, Object.class);
if (res.getStatusCode() != HttpStatus.OK) {
log.error("Zeppelin API: login failed with HTTP error: " + res);
throw new RuntimeException("Zeppelin API: login failed with HTTP error: " + res);
} else if (!res.getHeaders().containsKey(HttpHeaders.SET_COOKIE)) {
log.error("Zeppelin API: login failed (missing SET_COOKIE header)");
throw new RuntimeException("Zeppelin API: login failed (missing SET_COOKIE header)");
} else {
return res.getHeaders()
.get(HttpHeaders.SET_COOKIE)
.stream()
.map(s -> s.split(";"))
.flatMap(Arrays::stream)
.map(String::trim)
.filter(s -> s.startsWith("JSESSIONID="))
.map(s -> StringUtils.removeStart(s, "JSESSIONID="))
.filter(s -> !s.equalsIgnoreCase("deleteMe"))
.collect(Collectors.toSet());
}
}
public String getJsessionid() {

6
apps/dhp-mdstore-manager/src/main/java/eu/dnetlib/data/mdstore/manager/utils/zeppelin/HasStatus.java Normal file
View File

@ -0,0 +1,6 @@
package eu.dnetlib.data.mdstore.manager.utils.zeppelin;
public interface HasStatus {
String getStatus();
}

3
apps/dhp-mdstore-manager/src/main/java/eu/dnetlib/data/mdstore/manager/utils/zeppelin/ListResponse.java
View File

@ -3,12 +3,13 @@ package eu.dnetlib.data.mdstore.manager.utils.zeppelin;
import java.util.List;
import java.util.Map;
public class ListResponse {
public class ListResponse implements HasStatus {
private String status;
private String message;
private List<Map<String, String>> body;
@Override
public String getStatus() {
return status;
}

16
apps/dhp-mdstore-manager/src/main/java/eu/dnetlib/data/mdstore/manager/utils/zeppelin/SimpleResponse.java Normal file
View File

@ -0,0 +1,16 @@
package eu.dnetlib.data.mdstore.manager.utils.zeppelin;
public class SimpleResponse implements HasStatus {
private final String status;
public SimpleResponse(final String status) {
this.status = status;
}
@Override
public String getStatus() {
return status;
}
}

3
apps/dhp-mdstore-manager/src/main/java/eu/dnetlib/data/mdstore/manager/utils/zeppelin/StringResponse.java
View File

@ -1,11 +1,12 @@
package eu.dnetlib.data.mdstore.manager.utils.zeppelin;
public class StringResponse {
public class StringResponse implements HasStatus {
private String status;
private String message;
private String body;
@Override
public String getStatus() {
return status;
}

10
apps/dhp-mdstore-manager/src/main/resources/application.properties
View File

@ -31,9 +31,13 @@ dhp.mdstore-manager.hadoop.cluster = GARR
dhp.mdstore-manager.hdfs.base-path = /data/dnet.dev/mdstore
dhp.mdstore-manager.hadoop.user = dnet.dev
dhp.mdstore-manager.hadoop.zeppelin.base-url = https://iis-cdh5-test-gw.ocean.icm.edu.pl/zeppelin
dhp.mdstore-manager.hadoop.zeppelin.login =
dhp.mdstore-manager.hadoop.zeppelin.password =
#dhp.mdstore-manager.hadoop.zeppelin.base-url = https://iis-cdh5-test-gw.ocean.icm.edu.pl/zeppelin
#dhp.mdstore-manager.hadoop.zeppelin.login =
#dhp.mdstore-manager.hadoop.zeppelin.password =
dhp.mdstore-manager.hadoop.zeppelin.base-url = https://hadoop-zeppelin.garr-pa1.d4science.org
dhp.mdstore-manager.hadoop.zeppelin.login = zeppelin-api
dhp.mdstore-manager.hadoop.zeppelin.password = bf5a13ed84a716a
dhp.mdstore-manager.hadoop.zeppelin.name-prefix = mdstoreManager
dhp.mdstore-manager.inspector.records.max = 1000

5
apps/dhp-mdstore-manager/src/main/resources/static/index.html
View File

@ -81,10 +81,7 @@
<div class="btn-group">
<button class="btn btn-sm btn-warning dropdown-toggle" data-toggle="dropdown">zeppelin <span class="caret"></span></button>
<div class="dropdown-menu dropdown-menu-right">
<a class="dropdown-item small" href="./zeppelin/{{md.id}}/default" target="_blank">default note</a>
<a class="dropdown-item small" href="./zeppelin/{{md.id}}/dc_native" target="_blank">note for native stores (oai_dc)</a>
<a class="dropdown-item small" href="./zeppelin/{{md.id}}/datacite_native" target="_blank">note for native stores (datacite)</a>
<a class="dropdown-item small" href="./zeppelin/{{md.id}}/oaf_cleaned" target="_blank">note for transformed stores</a>
<a class="dropdown-item small" href="./zeppelin/{{md.id}}/{{t}}" target="_blank" ng-repeat="t in zeppelinTemplates">{{t}}</a>
</div>
</div>
</div>

14
apps/dhp-mdstore-manager/src/main/resources/static/mdstoremanager.js
View File

@ -5,6 +5,7 @@ app.controller('mdstoreManagerController', function($scope, $http) {
$scope.versions = [];
$scope.openMdstore = '';
$scope.openCurrentVersion = ''
$scope.zeppelinTemplates = [];
$scope.forceVersionDelete = false;
@ -14,7 +15,16 @@ app.controller('mdstoreManagerController', function($scope, $http) {
}, function errorCallback(res) {
alert('ERROR: ' + res.data.message);
});
};
};
$scope.obtainZeppelinTemplates = function() {
$http.get('./zeppelin/templates?' + $.now()).then(function successCallback(res) {
$scope.zeppelinTemplates = res.data;
}, function errorCallback(res) {
alert('ERROR: ' + res.data.message);
});
};
$scope.newMdstore = function(format, layout, interpretation, dsName, dsId, apiId) {
var url = './mdstores/new/' + encodeURIComponent(format) + '/' + encodeURIComponent(layout) + '/' + encodeURIComponent(interpretation);
@ -112,5 +122,5 @@ app.controller('mdstoreManagerController', function($scope, $http) {
};
$scope.reload();
$scope.obtainZeppelinTemplates();
});