244 lines
9.5 KiB
Java
244 lines
9.5 KiB
Java
package eu.dnetlib.data.mdstore.manager.utils;
|
|
|
|
import java.io.IOException;
|
|
import java.util.Arrays;
|
|
import java.util.List;
|
|
import java.util.Map;
|
|
import java.util.Objects;
|
|
import java.util.Optional;
|
|
|
|
import org.apache.commons.io.IOUtils;
|
|
import org.apache.commons.lang3.StringUtils;
|
|
import org.apache.commons.logging.Log;
|
|
import org.apache.commons.logging.LogFactory;
|
|
import org.springframework.beans.factory.annotation.Value;
|
|
import org.springframework.http.HttpEntity;
|
|
import org.springframework.http.HttpHeaders;
|
|
import org.springframework.http.HttpStatus;
|
|
import org.springframework.http.MediaType;
|
|
import org.springframework.http.ResponseEntity;
|
|
import org.springframework.stereotype.Component;
|
|
import org.springframework.util.LinkedMultiValueMap;
|
|
import org.springframework.util.MultiValueMap;
|
|
import org.springframework.web.client.RestTemplate;
|
|
|
|
import eu.dnetlib.data.mdstore.manager.exceptions.MDStoreManagerException;
|
|
import eu.dnetlib.data.mdstore.manager.utils.zeppelin.ListResponse;
|
|
import eu.dnetlib.data.mdstore.manager.utils.zeppelin.Note;
|
|
import eu.dnetlib.data.mdstore.manager.utils.zeppelin.Paragraph;
|
|
import eu.dnetlib.data.mdstore.manager.utils.zeppelin.StringResponse;
|
|
|
|
@Component
|
|
public class ZeppelinClient {
|
|
|
|
@Value("${dhp.mdstore-manager.hadoop.zeppelin.login}")
|
|
private String zeppelinLogin;
|
|
|
|
@Value("${dhp.mdstore-manager.hadoop.zeppelin.password}")
|
|
private String zeppelinPassword;
|
|
|
|
@Value("${dhp.mdstore-manager.hadoop.zeppelin.base-url}")
|
|
private String zeppelinBaseUrl;
|
|
|
|
@Value("${dhp.mdstore-manager.hadoop.zeppelin.name-prefix}")
|
|
private String zeppelinNamePrefix;
|
|
|
|
private static final Log log = LogFactory.getLog(ZeppelinClient.class);
|
|
|
|
public String zeppelinNote(final String note, final String mdId, final String currentVersion, final String currentVersionPath)
|
|
throws MDStoreManagerException {
|
|
final String jsessionid = obtainJsessionID();
|
|
|
|
final String newName = zeppelinNamePrefix + "/notes/" + note + "/" + currentVersion;
|
|
|
|
final Optional<String> oldNoteId = listNotes(jsessionid).stream()
|
|
.filter(Objects::nonNull)
|
|
.filter(map -> newName.equals(map.get("name")))
|
|
.map(map -> map.get("id"))
|
|
.findFirst();
|
|
|
|
if (oldNoteId.isPresent()) {
|
|
log.info("Returning existing note: " + oldNoteId.get());
|
|
return zeppelinBaseUrl + "/#/notebook/" + oldNoteId.get();
|
|
}
|
|
|
|
final String templateNoteId = findTemplateNoteId(note, jsessionid);
|
|
|
|
final String newId = cloneNote(templateNoteId, newName, jsessionid);
|
|
|
|
log.info("New note created, id: " + newId + ", name: " + newName);
|
|
|
|
addParagraph(newId, confParagraph(mdId, currentVersion, currentVersionPath), jsessionid);
|
|
|
|
return zeppelinBaseUrl + "/#/notebook/" + newId;
|
|
|
|
}
|
|
|
|
// TODO: prepare the cron job
|
|
public void cleanExpiredNotes() {
|
|
try {
|
|
final String jsessionid = obtainJsessionID();
|
|
|
|
for (final Map<String, String> n : listNotes(jsessionid)) {
|
|
final String id = n.get("id");
|
|
if (n.get("name").startsWith(zeppelinNamePrefix + "/notes/") && isExpired(id, jsessionid)) {
|
|
deleteNote(id, jsessionid);
|
|
}
|
|
}
|
|
} catch (final Exception e) {
|
|
log.error("Error cleaning expired notes", e);
|
|
}
|
|
}
|
|
|
|
private String obtainJsessionID() throws MDStoreManagerException {
|
|
|
|
final HttpHeaders headers = new HttpHeaders();
|
|
headers.setContentType(MediaType.APPLICATION_FORM_URLENCODED);
|
|
|
|
final MultiValueMap<String, String> map = new LinkedMultiValueMap<>();
|
|
map.add("userName", zeppelinLogin);
|
|
map.add("password", zeppelinPassword);
|
|
final HttpEntity<MultiValueMap<String, String>> request = new HttpEntity<>(map, headers);
|
|
|
|
final String url = zeppelinBaseUrl + "/api/login";
|
|
final ResponseEntity<?> res = new RestTemplate().postForEntity(url, request, Object.class);
|
|
|
|
if (res.getStatusCode() != HttpStatus.OK) {
|
|
log.error("Zeppelin API: login failed with HTTP error: " + res);
|
|
throw new MDStoreManagerException("Zeppelin API: login failed with HTTP error: " + res);
|
|
} else if (!res.getHeaders().containsKey(HttpHeaders.SET_COOKIE)) {
|
|
log.error("Zeppelin API: login failed (missing SET_COOKIE header)");
|
|
throw new MDStoreManagerException("Zeppelin API: login failed (missing SET_COOKIE header)");
|
|
} else {
|
|
return res.getHeaders()
|
|
.get(HttpHeaders.SET_COOKIE)
|
|
.stream()
|
|
.map(s -> s.split(";"))
|
|
.flatMap(Arrays::stream)
|
|
.map(String::trim)
|
|
.filter(s -> s.startsWith("JSESSIONID="))
|
|
.map(s -> StringUtils.removeStart(s, "JSESSIONID="))
|
|
.filter(s -> !s.equalsIgnoreCase("deleteMe"))
|
|
.distinct()
|
|
.filter(this::testConnection)
|
|
.findFirst()
|
|
.orElseThrow(() -> new MDStoreManagerException("Zeppelin API: login failed (invalid jsessionid)"));
|
|
}
|
|
}
|
|
|
|
private boolean testConnection(final String jsessionid) {
|
|
|
|
final String url = zeppelinBaseUrl + "/api/notebook;JSESSIONID=" + jsessionid;
|
|
log.info("Performing GET: " + url);
|
|
|
|
final ResponseEntity<ListResponse> res = new RestTemplate().getForEntity(url, ListResponse.class);
|
|
|
|
if (res.getStatusCode() != HttpStatus.OK) {
|
|
return false;
|
|
} else if (res.getBody() == null) {
|
|
return false;
|
|
} else if (!res.getBody().getStatus().equals("OK")) {
|
|
return false;
|
|
} else {
|
|
log.info("Connected to zeppelin: " + res.getBody());
|
|
log.info("Found JSESSIONID: " + jsessionid);
|
|
return true;
|
|
}
|
|
}
|
|
|
|
private List<Map<String, String>> listNotes(final String jsessionid) throws MDStoreManagerException {
|
|
final String url = zeppelinBaseUrl + "/api/notebook;JSESSIONID=" + jsessionid;
|
|
log.info("Performing GET: " + url);
|
|
|
|
final ResponseEntity<ListResponse> res = new RestTemplate().getForEntity(url, ListResponse.class);
|
|
|
|
if (res.getStatusCode() != HttpStatus.OK) {
|
|
log.error("Zeppelin API failed with HTTP error: " + res);
|
|
throw new MDStoreManagerException("Zeppelin API failed with HTTP error: " + res);
|
|
} else if (res.getBody() == null) {
|
|
log.error("Zeppelin API returned a null response");
|
|
throw new MDStoreManagerException("Zeppelin API returned a null response");
|
|
} else if (!res.getBody().getStatus().equals("OK")) {
|
|
log.error("Registration of zeppelin note failed: " + res.getBody());
|
|
throw new MDStoreManagerException("Registration of zeppelin note failed: " + res.getBody());
|
|
} else {
|
|
return res.getBody().getBody();
|
|
}
|
|
}
|
|
|
|
private String findTemplateNoteId(final String noteTemplate, final String jsessionid) throws MDStoreManagerException {
|
|
final String templateName = zeppelinNamePrefix + "/templates/" + noteTemplate;
|
|
|
|
return listNotes(jsessionid).stream()
|
|
.filter(map -> map.get("name").equals(templateName))
|
|
.map(map -> map.get("id"))
|
|
.findFirst()
|
|
.orElseThrow(() -> new MDStoreManagerException("Template Note not found: " + templateName));
|
|
}
|
|
|
|
private String cloneNote(final String noteId, final String newName, final String jsessionid) throws MDStoreManagerException {
|
|
final String url = zeppelinBaseUrl + "/api/notebook/" + noteId + ";JSESSIONID=" + jsessionid;
|
|
log.debug("Performing POST: " + url);
|
|
|
|
final ResponseEntity<StringResponse> res = new RestTemplate().postForEntity(url, new Note(newName), StringResponse.class);
|
|
|
|
if (res.getStatusCode() != HttpStatus.OK) {
|
|
log.error("Zeppelin API failed with HTTP error: " + res);
|
|
throw new MDStoreManagerException("Zeppelin API failed with HTTP error: " + res);
|
|
} else if (res.getBody() == null) {
|
|
log.error("Zeppelin API returned a null response");
|
|
throw new MDStoreManagerException("Zeppelin API returned a null response");
|
|
} else if (!res.getBody().getStatus().equals("OK")) {
|
|
log.error("Registration of zeppelin note failed: " + res.getBody());
|
|
throw new MDStoreManagerException("Registration of zeppelin note failed: " + res.getBody());
|
|
} else {
|
|
return res.getBody().getBody();
|
|
}
|
|
}
|
|
|
|
private Paragraph confParagraph(final String mdId, final String currentVersion, final String currentVersionPath) throws MDStoreManagerException {
|
|
try {
|
|
final String code = IOUtils.toString(getClass().getResourceAsStream("/zeppelin/conf.tmpl.py"))
|
|
.replaceAll("__MDSTORE_ID__", mdId)
|
|
.replaceAll("__VERSION__", currentVersion)
|
|
.replaceAll("__PATH__", currentVersionPath);
|
|
return new Paragraph("Configuration", code, 0);
|
|
} catch (final IOException e) {
|
|
log.error("Error preparing configuration paragraph", e);
|
|
throw new MDStoreManagerException("Error preparing configuration paragraph", e);
|
|
}
|
|
}
|
|
|
|
private String addParagraph(final String noteId, final Paragraph paragraph, final String jsessionid) throws MDStoreManagerException {
|
|
final String url = zeppelinBaseUrl + "/api/notebook/" + noteId + "/paragraph;JSESSIONID=" + jsessionid;
|
|
log.debug("Performing POST: " + url);
|
|
|
|
final ResponseEntity<StringResponse> res = new RestTemplate().postForEntity(url, paragraph, StringResponse.class);
|
|
|
|
if (res.getStatusCode() != HttpStatus.OK) {
|
|
log.error("Zeppelin API failed with HTTP error: " + res);
|
|
throw new MDStoreManagerException("Zeppelin API failed with HTTP error: " + res);
|
|
} else if (res.getBody() == null) {
|
|
log.error("Zeppelin API returned a null response");
|
|
throw new MDStoreManagerException("Zeppelin API returned a null response");
|
|
} else if (!res.getBody().getStatus().equals("OK")) {
|
|
log.error("Registration of zeppelin note failed: " + res.getBody());
|
|
throw new MDStoreManagerException("Registration of zeppelin note failed: " + res.getBody());
|
|
} else {
|
|
return res.getBody().getBody();
|
|
}
|
|
}
|
|
|
|
private void deleteNote(final String id, final String jsessionid) {
|
|
final String url = zeppelinBaseUrl + "/api/notebook/" + id + ";JSESSIONID=" + jsessionid;
|
|
log.debug("Performing DELETE: " + url);
|
|
new RestTemplate().delete(url);
|
|
}
|
|
|
|
private boolean isExpired(final String id, final String jsessionid) {
|
|
// TODO Auto-generated method stub
|
|
return false;
|
|
}
|
|
|
|
}
|