From b10bd1f385ed4ac187e7839057464dd1b5932d62 Mon Sep 17 00:00:00 2001 From: "michele.artini" Date: Fri, 29 Oct 2021 10:43:42 +0200 Subject: [PATCH] zeppelin notes --- .../manager/controller/MDStoreController.java | 10 +++--- .../controller/ZeppelinController.java | 6 ++-- .../mdstore/manager/utils/ZeppelinClient.java | 33 ++++++++++++++----- .../src/main/resources/zeppelin/conf.tmpl.py | 6 ++++ 4 files changed, 40 insertions(+), 15 deletions(-) diff --git a/apps/dhp-mdstore-manager/src/main/java/eu/dnetlib/data/mdstore/manager/controller/MDStoreController.java b/apps/dhp-mdstore-manager/src/main/java/eu/dnetlib/data/mdstore/manager/controller/MDStoreController.java index 1803542d..bf5ee22f 100644 --- a/apps/dhp-mdstore-manager/src/main/java/eu/dnetlib/data/mdstore/manager/controller/MDStoreController.java +++ b/apps/dhp-mdstore-manager/src/main/java/eu/dnetlib/data/mdstore/manager/controller/MDStoreController.java @@ -18,11 +18,11 @@ import org.springframework.web.bind.annotation.RestController; import com.google.common.collect.Sets; import eu.dnetlib.common.controller.AbstractDnetController; -import eu.dnetlib.dhp.schema.mdstore.MDStoreVersion; -import eu.dnetlib.dhp.schema.mdstore.MDStoreWithInfo; import eu.dnetlib.data.mdstore.manager.exceptions.MDStoreManagerException; import eu.dnetlib.data.mdstore.manager.utils.DatabaseUtils; import eu.dnetlib.data.mdstore.manager.utils.HdfsClient; +import eu.dnetlib.dhp.schema.mdstore.MDStoreVersion; +import eu.dnetlib.dhp.schema.mdstore.MDStoreWithInfo; import io.swagger.annotations.Api; import io.swagger.annotations.ApiOperation; import io.swagger.annotations.ApiParam; @@ -72,9 +72,9 @@ public class MDStoreController extends AbstractDnetController { @ApiParam("mdstore format") @PathVariable final String format, @ApiParam("mdstore layout") @PathVariable final String layout, @ApiParam("mdstore interpretation") @PathVariable final String interpretation, - @ApiParam("datasource name") @RequestParam(required = false) final String dsName, - @ApiParam("datasource id") @RequestParam(required = false) final String dsId, - @ApiParam("api id") @RequestParam(required = false) final String apiId) throws MDStoreManagerException { + @ApiParam("datasource name") @RequestParam(required = true) final String dsName, + @ApiParam("datasource id") @RequestParam(required = true) final String dsId, + @ApiParam("api id") @RequestParam(required = true) final String apiId) throws MDStoreManagerException { final String id = databaseUtils.createMDStore(format, layout, interpretation, dsName, dsId, apiId); return databaseUtils.findMdStore(id); } diff --git a/apps/dhp-mdstore-manager/src/main/java/eu/dnetlib/data/mdstore/manager/controller/ZeppelinController.java b/apps/dhp-mdstore-manager/src/main/java/eu/dnetlib/data/mdstore/manager/controller/ZeppelinController.java index 4956a71b..042bd38a 100644 --- a/apps/dhp-mdstore-manager/src/main/java/eu/dnetlib/data/mdstore/manager/controller/ZeppelinController.java +++ b/apps/dhp-mdstore-manager/src/main/java/eu/dnetlib/data/mdstore/manager/controller/ZeppelinController.java @@ -13,6 +13,7 @@ import eu.dnetlib.data.mdstore.manager.exceptions.MDStoreManagerException; import eu.dnetlib.data.mdstore.manager.utils.ControllerUtils; import eu.dnetlib.data.mdstore.manager.utils.DatabaseUtils; import eu.dnetlib.data.mdstore.manager.utils.ZeppelinClient; +import eu.dnetlib.dhp.schema.mdstore.MDStoreWithInfo; @Controller @RequestMapping("/zeppelin") @@ -26,9 +27,10 @@ public class ZeppelinController { @RequestMapping("/{mdId}/{note}") public String goToZeppelin(@PathVariable final String mdId, final @PathVariable String note) throws MDStoreManagerException { - final String currentVersion = databaseUtils.findMdStore(mdId).getCurrentVersion(); + final MDStoreWithInfo mdstore = databaseUtils.findMdStore(mdId); + final String currentVersion = mdstore.getCurrentVersion(); final String path = databaseUtils.findVersion(currentVersion).getHdfsPath() + "/store"; - return "redirect:" + zeppelinClient.zeppelinNote(note, mdId, currentVersion, path); + return "redirect:" + zeppelinClient.zeppelinNote(note, mdstore, path); } @ExceptionHandler(Exception.class) diff --git a/apps/dhp-mdstore-manager/src/main/java/eu/dnetlib/data/mdstore/manager/utils/ZeppelinClient.java b/apps/dhp-mdstore-manager/src/main/java/eu/dnetlib/data/mdstore/manager/utils/ZeppelinClient.java index 83e5afa4..e68ac1af 100644 --- a/apps/dhp-mdstore-manager/src/main/java/eu/dnetlib/data/mdstore/manager/utils/ZeppelinClient.java +++ b/apps/dhp-mdstore-manager/src/main/java/eu/dnetlib/data/mdstore/manager/utils/ZeppelinClient.java @@ -1,6 +1,7 @@ package eu.dnetlib.data.mdstore.manager.utils; import java.io.IOException; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Arrays; import java.util.LinkedHashMap; @@ -10,6 +11,7 @@ import java.util.Objects; import java.util.Optional; import org.apache.commons.io.IOUtils; +import org.apache.commons.lang.StringEscapeUtils; import org.apache.commons.lang3.StringUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -29,6 +31,7 @@ import eu.dnetlib.data.mdstore.manager.utils.zeppelin.ListResponse; import eu.dnetlib.data.mdstore.manager.utils.zeppelin.Note; import eu.dnetlib.data.mdstore.manager.utils.zeppelin.Paragraph; import eu.dnetlib.data.mdstore.manager.utils.zeppelin.StringResponse; +import eu.dnetlib.dhp.schema.mdstore.MDStoreWithInfo; @Component public class ZeppelinClient { @@ -47,11 +50,14 @@ public class ZeppelinClient { private static final Log log = LogFactory.getLog(ZeppelinClient.class); - public String zeppelinNote(final String note, final String mdId, final String currentVersion, final String currentVersionPath) - throws MDStoreManagerException { + private String jsessionid; + + public String zeppelinNote(final String note, final MDStoreWithInfo mdstore, final String currentVersionPath) throws MDStoreManagerException { final String jsessionid = obtainJsessionID(); - final String newName = zeppelinNamePrefix + "/notes/" + note + "/" + currentVersion; + final String newName = + StringUtils.join(Arrays.asList(zeppelinNamePrefix, "notes", mdstore.getDatasourceName().replaceAll("/", "-"), mdstore.getApiId() + .replaceAll("/", "-"), note.replaceAll("/", "-"), mdstore.getCurrentVersion().replaceAll("/", "-")), "/"); final Optional oldNoteId = listNotes(jsessionid).stream() .filter(Objects::nonNull) @@ -70,7 +76,7 @@ public class ZeppelinClient { log.info("New note created, id: " + newId + ", name: " + newName); - addParagraph(newId, confParagraph(mdId, currentVersion, currentVersionPath), jsessionid); + addParagraph(newId, confParagraph(mdstore, currentVersionPath), jsessionid); reassignRights(newId, jsessionid); @@ -200,11 +206,14 @@ public class ZeppelinClient { } } - private Paragraph confParagraph(final String mdId, final String currentVersion, final String currentVersionPath) throws MDStoreManagerException { + private Paragraph confParagraph(final MDStoreWithInfo mdstore, final String currentVersionPath) throws MDStoreManagerException { try { - final String code = IOUtils.toString(getClass().getResourceAsStream("/zeppelin/conf.tmpl.py")) - .replaceAll("__MDSTORE_ID__", mdId) - .replaceAll("__VERSION__", currentVersion) + final String code = IOUtils.toString(getClass().getResourceAsStream("/zeppelin/conf.tmpl.py"), StandardCharsets.UTF_8) + .replaceAll("__DS_NAME__", StringEscapeUtils.escapeJava(mdstore.getDatasourceName())) + .replaceAll("__DS_ID__", StringEscapeUtils.escapeJava(mdstore.getDatasourceId())) + .replaceAll("__API_ID__", StringEscapeUtils.escapeJava(mdstore.getApiId())) + .replaceAll("__MDSTORE_ID__", mdstore.getId()) + .replaceAll("__VERSION__", mdstore.getCurrentVersion()) .replaceAll("__PATH__", currentVersionPath); return new Paragraph("Configuration", code, 0); } catch (final IOException e) { @@ -257,4 +266,12 @@ public class ZeppelinClient { return false; } + public String getJsessionid() { + return jsessionid; + } + + public void setJsessionid(final String jsessionid) { + this.jsessionid = jsessionid; + } + } diff --git a/apps/dhp-mdstore-manager/src/main/resources/zeppelin/conf.tmpl.py b/apps/dhp-mdstore-manager/src/main/resources/zeppelin/conf.tmpl.py index a1f8bc8f..6907e5ab 100644 --- a/apps/dhp-mdstore-manager/src/main/resources/zeppelin/conf.tmpl.py +++ b/apps/dhp-mdstore-manager/src/main/resources/zeppelin/conf.tmpl.py @@ -1,9 +1,15 @@ %pyspark +dsName = "__DS_NAME__" +dsId = "__DS_ID__" +apiId = "__API_ID__" mdId = "__MDSTORE_ID__" mdVersion = "__VERSION__" path = "__PATH__" +print "Datasource Name:", dsName +print "Datasource ID:", dsId +print "Api ID:", apiId print "MdStore ID:", mdId print "Version ID:", mdVersion print "Version Data Path:", path