zeppelin notes

This commit is contained in:
Michele Artini 2021-10-29 10:43:42 +02:00
parent 51bfb5866b
commit b10bd1f385
4 changed files with 40 additions and 15 deletions

View File

@ -18,11 +18,11 @@ import org.springframework.web.bind.annotation.RestController;
import com.google.common.collect.Sets; import com.google.common.collect.Sets;
import eu.dnetlib.common.controller.AbstractDnetController; import eu.dnetlib.common.controller.AbstractDnetController;
import eu.dnetlib.dhp.schema.mdstore.MDStoreVersion;
import eu.dnetlib.dhp.schema.mdstore.MDStoreWithInfo;
import eu.dnetlib.data.mdstore.manager.exceptions.MDStoreManagerException; import eu.dnetlib.data.mdstore.manager.exceptions.MDStoreManagerException;
import eu.dnetlib.data.mdstore.manager.utils.DatabaseUtils; import eu.dnetlib.data.mdstore.manager.utils.DatabaseUtils;
import eu.dnetlib.data.mdstore.manager.utils.HdfsClient; import eu.dnetlib.data.mdstore.manager.utils.HdfsClient;
import eu.dnetlib.dhp.schema.mdstore.MDStoreVersion;
import eu.dnetlib.dhp.schema.mdstore.MDStoreWithInfo;
import io.swagger.annotations.Api; import io.swagger.annotations.Api;
import io.swagger.annotations.ApiOperation; import io.swagger.annotations.ApiOperation;
import io.swagger.annotations.ApiParam; import io.swagger.annotations.ApiParam;
@ -72,9 +72,9 @@ public class MDStoreController extends AbstractDnetController {
@ApiParam("mdstore format") @PathVariable final String format, @ApiParam("mdstore format") @PathVariable final String format,
@ApiParam("mdstore layout") @PathVariable final String layout, @ApiParam("mdstore layout") @PathVariable final String layout,
@ApiParam("mdstore interpretation") @PathVariable final String interpretation, @ApiParam("mdstore interpretation") @PathVariable final String interpretation,
@ApiParam("datasource name") @RequestParam(required = false) final String dsName, @ApiParam("datasource name") @RequestParam(required = true) final String dsName,
@ApiParam("datasource id") @RequestParam(required = false) final String dsId, @ApiParam("datasource id") @RequestParam(required = true) final String dsId,
@ApiParam("api id") @RequestParam(required = false) final String apiId) throws MDStoreManagerException { @ApiParam("api id") @RequestParam(required = true) final String apiId) throws MDStoreManagerException {
final String id = databaseUtils.createMDStore(format, layout, interpretation, dsName, dsId, apiId); final String id = databaseUtils.createMDStore(format, layout, interpretation, dsName, dsId, apiId);
return databaseUtils.findMdStore(id); return databaseUtils.findMdStore(id);
} }

View File

@ -13,6 +13,7 @@ import eu.dnetlib.data.mdstore.manager.exceptions.MDStoreManagerException;
import eu.dnetlib.data.mdstore.manager.utils.ControllerUtils; import eu.dnetlib.data.mdstore.manager.utils.ControllerUtils;
import eu.dnetlib.data.mdstore.manager.utils.DatabaseUtils; import eu.dnetlib.data.mdstore.manager.utils.DatabaseUtils;
import eu.dnetlib.data.mdstore.manager.utils.ZeppelinClient; import eu.dnetlib.data.mdstore.manager.utils.ZeppelinClient;
import eu.dnetlib.dhp.schema.mdstore.MDStoreWithInfo;
@Controller @Controller
@RequestMapping("/zeppelin") @RequestMapping("/zeppelin")
@ -26,9 +27,10 @@ public class ZeppelinController {
@RequestMapping("/{mdId}/{note}") @RequestMapping("/{mdId}/{note}")
public String goToZeppelin(@PathVariable final String mdId, final @PathVariable String note) throws MDStoreManagerException { public String goToZeppelin(@PathVariable final String mdId, final @PathVariable String note) throws MDStoreManagerException {
final String currentVersion = databaseUtils.findMdStore(mdId).getCurrentVersion(); final MDStoreWithInfo mdstore = databaseUtils.findMdStore(mdId);
final String currentVersion = mdstore.getCurrentVersion();
final String path = databaseUtils.findVersion(currentVersion).getHdfsPath() + "/store"; final String path = databaseUtils.findVersion(currentVersion).getHdfsPath() + "/store";
return "redirect:" + zeppelinClient.zeppelinNote(note, mdId, currentVersion, path); return "redirect:" + zeppelinClient.zeppelinNote(note, mdstore, path);
} }
@ExceptionHandler(Exception.class) @ExceptionHandler(Exception.class)

View File

@ -1,6 +1,7 @@
package eu.dnetlib.data.mdstore.manager.utils; package eu.dnetlib.data.mdstore.manager.utils;
import java.io.IOException; import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Arrays; import java.util.Arrays;
import java.util.LinkedHashMap; import java.util.LinkedHashMap;
@ -10,6 +11,7 @@ import java.util.Objects;
import java.util.Optional; import java.util.Optional;
import org.apache.commons.io.IOUtils; import org.apache.commons.io.IOUtils;
import org.apache.commons.lang.StringEscapeUtils;
import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.StringUtils;
import org.apache.commons.logging.Log; import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory; import org.apache.commons.logging.LogFactory;
@ -29,6 +31,7 @@ import eu.dnetlib.data.mdstore.manager.utils.zeppelin.ListResponse;
import eu.dnetlib.data.mdstore.manager.utils.zeppelin.Note; import eu.dnetlib.data.mdstore.manager.utils.zeppelin.Note;
import eu.dnetlib.data.mdstore.manager.utils.zeppelin.Paragraph; import eu.dnetlib.data.mdstore.manager.utils.zeppelin.Paragraph;
import eu.dnetlib.data.mdstore.manager.utils.zeppelin.StringResponse; import eu.dnetlib.data.mdstore.manager.utils.zeppelin.StringResponse;
import eu.dnetlib.dhp.schema.mdstore.MDStoreWithInfo;
@Component @Component
public class ZeppelinClient { public class ZeppelinClient {
@ -47,11 +50,14 @@ public class ZeppelinClient {
private static final Log log = LogFactory.getLog(ZeppelinClient.class); private static final Log log = LogFactory.getLog(ZeppelinClient.class);
public String zeppelinNote(final String note, final String mdId, final String currentVersion, final String currentVersionPath) private String jsessionid;
throws MDStoreManagerException {
public String zeppelinNote(final String note, final MDStoreWithInfo mdstore, final String currentVersionPath) throws MDStoreManagerException {
final String jsessionid = obtainJsessionID(); final String jsessionid = obtainJsessionID();
final String newName = zeppelinNamePrefix + "/notes/" + note + "/" + currentVersion; final String newName =
StringUtils.join(Arrays.asList(zeppelinNamePrefix, "notes", mdstore.getDatasourceName().replaceAll("/", "-"), mdstore.getApiId()
.replaceAll("/", "-"), note.replaceAll("/", "-"), mdstore.getCurrentVersion().replaceAll("/", "-")), "/");
final Optional<String> oldNoteId = listNotes(jsessionid).stream() final Optional<String> oldNoteId = listNotes(jsessionid).stream()
.filter(Objects::nonNull) .filter(Objects::nonNull)
@ -70,7 +76,7 @@ public class ZeppelinClient {
log.info("New note created, id: " + newId + ", name: " + newName); log.info("New note created, id: " + newId + ", name: " + newName);
addParagraph(newId, confParagraph(mdId, currentVersion, currentVersionPath), jsessionid); addParagraph(newId, confParagraph(mdstore, currentVersionPath), jsessionid);
reassignRights(newId, jsessionid); reassignRights(newId, jsessionid);
@ -200,11 +206,14 @@ public class ZeppelinClient {
} }
} }
private Paragraph confParagraph(final String mdId, final String currentVersion, final String currentVersionPath) throws MDStoreManagerException { private Paragraph confParagraph(final MDStoreWithInfo mdstore, final String currentVersionPath) throws MDStoreManagerException {
try { try {
final String code = IOUtils.toString(getClass().getResourceAsStream("/zeppelin/conf.tmpl.py")) final String code = IOUtils.toString(getClass().getResourceAsStream("/zeppelin/conf.tmpl.py"), StandardCharsets.UTF_8)
.replaceAll("__MDSTORE_ID__", mdId) .replaceAll("__DS_NAME__", StringEscapeUtils.escapeJava(mdstore.getDatasourceName()))
.replaceAll("__VERSION__", currentVersion) .replaceAll("__DS_ID__", StringEscapeUtils.escapeJava(mdstore.getDatasourceId()))
.replaceAll("__API_ID__", StringEscapeUtils.escapeJava(mdstore.getApiId()))
.replaceAll("__MDSTORE_ID__", mdstore.getId())
.replaceAll("__VERSION__", mdstore.getCurrentVersion())
.replaceAll("__PATH__", currentVersionPath); .replaceAll("__PATH__", currentVersionPath);
return new Paragraph("Configuration", code, 0); return new Paragraph("Configuration", code, 0);
} catch (final IOException e) { } catch (final IOException e) {
@ -257,4 +266,12 @@ public class ZeppelinClient {
return false; return false;
} }
public String getJsessionid() {
return jsessionid;
}
public void setJsessionid(final String jsessionid) {
this.jsessionid = jsessionid;
}
} }

View File

@ -1,9 +1,15 @@
%pyspark %pyspark
dsName = "__DS_NAME__"
dsId = "__DS_ID__"
apiId = "__API_ID__"
mdId = "__MDSTORE_ID__" mdId = "__MDSTORE_ID__"
mdVersion = "__VERSION__" mdVersion = "__VERSION__"
path = "__PATH__" path = "__PATH__"
print "Datasource Name:", dsName
print "Datasource ID:", dsId
print "Api ID:", apiId
print "MdStore ID:", mdId print "MdStore ID:", mdId
print "Version ID:", mdVersion print "Version ID:", mdVersion
print "Version Data Path:", path print "Version Data Path:", path