D-Net
/
dnet-applications
16
1
Fork 0
Code Issues Pull Requests Releases Wiki Activity

zeppelin notes

This commit is contained in:
Michele Artini 2021-10-29 10:43:42 +02:00
parent 51bfb5866b
commit b10bd1f385
4 changed files with 40 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
apps/dhp-mdstore-manager/src/main/java/eu/dnetlib/data/mdstore/manager/controller/MDStoreController.java
View File

@ -18,11 +18,11 @@ import org.springframework.web.bind.annotation.RestController;
import com.google.common.collect.Sets;
import eu.dnetlib.common.controller.AbstractDnetController;
import eu.dnetlib.dhp.schema.mdstore.MDStoreVersion;
import eu.dnetlib.dhp.schema.mdstore.MDStoreWithInfo;
import eu.dnetlib.data.mdstore.manager.exceptions.MDStoreManagerException;
import eu.dnetlib.data.mdstore.manager.utils.DatabaseUtils;
import eu.dnetlib.data.mdstore.manager.utils.HdfsClient;
import eu.dnetlib.dhp.schema.mdstore.MDStoreVersion;
import eu.dnetlib.dhp.schema.mdstore.MDStoreWithInfo;
import io.swagger.annotations.Api;
import io.swagger.annotations.ApiOperation;
import io.swagger.annotations.ApiParam;
@ -72,9 +72,9 @@ public class MDStoreController extends AbstractDnetController {
@ApiParam("mdstore format") @PathVariable final String format,
@ApiParam("mdstore layout") @PathVariable final String layout,
@ApiParam("mdstore interpretation") @PathVariable final String interpretation,
@ApiParam("datasource name") @RequestParam(required = false) final String dsName,
@ApiParam("datasource id") @RequestParam(required = false) final String dsId,
@ApiParam("api id") @RequestParam(required = false) final String apiId) throws MDStoreManagerException {
@ApiParam("datasource name") @RequestParam(required = true) final String dsName,
@ApiParam("datasource id") @RequestParam(required = true) final String dsId,
@ApiParam("api id") @RequestParam(required = true) final String apiId) throws MDStoreManagerException {
final String id = databaseUtils.createMDStore(format, layout, interpretation, dsName, dsId, apiId);
return databaseUtils.findMdStore(id);
}

6
apps/dhp-mdstore-manager/src/main/java/eu/dnetlib/data/mdstore/manager/controller/ZeppelinController.java
View File

@ -13,6 +13,7 @@ import eu.dnetlib.data.mdstore.manager.exceptions.MDStoreManagerException;
import eu.dnetlib.data.mdstore.manager.utils.ControllerUtils;
import eu.dnetlib.data.mdstore.manager.utils.DatabaseUtils;
import eu.dnetlib.data.mdstore.manager.utils.ZeppelinClient;
import eu.dnetlib.dhp.schema.mdstore.MDStoreWithInfo;
@Controller
@RequestMapping("/zeppelin")
@ -26,9 +27,10 @@ public class ZeppelinController {
@RequestMapping("/{mdId}/{note}")
public String goToZeppelin(@PathVariable final String mdId, final @PathVariable String note) throws MDStoreManagerException {
final String currentVersion = databaseUtils.findMdStore(mdId).getCurrentVersion();
final MDStoreWithInfo mdstore = databaseUtils.findMdStore(mdId);
final String currentVersion = mdstore.getCurrentVersion();
final String path = databaseUtils.findVersion(currentVersion).getHdfsPath() + "/store";
return "redirect:" + zeppelinClient.zeppelinNote(note, mdId, currentVersion, path);
return "redirect:" + zeppelinClient.zeppelinNote(note, mdstore, path);
}
@ExceptionHandler(Exception.class)

33
apps/dhp-mdstore-manager/src/main/java/eu/dnetlib/data/mdstore/manager/utils/ZeppelinClient.java
View File

@ -1,6 +1,7 @@
package eu.dnetlib.data.mdstore.manager.utils;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.LinkedHashMap;
@ -10,6 +11,7 @@ import java.util.Objects;
import java.util.Optional;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang.StringEscapeUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@ -29,6 +31,7 @@ import eu.dnetlib.data.mdstore.manager.utils.zeppelin.ListResponse;
import eu.dnetlib.data.mdstore.manager.utils.zeppelin.Note;
import eu.dnetlib.data.mdstore.manager.utils.zeppelin.Paragraph;
import eu.dnetlib.data.mdstore.manager.utils.zeppelin.StringResponse;
import eu.dnetlib.dhp.schema.mdstore.MDStoreWithInfo;
@Component
public class ZeppelinClient {
@ -47,11 +50,14 @@ public class ZeppelinClient {
private static final Log log = LogFactory.getLog(ZeppelinClient.class);
public String zeppelinNote(final String note, final String mdId, final String currentVersion, final String currentVersionPath)
throws MDStoreManagerException {
private String jsessionid;
public String zeppelinNote(final String note, final MDStoreWithInfo mdstore, final String currentVersionPath) throws MDStoreManagerException {
final String jsessionid = obtainJsessionID();
final String newName = zeppelinNamePrefix + "/notes/" + note + "/" + currentVersion;
final String newName =
StringUtils.join(Arrays.asList(zeppelinNamePrefix, "notes", mdstore.getDatasourceName().replaceAll("/", "-"), mdstore.getApiId()
.replaceAll("/", "-"), note.replaceAll("/", "-"), mdstore.getCurrentVersion().replaceAll("/", "-")), "/");
final Optional<String> oldNoteId = listNotes(jsessionid).stream()
.filter(Objects::nonNull)
@ -70,7 +76,7 @@ public class ZeppelinClient {
log.info("New note created, id: " + newId + ", name: " + newName);
addParagraph(newId, confParagraph(mdId, currentVersion, currentVersionPath), jsessionid);
addParagraph(newId, confParagraph(mdstore, currentVersionPath), jsessionid);
reassignRights(newId, jsessionid);
@ -200,11 +206,14 @@ public class ZeppelinClient {
}
}
private Paragraph confParagraph(final String mdId, final String currentVersion, final String currentVersionPath) throws MDStoreManagerException {
private Paragraph confParagraph(final MDStoreWithInfo mdstore, final String currentVersionPath) throws MDStoreManagerException {
try {
final String code = IOUtils.toString(getClass().getResourceAsStream("/zeppelin/conf.tmpl.py"))
.replaceAll("__MDSTORE_ID__", mdId)
.replaceAll("__VERSION__", currentVersion)
final String code = IOUtils.toString(getClass().getResourceAsStream("/zeppelin/conf.tmpl.py"), StandardCharsets.UTF_8)
.replaceAll("__DS_NAME__", StringEscapeUtils.escapeJava(mdstore.getDatasourceName()))
.replaceAll("__DS_ID__", StringEscapeUtils.escapeJava(mdstore.getDatasourceId()))
.replaceAll("__API_ID__", StringEscapeUtils.escapeJava(mdstore.getApiId()))
.replaceAll("__MDSTORE_ID__", mdstore.getId())
.replaceAll("__VERSION__", mdstore.getCurrentVersion())
.replaceAll("__PATH__", currentVersionPath);
return new Paragraph("Configuration", code, 0);
} catch (final IOException e) {
@ -257,4 +266,12 @@ public class ZeppelinClient {
return false;
}
public String getJsessionid() {
return jsessionid;
}
public void setJsessionid(final String jsessionid) {
this.jsessionid = jsessionid;
}
}

6
apps/dhp-mdstore-manager/src/main/resources/zeppelin/conf.tmpl.py
View File

@ -1,9 +1,15 @@
%pyspark
dsName = "__DS_NAME__"
dsId = "__DS_ID__"
apiId = "__API_ID__"
mdId = "__MDSTORE_ID__"
mdVersion = "__VERSION__"
path = "__PATH__"
print "Datasource Name:", dsName
print "Datasource ID:", dsId
print "Api ID:", apiId
print "MdStore ID:", mdId
print "Version ID:", mdVersion
print "Version Data Path:", path