From 81bef1295d72c9db3b6d1f578e053232af9f0db1 Mon Sep 17 00:00:00 2001 From: "michele.artini" Date: Thu, 28 Jan 2021 15:55:22 +0100 Subject: [PATCH] hdfsClient to delete mdstores and versions on hadoop --- apps/dhp-broker-public-application/pom.xml | 13 -- apps/dhp-mdstore-manager/pom.xml | 23 ++- .../manager/controller/MDStoreController.java | 30 +++- .../manager/controller/StatusResponse.java | 1 + .../mdstore/manager/utils/DatabaseUtils.java | 25 ++- .../mdstore/manager/utils/HdfsClient.java | 52 +++++++ .../resources/{ => hadoop}/GARR/core-site.xml | 0 .../GARR/garr-hadoop-conf.xml} | 0 .../main/resources/hadoop/OCEAN/core-site.xml | 145 ++++++++++++++++++ .../hadoop/OCEAN/ocean-hadoop-conf.xml | 101 ++++++++++++ .../src/main/resources/static/index.html | 9 +- .../resources/static/js/mdstoremanager.js | 11 +- pom.xml | 11 ++ 13 files changed, 379 insertions(+), 42 deletions(-) create mode 100644 apps/dhp-mdstore-manager/src/main/java/eu/dnetlib/data/mdstore/manager/utils/HdfsClient.java rename apps/dhp-mdstore-manager/src/main/resources/{ => hadoop}/GARR/core-site.xml (100%) rename apps/dhp-mdstore-manager/src/main/resources/{GARR/garr.xml => hadoop/GARR/garr-hadoop-conf.xml} (100%) create mode 100644 apps/dhp-mdstore-manager/src/main/resources/hadoop/OCEAN/core-site.xml create mode 100644 apps/dhp-mdstore-manager/src/main/resources/hadoop/OCEAN/ocean-hadoop-conf.xml diff --git a/apps/dhp-broker-public-application/pom.xml b/apps/dhp-broker-public-application/pom.xml index 34052933..8f9823dd 100644 --- a/apps/dhp-broker-public-application/pom.xml +++ b/apps/dhp-broker-public-application/pom.xml @@ -43,19 +43,6 @@ - - - - true - - - false - - cloudera - Cloudera Repository - https://repository.cloudera.com/artifactory/cloudera-repos - - diff --git a/apps/dhp-mdstore-manager/pom.xml b/apps/dhp-mdstore-manager/pom.xml index 1f5598ee..6be1e535 100644 --- a/apps/dhp-mdstore-manager/pom.xml +++ b/apps/dhp-mdstore-manager/pom.xml @@ -37,7 +37,28 @@ commons-io - + + + org.apache.hadoop + hadoop-client + 2.6.0-cdh5.9.2 + + + org.slf4j + slf4j-log4j12 + + + javax.servlet + servlet-api + + + com.google.guava + guava + + + + + eu.dnetlib.dhp dhp-common diff --git a/apps/dhp-mdstore-manager/src/main/java/eu/dnetlib/data/mdstore/manager/controller/MDStoreController.java b/apps/dhp-mdstore-manager/src/main/java/eu/dnetlib/data/mdstore/manager/controller/MDStoreController.java index a66fad11..e04d7a7c 100644 --- a/apps/dhp-mdstore-manager/src/main/java/eu/dnetlib/data/mdstore/manager/controller/MDStoreController.java +++ b/apps/dhp-mdstore-manager/src/main/java/eu/dnetlib/data/mdstore/manager/controller/MDStoreController.java @@ -10,7 +10,6 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.web.bind.annotation.DeleteMapping; import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.PathVariable; -import org.springframework.web.bind.annotation.PostMapping; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.RestController; @@ -20,6 +19,7 @@ import eu.dnetlib.data.mdstore.manager.common.model.MDStoreVersion; import eu.dnetlib.data.mdstore.manager.common.model.MDStoreWithInfo; import eu.dnetlib.data.mdstore.manager.exceptions.MDStoreManagerException; import eu.dnetlib.data.mdstore.manager.utils.DatabaseUtils; +import eu.dnetlib.data.mdstore.manager.utils.HdfsClient; import io.swagger.annotations.Api; import io.swagger.annotations.ApiOperation; import io.swagger.annotations.ApiParam; @@ -34,6 +34,9 @@ public class MDStoreController extends AbstractDnetController { @Autowired private DatabaseUtils databaseUtils; + @Autowired + private HdfsClient hdfsClient; + private static final Logger log = LoggerFactory.getLogger(DatabaseUtils.class); @ApiOperation("Return all the mdstores") @@ -76,7 +79,8 @@ public class MDStoreController extends AbstractDnetController { @ApiOperation("Delete a mdstore by id") @DeleteMapping("/mdstore/{mdId}") public StatusResponse delete(@ApiParam("the id of the mdstore that will be deleted") @PathVariable final String mdId) throws MDStoreManagerException { - databaseUtils.deleteMdStore(mdId); + final String hdfsPath = databaseUtils.deleteMdStore(mdId); + hdfsClient.deletePath(hdfsPath); return StatusResponse.DELETED; } @@ -99,12 +103,21 @@ public class MDStoreController extends AbstractDnetController { return databaseUtils.commitMdStoreVersion(versionId, size); } + @ApiOperation("Abort a preliminary version") + @GetMapping("/version/{versionId}/abort") + public StatusResponse commitVersion(@ApiParam("the id of the version to abort") @PathVariable final String versionId) throws MDStoreManagerException { + final String hdfsPath = databaseUtils.deleteMdStoreVersion(versionId, true); + hdfsClient.deletePath(hdfsPath); + return StatusResponse.ABORTED; + } + @ApiOperation("Delete a mdstore version") @DeleteMapping("/version/{versionId}") public StatusResponse deleteVersion(@ApiParam("the id of the version that has to be deleted") @PathVariable final String versionId, @ApiParam("if true, the controls on writing and readcount values will be skipped") @RequestParam(required = false, defaultValue = "false") final boolean force) throws MDStoreManagerException { - databaseUtils.deleteMdStoreVersion(versionId, force); + final String hdfsPath = databaseUtils.deleteMdStoreVersion(versionId, force); + hdfsClient.deletePath(hdfsPath); return StatusResponse.DELETED; } @@ -123,14 +136,15 @@ public class MDStoreController extends AbstractDnetController { } @ApiOperation("Delete expired versions") - @PostMapping("/versions/expired") + @DeleteMapping("/versions/expired") public StatusResponse deleteExpiredVersions() { new Thread(() -> { - for (final String v : databaseUtils.listExpiredVersions()) { + for (final String versionId : databaseUtils.listExpiredVersions()) { try { - databaseUtils.deleteMdStoreVersion(v, true); + final String hdfsPath = databaseUtils.deleteMdStoreVersion(versionId, true); + hdfsClient.deletePath(hdfsPath); } catch (final MDStoreManagerException e) { - log.warn("Error deleteting version " + v); + log.warn("Error deleteting version " + versionId, e); } } }).start(); @@ -143,7 +157,7 @@ public class MDStoreController extends AbstractDnetController { public Map info() { final Map info = new LinkedHashMap<>(); info.put("number_of_mdstores", databaseUtils.countMdStores()); - info.put("hadoop_cluster", databaseUtils.getHadoopCluster()); + info.put("hadoop_cluster", hdfsClient.getHadoopCluster()); info.put("hdfs_base_path", databaseUtils.getHdfsBasePath()); info.put("expired_versions", databaseUtils.listExpiredVersions()); return info; diff --git a/apps/dhp-mdstore-manager/src/main/java/eu/dnetlib/data/mdstore/manager/controller/StatusResponse.java b/apps/dhp-mdstore-manager/src/main/java/eu/dnetlib/data/mdstore/manager/controller/StatusResponse.java index 36d8051b..5501674e 100644 --- a/apps/dhp-mdstore-manager/src/main/java/eu/dnetlib/data/mdstore/manager/controller/StatusResponse.java +++ b/apps/dhp-mdstore-manager/src/main/java/eu/dnetlib/data/mdstore/manager/controller/StatusResponse.java @@ -4,6 +4,7 @@ public class StatusResponse { public static final StatusResponse DELETED = new StatusResponse("DELETED"); public static final StatusResponse DELETING = new StatusResponse("DELETING..."); + public static final StatusResponse ABORTED = new StatusResponse("ABORTED");; private String status; diff --git a/apps/dhp-mdstore-manager/src/main/java/eu/dnetlib/data/mdstore/manager/utils/DatabaseUtils.java b/apps/dhp-mdstore-manager/src/main/java/eu/dnetlib/data/mdstore/manager/utils/DatabaseUtils.java index 16d12dfe..0a028111 100644 --- a/apps/dhp-mdstore-manager/src/main/java/eu/dnetlib/data/mdstore/manager/utils/DatabaseUtils.java +++ b/apps/dhp-mdstore-manager/src/main/java/eu/dnetlib/data/mdstore/manager/utils/DatabaseUtils.java @@ -2,6 +2,7 @@ package eu.dnetlib.data.mdstore.manager.utils; import java.util.Date; import java.util.List; +import java.util.Optional; import java.util.stream.Collectors; import javax.transaction.Transactional; @@ -40,9 +41,6 @@ public class DatabaseUtils { @Value("${dhp.mdstore-manager.hdfs.base-path}") private String hdfsBasePath; - @Value("${dhp.mdstore-manager.hadoop.cluster}") - private String hadoopCluster; - private static final Logger log = LoggerFactory.getLogger(DatabaseUtils.class); public Iterable listMdStores() { @@ -90,8 +88,11 @@ public class DatabaseUtils { } @Transactional - public void deleteMdStore(final String mdId) throws MDStoreManagerException { - if (!mdstoreRepository.existsById(mdId)) { + public String deleteMdStore(final String mdId) throws MDStoreManagerException { + + final Optional md = mdstoreRepository.findById(mdId); + + if (!md.isPresent()) { log.error("MDStore not found: " + mdId); throw new MDStoreManagerException("MDStore not found: " + mdId); } @@ -109,6 +110,8 @@ public class DatabaseUtils { mdstoreCurrentVersionRepository.deleteById(mdId); mdstoreVersionRepository.deleteByMdstore(mdId); mdstoreRepository.deleteById(mdId); + + return md.get().getHdfsPath(); } @Transactional @@ -155,7 +158,7 @@ public class DatabaseUtils { } @Transactional - public void deleteMdStoreVersion(final String versionId, final boolean force) throws MDStoreManagerException { + public String deleteMdStoreVersion(final String versionId, final boolean force) throws MDStoreManagerException { final MDStoreVersion v = mdstoreVersionRepository.findById(versionId).orElseThrow(() -> new MDStoreManagerException("Version not found")); @@ -170,6 +173,8 @@ public class DatabaseUtils { } mdstoreVersionRepository.delete(v); + + return v.getHdfsPath(); } public String getHdfsBasePath() { @@ -180,12 +185,4 @@ public class DatabaseUtils { this.hdfsBasePath = hdfsBasePath; } - public String getHadoopCluster() { - return hadoopCluster; - } - - public void setHadoopCluster(final String hadoopCluster) { - this.hadoopCluster = hadoopCluster; - } - } diff --git a/apps/dhp-mdstore-manager/src/main/java/eu/dnetlib/data/mdstore/manager/utils/HdfsClient.java b/apps/dhp-mdstore-manager/src/main/java/eu/dnetlib/data/mdstore/manager/utils/HdfsClient.java new file mode 100644 index 00000000..06a58fb7 --- /dev/null +++ b/apps/dhp-mdstore-manager/src/main/java/eu/dnetlib/data/mdstore/manager/utils/HdfsClient.java @@ -0,0 +1,52 @@ +package eu.dnetlib.data.mdstore.manager.utils; + +import java.io.IOException; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; + +import eu.dnetlib.data.mdstore.manager.exceptions.MDStoreManagerException; + +@Component +public class HdfsClient { + + @Value("${dhp.mdstore-manager.hadoop.cluster}") + private String hadoopCluster; + + private static final Log log = LogFactory.getLog(HdfsClient.class); + + public void deletePath(final String path) throws MDStoreManagerException { + final Configuration conf = new Configuration(); + + if (hadoopCluster.equalsIgnoreCase("OCEAN")) { + conf.addResource(getClass().getResourceAsStream("/hadoop/OCEAN/core-site.xml")); + conf.addResource(getClass().getResourceAsStream("/hadoop/OCEAN/ocean-hadoop-conf.xml")); + } else if (hadoopCluster.equalsIgnoreCase("GARR")) { + conf.addResource(getClass().getResourceAsStream("/hadoop/GARR/core-site.xml")); + conf.addResource(getClass().getResourceAsStream("/hadoop/GARR/garr-hadoop-conf.xml")); + } else { + log.error("Invalid Haddop Cluster: " + hadoopCluster); + throw new MDStoreManagerException("Invalid Haddop Cluster: " + hadoopCluster); + } + + try (final FileSystem fs = FileSystem.get(conf)) { + fs.delete(new Path(path), true); + } catch (IllegalArgumentException | IOException e) { + log.error("Eror deleting path: " + path, e); + throw new MDStoreManagerException("Eror deleting path: " + path, e); + } + } + + public String getHadoopCluster() { + return hadoopCluster; + } + + public void setHadoopCluster(final String hadoopCluster) { + this.hadoopCluster = hadoopCluster; + } +} diff --git a/apps/dhp-mdstore-manager/src/main/resources/GARR/core-site.xml b/apps/dhp-mdstore-manager/src/main/resources/hadoop/GARR/core-site.xml similarity index 100% rename from apps/dhp-mdstore-manager/src/main/resources/GARR/core-site.xml rename to apps/dhp-mdstore-manager/src/main/resources/hadoop/GARR/core-site.xml diff --git a/apps/dhp-mdstore-manager/src/main/resources/GARR/garr.xml b/apps/dhp-mdstore-manager/src/main/resources/hadoop/GARR/garr-hadoop-conf.xml similarity index 100% rename from apps/dhp-mdstore-manager/src/main/resources/GARR/garr.xml rename to apps/dhp-mdstore-manager/src/main/resources/hadoop/GARR/garr-hadoop-conf.xml diff --git a/apps/dhp-mdstore-manager/src/main/resources/hadoop/OCEAN/core-site.xml b/apps/dhp-mdstore-manager/src/main/resources/hadoop/OCEAN/core-site.xml new file mode 100644 index 00000000..e9fde9e7 --- /dev/null +++ b/apps/dhp-mdstore-manager/src/main/resources/hadoop/OCEAN/core-site.xml @@ -0,0 +1,145 @@ + + + + + + fs.defaultFS + hdfs://nameservice1 + + + fs.trash.interval + 1 + + + io.compression.codecs + org.apache.hadoop.io.compress.DefaultCodec,org.apache.hadoop.io.compress.GzipCodec,org.apache.hadoop.io.compress.BZip2Codec,org.apache.hadoop.io.compress.DeflateCodec,org.apache.hadoop.io.compress.SnappyCodec,org.apache.hadoop.io.compress.Lz4Codec + + + hadoop.security.authentication + simple + + + hadoop.security.authorization + false + + + hadoop.rpc.protection + authentication + + + hadoop.security.auth_to_local + DEFAULT + + + hadoop.proxyuser.oozie.hosts + * + + + hadoop.proxyuser.oozie.groups + * + + + hadoop.proxyuser.mapred.hosts + * + + + hadoop.proxyuser.mapred.groups + * + + + hadoop.proxyuser.flume.hosts + * + + + hadoop.proxyuser.flume.groups + * + + + hadoop.proxyuser.HTTP.hosts + * + + + hadoop.proxyuser.HTTP.groups + * + + + hadoop.proxyuser.hive.hosts + * + + + hadoop.proxyuser.hive.groups + * + + + hadoop.proxyuser.hue.hosts + * + + + hadoop.proxyuser.hue.groups + * + + + hadoop.proxyuser.httpfs.hosts + * + + + hadoop.proxyuser.httpfs.groups + * + + + hadoop.proxyuser.hdfs.groups + * + + + hadoop.proxyuser.hdfs.hosts + * + + + hadoop.proxyuser.yarn.hosts + * + + + hadoop.proxyuser.yarn.groups + * + + + hadoop.security.group.mapping + org.apache.hadoop.security.ShellBasedUnixGroupsMapping + + + hadoop.security.instrumentation.requires.admin + false + + + net.topology.script.file.name + /etc/hadoop/conf.cloudera.yarn2/topology.py + + + io.file.buffer.size + 65536 + + + hadoop.ssl.enabled + false + + + hadoop.ssl.require.client.cert + false + true + + + hadoop.ssl.keystores.factory.class + org.apache.hadoop.security.ssl.FileBasedKeyStoresFactory + true + + + hadoop.ssl.server.conf + ssl-server.xml + true + + + hadoop.ssl.client.conf + ssl-client.xml + true + + diff --git a/apps/dhp-mdstore-manager/src/main/resources/hadoop/OCEAN/ocean-hadoop-conf.xml b/apps/dhp-mdstore-manager/src/main/resources/hadoop/OCEAN/ocean-hadoop-conf.xml new file mode 100644 index 00000000..aa5388fa --- /dev/null +++ b/apps/dhp-mdstore-manager/src/main/resources/hadoop/OCEAN/ocean-hadoop-conf.xml @@ -0,0 +1,101 @@ + + + + + + dfs.nameservices + nameservice1 + + + dfs.client.failover.proxy.provider.nameservice1 + org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider + + + dfs.ha.automatic-failover.enabled.nameservice1 + true + + + ha.zookeeper.quorum + iis-cdh5-test-m1.ocean.icm.edu.pl:2181,iis-cdh5-test-m2.ocean.icm.edu.pl:2181,iis-cdh5-test-m3.ocean.icm.edu.pl:2181 + + + dfs.ha.namenodes.nameservice1 + namenode528,namenode434 + + + dfs.namenode.rpc-address.nameservice1.namenode528 + iis-cdh5-test-m1.ocean.icm.edu.pl:8020 + + + dfs.namenode.servicerpc-address.nameservice1.namenode528 + iis-cdh5-test-m1.ocean.icm.edu.pl:8022 + + + dfs.namenode.http-address.nameservice1.namenode528 + iis-cdh5-test-m1.ocean.icm.edu.pl:50070 + + + dfs.namenode.https-address.nameservice1.namenode528 + iis-cdh5-test-m1.ocean.icm.edu.pl:50470 + + + dfs.namenode.rpc-address.nameservice1.namenode434 + iis-cdh5-test-m2.ocean.icm.edu.pl:8020 + + + dfs.namenode.servicerpc-address.nameservice1.namenode434 + iis-cdh5-test-m2.ocean.icm.edu.pl:8022 + + + dfs.namenode.http-address.nameservice1.namenode434 + iis-cdh5-test-m2.ocean.icm.edu.pl:50070 + + + dfs.namenode.https-address.nameservice1.namenode434 + iis-cdh5-test-m2.ocean.icm.edu.pl:50470 + + + dfs.replication + 3 + + + dfs.blocksize + 134217728 + + + dfs.client.use.datanode.hostname + false + + + fs.permissions.umask-mode + 022 + + + dfs.namenode.acls.enabled + false + + + dfs.client.use.legacy.blockreader + false + + + dfs.client.read.shortcircuit + false + + + dfs.domain.socket.path + /var/run/hdfs-sockets/dn + + + dfs.client.read.shortcircuit.skip.checksum + false + + + dfs.client.domain.socket.data.traffic + false + + + dfs.datanode.hdfs-blocks-metadata.enabled + true + + diff --git a/apps/dhp-mdstore-manager/src/main/resources/static/index.html b/apps/dhp-mdstore-manager/src/main/resources/static/index.html index 8802dd1d..04893b3d 100644 --- a/apps/dhp-mdstore-manager/src/main/resources/static/index.html +++ b/apps/dhp-mdstore-manager/src/main/resources/static/index.html @@ -153,7 +153,10 @@ {{v.id}}
- Path: {{v.hdfsPath}} + Path: {{v.hdfsPath}}
+ + + {{v.readCount}} @@ -161,10 +164,6 @@ {{v.lastUpdate | date:"MMM dd, yyyy 'at' HH:mm"}} {{v.size}} - - - - diff --git a/apps/dhp-mdstore-manager/src/main/resources/static/js/mdstoremanager.js b/apps/dhp-mdstore-manager/src/main/resources/static/js/mdstoremanager.js index 234e9e73..0a79fb5d 100644 --- a/apps/dhp-mdstore-manager/src/main/resources/static/js/mdstoremanager.js +++ b/apps/dhp-mdstore-manager/src/main/resources/static/js/mdstoremanager.js @@ -60,7 +60,16 @@ app.controller('mdstoreManagerController', function($scope, $http) { }); } }; - + + $scope.abortVersion = function(versionId) { + $http.get("/mdstores/version/" + versionId + "/abort?" + $.now()).success(function(data) { + $scope.reload(); + $scope.refreshVersions(); + }).error(function(err) { + alert('ERROR: ' + err.message); + }); + }; + $scope.resetReading = function(versionId) { $http.get("/mdstores/version/" + versionId + "/resetReading" + '?' + $.now()).success(function(data) { $scope.reload(); diff --git a/pom.xml b/pom.xml index 7e388e21..809db8a2 100644 --- a/pom.xml +++ b/pom.xml @@ -66,6 +66,17 @@ true + + + true + + + false + + cloudera + Cloudera Repository + https://repository.cloudera.com/artifactory/cloudera-repos +