forked from D-Net/dnet-hadoop
137 lines
5.0 KiB
Java
137 lines
5.0 KiB
Java
|
|
package eu.dnetlib.dhp.aggregation.mdstore;
|
|
|
|
import static eu.dnetlib.dhp.common.Constants.*;
|
|
import static eu.dnetlib.dhp.utils.DHPUtils.*;
|
|
|
|
import java.net.URI;
|
|
|
|
import org.apache.commons.io.IOUtils;
|
|
import org.apache.commons.lang3.StringUtils;
|
|
import org.apache.hadoop.fs.FSDataInputStream;
|
|
import org.apache.hadoop.fs.FileSystem;
|
|
import org.apache.hadoop.fs.Path;
|
|
import org.slf4j.Logger;
|
|
import org.slf4j.LoggerFactory;
|
|
|
|
import eu.dnetlib.dhp.application.ArgumentApplicationParser;
|
|
import eu.dnetlib.dhp.common.rest.DNetRestClient;
|
|
import eu.dnetlib.dhp.schema.mdstore.MDStoreVersion;
|
|
|
|
public class MDStoreActionNode {
|
|
private static final Logger log = LoggerFactory.getLogger(MDStoreActionNode.class);
|
|
|
|
enum MDAction {
|
|
NEW_VERSION, ROLLBACK, COMMIT, READ_LOCK, READ_UNLOCK
|
|
}
|
|
|
|
public static String NEW_VERSION_URI = "%s/mdstore/%s/newVersion";
|
|
|
|
public static final String COMMIT_VERSION_URL = "%s/version/%s/commit/%s";
|
|
public static final String ROLLBACK_VERSION_URL = "%s/version/%s/abort";
|
|
|
|
public static final String READ_LOCK_URL = "%s/mdstore/%s/startReading";
|
|
public static final String READ_UNLOCK_URL = "%s/version/%s/endReading";
|
|
|
|
private static final String MDSTOREVERSIONPARAM = "mdStoreVersion";
|
|
private static final String MDSTOREREADLOCKPARAM = "mdStoreReadLockVersion";
|
|
|
|
public static void main(String[] args) throws Exception {
|
|
final ArgumentApplicationParser argumentParser = new ArgumentApplicationParser(
|
|
IOUtils
|
|
.toString(
|
|
MDStoreActionNode.class
|
|
.getResourceAsStream(
|
|
"/eu/dnetlib/dhp/collection/mdstore_action_parameters.json")));
|
|
argumentParser.parseArgument(args);
|
|
|
|
log.info("Java Xmx: {}m", Runtime.getRuntime().maxMemory() / (1024 * 1024));
|
|
|
|
final MDAction action = MDAction.valueOf(argumentParser.get("action"));
|
|
log.info("Current action is {}", action);
|
|
|
|
final String mdStoreManagerURI = argumentParser.get("mdStoreManagerURI");
|
|
log.info("mdStoreManagerURI is {}", mdStoreManagerURI);
|
|
|
|
switch (action) {
|
|
case NEW_VERSION: {
|
|
final String mdStoreID = argumentParser.get("mdStoreID");
|
|
if (StringUtils.isBlank(mdStoreID)) {
|
|
throw new IllegalArgumentException("missing or empty argument mdStoreId");
|
|
}
|
|
final MDStoreVersion currentVersion = DNetRestClient
|
|
.doGET(String.format(NEW_VERSION_URI, mdStoreManagerURI, mdStoreID), MDStoreVersion.class);
|
|
populateOOZIEEnv(MDSTOREVERSIONPARAM, MAPPER.writeValueAsString(currentVersion));
|
|
break;
|
|
}
|
|
case COMMIT: {
|
|
|
|
final String hdfsuri = argumentParser.get("namenode");
|
|
if (StringUtils.isBlank(hdfsuri)) {
|
|
throw new IllegalArgumentException("missing or empty argument namenode");
|
|
}
|
|
final String mdStoreVersion_params = argumentParser.get("mdStoreVersion");
|
|
final MDStoreVersion mdStoreVersion = MAPPER.readValue(mdStoreVersion_params, MDStoreVersion.class);
|
|
|
|
if (StringUtils.isBlank(mdStoreVersion.getId())) {
|
|
throw new IllegalArgumentException(
|
|
"invalid MDStoreVersion value current is " + mdStoreVersion_params);
|
|
}
|
|
Path hdfstoreSizepath = new Path(mdStoreVersion.getHdfsPath() + MDSTORE_SIZE_PATH);
|
|
|
|
try (
|
|
FileSystem fs = FileSystem.get(URI.create(hdfsuri), getHadoopConfiguration(hdfsuri));
|
|
FSDataInputStream inputStream = fs.open(hdfstoreSizepath)) {
|
|
|
|
final Long mdStoreSize = Long.parseLong(IOUtils.toString(inputStream));
|
|
|
|
fs.create(hdfstoreSizepath);
|
|
DNetRestClient
|
|
.doGET(
|
|
String.format(COMMIT_VERSION_URL, mdStoreManagerURI, mdStoreVersion.getId(), mdStoreSize));
|
|
}
|
|
|
|
break;
|
|
}
|
|
case ROLLBACK: {
|
|
final String mdStoreVersion_params = argumentParser.get("mdStoreVersion");
|
|
final MDStoreVersion mdStoreVersion = MAPPER.readValue(mdStoreVersion_params, MDStoreVersion.class);
|
|
|
|
if (StringUtils.isBlank(mdStoreVersion.getId())) {
|
|
throw new IllegalArgumentException(
|
|
"invalid MDStoreVersion value current is " + mdStoreVersion_params);
|
|
}
|
|
DNetRestClient.doGET(String.format(ROLLBACK_VERSION_URL, mdStoreManagerURI, mdStoreVersion.getId()));
|
|
break;
|
|
}
|
|
|
|
case READ_LOCK: {
|
|
final String mdStoreID = argumentParser.get("mdStoreID");
|
|
if (StringUtils.isBlank(mdStoreID)) {
|
|
throw new IllegalArgumentException("missing or empty argument mdStoreId");
|
|
}
|
|
final MDStoreVersion currentVersion = DNetRestClient
|
|
.doGET(String.format(READ_LOCK_URL, mdStoreManagerURI, mdStoreID), MDStoreVersion.class);
|
|
populateOOZIEEnv(MDSTOREREADLOCKPARAM, MAPPER.writeValueAsString(currentVersion));
|
|
break;
|
|
}
|
|
case READ_UNLOCK: {
|
|
final String mdStoreVersion_params = argumentParser.get("readMDStoreId");
|
|
final MDStoreVersion mdStoreVersion = MAPPER.readValue(mdStoreVersion_params, MDStoreVersion.class);
|
|
|
|
if (StringUtils.isBlank(mdStoreVersion.getId())) {
|
|
throw new IllegalArgumentException(
|
|
"invalid MDStoreVersion value current is " + mdStoreVersion_params);
|
|
}
|
|
DNetRestClient.doGET(String.format(READ_UNLOCK_URL, mdStoreManagerURI, mdStoreVersion.getId()));
|
|
break;
|
|
}
|
|
|
|
default:
|
|
throw new IllegalArgumentException("invalid action");
|
|
}
|
|
|
|
}
|
|
|
|
}
|