106 lines
3.8 KiB
Python
106 lines
3.8 KiB
Python
|
import os
|
||
|
import argparse
|
||
|
import glob
|
||
|
from pathlib import Path
|
||
|
import re
|
||
|
import subprocess
|
||
|
|
||
|
regex = r"<\w*>(.*)<\/\w*>"
|
||
|
|
||
|
|
||
|
def get_jar_path():
|
||
|
for current_path in glob.glob("target/*.jar"):
|
||
|
return current_path
|
||
|
|
||
|
|
||
|
def extract_value(line):
|
||
|
matches = re.finditer(regex, line, re.MULTILINE)
|
||
|
for matchNum, match in enumerate(matches, start=1):
|
||
|
for groupNum in range(0, len(match.groups())):
|
||
|
return match.group(groupNum + 1)
|
||
|
|
||
|
|
||
|
def extract_dependencies():
|
||
|
with open("pom.xml") as f:
|
||
|
check_dependency = False
|
||
|
dep_to_add = []
|
||
|
for line in f:
|
||
|
if "<!-- JAR NEED -->" in line:
|
||
|
check_dependency = True
|
||
|
current_deps = {}
|
||
|
elif check_dependency:
|
||
|
if "groupId" in line:
|
||
|
current_deps["groupId"] = extract_value(line)
|
||
|
elif "artifactId" in line:
|
||
|
current_deps["artifactId"] = extract_value(line)
|
||
|
elif "version" in line:
|
||
|
current_deps["version"] = extract_value(line)
|
||
|
print("version", extract_value(line))
|
||
|
elif "</dependency>" in line:
|
||
|
dep_to_add.append(current_deps)
|
||
|
check_dependency = False
|
||
|
return dep_to_add
|
||
|
|
||
|
|
||
|
def to_path(dependency):
|
||
|
home_dir = Path.home()
|
||
|
base_path = os.path.join(home_dir, ".m2/repository/")
|
||
|
group_path = os.path.join(base_path, dependency["groupId"].replace(".", "/"))
|
||
|
data_path = os.path.join(group_path, dependency["artifactId"])
|
||
|
version_path = os.path.join(data_path, dependency["version"])
|
||
|
current_path = os.path.join(version_path, "{}-{}.jar".format(dependency["artifactId"], dependency["version"]))
|
||
|
if os.path.exists(current_path):
|
||
|
return "{}-{}.jar".format(dependency["artifactId"], dependency["version"]), current_path
|
||
|
return None,None
|
||
|
|
||
|
|
||
|
if __name__ == "__main__":
|
||
|
deps = extract_dependencies()
|
||
|
|
||
|
parser = argparse.ArgumentParser(
|
||
|
description="This scripts help you to publish, execute your check script in scala ")
|
||
|
parser.add_argument("user_name")
|
||
|
parser.add_argument("reference_class")
|
||
|
parser.add_argument("argument_file")
|
||
|
args = parser.parse_args()
|
||
|
|
||
|
print(f"reference_class = {args.reference_class} argument_file={args.argument_file}")
|
||
|
|
||
|
if os.path.exists(args.argument_file):
|
||
|
other_arguments = []
|
||
|
with open(args.argument_file) as f:
|
||
|
for line in f:
|
||
|
if len(line.strip()):
|
||
|
other_arguments.append(line.strip())
|
||
|
print("Cleaning Compile application ")
|
||
|
os.system('mvn clean compile package')
|
||
|
main_jar_path = get_jar_path()
|
||
|
if main_jar_path is None:
|
||
|
raise Exception("Unable to find the jar")
|
||
|
|
||
|
print("copy on your root folder")
|
||
|
|
||
|
os.system("ssh {}@iis-cdh5-test-gw.ocean.icm.edu.pl rm -rf sandro_nb".format(args.user_name))
|
||
|
|
||
|
os.system("ssh {}@iis-cdh5-test-gw.ocean.icm.edu.pl mkdir sandro_nb".format(args.user_name))
|
||
|
|
||
|
os.system(f"scp {main_jar_path} {args.user_name}@iis-cdh5-test-gw.ocean.icm.edu.pl:sandro_nb/")
|
||
|
jars = []
|
||
|
for item in deps:
|
||
|
name, p = to_path(item)
|
||
|
if p:
|
||
|
print(f"Copying dependencies {p} to lib deps")
|
||
|
os.system(f"scp {p} {args.user_name}@iis-cdh5-test-gw.ocean.icm.edu.pl:sandro_nb/")
|
||
|
jars.append(name)
|
||
|
|
||
|
j_name = ",".join(["sandro_nb/"+ item for item in jars])
|
||
|
name = main_jar_path.replace("target/", "")
|
||
|
command = f"spark2-submit --master yarn --jars {j_name} --class {args.reference_class} sandro_nb/{name}"
|
||
|
print(f"executing command {command}")
|
||
|
|
||
|
os.system(
|
||
|
"ssh {}@iis-cdh5-test-gw.ocean.icm.edu.pl {} {}".format(args.user_name, command, " ".join(other_arguments)))
|
||
|
|
||
|
else:
|
||
|
raise Exception(f"path not found {args.argument_file}")
|