From c1f9848953a056b2054e92d31f996b30b36aa0dc Mon Sep 17 00:00:00 2001 From: "miriam.baglioni" Date: Wed, 1 Mar 2023 15:29:11 +0100 Subject: [PATCH] [ECclassification] added new classes --- .../project/PrepareProgramme.java | 10 +- .../project/SparkAtomicActionJob.java | 9 +- ...tractProjects.java => ExtractFromZip.java} | 49 ++----- .../actionmanager/project/utils/ReadCSV.java | 7 +- .../project/utils/ReadProjects.java | 90 +++++++++++++ .../project/utils/ReadTopics.java | 92 +++++++++++++ .../project/utils/model/JsonTopic.java | 38 ++++++ ...s.json => extract_fromzip_parameters.json} | 8 +- .../project/oozie_app/download.sh | 3 + .../project/oozie_app/workflow.xml | 125 +++++++++++++----- .../project/read_parameters.json | 23 ++++ 11 files changed, 373 insertions(+), 81 deletions(-) rename dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/utils/{ExtractProjects.java => ExtractFromZip.java} (63%) create mode 100644 dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/utils/ReadProjects.java create mode 100644 dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/utils/ReadTopics.java create mode 100644 dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/utils/model/JsonTopic.java rename dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/project/{extract_project_parameters.json => extract_fromzip_parameters.json} (75%) create mode 100644 dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/project/oozie_app/download.sh create mode 100644 dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/project/read_parameters.json diff --git a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/PrepareProgramme.java b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/PrepareProgramme.java index 686b0fc7f..bb816a3a7 100644 --- a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/PrepareProgramme.java +++ b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/PrepareProgramme.java @@ -266,11 +266,15 @@ public class PrepareProgramme { String code = csvProgramme.getCode(); if (!code.endsWith(".") && !code.contains("Euratom") - && !code.equals("H2020-EC")) + && !code.equals("H2020-EC") && !code.equals("H2020") && + !code.equals("H2020-Topics")) code += "."; - csvProgramme.setClassification(map.get(code)._1()); - csvProgramme.setClassification_short(map.get(code)._2()); + if (map.containsKey(code)) { + csvProgramme.setClassification(map.get(code)._1()); + csvProgramme.setClassification_short(map.get(code)._2()); + } else + log.info("WARNING: No entry in map for code " + code); return csvProgramme; }).collect(); diff --git a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/SparkAtomicActionJob.java b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/SparkAtomicActionJob.java index cc1411b31..ee755d881 100644 --- a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/SparkAtomicActionJob.java +++ b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/SparkAtomicActionJob.java @@ -24,6 +24,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import eu.dnetlib.dhp.actionmanager.project.utils.model.CSVProgramme; import eu.dnetlib.dhp.actionmanager.project.utils.model.CSVProject; import eu.dnetlib.dhp.actionmanager.project.utils.model.EXCELTopic; +import eu.dnetlib.dhp.actionmanager.project.utils.model.JsonTopic; import eu.dnetlib.dhp.application.ArgumentApplicationParser; import eu.dnetlib.dhp.common.HdfsSupport; import eu.dnetlib.dhp.schema.action.AtomicAction; @@ -110,7 +111,7 @@ public class SparkAtomicActionJob { Dataset project = readPath(spark, projectPatH, CSVProject.class); Dataset programme = readPath(spark, programmePath, CSVProgramme.class); - Dataset topic = readPath(spark, topicPath, EXCELTopic.class); + Dataset topic = readPath(spark, topicPath, JsonTopic.class); Dataset aaproject = project .joinWith(programme, project.col("programme").equalTo(programme.col("code")), "left") @@ -144,9 +145,9 @@ public class SparkAtomicActionJob { .filter(Objects::nonNull); aaproject - .joinWith(topic, aaproject.col("h2020topiccode").equalTo(topic.col("code")), "left") - .map((MapFunction, Project>) p -> { - Optional op = Optional.ofNullable(p._2()); + .joinWith(topic, aaproject.col("id").equalTo(topic.col("projectId")), "left") + .map((MapFunction, Project>) p -> { + Optional op = Optional.ofNullable(p._2()); Project rp = p._1(); op.ifPresent(excelTopic -> rp.setH2020topicdescription(excelTopic.getTitle())); return rp; diff --git a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/utils/ExtractProjects.java b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/utils/ExtractFromZip.java similarity index 63% rename from dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/utils/ExtractProjects.java rename to dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/utils/ExtractFromZip.java index fd6bf3579..70686d4b0 100644 --- a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/utils/ExtractProjects.java +++ b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/utils/ExtractFromZip.java @@ -1,12 +1,13 @@ package eu.dnetlib.dhp.actionmanager.project.utils; -import java.io.*; +import java.io.BufferedWriter; +import java.io.IOException; +import java.io.OutputStreamWriter; +import java.io.Serializable; import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.List; -import java.util.Optional; -import java.util.zip.GZIPOutputStream; import java.util.zip.ZipEntry; import java.util.zip.ZipInputStream; @@ -30,7 +31,7 @@ import eu.dnetlib.dhp.application.ArgumentApplicationParser; * @author miriam.baglioni * @Date 28/02/23 */ -public class ExtractProjects implements Serializable { +public class ExtractFromZip implements Serializable { private static final Logger log = LoggerFactory.getLogger(PrepareProjects.class); private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); @@ -40,17 +41,17 @@ public class ExtractProjects implements Serializable { .toString( PrepareProjects.class .getResourceAsStream( - "/eu/dnetlib/dhp/actionmanager/project/extract_project_parameters.json")); + "/eu/dnetlib/dhp/actionmanager/project/extract_fromzip_parameters.json")); final ArgumentApplicationParser parser = new ArgumentApplicationParser(jsonConfiguration); parser.parseArgument(args); - final String projectPath = parser.get("projectPath"); - log.info("projectPath {}: ", projectPath); + final String inputPath = parser.get("inputPath"); + log.info("inputPath {}: ", inputPath); - final String workingPath = parser.get("workingPath"); - log.info("workingPath {}: ", workingPath); + final String outputPath = parser.get("outputPath"); + log.info("outputPath {}: ", outputPath); final String hdfsNameNode = parser.get("hdfsNameNode"); log.info("hdfsNameNode {}", hdfsNameNode); @@ -60,8 +61,8 @@ public class ExtractProjects implements Serializable { FileSystem fs = FileSystem.get(conf); - doExtract(projectPath, workingPath, fs); - readProjects(workingPath + "json/project.json", workingPath + "projects/h2020_projects_nld.json", fs); + doExtract(inputPath, outputPath, fs); + } private static void doExtract(String inputFile, String workingPath, FileSystem fileSystem) @@ -97,30 +98,4 @@ public class ExtractProjects implements Serializable { } - private static void readProjects(String inputFile, String workingPath, FileSystem fs) throws IOException { - Path hdfsreadpath = new Path(inputFile); - - FSDataInputStream inputStream = fs.open(hdfsreadpath); - - ArrayList projects = OBJECT_MAPPER - .readValue( - IOUtils.toString(inputStream, "UTF-8"), - new TypeReference>() { - }); - - Path hdfsWritePath = new Path(workingPath); - - if (fs.exists(hdfsWritePath)) { - fs.delete(hdfsWritePath, false); - } - FSDataOutputStream fos = fs.create(hdfsWritePath); - - try (BufferedWriter writer = new BufferedWriter(new OutputStreamWriter(fos, StandardCharsets.UTF_8))) { - - for (Project p : projects) { - writer.write(OBJECT_MAPPER.writeValueAsString(p)); - writer.newLine(); - } - } - } } diff --git a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/utils/ReadCSV.java b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/utils/ReadCSV.java index c967d4cae..31f443c38 100644 --- a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/utils/ReadCSV.java +++ b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/utils/ReadCSV.java @@ -6,7 +6,9 @@ import java.util.Optional; import org.apache.commons.io.IOUtils; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; import eu.dnetlib.dhp.application.ArgumentApplicationParser; import eu.dnetlib.dhp.common.collection.GetCSV; @@ -40,8 +42,11 @@ public class ReadCSV { conf.set("fs.defaultFS", hdfsNameNode); FileSystem fileSystem = FileSystem.get(conf); + + FSDataInputStream inputStream = fileSystem.open(new Path(fileURL)); + BufferedReader reader = new BufferedReader( - new InputStreamReader(new HttpConnector2().getInputSourceAsStream(fileURL))); + new InputStreamReader(inputStream)); GetCSV.getCsv(fileSystem, reader, hdfsPath, classForName, del); diff --git a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/utils/ReadProjects.java b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/utils/ReadProjects.java new file mode 100644 index 000000000..b24c4806a --- /dev/null +++ b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/utils/ReadProjects.java @@ -0,0 +1,90 @@ + +package eu.dnetlib.dhp.actionmanager.project.utils; + +import java.io.*; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; +import java.util.zip.ZipEntry; +import java.util.zip.ZipInputStream; + +import org.apache.commons.io.IOUtils; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FSDataInputStream; +import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; + +import eu.dnetlib.dhp.actionmanager.project.PrepareProjects; +import eu.dnetlib.dhp.actionmanager.project.utils.model.Project; +import eu.dnetlib.dhp.application.ArgumentApplicationParser; + +/** + * @author miriam.baglioni + * @Date 28/02/23 + */ +public class ReadProjects implements Serializable { + private static final Logger log = LoggerFactory.getLogger(ReadProjects.class); + private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); + + public static void main(String[] args) throws Exception { + + String jsonConfiguration = IOUtils + .toString( + PrepareProjects.class + .getResourceAsStream( + "/eu/dnetlib/dhp/actionmanager/project/read_parameters.json")); + + final ArgumentApplicationParser parser = new ArgumentApplicationParser(jsonConfiguration); + + parser.parseArgument(args); + + final String inputPath = parser.get("inputPath"); + log.info("inputPath {}: ", inputPath); + + final String outputPath = parser.get("outputPath"); + log.info("outputPath {}: ", outputPath); + + final String hdfsNameNode = parser.get("hdfsNameNode"); + log.info("hdfsNameNode {}", hdfsNameNode); + + Configuration conf = new Configuration(); + conf.set("fs.defaultFS", hdfsNameNode); + + FileSystem fs = FileSystem.get(conf); + + readProjects(inputPath, outputPath, fs); + } + + private static void readProjects(String inputFile, String workingPath, FileSystem fs) throws IOException { + Path hdfsreadpath = new Path(inputFile); + + FSDataInputStream inputStream = fs.open(hdfsreadpath); + + ArrayList projects = OBJECT_MAPPER + .readValue( + IOUtils.toString(inputStream, "UTF-8"), + new TypeReference>() { + }); + + Path hdfsWritePath = new Path(workingPath); + + if (fs.exists(hdfsWritePath)) { + fs.delete(hdfsWritePath, false); + } + FSDataOutputStream fos = fs.create(hdfsWritePath); + + try (BufferedWriter writer = new BufferedWriter(new OutputStreamWriter(fos, StandardCharsets.UTF_8))) { + + for (Project p : projects) { + writer.write(OBJECT_MAPPER.writeValueAsString(p)); + writer.newLine(); + } + } + } +} diff --git a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/utils/ReadTopics.java b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/utils/ReadTopics.java new file mode 100644 index 000000000..86ec22207 --- /dev/null +++ b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/utils/ReadTopics.java @@ -0,0 +1,92 @@ + +package eu.dnetlib.dhp.actionmanager.project.utils; + +import java.io.BufferedWriter; +import java.io.IOException; +import java.io.OutputStreamWriter; +import java.io.Serializable; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; + +import org.apache.commons.io.IOUtils; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FSDataInputStream; +import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; + +import eu.dnetlib.dhp.actionmanager.project.PrepareProjects; +import eu.dnetlib.dhp.actionmanager.project.utils.model.JsonTopic; +import eu.dnetlib.dhp.actionmanager.project.utils.model.Project; +import eu.dnetlib.dhp.application.ArgumentApplicationParser; + +/** + * @author miriam.baglioni + * @Date 28/02/23 + */ +public class ReadTopics implements Serializable { + private static final Logger log = LoggerFactory.getLogger(ReadTopics.class); + private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); + + public static void main(String[] args) throws Exception { + + String jsonConfiguration = IOUtils + .toString( + PrepareProjects.class + .getResourceAsStream( + "/eu/dnetlib/dhp/actionmanager/project/read_parameters.json")); + + final ArgumentApplicationParser parser = new ArgumentApplicationParser(jsonConfiguration); + + parser.parseArgument(args); + + final String inputPath = parser.get("inputPath"); + log.info("inputPath {}: ", inputPath); + + final String outputPath = parser.get("outputPath"); + log.info("outputPath {}: ", outputPath); + + final String hdfsNameNode = parser.get("hdfsNameNode"); + log.info("hdfsNameNode {}", hdfsNameNode); + + Configuration conf = new Configuration(); + conf.set("fs.defaultFS", hdfsNameNode); + + FileSystem fs = FileSystem.get(conf); + + readTopics(inputPath, outputPath, fs); + } + + private static void readTopics(String inputFile, String workingPath, FileSystem fs) throws IOException { + Path hdfsreadpath = new Path(inputFile); + + FSDataInputStream inputStream = fs.open(hdfsreadpath); + + ArrayList topics = OBJECT_MAPPER + .readValue( + IOUtils.toString(inputStream, "UTF-8"), + new TypeReference>() { + }); + + Path hdfsWritePath = new Path(workingPath); + + if (fs.exists(hdfsWritePath)) { + fs.delete(hdfsWritePath, false); + } + FSDataOutputStream fos = fs.create(hdfsWritePath); + + try (BufferedWriter writer = new BufferedWriter(new OutputStreamWriter(fos, StandardCharsets.UTF_8))) { + + for (JsonTopic p : topics) { + writer.write(OBJECT_MAPPER.writeValueAsString(p)); + writer.newLine(); + } + } + } +} diff --git a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/utils/model/JsonTopic.java b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/utils/model/JsonTopic.java new file mode 100644 index 000000000..8893e28d3 --- /dev/null +++ b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/utils/model/JsonTopic.java @@ -0,0 +1,38 @@ + +package eu.dnetlib.dhp.actionmanager.project.utils.model; + +import java.io.Serializable; + +/** + * @author miriam.baglioni + * @Date 28/02/23 + */ +public class JsonTopic implements Serializable { + private String projectID; + private String title; + private String topic; + + public String getProjectID() { + return projectID; + } + + public void setProjectID(String projectID) { + this.projectID = projectID; + } + + public String getTitle() { + return title; + } + + public void setTitle(String title) { + this.title = title; + } + + public String getTopic() { + return topic; + } + + public void setTopic(String topic) { + this.topic = topic; + } +} diff --git a/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/project/extract_project_parameters.json b/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/project/extract_fromzip_parameters.json similarity index 75% rename from dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/project/extract_project_parameters.json rename to dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/project/extract_fromzip_parameters.json index a690ba018..faf620527 100644 --- a/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/project/extract_project_parameters.json +++ b/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/project/extract_fromzip_parameters.json @@ -1,16 +1,16 @@ [ { -"paramName": "pjp", -"paramLongName": "projectPath", +"paramName": "ip", +"paramLongName": "inputPath", "paramDescription": "the path where the projects are stored ", "paramRequired": true }, { -"paramName": "wp", -"paramLongName": "workingPath", +"paramName": "op", +"paramLongName": "outputPath", "paramDescription": "the path for the extracted folder", "paramRequired": true }, diff --git a/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/project/oozie_app/download.sh b/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/project/oozie_app/download.sh new file mode 100644 index 000000000..240dbd622 --- /dev/null +++ b/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/project/oozie_app/download.sh @@ -0,0 +1,3 @@ +#!/bin/bash +hdfs dfs -rm $2 +curl -LSs $1 | hdfs dfs -put - $2 \ No newline at end of file diff --git a/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/project/oozie_app/workflow.xml b/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/project/oozie_app/workflow.xml index d0b149c77..21f9b23c5 100644 --- a/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/project/oozie_app/workflow.xml +++ b/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/project/oozie_app/workflow.xml @@ -14,7 +14,6 @@ outputPath - noneed path where to store the action set @@ -35,42 +34,104 @@ - + - - + - - - + + - - + + - - - eu.dnetlib.dhp.actionmanager.project.utils.ExtractProjects - --hdfsNameNode${nameNode} - --projectPath/tmp/miriam/cordis-h2020projects-json_.zip - - - --workingPath${workingDir}/ - - - + + + ${jobTracker} + ${nameNode} + + + mapred.job.queue.name + ${queueName} + + + download.sh + ${downloadH2020Projects} + ${projectPath} + HADOOP_USER_NAME=${wf:user()} + download.sh + + + - + + + eu.dnetlib.dhp.actionmanager.project.utils.ExtractFromZip + --hdfsNameNode${nameNode} + --inputPath${projectPath} + --outputPath${workingDir}/ + + + + + + + + + + + + + eu.dnetlib.dhp.actionmanager.project.utils.ReadProjects + --hdfsNameNode${nameNode} + --inputPath${workingDir}/json/project.json + --outputPath${workingDir}/projects + + + + + + + + ${jobTracker} + ${nameNode} + + + mapred.job.queue.name + ${queueName} + + + download.sh + ${downloadH2020Programme} + ${programmePath} + HADOOP_USER_NAME=${wf:user()} + download.sh + + + + + + + + eu.dnetlib.dhp.actionmanager.project.utils.ExtractFromZip + --hdfsNameNode${nameNode} + --inputPath${programmePath} + --outputPath${workingDir}/downloadedProgramme/ + + + + + eu.dnetlib.dhp.actionmanager.project.utils.ReadCSV --hdfsNameNode${nameNode} - --fileURL${programmeFileURL} + --fileURL${workingDir}/downloadedProgramme/csv/programme.csv --hdfsPath${workingDir}/programme --classForNameeu.dnetlib.dhp.actionmanager.project.utils.model.CSVProgramme @@ -78,20 +139,18 @@ - + - eu.dnetlib.dhp.actionmanager.project.utils.ReadExcel + eu.dnetlib.dhp.actionmanager.project.utils.ReadTopics --hdfsNameNode${nameNode} - --fileURL${topicFileURL} - --hdfsPath${workingDir}/topic - --sheetName${sheetName} - --classForNameeu.dnetlib.dhp.actionmanager.project.utils.model.EXCELTopic + --inputPath${workingDir}/json/topics.json + --outputPath${workingDir}/topic - + - + eu.dnetlib.dhp.actionmanager.project.ReadProjectsFromDB --hdfsPath${workingDir}/dbProjects @@ -125,9 +184,11 @@ --outputPath${workingDir}/preparedProgramme + + @@ -154,8 +215,8 @@ --outputPath${workingDir}/preparedProjects --dbProjectPath${workingDir}/dbProjects - - + + diff --git a/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/project/read_parameters.json b/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/project/read_parameters.json new file mode 100644 index 000000000..faf620527 --- /dev/null +++ b/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/project/read_parameters.json @@ -0,0 +1,23 @@ +[ + +{ +"paramName": "ip", +"paramLongName": "inputPath", +"paramDescription": "the path where the projects are stored ", +"paramRequired": true +}, + + + { +"paramName": "op", +"paramLongName": "outputPath", +"paramDescription": "the path for the extracted folder", +"paramRequired": true +}, + { + "paramName": "hnn", + "paramLongName": "hdfsNameNode", + "paramDescription": "the hdfs namenode", + "paramRequired": true + } +] \ No newline at end of file