removed not needed parameter
This commit is contained in:
parent
d9ca135c1f
commit
cc86f24372
|
@ -60,9 +60,6 @@ public class CreateContextRelation implements Serializable {
|
||||||
final String hdfsNameNode = parser.get("nameNode");
|
final String hdfsNameNode = parser.get("nameNode");
|
||||||
log.info("hdfsNameNode: {}", hdfsNameNode);
|
log.info("hdfsNameNode: {}", hdfsNameNode);
|
||||||
|
|
||||||
final String masterDuplicatePath = parser.get("masterDuplicate");
|
|
||||||
log.info("masterDuplicatePath: {}", masterDuplicatePath);
|
|
||||||
|
|
||||||
final CreateContextRelation cce = new CreateContextRelation(hdfsPath, hdfsNameNode);
|
final CreateContextRelation cce = new CreateContextRelation(hdfsPath, hdfsNameNode);
|
||||||
|
|
||||||
log.info("Creating relation for datasources and projects...");
|
log.info("Creating relation for datasources and projects...");
|
||||||
|
@ -74,20 +71,6 @@ public class CreateContextRelation implements Serializable {
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
private List<MasterDuplicate> readMasterDuplicate(String masterDuplicatePath) throws IOException {
|
|
||||||
FileSystem fileSystem = FileSystem.get(conf);
|
|
||||||
Path hdfsReadPath = new Path(masterDuplicatePath);
|
|
||||||
BufferedReader reader = new BufferedReader(new InputStreamReader(fileSystem.open(hdfsReadPath)));
|
|
||||||
List<MasterDuplicate> mdlist = new ArrayList<>();
|
|
||||||
ObjectMapper mapper = new ObjectMapper();
|
|
||||||
String line;
|
|
||||||
|
|
||||||
while ((line = reader.readLine()) != null) {
|
|
||||||
mdlist.add(mapper.readValue(line, MasterDuplicate.class));
|
|
||||||
}
|
|
||||||
return mdlist;
|
|
||||||
}
|
|
||||||
|
|
||||||
private void close() throws IOException {
|
private void close() throws IOException {
|
||||||
writer.close();
|
writer.close();
|
||||||
}
|
}
|
||||||
|
|
|
@ -0,0 +1,5 @@
|
||||||
|
#PROPERTIES FOR COPY THE GRAPH FROM HIVE
|
||||||
|
sourcePath=/tmp/prod_provision/graph/18_graph_blacklisted/
|
||||||
|
hivePath=/user/hive/warehouse/openaire_prod_20230802.db
|
||||||
|
readFormat=parquet
|
||||||
|
outputPath=/tmp/miriam/graphCopy
|
|
@ -0,0 +1,6 @@
|
||||||
|
#PROPERTIES FOR DUMP OF RESULTS RELATED TO A SPECIFIC COUNTRY AS SHOWN BY THE PORTAL
|
||||||
|
sourcePath=/tmp/miriam/graphCopy
|
||||||
|
outputPath=/tmp/miriam/graph_dumps/country_PT_2
|
||||||
|
country=PT
|
||||||
|
removeSet="merges;isMergedIn"
|
||||||
|
isLookUpUrl=http://services.openaire.eu:8280/is/services/isLookUp?wsdl
|
File diff suppressed because one or more lines are too long
|
@ -647,7 +647,6 @@
|
||||||
<main-class>eu.dnetlib.dhp.oa.graph.dump.complete.CreateContextRelation</main-class>
|
<main-class>eu.dnetlib.dhp.oa.graph.dump.complete.CreateContextRelation</main-class>
|
||||||
<arg>--hdfsPath</arg><arg>${workingDir}/dump/relation/context</arg>
|
<arg>--hdfsPath</arg><arg>${workingDir}/dump/relation/context</arg>
|
||||||
<arg>--nameNode</arg><arg>${nameNode}</arg>
|
<arg>--nameNode</arg><arg>${nameNode}</arg>
|
||||||
<arg>--masterDuplicate</arg><arg>${workingDir}/masterduplicate</arg>
|
|
||||||
</java>
|
</java>
|
||||||
<ok to="join_context"/>
|
<ok to="join_context"/>
|
||||||
<error to="Kill"/>
|
<error to="Kill"/>
|
||||||
|
|
Loading…
Reference in New Issue