D-Net
/
dhp-graph-dump
4
0
Fork 0
Code Issues Pull Requests 1 Projects Releases Wiki Activity

removed not needed parameter

This commit is contained in:
Miriam Baglioni 2023-11-13 11:38:56 +01:00
parent d9ca135c1f
commit cc86f24372
5 changed files with 41 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/complete/CreateContextRelation.java
View File

@ -60,9 +60,6 @@ public class CreateContextRelation implements Serializable {
final String hdfsNameNode = parser.get("nameNode");
log.info("hdfsNameNode: {}", hdfsNameNode);
final String masterDuplicatePath = parser.get("masterDuplicate");
log.info("masterDuplicatePath: {}", masterDuplicatePath);
final CreateContextRelation cce = new CreateContextRelation(hdfsPath, hdfsNameNode);
log.info("Creating relation for datasources and projects...");
@ -74,20 +71,6 @@ public class CreateContextRelation implements Serializable {
}
private List<MasterDuplicate> readMasterDuplicate(String masterDuplicatePath) throws IOException {
FileSystem fileSystem = FileSystem.get(conf);
Path hdfsReadPath = new Path(masterDuplicatePath);
BufferedReader reader = new BufferedReader(new InputStreamReader(fileSystem.open(hdfsReadPath)));
List<MasterDuplicate> mdlist = new ArrayList<>();
ObjectMapper mapper = new ObjectMapper();
String line;
while ((line = reader.readLine()) != null) {
mdlist.add(mapper.readValue(line, MasterDuplicate.class));
}
return mdlist;
}
private void close() throws IOException {
writer.close();
}

5
dump/src/main/resources/eu/dnetlib/dhp/oa/graph/dump/copygraph/job.properties Normal file
View File

@ -0,0 +1,5 @@
#PROPERTIES FOR COPY THE GRAPH FROM HIVE
sourcePath=/tmp/prod_provision/graph/18_graph_blacklisted/
hivePath=/user/hive/warehouse/openaire_prod_20230802.db
readFormat=parquet
outputPath=/tmp/miriam/graphCopy

6
dump/src/main/resources/eu/dnetlib/dhp/oa/graph/dump/countryresults/oozie_app/job.properties Normal file
View File

@ -0,0 +1,6 @@
#PROPERTIES FOR DUMP OF RESULTS RELATED TO A SPECIFIC COUNTRY AS SHOWN BY THE PORTAL
sourcePath=/tmp/miriam/graphCopy
outputPath=/tmp/miriam/graph_dumps/country_PT_2
country=PT
removeSet="merges;isMergedIn"
isLookUpUrl=http://services.openaire.eu:8280/is/services/isLookUp?wsdl

30
dump/src/main/resources/eu/dnetlib/dhp/oa/graph/dump/wf/subworkflows/subset/job.properties Normal file
View File

File diff suppressed because one or more lines are too long

1
dump/src/main/resources/eu/dnetlib/dhp/oa/graph/dump/wf/subworkflows/subset/oozie_app/workflow.xml
View File

@ -647,7 +647,6 @@
<main-class>eu.dnetlib.dhp.oa.graph.dump.complete.CreateContextRelation</main-class>
<arg>--hdfsPath</arg><arg>${workingDir}/dump/relation/context</arg>
<arg>--nameNode</arg><arg>${nameNode}</arg>
<arg>--masterDuplicate</arg><arg>${workingDir}/masterduplicate</arg>
</java>
<ok to="join_context"/>
<error to="Kill"/>