From e8f914f8b313c51b07f5b587a264a8aa18ee26bf Mon Sep 17 00:00:00 2001 From: "miriam.baglioni" Date: Mon, 22 Jun 2020 10:50:41 +0200 Subject: [PATCH] - --- .../dhp/oa/graph/dump/oozie_app/workflow.xml | 155 ++----------- .../dhp/oa/graph/dump/upload_zenodo.json | 39 ++++ .../dhp/oa/graph/dump/ZenodoUploadTest.java | 208 +++++++++++++----- 3 files changed, 210 insertions(+), 192 deletions(-) diff --git a/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/dump/oozie_app/workflow.xml b/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/dump/oozie_app/workflow.xml index be3156e0d..8b826832e 100644 --- a/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/dump/oozie_app/workflow.xml +++ b/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/dump/oozie_app/workflow.xml @@ -80,7 +80,7 @@ - + Action failed, error message[${wf:errorMessage(wf:lastErrorNode())}] @@ -366,146 +366,19 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + eu.dnetlib.dhp.oa.graph.dump.SendToZenodo + --hdfsPath${outputPath} + --hdfsNameNode${nameNode} + --accessToken${accessToken} + --connectionUrl${connectionUrl} + --metadata${metadata} + --isLookUpUrl${isLookUpUrl} + + + + diff --git a/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/dump/upload_zenodo.json b/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/dump/upload_zenodo.json index e69de29bb..d65df9584 100644 --- a/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/dump/upload_zenodo.json +++ b/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/dump/upload_zenodo.json @@ -0,0 +1,39 @@ + +[ +{ +"paramName":"is", +"paramLongName":"isLookUpUrl", +"paramDescription": "URL of the isLookUp Service", +"paramRequired": true +}, +{ +"paramName":"hdfsp", +"paramLongName":"hdfsPath", +"paramDescription": "the path of the folder tofind files to send to Zenodo", +"paramRequired": true +}, +{ +"paramName": "hdfsnn", +"paramLongName": "hdfsNameNode", +"paramDescription": "the name node", +"paramRequired": true +}, +{ +"paramName": "at", +"paramLongName": "accessToken", +"paramDescription": "the access token for the deposition", +"paramRequired": false +}, +{ +"paramName":"cu", +"paramLongName":"connectionUrl", +"paramDescription": "the url to connect to deposit", +"paramRequired": false +}, +{ +"paramName":"m", +"paramLongName":"metadata", +"paramDescription": "metadata associated to the deposition", +"paramRequired": false +} +] \ No newline at end of file diff --git a/dhp-workflows/dhp-graph-mapper/src/test/java/eu/dnetlib/dhp/oa/graph/dump/ZenodoUploadTest.java b/dhp-workflows/dhp-graph-mapper/src/test/java/eu/dnetlib/dhp/oa/graph/dump/ZenodoUploadTest.java index c6a3b0a4d..7694b9434 100644 --- a/dhp-workflows/dhp-graph-mapper/src/test/java/eu/dnetlib/dhp/oa/graph/dump/ZenodoUploadTest.java +++ b/dhp-workflows/dhp-graph-mapper/src/test/java/eu/dnetlib/dhp/oa/graph/dump/ZenodoUploadTest.java @@ -1,54 +1,180 @@ package eu.dnetlib.dhp.oa.graph.dump; -import java.io.IOException; +import java.io.*; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; import java.util.Arrays; +import org.apache.commons.io.IOUtils; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.*; +import org.apache.spark.SparkConf; +import org.apache.spark.sql.SparkSession; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; + import com.google.gson.Gson; + import eu.dnetlib.dhp.oa.graph.dump.zenodo.Creator; import eu.dnetlib.dhp.oa.graph.dump.zenodo.Metadata; import eu.dnetlib.dhp.oa.graph.dump.zenodo.ZenodoModel; -import org.apache.commons.io.IOUtils; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; public class ZenodoUploadTest { + private static String workingDir; + +// private static FileSystem fileSystem; +// + @BeforeAll + public static void beforeAll() throws IOException { + workingDir = Files + .createTempDirectory(eu.dnetlib.dhp.oa.graph.dump.UpdateProjectInfoTest.class.getSimpleName()) + .toString(); + } +// +// Configuration conf = new Configuration(); +// conf.set("fs.defaultFS", "localhost"); +// +// fileSystem = FileSystem.get(conf); +// +// FSDataOutputStream fsDataOutputStream = fileSystem.create(new org.apache.hadoop.fs.Path(workingDir + "/ni")); +// +// BufferedWriter writer = new BufferedWriter(new OutputStreamWriter(fsDataOutputStream, StandardCharsets.UTF_8)); +// +// +// writer.write(ZenodoUploadTest.class.getResourceAsStream("/eu/dnetlib/dhp/oa/graph/dump/zenodo/ni").toString()); +// } + + @Test + public void HDFSConnection() throws IOException { + CommunityMap communityMap = new CommunityMap(); + communityMap.put("ni", "Neuroinformatics"); + communityMap.put("dh-ch", "Digital Humanities and Cultural Heritage"); + LocalFileSystem fs = FileSystem.getLocal(new Configuration()); + + fs + .copyFromLocalFile( + false, new Path(getClass() + .getResource("/eu/dnetlib/dhp/oa/graph/dump/zenodo/ni") + .getPath()), + new Path(workingDir + "/zenodo/ni/ni")); + fs + .copyFromLocalFile( + false, new Path(getClass() + .getResource("/eu/dnetlib/dhp/oa/graph/dump/zenodo/dh-ch") + .getPath()), + new Path(workingDir + "/zenodo/dh-ch/dh-ch")); + + System.out.println("pr"); + +// Configuration conf = new Configuration(); +// conf.set("fs.defaultFS", "localhost"); +// +// APIClient s = new APIClient( + // "https://sandbox.zenodo.org/api/deposit/depositions"); +// +// s.connect(); +// s.upload(workingDir +"/ni", "Neuroinformatics", fs); + + APIClient client = new APIClient("https://sandbox.zenodo.org/api/deposit/depositions", + "5ImUj0VC1ICg4ifK5dc3AGzJhcfAB4osxrFlsr8WxHXxjaYgCE0hY8HZcDoe"); + client.connect(); + + // the second boolean parameter here sets the recursion to true + RemoteIterator fileStatusListIterator = fs + .listFiles( + new Path(workingDir + "/zenodo"), true); + while (fileStatusListIterator.hasNext()) { + LocatedFileStatus fileStatus = fileStatusListIterator.next(); + // do stuff with the file like ... + + // BufferedInputStream bis = new BufferedInputStream(fs.open( fileStatus.getPath())); + String p_string = fileStatus.getPath().toString(); + + int index = p_string.lastIndexOf("/"); + String community = p_string.substring(0, index); + community = community.substring(community.lastIndexOf("/") + 1); + String community_name = communityMap.get(community).replace(" ", "_"); + fs.copyToLocalFile(fileStatus.getPath(), new Path("/tmp/" + community_name)); + System.out.println(community); +// System.out.println(client.upload(bis, community)); + + File f = new File("/tmp/" + community_name); + client.upload(f, community_name); + + if (f.exists()) { + f.delete(); + } + + } + + ZenodoModel zenodo = new ZenodoModel(); + Metadata data = new Metadata(); + + data.setTitle("Dump of OpenAIRE Communities related graph"); + data.setUpload_type("dataset"); + data.setDescription("this is a fake uploade done for testing purposes"); + Creator c = new Creator(); + c.setName("Miriam Baglioni"); + c.setAffiliation("CNR _ISTI"); + data.setCreators(Arrays.asList(c)); + zenodo.setMetadata(data); + + System.out.println(client.sendMretadata(new Gson().toJson(zenodo))); + + System.out.println(client.publish()); + + } + + @Test + public void serializeMetadata() { + ZenodoModel zenodo = new ZenodoModel(); + Metadata data = new Metadata(); + + data.setTitle("Dump of OpenAIRE Communities related graph"); + data.setUpload_type("dataset"); + data.setDescription("this is a fake uploade done for testing purposes"); + Creator c = new Creator(); + c.setName("Miriam Baglioni"); + c.setAffiliation("CNR _ISTI"); + data.setCreators(Arrays.asList(c)); + zenodo.setMetadata(data); + + System.out.println(new Gson().toJson(zenodo)); + + } + @Test public void testConnection() throws IOException { + + // InputStream is = getClass().getClassLoader().getResourceAsStream("eu/dnetlib/dhp/oa/graph/dump/zenodo/ni"); APIClient s = new APIClient( - // "https://sandbox.zenodo.org/api/deposit/depositions?access_token=5ImUj0VC1ICg4ifK5dc3AGzJhcfAB4osxrFlsr8WxHXxjaYgCE0hY8HZcDoe"); - "https://sandbox.zenodo.org/api/deposit/depositions"); + "https://sandbox.zenodo.org/api/deposit/depositions", + "5ImUj0VC1ICg4ifK5dc3AGzJhcfAB4osxrFlsr8WxHXxjaYgCE0hY8HZcDoe"); Assertions.assertEquals(201, s.connect()); - s.upload(getClass() - .getResource("/eu/dnetlib/dhp/oa/graph/dump/zenodo/ni") - .getPath(), "Neuroinformatics"); + s + .upload( + new File(getClass() + .getResource("/eu/dnetlib/dhp/oa/graph/dump/zenodo/ni") + .getPath()), + "Neuroinformatics"); - s.upload(getClass() - .getResource("/eu/dnetlib/dhp/oa/graph/dump/zenodo/dh-ch") - .getPath(), "DigitalHumanitiesandCulturalHeritage"); +// s.upload(getClass() +// .getResource("/eu/dnetlib/dhp/oa/graph/dump/zenodo/dh-ch") +// .getPath(), "DigitalHumanitiesandCulturalHeritage"); +// +// s.upload(getClass() +// .getResource("/eu/dnetlib/dhp/oa/graph/dump/zenodo/egi") +// .getPath(), "EGI"); +// +// s.upload(getClass() +// .getResource("/eu/dnetlib/dhp/oa/graph/dump/zenodo/science-innovation-policy") +// .getPath(), "ScienceandInnovationPolicyStudies"); - s.upload(getClass() - .getResource("/eu/dnetlib/dhp/oa/graph/dump/zenodo/egi") - .getPath(), "EGI"); - - s.upload(getClass() - .getResource("/eu/dnetlib/dhp/oa/graph/dump/zenodo/science-innovation-policy") - .getPath(), "ScienceandInnovationPolicyStudies"); - -// -// -// String data = "{\"metadata\": {\"title\": \"My first upload\", " + -// "\"upload_type\": \"poster\", " + -// "\"description\": \"This is my first upload\", " + -// "\"creators\": [{\"name\": \"Doe, John\", " + -// "\"affiliation': 'Zenodo'}] -//... } -//... } -// -// ZenodoModel zenodo = new ZenodoModel(); Metadata data = new Metadata(); @@ -67,24 +193,4 @@ public class ZenodoUploadTest { } - - @Test - public void testPublish() throws IOException { - APIClient s = new APIClient("https://sandbox.zenodo.org/api/deposit/depositions"); - s.publish(); - } - @Test - public void testUpload() throws IOException { - - APIClient s = new APIClient( - "https://sandbox.zenodo.org/api/deposit/depositions?access_token=5ImUj0VC1ICg4ifK5dc3AGzJhcfAB4osxrFlsr8WxHXxjaYgCE0hY8HZcDoe"); - final String sourcePath = getClass() - .getResource("/eu/dnetlib/dhp/oa/graph/dump/zenodo/ni") - .getPath(); - - s.upload(sourcePath, "Neuroinformatics"); - - - - } }