sabeel
/
dnet-hadoop
forked from D-Net/dnet-hadoop
1
0
Fork 0
Code Pull Requests Activity

removed unuseful tests

This commit is contained in:
Miriam Baglioni 2020-07-07 18:25:00 +02:00
parent 9b20a21b24
commit a66aa9bd83
1 changed files with 9 additions and 116 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

125
dhp-workflows/dhp-graph-mapper/src/test/java/eu/dnetlib/dhp/oa/graph/dump/ZenodoUploadTest.java
View File

@ -2,9 +2,12 @@
package eu.dnetlib.dhp.oa.graph.dump; package eu.dnetlib.dhp.oa.graph.dump;
import java.io.*; import java.io.*;
import java.io.File;
import java.nio.charset.StandardCharsets; import java.nio.charset.StandardCharsets;
import java.nio.file.Files; import java.nio.file.Files;
import java.util.ArrayList;
import java.util.Arrays; import java.util.Arrays;
import java.util.List;
import org.apache.commons.io.IOUtils; import org.apache.commons.io.IOUtils;
import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configuration;
@ -17,16 +20,15 @@ import org.junit.jupiter.api.Test;
import com.google.gson.Gson; import com.google.gson.Gson;
import eu.dnetlib.dhp.oa.graph.dump.zenodo.Creator; import eu.dnetlib.dhp.oa.graph.dump.zenodo.*;
import eu.dnetlib.dhp.oa.graph.dump.zenodo.Metadata; import eu.dnetlib.dhp.schema.dump.oaf.*;
import eu.dnetlib.dhp.oa.graph.dump.zenodo.ZenodoModel;
public class ZenodoUploadTest { public class ZenodoUploadTest {
private static String workingDir; private static String workingDir;
private final String URL_STRING = "https://sandbox.zenodo.org/api/deposit/depositions"; private final String URL_STRING = "https://sandbox.zenodo.org/api/deposit/depositions";
private final String ACCESS_TOKEN = "5ImUj0VC1ICg4ifK5dc3AGzJhcfAB4osxrFlsr8WxHXxjaYgCE0hY8HZcDoe"; private final String ACCESS_TOKEN = "";
@BeforeAll @BeforeAll
public static void beforeAll() throws IOException { public static void beforeAll() throws IOException {
@ -34,19 +36,7 @@ public class ZenodoUploadTest {
.createTempDirectory(eu.dnetlib.dhp.oa.graph.dump.UpdateProjectInfoTest.class.getSimpleName()) .createTempDirectory(eu.dnetlib.dhp.oa.graph.dump.UpdateProjectInfoTest.class.getSimpleName())
.toString(); .toString();
} }
//
// Configuration conf = new Configuration();
// conf.set("fs.defaultFS", "localhost");
//
// fileSystem = FileSystem.get(conf);
//
// FSDataOutputStream fsDataOutputStream = fileSystem.create(new org.apache.hadoop.fs.Path(workingDir + "/ni"));
//
// BufferedWriter writer = new BufferedWriter(new OutputStreamWriter(fsDataOutputStream, StandardCharsets.UTF_8));
//
//
// writer.write(ZenodoUploadTest.class.getResourceAsStream("/eu/dnetlib/dhp/oa/graph/dump/zenodo/ni").toString());
// }
@Test @Test
public void HDFSConnection() throws IOException { public void HDFSConnection() throws IOException {
@ -68,16 +58,6 @@ public class ZenodoUploadTest {
.getPath()), .getPath()),
new Path(workingDir + "/zenodo/dh-ch/dh-ch")); new Path(workingDir + "/zenodo/dh-ch/dh-ch"));
System.out.println("pr");
// Configuration conf = new Configuration();
// conf.set("fs.defaultFS", "localhost");
//
// APIClient s = new APIClient(
// "https://sandbox.zenodo.org/api/deposit/depositions");
//
// s.connect();
// s.upload(workingDir +"/ni", "Neuroinformatics", fs);
APIClient client = new APIClient(URL_STRING, APIClient client = new APIClient(URL_STRING,
ACCESS_TOKEN); ACCESS_TOKEN);
@ -89,9 +69,7 @@ public class ZenodoUploadTest {
new Path(workingDir + "/zenodo"), true); new Path(workingDir + "/zenodo"), true);
while (fileStatusListIterator.hasNext()) { while (fileStatusListIterator.hasNext()) {
LocatedFileStatus fileStatus = fileStatusListIterator.next(); LocatedFileStatus fileStatus = fileStatusListIterator.next();
// do stuff with the file like ...
// BufferedInputStream bis = new BufferedInputStream(fs.open( fileStatus.getPath()));
String p_string = fileStatus.getPath().toString(); String p_string = fileStatus.getPath().toString();
int index = p_string.lastIndexOf("/"); int index = p_string.lastIndexOf("/");
@ -100,7 +78,6 @@ public class ZenodoUploadTest {
String community_name = communityMap.get(community).replace(" ", "_"); String community_name = communityMap.get(community).replace(" ", "_");
fs.copyToLocalFile(fileStatus.getPath(), new Path("/tmp/" + community_name)); fs.copyToLocalFile(fileStatus.getPath(), new Path("/tmp/" + community_name));
System.out.println(community); System.out.println(community);
// System.out.println(client.upload(bis, community));
File f = new File("/tmp/" + community_name); File f = new File("/tmp/" + community_name);
System.out.println(client.upload(f, community_name)); System.out.println(client.upload(f, community_name));
@ -111,98 +88,14 @@ public class ZenodoUploadTest {
} }
ZenodoModel zenodo = new ZenodoModel();
Metadata data = new Metadata();
data.setTitle("Dump of OpenAIRE Communities related graph"); String metadata = "{\"metadata\":{\"access_right\":\"open\",\"communities\":[{\"identifier\":\"openaire-research-graph\"}],\"creators\":[{\"affiliation\":\"CNR - ISTI\",\"name\":\"Manghi, Paolo\",\"orcid\":\"0000-0001-7291-3210\"},{\"affiliation\":\"CNR - ISTI\",\"name\":\"Atzori, Claudio\",\"orcid\":\"0000-0001-9613-6639\"},{\"affiliation\":\"CNR - ISTI\",\"name\":\"Bardi, Alessia\",\"orcid\":\"0000-0002-1112-1292\"},{\"affiliation\":\"ISTI - CNR\",\"name\":\"Baglioni, Miriam\",\"orcid\":\"0000-0002-2273-9004\"},{\"affiliation\":\"University of Bielefeld\",\"name\":\"Shirrwagen, Jochen\"},{\"affiliation\":\"Athena Research and Innovation Centre\",\"name\":\"Dimitropoulos, Harry\"},{\"affiliation\":\"CNR - ISTI\",\"name\":\"La Bruzzo, Sandro\",\"orcid\":\"0000-0003-2855-1245\"},{\"affiliation\":\"Athena Research and Innovation Centre\",\"name\":\"Foufoulas, Ioannis\"},{\"affiliation\":\"University of Bielefeld\",\"name\":\"Löhden, Aenne\"},{\"affiliation\":\"University of Bielefeld\",\"name\":\"Bäcker, Amelie\",\"orcid\":\"0000-0001-6015-2063\"},{\"affiliation\":\"CNR - ISTI\",\"name\":\"Mannocci, Andrea\",\"orcid\":\"0000-0002-5193-7851\"},{\"affiliation\":\"University of Warsaw\",\"name\":\"Horst, Marek\"},{\"affiliation\":\"University of Bielefeld\",\"name\":\"Czerniak, Andreas\",\"orcid\":\"0000-0003-3883-4169\"},{\"affiliation\":\"Athena Research and Innovation Centre\",\"name\":\"Kiatropoulou, Katerina\"},{\"affiliation\":\"Athena Research and Innovation Centre\",\"name\":\"Kokogiannaki, Argiro\",\"orcid\":\"0000-0002-3880-0244\"},{\"affiliation\":\"CNR - ISTI\",\"name\":\"De Bonis, Michele\"},{\"affiliation\":\"CNR - ISTI\",\"name\":\"Artini, Michele\"},{\"affiliation\":\"CNR - ISTI\",\"name\":\"Ottonello, Enrico\"},{\"affiliation\":\"Athena Research and Innovation Centre\",\"name\":\"Lempesis, Antonis\"},{\"affiliation\":\"CERN\",\"name\":\"Ioannidis, Alexandros\"},{\"affiliation\":\"University of Bielefeld\",\"name\":\"Summan, Friedrich\"}],\"description\":\"\\u003cp\\u003eThis dataset contains dumps of the OpenAIRE Research Graph containing metadata records relevant for the research communities and initiatives collaborating with OpenAIRE\\u003c/p\\u003e. \\u003cp\\u003eEach dataset is a zip containing a file with one json per line. Each json is compliant to the schema available at XXXX\\u003c/p\\u003e\",\"grants\":[{\"id\":\"777541\"},{\"id\":\"824091\"},{\"id\":\"824323\"}],\"keywords\":[\"Open Science\",\"Scholarly Communication\",\"Information Science\"],\"language\":\"eng\",\"license\":\"CC-BY-4.0\",\"title\":\"OpenAIRE Research Graph: Dumps for research communities and initiatives.\",\"upload_type\":\"dataset\",\"version\":\"1.0\"}}";
data.setUpload_type("dataset");
data.setDescription("this is a fake upload done for testing purposes");
Creator c = new Creator();
c.setName("Miriam Baglioni");
c.setAffiliation("CNR _ISTI");
data.setCreators(Arrays.asList(c));
zenodo.setMetadata(data);
System.out.println(client.sendMretadata(new Gson().toJson(zenodo))); System.out.println(client.sendMretadata(metadata));
System.out.println(client.publish()); System.out.println(client.publish());
} }
@Test
public void serializeMetadata() {
ZenodoModel zenodo = new ZenodoModel();
Metadata data = new Metadata();
data.setTitle("Dump of OpenAIRE Communities related graph");
data.setUpload_type("dataset");
data.setDescription("this is a fake uploade done for testing purposes");
Creator c = new Creator();
c.setName("Miriam Baglioni");
c.setAffiliation("CNR _ISTI");
data.setCreators(Arrays.asList(c));
zenodo.setMetadata(data);
System.out.println(new Gson().toJson(zenodo));
}
@Test
public void testConnection() throws IOException {
// InputStream is = getClass().getClassLoader().getResourceAsStream("eu/dnetlib/dhp/oa/graph/dump/zenodo/ni");
APIClient s = new APIClient(
"https://sandbox.zenodo.org/api/deposit/depositions",
"5ImUj0VC1ICg4ifK5dc3AGzJhcfAB4osxrFlsr8WxHXxjaYgCE0hY8HZcDoe");
Assertions.assertEquals(201, s.connect());
s
.upload(
new File(getClass()
.getResource("/eu/dnetlib/dhp/oa/graph/dump/zenodo/ni")
.getPath()),
"Neuroinformatics");
// s.upload(getClass()
// .getResource("/eu/dnetlib/dhp/oa/graph/dump/zenodo/dh-ch")
// .getPath(), "DigitalHumanitiesandCulturalHeritage");
//
// s.upload(getClass()
// .getResource("/eu/dnetlib/dhp/oa/graph/dump/zenodo/egi")
// .getPath(), "EGI");
//
// s.upload(getClass()
// .getResource("/eu/dnetlib/dhp/oa/graph/dump/zenodo/science-innovation-policy")
// .getPath(), "ScienceandInnovationPolicyStudies");
// ZenodoModel zenodo = new ZenodoModel();
// Metadata data = new Metadata();
//
// data.setTitle("Dump of OpenAIRE Communities related graph");
// data.setUpload_type("dataset");
// data.setDescription("this is a fake uploade done for testing purposes");
// Creator c = new Creator();
// c.setName("Miriam Baglioni");
// c.setAffiliation("CNR _ISTI");
// data.setCreators(Arrays.asList(c));
// zenodo.setMetadata(data);
//
// s.sendMretadata(new Gson().toJson(zenodo));
//
// s.publish();
}
@Test
public void testConnectionOnly() throws IOException {
APIClient s = new APIClient(
"https://sandbox.zenodo.org/api/deposit/depositions",
"5ImUj0VC1ICg4ifK5dc3AGzJhcfAB4osxrFlsr8WxHXxjaYgCE0hY8HZcDoe");
int tmp = s.connect();
System.out.println(tmp);
}
} }