changed test to upload the dump for covid-19 community

This commit is contained in:
Miriam Baglioni 2020-08-11 15:43:25 +02:00
parent 7dfd56df9d
commit 0603ec4757
3 changed files with 5 additions and 140 deletions

View File

@ -10,7 +10,7 @@ import org.junit.jupiter.api.Test;
public class ZenodoAPIClientTest { public class ZenodoAPIClientTest {
private final String URL_STRING = "https://sandbox.zenodo.org/api/deposit/depositions"; private final String URL_STRING = "https://sandbox.zenodo.org/api/deposit/depositions";
private final String ACCESS_TOKEN = ""; private final String ACCESS_TOKEN = "5ImUj0VC1ICg4ifK5dc3AGzJhcfAB4osxrFlsr8WxHXxjaYgCE0hY8HZcDoe";
private final String CONCEPT_REC_ID = "657113"; private final String CONCEPT_REC_ID = "657113";
@ -22,18 +22,18 @@ public class ZenodoAPIClientTest {
Assertions.assertEquals(201, client.newDeposition()); Assertions.assertEquals(201, client.newDeposition());
File file = new File(getClass() File file = new File(getClass()
.getResource("/eu/dnetlib/dhp/common/api/newDeposition") .getResource("/eu/dnetlib/dhp/common/api/COVID-19.json.gz")
.getPath()); .getPath());
InputStream is = new FileInputStream(file); InputStream is = new FileInputStream(file);
Assertions.assertEquals(200, client.uploadIS(is, "first_deposition", file.length())); Assertions.assertEquals(200, client.uploadIS(is, "COVID-19.json.gz", file.length()));
String metadata = "{\"metadata\":{\"access_right\":\"open\",\"creators\":[{\"affiliation\":\"ISTI - CNR\",\"name\":\"Baglioni, Miriam\",\"orcid\":\"0000-0002-2273-9004\"}],\"description\":\"\\u003cp\\u003eThis is a test for the automatic upload of files in a new deposition\\u003c/p\\u003e \",\"title\":\"Test.\",\"upload_type\":\"other\",\"version\":\"1.0\"}}"; String metadata = "{\"metadata\":{\"access_right\":\"open\",\"communities\":[{\"identifier\":\"openaire-research-graph\"}],\"creators\":[{\"affiliation\":\"ISTI - CNR\",\"name\":\"Bardi, Alessia\",\"orcid\":\"0000-0002-1112-1292\"},{\"affiliation\":\"eifl\", \"name\":\"Kuchma, Iryna\"},{\"affiliation\":\"BIH\", \"name\":\"Brobov, Evgeny\"},{\"affiliation\":\"GIDIF RBM\", \"name\":\"Truccolo, Ivana\"},{\"affiliation\":\"unesp\", \"name\":\"Monteiro, Elizabete\"},{\"affiliation\":\"und\", \"name\":\"Casalegno, Carlotta\"},{\"affiliation\":\"CARL ABRC\", \"name\":\"Clary, Erin\"},{\"affiliation\":\"The University of Edimburgh\", \"name\":\"Romanowski, Andrew\"},{\"affiliation\":\"ISTI - CNR\", \"name\":\"Pavone, Gina\"},{\"affiliation\":\"ISTI - CNR\", \"name\":\"Artini, Michele\"},{\"affiliation\":\"ISTI - CNR\",\"name\":\"Atzori, Claudio\",\"orcid\":\"0000-0001-9613-6639\"},{\"affiliation\":\"University of Bielefeld\",\"name\":\"Bäcker, Amelie\",\"orcid\":\"0000-0001-6015-2063\"},{\"affiliation\":\"ISTI - CNR\",\"name\":\"Baglioni, Miriam\",\"orcid\":\"0000-0002-2273-9004\"},{\"affiliation\":\"University of Bielefeld\",\"name\":\"Czerniak, Andreas\",\"orcid\":\"0000-0003-3883-4169\"},{\"affiliation\":\"ISTI - CNR\",\"name\":\"De Bonis, Michele\"},{\"affiliation\":\"Athena Research and Innovation Centre\",\"name\":\"Dimitropoulos, Harry\"},{\"affiliation\":\"Athena Research and Innovation Centre\",\"name\":\"Foufoulas, Ioannis\"},{\"affiliation\":\"University of Warsaw\",\"name\":\"Horst, Marek\"},{\"affiliation\":\"CERN\",\"name\":\"Ioannidis, Alexandros\"},{\"affiliation\":\"Athena Research and Innovation Centre\",\"name\":\"Kiatropoulou, Katerina\"},{\"affiliation\":\"Athena Research and Innovation Centre\",\"name\":\"Kokogiannaki, Argiro\", \"orcid\":\"0000-0002-3880-0244\"},{\"affiliation\":\"ISTI - CNR\",\"name\":\"La Bruzzo, Sandro\",\"orcid\":\"0000-0003-2855-1245\"},{\"affiliation\":\"ISTI - CNR\",\"name\":\"Lazzeri, Emma\"},{\"affiliation\":\"Athena Research and Innovation Centre\",\"name\":\"Lempesis, Antonis\"},{\"affiliation\":\"University of Bielefeld\",\"name\":\"Löhden, Aenne\"},{\"affiliation\":\"ISTI - CNR\",\"name\":\"Manghi, Paolo\",\"orcid\":\"0000-0001-7291-3210\"},{\"affiliation\":\"ISTI - CNR\",\"name\":\"Mannocci, Andrea\",\"orcid\":\"0000-0002-5193-7851\"},{\"affiliation\":\"ISTI - CNR\",\"name\":\"Ottonello, Enrico\"},{\"affiliation\":\"University of Bielefeld\",\"name\":\"Shirrwagen, Jochen\"},{\"affiliation\":\"University of Bielefeld\",\"name\":\"Summan, Friedrich\"}],\"description\":\"\\u003cp\\u003eThis dump provides access to the metadata records of publications, research data, projects and software that may be relevant to the Corona Virus Disease (COVID-19) research challenge. The dump contains records of the OpenAIRE COVID-19 Gateway (https://covid-19.openaire.eu/), identified via full-text mining and inference techniques applied to the OpenAIRE Research Graph (https://explore.openaire.eu/). The Graph is one of the largest Open Access collections of metadata records and link between publications, datasets, software, projects, funders, and organizations, aggregating 12,000+ scientific data sources world-wide, among which the Covid-19 data sources Zenodo COVID-19 Community, WHO (World Health Organization), BIP! FInder for COVID-19, Protein Data Bank, Dimensions, scienceOpen, and RSNA. \\u003cp\\u003eThe dump consists of a gzip file containing one json per line. Each json is compliant to the schema available at https://doi.org/10.5281/zenodo.3974226\\u003c/p\\u003e \",\"title\":\"OpenAIRE Covid-19 publications, datasets and software.\",\"upload_type\":\"dataset\",\"version\":\"1.0\"}}";
Assertions.assertEquals(200, client.sendMretadata(metadata)); Assertions.assertEquals(200, client.sendMretadata(metadata));
Assertions.assertEquals(202, client.publish()); //Assertions.assertEquals(202, client.publish());
} }

View File

@ -1,94 +0,0 @@
package eu.dnetlib.dhp.oa.graph.dump.graph;
import java.io.*;
import java.nio.charset.StandardCharsets;
import org.apache.commons.compress.archivers.ar.ArArchiveEntry;
import org.apache.commons.compress.archivers.ar.ArArchiveOutputStream;
import org.apache.commons.crypto.utils.IoUtils;
import org.apache.commons.io.IOUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.*;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import eu.dnetlib.dhp.application.ArgumentApplicationParser;
import eu.dnetlib.dhp.oa.graph.dump.APIClient;
public class MakeTar implements Serializable {
private static final Logger log = LoggerFactory.getLogger(MakeTar.class);
private final Configuration conf;
private final ArArchiveOutputStream ar;
public static void main(String[] args) throws Exception {
String jsonConfiguration = IOUtils
.toString(
MakeTar.class
.getResourceAsStream(
"/eu/dnetlib/dhp/oa/graph/dump_whole/input_maketar_parameter.json"));
final ArgumentApplicationParser parser = new ArgumentApplicationParser(jsonConfiguration);
parser.parseArgument(args);
final String hdfsPath = parser.get("hdfsPath");
log.info("hdfsPath: {}", hdfsPath);
final String hdfsNameNode = parser.get("hdfsNameNode");
log.info("nameNode: {}", hdfsNameNode);
final String inputPath = parser.get("sourcePath");
log.info("input path : {}", inputPath);
MakeTar mt = new MakeTar(hdfsPath, hdfsNameNode);
mt.execute(inputPath);
mt.close();
}
private void execute(String inputPath) throws IOException {
FileSystem fileSystem = FileSystem.get(conf);
RemoteIterator<LocatedFileStatus> fileStatusListIterator = fileSystem
.listFiles(
new Path(inputPath), true);
while (fileStatusListIterator.hasNext()) {
LocatedFileStatus fileStatus = fileStatusListIterator.next();
Path p = fileStatus.getPath();
String p_string = p.toString();
if (!p_string.endsWith("_SUCCESS")) {
String tmp = p_string.substring(0, p_string.lastIndexOf("/"));
String name = tmp.substring(tmp.lastIndexOf("/") + 1);
ar.putArchiveEntry(new ArArchiveEntry(name, fileStatus.getLen()));
InputStream is = fileSystem.open(fileStatus.getPath());
ar.write(IOUtils.toByteArray(is));
ar.closeArchiveEntry();
}
}
}
private void close() throws IOException {
ar.close();
}
public MakeTar(String hdfsPath, String hdfsNameNode) throws IOException {
this.conf = new Configuration();
this.conf.set("fs.defaultFS", hdfsNameNode);
FileSystem fileSystem = FileSystem.get(this.conf);
Path hdfsWritePath = new Path(hdfsPath);
FSDataOutputStream fsDataOutputStream = null;
if (fileSystem.exists(hdfsWritePath)) {
fsDataOutputStream = fileSystem.append(hdfsWritePath);
} else {
fsDataOutputStream = fileSystem.create(hdfsWritePath);
}
this.ar = new ArArchiveOutputStream(fsDataOutputStream.getWrappedStream());
}
}

View File

@ -1,41 +0,0 @@
package eu.dnetlib.dhp.oa.graph.dump.graph;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.LocalFileSystem;
import org.apache.hadoop.fs.Path;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test;
import java.io.IOException;
import java.nio.file.Files;
public class MakeTarTest {
private static String workingDir;
@BeforeAll
public static void beforeAll() throws IOException {
workingDir = Files
.createTempDirectory(eu.dnetlib.dhp.oa.graph.dump.graph.MakeTarTest.class.getSimpleName())
.toString();
}
@Test
public void testTar() throws IOException {
LocalFileSystem fs = FileSystem.getLocal(new Configuration());
fs
.copyFromLocalFile(
false, new Path(getClass()
.getResource("/eu/dnetlib/dhp/oa/graph/dump/zenodo/ni")
.getPath()),
new Path(workingDir + "/zenodo/ni/ni"));
fs
.copyFromLocalFile(
false, new Path(getClass()
.getResource("/eu/dnetlib/dhp/oa/graph/dump/zenodo/dh-ch")
.getPath()),
new Path(workingDir + "/zenodo/dh-ch/dh-ch"));
}
}