forked from D-Net/dnet-hadoop
changed test to upload the dump for covid-19 community
This commit is contained in:
parent
7dfd56df9d
commit
0603ec4757
|
|
@ -10,7 +10,7 @@ import org.junit.jupiter.api.Test;
|
||||||
public class ZenodoAPIClientTest {
|
public class ZenodoAPIClientTest {
|
||||||
|
|
||||||
private final String URL_STRING = "https://sandbox.zenodo.org/api/deposit/depositions";
|
private final String URL_STRING = "https://sandbox.zenodo.org/api/deposit/depositions";
|
||||||
private final String ACCESS_TOKEN = "";
|
private final String ACCESS_TOKEN = "5ImUj0VC1ICg4ifK5dc3AGzJhcfAB4osxrFlsr8WxHXxjaYgCE0hY8HZcDoe";
|
||||||
|
|
||||||
private final String CONCEPT_REC_ID = "657113";
|
private final String CONCEPT_REC_ID = "657113";
|
||||||
|
|
||||||
|
|
@ -22,18 +22,18 @@ public class ZenodoAPIClientTest {
|
||||||
Assertions.assertEquals(201, client.newDeposition());
|
Assertions.assertEquals(201, client.newDeposition());
|
||||||
|
|
||||||
File file = new File(getClass()
|
File file = new File(getClass()
|
||||||
.getResource("/eu/dnetlib/dhp/common/api/newDeposition")
|
.getResource("/eu/dnetlib/dhp/common/api/COVID-19.json.gz")
|
||||||
.getPath());
|
.getPath());
|
||||||
|
|
||||||
InputStream is = new FileInputStream(file);
|
InputStream is = new FileInputStream(file);
|
||||||
|
|
||||||
Assertions.assertEquals(200, client.uploadIS(is, "first_deposition", file.length()));
|
Assertions.assertEquals(200, client.uploadIS(is, "COVID-19.json.gz", file.length()));
|
||||||
|
|
||||||
String metadata = "{\"metadata\":{\"access_right\":\"open\",\"creators\":[{\"affiliation\":\"ISTI - CNR\",\"name\":\"Baglioni, Miriam\",\"orcid\":\"0000-0002-2273-9004\"}],\"description\":\"\\u003cp\\u003eThis is a test for the automatic upload of files in a new deposition\\u003c/p\\u003e \",\"title\":\"Test.\",\"upload_type\":\"other\",\"version\":\"1.0\"}}";
|
String metadata = "{\"metadata\":{\"access_right\":\"open\",\"communities\":[{\"identifier\":\"openaire-research-graph\"}],\"creators\":[{\"affiliation\":\"ISTI - CNR\",\"name\":\"Bardi, Alessia\",\"orcid\":\"0000-0002-1112-1292\"},{\"affiliation\":\"eifl\", \"name\":\"Kuchma, Iryna\"},{\"affiliation\":\"BIH\", \"name\":\"Brobov, Evgeny\"},{\"affiliation\":\"GIDIF RBM\", \"name\":\"Truccolo, Ivana\"},{\"affiliation\":\"unesp\", \"name\":\"Monteiro, Elizabete\"},{\"affiliation\":\"und\", \"name\":\"Casalegno, Carlotta\"},{\"affiliation\":\"CARL ABRC\", \"name\":\"Clary, Erin\"},{\"affiliation\":\"The University of Edimburgh\", \"name\":\"Romanowski, Andrew\"},{\"affiliation\":\"ISTI - CNR\", \"name\":\"Pavone, Gina\"},{\"affiliation\":\"ISTI - CNR\", \"name\":\"Artini, Michele\"},{\"affiliation\":\"ISTI - CNR\",\"name\":\"Atzori, Claudio\",\"orcid\":\"0000-0001-9613-6639\"},{\"affiliation\":\"University of Bielefeld\",\"name\":\"Bäcker, Amelie\",\"orcid\":\"0000-0001-6015-2063\"},{\"affiliation\":\"ISTI - CNR\",\"name\":\"Baglioni, Miriam\",\"orcid\":\"0000-0002-2273-9004\"},{\"affiliation\":\"University of Bielefeld\",\"name\":\"Czerniak, Andreas\",\"orcid\":\"0000-0003-3883-4169\"},{\"affiliation\":\"ISTI - CNR\",\"name\":\"De Bonis, Michele\"},{\"affiliation\":\"Athena Research and Innovation Centre\",\"name\":\"Dimitropoulos, Harry\"},{\"affiliation\":\"Athena Research and Innovation Centre\",\"name\":\"Foufoulas, Ioannis\"},{\"affiliation\":\"University of Warsaw\",\"name\":\"Horst, Marek\"},{\"affiliation\":\"CERN\",\"name\":\"Ioannidis, Alexandros\"},{\"affiliation\":\"Athena Research and Innovation Centre\",\"name\":\"Kiatropoulou, Katerina\"},{\"affiliation\":\"Athena Research and Innovation Centre\",\"name\":\"Kokogiannaki, Argiro\", \"orcid\":\"0000-0002-3880-0244\"},{\"affiliation\":\"ISTI - CNR\",\"name\":\"La Bruzzo, Sandro\",\"orcid\":\"0000-0003-2855-1245\"},{\"affiliation\":\"ISTI - CNR\",\"name\":\"Lazzeri, Emma\"},{\"affiliation\":\"Athena Research and Innovation Centre\",\"name\":\"Lempesis, Antonis\"},{\"affiliation\":\"University of Bielefeld\",\"name\":\"Löhden, Aenne\"},{\"affiliation\":\"ISTI - CNR\",\"name\":\"Manghi, Paolo\",\"orcid\":\"0000-0001-7291-3210\"},{\"affiliation\":\"ISTI - CNR\",\"name\":\"Mannocci, Andrea\",\"orcid\":\"0000-0002-5193-7851\"},{\"affiliation\":\"ISTI - CNR\",\"name\":\"Ottonello, Enrico\"},{\"affiliation\":\"University of Bielefeld\",\"name\":\"Shirrwagen, Jochen\"},{\"affiliation\":\"University of Bielefeld\",\"name\":\"Summan, Friedrich\"}],\"description\":\"\\u003cp\\u003eThis dump provides access to the metadata records of publications, research data, projects and software that may be relevant to the Corona Virus Disease (COVID-19) research challenge. The dump contains records of the OpenAIRE COVID-19 Gateway (https://covid-19.openaire.eu/), identified via full-text mining and inference techniques applied to the OpenAIRE Research Graph (https://explore.openaire.eu/). The Graph is one of the largest Open Access collections of metadata records and link between publications, datasets, software, projects, funders, and organizations, aggregating 12,000+ scientific data sources world-wide, among which the Covid-19 data sources Zenodo COVID-19 Community, WHO (World Health Organization), BIP! FInder for COVID-19, Protein Data Bank, Dimensions, scienceOpen, and RSNA. \\u003cp\\u003eThe dump consists of a gzip file containing one json per line. Each json is compliant to the schema available at https://doi.org/10.5281/zenodo.3974226\\u003c/p\\u003e \",\"title\":\"OpenAIRE Covid-19 publications, datasets and software.\",\"upload_type\":\"dataset\",\"version\":\"1.0\"}}";
|
||||||
|
|
||||||
Assertions.assertEquals(200, client.sendMretadata(metadata));
|
Assertions.assertEquals(200, client.sendMretadata(metadata));
|
||||||
|
|
||||||
Assertions.assertEquals(202, client.publish());
|
//Assertions.assertEquals(202, client.publish());
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -1,94 +0,0 @@
|
||||||
|
|
||||||
package eu.dnetlib.dhp.oa.graph.dump.graph;
|
|
||||||
|
|
||||||
import java.io.*;
|
|
||||||
import java.nio.charset.StandardCharsets;
|
|
||||||
|
|
||||||
import org.apache.commons.compress.archivers.ar.ArArchiveEntry;
|
|
||||||
import org.apache.commons.compress.archivers.ar.ArArchiveOutputStream;
|
|
||||||
import org.apache.commons.crypto.utils.IoUtils;
|
|
||||||
import org.apache.commons.io.IOUtils;
|
|
||||||
import org.apache.hadoop.conf.Configuration;
|
|
||||||
import org.apache.hadoop.fs.*;
|
|
||||||
import org.slf4j.Logger;
|
|
||||||
import org.slf4j.LoggerFactory;
|
|
||||||
|
|
||||||
import eu.dnetlib.dhp.application.ArgumentApplicationParser;
|
|
||||||
import eu.dnetlib.dhp.oa.graph.dump.APIClient;
|
|
||||||
|
|
||||||
public class MakeTar implements Serializable {
|
|
||||||
|
|
||||||
private static final Logger log = LoggerFactory.getLogger(MakeTar.class);
|
|
||||||
private final Configuration conf;
|
|
||||||
private final ArArchiveOutputStream ar;
|
|
||||||
|
|
||||||
public static void main(String[] args) throws Exception {
|
|
||||||
String jsonConfiguration = IOUtils
|
|
||||||
.toString(
|
|
||||||
MakeTar.class
|
|
||||||
.getResourceAsStream(
|
|
||||||
"/eu/dnetlib/dhp/oa/graph/dump_whole/input_maketar_parameter.json"));
|
|
||||||
|
|
||||||
final ArgumentApplicationParser parser = new ArgumentApplicationParser(jsonConfiguration);
|
|
||||||
parser.parseArgument(args);
|
|
||||||
|
|
||||||
final String hdfsPath = parser.get("hdfsPath");
|
|
||||||
log.info("hdfsPath: {}", hdfsPath);
|
|
||||||
|
|
||||||
final String hdfsNameNode = parser.get("hdfsNameNode");
|
|
||||||
log.info("nameNode: {}", hdfsNameNode);
|
|
||||||
|
|
||||||
final String inputPath = parser.get("sourcePath");
|
|
||||||
log.info("input path : {}", inputPath);
|
|
||||||
|
|
||||||
MakeTar mt = new MakeTar(hdfsPath, hdfsNameNode);
|
|
||||||
mt.execute(inputPath);
|
|
||||||
mt.close();
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
private void execute(String inputPath) throws IOException {
|
|
||||||
FileSystem fileSystem = FileSystem.get(conf);
|
|
||||||
|
|
||||||
RemoteIterator<LocatedFileStatus> fileStatusListIterator = fileSystem
|
|
||||||
.listFiles(
|
|
||||||
new Path(inputPath), true);
|
|
||||||
|
|
||||||
while (fileStatusListIterator.hasNext()) {
|
|
||||||
LocatedFileStatus fileStatus = fileStatusListIterator.next();
|
|
||||||
|
|
||||||
Path p = fileStatus.getPath();
|
|
||||||
String p_string = p.toString();
|
|
||||||
if (!p_string.endsWith("_SUCCESS")) {
|
|
||||||
String tmp = p_string.substring(0, p_string.lastIndexOf("/"));
|
|
||||||
String name = tmp.substring(tmp.lastIndexOf("/") + 1);
|
|
||||||
ar.putArchiveEntry(new ArArchiveEntry(name, fileStatus.getLen()));
|
|
||||||
InputStream is = fileSystem.open(fileStatus.getPath());
|
|
||||||
ar.write(IOUtils.toByteArray(is));
|
|
||||||
ar.closeArchiveEntry();
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private void close() throws IOException {
|
|
||||||
ar.close();
|
|
||||||
}
|
|
||||||
|
|
||||||
public MakeTar(String hdfsPath, String hdfsNameNode) throws IOException {
|
|
||||||
this.conf = new Configuration();
|
|
||||||
this.conf.set("fs.defaultFS", hdfsNameNode);
|
|
||||||
FileSystem fileSystem = FileSystem.get(this.conf);
|
|
||||||
Path hdfsWritePath = new Path(hdfsPath);
|
|
||||||
FSDataOutputStream fsDataOutputStream = null;
|
|
||||||
if (fileSystem.exists(hdfsWritePath)) {
|
|
||||||
fsDataOutputStream = fileSystem.append(hdfsWritePath);
|
|
||||||
} else {
|
|
||||||
fsDataOutputStream = fileSystem.create(hdfsWritePath);
|
|
||||||
}
|
|
||||||
|
|
||||||
this.ar = new ArArchiveOutputStream(fsDataOutputStream.getWrappedStream());
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
@ -1,41 +0,0 @@
|
||||||
package eu.dnetlib.dhp.oa.graph.dump.graph;
|
|
||||||
|
|
||||||
import org.apache.hadoop.conf.Configuration;
|
|
||||||
import org.apache.hadoop.fs.FileSystem;
|
|
||||||
import org.apache.hadoop.fs.LocalFileSystem;
|
|
||||||
import org.apache.hadoop.fs.Path;
|
|
||||||
import org.junit.jupiter.api.BeforeAll;
|
|
||||||
import org.junit.jupiter.api.Test;
|
|
||||||
|
|
||||||
import java.io.IOException;
|
|
||||||
import java.nio.file.Files;
|
|
||||||
|
|
||||||
public class MakeTarTest {
|
|
||||||
private static String workingDir;
|
|
||||||
|
|
||||||
@BeforeAll
|
|
||||||
public static void beforeAll() throws IOException {
|
|
||||||
workingDir = Files
|
|
||||||
.createTempDirectory(eu.dnetlib.dhp.oa.graph.dump.graph.MakeTarTest.class.getSimpleName())
|
|
||||||
.toString();
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
public void testTar() throws IOException {
|
|
||||||
LocalFileSystem fs = FileSystem.getLocal(new Configuration());
|
|
||||||
|
|
||||||
fs
|
|
||||||
.copyFromLocalFile(
|
|
||||||
false, new Path(getClass()
|
|
||||||
.getResource("/eu/dnetlib/dhp/oa/graph/dump/zenodo/ni")
|
|
||||||
.getPath()),
|
|
||||||
new Path(workingDir + "/zenodo/ni/ni"));
|
|
||||||
fs
|
|
||||||
.copyFromLocalFile(
|
|
||||||
false, new Path(getClass()
|
|
||||||
.getResource("/eu/dnetlib/dhp/oa/graph/dump/zenodo/dh-ch")
|
|
||||||
.getPath()),
|
|
||||||
new Path(workingDir + "/zenodo/dh-ch/dh-ch"));
|
|
||||||
|
|
||||||
}
|
|
||||||
}
|
|
||||||
Loading…
Reference in New Issue