From 0603ec4757662a4bf48d1613eca5d3ebf07fb165 Mon Sep 17 00:00:00 2001 From: "miriam.baglioni" Date: Tue, 11 Aug 2020 15:43:25 +0200 Subject: [PATCH] changed test to upload the dump for covid-19 community --- .../dhp/common/api/ZenodoAPIClientTest.java | 10 +- .../dhp/oa/graph/dump/graph/MakeTar.java | 94 ------------------- .../dhp/oa/graph/dump/graph/MakeTarTest.java | 41 -------- 3 files changed, 5 insertions(+), 140 deletions(-) delete mode 100644 dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/dump/graph/MakeTar.java delete mode 100644 dhp-workflows/dhp-graph-mapper/src/test/java/eu/dnetlib/dhp/oa/graph/dump/graph/MakeTarTest.java diff --git a/dhp-common/src/test/java/eu/dnetlib/dhp/common/api/ZenodoAPIClientTest.java b/dhp-common/src/test/java/eu/dnetlib/dhp/common/api/ZenodoAPIClientTest.java index a091e2668..fdbeec7b9 100644 --- a/dhp-common/src/test/java/eu/dnetlib/dhp/common/api/ZenodoAPIClientTest.java +++ b/dhp-common/src/test/java/eu/dnetlib/dhp/common/api/ZenodoAPIClientTest.java @@ -10,7 +10,7 @@ import org.junit.jupiter.api.Test; public class ZenodoAPIClientTest { private final String URL_STRING = "https://sandbox.zenodo.org/api/deposit/depositions"; - private final String ACCESS_TOKEN = ""; + private final String ACCESS_TOKEN = "5ImUj0VC1ICg4ifK5dc3AGzJhcfAB4osxrFlsr8WxHXxjaYgCE0hY8HZcDoe"; private final String CONCEPT_REC_ID = "657113"; @@ -22,18 +22,18 @@ public class ZenodoAPIClientTest { Assertions.assertEquals(201, client.newDeposition()); File file = new File(getClass() - .getResource("/eu/dnetlib/dhp/common/api/newDeposition") + .getResource("/eu/dnetlib/dhp/common/api/COVID-19.json.gz") .getPath()); InputStream is = new FileInputStream(file); - Assertions.assertEquals(200, client.uploadIS(is, "first_deposition", file.length())); + Assertions.assertEquals(200, client.uploadIS(is, "COVID-19.json.gz", file.length())); - String metadata = "{\"metadata\":{\"access_right\":\"open\",\"creators\":[{\"affiliation\":\"ISTI - CNR\",\"name\":\"Baglioni, Miriam\",\"orcid\":\"0000-0002-2273-9004\"}],\"description\":\"\\u003cp\\u003eThis is a test for the automatic upload of files in a new deposition\\u003c/p\\u003e \",\"title\":\"Test.\",\"upload_type\":\"other\",\"version\":\"1.0\"}}"; + String metadata = "{\"metadata\":{\"access_right\":\"open\",\"communities\":[{\"identifier\":\"openaire-research-graph\"}],\"creators\":[{\"affiliation\":\"ISTI - CNR\",\"name\":\"Bardi, Alessia\",\"orcid\":\"0000-0002-1112-1292\"},{\"affiliation\":\"eifl\", \"name\":\"Kuchma, Iryna\"},{\"affiliation\":\"BIH\", \"name\":\"Brobov, Evgeny\"},{\"affiliation\":\"GIDIF RBM\", \"name\":\"Truccolo, Ivana\"},{\"affiliation\":\"unesp\", \"name\":\"Monteiro, Elizabete\"},{\"affiliation\":\"und\", \"name\":\"Casalegno, Carlotta\"},{\"affiliation\":\"CARL ABRC\", \"name\":\"Clary, Erin\"},{\"affiliation\":\"The University of Edimburgh\", \"name\":\"Romanowski, Andrew\"},{\"affiliation\":\"ISTI - CNR\", \"name\":\"Pavone, Gina\"},{\"affiliation\":\"ISTI - CNR\", \"name\":\"Artini, Michele\"},{\"affiliation\":\"ISTI - CNR\",\"name\":\"Atzori, Claudio\",\"orcid\":\"0000-0001-9613-6639\"},{\"affiliation\":\"University of Bielefeld\",\"name\":\"Bäcker, Amelie\",\"orcid\":\"0000-0001-6015-2063\"},{\"affiliation\":\"ISTI - CNR\",\"name\":\"Baglioni, Miriam\",\"orcid\":\"0000-0002-2273-9004\"},{\"affiliation\":\"University of Bielefeld\",\"name\":\"Czerniak, Andreas\",\"orcid\":\"0000-0003-3883-4169\"},{\"affiliation\":\"ISTI - CNR\",\"name\":\"De Bonis, Michele\"},{\"affiliation\":\"Athena Research and Innovation Centre\",\"name\":\"Dimitropoulos, Harry\"},{\"affiliation\":\"Athena Research and Innovation Centre\",\"name\":\"Foufoulas, Ioannis\"},{\"affiliation\":\"University of Warsaw\",\"name\":\"Horst, Marek\"},{\"affiliation\":\"CERN\",\"name\":\"Ioannidis, Alexandros\"},{\"affiliation\":\"Athena Research and Innovation Centre\",\"name\":\"Kiatropoulou, Katerina\"},{\"affiliation\":\"Athena Research and Innovation Centre\",\"name\":\"Kokogiannaki, Argiro\", \"orcid\":\"0000-0002-3880-0244\"},{\"affiliation\":\"ISTI - CNR\",\"name\":\"La Bruzzo, Sandro\",\"orcid\":\"0000-0003-2855-1245\"},{\"affiliation\":\"ISTI - CNR\",\"name\":\"Lazzeri, Emma\"},{\"affiliation\":\"Athena Research and Innovation Centre\",\"name\":\"Lempesis, Antonis\"},{\"affiliation\":\"University of Bielefeld\",\"name\":\"Löhden, Aenne\"},{\"affiliation\":\"ISTI - CNR\",\"name\":\"Manghi, Paolo\",\"orcid\":\"0000-0001-7291-3210\"},{\"affiliation\":\"ISTI - CNR\",\"name\":\"Mannocci, Andrea\",\"orcid\":\"0000-0002-5193-7851\"},{\"affiliation\":\"ISTI - CNR\",\"name\":\"Ottonello, Enrico\"},{\"affiliation\":\"University of Bielefeld\",\"name\":\"Shirrwagen, Jochen\"},{\"affiliation\":\"University of Bielefeld\",\"name\":\"Summan, Friedrich\"}],\"description\":\"\\u003cp\\u003eThis dump provides access to the metadata records of publications, research data, projects and software that may be relevant to the Corona Virus Disease (COVID-19) research challenge. The dump contains records of the OpenAIRE COVID-19 Gateway (https://covid-19.openaire.eu/), identified via full-text mining and inference techniques applied to the OpenAIRE Research Graph (https://explore.openaire.eu/). The Graph is one of the largest Open Access collections of metadata records and link between publications, datasets, software, projects, funders, and organizations, aggregating 12,000+ scientific data sources world-wide, among which the Covid-19 data sources Zenodo COVID-19 Community, WHO (World Health Organization), BIP! FInder for COVID-19, Protein Data Bank, Dimensions, scienceOpen, and RSNA. \\u003cp\\u003eThe dump consists of a gzip file containing one json per line. Each json is compliant to the schema available at https://doi.org/10.5281/zenodo.3974226\\u003c/p\\u003e \",\"title\":\"OpenAIRE Covid-19 publications, datasets and software.\",\"upload_type\":\"dataset\",\"version\":\"1.0\"}}"; Assertions.assertEquals(200, client.sendMretadata(metadata)); - Assertions.assertEquals(202, client.publish()); + //Assertions.assertEquals(202, client.publish()); } diff --git a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/dump/graph/MakeTar.java b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/dump/graph/MakeTar.java deleted file mode 100644 index 33b1f035a..000000000 --- a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/dump/graph/MakeTar.java +++ /dev/null @@ -1,94 +0,0 @@ - -package eu.dnetlib.dhp.oa.graph.dump.graph; - -import java.io.*; -import java.nio.charset.StandardCharsets; - -import org.apache.commons.compress.archivers.ar.ArArchiveEntry; -import org.apache.commons.compress.archivers.ar.ArArchiveOutputStream; -import org.apache.commons.crypto.utils.IoUtils; -import org.apache.commons.io.IOUtils; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.*; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import eu.dnetlib.dhp.application.ArgumentApplicationParser; -import eu.dnetlib.dhp.oa.graph.dump.APIClient; - -public class MakeTar implements Serializable { - - private static final Logger log = LoggerFactory.getLogger(MakeTar.class); - private final Configuration conf; - private final ArArchiveOutputStream ar; - - public static void main(String[] args) throws Exception { - String jsonConfiguration = IOUtils - .toString( - MakeTar.class - .getResourceAsStream( - "/eu/dnetlib/dhp/oa/graph/dump_whole/input_maketar_parameter.json")); - - final ArgumentApplicationParser parser = new ArgumentApplicationParser(jsonConfiguration); - parser.parseArgument(args); - - final String hdfsPath = parser.get("hdfsPath"); - log.info("hdfsPath: {}", hdfsPath); - - final String hdfsNameNode = parser.get("hdfsNameNode"); - log.info("nameNode: {}", hdfsNameNode); - - final String inputPath = parser.get("sourcePath"); - log.info("input path : {}", inputPath); - - MakeTar mt = new MakeTar(hdfsPath, hdfsNameNode); - mt.execute(inputPath); - mt.close(); - - } - - private void execute(String inputPath) throws IOException { - FileSystem fileSystem = FileSystem.get(conf); - - RemoteIterator fileStatusListIterator = fileSystem - .listFiles( - new Path(inputPath), true); - - while (fileStatusListIterator.hasNext()) { - LocatedFileStatus fileStatus = fileStatusListIterator.next(); - - Path p = fileStatus.getPath(); - String p_string = p.toString(); - if (!p_string.endsWith("_SUCCESS")) { - String tmp = p_string.substring(0, p_string.lastIndexOf("/")); - String name = tmp.substring(tmp.lastIndexOf("/") + 1); - ar.putArchiveEntry(new ArArchiveEntry(name, fileStatus.getLen())); - InputStream is = fileSystem.open(fileStatus.getPath()); - ar.write(IOUtils.toByteArray(is)); - ar.closeArchiveEntry(); - } - - } - } - - private void close() throws IOException { - ar.close(); - } - - public MakeTar(String hdfsPath, String hdfsNameNode) throws IOException { - this.conf = new Configuration(); - this.conf.set("fs.defaultFS", hdfsNameNode); - FileSystem fileSystem = FileSystem.get(this.conf); - Path hdfsWritePath = new Path(hdfsPath); - FSDataOutputStream fsDataOutputStream = null; - if (fileSystem.exists(hdfsWritePath)) { - fsDataOutputStream = fileSystem.append(hdfsWritePath); - } else { - fsDataOutputStream = fileSystem.create(hdfsWritePath); - } - - this.ar = new ArArchiveOutputStream(fsDataOutputStream.getWrappedStream()); - - } - -} diff --git a/dhp-workflows/dhp-graph-mapper/src/test/java/eu/dnetlib/dhp/oa/graph/dump/graph/MakeTarTest.java b/dhp-workflows/dhp-graph-mapper/src/test/java/eu/dnetlib/dhp/oa/graph/dump/graph/MakeTarTest.java deleted file mode 100644 index b124fd148..000000000 --- a/dhp-workflows/dhp-graph-mapper/src/test/java/eu/dnetlib/dhp/oa/graph/dump/graph/MakeTarTest.java +++ /dev/null @@ -1,41 +0,0 @@ -package eu.dnetlib.dhp.oa.graph.dump.graph; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.LocalFileSystem; -import org.apache.hadoop.fs.Path; -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.Test; - -import java.io.IOException; -import java.nio.file.Files; - -public class MakeTarTest { - private static String workingDir; - - @BeforeAll - public static void beforeAll() throws IOException { - workingDir = Files - .createTempDirectory(eu.dnetlib.dhp.oa.graph.dump.graph.MakeTarTest.class.getSimpleName()) - .toString(); - } - - @Test - public void testTar() throws IOException { - LocalFileSystem fs = FileSystem.getLocal(new Configuration()); - - fs - .copyFromLocalFile( - false, new Path(getClass() - .getResource("/eu/dnetlib/dhp/oa/graph/dump/zenodo/ni") - .getPath()), - new Path(workingDir + "/zenodo/ni/ni")); - fs - .copyFromLocalFile( - false, new Path(getClass() - .getResource("/eu/dnetlib/dhp/oa/graph/dump/zenodo/dh-ch") - .getPath()), - new Path(workingDir + "/zenodo/dh-ch/dh-ch")); - - } -}