From 9454a46a68ea5e8a20291b167a288d654ea7aae7 Mon Sep 17 00:00:00 2001 From: Shane Kumpf Date: Wed, 4 Mar 2015 13:51:19 -0700 Subject: [PATCH] initial project layout and tests --- .idea/.name | 1 + .idea/compiler.xml | 32 +++++ .idea/copyright/profiles_settings.xml | 3 + .idea/encodings.xml | 6 + .idea/misc.xml | 178 ++++++++++++++++++++++++++ .idea/modules.xml | 8 ++ .idea/scopes/scope_settings.xml | 5 + .idea/vcs.xml | 6 + pom.xml | 7 + sequencefile-examples.iml | 113 ++++++++++++++++ test/main/java/SequenceFileTest.java | 88 +++++++++++++ 11 files changed, 447 insertions(+) create mode 100644 .idea/.name create mode 100644 .idea/compiler.xml create mode 100644 .idea/copyright/profiles_settings.xml create mode 100644 .idea/encodings.xml create mode 100644 .idea/misc.xml create mode 100644 .idea/modules.xml create mode 100644 .idea/scopes/scope_settings.xml create mode 100644 .idea/vcs.xml create mode 100644 sequencefile-examples.iml create mode 100644 test/main/java/SequenceFileTest.java diff --git a/.idea/.name b/.idea/.name new file mode 100644 index 0000000..b6b65c4 --- /dev/null +++ b/.idea/.name @@ -0,0 +1 @@ +sequencefile-examples \ No newline at end of file diff --git a/.idea/compiler.xml b/.idea/compiler.xml new file mode 100644 index 0000000..065602a --- /dev/null +++ b/.idea/compiler.xml @@ -0,0 +1,32 @@ + + + + + \ No newline at end of file diff --git a/.idea/copyright/profiles_settings.xml b/.idea/copyright/profiles_settings.xml new file mode 100644 index 0000000..e7bedf3 --- /dev/null +++ b/.idea/copyright/profiles_settings.xml @@ -0,0 +1,3 @@ + + + \ No newline at end of file diff --git a/.idea/encodings.xml b/.idea/encodings.xml new file mode 100644 index 0000000..d44269f --- /dev/null +++ b/.idea/encodings.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/.idea/misc.xml b/.idea/misc.xml new file mode 100644 index 0000000..90625bb --- /dev/null +++ b/.idea/misc.xml @@ -0,0 +1,178 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + localhost + 5050 + + + + + + + 1.7 + + + + + + + + \ No newline at end of file diff --git a/.idea/modules.xml b/.idea/modules.xml new file mode 100644 index 0000000..f516c2e --- /dev/null +++ b/.idea/modules.xml @@ -0,0 +1,8 @@ + + + + + + + + \ No newline at end of file diff --git a/.idea/scopes/scope_settings.xml b/.idea/scopes/scope_settings.xml new file mode 100644 index 0000000..922003b --- /dev/null +++ b/.idea/scopes/scope_settings.xml @@ -0,0 +1,5 @@ + + + + \ No newline at end of file diff --git a/.idea/vcs.xml b/.idea/vcs.xml new file mode 100644 index 0000000..35eb1dd --- /dev/null +++ b/.idea/vcs.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/pom.xml b/pom.xml index efb2c7c..39f9e18 100644 --- a/pom.xml +++ b/pom.xml @@ -12,6 +12,7 @@ 0.0.12 2.6.0.2.2.0.0-2041 6.1.26 + 4.11 @@ -48,6 +49,12 @@ ${hadoop-mini-clusters.version} test + + junit + junit + ${junit.version} + test + org.apache.hadoop hadoop-client diff --git a/sequencefile-examples.iml b/sequencefile-examples.iml new file mode 100644 index 0000000..4cc6502 --- /dev/null +++ b/sequencefile-examples.iml @@ -0,0 +1,113 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/test/main/java/SequenceFileTest.java b/test/main/java/SequenceFileTest.java new file mode 100644 index 0000000..13de248 --- /dev/null +++ b/test/main/java/SequenceFileTest.java @@ -0,0 +1,88 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import com.github.sakserv.minicluster.config.ConfigVars; +import com.github.sakserv.minicluster.impl.HdfsLocalCluster; +import org.apache.hadoop.conf.Configuration; + +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.io.IntWritable; +import org.apache.hadoop.io.SequenceFile; +import org.apache.hadoop.io.Text; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.io.IOException; + + +public class SequenceFileTest { + + HdfsLocalCluster hdfsLocalCluster; + + @Before + public void setUp() { + // Start HDFS + hdfsLocalCluster = new HdfsLocalCluster.Builder() + .setHdfsNamenodePort(12345) + .setHdfsTempDir("embedded_hdfs") + .setHdfsNumDatanodes(1) + .setHdfsEnablePermissions(false) + .setHdfsFormat(true) + .setHdfsConfig(new Configuration()) + .build(); + hdfsLocalCluster.start(); + } + + @After + public void tearDown() { + hdfsLocalCluster.stop(); + } + + @Test + public void testSequenceFile() throws IOException { + Configuration conf = hdfsLocalCluster.getHdfsConfig(); + FileSystem fs = hdfsLocalCluster.getHdfsFileSystemHandle(); + + + Path seqFileDir = new Path("/tmp/seq_file_test"); + fs.mkdirs(seqFileDir); + + Path seqFilePath = new Path(seqFileDir + "/file.seq"); + + SequenceFile.Writer writer = SequenceFile.createWriter(conf, + SequenceFile.Writer.file(seqFilePath), SequenceFile.Writer.keyClass(Text.class), + SequenceFile.Writer.valueClass(IntWritable.class)); + + writer.append(new Text("key1"), new IntWritable(1)); + writer.append(new Text("key2"), new IntWritable(2)); + + writer.close(); + + SequenceFile.Reader reader = new SequenceFile.Reader(conf, + SequenceFile.Reader.file(seqFilePath)); + + Text key = new Text(); + IntWritable val = new IntWritable(); + + while (reader.next(key, val)) { + System.out.println("SEQFILE KEY: " + key + "\t" + val); + } + + fs.mkdirs(new Path("/tmp/seq_file_test")); + + reader.close(); + } +}