26 lines
852 B
Java
26 lines
852 B
Java
package eu.dnetlib.graph.raw;
|
|
|
|
import org.apache.hadoop.io.Text;
|
|
import org.apache.spark.SparkConf;
|
|
import org.apache.spark.api.java.JavaSparkContext;
|
|
import org.apache.spark.sql.SparkSession;
|
|
|
|
public class CheckPath {
|
|
public static void main(String[] args) {
|
|
final SparkConf conf= new SparkConf();
|
|
final SparkSession spark = SparkSession
|
|
.builder()
|
|
.config(conf)
|
|
.appName(CheckPath.class.getSimpleName())
|
|
.master("yarn")
|
|
.getOrCreate();
|
|
|
|
|
|
final String sp ="/data/aggregator_contents/PROD_for_BETA/mdstore/*/*";
|
|
final JavaSparkContext sc = JavaSparkContext.fromSparkContext(spark.sparkContext());
|
|
final Long total = sc.sequenceFile(sp, Text.class, Text.class).count();
|
|
System.out.println("total = " + total);
|
|
|
|
}
|
|
}
|