2020-05-12 16:57:43 +02:00
|
|
|
|
|
|
|
package eu.dnetlib.doiboost.orcid;
|
|
|
|
|
|
|
|
import static eu.dnetlib.dhp.common.SparkSessionSupport.runWithSparkSession;
|
|
|
|
|
2020-05-15 19:49:26 +02:00
|
|
|
import java.io.IOException;
|
|
|
|
import java.text.SimpleDateFormat;
|
|
|
|
import java.util.Date;
|
2020-05-18 19:51:29 +02:00
|
|
|
import java.util.List;
|
2020-05-12 16:57:43 +02:00
|
|
|
import java.util.Optional;
|
|
|
|
|
|
|
|
import org.apache.commons.io.IOUtils;
|
2020-05-15 19:49:26 +02:00
|
|
|
import org.apache.hadoop.io.Text;
|
|
|
|
import org.apache.http.client.methods.CloseableHttpResponse;
|
|
|
|
import org.apache.http.client.methods.HttpGet;
|
|
|
|
import org.apache.http.impl.client.CloseableHttpClient;
|
|
|
|
import org.apache.http.impl.client.HttpClients;
|
2020-05-12 16:57:43 +02:00
|
|
|
import org.apache.spark.SparkConf;
|
2020-05-15 19:49:26 +02:00
|
|
|
import org.apache.spark.api.java.JavaRDD;
|
|
|
|
import org.apache.spark.api.java.JavaSparkContext;
|
|
|
|
import org.apache.spark.api.java.function.Function;
|
|
|
|
import org.apache.spark.sql.Encoders;
|
|
|
|
import org.apache.spark.sql.SaveMode;
|
2020-05-18 19:51:29 +02:00
|
|
|
import org.apache.spark.util.LongAccumulator;
|
2020-05-15 19:49:26 +02:00
|
|
|
import org.mortbay.log.Log;
|
2020-05-12 16:57:43 +02:00
|
|
|
import org.slf4j.Logger;
|
|
|
|
import org.slf4j.LoggerFactory;
|
|
|
|
|
|
|
|
import eu.dnetlib.dhp.application.ArgumentApplicationParser;
|
2020-05-15 19:49:26 +02:00
|
|
|
import eu.dnetlib.doiboost.orcid.model.DownloadedRecordData;
|
|
|
|
import scala.Tuple2;
|
2020-05-12 16:57:43 +02:00
|
|
|
|
|
|
|
public class SparkOrcidGenerateAuthors {
|
|
|
|
|
2020-05-15 19:49:26 +02:00
|
|
|
static final String DATE_FORMAT = "yyyy-MM-dd HH:mm:ss";
|
|
|
|
static final String lastUpdate = "2019-09-30 00:00:00";
|
|
|
|
|
|
|
|
public static void main(String[] args) throws IOException, Exception {
|
2020-05-12 16:57:43 +02:00
|
|
|
Logger logger = LoggerFactory.getLogger(SparkOrcidGenerateAuthors.class);
|
|
|
|
logger.info("[ SparkOrcidGenerateAuthors STARTED]");
|
|
|
|
|
2020-05-15 19:49:26 +02:00
|
|
|
final ArgumentApplicationParser parser = new ArgumentApplicationParser(
|
|
|
|
IOUtils
|
|
|
|
.toString(
|
|
|
|
SparkOrcidGenerateAuthors.class
|
|
|
|
.getResourceAsStream(
|
|
|
|
"/eu/dnetlib/dhp/doiboost/gen_orcid_authors_parameters.json")));
|
|
|
|
parser.parseArgument(args);
|
|
|
|
Boolean isSparkSessionManaged = Optional
|
|
|
|
.ofNullable(parser.get("isSparkSessionManaged"))
|
|
|
|
.map(Boolean::valueOf)
|
|
|
|
.orElse(Boolean.TRUE);
|
|
|
|
logger.info("isSparkSessionManaged: {}", isSparkSessionManaged);
|
|
|
|
final String workingPath = parser.get("workingPath");
|
|
|
|
logger.info("workingPath: ", workingPath);
|
|
|
|
final String outputAuthorsPath = parser.get("outputAuthorsPath");
|
|
|
|
logger.info("outputAuthorsPath: ", outputAuthorsPath);
|
|
|
|
final String token = parser.get("token");
|
2020-05-12 16:57:43 +02:00
|
|
|
|
2020-05-15 19:49:26 +02:00
|
|
|
SparkConf conf = new SparkConf();
|
|
|
|
runWithSparkSession(
|
|
|
|
conf,
|
|
|
|
isSparkSessionManaged,
|
|
|
|
spark -> {
|
|
|
|
JavaSparkContext sc = JavaSparkContext.fromSparkContext(spark.sparkContext());
|
2020-05-18 19:51:29 +02:00
|
|
|
|
|
|
|
LongAccumulator parsedRecordsAcc = sc.sc().longAccumulator("parsedRecords");
|
|
|
|
LongAccumulator modifiedRecordsAcc = sc.sc().longAccumulator("modifiedRecords");
|
|
|
|
LongAccumulator downloadedRecordsAcc = sc.sc().longAccumulator("downloadedRecords");
|
|
|
|
LongAccumulator alreadyDownloadedRecords = sc.sc().longAccumulator("alreadyDownloadedRecords");
|
|
|
|
JavaRDD<String> lamdaFileRDD = sc.textFile(workingPath + "lamdafiles");
|
|
|
|
|
|
|
|
JavaRDD<String> downloadedRDD = sc.textFile(workingPath + "downloaded");
|
|
|
|
Function<String, String> getOrcidIdFunction = line -> {
|
|
|
|
try {
|
|
|
|
String[] values = line.split(",");
|
|
|
|
return values[0].substring(1);
|
|
|
|
} catch (Exception e) {
|
|
|
|
return new String("");
|
|
|
|
}
|
|
|
|
};
|
|
|
|
List<String> downloadedRecords = downloadedRDD.map(getOrcidIdFunction).collect();
|
|
|
|
|
2020-05-15 19:49:26 +02:00
|
|
|
Function<String, Boolean> isModifiedAfterFilter = line -> {
|
|
|
|
String[] values = line.split(",");
|
|
|
|
String orcidId = values[0];
|
2020-05-18 19:51:29 +02:00
|
|
|
parsedRecordsAcc.add(1);
|
2020-05-15 19:49:26 +02:00
|
|
|
if (isModified(orcidId, values[3])) {
|
2020-05-18 19:51:29 +02:00
|
|
|
modifiedRecordsAcc.add(1);
|
2020-05-15 19:49:26 +02:00
|
|
|
return true;
|
|
|
|
}
|
|
|
|
return false;
|
|
|
|
};
|
2020-05-18 19:51:29 +02:00
|
|
|
Function<String, Boolean> isNotDownloadedFilter = line -> {
|
|
|
|
String[] values = line.split(",");
|
|
|
|
String orcidId = values[0];
|
|
|
|
if (downloadedRecords.contains(orcidId)) {
|
|
|
|
alreadyDownloadedRecords.add(1);
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
return true;
|
|
|
|
};
|
2020-05-15 19:49:26 +02:00
|
|
|
Function<String, Tuple2<String, String>> downloadRecordFunction = line -> {
|
|
|
|
String[] values = line.split(",");
|
|
|
|
String orcidId = values[0];
|
2020-05-18 19:51:29 +02:00
|
|
|
String modifiedDate = values[3];
|
|
|
|
return downloadRecord(orcidId, modifiedDate, token, downloadedRecordsAcc);
|
2020-05-15 19:49:26 +02:00
|
|
|
};
|
2020-05-12 16:57:43 +02:00
|
|
|
|
2020-05-15 19:49:26 +02:00
|
|
|
lamdaFileRDD
|
|
|
|
.filter(isModifiedAfterFilter)
|
2020-05-18 19:51:29 +02:00
|
|
|
.filter(isNotDownloadedFilter)
|
2020-05-15 19:49:26 +02:00
|
|
|
.map(downloadRecordFunction)
|
|
|
|
.rdd()
|
|
|
|
.saveAsTextFile(workingPath.concat(outputAuthorsPath));
|
|
|
|
});
|
2020-05-12 16:57:43 +02:00
|
|
|
|
|
|
|
}
|
|
|
|
|
2020-05-15 19:49:26 +02:00
|
|
|
private static boolean isModified(String orcidId, String modifiedDate) {
|
|
|
|
Date modifiedDateDt = null;
|
|
|
|
Date lastUpdateDt = null;
|
|
|
|
try {
|
|
|
|
if (modifiedDate.length() != 19) {
|
|
|
|
modifiedDate = modifiedDate.substring(0, 19);
|
|
|
|
}
|
|
|
|
modifiedDateDt = new SimpleDateFormat(DATE_FORMAT).parse(modifiedDate);
|
|
|
|
lastUpdateDt = new SimpleDateFormat(DATE_FORMAT).parse(lastUpdate);
|
|
|
|
} catch (Exception e) {
|
|
|
|
Log.warn("[" + orcidId + "] Parsing date: ", e.getMessage());
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
return modifiedDateDt.after(lastUpdateDt);
|
|
|
|
}
|
|
|
|
|
2020-05-18 19:51:29 +02:00
|
|
|
private static Tuple2<String, String> downloadRecord(String orcidId, String modifiedDate, String token,
|
|
|
|
LongAccumulator downloadedRecordsAcc) {
|
2020-05-15 19:49:26 +02:00
|
|
|
final DownloadedRecordData data = new DownloadedRecordData();
|
|
|
|
data.setOrcidId(orcidId);
|
2020-05-18 19:51:29 +02:00
|
|
|
data.setModifiedDate(modifiedDate);
|
2020-05-15 19:49:26 +02:00
|
|
|
try (CloseableHttpClient client = HttpClients.createDefault()) {
|
|
|
|
HttpGet httpGet = new HttpGet("https://api.orcid.org/v3.0/" + orcidId + "/record");
|
|
|
|
httpGet.addHeader("Accept", "application/vnd.orcid+xml");
|
|
|
|
httpGet.addHeader("Authorization", String.format("Bearer %s", token));
|
|
|
|
CloseableHttpResponse response = client.execute(httpGet);
|
|
|
|
int statusCode = response.getStatusLine().getStatusCode();
|
|
|
|
data.setStatusCode(statusCode);
|
|
|
|
if (statusCode != 200) {
|
|
|
|
Log
|
|
|
|
.warn(
|
|
|
|
"Downloading " + orcidId + " status code: " + response.getStatusLine().getStatusCode());
|
|
|
|
return data.toTuple2();
|
|
|
|
}
|
2020-05-18 19:51:29 +02:00
|
|
|
downloadedRecordsAcc.add(1);
|
2020-05-15 19:49:26 +02:00
|
|
|
data
|
|
|
|
.setCompressedData(
|
|
|
|
ArgumentApplicationParser.compressArgument(IOUtils.toString(response.getEntity().getContent())));
|
|
|
|
} catch (Throwable e) {
|
|
|
|
Log.warn("Downloading " + orcidId, e.getMessage());
|
|
|
|
data.setErrorMessage(e.getMessage());
|
|
|
|
return data.toTuple2();
|
|
|
|
}
|
|
|
|
return data.toTuple2();
|
|
|
|
}
|
2020-05-12 16:57:43 +02:00
|
|
|
}
|