forked from D-Net/dnet-hadoop
Compare commits
9 Commits
ee1f1eeca2
...
657b0208a2
| Author | SHA1 | Date |
|---|---|---|
 Enrico Ottonello
|
657b0208a2 | |
 Alessia Bardi
|
28a32facf6 | |
 Claudio Atzori
|
0c1cfee396 | |
 Miriam Baglioni
|
fae681fea1 | |
|
Miriam Baglioni
|
c09fcdb40b | |
|
Claudio Atzori
|
138d1dfbf8 | |
|
Claudio Atzori
|
072f192853 | |
|
Claudio Atzori
|
a88103bcf9 | |
|
Claudio Atzori
|
7da24c1dec |
|
|
@ -75,9 +75,14 @@ public class DHPUtils {
|
|||
|
||||
final HttpGet req = new HttpGet(url);
|
||||
|
||||
log.info("MDStoreManager request: {}", req);
|
||||
|
||||
try (final CloseableHttpClient client = HttpClients.createDefault()) {
|
||||
try (final CloseableHttpResponse response = client.execute(req)) {
|
||||
final String json = IOUtils.toString(response.getEntity().getContent());
|
||||
|
||||
log.info("MDStoreManager response: {}", json);
|
||||
|
||||
final MDStoreWithInfo[] mdstores = objectMapper.readValue(json, MDStoreWithInfo[].class);
|
||||
return Arrays
|
||||
.stream(mdstores)
|
||||
|
|
|
|||
|
|
@ -172,6 +172,61 @@ public class PromoteActionPayloadForGraphTableJobTest {
|
|||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void shouldPromoteActionPayload_custom() throws Exception {
|
||||
|
||||
Class<? extends Oaf> rowClazz = Publication.class;
|
||||
Class<? extends Oaf> actionPayloadClazz = Result.class;
|
||||
MergeAndGet.Strategy strategy = MergeAndGet.Strategy.MERGE_FROM_AND_GET;
|
||||
|
||||
// given
|
||||
Path inputGraphTableDir = createGraphTable(inputGraphRootDir, rowClazz);
|
||||
Path inputActionPayloadDir = createActionPayload(inputActionPayloadRootDir, rowClazz, actionPayloadClazz);
|
||||
Path outputGraphTableDir = outputDir.resolve("graph").resolve(rowClazz.getSimpleName().toLowerCase());
|
||||
|
||||
// when
|
||||
PromoteActionPayloadForGraphTableJob
|
||||
.main(
|
||||
new String[] {
|
||||
"-isSparkSessionManaged",
|
||||
Boolean.FALSE.toString(),
|
||||
"-inputGraphTablePath",
|
||||
inputGraphTableDir.toString(),
|
||||
"-graphTableClassName",
|
||||
rowClazz.getCanonicalName(),
|
||||
"-inputActionPayloadPath",
|
||||
inputActionPayloadDir.toString(),
|
||||
"-actionPayloadClassName",
|
||||
actionPayloadClazz.getCanonicalName(),
|
||||
"-outputGraphTablePath",
|
||||
outputGraphTableDir.toString(),
|
||||
"-mergeAndGetStrategy",
|
||||
strategy.name(),
|
||||
"--shouldGroupById",
|
||||
"true"
|
||||
});
|
||||
|
||||
// then
|
||||
assertTrue(Files.exists(outputGraphTableDir));
|
||||
|
||||
List<? extends Oaf> actualOutputRows = readGraphTableFromJobOutput(outputGraphTableDir.toString(), rowClazz)
|
||||
.collectAsList()
|
||||
.stream()
|
||||
.sorted(Comparator.comparingInt(Object::hashCode))
|
||||
.collect(Collectors.toList());
|
||||
|
||||
Publication p = actualOutputRows
|
||||
.stream()
|
||||
.map(o -> (Publication) o)
|
||||
.filter(o -> "50|4ScienceCRIS::6a67ed3daba1c380bf9de3c13ed9c879".equals(o.getId()))
|
||||
.findFirst()
|
||||
.get();
|
||||
|
||||
assertNotNull(p.getMeasures());
|
||||
assertTrue(p.getMeasures().size() > 0);
|
||||
|
||||
}
|
||||
|
||||
public static Stream<Arguments> promoteJobTestParams() {
|
||||
return Stream
|
||||
.of(
|
||||
|
|
|
|||
|
|
@ -17,4 +17,5 @@
|
|||
{"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"","inferenceprovenance":"","provenanceaction":{"classid":"","classname":"","schemeid":"","schemename":""}},"lastupdatetimestamp":1572018243405,"id":"50|CSC_________::00019460865d6cc381b36076131a5bc1","originalId":[],"collectedfrom":[],"pid":[],"dateofcollection":"","dateoftransformation":"","extraInfo":[],"oaiprovenance":null,"author":[],"resulttype":{"classid":"","classname":"","schemeid":"","schemename":""},"language":{"classid":"","classname":"","schemeid":"","schemename":""},"country":[],"subject":[{"value":"Computer Science::Networking and Internet Architecture","qualifier":{"classid":"arxiv","classname":"arxiv","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":{"invisible":false,"inferred":true,"deletedbyinference":false,"trust":"0.7416","inferenceprovenance":"iis::document_classes","provenanceaction":{"classid":"iis","classname":"iis","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"title":[],"relevantdate":[],"description":[],"dateofacceptance":{"value":"","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"","inferenceprovenance":"","provenanceaction":{"classid":"","classname":"","schemeid":"","schemename":""}}},"publisher":{"value":"","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"","inferenceprovenance":"","provenanceaction":{"classid":"","classname":"","schemeid":"","schemename":""}}},"embargoenddate":{"value":"","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"","inferenceprovenance":"","provenanceaction":{"classid":"","classname":"","schemeid":"","schemename":""}}},"source":[],"fulltext":[],"format":[],"contributor":[],"resourcetype":{"classid":"","classname":"","schemeid":"","schemename":""},"coverage":[],"bestaccessright":{"classid":"UNKNOWN","classname":"not available","schemeid":"dnet:access_modes","schemename":"dnet:access_modes"},"context":[],"externalReference":null,"instance":[]}
|
||||
{"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"","inferenceprovenance":"","provenanceaction":{"classid":"","classname":"","schemeid":"","schemename":""}},"lastupdatetimestamp":1572018243405,"id":"50|CSC_________::00019460865d6cc381b36076131a5bc1","originalId":[],"collectedfrom":[],"pid":[],"dateofcollection":"","dateoftransformation":"","extraInfo":[],"oaiprovenance":null,"author":[],"resulttype":{"classid":"","classname":"","schemeid":"","schemename":""},"language":{"classid":"","classname":"","schemeid":"","schemename":""},"country":[],"subject":[{"value":"Computer Science::Networking and Internet Architecture","qualifier":{"classid":"arxiv","classname":"arxiv","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":{"invisible":false,"inferred":true,"deletedbyinference":false,"trust":"0.7416","inferenceprovenance":"iis::document_classes","provenanceaction":{"classid":"iis","classname":"iis","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"title":[],"relevantdate":[],"description":[],"dateofacceptance":{"value":"","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"","inferenceprovenance":"","provenanceaction":{"classid":"","classname":"","schemeid":"","schemename":""}}},"publisher":{"value":"","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"","inferenceprovenance":"","provenanceaction":{"classid":"","classname":"","schemeid":"","schemename":""}}},"embargoenddate":{"value":"","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"","inferenceprovenance":"","provenanceaction":{"classid":"","classname":"","schemeid":"","schemename":""}}},"source":[],"fulltext":[],"format":[],"contributor":[],"resourcetype":{"classid":"","classname":"","schemeid":"","schemename":""},"coverage":[],"bestaccessright":{"classid":"UNKNOWN","classname":"not available","schemeid":"dnet:access_modes","schemename":"dnet:access_modes"},"context":[],"externalReference":null,"instance":[]}
|
||||
{"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"","inferenceprovenance":"","provenanceaction":{"classid":"","classname":"","schemeid":"","schemename":""}},"lastupdatetimestamp":1572018240982,"id":"50|CSC_________::0001d663c95c4132355e1765375a5275","originalId":[],"collectedfrom":[],"pid":[],"dateofcollection":"","dateoftransformation":"","extraInfo":[],"oaiprovenance":null,"author":[],"resulttype":{"classid":"","classname":"","schemeid":"","schemename":""},"language":{"classid":"","classname":"","schemeid":"","schemename":""},"country":[],"subject":[{"value":"animal diseases","qualifier":{"classid":"mesheuropmc","classname":"mesheuropmc","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":{"invisible":false,"inferred":true,"deletedbyinference":false,"trust":"0.7461","inferenceprovenance":"iis::document_classes","provenanceaction":{"classid":"iis","classname":"iis","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"title":[],"relevantdate":[],"description":[],"dateofacceptance":{"value":"","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"","inferenceprovenance":"","provenanceaction":{"classid":"","classname":"","schemeid":"","schemename":""}}},"publisher":{"value":"","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"","inferenceprovenance":"","provenanceaction":{"classid":"","classname":"","schemeid":"","schemename":""}}},"embargoenddate":{"value":"","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"","inferenceprovenance":"","provenanceaction":{"classid":"","classname":"","schemeid":"","schemename":""}}},"source":[],"fulltext":[],"format":[],"contributor":[],"resourcetype":{"classid":"","classname":"","schemeid":"","schemename":""},"coverage":[],"bestaccessright":{"classid":"UNKNOWN","classname":"not available","schemeid":"dnet:access_modes","schemename":"dnet:access_modes"},"context":[],"externalReference":null,"instance":[]}
|
||||
{"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"","inferenceprovenance":"","provenanceaction":{"classid":"","classname":"","schemeid":"","schemename":""}},"lastupdatetimestamp":1572018240982,"id":"50|CSC_________::0001d663c95c4132355e1765375a5275","originalId":[],"collectedfrom":[],"pid":[],"dateofcollection":"","dateoftransformation":"","extraInfo":[],"oaiprovenance":null,"author":[],"resulttype":{"classid":"","classname":"","schemeid":"","schemename":""},"language":{"classid":"","classname":"","schemeid":"","schemename":""},"country":[],"subject":[{"value":"animal diseases","qualifier":{"classid":"mesheuropmc","classname":"mesheuropmc","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":{"invisible":false,"inferred":true,"deletedbyinference":false,"trust":"0.7461","inferenceprovenance":"iis::document_classes","provenanceaction":{"classid":"iis","classname":"iis","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"title":[],"relevantdate":[],"description":[],"dateofacceptance":{"value":"","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"","inferenceprovenance":"","provenanceaction":{"classid":"","classname":"","schemeid":"","schemename":""}}},"publisher":{"value":"","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"","inferenceprovenance":"","provenanceaction":{"classid":"","classname":"","schemeid":"","schemename":""}}},"embargoenddate":{"value":"","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"","inferenceprovenance":"","provenanceaction":{"classid":"","classname":"","schemeid":"","schemename":""}}},"source":[],"fulltext":[],"format":[],"contributor":[],"resourcetype":{"classid":"","classname":"","schemeid":"","schemename":""},"coverage":[],"bestaccessright":{"classid":"UNKNOWN","classname":"not available","schemeid":"dnet:access_modes","schemename":"dnet:access_modes"},"context":[],"externalReference":null,"instance":[]}
|
||||
{"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"","inferenceprovenance":"","provenanceaction":{"classid":"","classname":"","schemeid":"","schemename":""}},"lastupdatetimestamp":1572018240982,"id":"50|CSC_________::0001d663c95c4132355e1765375a5275","originalId":[],"collectedfrom":[],"pid":[],"dateofcollection":"","dateoftransformation":"","extraInfo":[],"oaiprovenance":null,"author":[],"resulttype":{"classid":"","classname":"","schemeid":"","schemename":""},"language":{"classid":"","classname":"","schemeid":"","schemename":""},"country":[],"subject":[{"value":"animal diseases","qualifier":{"classid":"mesheuropmc","classname":"mesheuropmc","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":{"invisible":false,"inferred":true,"deletedbyinference":false,"trust":"0.7461","inferenceprovenance":"iis::document_classes","provenanceaction":{"classid":"iis","classname":"iis","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"title":[],"relevantdate":[],"description":[],"dateofacceptance":{"value":"","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"","inferenceprovenance":"","provenanceaction":{"classid":"","classname":"","schemeid":"","schemename":""}}},"publisher":{"value":"","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"","inferenceprovenance":"","provenanceaction":{"classid":"","classname":"","schemeid":"","schemename":""}}},"embargoenddate":{"value":"","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"","inferenceprovenance":"","provenanceaction":{"classid":"","classname":"","schemeid":"","schemename":""}}},"source":[],"fulltext":[],"format":[],"contributor":[],"resourcetype":{"classid":"","classname":"","schemeid":"","schemename":""},"coverage":[],"bestaccessright":{"classid":"UNKNOWN","classname":"not available","schemeid":"dnet:access_modes","schemename":"dnet:access_modes"},"context":[],"externalReference":null,"instance":[]}
|
||||
{"collectedfrom":null,"dataInfo":null,"lastupdatetimestamp":null,"id":"50|4ScienceCRIS::6a67ed3daba1c380bf9de3c13ed9c879","originalId":null,"pid":null,"dateofcollection":null,"dateoftransformation":null,"extraInfo":null,"oaiprovenance":null,"processingchargeamount":null,"processingchargecurrency":null,"measures":[{"id":"influence","unit":[{"key":"score","value":"1.64385446761e-08","dataInfo":{"invisible":false,"inferred":true,"deletedbyinference":false,"trust":"","inferenceprovenance":"update","provenanceaction":{"classid":"measure:bip","classname":"Inferred by OpenAIRE","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}]},{"id":"popularity_alt","unit":[{"key":"score","value":"18.9590813696","dataInfo":{"invisible":false,"inferred":true,"deletedbyinference":false,"trust":"","inferenceprovenance":"update","provenanceaction":{"classid":"measure:bip","classname":"Inferred by OpenAIRE","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}]},{"id":"popularity","unit":[{"key":"score","value":"6.00577981643e-08","dataInfo":{"invisible":false,"inferred":true,"deletedbyinference":false,"trust":"","inferenceprovenance":"update","provenanceaction":{"classid":"measure:bip","classname":"Inferred by OpenAIRE","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}]}],"author":null,"resulttype":null,"language":null,"country":null,"subject":null,"title":null,"relevantdate":null,"description":null,"dateofacceptance":null,"publisher":null,"embargoenddate":null,"source":null,"fulltext":null,"format":null,"contributor":null,"resourcetype":null,"coverage":null,"bestaccessright":null,"context":null,"externalReference":null,"instance":null}
|
||||
|
|
@ -3,6 +3,8 @@ package eu.dnetlib.doiboost.orcid;
|
|||
|
||||
import static eu.dnetlib.dhp.common.SparkSessionSupport.runWithSparkSession;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.time.LocalDate;
|
||||
import java.time.format.DateTimeFormatter;
|
||||
import java.util.*;
|
||||
|
|
@ -13,6 +15,7 @@ import org.apache.hadoop.io.Text;
|
|||
import org.apache.hadoop.io.compress.GzipCodec;
|
||||
import org.apache.spark.SparkConf;
|
||||
import org.apache.spark.api.java.JavaPairRDD;
|
||||
import org.apache.spark.api.java.JavaRDD;
|
||||
import org.apache.spark.api.java.JavaSparkContext;
|
||||
import org.apache.spark.api.java.function.FlatMapFunction;
|
||||
import org.apache.spark.api.java.function.Function;
|
||||
|
|
@ -20,6 +23,7 @@ import org.apache.spark.util.LongAccumulator;
|
|||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import com.google.common.collect.Lists;
|
||||
import com.google.gson.JsonElement;
|
||||
import com.google.gson.JsonParser;
|
||||
|
||||
|
|
@ -42,6 +46,7 @@ public class SparkDownloadOrcidWorks {
|
|||
public static final String ORCID_XML_DATETIME_FORMAT = "yyyy-MM-dd'T'HH:mm:ss.SSS'Z'";
|
||||
public static final DateTimeFormatter ORCID_XML_DATETIMEFORMATTER = DateTimeFormatter
|
||||
.ofPattern(ORCID_XML_DATETIME_FORMAT);
|
||||
public static final String DOWNLOAD_WORKS_REQUEST_SEPARATOR = ",";
|
||||
|
||||
public static void main(String[] args) throws Exception {
|
||||
|
||||
|
|
@ -56,7 +61,6 @@ public class SparkDownloadOrcidWorks {
|
|||
.ofNullable(parser.get("isSparkSessionManaged"))
|
||||
.map(Boolean::valueOf)
|
||||
.orElse(Boolean.TRUE);
|
||||
logger.info("isSparkSessionManaged: {}", isSparkSessionManaged);
|
||||
final String workingPath = parser.get("workingPath");
|
||||
logger.info("workingPath: {}", workingPath);
|
||||
final String outputPath = parser.get("outputPath");
|
||||
|
|
@ -69,16 +73,12 @@ public class SparkDownloadOrcidWorks {
|
|||
isSparkSessionManaged,
|
||||
spark -> {
|
||||
final String lastUpdateValue = HDFSUtil.readFromTextFile(hdfsServerUri, workingPath, "last_update.txt");
|
||||
logger.info("lastUpdateValue: ", lastUpdateValue);
|
||||
|
||||
JavaSparkContext sc = JavaSparkContext.fromSparkContext(spark.sparkContext());
|
||||
LongAccumulator updatedAuthorsAcc = spark.sparkContext().longAccumulator("updated_authors");
|
||||
LongAccumulator parsedAuthorsAcc = spark.sparkContext().longAccumulator("parsed_authors");
|
||||
LongAccumulator parsedWorksAcc = spark.sparkContext().longAccumulator("parsed_works");
|
||||
LongAccumulator modifiedWorksAcc = spark.sparkContext().longAccumulator("modified_works");
|
||||
LongAccumulator maxModifiedWorksLimitAcc = spark
|
||||
.sparkContext()
|
||||
.longAccumulator("max_modified_works_limit");
|
||||
LongAccumulator errorCodeFoundAcc = spark.sparkContext().longAccumulator("error_code_found");
|
||||
LongAccumulator errorLoadingJsonFoundAcc = spark
|
||||
.sparkContext()
|
||||
|
|
@ -94,7 +94,8 @@ public class SparkDownloadOrcidWorks {
|
|||
|
||||
JavaPairRDD<Text, Text> updatedAuthorsRDD = sc
|
||||
.sequenceFile(workingPath + "downloads/updated_authors/*", Text.class, Text.class);
|
||||
updatedAuthorsAcc.setValue(updatedAuthorsRDD.count());
|
||||
long authorsCount = updatedAuthorsRDD.count();
|
||||
updatedAuthorsAcc.setValue(authorsCount);
|
||||
|
||||
FlatMapFunction<Tuple2<Text, Text>, String> retrieveWorkUrlFunction = data -> {
|
||||
String orcidId = data._1().toString();
|
||||
|
|
@ -125,22 +126,38 @@ public class SparkDownloadOrcidWorks {
|
|||
errorCodeFoundAcc.add(1);
|
||||
}
|
||||
parsedAuthorsAcc.add(1);
|
||||
|
||||
workIdLastModifiedDate.forEach((k, v) -> {
|
||||
parsedWorksAcc.add(1);
|
||||
if (isModified(orcidId, v, lastUpdateValue)) {
|
||||
modifiedWorksAcc.add(1);
|
||||
workIds.add(orcidId.concat("/work/").concat(k));
|
||||
workIds.add(k);
|
||||
}
|
||||
});
|
||||
if (workIdLastModifiedDate.size() > 50) {
|
||||
maxModifiedWorksLimitAcc.add(1);
|
||||
if (workIds.isEmpty()) {
|
||||
return new ArrayList<String>().iterator();
|
||||
}
|
||||
return workIds.iterator();
|
||||
List<String> worksDownloadUrls = new ArrayList<>();
|
||||
|
||||
// Creation of url for reading multiple works (up to 100) with ORCID API
|
||||
// see this https://github.com/ORCID/ORCID-Source/blob/development/orcid-api-web/tutorial/works.md
|
||||
|
||||
List<List<String>> partitionedWorks = Lists.partition(workIds, 100);
|
||||
partitionedWorks.stream().forEach(p -> {
|
||||
String worksDownloadUrl = orcidId.concat("/works/");
|
||||
final StringBuffer buffer = new StringBuffer(worksDownloadUrl);
|
||||
p.forEach(id -> {
|
||||
buffer.append(id).append(DOWNLOAD_WORKS_REQUEST_SEPARATOR);
|
||||
});
|
||||
String finalUrl = buffer.substring(0, buffer.lastIndexOf(DOWNLOAD_WORKS_REQUEST_SEPARATOR));
|
||||
worksDownloadUrls.add(finalUrl);
|
||||
});
|
||||
return worksDownloadUrls.iterator();
|
||||
};
|
||||
|
||||
Function<String, Tuple2<String, String>> downloadWorkFunction = data -> {
|
||||
String relativeWorkUrl = data;
|
||||
String orcidId = relativeWorkUrl.split("/")[0];
|
||||
Function<String, Tuple2<String, String>> downloadWorksFunction = data -> {
|
||||
String relativeWorksUrl = data;
|
||||
String orcidId = relativeWorksUrl.split("/")[0];
|
||||
final DownloadedRecordData downloaded = new DownloadedRecordData();
|
||||
downloaded.setOrcidId(orcidId);
|
||||
downloaded.setLastModifiedDate(lastUpdateValue);
|
||||
|
|
@ -149,7 +166,7 @@ public class SparkDownloadOrcidWorks {
|
|||
httpConnector.setAuthMethod(MultiAttemptsHttpConnector.BEARER);
|
||||
httpConnector.setAcceptHeaderValue("application/vnd.orcid+xml");
|
||||
httpConnector.setAuthToken(token);
|
||||
String apiUrl = "https://api.orcid.org/v3.0/" + relativeWorkUrl;
|
||||
String apiUrl = "https://api.orcid.org/v3.0/" + relativeWorksUrl;
|
||||
DownloadsReport report = new DownloadsReport();
|
||||
long startReq = System.currentTimeMillis();
|
||||
boolean downloadCompleted = false;
|
||||
|
|
@ -185,18 +202,53 @@ public class SparkDownloadOrcidWorks {
|
|||
return downloaded.toTuple2();
|
||||
};
|
||||
|
||||
FlatMapFunction<Tuple2<String, String>, Tuple2<String, String>> splitWorksFunction = data -> {
|
||||
List<Tuple2<String, String>> splittedDownloadedWorks = new ArrayList<>();
|
||||
String jsonData = data._2().toString();
|
||||
JsonElement jElement = new JsonParser().parse(jsonData);
|
||||
String orcidId = data._1().toString();
|
||||
String statusCode = getJsonValue(jElement, "statusCode");
|
||||
String lastModifiedDate = getJsonValue(jElement, "lastModifiedDate");
|
||||
String compressedData = getJsonValue(jElement, "compressedData");
|
||||
String errorMessage = getJsonValue(jElement, "errorMessage");
|
||||
String works = ArgumentApplicationParser.decompressValue(compressedData);
|
||||
|
||||
// split a single xml containing multiple works into multiple xml (a single work for each xml)
|
||||
List<String> splittedWorks = XMLRecordParser
|
||||
.splitWorks(orcidId, works.getBytes(StandardCharsets.UTF_8));
|
||||
|
||||
splittedWorks.forEach(w -> {
|
||||
final DownloadedRecordData downloaded = new DownloadedRecordData();
|
||||
downloaded.setOrcidId(orcidId);
|
||||
downloaded.setLastModifiedDate(lastModifiedDate);
|
||||
downloaded.setStatusCode(Integer.parseInt(statusCode));
|
||||
downloaded.setErrorMessage(errorMessage);
|
||||
try {
|
||||
downloaded
|
||||
.setCompressedData(
|
||||
ArgumentApplicationParser
|
||||
.compressArgument(w));
|
||||
} catch (IOException e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
splittedDownloadedWorks.add(downloaded.toTuple2());
|
||||
});
|
||||
|
||||
return splittedDownloadedWorks.iterator();
|
||||
};
|
||||
|
||||
updatedAuthorsRDD
|
||||
.flatMap(retrieveWorkUrlFunction)
|
||||
.repartition(100)
|
||||
.map(downloadWorkFunction)
|
||||
.mapToPair(t -> new Tuple2<>(new Text(t._1()), new Text(t._2())))
|
||||
.map(downloadWorksFunction)
|
||||
.flatMap(splitWorksFunction)
|
||||
.mapToPair(w -> new Tuple2<>(new Text(w._1()), new Text(w._2())))
|
||||
.saveAsTextFile(workingPath.concat(outputPath), GzipCodec.class);
|
||||
|
||||
logger.info("updatedAuthorsAcc: {}", updatedAuthorsAcc.value());
|
||||
logger.info("parsedAuthorsAcc: {}", parsedAuthorsAcc.value());
|
||||
logger.info("parsedWorksAcc: {}", parsedWorksAcc.value());
|
||||
logger.info("modifiedWorksAcc: {}", modifiedWorksAcc.value());
|
||||
logger.info("maxModifiedWorksLimitAcc: {}", maxModifiedWorksLimitAcc.value());
|
||||
logger.info("errorCodeFoundAcc: {}", errorCodeFoundAcc.value());
|
||||
logger.info("errorLoadingJsonFoundAcc: {}", errorLoadingJsonFoundAcc.value());
|
||||
logger.info("errorLoadingXMLFoundAcc: {}", errorLoadingXMLFoundAcc.value());
|
||||
|
|
|
|||
|
|
@ -1,7 +1,11 @@
|
|||
|
||||
package eu.dnetlib.doiboost.orcid.xml;
|
||||
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.*;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.mortbay.log.Log;
|
||||
|
|
@ -34,6 +38,33 @@ public class XMLRecordParser {
|
|||
private static final String NS_WORK_URL = "http://www.orcid.org/ns/work";
|
||||
private static final String NS_HISTORY = "history";
|
||||
private static final String NS_HISTORY_URL = "http://www.orcid.org/ns/history";
|
||||
private static final String NS_BULK_URL = "http://www.orcid.org/ns/bulk";
|
||||
private static final String NS_BULK = "bulk";
|
||||
|
||||
private static final String namespaceList = " xmlns:internal=\"http://www.orcid.org/ns/internal\"\n" +
|
||||
" xmlns:education=\"http://www.orcid.org/ns/education\"\n" +
|
||||
" xmlns:distinction=\"http://www.orcid.org/ns/distinction\"\n" +
|
||||
" xmlns:deprecated=\"http://www.orcid.org/ns/deprecated\"\n" +
|
||||
" xmlns:other-name=\"http://www.orcid.org/ns/other-name\"\n" +
|
||||
" xmlns:membership=\"http://www.orcid.org/ns/membership\"\n" +
|
||||
" xmlns:error=\"http://www.orcid.org/ns/error\" xmlns:common=\"http://www.orcid.org/ns/common\"\n" +
|
||||
" xmlns:record=\"http://www.orcid.org/ns/record\"\n" +
|
||||
" xmlns:personal-details=\"http://www.orcid.org/ns/personal-details\"\n" +
|
||||
" xmlns:keyword=\"http://www.orcid.org/ns/keyword\" xmlns:email=\"http://www.orcid.org/ns/email\"\n" +
|
||||
" xmlns:external-identifier=\"http://www.orcid.org/ns/external-identifier\"\n" +
|
||||
" xmlns:funding=\"http://www.orcid.org/ns/funding\"\n" +
|
||||
" xmlns:preferences=\"http://www.orcid.org/ns/preferences\"\n" +
|
||||
" xmlns:address=\"http://www.orcid.org/ns/address\"\n" +
|
||||
" xmlns:invited-position=\"http://www.orcid.org/ns/invited-position\"\n" +
|
||||
" xmlns:work=\"http://www.orcid.org/ns/work\" xmlns:history=\"http://www.orcid.org/ns/history\"\n" +
|
||||
" xmlns:employment=\"http://www.orcid.org/ns/employment\"\n" +
|
||||
" xmlns:qualification=\"http://www.orcid.org/ns/qualification\"\n" +
|
||||
" xmlns:service=\"http://www.orcid.org/ns/service\" xmlns:person=\"http://www.orcid.org/ns/person\"\n" +
|
||||
" xmlns:activities=\"http://www.orcid.org/ns/activities\"\n" +
|
||||
" xmlns:researcher-url=\"http://www.orcid.org/ns/researcher-url\"\n" +
|
||||
" xmlns:peer-review=\"http://www.orcid.org/ns/peer-review\"\n" +
|
||||
" xmlns:bulk=\"http://www.orcid.org/ns/bulk\"\n" +
|
||||
" xmlns:research-resource=\"http://www.orcid.org/ns/research-resource\"";
|
||||
|
||||
private static final String NS_ERROR = "error";
|
||||
|
||||
|
|
@ -307,4 +338,65 @@ public class XMLRecordParser {
|
|||
}
|
||||
return authorHistory;
|
||||
}
|
||||
|
||||
public static List<String> splitWorks(String orcidId, byte[] bytes)
|
||||
throws ParseException, XPathParseException, NavException, XPathEvalException, VtdException, ModifyException,
|
||||
IOException, TranscodeException {
|
||||
|
||||
final VTDGen vg = new VTDGen();
|
||||
vg.setDoc(bytes);
|
||||
vg.parse(true);
|
||||
final VTDNav vn = vg.getNav();
|
||||
final AutoPilot ap = new AutoPilot(vn);
|
||||
ap.declareXPathNameSpace(NS_COMMON, NS_COMMON_URL);
|
||||
ap.declareXPathNameSpace(NS_WORK, NS_WORK_URL);
|
||||
ap.declareXPathNameSpace(NS_ERROR, NS_ERROR_URL);
|
||||
ap.declareXPathNameSpace(NS_BULK, NS_BULK_URL);
|
||||
|
||||
List<String> works = new ArrayList<>();
|
||||
try {
|
||||
ap.selectXPath("//work:work");
|
||||
while (ap.evalXPath() != -1) {
|
||||
ByteArrayOutputStream bos = new ByteArrayOutputStream();
|
||||
long l = vn.getElementFragment();
|
||||
String xmlHeader = "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"yes\"?>";
|
||||
bos.write(xmlHeader.getBytes(StandardCharsets.UTF_8));
|
||||
bos.write(vn.getXML().getBytes(), (int) l, (int) (l >> 32));
|
||||
works.add(bos.toString());
|
||||
bos.close();
|
||||
}
|
||||
} catch (Exception e) {
|
||||
throw new VtdException(e);
|
||||
}
|
||||
|
||||
List<VTDGen> vgModifiers = Arrays.asList(new VTDGen());
|
||||
List<XMLModifier> xmModifiers = Arrays.asList(new XMLModifier());
|
||||
List<ByteArrayOutputStream> buffer = Arrays.asList(new ByteArrayOutputStream());
|
||||
List<String> updatedWorks = works.stream().map(work -> {
|
||||
vgModifiers.get(0).setDoc(work.getBytes());
|
||||
try {
|
||||
vgModifiers.get(0).parse(false);
|
||||
final VTDNav vnModifier = vgModifiers.get(0).getNav();
|
||||
xmModifiers.get(0).bind(vnModifier);
|
||||
vnModifier.toElement(VTDNav.ROOT);
|
||||
int attr = vnModifier.getAttrVal("put-code");
|
||||
if (attr > -1) {
|
||||
xmModifiers
|
||||
.get(0)
|
||||
.insertAttribute(
|
||||
" path=\"/" + orcidId + "/work/" + vnModifier.toNormalizedString(attr) + "\""
|
||||
+ " " + namespaceList);
|
||||
}
|
||||
buffer.set(0, new ByteArrayOutputStream());
|
||||
xmModifiers.get(0).output(buffer.get(0));
|
||||
buffer.get(0).close();
|
||||
return buffer.get(0).toString();
|
||||
} catch (NavException | ModifyException | IOException | TranscodeException | ParseException e) {
|
||||
e.printStackTrace();
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
}).collect(Collectors.toList());
|
||||
|
||||
return updatedWorks;
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -78,7 +78,7 @@
|
|||
</configuration>
|
||||
</global>
|
||||
|
||||
<start to="ResetLambda"/>
|
||||
<start to="DownloadOrcidWorks"/>
|
||||
|
||||
<kill name="Kill">
|
||||
<message>Action failed, error message[${wf:errorMessage(wf:lastErrorNode())}]</message>
|
||||
|
|
@ -190,7 +190,7 @@
|
|||
<arg>-o</arg><arg>downloads/updated_works</arg>
|
||||
<arg>-t</arg><arg>${token}</arg>
|
||||
</spark>
|
||||
<ok to="ResetNewAuthors"/>
|
||||
<ok to="End"/>
|
||||
<error to="Kill"/>
|
||||
</action>
|
||||
|
||||
|
|
|
|||
|
|
@ -161,13 +161,11 @@ public class OrcidClientTest {
|
|||
|
||||
@Test
|
||||
@Disabled
|
||||
void testReadBase64CompressedRecord() throws Exception {
|
||||
void testReadBase64CompressedWork() throws Exception {
|
||||
final String base64CompressedRecord = IOUtils
|
||||
.toString(getClass().getResourceAsStream("0000-0003-3028-6161.compressed.base64"));
|
||||
.toString(getClass().getResourceAsStream("0000-0001-7281-6306.compressed.base64"));
|
||||
final String recordFromSeqFile = ArgumentApplicationParser.decompressValue(base64CompressedRecord);
|
||||
logToFile(testPath, "\n\ndownloaded \n\n" + recordFromSeqFile);
|
||||
final String downloadedRecord = testDownloadRecord("0000-0003-3028-6161", REQUEST_TYPE_RECORD);
|
||||
assertEquals(recordFromSeqFile, downloadedRecord);
|
||||
}
|
||||
|
||||
@Test
|
||||
|
|
@ -337,7 +335,7 @@ public class OrcidClientTest {
|
|||
@Ignore
|
||||
void testUpdatedRecord() throws Exception {
|
||||
final String base64CompressedRecord = IOUtils
|
||||
.toString(getClass().getResourceAsStream("0000-0003-3028-6161.compressed.base64"));
|
||||
.toString(getClass().getResourceAsStream("0000-0001-7281-6306.compressed.base64"));
|
||||
final String record = ArgumentApplicationParser.decompressValue(base64CompressedRecord);
|
||||
logToFile(testPath, "\n\nrecord updated \n\n" + record);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -108,4 +108,12 @@ public class XMLRecordParserTest {
|
|||
work.setBase64CompressData(ArgumentApplicationParser.compressArgument(xml));
|
||||
OrcidClientTest.logToFile(testPath, JsonWriter.create(work));
|
||||
}
|
||||
|
||||
@Test
|
||||
void testWorksSplit() throws Exception {
|
||||
String xml = IOUtils
|
||||
.toString(
|
||||
this.getClass().getResourceAsStream("multiple_downloaded_works.xml"));
|
||||
XMLRecordParser.splitWorks("0000-0001-7291-3210", xml.getBytes());
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1 @@
|
|||
H4sIAAAAAAAAAN1Y23LbNhB971dg+NAnkiKpSJZUS2luTerESSd22pm+QSQkISEBFiAlqxn9exe8CaRExkmc4Uw9Y9rEnrO72MUCC14+votCtCVCUs7mhms7BiLM5wFl67nx4fY3a2IgmWAW4JAzMjf2RBqPF5c7Lj7N1APFONnMjYEDPxb8utaFN3Gt8dAZD5R8MHWHk+nF1DUQAlNMzihLiGA4nBubJIlng8Fut7O58GkAz/WAyUGJMH5CqGSRIPVxkjnZRqsgOi+gMqHM72ZqoBqXxIKAShJ0UCuMzuTJhgiL4Yi0M48YnRmRaAnZ2NC4nXnE1CIkBBcd0VFio8D6PIq6ApLLde0wSS464pDLdUYMLnLIohWQBNNQtnObSF3LJ7LfdRouAOXMSAQaOqKgxLWo3eVrzaIBYQldUdIVw1OwrmuVsrxu2vgFoBYlQVZEQMmRrgAdQToXB4EgsoNXAHQOZVsKi9WKuaTdRdFE6lpUZbczlbTMxwZKi4t9O7gA1HISxSHfRxDirkSWGJ35T4pDSMuXdooaTOdLIrbU7yjaAmDU1viXVnYtZ7DLQFxpV7qPmHoFSoKFrzaNVHQs8TquXpEwJsiWkl2XyxVI5y7TsCPjSnrOV1AkeSq6InoCNVCcJhYcQUA6Hh5bKumShjSBpRSny5D6xiIzqH4u8/1q5guidmIrgOfCczzXcoaWN711vdnQm7mPbGfs/X05OIc+0RVimVgRHIRQ5UeNnuWMLce9dUDdaOY59tgdHjWe4ZzozSd5HD+VWX5IYV3DJlNH6chU0IWKqISQHsOZE6uz2LNG04lnTaaTYeWiIrZqVWf5ooudAVrpGy6TReVNRcqG6/Md3GvCjbCoo3Jx4/M4lchCL0KpFqlo6spQZ9VgCdWrKt7igq6p+uN/fYzPNDrfENxz7IcO7n3m2xqbLIxXXG5SjJ7idL1pV1uPeCMfmiDrGROahC35yUXPOHR/UcwFFnskU9hutziEnjSIOfSFcoaeMFQ0iMoJkEG5rVJJ1KigTFIfxaCDMoLWIeURRoKs4ZBR6pI02FcONly5HJxzMPf6I8xFnfu58C1JBbfeQZsc8vW+4NUhDb5Pk8zbxsRrMivZx2SxpMuE3BU666IuLsQoJYtfMSTGD8nnLGOe416YmTtojj7/8LgezCIEylo9RAdzD3u8Glc+HcwtD9Mo88qdHkyWqnZWvcFLjNdEZhLvYmq53sQ5mDhNNlzkk4BLyN5EtzaCKwl6gxkx0ZP85SlMnoTSRB+Kd56uViQx0Yv8/SUPgwgzE90UZHBpr95e2MXIb1yQDPHWfp2P/IH9T0SY6L19VSgVnFHpq7HC7DWEB6Ztoiu7MHSzoRsTPbOtQu2zDUDwOo1iHGITXeejr6COcBhWc3nJkwSLgCvrL/Oh5xseYkGB86rg8NUqc/BNqRln4XhaRgCyrhzJ2RzeMvT7asJ+Ji7YVxBLqch/ltNPQxzQysO/sICe00Svy4ldc/aRKPHh0Fyg+fpr1tLpsi82AbWcy4Ip1mxZfrWVXu2d2Ymfm6ofqzpKLbKFWmFViWcjp1tTu7pSldbpy/PGNET7pq2B8hoOOK28OBHeS00eadexXWc6HDCScuYPGL9znYuzmhuZ6VLNuIigMf6XBCgRGCo+68ATkRLjKwwetdzPqiBhlgl1n11IEq7Oaq2hzp93rRn5vpQRGjxIyjxLerZjTUbO0L2YjkfjRz8yX/e09n9LFpWSPUyBjbzhaDIeI/jHm4zcH1tcYMxS1h4+RzFsrxZ/2DSdk8rTPRRunwvt1iezzt0G4YCyHRx1xTcjG3CPocjmp0v2ZxzFv6gZMCJ+fz6/fju5fffk/Y3Wb4cnnRZX3coyTbhobtxN+Zlo5hBBAprkbe2x4SiPNE3YCFm3/m8yXzY4vRjXGqp+7B8buF7saw1jP8nXG9RePKg1xL14oDfg/SxCveHvxYPaBaMXD7QLTS/2Ty5QvXihXdh62o70C2IvLugX0n5ycLwA97QSywt3TydyccHvJ/vaB4W+DsTyA0Yv9rUPJj0dx9UHml7s6x+E+jkKyw9Q32P9VFZcFAqBeiz+A4MY5OQYIQAA
|
||||
File diff suppressed because one or more lines are too long
|
|
@ -0,0 +1,57 @@
|
|||
<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
|
||||
<bulk:bulk xmlns:internal="http://www.orcid.org/ns/internal" xmlns:education="http://www.orcid.org/ns/education" xmlns:distinction="http://www.orcid.org/ns/distinction" xmlns:deprecated="http://www.orcid.org/ns/deprecated" xmlns:other-name="http://www.orcid.org/ns/other-name" xmlns:membership="http://www.orcid.org/ns/membership" xmlns:error="http://www.orcid.org/ns/error" xmlns:common="http://www.orcid.org/ns/common" xmlns:record="http://www.orcid.org/ns/record" xmlns:personal-details="http://www.orcid.org/ns/personal-details" xmlns:keyword="http://www.orcid.org/ns/keyword" xmlns:email="http://www.orcid.org/ns/email" xmlns:external-identifier="http://www.orcid.org/ns/external-identifier" xmlns:funding="http://www.orcid.org/ns/funding" xmlns:preferences="http://www.orcid.org/ns/preferences" xmlns:address="http://www.orcid.org/ns/address" xmlns:invited-position="http://www.orcid.org/ns/invited-position" xmlns:work="http://www.orcid.org/ns/work" xmlns:history="http://www.orcid.org/ns/history" xmlns:employment="http://www.orcid.org/ns/employment" xmlns:qualification="http://www.orcid.org/ns/qualification" xmlns:service="http://www.orcid.org/ns/service" xmlns:person="http://www.orcid.org/ns/person" xmlns:activities="http://www.orcid.org/ns/activities" xmlns:researcher-url="http://www.orcid.org/ns/researcher-url" xmlns:peer-review="http://www.orcid.org/ns/peer-review" xmlns:bulk="http://www.orcid.org/ns/bulk" xmlns:research-resource="http://www.orcid.org/ns/research-resource">
|
||||
<work:work put-code="16639612" visibility="public">
|
||||
<common:created-date>2015-05-23T18:56:52.486Z</common:created-date>
|
||||
<common:last-modified-date>2017-02-28T08:22:12.454Z</common:last-modified-date>
|
||||
<common:source>
|
||||
<common:source-orcid>
|
||||
<common:uri>https://orcid.org/0000-0001-7291-3210</common:uri>
|
||||
<common:path>0000-0001-7291-3210</common:path>
|
||||
<common:host>orcid.org</common:host>
|
||||
</common:source-orcid>
|
||||
<common:source-name>Paolo Manghi</common:source-name>
|
||||
</common:source>
|
||||
<work:title>
|
||||
<common:title>The Query Language TQL</common:title>
|
||||
</work:title>
|
||||
<work:journal-title>5th International Workshop on Web and Data Bases (WebDB02) in conjunction with ACM SIGMOD 2002</work:journal-title>
|
||||
<work:citation>
|
||||
<work:citation-type>bibtex</work:citation-type>
|
||||
<work:citation-value>@inproceedings{Conforti2002, Author= {Giovanni Conforti and Giorgio Ghelli and Antonio Albano and Dario Colazzo and Paolo Manghi and Carlo Sartiani}, Bibsource= {DBLP, http://dblp.uni-trier.de}, Booktitle= {5th International Workshop on Web and Data Bases (WebDB02) in conjunction with ACM SIGMOD 2002}, Ee= {http://www.db.ucsd.edu/webdb2002/papers/43.pdf}, Pages= {13-18}, Title= {The Query Language TQL}, Year= {2002}}
|
||||
|
||||
</work:citation-value>
|
||||
</work:citation>
|
||||
<work:type>conference-paper</work:type>
|
||||
<common:publication-date>
|
||||
<common:year>2002</common:year>
|
||||
</common:publication-date>
|
||||
<common:external-ids/>
|
||||
</work:work>
|
||||
<work:work put-code="16639628" visibility="public">
|
||||
<common:created-date>2015-05-23T18:58:18.492Z</common:created-date>
|
||||
<common:last-modified-date>2017-02-28T08:22:12.455Z</common:last-modified-date>
|
||||
<common:source>
|
||||
<common:source-orcid>
|
||||
<common:uri>https://orcid.org/0000-0001-7291-3210</common:uri>
|
||||
<common:path>0000-0001-7291-3210</common:path>
|
||||
<common:host>orcid.org</common:host>
|
||||
</common:source-orcid>
|
||||
<common:source-name>Paolo Manghi</common:source-name>
|
||||
</common:source>
|
||||
<work:title>
|
||||
<common:title>The Query Language TQL - Demo Presentation</common:title>
|
||||
</work:title>
|
||||
<work:journal-title>X Convegno nazionale su Sistemi Evoluti per Basi di Dati (SEBD)</work:journal-title>
|
||||
<work:citation>
|
||||
<work:citation-type>bibtex</work:citation-type>
|
||||
<work:citation-value>@inproceedings{Conforti2002Demo, Address= {Portoferraio, Italy}, Author= {Giovanni Conforti and Giorgio Ghelli and Antonio Albano and Dario Colazzo and Paolo Manghi and Carlo Sartiani}, Bibsource= {DBLP, http://dblp.uni-trier.de}, Booktitle= {X Convegno nazionale su Sistemi Evoluti per Basi di Dati (SEBD)}, Month= {June}, Pages= {427-431}, Title= {The Query Language TQL - Demo Presentation}, Year= {2002}}
|
||||
|
||||
</work:citation-value>
|
||||
</work:citation>
|
||||
<work:type>conference-paper</work:type>
|
||||
<common:publication-date>
|
||||
<common:year>2002</common:year>
|
||||
</common:publication-date>
|
||||
<common:external-ids/>
|
||||
</work:work>
|
||||
</bulk:bulk>
|
||||
|
|
@ -6,6 +6,7 @@ import static eu.dnetlib.dhp.common.SparkSessionSupport.runWithSparkSession;
|
|||
|
||||
import java.util.Arrays;
|
||||
import java.util.List;
|
||||
import java.util.Optional;
|
||||
|
||||
import org.apache.commons.io.IOUtils;
|
||||
import org.apache.spark.SparkConf;
|
||||
|
|
@ -87,6 +88,8 @@ public class PrepareDatasourceCountryAssociation {
|
|||
Dataset<Datasource> datasource = readPath(spark, inputPath + "/datasource", Datasource.class)
|
||||
.filter(
|
||||
(FilterFunction<Datasource>) ds -> !ds.getDataInfo().getDeletedbyinference() &&
|
||||
Optional.ofNullable(ds.getDatasourcetype()).isPresent() &&
|
||||
Optional.ofNullable(ds.getDatasourcetype().getClassid()).isPresent() &&
|
||||
(allowedtypes.contains(ds.getDatasourcetype().getClassid()) ||
|
||||
whitelist.contains(ds.getId())));
|
||||
|
||||
|
|
|
|||
|
|
@ -379,7 +379,7 @@ public abstract class AbstractMdRecordToOafMapper {
|
|||
r.setPublisher(preparePublisher(doc, info));
|
||||
r.setEmbargoenddate(prepareField(doc, "//oaf:embargoenddate", info));
|
||||
r.setSource(prepareSources(doc, info));
|
||||
r.setFulltext(new ArrayList<>()); // NOT PRESENT IN MDSTORES
|
||||
r.setFulltext(prepareListFields(doc, "//oaf:fulltext", info));
|
||||
r.setFormat(prepareFormats(doc, info));
|
||||
r.setContributor(prepareContributors(doc, info));
|
||||
r.setResourcetype(prepareResourceType(doc, info));
|
||||
|
|
|
|||
|
|
@ -65,11 +65,19 @@ public class MigrateHdfsMdstoresApplication extends AbstractMigrationApplication
|
|||
log.info("isSparkSessionManaged: {}", isSparkSessionManaged);
|
||||
|
||||
final String mdstoreManagerUrl = parser.get("mdstoreManagerUrl");
|
||||
log.info("mdstoreManagerUrl: {}", mdstoreManagerUrl);
|
||||
|
||||
final String mdFormat = parser.get("mdFormat");
|
||||
log.info("mdFormat: {}", mdFormat);
|
||||
|
||||
final String mdLayout = parser.get("mdLayout");
|
||||
log.info("mdLayout: {}", mdLayout);
|
||||
|
||||
final String mdInterpretation = parser.get("mdInterpretation");
|
||||
log.info("mdInterpretation: {}", mdInterpretation);
|
||||
|
||||
final String hdfsPath = parser.get("hdfsPath");
|
||||
log.info("hdfsPath: {}", hdfsPath);
|
||||
|
||||
final Set<String> paths = mdstorePaths(mdstoreManagerUrl, mdFormat, mdLayout, mdInterpretation);
|
||||
|
||||
|
|
@ -95,6 +103,8 @@ public class MigrateHdfsMdstoresApplication extends AbstractMigrationApplication
|
|||
.filter(p -> HdfsSupport.exists(p, sc.hadoopConfiguration()))
|
||||
.toArray(size -> new String[size]);
|
||||
|
||||
log.info("Processing existing paths {}", Arrays.asList(validPaths));
|
||||
|
||||
if (validPaths.length > 0) {
|
||||
spark
|
||||
.read()
|
||||
|
|
|
|||
|
|
@ -122,6 +122,10 @@ class MappersTest {
|
|||
assertNotNull(p.getBestaccessright());
|
||||
assertEquals("OPEN", p.getBestaccessright().getClassid());
|
||||
|
||||
assertNotNull(p.getFulltext());
|
||||
assertEquals(1, p.getFulltext().size());
|
||||
assertEquals("https://oneecosystem.pensoft.net/article/13718/", p.getFulltext().get(0).getValue());
|
||||
|
||||
// RESULT PROJECT
|
||||
List<Relation> resultProject = list
|
||||
.stream()
|
||||
|
|
|
|||
|
|
@ -999,17 +999,11 @@ public class XmlRecordFactory implements Serializable {
|
|||
private List<String> measuresAsXml(List<Measure> measures) {
|
||||
return measures
|
||||
.stream()
|
||||
.flatMap(
|
||||
m -> m
|
||||
.getUnit()
|
||||
.stream()
|
||||
.map(
|
||||
u -> Lists
|
||||
.newArrayList(
|
||||
new Tuple2<>("id", m.getId()),
|
||||
new Tuple2<>("key", u.getKey()),
|
||||
new Tuple2<>("value", u.getValue())))
|
||||
.map(l -> XmlSerializationUtils.asXmlElement("measure", l)))
|
||||
.map(m -> {
|
||||
List<Tuple2<String, String>> l = Lists.newArrayList(new Tuple2<>("id", m.getId()));
|
||||
m.getUnit().forEach(kv -> l.add(new Tuple2<>(kv.getKey(), kv.getValue())));
|
||||
return XmlSerializationUtils.asXmlElement("measure", l);
|
||||
})
|
||||
.collect(Collectors.toList());
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -73,11 +73,20 @@ public class XmlRecordFactoryTest {
|
|||
assertEquals("EUR", doc.valueOf("//processingchargecurrency/text()"));
|
||||
|
||||
assertEquals(
|
||||
"1.00889953098e-08", doc.valueOf("//*[local-name() = 'result']/measure[./@id = 'influence']/@value"));
|
||||
"5.06690394631e-09", doc.valueOf("//*[local-name() = 'result']/measure[./@id = 'influence']/@score"));
|
||||
assertEquals(
|
||||
"30.6576853333", doc.valueOf("//*[local-name() = 'result']/measure[./@id = 'popularity_alt']/@value"));
|
||||
"C", doc.valueOf("//*[local-name() = 'result']/measure[./@id = 'influence']/@class"));
|
||||
|
||||
assertEquals(
|
||||
"4.62970429725e-08", doc.valueOf("//*[local-name() = 'result']/measure[./@id = 'popularity']/@value"));
|
||||
"0.0", doc.valueOf("//*[local-name() = 'result']/measure[./@id = 'popularity_alt']/@score"));
|
||||
assertEquals(
|
||||
"C", doc.valueOf("//*[local-name() = 'result']/measure[./@id = 'popularity_alt']/@class"));
|
||||
|
||||
assertEquals(
|
||||
"3.11855618382e-09", doc.valueOf("//*[local-name() = 'result']/measure[./@id = 'popularity']/@score"));
|
||||
assertEquals(
|
||||
"C", doc.valueOf("//*[local-name() = 'result']/measure[./@id = 'popularity']/@class"));
|
||||
|
||||
}
|
||||
|
||||
@Test
|
||||
|
|
|
|||
|
|
@ -5,7 +5,37 @@
|
|||
"unit": [
|
||||
{
|
||||
"key": "score",
|
||||
"value": "1.00889953098e-08"
|
||||
"value": "5.06690394631e-09",
|
||||
"dataInfo": {
|
||||
"invisible": false,
|
||||
"inferred": true,
|
||||
"deletedbyinference": false,
|
||||
"trust": "",
|
||||
"inferenceprovenance": "update",
|
||||
"provenanceaction": {
|
||||
"classid": "measure:bip",
|
||||
"classname": "Inferred by OpenAIRE",
|
||||
"schemeid": "dnet:provenanceActions",
|
||||
"schemename": "dnet:provenanceActions"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"key": "class",
|
||||
"value": "C",
|
||||
"dataInfo": {
|
||||
"invisible": false,
|
||||
"inferred": true,
|
||||
"deletedbyinference": false,
|
||||
"trust": "",
|
||||
"inferenceprovenance": "update",
|
||||
"provenanceaction": {
|
||||
"classid": "measure:bip",
|
||||
"classname": "Inferred by OpenAIRE",
|
||||
"schemeid": "dnet:provenanceActions",
|
||||
"schemename": "dnet:provenanceActions"
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
|
|
@ -14,7 +44,37 @@
|
|||
"unit": [
|
||||
{
|
||||
"key": "score",
|
||||
"value": "30.6576853333"
|
||||
"value": "0.0",
|
||||
"dataInfo": {
|
||||
"invisible": false,
|
||||
"inferred": true,
|
||||
"deletedbyinference": false,
|
||||
"trust": "",
|
||||
"inferenceprovenance": "update",
|
||||
"provenanceaction": {
|
||||
"classid": "measure:bip",
|
||||
"classname": "Inferred by OpenAIRE",
|
||||
"schemeid": "dnet:provenanceActions",
|
||||
"schemename": "dnet:provenanceActions"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"key": "class",
|
||||
"value": "C",
|
||||
"dataInfo": {
|
||||
"invisible": false,
|
||||
"inferred": true,
|
||||
"deletedbyinference": false,
|
||||
"trust": "",
|
||||
"inferenceprovenance": "update",
|
||||
"provenanceaction": {
|
||||
"classid": "measure:bip",
|
||||
"classname": "Inferred by OpenAIRE",
|
||||
"schemeid": "dnet:provenanceActions",
|
||||
"schemename": "dnet:provenanceActions"
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
|
|
@ -23,7 +83,115 @@
|
|||
"unit": [
|
||||
{
|
||||
"key": "score",
|
||||
"value": "4.62970429725e-08"
|
||||
"value": "3.11855618382e-09",
|
||||
"dataInfo": {
|
||||
"invisible": false,
|
||||
"inferred": true,
|
||||
"deletedbyinference": false,
|
||||
"trust": "",
|
||||
"inferenceprovenance": "update",
|
||||
"provenanceaction": {
|
||||
"classid": "measure:bip",
|
||||
"classname": "Inferred by OpenAIRE",
|
||||
"schemeid": "dnet:provenanceActions",
|
||||
"schemename": "dnet:provenanceActions"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"key": "class",
|
||||
"value": "C",
|
||||
"dataInfo": {
|
||||
"invisible": false,
|
||||
"inferred": true,
|
||||
"deletedbyinference": false,
|
||||
"trust": "",
|
||||
"inferenceprovenance": "update",
|
||||
"provenanceaction": {
|
||||
"classid": "measure:bip",
|
||||
"classname": "Inferred by OpenAIRE",
|
||||
"schemeid": "dnet:provenanceActions",
|
||||
"schemename": "dnet:provenanceActions"
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "influence_alt",
|
||||
"unit": [
|
||||
{
|
||||
"key": "score",
|
||||
"value": "0.0",
|
||||
"dataInfo": {
|
||||
"invisible": false,
|
||||
"inferred": true,
|
||||
"deletedbyinference": false,
|
||||
"trust": "",
|
||||
"inferenceprovenance": "update",
|
||||
"provenanceaction": {
|
||||
"classid": "measure:bip",
|
||||
"classname": "Inferred by OpenAIRE",
|
||||
"schemeid": "dnet:provenanceActions",
|
||||
"schemename": "dnet:provenanceActions"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"key": "class",
|
||||
"value": "C",
|
||||
"dataInfo": {
|
||||
"invisible": false,
|
||||
"inferred": true,
|
||||
"deletedbyinference": false,
|
||||
"trust": "",
|
||||
"inferenceprovenance": "update",
|
||||
"provenanceaction": {
|
||||
"classid": "measure:bip",
|
||||
"classname": "Inferred by OpenAIRE",
|
||||
"schemeid": "dnet:provenanceActions",
|
||||
"schemename": "dnet:provenanceActions"
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "impulse",
|
||||
"unit": [
|
||||
{
|
||||
"key": "score",
|
||||
"value": "0.0",
|
||||
"dataInfo": {
|
||||
"invisible": false,
|
||||
"inferred": true,
|
||||
"deletedbyinference": false,
|
||||
"trust": "",
|
||||
"inferenceprovenance": "update",
|
||||
"provenanceaction": {
|
||||
"classid": "measure:bip",
|
||||
"classname": "Inferred by OpenAIRE",
|
||||
"schemeid": "dnet:provenanceActions",
|
||||
"schemename": "dnet:provenanceActions"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"key": "class",
|
||||
"value": "C",
|
||||
"dataInfo": {
|
||||
"invisible": false,
|
||||
"inferred": true,
|
||||
"deletedbyinference": false,
|
||||
"trust": "",
|
||||
"inferenceprovenance": "update",
|
||||
"provenanceaction": {
|
||||
"classid": "measure:bip",
|
||||
"classname": "Inferred by OpenAIRE",
|
||||
"schemeid": "dnet:provenanceActions",
|
||||
"schemename": "dnet:provenanceActions"
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue