progress on call but not final
This commit is contained in:
parent
63bba067ba
commit
4d7836b4d7
|
@ -4,6 +4,7 @@ import eu.dnetlib.validator2.validation.XMLApplicationProfile;
|
||||||
import eu.dnetlib.validator2.validation.guideline.Guideline;
|
import eu.dnetlib.validator2.validation.guideline.Guideline;
|
||||||
import eu.dnetlib.validator2.validation.guideline.openaire.*;
|
import eu.dnetlib.validator2.validation.guideline.openaire.*;
|
||||||
import eu.dnetlib.validatorapi.entities.RuleInfo;
|
import eu.dnetlib.validatorapi.entities.RuleInfo;
|
||||||
|
import eu.dnetlib.validatorapi.entities.SummaryResult;
|
||||||
import eu.dnetlib.validatorapi.entities.ValidationJob;
|
import eu.dnetlib.validatorapi.entities.ValidationJob;
|
||||||
import eu.dnetlib.validatorapi.entities.ValidationJobResult;
|
import eu.dnetlib.validatorapi.entities.ValidationJobResult;
|
||||||
import eu.dnetlib.validatorapi.repository.ValidationJobRepository;
|
import eu.dnetlib.validatorapi.repository.ValidationJobRepository;
|
||||||
|
@ -11,8 +12,10 @@ import eu.dnetlib.validatorapi.repository.ValidationResultRepository;
|
||||||
import org.apache.logging.log4j.LogManager;
|
import org.apache.logging.log4j.LogManager;
|
||||||
import org.apache.logging.log4j.Logger;
|
import org.apache.logging.log4j.Logger;
|
||||||
import org.springframework.beans.factory.annotation.Autowired;
|
import org.springframework.beans.factory.annotation.Autowired;
|
||||||
|
import org.springframework.http.MediaType;
|
||||||
import org.springframework.web.bind.annotation.*;
|
import org.springframework.web.bind.annotation.*;
|
||||||
import org.w3c.dom.Document;
|
import org.w3c.dom.Document;
|
||||||
|
import org.w3c.dom.Element;
|
||||||
import org.w3c.dom.Node;
|
import org.w3c.dom.Node;
|
||||||
import org.w3c.dom.NodeList;
|
import org.w3c.dom.NodeList;
|
||||||
import org.xml.sax.InputSource;
|
import org.xml.sax.InputSource;
|
||||||
|
@ -28,8 +31,11 @@ import javax.xml.xpath.XPath;
|
||||||
import javax.xml.xpath.XPathConstants;
|
import javax.xml.xpath.XPathConstants;
|
||||||
import javax.xml.xpath.XPathExpression;
|
import javax.xml.xpath.XPathExpression;
|
||||||
import javax.xml.xpath.XPathFactory;
|
import javax.xml.xpath.XPathFactory;
|
||||||
|
import java.io.InputStream;
|
||||||
import java.io.StringReader;
|
import java.io.StringReader;
|
||||||
import java.io.StringWriter;
|
import java.io.StringWriter;
|
||||||
|
import java.net.HttpURLConnection;
|
||||||
|
import java.net.URL;
|
||||||
import java.util.*;
|
import java.util.*;
|
||||||
|
|
||||||
@RestController
|
@RestController
|
||||||
|
@ -46,20 +52,138 @@ public class ValidationController {
|
||||||
this.validationResultRepository = validationResultRepository;
|
this.validationResultRepository = validationResultRepository;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@RequestMapping(value = {"/realValidator"}, method = RequestMethod.POST)
|
||||||
@RequestMapping(value = {"/validateOAIPMH"}, method = RequestMethod.POST)
|
public void validateRealOAIPMH(@RequestParam(name = "guidelines") String guidelinesProfileName,
|
||||||
public void validateOAIPMH(@RequestParam(name = "guidelines") String guidelinesProfileName,
|
|
||||||
@RequestParam(name = "baseUrl", defaultValue = "localhost") String baseUrl, //not in use now
|
@RequestParam(name = "baseUrl", defaultValue = "localhost") String baseUrl, //not in use now
|
||||||
@RequestParam(name="numberOfRecords", defaultValue = "10") int numberOfRecords, //not in use now
|
@RequestParam(name="numberOfRecords", defaultValue = "10") int numberOfRecords //not in use now
|
||||||
@RequestBody String OAIPMHResponse) {
|
) {
|
||||||
|
|
||||||
ValidationJob validationJob = new ValidationJob(baseUrl, numberOfRecords);
|
ValidationJob validationJob = new ValidationJob(baseUrl, numberOfRecords);
|
||||||
|
|
||||||
List<RuleInfo> resultRules = null;
|
|
||||||
List<RuleInfo> fairRules = null;
|
|
||||||
AbstractOpenAireProfile profile = null;
|
AbstractOpenAireProfile profile = null;
|
||||||
AbstractOpenAireProfile fairProfile = null;
|
AbstractOpenAireProfile fairProfile = null;
|
||||||
|
|
||||||
|
profile = initializeProfile(guidelinesProfileName, profile, fairProfile);
|
||||||
|
validationJob.guidelines = profile.name();
|
||||||
|
System.out.println("Initial validation job "+ validationJob + " | " + validationJob.hashCode() + "\n");
|
||||||
|
|
||||||
|
//TODO uncomment
|
||||||
|
// validationJobRepository.save(validationJob);
|
||||||
|
int record = 0;
|
||||||
|
double resultSum = 0;
|
||||||
|
|
||||||
|
try {
|
||||||
|
System.out.println("making request....");
|
||||||
|
DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
|
||||||
|
DocumentBuilder db = dbf.newDocumentBuilder();
|
||||||
|
|
||||||
|
String baseURL = "http://repositorium.sdum.uminho.pt/oai/request"; // replace with your repository's base URL
|
||||||
|
String metadataPrefix = "oai_dc"; // replace with your desired metadata prefix
|
||||||
|
String set = null; // replace with your desired set, or null for all records
|
||||||
|
int batchSize = 50; // replace with your desired batch size
|
||||||
|
|
||||||
|
// Step 1: Make initial request
|
||||||
|
String url = baseURL + "?verb=ListRecords&metadataPrefix=" + metadataPrefix;
|
||||||
|
if (set != null) {
|
||||||
|
url += "&set=" + set;
|
||||||
|
}
|
||||||
|
// url += "&pageSize=" + batchSize;
|
||||||
|
System.out.println("URL " + url.toString());
|
||||||
|
HttpURLConnection conn = (HttpURLConnection) new URL(url).openConnection();
|
||||||
|
Document oaipmhResponse = parseResponse(conn.getInputStream());
|
||||||
|
|
||||||
|
int recordsRetrieved = 0;
|
||||||
|
// Step 2: Retrieve additional records using resumptionToken
|
||||||
|
while (recordsRetrieved < batchSize) {
|
||||||
|
System.out.println("True");
|
||||||
|
Element resumptionToken = (Element) oaipmhResponse.getElementsByTagName("resumptionToken").item(0);
|
||||||
|
if (resumptionToken == null || resumptionToken.getTextContent().isEmpty()) {
|
||||||
|
System.out.println("breaking");
|
||||||
|
break; // no more records to retrieve
|
||||||
|
}
|
||||||
|
url = baseURL + "?verb=ListRecords&resumptionToken=" + resumptionToken.getTextContent();
|
||||||
|
conn = (HttpURLConnection) new URL(url).openConnection();
|
||||||
|
oaipmhResponse = parseResponse(conn.getInputStream());
|
||||||
|
|
||||||
|
// TODO: process the records returned in this response
|
||||||
|
|
||||||
|
|
||||||
|
XPathFactory xfactory = XPathFactory.newInstance();
|
||||||
|
XPath xpath = xfactory.newXPath();
|
||||||
|
XPathExpression recordsExpression = xpath.compile("//record");
|
||||||
|
NodeList recordNodes = (NodeList) recordsExpression.evaluate(oaipmhResponse, XPathConstants.NODESET);
|
||||||
|
|
||||||
|
List<String> records = new ArrayList<String>();
|
||||||
|
for (int i = 0; i < recordNodes.getLength(); ++i) {
|
||||||
|
Node element = recordNodes.item(i);
|
||||||
|
|
||||||
|
StringWriter stringWriter = new StringWriter();
|
||||||
|
Transformer xform = TransformerFactory.newInstance().newTransformer();
|
||||||
|
xform.setOutputProperty(OutputKeys.OMIT_XML_DECLARATION, "yes"); // optional
|
||||||
|
xform.setOutputProperty(OutputKeys.INDENT, "yes"); // optional
|
||||||
|
xform.transform(new DOMSource(element), new StreamResult(stringWriter));
|
||||||
|
|
||||||
|
records.add(stringWriter.toString());
|
||||||
|
}
|
||||||
|
|
||||||
|
System.out.println("size " + records.size());
|
||||||
|
recordsRetrieved += records.size();
|
||||||
|
|
||||||
|
for (String recordXml : records) {
|
||||||
|
Document doc = db.parse(new InputSource(new StringReader(recordXml)));
|
||||||
|
if (profile != null) {
|
||||||
|
|
||||||
|
//what id is that?
|
||||||
|
XMLApplicationProfile.ValidationResult validationResult = profile.validate("id", doc);
|
||||||
|
|
||||||
|
Map<String, Guideline.Result> results = validationResult.results();
|
||||||
|
/* for (Map.Entry entry : results.entrySet()) {
|
||||||
|
|
||||||
|
ValidationJobResult validationJobResult = new ValidationJobResult();
|
||||||
|
validationJobResult.validationJobId = validationJob.id;
|
||||||
|
validationJobResult.ruleName = entry.getKey().toString();
|
||||||
|
validationJobResult.recordUrl = "localhost://records/record["+record+"]"; // silly id
|
||||||
|
Guideline.Result engineResult = (Guideline.Result) entry.getValue();
|
||||||
|
validationJobResult.score = engineResult.score();
|
||||||
|
validationJobResult.status = engineResult.status().toString();
|
||||||
|
validationJobResult.internalError = engineResult.internalError();
|
||||||
|
resultSum += engineResult.score();
|
||||||
|
|
||||||
|
validationJobResult.warnings = ((List<String>) engineResult.warnings()).toArray(new String[0]);
|
||||||
|
validationJobResult.errors = ((List<String>) engineResult.errors()).toArray(new String[0]);
|
||||||
|
System.out.println(validationJobResult + " | " + validationJobResult.hashCode() + "\n");
|
||||||
|
validationResultRepository.save(validationJobResult);
|
||||||
|
validationJobResults.add(validationJobResult);
|
||||||
|
|
||||||
|
|
||||||
|
}*/
|
||||||
|
|
||||||
|
}
|
||||||
|
record++;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
validationJob.status = "COMPLETED";
|
||||||
|
|
||||||
|
}
|
||||||
|
catch (Exception e) {
|
||||||
|
log.error("Validation job stopped unexpectedly.", e.getMessage());
|
||||||
|
System.out.println("ERROR " + e.getMessage());
|
||||||
|
validationJob.status = "STOPPED";
|
||||||
|
|
||||||
|
} finally {
|
||||||
|
validationJob.endDate = new Date();
|
||||||
|
System.out.println("Final validation job "+ validationJob.hashCode());
|
||||||
|
validationJob.recordsTested = record;
|
||||||
|
validationJob.score = resultSum / record;
|
||||||
|
//TODO uncomment
|
||||||
|
// validationJobRepository.save(validationJob);
|
||||||
|
}
|
||||||
|
|
||||||
|
//xmlValidationResponse.setRules(resultRules);
|
||||||
|
//xmlValidationResponse.setFairRules(fairRules);
|
||||||
|
}
|
||||||
|
|
||||||
|
private AbstractOpenAireProfile initializeProfile(@RequestParam(name = "guidelines") String guidelinesProfileName, AbstractOpenAireProfile profile, AbstractOpenAireProfile fairProfile) {
|
||||||
if (guidelinesProfileName.equals("dataArchiveGuidelinesV2Profile")) {
|
if (guidelinesProfileName.equals("dataArchiveGuidelinesV2Profile")) {
|
||||||
profile = new DataArchiveGuidelinesV2Profile();
|
profile = new DataArchiveGuidelinesV2Profile();
|
||||||
fairProfile = new FAIR_Data_GuidelinesProfile();
|
fairProfile = new FAIR_Data_GuidelinesProfile();
|
||||||
|
@ -78,6 +202,44 @@ public class ValidationController {
|
||||||
log.error("Exception: No valid guidelines");
|
log.error("Exception: No valid guidelines");
|
||||||
new Exception("Validation Job stopped unexpectedly. No valid guidelines were provided.");
|
new Exception("Validation Job stopped unexpectedly. No valid guidelines were provided.");
|
||||||
}
|
}
|
||||||
|
return profile;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
private Document parseResponse(InputStream stream)
|
||||||
|
throws Exception
|
||||||
|
{
|
||||||
|
DocumentBuilderFactory objDocumentBuilderFactory = null;
|
||||||
|
DocumentBuilder objDocumentBuilder = null;
|
||||||
|
Document doc = null;
|
||||||
|
try
|
||||||
|
{
|
||||||
|
objDocumentBuilderFactory = DocumentBuilderFactory.newInstance();
|
||||||
|
objDocumentBuilder = objDocumentBuilderFactory.newDocumentBuilder();
|
||||||
|
|
||||||
|
doc = objDocumentBuilder.parse(stream);
|
||||||
|
}
|
||||||
|
catch(Exception ex)
|
||||||
|
{
|
||||||
|
throw ex;
|
||||||
|
}
|
||||||
|
|
||||||
|
return doc;
|
||||||
|
}
|
||||||
|
@RequestMapping(value = {"/validateOAIPMH"}, method = RequestMethod.POST)
|
||||||
|
public void validateOAIPMH(@RequestParam(name = "guidelines") String guidelinesProfileName,
|
||||||
|
@RequestParam(name = "baseUrl", defaultValue = "localhost") String baseUrl, //not in use now
|
||||||
|
@RequestParam(name="numberOfRecords", defaultValue = "10") int numberOfRecords, //not in use now
|
||||||
|
@RequestBody String OAIPMHResponse) {
|
||||||
|
|
||||||
|
ValidationJob validationJob = new ValidationJob(baseUrl, numberOfRecords);
|
||||||
|
|
||||||
|
List<RuleInfo> resultRules = null;
|
||||||
|
List<RuleInfo> fairRules = null;
|
||||||
|
AbstractOpenAireProfile profile = null;
|
||||||
|
AbstractOpenAireProfile fairProfile = null;
|
||||||
|
|
||||||
|
profile = initializeProfile(guidelinesProfileName, profile, fairProfile);
|
||||||
|
|
||||||
validationJob.guidelines = profile.name();
|
validationJob.guidelines = profile.name();
|
||||||
System.out.println("Initial validation job "+ validationJob + " | " + validationJob.hashCode() + "\n");
|
System.out.println("Initial validation job "+ validationJob + " | " + validationJob.hashCode() + "\n");
|
||||||
|
@ -144,9 +306,12 @@ public class ValidationController {
|
||||||
//xmlValidationResponse.setFairRules(fairRules);
|
//xmlValidationResponse.setFairRules(fairRules);
|
||||||
}
|
}
|
||||||
|
|
||||||
@RequestMapping(value = {"/getResultsByJobId"}, method = RequestMethod.POST)
|
@RequestMapping(value = {"/getResultsByJobId"}, method = RequestMethod.POST,
|
||||||
public void getJobResults(@RequestParam(name = "jobId") String jobId){
|
produces = MediaType.APPLICATION_JSON_VALUE)
|
||||||
System.out.println(validationResultRepository.getSummarryResult().toString());
|
public List<SummaryResult> getJobResults(@RequestParam(name = "jobId") String jobId){
|
||||||
|
System.out.println(validationResultRepository.getSummaryResult().toString());
|
||||||
|
System.out.println("\n\n\n\n\n" + validationResultRepository.getSummaryResult().get(0));
|
||||||
|
return validationResultRepository.getSummaryResult();
|
||||||
}
|
}
|
||||||
|
|
||||||
public List<String> extractRecordXmls(String xml) throws Exception {
|
public List<String> extractRecordXmls(String xml) throws Exception {
|
||||||
|
|
|
@ -10,6 +10,7 @@ public class SummaryResult {
|
||||||
|
|
||||||
@Id
|
@Id
|
||||||
String rule_name;
|
String rule_name;
|
||||||
|
String rule_weight;
|
||||||
long passed_records;
|
long passed_records;
|
||||||
long failed_records;
|
long failed_records;
|
||||||
long errors;
|
long errors;
|
||||||
|
@ -23,14 +24,74 @@ public class SummaryResult {
|
||||||
this.warnings = warnings;
|
this.warnings = warnings;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public SummaryResult(String rule_name, String rule_weight, long passed_records, long failed_records, long errors, long warnings) {
|
||||||
|
this.rule_name = rule_name;
|
||||||
|
this.rule_weight = rule_weight;
|
||||||
|
this.passed_records = passed_records;
|
||||||
|
this.failed_records = failed_records;
|
||||||
|
this.errors = errors;
|
||||||
|
this.warnings = warnings;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getRule_name() {
|
||||||
|
return rule_name;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setRule_name(String rule_name) {
|
||||||
|
this.rule_name = rule_name;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getRule_weight() {
|
||||||
|
return rule_weight;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setRule_weight(String rule_weight) {
|
||||||
|
this.rule_weight = rule_weight;
|
||||||
|
}
|
||||||
|
|
||||||
|
public long getPassed_records() {
|
||||||
|
return passed_records;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setPassed_records(long passed_records) {
|
||||||
|
this.passed_records = passed_records;
|
||||||
|
}
|
||||||
|
|
||||||
|
public long getFailed_records() {
|
||||||
|
return failed_records;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setFailed_records(long failed_records) {
|
||||||
|
this.failed_records = failed_records;
|
||||||
|
}
|
||||||
|
|
||||||
|
public long getErrors() {
|
||||||
|
return errors;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setErrors(long errors) {
|
||||||
|
this.errors = errors;
|
||||||
|
}
|
||||||
|
|
||||||
|
public long getWarnings() {
|
||||||
|
return warnings;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setWarnings(long warnings) {
|
||||||
|
this.warnings = warnings;
|
||||||
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public String toString() {
|
public String toString() {
|
||||||
return "SummaryResult{" +
|
return "SummaryResult{" +
|
||||||
"rule_name='" + rule_name + '\'' +
|
"rule_name='" + rule_name + '\'' +
|
||||||
|
", rule_weight=" + rule_weight +
|
||||||
", passed_records=" + passed_records +
|
", passed_records=" + passed_records +
|
||||||
", failed_records=" + failed_records +
|
", failed_records=" + failed_records +
|
||||||
", errors=" + errors +
|
", errors=" + errors +
|
||||||
", warnings=" + warnings +
|
", warnings=" + warnings +
|
||||||
'}';
|
'}';
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
|
@ -23,7 +23,7 @@ public class ValidationJobResult implements Serializable {
|
||||||
|
|
||||||
@Id
|
@Id
|
||||||
@Column(name = "rule_weight")
|
@Column(name = "rule_weight")
|
||||||
public int ruleWeight;
|
public String ruleWeight;
|
||||||
|
|
||||||
@Id
|
@Id
|
||||||
@Column(name = "record_url")
|
@Column(name = "record_url")
|
||||||
|
|
|
@ -12,12 +12,12 @@ import java.util.List;
|
||||||
public interface ValidationResultRepository extends JpaRepository<ValidationJobResult, Long> {
|
public interface ValidationResultRepository extends JpaRepository<ValidationJobResult, Long> {
|
||||||
|
|
||||||
@Query(value =
|
@Query(value =
|
||||||
"SELECT NEW eu.dnetlib.validatorapi.entities.SummaryResult(sr.ruleName, " +
|
"SELECT NEW eu.dnetlib.validatorapi.entities.SummaryResult(sr.ruleName, sr.ruleWeight," +
|
||||||
"COUNT(CASE WHEN sr.status = 'SUCCESS' THEN 1 END) AS passed_records, " +
|
"COUNT(CASE WHEN sr.status = 'SUCCESS' THEN 1 END) AS passed_records, " +
|
||||||
"COUNT(CASE WHEN sr.status = 'FAILURE' THEN 1 END) AS failed_records, " +
|
"COUNT(CASE WHEN sr.status = 'FAILURE' THEN 1 END) AS failed_records, " +
|
||||||
"COUNT(CASE WHEN array_length(sr.errors, 1) > 0 THEN 1 END) AS errors, " +
|
"COUNT(CASE WHEN array_length(sr.errors, 1) > 0 THEN 1 END) AS errors, " +
|
||||||
"COUNT(CASE WHEN array_length(sr.warnings, 1) > 0 THEN 1 END) AS warnings) " +
|
"COUNT(CASE WHEN array_length(sr.warnings, 1) > 0 THEN 1 END) AS warnings) " +
|
||||||
"FROM ValidationJobResult sr " +
|
"FROM ValidationJobResult sr " +
|
||||||
"GROUP BY sr.ruleName")
|
"GROUP BY sr.ruleName, sr.ruleWeight")
|
||||||
List<SummaryResult> getSummarryResult();
|
List<SummaryResult> getSummaryResult();
|
||||||
}
|
}
|
||||||
|
|
|
@ -0,0 +1,33 @@
|
||||||
|
package eu.dnetlib.validatorapi.utils;
|
||||||
|
|
||||||
|
public class XmlProcessor { /* implements Processor {
|
||||||
|
private List<Element> elements;
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void process(Exchange exchange) throws Exception {
|
||||||
|
String xml = exchange.getIn().getBody(String.class);
|
||||||
|
|
||||||
|
DocumentBuilderFactory dbFactory = DocumentBuilderFactory.newInstance();
|
||||||
|
DocumentBuilder dBuilder = dbFactory.newDocumentBuilder();
|
||||||
|
Document doc = dBuilder.parse(new InputSource(new StringReader(xml)));
|
||||||
|
|
||||||
|
XPathFactory xfactory = XPathFactory.newInstance();
|
||||||
|
XPath xpath = xfactory.newXPath();
|
||||||
|
XPathExpression recordsExpression = xpath.compile("//record");
|
||||||
|
NodeList recordNodes = (NodeList) recordsExpression.evaluate(doc, XPathConstants.NODESET);
|
||||||
|
|
||||||
|
elements = nodeListToList(recordNodes); // assign the Elements to the variable
|
||||||
|
}
|
||||||
|
|
||||||
|
private List<Element> nodeListToList(NodeList nodeList) {
|
||||||
|
List<Element> list = new ArrayList<>();
|
||||||
|
for (int i = 0; i < nodeList.getLength(); i++) {
|
||||||
|
list.add((Element) nodeList.item(i));
|
||||||
|
}
|
||||||
|
return list;
|
||||||
|
}
|
||||||
|
|
||||||
|
public List<Element> getElements() {
|
||||||
|
return elements;
|
||||||
|
}*/
|
||||||
|
}
|
Loading…
Reference in New Issue