progress on call but not final
This commit is contained in:
Katerina
parent
63bba067ba
commit
4d7836b4d7
|
|
@ -4,6 +4,7 @@ import eu.dnetlib.validator2.validation.XMLApplicationProfile;
|
|||
import eu.dnetlib.validator2.validation.guideline.Guideline;
|
||||
import eu.dnetlib.validator2.validation.guideline.openaire.*;
|
||||
import eu.dnetlib.validatorapi.entities.RuleInfo;
|
||||
import eu.dnetlib.validatorapi.entities.SummaryResult;
|
||||
import eu.dnetlib.validatorapi.entities.ValidationJob;
|
||||
import eu.dnetlib.validatorapi.entities.ValidationJobResult;
|
||||
import eu.dnetlib.validatorapi.repository.ValidationJobRepository;
|
||||
|
|
@ -11,8 +12,10 @@ import eu.dnetlib.validatorapi.repository.ValidationResultRepository;
|
|||
import org.apache.logging.log4j.LogManager;
|
||||
import org.apache.logging.log4j.Logger;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.http.MediaType;
|
||||
import org.springframework.web.bind.annotation.*;
|
||||
import org.w3c.dom.Document;
|
||||
import org.w3c.dom.Element;
|
||||
import org.w3c.dom.Node;
|
||||
import org.w3c.dom.NodeList;
|
||||
import org.xml.sax.InputSource;
|
||||
|
|
@ -28,8 +31,11 @@ import javax.xml.xpath.XPath;
|
|||
import javax.xml.xpath.XPathConstants;
|
||||
import javax.xml.xpath.XPathExpression;
|
||||
import javax.xml.xpath.XPathFactory;
|
||||
import java.io.InputStream;
|
||||
import java.io.StringReader;
|
||||
import java.io.StringWriter;
|
||||
import java.net.HttpURLConnection;
|
||||
import java.net.URL;
|
||||
import java.util.*;
|
||||
|
||||
@RestController
|
||||
|
|
@ -46,20 +52,138 @@ public class ValidationController {
|
|||
this.validationResultRepository = validationResultRepository;
|
||||
}
|
||||
|
||||
|
||||
@RequestMapping(value = {"/validateOAIPMH"}, method = RequestMethod.POST)
|
||||
public void validateOAIPMH(@RequestParam(name = "guidelines") String guidelinesProfileName,
|
||||
@RequestParam(name = "baseUrl", defaultValue = "localhost") String baseUrl, //not in use now
|
||||
@RequestParam(name="numberOfRecords", defaultValue = "10") int numberOfRecords, //not in use now
|
||||
@RequestBody String OAIPMHResponse) {
|
||||
@RequestMapping(value = {"/realValidator"}, method = RequestMethod.POST)
|
||||
public void validateRealOAIPMH(@RequestParam(name = "guidelines") String guidelinesProfileName,
|
||||
@RequestParam(name = "baseUrl", defaultValue = "localhost") String baseUrl, //not in use now
|
||||
@RequestParam(name="numberOfRecords", defaultValue = "10") int numberOfRecords //not in use now
|
||||
) {
|
||||
|
||||
ValidationJob validationJob = new ValidationJob(baseUrl, numberOfRecords);
|
||||
|
||||
List<RuleInfo> resultRules = null;
|
||||
List<RuleInfo> fairRules = null;
|
||||
AbstractOpenAireProfile profile = null;
|
||||
AbstractOpenAireProfile fairProfile = null;
|
||||
|
||||
profile = initializeProfile(guidelinesProfileName, profile, fairProfile);
|
||||
validationJob.guidelines = profile.name();
|
||||
System.out.println("Initial validation job "+ validationJob + " | " + validationJob.hashCode() + "\n");
|
||||
|
||||
//TODO uncomment
|
||||
// validationJobRepository.save(validationJob);
|
||||
int record = 0;
|
||||
double resultSum = 0;
|
||||
|
||||
try {
|
||||
System.out.println("making request....");
|
||||
DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
|
||||
DocumentBuilder db = dbf.newDocumentBuilder();
|
||||
|
||||
String baseURL = "http://repositorium.sdum.uminho.pt/oai/request"; // replace with your repository's base URL
|
||||
String metadataPrefix = "oai_dc"; // replace with your desired metadata prefix
|
||||
String set = null; // replace with your desired set, or null for all records
|
||||
int batchSize = 50; // replace with your desired batch size
|
||||
|
||||
// Step 1: Make initial request
|
||||
String url = baseURL + "?verb=ListRecords&metadataPrefix=" + metadataPrefix;
|
||||
if (set != null) {
|
||||
url += "&set=" + set;
|
||||
}
|
||||
// url += "&pageSize=" + batchSize;
|
||||
System.out.println("URL " + url.toString());
|
||||
HttpURLConnection conn = (HttpURLConnection) new URL(url).openConnection();
|
||||
Document oaipmhResponse = parseResponse(conn.getInputStream());
|
||||
|
||||
int recordsRetrieved = 0;
|
||||
// Step 2: Retrieve additional records using resumptionToken
|
||||
while (recordsRetrieved < batchSize) {
|
||||
System.out.println("True");
|
||||
Element resumptionToken = (Element) oaipmhResponse.getElementsByTagName("resumptionToken").item(0);
|
||||
if (resumptionToken == null || resumptionToken.getTextContent().isEmpty()) {
|
||||
System.out.println("breaking");
|
||||
break; // no more records to retrieve
|
||||
}
|
||||
url = baseURL + "?verb=ListRecords&resumptionToken=" + resumptionToken.getTextContent();
|
||||
conn = (HttpURLConnection) new URL(url).openConnection();
|
||||
oaipmhResponse = parseResponse(conn.getInputStream());
|
||||
|
||||
// TODO: process the records returned in this response
|
||||
|
||||
|
||||
XPathFactory xfactory = XPathFactory.newInstance();
|
||||
XPath xpath = xfactory.newXPath();
|
||||
XPathExpression recordsExpression = xpath.compile("//record");
|
||||
NodeList recordNodes = (NodeList) recordsExpression.evaluate(oaipmhResponse, XPathConstants.NODESET);
|
||||
|
||||
List<String> records = new ArrayList<String>();
|
||||
for (int i = 0; i < recordNodes.getLength(); ++i) {
|
||||
Node element = recordNodes.item(i);
|
||||
|
||||
StringWriter stringWriter = new StringWriter();
|
||||
Transformer xform = TransformerFactory.newInstance().newTransformer();
|
||||
xform.setOutputProperty(OutputKeys.OMIT_XML_DECLARATION, "yes"); // optional
|
||||
xform.setOutputProperty(OutputKeys.INDENT, "yes"); // optional
|
||||
xform.transform(new DOMSource(element), new StreamResult(stringWriter));
|
||||
|
||||
records.add(stringWriter.toString());
|
||||
}
|
||||
|
||||
System.out.println("size " + records.size());
|
||||
recordsRetrieved += records.size();
|
||||
|
||||
for (String recordXml : records) {
|
||||
Document doc = db.parse(new InputSource(new StringReader(recordXml)));
|
||||
if (profile != null) {
|
||||
|
||||
//what id is that?
|
||||
XMLApplicationProfile.ValidationResult validationResult = profile.validate("id", doc);
|
||||
|
||||
Map<String, Guideline.Result> results = validationResult.results();
|
||||
/* for (Map.Entry entry : results.entrySet()) {
|
||||
|
||||
ValidationJobResult validationJobResult = new ValidationJobResult();
|
||||
validationJobResult.validationJobId = validationJob.id;
|
||||
validationJobResult.ruleName = entry.getKey().toString();
|
||||
validationJobResult.recordUrl = "localhost://records/record["+record+"]"; // silly id
|
||||
Guideline.Result engineResult = (Guideline.Result) entry.getValue();
|
||||
validationJobResult.score = engineResult.score();
|
||||
validationJobResult.status = engineResult.status().toString();
|
||||
validationJobResult.internalError = engineResult.internalError();
|
||||
resultSum += engineResult.score();
|
||||
|
||||
validationJobResult.warnings = ((List<String>) engineResult.warnings()).toArray(new String[0]);
|
||||
validationJobResult.errors = ((List<String>) engineResult.errors()).toArray(new String[0]);
|
||||
System.out.println(validationJobResult + " | " + validationJobResult.hashCode() + "\n");
|
||||
validationResultRepository.save(validationJobResult);
|
||||
validationJobResults.add(validationJobResult);
|
||||
|
||||
|
||||
}*/
|
||||
|
||||
}
|
||||
record++;
|
||||
}
|
||||
}
|
||||
validationJob.status = "COMPLETED";
|
||||
|
||||
}
|
||||
catch (Exception e) {
|
||||
log.error("Validation job stopped unexpectedly.", e.getMessage());
|
||||
System.out.println("ERROR " + e.getMessage());
|
||||
validationJob.status = "STOPPED";
|
||||
|
||||
} finally {
|
||||
validationJob.endDate = new Date();
|
||||
System.out.println("Final validation job "+ validationJob.hashCode());
|
||||
validationJob.recordsTested = record;
|
||||
validationJob.score = resultSum / record;
|
||||
//TODO uncomment
|
||||
// validationJobRepository.save(validationJob);
|
||||
}
|
||||
|
||||
//xmlValidationResponse.setRules(resultRules);
|
||||
//xmlValidationResponse.setFairRules(fairRules);
|
||||
}
|
||||
|
||||
private AbstractOpenAireProfile initializeProfile(@RequestParam(name = "guidelines") String guidelinesProfileName, AbstractOpenAireProfile profile, AbstractOpenAireProfile fairProfile) {
|
||||
if (guidelinesProfileName.equals("dataArchiveGuidelinesV2Profile")) {
|
||||
profile = new DataArchiveGuidelinesV2Profile();
|
||||
fairProfile = new FAIR_Data_GuidelinesProfile();
|
||||
|
|
@ -78,6 +202,44 @@ public class ValidationController {
|
|||
log.error("Exception: No valid guidelines");
|
||||
new Exception("Validation Job stopped unexpectedly. No valid guidelines were provided.");
|
||||
}
|
||||
return profile;
|
||||
}
|
||||
|
||||
|
||||
private Document parseResponse(InputStream stream)
|
||||
throws Exception
|
||||
{
|
||||
DocumentBuilderFactory objDocumentBuilderFactory = null;
|
||||
DocumentBuilder objDocumentBuilder = null;
|
||||
Document doc = null;
|
||||
try
|
||||
{
|
||||
objDocumentBuilderFactory = DocumentBuilderFactory.newInstance();
|
||||
objDocumentBuilder = objDocumentBuilderFactory.newDocumentBuilder();
|
||||
|
||||
doc = objDocumentBuilder.parse(stream);
|
||||
}
|
||||
catch(Exception ex)
|
||||
{
|
||||
throw ex;
|
||||
}
|
||||
|
||||
return doc;
|
||||
}
|
||||
@RequestMapping(value = {"/validateOAIPMH"}, method = RequestMethod.POST)
|
||||
public void validateOAIPMH(@RequestParam(name = "guidelines") String guidelinesProfileName,
|
||||
@RequestParam(name = "baseUrl", defaultValue = "localhost") String baseUrl, //not in use now
|
||||
@RequestParam(name="numberOfRecords", defaultValue = "10") int numberOfRecords, //not in use now
|
||||
@RequestBody String OAIPMHResponse) {
|
||||
|
||||
ValidationJob validationJob = new ValidationJob(baseUrl, numberOfRecords);
|
||||
|
||||
List<RuleInfo> resultRules = null;
|
||||
List<RuleInfo> fairRules = null;
|
||||
AbstractOpenAireProfile profile = null;
|
||||
AbstractOpenAireProfile fairProfile = null;
|
||||
|
||||
profile = initializeProfile(guidelinesProfileName, profile, fairProfile);
|
||||
|
||||
validationJob.guidelines = profile.name();
|
||||
System.out.println("Initial validation job "+ validationJob + " | " + validationJob.hashCode() + "\n");
|
||||
|
|
@ -144,9 +306,12 @@ public class ValidationController {
|
|||
//xmlValidationResponse.setFairRules(fairRules);
|
||||
}
|
||||
|
||||
@RequestMapping(value = {"/getResultsByJobId"}, method = RequestMethod.POST)
|
||||
public void getJobResults(@RequestParam(name = "jobId") String jobId){
|
||||
System.out.println(validationResultRepository.getSummarryResult().toString());
|
||||
@RequestMapping(value = {"/getResultsByJobId"}, method = RequestMethod.POST,
|
||||
produces = MediaType.APPLICATION_JSON_VALUE)
|
||||
public List<SummaryResult> getJobResults(@RequestParam(name = "jobId") String jobId){
|
||||
System.out.println(validationResultRepository.getSummaryResult().toString());
|
||||
System.out.println("\n\n\n\n\n" + validationResultRepository.getSummaryResult().get(0));
|
||||
return validationResultRepository.getSummaryResult();
|
||||
}
|
||||
|
||||
public List<String> extractRecordXmls(String xml) throws Exception {
|
||||
|
|
|
|||
|
|
@ -10,6 +10,7 @@ public class SummaryResult {
|
|||
|
||||
@Id
|
||||
String rule_name;
|
||||
String rule_weight;
|
||||
long passed_records;
|
||||
long failed_records;
|
||||
long errors;
|
||||
|
|
@ -23,14 +24,74 @@ public class SummaryResult {
|
|||
this.warnings = warnings;
|
||||
}
|
||||
|
||||
public SummaryResult(String rule_name, String rule_weight, long passed_records, long failed_records, long errors, long warnings) {
|
||||
this.rule_name = rule_name;
|
||||
this.rule_weight = rule_weight;
|
||||
this.passed_records = passed_records;
|
||||
this.failed_records = failed_records;
|
||||
this.errors = errors;
|
||||
this.warnings = warnings;
|
||||
}
|
||||
|
||||
public String getRule_name() {
|
||||
return rule_name;
|
||||
}
|
||||
|
||||
public void setRule_name(String rule_name) {
|
||||
this.rule_name = rule_name;
|
||||
}
|
||||
|
||||
public String getRule_weight() {
|
||||
return rule_weight;
|
||||
}
|
||||
|
||||
public void setRule_weight(String rule_weight) {
|
||||
this.rule_weight = rule_weight;
|
||||
}
|
||||
|
||||
public long getPassed_records() {
|
||||
return passed_records;
|
||||
}
|
||||
|
||||
public void setPassed_records(long passed_records) {
|
||||
this.passed_records = passed_records;
|
||||
}
|
||||
|
||||
public long getFailed_records() {
|
||||
return failed_records;
|
||||
}
|
||||
|
||||
public void setFailed_records(long failed_records) {
|
||||
this.failed_records = failed_records;
|
||||
}
|
||||
|
||||
public long getErrors() {
|
||||
return errors;
|
||||
}
|
||||
|
||||
public void setErrors(long errors) {
|
||||
this.errors = errors;
|
||||
}
|
||||
|
||||
public long getWarnings() {
|
||||
return warnings;
|
||||
}
|
||||
|
||||
public void setWarnings(long warnings) {
|
||||
this.warnings = warnings;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return "SummaryResult{" +
|
||||
"rule_name='" + rule_name + '\'' +
|
||||
", rule_weight=" + rule_weight +
|
||||
", passed_records=" + passed_records +
|
||||
", failed_records=" + failed_records +
|
||||
", errors=" + errors +
|
||||
", warnings=" + warnings +
|
||||
'}';
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -23,7 +23,7 @@ public class ValidationJobResult implements Serializable {
|
|||
|
||||
@Id
|
||||
@Column(name = "rule_weight")
|
||||
public int ruleWeight;
|
||||
public String ruleWeight;
|
||||
|
||||
@Id
|
||||
@Column(name = "record_url")
|
||||
|
|
|
|||
|
|
@ -12,12 +12,12 @@ import java.util.List;
|
|||
public interface ValidationResultRepository extends JpaRepository<ValidationJobResult, Long> {
|
||||
|
||||
@Query(value =
|
||||
"SELECT NEW eu.dnetlib.validatorapi.entities.SummaryResult(sr.ruleName, " +
|
||||
"SELECT NEW eu.dnetlib.validatorapi.entities.SummaryResult(sr.ruleName, sr.ruleWeight," +
|
||||
"COUNT(CASE WHEN sr.status = 'SUCCESS' THEN 1 END) AS passed_records, " +
|
||||
"COUNT(CASE WHEN sr.status = 'FAILURE' THEN 1 END) AS failed_records, " +
|
||||
"COUNT(CASE WHEN array_length(sr.errors, 1) > 0 THEN 1 END) AS errors, " +
|
||||
"COUNT(CASE WHEN array_length(sr.warnings, 1) > 0 THEN 1 END) AS warnings) " +
|
||||
"FROM ValidationJobResult sr " +
|
||||
"GROUP BY sr.ruleName")
|
||||
List<SummaryResult> getSummarryResult();
|
||||
"GROUP BY sr.ruleName, sr.ruleWeight")
|
||||
List<SummaryResult> getSummaryResult();
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,33 @@
|
|||
package eu.dnetlib.validatorapi.utils;
|
||||
|
||||
public class XmlProcessor { /* implements Processor {
|
||||
private List<Element> elements;
|
||||
|
||||
@Override
|
||||
public void process(Exchange exchange) throws Exception {
|
||||
String xml = exchange.getIn().getBody(String.class);
|
||||
|
||||
DocumentBuilderFactory dbFactory = DocumentBuilderFactory.newInstance();
|
||||
DocumentBuilder dBuilder = dbFactory.newDocumentBuilder();
|
||||
Document doc = dBuilder.parse(new InputSource(new StringReader(xml)));
|
||||
|
||||
XPathFactory xfactory = XPathFactory.newInstance();
|
||||
XPath xpath = xfactory.newXPath();
|
||||
XPathExpression recordsExpression = xpath.compile("//record");
|
||||
NodeList recordNodes = (NodeList) recordsExpression.evaluate(doc, XPathConstants.NODESET);
|
||||
|
||||
elements = nodeListToList(recordNodes); // assign the Elements to the variable
|
||||
}
|
||||
|
||||
private List<Element> nodeListToList(NodeList nodeList) {
|
||||
List<Element> list = new ArrayList<>();
|
||||
for (int i = 0; i < nodeList.getLength(); i++) {
|
||||
list.add((Element) nodeList.item(i));
|
||||
}
|
||||
return list;
|
||||
}
|
||||
|
||||
public List<Element> getElements() {
|
||||
return elements;
|
||||
}*/
|
||||
}
|
||||
Loading…
Reference in New Issue