diff --git a/src/main/java/eu/dnetlib/validatorapi/controllers/ValidationController.java b/src/main/java/eu/dnetlib/validatorapi/controllers/ValidationController.java index 66f4d0b..09fa5a4 100644 --- a/src/main/java/eu/dnetlib/validatorapi/controllers/ValidationController.java +++ b/src/main/java/eu/dnetlib/validatorapi/controllers/ValidationController.java @@ -4,6 +4,7 @@ import eu.dnetlib.validator2.validation.XMLApplicationProfile; import eu.dnetlib.validator2.validation.guideline.Guideline; import eu.dnetlib.validator2.validation.guideline.openaire.*; import eu.dnetlib.validatorapi.entities.RuleInfo; +import eu.dnetlib.validatorapi.entities.SummaryResult; import eu.dnetlib.validatorapi.entities.ValidationJob; import eu.dnetlib.validatorapi.entities.ValidationJobResult; import eu.dnetlib.validatorapi.repository.ValidationJobRepository; @@ -11,8 +12,10 @@ import eu.dnetlib.validatorapi.repository.ValidationResultRepository; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.http.MediaType; import org.springframework.web.bind.annotation.*; import org.w3c.dom.Document; +import org.w3c.dom.Element; import org.w3c.dom.Node; import org.w3c.dom.NodeList; import org.xml.sax.InputSource; @@ -28,8 +31,11 @@ import javax.xml.xpath.XPath; import javax.xml.xpath.XPathConstants; import javax.xml.xpath.XPathExpression; import javax.xml.xpath.XPathFactory; +import java.io.InputStream; import java.io.StringReader; import java.io.StringWriter; +import java.net.HttpURLConnection; +import java.net.URL; import java.util.*; @RestController @@ -46,20 +52,138 @@ public class ValidationController { this.validationResultRepository = validationResultRepository; } - - @RequestMapping(value = {"/validateOAIPMH"}, method = RequestMethod.POST) - public void validateOAIPMH(@RequestParam(name = "guidelines") String guidelinesProfileName, - @RequestParam(name = "baseUrl", defaultValue = "localhost") String baseUrl, //not in use now - @RequestParam(name="numberOfRecords", defaultValue = "10") int numberOfRecords, //not in use now - @RequestBody String OAIPMHResponse) { + @RequestMapping(value = {"/realValidator"}, method = RequestMethod.POST) + public void validateRealOAIPMH(@RequestParam(name = "guidelines") String guidelinesProfileName, + @RequestParam(name = "baseUrl", defaultValue = "localhost") String baseUrl, //not in use now + @RequestParam(name="numberOfRecords", defaultValue = "10") int numberOfRecords //not in use now + ) { ValidationJob validationJob = new ValidationJob(baseUrl, numberOfRecords); - List resultRules = null; - List fairRules = null; AbstractOpenAireProfile profile = null; AbstractOpenAireProfile fairProfile = null; + profile = initializeProfile(guidelinesProfileName, profile, fairProfile); + validationJob.guidelines = profile.name(); + System.out.println("Initial validation job "+ validationJob + " | " + validationJob.hashCode() + "\n"); + + //TODO uncomment + // validationJobRepository.save(validationJob); + int record = 0; + double resultSum = 0; + + try { + System.out.println("making request...."); + DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance(); + DocumentBuilder db = dbf.newDocumentBuilder(); + + String baseURL = "http://repositorium.sdum.uminho.pt/oai/request"; // replace with your repository's base URL + String metadataPrefix = "oai_dc"; // replace with your desired metadata prefix + String set = null; // replace with your desired set, or null for all records + int batchSize = 50; // replace with your desired batch size + + // Step 1: Make initial request + String url = baseURL + "?verb=ListRecords&metadataPrefix=" + metadataPrefix; + if (set != null) { + url += "&set=" + set; + } + // url += "&pageSize=" + batchSize; + System.out.println("URL " + url.toString()); + HttpURLConnection conn = (HttpURLConnection) new URL(url).openConnection(); + Document oaipmhResponse = parseResponse(conn.getInputStream()); + + int recordsRetrieved = 0; + // Step 2: Retrieve additional records using resumptionToken + while (recordsRetrieved < batchSize) { + System.out.println("True"); + Element resumptionToken = (Element) oaipmhResponse.getElementsByTagName("resumptionToken").item(0); + if (resumptionToken == null || resumptionToken.getTextContent().isEmpty()) { + System.out.println("breaking"); + break; // no more records to retrieve + } + url = baseURL + "?verb=ListRecords&resumptionToken=" + resumptionToken.getTextContent(); + conn = (HttpURLConnection) new URL(url).openConnection(); + oaipmhResponse = parseResponse(conn.getInputStream()); + + // TODO: process the records returned in this response + + + XPathFactory xfactory = XPathFactory.newInstance(); + XPath xpath = xfactory.newXPath(); + XPathExpression recordsExpression = xpath.compile("//record"); + NodeList recordNodes = (NodeList) recordsExpression.evaluate(oaipmhResponse, XPathConstants.NODESET); + + List records = new ArrayList(); + for (int i = 0; i < recordNodes.getLength(); ++i) { + Node element = recordNodes.item(i); + + StringWriter stringWriter = new StringWriter(); + Transformer xform = TransformerFactory.newInstance().newTransformer(); + xform.setOutputProperty(OutputKeys.OMIT_XML_DECLARATION, "yes"); // optional + xform.setOutputProperty(OutputKeys.INDENT, "yes"); // optional + xform.transform(new DOMSource(element), new StreamResult(stringWriter)); + + records.add(stringWriter.toString()); + } + + System.out.println("size " + records.size()); + recordsRetrieved += records.size(); + + for (String recordXml : records) { + Document doc = db.parse(new InputSource(new StringReader(recordXml))); + if (profile != null) { + + //what id is that? + XMLApplicationProfile.ValidationResult validationResult = profile.validate("id", doc); + + Map results = validationResult.results(); + /* for (Map.Entry entry : results.entrySet()) { + + ValidationJobResult validationJobResult = new ValidationJobResult(); + validationJobResult.validationJobId = validationJob.id; + validationJobResult.ruleName = entry.getKey().toString(); + validationJobResult.recordUrl = "localhost://records/record["+record+"]"; // silly id + Guideline.Result engineResult = (Guideline.Result) entry.getValue(); + validationJobResult.score = engineResult.score(); + validationJobResult.status = engineResult.status().toString(); + validationJobResult.internalError = engineResult.internalError(); + resultSum += engineResult.score(); + + validationJobResult.warnings = ((List) engineResult.warnings()).toArray(new String[0]); + validationJobResult.errors = ((List) engineResult.errors()).toArray(new String[0]); + System.out.println(validationJobResult + " | " + validationJobResult.hashCode() + "\n"); + validationResultRepository.save(validationJobResult); + validationJobResults.add(validationJobResult); + + + }*/ + + } + record++; + } + } + validationJob.status = "COMPLETED"; + + } + catch (Exception e) { + log.error("Validation job stopped unexpectedly.", e.getMessage()); + System.out.println("ERROR " + e.getMessage()); + validationJob.status = "STOPPED"; + + } finally { + validationJob.endDate = new Date(); + System.out.println("Final validation job "+ validationJob.hashCode()); + validationJob.recordsTested = record; + validationJob.score = resultSum / record; + //TODO uncomment + // validationJobRepository.save(validationJob); + } + + //xmlValidationResponse.setRules(resultRules); + //xmlValidationResponse.setFairRules(fairRules); + } + + private AbstractOpenAireProfile initializeProfile(@RequestParam(name = "guidelines") String guidelinesProfileName, AbstractOpenAireProfile profile, AbstractOpenAireProfile fairProfile) { if (guidelinesProfileName.equals("dataArchiveGuidelinesV2Profile")) { profile = new DataArchiveGuidelinesV2Profile(); fairProfile = new FAIR_Data_GuidelinesProfile(); @@ -78,6 +202,44 @@ public class ValidationController { log.error("Exception: No valid guidelines"); new Exception("Validation Job stopped unexpectedly. No valid guidelines were provided."); } + return profile; + } + + + private Document parseResponse(InputStream stream) + throws Exception + { + DocumentBuilderFactory objDocumentBuilderFactory = null; + DocumentBuilder objDocumentBuilder = null; + Document doc = null; + try + { + objDocumentBuilderFactory = DocumentBuilderFactory.newInstance(); + objDocumentBuilder = objDocumentBuilderFactory.newDocumentBuilder(); + + doc = objDocumentBuilder.parse(stream); + } + catch(Exception ex) + { + throw ex; + } + + return doc; + } + @RequestMapping(value = {"/validateOAIPMH"}, method = RequestMethod.POST) + public void validateOAIPMH(@RequestParam(name = "guidelines") String guidelinesProfileName, + @RequestParam(name = "baseUrl", defaultValue = "localhost") String baseUrl, //not in use now + @RequestParam(name="numberOfRecords", defaultValue = "10") int numberOfRecords, //not in use now + @RequestBody String OAIPMHResponse) { + + ValidationJob validationJob = new ValidationJob(baseUrl, numberOfRecords); + + List resultRules = null; + List fairRules = null; + AbstractOpenAireProfile profile = null; + AbstractOpenAireProfile fairProfile = null; + + profile = initializeProfile(guidelinesProfileName, profile, fairProfile); validationJob.guidelines = profile.name(); System.out.println("Initial validation job "+ validationJob + " | " + validationJob.hashCode() + "\n"); @@ -144,9 +306,12 @@ public class ValidationController { //xmlValidationResponse.setFairRules(fairRules); } - @RequestMapping(value = {"/getResultsByJobId"}, method = RequestMethod.POST) - public void getJobResults(@RequestParam(name = "jobId") String jobId){ - System.out.println(validationResultRepository.getSummarryResult().toString()); + @RequestMapping(value = {"/getResultsByJobId"}, method = RequestMethod.POST, + produces = MediaType.APPLICATION_JSON_VALUE) + public List getJobResults(@RequestParam(name = "jobId") String jobId){ + System.out.println(validationResultRepository.getSummaryResult().toString()); + System.out.println("\n\n\n\n\n" + validationResultRepository.getSummaryResult().get(0)); + return validationResultRepository.getSummaryResult(); } public List extractRecordXmls(String xml) throws Exception { diff --git a/src/main/java/eu/dnetlib/validatorapi/entities/SummaryResult.java b/src/main/java/eu/dnetlib/validatorapi/entities/SummaryResult.java index 5dfd08b..72060e6 100644 --- a/src/main/java/eu/dnetlib/validatorapi/entities/SummaryResult.java +++ b/src/main/java/eu/dnetlib/validatorapi/entities/SummaryResult.java @@ -10,6 +10,7 @@ public class SummaryResult { @Id String rule_name; + String rule_weight; long passed_records; long failed_records; long errors; @@ -23,14 +24,74 @@ public class SummaryResult { this.warnings = warnings; } + public SummaryResult(String rule_name, String rule_weight, long passed_records, long failed_records, long errors, long warnings) { + this.rule_name = rule_name; + this.rule_weight = rule_weight; + this.passed_records = passed_records; + this.failed_records = failed_records; + this.errors = errors; + this.warnings = warnings; + } + + public String getRule_name() { + return rule_name; + } + + public void setRule_name(String rule_name) { + this.rule_name = rule_name; + } + + public String getRule_weight() { + return rule_weight; + } + + public void setRule_weight(String rule_weight) { + this.rule_weight = rule_weight; + } + + public long getPassed_records() { + return passed_records; + } + + public void setPassed_records(long passed_records) { + this.passed_records = passed_records; + } + + public long getFailed_records() { + return failed_records; + } + + public void setFailed_records(long failed_records) { + this.failed_records = failed_records; + } + + public long getErrors() { + return errors; + } + + public void setErrors(long errors) { + this.errors = errors; + } + + public long getWarnings() { + return warnings; + } + + public void setWarnings(long warnings) { + this.warnings = warnings; + } + @Override public String toString() { return "SummaryResult{" + "rule_name='" + rule_name + '\'' + + ", rule_weight=" + rule_weight + ", passed_records=" + passed_records + ", failed_records=" + failed_records + ", errors=" + errors + ", warnings=" + warnings + '}'; } + + } diff --git a/src/main/java/eu/dnetlib/validatorapi/entities/ValidationJobResult.java b/src/main/java/eu/dnetlib/validatorapi/entities/ValidationJobResult.java index fb4fd72..2187200 100644 --- a/src/main/java/eu/dnetlib/validatorapi/entities/ValidationJobResult.java +++ b/src/main/java/eu/dnetlib/validatorapi/entities/ValidationJobResult.java @@ -23,7 +23,7 @@ public class ValidationJobResult implements Serializable { @Id @Column(name = "rule_weight") - public int ruleWeight; + public String ruleWeight; @Id @Column(name = "record_url") diff --git a/src/main/java/eu/dnetlib/validatorapi/repository/ValidationResultRepository.java b/src/main/java/eu/dnetlib/validatorapi/repository/ValidationResultRepository.java index 931dbc9..d4cce6f 100644 --- a/src/main/java/eu/dnetlib/validatorapi/repository/ValidationResultRepository.java +++ b/src/main/java/eu/dnetlib/validatorapi/repository/ValidationResultRepository.java @@ -12,12 +12,12 @@ import java.util.List; public interface ValidationResultRepository extends JpaRepository { @Query(value = - "SELECT NEW eu.dnetlib.validatorapi.entities.SummaryResult(sr.ruleName, " + + "SELECT NEW eu.dnetlib.validatorapi.entities.SummaryResult(sr.ruleName, sr.ruleWeight," + "COUNT(CASE WHEN sr.status = 'SUCCESS' THEN 1 END) AS passed_records, " + "COUNT(CASE WHEN sr.status = 'FAILURE' THEN 1 END) AS failed_records, " + "COUNT(CASE WHEN array_length(sr.errors, 1) > 0 THEN 1 END) AS errors, " + "COUNT(CASE WHEN array_length(sr.warnings, 1) > 0 THEN 1 END) AS warnings) " + "FROM ValidationJobResult sr " + - "GROUP BY sr.ruleName") - List getSummarryResult(); + "GROUP BY sr.ruleName, sr.ruleWeight") + List getSummaryResult(); } diff --git a/src/main/java/eu/dnetlib/validatorapi/utils/XmlProcessor.java b/src/main/java/eu/dnetlib/validatorapi/utils/XmlProcessor.java new file mode 100644 index 0000000..881558b --- /dev/null +++ b/src/main/java/eu/dnetlib/validatorapi/utils/XmlProcessor.java @@ -0,0 +1,33 @@ +package eu.dnetlib.validatorapi.utils; + +public class XmlProcessor { /* implements Processor { + private List elements; + + @Override + public void process(Exchange exchange) throws Exception { + String xml = exchange.getIn().getBody(String.class); + + DocumentBuilderFactory dbFactory = DocumentBuilderFactory.newInstance(); + DocumentBuilder dBuilder = dbFactory.newDocumentBuilder(); + Document doc = dBuilder.parse(new InputSource(new StringReader(xml))); + + XPathFactory xfactory = XPathFactory.newInstance(); + XPath xpath = xfactory.newXPath(); + XPathExpression recordsExpression = xpath.compile("//record"); + NodeList recordNodes = (NodeList) recordsExpression.evaluate(doc, XPathConstants.NODESET); + + elements = nodeListToList(recordNodes); // assign the Elements to the variable + } + + private List nodeListToList(NodeList nodeList) { + List list = new ArrayList<>(); + for (int i = 0; i < nodeList.getLength(); i++) { + list.add((Element) nodeList.item(i)); + } + return list; + } + + public List getElements() { + return elements; + }*/ +}