From 3faf80e84dbf7cf826d0ed806abce93c8a81257f Mon Sep 17 00:00:00 2001 From: "michele.artini" Date: Tue, 14 Feb 2023 16:55:42 +0100 Subject: [PATCH] cleaner --- .../data/mapping/MappingAjaxController.java | 43 +++++++++++++++++++ .../src/app/app-routing.module.ts | 4 +- .../dnet-is-application/src/app/app.module.ts | 4 +- .../cleaner-tester.component.css | 0 .../cleaner-tester.component.html | 25 +++++++++++ .../cleaner-tester.component.spec.ts | 23 ++++++++++ .../cleaner-tester.component.ts | 32 ++++++++++++++ .../src/app/common/is.service.ts | 13 ++++++ .../main-menu-panels.component.html | 11 ++--- .../dnetlib/data/mapping/cleaner/Cleaner.java | 2 +- 10 files changed, 149 insertions(+), 8 deletions(-) create mode 100644 apps/dnet-is-application/src/main/java/eu/dnetlib/data/mapping/MappingAjaxController.java create mode 100644 frontends/dnet-is-application/src/app/cleaner-tester/cleaner-tester.component.css create mode 100644 frontends/dnet-is-application/src/app/cleaner-tester/cleaner-tester.component.html create mode 100644 frontends/dnet-is-application/src/app/cleaner-tester/cleaner-tester.component.spec.ts create mode 100644 frontends/dnet-is-application/src/app/cleaner-tester/cleaner-tester.component.ts diff --git a/apps/dnet-is-application/src/main/java/eu/dnetlib/data/mapping/MappingAjaxController.java b/apps/dnet-is-application/src/main/java/eu/dnetlib/data/mapping/MappingAjaxController.java new file mode 100644 index 00000000..ba139131 --- /dev/null +++ b/apps/dnet-is-application/src/main/java/eu/dnetlib/data/mapping/MappingAjaxController.java @@ -0,0 +1,43 @@ +package eu.dnetlib.data.mapping; + +import java.nio.charset.StandardCharsets; + +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +import org.apache.commons.io.IOUtils; +import org.apache.commons.lang3.StringUtils; +import org.apache.http.HttpStatus; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.http.MediaType; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.RestController; + +import eu.dnetlib.data.mapping.cleaner.CleanerFactory; + +@RestController +@RequestMapping("/ajax/mapping") +public class MappingAjaxController { + + @Autowired + private CleanerFactory cleanerFactory; + + @PostMapping(value = "/clean", consumes = { + MediaType.TEXT_PLAIN_VALUE, MediaType.APPLICATION_XML_VALUE, + }, produces = MediaType.APPLICATION_XML_VALUE) + public void importResource(@RequestParam final String rule, final HttpServletRequest req, final HttpServletResponse res) throws Exception { + final String xmlIn = IOUtils.toString(req.getInputStream(), StandardCharsets.UTF_8); + final String xmlOut = cleanerFactory.obtainCleaningRule(rule).transform(xmlIn); + + res.setCharacterEncoding(StandardCharsets.UTF_8.name()); + res.setContentType(MediaType.APPLICATION_XML_VALUE); + + if (StringUtils.isNotBlank(xmlOut)) { + IOUtils.write(xmlOut, res.getOutputStream(), StandardCharsets.UTF_8.name()); + } else { + res.sendError(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Invalid record"); + } + } +} diff --git a/frontends/dnet-is-application/src/app/app-routing.module.ts b/frontends/dnet-is-application/src/app/app-routing.module.ts index 0c3a5aa2..ae8898c5 100644 --- a/frontends/dnet-is-application/src/app/app-routing.module.ts +++ b/frontends/dnet-is-application/src/app/app-routing.module.ts @@ -8,6 +8,7 @@ import { VocabulariesComponent, VocabularyEditorComponent } from './vocabularies import { ContextViewerComponent, ContextsComponent } from './contexts/contexts.component'; import { DsmSearchComponent, DsmResultsComponent, DsmApiComponent } from './dsm/dsm.component'; import { MdstoreInspectorComponent, MdstoresComponent } from './mdstores/mdstores.component'; +import { CleanerTesterComponent } from './cleaner-tester/cleaner-tester.component'; const routes: Routes = [ { path: "", redirectTo: 'info', pathMatch: 'full' }, @@ -22,7 +23,8 @@ const routes: Routes = [ { path: "dsm/search", component: DsmSearchComponent }, { path: "dsm/results/:page/:size", component: DsmResultsComponent }, { path: "mdstores", component: MdstoresComponent }, - { path: "mdrecords/:versionId/:limit", component: MdstoreInspectorComponent } + { path: "mdrecords/:versionId/:limit", component: MdstoreInspectorComponent }, + { path: "cleaner", component: CleanerTesterComponent } ]; @NgModule({ diff --git a/frontends/dnet-is-application/src/app/app.module.ts b/frontends/dnet-is-application/src/app/app.module.ts index 5b95a356..5ae2c16d 100644 --- a/frontends/dnet-is-application/src/app/app.module.ts +++ b/frontends/dnet-is-application/src/app/app.module.ts @@ -35,6 +35,7 @@ import { MatPaginatorModule } from '@angular/material/paginator'; import { MatProgressSpinnerModule } from '@angular/material/progress-spinner'; import { SpinnerHttpInterceptor } from './common/spinner.service'; import { MdstoresComponent, MdstoreInspectorComponent, MDStoreVersionsDialog, AddMDStoreDialog } from './mdstores/mdstores.component'; +import { CleanerTesterComponent } from './cleaner-tester/cleaner-tester.component'; @NgModule({ declarations: [ @@ -64,7 +65,8 @@ import { MdstoresComponent, MdstoreInspectorComponent, MDStoreVersionsDialog, Ad MdstoresComponent, MdstoreInspectorComponent, MDStoreVersionsDialog, - AddMDStoreDialog + AddMDStoreDialog, + CleanerTesterComponent ], imports: [ BrowserModule, diff --git a/frontends/dnet-is-application/src/app/cleaner-tester/cleaner-tester.component.css b/frontends/dnet-is-application/src/app/cleaner-tester/cleaner-tester.component.css new file mode 100644 index 00000000..e69de29b diff --git a/frontends/dnet-is-application/src/app/cleaner-tester/cleaner-tester.component.html b/frontends/dnet-is-application/src/app/cleaner-tester/cleaner-tester.component.html new file mode 100644 index 00000000..69ccdd36 --- /dev/null +++ b/frontends/dnet-is-application/src/app/cleaner-tester/cleaner-tester.component.html @@ -0,0 +1,25 @@ +

Cleaner Tester

+ +
+ + Cleaning rule + + {{i.name}} + + + + + Input Record + + + + Output Record + + + + + + {{ cleanForm.errors?.['serverError'] }} + + +
diff --git a/frontends/dnet-is-application/src/app/cleaner-tester/cleaner-tester.component.spec.ts b/frontends/dnet-is-application/src/app/cleaner-tester/cleaner-tester.component.spec.ts new file mode 100644 index 00000000..f3af4ee5 --- /dev/null +++ b/frontends/dnet-is-application/src/app/cleaner-tester/cleaner-tester.component.spec.ts @@ -0,0 +1,23 @@ +import { ComponentFixture, TestBed } from '@angular/core/testing'; + +import { CleanerTesterComponent } from './cleaner-tester.component'; + +describe('CleanerTesterComponent', () => { + let component: CleanerTesterComponent; + let fixture: ComponentFixture; + + beforeEach(async () => { + await TestBed.configureTestingModule({ + declarations: [ CleanerTesterComponent ] + }) + .compileComponents(); + + fixture = TestBed.createComponent(CleanerTesterComponent); + component = fixture.componentInstance; + fixture.detectChanges(); + }); + + it('should create', () => { + expect(component).toBeTruthy(); + }); +}); diff --git a/frontends/dnet-is-application/src/app/cleaner-tester/cleaner-tester.component.ts b/frontends/dnet-is-application/src/app/cleaner-tester/cleaner-tester.component.ts new file mode 100644 index 00000000..4d6d2f0d --- /dev/null +++ b/frontends/dnet-is-application/src/app/cleaner-tester/cleaner-tester.component.ts @@ -0,0 +1,32 @@ +import { Component, OnInit } from '@angular/core'; +import { FormGroup, FormControl, Validators } from '@angular/forms'; +import { MatDialog } from '@angular/material/dialog'; +import { ActivatedRoute } from '@angular/router'; +import { ResourceType, SimpleResource } from '../common/is.model'; +import { ISService } from '../common/is.service'; + +@Component({ + selector: 'app-cleaner-tester', + templateUrl: './cleaner-tester.component.html', + styleUrls: ['./cleaner-tester.component.css'] +}) +export class CleanerTesterComponent { + rules: SimpleResource[] = []; + + xmlOutput: string = '' + + cleanForm = new FormGroup({ + xmlIn: new FormControl('', [Validators.required]), + rule: new FormControl('', [Validators.required]), + xmlOut: new FormControl('') + }); + + constructor(public service: ISService, public route: ActivatedRoute, public dialog: MatDialog) { + this.service.loadSimpleResources("cleaning_rule", (data: SimpleResource[]) => this.rules = data); + } + + clean() { + this.service.testCleaning(this.cleanForm.get("rule")?.value!, this.cleanForm.get("xmlIn")?.value!, (data: string) => this.cleanForm.get("xmlOut")?.setValue(data), this.cleanForm); + } + +} diff --git a/frontends/dnet-is-application/src/app/common/is.service.ts b/frontends/dnet-is-application/src/app/common/is.service.ts index 7fba48a7..ecef739e 100644 --- a/frontends/dnet-is-application/src/app/common/is.service.ts +++ b/frontends/dnet-is-application/src/app/common/is.service.ts @@ -319,7 +319,20 @@ export class ISService { }); } + testCleaning(rule: string, xml: string, onSuccess: Function, relatedForm?: FormGroup): void { + var headers = new HttpHeaders().set('Content-Type', 'text/plain; charset=utf-8'); + + this.client.post('./ajax/mapping/clean?rule=' + encodeURIComponent(rule), xml, { headers, responseType: 'text' as 'json' }).subscribe({ + next: data => onSuccess(data), + error: error => this.showError(error, relatedForm) + }); + } + + private showError(error: any, form?: FormGroup) { + + console.log(error); + const msg = this.errorMessage(error); if (form) { form.setErrors({ serverError: msg }) diff --git a/frontends/dnet-is-application/src/app/main-menu-panels/main-menu-panels.component.html b/frontends/dnet-is-application/src/app/main-menu-panels/main-menu-panels.component.html index 6d70e2f4..416d80dd 100644 --- a/frontends/dnet-is-application/src/app/main-menu-panels/main-menu-panels.component.html +++ b/frontends/dnet-is-application/src/app/main-menu-panels/main-menu-panels.component.html @@ -8,14 +8,14 @@ Home - Home + Home Datasources
- Search + Search
@@ -56,7 +56,8 @@ Tools
- MDStore Inspector + MDStore Inspector + Cleaner Tester
@@ -65,7 +66,7 @@ Logs
- Workflow History + Workflow History
@@ -74,7 +75,7 @@ Info
- Container Info + Container Info
diff --git a/libs/dnet-data-services/src/main/java/eu/dnetlib/data/mapping/cleaner/Cleaner.java b/libs/dnet-data-services/src/main/java/eu/dnetlib/data/mapping/cleaner/Cleaner.java index 55bf9494..6946113b 100644 --- a/libs/dnet-data-services/src/main/java/eu/dnetlib/data/mapping/cleaner/Cleaner.java +++ b/libs/dnet-data-services/src/main/java/eu/dnetlib/data/mapping/cleaner/Cleaner.java @@ -39,7 +39,7 @@ public class Cleaner extends RecordTransformer { } catch (final Exception e) { log.error("Error evaluating rule", e); } - return ""; + return null; } private void markAsInvalid(final Document doc, final List> errors) {