csv export of duplicates and suggested orgs
This commit is contained in:
parent
3450f2764e
commit
878171b204
|
@ -1,5 +1,7 @@
|
||||||
<?xml version="1.0" encoding="UTF-8"?>
|
<?xml version="1.0" encoding="UTF-8"?>
|
||||||
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
<project xmlns="http://maven.apache.org/POM/4.0.0"
|
||||||
|
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||||
|
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||||
|
|
||||||
<parent>
|
<parent>
|
||||||
<groupId>eu.dnetlib.dhp</groupId>
|
<groupId>eu.dnetlib.dhp</groupId>
|
||||||
|
@ -58,6 +60,12 @@
|
||||||
<artifactId>jaxb-runtime</artifactId>
|
<artifactId>jaxb-runtime</artifactId>
|
||||||
</dependency>
|
</dependency>
|
||||||
|
|
||||||
|
<!-- CSV -->
|
||||||
|
<dependency>
|
||||||
|
<groupId>com.opencsv</groupId>
|
||||||
|
<artifactId>opencsv</artifactId>
|
||||||
|
<version>5.4</version>
|
||||||
|
</dependency>
|
||||||
|
|
||||||
<!-- hot swapping, disable cache for template, enable live reload -->
|
<!-- hot swapping, disable cache for template, enable live reload -->
|
||||||
<dependency>
|
<dependency>
|
||||||
|
|
|
@ -1,5 +1,6 @@
|
||||||
package eu.dnetlib.organizations.controller;
|
package eu.dnetlib.organizations.controller;
|
||||||
|
|
||||||
|
import java.io.IOException;
|
||||||
import java.time.OffsetDateTime;
|
import java.time.OffsetDateTime;
|
||||||
import java.util.ArrayList;
|
import java.util.ArrayList;
|
||||||
import java.util.Arrays;
|
import java.util.Arrays;
|
||||||
|
@ -12,6 +13,8 @@ import java.util.TreeMap;
|
||||||
import java.util.TreeSet;
|
import java.util.TreeSet;
|
||||||
import java.util.stream.Stream;
|
import java.util.stream.Stream;
|
||||||
|
|
||||||
|
import javax.servlet.http.HttpServletResponse;
|
||||||
|
|
||||||
import org.apache.commons.lang3.StringUtils;
|
import org.apache.commons.lang3.StringUtils;
|
||||||
import org.springframework.beans.factory.annotation.Autowired;
|
import org.springframework.beans.factory.annotation.Autowired;
|
||||||
import org.springframework.data.domain.Page;
|
import org.springframework.data.domain.Page;
|
||||||
|
@ -47,6 +50,7 @@ import eu.dnetlib.organizations.repository.readonly.OrganizationInfoViewReposito
|
||||||
import eu.dnetlib.organizations.repository.readonly.OrganizationSimpleViewRepository;
|
import eu.dnetlib.organizations.repository.readonly.OrganizationSimpleViewRepository;
|
||||||
import eu.dnetlib.organizations.repository.readonly.OrganizationViewRepository;
|
import eu.dnetlib.organizations.repository.readonly.OrganizationViewRepository;
|
||||||
import eu.dnetlib.organizations.repository.readonly.SuggestionInfoViewByCountryRepository;
|
import eu.dnetlib.organizations.repository.readonly.SuggestionInfoViewByCountryRepository;
|
||||||
|
import eu.dnetlib.organizations.utils.CSVConverter;
|
||||||
import eu.dnetlib.organizations.utils.DatabaseUtils;
|
import eu.dnetlib.organizations.utils.DatabaseUtils;
|
||||||
import eu.dnetlib.organizations.utils.OrganizationStatus;
|
import eu.dnetlib.organizations.utils.OrganizationStatus;
|
||||||
|
|
||||||
|
@ -184,7 +188,13 @@ public class OrganizationController extends AbstractDnetController {
|
||||||
} else {
|
} else {
|
||||||
throw new RuntimeException("User not authorized");
|
throw new RuntimeException("User not authorized");
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@GetMapping(value = "/duplicates/byCountry/{country}/csv", produces = "text/csv")
|
||||||
|
public void findDuplicatesByCountryCSV(@PathVariable final String country, final HttpServletResponse res, final Authentication authentication)
|
||||||
|
throws IOException {
|
||||||
|
final Iterable<DuplicateGroupView> list = findDuplicatesByCountry(country, authentication);
|
||||||
|
CSVConverter.writeCSV(res.getOutputStream(), list, DuplicateGroupView.class, "id", "name", "city", "country", "numberOfDuplicates");
|
||||||
}
|
}
|
||||||
|
|
||||||
private Collection<Set<OrganizationConflict>> groupConflicts(final Stream<ConflictGroupView> stream) {
|
private Collection<Set<OrganizationConflict>> groupConflicts(final Stream<ConflictGroupView> stream) {
|
||||||
|
@ -279,6 +289,16 @@ public class OrganizationController extends AbstractDnetController {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@GetMapping(value = "/byCountry/{status}/{code}/csv", produces = "text/csv")
|
||||||
|
public void findOrgsByStatusAndCountryCSV(@PathVariable final String status,
|
||||||
|
@PathVariable final String code,
|
||||||
|
final HttpServletResponse res,
|
||||||
|
final Authentication authentication) throws IOException {
|
||||||
|
final Iterable<OrganizationSimpleView> list = findOrgsByStatusAndCountry(status, code, authentication);
|
||||||
|
CSVConverter.writeCSV(res
|
||||||
|
.getOutputStream(), list, OrganizationSimpleView.class, "id", "name", "type", "city", "country", "acronyms", "urls", "status", "nSimilarDups", "nSuggestedDups", "nDifferentDups");
|
||||||
|
}
|
||||||
|
|
||||||
@GetMapping("/byType/{status}/{type}/{page}/{size}")
|
@GetMapping("/byType/{status}/{type}/{page}/{size}")
|
||||||
public Page<OrganizationSimpleView> findByType(@PathVariable final String status,
|
public Page<OrganizationSimpleView> findByType(@PathVariable final String status,
|
||||||
@PathVariable final String type,
|
@PathVariable final String type,
|
||||||
|
|
|
@ -0,0 +1,49 @@
|
||||||
|
package eu.dnetlib.organizations.utils;
|
||||||
|
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.io.OutputStream;
|
||||||
|
import java.io.StringWriter;
|
||||||
|
|
||||||
|
import org.apache.commons.io.IOUtils;
|
||||||
|
import org.apache.commons.lang3.StringUtils;
|
||||||
|
import org.apache.commons.logging.Log;
|
||||||
|
import org.apache.commons.logging.LogFactory;
|
||||||
|
|
||||||
|
import com.opencsv.CSVWriter;
|
||||||
|
import com.opencsv.bean.ColumnPositionMappingStrategy;
|
||||||
|
import com.opencsv.bean.StatefulBeanToCsv;
|
||||||
|
import com.opencsv.bean.StatefulBeanToCsvBuilder;
|
||||||
|
import com.opencsv.exceptions.CsvException;
|
||||||
|
|
||||||
|
public class CSVConverter {
|
||||||
|
|
||||||
|
private static final Log log = LogFactory.getLog(CSVConverter.class);
|
||||||
|
|
||||||
|
public static <T> void writeCSV(final OutputStream out, final Iterable<T> list, final Class<T> clazz, final String... columns) {
|
||||||
|
final StringWriter sw = new StringWriter();
|
||||||
|
|
||||||
|
sw.write(StringUtils.join(columns, ","));
|
||||||
|
sw.write(CSVWriter.DEFAULT_LINE_END);
|
||||||
|
|
||||||
|
try {
|
||||||
|
final ColumnPositionMappingStrategy<T> mapStrategy = new ColumnPositionMappingStrategy<>();
|
||||||
|
mapStrategy.setType(clazz);
|
||||||
|
mapStrategy.setColumnMapping(columns);
|
||||||
|
|
||||||
|
final StatefulBeanToCsv<T> btcsv = new StatefulBeanToCsvBuilder<T>(sw)
|
||||||
|
.withQuotechar(CSVWriter.DEFAULT_QUOTE_CHARACTER)
|
||||||
|
.withEscapechar(CSVWriter.DEFAULT_ESCAPE_CHARACTER)
|
||||||
|
.withMappingStrategy(mapStrategy)
|
||||||
|
.withSeparator(',')
|
||||||
|
.build();
|
||||||
|
|
||||||
|
btcsv.write(list.iterator());
|
||||||
|
|
||||||
|
IOUtils.write(sw.toString(), out);
|
||||||
|
|
||||||
|
} catch (final CsvException | IOException ex) {
|
||||||
|
log.error("Error mapping Bean to CSV", ex);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
|
@ -1,20 +1,26 @@
|
||||||
<h2>Duplicates</h2>
|
<h2>Duplicates</h2>
|
||||||
|
|
||||||
<div class="input-group input-group-sm mt-3 mb-3">
|
<div class="row mt-3 mb-3">
|
||||||
|
<div class="col">
|
||||||
<div class="input-group-append">
|
<div class="input-group input-group-sm">
|
||||||
<span class="input-group-text text-outline-primary">Current country:</span>
|
<div class="input-group-append">
|
||||||
<button class="btn btn-outline-primary dropdown-toggle" data-toggle="dropdown">{{country}}</button>
|
<span class="input-group-text text-outline-primary">Current country:</span>
|
||||||
<div class="dropdown-menu">
|
<button class="btn btn-outline-primary dropdown-toggle" data-toggle="dropdown">{{country}}</button>
|
||||||
<small>
|
<div class="dropdown-menu">
|
||||||
<a class="dropdown-item" href="#!/duplicates/{{c.code}}"
|
<small>
|
||||||
ng-repeat="c in info.data.byCountry"
|
<a class="dropdown-item" href="#!/duplicates/{{c.code}}"
|
||||||
ng-if="c.nDuplicates > 0">
|
ng-repeat="c in info.data.byCountry"
|
||||||
{{c.desc}} ({{c.code}}) <span class="badge badge-primary float-right">{{c.nDuplicates}}</span>
|
ng-if="c.nDuplicates > 0">
|
||||||
</a>
|
{{c.desc}} ({{c.code}}) <span class="badge badge-primary float-right">{{c.nDuplicates}}</span>
|
||||||
</small>
|
</a>
|
||||||
|
</small>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
<div class="col text-right" ng-show="duplicates.length > 0">
|
||||||
|
<a class="btn btn-sm btn-outline-primary" target="_blank" href="/api/organizations/duplicates/byCountry/{{country}}/csv" download="duplicates.{{country}}.csv">download as CSV</a>
|
||||||
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
<h5 class="text-muted" ng-if="!loading && duplicates.length == 0">No duplicates</h5>
|
<h5 class="text-muted" ng-if="!loading && duplicates.length == 0">No duplicates</h5>
|
||||||
|
|
|
@ -1,22 +1,29 @@
|
||||||
<h2>Pending Organizations</h2>
|
<h2>Suggested Organizations</h2>
|
||||||
|
|
||||||
<div class="input-group input-group-sm mt-3 mb-3">
|
<div class="row mt-3 mb-3">
|
||||||
<div class="input-group-append">
|
<div class="col">
|
||||||
<span class="input-group-text text-outline-primary">Current country:</span>
|
<div class="input-group input-group-sm">
|
||||||
<button class="btn btn-outline-primary dropdown-toggle" data-toggle="dropdown">{{country}}</button>
|
<div class="input-group-append">
|
||||||
<div class="dropdown-menu">
|
<span class="input-group-text text-outline-primary">Current country:</span>
|
||||||
<small>
|
<button class="btn btn-outline-primary dropdown-toggle" data-toggle="dropdown">{{country}}</button>
|
||||||
<a class="dropdown-item" href="#!/pendings/{{c.code}}"
|
<div class="dropdown-menu">
|
||||||
ng-repeat="c in info.data.byCountry"
|
<small>
|
||||||
ng-if="c.nPendingOrgs > 0">
|
<a class="dropdown-item" href="#!/pendings/{{c.code}}"
|
||||||
{{c.desc}} ({{c.code}}) <span class="badge badge-primary float-right">{{c.nPendingOrgs}}</span>
|
ng-repeat="c in info.data.byCountry"
|
||||||
</a>
|
ng-if="c.nPendingOrgs > 0">
|
||||||
</small>
|
{{c.desc}} ({{c.code}}) <span class="badge badge-primary float-right">{{c.nPendingOrgs}}</span>
|
||||||
</div>
|
</a>
|
||||||
</div>
|
</small>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="col text-right" ng-show="orgs.length > 0">
|
||||||
|
<a class="btn btn-sm btn-outline-primary" target="_blank" href="api/organizations/byCountry/suggested/{{country}}/csv" download="suggested_orgs.{{country}}.csv">download as CSV</a>
|
||||||
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
<h5 class="text-muted" ng-if="!loading && orgs.length == 0">No pending organizations</h5>
|
<h5 class="text-muted" ng-if="!loading && orgs.length == 0">No suggested organizations</h5>
|
||||||
|
|
||||||
<p ng-show="orgs.length > 0">
|
<p ng-show="orgs.length > 0">
|
||||||
<input type="text" class="form-control form-control-sm" ng-model="orgFilter" placeholder="Filter..."/>
|
<input type="text" class="form-control form-control-sm" ng-model="orgFilter" placeholder="Filter..."/>
|
||||||
|
|
|
@ -327,9 +327,9 @@ orgsModule.directive('orgConflicts', function($http, $window, $location, $route,
|
||||||
angular.forEach(scope.conflicts, function(o, pos) { ids.push(o.id); });
|
angular.forEach(scope.conflicts, function(o, pos) { ids.push(o.id); });
|
||||||
|
|
||||||
if (merge) {
|
if (merge) {
|
||||||
call_http_post($http, "/api/organizations/conflicts/fix/similar", ids, function(res) { $window.location.assign('#!/edit/0/' + res.data[0]); });
|
call_http_post($http, "api/organizations/conflicts/fix/similar", ids, function(res) { $window.location.assign('#!/edit/0/' + res.data[0]); });
|
||||||
} else {
|
} else {
|
||||||
call_http_post($http, "/api/organizations/conflicts/fix/different", ids, function(res) { $route.reload(); });
|
call_http_post($http, "api/organizations/conflicts/fix/different", ids, function(res) { $route.reload(); });
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
alert('Invalid group !!!');
|
alert('Invalid group !!!');
|
||||||
|
@ -740,7 +740,7 @@ orgsModule.controller('conflictsCtrl', function ($scope, $http, $routeParams, $l
|
||||||
var ids = [];
|
var ids = [];
|
||||||
angular.forEach(group, function(o, pos) { ids.push(o.id); });
|
angular.forEach(group, function(o, pos) { ids.push(o.id); });
|
||||||
|
|
||||||
var url = "/api/organizations/conflicts/fix/";
|
var url = "api/organizations/conflicts/fix/";
|
||||||
if (merge) { url += "similar"; }
|
if (merge) { url += "similar"; }
|
||||||
else { url += "different"; }
|
else { url += "different"; }
|
||||||
|
|
||||||
|
|
Loading…
Reference in New Issue