csv export of duplicates and suggested orgs

This commit is contained in:
Michele Artini 2021-07-01 10:51:55 +02:00
parent 3450f2764e
commit 878171b204
6 changed files with 124 additions and 34 deletions

View File

@ -1,5 +1,7 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<groupId>eu.dnetlib.dhp</groupId>
@ -58,6 +60,12 @@
<artifactId>jaxb-runtime</artifactId>
</dependency>
<!-- CSV -->
<dependency>
<groupId>com.opencsv</groupId>
<artifactId>opencsv</artifactId>
<version>5.4</version>
</dependency>
<!-- hot swapping, disable cache for template, enable live reload -->
<dependency>

View File

@ -1,5 +1,6 @@
package eu.dnetlib.organizations.controller;
import java.io.IOException;
import java.time.OffsetDateTime;
import java.util.ArrayList;
import java.util.Arrays;
@ -12,6 +13,8 @@ import java.util.TreeMap;
import java.util.TreeSet;
import java.util.stream.Stream;
import javax.servlet.http.HttpServletResponse;
import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.domain.Page;
@ -47,6 +50,7 @@ import eu.dnetlib.organizations.repository.readonly.OrganizationInfoViewReposito
import eu.dnetlib.organizations.repository.readonly.OrganizationSimpleViewRepository;
import eu.dnetlib.organizations.repository.readonly.OrganizationViewRepository;
import eu.dnetlib.organizations.repository.readonly.SuggestionInfoViewByCountryRepository;
import eu.dnetlib.organizations.utils.CSVConverter;
import eu.dnetlib.organizations.utils.DatabaseUtils;
import eu.dnetlib.organizations.utils.OrganizationStatus;
@ -184,7 +188,13 @@ public class OrganizationController extends AbstractDnetController {
} else {
throw new RuntimeException("User not authorized");
}
}
@GetMapping(value = "/duplicates/byCountry/{country}/csv", produces = "text/csv")
public void findDuplicatesByCountryCSV(@PathVariable final String country, final HttpServletResponse res, final Authentication authentication)
throws IOException {
final Iterable<DuplicateGroupView> list = findDuplicatesByCountry(country, authentication);
CSVConverter.writeCSV(res.getOutputStream(), list, DuplicateGroupView.class, "id", "name", "city", "country", "numberOfDuplicates");
}
private Collection<Set<OrganizationConflict>> groupConflicts(final Stream<ConflictGroupView> stream) {
@ -279,6 +289,16 @@ public class OrganizationController extends AbstractDnetController {
}
}
@GetMapping(value = "/byCountry/{status}/{code}/csv", produces = "text/csv")
public void findOrgsByStatusAndCountryCSV(@PathVariable final String status,
@PathVariable final String code,
final HttpServletResponse res,
final Authentication authentication) throws IOException {
final Iterable<OrganizationSimpleView> list = findOrgsByStatusAndCountry(status, code, authentication);
CSVConverter.writeCSV(res
.getOutputStream(), list, OrganizationSimpleView.class, "id", "name", "type", "city", "country", "acronyms", "urls", "status", "nSimilarDups", "nSuggestedDups", "nDifferentDups");
}
@GetMapping("/byType/{status}/{type}/{page}/{size}")
public Page<OrganizationSimpleView> findByType(@PathVariable final String status,
@PathVariable final String type,

View File

@ -0,0 +1,49 @@
package eu.dnetlib.organizations.utils;
import java.io.IOException;
import java.io.OutputStream;
import java.io.StringWriter;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import com.opencsv.CSVWriter;
import com.opencsv.bean.ColumnPositionMappingStrategy;
import com.opencsv.bean.StatefulBeanToCsv;
import com.opencsv.bean.StatefulBeanToCsvBuilder;
import com.opencsv.exceptions.CsvException;
public class CSVConverter {
private static final Log log = LogFactory.getLog(CSVConverter.class);
public static <T> void writeCSV(final OutputStream out, final Iterable<T> list, final Class<T> clazz, final String... columns) {
final StringWriter sw = new StringWriter();
sw.write(StringUtils.join(columns, ","));
sw.write(CSVWriter.DEFAULT_LINE_END);
try {
final ColumnPositionMappingStrategy<T> mapStrategy = new ColumnPositionMappingStrategy<>();
mapStrategy.setType(clazz);
mapStrategy.setColumnMapping(columns);
final StatefulBeanToCsv<T> btcsv = new StatefulBeanToCsvBuilder<T>(sw)
.withQuotechar(CSVWriter.DEFAULT_QUOTE_CHARACTER)
.withEscapechar(CSVWriter.DEFAULT_ESCAPE_CHARACTER)
.withMappingStrategy(mapStrategy)
.withSeparator(',')
.build();
btcsv.write(list.iterator());
IOUtils.write(sw.toString(), out);
} catch (final CsvException | IOException ex) {
log.error("Error mapping Bean to CSV", ex);
}
}
}

View File

@ -1,7 +1,8 @@
<h2>Duplicates</h2>
<div class="input-group input-group-sm mt-3 mb-3">
<div class="row mt-3 mb-3">
<div class="col">
<div class="input-group input-group-sm">
<div class="input-group-append">
<span class="input-group-text text-outline-primary">Current country:</span>
<button class="btn btn-outline-primary dropdown-toggle" data-toggle="dropdown">{{country}}</button>
@ -16,6 +17,11 @@
</div>
</div>
</div>
</div>
<div class="col text-right" ng-show="duplicates.length > 0">
<a class="btn btn-sm btn-outline-primary" target="_blank" href="/api/organizations/duplicates/byCountry/{{country}}/csv" download="duplicates.{{country}}.csv">download as CSV</a>
</div>
</div>
<h5 class="text-muted" ng-if="!loading && duplicates.length == 0">No duplicates</h5>

View File

@ -1,6 +1,8 @@
<h2>Pending Organizations</h2>
<h2>Suggested Organizations</h2>
<div class="input-group input-group-sm mt-3 mb-3">
<div class="row mt-3 mb-3">
<div class="col">
<div class="input-group input-group-sm">
<div class="input-group-append">
<span class="input-group-text text-outline-primary">Current country:</span>
<button class="btn btn-outline-primary dropdown-toggle" data-toggle="dropdown">{{country}}</button>
@ -15,8 +17,13 @@
</div>
</div>
</div>
</div>
<div class="col text-right" ng-show="orgs.length > 0">
<a class="btn btn-sm btn-outline-primary" target="_blank" href="api/organizations/byCountry/suggested/{{country}}/csv" download="suggested_orgs.{{country}}.csv">download as CSV</a>
</div>
</div>
<h5 class="text-muted" ng-if="!loading && orgs.length == 0">No pending organizations</h5>
<h5 class="text-muted" ng-if="!loading && orgs.length == 0">No suggested organizations</h5>
<p ng-show="orgs.length > 0">
<input type="text" class="form-control form-control-sm" ng-model="orgFilter" placeholder="Filter..."/>

View File

@ -327,9 +327,9 @@ orgsModule.directive('orgConflicts', function($http, $window, $location, $route,
angular.forEach(scope.conflicts, function(o, pos) { ids.push(o.id); });
if (merge) {
call_http_post($http, "/api/organizations/conflicts/fix/similar", ids, function(res) { $window.location.assign('#!/edit/0/' + res.data[0]); });
call_http_post($http, "api/organizations/conflicts/fix/similar", ids, function(res) { $window.location.assign('#!/edit/0/' + res.data[0]); });
} else {
call_http_post($http, "/api/organizations/conflicts/fix/different", ids, function(res) { $route.reload(); });
call_http_post($http, "api/organizations/conflicts/fix/different", ids, function(res) { $route.reload(); });
}
} else {
alert('Invalid group !!!');
@ -740,7 +740,7 @@ orgsModule.controller('conflictsCtrl', function ($scope, $http, $routeParams, $l
var ids = [];
angular.forEach(group, function(o, pos) { ids.push(o.id); });
var url = "/api/organizations/conflicts/fix/";
var url = "api/organizations/conflicts/fix/";
if (merge) { url += "similar"; }
else { url += "different"; }