From 878171b2046f49b902207212481a231e1ef5b45d Mon Sep 17 00:00:00 2001 From: "michele.artini" Date: Thu, 1 Jul 2021 10:51:55 +0200 Subject: [PATCH] csv export of duplicates and suggested orgs --- apps/dnet-orgs-database-application/pom.xml | 10 +++- .../controller/OrganizationController.java | 20 ++++++++ .../organizations/utils/CSVConverter.java | 49 +++++++++++++++++++ .../html/pages/advanced/duplicates.html | 34 +++++++------ .../html/pages/advanced/pendingOrgs.html | 39 +++++++++------ .../static/resources/js/organizations.js | 6 +-- 6 files changed, 124 insertions(+), 34 deletions(-) create mode 100644 apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/utils/CSVConverter.java diff --git a/apps/dnet-orgs-database-application/pom.xml b/apps/dnet-orgs-database-application/pom.xml index cf91bd80..4a9026f4 100644 --- a/apps/dnet-orgs-database-application/pom.xml +++ b/apps/dnet-orgs-database-application/pom.xml @@ -1,5 +1,7 @@ - + eu.dnetlib.dhp @@ -58,6 +60,12 @@ jaxb-runtime + + + com.opencsv + opencsv + 5.4 + diff --git a/apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/controller/OrganizationController.java b/apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/controller/OrganizationController.java index 671c99e5..ceb3230a 100644 --- a/apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/controller/OrganizationController.java +++ b/apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/controller/OrganizationController.java @@ -1,5 +1,6 @@ package eu.dnetlib.organizations.controller; +import java.io.IOException; import java.time.OffsetDateTime; import java.util.ArrayList; import java.util.Arrays; @@ -12,6 +13,8 @@ import java.util.TreeMap; import java.util.TreeSet; import java.util.stream.Stream; +import javax.servlet.http.HttpServletResponse; + import org.apache.commons.lang3.StringUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.data.domain.Page; @@ -47,6 +50,7 @@ import eu.dnetlib.organizations.repository.readonly.OrganizationInfoViewReposito import eu.dnetlib.organizations.repository.readonly.OrganizationSimpleViewRepository; import eu.dnetlib.organizations.repository.readonly.OrganizationViewRepository; import eu.dnetlib.organizations.repository.readonly.SuggestionInfoViewByCountryRepository; +import eu.dnetlib.organizations.utils.CSVConverter; import eu.dnetlib.organizations.utils.DatabaseUtils; import eu.dnetlib.organizations.utils.OrganizationStatus; @@ -184,7 +188,13 @@ public class OrganizationController extends AbstractDnetController { } else { throw new RuntimeException("User not authorized"); } + } + @GetMapping(value = "/duplicates/byCountry/{country}/csv", produces = "text/csv") + public void findDuplicatesByCountryCSV(@PathVariable final String country, final HttpServletResponse res, final Authentication authentication) + throws IOException { + final Iterable list = findDuplicatesByCountry(country, authentication); + CSVConverter.writeCSV(res.getOutputStream(), list, DuplicateGroupView.class, "id", "name", "city", "country", "numberOfDuplicates"); } private Collection> groupConflicts(final Stream stream) { @@ -279,6 +289,16 @@ public class OrganizationController extends AbstractDnetController { } } + @GetMapping(value = "/byCountry/{status}/{code}/csv", produces = "text/csv") + public void findOrgsByStatusAndCountryCSV(@PathVariable final String status, + @PathVariable final String code, + final HttpServletResponse res, + final Authentication authentication) throws IOException { + final Iterable list = findOrgsByStatusAndCountry(status, code, authentication); + CSVConverter.writeCSV(res + .getOutputStream(), list, OrganizationSimpleView.class, "id", "name", "type", "city", "country", "acronyms", "urls", "status", "nSimilarDups", "nSuggestedDups", "nDifferentDups"); + } + @GetMapping("/byType/{status}/{type}/{page}/{size}") public Page findByType(@PathVariable final String status, @PathVariable final String type, diff --git a/apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/utils/CSVConverter.java b/apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/utils/CSVConverter.java new file mode 100644 index 00000000..7849b606 --- /dev/null +++ b/apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/utils/CSVConverter.java @@ -0,0 +1,49 @@ +package eu.dnetlib.organizations.utils; + +import java.io.IOException; +import java.io.OutputStream; +import java.io.StringWriter; + +import org.apache.commons.io.IOUtils; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; + +import com.opencsv.CSVWriter; +import com.opencsv.bean.ColumnPositionMappingStrategy; +import com.opencsv.bean.StatefulBeanToCsv; +import com.opencsv.bean.StatefulBeanToCsvBuilder; +import com.opencsv.exceptions.CsvException; + +public class CSVConverter { + + private static final Log log = LogFactory.getLog(CSVConverter.class); + + public static void writeCSV(final OutputStream out, final Iterable list, final Class clazz, final String... columns) { + final StringWriter sw = new StringWriter(); + + sw.write(StringUtils.join(columns, ",")); + sw.write(CSVWriter.DEFAULT_LINE_END); + + try { + final ColumnPositionMappingStrategy mapStrategy = new ColumnPositionMappingStrategy<>(); + mapStrategy.setType(clazz); + mapStrategy.setColumnMapping(columns); + + final StatefulBeanToCsv btcsv = new StatefulBeanToCsvBuilder(sw) + .withQuotechar(CSVWriter.DEFAULT_QUOTE_CHARACTER) + .withEscapechar(CSVWriter.DEFAULT_ESCAPE_CHARACTER) + .withMappingStrategy(mapStrategy) + .withSeparator(',') + .build(); + + btcsv.write(list.iterator()); + + IOUtils.write(sw.toString(), out); + + } catch (final CsvException | IOException ex) { + log.error("Error mapping Bean to CSV", ex); + } + } + +} diff --git a/apps/dnet-orgs-database-application/src/main/resources/static/resources/html/pages/advanced/duplicates.html b/apps/dnet-orgs-database-application/src/main/resources/static/resources/html/pages/advanced/duplicates.html index 0b16e19c..008190b7 100644 --- a/apps/dnet-orgs-database-application/src/main/resources/static/resources/html/pages/advanced/duplicates.html +++ b/apps/dnet-orgs-database-application/src/main/resources/static/resources/html/pages/advanced/duplicates.html @@ -1,20 +1,26 @@

Duplicates

-
- -
- Current country: - -
- - - {{c.desc}} ({{c.code}}) {{c.nDuplicates}} - - +
+
+
+
+ Current country: + +
+ + + {{c.desc}} ({{c.code}}) {{c.nDuplicates}} + + +
+
-
+
+
+ download as CSV +
No duplicates
diff --git a/apps/dnet-orgs-database-application/src/main/resources/static/resources/html/pages/advanced/pendingOrgs.html b/apps/dnet-orgs-database-application/src/main/resources/static/resources/html/pages/advanced/pendingOrgs.html index 41be4376..dce2dfac 100644 --- a/apps/dnet-orgs-database-application/src/main/resources/static/resources/html/pages/advanced/pendingOrgs.html +++ b/apps/dnet-orgs-database-application/src/main/resources/static/resources/html/pages/advanced/pendingOrgs.html @@ -1,22 +1,29 @@ -

Pending Organizations

+

Suggested Organizations

-
-
- Current country: - -
- - - {{c.desc}} ({{c.code}}) {{c.nPendingOrgs}} - - -
-
+
+
+
+
+ Current country: + +
+ + + {{c.desc}} ({{c.code}}) {{c.nPendingOrgs}} + + +
+
+
+
+
+ download as CSV +
-
No pending organizations
+
No suggested organizations

diff --git a/apps/dnet-orgs-database-application/src/main/resources/static/resources/js/organizations.js b/apps/dnet-orgs-database-application/src/main/resources/static/resources/js/organizations.js index 6073f556..612fbe86 100644 --- a/apps/dnet-orgs-database-application/src/main/resources/static/resources/js/organizations.js +++ b/apps/dnet-orgs-database-application/src/main/resources/static/resources/js/organizations.js @@ -327,9 +327,9 @@ orgsModule.directive('orgConflicts', function($http, $window, $location, $route, angular.forEach(scope.conflicts, function(o, pos) { ids.push(o.id); }); if (merge) { - call_http_post($http, "/api/organizations/conflicts/fix/similar", ids, function(res) { $window.location.assign('#!/edit/0/' + res.data[0]); }); + call_http_post($http, "api/organizations/conflicts/fix/similar", ids, function(res) { $window.location.assign('#!/edit/0/' + res.data[0]); }); } else { - call_http_post($http, "/api/organizations/conflicts/fix/different", ids, function(res) { $route.reload(); }); + call_http_post($http, "api/organizations/conflicts/fix/different", ids, function(res) { $route.reload(); }); } } else { alert('Invalid group !!!'); @@ -740,7 +740,7 @@ orgsModule.controller('conflictsCtrl', function ($scope, $http, $routeParams, $l var ids = []; angular.forEach(group, function(o, pos) { ids.push(o.id); }); - var url = "/api/organizations/conflicts/fix/"; + var url = "api/organizations/conflicts/fix/"; if (merge) { url += "similar"; } else { url += "different"; }