D-Net
/
dnet-applications
23
1
Fork 0
Code Issues Pull Requests Releases Wiki Activity

[Community API extention] merging with master

This commit is contained in:
Miriam Baglioni 2022-12-19 12:45:50 +01:00
parent 50b2468a13 39d335439f
commit 36f858cd5b
55 changed files with 1017 additions and 211 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
apps/bioschemas-api/pom.xml
View File

@ -3,7 +3,7 @@
<parent>
<groupId>eu.dnetlib.dhp</groupId>
<artifactId>apps</artifactId>
<version>3.3.2-SNAPSHOT</version>
<version>3.3.3-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>

2
apps/dhp-broker-application/pom.xml
View File

@ -4,7 +4,7 @@
<parent>
<groupId>eu.dnetlib.dhp</groupId>
<artifactId>apps</artifactId>
<version>3.3.2-SNAPSHOT</version>
<version>3.3.3-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>

2
apps/dhp-broker-public-application/pom.xml
View File

@ -4,7 +4,7 @@
<parent>
<groupId>eu.dnetlib.dhp</groupId>
<artifactId>apps</artifactId>
<version>3.3.2-SNAPSHOT</version>
<version>3.3.3-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>

2
apps/dhp-mdstore-manager/pom.xml
View File

@ -4,7 +4,7 @@
<parent>
<groupId>eu.dnetlib.dhp</groupId>
<artifactId>apps</artifactId>
<version>3.3.2-SNAPSHOT</version>
<version>3.3.3-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>

2
apps/dnet-exporter-api/pom.xml
View File

@ -3,7 +3,7 @@
<parent>
<groupId>eu.dnetlib.dhp</groupId>
<artifactId>apps</artifactId>
<version>3.3.2-SNAPSHOT</version>
<version>3.3.3-SNAPSHOT</version>
<relativePath>../</relativePath>
</parent>

44
apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/common/AbstractExporterController.java
View File

@ -3,12 +3,9 @@ package eu.dnetlib.openaire.common;
import java.util.List;
import java.util.stream.Collectors;
import com.fasterxml.jackson.annotation.JsonAutoDetect;
import eu.dnetlib.enabling.datasources.common.DsmException;
import eu.dnetlib.enabling.datasources.common.DsmForbiddenException;
import eu.dnetlib.enabling.datasources.common.DsmNotFoundException;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.exception.ExceptionUtils;
import org.apache.commons.lang3.time.StopWatch;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.springframework.http.HttpStatus;
@ -17,6 +14,13 @@ import org.springframework.web.bind.annotation.ExceptionHandler;
import org.springframework.web.bind.annotation.ResponseBody;
import org.springframework.web.bind.annotation.ResponseStatus;
import com.fasterxml.jackson.annotation.JsonAutoDetect;
import eu.dnetlib.enabling.datasources.common.DsmException;
import eu.dnetlib.enabling.datasources.common.DsmForbiddenException;
import eu.dnetlib.enabling.datasources.common.DsmNotFoundException;
import eu.dnetlib.openaire.dsm.domain.Response;
/**
* Created by claudio on 18/07/2017.
*/
@ -25,7 +29,9 @@ public abstract class AbstractExporterController {
private static final Log log = LogFactory.getLog(AbstractExporterController.class); // NOPMD by marko on 11/24/08 5:02 PM
@ResponseBody
@ExceptionHandler({DsmException.class})
@ExceptionHandler({
DsmException.class
})
@ResponseStatus(value = HttpStatus.INTERNAL_SERVER_ERROR)
public ErrorMessage handleDSMException(final Exception e) {
return _handleError(e);
@ -39,7 +45,9 @@ public abstract class AbstractExporterController {
}
@ResponseBody
@ExceptionHandler({DsmNotFoundException.class})
@ExceptionHandler({
DsmNotFoundException.class
})
@ResponseStatus(value = HttpStatus.NOT_FOUND)
public ErrorMessage handleNotFoundException(final Exception e) {
return _handleError(e);
@ -50,23 +58,33 @@ public abstract class AbstractExporterController {
@ResponseStatus(HttpStatus.BAD_REQUEST)
public List<ErrorMessage> processValidationError(final MethodArgumentNotValidException e) {
return e.getBindingResult()
.getFieldErrors().stream()
.map(fe -> new ErrorMessage(
String.format("field '%s'", fe.getField()),
String.format("rejected value '%s'", fe.getRejectedValue()),
fe.getDefaultMessage()))
.collect(Collectors.toList());
.getFieldErrors()
.stream()
.map(fe -> new ErrorMessage(
String.format("field '%s'", fe.getField()),
String.format("rejected value '%s'", fe.getRejectedValue()),
fe.getDefaultMessage()))
.collect(Collectors.toList());
}
private ErrorMessage _handleError(final Exception e) {
log.error(e);
if (StringUtils.containsIgnoreCase(ExceptionUtils.getRootCauseMessage(e), "Broken pipe")) {
return null; //socket is closed, cannot return any response
return null; // socket is closed, cannot return any response
} else {
return new ErrorMessage(e);
}
}
// HELPERS
protected <T extends Response> T prepareResponse(final int page, final int size, final StopWatch stopWatch, final T rsp) {
rsp.getHeader()
.setTime(stopWatch.getTime())
.setPage(page)
.setSize(size);
return rsp;
}
@JsonAutoDetect
public class ErrorMessage {

81
apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/DsmApiController.java
View File

@ -16,10 +16,11 @@ import org.apache.http.HttpStatus;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.web.bind.annotation.CrossOrigin;
import org.springframework.web.bind.annotation.DeleteMapping;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestMethod;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RestController;
@ -36,10 +37,10 @@ import eu.dnetlib.openaire.dsm.domain.DatasourceDetails;
import eu.dnetlib.openaire.dsm.domain.DatasourceDetailsUpdate;
import eu.dnetlib.openaire.dsm.domain.DatasourceDetailsWithApis;
import eu.dnetlib.openaire.dsm.domain.DatasourceSnippetResponse;
import eu.dnetlib.openaire.dsm.domain.RegisteredDatasourceInfo;
import eu.dnetlib.openaire.dsm.domain.RequestFilter;
import eu.dnetlib.openaire.dsm.domain.RequestSort;
import eu.dnetlib.openaire.dsm.domain.RequestSortOrder;
import eu.dnetlib.openaire.dsm.domain.Response;
import eu.dnetlib.openaire.dsm.domain.SimpleResponse;
import eu.dnetlib.openaire.vocabularies.Country;
import io.swagger.v3.oas.annotations.Operation;
@ -58,9 +59,9 @@ public class DsmApiController extends AbstractExporterController {
@Autowired
private DsmCore dsmCore;
@RequestMapping(value = "/ds/countries", produces = {
@GetMapping(value = "/ds/countries", produces = {
"application/json"
}, method = RequestMethod.GET)
})
@Operation(summary = "list the datasource countries", description = "list the datasource countries", tags = {
DS, R
})
@ -72,9 +73,9 @@ public class DsmApiController extends AbstractExporterController {
return dsmCore.listCountries();
}
@RequestMapping(value = "/ds/searchdetails/{page}/{size}", produces = {
@PostMapping(value = "/ds/searchdetails/{page}/{size}", produces = {
"application/json"
}, method = RequestMethod.POST)
})
@Operation(summary = "search datasources", description = "Returns list of Datasource details.", tags = {
DS, R
})
@ -93,9 +94,9 @@ public class DsmApiController extends AbstractExporterController {
return prepareResponse(page, size, stop, rsp);
}
@RequestMapping(value = "/ds/aggregationhistory/{dsId}", produces = {
@GetMapping(value = "/ds/aggregationhistory/{dsId}", produces = {
"application/json"
}, method = RequestMethod.GET)
})
@Operation(summary = "search datasources", description = "Returns list of Datasource details.", tags = {
DS, R
})
@ -109,9 +110,9 @@ public class DsmApiController extends AbstractExporterController {
return prepareResponse(0, rsp.getAggregationInfo().size(), stop, rsp);
}
@RequestMapping(value = "/ds/searchsnippet/{page}/{size}", produces = {
@PostMapping(value = "/ds/searchsnippet/{page}/{size}", produces = {
"application/json"
}, method = RequestMethod.POST)
})
@Operation(summary = "search datasources", description = "Returns list of Datasource basic info.", tags = {
DS, R
})
@ -130,9 +131,9 @@ public class DsmApiController extends AbstractExporterController {
return prepareResponse(page, size, stop, rsp);
}
@RequestMapping(value = "/ds/searchregistered/{page}/{size}", produces = {
@PostMapping(value = "/ds/searchregistered/{page}/{size}", produces = {
"application/json"
}, method = RequestMethod.POST)
})
@Operation(summary = "search among registered datasources", description = "Returns list of Datasource basic info.", tags = {
DS,
R
@ -152,9 +153,9 @@ public class DsmApiController extends AbstractExporterController {
return prepareResponse(page, size, stop, rsp);
}
@RequestMapping(value = "/ds/recentregistered/{size}", produces = {
@GetMapping(value = "/ds/recentregistered/{size}", produces = {
"application/json"
}, method = RequestMethod.GET)
})
@Operation(summary = "return the latest datasources that were registered through Provide", description = "Returns list of Datasource basic info.", tags = {
DS,
R
@ -163,15 +164,15 @@ public class DsmApiController extends AbstractExporterController {
@ApiResponse(responseCode = "200", description = "OK"),
@ApiResponse(responseCode = "500", description = "unexpected error")
})
public SimpleResponse<?> recentRegistered(@PathVariable final int size) throws Throwable {
public SimpleResponse<RegisteredDatasourceInfo> recentRegistered(@PathVariable final int size) throws Throwable {
final StopWatch stop = StopWatch.createStarted();
final SimpleResponse<?> rsp = dsmCore.searchRecentRegistered(size);
final SimpleResponse<RegisteredDatasourceInfo> rsp = dsmCore.searchRecentRegistered(size);
return prepareResponse(1, size, stop, rsp);
}
@RequestMapping(value = "/ds/countregistered", produces = {
@GetMapping(value = "/ds/countregistered", produces = {
"application/json"
}, method = RequestMethod.GET)
})
@Operation(summary = "return the number of datasources registered after the given date", description = "Returns a number.", tags = {
DS,
R
@ -185,9 +186,9 @@ public class DsmApiController extends AbstractExporterController {
return dsmCore.countRegisteredAfter(fromDate, typologyFilter);
}
@RequestMapping(value = "/ds/api/{dsId}", produces = {
@GetMapping(value = "/ds/api/{dsId}", produces = {
"application/json"
}, method = RequestMethod.GET)
})
@Operation(summary = "get the list of API for a given datasource", description = "Returns the list of API for a given datasource.", tags = {
API,
R
@ -204,9 +205,9 @@ public class DsmApiController extends AbstractExporterController {
return prepareResponse(0, rsp.getApi().size(), stop, rsp);
}
@RequestMapping(value = "/api/baseurl/{page}/{size}", produces = {
@PostMapping(value = "/api/baseurl/{page}/{size}", produces = {
"application/json"
}, method = RequestMethod.POST)
})
@Operation(summary = "search for the list of base URLs of Datasource APIs managed by a user", description = "Returns the list of base URLs of Datasource APIs managed by a user", tags = {
DS, API, R
})
@ -222,7 +223,7 @@ public class DsmApiController extends AbstractExporterController {
return dsmCore.findBaseURLs(requestFilter, page, size);
}
@RequestMapping(value = "/ds/api/{apiId}", method = RequestMethod.DELETE)
@DeleteMapping(value = "/ds/api/{apiId}")
@Operation(summary = "delete an API", description = "delete an API, if removable", tags = {
API, W
})
@ -236,7 +237,7 @@ public class DsmApiController extends AbstractExporterController {
dsmCore.deleteApi(apiId);
}
@RequestMapping(value = "/ds/manage", method = RequestMethod.POST)
@PostMapping(value = "/ds/manage")
@Operation(summary = "set the managed status for a given datasource", description = "set the managed status for a given datasource", tags = {
DS, W
})
@ -251,7 +252,7 @@ public class DsmApiController extends AbstractExporterController {
dsmCore.setManaged(id, managed);
}
@RequestMapping(value = "/ds/managed/{id}", method = RequestMethod.GET)
@GetMapping(value = "/ds/managed/{id}")
@Operation(summary = "get the datasource managed status", description = "get the datasource managed status", tags = {
DS, R
})
@ -263,7 +264,7 @@ public class DsmApiController extends AbstractExporterController {
return dsmCore.isManaged(id);
}
@RequestMapping(value = "/ds/add", method = RequestMethod.POST)
@PostMapping(value = "/ds/add")
@Operation(summary = "add a new Datasource", description = "add a new Datasource", tags = {
DS, W
})
@ -280,7 +281,7 @@ public class DsmApiController extends AbstractExporterController {
dsmCore.save(datasource);
}
@RequestMapping(value = "/ds/addWithApis", method = RequestMethod.POST)
@PostMapping(value = "/ds/addWithApis")
@Operation(summary = "add a new Datasource and its apis", description = "add a new Datasource and its apis", tags = {
DS, W
})
@ -297,7 +298,7 @@ public class DsmApiController extends AbstractExporterController {
dsmCore.save(d);
}
@RequestMapping(value = "/ds/update", method = RequestMethod.POST)
@PostMapping(value = "/ds/update")
@Operation(summary = "update Datasource details", description = "update Datasource details", tags = {
DS, W
})
@ -311,7 +312,7 @@ public class DsmApiController extends AbstractExporterController {
dsmCore.updateDatasource(ds);
}
@RequestMapping(value = "/ds/api/baseurl", method = RequestMethod.POST)
@PostMapping(value = "/ds/api/baseurl")
@Operation(summary = "update the base URL of a datasource interface", description = "update the base URL of a datasource interface", tags = {
API, W
})
@ -327,7 +328,7 @@ public class DsmApiController extends AbstractExporterController {
dsmCore.updateApiBaseurl(dsId, apiId, baseUrl);
}
@RequestMapping(value = "/ds/api/compliance", method = RequestMethod.POST)
@PostMapping(value = "/ds/api/compliance")
@Operation(summary = "update the compatibility of a datasource interface", description = "update the compatibility of a datasource interface", tags = {
API, W
})
@ -344,7 +345,7 @@ public class DsmApiController extends AbstractExporterController {
dsmCore.updateApiCompatibility(dsId, apiId, compliance, override);
}
@RequestMapping(value = "/ds/api/oaiset", method = RequestMethod.POST)
@PostMapping(value = "/ds/api/oaiset")
@Operation(summary = "update the OAI set of a datasource interface", description = "update the OAI set of a datasource interface", tags = {
API, W
})
@ -360,7 +361,7 @@ public class DsmApiController extends AbstractExporterController {
dsmCore.updateApiOaiSet(dsId, apiId, oaiSet);
}
@RequestMapping(value = "/ds/api/add", method = RequestMethod.POST)
@PostMapping(value = "/ds/api/add")
@Operation(summary = "adds a new Interface to one Datasource", description = "adds an Interface to one Datasource", tags = {
API, W
})
@ -378,7 +379,7 @@ public class DsmApiController extends AbstractExporterController {
@Autowired
private OperationManager operationManager;
@RequestMapping(value = "/dsm/ops", method = RequestMethod.GET)
@GetMapping(value = "/dsm/ops")
@Operation(summary = "get the number of pending operations", description = "get the number of pending operations", tags = {
R, M
})
@ -390,7 +391,7 @@ public class DsmApiController extends AbstractExporterController {
return operationManager.getOpSize();
}
@RequestMapping(value = "/dsm/killops", method = RequestMethod.POST)
@PostMapping(value = "/dsm/killops")
@Operation(summary = "interrupts the pending operations", description = "return the number of interrupted operations", tags = {
W, M
})
@ -402,7 +403,7 @@ public class DsmApiController extends AbstractExporterController {
return operationManager.dropAll();
}
@RequestMapping(value = "/dsm/dropcache", method = RequestMethod.POST)
@PostMapping(value = "/dsm/dropcache")
@Operation(summary = "drop the caches", description = "drop the internal caches", tags = {
W, M
})
@ -414,12 +415,4 @@ public class DsmApiController extends AbstractExporterController {
dsmCore.dropCaches();
}
// HELPERS
private <T extends Response> T prepareResponse(final int page, final int size, final StopWatch stopWatch, final T rsp) {
rsp.getHeader()
.setTime(stopWatch.getTime())
.setPage(page)
.setSize(size);
return rsp;
}
}

86
apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/DsmApiControllerV2.java Normal file
View File

@ -0,0 +1,86 @@
package eu.dnetlib.openaire.dsm;
import static eu.dnetlib.openaire.common.ExporterConstants.DS;
import static eu.dnetlib.openaire.common.ExporterConstants.R;
import java.util.List;
import org.apache.commons.lang3.time.StopWatch;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.web.bind.annotation.CrossOrigin;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RestController;
import eu.dnetlib.openaire.common.AbstractExporterController;
import eu.dnetlib.openaire.dsm.dao.ResponseUtils;
import eu.dnetlib.openaire.dsm.domain.SimpleDatasourceInfo;
import eu.dnetlib.openaire.dsm.domain.SimpleResponse;
import io.swagger.v3.oas.annotations.Operation;
import io.swagger.v3.oas.annotations.responses.ApiResponse;
import io.swagger.v3.oas.annotations.responses.ApiResponses;
import io.swagger.v3.oas.annotations.tags.Tag;
@RestController
@CrossOrigin(origins = {
"*"
})
@ConditionalOnProperty(value = "openaire.exporter.enable.dsm", havingValue = "true")
@Tag(name = "OpenAIRE DSM API (version 2.0)", description = "the OpenAIRE Datasource Manager API 2.0")
@RequestMapping("/dsm/2.0")
public class DsmApiControllerV2 extends AbstractExporterController {
@Autowired
private DsmCore dsmCore;
@GetMapping("/recentregistered/{size}")
@Operation(summary = "return the latest datasources that were registered through Provide (v2)", description = "Returns list of Datasource basic info.", tags = {
DS,
R
})
@ApiResponses(value = {
@ApiResponse(responseCode = "200", description = "OK"),
@ApiResponse(responseCode = "500", description = "unexpected error")
})
public SimpleResponse<SimpleDatasourceInfo> recentRegisteredV2(@PathVariable final int size) throws Throwable {
final StopWatch stop = StopWatch.createStarted();
final SimpleResponse<SimpleDatasourceInfo> rsp = dsmCore.searchRecentRegisteredV2(size);
return prepareResponse(1, size, stop, rsp);
}
@GetMapping("/countfirstcollect")
@Operation(summary = "return the number of datasources registered after the given date", description = "Returns a number.", tags = {
DS,
R
})
@ApiResponses(value = {
@ApiResponse(responseCode = "200", description = "OK"),
@ApiResponse(responseCode = "500", description = "unexpected error")
})
public Long countFirstCollectAfter(@RequestParam final String fromDate,
@RequestParam(required = false) final String typologyFilter) throws Throwable {
return dsmCore.countFirstCollect(fromDate, typologyFilter);
}
@GetMapping("/firstCollected")
@Operation(summary = "return the datasources that were collected for the first time after the specified date", description = "Returns list of Datasource basic info.", tags = {
DS,
R
})
@ApiResponses(value = {
@ApiResponse(responseCode = "200", description = "OK"),
@ApiResponse(responseCode = "500", description = "unexpected error")
})
public SimpleResponse<SimpleDatasourceInfo> firstCollectedAfter(@RequestParam final String fromDate,
@RequestParam(required = false) final String typologyFilter) throws Throwable {
final StopWatch stop = StopWatch.createStarted();
final List<SimpleDatasourceInfo> list = dsmCore.getFirstCollectedAfter(fromDate, typologyFilter);
final SimpleResponse<SimpleDatasourceInfo> rsp = ResponseUtils.simpleResponse(list);
return prepareResponse(1, list.size(), stop, rsp);
}
}

123
apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/DsmCore.java
View File

@ -26,6 +26,7 @@ import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.data.domain.Page;
import org.springframework.jdbc.core.BeanPropertyRowMapper;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.jdbc.core.RowMapper;
import org.springframework.stereotype.Component;
import eu.dnetlib.enabling.datasources.common.AggregationInfo;
@ -51,6 +52,7 @@ import eu.dnetlib.openaire.dsm.domain.RegisteredDatasourceInfo;
import eu.dnetlib.openaire.dsm.domain.RequestFilter;
import eu.dnetlib.openaire.dsm.domain.RequestSort;
import eu.dnetlib.openaire.dsm.domain.RequestSortOrder;
import eu.dnetlib.openaire.dsm.domain.SimpleDatasourceInfo;
import eu.dnetlib.openaire.dsm.domain.SimpleResponse;
import eu.dnetlib.openaire.dsm.domain.db.ApiDbEntry;
import eu.dnetlib.openaire.dsm.domain.db.DatasourceDbEntry;
@ -275,7 +277,7 @@ public class DsmCore {
// HELPERS //////////////
public SimpleResponse<?> searchRecentRegistered(final int size) throws Throwable {
public SimpleResponse<RegisteredDatasourceInfo> searchRecentRegistered(final int size) throws Throwable {
try {
final String sql =
IOUtils.toString(getClass().getResourceAsStream("/eu/dnetlib/openaire/sql/recent_registered_datasources.sql.st"), Charset.defaultCharset());
@ -318,4 +320,123 @@ public class DsmCore {
return rsp;
}
public SimpleResponse<SimpleDatasourceInfo> searchRecentRegisteredV2(final int size) throws Throwable {
try {
final String sql =
IOUtils.toString(getClass().getResourceAsStream("/eu/dnetlib/openaire/sql/recent_registered_datasources_v2.sql.st"), Charset.defaultCharset());
final List<SimpleDatasourceInfo> list =
jdbcTemplate.query(sql, rowMapperForSimpleDatasourceInfo(), size);
return ResponseUtils.simpleResponse(list);
} catch (final Throwable e) {
log.error("error searching recent datasources", e);
throw e;
}
}
public Long countFirstCollect(final String fromDate, final String typeFilter) throws Throwable {
try {
if (StringUtils.isNotBlank(typeFilter)) {
final String sql =
IOUtils
.toString(getClass().getResourceAsStream("/eu/dnetlib/openaire/sql/count_first_collected_datasources_fromDate_typology.st.sql"), Charset
.defaultCharset());
return jdbcTemplate.queryForObject(sql, Long.class, typeFilter + "%", fromDate);
} else {
final String sql =
IOUtils.toString(getClass().getResourceAsStream("/eu/dnetlib/openaire/sql/count_first_collected_datasources_fromDate.st.sql"), Charset
.defaultCharset());
return jdbcTemplate.queryForObject(sql, Long.class, fromDate);
}
} catch (final Throwable e) {
log.error("error searching datasources using the first collection date", e);
throw e;
}
}
public List<SimpleDatasourceInfo> getFirstCollectedAfter(final String fromDate, final String typeFilter) throws Throwable {
try {
if (StringUtils.isNotBlank(typeFilter)) {
final String sql =
IOUtils
.toString(getClass().getResourceAsStream("/eu/dnetlib/openaire/sql/first_collected_datasources_fromDate_typology.st.sql"), Charset
.defaultCharset());
return jdbcTemplate.query(sql, rowMapperForSimpleDatasourceInfo(), typeFilter + "%", fromDate);
} else {
final String sql =
IOUtils
.toString(getClass().getResourceAsStream("/eu/dnetlib/openaire/sql/first_collected_datasources_fromDate.st.sql"), Charset
.defaultCharset());
return jdbcTemplate.query(sql, rowMapperForSimpleDatasourceInfo(), fromDate);
}
} catch (final Throwable e) {
log.error("error searching datasources using the first collection date", e);
throw e;
}
}
private RowMapper<SimpleDatasourceInfo> rowMapperForSimpleDatasourceInfo() {
return (rs, rowNum) -> {
final SimpleDatasourceInfo info = new SimpleDatasourceInfo();
info.setId(rs.getString("id"));
info.setOfficialName(rs.getString("officialName"));
info.setEnglishName(rs.getString("englishName"));
info.setTypology(rs.getString("typology"));
info.setEoscType(rs.getString("eoscType"));
info.setEoscDatasourceType(rs.getString("eoscDatasourceType"));
info.setRegisteredBy(rs.getString("registeredBy"));
info.setRegistrationDate(rs.getString("registrationDate"));
info.setFirstCollectionDate(rs.getString("firstCollectionDate"));
info.setLastCollectionDate(rs.getString("lastCollectionDate"));
info.setLastCollectionTotal(rs.getLong("lastCollectionTotal"));
final Set<String> compatibilities = new HashSet<>();
for (final String s : (String[]) rs.getArray("compatibilities").getArray()) {
compatibilities.add(s);
}
// The order of the condition is important
if (compatibilities.contains("openaire-cris_1.1")) {
info.setCompatibility("openaire-cris_1.1");
} else if (compatibilities.contains("openaire4.0")) {
info.setCompatibility("openaire4.0");
} else if (compatibilities.contains("driver") && compatibilities.contains("openaire2.0")) {
info.setCompatibility("driver-openaire2.0");
} else if (compatibilities.contains("driver")) {
info.setCompatibility("driver");
} else if (compatibilities.contains("openaire2.0")) {
info.setCompatibility("openaire2.0");
} else if (compatibilities.contains("openaire3.0")) {
info.setCompatibility("openaire3.0");
} else if (compatibilities.contains("openaire2.0_data")) {
info.setCompatibility("openaire2.0_data");
} else if (compatibilities.contains("native")) {
info.setCompatibility("native");
} else if (compatibilities.contains("hostedBy")) {
info.setCompatibility("hostedBy");
} else if (compatibilities.contains("notCompatible")) {
info.setCompatibility("notCompatible");
} else {
info.setCompatibility("UNKNOWN");
}
for (final String s : (String[]) rs.getArray("organizations").getArray()) {
info.getOrganizations().put(StringUtils.substringBefore(s, "@@@").trim(), StringUtils.substringAfter(s, "@@@").trim());
}
return info;
};
}
}

26
apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/DatasourceSpecs.java
View File

@ -1,5 +1,6 @@
package eu.dnetlib.openaire.dsm.dao;
import java.util.Arrays;
import java.util.List;
import java.util.Map.Entry;
@ -55,6 +56,11 @@ public class DatasourceSpecs {
case search:
expressions.add(likeSearch(ds, cb, e));
break;
case multiSearch:
expressions.add(likeSearchList(ds, cb, e));
break;
case searchOrgs:
// search by case insensitive organization's country
@ -100,6 +106,11 @@ public class DatasourceSpecs {
expressions.add(likeSearch(api, cb, e));
break;
case multiSearch:
expressions.add(likeSearchList(api, cb, e));
break;
case searchOrgs:
throw new DsmRuntimeException("not implemented");
}
@ -117,6 +128,21 @@ public class DatasourceSpecs {
return cb.like(cb.lower(r.get(e.getKey().name())), WILDCARD + getValue(e) + WILDCARD);
}
// substring, case insensitive, like based search
private static Predicate likeSearchList(final Root<?> r, final CriteriaBuilder cb, final Entry<FilterName, Object> e) {
final String key = e.getKey().name();
final Predicate[] arr =
Arrays.stream(e.getValue().toString().split(","))
.map(String::trim)
.map(String::toLowerCase)
.filter(StringUtils::isNotBlank)
.map(s -> cb.like(cb.lower(r.get(key)), WILDCARD + s + WILDCARD))
.toArray(size -> new Predicate[size]);
return cb.or(arr);
}
// search by ID, managed. exact match
private static Predicate exactSearch(final Root<?> r, final CriteriaBuilder cb, final Entry<FilterName, Object> e) {
return cb.equal(r.get(e.getKey().name()), getValue(e));

4
apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/ResponseUtils.java
View File

@ -52,8 +52,8 @@ public class ResponseUtils {
return header(Lists.newLinkedList(), total);
}
public static SimpleResponse<?> simpleResponse(final List<?> list) {
final SimpleResponse rsp = new SimpleResponse().setResponse(list);;
public static <T> SimpleResponse<T> simpleResponse(final List<T> list) {
final SimpleResponse<T> rsp = new SimpleResponse<T>().setResponse(list);;
rsp.setHeader(header(Lists.newLinkedList(), list.size()));
return rsp;
}

3
apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/FilterName.java
View File

@ -33,9 +33,10 @@ public enum FilterName {
case contactemail:
case registeredby:
case typology:
case eoscDatasourceType:
case platform:
return FilterType.search;
case eoscDatasourceType:
return FilterType.multiSearch;
case country:
return FilterType.searchOrgs;
default:

5
apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/FilterType.java
View File

@ -1,5 +1,8 @@
package eu.dnetlib.openaire.dsm.domain;
public enum FilterType {
exact, search, searchOrgs
exact,
search,
searchOrgs,
multiSearch
}

129
apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/SimpleDatasourceInfo.java Normal file
View File

@ -0,0 +1,129 @@
package eu.dnetlib.openaire.dsm.domain;
import java.util.LinkedHashMap;
import java.util.Map;
public class SimpleDatasourceInfo {
private String id;
private String officialName;
private String englishName;
private Map<String, String> organizations = new LinkedHashMap<>();
@Deprecated
private String typology;
private String eoscType;
private String eoscDatasourceType;
private String registeredBy;
private String registrationDate;
private String compatibility;
private String firstCollectionDate;
private String lastCollectionDate;
private long lastCollectionTotal;
public String getId() {
return id;
}
public void setId(final String id) {
this.id = id;
}
public String getOfficialName() {
return officialName;
}
public void setOfficialName(final String officialName) {
this.officialName = officialName;
}
public String getEnglishName() {
return englishName;
}
public void setEnglishName(final String englishName) {
this.englishName = englishName;
}
public Map<String, String> getOrganizations() {
return organizations;
}
public void setOrganizations(final Map<String, String> organizations) {
this.organizations = organizations;
}
@Deprecated
public String getTypology() {
return typology;
}
@Deprecated
public void setTypology(final String typology) {
this.typology = typology;
}
public String getEoscType() {
return eoscType;
}
public void setEoscType(final String eoscType) {
this.eoscType = eoscType;
}
public String getEoscDatasourceType() {
return eoscDatasourceType;
}
public void setEoscDatasourceType(final String eoscDatasourceType) {
this.eoscDatasourceType = eoscDatasourceType;
}
public String getRegisteredBy() {
return registeredBy;
}
public void setRegisteredBy(final String registeredBy) {
this.registeredBy = registeredBy;
}
public String getRegistrationDate() {
return registrationDate;
}
public void setRegistrationDate(final String registrationDate) {
this.registrationDate = registrationDate;
}
public String getCompatibility() {
return compatibility;
}
public void setCompatibility(final String compatibility) {
this.compatibility = compatibility;
}
public String getFirstCollectionDate() {
return firstCollectionDate;
}
public void setFirstCollectionDate(final String firstCollectionDate) {
this.firstCollectionDate = firstCollectionDate;
}
public String getLastCollectionDate() {
return lastCollectionDate;
}
public void setLastCollectionDate(final String lastCollectionDate) {
this.lastCollectionDate = lastCollectionDate;
}
public long getLastCollectionTotal() {
return lastCollectionTotal;
}
public void setLastCollectionTotal(final long lastCollectionTotal) {
this.lastCollectionTotal = lastCollectionTotal;
}
}

7
apps/dnet-exporter-api/src/main/resources/eu/dnetlib/openaire/sql/count_first_collected_datasources_fromDate.st.sql Normal file
View File

@ -0,0 +1,7 @@
select count(*) as count
from (
select d.id
from dsm_services d left outer join dsm_api a on (d.id = a.service)
group by d.id
having min(a.first_collection_date) >= cast(? as date)
) as t

8
apps/dnet-exporter-api/src/main/resources/eu/dnetlib/openaire/sql/count_first_collected_datasources_fromDate_typology.st.sql Normal file
View File

@ -0,0 +1,8 @@
select count(*) as count
from (
select d.id
from dsm_services d left outer join dsm_api a on (d.id = a.service)
where d._typology_to_remove_ like ?
group by d.id
having min(a.first_collection_date) >= cast(? as date)
) as t

29
apps/dnet-exporter-api/src/main/resources/eu/dnetlib/openaire/sql/first_collected_datasources_fromDate.st.sql Normal file
View File

@ -0,0 +1,29 @@
SELECT
s.id AS "id",
s.officialname AS "officialName",
s.englishname AS "englishName",
s._typology_to_remove_ AS "typology",
s.eosc_type AS "eoscType",
s.eosc_datasource_type AS "eoscDatasourceType",
s.registeredby AS "registeredBy",
s.registrationdate::text AS "registrationDate",
MIN(a.first_collection_date) AS "firstCollectionDate",
MAX(a.last_collection_date) AS "lastCollectionDate",
(array_remove(array_agg(a.last_collection_total order by a.last_collection_date desc), NULL))[1] AS "lastCollectionTotal",
array_remove(array_agg(DISTINCT coalesce(a.compatibility_override, a.compatibility)), NULL) AS "compatibilities",
array_remove(array_agg(DISTINCT o.id||' @@@ '||o.legalname), NULL) AS "organizations"
FROM
dsm_services s
left outer join dsm_api a on (s.id = a.service)
left outer join dsm_service_organization dso on (s.id = dso.service)
left outer join dsm_organizations o on (o.id = dso.organization)
GROUP BY
s.id,
s.officialname,
s.englishname,
s._typology_to_remove_,
s.eosc_type,
s.eosc_datasource_type,
s.registeredby,
s.registrationdate
HAVING MIN(a.first_collection_date) >= cast(? as date)

31
apps/dnet-exporter-api/src/main/resources/eu/dnetlib/openaire/sql/first_collected_datasources_fromDate_typology.st.sql Normal file
View File

@ -0,0 +1,31 @@
SELECT
s.id AS "id",
s.officialname AS "officialName",
s.englishname AS "englishName",
s._typology_to_remove_ AS "typology",
s.eosc_type AS "eoscType",
s.eosc_datasource_type AS "eoscDatasourceType",
s.registeredby AS "registeredBy",
s.registrationdate::text AS "registrationDate",
MIN(a.first_collection_date) AS "firstCollectionDate",
MAX(a.last_collection_date) AS "lastCollectionDate",
(array_remove(array_agg(a.last_collection_total order by a.last_collection_date desc), NULL))[1] AS "lastCollectionTotal",
array_remove(array_agg(DISTINCT coalesce(a.compatibility_override, a.compatibility)), NULL) AS "compatibilities",
array_remove(array_agg(DISTINCT o.id||' @@@ '||o.legalname), NULL) AS "organizations"
FROM
dsm_services s
left outer join dsm_api a on (s.id = a.service)
left outer join dsm_service_organization dso on (s.id = dso.service)
left outer join dsm_organizations o on (o.id = dso.organization)
WHERE
s._typology_to_remove_ like ?
GROUP BY
s.id,
s.officialname,
s.englishname,
s._typology_to_remove_,
s.eosc_type,
s.eosc_datasource_type,
s.registeredby,
s.registrationdate
HAVING MIN(a.first_collection_date) >= cast(? as date)

4
apps/dnet-exporter-api/src/main/resources/eu/dnetlib/openaire/sql/recent_registered_datasources_fromDate.st.sql
View File

@ -1,9 +1,7 @@
select count(d.id) as count
select count(DISTINCT d.id) as count
from
dsm_services d
left outer join dsm_api a on (d.id = a.service)
left outer join dsm_service_organization dso on (d.id = dso.service)
left outer join dsm_organizations o on (o.id = dso.organization)
where
d.registrationdate >= cast(? as date)
and d.registrationdate < a.last_collection_date

4
apps/dnet-exporter-api/src/main/resources/eu/dnetlib/openaire/sql/recent_registered_datasources_fromDate_typology.st.sql
View File

@ -1,9 +1,7 @@
select count(d.id) as count
select count(DISTINCT d.id) as count
from
dsm_services d
left outer join dsm_api a on (d.id = a.service)
left outer join dsm_service_organization dso on (d.id = dso.service)
left outer join dsm_organizations o on (o.id = dso.organization)
where
d.registrationdate >= cast(? as date)
and d._typology_to_remove_ like ?

38
apps/dnet-exporter-api/src/main/resources/eu/dnetlib/openaire/sql/recent_registered_datasources_v2.sql.st Normal file
View File

@ -0,0 +1,38 @@
SELECT
s.id AS "id",
s.officialname AS "officialName",
s.englishname AS "englishName",
s._typology_to_remove_ AS "typology",
s.eosc_type AS "eoscType",
s.eosc_datasource_type AS "eoscDatasourceType",
s.registeredby AS "registeredBy",
s.registrationdate::text AS "registrationDate",
MIN(a.first_collection_date) AS "firstCollectionDate",
MAX(a.last_collection_date) AS "lastCollectionDate",
(array_remove(array_agg(a.last_collection_total order by a.last_collection_date desc), NULL))[1] AS "lastCollectionTotal",
array_remove(array_agg(DISTINCT coalesce(a.compatibility_override, a.compatibility)), NULL) AS "compatibilities",
array_remove(array_agg(DISTINCT o.id||' @@@ '||o.legalname), NULL) AS "organizations"
FROM
dsm_services s
left outer join dsm_api a on (s.id = a.service)
left outer join dsm_service_organization dso on (s.id = dso.service)
left outer join dsm_organizations o on (o.id = dso.organization)
WHERE
s.registrationdate is not null
and s.registeredby is not null
and s.managed = true
GROUP BY
s.id,
s.officialname,
s.englishname,
s._typology_to_remove_,
s.eosc_type,
s.eosc_datasource_type,
s.registeredby,
s.registrationdate
HAVING
s.registrationdate < max(a.last_collection_date)
and sum(a.last_collection_total) > 0
ORDER BY s.registrationdate desc
LIMIT ?

2
apps/dnet-orgs-database-application/pom.xml
View File

@ -4,7 +4,7 @@
<parent>
<groupId>eu.dnetlib.dhp</groupId>
<artifactId>apps</artifactId>
<version>3.3.2-SNAPSHOT</version>
<version>3.3.3-SNAPSHOT</version>
<relativePath>../</relativePath>
</parent>

19
apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/controller/AdminController.java
View File

@ -22,6 +22,7 @@ import eu.dnetlib.organizations.importer.ImportExecution;
import eu.dnetlib.organizations.importer.ImportExecutor;
import eu.dnetlib.organizations.model.SystemConfiguration;
import eu.dnetlib.organizations.model.utils.VocabularyTerm;
import eu.dnetlib.organizations.model.view.PersistentOrganizationView;
import eu.dnetlib.organizations.model.view.UserView;
import eu.dnetlib.organizations.repository.SystemConfigurationRepository;
import eu.dnetlib.organizations.repository.UserRepository;
@ -202,4 +203,22 @@ public class AdminController extends AbstractDnetController {
throw new RuntimeException("User not authorized");
}
}
@GetMapping("/api/persistentOrgs")
public Iterable<PersistentOrganizationView> listPersistentOrgs() {
return dbUtils.listPersistentOrgs();
}
@PostMapping("/api/persistentOrgs")
public Iterable<PersistentOrganizationView> addPersistentOrgs(@RequestBody final List<String> ids) {
ids.forEach(id -> dbUtils.addPersistentOrgs(id));
return dbUtils.listPersistentOrgs();
}
@DeleteMapping("/api/persistentOrgs")
public Iterable<PersistentOrganizationView> deletePersistentOrgs(@RequestParam final String id) {
dbUtils.deletePersistentOrgs(id);
return dbUtils.listPersistentOrgs();
}
}

9
apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/controller/HomeController.java
View File

@ -11,6 +11,7 @@ import org.springframework.security.core.Authentication;
import org.springframework.stereotype.Controller;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.ModelAttribute;
import org.springframework.web.bind.annotation.PathVariable;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.core.type.TypeReference;
@ -30,6 +31,9 @@ public class HomeController extends AbstractDnetController {
@Autowired
private SystemConfigurationRepository systemConfigurationRepository;
@Value("${openaire.explore.organization.baseurl}")
private String openaireBaseUrl;
@Value("${openorgs.support.pages}")
private String supportPagesJson;
@ -40,6 +44,11 @@ public class HomeController extends AbstractDnetController {
return env.acceptsProfiles(Profiles.of("dev")) ? "redirect:main" : "home";
}
@GetMapping("/redirect/oa/{orgId}")
public String openaireUrl(@PathVariable final String orgId) {
return "redirect:" + String.format(openaireBaseUrl, orgId);
}
@GetMapping("/main")
public String main() {
return "main";

11
apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/controller/OrganizationController.java
View File

@ -17,7 +17,6 @@ import javax.servlet.http.HttpServletResponse;
import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.data.domain.Page;
import org.springframework.data.domain.PageRequest;
import org.springframework.security.core.Authentication;
@ -83,8 +82,6 @@ public class OrganizationController extends AbstractDnetController {
private JournalEntryRepository journalEntryRepository;
@Autowired
private DatabaseUtils databaseUtils;
@Value("${openaire.explore.organization.baseurl}")
private String oaBaseUrl;
@PostMapping("/save")
public List<String> save(@RequestBody final OrganizationView org, final Authentication authentication) {
@ -107,9 +104,7 @@ public class OrganizationController extends AbstractDnetController {
@GetMapping("/info")
public OrganizationInfoView infoById(@RequestParam final String id, final Authentication authentication) {
final OrganizationInfoView info = organizationInfoViewRepository.findById(id).get();
info.fillGraphNodeInfo(info.getId(), oaBaseUrl);
return info;
return organizationInfoViewRepository.findById(id).get();
}
@GetMapping("/suggestionsInfo")
@ -160,9 +155,7 @@ public class OrganizationController extends AbstractDnetController {
}
private List<OpenaireDuplicateView> listDuplicates(final String id) {
final List<OpenaireDuplicateView> list = openaireDuplicateViewRepository.findByLocalId(id);
list.forEach(d -> d.fillGraphNodeInfo(d.getOaOriginalId(), oaBaseUrl));
return list;
return openaireDuplicateViewRepository.findByLocalId(id);
}
@GetMapping("/conflicts/byCountry/{country}")

34
apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/model/PersistentOrganization.java Normal file
View File

@ -0,0 +1,34 @@
package eu.dnetlib.organizations.model;
import java.io.Serializable;
import javax.persistence.Column;
import javax.persistence.Entity;
import javax.persistence.Id;
import javax.persistence.Table;
@Entity
@Table(name = "persistent_orgs")
public class PersistentOrganization implements Serializable {
private static final long serialVersionUID = 7684478315366015099L;
@Id
@Column(name = "id")
private String id;
public PersistentOrganization() {}
public PersistentOrganization(final String id) {
this.id = id;
}
public String getId() {
return id;
}
public void setId(final String id) {
this.id = id;
}
}

32
apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/model/utils/OpenaireGraphNode.java
View File

@ -1,23 +1,31 @@
package eu.dnetlib.organizations.model.utils;
import org.apache.commons.codec.digest.DigestUtils;
import org.apache.commons.lang3.StringUtils;
import javax.persistence.Column;
import javax.persistence.MappedSuperclass;
public interface OpenaireGraphNode {
@MappedSuperclass
public abstract class OpenaireGraphNode {
String getOaGraphId();
@Column(name = "openaire_id")
private String openaireId;
void setOaGraphId(String oaGraphId);
@Column(name = "openaire_persistent")
private Boolean persistent = false;
String getOaGraphUrl();
public String getOpenaireId() {
return openaireId;
}
void setOaGraphUrl(String oaGraphUrl);
public void setOpenaireId(final String openaireId) {
this.openaireId = openaireId;
}
default void fillGraphNodeInfo(final String origId, final String baseUrl) {
final String oaGraphId = StringUtils.substringBefore(origId, "::") + "::"
+ DigestUtils.md5Hex(StringUtils.substringAfter(origId, "::"));
setOaGraphId(oaGraphId);
setOaGraphUrl(baseUrl + oaGraphId);
public Boolean getPersistent() {
return persistent;
}
public void setPersistent(final Boolean persistent) {
this.persistent = persistent;
}
}

29
apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/model/view/OpenaireDuplicateView.java
View File

@ -8,7 +8,6 @@ import javax.persistence.Entity;
import javax.persistence.Id;
import javax.persistence.IdClass;
import javax.persistence.Table;
import javax.persistence.Transient;
import org.hibernate.annotations.Type;
@ -18,7 +17,7 @@ import eu.dnetlib.organizations.model.utils.OpenaireGraphNode;
@Entity
@Table(name = "oa_duplicates_view")
@IdClass(OpenaireDuplicatePK.class)
public class OpenaireDuplicateView implements Serializable, OpenaireGraphNode {
public class OpenaireDuplicateView extends OpenaireGraphNode implements Serializable {
/**
*
@ -88,12 +87,6 @@ public class OpenaireDuplicateView implements Serializable, OpenaireGraphNode {
@Column(name = "ec_nutscode")
private Boolean ecNutscode;
@Transient
private String oaGraphId;
@Transient
private String oaGraphUrl;
public String getLocalId() {
return localId;
}
@ -254,24 +247,4 @@ public class OpenaireDuplicateView implements Serializable, OpenaireGraphNode {
this.ecNutscode = ecNutscode;
}
@Override
public String getOaGraphId() {
return oaGraphId;
}
@Override
public void setOaGraphId(final String oaGraphId) {
this.oaGraphId = oaGraphId;
}
@Override
public String getOaGraphUrl() {
return oaGraphUrl;
}
@Override
public void setOaGraphUrl(final String oaGraphUrl) {
this.oaGraphUrl = oaGraphUrl;
}
}

29
apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/model/view/OrganizationInfoView.java
View File

@ -7,13 +7,12 @@ import javax.persistence.Column;
import javax.persistence.Entity;
import javax.persistence.Id;
import javax.persistence.Table;
import javax.persistence.Transient;
import eu.dnetlib.organizations.model.utils.OpenaireGraphNode;
@Entity
@Table(name = "organizations_info_view")
public class OrganizationInfoView implements Serializable, OpenaireGraphNode {
public class OrganizationInfoView extends OpenaireGraphNode implements Serializable {
/**
*
@ -48,12 +47,6 @@ public class OrganizationInfoView implements Serializable, OpenaireGraphNode {
@Column(name = "note")
private boolean note;
@Transient
private String oaGraphId;
@Transient
private String oaGraphUrl;
public String getId() {
return id;
}
@ -126,24 +119,4 @@ public class OrganizationInfoView implements Serializable, OpenaireGraphNode {
this.note = note;
}
@Override
public String getOaGraphId() {
return oaGraphId;
}
@Override
public void setOaGraphId(final String oaGraphId) {
this.oaGraphId = oaGraphId;
}
@Override
public String getOaGraphUrl() {
return oaGraphUrl;
}
@Override
public void setOaGraphUrl(final String oaGraphUrl) {
this.oaGraphUrl = oaGraphUrl;
}
}

4
apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/model/view/OrganizationView.java
View File

@ -15,13 +15,15 @@ import org.hibernate.annotations.TypeDefs;
import com.vladmihalcea.hibernate.type.json.JsonBinaryType;
import com.vladmihalcea.hibernate.type.json.JsonStringType;
import eu.dnetlib.organizations.model.utils.OpenaireGraphNode;
@Entity
@Table(name = "organizations_view")
@TypeDefs({
@TypeDef(name = "json", typeClass = JsonStringType.class),
@TypeDef(name = "jsonb", typeClass = JsonBinaryType.class)
})
public class OrganizationView implements Serializable {
public class OrganizationView extends OpenaireGraphNode implements Serializable {
/**
*

63
apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/model/view/PersistentOrganizationView.java Normal file
View File

@ -0,0 +1,63 @@
package eu.dnetlib.organizations.model.view;
import java.io.Serializable;
import javax.persistence.Column;
import javax.persistence.Entity;
import javax.persistence.Id;
import javax.persistence.Table;
import eu.dnetlib.organizations.model.utils.OpenaireGraphNode;
@Entity
@Table(name = "persistent_orgs_view")
public class PersistentOrganizationView extends OpenaireGraphNode implements Serializable {
private static final long serialVersionUID = -8906936709574708538L;
@Id
@Column(name = "id")
private String id;
@Column(name = "name")
private String name;
@Column(name = "city")
private String city;
@Column(name = "country")
private String country;
public String getId() {
return id;
}
public void setId(final String id) {
this.id = id;
}
public String getName() {
return name;
}
public void setName(final String name) {
this.name = name;
}
public String getCity() {
return city;
}
public void setCity(final String city) {
this.city = city;
}
public String getCountry() {
return country;
}
public void setCountry(final String country) {
this.country = country;
}
}

3
apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/repository/OrganizationRepository.java
View File

@ -22,9 +22,10 @@ public interface OrganizationRepository extends JpaRepository<Organization, Stri
@Query("update Organization set status = ?2 where id = ?1")
void updateStatus(String id, String status);
// to override the generation strategy of the ID
@Modifying
@Query(value = "insert into organizations(id) values (?1)", nativeQuery = true)
void preparePendingOrg(String id);
void prepareOrgWithId(String id);
double countByStatus(String string);

11
apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/repository/PersistentOrganizationRepository.java Normal file
View File

@ -0,0 +1,11 @@
package eu.dnetlib.organizations.repository;
import org.springframework.data.jpa.repository.JpaRepository;
import org.springframework.stereotype.Repository;
import eu.dnetlib.organizations.model.PersistentOrganization;
@Repository
public interface PersistentOrganizationRepository extends JpaRepository<PersistentOrganization, String> {
}

3
apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/repository/readonly/OrganizationViewRepository.java
View File

@ -1,5 +1,7 @@
package eu.dnetlib.organizations.repository.readonly;
import java.util.Optional;
import org.springframework.stereotype.Repository;
import eu.dnetlib.organizations.model.view.OrganizationView;
@ -7,4 +9,5 @@ import eu.dnetlib.organizations.model.view.OrganizationView;
@Repository
public interface OrganizationViewRepository extends ReadOnlyRepository<OrganizationView, String> {
Optional<OrganizationView> findByOpenaireId(String openaireId);
}

10
apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/repository/readonly/PersistentOrganizationViewRepository.java Normal file
View File

@ -0,0 +1,10 @@
package eu.dnetlib.organizations.repository.readonly;
import org.springframework.stereotype.Repository;
import eu.dnetlib.organizations.model.view.PersistentOrganizationView;
@Repository
public interface PersistentOrganizationViewRepository extends ReadOnlyRepository<PersistentOrganizationView, String> {
}

169
apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/utils/DatabaseUtils.java
View File

@ -12,10 +12,12 @@ import java.util.Optional;
import java.util.Set;
import java.util.UUID;
import java.util.function.Function;
import java.util.function.Predicate;
import java.util.stream.Collectors;
import javax.transaction.Transactional;
import org.apache.commons.beanutils.BeanUtils;
import org.apache.commons.codec.digest.DigestUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.logging.Log;
@ -37,6 +39,7 @@ import eu.dnetlib.organizations.model.OpenaireDuplicate;
import eu.dnetlib.organizations.model.Organization;
import eu.dnetlib.organizations.model.OtherIdentifier;
import eu.dnetlib.organizations.model.OtherName;
import eu.dnetlib.organizations.model.PersistentOrganization;
import eu.dnetlib.organizations.model.Relationship;
import eu.dnetlib.organizations.model.Url;
import eu.dnetlib.organizations.model.User;
@ -46,6 +49,7 @@ import eu.dnetlib.organizations.model.utils.OrganizationConflict;
import eu.dnetlib.organizations.model.utils.TempBrowseEntry;
import eu.dnetlib.organizations.model.utils.VocabularyTerm;
import eu.dnetlib.organizations.model.view.OrganizationView;
import eu.dnetlib.organizations.model.view.PersistentOrganizationView;
import eu.dnetlib.organizations.model.view.UserView;
import eu.dnetlib.organizations.repository.AcronymRepository;
import eu.dnetlib.organizations.repository.JournalEntryRepository;
@ -54,11 +58,13 @@ import eu.dnetlib.organizations.repository.OpenaireDuplicateRepository;
import eu.dnetlib.organizations.repository.OrganizationRepository;
import eu.dnetlib.organizations.repository.OtherIdentifierRepository;
import eu.dnetlib.organizations.repository.OtherNameRepository;
import eu.dnetlib.organizations.repository.PersistentOrganizationRepository;
import eu.dnetlib.organizations.repository.RelationshipRepository;
import eu.dnetlib.organizations.repository.UrlRepository;
import eu.dnetlib.organizations.repository.UserCountryRepository;
import eu.dnetlib.organizations.repository.UserRepository;
import eu.dnetlib.organizations.repository.readonly.OrganizationViewRepository;
import eu.dnetlib.organizations.repository.readonly.PersistentOrganizationViewRepository;
@Component
public class DatabaseUtils {
@ -87,6 +93,10 @@ public class DatabaseUtils {
private OrganizationViewRepository organizationViewRepository;
@Autowired
private JournalEntryRepository journalEntryRepository;
@Autowired
private PersistentOrganizationRepository persistentOrganizationRepository;
@Autowired
private PersistentOrganizationViewRepository persistentOrganizationViewRepository;
@Autowired
private JdbcTemplate jdbcTemplate;
@ -129,7 +139,7 @@ public class DatabaseUtils {
final String pendingId = OpenOrgsConstants.OPENORGS_PENDING_PREFIX + UUID.randomUUID();
orgView.setId(pendingId);
// to override the generation strategy of the ID
organizationRepository.preparePendingOrg(pendingId);
organizationRepository.prepareOrgWithId(pendingId);
} else {
orgView.setId(null); // The ID is generated by the DB
}
@ -468,45 +478,97 @@ public class DatabaseUtils {
@Transactional
public String fixConflictSimilars(final List<String> similarIds, final String user) {
final OffsetDateTime now = OffsetDateTime.now();
final List<OrganizationView> views =
similarIds.stream().map(organizationViewRepository::findById).filter(Optional::isPresent).map(Optional::get).collect(Collectors.toList());
// I create a new org
final OrganizationView newOrg = new OrganizationView();
newOrg.setId(null);
newOrg.setStatus(null);
newOrg.setName(findFirstString(views, OrganizationView::getName));
newOrg.setType(findFirstString(views, OrganizationView::getType));
newOrg.setLat(findFirstNumber(views, OrganizationView::getLat));
newOrg.setLng(findFirstNumber(views, OrganizationView::getLng));
newOrg.setCity(findFirstString(views, OrganizationView::getCity));
newOrg.setCountry(findFirstString(views, OrganizationView::getCountry));
newOrg.setOtherIdentifiers(findAll(views, OrganizationView::getOtherIdentifiers));
newOrg.setOtherNames(findAll(views, OrganizationView::getOtherNames));
newOrg.setAcronyms(findAll(views, OrganizationView::getAcronyms));
newOrg.setUrls(findAll(views, OrganizationView::getUrls));
newOrg.setRelations(findAll(views, OrganizationView::getRelations));
final List<OrganizationView> persistents = views.stream().filter(v -> v.getPersistent()).collect(Collectors.toList());
newOrg.getOtherNames()
.addAll(views.stream()
final OrganizationView masterOrg = new OrganizationView();
if (persistents.size() > 1) {
throw new RuntimeException("Too many persintent organizations");
} else if (persistents.size() == 1) {
backupOrg(persistents.get(0), user);
masterOrg.setId(persistents.get(0).getId());
masterOrg.setStatus(OrganizationStatus.approved.toString());
} else {
masterOrg.setId(null);
masterOrg.setStatus(null);
}
return fixConflicts(masterOrg, views, user);
}
private String backupOrg(final OrganizationView org, final String user) {
final String origId = org.getId();
final String backupId = origId + "::" + OffsetDateTime.now().toEpochSecond();
organizationRepository.prepareOrgWithId(backupId);
try {
final OrganizationView backupOrg = (OrganizationView) BeanUtils.cloneBean(org);
backupOrg.setId(backupId);
insertOrUpdateOrganization(backupOrg, user, false);
organizationRepository.updateStatus(backupId, OrganizationStatus.hidden.toString());
journalEntryRepository
.save(new JournalEntry(origId, JournalOperations.BACKUP_ORG, "Saved a backup copy: " + backupId, user));
journalEntryRepository
.save(new JournalEntry(backupId, JournalOperations.BACKUP_ORG, "Saved a backup copy of " + origId, user));
return backupId;
} catch (final Exception e) {
log.error("Error performing the backup of " + origId, e);
throw new RuntimeException("Error performing the backup of " + origId, e);
}
}
private String fixConflicts(final OrganizationView masterOrg, final List<OrganizationView> orgs, final String user) {
final OffsetDateTime now = OffsetDateTime.now();
final String finalMessage = (masterOrg.getId() == null ? "New org created merging: " : "Merging in persistent org: ") +
orgs.stream()
.map(OrganizationView::getId)
.collect(Collectors.joining(", "));
masterOrg.setName(findFirstString(orgs, OrganizationView::getName));
masterOrg.setType(findFirstString(orgs, OrganizationView::getType));
masterOrg.setLat(findFirstNumber(orgs, OrganizationView::getLat));
masterOrg.setLng(findFirstNumber(orgs, OrganizationView::getLng));
masterOrg.setCity(findFirstString(orgs, OrganizationView::getCity));
masterOrg.setCountry(findFirstString(orgs, OrganizationView::getCountry));
masterOrg.setOtherIdentifiers(findAll(orgs, OrganizationView::getOtherIdentifiers));
masterOrg.setOtherNames(findAll(orgs, OrganizationView::getOtherNames));
masterOrg.setAcronyms(findAll(orgs, OrganizationView::getAcronyms));
masterOrg.setUrls(findAll(orgs, OrganizationView::getUrls));
masterOrg.setRelations(findAll(orgs, OrganizationView::getRelations, r -> !r.getType().equals(RelationType.Merged_In.toString())
&& !r.getType().equals(RelationType.Merged_In.toString())));
masterOrg.getOtherNames()
.addAll(orgs.stream()
.map(OrganizationView::getName)
.filter(StringUtils::isNotBlank)
.filter(s -> StringUtils.equalsIgnoreCase(s, newOrg.getName()))
.filter(s -> StringUtils.equalsIgnoreCase(s, masterOrg.getName()))
.map(s -> new eu.dnetlib.organizations.model.view.OtherName(s, "UNKNOWN"))
.collect(Collectors.toList()));
final String masterId = insertOrUpdateOrganization(newOrg, user, false);
final String masterId = insertOrUpdateOrganization(masterOrg, user, false);
// I hide the merged organizations
similarIds.forEach(id -> {
hideConflictOrgs(masterId, id);
journalEntryRepository.save(new JournalEntry(masterId, JournalOperations.FIX_CONFLICT, "The org has been hidded and merged in " + masterId, user));
});
orgs.stream()
.map(OrganizationView::getId)
.filter(id -> !id.equals(masterId))
.forEach(id -> {
hideConflictOrgs(masterId, id);
journalEntryRepository
.save(new JournalEntry(id, JournalOperations.FIX_CONFLICT, "The org has been hidded and merged in " + masterId, user));
});
// I reassign the duplicates to the new org
final List<OpenaireDuplicate> newDuplicates = similarIds.stream()
final List<OpenaireDuplicate> newDuplicates = orgs.stream()
.map(OrganizationView::getId)
.map(openaireDuplicateRepository::findByLocalId)
.flatMap(l -> l.stream())
.map(d -> new OpenaireDuplicate(masterId, d.getOaOriginalId(), d.getRelType(), d.getOaCollectedFrom()))
@ -518,18 +580,20 @@ public class DatabaseUtils {
openaireDuplicateRepository.updateModificationDate(d.getLocalId(), d.getOaOriginalId(), user, now);
});
for (final String similarId : similarIds) {
orgs.forEach(org -> {
final String similarId = org.getId();
openaireConflictRepository.updateMultipleStatusAndResetGroup(similarId, SimilarityType.is_different.toString(), user, now);
}
});
for (int i = 0; i < similarIds.size(); i++) {
for (int j = i + 1; j < similarIds.size(); j++) {
openaireConflictRepository.updateStatusAndResetGroup(similarIds.get(i), similarIds.get(j), SimilarityType.is_similar.toString(), user, now);
for (int i = 0; i < orgs.size(); i++) {
for (int j = i + 1; j < orgs.size(); j++) {
openaireConflictRepository
.updateStatusAndResetGroup(orgs.get(i).getId(), orgs.get(j).getId(), SimilarityType.is_similar.toString(), user, now);
}
}
journalEntryRepository
.save(new JournalEntry(masterId, JournalOperations.FIX_CONFLICT, "New org created merging: " + StringUtils.join(similarIds, ", "), user));
.save(new JournalEntry(masterId, JournalOperations.FIX_CONFLICT, finalMessage, user));
return masterId;
}
@ -572,6 +636,10 @@ public class DatabaseUtils {
return views.stream().map(mapper).flatMap(s -> s.stream()).collect(Collectors.toCollection(LinkedHashSet::new));
}
private <T> Set<T> findAll(final List<OrganizationView> views, final Function<OrganizationView, Set<T>> mapper, final Predicate<T> filter) {
return views.stream().map(mapper).flatMap(s -> s.stream()).filter(filter).collect(Collectors.toCollection(LinkedHashSet::new));
}
private List<Relationship> hideConflictOrgs(final String masterId, final String otherId) {
organizationRepository.updateStatus(otherId, OrganizationStatus.hidden.toString());
openaireConflictRepository.findById(new OpenaireConflictPK(masterId, otherId)).ifPresent(openaireConflictRepository::delete);
@ -585,4 +653,39 @@ public class DatabaseUtils {
return jdbcTemplate.queryForList(sql, String.class);
}
public Iterable<PersistentOrganizationView> listPersistentOrgs() {
return persistentOrganizationViewRepository.findAll();
}
public String addPersistentOrgs(final String id) {
final boolean valid;
final String ooid;
if (id.length() == 46) {
final Optional<OrganizationView> orgView = organizationViewRepository.findByOpenaireId(id);
valid = orgView.map(OrganizationView::getStatus)
.filter(s -> s.equals(OrganizationStatus.approved.toString()))
.isPresent();
ooid = orgView.get().getId();
} else {
valid = organizationRepository.findById(id)
.map(Organization::getStatus)
.filter(s -> s.equals(OrganizationStatus.approved.toString()))
.isPresent();
ooid = id;
}
if (valid) {
persistentOrganizationRepository.save(new PersistentOrganization(ooid));
return ooid;
} else {
throw new RuntimeException("The ID does not refer to an approved Organization");
}
}
public void deletePersistentOrgs(final String id) {
persistentOrganizationRepository.deleteById(id);
}
}

1
apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/utils/JournalOperations.java
View File

@ -9,5 +9,6 @@ public enum JournalOperations {
DUPLICATES,
FIX_CONFLICT,
NO_CONFLICT,
BACKUP_ORG,
UNKNOWN
}

2
apps/dnet-orgs-database-application/src/main/resources/application.properties
View File

@ -51,4 +51,4 @@ openorgs.support.pages = { "Ask a question": "https://www.openaire.eu/support/he
openaire.override.logout.url =
openaire.explore.organization.baseurl = https://explore.openaire.eu/search/organization?organizationId=
openaire.explore.organization.baseurl = https://explore.openaire.eu/search/organization?organizationId=%s

41
apps/dnet-orgs-database-application/src/main/resources/sql/schema.sql
View File

@ -5,6 +5,7 @@ DROP VIEW IF EXISTS users_view;
DROP VIEW IF EXISTS conflict_groups_view;
DROP VIEW IF EXISTS suggestions_info_by_country_view;
DROP VIEW IF EXISTS duplicate_groups_view;
DROP VIEW IF EXISTS persistent_orgs_view;
DROP TABLE IF EXISTS sysconf;
DROP TABLE IF EXISTS other_ids;
@ -14,6 +15,8 @@ DROP TABLE IF EXISTS relationships;
DROP TABLE IF EXISTS urls;
DROP TABLE IF EXISTS oa_duplicates;
DROP TABLE IF EXISTS oa_conflicts;
DROP TABLE IF EXISTS persistent_orgs;
DROP TABLE IF EXISTS organizations;
DROP TABLE IF EXISTS org_types;
@ -374,6 +377,10 @@ CREATE TABLE organizations (
CREATE INDEX organizations_type_idx ON organizations(type);
CREATE INDEX organizations_country_idx ON organizations(country);
CREATE TABLE persistent_orgs (
id text PRIMARY KEY REFERENCES organizations(id) ON UPDATE CASCADE ON DELETE CASCADE
);
CREATE TABLE other_ids (
id text REFERENCES organizations(id) ON UPDATE CASCADE ON DELETE CASCADE,
otherid text,
@ -464,7 +471,9 @@ CREATE VIEW oa_duplicates_view AS
o.ec_internationalorganization,
o.ec_enterprise,
o.ec_smevalidated,
o.ec_nutscode
o.ec_nutscode,
substr(d.oa_original_id, 1, 14)||md5(substr(d.oa_original_id, 15)) as openaire_id,
false as openaire_persistent
FROM
oa_duplicates d
LEFT OUTER JOIN organizations o ON (o.id = d.oa_original_id)
@ -524,6 +533,8 @@ CREATE VIEW organizations_view AS SELECT
org.ec_enterprise,
org.ec_smevalidated,
org.ec_nutscode,
substr(org.id, 1, 14)||md5(substr(org.id, 15)) as openaire_id,
count(po.id) > 0 as openaire_persistent,
COALESCE(jsonb_agg(DISTINCT jsonb_build_object('id', oid.otherid, 'type', oid.type)) FILTER (WHERE oid.otherid IS NOT NULL), '[]') AS other_ids,
COALESCE(jsonb_agg(DISTINCT jsonb_build_object('name', n.name, 'lang', n.lang)) FILTER (WHERE n.name IS NOT NULL), '[]') AS other_names,
COALESCE(jsonb_agg(DISTINCT a.acronym) FILTER (WHERE a.acronym IS NOT NULL), '[]') AS acronyms,
@ -537,6 +548,7 @@ FROM
LEFT OUTER JOIN urls u ON (org.id = u.id)
LEFT OUTER JOIN relationships r ON (org.id = r.id1)
LEFT OUTER JOIN organizations relorg ON (relorg.id = r.id2)
LEFT OUTER JOIN persistent_orgs po ON (org.id = po.id)
GROUP BY
org.id,
org.name,
@ -564,13 +576,16 @@ CREATE VIEW organizations_info_view AS SELECT
org.creation_date,
org.modified_by,
org.modification_date,
substr(org.id, 1, 14)||md5(substr(org.id, 15)) as openaire_id,
count(po.id) > 0 as openaire_persistent,
count(DISTINCT d.oa_original_id) as n_duplicates,
count(DISTINCT c.id2) as n_conflicts,
count(DISTINCT n.note) > 0 as note
FROM organizations org
LEFT OUTER JOIN oa_duplicates d ON (org.id = d.local_id AND d.reltype = 'suggested')
LEFT OUTER JOIN oa_conflicts c ON (org.id = c.id1 AND c.reltype = 'suggested')
LEFT OUTER JOIN notes n ON (org.id = n.id)
LEFT OUTER JOIN oa_duplicates d ON (org.id = d.local_id AND d.reltype = 'suggested')
LEFT OUTER JOIN oa_conflicts c ON (org.id = c.id1 AND c.reltype = 'suggested')
LEFT OUTER JOIN notes n ON (org.id = n.id)
LEFT OUTER JOIN persistent_orgs po ON (org.id = po.id)
GROUP BY org.id;
CREATE VIEW organizations_simple_view AS SELECT
@ -674,6 +689,18 @@ WHERE
GROUP BY o.id, o.name, o.city, o.country
ORDER BY o.name;
CREATE VIEW persistent_orgs_view AS SELECT
po.id,
substr(po.id, 1, 14)||md5(substr(po.id,15)) as openaire_id,
true as openaire_persistent,
o.name,
o.city,
o.country
FROM
persistent_orgs po
JOIN organizations o ON (po.id = o.id)
ORDER BY o.name;
CREATE TABLE org_index_search(id text PRIMARY KEY, txt tsvector);
CREATE INDEX org_index_search_txt_gin_idx ON org_index_search USING gin(txt);
@ -682,7 +709,7 @@ CREATE OR REPLACE FUNCTION refresh_index_search() RETURNS bigint AS $$
WITH d as (
INSERT INTO org_index_search(id, txt) SELECT
o.id,
to_tsvector(o.id||' '||o.name||' '||array_to_string(array_agg(DISTINCT n.name), ' ','')||' '||array_to_string(array_agg(DISTINCT i.otherid), ' ','')||' '||array_to_string(array_agg(DISTINCT a.acronym), ' ','')||' '||array_to_string(array_agg(DISTINCT u.url), ' ',''))
to_tsvector(o.id||' '||substr(o.id, 1, 14)||md5(substr(o.id, 15))||' '||o.name||' '||array_to_string(array_agg(DISTINCT n.name), ' ','')||' '||array_to_string(array_agg(DISTINCT i.otherid), ' ','')||' '||array_to_string(array_agg(DISTINCT a.acronym), ' ','')||' '||array_to_string(array_agg(DISTINCT u.url), ' ',''))
FROM organizations o
LEFT OUTER JOIN other_names n on (o.id = n.id)
LEFT OUTER JOIN other_ids i on (o.id = i.id)
@ -694,8 +721,6 @@ $$ LANGUAGE SQL;
SELECT refresh_index_search();
CREATE OR REPLACE FUNCTION delete_index_search() RETURNS trigger LANGUAGE plpgsql AS $$
BEGIN
DELETE FROM org_index_search WHERE id = old.id;
@ -707,7 +732,7 @@ CREATE OR REPLACE FUNCTION insert_or_update_index_search_trigger() RETURNS trigg
BEGIN
INSERT INTO org_index_search(id, txt) (SELECT
o.id,
to_tsvector(o.id||' '||o.name||' '||array_to_string(array_agg(DISTINCT n.name), ' ','')||' '||array_to_string(array_agg(DISTINCT i.otherid), ' ','')||' '||array_to_string(array_agg(DISTINCT a.acronym), ' ','')||' '||array_to_string(array_agg(DISTINCT u.url), ' ',''))
to_tsvector(o.id||' '||substr(o.id, 1, 14)||md5(substr(o.id, 15))||' '||o.name||' '||array_to_string(array_agg(DISTINCT n.name), ' ','')||' '||array_to_string(array_agg(DISTINCT i.otherid), ' ','')||' '||array_to_string(array_agg(DISTINCT a.acronym), ' ','')||' '||array_to_string(array_agg(DISTINCT u.url), ' ',''))
FROM organizations o
LEFT OUTER JOIN other_names n on (o.id = n.id)
LEFT OUTER JOIN other_ids i on (o.id = i.id)

56
apps/dnet-orgs-database-application/src/main/resources/static/resources/html/pages/admin/persistentOrgs.html Normal file
View File

@ -0,0 +1,56 @@
<h4>Persistent Organizations</h4>
<p>
<small>It is necessary to persist the identifiers of the organizations associated to an Institutional Dashboard</small>
</p>
<div class="row">
<div class="col-sm-12">
<input type="text" class="form-control form-control-sm mb-3" ng-model="poFilter" placeholder="Filter...">
<form>
<table class="table table-sm small">
<thead class="thead-light">
<tr>
<th style="width: 200px">ID</th>
<th style="width: 300px">OA Graph Node ID</th>
<th>Name</th>
<th style="width: 300px">Place</th>
<th style="width: 40px"></th>
</tr>
</thead>
<tbody ng-show="orgs.length > 0">
<tr ng-repeat="o in orgs | filter:poFilter">
<td><a href="#!/edit/0/{{o.id}}">{{o.id}}</a></td>
<td>{{o.openaireId}}</td>
<td>{{o.name}}</td>
<td><img ng-src="resources/images/flags/{{o.country}}.gif" /> {{o.city || '-'}}, {{o.country}}</td>
<td class="text-right">
<button type="button" class="btn btn-sm btn-outline-danger" ng-click="deletePersistentOrg(o.id)">
<i class="fa fa-trash"></i>
</button>
</td>
</tr>
</tbody>
<tbody ng-hide="orgs.length > 0">
<tr>
<td colspan="5" class="text-muted">No persistent organizazions</td>
</tr>
</tbody>
<tfoot>
<tr ng-init="newId=''">
<td colspan="4">
<input type="text" class="form-control form-control-sm" placeholder="example: openorgs____::0123456789 (OpenOrgs ID, 24 chars) or openorgs____::781e5e245d69b566979b86e28d23f2c7 (OA Graph Node ID, 64 chars) ..." ng-model="newId" />
</td>
<td class="text-right">
<button type="submit" class="btn btn-sm btn-outline-success" ng-click="addPersistentOrg(newId); newId=''" ng-disabled="!newId">
<i class="fa fa-plus"></i>
</button>
</td>
</tr>
</tfoot>
</table>
</form>
</div>
</div>

4
apps/dnet-orgs-database-application/src/main/resources/static/resources/html/pages/admin/users.html
View File

@ -82,11 +82,11 @@
</tr>
<tr class="d-flex">
<th class="col-3">Reference person</th>
<td class="col-9" ng-if="currentUser.referencePerson">>{{currentUser.referencePerson}}</td>
<td class="col-9" ng-if="currentUser.referencePerson">{{currentUser.referencePerson}}</td>
<td class="col-9 text-muted" ng-if="!currentUser.referencePerson"><i>not specified</i></td>
</tr>
<tr class="d-flex">
<th class="col-3">Registration Request</th>
<th class="col-3">Registration request</th>
<td class="col-9" ng-if="currentUser.requestMessage"><pre>{{currentUser.requestMessage}}</pre></td>
<td class="col-9 text-muted" ng-if="!currentUser.requestMessage"><i>not specified</i></td>
</tr>

4
apps/dnet-orgs-database-application/src/main/resources/static/resources/html/pages/edit/edit.html
View File

@ -6,7 +6,9 @@
<b>ID: </b>{{info.id}}<br />
<b>Created at</b> {{info.creationDate | date:'MMMM d, y HH:mm:ss'}} <b>by</b> {{info.createdBy}}<br />
<b>Modified at</b> {{info.modificationDate | date:'MMMM d, y HH:mm:ss'}} <b>by</b> {{info.modifiedBy}}<br />
<b>OA Graph Node ID: </b> {{info.oaGraphId}} <a href="{{info.oaGraphUrl}}" target="_blank">[try on OA Explore]</a><br/>
<b>OA Graph Node ID: </b> {{info.openaireId}} <a href="/redirect/oa/{{info.openaireId}}" target="_blank">[try on OA Explore]</a>
<span class="badge badge-info" title="It is probably related to an Institutional Dashboard" ng-show="info.persistent">persistent</span>
<br/>
</p>
<div class="card">

2
apps/dnet-orgs-database-application/src/main/resources/static/resources/html/parts/org_details.html
View File

@ -4,7 +4,7 @@
'text-white bg-success' : show == 'success',
'text-white bg-info' : show == 'info',
'bg-secondary' : show == 'secondary',
}">{{orgTitle}}</div>
}">{{orgTitle}} <span class="badge badge-warning" title="It is probably related to an Institutional Dashboard" ng-show="org.persistent">persistent ID</span></div>
<table class="table table-sm table-condensed" style="table-layout: fixed">
<tr class="d-flex">
<th class="col-4 pl-3">Name</th>

2
apps/dnet-orgs-database-application/src/main/resources/static/resources/html/parts/org_duplicates.html
View File

@ -36,7 +36,7 @@
<td class="col-2 text-center small"><img ng-src="resources/images/flags/{{sr.oaCountry}}.gif" /> {{sr.oaCountry}}</td>
<td class="col-3 small">
<b>Original Id:</b> <span class="text-monospace">{{sr.oaOriginalId}}</span><br />
<b>OA Graph Node ID: </b> <span class="text-monospace">{{sr.oaGraphId}}</span> <a href="{{sr.oaGraphUrl}}" target="_blank">[try]</a>
<b>OA Graph Node ID: </b> <span class="text-monospace">{{sr.openaireId}}</span> <a href="/redirect/oa/{{sr.openaireId}}" target="_blank">[try]</a>
<span ng-if="sr.oaCollectedFrom"><br /><b>Provenance:</b> {{sr.oaCollectedFrom}}</span>
<span ng-if="sr.createdBy && sr.createdBy != 'dedupWf'"><br /><b>Added by:</b> {{sr.createdBy}}</span>
</td>

36
apps/dnet-orgs-database-application/src/main/resources/static/resources/js/organizations.js
View File

@ -119,7 +119,12 @@ orgsModule.directive('resolveConflictsModal', function($http, $route, $window) {
call_http_post($http, 'api/organizations/conflicts/fix/similar', ids, function(res) {
$('#' + scope.modalId).modal('hide');
if (scope.openNewOrg == '1') {
$('#' + scope.modalId).on('hidden.bs.modal', function (e) { $window.location.assign('#!/edit/0/' + res.data[0]); });
$('#' + scope.modalId).on('hidden.bs.modal', function (e) {
var oldLoc = $window.location;
$window.location.assign('#!/edit/0/' + res.data[0]);
var newLoc = $window.location;
if (oldLoc == newLoc) $route.reload();
});
} else {
$('#' + scope.modalId).on('hidden.bs.modal', function (e) { $route.reload(); });
}
@ -323,7 +328,13 @@ orgsModule.directive('orgConflicts', function($http, $window, $location, $route,
angular.forEach(scope.conflicts, function(o, pos) { ids.push(o.id); });
if (merge) {
call_http_post($http, "api/organizations/conflicts/fix/similar", ids, function(res) { $window.location.assign('#!/edit/0/' + res.data[0]); });
call_http_post($http, "api/organizations/conflicts/fix/similar", ids, function(res) {
if (res.data[0] == scope.org.id) {
$route.reload();
} else {
$window.location.assign('#!/edit/0/' + res.data[0]);
}
});
} else {
call_http_post($http, "api/organizations/conflicts/fix/different", ids, function(res) { $route.reload(); });
}
@ -379,6 +390,7 @@ orgsModule.config(function($routeProvider) {
.when('/sysconf', { templateUrl: 'resources/html/pages/admin/sysConf.html', controller: 'sysConfCtrl' })
.when('/utils', { templateUrl: 'resources/html/pages/admin/utils.html', controller: 'utilsCtrl' })
.when('/lastImport', { templateUrl: 'resources/html/pages/admin/lastImport.html', controller: 'lastImportCtrl' })
.when('/persistentOrgs', { templateUrl: 'resources/html/pages/admin/persistentOrgs.html', controller: 'persistentOrgsCtrl' })
.otherwise({ redirectTo: '/search' });
});
@ -806,6 +818,26 @@ orgsModule.controller('lastImportCtrl', function ($scope, $http) {
$scope.refresh();
});
orgsModule.controller('persistentOrgsCtrl', function ($scope, $http) {
$scope.orgs = {};
$scope.refresh = function() {
call_http_get($http, 'api/persistentOrgs', function(res) { $scope.orgs = res.data; });
}
$scope.addPersistentOrg = function(id) {
call_http_post($http, 'api/persistentOrgs', [ id ], function(res) { $scope.orgs = res.data; });
}
$scope.deletePersistentOrg = function(id) {
if (confirm("Are you sure ?")) {
call_http_delete($http, 'api/persistentOrgs?id=' + id, function(res) { $scope.orgs = res.data; });
}
}
$scope.refresh();
});
orgsModule.controller('utilsCtrl', function ($scope, $http) {
$scope.fulltextIndexMessage = '';

2
apps/dnet-orgs-database-application/src/main/resources/templates/main.html
View File

@ -117,6 +117,8 @@ fieldset > legend { font-size : 1.2rem !important; }
<a class="dropdown-item" href="#!/utils" sec:authorize="hasRole('ROLE_OPENORGS_ADMIN')">System utilities</a>
<a class="dropdown-item" href="#!/lastImport" sec:authorize="hasRole('ROLE_OPENORGS_ADMIN')">Last import of suggestions</a>
<div class="dropdown-divider" sec:authorize="hasRole('ROLE_OPENORGS_ADMIN')"></div>
<a class="dropdown-item" href="#!/persistentOrgs">Configure persistent organizations</a>
<div class="dropdown-divider" sec:authorize="hasRole('ROLE_OPENORGS_ADMIN')"></div>
<a class="dropdown-item" href="#!/users">Manage users</a>
</div>
</li>

2
apps/pom.xml
View File

@ -3,7 +3,7 @@
<parent>
<groupId>eu.dnetlib.dhp</groupId>
<artifactId>dnet-applications</artifactId>
<version>3.3.2-SNAPSHOT</version>
<version>3.3.3-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>

2
apps/scholexplorer-api/pom.xml
View File

@ -3,7 +3,7 @@
<parent>
<groupId>eu.dnetlib.dhp</groupId>
<artifactId>apps</artifactId>
<version>3.3.2-SNAPSHOT</version>
<version>3.3.3-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>

2
cmd-line-apps/dhp-broker-client/pom.xml
View File

@ -4,7 +4,7 @@
<parent>
<groupId>eu.dnetlib.dhp</groupId>
<artifactId>cmd-line-apps</artifactId>
<version>3.3.2-SNAPSHOT</version>
<version>3.3.3-SNAPSHOT</version>
<relativePath>../</relativePath>
</parent>

2
cmd-line-apps/pom.xml
View File

@ -3,7 +3,7 @@
<parent>
<groupId>eu.dnetlib.dhp</groupId>
<artifactId>dnet-applications</artifactId>
<version>3.3.2-SNAPSHOT</version>
<version>3.3.3-SNAPSHOT</version>
<relativePath>../</relativePath>
</parent>

2
libs/dnet-apps-common/pom.xml
View File

@ -4,7 +4,7 @@
<parent>
<groupId>eu.dnetlib.dhp</groupId>
<artifactId>libs</artifactId>
<version>3.3.2-SNAPSHOT</version>
<version>3.3.3-SNAPSHOT</version>
<relativePath>../</relativePath>
</parent>

2
libs/dnet-broker-apps-common/pom.xml
View File

@ -4,7 +4,7 @@
<parent>
<groupId>eu.dnetlib.dhp</groupId>
<artifactId>libs</artifactId>
<version>3.3.2-SNAPSHOT</version>
<version>3.3.3-SNAPSHOT</version>
<relativePath>../</relativePath>
</parent>

2
libs/dnet-openaire-broker-common/pom.xml
View File

@ -3,7 +3,7 @@
<parent>
<groupId>eu.dnetlib.dhp</groupId>
<artifactId>libs</artifactId>
<version>3.3.2-SNAPSHOT</version>
<version>3.3.3-SNAPSHOT</version>
<relativePath>../</relativePath>
</parent>

2
libs/pom.xml
View File

@ -4,7 +4,7 @@
<parent>
<groupId>eu.dnetlib.dhp</groupId>
<artifactId>dnet-applications</artifactId>
<version>3.3.2-SNAPSHOT</version>
<version>3.3.3-SNAPSHOT</version>
<relativePath>../</relativePath>
</parent>

2
pom.xml
View File

@ -10,7 +10,7 @@
<modelVersion>4.0.0</modelVersion>
<groupId>eu.dnetlib.dhp</groupId>
<artifactId>dnet-applications</artifactId>
<version>3.3.2-SNAPSHOT</version>
<version>3.3.3-SNAPSHOT</version>
<packaging>pom</packaging>
<licenses>