D-Net
/
dnet-applications
16
1
Fork 0
Code Issues Pull Requests Releases Wiki Activity

new-exporter #5

Merged
michele.artini merged 21 commits from new-exporter into master 2022-04-22 10:27:22 +02:00
Conversation 0 Commits 21 Files Changed 176 +93 -203
15 changed files with 93 additions and 203 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files
Showing only changes of commit 00ed2922da - Show all commits

6
apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/DsmCore.java
View File

@ -262,14 +262,14 @@ public class DsmCore {
}
}
public Long countRegisteredAfter(final String fromDate, final String typologyFilter) throws Throwable {
public Long countRegisteredAfter(final String fromDate, final String typeFilter) throws Throwable {
try {
if (StringUtils.isNotBlank(typologyFilter)) {
if (StringUtils.isNotBlank(typeFilter)) {
final String sql =
IOUtils.toString(getClass().getResourceAsStream("/eu/dnetlib/openaire/sql/recent_registered_datasources_fromDate_typology.st.sql"), Charset
.defaultCharset());
return jdbcTemplate.queryForObject(sql, Long.class, fromDate, typologyFilter + "%");
return jdbcTemplate.queryForObject(sql, Long.class, fromDate, typeFilter + "%");
} else {
final String sql =
IOUtils.toString(getClass().getResourceAsStream("/eu/dnetlib/openaire/sql/recent_registered_datasources_fromDate.st.sql"), Charset

6
apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/DatasourceDao.java
View File

@ -2,6 +2,8 @@ package eu.dnetlib.openaire.dsm.dao;
import java.util.List;
import org.springframework.data.domain.Page;
import eu.dnetlib.enabling.datasources.common.Api;
import eu.dnetlib.enabling.datasources.common.Datasource;
import eu.dnetlib.enabling.datasources.common.DatasourceManagerCommon;
@ -10,7 +12,6 @@ import eu.dnetlib.openaire.dsm.domain.RequestFilter;
import eu.dnetlib.openaire.dsm.domain.RequestSort;
import eu.dnetlib.openaire.dsm.domain.RequestSortOrder;
import eu.dnetlib.openaire.vocabularies.Country;
import org.springframework.data.domain.Page;
public interface DatasourceDao<DS extends Datasource<?, ?>, API extends Api<?>> extends DatasourceManagerCommon<DS, API> {
@ -32,7 +33,7 @@ public interface DatasourceDao<DS extends Datasource<?, ?>, API extends Api<?>>
void updateTimezone(String dsId, String timezone) throws DsmException;
void updateTypology(String dsId, String timezone) throws DsmException;
void updateEoscDatasourceType(String dsId, String timezone) throws DsmException;
void updateRegisteringUser(String dsId, String registeredBy) throws DsmException;
@ -54,5 +55,6 @@ public interface DatasourceDao<DS extends Datasource<?, ?>, API extends Api<?>>
void updateApiBaseUrl(String apiId, String baseUrl) throws DsmException;
@Override
void addApi(final API api) throws DsmException;
}

8
apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/DatasourceDaoImpl.java
View File

@ -238,15 +238,15 @@ public class DatasourceDaoImpl implements DatasourceDao<DatasourceDbEntry, ApiDb
}
@Override
public void updateTypology(final String dsId, final String typology) throws DsmException {
public void updateEoscDatasourceType(final String dsId, final String type) throws DsmException {
final Vocabulary typologies = vocabularyClient.getDatasourceTypologies();
if (!typologies.hasCode(typology)) {
if (!typologies.hasCode(type)) {
throw new DsmException(
HttpStatus.SC_BAD_REQUEST,
String.format("invalid datasource typology '%s', provide one according to vocabulary %s", typology, config.getVocabularies()
String.format("invalid datasource type '%s', provide one according to vocabulary %s", type, config.getVocabularies()
.getDatasourceTypologiesEndpoint()));
}
dsRepository.setTypology(dsId, typology);
dsRepository.setEoscDatasourceType(dsId, type);
}
@Override

4
apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/DatasourceDbEntryRepository.java
View File

@ -53,8 +53,8 @@ public interface DatasourceDbEntryRepository extends JpaRepository<DatasourceDbE
@Modifying
@Transactional
@Query("update #{#entityName} d set d.typology = ?2 where d.id = ?1")
void setTypology(String dsId, String typology);
@Query("update #{#entityName} d set d.eosc_datasource_type = ?2 where d.id = ?1")
void setEoscDatasourceType(String dsId, String type);
@Modifying
@Transactional

122
apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/utils/DsmMappingUtils.java
View File

@ -7,8 +7,6 @@ import java.util.stream.Collectors;
import org.apache.commons.lang3.RandomStringUtils;
import org.apache.commons.lang3.StringUtils;
import org.dom4j.DocumentHelper;
import org.dom4j.Element;
import org.springframework.beans.BeanUtils;
import org.springframework.beans.BeanWrapper;
import org.springframework.beans.BeanWrapperImpl;
@ -16,7 +14,6 @@ import org.springframework.beans.BeanWrapperImpl;
import com.fasterxml.jackson.databind.ObjectMapper;
import eu.dnetlib.data.transform.xml.AbstractDNetXsltFunctions;
import eu.dnetlib.miscutils.datetime.DateUtils;
import eu.dnetlib.openaire.dsm.domain.ApiDetails;
import eu.dnetlib.openaire.dsm.domain.DatasourceDetails;
import eu.dnetlib.openaire.dsm.domain.DatasourceDetailsUpdate;
@ -50,7 +47,7 @@ public class DsmMappingUtils {
ds.setEnglishname(d.getEnglishname());
ds.setRegisteredby(d.getRegisteredby());
ds.setWebsiteurl(d.getWebsiteurl());
ds.setTypology(d.getTypology());
ds.setEoscDatasourceType(d.getEoscDatasourceType());
ds.setRegistrationdate(d.getRegistrationdate());
ds.setLogoUrl(d.getLogourl());
ds.setDescription(d.getDescription());
@ -104,112 +101,6 @@ public class DsmMappingUtils {
return _convert(d, DatasourceDbEntry.class);
}
public static String asRepositoryProfile(final DatasourceDetails ds) {
final Element root = DocumentHelper.createElement("RESOURCE_PROFILE");
final Element header = root.addElement("HEADER");
header.addElement("RESOURCE_IDENTIFIER").addAttribute("value", "");
header.addElement("RESOURCE_TYPE").addAttribute("value", "RepositoryServiceResourceType");
header.addElement("RESOURCE_KIND").addAttribute("value", "RepositoryServiceResources");
header.addElement("RESOURCE_URI").addAttribute("value", "");
header.addElement("DATE_OF_CREATION").addAttribute("value", DateUtils.now_ISO8601());
header.addElement("PROTOCOL");
final Element body = root.addElement("BODY");
final Element conf = body.addElement("CONFIGURATION");
conf.addElement("DATASOURCE_TYPE").setText(ds.getTypology());
final Element origId = conf.addElement("DATASOURCE_ORIGINAL_ID");
origId.addAttribute("provenance", "D-NET");
origId.setText(ds.getId());
conf.addElement("DATASOURCE_AGGREGATED").setText("false");
conf.addElement("ENVIRONMENTS").addElement("ENVIRONMENT").setText("OPENAIRE");
conf.addElement("TYPOLOGY").setText("" + ds.getTypology());
conf.addElement("MAX_SIZE_OF_DATASTRUCTURE").setText("0");
conf.addElement("AVAILABLE_DISKSPACE").setText("0");
conf.addElement("MAX_NUMBER_OF_DATASTRUCTURE").setText("0");
final String officialName = ds.getOfficialname();
conf.addElement("OFFICIAL_NAME").setText(officialName);
final String englishName = ds.getEnglishname();
conf.addElement("ENGLISH_NAME").setText(StringUtils.isNotBlank(englishName) ? englishName : officialName);
conf.addElement("ICON_URI").setText("" + ds.getLogourl());
final OrganizationDetails org = getOrganization(ds);
conf.addElement("COUNTRY").setText(org != null ? org.getCountry() : "");
final Element location = conf.addElement("LOCATION");
location.addElement("LONGITUDE").setText("" + ds.getLongitude());
location.addElement("LATITUDE").setText("" + ds.getLatitude());
location.addElement("TIMEZONE").setText("" + ds.getTimezone());
conf.addElement("REPOSITORY_WEBPAGE").setText(ds.getWebsiteurl());
getOrganization(ds);
conf.addElement("REPOSITORY_INSTITUTION").setText(org != null ? org.getLegalname() : "");
conf.addElement("ADMIN_INFO").setText(ds.getContactemail());
conf.addElement("INTERFACES");
final Element extraFields = conf.addElement("EXTRA_FIELDS");
addExtraField(extraFields, "OpenAireDataSourceId", ds.getId());
addExtraField(extraFields, "ACTIVATION_ID", ds.getActivationId());
addExtraField(extraFields, "NamespacePrefix", ds.getNamespaceprefix());
addExtraField(extraFields, "aggregatorName", ds.getAggregator());
addExtraField(extraFields, "dateOfCollection", "" + ds.getDateofcollection());
addExtraField(extraFields, "dateOfValidation", "" + ds.getDateofvalidation());
conf.addElement("REGISTERED_BY").setText(ds.getRegisteredby());
final Element status = body.addElement("STATUS");
status.addElement("NUMBER_OF_OBJECTS").setText("0");
status.addElement("LAST_UPDATE").addAttribute("value", DateUtils.now_ISO8601());
final Element qos = body.addElement("QOS");
qos.addElement("AVAILABILITY").setText("0");
qos.addElement("CAPACITY");
qos.addElement("THROUGHPUT").setText("0");
body.addElement("SECURITY_PARAMETERS");
body.addElement("BLACKBOARD");
return root.asXML();
}
public static String asRepositoryInterfce(final ApiDetails api) {
final Element iface = DocumentHelper.createElement("INTERFACE");
iface.addAttribute("active", String.valueOf(api.getActive()))
.addAttribute("compliance", api.getCompatibility())
.addAttribute("contentDescription", api.getContentdescription())
.addAttribute("id", api.getId())
.addAttribute("label", String.format("%s (%s)", api.getTypology(), api.getCompatibility()))
.addAttribute("removable", String.valueOf(api.getRemovable()))
.addAttribute("typology", api.getTypology());
iface.addElement("ACCESS_PROTOCOL").setText(api.getProtocol());
if (api.getApiParams() != null) {
final Element accessProtocol = (Element) iface.selectSingleNode("./ACCESS_PROTOCOL");
api.getApiParams().forEach(ap -> {
accessProtocol.addAttribute(ap.getParam(), ap.getValue());
});
}
iface.addElement("BASE_URL").setText(api.getBaseurl());
iface.addElement("INTERFACE_EXTRA_FIELD").addAttribute("name", "last_collection_date");
iface.addElement("INTERFACE_EXTRA_FIELD").addAttribute("name", "last_collection_mdId");
iface.addElement("INTERFACE_EXTRA_FIELD").addAttribute("name", "last_collection_total");
iface.addElement("INTERFACE_EXTRA_FIELD").addAttribute("name", "last_aggregation_date");
iface.addElement("INTERFACE_EXTRA_FIELD").addAttribute("name", "last_aggregation_mdId");
iface.addElement("INTERFACE_EXTRA_FIELD").addAttribute("name", "last_aggregation_total");
final Element mdPathNode = iface.addElement("INTERFACE_EXTRA_FIELD");
mdPathNode.addAttribute("name", "metadata_identifier_path");
if (StringUtils.isNotBlank(api.getMetadataIdentifierPath())) {
mdPathNode.setText(api.getMetadataIdentifierPath());
}
return iface.asXML();
}
// HELPERS
private static <T> T _convert(final Object o, final Class<T> clazz) {
@ -217,17 +108,6 @@ public class DsmMappingUtils {
return mapper.convertValue(o, clazz);
}
private static OrganizationDetails getOrganization(final DatasourceDetails ds) {
if (ds.getOrganizations() != null && !ds.getOrganizations().isEmpty()) { return ds.getOrganizations().stream().findFirst().get(); }
return null;
}
private static void addExtraField(final Element extraFields, final String field, final String value) {
final Element f = extraFields.addElement("FIELD");
f.addElement("key").setText(field);
f.addElement("value").setText(value != null ? value : "");
}
private static String asOpenaireId(final String id) {
final String prefix = StringUtils.substringBefore(id, ID_SEPARATOR);
final String md5 = StringUtils.substringAfter(id, ID_SEPARATOR);

29
apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/ApiDetails.java
View File

@ -25,7 +25,7 @@ public class ApiDetails extends ApiIgnoredProperties {
private String contentdescription = null;
@ApiModelProperty(position = 4)
private String typology = null;
private String eoscDatasourceType = null;
@ApiModelProperty(position = 5)
private String compatibility;
@ -79,10 +79,6 @@ public class ApiDetails extends ApiIgnoredProperties {
return contentdescription;
}
public String getTypology() {
return typology;
}
public String getCompatibility() {
return compatibility;
}
@ -135,11 +131,6 @@ public class ApiDetails extends ApiIgnoredProperties {
return this;
}
public ApiDetails setTypology(final String typology) {
this.typology = typology;
return this;
}
public ApiDetails setCompatibility(final String compatibility) {
this.compatibility = compatibility;
return this;
@ -192,23 +183,35 @@ public class ApiDetails extends ApiIgnoredProperties {
return compatibilityOverride;
}
public void setCompatibilityOverride(final String compatibilityOverride) {
public ApiDetails setCompatibilityOverride(final String compatibilityOverride) {
this.compatibilityOverride = compatibilityOverride;
return this;
}
public Boolean getRemovable() {
return removable;
}
public void setRemovable(final Boolean removable) {
public ApiDetails setRemovable(final Boolean removable) {
this.removable = removable;
return this;
}
public String getMetadataIdentifierPath() {
return metadataIdentifierPath;
}
public void setMetadataIdentifierPath(final String metadataIdentifierPath) {
public ApiDetails setMetadataIdentifierPath(final String metadataIdentifierPath) {
this.metadataIdentifierPath = metadataIdentifierPath;
return this;
}
public String getEoscDatasourceType() {
return eoscDatasourceType;
}
public ApiDetails setEoscDatasourceType(final String eoscDatasourceType) {
this.eoscDatasourceType = eoscDatasourceType;
return this;
}
}

10
apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/DatasourceDetails.java
View File

@ -66,7 +66,7 @@ public class DatasourceDetails extends DatasourceIgnoredProperties {
@NotBlank
@ApiModelProperty(position = 13)
private String typology;
private String eoscDatasourceType;
@ApiModelProperty(position = 14)
private Date dateofcollection;
@ -172,8 +172,8 @@ public class DatasourceDetails extends DatasourceIgnoredProperties {
return dateofvalidation;
}
public String getTypology() {
return typology;
public String getEoscDatasourceType() {
return eoscDatasourceType;
}
public Date getDateofcollection() {
@ -305,8 +305,8 @@ public class DatasourceDetails extends DatasourceIgnoredProperties {
return this;
}
public DatasourceDetails setTypology(final String typology) {
this.typology = typology;
public DatasourceDetails setEoscDatasourceType(final String eoscDatasourceType) {
this.eoscDatasourceType = eoscDatasourceType;
return this;
}

10
apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/DatasourceDetailsUpdate.java
View File

@ -51,7 +51,7 @@ public class DatasourceDetailsUpdate {
@NotBlank
@ApiModelProperty(position = 13)
private String typology;
private String eoscDatasourceType;
@ApiModelProperty(position = 15)
private String platform;
@ -114,8 +114,8 @@ public class DatasourceDetailsUpdate {
return timezone;
}
public String getTypology() {
return typology;
public String getEoscDatasourceType() {
return eoscDatasourceType;
}
public String getPlatform() {
@ -183,8 +183,8 @@ public class DatasourceDetailsUpdate {
return this;
}
public DatasourceDetailsUpdate setTypology(final String typology) {
this.typology = typology;
public DatasourceDetailsUpdate setEoscDatasourceType(final String eoscDatasourceType) {
this.eoscDatasourceType = eoscDatasourceType;
return this;
}

10
apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/DatasourceSnippetExtended.java
View File

@ -38,7 +38,7 @@ public class DatasourceSnippetExtended {
private Date registrationdate;
@ApiModelProperty(position = 7)
private String typology;
private String eoscDatasourceType;
@ApiModelProperty(position = 8)
private String logoUrl;
@ -106,12 +106,12 @@ public class DatasourceSnippetExtended {
this.registrationdate = registrationdate;
}
public String getTypology() {
return typology;
public String getEoscDatasourceType() {
return eoscDatasourceType;
}
public void setTypology(final String typology) {
this.typology = typology;
public void setEoscDatasourceType(final String eoscDatasourceType) {
this.eoscDatasourceType = eoscDatasourceType;
}
public String getLogoUrl() {

51
apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/FilterName.java
View File

@ -1,34 +1,43 @@
package eu.dnetlib.openaire.dsm.domain;
import com.fasterxml.jackson.annotation.JsonAutoDetect;
import io.swagger.annotations.ApiModel;
@JsonAutoDetect
@ApiModel(value = "Filter name", description = "List of the field names used to filter datasources")
public enum
FilterName {
id, managed, collectedfrom, // exact match
officialname, englishname, websiteurl, contactemail, registeredby, typology, platform, // like match
public enum FilterName {
id,
managed,
collectedfrom, // exact match
officialname,
englishname,
websiteurl,
contactemail,
registeredby,
eoscDatasourceType,
platform, // like match
country; // exact match on related organization
public static FilterType type(FilterName filterName) {
public static FilterType type(final FilterName filterName) {
switch (filterName) {
case id:
case managed:
case collectedfrom:
return FilterType.exact;
case officialname:
case englishname:
case websiteurl:
case contactemail:
case registeredby:
case typology:
case platform:
return FilterType.search;
case country:
return FilterType.searchOrgs;
default:
throw new IllegalStateException("unmapped filter type for: " + filterName);
case id:
case managed:
case collectedfrom:
return FilterType.exact;
case officialname:
case englishname:
case websiteurl:
case contactemail:
case registeredby:
case eoscDatasourceType:
case platform:
return FilterType.search;
case country:
return FilterType.searchOrgs;
default:
throw new IllegalStateException("unmapped filter type for: " + filterName);
}
}

11
apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/RegisteredDatasourceInfo.java
View File

@ -6,7 +6,7 @@ public class RegisteredDatasourceInfo {
private String officialName;
private String englishName;
private String organization;
private String typology;
private String eoscDatasourceType;
private String registeredBy;
private String registrationDate;
private String compatibility;
@ -45,12 +45,12 @@ public class RegisteredDatasourceInfo {
this.organization = organization;
}
public String getTypology() {
return typology;
public String getEoscDatasourceType() {
return eoscDatasourceType;
}
public void setTypology(final String typology) {
this.typology = typology;
public void setEoscDatasourceType(final String eoscDatasourceType) {
this.eoscDatasourceType = eoscDatasourceType;
}
public String getRegisteredBy() {
@ -92,4 +92,5 @@ public class RegisteredDatasourceInfo {
public void setLastCollectionTotal(final long lastCollectionTotal) {
this.lastCollectionTotal = lastCollectionTotal;
}
}

23
apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/db/DatasourceApiDbEntry.java
View File

@ -1,5 +1,6 @@
package eu.dnetlib.openaire.dsm.domain.db;
import javax.persistence.Column;
import javax.persistence.Entity;
import javax.persistence.Id;
import javax.persistence.Table;
@ -26,7 +27,8 @@ public class DatasourceApiDbEntry {
private String contactemail;
private String collectedfrom;
private String registeredby;
private String typology;
@Column(name = "eosc_datasource_type")
private String eoscDatasourceType;
private String platform;
private Boolean managed;
@ -34,7 +36,7 @@ public class DatasourceApiDbEntry {
protected String contentdescription = null;
protected Boolean active = false;
protected Boolean removable = false;
protected String apitypology = null;
protected String compatibility;
private String baseurl;
@ -96,12 +98,12 @@ public class DatasourceApiDbEntry {
this.websiteurl = websiteurl;
}
public String getTypology() {
return typology;
public String getEoscDatasourceType() {
return eoscDatasourceType;
}
public void setTypology(final String typology) {
this.typology = typology;
public void setEoscDatasourceType(final String eoscDatasourceType) {
this.eoscDatasourceType = eoscDatasourceType;
}
public String getPlatform() {
@ -152,14 +154,6 @@ public class DatasourceApiDbEntry {
this.removable = removable;
}
public String getApitypology() {
return apitypology;
}
public void setApitypology(final String apitypology) {
this.apitypology = apitypology;
}
public String getCompatibility() {
return compatibility;
}
@ -183,4 +177,5 @@ public class DatasourceApiDbEntry {
public void setCollectedfrom(final String collectedfrom) {
this.collectedfrom = collectedfrom;
}
}

2
apps/dnet-exporter-api/src/main/resources/eu/dnetlib/openaire/sql/recent_registered_datasources.sql.st
View File

@ -3,7 +3,7 @@ select
d.officialname as "officialName",
d.englishname as "englishName",
o.legalname as organization,
d.typology as typology,
d.eosc_datasource_type as eosc_datasource_type,
d.registeredby as "registeredBy",
d.registrationdate::text as "registrationDate",
a.compatibility as compatibility,

2
apps/dnet-exporter-api/src/main/resources/eu/dnetlib/openaire/sql/recent_registered_datasources_fromDate_typology.st.sql
View File

@ -6,7 +6,7 @@ from
left outer join dsm_organizations o on (o.id = dso.organization)
where
d.registrationdate >= cast(? as date)
and d.typology like ?
and d.eosc_datasource_type like ?
and d.registrationdate < a.last_collection_date
and d.registeredby is not null
and d.managed = true

2
apps/dnet-exporter-api/src/main/resources/global.properties
View File

@ -102,4 +102,4 @@ openaire.exporter.swaggerInfo.apiContactEmail = ${openaire.exporter.swaggerD
# VOCABULARIES
openaire.exporter.vocabularies.baseUrl = http://localhost:8980/provision/mvc/vocabularies
openaire.exporter.vocabularies.countriesEndpoint = ${openaire.exporter.vocabularies.baseUrl}/dnet:countries.json
openaire.exporter.vocabularies.datasourceTypologiesEndpoint = ${openaire.exporter.vocabularies.baseUrl}/dnet:datasource_typologies.json
openaire.exporter.vocabularies.datasourceTypologiesEndpoint = ${openaire.exporter.vocabularies.baseUrl}/dnet:eosc_datasource_types.json