MaDgIK
/
portal-search
14
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

cursor search first draft

This commit is contained in:
Katerina 2024-09-10 14:24:17 +03:00
parent c479a23c9e
commit 27ae04d4a6
10 changed files with 193 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

0
output.csv Normal file
View File

52
pom.xml
View File

@ -17,6 +17,58 @@
<properties>
<java.version>11</java.version>
</properties>
<repositories>
<repository>
<id>dnet45-releases</id>
<name>D-Net 45 releases</name>
<url>https://maven.d4science.org/nexus/content/repositories/dnet45-releases</url>
<layout>default</layout>
<snapshots>
<enabled>false</enabled>
</snapshots>
<releases>
<enabled>true</enabled>
</releases>
</repository>
<repository>
<id>dnet45-snapshots</id>
<name>D-Net 45 snapshots</name>
<url>https://maven.d4science.org/nexus/content/repositories/dnet45-snapshots</url>
<layout>default</layout>
<snapshots>
<enabled>true</enabled>
</snapshots>
<releases>
<enabled>false</enabled>
</releases>
</repository>
<repository>
<id>dnet45-bootstrap-snapshot</id>
<name>D-Net 45 Bootstrap Snapshot</name>
<url>https://maven.d4science.org/nexus/content/repositories/dnet45-bootstrap-snapshot/</url>
<releases>
<enabled>false</enabled>
</releases>
<snapshots>
<enabled>true</enabled>
</snapshots>
<layout>default</layout>
</repository>
<repository>
<id>dnet45-bootstrap-release</id>
<name>D-Net 45 Bootstrap Release</name>
<url>https://maven.d4science.org/nexus/content/repositories/dnet45-bootstrap-release/</url>
<releases>
<enabled>true</enabled>
</releases>
<snapshots>
<enabled>false</enabled>
</snapshots>
<layout>default</layout>
</repository>
</repositories>
<dependencies>
<dependency>
<groupId>org.springframework.boot</groupId>

1
src/main/java/com/example/CSVTest.java
View File

@ -25,6 +25,7 @@ public class CSVTest {
String jsonString = "[{\"datasourcetype\":{\"code\":\"pubsrepository::journal\",\"label\":\"Journal\"},\"datasourcetypeui\":{\"code\":\"Journal archive\",\"label\":\"Journal archive\"},\"eosctype\":{\"code\":\"Data Source\",\"label\":\"Data Source\"},\"eoscdatasourcetype\":{\"code\":\"Journal archive\",\"label\":\"Journal Archive\"},\"openairecompatibility\":{\"code\":\"hostedBy\",\"label\":\"collected from a compatible aggregator\"},\"officialname\":\"Journal of Biblical Text Research\",\"englishname\":\"Journal of Biblical Text Research\",\"namespaceprefix\":\"jrnl12265926\",\"latitude\":\"0.0\",\"longitude\":\"0.0\",\"odnumberofitems\":\"0.0\",\"versioning\":false,\"versioncontrol\":false,\"journal\":{\"name\":\"Journal of Biblical Text Research\",\"issnPrinted\":\"1226-5926\"},\"thematic\":false,\"contentpolicies\":[{\"code\":\"Journal articles\",\"label\":\"Journal articles\"}],\"consenttermsofuse\":false,\"fulltextdownload\":false}]";
SolrRecord solrRecord;
// List<eu.dnetlib.dhp.schema.solr.Datasource> dd = objectMapper.readValue(jsonString, new TypeReference<List<Datasource>>() {});

6
src/main/java/com/example/controllers/DatasourceController.java
View File

@ -35,4 +35,10 @@ public class DatasourceController {
@Valid @ValidFacetFields(type = FacetRequest.Type.datasource)FacetRequest facetRequest){
return datasourceService.facetSearch(datasourceRequest, facetRequest);
}
@GetMapping(path="/fetchCsv")
public CustomSolrResponse fetchCsv(DatasourceRequest datasourceRequest,
@Valid @ValidFacetFields(type = FacetRequest.Type.datasource)FacetRequest facetRequest){
return datasourceService.facetSearch(datasourceRequest, facetRequest);
}
}

1
src/main/java/com/example/dto/request/validators/FacetFieldsValidator.java
View File

@ -25,6 +25,7 @@ public class FacetFieldsValidator implements ConstraintValidator<ValidFacetField
@Override
public boolean isValid(FacetRequest facetRequest, ConstraintValidatorContext context) {
//TODO reconsider
if (facetRequest == null || facetRequest.facetFields == null || facetRequest.type == null) {
return true; // Consider null as valid, use @NotNull if required
}

54
src/main/java/com/example/helpers/csv/DatasourceCsv.java Normal file
View File

@ -0,0 +1,54 @@
package com.example.helpers.csv;
import lombok.Data;
import java.util.List;
@Data
public class DatasourceCsv {
String name;
String type;
String compatibility;
List<String> oaipmh;
String websiteUrl;
public String getName() {
return name;
}
public void setName(String name) {
this.name = name;
}
public String getType() {
return type;
}
public void setType(String type) {
this.type = type;
}
public String getCompatibility() {
return compatibility;
}
public void setCompatibility(String compatibility) {
this.compatibility = compatibility;
}
public List<String> getOaipmh() {
return oaipmh;
}
public void setOaipmh(List<String> oaipmh) {
this.oaipmh = oaipmh;
}
public String getWebsiteUrl() {
return websiteUrl;
}
public void setWebsiteUrl(String websiteUrl) {
this.websiteUrl = websiteUrl;
}
}

25
src/main/java/com/example/mappers/CustomSolrParamsMapper.java
View File

@ -56,6 +56,31 @@ public interface CustomSolrParamsMapper {
})
CustomSolrParams toCustomSolrParams(EntityRequest entityRequest);
/**
* queryOpts.add("start", "0");
* queryOpts.add("rows", "0");
* queryOpts.remove("rows");
* queryOpts.add("rows", rows+"");
* queryOpts.add("fl", "__result");
* queryOpts.add("shards.tolerant","true");
* queryOpts.add("cursorMark", "*");
* queryOpts.add("sort", "__indexrecordidentifier asc");
* @param entityRequest
* @return
*/
@Mappings({
@Mapping(target = "basicSolrQuery", expression = "java(mapBasicSolrQuery(entityRequest))"),
@Mapping(target = "start", constant = "0"),
@Mapping(target = "rows", constant = "500"),
@Mapping(target = "facet", constant = "false"), //TODO reconsider where these should be set
@Mapping(target = "shards.tolerant", constant = "true"),
// @Mapping(target = "cursorMark", constant = "*"), //TODO consider more flexible options for the future
@Mapping(target = "sort", constant = "__indexrecordidentifier asc"),
// @Mapping(target = "fieldList", constant = "__json")
})
CustomSolrParams toCustomCursorSolrParams(EntityRequest entityRequest);
default String mapQueryToQueryString(String query) {
return query != null ? query : "*:*";
}

16
src/main/java/com/example/mappers/csv/DatasourceCsvMapper.java Normal file
View File

@ -0,0 +1,16 @@
package com.example.mappers.csv;
import com.example.helpers.csv.DatasourceCsv;
import eu.dnetlib.dhp.schema.solr.SolrRecord;
import org.mapstruct.Mapper;
import org.mapstruct.Mapping;
@Mapper(componentModel = "spring")
public interface DatasourceCsvMapper {
@Mapping(target="name", source = "datasource.officialname")
@Mapping(target="websiteUrl", source="datasource.websiteurl")
@Mapping(target="type", source = "datasource.openairecompatibility.label")
@Mapping(target = "oaipmh", source = "datasource.accessinfopackage")
DatasourceCsv toDatasourceCsv(SolrRecord solrRecord);
}

33
src/main/java/com/example/repository/SolrRepository.java
View File

@ -134,7 +134,40 @@ public class SolrRepository {
System.out.println(query);
try {
QueryResponse response = solrConnectionManager.solrClient.query(query);
System.out.println("RESPONSE " + response.getResults().size());
return response;
} catch (SolrServerException | IOException e) {
e.printStackTrace();
return null;
}
}
public QueryResponse cursorQuery(CustomSolrParams customSolrParams) {
SolrQuery solrQuery = new SolrQuery();
SolrQuery query = solrQuery;
if(customSolrParams.getBasicSolrQuery()!=null)
query.setQuery(customSolrParams.getBasicSolrQuery().getQuery());
else
query.setQuery("*:*"); //TODO fix it on a mapper level
customSolrParams.getBasicSolrQuery().getFq().stream().forEach(query::addFilterQuery);
query.setFacet(false);
if (customSolrParams!=null && customSolrParams.getFacetFields()!=null) // TODO reconsider
customSolrParams.getFacetFields().stream().forEach(query::addFacetField);
query.setStart(customSolrParams.getStart());
query.setRows(customSolrParams.getRows());
//cursor mark + id
query.setShowDebugInfo(true);
System.out.println(query);
try {
QueryResponse response = solrConnectionManager.solrClient.query(query);

5
src/main/java/com/example/services/DatasourceService.java
View File

@ -53,4 +53,9 @@ public class DatasourceService {
CustomSolrParams customSolrParams = customSolrParamsMapper.toCustomSolrParams(datasourceRequest, facetRequest);
return customSolrResponseMapper.toCustomSolrResponse(solrRepository.query3(customSolrParams));
}
public String fetchCsv(DatasourceRequest datasourceRequest) {
CustomSolrParams customSolrParams = customSolrParamsMapper.toCustomSolrParams(datasourceRequest);
return "";
}
}