D-Net
/
dnet-applications
16
1
Fork 0
Code Issues Pull Requests Releases Wiki Activity

merge from master

This commit is contained in:
Michele Artini 2022-11-16 09:57:23 +01:00
parent 56b4e04ac3 39d335439f
commit 77d0e46652
342 changed files with 20663 additions and 875 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@ -5,6 +5,7 @@
*.iml
*.ipr
*.iws
*.java-version
*~
/**/*.sh
/**/my_application.properties

10
apps/bioschemas-api/deploy.info Normal file
View File

@ -0,0 +1,10 @@
{
"type_source": "SVN",
"goal": "package -U source:jar",
"url": "http://svn-public.driver.research-infrastructures.eu/driver/dnet50/modules/dnet-bioschemas-api/trunk/",
"deploy_repository": "dnet5-snapshots",
"version": "5",
"mail": "sandro.labruzzo@isti.cnr.it,michele.artini@isti.cnr.it, claudio.atzori@isti.cnr.it, alessia.bardi@isti.cnr.it, enrico.ottonello@isti.cnr.it",
"deploy_repository_url": "http://maven.research-infrastructures.eu/nexus/content/repositories/dnet5-snapshots",
"name": "dnet-ariadneplus-graphdb-publisher"
}

62
apps/bioschemas-api/pom.xml Normal file
View File

@ -0,0 +1,62 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<groupId>eu.dnetlib.dhp</groupId>
<artifactId>apps</artifactId>
<version>3.3.3-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
<modelVersion>4.0.0</modelVersion>
<packaging>jar</packaging>
<artifactId>bioschemas-api</artifactId>
<dependencies>
<dependency>
<groupId>hwu.elixir</groupId>
<artifactId>bmuse-core</artifactId>
</dependency>
<dependency>
<groupId>org.apache.any23</groupId>
<artifactId>apache-any23-core</artifactId>
</dependency>
<dependency>
<groupId>org.eclipse.rdf4j</groupId>
<artifactId>rdf4j-rio-rdfxml</artifactId>
</dependency>
<dependency>
<groupId>org.eclipse.rdf4j</groupId>
<artifactId>rdf4j-model</artifactId>
</dependency>
<dependency>
<groupId>org.jsoup</groupId>
<artifactId>jsoup</artifactId>
</dependency>
<dependency>
<groupId>org.seleniumhq.selenium</groupId>
<artifactId>selenium-java</artifactId>
</dependency>
<dependency>
<groupId>commons-io</groupId>
<artifactId>commons-io</artifactId>
<version>${bioschemas-commons-io-version}</version>
</dependency>
<dependency>
<groupId>commons-validator</groupId>
<artifactId>commons-validator</artifactId>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-help-plugin</artifactId>
</plugin>
<plugin>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-maven-plugin</artifactId>
</plugin>
</plugins>
</build>
</project>

8
apps/bioschemas-api/request_example.txt Normal file
View File

@ -0,0 +1,8 @@
https://mobidb.org/sitemap2.xml.gz
scrape?datasourceKey=mobidb&sitemapUrl=https%3A%2F%2Fmobidb.org%2Fsitemap2.xml.gz
https://proteinensemble.org/sitemap2.xml.gz
scrape?datasourceKey=ped&sitemapUrl=https%3A%2F%2Fproteinensemble.org%2Fsitemap2.xml.gz
https://disprot.org/sitemap2.xml.gz
scrape?datasourceKey=disprot&sitemapUrl=https%3A%2F%2Fdisprot.org%2Fsitemap2.xml.gz

14
apps/bioschemas-api/src/main/java/eu/dnetlib/bioschemas/api/AppConfigGarr.java Normal file
View File

@ -0,0 +1,14 @@
package eu.dnetlib.bioschemas.api;
import org.springframework.context.annotation.Configuration;
import org.springframework.context.annotation.Profile;
/**
* @author enrico.ottonello
*
*/
@Profile("garr")
@Configuration
public class AppConfigGarr {
}

52
apps/bioschemas-api/src/main/java/eu/dnetlib/bioschemas/api/MainApplication.java Normal file
View File

@ -0,0 +1,52 @@
package eu.dnetlib.bioschemas.api;
import io.swagger.v3.oas.models.tags.Tag;
import org.springdoc.core.GroupedOpenApi;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.cache.annotation.EnableCaching;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.ComponentScan;
import org.springframework.scheduling.annotation.EnableScheduling;
import eu.dnetlib.common.app.AbstractDnetApp;
import java.util.Arrays;
import java.util.List;
@SpringBootApplication
@EnableCaching
@EnableScheduling
@ComponentScan(basePackages = "eu.dnetlib")
public class MainApplication extends AbstractDnetApp {
public static final String BIOSCHEMAS_APIS = "D-Net Bioschemas Service APIs";
public static void main(final String[] args) {
SpringApplication.run(MainApplication.class, args);
}
@Bean
public GroupedOpenApi publicApi() {
return GroupedOpenApi.builder()
.group(BIOSCHEMAS_APIS)
.pathsToMatch("/api/**")
.build();
}
@Override
protected String swaggerTitle() {
return BIOSCHEMAS_APIS;
}
@Override
protected List<Tag> swaggerTags() {
return Arrays.asList(new Tag().name(BIOSCHEMAS_APIS).description(BIOSCHEMAS_APIS));
}
@Override
protected String swaggerDesc() {
return BIOSCHEMAS_APIS;
}
}

206
apps/bioschemas-api/src/main/java/eu/dnetlib/bioschemas/api/ServiceScrapeDriver.java Normal file
View File

@ -0,0 +1,206 @@
package eu.dnetlib.bioschemas.api;
import eu.dnetlib.bioschemas.api.crawl.CrawlRecord;
import eu.dnetlib.bioschemas.api.scraper.BMUSEScraper;
import eu.dnetlib.bioschemas.api.scraper.ScrapeState;
import eu.dnetlib.bioschemas.api.scraper.ScrapeThread;
import eu.dnetlib.bioschemas.api.utils.UrlParser;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import java.io.*;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.List;
import java.util.Objects;
import java.util.Properties;
import java.util.stream.Collectors;
import java.util.stream.Stream;
/**
* Runs the scrape. Collect a list of URLs (in the form of CrawlRecords) to scrape.
*
*/
public class ServiceScrapeDriver {
private static final String propertiesFile = "application.properties";
private int waitTime = 1;
private int numberOfPagesToCrawlInALoop;
private int totalNumberOfPagesToCrawlInASession;
private String outputFolder;
private int pagesCounter = 0;
private int scrapeVersion = 1;
private String sitemapUrl;
private String sitemapURLKey;
private String maxScrapedPages;
private String outputFilename;
private static SimpleDateFormat formatter = new SimpleDateFormat("yyyy-MM-dd 'at' HH:mm:ss z");
private static final Log logger = LogFactory.getLog(ServiceScrapeDriver.class);
public ServiceScrapeDriver(String sitemapUrl, String sitemapURLKey, String maxScrapedPages, String outputFilename, String outputFolder) {
this.sitemapUrl = sitemapUrl;
this.sitemapURLKey = sitemapURLKey;
this.maxScrapedPages = maxScrapedPages;
this.outputFilename = outputFilename;
this.outputFolder = outputFolder;
}
/**
* Fires off threads
* Originally designed as a multi-threaded process; now reduced to a single thread as
* the selenium webdriver is too expensive to run multi-threaded. However, the threading
* as been left in situ in case it is useful in the future.
*
*/
public void runScrape() throws IOException {
processProperties();
String url = sitemapUrl.toLowerCase();
Elements urls = UrlParser.getSitemapList(getSitemapUrl(), getSitemapURLKey());
Stream<Element> urlStream = null;
if (Objects.nonNull(maxScrapedPages)) {
urlStream = urls.stream().limit(Long.parseLong(maxScrapedPages));
} else {
urlStream = urls.stream();
}
List<Element> sites = urlStream.collect(Collectors.toList());
logger.info("Pages available for scraping: " + sites.size());
List<CrawlRecord> pagesToPull = generatePagesToPull(sites);
if (pagesToPull.isEmpty()) {
logger.error("Cannot retrieve URLs");
throw new RuntimeException("No pages found from sitemap");
}
ScrapeState scrapeState = new ScrapeState(pagesToPull);
logger.info("STARTING CRAWL: " + formatter.format(new Date(System.currentTimeMillis())));
while (pagesCounter < totalNumberOfPagesToCrawlInASession) {
logger.debug(pagesCounter + " scraped of " + totalNumberOfPagesToCrawlInASession);
ScrapeThread scrape1 = new ScrapeThread(new BMUSEScraper(), scrapeState, waitTime, scrapeVersion);
scrape1.setName("S1");
scrape1.start();
long startTime = System.nanoTime();
try {
scrape1.join();
} catch (InterruptedException e) {
logger.error("Exception waiting on thread");
e.printStackTrace();
return;
}
if(!scrape1.isFileWritten()) {
logger.error("Could not write output file so shutting down!");
Date date = new Date(System.currentTimeMillis());
logger.info("ENDING CRAWL after failure at: " + formatter.format(date));
return;
}
logger.debug("Value of isFileWritten: " + scrape1.isFileWritten());
long endTime = System.nanoTime();
long timeElapsed = endTime - startTime;
logger.debug("Time in s to complete: " + timeElapsed / 1e+9);
pagesCounter += numberOfPagesToCrawlInALoop;
logger.debug("ENDED loop");
}
logger.info("ENDING CRAWL: " + formatter.format(new Date(System.currentTimeMillis())));
File output = new File(outputFolder.concat("/").concat(outputFilename));
if (output.exists()) {
output.delete();
output.createNewFile();
}
FileWriter fileWriter;
BufferedWriter bufferedWriter;
fileWriter = new FileWriter(output.getAbsoluteFile(), true); // true to append
bufferedWriter = new BufferedWriter(fileWriter);
List<CrawlRecord> processed = scrapeState.getPagesProcessed();
for (int i=0;i<processed.size();i++) {
try {
bufferedWriter.write(processed.get(i).getNquads());
bufferedWriter.newLine();
bufferedWriter.flush();
} catch (IOException e) {
e.printStackTrace();
}
}
bufferedWriter.close();
logger.info(" Data stored into "+output.getAbsolutePath());
}
/**
* Get a list of URLs (in the form of CrawlRecords) that need to be scraped
*
* @return List of URLs to be scraped
* @see CrawlRecord
*/
private List<CrawlRecord> generatePagesToPull(List<Element> sites) {
List<CrawlRecord> crawls = sites
.stream()
.map(s -> {
CrawlRecord crawlRecord = new CrawlRecord(s.text());
String[] urlSplitted = crawlRecord.getUrl().split("/");
String name = urlSplitted[urlSplitted.length - 1];
crawlRecord.setName(name);
return crawlRecord;
})
.collect(Collectors.toList());
return crawls;
}
/**
* Updates properties based on properties file in src > main > resources
*
*/
private void processProperties() {
ClassLoader classLoader = ServiceScrapeDriver.class.getClassLoader();
InputStream is = classLoader.getResourceAsStream(propertiesFile);
if(is == null) {
logger.error(" Cannot find " + propertiesFile + " file");
throw new IllegalArgumentException(propertiesFile + "file is not found!");
}
Properties prop = new Properties();
try {
prop.load(is);
} catch (IOException e) {
logger.error(" Cannot load application.properties", e);
System.exit(0);
}
waitTime = Integer.parseInt(prop.getProperty("waitTime").trim());
logger.info(" waitTime: " + waitTime);
numberOfPagesToCrawlInALoop = Integer.parseInt(prop.getProperty("numberOfPagesToCrawlInALoop").trim());
logger.info(" numberOfPagesToCrawl: " + numberOfPagesToCrawlInALoop);
totalNumberOfPagesToCrawlInASession = Integer.parseInt(prop.getProperty("totalNumberOfPagesToCrawlInASession").trim());
logger.info(" totalNumberOfPagesToCrawlInASession: " + totalNumberOfPagesToCrawlInASession);
scrapeVersion = Integer.parseInt(prop.getProperty("scrapeVersion").trim());
logger.info(" scrapeVersion: " + scrapeVersion);
logger.info("\n\n\n");
}
public String getSitemapUrl() {
return sitemapUrl;
}
public String getSitemapURLKey() {
return sitemapURLKey;
}
private String getId(String pageUrl) {
String[] parts = pageUrl.split("/");
return parts[parts.length - 1];
}
}

98
apps/bioschemas-api/src/main/java/eu/dnetlib/bioschemas/api/controller/BioschemasAPIController.java Normal file
View File

@ -0,0 +1,98 @@
package eu.dnetlib.bioschemas.api.controller;
import eu.dnetlib.bioschemas.api.MainApplication;
import eu.dnetlib.bioschemas.api.scraper.ScrapingExecution;
import eu.dnetlib.bioschemas.api.scraper.ScrapingExecutor;
import eu.dnetlib.bioschemas.api.utils.BioschemasException;
import eu.dnetlib.common.controller.AbstractDnetController;
import io.swagger.v3.oas.annotations.Operation;
import io.swagger.v3.oas.annotations.Parameter;
import io.swagger.v3.oas.annotations.enums.ParameterIn;
import io.swagger.v3.oas.annotations.tags.Tag;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.LineIterator;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.web.bind.annotation.*;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import java.io.File;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
/**
* @author enrico.ottonello
*
*/
@RestController
@RequestMapping("/api")
@Tag(name = MainApplication.BIOSCHEMAS_APIS)
public class BioschemasAPIController extends AbstractDnetController {
@Value("${outputFolder}")
private String outputFolder;
@Value("${outputDataPattern}")
private String outputDataPattern;
private static Logger logger = LoggerFactory.getLogger(BioschemasAPIController.class);
@Autowired
private ScrapingExecutor scrapingExecutor;
private static final Log log = LogFactory.getLog(BioschemasAPIController.class);
@Operation(summary = "start the scraping operation", description = "<H1>Working input values are in the following table</H1><BR><TABLE><TR><TH>datasourceKey</TH><TH>sitemapUrl</TH></TR><TR><TD>ped</TD><TD>https://proteinensemble.org/sitemap2.xml.gz</TD></TR><TR><TD>disprot</TD><TD>https://disprot.org/sitemap2.xml.gz</TD></TR><TR><TD>mobidb</TD><TD>https://mobidb.org/sitemap2.xml.gz</TD></TR></TABLE>")
@GetMapping("/startScraping")
public ScrapingExecution startScraping(@Parameter(name = "datasourceKey") @RequestParam final String datasourceKey,
@Parameter(name = "sitemapUrl") @RequestParam final String sitemapUrl,
final HttpServletRequest req) {
logger.info("<STARTSCRAPING> datasourceKey: "+datasourceKey+" sitemapUrl:"+sitemapUrl);
return scrapingExecutor.startScraping(datasourceKey, sitemapUrl, getOutputDataPattern(), req.getRemoteAddr(), getOutputFolder());
}
@Operation(summary = "check the status of last scraping operation")
@GetMapping("/startScraping/status")
public final ScrapingExecution statusScraping() {
return scrapingExecutor.getLastScrapingExecution();
}
@Operation(summary = "retrieve the nquads downloaded for one specific provider")
@RequestMapping(value = "/getNQuads", method = RequestMethod.GET)
public String getNQuads(@Parameter(name = "datasourceKey") @RequestParam final String datasourceKey, HttpServletResponse response) throws BioschemasException, IOException {
logger.info("<GETNQUADS> datasourceKey: "+datasourceKey);
LineIterator it = FileUtils.lineIterator(new File(getOutputFolder().concat("/").concat(datasourceKey).concat(getOutputDataPattern())), "UTF-8");
try {
while (it.hasNext()) {
String line = it.nextLine();
response.getOutputStream().write(line.getBytes(StandardCharsets.UTF_8));
response.getOutputStream().println();
}
} finally {
}
return "";
}
public String getOutputFolder() {
return outputFolder;
}
public String getOutputDataPattern() {
return outputDataPattern;
}
public void setOutputFolder(String outputFolder) {
this.outputFolder = outputFolder;
}
public void setOutputDataPattern(String outputDataPattern) {
this.outputDataPattern = outputDataPattern;
}
}

17
apps/bioschemas-api/src/main/java/eu/dnetlib/bioschemas/api/controller/HomeController.java Normal file
View File

@ -0,0 +1,17 @@
package eu.dnetlib.bioschemas.api.controller;
import eu.dnetlib.common.controller.AbstractDnetController;
import org.springframework.stereotype.Controller;
import org.springframework.web.bind.annotation.GetMapping;
@Controller
public class HomeController extends AbstractDnetController {
@GetMapping({
"/doc", "/swagger"
})
public String apiDoc() {
return "redirect:swagger-ui/index.html";
}
}

136
apps/bioschemas-api/src/main/java/eu/dnetlib/bioschemas/api/crawl/CrawlRecord.java Normal file
View File

@ -0,0 +1,136 @@
package eu.dnetlib.bioschemas.api.crawl;
import java.util.Date;
import hwu.elixir.utils.Validation;
/**
*
* Store the current status of a single URL in the scrape service.
*
*
*/
public class CrawlRecord {
private Long id;
private String context = "";
private String url;
private Date dateScraped;
private StatusOfScrape status;
private boolean beingScraped;
private String name;
private String nquads;
public CrawlRecord() {
status = StatusOfScrape.UNTRIED;
}
public CrawlRecord(String url) {
Validation validation = new Validation();
if(validation.validateURI(url)) {
this.url = url;
context = "";
status = StatusOfScrape.UNTRIED;
dateScraped = null;
} else {
throw new IllegalArgumentException(url +" is not a valid url");
}
this.setId(System.currentTimeMillis());
}
public Long getId() {
return id;
}
public void setId(Long id) {
this.id = id;
}
public String getUrl() {
return url;
}
public Date getDateScraped() {
return dateScraped;
}
public void setDateScraped(Date dateScraped) {
this.dateScraped = dateScraped;
}
public StatusOfScrape getStatus() {
return status;
}
public void setStatus(StatusOfScrape status) {
this.status = status;
}
public String getContext() {
return context;
}
public void setContext(String context) {
this.context = context;
}
public boolean isBeingScraped() {
return beingScraped;
}
public void setBeingScraped(boolean beingScraped) {
this.beingScraped = beingScraped;
}
public String getName() {
return name;
}
public void setName(String name) {
this.name = name;
}
public String getNquads() {
return nquads;
}
public void setNquads(String nquads) {
this.nquads = nquads;
}
@Override
public boolean equals(Object o) {
if (this == o)
return true;
if (!(o instanceof CrawlRecord))
return false;
CrawlRecord otherCrawl = (CrawlRecord) o;
if(this.url.equals(otherCrawl.getUrl())) {
return true;
}
return false;
}
@Override
public int hashCode() {
int result = getId() != null ? getId().hashCode() : 0;
result = 31 * result + (getUrl() != null ? getUrl().hashCode() : 0);
result = 31 * result + (getContext() != null ? getContext().hashCode() : 0);
result = 31 * result + (getDateScraped() != null ? getDateScraped().hashCode() : 0);
return result;
}
}

19
apps/bioschemas-api/src/main/java/eu/dnetlib/bioschemas/api/crawl/StatusOfScrape.java Normal file
View File

@ -0,0 +1,19 @@
package eu.dnetlib.bioschemas.api.crawl;
/**
*
* {@link StatusOfScrape} describes the possible status levels the scrape for each URL/CrawlRecord.
*
* Each URL/CrawlRecord can have one of the following:
* DOES_NOT_EXIST = 404.
* HUMAN_INSPECTION = cannot parse for some reason; a human should see what is happening.
* UNTRIED = not scraped yet.
* FAILED = one failed attempt at scraping; will try again.
* GIVEN_UP = two failed attempts at scraping. Will not try again.
* SUCCESS = successfully scraped.
*
*/
public enum StatusOfScrape {
DOES_NOT_EXIST, HUMAN_INSPECTION, UNTRIED, FAILED, GIVEN_UP, SUCCESS;
}

87
apps/bioschemas-api/src/main/java/eu/dnetlib/bioschemas/api/scraper/BMUSEScraper.java Normal file
View File

@ -0,0 +1,87 @@
package eu.dnetlib.bioschemas.api.scraper;
import hwu.elixir.scrape.exceptions.MissingMarkupException;
import hwu.elixir.scrape.scraper.ScraperFilteredCore;
import org.apache.any23.Any23;
import org.apache.any23.extractor.ExtractionException;
import org.apache.any23.source.DocumentSource;
import org.apache.any23.source.StringDocumentSource;
import org.apache.any23.writer.NTriplesWriter;
import org.apache.any23.writer.TripleHandler;
import org.apache.any23.writer.TripleHandlerException;
import org.apache.commons.io.output.ByteArrayOutputStream;
import org.eclipse.rdf4j.model.IRI;
import org.eclipse.rdf4j.model.Model;
import org.eclipse.rdf4j.model.impl.SimpleValueFactory;
import org.eclipse.rdf4j.rio.RDFFormat;
import org.eclipse.rdf4j.rio.Rio;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.io.StringWriter;
public class BMUSEScraper extends ScraperFilteredCore {
private static Logger logger = LoggerFactory.getLogger(BMUSEScraper.class);
public String getNQUADSFromUrl(String url, Boolean dynamic) throws Exception {
logger.debug(url + " > scraping");
url = fixURL(url);
String html = "";
// The dynamic boolean determines if the scraper should start using selenium or JSOUP to scrape the information
// (dynamic and static respectively)
if (dynamic) {
html = wrapHTMLExtraction(url);
} else {
html = wrapHTMLExtractionStatic(url);
}
if (html == null || html.contentEquals(""))
throw new Exception("empty html");
html = injectId(html, url);
logger.debug(url + " > html scraped from " + url);
DocumentSource source = new StringDocumentSource(html, url);
String n3 = html2Triples(source, url);
if (n3 == null) {
throw new MissingMarkupException(url);
}
logger.debug(url + " > processing triples");
IRI sourceIRI = SimpleValueFactory.getInstance().createIRI(source.getDocumentIRI());
Model updatedModel = updatedModel = processTriples(n3, sourceIRI, 0l);
if (updatedModel == null) {
throw new Exception("rdf model null");
}
logger.debug(url + " > generating nquads");
try (StringWriter jsonLDWriter = new StringWriter()) {
Rio.write(updatedModel, jsonLDWriter, RDFFormat.NQUADS);
logger.debug(url + " > nquads generated");
return jsonLDWriter.toString();
} catch (Exception e) {
throw e;
}
}
private String html2Triples(DocumentSource source, String url) throws Exception {
Any23 runner = new Any23();
try (ByteArrayOutputStream out = new ByteArrayOutputStream();
TripleHandler handler = new NTriplesWriter(out);) {
runner.extract(source, handler);
return out.toString("UTF-8");
} catch (ExtractionException e) {
logger.error("Cannot extract triples", e);
} catch (IOException e1) {
logger.error(" IO error whilst extracting triples", e1);
} catch (TripleHandlerException e2) {
logger.error("TripleHanderException", e2);
}
return null;
}
}

157
apps/bioschemas-api/src/main/java/eu/dnetlib/bioschemas/api/scraper/ScrapeState.java Normal file
View File

@ -0,0 +1,157 @@
package eu.dnetlib.bioschemas.api.scraper;
import eu.dnetlib.bioschemas.api.crawl.StatusOfScrape;
import eu.dnetlib.bioschemas.api.crawl.CrawlRecord;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
/**
*/
public class ScrapeState {
private List<CrawlRecord> urlsToScrape = Collections.synchronizedList(new ArrayList<CrawlRecord>());
private List<CrawlRecord> urlsProcessed = Collections.synchronizedList(new ArrayList<CrawlRecord>()); // should this be a set?
private Map<String, Object> nquadsConcurrentHashMap = new ConcurrentHashMap<>();
/**
*
* @param pagesToBeScraped The list of sites to be scraped
* @see ScrapeThread
* @see CrawlRecord
*/
public ScrapeState(List<CrawlRecord> pagesToBeScraped) {
urlsToScrape.addAll(pagesToBeScraped);
}
/**
* Any pages/URLs left to scrape?
* @return True for yes & false for no
* @see CrawlRecord
*/
public synchronized boolean pagesLeftToScrape() {
return !urlsToScrape.isEmpty();
}
/**
* Returns the next URL/CrawlRecord to be scraped
*
* @return First page/URL that needs to be scraped next
* @see CrawlRecord
*/
public synchronized CrawlRecord getURLToProcess() {
if (urlsToScrape.isEmpty())
return null;
return urlsToScrape.remove(0);
}
/**
* Adds the given CrawlRecord to the list of CrawlRecords successfully scraped.
* Updates the status of the CrawlRecord to SUCCESS.
*
* @param url The latest URL/page that has been successfully scraped
* @see CrawlRecord
*/
public synchronized void addSuccessfulScrapedURL(CrawlRecord record) {
record.setStatus(StatusOfScrape.SUCCESS);
urlsProcessed.add(record);
}
/**
* Adds the given CrawlRecord to the list of CrawlRecords NOT successfully scraped.
* Updates the status of the CrawlRecord; if first failure the status is FAILED.
* If status is already FAILED it is changed to GIVEN_UP.
*
* If the status is FAILED, another try will be made in a future run.
*
*
* @param url The latest URL/page that has been unsuccessfully scraped
* @see CrawlRecord
*/
public synchronized void addFailedToScrapeURL(CrawlRecord record) {
if (record.getStatus().equals(StatusOfScrape.FAILED)) {
record.setStatus(StatusOfScrape.GIVEN_UP);
} else {
record.setStatus(StatusOfScrape.FAILED);
}
urlsProcessed.add(record);
}
/**
* Changes the status of the CrawlRecord to DOES_NOT_EXIST.
* As Selenium does not return the HTTP codes, it is questionable
* how useful this is.
*
*
* @param url The latest URL/page that has been 404'd
* @see CrawlRecord
*/
public synchronized void setStatusTo404(CrawlRecord record) {
record.setStatus(StatusOfScrape.DOES_NOT_EXIST);
urlsProcessed.add(record);
}
/**
*
* Changes the status of the CrawlRecord to HUMAN_INSPECTION.
* This captures the idea that the URLs may contain unexpected markup that needs a human to
* review and possibly update the scraper.
*
* @param url The latest URL/page that needs human inspection
* @see CrawlRecord
*/
public synchronized void setStatusToHumanInspection(CrawlRecord record) {
record.setStatus(StatusOfScrape.HUMAN_INSPECTION);
urlsProcessed.add(record);
}
/**
* Returns the number of URLs that are still to be scraped in this cycle.
* This does not return the number of URLs left to scrape in the DBMS, just in the current cycle.
*
* @return Number of URLs left to scrape in this cycle
* @see CrawlRecord
*/
public synchronized int getNumberPagesLeftToScrape() {
return urlsToScrape.size();
}
/**
* Gets the full list of URLs that have been processed in this cycle.
* This does not return the number of URLs that have been scraped in total across all cycles.
*
* @return
* @see CrawlRecord
*/
public synchronized List<CrawlRecord> getPagesProcessed() {
return urlsProcessed;
}
/**
* Gets the full list of URLs/CrawlRecords regardless of whether scraped or not in the current cycle.
*
* @return List of all CrawlRecords in this cycle.
* @see CrawlRecord
*/
public synchronized List<CrawlRecord> getPagesProcessedAndUnprocessed() {
List<CrawlRecord> urlsCombined = Collections.synchronizedList(new ArrayList<CrawlRecord>());
urlsCombined.addAll(urlsProcessed);
urlsCombined.addAll(urlsToScrape);
return urlsCombined;
}
public void addNquads(String key, String nquads) {
nquadsConcurrentHashMap.putIfAbsent(key, nquads);
}
public Map<String, Object> getNquadsConcurrentHashMap() {
return nquadsConcurrentHashMap;
}
}

103
apps/bioschemas-api/src/main/java/eu/dnetlib/bioschemas/api/scraper/ScrapeThread.java Normal file
View File

@ -0,0 +1,103 @@
package eu.dnetlib.bioschemas.api.scraper;
import eu.dnetlib.bioschemas.api.crawl.CrawlRecord;
import eu.dnetlib.bioschemas.api.utils.CompressorUtil;
import hwu.elixir.scrape.exceptions.CannotWriteException;
import hwu.elixir.scrape.exceptions.FourZeroFourException;
import hwu.elixir.scrape.exceptions.JsonLDInspectionException;
import hwu.elixir.scrape.exceptions.MissingMarkupException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.Date;
/**
*
* @see BMUSEScraper
* @see ScrapeState
*
*/
public class ScrapeThread extends Thread {
private ScrapeState scrapeState;
private BMUSEScraper process;
private int waitTime;
private boolean fileWritten = true;
private int scrapeVersion = 1;
private static Logger logger = LoggerFactory.getLogger(ScrapeThread.class);
/**
* Sets up a thread for actually scrapping.
*
* @param scraper Scraper that will actually do the scraping.
* @param scrapeState Object that maintains state across threads.
* @param waitTime How long (in seconds) thread should wait after scraping
* page before attempting new page.
* @param contextVersion The context URL used is 'https://bioschemas.org/crawl/CONTEXTVERSION/ID' Where ID is the id of the CrawlRecord pulled.
*
*/
public ScrapeThread(BMUSEScraper scraper, ScrapeState scrapeState, int waitTime, int contextVersion) {
this.scrapeState = scrapeState;
process = scraper;
this.waitTime = waitTime;
this.scrapeVersion = contextVersion;
}
@Override
/**
* Defines high-level process of scraping. Actual scraping done by an
* implementation of Scraper. If page scrape successful will add url to
* Scrape.sitesScraped
*
* @see Scraper
* @see SimpleScraper
*/
public void run() {
while (scrapeState.pagesLeftToScrape()) {
CrawlRecord record = scrapeState.getURLToProcess();
if (record == null)
break;
record.setContext("https://bioschemas.org/crawl/" + scrapeVersion +"/" + record.getId());
record.setDateScraped(new Date());
try {
String nquads = process.getNQUADSFromUrl(record.getUrl(), true);
logger.info("downloaded "+record.getUrl() + " leftToScrape:" + scrapeState.getNumberPagesLeftToScrape());
record.setNquads(CompressorUtil.compressValue(nquads));
if (!nquads.isEmpty()) {
scrapeState.addSuccessfulScrapedURL(record);
} else {
scrapeState.addFailedToScrapeURL(record);
}
} catch(FourZeroFourException fourZeroFourException) {
scrapeState.setStatusTo404(record);
fileWritten = false;
} catch (JsonLDInspectionException je) {
scrapeState.setStatusToHumanInspection(record);
fileWritten = false;
} catch (CannotWriteException cannotWrite) {
logger.error("Caught cannot read file, setting worked to false!");
fileWritten = false;
scrapeState.addFailedToScrapeURL(record);
return; // no point in continuing
} catch (MissingMarkupException e) {
logger.error("Cannot obtain markup from " + record.getUrl() +".");
fileWritten = false;
scrapeState.addFailedToScrapeURL(record);
} catch (Exception e) {
e.printStackTrace();
}
try {
ScrapeThread.sleep(100 * waitTime);
} catch (InterruptedException e) {
e.printStackTrace();
}
}
process.shutdown();
}
public boolean isFileWritten() {
return fileWritten;
}
}

99
apps/bioschemas-api/src/main/java/eu/dnetlib/bioschemas/api/scraper/ScrapingExecution.java Normal file
View File

@ -0,0 +1,99 @@
package eu.dnetlib.bioschemas.api.scraper;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import java.util.Date;
import java.util.UUID;
import java.util.concurrent.TimeUnit;
public class ScrapingExecution {
private String id;
private Long dateStart;
private Long dateEnd;
private ScrapingStatus status = ScrapingStatus.NOT_YET_STARTED;
private String message;
private static final Log log = LogFactory.getLog(ScrapingExecution.class);
public ScrapingExecution() {}
public ScrapingExecution(final String id, final Long dateStart, final Long dateEnd, final ScrapingStatus status, final String message) {
this.id = id;
this.dateStart = dateStart;
this.dateEnd = dateEnd;
this.status = status;
this.message = message;
}
public String getId() {
return id;
}
public void setId(final String id) {
this.id = id;
}
public Long getDateStart() {
return dateStart;
}
public void setDateStart(final Long dateStart) {
this.dateStart = dateStart;
}
public Long getDateEnd() {
return dateEnd;
}
public void setDateEnd(final Long dateEnd) {
this.dateEnd = dateEnd;
}
public ScrapingStatus getStatus() {
return status;
}
public void setStatus(final ScrapingStatus status) {
this.status = status;
}
public String getMessage() {
return message;
}
public void setMessage(final String message) {
this.message = message;
}
public void startNew(final String message) {
setId("scraping-" + UUID.randomUUID());
setDateStart(System.currentTimeMillis());
setDateEnd(null);
setStatus(ScrapingStatus.RUNNING);
setMessage(message);
log.info(message);
}
public void complete() {
setDateEnd(System.currentTimeMillis());
setStatus(ScrapingStatus.SUCCESS);
final long millis = getDateEnd() - getDateStart();
setMessage(String
.format("Scraping completed in %d min, %d sec", TimeUnit.MILLISECONDS.toMinutes(millis), TimeUnit.MILLISECONDS.toSeconds(millis) -
TimeUnit.MINUTES.toSeconds(TimeUnit.MILLISECONDS.toMinutes(millis))));
log.info(getMessage());
}
public void fail(final Throwable e) {
setDateEnd(new Date().getTime());
setStatus(ScrapingStatus.FAILED);
setMessage(e.getMessage());
log.error("Error scraping", e);
}
}

40
apps/bioschemas-api/src/main/java/eu/dnetlib/bioschemas/api/scraper/ScrapingExecutor.java Normal file
View File

@ -0,0 +1,40 @@
package eu.dnetlib.bioschemas.api.scraper;
import eu.dnetlib.bioschemas.api.ServiceScrapeDriver;
import org.springframework.stereotype.Component;
import javax.servlet.http.HttpServletRequest;
@Component
public class ScrapingExecutor {
private final ScrapingExecution lastScrapingExecution = new ScrapingExecution();
public ScrapingExecution getLastScrapingExecution() {
return lastScrapingExecution;
}
public ScrapingExecution startScraping(final String datasourceKey, final String sitemapUrl, final String outputDataPattern, final String remoteAddr, final String outputFolder) {
synchronized (lastScrapingExecution) {
if (lastScrapingExecution.getStatus() != ScrapingStatus.RUNNING) {
lastScrapingExecution.startNew("Scraping for " + datasourceKey + " " + sitemapUrl + " - request from " + remoteAddr);
new Thread(() -> {
try {
String sitemapUrlKey = "loc";
String outputFilename = datasourceKey.concat(outputDataPattern);
ServiceScrapeDriver service = new ServiceScrapeDriver(sitemapUrl, sitemapUrlKey, null, outputFilename, outputFolder);
service.runScrape();
lastScrapingExecution.complete();
} catch (final Throwable e) {
lastScrapingExecution.fail(e);
}
}).start();
} else {
final long now = System.currentTimeMillis();
return new ScrapingExecution(null, now, now, ScrapingStatus.NOT_LAUNCHED, "An other scraping is running");
}
}
return lastScrapingExecution;
}
}

9
apps/bioschemas-api/src/main/java/eu/dnetlib/bioschemas/api/scraper/ScrapingStatus.java Normal file
View File

@ -0,0 +1,9 @@
package eu.dnetlib.bioschemas.api.scraper;
public enum ScrapingStatus {
SUCCESS,
FAILED,
RUNNING,
NOT_LAUNCHED,
NOT_YET_STARTED
}

71
apps/bioschemas-api/src/main/java/eu/dnetlib/bioschemas/api/scraper/ServiceScraper.java Normal file
View File

@ -0,0 +1,71 @@
package eu.dnetlib.bioschemas.api.scraper;
import eu.dnetlib.bioschemas.api.crawl.StatusOfScrape;
import hwu.elixir.scrape.exceptions.*;
import hwu.elixir.scrape.scraper.ScraperFilteredCore;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* Provides the
* actual scraping functionality.
*
* Scrapes a given URL, converts into NQuads and writes to a file (name derived
* from URL). If the file already exists it will be overwritten.
*
*
* @see ScraperFilteredCore
*
*/
public class ServiceScraper extends ScraperFilteredCore {
private static Logger logger = LoggerFactory.getLogger(ServiceScraper.class);
private StatusOfScrape status= null;
/**
* Orchestrates the process of scraping a site before converting the extracted
* triples to NQuads and writing to a file.
*
* @param url Site to be scraped
* @param contextCounter Number used to generate the named graph/context and
* the URLs used to replace blank nodes.
* @param outputFolderName Location to which the NQuads will be written
* @return True if success; false otherwise
* @throws FourZeroFourException
* @throws JsonLDInspectionException
* @throws CannotWriteException
* @throws MissingMarkupException
*
*/
public boolean scrape(String url, Long contextCounter, String outputFolderName, String fileName, StatusOfScrape status) throws FourZeroFourException, JsonLDInspectionException, CannotWriteException, MissingMarkupException {
this.status = status;
logger.info("scraping "+url + " to "+fileName);
return scrape(url, outputFolderName, fileName, contextCounter, true);
}
@Override
/* Now takes account of StateOfCrawl
*/
protected String wrapHTMLExtraction(String url) throws FourZeroFourException {
String html = "";
if (status.equals(StatusOfScrape.UNTRIED) || status.equals(StatusOfScrape.FAILED)) {
try {
html = getHtmlViaSelenium(url);
} catch (SeleniumException e) {
// try again
try {
html = getHtmlViaSelenium(url);
} catch (SeleniumException e2) {
return "";
}
}
} else {
return "";
}
return html;
}
}

28
apps/bioschemas-api/src/main/java/eu/dnetlib/bioschemas/api/utils/BioschemasException.java Normal file
View File

@ -0,0 +1,28 @@
package eu.dnetlib.bioschemas.api.utils;
/**
* @author enrico.ottonello
*
*/
public class BioschemasException extends Exception{
public BioschemasException() {
}
public BioschemasException(final String message) {
super(message);
}
public BioschemasException(final String message, final Throwable cause) {
super(message, cause);
}
public BioschemasException(final Throwable cause) {
super(cause);
}
public BioschemasException(final String message, final Throwable cause, final boolean enableSuppression, final boolean writableStackTrace) {
super(message, cause, enableSuppression, writableStackTrace);
}
}

34
apps/bioschemas-api/src/main/java/eu/dnetlib/bioschemas/api/utils/CompressorUtil.java Normal file
View File

@ -0,0 +1,34 @@
package eu.dnetlib.bioschemas.api.utils;
import org.apache.commons.codec.binary.Base64;
import org.apache.commons.io.IOUtils;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.StringWriter;
import java.util.zip.GZIPInputStream;
import java.util.zip.GZIPOutputStream;
public class CompressorUtil {
public static String decompressValue(final String abstractCompressed) {
try {
byte[] byteArray = Base64.decodeBase64(abstractCompressed.getBytes());
GZIPInputStream gis = new GZIPInputStream(new ByteArrayInputStream(byteArray));
final StringWriter stringWriter = new StringWriter();
IOUtils.copy(gis, stringWriter);
return stringWriter.toString();
} catch (IOException e) {
throw new IllegalArgumentException(e);
}
}
public static String compressValue(final String value) throws IOException {
ByteArrayOutputStream out = new ByteArrayOutputStream();
GZIPOutputStream gzip = new GZIPOutputStream(out);
gzip.write(value.getBytes());
gzip.close();
return java.util.Base64.getEncoder().encodeToString(out.toByteArray());
}
}

64
apps/bioschemas-api/src/main/java/eu/dnetlib/bioschemas/api/utils/UrlParser.java Normal file
View File

@ -0,0 +1,64 @@
package eu.dnetlib.bioschemas.api.utils;
import hwu.elixir.utils.Helpers;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.select.Elements;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
public class UrlParser {
private static final Logger logger = LoggerFactory.getLogger(UrlParser.class.getName());
public static Elements getSitemapList(String url, String sitemapURLKey) throws IOException {
Document doc = new Document(url);
Document urlSitemapListsNested;
Elements elements = new Elements();
Elements sitemaps = new Elements();
boolean sitemapindex = false;
boolean urlset = false;
try {
int urlLength = url.length();
logger.info("parse sitemap list");
String sitemapExt = url.substring(urlLength - 3, urlLength);
if (sitemapExt.equalsIgnoreCase(".gz")) { // this checks only the extension at the ending
logger.info("compressed sitemap");
byte[] bytes = Jsoup.connect(url).ignoreContentType(true).execute().bodyAsBytes();
doc = Helpers.gzipFileDecompression(bytes);
} else {
doc = Jsoup.connect(url).maxBodySize(0).get();
}
} catch (IOException e) {
logger.error("Jsoup parsing exception: " + e.getMessage());
}
try {
elements = doc.select(sitemapURLKey);
// check the html if it is a sitemapindex or a urlset
sitemapindex = doc.outerHtml().contains("sitemapindex");
urlset = doc.outerHtml().contains("urlset");
} catch (NullPointerException e) {
logger.error(e.getMessage());
}
if (sitemapindex) {
// if sitemapindex get the loc of all the sitemaps
// added warning for sitemap index files
logger
.warn(
"please note this is a sitemapindex file which is not currently supported, please use the content (url) of the urlset instead");
sitemaps = doc.select(sitemapURLKey);
}
return elements;
}
}

27
apps/bioschemas-api/src/main/resources/application.properties Normal file
View File

@ -0,0 +1,27 @@
server.servlet.context-path=/bioschemas-api
server.port=8281
server.public_url = http://localhost:8281/bioschemas-api
server.public_desc = API Base URL
spring.profiles.active=garr
logging.file.name = /var/log/bioschemas-api/bioschemas.log
maven.pom.path = /META-INF/maven/eu.dnetlib.dhp/bioschemas-api/effective-pom.xml
spring.main.banner-mode = off
logging.level.root = INFO
management.endpoints.web.exposure.include = prometheus,health
management.endpoints.web.base-path = /
management.endpoints.web.path-mapping.prometheus = metrics
management.endpoints.web.path-mapping.health = health
waitTime=5
outputFolder=/data/bioschemas-harvest
outputDataPattern=_base64_gzipped_nquads.txt
numberOfPagesToCrawlInALoop=8
totalNumberOfPagesToCrawlInASession=32
scrapeVersion=1

37
apps/dhp-broker-application/pom.xml
View File

@ -4,8 +4,8 @@
<parent>
<groupId>eu.dnetlib.dhp</groupId>
<artifactId>apps</artifactId>
<version>3.2.2-SNAPSHOT</version>
<relativePath>../</relativePath>
<version>3.3.3-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
<modelVersion>4.0.0</modelVersion>
@ -18,14 +18,6 @@
<dependencies>
<!-- Mail -->
<dependency>
<groupId>javax.mail</groupId>
<artifactId>mail</artifactId>
<version>1.4.7</version>
</dependency>
<!-- Openaire -->
<dependency>
<groupId>eu.dnetlib.dhp</groupId>
@ -33,6 +25,31 @@
<version>${project.version}</version>
</dependency>
<!-- Tests -->
<dependency>
<groupId>org.junit.jupiter</groupId>
<artifactId>junit-jupiter</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-core</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-junit-jupiter</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-test</artifactId>
<scope>test</scope>
</dependency>
</dependencies>
<build>

51
apps/dhp-broker-application/src/main/java/eu/dnetlib/broker/LiteratureBrokerServiceApplication.java
View File

@ -1,14 +1,15 @@
package eu.dnetlib.broker;
import java.util.ArrayList;
import java.util.List;
import org.springdoc.core.GroupedOpenApi;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.context.annotation.Bean;
import eu.dnetlib.common.app.AbstractDnetApp;
import springfox.documentation.builders.ApiInfoBuilder;
import springfox.documentation.builders.RequestHandlerSelectors;
import springfox.documentation.service.ApiInfo;
import springfox.documentation.service.Tag;
import springfox.documentation.spring.web.plugins.Docket;
import io.swagger.v3.oas.models.tags.Tag;
@SpringBootApplication
public class LiteratureBrokerServiceApplication extends AbstractDnetApp {
@ -24,22 +25,28 @@ public class LiteratureBrokerServiceApplication extends AbstractDnetApp {
SpringApplication.run(LiteratureBrokerServiceApplication.class, args);
}
@Override
protected void configSwagger(final Docket docket) {
docket.select()
.apis(RequestHandlerSelectors.any())
.paths(p -> p.startsWith("/api/"))
.build()
.tags(new Tag(TAG_EVENTS, "Events management"), new Tag(TAG_SUBSCRIPTIONS, "Subscriptions management"), new Tag(TAG_NOTIFICATIONS,
"Notifications management"), new Tag(TAG_TOPIC_TYPES, "Topic types management"), new Tag(TAG_OPENAIRE, "OpenAIRE use case"))
.apiInfo(new ApiInfoBuilder()
.title("Literature Broker Service")
.description("APIs documentation")
.version("1.1")
.contact(ApiInfo.DEFAULT_CONTACT)
.license("Apache 2.0")
.licenseUrl("http://www.apache.org/licenses/LICENSE-2.0")
.build());
@Bean
public GroupedOpenApi publicApi() {
return GroupedOpenApi.builder()
.group("Broker APIs")
.pathsToMatch("/api/**")
.build();
}
@Override
protected String swaggerTitle() {
return "OpenAIRE Broker API";
}
@Override
protected List<Tag> swaggerTags() {
final List<Tag> tags = new ArrayList<>();
tags.add(new Tag().name(TAG_EVENTS).description("Events management"));
tags.add(new Tag().name(TAG_SUBSCRIPTIONS).description("Subscriptions management"));
tags.add(new Tag().name(TAG_NOTIFICATIONS).description("Notifications management"));
tags.add(new Tag().name(TAG_TOPIC_TYPES).description("Topic types management"));
tags.add(new Tag().name(TAG_OPENAIRE).description("OpenAIRE use case"));
return tags;
}
}

4
apps/dhp-broker-application/src/main/java/eu/dnetlib/broker/LiteratureBrokerServiceConfiguration.java
View File

@ -4,6 +4,7 @@ import org.elasticsearch.client.RestHighLevelClient;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.cache.annotation.EnableCaching;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.ComponentScan;
import org.springframework.context.annotation.Configuration;
import org.springframework.data.elasticsearch.client.ClientConfiguration;
import org.springframework.data.elasticsearch.client.RestClients;
@ -15,16 +16,15 @@ import org.springframework.transaction.annotation.EnableTransactionManagement;
import eu.dnetlib.broker.common.elasticsearch.Event;
import eu.dnetlib.broker.common.elasticsearch.Notification;
import eu.dnetlib.broker.common.properties.ElasticSearchProperties;
import springfox.documentation.swagger2.annotations.EnableSwagger2;
@Configuration
@EnableSwagger2
@EnableCaching
@EnableScheduling
@EnableTransactionManagement
@EnableElasticsearchRepositories(basePackageClasses = {
Event.class, Notification.class
})
@ComponentScan(basePackages = "eu.dnetlib")
public class LiteratureBrokerServiceConfiguration extends AbstractElasticsearchConfiguration {
@Autowired

2
apps/dhp-broker-application/src/main/java/eu/dnetlib/broker/controllers/ApiDocController.java
View File

@ -8,6 +8,6 @@ public class ApiDocController {
@GetMapping({ "/apidoc", "/api-doc", "/doc", "/swagger" })
public String apiDoc() {
return "redirect:swagger-ui/";
return "redirect:swagger-ui/index.html";
}
}

28
apps/dhp-broker-application/src/main/java/eu/dnetlib/broker/controllers/EventsController.java
View File

@ -31,12 +31,12 @@ import eu.dnetlib.broker.common.elasticsearch.EventStatsManager.BrowseEntry;
import eu.dnetlib.broker.common.subscriptions.Subscription;
import eu.dnetlib.broker.common.subscriptions.SubscriptionRepository;
import eu.dnetlib.common.controller.AbstractDnetController;
import io.swagger.annotations.Api;
import io.swagger.annotations.ApiOperation;
import io.swagger.v3.oas.annotations.Operation;
import io.swagger.v3.oas.annotations.tags.Tag;
@RestController
@RequestMapping("/api/events")
@Api(tags = LiteratureBrokerServiceApplication.TAG_EVENTS)
@Tag(name = LiteratureBrokerServiceApplication.TAG_EVENTS)
public class EventsController extends AbstractDnetController {
private static final Log log = LogFactory.getLog(AbstractDnetController.class);
@ -50,25 +50,25 @@ public class EventsController extends AbstractDnetController {
@Autowired
private EventStatsManager eventStatsManager;
@ApiOperation("Return an event by ID")
@Operation(summary = "Return an event by ID")
@GetMapping("/{id}")
public Event getEvent(@PathVariable final String id) {
return eventRepository.findById(id).get();
}
@ApiOperation("Delete an event by ID")
@Operation(summary = "Delete an event by ID")
@DeleteMapping("/{id}")
public void deleteEvent(@PathVariable final String id) {
eventRepository.deleteById(id);
}
@ApiOperation("Save an event by ID")
@Operation(summary = "Save an event by ID")
@PostMapping("/{id}")
public Event saveEvent(@RequestBody final Event event) {
return eventRepository.save(event);
}
@ApiOperation("Return a page of events")
@Operation(summary = "Return a page of events")
@GetMapping("/list/{page}/{pageSize}")
public List<Event> events(
@PathVariable final int page,
@ -76,7 +76,7 @@ public class EventsController extends AbstractDnetController {
return Lists.newArrayList(eventRepository.findAll(PageRequest.of(page, pageSize)));
}
@ApiOperation("Return a page of events by topic")
@Operation(summary = "Return a page of events by topic")
@GetMapping("/byTopic/{page}/{pageSize}")
public List<Event> eventsByTopic(
@PathVariable final int page,
@ -85,7 +85,7 @@ public class EventsController extends AbstractDnetController {
return Lists.newArrayList(eventRepository.findByTopic(topic, PageRequest.of(page, pageSize)));
}
@ApiOperation("Delete all the events")
@Operation(summary = "Delete all the events")
@DeleteMapping("/all")
public Map<String, Object> clearEvents() {
eventRepository.deleteAll();
@ -94,13 +94,13 @@ public class EventsController extends AbstractDnetController {
return res;
}
@ApiOperation("Delete the expired events")
@Operation(summary = "Delete the expired events")
@DeleteMapping("/expired")
public Map<String, Object> deleteExpiredEvents() {
return deleteEventsByExpiryDate(0, new Date().getTime());
}
@ApiOperation("Delete the events with the creationDate in a range")
@Operation(summary = "Delete the events with the creationDate in a range")
@DeleteMapping("/byCreationDate/{from}/{to}")
public Map<String, Long> deleteEventsByCreationDate(@PathVariable final long from, @PathVariable final long to) {
final Map<String, Long> res = new HashMap<>();
@ -113,7 +113,7 @@ public class EventsController extends AbstractDnetController {
return res;
}
@ApiOperation("Delete the events with the expiryDate in a range")
@Operation(summary = "Delete the events with the expiryDate in a range")
@DeleteMapping("/byExpiryDate/{from}/{to}")
public Map<String, Object> deleteEventsByExpiryDate(@PathVariable final long from, @PathVariable final long to) {
new Thread(() -> {
@ -128,13 +128,13 @@ public class EventsController extends AbstractDnetController {
return res;
}
@ApiOperation("Return the topics of the indexed events (all)")
@Operation(summary = "Return the topics of the indexed events (all)")
@GetMapping("/topics/all")
public List<BrowseEntry> browseTopics() {
return eventStatsManager.browseTopics();
}
@ApiOperation("Return the topics of the indexed events (only with subscriptions)")
@Operation(summary = "Return the topics of the indexed events (only with subscriptions)")
@GetMapping("/topics/withSubscriptions")
public List<BrowseEntry> browseTopicsWithSubscriptions() {

17
apps/dhp-broker-application/src/main/java/eu/dnetlib/broker/controllers/NotificationsController.java
View File

@ -14,16 +14,15 @@ import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
import eu.dnetlib.broker.LiteratureBrokerServiceApplication;
import eu.dnetlib.broker.common.elasticsearch.Notification;
import eu.dnetlib.broker.common.elasticsearch.NotificationRepository;
import eu.dnetlib.common.controller.AbstractDnetController;
import io.swagger.annotations.Api;
import io.swagger.annotations.ApiOperation;
import io.swagger.v3.oas.annotations.Operation;
import io.swagger.v3.oas.annotations.tags.Tag;
@RestController
@RequestMapping("/api/notifications")
@Api(tags = LiteratureBrokerServiceApplication.TAG_NOTIFICATIONS)
@Tag(name = "LiteratureBrokerServiceApplication.TAG_NOTIFICATIONS")
public class NotificationsController extends AbstractDnetController {
private static final Log log = LogFactory.getLog(NotificationsController.class);
@ -31,31 +30,31 @@ public class NotificationsController extends AbstractDnetController {
@Autowired
private NotificationRepository notificationRepository;
@ApiOperation("Return a notification by ID")
@Operation(summary = "Return a notification by ID")
@GetMapping("/{id}")
public Notification getNotification(@PathVariable final String id) {
return notificationRepository.findById(id).get();
}
@ApiOperation("Delete a notification by ID")
@Operation(summary = "Delete a notification by ID")
@DeleteMapping("/{id}")
public void deleteNotification(@PathVariable final String id) {
notificationRepository.deleteById(id);
}
@ApiOperation("Save a notification by ID")
@Operation(summary = "Save a notification by ID")
@PostMapping("/{id}")
public Notification saveNotification(@RequestBody final Notification notification) {
return notificationRepository.save(notification);
}
@ApiOperation("Delete all notifications")
@Operation(summary = "Delete all notifications")
@DeleteMapping("")
public void deleteAllNotifications() {
notificationRepository.deleteAll();
}
@ApiOperation("Delete the notifications with the date in a range")
@Operation(summary = "Delete the notifications with the date in a range")
@DeleteMapping("/byDate/{from}/{to}")
public Map<String, Object> deleteNotificationsByDate(@PathVariable final long from, @PathVariable final long to) {
new Thread(() -> {

10
apps/dhp-broker-application/src/main/java/eu/dnetlib/broker/controllers/StartMatchingController.java
View File

@ -16,13 +16,13 @@ import eu.dnetlib.broker.common.subscriptions.Subscription;
import eu.dnetlib.broker.common.subscriptions.SubscriptionRepository;
import eu.dnetlib.broker.matchers.SubscriptionEventMatcher;
import eu.dnetlib.common.controller.AbstractDnetController;
import io.swagger.annotations.Api;
import io.swagger.annotations.ApiOperation;
import io.swagger.v3.oas.annotations.Operation;
import io.swagger.v3.oas.annotations.tags.Tag;
@Profile("!openaire")
@RestController
@RequestMapping("/api/matching")
@Api(tags = LiteratureBrokerServiceApplication.TAG_MATCHING)
@Tag(name = LiteratureBrokerServiceApplication.TAG_MATCHING)
public class StartMatchingController extends AbstractDnetController {
@Autowired
@ -31,7 +31,7 @@ public class StartMatchingController extends AbstractDnetController {
@Autowired(required = false)
private SubscriptionEventMatcher subscriptionEventMatcher;
@ApiOperation("Launch the thread that produces new notifications")
@Operation(summary = "Launch the thread that produces new notifications")
@GetMapping("/start")
public List<String> startMatching() {
if (subscriptionEventMatcher != null) {
@ -42,7 +42,7 @@ public class StartMatchingController extends AbstractDnetController {
}
}
@ApiOperation("Launch the thread that produces new notifications by subscriptuion id")
@Operation(summary = "Launch the thread that produces new notifications by subscriptuion id")
@GetMapping("/start/{subscriptionId}")
public List<String> startMatching(@PathVariable final String subscriptionId) {
final Optional<Subscription> s = subscriptionRepo.findById(subscriptionId);

20
apps/dhp-broker-application/src/main/java/eu/dnetlib/broker/controllers/SubscriptionsController.java
View File

@ -29,12 +29,12 @@ import eu.dnetlib.broker.common.subscriptions.NotificationMode;
import eu.dnetlib.broker.common.subscriptions.Subscription;
import eu.dnetlib.broker.common.subscriptions.SubscriptionRepository;
import eu.dnetlib.common.controller.AbstractDnetController;
import io.swagger.annotations.Api;
import io.swagger.annotations.ApiOperation;
import io.swagger.v3.oas.annotations.Operation;
import io.swagger.v3.oas.annotations.tags.Tag;
@RestController
@RequestMapping("/api/subscriptions")
@Api(tags = LiteratureBrokerServiceApplication.TAG_SUBSCRIPTIONS)
@Tag(name = LiteratureBrokerServiceApplication.TAG_SUBSCRIPTIONS)
public class SubscriptionsController extends AbstractDnetController {
@Autowired
@ -53,26 +53,26 @@ public class SubscriptionsController extends AbstractDnetController {
}
};
@ApiOperation("Return the list of subscriptions")
@Operation(summary = "Return the list of subscriptions")
@GetMapping("")
public Iterable<Subscription> listSubscriptions() {
return subscriptionRepo.findAll();
}
@ApiOperation("Return a subscription by ID")
@Operation(summary = "Return a subscription by ID")
@GetMapping("/{id}")
public Subscription getSubscription(@PathVariable final String id) {
return subscriptionRepo.findById(id).get();
}
@ApiOperation("Delete a subscription by ID and its notifications")
@Operation(summary = "Delete a subscription by ID and its notifications")
@DeleteMapping("/{id}")
public void deleteSubscription(@PathVariable final String id) {
subscriptionRepo.deleteById(id);
notificationRepo.deleteBySubscriptionId(id);
}
@ApiOperation("Perform a new subscription")
@Operation(summary = "Perform a new subscription")
@PostMapping("")
public Subscription registerSubscription(@RequestBody final InSubscription inSub) {
final Subscription sub = inSub.asSubscription();
@ -80,7 +80,7 @@ public class SubscriptionsController extends AbstractDnetController {
return sub;
}
@ApiOperation("Delete all subscriptions and notifications")
@Operation(summary = "Delete all subscriptions and notifications")
@DeleteMapping("")
public Map<String, Object> clearSubscriptions() {
final Map<String, Object> res = new HashMap<>();
@ -90,7 +90,7 @@ public class SubscriptionsController extends AbstractDnetController {
return res;
}
@ApiOperation("Reset the last notification date")
@Operation(summary = "Reset the last notification date")
@DeleteMapping("/{id}/date")
public void deleteNotificationDate(@PathVariable final String id) {
final Subscription s = subscriptionRepo.findById(id).get();
@ -98,7 +98,7 @@ public class SubscriptionsController extends AbstractDnetController {
subscriptionRepo.save(s);
}
@ApiOperation("Reset all the last notification dates")
@Operation(summary = "Reset all the last notification dates")
@GetMapping("/resetLastNotificationDates")
public void deleteAllNotificationDates() {
for (final Subscription s : subscriptionRepo.findAll()) {

16
apps/dhp-broker-application/src/main/java/eu/dnetlib/broker/controllers/TopicsController.java
View File

@ -22,12 +22,12 @@ import eu.dnetlib.broker.LiteratureBrokerServiceApplication;
import eu.dnetlib.broker.common.topics.TopicType;
import eu.dnetlib.broker.common.topics.TopicTypeRepository;
import eu.dnetlib.common.controller.AbstractDnetController;
import io.swagger.annotations.Api;
import io.swagger.annotations.ApiOperation;
import io.swagger.v3.oas.annotations.Operation;
import io.swagger.v3.oas.annotations.tags.Tag;
@RestController
@RequestMapping("/api/topic-types")
@Api(tags = LiteratureBrokerServiceApplication.TAG_TOPIC_TYPES)
@Tag(name = LiteratureBrokerServiceApplication.TAG_TOPIC_TYPES)
public class TopicsController extends AbstractDnetController {
@Autowired
@ -36,13 +36,13 @@ public class TopicsController extends AbstractDnetController {
private final Predicate<String> verifyExpression =
Pattern.compile("^([a-zA-Z0-9._-]+|<[a-zA-Z0-9._-]+>)(\\/([a-zA-Z0-9._-]+|<[a-zA-Z0-9._-]+>))+$").asPredicate();
@ApiOperation("Return the list of topic types")
@Operation(summary = "Return the list of topic types")
@GetMapping("")
public Iterable<TopicType> listTopicTypes() {
return topicTypeRepo.findAll();
}
@ApiOperation("Register a new topic type")
@Operation(summary = "Register a new topic type")
@PostMapping("/add")
public TopicType registerTopicType(@RequestParam final String name,
@RequestParam final String expression,
@ -61,20 +61,20 @@ public class TopicsController extends AbstractDnetController {
return type;
}
@ApiOperation("Return a topic type by ID")
@Operation(summary = "Return a topic type by ID")
@GetMapping("/{id}")
public TopicType getTopicType(@PathVariable final String id) {
return topicTypeRepo.findById(id).get();
}
@ApiOperation("Delete a topic type by ID")
@Operation(summary = "Delete a topic type by ID")
@DeleteMapping("/{id}")
public List<String> deleteTopicType(@PathVariable final String id) {
topicTypeRepo.deleteById(id);
return Arrays.asList("Done.");
}
@ApiOperation("Delete all topic types")
@Operation(summary = "Delete all topic types")
@DeleteMapping("")
public Map<String, Object> clearTopicTypes() {
final Map<String, Object> res = new HashMap<>();

24
apps/dhp-broker-application/src/main/java/eu/dnetlib/broker/openaire/OpenaireBrokerController.java
View File

@ -56,13 +56,13 @@ import eu.dnetlib.broker.common.subscriptions.SubscriptionRepository;
import eu.dnetlib.broker.events.output.DispatcherManager;
import eu.dnetlib.broker.objects.OaBrokerEventPayload;
import eu.dnetlib.common.controller.AbstractDnetController;
import io.swagger.annotations.Api;
import io.swagger.annotations.ApiOperation;
import io.swagger.v3.oas.annotations.Operation;
import io.swagger.v3.oas.annotations.tags.Tag;
@Profile("openaire")
@RestController
@RequestMapping("/api/openaireBroker")
@Api(tags = LiteratureBrokerServiceApplication.TAG_OPENAIRE)
@Tag(name = LiteratureBrokerServiceApplication.TAG_OPENAIRE)
public class OpenaireBrokerController extends AbstractDnetController {
@Autowired
@ -85,7 +85,7 @@ public class OpenaireBrokerController extends AbstractDnetController {
private static final Log log = LogFactory.getLog(OpenaireBrokerController.class);
@ApiOperation("Return the datasources having events")
@Operation(summary = "Return the datasources having events")
@GetMapping("/datasources")
public List<BrowseEntry> findDatasourcesWithEvents(@RequestParam(defaultValue = "false", required = false) final boolean useIndex) {
return useIndex ? findDatasourcesWithEventsUsingIndex() : findDatasourcesWithEventsUsingDb();
@ -123,7 +123,7 @@ public class OpenaireBrokerController extends AbstractDnetController {
}
}
@ApiOperation("Return the topics of the events of a datasource")
@Operation(summary = "Return the topics of the events of a datasource")
@GetMapping("/topicsForDatasource")
public List<BrowseEntry> findTopicsForDatasource(@RequestParam final String ds,
@RequestParam(defaultValue = "false", required = false) final boolean useIndex) {
@ -163,7 +163,7 @@ public class OpenaireBrokerController extends AbstractDnetController {
}
}
@ApiOperation("Return a page of events of a datasource (by topic)")
@Operation(summary = "Return a page of events of a datasource (by topic)")
@GetMapping("/events/{nPage}/{size}")
public EventsPage showEvents(@RequestParam final String ds, @RequestParam final String topic, @PathVariable final int nPage, @PathVariable final int size) {
@ -191,7 +191,7 @@ public class OpenaireBrokerController extends AbstractDnetController {
return new EventsPage(ds, topic, nPage, overrideGetTotalPage(page, size), page.getTotalHits(), list);
}
@ApiOperation("Return a page of events of a datasource (by query)")
@Operation(summary = "Return a page of events of a datasource (by query)")
@PostMapping("/events/{nPage}/{size}")
public EventsPage advancedShowEvents(@PathVariable final int nPage, @PathVariable final int size, @RequestBody final AdvQueryObject qObj) {
@ -227,7 +227,7 @@ public class OpenaireBrokerController extends AbstractDnetController {
return new EventsPage(qObj.getDatasource(), qObj.getTopic(), nPage, overrideGetTotalPage(page, size), page.getTotalHits(), list);
}
@ApiOperation("Perform a subscription")
@Operation(summary = "Perform a subscription")
@PostMapping("/subscribe")
public Subscription registerSubscription(@RequestBody final OpenaireSubscription oSub) {
final Subscription sub = oSub.asSubscription();
@ -237,7 +237,7 @@ public class OpenaireBrokerController extends AbstractDnetController {
return sub;
}
@ApiOperation("Return the subscriptions of an user (by email and datasource (optional))")
@Operation(summary = "Return the subscriptions of an user (by email and datasource (optional))")
@GetMapping("/subscriptions")
public Map<String, List<SimpleSubscriptionDesc>> subscriptions(@RequestParam final String email, @RequestParam(required = false) final String ds) {
final Iterable<Subscription> iter = subscriptionRepo.findBySubscriber(email);
@ -247,7 +247,7 @@ public class OpenaireBrokerController extends AbstractDnetController {
.collect(Collectors.groupingBy(SimpleSubscriptionDesc::getDatasource));
}
@ApiOperation("Return a page of notifications")
@Operation(summary = "Return a page of notifications")
@GetMapping("/notifications/{subscrId}/{nPage}/{size}")
public EventsPage notifications(@PathVariable final String subscrId, @PathVariable final int nPage, @PathVariable final int size) {
@ -279,14 +279,14 @@ public class OpenaireBrokerController extends AbstractDnetController {
}
@ApiOperation("Send notifications")
@Operation(summary = "Send notifications")
@GetMapping("/notifications/send/{date}")
private List<String> sendMailForNotifications(@PathVariable final long date) {
new Thread(() -> innerSendMailForNotifications(date)).start();
return Arrays.asList("Sending ...");
}
@ApiOperation("Update stats")
@Operation(summary = "Update stats")
@GetMapping("/stats/update")
private List<String> updateStats() {
new Thread(() -> {

3
apps/dhp-broker-application/src/main/resources/application.properties
View File

@ -1,5 +1,8 @@
spring.profiles.active = dev,openaire
server.public_url =
server.public_desc = API Base URL
#logging.level.root=DEBUG
maven.pom.path = /META-INF/maven/eu.dnetlib.dhp/dhp-broker-application/effective-pom.xml

2
apps/dhp-broker-application/src/main/resources/static/index.html
View File

@ -27,7 +27,7 @@
<a class="nav-link dropdown-toggle" href="javascript:void(0)" data-toggle="dropdown">Tools <span class="caret"></span></a>
<div class="dropdown-menu dropdown-menu-right">
<a class="dropdown-item" href="{{t.url}}" target="_blank" ng-repeat="t in tools">{{t.name}}</a>
<a class="dropdown-item" href="/swagger-ui/" target="_blank">API documentation</a>
<a class="dropdown-item" href="/apidoc" target="_blank">API documentation</a>
</div>
</li>
</ul>

31
apps/dhp-broker-public-application/pom.xml
View File

@ -4,8 +4,8 @@
<parent>
<groupId>eu.dnetlib.dhp</groupId>
<artifactId>apps</artifactId>
<version>3.2.2-SNAPSHOT</version>
<relativePath>../</relativePath>
<version>3.3.3-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
<modelVersion>4.0.0</modelVersion>
@ -22,10 +22,37 @@
<artifactId>dnet-broker-apps-common</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client</artifactId>
</dependency>
<!-- Tests -->
<dependency>
<groupId>org.junit.jupiter</groupId>
<artifactId>junit-jupiter</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-core</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-junit-jupiter</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-test</artifactId>
<scope>test</scope>
</dependency>
</dependencies>
<build>

4
apps/dhp-broker-public-application/src/main/java/eu/dnetlib/broker/BrokerConfiguration.java
View File

@ -4,6 +4,7 @@ import org.elasticsearch.client.RestHighLevelClient;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.cache.annotation.EnableCaching;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.ComponentScan;
import org.springframework.context.annotation.Configuration;
import org.springframework.data.elasticsearch.client.ClientConfiguration;
import org.springframework.data.elasticsearch.client.RestClients;
@ -15,16 +16,15 @@ import org.springframework.transaction.annotation.EnableTransactionManagement;
import eu.dnetlib.broker.common.elasticsearch.Event;
import eu.dnetlib.broker.common.elasticsearch.Notification;
import eu.dnetlib.broker.common.properties.ElasticSearchProperties;
import springfox.documentation.swagger2.annotations.EnableSwagger2;
@Configuration
@EnableSwagger2
@EnableCaching
@EnableScheduling
@EnableTransactionManagement
@EnableElasticsearchRepositories(basePackageClasses = {
Event.class, Notification.class
})
@ComponentScan(basePackages = "eu.dnetlib")
public class BrokerConfiguration extends AbstractElasticsearchConfiguration {
@Autowired

45
apps/dhp-broker-public-application/src/main/java/eu/dnetlib/broker/BrokerPublicApplication.java
View File

@ -1,14 +1,15 @@
package eu.dnetlib.broker;
import java.util.Arrays;
import java.util.List;
import org.springdoc.core.GroupedOpenApi;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.context.annotation.Bean;
import eu.dnetlib.common.app.AbstractDnetApp;
import springfox.documentation.builders.ApiInfoBuilder;
import springfox.documentation.builders.RequestHandlerSelectors;
import springfox.documentation.service.ApiInfo;
import springfox.documentation.service.Tag;
import springfox.documentation.spring.web.plugins.Docket;
import io.swagger.v3.oas.models.tags.Tag;
@SpringBootApplication
public class BrokerPublicApplication extends AbstractDnetApp {
@ -19,22 +20,22 @@ public class BrokerPublicApplication extends AbstractDnetApp {
SpringApplication.run(BrokerPublicApplication.class, args);
}
@Override
protected void configSwagger(final Docket docket) {
docket.select()
.apis(RequestHandlerSelectors.any())
.paths(p -> p.startsWith("/"))
.build()
.tags(new Tag(OA_PUBLIC_APIS, OA_PUBLIC_APIS))
.apiInfo(new ApiInfoBuilder()
.title("OpenAIRE Public Broker API")
.description("APIs documentation")
.version("1.1")
.contact(ApiInfo.DEFAULT_CONTACT)
.license("Apache 2.0")
.licenseUrl("http://www.apache.org/licenses/LICENSE-2.0")
.build());
@Bean
public GroupedOpenApi publicApi() {
return GroupedOpenApi.builder()
.group("Broker Public APIs")
.pathsToMatch("/**")
.build();
}
@Override
protected String swaggerTitle() {
return "OpenAIRE Public Broker API";
}
@Override
protected List<Tag> swaggerTags() {
return Arrays.asList(new Tag().name(OA_PUBLIC_APIS).description(OA_PUBLIC_APIS));
}
}

2
apps/dhp-broker-public-application/src/main/java/eu/dnetlib/broker/oa/controllers/ApiDocController.java
View File

@ -8,6 +8,6 @@ public class ApiDocController {
@GetMapping({ "/apidoc", "/api-doc", "/doc", "/swagger" })
public String apiDoc() {
return "redirect:swagger-ui/";
return "redirect:swagger-ui/index.html";
}
}

20
apps/dhp-broker-public-application/src/main/java/eu/dnetlib/broker/oa/controllers/OpenairePublicController.java
View File

@ -63,13 +63,13 @@ import eu.dnetlib.broker.common.subscriptions.Subscription;
import eu.dnetlib.broker.common.subscriptions.SubscriptionRepository;
import eu.dnetlib.broker.objects.OaBrokerEventPayload;
import eu.dnetlib.common.controller.AbstractDnetController;
import io.swagger.annotations.Api;
import io.swagger.annotations.ApiOperation;
import io.swagger.v3.oas.annotations.Operation;
import io.swagger.v3.oas.annotations.tags.Tag;
@Profile("openaire")
@RestController
@RequestMapping("/")
@Api(tags = BrokerPublicApplication.OA_PUBLIC_APIS)
@Tag(name = BrokerPublicApplication.OA_PUBLIC_APIS)
public class OpenairePublicController extends AbstractDnetController {
@Autowired
@ -97,7 +97,7 @@ public class OpenairePublicController extends AbstractDnetController {
private static final Log log = LogFactory.getLog(OpenairePublicController.class);
@ApiOperation("Returns notifications by subscription using scrolls (first page)")
@Operation(summary = "Returns notifications by subscription using scrolls (first page)")
@GetMapping("/scroll/notifications/bySubscriptionId/{subscrId}")
public ScrollPage<ShortEventMessage> prepareScrollNotificationsBySubscrId(@PathVariable final String subscrId) {
@ -130,7 +130,7 @@ public class OpenairePublicController extends AbstractDnetController {
}
}
@ApiOperation("Returns notifications using scrolls (other pages)")
@Operation(summary = "Returns notifications using scrolls (other pages)")
@GetMapping("/scroll/notifications/{scrollId}")
public ScrollPage<ShortEventMessage> scrollNotifications(@PathVariable final String scrollId) {
@ -147,7 +147,7 @@ public class OpenairePublicController extends AbstractDnetController {
}
}
@ApiOperation("Returns notifications as file")
@Operation(summary = "Returns notifications as file")
@GetMapping(value = "/file/notifications/bySubscriptionId/{subscrId}", produces = "application/gzip")
public void notificationsAsFile(final HttpServletResponse res, @PathVariable final String subscrId) throws Exception {
@ -184,7 +184,7 @@ public class OpenairePublicController extends AbstractDnetController {
}
@ApiOperation("Returns events as file by opendoarId")
@Operation(summary = "Returns events as file by opendoarId")
@GetMapping(value = "/file/events/opendoar/{id}", produces = "application/gzip")
public void opendoarEventsAsFile(final HttpServletResponse res, @PathVariable final String id) {
@ -252,13 +252,13 @@ public class OpenairePublicController extends AbstractDnetController {
return first;
}
@ApiOperation("Returns the list of subscriptions by user email")
@Operation(summary = "Returns the list of subscriptions by user email")
@GetMapping(value = "/subscriptions")
private Iterable<Subscription> listSubscriptionsByUser(@RequestParam final String email) {
return subscriptionRepo.findBySubscriber(email);
}
@ApiOperation("Returns the status of the application")
@Operation(summary = "Returns the status of the application")
@GetMapping(value = "/status")
private Map<String, Long> status() {
final Map<String, Long> res = new LinkedHashMap<>();
@ -269,7 +269,7 @@ public class OpenairePublicController extends AbstractDnetController {
return res;
}
@ApiOperation("Store the feedback of an event (MOCK)")
@Operation(summary = "Store the feedback of an event (MOCK)")
@RequestMapping(value = "/feedback/events", method = {
RequestMethod.POST, RequestMethod.PATCH
})

3
apps/dhp-broker-public-application/src/main/resources/application.properties
View File

@ -1,5 +1,8 @@
spring.profiles.active = dev,openaire
server.public_url =
server.public_desc = API Base URL
#logging.level.root=DEBUG
maven.pom.path = /META-INF/maven/eu.dnetlib.dhp/dhp-broker-public-application/effective-pom.xml

2
apps/dhp-broker-public-application/src/main/resources/static/index.html
View File

@ -2,6 +2,6 @@
<html>
<head>
<title>OpenAIRE Broker Public API</title>
<meta http-equiv="refresh" content="2; url = ./swagger" />
<meta http-equiv="refresh" content="2; url = ./apidoc" />
</head>
</html>

29
apps/dhp-mdstore-manager/pom.xml
View File

@ -4,8 +4,8 @@
<parent>
<groupId>eu.dnetlib.dhp</groupId>
<artifactId>apps</artifactId>
<version>3.2.2-SNAPSHOT</version>
<relativePath>../</relativePath>
<version>3.3.3-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
<modelVersion>4.0.0</modelVersion>
@ -60,12 +60,31 @@
<artifactId>dhp-schemas</artifactId>
</dependency>
<!-- JUnit -->
<!-- Tests -->
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<groupId>org.junit.jupiter</groupId>
<artifactId>junit-jupiter</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-core</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-junit-jupiter</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-test</artifactId>
<scope>test</scope>
</dependency>
</dependencies>

43
apps/dhp-mdstore-manager/src/main/java/eu/dnetlib/data/mdstore/manager/MainApplication.java
View File

@ -1,51 +1,36 @@
package eu.dnetlib.data.mdstore.manager;
import org.springframework.beans.factory.annotation.Value;
import org.springdoc.core.GroupedOpenApi;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.boot.autoconfigure.domain.EntityScan;
import org.springframework.cache.annotation.EnableCaching;
import org.springframework.context.annotation.Bean;
import org.springframework.scheduling.annotation.EnableScheduling;
import eu.dnetlib.common.app.AbstractDnetApp;
import springfox.documentation.builders.ApiInfoBuilder;
import springfox.documentation.builders.RequestHandlerSelectors;
import springfox.documentation.service.ApiInfo;
import springfox.documentation.spring.web.plugins.Docket;
import springfox.documentation.swagger2.annotations.EnableSwagger2;
@SpringBootApplication
@EnableSwagger2
@EnableCaching
@EnableScheduling
@EntityScan("eu.dnetlib.dhp.schema.mdstore")
public class MainApplication extends AbstractDnetApp {
@Value("${dhp.swagger.api.host}")
private String swaggetHost;
@Value("${dhp.swagger.api.basePath}")
private String swaggerPath;
public static void main(final String[] args) {
SpringApplication.run(MainApplication.class, args);
}
@Bean
public GroupedOpenApi publicApi() {
return GroupedOpenApi.builder()
.group("MDStore APIs")
.pathsToMatch("/mdstores/**")
.build();
}
@Override
protected void configSwagger(final Docket docket) {
docket
.host(swaggetHost)
.pathMapping(swaggerPath)
.select()
.apis(RequestHandlerSelectors.any())
.paths(p -> p.startsWith("/mdstores"))
.build()
.apiInfo(new ApiInfoBuilder()
.title("MDStore Manager APIs")
.description("APIs documentation")
.version("1.1")
.contact(ApiInfo.DEFAULT_CONTACT)
.license("Apache 2.0")
.licenseUrl("http://www.apache.org/licenses/LICENSE-2.0")
.build());
protected String swaggerTitle() {
return "MDStore Manager APIs";
}
}

92
apps/dhp-mdstore-manager/src/main/java/eu/dnetlib/data/mdstore/manager/controller/MDStoreController.java
View File

@ -18,20 +18,18 @@ import org.springframework.web.bind.annotation.RestController;
import com.google.common.collect.Sets;
import eu.dnetlib.common.controller.AbstractDnetController;
import eu.dnetlib.dhp.schema.mdstore.MDStoreVersion;
import eu.dnetlib.dhp.schema.mdstore.MDStoreWithInfo;
import eu.dnetlib.data.mdstore.manager.exceptions.MDStoreManagerException;
import eu.dnetlib.data.mdstore.manager.utils.DatabaseUtils;
import eu.dnetlib.data.mdstore.manager.utils.HdfsClient;
import io.swagger.annotations.Api;
import io.swagger.annotations.ApiOperation;
import io.swagger.annotations.ApiParam;
import eu.dnetlib.dhp.schema.mdstore.MDStoreVersion;
import eu.dnetlib.dhp.schema.mdstore.MDStoreWithInfo;
import io.swagger.v3.oas.annotations.Operation;
import io.swagger.v3.oas.annotations.Parameter;
import io.swagger.v3.oas.annotations.tags.Tag;
@RestController
@RequestMapping("/mdstores")
@Api(tags = {
"Metadata Stores"
})
@Tag(name = "Metadata Stores")
public class MDStoreController extends AbstractDnetController {
@Autowired
@ -42,67 +40,67 @@ public class MDStoreController extends AbstractDnetController {
private static final Logger log = LoggerFactory.getLogger(DatabaseUtils.class);
@ApiOperation("Return all the mdstores")
@Operation(summary = "Return all the mdstores")
@GetMapping("/")
public Iterable<MDStoreWithInfo> find() {
return databaseUtils.listMdStores();
}
@ApiOperation("Return all the mdstore identifiers")
@Operation(summary = "Return all the mdstore identifiers")
@GetMapping("/ids")
public List<String> findIdentifiers() {
return databaseUtils.listMdStoreIDs();
}
@ApiOperation("Return a mdstores by id")
@Operation(summary = "Return a mdstores by id")
@GetMapping("/mdstore/{mdId}")
public MDStoreWithInfo getMdStore(@ApiParam("the mdstore identifier") @PathVariable final String mdId) throws MDStoreManagerException {
public MDStoreWithInfo getMdStore(@Parameter(name = "the mdstore identifier") @PathVariable final String mdId) throws MDStoreManagerException {
return databaseUtils.findMdStore(mdId);
}
@ApiOperation("Increase the read count of the current mdstore")
@Operation(summary = "Increase the read count of the current mdstore")
@GetMapping("/mdstore/{mdId}/startReading")
public MDStoreVersion startReading(@ApiParam("the mdstore identifier") @PathVariable final String mdId) throws MDStoreManagerException {
public MDStoreVersion startReading(@Parameter(name = "the mdstore identifier") @PathVariable final String mdId) throws MDStoreManagerException {
return databaseUtils.startReading(mdId);
}
@ApiOperation("Create a new mdstore")
@Operation(summary = "Create a new mdstore")
@GetMapping("/new/{format}/{layout}/{interpretation}")
public MDStoreWithInfo createMDStore(
@ApiParam("mdstore format") @PathVariable final String format,
@ApiParam("mdstore layout") @PathVariable final String layout,
@ApiParam("mdstore interpretation") @PathVariable final String interpretation,
@ApiParam("datasource name") @RequestParam(required = false) final String dsName,
@ApiParam("datasource id") @RequestParam(required = false) final String dsId,
@ApiParam("api id") @RequestParam(required = false) final String apiId) throws MDStoreManagerException {
@Parameter(name = "mdstore format") @PathVariable final String format,
@Parameter(name = "mdstore layout") @PathVariable final String layout,
@Parameter(name = "mdstore interpretation") @PathVariable final String interpretation,
@Parameter(name = "datasource name") @RequestParam(required = true) final String dsName,
@Parameter(name = "datasource id") @RequestParam(required = true) final String dsId,
@Parameter(name = "api id") @RequestParam(required = true) final String apiId) throws MDStoreManagerException {
final String id = databaseUtils.createMDStore(format, layout, interpretation, dsName, dsId, apiId);
return databaseUtils.findMdStore(id);
}
@ApiOperation("Delete a mdstore by id")
@Operation(summary = "Delete a mdstore by id")
@DeleteMapping("/mdstore/{mdId}")
public StatusResponse delete(@ApiParam("the id of the mdstore that will be deleted") @PathVariable final String mdId) throws MDStoreManagerException {
public StatusResponse delete(@Parameter(name = "the id of the mdstore that will be deleted") @PathVariable final String mdId) throws MDStoreManagerException {
final String hdfsPath = databaseUtils.deleteMdStore(mdId);
hdfsClient.deletePath(hdfsPath);
return StatusResponse.DELETED;
}
@ApiOperation("Return all the versions of a mdstore")
@Operation(summary = "Return all the versions of a mdstore")
@GetMapping("/mdstore/{mdId}/versions")
public Iterable<MDStoreVersion> listVersions(@PathVariable final String mdId) throws MDStoreManagerException {
return databaseUtils.listVersions(mdId);
}
@ApiOperation("Create a new preliminary version of a mdstore")
@Operation(summary = "Create a new preliminary version of a mdstore")
@GetMapping("/mdstore/{mdId}/newVersion")
public MDStoreVersion prepareNewVersion(@ApiParam("the id of the mdstore for which will be created a new version") @PathVariable final String mdId) {
public MDStoreVersion prepareNewVersion(@Parameter(name = "the id of the mdstore for which will be created a new version") @PathVariable final String mdId) {
return databaseUtils.prepareMdStoreVersion(mdId);
}
@ApiOperation("Promote a preliminary version to current")
@Operation(summary = "Promote a preliminary version to current")
@GetMapping("/version/{versionId}/commit/{size}")
public MDStoreVersion commitVersion(@ApiParam("the id of the version that will be promoted to the current version") @PathVariable final String versionId,
@ApiParam("the size of the new current mdstore") @PathVariable final long size) throws MDStoreManagerException {
public MDStoreVersion commitVersion(@Parameter(name = "the id of the version that will be promoted to the current version") @PathVariable final String versionId,
@Parameter(name = "the size of the new current mdstore") @PathVariable final long size) throws MDStoreManagerException {
try {
return databaseUtils.commitMdStoreVersion(versionId, size);
} finally {
@ -110,46 +108,46 @@ public class MDStoreController extends AbstractDnetController {
}
}
@ApiOperation("Abort a preliminary version")
@Operation(summary = "Abort a preliminary version")
@GetMapping("/version/{versionId}/abort")
public StatusResponse commitVersion(@ApiParam("the id of the version to abort") @PathVariable final String versionId) throws MDStoreManagerException {
public StatusResponse commitVersion(@Parameter(name = "the id of the version to abort") @PathVariable final String versionId) throws MDStoreManagerException {
final String hdfsPath = databaseUtils.deleteMdStoreVersion(versionId, true);
hdfsClient.deletePath(hdfsPath);
return StatusResponse.ABORTED;
}
@ApiOperation("Return an existing mdstore version")
@Operation(summary = "Return an existing mdstore version")
@GetMapping("/version/{versionId}")
public MDStoreVersion getVersion(@ApiParam("the id of the version that has to be deleted") @PathVariable final String versionId)
public MDStoreVersion getVersion(@Parameter(name = "the id of the version that has to be deleted") @PathVariable final String versionId)
throws MDStoreManagerException {
return databaseUtils.findVersion(versionId);
}
@ApiOperation("Delete a mdstore version")
@Operation(summary = "Delete a mdstore version")
@DeleteMapping("/version/{versionId}")
public StatusResponse deleteVersion(@ApiParam("the id of the version that has to be deleted") @PathVariable final String versionId,
@ApiParam("if true, the controls on writing and readcount values will be skipped") @RequestParam(required = false, defaultValue = "false") final boolean force)
public StatusResponse deleteVersion(@Parameter(name = "the id of the version that has to be deleted") @PathVariable final String versionId,
@Parameter(name = "if true, the controls on writing and readcount values will be skipped") @RequestParam(required = false, defaultValue = "false") final boolean force)
throws MDStoreManagerException {
final String hdfsPath = databaseUtils.deleteMdStoreVersion(versionId, force);
hdfsClient.deletePath(hdfsPath);
return StatusResponse.DELETED;
}
@ApiOperation("Decrease the read count of a mdstore version")
@Operation(summary = "Decrease the read count of a mdstore version")
@GetMapping("/version/{versionId}/endReading")
public MDStoreVersion endReading(@ApiParam("the id of the version that has been completely read") @PathVariable final String versionId)
public MDStoreVersion endReading(@Parameter(name = "the id of the version that has been completely read") @PathVariable final String versionId)
throws MDStoreManagerException {
return databaseUtils.endReading(versionId);
}
@ApiOperation("Reset the read count of a mdstore version")
@Operation(summary = "Reset the read count of a mdstore version")
@GetMapping("/version/{versionId}/resetReading")
public MDStoreVersion resetReading(@ApiParam("the id of the version") @PathVariable final String versionId)
public MDStoreVersion resetReading(@Parameter(name = "the id of the version") @PathVariable final String versionId)
throws MDStoreManagerException {
return databaseUtils.resetReading(versionId);
}
@ApiOperation("Delete expired versions")
@Operation(summary = "Delete expired versions")
@DeleteMapping("/versions/expired")
public StatusResponse deleteExpiredVersions() {
new Thread(this::performDeleteOfExpiredVersions).start();
@ -169,10 +167,10 @@ public class MDStoreController extends AbstractDnetController {
log.info("Done.");
}
@ApiOperation("Fix the inconsistencies on HDFS")
@Operation(summary = "Fix the inconsistencies on HDFS")
@GetMapping("/hdfs/inconsistencies")
public Set<String> fixHdfsInconsistencies(
@ApiParam("force the deletion of hdfs paths") @RequestParam(required = false, defaultValue = "false") final boolean delete)
@Parameter(name = "force the deletion of hdfs paths") @RequestParam(required = false, defaultValue = "false") final boolean delete)
throws MDStoreManagerException {
final Set<String> hdfsDirs = hdfsClient.listHadoopDirs();
@ -189,7 +187,7 @@ public class MDStoreController extends AbstractDnetController {
return toDelete;
}
@ApiOperation("Show informations")
@Operation(summary = "Show informations")
@GetMapping("/info")
public Map<String, Object> info() {
final Map<String, Object> info = new LinkedHashMap<>();
@ -201,21 +199,21 @@ public class MDStoreController extends AbstractDnetController {
return info;
}
@ApiOperation("list the file inside the path of a mdstore version")
@Operation(summary = "list the file inside the path of a mdstore version")
@GetMapping("/version/{versionId}/parquet/files")
public Set<String> listVersionFiles(@PathVariable final String versionId) throws MDStoreManagerException {
final String path = databaseUtils.findVersion(versionId).getHdfsPath();
return hdfsClient.listContent(path + "/store", HdfsClient::isParquetFile);
}
@ApiOperation("read the parquet file of a mdstore version")
@Operation(summary = "read the parquet file of a mdstore version")
@GetMapping("/version/{versionId}/parquet/content/{limit}")
public List<Map<String, String>> listVersionParquet(@PathVariable final String versionId, @PathVariable final long limit) throws MDStoreManagerException {
final String path = databaseUtils.findVersion(versionId).getHdfsPath();
return hdfsClient.readParquetFiles(path + "/store", limit);
}
@ApiOperation("read the parquet file of a mdstore (current version)")
@Operation(summary = "read the parquet file of a mdstore (current version)")
@GetMapping("/mdstore/{mdId}/parquet/content/{limit}")
public List<Map<String, String>> listMdstoreParquet(@PathVariable final String mdId, @PathVariable final long limit) throws MDStoreManagerException {
final String versionId = databaseUtils.findMdStore(mdId).getCurrentVersion();

2
apps/dhp-mdstore-manager/src/main/java/eu/dnetlib/data/mdstore/manager/controller/SwaggerController.java
View File

@ -11,6 +11,6 @@ public class SwaggerController {
"/apidoc", "/api-doc", "/doc", "/swagger"
}, method = RequestMethod.GET)
public String apiDoc() {
return "redirect:swagger-ui/";
return "redirect:swagger-ui/index.html";
}
}

9
apps/dhp-mdstore-manager/src/main/java/eu/dnetlib/data/mdstore/manager/controller/ZeppelinController.java
View File

@ -13,9 +13,9 @@ import eu.dnetlib.data.mdstore.manager.exceptions.MDStoreManagerException;
import eu.dnetlib.data.mdstore.manager.utils.ControllerUtils;
import eu.dnetlib.data.mdstore.manager.utils.DatabaseUtils;
import eu.dnetlib.data.mdstore.manager.utils.ZeppelinClient;
import eu.dnetlib.dhp.schema.mdstore.MDStoreWithInfo;
@Controller
@RequestMapping("/zeppelin")
public class ZeppelinController {
@Autowired
@ -24,11 +24,12 @@ public class ZeppelinController {
@Autowired
private DatabaseUtils databaseUtils;
@RequestMapping("/{mdId}/{note}")
@RequestMapping("/zeppelin/{mdId}/{note}")
public String goToZeppelin(@PathVariable final String mdId, final @PathVariable String note) throws MDStoreManagerException {
final String currentVersion = databaseUtils.findMdStore(mdId).getCurrentVersion();
final MDStoreWithInfo mdstore = databaseUtils.findMdStore(mdId);
final String currentVersion = mdstore.getCurrentVersion();
final String path = databaseUtils.findVersion(currentVersion).getHdfsPath() + "/store";
return "redirect:" + zeppelinClient.zeppelinNote(note, mdId, currentVersion, path);
return "redirect:" + zeppelinClient.zeppelinNote(note, mdstore, path);
}
@ExceptionHandler(Exception.class)

29
apps/dhp-mdstore-manager/src/main/java/eu/dnetlib/data/mdstore/manager/controller/ZeppelinRestController.java Normal file
View File

@ -0,0 +1,29 @@
package eu.dnetlib.data.mdstore.manager.controller;
import java.util.List;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.RestController;
import eu.dnetlib.common.controller.AbstractDnetController;
import eu.dnetlib.data.mdstore.manager.exceptions.MDStoreManagerException;
import eu.dnetlib.data.mdstore.manager.utils.ZeppelinClient;
@RestController
public class ZeppelinRestController extends AbstractDnetController {
@Autowired
private ZeppelinClient zeppelinClient;
@GetMapping("/zeppelin/templates")
public List<String> getTemplates() throws MDStoreManagerException {
try {
// if (zeppelinClient.get)
return zeppelinClient.listTemplates();
} catch (final Throwable e) {
throw new MDStoreManagerException("Zeppelin is unreachable", e);
}
}
}

8
apps/dhp-mdstore-manager/src/main/java/eu/dnetlib/data/mdstore/manager/utils/DatabaseUtils.java
View File

@ -16,15 +16,15 @@ import org.springframework.beans.factory.annotation.Value;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.stereotype.Service;
import eu.dnetlib.dhp.schema.mdstore.MDStore;
import eu.dnetlib.dhp.schema.mdstore.MDStoreCurrentVersion;
import eu.dnetlib.dhp.schema.mdstore.MDStoreVersion;
import eu.dnetlib.dhp.schema.mdstore.MDStoreWithInfo;
import eu.dnetlib.data.mdstore.manager.exceptions.MDStoreManagerException;
import eu.dnetlib.data.mdstore.manager.repository.MDStoreCurrentVersionRepository;
import eu.dnetlib.data.mdstore.manager.repository.MDStoreRepository;
import eu.dnetlib.data.mdstore.manager.repository.MDStoreVersionRepository;
import eu.dnetlib.data.mdstore.manager.repository.MDStoreWithInfoRepository;
import eu.dnetlib.dhp.schema.mdstore.MDStore;
import eu.dnetlib.dhp.schema.mdstore.MDStoreCurrentVersion;
import eu.dnetlib.dhp.schema.mdstore.MDStoreVersion;
import eu.dnetlib.dhp.schema.mdstore.MDStoreWithInfo;
@Service
public class DatabaseUtils {

357
apps/dhp-mdstore-manager/src/main/java/eu/dnetlib/data/mdstore/manager/utils/ZeppelinClient.java
View File

@ -1,34 +1,46 @@
package eu.dnetlib.data.mdstore.manager.utils;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
import java.util.Set;
import java.util.stream.Collectors;
import javax.annotation.PostConstruct;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang.StringEscapeUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.http.HttpEntity;
import org.springframework.http.HttpHeaders;
import org.springframework.http.HttpMethod;
import org.springframework.http.HttpStatus;
import org.springframework.http.MediaType;
import org.springframework.http.ResponseEntity;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Component;
import org.springframework.util.LinkedMultiValueMap;
import org.springframework.util.MultiValueMap;
import org.springframework.web.client.RestTemplate;
import eu.dnetlib.data.mdstore.manager.exceptions.MDStoreManagerException;
import eu.dnetlib.data.mdstore.manager.utils.zeppelin.HasStatus;
import eu.dnetlib.data.mdstore.manager.utils.zeppelin.ListResponse;
import eu.dnetlib.data.mdstore.manager.utils.zeppelin.Note;
import eu.dnetlib.data.mdstore.manager.utils.zeppelin.Paragraph;
import eu.dnetlib.data.mdstore.manager.utils.zeppelin.SimpleResponse;
import eu.dnetlib.data.mdstore.manager.utils.zeppelin.StringResponse;
import eu.dnetlib.dhp.schema.mdstore.MDStoreWithInfo;
@Component
public class ZeppelinClient {
@ -47,46 +59,126 @@ public class ZeppelinClient {
private static final Log log = LogFactory.getLog(ZeppelinClient.class);
public String zeppelinNote(final String note, final String mdId, final String currentVersion, final String currentVersionPath)
throws MDStoreManagerException {
final String jsessionid = obtainJsessionID();
private static final Map<String, List<String>> DEFAULT_RIGHTS = new LinkedHashMap<>();
final String newName = zeppelinNamePrefix + "/notes/" + note + "/" + currentVersion;
private static final Integer MAX_NUMBER_OF_MD_NOTES = 2;
final Optional<String> oldNoteId = listNotes(jsessionid).stream()
@PostConstruct
public void init() {
DEFAULT_RIGHTS.put("owners", Arrays.asList(zeppelinLogin));
DEFAULT_RIGHTS.put("readers", new ArrayList<>()); // ALL
DEFAULT_RIGHTS.put("runners", new ArrayList<>()); // ALL
DEFAULT_RIGHTS.put("writers", new ArrayList<>()); // ALL
}
private String jsessionid;
public String zeppelinNote(final String note, final MDStoreWithInfo mdstore, final String currentVersionPath) throws MDStoreManagerException {
if (notConfigured()) { throw new MDStoreManagerException("A zeppelin property is empty"); }
final String newName =
StringUtils.join(Arrays.asList(zeppelinNamePrefix, "notes", mdstore.getDatasourceName().replaceAll("/", "-"), mdstore.getApiId()
.replaceAll("/", "-"), note.replaceAll("/", "-"), mdstore.getCurrentVersion().replaceAll("/", "-")), "/");
final List<Map<String, String>> notes = listNotes();
final Optional<String> oldNoteId = notes.stream()
.filter(Objects::nonNull)
.filter(map -> newName.equals(map.get("name")))
.map(map -> map.get("id"))
.findFirst();
if (oldNoteId.isPresent()) {
log.info("Returning existing note: " + oldNoteId.get());
log.debug("Returning existing note: " + oldNoteId.get());
return zeppelinBaseUrl + "/#/notebook/" + oldNoteId.get();
}
final String templateNoteId = findTemplateNoteId(note, jsessionid);
final String templateName = zeppelinNamePrefix + "/templates/" + note;
final String templateNoteId = notes.stream()
.filter(map -> map.get("name").equals(templateName))
.map(map -> map.get("id"))
.findFirst()
.orElseThrow(() -> new MDStoreManagerException("Template Note not found: " + templateName));
final String newId = cloneNote(templateNoteId, newName, jsessionid);
log.info("New note created, id: " + newId + ", name: " + newName);
addParagraph(newId, confParagraph(mdId, currentVersion, currentVersionPath), jsessionid);
reassignRights(newId, jsessionid);
final String newId = cloneNote(templateNoteId, newName, mdstore, currentVersionPath);
return zeppelinBaseUrl + "/#/notebook/" + newId;
}
// TODO: prepare the cron job
public void cleanExpiredNotes() {
try {
final String jsessionid = obtainJsessionID();
public List<String> listTemplates() {
final String prefix = zeppelinNamePrefix + "/templates/";
for (final Map<String, String> n : listNotes(jsessionid)) {
final String id = n.get("id");
if (n.get("name").startsWith(zeppelinNamePrefix + "/notes/") && isExpired(id, jsessionid)) {
deleteNote(id, jsessionid);
if (notConfigured()) {
return new ArrayList<>();
} else {
return listNotes().stream()
.map(map -> map.get("name"))
.filter(s -> s.startsWith(prefix))
.map(s -> StringUtils.substringAfter(s, prefix))
.sorted()
.collect(Collectors.toList());
}
}
private List<Map<String, String>> listNotes() {
return callApi(HttpMethod.GET, "notebook", ListResponse.class, null).getBody();
}
private String cloneNote(final String noteId, final String newName, final MDStoreWithInfo mdstore, final String currentVersionPath)
throws MDStoreManagerException {
final String newId = callApi(HttpMethod.POST, "notebook/" + noteId, StringResponse.class, new Note(newName)).getBody();
callApi(HttpMethod.POST, "notebook/" + newId + "/paragraph", StringResponse.class, confParagraph(mdstore, currentVersionPath)).getBody();
callApi(HttpMethod.PUT, "notebook/" + newId + "/permissions", SimpleResponse.class, DEFAULT_RIGHTS);
log.info("New note created, id: " + newId + ", name: " + newName);
return newId;
}
private Paragraph confParagraph(final MDStoreWithInfo mdstore, final String currentVersionPath) throws MDStoreManagerException {
try {
final String code = IOUtils.toString(getClass().getResourceAsStream("/zeppelin/paragraph_conf.tmpl"), StandardCharsets.UTF_8)
.replaceAll("__DS_NAME__", StringEscapeUtils.escapeJava(mdstore.getDatasourceName()))
.replaceAll("__DS_ID__", StringEscapeUtils.escapeJava(mdstore.getDatasourceId()))
.replaceAll("__API_ID__", StringEscapeUtils.escapeJava(mdstore.getApiId()))
.replaceAll("__MDSTORE_ID__", mdstore.getId())
.replaceAll("__VERSION__", mdstore.getCurrentVersion())
.replaceAll("__PATH__", currentVersionPath);
return new Paragraph("Configuration", code, 0);
} catch (final IOException e) {
log.error("Error preparing configuration paragraph", e);
throw new MDStoreManagerException("Error preparing configuration paragraph", e);
}
}
@Scheduled(fixedRate = 12 * 60 * 60 * 1000) // 12 hours
public void cleanExpiredNotes() {
if (notConfigured()) { return; }
try {
// I sort the notes according to the version datestamp (more recent first)
final List<Map<String, String>> notes = listNotes()
.stream()
.filter(n -> n.get("name").startsWith(zeppelinNamePrefix + "/notes/"))
.sorted((o1, o2) -> StringUtils.compare(o2.get("name"), o1.get("name")))
.collect(Collectors.toList());
final Map<String, Integer> map = new HashMap<>();
for (final Map<String, String> n : notes) {
final String firstPart = StringUtils.substringBeforeLast(n.get("name"), "-");
if (!map.containsKey(firstPart)) {
log.debug("Evaluating note " + n.get("name") + " for deletion: CONFIRMED");
map.put(firstPart, 1);
} else if (map.get(firstPart) < MAX_NUMBER_OF_MD_NOTES) {
log.debug("Evaluating note " + n.get("name") + " for deletion: CONFIRMED");
map.put(firstPart, map.get(firstPart) + 1);
} else {
log.debug("Evaluating note " + n.get("name") + " for deletion: TO_DELETE");
callApi(HttpMethod.DELETE, "notebook/" + n.get("id"), SimpleResponse.class, null);
}
}
} catch (final Exception e) {
@ -94,7 +186,90 @@ public class ZeppelinClient {
}
}
private String obtainJsessionID() throws MDStoreManagerException {
private <T extends HasStatus> T callApi(final HttpMethod method, final String api, final Class<T> resClazz, final Object objRequest) {
if (jsessionid == null) {
final T res = findNewJsessionId(method, api, resClazz, objRequest);
if (res != null) { return res; }
} else {
try {
return callApi(method, api, resClazz, objRequest, jsessionid);
} catch (final MDStoreManagerException e) {
final T res = findNewJsessionId(method, api, resClazz, objRequest);
if (res != null) { return res; }
}
}
throw new RuntimeException("All attempted calls are failed");
}
@SuppressWarnings("unchecked")
private <T extends HasStatus> T callApi(final HttpMethod method,
final String api,
final Class<T> resClazz,
final Object objRequest,
final String jsessionid)
throws MDStoreManagerException {
final String url = String.format("%s/api/%s;JSESSIONID=%s", zeppelinBaseUrl, api, jsessionid);
final RestTemplate restTemplate = new RestTemplate();
ResponseEntity<T> res = null;
switch (method) {
case GET:
log.debug("Performing GET: " + url);
res = restTemplate.getForEntity(url, resClazz);
break;
case POST:
log.debug("Performing POST: " + url);
res = restTemplate.postForEntity(url, objRequest, resClazz);
break;
case PUT:
log.debug("Performing PUT: " + url);
restTemplate.put(url, objRequest);
break;
case DELETE:
log.debug("Performing DELETE: " + url);
restTemplate.delete(url);
break;
default:
throw new RuntimeException("Unsupported method: " + method);
}
if (method == HttpMethod.PUT || method == HttpMethod.DELETE) {
return (T) new SimpleResponse("OK");
} else if (res == null) {
log.error("NULL response from the API");
throw new MDStoreManagerException("NULL response from the API");
} else if (res.getStatusCode() != HttpStatus.OK) {
log.error("Zeppelin API failed with HTTP error: " + res);
throw new MDStoreManagerException("Zeppelin API failed with HTTP error: " + res);
} else if (res.getBody() == null) {
log.error("Zeppelin API returned a null response");
throw new MDStoreManagerException("Zeppelin API returned a null response");
} else if (!res.getBody().getStatus().equals("OK")) {
log.error("Zeppelin API Operation failed: " + res.getBody());
throw new MDStoreManagerException("Registration of zeppelin note failed: " + res.getBody());
} else {
return res.getBody();
}
}
private <T extends HasStatus> T findNewJsessionId(final HttpMethod method, final String api, final Class<T> resClazz, final Object objRequest) {
for (final String id : obtainJsessionIDs()) {
try {
final T res = callApi(method, api, resClazz, objRequest, id);
setJsessionid(id);
return res;
} catch (final MDStoreManagerException e) {
log.warn("Skipping invalid jsessionid: " + id);
}
}
return null;
}
private Set<String> obtainJsessionIDs() {
final HttpHeaders headers = new HttpHeaders();
headers.setContentType(MediaType.APPLICATION_FORM_URLENCODED);
@ -109,10 +284,10 @@ public class ZeppelinClient {
if (res.getStatusCode() != HttpStatus.OK) {
log.error("Zeppelin API: login failed with HTTP error: " + res);
throw new MDStoreManagerException("Zeppelin API: login failed with HTTP error: " + res);
throw new RuntimeException("Zeppelin API: login failed with HTTP error: " + res);
} else if (!res.getHeaders().containsKey(HttpHeaders.SET_COOKIE)) {
log.error("Zeppelin API: login failed (missing SET_COOKIE header)");
throw new MDStoreManagerException("Zeppelin API: login failed (missing SET_COOKIE header)");
throw new RuntimeException("Zeppelin API: login failed (missing SET_COOKIE header)");
} else {
return res.getHeaders()
.get(HttpHeaders.SET_COOKIE)
@ -123,138 +298,20 @@ public class ZeppelinClient {
.filter(s -> s.startsWith("JSESSIONID="))
.map(s -> StringUtils.removeStart(s, "JSESSIONID="))
.filter(s -> !s.equalsIgnoreCase("deleteMe"))
.distinct()
.filter(this::testConnection)
.findFirst()
.orElseThrow(() -> new MDStoreManagerException("Zeppelin API: login failed (invalid jsessionid)"));
.collect(Collectors.toSet());
}
}
private boolean testConnection(final String jsessionid) {
final String url = zeppelinBaseUrl + "/api/notebook;JSESSIONID=" + jsessionid;
log.info("Performing GET: " + url);
final ResponseEntity<ListResponse> res = new RestTemplate().getForEntity(url, ListResponse.class);
if (res.getStatusCode() != HttpStatus.OK) {
return false;
} else if (res.getBody() == null) {
return false;
} else if (!res.getBody().getStatus().equals("OK")) {
return false;
} else {
log.info("Connected to zeppelin: " + res.getBody());
log.info("Found JSESSIONID: " + jsessionid);
return true;
}
public String getJsessionid() {
return jsessionid;
}
private List<Map<String, String>> listNotes(final String jsessionid) throws MDStoreManagerException {
final String url = zeppelinBaseUrl + "/api/notebook;JSESSIONID=" + jsessionid;
log.info("Performing GET: " + url);
final ResponseEntity<ListResponse> res = new RestTemplate().getForEntity(url, ListResponse.class);
if (res.getStatusCode() != HttpStatus.OK) {
log.error("Zeppelin API failed with HTTP error: " + res);
throw new MDStoreManagerException("Zeppelin API failed with HTTP error: " + res);
} else if (res.getBody() == null) {
log.error("Zeppelin API returned a null response");
throw new MDStoreManagerException("Zeppelin API returned a null response");
} else if (!res.getBody().getStatus().equals("OK")) {
log.error("Registration of zeppelin note failed: " + res.getBody());
throw new MDStoreManagerException("Registration of zeppelin note failed: " + res.getBody());
} else {
return res.getBody().getBody();
}
public void setJsessionid(final String jsessionid) {
this.jsessionid = jsessionid;
}
private String findTemplateNoteId(final String noteTemplate, final String jsessionid) throws MDStoreManagerException {
final String templateName = zeppelinNamePrefix + "/templates/" + noteTemplate;
return listNotes(jsessionid).stream()
.filter(map -> map.get("name").equals(templateName))
.map(map -> map.get("id"))
.findFirst()
.orElseThrow(() -> new MDStoreManagerException("Template Note not found: " + templateName));
}
private String cloneNote(final String noteId, final String newName, final String jsessionid) throws MDStoreManagerException {
final String url = zeppelinBaseUrl + "/api/notebook/" + noteId + ";JSESSIONID=" + jsessionid;
log.debug("Performing POST: " + url);
final ResponseEntity<StringResponse> res = new RestTemplate().postForEntity(url, new Note(newName), StringResponse.class);
if (res.getStatusCode() != HttpStatus.OK) {
log.error("Zeppelin API failed with HTTP error: " + res);
throw new MDStoreManagerException("Zeppelin API failed with HTTP error: " + res);
} else if (res.getBody() == null) {
log.error("Zeppelin API returned a null response");
throw new MDStoreManagerException("Zeppelin API returned a null response");
} else if (!res.getBody().getStatus().equals("OK")) {
log.error("Registration of zeppelin note failed: " + res.getBody());
throw new MDStoreManagerException("Registration of zeppelin note failed: " + res.getBody());
} else {
return res.getBody().getBody();
}
}
private Paragraph confParagraph(final String mdId, final String currentVersion, final String currentVersionPath) throws MDStoreManagerException {
try {
final String code = IOUtils.toString(getClass().getResourceAsStream("/zeppelin/conf.tmpl.py"))
.replaceAll("__MDSTORE_ID__", mdId)
.replaceAll("__VERSION__", currentVersion)
.replaceAll("__PATH__", currentVersionPath);
return new Paragraph("Configuration", code, 0);
} catch (final IOException e) {
log.error("Error preparing configuration paragraph", e);
throw new MDStoreManagerException("Error preparing configuration paragraph", e);
}
}
private String addParagraph(final String noteId, final Paragraph paragraph, final String jsessionid) throws MDStoreManagerException {
final String url = zeppelinBaseUrl + "/api/notebook/" + noteId + "/paragraph;JSESSIONID=" + jsessionid;
log.debug("Performing POST: " + url);
final ResponseEntity<StringResponse> res = new RestTemplate().postForEntity(url, paragraph, StringResponse.class);
if (res.getStatusCode() != HttpStatus.OK) {
log.error("Zeppelin API failed with HTTP error: " + res);
throw new MDStoreManagerException("Zeppelin API failed with HTTP error: " + res);
} else if (res.getBody() == null) {
log.error("Zeppelin API returned a null response");
throw new MDStoreManagerException("Zeppelin API returned a null response");
} else if (!res.getBody().getStatus().equals("OK")) {
log.error("Registration of zeppelin note failed: " + res.getBody());
throw new MDStoreManagerException("Registration of zeppelin note failed: " + res.getBody());
} else {
return res.getBody().getBody();
}
}
private void reassignRights(final String noteId, final String jsessionid) {
final String url = zeppelinBaseUrl + "/api/notebook/" + noteId + "/permissions;JSESSIONID=" + jsessionid;
log.info("Performing PUT: " + url);
final Map<String, List<String>> rights = new LinkedHashMap<>();
rights.put("owners", Arrays.asList(zeppelinLogin));
rights.put("readers", new ArrayList<>()); // ALL
rights.put("runners", new ArrayList<>()); // ALL
rights.put("writers", new ArrayList<>()); // ALL
new RestTemplate().put(url, rights);
}
private void deleteNote(final String id, final String jsessionid) {
final String url = zeppelinBaseUrl + "/api/notebook/" + id + ";JSESSIONID=" + jsessionid;
log.debug("Performing DELETE: " + url);
new RestTemplate().delete(url);
}
private boolean isExpired(final String id, final String jsessionid) {
// TODO Auto-generated method stub
return false;
private boolean notConfigured() {
return StringUtils.isAnyBlank(zeppelinBaseUrl, zeppelinLogin, zeppelinPassword, zeppelinNamePrefix);
}
}

6
apps/dhp-mdstore-manager/src/main/java/eu/dnetlib/data/mdstore/manager/utils/zeppelin/HasStatus.java Normal file
View File

@ -0,0 +1,6 @@
package eu.dnetlib.data.mdstore.manager.utils.zeppelin;
public interface HasStatus {
String getStatus();
}

3
apps/dhp-mdstore-manager/src/main/java/eu/dnetlib/data/mdstore/manager/utils/zeppelin/ListResponse.java
View File

@ -3,12 +3,13 @@ package eu.dnetlib.data.mdstore.manager.utils.zeppelin;
import java.util.List;
import java.util.Map;
public class ListResponse {
public class ListResponse implements HasStatus {
private String status;
private String message;
private List<Map<String, String>> body;
@Override
public String getStatus() {
return status;
}

16
apps/dhp-mdstore-manager/src/main/java/eu/dnetlib/data/mdstore/manager/utils/zeppelin/SimpleResponse.java Normal file
View File

@ -0,0 +1,16 @@
package eu.dnetlib.data.mdstore.manager.utils.zeppelin;
public class SimpleResponse implements HasStatus {
private final String status;
public SimpleResponse(final String status) {
this.status = status;
}
@Override
public String getStatus() {
return status;
}
}

3
apps/dhp-mdstore-manager/src/main/java/eu/dnetlib/data/mdstore/manager/utils/zeppelin/StringResponse.java
View File

@ -1,11 +1,12 @@
package eu.dnetlib.data.mdstore.manager.utils.zeppelin;
public class StringResponse {
public class StringResponse implements HasStatus {
private String status;
private String message;
private String body;
@Override
public String getStatus() {
return status;
}

17
apps/dhp-mdstore-manager/src/main/resources/application.properties
View File

@ -1,5 +1,8 @@
spring.main.banner-mode = console
server.public_url =
server.public_desc = API Base URL
logging.level.root = INFO
maven.pom.path = /META-INF/maven/eu.dnetlib.dhp/dhp-mdstore-manager/effective-pom.xml
@ -12,8 +15,8 @@ management.endpoints.web.path-mapping.prometheus = metrics
management.endpoints.web.path-mapping.health = health
spring.datasource.url=jdbc:postgresql://localhost:5432/mdstoremanager
spring.datasource.username=dnet
spring.datasource.password=dnetPwd
spring.datasource.username=
spring.datasource.password=
spring.jpa.properties.hibernate.dialect = org.hibernate.dialect.PostgreSQLDialect
@ -31,12 +34,16 @@ dhp.mdstore-manager.hadoop.cluster = GARR
dhp.mdstore-manager.hdfs.base-path = /data/dnet.dev/mdstore
dhp.mdstore-manager.hadoop.user = dnet.dev
dhp.mdstore-manager.hadoop.zeppelin.base-url = https://iis-cdh5-test-gw.ocean.icm.edu.pl/zeppelin
#dhp.mdstore-manager.hadoop.zeppelin.base-url = https://iis-cdh5-test-gw.ocean.icm.edu.pl/zeppelin
#dhp.mdstore-manager.hadoop.zeppelin.login =
#dhp.mdstore-manager.hadoop.zeppelin.password =
dhp.mdstore-manager.hadoop.zeppelin.base-url = https://hadoop-zeppelin.garr-pa1.d4science.org
dhp.mdstore-manager.hadoop.zeppelin.login =
dhp.mdstore-manager.hadoop.zeppelin.password =
dhp.mdstore-manager.hadoop.zeppelin.name-prefix = mdstoreManager
dhp.mdstore-manager.inspector.records.max = 1000
dhp.swagger.api.host = localhost
dhp.swagger.api.basePath = /
# dhp.swagger.api.host = localhost
dhp.swagger.api.basePath = /**

11
apps/dhp-mdstore-manager/src/main/resources/static/index.html
View File

@ -20,7 +20,7 @@
<h1>Metadata Store Manager</h1>
<hr />
<a href="./swagger-ui/" target="_blank">API documentation</a>
<a href="./apidoc" target="_blank">API documentation</a>
<hr />
<a href="javascript:void(0)" data-toggle="modal" data-target="#newMdstoreModal">create a new mdstore</a>
<hr />
@ -77,14 +77,11 @@
<div class="float-right">
<a href="./mdrecords/{{md.id}}/50" class="btn btn-sm btn-primary" target="_blank">inspect</a>
<div class="btn-group">
<button class="btn btn-sm btn-outline-warning dropdown-toggle" disabled="disabled" ng-show="zeppelinTemplates.length == 0">zeppelin <span class="caret"></span></button>
<div class="btn-group" ng-show="zeppelinTemplates.length > 0">
<button class="btn btn-sm btn-warning dropdown-toggle" data-toggle="dropdown">zeppelin <span class="caret"></span></button>
<div class="dropdown-menu dropdown-menu-right">
<a class="dropdown-item small" href="./zeppelin/{{md.id}}/default" target="_blank">default note</a>
<a class="dropdown-item small" href="./zeppelin/{{md.id}}/dc_native" target="_blank">note for native stores (oai_dc)</a>
<a class="dropdown-item small" href="./zeppelin/{{md.id}}/datacite_native" target="_blank">note for native stores (datacite)</a>
<a class="dropdown-item small" href="./zeppelin/{{md.id}}/oaf_cleaned" target="_blank">note for transformed stores</a>
<a class="dropdown-item small" href="./zeppelin/{{md.id}}/{{t}}" target="_blank" ng-repeat="t in zeppelinTemplates">{{t}}</a>
</div>
</div>
</div>

14
apps/dhp-mdstore-manager/src/main/resources/static/mdstoremanager.js
View File

@ -5,6 +5,7 @@ app.controller('mdstoreManagerController', function($scope, $http) {
$scope.versions = [];
$scope.openMdstore = '';
$scope.openCurrentVersion = ''
$scope.zeppelinTemplates = [];
$scope.forceVersionDelete = false;
@ -14,7 +15,16 @@ app.controller('mdstoreManagerController', function($scope, $http) {
}, function errorCallback(res) {
alert('ERROR: ' + res.data.message);
});
};
};
$scope.obtainZeppelinTemplates = function() {
$http.get('./zeppelin/templates?' + $.now()).then(function successCallback(res) {
$scope.zeppelinTemplates = res.data;
}, function errorCallback(res) {
alert('ERROR: ' + res.data.message);
});
};
$scope.newMdstore = function(format, layout, interpretation, dsName, dsId, apiId) {
var url = './mdstores/new/' + encodeURIComponent(format) + '/' + encodeURIComponent(layout) + '/' + encodeURIComponent(interpretation);
@ -112,5 +122,5 @@ app.controller('mdstoreManagerController', function($scope, $http) {
};
$scope.reload();
$scope.obtainZeppelinTemplates();
});

37
apps/dhp-mdstore-manager/src/main/resources/templates/inspector.html
View File

@ -40,6 +40,10 @@
<script th:inline="javascript">
/*<![CDATA[*/
function mdId() {
return /*[[${mdId}]]*/ '';
}
function versionId() {
return /*[[${versionId}]]*/ '';
}
@ -62,7 +66,16 @@
<div class="col">
<h1>Metadata Inspector</h1>
<br />
<hr />
<div ng-show="zeppelinTemplates.length > 0">
<div class="btn-group">
<button class="btn btn-sm btn-warning dropdown-toggle" data-toggle="dropdown">zeppelin <span class="caret"></span></button>
<div class="dropdown-menu dropdown-menu-right">
<a class="dropdown-item small" href="../../zeppelin/{{mdId}}/{{t}}" target="_blank" ng-repeat="t in zeppelinTemplates">{{t}}</a>
</div>
</div>
<hr />
</div>
<table class="table table-condensed table-sm small">
<tr>
@ -111,8 +124,12 @@
<br />
<div class="card mt-4" ng-repeat="rec in records | filter:recordsFilter">
<div class="card-header text-white bg-primary small">{{rec.id}}</div>
<table class="table table-condensed table-striped small">
<div class="card-header text-white bg-primary small">
<span ng-show="rec.id">{{rec.id}}</span>
<span ng-hide="rec.id">the record is unreadable</span>
</div>
<div class="card-body" ng-hide="rec.id">Invalid record format</div>
<table class="table table-condensed table-striped small" ng-show="rec.id">
<tr>
<th class="col-xs-3">Original Id</th>
<td class="col-xs-9">{{rec.originalId}}</td>
@ -134,7 +151,7 @@
</td>
</tr>
</table>
<div class="card-body">
<div class="card-body" ng-show="rec.id">
<span class="badge badge-success float-right">{{rec.encoding}}</span>
<br />
<pre class="small">{{rec.body}}</pre>
@ -181,9 +198,11 @@
var app = angular.module('mdInspectorApp', []);
app.controller('mdInspectorController', function($scope, $http) {
$scope.mdId = mdId();
$scope.records = [];
$scope.versionId = versionId();
$scope.limit = limit();
$scope.zeppelinTemplates = [];
$scope.reload = function() {
showSpinner();
@ -199,9 +218,17 @@
alert('ERROR: ' + res.data.message);
});
};
$scope.obtainZeppelinTemplates = function() {
$http.get('../../zeppelin/templates?' + $.now()).then(function successCallback(res) {
$scope.zeppelinTemplates = res.data;
}, function errorCallback(res) {
alert('ERROR: ' + res.data.message);
});
};
$scope.reload();
$scope.obtainZeppelinTemplates();
});
</script>

9
apps/dhp-mdstore-manager/src/main/resources/zeppelin/conf.tmpl.py
View File

@ -1,9 +0,0 @@
%pyspark
mdId = "__MDSTORE_ID__"
mdVersion = "__VERSION__"
path = "__PATH__"
print "MdStore ID:", mdId
print "Version ID:", mdVersion
print "Version Data Path:", path

8
apps/dhp-mdstore-manager/src/main/resources/zeppelin/paragraph_conf.tmpl Normal file
View File

@ -0,0 +1,8 @@
%spark
val dsName = "__DS_NAME__"
val dsId = "__DS_ID__"
val apiId = "__API_ID__"
val mdId = "__MDSTORE_ID__"
val mdVersion = "__VERSION__"
val path = "__PATH__"

2
apps/dhp-mdstore-manager/src/test/java/eu/dnetlib/data/mdstore/manager/controller/MDStoreControllerTest.java
View File

@ -1,8 +1,8 @@
package eu.dnetlib.data.mdstore.manager.controller;
import static org.junit.Assert.assertTrue;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotEquals;
import static org.junit.jupiter.api.Assertions.assertTrue;
import java.util.Date;

2
apps/dhp-mdstore-manager/src/test/java/eu/dnetlib/data/mdstore/manager/controller/MDStoreControllerWithMocksTest.java
View File

@ -1,7 +1,7 @@
package eu.dnetlib.data.mdstore.manager.controller;
import static org.junit.Assert.assertTrue;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.verify;

6
apps/dnet-exporter-api/README.md Normal file
View File

@ -0,0 +1,6 @@
SpringBoot application implementing OpenAIRE REST API to manage
- Datasources
- Contexts
- Communities
- Funders
- Projects

149
apps/dnet-exporter-api/pom.xml Normal file
View File

@ -0,0 +1,149 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<groupId>eu.dnetlib.dhp</groupId>
<artifactId>apps</artifactId>
<version>3.3.3-SNAPSHOT</version>
<relativePath>../</relativePath>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>dnet-exporter-api</artifactId>
<packaging>jar</packaging>
<!-- Add typical dependencies for a web application -->
<dependencies>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-jdbc</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-data-jpa</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-cache</artifactId>
</dependency>
<dependency>
<groupId>eu.dnetlib</groupId>
<artifactId>cnr-rmi-api</artifactId>
<version>[2.0.0,3.0.0)</version>
</dependency>
<dependency>
<groupId>org.apache.cxf</groupId>
<artifactId>cxf-rt-transports-http</artifactId>
<version>3.1.5</version>
</dependency>
<dependency>
<groupId>eu.dnetlib</groupId>
<artifactId>cnr-service-common</artifactId>
<version>[2.0.0,3.0.0)</version>
</dependency>
<dependency>
<groupId>eu.dnetlib</groupId>
<artifactId>dnet-openaireplus-mapping-utils</artifactId>
<version>[6.3.0,7.0.0)</version>
<exclusions>
<exclusion>
<groupId>com.sun.jersey</groupId>
<artifactId>jersey-client</artifactId>
</exclusion>
<exclusion>
<groupId>eu.dnetlib</groupId>
<artifactId>dnet-hadoop-commons</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>eu.dnetlib</groupId>
<artifactId>dnet-objectstore-rmi</artifactId>
<version>[2.0.0,3.0.0)</version>
</dependency>
<dependency>
<groupId>org.apache.solr</groupId>
<artifactId>solr-solrj</artifactId>
<version>7.5.0</version>
</dependency>
<dependency>
<groupId>org.postgresql</groupId>
<artifactId>postgresql</artifactId>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-dbcp2</artifactId>
</dependency>
<dependency>
<groupId>org.antlr</groupId>
<artifactId>stringtemplate</artifactId>
<version>3.2.1</version>
</dependency>
<dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpclient</artifactId>
</dependency>
<dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpcore</artifactId>
</dependency>
<dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpmime</artifactId>
</dependency>
<dependency>
<groupId>net.sf.supercsv</groupId>
<artifactId>super-csv</artifactId>
<version>2.4.0</version>
</dependency>
<dependency>
<groupId>com.google.code.gson</groupId>
<artifactId>gson</artifactId>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.datatype</groupId>
<artifactId>jackson-datatype-joda</artifactId>
</dependency>
<dependency>
<groupId>joda-time</groupId>
<artifactId>joda-time</artifactId>
<version>2.8.2</version>
</dependency>
<dependency>
<groupId>org.mongodb</groupId>
<artifactId>mongo-java-driver</artifactId>
<version>3.4.2</version>
</dependency>
<dependency>
<groupId>eu.dnetlib</groupId>
<artifactId>dnet-datasource-manager-common</artifactId>
<version>[2.0.1,3.0.0)</version>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-test</artifactId>
<scope>test</scope>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-help-plugin</artifactId>
</plugin>
<plugin>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-maven-plugin</artifactId>
</plugin>
</plugins>
</build>
</project>

29
apps/dnet-exporter-api/src/main/java/eu/dnetlib/CacheCustomizer.java Normal file
View File

@ -0,0 +1,29 @@
package eu.dnetlib;
import org.springframework.boot.autoconfigure.cache.CacheManagerCustomizer;
import org.springframework.cache.concurrent.ConcurrentMapCacheManager;
import org.springframework.stereotype.Component;
import static java.util.Arrays.asList;
@Component
public class CacheCustomizer implements CacheManagerCustomizer<ConcurrentMapCacheManager> {
@Override
public void customize(final ConcurrentMapCacheManager cacheManager) {
cacheManager.setCacheNames(
asList(
"fundingpath-ids",
"indexdsinfo-cache",
"objectstoreid-cache",
"context-cache",
"context-cache-funder",
"context-cache-community",
"dsm-aggregationhistory-cache",
"dsm-firstharvestdate-cache",
"vocabularies-cache",
"community-cache",
"info"));
}
}

88
apps/dnet-exporter-api/src/main/java/eu/dnetlib/DNetOpenaireExporterApplication.java Normal file
View File

@ -0,0 +1,88 @@
package eu.dnetlib;
import org.springdoc.core.GroupedOpenApi;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.boot.autoconfigure.solr.SolrAutoConfiguration;
import org.springframework.cache.annotation.EnableCaching;
import org.springframework.context.annotation.Bean;
import org.springframework.scheduling.annotation.EnableScheduling;
import eu.dnetlib.common.app.AbstractDnetApp;
import eu.dnetlib.openaire.community.CommunityApiController;
import eu.dnetlib.openaire.context.ContextApiController;
import eu.dnetlib.openaire.dsm.DsmApiController;
import eu.dnetlib.openaire.funders.FundersApiController;
import eu.dnetlib.openaire.info.InfoController;
import eu.dnetlib.openaire.project.ProjectsController;
@EnableCaching
@EnableScheduling
@SpringBootApplication
@EnableAutoConfiguration(exclude = {
SolrAutoConfiguration.class
})
public class DNetOpenaireExporterApplication extends AbstractDnetApp {
public static final String V1 = "1.0.0";
public static void main(final String[] args) throws Exception {
SpringApplication.run(DNetOpenaireExporterApplication.class, args);
}
@Override
protected String swaggerTitle() {
return "D-Net Exporter APIs";
}
@Override
protected String swaggerVersion() {
return V1;
}
@Bean
@ConditionalOnProperty(value = "openaire.exporter.enable.dsm", havingValue = "true")
public GroupedOpenApi dsm() {
return newGroupedOpenApi("Datasource Manager", DsmApiController.class.getPackage().getName());
}
@Bean
@ConditionalOnProperty(value = "openaire.exporter.enable.project", havingValue = "true")
public GroupedOpenApi projects() {
return newGroupedOpenApi("OpenAIRE Projects", ProjectsController.class.getPackage().getName());
}
@Bean
@ConditionalOnProperty(value = "openaire.exporter.enable.funders", havingValue = "true")
public GroupedOpenApi funders() {
return newGroupedOpenApi("OpenAIRE Funders", FundersApiController.class.getPackage().getName());
}
@Bean
@ConditionalOnProperty(value = "openaire.exporter.enable.community", havingValue = "true")
public GroupedOpenApi communities() {
return newGroupedOpenApi("OpenAIRE Communities", CommunityApiController.class.getPackage().getName());
}
@Bean
@ConditionalOnProperty(value = "openaire.exporter.enable.context", havingValue = "true")
public GroupedOpenApi contexts() {
return newGroupedOpenApi("OpenAIRE Contexts", ContextApiController.class.getPackage().getName());
}
@Bean
@ConditionalOnProperty(value = "openaire.exporter.enable.info", havingValue = "true")
public GroupedOpenApi info() {
return newGroupedOpenApi("OpenAIRE Info", InfoController.class.getPackage().getName());
}
private GroupedOpenApi newGroupedOpenApi(final String groupName, final String controllerPackage) {
return GroupedOpenApi.builder()
.group(groupName)
.packagesToScan(controllerPackage)
.build();
}
}

106
apps/dnet-exporter-api/src/main/java/eu/dnetlib/DNetOpenaireExporterConfiguration.java Normal file
View File

@ -0,0 +1,106 @@
package eu.dnetlib;
import javax.sql.DataSource;
import org.apache.commons.dbcp2.BasicDataSource;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.cxf.endpoint.Client;
import org.apache.cxf.frontend.ClientProxy;
import org.apache.cxf.jaxws.JaxWsProxyFactoryBean;
import org.apache.cxf.transport.http.HTTPConduit;
import org.apache.cxf.transports.http.configuration.HTTPClientPolicy;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import com.mongodb.MongoClient;
import com.mongodb.MongoClientOptions;
import com.mongodb.ServerAddress;
import eu.dnetlib.DnetOpenaireExporterProperties.Jdbc;
import eu.dnetlib.data.objectstore.rmi.ObjectStoreService;
import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpService;
import eu.dnetlib.enabling.is.registry.rmi.ISRegistryService;
/**
* Created by claudio on 07/07/2017.
*/
@Configuration
public class DNetOpenaireExporterConfiguration {
private static final Log log = LogFactory.getLog(DNetOpenaireExporterConfiguration.class);
@Autowired
private DnetOpenaireExporterProperties props;
@Bean
public ISLookUpService getLookUpService() {
return getServiceStub(ISLookUpService.class, props.getIsLookupUrl());
}
@Bean
public ObjectStoreService getObjectStoreService() {
return getServiceStub(ObjectStoreService.class, props.getObjectStoreServiceUrl());
}
@Bean
public ISRegistryService getRegistryService() {
return getServiceStub(ISRegistryService.class, props.getIsRegistryServiceUrl());
}
@SuppressWarnings("unchecked")
private <T> T getServiceStub(final Class<T> clazz, final String endpoint) {
log.info(String.format("Initializing service stub %s, endpoint %s", clazz.toString(), endpoint));
final JaxWsProxyFactoryBean jaxWsProxyFactory = new JaxWsProxyFactoryBean();
jaxWsProxyFactory.setServiceClass(clazz);
jaxWsProxyFactory.setAddress(endpoint);
final T service = (T) jaxWsProxyFactory.create();
final Client client = ClientProxy.getClient(service);
if (client != null) {
final HTTPConduit conduit = (HTTPConduit) client.getConduit();
final HTTPClientPolicy policy = new HTTPClientPolicy();
log.info(String.format("setting connectTimeout to %s, receiveTimeout to %s for service %s", props.getCxfClientConnectTimeout(), props
.getCxfClientReceiveTimeout(), clazz.getCanonicalName()));
policy.setConnectionTimeout(props.getCxfClientConnectTimeout());
policy.setReceiveTimeout(props.getCxfClientReceiveTimeout());
conduit.setClient(policy);
}
return service;
}
@Bean
public DataSource getSqlDataSource() {
final Jdbc jdbc = props.getJdbc();
return getDatasource(jdbc.getDriverClassName(), jdbc.getUrl(), jdbc.getUser(), jdbc.getPwd(), jdbc.getMinIdle(), jdbc.getMaxRows());
}
private BasicDataSource getDatasource(final String driverClassName,
final String jdbcUrl,
final String jdbcUser,
final String jdbcPwd,
final int jdbcMinIdle,
final int jdbcMaxIdle) {
final BasicDataSource d = new BasicDataSource();
d.setDriverClassName(driverClassName);
d.setUrl(jdbcUrl);
d.setUsername(jdbcUser);
d.setPassword(jdbcPwd);
d.setMinIdle(jdbcMinIdle);
d.setMaxIdle(jdbcMaxIdle);
return d;
}
@Bean
public MongoClient getMongoClient() {
return new MongoClient(
new ServerAddress(props.getDatasource().getMongoHost(), props.getDatasource().getMongoPort()),
MongoClientOptions.builder().connectionsPerHost(props.getDatasource().getMongoConnectionsPerHost()).build());
}
}

489
apps/dnet-exporter-api/src/main/java/eu/dnetlib/DnetOpenaireExporterProperties.java Normal file
View File

@ -0,0 +1,489 @@
package eu.dnetlib;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.context.annotation.Configuration;
import org.springframework.context.annotation.PropertySource;
import org.springframework.core.io.ClassPathResource;
import org.springframework.core.io.Resource;
/**
* Created by Alessia Bardi on 31/03/17.
*
* @author Alessia Bardi, Claudio Atzori
*/
@Configuration
@PropertySource("classpath:global.properties")
@ConfigurationProperties(prefix = "openaire.exporter")
public class DnetOpenaireExporterProperties {
// ISLOOKUP
private ClassPathResource findSolrIndexUrl;
private ClassPathResource findIndexDsInfo;
private ClassPathResource findObjectStore;
private ClassPathResource findFunderContexts;
private ClassPathResource findCommunityContexts;
private ClassPathResource findContextProfiles;
private ClassPathResource findContextProfilesByType;
private ClassPathResource getRepoProfile;
private String isLookupUrl;
private String objectStoreServiceUrl;
private String isRegistryServiceUrl;
private int requestWorkers = 100;
private int requestTimeout = 10;
private int cxfClientConnectTimeout = 120;
private int cxfClientReceiveTimeout = 120;
private Datasource datasource;
private Project project;
private Jdbc jdbc;
private Vocabularies vocabularies;
public static class Datasource {
// MONGODB
private String mongoHost;
private int mongoPort;
private String mongoCollectionName;
private String mongoDbName;
private int mongoConnectionsPerHost;
private int mongoQueryLimit;
public String getMongoHost() {
return mongoHost;
}
public void setMongoHost(final String mongoHost) {
this.mongoHost = mongoHost;
}
public int getMongoPort() {
return mongoPort;
}
public void setMongoPort(final int mongoPort) {
this.mongoPort = mongoPort;
}
public String getMongoCollectionName() {
return mongoCollectionName;
}
public void setMongoCollectionName(final String mongoCollectionName) {
this.mongoCollectionName = mongoCollectionName;
}
public String getMongoDbName() {
return mongoDbName;
}
public void setMongoDbName(final String mongoDbName) {
this.mongoDbName = mongoDbName;
}
public int getMongoConnectionsPerHost() {
return mongoConnectionsPerHost;
}
public void setMongoConnectionsPerHost(final int mongoConnectionsPerHost) {
this.mongoConnectionsPerHost = mongoConnectionsPerHost;
}
public int getMongoQueryLimit() {
return mongoQueryLimit;
}
public void setMongoQueryLimit(final int mongoQueryLimit) {
this.mongoQueryLimit = mongoQueryLimit;
}
}
public static class Project {
private int flushSize;
private String tsvFields;
private Resource projectsFundingQueryTemplate;
private Resource dspaceTemplate;
private Resource dspaceHeadTemplate;
private Resource dspaceTailTemplate;
private Resource eprintsTemplate;
public int getFlushSize() {
return flushSize;
}
public void setFlushSize(final int flushSize) {
this.flushSize = flushSize;
}
public String getTsvFields() {
return tsvFields;
}
public void setTsvFields(final String tsvFields) {
this.tsvFields = tsvFields;
}
public Resource getProjectsFundingQueryTemplate() {
return projectsFundingQueryTemplate;
}
public void setProjectsFundingQueryTemplate(final Resource projectsFundingQueryTemplate) {
this.projectsFundingQueryTemplate = projectsFundingQueryTemplate;
}
public Resource getDspaceTemplate() {
return dspaceTemplate;
}
public void setDspaceTemplate(final Resource dspaceTemplate) {
this.dspaceTemplate = dspaceTemplate;
}
public Resource getDspaceHeadTemplate() {
return dspaceHeadTemplate;
}
public void setDspaceHeadTemplate(final Resource dspaceHeadTemplate) {
this.dspaceHeadTemplate = dspaceHeadTemplate;
}
public Resource getDspaceTailTemplate() {
return dspaceTailTemplate;
}
public void setDspaceTailTemplate(final Resource dspaceTailTemplate) {
this.dspaceTailTemplate = dspaceTailTemplate;
}
public Resource getEprintsTemplate() {
return eprintsTemplate;
}
public void setEprintsTemplate(final Resource eprintsTemplate) {
this.eprintsTemplate = eprintsTemplate;
}
}
public static class Jdbc {
// JDBC
@Value("${spring.datasource.driverClassName}")
private String driverClassName;
private String url;
private String user;
private String pwd;
private int minIdle;
private int maxidle;
private int maxRows;
public String getDriverClassName() {
return driverClassName;
}
public String getUrl() {
return url;
}
public void setUrl(final String url) {
this.url = url;
}
public String getUser() {
return user;
}
public void setUser(final String user) {
this.user = user;
}
public String getPwd() {
return pwd;
}
public void setPwd(final String pwd) {
this.pwd = pwd;
}
public int getMinIdle() {
return minIdle;
}
public void setMinIdle(final int minIdle) {
this.minIdle = minIdle;
}
public int getMaxidle() {
return maxidle;
}
public void setMaxidle(final int maxidle) {
this.maxidle = maxidle;
}
public int getMaxRows() {
return maxRows;
}
public void setMaxRows(final int maxRows) {
this.maxRows = maxRows;
}
}
public static class Swagger {
private String apiTitle;
private String apiDescription;
private String apiLicense;
private String apiLicenseUrl;
private String apiContactName;
private String apiContactUrl;
private String apiContactEmail;
public String getApiTitle() {
return apiTitle;
}
public void setApiTitle(final String apiTitle) {
this.apiTitle = apiTitle;
}
public String getApiDescription() {
return apiDescription;
}
public void setApiDescription(final String apiDescription) {
this.apiDescription = apiDescription;
}
public String getApiLicense() {
return apiLicense;
}
public void setApiLicense(final String apiLicense) {
this.apiLicense = apiLicense;
}
public String getApiLicenseUrl() {
return apiLicenseUrl;
}
public void setApiLicenseUrl(final String apiLicenseUrl) {
this.apiLicenseUrl = apiLicenseUrl;
}
public String getApiContactName() {
return apiContactName;
}
public void setApiContactName(final String apiContactName) {
this.apiContactName = apiContactName;
}
public String getApiContactUrl() {
return apiContactUrl;
}
public void setApiContactUrl(final String apiContactUrl) {
this.apiContactUrl = apiContactUrl;
}
public String getApiContactEmail() {
return apiContactEmail;
}
public void setApiContactEmail(final String apiContactEmail) {
this.apiContactEmail = apiContactEmail;
}
}
public static class Vocabularies {
private String baseUrl;
private String countriesEndpoint;
private String datasourceTypologiesEndpoint;
public String getCountriesEndpoint() {
return countriesEndpoint;
}
public void setCountriesEndpoint(final String countriesEndpoint) {
this.countriesEndpoint = countriesEndpoint;
}
public String getBaseUrl() {
return baseUrl;
}
public void setBaseUrl(final String baseUrl) {
this.baseUrl = baseUrl;
}
public String getDatasourceTypologiesEndpoint() {
return datasourceTypologiesEndpoint;
}
public void setDatasourceTypologiesEndpoint(final String datasourceTypologiesEndpoint) {
this.datasourceTypologiesEndpoint = datasourceTypologiesEndpoint;
}
}
public ClassPathResource getFindSolrIndexUrl() {
return findSolrIndexUrl;
}
public void setFindSolrIndexUrl(final ClassPathResource findSolrIndexUrl) {
this.findSolrIndexUrl = findSolrIndexUrl;
}
public ClassPathResource getFindIndexDsInfo() {
return findIndexDsInfo;
}
public ClassPathResource getFindObjectStore() {
return findObjectStore;
}
public void setFindObjectStore(final ClassPathResource findObjectStore) {
this.findObjectStore = findObjectStore;
}
public void setFindIndexDsInfo(final ClassPathResource findIndexDsInfo) {
this.findIndexDsInfo = findIndexDsInfo;
}
public ClassPathResource getFindFunderContexts() {
return findFunderContexts;
}
public void setFindFunderContexts(final ClassPathResource findFunderContexts) {
this.findFunderContexts = findFunderContexts;
}
public ClassPathResource getFindCommunityContexts() {
return findCommunityContexts;
}
public ClassPathResource getFindContextProfiles() {
return findContextProfiles;
}
public ClassPathResource getFindContextProfilesByType() {
return findContextProfilesByType;
}
public void setFindContextProfiles(final ClassPathResource findContextProfiles) {
this.findContextProfiles = findContextProfiles;
}
public void setFindContextProfilesByType(final ClassPathResource findContextProfilesByType) {
this.findContextProfilesByType = findContextProfilesByType;
}
public void setFindCommunityContexts(final ClassPathResource findCommunityContexts) {
this.findCommunityContexts = findCommunityContexts;
}
public ClassPathResource getGetRepoProfile() {
return getRepoProfile;
}
public void setGetRepoProfile(final ClassPathResource getRepoProfile) {
this.getRepoProfile = getRepoProfile;
}
public String getIsLookupUrl() {
return isLookupUrl;
}
public void setIsLookupUrl(final String isLookupUrl) {
this.isLookupUrl = isLookupUrl;
}
public String getObjectStoreServiceUrl() {
return objectStoreServiceUrl;
}
public void setObjectStoreServiceUrl(final String objectStoreServiceUrl) {
this.objectStoreServiceUrl = objectStoreServiceUrl;
}
public String getIsRegistryServiceUrl() {
return isRegistryServiceUrl;
}
public void setIsRegistryServiceUrl(final String isRegistryServiceUrl) {
this.isRegistryServiceUrl = isRegistryServiceUrl;
}
public int getRequestWorkers() {
return requestWorkers;
}
public void setRequestWorkers(final int requestWorkers) {
this.requestWorkers = requestWorkers;
}
public int getRequestTimeout() {
return requestTimeout;
}
public void setRequestTimeout(final int requestTimeout) {
this.requestTimeout = requestTimeout;
}
public int getCxfClientConnectTimeout() {
return cxfClientConnectTimeout;
}
public void setCxfClientConnectTimeout(final int cxfClientConnectTimeout) {
this.cxfClientConnectTimeout = cxfClientConnectTimeout;
}
public int getCxfClientReceiveTimeout() {
return cxfClientReceiveTimeout;
}
public void setCxfClientReceiveTimeout(final int cxfClientReceiveTimeout) {
this.cxfClientReceiveTimeout = cxfClientReceiveTimeout;
}
public Datasource getDatasource() {
return datasource;
}
public void setDatasource(final Datasource datasource) {
this.datasource = datasource;
}
public Project getProject() {
return project;
}
public void setProject(final Project project) {
this.project = project;
}
public Jdbc getJdbc() {
return jdbc;
}
public void setJdbc(final Jdbc jdbc) {
this.jdbc = jdbc;
}
public Vocabularies getVocabularies() {
return vocabularies;
}
public void setVocabularies(final Vocabularies vocabularies) {
this.vocabularies = vocabularies;
}
}

16
apps/dnet-exporter-api/src/main/java/eu/dnetlib/SwaggerController.java Normal file
View File

@ -0,0 +1,16 @@
package eu.dnetlib;
import org.springframework.stereotype.Controller;
import org.springframework.web.bind.annotation.RequestMapping;
@Controller
public class SwaggerController {
@RequestMapping(value = {
"/", "/docs", "swagger-ui.html", "swagger-ui/"
})
public String index() {
return "redirect:swagger-ui/index.html";
}
}

118
apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/common/AbstractExporterController.java Normal file
View File

@ -0,0 +1,118 @@
package eu.dnetlib.openaire.common;
import java.util.List;
import java.util.stream.Collectors;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.exception.ExceptionUtils;
import org.apache.commons.lang3.time.StopWatch;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.springframework.http.HttpStatus;
import org.springframework.web.bind.MethodArgumentNotValidException;
import org.springframework.web.bind.annotation.ExceptionHandler;
import org.springframework.web.bind.annotation.ResponseBody;
import org.springframework.web.bind.annotation.ResponseStatus;
import com.fasterxml.jackson.annotation.JsonAutoDetect;
import eu.dnetlib.enabling.datasources.common.DsmException;
import eu.dnetlib.enabling.datasources.common.DsmForbiddenException;
import eu.dnetlib.enabling.datasources.common.DsmNotFoundException;
import eu.dnetlib.openaire.dsm.domain.Response;
/**
* Created by claudio on 18/07/2017.
*/
public abstract class AbstractExporterController {
private static final Log log = LogFactory.getLog(AbstractExporterController.class); // NOPMD by marko on 11/24/08 5:02 PM
@ResponseBody
@ExceptionHandler({
DsmException.class
})
@ResponseStatus(value = HttpStatus.INTERNAL_SERVER_ERROR)
public ErrorMessage handleDSMException(final Exception e) {
return _handleError(e);
}
@ResponseBody
@ExceptionHandler(DsmForbiddenException.class)
@ResponseStatus(value = HttpStatus.FORBIDDEN)
public ErrorMessage handleForbiddenException(final Exception e) {
return _handleError(e);
}
@ResponseBody
@ExceptionHandler({
DsmNotFoundException.class
})
@ResponseStatus(value = HttpStatus.NOT_FOUND)
public ErrorMessage handleNotFoundException(final Exception e) {
return _handleError(e);
}
@ResponseBody
@ExceptionHandler(MethodArgumentNotValidException.class)
@ResponseStatus(HttpStatus.BAD_REQUEST)
public List<ErrorMessage> processValidationError(final MethodArgumentNotValidException e) {
return e.getBindingResult()
.getFieldErrors()
.stream()
.map(fe -> new ErrorMessage(
String.format("field '%s'", fe.getField()),
String.format("rejected value '%s'", fe.getRejectedValue()),
fe.getDefaultMessage()))
.collect(Collectors.toList());
}
private ErrorMessage _handleError(final Exception e) {
log.error(e);
if (StringUtils.containsIgnoreCase(ExceptionUtils.getRootCauseMessage(e), "Broken pipe")) {
return null; // socket is closed, cannot return any response
} else {
return new ErrorMessage(e);
}
}
// HELPERS
protected <T extends Response> T prepareResponse(final int page, final int size, final StopWatch stopWatch, final T rsp) {
rsp.getHeader()
.setTime(stopWatch.getTime())
.setPage(page)
.setSize(size);
return rsp;
}
@JsonAutoDetect
public class ErrorMessage {
private final String message;
private final String details;
private final String stacktrace;
public ErrorMessage(final Exception e) {
this(e.getMessage(), "", ExceptionUtils.getStackTrace(e));
}
public ErrorMessage(final String message, final String details, final String stacktrace) {
this.message = message;
this.details = details;
this.stacktrace = stacktrace;
}
public String getMessage() {
return this.message;
}
public String getStacktrace() {
return this.stacktrace;
}
public String getDetails() {
return details;
}
}
}

58
apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/common/ConverterTextArray.java Normal file
View File

@ -0,0 +1,58 @@
package eu.dnetlib.openaire.common;
import java.sql.Array;
import java.sql.Connection;
import java.sql.SQLException;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
import javax.persistence.AttributeConverter;
import javax.persistence.Converter;
import javax.sql.DataSource;
import org.springframework.beans.BeansException;
import org.springframework.context.ApplicationContext;
import org.springframework.context.ApplicationContextAware;
/**
* Created by claudio on 05/07/2017.
*/
@Converter
public class ConverterTextArray implements AttributeConverter<List<String>, Array>, ApplicationContextAware {
private ApplicationContext applicationContext;
@Override
public Array convertToDatabaseColumn(List<String> attribute) {
final Map<String, DataSource> datasources = applicationContext.getBeansOfType(DataSource.class);
DataSource source = datasources.values().stream().findFirst().get();
try {
Connection conn = source.getConnection();
return conn.createArrayOf("text", attribute.toArray());
} catch (SQLException e) {
e.printStackTrace();
}
return null;
}
@Override
public List<String> convertToEntityAttribute(Array dbData) {
try {
return Arrays.stream((Object[]) dbData.getArray()).map(d -> (String) d).collect(Collectors.toList());
} catch (SQLException e) {
e.printStackTrace();
return null;
}
}
@Override
public void setApplicationContext(final ApplicationContext applicationContext) throws BeansException {
this.applicationContext = applicationContext;
}
}

30
apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/common/ExporterConstants.java Normal file
View File

@ -0,0 +1,30 @@
package eu.dnetlib.openaire.common;
public class ExporterConstants {
/*
* Tags used to group the operations on the swagger UI
*/
public final static String C = "Community";
public final static String C_CP = "Community content providers";
public final static String C_PJ = "Community projects";
public final static String C_ZC = "Community Zenodo Communities";
public final static String C_O = "Community Organizations";
public final static String DS = "Datasource";
public final static String API = "Interface";
public final static String R = "Read";
public final static String W = "Write";
public final static String D = "Deprecated";
public final static String M = "Management";
public final static String DSPACE = "DSpace";
public final static String EPRINT = "EPrints";
public final static String TSV = "TSV";
public final static String STREAMING = "Streaming";
public static final String OAI = "oai";
public static final String SET = "set";
}

106
apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/common/GenericArrayUserType.java Normal file
View File

@ -0,0 +1,106 @@
package eu.dnetlib.openaire.common;
import java.io.Serializable;
import java.sql.*;
import org.hibernate.HibernateException;
import org.hibernate.engine.spi.SharedSessionContractImplementor;
import org.hibernate.usertype.UserType;
/**
* Created by claudio on 05/07/2017.
*/
public class GenericArrayUserType<T extends Serializable> implements UserType {
protected static final int[] SQL_TYPES = { Types.ARRAY };
private Class<T> typeParameterClass;
@Override
public Object assemble(Serializable cached, Object owner) throws HibernateException {
return this.deepCopy(cached);
}
@Override
public Object deepCopy(Object value) throws HibernateException {
return value;
}
@SuppressWarnings("unchecked")
@Override
public Serializable disassemble(Object value) throws HibernateException {
return (T) this.deepCopy(value);
}
@Override
public boolean equals(Object x, Object y) throws HibernateException {
if (x == null) {
return y == null;
}
return x.equals(y);
}
@Override
public int hashCode(Object x) throws HibernateException {
return x.hashCode();
}
@Override
public Object nullSafeGet(final ResultSet resultSet,
final String[] names,
final SharedSessionContractImplementor sharedSessionContractImplementor,
final Object o)
throws HibernateException, SQLException {
if (resultSet.wasNull()) {
return null;
}
if (resultSet.getArray(names[0]) == null) {
return new Integer[0];
}
Array array = resultSet.getArray(names[0]);
@SuppressWarnings("unchecked")
T javaArray = (T) array.getArray();
return javaArray;
}
@Override
public void nullSafeSet(final PreparedStatement statement,
final Object value,
final int index,
final SharedSessionContractImplementor session)
throws HibernateException, SQLException {
Connection connection = statement.getConnection();
if (value == null) {
statement.setNull(index, SQL_TYPES[0]);
} else {
@SuppressWarnings("unchecked")
T castObject = (T) value;
Array array = connection.createArrayOf("integer", (Object[]) castObject);
statement.setArray(index, array);
}
}
@Override
public boolean isMutable() {
return true;
}
@Override
public Object replace(Object original, Object target, Object owner) throws HibernateException {
return original;
}
@Override
public Class<T> returnedClass() {
return typeParameterClass;
}
@Override
public int[] sqlTypes() {
return new int[] { Types.ARRAY };
}
}

48
apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/common/ISClient.java Normal file
View File

@ -0,0 +1,48 @@
package eu.dnetlib.openaire.common;
import java.io.IOException;
import java.util.List;
import java.util.Map;
import eu.dnetlib.enabling.datasources.common.DsmException;
import eu.dnetlib.openaire.context.Context;
import eu.dnetlib.openaire.dsm.dao.utils.IndexDsInfo;
public interface ISClient {
IndexDsInfo calculateCurrentIndexDsInfo() throws DsmException;
String getObjectStoreId(String dsId) throws DsmException;
Map<String, Context> getFunderContextMap() throws IOException;
Map<String, Context> getCommunityContextMap() throws IOException;
Map<String, Context> getContextMap(final List<String> type) throws IOException;
void updateContextParam(String id, String name, String value);
void updateContextAttribute(String id, String name, String value);
void addConcept(String id, String categoryId, String data);
void removeConcept(String id, String categoryId, String conceptId);
void dropCache();
/**
*
* @param id
* id of the concept to be updated (i.e. ni::projects::2)
* @param name
* name of the attribute to be updated
* @param value
* new value for the attribute
*/
void updateConceptAttribute(String id, String name, String value);
void updateConceptParam(String id, String name, String value);
void updateConceptParamNoEscape(String id, String name, String value);
}

297
apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/common/ISClientImpl.java Normal file
View File

@ -0,0 +1,297 @@
package eu.dnetlib.openaire.common;
import static eu.dnetlib.openaire.common.Utils.escape;
import java.io.IOException;
import java.nio.charset.Charset;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Queue;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.function.Function;
import java.util.stream.Collectors;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.cache.annotation.CacheEvict;
import org.springframework.cache.annotation.Cacheable;
import org.springframework.core.io.ClassPathResource;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Component;
import com.google.common.collect.Lists;
import com.google.common.escape.Escaper;
import com.google.common.xml.XmlEscapers;
import eu.dnetlib.DnetOpenaireExporterProperties;
import eu.dnetlib.enabling.datasources.common.DsmException;
import eu.dnetlib.enabling.datasources.common.DsmRuntimeException;
import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpException;
import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpService;
import eu.dnetlib.openaire.context.Context;
import eu.dnetlib.openaire.context.ContextMappingUtils;
import eu.dnetlib.openaire.dsm.dao.utils.IndexDsInfo;
/**
* Created by claudio on 20/10/2016.
*/
@Component
public class ISClientImpl implements ISClient {
private static final Log log = LogFactory.getLog(ISClientImpl.class);
@Autowired
private DnetOpenaireExporterProperties config;
@Autowired
private ISLookUpService isLookUpService;
@Override
@Cacheable("indexdsinfo-cache")
public IndexDsInfo calculateCurrentIndexDsInfo() throws DsmException {
log.warn("calculateCurrentIndexDsInfo(): not using cache");
final String[] arr;
try {
arr = _isLookUp(_getQuery(config.getFindIndexDsInfo())).split("@@@");
return new IndexDsInfo(
_isLookUp(_getQuery(config.getFindSolrIndexUrl())),
arr[0].trim(), arr[1].trim(), arr[2].trim());
} catch (IOException | ISLookUpException e) {
throw new DsmException("unable fetch index DS information from IS");
}
}
@Override
@Cacheable("objectstoreid-cache")
public String getObjectStoreId(final String dsId) throws DsmException {
log.warn(String.format("getObjectStoreId(%s): not using cache", dsId));
try {
final String xqueryTemplate = _getQuery(config.getFindObjectStore());
return _isLookUp(String.format(xqueryTemplate, dsId));
} catch (IOException | ISLookUpException e) {
throw new DsmException("unble to find objectstore for ds " + dsId);
}
}
@Override
@Cacheable("context-cache-funder")
public Map<String, Context> getFunderContextMap() throws IOException {
return _processContext(_getQuery(config.getFindFunderContexts()));
}
@Override
@Cacheable("context-cache-community")
public Map<String, Context> getCommunityContextMap() throws IOException {
return _processContext(_getQuery(config.getFindCommunityContexts()));
}
@Override
@Cacheable("context-cache")
public Map<String, Context> getContextMap(final List<String> type) throws IOException {
if (Objects.isNull(type) || type.isEmpty()) {
return _processContext(_getQuery(config.getFindContextProfiles()));
} else {
final String xqueryTemplate = _getQuery(config.getFindContextProfilesByType());
final String xquery = String.format(xqueryTemplate, type.stream()
.map(t -> String.format("./RESOURCE_PROFILE/BODY/CONFIGURATION/context/@type = '%s'", t))
.collect(Collectors.joining(" or ")));
return _processContext(xquery);
}
}
@Override
@CacheEvict(value = {
"context-cache", "context-cache-funder"
}, allEntries = true)
public void updateContextParam(final String id, final String name, final String value) {
try {
_quickSeachProfile(getXQuery(id, name, value));
} catch (final ISLookUpException e) {
throw new DsmRuntimeException(String.format("unable update context param [id: %s, name: %s, value: %s]", id, name, value), e);
}
}
@Override
@CacheEvict(value = {
"context-cache", "context-cache-funder"
}, allEntries = true)
public void updateContextAttribute(final String id, final String name, final String value) {
final Escaper esc = XmlEscapers.xmlAttributeEscaper();
try {
_quickSeachProfile(String.format("update value collection('/db/DRIVER/ContextDSResources/ContextDSResourceType')" +
"/RESOURCE_PROFILE/BODY/CONFIGURATION/context[./@id = '%s']/@%s with '%s'", id, name, escape(esc, value)));
} catch (final ISLookUpException e) {
throw new DsmRuntimeException(String.format("unable update context attribute [id: %s, name: %s, data: %s]", id, name, value), e);
}
}
@Override
@CacheEvict(value = {
"context-cache", "context-cache-funder"
}, allEntries = true)
public void addConcept(final String id, final String categoryId, final String data) {
try {
_quickSeachProfile(String.format("update insert %s into collection('/db/DRIVER/ContextDSResources/ContextDSResourceType')" +
"/RESOURCE_PROFILE/BODY/CONFIGURATION/context[./@id = '%s']/category[./@id = '%s']", data, id, categoryId));
} catch (final ISLookUpException e) {
throw new DsmRuntimeException(String.format("unable add concept [id: %s, categoryId: %s, data: %s]", id, categoryId, data), e);
}
}
@Override
@CacheEvict(value = {
"context-cache", "context-cache-funder"
}, allEntries = true)
public void removeConcept(final String id, final String categoryId, final String conceptId) {
try {
_quickSeachProfile(String.format("for $concept in collection('/db/DRIVER/ContextDSResources/ContextDSResourceType')" +
"/RESOURCE_PROFILE/BODY/CONFIGURATION/context[./@id = '%s']" +
"/category[./@id = '%s']/concept[./@id = '%s'] " +
"return update delete $concept", id, categoryId, conceptId));
} catch (final ISLookUpException e) {
throw new DsmRuntimeException(String.format("unable remove concept [id: %s, categoryId: %s, conceptId: %s]", id, categoryId, conceptId), e);
}
}
@Override
@CacheEvict(value = {
"context-cache", "context-cache-community", "context-cache-funder"
}, allEntries = true)
public void updateConceptAttribute(final String id, final String name, final String value) {
final Escaper esc = XmlEscapers.xmlAttributeEscaper();
try {
_quickSeachProfile(String.format("update value collection('/db/DRIVER/ContextDSResources/ContextDSResourceType')" +
"/RESOURCE_PROFILE/BODY/CONFIGURATION/context/category/concept[./@id = '%s']/@%s with '%s'", id, name, escape(esc, value)));
} catch (final ISLookUpException e) {
throw new DsmRuntimeException(String.format("unable update concept attribute [id: %s, name: %s, value: %s]", id, name, value), e);
}
}
@Override
@CacheEvict(value = {
"context-cache", "context-cache-funder"
}, allEntries = true)
public void updateConceptParam(final String id, final String name, final String value) {
try {
_quickSeachProfile(getConceptXQuery(id, name, value));
} catch (final ISLookUpException e) {
throw new DsmRuntimeException(String.format("unable update concept param [id: %s, name: %s, value: %s]", id, name, value), e);
}
}
@Override
@CacheEvict(value = {
"context-cache", "context-cache-funder"
}, allEntries = true)
public void updateConceptParamNoEscape(final String id, final String name, final String value) {
try {
_quickSeachProfile(getConceptXQueryNoEscape(id, name, value));
} catch (final ISLookUpException e) {
throw new DsmRuntimeException(String.format("unable update concept param [id: %s, name: %s, value: %s]", id, name, value), e);
}
}
/// HELPERS
private String getXQuery(final String id, final String name, final String value) {
final Escaper esc = XmlEscapers.xmlContentEscaper();
if (StringUtils.isNotBlank(value)) {
return String.format("update replace collection('/db/DRIVER/ContextDSResources/ContextDSResourceType')" +
"/RESOURCE_PROFILE/BODY/CONFIGURATION/context[./@id = '%s']/param[./@name = '%s'] with <param name='%s'>%s</param>", id, name, name, escape(esc, value));
} else {
return String.format("update replace collection('/db/DRIVER/ContextDSResources/ContextDSResourceType')" +
"/RESOURCE_PROFILE/BODY/CONFIGURATION/context[./@id = '%s']/param[./@name = '%s'] with <param name='%s'/>", id, name, name);
}
}
private String getConceptXQuery(final String id, final String name, final String value) {
final Escaper esc = XmlEscapers.xmlContentEscaper();
if (StringUtils.isNotBlank(value)) {
return String.format("update replace collection('/db/DRIVER/ContextDSResources/ContextDSResourceType')//" +
"concept[./@id = '%s']/param[./@name = '%s'] with <param name='%s'>%s</param>", id, name, name, escape(esc, value));
} else {
return String
.format("update replace collection('/db/DRIVER/ContextDSResources/ContextDSResourceType')//concept[./@id = '%s']/param[./@name = '%s'] with <param name='%s'/>", id, name, name);
}
}
private String getConceptXQueryNoEscape(final String id, final String name, final String value) {
if (StringUtils.isNotBlank(value)) {
return String.format("update replace collection('/db/DRIVER/ContextDSResources/ContextDSResourceType')//" +
"concept[./@id = '%s']/param[./@name = '%s'] with <param name='%s'>%s</param>", id, name, name, value);
} else {
return String
.format("update replace collection('/db/DRIVER/ContextDSResources/ContextDSResourceType')//concept[./@id = '%s']/param[./@name = '%s'] with <param name='%s'/>", id, name, name);
}
}
private Map<String, Context> _processContext(final String xquery) throws IOException {
return _processContext(new LinkedBlockingQueue<>(), xquery);
}
private Map<String, Context> _processContext(final Queue<Throwable> errors, final String xquery) throws IOException {
try {
return getContextProfiles(errors, xquery).stream()
.filter(StringUtils::isNotBlank)
.map(s -> ContextMappingUtils.parseContext(s, errors))
.collect(Collectors.toMap(Context::getId, Function.identity(), (c1, c2) -> {
log.warn(String.format("found duplicate context profile '%s'", c1.getId()));
return c1;
}));
} finally {
if (!errors.isEmpty()) {
log.error(errors);
errors.forEach(Throwable::printStackTrace);
}
}
}
private List<String> getContextProfiles(final Queue<Throwable> errors, final String xquery) throws IOException {
log.warn("getContextProfiles(): not using cache");
try {
return _quickSeachProfile(xquery);
} catch (final ISLookUpException e) {
throw new DsmRuntimeException("unable to get context profiles", e);
}
}
private String _getQuery(final ClassPathResource resource) throws IOException {
return IOUtils.toString(resource.getInputStream(), Charset.defaultCharset());
}
private String _isLookUp(final String xquery) throws ISLookUpException {
log.debug(String.format("running xquery:\n%s", xquery));
// log.debug(String.format("query result: %s", res));
return isLookUpService.getResourceProfileByQuery(xquery);
}
private List<String> _quickSeachProfile(final String xquery) throws ISLookUpException {
final List<String> res = Lists.newArrayList();
log.debug(String.format("running xquery:\n%s", xquery));
final List<String> list = isLookUpService.quickSearchProfile(xquery);
if (list != null) {
res.addAll(list);
}
log.debug(String.format("query result size: %s", res.size()));
return res;
}
@Override
@CacheEvict(cacheNames = {
"context-cache", "indexdsinfo-cache", "objectstoreid-cache"
}, allEntries = true)
@Scheduled(fixedDelayString = "${openaire.exporter.cache.ttl}")
public void dropCache() {
log.debug("dropped dsManager IS cache");
}
}

60
apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/common/OperationManager.java Normal file
View File

@ -0,0 +1,60 @@
package eu.dnetlib.openaire.common;
import java.util.List;
import java.util.concurrent.*;
import javax.annotation.PostConstruct;
import javax.annotation.PreDestroy;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.springframework.stereotype.Component;
@Component
public class OperationManager {
private static final Log log = LogFactory.getLog(OperationManager.class);
private static final long SLEEP_TIME = 1000;
private static final int Q_SIZE = 100;
private static final int POOL_SIZE = 5;
private final BlockingQueue<Runnable> ops = new ArrayBlockingQueue<>(Q_SIZE);
private ExecutorService executor;
@PostConstruct
public void init() {
executor = getExecutor();
}
public int dropAll() {
final List<Runnable> lostOperations = executor.shutdownNow();
log.warn(String.format("discarding %s operations", lostOperations.size()));
executor = getExecutor();
return lostOperations.size();
}
public int getOpSize() {
return ops.size();
}
public void addOperation(final Runnable op) {
executor.execute(op);
}
@PreDestroy
public void tearDown() throws InterruptedException {
executor.shutdown();
final boolean done = executor.awaitTermination(SLEEP_TIME, TimeUnit.MILLISECONDS);
log.debug(String.format("All operations were completed so far? %s", done));
}
// HELPERS
private ThreadPoolExecutor getExecutor() {
return new ThreadPoolExecutor(POOL_SIZE, POOL_SIZE,0L, TimeUnit.MILLISECONDS, ops);
}
}

63
apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/common/RFC3339DateFormat.java Normal file
View File

@ -0,0 +1,63 @@
package eu.dnetlib.openaire.common;
import java.text.FieldPosition;
import java.util.Calendar;
import java.util.Date;
import java.util.GregorianCalendar;
import java.util.Locale;
import java.util.TimeZone;
import com.fasterxml.jackson.databind.util.StdDateFormat;
public class RFC3339DateFormat extends StdDateFormat {
/**
*
*/
private static final long serialVersionUID = 8174507696046505992L;
private static final TimeZone TIMEZONE_Z = TimeZone.getTimeZone("UTC");
// Same as ISO8601DateFormat but serializing milliseconds.
@Override
public StringBuffer format(final Date date, final StringBuffer toAppendTo, final FieldPosition fieldPosition) {
final String value = format(date, true, TIMEZONE_Z, Locale.US);
toAppendTo.append(value);
return toAppendTo;
}
/**
* Format date into yyyy-MM-ddThh:mm:ss[.sss][Z|[+-]hh:mm]
*
* @param date
* the date to format
* @param millis
* true to include millis precision otherwise false
* @param tz
* timezone to use for the formatting (UTC will produce 'Z')
* @return the date formatted as yyyy-MM-ddThh:mm:ss[.sss][Z|[+-]hh:mm]
*/
private static String format(final Date date, final boolean millis, final TimeZone tz, final Locale loc) {
final Calendar calendar = new GregorianCalendar(tz, loc);
calendar.setTime(date);
// estimate capacity of buffer as close as we can (yeah, that's pedantic ;)
final StringBuilder sb = new StringBuilder(30);
sb.append(String.format("%04d-%02d-%02dT%02d:%02d:%02d", calendar.get(Calendar.YEAR), calendar.get(Calendar.MONTH) + 1, calendar
.get(Calendar.DAY_OF_MONTH), calendar.get(Calendar.HOUR_OF_DAY), calendar.get(Calendar.MINUTE), calendar.get(Calendar.SECOND)));
if (millis) {
sb.append(String.format(".%03d", calendar.get(Calendar.MILLISECOND)));
}
final int offset = tz.getOffset(calendar.getTimeInMillis());
if (offset != 0) {
final int hours = Math.abs(offset / (60 * 1000) / 60);
final int minutes = Math.abs(offset / (60 * 1000) % 60);
sb.append(String.format("%c%02d:%02d", offset < 0 ? '-' : '+', hours, minutes));
} else {
sb.append('Z');
}
return sb.toString();
}
}

22
apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/common/Utils.java Normal file
View File

@ -0,0 +1,22 @@
package eu.dnetlib.openaire.common;
import java.util.Iterator;
import java.util.Spliterator;
import java.util.Spliterators;
import java.util.stream.Stream;
import java.util.stream.StreamSupport;
import com.google.common.escape.Escaper;
import org.apache.commons.lang3.StringUtils;
public class Utils {
public static <T> Stream<T> stream(Iterator<T> iterator) {
return StreamSupport.stream(Spliterators.spliteratorUnknownSize(iterator, Spliterator.ORDERED), false);
}
public static String escape(final Escaper esc, final String value) {
return StringUtils.isNotBlank(value) ? esc.escape(value) : "";
}
}

347
apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityApiController.java Normal file
View File

@ -0,0 +1,347 @@
package eu.dnetlib.openaire.community;
import static eu.dnetlib.openaire.common.ExporterConstants.C;
import static eu.dnetlib.openaire.common.ExporterConstants.C_CP;
import static eu.dnetlib.openaire.common.ExporterConstants.C_O;
import static eu.dnetlib.openaire.common.ExporterConstants.C_PJ;
import static eu.dnetlib.openaire.common.ExporterConstants.C_ZC;
import static eu.dnetlib.openaire.common.ExporterConstants.R;
import static eu.dnetlib.openaire.common.ExporterConstants.W;
import java.util.List;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.web.bind.annotation.CrossOrigin;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestMethod;
import org.springframework.web.bind.annotation.RestController;
import io.swagger.v3.oas.annotations.Operation;
import io.swagger.v3.oas.annotations.responses.ApiResponse;
import io.swagger.v3.oas.annotations.responses.ApiResponses;
import io.swagger.v3.oas.annotations.tags.Tag;
@RestController
@CrossOrigin(origins = {
"*"
})
@ConditionalOnProperty(value = "openaire.exporter.enable.community", havingValue = "true")
@Tag(name = "OpenAIRE Communities API", description = "the OpenAIRE Community API")
public class CommunityApiController {
@Autowired
private CommunityApiCore communityApiCore;
@RequestMapping(value = "/community/communities", produces = {
"application/json"
}, method = RequestMethod.GET)
@Operation(summary = "get all community profiles", description = "get all community profiles", tags = {
C, R
})
@ApiResponses(value = {
@ApiResponse(responseCode = "200", description = "OK"),
@ApiResponse(responseCode = "500", description = "unexpected error")
})
public List<CommunitySummary> listCommunities() throws CommunityException {
return communityApiCore.listCommunities();
}
@RequestMapping(value = "/community/{id}", produces = {
"application/json"
}, method = RequestMethod.GET)
@Operation(summary = "get community profile", description = "get community profile", tags = {
C, R
})
@ApiResponses(value = {
@ApiResponse(responseCode = "200", description = "OK"),
@ApiResponse(responseCode = "404", description = "not found"),
@ApiResponse(responseCode = "500", description = "unexpected error")
})
public CommunityDetails getCommunity(@PathVariable final String id) throws CommunityException, CommunityNotFoundException {
return communityApiCore.getCommunity(id);
}
@RequestMapping(value = "/community/{id}", produces = {
"application/json"
}, method = RequestMethod.POST)
@Operation(summary = "update community details", description = "update community details", tags = {
C, R
})
@ApiResponses(value = {
@ApiResponse(responseCode = "200", description = "OK"),
@ApiResponse(responseCode = "404", description = "not found"),
@ApiResponse(responseCode = "500", description = "unexpected error")
})
public void setCommunity(
@PathVariable final String id,
@RequestBody final CommunityWritableProperties properties) throws CommunityException, CommunityNotFoundException {
communityApiCore.setCommunity(id, properties);
}
@RequestMapping(value = "/community/{id}/projects", produces = {
"application/json"
}, method = RequestMethod.GET)
@Operation(summary = "get community projects", description = "get community projects", tags = {
C_PJ, R
})
@ApiResponses(value = {
@ApiResponse(responseCode = "200", description = "OK"),
@ApiResponse(responseCode = "404", description = "not found"),
@ApiResponse(responseCode = "500", description = "unexpected error")
})
public List<CommunityProject> getCommunityProjects(@PathVariable final String id) throws CommunityException, CommunityNotFoundException {
return communityApiCore.getCommunityProjects(id);
}
@RequestMapping(value = "/community/{id}/projects", produces = {
"application/json"
}, method = RequestMethod.POST)
@Operation(summary = "associate a project to the community", description = "associate a project to the community", tags = {
C_PJ, W
})
@ApiResponses(value = {
@ApiResponse(responseCode = "200", description = "OK"),
@ApiResponse(responseCode = "404", description = "not found"),
@ApiResponse(responseCode = "500", description = "unexpected error")
})
public CommunityProject addCommunityProject(
@PathVariable final String id,
@RequestBody final CommunityProject project) throws CommunityException, CommunityNotFoundException {
return communityApiCore.addCommunityProject(id, project);
}
@RequestMapping(value = "/community/{id}/projects", produces = {
"application/json"
}, method = RequestMethod.DELETE)
@Operation(summary = "remove a project from the community", description = "remove a project from the community", tags = {
C_PJ, W
})
@ApiResponses(value = {
@ApiResponse(responseCode = "200", description = "OK"),
@ApiResponse(responseCode = "404", description = "not found"),
@ApiResponse(responseCode = "500", description = "unexpected error")
})
public void deleteCommunityProject(
@PathVariable final String id,
@RequestBody final Integer projectId) throws CommunityException, CommunityNotFoundException {
communityApiCore.removeCommunityProject(id, projectId);
}
@RequestMapping(value = "/community/{id}/contentproviders", produces = {
"application/json"
}, method = RequestMethod.GET)
@Operation(summary = "get the list of content providers associated to a given community", description = "get the list of content providers associated to a given community", tags = {
C_CP, R
})
@ApiResponses(value = {
@ApiResponse(responseCode = "200", description = "OK"),
@ApiResponse(responseCode = "404", description = "not found"),
@ApiResponse(responseCode = "500", description = "unexpected error")
})
public List<CommunityContentprovider> getCommunityContentproviders(@PathVariable final String id) throws CommunityException, CommunityNotFoundException {
return communityApiCore.getCommunityContentproviders(id);
}
@RequestMapping(value = "/community/{id}/contentproviders", produces = {
"application/json"
}, method = RequestMethod.POST)
@Operation(summary = "associate a content provider to the community", description = "associate a content provider to the community", tags = {
C_CP, W
})
@ApiResponses(value = {
@ApiResponse(responseCode = "200", description = "OK"),
@ApiResponse(responseCode = "404", description = "not found"),
@ApiResponse(responseCode = "500", description = "unexpected error")
})
public CommunityContentprovider addCommunityContentprovider(
@PathVariable final String id,
@RequestBody final CommunityContentprovider contentprovider) throws CommunityException, CommunityNotFoundException {
return communityApiCore.addCommunityContentprovider(id, contentprovider);
}
@RequestMapping(value = "/community/{id}/contentproviders", produces = {
"application/json"
}, method = RequestMethod.DELETE)
@Operation(summary = "remove the association between a content provider and the community", description = "remove the association between a content provider and the community", tags = {
C_CP, W
})
@ApiResponses(value = {
@ApiResponse(responseCode = "200", description = "OK"),
@ApiResponse(responseCode = "404", description = "not found"),
@ApiResponse(responseCode = "500", description = "unexpected error")
})
public void removeCommunityContentprovider(
@PathVariable final String id,
@RequestBody final Integer contentproviderId) throws CommunityException, CommunityNotFoundException {
communityApiCore.removeCommunityContentProvider(id, contentproviderId);
}
// ADDING CODE FOR COMMUNITY ORGANIZATIONS
@RequestMapping(value = "/community/{id}/organizations", produces = {
"application/json"
}, method = RequestMethod.GET)
@Operation(summary = "get the list of organizations for a given community", description = "get the list of organizations for a given community", tags = {
C_O, R
})
@ApiResponses(value = {
@ApiResponse(responseCode = "200", description = "OK"),
@ApiResponse(responseCode = "404", description = "not found"),
@ApiResponse(responseCode = "500", description = "unexpected error")
})
public List<CommunityOrganization> getCommunityOrganizations(@PathVariable final String id) throws CommunityException, CommunityNotFoundException {
return communityApiCore.getCommunityOrganizations(id);
}
@RequestMapping(value = "/community/{id}/organizations", produces = {
"application/json"
}, method = RequestMethod.POST)
@Operation(summary = "associate an organization to the community", description = "associate an organization to the community", tags = {
C_O, W
})
@ApiResponses(value = {
@ApiResponse(responseCode = "200", description = "OK"),
@ApiResponse(responseCode = "404", description = "not found"),
@ApiResponse(responseCode = "500", description = "unexpected error")
})
public CommunityOrganization addCommunityOrganization(
@PathVariable final String id,
@RequestBody final CommunityOrganization organization) throws CommunityException, CommunityNotFoundException {
return communityApiCore.addCommunityOrganization(id, organization);
}
@RequestMapping(value = "/community/{id}/organizations", produces = {
"application/json"
}, method = RequestMethod.DELETE)
@Operation(summary = "remove the association between an organization and the community", description = "remove the association between an organization and the community", tags = {
C_O, W
})
@ApiResponses(value = {
@ApiResponse(responseCode = "200", description = "OK"),
@ApiResponse(responseCode = "404", description = "not found"),
@ApiResponse(responseCode = "500", description = "unexpected error")
})
public void removeCommunityOrganization(
@PathVariable final String id,
@RequestBody final Integer organizationId) throws CommunityException, CommunityNotFoundException {
communityApiCore.removeCommunityOrganization(id, organizationId);
}
// **********************
@RequestMapping(value = "/community/{id}/subjects", produces = {
"application/json"
}, method = RequestMethod.POST)
@Operation(summary = "associate a subject to the community", description = "associate a subject to the community", tags = {
C, W
})
@ApiResponses(value = {
@ApiResponse(responseCode = "200", description = "OK"),
@ApiResponse(responseCode = "404", description = "not found"),
@ApiResponse(responseCode = "500", description = "unexpected error")
})
public CommunityDetails addCommunitySubjects(
@PathVariable final String id,
@RequestBody final List<String> subjects) throws CommunityException, CommunityNotFoundException {
return communityApiCore.addCommunitySubjects(id, subjects);
}
@RequestMapping(value = "/community/{id}/subjects", produces = {
"application/json"
}, method = RequestMethod.DELETE)
@Operation(summary = "remove subjects from a community", description = "remove subjects from a community", tags = {
C, W
})
@ApiResponses(value = {
@ApiResponse(responseCode = "200", description = "OK"),
@ApiResponse(responseCode = "404", description = "not found"),
@ApiResponse(responseCode = "500", description = "unexpected error")
})
public CommunityDetails removeCommunitySubjects(
@PathVariable final String id,
@RequestBody final List<String> subjects) throws CommunityException, CommunityNotFoundException {
return communityApiCore.removeCommunitySubjects(id, subjects);
}
@RequestMapping(value = "/community/{id}/zenodocommunities", produces = {
"application/json"
}, method = RequestMethod.GET)
@Operation(summary = "get the list of Zenodo communities associated to a given community", description = "get the list of Zenodo communities associated to a given community", tags = {
C_ZC, R
})
@ApiResponses(value = {
@ApiResponse(responseCode = "200", description = "OK"),
@ApiResponse(responseCode = "404", description = "not found"),
@ApiResponse(responseCode = "500", description = "unexpected error")
})
public List<CommunityZenodoCommunity> getCommunityZenodoCommunities(@PathVariable final String id) throws CommunityException, CommunityNotFoundException {
return communityApiCore.getCommunityZenodoCommunities(id);
}
@RequestMapping(value = "/community/{id}/zenodocommunities", produces = {
"application/json"
}, method = RequestMethod.POST)
@Operation(summary = "associate a Zenodo community to the community", description = "associate a Zenodo community to the community", tags = {
C_ZC, W
})
@ApiResponses(value = {
@ApiResponse(responseCode = "200", description = "OK"),
@ApiResponse(responseCode = "404", description = "not found"),
@ApiResponse(responseCode = "500", description = "unexpected error")
})
public CommunityZenodoCommunity addCommunityZenodoCommunity(
@PathVariable final String id,
@RequestBody final CommunityZenodoCommunity zenodocommunity) throws CommunityException, CommunityNotFoundException {
return communityApiCore.addCommunityZenodoCommunity(id, zenodocommunity);
}
@RequestMapping(value = "/community/{id}/zenodocommunities", produces = {
"application/json"
}, method = RequestMethod.DELETE)
@Operation(summary = "remove a Zenodo community from a community", description = "remove a Zenodo community from a community", tags = {
C_ZC, W
})
@ApiResponses(value = {
@ApiResponse(responseCode = "200", description = "OK"),
@ApiResponse(responseCode = "404", description = "not found"),
@ApiResponse(responseCode = "500", description = "unexpected error")
})
public void removeCommunityZenodoCommunity(
@PathVariable final String id,
@RequestBody final Integer zenodoCommId) throws CommunityException, CommunityNotFoundException {
communityApiCore.removeCommunityZenodoCommunity(id, zenodoCommId);
}
@RequestMapping(value = "/community/{zenodoId}/openairecommunities", produces = {
"application/json"
}, method = RequestMethod.GET)
@Operation(summary = "get the list of OpenAIRE communities associated to a given Zenodo community", description = "get the list of OpenAIRE communities associated to a given Zenodo community", tags = {
C_ZC, R
})
@ApiResponses(value = {
@ApiResponse(responseCode = "200", description = "OK"),
@ApiResponse(responseCode = "404", description = "not found"),
@ApiResponse(responseCode = "500", description = "unexpected error")
})
public CommunityOpenAIRECommunities getOpenAireCommunities(
@PathVariable final String zenodoId) throws CommunityException, CommunityNotFoundException {
return communityApiCore.getOpenAIRECommunities(zenodoId);
}
}

368
apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityApiCore.java Normal file
View File

@ -0,0 +1,368 @@
package eu.dnetlib.openaire.community;
import static eu.dnetlib.openaire.community.CommunityConstants.CCONTENTPROVIDER_NAME;
import static eu.dnetlib.openaire.community.CommunityConstants.CCONTENTPROVIDER_OFFICIALNAME;
import static eu.dnetlib.openaire.community.CommunityConstants.CCONTENTPROVIDER_SELCRITERIA;
import static eu.dnetlib.openaire.community.CommunityConstants.CLABEL;
import static eu.dnetlib.openaire.community.CommunityConstants.CONTENTPROVIDERS_ID_SUFFIX;
import static eu.dnetlib.openaire.community.CommunityConstants.CORGANIZATION_LOGOURL;
import static eu.dnetlib.openaire.community.CommunityConstants.CORGANIZATION_NAME;
import static eu.dnetlib.openaire.community.CommunityConstants.CORGANIZATION_WEBSITEURL;
import static eu.dnetlib.openaire.community.CommunityConstants.CPROFILE_SUBJECT;
import static eu.dnetlib.openaire.community.CommunityConstants.CPROJECT_ACRONYM;
import static eu.dnetlib.openaire.community.CommunityConstants.CPROJECT_FULLNAME;
import static eu.dnetlib.openaire.community.CommunityConstants.CPROJECT_FUNDER;
import static eu.dnetlib.openaire.community.CommunityConstants.CPROJECT_NUMBER;
import static eu.dnetlib.openaire.community.CommunityConstants.CSUMMARY_DESCRIPTION;
import static eu.dnetlib.openaire.community.CommunityConstants.CSUMMARY_LOGOURL;
import static eu.dnetlib.openaire.community.CommunityConstants.CSUMMARY_NAME;
import static eu.dnetlib.openaire.community.CommunityConstants.CSUMMARY_STATUS;
import static eu.dnetlib.openaire.community.CommunityConstants.CSUMMARY_ZENODOC;
import static eu.dnetlib.openaire.community.CommunityConstants.CSV_DELIMITER;
import static eu.dnetlib.openaire.community.CommunityConstants.ID_SEPARATOR;
import static eu.dnetlib.openaire.community.CommunityConstants.OPENAIRE_ID;
import static eu.dnetlib.openaire.community.CommunityConstants.ORGANIZATION_ID_SUFFIX;
import static eu.dnetlib.openaire.community.CommunityConstants.PROJECTS_ID_SUFFIX;
import static eu.dnetlib.openaire.community.CommunityConstants.ZENODOCOMMUNITY_ID_SUFFIX;
import java.util.Base64;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.TreeMap;
import java.util.stream.Collectors;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.cache.annotation.CacheEvict;
import org.springframework.stereotype.Component;
import com.google.common.base.Functions;
import com.google.common.base.Joiner;
import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
import eu.dnetlib.openaire.common.ISClient;
@Component
@ConditionalOnProperty(value = "openaire.exporter.enable.community", havingValue = "true")
public class CommunityApiCore {// implements CommunityClient{
private static final Log log = LogFactory.getLog(CommunityApiCore.class);
@Autowired
private CommunityClient cci;
@Autowired
private ISClient isClient;
@Autowired
private CommunityCommon cc;
public List<CommunitySummary> listCommunities() throws CommunityException {
return cc.listCommunities();
}
public CommunityDetails getCommunity(final String id) throws CommunityException, CommunityNotFoundException {
return cc.getCommunity(id);
}
public void setCommunity(final String id, final CommunityWritableProperties details) throws CommunityException, CommunityNotFoundException {
cc.getCommunity(id); // ensure the community exists.
if (details.getShortName() != null) {
isClient.updateContextAttribute(id, CLABEL, details.getShortName());
}
if (details.getName() != null) {
isClient.updateContextParam(id, CSUMMARY_NAME, details.getName());
}
if (details.getDescription() != null) {
isClient.updateContextParam(id, CSUMMARY_DESCRIPTION, details.getDescription());
}
if (details.getLogoUrl() != null) {
isClient.updateContextParam(id, CSUMMARY_LOGOURL, details.getLogoUrl());
}
if (details.getStatus() != null) {
isClient.updateContextParam(id, CSUMMARY_STATUS, details.getStatus().name());
}
if (details.getSubjects() != null) {
isClient.updateContextParam(id, CPROFILE_SUBJECT, Joiner.on(CSV_DELIMITER).join(details.getSubjects()));
}
if (details.getMainZenodoCommunity() != null) {
isClient.updateContextParam(id, CSUMMARY_ZENODOC, details.getMainZenodoCommunity());
}
cc.updateCommunity(id, details);
}
public List<CommunityProject> getCommunityProjects(final String id) throws CommunityException, CommunityNotFoundException {
cc.getCommunity(id); // ensure the community exists.
return cc.getCommunityInfo(id, PROJECTS_ID_SUFFIX, c -> CommunityMappingUtils.asCommunityProject(id, c));
}
public CommunityProject addCommunityProject(final String id, final CommunityProject project) throws CommunityException, CommunityNotFoundException {
if (!StringUtils.equalsIgnoreCase(id, project.getCommunityId())) {
throw new CommunityException("parameters 'id' and project.communityId must be coherent");
}
final TreeMap<Integer, CommunityProject> projects = getCommunityProjectMap(id);
final String project_id = project.getId();
if (project_id != null && projects.keySet().contains(Integer.valueOf(project_id))) {
if (project.getName() != null) {
isClient.updateConceptParam(id + PROJECTS_ID_SUFFIX + ID_SEPARATOR + project_id, CPROJECT_FULLNAME, project.getName());
}
if (project.getAcronym() != null) {
isClient.updateConceptParam(id + PROJECTS_ID_SUFFIX + ID_SEPARATOR + project_id, CPROJECT_ACRONYM, project.getAcronym());
}
if (project.getOpenaireId() != null) {
isClient.updateConceptParam(id + PROJECTS_ID_SUFFIX + ID_SEPARATOR + project_id, OPENAIRE_ID, project.getOpenaireId());
}
if (project.getFunder() != null) {
isClient.updateConceptParam(id + PROJECTS_ID_SUFFIX + ID_SEPARATOR + project_id, CPROJECT_FUNDER, project.getFunder());
}
if (project.getGrantId() != null) {
isClient.updateConceptParam(id + PROJECTS_ID_SUFFIX + ID_SEPARATOR + project_id, CPROJECT_NUMBER, project.getGrantId());
}
} else {
project.setId(nextId(projects != null && !projects.isEmpty() ? projects.lastKey() : 0));
isClient.addConcept(id, id + PROJECTS_ID_SUFFIX, CommunityMappingUtils.asProjectXML(id, project));
}
cc.updateProject(id, project);
return project;
}
private String nextId(final Integer id) {
return String.valueOf(id + 1);
}
public void removeCommunityProject(final String id, final Integer projectId) throws CommunityException, CommunityNotFoundException {
final Map<Integer, CommunityProject> projects = getCommunityProjectMap(id);
if (!projects.containsKey(projectId)) {
throw new CommunityNotFoundException(String.format("project '%s' doesn't exist within context '%s'", projectId, id));
}
isClient.removeConcept(id, id + PROJECTS_ID_SUFFIX, id + PROJECTS_ID_SUFFIX + ID_SEPARATOR + projectId);
cc.removeFromCategory(id, PROJECTS_ID_SUFFIX, String.valueOf(projectId));
}
public List<CommunityContentprovider> getCommunityContentproviders(final String id) throws CommunityException, CommunityNotFoundException {
cc.getCommunity(id); // ensure the community exists.
return cc.getCommunityInfo(id, CONTENTPROVIDERS_ID_SUFFIX, c -> CommunityMappingUtils.asCommunityDataprovider(id, c));
}
public CommunityContentprovider addCommunityContentprovider(final String id, final CommunityContentprovider cp)
throws CommunityException, CommunityNotFoundException {
log.info("content provider to add " + cp.toString());
if (!StringUtils.equalsIgnoreCase(id, cp.getCommunityId())) { throw new CommunityException("parameters 'id' and cp.communityId must be coherent"); }
final TreeMap<Integer, CommunityContentprovider> cps = getCommunityContentproviderMap(id);
final String concept_id = cp.getId();
if (concept_id != null && cps.keySet().contains(Integer.valueOf(concept_id))) {
if (cp.getName() != null) {
isClient.updateConceptParam(id + CONTENTPROVIDERS_ID_SUFFIX + ID_SEPARATOR + concept_id, CCONTENTPROVIDER_NAME, cp.getName());
}
if (cp.getOfficialname() != null) {
isClient.updateConceptParam(id + CONTENTPROVIDERS_ID_SUFFIX + ID_SEPARATOR + concept_id, CCONTENTPROVIDER_OFFICIALNAME, cp.getOfficialname());
}
if (cp.getOpenaireId() != null) {
isClient.updateConceptParam(id + CONTENTPROVIDERS_ID_SUFFIX + ID_SEPARATOR + concept_id, OPENAIRE_ID, cp.getOpenaireId());
}
if (cp.getSelectioncriteria() != null) {
isClient.updateConceptParamNoEscape(id + CONTENTPROVIDERS_ID_SUFFIX + ID_SEPARATOR + concept_id, CCONTENTPROVIDER_SELCRITERIA, cp.toXML());
}
} else {
log.info("adding new concept for community " + id);
cp.setId(nextId(!cps.isEmpty() ? cps.lastKey() : 0));
isClient.addConcept(id, id + CONTENTPROVIDERS_ID_SUFFIX, CommunityMappingUtils.asContentProviderXML(id, cp));
}
cc.updateDatasource(id, cp);
return cp;
}
public void removeCommunityContentProvider(final String id, final Integer contentproviderId) throws CommunityException, CommunityNotFoundException {
final Map<Integer, CommunityContentprovider> providers = getCommunityContentproviderMap(id);
if (!providers.containsKey(contentproviderId)) {
throw new CommunityNotFoundException(String.format("content provider '%s' doesn't exist within context '%s'", contentproviderId, id));
}
isClient.removeConcept(id, id + CONTENTPROVIDERS_ID_SUFFIX, id + CONTENTPROVIDERS_ID_SUFFIX + ID_SEPARATOR + contentproviderId);
cc.removeFromCategory(id, CONTENTPROVIDERS_ID_SUFFIX, String.valueOf(contentproviderId));
}
public void removeCommunityOrganization(final String id, final Integer organizationId) throws CommunityException, CommunityNotFoundException {
final Map<Integer, CommunityOrganization> organizations = getCommunityOrganizationMap(id);
if (!organizations.containsKey(organizationId)) {
throw new CommunityNotFoundException(String.format("organization '%s' doesn't exist within context '%s'", organizationId, id));
}
isClient.removeConcept(id, id + ORGANIZATION_ID_SUFFIX, id + ORGANIZATION_ID_SUFFIX + ID_SEPARATOR + organizationId);
cc.removeFromCategory(id, ORGANIZATION_ID_SUFFIX, String.valueOf(organizationId));
}
public List<CommunityZenodoCommunity> getCommunityZenodoCommunities(final String id) throws CommunityException, CommunityNotFoundException {
return cc.getCommunityZenodoCommunities(id);
}
public List<CommunityOrganization> getCommunityOrganizations(final String id) throws CommunityException, CommunityNotFoundException {
cc.getCommunity(id);
return cc.getCommunityInfo(id, ORGANIZATION_ID_SUFFIX, c -> CommunityMappingUtils.asCommunityOrganization(id, c));
}
public CommunityDetails addCommunitySubjects(final String id, final List<String> subjects) throws CommunityException, CommunityNotFoundException {
final CommunityDetails cd = new CommunityDetails();
final Set<String> current = Sets.newHashSet(cc.getCommunity(id).getSubjects());
current.addAll(subjects);
cd.setSubjects(Lists.newArrayList(current));
setCommunity(id, CommunityWritableProperties.fromDetails(cd));
return cd;
}
public CommunityDetails removeCommunitySubjects(final String id, final List<String> subjects) throws CommunityException, CommunityNotFoundException {
final CommunityDetails cd = new CommunityDetails();
final Set<String> current = Sets.newHashSet(cc.getCommunity(id).getSubjects());
current.removeAll(subjects);
cd.setSubjects(Lists.newArrayList(current));
setCommunity(id, CommunityWritableProperties.fromDetails(cd));
return cd;
}
@CacheEvict(value = "community-cache", allEntries = true)
public void removeCommunityZenodoCommunity(final String id, final Integer zenodoCommId) throws CommunityException, CommunityNotFoundException {
final Map<Integer, CommunityZenodoCommunity> zcomms = getZenodoCommunityMap(id);
if (!zcomms.containsKey(zenodoCommId)) {
throw new CommunityNotFoundException(String.format("Zenodo community '%s' doesn't exist within context '%s'", zenodoCommId, id));
}
isClient.removeConcept(id, id + ZENODOCOMMUNITY_ID_SUFFIX, id + ZENODOCOMMUNITY_ID_SUFFIX + ID_SEPARATOR + zenodoCommId);
cc.removeFromCategory(id, ZENODOCOMMUNITY_ID_SUFFIX, String.valueOf(zenodoCommId));
}
@CacheEvict(value = "community-cache", allEntries = true)
public CommunityZenodoCommunity addCommunityZenodoCommunity(final String id, final CommunityZenodoCommunity zc)
throws CommunityException, CommunityNotFoundException {
if (!StringUtils.equalsIgnoreCase(id, zc.getCommunityId())) { throw new CommunityException("parameters 'id' and zc.communityId must be coherent"); }
if (!StringUtils.isNotBlank(zc.getZenodoid())) { throw new CommunityException("parameter zenodoid cannot be null or empty"); }
final TreeMap<Integer, CommunityZenodoCommunity> zcs = getZenodoCommunityMap(id);
for (final CommunityZenodoCommunity czc : zcs.values()) {
if (czc.getZenodoid().equals(zc.getZenodoid())) { throw new CommunityException("Zenodo community already associated to the RCD"); }
}
zc.setId(nextId(!zcs.isEmpty() ? zcs.lastKey() : 0));
isClient.addConcept(id, id + ZENODOCOMMUNITY_ID_SUFFIX, CommunityMappingUtils.asZenodoCommunityXML(id, zc));
cc.updateZenodoCommunity(id, zc);
return zc;
}
public CommunityOpenAIRECommunities getOpenAIRECommunities(final String zenodoId) throws CommunityException, CommunityNotFoundException {
if (cci.getInverseZenodoCommunityMap().containsKey(zenodoId)) {
return new CommunityOpenAIRECommunities().setZenodoid(zenodoId)
.setOpenAirecommunitylist(cci.getInverseZenodoCommunityMap().get(zenodoId).stream().collect(Collectors.toList()));
}
return new CommunityOpenAIRECommunities();
}
// HELPERS
private TreeMap<Integer, CommunityProject> getCommunityProjectMap(final String id) throws CommunityException, CommunityNotFoundException {
return getCommunityProjects(id).stream()
.collect(Collectors.toMap(p -> Integer.valueOf(p.getId()), Functions.identity(), (p1, p2) -> {
log.warn(String.format("duplicate project found: '%s'", p1.getId()));
return p2;
}, TreeMap::new));
}
private TreeMap<Integer, CommunityContentprovider> getCommunityContentproviderMap(final String id) throws CommunityException, CommunityNotFoundException {
log.info("getting community content provider map");
return getCommunityContentproviders(id).stream()
.collect(Collectors.toMap(cp -> Integer.valueOf(cp.getId()), Functions.identity(), (cp1, cp2) -> {
log.warn(String.format("duplicate content provider found: '%s'", cp1.getId()));
return cp2;
}, TreeMap::new));
}
private TreeMap<Integer, CommunityZenodoCommunity> getZenodoCommunityMap(final String id) throws CommunityException, CommunityNotFoundException {
return getCommunityZenodoCommunities(id).stream()
.collect(Collectors.toMap(cp -> Integer.valueOf(cp.getId()), Functions.identity(), (cp1, cp2) -> {
log.warn(String.format("duplicate Zenodo community found: '%s'", cp1.getId()));
return cp2;
}, TreeMap::new));
}
private TreeMap<Integer, CommunityOrganization> getCommunityOrganizationMap(final String id) throws CommunityException, CommunityNotFoundException {
return getCommunityOrganizations(id).stream()
.collect(Collectors.toMap(o -> Integer.valueOf(o.getId()), Functions.identity(), (o1, o2) -> {
log.warn(String.format("duplicate content provider found: '%s'", o1.getId()));
return o2;
}, TreeMap::new));
}
public CommunityOrganization addCommunityOrganization(final String id, final CommunityOrganization organization)
throws CommunityException, CommunityNotFoundException {
if (!StringUtils.equalsIgnoreCase(id, organization.getCommunityId())) {
throw new CommunityException("parameters 'id' and organization.communityId must be coherent");
}
final TreeMap<Integer, CommunityOrganization> cps = getCommunityOrganizationMap(id);
final String organization_id = organization.getId();
if (organization_id != null && cps.keySet().contains(Integer.valueOf(organization_id))) {
if (organization.getName() != null) {
isClient.updateConceptParam(id + ORGANIZATION_ID_SUFFIX + ID_SEPARATOR + organization_id, CORGANIZATION_NAME, organization.getName());
}
if (organization.getLogo_url() != null) {
isClient.updateConceptParam(id + ORGANIZATION_ID_SUFFIX + ID_SEPARATOR + organization_id, CORGANIZATION_LOGOURL, Base64.getEncoder()
.encodeToString(organization.getLogo_url().getBytes()));
}
if (organization.getWebsite_url() != null) {
isClient.updateConceptParam(id + ORGANIZATION_ID_SUFFIX + ID_SEPARATOR + organization_id, CORGANIZATION_WEBSITEURL, Base64.getEncoder()
.encodeToString(organization.getWebsite_url().getBytes()));
}
} else {
organization.setId(nextId(!cps.isEmpty() ? cps.lastKey() : 0));
isClient.addConcept(id, id + ORGANIZATION_ID_SUFFIX, CommunityMappingUtils.asOrganizationXML(id, organization));
}
cc.updateOrganization(id, organization);
return organization;
}
}

12
apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityClient.java Normal file
View File

@ -0,0 +1,12 @@
package eu.dnetlib.openaire.community;
import java.util.Map;
import java.util.Set;
public interface CommunityClient {
Map<String, Set<String>> getInverseZenodoCommunityMap() throws CommunityException, CommunityNotFoundException;
void dropCache();
}

58
apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityClientImpl.java Normal file
View File

@ -0,0 +1,58 @@
package eu.dnetlib.openaire.community;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.cache.annotation.CacheEvict;
import org.springframework.cache.annotation.Cacheable;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Component;
import java.util.*;
@Component
public class CommunityClientImpl implements CommunityClient {
private static final Log log = LogFactory.getLog(CommunityClient.class);
@Autowired
private CommunityCommon communityCommon;
@Override
@Cacheable("community-cache")
public Map<String, Set<String>> getInverseZenodoCommunityMap () throws CommunityException, CommunityNotFoundException {
log.info("Creating the data structure. Not using cache");
final Map<String, Set<String>> inverseListMap = new HashMap<>();
final List<CommunitySummary> communityList = communityCommon.listCommunities();
for(CommunitySummary cs :communityList){
final String communityId = cs.getId();
List<CommunityZenodoCommunity> czc = communityCommon.getCommunityZenodoCommunities(communityId);
for(CommunityZenodoCommunity zc:czc){
final String zenodoId = zc.getZenodoid();
if(!inverseListMap.containsKey(zenodoId)) {
inverseListMap.put(zc.getZenodoid(),new HashSet<>());
}
inverseListMap.get(zc.getZenodoid()).add(communityId);
}
final String zenodoMainCommunity = communityCommon.getCommunity(communityId).getZenodoCommunity();
if(!inverseListMap.containsKey(zenodoMainCommunity)) {
inverseListMap.put(zenodoMainCommunity,new HashSet<>());
}
inverseListMap.get(zenodoMainCommunity).add(communityId);
}
return inverseListMap;
}
@Override
@CacheEvict(cacheNames = { "community-cache", "context-cache-community"}, allEntries = true)
@Scheduled(fixedDelayString = "${openaire.exporter.cache.ttl}")
public void dropCache(){
log.debug("dropped community cache");
}
}

438
apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityCommon.java Normal file
View File

@ -0,0 +1,438 @@
package eu.dnetlib.openaire.community;
import com.google.common.base.Joiner;
import com.google.common.collect.Lists;
import eu.dnetlib.openaire.common.ISClient;
import eu.dnetlib.openaire.context.Category;
import eu.dnetlib.openaire.context.Concept;
import eu.dnetlib.openaire.context.Context;
import eu.dnetlib.openaire.context.Param;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import java.io.IOException;
import java.util.*;
import java.util.function.Function;
import java.util.stream.Collectors;
import static eu.dnetlib.openaire.community.CommunityConstants.*;
@Component
public class CommunityCommon {
@Autowired
private ISClient isClient;
public Map<String, Context> getContextMap() throws CommunityException {
try {
return isClient.getCommunityContextMap();
} catch (IOException e) {
throw new CommunityException(e);
}
}
public List<CommunitySummary> listCommunities() throws CommunityException {
return getContextMap().values().stream()
.filter(context -> !communityBlackList.contains(context.getId()))
.map(CommunityMappingUtils::asCommunitySummary)
.collect(Collectors.toList());
}
public <R> List<R> getCommunityInfo(final String id, final String idSuffix, final Function<Concept, R> mapping) throws CommunityException {
final Map<String, Context> contextMap = getContextMap();
final Context context = contextMap.get(id);
if (context != null) {
final Map<String, Category> categories = context.getCategories();
final Category category = categories.get(id + idSuffix);
if (category != null) {
return category.getConcepts().stream()
.map(mapping)
.collect(Collectors.toList());
}
}
return Lists.newArrayList();
}
public CommunityDetails getCommunity(final String id) throws CommunityException, CommunityNotFoundException {
final Context context = getContextMap().get(id);
if (context == null || CommunityConstants.communityBlackList.contains(id)) {
throw new CommunityNotFoundException(String.format("community '%s' does not exist", id));
}
return CommunityMappingUtils.asCommunityProfile(context);
}
public List<CommunityZenodoCommunity> getCommunityZenodoCommunities(final String id) throws CommunityException, CommunityNotFoundException {
getCommunity(id); // ensure the community exists.
return getCommunityInfo(id, ZENODOCOMMUNITY_ID_SUFFIX, c -> CommunityMappingUtils.asCommunityZenodoCommunity(id, c));
}
public void updateProject(String communityId, CommunityProject project) throws CommunityException {
final Context context = getContextMap().get(communityId);
Category prj = context.getCategories().get(communityId + PROJECTS_ID_SUFFIX);
if (prj.getConcepts().stream().map(c -> c.getId()).collect(Collectors.toList())
.contains(communityId + PROJECTS_ID_SUFFIX + ID_SEPARATOR + project.getId())){
prj.getConcepts().forEach(concept -> {
if (concept.getId().equals(communityId + PROJECTS_ID_SUFFIX + ID_SEPARATOR + project.getId())) {
if (project.getName() != null) {
concept.getParams().replace(CPROJECT_FULLNAME, Arrays.asList(new Param()
.setName(CPROJECT_FULLNAME).setValue(project.getName())));
}
if (project.getAcronym() != null) {
if(concept.getParams().keySet().contains(CPROJECT_ACRONYM)){
concept.getParams().replace(CPROJECT_ACRONYM, Arrays.asList(new Param()
.setName(CPROJECT_ACRONYM).setValue(project.getAcronym())));
}
else{
concept.getParams().put(CPROJECT_ACRONYM, Arrays.asList(new Param()
.setName(CPROJECT_ACRONYM).setValue(project.getAcronym())));
}
}
if (project.getOpenaireId() != null) {
if(concept.getParams().keySet().contains(OPENAIRE_ID)){
concept.getParams().replace(OPENAIRE_ID, Arrays.asList(new Param()
.setName(OPENAIRE_ID).setValue(project.getOpenaireId())));
}
else{
concept.getParams().put(OPENAIRE_ID, Arrays.asList(new Param()
.setName(OPENAIRE_ID).setValue(project.getOpenaireId())));
}
}
if (project.getFunder() != null) {
concept.getParams().replace(CPROJECT_FUNDER, Arrays.asList(new Param()
.setName(CPROJECT_FUNDER).setValue(project.getFunder())));
}
if (project.getGrantId() != null) {
concept.getParams().replace(CPROJECT_NUMBER, Arrays.asList(new Param()
.setName(CPROJECT_NUMBER).setValue(project.getGrantId())));
}
}
});
}
else{
Concept concept = new Concept();
concept.setId(communityId + PROJECTS_ID_SUFFIX + ID_SEPARATOR + project.getId());
concept.setClaim(false);
if(project.getAcronym() != null)
concept.setLabel(project.getAcronym());
else
concept.setLabel("");
Map<String, List<Param>> params = new TreeMap<>();
if(project.getAcronym() != null){
params.put(CPROJECT_ACRONYM, Arrays.asList(new Param().setName(CPROJECT_ACRONYM)
.setValue(project.getAcronym())));
}
if (project.getName() != null){
params.put(CPROJECT_FULLNAME, Arrays.asList(new Param()
.setName(CPROJECT_FULLNAME)
.setValue(project.getName())
));
}
if (project.getOpenaireId() != null){
params.put(OPENAIRE_ID, Arrays.asList(new Param()
.setName(OPENAIRE_ID)
.setValue(project.getOpenaireId())
));
}
if(project.getFunder() != null){
params.put(CPROJECT_FUNDER, Arrays.asList(new Param()
.setName(CPROJECT_FUNDER)
.setValue(project.getFunder())
));
}
if (project.getGrantId()!=null){
params.put(CPROJECT_NUMBER, Arrays.asList(new Param()
.setName(CPROJECT_NUMBER)
.setValue(project.getGrantId())
));
}
concept.setParams(params);
prj.getConcepts().add(concept);
}
}
public void updateCommunity(String id, CommunityWritableProperties community) throws CommunityException {
final Context context = getContextMap().get(id);
if(community.getShortName() != null) {
context.setLabel(community.getShortName());
}
if (community.getName() != null){
context.getParams().replace(CSUMMARY_NAME, Arrays.asList(new Param()
.setValue(community.getName()).setName(CSUMMARY_NAME)));
}
if(community.getDescription() != null) {
context.getParams()
.replace(CSUMMARY_DESCRIPTION, Arrays.asList(new Param()
.setName(CSUMMARY_DESCRIPTION).setValue(community.getDescription())));
}
if(community.getLogoUrl() != null){
context.getParams()
.replace(CSUMMARY_LOGOURL, Arrays.asList(new Param()
.setName(CSUMMARY_LOGOURL).setValue(community.getLogoUrl())));
}
if (community.getStatus() != null) {
context.getParams()
.replace(CSUMMARY_STATUS, Arrays.asList(new Param()
.setName(CSUMMARY_STATUS).setValue(community.getStatus().name())));
}
if (community.getSubjects() != null) {
context.getParams()
.replace(CPROFILE_SUBJECT, Arrays.asList(new Param().setName(CPROFILE_SUBJECT)
.setValue(Joiner.on(CSV_DELIMITER)
.join(community.getSubjects()))));
}
if(community.getMainZenodoCommunity() != null){
context.getParams()
.replace(CSUMMARY_ZENODOC, Arrays.asList(new Param()
.setName(CSUMMARY_ZENODOC).setValue(community.getMainZenodoCommunity())));
}
}
public void removeFromCategory(String communityId, String category, String conceptId) throws CommunityException {
Map<String, Context> cmap = getContextMap();
Context context = cmap.get(communityId);
Map<String, Category> cat = context.getCategories();
List<Concept> concepts = cat.get(communityId + category).getConcepts()
.stream().filter(c -> !c.getId().equals(communityId + category + ID_SEPARATOR + conceptId)).collect(Collectors.toList());
cat.get(communityId + category).setConcepts(concepts);
}
public void updateDatasource(String communityId, CommunityContentprovider cp) throws CommunityException {
final Context context = getContextMap().get(communityId);
Category dts = context.getCategories().get(communityId + CONTENTPROVIDERS_ID_SUFFIX);
if (dts.getConcepts().stream().map(c -> c.getId()).collect(Collectors.toList())
.contains(communityId + CONTENTPROVIDERS_ID_SUFFIX + ID_SEPARATOR + cp.getId())){
dts.getConcepts().forEach(concept -> {
if (concept.getId().equals(communityId + CONTENTPROVIDERS_ID_SUFFIX + ID_SEPARATOR + cp.getId())) {
if (cp.getName() != null) {
if(concept.getParams().keySet().contains(CCONTENTPROVIDER_NAME)){
concept.getParams().replace(CCONTENTPROVIDER_NAME, Arrays.asList(new Param()
.setName(CCONTENTPROVIDER_NAME).setValue(cp.getName())));
}
else{
concept.getParams().put(CCONTENTPROVIDER_NAME, Arrays.asList(new Param()
.setName(CCONTENTPROVIDER_NAME).setValue(cp.getName())));
}
}
if (cp.getOfficialname() != null) {
if(concept.getParams().keySet().contains(CCONTENTPROVIDER_OFFICIALNAME)){
concept.getParams().replace(CCONTENTPROVIDER_OFFICIALNAME, Arrays.asList(new Param()
.setName(CCONTENTPROVIDER_OFFICIALNAME).setValue(cp.getOfficialname())));
}
else{
concept.getParams().put(CCONTENTPROVIDER_OFFICIALNAME, Arrays.asList(new Param()
.setName(CCONTENTPROVIDER_OFFICIALNAME).setValue(cp.getOfficialname())));
}
}
if (cp.getOpenaireId() != null) {
if(concept.getParams().keySet().contains(OPENAIRE_ID)){
concept.getParams().replace(OPENAIRE_ID, Arrays.asList(new Param()
.setName(OPENAIRE_ID).setValue(cp.getOpenaireId())));
}
else{
concept.getParams().put(OPENAIRE_ID, Arrays.asList(new Param()
.setName(OPENAIRE_ID).setValue(cp.getOpenaireId())));
}
}
if (cp.getSelectioncriteria() != null) {
if(concept.getParams().keySet().contains(CCONTENTPROVIDER_SELCRITERIA)){
concept.getParams().replace(CCONTENTPROVIDER_SELCRITERIA, Arrays.asList(new Param()
.setName(CCONTENTPROVIDER_SELCRITERIA).setValue(cp.toJson())));
}
else{
concept.getParams().put(CCONTENTPROVIDER_SELCRITERIA, Arrays.asList(new Param()
.setName(CCONTENTPROVIDER_SELCRITERIA).setValue(cp.toJson())));
}
}
}
});
}
else{
Concept concept = new Concept();
concept.setId(communityId + CONTENTPROVIDERS_ID_SUFFIX + ID_SEPARATOR + cp.getId());
concept.setClaim(false);
concept.setLabel("");
Map<String, List<Param>> params = new TreeMap<>();
if (cp.getName() != null) {
params.put( CCONTENTPROVIDER_NAME, Arrays.asList(new Param().setValue(cp.getName()).setName(CCONTENTPROVIDER_NAME)));
}
if(cp.getOfficialname()!= null){
params.put( CCONTENTPROVIDER_OFFICIALNAME, Arrays.asList(new Param().setValue(cp.getOfficialname()).setName(CCONTENTPROVIDER_OFFICIALNAME)));
}
if (cp.getOpenaireId() != null){
params.put( OPENAIRE_ID, Arrays.asList(new Param().setValue(cp.getOpenaireId()).setName(OPENAIRE_ID)));
}
if(cp.getSelectioncriteria() != null){
params.put( CCONTENTPROVIDER_SELCRITERIA, Arrays.asList(new Param().setValue(cp.toJson()).setName(CCONTENTPROVIDER_SELCRITERIA)));
}
concept.setParams(params);
dts.getConcepts().add(concept);
}
}
public void updateOrganization(String communityId, CommunityOrganization organization) throws CommunityException {
final Context context = getContextMap().get(communityId);
Category orgs = context.getCategories().get(communityId + ORGANIZATION_ID_SUFFIX);
if (orgs.getConcepts().stream().map(c -> c.getId()).collect(Collectors.toList())
.contains(communityId + ORGANIZATION_ID_SUFFIX + ID_SEPARATOR + organization.getId())){
orgs.getConcepts().forEach(concept -> {
if (concept.getId().equals(communityId + ORGANIZATION_ID_SUFFIX + ID_SEPARATOR + organization.getId())) {
if (organization.getName() != null) {
if(concept.getParams().keySet().contains(CORGANIZATION_NAME)){
concept.getParams().replace(CORGANIZATION_NAME, Arrays.asList(new Param()
.setName(CORGANIZATION_NAME).setValue(organization.getName())));
}
else{
concept.getParams().put(CORGANIZATION_NAME, Arrays.asList(new Param()
.setName(CORGANIZATION_NAME).setValue(organization.getName())));
}
}
if (organization.getLogo_url() != null) {
if(concept.getParams().keySet().contains(CORGANIZATION_LOGOURL)){
concept.getParams().replace(CORGANIZATION_LOGOURL, Arrays.asList(new Param()
.setName(CORGANIZATION_LOGOURL).setValue(Base64.getEncoder().encodeToString(organization.getLogo_url().getBytes()))));
}
else{
concept.getParams().put(CORGANIZATION_LOGOURL, Arrays.asList(new Param()
.setName(CORGANIZATION_LOGOURL).setValue(Base64.getEncoder().encodeToString(organization.getLogo_url().getBytes()))));
}
}
if (organization.getWebsite_url() != null) {
if(concept.getParams().keySet().contains(CORGANIZATION_WEBSITEURL)){
concept.getParams().replace(CORGANIZATION_WEBSITEURL, Arrays.asList(new Param()
.setName(CORGANIZATION_WEBSITEURL).setValue(Base64.getEncoder().encodeToString(organization.getWebsite_url().getBytes()))));
}
else{
concept.getParams().put(CORGANIZATION_WEBSITEURL, Arrays.asList(new Param()
.setName(CORGANIZATION_WEBSITEURL).setValue(Base64.getEncoder().encodeToString(organization.getWebsite_url().getBytes()))));
}
}
}
});
}
else{
Concept concept = new Concept();
concept.setId(communityId + ORGANIZATION_ID_SUFFIX + ID_SEPARATOR + organization.getId());
concept.setClaim(false);
concept.setLabel("");
Map<String, List<Param>> params = new TreeMap<>();
if (organization.getName() != null) {
params.put( CORGANIZATION_NAME, Arrays.asList(new Param().setValue(organization.getName()).setName(CORGANIZATION_NAME)));
}
if(organization.getLogo_url()!= null){
params.put( CORGANIZATION_LOGOURL, Arrays.asList(new Param().setValue(Base64.getEncoder().encodeToString(organization.getLogo_url().getBytes())).setName(CORGANIZATION_LOGOURL)));
}
if (organization.getWebsite_url() != null){
params.put( CORGANIZATION_WEBSITEURL, Arrays.asList(new Param().setValue(Base64.getEncoder().encodeToString(organization.getWebsite_url().getBytes())).setName(CORGANIZATION_WEBSITEURL)));
}
concept.setParams(params);
orgs.getConcepts().add(concept);
}
}
public void updateZenodoCommunity(String communityId, CommunityZenodoCommunity zc) throws CommunityException {
final Context context = getContextMap().get(communityId);
Category zcs = context.getCategories().get(communityId + ZENODOCOMMUNITY_ID_SUFFIX);
if (zcs.getConcepts().stream().map(c -> c.getId()).collect(Collectors.toList())
.contains(communityId + ZENODOCOMMUNITY_ID_SUFFIX + ID_SEPARATOR + zc.getId())){
zcs.getConcepts().forEach(concept -> {
if (concept.getId().equals(communityId + ZENODOCOMMUNITY_ID_SUFFIX + ID_SEPARATOR + zc.getId())) {
if (zc.getZenodoid() != null) {
if(concept.getParams().keySet().contains(CZENODOCOMMUNITY_ID)){
concept.getParams().replace(CZENODOCOMMUNITY_ID, Arrays.asList(new Param()
.setName(CZENODOCOMMUNITY_ID).setValue(zc.getZenodoid())));
}
else{
concept.getParams().put(CZENODOCOMMUNITY_ID, Arrays.asList(new Param()
.setName(CZENODOCOMMUNITY_ID).setValue(zc.getZenodoid())));
}
}
}
});
}
else{
Concept concept = new Concept();
concept.setId(communityId + ZENODOCOMMUNITY_ID_SUFFIX + ID_SEPARATOR + zc.getId());
concept.setClaim(false);
Map<String, List<Param>> params = new TreeMap<>();
if (zc.getZenodoid() != null) {
params.put( CZENODOCOMMUNITY_ID, Arrays.asList(new Param().setValue(zc.getZenodoid()).setName(CZENODOCOMMUNITY_ID)));
concept.setLabel(zc.getZenodoid());
}else{
concept.setLabel("");
}
concept.setParams(params);
zcs.getConcepts().add(concept);
}
}
}

58
apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityConstants.java Normal file
View File

@ -0,0 +1,58 @@
package eu.dnetlib.openaire.community;
import java.util.Set;
import com.google.common.collect.Sets;
public class CommunityConstants {
public final static Set<String> communityBlackList = Sets.newHashSet("fet-fp7", "fet-h2020");
// common
public final static String OPENAIRE_ID = "openaireId";
public final static String PIPE_SEPARATOR = "||";
public final static String ID_SEPARATOR = "::";
public final static String CSV_DELIMITER = ",";
public final static String CLABEL = "label";
// id suffixes
public final static String PROJECTS_ID_SUFFIX = ID_SEPARATOR + "projects";
public final static String CONTENTPROVIDERS_ID_SUFFIX = ID_SEPARATOR + "contentproviders";
public final static String ZENODOCOMMUNITY_ID_SUFFIX = ID_SEPARATOR + "zenodocommunities";
public final static String ORGANIZATION_ID_SUFFIX = ID_SEPARATOR + "organizations";
// community summary
public final static String CSUMMARY_DESCRIPTION = "description";
public final static String CSUMMARY_LOGOURL = "logourl";
public final static String CSUMMARY_STATUS = "status";
public final static String CSUMMARY_NAME = "name";
public final static String CSUMMARY_MANAGER = "manager";
public final static String CSUMMARY_ZENODOC = "zenodoCommunity";
// community profile
public final static String CPROFILE_SUBJECT = "subject";
public final static String CPROFILE_CREATIONDATE = "creationdate";
// community project
public final static String CPROJECT_FUNDER = "funder";
public final static String CPROJECT_NUMBER = "CD_PROJECT_NUMBER";
public final static String CPROJECT_FULLNAME = "projectfullname";
public final static String CPROJECT_ACRONYM = "acronym";
// community content provider
public final static String CCONTENTPROVIDER_NAME = "name";
public final static String CCONTENTPROVIDER_OFFICIALNAME = "officialname";
public final static String CCONTENTPROVIDER_ENABLED = "enabled";
public final static String CCONTENTPROVIDERENABLED_DEFAULT = "true";
public final static String CCONTENTPROVIDER_SELCRITERIA = "selcriteria";
//community zenodo community
public final static String CZENODOCOMMUNITY_ID = "zenodoid";
//community organization
public final static String CORGANIZATION_NAME = "name";
public final static String CORGANIZATION_LOGOURL = "logourl";
public final static String CORGANIZATION_WEBSITEURL = "websiteurl";
}

100
apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityContentprovider.java Normal file
View File

@ -0,0 +1,100 @@
package eu.dnetlib.openaire.community;
import javax.validation.constraints.NotNull;
import com.fasterxml.jackson.annotation.JsonAutoDetect;
import com.google.gson.Gson;
import eu.dnetlib.openaire.community.selectioncriteria.SelectionCriteria;
import io.swagger.v3.oas.annotations.media.Schema;
@JsonAutoDetect
public class CommunityContentprovider {
@Schema(description = "OpenAIRE identifier for this content provider, if available", required = false)
private String openaireId;
@NotNull
@Schema(description = "the community identifier this content provider belongs to", required = true)
private String communityId;
@NotNull
@Schema(description = "identifies this content provider within the context it belongs to", required = true)
private String id;
@Schema(description = "content provider name", required = false)
private String name;
@NotNull
@Schema(description = "content provider official name", required = true)
private String officialname;
// @NotNull
@Schema(description = "content provider selection criteria", required = false)
private SelectionCriteria selectioncriteria;
public String getOpenaireId() {
return openaireId;
}
public void setOpenaireId(final String openaireId) {
this.openaireId = openaireId;
}
public String getCommunityId() {
return communityId;
}
public void setCommunityId(final String communityId) {
this.communityId = communityId;
}
public String getId() {
return id;
}
public void setId(final String id) {
this.id = id;
}
public String getName() {
return name;
}
public void setName(final String name) {
this.name = name;
}
public String getOfficialname() {
return officialname;
}
public void setOfficialname(final String officialname) {
this.officialname = officialname;
}
public SelectionCriteria getSelectioncriteria() {
return this.selectioncriteria;
}
public void setSelectioncriteria(final SelectionCriteria selectioncriteria) {
this.selectioncriteria = selectioncriteria;
}
@Override
public String toString() {
return String.format("id %s, name %s, selection criteria %s", this.id, this.name, toJson());
}
public String toJson() {
if (selectioncriteria == null) { return ""; }
return new Gson().toJson(selectioncriteria);
}
public String toXML() {
if (selectioncriteria == null) { return ""; }
return "<![CDATA[" + toJson() + "]]>";
}
}

55
apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityDetails.java Normal file
View File

@ -0,0 +1,55 @@
package eu.dnetlib.openaire.community;
import java.util.Date;
import java.util.List;
import com.fasterxml.jackson.annotation.JsonAutoDetect;
import io.swagger.v3.oas.annotations.media.Schema;
@JsonAutoDetect
public class CommunityDetails extends CommunitySummary {
@Schema(description = "date of creation for this community")
private Date creationDate;
@Schema(description = "date of the last update for this communityu")
private Date lastUpdateDate;
@Schema(description = "list of subjects (keywords) that characterise this community")
private List<String> subjects;
public CommunityDetails() {}
public CommunityDetails(final CommunitySummary summary) {
super(summary);
}
@Override
public Date getCreationDate() {
return creationDate;
}
@Override
public void setCreationDate(final Date creationDate) {
this.creationDate = creationDate;
}
public List<String> getSubjects() {
return subjects;
}
public void setSubjects(final List<String> subjects) {
this.subjects = subjects;
}
@Override
public Date getLastUpdateDate() {
return lastUpdateDate;
}
@Override
public void setLastUpdateDate(final Date lastUpdateDate) {
this.lastUpdateDate = lastUpdateDate;
}
}

25
apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityException.java Normal file
View File

@ -0,0 +1,25 @@
package eu.dnetlib.openaire.community;
import java.io.IOException;
import org.springframework.http.HttpStatus;
import org.springframework.web.bind.annotation.ResponseBody;
import org.springframework.web.bind.annotation.ResponseStatus;
@ResponseBody
@ResponseStatus(value = HttpStatus.INTERNAL_SERVER_ERROR)
public class CommunityException extends Exception {
/**
*
*/
private static final long serialVersionUID = -4961233580574761346L;
public CommunityException(final String message) {
super(message);
}
public CommunityException(final IOException e) {
super(e);
}
}

231
apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityMappingUtils.java Normal file
View File

@ -0,0 +1,231 @@
package eu.dnetlib.openaire.community;
import java.text.ParseException;
import java.util.*;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import com.google.common.escape.Escaper;
import com.google.common.xml.XmlEscapers;
import eu.dnetlib.openaire.community.selectioncriteria.SelectionCriteria;
import eu.dnetlib.openaire.context.Concept;
import eu.dnetlib.openaire.context.Context;
import eu.dnetlib.openaire.context.Param;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import static eu.dnetlib.openaire.common.Utils.escape;
import static eu.dnetlib.openaire.community.CommunityConstants.*;
public class CommunityMappingUtils {
private final static String pattern = "yyyy-MM-dd'T'hh:mm:ss";
private static final Log log = LogFactory.getLog(CommunityMappingUtils.class);
public static CommunitySummary asCommunitySummary(final Context c) {
final CommunitySummary summary = new CommunitySummary();
summary.setId(c.getId());
summary.setShortName(c.getLabel());
summary.setLastUpdateDate(c.getLastUpdateDate());
summary.setCreationDate(c.getCreationDate());
summary.setQueryId(c.getId() + PIPE_SEPARATOR + c.getLabel());
summary.setType(c.getType());
final Map<String, List<Param>> params = c.getParams();
if (params.containsKey(CSUMMARY_DESCRIPTION)) {
summary.setDescription(asCsv(params.get(CSUMMARY_DESCRIPTION)));
}
if (params.containsKey(CSUMMARY_LOGOURL)) {
summary.setLogoUrl(asCsv(params.get(CSUMMARY_LOGOURL)));
}
if (params.containsKey(CSUMMARY_STATUS)) {
summary.setStatus(CommunityStatus.valueOf(firstValue(params, CSUMMARY_STATUS)));
}
if (params.containsKey(CSUMMARY_NAME)) {
summary.setName(asCsv(params.get(CSUMMARY_NAME)));
}
if (params.containsKey(CSUMMARY_ZENODOC)) {
summary.setZenodoCommunity(asCsv(params.get(CSUMMARY_ZENODOC)));
}
return summary;
}
public static CommunityDetails asCommunityProfile(final Context c) {
final CommunityDetails p = new CommunityDetails(asCommunitySummary(c));
p.setLastUpdateDate(c.getLastUpdateDate());
final Map<String, List<Param>> params = c.getParams();
if (params.containsKey(CPROFILE_SUBJECT)) {
p.setSubjects(splitValues(asValues(params.get(CPROFILE_SUBJECT)), CSV_DELIMITER));
}
if (params.containsKey(CPROFILE_CREATIONDATE)){
try {
p.setCreationDate(org.apache.commons.lang3.time.DateUtils.parseDate(asCsv(params.get(CPROFILE_CREATIONDATE)), pattern));
}catch(ParseException e) {
e.printStackTrace();
}
}
return p;
}
public static CommunityProject asCommunityProject(final String communityId, final Concept c) {
final Map<String, List<Param>> p = c.getParams();
final CommunityProject project = new CommunityProject();
project.setCommunityId(communityId);
project.setId(StringUtils.substringAfterLast(c.getId(), ID_SEPARATOR));
project.setOpenaireId(firstValue(p, OPENAIRE_ID));
project.setFunder(firstValue(p, CPROJECT_FUNDER));
project.setGrantId(firstValue(p, CPROJECT_NUMBER));
project.setName(firstValue(p, CPROJECT_FULLNAME));
project.setAcronym(firstValue(p, CPROJECT_ACRONYM));
return project;
}
public static CommunityContentprovider asCommunityDataprovider(final String communityId, final Concept c) {
final Map<String, List<Param>> p = c.getParams();
final CommunityContentprovider d = new CommunityContentprovider();
d.setCommunityId(communityId);
d.setId(StringUtils.substringAfterLast(c.getId(), ID_SEPARATOR));
d.setOpenaireId(firstValue(p, OPENAIRE_ID));
d.setName(firstValue(p, CCONTENTPROVIDER_NAME));
d.setOfficialname(firstValue(p, CCONTENTPROVIDER_OFFICIALNAME));
d.setSelectioncriteria(SelectionCriteria.fromJson(firstValue(p, CCONTENTPROVIDER_SELCRITERIA)));
return d;
}
public static CommunityZenodoCommunity asCommunityZenodoCommunity(final String communityId, final Concept c){
final CommunityZenodoCommunity z = new CommunityZenodoCommunity();
final Map<String, List<Param>> p = c.getParams();
z.setCommunityId(communityId);
z.setId(StringUtils.substringAfterLast(c.getId(), ID_SEPARATOR));
z.setZenodoid(firstValue(p,CZENODOCOMMUNITY_ID));
//z.setName(c.getLabel());
return z;
}
public static CommunityOrganization asCommunityOrganization(String id, Concept c) {
final Map<String, List<Param>> p = c.getParams();
final CommunityOrganization o = new CommunityOrganization();
o.setCommunityId(id);
o.setId(StringUtils.substringAfterLast(c.getId(), ID_SEPARATOR));
o.setName(firstValue(p,CORGANIZATION_NAME));
o.setLogo_url(getDecodedUrl(firstValue(p,CORGANIZATION_LOGOURL)));
o.setWebsite_url(getDecodedUrl(firstValue(p,CORGANIZATION_WEBSITEURL)));
return o;
}
private static String getDecodedUrl(final String encoded_url){
if(encoded_url == null){
return encoded_url;
}
return new String(Base64.getDecoder().decode(encoded_url));
}
private static List<String> splitValues(final Stream<String> stream, final String separator) {
return stream.map(s -> s.split(separator))
.map(Arrays::asList)
.flatMap(List::stream)
.filter(StringUtils::isNotBlank)
.map(StringUtils::trim)
.collect(Collectors.toList());
}
private static String firstValue(final Map<String, List<Param>> p, final String paramName) {
return asValues(p.get(paramName)).findFirst().orElse(null);
}
private static String asCsv(final List<Param> params) {
return asValues(params)
.collect(Collectors.joining(CSV_DELIMITER));
}
private static Stream<String> asValues(final List<Param> params) {
return params == null ? Stream.empty() : params.stream()
.map(Param::getValue)
.map(StringUtils::trim)
.distinct();
}
public static String asProjectXML(final String contextId, final CommunityProject project) {
final Escaper esc = XmlEscapers.xmlAttributeEscaper();
final StringBuilder sb = new StringBuilder();
sb.append(
String.format(
"<concept claim='false' id='%s%s%s%s' label='%s'>\n",
escape(esc, contextId), PROJECTS_ID_SUFFIX, ID_SEPARATOR, escape(esc, String.valueOf(project.getId())), escape(esc, project.getAcronym())));
sb.append(paramXML(CPROJECT_FULLNAME, project.getName()));
sb.append(paramXML(CPROJECT_ACRONYM, project.getAcronym()));
sb.append(paramXML(CPROJECT_NUMBER, project.getGrantId()));
sb.append(paramXML(CPROJECT_FUNDER, project.getFunder()));
sb.append(paramXML(OPENAIRE_ID, project.getOpenaireId()));
sb.append("</concept>\n");
return sb.toString();
}
public static String asContentProviderXML(final String contextId, final CommunityContentprovider ccp) {
log.info("creating the XML for the content provider");
final Escaper esc = XmlEscapers.xmlAttributeEscaper();
final StringBuilder sb = new StringBuilder();
sb.append(
String.format(
"<concept claim='false' id='%s%s%s%s' label='%s'>\n",
escape(esc, contextId), CONTENTPROVIDERS_ID_SUFFIX, ID_SEPARATOR, escape(esc, String.valueOf(ccp.getId())), escape(esc, ccp.getName())));
sb.append(paramXML(OPENAIRE_ID, ccp.getOpenaireId()));
sb.append(paramXML(CCONTENTPROVIDER_NAME, ccp.getName()));
sb.append(paramXML(CCONTENTPROVIDER_OFFICIALNAME, ccp.getOfficialname()));
sb.append(paramXML(CCONTENTPROVIDER_ENABLED,CCONTENTPROVIDERENABLED_DEFAULT));
sb.append(paramXMLNoEscape(CCONTENTPROVIDER_SELCRITERIA, ccp.toXML()));
sb.append("</concept>\n");
log.info(sb.toString());
return sb.toString();
}
public static String asZenodoCommunityXML(final String contextId, final CommunityZenodoCommunity zc) {
final Escaper esc = XmlEscapers.xmlAttributeEscaper();
final StringBuilder sb = new StringBuilder();
sb.append(
String.format(
"<concept claim='false' id='%s%s%s%s' label='%s'>\n",
escape(esc, contextId), ZENODOCOMMUNITY_ID_SUFFIX, ID_SEPARATOR, escape(esc, String.valueOf(zc.getId())), escape(esc, zc.getZenodoid())));
sb.append(paramXML(CZENODOCOMMUNITY_ID, zc.getZenodoid()));
sb.append("</concept>\n");
return sb.toString();
}
public static String asOrganizationXML(final String contextId, CommunityOrganization organization) {
final Escaper esc = XmlEscapers.xmlAttributeEscaper();
final StringBuilder sb = new StringBuilder();
sb.append(
String.format(
"<concept claim='false' id='%s%s%s%s' label='%s'>\n",
escape(esc, contextId), ORGANIZATION_ID_SUFFIX, ID_SEPARATOR, escape(esc, String.valueOf(organization.getId())), escape(esc, organization.getName())));
sb.append(paramXML(CORGANIZATION_NAME, organization.getName()));
sb.append(paramXML(CORGANIZATION_LOGOURL, Base64.getEncoder().encodeToString(organization.getLogo_url().getBytes())));
sb.append(paramXML(CORGANIZATION_WEBSITEURL,Base64.getEncoder().encodeToString(organization.getWebsite_url().getBytes())));
sb.append("</concept>\n");
return sb.toString();
}
private static String paramXML(final String paramName, final String value) {
return String.format("<param name='%s'>%s</param>\n", paramName, escape(XmlEscapers.xmlContentEscaper(), value));
}
private static String paramXMLNoEscape(final String paramName, final String value) {
return String.format("<param name='%s'>%s</param>\n", paramName, value);
}
}

24
apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityNotFoundException.java Normal file
View File

@ -0,0 +1,24 @@
package eu.dnetlib.openaire.community;
import org.springframework.http.HttpStatus;
import org.springframework.web.bind.annotation.ResponseBody;
import org.springframework.web.bind.annotation.ResponseStatus;
@ResponseBody
@ResponseStatus(value = HttpStatus.NOT_FOUND)
public class CommunityNotFoundException extends Exception {
/**
*
*/
private static final long serialVersionUID = -5605421323034135778L;
public CommunityNotFoundException(final String msg) {
super(msg);
}
public CommunityNotFoundException(final Exception e) {
super(e);
}
}

43
apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityOpenAIRECommunities.java Normal file
View File

@ -0,0 +1,43 @@
package eu.dnetlib.openaire.community;
import java.util.ArrayList;
import java.util.List;
import javax.validation.constraints.NotNull;
import io.swagger.v3.oas.annotations.media.Schema;
public class CommunityOpenAIRECommunities {
@NotNull
@Schema(description = "the zenodo community identifier", required = true)
private String zenodoid;
@NotNull
@Schema(description = "identifies this zenodo community within the context it belongs to", required = true)
private List<String> openAirecommunitylist;
public CommunityOpenAIRECommunities() {
this.zenodoid = "";
openAirecommunitylist = new ArrayList<>();
}
public List<String> getOpenAirecommunitylist() {
return openAirecommunitylist;
}
public CommunityOpenAIRECommunities setOpenAirecommunitylist(final List<String> openAirecommunitylist) {
this.openAirecommunitylist = openAirecommunitylist;
return this;
}
public String getZenodoid() {
return zenodoid;
}
public CommunityOpenAIRECommunities setZenodoid(final String zenodoid) {
this.zenodoid = zenodoid;
return this;
}
}

76
apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityOrganization.java Normal file
View File

@ -0,0 +1,76 @@
package eu.dnetlib.openaire.community;
import javax.validation.constraints.NotNull;
import com.fasterxml.jackson.annotation.JsonAutoDetect;
import io.swagger.v3.oas.annotations.media.Schema;
@JsonAutoDetect
public class CommunityOrganization {
@NotNull
@Schema(description = "the community identifier this organization belongs to", required = true)
private String communityId;
@NotNull
@Schema(description = "name of the organization", required = true)
private String name;
@NotNull
@Schema(description = "identifies this organization within the context it belongs to", required = true)
private String id;
@NotNull
@Schema(description = "url of the logo for this organization", required = true)
private String logo_url;
@NotNull
@Schema(description = "website url for this organization", required = true)
private String website_url;
public String getCommunityId() {
return communityId;
}
public CommunityOrganization setCommunityId(final String communityId) {
this.communityId = communityId;
return this;
}
public String getName() {
return name;
}
public CommunityOrganization setName(final String name) {
this.name = name;
return this;
}
public String getId() {
return id;
}
public CommunityOrganization setId(final String id) {
this.id = id;
return this;
}
public String getLogo_url() {
return logo_url;
}
public CommunityOrganization setLogo_url(final String logo_url) {
this.logo_url = logo_url;
return this;
}
public String getWebsite_url() {
return website_url;
}
public CommunityOrganization setWebsite_url(final String website_url) {
this.website_url = website_url;
return this;
}
}

87
apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityProject.java Normal file
View File

@ -0,0 +1,87 @@
package eu.dnetlib.openaire.community;
import com.fasterxml.jackson.annotation.JsonAutoDetect;
import io.swagger.v3.oas.annotations.media.Schema;
@JsonAutoDetect
public class CommunityProject {
@Schema(description = "OpenAIRE identifier for this project, if available", required = false)
private String openaireId;
@Schema(description = "the community identifier this project belongs to", required = true)
private String communityId;
@Schema(description = "identifies this project within the context it belongs to", required = true)
private String id;
@Schema(description = "project name", required = true)
private String name;
@Schema(description = "project acronym", required = false)
private String acronym;
@Schema(description = "project funder", required = true)
private String funder;
@Schema(description = "project grant id", required = true)
private String grantId;
public String getOpenaireId() {
return openaireId;
}
public void setOpenaireId(final String openaireId) {
this.openaireId = openaireId;
}
public String getCommunityId() {
return communityId;
}
public void setCommunityId(final String communityId) {
this.communityId = communityId;
}
public String getId() {
return id;
}
public void setId(final String id) {
this.id = id;
}
public String getName() {
return name;
}
public void setName(final String name) {
this.name = name;
}
public String getAcronym() {
return acronym;
}
public void setAcronym(final String acronym) {
this.acronym = acronym;
}
public String getFunder() {
return funder;
}
public void setFunder(final String funder) {
this.funder = funder;
}
public String getGrantId() {
return grantId;
}
public void setGrantId(final String grantId) {
this.grantId = grantId;
}
}

18
apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityStatus.java Normal file
View File

@ -0,0 +1,18 @@
package eu.dnetlib.openaire.community;
import com.fasterxml.jackson.annotation.JsonAutoDetect;
import io.swagger.v3.oas.annotations.media.Schema;
@JsonAutoDetect
public enum CommunityStatus {
@Schema(description = "restricted visibility")
hidden,
@Schema(description = "visible only to RCD managers")
manager,
@Schema(description = "visible to RCD managers and to the community users")
all
}

174
apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunitySummary.java Normal file
View File

@ -0,0 +1,174 @@
package eu.dnetlib.openaire.community;
import java.util.Date;
import com.fasterxml.jackson.annotation.JsonAutoDetect;
import io.swagger.v3.oas.annotations.media.Schema;
@JsonAutoDetect
public class CommunitySummary {
@Schema(description = "identifies the community")
protected String id;
@Schema(description = "values for this field reflect the index field _community_ in the index, e.g. 'egi||EGI Federation'")
protected String queryId;
@Schema(description = "community type")
protected String type;
@Schema(description = "community name")
protected String name;
@Schema(description = "community short name")
protected String shortName;
@Schema(description = "community creation date")
protected Date creationDate;
@Schema(description = "community last update date")
protected Date lastUpdateDate;
@Schema(description = "community description")
protected String description;
@Schema(description = "http url for the community logo")
protected String logoUrl;
@Schema(description = "status of the community, drives its visibility")
protected CommunityStatus status;
@Schema(description = "Zenodo community associated to this community")
protected String zenodoCommunity;
public CommunitySummary() {}
public CommunitySummary(
final String id,
final String queryId,
final String type,
final String name,
final String shortName,
final Date creationDate,
final Date lastUpdateDate,
final String description,
final String logoUrl,
final CommunityStatus status,
final String zenodoCommunity) {
this.id = id;
this.queryId = queryId;
this.type = type;
this.name = name;
this.shortName = shortName;
this.creationDate = creationDate;
this.lastUpdateDate = lastUpdateDate;
this.description = description;
this.logoUrl = logoUrl;
this.status = status;
this.zenodoCommunity = zenodoCommunity;
}
public CommunitySummary(final CommunitySummary summary) {
this(summary.getId(),
summary.getQueryId(),
summary.getType(),
summary.getName(),
summary.getShortName(),
summary.getCreationDate(),
summary.getLastUpdateDate(),
summary.getDescription(),
summary.getLogoUrl(),
summary.getStatus(),
summary.getZenodoCommunity());
}
public String getId() {
return id;
}
public void setId(final String id) {
this.id = id;
}
public String getQueryId() {
return queryId;
}
public void setQueryId(final String queryId) {
this.queryId = queryId;
}
public String getType() {
return type;
}
public void setType(final String type) {
this.type = type;
}
public String getName() {
return name;
}
public void setName(final String name) {
this.name = name;
}
public String getShortName() {
return shortName;
}
public void setShortName(final String shortName) {
this.shortName = shortName;
}
public Date getCreationDate() {
return creationDate;
}
public void setCreationDate(final Date creationDate) {
this.creationDate = creationDate;
}
public Date getLastUpdateDate() {
return lastUpdateDate;
}
public void setLastUpdateDate(final Date lastUpdateDate) {
this.lastUpdateDate = lastUpdateDate;
}
public String getDescription() {
return description;
}
public void setDescription(final String description) {
this.description = description;
}
public String getLogoUrl() {
return logoUrl;
}
public void setLogoUrl(final String logoUrl) {
this.logoUrl = logoUrl;
}
public CommunityStatus getStatus() {
return status;
}
public void setStatus(final CommunityStatus status) {
this.status = status;
}
public String getZenodoCommunity() {
return zenodoCommunity;
}
public void setZenodoCommunity(final String zenodoCommunity) {
this.zenodoCommunity = zenodoCommunity;
}
}

100
apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityWritableProperties.java Normal file
View File

@ -0,0 +1,100 @@
package eu.dnetlib.openaire.community;
import java.util.List;
import com.fasterxml.jackson.annotation.JsonAutoDetect;
import io.swagger.v3.oas.annotations.media.Schema;
@JsonAutoDetect
public class CommunityWritableProperties {
@Schema(description = "community name")
private String name;
@Schema(description = "community short name")
private String shortName;
@Schema(description = "community description")
private String description;
@Schema(description = "http url for the community logo")
private String logoUrl;
@Schema(description = "list of subjects (keywords) that characterise this community")
private List<String> subjects;
@Schema(description = "status of the community, drives its visibility")
private CommunityStatus status;
@Schema(description = "id of the main Zenodo community")
private String mainZenodoCommunity;
public static CommunityWritableProperties fromDetails(final CommunityDetails details) {
final CommunityWritableProperties p = new CommunityWritableProperties();
p.setName(details.getName());
p.setShortName(details.getShortName());
p.setDescription(details.getDescription());
p.setLogoUrl(details.getLogoUrl());
p.setSubjects(details.getSubjects());
p.setStatus(details.getStatus());
p.setMainZenodoCommunity(details.getZenodoCommunity());
return p;
}
public String getName() {
return name;
}
public void setName(final String name) {
this.name = name;
}
public String getShortName() {
return shortName;
}
public void setShortName(final String shortName) {
this.shortName = shortName;
}
public String getDescription() {
return description;
}
public void setDescription(final String description) {
this.description = description;
}
public String getLogoUrl() {
return logoUrl;
}
public void setLogoUrl(final String logoUrl) {
this.logoUrl = logoUrl;
}
public List<String> getSubjects() {
return subjects;
}
public void setSubjects(final List<String> subjects) {
this.subjects = subjects;
}
public CommunityStatus getStatus() {
return status;
}
public void setStatus(final CommunityStatus status) {
this.status = status;
}
public String getMainZenodoCommunity() {
return mainZenodoCommunity;
}
public void setMainZenodoCommunity(final String mainZenodoCommunity) {
this.mainZenodoCommunity = mainZenodoCommunity;
}
}

48
apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityZenodoCommunity.java Normal file
View File

@ -0,0 +1,48 @@
package eu.dnetlib.openaire.community;
import javax.validation.constraints.NotNull;
import com.fasterxml.jackson.annotation.JsonAutoDetect;
import io.swagger.v3.oas.annotations.media.Schema;
@JsonAutoDetect
public class CommunityZenodoCommunity {
@NotNull
@Schema(description = "the community identifier this zenodo Community belongs to", required = true)
private String communityId;
@NotNull
@Schema(description = "Zenodo identifier for this community", required = true)
private String zenodoid;
@NotNull
@Schema(description = "identifies this zenodo community within the context it belongs to", required = true)
private String id;
public String getZenodoid() {
return zenodoid;
}
public void setZenodoid(final String zenodoid) {
this.zenodoid = zenodoid;
}
public String getId() {
return id;
}
public void setId(final String id) {
this.id = id;
}
public String getCommunityId() {
return communityId;
}
public void setCommunityId(final String communityId) {
this.communityId = communityId;
}
}

45
apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/selectioncriteria/Constraint.java Normal file
View File

@ -0,0 +1,45 @@
package eu.dnetlib.openaire.community.selectioncriteria;
import java.io.Serializable;
import com.fasterxml.jackson.annotation.JsonAutoDetect;
@JsonAutoDetect
public class Constraint implements Serializable {
/**
*
*/
private static final long serialVersionUID = -5996232267609464747L;
private String verb;
private String field;
private String value;
public Constraint() {}
public String getVerb() {
return verb;
}
public void setVerb(final String verb) {
this.verb = verb;
}
public String getField() {
return field;
}
public void setField(final String field) {
this.field = field;
}
public String getValue() {
return value;
}
public void setValue(final String value) {
this.value = value;
}
}

27
apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/selectioncriteria/Constraints.java Normal file
View File

@ -0,0 +1,27 @@
package eu.dnetlib.openaire.community.selectioncriteria;
import java.io.Serializable;
import java.util.List;
import com.fasterxml.jackson.annotation.JsonAutoDetect;
@JsonAutoDetect
public class Constraints implements Serializable {
/**
*
*/
private static final long serialVersionUID = 2694950017620361195L;
private List<Constraint> constraint;
public Constraints() {}
public List<Constraint> getConstraint() {
return constraint;
}
public void setConstraint(final List<Constraint> constraint) {
this.constraint = constraint;
}
}

33
apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/selectioncriteria/SelectionCriteria.java Normal file
View File

@ -0,0 +1,33 @@
package eu.dnetlib.openaire.community.selectioncriteria;
import java.io.Serializable;
import java.util.List;
import com.fasterxml.jackson.annotation.JsonAutoDetect;
import com.google.gson.Gson;
@JsonAutoDetect
public class SelectionCriteria implements Serializable {
/**
*
*/
private static final long serialVersionUID = 4303936216579280542L;
private List<Constraints> criteria;
public SelectionCriteria() {}
public List<Constraints> getCriteria() {
return criteria;
}
public void setCriteria(final List<Constraints> criteria) {
this.criteria = criteria;
}
public static SelectionCriteria fromJson(final String json) {
return new Gson().fromJson(json, SelectionCriteria.class);
}
}

67
apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/context/Category.java Normal file
View File

@ -0,0 +1,67 @@
package eu.dnetlib.openaire.context;
import java.util.List;
import java.util.Map;
public class Category {
private String id;
private String label;
private boolean claim;
private Map<String, List<Param>> params;
private List<Concept> concepts;
public String getId() {
return id;
}
public String getLabel() {
return label;
}
public boolean isClaim() {
return claim;
}
public boolean hasConcepts() {
return getConcepts() != null && !getConcepts().isEmpty();
}
public Map<String, List<Param>> getParams() {
return params;
}
public List<Concept> getConcepts() {
return concepts;
}
public Category setId(final String id) {
this.id = id;
return this;
}
public Category setLabel(final String label) {
this.label = label;
return this;
}
public Category setClaim(final boolean claim) {
this.claim = claim;
return this;
}
public Category setParams(final Map<String, List<Param>> params) {
this.params = params;
return this;
}
public Category setConcepts(final List<Concept> concepts) {
this.concepts = concepts;
return this;
}
}

38
apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/context/CategorySummary.java Normal file
View File

@ -0,0 +1,38 @@
package eu.dnetlib.openaire.context;
public class CategorySummary {
private String id;
private String label;
private boolean hasConcept;
public String getId() {
return id;
}
public String getLabel() {
return label;
}
public boolean isHasConcept() {
return hasConcept;
}
public CategorySummary setId(final String id) {
this.id = id;
return this;
}
public CategorySummary setLabel(final String label) {
this.label = label;
return this;
}
public CategorySummary setHasConcept(final boolean hasConcept) {
this.hasConcept = hasConcept;
return this;
}
}

Some files were not shown because too many files have changed in this diff Show More