apps/bioschemas-api/src/main/java/eu/dnetlib/bioschemas/api/ServiceScrapeDriver.java

@@ -44,12 +44,19 @@ public class ServiceScrapeDriver {
 
 	private static final Log logger = LogFactory.getLog(ServiceScrapeDriver.class);
 
-	public ServiceScrapeDriver(String sitemapUrl, String sitemapURLKey, String maxScrapedPages, String outputFilename, String outputFolder) {
+	public ServiceScrapeDriver(String sitemapUrl, String sitemapURLKey, String maxScrapedPages, String outputFilename) {
 		this.sitemapUrl = sitemapUrl;
 		this.sitemapURLKey = sitemapURLKey;
 		this.maxScrapedPages = maxScrapedPages;
 		this.outputFilename = outputFilename;
-		this.outputFolder = outputFolder;
+	}
+
+	/**
+	 * Runs the scrape process
+	 *
+	 */
+	public void start() throws IOException {
+		runScrape();
 	}
 	
 	/** 
@@ -59,7 +66,7 @@ public class ServiceScrapeDriver {
 	 * as been left in situ in case it is useful in the future.
 	 * 
 	 */
-	public void runScrape() throws IOException {
+	private void runScrape() throws IOException {
 		processProperties();
 		String url = sitemapUrl.toLowerCase();
 		Elements urls = UrlParser.getSitemapList(getSitemapUrl(), getSitemapURLKey());
@@ -182,6 +189,8 @@ public class ServiceScrapeDriver {
 
 		waitTime = Integer.parseInt(prop.getProperty("waitTime").trim());
 		logger.info("     waitTime: " + waitTime);
+		outputFolder = prop.getProperty("outputFolder").trim();
+		logger.info("     outputFolder: " + outputFolder);		
 		numberOfPagesToCrawlInALoop = Integer.parseInt(prop.getProperty("numberOfPagesToCrawlInALoop").trim());
 		logger.info("     numberOfPagesToCrawl: " + numberOfPagesToCrawlInALoop);
 		totalNumberOfPagesToCrawlInASession = Integer.parseInt(prop.getProperty("totalNumberOfPagesToCrawlInASession").trim());

apps/bioschemas-api/src/main/java/eu/dnetlib/bioschemas/api/controller/BioschemasAPIController.java

@@ -44,7 +44,7 @@ public class BioschemasAPIController extends AbstractDnetController {
 	@GetMapping("/startScraping")
 	public ScrapingExecution startScraping(@RequestParam final String datasourceKey, @RequestParam final String sitemapUrl, final HttpServletRequest req) {
 		logger.info("<STARTSCRAPING> datasourceKey: "+datasourceKey+" sitemapUrl:"+sitemapUrl);
-		return scrapingExecutor.startScraping(datasourceKey, sitemapUrl, getOutputDataPattern(), req.getRemoteAddr(), getOutputFolder());
+		return scrapingExecutor.startScraping(datasourceKey, sitemapUrl, getOutputDataPattern(), req.getRemoteAddr());
 	}
 
 	@GetMapping("/startScraping/status")
@@ -76,12 +76,4 @@ public class BioschemasAPIController extends AbstractDnetController {
 	public String getOutputDataPattern() {
 		return outputDataPattern;
 	}
-
-	public void setOutputFolder(String outputFolder) {
-		this.outputFolder = outputFolder;
-	}
-
-	public void setOutputDataPattern(String outputDataPattern) {
-		this.outputDataPattern = outputDataPattern;
-	}
 }

apps/bioschemas-api/src/main/java/eu/dnetlib/bioschemas/api/scraper/ScrapingExecutor.java

@@ -14,7 +14,7 @@ public class ScrapingExecutor {
 		return lastScrapingExecution;
 	}
 
-	public ScrapingExecution startScraping(final String datasourceKey, final String sitemapUrl, final String outputDataPattern, final String remoteAddr, final String outputFolder) {
+	public ScrapingExecution startScraping(final String datasourceKey, final String sitemapUrl, final String outputDataPattern, final String remoteAddr) {
 		synchronized (lastScrapingExecution) {
 			if (lastScrapingExecution.getStatus() != ScrapingStatus.RUNNING) {
 				lastScrapingExecution.startNew("Scraping for " + datasourceKey + " " + sitemapUrl + " - request from " + remoteAddr);
@@ -22,8 +22,8 @@ public class ScrapingExecutor {
 					try {
 						String sitemapUrlKey = "loc";
 						String outputFilename = datasourceKey.concat(outputDataPattern);
-						ServiceScrapeDriver service = new ServiceScrapeDriver(sitemapUrl, sitemapUrlKey, null, outputFilename, outputFolder);
-						service.runScrape();
+						ServiceScrapeDriver service = new ServiceScrapeDriver(sitemapUrl, sitemapUrlKey, null, outputFilename);
+						service.start();
 						lastScrapingExecution.complete();
 					} catch (final Throwable e) {
 						lastScrapingExecution.fail(e);

apps/bioschemas-api/src/main/resources/application.properties

@@ -1,9 +1,9 @@
-server.servlet.context-path=/bioschemas-api
+server.servlet.context-path=/bioschemas
 server.port=8281
 
 spring.profiles.active=garr
 
-logging.file.name = /var/log/bioschemas-api/bioschemas.log
+logging.file.name = /var/log/bioschemas/log/bioschemas-api.log
 
 maven.pom.path = /META-INF/maven/eu.dnetlib.dhp/bioschemas-api/effective-pom.xml
 
@@ -17,8 +17,9 @@ management.endpoints.web.path-mapping.prometheus    =   metrics
 management.endpoints.web.path-mapping.health        =   health
 
 waitTime=5
-outputFolder=/data/bioschemas-harvest
+outputFolder=/data
 outputDataPattern=_base64_gzipped_nquads.txt
 numberOfPagesToCrawlInALoop=8
 totalNumberOfPagesToCrawlInASession=32
+chromiumDriverLocation = /usr/local/bin/chromedriver
 scrapeVersion=1