Compare commits

..

No commits in common. "3f61e6fce22fbbb851d5937d083af54d530b2d68" and "7511ef94c2aa4b0d441ad218330d5e79bbec0d2d" have entirely different histories.

4 changed files with 20 additions and 18 deletions

View File

@ -44,12 +44,19 @@ public class ServiceScrapeDriver {
private static final Log logger = LogFactory.getLog(ServiceScrapeDriver.class);
public ServiceScrapeDriver(String sitemapUrl, String sitemapURLKey, String maxScrapedPages, String outputFilename, String outputFolder) {
public ServiceScrapeDriver(String sitemapUrl, String sitemapURLKey, String maxScrapedPages, String outputFilename) {
this.sitemapUrl = sitemapUrl;
this.sitemapURLKey = sitemapURLKey;
this.maxScrapedPages = maxScrapedPages;
this.outputFilename = outputFilename;
this.outputFolder = outputFolder;
}
/**
* Runs the scrape process
*
*/
public void start() throws IOException {
runScrape();
}
/**
@ -59,7 +66,7 @@ public class ServiceScrapeDriver {
* as been left in situ in case it is useful in the future.
*
*/
public void runScrape() throws IOException {
private void runScrape() throws IOException {
processProperties();
String url = sitemapUrl.toLowerCase();
Elements urls = UrlParser.getSitemapList(getSitemapUrl(), getSitemapURLKey());
@ -182,6 +189,8 @@ public class ServiceScrapeDriver {
waitTime = Integer.parseInt(prop.getProperty("waitTime").trim());
logger.info(" waitTime: " + waitTime);
outputFolder = prop.getProperty("outputFolder").trim();
logger.info(" outputFolder: " + outputFolder);
numberOfPagesToCrawlInALoop = Integer.parseInt(prop.getProperty("numberOfPagesToCrawlInALoop").trim());
logger.info(" numberOfPagesToCrawl: " + numberOfPagesToCrawlInALoop);
totalNumberOfPagesToCrawlInASession = Integer.parseInt(prop.getProperty("totalNumberOfPagesToCrawlInASession").trim());

View File

@ -44,7 +44,7 @@ public class BioschemasAPIController extends AbstractDnetController {
@GetMapping("/startScraping")
public ScrapingExecution startScraping(@RequestParam final String datasourceKey, @RequestParam final String sitemapUrl, final HttpServletRequest req) {
logger.info("<STARTSCRAPING> datasourceKey: "+datasourceKey+" sitemapUrl:"+sitemapUrl);
return scrapingExecutor.startScraping(datasourceKey, sitemapUrl, getOutputDataPattern(), req.getRemoteAddr(), getOutputFolder());
return scrapingExecutor.startScraping(datasourceKey, sitemapUrl, getOutputDataPattern(), req.getRemoteAddr());
}
@GetMapping("/startScraping/status")
@ -76,12 +76,4 @@ public class BioschemasAPIController extends AbstractDnetController {
public String getOutputDataPattern() {
return outputDataPattern;
}
public void setOutputFolder(String outputFolder) {
this.outputFolder = outputFolder;
}
public void setOutputDataPattern(String outputDataPattern) {
this.outputDataPattern = outputDataPattern;
}
}

View File

@ -14,7 +14,7 @@ public class ScrapingExecutor {
return lastScrapingExecution;
}
public ScrapingExecution startScraping(final String datasourceKey, final String sitemapUrl, final String outputDataPattern, final String remoteAddr, final String outputFolder) {
public ScrapingExecution startScraping(final String datasourceKey, final String sitemapUrl, final String outputDataPattern, final String remoteAddr) {
synchronized (lastScrapingExecution) {
if (lastScrapingExecution.getStatus() != ScrapingStatus.RUNNING) {
lastScrapingExecution.startNew("Scraping for " + datasourceKey + " " + sitemapUrl + " - request from " + remoteAddr);
@ -22,8 +22,8 @@ public class ScrapingExecutor {
try {
String sitemapUrlKey = "loc";
String outputFilename = datasourceKey.concat(outputDataPattern);
ServiceScrapeDriver service = new ServiceScrapeDriver(sitemapUrl, sitemapUrlKey, null, outputFilename, outputFolder);
service.runScrape();
ServiceScrapeDriver service = new ServiceScrapeDriver(sitemapUrl, sitemapUrlKey, null, outputFilename);
service.start();
lastScrapingExecution.complete();
} catch (final Throwable e) {
lastScrapingExecution.fail(e);

View File

@ -1,9 +1,9 @@
server.servlet.context-path=/bioschemas-api
server.servlet.context-path=/bioschemas
server.port=8281
spring.profiles.active=garr
logging.file.name = /var/log/bioschemas-api/bioschemas.log
logging.file.name = /var/log/bioschemas/log/bioschemas-api.log
maven.pom.path = /META-INF/maven/eu.dnetlib.dhp/bioschemas-api/effective-pom.xml
@ -17,8 +17,9 @@ management.endpoints.web.path-mapping.prometheus = metrics
management.endpoints.web.path-mapping.health = health
waitTime=5
outputFolder=/data/bioschemas-harvest
outputFolder=/data
outputDataPattern=_base64_gzipped_nquads.txt
numberOfPagesToCrawlInALoop=8
totalNumberOfPagesToCrawlInASession=32
chromiumDriverLocation = /usr/local/bin/chromedriver
scrapeVersion=1