package eu.dnetlib.bioschemas.api.controller; import eu.dnetlib.bioschemas.api.scraper.ScrapingExecution; import eu.dnetlib.bioschemas.api.scraper.ScrapingExecutor; import eu.dnetlib.bioschemas.api.utils.BioschemasException; import eu.dnetlib.common.controller.AbstractDnetController; import org.apache.commons.io.FileUtils; import org.apache.commons.io.LineIterator; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.web.bind.annotation.*; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; import java.io.File; import java.io.IOException; import java.nio.charset.StandardCharsets; /** * @author enrico.ottonello * */ @RestController @RequestMapping("/api") public class BioschemasAPIController extends AbstractDnetController { @Value("${outputFolder}") private String outputFolder; @Value("${outputDataPattern}") private String outputDataPattern; private static Logger logger = LoggerFactory.getLogger(BioschemasAPIController.class); @Autowired private ScrapingExecutor scrapingExecutor; private static final Log log = LogFactory.getLog(BioschemasAPIController.class); @GetMapping("/startScraping") public ScrapingExecution startScraping(@RequestParam final String datasourceKey, @RequestParam final String sitemapUrl, final HttpServletRequest req) { logger.info(" datasourceKey: "+datasourceKey+" sitemapUrl:"+sitemapUrl); return scrapingExecutor.startScraping(datasourceKey, sitemapUrl, getOutputDataPattern(), req.getRemoteAddr()); } @GetMapping("/startScraping/status") public final ScrapingExecution statusScraping() { return scrapingExecutor.getLastScrapingExecution(); } @RequestMapping(value = "/getNQuads", method = RequestMethod.GET) public String getNQuads(@RequestParam final String datasourceKey, HttpServletResponse response) throws BioschemasException, IOException { logger.info(" datasourceKey: "+datasourceKey); LineIterator it = FileUtils.lineIterator(new File(getOutputFolder().concat("/").concat(datasourceKey).concat(getOutputDataPattern())), "UTF-8"); try { while (it.hasNext()) { String line = it.nextLine(); response.getOutputStream().write(line.getBytes(StandardCharsets.UTF_8)); response.getOutputStream().println(); } } finally { } return ""; } public String getOutputFolder() { return outputFolder; } public String getOutputDataPattern() { return outputDataPattern; } }