package eu.dnetlib.bmuse_webapp.publisher; import eu.dnetlib.bmuse_webapp.ServiceScrapeDriver; import eu.dnetlib.common.controller.AbstractDnetController; import org.apache.commons.io.FileUtils; import org.apache.commons.io.IOUtils; import org.apache.commons.io.LineIterator; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.tomcat.jni.FileInfo; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.web.bind.annotation.*; import javax.servlet.http.HttpServletResponse; import java.io.File; import java.io.FileInputStream; import java.io.FileReader; import java.io.IOException; import java.nio.charset.StandardCharsets; import java.util.List; /** * @author enrico.ottonello * */ @RestController @RequestMapping("/api") public class BMUSEWebappController extends AbstractDnetController { private static final Log log = LogFactory.getLog(BMUSEWebappController.class); @RequestMapping(value = "/version", method = RequestMethod.GET) public String version() throws BMUSEWebappException { return "1.0.0-SNAPSHOT"; } @RequestMapping(value = "/scrape", method = RequestMethod.GET) public String scrape(@RequestParam final String datasourceKey, @RequestParam final String sitemapUrl) throws BMUSEWebappException, IOException { log.info("datasourceKey: "+datasourceKey+" sitemapUrl:"+sitemapUrl); // String sitemapUrl = "https://mobidb.org/sitemap2.xml.gz"; scrape?datasourceKey=mobidb&sitemapUrl=https%3A%2F%2Fmobidb.org%2Fsitemap2.xml.gz // String sitemapUrl = "https://proteinensemble.org/sitemap2.xml.gz"; scrape?datasourceKey=ped&sitemapUrl=https%3A%2F%2Fproteinensemble.org%2Fsitemap2.xml.gz // String sitemapUrl = "https://disprot.org/sitemap2.xml.gz"; scrape?datasourceKey=disprot&sitemapUrl=https%3A%2F%2Fdisprot.org%2Fsitemap2.xml.gz String sitemapUrlKey = "loc"; String outputFilename = datasourceKey.concat("_base64_gzipped_nquads.txt"); ServiceScrapeDriver service = new ServiceScrapeDriver(sitemapUrl, sitemapUrlKey, null, outputFilename); service.start(); return "started"; } @RequestMapping(value = "/nquads", method = RequestMethod.GET) public String nquads(HttpServletResponse response) throws BMUSEWebappException, IOException { LineIterator it = FileUtils.lineIterator(new File("/Users/enrico.ottonello/data/bmuse-output/output.nq"), "UTF-8"); try { while (it.hasNext()) { String line = it.nextLine(); response.getOutputStream().write(line.getBytes(StandardCharsets.UTF_8)); response.getOutputStream().println(); } } finally { } return ""; } }