65 lines
2.5 KiB
Java
65 lines
2.5 KiB
Java
package eu.dnetlib.bmuse_webapp.publisher;
|
|
|
|
import eu.dnetlib.bmuse_webapp.ServiceScrapeDriver;
|
|
import eu.dnetlib.common.controller.AbstractDnetController;
|
|
import org.apache.commons.io.FileUtils;
|
|
import org.apache.commons.io.IOUtils;
|
|
import org.apache.commons.io.LineIterator;
|
|
import org.apache.commons.logging.Log;
|
|
import org.apache.commons.logging.LogFactory;
|
|
import org.apache.tomcat.jni.FileInfo;
|
|
import org.springframework.beans.factory.annotation.Autowired;
|
|
import org.springframework.web.bind.annotation.*;
|
|
|
|
import javax.servlet.http.HttpServletResponse;
|
|
import java.io.File;
|
|
import java.io.FileInputStream;
|
|
import java.io.FileReader;
|
|
import java.io.IOException;
|
|
import java.nio.charset.StandardCharsets;
|
|
import java.util.List;
|
|
|
|
/**
|
|
* @author enrico.ottonello
|
|
*
|
|
*/
|
|
|
|
@RestController
|
|
@RequestMapping("/api")
|
|
public class BMUSEWebappController extends AbstractDnetController {
|
|
|
|
private static final Log log = LogFactory.getLog(BMUSEWebappController.class);
|
|
|
|
@RequestMapping(value = "/version", method = RequestMethod.GET)
|
|
public String version() throws BMUSEWebappException {
|
|
return "1.0.0-SNAPSHOT";
|
|
}
|
|
|
|
@RequestMapping(value = "/scrape", method = RequestMethod.GET)
|
|
public String scrape(@RequestParam final String datasourceKey, @RequestParam final String sitemapUrl) throws BMUSEWebappException, IOException {
|
|
|
|
log.info("datasourceKey: "+datasourceKey+" sitemapUrl:"+sitemapUrl);
|
|
// String sitemapUrl = "https://mobidb.org/sitemap2.xml.gz"; scrape?datasourceKey=mobidb&sitemapUrl=https%3A%2F%2Fmobidb.org%2Fsitemap2.xml.gz
|
|
// String sitemapUrl = "https://proteinensemble.org/sitemap2.xml.gz"; scrape?datasourceKey=ped&sitemapUrl=https%3A%2F%2Fproteinensemble.org%2Fsitemap2.xml.gz
|
|
// String sitemapUrl = "https://disprot.org/sitemap2.xml.gz"; scrape?datasourceKey=disprot&sitemapUrl=https%3A%2F%2Fdisprot.org%2Fsitemap2.xml.gz
|
|
String sitemapUrlKey = "loc";
|
|
String outputFilename = datasourceKey.concat("_base64_gzipped_nquads.txt");
|
|
ServiceScrapeDriver service = new ServiceScrapeDriver(sitemapUrl, sitemapUrlKey, null, outputFilename);
|
|
service.start();
|
|
return "started";
|
|
}
|
|
|
|
@RequestMapping(value = "/nquads", method = RequestMethod.GET)
|
|
public String nquads(HttpServletResponse response) throws BMUSEWebappException, IOException {
|
|
LineIterator it = FileUtils.lineIterator(new File("/Users/enrico.ottonello/data/bmuse-output/output.nq"), "UTF-8");
|
|
try {
|
|
while (it.hasNext()) {
|
|
String line = it.nextLine();
|
|
response.getOutputStream().write(line.getBytes(StandardCharsets.UTF_8));
|
|
response.getOutputStream().println();
|
|
}
|
|
} finally {
|
|
}
|
|
return "";
|
|
}
|
|
} |