diff --git a/dnet-ariadneplus/src/main/java/eu/dnetlib/ariadneplus/workflows/nodes/ClarinAPIGeneratorJobNode.java b/dnet-ariadneplus/src/main/java/eu/dnetlib/ariadneplus/workflows/nodes/ClarinAPIGeneratorJobNode.java deleted file mode 100644 index 14592c5..0000000 --- a/dnet-ariadneplus/src/main/java/eu/dnetlib/ariadneplus/workflows/nodes/ClarinAPIGeneratorJobNode.java +++ /dev/null @@ -1,122 +0,0 @@ -package eu.dnetlib.ariadneplus.workflows.nodes; - -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.List; - -import eu.dnetlib.clients.enabling.ISLookUpClient; -import eu.dnetlib.enabling.locators.UniqueServiceLocator; -import eu.dnetlib.msro.workflows.graph.Arc; -import eu.dnetlib.msro.workflows.nodes.AsyncJobNode; -import eu.dnetlib.msro.workflows.procs.Env; -import eu.dnetlib.msro.workflows.util.WorkflowsConstants; -import eu.dnetlib.rmi.datasource.DatasourceManagerService; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.springframework.beans.factory.annotation.Autowired; - -/** - * Created by Alessia Bardi on 12/01/2018. - * - * @author Alessia Bardi - */ -public class ClarinAPIGeneratorJobNode extends AsyncJobNode { - - private static final Log log = LogFactory.getLog(ClarinAPIGeneratorJobNode.class); - - /** Path to a local folder containing the subfolders with Clarin mapping files **/ - private String folderPath; - /** Path to a local folder containing the subfolders with Clarin input files **/ - private String inputBaseUrlPrefix; - - private String metadataIdentifierPath; - - private String clarinDatasourceProfileID; - private String clarinDatasourceOriginalId; - - @Autowired - private UniqueServiceLocator serviceLocator; - @Autowired - private ISLookUpClient isLookupClient; - - - @Override - protected String execute(final Env env) throws Exception { - Path folder = Paths.get(getFolderPath()); - if (!Files.isDirectory(folder)){ - throw new IllegalArgumentException(getFolderPath()+" must be a directory"); - } - final DatasourceManagerService dsMan = getServiceLocator().getService(DatasourceManagerService.class); - - List apiIds = getIsLookupClient().search("//RESOURCE_PROFILE[./HEADER/RESOURCE_IDENTIFIER/@value='"+getClarinDatasourceProfileID()+"']//INTERFACE/@id/string()"); - - ClarinFileVisitor visitor = new ClarinFileVisitor(getClarinDatasourceProfileID(), getClarinDatasourceOriginalId(), - getInputBaseUrlPrefix(), getMetadataIdentifierPath(), apiIds, getServiceLocator(), getIsLookupClient()); - Files.walkFileTree(folder, visitor); - - env.setAttribute(WorkflowsConstants.MAIN_LOG_PREFIX +"visitedFiles", visitor.getCountVisitedFiles()); - env.setAttribute(WorkflowsConstants.MAIN_LOG_PREFIX +"createdTDS", visitor.getCountCreatedTDS()); - env.setAttribute(WorkflowsConstants.MAIN_LOG_PREFIX +"updatedTDS", visitor.getCountUpdatedTDS()); - env.setAttribute(WorkflowsConstants.MAIN_LOG_PREFIX +"visitedFolders", visitor.getCountVisitedFolders()); - env.setAttribute(WorkflowsConstants.MAIN_LOG_PREFIX +"createdInterfaces", visitor.getCountCreatedInterfaces()); - return Arc.DEFAULT_ARC; - - } - - public String getFolderPath() { - return folderPath; - } - - public void setFolderPath(final String folderPath) { - this.folderPath = folderPath; - } - - public String getInputBaseUrlPrefix() { - return inputBaseUrlPrefix; - } - - public void setInputBaseUrlPrefix(final String inputBaseUrlPrefix) { - this.inputBaseUrlPrefix = inputBaseUrlPrefix; - } - - public String getMetadataIdentifierPath() { - return metadataIdentifierPath; - } - - public void setMetadataIdentifierPath(final String metadataIdentifierPath) { - this.metadataIdentifierPath = metadataIdentifierPath; - } - - public String getClarinDatasourceProfileID() { - return clarinDatasourceProfileID; - } - - public void setClarinDatasourceProfileID(final String clarinDatasourceProfileID) { - this.clarinDatasourceProfileID = clarinDatasourceProfileID; - } - - public String getClarinDatasourceOriginalId() { - return clarinDatasourceOriginalId; - } - - public void setClarinDatasourceOriginalId(final String clarinDatasourceOriginalId) { - this.clarinDatasourceOriginalId = clarinDatasourceOriginalId; - } - - public UniqueServiceLocator getServiceLocator() { - return serviceLocator; - } - - public void setServiceLocator(final UniqueServiceLocator serviceLocator) { - this.serviceLocator = serviceLocator; - } - - public ISLookUpClient getIsLookupClient() { - return isLookupClient; - } - - public void setIsLookupClient(final ISLookUpClient isLookupClient) { - this.isLookupClient = isLookupClient; - } -} diff --git a/dnet-ariadneplus/src/main/java/eu/dnetlib/ariadneplus/workflows/nodes/ClarinFileVisitor.java b/dnet-ariadneplus/src/main/java/eu/dnetlib/ariadneplus/workflows/nodes/ClarinFileVisitor.java deleted file mode 100644 index 9eaa603..0000000 --- a/dnet-ariadneplus/src/main/java/eu/dnetlib/ariadneplus/workflows/nodes/ClarinFileVisitor.java +++ /dev/null @@ -1,263 +0,0 @@ -package eu.dnetlib.ariadneplus.workflows.nodes; - -import java.io.IOException; -import java.nio.charset.Charset; -import java.nio.file.FileVisitResult; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.SimpleFileVisitor; -import java.nio.file.attribute.BasicFileAttributes; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; - -import com.google.common.collect.Lists; -import com.google.common.collect.Maps; -import eu.dnetlib.clients.enabling.ISLookUpClient; -import eu.dnetlib.enabling.locators.UniqueServiceLocator; -import eu.dnetlib.miscutils.datetime.DateUtils; -import eu.dnetlib.rmi.datasource.DatasourceManagerService; -import eu.dnetlib.rmi.datasource.DatasourceManagerServiceException; -import eu.dnetlib.rmi.datasource.IfaceDesc; -import eu.dnetlib.rmi.enabling.ISLookUpException; -import eu.dnetlib.rmi.enabling.ISRegistryException; -import eu.dnetlib.rmi.enabling.ISRegistryService; -import org.antlr.stringtemplate.StringTemplate; -import org.apache.commons.io.IOUtils; -import org.apache.commons.lang3.StringUtils; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; - -import static java.nio.file.FileVisitResult.CONTINUE; - -/** - * Created by Alessia Bardi on 12/01/2018. - * - * @author Alessia Bardi - */ -public class ClarinFileVisitor extends SimpleFileVisitor { - - private static final Log log = LogFactory.getLog(ClarinFileVisitor.class); - private static final String API_PREFIX = "api_________::"; - private static final String TDS_TEMPLATE ="/eu/dnetlib/ariadneplus/workflows/nodes/clarin_tds.xml.st"; - - private List interfaces = Lists.newArrayList(); - private String clarinDatasourceProfileID; - private String clarinDatasourceOriginalId; - private DatasourceManagerService dsMan = null; - private String inputBaseUrlPrefix; - private String metadataIdentifierPath; - private ISLookUpClient lookupClient; - private ISRegistryService registryService; - - private int countVisitedFiles = 0; - private int countCreatedTDS = 0; - private int countUpdatedTDS = 0; - private int countVisitedFolders = 0; - private int countCreatedInterfaces = 0; - - - @Override - public FileVisitResult preVisitDirectory(final Path dir, final BasicFileAttributes attrs) throws IOException { - log.info("Processing " + dir.toString()); - countVisitedFolders++; - String dirName = dir.getFileName().toString(); - if(dirName.equalsIgnoreCase("x3ml-mappings")) return CONTINUE; - String apiId = API_PREFIX + getClarinDatasourceOriginalId() + "::" + dirName; - if (!interfaces.contains(apiId)) { - //API TO BE CREATED - IfaceDesc iface = new IfaceDesc(); - iface.setActive(false); - iface.setCompliance("metadata"); - iface.setContentDescription("metadata"); - iface.setId(apiId); - iface.setRemovable(true); - iface.setTypology("dnet:repository::clarin"); - iface.setAccessProtocol("filesystem"); - Map accessParams = Maps.newHashMap(); - accessParams.put("extensions", "xml"); - iface.setAccessParams(accessParams); - iface.setBaseUrl(StringUtils.appendIfMissing(getInputBaseUrlPrefix(), "/") + dirName); - Map extraFields = Maps.newHashMap(); - extraFields.put("metadata_identifier_path", getMetadataIdentifierPath()); - iface.setExtraFields(extraFields); - try { - dsMan.addInterface(getClarinDatasourceProfileID(), iface); - countCreatedInterfaces++; - log.info("CREATED NEW INTERFACE " + iface.getId() + " for " + getClarinDatasourceOriginalId() + "(" + getClarinDatasourceProfileID() + ")"); - } catch (DatasourceManagerServiceException e) { - log.error("Can't add interface " + iface.getId() + " to " + getClarinDatasourceOriginalId() + "(" + getClarinDatasourceProfileID() + ")", e); - return CONTINUE; - } - } else { - if (log.isDebugEnabled()) { - log.debug("Interface " + apiId + " already exists"); - } - } - return CONTINUE; - } - - @Override - public FileVisitResult visitFile(final Path file, final BasicFileAttributes attrs) throws IOException { - String filename = file.getFileName().toString(); - log.info("Processing " + file.toString()); - countVisitedFiles++; - String tdsTitle = filename; - //call Files.lines which will use a stream to iterate over each line of the file. - //Next we will convert the stream to a string by calling Collectors.joining() which will join all the strings together. - String updatedCode = Files.lines(file).collect(Collectors.joining()).replace("", ""); - try { - List res = this.lookupClient.search("//RESOURCE_PROFILE[.//RESOURCE_TYPE/@value=\"TransformationRuleDSResourceType\" and .//CONFIGURATION/SCRIPT/TITLE/string()=\""+tdsTitle+"\"]/HEADER/RESOURCE_IDENTIFIER/@value/string()"); - if(res == null || res.isEmpty()){ - log.debug("Creating new TDS profile for "+filename); - final String template = IOUtils.toString(getClass().getResourceAsStream(TDS_TEMPLATE), Charset.forName("UTF-8")); - final StringTemplate st = new StringTemplate(template); - st.setAttribute("date", DateUtils.calculate_ISO8601(DateUtils.now())); - st.setAttribute("title", tdsTitle); - st.setAttribute("mapping", ""); - String profId = this.registryService.registerProfile(st.toString()); - countCreatedTDS++; - log.info("REGISTERED NEW TDS FOR "+filename+": "+profId); - } - else{ - String tdsProfileId = res.get(0); - log.debug("Updating TDS profile "+tdsProfileId+"for "+filename); - - boolean done = this.registryService.updateProfileNode(tdsProfileId, "//CONFIGURATION/SCRIPT/CODE", ""); - if(done){ - log.info("TDS PROFILE "+tdsProfileId+" UPDATED with contents from "+filename); - countUpdatedTDS++; - } - if(!done){ - log.error("!!! TDS PROFILE "+tdsProfileId+" COULD NOT BE UPDATED with contents from "+filename); - } - } - } catch (ISLookUpException | ISRegistryException e) { - log.error("CANNOT UPDATE/CREATE TDS PROFILE FOR "+filename, e); - } - - return CONTINUE; - } - - protected ClarinFileVisitor() { - super(); - } - - public ClarinFileVisitor(final String clarinDatasourceProfileID, final String clarinDatasourceOriginalId, - final String inputBaseUrlPrefix, final String metadataIdentifierPath, final List interfaces, - final UniqueServiceLocator locator, final ISLookUpClient lookupClient) { - super(); - this.clarinDatasourceOriginalId = clarinDatasourceOriginalId; - this.clarinDatasourceProfileID = clarinDatasourceProfileID; - this.inputBaseUrlPrefix = inputBaseUrlPrefix; - this.metadataIdentifierPath = metadataIdentifierPath; - this.interfaces = interfaces; - this.dsMan = locator.getService(DatasourceManagerService.class); - this.registryService = locator.getService(ISRegistryService.class); - this.lookupClient = lookupClient; - } - - public List getInterfaces() { - return interfaces; - } - - public void setInterfaces(final List interfaces) { - this.interfaces = interfaces; - } - - public String getClarinDatasourceOriginalId() { - return clarinDatasourceOriginalId; - } - - public void setClarinDatasourceOriginalId(final String clarinDatasourceOriginalId) { - this.clarinDatasourceOriginalId = clarinDatasourceOriginalId; - } - - public DatasourceManagerService getDsMan() { - return dsMan; - } - - public void setDsMan(final DatasourceManagerService dsMan) { - this.dsMan = dsMan; - } - - public String getInputBaseUrlPrefix() { - return inputBaseUrlPrefix; - } - - public void setInputBaseUrlPrefix(final String inputBaseUrlPrefix) { - this.inputBaseUrlPrefix = inputBaseUrlPrefix; - } - - public String getMetadataIdentifierPath() { - return metadataIdentifierPath; - } - - public void setMetadataIdentifierPath(final String metadataIdentifierPath) { - this.metadataIdentifierPath = metadataIdentifierPath; - } - - public String getClarinDatasourceProfileID() { - return clarinDatasourceProfileID; - } - - public void setClarinDatasourceProfileID(final String clarinDatasourceProfileID) { - this.clarinDatasourceProfileID = clarinDatasourceProfileID; - } - - public ISLookUpClient getLookupClient() { - return lookupClient; - } - - public void setLookupClient(final ISLookUpClient lookupClient) { - this.lookupClient = lookupClient; - } - - public ISRegistryService getRegistryService() { - return registryService; - } - - public void setRegistryService(final ISRegistryService registryService) { - this.registryService = registryService; - } - - public int getCountVisitedFiles() { - return countVisitedFiles; - } - - public void setCountVisitedFiles(final int countVisitedFiles) { - this.countVisitedFiles = countVisitedFiles; - } - - public int getCountCreatedTDS() { - return countCreatedTDS; - } - - public void setCountCreatedTDS(final int countCreatedTDS) { - this.countCreatedTDS = countCreatedTDS; - } - - public int getCountUpdatedTDS() { - return countUpdatedTDS; - } - - public void setCountUpdatedTDS(final int countUpdatedTDS) { - this.countUpdatedTDS = countUpdatedTDS; - } - - public int getCountVisitedFolders() { - return countVisitedFolders; - } - - public void setCountVisitedFolders(final int countVisitedFolders) { - this.countVisitedFolders = countVisitedFolders; - } - - public int getCountCreatedInterfaces() { - return countCreatedInterfaces; - } - - public void setCountCreatedInterfaces(final int countCreatedInterfaces) { - this.countCreatedInterfaces = countCreatedInterfaces; - } -} diff --git a/dnet-ariadneplus/src/main/java/eu/dnetlib/ariadneplus/workflows/nodes/ReadVirtuosoByTypeJobNode.java b/dnet-ariadneplus/src/main/java/eu/dnetlib/ariadneplus/workflows/nodes/ReadVirtuosoByTypeJobNode.java deleted file mode 100644 index 6ad9c9d..0000000 --- a/dnet-ariadneplus/src/main/java/eu/dnetlib/ariadneplus/workflows/nodes/ReadVirtuosoByTypeJobNode.java +++ /dev/null @@ -1,51 +0,0 @@ -package eu.dnetlib.ariadneplus.workflows.nodes; - -import eu.dnetlib.msro.workflows.graph.Arc; -import eu.dnetlib.msro.workflows.procs.Env; -import eu.dnetlib.rmi.common.ResultSet; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.springframework.web.client.RestTemplate; - -/** - * Created by Alessia Bardi on 26/01/2018. - * - * @author Alessia Bardi - */ -public class ReadVirtuosoByTypeJobNode extends ReadVirtuosoJobNode { - - private static final Log log = LogFactory.getLog(ReadVirtuosoByTypeJobNode.class); - - private String typeNamespace; - private String typeName; - - @Override - protected String execute(final Env env){ - log.info("Using virtuoso reader at : " + getVirtuosoReaderAPIUrl()); - RestTemplate restTemplate = new RestTemplate(getClientHttpRequestFactory()); - - VirtuosoAriadnePlusByTypeIterator iterator = new VirtuosoAriadnePlusByTypeIterator().typeNamespace(typeNamespace).typeName(typeName).datasourceInterface(getDatasourceInterface()).virtuosoReaderAPIUrl(getVirtuosoReaderAPIUrl()).datasourceName(getDatasourceName()).restTemplate(restTemplate); - log.debug(String.format("Created iterator for %s:%s : ", typeNamespace, typeName )); - ResultSet rs = getRsFactory().createResultSet(() -> iterator); - - env.setAttribute("virtuoso_rs", rs); - - return Arc.DEFAULT_ARC; - } - - public String getTypeNamespace() { - return typeNamespace; - } - - public void setTypeNamespace(final String typeNamespace) { - this.typeNamespace = typeNamespace; - } - - public String getTypeName() { - return typeName; - } - - public void setTypeName(final String typeName) { - this.typeName = typeName; - } -} diff --git a/dnet-ariadneplus/src/main/java/eu/dnetlib/ariadneplus/workflows/nodes/ReadVirtuosoJobNode.java b/dnet-ariadneplus/src/main/java/eu/dnetlib/ariadneplus/workflows/nodes/ReadVirtuosoJobNode.java deleted file mode 100644 index 4c2c45d..0000000 --- a/dnet-ariadneplus/src/main/java/eu/dnetlib/ariadneplus/workflows/nodes/ReadVirtuosoJobNode.java +++ /dev/null @@ -1,87 +0,0 @@ -package eu.dnetlib.ariadneplus.workflows.nodes; - -import eu.dnetlib.enabling.resultset.factory.ResultSetFactory; -import eu.dnetlib.msro.workflows.graph.Arc; -import eu.dnetlib.msro.workflows.nodes.AsyncJobNode; -import eu.dnetlib.msro.workflows.procs.Env; -import eu.dnetlib.rmi.common.ResultSet; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.http.client.ClientHttpRequestFactory; -import org.springframework.http.client.HttpComponentsClientHttpRequestFactory; -import org.springframework.web.client.RestTemplate; - -/** - * Created by Alessia Bardi on 26/01/2018. - * - * @author Alessia Bardi - */ -public class ReadVirtuosoJobNode extends AsyncJobNode { - - private static final Log log = LogFactory.getLog(ReadVirtuosoJobNode.class); - - private String datasourceName; - private String datasourceInterface; - private String virtuosoReaderAPIUrl; - private int readTimeout = 30000; - - @Autowired - private ResultSetFactory rsFactory; - - @Override - protected String execute(final Env env){ - log.info("Using virtuoso reader at : " + getVirtuosoReaderAPIUrl()); - RestTemplate restTemplate = new RestTemplate(getClientHttpRequestFactory()); - - VirtuosoAriadnePlusIterator iterator = new VirtuosoAriadnePlusIterator().datasourceInterface(getDatasourceInterface()).virtuosoReaderAPIUrl(getVirtuosoReaderAPIUrl()).datasourceName(getDatasourceName()).restTemplate(restTemplate); - ResultSet rs = rsFactory.createResultSet(() -> iterator); - - env.setAttribute("virtuoso_rs", rs); - - return Arc.DEFAULT_ARC; - } - - - protected ClientHttpRequestFactory getClientHttpRequestFactory() { - HttpComponentsClientHttpRequestFactory clientHttpRequestFactory = new HttpComponentsClientHttpRequestFactory(); - clientHttpRequestFactory.setReadTimeout(readTimeout); - return clientHttpRequestFactory; - } - - public String getDatasourceName() { - return datasourceName; - } - - public void setDatasourceName(final String datasourceName) { - this.datasourceName = datasourceName; - } - - public String getDatasourceInterface() { - return datasourceInterface; - } - - public void setDatasourceInterface(final String datasourceInterface) { - this.datasourceInterface = datasourceInterface; - } - - public String getVirtuosoReaderAPIUrl() { - return virtuosoReaderAPIUrl; - } - - public void setVirtuosoReaderAPIUrl(final String virtuosoReaderAPIUrl) { - this.virtuosoReaderAPIUrl = virtuosoReaderAPIUrl; - } - - public int getReadTimeout() { - return readTimeout; - } - - public void setReadTimeout(final int readTimeout) { - this.readTimeout = readTimeout; - } - - public ResultSetFactory getRsFactory() { - return rsFactory; - } -} diff --git a/dnet-ariadneplus/src/main/java/eu/dnetlib/ariadneplus/workflows/nodes/RegisterClarinWorkflowFromTemplateJobNode.java b/dnet-ariadneplus/src/main/java/eu/dnetlib/ariadneplus/workflows/nodes/RegisterClarinWorkflowFromTemplateJobNode.java deleted file mode 100644 index 8e39398..0000000 --- a/dnet-ariadneplus/src/main/java/eu/dnetlib/ariadneplus/workflows/nodes/RegisterClarinWorkflowFromTemplateJobNode.java +++ /dev/null @@ -1,108 +0,0 @@ -package eu.dnetlib.ariadneplus.workflows.nodes; - -import java.nio.charset.Charset; - -import com.google.common.base.Joiner; -import eu.dnetlib.clients.enabling.ISLookUpClient; -import eu.dnetlib.enabling.locators.UniqueServiceLocator; -import eu.dnetlib.msro.workflows.graph.Arc; -import eu.dnetlib.msro.workflows.nodes.SimpleJobNode; -import eu.dnetlib.msro.workflows.procs.Env; -import eu.dnetlib.msro.workflows.procs.ProcessAware; -import eu.dnetlib.msro.workflows.procs.WorkflowProcess; -import eu.dnetlib.msro.workflows.util.WorkflowsConstants; -import eu.dnetlib.rmi.enabling.ISLookUpException; -import eu.dnetlib.rmi.enabling.ISRegistryService; -import org.antlr.stringtemplate.StringTemplate; -import org.apache.commons.io.IOUtils; -import org.apache.commons.lang3.StringEscapeUtils; -import org.apache.commons.lang3.StringUtils; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.springframework.beans.factory.annotation.Autowired; - -/** - * Copied from eu.dnetlib.msro.workflows.nodes.repohi.RegisterWorkflowFromTemplateJobNode - * - * Clarin specific implementation for setting the transformation rules automatically. - * - * @author alessia - */ -public class RegisterClarinWorkflowFromTemplateJobNode extends SimpleJobNode implements ProcessAware { - - private static final Log log = LogFactory.getLog(RegisterClarinWorkflowFromTemplateJobNode.class); - private String wfName; - private String wfTemplate; - private String description; - private WorkflowProcess process; - @Autowired - private UniqueServiceLocator serviceLocator; - @Autowired - private ISLookUpClient isLookUpClient; - - public String getWfName() { - return this.wfName; - } - - public void setWfName(final String wfName) { - this.wfName = wfName; - } - - public String getWfTemplate() { - return this.wfTemplate; - } - - public void setWfTemplate(final String wfTemplate) { - this.wfTemplate = wfTemplate; - } - - public String getDescription() { - return this.description; - } - - public void setDescription(final String description) { - this.description = description; - } - - @Override - protected String execute(final Env env) throws Exception { - final String dsId = this.process.getDsId(); - final String ifaceId = this.process.getDsInterface(); - final String dsName = this.process.getDsName(); - - final StringTemplate profTemplate = new StringTemplate(IOUtils.toString(getClass().getResourceAsStream(getWfTemplate()), Charset.forName("UTF-8"))); - profTemplate.setAttribute("name", StringEscapeUtils.escapeXml11(this.wfName)); - profTemplate.setAttribute("desc", StringEscapeUtils.escapeXml11(this.description)); - profTemplate.setAttribute("priority", WorkflowsConstants.DEFAULT_WF_PRIORITY); - profTemplate.setAttribute("dsId", StringEscapeUtils.escapeXml11(dsId)); - profTemplate.setAttribute("interface", StringEscapeUtils.escapeXml11(ifaceId)); - profTemplate.setAttribute("dsName", StringEscapeUtils.escapeXml11(dsName)); - profTemplate.setAttribute("tdsCsv", getListOfTDS(ifaceId)); - - final String profId = this.serviceLocator.getService(ISRegistryService.class).registerProfile(profTemplate.toString()); - - env.setAttribute("repoWfId", profId); - - log.info("A new repo wf has been registered, id: " + profId); - - return Arc.DEFAULT_ARC; - } - - private String getListOfTDS(final String ifaceId) throws ISLookUpException { - String lastAPIPart = StringUtils.substringAfterLast(ifaceId, "::"); - String query = "for $x in collection('/db/DRIVER/TransformationRuleDSResources/TransformationRuleDSResourceType') return $x[starts-with(.//TITLE/text() , '"+lastAPIPart+"')]//RESOURCE_IDENTIFIER/@value/string()"; - return Joiner.on(',').join(isLookUpClient.search(query)); - } - - public WorkflowProcess getProcess() { - return this.process; - } - - @Override - public void setProcess(final WorkflowProcess process) { - this.process = process; - } - - - -} diff --git a/dnet-ariadneplus/src/main/java/eu/dnetlib/ariadneplus/workflows/nodes/ReportReaderJobNode.java b/dnet-ariadneplus/src/main/java/eu/dnetlib/ariadneplus/workflows/nodes/ReportReaderJobNode.java deleted file mode 100644 index b31aedc..0000000 --- a/dnet-ariadneplus/src/main/java/eu/dnetlib/ariadneplus/workflows/nodes/ReportReaderJobNode.java +++ /dev/null @@ -1,50 +0,0 @@ -package eu.dnetlib.ariadneplus.workflows.nodes; - -import java.util.Map; - -import com.google.gson.Gson; -import eu.dnetlib.msro.workflows.graph.Arc; -import eu.dnetlib.msro.workflows.nodes.AsyncJobNode; -import eu.dnetlib.msro.workflows.nodes.SimpleJobNode; -import eu.dnetlib.msro.workflows.procs.Env; -import eu.dnetlib.msro.workflows.util.WorkflowsConstants; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; - -/** - * Created by Alessia Bardi on 05/04/17. - * - * @author Alessia Bardi - */ -public class ReportReaderJobNode extends AsyncJobNode { - - private static final Log log = LogFactory.getLog(ReportReaderJobNode.class); - private String reportEnvParam = "validationReport"; - private String outputParam = reportEnvParam; - - @Override - protected String execute(final Env env) throws Exception { - Map report = env.getAttribute(reportEnvParam, Map.class); - Gson gson = new Gson(); - String theReport = gson.toJson(report); - env.setAttribute( WorkflowsConstants.MAIN_LOG_PREFIX+outputParam, theReport); - log.debug("Found the report: "+theReport); - return Arc.DEFAULT_ARC; - } - - public String getReportEnvParam() { - return reportEnvParam; - } - - public void setReportEnvParam(final String reportEnvParam) { - this.reportEnvParam = reportEnvParam; - } - - public String getOutputParam() { - return outputParam; - } - - public void setOutputParam(final String outputParam) { - this.outputParam = outputParam; - } -} diff --git a/dnet-ariadneplus/src/main/java/eu/dnetlib/ariadneplus/workflows/nodes/VirtuosoAriadnePlusByTypeIterator.java b/dnet-ariadneplus/src/main/java/eu/dnetlib/ariadneplus/workflows/nodes/VirtuosoAriadnePlusByTypeIterator.java deleted file mode 100644 index fbf090f..0000000 --- a/dnet-ariadneplus/src/main/java/eu/dnetlib/ariadneplus/workflows/nodes/VirtuosoAriadnePlusByTypeIterator.java +++ /dev/null @@ -1,91 +0,0 @@ -package eu.dnetlib.ariadneplus.workflows.nodes; - -import java.net.URI; -import java.net.URISyntaxException; -import java.util.Map; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.http.client.utils.URIBuilder; -import org.springframework.web.client.RestTemplate; - -/** - * Created by Alessia Bardi on 01/03/2018. - * - * @author Alessia Bardi - */ -public class VirtuosoAriadnePlusByTypeIterator extends VirtuosoAriadnePlusIterator { - - private static final Log log = LogFactory.getLog(VirtuosoAriadnePlusByTypeIterator.class); - private String typeName; - private String typeNamespace; - - @Override - protected URI getURIForSubjectList(final int offset) throws URISyntaxException { - URIBuilder builder = new URIBuilder(getVirtuosoReaderAPIUrl() + "/apiSubjectsWithType"); - builder.addParameter("api", getDatasourceInterface()); - builder.addParameter("typeNamespace", typeNamespace); - builder.addParameter("typeName", typeName); - builder.addParameter("limit", Integer.toString(LIMIT)); - builder.addParameter("offset", Integer.toString(offset)); - return builder.build(); - } - - @Override - protected URI getURIForRDFRequest(final String subjectURL) throws URISyntaxException { - URIBuilder builder = new URIBuilder(getVirtuosoReaderAPIUrl() + "/subject").addParameter("subjectURL", subjectURL).addParameter("typeName", typeName).addParameter("timeout", ANY_TIME_QUERY_MS); - return builder.build(); - } - - public String getTypeName() { - return typeName; - } - - public void setTypeName(final String typeName) { - this.typeName = typeName; - } - - public String getTypeNamespace() { - return typeNamespace; - } - - public void setTypeNamespace(final String typeNamespace) { - this.typeNamespace = typeNamespace; - } - - public VirtuosoAriadnePlusByTypeIterator typeName(final String typeName) { - this.typeName = typeName; - return this; - } - - public VirtuosoAriadnePlusByTypeIterator typeNamespace(final String typeNamespace) { - this.typeNamespace = typeNamespace; - return this; - } - - @Override - public VirtuosoAriadnePlusByTypeIterator restTemplate(final RestTemplate restTemplate) { - setRestTemplate(restTemplate); - return this; - } - @Override - public VirtuosoAriadnePlusByTypeIterator datasourceInterface(final String datasourceInterface) { - setDatasourceInterface(datasourceInterface); - return this; - } - @Override - public VirtuosoAriadnePlusByTypeIterator virtuosoReaderAPIUrl(final String virtuosoReaderAPIUrl) { - setVirtuosoReaderAPIUrl(virtuosoReaderAPIUrl); - return this; - } - @Override - public VirtuosoAriadnePlusByTypeIterator datasourceName(final String datasourceName) { - setDatasourceName(datasourceName); - return this; - } - @Override - public VirtuosoAriadnePlusByTypeIterator errors(final Map errors) { - setErrors(errors); - return this; - } -} diff --git a/dnet-ariadneplus/src/main/java/eu/dnetlib/ariadneplus/workflows/nodes/VirtuosoAriadnePlusException.java b/dnet-ariadneplus/src/main/java/eu/dnetlib/ariadneplus/workflows/nodes/VirtuosoAriadnePlusException.java deleted file mode 100644 index daaef3c..0000000 --- a/dnet-ariadneplus/src/main/java/eu/dnetlib/ariadneplus/workflows/nodes/VirtuosoAriadnePlusException.java +++ /dev/null @@ -1,24 +0,0 @@ -package eu.dnetlib.ariadneplus.workflows.nodes; - -/** - * Created by Alessia Bardi on 17/02/2018. - * - * @author Alessia Bardi - */ -public class VirtuosoAriadnePlusException extends Exception{ - - public VirtuosoAriadnePlusException() { - } - - public VirtuosoAriadnePlusException(final String message) { - super(message); - } - - public VirtuosoAriadnePlusException(final String message, final Throwable cause) { - super(message, cause); - } - - public VirtuosoAriadnePlusException(final Throwable cause) { - super(cause); - } -} diff --git a/dnet-ariadneplus/src/main/java/eu/dnetlib/ariadneplus/workflows/nodes/VirtuosoAriadnePlusIterator.java b/dnet-ariadneplus/src/main/java/eu/dnetlib/ariadneplus/workflows/nodes/VirtuosoAriadnePlusIterator.java deleted file mode 100644 index 7ccef34..0000000 --- a/dnet-ariadneplus/src/main/java/eu/dnetlib/ariadneplus/workflows/nodes/VirtuosoAriadnePlusIterator.java +++ /dev/null @@ -1,352 +0,0 @@ -package eu.dnetlib.ariadneplus.workflows.nodes; - -import java.net.URI; -import java.net.URISyntaxException; -import java.util.List; -import java.util.Map; -import java.util.NoSuchElementException; -import java.util.concurrent.BlockingQueue; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.TimeUnit; - -import com.google.common.collect.Lists; -import com.google.common.collect.Maps; -import com.google.common.collect.Queues; -import eu.dnetlib.data.collector.ThreadSafeIterator; -import eu.dnetlib.rmi.data.CollectorServiceRuntimeException; -import org.apache.commons.lang3.StringEscapeUtils; -import org.apache.commons.lang3.StringUtils; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.http.client.utils.URIBuilder; -import org.springframework.http.*; -import org.springframework.web.client.ResourceAccessException; -import org.springframework.web.client.RestClientException; -import org.springframework.web.client.RestTemplate; - -/** - * Created by Alessia Bardi on 31/01/2018. - * - * @author Alessia Bardi - */ -public class VirtuosoAriadnePlusIterator extends ThreadSafeIterator { - - private static final Log log = LogFactory.getLog(VirtuosoAriadnePlusIterator.class); - protected static final String ANY_TIME_QUERY_MS = "1800000"; //1800000 == 3 mins - protected static final int QUEUE_TIMEOUT_SECONDS = 600; - public final static String TERMINATOR = "ARNOLD"; - public final static String ERROR_TERMINATOR = "SCHWARZ"; - protected final static int SLEEP_MS = 5000; - protected final static int MAX_RETRIES = 3; - protected final static int LIMIT = 100; - - private String datasourceName; - private String datasourceInterface; - private String virtuosoReaderAPIUrl; - private boolean started = false; - private Map errors = Maps.newHashMap(); - private List listForClass = Lists.newArrayList(); - private BlockingQueue elements = Queues.newArrayBlockingQueue(10); - - private String currentElement = null; - private ExecutorService executor = Executors.newSingleThreadExecutor(); - - private RestTemplate restTemplate; - - - private synchronized void verifyStarted(){ - if (!this.started) { - this.started = true; - fillQueue(); - getNextElement(MAX_RETRIES); - } - } - - protected void fillQueue(){ - log.info("Virtuoso reader at : " + getVirtuosoReaderAPIUrl()); - getExecutor().submit(() -> { - try { - int offset = 0; - boolean again; - do { - List subjectList = getSubjectList(offset); - for (String subject : subjectList) { - String xmlFile = tryGetRDF(subject, MAX_RETRIES); - if (StringUtils.isBlank(xmlFile)) { - log.warn("Skipping blank RDF for " + subject); - } else { - getElements().offer(xmlFile, QUEUE_TIMEOUT_SECONDS, TimeUnit.SECONDS); - } - } - again = subjectList.size() == LIMIT; - offset += LIMIT; - } while(again); - log.debug("End of subject list, adding terminator to the queue"); - getElements().offer(TERMINATOR, QUEUE_TIMEOUT_SECONDS, TimeUnit.SECONDS); - } catch (Exception e) { - log.error(e.getMessage()); - try { - getElements().offer(ERROR_TERMINATOR, QUEUE_TIMEOUT_SECONDS, TimeUnit.SECONDS); - } catch (InterruptedException e1) { - log.error(e1.getMessage()); - } - } - - }); - getExecutor().shutdown(); - } - - - protected String tryGetRDF(final String subjectURL, int attempt) throws URISyntaxException, InterruptedException, VirtuosoAriadnePlusException { - //this is something George said: if it has no http URI, then it is not to be considered relevant by itself - if(!subjectURL.startsWith("http")){ - log.debug("Skipping as non-http url: "+subjectURL); - errors.merge("Non-http URLs", 1, Integer::sum); - return null; - } - log.debug("Querying Api, remaining attempts: "+attempt); - if (attempt <= 0) { - errors.merge("Failed tryGetRDF", 1, Integer::sum); - return null; - } - ResponseEntity response = null; - try { - response = getRDF(subjectURL); - }catch(ResourceAccessException e){ - //request timed out --> sleep and try again - log.warn("Request timeout for "+subjectURL+": I'll sleep and then try again"); - Thread.sleep(SLEEP_MS); - return tryGetRDF(subjectURL, --attempt); - } - HttpStatus responseStatus = response.getStatusCode(); - if (responseStatus.is2xxSuccessful()) { - String rdfFile = response.getBody(); - if(StringUtils.isBlank(rdfFile)){ - log.warn("Got blank RDF for "+subjectURL+" , let's try again..."); - Thread.sleep(SLEEP_MS); - return tryGetRDF(subjectURL, --attempt); - } - else { - final String xmlFile = completeXML(rdfFile, subjectURL); - return xmlFile; - } - } else { - if (responseStatus.is5xxServerError()) { - //sleep for a while and re-try - log.warn("HTTP ERROR: "+responseStatus.value() + ": " + responseStatus.getReasonPhrase()+": I'll sleep and then try again"); - Thread.sleep(SLEEP_MS); - return tryGetRDF(subjectURL, --attempt); - } else { - log.error("ERROR: Can't get the RDF for " + subjectURL + " " + responseStatus.value() + ": " + responseStatus.getReasonPhrase()); - errors.merge(responseStatus.value() + ": " + responseStatus.getReasonPhrase(), 1, Integer::sum); - } - } - return null; - } - - protected URI getURIForSubjectList(final int offset) throws URISyntaxException { - URIBuilder builder = new URIBuilder(getVirtuosoReaderAPIUrl() + "/apiSubjects"); - builder.addParameter("api", getDatasourceInterface()); - builder.addParameter("limit", Integer.toString(LIMIT)); - builder.addParameter("offset", Integer.toString(offset)); - return builder.build(); - } - - protected List getSubjectList(final int offset) throws URISyntaxException, VirtuosoAriadnePlusException { - URI uri = getURIForSubjectList(offset); - log.debug("fillQueue -- Calling for subject list: " + uri.toString()); - List subjectList; - try { - subjectList = getRestTemplate().getForObject(uri, getListForClass().getClass()); - }catch(RestClientException rce){ - throw new VirtuosoAriadnePlusException(rce); - } - return subjectList; - } - - protected URI getURIForRDFRequest(final String subjectURL) throws URISyntaxException { - URIBuilder builder = new URIBuilder(getVirtuosoReaderAPIUrl() + "/subject").addParameter("subjectURL", subjectURL).addParameter("timeout", ANY_TIME_QUERY_MS); - return builder.build(); - } - - protected ResponseEntity getRDF(final String subjectURL) throws URISyntaxException { - HttpHeaders headers = new HttpHeaders(); - headers.setAccept(Lists.newArrayList(MediaType.APPLICATION_XML)); - URI uri = getURIForRDFRequest(subjectURL); - log.debug("fillQueue -- Calling for subject RDF: " + uri.toString()); - HttpEntity entity = new HttpEntity<>("parameters", headers); - return restTemplate.exchange(uri, HttpMethod.GET, entity, String.class); - } - - - public String completeXML(final String rdfFile, final String url) { - String xmlEscapedURL = StringEscapeUtils.escapeXml11(url); - String rdfFileNoXmlDecl = rdfFile.replaceAll("\\<\\?xml(.+?)\\?\\>", "").trim(); - return "
" - + xmlEscapedURL + "" + datasourceInterface + "" + datasourceName - + "
" + rdfFileNoXmlDecl + ""; - } - - @Override - public boolean doHasNext() { - try { - verifyStarted(); - } catch (Exception e) { - getExecutor().shutdownNow(); - throw new CollectorServiceRuntimeException(e); - } - switch(currentElement){ - case TERMINATOR: - if(!executor.isTerminated()) executor.shutdownNow(); - return false; - case ERROR_TERMINATOR: - executor.shutdownNow(); - throw new CollectorServiceRuntimeException("Error getting elements from virtuoso"); - default: - return true; - } - } - - @Override - public String doNext() { - if(!hasNext()) { - log.error("Next called but hasNext is false", new NoSuchElementException()); - throw new NoSuchElementException(); - } - switch(currentElement){ - case TERMINATOR: - case ERROR_TERMINATOR: - executor.shutdownNow(); - throw new NoSuchElementException(); - default: - String res = currentElement; - getNextElement(MAX_RETRIES); - return res; - } - } - - private void getNextElement(int attempt){ - log.debug("polling from queue, remaining attempts: "+attempt); - if(attempt <= 0) currentElement = ERROR_TERMINATOR; - else{ - try { - currentElement = elements.take(); - } catch (InterruptedException e) { - //current thread interrupted. Let's end. - currentElement = ERROR_TERMINATOR; - executor.shutdownNow(); - } - } - } - - @Override - public void remove() { - throw new UnsupportedOperationException(); - } - - public VirtuosoAriadnePlusIterator datasourceInterface(final String datasourceInterface) { - this.datasourceInterface = datasourceInterface; - return this; - } - - public VirtuosoAriadnePlusIterator virtuosoReaderAPIUrl(final String virtuosoReaderAPIUrl) { - this.virtuosoReaderAPIUrl = virtuosoReaderAPIUrl; - return this; - } - - public VirtuosoAriadnePlusIterator datasourceName(final String datasourceName) { - this.datasourceName = datasourceName; - return this; - } - - public VirtuosoAriadnePlusIterator errors(final Map errors) { - this.errors = errors; - return this; - } - - public String getDatasourceInterface() { - return datasourceInterface; - } - - public String getVirtuosoReaderAPIUrl() { - return virtuosoReaderAPIUrl; - } - - public Map getErrors() { - return errors; - } - - public BlockingQueue getElements() { - return elements; - } - - public RestTemplate getRestTemplate() { - return restTemplate; - } - - public VirtuosoAriadnePlusIterator restTemplate(final RestTemplate restTemplate) { - this.restTemplate = restTemplate; - return this; - } - - public String getDatasourceName() { - return datasourceName; - } - - public boolean isStarted() { - return started; - } - - public List getListForClass() { - return listForClass; - } - - public String getCurrentElement() { - return currentElement; - } - - public ExecutorService getExecutor() { - return executor; - } - - public void setDatasourceName(final String datasourceName) { - this.datasourceName = datasourceName; - } - - public void setDatasourceInterface(final String datasourceInterface) { - this.datasourceInterface = datasourceInterface; - } - - public void setVirtuosoReaderAPIUrl(final String virtuosoReaderAPIUrl) { - this.virtuosoReaderAPIUrl = virtuosoReaderAPIUrl; - } - - public void setStarted(final boolean started) { - this.started = started; - } - - public void setErrors(final Map errors) { - this.errors = errors; - } - - public void setListForClass(final List listForClass) { - this.listForClass = listForClass; - } - - public void setElements(final BlockingQueue elements) { - this.elements = elements; - } - - public void setCurrentElement(final String currentElement) { - this.currentElement = currentElement; - } - - public void setExecutor(final ExecutorService executor) { - this.executor = executor; - } - - public void setRestTemplate(final RestTemplate restTemplate) { - this.restTemplate = restTemplate; - } -} diff --git a/dnet-ariadneplus/src/main/java/eu/dnetlib/data/collector/plugins/ariadneplus/ehri/EHRICollectorPlugin.java b/dnet-ariadneplus/src/main/java/eu/dnetlib/data/collector/plugins/ariadneplus/ehri/EHRICollectorPlugin.java deleted file mode 100644 index 4533a6d..0000000 --- a/dnet-ariadneplus/src/main/java/eu/dnetlib/data/collector/plugins/ariadneplus/ehri/EHRICollectorPlugin.java +++ /dev/null @@ -1,41 +0,0 @@ -package eu.dnetlib.data.collector.plugins.ariadneplus.ehri; - -import eu.dnetlib.rmi.data.CollectorServiceException; -import eu.dnetlib.rmi.data.InterfaceDescriptor; -import eu.dnetlib.rmi.data.plugin.AbstractCollectorPlugin; - -/** - * This collector plugins collects metadata records from the EHRI portal export URL in the form https://portal.ehri-project.eu/units//export. - * IDs of units are retrieved from the EHRI GraphQL API, see docs at https://portal.ehri-project.eu/api/graphql - * Created by Alessia Bardi on 19/04/2017. - * - * @author Alessia Bardi - */ -public class EHRICollectorPlugin extends AbstractCollectorPlugin { - - private EHRIIteratorFactory ehriIteratorFactory; - - @Override - public Iterable collect(final InterfaceDescriptor interfaceDescriptor, final String fromDate, final String untilDate) - throws CollectorServiceException { - - // https://portal.ehri-project.eu/units - final String baseUrl = interfaceDescriptor.getBaseUrl(); - // export - final String suffixToUrl = interfaceDescriptor.getParams().get("suffixToBaseUrl"); - - //info to get the identifiers from the graphQL API: - final String graphQLURL = interfaceDescriptor.getParams().get("graphQLURL"); - final String graphQLQuery = interfaceDescriptor.getParams().get("graphQLQuery"); - - return () -> ehriIteratorFactory.newIterator(graphQLURL, graphQLQuery, baseUrl, suffixToUrl); - } - - public EHRIIteratorFactory getEhriIteratorFactory() { - return ehriIteratorFactory; - } - - public void setEhriIteratorFactory(final EHRIIteratorFactory ehriIteratorFactory) { - this.ehriIteratorFactory = ehriIteratorFactory; - } -} diff --git a/dnet-ariadneplus/src/main/java/eu/dnetlib/data/collector/plugins/ariadneplus/ehri/EHRIGraphQLClient.java b/dnet-ariadneplus/src/main/java/eu/dnetlib/data/collector/plugins/ariadneplus/ehri/EHRIGraphQLClient.java deleted file mode 100644 index 9d70093..0000000 --- a/dnet-ariadneplus/src/main/java/eu/dnetlib/data/collector/plugins/ariadneplus/ehri/EHRIGraphQLClient.java +++ /dev/null @@ -1,77 +0,0 @@ -package eu.dnetlib.data.collector.plugins.ariadneplus.ehri; - -import java.io.IOException; -import java.io.InputStreamReader; -import java.net.URI; -import java.net.URISyntaxException; -import java.util.Iterator; - -import com.google.common.collect.Iterables; -import com.google.common.collect.Lists; -import com.google.gson.JsonArray; -import com.google.gson.JsonObject; -import com.google.gson.JsonParser; -import eu.dnetlib.rmi.data.CollectorServiceException; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.http.HttpEntity; -import org.apache.http.client.methods.CloseableHttpResponse; -import org.apache.http.client.methods.HttpPost; -import org.apache.http.entity.StringEntity; -import org.apache.http.impl.client.CloseableHttpClient; -import org.apache.http.impl.client.HttpClients; -import org.apache.http.util.EntityUtils; - -/** - * IDs of units are retrieved from the EHRI GraphQL API, see docs at https://portal.ehri-project.eu/api/graphql - * - * Created by Alessia Bardi on 19/04/2017. - * - * @author Alessia Bardi - */ -public class EHRIGraphQLClient { - - private static final Log log = LogFactory.getLog(EHRIGraphQLClient.class); - - public Iterator collect(final String baseURL, final String graphQLQuery) throws URISyntaxException, IOException, CollectorServiceException { - /* - The curl equivalent: - curl --header X-Stream:true \ - --header Content-type:application/json \ - https://portal.ehri-project.eu/api/graphql \ - --data-binary '{"query":"{topLevelDocumentaryUnits{items{id}}}"}' - */ - CloseableHttpClient httpClient = HttpClients.createDefault(); - URI baseURI = new URI(baseURL); - HttpPost httpPost = new HttpPost(); - httpPost.setURI(baseURI); - httpPost.setHeader("X-Stream", "true"); - httpPost.setHeader("Content-type", "application/json" ); - log.info(graphQLQuery); - StringEntity postQuery = new StringEntity(graphQLQuery); - httpPost.setEntity(postQuery); - HttpEntity entity = null; - try(CloseableHttpResponse response = httpClient.execute(httpPost)) { - switch(response.getStatusLine().getStatusCode()){ - case 200: - entity = response.getEntity(); - InputStreamReader reader = new InputStreamReader(entity.getContent()); - return getIdentifiers(reader); - default: - log.error(httpPost); - log.error(response.getStatusLine()); - throw new CollectorServiceException(response.getStatusLine().toString()); - } - } finally { - if(entity != null) EntityUtils.consume(entity); - } - } - - protected Iterator getIdentifiers(final InputStreamReader input){ - JsonObject jsonObject = new JsonParser().parse(input).getAsJsonObject(); - JsonArray items = jsonObject.getAsJsonObject("data").getAsJsonObject("topLevelDocumentaryUnits").getAsJsonArray("items"); - log.debug(items); - return Lists.newArrayList(Iterables.transform(items, jelem -> jelem.getAsJsonObject().get("id").getAsString())).iterator(); - } - -} diff --git a/dnet-ariadneplus/src/main/java/eu/dnetlib/data/collector/plugins/ariadneplus/ehri/EHRIIterator.java b/dnet-ariadneplus/src/main/java/eu/dnetlib/data/collector/plugins/ariadneplus/ehri/EHRIIterator.java deleted file mode 100644 index c09b56f..0000000 --- a/dnet-ariadneplus/src/main/java/eu/dnetlib/data/collector/plugins/ariadneplus/ehri/EHRIIterator.java +++ /dev/null @@ -1,107 +0,0 @@ -package eu.dnetlib.data.collector.plugins.ariadneplus.ehri; - -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.net.URL; -import java.util.Iterator; - -import com.ximpleware.*; -import eu.dnetlib.data.collector.ThreadSafeIterator; -import eu.dnetlib.rmi.data.CollectorServiceRuntimeException; -import org.apache.commons.io.IOUtils; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; - -/** - * Created by Alessia Bardi on 03/05/2017. - * - * @author Alessia Bardi - */ -public class EHRIIterator extends ThreadSafeIterator { - - private static final Log log = LogFactory.getLog(EHRIIterator.class); - private static int MAX_FAILED = 100; - private Iterator identifiers; - private String baseUrl; - private String suffix; - private int failed = 0; - - - public EHRIIterator(final Iterator idIterator, final String baseUrl, final String suffix){ - this.identifiers = idIterator; - this.baseUrl = baseUrl; - this.suffix = suffix; - } - - @Override - public boolean doHasNext() { - return identifiers.hasNext(); - } - - @Override - public String doNext() { - String target = baseUrl+"/"+identifiers.next()+"/"+suffix; - log.debug("Getting "+target); - try { - URL url = new URL(target); - String record = IOUtils.toString(url, "UTF-8"); - return removeDefaultEADNamespace(record); - } catch (IOException e) { - log.error("Unable to get "+target, e); - failed++; - if(failed > MAX_FAILED){ - throw new CollectorServiceRuntimeException("Could not download more than "+MAX_FAILED+"documents from EHRI. Stopping."); - } - if(this.hasNext()){ - return this.next(); - } - else return ""; - } - } - - protected String removeDefaultEADNamespace(final String xml) { - try { - VTDGen vg = new VTDGen(); - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - vg.setDoc(xml.getBytes()); - vg.parse(false); // turn off namespace awareness so that - VTDNav vn = vg.getNav(); - AutoPilot ap = new AutoPilot(vn); - XMLModifier xm = new XMLModifier(vn); - ap.selectXPath("/ead/@xmlns"); - int i = 0; - while ((i = ap.evalXPath()) != -1) { - xm.remove(); - } - xm.output(baos); - return baos.toString(); - }catch(Exception e){ - log.error("Cannot remove default namespace from ead element: "+xml); - throw new CollectorServiceRuntimeException("Cannot remove default namespace from ead element", e); - } - } - - public Iterator getIdentifiers() { - return identifiers; - } - - public void setIdentifiers(final Iterator identifiers) { - this.identifiers = identifiers; - } - - public String getBaseUrl() { - return baseUrl; - } - - public void setBaseUrl(final String baseUrl) { - this.baseUrl = baseUrl; - } - - public String getSuffix() { - return suffix; - } - - public void setSuffix(final String suffix) { - this.suffix = suffix; - } -} diff --git a/dnet-ariadneplus/src/main/java/eu/dnetlib/data/collector/plugins/ariadneplus/ehri/EHRIIteratorFactory.java b/dnet-ariadneplus/src/main/java/eu/dnetlib/data/collector/plugins/ariadneplus/ehri/EHRIIteratorFactory.java deleted file mode 100644 index 0d65ecc..0000000 --- a/dnet-ariadneplus/src/main/java/eu/dnetlib/data/collector/plugins/ariadneplus/ehri/EHRIIteratorFactory.java +++ /dev/null @@ -1,38 +0,0 @@ -package eu.dnetlib.data.collector.plugins.ariadneplus.ehri; - -import java.io.IOException; -import java.net.URISyntaxException; -import java.util.Iterator; - -import eu.dnetlib.rmi.data.CollectorServiceException; -import org.springframework.beans.factory.annotation.Autowired; - -/** - * Created by Alessia Bardi on 03/05/2017. - * - * @author Alessia Bardi - */ -public class EHRIIteratorFactory { - - @Autowired - private EHRIGraphQLClient ehriGraphQLClient; - - public Iterator newIterator(final String baseURLIdentifiers, final String queryIdentifiers, - final String baseUrl, - final String suffix) { - try { - Iterator ids = ehriGraphQLClient.collect(baseURLIdentifiers, queryIdentifiers); - return new EHRIIterator(ids, baseUrl, suffix); - } catch (CollectorServiceException | IOException | URISyntaxException e) { - throw new RuntimeException(e); - } - } - - public EHRIGraphQLClient getEhriGraphQLClient() { - return ehriGraphQLClient; - } - - public void setEhriGraphQLClient(final EHRIGraphQLClient ehriGraphQLClient) { - this.ehriGraphQLClient = ehriGraphQLClient; - } -} diff --git a/dnet-ariadneplus/src/main/java/eu/dnetlib/data/collector/plugins/ariadneplus/isidore/IsidoreCollectorPlugin.java b/dnet-ariadneplus/src/main/java/eu/dnetlib/data/collector/plugins/ariadneplus/isidore/IsidoreCollectorPlugin.java deleted file mode 100644 index 642dccc..0000000 --- a/dnet-ariadneplus/src/main/java/eu/dnetlib/data/collector/plugins/ariadneplus/isidore/IsidoreCollectorPlugin.java +++ /dev/null @@ -1,53 +0,0 @@ -package eu.dnetlib.data.collector.plugins.ariadneplus.isidore; - -import eu.dnetlib.miscutils.functional.xml.SaxonHelper; -import eu.dnetlib.rmi.data.CollectorServiceException; -import eu.dnetlib.rmi.data.InterfaceDescriptor; -import eu.dnetlib.rmi.data.plugin.AbstractCollectorPlugin; -import org.apache.commons.lang3.StringUtils; -import org.springframework.beans.factory.annotation.Autowired; - -/** - * @author alessia bardi - * - */ -public class IsidoreCollectorPlugin extends AbstractCollectorPlugin { - - @Autowired - private SaxonHelper saxonHelper; - - @Override - public Iterable collect(InterfaceDescriptor ifDescriptor, String arg1, String arg2) - throws CollectorServiceException { - final String baseUrl = ifDescriptor.getBaseUrl(); - final String queryParams = ifDescriptor.getParams().get("queryParams"); - final String pageParam = ifDescriptor.getParams().get("pageParam"); - final String startFromPage = ifDescriptor.getParams().get("startFromPage"); - final String nextPagePath = ifDescriptor.getParams().get("nextPagePath"); - final String pageSizeParam = ifDescriptor.getParams().get("pageSizeParam"); - final String pageSize = ifDescriptor.getParams().get("pageSize"); - final String resultTotalXpath = ifDescriptor.getParams().get("resultTotalXpath"); - final String entityXpath = ifDescriptor.getParams().get("entityXpath"); - - if (StringUtils.isBlank(baseUrl)) {throw new CollectorServiceException("Param 'baseUrl' is null or empty");} - if (StringUtils.isBlank(pageParam)) {throw new CollectorServiceException("Param 'pageParam' is null or empty");} - if (StringUtils.isBlank(startFromPage)) {throw new CollectorServiceException("Param 'startFromPage' is null or empty");} - if (StringUtils.isBlank(nextPagePath)) {throw new CollectorServiceException("Param 'nextPagePath' is null or empty");} - if (StringUtils.isBlank(pageSizeParam)) {throw new CollectorServiceException("Param 'pageSizeParam' is null or empty");} - if (StringUtils.isBlank(pageSize)) {throw new CollectorServiceException("Param 'pageSize' is null or empty");} - if (StringUtils.isBlank(resultTotalXpath)) {throw new CollectorServiceException("Param 'resultTotalXpath' is null or empty");} - if (StringUtils.isBlank(entityXpath)) {throw new CollectorServiceException("Param 'entityXpath' is null or empty");} - - return () -> new IsidoreIterator( - saxonHelper, - baseUrl, - queryParams, - pageParam, - Integer.parseInt(startFromPage), - nextPagePath, - pageSizeParam, - Integer.parseInt(pageSize), - resultTotalXpath, - entityXpath); - } -} diff --git a/dnet-ariadneplus/src/main/java/eu/dnetlib/data/collector/plugins/ariadneplus/isidore/IsidoreIterator.java b/dnet-ariadneplus/src/main/java/eu/dnetlib/data/collector/plugins/ariadneplus/isidore/IsidoreIterator.java deleted file mode 100644 index 0f92726..0000000 --- a/dnet-ariadneplus/src/main/java/eu/dnetlib/data/collector/plugins/ariadneplus/isidore/IsidoreIterator.java +++ /dev/null @@ -1,169 +0,0 @@ -package eu.dnetlib.data.collector.plugins.ariadneplus.isidore; /** - * log.debug(...) equal to log.trace(...) in the application-logs - *

- * known bug: at resumptionType 'discover' if the (resultTotal % resultSizeValue) == 0 the collecting fails -> change the resultSizeValue - */ - -import java.io.InputStream; -import java.net.URL; -import java.util.Iterator; -import java.util.Queue; -import java.util.concurrent.PriorityBlockingQueue; - -import com.google.common.collect.Maps; -import eu.dnetlib.miscutils.functional.xml.SaxonHelper; -import eu.dnetlib.rmi.data.CollectorServiceException; -import net.sf.saxon.s9api.*; -import org.apache.commons.io.IOUtils; -import org.apache.commons.lang3.StringUtils; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; - -/** - * @author Jochen Schirrwagen, Aenne Loehden, Andreas Czerniak, Alessia Bardi - * @date 2018-09-03 - * - */ -public class IsidoreIterator implements Iterator { - - private static final Log log = LogFactory.getLog(IsidoreIterator.class); // NOPMD by marko on 11/24/08 5:02 PM - - private SaxonHelper saxonHelper; - private Queue recordQueue = new PriorityBlockingQueue(); - private InputStream resultStream; - private String query; - private String querySize; - private String queryPage; - private int resultTotal = -1; - - private String baseUrl; - private String queryParams; - private String pageParam; - private int page; - private String pageSizeParam; - private int pageSize; - - private XPathSelector xprEntity; - private XPathSelector xprResultTotalPath; - private XPathSelector xprNextPagePath; - - public IsidoreIterator( - final SaxonHelper saxonHelper, - final String baseUrl, - final String queryParams, - final String pageParam, - final int startFromPage, - final String nextPagePath, - final String pageSizeParam, - final int pageSize, - final String resultTotalXpath, - final String entityXpath - ) { - this.saxonHelper = saxonHelper; - this.baseUrl = baseUrl; - this.queryParams = queryParams; - this.pageParam = pageParam; - this.page = startFromPage; - this.queryPage = this.pageParam+"="+this.page; - this.pageSizeParam = pageSizeParam; - this.pageSize = pageSize; - this.querySize = this.pageSizeParam+"="+this.pageSize; - try { - initXpathSelector(resultTotalXpath, nextPagePath, entityXpath); - } catch (Exception e) { - throw new IllegalStateException("xml transformation init failed: " + e.getMessage()); - } - initQuery(); - } - - private void initXpathSelector(final String resultTotalXpath,final String nextPagePath, final String entityXpath) - throws SaxonApiException { - xprResultTotalPath = this.saxonHelper.help().prepareXPathSelector(resultTotalXpath, Maps.newHashMap()); - xprNextPagePath = this.saxonHelper.help().prepareXPathSelector(nextPagePath, Maps.newHashMap()); - xprEntity = this.saxonHelper.help().prepareXPathSelector(entityXpath, Maps.newHashMap()); - } - - private void initQuery() { - query = baseUrl + "?" + queryParams +"&"+ querySize +"&"+ queryPage; - } - - private void disconnect() { - // TODO close inputstream - } - - /* (non-Javadoc) - * @see java.util.Iterator#hasNext() - */ - @Override - public boolean hasNext() { - if (recordQueue.isEmpty() && query.isEmpty()) { - disconnect(); - return false; - } else { - return true; - } - } - - /* (non-Javadoc) - * @see java.util.Iterator#next() - */ - @Override - public String next() { - synchronized (recordQueue) { - while (recordQueue.isEmpty() && !query.isEmpty()) { - try { - log.info("get Query: " + query); - query = downloadPage(query); - log.debug("next queryURL from downloadPage(): " + query); - } catch (CollectorServiceException e) { - log.debug("CollectorPlugin.next()-Exception: " + e); - throw new RuntimeException(e); - } - } - return recordQueue.poll(); - } - } - - /* - * download page and return nextQuery - */ - private String downloadPage(String query) throws CollectorServiceException { - String nextQuery = ""; - XdmValue nodeList; - - try { - URL qUrl = new URL(query); - resultStream = qUrl.openStream(); - String resultPage = IOUtils.toString(resultStream); - nodeList = this.saxonHelper.help().evaluate(resultPage, xprEntity); - log.debug("nodeList.size: " + nodeList.size()); - for(XdmItem entity : nodeList){ - recordQueue.add(this.saxonHelper.help().serialize((XdmNode) entity)); - } - - String nextPage = this.saxonHelper.help().evaluateSingleAsString(resultPage, xprNextPagePath); - if(StringUtils.isBlank(nextPage)){ - log.info("No next page available, we reached the end"); - } - else{ - this.queryPage = pageParam+"="+nextPage; - return baseUrl + "?" + queryParams +"&"+ querySize +"&"+ queryPage; - } - - if (resultTotal == -1) { - String tot = this.saxonHelper.help().evaluateSingleAsString(resultPage, xprResultTotalPath); - resultTotal = Integer.parseInt(tot); - log.info("resultTotal was -1 is now: " + resultTotal); - } - log.info("resultTotal: " + resultTotal); - log.debug("nextQueryUrl: " + nextQuery); - return nextQuery; - - } catch (Exception e) { - log.error(e); - throw new IllegalStateException("collection failed: " + e.getMessage()); - } - } - - -} diff --git a/dnet-ariadneplus/src/main/resources/eu/dnetlib/ariadneplus/workflows/nodes/applicationContext-ariadneplus-msro-nodes.xml b/dnet-ariadneplus/src/main/resources/eu/dnetlib/ariadneplus/workflows/nodes/applicationContext-ariadneplus-msro-nodes.xml index 34caa3e..187bc05 100644 --- a/dnet-ariadneplus/src/main/resources/eu/dnetlib/ariadneplus/workflows/nodes/applicationContext-ariadneplus-msro-nodes.xml +++ b/dnet-ariadneplus/src/main/resources/eu/dnetlib/ariadneplus/workflows/nodes/applicationContext-ariadneplus-msro-nodes.xml @@ -5,15 +5,11 @@ xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans.xsd http://cxf.apache.org/transports/http/configuration http://cxf.apache.org/schemas/configuration/http-conf.xsd"> - - - - diff --git a/dnet-ariadneplus/src/test/java/eu/dnetlib/ariadneplus/workflows/nodes/VirtuosoAriadnePlusIteratorTest.java b/dnet-ariadneplus/src/test/java/eu/dnetlib/ariadneplus/workflows/nodes/VirtuosoAriadnePlusIteratorTest.java deleted file mode 100644 index 7f9e73b..0000000 --- a/dnet-ariadneplus/src/test/java/eu/dnetlib/ariadneplus/workflows/nodes/VirtuosoAriadnePlusIteratorTest.java +++ /dev/null @@ -1,69 +0,0 @@ -package eu.dnetlib.ariadneplus.workflows.nodes; - -import java.util.Iterator; -import java.util.Map; -import java.util.Map.Entry; - -import org.junit.Before; -import org.junit.Test; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; - -/** - * Created by Alessia Bardi on 09/02/2018. - * - * @author Alessia Bardi - */ -public class VirtuosoAriadnePlusIteratorTest { - - private VirtuosoAriadnePlusIterator it; - - @Before - public void setup(){ - it = new VirtuosoAriadnePlusMockIterator(); - } - - @Test - public void fillQueueTest() throws Exception { - it.fillQueue(); - String s = it.getElements().take(); - while(!s.equals(VirtuosoAriadnePlusIterator.TERMINATOR)){ - System.out.println(s); - s = it.getElements().take(); - } - Map errors = it.getErrors(); - for (Iterator> it1 = errors.entrySet().iterator(); it1.hasNext(); ) { - final Entry e = it1.next(); - System.out.println(e.getKey() +" - "+e.getValue()); - } - } - - @Test - public void testHasNext(){ - assertTrue(it.hasNext()); - } - - @Test - public void testIterate(){ - while(it.hasNext()) - System.out.println(it.next()); - } - - @Test - public void testCompleteXml(){ - String s = ""; - String completed = it.completeXML(s, "http://ariadneplus.d4science.org/handle/AriadnePlus/REG/Thing/Appellation/urp%40regione.umbria.it"); - System.out.println(completed); - } - - @Test - public void testRemoveXmlDeclaration(){ - String s = "", "").trim(); - assertEquals("", "").trim(); - assertEquals(" elements = Lists.newArrayList("rdf1", "rdf2", "rdf3", "rdf4", "rdf5", "rdf6"); - - @Override - protected void fillQueue() { - new Thread(() -> { - try { - for (String e : elements) { - - getElements().offer(e, QUEUE_TIMEOUT_SECONDS, TimeUnit.SECONDS); - } - } catch (InterruptedException ee) { - ee.printStackTrace(); - } finally { - try { - getElements().offer(TERMINATOR, QUEUE_TIMEOUT_SECONDS, TimeUnit.SECONDS); - } catch (InterruptedException ee) { - System.out.println(ee); - } - } - }).start(); - } -} diff --git a/dnet-ariadneplus/src/test/java/eu/dnetlib/ariadneplus/workflows/nodes/X3MTransformAriadnePlusJobNodeTest.java b/dnet-ariadneplus/src/test/java/eu/dnetlib/ariadneplus/workflows/nodes/X3MTransformAriadnePlusJobNodeTest.java new file mode 100644 index 0000000..9658cf4 --- /dev/null +++ b/dnet-ariadneplus/src/test/java/eu/dnetlib/ariadneplus/workflows/nodes/X3MTransformAriadnePlusJobNodeTest.java @@ -0,0 +1,134 @@ +package eu.dnetlib.ariadneplus.workflows.nodes; + +import org.junit.Test; + +import java.time.Duration; +import java.time.LocalDateTime; + +/** + * Created by Alessia Bardi on 13/04/2017. + * + * @author Alessia Bardi + */ +public class X3MTransformAriadnePlusJobNodeTest { + + private X3MTransformAriadnePlusJobNode transformJob = new X3MTransformAriadnePlusJobNode(); + private String header = "\n" + + " ariadne_mock::0000023f507999464aa2b78875b7e5d6\n" + + " 2420500\n" + + " 2017-04-10T18:44:46.85+02:00\n" + + " ariadne_mock\n" + + " Ariadne Mock\n" + + " 2017-04-12T16:31:45.766\n" + + " \n" + + " \n" + + " \n" + + " "; + private String footer = "\n" + + " \n" + + " \n" + + " sftp%3A%2F%2Fariadne2.isti.cnr.it%2F..%2F..%2Fdata%2Ftransform%2Facdm_correct\n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " "; + private String metadata=" \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " CAIRN\n" + + " \n" + + " \n" + + " \n" + + " \n" + + " HUT CIRCLE\n" + + " \n" + + " \n" + + " ADS Terms and Conditions\n" + + " \n" + + " \n" + + " SHEEPFOLD\n" + + " \n" + + " \n" + + " \n" + + " \n" + + " FIELD SYSTEM\n" + + " \n" + + " \n" + + " MID GLEN CROE\n" + + " \n" + + " \n" + + " en\n" + + " \n" + + " \n" + + " \n" + + " \n" + + " Sites and monuments databases or\n" + + " inventories\n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " 2013-12-09\n" + + " 00:00:00.0\n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " 2013-12-09\n" + + " 00:00:00.0\n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " BUILDING\n" + + " \n" + + " \n" + + " 2420500\n" + + " http://registry.ariadne-infrastructure.eu/collection/22721290\n" + + " Multiple instances of: SHEEPFOLD<br /><br />Multiple\n" + + " instances of: BUILDING<br />Multiple instances of: FIELD SYSTEM<br\n" + + " />Possible instance of: CAIRN<br />Multiple instances of: HUT\n" + + " CIRCLE<br />Possible instance of: HUT CIRCLE\n" + + " \n" + + " "; + + @Test + public void testBuildXML(){ + LocalDateTime now = LocalDateTime.now(); + String res = transformJob.buildXML(header, now.toString(), metadata, footer); + LocalDateTime end = LocalDateTime.now(); + System.out.println("Building XML took:"+ Duration.between(now, end).toMillis()); + System.out.println(res); + } +} diff --git a/dnet-ariadneplus/src/test/java/eu/dnetlib/data/collector/plugins/ariadneplus/ehri/EHRIGraphQLClientTest.java b/dnet-ariadneplus/src/test/java/eu/dnetlib/data/collector/plugins/ariadneplus/ehri/EHRIGraphQLClientTest.java deleted file mode 100644 index 0691021..0000000 --- a/dnet-ariadneplus/src/test/java/eu/dnetlib/data/collector/plugins/ariadneplus/ehri/EHRIGraphQLClientTest.java +++ /dev/null @@ -1,72 +0,0 @@ -package eu.dnetlib.data.collector.plugins.ariadneplus.ehri; - -import java.io.IOException; -import java.io.InputStreamReader; -import java.net.URISyntaxException; -import java.net.URL; -import java.util.Iterator; - -import eu.dnetlib.rmi.data.CollectorServiceException; -import org.apache.commons.io.IOUtils; -import org.junit.Ignore; -import org.junit.Test; - -/** - * Created by Alessia Bardi on 19/04/2017. - * - * @author Alessia Bardi - */ -public class EHRIGraphQLClientTest { - - private EHRIGraphQLClient client = new EHRIGraphQLClient(); - private String baseUrl = "https://portal.ehri-project.eu/api/graphql"; - private String query = "{\"query\":\"{topLevelDocumentaryUnits{items{id}}}\"}"; - - private String simpleOutput = "{\n" - + " \"data\" : {\n" - + " \"topLevelDocumentaryUnits\" : {\n" - + " \"items\" : [ {\n" - + " \"id\" : \"at-001985-ikg_ar_1\"\n" - + " }, {\n" - + " \"id\" : \"at-001985-ikg_ar_11\"\n" - + " }]\n" - + " }\n" - + " }\n" - + "}"; - - @Test - public void testGetIdentifiers() throws IOException { - Iterator it = client.getIdentifiers(new InputStreamReader(IOUtils.toInputStream(simpleOutput, "UTF-8"))); - while(it.hasNext()) - System.out.println(it.next()); - } - - @Ignore - @Test - public void testRemoteCollect() throws CollectorServiceException, IOException, URISyntaxException { - Iterator it =client.collect(baseUrl, query); - int stopAt = 5; - for(int i = 0; i < stopAt && it.hasNext(); i++){ - System.out.println(it.next()); - } - } - - @Ignore - @Test - public void testGetResource() throws IOException { - String id = "at-001985-ikg_ar_11"; - String target = "https://portal.ehri-project.eu/units/"+id+"/export"; - URL url = new URL(target); - String res = IOUtils.toString(url, "UTF-8"); - System.out.println(res); - } - -// @Test -// public void testDotNotation(){ -// JsonObject jsonObject = new JsonParser().parse(simpleOutput).getAsJsonObject(); -// JsonArray items = jsonObject.getAsJsonArray("data.topLevelDocumentaryUnits.items"); -// for(JsonElement id : items){ -// System.out.println(id.getAsJsonObject().get("id")); -// } -// } -} diff --git a/dnet-ariadneplus/src/test/java/eu/dnetlib/data/collector/plugins/ariadneplus/ehri/EHRIIteratorTest.java b/dnet-ariadneplus/src/test/java/eu/dnetlib/data/collector/plugins/ariadneplus/ehri/EHRIIteratorTest.java deleted file mode 100644 index 718ac33..0000000 --- a/dnet-ariadneplus/src/test/java/eu/dnetlib/data/collector/plugins/ariadneplus/ehri/EHRIIteratorTest.java +++ /dev/null @@ -1,32 +0,0 @@ -package eu.dnetlib.data.collector.plugins.ariadneplus.ehri; - -import org.junit.Test; - -/** - * Created by Alessia Bardi on 13/06/2017. - * - * @author Alessia Bardi - */ -public class EHRIIteratorTest { - - - private EHRIIterator ehriIterator = new EHRIIterator(null, null, null); - final String test = "\n" - + " \n" - + " us-005578-irn516886\n" - + "\t\n" - + "\t"; - - @Test - public void testRemoveDefaultEADNamespace(){ - System.out.println(ehriIterator.removeDefaultEADNamespace(test)); - } -} diff --git a/dnet-ariadneplus/src/test/java/eu/dnetlib/data/collector/plugins/ariadneplus/isidore/IsidoreIteratorTest.java b/dnet-ariadneplus/src/test/java/eu/dnetlib/data/collector/plugins/ariadneplus/isidore/IsidoreIteratorTest.java deleted file mode 100644 index f486dc3..0000000 --- a/dnet-ariadneplus/src/test/java/eu/dnetlib/data/collector/plugins/ariadneplus/isidore/IsidoreIteratorTest.java +++ /dev/null @@ -1,47 +0,0 @@ -package eu.dnetlib.data.collector.plugins.ariadneplus.isidore; - -import javax.xml.transform.TransformerFactory; - -import eu.dnetlib.miscutils.functional.xml.SaxonHelper; -import org.junit.Before; -import org.junit.Ignore; -import org.junit.Test; - -/** - * Created by Alessia Bardi on 17/11/2018. - * - * @author Alessia Bardi - */ - -public class IsidoreIteratorTest { - - TransformerFactory tf = TransformerFactory.newInstance(); - SaxonHelper saxonHelper = new SaxonHelper(); - - private String baseUrl = "https://api.rechercheisidore.fr/resource/search"; - private String queryParams = "source_tree=10670/3.ji0kr6|10670/2.rao8tk"; - private String pageParam = "page"; - private int startFromPage = 1; - private String nextPagePath = "//page/@next"; - private String pageSizeParam = "size"; - private int pageSize = 10; - private String resultTotalXpath = "//replies/meta/@items"; - private String entityXpath = "//reply"; - - @Before - public void setup(){ - saxonHelper.setSaxonTransformerFactory(tf); - } - - @Ignore - @Test - public void test(){ - IsidoreIterator rip = new IsidoreIterator(saxonHelper, baseUrl, queryParams,pageParam, startFromPage, - nextPagePath, pageSizeParam, pageSize, resultTotalXpath, entityXpath); - while(rip.hasNext()){ - String record = rip.next(); - System.out.println(record); - } - } - -}