[HostedByMap] changed the file for doaj to be downloaded. Now it is the json one since it has information about the oa_start for the jpurnal. This information with the review process one has been added to the hostedbymap.

This commit is contained in:
Miriam Baglioni 2022-02-16 12:12:27 +01:00
parent 91d3a47110
commit 59e448205f
34 changed files with 1302 additions and 148 deletions

View File

@ -0,0 +1,104 @@
package eu.dnetlib.dhp.oa.graph.hostedbymap;
import static eu.dnetlib.dhp.common.collection.DecompressTarGz.doExtract;
import java.io.BufferedOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.PrintWriter;
import java.util.Arrays;
import java.util.Objects;
import java.util.stream.Stream;
import java.util.zip.GZIPOutputStream;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import eu.dnetlib.dhp.oa.graph.hostedbymap.model.DOAJModel;
import eu.dnetlib.dhp.oa.graph.hostedbymap.model.doaj.DOAJEntry;
import org.apache.commons.io.IOUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.*;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import eu.dnetlib.dhp.application.ArgumentApplicationParser;
public class ExtractAndMapDoajJson {
private static final Logger log = LoggerFactory.getLogger(ExtractAndMapDoajJson.class);
public static void main(final String[] args) throws Exception {
final ArgumentApplicationParser parser = new ArgumentApplicationParser(
IOUtils
.toString(
Objects
.requireNonNull(
ExtractAndMapDoajJson.class
.getResourceAsStream(
"/eu/dnetlib/dhp/oa/graph/hostedbymap/download_json_parameters.json"))));
parser.parseArgument(args);
final String compressedInput = parser.get("compressedFile");
log.info("compressedInput {}", compressedInput);
final String hdfsNameNode = parser.get("hdfsNameNode");
log.info("hdfsNameNode {}", hdfsNameNode);
final String outputPath = parser.get("outputPath");
log.info("outputPath {}", outputPath);
final String workingPath = parser.get("workingPath");
log.info("workingPath {}", workingPath);
Configuration conf = new Configuration();
conf.set("fs.defaultFS", hdfsNameNode);
FileSystem fs = FileSystem.get(conf);
doExtract(fs, workingPath, compressedInput);
doMap(fs, workingPath, outputPath);
}
private static void doMap(FileSystem fs, String workingPath, String outputPath) throws IOException {
RemoteIterator<LocatedFileStatus> fileStatusListIterator = fs.listFiles(
new Path(workingPath), true);
Path hdfsWritePath = new Path(outputPath.concat(outputPath));
if (fs.exists(hdfsWritePath)) {
fs.delete(hdfsWritePath, true);
}
try (
FSDataOutputStream out = fs
.create(hdfsWritePath);
PrintWriter writer = new PrintWriter(new BufferedOutputStream(out))) {
while(fileStatusListIterator.hasNext()){
FSDataInputStream is = fs.open(fileStatusListIterator.next().getPath());
DOAJEntry[] doajEntries = new ObjectMapper().readValue((InputStream)is, DOAJEntry[].class);
Arrays.stream(doajEntries).forEach(doaj -> {
DOAJModel doajModel = new DOAJModel();
doajModel.setOaStart(doaj.getBibjson().getOa_start());
doajModel.setEissn(doaj.getBibjson().getEissn());
doajModel.setIssn(doaj.getBibjson().getPissn());
doajModel.setJournalTitle(doaj.getBibjson().getTitle());
doajModel.setReviewProcess(doaj.getBibjson().getEditorial().getReview_process());
try {
writer.println(new ObjectMapper().writeValueAsString(doajModel));
} catch (JsonProcessingException e) {
e.printStackTrace();
}
});
}
}
}
}

View File

@ -1,55 +0,0 @@
package eu.dnetlib.dhp.oa.graph.hostedbymap;
import static eu.dnetlib.dhp.common.collection.DecompressTarGz.doExtract;
import java.io.*;
import java.nio.charset.StandardCharsets;
import java.util.Objects;
import org.apache.commons.io.IOUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import eu.dnetlib.dhp.application.ArgumentApplicationParser;
import eu.dnetlib.dhp.common.collection.CollectorException;
import eu.dnetlib.dhp.common.collection.HttpConnector2;
public class ExtractDoajJson {
private static final Logger log = LoggerFactory.getLogger(ExtractDoajJson.class);
public static void main(final String[] args) throws Exception {
final ArgumentApplicationParser parser = new ArgumentApplicationParser(
IOUtils
.toString(
Objects
.requireNonNull(
ExtractDoajJson.class
.getResourceAsStream(
"/eu/dnetlib/dhp/oa/graph/hostedbymap/download_json_parameters.json"))));
parser.parseArgument(args);
final String compressedInput = parser.get("compressedFile");
log.info("compressedInput {}", compressedInput);
final String hdfsNameNode = parser.get("hdfsNameNode");
log.info("hdfsNameNode {}", hdfsNameNode);
final String outputPath = parser.get("outputPath");
log.info("outputPath {}", outputPath);
Configuration conf = new Configuration();
conf.set("fs.defaultFS", hdfsNameNode);
FileSystem fs = FileSystem.get(conf);
doExtract(fs, outputPath, compressedInput);
}
}

View File

@ -2,6 +2,7 @@
package eu.dnetlib.dhp.oa.graph.hostedbymap.model;
import java.io.Serializable;
import java.util.List;
import com.opencsv.bean.CsvBindByName;
@ -17,7 +18,17 @@ public class DOAJModel implements Serializable {
private String eissn;
@CsvBindByName(column = "Review process")
private String reviewProcess;
private List<String> reviewProcess;
private Integer oaStart ;
public Integer getOaStart() {
return oaStart;
}
public void setOaStart(Integer oaStart) {
this.oaStart = oaStart;
}
public String getJournalTitle() {
return journalTitle;
@ -43,11 +54,11 @@ public class DOAJModel implements Serializable {
this.eissn = eissn;
}
public String getReviewProcess() {
public List<String> getReviewProcess() {
return reviewProcess;
}
public void setReviewProcess(String reviewProcess) {
public void setReviewProcess(List<String> reviewProcess) {
this.reviewProcess = reviewProcess;
}
}

View File

@ -0,0 +1,34 @@
package eu.dnetlib.dhp.oa.graph.hostedbymap.model.doaj;
import java.io.Serializable;
import java.util.List;
public class APC implements Serializable {
private Boolean has_apc;
private String url;
private List<Max> max;
public List<Max> getMax() {
return max;
}
public void setMax(List<Max> max) {
this.max = max;
}
public Boolean getHas_apc() {
return has_apc;
}
public void setHas_apc(Boolean has_apc) {
this.has_apc = has_apc;
}
public String getUrl() {
return url;
}
public void setUrl(String url) {
this.url = url;
}
}

View File

@ -0,0 +1,24 @@
package eu.dnetlib.dhp.oa.graph.hostedbymap.model.doaj;
import java.io.Serializable;
public class Admin implements Serializable {
private Boolean ticked;
private Boolean seal;
public Boolean getTicked() {
return ticked;
}
public void setTicked(Boolean ticked) {
this.ticked = ticked;
}
public Boolean getSeal() {
return seal;
}
public void setSeal(Boolean seal) {
this.seal = seal;
}
}

View File

@ -0,0 +1,43 @@
package eu.dnetlib.dhp.oa.graph.hostedbymap.model.doaj;
import java.io.Serializable;
import java.util.List;
public class Article implements Serializable {
private String license_display_example_url;
private List<String> license_display;
private Boolean orcid;
private Boolean i4oc_open_citations;
public String getLicense_display_example_url() {
return license_display_example_url;
}
public void setLicense_display_example_url(String license_display_example_url) {
this.license_display_example_url = license_display_example_url;
}
public List<String> getLicense_display() {
return license_display;
}
public void setLicense_display(List<String> license_display) {
this.license_display = license_display;
}
public Boolean getOrcid() {
return orcid;
}
public void setOrcid(Boolean orcid) {
this.orcid = orcid;
}
public Boolean getI4oc_open_citations() {
return i4oc_open_citations;
}
public void setI4oc_open_citations(Boolean i4oc_open_citations) {
this.i4oc_open_citations = i4oc_open_citations;
}
}

View File

@ -0,0 +1,255 @@
package eu.dnetlib.dhp.oa.graph.hostedbymap.model.doaj;
import com.fasterxml.jackson.annotation.JsonIgnore;
import java.io.Serializable;
import java.util.List;
public class BibJson implements Serializable {
private Editorial editorial;
private PidScheme pid_scheme;
private Copyright copyright;
private List<String> keywords;
private Plagiarism plagiarism;
private List<Subject> subject;
private String eissn;
private String pissn;
private List<String> language;
private String title;
private Article article ;
private Institution institution;
private Preservation preservation;
private List<License> license;
private Ref ref;
private Integer oa_start;
private APC apc;
private OtherCharges other_charges;
private Integer publication_time_weeks;
private DepositPolicy deposit_policy;
private Publisher publisher;
private Boolean boai;
private Waiver waiver;
private String alternative_title;
private List<String> is_replaced_by;
private List<String> replaces;
private String discontinued_date;
public String getDiscontinued_date() {
return discontinued_date;
}
public void setDiscontinued_date(String discontinued_date) {
this.discontinued_date = discontinued_date;
}
public List<String> getReplaces() {
return replaces;
}
public void setReplaces(List<String> replaces) {
this.replaces = replaces;
}
public List<String> getIs_replaced_by() {
return is_replaced_by;
}
public void setIs_replaced_by(List<String> is_replaced_by) {
this.is_replaced_by = is_replaced_by;
}
public String getAlternative_title() {
return alternative_title;
}
public void setAlternative_title(String alternative_title) {
this.alternative_title = alternative_title;
}
public String getPissn() {
return pissn;
}
public void setPissn(String pissn) {
this.pissn = pissn;
}
public Editorial getEditorial() {
return editorial;
}
public void setEditorial(Editorial editorial) {
this.editorial = editorial;
}
public PidScheme getPid_scheme() {
return pid_scheme;
}
public void setPid_scheme(PidScheme pid_scheme) {
this.pid_scheme = pid_scheme;
}
public Copyright getCopyright() {
return copyright;
}
public void setCopyright(Copyright copyright) {
this.copyright = copyright;
}
public List<String> getKeywords() {
return keywords;
}
public void setKeywords(List<String> keywords) {
this.keywords = keywords;
}
public Plagiarism getPlagiarism() {
return plagiarism;
}
public void setPlagiarism(Plagiarism plagiarism) {
this.plagiarism = plagiarism;
}
public List<Subject> getSubject() {
return subject;
}
public void setSubject(List<Subject> subject) {
this.subject = subject;
}
public String getEissn() {
return eissn;
}
public void setEissn(String eissn) {
this.eissn = eissn;
}
public List<String> getLanguage() {
return language;
}
public void setLanguage(List<String> language) {
this.language = language;
}
public String getTitle() {
return title;
}
public void setTitle(String title) {
this.title = title;
}
public Article getArticle() {
return article;
}
public void setArticle(Article article) {
this.article = article;
}
public Institution getInstitution() {
return institution;
}
public void setInstitution(Institution institution) {
this.institution = institution;
}
public Preservation getPreservation() {
return preservation;
}
public void setPreservation(Preservation preservation) {
this.preservation = preservation;
}
public List<License> getLicense() {
return license;
}
public void setLicense(List<License> license) {
this.license = license;
}
public Ref getRef() {
return ref;
}
public void setRef(Ref ref) {
this.ref = ref;
}
public Integer getOa_start() {
return oa_start;
}
public void setOa_start(Integer oa_start) {
this.oa_start = oa_start;
}
public APC getApc() {
return apc;
}
public void setApc(APC apc) {
this.apc = apc;
}
public OtherCharges getOther_charges() {
return other_charges;
}
public void setOther_charges(OtherCharges other_charges) {
this.other_charges = other_charges;
}
public Integer getPublication_time_weeks() {
return publication_time_weeks;
}
public void setPublication_time_weeks(Integer publication_time_weeks) {
this.publication_time_weeks = publication_time_weeks;
}
public DepositPolicy getDeposit_policy() {
return deposit_policy;
}
public void setDeposit_policy(DepositPolicy deposit_policy) {
this.deposit_policy = deposit_policy;
}
public Publisher getPublisher() {
return publisher;
}
public void setPublisher(Publisher publisher) {
this.publisher = publisher;
}
public Boolean getBoai() {
return boai;
}
public void setBoai(Boolean boai) {
this.boai = boai;
}
public Waiver getWaiver() {
return waiver;
}
public void setWaiver(Waiver waiver) {
this.waiver = waiver;
}
}

View File

@ -0,0 +1,24 @@
package eu.dnetlib.dhp.oa.graph.hostedbymap.model.doaj;
import java.io.Serializable;
public class Copyright implements Serializable {
private Boolean author_retains;
private String url;
public Boolean getAuthor_retains() {
return author_retains;
}
public void setAuthor_retains(Boolean author_retains) {
this.author_retains = author_retains;
}
public String getUrl() {
return url;
}
public void setUrl(String url) {
this.url = url;
}
}

View File

@ -0,0 +1,51 @@
package eu.dnetlib.dhp.oa.graph.hostedbymap.model.doaj;
import java.io.Serializable;
public class DOAJEntry implements Serializable {
private String last_updated;
private BibJson bibjson;
private Admin admin;
private String created_date;
private String id;
public String getLast_updated() {
return last_updated;
}
public void setLast_updated(String last_updated) {
this.last_updated = last_updated;
}
public BibJson getBibjson() {
return bibjson;
}
public void setBibjson(BibJson bibjson) {
this.bibjson = bibjson;
}
public Admin getAdmin() {
return admin;
}
public void setAdmin(Admin admin) {
this.admin = admin;
}
public String getCreated_date() {
return created_date;
}
public void setCreated_date(String created_date) {
this.created_date = created_date;
}
public String getId() {
return id;
}
public void setId(String id) {
this.id = id;
}
}

View File

@ -0,0 +1,34 @@
package eu.dnetlib.dhp.oa.graph.hostedbymap.model.doaj;
import java.io.Serializable;
import java.util.List;
public class DepositPolicy implements Serializable {
private List<String> service;
private String url;
private Boolean has_policy;
public List<String> getService() {
return service;
}
public void setService(List<String> service) {
this.service = service;
}
public String getUrl() {
return url;
}
public void setUrl(String url) {
this.url = url;
}
public Boolean getHas_policy() {
return has_policy;
}
public void setHas_policy(Boolean has_policy) {
this.has_policy = has_policy;
}
}

View File

@ -0,0 +1,34 @@
package eu.dnetlib.dhp.oa.graph.hostedbymap.model.doaj;
import java.io.Serializable;
import java.util.List;
public class Editorial implements Serializable {
private List<String> review_process;
private String review_url;
private String board_url;
public List<String> getReview_process() {
return review_process;
}
public void setReview_process(List<String> review_process) {
this.review_process = review_process;
}
public String getReview_url() {
return review_url;
}
public void setReview_url(String review_url) {
this.review_url = review_url;
}
public String getBoard_url() {
return board_url;
}
public void setBoard_url(String board_url) {
this.board_url = board_url;
}
}

View File

@ -0,0 +1,24 @@
package eu.dnetlib.dhp.oa.graph.hostedbymap.model.doaj;
import java.io.Serializable;
public class Institution implements Serializable {
private String country;
private String name;
public String getCountry() {
return country;
}
public void setCountry(String country) {
this.country = country;
}
public String getName() {
return name;
}
public void setName(String name) {
this.name = name;
}
}

View File

@ -0,0 +1,66 @@
package eu.dnetlib.dhp.oa.graph.hostedbymap.model.doaj;
import com.fasterxml.jackson.annotation.JsonProperty;
import java.io.Serializable;
public class License implements Serializable {
private Boolean nc;
private Boolean nd;
private Boolean by;
private String type;
private Boolean sa;
private String url;
public Boolean getnC() {
return nc;
}
@JsonProperty("NC")
public void setnC(Boolean NC) {
this.nc = NC;
}
public String getType() {
return type;
}
public void setType(String type) {
this.type = type;
}
public Boolean getNd() {
return nd;
}
@JsonProperty("ND")
public void setNd(Boolean nd) {
this.nd = nd;
}
public Boolean getBy() {
return by;
}
@JsonProperty("BY")
public void setBy(Boolean by) {
this.by = by;
}
public Boolean getSa() {
return sa;
}
@JsonProperty("SA")
public void setSa(Boolean sa) {
this.sa = sa;
}
public String getUrl() {
return url;
}
public void setUrl(String url) {
this.url = url;
}
}

View File

@ -0,0 +1,24 @@
package eu.dnetlib.dhp.oa.graph.hostedbymap.model.doaj;
import java.io.Serializable;
public class Max implements Serializable {
private Integer price;
private String currency;
public Integer getPrice() {
return price;
}
public void setPrice(Integer price) {
this.price = price;
}
public String getCurrency() {
return currency;
}
public void setCurrency(String currency) {
this.currency = currency;
}
}

View File

@ -0,0 +1,24 @@
package eu.dnetlib.dhp.oa.graph.hostedbymap.model.doaj;
import java.io.Serializable;
public class OtherCharges implements Serializable {
private Boolean has_other_charges;
private String url;
public String getUrl() {
return url;
}
public void setUrl(String url) {
this.url = url;
}
public Boolean getHas_other_charges() {
return has_other_charges;
}
public void setHas_other_charges(Boolean has_other_charges) {
this.has_other_charges = has_other_charges;
}
}

View File

@ -0,0 +1,25 @@
package eu.dnetlib.dhp.oa.graph.hostedbymap.model.doaj;
import java.io.Serializable;
import java.util.List;
public class PidScheme implements Serializable {
private List<String> scheme;
private Boolean has_pid_scheme;
public List<String> getScheme() {
return scheme;
}
public void setScheme(List<String> scheme) {
this.scheme = scheme;
}
public Boolean getHas_pid_scheme() {
return has_pid_scheme;
}
public void setHas_pid_scheme(Boolean has_pid_scheme) {
this.has_pid_scheme = has_pid_scheme;
}
}

View File

@ -0,0 +1,25 @@
package eu.dnetlib.dhp.oa.graph.hostedbymap.model.doaj;
import javax.sql.rowset.serial.SerialArray;
import java.io.Serializable;
public class Plagiarism implements Serializable{
private Boolean detection;
private String url;
public Boolean getDetection() {
return detection;
}
public void setDetection(Boolean detection) {
this.detection = detection;
}
public String getUrl() {
return url;
}
public void setUrl(String url) {
this.url = url;
}
}

View File

@ -0,0 +1,43 @@
package eu.dnetlib.dhp.oa.graph.hostedbymap.model.doaj;
import java.io.Serializable;
import java.util.List;
public class Preservation implements Serializable {
private Boolean has_preservation;
private List<String> service;
private List<String> national_library;
private String url;
public String getUrl() {
return url;
}
public void setUrl(String url) {
this.url = url;
}
public Boolean getHas_preservation() {
return has_preservation;
}
public void setHas_preservation(Boolean has_preservation) {
this.has_preservation = has_preservation;
}
public List<String> getService() {
return service;
}
public void setService(List<String> service) {
this.service = service;
}
public List<String> getNational_library() {
return national_library;
}
public void setNational_library(List<String> national_library) {
this.national_library = national_library;
}
}

View File

@ -0,0 +1,24 @@
package eu.dnetlib.dhp.oa.graph.hostedbymap.model.doaj;
import java.io.Serializable;
public class Publisher implements Serializable {
private String country;
private String name;
public String getCountry() {
return country;
}
public void setCountry(String country) {
this.country = country;
}
public String getName() {
return name;
}
public void setName(String name) {
this.name = name;
}
}

View File

@ -0,0 +1,51 @@
package eu.dnetlib.dhp.oa.graph.hostedbymap.model.doaj;
import java.io.Serializable;
public class Ref implements Serializable {
private String aims_scope;
private String journal;
private String oa_statement;
private String author_instructions;
private String license_terms;
public String getAims_scope() {
return aims_scope;
}
public void setAims_scope(String aims_scope) {
this.aims_scope = aims_scope;
}
public String getJournal() {
return journal;
}
public void setJournal(String journal) {
this.journal = journal;
}
public String getOa_statement() {
return oa_statement;
}
public void setOa_statement(String oa_statement) {
this.oa_statement = oa_statement;
}
public String getAuthor_instructions() {
return author_instructions;
}
public void setAuthor_instructions(String author_instructions) {
this.author_instructions = author_instructions;
}
public String getLicense_terms() {
return license_terms;
}
public void setLicense_terms(String license_terms) {
this.license_terms = license_terms;
}
}

View File

@ -0,0 +1,33 @@
package eu.dnetlib.dhp.oa.graph.hostedbymap.model.doaj;
import java.io.Serializable;
public class Subject implements Serializable {
private String code;
private String scheme;
private String term;
public String getCode() {
return code;
}
public void setCode(String code) {
this.code = code;
}
public String getScheme() {
return scheme;
}
public void setScheme(String scheme) {
this.scheme = scheme;
}
public String getTerm() {
return term;
}
public void setTerm(String term) {
this.term = term;
}
}

View File

@ -0,0 +1,24 @@
package eu.dnetlib.dhp.oa.graph.hostedbymap.model.doaj;
import java.io.Serializable;
public class Waiver implements Serializable {
private Boolean has_waiver;
private String url;
public String getUrl() {
return url;
}
public void setUrl(String url) {
this.url = url;
}
public Boolean getHas_waiver() {
return has_waiver;
}
public void setHas_waiver(Boolean has_waiver) {
this.has_waiver = has_waiver;
}
}

View File

@ -6,6 +6,7 @@
"paramDescription": "the output json file produced by the CSV downlaod procedure",
"paramRequired": true
},
{
"paramName": "hnn",
"paramLongName": "hdfsNameNode",
@ -16,6 +17,11 @@
"paramLongName": "compressedFile",
"paramDescription": "the path used to store the HostedByMap",
"paramRequired": true
},{
"paramName":"wp",
"paramLongName":"workingPath",
"paramDescription": "the output json file produced by the CSV downlaod procedure",
"paramRequired": true
}
]

View File

@ -0,0 +1,3 @@
## This is a classpath-based import file (this header is required)
download_doaj classpath eu/dnetlib/dhp/oa/graph/hostedbymap/subworkflows/doaj/oozie_app
download_unibi classpath eu/dnetlib/dhp/oa/graph/hostedbymap/subworkflows/unibi/oozie_app

View File

@ -74,7 +74,7 @@
<decision name="resume_from">
<switch>
<case to="produceHBM">${wf:conf('resumeFrom') eq 'ProduceHBM'}</case>
<case to="remove_hbmpath">${wf:conf('resumeFrom') eq 'download_csv'}</case>
<case to="download_files">${wf:conf('resumeFrom') eq 'Download'}</case>
<default to="prepareInfo"/>
</switch>
</decision>
@ -83,18 +83,17 @@
<message>Action failed, error message[${wf:errorMessage(wf:lastErrorNode())}]</message>
</kill>
<action name="remove_hbmpath">
<fs>
<delete path="${hostedByMapPath}"/>
<!-- <mkdir path="${hostedByMapPath}"/>-->
</fs>
<ok to="fork_downloads_csv"/>
<error to="Kill"/>
</action>
<decision name="download_files">
<switch>
<case to="fork_downloads_csv">${wf:conf('download') eq 'both'}</case>
<case to="downloadGold">${wf:conf('download') eq 'gold'}</case>
<default to="downloadDOAJ"/>
</switch>
</decision>
<fork name="fork_downloads_csv">
<path start="download_gold"/>
<path start="download_doaj"/>
<path start="download_doaj_json"/>
</fork>
<action name="download_gold">
@ -103,27 +102,13 @@
<arg>--hdfsNameNode</arg><arg>${nameNode}</arg>
<arg>--fileURL</arg><arg>${unibiFileURL}</arg>
<arg>--tmpFile</arg><arg>/tmp/unibi_gold_replaced.csv</arg>
<arg>--outputFile</arg><arg>${workingDir}/unibi_gold.json</arg>
<arg>--outputFile</arg><arg>/user/${wf:user()}/data/unibi_gold.json</arg>
<arg>--classForName</arg><arg>eu.dnetlib.dhp.oa.graph.hostedbymap.model.UnibiGoldModel</arg>
</java>
<ok to="join_download"/>
<error to="Kill"/>
</action>
<action name="download_doaj">
<java>
<main-class>eu.dnetlib.dhp.oa.graph.hostedbymap.DownloadCSV2</main-class>
<arg>--hdfsNameNode</arg><arg>${nameNode}</arg>
<arg>--fileURL</arg><arg>${doajFileURL}</arg>
<arg>--tmpFile</arg><arg>/tmp/doaj_replaced.csv</arg>
<arg>--outputFile</arg><arg>${workingDir}/doaj.json</arg>
<arg>--classForName</arg><arg>eu.dnetlib.dhp.oa.graph.hostedbymap.model.DOAJModel</arg>
</java>
<ok to="join_download"/>
<error to="Kill"/>
</action>
<action name="download_doaj_json">
<shell xmlns="uri:oozie:shell-action:0.2">
<job-tracker>${jobTracker}</job-tracker>
@ -142,24 +127,72 @@
<file>download.sh</file>
<capture-output/>
</shell>
<ok to="extractTarGz"/>
<ok to="extractTarGzAndMap"/>
<error to="Kill"/>
</action>
<action name="extractTarGz">
<action name="extractTarGzAndMap">
<java>
<main-class>eu.dnetlib.dhp.oa.graph.hostedbymap.ExtractDoajJson</main-class>
<main-class>eu.dnetlib.dhp.oa.graph.hostedbymap.ExtractAndMapDoajJson</main-class>
<arg>--hdfsNameNode</arg><arg>${nameNode}</arg>
<arg>--compressedFile</arg><arg>${dumpPath}/${dumpFileName}</arg>
<arg>--outputPath</arg><arg>${workingDir}/DOAJ/</arg>
<arg>--workingPath</arg><arg>${workingDir}/DOAJ/</arg>
<arg>--outputFile</arg><arg>/user/${wf:user()}/data/doaj.json</arg>
</java>
<ok to="End"/>
<ok to="join_download"/>
<error to="Kill"/>
</action>
<join name="join_download" to="produceHBM"/>
<action name="downloadGold">
<java>
<main-class>eu.dnetlib.dhp.oa.graph.hostedbymap.DownloadCSV2</main-class>
<arg>--hdfsNameNode</arg><arg>${nameNode}</arg>
<arg>--fileURL</arg><arg>${unibiFileURL}</arg>
<arg>--tmpFile</arg><arg>/tmp/unibi_gold_replaced.csv</arg>
<arg>--outputFile</arg><arg>/user/${wf:user()}/data/unibi_gold.json</arg>
<arg>--classForName</arg><arg>eu.dnetlib.dhp.oa.graph.hostedbymap.model.UnibiGoldModel</arg>
</java>
<ok to="produceHBM"/>
<error to="Kill"/>
</action>
<action name="downloadDOAJ">
<shell xmlns="uri:oozie:shell-action:0.2">
<job-tracker>${jobTracker}</job-tracker>
<name-node>${nameNode}</name-node>
<configuration>
<property>
<name>mapred.job.queue.name</name>
<value>${queueName}</value>
</property>
</configuration>
<exec>download.sh</exec>
<argument>${doajJsonFileURL}</argument>
<argument>${dumpPath}</argument>
<argument>${dumpFileName}</argument>
<env-var>HADOOP_USER_NAME=${wf:user()}</env-var>
<file>download.sh</file>
<capture-output/>
</shell>
<ok to="extract"/>
<error to="Kill"/>
</action>
<action name="extract">
<java>
<main-class>eu.dnetlib.dhp.oa.graph.hostedbymap.ExtractAndMapDoajJson</main-class>
<arg>--hdfsNameNode</arg><arg>${nameNode}</arg>
<arg>--compressedFile</arg><arg>${dumpPath}/${dumpFileName}</arg>
<arg>--workingPath</arg><arg>${workingDir}/DOAJ/</arg>
<arg>--outputFile</arg><arg>/user/${wf:user()}/data/doaj.json</arg>
</java>
<ok to="produceHBM"/>
<error to="Kill"/>
</action>
<action name="produceHBM">
<spark xmlns="uri:oozie:spark-action:0.2">
<master>yarn-cluster</master>

View File

@ -0,0 +1,30 @@
<configuration>
<property>
<name>jobTracker</name>
<value>yarnRM</value>
</property>
<property>
<name>nameNode</name>
<value>hdfs://nameservice1</value>
</property>
<property>
<name>oozie.use.system.libpath</name>
<value>true</value>
</property>
<property>
<name>hiveMetastoreUris</name>
<value>thrift://iis-cdh5-test-m3.ocean.icm.edu.pl:9083</value>
</property>
<property>
<name>hiveJdbcUrl</name>
<value>jdbc:hive2://iis-cdh5-test-m3.ocean.icm.edu.pl:10000</value>
</property>
<property>
<name>hiveDbName</name>
<value>openaire</value>
</property>
<property>
<name>oozie.launcher.mapreduce.user.classpath.first</name>
<value>true</value>
</property>
</configuration>

View File

@ -0,0 +1,114 @@
<workflow-app name="hosted_by_map" xmlns="uri:oozie:workflow:0.5">
<parameters>
<property>
<name>sourcePath</name>
<description>the source path</description>
</property>
<property>
<name>outputPath</name>
<description>the output path</description>
</property>
<property>
<name>sparkDriverMemory</name>
<description>memory for driver process</description>
</property>
<property>
<name>sparkExecutorMemory</name>
<description>memory for individual executor</description>
</property>
<property>
<name>sparkExecutorCores</name>
<description>number of cores used by single executor</description>
</property>
<property>
<name>oozieActionShareLibForSpark2</name>
<description>oozie action sharelib for spark 2.*</description>
</property>
<property>
<name>spark2ExtraListeners</name>
<value>com.cloudera.spark.lineage.NavigatorAppListener</value>
<description>spark 2.* extra listeners classname</description>
</property>
<property>
<name>spark2SqlQueryExecutionListeners</name>
<value>com.cloudera.spark.lineage.NavigatorQueryListener</value>
<description>spark 2.* sql query execution listeners classname</description>
</property>
<property>
<name>spark2YarnHistoryServerAddress</name>
<description>spark 2.* yarn history server address</description>
</property>
<property>
<name>spark2EventLogDir</name>
<description>spark 2.* event log dir location</description>
</property>
</parameters>
<global>
<job-tracker>${jobTracker}</job-tracker>
<name-node>${nameNode}</name-node>
<configuration>
<property>
<name>mapreduce.job.queuename</name>
<value>${queueName}</value>
</property>
<property>
<name>oozie.launcher.mapred.job.queue.name</name>
<value>${oozieLauncherQueueName}</value>
</property>
<property>
<name>oozie.action.sharelib.for.spark</name>
<value>${oozieActionShareLibForSpark2}</value>
</property>
</configuration>
</global>
<start to="downloadDOAJ"/>
<kill name="Kill">
<message>Action failed, error message[${wf:errorMessage(wf:lastErrorNode())}]</message>
</kill>
<action name="downloadDOAJ">
<shell xmlns="uri:oozie:shell-action:0.2">
<job-tracker>${jobTracker}</job-tracker>
<name-node>${nameNode}</name-node>
<configuration>
<property>
<name>mapred.job.queue.name</name>
<value>${queueName}</value>
</property>
</configuration>
<exec>download.sh</exec>
<argument>${doajJsonFileURL}</argument>
<argument>${dumpPath}</argument>
<argument>${dumpFileName}</argument>
<env-var>HADOOP_USER_NAME=${wf:user()}</env-var>
<file>download.sh</file>
<capture-output/>
</shell>
<ok to="extractTarGzAndMap"/>
<error to="Kill"/>
</action>
<action name="extractTarGzAndMap">
<java>
<main-class>eu.dnetlib.dhp.oa.graph.hostedbymap.ExtractAndMapDoajJson</main-class>
<arg>--hdfsNameNode</arg><arg>${nameNode}</arg>
<arg>--compressedFile</arg><arg>${dumpPath}/${dumpFileName}</arg>
<arg>--workingPath</arg><arg>${workingDir}/DOAJ/</arg>
<arg>--outputFile</arg><arg>/user/${wf:user()}/data/doaj.json</arg>
</java>
<ok to="produceHBM"/>
<error to="End"/>
</action>
<end name="End"/>
</workflow-app>

View File

@ -10,7 +10,9 @@ case class HostedByItemType(
issn: String,
eissn: String,
lissn: String,
openAccess: Boolean
openAccess: Boolean,
oaDate: Int,
reviewProcess: List[String]
) {}
case class HostedByInfo(
@ -58,7 +60,7 @@ object Aggregators {
] {
override def zero: (String, HostedByItemType) =
("", HostedByItemType("", "", "", "", "", false))
("", HostedByItemType("", "", "", "", "", false, -1, List[String]()))
override def reduce(
b: (String, HostedByItemType),
@ -78,6 +80,20 @@ object Aggregators {
return b1
}
if (b1._2.id.startsWith("10|")) {
if(b2._2.oaDate != -1){
return (
b1._1,
HostedByItemType(
b1._2.id,
b1._2.officialname,
b1._2.issn,
b1._2.eissn,
b1._2.lissn,
b1._2.openAccess || b2._2.openAccess,
b2._2.oaDate,
b2._2.reviewProcess
))
}
return (
b1._1,
HostedByItemType(
@ -86,11 +102,28 @@ object Aggregators {
b1._2.issn,
b1._2.eissn,
b1._2.lissn,
b1._2.openAccess || b2._2.openAccess
b1._2.openAccess || b2._2.openAccess,
b1._2.oaDate,
b1._2.reviewProcess
)
)
}
if(b1._2.oaDate != -1){
return (
b2._1,
HostedByItemType(
b2._2.id,
b2._2.officialname,
b2._2.issn,
b2._2.eissn,
b2._2.lissn,
b1._2.openAccess || b2._2.openAccess,
b1._2.oaDate,
b1._2.reviewProcess
)
)
}
return (
b2._1,
HostedByItemType(
@ -99,10 +132,13 @@ object Aggregators {
b2._2.issn,
b2._2.eissn,
b2._2.lissn,
b1._2.openAccess || b2._2.openAccess
b1._2.openAccess || b2._2.openAccess,
b2._2.oaDate,
b2._2.reviewProcess
)
)
}
override def finish(reduction: (String, HostedByItemType)): (String, HostedByItemType) =

View File

@ -4,7 +4,7 @@ import com.fasterxml.jackson.databind.ObjectMapper
import eu.dnetlib.dhp.application.ArgumentApplicationParser
import eu.dnetlib.dhp.oa.graph.hostedbymap.model.{DOAJModel, UnibiGoldModel}
import eu.dnetlib.dhp.schema.oaf.Datasource
import org.apache.commons.io.IOUtils
import org.apache.commons.io.{FileUtils, IOUtils}
import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.fs.{FileSystem, Path}
import org.apache.hadoop.io.compress.GzipCodec
@ -13,7 +13,7 @@ import org.apache.spark.sql.{Dataset, Encoder, Encoders, SparkSession}
import org.json4s.DefaultFormats
import org.slf4j.{Logger, LoggerFactory}
import java.io.PrintWriter
import java.io.{File, PrintWriter}
object SparkProduceHostedByMap {
@ -34,7 +34,9 @@ object SparkProduceHostedByMap {
openaire.journal_id,
"",
"",
isOpenAccess
isOpenAccess,
-1,
List[String]()
)
case Constants.EISSN =>
HostedByItemType(
@ -43,7 +45,9 @@ object SparkProduceHostedByMap {
"",
openaire.journal_id,
"",
isOpenAccess
isOpenAccess,
-1,
List[String]()
)
case Constants.ISSNL =>
HostedByItemType(
@ -52,7 +56,9 @@ object SparkProduceHostedByMap {
"",
"",
openaire.journal_id,
isOpenAccess
isOpenAccess,
-1,
List[String]()
)
// catch the default with a variable so you can print it
@ -77,34 +83,36 @@ object SparkProduceHostedByMap {
issn: String,
eissn: String,
issnl: String,
oa: Boolean
oa: Boolean,
oaDate: Int,
reviewProcess: List[String]
): HostedByItemType = {
if (issn != null) {
if (eissn != null) {
if (issnl != null) {
HostedByItemType(id, officialname, issn, eissn, issnl, oa)
HostedByItemType(id, officialname, issn, eissn, issnl, oa, oaDate, reviewProcess)
} else {
HostedByItemType(id, officialname, issn, eissn, "", oa)
HostedByItemType(id, officialname, issn, eissn, "", oa, oaDate, reviewProcess)
}
} else {
if (issnl != null) {
HostedByItemType(id, officialname, issn, "", issnl, oa)
HostedByItemType(id, officialname, issn, "", issnl, oa, oaDate, reviewProcess)
} else {
HostedByItemType(id, officialname, issn, "", "", oa)
HostedByItemType(id, officialname, issn, "", "", oa, oaDate, reviewProcess)
}
}
} else {
if (eissn != null) {
if (issnl != null) {
HostedByItemType(id, officialname, "", eissn, issnl, oa)
HostedByItemType(id, officialname, "", eissn, issnl, oa, oaDate, reviewProcess)
} else {
HostedByItemType(id, officialname, "", eissn, "", oa)
HostedByItemType(id, officialname, "", eissn, "", oa, oaDate, reviewProcess)
}
} else {
if (issnl != null) {
HostedByItemType(id, officialname, "", "", issnl, oa)
HostedByItemType(id, officialname, "", "", issnl, oa, oaDate, reviewProcess)
} else {
HostedByItemType("", "", "", "", "", oa)
HostedByItemType("", "", "", "", "", oa, oaDate, reviewProcess)
}
}
}
@ -119,10 +127,12 @@ object SparkProduceHostedByMap {
dats.getJournal.getIssnPrinted,
dats.getJournal.getIssnOnline,
dats.getJournal.getIssnLinking,
false
false,
-1,
List[String]()
)
}
HostedByItemType("", "", "", "", "", false)
HostedByItemType("", "", "", "", "", false, -1, List[String]())
}
def oaHostedByDataset(spark: SparkSession, datasourcePath: String): Dataset[HostedByItemType] = {
@ -148,7 +158,9 @@ object SparkProduceHostedByMap {
gold.getIssn,
"",
gold.getIssnL,
true
true,
-1,
List[String]()
)
}
@ -178,7 +190,9 @@ object SparkProduceHostedByMap {
doaj.getIssn,
doaj.getEissn,
"",
true
true,
doaj.getOaStart,
doaj.getReviewProcess
)
}
@ -252,6 +266,12 @@ object SparkProduceHostedByMap {
val workingDirPath = parser.get("workingPath")
val outputPath = parser.get("outputPath")
val file = new File(outputPath);
if(file.exists() && file.isDirectory){
FileUtils.deleteDirectory(file)
}
implicit val formats = DefaultFormats
logger.info("Getting the Datasources")

View File

@ -10,6 +10,9 @@ import java.io.IOException;
import java.io.InputStreamReader;
import java.nio.file.Files;
import eu.dnetlib.dhp.oa.graph.hostedbymap.model.doaj.DOAJEntry;
import eu.dnetlib.dhp.oa.graph.hostedbymap.model.doaj.License;
import org.apache.commons.io.Charsets;
import org.apache.commons.io.FileUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
@ -27,6 +30,7 @@ import com.fasterxml.jackson.databind.ObjectMapper;
import eu.dnetlib.dhp.common.collection.CollectorException;
import eu.dnetlib.dhp.oa.graph.hostedbymap.model.DOAJModel;
import eu.dnetlib.dhp.oa.graph.hostedbymap.model.UnibiGoldModel;
import org.spark_project.guava.io.Resources;
public class DownloadCsvTest {
@ -146,4 +150,18 @@ public class DownloadCsvTest {
FileUtils.deleteQuietly(new File(workingDir));
}
@Test
public void testJsonParser() throws IOException {
String input2 = Resources.toString(getClass().getResource("/eu/dnetlib/dhp/oa/graph/hostedbymap/journal_batch_1.json") , Charsets.UTF_8) ;
DOAJEntry[] doaj = new ObjectMapper().readValue(input2, DOAJEntry[].class);
System.out.println(doaj.length);
}
}

View File

@ -0,0 +1,10 @@
[{"last_updated": "2021-07-13T13:25:09Z", "bibjson": {"editorial": {"review_process": ["Double blind peer review"], "review_url": "https://epress.lib.uts.edu.au/journals/index.php/mcs/about/editorialPolicies#peerReviewProcess", "board_url": "https://epress.lib.uts.edu.au/journals/index.php/mcs/about/editorialTeam"}, "pid_scheme": {"scheme": ["DOI"], "has_pid_scheme": true}, "copyright": {"author_retains": true, "url": "https://epress.lib.uts.edu.au/journals/index.php/mcs/about/submissions#copyrightNotice"}, "keywords": ["cultural cohesion", "cosmopolitan societies", "civil society", "dialogue", "recognition", "inclusion"], "plagiarism": {"detection": true, "url": "https://epress.lib.uts.edu.au/journals/index.php/mcs/about/editorialPolicies#custom-4"}, "subject": [{"code": "HM401-1281", "scheme": "LCC", "term": "Sociology (General)"}], "eissn": "1837-5391", "language": ["EN"], "title": "Cosmopolitan Civil Societies: An Interdisciplinary Journal", "article": {"license_display_example_url": "https://epress.lib.uts.edu.au/journals/index.php/mcs/article/view/7644", "license_display": ["Embed"], "orcid": true, "i4oc_open_citations": false}, "institution": {"country": "AU", "name": "University of Technology Sydney"}, "preservation": {"has_preservation": true, "service": ["CLOCKSS"], "national_library": ["Trove - National Library of Australia"], "url": "https://epress.lib.uts.edu.au/journals/index.php/mcs/about/editorialPolicies#archiving"}, "license": [{"NC": false, "ND": false, "BY": true, "type": "CC BY", "SA": false, "url": "https://creativecommons.org/licenses/by/4.0/"}], "ref": {"aims_scope": "https://epress.lib.uts.edu.au/journals/index.php/mcs/about/editorialPolicies#focusAndScope", "journal": "https://epress.lib.uts.edu.au/journals/index.php/mcs", "oa_statement": "https://epress.lib.uts.edu.au/journals/index.php/mcs/about/editorialPolicies#openAccessPolicy", "author_instructions": "https://epress.lib.uts.edu.au/journals/index.php/mcs/about/submissions#authorGuidelines", "license_terms": "https://epress.lib.uts.edu.au/journals/index.php/mcs/about/submissions#copyrightNotice"}, "oa_start": 2013, "apc": {"has_apc": false, "url": "https://epress.lib.uts.edu.au/journals/index.php/mcs"}, "other_charges": {"has_other_charges": false}, "publication_time_weeks": 9, "deposit_policy": {"service": ["Sherpa/Romeo"], "url": "https://v2.sherpa.ac.uk/id/publication/27499", "has_policy": true}, "publisher": {"country": "AU", "name": "UTS ePRESS"}, "boai": true, "waiver": {"has_waiver": false}}, "admin": {"ticked": true, "seal": true}, "created_date": "2014-10-02T16:36:18Z", "id": "a82bf9c2802f491f93181d1585559c03"},
{"last_updated": "2021-04-29T13:15:36Z", "bibjson": {"editorial": {"review_process": ["Peer review"], "review_url": "http://journals.pu.if.ua/index.php/pcss/about/editorialPolicies#peerReviewProcess", "board_url": "http://journals.pu.if.ua/index.php/pcss/about/editorialTeam"}, "pid_scheme": {"scheme": ["DOI"], "has_pid_scheme": true}, "copyright": {"author_retains": true, "url": "http://journals.pu.if.ua/index.php/pcss/about/submissions#copyrightNotice"}, "keywords": ["solid state", "condensed matter physics", "material science", "chemistry engineering"], "plagiarism": {"detection": false}, "subject": [{"code": "QC1-999", "scheme": "LCC", "term": "Physics"}], "eissn": "2309-8589", "language": ["EN", "UK"], "title": "\u0424\u0456\u0437\u0438\u043a\u0430 \u0456 \u0445\u0456\u043c\u0456\u044f \u0442\u0432\u0435\u0440\u0434\u043e\u0433\u043e \u0442\u0456\u043b\u0430", "pissn": "1729-4428", "article": {"license_display": ["No"]}, "preservation": {"has_preservation": true, "national_library": ["http://www.nbuv.gov.ua/node/1087"], "url": "http://journals.pu.if.ua/index.php/pcss/about/editorialPolicies#archiving"}, "license": [{"NC": false, "ND": false, "BY": true, "type": "CC BY", "SA": false}], "ref": {"aims_scope": "http://journals.pu.if.ua/index.php/pcss/about/editorialPolicies#focusAndScope", "journal": "http://journals.pu.if.ua/index.php/pcss/index", "oa_statement": "http://journals.pu.if.ua/index.php/pcss/about/editorialPolicies#openAccessPolicy", "author_instructions": "http://journals.pu.if.ua/index.php/pcss/about/submissions#authorGuidelines", "license_terms": "http://journals.pu.if.ua/index.php/pcss/about/submissions#copyrightNotice"}, "oa_start": 2000, "alternative_title": "Physics and Chemistry of Solid State", "apc": {"has_apc": false, "url": "http://journals.pu.if.ua/index.php/pcss/about/editorialPolicies#custom-2"}, "other_charges": {"has_other_charges": false, "url": "http://journals.pu.if.ua/index.php/pcss/about/editorialPolicies#custom-2"}, "publication_time_weeks": 12, "deposit_policy": {"has_policy": false}, "publisher": {"country": "UA", "name": "Vasyl Stefanyk Precarpathian National University"}, "boai": true, "waiver": {"has_waiver": false}}, "admin": {"ticked": true, "seal": false}, "id": "61dcea2b5b5f43e9a90185c4bae7af8c", "created_date": "2018-08-23T15:30:32Z"},
{"last_updated": "2021-04-29T13:15:36Z", "bibjson": {"editorial": {"review_process": ["Double blind peer review"], "review_url": "http://www.hum.unrc.edu.ar/ojs/index.php/tefros/about/editorialPolicies#peerReviewProcess", "board_url": "http://www.hum.unrc.edu.ar/ojs/index.php/tefros/about/editorialTeam"}, "pid_scheme": {"has_pid_scheme": false}, "copyright": {"author_retains": true, "url": "http://hum.unrc.edu.ar/ojs/index.php/tefros/about/submissions#copyrightNotice"}, "keywords": ["etnohistory", "indigenous"], "plagiarism": {"detection": false}, "subject": [{"code": "F1201-3799", "scheme": "LCC", "term": "Latin America. Spanish America"}, {"code": "HN1-995", "scheme": "LCC", "term": "Social history and conditions. Social problems. Social reform"}], "eissn": "1669-726X", "language": ["PT", "ES"], "title": "Revista TEFROS", "article": {"license_display_example_url": "http://www.hum.unrc.edu.ar/ojs/index.php/tefros/article/view/628/596", "license_display": ["Embed"]}, "institution": {"name": "Universidad Nacional de R\u00edo Cuarto, Taller de Etnohistoria de la Frontera Sur"}, "preservation": {"has_preservation": false}, "license": [{"NC": true, "ND": false, "BY": true, "type": "CC BY-NC-SA", "SA": true}], "ref": {"aims_scope": "http://www.hum.unrc.edu.ar/ojs/index.php/tefros/about/editorialPolicies#focusAndScope", "journal": "http://hum.unrc.edu.ar/ojs/index.php/tefros/index", "oa_statement": "http://hum.unrc.edu.ar/ojs/index.php/tefros/about/editorialPolicies#openAccessPolicy", "author_instructions": "http://www.hum.unrc.edu.ar/ojs/index.php/tefros/about/submissions#authorGuidelines", "license_terms": "http://www.hum.unrc.edu.ar/ojs/index.php/tefros/index"}, "oa_start": 2005, "apc": {"has_apc": false, "url": "http://hum.unrc.edu.ar/ojs/index.php/tefros/about/editorialPolicies#openAccessPolicy"}, "other_charges": {"has_other_charges": false, "url": "http://hum.unrc.edu.ar/ojs/index.php/tefros/about/editorialPolicies#openAccessPolicy"}, "publication_time_weeks": 20, "deposit_policy": {"has_policy": false}, "publisher": {"country": "AR", "name": "Universidad Nacional de R\u00edo Cuarto"}, "boai": true, "waiver": {"has_waiver": false}}, "admin": {"ticked": true, "seal": false}, "id": "093da7b36cee46789cd037433cb9cc0f", "created_date": "2018-08-31T17:34:34Z"},
{"last_updated": "2021-04-29T13:15:36Z", "bibjson": {"copyright": {"author_retains": true, "url": "https://ojs.tdmu.edu.ua/index.php/pharm-chas/about/submissions#copyrightNotice"}, "keywords": ["pharmacy", "pharmacoeconomics", "phytochemistry", "synthesis", "pharmacodynamics", "pharmacokinetics"], "subject": [{"code": "RS1-441", "scheme": "LCC", "term": "Pharmacy and materia medica"}], "language": ["EN", "UK"], "title": "\u0424\u0430\u0440\u043c\u0430\u0446\u0435\u0432\u0442\u0438\u0447\u043d\u0438\u0439 \u0447\u0430\u0441\u043e\u043f\u0438\u0441", "institution": {"name": "Ternopil State Medical University"}, "preservation": {"has_preservation": true, "national_library": ["http://www.irbis-nbuv.gov.ua/cgi-bin/irbis_nbuv/cgiirbis_64.exe?Z21ID=&I21DBN=UJRN&P21DBN=UJRN&S21STN=1&S21REF=10&S21FMT=juu_all&C21COM=S&S21CNR=20&S21P01=0&S21P02=0&S21P03=PREF=&S21COLORTERMS=0&S21STR=Phch"], "url": "https://ojs.tdmu.edu.ua/index.php/pharm-chas/about/editorialPolicies#archiving"}, "ref": {"aims_scope": "https://ojs.tdmu.edu.ua/index.php/pharm-chas/about/editorialPolicies#focusAndScope", "journal": "https://ojs.tdmu.edu.ua/index.php/pharm-chas", "oa_statement": "https://ojs.tdmu.edu.ua/index.php/pharm-chas/about/editorialPolicies#openAccessPolicy", "author_instructions": "https://ojs.tdmu.edu.ua/index.php/pharm-chas/about/submissions#authorGuidelines", "license_terms": "https://ojs.tdmu.edu.ua/index.php/pharm-chas/about/submissions#copyrightNotice"}, "alternative_title": "Pharmaceutical Review", "apc": {"has_apc": true, "max": [{"price": 1800, "currency": "UAH"}], "url": "https://ojs.tdmu.edu.ua/index.php/pharm-chas/about/submissions#authorFees"}, "other_charges": {"has_other_charges": false, "url": "https://ojs.tdmu.edu.ua/index.php/pharm-chas/about/submissions#authorGuidelines"}, "publication_time_weeks": 12, "deposit_policy": {"has_policy": false}, "boai": true, "editorial": {"review_process": ["Double blind peer review"], "review_url": "https://ojs.tdmu.edu.ua/index.php/pharm-chas/about/editorialPolicies#peerReviewProcess", "board_url": "https://ojs.tdmu.edu.ua/index.php/pharm-chas/about/editorialTeam"}, "pid_scheme": {"scheme": ["DOI"], "has_pid_scheme": true}, "plagiarism": {"detection": false}, "eissn": "2414-9926", "pissn": "2312-0967", "article": {"license_display": ["No"]}, "license": [{"NC": false, "ND": false, "BY": true, "type": "CC BY", "SA": false}], "oa_start": 2007, "publisher": {"country": "UA", "name": "Ukrmedknyha Publishing House"}, "waiver": {"has_waiver": false}}, "admin": {"ticked": true, "seal": false}, "id": "2f3357c422ba4c23ae4ea0e5e44cab77", "created_date": "2018-08-27T17:39:23Z"},
{"last_updated": "2021-04-29T13:15:36Z", "bibjson": {"editorial": {"review_process": ["Double blind peer review"], "review_url": "https://cerac.unlpam.edu.ar/index.php/anclajes/about/editorialPolicies#peerReviewProcess", "board_url": "https://cerac.unlpam.edu.ar/index.php/anclajes/about/editorialTeam"}, "pid_scheme": {"has_pid_scheme": false}, "copyright": {"author_retains": true, "url": "https://cerac.unlpam.edu.ar/index.php/anclajes/about/submissions#copyrightNotice"}, "keywords": ["literature", "literary criticism", "discourse analysis", "latin american literature", "argentine literature", "literary history"], "plagiarism": {"detection": true, "url": "https://cerac.unlpam.edu.ar/index.php/anclajes/about/editorialPolicies#custom-2"}, "subject": [{"code": "P1-1091", "scheme": "LCC", "term": "Philology. Linguistics"}, {"code": "PQ1-3999", "scheme": "LCC", "term": "French literature - Italian literature - Spanish literature - Portuguese literature"}], "eissn": "1851-4669", "language": ["ES"], "title": "Anclajes", "pissn": "0329-3807", "article": {"license_display_example_url": "https://cerac.unlpam.edu.ar/index.php/anclajes/article/view/2732/3231", "license_display": ["Embed"]}, "institution": {"name": "Instituto de Investigaciones Literarias y Discursivas"}, "preservation": {"has_preservation": false}, "license": [{"NC": true, "ND": false, "BY": true, "type": "CC BY-NC-SA", "SA": true}], "ref": {"aims_scope": "https://cerac.unlpam.edu.ar/index.php/anclajes/about/editorialPolicies#focusAndScope", "journal": "https://cerac.unlpam.edu.ar/index.php/anclajes", "oa_statement": "https://cerac.unlpam.edu.ar/index.php/anclajes/about/editorialPolicies#openAccessPolicy", "author_instructions": "https://cerac.unlpam.edu.ar/index.php/anclajes/about/submissions#authorGuidelines", "license_terms": "https://cerac.unlpam.edu.ar/index.php/anclajes/about/submissions#copyrightNotice"}, "oa_start": 2010, "apc": {"has_apc": false, "url": "https://cerac.unlpam.edu.ar/index.php/anclajes/about/submissions#copyrightNotice"}, "other_charges": {"has_other_charges": false, "url": "https://cerac.unlpam.edu.ar/index.php/anclajes/about/submissions#copyrightNotice"}, "publication_time_weeks": 48, "deposit_policy": {"has_policy": false}, "publisher": {"country": "AR", "name": "Universidad Nacional de La Pampa"}, "boai": true, "waiver": {"has_waiver": false}}, "admin": {"ticked": true, "seal": false}, "id": "2ff9a11c1d1748bb94586a2ba5617c85", "created_date": "2018-11-06T17:51:27Z"},
{"last_updated": "2021-04-29T13:19:04Z", "bibjson": {"editorial": {"review_process": ["Blind peer review"], "review_url": "http://www.keaipublishing.com/en/journals/translational-medicine-of-aging/guide-for-authors/#1267247", "board_url": "http://www.keaipublishing.com/en/journals/translational-medicine-of-aging/editorial-board/"}, "pid_scheme": {"scheme": ["DOI"], "has_pid_scheme": true}, "copyright": {"author_retains": false}, "keywords": ["aging", "senescence", "lifespan", "metabolism", "genomic maintenance", "dna repair"], "plagiarism": {"detection": true, "url": "https://www.elsevier.com/editors/perk/plagiarism-complaints"}, "subject": [{"code": "R", "scheme": "LCC", "term": "Medicine"}], "eissn": "2468-5011", "language": ["EN"], "title": "Translational Medicine of Aging", "article": {"license_display_example_url": "https://www.sciencedirect.com/science/article/pii/S2468501117300251", "license_display": ["Embed"]}, "preservation": {"has_preservation": true, "service": ["CLOCKSS", "Portico"], "national_library": ["Koninklijke Bibliotheek"], "url": "https://www.elsevier.com/about/our-business/policies/digital-archive"}, "license": [{"NC": false, "ND": false, "BY": true, "type": "CC BY", "SA": false, "url": "https://creativecommons.org/licenses/by/4.0/"}, {"NC": true, "ND": true, "BY": true, "type": "CC BY-NC-ND", "SA": false, "url": "https://creativecommons.org/licenses/by-nc-nd/4.0/"}], "ref": {"aims_scope": "http://www.keaipublishing.com/en/journals/translational-medicine-of-aging/", "journal": "http://www.keaipublishing.com/en/journals/translational-medicine-of-aging/", "oa_statement": "http://www.keaipublishing.com/en/journals/translational-medicine-of-aging/open-access-journal/", "author_instructions": "http://www.keaipublishing.com/en/journals/translational-medicine-of-aging/guide-for-authors/", "license_terms": "http://www.keaipublishing.com/en/journals/translational-medicine-of-aging/open-access-journal/"}, "oa_start": 2017, "apc": {"has_apc": true, "max": [{"price": 980, "currency": "USD"}], "url": "http://www.keaipublishing.com/en/journals/translational-medicine-of-aging/open-access-journal/"}, "other_charges": {"has_other_charges": false, "url": "http://www.keaipublishing.com/en/journals/translational-medicine-of-aging/open-access-journal/"}, "publication_time_weeks": 15, "deposit_policy": {"has_policy": false}, "publisher": {"country": "CN", "name": "KeAi Communications Co., Ltd."}, "boai": true, "waiver": {"has_waiver": false}}, "admin": {"ticked": true, "seal": false}, "created_date": "2018-11-12T13:51:36Z", "id": "b3b853b478c54f61ab31bdb0dae28757"},
{"last_updated": "2021-04-29T13:15:36Z", "bibjson": {"editorial": {"review_process": ["Peer review"], "review_url": "https://revistas.unc.edu.ar/index.php/intersticios/about/editorialPolicies#peerReviewProcess", "board_url": "https://revistas.unc.edu.ar/index.php/intersticios/about/editorialTeam"}, "pid_scheme": {"has_pid_scheme": false}, "copyright": {"author_retains": false}, "keywords": ["history of ideas", "philosophy", "social sciences", "latin america"], "plagiarism": {"detection": false}, "subject": [{"code": "F1201-3799", "scheme": "LCC", "term": "Latin America. Spanish America"}, {"code": "H", "scheme": "LCC", "term": "Social Sciences"}], "eissn": "2250-6543", "language": ["PT", "ES"], "title": "Intersticios", "article": {"license_display": ["No"]}, "institution": {"name": "Universidad Nacional de C\u00f3rdoba, Facultad de Filosof\u00eda y Humanidades, Centro de Investigaciones"}, "preservation": {"has_preservation": false}, "license": [{"NC": true, "ND": false, "BY": true, "type": "CC BY-NC-SA", "SA": true}], "ref": {"aims_scope": "https://revistas.unc.edu.ar/index.php/intersticios/about/editorialPolicies#focusAndScope", "journal": "https://revistas.unc.edu.ar/index.php/intersticios", "oa_statement": "https://revistas.unc.edu.ar/index.php/intersticios/about/editorialPolicies#openAccessPolicy", "author_instructions": "https://revistas.unc.edu.ar/index.php/intersticios/about/submissions#authorGuidelines", "license_terms": "https://revistas.unc.edu.ar/index.php/intersticios/about/editorialPolicies#openAccessPolicy"}, "oa_start": 2012, "alternative_title": "Intersticios de la Pol\u00edtica y la Cultura: Intervenciones Latinoamericanas", "apc": {"has_apc": false, "url": "https://revistas.unc.edu.ar/index.php/intersticios/about/editorialPolicies#openAccessPolicy"}, "other_charges": {"has_other_charges": false, "url": "https://revistas.unc.edu.ar/index.php/intersticios/about/editorialPolicies#openAccessPolicy"}, "publication_time_weeks": 7, "deposit_policy": {"has_policy": false}, "publisher": {"country": "AR", "name": "Universidad Nacional de C\u00f3rdoba"}, "boai": true, "waiver": {"has_waiver": false}}, "admin": {"ticked": true, "seal": false}, "id": "3a66d2ab9e0b421a905930421a7eb222", "created_date": "2018-11-08T15:51:44Z"},
{"last_updated": "2021-04-29T13:15:36Z", "bibjson": {"editorial": {"review_process": ["Peer review"], "review_url": "http://www.scientificexploration.org/journal-submissions", "board_url": "http://www.scientificexploration.org/journal"}, "pid_scheme": {"scheme": ["DOI"], "has_pid_scheme": true}, "copyright": {"author_retains": true, "url": "http://www.scientificexploration.org/journal"}, "keywords": ["scientific anomalies", "philosophy of science", "paraspychology"], "plagiarism": {"detection": true, "url": "http://www.scientificexploration.org/journal"}, "subject": [{"code": "BD10-701", "scheme": "LCC", "term": "Speculative philosophy"}], "language": ["EN"], "title": "Journal of Scientific Exploration", "pissn": "0892-3310", "article": {"license_display_example_url": "http://www.scientificexploration.org/journal-library", "license_display": ["Embed"]}, "institution": {"name": "Society for Scientific Exploration"}, "preservation": {"has_preservation": false}, "license": [{"NC": true, "ND": false, "BY": true, "type": "CC BY-NC", "SA": false}], "ref": {"aims_scope": "http://www.scientificexploration.org/journal", "journal": "http://www.scientificexploration.org/journal-library", "oa_statement": "http://www.scientificexploration.org/journal-library", "author_instructions": "http://www.scientificexploration.org/journal-submissions", "license_terms": "https://www.scientificexploration.org/docs/JSE_Ethics_Policy.pdf"}, "oa_start": 2018, "apc": {"has_apc": false, "url": "http://www.scientificexploration.org/journal-library"}, "other_charges": {"has_other_charges": false, "url": "http://www.scientificexploration.org/journal-library"}, "publication_time_weeks": 32, "deposit_policy": {"service": ["Sherpa/Romeo"], "has_policy": true}, "publisher": {"country": "US", "name": "SSE"}, "boai": true, "waiver": {"has_waiver": false}}, "admin": {"ticked": true, "seal": false}, "id": "1a2812cdd9644ac2b8ecf626e4216479", "created_date": "2018-12-16T22:27:11Z"},
{"last_updated": "2021-07-16T11:47:53Z", "bibjson": {"copyright": {"author_retains": true, "url": "https://vestnik.donstu.ru/jour/about/submissions#copyrightNotice"}, "keywords": ["mechanics", "mathematics", "simulation dynamics", "machine building", "friction and wear", "automation of technological processes"], "subject": [{"code": "TA349-359", "scheme": "LCC", "term": "Mechanics of engineering. Applied mechanics"}], "language": ["RU"], "title": "\u0412\u0435\u0441\u0442\u043d\u0438\u043a \u0414\u043e\u043d\u0441\u043a\u043e\u0433\u043e \u0433\u043e\u0441\u0443\u0434\u0430\u0440\u0441\u0442\u0432\u0435\u043d\u043d\u043e\u0433\u043e \u0442\u0435\u0445\u043d\u0438\u0447\u0435\u0441\u043a\u043e\u0433\u043e \u0443\u043d\u0438\u0432\u0435\u0440\u0441\u0438\u0442\u0435\u0442\u0430", "preservation": {"has_preservation": true, "service": ["elibrary.ru"], "url": "https://elibrary.ru/title_about.asp?id=8483"}, "ref": {"aims_scope": "https://vestnik.donstu.ru/jour/about/editorialPolicies#focusAndScope", "journal": "https://vestnik.donstu.ru", "oa_statement": "https://vestnik.donstu.ru/jour/about/editorialPolicies#openAccessPolicy", "author_instructions": "https://vestnik.donstu.ru/jour/about/submissions", "license_terms": "https://vestnik.donstu.ru/jour/about/submissions#copyrightNotice"}, "alternative_title": "Vestnik of Don State Technical University", "apc": {"has_apc": false, "url": "https://vestnik.donstu.ru/jour/about/editorialPolicies#custom-4"}, "other_charges": {"has_other_charges": false}, "publication_time_weeks": 4, "deposit_policy": {"has_policy": false}, "is_replaced_by": ["2687-1653"], "boai": true, "editorial": {"review_process": ["Double blind peer review"], "review_url": "https://vestnik.donstu.ru/jour/about/editorialPolicies#custom-0", "board_url": "https://vestnik.donstu.ru/jour/pages/view/EditorialC"}, "pid_scheme": {"scheme": ["DOI"], "has_pid_scheme": true}, "plagiarism": {"detection": true, "url": "https://vestnik.donstu.ru/jour/about/editorialPolicies#custom-6"}, "eissn": "1992-6006", "pissn": "1992-5980", "article": {"license_display_example_url": "https://vestnik.donstu.ru/jour/article/view/1343/1326", "license_display": ["Embed"]}, "license": [{"NC": false, "ND": false, "BY": true, "type": "CC BY", "SA": false, "url": "https://creativecommons.org/licenses/by/4.0/"}], "oa_start": 2006, "publisher": {"country": "RU", "name": "Don State Technical University"}, "waiver": {"has_waiver": false}}, "admin": {"ticked": true, "seal": false}, "created_date": "2018-12-07T12:19:11Z", "id": "ffa2368ed10748e5b7ec2ee930a30142"},
{"last_updated": "2021-04-29T13:15:36Z", "bibjson": {"copyright": {"author_retains": true, "url": "https://www.ophthalmosurgery.ru/jour/about/submissions#copyrightNotice"}, "keywords": ["cataract", "iol implantation", "vitreoretinal surgery", "choroid melanoma", "macular edema", "glaucoma"], "subject": [{"code": "RE1-994", "scheme": "LCC", "term": "Ophthalmology"}], "language": ["RU"], "title": "\u041e\u0444\u0442\u0430\u043b\u044c\u043c\u043e\u0445\u0438\u0440\u0443\u0440\u0433\u0438\u044f", "institution": {"name": "Russian Society of Ophthalmologists"}, "preservation": {"has_preservation": true, "service": ["elibrary.ru"], "url": "https://elibrary.ru/title_about.asp?id=8970"}, "ref": {"aims_scope": "https://www.ophthalmosurgery.ru/jour/about/editorialPolicies#focusAndScope", "journal": "https://www.ophthalmosurgery.ru/jour", "oa_statement": "https://www.ophthalmosurgery.ru/jour/about/editorialPolicies#openAccessPolicy", "author_instructions": "https://www.ophthalmosurgery.ru/jour/about/submissions", "license_terms": "https://www.ophthalmosurgery.ru/jour/about/submissions#copyrightNotice"}, "alternative_title": "Fyodorov Journal of Ophthalmic Surgery", "apc": {"has_apc": false, "url": "https://www.ophthalmosurgery.ru/jour/about/editorialPolicies#custom-4"}, "other_charges": {"has_other_charges": false, "url": "https://www.ophthalmosurgery.ru/jour/about/editorialPolicies#custom-4"}, "publication_time_weeks": 8, "deposit_policy": {"has_policy": false}, "boai": true, "editorial": {"review_process": ["Blind peer review"], "review_url": "https://www.ophthalmosurgery.ru/jour/about/editorialPolicies#custom-0", "board_url": "https://www.ophthalmosurgery.ru/index.php/jour/pages/view/EditorialC"}, "pid_scheme": {"scheme": ["DOI"], "has_pid_scheme": true}, "plagiarism": {"detection": true, "url": "https://www.ophthalmosurgery.ru/jour/about/editorialPolicies#custom-6"}, "eissn": "2312-4970", "pissn": "0235-4160", "article": {"license_display_example_url": "https://www.ophthalmosurgery.ru/jour/article/view/434", "license_display": ["Embed"]}, "license": [{"NC": false, "ND": false, "BY": true, "type": "CC BY", "SA": false}], "oa_start": 2013, "publisher": {"country": "RU", "name": "Publishing house\u00a0\"Ophthalmology\""}, "waiver": {"has_waiver": false}}, "admin": {"ticked": true, "seal": false}, "id": "b8b36bc88e0543d1b44ac0cb509b964e", "created_date": "2019-01-18T10:37:20Z"}]