D-Net
/
dhp-graph-dump
4
0
Fork 0
Code Issues Pull Requests 1 Projects Releases Wiki Activity

refactoring

This commit is contained in:
Miriam Baglioni 2023-07-01 11:07:41 +02:00
parent 2ac5c4a9ab
commit 3fba247c38
14 changed files with 477 additions and 422 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/SendToZenodoHDFS.java
View File

@ -4,14 +4,14 @@ package eu.dnetlib.dhp.oa.graph.dump;
import java.io.Serializable;
import java.util.Optional;
import eu.dnetlib.dhp.oa.zenodoapi.MissingConceptDoiException;
import eu.dnetlib.dhp.oa.zenodoapi.ZenodoAPIClient;
import org.apache.commons.io.IOUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.*;
import eu.dnetlib.dhp.application.ArgumentApplicationParser;
import eu.dnetlib.dhp.oa.graph.dump.exceptions.NoAvailableEntityTypeException;
import eu.dnetlib.dhp.oa.zenodoapi.MissingConceptDoiException;
import eu.dnetlib.dhp.oa.zenodoapi.ZenodoAPIClient;
public class SendToZenodoHDFS implements Serializable {

66
dump/src/main/java/eu/dnetlib/dhp/oa/zenodoapi/InputStreamRequestBody.java
View File

@ -1,5 +1,11 @@
package eu.dnetlib.dhp.oa.zenodoapi;
/**
* @author miriam.baglioni
* @Date 01/07/23
*/
/**
* @author miriam.baglioni
* @Date 01/07/23
@ -16,41 +22,41 @@ import okio.Source;
public class InputStreamRequestBody extends RequestBody {
private final InputStream inputStream;
private final MediaType mediaType;
private final long lenght;
private final InputStream inputStream;
private final MediaType mediaType;
private final long lenght;
public static RequestBody create(final MediaType mediaType, final InputStream inputStream, final long len) {
public static RequestBody create(final MediaType mediaType, final InputStream inputStream, final long len) {
return new InputStreamRequestBody(inputStream, mediaType, len);
}
return new InputStreamRequestBody(inputStream, mediaType, len);
}
private InputStreamRequestBody(InputStream inputStream, MediaType mediaType, long len) {
this.inputStream = inputStream;
this.mediaType = mediaType;
this.lenght = len;
}
private InputStreamRequestBody(InputStream inputStream, MediaType mediaType, long len) {
this.inputStream = inputStream;
this.mediaType = mediaType;
this.lenght = len;
}
@Override
public MediaType contentType() {
return mediaType;
}
@Override
public MediaType contentType() {
return mediaType;
}
@Override
public long contentLength() {
@Override
public long contentLength() {
return lenght;
return lenght;
}
}
@Override
public void writeTo(BufferedSink sink) throws IOException {
Source source = null;
try {
source = Okio.source(inputStream);
sink.writeAll(source);
} finally {
Util.closeQuietly(source);
}
}
}
@Override
public void writeTo(BufferedSink sink) throws IOException {
Source source = null;
try {
source = Okio.source(inputStream);
sink.writeAll(source);
} finally {
Util.closeQuietly(source);
}
}
}

4
dump/src/main/java/eu/dnetlib/dhp/oa/zenodoapi/ZenodoAPIClient.java
View File

@ -8,8 +8,6 @@ import java.net.URL;
import java.nio.charset.StandardCharsets;
import java.util.concurrent.TimeUnit;
import eu.dnetlib.dhp.oa.zenodoapi.model.ZenodoModel;
import eu.dnetlib.dhp.oa.zenodoapi.model.ZenodoModelList;
import org.apache.http.HttpHeaders;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
@ -26,6 +24,8 @@ import org.jetbrains.annotations.NotNull;
import com.google.gson.Gson;
import eu.dnetlib.dhp.oa.zenodoapi.model.ZenodoModel;
import eu.dnetlib.dhp.oa.zenodoapi.model.ZenodoModelList;
import okhttp3.*;
public class ZenodoAPIClient implements Serializable {

15
dump/src/main/java/eu/dnetlib/dhp/oa/zenodoapi/model/Community.java
View File

@ -1,3 +1,4 @@
package eu.dnetlib.dhp.oa.zenodoapi.model;
/**
@ -5,13 +6,13 @@ package eu.dnetlib.dhp.oa.zenodoapi.model;
* @Date 01/07/23
*/
public class Community {
private String identifier;
private String identifier;
public String getIdentifier() {
return identifier;
}
public String getIdentifier() {
return identifier;
}
public void setIdentifier(String identifier) {
this.identifier = identifier;
}
public void setIdentifier(String identifier) {
this.identifier = identifier;
}
}

70
dump/src/main/java/eu/dnetlib/dhp/oa/zenodoapi/model/Creator.java
View File

@ -1,3 +1,4 @@
package eu.dnetlib.dhp.oa.zenodoapi.model;
/**
@ -5,47 +6,46 @@ package eu.dnetlib.dhp.oa.zenodoapi.model;
* @Date 01/07/23
*/
public class Creator {
private String affiliation;
private String name;
private String orcid;
private String affiliation;
private String name;
private String orcid;
public String getAffiliation() {
return affiliation;
}
public String getAffiliation() {
return affiliation;
}
public void setAffiliation(String affiliation) {
this.affiliation = affiliation;
}
public void setAffiliation(String affiliation) {
this.affiliation = affiliation;
}
public String getName() {
return name;
}
public String getName() {
return name;
}
public void setName(String name) {
this.name = name;
}
public void setName(String name) {
this.name = name;
}
public String getOrcid() {
return orcid;
}
public String getOrcid() {
return orcid;
}
public void setOrcid(String orcid) {
this.orcid = orcid;
}
public void setOrcid(String orcid) {
this.orcid = orcid;
}
public static Creator newInstance(String name, String affiliation, String orcid) {
Creator c = new Creator();
if (name != null) {
c.name = name;
}
if (affiliation != null) {
c.affiliation = affiliation;
}
if (orcid != null) {
c.orcid = orcid;
}
public static Creator newInstance(String name, String affiliation, String orcid) {
Creator c = new Creator();
if (name != null) {
c.name = name;
}
if (affiliation != null) {
c.affiliation = affiliation;
}
if (orcid != null) {
c.orcid = orcid;
}
return c;
}
return c;
}
}

62
dump/src/main/java/eu/dnetlib/dhp/oa/zenodoapi/model/File.java
View File

@ -1,5 +1,11 @@
package eu.dnetlib.dhp.oa.zenodoapi.model;
/**
* @author miriam.baglioni
* @Date 01/07/23
*/
/**
* @author miriam.baglioni
* @Date 01/07/23
@ -7,41 +13,41 @@ package eu.dnetlib.dhp.oa.zenodoapi.model;
import java.io.Serializable;
public class File implements Serializable {
private String checksum;
private String filename;
private long filesize;
private String id;
private String checksum;
private String filename;
private long filesize;
private String id;
public String getChecksum() {
return checksum;
}
public String getChecksum() {
return checksum;
}
public void setChecksum(String checksum) {
this.checksum = checksum;
}
public void setChecksum(String checksum) {
this.checksum = checksum;
}
public String getFilename() {
return filename;
}
public String getFilename() {
return filename;
}
public void setFilename(String filename) {
this.filename = filename;
}
public void setFilename(String filename) {
this.filename = filename;
}
public long getFilesize() {
return filesize;
}
public long getFilesize() {
return filesize;
}
public void setFilesize(long filesize) {
this.filesize = filesize;
}
public void setFilesize(long filesize) {
this.filesize = filesize;
}
public String getId() {
return id;
}
public String getId() {
return id;
}
public void setId(String id) {
this.id = id;
}
public void setId(String id) {
this.id = id;
}
}

30
dump/src/main/java/eu/dnetlib/dhp/oa/zenodoapi/model/Grant.java
View File

@ -1,5 +1,11 @@
package eu.dnetlib.dhp.oa.zenodoapi.model;
/**
* @author miriam.baglioni
* @Date 01/07/23
*/
/**
* @author miriam.baglioni
* @Date 01/07/23
@ -7,20 +13,20 @@ package eu.dnetlib.dhp.oa.zenodoapi.model;
import java.io.Serializable;
public class Grant implements Serializable {
private String id;
private String id;
public String getId() {
return id;
}
public String getId() {
return id;
}
public void setId(String id) {
this.id = id;
}
public void setId(String id) {
this.id = id;
}
public static Grant newInstance(String id) {
Grant g = new Grant();
g.id = id;
public static Grant newInstance(String id) {
Grant g = new Grant();
g.id = id;
return g;
}
return g;
}
}

134
dump/src/main/java/eu/dnetlib/dhp/oa/zenodoapi/model/Links.java
View File

@ -1,5 +1,11 @@
package eu.dnetlib.dhp.oa.zenodoapi.model;
/**
* @author miriam.baglioni
* @Date 01/07/23
*/
/**
* @author miriam.baglioni
* @Date 01/07/23
@ -8,88 +14,88 @@ import java.io.Serializable;
public class Links implements Serializable {
private String bucket;
private String bucket;
private String discard;
private String discard;
private String edit;
private String files;
private String html;
private String latest_draft;
private String latest_draft_html;
private String publish;
private String edit;
private String files;
private String html;
private String latest_draft;
private String latest_draft_html;
private String publish;
private String self;
private String self;
public String getBucket() {
return bucket;
}
public String getBucket() {
return bucket;
}
public void setBucket(String bucket) {
this.bucket = bucket;
}
public void setBucket(String bucket) {
this.bucket = bucket;
}
public String getDiscard() {
return discard;
}
public String getDiscard() {
return discard;
}
public void setDiscard(String discard) {
this.discard = discard;
}
public void setDiscard(String discard) {
this.discard = discard;
}
public String getEdit() {
return edit;
}
public String getEdit() {
return edit;
}
public void setEdit(String edit) {
this.edit = edit;
}
public void setEdit(String edit) {
this.edit = edit;
}
public String getFiles() {
return files;
}
public String getFiles() {
return files;
}
public void setFiles(String files) {
this.files = files;
}
public void setFiles(String files) {
this.files = files;
}
public String getHtml() {
return html;
}
public String getHtml() {
return html;
}
public void setHtml(String html) {
this.html = html;
}
public void setHtml(String html) {
this.html = html;
}
public String getLatest_draft() {
return latest_draft;
}
public String getLatest_draft() {
return latest_draft;
}
public void setLatest_draft(String latest_draft) {
this.latest_draft = latest_draft;
}
public void setLatest_draft(String latest_draft) {
this.latest_draft = latest_draft;
}
public String getLatest_draft_html() {
return latest_draft_html;
}
public String getLatest_draft_html() {
return latest_draft_html;
}
public void setLatest_draft_html(String latest_draft_html) {
this.latest_draft_html = latest_draft_html;
}
public void setLatest_draft_html(String latest_draft_html) {
this.latest_draft_html = latest_draft_html;
}
public String getPublish() {
return publish;
}
public String getPublish() {
return publish;
}
public void setPublish(String publish) {
this.publish = publish;
}
public void setPublish(String publish) {
this.publish = publish;
}
public String getSelf() {
return self;
}
public String getSelf() {
return self;
}
public void setSelf(String self) {
this.self = self;
}
}
public void setSelf(String self) {
this.self = self;
}
}

230
dump/src/main/java/eu/dnetlib/dhp/oa/zenodoapi/model/Metadata.java
View File

@ -1,5 +1,11 @@
package eu.dnetlib.dhp.oa.zenodoapi.model;
/**
* @author miriam.baglioni
* @Date 01/07/23
*/
/**
* @author miriam.baglioni
* @Date 01/07/23
@ -9,148 +15,148 @@ import java.util.List;
public class Metadata implements Serializable {
private String access_right;
private List<Community> communities;
private List<Creator> creators;
private String description;
private String doi;
private List<Grant> grants;
private List<String> keywords;
private String language;
private String license;
private PrereserveDoi prereserve_doi;
private String publication_date;
private List<String> references;
private List<RelatedIdentifier> related_identifiers;
private String title;
private String upload_type;
private String version;
private String access_right;
private List<Community> communities;
private List<Creator> creators;
private String description;
private String doi;
private List<Grant> grants;
private List<String> keywords;
private String language;
private String license;
private PrereserveDoi prereserve_doi;
private String publication_date;
private List<String> references;
private List<RelatedIdentifier> related_identifiers;
private String title;
private String upload_type;
private String version;
public String getUpload_type() {
return upload_type;
}
public String getUpload_type() {
return upload_type;
}
public void setUpload_type(String upload_type) {
this.upload_type = upload_type;
}
public void setUpload_type(String upload_type) {
this.upload_type = upload_type;
}
public String getVersion() {
return version;
}
public String getVersion() {
return version;
}
public void setVersion(String version) {
this.version = version;
}
public void setVersion(String version) {
this.version = version;
}
public String getAccess_right() {
return access_right;
}
public String getAccess_right() {
return access_right;
}
public void setAccess_right(String access_right) {
this.access_right = access_right;
}
public void setAccess_right(String access_right) {
this.access_right = access_right;
}
public List<Community> getCommunities() {
return communities;
}
public List<Community> getCommunities() {
return communities;
}
public void setCommunities(List<Community> communities) {
this.communities = communities;
}
public void setCommunities(List<Community> communities) {
this.communities = communities;
}
public List<Creator> getCreators() {
return creators;
}
public List<Creator> getCreators() {
return creators;
}
public void setCreators(List<Creator> creators) {
this.creators = creators;
}
public void setCreators(List<Creator> creators) {
this.creators = creators;
}
public String getDescription() {
return description;
}
public String getDescription() {
return description;
}
public void setDescription(String description) {
this.description = description;
}
public void setDescription(String description) {
this.description = description;
}
public String getDoi() {
return doi;
}
public String getDoi() {
return doi;
}
public void setDoi(String doi) {
this.doi = doi;
}
public void setDoi(String doi) {
this.doi = doi;
}
public List<Grant> getGrants() {
return grants;
}
public List<Grant> getGrants() {
return grants;
}
public void setGrants(List<Grant> grants) {
this.grants = grants;
}
public void setGrants(List<Grant> grants) {
this.grants = grants;
}
public List<String> getKeywords() {
return keywords;
}
public List<String> getKeywords() {
return keywords;
}
public void setKeywords(List<String> keywords) {
this.keywords = keywords;
}
public void setKeywords(List<String> keywords) {
this.keywords = keywords;
}
public String getLanguage() {
return language;
}
public String getLanguage() {
return language;
}
public void setLanguage(String language) {
this.language = language;
}
public void setLanguage(String language) {
this.language = language;
}
public String getLicense() {
return license;
}
public String getLicense() {
return license;
}
public void setLicense(String license) {
this.license = license;
}
public void setLicense(String license) {
this.license = license;
}
public PrereserveDoi getPrereserve_doi() {
return prereserve_doi;
}
public PrereserveDoi getPrereserve_doi() {
return prereserve_doi;
}
public void setPrereserve_doi(PrereserveDoi prereserve_doi) {
this.prereserve_doi = prereserve_doi;
}
public void setPrereserve_doi(PrereserveDoi prereserve_doi) {
this.prereserve_doi = prereserve_doi;
}
public String getPublication_date() {
return publication_date;
}
public String getPublication_date() {
return publication_date;
}
public void setPublication_date(String publication_date) {
this.publication_date = publication_date;
}
public void setPublication_date(String publication_date) {
this.publication_date = publication_date;
}
public List<String> getReferences() {
return references;
}
public List<String> getReferences() {
return references;
}
public void setReferences(List<String> references) {
this.references = references;
}
public void setReferences(List<String> references) {
this.references = references;
}
public List<RelatedIdentifier> getRelated_identifiers() {
return related_identifiers;
}
public List<RelatedIdentifier> getRelated_identifiers() {
return related_identifiers;
}
public void setRelated_identifiers(List<RelatedIdentifier> related_identifiers) {
this.related_identifiers = related_identifiers;
}
public void setRelated_identifiers(List<RelatedIdentifier> related_identifiers) {
this.related_identifiers = related_identifiers;
}
public String getTitle() {
return title;
}
public String getTitle() {
return title;
}
public void setTitle(String title) {
this.title = title;
}
public void setTitle(String title) {
this.title = title;
}
}

34
dump/src/main/java/eu/dnetlib/dhp/oa/zenodoapi/model/PrereserveDoi.java
View File

@ -1,5 +1,11 @@
package eu.dnetlib.dhp.oa.zenodoapi.model;
/**
* @author miriam.baglioni
* @Date 01/07/23
*/
/**
* @author miriam.baglioni
* @Date 01/07/23
@ -7,22 +13,22 @@ package eu.dnetlib.dhp.oa.zenodoapi.model;
import java.io.Serializable;
public class PrereserveDoi implements Serializable {
private String doi;
private String recid;
private String doi;
private String recid;
public String getDoi() {
return doi;
}
public String getDoi() {
return doi;
}
public void setDoi(String doi) {
this.doi = doi;
}
public void setDoi(String doi) {
this.doi = doi;
}
public String getRecid() {
return recid;
}
public String getRecid() {
return recid;
}
public void setRecid(String recid) {
this.recid = recid;
}
public void setRecid(String recid) {
this.recid = recid;
}
}

64
dump/src/main/java/eu/dnetlib/dhp/oa/zenodoapi/model/RelatedIdentifier.java
View File

@ -1,5 +1,11 @@
package eu.dnetlib.dhp.oa.zenodoapi.model;
/**
* @author miriam.baglioni
* @Date 01/07/23
*/
/**
* @author miriam.baglioni
* @Date 01/07/23
@ -7,40 +13,40 @@ package eu.dnetlib.dhp.oa.zenodoapi.model;
import java.io.Serializable;
public class RelatedIdentifier implements Serializable {
private String identifier;
private String relation;
private String resource_type;
private String scheme;
private String identifier;
private String relation;
private String resource_type;
private String scheme;
public String getIdentifier() {
return identifier;
}
public String getIdentifier() {
return identifier;
}
public void setIdentifier(String identifier) {
this.identifier = identifier;
}
public void setIdentifier(String identifier) {
this.identifier = identifier;
}
public String getRelation() {
return relation;
}
public String getRelation() {
return relation;
}
public void setRelation(String relation) {
this.relation = relation;
}
public void setRelation(String relation) {
this.relation = relation;
}
public String getResource_type() {
return resource_type;
}
public String getResource_type() {
return resource_type;
}
public void setResource_type(String resource_type) {
this.resource_type = resource_type;
}
public void setResource_type(String resource_type) {
this.resource_type = resource_type;
}
public String getScheme() {
return scheme;
}
public String getScheme() {
return scheme;
}
public void setScheme(String scheme) {
this.scheme = scheme;
}
}
public void setScheme(String scheme) {
this.scheme = scheme;
}
}

176
dump/src/main/java/eu/dnetlib/dhp/oa/zenodoapi/model/ZenodoModel.java
View File

@ -1,5 +1,11 @@
package eu.dnetlib.dhp.oa.zenodoapi.model;
/**
* @author miriam.baglioni
* @Date 01/07/23
*/
/**
* @author miriam.baglioni
* @Date 01/07/23
@ -9,113 +15,113 @@ import java.util.List;
public class ZenodoModel implements Serializable {
private String conceptrecid;
private String created;
private String conceptrecid;
private String created;
private List<File> files;
private String id;
private Links links;
private Metadata metadata;
private String modified;
private String owner;
private String record_id;
private String state;
private boolean submitted;
private String title;
private List<File> files;
private String id;
private Links links;
private Metadata metadata;
private String modified;
private String owner;
private String record_id;
private String state;
private boolean submitted;
private String title;
public String getConceptrecid() {
return conceptrecid;
}
public String getConceptrecid() {
return conceptrecid;
}
public void setConceptrecid(String conceptrecid) {
this.conceptrecid = conceptrecid;
}
public void setConceptrecid(String conceptrecid) {
this.conceptrecid = conceptrecid;
}
public String getCreated() {
return created;
}
public String getCreated() {
return created;
}
public void setCreated(String created) {
this.created = created;
}
public void setCreated(String created) {
this.created = created;
}
public List<File> getFiles() {
return files;
}
public List<File> getFiles() {
return files;
}
public void setFiles(List<File> files) {
this.files = files;
}
public void setFiles(List<File> files) {
this.files = files;
}
public String getId() {
return id;
}
public String getId() {
return id;
}
public void setId(String id) {
this.id = id;
}
public void setId(String id) {
this.id = id;
}
public Links getLinks() {
return links;
}
public Links getLinks() {
return links;
}
public void setLinks(Links links) {
this.links = links;
}
public void setLinks(Links links) {
this.links = links;
}
public Metadata getMetadata() {
return metadata;
}
public Metadata getMetadata() {
return metadata;
}
public void setMetadata(Metadata metadata) {
this.metadata = metadata;
}
public void setMetadata(Metadata metadata) {
this.metadata = metadata;
}
public String getModified() {
return modified;
}
public String getModified() {
return modified;
}
public void setModified(String modified) {
this.modified = modified;
}
public void setModified(String modified) {
this.modified = modified;
}
public String getOwner() {
return owner;
}
public String getOwner() {
return owner;
}
public void setOwner(String owner) {
this.owner = owner;
}
public void setOwner(String owner) {
this.owner = owner;
}
public String getRecord_id() {
return record_id;
}
public String getRecord_id() {
return record_id;
}
public void setRecord_id(String record_id) {
this.record_id = record_id;
}
public void setRecord_id(String record_id) {
this.record_id = record_id;
}
public String getState() {
return state;
}
public String getState() {
return state;
}
public void setState(String state) {
this.state = state;
}
public void setState(String state) {
this.state = state;
}
public boolean isSubmitted() {
return submitted;
}
public boolean isSubmitted() {
return submitted;
}
public void setSubmitted(boolean submitted) {
this.submitted = submitted;
}
public void setSubmitted(boolean submitted) {
this.submitted = submitted;
}
public String getTitle() {
return title;
}
public String getTitle() {
return title;
}
public void setTitle(String title) {
this.title = title;
}
}
public void setTitle(String title) {
this.title = title;
}
}

6
dump/src/main/java/eu/dnetlib/dhp/oa/zenodoapi/model/ZenodoModelList.java
View File

@ -1,5 +1,11 @@
package eu.dnetlib.dhp.oa.zenodoapi.model;
/**
* @author miriam.baglioni
* @Date 01/07/23
*/
/**
* @author miriam.baglioni
* @Date 01/07/23

4
dump/src/test/java/eu/dnetlib/dhp/oa/graph/dump/ZenodoUploadTest.java
View File

@ -7,7 +7,6 @@ import java.io.IOException;
import java.io.InputStream;
import java.nio.file.Files;
import eu.dnetlib.dhp.oa.zenodoapi.ZenodoAPIClient;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.*;
import org.junit.jupiter.api.Assertions;
@ -19,6 +18,7 @@ import com.google.gson.Gson;
import eu.dnetlib.dhp.common.api.MissingConceptDoiException;
import eu.dnetlib.dhp.oa.graph.dump.community.CommunityMap;
import eu.dnetlib.dhp.oa.zenodoapi.ZenodoAPIClient;
@Disabled
public class ZenodoUploadTest {
@ -157,7 +157,7 @@ public class ZenodoUploadTest {
@Test
void depositBigFile() throws MissingConceptDoiException, IOException {
ZenodoAPIClient client = new ZenodoAPIClient(URL_STRING,
ACCESS_TOKEN);
ACCESS_TOKEN);
Assertions.assertEquals(201, client.newDeposition());