[dump] refactoring
This commit is contained in:
parent
2566b97138
commit
a175ac2c7f
|
@ -53,7 +53,7 @@ public class Utils {
|
|||
|
||||
return String
|
||||
.format(
|
||||
"%s|%s::%s", Constants.CONTEXT_ID, Constants.CONTEXT_NS_PREFIX,
|
||||
"%s::%s", Constants.CONTEXT_NS_PREFIX,
|
||||
DHPUtils.md5(id));
|
||||
}
|
||||
|
||||
|
|
|
@ -1,10 +1,10 @@
|
|||
|
||||
package eu.dnetlib.dhp.oa.graph.dump.csv;
|
||||
|
||||
import org.apache.commons.lang.StringUtils;
|
||||
|
||||
import java.io.Serializable;
|
||||
|
||||
import org.apache.commons.lang.StringUtils;
|
||||
|
||||
/**
|
||||
* @author miriam.baglioni
|
||||
* @Date 10/05/23
|
||||
|
@ -13,8 +13,8 @@ public class Constants implements Serializable {
|
|||
public final static String SEP = "\t";
|
||||
|
||||
public static final String addQuotes(String id) {
|
||||
if(StringUtils.isNotEmpty(id))
|
||||
return "\"" + id + "\"";
|
||||
// if (StringUtils.isNotEmpty(id))
|
||||
// return "\"" + id + "\"";
|
||||
return id;
|
||||
}
|
||||
}
|
||||
|
|
|
@ -347,12 +347,15 @@ public class SparkDumpResults implements Serializable {
|
|||
}
|
||||
|
||||
private static String removeBreaks(String input) {
|
||||
return input.replace("\n", " ")
|
||||
if (StringUtils.isNotEmpty(input))
|
||||
return input
|
||||
.replace("\n", " ")
|
||||
.replace("\t", " ")
|
||||
.replace("\r", " ")
|
||||
.replace("\\\"", " ")
|
||||
.replace("\"", " ")
|
||||
;
|
||||
//.replace("\\", " ")
|
||||
.replace("\"", " ");
|
||||
|
||||
return input;
|
||||
|
||||
}
|
||||
|
||||
|
|
|
@ -1,10 +1,10 @@
|
|||
|
||||
package eu.dnetlib.dhp.oa.graph.dump.csv.model;
|
||||
|
||||
import eu.dnetlib.dhp.oa.graph.dump.csv.Constants;
|
||||
|
||||
import java.io.Serializable;
|
||||
|
||||
import eu.dnetlib.dhp.oa.graph.dump.csv.Constants;
|
||||
|
||||
/**
|
||||
* @author miriam.baglioni
|
||||
* @Date 11/05/23
|
||||
|
|
|
@ -1,10 +1,10 @@
|
|||
|
||||
package eu.dnetlib.dhp.oa.graph.dump.csv.model;
|
||||
|
||||
import eu.dnetlib.dhp.oa.graph.dump.csv.Constants;
|
||||
|
||||
import java.io.Serializable;
|
||||
|
||||
import eu.dnetlib.dhp.oa.graph.dump.csv.Constants;
|
||||
|
||||
/**
|
||||
* @author miriam.baglioni
|
||||
* @Date 11/05/23
|
||||
|
|
|
@ -1,10 +1,10 @@
|
|||
|
||||
package eu.dnetlib.dhp.oa.graph.dump.csv.model;
|
||||
|
||||
import eu.dnetlib.dhp.oa.graph.dump.csv.Constants;
|
||||
|
||||
import java.io.Serializable;
|
||||
|
||||
import eu.dnetlib.dhp.oa.graph.dump.csv.Constants;
|
||||
|
||||
/**
|
||||
* @author miriam.baglioni
|
||||
* @Date 11/05/23
|
||||
|
|
|
@ -1,10 +1,10 @@
|
|||
|
||||
package eu.dnetlib.dhp.oa.graph.dump.csv.model;
|
||||
|
||||
import eu.dnetlib.dhp.oa.graph.dump.csv.Constants;
|
||||
|
||||
import java.io.Serializable;
|
||||
|
||||
import eu.dnetlib.dhp.oa.graph.dump.csv.Constants;
|
||||
|
||||
/**
|
||||
* @author miriam.baglioni
|
||||
* @Date 11/05/23
|
||||
|
|
|
@ -1,10 +1,10 @@
|
|||
|
||||
package eu.dnetlib.dhp.oa.graph.dump.csv.model;
|
||||
|
||||
import eu.dnetlib.dhp.oa.graph.dump.csv.Constants;
|
||||
|
||||
import java.io.Serializable;
|
||||
|
||||
import eu.dnetlib.dhp.oa.graph.dump.csv.Constants;
|
||||
|
||||
/**
|
||||
* @author miriam.baglioni
|
||||
* @Date 11/05/23
|
||||
|
|
|
@ -3,13 +3,13 @@ package eu.dnetlib.dhp.oa.graph.dump.csv.model;
|
|||
|
||||
import java.io.Serializable;
|
||||
|
||||
import eu.dnetlib.dhp.oa.graph.dump.csv.Constants;
|
||||
import org.apache.commons.lang.StringUtils;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonGetter;
|
||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||
import com.fasterxml.jackson.annotation.JsonSetter;
|
||||
|
||||
import eu.dnetlib.dhp.oa.graph.dump.csv.Constants;
|
||||
import eu.dnetlib.dhp.schema.oaf.Country;
|
||||
import eu.dnetlib.dhp.schema.oaf.StructuredProperty;
|
||||
import sun.swing.StringUIClientPropertyKey;
|
||||
|
@ -38,7 +38,6 @@ public class CSVResult implements Serializable {
|
|||
this.id = Constants.addQuotes(id);
|
||||
}
|
||||
|
||||
|
||||
public String getType() {
|
||||
return type;
|
||||
}
|
||||
|
@ -52,8 +51,7 @@ public class CSVResult implements Serializable {
|
|||
}
|
||||
|
||||
public void setTitle(String title) {
|
||||
this.title =
|
||||
Constants.addQuotes(title);
|
||||
this.title = Constants.addQuotes(title);
|
||||
}
|
||||
|
||||
public String getDescription() {
|
||||
|
@ -61,8 +59,7 @@ public class CSVResult implements Serializable {
|
|||
}
|
||||
|
||||
public void setDescription(String description) {
|
||||
this.description =
|
||||
Constants.addQuotes(description);
|
||||
this.description = Constants.addQuotes(description);
|
||||
}
|
||||
|
||||
public String getAccessright() {
|
||||
|
|
|
@ -18,7 +18,7 @@ import org.apache.http.entity.ContentType;
|
|||
import org.apache.http.entity.InputStreamEntity;
|
||||
import org.apache.http.entity.StringEntity;
|
||||
import org.apache.http.impl.client.CloseableHttpClient;
|
||||
//import org.apache.http.impl.client.HttpClients;
|
||||
// import org.apache.http.impl.client.HttpClients;
|
||||
import org.apache.http.util.EntityUtils;
|
||||
import org.jetbrains.annotations.NotNull;
|
||||
|
||||
|
@ -108,11 +108,11 @@ public class ZenodoAPIClient implements Serializable {
|
|||
RequestBody body = RequestBody.create(json, MEDIA_TYPE_JSON);
|
||||
|
||||
Request request = new Request.Builder()
|
||||
.url(urlString)
|
||||
.addHeader("Content-Type", "application/json") // add request headers
|
||||
.addHeader("Authorization", "Bearer " + access_token)
|
||||
.post(body)
|
||||
.build();
|
||||
.url(urlString)
|
||||
.addHeader("Content-Type", "application/json") // add request headers
|
||||
.addHeader("Authorization", "Bearer " + access_token)
|
||||
.post(body)
|
||||
.build();
|
||||
|
||||
try (Response response = httpClient.newCall(request).execute()) {
|
||||
|
||||
|
@ -122,7 +122,8 @@ public class ZenodoAPIClient implements Serializable {
|
|||
// Get response body
|
||||
json = response.body().string();
|
||||
|
||||
eu.dnetlib.dhp.common.api.zenodo.ZenodoModel newSubmission = new Gson().fromJson(json, eu.dnetlib.dhp.common.api.zenodo.ZenodoModel.class);
|
||||
eu.dnetlib.dhp.common.api.zenodo.ZenodoModel newSubmission = new Gson()
|
||||
.fromJson(json, eu.dnetlib.dhp.common.api.zenodo.ZenodoModel.class);
|
||||
this.bucket = newSubmission.getLinks().getBucket();
|
||||
this.deposition_id = newSubmission.getId();
|
||||
|
||||
|
@ -132,7 +133,6 @@ public class ZenodoAPIClient implements Serializable {
|
|||
|
||||
}
|
||||
|
||||
|
||||
// public int uploadIS2(InputStream is, String fileName) throws IOException {
|
||||
//
|
||||
// final String crlf = "\r\n";
|
||||
|
@ -340,10 +340,10 @@ public class ZenodoAPIClient implements Serializable {
|
|||
RequestBody body = RequestBody.create(json, MEDIA_TYPE_JSON);
|
||||
|
||||
Request request = new Request.Builder()
|
||||
.url(urlString + "/" + deposition_id + "/actions/newversion")
|
||||
.addHeader(HttpHeaders.AUTHORIZATION, "Bearer " + access_token)
|
||||
.post(body)
|
||||
.build();
|
||||
.url(urlString + "/" + deposition_id + "/actions/newversion")
|
||||
.addHeader(HttpHeaders.AUTHORIZATION, "Bearer " + access_token)
|
||||
.post(body)
|
||||
.build();
|
||||
|
||||
try (Response response = httpClient.newCall(request).execute()) {
|
||||
|
||||
|
@ -445,11 +445,11 @@ public class ZenodoAPIClient implements Serializable {
|
|||
String url = urlBuilder.build().toString();
|
||||
|
||||
Request request = new Request.Builder()
|
||||
.url(url)
|
||||
.addHeader(HttpHeaders.CONTENT_TYPE, ContentType.APPLICATION_JSON.toString()) // add request headers
|
||||
.addHeader(HttpHeaders.AUTHORIZATION, "Bearer " + access_token)
|
||||
.get()
|
||||
.build();
|
||||
.url(url)
|
||||
.addHeader(HttpHeaders.CONTENT_TYPE, ContentType.APPLICATION_JSON.toString()) // add request headers
|
||||
.addHeader(HttpHeaders.AUTHORIZATION, "Bearer " + access_token)
|
||||
.get()
|
||||
.build();
|
||||
|
||||
try (Response response = httpClient.newCall(request).execute()) {
|
||||
|
||||
|
|
|
@ -7,8 +7,6 @@ import java.io.IOException;
|
|||
import java.io.InputStream;
|
||||
import java.nio.file.Files;
|
||||
|
||||
import eu.dnetlib.dhp.oa.zenodoapi.ZenodoAPIClient;
|
||||
import eu.dnetlib.dhp.oa.zenodoapi.MissingConceptDoiException;
|
||||
import org.apache.hadoop.conf.Configuration;
|
||||
import org.apache.hadoop.fs.*;
|
||||
import org.junit.jupiter.api.Assertions;
|
||||
|
@ -19,7 +17,8 @@ import org.junit.jupiter.api.Test;
|
|||
import com.google.gson.Gson;
|
||||
|
||||
import eu.dnetlib.dhp.oa.graph.dump.community.CommunityMap;
|
||||
|
||||
import eu.dnetlib.dhp.oa.zenodoapi.MissingConceptDoiException;
|
||||
import eu.dnetlib.dhp.oa.zenodoapi.ZenodoAPIClient;
|
||||
|
||||
@Disabled
|
||||
public class ZenodoUploadTest {
|
||||
|
@ -86,8 +85,7 @@ public class ZenodoUploadTest {
|
|||
|
||||
System.out.println(client.sendMretadata(metadata));
|
||||
|
||||
|
||||
//System.out.println(client.publish());
|
||||
// System.out.println(client.publish());
|
||||
|
||||
}
|
||||
|
||||
|
@ -138,7 +136,7 @@ public class ZenodoUploadTest {
|
|||
|
||||
}
|
||||
|
||||
//System.out.println(client.publish());
|
||||
// System.out.println(client.publish());
|
||||
|
||||
}
|
||||
|
||||
|
@ -146,7 +144,7 @@ public class ZenodoUploadTest {
|
|||
void testNewVersion2() throws Exception, MissingConceptDoiException {
|
||||
|
||||
ZenodoAPIClient client = new ZenodoAPIClient(URL_STRING,
|
||||
ACCESS_TOKEN);
|
||||
ACCESS_TOKEN);
|
||||
|
||||
client.newVersion("1210237");
|
||||
|
||||
|
@ -162,6 +160,7 @@ public class ZenodoUploadTest {
|
|||
// Assertions.assertEquals(202, client.publish());
|
||||
|
||||
}
|
||||
|
||||
@Test
|
||||
void readCommunityMap() throws IOException {
|
||||
LocalFileSystem fs = FileSystem.getLocal(new Configuration());
|
||||
|
|
|
@ -347,4 +347,12 @@ public class DumpResultTest {
|
|||
builder.append(((Node) (root.selectNodes("//description").get(0))).getText());
|
||||
System.out.println(builder.toString());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void provaReplacement(){
|
||||
String st = "\"Person in the sand " + "\"";
|
||||
System.out.println(st);
|
||||
|
||||
System.out.println(st.replace("\\\"", " "));
|
||||
}
|
||||
}
|
||||
|
|
Loading…
Reference in New Issue