spyros
/
dnet-hadoop
forked from D-Net/dnet-hadoop
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Changes to create properly directories for downloaded files

This commit is contained in:
Spyros Zoupanos 2020-09-19 22:04:42 +03:00
parent 03fb2b9e01
commit ed4e9f46d9
1 changed files with 47 additions and 39 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

86
dhp-workflows/dhp-usage-stats-update/src/main/java/eu/dnetlib/oa/graph/usagestats/export/SarcStats.java
View File

@ -142,7 +142,7 @@ public class SarcStats {
System.out.println("====> Created sarc_sushilogtmp_json_non_array_" + issn.replace("-", "_") + " table"); System.out.println("====> Created sarc_sushilogtmp_json_non_array_" + issn.replace("-", "_") + " table");
System.out.println("====> Creating sarc_sushilogtmp table"); System.out.println("====> Creating sarc_sushilogtmp table");
String create_sarc_sushilogtmp = "CREATE TABLE " + ConnectDB.getUsageStatsDBSchema() String create_sarc_sushilogtmp = "CREATE TABLE IF NOT EXISTS " + ConnectDB.getUsageStatsDBSchema()
+ ".sarc_sushilogtmp(source STRING, repository STRING, " + + ".sarc_sushilogtmp(source STRING, repository STRING, " +
"rid STRING, date STRING, metric_type STRING, count INT) clustered by (source) into 100 buckets stored as orc " "rid STRING, date STRING, metric_type STRING, count INT) clustered by (source) into 100 buckets stored as orc "
+ +
@ -150,26 +150,6 @@ public class SarcStats {
stmt.executeUpdate(create_sarc_sushilogtmp); stmt.executeUpdate(create_sarc_sushilogtmp);
System.out.println("====> Created sarc_sushilogtmp table"); System.out.println("====> Created sarc_sushilogtmp table");
// "INSERT INTO sushilog (source, repository, rid, date, metric_type, count) VALUES (?,?,?,?,?,?)");
// JSONObject itemPerformance = (JSONObject) jsonObjectRow.get("c:ItemPerformance");
// // for (Object perf : itemPerformance) {
// JSONObject performance = (JSONObject) itemPerformance;
// JSONObject periodObj = (JSONObject) performance.get("c:Period");
// String period = periodObj.get("c:Begin").toString();
// JSONObject instanceObj = (JSONObject) performance.get("c:Instance");
// String type = instanceObj.get("c:MetricType").toString();
// String count = instanceObj.get("c:Count").toString();
// // System.out.println(rid + " : " + period + " : " + count);
//
// preparedStatement.setString(1, "SARC-OJS");
// preparedStatement.setString(2, issn);
// // preparedStatement.setString(2, url);
// preparedStatement.setString(3, rid);
// preparedStatement.setString(4, period);
// preparedStatement.setString(5, type);
// preparedStatement.setInt(6, Integer.parseInt(count));
System.out.println("====> Inserting to sarc_sushilogtmp table (sarc_sushilogtmp_json_array)"); System.out.println("====> Inserting to sarc_sushilogtmp table (sarc_sushilogtmp_json_array)");
String insert_sarc_sushilogtmp = "INSERT INTO " + ConnectDB.getUsageStatsDBSchema() + ".sarc_sushilogtmp " + String insert_sarc_sushilogtmp = "INSERT INTO " + ConnectDB.getUsageStatsDBSchema() + ".sarc_sushilogtmp " +
"SELECT 'SARC-OJS', '" + issn + "' , `ItemIdent`.`Value`, `ItemPerformance`.`Period`.`Begin`, " + "SELECT 'SARC-OJS', '" + issn + "' , `ItemIdent`.`Value`, `ItemPerformance`.`Period`.`Begin`, " +
@ -189,6 +169,10 @@ public class SarcStats {
System.out.println("====> Inserted to sarc_sushilogtmp table (sarc_sushilogtmp_json_non_array)"); System.out.println("====> Inserted to sarc_sushilogtmp table (sarc_sushilogtmp_json_non_array)");
ConnectDB.getConnection().close(); ConnectDB.getConnection().close();
////////////////////////////////////
// Add everything to the sushilog table!!!!
////////////////////////////////////
} }
public void getAndProcessSarc(String sarcsReportPathArray, String sarcsReportPathNonArray) throws Exception { public void getAndProcessSarc(String sarcsReportPathArray, String sarcsReportPathNonArray) throws Exception {
@ -204,18 +188,18 @@ public class SarcStats {
ConnectDB.getConnection().close(); ConnectDB.getConnection().close();
List<String[]> issnAndUrls = new ArrayList<String[]>(); List<String[]> issnAndUrls = new ArrayList<String[]>();
issnAndUrls.add(new String[] { // issnAndUrls.add(new String[] {
"https://revistas.rcaap.pt/motricidade/sushiLite/v1_7/", "1646-107X" // "https://revistas.rcaap.pt/motricidade/sushiLite/v1_7/", "1646-107X"
}); // });
issnAndUrls.add(new String[] { // issnAndUrls.add(new String[] {
"https://revistas.rcaap.pt/antropologicas/sushiLite/v1_7/", "0873-819X" // "https://revistas.rcaap.pt/antropologicas/sushiLite/v1_7/", "0873-819X"
}); // });
issnAndUrls.add(new String[] { // issnAndUrls.add(new String[] {
"https://revistas.rcaap.pt/interaccoes/sushiLite/v1_7/", "1646-2335" // "https://revistas.rcaap.pt/interaccoes/sushiLite/v1_7/", "1646-2335"
}); // });
issnAndUrls.add(new String[] { // issnAndUrls.add(new String[] {
"https://revistas.rcaap.pt/cct/sushiLite/v1_7/", "2182-3030" // "https://revistas.rcaap.pt/cct/sushiLite/v1_7/", "2182-3030"
}); // });
issnAndUrls.add(new String[] { issnAndUrls.add(new String[] {
"https://actapediatrica.spp.pt/sushiLite/v1_7/", "0873-9781" "https://actapediatrica.spp.pt/sushiLite/v1_7/", "0873-9781"
}); });
@ -315,9 +299,28 @@ public class SarcStats {
"INSERT INTO sushilog (source, repository, rid, date, metric_type, count) VALUES (?,?,?,?,?,?)"); "INSERT INTO sushilog (source, repository, rid, date, metric_type, count) VALUES (?,?,?,?,?,?)");
int batch_size = 0; int batch_size = 0;
// Creating the needed configuration for the correct storing of data
Configuration config = new Configuration();
config.addResource(new Path("/etc/hadoop/conf/core-site.xml"));
config.addResource(new Path("/etc/hadoop/conf/hdfs-site.xml"));
config
.set(
"fs.hdfs.impl",
org.apache.hadoop.hdfs.DistributedFileSystem.class.getName());
config
.set(
"fs.file.impl",
org.apache.hadoop.fs.LocalFileSystem.class.getName());
FileSystem dfs = FileSystem.get(config);
// Creating the directory to save the files // Creating the directory to save the files
new File(sarcsReportPathArray + "/" + issn).mkdirs(); dfs.mkdirs(new Path(sarcsReportPathArray + "/" + issn));
new File(sarcsReportPathNonArray + "/" + issn).mkdirs(); dfs.mkdirs(new Path(sarcsReportPathNonArray + "/" + issn));
// System.out.println("====> END ===== ");
//
// if (true)
// System.exit(0);
while (start.before(end)) { while (start.before(end)) {
// String reportUrl = // String reportUrl =
@ -372,18 +375,18 @@ public class SarcStats {
} }
// Creating the file in the filesystem for the ItemIdentifier as array object // Creating the file in the filesystem for the ItemIdentifier as array object
FileSystem fsArray = FileSystem.get(new Configuration()); // FileSystem fsArray = FileSystem.get(new Configuration());
String filePathArray = sarcsReportPathArray + "/" + issn + "/" + "SarcsARReport" + String filePathArray = sarcsReportPathArray + "/" + issn + "/" + "SarcsARReport" +
simpleDateFormat.format(start.getTime()) + ".json"; simpleDateFormat.format(start.getTime()) + ".json";
System.out.println("Storing to file: " + filePathArray); System.out.println("Storing to file: " + filePathArray);
FSDataOutputStream finArray = fsArray.create(new Path(filePathArray), true); FSDataOutputStream finArray = dfs.create(new Path(filePathArray), true);
// Creating the file in the filesystem for the ItemIdentifier as array object // Creating the file in the filesystem for the ItemIdentifier as array object
FileSystem fsNonArray = FileSystem.get(new Configuration()); // FileSystem fsNonArray = FileSystem.get(new Configuration());
String filePathNonArray = sarcsReportPathNonArray + "/" + issn + "/" + "SarcsARReport" + String filePathNonArray = sarcsReportPathNonArray + "/" + issn + "/" + "SarcsARReport" +
simpleDateFormat.format(start.getTime()) + ".json"; simpleDateFormat.format(start.getTime()) + ".json";
System.out.println("Storing to file: " + filePathNonArray); System.out.println("Storing to file: " + filePathNonArray);
FSDataOutputStream finNonArray = fsNonArray.create(new Path(filePathNonArray), true); FSDataOutputStream finNonArray = dfs.create(new Path(filePathNonArray), true);
for (Object aJsonArray : jsonArray) { for (Object aJsonArray : jsonArray) {
@ -411,6 +414,9 @@ public class SarcStats {
if (fileNonArray.length() == 0) if (fileNonArray.length() == 0)
fileNonArray.delete(); fileNonArray.delete();
// fsArray.close();
// fsNonArray.close();
////////////////// //////////////////
// JSONObject jsonObjectRow = (JSONObject) aJsonArray; // JSONObject jsonObjectRow = (JSONObject) aJsonArray;
// JSONArray itemIdentifier = new JSONArray(); // JSONArray itemIdentifier = new JSONArray();
@ -465,6 +471,8 @@ public class SarcStats {
// break; // break;
} }
dfs.close();
// preparedStatement.executeBatch(); // preparedStatement.executeBatch();
// ConnectDB.getConnection().commit(); // ConnectDB.getConnection().commit();
ConnectDB.getConnection().close(); ConnectDB.getConnection().close();