forked from D-Net/dnet-hadoop
Renamings on Sarcs and deletion of empty files (to be double checked)
This commit is contained in:
parent
17f2748eb4
commit
1dcb197f02
|
|
@ -65,57 +65,61 @@ public class SarcStats {
|
|||
}
|
||||
}
|
||||
|
||||
public void processSarc(String sarcsReportPathArray, String sarcsReportPathNonArray) throws Exception {
|
||||
public void processSarc() {
|
||||
|
||||
}
|
||||
|
||||
public void getSarc(String sarcsReportPathArray, String sarcsReportPathNonArray) throws Exception {
|
||||
// There was a problem to download the following file
|
||||
processARReport(
|
||||
getARReport(
|
||||
sarcsReportPathArray, sarcsReportPathNonArray, "https://revistas.rcaap.pt/motricidade/sushiLite/v1_7/",
|
||||
"1646-107X");
|
||||
processARReport(
|
||||
getARReport(
|
||||
sarcsReportPathArray, sarcsReportPathNonArray, "https://revistas.rcaap.pt/antropologicas/sushiLite/v1_7/",
|
||||
"0873-819X");
|
||||
processARReport(
|
||||
getARReport(
|
||||
sarcsReportPathArray, sarcsReportPathNonArray, "https://revistas.rcaap.pt/interaccoes/sushiLite/v1_7/",
|
||||
"1646-2335");
|
||||
processARReport(
|
||||
getARReport(
|
||||
sarcsReportPathArray, sarcsReportPathNonArray, "https://revistas.rcaap.pt/cct/sushiLite/v1_7/",
|
||||
"2182-3030");
|
||||
processARReport(
|
||||
getARReport(
|
||||
sarcsReportPathArray, sarcsReportPathNonArray, "https://actapediatrica.spp.pt/sushiLite/v1_7/",
|
||||
"0873-9781");
|
||||
processARReport(
|
||||
getARReport(
|
||||
sarcsReportPathArray, sarcsReportPathNonArray, "https://revistas.rcaap.pt/sociologiapp/sushiLite/v1_7/",
|
||||
"0873-6529");
|
||||
processARReport(
|
||||
getARReport(
|
||||
sarcsReportPathArray, sarcsReportPathNonArray, "https://revistas.rcaap.pt/finisterra/sushiLite/v1_7/",
|
||||
"0430-5027");
|
||||
processARReport(
|
||||
getARReport(
|
||||
sarcsReportPathArray, sarcsReportPathNonArray, "https://revistas.rcaap.pt/sisyphus/sushiLite/v1_7/",
|
||||
"2182-8474");
|
||||
processARReport(
|
||||
getARReport(
|
||||
sarcsReportPathArray, sarcsReportPathNonArray, "https://revistas.rcaap.pt/anestesiologia/sushiLite/v1_7/",
|
||||
"0871-6099");
|
||||
processARReport(
|
||||
getARReport(
|
||||
sarcsReportPathArray, sarcsReportPathNonArray, "https://revistas.rcaap.pt/rpe/sushiLite/v1_7/",
|
||||
"0871-9187");
|
||||
processARReport(
|
||||
getARReport(
|
||||
sarcsReportPathArray, sarcsReportPathNonArray, "https://revistas.rcaap.pt/psilogos/sushiLite/v1_7/",
|
||||
"1646-091X");
|
||||
processARReport(
|
||||
getARReport(
|
||||
sarcsReportPathArray, sarcsReportPathNonArray, "https://revistas.rcaap.pt/juridica/sushiLite/v1_7/",
|
||||
"2183-5799");
|
||||
processARReport(
|
||||
getARReport(
|
||||
sarcsReportPathArray, sarcsReportPathNonArray, "https://revistas.rcaap.pt/ecr/sushiLite/v1_7/",
|
||||
"1647-2098");
|
||||
processARReport(
|
||||
getARReport(
|
||||
sarcsReportPathArray, sarcsReportPathNonArray, "https://revistas.rcaap.pt/nascercrescer/sushiLite/v1_7/",
|
||||
"0872-0754");
|
||||
processARReport(
|
||||
getARReport(
|
||||
sarcsReportPathArray, sarcsReportPathNonArray, "https://revistas.rcaap.pt/cea/sushiLite/v1_7/",
|
||||
"1645-3794");
|
||||
processARReport(
|
||||
getARReport(
|
||||
sarcsReportPathArray, sarcsReportPathNonArray, "https://revistas.rcaap.pt/proelium/sushiLite/v1_7/",
|
||||
"1645-8826");
|
||||
processARReport(
|
||||
getARReport(
|
||||
sarcsReportPathArray, sarcsReportPathNonArray, "https://revistas.rcaap.pt/millenium/sushiLite/v1_7/",
|
||||
"0873-3015");
|
||||
}
|
||||
|
|
@ -136,7 +140,7 @@ public class SarcStats {
|
|||
ConnectDB.getConnection().close();
|
||||
}
|
||||
|
||||
public void processARReport(String sarcsReportPathArray, String sarcsReportPathNonArray,
|
||||
public void getARReport(String sarcsReportPathArray, String sarcsReportPathNonArray,
|
||||
String url, String issn) throws Exception {
|
||||
log.info("Processing SARC! issn: " + issn + " with url: " + url);
|
||||
ConnectDB.getConnection().setAutoCommit(false);
|
||||
|
|
@ -192,6 +196,7 @@ public class SarcStats {
|
|||
* PrintWriter wr = new PrintWriter(new FileWriter("logs/" + simpleDateFormat.format(start.getTime()) +
|
||||
* ".json")); wr.print(text); wr.close();
|
||||
*/
|
||||
|
||||
System.out.println("AAAAAAAAAAA text " + text);
|
||||
|
||||
JSONParser parser = new JSONParser();
|
||||
|
|
@ -225,14 +230,14 @@ public class SarcStats {
|
|||
|
||||
// Creating the file in the filesystem for the ItemIdentifier as array object
|
||||
FileSystem fsArray = FileSystem.get(new Configuration());
|
||||
String filePathArray = sarcsReportPathArray + "/" + "SarcsARReport_" +
|
||||
String filePathArray = sarcsReportPathArray + "/" + "SarcsARReport_" + issn + "_" +
|
||||
simpleDateFormat.format(start.getTime()) + ".json";
|
||||
System.out.println("Storing to file: " + filePathArray);
|
||||
FSDataOutputStream finArray = fsArray.create(new Path(filePathArray), true);
|
||||
|
||||
// Creating the file in the filesystem for the ItemIdentifier as array object
|
||||
FileSystem fsNonArray = FileSystem.get(new Configuration());
|
||||
String filePathNonArray = sarcsReportPathNonArray + "/" + "SarcsARReport_" +
|
||||
String filePathNonArray = sarcsReportPathNonArray + "/" + "SarcsARReport_" + issn + "_" +
|
||||
simpleDateFormat.format(start.getTime()) + ".json";
|
||||
System.out.println("Storing to file: " + filePathNonArray);
|
||||
FSDataOutputStream finNonArray = fsNonArray.create(new Path(filePathNonArray), true);
|
||||
|
|
@ -252,7 +257,15 @@ public class SarcStats {
|
|||
finArray.writeChar('\n');
|
||||
}
|
||||
}
|
||||
|
||||
if (finArray.size() == 0)
|
||||
fsArray.deleteOnExit(new Path(filePathArray));
|
||||
|
||||
if (finNonArray.size() == 0)
|
||||
fsNonArray.deleteOnExit(new Path(filePathNonArray));
|
||||
|
||||
finArray.close();
|
||||
finNonArray.close();
|
||||
|
||||
//////////////////
|
||||
// JSONObject jsonObjectRow = (JSONObject) aJsonArray;
|
||||
|
|
@ -313,37 +326,38 @@ public class SarcStats {
|
|||
ConnectDB.getConnection().close();
|
||||
|
||||
}
|
||||
|
||||
|
||||
private void renameKeysRecursively(String delimiter, JSONArray givenJsonObj) throws Exception {
|
||||
for(Object jjval : givenJsonObj) {
|
||||
if(jjval instanceof JSONArray)
|
||||
renameKeysRecursively(delimiter, (JSONArray)jjval);
|
||||
else if(jjval instanceof JSONObject)
|
||||
renameKeysRecursively(delimiter, (JSONObject)jjval);
|
||||
for (Object jjval : givenJsonObj) {
|
||||
if (jjval instanceof JSONArray)
|
||||
renameKeysRecursively(delimiter, (JSONArray) jjval);
|
||||
else if (jjval instanceof JSONObject)
|
||||
renameKeysRecursively(delimiter, (JSONObject) jjval);
|
||||
// All other types of vals
|
||||
else;
|
||||
else
|
||||
;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
private void renameKeysRecursively(String delimiter, JSONObject givenJsonObj) throws Exception {
|
||||
Set<String> jkeys = new HashSet<String>(givenJsonObj.keySet());
|
||||
for (String jkey : jkeys) {
|
||||
System.out.println("++++> " + jkey);
|
||||
|
||||
|
||||
String[] splitArray = jkey.split(delimiter);
|
||||
String newJkey = splitArray[splitArray.length - 1];
|
||||
System.out.println("New jkey: " + jkey);
|
||||
|
||||
|
||||
Object jval = givenJsonObj.get(jkey);
|
||||
System.out.println("jval ===> " + jval.getClass().getName());
|
||||
givenJsonObj.remove(jkey);
|
||||
givenJsonObj.put(newJkey, jval);
|
||||
|
||||
if(jval instanceof JSONObject)
|
||||
renameKeysRecursively(delimiter, (JSONObject)jval);
|
||||
|
||||
if(jval instanceof JSONArray) {
|
||||
renameKeysRecursively(delimiter, (JSONArray)jval);
|
||||
|
||||
if (jval instanceof JSONObject)
|
||||
renameKeysRecursively(delimiter, (JSONObject) jval);
|
||||
|
||||
if (jval instanceof JSONArray) {
|
||||
renameKeysRecursively(delimiter, (JSONArray) jval);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -60,7 +60,7 @@ public class UsageStatsExporter {
|
|||
// log.info("irus done");
|
||||
|
||||
SarcStats sarcStats = new SarcStats();
|
||||
sarcStats.processSarc(sarcsReportPathArray, sarcsReportPathNonArray);
|
||||
sarcStats.getSarc(sarcsReportPathArray, sarcsReportPathNonArray);
|
||||
sarcStats.sarcStats();
|
||||
log.info("sarc done");
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue