enrico.ottonello
/
dnet-hadoop
forked from D-Net/dnet-hadoop
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Collection on Refresh WORKS!!!

This commit is contained in:
Sandro La Bruzzo 2021-01-29 17:02:46 +01:00
parent 0276180039
commit 8ee82576c6
3 changed files with 24 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/aggregation/mdstore/MDStoreActionNode.java
View File

@ -38,8 +38,8 @@ public class MDStoreActionNode {
public static final String COMMIT_VERSION_URL = "%s/version/%s/commit/%s"; public static final String COMMIT_VERSION_URL = "%s/version/%s/commit/%s";
public static final String ROLLBACK_VERSION_URL = "%s/version/%s/abort"; public static final String ROLLBACK_VERSION_URL = "%s/version/%s/abort";
public static final String READ_LOCK_URL = "%s/mdstores/mdstore/%s/startReading"; public static final String READ_LOCK_URL = "%s/mdstore/%s/startReading";
public static final String READ_UNLOCK_URL = "%s/mdstores/version/%s/endReading"; public static final String READ_UNLOCK_URL = "%s/version/%s/endReading";
private static final String MDSTOREVERSIONPARAM = "mdStoreVersion"; private static final String MDSTOREVERSIONPARAM = "mdStoreVersion";
private static final String MDSTOREREADLOCKPARAM = "mdStoreReadLockVersion"; private static final String MDSTOREREADLOCKPARAM = "mdStoreReadLockVersion";
@ -94,11 +94,7 @@ public class MDStoreActionNode {
System.setProperty("hadoop.home.dir", "/"); System.setProperty("hadoop.home.dir", "/");
// Get the filesystem - HDFS // Get the filesystem - HDFS
FileSystem fs = FileSystem.get(URI.create(hdfsuri), conf); FileSystem fs = FileSystem.get(URI.create(hdfsuri), conf);
String mdStoreSizeParam = argumentParser.get("mdStoreSize");
if (StringUtils.isBlank(mdStoreSizeParam)) {
throw new IllegalArgumentException("missing or empty argument mdStoreSize");
}
Path hdfstoreSizepath = new Path(mdStoreVersion.getHdfsPath() + "/size"); Path hdfstoreSizepath = new Path(mdStoreVersion.getHdfsPath() + "/size");
FSDataInputStream inputStream = fs.open(hdfstoreSizepath); FSDataInputStream inputStream = fs.open(hdfstoreSizepath);

13
dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/collection/CollectionJobTest.java
View File

@ -16,6 +16,8 @@ import org.junit.jupiter.api.io.TempDir;
import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectMapper;
import eu.dnetlib.data.mdstore.manager.common.model.MDStoreCurrentVersion;
import eu.dnetlib.data.mdstore.manager.common.model.MDStoreVersion;
import eu.dnetlib.dhp.model.mdstore.MetadataRecord; import eu.dnetlib.dhp.model.mdstore.MetadataRecord;
import eu.dnetlib.dhp.model.mdstore.Provenance; import eu.dnetlib.dhp.model.mdstore.Provenance;
import eu.dnetlib.dhp.schema.common.ModelSupport; import eu.dnetlib.dhp.schema.common.ModelSupport;
@ -37,6 +39,17 @@ public class CollectionJobTest {
spark.stop(); spark.stop();
} }
@Test
public void testJSONSerialization() throws Exception {
final String s = IOUtils.toString(getClass().getResourceAsStream("input.json"));
System.out.println("s = " + s);
final ObjectMapper mapper = new ObjectMapper();
MDStoreVersion mi = mapper.readValue(s, MDStoreVersion.class);
assertNotNull(mi);
}
@Test @Test
public void tesCollection(@TempDir Path testDir) throws Exception { public void tesCollection(@TempDir Path testDir) throws Exception {
final Provenance provenance = new Provenance("pippo", "puppa", "ns_prefix"); final Provenance provenance = new Provenance("pippo", "puppa", "ns_prefix");

9
dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/collection/input.json Normal file
View File

@ -0,0 +1,9 @@
{
"id": "md-7557225f-77cc-407d-bdf4-d2fe03131464-1611935085410",
"mdstore": "md-7557225f-77cc-407d-bdf4-d2fe03131464",
"writing": true,
"readCount": 0,
"lastUpdate": null,
"size": 0,
"hdfsPath": "/data/dnet.dev/mdstore/md-7557225f-77cc-407d-bdf4-d2fe03131464/md-7557225f-77cc-407d-bdf4-d2fe03131464-1611935085410"
}