forked from D-Net/dnet-hadoop
switched automatic code formatting plugin to net.revelc.code.formatter:formatter-maven-plugin
This commit is contained in:
parent
d3fd05e3c5
commit
a0bdbacdae
|
|
@ -1,3 +1,4 @@
|
|||
|
||||
package eu.dnetlib.maven.plugin.properties;
|
||||
|
||||
import java.io.File;
|
||||
|
|
@ -20,18 +21,21 @@ public class GenerateOoziePropertiesMojo extends AbstractMojo {
|
|||
public static final String PROPERTY_NAME_WF_SOURCE_DIR = "workflow.source.dir";
|
||||
public static final String PROPERTY_NAME_SANDBOX_NAME = "sandboxName";
|
||||
|
||||
private final String[] limiters = {"dhp", "dnetlib", "eu"};
|
||||
private final String[] limiters = {
|
||||
"dhp", "dnetlib", "eu"
|
||||
};
|
||||
|
||||
@Override
|
||||
public void execute() throws MojoExecutionException, MojoFailureException {
|
||||
if (System.getProperties().containsKey(PROPERTY_NAME_WF_SOURCE_DIR)
|
||||
&& !System.getProperties().containsKey(PROPERTY_NAME_SANDBOX_NAME)) {
|
||||
String generatedSandboxName =
|
||||
generateSandboxName(System.getProperties().getProperty(PROPERTY_NAME_WF_SOURCE_DIR));
|
||||
String generatedSandboxName = generateSandboxName(
|
||||
System.getProperties().getProperty(PROPERTY_NAME_WF_SOURCE_DIR));
|
||||
if (generatedSandboxName != null) {
|
||||
System.getProperties().setProperty(PROPERTY_NAME_SANDBOX_NAME, generatedSandboxName);
|
||||
} else {
|
||||
System.out.println(
|
||||
System.out
|
||||
.println(
|
||||
"unable to generate sandbox name from path: "
|
||||
+ System.getProperties().getProperty(PROPERTY_NAME_WF_SOURCE_DIR));
|
||||
}
|
||||
|
|
|
|||
|
|
@ -9,6 +9,7 @@
|
|||
* express or implied. See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package eu.dnetlib.maven.plugin.properties;
|
||||
|
||||
import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
|
||||
|
|
@ -70,33 +71,31 @@ public class WritePredefinedProjectProperties extends AbstractMojo {
|
|||
protected File outputFile;
|
||||
|
||||
/**
|
||||
* If true, the plugin will silently ignore any non-existent properties files, and the build will
|
||||
* continue
|
||||
* If true, the plugin will silently ignore any non-existent properties files, and the build will continue
|
||||
*
|
||||
* @parameter property="properties.quiet" default-value="true"
|
||||
*/
|
||||
private boolean quiet;
|
||||
|
||||
/**
|
||||
* Comma separated list of characters to escape when writing property values. cr=carriage return,
|
||||
* lf=linefeed, tab=tab. Any other values are taken literally.
|
||||
* Comma separated list of characters to escape when writing property values. cr=carriage return, lf=linefeed,
|
||||
* tab=tab. Any other values are taken literally.
|
||||
*
|
||||
* @parameter default-value="cr,lf,tab" property="properties.escapeChars"
|
||||
*/
|
||||
private String escapeChars;
|
||||
|
||||
/**
|
||||
* If true, the plugin will include system properties when writing the properties file. System
|
||||
* properties override both environment variables and project properties.
|
||||
* If true, the plugin will include system properties when writing the properties file. System properties override
|
||||
* both environment variables and project properties.
|
||||
*
|
||||
* @parameter default-value="false" property="properties.includeSystemProperties"
|
||||
*/
|
||||
private boolean includeSystemProperties;
|
||||
|
||||
/**
|
||||
* If true, the plugin will include environment variables when writing the properties file.
|
||||
* Environment variables are prefixed with "env". Environment variables override project
|
||||
* properties.
|
||||
* If true, the plugin will include environment variables when writing the properties file. Environment variables
|
||||
* are prefixed with "env". Environment variables override project properties.
|
||||
*
|
||||
* @parameter default-value="false" property="properties.includeEnvironmentVariables"
|
||||
*/
|
||||
|
|
@ -110,8 +109,8 @@ public class WritePredefinedProjectProperties extends AbstractMojo {
|
|||
private String exclude;
|
||||
|
||||
/**
|
||||
* Comma separated set of properties to write to the properties file. If provided, only the
|
||||
* properties matching those supplied here will be written to the properties file.
|
||||
* Comma separated set of properties to write to the properties file. If provided, only the properties matching
|
||||
* those supplied here will be written to the properties file.
|
||||
*
|
||||
* @parameter property="properties.include"
|
||||
*/
|
||||
|
|
@ -122,7 +121,9 @@ public class WritePredefinedProjectProperties extends AbstractMojo {
|
|||
* @see org.apache.maven.plugin.AbstractMojo#execute()
|
||||
*/
|
||||
@Override
|
||||
@SuppressFBWarnings({"NP_UNWRITTEN_FIELD", "UWF_UNWRITTEN_FIELD"})
|
||||
@SuppressFBWarnings({
|
||||
"NP_UNWRITTEN_FIELD", "UWF_UNWRITTEN_FIELD"
|
||||
})
|
||||
public void execute() throws MojoExecutionException, MojoFailureException {
|
||||
Properties properties = new Properties();
|
||||
// Add project properties
|
||||
|
|
@ -437,8 +438,8 @@ public class WritePredefinedProjectProperties extends AbstractMojo {
|
|||
*/
|
||||
public void setIncludePropertyKeysFromFiles(String[] includePropertyKeysFromFiles) {
|
||||
if (includePropertyKeysFromFiles != null) {
|
||||
this.includePropertyKeysFromFiles =
|
||||
Arrays.copyOf(includePropertyKeysFromFiles, includePropertyKeysFromFiles.length);
|
||||
this.includePropertyKeysFromFiles = Arrays
|
||||
.copyOf(includePropertyKeysFromFiles, includePropertyKeysFromFiles.length);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
|
||||
package eu.dnetlib.maven.plugin.properties;
|
||||
|
||||
import static eu.dnetlib.maven.plugin.properties.GenerateOoziePropertiesMojo.PROPERTY_NAME_SANDBOX_NAME;
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
|
||||
package eu.dnetlib.maven.plugin.properties;
|
||||
|
||||
import static eu.dnetlib.maven.plugin.properties.WritePredefinedProjectProperties.PROPERTY_PREFIX_ENV;
|
||||
|
|
@ -20,7 +21,8 @@ import org.mockito.junit.jupiter.MockitoExtension;
|
|||
@ExtendWith(MockitoExtension.class)
|
||||
public class WritePredefinedProjectPropertiesTest {
|
||||
|
||||
@Mock private MavenProject mavenProject;
|
||||
@Mock
|
||||
private MavenProject mavenProject;
|
||||
|
||||
private WritePredefinedProjectProperties mojo;
|
||||
|
||||
|
|
@ -145,7 +147,9 @@ public class WritePredefinedProjectPropertiesTest {
|
|||
includedProperties.setProperty(includedKey, "irrelevantValue");
|
||||
includedProperties.store(new FileWriter(includedPropertiesFile), null);
|
||||
|
||||
mojo.setIncludePropertyKeysFromFiles(new String[] {includedPropertiesFile.getAbsolutePath()});
|
||||
mojo.setIncludePropertyKeysFromFiles(new String[] {
|
||||
includedPropertiesFile.getAbsolutePath()
|
||||
});
|
||||
|
||||
// execute
|
||||
mojo.execute();
|
||||
|
|
@ -171,8 +175,11 @@ public class WritePredefinedProjectPropertiesTest {
|
|||
projectProperties.setProperty(includedKey, includedValue);
|
||||
doReturn(projectProperties).when(mavenProject).getProperties();
|
||||
|
||||
mojo.setIncludePropertyKeysFromFiles(
|
||||
new String[] {"/eu/dnetlib/maven/plugin/properties/included.properties"});
|
||||
mojo
|
||||
.setIncludePropertyKeysFromFiles(
|
||||
new String[] {
|
||||
"/eu/dnetlib/maven/plugin/properties/included.properties"
|
||||
});
|
||||
|
||||
// execute
|
||||
mojo.execute();
|
||||
|
|
@ -197,7 +204,9 @@ public class WritePredefinedProjectPropertiesTest {
|
|||
projectProperties.setProperty(includedKey, includedValue);
|
||||
doReturn(projectProperties).when(mavenProject).getProperties();
|
||||
|
||||
mojo.setIncludePropertyKeysFromFiles(new String[] {""});
|
||||
mojo.setIncludePropertyKeysFromFiles(new String[] {
|
||||
""
|
||||
});
|
||||
|
||||
// execute
|
||||
Assertions.assertThrows(MojoExecutionException.class, () -> mojo.execute());
|
||||
|
|
@ -221,7 +230,9 @@ public class WritePredefinedProjectPropertiesTest {
|
|||
includedProperties.setProperty(includedKey, "irrelevantValue");
|
||||
includedProperties.storeToXML(new FileOutputStream(includedPropertiesFile), null);
|
||||
|
||||
mojo.setIncludePropertyKeysFromFiles(new String[] {includedPropertiesFile.getAbsolutePath()});
|
||||
mojo.setIncludePropertyKeysFromFiles(new String[] {
|
||||
includedPropertiesFile.getAbsolutePath()
|
||||
});
|
||||
|
||||
// execute
|
||||
mojo.execute();
|
||||
|
|
@ -252,7 +263,9 @@ public class WritePredefinedProjectPropertiesTest {
|
|||
includedProperties.setProperty(includedKey, "irrelevantValue");
|
||||
includedProperties.store(new FileOutputStream(includedPropertiesFile), null);
|
||||
|
||||
mojo.setIncludePropertyKeysFromFiles(new String[] {includedPropertiesFile.getAbsolutePath()});
|
||||
mojo.setIncludePropertyKeysFromFiles(new String[] {
|
||||
includedPropertiesFile.getAbsolutePath()
|
||||
});
|
||||
|
||||
// execute
|
||||
Assertions.assertThrows(MojoExecutionException.class, () -> mojo.execute());
|
||||
|
|
@ -262,7 +275,9 @@ public class WritePredefinedProjectPropertiesTest {
|
|||
public void testExecuteWithQuietModeOn(@TempDir File testFolder) throws Exception {
|
||||
// given
|
||||
mojo.setQuiet(true);
|
||||
mojo.setIncludePropertyKeysFromFiles(new String[] {"invalid location"});
|
||||
mojo.setIncludePropertyKeysFromFiles(new String[] {
|
||||
"invalid location"
|
||||
});
|
||||
|
||||
// execute
|
||||
mojo.execute();
|
||||
|
|
@ -276,7 +291,9 @@ public class WritePredefinedProjectPropertiesTest {
|
|||
@Test
|
||||
public void testExecuteIncludingPropertyKeysFromInvalidFile() {
|
||||
// given
|
||||
mojo.setIncludePropertyKeysFromFiles(new String[] {"invalid location"});
|
||||
mojo.setIncludePropertyKeysFromFiles(new String[] {
|
||||
"invalid location"
|
||||
});
|
||||
|
||||
// execute
|
||||
Assertions.assertThrows(MojoExecutionException.class, () -> mojo.execute());
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
|
||||
package eu.dnetlib.collector.worker.model;
|
||||
|
||||
import java.util.HashMap;
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
|
||||
package eu.dnetlib.data.mdstore.manager.common.model;
|
||||
|
||||
import java.io.Serializable;
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
|
||||
package eu.dnetlib.data.mdstore.manager.common.model;
|
||||
|
||||
import java.io.Serializable;
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
|
||||
package eu.dnetlib.data.mdstore.manager.common.model;
|
||||
|
||||
import java.io.Serializable;
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
|
||||
package eu.dnetlib.data.mdstore.manager.common.model;
|
||||
|
||||
import java.io.Serializable;
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
|
||||
package eu.dnetlib.dhp.application;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
|
|
@ -21,8 +22,7 @@ public class ArgumentApplicationParser implements Serializable {
|
|||
|
||||
public ArgumentApplicationParser(final String json_configuration) throws Exception {
|
||||
final ObjectMapper mapper = new ObjectMapper();
|
||||
final OptionsParameter[] configuration =
|
||||
mapper.readValue(json_configuration, OptionsParameter[].class);
|
||||
final OptionsParameter[] configuration = mapper.readValue(json_configuration, OptionsParameter[].class);
|
||||
createOptionMap(configuration);
|
||||
}
|
||||
|
||||
|
|
@ -32,7 +32,8 @@ public class ArgumentApplicationParser implements Serializable {
|
|||
|
||||
private void createOptionMap(final OptionsParameter[] configuration) {
|
||||
|
||||
Arrays.stream(configuration)
|
||||
Arrays
|
||||
.stream(configuration)
|
||||
.map(
|
||||
conf -> {
|
||||
final Option o = new Option(conf.getParamName(), true, conf.getParamDescription());
|
||||
|
|
@ -74,10 +75,11 @@ public class ArgumentApplicationParser implements Serializable {
|
|||
public void parseArgument(final String[] args) throws Exception {
|
||||
CommandLineParser parser = new BasicParser();
|
||||
CommandLine cmd = parser.parse(options, args);
|
||||
Arrays.stream(cmd.getOptions())
|
||||
Arrays
|
||||
.stream(cmd.getOptions())
|
||||
.forEach(
|
||||
it ->
|
||||
objectMap.put(
|
||||
it -> objectMap
|
||||
.put(
|
||||
it.getLongOpt(),
|
||||
compressedValues.contains(it.getLongOpt())
|
||||
? decompressValue(it.getValue())
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
|
||||
package eu.dnetlib.dhp.application;
|
||||
|
||||
public class OptionsParameter {
|
||||
|
|
@ -8,7 +9,8 @@ public class OptionsParameter {
|
|||
private boolean paramRequired;
|
||||
private boolean compressed;
|
||||
|
||||
public OptionsParameter() {}
|
||||
public OptionsParameter() {
|
||||
}
|
||||
|
||||
public String getParamName() {
|
||||
return paramName;
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
|
||||
package eu.dnetlib.dhp.common;
|
||||
|
||||
import java.io.Serializable;
|
||||
|
|
@ -6,16 +7,18 @@ import java.util.function.Supplier;
|
|||
/** Provides serializable and throwing extensions to standard functional interfaces. */
|
||||
public class FunctionalInterfaceSupport {
|
||||
|
||||
private FunctionalInterfaceSupport() {}
|
||||
private FunctionalInterfaceSupport() {
|
||||
}
|
||||
|
||||
/**
|
||||
* Serializable supplier of any kind of objects. To be used withing spark processing pipelines
|
||||
* when supplying functions externally.
|
||||
* Serializable supplier of any kind of objects. To be used withing spark processing pipelines when supplying
|
||||
* functions externally.
|
||||
*
|
||||
* @param <T>
|
||||
*/
|
||||
@FunctionalInterface
|
||||
public interface SerializableSupplier<T> extends Supplier<T>, Serializable {}
|
||||
public interface SerializableSupplier<T> extends Supplier<T>, Serializable {
|
||||
}
|
||||
|
||||
/**
|
||||
* Extension of consumer accepting functions throwing an exception.
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
|
||||
package eu.dnetlib.dhp.common;
|
||||
|
||||
import static eu.dnetlib.dhp.common.ThrowingSupport.rethrowAsRuntimeException;
|
||||
|
|
@ -16,7 +17,8 @@ import org.slf4j.LoggerFactory;
|
|||
public class HdfsSupport {
|
||||
private static final Logger logger = LoggerFactory.getLogger(HdfsSupport.class);
|
||||
|
||||
private HdfsSupport() {}
|
||||
private HdfsSupport() {
|
||||
}
|
||||
|
||||
/**
|
||||
* Checks a path (file or dir) exists on HDFS.
|
||||
|
|
@ -62,8 +64,8 @@ public class HdfsSupport {
|
|||
public static List<String> listFiles(String path, Configuration configuration) {
|
||||
logger.info("Listing files in path: {}", path);
|
||||
return rethrowAsRuntimeException(
|
||||
() ->
|
||||
Arrays.stream(FileSystem.get(configuration).listStatus(new Path(path)))
|
||||
() -> Arrays
|
||||
.stream(FileSystem.get(configuration).listStatus(new Path(path)))
|
||||
.filter(FileStatus::isDirectory)
|
||||
.map(x -> x.getPath().toString())
|
||||
.collect(Collectors.toList()));
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
|
||||
package eu.dnetlib.dhp.common;
|
||||
|
||||
import eu.dnetlib.dhp.common.FunctionalInterfaceSupport.ThrowingConsumer;
|
||||
|
|
@ -9,12 +10,12 @@ import org.apache.spark.sql.SparkSession;
|
|||
/** SparkSession utility methods. */
|
||||
public class SparkSessionSupport {
|
||||
|
||||
private SparkSessionSupport() {}
|
||||
private SparkSessionSupport() {
|
||||
}
|
||||
|
||||
/**
|
||||
* Runs a given function using SparkSession created using default builder and supplied SparkConf.
|
||||
* Stops SparkSession when SparkSession is managed. Allows to reuse SparkSession created
|
||||
* externally.
|
||||
* Runs a given function using SparkSession created using default builder and supplied SparkConf. Stops SparkSession
|
||||
* when SparkSession is managed. Allows to reuse SparkSession created externally.
|
||||
*
|
||||
* @param conf SparkConf instance
|
||||
* @param isSparkSessionManaged When true will stop SparkSession
|
||||
|
|
@ -27,9 +28,8 @@ public class SparkSessionSupport {
|
|||
}
|
||||
|
||||
/**
|
||||
* Runs a given function using SparkSession created with hive support and using default builder
|
||||
* and supplied SparkConf. Stops SparkSession when SparkSession is managed. Allows to reuse
|
||||
* SparkSession created externally.
|
||||
* Runs a given function using SparkSession created with hive support and using default builder and supplied
|
||||
* SparkConf. Stops SparkSession when SparkSession is managed. Allows to reuse SparkSession created externally.
|
||||
*
|
||||
* @param conf SparkConf instance
|
||||
* @param isSparkSessionManaged When true will stop SparkSession
|
||||
|
|
@ -45,9 +45,8 @@ public class SparkSessionSupport {
|
|||
}
|
||||
|
||||
/**
|
||||
* Runs a given function using SparkSession created using supplied builder and supplied SparkConf.
|
||||
* Stops SparkSession when SparkSession is managed. Allows to reuse SparkSession created
|
||||
* externally.
|
||||
* Runs a given function using SparkSession created using supplied builder and supplied SparkConf. Stops
|
||||
* SparkSession when SparkSession is managed. Allows to reuse SparkSession created externally.
|
||||
*
|
||||
* @param sparkSessionBuilder Builder of SparkSession
|
||||
* @param conf SparkConf instance
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
|
||||
package eu.dnetlib.dhp.common;
|
||||
|
||||
import eu.dnetlib.dhp.common.FunctionalInterfaceSupport.ThrowingRunnable;
|
||||
|
|
@ -6,7 +7,8 @@ import eu.dnetlib.dhp.common.FunctionalInterfaceSupport.ThrowingSupplier;
|
|||
/** Exception handling utility methods. */
|
||||
public class ThrowingSupport {
|
||||
|
||||
private ThrowingSupport() {}
|
||||
private ThrowingSupport() {
|
||||
}
|
||||
|
||||
/**
|
||||
* Executes given runnable and rethrows any exceptions as RuntimeException.
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
|
||||
package eu.dnetlib.dhp.model.mdstore;
|
||||
|
||||
import eu.dnetlib.dhp.utils.DHPUtils;
|
||||
|
|
@ -16,8 +17,7 @@ public class MetadataRecord implements Serializable {
|
|||
private String encoding;
|
||||
|
||||
/**
|
||||
* The information about the provenance of the record see @{@link Provenance} for the model of
|
||||
* this information
|
||||
* The information about the provenance of the record see @{@link Provenance} for the model of this information
|
||||
*/
|
||||
private Provenance provenance;
|
||||
|
||||
|
|
|
|||
|
|
@ -1,11 +1,13 @@
|
|||
|
||||
package eu.dnetlib.dhp.model.mdstore;
|
||||
|
||||
import java.io.Serializable;
|
||||
|
||||
/**
|
||||
* @author Sandro La Bruzzo
|
||||
* <p>Provenace class models the provenance of the record in the metadataStore It contains the
|
||||
* identifier and the name of the datasource that gives the record
|
||||
* <p>
|
||||
* Provenace class models the provenance of the record in the metadataStore It contains the identifier and the
|
||||
* name of the datasource that gives the record
|
||||
*/
|
||||
public class Provenance implements Serializable {
|
||||
|
||||
|
|
@ -15,7 +17,8 @@ public class Provenance implements Serializable {
|
|||
|
||||
private String nsPrefix;
|
||||
|
||||
public Provenance() {}
|
||||
public Provenance() {
|
||||
}
|
||||
|
||||
public Provenance(String datasourceId, String datasourceName, String nsPrefix) {
|
||||
this.datasourceId = datasourceId;
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
|
||||
package eu.dnetlib.dhp.parser.utility;
|
||||
|
||||
public class VtdException extends Exception {
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
|
||||
package eu.dnetlib.dhp.parser.utility;
|
||||
|
||||
import com.ximpleware.AutoPilot;
|
||||
|
|
@ -36,7 +37,8 @@ public class VtdUtilityParser {
|
|||
final Map<String, String> currentAttributes = new HashMap<>();
|
||||
if (attributes != null) {
|
||||
|
||||
attributes.forEach(
|
||||
attributes
|
||||
.forEach(
|
||||
attributeKey -> {
|
||||
try {
|
||||
int attr = vn.getAttrVal(attributeKey);
|
||||
|
|
@ -58,7 +60,8 @@ public class VtdUtilityParser {
|
|||
ap.selectXPath(xpath);
|
||||
while (ap.evalXPath() != -1) {
|
||||
int t = vn.getText();
|
||||
if (t > -1) results.add(vn.toNormalizedString(t));
|
||||
if (t > -1)
|
||||
results.add(vn.toNormalizedString(t));
|
||||
}
|
||||
return results;
|
||||
} catch (Exception e) {
|
||||
|
|
@ -72,7 +75,8 @@ public class VtdUtilityParser {
|
|||
ap.selectXPath(xpath);
|
||||
while (ap.evalXPath() != -1) {
|
||||
int it = nav.getText();
|
||||
if (it > -1) return nav.toNormalizedString(it);
|
||||
if (it > -1)
|
||||
return nav.toNormalizedString(it);
|
||||
}
|
||||
return null;
|
||||
} catch (Exception e) {
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
|
||||
package eu.dnetlib.dhp.utils;
|
||||
|
||||
import com.jayway.jsonpath.JsonPath;
|
||||
|
|
@ -59,7 +60,8 @@ public class DHPUtils {
|
|||
public static String getJPathString(final String jsonPath, final String json) {
|
||||
try {
|
||||
Object o = JsonPath.read(json, jsonPath);
|
||||
if (o instanceof String) return (String) o;
|
||||
if (o instanceof String)
|
||||
return (String) o;
|
||||
if (o instanceof JSONArray && ((JSONArray) o).size() > 0)
|
||||
return (String) ((JSONArray) o).get(0);
|
||||
return o.toString();
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
|
||||
package eu.dnetlib.dhp.utils;
|
||||
|
||||
import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpService;
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
|
||||
package eu.dnetlib.dhp.utils.saxon;
|
||||
|
||||
import net.sf.saxon.expr.XPathContext;
|
||||
|
|
@ -9,8 +10,7 @@ import net.sf.saxon.trans.XPathException;
|
|||
|
||||
public abstract class AbstractExtensionFunction extends ExtensionFunctionDefinition {
|
||||
|
||||
public static String DEFAULT_SAXON_EXT_NS_URI =
|
||||
"http://www.d-net.research-infrastructures.eu/saxon-extension";
|
||||
public static String DEFAULT_SAXON_EXT_NS_URI = "http://www.d-net.research-infrastructures.eu/saxon-extension";
|
||||
|
||||
public abstract String getName();
|
||||
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
|
||||
package eu.dnetlib.dhp.utils.saxon;
|
||||
|
||||
import java.text.ParseException;
|
||||
|
|
@ -13,7 +14,9 @@ import net.sf.saxon.value.StringValue;
|
|||
|
||||
public class ExtractYear extends AbstractExtensionFunction {
|
||||
|
||||
private static final String[] dateFormats = {"yyyy-MM-dd", "yyyy/MM/dd"};
|
||||
private static final String[] dateFormats = {
|
||||
"yyyy-MM-dd", "yyyy/MM/dd"
|
||||
};
|
||||
|
||||
@Override
|
||||
public String getName() {
|
||||
|
|
@ -44,7 +47,9 @@ public class ExtractYear extends AbstractExtensionFunction {
|
|||
|
||||
@Override
|
||||
public SequenceType[] getArgumentTypes() {
|
||||
return new SequenceType[] {SequenceType.OPTIONAL_ITEM};
|
||||
return new SequenceType[] {
|
||||
SequenceType.OPTIONAL_ITEM
|
||||
};
|
||||
}
|
||||
|
||||
@Override
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
|
||||
package eu.dnetlib.dhp.utils.saxon;
|
||||
|
||||
import java.text.ParseException;
|
||||
|
|
@ -43,7 +44,9 @@ public class NormalizeDate extends AbstractExtensionFunction {
|
|||
|
||||
@Override
|
||||
public SequenceType[] getArgumentTypes() {
|
||||
return new SequenceType[] {SequenceType.OPTIONAL_ITEM};
|
||||
return new SequenceType[] {
|
||||
SequenceType.OPTIONAL_ITEM
|
||||
};
|
||||
}
|
||||
|
||||
@Override
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
|
||||
package eu.dnetlib.dhp.utils.saxon;
|
||||
|
||||
import net.sf.saxon.expr.XPathContext;
|
||||
|
|
@ -49,7 +50,9 @@ public class PickFirst extends AbstractExtensionFunction {
|
|||
|
||||
@Override
|
||||
public SequenceType[] getArgumentTypes() {
|
||||
return new SequenceType[] {SequenceType.OPTIONAL_ITEM};
|
||||
return new SequenceType[] {
|
||||
SequenceType.OPTIONAL_ITEM
|
||||
};
|
||||
}
|
||||
|
||||
@Override
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
|
||||
package eu.dnetlib.dhp.utils.saxon;
|
||||
|
||||
import java.io.StringReader;
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
|
||||
package eu.dnetlib.message;
|
||||
|
||||
import com.fasterxml.jackson.core.JsonProcessingException;
|
||||
|
|
@ -20,7 +21,8 @@ public class Message {
|
|||
return jsonMapper.readValue(json, Message.class);
|
||||
}
|
||||
|
||||
public Message() {}
|
||||
public Message() {
|
||||
}
|
||||
|
||||
public Message(String workflowId, String jobName, MessageType type, Map<String, String> body) {
|
||||
this.workflowId = workflowId;
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
|
||||
package eu.dnetlib.message;
|
||||
|
||||
import com.rabbitmq.client.AMQP;
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
|
||||
package eu.dnetlib.message;
|
||||
|
||||
import com.rabbitmq.client.Channel;
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
|
||||
package eu.dnetlib.message;
|
||||
|
||||
public enum MessageType {
|
||||
ONGOING,
|
||||
REPORT
|
||||
ONGOING, REPORT
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
|
||||
package eu.dnetlib.scholexplorer.relation;
|
||||
|
||||
import java.io.Serializable;
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
|
||||
package eu.dnetlib.scholexplorer.relation;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
|
|
@ -9,8 +10,7 @@ public class RelationMapper extends HashMap<String, RelInfo> implements Serializ
|
|||
|
||||
public static RelationMapper load() throws Exception {
|
||||
|
||||
final String json =
|
||||
IOUtils.toString(RelationMapper.class.getResourceAsStream("relations.json"));
|
||||
final String json = IOUtils.toString(RelationMapper.class.getResourceAsStream("relations.json"));
|
||||
|
||||
ObjectMapper mapper = new ObjectMapper();
|
||||
return mapper.readValue(json, RelationMapper.class);
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
|
||||
package eu.dnetlib.dhp.application;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
|
|
@ -10,12 +11,13 @@ public class ArgumentApplicationParserTest {
|
|||
|
||||
@Test
|
||||
public void testParseParameter() throws Exception {
|
||||
final String jsonConfiguration =
|
||||
IOUtils.toString(
|
||||
final String jsonConfiguration = IOUtils
|
||||
.toString(
|
||||
this.getClass().getResourceAsStream("/eu/dnetlib/application/parameters.json"));
|
||||
assertNotNull(jsonConfiguration);
|
||||
ArgumentApplicationParser parser = new ArgumentApplicationParser(jsonConfiguration);
|
||||
parser.parseArgument(
|
||||
parser
|
||||
.parseArgument(
|
||||
new String[] {
|
||||
"-p",
|
||||
"value0",
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
|
||||
package eu.dnetlib.dhp.common;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.*;
|
||||
|
|
@ -65,8 +66,9 @@ public class HdfsSupportTest {
|
|||
|
||||
// then
|
||||
assertEquals(2, paths.size());
|
||||
List<String> expecteds =
|
||||
Arrays.stream(new String[] {subDir1.toString(), subDir2.toString()})
|
||||
List<String> expecteds = Arrays.stream(new String[] {
|
||||
subDir1.toString(), subDir2.toString()
|
||||
})
|
||||
.sorted()
|
||||
.collect(Collectors.toList());
|
||||
List<String> actuals = paths.stream().sorted().collect(Collectors.toList());
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
|
||||
package eu.dnetlib.dhp.common;
|
||||
|
||||
import static org.mockito.Mockito.*;
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
|
||||
package eu.dnetlib.dhp.model.mdstore;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
|
||||
package eu.dnetlib.message;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.*;
|
||||
|
|
@ -33,8 +34,7 @@ public class MessageTest {
|
|||
|
||||
@Test
|
||||
public void toStringTest() {
|
||||
final String expectedJson =
|
||||
"{\"workflowId\":\"wId\",\"jobName\":\"Collection\",\"type\":\"ONGOING\",\"body\":{\"ExecutionTime\":\"30s\",\"parsedItem\":\"300\"}}";
|
||||
final String expectedJson = "{\"workflowId\":\"wId\",\"jobName\":\"Collection\",\"type\":\"ONGOING\",\"body\":{\"ExecutionTime\":\"30s\",\"parsedItem\":\"300\"}}";
|
||||
Message m = new Message();
|
||||
m.setWorkflowId("wId");
|
||||
m.setType(MessageType.ONGOING);
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
|
||||
package eu.dnetlib.scholexplorer.relation;
|
||||
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
|
||||
package eu.dnetlib.dhp.schema.action;
|
||||
|
||||
import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
|
||||
|
|
@ -11,7 +12,8 @@ public class AtomicAction<T extends Oaf> implements Serializable {
|
|||
|
||||
private T payload;
|
||||
|
||||
public AtomicAction() {}
|
||||
public AtomicAction() {
|
||||
}
|
||||
|
||||
public AtomicAction(Class<T> clazz, T payload) {
|
||||
this.clazz = clazz;
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
|
||||
package eu.dnetlib.dhp.schema.action;
|
||||
|
||||
import com.fasterxml.jackson.core.JsonParser;
|
||||
|
|
|
|||
|
|
@ -1,16 +1,11 @@
|
|||
|
||||
package eu.dnetlib.dhp.schema.common;
|
||||
|
||||
import eu.dnetlib.dhp.schema.oaf.OafEntity;
|
||||
|
||||
/** Actual entity types in the Graph */
|
||||
public enum EntityType {
|
||||
publication,
|
||||
dataset,
|
||||
otherresearchproduct,
|
||||
software,
|
||||
datasource,
|
||||
organization,
|
||||
project;
|
||||
publication, dataset, otherresearchproduct, software, datasource, organization, project;
|
||||
|
||||
/**
|
||||
* Resolves the EntityType, given the relative class name
|
||||
|
|
|
|||
|
|
@ -1,9 +1,7 @@
|
|||
|
||||
package eu.dnetlib.dhp.schema.common;
|
||||
|
||||
/** Main entity types in the Graph */
|
||||
public enum MainEntityType {
|
||||
result,
|
||||
datasource,
|
||||
organization,
|
||||
project
|
||||
result, datasource, organization, project
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
|
||||
package eu.dnetlib.dhp.schema.common;
|
||||
|
||||
import eu.dnetlib.dhp.schema.oaf.Qualifier;
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
|
||||
package eu.dnetlib.dhp.schema.common;
|
||||
|
||||
import com.google.common.collect.Maps;
|
||||
|
|
@ -52,7 +53,8 @@ public class ModelSupport {
|
|||
|
||||
private static final String schemeTemplate = "dnet:%s_%s_relations";
|
||||
|
||||
private ModelSupport() {}
|
||||
private ModelSupport() {
|
||||
}
|
||||
|
||||
/**
|
||||
* Checks subclass-superclass relationship.
|
||||
|
|
@ -142,7 +144,8 @@ public class ModelSupport {
|
|||
}
|
||||
|
||||
public static String getScheme(final String sourceType, final String targetType) {
|
||||
return String.format(
|
||||
return String
|
||||
.format(
|
||||
schemeTemplate,
|
||||
entityMapping.get(EntityType.valueOf(sourceType)).name(),
|
||||
entityMapping.get(EntityType.valueOf(targetType)).name());
|
||||
|
|
@ -159,29 +162,31 @@ public class ModelSupport {
|
|||
|
||||
private static <T extends Oaf> String idFnForRelation(T t) {
|
||||
Relation r = (Relation) t;
|
||||
return Optional.ofNullable(r.getSource())
|
||||
return Optional
|
||||
.ofNullable(r.getSource())
|
||||
.map(
|
||||
source ->
|
||||
Optional.ofNullable(r.getTarget())
|
||||
source -> Optional
|
||||
.ofNullable(r.getTarget())
|
||||
.map(
|
||||
target ->
|
||||
Optional.ofNullable(r.getRelType())
|
||||
target -> Optional
|
||||
.ofNullable(r.getRelType())
|
||||
.map(
|
||||
relType ->
|
||||
Optional.ofNullable(r.getSubRelType())
|
||||
relType -> Optional
|
||||
.ofNullable(r.getSubRelType())
|
||||
.map(
|
||||
subRelType ->
|
||||
Optional.ofNullable(r.getRelClass())
|
||||
subRelType -> Optional
|
||||
.ofNullable(r.getRelClass())
|
||||
.map(
|
||||
relClass ->
|
||||
String.join(
|
||||
relClass -> String
|
||||
.join(
|
||||
source,
|
||||
target,
|
||||
relType,
|
||||
subRelType,
|
||||
relClass))
|
||||
.orElse(
|
||||
String.join(
|
||||
String
|
||||
.join(
|
||||
source,
|
||||
target,
|
||||
relType,
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
|
||||
package eu.dnetlib.dhp.schema.oaf;
|
||||
|
||||
import java.io.Serializable;
|
||||
|
|
@ -68,8 +69,10 @@ public class Author implements Serializable {
|
|||
|
||||
@Override
|
||||
public boolean equals(Object o) {
|
||||
if (this == o) return true;
|
||||
if (o == null || getClass() != o.getClass()) return false;
|
||||
if (this == o)
|
||||
return true;
|
||||
if (o == null || getClass() != o.getClass())
|
||||
return false;
|
||||
Author author = (Author) o;
|
||||
return Objects.equals(fullname, author.fullname)
|
||||
&& Objects.equals(name, author.name)
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
|
||||
package eu.dnetlib.dhp.schema.oaf;
|
||||
|
||||
import java.io.Serializable;
|
||||
|
|
@ -31,9 +32,12 @@ public class Context implements Serializable {
|
|||
|
||||
@Override
|
||||
public boolean equals(Object obj) {
|
||||
if (this == obj) return true;
|
||||
if (obj == null) return false;
|
||||
if (getClass() != obj.getClass()) return false;
|
||||
if (this == obj)
|
||||
return true;
|
||||
if (obj == null)
|
||||
return false;
|
||||
if (getClass() != obj.getClass())
|
||||
return false;
|
||||
|
||||
Context other = (Context) obj;
|
||||
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
|
||||
package eu.dnetlib.dhp.schema.oaf;
|
||||
|
||||
import java.util.Objects;
|
||||
|
|
@ -16,9 +17,12 @@ public class Country extends Qualifier {
|
|||
|
||||
@Override
|
||||
public boolean equals(Object o) {
|
||||
if (this == o) return true;
|
||||
if (o == null || getClass() != o.getClass()) return false;
|
||||
if (!super.equals(o)) return false;
|
||||
if (this == o)
|
||||
return true;
|
||||
if (o == null || getClass() != o.getClass())
|
||||
return false;
|
||||
if (!super.equals(o))
|
||||
return false;
|
||||
Country country = (Country) o;
|
||||
return Objects.equals(dataInfo, country.dataInfo);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
|
||||
package eu.dnetlib.dhp.schema.oaf;
|
||||
|
||||
import java.io.Serializable;
|
||||
|
|
@ -62,8 +63,10 @@ public class DataInfo implements Serializable {
|
|||
|
||||
@Override
|
||||
public boolean equals(Object o) {
|
||||
if (this == o) return true;
|
||||
if (o == null || getClass() != o.getClass()) return false;
|
||||
if (this == o)
|
||||
return true;
|
||||
if (o == null || getClass() != o.getClass())
|
||||
return false;
|
||||
DataInfo dataInfo = (DataInfo) o;
|
||||
return Objects.equals(invisible, dataInfo.invisible)
|
||||
&& Objects.equals(inferred, dataInfo.inferred)
|
||||
|
|
@ -75,7 +78,8 @@ public class DataInfo implements Serializable {
|
|||
|
||||
@Override
|
||||
public int hashCode() {
|
||||
return Objects.hash(
|
||||
return Objects
|
||||
.hash(
|
||||
invisible, inferred, deletedbyinference, trust, inferenceprovenance, provenanceaction);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
|
||||
package eu.dnetlib.dhp.schema.oaf;
|
||||
|
||||
import eu.dnetlib.dhp.schema.common.ModelConstants;
|
||||
|
|
@ -90,8 +91,7 @@ public class Dataset extends Result implements Serializable {
|
|||
|
||||
final Dataset d = (Dataset) e;
|
||||
|
||||
storagedate =
|
||||
d.getStoragedate() != null && compareTrust(this, e) < 0 ? d.getStoragedate() : storagedate;
|
||||
storagedate = d.getStoragedate() != null && compareTrust(this, e) < 0 ? d.getStoragedate() : storagedate;
|
||||
|
||||
device = d.getDevice() != null && compareTrust(this, e) < 0 ? d.getDevice() : device;
|
||||
|
||||
|
|
@ -99,13 +99,11 @@ public class Dataset extends Result implements Serializable {
|
|||
|
||||
version = d.getVersion() != null && compareTrust(this, e) < 0 ? d.getVersion() : version;
|
||||
|
||||
lastmetadataupdate =
|
||||
d.getLastmetadataupdate() != null && compareTrust(this, e) < 0
|
||||
lastmetadataupdate = d.getLastmetadataupdate() != null && compareTrust(this, e) < 0
|
||||
? d.getLastmetadataupdate()
|
||||
: lastmetadataupdate;
|
||||
|
||||
metadataversionnumber =
|
||||
d.getMetadataversionnumber() != null && compareTrust(this, e) < 0
|
||||
metadataversionnumber = d.getMetadataversionnumber() != null && compareTrust(this, e) < 0
|
||||
? d.getMetadataversionnumber()
|
||||
: metadataversionnumber;
|
||||
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
|
||||
package eu.dnetlib.dhp.schema.oaf;
|
||||
|
||||
import java.io.Serializable;
|
||||
|
|
@ -372,120 +373,93 @@ public class Datasource extends OafEntity implements Serializable {
|
|||
|
||||
Datasource d = (Datasource) e;
|
||||
|
||||
datasourcetype =
|
||||
d.getDatasourcetype() != null && compareTrust(this, e) < 0
|
||||
datasourcetype = d.getDatasourcetype() != null && compareTrust(this, e) < 0
|
||||
? d.getDatasourcetype()
|
||||
: datasourcetype;
|
||||
openairecompatibility =
|
||||
d.getOpenairecompatibility() != null && compareTrust(this, e) < 0
|
||||
openairecompatibility = d.getOpenairecompatibility() != null && compareTrust(this, e) < 0
|
||||
? d.getOpenairecompatibility()
|
||||
: openairecompatibility;
|
||||
officialname =
|
||||
d.getOfficialname() != null && compareTrust(this, e) < 0
|
||||
officialname = d.getOfficialname() != null && compareTrust(this, e) < 0
|
||||
? d.getOfficialname()
|
||||
: officialname;
|
||||
englishname =
|
||||
d.getEnglishname() != null && compareTrust(this, e) < 0 ? d.getEnglishname() : officialname;
|
||||
websiteurl =
|
||||
d.getWebsiteurl() != null && compareTrust(this, e) < 0 ? d.getWebsiteurl() : websiteurl;
|
||||
englishname = d.getEnglishname() != null && compareTrust(this, e) < 0 ? d.getEnglishname() : officialname;
|
||||
websiteurl = d.getWebsiteurl() != null && compareTrust(this, e) < 0 ? d.getWebsiteurl() : websiteurl;
|
||||
logourl = d.getLogourl() != null && compareTrust(this, e) < 0 ? d.getLogourl() : getLogourl();
|
||||
contactemail =
|
||||
d.getContactemail() != null && compareTrust(this, e) < 0
|
||||
contactemail = d.getContactemail() != null && compareTrust(this, e) < 0
|
||||
? d.getContactemail()
|
||||
: contactemail;
|
||||
namespaceprefix =
|
||||
d.getNamespaceprefix() != null && compareTrust(this, e) < 0
|
||||
namespaceprefix = d.getNamespaceprefix() != null && compareTrust(this, e) < 0
|
||||
? d.getNamespaceprefix()
|
||||
: namespaceprefix;
|
||||
latitude = d.getLatitude() != null && compareTrust(this, e) < 0 ? d.getLatitude() : latitude;
|
||||
longitude =
|
||||
d.getLongitude() != null && compareTrust(this, e) < 0 ? d.getLongitude() : longitude;
|
||||
dateofvalidation =
|
||||
d.getDateofvalidation() != null && compareTrust(this, e) < 0
|
||||
longitude = d.getLongitude() != null && compareTrust(this, e) < 0 ? d.getLongitude() : longitude;
|
||||
dateofvalidation = d.getDateofvalidation() != null && compareTrust(this, e) < 0
|
||||
? d.getDateofvalidation()
|
||||
: dateofvalidation;
|
||||
description =
|
||||
d.getDescription() != null && compareTrust(this, e) < 0 ? d.getDescription() : description;
|
||||
description = d.getDescription() != null && compareTrust(this, e) < 0 ? d.getDescription() : description;
|
||||
subjects = mergeLists(subjects, d.getSubjects());
|
||||
|
||||
// opendoar specific fields (od*)
|
||||
odnumberofitems =
|
||||
d.getOdnumberofitems() != null && compareTrust(this, e) < 0
|
||||
odnumberofitems = d.getOdnumberofitems() != null && compareTrust(this, e) < 0
|
||||
? d.getOdnumberofitems()
|
||||
: odnumberofitems;
|
||||
odnumberofitemsdate =
|
||||
d.getOdnumberofitemsdate() != null && compareTrust(this, e) < 0
|
||||
odnumberofitemsdate = d.getOdnumberofitemsdate() != null && compareTrust(this, e) < 0
|
||||
? d.getOdnumberofitemsdate()
|
||||
: odnumberofitemsdate;
|
||||
odpolicies =
|
||||
d.getOdpolicies() != null && compareTrust(this, e) < 0 ? d.getOdpolicies() : odpolicies;
|
||||
odpolicies = d.getOdpolicies() != null && compareTrust(this, e) < 0 ? d.getOdpolicies() : odpolicies;
|
||||
odlanguages = mergeLists(odlanguages, d.getOdlanguages());
|
||||
odcontenttypes = mergeLists(odcontenttypes, d.getOdcontenttypes());
|
||||
accessinfopackage = mergeLists(accessinfopackage, d.getAccessinfopackage());
|
||||
|
||||
// re3data fields
|
||||
releasestartdate =
|
||||
d.getReleasestartdate() != null && compareTrust(this, e) < 0
|
||||
releasestartdate = d.getReleasestartdate() != null && compareTrust(this, e) < 0
|
||||
? d.getReleasestartdate()
|
||||
: releasestartdate;
|
||||
releaseenddate =
|
||||
d.getReleaseenddate() != null && compareTrust(this, e) < 0
|
||||
releaseenddate = d.getReleaseenddate() != null && compareTrust(this, e) < 0
|
||||
? d.getReleaseenddate()
|
||||
: releaseenddate;
|
||||
missionstatementurl =
|
||||
d.getMissionstatementurl() != null && compareTrust(this, e) < 0
|
||||
missionstatementurl = d.getMissionstatementurl() != null && compareTrust(this, e) < 0
|
||||
? d.getMissionstatementurl()
|
||||
: missionstatementurl;
|
||||
dataprovider =
|
||||
d.getDataprovider() != null && compareTrust(this, e) < 0
|
||||
dataprovider = d.getDataprovider() != null && compareTrust(this, e) < 0
|
||||
? d.getDataprovider()
|
||||
: dataprovider;
|
||||
serviceprovider =
|
||||
d.getServiceprovider() != null && compareTrust(this, e) < 0
|
||||
serviceprovider = d.getServiceprovider() != null && compareTrust(this, e) < 0
|
||||
? d.getServiceprovider()
|
||||
: serviceprovider;
|
||||
|
||||
// {open, restricted or closed}
|
||||
databaseaccesstype =
|
||||
d.getDatabaseaccesstype() != null && compareTrust(this, e) < 0
|
||||
databaseaccesstype = d.getDatabaseaccesstype() != null && compareTrust(this, e) < 0
|
||||
? d.getDatabaseaccesstype()
|
||||
: databaseaccesstype;
|
||||
|
||||
// {open, restricted or closed}
|
||||
datauploadtype =
|
||||
d.getDatauploadtype() != null && compareTrust(this, e) < 0
|
||||
datauploadtype = d.getDatauploadtype() != null && compareTrust(this, e) < 0
|
||||
? d.getDatauploadtype()
|
||||
: datauploadtype;
|
||||
|
||||
// {feeRequired, registration, other}
|
||||
databaseaccessrestriction =
|
||||
d.getDatabaseaccessrestriction() != null && compareTrust(this, e) < 0
|
||||
databaseaccessrestriction = d.getDatabaseaccessrestriction() != null && compareTrust(this, e) < 0
|
||||
? d.getDatabaseaccessrestriction()
|
||||
: databaseaccessrestriction;
|
||||
|
||||
// {feeRequired, registration, other}
|
||||
datauploadrestriction =
|
||||
d.getDatauploadrestriction() != null && compareTrust(this, e) < 0
|
||||
datauploadrestriction = d.getDatauploadrestriction() != null && compareTrust(this, e) < 0
|
||||
? d.getDatauploadrestriction()
|
||||
: datauploadrestriction;
|
||||
|
||||
versioning =
|
||||
d.getVersioning() != null && compareTrust(this, e) < 0 ? d.getVersioning() : versioning;
|
||||
citationguidelineurl =
|
||||
d.getCitationguidelineurl() != null && compareTrust(this, e) < 0
|
||||
versioning = d.getVersioning() != null && compareTrust(this, e) < 0 ? d.getVersioning() : versioning;
|
||||
citationguidelineurl = d.getCitationguidelineurl() != null && compareTrust(this, e) < 0
|
||||
? d.getCitationguidelineurl()
|
||||
: citationguidelineurl;
|
||||
|
||||
// {yes, no, unknown}
|
||||
qualitymanagementkind =
|
||||
d.getQualitymanagementkind() != null && compareTrust(this, e) < 0
|
||||
qualitymanagementkind = d.getQualitymanagementkind() != null && compareTrust(this, e) < 0
|
||||
? d.getQualitymanagementkind()
|
||||
: qualitymanagementkind;
|
||||
pidsystems =
|
||||
d.getPidsystems() != null && compareTrust(this, e) < 0 ? d.getPidsystems() : pidsystems;
|
||||
pidsystems = d.getPidsystems() != null && compareTrust(this, e) < 0 ? d.getPidsystems() : pidsystems;
|
||||
|
||||
certificates =
|
||||
d.getCertificates() != null && compareTrust(this, e) < 0
|
||||
certificates = d.getCertificates() != null && compareTrust(this, e) < 0
|
||||
? d.getCertificates()
|
||||
: certificates;
|
||||
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
|
||||
package eu.dnetlib.dhp.schema.oaf;
|
||||
|
||||
import java.io.Serializable;
|
||||
|
|
@ -94,8 +95,10 @@ public class ExternalReference implements Serializable {
|
|||
|
||||
@Override
|
||||
public boolean equals(Object o) {
|
||||
if (this == o) return true;
|
||||
if (o == null || getClass() != o.getClass()) return false;
|
||||
if (this == o)
|
||||
return true;
|
||||
if (o == null || getClass() != o.getClass())
|
||||
return false;
|
||||
ExternalReference that = (ExternalReference) o;
|
||||
return Objects.equals(sitename, that.sitename)
|
||||
&& Objects.equals(label, that.label)
|
||||
|
|
@ -109,7 +112,8 @@ public class ExternalReference implements Serializable {
|
|||
|
||||
@Override
|
||||
public int hashCode() {
|
||||
return Objects.hash(
|
||||
return Objects
|
||||
.hash(
|
||||
sitename, label, url, description, qualifier, refidentifier, query, dataInfo);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
|
||||
package eu.dnetlib.dhp.schema.oaf;
|
||||
|
||||
import java.io.Serializable;
|
||||
|
|
@ -57,8 +58,10 @@ public class ExtraInfo implements Serializable {
|
|||
|
||||
@Override
|
||||
public boolean equals(Object o) {
|
||||
if (this == o) return true;
|
||||
if (o == null || getClass() != o.getClass()) return false;
|
||||
if (this == o)
|
||||
return true;
|
||||
if (o == null || getClass() != o.getClass())
|
||||
return false;
|
||||
ExtraInfo extraInfo = (ExtraInfo) o;
|
||||
return Objects.equals(name, extraInfo.name)
|
||||
&& Objects.equals(typology, extraInfo.typology)
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
|
||||
package eu.dnetlib.dhp.schema.oaf;
|
||||
|
||||
import java.io.Serializable;
|
||||
|
|
@ -31,9 +32,12 @@ public class Field<T> implements Serializable {
|
|||
|
||||
@Override
|
||||
public boolean equals(Object obj) {
|
||||
if (this == obj) return true;
|
||||
if (obj == null) return false;
|
||||
if (getClass() != obj.getClass()) return false;
|
||||
if (this == obj)
|
||||
return true;
|
||||
if (obj == null)
|
||||
return false;
|
||||
if (getClass() != obj.getClass())
|
||||
return false;
|
||||
Field<T> other = (Field<T>) obj;
|
||||
return getValue().equals(other.getValue());
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
|
||||
package eu.dnetlib.dhp.schema.oaf;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonIgnore;
|
||||
|
|
@ -44,7 +45,8 @@ public class GeoLocation implements Serializable {
|
|||
public String toComparableString() {
|
||||
return isBlank()
|
||||
? ""
|
||||
: String.format(
|
||||
: String
|
||||
.format(
|
||||
"%s::%s%s",
|
||||
point != null ? point.toLowerCase() : "",
|
||||
box != null ? box.toLowerCase() : "",
|
||||
|
|
@ -58,9 +60,12 @@ public class GeoLocation implements Serializable {
|
|||
|
||||
@Override
|
||||
public boolean equals(Object obj) {
|
||||
if (this == obj) return true;
|
||||
if (obj == null) return false;
|
||||
if (getClass() != obj.getClass()) return false;
|
||||
if (this == obj)
|
||||
return true;
|
||||
if (obj == null)
|
||||
return false;
|
||||
if (getClass() != obj.getClass())
|
||||
return false;
|
||||
|
||||
GeoLocation other = (GeoLocation) obj;
|
||||
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
|
||||
package eu.dnetlib.dhp.schema.oaf;
|
||||
|
||||
import java.io.Serializable;
|
||||
|
|
@ -121,7 +122,8 @@ public class Instance implements Serializable {
|
|||
}
|
||||
|
||||
public String toComparableString() {
|
||||
return String.format(
|
||||
return String
|
||||
.format(
|
||||
"%s::%s::%s::%s",
|
||||
hostedby != null && hostedby.getKey() != null ? hostedby.getKey().toLowerCase() : "",
|
||||
accessright != null && accessright.getClassid() != null ? accessright.getClassid() : "",
|
||||
|
|
@ -136,9 +138,12 @@ public class Instance implements Serializable {
|
|||
|
||||
@Override
|
||||
public boolean equals(Object obj) {
|
||||
if (this == obj) return true;
|
||||
if (obj == null) return false;
|
||||
if (getClass() != obj.getClass()) return false;
|
||||
if (this == obj)
|
||||
return true;
|
||||
if (obj == null)
|
||||
return false;
|
||||
if (getClass() != obj.getClass())
|
||||
return false;
|
||||
|
||||
Instance other = (Instance) obj;
|
||||
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
|
||||
package eu.dnetlib.dhp.schema.oaf;
|
||||
|
||||
import java.io.Serializable;
|
||||
|
|
@ -127,8 +128,10 @@ public class Journal implements Serializable {
|
|||
|
||||
@Override
|
||||
public boolean equals(Object o) {
|
||||
if (this == o) return true;
|
||||
if (o == null || getClass() != o.getClass()) return false;
|
||||
if (this == o)
|
||||
return true;
|
||||
if (o == null || getClass() != o.getClass())
|
||||
return false;
|
||||
Journal journal = (Journal) o;
|
||||
return Objects.equals(name, journal.name)
|
||||
&& Objects.equals(issnPrinted, journal.issnPrinted)
|
||||
|
|
@ -146,7 +149,8 @@ public class Journal implements Serializable {
|
|||
|
||||
@Override
|
||||
public int hashCode() {
|
||||
return Objects.hash(
|
||||
return Objects
|
||||
.hash(
|
||||
name,
|
||||
issnPrinted,
|
||||
issnOnline,
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
|
||||
package eu.dnetlib.dhp.schema.oaf;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonIgnore;
|
||||
|
|
@ -39,7 +40,8 @@ public class KeyValue implements Serializable {
|
|||
public String toComparableString() {
|
||||
return isBlank()
|
||||
? ""
|
||||
: String.format(
|
||||
: String
|
||||
.format(
|
||||
"%s::%s",
|
||||
key != null ? key.toLowerCase() : "", value != null ? value.toLowerCase() : "");
|
||||
}
|
||||
|
|
@ -56,9 +58,12 @@ public class KeyValue implements Serializable {
|
|||
|
||||
@Override
|
||||
public boolean equals(Object obj) {
|
||||
if (this == obj) return true;
|
||||
if (obj == null) return false;
|
||||
if (getClass() != obj.getClass()) return false;
|
||||
if (this == obj)
|
||||
return true;
|
||||
if (obj == null)
|
||||
return false;
|
||||
if (getClass() != obj.getClass())
|
||||
return false;
|
||||
|
||||
KeyValue other = (KeyValue) obj;
|
||||
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
|
||||
package eu.dnetlib.dhp.schema.oaf;
|
||||
|
||||
import java.io.Serializable;
|
||||
|
|
@ -17,8 +18,10 @@ public class OAIProvenance implements Serializable {
|
|||
|
||||
@Override
|
||||
public boolean equals(Object o) {
|
||||
if (this == o) return true;
|
||||
if (o == null || getClass() != o.getClass()) return false;
|
||||
if (this == o)
|
||||
return true;
|
||||
if (o == null || getClass() != o.getClass())
|
||||
return false;
|
||||
OAIProvenance that = (OAIProvenance) o;
|
||||
return Objects.equals(originDescription, that.originDescription);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
|
||||
package eu.dnetlib.dhp.schema.oaf;
|
||||
|
||||
import java.io.Serializable;
|
||||
|
|
@ -37,11 +38,13 @@ public abstract class Oaf implements Serializable {
|
|||
}
|
||||
|
||||
public void mergeOAFDataInfo(Oaf e) {
|
||||
if (e.getDataInfo() != null && compareTrust(this, e) < 0) dataInfo = e.getDataInfo();
|
||||
if (e.getDataInfo() != null && compareTrust(this, e) < 0)
|
||||
dataInfo = e.getDataInfo();
|
||||
}
|
||||
|
||||
protected String extractTrust(Oaf e) {
|
||||
if (e == null || e.getDataInfo() == null || e.getDataInfo().getTrust() == null) return "0.0";
|
||||
if (e == null || e.getDataInfo() == null || e.getDataInfo().getTrust() == null)
|
||||
return "0.0";
|
||||
return e.getDataInfo().getTrust();
|
||||
}
|
||||
|
||||
|
|
@ -51,8 +54,10 @@ public abstract class Oaf implements Serializable {
|
|||
|
||||
@Override
|
||||
public boolean equals(Object o) {
|
||||
if (this == o) return true;
|
||||
if (o == null || getClass() != o.getClass()) return false;
|
||||
if (this == o)
|
||||
return true;
|
||||
if (o == null || getClass() != o.getClass())
|
||||
return false;
|
||||
Oaf oaf = (Oaf) o;
|
||||
return Objects.equals(dataInfo, oaf.dataInfo)
|
||||
&& Objects.equals(lastupdatetimestamp, oaf.lastupdatetimestamp);
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
|
||||
package eu.dnetlib.dhp.schema.oaf;
|
||||
|
||||
import java.io.Serializable;
|
||||
|
|
@ -78,7 +79,8 @@ public abstract class OafEntity extends Oaf implements Serializable {
|
|||
|
||||
public void mergeFrom(OafEntity e) {
|
||||
|
||||
if (e == null) return;
|
||||
if (e == null)
|
||||
return;
|
||||
|
||||
originalId = mergeLists(originalId, e.getOriginalId());
|
||||
|
||||
|
|
@ -100,7 +102,8 @@ public abstract class OafEntity extends Oaf implements Serializable {
|
|||
|
||||
protected <T> List<T> mergeLists(final List<T>... lists) {
|
||||
|
||||
return Arrays.stream(lists)
|
||||
return Arrays
|
||||
.stream(lists)
|
||||
.filter(Objects::nonNull)
|
||||
.flatMap(List::stream)
|
||||
.distinct()
|
||||
|
|
@ -109,9 +112,12 @@ public abstract class OafEntity extends Oaf implements Serializable {
|
|||
|
||||
@Override
|
||||
public boolean equals(Object o) {
|
||||
if (this == o) return true;
|
||||
if (o == null || getClass() != o.getClass()) return false;
|
||||
if (!super.equals(o)) return false;
|
||||
if (this == o)
|
||||
return true;
|
||||
if (o == null || getClass() != o.getClass())
|
||||
return false;
|
||||
if (!super.equals(o))
|
||||
return false;
|
||||
OafEntity oafEntity = (OafEntity) o;
|
||||
return Objects.equals(id, oafEntity.id);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
|
||||
package eu.dnetlib.dhp.schema.oaf;
|
||||
|
||||
import java.io.Serializable;
|
||||
|
|
@ -175,50 +176,38 @@ public class Organization extends OafEntity implements Serializable {
|
|||
}
|
||||
|
||||
final Organization o = (Organization) e;
|
||||
legalshortname =
|
||||
o.getLegalshortname() != null && compareTrust(this, e) < 0
|
||||
legalshortname = o.getLegalshortname() != null && compareTrust(this, e) < 0
|
||||
? o.getLegalshortname()
|
||||
: legalshortname;
|
||||
legalname =
|
||||
o.getLegalname() != null && compareTrust(this, e) < 0 ? o.getLegalname() : legalname;
|
||||
legalname = o.getLegalname() != null && compareTrust(this, e) < 0 ? o.getLegalname() : legalname;
|
||||
alternativeNames = mergeLists(o.getAlternativeNames(), alternativeNames);
|
||||
websiteurl =
|
||||
o.getWebsiteurl() != null && compareTrust(this, e) < 0 ? o.getWebsiteurl() : websiteurl;
|
||||
websiteurl = o.getWebsiteurl() != null && compareTrust(this, e) < 0 ? o.getWebsiteurl() : websiteurl;
|
||||
logourl = o.getLogourl() != null && compareTrust(this, e) < 0 ? o.getLogourl() : logourl;
|
||||
eclegalbody =
|
||||
o.getEclegalbody() != null && compareTrust(this, e) < 0 ? o.getEclegalbody() : eclegalbody;
|
||||
eclegalperson =
|
||||
o.getEclegalperson() != null && compareTrust(this, e) < 0
|
||||
eclegalbody = o.getEclegalbody() != null && compareTrust(this, e) < 0 ? o.getEclegalbody() : eclegalbody;
|
||||
eclegalperson = o.getEclegalperson() != null && compareTrust(this, e) < 0
|
||||
? o.getEclegalperson()
|
||||
: eclegalperson;
|
||||
ecnonprofit =
|
||||
o.getEcnonprofit() != null && compareTrust(this, e) < 0 ? o.getEcnonprofit() : ecnonprofit;
|
||||
ecresearchorganization =
|
||||
o.getEcresearchorganization() != null && compareTrust(this, e) < 0
|
||||
ecnonprofit = o.getEcnonprofit() != null && compareTrust(this, e) < 0 ? o.getEcnonprofit() : ecnonprofit;
|
||||
ecresearchorganization = o.getEcresearchorganization() != null && compareTrust(this, e) < 0
|
||||
? o.getEcresearchorganization()
|
||||
: ecresearchorganization;
|
||||
echighereducation =
|
||||
o.getEchighereducation() != null && compareTrust(this, e) < 0
|
||||
echighereducation = o.getEchighereducation() != null && compareTrust(this, e) < 0
|
||||
? o.getEchighereducation()
|
||||
: echighereducation;
|
||||
ecinternationalorganizationeurinterests =
|
||||
o.getEcinternationalorganizationeurinterests() != null && compareTrust(this, e) < 0
|
||||
ecinternationalorganizationeurinterests = o.getEcinternationalorganizationeurinterests() != null
|
||||
&& compareTrust(this, e) < 0
|
||||
? o.getEcinternationalorganizationeurinterests()
|
||||
: ecinternationalorganizationeurinterests;
|
||||
ecinternationalorganization =
|
||||
o.getEcinternationalorganization() != null && compareTrust(this, e) < 0
|
||||
ecinternationalorganization = o.getEcinternationalorganization() != null && compareTrust(this, e) < 0
|
||||
? o.getEcinternationalorganization()
|
||||
: ecinternationalorganization;
|
||||
ecenterprise =
|
||||
o.getEcenterprise() != null && compareTrust(this, e) < 0
|
||||
ecenterprise = o.getEcenterprise() != null && compareTrust(this, e) < 0
|
||||
? o.getEcenterprise()
|
||||
: ecenterprise;
|
||||
ecsmevalidated =
|
||||
o.getEcsmevalidated() != null && compareTrust(this, e) < 0
|
||||
ecsmevalidated = o.getEcsmevalidated() != null && compareTrust(this, e) < 0
|
||||
? o.getEcsmevalidated()
|
||||
: ecsmevalidated;
|
||||
ecnutscode =
|
||||
o.getEcnutscode() != null && compareTrust(this, e) < 0 ? o.getEcnutscode() : ecnutscode;
|
||||
ecnutscode = o.getEcnutscode() != null && compareTrust(this, e) < 0 ? o.getEcnutscode() : ecnutscode;
|
||||
country = o.getCountry() != null && compareTrust(this, e) < 0 ? o.getCountry() : country;
|
||||
mergeOAFDataInfo(o);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
|
||||
package eu.dnetlib.dhp.schema.oaf;
|
||||
|
||||
import java.io.Serializable;
|
||||
|
|
@ -67,8 +68,10 @@ public class OriginDescription implements Serializable {
|
|||
|
||||
@Override
|
||||
public boolean equals(Object o) {
|
||||
if (this == o) return true;
|
||||
if (o == null || getClass() != o.getClass()) return false;
|
||||
if (this == o)
|
||||
return true;
|
||||
if (o == null || getClass() != o.getClass())
|
||||
return false;
|
||||
OriginDescription that = (OriginDescription) o;
|
||||
return Objects.equals(harvestDate, that.harvestDate)
|
||||
&& Objects.equals(altered, that.altered)
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
|
||||
package eu.dnetlib.dhp.schema.oaf;
|
||||
|
||||
import eu.dnetlib.dhp.schema.common.ModelConstants;
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
|
||||
package eu.dnetlib.dhp.schema.oaf;
|
||||
|
||||
import java.io.Serializable;
|
||||
|
|
@ -275,63 +276,48 @@ public class Project extends OafEntity implements Serializable {
|
|||
|
||||
Project p = (Project) e;
|
||||
|
||||
websiteurl =
|
||||
p.getWebsiteurl() != null && compareTrust(this, e) < 0 ? p.getWebsiteurl() : websiteurl;
|
||||
websiteurl = p.getWebsiteurl() != null && compareTrust(this, e) < 0 ? p.getWebsiteurl() : websiteurl;
|
||||
code = p.getCode() != null && compareTrust(this, e) < 0 ? p.getCode() : code;
|
||||
acronym = p.getAcronym() != null && compareTrust(this, e) < 0 ? p.getAcronym() : acronym;
|
||||
title = p.getTitle() != null && compareTrust(this, e) < 0 ? p.getTitle() : title;
|
||||
startdate =
|
||||
p.getStartdate() != null && compareTrust(this, e) < 0 ? p.getStartdate() : startdate;
|
||||
startdate = p.getStartdate() != null && compareTrust(this, e) < 0 ? p.getStartdate() : startdate;
|
||||
enddate = p.getEnddate() != null && compareTrust(this, e) < 0 ? p.getEnddate() : enddate;
|
||||
callidentifier =
|
||||
p.getCallidentifier() != null && compareTrust(this, e) < 0
|
||||
callidentifier = p.getCallidentifier() != null && compareTrust(this, e) < 0
|
||||
? p.getCallidentifier()
|
||||
: callidentifier;
|
||||
keywords = p.getKeywords() != null && compareTrust(this, e) < 0 ? p.getKeywords() : keywords;
|
||||
duration = p.getDuration() != null && compareTrust(this, e) < 0 ? p.getDuration() : duration;
|
||||
ecsc39 = p.getEcsc39() != null && compareTrust(this, e) < 0 ? p.getEcsc39() : ecsc39;
|
||||
oamandatepublications =
|
||||
p.getOamandatepublications() != null && compareTrust(this, e) < 0
|
||||
oamandatepublications = p.getOamandatepublications() != null && compareTrust(this, e) < 0
|
||||
? p.getOamandatepublications()
|
||||
: oamandatepublications;
|
||||
ecarticle29_3 =
|
||||
p.getEcarticle29_3() != null && compareTrust(this, e) < 0
|
||||
ecarticle29_3 = p.getEcarticle29_3() != null && compareTrust(this, e) < 0
|
||||
? p.getEcarticle29_3()
|
||||
: ecarticle29_3;
|
||||
subjects = mergeLists(subjects, p.getSubjects());
|
||||
fundingtree = mergeLists(fundingtree, p.getFundingtree());
|
||||
contracttype =
|
||||
p.getContracttype() != null && compareTrust(this, e) < 0
|
||||
contracttype = p.getContracttype() != null && compareTrust(this, e) < 0
|
||||
? p.getContracttype()
|
||||
: contracttype;
|
||||
optional1 =
|
||||
p.getOptional1() != null && compareTrust(this, e) < 0 ? p.getOptional1() : optional1;
|
||||
optional2 =
|
||||
p.getOptional2() != null && compareTrust(this, e) < 0 ? p.getOptional2() : optional2;
|
||||
jsonextrainfo =
|
||||
p.getJsonextrainfo() != null && compareTrust(this, e) < 0
|
||||
optional1 = p.getOptional1() != null && compareTrust(this, e) < 0 ? p.getOptional1() : optional1;
|
||||
optional2 = p.getOptional2() != null && compareTrust(this, e) < 0 ? p.getOptional2() : optional2;
|
||||
jsonextrainfo = p.getJsonextrainfo() != null && compareTrust(this, e) < 0
|
||||
? p.getJsonextrainfo()
|
||||
: jsonextrainfo;
|
||||
contactfullname =
|
||||
p.getContactfullname() != null && compareTrust(this, e) < 0
|
||||
contactfullname = p.getContactfullname() != null && compareTrust(this, e) < 0
|
||||
? p.getContactfullname()
|
||||
: contactfullname;
|
||||
contactfax =
|
||||
p.getContactfax() != null && compareTrust(this, e) < 0 ? p.getContactfax() : contactfax;
|
||||
contactphone =
|
||||
p.getContactphone() != null && compareTrust(this, e) < 0
|
||||
contactfax = p.getContactfax() != null && compareTrust(this, e) < 0 ? p.getContactfax() : contactfax;
|
||||
contactphone = p.getContactphone() != null && compareTrust(this, e) < 0
|
||||
? p.getContactphone()
|
||||
: contactphone;
|
||||
contactemail =
|
||||
p.getContactemail() != null && compareTrust(this, e) < 0
|
||||
contactemail = p.getContactemail() != null && compareTrust(this, e) < 0
|
||||
? p.getContactemail()
|
||||
: contactemail;
|
||||
summary = p.getSummary() != null && compareTrust(this, e) < 0 ? p.getSummary() : summary;
|
||||
currency = p.getCurrency() != null && compareTrust(this, e) < 0 ? p.getCurrency() : currency;
|
||||
totalcost =
|
||||
p.getTotalcost() != null && compareTrust(this, e) < 0 ? p.getTotalcost() : totalcost;
|
||||
fundedamount =
|
||||
p.getFundedamount() != null && compareTrust(this, e) < 0
|
||||
totalcost = p.getTotalcost() != null && compareTrust(this, e) < 0 ? p.getTotalcost() : totalcost;
|
||||
fundedamount = p.getFundedamount() != null && compareTrust(this, e) < 0
|
||||
? p.getFundedamount()
|
||||
: fundedamount;
|
||||
mergeOAFDataInfo(e);
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
|
||||
package eu.dnetlib.dhp.schema.oaf;
|
||||
|
||||
import eu.dnetlib.dhp.schema.common.ModelConstants;
|
||||
|
|
@ -30,7 +31,8 @@ public class Publication extends Result implements Serializable {
|
|||
|
||||
Publication p = (Publication) e;
|
||||
|
||||
if (p.getJournal() != null && compareTrust(this, e) < 0) journal = p.getJournal();
|
||||
if (p.getJournal() != null && compareTrust(this, e) < 0)
|
||||
journal = p.getJournal();
|
||||
mergeOAFDataInfo(e);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
|
||||
package eu.dnetlib.dhp.schema.oaf;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonIgnore;
|
||||
|
|
@ -46,7 +47,8 @@ public class Qualifier implements Serializable {
|
|||
public String toComparableString() {
|
||||
return isBlank()
|
||||
? ""
|
||||
: String.format(
|
||||
: String
|
||||
.format(
|
||||
"%s::%s::%s::%s",
|
||||
classid != null ? classid : "",
|
||||
classname != null ? classname : "",
|
||||
|
|
@ -69,9 +71,12 @@ public class Qualifier implements Serializable {
|
|||
|
||||
@Override
|
||||
public boolean equals(Object obj) {
|
||||
if (this == obj) return true;
|
||||
if (obj == null) return false;
|
||||
if (getClass() != obj.getClass()) return false;
|
||||
if (this == obj)
|
||||
return true;
|
||||
if (obj == null)
|
||||
return false;
|
||||
if (getClass() != obj.getClass())
|
||||
return false;
|
||||
|
||||
Qualifier other = (Qualifier) obj;
|
||||
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
|
||||
package eu.dnetlib.dhp.schema.oaf;
|
||||
|
||||
import static com.google.common.base.Preconditions.checkArgument;
|
||||
|
|
@ -68,11 +69,14 @@ public class Relation extends Oaf {
|
|||
checkArgument(Objects.equals(getRelClass(), r.getRelClass()), "relClass(es) must be equal");
|
||||
|
||||
setCollectedfrom(
|
||||
Stream.concat(
|
||||
Optional.ofNullable(getCollectedfrom())
|
||||
Stream
|
||||
.concat(
|
||||
Optional
|
||||
.ofNullable(getCollectedfrom())
|
||||
.map(Collection::stream)
|
||||
.orElse(Stream.empty()),
|
||||
Optional.ofNullable(r.getCollectedfrom())
|
||||
Optional
|
||||
.ofNullable(r.getCollectedfrom())
|
||||
.map(Collection::stream)
|
||||
.orElse(Stream.empty()))
|
||||
.distinct() // relies on KeyValue.equals
|
||||
|
|
@ -81,8 +85,10 @@ public class Relation extends Oaf {
|
|||
|
||||
@Override
|
||||
public boolean equals(Object o) {
|
||||
if (this == o) return true;
|
||||
if (o == null || getClass() != o.getClass()) return false;
|
||||
if (this == o)
|
||||
return true;
|
||||
if (o == null || getClass() != o.getClass())
|
||||
return false;
|
||||
Relation relation = (Relation) o;
|
||||
return relType.equals(relation.relType)
|
||||
&& subRelType.equals(relation.subRelType)
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
|
||||
package eu.dnetlib.dhp.schema.oaf;
|
||||
|
||||
import java.io.Serializable;
|
||||
|
|
@ -233,9 +234,11 @@ public class Result extends OafEntity implements Serializable {
|
|||
if (r.getBestaccessright() != null && compareTrust(this, r) < 0)
|
||||
bestaccessright = r.getBestaccessright();
|
||||
|
||||
if (r.getResulttype() != null && compareTrust(this, r) < 0) resulttype = r.getResulttype();
|
||||
if (r.getResulttype() != null && compareTrust(this, r) < 0)
|
||||
resulttype = r.getResulttype();
|
||||
|
||||
if (r.getLanguage() != null && compareTrust(this, r) < 0) language = r.getLanguage();
|
||||
if (r.getLanguage() != null && compareTrust(this, r) < 0)
|
||||
language = r.getLanguage();
|
||||
|
||||
country = mergeLists(country, r.getCountry());
|
||||
|
||||
|
|
@ -247,7 +250,8 @@ public class Result extends OafEntity implements Serializable {
|
|||
|
||||
description = longestLists(description, r.getDescription());
|
||||
|
||||
if (r.getPublisher() != null && compareTrust(this, r) < 0) publisher = r.getPublisher();
|
||||
if (r.getPublisher() != null && compareTrust(this, r) < 0)
|
||||
publisher = r.getPublisher();
|
||||
|
||||
if (r.getEmbargoenddate() != null && compareTrust(this, r) < 0)
|
||||
embargoenddate = r.getEmbargoenddate();
|
||||
|
|
@ -260,7 +264,8 @@ public class Result extends OafEntity implements Serializable {
|
|||
|
||||
contributor = mergeLists(contributor, r.getContributor());
|
||||
|
||||
if (r.getResourcetype() != null) resourcetype = r.getResourcetype();
|
||||
if (r.getResourcetype() != null)
|
||||
resourcetype = r.getResourcetype();
|
||||
|
||||
coverage = mergeLists(coverage, r.getCoverage());
|
||||
|
||||
|
|
@ -270,16 +275,17 @@ public class Result extends OafEntity implements Serializable {
|
|||
}
|
||||
|
||||
private List<Field<String>> longestLists(List<Field<String>> a, List<Field<String>> b) {
|
||||
if (a == null || b == null) return a == null ? b : a;
|
||||
if (a == null || b == null)
|
||||
return a == null ? b : a;
|
||||
if (a.size() == b.size()) {
|
||||
int msa =
|
||||
a.stream()
|
||||
int msa = a
|
||||
.stream()
|
||||
.filter(i -> i.getValue() != null)
|
||||
.map(i -> i.getValue().length())
|
||||
.max(Comparator.naturalOrder())
|
||||
.orElse(0);
|
||||
int msb =
|
||||
b.stream()
|
||||
int msb = b
|
||||
.stream()
|
||||
.filter(i -> i.getValue() != null)
|
||||
.map(i -> i.getValue().length())
|
||||
.max(Comparator.naturalOrder())
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
|
||||
package eu.dnetlib.dhp.schema.oaf;
|
||||
|
||||
import eu.dnetlib.dhp.schema.common.ModelConstants;
|
||||
|
|
@ -63,13 +64,11 @@ public class Software extends Result implements Serializable {
|
|||
|
||||
license = mergeLists(license, s.getLicense());
|
||||
|
||||
codeRepositoryUrl =
|
||||
s.getCodeRepositoryUrl() != null && compareTrust(this, s) < 0
|
||||
codeRepositoryUrl = s.getCodeRepositoryUrl() != null && compareTrust(this, s) < 0
|
||||
? s.getCodeRepositoryUrl()
|
||||
: codeRepositoryUrl;
|
||||
|
||||
programmingLanguage =
|
||||
s.getProgrammingLanguage() != null && compareTrust(this, s) < 0
|
||||
programmingLanguage = s.getProgrammingLanguage() != null && compareTrust(this, s) < 0
|
||||
? s.getProgrammingLanguage()
|
||||
: programmingLanguage;
|
||||
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
|
||||
package eu.dnetlib.dhp.schema.oaf;
|
||||
|
||||
import java.io.Serializable;
|
||||
|
|
@ -45,9 +46,12 @@ public class StructuredProperty implements Serializable {
|
|||
|
||||
@Override
|
||||
public boolean equals(Object obj) {
|
||||
if (this == obj) return true;
|
||||
if (obj == null) return false;
|
||||
if (getClass() != obj.getClass()) return false;
|
||||
if (this == obj)
|
||||
return true;
|
||||
if (obj == null)
|
||||
return false;
|
||||
if (getClass() != obj.getClass())
|
||||
return false;
|
||||
|
||||
StructuredProperty other = (StructuredProperty) obj;
|
||||
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
|
||||
package eu.dnetlib.dhp.schema.scholexplorer;
|
||||
|
||||
import eu.dnetlib.dhp.schema.oaf.Dataset;
|
||||
|
|
@ -46,7 +47,8 @@ public class DLIDataset extends Dataset {
|
|||
DLIDataset p = (DLIDataset) e;
|
||||
if (StringUtils.isBlank(completionStatus) && StringUtils.isNotBlank(p.completionStatus))
|
||||
completionStatus = p.completionStatus;
|
||||
if ("complete".equalsIgnoreCase(p.completionStatus)) completionStatus = "complete";
|
||||
if ("complete".equalsIgnoreCase(p.completionStatus))
|
||||
completionStatus = "complete";
|
||||
dlicollectedfrom = mergeProvenance(dlicollectedfrom, p.getDlicollectedfrom());
|
||||
}
|
||||
|
||||
|
|
@ -54,7 +56,8 @@ public class DLIDataset extends Dataset {
|
|||
final List<ProvenaceInfo> a, final List<ProvenaceInfo> b) {
|
||||
Map<String, ProvenaceInfo> result = new HashMap<>();
|
||||
if (a != null)
|
||||
a.forEach(
|
||||
a
|
||||
.forEach(
|
||||
p -> {
|
||||
if (p != null && StringUtils.isNotBlank(p.getId()) && result.containsKey(p.getId())) {
|
||||
if ("incomplete".equalsIgnoreCase(result.get(p.getId()).getCompletionStatus())
|
||||
|
|
@ -66,7 +69,8 @@ public class DLIDataset extends Dataset {
|
|||
result.put(p.getId(), p);
|
||||
});
|
||||
if (b != null)
|
||||
b.forEach(
|
||||
b
|
||||
.forEach(
|
||||
p -> {
|
||||
if (p != null && StringUtils.isNotBlank(p.getId()) && result.containsKey(p.getId())) {
|
||||
if ("incomplete".equalsIgnoreCase(result.get(p.getId()).getCompletionStatus())
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
|
||||
package eu.dnetlib.dhp.schema.scholexplorer;
|
||||
|
||||
import eu.dnetlib.dhp.schema.oaf.OafEntity;
|
||||
|
|
@ -44,7 +45,8 @@ public class DLIPublication extends Publication implements Serializable {
|
|||
DLIPublication p = (DLIPublication) e;
|
||||
if (StringUtils.isBlank(completionStatus) && StringUtils.isNotBlank(p.completionStatus))
|
||||
completionStatus = p.completionStatus;
|
||||
if ("complete".equalsIgnoreCase(p.completionStatus)) completionStatus = "complete";
|
||||
if ("complete".equalsIgnoreCase(p.completionStatus))
|
||||
completionStatus = "complete";
|
||||
dlicollectedfrom = mergeProvenance(dlicollectedfrom, p.getDlicollectedfrom());
|
||||
}
|
||||
|
||||
|
|
@ -52,7 +54,8 @@ public class DLIPublication extends Publication implements Serializable {
|
|||
final List<ProvenaceInfo> a, final List<ProvenaceInfo> b) {
|
||||
Map<String, ProvenaceInfo> result = new HashMap<>();
|
||||
if (a != null)
|
||||
a.forEach(
|
||||
a
|
||||
.forEach(
|
||||
p -> {
|
||||
if (p != null && StringUtils.isNotBlank(p.getId()) && result.containsKey(p.getId())) {
|
||||
if ("incomplete".equalsIgnoreCase(result.get(p.getId()).getCompletionStatus())
|
||||
|
|
@ -64,7 +67,8 @@ public class DLIPublication extends Publication implements Serializable {
|
|||
result.put(p.getId(), p);
|
||||
});
|
||||
if (b != null)
|
||||
b.forEach(
|
||||
b
|
||||
.forEach(
|
||||
p -> {
|
||||
if (p != null && StringUtils.isNotBlank(p.getId()) && result.containsKey(p.getId())) {
|
||||
if ("incomplete".equalsIgnoreCase(result.get(p.getId()).getCompletionStatus())
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
|
||||
package eu.dnetlib.dhp.schema.scholexplorer;
|
||||
|
||||
import eu.dnetlib.dhp.schema.oaf.Relation;
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
|
||||
package eu.dnetlib.dhp.schema.scholexplorer;
|
||||
|
||||
import eu.dnetlib.dhp.schema.oaf.Oaf;
|
||||
|
|
@ -72,7 +73,8 @@ public class DLIUnknown extends Oaf implements Serializable {
|
|||
}
|
||||
|
||||
public void mergeFrom(DLIUnknown p) {
|
||||
if ("complete".equalsIgnoreCase(p.completionStatus)) completionStatus = "complete";
|
||||
if ("complete".equalsIgnoreCase(p.completionStatus))
|
||||
completionStatus = "complete";
|
||||
dlicollectedfrom = mergeProvenance(dlicollectedfrom, p.getDlicollectedfrom());
|
||||
}
|
||||
|
||||
|
|
@ -80,7 +82,8 @@ public class DLIUnknown extends Oaf implements Serializable {
|
|||
final List<ProvenaceInfo> a, final List<ProvenaceInfo> b) {
|
||||
Map<String, ProvenaceInfo> result = new HashMap<>();
|
||||
if (a != null)
|
||||
a.forEach(
|
||||
a
|
||||
.forEach(
|
||||
p -> {
|
||||
if (p != null && StringUtils.isNotBlank(p.getId()) && result.containsKey(p.getId())) {
|
||||
if ("incomplete".equalsIgnoreCase(result.get(p.getId()).getCompletionStatus())
|
||||
|
|
@ -92,7 +95,8 @@ public class DLIUnknown extends Oaf implements Serializable {
|
|||
result.put(p.getId(), p);
|
||||
});
|
||||
if (b != null)
|
||||
b.forEach(
|
||||
b
|
||||
.forEach(
|
||||
p -> {
|
||||
if (p != null && StringUtils.isNotBlank(p.getId()) && result.containsKey(p.getId())) {
|
||||
if ("incomplete".equalsIgnoreCase(result.get(p.getId()).getCompletionStatus())
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
|
||||
package eu.dnetlib.dhp.schema.scholexplorer;
|
||||
|
||||
import java.io.Serializable;
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
|
||||
package eu.dnetlib.dhp.schema.action;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.*;
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
|
||||
package eu.dnetlib.dhp.schema.common;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
|
||||
package eu.dnetlib.dhp.schema.oaf;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.*;
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
|
||||
package eu.dnetlib.dhp.schema.scholexplorer;
|
||||
|
||||
import com.fasterxml.jackson.core.JsonProcessingException;
|
||||
|
|
@ -22,13 +23,17 @@ public class DLItest {
|
|||
a1.setCompletionStatus("complete");
|
||||
|
||||
DLIPublication a = new DLIPublication();
|
||||
a.setPid(
|
||||
Arrays.asList(
|
||||
a
|
||||
.setPid(
|
||||
Arrays
|
||||
.asList(
|
||||
createSP("10.11", "doi", "dnet:pid_types"),
|
||||
createSP("123456", "pdb", "dnet:pid_types")));
|
||||
a.setTitle(Collections.singletonList(createSP("A Title", "title", "dnetTitle")));
|
||||
a.setDlicollectedfrom(
|
||||
Arrays.asList(
|
||||
a
|
||||
.setDlicollectedfrom(
|
||||
Arrays
|
||||
.asList(
|
||||
createCollectedFrom("dct", "datacite", "complete"),
|
||||
createCollectedFrom("dct", "datacite", "incomplete")));
|
||||
a.setCompletionStatus("incomplete");
|
||||
|
|
@ -42,8 +47,7 @@ public class DLItest {
|
|||
@Test
|
||||
public void testDeserialization() throws IOException {
|
||||
|
||||
final String json =
|
||||
"{\"dataInfo\":{\"invisible\":false,\"inferred\":null,\"deletedbyinference\":false,\"trust\":\"0.9\",\"inferenceprovenance\":null,\"provenanceaction\":null},\"lastupdatetimestamp\":null,\"id\":\"60|bd9352547098929a394655ad1a44a479\",\"originalId\":[\"bd9352547098929a394655ad1a44a479\"],\"collectedfrom\":[{\"key\":\"dli_________::datacite\",\"value\":\"Datasets in Datacite\",\"dataInfo\":null,\"blank\":false}],\"pid\":[{\"value\":\"10.7925/DRS1.DUCHAS_5078760\",\"qualifier\":{\"classid\":\"doi\",\"classname\":\"doi\",\"schemeid\":\"dnet:pid_types\",\"schemename\":\"dnet:pid_types\",\"blank\":false},\"dataInfo\":null}],\"dateofcollection\":\"2020-01-09T08:29:31.885Z\",\"dateoftransformation\":null,\"extraInfo\":null,\"oaiprovenance\":null,\"author\":[{\"fullname\":\"Cathail, S. Ó\",\"name\":null,\"surname\":null,\"rank\":null,\"pid\":null,\"affiliation\":null},{\"fullname\":\"Donnell, Breda Mc\",\"name\":null,\"surname\":null,\"rank\":null,\"pid\":null,\"affiliation\":null},{\"fullname\":\"Ireland. Department of Arts, Culture, and the Gaeltacht\",\"name\":null,\"surname\":null,\"rank\":null,\"pid\":null,\"affiliation\":null},{\"fullname\":\"University College Dublin\",\"name\":null,\"surname\":null,\"rank\":null,\"pid\":null,\"affiliation\":null},{\"fullname\":\"National Folklore Foundation\",\"name\":null,\"surname\":null,\"rank\":null,\"pid\":null,\"affiliation\":null},{\"fullname\":\"Cathail, S. Ó\",\"name\":null,\"surname\":null,\"rank\":null,\"pid\":null,\"affiliation\":null},{\"fullname\":\"Donnell, Breda Mc\",\"name\":null,\"surname\":null,\"rank\":null,\"pid\":null,\"affiliation\":null}],\"resulttype\":null,\"language\":null,\"country\":null,\"subject\":[{\"value\":\"Recreation\",\"qualifier\":{\"classid\":\"dnet:subject\",\"classname\":\"dnet:subject\",\"schemeid\":\"unknown\",\"schemename\":\"unknown\",\"blank\":false},\"dataInfo\":null},{\"value\":\"Entertainments and recreational activities\",\"qualifier\":{\"classid\":\"dnet:subject\",\"classname\":\"dnet:subject\",\"schemeid\":\"unknown\",\"schemename\":\"unknown\",\"blank\":false},\"dataInfo\":null},{\"value\":\"Siamsaíocht agus caitheamh aimsire\",\"qualifier\":{\"classid\":\"dnet:subject\",\"classname\":\"dnet:subject\",\"schemeid\":\"unknown\",\"schemename\":\"unknown\",\"blank\":false},\"dataInfo\":null}],\"title\":[{\"value\":\"Games We Play\",\"qualifier\":null,\"dataInfo\":null}],\"relevantdate\":[{\"value\":\"1938-09-28\",\"qualifier\":{\"classid\":\"date\",\"classname\":\"date\",\"schemeid\":\"dnet::date\",\"schemename\":\"dnet::date\",\"blank\":false},\"dataInfo\":null}],\"description\":[{\"value\":\"Story collected by Breda Mc Donnell, a student at Tenure school (Tinure, Co. Louth) (no informant identified).\",\"dataInfo\":null}],\"dateofacceptance\":null,\"publisher\":{\"value\":\"University College Dublin\",\"dataInfo\":null},\"embargoenddate\":null,\"source\":null,\"fulltext\":null,\"format\":null,\"contributor\":null,\"resourcetype\":null,\"coverage\":null,\"refereed\":null,\"context\":null,\"processingchargeamount\":null,\"processingchargecurrency\":null,\"externalReference\":null,\"instance\":[],\"storagedate\":null,\"device\":null,\"size\":null,\"version\":null,\"lastmetadataupdate\":null,\"metadataversionnumber\":null,\"geolocation\":null,\"dlicollectedfrom\":[{\"id\":\"dli_________::datacite\",\"name\":\"Datasets in Datacite\",\"completionStatus\":\"complete\",\"collectionMode\":\"resolved\"}],\"completionStatus\":\"complete\"}";
|
||||
final String json = "{\"dataInfo\":{\"invisible\":false,\"inferred\":null,\"deletedbyinference\":false,\"trust\":\"0.9\",\"inferenceprovenance\":null,\"provenanceaction\":null},\"lastupdatetimestamp\":null,\"id\":\"60|bd9352547098929a394655ad1a44a479\",\"originalId\":[\"bd9352547098929a394655ad1a44a479\"],\"collectedfrom\":[{\"key\":\"dli_________::datacite\",\"value\":\"Datasets in Datacite\",\"dataInfo\":null,\"blank\":false}],\"pid\":[{\"value\":\"10.7925/DRS1.DUCHAS_5078760\",\"qualifier\":{\"classid\":\"doi\",\"classname\":\"doi\",\"schemeid\":\"dnet:pid_types\",\"schemename\":\"dnet:pid_types\",\"blank\":false},\"dataInfo\":null}],\"dateofcollection\":\"2020-01-09T08:29:31.885Z\",\"dateoftransformation\":null,\"extraInfo\":null,\"oaiprovenance\":null,\"author\":[{\"fullname\":\"Cathail, S. Ó\",\"name\":null,\"surname\":null,\"rank\":null,\"pid\":null,\"affiliation\":null},{\"fullname\":\"Donnell, Breda Mc\",\"name\":null,\"surname\":null,\"rank\":null,\"pid\":null,\"affiliation\":null},{\"fullname\":\"Ireland. Department of Arts, Culture, and the Gaeltacht\",\"name\":null,\"surname\":null,\"rank\":null,\"pid\":null,\"affiliation\":null},{\"fullname\":\"University College Dublin\",\"name\":null,\"surname\":null,\"rank\":null,\"pid\":null,\"affiliation\":null},{\"fullname\":\"National Folklore Foundation\",\"name\":null,\"surname\":null,\"rank\":null,\"pid\":null,\"affiliation\":null},{\"fullname\":\"Cathail, S. Ó\",\"name\":null,\"surname\":null,\"rank\":null,\"pid\":null,\"affiliation\":null},{\"fullname\":\"Donnell, Breda Mc\",\"name\":null,\"surname\":null,\"rank\":null,\"pid\":null,\"affiliation\":null}],\"resulttype\":null,\"language\":null,\"country\":null,\"subject\":[{\"value\":\"Recreation\",\"qualifier\":{\"classid\":\"dnet:subject\",\"classname\":\"dnet:subject\",\"schemeid\":\"unknown\",\"schemename\":\"unknown\",\"blank\":false},\"dataInfo\":null},{\"value\":\"Entertainments and recreational activities\",\"qualifier\":{\"classid\":\"dnet:subject\",\"classname\":\"dnet:subject\",\"schemeid\":\"unknown\",\"schemename\":\"unknown\",\"blank\":false},\"dataInfo\":null},{\"value\":\"Siamsaíocht agus caitheamh aimsire\",\"qualifier\":{\"classid\":\"dnet:subject\",\"classname\":\"dnet:subject\",\"schemeid\":\"unknown\",\"schemename\":\"unknown\",\"blank\":false},\"dataInfo\":null}],\"title\":[{\"value\":\"Games We Play\",\"qualifier\":null,\"dataInfo\":null}],\"relevantdate\":[{\"value\":\"1938-09-28\",\"qualifier\":{\"classid\":\"date\",\"classname\":\"date\",\"schemeid\":\"dnet::date\",\"schemename\":\"dnet::date\",\"blank\":false},\"dataInfo\":null}],\"description\":[{\"value\":\"Story collected by Breda Mc Donnell, a student at Tenure school (Tinure, Co. Louth) (no informant identified).\",\"dataInfo\":null}],\"dateofacceptance\":null,\"publisher\":{\"value\":\"University College Dublin\",\"dataInfo\":null},\"embargoenddate\":null,\"source\":null,\"fulltext\":null,\"format\":null,\"contributor\":null,\"resourcetype\":null,\"coverage\":null,\"refereed\":null,\"context\":null,\"processingchargeamount\":null,\"processingchargecurrency\":null,\"externalReference\":null,\"instance\":[],\"storagedate\":null,\"device\":null,\"size\":null,\"version\":null,\"lastmetadataupdate\":null,\"metadataversionnumber\":null,\"geolocation\":null,\"dlicollectedfrom\":[{\"id\":\"dli_________::datacite\",\"name\":\"Datasets in Datacite\",\"completionStatus\":\"complete\",\"collectionMode\":\"resolved\"}],\"completionStatus\":\"complete\"}";
|
||||
|
||||
ObjectMapper mapper = new ObjectMapper();
|
||||
mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
|
||||
package eu.dnetlib.dhp.actionmanager;
|
||||
|
||||
import com.google.common.base.Splitter;
|
||||
|
|
@ -24,8 +25,7 @@ import org.slf4j.LoggerFactory;
|
|||
|
||||
public class ISClient implements Serializable {
|
||||
|
||||
private static final Logger log =
|
||||
LoggerFactory.getLogger(PartitionActionSetsByPayloadTypeJob.class);
|
||||
private static final Logger log = LoggerFactory.getLogger(PartitionActionSetsByPayloadTypeJob.class);
|
||||
|
||||
private static final String INPUT_ACTION_SET_ID_SEPARATOR = ",";
|
||||
|
||||
|
|
@ -37,14 +37,16 @@ public class ISClient implements Serializable {
|
|||
|
||||
public List<String> getLatestRawsetPaths(String setIds) {
|
||||
|
||||
List<String> ids =
|
||||
Lists.newArrayList(
|
||||
Splitter.on(INPUT_ACTION_SET_ID_SEPARATOR)
|
||||
List<String> ids = Lists
|
||||
.newArrayList(
|
||||
Splitter
|
||||
.on(INPUT_ACTION_SET_ID_SEPARATOR)
|
||||
.omitEmptyStrings()
|
||||
.trimResults()
|
||||
.split(setIds));
|
||||
|
||||
return ids.stream()
|
||||
return ids
|
||||
.stream()
|
||||
.map(id -> getSet(isLookup, id))
|
||||
.map(as -> as.getPathToLatest())
|
||||
.collect(Collectors.toCollection(ArrayList::new));
|
||||
|
|
@ -52,8 +54,7 @@ public class ISClient implements Serializable {
|
|||
|
||||
private ActionManagerSet getSet(ISLookUpService isLookup, final String setId) {
|
||||
|
||||
final String q =
|
||||
"for $x in collection('/db/DRIVER/ActionManagerSetDSResources/ActionManagerSetDSResourceType') "
|
||||
final String q = "for $x in collection('/db/DRIVER/ActionManagerSetDSResources/ActionManagerSetDSResourceType') "
|
||||
+ "where $x//SET/@id = '"
|
||||
+ setId
|
||||
+ "' return $x";
|
||||
|
|
@ -78,7 +79,8 @@ public class ISClient implements Serializable {
|
|||
set.setId(doc.valueOf("//SET/@id").trim());
|
||||
set.setName(doc.valueOf("//SET").trim());
|
||||
set.setImpact(ImpactTypes.valueOf(doc.valueOf("//IMPACT").trim()));
|
||||
set.setLatest(
|
||||
set
|
||||
.setLatest(
|
||||
doc.valueOf("//RAW_SETS/LATEST/@id"),
|
||||
doc.valueOf("//RAW_SETS/LATEST/@creationDate"),
|
||||
doc.valueOf("//RAW_SETS/LATEST/@lastUpdate"));
|
||||
|
|
@ -87,7 +89,8 @@ public class ISClient implements Serializable {
|
|||
if (expiredNodes != null) {
|
||||
for (int i = 0; i < expiredNodes.size(); i++) {
|
||||
Element ex = (Element) expiredNodes.get(i);
|
||||
set.addExpired(
|
||||
set
|
||||
.addExpired(
|
||||
ex.attributeValue("id"),
|
||||
ex.attributeValue("creationDate"),
|
||||
ex.attributeValue("lastUpdate"));
|
||||
|
|
@ -114,8 +117,7 @@ public class ISClient implements Serializable {
|
|||
|
||||
private String queryServiceProperty(ISLookUpService isLookup, final String propertyName)
|
||||
throws ActionManagerException {
|
||||
final String q =
|
||||
"for $x in /RESOURCE_PROFILE[.//RESOURCE_TYPE/@value='ActionManagerServiceResourceType'] return $x//SERVICE_PROPERTIES/PROPERTY[./@ key='"
|
||||
final String q = "for $x in /RESOURCE_PROFILE[.//RESOURCE_TYPE/@value='ActionManagerServiceResourceType'] return $x//SERVICE_PROPERTIES/PROPERTY[./@ key='"
|
||||
+ propertyName
|
||||
+ "']/@value/string()";
|
||||
log.debug("quering for service property: " + q);
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
|
||||
package eu.dnetlib.dhp.actionmanager.migration;
|
||||
|
||||
import eu.dnetlib.data.proto.FieldTypeProtos.Qualifier;
|
||||
|
|
@ -8,38 +9,58 @@ public class LicenseComparator implements Comparator<Qualifier> {
|
|||
@Override
|
||||
public int compare(Qualifier left, Qualifier right) {
|
||||
|
||||
if (left == null && right == null) return 0;
|
||||
if (left == null) return 1;
|
||||
if (right == null) return -1;
|
||||
if (left == null && right == null)
|
||||
return 0;
|
||||
if (left == null)
|
||||
return 1;
|
||||
if (right == null)
|
||||
return -1;
|
||||
|
||||
String lClass = left.getClassid();
|
||||
String rClass = right.getClassid();
|
||||
|
||||
if (lClass.equals(rClass)) return 0;
|
||||
if (lClass.equals(rClass))
|
||||
return 0;
|
||||
|
||||
if (lClass.equals("OPEN SOURCE")) return -1;
|
||||
if (rClass.equals("OPEN SOURCE")) return 1;
|
||||
if (lClass.equals("OPEN SOURCE"))
|
||||
return -1;
|
||||
if (rClass.equals("OPEN SOURCE"))
|
||||
return 1;
|
||||
|
||||
if (lClass.equals("OPEN")) return -1;
|
||||
if (rClass.equals("OPEN")) return 1;
|
||||
if (lClass.equals("OPEN"))
|
||||
return -1;
|
||||
if (rClass.equals("OPEN"))
|
||||
return 1;
|
||||
|
||||
if (lClass.equals("6MONTHS")) return -1;
|
||||
if (rClass.equals("6MONTHS")) return 1;
|
||||
if (lClass.equals("6MONTHS"))
|
||||
return -1;
|
||||
if (rClass.equals("6MONTHS"))
|
||||
return 1;
|
||||
|
||||
if (lClass.equals("12MONTHS")) return -1;
|
||||
if (rClass.equals("12MONTHS")) return 1;
|
||||
if (lClass.equals("12MONTHS"))
|
||||
return -1;
|
||||
if (rClass.equals("12MONTHS"))
|
||||
return 1;
|
||||
|
||||
if (lClass.equals("EMBARGO")) return -1;
|
||||
if (rClass.equals("EMBARGO")) return 1;
|
||||
if (lClass.equals("EMBARGO"))
|
||||
return -1;
|
||||
if (rClass.equals("EMBARGO"))
|
||||
return 1;
|
||||
|
||||
if (lClass.equals("RESTRICTED")) return -1;
|
||||
if (rClass.equals("RESTRICTED")) return 1;
|
||||
if (lClass.equals("RESTRICTED"))
|
||||
return -1;
|
||||
if (rClass.equals("RESTRICTED"))
|
||||
return 1;
|
||||
|
||||
if (lClass.equals("CLOSED")) return -1;
|
||||
if (rClass.equals("CLOSED")) return 1;
|
||||
if (lClass.equals("CLOSED"))
|
||||
return -1;
|
||||
if (rClass.equals("CLOSED"))
|
||||
return 1;
|
||||
|
||||
if (lClass.equals("UNKNOWN")) return -1;
|
||||
if (rClass.equals("UNKNOWN")) return 1;
|
||||
if (lClass.equals("UNKNOWN"))
|
||||
return -1;
|
||||
if (rClass.equals("UNKNOWN"))
|
||||
return 1;
|
||||
|
||||
// Else (but unlikely), lexicographical ordering will do.
|
||||
return lClass.compareTo(rClass);
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
|
||||
package eu.dnetlib.dhp.actionmanager.migration;
|
||||
|
||||
import com.google.common.base.Splitter;
|
||||
|
|
@ -34,10 +35,11 @@ public class MigrateActionSet {
|
|||
private static final String RAWSET_PREFIX = "rawset_";
|
||||
|
||||
public static void main(String[] args) throws Exception {
|
||||
final ArgumentApplicationParser parser =
|
||||
new ArgumentApplicationParser(
|
||||
IOUtils.toString(
|
||||
MigrateActionSet.class.getResourceAsStream(
|
||||
final ArgumentApplicationParser parser = new ArgumentApplicationParser(
|
||||
IOUtils
|
||||
.toString(
|
||||
MigrateActionSet.class
|
||||
.getResourceAsStream(
|
||||
"/eu/dnetlib/dhp/actionmanager/migration/migrate_actionsets_parameters.json")));
|
||||
parser.parseArgument(args);
|
||||
|
||||
|
|
@ -68,8 +70,7 @@ public class MigrateActionSet {
|
|||
Configuration conf = getConfiguration(distcp_task_timeout, distcp_memory_mb, distcp_num_maps);
|
||||
FileSystem targetFS = FileSystem.get(conf);
|
||||
|
||||
Configuration sourceConf =
|
||||
getConfiguration(distcp_task_timeout, distcp_memory_mb, distcp_num_maps);
|
||||
Configuration sourceConf = getConfiguration(distcp_task_timeout, distcp_memory_mb, distcp_num_maps);
|
||||
sourceConf.set(FileSystem.FS_DEFAULT_NAME_KEY, sourceNN);
|
||||
FileSystem sourceFS = FileSystem.get(sourceConf);
|
||||
|
||||
|
|
@ -78,7 +79,8 @@ public class MigrateActionSet {
|
|||
List<Path> targetPaths = new ArrayList<>();
|
||||
|
||||
final List<Path> sourcePaths = getSourcePaths(sourceNN, isLookUp);
|
||||
log.info(
|
||||
log
|
||||
.info(
|
||||
"paths to process:\n{}",
|
||||
sourcePaths.stream().map(p -> p.toString()).collect(Collectors.joining("\n")));
|
||||
for (Path source : sourcePaths) {
|
||||
|
|
@ -87,8 +89,7 @@ public class MigrateActionSet {
|
|||
log.warn("skipping unexisting path: {}", source);
|
||||
} else {
|
||||
|
||||
LinkedList<String> pathQ =
|
||||
Lists.newLinkedList(Splitter.on(SEPARATOR).split(source.toUri().getPath()));
|
||||
LinkedList<String> pathQ = Lists.newLinkedList(Splitter.on(SEPARATOR).split(source.toUri().getPath()));
|
||||
|
||||
final String rawSet = pathQ.pollLast();
|
||||
log.info("got RAWSET: {}", rawSet);
|
||||
|
|
@ -97,8 +98,8 @@ public class MigrateActionSet {
|
|||
|
||||
final String actionSetDirectory = pathQ.pollLast();
|
||||
|
||||
final Path targetPath =
|
||||
new Path(targetNN + workDir + SEPARATOR + actionSetDirectory + SEPARATOR + rawSet);
|
||||
final Path targetPath = new Path(
|
||||
targetNN + workDir + SEPARATOR + actionSetDirectory + SEPARATOR + rawSet);
|
||||
|
||||
log.info("using TARGET PATH: {}", targetPath);
|
||||
|
||||
|
|
@ -115,7 +116,8 @@ public class MigrateActionSet {
|
|||
}
|
||||
}
|
||||
|
||||
props.setProperty(
|
||||
props
|
||||
.setProperty(
|
||||
TARGET_PATHS, targetPaths.stream().map(p -> p.toString()).collect(Collectors.joining(",")));
|
||||
File file = new File(System.getProperty("oozie.action.output.properties"));
|
||||
|
||||
|
|
@ -140,8 +142,8 @@ public class MigrateActionSet {
|
|||
op.preserve(DistCpOptions.FileAttribute.REPLICATION);
|
||||
op.preserve(DistCpOptions.FileAttribute.CHECKSUMTYPE);
|
||||
|
||||
int res =
|
||||
ToolRunner.run(
|
||||
int res = ToolRunner
|
||||
.run(
|
||||
new DistCp(conf, op),
|
||||
new String[] {
|
||||
"-Dmapred.task.timeout=" + distcp_task_timeout,
|
||||
|
|
@ -171,8 +173,7 @@ public class MigrateActionSet {
|
|||
|
||||
private List<Path> getSourcePaths(String sourceNN, ISLookUpService isLookUp)
|
||||
throws ISLookUpException {
|
||||
String XQUERY =
|
||||
"distinct-values(\n"
|
||||
String XQUERY = "distinct-values(\n"
|
||||
+ "let $basePath := collection('/db/DRIVER/ServiceResources/ActionManagerServiceResourceType')//SERVICE_PROPERTIES/PROPERTY[@key = 'basePath']/@value/string()\n"
|
||||
+ "for $x in collection('/db/DRIVER/ActionManagerSetDSResources/ActionManagerSetDSResourceType') \n"
|
||||
+ "let $setDir := $x//SET/@directory/string()\n"
|
||||
|
|
@ -180,7 +181,9 @@ public class MigrateActionSet {
|
|||
+ "return concat($basePath, '/', $setDir, '/', $rawSet))";
|
||||
|
||||
log.info(String.format("running xquery:\n%s", XQUERY));
|
||||
return isLookUp.quickSearchProfile(XQUERY).stream()
|
||||
return isLookUp
|
||||
.quickSearchProfile(XQUERY)
|
||||
.stream()
|
||||
.map(p -> sourceNN + p)
|
||||
.map(Path::new)
|
||||
.collect(Collectors.toList());
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
|
||||
package eu.dnetlib.dhp.actionmanager.migration;
|
||||
|
||||
import static eu.dnetlib.data.proto.KindProtos.Kind.entity;
|
||||
|
|
@ -46,7 +47,8 @@ public class ProtoConverter implements Serializable {
|
|||
rel.setRelType(r.getRelType().toString());
|
||||
rel.setSubRelType(r.getSubRelType().toString());
|
||||
rel.setRelClass(r.getRelClass());
|
||||
rel.setCollectedfrom(
|
||||
rel
|
||||
.setCollectedfrom(
|
||||
r.getCollectedfromCount() > 0
|
||||
? r.getCollectedfromList().stream().map(kv -> mapKV(kv)).collect(Collectors.toList())
|
||||
: null);
|
||||
|
|
@ -97,14 +99,16 @@ public class ProtoConverter implements Serializable {
|
|||
}
|
||||
|
||||
private static Organization convertOrganization(OafProtos.Oaf oaf) {
|
||||
final OrganizationProtos.Organization.Metadata m =
|
||||
oaf.getEntity().getOrganization().getMetadata();
|
||||
final OrganizationProtos.Organization.Metadata m = oaf.getEntity().getOrganization().getMetadata();
|
||||
final Organization org = setOaf(new Organization(), oaf);
|
||||
setEntity(org, oaf);
|
||||
org.setLegalshortname(mapStringField(m.getLegalshortname()));
|
||||
org.setLegalname(mapStringField(m.getLegalname()));
|
||||
org.setAlternativeNames(
|
||||
m.getAlternativeNamesList().stream()
|
||||
org
|
||||
.setAlternativeNames(
|
||||
m
|
||||
.getAlternativeNamesList()
|
||||
.stream()
|
||||
.map(ProtoConverter::mapStringField)
|
||||
.collect(Collectors.toList()));
|
||||
org.setWebsiteurl(mapStringField(m.getWebsiteurl()));
|
||||
|
|
@ -114,7 +118,8 @@ public class ProtoConverter implements Serializable {
|
|||
org.setEcnonprofit(mapStringField(m.getEcnonprofit()));
|
||||
org.setEcresearchorganization(mapStringField(m.getEcresearchorganization()));
|
||||
org.setEchighereducation(mapStringField(m.getEchighereducation()));
|
||||
org.setEcinternationalorganizationeurinterests(
|
||||
org
|
||||
.setEcinternationalorganizationeurinterests(
|
||||
mapStringField(m.getEcinternationalorganizationeurinterests()));
|
||||
org.setEcinternationalorganization(mapStringField(m.getEcinternationalorganization()));
|
||||
org.setEcenterprise(mapStringField(m.getEcenterprise()));
|
||||
|
|
@ -129,8 +134,11 @@ public class ProtoConverter implements Serializable {
|
|||
final DatasourceProtos.Datasource.Metadata m = oaf.getEntity().getDatasource().getMetadata();
|
||||
final Datasource datasource = setOaf(new Datasource(), oaf);
|
||||
setEntity(datasource, oaf);
|
||||
datasource.setAccessinfopackage(
|
||||
m.getAccessinfopackageList().stream()
|
||||
datasource
|
||||
.setAccessinfopackage(
|
||||
m
|
||||
.getAccessinfopackageList()
|
||||
.stream()
|
||||
.map(ProtoConverter::mapStringField)
|
||||
.collect(Collectors.toList()));
|
||||
datasource.setCertificates(mapStringField(m.getCertificates()));
|
||||
|
|
@ -151,12 +159,18 @@ public class ProtoConverter implements Serializable {
|
|||
datasource.setLogourl(mapStringField(m.getLogourl()));
|
||||
datasource.setMissionstatementurl(mapStringField(m.getMissionstatementurl()));
|
||||
datasource.setNamespaceprefix(mapStringField(m.getNamespaceprefix()));
|
||||
datasource.setOdcontenttypes(
|
||||
m.getOdcontenttypesList().stream()
|
||||
datasource
|
||||
.setOdcontenttypes(
|
||||
m
|
||||
.getOdcontenttypesList()
|
||||
.stream()
|
||||
.map(ProtoConverter::mapStringField)
|
||||
.collect(Collectors.toList()));
|
||||
datasource.setOdlanguages(
|
||||
m.getOdlanguagesList().stream()
|
||||
datasource
|
||||
.setOdlanguages(
|
||||
m
|
||||
.getOdlanguagesList()
|
||||
.stream()
|
||||
.map(ProtoConverter::mapStringField)
|
||||
.collect(Collectors.toList()));
|
||||
datasource.setOdnumberofitems(mapStringField(m.getOdnumberofitems()));
|
||||
|
|
@ -165,14 +179,18 @@ public class ProtoConverter implements Serializable {
|
|||
datasource.setOfficialname(mapStringField(m.getOfficialname()));
|
||||
datasource.setOpenairecompatibility(mapQualifier(m.getOpenairecompatibility()));
|
||||
datasource.setPidsystems(mapStringField(m.getPidsystems()));
|
||||
datasource.setPolicies(
|
||||
datasource
|
||||
.setPolicies(
|
||||
m.getPoliciesList().stream().map(ProtoConverter::mapKV).collect(Collectors.toList()));
|
||||
datasource.setQualitymanagementkind(mapStringField(m.getQualitymanagementkind()));
|
||||
datasource.setReleaseenddate(mapStringField(m.getReleaseenddate()));
|
||||
datasource.setServiceprovider(mapBoolField(m.getServiceprovider()));
|
||||
datasource.setReleasestartdate(mapStringField(m.getReleasestartdate()));
|
||||
datasource.setSubjects(
|
||||
m.getSubjectsList().stream()
|
||||
datasource
|
||||
.setSubjects(
|
||||
m
|
||||
.getSubjectsList()
|
||||
.stream()
|
||||
.map(ProtoConverter::mapStructuredProperty)
|
||||
.collect(Collectors.toList()));
|
||||
datasource.setVersioning(mapBoolField(m.getVersioning()));
|
||||
|
|
@ -204,13 +222,17 @@ public class ProtoConverter implements Serializable {
|
|||
project.setFundedamount(m.getFundedamount());
|
||||
project.setTotalcost(m.getTotalcost());
|
||||
project.setKeywords(mapStringField(m.getKeywords()));
|
||||
project.setSubjects(
|
||||
m.getSubjectsList().stream()
|
||||
project
|
||||
.setSubjects(
|
||||
m
|
||||
.getSubjectsList()
|
||||
.stream()
|
||||
.map(sp -> mapStructuredProperty(sp))
|
||||
.collect(Collectors.toList()));
|
||||
project.setTitle(mapStringField(m.getTitle()));
|
||||
project.setWebsiteurl(mapStringField(m.getWebsiteurl()));
|
||||
project.setFundingtree(
|
||||
project
|
||||
.setFundingtree(
|
||||
m.getFundingtreeList().stream().map(f -> mapStringField(f)).collect(Collectors.toList()));
|
||||
project.setJsonextrainfo(mapStringField(m.getJsonextrainfo()));
|
||||
project.setSummary(mapStringField(m.getSummary()));
|
||||
|
|
@ -242,12 +264,18 @@ public class ProtoConverter implements Serializable {
|
|||
setEntity(software, oaf);
|
||||
setResult(software, oaf);
|
||||
|
||||
software.setDocumentationUrl(
|
||||
m.getDocumentationUrlList().stream()
|
||||
software
|
||||
.setDocumentationUrl(
|
||||
m
|
||||
.getDocumentationUrlList()
|
||||
.stream()
|
||||
.map(ProtoConverter::mapStringField)
|
||||
.collect(Collectors.toList()));
|
||||
software.setLicense(
|
||||
m.getLicenseList().stream()
|
||||
software
|
||||
.setLicense(
|
||||
m
|
||||
.getLicenseList()
|
||||
.stream()
|
||||
.map(ProtoConverter::mapStructuredProperty)
|
||||
.collect(Collectors.toList()));
|
||||
software.setCodeRepositoryUrl(mapStringField(m.getCodeRepositoryUrl()));
|
||||
|
|
@ -260,15 +288,22 @@ public class ProtoConverter implements Serializable {
|
|||
OtherResearchProduct otherResearchProducts = setOaf(new OtherResearchProduct(), oaf);
|
||||
setEntity(otherResearchProducts, oaf);
|
||||
setResult(otherResearchProducts, oaf);
|
||||
otherResearchProducts.setContactperson(
|
||||
m.getContactpersonList().stream()
|
||||
otherResearchProducts
|
||||
.setContactperson(
|
||||
m
|
||||
.getContactpersonList()
|
||||
.stream()
|
||||
.map(ProtoConverter::mapStringField)
|
||||
.collect(Collectors.toList()));
|
||||
otherResearchProducts.setContactgroup(
|
||||
m.getContactgroupList().stream()
|
||||
otherResearchProducts
|
||||
.setContactgroup(
|
||||
m
|
||||
.getContactgroupList()
|
||||
.stream()
|
||||
.map(ProtoConverter::mapStringField)
|
||||
.collect(Collectors.toList()));
|
||||
otherResearchProducts.setTool(
|
||||
otherResearchProducts
|
||||
.setTool(
|
||||
m.getToolList().stream().map(ProtoConverter::mapStringField).collect(Collectors.toList()));
|
||||
|
||||
return otherResearchProducts;
|
||||
|
|
@ -296,8 +331,11 @@ public class ProtoConverter implements Serializable {
|
|||
dataset.setVersion(mapStringField(m.getVersion()));
|
||||
dataset.setLastmetadataupdate(mapStringField(m.getLastmetadataupdate()));
|
||||
dataset.setMetadataversionnumber(mapStringField(m.getMetadataversionnumber()));
|
||||
dataset.setGeolocation(
|
||||
m.getGeolocationList().stream()
|
||||
dataset
|
||||
.setGeolocation(
|
||||
m
|
||||
.getGeolocationList()
|
||||
.stream()
|
||||
.map(ProtoConverter::mapGeolocation)
|
||||
.collect(Collectors.toList()));
|
||||
return dataset;
|
||||
|
|
@ -314,16 +352,23 @@ public class ProtoConverter implements Serializable {
|
|||
final OafProtos.OafEntity e = oaf.getEntity();
|
||||
entity.setId(e.getId());
|
||||
entity.setOriginalId(e.getOriginalIdList());
|
||||
entity.setCollectedfrom(
|
||||
entity
|
||||
.setCollectedfrom(
|
||||
e.getCollectedfromList().stream().map(ProtoConverter::mapKV).collect(Collectors.toList()));
|
||||
entity.setPid(
|
||||
e.getPidList().stream()
|
||||
entity
|
||||
.setPid(
|
||||
e
|
||||
.getPidList()
|
||||
.stream()
|
||||
.map(ProtoConverter::mapStructuredProperty)
|
||||
.collect(Collectors.toList()));
|
||||
entity.setDateofcollection(e.getDateofcollection());
|
||||
entity.setDateoftransformation(e.getDateoftransformation());
|
||||
entity.setExtraInfo(
|
||||
e.getExtraInfoList().stream()
|
||||
entity
|
||||
.setExtraInfo(
|
||||
e
|
||||
.getExtraInfoList()
|
||||
.stream()
|
||||
.map(ProtoConverter::mapExtraInfo)
|
||||
.collect(Collectors.toList()));
|
||||
return entity;
|
||||
|
|
@ -332,55 +377,87 @@ public class ProtoConverter implements Serializable {
|
|||
public static <T extends Result> T setResult(T entity, OafProtos.Oaf oaf) {
|
||||
// setting Entity fields
|
||||
final ResultProtos.Result.Metadata m = oaf.getEntity().getResult().getMetadata();
|
||||
entity.setAuthor(
|
||||
entity
|
||||
.setAuthor(
|
||||
m.getAuthorList().stream().map(ProtoConverter::mapAuthor).collect(Collectors.toList()));
|
||||
entity.setResulttype(mapQualifier(m.getResulttype()));
|
||||
entity.setLanguage(mapQualifier(m.getLanguage()));
|
||||
entity.setCountry(
|
||||
m.getCountryList().stream()
|
||||
entity
|
||||
.setCountry(
|
||||
m
|
||||
.getCountryList()
|
||||
.stream()
|
||||
.map(ProtoConverter::mapQualifierAsCountry)
|
||||
.collect(Collectors.toList()));
|
||||
entity.setSubject(
|
||||
m.getSubjectList().stream()
|
||||
entity
|
||||
.setSubject(
|
||||
m
|
||||
.getSubjectList()
|
||||
.stream()
|
||||
.map(ProtoConverter::mapStructuredProperty)
|
||||
.collect(Collectors.toList()));
|
||||
entity.setTitle(
|
||||
m.getTitleList().stream()
|
||||
entity
|
||||
.setTitle(
|
||||
m
|
||||
.getTitleList()
|
||||
.stream()
|
||||
.map(ProtoConverter::mapStructuredProperty)
|
||||
.collect(Collectors.toList()));
|
||||
entity.setRelevantdate(
|
||||
m.getRelevantdateList().stream()
|
||||
entity
|
||||
.setRelevantdate(
|
||||
m
|
||||
.getRelevantdateList()
|
||||
.stream()
|
||||
.map(ProtoConverter::mapStructuredProperty)
|
||||
.collect(Collectors.toList()));
|
||||
entity.setDescription(
|
||||
m.getDescriptionList().stream()
|
||||
entity
|
||||
.setDescription(
|
||||
m
|
||||
.getDescriptionList()
|
||||
.stream()
|
||||
.map(ProtoConverter::mapStringField)
|
||||
.collect(Collectors.toList()));
|
||||
entity.setDateofacceptance(mapStringField(m.getDateofacceptance()));
|
||||
entity.setPublisher(mapStringField(m.getPublisher()));
|
||||
entity.setEmbargoenddate(mapStringField(m.getEmbargoenddate()));
|
||||
entity.setSource(
|
||||
m.getSourceList().stream()
|
||||
entity
|
||||
.setSource(
|
||||
m
|
||||
.getSourceList()
|
||||
.stream()
|
||||
.map(ProtoConverter::mapStringField)
|
||||
.collect(Collectors.toList()));
|
||||
entity.setFulltext(
|
||||
m.getFulltextList().stream()
|
||||
entity
|
||||
.setFulltext(
|
||||
m
|
||||
.getFulltextList()
|
||||
.stream()
|
||||
.map(ProtoConverter::mapStringField)
|
||||
.collect(Collectors.toList()));
|
||||
entity.setFormat(
|
||||
m.getFormatList().stream()
|
||||
entity
|
||||
.setFormat(
|
||||
m
|
||||
.getFormatList()
|
||||
.stream()
|
||||
.map(ProtoConverter::mapStringField)
|
||||
.collect(Collectors.toList()));
|
||||
entity.setContributor(
|
||||
m.getContributorList().stream()
|
||||
entity
|
||||
.setContributor(
|
||||
m
|
||||
.getContributorList()
|
||||
.stream()
|
||||
.map(ProtoConverter::mapStringField)
|
||||
.collect(Collectors.toList()));
|
||||
entity.setResourcetype(mapQualifier(m.getResourcetype()));
|
||||
entity.setCoverage(
|
||||
m.getCoverageList().stream()
|
||||
entity
|
||||
.setCoverage(
|
||||
m
|
||||
.getCoverageList()
|
||||
.stream()
|
||||
.map(ProtoConverter::mapStringField)
|
||||
.collect(Collectors.toList()));
|
||||
entity.setContext(
|
||||
entity
|
||||
.setContext(
|
||||
m.getContextList().stream().map(ProtoConverter::mapContext).collect(Collectors.toList()));
|
||||
|
||||
entity.setBestaccessright(getBestAccessRights(oaf.getEntity().getResult().getInstanceList()));
|
||||
|
|
@ -390,8 +467,10 @@ public class ProtoConverter implements Serializable {
|
|||
|
||||
private static Qualifier getBestAccessRights(List<ResultProtos.Result.Instance> instanceList) {
|
||||
if (instanceList != null) {
|
||||
final Optional<FieldTypeProtos.Qualifier> min =
|
||||
instanceList.stream().map(i -> i.getAccessright()).min(new LicenseComparator());
|
||||
final Optional<FieldTypeProtos.Qualifier> min = instanceList
|
||||
.stream()
|
||||
.map(i -> i.getAccessright())
|
||||
.min(new LicenseComparator());
|
||||
|
||||
final Qualifier rights = min.isPresent() ? mapQualifier(min.get()) : new Qualifier();
|
||||
|
||||
|
|
@ -418,8 +497,11 @@ public class ProtoConverter implements Serializable {
|
|||
|
||||
final Context entity = new Context();
|
||||
entity.setId(context.getId());
|
||||
entity.setDataInfo(
|
||||
context.getDataInfoList().stream()
|
||||
entity
|
||||
.setDataInfo(
|
||||
context
|
||||
.getDataInfoList()
|
||||
.stream()
|
||||
.map(ProtoConverter::mapDataInfo)
|
||||
.collect(Collectors.toList()));
|
||||
return entity;
|
||||
|
|
@ -543,8 +625,11 @@ public class ProtoConverter implements Serializable {
|
|||
entity.setName(author.getName());
|
||||
entity.setSurname(author.getSurname());
|
||||
entity.setRank(author.getRank());
|
||||
entity.setPid(
|
||||
author.getPidList().stream()
|
||||
entity
|
||||
.setPid(
|
||||
author
|
||||
.getPidList()
|
||||
.stream()
|
||||
.map(
|
||||
kv -> {
|
||||
final StructuredProperty sp = new StructuredProperty();
|
||||
|
|
@ -556,8 +641,11 @@ public class ProtoConverter implements Serializable {
|
|||
return sp;
|
||||
})
|
||||
.collect(Collectors.toList()));
|
||||
entity.setAffiliation(
|
||||
author.getAffiliationList().stream()
|
||||
entity
|
||||
.setAffiliation(
|
||||
author
|
||||
.getAffiliationList()
|
||||
.stream()
|
||||
.map(ProtoConverter::mapStringField)
|
||||
.collect(Collectors.toList()));
|
||||
return entity;
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
|
||||
package eu.dnetlib.dhp.actionmanager.migration;
|
||||
|
||||
import static eu.dnetlib.dhp.common.SparkSessionSupport.runWithSparkSession;
|
||||
|
|
@ -40,15 +41,16 @@ public class TransformActions implements Serializable {
|
|||
private static final String SEPARATOR = "/";
|
||||
|
||||
public static void main(String[] args) throws Exception {
|
||||
final ArgumentApplicationParser parser =
|
||||
new ArgumentApplicationParser(
|
||||
IOUtils.toString(
|
||||
MigrateActionSet.class.getResourceAsStream(
|
||||
final ArgumentApplicationParser parser = new ArgumentApplicationParser(
|
||||
IOUtils
|
||||
.toString(
|
||||
MigrateActionSet.class
|
||||
.getResourceAsStream(
|
||||
"/eu/dnetlib/dhp/actionmanager/migration/transform_actionsets_parameters.json")));
|
||||
parser.parseArgument(args);
|
||||
|
||||
Boolean isSparkSessionManaged =
|
||||
Optional.ofNullable(parser.get("isSparkSessionManaged"))
|
||||
Boolean isSparkSessionManaged = Optional
|
||||
.ofNullable(parser.get("isSparkSessionManaged"))
|
||||
.map(Boolean::valueOf)
|
||||
.orElse(Boolean.TRUE);
|
||||
log.info("isSparkSessionManaged: {}", isSparkSessionManaged);
|
||||
|
|
@ -83,8 +85,7 @@ public class TransformActions implements Serializable {
|
|||
final String rawset = pathQ.pollLast();
|
||||
final String actionSetDirectory = pathQ.pollLast();
|
||||
|
||||
final Path targetDirectory =
|
||||
new Path(targetBaseDir + SEPARATOR + actionSetDirectory + SEPARATOR + rawset);
|
||||
final Path targetDirectory = new Path(targetBaseDir + SEPARATOR + actionSetDirectory + SEPARATOR + rawset);
|
||||
|
||||
if (fs.exists(targetDirectory)) {
|
||||
log.info("found target directory '{}", targetDirectory);
|
||||
|
|
@ -94,7 +95,8 @@ public class TransformActions implements Serializable {
|
|||
|
||||
log.info("transforming actions from '{}' to '{}'", sourcePath, targetDirectory);
|
||||
|
||||
sc.sequenceFile(sourcePath, Text.class, Text.class)
|
||||
sc
|
||||
.sequenceFile(sourcePath, Text.class, Text.class)
|
||||
.map(a -> eu.dnetlib.actionmanager.actions.AtomicAction.fromJSON(a._2().toString()))
|
||||
.map(TransformActions::doTransform)
|
||||
.filter(Objects::nonNull)
|
||||
|
|
@ -129,8 +131,12 @@ public class TransformActions implements Serializable {
|
|||
case project:
|
||||
return new AtomicAction<>(Project.class, (Project) oaf);
|
||||
case result:
|
||||
final String resulttypeid =
|
||||
proto_oaf.getEntity().getResult().getMetadata().getResulttype().getClassid();
|
||||
final String resulttypeid = proto_oaf
|
||||
.getEntity()
|
||||
.getResult()
|
||||
.getMetadata()
|
||||
.getResulttype()
|
||||
.getClassid();
|
||||
switch (resulttypeid) {
|
||||
case "publication":
|
||||
return new AtomicAction<>(Publication.class, (Publication) oaf);
|
||||
|
|
@ -157,8 +163,7 @@ public class TransformActions implements Serializable {
|
|||
|
||||
private static String getTargetBaseDir(String isLookupUrl) throws ISLookUpException {
|
||||
ISLookUpService isLookUp = ISLookupClientFactory.getLookUpService(isLookupUrl);
|
||||
String XQUERY =
|
||||
"collection('/db/DRIVER/ServiceResources/ActionManagerServiceResourceType')//SERVICE_PROPERTIES/PROPERTY[@key = 'basePath']/@value/string()";
|
||||
String XQUERY = "collection('/db/DRIVER/ServiceResources/ActionManagerServiceResourceType')//SERVICE_PROPERTIES/PROPERTY[@key = 'basePath']/@value/string()";
|
||||
return isLookUp.getResourceProfileByQuery(XQUERY);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
|
||||
package eu.dnetlib.dhp.actionmanager.partition;
|
||||
|
||||
import static eu.dnetlib.dhp.common.SparkSessionSupport.runWithSparkSession;
|
||||
|
|
@ -23,20 +24,22 @@ import org.slf4j.LoggerFactory;
|
|||
/** Partitions given set of action sets by payload type. */
|
||||
public class PartitionActionSetsByPayloadTypeJob {
|
||||
|
||||
private static final Logger logger =
|
||||
LoggerFactory.getLogger(PartitionActionSetsByPayloadTypeJob.class);
|
||||
private static final Logger logger = LoggerFactory.getLogger(PartitionActionSetsByPayloadTypeJob.class);
|
||||
|
||||
private static final StructType KV_SCHEMA =
|
||||
StructType$.MODULE$.apply(
|
||||
Arrays.asList(
|
||||
private static final StructType KV_SCHEMA = StructType$.MODULE$
|
||||
.apply(
|
||||
Arrays
|
||||
.asList(
|
||||
StructField$.MODULE$.apply("key", DataTypes.StringType, false, Metadata.empty()),
|
||||
StructField$.MODULE$.apply("value", DataTypes.StringType, false, Metadata.empty())));
|
||||
|
||||
private static final StructType ATOMIC_ACTION_SCHEMA =
|
||||
StructType$.MODULE$.apply(
|
||||
Arrays.asList(
|
||||
private static final StructType ATOMIC_ACTION_SCHEMA = StructType$.MODULE$
|
||||
.apply(
|
||||
Arrays
|
||||
.asList(
|
||||
StructField$.MODULE$.apply("clazz", DataTypes.StringType, false, Metadata.empty()),
|
||||
StructField$.MODULE$.apply(
|
||||
StructField$.MODULE$
|
||||
.apply(
|
||||
"payload", DataTypes.StringType, false, Metadata.empty())));
|
||||
|
||||
private ISClient isClient;
|
||||
|
|
@ -45,18 +48,20 @@ public class PartitionActionSetsByPayloadTypeJob {
|
|||
this.isClient = new ISClient(isLookupUrl);
|
||||
}
|
||||
|
||||
public PartitionActionSetsByPayloadTypeJob() {}
|
||||
public PartitionActionSetsByPayloadTypeJob() {
|
||||
}
|
||||
|
||||
public static void main(String[] args) throws Exception {
|
||||
String jsonConfiguration =
|
||||
IOUtils.toString(
|
||||
PromoteActionPayloadForGraphTableJob.class.getResourceAsStream(
|
||||
String jsonConfiguration = IOUtils
|
||||
.toString(
|
||||
PromoteActionPayloadForGraphTableJob.class
|
||||
.getResourceAsStream(
|
||||
"/eu/dnetlib/dhp/actionmanager/partition/partition_action_sets_by_payload_type_input_parameters.json"));
|
||||
final ArgumentApplicationParser parser = new ArgumentApplicationParser(jsonConfiguration);
|
||||
parser.parseArgument(args);
|
||||
|
||||
Boolean isSparkSessionManaged =
|
||||
Optional.ofNullable(parser.get("isSparkSessionManaged"))
|
||||
Boolean isSparkSessionManaged = Optional
|
||||
.ofNullable(parser.get("isSparkSessionManaged"))
|
||||
.map(Boolean::valueOf)
|
||||
.orElse(Boolean.TRUE);
|
||||
logger.info("isSparkSessionManaged: {}", isSparkSessionManaged);
|
||||
|
|
@ -97,7 +102,8 @@ public class PartitionActionSetsByPayloadTypeJob {
|
|||
|
||||
private static void readAndWriteActionSetsFromPaths(
|
||||
SparkSession spark, List<String> inputActionSetPaths, String outputPath) {
|
||||
inputActionSetPaths.stream()
|
||||
inputActionSetPaths
|
||||
.stream()
|
||||
.filter(path -> HdfsSupport.exists(path, spark.sparkContext().hadoopConfiguration()))
|
||||
.forEach(
|
||||
inputActionSetPath -> {
|
||||
|
|
@ -111,8 +117,8 @@ public class PartitionActionSetsByPayloadTypeJob {
|
|||
|
||||
JavaSparkContext sc = JavaSparkContext.fromSparkContext(spark.sparkContext());
|
||||
|
||||
JavaRDD<Row> rdd =
|
||||
sc.sequenceFile(path, Text.class, Text.class)
|
||||
JavaRDD<Row> rdd = sc
|
||||
.sequenceFile(path, Text.class, Text.class)
|
||||
.map(x -> RowFactory.create(x._1().toString(), x._2().toString()));
|
||||
|
||||
return spark
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
|
||||
package eu.dnetlib.dhp.actionmanager.promote;
|
||||
|
||||
import static eu.dnetlib.dhp.schema.common.ModelSupport.isSubClass;
|
||||
|
|
@ -11,17 +12,17 @@ import java.util.function.BiFunction;
|
|||
/** OAF model merging support. */
|
||||
public class MergeAndGet {
|
||||
|
||||
private MergeAndGet() {}
|
||||
private MergeAndGet() {
|
||||
}
|
||||
|
||||
/**
|
||||
* Strategy for merging OAF model objects.
|
||||
*
|
||||
* <p>MERGE_FROM_AND_GET: use OAF 'mergeFrom' method SELECT_NEWER_AND_GET: use last update
|
||||
* timestamp to return newer instance
|
||||
* <p>
|
||||
* MERGE_FROM_AND_GET: use OAF 'mergeFrom' method SELECT_NEWER_AND_GET: use last update timestamp to return newer
|
||||
* instance
|
||||
*/
|
||||
public enum Strategy {
|
||||
MERGE_FROM_AND_GET,
|
||||
SELECT_NEWER_AND_GET
|
||||
MERGE_FROM_AND_GET, SELECT_NEWER_AND_GET
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -32,8 +33,8 @@ public class MergeAndGet {
|
|||
* @param <A> Action payload type
|
||||
* @return BiFunction to be used to merge OAF objects
|
||||
*/
|
||||
public static <G extends Oaf, A extends Oaf>
|
||||
SerializableSupplier<BiFunction<G, A, G>> functionFor(Strategy strategy) {
|
||||
public static <G extends Oaf, A extends Oaf> SerializableSupplier<BiFunction<G, A, G>> functionFor(
|
||||
Strategy strategy) {
|
||||
switch (strategy) {
|
||||
case MERGE_FROM_AND_GET:
|
||||
return () -> MergeAndGet::mergeFromAndGet;
|
||||
|
|
@ -54,7 +55,8 @@ public class MergeAndGet {
|
|||
return x;
|
||||
}
|
||||
throw new RuntimeException(
|
||||
String.format(
|
||||
String
|
||||
.format(
|
||||
"MERGE_FROM_AND_GET incompatible types: %s, %s",
|
||||
x.getClass().getCanonicalName(), y.getClass().getCanonicalName()));
|
||||
}
|
||||
|
|
@ -70,12 +72,14 @@ public class MergeAndGet {
|
|||
return x;
|
||||
} else if (isSubClass(x, y) && x.getLastupdatetimestamp() < y.getLastupdatetimestamp()) {
|
||||
throw new RuntimeException(
|
||||
String.format(
|
||||
String
|
||||
.format(
|
||||
"SELECT_NEWER_AND_GET cannot return right type when it is not the same as left type: %s, %s",
|
||||
x.getClass().getCanonicalName(), y.getClass().getCanonicalName()));
|
||||
}
|
||||
throw new RuntimeException(
|
||||
String.format(
|
||||
String
|
||||
.format(
|
||||
"SELECT_NEWER_AND_GET cannot be used when left is not subtype of right: %s, %s",
|
||||
x.getClass().getCanonicalName(), y.getClass().getCanonicalName()));
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
|
||||
package eu.dnetlib.dhp.actionmanager.promote;
|
||||
|
||||
import static eu.dnetlib.dhp.common.SparkSessionSupport.runWithSparkSession;
|
||||
|
|
@ -25,21 +26,21 @@ import org.slf4j.LoggerFactory;
|
|||
|
||||
/** Applies a given action payload file to graph table of compatible type. */
|
||||
public class PromoteActionPayloadForGraphTableJob {
|
||||
private static final Logger logger =
|
||||
LoggerFactory.getLogger(PromoteActionPayloadForGraphTableJob.class);
|
||||
private static final Logger logger = LoggerFactory.getLogger(PromoteActionPayloadForGraphTableJob.class);
|
||||
|
||||
private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
|
||||
|
||||
public static void main(String[] args) throws Exception {
|
||||
String jsonConfiguration =
|
||||
IOUtils.toString(
|
||||
PromoteActionPayloadForGraphTableJob.class.getResourceAsStream(
|
||||
String jsonConfiguration = IOUtils
|
||||
.toString(
|
||||
PromoteActionPayloadForGraphTableJob.class
|
||||
.getResourceAsStream(
|
||||
"/eu/dnetlib/dhp/actionmanager/promote/promote_action_payload_for_graph_table_input_parameters.json"));
|
||||
final ArgumentApplicationParser parser = new ArgumentApplicationParser(jsonConfiguration);
|
||||
parser.parseArgument(args);
|
||||
|
||||
Boolean isSparkSessionManaged =
|
||||
Optional.ofNullable(parser.get("isSparkSessionManaged"))
|
||||
Boolean isSparkSessionManaged = Optional
|
||||
.ofNullable(parser.get("isSparkSessionManaged"))
|
||||
.map(Boolean::valueOf)
|
||||
.orElse(Boolean.TRUE);
|
||||
logger.info("isSparkSessionManaged: {}", isSparkSessionManaged);
|
||||
|
|
@ -59,13 +60,11 @@ public class PromoteActionPayloadForGraphTableJob {
|
|||
String outputGraphTablePath = parser.get("outputGraphTablePath");
|
||||
logger.info("outputGraphTablePath: {}", outputGraphTablePath);
|
||||
|
||||
MergeAndGet.Strategy strategy =
|
||||
MergeAndGet.Strategy.valueOf(parser.get("mergeAndGetStrategy").toUpperCase());
|
||||
MergeAndGet.Strategy strategy = MergeAndGet.Strategy.valueOf(parser.get("mergeAndGetStrategy").toUpperCase());
|
||||
logger.info("strategy: {}", strategy);
|
||||
|
||||
Class<? extends Oaf> rowClazz = (Class<? extends Oaf>) Class.forName(graphTableClassName);
|
||||
Class<? extends Oaf> actionPayloadClazz =
|
||||
(Class<? extends Oaf>) Class.forName(actionPayloadClassName);
|
||||
Class<? extends Oaf> actionPayloadClazz = (Class<? extends Oaf>) Class.forName(actionPayloadClassName);
|
||||
|
||||
throwIfGraphTableClassIsNotSubClassOfActionPayloadClass(rowClazz, actionPayloadClazz);
|
||||
|
||||
|
|
@ -92,8 +91,8 @@ public class PromoteActionPayloadForGraphTableJob {
|
|||
private static void throwIfGraphTableClassIsNotSubClassOfActionPayloadClass(
|
||||
Class<? extends Oaf> rowClazz, Class<? extends Oaf> actionPayloadClazz) {
|
||||
if (!isSubClass(rowClazz, actionPayloadClazz)) {
|
||||
String msg =
|
||||
String.format(
|
||||
String msg = String
|
||||
.format(
|
||||
"graph table class is not a subclass of action payload class: graph=%s, action=%s",
|
||||
rowClazz.getCanonicalName(), actionPayloadClazz.getCanonicalName());
|
||||
throw new RuntimeException(msg);
|
||||
|
|
@ -113,11 +112,9 @@ public class PromoteActionPayloadForGraphTableJob {
|
|||
Class<G> rowClazz,
|
||||
Class<A> actionPayloadClazz) {
|
||||
Dataset<G> rowDS = readGraphTable(spark, inputGraphTablePath, rowClazz);
|
||||
Dataset<A> actionPayloadDS =
|
||||
readActionPayload(spark, inputActionPayloadPath, actionPayloadClazz);
|
||||
Dataset<A> actionPayloadDS = readActionPayload(spark, inputActionPayloadPath, actionPayloadClazz);
|
||||
|
||||
Dataset<G> result =
|
||||
promoteActionPayloadForGraphTable(
|
||||
Dataset<G> result = promoteActionPayloadForGraphTable(
|
||||
rowDS, actionPayloadDS, strategy, rowClazz, actionPayloadClazz)
|
||||
.map((MapFunction<G, G>) value -> value, Encoders.bean(rowClazz));
|
||||
|
||||
|
|
@ -147,9 +144,8 @@ public class PromoteActionPayloadForGraphTableJob {
|
|||
.read()
|
||||
.parquet(path)
|
||||
.map(
|
||||
(MapFunction<Row, A>)
|
||||
value ->
|
||||
OBJECT_MAPPER.readValue(value.<String>getAs("payload"), actionPayloadClazz),
|
||||
(MapFunction<Row, A>) value -> OBJECT_MAPPER
|
||||
.readValue(value.<String> getAs("payload"), actionPayloadClazz),
|
||||
Encoders.bean(actionPayloadClazz));
|
||||
}
|
||||
|
||||
|
|
@ -159,22 +155,21 @@ public class PromoteActionPayloadForGraphTableJob {
|
|||
MergeAndGet.Strategy strategy,
|
||||
Class<G> rowClazz,
|
||||
Class<A> actionPayloadClazz) {
|
||||
logger.info(
|
||||
logger
|
||||
.info(
|
||||
"Promoting action payload for graph table: payload={}, table={}",
|
||||
actionPayloadClazz.getSimpleName(),
|
||||
rowClazz.getSimpleName());
|
||||
|
||||
SerializableSupplier<Function<G, String>> rowIdFn = ModelSupport::idFn;
|
||||
SerializableSupplier<Function<A, String>> actionPayloadIdFn = ModelSupport::idFn;
|
||||
SerializableSupplier<BiFunction<G, A, G>> mergeRowWithActionPayloadAndGetFn =
|
||||
MergeAndGet.functionFor(strategy);
|
||||
SerializableSupplier<BiFunction<G, A, G>> mergeRowWithActionPayloadAndGetFn = MergeAndGet.functionFor(strategy);
|
||||
SerializableSupplier<BiFunction<G, G, G>> mergeRowsAndGetFn = MergeAndGet.functionFor(strategy);
|
||||
SerializableSupplier<G> zeroFn = zeroFn(rowClazz);
|
||||
SerializableSupplier<Function<G, Boolean>> isNotZeroFn =
|
||||
PromoteActionPayloadForGraphTableJob::isNotZeroFnUsingIdOrSource;
|
||||
SerializableSupplier<Function<G, Boolean>> isNotZeroFn = PromoteActionPayloadForGraphTableJob::isNotZeroFnUsingIdOrSource;
|
||||
|
||||
Dataset<G> joinedAndMerged =
|
||||
PromoteActionPayloadFunctions.joinGraphTableWithActionPayloadAndMerge(
|
||||
Dataset<G> joinedAndMerged = PromoteActionPayloadFunctions
|
||||
.joinGraphTableWithActionPayloadAndMerge(
|
||||
rowDS,
|
||||
actionPayloadDS,
|
||||
rowIdFn,
|
||||
|
|
@ -183,7 +178,8 @@ public class PromoteActionPayloadForGraphTableJob {
|
|||
rowClazz,
|
||||
actionPayloadClazz);
|
||||
|
||||
return PromoteActionPayloadFunctions.groupGraphTableByIdAndMerge(
|
||||
return PromoteActionPayloadFunctions
|
||||
.groupGraphTableByIdAndMerge(
|
||||
joinedAndMerged, rowIdFn, mergeRowsAndGetFn, zeroFn, isNotZeroFn, rowClazz);
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
|
||||
package eu.dnetlib.dhp.actionmanager.promote;
|
||||
|
||||
import static eu.dnetlib.dhp.schema.common.ModelSupport.isSubClass;
|
||||
|
|
@ -20,11 +21,11 @@ import scala.Tuple2;
|
|||
/** Promote action payload functions. */
|
||||
public class PromoteActionPayloadFunctions {
|
||||
|
||||
private PromoteActionPayloadFunctions() {}
|
||||
private PromoteActionPayloadFunctions() {
|
||||
}
|
||||
|
||||
/**
|
||||
* Joins dataset representing graph table with dataset representing action payload using supplied
|
||||
* functions.
|
||||
* Joins dataset representing graph table with dataset representing action payload using supplied functions.
|
||||
*
|
||||
* @param rowDS Dataset representing graph table
|
||||
* @param actionPayloadDS Dataset representing action payload
|
||||
|
|
@ -51,8 +52,8 @@ public class PromoteActionPayloadFunctions {
|
|||
}
|
||||
|
||||
Dataset<Tuple2<String, G>> rowWithIdDS = mapToTupleWithId(rowDS, rowIdFn, rowClazz);
|
||||
Dataset<Tuple2<String, A>> actionPayloadWithIdDS =
|
||||
mapToTupleWithId(actionPayloadDS, actionPayloadIdFn, actionPayloadClazz);
|
||||
Dataset<Tuple2<String, A>> actionPayloadWithIdDS = mapToTupleWithId(
|
||||
actionPayloadDS, actionPayloadIdFn, actionPayloadClazz);
|
||||
|
||||
return rowWithIdDS
|
||||
.joinWith(
|
||||
|
|
@ -60,21 +61,17 @@ public class PromoteActionPayloadFunctions {
|
|||
rowWithIdDS.col("_1").equalTo(actionPayloadWithIdDS.col("_1")),
|
||||
"full_outer")
|
||||
.map(
|
||||
(MapFunction<Tuple2<Tuple2<String, G>, Tuple2<String, A>>, G>)
|
||||
value -> {
|
||||
(MapFunction<Tuple2<Tuple2<String, G>, Tuple2<String, A>>, G>) value -> {
|
||||
Optional<G> rowOpt = Optional.ofNullable(value._1()).map(Tuple2::_2);
|
||||
Optional<A> actionPayloadOpt = Optional.ofNullable(value._2()).map(Tuple2::_2);
|
||||
return rowOpt
|
||||
.map(
|
||||
row ->
|
||||
actionPayloadOpt
|
||||
row -> actionPayloadOpt
|
||||
.map(
|
||||
actionPayload ->
|
||||
mergeAndGetFn.get().apply(row, actionPayload))
|
||||
actionPayload -> mergeAndGetFn.get().apply(row, actionPayload))
|
||||
.orElse(row))
|
||||
.orElseGet(
|
||||
() ->
|
||||
actionPayloadOpt
|
||||
() -> actionPayloadOpt
|
||||
.filter(
|
||||
actionPayload -> actionPayload.getClass().equals(rowClazz))
|
||||
.map(rowClazz::cast)
|
||||
|
|
@ -86,7 +83,8 @@ public class PromoteActionPayloadFunctions {
|
|||
|
||||
private static <T extends Oaf> Dataset<Tuple2<String, T>> mapToTupleWithId(
|
||||
Dataset<T> ds, SerializableSupplier<Function<T, String>> idFn, Class<T> clazz) {
|
||||
return ds.map(
|
||||
return ds
|
||||
.map(
|
||||
(MapFunction<T, Tuple2<String, T>>) value -> new Tuple2<>(idFn.get().apply(value), value),
|
||||
Encoders.tuple(Encoders.STRING(), Encoders.kryo(clazz)));
|
||||
}
|
||||
|
|
@ -110,8 +108,7 @@ public class PromoteActionPayloadFunctions {
|
|||
SerializableSupplier<G> zeroFn,
|
||||
SerializableSupplier<Function<G, Boolean>> isNotZeroFn,
|
||||
Class<G> rowClazz) {
|
||||
TypedColumn<G, G> aggregator =
|
||||
new TableAggregator<>(zeroFn, mergeAndGetFn, isNotZeroFn, rowClazz).toColumn();
|
||||
TypedColumn<G, G> aggregator = new TableAggregator<>(zeroFn, mergeAndGetFn, isNotZeroFn, rowClazz).toColumn();
|
||||
return rowDS
|
||||
.groupByKey((MapFunction<G, String>) x -> rowIdFn.get().apply(x), Encoders.STRING())
|
||||
.agg(aggregator)
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
|
||||
package eu.dnetlib.dhp.actionmanager.partition;
|
||||
|
||||
import static eu.dnetlib.dhp.common.ThrowingSupport.rethrowAsRuntimeException;
|
||||
|
|
@ -37,19 +38,20 @@ import scala.collection.mutable.Seq;
|
|||
|
||||
@ExtendWith(MockitoExtension.class)
|
||||
public class PartitionActionSetsByPayloadTypeJobTest {
|
||||
private static final ClassLoader cl =
|
||||
PartitionActionSetsByPayloadTypeJobTest.class.getClassLoader();
|
||||
private static final ClassLoader cl = PartitionActionSetsByPayloadTypeJobTest.class.getClassLoader();
|
||||
|
||||
private static Configuration configuration;
|
||||
private static SparkSession spark;
|
||||
|
||||
private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
|
||||
|
||||
private static final StructType ATOMIC_ACTION_SCHEMA =
|
||||
StructType$.MODULE$.apply(
|
||||
Arrays.asList(
|
||||
private static final StructType ATOMIC_ACTION_SCHEMA = StructType$.MODULE$
|
||||
.apply(
|
||||
Arrays
|
||||
.asList(
|
||||
StructField$.MODULE$.apply("clazz", DataTypes.StringType, false, Metadata.empty()),
|
||||
StructField$.MODULE$.apply(
|
||||
StructField$.MODULE$
|
||||
.apply(
|
||||
"payload", DataTypes.StringType, false, Metadata.empty())));
|
||||
|
||||
@BeforeAll
|
||||
|
|
@ -71,7 +73,8 @@ public class PartitionActionSetsByPayloadTypeJobTest {
|
|||
@Nested
|
||||
class Main {
|
||||
|
||||
@Mock private ISClient isClient;
|
||||
@Mock
|
||||
private ISClient isClient;
|
||||
|
||||
@Test
|
||||
public void shouldPartitionActionSetsByPayloadType(@TempDir Path workingDir) throws Exception {
|
||||
|
|
@ -84,12 +87,14 @@ public class PartitionActionSetsByPayloadTypeJobTest {
|
|||
List<String> inputActionSetsPaths = resolveInputActionSetPaths(inputActionSetsBaseDir);
|
||||
|
||||
// when
|
||||
Mockito.when(isClient.getLatestRawsetPaths(Mockito.anyString()))
|
||||
Mockito
|
||||
.when(isClient.getLatestRawsetPaths(Mockito.anyString()))
|
||||
.thenReturn(inputActionSetsPaths);
|
||||
|
||||
PartitionActionSetsByPayloadTypeJob job = new PartitionActionSetsByPayloadTypeJob();
|
||||
job.setIsClient(isClient);
|
||||
job.run(
|
||||
job
|
||||
.run(
|
||||
Boolean.FALSE,
|
||||
"", // it can be empty we're mocking the response from isClient
|
||||
// to
|
||||
|
|
@ -114,7 +119,8 @@ public class PartitionActionSetsByPayloadTypeJobTest {
|
|||
|
||||
private List<String> resolveInputActionSetPaths(Path inputActionSetsBaseDir) throws IOException {
|
||||
Path inputActionSetJsonDumpsDir = getInputActionSetJsonDumpsDir();
|
||||
return Files.list(inputActionSetJsonDumpsDir)
|
||||
return Files
|
||||
.list(inputActionSetJsonDumpsDir)
|
||||
.map(
|
||||
path -> {
|
||||
String inputActionSetId = path.getFileName().toString();
|
||||
|
|
@ -128,29 +134,31 @@ public class PartitionActionSetsByPayloadTypeJobTest {
|
|||
Path inputActionSetJsonDumpsDir = getInputActionSetJsonDumpsDir();
|
||||
|
||||
Map<String, List<String>> oafsByType = new HashMap<>();
|
||||
Files.list(inputActionSetJsonDumpsDir)
|
||||
Files
|
||||
.list(inputActionSetJsonDumpsDir)
|
||||
.forEach(
|
||||
inputActionSetJsonDumpFile -> {
|
||||
String inputActionSetId = inputActionSetJsonDumpFile.getFileName().toString();
|
||||
Path inputActionSetDir = inputActionSetsDir.resolve(inputActionSetId);
|
||||
|
||||
Dataset<String> actionDS =
|
||||
readActionsFromJsonDump(inputActionSetJsonDumpFile.toString()).cache();
|
||||
Dataset<String> actionDS = readActionsFromJsonDump(inputActionSetJsonDumpFile.toString()).cache();
|
||||
|
||||
writeActionsAsJobInput(actionDS, inputActionSetId, inputActionSetDir.toString());
|
||||
|
||||
Map<String, List<String>> actionSetOafsByType =
|
||||
actionDS
|
||||
Map<String, List<String>> actionSetOafsByType = actionDS
|
||||
.withColumn("atomic_action", from_json(col("value"), ATOMIC_ACTION_SCHEMA))
|
||||
.select(expr("atomic_action.*")).groupBy(col("clazz"))
|
||||
.agg(collect_list(col("payload")).as("payload_list")).collectAsList().stream()
|
||||
.select(expr("atomic_action.*"))
|
||||
.groupBy(col("clazz"))
|
||||
.agg(collect_list(col("payload")).as("payload_list"))
|
||||
.collectAsList()
|
||||
.stream()
|
||||
.map(
|
||||
row ->
|
||||
new AbstractMap.SimpleEntry<>(
|
||||
row -> new AbstractMap.SimpleEntry<>(
|
||||
row.<String> getAs("clazz"),
|
||||
mutableSeqAsJavaList(row.<Seq<String>> getAs("payload_list"))))
|
||||
.collect(
|
||||
Collectors.toMap(
|
||||
Collectors
|
||||
.toMap(
|
||||
AbstractMap.SimpleEntry::getKey, AbstractMap.SimpleEntry::getValue));
|
||||
|
||||
actionSetOafsByType
|
||||
|
|
@ -172,8 +180,10 @@ public class PartitionActionSetsByPayloadTypeJobTest {
|
|||
}
|
||||
|
||||
private static Path getInputActionSetJsonDumpsDir() {
|
||||
return Paths.get(
|
||||
Objects.requireNonNull(cl.getResource("eu/dnetlib/dhp/actionmanager/partition/input/"))
|
||||
return Paths
|
||||
.get(
|
||||
Objects
|
||||
.requireNonNull(cl.getResource("eu/dnetlib/dhp/actionmanager/partition/input/"))
|
||||
.getFile());
|
||||
}
|
||||
|
||||
|
|
@ -195,12 +205,12 @@ public class PartitionActionSetsByPayloadTypeJobTest {
|
|||
Path outputDatasetDir = outputDir.resolve(String.format("clazz=%s", clazz.getCanonicalName()));
|
||||
Files.exists(outputDatasetDir);
|
||||
|
||||
List<T> actuals =
|
||||
readActionPayloadFromJobOutput(outputDatasetDir.toString(), clazz).collectAsList();
|
||||
List<T> actuals = readActionPayloadFromJobOutput(outputDatasetDir.toString(), clazz).collectAsList();
|
||||
actuals.sort(Comparator.comparingInt(Object::hashCode));
|
||||
|
||||
List<T> expecteds =
|
||||
oafsByClassName.get(clazz.getCanonicalName()).stream()
|
||||
List<T> expecteds = oafsByClassName
|
||||
.get(clazz.getCanonicalName())
|
||||
.stream()
|
||||
.map(json -> mapToOaf(json, clazz))
|
||||
.sorted(Comparator.comparingInt(Object::hashCode))
|
||||
.collect(Collectors.toList());
|
||||
|
|
@ -214,15 +224,15 @@ public class PartitionActionSetsByPayloadTypeJobTest {
|
|||
.read()
|
||||
.parquet(path)
|
||||
.map(
|
||||
(MapFunction<Row, T>)
|
||||
value -> OBJECT_MAPPER.readValue(value.<String>getAs("payload"), clazz),
|
||||
(MapFunction<Row, T>) value -> OBJECT_MAPPER.readValue(value.<String> getAs("payload"), clazz),
|
||||
Encoders.bean(clazz));
|
||||
}
|
||||
|
||||
private static <T extends Oaf> T mapToOaf(String json, Class<T> clazz) {
|
||||
return rethrowAsRuntimeException(
|
||||
() -> OBJECT_MAPPER.readValue(json, clazz),
|
||||
String.format(
|
||||
String
|
||||
.format(
|
||||
"failed to map json to class: json=%s, class=%s", json, clazz.getCanonicalName()));
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
|
||||
package eu.dnetlib.dhp.actionmanager.promote;
|
||||
|
||||
import static eu.dnetlib.dhp.actionmanager.promote.MergeAndGet.Strategy;
|
||||
|
|
@ -126,8 +127,10 @@ public class MergeAndGetTest {
|
|||
@Test
|
||||
public void shouldThrowForOafEntityAndOafEntityButNotSubclasses() {
|
||||
// given
|
||||
class OafEntitySub1 extends OafEntity {}
|
||||
class OafEntitySub2 extends OafEntity {}
|
||||
class OafEntitySub1 extends OafEntity {
|
||||
}
|
||||
class OafEntitySub2 extends OafEntity {
|
||||
}
|
||||
|
||||
OafEntitySub1 a = mock(OafEntitySub1.class);
|
||||
OafEntitySub2 b = mock(OafEntitySub2.class);
|
||||
|
|
@ -166,8 +169,7 @@ public class MergeAndGetTest {
|
|||
Relation b = mock(Relation.class);
|
||||
|
||||
// when
|
||||
SerializableSupplier<BiFunction<Oaf, Oaf, Oaf>> fn =
|
||||
functionFor(Strategy.SELECT_NEWER_AND_GET);
|
||||
SerializableSupplier<BiFunction<Oaf, Oaf, Oaf>> fn = functionFor(Strategy.SELECT_NEWER_AND_GET);
|
||||
|
||||
// then
|
||||
assertThrows(RuntimeException.class, () -> fn.get().apply(a, b));
|
||||
|
|
@ -180,8 +182,7 @@ public class MergeAndGetTest {
|
|||
OafEntity b = mock(OafEntity.class);
|
||||
|
||||
// when
|
||||
SerializableSupplier<BiFunction<Oaf, Oaf, Oaf>> fn =
|
||||
functionFor(Strategy.SELECT_NEWER_AND_GET);
|
||||
SerializableSupplier<BiFunction<Oaf, Oaf, Oaf>> fn = functionFor(Strategy.SELECT_NEWER_AND_GET);
|
||||
|
||||
// then
|
||||
assertThrows(RuntimeException.class, () -> fn.get().apply(a, b));
|
||||
|
|
@ -194,8 +195,7 @@ public class MergeAndGetTest {
|
|||
Result b = mock(Result.class);
|
||||
|
||||
// when
|
||||
SerializableSupplier<BiFunction<Oaf, Oaf, Oaf>> fn =
|
||||
functionFor(Strategy.SELECT_NEWER_AND_GET);
|
||||
SerializableSupplier<BiFunction<Oaf, Oaf, Oaf>> fn = functionFor(Strategy.SELECT_NEWER_AND_GET);
|
||||
|
||||
// then
|
||||
assertThrows(RuntimeException.class, () -> fn.get().apply(a, b));
|
||||
|
|
@ -212,8 +212,7 @@ public class MergeAndGetTest {
|
|||
b.setLastupdatetimestamp(2L);
|
||||
|
||||
// when
|
||||
SerializableSupplier<BiFunction<Oaf, Oaf, Oaf>> fn =
|
||||
functionFor(Strategy.SELECT_NEWER_AND_GET);
|
||||
SerializableSupplier<BiFunction<Oaf, Oaf, Oaf>> fn = functionFor(Strategy.SELECT_NEWER_AND_GET);
|
||||
|
||||
// then
|
||||
assertThrows(RuntimeException.class, () -> fn.get().apply(a, b));
|
||||
|
|
@ -228,8 +227,7 @@ public class MergeAndGetTest {
|
|||
when(b.getLastupdatetimestamp()).thenReturn(2L);
|
||||
|
||||
// when
|
||||
SerializableSupplier<BiFunction<Oaf, Oaf, Oaf>> fn =
|
||||
functionFor(Strategy.SELECT_NEWER_AND_GET);
|
||||
SerializableSupplier<BiFunction<Oaf, Oaf, Oaf>> fn = functionFor(Strategy.SELECT_NEWER_AND_GET);
|
||||
|
||||
// then
|
||||
Oaf x = fn.get().apply(a, b);
|
||||
|
|
@ -246,8 +244,7 @@ public class MergeAndGetTest {
|
|||
when(b.getLastupdatetimestamp()).thenReturn(1L);
|
||||
|
||||
// when
|
||||
SerializableSupplier<BiFunction<Oaf, Oaf, Oaf>> fn =
|
||||
functionFor(Strategy.SELECT_NEWER_AND_GET);
|
||||
SerializableSupplier<BiFunction<Oaf, Oaf, Oaf>> fn = functionFor(Strategy.SELECT_NEWER_AND_GET);
|
||||
|
||||
// then
|
||||
Oaf x = fn.get().apply(a, b);
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
|
||||
package eu.dnetlib.dhp.actionmanager.promote;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.*;
|
||||
|
|
@ -27,8 +28,7 @@ import org.junit.jupiter.params.provider.Arguments;
|
|||
import org.junit.jupiter.params.provider.MethodSource;
|
||||
|
||||
public class PromoteActionPayloadForGraphTableJobTest {
|
||||
private static final ClassLoader cl =
|
||||
PromoteActionPayloadForGraphTableJobTest.class.getClassLoader();
|
||||
private static final ClassLoader cl = PromoteActionPayloadForGraphTableJobTest.class.getClassLoader();
|
||||
|
||||
private static SparkSession spark;
|
||||
|
||||
|
|
@ -52,8 +52,7 @@ public class PromoteActionPayloadForGraphTableJobTest {
|
|||
|
||||
@BeforeEach
|
||||
public void beforeEach() throws IOException {
|
||||
workingDir =
|
||||
Files.createTempDirectory(PromoteActionPayloadForGraphTableJobTest.class.getSimpleName());
|
||||
workingDir = Files.createTempDirectory(PromoteActionPayloadForGraphTableJobTest.class.getSimpleName());
|
||||
inputDir = workingDir.resolve("input");
|
||||
inputGraphRootDir = inputDir.resolve("graph");
|
||||
inputActionPayloadRootDir = inputDir.resolve("action_payload");
|
||||
|
|
@ -81,11 +80,10 @@ public class PromoteActionPayloadForGraphTableJobTest {
|
|||
Class<OafEntity> actionPayloadClazz = OafEntity.class;
|
||||
|
||||
// when
|
||||
RuntimeException exception =
|
||||
assertThrows(
|
||||
RuntimeException exception = assertThrows(
|
||||
RuntimeException.class,
|
||||
() ->
|
||||
PromoteActionPayloadForGraphTableJob.main(
|
||||
() -> PromoteActionPayloadForGraphTableJob
|
||||
.main(
|
||||
new String[] {
|
||||
"-isSparkSessionManaged",
|
||||
Boolean.FALSE.toString(),
|
||||
|
|
@ -104,16 +102,15 @@ public class PromoteActionPayloadForGraphTableJobTest {
|
|||
}));
|
||||
|
||||
// then
|
||||
String msg =
|
||||
String.format(
|
||||
String msg = String
|
||||
.format(
|
||||
"graph table class is not a subclass of action payload class: graph=%s, action=%s",
|
||||
rowClazz.getCanonicalName(), actionPayloadClazz.getCanonicalName());
|
||||
assertTrue(exception.getMessage().contains(msg));
|
||||
}
|
||||
|
||||
@ParameterizedTest(name = "strategy: {0}, graph table: {1}, action payload: {2}")
|
||||
@MethodSource(
|
||||
"eu.dnetlib.dhp.actionmanager.promote.PromoteActionPayloadForGraphTableJobTest#promoteJobTestParams")
|
||||
@MethodSource("eu.dnetlib.dhp.actionmanager.promote.PromoteActionPayloadForGraphTableJobTest#promoteJobTestParams")
|
||||
public void shouldPromoteActionPayloadForGraphTable(
|
||||
MergeAndGet.Strategy strategy,
|
||||
Class<? extends Oaf> rowClazz,
|
||||
|
|
@ -121,13 +118,12 @@ public class PromoteActionPayloadForGraphTableJobTest {
|
|||
throws Exception {
|
||||
// given
|
||||
Path inputGraphTableDir = createGraphTable(inputGraphRootDir, rowClazz);
|
||||
Path inputActionPayloadDir =
|
||||
createActionPayload(inputActionPayloadRootDir, rowClazz, actionPayloadClazz);
|
||||
Path outputGraphTableDir =
|
||||
outputDir.resolve("graph").resolve(rowClazz.getSimpleName().toLowerCase());
|
||||
Path inputActionPayloadDir = createActionPayload(inputActionPayloadRootDir, rowClazz, actionPayloadClazz);
|
||||
Path outputGraphTableDir = outputDir.resolve("graph").resolve(rowClazz.getSimpleName().toLowerCase());
|
||||
|
||||
// when
|
||||
PromoteActionPayloadForGraphTableJob.main(
|
||||
PromoteActionPayloadForGraphTableJob
|
||||
.main(
|
||||
new String[] {
|
||||
"-isSparkSessionManaged",
|
||||
Boolean.FALSE.toString(),
|
||||
|
|
@ -148,20 +144,21 @@ public class PromoteActionPayloadForGraphTableJobTest {
|
|||
// then
|
||||
assertTrue(Files.exists(outputGraphTableDir));
|
||||
|
||||
List<? extends Oaf> actualOutputRows =
|
||||
readGraphTableFromJobOutput(outputGraphTableDir.toString(), rowClazz).collectAsList()
|
||||
List<? extends Oaf> actualOutputRows = readGraphTableFromJobOutput(outputGraphTableDir.toString(), rowClazz)
|
||||
.collectAsList()
|
||||
.stream()
|
||||
.sorted(Comparator.comparingInt(Object::hashCode))
|
||||
.collect(Collectors.toList());
|
||||
String expectedOutputGraphTableJsonDumpPath =
|
||||
resultFileLocation(strategy, rowClazz, actionPayloadClazz);
|
||||
Path expectedOutputGraphTableJsonDumpFile =
|
||||
Paths.get(
|
||||
Objects.requireNonNull(cl.getResource(expectedOutputGraphTableJsonDumpPath))
|
||||
String expectedOutputGraphTableJsonDumpPath = resultFileLocation(strategy, rowClazz, actionPayloadClazz);
|
||||
Path expectedOutputGraphTableJsonDumpFile = Paths
|
||||
.get(
|
||||
Objects
|
||||
.requireNonNull(cl.getResource(expectedOutputGraphTableJsonDumpPath))
|
||||
.getFile());
|
||||
List<? extends Oaf> expectedOutputRows =
|
||||
readGraphTableFromJsonDump(expectedOutputGraphTableJsonDumpFile.toString(), rowClazz)
|
||||
.collectAsList().stream()
|
||||
List<? extends Oaf> expectedOutputRows = readGraphTableFromJsonDump(
|
||||
expectedOutputGraphTableJsonDumpFile.toString(), rowClazz)
|
||||
.collectAsList()
|
||||
.stream()
|
||||
.sorted(Comparator.comparingInt(Object::hashCode))
|
||||
.collect(Collectors.toList());
|
||||
assertIterableEquals(expectedOutputRows, actualOutputRows);
|
||||
|
|
@ -169,7 +166,8 @@ public class PromoteActionPayloadForGraphTableJobTest {
|
|||
}
|
||||
|
||||
public static Stream<Arguments> promoteJobTestParams() {
|
||||
return Stream.of(
|
||||
return Stream
|
||||
.of(
|
||||
arguments(
|
||||
MergeAndGet.Strategy.MERGE_FROM_AND_GET,
|
||||
eu.dnetlib.dhp.schema.oaf.Dataset.class,
|
||||
|
|
@ -196,8 +194,8 @@ public class PromoteActionPayloadForGraphTableJobTest {
|
|||
|
||||
private static <G extends Oaf> Path createGraphTable(Path inputGraphRootDir, Class<G> rowClazz) {
|
||||
String inputGraphTableJsonDumpPath = inputGraphTableJsonDumpLocation(rowClazz);
|
||||
Path inputGraphTableJsonDumpFile =
|
||||
Paths.get(Objects.requireNonNull(cl.getResource(inputGraphTableJsonDumpPath)).getFile());
|
||||
Path inputGraphTableJsonDumpFile = Paths
|
||||
.get(Objects.requireNonNull(cl.getResource(inputGraphTableJsonDumpPath)).getFile());
|
||||
Dataset<G> rowDS = readGraphTableFromJsonDump(inputGraphTableJsonDumpFile.toString(), rowClazz);
|
||||
String inputGraphTableName = rowClazz.getSimpleName().toLowerCase();
|
||||
Path inputGraphTableDir = inputGraphRootDir.resolve(inputGraphTableName);
|
||||
|
|
@ -206,7 +204,8 @@ public class PromoteActionPayloadForGraphTableJobTest {
|
|||
}
|
||||
|
||||
private static String inputGraphTableJsonDumpLocation(Class<? extends Oaf> rowClazz) {
|
||||
return String.format(
|
||||
return String
|
||||
.format(
|
||||
"%s/%s.json",
|
||||
"eu/dnetlib/dhp/actionmanager/promote/input/graph", rowClazz.getSimpleName().toLowerCase());
|
||||
}
|
||||
|
|
@ -227,14 +226,12 @@ public class PromoteActionPayloadForGraphTableJobTest {
|
|||
|
||||
private static <G extends Oaf, A extends Oaf> Path createActionPayload(
|
||||
Path inputActionPayloadRootDir, Class<G> rowClazz, Class<A> actionPayloadClazz) {
|
||||
String inputActionPayloadJsonDumpPath =
|
||||
inputActionPayloadJsonDumpLocation(rowClazz, actionPayloadClazz);
|
||||
Path inputActionPayloadJsonDumpFile =
|
||||
Paths.get(Objects.requireNonNull(cl.getResource(inputActionPayloadJsonDumpPath)).getFile());
|
||||
Dataset<String> actionPayloadDS =
|
||||
readActionPayloadFromJsonDump(inputActionPayloadJsonDumpFile.toString());
|
||||
Path inputActionPayloadDir =
|
||||
inputActionPayloadRootDir.resolve(actionPayloadClazz.getSimpleName().toLowerCase());
|
||||
String inputActionPayloadJsonDumpPath = inputActionPayloadJsonDumpLocation(rowClazz, actionPayloadClazz);
|
||||
Path inputActionPayloadJsonDumpFile = Paths
|
||||
.get(Objects.requireNonNull(cl.getResource(inputActionPayloadJsonDumpPath)).getFile());
|
||||
Dataset<String> actionPayloadDS = readActionPayloadFromJsonDump(inputActionPayloadJsonDumpFile.toString());
|
||||
Path inputActionPayloadDir = inputActionPayloadRootDir
|
||||
.resolve(actionPayloadClazz.getSimpleName().toLowerCase());
|
||||
writeActionPayloadAsJobInput(actionPayloadDS, inputActionPayloadDir.toString());
|
||||
return inputActionPayloadDir;
|
||||
}
|
||||
|
|
@ -242,7 +239,8 @@ public class PromoteActionPayloadForGraphTableJobTest {
|
|||
private static String inputActionPayloadJsonDumpLocation(
|
||||
Class<? extends Oaf> rowClazz, Class<? extends Oaf> actionPayloadClazz) {
|
||||
|
||||
return String.format(
|
||||
return String
|
||||
.format(
|
||||
"eu/dnetlib/dhp/actionmanager/promote/input/action_payload/%s_table/%s.json",
|
||||
rowClazz.getSimpleName().toLowerCase(), actionPayloadClazz.getSimpleName().toLowerCase());
|
||||
}
|
||||
|
|
@ -269,7 +267,8 @@ public class PromoteActionPayloadForGraphTableJobTest {
|
|||
MergeAndGet.Strategy strategy,
|
||||
Class<? extends Oaf> rowClazz,
|
||||
Class<? extends Oaf> actionPayloadClazz) {
|
||||
return String.format(
|
||||
return String
|
||||
.format(
|
||||
"eu/dnetlib/dhp/actionmanager/promote/output/graph/%s/%s/%s_action_payload/result.json",
|
||||
strategy.name().toLowerCase(),
|
||||
rowClazz.getSimpleName().toLowerCase(),
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
|
||||
package eu.dnetlib.dhp.actionmanager.promote;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
|
|
@ -43,13 +44,14 @@ public class PromoteActionPayloadFunctionsTest {
|
|||
@Test
|
||||
public void shouldThrowWhenTableTypeIsNotSubtypeOfActionPayloadType() {
|
||||
// given
|
||||
class OafImpl extends Oaf {}
|
||||
class OafImpl extends Oaf {
|
||||
}
|
||||
|
||||
// when
|
||||
assertThrows(
|
||||
RuntimeException.class,
|
||||
() ->
|
||||
PromoteActionPayloadFunctions.joinGraphTableWithActionPayloadAndMerge(
|
||||
() -> PromoteActionPayloadFunctions
|
||||
.joinGraphTableWithActionPayloadAndMerge(
|
||||
null, null, null, null, null, OafImplSubSub.class, OafImpl.class));
|
||||
}
|
||||
|
||||
|
|
@ -61,17 +63,16 @@ public class PromoteActionPayloadFunctionsTest {
|
|||
String id2 = "id2";
|
||||
String id3 = "id3";
|
||||
String id4 = "id4";
|
||||
List<OafImplSubSub> rowData =
|
||||
Arrays.asList(
|
||||
List<OafImplSubSub> rowData = Arrays
|
||||
.asList(
|
||||
createOafImplSubSub(id0),
|
||||
createOafImplSubSub(id1),
|
||||
createOafImplSubSub(id2),
|
||||
createOafImplSubSub(id3));
|
||||
Dataset<OafImplSubSub> rowDS =
|
||||
spark.createDataset(rowData, Encoders.bean(OafImplSubSub.class));
|
||||
Dataset<OafImplSubSub> rowDS = spark.createDataset(rowData, Encoders.bean(OafImplSubSub.class));
|
||||
|
||||
List<OafImplSubSub> actionPayloadData =
|
||||
Arrays.asList(
|
||||
List<OafImplSubSub> actionPayloadData = Arrays
|
||||
.asList(
|
||||
createOafImplSubSub(id1),
|
||||
createOafImplSubSub(id2),
|
||||
createOafImplSubSub(id2),
|
||||
|
|
@ -82,22 +83,20 @@ public class PromoteActionPayloadFunctionsTest {
|
|||
createOafImplSubSub(id4),
|
||||
createOafImplSubSub(id4),
|
||||
createOafImplSubSub(id4));
|
||||
Dataset<OafImplSubSub> actionPayloadDS =
|
||||
spark.createDataset(actionPayloadData, Encoders.bean(OafImplSubSub.class));
|
||||
Dataset<OafImplSubSub> actionPayloadDS = spark
|
||||
.createDataset(actionPayloadData, Encoders.bean(OafImplSubSub.class));
|
||||
|
||||
SerializableSupplier<Function<OafImplSubSub, String>> rowIdFn = () -> OafImplRoot::getId;
|
||||
SerializableSupplier<Function<OafImplSubSub, String>> actionPayloadIdFn =
|
||||
() -> OafImplRoot::getId;
|
||||
SerializableSupplier<BiFunction<OafImplSubSub, OafImplSubSub, OafImplSubSub>> mergeAndGetFn =
|
||||
() ->
|
||||
(x, y) -> {
|
||||
SerializableSupplier<Function<OafImplSubSub, String>> actionPayloadIdFn = () -> OafImplRoot::getId;
|
||||
SerializableSupplier<BiFunction<OafImplSubSub, OafImplSubSub, OafImplSubSub>> mergeAndGetFn = () -> (x,
|
||||
y) -> {
|
||||
x.merge(y);
|
||||
return x;
|
||||
};
|
||||
|
||||
// when
|
||||
List<OafImplSubSub> results =
|
||||
PromoteActionPayloadFunctions.joinGraphTableWithActionPayloadAndMerge(
|
||||
List<OafImplSubSub> results = PromoteActionPayloadFunctions
|
||||
.joinGraphTableWithActionPayloadAndMerge(
|
||||
rowDS,
|
||||
actionPayloadDS,
|
||||
rowIdFn,
|
||||
|
|
@ -115,7 +114,8 @@ public class PromoteActionPayloadFunctionsTest {
|
|||
assertEquals(3, results.stream().filter(x -> x.getId().equals(id3)).count());
|
||||
assertEquals(4, results.stream().filter(x -> x.getId().equals(id4)).count());
|
||||
|
||||
results.forEach(
|
||||
results
|
||||
.forEach(
|
||||
result -> {
|
||||
switch (result.getId()) {
|
||||
case "id0":
|
||||
|
|
@ -143,17 +143,16 @@ public class PromoteActionPayloadFunctionsTest {
|
|||
String id2 = "id2";
|
||||
String id3 = "id3";
|
||||
String id4 = "id4";
|
||||
List<OafImplSubSub> rowData =
|
||||
Arrays.asList(
|
||||
List<OafImplSubSub> rowData = Arrays
|
||||
.asList(
|
||||
createOafImplSubSub(id0),
|
||||
createOafImplSubSub(id1),
|
||||
createOafImplSubSub(id2),
|
||||
createOafImplSubSub(id3));
|
||||
Dataset<OafImplSubSub> rowDS =
|
||||
spark.createDataset(rowData, Encoders.bean(OafImplSubSub.class));
|
||||
Dataset<OafImplSubSub> rowDS = spark.createDataset(rowData, Encoders.bean(OafImplSubSub.class));
|
||||
|
||||
List<OafImplSub> actionPayloadData =
|
||||
Arrays.asList(
|
||||
List<OafImplSub> actionPayloadData = Arrays
|
||||
.asList(
|
||||
createOafImplSub(id1),
|
||||
createOafImplSub(id2),
|
||||
createOafImplSub(id2),
|
||||
|
|
@ -164,22 +163,19 @@ public class PromoteActionPayloadFunctionsTest {
|
|||
createOafImplSub(id4),
|
||||
createOafImplSub(id4),
|
||||
createOafImplSub(id4));
|
||||
Dataset<OafImplSub> actionPayloadDS =
|
||||
spark.createDataset(actionPayloadData, Encoders.bean(OafImplSub.class));
|
||||
Dataset<OafImplSub> actionPayloadDS = spark
|
||||
.createDataset(actionPayloadData, Encoders.bean(OafImplSub.class));
|
||||
|
||||
SerializableSupplier<Function<OafImplSubSub, String>> rowIdFn = () -> OafImplRoot::getId;
|
||||
SerializableSupplier<Function<OafImplSub, String>> actionPayloadIdFn =
|
||||
() -> OafImplRoot::getId;
|
||||
SerializableSupplier<BiFunction<OafImplSubSub, OafImplSub, OafImplSubSub>> mergeAndGetFn =
|
||||
() ->
|
||||
(x, y) -> {
|
||||
SerializableSupplier<Function<OafImplSub, String>> actionPayloadIdFn = () -> OafImplRoot::getId;
|
||||
SerializableSupplier<BiFunction<OafImplSubSub, OafImplSub, OafImplSubSub>> mergeAndGetFn = () -> (x, y) -> {
|
||||
x.merge(y);
|
||||
return x;
|
||||
};
|
||||
|
||||
// when
|
||||
List<OafImplSubSub> results =
|
||||
PromoteActionPayloadFunctions.joinGraphTableWithActionPayloadAndMerge(
|
||||
List<OafImplSubSub> results = PromoteActionPayloadFunctions
|
||||
.joinGraphTableWithActionPayloadAndMerge(
|
||||
rowDS,
|
||||
actionPayloadDS,
|
||||
rowIdFn,
|
||||
|
|
@ -197,7 +193,8 @@ public class PromoteActionPayloadFunctionsTest {
|
|||
assertEquals(3, results.stream().filter(x -> x.getId().equals(id3)).count());
|
||||
assertEquals(0, results.stream().filter(x -> x.getId().equals(id4)).count());
|
||||
|
||||
results.forEach(
|
||||
results
|
||||
.forEach(
|
||||
result -> {
|
||||
switch (result.getId()) {
|
||||
case "id0":
|
||||
|
|
@ -224,31 +221,28 @@ public class PromoteActionPayloadFunctionsTest {
|
|||
String id1 = "id1";
|
||||
String id2 = "id2";
|
||||
String id3 = "id3";
|
||||
List<OafImplSubSub> rowData =
|
||||
Arrays.asList(
|
||||
List<OafImplSubSub> rowData = Arrays
|
||||
.asList(
|
||||
createOafImplSubSub(id1),
|
||||
createOafImplSubSub(id2),
|
||||
createOafImplSubSub(id2),
|
||||
createOafImplSubSub(id3),
|
||||
createOafImplSubSub(id3),
|
||||
createOafImplSubSub(id3));
|
||||
Dataset<OafImplSubSub> rowDS =
|
||||
spark.createDataset(rowData, Encoders.bean(OafImplSubSub.class));
|
||||
Dataset<OafImplSubSub> rowDS = spark.createDataset(rowData, Encoders.bean(OafImplSubSub.class));
|
||||
|
||||
SerializableSupplier<Function<OafImplSubSub, String>> rowIdFn = () -> OafImplRoot::getId;
|
||||
SerializableSupplier<BiFunction<OafImplSubSub, OafImplSubSub, OafImplSubSub>> mergeAndGetFn =
|
||||
() ->
|
||||
(x, y) -> {
|
||||
SerializableSupplier<BiFunction<OafImplSubSub, OafImplSubSub, OafImplSubSub>> mergeAndGetFn = () -> (x,
|
||||
y) -> {
|
||||
x.merge(y);
|
||||
return x;
|
||||
};
|
||||
SerializableSupplier<OafImplSubSub> zeroFn = OafImplSubSub::new;
|
||||
SerializableSupplier<Function<OafImplSubSub, Boolean>> isNotZeroFn =
|
||||
() -> x -> Objects.nonNull(x.getId());
|
||||
SerializableSupplier<Function<OafImplSubSub, Boolean>> isNotZeroFn = () -> x -> Objects.nonNull(x.getId());
|
||||
|
||||
// when
|
||||
List<OafImplSubSub> results =
|
||||
PromoteActionPayloadFunctions.groupGraphTableByIdAndMerge(
|
||||
List<OafImplSubSub> results = PromoteActionPayloadFunctions
|
||||
.groupGraphTableByIdAndMerge(
|
||||
rowDS, rowIdFn, mergeAndGetFn, zeroFn, isNotZeroFn, OafImplSubSub.class)
|
||||
.collectAsList();
|
||||
|
||||
|
|
@ -258,7 +252,8 @@ public class PromoteActionPayloadFunctionsTest {
|
|||
assertEquals(1, results.stream().filter(x -> x.getId().equals(id2)).count());
|
||||
assertEquals(1, results.stream().filter(x -> x.getId().equals(id3)).count());
|
||||
|
||||
results.forEach(
|
||||
results
|
||||
.forEach(
|
||||
result -> {
|
||||
switch (result.getId()) {
|
||||
case "id1":
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
|
||||
package eu.dnetlib.dhp.collection;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
|
|
@ -40,20 +41,22 @@ public class GenerateNativeStoreSparkJob {
|
|||
final LongAccumulator totalItems,
|
||||
final LongAccumulator invalidRecords) {
|
||||
|
||||
if (totalItems != null) totalItems.add(1);
|
||||
if (totalItems != null)
|
||||
totalItems.add(1);
|
||||
try {
|
||||
SAXReader reader = new SAXReader();
|
||||
Document document =
|
||||
reader.read(new ByteArrayInputStream(input.getBytes(StandardCharsets.UTF_8)));
|
||||
Document document = reader.read(new ByteArrayInputStream(input.getBytes(StandardCharsets.UTF_8)));
|
||||
Node node = document.selectSingleNode(xpath);
|
||||
final String originalIdentifier = node.getText();
|
||||
if (StringUtils.isBlank(originalIdentifier)) {
|
||||
if (invalidRecords != null) invalidRecords.add(1);
|
||||
if (invalidRecords != null)
|
||||
invalidRecords.add(1);
|
||||
return null;
|
||||
}
|
||||
return new MetadataRecord(originalIdentifier, encoding, provenance, input, dateOfCollection);
|
||||
} catch (Throwable e) {
|
||||
if (invalidRecords != null) invalidRecords.add(1);
|
||||
if (invalidRecords != null)
|
||||
invalidRecords.add(1);
|
||||
e.printStackTrace();
|
||||
return null;
|
||||
}
|
||||
|
|
@ -61,18 +64,19 @@ public class GenerateNativeStoreSparkJob {
|
|||
|
||||
public static void main(String[] args) throws Exception {
|
||||
|
||||
final ArgumentApplicationParser parser =
|
||||
new ArgumentApplicationParser(
|
||||
IOUtils.toString(
|
||||
GenerateNativeStoreSparkJob.class.getResourceAsStream(
|
||||
final ArgumentApplicationParser parser = new ArgumentApplicationParser(
|
||||
IOUtils
|
||||
.toString(
|
||||
GenerateNativeStoreSparkJob.class
|
||||
.getResourceAsStream(
|
||||
"/eu/dnetlib/dhp/collection/collection_input_parameters.json")));
|
||||
parser.parseArgument(args);
|
||||
final ObjectMapper jsonMapper = new ObjectMapper();
|
||||
final Provenance provenance = jsonMapper.readValue(parser.get("provenance"), Provenance.class);
|
||||
final long dateOfCollection = new Long(parser.get("dateOfCollection"));
|
||||
|
||||
final SparkSession spark =
|
||||
SparkSession.builder()
|
||||
final SparkSession spark = SparkSession
|
||||
.builder()
|
||||
.appName("GenerateNativeStoreSparkJob")
|
||||
.master(parser.get("master"))
|
||||
.getOrCreate();
|
||||
|
|
@ -80,20 +84,18 @@ public class GenerateNativeStoreSparkJob {
|
|||
final Map<String, String> ongoingMap = new HashMap<>();
|
||||
final Map<String, String> reportMap = new HashMap<>();
|
||||
|
||||
final boolean test =
|
||||
parser.get("isTest") == null ? false : Boolean.valueOf(parser.get("isTest"));
|
||||
final boolean test = parser.get("isTest") == null ? false : Boolean.valueOf(parser.get("isTest"));
|
||||
|
||||
final JavaSparkContext sc = new JavaSparkContext(spark.sparkContext());
|
||||
|
||||
final JavaPairRDD<IntWritable, Text> inputRDD =
|
||||
sc.sequenceFile(parser.get("input"), IntWritable.class, Text.class);
|
||||
final JavaPairRDD<IntWritable, Text> inputRDD = sc
|
||||
.sequenceFile(parser.get("input"), IntWritable.class, Text.class);
|
||||
|
||||
final LongAccumulator totalItems = sc.sc().longAccumulator("TotalItems");
|
||||
|
||||
final LongAccumulator invalidRecords = sc.sc().longAccumulator("InvalidRecords");
|
||||
|
||||
final MessageManager manager =
|
||||
new MessageManager(
|
||||
final MessageManager manager = new MessageManager(
|
||||
parser.get("rabbitHost"),
|
||||
parser.get("rabbitUser"),
|
||||
parser.get("rabbitPassword"),
|
||||
|
|
@ -101,11 +103,9 @@ public class GenerateNativeStoreSparkJob {
|
|||
false,
|
||||
null);
|
||||
|
||||
final JavaRDD<MetadataRecord> mappeRDD =
|
||||
inputRDD
|
||||
final JavaRDD<MetadataRecord> mappeRDD = inputRDD
|
||||
.map(
|
||||
item ->
|
||||
parseRecord(
|
||||
item -> parseRecord(
|
||||
item._2().toString(),
|
||||
parser.get("xpath"),
|
||||
parser.get("encoding"),
|
||||
|
|
@ -118,7 +118,8 @@ public class GenerateNativeStoreSparkJob {
|
|||
|
||||
ongoingMap.put("ongoing", "0");
|
||||
if (!test) {
|
||||
manager.sendMessage(
|
||||
manager
|
||||
.sendMessage(
|
||||
new Message(
|
||||
parser.get("workflowId"), "DataFrameCreation", MessageType.ONGOING, ongoingMap),
|
||||
parser.get("rabbitOngoingQueue"),
|
||||
|
|
@ -132,7 +133,8 @@ public class GenerateNativeStoreSparkJob {
|
|||
mdStoreRecords.add(mdstore.count());
|
||||
ongoingMap.put("ongoing", "" + totalItems.value());
|
||||
if (!test) {
|
||||
manager.sendMessage(
|
||||
manager
|
||||
.sendMessage(
|
||||
new Message(
|
||||
parser.get("workflowId"), "DataFrameCreation", MessageType.ONGOING, ongoingMap),
|
||||
parser.get("rabbitOngoingQueue"),
|
||||
|
|
@ -144,7 +146,8 @@ public class GenerateNativeStoreSparkJob {
|
|||
reportMap.put("invalidRecords", "" + invalidRecords.value());
|
||||
reportMap.put("mdStoreSize", "" + mdStoreRecords.value());
|
||||
if (!test) {
|
||||
manager.sendMessage(
|
||||
manager
|
||||
.sendMessage(
|
||||
new Message(parser.get("workflowId"), "Collection", MessageType.REPORT, reportMap),
|
||||
parser.get("rabbitReportQueue"),
|
||||
true,
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
|
||||
package eu.dnetlib.dhp.collection.plugin;
|
||||
|
||||
import eu.dnetlib.collector.worker.model.ApiDescriptor;
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
|
||||
package eu.dnetlib.dhp.collection.plugin.oai;
|
||||
|
||||
import com.google.common.base.Splitter;
|
||||
|
|
@ -33,7 +34,8 @@ public class OaiCollectorPlugin implements CollectorPlugin {
|
|||
|
||||
final List<String> sets = new ArrayList<>();
|
||||
if (setParam != null) {
|
||||
sets.addAll(
|
||||
sets
|
||||
.addAll(
|
||||
Lists.newArrayList(Splitter.on(",").omitEmptyStrings().trimResults().split(setParam)));
|
||||
}
|
||||
if (sets.isEmpty()) {
|
||||
|
|
@ -57,15 +59,15 @@ public class OaiCollectorPlugin implements CollectorPlugin {
|
|||
throw new DnetCollectorException("Invalid date (YYYY-MM-DD): " + untilDate);
|
||||
}
|
||||
|
||||
final Iterator<Iterator<String>> iters =
|
||||
sets.stream()
|
||||
final Iterator<Iterator<String>> iters = sets
|
||||
.stream()
|
||||
.map(
|
||||
set ->
|
||||
getOaiIteratorFactory()
|
||||
set -> getOaiIteratorFactory()
|
||||
.newIterator(baseUrl, mdFormat, set, fromDate, untilDate))
|
||||
.iterator();
|
||||
|
||||
return StreamSupport.stream(
|
||||
return StreamSupport
|
||||
.stream(
|
||||
Spliterators.spliteratorUnknownSize(Iterators.concat(iters), Spliterator.ORDERED), false);
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
|
||||
package eu.dnetlib.dhp.collection.plugin.oai;
|
||||
|
||||
import eu.dnetlib.dhp.collection.worker.DnetCollectorException;
|
||||
|
|
@ -86,12 +87,12 @@ public class OaiIterator implements Iterator<String> {
|
|||
}
|
||||
|
||||
@Override
|
||||
public void remove() {}
|
||||
public void remove() {
|
||||
}
|
||||
|
||||
private String firstPage() throws DnetCollectorException {
|
||||
try {
|
||||
String url =
|
||||
baseUrl + "?verb=ListRecords&metadataPrefix=" + URLEncoder.encode(mdFormat, "UTF-8");
|
||||
String url = baseUrl + "?verb=ListRecords&metadataPrefix=" + URLEncoder.encode(mdFormat, "UTF-8");
|
||||
if (set != null && !set.isEmpty()) {
|
||||
url += "&set=" + URLEncoder.encode(set, "UTF-8");
|
||||
}
|
||||
|
|
@ -154,8 +155,7 @@ public class OaiIterator implements Iterator<String> {
|
|||
}
|
||||
}
|
||||
|
||||
final Node errorNode =
|
||||
doc.selectSingleNode("/*[local-name()='OAI-PMH']/*[local-name()='error']");
|
||||
final Node errorNode = doc.selectSingleNode("/*[local-name()='OAI-PMH']/*[local-name()='error']");
|
||||
if (errorNode != null) {
|
||||
final String code = errorNode.valueOf("@code");
|
||||
if ("noRecordsMatch".equalsIgnoreCase(code.trim())) {
|
||||
|
|
@ -166,8 +166,7 @@ public class OaiIterator implements Iterator<String> {
|
|||
}
|
||||
}
|
||||
|
||||
for (final Object o :
|
||||
doc.selectNodes("//*[local-name()='ListRecords']/*[local-name()='record']")) {
|
||||
for (final Object o : doc.selectNodes("//*[local-name()='ListRecords']/*[local-name()='record']")) {
|
||||
queue.add(((Node) o).asXML());
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
|
||||
package eu.dnetlib.dhp.collection.plugin.oai;
|
||||
|
||||
import eu.dnetlib.dhp.collection.worker.utils.HttpConnector;
|
||||
|
|
@ -17,7 +18,8 @@ public class OaiIteratorFactory {
|
|||
}
|
||||
|
||||
private HttpConnector getHttpConnector() {
|
||||
if (httpConnector == null) httpConnector = new HttpConnector();
|
||||
if (httpConnector == null)
|
||||
httpConnector = new HttpConnector();
|
||||
return httpConnector;
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
|
||||
package eu.dnetlib.dhp.collection.worker;
|
||||
|
||||
public class DnetCollectorException extends Exception {
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
|
||||
package eu.dnetlib.dhp.collection.worker;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
|
|
@ -45,8 +46,7 @@ public class DnetCollectorWorker {
|
|||
public void collect() throws DnetCollectorException {
|
||||
try {
|
||||
final ObjectMapper jsonMapper = new ObjectMapper();
|
||||
final ApiDescriptor api =
|
||||
jsonMapper.readValue(argumentParser.get("apidescriptor"), ApiDescriptor.class);
|
||||
final ApiDescriptor api = jsonMapper.readValue(argumentParser.get("apidescriptor"), ApiDescriptor.class);
|
||||
|
||||
final CollectorPlugin plugin = collectorPluginFactory.getPluginByProtocol(api.getProtocol());
|
||||
|
||||
|
|
@ -71,8 +71,8 @@ public class DnetCollectorWorker {
|
|||
final Map<String, String> ongoingMap = new HashMap<>();
|
||||
final Map<String, String> reportMap = new HashMap<>();
|
||||
final AtomicInteger counter = new AtomicInteger(0);
|
||||
try (SequenceFile.Writer writer =
|
||||
SequenceFile.createWriter(
|
||||
try (SequenceFile.Writer writer = SequenceFile
|
||||
.createWriter(
|
||||
conf,
|
||||
SequenceFile.Writer.file(hdfswritepath),
|
||||
SequenceFile.Writer.keyClass(IntWritable.class),
|
||||
|
|
@ -88,9 +88,11 @@ public class DnetCollectorWorker {
|
|||
if (counter.get() % 10 == 0) {
|
||||
try {
|
||||
ongoingMap.put("ongoing", "" + counter.get());
|
||||
log.debug(
|
||||
log
|
||||
.debug(
|
||||
"Sending message: "
|
||||
+ manager.sendMessage(
|
||||
+ manager
|
||||
.sendMessage(
|
||||
new Message(
|
||||
argumentParser.get("workflowId"),
|
||||
"Collection",
|
||||
|
|
@ -111,14 +113,16 @@ public class DnetCollectorWorker {
|
|||
});
|
||||
}
|
||||
ongoingMap.put("ongoing", "" + counter.get());
|
||||
manager.sendMessage(
|
||||
manager
|
||||
.sendMessage(
|
||||
new Message(
|
||||
argumentParser.get("workflowId"), "Collection", MessageType.ONGOING, ongoingMap),
|
||||
argumentParser.get("rabbitOngoingQueue"),
|
||||
true,
|
||||
false);
|
||||
reportMap.put("collected", "" + counter.get());
|
||||
manager.sendMessage(
|
||||
manager
|
||||
.sendMessage(
|
||||
new Message(
|
||||
argumentParser.get("workflowId"), "Collection", MessageType.REPORT, reportMap),
|
||||
argumentParser.get("rabbitOngoingQueue"),
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
|
||||
package eu.dnetlib.dhp.collection.worker;
|
||||
|
||||
import eu.dnetlib.dhp.application.ArgumentApplicationParser;
|
||||
|
|
@ -8,9 +9,9 @@ import org.slf4j.Logger;
|
|||
import org.slf4j.LoggerFactory;
|
||||
|
||||
/**
|
||||
* DnetCollectortWorkerApplication is the main class responsible to start the Dnet Collection into
|
||||
* HDFS. This module will be executed on the hadoop cluster and taking in input some parameters that
|
||||
* tells it which is the right collector plugin to use and where store the data into HDFS path
|
||||
* DnetCollectortWorkerApplication is the main class responsible to start the Dnet Collection into HDFS. This module
|
||||
* will be executed on the hadoop cluster and taking in input some parameters that tells it which is the right collector
|
||||
* plugin to use and where store the data into HDFS path
|
||||
*
|
||||
* @author Sandro La Bruzzo
|
||||
*/
|
||||
|
|
@ -25,24 +26,23 @@ public class DnetCollectorWorkerApplication {
|
|||
/** @param args */
|
||||
public static void main(final String[] args) throws Exception {
|
||||
|
||||
argumentParser =
|
||||
new ArgumentApplicationParser(
|
||||
IOUtils.toString(
|
||||
DnetCollectorWorker.class.getResourceAsStream(
|
||||
argumentParser = new ArgumentApplicationParser(
|
||||
IOUtils
|
||||
.toString(
|
||||
DnetCollectorWorker.class
|
||||
.getResourceAsStream(
|
||||
"/eu/dnetlib/collector/worker/collector_parameter.json")));
|
||||
argumentParser.parseArgument(args);
|
||||
log.info("hdfsPath =" + argumentParser.get("hdfsPath"));
|
||||
log.info("json = " + argumentParser.get("apidescriptor"));
|
||||
final MessageManager manager =
|
||||
new MessageManager(
|
||||
final MessageManager manager = new MessageManager(
|
||||
argumentParser.get("rabbitHost"),
|
||||
argumentParser.get("rabbitUser"),
|
||||
argumentParser.get("rabbitPassword"),
|
||||
false,
|
||||
false,
|
||||
null);
|
||||
final DnetCollectorWorker worker =
|
||||
new DnetCollectorWorker(collectorPluginFactory, argumentParser, manager);
|
||||
final DnetCollectorWorker worker = new DnetCollectorWorker(collectorPluginFactory, argumentParser, manager);
|
||||
worker.collect();
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue