forked from D-Net/dnet-hadoop
Compare commits
42 Commits
cd9c51fd7a
...
22eaf211e8
Author | SHA1 | Date |
---|---|---|
dimitrispie | 22eaf211e8 | |
dimitrispie | 6b247524a8 | |
dimitrispie | 566e2459a8 | |
dimitrispie | 733abdffe3 | |
dimitrispie | 7a7eb30a3f | |
dimitrispie | 0654343479 | |
dimitrispie | fc40a4ddf7 | |
dimitrispie | 26f5a5209c | |
dimitrispie | dbb12314fe | |
Claudio Atzori | c1b6ae47cd | |
dimitrispie | 195111119e | |
dimitrispie | 2036f8247c | |
dimitrispie | 3d42652c44 | |
dimitrispie | 264939a01a | |
dimitrispie | dd1098255e | |
dimitrispie | cdf11c8465 | |
dimitrispie | 553880f3f1 | |
dimitrispie | bc40663b61 | |
dimitrispie | 6418208d57 | |
dimitrispie | 50cda834bd | |
dimitrispie | bb2ef1b74d | |
dimitrispie | 2cd0514397 | |
Miriam Baglioni | 59c36eb185 | |
Claudio Atzori | 6b8c357381 | |
Claudio Atzori | c0d2b62e46 | |
Claudio Atzori | a3948c1f6e | |
Claudio Atzori | fddbc8364e | |
Alessia Bardi | 6208b04f1d | |
Sandro La Bruzzo | 9ca438d9b1 | |
Sandro La Bruzzo | 42ff7a5665 | |
Sandro La Bruzzo | ebe6aa6d38 | |
dimitrispie | da1f123d7b | |
dimitrispie | c90dd653c2 | |
Dimitris | 6cb4cbb75e | |
Dimitris | 1e06815cdb | |
dimitrispie | 4125b71661 | |
dimitrispie | 533bde3228 | |
Dimitris | ac5b7f89b5 | |
Dimitris | b081e01dad | |
Claudio Atzori | a4cfabdbc6 | |
Claudio Atzori | 338327171d | |
Claudio Atzori | 6cbda49112 |
|
@ -25,6 +25,10 @@
|
|||
/build
|
||||
spark-warehouse
|
||||
/**/job-override.properties
|
||||
<<<<<<< HEAD
|
||||
/**/*.log
|
||||
=======
|
||||
/**/*.log
|
||||
/**/.factorypath
|
||||
|
||||
>>>>>>> upstream/master
|
||||
|
|
0
dhp-build/dhp-build-assembly-resources/src/main/resources/assemblies/oozie-installer.xml
Normal file → Executable file
0
dhp-build/dhp-build-assembly-resources/src/main/resources/assemblies/oozie-installer.xml
Normal file → Executable file
0
dhp-build/dhp-build-assembly-resources/src/main/resources/assemblies/tests.xml
Normal file → Executable file
0
dhp-build/dhp-build-assembly-resources/src/main/resources/assemblies/tests.xml
Normal file → Executable file
0
dhp-build/dhp-build-assembly-resources/src/main/resources/commands/get_working_dir.sh
Normal file → Executable file
0
dhp-build/dhp-build-assembly-resources/src/main/resources/commands/get_working_dir.sh
Normal file → Executable file
0
dhp-build/dhp-build-assembly-resources/src/main/resources/commands/print_working_dir.sh
Normal file → Executable file
0
dhp-build/dhp-build-assembly-resources/src/main/resources/commands/print_working_dir.sh
Normal file → Executable file
0
dhp-build/dhp-build-assembly-resources/src/main/resources/commands/readme.markdown
Normal file → Executable file
0
dhp-build/dhp-build-assembly-resources/src/main/resources/commands/readme.markdown
Normal file → Executable file
0
dhp-build/dhp-build-assembly-resources/src/main/resources/commands/run_workflow.sh
Normal file → Executable file
0
dhp-build/dhp-build-assembly-resources/src/main/resources/commands/run_workflow.sh
Normal file → Executable file
0
dhp-build/dhp-build-assembly-resources/src/main/resources/commands/upload_workflow.sh
Normal file → Executable file
0
dhp-build/dhp-build-assembly-resources/src/main/resources/commands/upload_workflow.sh
Normal file → Executable file
0
dhp-build/dhp-build-assembly-resources/src/main/resources/project-default.properties
Normal file → Executable file
0
dhp-build/dhp-build-assembly-resources/src/main/resources/project-default.properties
Normal file → Executable file
0
dhp-build/dhp-build-properties-maven-plugin/src/main/java/eu/dnetlib/maven/plugin/properties/GenerateOoziePropertiesMojo.java
Normal file → Executable file
0
dhp-build/dhp-build-properties-maven-plugin/src/main/java/eu/dnetlib/maven/plugin/properties/GenerateOoziePropertiesMojo.java
Normal file → Executable file
0
dhp-build/dhp-build-properties-maven-plugin/src/test/resources/eu/dnetlib/maven/plugin/properties/included.properties
Normal file → Executable file
0
dhp-build/dhp-build-properties-maven-plugin/src/test/resources/eu/dnetlib/maven/plugin/properties/included.properties
Normal file → Executable file
0
dhp-build/dhp-code-style/src/main/resources/eclipse/formatter_aosp.xml
Normal file → Executable file
0
dhp-build/dhp-code-style/src/main/resources/eclipse/formatter_aosp.xml
Normal file → Executable file
0
dhp-build/dhp-code-style/src/main/resources/eclipse/formatter_dnet.xml
Normal file → Executable file
0
dhp-build/dhp-code-style/src/main/resources/eclipse/formatter_dnet.xml
Normal file → Executable file
0
dhp-build/dhp-code-style/src/main/resources/eclipse/formatter_google.xml
Normal file → Executable file
0
dhp-build/dhp-code-style/src/main/resources/eclipse/formatter_google.xml
Normal file → Executable file
0
dhp-common/src/main/java/eu/dnetlib/dhp/application/ArgumentApplicationParser.java
Normal file → Executable file
0
dhp-common/src/main/java/eu/dnetlib/dhp/application/ArgumentApplicationParser.java
Normal file → Executable file
0
dhp-common/src/main/java/eu/dnetlib/dhp/application/OptionsParameter.java
Normal file → Executable file
0
dhp-common/src/main/java/eu/dnetlib/dhp/application/OptionsParameter.java
Normal file → Executable file
0
dhp-common/src/main/java/eu/dnetlib/dhp/application/SparkScalaApplication.scala
Normal file → Executable file
0
dhp-common/src/main/java/eu/dnetlib/dhp/application/SparkScalaApplication.scala
Normal file → Executable file
0
dhp-common/src/main/java/eu/dnetlib/dhp/collection/ApiDescriptor.java
Normal file → Executable file
0
dhp-common/src/main/java/eu/dnetlib/dhp/collection/ApiDescriptor.java
Normal file → Executable file
0
dhp-common/src/main/java/eu/dnetlib/dhp/common/FunctionalInterfaceSupport.java
Normal file → Executable file
0
dhp-common/src/main/java/eu/dnetlib/dhp/common/FunctionalInterfaceSupport.java
Normal file → Executable file
0
dhp-common/src/main/java/eu/dnetlib/dhp/common/GraphResultMapper.java
Normal file → Executable file
0
dhp-common/src/main/java/eu/dnetlib/dhp/common/GraphResultMapper.java
Normal file → Executable file
0
dhp-common/src/main/java/eu/dnetlib/dhp/common/SparkSessionSupport.java
Normal file → Executable file
0
dhp-common/src/main/java/eu/dnetlib/dhp/common/SparkSessionSupport.java
Normal file → Executable file
0
dhp-common/src/main/java/eu/dnetlib/dhp/common/aggregation/AggregatorReport.java
Normal file → Executable file
0
dhp-common/src/main/java/eu/dnetlib/dhp/common/aggregation/AggregatorReport.java
Normal file → Executable file
0
dhp-common/src/main/java/eu/dnetlib/dhp/common/api/InputStreamRequestBody.java
Normal file → Executable file
0
dhp-common/src/main/java/eu/dnetlib/dhp/common/api/InputStreamRequestBody.java
Normal file → Executable file
0
dhp-common/src/main/java/eu/dnetlib/dhp/common/api/MissingConceptDoiException.java
Normal file → Executable file
0
dhp-common/src/main/java/eu/dnetlib/dhp/common/api/MissingConceptDoiException.java
Normal file → Executable file
0
dhp-common/src/main/java/eu/dnetlib/dhp/common/api/ZenodoAPIClient.java
Normal file → Executable file
0
dhp-common/src/main/java/eu/dnetlib/dhp/common/api/ZenodoAPIClient.java
Normal file → Executable file
0
dhp-common/src/main/java/eu/dnetlib/dhp/common/api/zenodo/Community.java
Normal file → Executable file
0
dhp-common/src/main/java/eu/dnetlib/dhp/common/api/zenodo/Community.java
Normal file → Executable file
0
dhp-common/src/main/java/eu/dnetlib/dhp/common/api/zenodo/Creator.java
Normal file → Executable file
0
dhp-common/src/main/java/eu/dnetlib/dhp/common/api/zenodo/Creator.java
Normal file → Executable file
0
dhp-common/src/main/java/eu/dnetlib/dhp/common/api/zenodo/Grant.java
Normal file → Executable file
0
dhp-common/src/main/java/eu/dnetlib/dhp/common/api/zenodo/Grant.java
Normal file → Executable file
0
dhp-common/src/main/java/eu/dnetlib/dhp/common/api/zenodo/Links.java
Normal file → Executable file
0
dhp-common/src/main/java/eu/dnetlib/dhp/common/api/zenodo/Links.java
Normal file → Executable file
0
dhp-common/src/main/java/eu/dnetlib/dhp/common/api/zenodo/Metadata.java
Normal file → Executable file
0
dhp-common/src/main/java/eu/dnetlib/dhp/common/api/zenodo/Metadata.java
Normal file → Executable file
0
dhp-common/src/main/java/eu/dnetlib/dhp/common/api/zenodo/PrereserveDoi.java
Normal file → Executable file
0
dhp-common/src/main/java/eu/dnetlib/dhp/common/api/zenodo/PrereserveDoi.java
Normal file → Executable file
0
dhp-common/src/main/java/eu/dnetlib/dhp/common/api/zenodo/RelatedIdentifier.java
Normal file → Executable file
0
dhp-common/src/main/java/eu/dnetlib/dhp/common/api/zenodo/RelatedIdentifier.java
Normal file → Executable file
0
dhp-common/src/main/java/eu/dnetlib/dhp/common/api/zenodo/ZenodoModel.java
Normal file → Executable file
0
dhp-common/src/main/java/eu/dnetlib/dhp/common/api/zenodo/ZenodoModel.java
Normal file → Executable file
0
dhp-common/src/main/java/eu/dnetlib/dhp/common/api/zenodo/ZenodoModelList.java
Normal file → Executable file
0
dhp-common/src/main/java/eu/dnetlib/dhp/common/api/zenodo/ZenodoModelList.java
Normal file → Executable file
0
dhp-common/src/main/java/eu/dnetlib/dhp/common/collection/CollectorException.java
Normal file → Executable file
0
dhp-common/src/main/java/eu/dnetlib/dhp/common/collection/CollectorException.java
Normal file → Executable file
0
dhp-common/src/main/java/eu/dnetlib/dhp/common/collection/GetCSV.java
Normal file → Executable file
0
dhp-common/src/main/java/eu/dnetlib/dhp/common/collection/GetCSV.java
Normal file → Executable file
0
dhp-common/src/main/java/eu/dnetlib/dhp/common/collection/HttpClientParams.java
Normal file → Executable file
0
dhp-common/src/main/java/eu/dnetlib/dhp/common/collection/HttpClientParams.java
Normal file → Executable file
0
dhp-common/src/main/java/eu/dnetlib/dhp/common/collection/HttpConnector2.java
Normal file → Executable file
0
dhp-common/src/main/java/eu/dnetlib/dhp/common/collection/HttpConnector2.java
Normal file → Executable file
0
dhp-common/src/main/java/eu/dnetlib/dhp/common/rest/DNetRestClient.java
Normal file → Executable file
0
dhp-common/src/main/java/eu/dnetlib/dhp/common/rest/DNetRestClient.java
Normal file → Executable file
0
dhp-common/src/main/java/eu/dnetlib/dhp/common/vocabulary/Vocabulary.java
Normal file → Executable file
0
dhp-common/src/main/java/eu/dnetlib/dhp/common/vocabulary/Vocabulary.java
Normal file → Executable file
0
dhp-common/src/main/java/eu/dnetlib/dhp/common/vocabulary/VocabularyGroup.java
Normal file → Executable file
0
dhp-common/src/main/java/eu/dnetlib/dhp/common/vocabulary/VocabularyGroup.java
Normal file → Executable file
0
dhp-common/src/main/java/eu/dnetlib/dhp/common/vocabulary/VocabularyTerm.java
Normal file → Executable file
0
dhp-common/src/main/java/eu/dnetlib/dhp/common/vocabulary/VocabularyTerm.java
Normal file → Executable file
|
@ -120,9 +120,17 @@ public class AuthorMerger {
|
|||
}
|
||||
|
||||
public static String pidToComparableString(StructuredProperty pid) {
|
||||
<<<<<<< HEAD
|
||||
if (pid == null)
|
||||
return "";
|
||||
return (pid.getQualifier() != null
|
||||
? pid.getQualifier().getClassid() != null ? pid.getQualifier().getClassid().toLowerCase() : ""
|
||||
: "")
|
||||
=======
|
||||
final String classid = pid.getQualifier().getClassid() != null ? pid.getQualifier().getClassid().toLowerCase()
|
||||
: "";
|
||||
return (pid.getQualifier() != null ? classid : "")
|
||||
>>>>>>> upstream/master
|
||||
+ (pid.getValue() != null ? pid.getValue().toLowerCase() : "");
|
||||
}
|
||||
|
||||
|
|
0
dhp-common/src/main/java/eu/dnetlib/dhp/parser/utility/VtdException.java
Normal file → Executable file
0
dhp-common/src/main/java/eu/dnetlib/dhp/parser/utility/VtdException.java
Normal file → Executable file
0
dhp-common/src/main/java/eu/dnetlib/dhp/parser/utility/VtdUtilityParser.java
Normal file → Executable file
0
dhp-common/src/main/java/eu/dnetlib/dhp/parser/utility/VtdUtilityParser.java
Normal file → Executable file
36
dhp-common/src/main/java/eu/dnetlib/dhp/schema/oaf/utils/GraphCleaningFunctions.java
Normal file → Executable file
36
dhp-common/src/main/java/eu/dnetlib/dhp/schema/oaf/utils/GraphCleaningFunctions.java
Normal file → Executable file
|
@ -16,6 +16,8 @@ import com.github.sisyphsu.dateparser.DateParserUtils;
|
|||
import com.google.common.collect.Lists;
|
||||
import com.google.common.collect.Sets;
|
||||
|
||||
import eu.dnetlib.dhp.common.vocabulary.Vocabulary;
|
||||
import eu.dnetlib.dhp.common.vocabulary.VocabularyGroup;
|
||||
import eu.dnetlib.dhp.schema.common.ModelConstants;
|
||||
import eu.dnetlib.dhp.schema.common.ModelSupport;
|
||||
import eu.dnetlib.dhp.schema.oaf.*;
|
||||
|
@ -115,7 +117,7 @@ public class GraphCleaningFunctions extends CleaningFunctions {
|
|||
return true;
|
||||
}
|
||||
|
||||
public static <T extends Oaf> T cleanup(T value) {
|
||||
public static <T extends Oaf> T cleanup(T value, VocabularyGroup vocs) {
|
||||
if (value instanceof Datasource) {
|
||||
// nothing to clean here
|
||||
} else if (value instanceof Project) {
|
||||
|
@ -234,6 +236,38 @@ public class GraphCleaningFunctions extends CleaningFunctions {
|
|||
if (Objects.nonNull(r.getInstance())) {
|
||||
|
||||
for (Instance i : r.getInstance()) {
|
||||
if (!vocs.termExists(ModelConstants.DNET_PUBLICATION_RESOURCE, i.getInstancetype().getClassid())) {
|
||||
if (r instanceof Publication) {
|
||||
i
|
||||
.setInstancetype(
|
||||
OafMapperUtils
|
||||
.qualifier(
|
||||
"0038", "Other literature type", ModelConstants.DNET_PUBLICATION_RESOURCE,
|
||||
ModelConstants.DNET_PUBLICATION_RESOURCE));
|
||||
} else if (r instanceof Dataset) {
|
||||
i
|
||||
.setInstancetype(
|
||||
OafMapperUtils
|
||||
.qualifier(
|
||||
"0039", "Other dataset type", ModelConstants.DNET_PUBLICATION_RESOURCE,
|
||||
ModelConstants.DNET_PUBLICATION_RESOURCE));
|
||||
} else if (r instanceof Software) {
|
||||
i
|
||||
.setInstancetype(
|
||||
OafMapperUtils
|
||||
.qualifier(
|
||||
"0040", "Other software type", ModelConstants.DNET_PUBLICATION_RESOURCE,
|
||||
ModelConstants.DNET_PUBLICATION_RESOURCE));
|
||||
} else if (r instanceof OtherResearchProduct) {
|
||||
i
|
||||
.setInstancetype(
|
||||
OafMapperUtils
|
||||
.qualifier(
|
||||
"0020", "Other ORP type", ModelConstants.DNET_PUBLICATION_RESOURCE,
|
||||
ModelConstants.DNET_PUBLICATION_RESOURCE));
|
||||
}
|
||||
}
|
||||
|
||||
if (Objects.nonNull(i.getPid())) {
|
||||
i.setPid(processPidCleaning(i.getPid()));
|
||||
}
|
||||
|
|
0
dhp-common/src/main/java/eu/dnetlib/dhp/schema/oaf/utils/OafMapperUtils.java
Normal file → Executable file
0
dhp-common/src/main/java/eu/dnetlib/dhp/schema/oaf/utils/OafMapperUtils.java
Normal file → Executable file
0
dhp-common/src/main/java/eu/dnetlib/dhp/utils/ISLookupClientFactory.java
Normal file → Executable file
0
dhp-common/src/main/java/eu/dnetlib/dhp/utils/ISLookupClientFactory.java
Normal file → Executable file
0
dhp-common/src/main/java/eu/dnetlib/dhp/utils/saxon/AbstractExtensionFunction.java
Normal file → Executable file
0
dhp-common/src/main/java/eu/dnetlib/dhp/utils/saxon/AbstractExtensionFunction.java
Normal file → Executable file
0
dhp-common/src/main/java/eu/dnetlib/dhp/utils/saxon/ExtractYear.java
Normal file → Executable file
0
dhp-common/src/main/java/eu/dnetlib/dhp/utils/saxon/ExtractYear.java
Normal file → Executable file
0
dhp-common/src/main/java/eu/dnetlib/dhp/utils/saxon/NormalizeDate.java
Normal file → Executable file
0
dhp-common/src/main/java/eu/dnetlib/dhp/utils/saxon/NormalizeDate.java
Normal file → Executable file
0
dhp-common/src/main/java/eu/dnetlib/dhp/utils/saxon/SaxonTransformerFactory.java
Normal file → Executable file
0
dhp-common/src/main/java/eu/dnetlib/dhp/utils/saxon/SaxonTransformerFactory.java
Normal file → Executable file
0
dhp-common/src/main/java/eu/dnetlib/scholexplorer/relation/RelInfo.java
Normal file → Executable file
0
dhp-common/src/main/java/eu/dnetlib/scholexplorer/relation/RelInfo.java
Normal file → Executable file
0
dhp-common/src/main/java/eu/dnetlib/scholexplorer/relation/RelationMapper.java
Normal file → Executable file
0
dhp-common/src/main/java/eu/dnetlib/scholexplorer/relation/RelationMapper.java
Normal file → Executable file
0
dhp-common/src/main/resources/eu/dnetlib/dhp/schema/oaf/utils/pid_blacklist.json
Normal file → Executable file
0
dhp-common/src/main/resources/eu/dnetlib/dhp/schema/oaf/utils/pid_blacklist.json
Normal file → Executable file
0
dhp-common/src/main/resources/eu/dnetlib/scholexplorer/relation/relations.json
Normal file → Executable file
0
dhp-common/src/main/resources/eu/dnetlib/scholexplorer/relation/relations.json
Normal file → Executable file
0
dhp-common/src/test/java/eu/dnetlib/dhp/application/ArgumentApplicationParserTest.java
Normal file → Executable file
0
dhp-common/src/test/java/eu/dnetlib/dhp/application/ArgumentApplicationParserTest.java
Normal file → Executable file
0
dhp-common/src/test/java/eu/dnetlib/dhp/common/SparkSessionSupportTest.java
Normal file → Executable file
0
dhp-common/src/test/java/eu/dnetlib/dhp/common/SparkSessionSupportTest.java
Normal file → Executable file
0
dhp-common/src/test/java/eu/dnetlib/dhp/common/api/ZenodoAPIClientTest.java
Normal file → Executable file
0
dhp-common/src/test/java/eu/dnetlib/dhp/common/api/ZenodoAPIClientTest.java
Normal file → Executable file
0
dhp-common/src/test/java/eu/dnetlib/dhp/oa/merge/AuthorMergerTest.java
Normal file → Executable file
0
dhp-common/src/test/java/eu/dnetlib/dhp/oa/merge/AuthorMergerTest.java
Normal file → Executable file
0
dhp-common/src/test/java/eu/dnetlib/dhp/schema/oaf/utils/OafMapperUtilsTest.java
Normal file → Executable file
0
dhp-common/src/test/java/eu/dnetlib/dhp/schema/oaf/utils/OafMapperUtilsTest.java
Normal file → Executable file
0
dhp-common/src/test/java/eu/dnetlib/scholexplorer/relation/RelationMapperTest.java
Normal file → Executable file
0
dhp-common/src/test/java/eu/dnetlib/scholexplorer/relation/RelationMapperTest.java
Normal file → Executable file
0
dhp-common/src/test/resources/eu/dnetlib/application/parameters.json
Normal file → Executable file
0
dhp-common/src/test/resources/eu/dnetlib/application/parameters.json
Normal file → Executable file
0
dhp-common/src/test/resources/eu/dnetlib/dhp/common/api/metadata.json
Normal file → Executable file
0
dhp-common/src/test/resources/eu/dnetlib/dhp/common/api/metadata.json
Normal file → Executable file
0
dhp-common/src/test/resources/eu/dnetlib/dhp/common/api/newDeposition
Normal file → Executable file
0
dhp-common/src/test/resources/eu/dnetlib/dhp/common/api/newDeposition
Normal file → Executable file
0
dhp-common/src/test/resources/eu/dnetlib/dhp/oa/merge/publications_with_authors.json
Normal file → Executable file
0
dhp-common/src/test/resources/eu/dnetlib/dhp/oa/merge/publications_with_authors.json
Normal file → Executable file
0
dhp-common/src/test/resources/eu/dnetlib/dhp/schema/oaf/utils/dataset_1.json
Normal file → Executable file
0
dhp-common/src/test/resources/eu/dnetlib/dhp/schema/oaf/utils/dataset_1.json
Normal file → Executable file
0
dhp-common/src/test/resources/eu/dnetlib/dhp/schema/oaf/utils/dataset_2.json
Normal file → Executable file
0
dhp-common/src/test/resources/eu/dnetlib/dhp/schema/oaf/utils/dataset_2.json
Normal file → Executable file
0
dhp-common/src/test/resources/eu/dnetlib/dhp/schema/oaf/utils/publication_1.json
Normal file → Executable file
0
dhp-common/src/test/resources/eu/dnetlib/dhp/schema/oaf/utils/publication_1.json
Normal file → Executable file
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue