forked from D-Net/dnet-hadoop
91 lines
4.0 KiB
Java
91 lines
4.0 KiB
Java
|
package eu.dnetlib.dhp.oa.dedup;
|
||
|
|
||
|
import com.google.common.collect.Lists;
|
||
|
import eu.dnetlib.dhp.schema.common.EntityType;
|
||
|
import eu.dnetlib.dhp.schema.common.ModelSupport;
|
||
|
import eu.dnetlib.dhp.schema.oaf.Field;
|
||
|
import eu.dnetlib.dhp.schema.oaf.OafEntity;
|
||
|
import eu.dnetlib.dhp.schema.oaf.Result;
|
||
|
import eu.dnetlib.dhp.schema.oaf.StructuredProperty;
|
||
|
import org.apache.commons.lang.NullArgumentException;
|
||
|
import org.apache.commons.lang.StringUtils;
|
||
|
|
||
|
import java.io.Serializable;
|
||
|
import java.text.ParseException;
|
||
|
import java.text.SimpleDateFormat;
|
||
|
import java.util.*;
|
||
|
|
||
|
public class IdGenerator implements Serializable {
|
||
|
|
||
|
private static SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd");
|
||
|
public static String CROSSREF_ID = "10|openaire____::081b82f96300b6a6e3d282bad31cb6e2";
|
||
|
public static String DATACITE_ID = "10|openaire____::9e3be59865b2c1c335d32dae2fe7b254";
|
||
|
|
||
|
//pick the best pid from the list (consider date and pidtype)
|
||
|
public static String generate(List<Identifier> pids, String defaultID) {
|
||
|
if (pids == null || pids.size() == 0)
|
||
|
return defaultID;
|
||
|
|
||
|
Optional<Identifier> bp = pids.stream()
|
||
|
.max(Identifier::compareTo);
|
||
|
|
||
|
if (bp.get().isUseOriginal() || bp.get().getPid().getValue() == null) {
|
||
|
return bp.get().getOriginalID().split("\\|")[0] + "|dedup_wf_001::" + DedupUtility.md5(bp.get().getOriginalID());
|
||
|
} else {
|
||
|
return bp.get().getOriginalID().split("\\|")[0] + "|" + createPrefix(bp.get().getPid().getQualifier().getClassid()) + "::" + DedupUtility.md5(bp.get().getPid().getValue());
|
||
|
}
|
||
|
|
||
|
}
|
||
|
|
||
|
//pick the best pid from the entity. Returns a list (length 1) to save time in the call
|
||
|
public static <T extends OafEntity> List<Identifier> bestPidtoIdentifier(T entity) {
|
||
|
|
||
|
if (entity.getPid() == null || entity.getPid().size() == 0)
|
||
|
return Lists.newArrayList(new Identifier(new StructuredProperty(), new Date(), PidType.original, entity.getCollectedfrom(), EntityType.fromClass(entity.getClass()), entity.getId()));
|
||
|
|
||
|
Optional<StructuredProperty> bp = entity.getPid().stream()
|
||
|
.filter(pid -> PidType.classidValueOf(pid.getQualifier().getClassid()) != PidType.undefined)
|
||
|
.max(Comparator.comparing(pid -> PidType.classidValueOf(pid.getQualifier().getClassid())));
|
||
|
|
||
|
return bp.map(structuredProperty ->
|
||
|
Lists.newArrayList(new Identifier(structuredProperty, extractDate(entity, sdf), PidType.classidValueOf(structuredProperty.getQualifier().getClassid()), entity.getCollectedfrom(), EntityType.fromClass(entity.getClass()), entity.getId()))
|
||
|
).orElseGet(() -> Lists.newArrayList(new Identifier(new StructuredProperty(), new Date(), PidType.original, entity.getCollectedfrom(), EntityType.fromClass(entity.getClass()), entity.getId())));
|
||
|
|
||
|
}
|
||
|
|
||
|
//create the prefix (length = 12): dedup_+ pidType
|
||
|
public static String createPrefix(String pidType) {
|
||
|
|
||
|
StringBuilder prefix = new StringBuilder("dedup_" + pidType);
|
||
|
|
||
|
while (prefix.length() < 12) {
|
||
|
prefix.append("_");
|
||
|
}
|
||
|
return prefix.toString().substring(0, 12);
|
||
|
|
||
|
}
|
||
|
|
||
|
//extracts the date from the record. If the date is not available or is not wellformed, it returns a base date: 00-01-01
|
||
|
public static <T extends OafEntity> Date extractDate(T duplicate, SimpleDateFormat sdf){
|
||
|
|
||
|
String date = "2000-01-01";
|
||
|
if (ModelSupport.isSubClass(duplicate, Result.class)) {
|
||
|
Result result = (Result) duplicate;
|
||
|
if (isWellformed(result.getDateofacceptance())){
|
||
|
date = result.getDateofacceptance().getValue();
|
||
|
}
|
||
|
}
|
||
|
|
||
|
try {
|
||
|
return sdf.parse(date);
|
||
|
} catch (ParseException e) {
|
||
|
return new Date();
|
||
|
}
|
||
|
|
||
|
}
|
||
|
|
||
|
public static boolean isWellformed(Field<String> date) {
|
||
|
return date != null && StringUtils.isNotBlank(date.getValue()) && date.getValue().matches(DatePicker.DATE_PATTERN) && DatePicker.inRange(date.getValue());
|
||
|
}
|
||
|
}
|