diff --git a/src/main/java/eu/openaire/urls_worker/models/Payload.java b/src/main/java/eu/openaire/urls_worker/models/Payload.java index e2095d8..33fdabf 100644 --- a/src/main/java/eu/openaire/urls_worker/models/Payload.java +++ b/src/main/java/eu/openaire/urls_worker/models/Payload.java @@ -49,8 +49,8 @@ public class Payload { @JsonProperty("provenance") private String provenance; // "crawl:" - @JsonProperty("provenance") - private String datasourceId; // "crawl:" + @JsonProperty("datasourceId") + private String datasourceId; public Payload() {} diff --git a/src/main/java/eu/openaire/urls_worker/plugins/PublicationsRetrieverPlugin.java b/src/main/java/eu/openaire/urls_worker/plugins/PublicationsRetrieverPlugin.java index 63c9501..d7d7424 100644 --- a/src/main/java/eu/openaire/urls_worker/plugins/PublicationsRetrieverPlugin.java +++ b/src/main/java/eu/openaire/urls_worker/plugins/PublicationsRetrieverPlugin.java @@ -223,6 +223,9 @@ public class PublicationsRetrieverPlugin { String urlId = data.getUrlId(); String datasourceId = urlIdsWithDatasourceIds.get(urlId); + // Each record will have the urlID, the datasourceID and possibly one filename, which may contain a different urlID. + // The Controller will select the correct datasourceID for before adding it inside the S3-ObjectStore filename. + Payload payload = new Payload(urlId, data.getSourceUrl(), docOrDatasetUrl, timestamp, mimeType, size, hash, fileLocation, "crawl:PublicationsRetriever", datasourceId); // TODO - If support is added for other doc-formats other than "pdf", then make sure the "mime_type" is correctly specified.