Compare commits
No commits in common. "master" and "2.0" have entirely different histories.
|
@ -4,4 +4,4 @@ COPY build/libs/*-SNAPSHOT.jar urls_controller.jar
|
|||
|
||||
EXPOSE 1880
|
||||
|
||||
ENTRYPOINT ["java", "-Xms1024m", "-Xmx4096m", "-jar", "/urls_controller.jar", "--spring.config.location=file:///mnt/config/application.yml"]
|
||||
ENTRYPOINT ["java","-jar","/urls_controller.jar", "--spring.config.location=file:///mnt/config/application.yml"]
|
||||
|
|
201
LICENSE
201
LICENSE
|
@ -1,201 +0,0 @@
|
|||
Apache License
|
||||
Version 2.0, January 2004
|
||||
http://www.apache.org/licenses/
|
||||
|
||||
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
||||
|
||||
1. Definitions.
|
||||
|
||||
"License" shall mean the terms and conditions for use, reproduction,
|
||||
and distribution as defined by Sections 1 through 9 of this document.
|
||||
|
||||
"Licensor" shall mean the copyright owner or entity authorized by
|
||||
the copyright owner that is granting the License.
|
||||
|
||||
"Legal Entity" shall mean the union of the acting entity and all
|
||||
other entities that control, are controlled by, or are under common
|
||||
control with that entity. For the purposes of this definition,
|
||||
"control" means (i) the power, direct or indirect, to cause the
|
||||
direction or management of such entity, whether by contract or
|
||||
otherwise, or (ii) ownership of fifty percent (50%) or more of the
|
||||
outstanding shares, or (iii) beneficial ownership of such entity.
|
||||
|
||||
"You" (or "Your") shall mean an individual or Legal Entity
|
||||
exercising permissions granted by this License.
|
||||
|
||||
"Source" form shall mean the preferred form for making modifications,
|
||||
including but not limited to software source code, documentation
|
||||
source, and configuration files.
|
||||
|
||||
"Object" form shall mean any form resulting from mechanical
|
||||
transformation or translation of a Source form, including but
|
||||
not limited to compiled object code, generated documentation,
|
||||
and conversions to other media types.
|
||||
|
||||
"Work" shall mean the work of authorship, whether in Source or
|
||||
Object form, made available under the License, as indicated by a
|
||||
copyright notice that is included in or attached to the work
|
||||
(an example is provided in the Appendix below).
|
||||
|
||||
"Derivative Works" shall mean any work, whether in Source or Object
|
||||
form, that is based on (or derived from) the Work and for which the
|
||||
editorial revisions, annotations, elaborations, or other modifications
|
||||
represent, as a whole, an original work of authorship. For the purposes
|
||||
of this License, Derivative Works shall not include works that remain
|
||||
separable from, or merely link (or bind by name) to the interfaces of,
|
||||
the Work and Derivative Works thereof.
|
||||
|
||||
"Contribution" shall mean any work of authorship, including
|
||||
the original version of the Work and any modifications or additions
|
||||
to that Work or Derivative Works thereof, that is intentionally
|
||||
submitted to Licensor for inclusion in the Work by the copyright owner
|
||||
or by an individual or Legal Entity authorized to submit on behalf of
|
||||
the copyright owner. For the purposes of this definition, "submitted"
|
||||
means any form of electronic, verbal, or written communication sent
|
||||
to the Licensor or its representatives, including but not limited to
|
||||
communication on electronic mailing lists, source code control systems,
|
||||
and issue tracking systems that are managed by, or on behalf of, the
|
||||
Licensor for the purpose of discussing and improving the Work, but
|
||||
excluding communication that is conspicuously marked or otherwise
|
||||
designated in writing by the copyright owner as "Not a Contribution."
|
||||
|
||||
"Contributor" shall mean Licensor and any individual or Legal Entity
|
||||
on behalf of whom a Contribution has been received by Licensor and
|
||||
subsequently incorporated within the Work.
|
||||
|
||||
2. Grant of Copyright License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
copyright license to reproduce, prepare Derivative Works of,
|
||||
publicly display, publicly perform, sublicense, and distribute the
|
||||
Work and such Derivative Works in Source or Object form.
|
||||
|
||||
3. Grant of Patent License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
(except as stated in this section) patent license to make, have made,
|
||||
use, offer to sell, sell, import, and otherwise transfer the Work,
|
||||
where such license applies only to those patent claims licensable
|
||||
by such Contributor that are necessarily infringed by their
|
||||
Contribution(s) alone or by combination of their Contribution(s)
|
||||
with the Work to which such Contribution(s) was submitted. If You
|
||||
institute patent litigation against any entity (including a
|
||||
cross-claim or counterclaim in a lawsuit) alleging that the Work
|
||||
or a Contribution incorporated within the Work constitutes direct
|
||||
or contributory patent infringement, then any patent licenses
|
||||
granted to You under this License for that Work shall terminate
|
||||
as of the date such litigation is filed.
|
||||
|
||||
4. Redistribution. You may reproduce and distribute copies of the
|
||||
Work or Derivative Works thereof in any medium, with or without
|
||||
modifications, and in Source or Object form, provided that You
|
||||
meet the following conditions:
|
||||
|
||||
(a) You must give any other recipients of the Work or
|
||||
Derivative Works a copy of this License; and
|
||||
|
||||
(b) You must cause any modified files to carry prominent notices
|
||||
stating that You changed the files; and
|
||||
|
||||
(c) You must retain, in the Source form of any Derivative Works
|
||||
that You distribute, all copyright, patent, trademark, and
|
||||
attribution notices from the Source form of the Work,
|
||||
excluding those notices that do not pertain to any part of
|
||||
the Derivative Works; and
|
||||
|
||||
(d) If the Work includes a "NOTICE" text file as part of its
|
||||
distribution, then any Derivative Works that You distribute must
|
||||
include a readable copy of the attribution notices contained
|
||||
within such NOTICE file, excluding those notices that do not
|
||||
pertain to any part of the Derivative Works, in at least one
|
||||
of the following places: within a NOTICE text file distributed
|
||||
as part of the Derivative Works; within the Source form or
|
||||
documentation, if provided along with the Derivative Works; or,
|
||||
within a display generated by the Derivative Works, if and
|
||||
wherever such third-party notices normally appear. The contents
|
||||
of the NOTICE file are for informational purposes only and
|
||||
do not modify the License. You may add Your own attribution
|
||||
notices within Derivative Works that You distribute, alongside
|
||||
or as an addendum to the NOTICE text from the Work, provided
|
||||
that such additional attribution notices cannot be construed
|
||||
as modifying the License.
|
||||
|
||||
You may add Your own copyright statement to Your modifications and
|
||||
may provide additional or different license terms and conditions
|
||||
for use, reproduction, or distribution of Your modifications, or
|
||||
for any such Derivative Works as a whole, provided Your use,
|
||||
reproduction, and distribution of the Work otherwise complies with
|
||||
the conditions stated in this License.
|
||||
|
||||
5. Submission of Contributions. Unless You explicitly state otherwise,
|
||||
any Contribution intentionally submitted for inclusion in the Work
|
||||
by You to the Licensor shall be under the terms and conditions of
|
||||
this License, without any additional terms or conditions.
|
||||
Notwithstanding the above, nothing herein shall supersede or modify
|
||||
the terms of any separate license agreement you may have executed
|
||||
with Licensor regarding such Contributions.
|
||||
|
||||
6. Trademarks. This License does not grant permission to use the trade
|
||||
names, trademarks, service marks, or product names of the Licensor,
|
||||
except as required for reasonable and customary use in describing the
|
||||
origin of the Work and reproducing the content of the NOTICE file.
|
||||
|
||||
7. Disclaimer of Warranty. Unless required by applicable law or
|
||||
agreed to in writing, Licensor provides the Work (and each
|
||||
Contributor provides its Contributions) on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
implied, including, without limitation, any warranties or conditions
|
||||
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
|
||||
PARTICULAR PURPOSE. You are solely responsible for determining the
|
||||
appropriateness of using or redistributing the Work and assume any
|
||||
risks associated with Your exercise of permissions under this License.
|
||||
|
||||
8. Limitation of Liability. In no event and under no legal theory,
|
||||
whether in tort (including negligence), contract, or otherwise,
|
||||
unless required by applicable law (such as deliberate and grossly
|
||||
negligent acts) or agreed to in writing, shall any Contributor be
|
||||
liable to You for damages, including any direct, indirect, special,
|
||||
incidental, or consequential damages of any character arising as a
|
||||
result of this License or out of the use or inability to use the
|
||||
Work (including but not limited to damages for loss of goodwill,
|
||||
work stoppage, computer failure or malfunction, or any and all
|
||||
other commercial damages or losses), even if such Contributor
|
||||
has been advised of the possibility of such damages.
|
||||
|
||||
9. Accepting Warranty or Additional Liability. While redistributing
|
||||
the Work or Derivative Works thereof, You may choose to offer,
|
||||
and charge a fee for, acceptance of support, warranty, indemnity,
|
||||
or other liability obligations and/or rights consistent with this
|
||||
License. However, in accepting such obligations, You may act only
|
||||
on Your own behalf and on Your sole responsibility, not on behalf
|
||||
of any other Contributor, and only if You agree to indemnify,
|
||||
defend, and hold each Contributor harmless for any liability
|
||||
incurred by, or claims asserted against, such Contributor by reason
|
||||
of your accepting any such warranty or additional liability.
|
||||
|
||||
END OF TERMS AND CONDITIONS
|
||||
|
||||
APPENDIX: How to apply the Apache License to your work.
|
||||
|
||||
To apply the Apache License to your work, attach the following
|
||||
boilerplate notice, with the fields enclosed by brackets "[]"
|
||||
replaced with your own identifying information. (Don't include
|
||||
the brackets!) The text should be enclosed in the appropriate
|
||||
comment syntax for the file format. We also recommend that a
|
||||
file or class name and description of purpose be included on the
|
||||
same "printed page" as the copyright notice for easier
|
||||
identification within third-party archives.
|
||||
|
||||
Copyright 2021-2024 OpenAIRE AMKE
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
79
README.md
79
README.md
|
@ -1,82 +1,39 @@
|
|||
# UrlsController
|
||||
## [![Jenkins build status](https://jenkins-dnet.d4science.org/buildStatus/icon?job=UrlsController)](https://jenkins-dnet.d4science.org/job/UrlsController/)
|
||||
|
||||
The Controller's Application receives requests coming from the [**Workers**](https://code-repo.d4science.org/lsmyrnaios/UrlsWorker) (deployed on the cloud), constructs an assignments-list with data received from a database and returns the list to the workers.<br>
|
||||
The Controller's Application receives requests coming from the [Workers](https://code-repo.d4science.org/lsmyrnaios/UrlsWorker) , constructs an assignments-list with data received from a database and returns the list to the workers.<br>
|
||||
Then, it receives the "WorkerReports", it requests the full-texts from the workers, in batches, and uploads them on the S3-Object-Store. Finally, it writes the related reports, along with the updated file-locations into the database.<br>
|
||||
<br>
|
||||
It can also process **Bulk-Import** requests, from compatible data sources, in which case it receives the full-text files immediately, without offloading crawling jobs to Workers.<br>
|
||||
<br>
|
||||
For managing and generating data, we use [**Impala**](https://impala.apache.org/) JDBC and WebHDFS.<br>
|
||||
For interacting with the database we use [Impala](https://impala.apache.org/).<br>
|
||||
<br>
|
||||
|
||||
|
||||
**To install and run the application**:
|
||||
- Run ```git clone``` and then ```cd UrlsController```.
|
||||
- Set the preferable values inside the [__application.yml__](https://code-repo.d4science.org/lsmyrnaios/UrlsController/src/branch/master/src/main/resources/application.yml) file. Specifically, for tests, set the ***services.pdfaggregation.controller.isTestEnvironment*** property to "**true**" and make sure the "***services.pdfaggregation.controller.db.testDatabaseName***" property is set to a test-database.
|
||||
- Execute the ```installAndRun.sh``` script which builds and runs the app.<br>
|
||||
If you want to just run the app, then run the script with the argument "1": ```./installAndRun.sh 1```.<br>
|
||||
If you want to build and run the app on a **Docker Container**, then run the script with the argument "0" followed by the argument "1": ```./installAndRun.sh 0 1```.<br>
|
||||
Additionally, if you want to test/visualize the exposed metrics on Prometheus and Grafana, you can deploy their instances on docker containers,
|
||||
by enabling the "runPrometheusAndGrafanaContainers" switch, inside the "./installAndRun.sh" script.<br>
|
||||
<br>
|
||||
|
||||
|
||||
**BulkImport API**:
|
||||
- "**bulkImportFullTexts**" endpoint: **http://\<IP\>:\<PORT\>/api/bulkImportFullTexts?provenance=\<provenance\>&bulkImportDir=\<bulkImportDir\>&shouldDeleteFilesOnFinish={true|false}** <br>
|
||||
This endpoint loads the right configuration with the help of the "provenance" parameter, delegates the processing to a background thread and immediately returns a message with useful information, including the "reportFileID", which can be used at any moment to request a report about the progress of the bulk-import procedure.<br>
|
||||
The processing job starts running after 30-60 minutes and processes the full-texts files inside the given directory, in the following way: it generates the OpenAIRE-IDs, uploads the files to the S3 Object Store, generates and stores the "payload" records in the database. If it is requested by the user, it removes the successfully imported full-texts from the directory.
|
||||
- "**getBulkImportReport**" endpoint: **http://\<IP\>:\<PORT\>/api/getBulkImportReport?id=\<reportFileID\>** <br>
|
||||
This endpoint returns the bulkImport report, which corresponds to the given reportFileID, in JSON format.
|
||||
<br>
|
||||
|
||||
**How to add a bulk-import datasource**:
|
||||
- Open the [__application.yml__](https://code-repo.d4science.org/lsmyrnaios/UrlsController/src/branch/master/src/main/resources/application.yml) file.
|
||||
- Add a new object under the "bulk-import.bulkImportSources" property.
|
||||
- Read the comments written in the end of the "bulk-import" property and make sure all requirements are met.
|
||||
- "**bulkImportFullTexts**" endpoint: **http://\<IP\>:<PORT>/api/bulkImportFullTexts?provenance=\<provenance\>&bulkImportDir=\<bulkImportDir\>&shouldDeleteFilesOnFinish={true|false}** <br>
|
||||
This endpoint loads the right configuration with the help of the "provenance" param and then processes the full-texts files inside the given directory, in the following way: it generates the openAireIDs, uploads the files to the S3 ObjectStore, generates and stores the the "payload" records in the database. If it is specified, it removes the successfully imported full-texts from the directory.
|
||||
- "**getBulkImportReport**" endpoint: **http://\<IP\>:<PORT>/api/getBulkImportReport?id=\<bulkImportReportId\>** <br>
|
||||
This endpoint returns the bulkImport report, which corresponds to the given ID, in JSON format.
|
||||
<br>
|
||||
<br>
|
||||
|
||||
**Statistics API**:
|
||||
- "**getNumberOfAllPayloads**" endpoint: **http://\<IP\>:\<PORT\>/api/stats/getNumberOfAllPayloads** <br>
|
||||
- "**getNumberOfAllPayloads**" endpoint: **http://\<IP\>:<PORT>/api/stats/getNumberOfAllPayloads** <br>
|
||||
This endpoint returns the total number of payloads existing in the database, independently of the way they were aggregated. This includes the payloads created by other pieces of software, before the PDF-Aggregation-Service was created.
|
||||
- "**getNumberOfPayloadsAggregatedByServiceThroughCrawling**" endpoint: **http://\<IP\>:\<PORT\>/api/stats/getNumberOfPayloadsAggregatedByServiceThroughCrawling** <br>
|
||||
This endpoint returns the number of payloads aggregated by the PDF-Aggregated-Service itself, through crawling.
|
||||
- "**getNumberOfPayloadsAggregatedByServiceThroughBulkImport**" endpoint: **http://\<IP\>:\<PORT\>/api/stats/getNumberOfPayloadsAggregatedByServiceThroughBulkImport** <br>
|
||||
This endpoint returns the number of payloads aggregated by the PDF-Aggregated-Service itself, through bulk-import procedures, from compatible datasources.
|
||||
- "**getNumberOfPayloadsAggregatedByService**" endpoint: **http://\<IP\>:\<PORT\>/api/stats/getNumberOfPayloadsAggregatedByService** <br>
|
||||
This endpoint returns the number of payloads aggregated by the PDF-Aggregated-Service itself, both through crawling and bulk-import procedures.
|
||||
- "**getNumberOfLegacyPayloads**" endpoint: **http://\<IP\>:\<PORT\>/api/stats/getNumberOfLegacyPayloads** <br>
|
||||
This endpoint returns the number of payloads which were aggregated by methods other than the PDF Aggregation Service.
|
||||
- "**getNumberOfPayloadsForDatasource**" endpoint: **http://\<IP\>:\<PORT\>/api/stats/getNumberOfPayloadsForDatasource?datasourceId=\<givenDatasourceId\>** <br>
|
||||
- "**getNumberOfPayloadsAggregatedByService**" endpoint: **http://\<IP\>:<PORT>/api/stats/getNumberOfPayloadsAggregatedByService** <br>
|
||||
This endpoint returns the number of payloads aggregated by the PDF-Aggregated-Service itself. It excludes the payloads aggregated by other methods, by applying a Date-filter for the records created in 2021 or later.
|
||||
- "**getNumberOfPayloadsForDatasource**" endpoint: **http://\<IP\>:<PORT>/api/stats/getNumberOfPayloadsForDatasource?datasourceId=\<givenDatasourceId\>** <br>
|
||||
This endpoint returns the number of payloads which belong to the datasource specified by the given datasourceID.
|
||||
- "**getNumberOfRecordsInspectedByServiceThroughCrawling**" endpoint: **http://\<IP\>:\<PORT\>/api/stats/getNumberOfRecordsInspectedByServiceThroughCrawling** <br>
|
||||
This endpoint returns the number of records inspected by the PDF-Aggregation-Service through crawling.
|
||||
- "**getNumberOfRecordsInspected**" endpoint: **http://\<IP\>:<PORT>/api/stats/getNumberOfRecordsInspected** <br>
|
||||
This endpoint returns the number of records inspected by the PDF-Aggregation-Service.
|
||||
<br>
|
||||
<br>
|
||||
|
||||
**Shutdown Service API**:
|
||||
- "**shutdownService**" endpoint: **http://localhost:\<PORT\>/api/shutdownService** <br>
|
||||
This endpoint sends "shutdownWorker" requests to all the Workers which are actively participating in the Service. The Workers will shut down after finishing their work-in-progress and all full-texts have been either transferred to the Controller or deleted, in case an error has appeared.<br>
|
||||
Once the Workers are about to shut down, they send a "shutdownReport" to the Controller. A scheduling task runs in the Controller, every 2 hours, and if the user has specified that the Controller must shut down and all the Workers participating in the Service have shutdown, then it gently shuts down the Controller.
|
||||
- "**cancelShutdownService**" endpoint: **http://localhost:\<PORT\>/api/cancelShutdownService** <br>
|
||||
This endpoint specifies that the Controller will not shut down, and sends "cancelShutdownWorker" requests to all the Workers which are actively participating in the Service (have not shut down yet), so that they can continue to request assignments.
|
||||
<br>
|
||||
|
||||
Note: The Shutdown Service API is accessible by the Controller's host machine.
|
||||
<br>
|
||||
<br>
|
||||
|
||||
|
||||
**Prometheus Metrics**:
|
||||
- "**numOfAllPayloads**"
|
||||
- "**numOfPayloadsAggregatedByServiceThroughCrawling**"
|
||||
- "**numOfPayloadsAggregatedByServiceThroughBulkImport**"
|
||||
- "**numOfPayloadsAggregatedByService**"
|
||||
- "**numOfLegacyPayloads**"
|
||||
- "**numOfRecordsInspectedByServiceThroughCrawling**"
|
||||
- "**getAssignments_time_seconds_max**": Time taken to return the assignments.
|
||||
- "**addWorkerReport_time_seconds**": Time taken to add the WorkerReport.
|
||||
<br>
|
||||
**To install and run the application**:
|
||||
- Run ```git clone``` and then ```cd UrlsController```.
|
||||
- Set the preferable values inside the [__application.yml__](https://code-repo.d4science.org/lsmyrnaios/UrlsController/src/branch/master/src/main/resources/application.yml) file.
|
||||
- Execute the ```installAndRun.sh``` script which builds and runs the app.<br>
|
||||
If you want to just run the app, then run the script with the argument "1": ```./installAndRun.sh 1```.<br>
|
||||
If you want to build and run the app on a **Docker Container**, then run the script with the argument "0" followed by the argument "1": ```./installAndRun.sh 0 1```.<br>
|
||||
<br>
|
||||
|
||||
Implementation notes:
|
||||
|
|
52
build.gradle
52
build.gradle
|
@ -1,25 +1,23 @@
|
|||
plugins {
|
||||
id 'org.springframework.boot' version '2.7.18'
|
||||
id 'io.spring.dependency-management' version '1.1.5'
|
||||
id 'org.springframework.boot' version '2.7.12'
|
||||
id 'io.spring.dependency-management' version '1.1.0'
|
||||
id 'java'
|
||||
}
|
||||
|
||||
java {
|
||||
group = 'eu.openaire.urls_controller'
|
||||
version = '2.7.0-SNAPSHOT'
|
||||
sourceCompatibility = JavaVersion.VERSION_1_8
|
||||
}
|
||||
group = 'eu.openaire.urls_controller'
|
||||
version = '1.0.0-SNAPSHOT'
|
||||
sourceCompatibility = '1.8'
|
||||
|
||||
repositories {
|
||||
mavenCentral()
|
||||
maven {
|
||||
name "madgik"
|
||||
url "https://repo.madgik.di.uoa.gr/content/repositories/thirdparty/"
|
||||
name "omtd"
|
||||
url "https://repo.openminted.eu/content/repositories/releases/"
|
||||
}
|
||||
maven {
|
||||
name "pentaho-repo"
|
||||
url "https://public.nexus.pentaho.org/content/groups/omni/"
|
||||
}
|
||||
}
|
||||
|
||||
ext {
|
||||
hadoopVersion = '3.4.0'
|
||||
}
|
||||
|
||||
dependencies {
|
||||
|
@ -42,19 +40,18 @@ dependencies {
|
|||
// Enable the validation annotations.
|
||||
//implementation group: 'jakarta.validation', name: 'jakarta.validation-api', version: '3.0.2'
|
||||
|
||||
implementation "org.projectlombok:lombok:1.18.28"
|
||||
|
||||
// https://mvnrepository.com/artifact/com.google.guava/guava
|
||||
implementation group: 'com.google.guava', name: 'guava', version: '33.2.0-jre'
|
||||
implementation group: 'com.google.guava', name: 'guava', version: '32.0.0-jre'
|
||||
|
||||
// https://mvnrepository.com/artifact/org.apache.commons/commons-lang3
|
||||
implementation group: 'org.apache.commons', name: 'commons-lang3', version: '3.14.0'
|
||||
implementation group: 'org.apache.commons', name: 'commons-lang3', version: '3.12.0'
|
||||
|
||||
// https://mvnrepository.com/artifact/org.apache.commons/commons-compress
|
||||
implementation("org.apache.commons:commons-compress:1.26.2") {
|
||||
exclude group: 'com.github.luben', module: 'zstd-jni'
|
||||
}
|
||||
implementation 'com.github.luben:zstd-jni:1.5.6-3' // Even though this is part of the above dependency, the Apache commons rarely updates it, while the zstd team makes improvements very often.
|
||||
implementation 'org.apache.commons:commons-compress:1.23.0'
|
||||
|
||||
implementation 'io.minio:minio:8.5.10'
|
||||
implementation 'io.minio:minio:8.5.3'
|
||||
|
||||
// https://mvnrepository.com/artifact/com.cloudera.impala/jdbc
|
||||
implementation("com.cloudera.impala:jdbc:2.5.31") {
|
||||
|
@ -77,10 +74,10 @@ dependencies {
|
|||
}
|
||||
|
||||
// https://mvnrepository.com/artifact/org.apache.parquet/parquet-avro
|
||||
implementation('org.apache.parquet:parquet-avro:1.14.0')
|
||||
implementation('org.apache.parquet:parquet-avro:1.13.1')
|
||||
|
||||
// https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-common
|
||||
implementation("org.apache.hadoop:hadoop-common:$hadoopVersion") {
|
||||
implementation('org.apache.hadoop:hadoop-common:3.3.5') {
|
||||
exclude group: 'org.apache.parquet', module: 'parquet-avro'
|
||||
exclude group: 'org.apache.avro', module: 'avro'
|
||||
exclude group: 'org.slf4j', module: 'slf4j-api'
|
||||
|
@ -96,7 +93,7 @@ dependencies {
|
|||
}
|
||||
|
||||
// https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-mapreduce-client-core
|
||||
implementation("org.apache.hadoop:hadoop-mapreduce-client-core:$hadoopVersion") {
|
||||
implementation('org.apache.hadoop:hadoop-mapreduce-client-core:3.3.5') {
|
||||
exclude group: 'org.apache.parquet', module: 'parquet-avro'
|
||||
exclude group: 'org.apache.avro', module: 'avro'
|
||||
exclude group: 'org.slf4j', module: 'slf4j-api'
|
||||
|
@ -110,16 +107,17 @@ dependencies {
|
|||
|
||||
// Add back some updated version of the needed dependencies.
|
||||
implementation 'org.apache.thrift:libthrift:0.17.0' // Newer versions (>=0.18.X) are not compatible with JAVA 8.
|
||||
implementation 'com.fasterxml.woodstox:woodstox-core:6.6.2'
|
||||
implementation 'com.fasterxml.woodstox:woodstox-core:6.5.1'
|
||||
|
||||
// https://mvnrepository.com/artifact/org.json/json
|
||||
implementation 'org.json:json:20240303' // This is used only in "ParquetFileUtils.createRemoteParquetDirectories()". TODO - Replace it with "gson".
|
||||
implementation 'org.json:json:20230227'
|
||||
|
||||
// https://mvnrepository.com/artifact/com.google.code.gson/gson
|
||||
implementation 'com.google.code.gson:gson:2.11.0'
|
||||
implementation 'com.google.code.gson:gson:2.10.1'
|
||||
|
||||
|
||||
// https://mvnrepository.com/artifact/io.micrometer/micrometer-registry-prometheus
|
||||
runtimeOnly 'io.micrometer:micrometer-registry-prometheus:1.13.0'
|
||||
runtimeOnly 'io.micrometer:micrometer-registry-prometheus:1.11.0'
|
||||
|
||||
testImplementation 'org.springframework.security:spring-security-test'
|
||||
testImplementation "org.springframework.boot:spring-boot-starter-test"
|
||||
|
|
|
@ -4,7 +4,6 @@ services:
|
|||
|
||||
urls_controller:
|
||||
image: 'pdf_aggregation_service/urls_controller:latest'
|
||||
container_name: urls_controller
|
||||
ports:
|
||||
- '1880:1880'
|
||||
volumes:
|
||||
|
|
Binary file not shown.
|
@ -1,7 +1,6 @@
|
|||
distributionBase=GRADLE_USER_HOME
|
||||
distributionPath=wrapper/dists
|
||||
distributionUrl=https\://services.gradle.org/distributions/gradle-8.7-bin.zip
|
||||
distributionUrl=https\://services.gradle.org/distributions/gradle-8.1.1-bin.zip
|
||||
networkTimeout=10000
|
||||
validateDistributionUrl=true
|
||||
zipStoreBase=GRADLE_USER_HOME
|
||||
zipStorePath=wrapper/dists
|
||||
|
|
|
@ -1,249 +0,0 @@
|
|||
#!/bin/sh
|
||||
|
||||
#
|
||||
# Copyright © 2015-2021 the original authors.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# https://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
##############################################################################
|
||||
#
|
||||
# Gradle start up script for POSIX generated by Gradle.
|
||||
#
|
||||
# Important for running:
|
||||
#
|
||||
# (1) You need a POSIX-compliant shell to run this script. If your /bin/sh is
|
||||
# noncompliant, but you have some other compliant shell such as ksh or
|
||||
# bash, then to run this script, type that shell name before the whole
|
||||
# command line, like:
|
||||
#
|
||||
# ksh Gradle
|
||||
#
|
||||
# Busybox and similar reduced shells will NOT work, because this script
|
||||
# requires all of these POSIX shell features:
|
||||
# * functions;
|
||||
# * expansions «$var», «${var}», «${var:-default}», «${var+SET}»,
|
||||
# «${var#prefix}», «${var%suffix}», and «$( cmd )»;
|
||||
# * compound commands having a testable exit status, especially «case»;
|
||||
# * various built-in commands including «command», «set», and «ulimit».
|
||||
#
|
||||
# Important for patching:
|
||||
#
|
||||
# (2) This script targets any POSIX shell, so it avoids extensions provided
|
||||
# by Bash, Ksh, etc; in particular arrays are avoided.
|
||||
#
|
||||
# The "traditional" practice of packing multiple parameters into a
|
||||
# space-separated string is a well documented source of bugs and security
|
||||
# problems, so this is (mostly) avoided, by progressively accumulating
|
||||
# options in "$@", and eventually passing that to Java.
|
||||
#
|
||||
# Where the inherited environment variables (DEFAULT_JVM_OPTS, JAVA_OPTS,
|
||||
# and GRADLE_OPTS) rely on word-splitting, this is performed explicitly;
|
||||
# see the in-line comments for details.
|
||||
#
|
||||
# There are tweaks for specific operating systems such as AIX, CygWin,
|
||||
# Darwin, MinGW, and NonStop.
|
||||
#
|
||||
# (3) This script is generated from the Groovy template
|
||||
# https://github.com/gradle/gradle/blob/HEAD/subprojects/plugins/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt
|
||||
# within the Gradle project.
|
||||
#
|
||||
# You can find Gradle at https://github.com/gradle/gradle/.
|
||||
#
|
||||
##############################################################################
|
||||
|
||||
# Attempt to set APP_HOME
|
||||
|
||||
# Resolve links: $0 may be a link
|
||||
app_path=$0
|
||||
|
||||
# Need this for daisy-chained symlinks.
|
||||
while
|
||||
APP_HOME=${app_path%"${app_path##*/}"} # leaves a trailing /; empty if no leading path
|
||||
[ -h "$app_path" ]
|
||||
do
|
||||
ls=$( ls -ld "$app_path" )
|
||||
link=${ls#*' -> '}
|
||||
case $link in #(
|
||||
/*) app_path=$link ;; #(
|
||||
*) app_path=$APP_HOME$link ;;
|
||||
esac
|
||||
done
|
||||
|
||||
# This is normally unused
|
||||
# shellcheck disable=SC2034
|
||||
APP_BASE_NAME=${0##*/}
|
||||
# Discard cd standard output in case $CDPATH is set (https://github.com/gradle/gradle/issues/25036)
|
||||
APP_HOME=$( cd "${APP_HOME:-./}" > /dev/null && pwd -P ) || exit
|
||||
|
||||
# Use the maximum available, or set MAX_FD != -1 to use that value.
|
||||
MAX_FD=maximum
|
||||
|
||||
warn () {
|
||||
echo "$*"
|
||||
} >&2
|
||||
|
||||
die () {
|
||||
echo
|
||||
echo "$*"
|
||||
echo
|
||||
exit 1
|
||||
} >&2
|
||||
|
||||
# OS specific support (must be 'true' or 'false').
|
||||
cygwin=false
|
||||
msys=false
|
||||
darwin=false
|
||||
nonstop=false
|
||||
case "$( uname )" in #(
|
||||
CYGWIN* ) cygwin=true ;; #(
|
||||
Darwin* ) darwin=true ;; #(
|
||||
MSYS* | MINGW* ) msys=true ;; #(
|
||||
NONSTOP* ) nonstop=true ;;
|
||||
esac
|
||||
|
||||
CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar
|
||||
|
||||
|
||||
# Determine the Java command to use to start the JVM.
|
||||
if [ -n "$JAVA_HOME" ] ; then
|
||||
if [ -x "$JAVA_HOME/jre/sh/java" ] ; then
|
||||
# IBM's JDK on AIX uses strange locations for the executables
|
||||
JAVACMD=$JAVA_HOME/jre/sh/java
|
||||
else
|
||||
JAVACMD=$JAVA_HOME/bin/java
|
||||
fi
|
||||
if [ ! -x "$JAVACMD" ] ; then
|
||||
die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME
|
||||
|
||||
Please set the JAVA_HOME variable in your environment to match the
|
||||
location of your Java installation."
|
||||
fi
|
||||
else
|
||||
JAVACMD=java
|
||||
if ! command -v java >/dev/null 2>&1
|
||||
then
|
||||
die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
|
||||
|
||||
Please set the JAVA_HOME variable in your environment to match the
|
||||
location of your Java installation."
|
||||
fi
|
||||
fi
|
||||
|
||||
# Increase the maximum file descriptors if we can.
|
||||
if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then
|
||||
case $MAX_FD in #(
|
||||
max*)
|
||||
# In POSIX sh, ulimit -H is undefined. That's why the result is checked to see if it worked.
|
||||
# shellcheck disable=SC2039,SC3045
|
||||
MAX_FD=$( ulimit -H -n ) ||
|
||||
warn "Could not query maximum file descriptor limit"
|
||||
esac
|
||||
case $MAX_FD in #(
|
||||
'' | soft) :;; #(
|
||||
*)
|
||||
# In POSIX sh, ulimit -n is undefined. That's why the result is checked to see if it worked.
|
||||
# shellcheck disable=SC2039,SC3045
|
||||
ulimit -n "$MAX_FD" ||
|
||||
warn "Could not set maximum file descriptor limit to $MAX_FD"
|
||||
esac
|
||||
fi
|
||||
|
||||
# Collect all arguments for the java command, stacking in reverse order:
|
||||
# * args from the command line
|
||||
# * the main class name
|
||||
# * -classpath
|
||||
# * -D...appname settings
|
||||
# * --module-path (only if needed)
|
||||
# * DEFAULT_JVM_OPTS, JAVA_OPTS, and GRADLE_OPTS environment variables.
|
||||
|
||||
# For Cygwin or MSYS, switch paths to Windows format before running java
|
||||
if "$cygwin" || "$msys" ; then
|
||||
APP_HOME=$( cygpath --path --mixed "$APP_HOME" )
|
||||
CLASSPATH=$( cygpath --path --mixed "$CLASSPATH" )
|
||||
|
||||
JAVACMD=$( cygpath --unix "$JAVACMD" )
|
||||
|
||||
# Now convert the arguments - kludge to limit ourselves to /bin/sh
|
||||
for arg do
|
||||
if
|
||||
case $arg in #(
|
||||
-*) false ;; # don't mess with options #(
|
||||
/?*) t=${arg#/} t=/${t%%/*} # looks like a POSIX filepath
|
||||
[ -e "$t" ] ;; #(
|
||||
*) false ;;
|
||||
esac
|
||||
then
|
||||
arg=$( cygpath --path --ignore --mixed "$arg" )
|
||||
fi
|
||||
# Roll the args list around exactly as many times as the number of
|
||||
# args, so each arg winds up back in the position where it started, but
|
||||
# possibly modified.
|
||||
#
|
||||
# NB: a `for` loop captures its iteration list before it begins, so
|
||||
# changing the positional parameters here affects neither the number of
|
||||
# iterations, nor the values presented in `arg`.
|
||||
shift # remove old arg
|
||||
set -- "$@" "$arg" # push replacement arg
|
||||
done
|
||||
fi
|
||||
|
||||
|
||||
# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
|
||||
DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"'
|
||||
|
||||
# Collect all arguments for the java command:
|
||||
# * DEFAULT_JVM_OPTS, JAVA_OPTS, JAVA_OPTS, and optsEnvironmentVar are not allowed to contain shell fragments,
|
||||
# and any embedded shellness will be escaped.
|
||||
# * For example: A user cannot expect ${Hostname} to be expanded, as it is an environment variable and will be
|
||||
# treated as '${Hostname}' itself on the command line.
|
||||
|
||||
set -- \
|
||||
"-Dorg.gradle.appname=$APP_BASE_NAME" \
|
||||
-classpath "$CLASSPATH" \
|
||||
org.gradle.wrapper.GradleWrapperMain \
|
||||
"$@"
|
||||
|
||||
# Stop when "xargs" is not available.
|
||||
if ! command -v xargs >/dev/null 2>&1
|
||||
then
|
||||
die "xargs is not available"
|
||||
fi
|
||||
|
||||
# Use "xargs" to parse quoted args.
|
||||
#
|
||||
# With -n1 it outputs one arg per line, with the quotes and backslashes removed.
|
||||
#
|
||||
# In Bash we could simply go:
|
||||
#
|
||||
# readarray ARGS < <( xargs -n1 <<<"$var" ) &&
|
||||
# set -- "${ARGS[@]}" "$@"
|
||||
#
|
||||
# but POSIX shell has neither arrays nor command substitution, so instead we
|
||||
# post-process each arg (as a line of input to sed) to backslash-escape any
|
||||
# character that might be a shell metacharacter, then use eval to reverse
|
||||
# that process (while maintaining the separation between arguments), and wrap
|
||||
# the whole thing up as a single "set" statement.
|
||||
#
|
||||
# This will of course break if any of these variables contains a newline or
|
||||
# an unmatched quote.
|
||||
#
|
||||
|
||||
eval "set -- $(
|
||||
printf '%s\n' "$DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS" |
|
||||
xargs -n1 |
|
||||
sed ' s~[^-[:alnum:]+,./:=@_]~\\&~g; ' |
|
||||
tr '\n' ' '
|
||||
)" '"$@"'
|
||||
|
||||
exec "$JAVACMD" "$@"
|
|
@ -1,92 +0,0 @@
|
|||
@rem
|
||||
@rem Copyright 2015 the original author or authors.
|
||||
@rem
|
||||
@rem Licensed under the Apache License, Version 2.0 (the "License");
|
||||
@rem you may not use this file except in compliance with the License.
|
||||
@rem You may obtain a copy of the License at
|
||||
@rem
|
||||
@rem https://www.apache.org/licenses/LICENSE-2.0
|
||||
@rem
|
||||
@rem Unless required by applicable law or agreed to in writing, software
|
||||
@rem distributed under the License is distributed on an "AS IS" BASIS,
|
||||
@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
@rem See the License for the specific language governing permissions and
|
||||
@rem limitations under the License.
|
||||
@rem
|
||||
|
||||
@if "%DEBUG%"=="" @echo off
|
||||
@rem ##########################################################################
|
||||
@rem
|
||||
@rem Gradle startup script for Windows
|
||||
@rem
|
||||
@rem ##########################################################################
|
||||
|
||||
@rem Set local scope for the variables with windows NT shell
|
||||
if "%OS%"=="Windows_NT" setlocal
|
||||
|
||||
set DIRNAME=%~dp0
|
||||
if "%DIRNAME%"=="" set DIRNAME=.
|
||||
@rem This is normally unused
|
||||
set APP_BASE_NAME=%~n0
|
||||
set APP_HOME=%DIRNAME%
|
||||
|
||||
@rem Resolve any "." and ".." in APP_HOME to make it shorter.
|
||||
for %%i in ("%APP_HOME%") do set APP_HOME=%%~fi
|
||||
|
||||
@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
|
||||
set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m"
|
||||
|
||||
@rem Find java.exe
|
||||
if defined JAVA_HOME goto findJavaFromJavaHome
|
||||
|
||||
set JAVA_EXE=java.exe
|
||||
%JAVA_EXE% -version >NUL 2>&1
|
||||
if %ERRORLEVEL% equ 0 goto execute
|
||||
|
||||
echo. 1>&2
|
||||
echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 1>&2
|
||||
echo. 1>&2
|
||||
echo Please set the JAVA_HOME variable in your environment to match the 1>&2
|
||||
echo location of your Java installation. 1>&2
|
||||
|
||||
goto fail
|
||||
|
||||
:findJavaFromJavaHome
|
||||
set JAVA_HOME=%JAVA_HOME:"=%
|
||||
set JAVA_EXE=%JAVA_HOME%/bin/java.exe
|
||||
|
||||
if exist "%JAVA_EXE%" goto execute
|
||||
|
||||
echo. 1>&2
|
||||
echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% 1>&2
|
||||
echo. 1>&2
|
||||
echo Please set the JAVA_HOME variable in your environment to match the 1>&2
|
||||
echo location of your Java installation. 1>&2
|
||||
|
||||
goto fail
|
||||
|
||||
:execute
|
||||
@rem Setup the command line
|
||||
|
||||
set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar
|
||||
|
||||
|
||||
@rem Execute Gradle
|
||||
"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %*
|
||||
|
||||
:end
|
||||
@rem End local scope for the variables with windows NT shell
|
||||
if %ERRORLEVEL% equ 0 goto mainEnd
|
||||
|
||||
:fail
|
||||
rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of
|
||||
rem the _cmd.exe /c_ return code!
|
||||
set EXIT_CODE=%ERRORLEVEL%
|
||||
if %EXIT_CODE% equ 0 set EXIT_CODE=1
|
||||
if not ""=="%GRADLE_EXIT_CONSOLE%" exit %EXIT_CODE%
|
||||
exit /b %EXIT_CODE%
|
||||
|
||||
:mainEnd
|
||||
if "%OS%"=="Windows_NT" endlocal
|
||||
|
||||
:omega
|
|
@ -9,33 +9,32 @@ handle_error () {
|
|||
# Change the working directory to the script's directory, when running from another location.
|
||||
cd "${0%/*}" || handle_error "Could not change-dir to this script's dir!" 1
|
||||
|
||||
justRun=0
|
||||
justInstall=0
|
||||
shouldRunInDocker=0
|
||||
|
||||
if [[ $# -eq 1 ]]; then
|
||||
justRun=$1
|
||||
justInstall=$1
|
||||
elif [[ $# -eq 2 ]]; then
|
||||
justRun=$1
|
||||
justInstall=$1
|
||||
shouldRunInDocker=$2
|
||||
elif [[ $# -gt 2 ]]; then
|
||||
echo -e "Wrong number of arguments given: ${#}\nPlease execute it like: installAndRun.sh <justRun: 0 | 1> <shouldRunInDocker: 0 | 1>"; exit 2
|
||||
echo -e "Wrong number of arguments given: ${#}\nPlease execute it like: script.sh <justInstall: 0 | 1> <shouldRunInDocker: 0 | 1>"; exit 2
|
||||
fi
|
||||
|
||||
if [[ justRun -eq 1 && shouldRunInDocker -eq 1 ]]; then
|
||||
echo -e "Cannot run in docker without re-building the project (just to be safe). Setting \"justRun\" to < 0 >"
|
||||
justRun=0
|
||||
if [[ justInstall -eq 1 && shouldRunInDocker -eq 1 ]]; then
|
||||
echo -e "Cannot run in docker without re-building the project (just to be safe). Setting \"justInstall\" to < 0 >"
|
||||
justInstall=0
|
||||
fi
|
||||
|
||||
gradleVersion="8.7"
|
||||
gradleVersion="8.1.1"
|
||||
|
||||
if [[ justRun -eq 0 ]]; then
|
||||
if [[ justInstall -eq 0 ]]; then
|
||||
|
||||
if [[ ! -d /opt/gradle/gradle-${gradleVersion} ]]; then
|
||||
wget https://services.gradle.org/distributions/gradle-${gradleVersion}-bin.zip
|
||||
echo -e "\nAsking for sudo, in order to install 'gradle'..\n"
|
||||
sudo mkdir /opt/gradle
|
||||
sudo apt install -y unzip && sudo unzip -d /opt/gradle gradle-${gradleVersion}-bin.zip
|
||||
sudo rm -rf gradle-${gradleVersion}-bin.zip
|
||||
#ls /opt/gradle/gradle-${gradleVersion} # For debugging installation
|
||||
fi
|
||||
|
||||
|
@ -46,7 +45,7 @@ if [[ justRun -eq 0 ]]; then
|
|||
#gradle tasks # For debugging installation
|
||||
#gradle -v # For debugging installation
|
||||
|
||||
gradle clean build # --refresh-dependencies # --info
|
||||
gradle clean build
|
||||
|
||||
if [[ shouldRunInDocker -eq 1 ]]; then
|
||||
|
||||
|
@ -64,14 +63,11 @@ if [[ justRun -eq 0 ]]; then
|
|||
(sudo docker compose -f prometheus/docker-compose-prometheus.yml up --build -d && echo -e "\nThe Prometheus and Grafana docker-containers started running.\n") || handle_error "Could not build and/or run the 'prometheus' and/or 'grafana' docker containers!" 5
|
||||
fi
|
||||
|
||||
echo -e "Waiting 20 seconds before getting the status..\n"
|
||||
sleep 20
|
||||
# 20 seconds are enough to check if there is an immediate fatal error with one of the docker images. This will cover the problematic configuration case for Prometheus and Grafana containers.
|
||||
echo -e "Waiting 65 seconds before getting the status..\n"
|
||||
sleep 65
|
||||
sudo docker ps -a || handle_error "Could not get the status of docker-containers!" 6 # Using -a to get the status of failed containers as well.
|
||||
echo -e "\n\nGetting the logs of docker-container \"urls_controller\":\n"
|
||||
sudo docker logs -f urls_controller || handle_error "Could not get the logs of docker-container \"urls_controller\"!" 7 # Using "regex anchors" to avoid false-positives. Works even if the container is not running, thus showing the error-log.
|
||||
# Use just the container-name and the "-f" parameter to indicate that we want to follow on logs updates, until we specify to unfollow them (with ctrl+c).
|
||||
# This way we do not need to run the "docker logs" again and again, not checking the the container-id each time.
|
||||
echo -e "\n\nGetting the logs of docker-container \"urlscontroller-urls_controller-1\":\n"
|
||||
sudo docker logs "$(sudo docker ps -aqf "name=^urlscontroller-urls_controller-1$")" || handle_error "Could not get the logs of docker-container \"urlscontroller-urls_controller-1\"!" 7 # Using "regex anchors" to avoid false-positives. Works even if the container is not running, thus showing the error-log.
|
||||
fi
|
||||
else
|
||||
export PATH=/opt/gradle/gradle-${gradleVersion}/bin:$PATH # Make sure the gradle is still accessible (it usually isn't without the "export").
|
||||
|
|
|
@ -4,7 +4,6 @@ services:
|
|||
|
||||
prometheus:
|
||||
image: 'prom/prometheus:latest'
|
||||
container_name: prometheus
|
||||
ports:
|
||||
- '9090:9090'
|
||||
command: '--config.file=/etc/prometheus/config.yml'
|
||||
|
@ -13,7 +12,6 @@ services:
|
|||
|
||||
grafana:
|
||||
image: 'grafana/grafana:latest'
|
||||
container_name: grafana
|
||||
ports:
|
||||
- '3000:3000'
|
||||
depends_on:
|
||||
|
|
|
@ -0,0 +1,8 @@
|
|||
echo "Running compose down.."
|
||||
sudo docker compose -f ./prometheus/docker-compose-prometheus.yml down
|
||||
sudo docker compose -f docker-compose.yml -f ./prometheus/docker-compose-prometheus.yml down
|
||||
|
||||
|
||||
# In case we need to hard-remove the containers, use the following commands:
|
||||
#sudo docker stop $(sudo docker ps -aqf "name=^(?:urlscontroller-urls_controller|prometheus-(?:prometheus|grafana))-1$") || true # There may be no active containers
|
||||
#sudo docker rm $(sudo docker ps -aqf "name=^(?:urlscontroller-urls_controller|prometheus-(?:prometheus|grafana))-1$") || true # All containers may be already removed.
|
|
@ -1,40 +0,0 @@
|
|||
# This script shuts down (ONLY!) the Controller, by stopping and killing the related containers.
|
||||
# It is used during testing.
|
||||
# It does not shuts down the whole service! The workers will keep running and their work will be lost.
|
||||
|
||||
# For error-handling, we cannot use the "set -e" since: it has problems https://mywiki.wooledge.org/BashFAQ/105
|
||||
# So we have our own function, for use when a single command fails.
|
||||
handle_error () {
|
||||
echo -e "\n\n$1\n\n"; exit $2
|
||||
}
|
||||
|
||||
# Change the working directory to the script's directory, when running from another location.
|
||||
cd "${0%/*}" || handle_error "Could not change-dir to this script's dir!" 1
|
||||
|
||||
forceControllerShutdown=0
|
||||
|
||||
if [[ $# -eq 1 ]]; then
|
||||
forceControllerShutdown=$1
|
||||
elif [[ $# -gt 1 ]]; then
|
||||
echo -e "Wrong number of arguments given: ${#}\nPlease execute it like: shutdownService.sh <forceControllerShutdown: 0 | 1>"; exit 2
|
||||
fi
|
||||
# We may have no arguments, if we do not want to force the Controller to shutdown.
|
||||
|
||||
# Shutdown Prometheus, if it's running.
|
||||
sudo docker compose -f ./prometheus/docker-compose-prometheus.yml down
|
||||
|
||||
if [[ forceControllerShutdown -eq 1 ]]; then
|
||||
echo "Shutting down the Controller.."
|
||||
sudo docker compose -f docker-compose.yml down
|
||||
else
|
||||
echo "Shutting down the Service.."
|
||||
sudo apt install -y curl
|
||||
curl -X POST -i 'http://localhost:1880/api/shutdownService'
|
||||
|
||||
# Follow the logs until shutdown.
|
||||
sudo docker logs -f urls_controller || handle_error "Could not get the logs of docker-container \"urls_controller\"!" 3 # Using "regex anchors" to avoid false-positives. Works even if the container is not running, thus showing the error-log.
|
||||
fi
|
||||
|
||||
# In case we need to hard-remove the containers, use the following commands:
|
||||
#sudo docker stop $(sudo docker ps -aqf "name=^(?:urlscontroller-urls_controller|prometheus-(?:prometheus|grafana))-1$") || true # There may be no active containers
|
||||
#sudo docker rm $(sudo docker ps -aqf "name=^(?:urlscontroller-urls_controller|prometheus-(?:prometheus|grafana))-1$") || true # All containers may be already removed.
|
|
@ -1,8 +1,7 @@
|
|||
package eu.openaire.urls_controller;
|
||||
|
||||
import com.zaxxer.hikari.HikariDataSource;
|
||||
import eu.openaire.urls_controller.controllers.BulkImportController;
|
||||
import eu.openaire.urls_controller.controllers.UrlsController;
|
||||
import eu.openaire.urls_controller.services.BulkImportServiceImpl;
|
||||
import eu.openaire.urls_controller.services.UrlsServiceImpl;
|
||||
import eu.openaire.urls_controller.util.FileUtils;
|
||||
import eu.openaire.urls_controller.util.UriBuilder;
|
||||
|
@ -10,7 +9,6 @@ import io.micrometer.core.aop.TimedAspect;
|
|||
import io.micrometer.core.instrument.MeterRegistry;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.boot.CommandLineRunner;
|
||||
import org.springframework.boot.SpringApplication;
|
||||
import org.springframework.boot.autoconfigure.SpringBootApplication;
|
||||
|
@ -31,18 +29,15 @@ import java.util.concurrent.TimeUnit;
|
|||
|
||||
@SpringBootApplication
|
||||
@EnableScheduling
|
||||
public class UrlsControllerApplication {
|
||||
public class Application {
|
||||
|
||||
private static final Logger logger = LoggerFactory.getLogger(UrlsControllerApplication.class);
|
||||
|
||||
@Autowired
|
||||
HikariDataSource hikariDataSource;
|
||||
private static final Logger logger = LoggerFactory.getLogger(Application.class);
|
||||
|
||||
private static ConfigurableApplicationContext context;
|
||||
|
||||
|
||||
public static void main(String[] args) {
|
||||
context = SpringApplication.run(UrlsControllerApplication.class, args);
|
||||
context = SpringApplication.run(Application.class, args);
|
||||
}
|
||||
|
||||
@Bean
|
||||
|
@ -58,10 +53,8 @@ public class UrlsControllerApplication {
|
|||
}
|
||||
|
||||
|
||||
public void gentleAppShutdown()
|
||||
public static void gentleAppShutdown()
|
||||
{
|
||||
logger.info("Shutting down the app..");
|
||||
shutdownThreads();
|
||||
int exitCode = 0;
|
||||
try {
|
||||
exitCode = SpringApplication.exit(context, () -> 0); // The "PreDestroy" method will be called. (the "context" will be closed automatically (I checked it))
|
||||
|
@ -72,55 +65,34 @@ public class UrlsControllerApplication {
|
|||
}
|
||||
|
||||
|
||||
private boolean haveThreadsShutdown = false;
|
||||
|
||||
@PreDestroy
|
||||
public void shutdownThreads()
|
||||
{
|
||||
// Normally this methods will have already been executed by "gentleAppShutdown()" just before the "PreDestroy" has called this method again.
|
||||
// BUT, in case the service was shutdown from the OS, without the use of the "ShutdownService-API, then this will be the 1st time this method is called.
|
||||
if ( haveThreadsShutdown )
|
||||
return;
|
||||
|
||||
public void preDestroy() {
|
||||
logger.info("Shutting down the threads..");
|
||||
|
||||
shutdownThreadsForExecutorService(UrlsServiceImpl.insertsExecutor);
|
||||
shutdownThreadsForExecutorService(FileUtils.hashMatchingExecutor);
|
||||
shutdownThreadsForExecutorService(BulkImportController.bulkImportExecutor);
|
||||
shutdownThreadsForExecutorService(UrlsController.backgroundExecutor);
|
||||
|
||||
haveThreadsShutdown = true;
|
||||
|
||||
// For some reason the Hikari Datasource cannot close properly by Spring Boot, unless we explicitly call close here.
|
||||
hikariDataSource.close();
|
||||
shutdownThreads(UrlsServiceImpl.insertsExecutor);
|
||||
shutdownThreads(FileUtils.hashMatchingExecutor);
|
||||
shutdownThreads(UrlsController.backgroundExecutor);
|
||||
shutdownThreads(BulkImportServiceImpl.bulkImportExecutor);
|
||||
|
||||
logger.info("Exiting..");
|
||||
}
|
||||
|
||||
|
||||
private void shutdownThreadsForExecutorService(ExecutorService executorService) throws RuntimeException
|
||||
private void shutdownThreads(ExecutorService executorService)
|
||||
{
|
||||
executorService.shutdown(); // Define that no new tasks will be scheduled.
|
||||
try {
|
||||
if ( ! executorService.awaitTermination(2, TimeUnit.MINUTES) ) {
|
||||
logger.warn("The working threads did not finish on time! Stopping them immediately..");
|
||||
executorService.shutdownNow();
|
||||
// Wait a while for tasks to respond to being cancelled (thus terminated).
|
||||
if ( ! executorService.awaitTermination(1, TimeUnit.MINUTES) )
|
||||
logger.warn("The executor " + executorService + " could not be terminated!");
|
||||
}
|
||||
} catch (SecurityException se) {
|
||||
logger.error("Could not shutdown the threads in any way..!", se);
|
||||
} catch (InterruptedException ie) {
|
||||
try {
|
||||
executorService.shutdownNow();
|
||||
// Wait a while for tasks to respond to being cancelled (thus terminated).
|
||||
if ( ! executorService.awaitTermination(1, TimeUnit.MINUTES) )
|
||||
logger.warn("The executor " + executorService + " could not be terminated!");
|
||||
} catch (SecurityException se) {
|
||||
logger.error("Could not shutdown the threads in any way..!", se);
|
||||
} catch (InterruptedException e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
}
|
||||
}
|
|
@ -14,8 +14,6 @@ public class BulkImport {
|
|||
|
||||
private String bulkImportReportLocation;
|
||||
|
||||
private int numOfThreadsForBulkImportProcedures;
|
||||
|
||||
private Map<String, BulkImportSource> bulkImportSources;
|
||||
|
||||
public BulkImport() {
|
||||
|
@ -37,14 +35,6 @@ public class BulkImport {
|
|||
this.bulkImportReportLocation = bulkImportReportLocation;
|
||||
}
|
||||
|
||||
public int getNumOfThreadsForBulkImportProcedures() {
|
||||
return numOfThreadsForBulkImportProcedures;
|
||||
}
|
||||
|
||||
public void setNumOfThreadsForBulkImportProcedures(int numOfThreadsForBulkImportProcedures) {
|
||||
this.numOfThreadsForBulkImportProcedures = numOfThreadsForBulkImportProcedures;
|
||||
}
|
||||
|
||||
public Map<String, BulkImportSource> getBulkImportSources() {
|
||||
return bulkImportSources;
|
||||
}
|
||||
|
@ -58,23 +48,22 @@ public class BulkImport {
|
|||
return "BulkImport{" +
|
||||
"baseBulkImportLocation='" + baseBulkImportLocation + '\'' +
|
||||
", bulkImportReportLocation='" + bulkImportReportLocation + '\'' +
|
||||
", numOfThreadsForBulkImportProcedures=" + numOfThreadsForBulkImportProcedures +
|
||||
", bulkImportSources=" + bulkImportSources +
|
||||
'}';
|
||||
}
|
||||
|
||||
|
||||
public static class BulkImportSource {
|
||||
private String datasourceID;
|
||||
private String datasourcePrefix;
|
||||
private String pdfUrlPrefix;
|
||||
private String mimeType;
|
||||
private boolean isAuthoritative;
|
||||
String datasourceID;
|
||||
String datasourcePrefix;
|
||||
String pdfUrlPrefix;
|
||||
String mimeType;
|
||||
|
||||
|
||||
public BulkImportSource() {
|
||||
}
|
||||
|
||||
|
||||
public String getDatasourceID() {
|
||||
return datasourceID;
|
||||
}
|
||||
|
@ -107,13 +96,6 @@ public class BulkImport {
|
|||
this.mimeType = mimeType;
|
||||
}
|
||||
|
||||
public boolean getIsAuthoritative() {
|
||||
return isAuthoritative;
|
||||
}
|
||||
|
||||
public void setIsAuthoritative(boolean isAuthoritative) {
|
||||
this.isAuthoritative = isAuthoritative;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
|
@ -122,7 +104,6 @@ public class BulkImport {
|
|||
", datasourcePrefix='" + datasourcePrefix + '\'' +
|
||||
", pdfUrlPrefix='" + pdfUrlPrefix + '\'' +
|
||||
", mimeType='" + mimeType + '\'' +
|
||||
", isAuthoritative=" + isAuthoritative +
|
||||
'}';
|
||||
}
|
||||
}
|
||||
|
|
|
@ -1,38 +1,25 @@
|
|||
package eu.openaire.urls_controller.components;
|
||||
|
||||
import com.google.gson.Gson;
|
||||
import com.google.gson.JsonSyntaxException;
|
||||
import eu.openaire.urls_controller.UrlsControllerApplication;
|
||||
import eu.openaire.urls_controller.configuration.DatabaseConnector;
|
||||
import eu.openaire.urls_controller.controllers.BulkImportController;
|
||||
import eu.openaire.urls_controller.Application;
|
||||
import eu.openaire.urls_controller.controllers.ShutdownController;
|
||||
import eu.openaire.urls_controller.controllers.StatsController;
|
||||
import eu.openaire.urls_controller.controllers.UrlsController;
|
||||
import eu.openaire.urls_controller.models.WorkerInfo;
|
||||
import eu.openaire.urls_controller.payloads.requests.WorkerReport;
|
||||
import eu.openaire.urls_controller.services.UrlsServiceImpl;
|
||||
import eu.openaire.urls_controller.util.FileUtils;
|
||||
import eu.openaire.urls_controller.util.GenericUtils;
|
||||
import io.micrometer.core.instrument.MeterRegistry;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.http.ResponseEntity;
|
||||
import org.springframework.scheduling.annotation.Scheduled;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
import java.io.BufferedReader;
|
||||
import java.io.File;
|
||||
import java.io.FileReader;
|
||||
import java.text.DecimalFormat;
|
||||
import java.util.*;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Date;
|
||||
import java.util.List;
|
||||
import java.util.concurrent.Callable;
|
||||
import java.util.concurrent.CancellationException;
|
||||
import java.util.concurrent.ExecutionException;
|
||||
import java.util.concurrent.Future;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
import java.util.regex.Matcher;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
|
||||
@Component
|
||||
|
@ -43,456 +30,123 @@ public class ScheduledTasks {
|
|||
@Autowired
|
||||
FileUtils fileUtils;
|
||||
|
||||
@Autowired
|
||||
UrlsServiceImpl urlsService;
|
||||
|
||||
@Autowired
|
||||
private UrlsControllerApplication urlsControllerApplication;
|
||||
|
||||
private final StatsController statsController;
|
||||
|
||||
private final UrlsController urlsController;
|
||||
|
||||
@Value("${services.pdfaggregation.controller.assignmentLimit}")
|
||||
private int assignmentsLimit;
|
||||
|
||||
public static final DecimalFormat df = new DecimalFormat("0.00");
|
||||
|
||||
private final String workerReportsDirPath;
|
||||
|
||||
public static final AtomicInteger numOfAllPayloads = new AtomicInteger(0);
|
||||
public static final AtomicInteger numOfPayloadsAggregatedByServiceThroughCrawling = new AtomicInteger(0);
|
||||
public static final AtomicInteger numOfPayloadsAggregatedByServiceThroughBulkImport = new AtomicInteger(0);
|
||||
public static final AtomicInteger numOfPayloadsAggregatedByService = new AtomicInteger(0);
|
||||
public static final AtomicInteger numOfLegacyPayloads = new AtomicInteger(0);
|
||||
|
||||
public static final AtomicInteger numOfRecordsInspectedByServiceThroughCrawling = new AtomicInteger(0);
|
||||
|
||||
|
||||
public ScheduledTasks(@Value("${services.pdfaggregation.controller.workerReportsDirPath}") String workerReportsDirPath, StatsController statsController, UrlsController urlsController, MeterRegistry registry)
|
||||
public ScheduledTasks(@Value("${services.pdfaggregation.controller.workerReportsDirPath}") String workerReportsDirPath)
|
||||
{
|
||||
if ( !workerReportsDirPath.endsWith("/") )
|
||||
workerReportsDirPath += "/";
|
||||
|
||||
this.workerReportsDirPath = workerReportsDirPath; // This dir will be created later.
|
||||
this.statsController = statsController;
|
||||
this.urlsController = urlsController;
|
||||
jsonStringBuilder = new StringBuilder(assignmentsLimit * 500);
|
||||
|
||||
registry.gauge("numOfAllPayloads", numOfAllPayloads);
|
||||
registry.gauge("numOfPayloadsAggregatedByServiceThroughCrawling", numOfPayloadsAggregatedByServiceThroughCrawling);
|
||||
registry.gauge("numOfPayloadsAggregatedByServiceThroughBulkImport", numOfPayloadsAggregatedByServiceThroughBulkImport);
|
||||
registry.gauge("numOfPayloadsAggregatedByService", numOfPayloadsAggregatedByService);
|
||||
registry.gauge("numOfLegacyPayloads", numOfLegacyPayloads);
|
||||
registry.gauge("numOfRecordsInspectedByServiceThroughCrawling", numOfRecordsInspectedByServiceThroughCrawling);
|
||||
}
|
||||
|
||||
|
||||
@Scheduled(initialDelay = 1_800_000, fixedDelay = 900_000) // Execute this method 30 mins from the start and 15 mins after each last execution, in order for some tasks to have been gathered.
|
||||
//@Scheduled(initialDelay = 60_000, fixedDelay = 20_000) // Just for testing (every 20 secs).
|
||||
// The initial delay is larger, because we have to wait some time for at least one worker to finish retrieving the full-texts from thousands of publications, whereas, later we will have a lot of workerReports waiting to be processed.
|
||||
public void checkResultsOfBackgroundTasks()
|
||||
@Scheduled(initialDelay = 1_800_000, fixedDelay = 1_800_000) // Execute this method 30 mins from the start and 30 mins after each last execution, in order for some tasks to have been gathered.
|
||||
//@Scheduled(initialDelay = 20_000, fixedDelay = 20_000) // Just for testing (every 20 secs).
|
||||
public void executeBackgroundTasks()
|
||||
{
|
||||
int sizeOfFutures = UrlsController.futuresOfBackgroundTasks.size();
|
||||
if ( sizeOfFutures == 0 )
|
||||
List<Callable<Boolean>> tempList = new ArrayList<>(UrlsController.backgroundCallableTasks); // Copy the list in order to know what was executed.
|
||||
// So the items added while this execution happens, will be remain in the global-list, while the other will have already be deleted.
|
||||
int numOfTasks = tempList.size(); // Since the temp-list is a deep-copy and not a reference, new tasks that are added will not be executed.
|
||||
if ( numOfTasks == 0 )
|
||||
return;
|
||||
|
||||
logger.debug("Going to check the results of " + sizeOfFutures + " background tasks.");
|
||||
|
||||
// Calling ".get()" on each future will block the current scheduling thread until a result is returned (if the callable task succeeded or failed).
|
||||
// So this scheduled-task will not be executed again until all current callableTasks have finished executing.
|
||||
|
||||
int numFailedTasks = 0;
|
||||
|
||||
List<Future<Boolean>> futuresToDelete = new ArrayList<>(sizeOfFutures);
|
||||
|
||||
for ( int i=0; i < sizeOfFutures; ++i ) { // Check the futures up to the current "sizeOfFutures". The size of the list may increase while we check, but here we wil check only the first e.g. 10 futures.
|
||||
Future<Boolean> future = null;
|
||||
try {
|
||||
future = UrlsController.futuresOfBackgroundTasks.get(i);
|
||||
if ( ! future.get() ) // Get and see if an exception is thrown. This blocks the current thread, until the task of the future has finished.
|
||||
numFailedTasks ++;
|
||||
} catch (ExecutionException ee) { // These can be serious errors like an "out of memory exception" (Java HEAP).
|
||||
logger.error(GenericUtils.getSelectedStackTraceForCausedException(ee, "Background_task_" + i + " failed with: ", null, 30));
|
||||
numFailedTasks ++;
|
||||
} catch (CancellationException ce) {
|
||||
logger.error("Background_task_" + i + " was cancelled: " + ce.getMessage());
|
||||
numFailedTasks ++;
|
||||
} catch (InterruptedException ie) {
|
||||
logger.error("Background_task_" + i + " was interrupted: " + ie.getMessage());
|
||||
numFailedTasks ++;
|
||||
} catch (IndexOutOfBoundsException ioobe) {
|
||||
logger.error("IOOBE for background_task_" + i + " in the futures-list! " + ioobe.getMessage());
|
||||
// Only here, the "future" will be null.
|
||||
} finally {
|
||||
if ( future != null ) // It may be null in case we have a IOBE.
|
||||
futuresToDelete.add(future); // Do not delete them directly here, as the indexes will get messed up and we will get "IOOBE".
|
||||
}
|
||||
// Immediately delete the selected tasks form the global list, so that if these tasks are not finished before the scheduler runs again, they will not be re-executed.
|
||||
for ( Callable<Boolean> selectedTask : tempList ) {
|
||||
UrlsController.backgroundCallableTasks.remove(selectedTask);
|
||||
}
|
||||
|
||||
for ( Future<Boolean> future : futuresToDelete )
|
||||
UrlsController.futuresOfBackgroundTasks.remove(future); // We do not need it anymore. This is the safest way to delete them without removing newly added futures as well.
|
||||
|
||||
if ( numFailedTasks > 0 )
|
||||
logger.warn(numFailedTasks + " out of " + sizeOfFutures + " background tasks have failed!");
|
||||
else if ( logger.isTraceEnabled() )
|
||||
logger.trace("All of the " + sizeOfFutures + " background tasks have succeeded.");
|
||||
}
|
||||
|
||||
|
||||
@Scheduled(initialDelay = 600_000, fixedDelay = 7_200_000) // Check every 2 hours. The initial delay is 10 minutes, to allow to shut down quickly in case of problem when starting, but also account for the initial communication with the Workers, where a problem may appear.
|
||||
//@Scheduled(initialDelay = 60_000, fixedDelay = 20_000) // Just for testing (every 20 secs).
|
||||
public void checkIfServiceIsReadyForShutdown()
|
||||
{
|
||||
if ( ! ShutdownController.shouldShutdownService )
|
||||
return; // Either the service was never instructed to shut down, or the user canceled the request.
|
||||
|
||||
// Check whether there are still background tasks to be processed. Either workerReport or Bulk-import requests.
|
||||
int numOfFutures = UrlsController.futuresOfBackgroundTasks.size();
|
||||
if ( numOfFutures > 0 ) {
|
||||
logger.debug("There are still " + numOfFutures + " backgroundTasks waiting to be executed or have their status checked..");
|
||||
return;
|
||||
}
|
||||
|
||||
// Here, the above may have given a result of < 0 >, but a new task may be asked for execution right next and still await for execution..
|
||||
// The crawling-jobs can be safely finish, by avoiding to shut-down as long as at least one worker is still running (waiting for the Controller to verify that the assignments-batch is completed).
|
||||
// The bulk-import procedures have their bulkImport DIR registered in the "bulkImportDirsUnderProcessing", before their takes is being submitted for execution.
|
||||
// So the Controller will now shut down if either of takes-types have not finished.
|
||||
|
||||
// Check whether there are any active bulk-import procedures.
|
||||
int numOfBulkImportDirsUnderProcessing = BulkImportController.bulkImportDirsUnderProcessing.size();
|
||||
if ( numOfBulkImportDirsUnderProcessing > 0 ) {
|
||||
logger.debug("There are still " + numOfBulkImportDirsUnderProcessing + " bulkImportDirsUnderProcessing..");
|
||||
return;
|
||||
}
|
||||
|
||||
// Check whether the workers have not shutdown yet, which means that they either crawl assignments or/and they are waiting for the Controller to process the WorkerReport and then shutdown.
|
||||
Set<String> workerIds = UrlsController.workersInfoMap.keySet();
|
||||
if ( workerIds.size() > 0 ) {
|
||||
for ( String workerId : workerIds ) {
|
||||
WorkerInfo workerInfo = UrlsController.workersInfoMap.get(workerId); // The workerId is certainly inside the map and has a workerInfo value.
|
||||
if ( ! workerInfo.getHasShutdown() ) {
|
||||
logger.debug("At least one worker (with IP: " + workerInfo.getWorkerIP() + ") is still active. Waiting for all workers to shutdown, before shutting down the service.");
|
||||
return; // If at least 1 worker is still active, then do not shut down the Controller.
|
||||
}
|
||||
}
|
||||
logger.info("All workers have already shutdown. Shutting down the Controller..");
|
||||
} else
|
||||
logger.info("No workers have participated in the service yet. The Controller will shut-down now.");
|
||||
|
||||
// IMPORTANT: If one worker has crashed, then it will have not informed the Controller. So the controller will think that it is still running and will not shut down..!
|
||||
// In this case, we have to manually shut-down the service, from Docker cli.
|
||||
|
||||
// Any left-over worker-reports are kept to be retried next time the Controller starts.
|
||||
|
||||
urlsControllerApplication.gentleAppShutdown();
|
||||
}
|
||||
|
||||
|
||||
@Value("${services.pdfaggregation.controller.isTestEnvironment}")
|
||||
private boolean isTestEnvironment;
|
||||
|
||||
|
||||
@Scheduled(initialDelay = 43_200_000, fixedDelay = Long.MAX_VALUE) // Run 12 hours after startup (in order for all workers to be back online) and then never again.
|
||||
//@Scheduled(initialDelay = 420_000, fixedDelay = Long.MAX_VALUE) // Just for testing (7 mins after startup, only once).
|
||||
public void checkAndProcessOldAndUnsuccessfulWorkerReports()
|
||||
{
|
||||
// We make sure an initial delay of some minutes is in place before this is executed, since we have to make sure all workers are up and running in order for them to be able to answer the full-texts-requests.
|
||||
|
||||
if ( UrlsController.numOfActiveWorkers.get() == 0 ) {
|
||||
long timeToWait = (isTestEnvironment ? 1_200_000 : 43_200_000); // 10 mins | 12 hours
|
||||
logger.warn("None of the workers is participating in the service, at the moment. Will wait " + ((timeToWait /1000) /60) + " minutes and try again..");
|
||||
try {
|
||||
Thread.sleep(timeToWait);
|
||||
} catch (InterruptedException ie) {
|
||||
logger.warn("The wait-period was interrupted! Will try either way.");
|
||||
}
|
||||
if ( UrlsController.numOfActiveWorkers.get() == 0 ) {
|
||||
logger.error("None of the workers is participating in the service, yet again. Will not process any leftover workerReports!");
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
// In case the Controller processes older, failed workerReports, it may be the case that the worker to which the report comes from, is not currently online.
|
||||
// So its IP won't be stored in the map. Even if we have its IP in the report itself, it's of no use if the worker is offline, since we cannot get the full-texts.
|
||||
// We do not care for attempts, if the payloads are not there.
|
||||
|
||||
inspectWorkerReportsAndTakeAction(ActionForWorkerReports.process_previous_failed);
|
||||
|
||||
// In case a worker-report among those "left-overs" fails again, then it will just be removed by the scheduler, when 7 days pass, and it's still there.
|
||||
}
|
||||
|
||||
|
||||
@Scheduled(initialDelay = 86_400_000, fixedDelay = 86_400_000) // Run after one day, every day.
|
||||
//@Scheduled(initialDelay = 1_200_000, fixedDelay = 1_200_000) // Just for testing (every 1200 secs).
|
||||
public void checkAndProcessRecentUnsuccessfulWorkerReports()
|
||||
{
|
||||
// After some hours from their failure and before the worker delete the full-texts as obsolete, we re-try these workerReports,
|
||||
// hopping that any previous error was temporary.
|
||||
|
||||
inspectWorkerReportsAndTakeAction(ActionForWorkerReports.process_current_failed);
|
||||
}
|
||||
|
||||
|
||||
@Scheduled(initialDelay = 604_800_000, fixedDelay = 604_800_000) // Run every 7 days.
|
||||
//@Scheduled(initialDelay = 1_200_000, fixedDelay = 1_200_000) // Just for testing (every 1200 secs).
|
||||
public void checkAndDeleteOldWorkerReports()
|
||||
{
|
||||
// The failed workerReports are kept for 7 days, for manual debugging purposes. Even if the service does not make more than 2 attempts to import them.
|
||||
|
||||
inspectWorkerReportsAndTakeAction(ActionForWorkerReports.delete_old);
|
||||
}
|
||||
|
||||
|
||||
@Scheduled(initialDelay = 259_200_000, fixedDelay = 259_200_000) // Run every 3 days. 3 days after startup.
|
||||
//@Scheduled(initialDelay = 1_200_000, fixedDelay = 1_200_000) // Just for testing (every 1200 secs).
|
||||
public void checkAndDeleteUnhandledAssignments()
|
||||
{
|
||||
// Remove the assignments having a "date" older than 3 days.
|
||||
|
||||
// For some reason, sometimes, the worker cannot receive the assignments in time.
|
||||
// In this case, no job is created for those assignments nd no workerReport gets stored in storage.
|
||||
// The assignments just remain in the table, and the urls cannot be rechecked.
|
||||
|
||||
Calendar calendar = Calendar.getInstance();
|
||||
calendar.add(Calendar.DAY_OF_MONTH, -3); // Subtract 3 from current Date.
|
||||
|
||||
DatabaseConnector.databaseLock.lock();
|
||||
urlsService.deleteAssignmentsWithOlderDate(calendar.getTimeInMillis()); // Any error-log is written inside.
|
||||
DatabaseConnector.databaseLock.unlock();
|
||||
}
|
||||
|
||||
|
||||
// Scheduled Metrics for Prometheus.
|
||||
// Prometheus scrapes for metrics usually every 15 seconds, but that is an extremely short time-period for DB-statistics.
|
||||
|
||||
@Scheduled(fixedDelay = 21_600_000) // Every 6 hours run the following queries to the database and register the metric.
|
||||
//@Scheduled(initialDelay = 60_000, fixedDelay = 1_200_000) // For general testing only.
|
||||
//@Scheduled(initialDelay = 60_000, fixedDelay = 120_000) // For debug testing only.
|
||||
public void updatePrometheusMetrics()
|
||||
{
|
||||
ResponseEntity<?> responseEntity = statsController.getNumberOfAllPayloads(true);
|
||||
if ( responseEntity.getStatusCode().value() == 200 ) {
|
||||
numOfAllPayloads.set(Integer.parseInt(responseEntity.getBody().toString())); // (any other cast method fails)
|
||||
} // Any error is already logged.
|
||||
|
||||
responseEntity = statsController.getNumberOfPayloadsAggregatedByServiceThroughCrawling(true);
|
||||
if ( responseEntity.getStatusCode().value() == 200 ) {
|
||||
numOfPayloadsAggregatedByServiceThroughCrawling.set(Integer.parseInt(responseEntity.getBody().toString())); // (any other cast method fails)
|
||||
} // Any error is already logged.
|
||||
|
||||
responseEntity = statsController.getNumberOfPayloadsAggregatedByServiceThroughBulkImport(true);
|
||||
if ( responseEntity.getStatusCode().value() == 200 ) {
|
||||
numOfPayloadsAggregatedByServiceThroughBulkImport.set(Integer.parseInt(responseEntity.getBody().toString())); // (any other cast method fails)
|
||||
} // Any error is already logged.
|
||||
|
||||
responseEntity = statsController.getNumberOfPayloadsAggregatedByService(true);
|
||||
if ( responseEntity.getStatusCode().value() == 200 ) {
|
||||
numOfPayloadsAggregatedByService.set(Integer.parseInt(responseEntity.getBody().toString())); // (any other cast method fails)
|
||||
} // Any error is already logged.
|
||||
|
||||
responseEntity = statsController.getNumberOfLegacyPayloads(true);
|
||||
if ( responseEntity.getStatusCode().value() == 200 ) {
|
||||
numOfLegacyPayloads.set(Integer.parseInt(responseEntity.getBody().toString())); // (any other cast method fails)
|
||||
} // Any error is already logged.
|
||||
|
||||
responseEntity = statsController.getNumberOfRecordsInspectedByServiceThroughCrawling(true);
|
||||
if ( responseEntity.getStatusCode().value() == 200 ) {
|
||||
numOfRecordsInspectedByServiceThroughCrawling.set(Integer.parseInt(responseEntity.getBody().toString())); // (any other cast method fails)
|
||||
} // Any error is already logged.
|
||||
|
||||
|
||||
// TODO - Export more complex data; <numOfAllPayloadsPerDatasource>, <numOfAllPayloadsPerPublicationYear>,
|
||||
// <numOfAggregatedPayloadsPerDatasource>, ..., <numOfBulkImportedPayloadsPerDatasource>, ...
|
||||
}
|
||||
|
||||
|
||||
enum ActionForWorkerReports {process_previous_failed, process_current_failed, delete_old}
|
||||
|
||||
// TODO - Maybe make these numbers configurable from the "application.yml" file.
|
||||
|
||||
private static final double daysToWaitBeforeProcessing = 0.5; // 12 hours
|
||||
|
||||
private static final double maxDaysToAllowProcessing = 1.9; // 45.6 hours
|
||||
// The Workers wait at most 48 hours before deleting the full-text files. So there is no point to try and process the report after that time-frame.
|
||||
// These reports will have to wait a bit for the scheduler to assign them to threads, before actually being processed.
|
||||
|
||||
|
||||
private static final Gson gson = new Gson(); // This is "transient" by default.
|
||||
private static StringBuilder jsonStringBuilder = null;
|
||||
private static final int daysDivisor = (1000 * 60 * 60 * 24); // In order to get the time-difference in days. We divide with: /1000 to get seconds, /60 to get minutes, /60 to get hours and /24 to get days.
|
||||
|
||||
private static final double daysToWaitBeforeDeletion = 7.0;
|
||||
|
||||
|
||||
private void inspectWorkerReportsAndTakeAction(ActionForWorkerReports actionForWorkerReports)
|
||||
{
|
||||
if ( actionForWorkerReports == ActionForWorkerReports.process_previous_failed )
|
||||
logger.debug("Going to check and process any unsuccessful workerReports from the previous run.");
|
||||
else if ( actionForWorkerReports == ActionForWorkerReports.process_current_failed )
|
||||
logger.debug("Going to check and process any unsuccessful workerReports which are between " + daysToWaitBeforeProcessing + " and " + daysToWaitBeforeDeletion + " days old (inclusive).");
|
||||
else
|
||||
logger.debug("Going to check and remove any leftover workerReports, which are more than " + daysToWaitBeforeDeletion + " days old.");
|
||||
// The failed workerReports are kept for 7 days, for manual debugging purposes. Even if the service does not make more than 2 attempts to import them.
|
||||
|
||||
logger.debug(numOfTasks + " background tasks were found inside the \"backgroundCallableTasks\" list and are about to be executed.");
|
||||
// Execute the tasks and wait for them to finish.
|
||||
try {
|
||||
File[] workerReportSubDirs = getWorkerReportSubDirs();
|
||||
if ( workerReportSubDirs == null )
|
||||
return;
|
||||
|
||||
long currentTime = 0L;
|
||||
if ( actionForWorkerReports != ActionForWorkerReports.process_previous_failed )
|
||||
currentTime = System.currentTimeMillis();
|
||||
|
||||
int numWorkerReportsToBeHandled = 0;
|
||||
int numWorkerReportsHandled = 0;
|
||||
|
||||
for ( File workerReportSubDir : workerReportSubDirs )
|
||||
{
|
||||
File[] workerReportFiles = workerReportSubDir.listFiles(File::isFile);
|
||||
if ( workerReportFiles == null ) {
|
||||
logger.error("There was an error when getting the workerReports of \"workerReportSubDir\": " + workerReportSubDir);
|
||||
return;
|
||||
} else if (workerReportFiles.length == 0) {
|
||||
logger.debug("The \"workerReportsDir\" is empty, so there is nothing to take action on.");
|
||||
return;
|
||||
List<Future<Boolean>> futures = UrlsController.backgroundExecutor.invokeAll(tempList);
|
||||
int sizeOfFutures = futures.size();
|
||||
for ( int i = 0; i < sizeOfFutures; ++i ) {
|
||||
try {
|
||||
Boolean value = futures.get(i).get(); // Get and see if an exception is thrown..
|
||||
// Add check for the value, if wanted.. (we don't care at the moment)
|
||||
} catch (ExecutionException ee) {
|
||||
String stackTraceMessage = GenericUtils.getSelectiveStackTrace(ee, null, 15); // These can be serious errors like an "out of memory exception" (Java HEAP).
|
||||
logger.error("Task_" + (i+1) + " failed with: " + ee.getMessage() + "\n" + stackTraceMessage);
|
||||
} catch (CancellationException ce) {
|
||||
logger.error("Task_" + (i+1) + " was cancelled: " + ce.getMessage());
|
||||
} catch (IndexOutOfBoundsException ioobe) {
|
||||
logger.error("IOOBE for task_" + i + " in the futures-list! " + ioobe.getMessage());
|
||||
}
|
||||
|
||||
for ( File workerReportFile : workerReportFiles )
|
||||
{
|
||||
String workerReportName = workerReportFile.getName();
|
||||
|
||||
if ( actionForWorkerReports == ActionForWorkerReports.process_previous_failed ) {
|
||||
if ( workerReportName.contains("failed") ) {
|
||||
numWorkerReportsToBeHandled ++;
|
||||
if ( processFailedWorkerReport(workerReportFile, workerReportName) )
|
||||
numWorkerReportsHandled ++;
|
||||
}
|
||||
} else {
|
||||
long lastModified = workerReportFile.lastModified();
|
||||
|
||||
if ( logger.isTraceEnabled() )
|
||||
logger.trace("The workerReport \"" + workerReportName + "\" was last accessed in: " + new Date(lastModified));
|
||||
|
||||
double elapsedDays = (double) (currentTime - lastModified) / daysDivisor;
|
||||
|
||||
if ( actionForWorkerReports == ActionForWorkerReports.process_current_failed ) {
|
||||
if ( (elapsedDays >= daysToWaitBeforeProcessing) && (elapsedDays <= maxDaysToAllowProcessing)
|
||||
&& workerReportName.contains("failed") ) {
|
||||
numWorkerReportsToBeHandled ++;
|
||||
if ( processFailedWorkerReport(workerReportFile, workerReportName) )
|
||||
numWorkerReportsHandled ++;
|
||||
}
|
||||
} else { // Deletion..
|
||||
if ( elapsedDays > daysToWaitBeforeDeletion ) {
|
||||
// Enough time has passed, the directory should be deleted immediately.
|
||||
logger.warn("The workerReport \"" + workerReportName + "\" was accessed " + df.format(elapsedDays) + " days ago (passed the " + daysToWaitBeforeDeletion + " days limit) and will be deleted.");
|
||||
numWorkerReportsToBeHandled ++;
|
||||
if ( fileUtils.deleteFile(workerReportFile.getAbsolutePath()) // Either successful or failed.
|
||||
&& !workerReportName.contains("successful") // If this has failed or its state is unknown (it was never renamed), then delete the assignment-records. For the successful, they have already been deleted.
|
||||
&& extractAssignmentsCounterAndDeleteRelatedAssignmentRecords(workerReportName) )
|
||||
numWorkerReportsHandled ++;
|
||||
}
|
||||
}
|
||||
}
|
||||
}// end reports loop
|
||||
}// end sub-dirs loop
|
||||
|
||||
String initMsg = "The action \"" + actionForWorkerReports.toString() + "\" was imposed to " + numWorkerReportsHandled + " workerReports.";
|
||||
int numWorkerReportsFailedToBeHandled = (numWorkerReportsToBeHandled - numWorkerReportsHandled);
|
||||
if ( numWorkerReportsFailedToBeHandled > 0 )
|
||||
logger.warn(initMsg + " " + numWorkerReportsFailedToBeHandled + " workerReports failed to be handled!");
|
||||
else
|
||||
logger.debug(initMsg);
|
||||
}
|
||||
} catch (Exception e) {
|
||||
logger.error("", e);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
private File[] getWorkerReportSubDirs() throws Exception
|
||||
@Scheduled(initialDelay = 60_000, fixedDelay = 7_200_000) // Check every 2 hours.
|
||||
//@Scheduled(initialDelay = 60_000, fixedDelay = 20_000) // Just for testing (every 20 secs).
|
||||
public void checkIfServiceIsReadyForShutdown()
|
||||
{
|
||||
File workerReportsDir = new File(workerReportsDirPath);
|
||||
if ( !workerReportsDir.isDirectory() ) {
|
||||
logger.error("The \"workerReportsDir\" (" + workerReportsDirPath + ") does not exist!"); // This base dir should always exist!
|
||||
return null;
|
||||
if ( ! ShutdownController.shouldShutdownService )
|
||||
return; // Either the service was never instructed to shut down, or the user canceled the request.
|
||||
|
||||
// If the workers have shutdown on their own, without been instructed to by the Controller, then the Controller will keep running.
|
||||
|
||||
for ( String workerId : UrlsController.workersInfoMap.keySet() ) {
|
||||
if ( ! UrlsController.workersInfoMap.get(workerId).getHasShutdown() ) // The workerId is certainly inside the map and has a workerInfo value.
|
||||
return; // If at least 1 worker is still active, then do not shut down the server.
|
||||
}
|
||||
// The worker reports are inside "worker_X" sub-dirs.
|
||||
File[] workerReportSubDirs = workerReportsDir.listFiles(File::isDirectory);
|
||||
if ( workerReportSubDirs == null ) {
|
||||
logger.error("There was an error when getting the subDirs of \"workerReportsDir\": " + workerReportsDir);
|
||||
return null;
|
||||
} else if ( workerReportSubDirs.length == 0 ) { // The worker_X sub-dirs do not normally get deleted. So this is a warning that either no workers are running (wasting time) or that something bad happened to thios directories.
|
||||
logger.warn("The \"workerReportsDir\" is empty. None of the workers returned a workerReport, so there is nothing to process.");
|
||||
return null;
|
||||
} else
|
||||
return workerReportSubDirs;
|
||||
|
||||
logger.info("All workers have already shutdown. Shutting down the Controller..");
|
||||
Application.gentleAppShutdown();
|
||||
}
|
||||
|
||||
|
||||
private boolean processFailedWorkerReport(File workerReportFile, String workerReportName)
|
||||
{
|
||||
logger.debug("Going to load and parse the failed workerReport: " + workerReportName);
|
||||
private static final double daysToWaitBeforeDeletion = 7.0;
|
||||
|
||||
// Load the file's json content into a "WorkerReport" object.
|
||||
try ( BufferedReader bfRead = new BufferedReader(new FileReader(workerReportFile), FileUtils.halfMb) ) {
|
||||
String line;
|
||||
while ( (line = bfRead.readLine()) != null ) // The line, without any line-termination-characters.
|
||||
jsonStringBuilder.append(line).append(GenericUtils.endOfLine);
|
||||
|
||||
@Scheduled(initialDelay = 604_800_000, fixedDelay = 604_800_000) // Run every 7 days.
|
||||
//@Scheduled(initialDelay = 1_200_000, fixedDelay = 1_200_000) // Just for testing (every 1200 secs).
|
||||
public void checkAndDeleteUnsuccessfulWorkerReports()
|
||||
{
|
||||
logger.debug("Going to check and remove any unsuccessful workerReports, which are more than 7 days old.");
|
||||
try {
|
||||
File workerReportsDir = new File(workerReportsDirPath);
|
||||
if ( !workerReportsDir.isDirectory() ) {
|
||||
logger.error("The \"workerReportsDir\" (" + workerReportsDirPath + ") does not exist!"); // This base dir should always exist!
|
||||
return;
|
||||
}
|
||||
|
||||
File[] workerReports = workerReportsDir.listFiles(File::isFile);
|
||||
if ( workerReports == null ) {
|
||||
logger.error("There was an error when getting the subDirs of \"workerReportsDir\": " + workerReportsDir);
|
||||
return;
|
||||
}
|
||||
else if ( workerReports.length == 0 ) {
|
||||
logger.debug("The \"workerReportsDir\" is empty, so there is nothing to delete.");
|
||||
return;
|
||||
}
|
||||
|
||||
long currentTime = System.currentTimeMillis();
|
||||
|
||||
// Loop through the array and print only the directories
|
||||
for ( File workerReport : workerReports ) {
|
||||
long lastModified = workerReport.lastModified();
|
||||
|
||||
if ( logger.isTraceEnabled() )
|
||||
logger.trace("The workerReport \"" + workerReport.getName() + "\" was last accessed in: " + new Date(lastModified));
|
||||
|
||||
// Get the difference in hours. /1000 to get seconds, /60 to get minutes, /60 to get hours and /24 to get days.
|
||||
double elapsedDays = (double) (currentTime - lastModified) / (1000 * 60 * 60 * 24);
|
||||
if ( elapsedDays > daysToWaitBeforeDeletion ) {
|
||||
// Enough time has passed, the directory should be deleted immediately.
|
||||
String workerReportName = workerReport.getName();
|
||||
logger.warn("The workerReport \"" + workerReportName + "\" was accessed " + elapsedDays + " days ago (passed the " + daysToWaitBeforeDeletion + " days limit) and will be deleted.");
|
||||
fileUtils.deleteFile(workerReport.getAbsolutePath());
|
||||
}
|
||||
}
|
||||
} catch (Exception e) {
|
||||
logger.error("Problem when acquiring the contents of workerReport \"" + workerReportName + "\"");
|
||||
jsonStringBuilder.setLength(0); // Reset the StringBuilder without de-allocating.
|
||||
// This failed report will be retried by the scheduler, inside the allowed time-period.
|
||||
// If it never reached success, then its file and the assignment-records will be deleted by the scheduler in the following days.
|
||||
return false;
|
||||
logger.error("", e);
|
||||
}
|
||||
|
||||
WorkerReport workerReport = null;
|
||||
try {
|
||||
workerReport = gson.fromJson(jsonStringBuilder.toString(), WorkerReport.class);
|
||||
} catch (JsonSyntaxException jse) {
|
||||
logger.error("Problem when parsing the workerReport \"" + workerReportName + "\": " + jse.getMessage());
|
||||
// Same handling as mentioned in the above "catch".
|
||||
return false;
|
||||
} finally {
|
||||
jsonStringBuilder.setLength(0); // Reset the StringBuilder without de-allocating.
|
||||
}
|
||||
|
||||
this.urlsController.addWorkerReport(workerReport); // This will check and add the workerReport to the background jobs' scheduler.
|
||||
return true;
|
||||
}
|
||||
|
||||
|
||||
private static final Pattern ASSIGNMENTS_COUNTER_REPORT_FILTER = Pattern.compile(".*([\\d]+)_report[\\w]*.json$");
|
||||
|
||||
private boolean extractAssignmentsCounterAndDeleteRelatedAssignmentRecords(String workerReportName)
|
||||
{
|
||||
// We need to delete the records from the "assignment" table, in order for them to be retried in the future.
|
||||
|
||||
// Take the counter from workerReportName.
|
||||
Matcher matcher = ASSIGNMENTS_COUNTER_REPORT_FILTER.matcher(workerReportName);
|
||||
if ( ! matcher.matches() ) {
|
||||
logger.error("Could not match the report \"" + workerReportName + "\" with regex: " + ASSIGNMENTS_COUNTER_REPORT_FILTER);
|
||||
return false;
|
||||
}
|
||||
String counterString = matcher.group(1);
|
||||
if ( (counterString == null) || counterString.isEmpty() ) {
|
||||
logger.error("Could not extract the \"assignmentCounter\" from report: " + workerReportName);
|
||||
return false;
|
||||
}
|
||||
int curReportAssignmentsCounter;
|
||||
try {
|
||||
curReportAssignmentsCounter = Integer.parseInt(counterString);
|
||||
} catch (NumberFormatException nfe) {
|
||||
logger.error("Could not parse the \"curReportAssignmentsCounter\" (" + counterString + ") which was extracted from report: " + workerReportName);
|
||||
return false;
|
||||
}
|
||||
|
||||
logger.debug("Will delete the assignments of the old, not-successful, workerReport: " + workerReportName);
|
||||
|
||||
DatabaseConnector.databaseLock.lock();
|
||||
urlsService.deleteAssignmentsBatch(curReportAssignmentsCounter); // Any error-log is written inside.
|
||||
DatabaseConnector.databaseLock.unlock();
|
||||
return true;
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
@ -12,9 +12,9 @@ import java.util.concurrent.locks.Lock;
|
|||
import java.util.concurrent.locks.ReentrantLock;
|
||||
|
||||
@Repository
|
||||
public class DatabaseConnector {
|
||||
public class ImpalaConnector {
|
||||
|
||||
private static final Logger logger = LoggerFactory.getLogger(DatabaseConnector.class);
|
||||
private static final Logger logger = LoggerFactory.getLogger(ImpalaConnector.class);
|
||||
|
||||
@Autowired
|
||||
private JdbcTemplate jdbcTemplate;
|
||||
|
@ -28,9 +28,9 @@ public class DatabaseConnector {
|
|||
public static final Lock databaseLock = new ReentrantLock(true); // This lock is locking the threads trying to execute queries in the database.
|
||||
|
||||
|
||||
public DatabaseConnector(@Value("${services.pdfaggregation.controller.isTestEnvironment}") boolean isTestEnvironment,
|
||||
@Value("${services.pdfaggregation.controller.db.initialDatabaseName}") String initialDatabaseName,
|
||||
@Value("${services.pdfaggregation.controller.db.testDatabaseName}") String testDatabaseName) {
|
||||
public ImpalaConnector(@Value("${services.pdfaggregation.controller.isTestEnvironment}") boolean isTestEnvironment,
|
||||
@Value("${services.pdfaggregation.controller.db.initialDatabaseName}") String initialDatabaseName,
|
||||
@Value("${services.pdfaggregation.controller.db.testDatabaseName}") String testDatabaseName) {
|
||||
this.isTestEnvironment = isTestEnvironment;
|
||||
this.initialDatabaseName = initialDatabaseName;
|
||||
this.testDatabaseName = testDatabaseName;
|
||||
|
@ -58,29 +58,24 @@ public class DatabaseConnector {
|
|||
logger.info("Going to create (if not exist) the TEST-database \"" + testDatabaseName + "\" and its tables. Also will fill some tables with data from the initial-database \"" + initialDatabaseName + "\".");
|
||||
jdbcTemplate.execute("CREATE DATABASE IF NOT EXISTS " + testDatabaseName);
|
||||
|
||||
try { // Metastore takes some time to recognize the DB has been created, in order to use it later..
|
||||
Thread.sleep(1000);
|
||||
} catch (InterruptedException ignore) {}
|
||||
jdbcTemplate.execute("CREATE TABLE IF NOT EXISTS " + testDatabaseName + ".publication stored as parquet as select * from " + initialDatabaseName + ".publication");
|
||||
jdbcTemplate.execute("COMPUTE STATS " + testDatabaseName + ".publication");
|
||||
|
||||
jdbcTemplate.update("INVALIDATE METADATA");
|
||||
jdbcTemplate.execute("CREATE TABLE IF NOT EXISTS " + testDatabaseName + ".publication_pids stored as parquet as select * from " + initialDatabaseName + ".publication_pids");
|
||||
jdbcTemplate.execute("COMPUTE STATS " + testDatabaseName + ".publication_pids");
|
||||
|
||||
try { // Metastore takes some time to recognize the DB has been created, in order to use it later..
|
||||
Thread.sleep(1000);
|
||||
} catch (InterruptedException ignore) {}
|
||||
jdbcTemplate.execute("CREATE TABLE IF NOT EXISTS " + testDatabaseName + ".publication_urls stored as parquet as select * from " + initialDatabaseName + ".publication_urls");
|
||||
jdbcTemplate.execute("COMPUTE STATS " + testDatabaseName + ".publication_urls");
|
||||
|
||||
// Create VIEWs of the original data. We just READ from it, so it's safe for our testing environment..
|
||||
|
||||
jdbcTemplate.execute("CREATE VIEW IF NOT EXISTS " + testDatabaseName + ".publication as select * from " + initialDatabaseName + ".publication");
|
||||
// TODO - Should we add the "publication_oids" table? Will it be used in the "getAssignments" query?
|
||||
|
||||
jdbcTemplate.execute("CREATE VIEW IF NOT EXISTS " + testDatabaseName + ".publication_pids as select * from " + initialDatabaseName + ".publication_pids");
|
||||
|
||||
jdbcTemplate.execute("CREATE VIEW IF NOT EXISTS " + testDatabaseName + ".publication_urls as select * from " + initialDatabaseName + ".publication_urls");
|
||||
jdbcTemplate.execute("CREATE TABLE IF NOT EXISTS " + testDatabaseName + ".datasource stored as parquet as select * from " + initialDatabaseName + ".datasource");
|
||||
jdbcTemplate.execute("COMPUTE STATS " + testDatabaseName + ".datasource");
|
||||
|
||||
jdbcTemplate.execute("CREATE VIEW IF NOT EXISTS " + testDatabaseName + ".publication_boost as select * from " + initialDatabaseName + ".publication_boost");
|
||||
|
||||
jdbcTemplate.execute("CREATE VIEW IF NOT EXISTS " + testDatabaseName + ".datasource as select * from " + initialDatabaseName + ".datasource");
|
||||
|
||||
jdbcTemplate.execute("CREATE VIEW IF NOT EXISTS " + testDatabaseName + ".payload_legacy as select * from " + initialDatabaseName + ".payload_legacy");
|
||||
jdbcTemplate.execute("CREATE TABLE IF NOT EXISTS " + testDatabaseName + ".payload_legacy stored as parquet as select * from " + initialDatabaseName + ".payload_legacy");
|
||||
jdbcTemplate.execute("COMPUTE STATS " + testDatabaseName + ".payload_legacy");
|
||||
|
||||
databaseName = testDatabaseName;
|
||||
} else {
|
||||
|
@ -94,7 +89,7 @@ public class DatabaseConnector {
|
|||
// Drop the "current_assignment" table. It is a temporary table which is created on-demand during execution.
|
||||
jdbcTemplate.execute("DROP TABLE IF EXISTS " + databaseName + ".current_assignment PURGE");
|
||||
|
||||
jdbcTemplate.execute("CREATE TABLE IF NOT EXISTS " + databaseName + ".assignment (id string, original_url string, workerid string, assignments_batch_counter bigint, `date` bigint) stored as parquet");
|
||||
jdbcTemplate.execute("CREATE TABLE IF NOT EXISTS " + databaseName + ".assignment (id string, original_url string, workerid string, `date` bigint) stored as parquet");
|
||||
jdbcTemplate.execute("COMPUTE STATS " + databaseName + ".assignment");
|
||||
|
||||
jdbcTemplate.execute("CREATE TABLE IF NOT EXISTS " + databaseName + ".attempt (id string, original_url string, `date` bigint, status string, error_class string, error_message string) stored as parquet");
|
||||
|
@ -113,21 +108,9 @@ public class DatabaseConnector {
|
|||
jdbcTemplate.execute("CREATE TABLE IF NOT EXISTS " + databaseName + ".payload_bulk_import (id string, original_url string, actual_url string, `date` bigint, mimetype string, size string, `hash` string, `location` string, provenance string) stored as parquet");
|
||||
jdbcTemplate.execute("COMPUTE STATS " + databaseName + ".payload_bulk_import");
|
||||
|
||||
|
||||
try { // Metastore takes some time to recognize the tables have been created, in order to use them in the view.
|
||||
Thread.sleep(1000);
|
||||
} catch (InterruptedException ignore) {}
|
||||
|
||||
jdbcTemplate.update("INVALIDATE METADATA");
|
||||
|
||||
try { // Metastore takes some time to recognize the tables have been created, in order to use them in the view.
|
||||
Thread.sleep(1000);
|
||||
} catch (InterruptedException ignore) {}
|
||||
|
||||
|
||||
jdbcTemplate.execute("CREATE VIEW IF NOT EXISTS " + databaseName + ".payload\n" +
|
||||
"AS SELECT * from " + databaseName + ".payload_legacy\n" +
|
||||
"UNION ALL SELECT * FROM " + databaseName +".payload_aggregated\n" +
|
||||
jdbcTemplate.execute("CREATE VIEW IF NOT EXISTS " + databaseName + ".payload " +
|
||||
"AS SELECT * from " + databaseName + ".payload_legacy " +
|
||||
"UNION ALL SELECT * FROM " + databaseName +".payload_aggregated " +
|
||||
"UNION ALL SELECT * FROM " + databaseName + ".payload_bulk_import");
|
||||
// We do not do the "compute stats" for the view, since we get the following error: "COMPUTE STATS not supported for view: pdfaggregationdatabase_payloads_view.payload".
|
||||
|
|
@ -1,12 +1,7 @@
|
|||
package eu.openaire.urls_controller.controllers;
|
||||
|
||||
import com.google.gson.Gson;
|
||||
import com.google.gson.GsonBuilder;
|
||||
import com.google.gson.JsonParseException;
|
||||
import com.google.gson.JsonParser;
|
||||
import eu.openaire.urls_controller.components.BulkImport;
|
||||
import eu.openaire.urls_controller.models.BulkImportReport;
|
||||
import eu.openaire.urls_controller.models.BulkImportResponse;
|
||||
import eu.openaire.urls_controller.services.BulkImportService;
|
||||
import eu.openaire.urls_controller.util.FileUtils;
|
||||
import eu.openaire.urls_controller.util.GenericUtils;
|
||||
|
@ -14,10 +9,11 @@ import org.slf4j.Logger;
|
|||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.http.HttpStatus;
|
||||
import org.springframework.http.MediaType;
|
||||
import org.springframework.http.ResponseEntity;
|
||||
import org.springframework.web.bind.MissingServletRequestParameterException;
|
||||
import org.springframework.web.bind.annotation.*;
|
||||
import org.springframework.web.bind.annotation.GetMapping;
|
||||
import org.springframework.web.bind.annotation.RequestMapping;
|
||||
import org.springframework.web.bind.annotation.RequestParam;
|
||||
import org.springframework.web.bind.annotation.RestController;
|
||||
|
||||
import java.io.BufferedReader;
|
||||
import java.io.File;
|
||||
|
@ -27,9 +23,6 @@ import java.util.Collections;
|
|||
import java.util.HashMap;
|
||||
import java.util.Set;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
import java.util.concurrent.ExecutorService;
|
||||
import java.util.concurrent.Executors;
|
||||
import java.util.concurrent.RejectedExecutionException;
|
||||
import java.util.regex.Matcher;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
|
@ -50,49 +43,31 @@ public class BulkImportController {
|
|||
|
||||
private final HashMap<String, BulkImport.BulkImportSource> bulkImportSources;
|
||||
|
||||
public static final Set<String> bulkImportDirsUnderProcessing = Collections.newSetFromMap(new ConcurrentHashMap<String, Boolean>());
|
||||
public static final Set<String> bulkImportDirs = Collections.newSetFromMap(new ConcurrentHashMap<String, Boolean>());
|
||||
|
||||
public static int numOfThreadsForBulkImportProcedures;
|
||||
public static ExecutorService bulkImportExecutor;
|
||||
|
||||
|
||||
public BulkImportController(BulkImportService bulkImportService, BulkImport bulkImport)
|
||||
{
|
||||
String bulkImportReportLocationTemp;
|
||||
String bulkImportReportLocation1;
|
||||
this.baseBulkImportLocation = bulkImport.getBaseBulkImportLocation();
|
||||
|
||||
this.bulkImportSources = new HashMap<>(bulkImport.getBulkImportSources());
|
||||
|
||||
bulkImportReportLocationTemp = bulkImport.getBulkImportReportLocation();
|
||||
if ( !bulkImportReportLocationTemp.endsWith("/") )
|
||||
bulkImportReportLocationTemp += "/";
|
||||
this.bulkImportReportLocation = bulkImportReportLocationTemp;
|
||||
bulkImportReportLocation1 = bulkImport.getBulkImportReportLocation();
|
||||
if ( !bulkImportReportLocation1.endsWith("/") )
|
||||
bulkImportReportLocation1 += "/";
|
||||
this.bulkImportReportLocation = bulkImportReportLocation1;
|
||||
|
||||
this.bulkImportService = bulkImportService;
|
||||
|
||||
numOfThreadsForBulkImportProcedures = bulkImport.getNumOfThreadsForBulkImportProcedures();
|
||||
if ( numOfThreadsForBulkImportProcedures <= 0 )
|
||||
throw new RuntimeException("The given \"numOfThreadsForBulkImportProcedures\" is not a positive number: " + numOfThreadsForBulkImportProcedures);
|
||||
|
||||
logger.info("Will use " + numOfThreadsForBulkImportProcedures + " threads for the bulk-import procedures.");
|
||||
bulkImportExecutor = Executors.newFixedThreadPool(numOfThreadsForBulkImportProcedures); // At most < numOfThreadsForBulkImportProcedures > threads will be used per bulk-import procedure..
|
||||
}
|
||||
|
||||
|
||||
private static final Pattern LAST_DIR_REGEX = Pattern.compile("^.*/([^/]+/)$");
|
||||
private static final Pattern LAST_DIR_REGEX = Pattern.compile("^.*/([^/]+[/]?)$");
|
||||
|
||||
@GetMapping("bulkImportFullTexts")
|
||||
public ResponseEntity<?> bulkImportFullTexts(@RequestParam String provenance, @RequestParam String bulkImportDir, @RequestParam boolean shouldDeleteFilesOnFinish) {
|
||||
|
||||
// This method shows the parameters which are missing when dealing with the bulk-import API.
|
||||
// Spring Boot does not show any specific messages to the user (like stacktraces), for security reasons.
|
||||
@ExceptionHandler(MissingServletRequestParameterException.class)
|
||||
public ResponseEntity<?> handleMissingParams(MissingServletRequestParameterException ex) {
|
||||
return ResponseEntity.badRequest().body(String.format("Missing parameter: %s\n", ex.getParameterName()));
|
||||
}
|
||||
|
||||
|
||||
@PostMapping("bulkImportFullTexts")
|
||||
public ResponseEntity<?> bulkImportFullTexts(@RequestParam String provenance, @RequestParam String bulkImportDir, @RequestParam boolean shouldDeleteFilesOnFinish)
|
||||
{
|
||||
BulkImport.BulkImportSource bulkImportSource = bulkImportSources.get(provenance);
|
||||
if ( bulkImportSource == null ) {
|
||||
String errorMsg = "The provided provenance \"" + provenance + "\" is not in the list of the bulk-imported sources, so no configuration-rules are available!";
|
||||
|
@ -107,7 +82,7 @@ public class BulkImportController {
|
|||
return ResponseEntity.badRequest().body(errorMsg);
|
||||
}
|
||||
|
||||
String givenBulkDir = bulkImportDir; // Keep the given value here, to not expose the full-path, in case the user has not provided an absolute path.
|
||||
String givenBulkDir = bulkImportDir; // Keep the given value here, to not expose the full-path, in case the user has not provided an absolut path.
|
||||
|
||||
// Make sure the whole path ends with "/", so that we can easily append file-names later.
|
||||
if ( !bulkImportDir.endsWith("/") )
|
||||
|
@ -135,7 +110,6 @@ public class BulkImportController {
|
|||
}
|
||||
|
||||
// The "relativeBulkImportDir" should always be guaranteed to end with "/"! Otherwise, the import-procedure will fail.
|
||||
|
||||
logger.info("Received a \"bulkImportFullTexts\" request for \"" + provenance + "\" procedure and bulkImportDir: \"" + bulkImportDir + "\".");
|
||||
|
||||
// Check whether the given directory is accessible.
|
||||
|
@ -154,94 +128,66 @@ public class BulkImportController {
|
|||
return ResponseEntity.badRequest().body(errorMsg);
|
||||
}
|
||||
// The above check does not catch the case were the directory has at least one subdirectory, but no full-texts files.
|
||||
// The "iterator()" will have a "next" entry, but no full-text file will exist. Although, that case will be rare and will be caught later on, after this procedure has been accepted.
|
||||
// The "iterator()" will have a "next" entry, but no full-text file will exist. Although, that case will be rare.
|
||||
} catch (Exception e) {
|
||||
String errorMsg = "Error when checking if the givenDir \"" + givenDir + "\" is empty!";
|
||||
logger.error(errorMsg);
|
||||
return ResponseEntity.internalServerError().body(errorMsg);
|
||||
}
|
||||
|
||||
// After applying all the logic-checks and informing the user of any mistake, then we check and inform if the Service is about to shut down.
|
||||
if ( ShutdownController.shouldShutdownService ) {
|
||||
String warnMsg = "The Service is about to shutdown, after all under-processing assignments and/or bulkImport requests are handled. No new requests are accepted!";
|
||||
logger.warn(warnMsg);
|
||||
return ResponseEntity.status(HttpStatus.CONFLICT).body(warnMsg);
|
||||
}
|
||||
|
||||
// Detect if the same directory is scheduled for being processed. In that case, return a 429.
|
||||
if ( ! bulkImportDirsUnderProcessing.add(bulkImportDir) ) {
|
||||
if ( ! bulkImportDirs.add(bulkImportDir) ) {
|
||||
// We allow multiple jobs for the same provenance, running at the same time, but not multiple jobs for the same bulkImportDirectory.
|
||||
String errorMsg = "There is a bulk-import request for the directory \"" + bulkImportDir + "\" that is being handled at the moment. Please wait until it's finished being processed, before making another request.";
|
||||
logger.error(errorMsg);
|
||||
return ResponseEntity.status(HttpStatus.TOO_MANY_REQUESTS).body(errorMsg);
|
||||
}
|
||||
|
||||
Path currentBulkImportReportLocationDir = Paths.get(this.bulkImportReportLocation, provenance, relativeBulkImportDir);
|
||||
Path currentBulkImportReportLocationDir = Paths.get(this.bulkImportReportLocation, provenance);
|
||||
try {
|
||||
Files.createDirectories(currentBulkImportReportLocationDir); // No-op if dir exists. It does not throw a "alreadyExistsException"
|
||||
} catch (Exception e) {
|
||||
String errorMsg = "Could not create the \"bulkImportReportLocation\" for provenance \"" + provenance + "\" : " + currentBulkImportReportLocationDir;
|
||||
String errorMsg = "Could nor create the \"bulkImportReportLocation\" for provenance \"" + provenance + "\" : " + currentBulkImportReportLocationDir;
|
||||
logger.error(errorMsg, e);
|
||||
bulkImportDirsUnderProcessing.remove(bulkImportDir);
|
||||
return ResponseEntity.internalServerError().body(errorMsg);
|
||||
}
|
||||
|
||||
// Generate the "bulkImportReportID". We are removing the ending "slash" ("/") from the "relativeBulkImportDir".
|
||||
String bulkImportReportID = provenance + "/" + relativeBulkImportDir + "report_" + GenericUtils.getRandomNumber(10000, 99999);
|
||||
String bulkImportReportID = provenance + "/" + relativeBulkImportDir.substring(0, (relativeBulkImportDir.length() -1)) + "_report_" + GenericUtils.getRandomNumber(10000, 99999);
|
||||
String bulkImportReportFullPath = this.bulkImportReportLocation + bulkImportReportID + ".json";
|
||||
|
||||
String msg = "The bulkImportFullTexts request for " + provenance + " procedure and bulkImportDir: " + givenBulkDir + " was accepted and will be submitted for execution. "
|
||||
String msg = "The 'bulkImportFullTexts' request for '" + provenance + "' procedure and bulkImportDir: '" + givenBulkDir + "' was accepted and will be scheduled for execution. "
|
||||
+ (shouldDeleteFilesOnFinish ? "The successfully imported files will be deleted." : "All files will remain inside the directory after processing.")
|
||||
+ " You can request a report at any moment, using the reportID.";
|
||||
+ " You can request a report at any moment, using this reportFileID: " + bulkImportReportID;
|
||||
|
||||
BulkImportReport bulkImportReport = new BulkImportReport(provenance, bulkImportReportFullPath, bulkImportReportID);
|
||||
bulkImportReport.addEvent(msg);
|
||||
|
||||
String errorMsg = fileUtils.writeToFile(bulkImportReportFullPath, bulkImportReport.getJsonReport(), true);
|
||||
if ( errorMsg != null ) {
|
||||
bulkImportDirsUnderProcessing.remove(bulkImportDir);
|
||||
if ( errorMsg != null )
|
||||
return ResponseEntity.internalServerError().body(errorMsg);
|
||||
}
|
||||
|
||||
logger.info(msg + " \"bulkImportReportID\": " + bulkImportReportID);
|
||||
logger.info(msg);
|
||||
|
||||
// Add this to a background job, since it will take a lot of time to be completed, and the caller will get a "read-timeout" at least and a socket-timeout at most (in case of a network failure during those hours).
|
||||
String finalBulkImportDir = bulkImportDir;
|
||||
String finalRelativeBulkImportDir = relativeBulkImportDir;
|
||||
try {
|
||||
UrlsController.futuresOfBackgroundTasks.add(
|
||||
UrlsController.backgroundExecutor.submit(
|
||||
() -> bulkImportService.bulkImportFullTextsFromDirectory(bulkImportReport, finalRelativeBulkImportDir, finalBulkImportDir, givenDir, provenance, bulkImportSource, shouldDeleteFilesOnFinish)
|
||||
)
|
||||
);
|
||||
} catch (RejectedExecutionException ree) {
|
||||
errorMsg = "The bulkImport request for bulkImportReportLocation \"" + bulkImportReportLocation + "\" and provenance " + provenance + " has failed to be executed!";
|
||||
bulkImportReport.addEvent(msg);
|
||||
logger.error(errorMsg, ree);
|
||||
return ResponseEntity.internalServerError().body(errorMsg);
|
||||
}
|
||||
UrlsController.backgroundCallableTasks.add(() ->
|
||||
bulkImportService.bulkImportFullTextsFromDirectory(bulkImportReport, finalRelativeBulkImportDir, finalBulkImportDir, givenDir, provenance, bulkImportSource, shouldDeleteFilesOnFinish)
|
||||
);
|
||||
|
||||
// This directory, will be removed from "bulkImportDirsUnderProcessing", when the background job finishes.
|
||||
return ResponseEntity.ok().body(new BulkImportResponse(msg, bulkImportReportID)); // The response is automatically serialized to json, and it has the type "application/json".
|
||||
return ResponseEntity.ok().body(msg);
|
||||
}
|
||||
|
||||
|
||||
@GetMapping(value = "getBulkImportReport", produces = MediaType.APPLICATION_JSON_VALUE)
|
||||
public ResponseEntity<?> getBulkImportReport(@RequestParam("id") String bulkImportReportId, @RequestParam(name = "pretty", defaultValue = "false") boolean prettyFormatting)
|
||||
@GetMapping("getBulkImportReport")
|
||||
public ResponseEntity<?> getBulkImportReport(@RequestParam("id") String bulkImportReportId)
|
||||
{
|
||||
logger.info("Received a \"getBulkImportReport\" request for \"bulkImportReportId\": \"" + bulkImportReportId + "\"." + (prettyFormatting ? " Will return the report pretty-formatted." : ""));
|
||||
|
||||
// Even if the Service is set to shut down soon, we allow this endpoint to return the report up to the last minute,
|
||||
// since the Service may be up for another hour, running a bulk-import procedure, for which we want to check its progress.
|
||||
|
||||
// Write the contents of the report-file to a string (efficiently!) and return the whole content as an HTTP-response.
|
||||
final StringBuilder stringBuilder = new StringBuilder(25_000);
|
||||
StringBuilder stringBuilder = new StringBuilder(2_000);
|
||||
String line;
|
||||
|
||||
FileUtils.fileAccessLock.lock();
|
||||
try ( BufferedReader in = new BufferedReader(new InputStreamReader(Files.newInputStream(Paths.get(this.bulkImportReportLocation, bulkImportReportId + ".json"))), FileUtils.twentyFiveKb) ) {
|
||||
try ( BufferedReader in = new BufferedReader(new InputStreamReader(Files.newInputStream(Paths.get(this.bulkImportReportLocation, bulkImportReportId + ".json"))), FileUtils.tenMb) ) {
|
||||
while ( (line = in.readLine()) != null )
|
||||
stringBuilder.append(line).append(GenericUtils.endOfLine); // The "readLine()" does not return the line-term char.
|
||||
stringBuilder.append(line).append("\n"); // The "readLine()" does not return the line-term char.
|
||||
} catch (NoSuchFileException nsfe) {
|
||||
logger.warn("The requested report-file with ID: \"" + bulkImportReportId + "\" was not found!");
|
||||
return ResponseEntity.notFound().build();
|
||||
|
@ -249,20 +195,9 @@ public class BulkImportController {
|
|||
String errorMsg = "Failed to read the contents of report-file with ID: " + bulkImportReportId;
|
||||
logger.error(errorMsg, e);
|
||||
return ResponseEntity.internalServerError().body(errorMsg); // It's ok to give the file-path to the user, since the report already contains the file-path.
|
||||
} finally {
|
||||
FileUtils.fileAccessLock.unlock();
|
||||
}
|
||||
|
||||
String json = stringBuilder.toString().trim();
|
||||
if ( prettyFormatting ) {
|
||||
final Gson gson = new GsonBuilder().setPrettyPrinting().create();
|
||||
try {
|
||||
json = gson.toJson(JsonParser.parseString(json));
|
||||
} catch (JsonParseException jpe) {
|
||||
logger.error("Problem when parsing the json-string: " + jpe.getMessage() + "\nIt is not a valid json!\n" + json);
|
||||
}
|
||||
}
|
||||
return ResponseEntity.ok().body(json);
|
||||
return ResponseEntity.ok().body(stringBuilder.toString());
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
@ -2,7 +2,6 @@ package eu.openaire.urls_controller.controllers;
|
|||
|
||||
import eu.openaire.urls_controller.models.WorkerInfo;
|
||||
import eu.openaire.urls_controller.services.ShutdownService;
|
||||
import eu.openaire.urls_controller.util.GenericUtils;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
|
@ -26,140 +25,81 @@ public class ShutdownController {
|
|||
ShutdownService shutdownService;
|
||||
|
||||
public static boolean shouldShutdownService = false;
|
||||
public static boolean shouldShutdownAllWorkers = false;
|
||||
|
||||
|
||||
@PostMapping("shutdownService")
|
||||
public ResponseEntity<?> shutdownServiceGracefully(HttpServletRequest request)
|
||||
{
|
||||
String initMsg = "Received a \"shutdownService\" request ";
|
||||
String remoteAddr = GenericUtils.getRequestorAddress(request);
|
||||
initMsg += "from [" + remoteAddr + "]. ";
|
||||
ResponseEntity<?> responseEntity = shutdownService.passSecurityChecks(remoteAddr, initMsg);
|
||||
String initMsg = "Received a \"shutdownService\" request. ";
|
||||
ResponseEntity<?> responseEntity = shutdownService.passSecurityChecks(request, initMsg);
|
||||
if ( responseEntity != null )
|
||||
return responseEntity;
|
||||
|
||||
String endingMsg;
|
||||
if ( shouldShutdownService ) {
|
||||
endingMsg = "The controller has already received a \"shutdownService\" request (which was not canceled afterwards).";
|
||||
logger.info(initMsg + endingMsg);
|
||||
} else {
|
||||
String finalMsg = "";
|
||||
if ( shouldShutdownService )
|
||||
finalMsg = "The controller has already received a \"shutdownService\" (which was not canceled afterwards).";
|
||||
else {
|
||||
shouldShutdownService = true;
|
||||
endingMsg = "The service will shutdown, after finishing current work.";
|
||||
logger.info(initMsg + endingMsg);
|
||||
|
||||
shutdownService.postShutdownOrCancelRequestsToAllWorkers(false);
|
||||
// Send "shutdownWorker" requests to all Workers.
|
||||
for ( String workerId : UrlsController.workersInfoMap.keySet() ) {
|
||||
if ( ! UrlsController.workersInfoMap.get(workerId).getHasShutdown() )
|
||||
shutdownService.postShutdownOrCancelRequestToWorker(workerId, UrlsController.workersInfoMap.get(workerId).getWorkerIP(), false);
|
||||
}
|
||||
|
||||
// That's it for now. The workers may take some hours to finish their work (including delivering the full-text files).
|
||||
// A scheduler monitors the shutdown of the workers. Once all worker have shutdown, the Controller shuts down as well.
|
||||
}
|
||||
|
||||
return ResponseEntity.ok().body(endingMsg + GenericUtils.endOfLine);
|
||||
finalMsg += "The service will shutdown, after finishing current work.";
|
||||
logger.info(initMsg + finalMsg);
|
||||
return ResponseEntity.ok().body(finalMsg + "\n");
|
||||
}
|
||||
|
||||
|
||||
@PostMapping("cancelShutdownService")
|
||||
public ResponseEntity<?> cancelShutdownServiceGracefully(HttpServletRequest request)
|
||||
{
|
||||
String initMsg = "Received a \"cancelShutdownService\" request ";
|
||||
String remoteAddr = GenericUtils.getRequestorAddress(request);
|
||||
initMsg += "from [" + remoteAddr + "]. ";
|
||||
ResponseEntity<?> responseEntity = shutdownService.passSecurityChecks(remoteAddr, initMsg);
|
||||
String initMsg = "Received a \"cancelShutdownService\" request. ";
|
||||
ResponseEntity<?> responseEntity = shutdownService.passSecurityChecks(request, initMsg);
|
||||
if ( responseEntity != null )
|
||||
return responseEntity;
|
||||
|
||||
shouldShutdownService = false;
|
||||
String endingMsg = "Any previous \"shutdownService\"-request is canceled.";
|
||||
logger.info(initMsg + endingMsg);
|
||||
// Send "cancelShutdownWorker" requests to all Workers.
|
||||
for ( String workerId : UrlsController.workersInfoMap.keySet() )
|
||||
shutdownService.postShutdownOrCancelRequestToWorker(workerId, UrlsController.workersInfoMap.get(workerId).getWorkerIP(), true);
|
||||
|
||||
// Cancel the shutdown of the workers, if we are able to catch up with them before they have already shutdown..
|
||||
shutdownService.postShutdownOrCancelRequestsToAllWorkers(true);
|
||||
|
||||
return ResponseEntity.ok().body(endingMsg + GenericUtils.endOfLine);
|
||||
}
|
||||
|
||||
|
||||
// The "shutdownAllWorkers" and a "cancelShutdownAllWorkers" endpoints help when updating only the workers,
|
||||
// while keeping the Controller running and accepting bulk-import requests.
|
||||
|
||||
@PostMapping("shutdownAllWorkers")
|
||||
public ResponseEntity<?> shutdownAllWorkersGracefully(HttpServletRequest request)
|
||||
{
|
||||
String initMsg = "Received a \"shutdownAllWorkers\" request ";
|
||||
String remoteAddr = GenericUtils.getRequestorAddress(request);
|
||||
initMsg += "from [" + remoteAddr + "]. ";
|
||||
ResponseEntity<?> responseEntity = shutdownService.passSecurityChecks(remoteAddr, initMsg);
|
||||
if ( responseEntity != null )
|
||||
return responseEntity;
|
||||
|
||||
String endingMsg;
|
||||
if ( shouldShutdownAllWorkers ) {
|
||||
endingMsg = "The controller has already received a \"shutdownAllWorkers\" request (which was not canceled afterwards).";
|
||||
logger.info(initMsg + endingMsg);
|
||||
} else {
|
||||
shouldShutdownAllWorkers = true;
|
||||
endingMsg = "All workers will shutdown, after finishing current work.";
|
||||
logger.info(initMsg + endingMsg);
|
||||
|
||||
shutdownService.postShutdownOrCancelRequestsToAllWorkers(false);
|
||||
|
||||
// That's it for now. The workers may take some hours to finish their work (including delivering the full-text files).
|
||||
// The service will continue to run and handle bulk-import requests.
|
||||
// Once the workers are ready to work again, they can be started without any additional configuration.
|
||||
}
|
||||
|
||||
return ResponseEntity.ok().body(endingMsg + GenericUtils.endOfLine);
|
||||
}
|
||||
|
||||
|
||||
@PostMapping("cancelShutdownAllWorkers")
|
||||
public ResponseEntity<?> cancelShutdownAllWorkersGracefully(HttpServletRequest request)
|
||||
{
|
||||
String initMsg = "Received a \"cancelShutdownAllWorkers\" request ";
|
||||
String remoteAddr = GenericUtils.getRequestorAddress(request);
|
||||
initMsg += "from [" + remoteAddr + "]. ";
|
||||
ResponseEntity<?> responseEntity = shutdownService.passSecurityChecks(remoteAddr, initMsg);
|
||||
if ( responseEntity != null )
|
||||
return responseEntity;
|
||||
|
||||
shouldShutdownAllWorkers = false;
|
||||
String endingMsg = "Any previous \"shutdownAllWorkers\"-request is canceled.";
|
||||
logger.info(initMsg + endingMsg);
|
||||
|
||||
// Cancel the shutdown of the workers, if we are able to catch up with them before they have already shutdown..
|
||||
shutdownService.postShutdownOrCancelRequestsToAllWorkers(true);
|
||||
|
||||
return ResponseEntity.ok().body(endingMsg + GenericUtils.endOfLine);
|
||||
String finalMsg = "Any previous \"shutdownService\"-request is canceled.";
|
||||
logger.info(initMsg + finalMsg);
|
||||
return ResponseEntity.ok().body(finalMsg + "\n");
|
||||
}
|
||||
|
||||
|
||||
@PostMapping("workerShutdownReport")
|
||||
public ResponseEntity<?> workerShutdownReport(@RequestParam String workerId, HttpServletRequest request)
|
||||
{
|
||||
String remoteAddr = GenericUtils.getRequestorAddress(request);
|
||||
String initMsg = "Received a \"workerShutdownReport\" from worker: \"" + workerId + "\" [IP: " + remoteAddr + "].";
|
||||
String initMsg = "Received a \"workerShutdownReport\" from worker: \"" + workerId + "\".";
|
||||
WorkerInfo workerInfo = UrlsController.workersInfoMap.get(workerId);
|
||||
if ( workerInfo == null ) {
|
||||
String errorMsg = "The worker with id \"" + workerId + "\" has not participated in the PDF-Aggregation-Service!";
|
||||
logger.warn(initMsg + GenericUtils.endOfLine + errorMsg);
|
||||
String errorMsg = "The worker with id \"" + workerId + "\" has not participated in the PDF-Aggregation-Service";
|
||||
logger.warn(initMsg + "\n" + errorMsg);
|
||||
return ResponseEntity.badRequest().body(errorMsg);
|
||||
}
|
||||
|
||||
String remoteAddr = request.getHeader("X-FORWARDED-FOR");
|
||||
if ( remoteAddr == null || remoteAddr.isEmpty() )
|
||||
remoteAddr = request.getRemoteAddr();
|
||||
|
||||
if ( ! remoteAddr.equals(workerInfo.getWorkerIP()) ) {
|
||||
logger.error(initMsg + " The request came from an IP different from the one this worker was registered with: " + workerInfo.getWorkerIP());
|
||||
logger.error(initMsg + " The request came from another IP: " + remoteAddr + " | while this worker was registered with the IP: " + workerInfo.getWorkerIP());
|
||||
return ResponseEntity.status(HttpStatus.FORBIDDEN).build();
|
||||
}
|
||||
|
||||
logger.info(initMsg);
|
||||
|
||||
workerInfo.setHasShutdown(true); // This will update the map.
|
||||
int numActiveWorkers = UrlsController.numOfActiveWorkers.decrementAndGet();
|
||||
if ( (numActiveWorkers == 0) && shouldShutdownAllWorkers ) { // If all workers have shutdown and the "shouldShutdownAllWorkers" was set, then reset the indicator to allow for future shutdowns.
|
||||
logger.info("All workers have shutdown.");
|
||||
shouldShutdownAllWorkers = false; // Make sure we can request that all the workers will shut-down again, when the user starts a couple of workers afterwards and sometime in the future he wants to shut them down.
|
||||
}
|
||||
|
||||
// Return "HTTP-OK" to this worker. If this was part of a shutdown-service request, then wait for the scheduler to check and shutdown the service.
|
||||
// Return "HTTP-OK" to this worker and wait for the scheduler to check and shutdown the service.
|
||||
return ResponseEntity.ok().build();
|
||||
}
|
||||
|
||||
|
|
|
@ -1,16 +1,16 @@
|
|||
package eu.openaire.urls_controller.controllers;
|
||||
|
||||
|
||||
import eu.openaire.urls_controller.configuration.DatabaseConnector;
|
||||
import eu.openaire.urls_controller.configuration.ImpalaConnector;
|
||||
import eu.openaire.urls_controller.services.StatsService;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.http.MediaType;
|
||||
import org.springframework.http.ResponseEntity;
|
||||
import org.springframework.web.bind.MissingServletRequestParameterException;
|
||||
import org.springframework.web.bind.annotation.*;
|
||||
|
||||
import org.springframework.web.bind.annotation.GetMapping;
|
||||
import org.springframework.web.bind.annotation.RequestMapping;
|
||||
import org.springframework.web.bind.annotation.RequestParam;
|
||||
import org.springframework.web.bind.annotation.RestController;
|
||||
|
||||
/**
|
||||
* This controller returns statistics for the database.
|
||||
|
@ -25,110 +25,51 @@ public class StatsController {
|
|||
private StatsService statsService;
|
||||
|
||||
|
||||
// This method shows the parameters which are missing when dealing with the bulk-import API.
|
||||
// Spring Boot does not show any specific messages to the user (like stacktraces), for security reasons.
|
||||
@ExceptionHandler(MissingServletRequestParameterException.class)
|
||||
public ResponseEntity<?> handleMissingParams(MissingServletRequestParameterException ex) {
|
||||
return ResponseEntity.badRequest().body(String.format("Missing parameter: %s\n", ex.getParameterName()));
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* This endpoint returns the total number of payloads existing in the database, independently of the way they were aggregated.
|
||||
* This includes the payloads created by other pieces of software, before the PDF-Aggregation-Service was created.
|
||||
* */
|
||||
@GetMapping(value = "getNumberOfAllPayloads", produces = MediaType.TEXT_PLAIN_VALUE)
|
||||
public ResponseEntity<?> getNumberOfAllPayloads(boolean isCalledFromScheduler)
|
||||
{
|
||||
if ( ! isCalledFromScheduler )
|
||||
logger.info("Received a \"getNumberOfAllPayloads\" request.");
|
||||
|
||||
final String getAllPayloadsNumberQuery = "select count(id) from " + DatabaseConnector.databaseName + ".payload";
|
||||
return statsService.getNumberOfPayloads(getAllPayloadsNumberQuery, "all payloads", 0);
|
||||
@GetMapping("getNumberOfAllPayloads")
|
||||
public ResponseEntity<?> getNumberOfAllPayloads() {
|
||||
logger.info("Received a \"getNumberOfAllPayloads\" request.");
|
||||
final String getPayloadsNumberQuery = "select count(id) from " + ImpalaConnector.databaseName + ".payload";
|
||||
return statsService.getNumberOfPayloads(getPayloadsNumberQuery, "payloads");
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* This endpoint returns the number of payloads aggregated by the PDF-Aggregated-Service itself, through crawling.
|
||||
* This endpoint returns the number of payloads aggregated by the PDF-Aggregated-Service itself.
|
||||
* It excludes the payloads aggregated by other methods, by applying a Date-filter for the records created in 2021 or later.
|
||||
* */
|
||||
@GetMapping(value = "getNumberOfPayloadsAggregatedByServiceThroughCrawling", produces = MediaType.TEXT_PLAIN_VALUE)
|
||||
public ResponseEntity<?> getNumberOfPayloadsAggregatedByServiceThroughCrawling(boolean isCalledFromScheduler)
|
||||
{
|
||||
if ( ! isCalledFromScheduler )
|
||||
logger.info("Received a \"getNumberOfPayloadsAggregatedByServiceThroughCrawling\" request.");
|
||||
|
||||
String getNumOfPayloadsAggregatedByServiceThroughCrawlingQuery = "select count(id) from " + DatabaseConnector.databaseName + ".payload_aggregated";
|
||||
return statsService.getNumberOfPayloads(getNumOfPayloadsAggregatedByServiceThroughCrawlingQuery, "payloads aggregated by the Service through crawling", 0);
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* This endpoint returns the number of payloads aggregated by this Service, through BulkImport procedures with compatible datasources..
|
||||
* */
|
||||
@GetMapping(value = "getNumberOfPayloadsAggregatedByServiceThroughBulkImport", produces = MediaType.TEXT_PLAIN_VALUE)
|
||||
public ResponseEntity<?> getNumberOfPayloadsAggregatedByServiceThroughBulkImport(boolean isCalledFromScheduler)
|
||||
{
|
||||
if ( ! isCalledFromScheduler )
|
||||
logger.info("Received a \"getNumberOfPayloadsAggregatedByServiceThroughBulkImport\" request.");
|
||||
|
||||
String getNumOfPayloadsAggregatedByServiceThroughBulkImportQuery = "select count(id) from " + DatabaseConnector.databaseName + ".payload_bulk_import";
|
||||
return statsService.getNumberOfPayloads(getNumOfPayloadsAggregatedByServiceThroughBulkImportQuery, "payloads aggregated by the Service through BulkImport procedures", 0);
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* This endpoint returns the number of payloads aggregated by the PDF-Aggregated-Service itself, through crawling AND bulk-import procedures.
|
||||
* */
|
||||
@GetMapping(value = "getNumberOfPayloadsAggregatedByService", produces = MediaType.TEXT_PLAIN_VALUE)
|
||||
public ResponseEntity<?> getNumberOfPayloadsAggregatedByService(boolean isCalledFromScheduler)
|
||||
{
|
||||
if ( ! isCalledFromScheduler )
|
||||
logger.info("Received a \"getNumberOfPayloadsAggregatedByService\" request.");
|
||||
|
||||
String getNumOfPayloadsAggregatedByServiceQuery = "select count(id) from\n" +
|
||||
" (select id from " + DatabaseConnector.databaseName + ".payload_aggregated\n" +
|
||||
" union all\n" +
|
||||
" select id from " + DatabaseConnector.databaseName + ".payload_bulk_import)\n" +
|
||||
" as payloads_from_service";
|
||||
return statsService.getNumberOfPayloads(getNumOfPayloadsAggregatedByServiceQuery, "payloads aggregated by the Service, through both crawling and bulk-import procedures", 0);
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* This endpoint returns the number of legacy payloads, which were aggregated by methods other thant the PDF Aggregation Service.
|
||||
* */
|
||||
@GetMapping(value = "getNumberOfLegacyPayloads", produces = MediaType.TEXT_PLAIN_VALUE)
|
||||
public ResponseEntity<?> getNumberOfLegacyPayloads(boolean isCalledFromScheduler)
|
||||
{
|
||||
if ( ! isCalledFromScheduler )
|
||||
logger.info("Received a \"getNumberOfLegacyPayloads\" request.");
|
||||
|
||||
String getNumOfLegacyPayloadsQuery = "select count(id) from " + DatabaseConnector.databaseName + ".payload_legacy";
|
||||
return statsService.getNumberOfPayloads(getNumOfLegacyPayloadsQuery, "legacy payloads", 0);
|
||||
@GetMapping("getNumberOfPayloadsAggregatedByService")
|
||||
public ResponseEntity<?> getNumberOfPayloadsAggregatedByService() {
|
||||
logger.info("Received a \"getNumberOfPayloadsAggregatedByService\" request.");
|
||||
String getPayloadsAggregatedQuery = "select count(id) from " + ImpalaConnector.databaseName + ".payload where `date` >= cast(cast('2021-01-01' as timestamp) as bigint)";
|
||||
return statsService.getNumberOfPayloads(getPayloadsAggregatedQuery, "payloads retrieved by the PDF Aggregation Service");
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* This endpoint returns the number of payloads related to the given datasourceID.
|
||||
* */
|
||||
@GetMapping(value = "getNumberOfPayloadsForDatasource", produces = MediaType.TEXT_PLAIN_VALUE)
|
||||
@GetMapping("getNumberOfPayloadsForDatasource")
|
||||
public ResponseEntity<?> getNumberOfPayloadsForDatasource(@RequestParam String datasourceId) {
|
||||
logger.info("Received a \"getNumberOfPayloadsForDatasource\" request.");
|
||||
final String getNumOfPayloadsForDatasourceQuery =
|
||||
"select count(p.id) from " + DatabaseConnector.databaseName + ".payload p\n" +
|
||||
" join " + DatabaseConnector.databaseName + ".publication pu on pu.id=p.id and pu.datasourceid=\"" + datasourceId + "\"";
|
||||
final String getPayloadsNumberForDatasourceQuery =
|
||||
"select count(p.id) from " + ImpalaConnector.databaseName + ".payload p\n" +
|
||||
" join " + ImpalaConnector.databaseName + ".publication pu on pu.id=p.id and pu.datasourceid=\"" + datasourceId + "\"";
|
||||
|
||||
if ( logger.isTraceEnabled() )
|
||||
logger.trace("getNumOfPayloadsForDatasourceQuery:\n" + getNumOfPayloadsForDatasourceQuery);
|
||||
logger.trace("getPayloadsNumberForDatasourceQuery:\n" + getPayloadsNumberForDatasourceQuery);
|
||||
|
||||
return statsService.getNumberOfPayloads(getNumOfPayloadsForDatasourceQuery, "payloads related to datasourceId \"" + datasourceId + "\"", 0);
|
||||
return statsService.getNumberOfPayloads(getPayloadsNumberForDatasourceQuery, "payloads related to datasourceId \"" + datasourceId + "\"");
|
||||
}
|
||||
|
||||
|
||||
|
||||
// TODO - Add an endpoint to get the publication year as a param and return the number of payloads for the publications of that year.
|
||||
// select count(p.id) from payload p
|
||||
// join publication pu on pu.id=p.id and pu.year=<GIVEN_YEAR>
|
||||
// join publication pu on pu.id=p.id and pu.pub_year=<GIVEN_YEAR>
|
||||
|
||||
|
||||
|
||||
|
@ -137,41 +78,32 @@ public class StatsController {
|
|||
// In case the "numTopDatasources" param is not given or is less or equal to 0, then no limit will be added to the query.
|
||||
/*
|
||||
select d.id, d.name, d.type, d.allow_harvest, count(p.id) as payload_count from datasource d
|
||||
join publication pu on pu.datasourceid=d.id -- We want the datasources with at least 1 publication.
|
||||
left join payload p on p.id=pu.id -- We want the datasources with 0 payloads too, so we use "left join".
|
||||
join publication pu on pu.datasourceid=d.id
|
||||
left join payload p on p.id=pu.id -- We want the datasources with 0 payloads too.
|
||||
group by d.id, d.name, d.type, d.allow_harvest
|
||||
order by payload_count desc
|
||||
*/
|
||||
|
||||
|
||||
|
||||
// TODO - Add an endpoint to return the number of payloads found for each publication-year, in descending order..
|
||||
// For example the number of payloads for publications published in 2016 is <number>
|
||||
// --//-- the number for 2017 is <number>
|
||||
// Add a "limit" parameter for the user to specify that wants only the last 5 years (2019-2023).
|
||||
|
||||
|
||||
/**
|
||||
* This endpoint returns the total number of distinct full-text files existing in the database.
|
||||
* */
|
||||
@GetMapping(value = "getNumberOfAllDistinctFullTexts", produces = MediaType.TEXT_PLAIN_VALUE)
|
||||
@GetMapping("getNumberOfAllDistinctFullTexts")
|
||||
public ResponseEntity<?> getNumberOfAllDistinctFullTexts() {
|
||||
logger.info("Received a \"getNumberOfAllDistinctFullTexts\" request.");
|
||||
final String getPayloadsNumberQuery = "select count(distinct `hash`) from " + DatabaseConnector.databaseName + ".payload";
|
||||
return statsService.getNumberOfPayloads(getPayloadsNumberQuery, "distinct full-text files", 0);
|
||||
final String getPayloadsNumberQuery = "select count(distinct `hash`) from " + ImpalaConnector.databaseName + ".payload";
|
||||
return statsService.getNumberOfPayloads(getPayloadsNumberQuery, "distinct full-text files");
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* This endpoint returns the number of records inspected by the PDF-Aggregation-Service, through crawling.
|
||||
* This endpoint returns the number of records inspected by the PDF-Aggregation-Service.
|
||||
* */
|
||||
@GetMapping(value = "getNumberOfRecordsInspectedByServiceThroughCrawling", produces = MediaType.TEXT_PLAIN_VALUE)
|
||||
public ResponseEntity<?> getNumberOfRecordsInspectedByServiceThroughCrawling(boolean isCalledFromScheduler)
|
||||
@GetMapping("getNumberOfRecordsInspected")
|
||||
public ResponseEntity<?> getNumberOfRecordsInspected()
|
||||
{
|
||||
if ( ! isCalledFromScheduler )
|
||||
logger.info("Received a \"getNumberOfRecordsInspectedByServiceThroughCrawling\" request.");
|
||||
|
||||
return statsService.getNumberOfRecordsInspectedByServiceThroughCrawling(0);
|
||||
return statsService.getNumberOfRecordsInspected();
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
@ -1,7 +1,7 @@
|
|||
package eu.openaire.urls_controller.controllers;
|
||||
|
||||
import com.google.common.collect.HashMultimap;
|
||||
import eu.openaire.urls_controller.configuration.DatabaseConnector;
|
||||
import eu.openaire.urls_controller.configuration.ImpalaConnector;
|
||||
import eu.openaire.urls_controller.models.Assignment;
|
||||
import eu.openaire.urls_controller.models.Datasource;
|
||||
import eu.openaire.urls_controller.payloads.responces.AssignmentsResponse;
|
||||
|
@ -15,7 +15,10 @@ import org.springframework.beans.factory.annotation.Value;
|
|||
import org.springframework.http.HttpStatus;
|
||||
import org.springframework.http.ResponseEntity;
|
||||
import org.springframework.jdbc.core.JdbcTemplate;
|
||||
import org.springframework.web.bind.annotation.*;
|
||||
import org.springframework.web.bind.annotation.GetMapping;
|
||||
import org.springframework.web.bind.annotation.RequestMapping;
|
||||
import org.springframework.web.bind.annotation.RequestParam;
|
||||
import org.springframework.web.bind.annotation.RestController;
|
||||
|
||||
import java.io.File;
|
||||
import java.sql.Timestamp;
|
||||
|
@ -56,7 +59,6 @@ public class TestController {
|
|||
boolean isFirstRun = true;
|
||||
boolean assignmentsLimitReached = false;
|
||||
Timestamp timestamp = new Timestamp(System.currentTimeMillis()); // Store it here, in order to have the same for all current records.
|
||||
long curAssignmentsBatchCounter = assignmentsBatchCounter.incrementAndGet();
|
||||
|
||||
// Start loading urls.
|
||||
while ( true ) {
|
||||
|
@ -76,7 +78,7 @@ public class TestController {
|
|||
}
|
||||
|
||||
int randomNum = GenericUtils.getRandomNumber(1, 5);
|
||||
assignments.add(new Assignment(pair.getKey(), pair.getValue(), new Datasource("ID_" + randomNum, "NAME_" + randomNum), workerId, curAssignmentsBatchCounter, timestamp));
|
||||
assignments.add(new Assignment(pair.getKey(), pair.getValue(), new Datasource("ID_" + randomNum, "NAME_" + randomNum), workerId, timestamp));
|
||||
}// end pairs-for-loop
|
||||
|
||||
if ( assignmentsLimitReached ) {
|
||||
|
@ -89,6 +91,7 @@ public class TestController {
|
|||
if ( scanner != null ) // Check if the initial value is null.
|
||||
scanner.close();
|
||||
|
||||
long curAssignmentsBatchCounter = assignmentsBatchCounter.incrementAndGet();
|
||||
logger.info("Sending batch_" + curAssignmentsBatchCounter + " with " + assignments.size() + " assignments (" + testFileUtils.duplicateIdUrlEntries.get() + " more assignments were discarded as duplicates), to worker with ID: " + workerId);
|
||||
return ResponseEntity.status(HttpStatus.OK).header("Content-Type", "application/json").body(new AssignmentsResponse(curAssignmentsBatchCounter, assignments));
|
||||
}
|
||||
|
@ -97,7 +100,7 @@ public class TestController {
|
|||
@GetMapping("get10PublicationIdsTest")
|
||||
public ResponseEntity<?> get10PublicationIdsTest() {
|
||||
|
||||
String query = "SELECT id FROM " + DatabaseConnector.databaseName + ".publication LIMIT 10;";
|
||||
String query = "SELECT id FROM " + ImpalaConnector.databaseName + ".publication LIMIT 10;";
|
||||
|
||||
try {
|
||||
List<String> publications = jdbcTemplate.queryForList(query, String.class);
|
||||
|
@ -110,7 +113,7 @@ public class TestController {
|
|||
}
|
||||
|
||||
|
||||
@PostMapping("uploadParquetFile")
|
||||
@GetMapping("parquet_upload")
|
||||
public ResponseEntity<?> uploadParquetFile() {
|
||||
|
||||
logger.debug("We got a \"parquet_upload\" request.");
|
||||
|
|
|
@ -5,7 +5,7 @@ import eu.openaire.urls_controller.models.WorkerInfo;
|
|||
import eu.openaire.urls_controller.payloads.requests.WorkerReport;
|
||||
import eu.openaire.urls_controller.services.UrlsService;
|
||||
import eu.openaire.urls_controller.util.FileUtils;
|
||||
import eu.openaire.urls_controller.util.GenericUtils;
|
||||
import eu.openaire.urls_controller.util.ParquetFileUtils;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
|
@ -21,8 +21,10 @@ import java.nio.file.Paths;
|
|||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
import java.util.List;
|
||||
import java.util.concurrent.*;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
import java.util.concurrent.Callable;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
import java.util.concurrent.ExecutorService;
|
||||
import java.util.concurrent.Executors;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
|
||||
|
@ -38,6 +40,9 @@ public class UrlsController {
|
|||
@Autowired
|
||||
private FileUtils fileUtils;
|
||||
|
||||
@Autowired
|
||||
private ParquetFileUtils parquetFileUtils;
|
||||
|
||||
private static final Pattern MALICIOUS_INPUT_STRING = Pattern.compile(".*[';`\"]+.*");
|
||||
|
||||
@Value("${services.pdfaggregation.controller.assignmentLimit}")
|
||||
|
@ -45,28 +50,20 @@ public class UrlsController {
|
|||
|
||||
public static final ConcurrentHashMap<String, WorkerInfo> workersInfoMap = new ConcurrentHashMap<>(6);
|
||||
|
||||
public static final AtomicInteger numOfActiveWorkers = new AtomicInteger(0);
|
||||
|
||||
public static ExecutorService backgroundExecutor;
|
||||
public static final ExecutorService backgroundExecutor = Executors.newFixedThreadPool(4); // At most 4 threads will be used.
|
||||
|
||||
public static final List<Future<Boolean>> futuresOfBackgroundTasks = Collections.synchronizedList(new ArrayList<>());
|
||||
public static final List<Callable<Boolean>> backgroundCallableTasks = Collections.synchronizedList(new ArrayList<>());
|
||||
|
||||
private final String workerReportsDirPath;
|
||||
|
||||
|
||||
public UrlsController(@Value("${services.pdfaggregation.controller.workerReportsDirPath}") String workerReportsDirPath,
|
||||
@Value("${services.pdfaggregation.controller.numOfBackgroundThreads}") int numOfBackgroundThreads)
|
||||
public UrlsController(@Value("${services.pdfaggregation.controller.workerReportsDirPath}") String workerReportsDirPath)
|
||||
{
|
||||
if ( !workerReportsDirPath.endsWith("/") )
|
||||
workerReportsDirPath += "/";
|
||||
|
||||
this.workerReportsDirPath = workerReportsDirPath; // This dir will be created later.
|
||||
|
||||
if ( numOfBackgroundThreads <= 0 )
|
||||
throw new RuntimeException("The given \"numOfBackgroundThreads\" is not a positive number: " + numOfBackgroundThreads);
|
||||
|
||||
logger.info("Will use " + numOfBackgroundThreads + " threads for background tasks, such as processing worker-reports or bulk-import procedures.");
|
||||
backgroundExecutor = Executors.newFixedThreadPool(numOfBackgroundThreads); // At most < numOfBackgroundThreads > tasks will be running in parallel.
|
||||
}
|
||||
|
||||
|
||||
|
@ -80,7 +77,7 @@ public class UrlsController {
|
|||
return ResponseEntity.status(HttpStatus.FORBIDDEN).body(errorMsg);
|
||||
}
|
||||
|
||||
logger.info("Worker with id: \"" + workerId + "\", requested up to " + workerAssignmentsLimit + " assignments. The assignments-limit of the controller is: " + assignmentLimit);
|
||||
logger.info("Worker with id: \"" + workerId + "\", requested " + workerAssignmentsLimit + " assignments. The assignments-limit of the controller is: " + assignmentLimit);
|
||||
|
||||
// Sanitize the "assignmentsLimit". Do not let an overload happen in the Controller's or the Impala's server.
|
||||
int assignmentsLimit = workerAssignmentsLimit;
|
||||
|
@ -95,19 +92,20 @@ public class UrlsController {
|
|||
|
||||
if ( ShutdownController.shouldShutdownService ) {
|
||||
// There might be the case that the Controller has not sent shutDown requests to the Workers yet, or it has, BUT:
|
||||
// 1) A worker requests for new assignments, before it can handle the shutDown request given to it.
|
||||
// 1) A worker requests for new assignments before the shutDown request in handled by its side.
|
||||
// 2) A new Worker joins the Service (unexpected, but anyway).
|
||||
String warnMsg = "The Service is about to shutdown, after all under-processing assignments and/or bulkImport requests are handled. No new requests are accepted!";
|
||||
String warnMsg = "The Service is about to shutdown, after all under-processing assignments are handled. No new requests are accepted!";
|
||||
logger.warn(warnMsg); // It's likely not an actual error, but still it's not accepted.
|
||||
return ResponseEntity.status(HttpStatus.CONFLICT).body(warnMsg); // The worker will wait 15 mins and upon going to retry it will notice that it should not do a new request then or it may have already shutdown in the meantime.
|
||||
return ResponseEntity.status(HttpStatus.CONFLICT).body(warnMsg); // The worker will wait 15 mins and upon going to retry it will notice that it should not do a new request or it may have shutdown in the meantime.
|
||||
}
|
||||
// Do not apply any check for the "ShutdownController.shouldShutdownAllWorkers", since then we have to also make sure it is set to false after all workers have been shutdown, in order for updated workers to ba able to request assignments after they are started again..
|
||||
|
||||
if ( request == null ) {
|
||||
logger.error("The \"HttpServletRequest\" is null!");
|
||||
return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR).build();
|
||||
}
|
||||
String remoteAddr = GenericUtils.getRequestorAddress(request);
|
||||
String remoteAddr = request.getHeader("X-FORWARDED-FOR");
|
||||
if ( (remoteAddr == null) || remoteAddr.isEmpty() )
|
||||
remoteAddr = request.getRemoteAddr();
|
||||
|
||||
WorkerInfo workerInfo = workersInfoMap.get(workerId);
|
||||
if ( workerInfo != null ) { // This worker has already been identified.
|
||||
|
@ -115,17 +113,15 @@ public class UrlsController {
|
|||
if ( !savedWorkerIp.equals(remoteAddr) ) {
|
||||
logger.warn("The worker with id \"" + workerId + "\" has changed IP from \"" + savedWorkerIp + "\" to \"" + remoteAddr + "\".");
|
||||
workerInfo.setWorkerIP(remoteAddr); // Set the new IP. The update will be reflected in the map.
|
||||
} // In this case, the worker may have previously informed the Controller it has shutdown or it may have crashed.
|
||||
} // In this case, the worker may has previously informed the Controller it has shutdown or it may have crashed.
|
||||
|
||||
if ( workerInfo.getHasShutdown() ) {
|
||||
logger.info("The worker with id \"" + workerId + "\" was restarted.");
|
||||
workerInfo.setHasShutdown(false);
|
||||
numOfActiveWorkers.incrementAndGet();
|
||||
}
|
||||
} else {
|
||||
logger.info("The worker \"" + workerId + "\" is requesting assignments for the first time. Going to store its IP [" + remoteAddr + "] in memory.");
|
||||
logger.info("The worker \"" + workerId + "\" is requesting assignments for the first time. Going to store its IP [" + remoteAddr + "].");
|
||||
workersInfoMap.put(workerId, new WorkerInfo(remoteAddr, false));
|
||||
numOfActiveWorkers.incrementAndGet();
|
||||
}
|
||||
|
||||
return urlsService.getAssignments(workerId, assignmentsLimit);
|
||||
|
@ -155,23 +151,23 @@ public class UrlsController {
|
|||
return ResponseEntity.status(HttpStatus.FORBIDDEN).body(errorMsg);
|
||||
}
|
||||
|
||||
int sizeOfUrlReports = 0;
|
||||
int sizeOUrlReports = 0;
|
||||
List<UrlReport> urlReports = workerReport.getUrlReports();
|
||||
if ( (urlReports == null) || ((sizeOfUrlReports = urlReports.size()) == 0) ) {
|
||||
if ( (urlReports == null) || ((sizeOUrlReports = urlReports.size()) == 0) ) {
|
||||
String errorMsg = "The given \"WorkerReport\" from worker with ID \"" + curWorkerId + "\" was empty (without any UrlReports)!";
|
||||
logger.error(errorMsg);
|
||||
return ResponseEntity.status(HttpStatus.BAD_REQUEST).body(errorMsg);
|
||||
}
|
||||
|
||||
long curReportAssignmentsCounter = workerReport.getAssignmentRequestCounter();
|
||||
logger.info("Received the WorkerReport for batch-assignments_" + curReportAssignmentsCounter + ", from the worker with id: " + curWorkerId + ". It contains " + sizeOfUrlReports + " urlReports. Going to request the fullTexts from the Worker and insert the UrlReports into the database.");
|
||||
logger.info("Received the WorkerReport for batch-assignments_" + curReportAssignmentsCounter + ", from the worker with id: " + curWorkerId + ". It contains " + sizeOUrlReports + " urlReports. Going to request the fullTexts from the Worker and insert the UrlReports into the database.");
|
||||
|
||||
// Make sure this worker's report directory is created.
|
||||
Path currentWorkerReportLocationDir = Paths.get(this.workerReportsDirPath, curWorkerId);
|
||||
try {
|
||||
Files.createDirectories(currentWorkerReportLocationDir); // No-op if dir exists. It does not throw a "alreadyExistsException"
|
||||
} catch (Exception e) {
|
||||
String errorMsg = "Could not create the \"currentWorkerReportLocationDir\" for worker \"" + curWorkerId + "\" : " + currentWorkerReportLocationDir;
|
||||
String errorMsg = "Could nor create the \"currentWorkerReportLocationDir\" for worker \"" + curWorkerId + "\" : " + currentWorkerReportLocationDir;
|
||||
logger.error(errorMsg, e);
|
||||
return ResponseEntity.internalServerError().body(errorMsg);
|
||||
}
|
||||
|
@ -183,20 +179,12 @@ public class UrlsController {
|
|||
fileUtils.writeToFile(workerReportFile, workerReport.getJsonReport(), false); // Only one thread is writing to this specific file.
|
||||
// The above method will overwrite a possibly existing file. So in case of a crash, it's better to back up the reports before starting the Controller again (as the assignments-counter will start over, from 0).
|
||||
|
||||
int finalSizeOUrlReports = sizeOfUrlReports;
|
||||
try {
|
||||
UrlsController.futuresOfBackgroundTasks.add(
|
||||
backgroundExecutor.submit(
|
||||
() -> urlsService.addWorkerReport(curWorkerId, curReportAssignmentsCounter, urlReports, finalSizeOUrlReports)
|
||||
)
|
||||
);
|
||||
} catch (RejectedExecutionException ree) {
|
||||
String errorMsg = "The WorkerReport from worker \"" + curWorkerId + "\" and assignments_" + curReportAssignmentsCounter + " have failed to be executed!";
|
||||
logger.error(errorMsg, ree);
|
||||
return ResponseEntity.internalServerError().body(errorMsg);
|
||||
}
|
||||
int finalSizeOUrlReports = sizeOUrlReports;
|
||||
UrlsController.backgroundCallableTasks.add(() ->
|
||||
urlsService.addWorkerReport(curWorkerId, curReportAssignmentsCounter, urlReports, finalSizeOUrlReports)
|
||||
);
|
||||
|
||||
String msg = "The 'addWorkerReport' request for worker with id: '" + curWorkerId + "' and assignments_" + curReportAssignmentsCounter + " , was accepted and submitted for execution.";
|
||||
String msg = "The 'addWorkerReport' request for worker with id: '" + curWorkerId + "' and assignments_" + curReportAssignmentsCounter + " , was accepted and will be scheduled for execution.";
|
||||
logger.info(msg);
|
||||
return ResponseEntity.ok().body(msg);
|
||||
}
|
||||
|
|
|
@ -13,7 +13,6 @@ import java.sql.Timestamp;
|
|||
"original_url",
|
||||
"datasource",
|
||||
"workerId",
|
||||
"assignments_batch_counter",
|
||||
"timestamp"
|
||||
})
|
||||
public class Assignment {
|
||||
|
@ -30,22 +29,17 @@ public class Assignment {
|
|||
@JsonProperty("workerid")
|
||||
private String workerId;
|
||||
|
||||
@JsonProperty("assignments_batch_counter")
|
||||
private long assignmentsBatchCounter;
|
||||
|
||||
@JsonProperty("timestamp")
|
||||
private Timestamp timestamp;
|
||||
|
||||
|
||||
public Assignment() {}
|
||||
|
||||
|
||||
public Assignment(String id, String originalUrl, Datasource datasource, String workerId, long assignmentsBatchCounter, Timestamp timestamp) {
|
||||
public Assignment(String id, String originalUrl, Datasource datasource, String workerId, Timestamp timestamp) {
|
||||
this.id = id;
|
||||
this.originalUrl = originalUrl;
|
||||
this.datasource = datasource;
|
||||
this.workerId = workerId;
|
||||
this.assignmentsBatchCounter = assignmentsBatchCounter;
|
||||
this.timestamp = timestamp;
|
||||
}
|
||||
|
||||
|
@ -82,14 +76,6 @@ public class Assignment {
|
|||
this.workerId = workerId;
|
||||
}
|
||||
|
||||
public long getAssignmentsBatchCounter() {
|
||||
return assignmentsBatchCounter;
|
||||
}
|
||||
|
||||
public void setAssignmentsBatchCounter(long assignmentsBatchCounter) {
|
||||
this.assignmentsBatchCounter = assignmentsBatchCounter;
|
||||
}
|
||||
|
||||
public Timestamp getTimestamp() {
|
||||
return timestamp;
|
||||
}
|
||||
|
@ -105,9 +91,7 @@ public class Assignment {
|
|||
", originalUrl='" + originalUrl + '\'' +
|
||||
", datasource=" + datasource +
|
||||
", workerId='" + workerId + '\'' +
|
||||
", assignmentsBatchCounter=" + assignmentsBatchCounter +
|
||||
", timestamp=" + timestamp +
|
||||
'}';
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
@ -9,10 +9,7 @@ import com.google.gson.Gson;
|
|||
import eu.openaire.urls_controller.util.GenericUtils;
|
||||
|
||||
import java.util.Collection;
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.locks.Lock;
|
||||
import java.util.concurrent.locks.ReentrantLock;
|
||||
|
||||
|
||||
@JsonInclude(JsonInclude.Include.NON_NULL)
|
||||
|
@ -37,8 +34,6 @@ public class BulkImportReport {
|
|||
@JsonProperty
|
||||
private Map<String, Collection<String>> eventsMap;
|
||||
|
||||
transient private final Lock reportLock = new ReentrantLock(true);
|
||||
|
||||
|
||||
public BulkImportReport(String provenance, String reportLocation, String reportID) {
|
||||
this.provenance = provenance;
|
||||
|
@ -48,20 +43,14 @@ public class BulkImportReport {
|
|||
|
||||
|
||||
public void addEvent(String event) {
|
||||
eventsMultimap.put(GenericUtils.getReadableCurrentTimeAndZone(), event); // This is synchronized.
|
||||
eventsMultimap.put(GenericUtils.getReadableCurrentTimeAndZone(), event);
|
||||
}
|
||||
|
||||
/**
|
||||
* Synchronize it with a lock, to avoid concurrency issues when concurrent calls are made to the same bulkImport-Report object.
|
||||
* */
|
||||
public String getJsonReport()
|
||||
{
|
||||
reportLock.lock();
|
||||
//Convert the LinkedHashMultiMap<String, String> to Map<String, Collection<String>>, since Gson cannot serialize Multimaps.
|
||||
eventsMap = new HashMap<>(eventsMultimap.asMap()); // Make sure we use a clone of the original data, in order to avoid any exception in the "gson.toJson()" method, when at the same time another thread modifies the "eventsMultimap".
|
||||
String reportToReturn = gson.toJson(this, BulkImportReport.class);
|
||||
reportLock.unlock();
|
||||
return reportToReturn;
|
||||
eventsMap = eventsMultimap.asMap();
|
||||
return gson.toJson(this);
|
||||
}
|
||||
|
||||
public String getProvenance() {
|
||||
|
|
|
@ -1,51 +0,0 @@
|
|||
package eu.openaire.urls_controller.models;
|
||||
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonInclude;
|
||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||
import com.fasterxml.jackson.annotation.JsonPropertyOrder;
|
||||
|
||||
@JsonInclude(JsonInclude.Include.NON_NULL)
|
||||
@JsonPropertyOrder({
|
||||
"message",
|
||||
"reportID"
|
||||
})
|
||||
public class BulkImportResponse {
|
||||
|
||||
@JsonProperty("message")
|
||||
String message;
|
||||
|
||||
@JsonProperty("reportID")
|
||||
String reportID;
|
||||
|
||||
public BulkImportResponse() {}
|
||||
|
||||
public BulkImportResponse(String message, String bulkImportReportID) {
|
||||
this.message = message;
|
||||
this.reportID = bulkImportReportID;
|
||||
}
|
||||
|
||||
public String getMessage() {
|
||||
return message;
|
||||
}
|
||||
|
||||
public void setMessage(String message) {
|
||||
this.message = message;
|
||||
}
|
||||
|
||||
public String getReportID() {
|
||||
return reportID;
|
||||
}
|
||||
|
||||
public void setReportID(String reportID) {
|
||||
this.reportID = reportID;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return "BulkImportResponse{" +
|
||||
"message='" + message + '\'' +
|
||||
", reportID='" + reportID + '\'' +
|
||||
'}';
|
||||
}
|
||||
}
|
|
@ -18,9 +18,9 @@ public class FileLocationData {
|
|||
|
||||
public FileLocationData(String fileLocation) throws RuntimeException {
|
||||
// Extract and set LocationData.
|
||||
Matcher matcher = FileUtils.FILEPATH_ID_EXTENSION.matcher(fileLocation);
|
||||
Matcher matcher = FileUtils.FILENAME_ID_EXTENSION.matcher(fileLocation);
|
||||
if ( !matcher.matches() )
|
||||
throw new RuntimeException("Failed to match the \"" + fileLocation + "\" with the regex: " + FileUtils.FILEPATH_ID_EXTENSION);
|
||||
throw new RuntimeException("Failed to match the \"" + fileLocation + "\" with the regex: " + FileUtils.FILENAME_ID_EXTENSION);
|
||||
fileDir = matcher.group(1);
|
||||
if ( (fileDir == null) || fileDir.isEmpty() )
|
||||
throw new RuntimeException("Failed to extract the \"fileDir\" from \"" + fileLocation + "\".");
|
||||
|
|
|
@ -68,14 +68,6 @@ public class Payload {
|
|||
this.datasourceId = datasourceId;
|
||||
}
|
||||
|
||||
|
||||
public Payload clone()
|
||||
{
|
||||
// Return a new object with the same values. Clone whatever objects are not immutable.!
|
||||
return new Payload(this.id, this.original_url, this.actual_url, (Timestamp) this.timestamp_acquired.clone(), this.mime_type, this.size, this.hash,
|
||||
this.location, this.provenance, this.datasourceId);
|
||||
}
|
||||
|
||||
public String getId() {
|
||||
return id;
|
||||
}
|
||||
|
|
|
@ -13,6 +13,6 @@ public interface BulkImportService {
|
|||
|
||||
List<String> getFileLocationsInsideDir(String directory);
|
||||
|
||||
String getMD5Hash(String string);
|
||||
String getMD5hash(String string);
|
||||
|
||||
}
|
||||
|
|
|
@ -3,7 +3,7 @@ package eu.openaire.urls_controller.services;
|
|||
|
||||
import com.google.common.collect.Lists;
|
||||
import eu.openaire.urls_controller.components.BulkImport;
|
||||
import eu.openaire.urls_controller.configuration.DatabaseConnector;
|
||||
import eu.openaire.urls_controller.configuration.ImpalaConnector;
|
||||
import eu.openaire.urls_controller.controllers.BulkImportController;
|
||||
import eu.openaire.urls_controller.models.BulkImportReport;
|
||||
import eu.openaire.urls_controller.models.DocFileData;
|
||||
|
@ -15,21 +15,21 @@ import org.apache.avro.generic.GenericData;
|
|||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.dao.EmptyResultDataAccessException;
|
||||
import org.springframework.jdbc.core.JdbcTemplate;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import javax.xml.bind.DatatypeConverter;
|
||||
import java.io.File;
|
||||
import java.net.ConnectException;
|
||||
import java.net.UnknownHostException;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.nio.file.Paths;
|
||||
import java.security.MessageDigest;
|
||||
import java.util.*;
|
||||
import java.util.concurrent.Callable;
|
||||
import java.util.concurrent.CancellationException;
|
||||
import java.util.concurrent.ExecutionException;
|
||||
import java.util.concurrent.Future;
|
||||
import java.sql.Types;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.concurrent.*;
|
||||
import java.util.stream.Collectors;
|
||||
import java.util.stream.Stream;
|
||||
|
||||
|
@ -39,12 +39,21 @@ public class BulkImportServiceImpl implements BulkImportService {
|
|||
|
||||
private static final Logger logger = LoggerFactory.getLogger(BulkImportServiceImpl.class);
|
||||
|
||||
|
||||
@Autowired
|
||||
private FileUtils fileUtils;
|
||||
|
||||
|
||||
@Autowired
|
||||
private ParquetFileUtils parquetFileUtils;
|
||||
|
||||
@Autowired
|
||||
private JdbcTemplate jdbcTemplate;
|
||||
|
||||
|
||||
private static final int numOfBulkImportThreads = 4;
|
||||
public static final ExecutorService bulkImportExecutor = Executors.newFixedThreadPool(numOfBulkImportThreads); // At most 4 threads will be used.
|
||||
|
||||
|
||||
/**
|
||||
* Given a directory with full-text-files, this method imports the full-texts files in the PDF Aggregation Service.
|
||||
|
@ -55,30 +64,24 @@ public class BulkImportServiceImpl implements BulkImportService {
|
|||
String bulkImportReportLocation = bulkImportReport.getReportLocation();
|
||||
|
||||
// Write to bulkImport-report file.
|
||||
String msg = "Initializing the bulkImport " + provenance + " procedure with bulkImportDir: " + bulkImportDirName + ".";
|
||||
logger.info(msg);
|
||||
bulkImportReport.addEvent(msg);
|
||||
bulkImportReport.addEvent("Initializing the bulkImport '" + provenance + "' procedure with bulkImportDir '" + bulkImportDirName + "'.");
|
||||
// Do not write immediately to the file, wait for the following checks.
|
||||
|
||||
String additionalLoggingMsg = " | provenance: \"" + provenance + "\" | bulkImportDir: \"" + bulkImportDirName + "\"";
|
||||
|
||||
if ( (ParquetFileUtils.payloadsSchema == null) // Parse the schema if it's not already parsed.
|
||||
&& ((ParquetFileUtils.payloadsSchema = ParquetFileUtils.parseSchema(ParquetFileUtils.payloadSchemaFilePath)) == null ) ) {
|
||||
String errorMsg = "The payloadsSchema could not be parsed!";
|
||||
logger.error(errorMsg + additionalLoggingMsg);
|
||||
String errorMsg = "The 'payloadsSchema' could not be parsed!";
|
||||
logger.error(errorMsg);
|
||||
bulkImportReport.addEvent(errorMsg);
|
||||
fileUtils.writeToFile(bulkImportReportLocation, bulkImportReport.getJsonReport(), false);
|
||||
BulkImportController.bulkImportDirsUnderProcessing.remove(bulkImportDirName);
|
||||
fileUtils.writeToFile(bulkImportReportLocation, bulkImportReport.getJsonReport(), true);
|
||||
BulkImportController.bulkImportDirs.remove(bulkImportDirName);
|
||||
return false;
|
||||
}
|
||||
|
||||
List<String> fileLocations = getFileLocationsInsideDir(bulkImportDirName); // the error-msg has already been written
|
||||
List<String> fileLocations = getFileLocationsInsideDir(bulkImportDirName);
|
||||
if ( fileLocations == null ) {
|
||||
String errorMsg = "Could not retrieve the files for bulk-import!";
|
||||
logger.error(errorMsg + additionalLoggingMsg);
|
||||
bulkImportReport.addEvent(errorMsg);
|
||||
fileUtils.writeToFile(bulkImportReportLocation, bulkImportReport.getJsonReport(), false);
|
||||
BulkImportController.bulkImportDirsUnderProcessing.remove(bulkImportDirName);
|
||||
bulkImportReport.addEvent("Could not retrieve the files for bulk-import!");
|
||||
fileUtils.writeToFile(bulkImportReportLocation, bulkImportReport.getJsonReport(), true);
|
||||
BulkImportController.bulkImportDirs.remove(bulkImportDirName);
|
||||
return false;
|
||||
}
|
||||
|
||||
|
@ -87,98 +90,81 @@ public class BulkImportServiceImpl implements BulkImportService {
|
|||
String errorMsg = "No files were found inside the bulkImportDir: " + bulkImportDirName;
|
||||
logger.warn(errorMsg);
|
||||
bulkImportReport.addEvent(errorMsg);
|
||||
fileUtils.writeToFile(bulkImportReportLocation, bulkImportReport.getJsonReport(), false);
|
||||
BulkImportController.bulkImportDirsUnderProcessing.remove(bulkImportDirName);
|
||||
fileUtils.writeToFile(bulkImportReportLocation, bulkImportReport.getJsonReport(), true);
|
||||
BulkImportController.bulkImportDirs.remove(bulkImportDirName);
|
||||
return false;
|
||||
}
|
||||
|
||||
if ( logger.isTraceEnabled() )
|
||||
logger.trace("fileLocations: " + additionalLoggingMsg + GenericUtils.endOfLine + fileLocations);
|
||||
logger.trace("fileLocations:\n" + fileLocations);
|
||||
|
||||
String localParquetDir = parquetFileUtils.parquetBaseLocalDirectoryPath + "bulk_import_" + provenance + File.separator + relativeBulkImportDir; // This ends with "/".
|
||||
try {
|
||||
Files.createDirectories(Paths.get(localParquetDir)); // No-op if it already exists.
|
||||
} catch (Exception e) {
|
||||
String errorMsg = "Could not create the local parquet-directory: " + localParquetDir;
|
||||
logger.error(errorMsg + additionalLoggingMsg, e);
|
||||
logger.error(errorMsg, e);
|
||||
bulkImportReport.addEvent(errorMsg);
|
||||
fileUtils.writeToFile(bulkImportReportLocation, bulkImportReport.getJsonReport(), false);
|
||||
BulkImportController.bulkImportDirsUnderProcessing.remove(bulkImportDirName);
|
||||
fileUtils.writeToFile(bulkImportReportLocation, bulkImportReport.getJsonReport(), true);
|
||||
BulkImportController.bulkImportDirs.remove(bulkImportDirName);
|
||||
return false;
|
||||
}
|
||||
|
||||
// Create a new directory on HDFS, with this bulkImportDir name. So, that there will not be any "load data" operation to fail because another thread has loaded that base-dir right before.
|
||||
String currentBulkImportHdfsDir = parquetFileUtils.parquetHDFSDirectoryPathPayloadsBulkImport + relativeBulkImportDir;
|
||||
if ( ! parquetFileUtils.applyHDFSOperation(parquetFileUtils.webHDFSBaseUrl + currentBulkImportHdfsDir + parquetFileUtils.mkDirsAndParams) ) { // N0-op if it already exists. It is very quick.
|
||||
String errorMsg = "Could not create the remote HDFS-directory: " + currentBulkImportHdfsDir;
|
||||
logger.error(errorMsg + additionalLoggingMsg);
|
||||
if ( ! parquetFileUtils.applyHDFOperation(parquetFileUtils.webHDFSBaseUrl + currentBulkImportHdfsDir + parquetFileUtils.mkDirsAndParams) ) { // N0-op if it already exists. It is very quick.
|
||||
String errorMsg = "Could not create the hdfs-directory: " + currentBulkImportHdfsDir;
|
||||
logger.error(errorMsg);
|
||||
bulkImportReport.addEvent(errorMsg);
|
||||
fileUtils.writeToFile(bulkImportReportLocation, bulkImportReport.getJsonReport(), false);
|
||||
BulkImportController.bulkImportDirsUnderProcessing.remove(bulkImportDirName);
|
||||
fileUtils.writeToFile(bulkImportReportLocation, bulkImportReport.getJsonReport(), true);
|
||||
BulkImportController.bulkImportDirs.remove(bulkImportDirName);
|
||||
return false;
|
||||
}
|
||||
|
||||
long timeMillis = System.currentTimeMillis(); // Store it here, in order to have the same for all current records.
|
||||
|
||||
List<Callable<Integer>> callableTasksForFileSegments = new ArrayList<>(numOfFiles);
|
||||
int sizeOfEachSegment = (numOfFiles / BulkImportController.numOfThreadsForBulkImportProcedures);
|
||||
if ( sizeOfEachSegment == 0 ) // In case the numFiles are fewer than the numOfThreads to handle them.
|
||||
sizeOfEachSegment = 1;
|
||||
List<List<String>> subLists = Lists.partition(fileLocations, sizeOfEachSegment); // Divide the initial list to "numOfThreadsForBulkImportProcedures" subLists. The last one may have marginally fewer files.
|
||||
List<Callable<Integer>> callables = new ArrayList<>(numOfFiles);
|
||||
List<List<String>> subLists = Lists.partition(fileLocations, numOfBulkImportThreads); // Divide the initial list to "numOfBulkImportThreads" subLists. The last one may have marginally fewer files.
|
||||
int subListsSize = subLists.size();
|
||||
|
||||
msg = "Going to bulk-import the " + numOfFiles + " files in parallel, after dividing them in " + subListsSize + " segments.";
|
||||
logger.debug(msg + additionalLoggingMsg);
|
||||
bulkImportReport.addEvent(msg);
|
||||
fileUtils.writeToFile(bulkImportReportLocation, bulkImportReport.getJsonReport(), false);
|
||||
bulkImportReport.addEvent("Going to import the files in " + subListsSize + " segments, in parallel.");
|
||||
fileUtils.writeToFile(bulkImportReportLocation, bulkImportReport.getJsonReport(), true);
|
||||
|
||||
for ( int i = 0; i < subListsSize; ++i ) {
|
||||
int finalI = i;
|
||||
callableTasksForFileSegments.add(() -> { // Handle inserts to the "attempt" table. Insert 20% of the "attempt" queries.
|
||||
return processBulkImportedFilesSegment(bulkImportReport, finalI, subLists.get(finalI), bulkImportDirName, localParquetDir, currentBulkImportHdfsDir, provenance, bulkImportSource, timeMillis, shouldDeleteFilesOnFinish, additionalLoggingMsg);
|
||||
callables.add(() -> { // Handle inserts to the "attempt" table. Insert 20% of the "attempt" queries.
|
||||
return processBulkImportedFilesSegment(bulkImportReport, finalI, subLists.get(finalI), bulkImportDirName, localParquetDir, currentBulkImportHdfsDir, provenance, bulkImportSource, timeMillis, shouldDeleteFilesOnFinish);
|
||||
});
|
||||
}
|
||||
|
||||
int numFailedSegments = 0;
|
||||
int numFailedFilesForSegment = 0;
|
||||
int numAllFailedFiles = 0;
|
||||
int numFailedFiles = 0;
|
||||
try {
|
||||
List<Future<Integer>> futures = BulkImportController.bulkImportExecutor.invokeAll(callableTasksForFileSegments); // This waits for all tasks to finish.
|
||||
int sizeOfFutures = futures.size(); // This is the same as the "subListsSize".
|
||||
for ( int i = 0; i < sizeOfFutures; ++i )
|
||||
{ // For each segment..
|
||||
List<Future<Integer>> futures = bulkImportExecutor.invokeAll(callables); // This waits for all tasks to finish.
|
||||
int sizeOfFutures = futures.size();
|
||||
for ( int i = 0; i < sizeOfFutures; ++i ) {
|
||||
try {
|
||||
numFailedFilesForSegment = futures.get(i).get();
|
||||
numAllFailedFiles += numFailedFilesForSegment;
|
||||
if ( numFailedFilesForSegment == subLists.get(i).size() )
|
||||
numFailedFiles += futures.get(i).get();
|
||||
if ( numFailedFiles == subLists.get(i).size() ) { // Get and see if it was successfully or not, or if an exception is thrown..
|
||||
numFailedSegments++;
|
||||
}
|
||||
// In case all the files failed to be bulk-imported, then we will detect it in the "numSuccessfulSegments"-check later.
|
||||
// The failed-to-be-imported files, will not be deleted, even if the user specifies that he wants to delete the directory.
|
||||
} catch (ExecutionException ee) { // These can be serious errors like an "out of memory exception" (Java HEAP).
|
||||
numFailedSegments ++;
|
||||
numAllFailedFiles += subLists.get(i).size(); // We assume all files of this segment failed, as all are passed through the same parts of code, so any serious exception should arise from the 1st files being processed and the rest of the files wil be skipped..
|
||||
logger.error(GenericUtils.getSelectedStackTraceForCausedException(ee, "Task_" + i + " failed with: ", additionalLoggingMsg, 20));
|
||||
bulkImportReport.addEvent("Segment_" + i + " failed with: " + ee.getCause().getMessage());
|
||||
} catch (ExecutionException ee) {
|
||||
String stackTraceMessage = GenericUtils.getSelectiveStackTrace(ee, null, 15); // These can be serious errors like an "out of memory exception" (Java HEAP).
|
||||
logger.error("Task_" + (i+1) + " failed with: " + ee.getMessage() + "\n" + stackTraceMessage);
|
||||
} catch (CancellationException ce) {
|
||||
numFailedSegments ++;
|
||||
numAllFailedFiles += subLists.get(i).size(); // We assume all files have failed.
|
||||
logger.error("Task_" + i + " was cancelled: " + ce.getMessage() + additionalLoggingMsg);
|
||||
bulkImportReport.addEvent("Segment_" + i + " failed with: " + ce.getMessage());
|
||||
} catch (InterruptedException ie) {
|
||||
numFailedSegments ++;
|
||||
numAllFailedFiles += (subLists.get(i).size() / 3); // In this case, only some of the files will have failed (do not know how many).
|
||||
logger.error("Task_" + i + " was interrupted: " + ie.getMessage());
|
||||
bulkImportReport.addEvent("Segment_" + i + " failed with: " + ie.getMessage());
|
||||
logger.error("Task_" + (i+1) + " was cancelled: " + ce.getMessage());
|
||||
} catch (IndexOutOfBoundsException ioobe) {
|
||||
logger.error("IOOBE for task_" + i + " in the futures-list! " + ioobe.getMessage() + additionalLoggingMsg);
|
||||
logger.error("IOOBE for task_" + i + " in the futures-list! " + ioobe.getMessage());
|
||||
}
|
||||
}
|
||||
} catch (Exception e) {
|
||||
String errorMsg = "An error occurred when trying to bulk-import data from bulkImportDir: " + bulkImportDirName;
|
||||
logger.error(errorMsg, e);
|
||||
bulkImportReport.addEvent(errorMsg);
|
||||
fileUtils.writeToFile(bulkImportReportLocation, bulkImportReport.getJsonReport(), false);
|
||||
BulkImportController.bulkImportDirsUnderProcessing.remove(bulkImportDirName);
|
||||
fileUtils.writeToFile(bulkImportReportLocation, bulkImportReport.getJsonReport(), true);
|
||||
BulkImportController.bulkImportDirs.remove(bulkImportDirName);
|
||||
return false;
|
||||
} finally {
|
||||
logger.debug("Deleting local parquet directory: " + localParquetDir);
|
||||
|
@ -186,95 +172,59 @@ public class BulkImportServiceImpl implements BulkImportService {
|
|||
}
|
||||
|
||||
// Check the results.
|
||||
if ( numAllFailedFiles == numOfFiles ) {
|
||||
String errorMsg = "None of the files inside the bulkImportDir: " + bulkImportDirName + " were imported!";
|
||||
String msg;
|
||||
if ( numFailedFiles == numOfFiles ) {
|
||||
String errorMsg = "None of the files inside the bulkImportDir '" + bulkImportDirName + "' were imported!";
|
||||
logger.error(errorMsg);
|
||||
bulkImportReport.addEvent(errorMsg);
|
||||
fileUtils.writeToFile(bulkImportReportLocation, bulkImportReport.getJsonReport(), false);
|
||||
BulkImportController.bulkImportDirsUnderProcessing.remove(bulkImportDirName);
|
||||
fileUtils.writeToFile(bulkImportReportLocation, bulkImportReport.getJsonReport(), true);
|
||||
BulkImportController.bulkImportDirs.remove(bulkImportDirName);
|
||||
return false;
|
||||
} else if ( numAllFailedFiles > 0 ) { // Some failed, but not all.
|
||||
msg = numAllFailedFiles + " files" + (numFailedSegments > 0 ? (" and " + numFailedSegments + " whole segments") : "") + " failed to be bulk-imported, from the bulkImportDir: " + bulkImportDirName;
|
||||
} else if ( numFailedFiles > 0 ) { // Some failed, but not all.
|
||||
msg = numFailedFiles + " files" + (numFailedSegments > 0 ? (" and " + numFailedSegments + " whole segments") : "") + " failed to be bulk-imported, from the bulkImportDir: " + bulkImportDirName;
|
||||
logger.warn(msg);
|
||||
} else {
|
||||
msg = "All " + numOfFiles + " files, from bulkImportDir: " + bulkImportDirName + " were bulkImported successfully.";
|
||||
msg = "All " + numOfFiles + " files, from bulkImportDir '" + bulkImportDirName + "' were bulkImported successfully.";
|
||||
logger.info(msg);
|
||||
}
|
||||
bulkImportReport.addEvent(msg);
|
||||
fileUtils.writeToFile(bulkImportReportLocation, bulkImportReport.getJsonReport(), false);
|
||||
fileUtils.writeToFile(bulkImportReportLocation, bulkImportReport.getJsonReport(), true);
|
||||
|
||||
// Merge the parquet files inside the table "payload_bulk_import", to improve performance of future operations.
|
||||
DatabaseConnector.databaseLock.lock();
|
||||
String mergeErrorMsg = parquetFileUtils.mergeParquetFilesOfTable("payload_bulk_import", "", null); // msg is already logged
|
||||
DatabaseConnector.databaseLock.unlock();
|
||||
if ( mergeErrorMsg != null ) { // the message in already logged
|
||||
ImpalaConnector.databaseLock.lock();
|
||||
String mergeErrorMsg = fileUtils.mergeParquetFiles("payload_bulk_import", "", null);
|
||||
if ( mergeErrorMsg != null ) {
|
||||
ImpalaConnector.databaseLock.unlock();
|
||||
bulkImportReport.addEvent(mergeErrorMsg);
|
||||
fileUtils.writeToFile(bulkImportReportLocation, bulkImportReport.getJsonReport(), false);
|
||||
BulkImportController.bulkImportDirsUnderProcessing.remove(bulkImportDirName);
|
||||
fileUtils.writeToFile(bulkImportReportLocation, bulkImportReport.getJsonReport(), true);
|
||||
BulkImportController.bulkImportDirs.remove(bulkImportDirName);
|
||||
return false;
|
||||
}
|
||||
ImpalaConnector.databaseLock.unlock();
|
||||
|
||||
String successMsg = "Finished the bulk-import procedure for " + provenance + " and bulkImportDir: " + bulkImportDirName;
|
||||
String successMsg = "Finished the bulk-import procedure for '" + provenance + "' and bulkImportDir: " + bulkImportDirName;
|
||||
logger.info(successMsg);
|
||||
bulkImportReport.addEvent(successMsg);
|
||||
fileUtils.writeToFile(bulkImportReportLocation, bulkImportReport.getJsonReport(), false);
|
||||
fileUtils.writeToFile(bulkImportReportLocation, bulkImportReport.getJsonReport(), true);
|
||||
// The report-file will be overwritten every now and then, instead of appended, since we want to add an update new JSON report-object each time.
|
||||
// Also, we do not want to write the object in the end (in its final form), since we want the user to have the ability to request the report at any time,
|
||||
// after submitting the bulk-import request, to see its progress (since the number of file may be very large and the processing may take many hours).
|
||||
|
||||
BulkImportController.bulkImportDirsUnderProcessing.remove(bulkImportDirName);
|
||||
BulkImportController.bulkImportDirs.remove(bulkImportDirName);
|
||||
return true;
|
||||
}
|
||||
|
||||
|
||||
private int processBulkImportedFilesSegment(BulkImportReport bulkImportReport, int segmentCounter, List<String> fileLocationsSegment, String bulkImportDirName, String localParquetDir, String currentBulkImportHdfsDir,
|
||||
String provenance, BulkImport.BulkImportSource bulkImportSource, long timeMillis, boolean shouldDeleteFilesOnFinish, String additionalLoggingMsg)
|
||||
String provenance, BulkImport.BulkImportSource bulkImportSource, long timeMillis, boolean shouldDeleteFilesOnFinish)
|
||||
{
|
||||
// Inside this thread, process a segment of the files.
|
||||
String bulkImportReportLocation = bulkImportReport.getReportLocation();
|
||||
|
||||
int numOfFilesInSegment = fileLocationsSegment.size();
|
||||
String msg = "Going to import " + numOfFilesInSegment + " files, for segment-" + segmentCounter;
|
||||
logger.debug(msg + additionalLoggingMsg);
|
||||
String msg = "Going to import " + numOfFilesInSegment + " files for segment-" + segmentCounter + " , of bulkImport procedure '" + provenance + "' | dir: '" + bulkImportDirName + "'..";
|
||||
logger.debug(msg);
|
||||
bulkImportReport.addEvent(msg);
|
||||
fileUtils.writeToFile(bulkImportReportLocation, bulkImportReport.getJsonReport(), true);
|
||||
|
||||
HashMap<String, DocFileData> fileLocationsWithData = new HashMap<>(numOfFilesInSegment); // Map the fileLocations with hashes in order to quickly retrieve them later, when constructing the s3-file-location.
|
||||
Set<String> fileHashes = new HashSet<>(numOfFilesInSegment);
|
||||
|
||||
// Get file-data for this segment's files.
|
||||
for ( String fileLocation : fileLocationsSegment ) {
|
||||
DocFileData docFileData = new DocFileData(new File(fileLocation), null, null, fileLocation);
|
||||
docFileData.calculateAndSetHashAndSize();
|
||||
String fileHash = docFileData.getHash();
|
||||
if ( fileHash != null ) { // We will not work with files for which we cannot produce the hash, since the s3-file-location requires it.
|
||||
fileHashes.add(fileHash);
|
||||
fileLocationsWithData.put(fileLocation, docFileData);
|
||||
}
|
||||
}
|
||||
int fileHashesSetSize = fileHashes.size();
|
||||
if ( fileHashesSetSize == 0 ) {
|
||||
msg = "No fileHashes were retrieved for segment_" + segmentCounter + ", no files will be processed from it.";
|
||||
logger.warn(msg + additionalLoggingMsg);
|
||||
bulkImportReport.addEvent(msg);
|
||||
fileUtils.writeToFile(bulkImportReportLocation, bulkImportReport.getJsonReport(), true);
|
||||
return numOfFilesInSegment; // All files of this segment failed.
|
||||
}
|
||||
|
||||
// Get the "alreadyFound" fileLocations for the fileHashes which exist in the DB.
|
||||
HashMap<String, String> hashWithExistingLocationMap = fileUtils.getHashLocationMap(fileHashes, fileHashesSetSize, segmentCounter, "segment");
|
||||
int numAlreadyRetrievedFiles = hashWithExistingLocationMap.size();
|
||||
if ( numAlreadyRetrievedFiles > 0 ) {
|
||||
msg = numAlreadyRetrievedFiles + " files from segment_" + segmentCounter + ", have been already retrieved in the past.";
|
||||
logger.warn(msg + additionalLoggingMsg);
|
||||
bulkImportReport.addEvent(msg);
|
||||
fileUtils.writeToFile(bulkImportReportLocation, bulkImportReport.getJsonReport(), true);
|
||||
}
|
||||
|
||||
// Generate the payload-records to be inserted in the DB.
|
||||
// The OpenAireID which is synthesised from the fileName, may be different from the ID related to the exact same file which was retrieved in the past.
|
||||
// The same file may be related with multiple publications from different datasources. The deduplication is out of the scope of this service.
|
||||
// All different OpenAIRE IDs should be saved, independently of the fact that their urls or files may be related with other publications as well.
|
||||
|
||||
List<GenericData.Record> payloadRecords = new ArrayList<>(numOfFilesInSegment);
|
||||
|
||||
|
@ -283,46 +233,24 @@ public class BulkImportServiceImpl implements BulkImportService {
|
|||
int counter = 0;
|
||||
|
||||
// Upload files to S3 and collect payloadRecords.
|
||||
for ( int i=0; i < numOfFilesInSegment; ++i ) {
|
||||
String fileLocation = fileLocationsSegment.get(i);
|
||||
DocFileData docFileData = fileLocationsWithData.get(fileLocation);
|
||||
if ( docFileData == null ) {
|
||||
for ( String fileLocation: fileLocationsSegment ) {
|
||||
GenericData.Record record = processBulkImportedFile(fileLocation, provenance, bulkImportSource, timeMillis);
|
||||
if ( record != null )
|
||||
payloadRecords.add(record);
|
||||
else {
|
||||
bulkImportReport.addEvent("An error caused the file: '" + fileLocation + "' to not be imported!");
|
||||
failedFiles.add(fileLocation);
|
||||
} else {
|
||||
String alreadyRetrievedFileLocation = hashWithExistingLocationMap.get(docFileData.getHash());
|
||||
GenericData.Record record = null;
|
||||
try {
|
||||
record = processBulkImportedFile(docFileData, alreadyRetrievedFileLocation, provenance, bulkImportSource, timeMillis, additionalLoggingMsg);
|
||||
} catch (Exception e) {
|
||||
String errorMsg = "Exception when uploading the files of segment_" + segmentCounter + " to the S3 Object Store. Will avoid uploading the rest of the files for this segment.. " + e.getMessage();
|
||||
logger.error(errorMsg + additionalLoggingMsg);
|
||||
bulkImportReport.addEvent(errorMsg);
|
||||
fileUtils.writeToFile(bulkImportReportLocation, bulkImportReport.getJsonReport(), true);
|
||||
for ( int j=i; j < numOfFilesInSegment; ++j )
|
||||
failedFiles.add(fileLocationsSegment.get(j)); // The rest of the files are considered "failed".
|
||||
break;
|
||||
}
|
||||
|
||||
if ( record != null )
|
||||
payloadRecords.add(record);
|
||||
else {
|
||||
bulkImportReport.addEvent("An error caused the file: " + fileLocation + " to not be imported!");
|
||||
failedFiles.add(fileLocation);
|
||||
}
|
||||
}
|
||||
|
||||
if ( ((++counter) % 150) == 0 ) { // Every 150 files, report the status for this segment and write it to the file.
|
||||
msg = "Progress for segment-" + segmentCounter + " : " + payloadRecords.size() + " files have been imported and " + failedFiles.size() + " have failed, out of " + numOfFilesInSegment + " files.";
|
||||
if ( logger.isTraceEnabled() )
|
||||
logger.trace(msg + additionalLoggingMsg);
|
||||
bulkImportReport.addEvent(msg);
|
||||
if ( ((++counter) % 100) == 0 ) { // Every 100 files, report the status.
|
||||
bulkImportReport.addEvent("Progress for segment-" + segmentCounter + " : " + payloadRecords.size() + " files have been imported and " + failedFiles.size() + " have failed, out of " + numOfFilesInSegment + " files.");
|
||||
fileUtils.writeToFile(bulkImportReportLocation, bulkImportReport.getJsonReport(), true);
|
||||
}
|
||||
} // End of processing files for this segment.
|
||||
}
|
||||
|
||||
int numOfPayloadRecords = payloadRecords.size();
|
||||
if ( numOfPayloadRecords == 0 ) {
|
||||
String errorMsg = "No payload-records were generated for any of the files, of segment-" + segmentCounter + ", inside the bulkImportDir: " + bulkImportDirName;
|
||||
String errorMsg = "No payload-records were generated for any of the files inside the bulkImportDir: " + bulkImportDirName;
|
||||
logger.warn(errorMsg);
|
||||
bulkImportReport.addEvent(errorMsg);
|
||||
fileUtils.writeToFile(bulkImportReportLocation, bulkImportReport.getJsonReport(), true);
|
||||
|
@ -330,125 +258,165 @@ public class BulkImportServiceImpl implements BulkImportService {
|
|||
return numOfFilesInSegment;
|
||||
} else if ( numOfPayloadRecords != numOfFilesInSegment ) {
|
||||
// Write this important note here, in order to certainly be in the report, even if a parquet-file failure happens and the method exists early.
|
||||
String errorMsg = failedFiles.size() + " out of " + numOfFilesInSegment + " files failed to be bulk-imported, for segment-" + segmentCounter + " !";
|
||||
logger.warn(errorMsg + additionalLoggingMsg);
|
||||
String errorMsg = failedFiles.size() + " out of " + numOfFilesInSegment + " files failed to be imported, for segment-" + segmentCounter + "!";
|
||||
logger.warn(errorMsg);
|
||||
bulkImportReport.addEvent(errorMsg);
|
||||
fileUtils.writeToFile(bulkImportReportLocation, bulkImportReport.getJsonReport(), true);
|
||||
}
|
||||
|
||||
// Construct the parquet file, upload it to HDFS and load it in the "payload_bulk_import" table.
|
||||
// Construct the parquet file, upload it to HDFS and load them it in the "payload_bulk_import" table.
|
||||
String parquetFileName = "payloads_" + segmentCounter + ".parquet";
|
||||
String fullLocalParquetFilePath = localParquetDir + parquetFileName;
|
||||
|
||||
if ( logger.isTraceEnabled() )
|
||||
logger.trace("Going to write " + numOfPayloadRecords + " payload-records to the parquet file: " + fullLocalParquetFilePath + additionalLoggingMsg); // DEBUG!
|
||||
logger.trace("Going to write " + numOfPayloadRecords + " payload-records to the parquet file: " + fullLocalParquetFilePath); // DEBUG!
|
||||
|
||||
if ( ! parquetFileUtils.writeToParquet(payloadRecords, ParquetFileUtils.payloadsSchema, fullLocalParquetFilePath) ) { // Any errorMsg is already logged inside.
|
||||
String errorMsg = "Could not write the payload-records for segment-" + segmentCounter + " to the parquet-file: " + parquetFileName + " !";
|
||||
logger.error(errorMsg + additionalLoggingMsg);
|
||||
bulkImportReport.addEvent(errorMsg);
|
||||
if ( ! parquetFileUtils.writeToParquet(payloadRecords, ParquetFileUtils.payloadsSchema, fullLocalParquetFilePath) ) {
|
||||
bulkImportReport.addEvent("Could not write the payload-records to the parquet-file: '" + parquetFileName + "'!");
|
||||
fileUtils.writeToFile(bulkImportReportLocation, bulkImportReport.getJsonReport(), true);
|
||||
// None of the files of this segment will be deleted, in any case.
|
||||
return numOfFilesInSegment; // All files of this segment have failed.
|
||||
return numOfFilesInSegment;
|
||||
}
|
||||
|
||||
if ( logger.isTraceEnabled() )
|
||||
logger.trace("Going to upload the parquet file: " + fullLocalParquetFilePath + " to HDFS." + additionalLoggingMsg); // DEBUG!
|
||||
logger.trace("Going to upload the parquet file: " + fullLocalParquetFilePath + " to HDFS."); // DEBUG!
|
||||
|
||||
// Upload and insert the data to the "payload" Impala table. (no database-locking is required)
|
||||
String errorMsg = parquetFileUtils.uploadParquetFileToHDFS(fullLocalParquetFilePath, parquetFileName, currentBulkImportHdfsDir); // The returned message is already logged inside.
|
||||
String errorMsg = parquetFileUtils.uploadParquetFileToHDFS(fullLocalParquetFilePath, parquetFileName, currentBulkImportHdfsDir);
|
||||
if ( errorMsg != null ) { // The possible error-message returned, is already logged by the Controller.
|
||||
errorMsg = "Could not upload the parquet-file " + parquetFileName + " to HDFS!";
|
||||
logger.error(errorMsg + additionalLoggingMsg);
|
||||
bulkImportReport.addEvent(errorMsg);
|
||||
bulkImportReport.addEvent("Could not upload the parquet-file '" + parquetFileName + "' to HDFS!");
|
||||
fileUtils.writeToFile(bulkImportReportLocation, bulkImportReport.getJsonReport(), true);
|
||||
// None of the files of this segment will be deleted, in any case.
|
||||
return numOfFilesInSegment; // All files of this segment have failed.
|
||||
return numOfFilesInSegment;
|
||||
}
|
||||
|
||||
if ( logger.isTraceEnabled() )
|
||||
logger.trace("Going to load the data of parquet-file: \"" + parquetFileName + "\" to the database-table: \"payload_bulk_import\"." + additionalLoggingMsg); // DEBUG!
|
||||
logger.trace("Going to load the data of parquet-file: \"" + parquetFileName + "\" to the database-table: \"payload_bulk_import\"."); // DEBUG!
|
||||
|
||||
DatabaseConnector.databaseLock.lock();
|
||||
boolean parquetDataLoaded = parquetFileUtils.loadParquetDataIntoTable((currentBulkImportHdfsDir + parquetFileName), "payload_bulk_import");
|
||||
DatabaseConnector.databaseLock.unlock();
|
||||
if ( !parquetDataLoaded ) {
|
||||
errorMsg = "Could not load the payload-records to the database, for segment-" + segmentCounter + "!";
|
||||
logger.error(errorMsg + additionalLoggingMsg);
|
||||
bulkImportReport.addEvent(errorMsg);
|
||||
ImpalaConnector.databaseLock.lock();
|
||||
if ( !parquetFileUtils.loadParquetDataIntoTable((currentBulkImportHdfsDir + parquetFileName), "payload_bulk_import") ) {
|
||||
ImpalaConnector.databaseLock.unlock();
|
||||
bulkImportReport.addEvent("Could not load the payload-records to the database!");
|
||||
fileUtils.writeToFile(bulkImportReportLocation, bulkImportReport.getJsonReport(), true);
|
||||
// None of the files of this segment will be deleted, in any case.
|
||||
return numOfFilesInSegment; // All files of this segment have failed.
|
||||
return numOfFilesInSegment;
|
||||
}
|
||||
ImpalaConnector.databaseLock.unlock();
|
||||
|
||||
String segmentSuccessMsg = "Finished importing " + numOfPayloadRecords + " files, out of " + numOfFilesInSegment + ", for segment-" + segmentCounter + ".";
|
||||
logger.info(segmentSuccessMsg + additionalLoggingMsg);
|
||||
String segmentSuccessMsg = "Finished importing " + numOfPayloadRecords + " files, out of " + numOfFilesInSegment + " , for segment-" + segmentCounter + ".";
|
||||
logger.info(segmentSuccessMsg);
|
||||
bulkImportReport.addEvent(segmentSuccessMsg);
|
||||
|
||||
if ( shouldDeleteFilesOnFinish ) {
|
||||
segmentSuccessMsg = "As the user requested, the successfully imported files of " + provenance + " procedure, of bulk-import segment-" + segmentCounter + ", from directory " + bulkImportDirName + ", will be deleted.";
|
||||
segmentSuccessMsg = "As the user requested, the successfully imported files of '" + provenance + "' procedure, of bulk-import segment-" + segmentCounter + ", from directory '" + bulkImportDirName + "', will be deleted.";
|
||||
logger.info(segmentSuccessMsg);
|
||||
bulkImportReport.addEvent(segmentSuccessMsg);
|
||||
// Delete all files except the ones in the "failedHashSet".
|
||||
|
||||
// Delete all files except the ones in the "failedHashSet"
|
||||
for ( String fileLocation : fileLocationsSegment ) {
|
||||
if ( !failedFiles.contains(fileLocation) )
|
||||
if ( !fileUtils.deleteFile(fileLocation) ) // The "error-log-message" is shown inside.
|
||||
bulkImportReport.addEvent("The file " + fileLocation + " could not be deleted! Please make sure you have provided the WRITE-permission.");
|
||||
if ( !fileUtils.deleteFile(fileLocation) )
|
||||
bulkImportReport.addEvent("The file '" + fileLocation + "' could not be deleted! Please make sure you have provided the WRITE-permission.");
|
||||
}
|
||||
}
|
||||
|
||||
fileUtils.writeToFile(bulkImportReportLocation, bulkImportReport.getJsonReport(), true);
|
||||
return (numOfFilesInSegment - numOfPayloadRecords); // Return the numOfFailedFiles.
|
||||
}
|
||||
|
||||
|
||||
private GenericData.Record processBulkImportedFile(DocFileData docFileData, String alreadyRetrievedFileLocation, String provenance, BulkImport.BulkImportSource bulkImportSource, long timeMillis, String additionalLoggingMsg)
|
||||
throws ConnectException, UnknownHostException
|
||||
private GenericData.Record processBulkImportedFile(String fileLocation, String provenance, BulkImport.BulkImportSource bulkImportSource, long timeMillis)
|
||||
{
|
||||
File fullTextFile = new File(fileLocation);
|
||||
DocFileData docFileData = new DocFileData(fullTextFile, null, null, null);
|
||||
docFileData.calculateAndSetHashAndSize();
|
||||
|
||||
// Check if this file is already found by crawling. Even though we started excluding this datasource from crawling, many full-texts have already been downloaded.
|
||||
// Also, it may be the case that this file was downloaded by another datasource.
|
||||
|
||||
FileLocationData fileLocationData;
|
||||
try {
|
||||
fileLocationData = new FileLocationData(docFileData.getLocation());
|
||||
fileLocationData = new FileLocationData(fileLocation);
|
||||
} catch (RuntimeException re) {
|
||||
logger.error(re.getMessage() + additionalLoggingMsg);
|
||||
logger.error(re.getMessage());
|
||||
return null;
|
||||
}
|
||||
|
||||
String fileHash = docFileData.getHash();
|
||||
if ( fileHash == null )
|
||||
return null; // No check of past found full-text can be made nor the S3-fileName can be created.
|
||||
|
||||
String datasourceId = bulkImportSource.getDatasourceID();
|
||||
String datasourcePrefix = bulkImportSource.getDatasourcePrefix();
|
||||
String fileNameID = fileLocationData.getFileNameID();
|
||||
|
||||
String openAireId = generateOpenaireId(fileNameID, bulkImportSource.getDatasourcePrefix(), bulkImportSource.getIsAuthoritative());
|
||||
if ( openAireId == null )
|
||||
return null;
|
||||
|
||||
String fileHash = docFileData.getHash(); // It's guaranteed to NOT be null at this point.
|
||||
|
||||
String s3Url = null;
|
||||
if ( alreadyRetrievedFileLocation != null ) // If the full-text of this record is already-found and uploaded.
|
||||
{
|
||||
// This full-text was found to already be in the database.
|
||||
// If it has the same datasourceID, then it likely was crawled before from an ID belonging to this datasource.
|
||||
// If also, has the same ID, then the exact same record from that datasource was retrieved previously.
|
||||
// Else, the file was downloaded by another record of this datasource.
|
||||
// ELse if the datasourceID is not the same, then the same file was retrieved from another datasource.
|
||||
// The above analysis is educational, it does not need to take place and is not currently used.
|
||||
s3Url = alreadyRetrievedFileLocation;
|
||||
} else {
|
||||
s3Url = fileUtils.constructS3FilenameAndUploadToS3(fileLocationData.getFileDir(), fileLocationData.getFileName(), openAireId, fileLocationData.getDotFileExtension(), bulkImportSource.getDatasourceID(), fileHash);
|
||||
if ( s3Url == null )
|
||||
return null;
|
||||
}
|
||||
|
||||
// TODO - If another url-schema is introduced for other datasources, have a "switch"-statement and perform the right "actualUrl"-creation based on current schema.
|
||||
String actualUrl = (bulkImportSource.getPdfUrlPrefix() + fileNameID); // This string-concatenation, works with urls of Arvix. A different construction may be needed for other datasources.
|
||||
String originalUrl = actualUrl; // We have the full-text files from bulk-import, so let's assume the original-url is also the full-text-link.
|
||||
|
||||
return parquetFileUtils.getPayloadParquetRecord(openAireId, originalUrl, actualUrl, timeMillis, bulkImportSource.getMimeType(),
|
||||
docFileData.getSize(), fileHash, s3Url, provenance, true); // It may return null.
|
||||
final String getFileLocationForHashQuery = "select `location` from " + ImpalaConnector.databaseName + ".payload where `hash` = ? limit 1";
|
||||
final int[] hashArgType = new int[] {Types.VARCHAR};
|
||||
String alreadyFoundFileLocation = null;
|
||||
ImpalaConnector.databaseLock.lock();
|
||||
try {
|
||||
alreadyFoundFileLocation = jdbcTemplate.queryForObject(getFileLocationForHashQuery, new Object[]{fileHash}, hashArgType, String.class);
|
||||
} catch (EmptyResultDataAccessException erdae) {
|
||||
// No fileLocation is found, it's ok. It will be null by default.
|
||||
} catch (Exception e) {
|
||||
logger.error("Error when executing or acquiring data from the the 'getFileLocationForHashQuery'!\n", e);
|
||||
// Continue with bulk-importing the file and uploading it to S3.
|
||||
} finally {
|
||||
ImpalaConnector.databaseLock.unlock();
|
||||
}
|
||||
|
||||
String idMd5hash = getMD5hash(fileNameID.toLowerCase());
|
||||
if ( idMd5hash == null )
|
||||
return null;
|
||||
|
||||
// openaire id = <datasourcePrefix> + "::" + <md5(lowercase(arxivId))>
|
||||
String openAireId = (datasourcePrefix + "::" + idMd5hash);
|
||||
|
||||
String s3Url = null;
|
||||
|
||||
if ( alreadyFoundFileLocation != null ) // If the full-text of this record is already-found and uploaded.
|
||||
{
|
||||
// This full-text was found to already be in the database.
|
||||
// If it has the same datasourceID, then it likely was crawled before from an ID belonging to this datasource.
|
||||
// If also has the same ID, then the exact same record from that datasource was retrieved previously.
|
||||
// Else, the file was downloaded by another record of this datasource.
|
||||
// ELse if the datasourceID is not the same, then the same file was retrieved from another datasource.
|
||||
// The above analysis is educational, it does not need to take place and is not currently used.
|
||||
|
||||
s3Url = alreadyFoundFileLocation;
|
||||
} else {
|
||||
try {
|
||||
s3Url = fileUtils.constructFileNameAndUploadToS3(fileLocationData.getFileDir(), fileLocationData.getFileName(), openAireId, fileLocationData.getDotFileExtension(), datasourceId, fileHash); // This throws Exception, in case the uploading failed.
|
||||
if ( s3Url == null )
|
||||
return null; // In case the 'datasourceID' or 'hash' is null. Which should never happen here, since both of them are checked before the execution reaches here.
|
||||
} catch (Exception e) {
|
||||
logger.error("Could not upload the file '" + fileLocationData.getFileName() + "' to the S3 ObjectStore!", e);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
GenericData.Record record = new GenericData.Record(ParquetFileUtils.payloadsSchema);
|
||||
record.put("id", openAireId);
|
||||
record.put("original_url", originalUrl);
|
||||
record.put("actual_url", actualUrl);
|
||||
record.put("date", timeMillis);
|
||||
record.put("mimetype", bulkImportSource.getMimeType());
|
||||
Long size = docFileData.getSize();
|
||||
record.put("size", ((size != null) ? String.valueOf(size) : null));
|
||||
record.put("hash", fileHash); // This is already checked and will not be null here.
|
||||
record.put("location", s3Url);
|
||||
record.put("provenance", provenance);
|
||||
|
||||
return record;
|
||||
}
|
||||
|
||||
|
||||
public List<String> getFileLocationsInsideDir(String directory)
|
||||
{
|
||||
List<String> fileLocations = null;
|
||||
|
||||
try ( Stream<Path> walkStream = Files.find(Paths.get(directory), Integer.MAX_VALUE, (filePath, fileAttr) -> fileAttr.isRegularFile()) )
|
||||
// In case we ever include other type-of-Files inside the same directory, we need to add this filter: "&& !filePath.toString().endsWith("name.ext")"
|
||||
{
|
||||
|
@ -458,11 +426,12 @@ public class BulkImportServiceImpl implements BulkImportService {
|
|||
logger.error(errorMsg, e);
|
||||
return null;
|
||||
}
|
||||
|
||||
return fileLocations;
|
||||
}
|
||||
|
||||
|
||||
public String getMD5Hash(String string)
|
||||
public String getMD5hash(String string)
|
||||
{
|
||||
String md5 = null;
|
||||
try {
|
||||
|
@ -476,21 +445,4 @@ public class BulkImportServiceImpl implements BulkImportService {
|
|||
return md5;
|
||||
}
|
||||
|
||||
|
||||
public String generateOpenaireId(String id, String datasourcePrefix, boolean isAuthoritative)
|
||||
{
|
||||
// If the "provenance" relates to an "authoritative" source, then its id has to be lowercase, before the md5() is applied to it.
|
||||
// general_openaire_id = <datasourcePrefix> + "::" + <md5(ID)>
|
||||
// authoritative_openaire_id = <datasourcePrefix> + "::" + <md5(lowercase(ID))>
|
||||
|
||||
if ( isAuthoritative )
|
||||
id = id.toLowerCase();
|
||||
|
||||
String idMd5Hash = getMD5Hash(id);
|
||||
if ( idMd5Hash == null )
|
||||
return null;
|
||||
|
||||
return (datasourcePrefix + "::" + idMd5Hash);
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
@ -2,11 +2,11 @@ package eu.openaire.urls_controller.services;
|
|||
|
||||
import org.springframework.http.ResponseEntity;
|
||||
|
||||
import javax.servlet.http.HttpServletRequest;
|
||||
|
||||
public interface ShutdownService {
|
||||
|
||||
ResponseEntity<?> passSecurityChecks(String remoteAddr, String initMsg);
|
||||
|
||||
void postShutdownOrCancelRequestsToAllWorkers(boolean shouldCancel);
|
||||
ResponseEntity<?> passSecurityChecks(HttpServletRequest request, String initMsg);
|
||||
|
||||
boolean postShutdownOrCancelRequestToWorker(String workerId, String workerIp, boolean shouldCancel);
|
||||
|
||||
|
|
|
@ -1,19 +1,17 @@
|
|||
package eu.openaire.urls_controller.services;
|
||||
|
||||
import eu.openaire.urls_controller.controllers.UrlsController;
|
||||
import eu.openaire.urls_controller.models.WorkerInfo;
|
||||
import eu.openaire.urls_controller.util.UriBuilder;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.http.HttpStatus;
|
||||
import org.springframework.http.ResponseEntity;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.web.client.HttpServerErrorException;
|
||||
import org.springframework.web.client.RestTemplate;
|
||||
|
||||
import javax.servlet.http.HttpServletRequest;
|
||||
import java.net.ConnectException;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
|
||||
@Service
|
||||
|
@ -22,46 +20,33 @@ public class ShutdownServiceImpl implements ShutdownService {
|
|||
|
||||
private static final Logger logger = LoggerFactory.getLogger(ShutdownServiceImpl.class);
|
||||
|
||||
// Private Addresses, according to RFC 1918: https://www.rfc-editor.org/rfc/rfc1918
|
||||
private static final Pattern PRIVATE_IP_ADDRESSES_RFC_1918 = Pattern.compile("(?:10.|172.(?:1[6-9]|2[0-9]|3[0-1])|192.168.)[0-9.]+");
|
||||
|
||||
|
||||
@Value("${services.pdfaggregation.worker.port}")
|
||||
private String workerPort;
|
||||
|
||||
|
||||
public ResponseEntity<?> passSecurityChecks(String remoteAddr, String initMsg)
|
||||
public ResponseEntity<?> passSecurityChecks(HttpServletRequest request, String initMsg)
|
||||
{
|
||||
if ( request == null ) {
|
||||
logger.error(initMsg + "The \"HttpServletRequest\" is null!");
|
||||
return ResponseEntity.internalServerError().build();
|
||||
}
|
||||
String remoteAddr = request.getHeader("X-FORWARDED-FOR");
|
||||
if ( remoteAddr == null || remoteAddr.isEmpty() )
|
||||
remoteAddr = request.getRemoteAddr();
|
||||
|
||||
// In case the Controller is running inside a docker container, and we want to send the "shutdownServiceRequest" from the terminal (with curl), without entering inside the container,
|
||||
// then the request will appear coming from a local (private) IP, instead of localhost.
|
||||
if ( ! (remoteAddr.equals("127.0.0.1") || remoteAddr.equals(UriBuilder.ip) || PRIVATE_IP_ADDRESSES_RFC_1918.matcher(remoteAddr).matches()) ) {
|
||||
// then the request will appear coming from a local IP (192.168.X.Y), instead of localhost.
|
||||
if ( ! (remoteAddr.equals("127.0.0.1") || remoteAddr.startsWith("192.168.") || remoteAddr.equals(UriBuilder.ip) ) ) {
|
||||
logger.error(initMsg + "The request came from another IP: " + remoteAddr + " | while the Controller has the IP: " + UriBuilder.ip);
|
||||
return ResponseEntity.status(HttpStatus.FORBIDDEN).build();
|
||||
}
|
||||
|
||||
return null; // The checks are passing.
|
||||
}
|
||||
|
||||
|
||||
public void postShutdownOrCancelRequestsToAllWorkers(boolean shouldCancel)
|
||||
{
|
||||
// Send "shutdownWorker" requests to all active Workers.
|
||||
for ( String workerId : UrlsController.workersInfoMap.keySet() ) {
|
||||
WorkerInfo workerInfo = UrlsController.workersInfoMap.get(workerId);
|
||||
if ( ! workerInfo.getHasShutdown() ) // A worker may have shutdown on its own (by sending it a shutDown request manually), so it will have told the Controller when it shut down. In case of a Worker-crash, the Controller will not know about it.
|
||||
postShutdownOrCancelRequestToWorker(workerId, workerInfo.getWorkerIP(), shouldCancel);
|
||||
else
|
||||
logger.warn("Will not post " + (shouldCancel ? "Cancel-" : "") + "ShutdownRequest to Worker \"" + workerId + "\", since is it has already shutdown.");
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
private static final RestTemplate restTemplate = new RestTemplate();
|
||||
|
||||
public boolean postShutdownOrCancelRequestToWorker(String workerId, String workerIp, boolean shouldCancel)
|
||||
{
|
||||
String url = "http://" + workerIp + ":" + workerPort + "/api/" + (shouldCancel ? "cancelShutdownWorker" : "shutdownWorker");
|
||||
String url = "http://" + workerIp + ":1881/api/" + (shouldCancel ? "cancelShutdownWorker" : "shutdownWorker");
|
||||
try {
|
||||
ResponseEntity<?> responseEntity = restTemplate.postForEntity(url, null, String.class);
|
||||
ResponseEntity<?> responseEntity = new RestTemplate().postForEntity(url, null, String.class);
|
||||
int responseCode = responseEntity.getStatusCodeValue();
|
||||
if ( responseCode != HttpStatus.OK.value() ) {
|
||||
logger.error("HTTP-Connection problem with the submission of the \"postShutdownOrCancelRequestToWorker\" of worker \"" + workerId + "\"! Error-code was: " + responseCode);
|
||||
|
|
|
@ -4,8 +4,8 @@ import org.springframework.http.ResponseEntity;
|
|||
|
||||
public interface StatsService {
|
||||
|
||||
ResponseEntity<?> getNumberOfPayloads(String getPayloadsNumberQuery, String extraMsg, int retryCount);
|
||||
ResponseEntity<?> getNumberOfPayloads(String getPayloadsNumberQuery, String extraMsg);
|
||||
|
||||
ResponseEntity<?> getNumberOfRecordsInspectedByServiceThroughCrawling(int retryCount);
|
||||
ResponseEntity<?> getNumberOfRecordsInspected();
|
||||
|
||||
}
|
||||
|
|
|
@ -1,6 +1,6 @@
|
|||
package eu.openaire.urls_controller.services;
|
||||
|
||||
import eu.openaire.urls_controller.configuration.DatabaseConnector;
|
||||
import eu.openaire.urls_controller.configuration.ImpalaConnector;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
|
@ -19,35 +19,20 @@ public class StatsServiceImpl implements StatsService {
|
|||
private JdbcTemplate jdbcTemplate;
|
||||
|
||||
// No DB-lock is required for these READ-operations.
|
||||
// BUT! There is an issue.. these queries may run while a "table-merging" operation is in progress.. thus resulting in "no table reference" and "no file found (fieName.parquet)"
|
||||
|
||||
// Thus, we need to have an "error-detection-and-retry" mechanism, in order to avoid returning error that we know will exist in certain times and we can overcome them.
|
||||
// The final time-to-return of the results-retrieval methods may be somewhat large, but the alternative of returning predictable errors or locking the DB and slowing down the aggregation system are even worse.
|
||||
|
||||
public ResponseEntity<?> getNumberOfPayloads(String getNumberQuery, String message, int retryCount)
|
||||
{
|
||||
if ( retryCount > 10 ) {
|
||||
String errorMsg = "Could not find the requested payload-type table in an non-merging state, after " + (retryCount -1) + " retries!";
|
||||
logger.error(errorMsg);
|
||||
return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR).body(errorMsg);
|
||||
}
|
||||
|
||||
public ResponseEntity<?> getNumberOfPayloads(String getNumberQuery, String message) {
|
||||
try {
|
||||
Object result = jdbcTemplate.queryForObject(getNumberQuery, Integer.class);
|
||||
if ( result != null ) {
|
||||
int numOfPayloads = (int) result;
|
||||
logger.info("The number of " + message + " in the database \"" + DatabaseConnector.databaseName + "\" is " + numOfPayloads);
|
||||
return ResponseEntity.ok(Integer.toString(numOfPayloads));
|
||||
logger.info("The number of " + message + " in the database \"" + ImpalaConnector.databaseName + "\" is " + numOfPayloads);
|
||||
return new ResponseEntity<>(numOfPayloads, HttpStatus.OK);
|
||||
} else
|
||||
return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR).body("The number of " + message + " could not be retrieved (it was null) from the database \"" + DatabaseConnector.databaseName + "\" using the getNumberQuery: " + getNumberQuery);
|
||||
return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR).body("The number of " + message + " could not be retrieved from the database \"" + ImpalaConnector.databaseName + "\" using the getNumberQuery: " + getNumberQuery);
|
||||
} catch (EmptyResultDataAccessException erdae) {
|
||||
return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR).body("The number of " + message + " could not be retrieved (empty result) from the database \"" + DatabaseConnector.databaseName + "\" using the getNumberQuery: " + getNumberQuery);
|
||||
return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR).body("The number of " + message + " could not be retrieved from the database \"" + ImpalaConnector.databaseName + "\" using the getNumberQuery: " + getNumberQuery);
|
||||
} catch (Exception e) {
|
||||
String exMsg = e.getMessage();
|
||||
if ( (exMsg != null) && (exMsg.contains("Could not resolve table reference") || exMsg.contains("Failed to open HDFS file")) ) {
|
||||
sleep1min();
|
||||
return getNumberOfPayloads(getNumberQuery, message, (++retryCount));
|
||||
}
|
||||
String errorMsg = "Problem when executing \"getNumberQuery\": " + getNumberQuery;
|
||||
logger.error(errorMsg, e);
|
||||
return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR).body(errorMsg);
|
||||
|
@ -56,36 +41,27 @@ public class StatsServiceImpl implements StatsService {
|
|||
}
|
||||
|
||||
|
||||
public ResponseEntity<?> getNumberOfRecordsInspectedByServiceThroughCrawling(int retryCount)
|
||||
public ResponseEntity<?> getNumberOfRecordsInspected()
|
||||
{
|
||||
if ( retryCount > 10 ) {
|
||||
String errorMsg = "Could not find the requested attempt table in an non-merging state, after " + (retryCount -1) + " retries!";
|
||||
logger.error(errorMsg);
|
||||
return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR).body(errorMsg);
|
||||
}
|
||||
|
||||
// Note that until all the records are inspected, the "attempt" table contains all the inspected records PLUS very few duplicates (id-url) which come from the publications-database.
|
||||
// After all the records are inspected, it contains duplicate records of more and more id-urls, as time goes on, since for every eligible record the Service re-attempts to get the full-text.
|
||||
// So in order to get the number of inspected records, we want the distinct number, which at some point it will remain stable, even though the Service will try again and again some records.
|
||||
// Before all the records are inspected, this endpoint will report all the inspected records MINUS the duplicate records which come straight from the "publication" table.
|
||||
|
||||
final String getInspectedRecordsNumberQuery = "select count(dist.id) from (select distinct id, original_url from " + DatabaseConnector.databaseName + ".attempt) as dist";
|
||||
logger.info("Received a \"getNumberOfRecordsInspected\" request.");
|
||||
|
||||
final String getInspectedRecordsNumberQuery = "select count(dist.id) from (select distinct id, original_url from " + ImpalaConnector.databaseName + ".attempt) as dist";
|
||||
try {
|
||||
Object result = jdbcTemplate.queryForObject(getInspectedRecordsNumberQuery, Integer.class);
|
||||
if ( result != null ) {
|
||||
int numOfInspectedRecords = (int) result;
|
||||
logger.info("Number of crawling-inspected records from the database \"" + DatabaseConnector.databaseName + "\" is " + numOfInspectedRecords);
|
||||
return ResponseEntity.ok(Integer.toString(numOfInspectedRecords));
|
||||
logger.info("Number of inspected records from the database \"" + ImpalaConnector.databaseName + "\" is " + numOfInspectedRecords);
|
||||
return new ResponseEntity<>(numOfInspectedRecords, HttpStatus.OK);
|
||||
} else
|
||||
return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR).body("The inspected records' number could not be retrieved from the database \"" + DatabaseConnector.databaseName + "\" using the getInspectedRecordsNumberQuery: " + getInspectedRecordsNumberQuery);
|
||||
return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR).body("The inspected records' number could not be retrieved from the database \"" + ImpalaConnector.databaseName + "\" using the getInspectedRecordsNumberQuery: " + getInspectedRecordsNumberQuery);
|
||||
} catch (EmptyResultDataAccessException erdae) {
|
||||
return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR).body("The inspected records' number could not be retrieved from the database \"" + DatabaseConnector.databaseName + "\" using the getInspectedRecordsNumberQuery: " + getInspectedRecordsNumberQuery);
|
||||
return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR).body("The inspected records' number could not be retrieved from the database \"" + ImpalaConnector.databaseName + "\" using the getInspectedRecordsNumberQuery: " + getInspectedRecordsNumberQuery);
|
||||
} catch (Exception e) {
|
||||
String exMsg = e.getMessage();
|
||||
if ( (exMsg != null) && (exMsg.contains("Could not resolve table reference") || exMsg.contains("Failed to open HDFS file")) ) {
|
||||
sleep1min();
|
||||
return getNumberOfRecordsInspectedByServiceThroughCrawling(++retryCount);
|
||||
}
|
||||
String errorMsg = "Problem when executing \"getInspectedRecordsNumberQuery\": " + getInspectedRecordsNumberQuery;
|
||||
logger.error(errorMsg, e);
|
||||
return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR).body(errorMsg);
|
||||
|
@ -93,18 +69,7 @@ public class StatsServiceImpl implements StatsService {
|
|||
}
|
||||
}
|
||||
|
||||
|
||||
// To get the human-friendly timestamp format from the BigInt in the database:
|
||||
// select from_timestamp(CAST(CAST(`date` as decimal(30,0))/1000 AS timestamp), "yyyy-MM-dd HH:mm:ss.SSS") from payload
|
||||
// Or simpler: select from_timestamp(CAST((`date`/1000) AS timestamp), "yyyy-MM-dd HH:mm:ss.SSS") from payload
|
||||
|
||||
|
||||
private void sleep1min() {
|
||||
try {
|
||||
Thread.sleep(60_000); // Sleep for 1 min.
|
||||
} catch (InterruptedException ie) {
|
||||
logger.warn("Sleeping was interrupted!");
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
@ -1,25 +1,21 @@
|
|||
package eu.openaire.urls_controller.services;
|
||||
|
||||
import eu.openaire.urls_controller.components.BulkImport;
|
||||
import eu.openaire.urls_controller.configuration.DatabaseConnector;
|
||||
import eu.openaire.urls_controller.configuration.ImpalaConnector;
|
||||
import eu.openaire.urls_controller.controllers.UrlsController;
|
||||
import eu.openaire.urls_controller.models.*;
|
||||
import eu.openaire.urls_controller.payloads.responces.AssignmentsResponse;
|
||||
import eu.openaire.urls_controller.util.FileUtils;
|
||||
import eu.openaire.urls_controller.util.FilesHandler;
|
||||
import eu.openaire.urls_controller.util.GenericUtils;
|
||||
import eu.openaire.urls_controller.util.ParquetFileUtils;
|
||||
import io.micrometer.core.annotation.Timed;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.dao.EmptyResultDataAccessException;
|
||||
import org.springframework.http.HttpStatus;
|
||||
import org.springframework.http.ResponseEntity;
|
||||
import org.springframework.jdbc.core.JdbcTemplate;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.web.client.HttpClientErrorException;
|
||||
import org.springframework.web.client.HttpServerErrorException;
|
||||
import org.springframework.web.client.RestTemplate;
|
||||
|
||||
|
@ -31,7 +27,6 @@ import java.sql.PreparedStatement;
|
|||
import java.sql.SQLException;
|
||||
import java.sql.Timestamp;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Calendar;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.concurrent.Callable;
|
||||
|
@ -50,9 +45,6 @@ public class UrlsServiceImpl implements UrlsService {
|
|||
@Autowired
|
||||
private JdbcTemplate jdbcTemplate;
|
||||
|
||||
@Autowired
|
||||
private FilesHandler filesHandler;
|
||||
|
||||
@Autowired
|
||||
private FileUtils fileUtils;
|
||||
|
||||
|
@ -62,20 +54,12 @@ public class UrlsServiceImpl implements UrlsService {
|
|||
@Value("${services.pdfaggregation.controller.workerReportsDirPath}")
|
||||
private String workerReportsDirPath;
|
||||
|
||||
@Value("${services.pdfaggregation.worker.port}")
|
||||
private String workerPort;
|
||||
|
||||
public static final AtomicLong assignmentsBatchCounter = new AtomicLong(0);
|
||||
|
||||
private final AtomicInteger maxAttemptsPerRecordAtomic;
|
||||
|
||||
private static String excludedDatasourceIDsStringList = null;
|
||||
|
||||
|
||||
private static final String DOC_URL_FILTER = ".+(?:pdf|download|/doc|document|(?:/|[?]|&)file|/fulltext|attachment|/paper|view(?:file|doc)|/get|cgi/viewcontent.cgi\\?|t[ée]l[ée]charger|descargar).*";
|
||||
// "DOC_URL_FILTER" works for lowerCase Strings (we use the "ignore-case" indicator in the "regexp_like()" method).
|
||||
|
||||
|
||||
public static final ExecutorService insertsExecutor = Executors.newFixedThreadPool(6);
|
||||
// TODO - Unify this ExecutorService with the hash-matching executorService. Since one will ALWAYS be called after the other. So why having two ExecServices to handle?
|
||||
|
||||
|
@ -100,65 +84,69 @@ public class UrlsServiceImpl implements UrlsService {
|
|||
throw new RuntimeException("One of the bulk-imported datasourceIDs was not found! | source: " + source);
|
||||
excludedIDs.add(datasourceID);
|
||||
}
|
||||
|
||||
int exclusionListSize = excludedIDs.size(); // This list will not be empty.
|
||||
|
||||
// Prepare the "excludedDatasourceIDsStringList" to be used inside the "findAssignmentsQuery". Create the following string-pattern:
|
||||
// ("ID_1", "ID_2", ...)
|
||||
int stringBuilderCapacity = ((exclusionListSize * 46) + (exclusionListSize -1) +2);
|
||||
excludedDatasourceIDsStringList = FileUtils.getQueryListString(excludedIDs, exclusionListSize, stringBuilderCapacity);
|
||||
logger.info("The following bulkImport data-sources will be excluded from crawling: " + excludedDatasourceIDsStringList);
|
||||
final StringBuilder sb = new StringBuilder((exclusionListSize * 46) + (exclusionListSize -1) +2 );
|
||||
sb.append("(");
|
||||
for ( int i=0; i < exclusionListSize; ++i ) {
|
||||
sb.append("\"").append(excludedIDs.get(i)).append("\"");
|
||||
if ( i < (exclusionListSize -1) )
|
||||
sb.append(", ");
|
||||
}
|
||||
sb.append(")");
|
||||
|
||||
excludedDatasourceIDsStringList = sb.toString();
|
||||
logger.info("The following bulkImport-datasources will be excluded from crawling: " + excludedDatasourceIDsStringList);
|
||||
}
|
||||
|
||||
|
||||
@Timed(value = "getAssignments.time", description = "Time taken to return the assignments.")
|
||||
public ResponseEntity<?> getAssignments(String workerId, int assignmentsLimit)
|
||||
{
|
||||
int currentYear = Calendar.getInstance().get(Calendar.YEAR);
|
||||
|
||||
// Create the Assignments from the id-urls stored in the database up to the < assignmentsLimit >.
|
||||
String findAssignmentsQuery =
|
||||
"select pubid, url, datasourceid, datasourcename\n" + // Select the final sorted data with "assignmentsLimit".
|
||||
"from (select distinct p.id as pubid, pu.url as url, pb.level as level, attempts.counts as attempt_count, p.year as pub_year, d.id as datasourceid, d.name as datasourcename\n" + // Select the distinct id-url data. Beware that this will return duplicate id-url pairs, wince one pair may be associated with multiple datasources.
|
||||
" from " + DatabaseConnector.databaseName + ".publication_urls pu\n" +
|
||||
" join " + DatabaseConnector.databaseName + ".publication p on p.id=pu.id\n" +
|
||||
" join " + DatabaseConnector.databaseName + ".datasource d on d.id=p.datasourceid and d.allow_harvest=true"+
|
||||
((excludedDatasourceIDsStringList != null) ? // If we have an exclusion-list, use it below.
|
||||
(" and d.id not in " + excludedDatasourceIDsStringList + GenericUtils.endOfLine) : "") +
|
||||
" left anti join (select a.original_url from " + DatabaseConnector.databaseName + ".assignment a\n" +
|
||||
"select pubid, url, datasourceid, datasourcename\n" + // The datsourceName is currently not used. It may be used by the Worker, in the future to apply a datasource-specific aggregation plugin to take the full-texts quickly, instead of using the general crawling one.
|
||||
"from (select distinct pubid, url, datasourceid, datasourcename, attempt_count, pub_year\n" +
|
||||
" from (select p.id as pubid, p.year as pub_year, pu.url as url, d.id as datasourceid, d.name as datasourcename, attempts.counts as attempt_count\n" +
|
||||
" from " + ImpalaConnector.databaseName + ".publication p\n" +
|
||||
" join " + ImpalaConnector.databaseName + ".publication_urls pu on pu.id=p.id\n" +
|
||||
" join " + ImpalaConnector.databaseName + ".datasource d on d.id=p.datasourceid\n" +
|
||||
" left outer join (select count(a.id) as counts, a.id from " + ImpalaConnector.databaseName + ".attempt a group by a.id) as attempts\n" +
|
||||
" on attempts.id=p.id\n" +
|
||||
" left outer join (\n" +
|
||||
" select a.id, a.original_url from " + ImpalaConnector.databaseName + ".assignment a\n" +
|
||||
" union all\n" +
|
||||
" select pl.original_url from " + DatabaseConnector.databaseName + ".payload pl\n" + // Here we access the payload-VIEW which includes the three payload-tables.
|
||||
" ) as existing\n" +
|
||||
" on existing.original_url=pu.url\n" +
|
||||
" left outer join " + DatabaseConnector.databaseName + ".publication_boost pb\n" +
|
||||
" on p.id=pb.id\n" +
|
||||
" left outer join (select count(at.original_url) as counts, at.original_url from " + DatabaseConnector.databaseName + ".attempt at group by at.original_url) as attempts\n" +
|
||||
" on attempts.original_url=pu.url\n" +
|
||||
" where coalesce(attempts.counts, 0) <= " + maxAttemptsPerRecordAtomic.get() + GenericUtils.endOfLine +
|
||||
" and not exists (select 1 from " + DatabaseConnector.databaseName + ".attempt a where a.original_url=pu.url and a.error_class = 'noRetry' limit 1)\n" +
|
||||
" and (p.year <= " + currentYear + " or p.year > " + (currentYear + 5) + ")\n" + // Exclude the pubs which will be published in the next 5 years. They don't provide full-texts now. (We don't exclude all future pubs, since, some have invalid year, like "9999").
|
||||
") as distinct_results\n" +
|
||||
"order by coalesce(attempt_count, 0), coalesce(level, 0) desc, coalesce(pub_year, 0) desc, (case when regexp_like(url, '" + DOC_URL_FILTER + "', 'i') then 1 else 0 end) desc, reverse(pubid), url\n" + // We also order by reverse "pubid" and "url", in order to get the exactly same records for consecutive runs, all things being equal.
|
||||
"limit " + assignmentsLimit;
|
||||
" select pl.id, pl.original_url from " + ImpalaConnector.databaseName + ".payload pl\n" +
|
||||
" ) as existing\n" + // Here we access the payload-VIEW which includes the three payload-tables.
|
||||
" on existing.id=p.id and existing.original_url=pu.url\n" +
|
||||
" where d.allow_harvest=true and existing.id is null\n" +
|
||||
((excludedDatasourceIDsStringList != null) ? // If we have an exclusion-list, use it below.
|
||||
(" and d.id not in " + excludedDatasourceIDsStringList + "\n") : "") +
|
||||
" and coalesce(attempts.counts, 0) <= " + maxAttemptsPerRecordAtomic.get() + "\n" +
|
||||
" and not exists (select 1 from " + ImpalaConnector.databaseName + ".attempt a where a.id=p.id and a.error_class = 'noRetry' limit 1)\n" +
|
||||
" and pu.url != '' and pu.url is not null\n" + // Some IDs have empty-string urls, there are no "null" urls, but keep the relevant check for future-proofing.
|
||||
" limit " + (assignmentsLimit * 10) + ")\n" +
|
||||
" as non_distinct_results\n" +
|
||||
" order by coalesce(attempt_count, 0), coalesce(pub_year, 0) desc, reverse(pubid), url\n" +
|
||||
" limit " + assignmentsLimit + ")\n" +
|
||||
"as findAssignmentsQuery";
|
||||
|
||||
// The datasourceID and datasourceName are currently not used during the processing in the Worker. They may be used by the Worker, in the future to apply a datasource-specific aggregation plugin to take the full-texts quickly, instead of using the general crawling one.
|
||||
// However, the "datasourceid" is useful to be able to generate the fileNames for the S3, without needing to perform additional select queries (with JOINs) at that phase.
|
||||
|
||||
// The "order by" in the end makes sure the older attempted records will be re-attempted after a long time.
|
||||
//logger.trace("findAssignmentsQuery:\n" + findAssignmentsQuery); // DEBUG!
|
||||
|
||||
final String getAssignmentsQuery = "select * from " + DatabaseConnector.databaseName + ".current_assignment";
|
||||
final String getAssignmentsQuery = "select * from " + ImpalaConnector.databaseName + ".current_assignment";
|
||||
|
||||
final List<Assignment> assignments = new ArrayList<>(assignmentsLimit);
|
||||
List<Assignment> assignments = new ArrayList<>(assignmentsLimit);
|
||||
|
||||
long curAssignmentsBatchCounter = assignmentsBatchCounter.incrementAndGet();
|
||||
ImpalaConnector.databaseLock.lock();
|
||||
|
||||
DatabaseConnector.databaseLock.lock();
|
||||
|
||||
// For performance reasons, we collect the returned assignments and create a temp-table with them, so that later they can be copied efficiently to the "a-bit-more-permanent" assignment table.
|
||||
// This way, we avoid having to manually insert thousands of assignment records there. Instead, we create a new table AS the results from the "findAssignmentsQuery".
|
||||
String errorMsg = createAndInitializeCurrentAssignmentsTable(findAssignmentsQuery);
|
||||
if ( errorMsg != null ) {
|
||||
DatabaseConnector.databaseLock.unlock();
|
||||
ImpalaConnector.databaseLock.unlock();
|
||||
return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR).body(errorMsg);
|
||||
}
|
||||
|
||||
|
@ -169,10 +157,9 @@ public class UrlsServiceImpl implements UrlsService {
|
|||
jdbcTemplate.query(getAssignmentsQuery, rs -> {
|
||||
Assignment assignment = new Assignment();
|
||||
assignment.setWorkerId(workerId);
|
||||
assignment.setAssignmentsBatchCounter(curAssignmentsBatchCounter);
|
||||
assignment.setTimestamp(timestamp);
|
||||
Datasource datasource = new Datasource();
|
||||
try { // For each of the 4 columns returned, do the following. The column-indexing starts from 1.
|
||||
try { // For each of the 4 columns returned. The indexing starts from 1
|
||||
assignment.setId(rs.getString(1));
|
||||
assignment.setOriginalUrl(rs.getString(2));
|
||||
datasource.setId(rs.getString(3));
|
||||
|
@ -183,275 +170,256 @@ public class UrlsServiceImpl implements UrlsService {
|
|||
assignment.setDatasource(datasource);
|
||||
assignments.add(assignment);
|
||||
});
|
||||
} catch (EmptyResultDataAccessException erdae) {
|
||||
String tmpErrMsg = dropCurrentAssignmentTable();
|
||||
DatabaseConnector.databaseLock.unlock();
|
||||
errorMsg = "No results retrieved from the \"getAssignmentsQuery\" for worker with id: " + workerId + ". Will increase the \"maxAttempts\" to " + maxAttemptsPerRecordAtomic.incrementAndGet() + " for the next requests.";
|
||||
logger.error(errorMsg);
|
||||
if ( tmpErrMsg != null ) {
|
||||
errorMsg += GenericUtils.endOfLine + tmpErrMsg; // The additional error-msg is already logged.
|
||||
return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR).body(errorMsg);
|
||||
} else
|
||||
return ResponseEntity.status(HttpStatus.NO_CONTENT).body(new AssignmentsResponse((long) -1, null));
|
||||
} catch (Exception e) {
|
||||
errorMsg = ImpalaConnector.handleQueryException("getAssignmentsQuery", getAssignmentsQuery, e);
|
||||
String tmpErrMsg = dropCurrentAssignmentTable();
|
||||
DatabaseConnector.databaseLock.unlock();
|
||||
errorMsg = DatabaseConnector.handleQueryException("getAssignmentsQuery", getAssignmentsQuery, e);
|
||||
if ( tmpErrMsg != null )
|
||||
errorMsg += GenericUtils.endOfLine + tmpErrMsg;
|
||||
errorMsg += "\n" + tmpErrMsg;
|
||||
ImpalaConnector.databaseLock.unlock();
|
||||
return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR).body(errorMsg);
|
||||
}
|
||||
|
||||
int assignmentsSize = assignments.size();
|
||||
if ( assignmentsSize == 0 ) {
|
||||
String tmpErrMsg = dropCurrentAssignmentTable();
|
||||
DatabaseConnector.databaseLock.unlock();
|
||||
errorMsg = "Some results were retrieved from the \"getAssignmentsQuery\", but no data could be extracted from them, for worker with id: " + workerId;
|
||||
if ( tmpErrMsg != null )
|
||||
errorMsg += GenericUtils.endOfLine + tmpErrMsg;
|
||||
errorMsg = "No results retrieved from the \"findAssignmentsQuery\" for worker with id: " + workerId + ". Will increase the \"maxAttempts\" to " + maxAttemptsPerRecordAtomic.incrementAndGet() + " for the next requests.";
|
||||
logger.error(errorMsg);
|
||||
return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR).body(errorMsg);
|
||||
}
|
||||
else if ( assignmentsSize < assignmentsLimit )
|
||||
logger.warn("The retrieved results were fewer (" + assignmentsSize + ") than the \"assignmentsLimit\" (" + assignmentsLimit + "), for worker with id: " + workerId + ". Will increase the \"maxAttempts\" to " + maxAttemptsPerRecordAtomic.incrementAndGet() + ", for the next requests.");
|
||||
String tmpErrMsg = dropCurrentAssignmentTable();
|
||||
ImpalaConnector.databaseLock.unlock();
|
||||
if ( tmpErrMsg != null ) {
|
||||
errorMsg += "\n" + tmpErrMsg; // The additional error-msg is already logged.
|
||||
return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR).body(errorMsg);
|
||||
} else
|
||||
return ResponseEntity.status(HttpStatus.MULTI_STATUS).body(new AssignmentsResponse((long) -1, null));
|
||||
} else if ( assignmentsSize < assignmentsLimit )
|
||||
logger.warn("The retrieved results were fewer (" + assignmentsSize + ") than the \"assignmentsLimit\" (" + assignmentsLimit + "), for worker with id: " + workerId + ". Will increase the \"maxAttempts\" to " + maxAttemptsPerRecordAtomic.incrementAndGet() + " for the next requests.");
|
||||
|
||||
logger.debug("Finished gathering " + assignmentsSize + " assignments for worker with id \"" + workerId + "\". Going to insert them into the \"assignment\" table and then return them to the worker. | assignments_" + curAssignmentsBatchCounter);
|
||||
logger.debug("Finished gathering " + assignmentsSize + " assignments for worker with id \"" + workerId + "\". Going to insert them into the \"assignment\" table and then return them to the worker.");
|
||||
|
||||
// Write the Assignment details to the assignment-table.
|
||||
String insertAssignmentsQuery = "insert into " + DatabaseConnector.databaseName + ".assignment \n select pubid, url, '" + workerId + "', " + curAssignmentsBatchCounter + ", " + timestampMillis
|
||||
+ "\nfrom " + DatabaseConnector.databaseName + ".current_assignment";
|
||||
String insertAssignmentsQuery = "insert into " + ImpalaConnector.databaseName + ".assignment \n select pub_data.pubid, pub_data.url, '" + workerId + "', " + timestampMillis + "\n"
|
||||
+ "from (\n select pubid, url from " + ImpalaConnector.databaseName + ".current_assignment) as pub_data";
|
||||
|
||||
try {
|
||||
jdbcTemplate.execute(insertAssignmentsQuery);
|
||||
} catch (Exception e) {
|
||||
errorMsg = ImpalaConnector.handleQueryException("insertAssignmentsQuery", insertAssignmentsQuery, e);
|
||||
String tmpErrMsg = dropCurrentAssignmentTable();
|
||||
DatabaseConnector.databaseLock.unlock();
|
||||
errorMsg = DatabaseConnector.handleQueryException("insertAssignmentsQuery", insertAssignmentsQuery, e);
|
||||
if ( tmpErrMsg != null )
|
||||
errorMsg += GenericUtils.endOfLine + tmpErrMsg;
|
||||
errorMsg += "\n" + tmpErrMsg;
|
||||
ImpalaConnector.databaseLock.unlock();
|
||||
return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR).body(errorMsg);
|
||||
}
|
||||
|
||||
errorMsg = dropCurrentAssignmentTable();
|
||||
DatabaseConnector.databaseLock.unlock();
|
||||
if ( errorMsg != null )
|
||||
if ( errorMsg != null ) {
|
||||
ImpalaConnector.databaseLock.unlock();
|
||||
return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR).body(errorMsg);
|
||||
}
|
||||
|
||||
// We do not need to "merge" the parquet files for the "assignment" table here, since this happens every time we delete the assignments of a specific batch.
|
||||
logger.debug("Finished inserting " + assignmentsSize + " assignments into the \"assignment\"-table. Going to merge the parquet files for this table.");
|
||||
|
||||
logger.debug("Finished inserting " + assignmentsSize + " assignments into the \"assignment\"-table. | curAssignmentsBatchCounter: " + curAssignmentsBatchCounter + " | worker: " + workerId);
|
||||
String mergeErrorMsg = fileUtils.mergeParquetFiles("assignment", "", null);
|
||||
if ( mergeErrorMsg != null ) {
|
||||
ImpalaConnector.databaseLock.unlock();
|
||||
return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR).body(mergeErrorMsg);
|
||||
}
|
||||
|
||||
// Due to the fact that one publication with an id-url pair can be connected with multiple datasources, the results returned from the query may be duplicates.
|
||||
// So, we apply a post-processing step where we collect only one instance of each id-url pair and send it to the Worker.
|
||||
ImpalaConnector.databaseLock.unlock();
|
||||
|
||||
// We would certainly like the query to return assignments with unique id-url pairs, but for now there is no other solution.
|
||||
// There is no logic-error though, since the worker requests a "LIMIT" of 5000 not "exactly 5000" nor it matters the actual number it will process.
|
||||
// So, it will ask for "up to 5000" and the Controller may return 4700.
|
||||
// Since the average number of duplicates will be the same for each worker-request, none of the Workers will have any advantage over the other in the long run,
|
||||
// so there will be no conflicting performance data between them.
|
||||
|
||||
final HashMap<String, Assignment> uniquePairsAndAssignments = new HashMap<>((int) (assignmentsLimit * 0.9));
|
||||
|
||||
for ( Assignment assignment : assignments )
|
||||
uniquePairsAndAssignments.put(assignment.getId() + "_" + assignment.getOriginalUrl(), assignment);
|
||||
// This will just update the duplicate record with another "assignment object", containing a different datasource.
|
||||
|
||||
List<Assignment> distinctAssignments = new ArrayList<>(uniquePairsAndAssignments.values());
|
||||
int distinctAssignmentsSize = distinctAssignments.size();
|
||||
|
||||
if ( logger.isTraceEnabled() )
|
||||
logger.trace("numDuplicates in returned assignments_" + curAssignmentsBatchCounter + " = " + (assignmentsSize - distinctAssignmentsSize));
|
||||
|
||||
logger.info("Sending batch-assignments_" + curAssignmentsBatchCounter + ", with " + distinctAssignmentsSize + " assignments, to worker with ID: " + workerId + ".");
|
||||
return ResponseEntity.status(HttpStatus.OK).body(new AssignmentsResponse(curAssignmentsBatchCounter, distinctAssignments));
|
||||
long curAssignmentsBatchCounter = assignmentsBatchCounter.incrementAndGet();
|
||||
logger.info("Sending batch-assignments_" + curAssignmentsBatchCounter + " with " + assignmentsSize + " assignments to worker with ID: " + workerId + ".");
|
||||
return ResponseEntity.status(HttpStatus.OK).body(new AssignmentsResponse(curAssignmentsBatchCounter, assignments));
|
||||
}
|
||||
|
||||
|
||||
|
||||
public static final AtomicLong numOfWorkerReportsProcessed = new AtomicLong(0);
|
||||
|
||||
|
||||
@Timed(value = "addWorkerReport.time", description = "Time taken to add the WorkerReport.")
|
||||
public Boolean addWorkerReport(String curWorkerId, long curReportAssignmentsCounter, List<UrlReport> urlReports, int sizeOfUrlReports)
|
||||
public Boolean addWorkerReport(String curWorkerId, long curReportAssignments, List<UrlReport> urlReports, int sizeOfUrlReports)
|
||||
{
|
||||
long currentNumOfWorkerReportsProcessed = numOfWorkerReportsProcessed.incrementAndGet(); // Increment it when it is actually being pressed, not when it arrives at the endpoint.
|
||||
|
||||
logger.info("Initializing the addition of the worker's (" + curWorkerId + ") report for assignments_" + curReportAssignmentsCounter);
|
||||
|
||||
boolean hasFulltexts = true;
|
||||
logger.info("Initializing the addition of the worker's (" + curWorkerId + ") report for assignments_" + assignmentsBatchCounter);
|
||||
|
||||
// Before continuing with inserts, take and upload the fullTexts from the Worker. Also, update the file-"location".
|
||||
FileUtils.UploadFullTextsResponse uploadFullTextsResponse = filesHandler.getAndUploadFullTexts(urlReports, sizeOfUrlReports, curReportAssignmentsCounter, curWorkerId);
|
||||
if ( uploadFullTextsResponse == null ) {
|
||||
// Nothing to post to the Worker, since we do not have the worker's info.
|
||||
// Rename the worker-report-file to indicate its "failure", so that the scheduler can pick it up and retry processing it.
|
||||
String workerReportBaseName = this.workerReportsDirPath + File.separator + curWorkerId + File.separator + curWorkerId + "_assignments_" + curReportAssignmentsCounter + "_report";
|
||||
renameAndGetWorkerReportFile(workerReportBaseName, new File(workerReportBaseName + ".json"), "No info was found for worker: " + curWorkerId); // It may return null.
|
||||
FileUtils.UploadFullTextsResponse uploadFullTextsResponse = fileUtils.getAndUploadFullTexts(urlReports, curReportAssignments, curWorkerId);
|
||||
if ( uploadFullTextsResponse == FileUtils.UploadFullTextsResponse.databaseError ) {
|
||||
postReportResultToWorker(curWorkerId, curReportAssignments, "Problem with the Impala-database!");
|
||||
return false;
|
||||
} else if ( uploadFullTextsResponse == FileUtils.UploadFullTextsResponse.unsuccessful ) {
|
||||
logger.error("Failed to get and/or upload the fullTexts for batch-assignments_" + curReportAssignmentsCounter);
|
||||
}
|
||||
else if ( uploadFullTextsResponse == FileUtils.UploadFullTextsResponse.unsuccessful ) {
|
||||
logger.error("Failed to get and/or upload the fullTexts for batch-assignments_" + curReportAssignments);
|
||||
// The docUrls were still found! Just update ALL the fileLocations, sizes, hashes and mimetypes, to show that the files are not available.
|
||||
filesHandler.removeUnretrievedFullTextsFromUrlReports(urlReports, false);
|
||||
fileUtils.updateUrlReportsToHaveNoFullTextFiles(urlReports, false);
|
||||
// We write only the payloads which are connected with retrieved full-texts, uploaded to S3-Object-Store.
|
||||
// We continue with writing the "attempts", as we want to avoid re-checking the failed-urls later.
|
||||
// The urls which give full-text (no matter if we could not get it from the worker), are flagged as "couldRetry" anyway, so they will be picked-up to be checked again later.
|
||||
hasFulltexts = false;
|
||||
} else if ( uploadFullTextsResponse == FileUtils.UploadFullTextsResponse.successful_without_fulltexts )
|
||||
hasFulltexts = false;
|
||||
else
|
||||
logger.debug("Finished uploading the full-texts from batch-assignments_" + curReportAssignmentsCounter);
|
||||
} else
|
||||
logger.debug("Finished uploading the full-texts from batch-assignments_" + curReportAssignments);
|
||||
|
||||
String localParquetPath = parquetFileUtils.parquetBaseLocalDirectoryPath + "assignments_" + curReportAssignmentsCounter + File.separator;
|
||||
String localParquetPath = parquetFileUtils.parquetBaseLocalDirectoryPath + "assignments_" + curReportAssignments + File.separator;
|
||||
try {
|
||||
Files.createDirectories(Paths.get(localParquetPath)); // No-op if it already exists. It does not throw a "alreadyExistsException"
|
||||
} catch (Exception e) {
|
||||
String errorMsg = "Could not create the parquet-directory: " + localParquetPath;
|
||||
logger.error(errorMsg, e);
|
||||
postReportResultToWorker(curWorkerId, curReportAssignmentsCounter, errorMsg);
|
||||
postReportResultToWorker(curWorkerId, curReportAssignments, errorMsg);
|
||||
return false;
|
||||
}
|
||||
|
||||
logger.debug("Going to write the results in the parquet files, then upload them to HDFS, and then load them into the database's tables. For batch-assignments_" + curReportAssignmentsCounter);
|
||||
logger.debug("Going to write the results in the parquet files, then upload them to HDFS, and then load them into the database's tables. For batch-assignments_" + curReportAssignments);
|
||||
|
||||
List<Callable<ParquetReport>> callableTasks = parquetFileUtils.getTasksForCreatingAndUploadingParquetFiles(urlReports, sizeOfUrlReports, curReportAssignments, localParquetPath, uploadFullTextsResponse);
|
||||
|
||||
// Create HDFS subDirs for these assignments. Other background threads handling other assignments will not interfere with loading of parquetFiles to the DB tables.
|
||||
String endingMkDirAndParams = curReportAssignmentsCounter + "/" + parquetFileUtils.mkDirsAndParams;
|
||||
if ( !parquetFileUtils.applyHDFSOperation(parquetFileUtils.webHDFSBaseUrl + parquetFileUtils.parquetHDFSDirectoryPathAttempts + endingMkDirAndParams)
|
||||
|| (hasFulltexts && !parquetFileUtils.applyHDFSOperation(parquetFileUtils.webHDFSBaseUrl + parquetFileUtils.parquetHDFSDirectoryPathPayloadsAggregated + endingMkDirAndParams)) )
|
||||
String endingMkDirAndParams = curReportAssignments + "/" + parquetFileUtils.mkDirsAndParams;
|
||||
if ( !parquetFileUtils.applyHDFOperation(parquetFileUtils.webHDFSBaseUrl + parquetFileUtils.parquetHDFSDirectoryPathAttempts + endingMkDirAndParams)
|
||||
|| !parquetFileUtils.applyHDFOperation(parquetFileUtils.webHDFSBaseUrl + parquetFileUtils.parquetHDFSDirectoryPathPayloadsAggregated + endingMkDirAndParams) )
|
||||
{
|
||||
postReportResultToWorker(curWorkerId, curReportAssignmentsCounter, "Error when creating the HDFS sub-directories for assignments_" + curReportAssignmentsCounter);
|
||||
postReportResultToWorker(curWorkerId, curReportAssignments, "Error when creating the HDFS sub-directories for assignments_" + curReportAssignments);
|
||||
return false;
|
||||
}
|
||||
|
||||
boolean hasAttemptParquetFileProblem = false;
|
||||
boolean hasPayloadParquetFileProblem = false;
|
||||
|
||||
DatabaseConnector.databaseLock.lock();
|
||||
// Lock the DB here so the prefilled-Payloads which will be generated inside the "getTasksForCreatingAndUploadingParquetFiles()" method (using a dedicated query)
|
||||
// will be synchronized with the insert of all attempt and payload records to the DB. This action is NOT a callable-task, so it runs during the execution of this method.
|
||||
// This is important in order to avoid having workers take these records as assignments, when we know that payloads are ready to be inserted for them.
|
||||
|
||||
List<Callable<ParquetReport>> callableTasks = parquetFileUtils.getTasksForCreatingAndUploadingParquetFiles(urlReports, sizeOfUrlReports, curReportAssignmentsCounter, localParquetPath, uploadFullTextsResponse);
|
||||
|
||||
try { // Invoke all the tasks and wait for them to finish.
|
||||
List<Future<ParquetReport>> futures = insertsExecutor.invokeAll(callableTasks);
|
||||
|
||||
SumParquetSuccess sumParquetSuccess = parquetFileUtils.checkParquetFilesSuccess(futures);
|
||||
ResponseEntity<?> errorResponseEntity = sumParquetSuccess.getResponseEntity();
|
||||
if ( errorResponseEntity != null ) { // The related log is already shown in this case.
|
||||
DatabaseConnector.databaseLock.unlock();
|
||||
postReportResultToWorker(curWorkerId, curReportAssignmentsCounter, "Error when creating or uploading the parquet files!");
|
||||
if ( errorResponseEntity != null ) { // The related log is already shown.
|
||||
postReportResultToWorker(curWorkerId, curReportAssignments, "Error when creating or uploading the parquet files!");
|
||||
return false;
|
||||
}
|
||||
hasAttemptParquetFileProblem = sumParquetSuccess.isAttemptParquetFileProblem();
|
||||
hasPayloadParquetFileProblem = sumParquetSuccess.isPayloadParquetFileProblem();
|
||||
|
||||
if ( hasAttemptParquetFileProblem && hasPayloadParquetFileProblem ) {
|
||||
DatabaseConnector.databaseLock.unlock();
|
||||
throw new RuntimeException("All of the parquet files failed to be created or uploaded! Will avoid to execute load-requests into the database, for batch-assignments_" + curReportAssignmentsCounter);
|
||||
} else {
|
||||
if ( hasAttemptParquetFileProblem && hasPayloadParquetFileProblem )
|
||||
throw new RuntimeException("All of the parquet files failed to be created or uploaded! Will avoid to execute load-requests into the database, for batch-assignments_" + curReportAssignments);
|
||||
else {
|
||||
if ( hasAttemptParquetFileProblem )
|
||||
logger.error("All of the attempt-parquet files failed to be created or uploaded! Will avoid to execute load-requests into the database-table \"attempt\", for batch-assignments_" + curReportAssignmentsCounter);
|
||||
logger.error("All of the attempt-parquet files failed to be created or uploaded! Will avoid to execute load-requests into the database-table \"attempt\", for batch-assignments_" + curReportAssignments);
|
||||
else if ( hasPayloadParquetFileProblem )
|
||||
logger.error("All of the payload-parquet files failed to be created or uploaded! Will avoid to execute load-requests into the database-table \"payload_aggregated\", for batch-assignments_" + curReportAssignmentsCounter);
|
||||
logger.error("The single payload-parquet-file failed to be created or uploaded! Will avoid to execute load-requests into the database-table \"payload_aggregated\", for batch-assignments_" + curReportAssignments);
|
||||
else
|
||||
logger.debug("Going to execute \"load\"-requests on the database, for the uploaded parquet-files, for batch-assignments_" + curReportAssignmentsCounter);
|
||||
logger.debug("Going to execute \"load\"-requests on the database, for the uploaded parquet-files.");
|
||||
}
|
||||
|
||||
// Load all the parquet files of each type into its table.
|
||||
ImpalaConnector.databaseLock.lock();
|
||||
|
||||
if ( ! hasAttemptParquetFileProblem )
|
||||
hasAttemptParquetFileProblem = ! parquetFileUtils.loadParquetDataIntoTable(parquetFileUtils.parquetHDFSDirectoryPathAttempts + curReportAssignmentsCounter + "/", "attempt");
|
||||
hasAttemptParquetFileProblem = ! parquetFileUtils.loadParquetDataIntoTable(parquetFileUtils.parquetHDFSDirectoryPathAttempts + curReportAssignments + "/", "attempt");
|
||||
|
||||
if ( hasFulltexts && ! hasPayloadParquetFileProblem )
|
||||
hasPayloadParquetFileProblem = ! parquetFileUtils.loadParquetDataIntoTable(parquetFileUtils.parquetHDFSDirectoryPathPayloadsAggregated + curReportAssignmentsCounter + "/", "payload_aggregated");
|
||||
if ( ! hasPayloadParquetFileProblem )
|
||||
hasPayloadParquetFileProblem = ! parquetFileUtils.loadParquetDataIntoTable(parquetFileUtils.parquetHDFSDirectoryPathPayloadsAggregated + curReportAssignments + "/", "payload_aggregated");
|
||||
|
||||
DatabaseConnector.databaseLock.unlock();
|
||||
ImpalaConnector.databaseLock.unlock();
|
||||
|
||||
if ( hasAttemptParquetFileProblem && hasPayloadParquetFileProblem )
|
||||
throw new RuntimeException("The data from the HDFS parquet sub-directories COULD NOT be loaded into the \"attempt\" and the \"payload_aggregated\" tables, for batch-assignments_" + curReportAssignmentsCounter);
|
||||
throw new RuntimeException("The data from the HDFS parquet sub-directories COULD NOT be loaded into the \"attempt\" and the \"payload_aggregated\" tables, for batch-assignments_" + curReportAssignments);
|
||||
else if ( hasAttemptParquetFileProblem || hasPayloadParquetFileProblem )
|
||||
logger.error("The data from the HDFS parquet sub-directories COULD NOT be loaded into the \"attempt\" or the \"payload_aggregated\" table, for batch-assignments_" + curReportAssignmentsCounter);
|
||||
logger.error("The data from the HDFS parquet sub-directories COULD NOT be loaded into the \"attempt\" or the \"payload_aggregated\" table, for batch-assignments_" + curReportAssignments);
|
||||
else
|
||||
logger.debug("The data from the HDFS parquet sub-directories was loaded into the \"attempt\" " + (hasFulltexts ? "and the \"payload_aggregated\" tables" : "table") + ", for batch-assignments_" + curReportAssignmentsCounter);
|
||||
logger.debug("The data from the HDFS parquet sub-directories was loaded into the \"attempt\" and the \"payload_aggregated\" tables, for batch-assignments_" + curReportAssignments);
|
||||
|
||||
} catch (InterruptedException ie) { // Thrown by "insertsExecutor.invokeAll()". In this case, any unfinished tasks are cancelled.
|
||||
DatabaseConnector.databaseLock.unlock();
|
||||
} catch (InterruptedException ie) { // In this case, any unfinished tasks are cancelled.
|
||||
logger.warn("The current thread was interrupted when waiting for the worker-threads to finish inserting into the tables: " + ie.getMessage());
|
||||
// This is a very rare case. At the moment, we just move on with table-merging.
|
||||
} catch (RuntimeException re) {
|
||||
// Only one of the REs is inside the DB-locked code, so the "unlock" happens before it's thrown, there.
|
||||
String errorMsg = re.getMessage();
|
||||
ImpalaConnector.databaseLock.lock();
|
||||
String assignmentErrorMsg = deleteWorkerAssignments(curWorkerId);
|
||||
ImpalaConnector.databaseLock.unlock();
|
||||
if ( assignmentErrorMsg != null )
|
||||
errorMsg += "\n" + assignmentErrorMsg;
|
||||
logger.error(errorMsg);
|
||||
postReportResultToWorker(curWorkerId, curReportAssignmentsCounter, errorMsg);
|
||||
postReportResultToWorker(curWorkerId, curReportAssignments, errorMsg);
|
||||
return false;
|
||||
} catch (Exception e) { // Thrown by "insertsExecutor.invokeAll()".
|
||||
DatabaseConnector.databaseLock.unlock();
|
||||
} catch (Exception e) {
|
||||
String errorMsg = "Unexpected error when inserting into the \"attempt\" and \"payload_aggregated\" tables in parallel! " + e.getMessage();
|
||||
logger.error(errorMsg, e);
|
||||
postReportResultToWorker(curWorkerId, curReportAssignmentsCounter, errorMsg);
|
||||
postReportResultToWorker(curWorkerId, curReportAssignments, errorMsg);
|
||||
return false; // No tables-merging is happening.
|
||||
} finally {
|
||||
logger.debug("Deleting parquet directory: " + localParquetPath);
|
||||
fileUtils.deleteDirectory(new File(localParquetPath));
|
||||
// Delete the HDFS subDirs for this Report.
|
||||
String endingRmDirAndParams = curReportAssignmentsCounter + "/" + parquetFileUtils.rmDirsAndParams;
|
||||
if ( !parquetFileUtils.applyHDFSOperation(parquetFileUtils.webHDFSBaseUrl + parquetFileUtils.parquetHDFSDirectoryPathAttempts + endingRmDirAndParams)
|
||||
|| (hasFulltexts && !parquetFileUtils.applyHDFSOperation(parquetFileUtils.webHDFSBaseUrl + parquetFileUtils.parquetHDFSDirectoryPathPayloadsAggregated + endingRmDirAndParams)) )
|
||||
String endingRmDirAndParams = curReportAssignments + "/" + parquetFileUtils.rmDirsAndParams;
|
||||
if ( !parquetFileUtils.applyHDFOperation(parquetFileUtils.webHDFSBaseUrl + parquetFileUtils.parquetHDFSDirectoryPathAttempts + endingRmDirAndParams)
|
||||
|| !parquetFileUtils.applyHDFOperation(parquetFileUtils.webHDFSBaseUrl + parquetFileUtils.parquetHDFSDirectoryPathPayloadsAggregated + endingRmDirAndParams) )
|
||||
{
|
||||
logger.error("Error when deleting the HDFS sub-directories for assignments_" + curReportAssignmentsCounter); // A directory-specific log has already appeared.
|
||||
logger.error("Error when deleting the HDFS sub-directories for assignments_" + curReportAssignments); // A directory-specific log has already appeared.
|
||||
// The failure to delete the assignments_subDirs is not that of a problem and should not erase the whole process. So all goes as planned (the worker deletes any remaining files).
|
||||
// The worst case is that a few subDirs will be left back in the HDFS, although, without their parquetFiles, since they have already moved inside the DB tables.
|
||||
}
|
||||
}
|
||||
|
||||
// Delete the assignments each time, as they are bound to the "current" assignmentsCounter. Otherwise, they will never be deleted!
|
||||
// If this method exits sooner, due to an error, then the assignments are not deleted in order to wait for the schedulers to retry them and not be given to workers, to avoid reprocessing the urls.
|
||||
DatabaseConnector.databaseLock.lock();
|
||||
String deleteErrorMsg = deleteAssignmentsBatch(curReportAssignmentsCounter);
|
||||
DatabaseConnector.databaseLock.unlock();
|
||||
if ( deleteErrorMsg != null ) {
|
||||
postReportResultToWorker(curWorkerId, curReportAssignmentsCounter, deleteErrorMsg);
|
||||
logger.debug("Going to merge the parquet files for the tables which were altered.");
|
||||
// When the uploaded parquet files are "loaded" into the tables, they are actually moved into the directory which contains the data of the table.
|
||||
|
||||
String mergeErrorMsg;
|
||||
|
||||
ImpalaConnector.databaseLock.lock();
|
||||
|
||||
if ( ! hasAttemptParquetFileProblem ) {
|
||||
mergeErrorMsg = fileUtils.mergeParquetFiles("attempt", "", null);
|
||||
if ( mergeErrorMsg != null ) {
|
||||
ImpalaConnector.databaseLock.unlock();
|
||||
postReportResultToWorker(curWorkerId, curReportAssignments, mergeErrorMsg);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
if ( ! hasPayloadParquetFileProblem ) {
|
||||
mergeErrorMsg = fileUtils.mergeParquetFiles("payload_aggregated", "", null);
|
||||
if ( mergeErrorMsg != null ) {
|
||||
ImpalaConnector.databaseLock.unlock();
|
||||
postReportResultToWorker(curWorkerId, curReportAssignments, mergeErrorMsg);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
mergeErrorMsg = deleteWorkerAssignments(curWorkerId);
|
||||
if ( mergeErrorMsg != null ) {
|
||||
ImpalaConnector.databaseLock.unlock();
|
||||
postReportResultToWorker(curWorkerId, curReportAssignments, mergeErrorMsg);
|
||||
return false;
|
||||
}
|
||||
|
||||
// For every "numOfWorkers" assignment-batches that get processed, we merge the "attempts" and "payload_aggregated" tables.
|
||||
if ( (currentNumOfWorkerReportsProcessed % UrlsController.numOfActiveWorkers.get()) == 0 ) // The workersNum will not be zero!
|
||||
if ( ! mergeWorkerRelatedTables(curWorkerId, curReportAssignmentsCounter, hasAttemptParquetFileProblem, hasPayloadParquetFileProblem, hasFulltexts) )
|
||||
// The "postReportResultToWorker()" was called inside.
|
||||
return false;
|
||||
ImpalaConnector.databaseLock.unlock();
|
||||
|
||||
logger.debug("Finished merging the database tables.");
|
||||
if ( uploadFullTextsResponse == FileUtils.UploadFullTextsResponse.unsuccessful ) {
|
||||
postReportResultToWorker(curWorkerId, curReportAssignmentsCounter, "The full-text files failed to be acquired from the worker!");
|
||||
postReportResultToWorker(curWorkerId, curReportAssignments, "The full-text files failed to be acquired from the worker!");
|
||||
return false;
|
||||
}
|
||||
|
||||
// Notify the Worker that the processing of this report was successful, so that the Worker can delete the files.
|
||||
postReportResultToWorker(curWorkerId, curReportAssignmentsCounter, null);
|
||||
postReportResultToWorker(curWorkerId, curReportAssignments, null);
|
||||
return true;
|
||||
}
|
||||
|
||||
|
||||
private String createAndInitializeCurrentAssignmentsTable(String findAssignmentsQuery)
|
||||
{
|
||||
String createCurrentAssignmentsQuery = "create table " + DatabaseConnector.databaseName + ".current_assignment as \n" + findAssignmentsQuery;
|
||||
final String computeCurrentAssignmentsStatsQuery = "COMPUTE STATS " + DatabaseConnector.databaseName + ".current_assignment";
|
||||
String createCurrentAssignmentsQuery = "create table " + ImpalaConnector.databaseName + ".current_assignment as \n" + findAssignmentsQuery;
|
||||
String computeCurrentAssignmentsStatsQuery = "COMPUTE STATS " + ImpalaConnector.databaseName + ".current_assignment";
|
||||
|
||||
try {
|
||||
jdbcTemplate.execute(createCurrentAssignmentsQuery);
|
||||
} catch (Exception e) {
|
||||
String errorMsg = DatabaseConnector.handleQueryException("createCurrentAssignmentsQuery", createCurrentAssignmentsQuery, e);
|
||||
String errorMsg = ImpalaConnector.handleQueryException("createCurrentAssignmentsQuery", createCurrentAssignmentsQuery, e);
|
||||
String tmpErrMsg = dropCurrentAssignmentTable(); // The table may be partially created, e.g. in case of an "out of memory" error in the database-server, during the creation, resulting in an empty table (yes it has happened).
|
||||
if ( tmpErrMsg != null )
|
||||
errorMsg += GenericUtils.endOfLine + tmpErrMsg;
|
||||
errorMsg += "\n" + tmpErrMsg;
|
||||
return errorMsg;
|
||||
}
|
||||
|
||||
try {
|
||||
jdbcTemplate.execute(computeCurrentAssignmentsStatsQuery);
|
||||
} catch (Exception e) {
|
||||
String errorMsg = DatabaseConnector.handleQueryException("computeCurrentAssignmentsStatsQuery", computeCurrentAssignmentsStatsQuery, e);
|
||||
String errorMsg = ImpalaConnector.handleQueryException("computeCurrentAssignmentsStatsQuery", computeCurrentAssignmentsStatsQuery, e);
|
||||
String tmpErrMsg = dropCurrentAssignmentTable();
|
||||
if ( tmpErrMsg != null )
|
||||
errorMsg += GenericUtils.endOfLine + tmpErrMsg;
|
||||
errorMsg += "\n" + tmpErrMsg;
|
||||
return errorMsg;
|
||||
}
|
||||
|
||||
|
@ -460,147 +428,58 @@ public class UrlsServiceImpl implements UrlsService {
|
|||
|
||||
|
||||
private String dropCurrentAssignmentTable() {
|
||||
final String dropCurrentAssignmentsQuery = "DROP TABLE IF EXISTS " + DatabaseConnector.databaseName + ".current_assignment PURGE";
|
||||
String dropCurrentAssignmentsQuery = "DROP TABLE IF EXISTS " + ImpalaConnector.databaseName + ".current_assignment PURGE";
|
||||
try {
|
||||
jdbcTemplate.execute(dropCurrentAssignmentsQuery);
|
||||
return null; // All good. No error-message.
|
||||
} catch (Exception e) {
|
||||
return DatabaseConnector.handleQueryException("dropCurrentAssignmentsQuery", dropCurrentAssignmentsQuery, e); // The error is already logged inside.
|
||||
return ImpalaConnector.handleQueryException("dropCurrentAssignmentsQuery", dropCurrentAssignmentsQuery, e); // The error is already logged inside.
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
private boolean mergeWorkerRelatedTables(String curWorkerId, long curReportAssignmentsCounter, boolean hasAttemptParquetFileProblem, boolean hasPayloadParquetFileProblem, boolean hasFulltexts)
|
||||
private String deleteWorkerAssignments(String curWorkerId)
|
||||
{
|
||||
logger.debug("Going to merge the parquet files for the tables which were altered, for batch-assignments_" + curReportAssignmentsCounter);
|
||||
// When the uploaded parquet files are "loaded" into the tables, they are actually moved into the directory which contains the data of the table.
|
||||
// This means that over time a table may have thousands of parquet files and the search through them will be very slow. Thus, we merge them regularly.
|
||||
|
||||
String mergeErrorMsg;
|
||||
|
||||
DatabaseConnector.databaseLock.lock();
|
||||
|
||||
if ( ! hasAttemptParquetFileProblem ) {
|
||||
mergeErrorMsg = parquetFileUtils.mergeParquetFilesOfTable("attempt", "", null);
|
||||
if ( mergeErrorMsg != null ) {
|
||||
DatabaseConnector.databaseLock.unlock();
|
||||
postReportResultToWorker(curWorkerId, curReportAssignmentsCounter, mergeErrorMsg);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
if ( hasFulltexts && ! hasPayloadParquetFileProblem ) {
|
||||
mergeErrorMsg = parquetFileUtils.mergeParquetFilesOfTable("payload_aggregated", "", null);
|
||||
if ( mergeErrorMsg != null ) {
|
||||
DatabaseConnector.databaseLock.unlock();
|
||||
postReportResultToWorker(curWorkerId, curReportAssignmentsCounter, mergeErrorMsg);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
DatabaseConnector.databaseLock.unlock();
|
||||
logger.debug("Finished merging the database tables, for batch-assignments_" + curReportAssignmentsCounter);
|
||||
return true;
|
||||
}
|
||||
|
||||
|
||||
public String deleteAssignmentsBatch(long givenAssignmentsBatchCounter)
|
||||
{
|
||||
// This will delete the rows of the "assignment" table which refer to the "givenAssignmentsBatchCounter".
|
||||
// As we have non-KUDU Impala tables, the Delete operation can only succeed through a "merge" operation of the rest of the data.
|
||||
// Only the rows referring to OTHER "givenAssignmentsBatchCounter" get stored in a temp-table, while the "assignment" table gets deleted. Then, the temp_table becomes the "assignment" table.
|
||||
// We don't need to keep the assignment-info forever, as the "findAssignmentsQuery" checks the "payload" table for previously handled tasks.
|
||||
return parquetFileUtils.mergeParquetFilesOfTable("assignment", " WHERE assignments_batch_counter != ", givenAssignmentsBatchCounter);
|
||||
}
|
||||
|
||||
|
||||
public String deleteAssignmentsWithOlderDate(long givenDate)
|
||||
{
|
||||
// This will delete the rows of the "assignment" table which are older than "givenDate".
|
||||
// As we have non-KUDU Impala tables, the Delete operation can only succeed through a "merge" operation of the rest of the data.
|
||||
// Only the rows referring to NEWER than "givenDate" get stored in a temp-table, while the "assignment" table gets deleted. Then, the temp_table becomes the "assignment" table.
|
||||
// We don't need to keep the assignment-info forever, as the "findAssignmentsQuery" checks the "payload" table for previously handled tasks.
|
||||
return parquetFileUtils.mergeParquetFilesOfTable("assignment", " WHERE `date` >= ", givenDate);
|
||||
// This will delete the rows of the "assignment" table which refer to the "curWorkerId". As we have non-KUDU Impala tables, the Delete operation can only succeed through a "merge" operation of the rest of the data.
|
||||
// Only the rows referring to OTHER workerIDs get stored in a temp-table, while the "assignment" table gets deleted. Then, the temp_table becomes the "assignment" table.
|
||||
// We don't need to keep the assignment-info anymore, the "findAssignmentsQuery" checks the "payload_aggregated" table for previously handled tasks.
|
||||
return fileUtils.mergeParquetFiles("assignment", " WHERE workerid != ", curWorkerId);
|
||||
}
|
||||
|
||||
|
||||
private static final RestTemplate restTemplate = new RestTemplate();
|
||||
|
||||
private boolean postReportResultToWorker(String workerId, long assignmentRequestCounter, String errorMsg)
|
||||
{
|
||||
// Rename the worker-report to indicate success or failure.
|
||||
String workerReportBaseName = this.workerReportsDirPath + File.separator + workerId + File.separator + workerId + "_assignments_" + assignmentRequestCounter + "_report";
|
||||
File workerReport = new File(workerReportBaseName + ".json");
|
||||
File renamedWorkerReport = renameAndGetWorkerReportFile(workerReportBaseName, workerReport, errorMsg); // It may return null.
|
||||
|
||||
// Get the IP of this worker.
|
||||
WorkerInfo workerInfo = UrlsController.workersInfoMap.get(workerId);
|
||||
if ( workerInfo == null ) {
|
||||
logger.error("Could not find any info for worker with id: \"" + workerId +"\".");
|
||||
return false;
|
||||
}
|
||||
String url = "http://" + workerInfo.getWorkerIP() + ":" + workerPort + "/api/addReportResultToWorker/" + assignmentRequestCounter; // This workerIP will NOT be null.
|
||||
String url = "http://" + workerInfo.getWorkerIP() + ":1881/api/addReportResultToWorker/" + assignmentRequestCounter; // This workerIP will not be null.
|
||||
|
||||
if ( logger.isTraceEnabled() )
|
||||
logger.trace("Going to \"postReportResultToWorker\": \"" + workerId + "\", for assignments_" + assignmentRequestCounter + ((errorMsg != null) ? "\nError: " + errorMsg : ""));
|
||||
|
||||
try {
|
||||
ResponseEntity<String> responseEntity = restTemplate.postForEntity(url, errorMsg, String.class); // We may pass a "null" entity.
|
||||
ResponseEntity<String> responseEntity = new RestTemplate().postForEntity(url, errorMsg, String.class); // We may pass a "null" entity.
|
||||
int responseCode = responseEntity.getStatusCodeValue();
|
||||
if ( responseCode != 200 ) {
|
||||
if ( responseCode != HttpStatus.OK.value() ) {
|
||||
logger.error("HTTP-Connection problem with the submission of the \"postReportResultToWorker\" of worker \"" + workerId + "\" and assignments_" + assignmentRequestCounter + "! Error-code was: " + responseCode);
|
||||
return false;
|
||||
} else if ( errorMsg == null ) // If the worker was notified, then go delete the successful workerReport.
|
||||
fileUtils.deleteFile((renamedWorkerReport != null) ? renamedWorkerReport.getAbsolutePath() : workerReport.getAbsolutePath());
|
||||
return true;
|
||||
} else {
|
||||
fileUtils.deleteFile(workerReportsDirPath + "/" + workerId + "/" + workerId + "_assignments_" + assignmentRequestCounter + "_report.json");
|
||||
return true;
|
||||
}
|
||||
} catch (HttpServerErrorException hsee) {
|
||||
logger.error("The Worker \"" + workerId + "\" failed to handle the \"postReportResultToWorker\", of assignments_" + assignmentRequestCounter + ": " + hsee.getMessage());
|
||||
return false;
|
||||
} catch (HttpClientErrorException hcee) {
|
||||
logger.error("The Controller did something wrong when sending the report result to the Worker (" + workerId + "). | url: " + url + GenericUtils.endOfLine + hcee.getMessage());
|
||||
return false;
|
||||
} catch (Exception e) {
|
||||
errorMsg = "Error for \"postReportResultToWorker\", of assignments_" + assignmentRequestCounter + " to the Worker: " + workerId;
|
||||
Throwable cause = e.getCause();
|
||||
String exMsg;
|
||||
if ( (cause != null) && ((exMsg = cause.getMessage()) != null) && exMsg.contains("Connection refused") ) {
|
||||
logger.error(errorMsg + " | The worker has probably crashed, since we received a \"Connection refused\" message!");
|
||||
workerInfo.setHasShutdown(true); // Avoid sending possible shutdown-Requests later on. Also show a specific message if this Worker requests new assignments in the future.
|
||||
} else
|
||||
logger.error(errorMsg, e);
|
||||
logger.error("Error for \"postReportResultToWorker\", of assignments_" + assignmentRequestCounter + " to the Worker: " + workerId, e);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
private static File renameAndGetWorkerReportFile(String workerReportBaseName, File workerReport, String errorMsg)
|
||||
{
|
||||
File renamedWorkerReport = null;
|
||||
try {
|
||||
// Check if the workerReport does not exist under this name, as it may have been renamed previously to "failed" and now is the 2nd try.
|
||||
if ( !workerReport.isFile() ) {
|
||||
// Then this is the 2nd try and the report has already been renamed to "failed" the 1st time.
|
||||
workerReport = new File(workerReportBaseName + "_failed.json");
|
||||
if ( !workerReport.isFile() ) { // In this case, an error exists, since this file was deleted before its time.
|
||||
logger.error("The workerReport file \"" + workerReport.getAbsolutePath() + "\" does not exist!");
|
||||
// TODO - Do we need additional handling? This report may be either successful or failed but the file was deleted.
|
||||
return null; // Do not proceed on renaming the non-existing file.
|
||||
}
|
||||
}
|
||||
// Only if the file exists, proceed to renaming it.
|
||||
renamedWorkerReport = new File(workerReportBaseName + ((errorMsg == null) ? "_successful.json" : "_failed.json"));
|
||||
if ( ! workerReport.renameTo(renamedWorkerReport) ) {
|
||||
logger.warn("There was a problem when renaming the workerReport: " + workerReport.getName());
|
||||
return null;
|
||||
}
|
||||
} catch (Exception e) {
|
||||
logger.error("There was a problem when renaming the workerReport: " + workerReport.getName(), e);
|
||||
return null;
|
||||
}
|
||||
return renamedWorkerReport;
|
||||
}
|
||||
|
||||
|
||||
// The "batchExecute" does not work in this Impala-Database, so this is a "giant-query" solution.
|
||||
// Note: this causes an "Out of memory"-ERROR in the current version of the Impala JDBC driver. If a later version is provided, then this code should be tested.
|
||||
private static PreparedStatement constructLargeInsertQuery(Connection con, String baseInsertQuery, int dataSize, int numParamsPerRow) throws RuntimeException {
|
||||
|
|
|
@ -1,64 +1,29 @@
|
|||
package eu.openaire.urls_controller.util;
|
||||
|
||||
import com.google.common.collect.SetMultimap;
|
||||
import eu.openaire.urls_controller.models.Payload;
|
||||
import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
|
||||
import org.apache.commons.compress.archivers.tar.TarArchiveInputStream;
|
||||
import org.apache.commons.compress.compressors.zstandard.ZstdCompressorInputStream;
|
||||
import org.apache.commons.lang.StringUtils;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
import java.io.BufferedInputStream;
|
||||
import java.io.BufferedOutputStream;
|
||||
import java.io.File;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.nio.file.Paths;
|
||||
import java.util.List;
|
||||
|
||||
@Component
|
||||
public class FileDecompressor {
|
||||
|
||||
private static final Logger logger = LoggerFactory.getLogger(FileDecompressor.class);
|
||||
|
||||
@Autowired
|
||||
private FileUtils fileUtils;
|
||||
|
||||
|
||||
boolean decompressAndUploadFullTexts(String zstdFileFullPath, Path curBatchPath, String targetDirectory, List<String> fileNamesForCurBatch, int batchCounter, SetMultimap<String, Payload> allFileNamesWithPayloads, long assignmentsBatchCounter)
|
||||
{
|
||||
try {
|
||||
decompressFiles(zstdFileFullPath, curBatchPath);
|
||||
String[] extractedFileNames = new File(targetDirectory).list();
|
||||
if ( extractedFileNames == null ) {
|
||||
logger.error("There was an error when acquiring the list of extracted files of directory: " + targetDirectory);
|
||||
return false;
|
||||
} else if ( extractedFileNames.length == 0 ) { // The directory might have only two files, the "tar-file" and the "tar.zstd-file", if the full-texts failed to be decompressed or untarred..
|
||||
logger.error("No full-texts' fleNames where extracted from directory: " + targetDirectory);
|
||||
return false;
|
||||
} else if ( extractedFileNames.length != fileNamesForCurBatch.size() ) {
|
||||
logger.warn("The number of extracted files (" + extractedFileNames.length + ") was not equal to the number of files (" + fileNamesForCurBatch.size() + ") of the current batch_" + batchCounter);
|
||||
// We do NOT have to find and cross-reference the missing files with the urlReports, in order to set their locations to <null>,
|
||||
// since, in the end of each assignments-batch, an iteration will be made and for all the non-retrieved and non-uploaded full-texts, the app will set them to null.
|
||||
}
|
||||
fileUtils.uploadFullTexts(extractedFileNames, targetDirectory, allFileNamesWithPayloads, batchCounter);
|
||||
return true;
|
||||
} catch (Exception e) {
|
||||
logger.error("Could not extract and upload the full-texts for batch_" + batchCounter + " of assignments_" + assignmentsBatchCounter + GenericUtils.endOfLine + e.getMessage(), e); // It shows the response body (after Spring v.2.5.6).
|
||||
return false;
|
||||
} finally {
|
||||
fileUtils.deleteDirectory(curBatchPath.toFile());
|
||||
}
|
||||
}
|
||||
|
||||
public void decompressFiles(String zstdSource, Path targetDir) throws Exception
|
||||
{
|
||||
// Decompress the zstd file.
|
||||
String tarPathStr = StringUtils.replace(zstdSource, ".zstd", "", 1); // Remove the ".zstd" extension.
|
||||
Path tarPath = Paths.get(tarPathStr);
|
||||
Path tarPath = Paths.get(StringUtils.replace(zstdSource, ".zstd", "", 1)); // Remove the ".zstd" extension.
|
||||
int readByte = -1;
|
||||
|
||||
try ( ZstdCompressorInputStream zsIn = new ZstdCompressorInputStream(new BufferedInputStream(Files.newInputStream(Paths.get(zstdSource)), FileUtils.tenMb));
|
||||
|
@ -66,8 +31,6 @@ public class FileDecompressor {
|
|||
{
|
||||
while ( (readByte = zsIn.read()) != -1 )
|
||||
out.write(readByte);
|
||||
} finally {
|
||||
fileUtils.deleteFile(zstdSource); // Delete the initial zstd file.
|
||||
}
|
||||
|
||||
// Now we have a decompressed tar-file, which we will Un-tar, in order to extract the full-text files.
|
||||
|
@ -83,8 +46,6 @@ public class FileDecompressor {
|
|||
} // The exception will be given to the caller method.
|
||||
// No need to close the tarEntry (no "close()" method is provided).
|
||||
}
|
||||
} finally {
|
||||
fileUtils.deleteFile(tarPathStr); // Delete the decompressed tar file.
|
||||
}
|
||||
|
||||
// Now we have a batch-directory which contains the tar-file along with the extracted full-text files.
|
||||
|
|
|
@ -1,13 +1,14 @@
|
|||
package eu.openaire.urls_controller.util;
|
||||
|
||||
import com.google.common.collect.HashMultimap;
|
||||
import com.google.common.collect.Multimaps;
|
||||
import com.google.common.collect.SetMultimap;
|
||||
import eu.openaire.urls_controller.configuration.DatabaseConnector;
|
||||
import eu.openaire.urls_controller.models.Error;
|
||||
import eu.openaire.urls_controller.configuration.ImpalaConnector;
|
||||
import eu.openaire.urls_controller.controllers.UrlsController;
|
||||
import eu.openaire.urls_controller.models.Payload;
|
||||
import eu.openaire.urls_controller.models.UrlReport;
|
||||
import eu.openaire.urls_controller.models.WorkerInfo;
|
||||
import org.apache.commons.io.FileDeleteStrategy;
|
||||
import org.jetbrains.annotations.NotNull;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
|
@ -17,25 +18,25 @@ import org.springframework.jdbc.core.JdbcTemplate;
|
|||
import org.springframework.stereotype.Component;
|
||||
|
||||
import java.io.*;
|
||||
import java.net.ConnectException;
|
||||
import java.net.HttpURLConnection;
|
||||
import java.net.UnknownHostException;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.net.URL;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.nio.file.Paths;
|
||||
import java.sql.SQLException;
|
||||
import java.sql.Types;
|
||||
import java.text.DecimalFormat;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
import java.util.concurrent.Callable;
|
||||
import java.util.concurrent.ExecutorService;
|
||||
import java.util.concurrent.Executors;
|
||||
import java.util.concurrent.Future;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
import java.util.concurrent.locks.Lock;
|
||||
import java.util.concurrent.locks.ReentrantLock;
|
||||
import java.util.regex.Matcher;
|
||||
import java.util.regex.Pattern;
|
||||
import java.util.stream.Collectors;
|
||||
import java.util.stream.Stream;
|
||||
|
||||
@Component
|
||||
public class FileUtils {
|
||||
|
@ -48,113 +49,326 @@ public class FileUtils {
|
|||
@Autowired
|
||||
private S3ObjectStore s3ObjectStore;
|
||||
|
||||
@Autowired
|
||||
private FileDecompressor fileDecompressor;
|
||||
|
||||
public enum UploadFullTextsResponse {successful, successful_without_fulltexts, unsuccessful}
|
||||
|
||||
public enum UploadFullTextsResponse {successful, unsuccessful, databaseError}
|
||||
|
||||
public String baseFilesLocation;
|
||||
|
||||
public static final String workingDir = System.getProperty("user.dir") + File.separator;
|
||||
|
||||
private final boolean isTestEnvironment;
|
||||
private boolean isTestEnvironment;
|
||||
|
||||
|
||||
public FileUtils (@Value("${services.pdfaggregation.controller.isTestEnvironment}") boolean isTestEnvironment) {
|
||||
public FileUtils (@Value("${services.pdfaggregation.controller.baseFilesLocation}") String baseFilesLocation, @Value("${services.pdfaggregation.controller.isTestEnvironment}") boolean isTestEnvironment) {
|
||||
if ( !baseFilesLocation.endsWith(File.separator) )
|
||||
baseFilesLocation += File.separator;
|
||||
|
||||
if ( !baseFilesLocation.startsWith(File.separator) )
|
||||
baseFilesLocation = workingDir + baseFilesLocation;
|
||||
|
||||
this.baseFilesLocation = baseFilesLocation;
|
||||
|
||||
this.isTestEnvironment = isTestEnvironment;
|
||||
}
|
||||
|
||||
|
||||
// The following regex might be useful in a future scenario. It extracts the "plain-filename" and "file-ID" and the "file-extension".
|
||||
/**
|
||||
* In each insertion, a new parquet-file is created, so we end up with millions of files. Parquet is great for fast-select, so have to stick with it and merge those files..
|
||||
* This method, creates a clone of the original table in order to have only one parquet file in the end. Drops the original table.
|
||||
* Renames the clone to the original's name.
|
||||
* Returns the errorMsg, if an error appears, otherwise is returns "null".
|
||||
*/
|
||||
public String mergeParquetFiles(String tableName, String whereClause, String parameter) {
|
||||
String errorMsg;
|
||||
if ( (tableName == null) || tableName.isEmpty() ) {
|
||||
errorMsg = "No tableName was given. Do not know the tableName for which we should merger the underlying files for!";
|
||||
logger.error(errorMsg);
|
||||
return errorMsg;
|
||||
}
|
||||
|
||||
// Make sure the following are empty strings (in case another method call this one in the future with a null-value).
|
||||
if ( whereClause == null )
|
||||
whereClause = "";
|
||||
|
||||
if ( parameter == null )
|
||||
parameter = "";
|
||||
else
|
||||
parameter = " '" + parameter + "'"; // This will be a "string-check", thus the single-quotes.
|
||||
|
||||
try {
|
||||
jdbcTemplate.execute("CREATE TABLE " + ImpalaConnector.databaseName + "." + tableName + "_tmp stored as parquet AS SELECT * FROM " + ImpalaConnector.databaseName + "." + tableName + " " + whereClause + parameter);
|
||||
jdbcTemplate.execute("DROP TABLE " + ImpalaConnector.databaseName + "." + tableName + " PURGE");
|
||||
jdbcTemplate.execute("ALTER TABLE " + ImpalaConnector.databaseName + "." + tableName + "_tmp RENAME TO " + ImpalaConnector.databaseName + "." + tableName);
|
||||
jdbcTemplate.execute("COMPUTE STATS " + ImpalaConnector.databaseName + "." + tableName);
|
||||
} catch (Exception e) {
|
||||
errorMsg = "Problem when executing the \"clone-drop-rename\" queries!\n";
|
||||
logger.error(errorMsg, e);
|
||||
return errorMsg;
|
||||
}
|
||||
|
||||
return null; // No errorMsg, everything is fine.
|
||||
}
|
||||
|
||||
|
||||
public static DecimalFormat df = new DecimalFormat("0.00");
|
||||
|
||||
// The following regex might be usefull in a future scenario. It extracts the "plain-filename" and "file-ID" and the "file-extension".
|
||||
// Possible full-filenames are: "path1/path2/ID.pdf", "ID2.pdf", "path1/path2/ID(12).pdf", "ID2(25).pdf"
|
||||
public static final Pattern FILEPATH_ID_EXTENSION = Pattern.compile("([^.()]+/)?((([^/()]+)[^./]*)(\\.[\\w]{2,10}))$");
|
||||
public static final Pattern FILENAME_ID_EXTENSION = Pattern.compile("(?:([^.()]+)/)?((([^/()]+)[^./]*)(\\.[\\w]{2,10}))$");
|
||||
|
||||
private static final int numOfFullTextsPerBatch = 70; // The HTTP-headers cannot be too large (It failed with 100 fileNames).
|
||||
|
||||
public static final ExecutorService hashMatchingExecutor = Executors.newFixedThreadPool(6);
|
||||
// TODO - Unify this ExecutorService with the hash-matching executorService. Since one will ALWAYS be called after the other. So why having two ExecServices to handle?
|
||||
|
||||
|
||||
public HashMultimap<String, Payload> getHashesWithPayloads(List<UrlReport> urlReports, int sizeOfUrlReports)
|
||||
{
|
||||
HashMultimap<String, Payload> hashesWithPayloads = HashMultimap.create((sizeOfUrlReports / 5), 3); // Holds multiple payloads for the same fileHash.
|
||||
// The "Hash" part of the multimap helps with avoiding duplicate fileHashes.
|
||||
public UploadFullTextsResponse getAndUploadFullTexts(List<UrlReport> urlReports, long assignmentsBatchCounter, String workerId) {
|
||||
// The Controller have to request the files from the Worker, in order to upload them to the S3.
|
||||
// We will have to UPDATE the "location" of each of those files in the UrlReports and then insert them all into the database.
|
||||
|
||||
String workerIp = null;
|
||||
WorkerInfo workerInfo = UrlsController.workersInfoMap.get(workerId);
|
||||
if ( workerInfo == null ) {
|
||||
logger.error("Could not retrieve the info of worker: " + workerId);
|
||||
return null;
|
||||
} else
|
||||
workerIp = workerInfo.getWorkerIP(); // This won't be null.
|
||||
|
||||
// Get the file-locations.
|
||||
AtomicInteger numFullTextsFound = new AtomicInteger();
|
||||
AtomicInteger numFilesFoundFromPreviousAssignmentsBatches = new AtomicInteger();
|
||||
int urlReportsSize = urlReports.size();
|
||||
|
||||
SetMultimap<String, Payload> allFileNamesWithPayloads = Multimaps.synchronizedSetMultimap(HashMultimap.create((urlReportsSize / 5), 3)); // Holds multiple values for any key, if a fileName(key) has many IDs(values) associated with it.
|
||||
|
||||
final String getFileLocationForHashQuery = "select `location` from " + ImpalaConnector.databaseName + ".payload" + (isTestEnvironment ? "_aggregated" : "") + " where `hash` = ? limit 1";
|
||||
final int[] hashArgType = new int[] {Types.VARCHAR};
|
||||
|
||||
List<Callable<Void>> callableTasks = new ArrayList<>(6);
|
||||
|
||||
for ( UrlReport urlReport : urlReports )
|
||||
{
|
||||
Payload payload = urlReport.getPayload();
|
||||
if ( payload == null )
|
||||
continue;
|
||||
callableTasks.add(() -> {
|
||||
Payload payload = urlReport.getPayload();
|
||||
if ( payload == null )
|
||||
return null;
|
||||
|
||||
String fileLocation = payload.getLocation();
|
||||
if ( fileLocation == null )
|
||||
continue; // The full-text was not retrieved for this UrlReport.
|
||||
String fileLocation = payload.getLocation();
|
||||
if ( fileLocation == null )
|
||||
return null; // The full-text was not retrieved, go to the next UrlReport.
|
||||
|
||||
// Query the payload-table FOR EACH RECORD to get the fileLocation of A PREVIOUS RECORD WITH THE SAME FILE-HASH.
|
||||
// If no result is returned, then this record is not previously found, so go ahead and add it in the list of files to request from the worker.
|
||||
// If a file-location IS returned (for this hash), then this file is already uploaded to the S3. Update the record to point to that file-location and do not request that file from the Worker.
|
||||
String fileHash = payload.getHash();
|
||||
if ( fileHash != null )
|
||||
{
|
||||
hashesWithPayloads.put(fileHash, payload); // Hold multiple payloads per fileHash.
|
||||
// There are 2 cases, which contribute to that:
|
||||
// 1) Different publication-IDs end up giving the same full-text-url, resulting in the same file. Those duplicates are not saved, but instead, the location, hash and size of the file is copied to the other payload.
|
||||
// 2) Different publication-IDs end up giving different full-text-urls which point to the same file. Although very rare, in this case, the file is downloaded again by the Worker and has a different name.
|
||||
// Query the payload-table FOR EACH RECORD to get the fileLocation of A PREVIOUS RECORD WITH THE SAME FILE-HASH.
|
||||
// If no result is returned, then this record is not previously found, so go ahead and add it in the list of files to request from the worker.
|
||||
// If a file-location IS returned (for this hash), then this file is already uploaded to the S3. Update the record to point to that file-location and do not request that file from the Worker.
|
||||
String fileHash = payload.getHash();
|
||||
if ( fileHash != null ) {
|
||||
String alreadyFoundFileLocation = null;
|
||||
try {
|
||||
alreadyFoundFileLocation = jdbcTemplate.queryForObject(getFileLocationForHashQuery, new Object[] {fileHash}, hashArgType, String.class);
|
||||
} catch (EmptyResultDataAccessException erdae) {
|
||||
// No fileLocation is found, it's ok. It will be null by default.
|
||||
} catch (Exception e) {
|
||||
logger.error("Error when executing or acquiring data from the the \"getFileLocationForHashQuery\"!\n", e);
|
||||
// TODO - SHOULD WE RETURN A "UploadFullTextsResponse.databaseError" AND force the caller to not even insert the payloads to the database??
|
||||
// TODO - The idea is that since the database will have problems.. there is no point in trying to insert the payloads to Impala (we will handle it like: we tried to insert and got an error).
|
||||
// Unless we do what it is said above, do not continue to the next UrlReport, this query-exception should not disrupt the normal full-text processing.
|
||||
}
|
||||
|
||||
// In either case, the duplicate file will not be transferred to the Controller, but in the 2nd one it takes up extra space, at least for some time.
|
||||
// TODO - Implement a fileHash-check algorithm in the Worker's side ("PublicationsRetriever"), to avoid keeping those files in storage.
|
||||
if ( alreadyFoundFileLocation != null ) { // If the full-text of this record is already-found and uploaded.
|
||||
payload.setLocation(alreadyFoundFileLocation); // Set the location to the older identical file, which was uploaded to S3. The other file-data is identical.
|
||||
if ( logger.isTraceEnabled() )
|
||||
logger.trace("The record with ID \"" + payload.getId() + "\" has an \"alreadyRetrieved\" file, with hash \"" + fileHash + "\" and location \"" + alreadyFoundFileLocation + "\"."); // DEBUG!
|
||||
numFilesFoundFromPreviousAssignmentsBatches.incrementAndGet();
|
||||
numFullTextsFound.incrementAndGet();
|
||||
return null; // Do not request the file from the worker, it's already uploaded. Move on. The "location" will be filled my the "setFullTextForMultiplePayloads()" method, later.
|
||||
}
|
||||
}
|
||||
|
||||
} else // This should never happen..
|
||||
logger.error("Payload: " + payload + " has a null fileHash!");
|
||||
// Extract the "fileNameWithExtension" to be added in the HashMultimap.
|
||||
Matcher matcher = FILENAME_ID_EXTENSION.matcher(fileLocation);
|
||||
if ( ! matcher.matches() ) {
|
||||
logger.error("Failed to match the \"fileLocation\": \"" + fileLocation + "\" of id: \"" + payload.getId() + "\", originalUrl: \"" + payload.getOriginal_url() + "\", using this regex: " + FILENAME_ID_EXTENSION);
|
||||
return null;
|
||||
}
|
||||
String fileNameWithExtension = matcher.group(2);
|
||||
if ( (fileNameWithExtension == null) || fileNameWithExtension.isEmpty() ) {
|
||||
logger.error("Failed to extract the \"fileNameWithExtension\" from \"fileLocation\": \"" + fileLocation + "\", of id: \"" + payload.getId() + "\", originalUrl: \"" + payload.getOriginal_url() + "\", using this regex: " + FILENAME_ID_EXTENSION);
|
||||
return null;
|
||||
}
|
||||
|
||||
numFullTextsFound.incrementAndGet();
|
||||
allFileNamesWithPayloads.put(fileNameWithExtension, payload); // The keys and the values are not duplicate.
|
||||
// Task with ID-1 might have an "ID-1.pdf" file, while a task with ID-2 can also have an "ID-1.pdf" file, as the pdf-url-2 might be the same with pdf-url-1, thus, the ID-2 file was not downloaded again.
|
||||
return null;
|
||||
});
|
||||
}// end-for
|
||||
|
||||
return hashesWithPayloads;
|
||||
}
|
||||
|
||||
|
||||
public HashMap<String, String> getHashLocationMap(Set<String> fileHashes, int fileHashesSetSize, long batchCounter, String groupType)
|
||||
{
|
||||
// Prepare the "fileHashListString" to be used inside the "getHashLocationsQuery". Create the following string-pattern:
|
||||
// ("HASH_1", "HASH_2", ...)
|
||||
int stringBuilderCapacity = ((fileHashesSetSize * 32) + (fileHashesSetSize -1) +2);
|
||||
|
||||
String getHashLocationsQuery = "select distinct `hash`, `location` from " + DatabaseConnector.databaseName + ".payload where `hash` in "
|
||||
+ getQueryListString(new ArrayList<>(fileHashes), fileHashesSetSize, stringBuilderCapacity);
|
||||
|
||||
HashMap<String, String> hashLocationMap = new HashMap<>(fileHashesSetSize/2); // No multimap is needed since only one location is returned for each fileHash.
|
||||
|
||||
DatabaseConnector.databaseLock.lock(); // The execution uses the database.
|
||||
try {
|
||||
jdbcTemplate.query(getHashLocationsQuery, rs -> {
|
||||
try { // For each of the 4 columns returned, do the following. The column-indexing starts from 1.
|
||||
hashLocationMap.put(rs.getString(1), rs.getString(2));
|
||||
} catch (SQLException sqle) {
|
||||
logger.error("No value was able to be retrieved from one of the columns of row_" + rs.getRow(), sqle);
|
||||
ImpalaConnector.databaseLock.lock(); // The execution uses the database.
|
||||
try { // Invoke all the tasks and wait for them to finish before moving to the next batch.
|
||||
List<Future<Void>> futures = hashMatchingExecutor.invokeAll(callableTasks);
|
||||
for ( Future<Void> future : futures ) {
|
||||
try {
|
||||
Void result = future.get(); // The result is always "null" as we have a "Void" type.
|
||||
} catch (Exception e) {
|
||||
logger.error("", e);
|
||||
}
|
||||
});
|
||||
} catch (EmptyResultDataAccessException erdae) {
|
||||
logger.warn("No previously-found hash-locations where found for " + groupType + "_" + batchCounter);
|
||||
}
|
||||
} catch (InterruptedException ie) { // In this case, any unfinished tasks are cancelled.
|
||||
logger.warn("The current thread was interrupted when waiting for the worker-threads to finish checking for already-found file-hashes: " + ie.getMessage());
|
||||
// This is a very rare case. At the moment, we just move on with what we have so far.
|
||||
} catch (Exception e) {
|
||||
logger.error("Unexpected error when checking for already-found file-hashes, for " + groupType + "_" + batchCounter, e);
|
||||
// We will continue with storing the files, we do not want to lose them.
|
||||
String errorMsg = "Unexpected error when checking for already-found file-hashes in parallel! " + e.getMessage();
|
||||
logger.error(errorMsg, e);
|
||||
return UploadFullTextsResponse.unsuccessful;
|
||||
} finally {
|
||||
DatabaseConnector.databaseLock.unlock();
|
||||
ImpalaConnector.databaseLock.unlock(); // The remaining work of this function does not use the database.
|
||||
}
|
||||
|
||||
return hashLocationMap;
|
||||
logger.info("NumFullTextsFound by assignments_" + assignmentsBatchCounter + " = " + numFullTextsFound.get() + " (out of " + urlReportsSize + " | about " + df.format(numFullTextsFound.get() * 100.0 / urlReportsSize) + "%).");
|
||||
logger.debug("NumFilesFoundFromPreviousAssignmentsBatches = " + numFilesFoundFromPreviousAssignmentsBatches.get());
|
||||
|
||||
ArrayList<String> allFileNames = new ArrayList<>(allFileNamesWithPayloads.keySet());
|
||||
int numAllFullTexts = allFileNames.size();
|
||||
if ( numAllFullTexts == 0 ) {
|
||||
logger.warn("No full-text files were retrieved for assignments_" + assignmentsBatchCounter + " | from worker: " + workerId);
|
||||
return UploadFullTextsResponse.successful; // It was handled, no error.
|
||||
}
|
||||
|
||||
// Request the full-texts in batches, compressed in a zstd tar file.
|
||||
int numOfBatches = (numAllFullTexts / numOfFullTextsPerBatch);
|
||||
int remainingFiles = (numAllFullTexts % numOfFullTextsPerBatch);
|
||||
if ( remainingFiles > 0 ) { // Add an extra batch for the remaining files. This guarantees at least one batch will exist no matter how few (>0) the files are.
|
||||
numOfBatches++;
|
||||
logger.debug("The assignments_" + assignmentsBatchCounter + " have " + numAllFullTexts + " distinct non-already-uploaded fullTexts (total is: " + numFullTextsFound.get() + "). Going to request them from the Worker \"" + workerId + "\", in " + numOfBatches + " batches (" + numOfFullTextsPerBatch + " files each, except for the final batch, which will have " + remainingFiles + " files).");
|
||||
} else
|
||||
logger.debug("The assignments_" + assignmentsBatchCounter + " have " + numAllFullTexts + " distinct non-already-uploaded fullTexts (total is: " + numFullTextsFound.get() + "). Going to request them from the Worker \"" + workerId + "\", in " + numOfBatches + " batches (" + numOfFullTextsPerBatch + " files each).");
|
||||
|
||||
// Check if one full text is left out because of the division. Put it int the last batch.
|
||||
String baseUrl = "http://" + workerIp + ":1881/api/full-texts/getFullTexts/" + assignmentsBatchCounter + "/" + numOfBatches + "/";
|
||||
|
||||
// TODO - The worker should send the port in which it accepts requests, along with the current request.
|
||||
// TODO - The least we have to do it to expose the port-assignment somewhere more obvious like inside the "application.yml" file.
|
||||
|
||||
String curAssignmentsBaseLocation = baseFilesLocation + "assignments_" + assignmentsBatchCounter + File.separator;
|
||||
// Note: the "curAssignmentsBaseLocation"-directory will be created once the first batch subdirectory is called for creation.
|
||||
|
||||
int failedBatches = 0;
|
||||
for ( int batchCounter = 1; batchCounter <= numOfBatches; ++batchCounter ) {
|
||||
List<String> fileNamesForCurBatch = getFileNamesForBatch(allFileNames, numAllFullTexts, batchCounter);
|
||||
HttpURLConnection conn;
|
||||
try {
|
||||
conn = getConnection(baseUrl, assignmentsBatchCounter, batchCounter, fileNamesForCurBatch, numOfBatches, workerId);
|
||||
if ( conn == null ) {
|
||||
failedBatches ++;
|
||||
continue; // To the next batch.
|
||||
}
|
||||
} catch (RuntimeException re) {
|
||||
// The "cause" was logged inside "getConnection()".
|
||||
failedBatches += (1 + (numOfBatches - batchCounter)); // The "failedBatches" will have the previously failedBatches + this one + the remaining batches which will likely fail too, thus, they will not be tested.
|
||||
break; // The rest of the batches will likely fail as well.
|
||||
}
|
||||
|
||||
// Get the extracted files.
|
||||
String targetDirectory = curAssignmentsBaseLocation + "batch_" + batchCounter + File.separator;
|
||||
Path curBatchPath = null;
|
||||
try {
|
||||
// Create this batch-directory.
|
||||
curBatchPath = Files.createDirectories(Paths.get(targetDirectory));
|
||||
// The base-directory will be created along with the first batch directory.
|
||||
|
||||
// Save and decompress the zstd file. Iterate over the PDFs and upload each one of them and get the S3-Url.
|
||||
String zstdFileFullPath = targetDirectory + "fullTexts_" + assignmentsBatchCounter + "_" + batchCounter + ".tar.zstd";
|
||||
File zstdFile = new File(zstdFileFullPath);
|
||||
|
||||
if ( ! saveArchive(conn, zstdFile) ) {
|
||||
failedBatches ++;
|
||||
continue; // To the next batch.
|
||||
}
|
||||
//logger.debug("The zstd file has been saved: " + zstdFileFullPath); // DEBUG!
|
||||
|
||||
// We do not call "conn.disconnect()", since more request are about to be made to the worker, in the near future.
|
||||
|
||||
fileDecompressor.decompressFiles(zstdFileFullPath, curBatchPath);
|
||||
|
||||
String[] extractedFileNames = new File(targetDirectory).list();
|
||||
if ( (extractedFileNames == null) || (extractedFileNames.length <= 2) ) { // The directory might have only two files, the "tar-file" and the "tar.zstd-file", if the full-texts failed to be decompressed or untarred..
|
||||
logger.error("No full-texts' fleNames where extracted from directory: " + targetDirectory);
|
||||
failedBatches ++;
|
||||
continue; // To the next batch.
|
||||
}
|
||||
else if ( (extractedFileNames.length -2) != fileNamesForCurBatch.size() ) {
|
||||
logger.warn("The number of extracted files (" + (extractedFileNames.length -2) + ") was not equal to the number of the current-batch's (" + batchCounter + ") files (" + fileNamesForCurBatch.size() + ").");
|
||||
// We do NOT have to find and cross-reference the missing files with the urlReports, in order to set their locations to <null>,
|
||||
// since, in the end of each assignments-batch, an iteration will be made and for all the non-retrieved and non-uploaded full-texts, the app will set them to null.
|
||||
}
|
||||
|
||||
uploadFullTexts(extractedFileNames, targetDirectory, allFileNamesWithPayloads);
|
||||
|
||||
} catch (Exception e) {
|
||||
logger.error("Could not extract and upload the full-texts for batch_" + batchCounter + " of assignments_" + assignmentsBatchCounter + "\n" + e.getMessage(), e); // It shows the response body (after Spring v.2.5.6).
|
||||
failedBatches ++;
|
||||
} finally {
|
||||
if ( curBatchPath != null )
|
||||
deleteDirectory(curBatchPath.toFile());
|
||||
}
|
||||
} // End of batches.
|
||||
|
||||
updateUrlReportsToHaveNoFullTextFiles(urlReports, true); // Make sure all records without an S3-Url have < null > file-data (some batches or uploads might have failed).
|
||||
deleteDirectory(new File(curAssignmentsBaseLocation));
|
||||
|
||||
if ( failedBatches == numOfBatches ) {
|
||||
logger.error("None of the " + numOfBatches + " batches could be handled for assignments_" + assignmentsBatchCounter + ", for worker: " + workerId);
|
||||
return UploadFullTextsResponse.unsuccessful;
|
||||
} else
|
||||
return UploadFullTextsResponse.successful;
|
||||
}
|
||||
|
||||
|
||||
void uploadFullTexts(String[] fileNames, String targetDirectory, SetMultimap<String, Payload> allFileNamesWithPayloads, int batchCounter)
|
||||
private HttpURLConnection getConnection(String baseUrl, long assignmentsBatchCounter, int batchNum, List<String> fileNamesForCurBatch, int totalBatches, String workerId) throws RuntimeException
|
||||
{
|
||||
baseUrl += batchNum + "/";
|
||||
String requestUrl = getRequestUrlForBatch(baseUrl, fileNamesForCurBatch);
|
||||
//logger.debug("Going to request the batch_" + batchNum + " (out of " + totalBatches + ") with " + fileNamesForCurBatch.size() + " fullTexts, of assignments_" + assignmentsBatchCounter + " from the Worker with ID \"" + workerId + "\" and baseRequestUrl: " + baseUrl + "[fileNames]");
|
||||
try {
|
||||
HttpURLConnection conn = (HttpURLConnection) new URL(requestUrl).openConnection();
|
||||
conn.setRequestMethod("GET");
|
||||
conn.setRequestProperty("User-Agent", "UrlsController");
|
||||
conn.connect();
|
||||
int statusCode = conn.getResponseCode();
|
||||
if ( statusCode == -1 ) { // Invalid HTTP-Response.
|
||||
logger.warn("Problem when getting the \"status-code\" for url: " + requestUrl);
|
||||
throw new RuntimeException(); // Avoid any other batches.
|
||||
} else if ( statusCode != 200 ) {
|
||||
logger.warn("HTTP-" + statusCode + ": " + getMessageFromResponseBody(conn, true) + "\n\nProblem when requesting the ZstdFile of batch_" + batchNum + " from the Worker with ID \"" + workerId + "\" and requestUrl: " + requestUrl);
|
||||
if ( (statusCode >= 500) && (statusCode <= 599) )
|
||||
throw new RuntimeException(); // Throw an exception to indicate that the Worker has problems and all remaining batches will fail as well.
|
||||
return null;
|
||||
} else
|
||||
return conn;
|
||||
} catch (RuntimeException re) {
|
||||
throw re;
|
||||
} catch (Exception e) {
|
||||
String exMessage = e.getMessage();
|
||||
logger.warn("Problem when requesting the ZstdFile of batch_" + batchNum + " of assignments_" + assignmentsBatchCounter + " from the Worker with ID \"" + workerId + "\" and requestUrl: " + requestUrl + "\n" + exMessage);
|
||||
if ( exMessage.contains("Connection refused") ) {
|
||||
logger.error("Since we received a \"Connection refused\", all of the remaining batches (" + (totalBatches - batchNum) + ") will not be requested!");
|
||||
throw new RuntimeException();
|
||||
}
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
private void uploadFullTexts(String[] fileNames, String targetDirectory, SetMultimap<String, Payload> allFileNamesWithPayloads)
|
||||
{
|
||||
// Iterate over the files and upload them to S3.
|
||||
|
||||
// TODO - Make the uploads run in parallel, using at most 4 threads.
|
||||
// There might also be uploads from other assignment-batches (from other workers) running at the same time.
|
||||
// But still, there are currently at most 3 (?) upload operations, ast the same time, each using 1 thread.
|
||||
// It's most common to have just 1 or 2 of them. So each can easily use 2 or 4 threads to upload its own files.
|
||||
|
||||
//int numUploadedFiles = 0;
|
||||
for ( String fileName : fileNames )
|
||||
for( String fileName : fileNames )
|
||||
{
|
||||
// Check if any of the ".tar" files (".tar" and ".tar.zstd") are still in the directory (in case their deletion failed) and exclude them from being uploaded.
|
||||
if ( fileName.contains(".tar") )
|
||||
if ( fileName.contains(".tar") ) // Exclude the tar-files from uploading.
|
||||
continue;
|
||||
|
||||
// Check if this stored file is related to one or more Payloads from the Set. Defend against malicious file injection. It does not add more overhead, since we already need the "fileRelatedPayloads".
|
||||
|
@ -165,100 +379,74 @@ public class FileUtils {
|
|||
}
|
||||
|
||||
// Let's try to upload the file to S3 and update the payloads, either in successful file-uploads (right-away) or not (in the end).
|
||||
|
||||
// Prepare the filename as: "datasourceid/publicationid::hash.pdf"
|
||||
// All related payloads point to this exact same file, BUT, may be related with different urlIDs, which in turn be related with different datasourceIDs.
|
||||
// This file could have been found from different urlIds and thus be related to multiple datasourceIds.
|
||||
// BUT, since the filename contains a specific urlID, the datasourceId should be the one related to that specific urlID.
|
||||
// So, we extract this urlID, search the payload inside the "fileRelatedPayloads" and get the related datasourceID (instead of taking the first or a random datasourceID).
|
||||
|
||||
Matcher matcher = FILEPATH_ID_EXTENSION.matcher(fileName);
|
||||
if ( !matcher.matches() ) {
|
||||
logger.error("Failed to match the \"" + fileName + "\" with the regex: " + FILEPATH_ID_EXTENSION);
|
||||
continue;
|
||||
}
|
||||
// The "matcher.group(3)" returns the "filenameWithoutExtension", which is currently not used.
|
||||
// Use the "fileNameID" and not the "filenameWithoutExtension", as we want to avoid keeping the possible "parenthesis" with the increasing number (about the duplication of ID-fileName).
|
||||
String fileNameID = matcher.group(4); // The "fileNameID" is the OpenAIRE_ID for this file.
|
||||
if ( (fileNameID == null) || fileNameID.isEmpty() ) {
|
||||
logger.error("Failed to extract the \"fileNameID\" from \"" + fileName + "\".");
|
||||
continue;
|
||||
}
|
||||
String dotFileExtension = matcher.group(5);
|
||||
if ( (dotFileExtension == null) || dotFileExtension.isEmpty() ) {
|
||||
logger.error("Failed to extract the \"dotFileExtension\" from \"" + fileName + "\".");
|
||||
continue;
|
||||
}
|
||||
|
||||
// This file is related with some payloads, in a sense that these payloads have urls which lead to the same full-text url.
|
||||
// These payloads might have different IDs and sourceUrls. But, in the end, the different sourceUrls give the same full-text.
|
||||
// Below, we make sure we pick the "datasource" from the payload, which has the same id as the full-text's name.
|
||||
// If there are multiple payloads with the same id, which point to the same file, then we can take whatever datasource we want from those payloads.
|
||||
// It is possible that payloads with same IDs, but different sourceUrls pointing to the same full-text, can be related with different datasources
|
||||
// (especially for IDs of type: "doiboost_____::XXXXXXXXXXXXXXXXXXXXX").
|
||||
// It does not really matter, since the first-ever payload to give this full-text could very well be another one,
|
||||
// since the crawling happens in multiple threads which compete with each other for CPU time.
|
||||
|
||||
String datasourceId = null;
|
||||
String hash = null;
|
||||
boolean isFound = false;
|
||||
for ( Payload payload : fileRelatedPayloads ) {
|
||||
if ( fileNameID.equals(payload.getId()) ) {
|
||||
datasourceId = payload.getDatasourceId();
|
||||
hash = payload.getHash();
|
||||
isFound = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if ( !isFound ) { // This should never normally happen. If it does, then a very bad change will have taken place.
|
||||
logger.error("The \"fileNameID\" (" + fileNameID + ") was not found inside the \"fileRelatedPayloads\" for fileName: " + fileName);
|
||||
continue;
|
||||
}
|
||||
|
||||
try {
|
||||
String s3Url = constructS3FilenameAndUploadToS3(targetDirectory, fileName, fileNameID, dotFileExtension, datasourceId, hash);
|
||||
if ( s3Url != null ) {
|
||||
setFullTextForMultiplePayloads(fileRelatedPayloads, s3Url);
|
||||
//numUploadedFiles ++;
|
||||
// Prepare the filename as: "datasourceid/publicationid::hash.pdf"
|
||||
// All related payloads point to this exact same file, BUT, may be related with different urlIDs, which in turn be related with different datasourceIDs.
|
||||
// This file could have been found from different urlIds and thus be related to multiple datasourceIds.
|
||||
// BUT, since the filename contains a specific urlID, the datasourceId should be the one related to that specific urlID.
|
||||
// So, we extract this urlID, search the payload inside the "fileRelatedPayloads" and get the related datasourceID (instead of taking the first or a random datasourceID).
|
||||
|
||||
Matcher matcher = FILENAME_ID_EXTENSION.matcher(fileName);
|
||||
if ( !matcher.matches() ) {
|
||||
logger.error("Failed to match the \"" + fileName + "\" with the regex: " + FILENAME_ID_EXTENSION);
|
||||
continue;
|
||||
}
|
||||
// The "matcher.group(3)" returns the "filenameWithoutExtension", which is currently not used.
|
||||
String fileNameID = matcher.group(4);
|
||||
if ( (fileNameID == null) || fileNameID.isEmpty() ) {
|
||||
logger.error("Failed to extract the \"fileNameID\" from \"" + fileName + "\".");
|
||||
continue;
|
||||
}
|
||||
String dotFileExtension = matcher.group(5);
|
||||
if ( (dotFileExtension == null) || dotFileExtension.isEmpty() ) {
|
||||
logger.error("Failed to extract the \"dotFileExtension\" from \"" + fileName + "\".");
|
||||
continue;
|
||||
}
|
||||
|
||||
// This file is related with some payloads, in a sense that these payloads have urls which lead to the same full-text url.
|
||||
// These payloads might have different IDs and sourceUrls. But, in the end, the different sourceUrls give the same full-text.
|
||||
// Below, we make sure we pick the "datasource" from the payload, which has the same id as the full-text's name.
|
||||
// If there are multiple payloads with the same id, which point to the same file, then we can take whatever datasource we want from those payloads.
|
||||
// It is possible that payloads with same IDs, but different sourceUrls pointing to the same full-text, can be related with different datasources
|
||||
// (especially for IDs of type: "doiboost_____::XXXXXXXXXXXXXXXXXXXXX").
|
||||
// It does not really matter, since the first-ever payload to give this full-text could very well be another one,
|
||||
// since the crawling happens in multiple threads which compete with each other for CPU time.
|
||||
|
||||
String datasourceId = null;
|
||||
String hash = null;
|
||||
boolean isFound = false;
|
||||
for ( Payload payload : fileRelatedPayloads ) {
|
||||
if ( fileNameID.equals(payload.getId()) ) {
|
||||
datasourceId = payload.getDatasourceId();
|
||||
hash = payload.getHash();
|
||||
isFound = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if ( !isFound ) { // This should never normally happen. If it does, then a very bad change will have taken place.
|
||||
logger.error("The \"fileNameID\" (" + fileNameID + ") was not found inside the \"fileRelatedPayloads\" for fileName: " + fileName);
|
||||
continue;
|
||||
}
|
||||
|
||||
String s3Url = constructFileNameAndUploadToS3(targetDirectory, fileName, fileNameID, dotFileExtension, datasourceId, hash);
|
||||
if ( s3Url == null )
|
||||
continue;
|
||||
|
||||
setFullTextForMultiplePayloads(fileRelatedPayloads, s3Url);
|
||||
//numUploadedFiles ++;
|
||||
} catch (Exception e) {
|
||||
logger.error("Avoid uploading the rest of the files of batch_" + batchCounter + " | " + e.getMessage());
|
||||
break;
|
||||
logger.error("Could not upload the file \"" + fileName + "\" to the S3 ObjectStore!", e);
|
||||
}
|
||||
// Else, the record will have its file-data set to "null", in the end of the caller method (as it will not have an s3Url as its location).
|
||||
}//end filenames-for-loop
|
||||
|
||||
//logger.debug("Finished uploading " + numUploadedFiles + " full-texts (out of " + fileNames.length + " distinct files) from assignments_" + assignmentsBatchCounter + ", batch_" + batchCounter + " on S3-ObjectStore.");
|
||||
}
|
||||
|
||||
|
||||
public String constructS3FilenameAndUploadToS3(String targetDirectory, String fileName, String openAireId,
|
||||
String dotFileExtension, String datasourceId, String hash) throws ConnectException, UnknownHostException
|
||||
{
|
||||
String filenameForS3 = constructS3FileName(fileName, openAireId, dotFileExtension, datasourceId, hash); // This name is for the uploaded file, in the S3 Object Store.
|
||||
if ( filenameForS3 == null ) // The error is logged inside.
|
||||
return null;
|
||||
|
||||
String fileFullPath = targetDirectory + fileName; // The fullPath to the local file (which has the previous name).
|
||||
String s3Url = null;
|
||||
try {
|
||||
s3Url = s3ObjectStore.uploadToS3(filenameForS3, fileFullPath);
|
||||
} catch (ConnectException ce) {
|
||||
logger.error("Could not connect with the S3 Object Store! " + ce.getMessage());
|
||||
throw ce;
|
||||
} catch (UnknownHostException uhe) {
|
||||
logger.error("The S3 Object Store could not be found! " + uhe.getMessage());
|
||||
throw uhe;
|
||||
} catch (Exception e) {
|
||||
logger.error("Could not upload the local-file \"" + fileFullPath + "\" to the S3 ObjectStore, with S3-filename: \"" + filenameForS3 + "\"!", e);
|
||||
return null;
|
||||
// Else, the record will have its file-data set to "null", in the end of this method.
|
||||
}
|
||||
return s3Url;
|
||||
|
||||
//logger.debug("Finished uploading " + numUploadedFiles + " full-texts (out of " + (fileNames.length -2) + " distinct files) from assignments_" + assignmentsBatchCounter + ", batch_" + batchCounter + " on S3-ObjectStore.");
|
||||
// (fileNames.length -2) --> minus the zstd and the tar files.
|
||||
}
|
||||
|
||||
|
||||
public String constructS3FileName(String fileName, String openAireID, String dotFileExtension, String datasourceId, String hash)
|
||||
public String constructFileNameAndUploadToS3(String fileDir, String fileName, String openAireID, String dotFileExtension, String datasourceId, String hash) throws Exception
|
||||
{
|
||||
if ( datasourceId == null ) {
|
||||
logger.error("The retrieved \"datasourceId\" was \"null\" for file: " + fileName);
|
||||
|
@ -269,15 +457,19 @@ public class FileUtils {
|
|||
return null;
|
||||
}
|
||||
|
||||
String fileFullPath = fileDir + File.separator + fileName; // The fullPath to the local file.
|
||||
|
||||
// Use the "fileNameID" and not the "filenameWithoutExtension", as we want to avoid keeping the possible "parenthesis" with the increasing number (about the duplication of ID-fileName).
|
||||
// Now we append the file-hash, so it is guaranteed that the filename will be unique.
|
||||
return datasourceId + "/" + openAireID + "::" + hash + dotFileExtension; // This is the fileName to be used in the objectStore, not of the local file!
|
||||
fileName = datasourceId + "/" + openAireID + "::" + hash + dotFileExtension; // This is the fileName to be used in the objectStore, not of the local file!
|
||||
|
||||
return s3ObjectStore.uploadToS3(fileName, fileFullPath);
|
||||
}
|
||||
|
||||
|
||||
public String getMessageFromResponseBody(HttpURLConnection conn, boolean isError)
|
||||
{
|
||||
public String getMessageFromResponseBody(HttpURLConnection conn, boolean isError) {
|
||||
final StringBuilder msgStrB = new StringBuilder(500);
|
||||
try ( BufferedReader br = new BufferedReader(new InputStreamReader((isError ? conn.getErrorStream() : conn.getInputStream()), StandardCharsets.UTF_8)) ) {
|
||||
try ( BufferedReader br = new BufferedReader(new InputStreamReader((isError ? conn.getErrorStream() : conn.getInputStream()))) ) { // Try-with-resources
|
||||
String inputLine;
|
||||
while ( (inputLine = br.readLine()) != null )
|
||||
{
|
||||
|
@ -295,13 +487,42 @@ public class FileUtils {
|
|||
}
|
||||
|
||||
|
||||
public static final int twentyFiveKb = 25_600; // 25 Kb
|
||||
public static final int halfMb = 524_288; // 0.5 Mb = 512 Kb = 524_288 bytes
|
||||
private List<String> getFileNamesForBatch(List<String> allFileNames, int numAllFullTexts, int curBatch) {
|
||||
int initialIndex = ((curBatch-1) * numOfFullTextsPerBatch);
|
||||
int endingIndex = (curBatch * numOfFullTextsPerBatch);
|
||||
if ( endingIndex > numAllFullTexts ) // This might be the case, when the "numAllFullTexts" is too small.
|
||||
endingIndex = numAllFullTexts;
|
||||
|
||||
List<String> fileNamesOfCurBatch = new ArrayList<>(numOfFullTextsPerBatch);
|
||||
for ( int i = initialIndex; i < endingIndex; ++i ) {
|
||||
try {
|
||||
fileNamesOfCurBatch.add(allFileNames.get(i));
|
||||
} catch (IndexOutOfBoundsException ioobe) {
|
||||
logger.error("IOOBE for i=" + i + "\n" + ioobe.getMessage(), ioobe);
|
||||
}
|
||||
}
|
||||
return fileNamesOfCurBatch;
|
||||
}
|
||||
|
||||
|
||||
private String getRequestUrlForBatch(String baseUrl, List<String> fileNamesForCurBatch) {
|
||||
final StringBuilder sb = new StringBuilder(numOfFullTextsPerBatch * 50);
|
||||
sb.append(baseUrl);
|
||||
int numFullTextsCurBatch = fileNamesForCurBatch.size();
|
||||
for ( int j=0; j < numFullTextsCurBatch; ++j ){
|
||||
sb.append(fileNamesForCurBatch.get(j));
|
||||
if ( j < (numFullTextsCurBatch -1) )
|
||||
sb.append(",");
|
||||
}
|
||||
return sb.toString();
|
||||
}
|
||||
|
||||
|
||||
public static final int tenMb = (10 * 1_048_576);
|
||||
|
||||
public boolean saveArchive(InputStream inputStream, File zstdFile)
|
||||
public boolean saveArchive(HttpURLConnection conn, File zstdFile)
|
||||
{
|
||||
try ( BufferedInputStream inStream = new BufferedInputStream(inputStream, tenMb);
|
||||
try ( BufferedInputStream inStream = new BufferedInputStream(conn.getInputStream(), tenMb);
|
||||
BufferedOutputStream outStream = new BufferedOutputStream(Files.newOutputStream(zstdFile.toPath()), tenMb) )
|
||||
{
|
||||
int readBytes;
|
||||
|
@ -317,102 +538,31 @@ public class FileUtils {
|
|||
|
||||
|
||||
/**
|
||||
* This method searches the backlog of publications for the ones that have the same "original_url" or their "original_url" is equal to the "actual_url" or an existing payload.
|
||||
* Then, it generates a new "UrlReport" object, which has as a payload a previously generated one, which has different "id", "original_url".
|
||||
* Then, the program automatically generates "attempt" and "payload" records for these additional UrlReport-records.
|
||||
* It must be executed inside the same "database-locked" block of code, along with the inserts of the attempt and payload records.
|
||||
* */
|
||||
public boolean addUrlReportsByMatchingRecordsFromBacklog(List<UrlReport> urlReports, List<Payload> initialPayloads, int numInitialPayloads, long assignmentsBatchCounter)
|
||||
{
|
||||
logger.debug("numInitialPayloads: " + numInitialPayloads + " | assignmentsBatchCounter: " + assignmentsBatchCounter);
|
||||
* This method updates the UrlReports to not point to any downloaded fullText files.
|
||||
* This is useful when the uploading process of the fullTexts to the S3-ObjectStore fails, and we don't want any "links" to locally stored files, which will be deleted.
|
||||
* If the "shouldCheckAndKeepS3UploadedFiles" is set to "true", then the payloads which have their file uploaded to the S3-ObjectStore, are excluded.
|
||||
* @param urlReports
|
||||
* @param shouldCheckAndKeepS3UploadedFiles
|
||||
*/
|
||||
public void updateUrlReportsToHaveNoFullTextFiles(List<UrlReport> urlReports, boolean shouldCheckAndKeepS3UploadedFiles) {
|
||||
for ( UrlReport urlReport : urlReports ) {
|
||||
Payload payload = urlReport.getPayload();
|
||||
if ( payload == null )
|
||||
continue;
|
||||
|
||||
// Create a HashMultimap, containing the "original_url" or "actual_url" as the key and the related "payload" objects as its values.
|
||||
final HashMultimap<String, Payload> urlToPayloadsMultimap = HashMultimap.create((numInitialPayloads / 3), 3); // Holds multiple values for any key, if a fileName(key) has many IDs(values) associated with it.
|
||||
for ( Payload payload : initialPayloads ) {
|
||||
String original_url = payload.getOriginal_url();
|
||||
String actual_url = payload.getActual_url();
|
||||
if ( shouldCheckAndKeepS3UploadedFiles ) {
|
||||
String fileLocation = payload.getLocation();
|
||||
if ( (fileLocation == null) || s3ObjectStore.isLocationInStore(fileLocation) )
|
||||
continue;
|
||||
}
|
||||
|
||||
// Link this payload with both the original and actual urls (in case they are different).
|
||||
urlToPayloadsMultimap.put(original_url, payload);
|
||||
if ( ! actual_url.equals(original_url) )
|
||||
urlToPayloadsMultimap.put(actual_url, payload);
|
||||
// Mark this full-text as not-retrieved, since it will be deleted from local-storage. The retrieved link to the full-text ("actual_url") will be kept, for now.
|
||||
payload.setLocation(null); // This will cause the payload to not be inserted into the "payload" table in the database. Only the "attempt" record will be inserted.
|
||||
payload.setHash(null);
|
||||
payload.setMime_type(null);
|
||||
payload.setSize(null);
|
||||
// The id-url record will be called as a new assignment in the future.
|
||||
}
|
||||
// A url may be related to different payloads, in the urlReports. For example, in one payload the url was the original_url
|
||||
// but in another payload the url was only the actual-url (the original was different)
|
||||
|
||||
// Gather the original and actual urls of the current payloads and add them in a list usable by the query.
|
||||
List<String> urlsToRetrieveRelatedIDs = initialPayloads.parallelStream()
|
||||
.flatMap(payload -> Stream.of(payload.getOriginal_url(), payload.getActual_url())) // Add both "original_url" and "actual_url" in the final results.
|
||||
.collect(Collectors.toList());
|
||||
|
||||
// Prepare the "urlsToRetrieveRelatedIDs" to be used inside the "getDataForPayloadPrefillQuery". Create the following string-pattern: ("URL_1", "URL_2", ...)
|
||||
int urlsToRetrieveRelatedIDsSize = urlsToRetrieveRelatedIDs.size();
|
||||
int stringBuilderCapacity = (urlsToRetrieveRelatedIDsSize * 100);
|
||||
|
||||
// Get the id and url of any
|
||||
String getDataForPayloadPrefillQuery = "select distinct pu.id, pu.url\n" +
|
||||
"from " + DatabaseConnector.databaseName + ".publication_urls pu\n" +
|
||||
// Exclude the "already-processed" pairs.
|
||||
"left anti join " + DatabaseConnector.databaseName + ".attempt a on a.id=pu.id and a.original_url=pu.url\n" +
|
||||
"left anti join " + DatabaseConnector.databaseName + ".payload p on p.id=pu.id and p.original_url=pu.url\n" +
|
||||
"left anti join " + DatabaseConnector.databaseName + ".assignment asgn on asgn.id=pu.id and asgn.original_url=pu.url\n" +
|
||||
// Limit the urls to the ones matching to the payload-urls found for the current assignments.
|
||||
"where pu.url in " + getQueryListString(urlsToRetrieveRelatedIDs, urlsToRetrieveRelatedIDsSize, stringBuilderCapacity);
|
||||
|
||||
//logger.trace("getDataForPayloadPrefillQuery:\n" + getDataForPayloadPrefillQuery);
|
||||
|
||||
final List<Payload> prefilledPayloads = new ArrayList<>(1000);
|
||||
try {
|
||||
jdbcTemplate.query(getDataForPayloadPrefillQuery, rs -> {
|
||||
String id;
|
||||
String original_url;
|
||||
try { // For each of the 2 columns returned, do the following. The column-indexing starts from 1.
|
||||
id = rs.getString(1);
|
||||
original_url = rs.getString(2);
|
||||
} catch (SQLException sqle) {
|
||||
logger.error("No value was able to be retrieved from one of the columns of row_" + rs.getRow(), sqle);
|
||||
return; // Move to the next payload.
|
||||
}
|
||||
Set<Payload> foundPayloads = urlToPayloadsMultimap.get(original_url); // It doesn't return null, on error, but an empty set.
|
||||
if ( foundPayloads.size() == 0 ) {
|
||||
logger.error("No payloads associated with \"original_url\" = \"" + original_url + "\"!");
|
||||
return;
|
||||
}
|
||||
// Select a random "foundPayload" to use its data to fill the "prefilledPayload" (in a "Set" the first element is pseudo-random).
|
||||
// Since the "foundPayloads" is not-empty at this point, the, we do not have to check if the first element is present.
|
||||
Payload prefilledPayload = foundPayloads.stream().findFirst().get().clone(); // We take a clone of the original payload and change just the id and the original_url.
|
||||
prefilledPayload.setId(id);
|
||||
prefilledPayload.setOriginal_url(original_url);
|
||||
prefilledPayloads.add(prefilledPayload);
|
||||
});
|
||||
} catch (EmptyResultDataAccessException erdae) {
|
||||
logger.error("No results retrieved from the \"getDataForPayloadPrefillQuery\", when trying to prefill payloads, from assignment_" + assignmentsBatchCounter + ".");
|
||||
return false;
|
||||
} catch (Exception e) {
|
||||
DatabaseConnector.handleQueryException("getDataForPayloadPrefillQuery", getDataForPayloadPrefillQuery, e);
|
||||
return false;
|
||||
}
|
||||
|
||||
int numPrefilledPayloads = prefilledPayloads.size();
|
||||
if ( numPrefilledPayloads == 0 ) {
|
||||
logger.error("Some results were retrieved from the \"getDataForPayloadPrefillQuery\", but no data could be extracted from them, when trying to prefill payloads, from assignment_" + assignmentsBatchCounter + ".");
|
||||
return false;
|
||||
}
|
||||
|
||||
logger.debug("numPrefilledPayloads: " + numPrefilledPayloads + " | assignmentsBatchCounter: " + assignmentsBatchCounter);
|
||||
|
||||
// Add the prefilled to the UrlReports.
|
||||
final Error noError = new Error(null, null);
|
||||
for ( Payload prefilledPayload : prefilledPayloads )
|
||||
{
|
||||
urlReports.add(new UrlReport(UrlReport.StatusType.accessible, prefilledPayload, noError));
|
||||
}
|
||||
|
||||
logger.debug("Final number of UrlReports is " + urlReports.size() + " | assignmentsBatchCounter: " + assignmentsBatchCounter);
|
||||
|
||||
// In order to avoid assigning these "prefill" records to workers, before they are inserted in the attempt and payload tables..
|
||||
// We have to make sure this method is called inside a "DB-locked" code block and the "DB-unlock" happens only after all records are loaded into the DB-tables.
|
||||
return true;
|
||||
}
|
||||
|
||||
|
||||
|
@ -421,7 +571,7 @@ public class FileUtils {
|
|||
* @param filePayloads
|
||||
* @param s3Url
|
||||
*/
|
||||
public void setFullTextForMultiplePayloads(@NotNull Set<Payload> filePayloads, String s3Url) {
|
||||
public void setFullTextForMultiplePayloads(Set<Payload> filePayloads, String s3Url) {
|
||||
for ( Payload payload : filePayloads )
|
||||
if ( payload != null )
|
||||
payload.setLocation(s3Url); // Update the file-location to the new S3-url. All the other file-data is already set from the Worker.
|
||||
|
@ -455,17 +605,14 @@ public class FileUtils {
|
|||
}
|
||||
|
||||
|
||||
public static final Lock fileAccessLock = new ReentrantLock(true);
|
||||
Lock fileWriteLock = new ReentrantLock(true);
|
||||
|
||||
public String writeToFile(String fileFullPath, String stringToWrite, boolean shouldLockThreads)
|
||||
{
|
||||
if ( shouldLockThreads ) // In case multiple threads write to the same file. for ex. during the bulk-import procedure.
|
||||
fileAccessLock.lock();
|
||||
if ( shouldLockThreads )
|
||||
fileWriteLock.lock();
|
||||
|
||||
// TODO - Make this method to be synchronized for the specific file, not in general.
|
||||
// TODO - NOW: Multiple bulkImport procedures (with diff DIRs), are blocked while writing to DIFFERENT files..
|
||||
|
||||
try ( BufferedWriter bufferedWriter = new BufferedWriter(Files.newBufferedWriter(Paths.get(fileFullPath)), halfMb) )
|
||||
try ( BufferedWriter bufferedWriter = new BufferedWriter(Files.newBufferedWriter(Paths.get(fileFullPath)), FileUtils.tenMb) )
|
||||
{
|
||||
bufferedWriter.write(stringToWrite); // This will overwrite the file. If the new string is smaller, then it does not matter.
|
||||
} catch (Exception e) {
|
||||
|
@ -474,22 +621,9 @@ public class FileUtils {
|
|||
return errorMsg;
|
||||
} finally {
|
||||
if ( shouldLockThreads )
|
||||
fileAccessLock.unlock();
|
||||
fileWriteLock.unlock();
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
|
||||
public static String getQueryListString(List<String> list, int fileHashesListSize, int stringBuilderCapacity) {
|
||||
StringBuilder sb = new StringBuilder(stringBuilderCapacity);
|
||||
sb.append("(");
|
||||
for ( int i=0; i < fileHashesListSize; ++i ) {
|
||||
sb.append("\"").append(list.get(i)).append("\"");
|
||||
if ( i < (fileHashesListSize -1) )
|
||||
sb.append(", ");
|
||||
}
|
||||
sb.append(")");
|
||||
return sb.toString();
|
||||
}
|
||||
|
||||
}
|
||||
}
|
|
@ -1,404 +0,0 @@
|
|||
package eu.openaire.urls_controller.util;
|
||||
|
||||
import com.google.common.collect.HashMultimap;
|
||||
import eu.openaire.urls_controller.controllers.UrlsController;
|
||||
import eu.openaire.urls_controller.models.Payload;
|
||||
import eu.openaire.urls_controller.models.UrlReport;
|
||||
import eu.openaire.urls_controller.models.WorkerInfo;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.net.HttpURLConnection;
|
||||
import java.net.URL;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.nio.file.Paths;
|
||||
import java.text.DecimalFormat;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
import java.util.regex.Matcher;
|
||||
|
||||
@Component
|
||||
public class FilesHandler {
|
||||
|
||||
private static final Logger logger = LoggerFactory.getLogger(FilesHandler.class);
|
||||
|
||||
@Autowired
|
||||
private FileUtils fileUtils;
|
||||
|
||||
@Autowired
|
||||
private FileDecompressor fileDecompressor;
|
||||
|
||||
@Autowired
|
||||
private S3ObjectStore s3ObjectStore;
|
||||
|
||||
@Value("${services.pdfaggregation.worker.port}")
|
||||
private String workerPort;
|
||||
|
||||
public static final DecimalFormat df = new DecimalFormat("0.00");
|
||||
|
||||
|
||||
private static final int numOfFullTextsPerBatch = 70; // The HTTP-headers cannot be too large (It failed with 100 fileNames).
|
||||
|
||||
public String baseFilesLocation;
|
||||
|
||||
|
||||
public FilesHandler (@Value("${services.pdfaggregation.controller.baseFilesLocation}") String baseFilesLocation) {
|
||||
if ( !baseFilesLocation.endsWith(File.separator) )
|
||||
baseFilesLocation += File.separator;
|
||||
|
||||
if ( !baseFilesLocation.startsWith(File.separator) )
|
||||
baseFilesLocation = FileUtils.workingDir + baseFilesLocation;
|
||||
|
||||
this.baseFilesLocation = baseFilesLocation;
|
||||
}
|
||||
|
||||
|
||||
private HttpURLConnection getConnectionForFullTextBatch(String baseUrl, long assignmentsBatchCounter, int batchNum, List<String> fileNamesForCurBatch, int totalBatches, String workerId) throws RuntimeException
|
||||
{
|
||||
baseUrl += batchNum + "/";
|
||||
String requestUrl = getRequestUrlForBatch(baseUrl, fileNamesForCurBatch);
|
||||
//logger.debug("Going to request the batch_" + batchNum + " (out of " + totalBatches + ") with " + fileNamesForCurBatch.size() + " fullTexts, of assignments_" + assignmentsBatchCounter + " from the Worker with ID \"" + workerId + "\" and baseRequestUrl: " + baseUrl + "[fileNames]");
|
||||
try {
|
||||
HttpURLConnection conn = (HttpURLConnection) new URL(requestUrl).openConnection();
|
||||
conn.setRequestMethod("GET");
|
||||
conn.setRequestProperty("User-Agent", "UrlsController");
|
||||
|
||||
// TODO - Write the fileNames in the RequestBody, so that we can include as many as we want in each request.
|
||||
// Right now, we can include only up to 70-80 fileNames in the url-string.
|
||||
// TODO - We need to add the fileNames in the requestBody BEFORE we connect. So we will need to refactor the code to work in that order.
|
||||
/*OutputStream os = conn.getOutputStream();
|
||||
OutputStreamWriter osw = new OutputStreamWriter(os, StandardCharsets.UTF_8);
|
||||
osw.write(fileNamesForCurBatch_separated_by_comma);
|
||||
osw.flush();
|
||||
osw.close();
|
||||
os.close();*/
|
||||
|
||||
|
||||
// TODO - The above update will also enable is to improve edge-case management like making sure we do not create a whole new batch for just a few files..
|
||||
// Check this case for example, where we have one extra batch with the network, compression-decompression, transfer, uploading ect overhead:
|
||||
// 2023-02-06 12:17:26.893 [http-nio-1880-exec-8] DEBUG e.o.urls_controller.util.FileUtils.getAndUploadFullTexts(@235) - The assignments_12 have 211 distinct non-already-uploaded fullTexts.
|
||||
// Going to request them from the Worker "worker_X", in 4 batches (70 files each, except for the final batch, which will have 1 files).
|
||||
|
||||
// If we are not limited by the url-length we can easily say that if less than 10 files remain for the last batch, then add them to the previous batch (eg. the last batch will have 79 files)
|
||||
// If equal to 10 or more files remain, then we will make an extra batch.
|
||||
|
||||
conn.connect();
|
||||
int statusCode = conn.getResponseCode();
|
||||
if ( statusCode == -1 ) { // Invalid HTTP-Response.
|
||||
logger.warn("Problem when getting the \"status-code\" for url: " + requestUrl);
|
||||
throw new RuntimeException(); // Avoid any other batches.
|
||||
} else if ( statusCode != 200 ) {
|
||||
String errMsg = fileUtils.getMessageFromResponseBody(conn, true);
|
||||
logger.warn("HTTP-" + statusCode + ": " + errMsg + "\n\nProblem when requesting the ZstdFile of batch_" + batchNum + " from the Worker with ID \"" + workerId + "\" and requestUrl: " + requestUrl);
|
||||
if ( ((statusCode >= 500) && (statusCode <= 599))
|
||||
|| ((statusCode == 400) && ((errMsg != null) && errMsg.contains("The base directory for assignments_" + assignmentsBatchCounter + " was not found"))) )
|
||||
throw new RuntimeException(); // Throw an exception to indicate that the Worker has problems and all remaining batches will fail as well.
|
||||
return null;
|
||||
} else
|
||||
return conn;
|
||||
} catch (RuntimeException re) {
|
||||
throw re;
|
||||
} catch (Exception e) {
|
||||
String exMessage = e.getMessage();
|
||||
logger.warn("Problem when requesting the ZstdFile of batch_" + batchNum + " of assignments_" + assignmentsBatchCounter + " from the Worker with ID \"" + workerId + "\" and requestUrl: " + requestUrl + GenericUtils.endOfLine + exMessage);
|
||||
if ( exMessage.contains("Connection refused") ) {
|
||||
logger.error("Since we received a \"Connection refused\", from \"" + workerId + "\", all of the remaining batches (" + (totalBatches - batchNum) + ") will not be requested!");
|
||||
throw new RuntimeException();
|
||||
}
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
public FileUtils.UploadFullTextsResponse getAndUploadFullTexts(List<UrlReport> urlReports, int sizeOfUrlReports, long assignmentsBatchCounter, String workerId) throws RuntimeException
|
||||
{
|
||||
// The Controller have to request the files from the Worker, in order to upload them to the S3.
|
||||
// We UPDATE the "location" of each of those files in the UrlReports and then insert them all into the database.
|
||||
|
||||
String workerIp = null;
|
||||
WorkerInfo workerInfo = UrlsController.workersInfoMap.get(workerId);
|
||||
if ( workerInfo == null ) {
|
||||
logger.error("Could not retrieve the info of worker: " + workerId);
|
||||
return null;
|
||||
} else
|
||||
workerIp = workerInfo.getWorkerIP(); // This won't be null.
|
||||
|
||||
// Get the file-locations.
|
||||
int numValidFullTextsFound = 0;
|
||||
int numFilesFoundFromPreviousAssignmentsBatches = 0;
|
||||
int numFullTextsWithProblematicLocations = 0;
|
||||
|
||||
HashMultimap<String, Payload> hashesWithPayloads = fileUtils.getHashesWithPayloads(urlReports, sizeOfUrlReports); // Holds multiple payloads for the same fileHash.
|
||||
Set<String> fileHashes = hashesWithPayloads.keySet();
|
||||
int fileHashesSetSize = fileHashes.size(); // Get the size of the keysSet, instead of the whole multimap.
|
||||
if ( fileHashesSetSize == 0 ) {
|
||||
logger.warn("No fulltexts were retrieved for assignments_" + assignmentsBatchCounter + ", from worker: \"" + workerId + "\".");
|
||||
return FileUtils.UploadFullTextsResponse.successful_without_fulltexts; // It was handled, no error.
|
||||
}
|
||||
|
||||
HashMap<String, String> hashLocationMap = fileUtils.getHashLocationMap(fileHashes, fileHashesSetSize, assignmentsBatchCounter, "assignments");
|
||||
HashMultimap<String, Payload> allFileNamesWithPayloads = HashMultimap.create((sizeOfUrlReports / 5), 3); // Holds multiple values for any key, if a fileName(key) has many IDs(values) associated with it.
|
||||
|
||||
for ( String fileHash : fileHashes )
|
||||
{
|
||||
for ( Payload payload : hashesWithPayloads.get(fileHash) )
|
||||
{
|
||||
String alreadyFoundFileLocation = hashLocationMap.get(fileHash); // Only one location has been retrieved per fileHash.
|
||||
if ( alreadyFoundFileLocation != null ) {
|
||||
// Fill the payloads with locations from the "previously-found-hashes."
|
||||
payload.setLocation(alreadyFoundFileLocation);
|
||||
if ( logger.isTraceEnabled() )
|
||||
logger.trace("The record with ID \"" + payload.getId() + "\" has an \"alreadyRetrieved\" file, with hash \"" + fileHash + "\" and location \"" + alreadyFoundFileLocation + "\"."); // DEBUG!
|
||||
numFilesFoundFromPreviousAssignmentsBatches ++;
|
||||
numValidFullTextsFound ++; // We trust the location being valid..
|
||||
}
|
||||
else { // This file has not been found before..
|
||||
// Extract the "fileNameWithExtension" to be added in the HashMultimap.
|
||||
String fileLocation = payload.getLocation();
|
||||
Matcher matcher = FileUtils.FILEPATH_ID_EXTENSION.matcher(fileLocation);
|
||||
if ( ! matcher.matches() ) {
|
||||
logger.error("Failed to match the \"fileLocation\": \"" + fileLocation + "\" of id: \"" + payload.getId() + "\", originalUrl: \"" + payload.getOriginal_url() + "\", using this regex: " + FileUtils.FILEPATH_ID_EXTENSION);
|
||||
numFullTextsWithProblematicLocations ++;
|
||||
continue;
|
||||
}
|
||||
String fileNameWithExtension = matcher.group(2);
|
||||
if ( (fileNameWithExtension == null) || fileNameWithExtension.isEmpty() ) {
|
||||
logger.error("Failed to extract the \"fileNameWithExtension\" from \"fileLocation\": \"" + fileLocation + "\", of id: \"" + payload.getId() + "\", originalUrl: \"" + payload.getOriginal_url() + "\", using this regex: " + FileUtils.FILEPATH_ID_EXTENSION);
|
||||
numFullTextsWithProblematicLocations ++;
|
||||
continue;
|
||||
}
|
||||
|
||||
numValidFullTextsFound ++;
|
||||
allFileNamesWithPayloads.put(fileNameWithExtension, payload); // The keys and the values are not duplicate.
|
||||
// Task with ID-1 might have an "ID-1.pdf" file, while a task with ID-2 can also have an "ID-1.pdf" file, as the pdf-url-2 might be the same with pdf-url-1, thus, the ID-2 file was not downloaded again.
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if ( numFullTextsWithProblematicLocations > 0 )
|
||||
logger.warn(numFullTextsWithProblematicLocations + " files had problematic names.");
|
||||
|
||||
if ( numValidFullTextsFound == 0 ) {
|
||||
logger.warn("No full-text files were retrieved for assignments_" + assignmentsBatchCounter + " | from worker: " + workerId);
|
||||
return FileUtils.UploadFullTextsResponse.successful_without_fulltexts; // It's not what we want, but it's not an error either.
|
||||
}
|
||||
|
||||
ArrayList<String> allFileNames = new ArrayList<>(allFileNamesWithPayloads.keySet()); // The number of fulltexts are lower than the number of payloads, since multiple payloads may lead to the same file.
|
||||
int numFullTextsToBeRequested = allFileNames.size();
|
||||
if ( numFullTextsToBeRequested == 0 ) {
|
||||
logger.info(numValidFullTextsFound + " fulltexts were retrieved for assignments_" + assignmentsBatchCounter + ", from worker: \"" + workerId + "\", but all of them have been retrieved before.");
|
||||
return FileUtils.UploadFullTextsResponse.successful_without_fulltexts; // It was handled, no error.
|
||||
}
|
||||
|
||||
logger.info("NumFullTextsFound by assignments_" + assignmentsBatchCounter + " = " + numValidFullTextsFound + " (out of " + sizeOfUrlReports + " | about " + df.format(numValidFullTextsFound * 100.0 / sizeOfUrlReports) + "%).");
|
||||
|
||||
// TODO - Have a prometheus GAUGE to hold the value of the above percentage, so that we can track the success-rates over time..
|
||||
|
||||
logger.debug("NumFilesFoundFromPreviousAssignmentsBatches = " + numFilesFoundFromPreviousAssignmentsBatches);
|
||||
|
||||
// Request the full-texts in batches, compressed in a zstd tar file.
|
||||
int numOfBatches = (numFullTextsToBeRequested / numOfFullTextsPerBatch);
|
||||
int remainingFiles = (numFullTextsToBeRequested % numOfFullTextsPerBatch);
|
||||
if ( remainingFiles > 0 ) { // Add an extra batch for the remaining files. This guarantees at least one batch will exist no matter how few (>0) the files are.
|
||||
numOfBatches++;
|
||||
logger.debug("The assignments_" + assignmentsBatchCounter + " have " + numFullTextsToBeRequested + " distinct, non-already-uploaded fullTexts (total is: " + numValidFullTextsFound + "). Going to request them from the Worker \"" + workerId + "\", in " + numOfBatches + " batches (" + numOfFullTextsPerBatch + " files each, except for the final batch, which will have " + remainingFiles + " files).");
|
||||
} else
|
||||
logger.debug("The assignments_" + assignmentsBatchCounter + " have " + numFullTextsToBeRequested + " distinct, non-already-uploaded fullTexts (total is: " + numValidFullTextsFound + "). Going to request them from the Worker \"" + workerId + "\", in " + numOfBatches + " batches (" + numOfFullTextsPerBatch + " files each).");
|
||||
|
||||
// Check if one full text is left out because of the division. Put it int the last batch.
|
||||
String baseUrl = "http://" + workerIp + ":" + workerPort + "/api/full-texts/getFullTexts/" + assignmentsBatchCounter + "/" + numOfBatches + "/";
|
||||
|
||||
// TODO - The worker should send the port in which it accepts requests, along with the current request.
|
||||
// TODO - The least we have to do it to expose the port-assignment somewhere more obvious like inside the "application.yml" file.
|
||||
|
||||
String curAssignmentsBaseLocation = baseFilesLocation + "assignments_" + assignmentsBatchCounter + File.separator;
|
||||
// Note: the "curAssignmentsBaseLocation"-directory will be created once the first batch subdirectory is called for creation.
|
||||
|
||||
int failedBatches = 0;
|
||||
for ( int batchCounter = 1; batchCounter <= numOfBatches; ++batchCounter ) {
|
||||
// TODO - Currently, for big assignments (e.g. 10000), it takes 2 mins (actually 1,5 mins after using the Zstandard compression) for the worker to zstd the files and return them FOR EACH BATCH
|
||||
// Also it takes around 3 mins for the Controller to process the received files FOR EACH BATCH
|
||||
// So, for 24 batches, it takes around 24 * 2 * 3 = 144 mins to process all the full-texts for each assignments-batch.
|
||||
|
||||
// TODO - What if we could passing a new thread for each full-texts-batch, and make them "FIRE" one after the other.
|
||||
// TODO - So, the 1st thread with the first batch starts and waits for the the first zstd file from the worker,
|
||||
// Once it takes the zstd file it continues, but now the Worker is just sitting waiting.. So the 2nd thread fires and asks the new zstd.
|
||||
// So, the worker will never be left waiting and the Controller will not wait for the Worker either..!
|
||||
|
||||
// The worker will not have 2 parallel requests for zstd files, so the single CPU there will not be stressed to zstd many files in parallel.
|
||||
// Yes the Controller may have the situation in which before finishing uploading the previously receive files to S3, it receives the new zstd from the Worker.
|
||||
// TODO - BUT, we can make the new thread "WAIT" for the previous to finish.
|
||||
|
||||
String targetDirectory = curAssignmentsBaseLocation + "batch_" + batchCounter + File.separator;
|
||||
Path curBatchPath;
|
||||
try { // Create this batch-directory.
|
||||
curBatchPath = Files.createDirectories(Paths.get(targetDirectory));
|
||||
// The base-directory will be created along with the first batch-directory.
|
||||
} catch (Exception e) {
|
||||
logger.error("Could not create the \"curBatchPath\" directory: " + targetDirectory + GenericUtils.endOfLine + e.getMessage(), e); // It shows the response body (after Spring v.2.5.6).
|
||||
failedBatches ++;
|
||||
continue;
|
||||
}
|
||||
|
||||
List<String> fileNamesForCurBatch = getFileNamesForBatch(allFileNames, numFullTextsToBeRequested, batchCounter);
|
||||
String zstdFileFullPath = targetDirectory + "fullTexts_" + assignmentsBatchCounter + "_" + batchCounter + ".tar.zstd";
|
||||
try {
|
||||
if ( ! getAndSaveFullTextBatch(fileNamesForCurBatch, baseUrl, assignmentsBatchCounter, batchCounter, numOfBatches, zstdFileFullPath, workerId) ) {
|
||||
failedBatches ++;
|
||||
continue;
|
||||
}
|
||||
} catch (RuntimeException re) {
|
||||
failedBatches += (1 + (numOfBatches - batchCounter)); // The "failedBatches" will have the previously failedBatches + this one + the remaining batches which will likely fail too, thus, they will not be tested. Some initial batches may have succeeded.
|
||||
break;
|
||||
}
|
||||
|
||||
if ( ! fileDecompressor.decompressAndUploadFullTexts(zstdFileFullPath, curBatchPath, targetDirectory, fileNamesForCurBatch, batchCounter, allFileNamesWithPayloads, assignmentsBatchCounter) )
|
||||
failedBatches ++;
|
||||
} // End of batches.
|
||||
|
||||
if ( failedBatches == numOfBatches )
|
||||
logger.error("None of the " + numOfBatches + " batches could be handled for assignments_" + assignmentsBatchCounter + ", for worker: " + workerId);
|
||||
|
||||
removeUnretrievedFullTextsFromUrlReports(urlReports, true); // Make sure all records without an S3-Url have < null > file-data (some batches or uploads might have failed).
|
||||
fileUtils.deleteDirectory(new File(curAssignmentsBaseLocation));
|
||||
|
||||
// Check and warn about the number of failed payloads.
|
||||
// Possible reasons: failed to check their hash in the DB, the file was not found inside the worker, whole batch failed to be delivered from the worker, files failed t be uploaded to S3
|
||||
// Retrieve the payloads from the existing urlReports.
|
||||
long finalPayloadsCounter = urlReports.parallelStream()
|
||||
.map(UrlReport::getPayload).filter(payload -> ((payload != null) && (payload.getLocation() != null)))
|
||||
.count();
|
||||
int numInitialPayloads = (numValidFullTextsFound + numFullTextsWithProblematicLocations);
|
||||
long numFailedPayloads = (numInitialPayloads - finalPayloadsCounter);
|
||||
if ( numFailedPayloads == numInitialPayloads ) {
|
||||
// This will also be the case if there was no DB failure, but all the batches have failed.
|
||||
logger.error("None of the " + numInitialPayloads + " payloads could be handled for assignments_" + assignmentsBatchCounter + ", for worker: " + workerId);
|
||||
return FileUtils.UploadFullTextsResponse.unsuccessful;
|
||||
} else if ( numFailedPayloads > 0 )
|
||||
logger.warn(numFailedPayloads + " payloads (out of " + numInitialPayloads + ") failed to be processed for assignments_" + assignmentsBatchCounter + ", for worker: " + workerId);
|
||||
|
||||
return FileUtils.UploadFullTextsResponse.successful;
|
||||
}
|
||||
|
||||
|
||||
private boolean getAndSaveFullTextBatch(List<String> fileNamesForCurBatch, String baseUrl, long assignmentsBatchCounter, int batchCounter, int numOfBatches,
|
||||
String zstdFileFullPath, String workerId) throws RuntimeException
|
||||
{
|
||||
HttpURLConnection conn;
|
||||
InputStream inputStream;
|
||||
try {
|
||||
if ( (conn = getConnectionForFullTextBatch(baseUrl, assignmentsBatchCounter, batchCounter, fileNamesForCurBatch, numOfBatches, workerId)) == null )
|
||||
return false;
|
||||
|
||||
inputStream = conn.getInputStream();
|
||||
} catch (RuntimeException re) {
|
||||
// The "cause" was logged inside "getConnection()".
|
||||
throw re;
|
||||
} catch (IOException ioe) {
|
||||
throw new RuntimeException(ioe.getMessage());
|
||||
}
|
||||
|
||||
// Save and decompress the zstd file. Iterate over the PDFs and upload each one of them and get the S3-Url.
|
||||
return fileUtils.saveArchive(inputStream, new File(zstdFileFullPath));
|
||||
// We do not call "conn.disconnect()", since more request are about to be made to the worker, in the near future.
|
||||
|
||||
//private static final RestTemplate restTemplate = new RestTemplate();
|
||||
|
||||
|
||||
//baseUrl += batchCounter + "/";
|
||||
//String requestUrl = getRequestUrlForBatch(baseUrl, fileNamesForCurBatch);
|
||||
|
||||
// Define a response extractor that returns an input stream
|
||||
//ResponseExtractor<InputStream> responseExtractor = HttpInputMessage::getBody;
|
||||
|
||||
/*ResponseExtractor<InputStream> responseExtractor = new ResponseExtractor<InputStream>() {
|
||||
@Override
|
||||
public InputStream extractData(ClientHttpResponse response) throws IOException {
|
||||
return response.getBody();
|
||||
}
|
||||
};
|
||||
|
||||
try {
|
||||
//ResponseEntity<Object> responseEntity = restTemplate.exchange(requestUrl, HttpMethod.GET, null, Object.class);
|
||||
InputStream inputStream = restTemplate.exchange(requestUrl, HttpMethod.GET, null, responseExtractor);
|
||||
//InputStream inputStream = (InputStream) responseEntity.getBody();
|
||||
return saveArchive(inputStream, new File(zstdFileFullPath));
|
||||
} catch (Exception e) {
|
||||
logger.error("Could not get the file from the response body!", e);
|
||||
return false;
|
||||
}*/
|
||||
}
|
||||
|
||||
|
||||
private String getRequestUrlForBatch(String baseUrl, List<String> fileNamesForCurBatch)
|
||||
{
|
||||
final StringBuilder sb = new StringBuilder(numOfFullTextsPerBatch * 50);
|
||||
sb.append(baseUrl);
|
||||
int numFullTextsCurBatch = fileNamesForCurBatch.size();
|
||||
for ( int j=0; j < numFullTextsCurBatch; ++j ){
|
||||
sb.append(fileNamesForCurBatch.get(j));
|
||||
if ( j < (numFullTextsCurBatch -1) )
|
||||
sb.append(",");
|
||||
}
|
||||
return sb.toString();
|
||||
}
|
||||
|
||||
|
||||
private List<String> getFileNamesForBatch(List<String> allFileNames, int numAllFullTexts, int curBatch)
|
||||
{
|
||||
int initialIndex = ((curBatch-1) * numOfFullTextsPerBatch);
|
||||
int endingIndex = (curBatch * numOfFullTextsPerBatch);
|
||||
if ( endingIndex > numAllFullTexts ) // This might be the case, when the "numAllFullTexts" is too small.
|
||||
endingIndex = numAllFullTexts;
|
||||
|
||||
final List<String> fileNamesOfCurBatch = new ArrayList<>(numOfFullTextsPerBatch);
|
||||
for ( int i = initialIndex; i < endingIndex; ++i ) {
|
||||
try {
|
||||
fileNamesOfCurBatch.add(allFileNames.get(i));
|
||||
} catch (IndexOutOfBoundsException ioobe) {
|
||||
logger.error("IOOBE for i=" + i + GenericUtils.endOfLine + ioobe.getMessage(), ioobe);
|
||||
}
|
||||
}
|
||||
return fileNamesOfCurBatch;
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* This method updates the UrlReports to not point to any downloaded fullText files.
|
||||
* This is useful when the uploading process of the fullTexts to the S3-ObjectStore fails, and we don't want any "links" to locally stored files, which will be deleted.
|
||||
* If the "shouldCheckAndKeepS3UploadedFiles" is set to "true", then the payloads which have their file uploaded to the S3-ObjectStore, are excluded.
|
||||
* @param urlReports
|
||||
* @param shouldCheckAndKeepS3UploadedFiles
|
||||
*/
|
||||
public void removeUnretrievedFullTextsFromUrlReports(List<UrlReport> urlReports, boolean shouldCheckAndKeepS3UploadedFiles)
|
||||
{
|
||||
for ( UrlReport urlReport : urlReports ) {
|
||||
Payload payload = urlReport.getPayload();
|
||||
if ( payload == null )
|
||||
continue;
|
||||
|
||||
if ( shouldCheckAndKeepS3UploadedFiles ) {
|
||||
String fileLocation = payload.getLocation();
|
||||
if ( (fileLocation == null) || s3ObjectStore.isLocationInStore(fileLocation) )
|
||||
continue;
|
||||
}
|
||||
|
||||
// Mark this full-text as not-retrieved, since it will be deleted from local-storage. The retrieved link to the full-text ("actual_url") will be kept, for now.
|
||||
payload.setLocation(null); // This will cause the payload to not be inserted into the "payload" table in the database. Only the "attempt" record will be inserted.
|
||||
payload.setHash(null);
|
||||
payload.setMime_type(null);
|
||||
payload.setSize(null);
|
||||
// The id-url record will be called as a new assignment in the future.
|
||||
}
|
||||
}
|
||||
|
||||
}
|
|
@ -1,22 +1,11 @@
|
|||
package eu.openaire.urls_controller.util;
|
||||
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import javax.servlet.http.HttpServletRequest;
|
||||
import java.text.SimpleDateFormat;
|
||||
import java.util.Date;
|
||||
|
||||
public class GenericUtils {
|
||||
|
||||
private static final Logger logger = LoggerFactory.getLogger(GenericUtils.class);
|
||||
|
||||
|
||||
public static final String endOfLine = "\n";
|
||||
public static final String tab = "\t";
|
||||
|
||||
private static final SimpleDateFormat simpleDateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSS z");
|
||||
|
||||
private static final SimpleDateFormat simpleDateFormat = new SimpleDateFormat("yyyy-MM-dd 'at' HH:mm:ss.SSS z");
|
||||
|
||||
public static String getReadableCurrentTimeAndZone() {
|
||||
return (simpleDateFormat.format(new Date(System.currentTimeMillis())));
|
||||
|
@ -28,41 +17,17 @@ public class GenericUtils {
|
|||
}
|
||||
|
||||
|
||||
|
||||
public static String getSelectedStackTraceForCausedException(Throwable thr, String firstMessage, String additionalMessage, int numOfLines)
|
||||
{
|
||||
// The stacktrace of the "ExecutionException" is the one of the current code and not the code which ran inside the background-task. Try to get the cause.
|
||||
Throwable causedThrowable = thr.getCause();
|
||||
if ( causedThrowable == null ) {
|
||||
logger.warn("No cause was retrieved for the \"ExecutionException\"!");
|
||||
causedThrowable = thr;
|
||||
}
|
||||
String initialMessage = firstMessage + causedThrowable.getMessage() + ((additionalMessage != null) ? additionalMessage : "");
|
||||
return getSelectiveStackTrace(causedThrowable, initialMessage, numOfLines);
|
||||
}
|
||||
|
||||
|
||||
public static String getSelectiveStackTrace(Throwable thr, String initialMessage, int numOfLines)
|
||||
{
|
||||
StackTraceElement[] stels = thr.getStackTrace();
|
||||
StringBuilder sb = new StringBuilder(numOfLines *100); // This StringBuilder is thread-safe as a local-variable.
|
||||
StringBuilder sb = new StringBuilder(numOfLines *100);
|
||||
if ( initialMessage != null )
|
||||
sb.append(initialMessage).append(endOfLine);
|
||||
sb.append("Stacktrace:").append(endOfLine);
|
||||
for ( int i = 0; (i < stels.length) && (i < numOfLines); ++i ) {
|
||||
sb.append(tab).append(stels[i]);
|
||||
if (i < (numOfLines -1)) sb.append(endOfLine);
|
||||
sb.append(initialMessage).append(" Stacktrace:").append("\n"); // This StringBuilder is thread-safe as a local-variable.
|
||||
for ( int i = 0; (i < stels.length) && (i <= numOfLines); ++i ) {
|
||||
sb.append(stels[i]);
|
||||
if (i < numOfLines) sb.append("\n");
|
||||
}
|
||||
return sb.toString();
|
||||
}
|
||||
|
||||
|
||||
public static String getRequestorAddress(HttpServletRequest request)
|
||||
{
|
||||
String remoteAddr = request.getHeader("X-FORWARDED-FOR"); // This retrieves the original IP address, if the request passes through a proxy server.
|
||||
if ( remoteAddr == null )
|
||||
remoteAddr = request.getRemoteAddr();
|
||||
return remoteAddr;
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
@ -1,9 +1,10 @@
|
|||
package eu.openaire.urls_controller.util;
|
||||
|
||||
import com.google.common.collect.Lists;
|
||||
import eu.openaire.urls_controller.configuration.DatabaseConnector;
|
||||
import eu.openaire.urls_controller.configuration.ImpalaConnector;
|
||||
import eu.openaire.urls_controller.models.Error;
|
||||
import eu.openaire.urls_controller.models.*;
|
||||
import eu.openaire.urls_controller.services.UrlsServiceImpl;
|
||||
import org.apache.avro.Schema;
|
||||
import org.apache.avro.generic.GenericData;
|
||||
import org.apache.avro.generic.GenericRecord;
|
||||
|
@ -23,7 +24,6 @@ import org.slf4j.LoggerFactory;
|
|||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.core.io.ClassPathResource;
|
||||
import org.springframework.dao.EmptyResultDataAccessException;
|
||||
import org.springframework.http.HttpStatus;
|
||||
import org.springframework.http.ResponseEntity;
|
||||
import org.springframework.jdbc.core.JdbcTemplate;
|
||||
|
@ -44,10 +44,7 @@ import java.util.ArrayList;
|
|||
import java.util.Base64;
|
||||
import java.util.List;
|
||||
import java.util.concurrent.Callable;
|
||||
import java.util.concurrent.CancellationException;
|
||||
import java.util.concurrent.ExecutionException;
|
||||
import java.util.concurrent.Future;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
@Component
|
||||
public class ParquetFileUtils {
|
||||
|
@ -70,10 +67,10 @@ public class ParquetFileUtils {
|
|||
private final String hdfsUserName;
|
||||
|
||||
|
||||
private static final String attemptSchemaFilePath = "schemas/attempt.avsc";
|
||||
|
||||
public static final String payloadSchemaFilePath = "schemas/payload.avsc";
|
||||
|
||||
private static final String attemptSchemaFilePath = "schemas/attempt.avsc";
|
||||
|
||||
public static Schema payloadsSchema = null;
|
||||
public Schema attemptsSchema;
|
||||
|
||||
|
@ -157,89 +154,58 @@ public class ParquetFileUtils {
|
|||
}
|
||||
|
||||
|
||||
/**
|
||||
* The tasks created below must be executed inside a "database-lock" block.
|
||||
* */
|
||||
public List<Callable<ParquetReport>> getTasksForCreatingAndUploadingParquetFiles(List<UrlReport> urlReports, int sizeOfUrlReports, long curReportAssignments, String currentParquetPath, FileUtils.UploadFullTextsResponse uploadFullTextsResponse)
|
||||
{
|
||||
if ( (attemptsSchema == null) // Parse the schema if it's not already parsed.
|
||||
&& ((attemptsSchema = parseSchema(attemptSchemaFilePath)) == null ) ) {
|
||||
logger.error("Nothing can be done without the attemptsSchema! Exiting.."); // The cause is already logged inside the above method.
|
||||
System.exit(88); // Exit the whole app, as it cannot add the results to the database!
|
||||
}
|
||||
// Split the "UrlReports" into some sub-lists.
|
||||
List<List<UrlReport>> subLists;
|
||||
|
||||
// Pre-define the tasks to run in multiple threads.
|
||||
final List<Callable<ParquetReport>> callableTasks = new ArrayList<>(8); // 5 threads will handle the attempts and 3 the payloads.
|
||||
// Pre-define the tasks to run.
|
||||
List<Callable<ParquetReport>> callableTasks = new ArrayList<>(6);
|
||||
// One thread will handle the inserts to the "payload" table and the others to the "attempt" table. This way there will be as little blocking as possible (from the part of Impala).
|
||||
|
||||
if ( uploadFullTextsResponse == FileUtils.UploadFullTextsResponse.successful )
|
||||
int sizeOfEachSubList = (int)(sizeOfUrlReports * 0.2);
|
||||
if ( sizeOfEachSubList > 10 )
|
||||
{
|
||||
List<Payload> initialPayloads = urlReports.parallelStream()
|
||||
.map(UrlReport::getPayload).filter(payload -> ((payload != null) && (payload.getLocation() != null)))
|
||||
.collect(Collectors.toList());
|
||||
|
||||
int numInitialPayloads = initialPayloads.size();
|
||||
if ( numInitialPayloads > 0 ) // If at least 1 payload was created by the processed records..
|
||||
{ // (it's ok to have no payloads, if there were no full-texts available)
|
||||
// At this point we know there was no problem with the full-texts, but we do not know if at least one full-text was retrieved FROM the worker.
|
||||
if ( (payloadsSchema == null) // Parse the schema if it's not already parsed.
|
||||
&& ((payloadsSchema = parseSchema(payloadSchemaFilePath)) == null ) ) {
|
||||
logger.error("Nothing can be done without the payloadsSchema! Exiting.."); // The cause is already logged inside the above method.
|
||||
System.exit(89); // Exit the whole app, as it cannot add the results to the database!
|
||||
}
|
||||
|
||||
// The UrlsReports for the pre-filled are added only here, since we do not want attempt records fo these.
|
||||
fileUtils.addUrlReportsByMatchingRecordsFromBacklog(urlReports, initialPayloads, numInitialPayloads, curReportAssignments); // This will add more Object in the update the "urlReports" list.
|
||||
// In case the above method returns an error, nothing happens. We just have only the initial payloads to insert to the DB.
|
||||
|
||||
int sizeOfEachSubList = (int)(sizeOfUrlReports * 0.33); // We want 3 sub-lists for the payloads.
|
||||
if ( sizeOfEachSubList == 0 ) // If the "sizeOfUrlReports" is <= 3.
|
||||
sizeOfEachSubList = 1;
|
||||
// There may be 1 more with very few elements, due to non-persisted splitting. Unfortunately, we cannot st the number of splits, only the size of most splits.
|
||||
if ( sizeOfEachSubList > 10 ) {
|
||||
List<List<UrlReport>> finalSubLists = Lists.partition(urlReports, sizeOfEachSubList); // This needs the "sizeOfEachSubList" to be above < 0 >.
|
||||
int numSubListsPayload = finalSubLists.size();
|
||||
// We will run <numSubListsPayload> tasks for the payloads.
|
||||
// Since the payloads are not evenly destributed in the urlReports, there may be some sub-lists of urlReports without any payloads inside.
|
||||
for ( int i = 0; i < numSubListsPayload; ++i ) {
|
||||
int finalI = i;
|
||||
callableTasks.add(() -> { // Handle inserts to the "payload" table. Around 20% of the total amount.
|
||||
return new ParquetReport(ParquetReport.ParquetType.payload, createPayloadParquetDataAndUploadToHDFS(finalI, finalSubLists.get(finalI), curReportAssignments, currentParquetPath, (parquetHDFSDirectoryPathPayloadsAggregated + curReportAssignments + "/")));
|
||||
});
|
||||
}
|
||||
} else {
|
||||
// If the "urlReports" are so few, that we cannot get big "sublists", assign a single task to handle all the payload (sizeOfEachSubList * 5).
|
||||
callableTasks.add(() -> { // Handle inserts to the "payload" table. Around 20% of the total amount.
|
||||
return new ParquetReport(ParquetReport.ParquetType.payload, createPayloadParquetDataAndUploadToHDFS(0, urlReports, curReportAssignments, currentParquetPath, (parquetHDFSDirectoryPathPayloadsAggregated + curReportAssignments + "/")));
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
int sizeOfEachSubList = (int)(sizeOfUrlReports * 0.2); // We want 5 sub-lists for the attempts.
|
||||
// There may be 1 more with very few elements, due to non-persisted splitting. Unfortunately, we cannot st the number of splits, only the size of most splits.
|
||||
if ( sizeOfEachSubList > 10 ) {
|
||||
List<List<UrlReport>> finalSubLists = Lists.partition(urlReports, sizeOfEachSubList); // This needs the "sizeOfEachSubList" to be above < 0 >.
|
||||
subLists = Lists.partition(urlReports, sizeOfEachSubList); // This needs the "sizeOfEachSubList" to be above < 0 >.
|
||||
// The above will create some sub-lists, each one containing 20% of total amount.
|
||||
int numSubListsAttempt = finalSubLists.size();
|
||||
// We will run <numSubListsAttempt> tasks for the payloads.
|
||||
for ( int i = 0; i < numSubListsAttempt; ++i ) {
|
||||
int subListsSize = subLists.size();
|
||||
for ( int i = 0; i < subListsSize; ++i ) {
|
||||
int finalI = i;
|
||||
callableTasks.add(() -> { // Handle inserts to the "attempt" table. Insert 20% of the "attempt" queries.
|
||||
return new ParquetReport(ParquetReport.ParquetType.attempt, createAttemptParquetDataAndUploadToHDFS(finalI, finalSubLists.get(finalI), curReportAssignments, currentParquetPath));
|
||||
return new ParquetReport(ParquetReport.ParquetType.attempt, createAndLoadParquetDataIntoAttemptTable(finalI, subLists.get(finalI), curReportAssignments, currentParquetPath));
|
||||
});
|
||||
}
|
||||
} else {
|
||||
// If the "urlReports" are so few, that we cannot get big "sublists", assign a single task to handle all the attempts (sizeOfEachSubList * 5).
|
||||
callableTasks.add(() -> { // Handle inserts to the "attempt" table.
|
||||
return new ParquetReport(ParquetReport.ParquetType.attempt, createAttemptParquetDataAndUploadToHDFS(0, urlReports, curReportAssignments, currentParquetPath));
|
||||
// If the "urlReports" are so few, that we cannot get big "sublists", assign a single task to handle all the attempts.
|
||||
callableTasks.add(() -> { // Handle inserts to the "attempt" table. Insert 20% of the "attempt" queries.
|
||||
return new ParquetReport(ParquetReport.ParquetType.attempt, createAndLoadParquetDataIntoAttemptTable(0, urlReports, curReportAssignments, currentParquetPath));
|
||||
});
|
||||
}
|
||||
|
||||
if ( uploadFullTextsResponse == FileUtils.UploadFullTextsResponse.successful )
|
||||
{
|
||||
// At this point we know there was no problem with the full-texts, but we do not know if at least one full-text was retrieved.
|
||||
if ( (payloadsSchema == null) // Parse the schema if it's not already parsed.
|
||||
&& ((payloadsSchema = parseSchema(payloadSchemaFilePath)) == null ) ) {
|
||||
logger.error("Nothing can be done without the payloadsSchema! Exiting.."); // The cause is already logged inside the above method.
|
||||
System.exit(88); // Exit the whole app, as it cannot add the results to the database!
|
||||
}
|
||||
callableTasks.add(() -> { // Handle inserts to the "payload" table. Around 20% of the total amount.
|
||||
return new ParquetReport(ParquetReport.ParquetType.payload, createAndLoadParquetDataIntoPayloadTable(urlReports, curReportAssignments, currentParquetPath, (parquetHDFSDirectoryPathPayloadsAggregated + curReportAssignments + "/")));
|
||||
});
|
||||
}
|
||||
|
||||
if ( (attemptsSchema == null) // Parse the schema if it's not already parsed.
|
||||
&& ((attemptsSchema = parseSchema(attemptSchemaFilePath)) == null ) ) {
|
||||
logger.error("Nothing can be done without the attemptsSchema! Exiting.."); // The cause is already logged inside the above method.
|
||||
System.exit(89); // Exit the whole app, as it cannot add the results to the database!
|
||||
}
|
||||
|
||||
return callableTasks;
|
||||
}
|
||||
|
||||
|
||||
public boolean createAttemptParquetDataAndUploadToHDFS(int attemptsIncNum, List<UrlReport> urlReports, long curReportAssignmentsCounter, String localParquetPath)
|
||||
public boolean createAndLoadParquetDataIntoAttemptTable(int attemptsIncNum, List<UrlReport> urlReports, long curReportAssignments, String localParquetPath)
|
||||
{
|
||||
List<GenericData.Record> recordList = new ArrayList<>(urlReports.size());
|
||||
GenericData.Record record;
|
||||
|
@ -247,7 +213,7 @@ public class ParquetFileUtils {
|
|||
for ( UrlReport urlReport : urlReports ) {
|
||||
Payload payload = urlReport.getPayload();
|
||||
if ( payload == null ) {
|
||||
logger.warn("Payload was \"null\" for a \"urlReport\", in assignments_" + curReportAssignmentsCounter + GenericUtils.endOfLine + urlReport);
|
||||
logger.warn("Payload was \"null\" for a \"urlReport\", in assignments_" + curReportAssignments + "\n" + urlReport);
|
||||
continue;
|
||||
}
|
||||
|
||||
|
@ -274,11 +240,11 @@ public class ParquetFileUtils {
|
|||
|
||||
int recordsSize = recordList.size();
|
||||
if ( recordsSize == 0 ) {
|
||||
logger.error("No attempt-parquet-records could be created in order to be inserted to the database!");
|
||||
logger.warn("No attempts are available to be inserted to the database!");
|
||||
return false;
|
||||
}
|
||||
|
||||
String fileName = curReportAssignmentsCounter + "_attempts_" + attemptsIncNum + ".parquet";
|
||||
String fileName = UrlsServiceImpl.assignmentsBatchCounter.get() + "_attempts_" + attemptsIncNum + ".parquet";
|
||||
//logger.trace("Going to write " + recordsSize + " attempt-records to the parquet file: " + fileName); // DEBUG!
|
||||
|
||||
String fullFilePath = localParquetPath + fileName;
|
||||
|
@ -286,7 +252,7 @@ public class ParquetFileUtils {
|
|||
//logger.trace("Parquet file \"" + fileName + "\" was created and filled."); // DEBUG!
|
||||
|
||||
// Upload and insert the data to the "attempt" Impala table.
|
||||
String errorMsg = uploadParquetFileToHDFS(fullFilePath, fileName, (parquetHDFSDirectoryPathAttempts + curReportAssignmentsCounter + "/"));
|
||||
String errorMsg = uploadParquetFileToHDFS(fullFilePath, fileName, (parquetHDFSDirectoryPathAttempts + curReportAssignments + "/"));
|
||||
return (errorMsg == null); // The possible error-message returned, is already logged by the Controller.
|
||||
} else
|
||||
return false;
|
||||
|
@ -296,17 +262,16 @@ public class ParquetFileUtils {
|
|||
}
|
||||
|
||||
|
||||
public boolean createPayloadParquetDataAndUploadToHDFS(int payloadsCounter, List<UrlReport> urlReports, long curReportAssignmentsCounter, String localParquetPath, String parquetHDFSDirectoryPathPayloads)
|
||||
public boolean createAndLoadParquetDataIntoPayloadTable(List<UrlReport> urlReports, long curReportAssignments, String localParquetPath, String parquetHDFSDirectoryPathPayloads)
|
||||
{
|
||||
List<GenericData.Record> recordList = new ArrayList<>((int) (urlReports.size() * 0.2));
|
||||
GenericData.Record record;
|
||||
int numPayloadsInsideUrlReports = 0;
|
||||
|
||||
for ( UrlReport urlReport : urlReports )
|
||||
{
|
||||
Payload payload = urlReport.getPayload();
|
||||
if ( payload == null ) {
|
||||
logger.warn("Payload was \"null\" for a \"urlReport\", in assignments_" + curReportAssignmentsCounter + GenericUtils.endOfLine + urlReport);
|
||||
logger.warn("Payload was \"null\" for a \"urlReport\", in assignments_" + curReportAssignments + "\n" + urlReport);
|
||||
continue;
|
||||
}
|
||||
|
||||
|
@ -314,27 +279,32 @@ public class ParquetFileUtils {
|
|||
if ( fileLocation == null ) // We want only the records with uploaded full-texts in the "payload" table.
|
||||
continue;
|
||||
|
||||
numPayloadsInsideUrlReports ++;
|
||||
|
||||
Timestamp timestamp = payload.getTimestamp_acquired();
|
||||
record = getPayloadParquetRecord(payload.getId(), payload.getOriginal_url(), payload.getActual_url(),
|
||||
(timestamp != null) ? timestamp.getTime() : System.currentTimeMillis(),
|
||||
payload.getMime_type(), payload.getSize(), payload.getHash(), fileLocation, payload.getProvenance(), false);
|
||||
|
||||
if ( record != null )
|
||||
try {
|
||||
record = new GenericData.Record(payloadsSchema);
|
||||
record.put("id", payload.getId());
|
||||
record.put("original_url", payload.getOriginal_url());
|
||||
record.put("actual_url", payload.getActual_url());
|
||||
Timestamp timestamp = payload.getTimestamp_acquired();
|
||||
record.put("date", (timestamp != null) ? timestamp.getTime() : System.currentTimeMillis());
|
||||
record.put("mimetype", payload.getMime_type());
|
||||
Long size = payload.getSize();
|
||||
record.put("size", ((size != null) ? String.valueOf(size) : null));
|
||||
record.put("hash", payload.getHash());
|
||||
record.put("location", fileLocation);
|
||||
record.put("provenance", payload.getProvenance());
|
||||
recordList.add(record);
|
||||
} catch (Exception e) {
|
||||
logger.error("Failed to create a payload record!", e);
|
||||
}
|
||||
}
|
||||
|
||||
if ( numPayloadsInsideUrlReports == 0 )
|
||||
return true; // This urlsReports-sublist does not have any payloads inside to use. That's fine.
|
||||
|
||||
int recordsSize = recordList.size();
|
||||
if ( recordsSize == 0 ) {
|
||||
logger.error("No payload-parquet-records could be created in order to be inserted to the database!");
|
||||
logger.warn("No payloads are available to be inserted to the database!");
|
||||
return false;
|
||||
}
|
||||
|
||||
String fileName = curReportAssignmentsCounter + "_payloads_" + payloadsCounter + ".parquet";
|
||||
String fileName = UrlsServiceImpl.assignmentsBatchCounter.get() + "_payloads.parquet";
|
||||
//logger.trace("Going to write " + recordsSize + " payload-records to the parquet file: " + fileName); // DEBUG!
|
||||
|
||||
String fullFilePath = localParquetPath + fileName;
|
||||
|
@ -352,30 +322,6 @@ public class ParquetFileUtils {
|
|||
}
|
||||
|
||||
|
||||
public GenericData.Record getPayloadParquetRecord(String id, String original_url, String actual_url, long timeMillis, String mimetype, Long size,
|
||||
String hash, String fileLocation, String provenance, boolean isForBulkImport)
|
||||
{
|
||||
GenericData.Record record;
|
||||
try {
|
||||
record = new GenericData.Record(payloadsSchema);
|
||||
record.put("id", id);
|
||||
record.put("original_url", original_url);
|
||||
record.put("actual_url", actual_url);
|
||||
record.put("date", timeMillis);
|
||||
record.put("mimetype", mimetype);
|
||||
record.put("size", ((size != null) ? String.valueOf(size) : null));
|
||||
record.put("hash", hash);
|
||||
record.put("location", fileLocation);
|
||||
record.put("provenance", (isForBulkImport ? "bulk:" : "") + provenance);
|
||||
// Add the "bulk:" prefix in order to be more clear that this record comes from bulkImport, when looking all records in the "payload" VIEW.
|
||||
return record;
|
||||
} catch (Exception e) {
|
||||
logger.error("Failed to create a payload record!", e);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
public boolean writeToParquet(List<GenericData.Record> recordList, Schema schema, String fullFilePath)
|
||||
{
|
||||
OutputFile outputFile;
|
||||
|
@ -401,7 +347,7 @@ public class ParquetFileUtils {
|
|||
} catch (Throwable e) { // The simple "Exception" may not be thrown here, but an "Error" may be thrown. "Throwable" catches EVERYTHING!
|
||||
String errorMsg = "Problem when creating the \"ParquetWriter\" object or when writing the records with it!";
|
||||
if ( e instanceof org.apache.hadoop.fs.FileAlreadyExistsException )
|
||||
logger.error(errorMsg + GenericUtils.endOfLine + e.getMessage());
|
||||
logger.error(errorMsg + "\n" + e.getMessage());
|
||||
else
|
||||
logger.error(errorMsg, e);
|
||||
|
||||
|
@ -461,9 +407,7 @@ public class ParquetFileUtils {
|
|||
}
|
||||
//logger.trace("The target location is: " + location + "\nWill do a silent redirect to HTTPS."); // DEBUG!
|
||||
|
||||
// In case the WebHDFS uses https, then perform the offline redirect to https here (to avoid the live redirection.)
|
||||
if ( webHDFSBaseUrl.startsWith("https:") )
|
||||
location = StringUtils.replace(location, "http:", "https:", 1);
|
||||
location = StringUtils.replace(location, "http:", "https:", 1);
|
||||
// Unless we handle this here, we have to either complicate the process by handling the https-redirect or in any-way getting a hit in performance by having one more step each time ww want to upload a file.
|
||||
|
||||
conn = (HttpURLConnection) (new URL(location)).openConnection(); // This already contains the "user.name" parameter.
|
||||
|
@ -502,9 +446,9 @@ public class ParquetFileUtils {
|
|||
// Using the "load data inpath" command, he files are MOVED, not copied! So we don't have to delete them afterwards.
|
||||
// See: https://docs.cloudera.com/documentation/enterprise/latest/topics/impala_load_data.html
|
||||
} catch (Throwable e) {
|
||||
String errorMsg = "Error while uploading parquet file \"" + parquetFileFullLocalPath + "\" to HDFS!\n";
|
||||
logger.error(errorMsg, e);
|
||||
return errorMsg + e.getMessage();
|
||||
String errorMsg = "Error while uploading parquet file \"" + parquetFileFullLocalPath + "\" to HDFS!\n" + e;
|
||||
logger.error(errorMsg);
|
||||
return errorMsg;
|
||||
}
|
||||
|
||||
// The local parquet file will be deleted later.
|
||||
|
@ -515,7 +459,7 @@ public class ParquetFileUtils {
|
|||
public boolean loadParquetDataIntoTable(String remoteParquetDataLocation, String tableName)
|
||||
{
|
||||
// Import the data from the parquet file into the database's table.
|
||||
String loadParquetInTableQuery = "load data inpath '" + remoteParquetDataLocation + "' into table " + DatabaseConnector.databaseName + "." + tableName;
|
||||
String loadParquetInTableQuery = "load data inpath '" + remoteParquetDataLocation + "' into table " + ImpalaConnector.databaseName + "." + tableName;
|
||||
try {
|
||||
jdbcTemplate.execute(loadParquetInTableQuery);
|
||||
} catch (Exception e) {
|
||||
|
@ -528,7 +472,7 @@ public class ParquetFileUtils {
|
|||
return false; // Since each thread is using a different subDir, by design, This error is unacceptable.
|
||||
}
|
||||
}
|
||||
DatabaseConnector.handleQueryException("loadParquetInTableQuery", loadParquetInTableQuery, e); // It's already logged.
|
||||
ImpalaConnector.handleQueryException("loadParquetInTableQuery", loadParquetInTableQuery, e); // It's already logged.
|
||||
return false;
|
||||
}
|
||||
//logger.trace("The data from \"" + remoteParquetDataLocation + "\" was loaded into the " + tableName + " table."); // DEBUG!
|
||||
|
@ -544,7 +488,6 @@ public class ParquetFileUtils {
|
|||
// So with one request we will create the "parquet_uploads/" and the "parquet_uploads/attempts/" and with the seconds request, the "parquet_uploads/payloads/" directory.
|
||||
|
||||
logger.info("Going to check if the remote parquet directories exist.");
|
||||
String initErrMsg = "The remote parquet HDFS-directory ";
|
||||
|
||||
String listMainDirectoryUrl = webHDFSBaseUrl + parquetBaseRemoteDirectory + "?op=LISTSTATUS&user.name=" + hdfsUserName;
|
||||
|
||||
|
@ -569,10 +512,10 @@ public class ParquetFileUtils {
|
|||
}
|
||||
|
||||
if ( statusCode == 404 ) {
|
||||
logger.info(initErrMsg + "\"" + parquetBaseRemoteDirectory + "\" does not exist. We will create it, along with its sub-directories.");
|
||||
attemptCreationSuccessful = applyHDFSOperation(webHDFSBaseUrl + parquetHDFSDirectoryPathAttempts + mkDirsAndParams);
|
||||
payloadAggregatedCreationSuccessful = applyHDFSOperation(webHDFSBaseUrl + parquetHDFSDirectoryPathPayloadsAggregated + mkDirsAndParams);
|
||||
payloadBulkImportCreationSuccessful = applyHDFSOperation(webHDFSBaseUrl + parquetHDFSDirectoryPathPayloadsBulkImport + mkDirsAndParams);
|
||||
logger.info("The directory \"" + parquetBaseRemoteDirectory + "\" does not exist. We will create it, along with its sub-directories.");
|
||||
attemptCreationSuccessful = applyHDFOperation(webHDFSBaseUrl + parquetHDFSDirectoryPathAttempts + mkDirsAndParams);
|
||||
payloadAggregatedCreationSuccessful = applyHDFOperation(webHDFSBaseUrl + parquetHDFSDirectoryPathPayloadsAggregated + mkDirsAndParams);
|
||||
payloadBulkImportCreationSuccessful = applyHDFOperation(webHDFSBaseUrl + parquetHDFSDirectoryPathPayloadsBulkImport + mkDirsAndParams);
|
||||
return (attemptCreationSuccessful && payloadAggregatedCreationSuccessful && payloadBulkImportCreationSuccessful);
|
||||
// We need all directories to be created in order for the app to function properly!
|
||||
}
|
||||
|
@ -615,10 +558,10 @@ public class ParquetFileUtils {
|
|||
foundPayloadsAggregatedDir = true;
|
||||
else if ( dirPath.equals("payloads_bulk_import") )
|
||||
foundPayloadsBulkImportDir = true;
|
||||
else if ( ! dirPath.equals("test") ) // The "test" directory helps with testing the service, without interfering with the production directories.
|
||||
logger.warn("Unknown remote parquet HDFS-directory found: " + dirPath);
|
||||
else
|
||||
logger.warn("Unknown remote parquet directory found: " + dirPath);
|
||||
}
|
||||
} catch (JSONException je) { // In case any of the above "json-keys" was not found.
|
||||
} catch ( JSONException je ) { // In case any of the above "json-keys" was not found.
|
||||
logger.warn("JSON Exception was thrown while trying to retrieve the subdirectories \"attempts\" and \"payloads\": " + je.getMessage() + "\n\nJsonResponse: " + jsonResponse);
|
||||
return false;
|
||||
}
|
||||
|
@ -628,22 +571,22 @@ public class ParquetFileUtils {
|
|||
|
||||
// For each missing subdirectories, run the mkDirs-request.
|
||||
if ( !foundAttemptsDir ) {
|
||||
logger.debug(initErrMsg + "\"" + parquetHDFSDirectoryPathAttempts + "\" does not exist! Going to create it.");
|
||||
attemptCreationSuccessful = applyHDFSOperation(webHDFSBaseUrl + parquetHDFSDirectoryPathAttempts + mkDirsAndParams);
|
||||
logger.debug("The remote parquet directory \"" + parquetHDFSDirectoryPathAttempts + "\" does not exist! Going to create it.");
|
||||
attemptCreationSuccessful = applyHDFOperation(webHDFSBaseUrl + parquetHDFSDirectoryPathAttempts + mkDirsAndParams);
|
||||
} else
|
||||
logger.info(initErrMsg + "\"" + parquetHDFSDirectoryPathAttempts + "\" exists.");
|
||||
logger.info("The remote parquet directory \"" + parquetHDFSDirectoryPathAttempts + "\" exists.");
|
||||
|
||||
if ( !foundPayloadsAggregatedDir ) {
|
||||
logger.debug(initErrMsg + "\"" + parquetHDFSDirectoryPathPayloadsAggregated + "\" does not exist! Going to create it.");
|
||||
payloadAggregatedCreationSuccessful = applyHDFSOperation(webHDFSBaseUrl + parquetHDFSDirectoryPathPayloadsAggregated + mkDirsAndParams);
|
||||
logger.debug("The remote parquet directory \"" + parquetHDFSDirectoryPathPayloadsAggregated + "\" does not exist! Going to create it.");
|
||||
payloadAggregatedCreationSuccessful = applyHDFOperation(webHDFSBaseUrl + parquetHDFSDirectoryPathPayloadsAggregated + mkDirsAndParams);
|
||||
} else
|
||||
logger.info(initErrMsg + "\"" + parquetHDFSDirectoryPathPayloadsAggregated + "\" exists.");
|
||||
logger.info("The remote parquet directory \"" + parquetHDFSDirectoryPathPayloadsAggregated + "\" exists.");
|
||||
|
||||
if ( !foundPayloadsBulkImportDir ) {
|
||||
logger.debug(initErrMsg + "\"" + parquetHDFSDirectoryPathPayloadsBulkImport + "\" does not exist! Going to create it.");
|
||||
payloadBulkImportCreationSuccessful = applyHDFSOperation(webHDFSBaseUrl + parquetHDFSDirectoryPathPayloadsBulkImport + mkDirsAndParams);
|
||||
logger.debug("The remote parquet directory \"" + parquetHDFSDirectoryPathPayloadsBulkImport + "\" does not exist! Going to create it.");
|
||||
payloadBulkImportCreationSuccessful = applyHDFOperation(webHDFSBaseUrl + parquetHDFSDirectoryPathPayloadsBulkImport + mkDirsAndParams);
|
||||
} else
|
||||
logger.info(initErrMsg + "\"" + parquetHDFSDirectoryPathPayloadsBulkImport + "\" exists.");
|
||||
logger.info("The remote parquet directory \"" + parquetHDFSDirectoryPathPayloadsBulkImport + "\" exists.");
|
||||
|
||||
return (attemptCreationSuccessful && payloadAggregatedCreationSuccessful && payloadBulkImportCreationSuccessful);
|
||||
// We need all directories to be created in order for the app to function properly!
|
||||
|
@ -654,7 +597,7 @@ public class ParquetFileUtils {
|
|||
}
|
||||
|
||||
|
||||
public boolean applyHDFSOperation(String hdfsOperationUrl)
|
||||
public boolean applyHDFOperation(String hdfsOperationUrl)
|
||||
{
|
||||
try {
|
||||
URL url = new URL(hdfsOperationUrl);
|
||||
|
@ -674,7 +617,7 @@ public class ParquetFileUtils {
|
|||
return false;
|
||||
}
|
||||
if ( logger.isTraceEnabled() )
|
||||
logger.trace("The Operation was successful for hdfs-op-url: " + hdfsOperationUrl + GenericUtils.endOfLine + fileUtils.getMessageFromResponseBody(conn, false));
|
||||
logger.trace("The Operation was successful for hdfs-op-url: " + hdfsOperationUrl + "\n" + fileUtils.getMessageFromResponseBody(conn, false));
|
||||
} catch (Exception e) {
|
||||
logger.error("", e);
|
||||
return false;
|
||||
|
@ -691,11 +634,11 @@ public class ParquetFileUtils {
|
|||
int numOfAllPayloadParquetFileCreations = 0;
|
||||
int numOfFailedPayloadParquetFileCreations = 0;
|
||||
|
||||
int sizeOfFutures = futures.size();
|
||||
for ( int i = 0; i < sizeOfFutures; ++i )
|
||||
for ( Future<ParquetReport> future : futures )
|
||||
{
|
||||
ParquetReport parquetReport = null;
|
||||
try {
|
||||
ParquetReport parquetReport = futures.get(i).get();
|
||||
parquetReport = future.get();
|
||||
boolean hasProblems = (! parquetReport.isSuccessful());
|
||||
ParquetReport.ParquetType parquetType = parquetReport.getParquetType();
|
||||
if ( parquetType.equals(ParquetReport.ParquetType.attempt) ) {
|
||||
|
@ -711,17 +654,11 @@ public class ParquetFileUtils {
|
|||
logger.error(errMsg);
|
||||
return new SumParquetSuccess(false, false, ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR).body(errMsg));
|
||||
}
|
||||
} catch (ExecutionException ee) { // These can be serious errors like an "out of memory exception" (Java HEAP).
|
||||
logger.error(GenericUtils.getSelectedStackTraceForCausedException(ee, "Parquet_task_" + i + " failed with: ", null, 15));
|
||||
} catch (CancellationException ce) {
|
||||
logger.error("Parquet_task_" + i + " was cancelled: " + ce.getMessage());
|
||||
} catch (InterruptedException ie) {
|
||||
logger.error("Parquet_task_" + i + " was interrupted: " + ie.getMessage());
|
||||
} catch (IndexOutOfBoundsException ioobe) {
|
||||
logger.error("IOOBE for parquet_task_" + i + " in the futures-list! " + ioobe.getMessage());
|
||||
} catch (Exception e) {
|
||||
logger.error("", e);
|
||||
// We do not know if the failed "future" refers to a "payload" or to a "attempt".
|
||||
// So we cannot increase a specific counter. That's ok, the only drawback if that we may try to "load" the non-existent data and get an exception.
|
||||
}
|
||||
// We do not know if the failed "future" refers to a "payload" or to a "attempt".
|
||||
// So we cannot increase a specific counter. That's ok, the only drawback if that we may try to "load" the non-existent data and get an exception.
|
||||
} // End-for
|
||||
|
||||
boolean hasAttemptParquetFileProblem = (numOfFailedAttemptParquetFileCreations == numOfAllAttemptParquetFileCreations);
|
||||
|
@ -730,110 +667,4 @@ public class ParquetFileUtils {
|
|||
return new SumParquetSuccess(hasAttemptParquetFileProblem, hasPayloadParquetFileProblem, null);
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* In each insertion, a new parquet-file is created, so we end up with millions of files. Parquet is great for fast-select, so have to stick with it and merge those files..
|
||||
* This method, creates a clone of the original table in order to have only one parquet file in the end. Drops the original table.
|
||||
* Renames the clone to the original's name.
|
||||
* Returns the errorMsg, if an error appears, otherwise is returns "null".
|
||||
*/
|
||||
public String mergeParquetFilesOfTable(String tableName, String whereClause, Object parameter) {
|
||||
String errorMsg;
|
||||
if ( (tableName == null) || tableName.isEmpty() ) {
|
||||
errorMsg = "No tableName was given. Do not know the tableName for which we should merger the underlying files for!";
|
||||
logger.error(errorMsg);
|
||||
return errorMsg; // Return the error-msg to indicate that something went wrong and pass it down to the Worker.
|
||||
}
|
||||
|
||||
// Make sure the following are empty strings.
|
||||
whereClause = (whereClause != null) ? (whereClause + " ") : "";
|
||||
|
||||
if ( parameter == null )
|
||||
parameter = "";
|
||||
else if ( parameter instanceof String )
|
||||
parameter = "'" + parameter + "'"; // This will be a "string-check", thus the single-quotes.
|
||||
// Else it is a "long", it will be used as is.
|
||||
|
||||
// Create a temp-table as a copy of the initial table.
|
||||
try {
|
||||
jdbcTemplate.execute("CREATE TABLE " + DatabaseConnector.databaseName + "." + tableName + "_tmp stored as parquet AS SELECT * FROM " + DatabaseConnector.databaseName + "." + tableName + " " + whereClause + parameter);
|
||||
} catch (Exception e) {
|
||||
errorMsg = "Problem when copying the contents of \"" + tableName + "\" table to a newly created \"" + tableName + "_tmp\" table, when merging the parquet-files!\n";
|
||||
logger.error(errorMsg, e);
|
||||
try { // Make sure we delete the possibly half-created temp-table.
|
||||
jdbcTemplate.execute("DROP TABLE IF EXISTS " + DatabaseConnector.databaseName + "." + tableName + "_tmp PURGE");
|
||||
// We cannot move on with merging, but no harm happened, since the "table_tmp" name is still reserved for future use (after it was dropped immediately)..
|
||||
} catch (Exception e1) {
|
||||
logger.error("Failed to drop the \"" + tableName + "_tmp\" table!", e1);
|
||||
// TODO - Should we shutdown the service? It is highly unlikely that anyone will observe this error live (and act immediately)..!
|
||||
}
|
||||
return errorMsg; // We return only the initial error to the Worker, which is easily distinguished indie the "merge-queries".
|
||||
}
|
||||
|
||||
// Drop the initial table.
|
||||
try {
|
||||
jdbcTemplate.execute("DROP TABLE " + DatabaseConnector.databaseName + "." + tableName + " PURGE");
|
||||
} catch (Exception e) {
|
||||
errorMsg = "Problem when dropping the initial \"" + tableName + "\" table, when merging the parquet-files!\n";
|
||||
logger.error(errorMsg, e);
|
||||
// The original table could not be dropped, so the temp-table cannot be renamed to the original..!
|
||||
try { // Make sure we delete the already created temp-table, in order to be able to use it in the future. The merging has failed nevertheless.
|
||||
jdbcTemplate.execute("DROP TABLE " + DatabaseConnector.databaseName + "." + tableName + "_tmp PURGE");
|
||||
} catch (Exception e1) {
|
||||
logger.error((errorMsg += "Failed to drop the \"" + tableName + "_tmp\" table!"), e1); // Add this error to the original, both are very important.
|
||||
}
|
||||
// Here, the original table is created.
|
||||
return errorMsg;
|
||||
// TODO - Should we shutdown the service? It is highly unlikely that anyone will observe this error live (and act immediately)..!
|
||||
}
|
||||
|
||||
// Rename the temp-table to have the initial-table's name.
|
||||
try {
|
||||
jdbcTemplate.execute("ALTER TABLE " + DatabaseConnector.databaseName + "." + tableName + "_tmp RENAME TO " + DatabaseConnector.databaseName + "." + tableName);
|
||||
} catch (Exception e) {
|
||||
errorMsg = "Problem in renaming the \"" + tableName + "_tmp\" table to \"" + tableName + "\", when merging the parquet-files!\n";
|
||||
logger.error(errorMsg, e);
|
||||
// At this point we only have a "temp-table", the original is already deleted..
|
||||
// Try to create the original, as a copy of the temp-table. If that succeeds, then try to delete the temp-table.
|
||||
try {
|
||||
jdbcTemplate.execute("CREATE TABLE " + DatabaseConnector.databaseName + "." + tableName + " stored as parquet AS SELECT * FROM " + DatabaseConnector.databaseName + "." + tableName + "_tmp");
|
||||
} catch (Exception e1) {
|
||||
errorMsg = "Problem when copying the contents of \"" + tableName + "_tmp\" table to a newly created \"" + tableName + "\" table, when merging the parquet-files!\n";
|
||||
logger.error(errorMsg, e1);
|
||||
// If the original table was not created, then we have to intervene manually, if it was created but without any data, then we can safely move on handling other assignments and workerReports, but the data will be lost! So this workerReport failed to be handled.
|
||||
try { // The below query normally returns a list, as it takes a "regex-pattern" as an input. BUT, we give just the table name, without wildcards. So the result is either the tableName itself or none (not any other table).
|
||||
jdbcTemplate.queryForObject("SHOW TABLES IN " + DatabaseConnector.databaseName + " LIKE '" + tableName + "'", List.class);
|
||||
} catch (EmptyResultDataAccessException erdae) {
|
||||
// The table does not exist, so it was not even half-created by the previous query.
|
||||
// Not having the original table anymore is a serious error. A manual action is needed!
|
||||
logger.error((errorMsg += "The original table \"" + tableName + "\" must be created manually! Serious problems may appear otherwise!"));
|
||||
// TODO - Should we shutdown the service? It is highly unlikely that anyone will observe this error live (and fix it immediately to avoid other errors in the Service)..!
|
||||
}
|
||||
// Here, the original-table exists in the DB, BUT without any data inside! This workerReport failed to be handled! (some of its data could not be loaded to the database, and all previous data was lost).
|
||||
return errorMsg;
|
||||
}
|
||||
|
||||
// The creation of the original table was successful. Try to delete the temp-table.
|
||||
try {
|
||||
jdbcTemplate.execute("DROP TABLE " + DatabaseConnector.databaseName + "." + tableName + "_tmp PURGE");
|
||||
} catch (Exception e2) {
|
||||
logger.error((errorMsg += "Problem when dropping the \"" + tableName + "_tmp\" table, when merging the parquet-files!\n"), e2);
|
||||
// Manual deletion should be performed!
|
||||
return errorMsg; // Return both errors here, as the second is so important that if it did not happen then we could move on with this workerReport.
|
||||
// TODO - Should we shutdown the service? It is highly unlikely that anyone will observe this error live (and act immediately)..!
|
||||
}
|
||||
// Here the original table exists and the temp-table is deleted. We eventually have the same state as if the "ALTER TABLE" succeeded.
|
||||
}
|
||||
|
||||
// Gather information to be used for queries-optimization.
|
||||
try {
|
||||
jdbcTemplate.execute("COMPUTE STATS " + DatabaseConnector.databaseName + "." + tableName);
|
||||
} catch (Exception e) {
|
||||
logger.error("Problem when gathering information from table \"" + tableName + "\" to be used for queries-optimization.", e);
|
||||
// In this case the error is not so important to the whole operation.. It's only that the performance of this specific table will be less optimal, only temporarily, unless every "COMPUTE STATS" query fails for future workerReports too.
|
||||
}
|
||||
|
||||
return null; // No errorMsg, everything is fine.
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
@ -48,8 +48,8 @@ public class S3ObjectStore {
|
|||
boolean bucketExists = minioClient.bucketExists(BucketExistsArgs.builder().bucket(bucketName).build());
|
||||
|
||||
// Keep this commented-out to avoid objects-deletion by accident. The code is open-sourced, so it's easy to enable this ability if we really want it (e.g. for testing).
|
||||
/*if ( shouldEmptyBucket && isTestEnvironment && bucketExists )
|
||||
emptyBucket(bucketName, false);*/
|
||||
if ( shouldEmptyBucket && isTestEnvironment && bucketExists )
|
||||
emptyBucket(bucketName, false);
|
||||
|
||||
// Make the bucket, if not exist.
|
||||
if ( !bucketExists ) {
|
||||
|
@ -117,57 +117,30 @@ public class S3ObjectStore {
|
|||
}
|
||||
|
||||
|
||||
public void emptyBucket(String bucketName, boolean shouldDeleteBucket)
|
||||
{
|
||||
public void emptyBucket(String bucketName, boolean shouldDeleteBucket) throws Exception {
|
||||
logger.warn("Going to " + (shouldDeleteBucket ? "delete" : "empty") + " bucket \"" + bucketName + "\"!");
|
||||
|
||||
// First list the objects of the bucket.
|
||||
Iterable<Result<Item>> results;
|
||||
try {
|
||||
results = minioClient.listObjects(ListObjectsArgs.builder().bucket(bucketName).build());
|
||||
} catch (Exception e) {
|
||||
logger.error("Could not retrieve the list of objects of bucket \"" + bucketName + "\"!");
|
||||
return;
|
||||
}
|
||||
|
||||
int countDeletedFiles = 0;
|
||||
int countFilesNotDeleted = 0;
|
||||
long totalSize = 0;
|
||||
Item item;
|
||||
Iterable<Result<Item>> results = minioClient.listObjects(ListObjectsArgs.builder().bucket(bucketName).build());
|
||||
|
||||
// Then, delete the objects.
|
||||
for ( Result<Item> resultItem : results ) {
|
||||
try {
|
||||
item = resultItem.get();
|
||||
if ( !deleteFile(resultItem.get().objectName(), bucketName) ) {
|
||||
logger.error("Cannot proceed with bucket deletion, since only an empty bucket can be removed!");
|
||||
return;
|
||||
}
|
||||
} catch (Exception e) {
|
||||
logger.error("Could not get the item-object of one of the S3-Objects returned from the bucket!", e);
|
||||
countFilesNotDeleted ++;
|
||||
continue;
|
||||
logger.warn("Could not remove " + resultItem.get().objectName());
|
||||
}
|
||||
totalSize += item.size();
|
||||
|
||||
if ( !deleteFile(item.objectName(), bucketName) ) { // The reason and for what object, is already logged.
|
||||
logger.error("Cannot proceed with bucket deletion, since only an empty bucket can be removed!");
|
||||
countFilesNotDeleted ++;
|
||||
} else
|
||||
countDeletedFiles ++;
|
||||
}
|
||||
|
||||
if ( shouldDeleteBucket ) {
|
||||
if ( countFilesNotDeleted == 0 ) {
|
||||
// Lastly, delete the empty bucket. We need to do this last, as in case it's not empty, we get an error!
|
||||
try {
|
||||
minioClient.removeBucket(RemoveBucketArgs.builder().bucket(bucketName).build());
|
||||
logger.info("Bucket \"" + bucketName + "\" was deleted!");
|
||||
} catch (Exception e) {
|
||||
logger.error("Bucket \"" + bucketName + "\" could not be deleted!", e);
|
||||
}
|
||||
} else
|
||||
logger.error("Cannot execute the \"removeBucket\" command for bucket \"" + bucketName + "\", as " + countFilesNotDeleted + " files failed to be deleted!");
|
||||
} else
|
||||
logger.info("Bucket \"" + bucketName + "\" was emptied!");
|
||||
// Lastly, delete the empty bucket.
|
||||
minioClient.removeBucket(RemoveBucketArgs.builder().bucket(bucketName).build());
|
||||
}
|
||||
|
||||
logger.info(countDeletedFiles + " files were deleted, amounting to " + ((totalSize/1024)/1024) + " MB.");
|
||||
logger.info("Bucket " + bucketName + " was " + (shouldDeleteBucket ? "deleted!" : "emptied!"));
|
||||
}
|
||||
|
||||
|
||||
|
|
|
@ -15,14 +15,10 @@ services:
|
|||
|
||||
db:
|
||||
initialDatabaseName: pdfaggregation_i
|
||||
testDatabaseName: pdfaggregationdatabase_test_new
|
||||
testDatabaseName: pdfaggregationdatabase_payloads_view_test
|
||||
|
||||
assignmentLimit: 10000
|
||||
maxAttemptsPerRecord: 3
|
||||
|
||||
numOfBackgroundThreads: 8
|
||||
# This refers to the number of threads running in the background and processing workerReports and bulkImport procedures.
|
||||
|
||||
baseFilesLocation: /tmp/
|
||||
workerReportsDirPath: /reports/workerReports/
|
||||
parquetLocalDirectoryPath: ${services.pdfaggregation.controller.baseFilesLocation}parquetFiles/
|
||||
|
@ -32,34 +28,24 @@ services:
|
|||
secretKey: XA
|
||||
region: XA
|
||||
bucketName: XA
|
||||
shouldEmptyBucket: false # For data-security, in order to use this ability, the relevant code has to be uncommented in "util.S3ObjectStore.java"
|
||||
shouldEmptyBucket: false
|
||||
shouldShowAllS3Buckets: true
|
||||
worker:
|
||||
port: 1881
|
||||
|
||||
|
||||
bulk-import:
|
||||
baseBulkImportLocation: /mnt/bulk_import/
|
||||
bulkImportReportLocation: /reports/bulkImportReports/
|
||||
numOfThreadsForBulkImportProcedures: 6
|
||||
bulkImportSources: # These sources are accepted for bulk-import requests and are excluded from crawling.
|
||||
arxivImport:
|
||||
datasourceID: opendoar____::6f4922f45568161a8cdf4ad2299f6d23
|
||||
datasourcePrefix: arXiv_______ # For PID-providing datasource, we use the PID-prefix here. (so not the datasource-prefix: "od________18")
|
||||
pdfUrlPrefix: https://arxiv.org/pdf/
|
||||
mimeType: application/pdf
|
||||
isAuthoritative: true
|
||||
# otherImport:
|
||||
# datasourceID: othersource__::0123
|
||||
# datasourcePrefix: other_______
|
||||
# pdfUrlPrefix: https://example.org/pdf/
|
||||
# mimeType: application/pdf
|
||||
# isAuthoritative: false
|
||||
|
||||
# For "authoritative" sources, a special prefix is selected, from: https://graph.openaire.eu/docs/data-model/pids-and-identifiers/#identifiers-in-the-graph
|
||||
# For the rest, the "datasource_prefix" is selected, using this query:
|
||||
# select datasource.namespaceprefix.value
|
||||
# from openaire_prod_<PROD_DATE>.datasource -- Here use the production-table with the latest date.
|
||||
# where officialname.value = 'datasourceOfficialName';
|
||||
|
||||
|
||||
spring:
|
||||
|
@ -81,7 +67,7 @@ spring:
|
|||
ansi:
|
||||
enabled: always
|
||||
lifecycle:
|
||||
timeout-per-shutdown-phase: 5m
|
||||
timeout-per-shutdown-phase: 2m
|
||||
|
||||
|
||||
hdfs:
|
||||
|
@ -116,8 +102,6 @@ management:
|
|||
application: ${spring.application.name}
|
||||
|
||||
logging:
|
||||
file:
|
||||
path: logs
|
||||
level:
|
||||
root: INFO
|
||||
eu:
|
||||
|
|
|
@ -1,16 +1,16 @@
|
|||
<configuration debug="false">
|
||||
|
||||
<appender name="RollingFile" class="ch.qos.logback.core.rolling.RollingFileAppender">
|
||||
<file>${LOG_PATH}/UrlsController.log</file>
|
||||
<file>logs/UrlsController.log</file>
|
||||
|
||||
<rollingPolicy class="ch.qos.logback.core.rolling.FixedWindowRollingPolicy">
|
||||
<fileNamePattern>${LOG_PATH}/UrlsController.%i.log.zip</fileNamePattern>
|
||||
<fileNamePattern>logs/UrlsController.%i.log.zip</fileNamePattern>
|
||||
<minIndex>1</minIndex>
|
||||
<maxIndex>10</maxIndex>
|
||||
<maxIndex>20</maxIndex>
|
||||
</rollingPolicy>
|
||||
|
||||
<triggeringPolicy class="ch.qos.logback.core.rolling.SizeBasedTriggeringPolicy">
|
||||
<maxFileSize>100MB</maxFileSize>
|
||||
<maxFileSize>50MB</maxFileSize>
|
||||
</triggeringPolicy>
|
||||
|
||||
<encoder>
|
||||
|
|
Loading…
Reference in New Issue