master #59

Closed
claudio.atzori wants to merge 3221 commits from master into stable_ids
9 changed files with 38 additions and 8 deletions
Showing only changes of commit e9f24df21c - Show all commits

View File

@ -63,9 +63,12 @@ public class ArchiveRepositoryURLs {
final Integer archiveThresholdInDays = Integer.parseInt(argumentParser.get("archiveThresholdInDays")); final Integer archiveThresholdInDays = Integer.parseInt(argumentParser.get("archiveThresholdInDays"));
log.info("archiveThresholdInDays: {}", archiveThresholdInDays); log.info("archiveThresholdInDays: {}", archiveThresholdInDays);
final String apiAccessToken = argumentParser.get("apiAccessToken");
log.info("apiAccessToken: {}", apiAccessToken);
final HttpClientParams clientParams = SWHUtils.getClientParams(argumentParser); final HttpClientParams clientParams = SWHUtils.getClientParams(argumentParser);
swhConnection = new SWHConnection(clientParams); swhConnection = new SWHConnection(clientParams, apiAccessToken);
final FileSystem fs = FileSystem.get(getHadoopConfiguration(hdfsuri)); final FileSystem fs = FileSystem.get(getHadoopConfiguration(hdfsuri));

View File

@ -55,9 +55,12 @@ public class CollectLastVisitRepositoryData {
final String outputPath = argumentParser.get("lastVisitsPath"); final String outputPath = argumentParser.get("lastVisitsPath");
log.info("outputPath: {}", outputPath); log.info("outputPath: {}", outputPath);
final String apiAccessToken = argumentParser.get("apiAccessToken");
log.info("apiAccessToken: {}", apiAccessToken);
final HttpClientParams clientParams = SWHUtils.getClientParams(argumentParser); final HttpClientParams clientParams = SWHUtils.getClientParams(argumentParser);
swhConnection = new SWHConnection(clientParams); swhConnection = new SWHConnection(clientParams, apiAccessToken);
final FileSystem fs = FileSystem.get(getHadoopConfiguration(hdfsuri)); final FileSystem fs = FileSystem.get(getHadoopConfiguration(hdfsuri));

View File

@ -14,13 +14,15 @@ public class SWHConnection {
HttpConnector2 conn; HttpConnector2 conn;
public SWHConnection(HttpClientParams clientParams) { public SWHConnection(HttpClientParams clientParams, String accessToken) {
// set custom headers // set custom headers
Map<String, String> headers = new HashMap<String, String>() { Map<String, String> headers = new HashMap<String, String>() {
{ {
put(HttpHeaders.ACCEPT, "application/json"); put(HttpHeaders.ACCEPT, "application/json");
put(HttpHeaders.AUTHORIZATION, String.format("Bearer %s", SWHConstants.ACCESS_TOKEN)); if (accessToken != null) {
put(HttpHeaders.AUTHORIZATION, String.format("Bearer %s", accessToken));
}
} }
}; };

View File

@ -6,8 +6,6 @@ public class SWHConstants {
public static final String SWH_ARCHIVE_URL = "https://archive.softwareheritage.org/api/1/origin/save/%s/url/%s/"; public static final String SWH_ARCHIVE_URL = "https://archive.softwareheritage.org/api/1/origin/save/%s/url/%s/";
public static final String ACCESS_TOKEN = "eyJhbGciOiJIUzI1NiIsInR5cCIgOiAiSldUIiwia2lkIiA6ICJhMTMxYTQ1My1hM2IyLTQwMTUtODQ2Ny05MzAyZjk3MTFkOGEifQ.eyJpYXQiOjE2OTQ2MzYwMjAsImp0aSI6IjkwZjdkNTNjLTQ5YTktNGFiMy1hY2E0LTcwMTViMjEyZTNjNiIsImlzcyI6Imh0dHBzOi8vYXV0aC5zb2Z0d2FyZWhlcml0YWdlLm9yZy9hdXRoL3JlYWxtcy9Tb2Z0d2FyZUhlcml0YWdlIiwiYXVkIjoiaHR0cHM6Ly9hdXRoLnNvZnR3YXJlaGVyaXRhZ2Uub3JnL2F1dGgvcmVhbG1zL1NvZnR3YXJlSGVyaXRhZ2UiLCJzdWIiOiIzMTY5OWZkNC0xNmE0LTQxOWItYTdhMi00NjI5MDY4ZjI3OWEiLCJ0eXAiOiJPZmZsaW5lIiwiYXpwIjoic3doLXdlYiIsInNlc3Npb25fc3RhdGUiOiIzMjYzMzEwMS00ZDRkLTQwMjItODU2NC1iMzNlMTJiNTE3ZDkiLCJzY29wZSI6Im9wZW5pZCBvZmZsaW5lX2FjY2VzcyBwcm9maWxlIGVtYWlsIn0.XHj1VIZu1dZ4Ej32-oU84mFmaox9cLNjXosNxwZM0Xs";
public static final String DEFAULT_VISIT_TYPE = "git"; public static final String DEFAULT_VISIT_TYPE = "git";
public static final String VISIT_STATUS_NOT_FOUND = "not_found"; public static final String VISIT_STATUS_NOT_FOUND = "not_found";

View File

@ -46,5 +46,11 @@
"paramLongName": "archiveThresholdInDays", "paramLongName": "archiveThresholdInDays",
"paramDescription": "the thershold (in days) required to issue an archive request", "paramDescription": "the thershold (in days) required to issue an archive request",
"paramRequired": false "paramRequired": false
},
{
"paramName": "aat",
"paramLongName": "apiAccessToken",
"paramDescription": "the API access token of the SWH API",
"paramRequired": false
} }
] ]

View File

@ -40,5 +40,11 @@
"paramLongName": "requestMethod", "paramLongName": "requestMethod",
"paramDescription": "the method of the requests to perform", "paramDescription": "the method of the requests to perform",
"paramRequired": false "paramRequired": false
},
{
"paramName": "aat",
"paramLongName": "apiAccessToken",
"paramDescription": "the API access token of the SWH API",
"paramRequired": false
} }
] ]

View File

@ -8,6 +8,8 @@ archiveRequestsPath=${workingDir}/3_archive_requests.seq
actionsetsPath=${workingDir}/4_actionsets actionsetsPath=${workingDir}/4_actionsets
graphPath=/tmp/prod_provision/graph/18_graph_blacklisted graphPath=/tmp/prod_provision/graph/18_graph_blacklisted
apiAccessToken=eyJhbGciOiJIUzI1NiIsInR5cCIgOiAiSldUIiwia2lkIiA6ICJhMTMxYTQ1My1hM2IyLTQwMTUtODQ2Ny05MzAyZjk3MTFkOGEifQ.eyJpYXQiOjE2OTQ2MzYwMjAsImp0aSI6IjkwZjdkNTNjLTQ5YTktNGFiMy1hY2E0LTcwMTViMjEyZTNjNiIsImlzcyI6Imh0dHBzOi8vYXV0aC5zb2Z0d2FyZWhlcml0YWdlLm9yZy9hdXRoL3JlYWxtcy9Tb2Z0d2FyZUhlcml0YWdlIiwiYXVkIjoiaHR0cHM6Ly9hdXRoLnNvZnR3YXJlaGVyaXRhZ2Uub3JnL2F1dGgvcmVhbG1zL1NvZnR3YXJlSGVyaXRhZ2UiLCJzdWIiOiIzMTY5OWZkNC0xNmE0LTQxOWItYTdhMi00NjI5MDY4ZjI3OWEiLCJ0eXAiOiJPZmZsaW5lIiwiYXpwIjoic3doLXdlYiIsInNlc3Npb25fc3RhdGUiOiIzMjYzMzEwMS00ZDRkLTQwMjItODU2NC1iMzNlMTJiNTE3ZDkiLCJzY29wZSI6Im9wZW5pZCBvZmZsaW5lX2FjY2VzcyBwcm9maWxlIGVtYWlsIn0.XHj1VIZu1dZ4Ej32-oU84mFmaox9cLNjXosNxwZM0Xs
maxNumberOfRetry=2 maxNumberOfRetry=2
retryDelay=1 retryDelay=1
requestDelay=100 requestDelay=100

View File

@ -38,6 +38,10 @@
<name>requestDelay</name> <name>requestDelay</name>
<description>Delay between API requests (in ms)</description> <description>Delay between API requests (in ms)</description>
</property> </property>
<property>
<name>apiAccessToken</name>
<description>The API Key of the SWH API</description>
</property>
<property> <property>
<name>softwareLimit</name> <name>softwareLimit</name>
<description>Limit on the number of repo URLs to use (Optional); for debug purposes</description> <description>Limit on the number of repo URLs to use (Optional); for debug purposes</description>
@ -61,6 +65,10 @@
<name>actionsetsPath</name> <name>actionsetsPath</name>
<value>${actionsetsPath}</value> <value>${actionsetsPath}</value>
</property> </property>
<property>
<name>apiAccessToken</name>
<value>${apiAccessToken}</value>
</property>
</configuration> </configuration>
</global> </global>
@ -117,6 +125,7 @@
<arg>--requestDelay</arg><arg>${requestDelay}</arg> <arg>--requestDelay</arg><arg>${requestDelay}</arg>
<arg>--retryDelay</arg><arg>${retryDelay}</arg> <arg>--retryDelay</arg><arg>${retryDelay}</arg>
<arg>--requestMethod</arg><arg>GET</arg> <arg>--requestMethod</arg><arg>GET</arg>
<arg>--apiAccessToken</arg><arg>${apiAccessToken}</arg>
</java> </java>
<ok to="archive-repository-urls"/> <ok to="archive-repository-urls"/>
@ -136,6 +145,7 @@
<arg>--requestDelay</arg><arg>${requestDelay}</arg> <arg>--requestDelay</arg><arg>${requestDelay}</arg>
<arg>--retryDelay</arg><arg>${retryDelay}</arg> <arg>--retryDelay</arg><arg>${retryDelay}</arg>
<arg>--requestMethod</arg><arg>POST</arg> <arg>--requestMethod</arg><arg>POST</arg>
<arg>--apiAccessToken</arg><arg>${apiAccessToken}</arg>
</java> </java>
<ok to="create-swh-actionsets"/> <ok to="create-swh-actionsets"/>

View File

@ -25,7 +25,7 @@ public class SWHConnectionTest {
HttpClientParams clientParams = new HttpClientParams(); HttpClientParams clientParams = new HttpClientParams();
clientParams.setRequestMethod("GET"); clientParams.setRequestMethod("GET");
SWHConnection swhConnection = new SWHConnection(clientParams); SWHConnection swhConnection = new SWHConnection(clientParams, null);
String repoUrl = "https://github.com/stanford-futuredata/FAST"; String repoUrl = "https://github.com/stanford-futuredata/FAST";
URL url = new URL(String.format(SWHConstants.SWH_LATEST_VISIT_URL, repoUrl)); URL url = new URL(String.format(SWHConstants.SWH_LATEST_VISIT_URL, repoUrl));
@ -43,7 +43,7 @@ public class SWHConnectionTest {
HttpClientParams clientParams = new HttpClientParams(); HttpClientParams clientParams = new HttpClientParams();
clientParams.setRequestMethod("POST"); clientParams.setRequestMethod("POST");
SWHConnection swhConnection = new SWHConnection(clientParams); SWHConnection swhConnection = new SWHConnection(clientParams, null);
String repoUrl = "https://github.com/stanford-futuredata/FAST"; String repoUrl = "https://github.com/stanford-futuredata/FAST";
URL url = new URL(String.format(SWHConstants.SWH_ARCHIVE_URL, SWHConstants.DEFAULT_VISIT_TYPE, repoUrl)); URL url = new URL(String.format(SWHConstants.SWH_ARCHIVE_URL, SWHConstants.DEFAULT_VISIT_TYPE, repoUrl));