sabeel
/
dnet-hadoop
forked from D-Net/dnet-hadoop
1
0
Fork 0
Code Pull Requests Activity

Add support to cretate/update solr collection aliases

This commit is contained in:
Serafeim Chatzopoulos 2024-06-20 16:03:15 +03:00
parent 6421f8fece
commit 9f6e16a03c
5 changed files with 118 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/provision/ProvisionConstants.java
View File

@ -14,4 +14,7 @@ public class ProvisionConstants {
return format + SEPARATOR + LAYOUT + SEPARATOR + INTERPRETATION;
}
public static final String PUBLIC_ALIAS_NAME = "public";
public static final String SHADOW_ALIAS_NAME = "shadow";
}

55
dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/provision/SolrAdminApplication.java
View File

@ -9,6 +9,7 @@ import org.apache.commons.io.IOUtils;
import org.apache.solr.client.solrj.SolrResponse;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.impl.CloudSolrClient;
import org.apache.solr.client.solrj.request.CollectionAdminRequest;
import org.apache.solr.client.solrj.response.UpdateResponse;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -23,7 +24,7 @@ public class SolrAdminApplication implements Closeable {
private static final Logger log = LoggerFactory.getLogger(SolrAdminApplication.class);
enum Action {
DELETE_BY_QUERY, COMMIT
DELETE_BY_QUERY, COMMIT, UPDATE_ALIASES
}
private final CloudSolrClient solrClient;
@ -62,8 +63,21 @@ public class SolrAdminApplication implements Closeable {
final String collection = ProvisionConstants.getCollectionName(format);
log.info("collection: {}", collection);
final String publicFormat = parser.get("publicFormat");
log.info("publicFormat: {}", publicFormat);
final String shadowFormat = parser.get("shadowFormat");
log.info("shadowFormat: {}", shadowFormat);
// get collection names from metadata format profiles names
final String publicCollection = ProvisionConstants.getCollectionName(publicFormat);
log.info("publicCollection: {}", publicCollection);
final String shadowCollection = ProvisionConstants.getCollectionName(shadowFormat);
log.info("shadowCollection: {}", shadowCollection);
try (SolrAdminApplication app = new SolrAdminApplication(zkHost)) {
app.execute(action, collection, query, commit);
app.execute(action, collection, query, commit, publicCollection, shadowCollection);
}
}
@ -73,21 +87,28 @@ public class SolrAdminApplication implements Closeable {
}
public SolrResponse commit(String collection) throws IOException, SolrServerException {
return execute(Action.COMMIT, collection, null, true);
return execute(Action.COMMIT, collection, null, true, null, null);
}
public SolrResponse execute(Action action, String collection, String query, boolean commit)
public SolrResponse execute(Action action, String collection, String query, boolean commit,
String publicCollection, String shadowCollection)
throws IOException, SolrServerException {
switch (action) {
case DELETE_BY_QUERY:
UpdateResponse rsp = solrClient.deleteByQuery(collection, query);
if (commit) {
solrClient.commit(collection);
return solrClient.commit(collection);
}
return rsp;
case COMMIT:
return solrClient.commit(collection);
case UPDATE_ALIASES:
this.updateAliases(publicCollection, shadowCollection);
return null;
default:
throw new IllegalArgumentException("action not managed: " + action);
}
@ -98,4 +119,28 @@ public class SolrAdminApplication implements Closeable {
solrClient.close();
}
private void updateAliases(String publicCollection, String shadowCollection)
throws SolrServerException, IOException {
// delete current aliases
this.deleteAlias(ProvisionConstants.PUBLIC_ALIAS_NAME);
this.deleteAlias(ProvisionConstants.SHADOW_ALIAS_NAME);
// create aliases
this.createAlias(ProvisionConstants.PUBLIC_ALIAS_NAME, publicCollection);
this.createAlias(ProvisionConstants.SHADOW_ALIAS_NAME, shadowCollection);
}
public SolrResponse deleteAlias(String aliasName) throws SolrServerException, IOException {
CollectionAdminRequest.DeleteAlias deleteAliasRequest = CollectionAdminRequest.deleteAlias(aliasName);
return deleteAliasRequest.process(solrClient);
}
public SolrResponse createAlias(String aliasName, String collection) throws IOException, SolrServerException {
CollectionAdminRequest.CreateAlias createAliasRequest = CollectionAdminRequest
.createAlias(aliasName, collection);
return createAliasRequest.process(solrClient);
}
}

13
dhp-workflows/dhp-graph-provision/src/main/resources/eu/dnetlib/dhp/oa/provision/input_solradmin_parameters.json
View File

@ -28,5 +28,18 @@
"paramLongName": "commit",
"paramDescription": "should the action be followed by a commit?",
"paramRequired": false
},
{
"paramName": "pf",
"paramLongName": "publicFormat",
"paramDescription": "the name of the public metadata format profile - used to create an alias",
"paramRequired": false
},
{
"paramName": "sf",
"paramLongName": "shadowFormat",
"paramDescription": "the name of the shadow metadata format profile - used to create an alias",
"paramRequired": false
}
]

29
dhp-workflows/dhp-graph-provision/src/main/resources/eu/dnetlib/dhp/oa/provision/oozie_app/workflow.xml
View File

@ -110,6 +110,14 @@
<name>sparkNetworkTimeout</name>
<description>configures spark.network.timeout</description>
</property>
<property>
<name>publicFormat</name>
<description>the public metadata format - used to create the public collection alias</description>
</property>
<property>
<name>shadowFormat</name>
<description>the shadow metadata format - used to create the shadow collection alias</description>
</property>
</parameters>
<global>
@ -133,6 +141,7 @@
<case to="create_payloads">${wf:conf('resumeFrom') eq 'create_payloads'}</case>
<case to="drop_solr_collection">${wf:conf('resumeFrom') eq 'drop_solr_collection'}</case>
<case to="to_solr_index">${wf:conf('resumeFrom') eq 'to_solr_index'}</case>
<case to="update_solr_aliases">${wf:conf('resumeFrom') eq 'update_solr_aliases'}</case>
<default to="prepare_relations"/>
</switch>
</decision>
@ -721,5 +730,25 @@
<error to="Kill"/>
</action>
<!-- Action that updates the solr core aliases - out of order execution, only using the 'resume_from' param -->
<action name="update_solr_aliases">
<java>
<configuration>
<property>
<name>oozie.launcher.mapreduce.user.classpath.first</name>
<value>true</value>
</property>
</configuration>
<main-class>eu.dnetlib.dhp.oa.provision.SolrAdminApplication</main-class>
<arg>--isLookupUrl</arg><arg>${isLookupUrl}</arg>
<arg>--format</arg><arg>${format}</arg>
<arg>--action</arg><arg>UPDATE_ALIASES</arg>
<arg>--publicFormat</arg><arg>${publicFormat}</arg>
<arg>--shadowFormat</arg><arg>${shadowFormat}</arg>
</java>
<ok to="End"/>
<error to="Kill"/>
</action>
<end name="End"/>
</workflow-app>

25
dhp-workflows/dhp-graph-provision/src/test/java/eu/dnetlib/dhp/oa/provision/SolrAdminApplicationTest.java
View File

@ -4,9 +4,9 @@ package eu.dnetlib.dhp.oa.provision;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertTrue;
import org.apache.solr.client.solrj.response.CollectionAdminResponse;
import org.apache.solr.client.solrj.response.SolrPingResponse;
import org.apache.solr.client.solrj.response.UpdateResponse;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
class SolrAdminApplicationTest extends SolrTest {
@ -24,7 +24,7 @@ class SolrAdminApplicationTest extends SolrTest {
SolrAdminApplication admin = new SolrAdminApplication(miniCluster.getSolrClient().getZkHost());
UpdateResponse rsp = (UpdateResponse) admin
.execute(SolrAdminApplication.Action.DELETE_BY_QUERY, DEFAULT_COLLECTION, "*:*", false);
.execute(SolrAdminApplication.Action.DELETE_BY_QUERY, DEFAULT_COLLECTION, "*:*", false, null, null);
assertEquals(0, rsp.getStatus());
}
@ -39,4 +39,25 @@ class SolrAdminApplicationTest extends SolrTest {
assertEquals(0, rsp.getStatus());
}
@Test
void testAdminApplication_CREATE_ALIAS() throws Exception {
SolrAdminApplication admin = new SolrAdminApplication(miniCluster.getSolrClient().getZkHost());
CollectionAdminResponse rsp = (CollectionAdminResponse) admin
.createAlias(ProvisionConstants.PUBLIC_ALIAS_NAME, DEFAULT_COLLECTION);
assertEquals(0, rsp.getStatus());
}
@Test
void testAdminApplication_DELETE_ALIAS() throws Exception {
SolrAdminApplication admin = new SolrAdminApplication(miniCluster.getSolrClient().getZkHost());
CollectionAdminResponse rsp = (CollectionAdminResponse) admin.deleteAlias(ProvisionConstants.PUBLIC_ALIAS_NAME);
assertEquals(0, rsp.getStatus());
}
}