alfredo.oliviero
/
dockerizing_cassandra
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

variabili

This commit is contained in:
Alfredo Oliviero 2024-12-06 10:38:42 +01:00
parent 1ff213ba57
commit 81ea051f82
7 changed files with 47 additions and 50 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
.env_default
View File

@ -1,11 +1,8 @@
CASSANDRA_CLUSTER_NAME=TestCluster
CASSANDRA_SEEDS=cassandra1,cassandra2,cassandra3
CASSANDRA_NUM_TOKENS=256
KEYSPACE=dev_keyspace_1
DUMP_DIR=/dump
CASSANDRA_KEYSPACE=dev_keyspace_1
CASSANDRA_DUMP_DIR=/dump
CASSANDRA_DC=DC1
CASSANDRA_RACK=RAC1

14
dump.sh
View File

@ -1,12 +1,12 @@
#!/bin/bash
# Configuration
KEYSPACE="dev_keyspace_1"
CASSANDRA_KEYSPACE="dev_keyspace_1"
LOCAL_DIR="data/dumps"
NODES=("cass-dev-01" "cass-dev-02" "cass-dev-03")
SSH_USER="alfredo.oliviero"
DATA_PATH="/data"
SNAPSHOT_PATH="${DATA_PATH}/${KEYSPACE}"
SNAPSHOT_PATH="${DATA_PATH}/${CASSANDRA_KEYSPACE}"
NODETOOL="/home/alfredo.oliviero/apache-cassandra-4.1.3/bin/nodetool"
CQLSH="/home/alfredo.oliviero/apache-cassandra-4.1.3/bin/cqlsh"
CQLSH_IP="10.1.28.100"
@ -40,10 +40,10 @@ snapshot_and_copy() {
rm -rf $LOCAL_DIR/$NODE_NAME
log "Removing old snapshots on $NODE"
ssh $SSH_USER@$NODE "sudo $NODETOOL clearsnapshot -t $DUMP_TAG -- $KEYSPACE"
ssh $SSH_USER@$NODE "sudo $NODETOOL clearsnapshot -t $DUMP_TAG -- $CASSANDRA_KEYSPACE"
log "Creating snapshot on $NODE"
ssh $SSH_USER@$NODE "sudo $NODETOOL snapshot -t $DUMP_TAG $KEYSPACE"
ssh $SSH_USER@$NODE "sudo $NODETOOL snapshot -t $DUMP_TAG $CASSANDRA_KEYSPACE"
# Find the snapshot path and copy it if it exists
TABLES=$(ssh $SSH_USER@$NODE "find $SNAPSHOT_PATH -name $DUMP_TAG")
@ -70,13 +70,13 @@ cleanup() {
# Set trap to catch signals and run cleanup
trap cleanup SIGINT SIGTERM
log "Starting snapshot creation for keyspace $KEYSPACE"
log "Starting snapshot creation for keyspace $CASSANDRA_KEYSPACE"
# Export keyspace schema from the first node
log "Exporting keyspace schema for $KEYSPACE from ${NODES[0]}"
log "Exporting keyspace schema for $CASSANDRA_KEYSPACE from ${NODES[0]}"
mkdir -p ${LOCAL_DIR}/schema
ssh $SSH_USER@${NODES[0]} "sudo $CQLSH $CQLSH_IP -e 'DESCRIBE KEYSPACE $KEYSPACE;'" > "${LOCAL_DIR}/schema/${KEYSPACE}_schema.cql"
ssh $SSH_USER@${NODES[0]} "sudo $CQLSH $CQLSH_IP -e 'DESCRIBE CASSANDRA_KEYSPACE $CASSANDRA_KEYSPACE;'" > "${LOCAL_DIR}/schema/${CASSANDRA_KEYSPACE}_schema.cql"
# Perform snapshot and copy for each node in parallel
NODE_INDEX=1

6
scripts/import.sh
View File

@ -1,7 +1,7 @@
#!/bin/bash
SNAPSHOT_DIR="$DUMP_DIR/snapshot"
SNAPSHOT_DIR="$CASSANDRA_DUMP_DIR/snapshot"
IP_ADDRESS=$(hostname -I | awk '{print $1}')
# Define a logging function
@ -14,6 +14,6 @@ log "Importing snapshots using sstableloader..."
for TABLE_DIR in $(ls $SNAPSHOT_DIR); do
TABLE_NAME=$(basename $TABLE_DIR) # Extract table name from directory name
log "Importing table: $TABLE_NAME from directory: $SNAPSHOT_DIR/$TABLE_DIR"
sstableloader -d "$CASSANDRA_SEEDS" -v -k "$KEYSPACE" "$SNAPSHOT_DIR/$TABLE_DIR"
cqlsh $IP_ADDRESS -k "$KEYSPACE" -e "select count(*) from $TABLE_NAME;" >&2
sstableloader -d "$CASSANDRA_SEEDS" -v -k "$CASSANDRA_KEYSPACE" "$SNAPSHOT_DIR/$TABLE_DIR"
cqlsh $IP_ADDRESS -k "$CASSANDRA_KEYSPACE" -e "select count(*) from $TABLE_NAME;" >&2
done

14
scripts/is_keyspace_exists.sh
View File

@ -3,13 +3,13 @@
# Usage: is_keyspace_exists.sh [--keyspace <keyspace>]
# Example: is_keyspace_exists.sh --keyspace dev_keyspace_1
KEYSPACE=${KEYSPACE:-}
CASSANDRA_KEYSPACE=${CASSANDRA_KEYSPACE:-}
# Parse arguments
while [ $# -gt 0 ]; do
case "$1" in
--keyspace)
KEYSPACE="$2"
CASSANDRA_KEYSPACE="$2"
shift 2
;;
*)
@ -20,16 +20,16 @@ while [ $# -gt 0 ]; do
done
# Check for required arguments or environment variables
if [ -z "$KEYSPACE" ]; then
echo "KEYSPACE is not set. Set it via --keyspace or KEYSPACE environment variable."
if [ -z "$CASSANDRA_KEYSPACE" ]; then
echo "CASSANDRA_KEYSPACE is not set. Set it via --keyspace or CASSANDRA_KEYSPACE environment variable."
exit 1
fi
IP_ADDRESS=$(hostname -I | awk '{print $1}')
if cqlsh $IP_ADDRESS -e "DESCRIBE KEYSPACE $KEYSPACE;" > /dev/null 2>&1; then
echo "Keyspace $KEYSPACE EXISTS"
if cqlsh $IP_ADDRESS -e "DESCRIBE CASSANDRA_KEYSPACE $CASSANDRA_KEYSPACE;" > /dev/null 2>&1; then
echo "Keyspace $CASSANDRA_KEYSPACE EXISTS"
exit 0
fi
echo "Keyspace $KEYSPACE DOES NOT EXIST"
echo "Keyspace $CASSANDRA_KEYSPACE DOES NOT EXIST"
exit 1

2
scripts/is_schema_agreed.sh
View File

@ -13,7 +13,7 @@ log() {
log "Checking if schema is agreed..."
is_schema_agreed() {
if cqlsh $IP_ADDRESS -e "DESCRIBE KEYSPACE $KEYSPACE;" > /dev/null 2>&1; then
if cqlsh $IP_ADDRESS -e "DESCRIBE CASSANDRA_KEYSPACE $CASSANDRA_KEYSPACE;" > /dev/null 2>&1; then
SCHEMA_NODES=$(nodetool describecluster | grep -A 1 "Schema versions:" | grep -o '\[.*\]' | tr -d '[]' | tr ',' '\n' | wc -l)
if [ "$SCHEMA_NODES" -eq "${#SEEDS[@]}" ]; then
return 0

28
scripts/setup.sh
View File

@ -9,13 +9,13 @@ log() {
log "RUNNING SETUP"
# Configuration
KEYSPACE="dev_keyspace_1"
DUMP_DIR="/dump" # Ensure DUMP_DIR is defined
SCHEMA_PATH="$DUMP_DIR/schema/${KEYSPACE}_schema.cql"
CASSANDRA_KEYSPACE="dev_keyspace_1"
CASSANDRA_DUMP_DIR="/dump" # Ensure CASSANDRA_DUMP_DIR is defined
SCHEMA_PATH="$CASSANDRA_DUMP_DIR/schema/${CASSANDRA_KEYSPACE}_schema.cql"
CASSANDRA_SEEDS="cassandra1,cassandra2,cassandra3"
IP_ADDRESS=$(hostname -I | awk '{print $1}')
DATA_DIR="/var/lib/cassandra/data/$KEYSPACE"
SNAPSHOT_DIR="$DUMP_DIR/snapshot"
DATA_DIR="/var/lib/cassandra/data/$CASSANDRA_KEYSPACE"
SNAPSHOT_DIR="$CASSANDRA_DUMP_DIR/snapshot"
PRIMARY_NODE=${PRIMARY_NODE:-false} # Default to false if not set
SLEEP_DURATION=5 # Sleep duration in seconds for waits
TIMEOUT=3000 # Timeout in seconds for waits
@ -72,7 +72,7 @@ wait_for_all_nodes_up() {
# Function to check for schema agreement and if schema exists
is_schema_agreed() {
if cqlsh $IP_ADDRESS -e "DESCRIBE KEYSPACE $KEYSPACE;" > /dev/null 2>&1; then
if cqlsh $IP_ADDRESS -e "DESCRIBE CASSANDRA_KEYSPACE $CASSANDRA_KEYSPACE;" > /dev/null 2>&1; then
SCHEMA_NODES=$(nodetool describecluster | grep -A 1 "Schema versions:" | grep -o '\[.*\]' | tr -d '[]' | tr ',' '\n' | wc -l)
if [ "$SCHEMA_NODES" -eq "${#SEEDS[@]}" ]; then
return 0
@ -83,14 +83,14 @@ is_schema_agreed() {
# Function to check if keyspace exists
is_keyspace_exists() {
if cqlsh $IP_ADDRESS -e "DESCRIBE KEYSPACE $KEYSPACE;" > /dev/null 2>&1; then
if cqlsh $IP_ADDRESS -e "DESCRIBE CASSANDRA_KEYSPACE $CASSANDRA_KEYSPACE;" > /dev/null 2>&1; then
return 0
fi
return 1
}
log "setup KEYSPACE: $KEYSPACE"
log "setup DUMP_DIR: $DUMP_DIR"
log "setup CASSANDRA_KEYSPACE: $CASSANDRA_KEYSPACE"
log "setup CASSANDRA_DUMP_DIR: $CASSANDRA_DUMP_DIR"
log "setup SCHEMA_PATH: $SCHEMA_PATH"
log "setup CASSANDRA_SEEDS: $CASSANDRA_SEEDS"
@ -123,12 +123,12 @@ wait_for_all_nodes_up
# Step 2: Create keyspace and schema on the primary node
if [ "$PRIMARY_NODE" = "true" ]; then
log "Checking if keyspace $KEYSPACE exists..."
log "Checking if keyspace $CASSANDRA_KEYSPACE exists..."
if ! is_keyspace_exists; then
log "Keyspace $KEYSPACE does not exist. Creating keyspace and tables..."
log "Keyspace $CASSANDRA_KEYSPACE does not exist. Creating keyspace and tables..."
cqlsh $IP_ADDRESS -f "$SCHEMA_PATH"
else
log "Keyspace $KEYSPACE already exists. Ensuring tables exist..."
log "Keyspace $CASSANDRA_KEYSPACE already exists. Ensuring tables exist..."
fi
fi
@ -142,8 +142,8 @@ wait_for_command "is_schema_agreed" $TIMEOUT
for TABLE_DIR in $(ls $SNAPSHOT_DIR); do
TABLE_NAME=$(basename $TABLE_DIR) # Extract table name from directory name
log "Importing table: $TABLE_NAME from directory: $SNAPSHOT_DIR/$TABLE_DIR"
sstableloader -d "$CASSANDRA_SEEDS" -v -k "$KEYSPACE" "$SNAPSHOT_DIR/$TABLE_DIR"
cqlsh $IP_ADDRESS -k "$KEYSPACE" -e "select count(*) from $TABLE_NAME;" >&2
sstableloader -d "$CASSANDRA_SEEDS" -v -k "$CASSANDRA_KEYSPACE" "$SNAPSHOT_DIR/$TABLE_DIR"
cqlsh $IP_ADDRESS -k "$CASSANDRA_KEYSPACE" -e "select count(*) from $TABLE_NAME;" >&2
done
# fi

26
scripts/setup_orig.sh
View File

@ -9,16 +9,16 @@ log() {
log "RUNNING SETUP"
# Configuration
KEYSPACE=${KEYSPACE:-dev_keyspace_1}
DUMP_DIR=${DUMP_DIR:-/dump} # Ensure DUMP_DIR is defined
CASSANDRA_KEYSPACE=${CASSANDRA_KEYSPACE:-dev_keyspace_1}
CASSANDRA_DUMP_DIR=${CASSANDRA_DUMP_DIR:-/dump} # Ensure CASSANDRA_DUMP_DIR is defined
CASSANDRA_SEEDS=${CASSANDRA_SEEDS:-cassandra1,cassandra2,cassandra3}
PRIMARY_NODE=${PRIMARY_NODE:-false} # Default to false if not set
IP_ADDRESS=$(hostname -I | awk '{print $1}')
SCHEMA_PATH="$DUMP_DIR/schema/${KEYSPACE}_schema.cql"
DATA_DIR="/var/lib/cassandra/data/$KEYSPACE"
SNAPSHOT_DIR="$DUMP_DIR/snapshot"
SCHEMA_PATH="$CASSANDRA_DUMP_DIR/schema/${CASSANDRA_KEYSPACE}_schema.cql"
DATA_DIR="/var/lib/cassandra/data/$CASSANDRA_KEYSPACE"
SNAPSHOT_DIR="$CASSANDRA_DUMP_DIR/snapshot"
# Initialize SEEDS array
SEEDS=(${CASSANDRA_SEEDS//,/ })
@ -50,8 +50,8 @@ wait_for_command() {
done
}
log "setup KEYSPACE: $KEYSPACE"
log "setup DUMP_DIR: $DUMP_DIR"
log "setup CASSANDRA_KEYSPACE: $CASSANDRA_KEYSPACE"
log "setup CASSANDRA_DUMP_DIR: $CASSANDRA_DUMP_DIR"
log "setup SCHEMA_PATH: $SCHEMA_PATH"
log "setup CASSANDRA_SEEDS: $CASSANDRA_SEEDS"
@ -95,23 +95,23 @@ wait_for_all_nodes_up
# Step 2: Create keyspace and schema on the primary node
if [ "$PRIMARY_NODE" = "true" ]; then
log "Checking if keyspace $KEYSPACE exists..."
if ! /scripts/is_keyspace_exists.sh --keyspace "$KEYSPACE"; then
log "Keyspace $KEYSPACE does not exist. Creating keyspace and tables..."
log "Checking if keyspace $CASSANDRA_KEYSPACE exists..."
if ! /scripts/is_keyspace_exists.sh --keyspace "$CASSANDRA_KEYSPACE"; then
log "Keyspace $CASSANDRA_KEYSPACE does not exist. Creating keyspace and tables..."
cqlsh $IP_ADDRESS -f "$SCHEMA_PATH"
else
log "Keyspace $KEYSPACE already exists. Ensuring tables exist..."
log "Keyspace $CASSANDRA_KEYSPACE already exists. Ensuring tables exist..."
fi
fi
# Step 3: Wait for schema to be created and agreed upon across all nodes
log "Waiting for schema agreement across all nodes..."
wait_for_command "/scripts/is_schema_agreed.sh --keyspace $KEYSPACE --cassandra_seeds $CASSANDRA_SEEDS" $TIMEOUT
wait_for_command "/scripts/is_schema_agreed.sh --keyspace $CASSANDRA_KEYSPACE --cassandra_seeds $CASSANDRA_SEEDS" $TIMEOUT
# Step 4: Import data using sstableloader if not previously imported
if [ "$EMPTY_DB" = true ]; then
log "Importing snapshots using sstableloader..."
/scripts/import.sh --keyspace "$KEYSPACE" --dump_dir "$SNAPSHOT_DIR" --cassandra_seeds "$CASSANDRA_SEEDS"
/scripts/import.sh --keyspace "$CASSANDRA_KEYSPACE" --CASSANDRA_DUMP_DIR "$SNAPSHOT_DIR" --cassandra_seeds "$CASSANDRA_SEEDS"
fi
log "FINISHED IMPORT"