From 81ea051f8274ba8c61aac7cde5d2f569c1366c6c Mon Sep 17 00:00:00 2001 From: Alfredo Oliviero Date: Fri, 6 Dec 2024 10:38:42 +0100 Subject: [PATCH] variabili --- .env_default | 7 ++----- dump.sh | 14 +++++++------- scripts/import.sh | 6 +++--- scripts/is_keyspace_exists.sh | 14 +++++++------- scripts/is_schema_agreed.sh | 2 +- scripts/setup.sh | 28 ++++++++++++++-------------- scripts/setup_orig.sh | 26 +++++++++++++------------- 7 files changed, 47 insertions(+), 50 deletions(-) diff --git a/.env_default b/.env_default index 6ac89f3..eb090e8 100644 --- a/.env_default +++ b/.env_default @@ -1,11 +1,8 @@ CASSANDRA_CLUSTER_NAME=TestCluster CASSANDRA_SEEDS=cassandra1,cassandra2,cassandra3 CASSANDRA_NUM_TOKENS=256 - -KEYSPACE=dev_keyspace_1 - -DUMP_DIR=/dump - +CASSANDRA_KEYSPACE=dev_keyspace_1 +CASSANDRA_DUMP_DIR=/dump CASSANDRA_DC=DC1 CASSANDRA_RACK=RAC1 diff --git a/dump.sh b/dump.sh index 032cd7c..8d6d7ba 100755 --- a/dump.sh +++ b/dump.sh @@ -1,12 +1,12 @@ #!/bin/bash # Configuration -KEYSPACE="dev_keyspace_1" +CASSANDRA_KEYSPACE="dev_keyspace_1" LOCAL_DIR="data/dumps" NODES=("cass-dev-01" "cass-dev-02" "cass-dev-03") SSH_USER="alfredo.oliviero" DATA_PATH="/data" -SNAPSHOT_PATH="${DATA_PATH}/${KEYSPACE}" +SNAPSHOT_PATH="${DATA_PATH}/${CASSANDRA_KEYSPACE}" NODETOOL="/home/alfredo.oliviero/apache-cassandra-4.1.3/bin/nodetool" CQLSH="/home/alfredo.oliviero/apache-cassandra-4.1.3/bin/cqlsh" CQLSH_IP="10.1.28.100" @@ -40,10 +40,10 @@ snapshot_and_copy() { rm -rf $LOCAL_DIR/$NODE_NAME log "Removing old snapshots on $NODE" - ssh $SSH_USER@$NODE "sudo $NODETOOL clearsnapshot -t $DUMP_TAG -- $KEYSPACE" + ssh $SSH_USER@$NODE "sudo $NODETOOL clearsnapshot -t $DUMP_TAG -- $CASSANDRA_KEYSPACE" log "Creating snapshot on $NODE" - ssh $SSH_USER@$NODE "sudo $NODETOOL snapshot -t $DUMP_TAG $KEYSPACE" + ssh $SSH_USER@$NODE "sudo $NODETOOL snapshot -t $DUMP_TAG $CASSANDRA_KEYSPACE" # Find the snapshot path and copy it if it exists TABLES=$(ssh $SSH_USER@$NODE "find $SNAPSHOT_PATH -name $DUMP_TAG") @@ -70,13 +70,13 @@ cleanup() { # Set trap to catch signals and run cleanup trap cleanup SIGINT SIGTERM -log "Starting snapshot creation for keyspace $KEYSPACE" +log "Starting snapshot creation for keyspace $CASSANDRA_KEYSPACE" # Export keyspace schema from the first node -log "Exporting keyspace schema for $KEYSPACE from ${NODES[0]}" +log "Exporting keyspace schema for $CASSANDRA_KEYSPACE from ${NODES[0]}" mkdir -p ${LOCAL_DIR}/schema -ssh $SSH_USER@${NODES[0]} "sudo $CQLSH $CQLSH_IP -e 'DESCRIBE KEYSPACE $KEYSPACE;'" > "${LOCAL_DIR}/schema/${KEYSPACE}_schema.cql" +ssh $SSH_USER@${NODES[0]} "sudo $CQLSH $CQLSH_IP -e 'DESCRIBE CASSANDRA_KEYSPACE $CASSANDRA_KEYSPACE;'" > "${LOCAL_DIR}/schema/${CASSANDRA_KEYSPACE}_schema.cql" # Perform snapshot and copy for each node in parallel NODE_INDEX=1 diff --git a/scripts/import.sh b/scripts/import.sh index cbf9b0d..c4a1b86 100755 --- a/scripts/import.sh +++ b/scripts/import.sh @@ -1,7 +1,7 @@ #!/bin/bash -SNAPSHOT_DIR="$DUMP_DIR/snapshot" +SNAPSHOT_DIR="$CASSANDRA_DUMP_DIR/snapshot" IP_ADDRESS=$(hostname -I | awk '{print $1}') # Define a logging function @@ -14,6 +14,6 @@ log "Importing snapshots using sstableloader..." for TABLE_DIR in $(ls $SNAPSHOT_DIR); do TABLE_NAME=$(basename $TABLE_DIR) # Extract table name from directory name log "Importing table: $TABLE_NAME from directory: $SNAPSHOT_DIR/$TABLE_DIR" - sstableloader -d "$CASSANDRA_SEEDS" -v -k "$KEYSPACE" "$SNAPSHOT_DIR/$TABLE_DIR" - cqlsh $IP_ADDRESS -k "$KEYSPACE" -e "select count(*) from $TABLE_NAME;" >&2 + sstableloader -d "$CASSANDRA_SEEDS" -v -k "$CASSANDRA_KEYSPACE" "$SNAPSHOT_DIR/$TABLE_DIR" + cqlsh $IP_ADDRESS -k "$CASSANDRA_KEYSPACE" -e "select count(*) from $TABLE_NAME;" >&2 done diff --git a/scripts/is_keyspace_exists.sh b/scripts/is_keyspace_exists.sh index 6e43516..964aa09 100755 --- a/scripts/is_keyspace_exists.sh +++ b/scripts/is_keyspace_exists.sh @@ -3,13 +3,13 @@ # Usage: is_keyspace_exists.sh [--keyspace ] # Example: is_keyspace_exists.sh --keyspace dev_keyspace_1 -KEYSPACE=${KEYSPACE:-} +CASSANDRA_KEYSPACE=${CASSANDRA_KEYSPACE:-} # Parse arguments while [ $# -gt 0 ]; do case "$1" in --keyspace) - KEYSPACE="$2" + CASSANDRA_KEYSPACE="$2" shift 2 ;; *) @@ -20,16 +20,16 @@ while [ $# -gt 0 ]; do done # Check for required arguments or environment variables -if [ -z "$KEYSPACE" ]; then - echo "KEYSPACE is not set. Set it via --keyspace or KEYSPACE environment variable." +if [ -z "$CASSANDRA_KEYSPACE" ]; then + echo "CASSANDRA_KEYSPACE is not set. Set it via --keyspace or CASSANDRA_KEYSPACE environment variable." exit 1 fi IP_ADDRESS=$(hostname -I | awk '{print $1}') -if cqlsh $IP_ADDRESS -e "DESCRIBE KEYSPACE $KEYSPACE;" > /dev/null 2>&1; then - echo "Keyspace $KEYSPACE EXISTS" +if cqlsh $IP_ADDRESS -e "DESCRIBE CASSANDRA_KEYSPACE $CASSANDRA_KEYSPACE;" > /dev/null 2>&1; then + echo "Keyspace $CASSANDRA_KEYSPACE EXISTS" exit 0 fi -echo "Keyspace $KEYSPACE DOES NOT EXIST" +echo "Keyspace $CASSANDRA_KEYSPACE DOES NOT EXIST" exit 1 diff --git a/scripts/is_schema_agreed.sh b/scripts/is_schema_agreed.sh index 0e4bf0a..73778c4 100755 --- a/scripts/is_schema_agreed.sh +++ b/scripts/is_schema_agreed.sh @@ -13,7 +13,7 @@ log() { log "Checking if schema is agreed..." is_schema_agreed() { - if cqlsh $IP_ADDRESS -e "DESCRIBE KEYSPACE $KEYSPACE;" > /dev/null 2>&1; then + if cqlsh $IP_ADDRESS -e "DESCRIBE CASSANDRA_KEYSPACE $CASSANDRA_KEYSPACE;" > /dev/null 2>&1; then SCHEMA_NODES=$(nodetool describecluster | grep -A 1 "Schema versions:" | grep -o '\[.*\]' | tr -d '[]' | tr ',' '\n' | wc -l) if [ "$SCHEMA_NODES" -eq "${#SEEDS[@]}" ]; then return 0 diff --git a/scripts/setup.sh b/scripts/setup.sh index d0a54eb..02b1a78 100755 --- a/scripts/setup.sh +++ b/scripts/setup.sh @@ -9,13 +9,13 @@ log() { log "RUNNING SETUP" # Configuration -KEYSPACE="dev_keyspace_1" -DUMP_DIR="/dump" # Ensure DUMP_DIR is defined -SCHEMA_PATH="$DUMP_DIR/schema/${KEYSPACE}_schema.cql" +CASSANDRA_KEYSPACE="dev_keyspace_1" +CASSANDRA_DUMP_DIR="/dump" # Ensure CASSANDRA_DUMP_DIR is defined +SCHEMA_PATH="$CASSANDRA_DUMP_DIR/schema/${CASSANDRA_KEYSPACE}_schema.cql" CASSANDRA_SEEDS="cassandra1,cassandra2,cassandra3" IP_ADDRESS=$(hostname -I | awk '{print $1}') -DATA_DIR="/var/lib/cassandra/data/$KEYSPACE" -SNAPSHOT_DIR="$DUMP_DIR/snapshot" +DATA_DIR="/var/lib/cassandra/data/$CASSANDRA_KEYSPACE" +SNAPSHOT_DIR="$CASSANDRA_DUMP_DIR/snapshot" PRIMARY_NODE=${PRIMARY_NODE:-false} # Default to false if not set SLEEP_DURATION=5 # Sleep duration in seconds for waits TIMEOUT=3000 # Timeout in seconds for waits @@ -72,7 +72,7 @@ wait_for_all_nodes_up() { # Function to check for schema agreement and if schema exists is_schema_agreed() { - if cqlsh $IP_ADDRESS -e "DESCRIBE KEYSPACE $KEYSPACE;" > /dev/null 2>&1; then + if cqlsh $IP_ADDRESS -e "DESCRIBE CASSANDRA_KEYSPACE $CASSANDRA_KEYSPACE;" > /dev/null 2>&1; then SCHEMA_NODES=$(nodetool describecluster | grep -A 1 "Schema versions:" | grep -o '\[.*\]' | tr -d '[]' | tr ',' '\n' | wc -l) if [ "$SCHEMA_NODES" -eq "${#SEEDS[@]}" ]; then return 0 @@ -83,14 +83,14 @@ is_schema_agreed() { # Function to check if keyspace exists is_keyspace_exists() { - if cqlsh $IP_ADDRESS -e "DESCRIBE KEYSPACE $KEYSPACE;" > /dev/null 2>&1; then + if cqlsh $IP_ADDRESS -e "DESCRIBE CASSANDRA_KEYSPACE $CASSANDRA_KEYSPACE;" > /dev/null 2>&1; then return 0 fi return 1 } -log "setup KEYSPACE: $KEYSPACE" -log "setup DUMP_DIR: $DUMP_DIR" +log "setup CASSANDRA_KEYSPACE: $CASSANDRA_KEYSPACE" +log "setup CASSANDRA_DUMP_DIR: $CASSANDRA_DUMP_DIR" log "setup SCHEMA_PATH: $SCHEMA_PATH" log "setup CASSANDRA_SEEDS: $CASSANDRA_SEEDS" @@ -123,12 +123,12 @@ wait_for_all_nodes_up # Step 2: Create keyspace and schema on the primary node if [ "$PRIMARY_NODE" = "true" ]; then - log "Checking if keyspace $KEYSPACE exists..." + log "Checking if keyspace $CASSANDRA_KEYSPACE exists..." if ! is_keyspace_exists; then - log "Keyspace $KEYSPACE does not exist. Creating keyspace and tables..." + log "Keyspace $CASSANDRA_KEYSPACE does not exist. Creating keyspace and tables..." cqlsh $IP_ADDRESS -f "$SCHEMA_PATH" else - log "Keyspace $KEYSPACE already exists. Ensuring tables exist..." + log "Keyspace $CASSANDRA_KEYSPACE already exists. Ensuring tables exist..." fi fi @@ -142,8 +142,8 @@ wait_for_command "is_schema_agreed" $TIMEOUT for TABLE_DIR in $(ls $SNAPSHOT_DIR); do TABLE_NAME=$(basename $TABLE_DIR) # Extract table name from directory name log "Importing table: $TABLE_NAME from directory: $SNAPSHOT_DIR/$TABLE_DIR" - sstableloader -d "$CASSANDRA_SEEDS" -v -k "$KEYSPACE" "$SNAPSHOT_DIR/$TABLE_DIR" - cqlsh $IP_ADDRESS -k "$KEYSPACE" -e "select count(*) from $TABLE_NAME;" >&2 + sstableloader -d "$CASSANDRA_SEEDS" -v -k "$CASSANDRA_KEYSPACE" "$SNAPSHOT_DIR/$TABLE_DIR" + cqlsh $IP_ADDRESS -k "$CASSANDRA_KEYSPACE" -e "select count(*) from $TABLE_NAME;" >&2 done # fi diff --git a/scripts/setup_orig.sh b/scripts/setup_orig.sh index 3283ec3..d81db91 100755 --- a/scripts/setup_orig.sh +++ b/scripts/setup_orig.sh @@ -9,16 +9,16 @@ log() { log "RUNNING SETUP" # Configuration -KEYSPACE=${KEYSPACE:-dev_keyspace_1} -DUMP_DIR=${DUMP_DIR:-/dump} # Ensure DUMP_DIR is defined +CASSANDRA_KEYSPACE=${CASSANDRA_KEYSPACE:-dev_keyspace_1} +CASSANDRA_DUMP_DIR=${CASSANDRA_DUMP_DIR:-/dump} # Ensure CASSANDRA_DUMP_DIR is defined CASSANDRA_SEEDS=${CASSANDRA_SEEDS:-cassandra1,cassandra2,cassandra3} PRIMARY_NODE=${PRIMARY_NODE:-false} # Default to false if not set IP_ADDRESS=$(hostname -I | awk '{print $1}') -SCHEMA_PATH="$DUMP_DIR/schema/${KEYSPACE}_schema.cql" -DATA_DIR="/var/lib/cassandra/data/$KEYSPACE" -SNAPSHOT_DIR="$DUMP_DIR/snapshot" +SCHEMA_PATH="$CASSANDRA_DUMP_DIR/schema/${CASSANDRA_KEYSPACE}_schema.cql" +DATA_DIR="/var/lib/cassandra/data/$CASSANDRA_KEYSPACE" +SNAPSHOT_DIR="$CASSANDRA_DUMP_DIR/snapshot" # Initialize SEEDS array SEEDS=(${CASSANDRA_SEEDS//,/ }) @@ -50,8 +50,8 @@ wait_for_command() { done } -log "setup KEYSPACE: $KEYSPACE" -log "setup DUMP_DIR: $DUMP_DIR" +log "setup CASSANDRA_KEYSPACE: $CASSANDRA_KEYSPACE" +log "setup CASSANDRA_DUMP_DIR: $CASSANDRA_DUMP_DIR" log "setup SCHEMA_PATH: $SCHEMA_PATH" log "setup CASSANDRA_SEEDS: $CASSANDRA_SEEDS" @@ -95,23 +95,23 @@ wait_for_all_nodes_up # Step 2: Create keyspace and schema on the primary node if [ "$PRIMARY_NODE" = "true" ]; then - log "Checking if keyspace $KEYSPACE exists..." - if ! /scripts/is_keyspace_exists.sh --keyspace "$KEYSPACE"; then - log "Keyspace $KEYSPACE does not exist. Creating keyspace and tables..." + log "Checking if keyspace $CASSANDRA_KEYSPACE exists..." + if ! /scripts/is_keyspace_exists.sh --keyspace "$CASSANDRA_KEYSPACE"; then + log "Keyspace $CASSANDRA_KEYSPACE does not exist. Creating keyspace and tables..." cqlsh $IP_ADDRESS -f "$SCHEMA_PATH" else - log "Keyspace $KEYSPACE already exists. Ensuring tables exist..." + log "Keyspace $CASSANDRA_KEYSPACE already exists. Ensuring tables exist..." fi fi # Step 3: Wait for schema to be created and agreed upon across all nodes log "Waiting for schema agreement across all nodes..." -wait_for_command "/scripts/is_schema_agreed.sh --keyspace $KEYSPACE --cassandra_seeds $CASSANDRA_SEEDS" $TIMEOUT +wait_for_command "/scripts/is_schema_agreed.sh --keyspace $CASSANDRA_KEYSPACE --cassandra_seeds $CASSANDRA_SEEDS" $TIMEOUT # Step 4: Import data using sstableloader if not previously imported if [ "$EMPTY_DB" = true ]; then log "Importing snapshots using sstableloader..." - /scripts/import.sh --keyspace "$KEYSPACE" --dump_dir "$SNAPSHOT_DIR" --cassandra_seeds "$CASSANDRA_SEEDS" + /scripts/import.sh --keyspace "$CASSANDRA_KEYSPACE" --CASSANDRA_DUMP_DIR "$SNAPSHOT_DIR" --cassandra_seeds "$CASSANDRA_SEEDS" fi log "FINISHED IMPORT"