mariadb-demo/scripts/cluster.sh

#!/usr/bin/env bash
# ── MariaDB HA Cluster Control ───────────────────────────
# Manages 6 MariaDB nodes (DC1+DC2) + 2 MaxScale proxies.
# Idempotent: create if missing, start if stopped, skip if running.
#
# Usage: cluster.sh {start|stop|status}
#
# Persists across reboots — all volumes under $HOME, not /tmp.
set -euo pipefail

readonly SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
readonly CONFIG_DIR="${SCRIPT_DIR}/../cluster-config"

# ── Container definitions ────────────────────────────────
# Format: name|image|host_port|container_port|config_file|admin_port(optional)
readonly NODES_DC1=(
  "mariadb1|mariadb:latest|3307|3306|${CONFIG_DIR}/server1.cnf"
  "mariadb2|mariadb:latest|3308|3306|${CONFIG_DIR}/server2.cnf"
  "mariadb3|mariadb:latest|3309|3306|${CONFIG_DIR}/server3.cnf"
)
readonly NODES_DC2=(
  "mariadb4|mariadb:latest|3310|3306|${CONFIG_DIR}/server4.cnf"
  "mariadb5|mariadb:latest|3311|3306|${CONFIG_DIR}/server5.cnf"
  "mariadb6|mariadb:latest|3312|3306|${CONFIG_DIR}/server6.cnf"
)

readonly PROXIES=(
  "maxscale1|mariadb/maxscale:latest|4000|4000|${CONFIG_DIR}/maxscale.cnf|8989"
)

readonly MARIADB_USER="root"
readonly MARIADB_PASS="monitor123"
readonly MAXSCALE_PORT=4000
readonly MARIADB_TEST_PORT=3307
readonly MAX_POLL_SECONDS=60
readonly POLL_INTERVAL=2

# ── Pure helpers ─────────────────────────────────────────

_container_exists() {
  local name="$1"
  docker ps -a --format '{{.Names}}' --filter "name=^${name}$" | grep -q .
}

_container_running() {
  local name="$1"
  docker ps --format '{{.Names}}' --filter "name=^${name}$" | grep -q .
}

_mariadb_healthy() {
  mariadb -h 127.0.0.1 -P "${MARIADB_TEST_PORT}" \
    -u "${MARIADB_USER}" -p"${MARIADB_PASS}" \
    --skip-ssl -e "SELECT 1" &>/dev/null
}

_maxscale_healthy() {
  mariadb -h 127.0.0.1 -P "${MAXSCALE_PORT}" \
    -u "${MARIADB_USER}" -p"${MARIADB_PASS}" \
    --skip-ssl -e "SELECT 1" &>/dev/null
}

_poll_health() {
  local label="$1"
  local check_fn="$2"
  local elapsed=0

  while (( elapsed < MAX_POLL_SECONDS )); do
    if ${check_fn}; then
      echo "  [OK] ${label} ready (${elapsed}s)"
      return 0
    fi
    sleep "${POLL_INTERVAL}"
    elapsed=$(( elapsed + POLL_INTERVAL ))
  done
  echo "  [FAIL] ${label} not ready after ${MAX_POLL_SECONDS}s" >&2
  return 1
}

_create_node() {
  local name="$1" image="$2" host_port="$3" cont_port="$4" config="$5"

  if _container_exists "${name}"; then
    if _container_running "${name}"; then
      echo "  [SKIP] ${name} already running"
      return 0
    fi
    echo "  [START] ${name} (existing container)"
    docker start "${name}" >/dev/null
    return 0
  fi

  echo "  [CREATE] ${name} (port ${host_port})"
  docker run -d --name "${name}" \
    -e MYSQL_ROOT_PASSWORD="${MARIADB_PASS}" \
    -p "${host_port}:${cont_port}" \
    -v "${config}:/etc/mysql/conf.d/seb.cnf:ro" \
    "${image}" >/dev/null
}

_create_proxy() {
  local name="$1" image="$2" host_port="$3" cont_port="$4" config="$5" admin_port="$6"

  if _container_exists "${name}"; then
    if _container_running "${name}"; then
      echo "  [SKIP] ${name} already running"
      return 0
    fi
    echo "  [START] ${name} (existing container)"
    docker start "${name}" >/dev/null
    return 0
  fi

  echo "  [CREATE] ${name} (port ${host_port}, admin :${admin_port})"
  docker run -d --name "${name}" \
    -p "${host_port}:${cont_port}" \
    -p "${admin_port}:8989" \
    -v "${config}:/etc/maxscale.cnf:ro" \
    "${image}" >/dev/null

  # Create REST API user after container is ready (only for first container)
  if [[ "${name}" == "maxscale1" ]]; then
    sleep 3
    docker exec "${name}" maxctrl create user monitor monitor123 --type=admin 2>/dev/null || true
  fi
}

# ── Commands ─────────────────────────────────────────────

cmd_start() {
  echo "=== MariaDB Cluster: Starting ==="

  # 1. Start all MariaDB nodes (DC1 + DC2)
  for entry in "${NODES_DC1[@]}" "${NODES_DC2[@]}"; do
    IFS='|' read -r name image host_port cont_port config <<< "${entry}"
    _create_node "${name}" "${image}" "${host_port}" "${cont_port}" "${config}"
  done

  # 2. Wait for at least one MariaDB node
  echo "  [WAIT] Polling MariaDB on :${MARIADB_TEST_PORT} ..."
  _poll_health "MariaDB" _mariadb_healthy || exit 1

  # 3. Generate MaxScale config (after all node IPs are known)
  local mx_config="${CONFIG_DIR}/maxscale.cnf"
  if ! _container_exists "maxscale1"; then
    echo "  [CONFIG] Generating ${mx_config}"
    _generate_maxscale_config "${mx_config}"
  fi

  # 4. Start MaxScale proxies
  for entry in "${PROXIES[@]}"; do
    IFS='|' read -r name image host_port cont_port config admin_port <<< "${entry}"
    _create_proxy "${name}" "${image}" "${host_port}" "${cont_port}" "${config}" "${admin_port}"
  done

  # 5. Wait for MaxScale
  echo "  [WAIT] Polling MaxScale on :${MAXSCALE_PORT} ..."
  _poll_health "MaxScale" _maxscale_healthy || exit 1

  # 6. Setup replication if needed (idempotent)
  _setup_replication

  echo "=== Cluster ready ==="
}

cmd_stop() {
  echo "=== MariaDB Cluster: Stopping ==="

  # Stop proxies first, then nodes
  local containers=()
  for entry in "${PROXIES[@]}"; do
    IFS='|' read -r name _ <<< "${entry}"
    containers+=("${name}")
  done
  for entry in "${NODES_DC1[@]}" "${NODES_DC2[@]}"; do
    IFS='|' read -r name _ <<< "${entry}"
    containers+=("${name}")
  done

  for name in "${containers[@]}"; do
    if _container_running "${name}"; then
      echo "  [STOP] ${name}"
      docker stop "${name}" >/dev/null
    else
      echo "  [SKIP] ${name} not running"
    fi
  done

  echo "=== Cluster stopped ==="
}

cmd_status() {
  echo "=== MariaDB Cluster: Status ==="
  echo ""

  echo "Cluster 1 (ports 3307-3309):"
  for entry in "${NODES_DC1[@]}"; do
    IFS='|' read -r name _ host_port _ _ <<< "${entry}"
    if _container_running "${name}"; then
      echo "  ${name} :${host_port}  [RUNNING]"
    elif _container_exists "${name}"; then
      echo "  ${name} :${host_port}  [STOPPED]"
    else
      echo "  ${name} :${host_port}  [MISSING]"
    fi
  done

  echo ""
  echo "Cluster 2 (ports 3310-3312):"
  for entry in "${NODES_DC2[@]}"; do
    IFS='|' read -r name _ host_port _ _ <<< "${entry}"
    if _container_running "${name}"; then
      echo "  ${name} :${host_port}  [RUNNING]"
    elif _container_exists "${name}"; then
      echo "  ${name} :${host_port}  [STOPPED]"
    else
      echo "  ${name} :${host_port}  [MISSING]"
    fi
  done

  echo ""
  echo "MaxScale:"
  for entry in "${PROXIES[@]}"; do
    IFS='|' read -r name _ host_port _ _ admin_port <<< "${entry}"
    if _container_running "${name}"; then
      echo "  ${name} :${host_port} (admin :${admin_port})  [RUNNING]"
    elif _container_exists "${name}"; then
      echo "  ${name} :${host_port} (admin :${admin_port})  [STOPPED]"
    else
      echo "  ${name} :${host_port} (admin :${admin_port})  [MISSING]"
    fi
  done

  echo ""
  if _maxscale_healthy; then
    echo "Health: ALL HEALTHY"
  elif _mariadb_healthy; then
    echo "Health: MariaDB UP, MaxScale DOWN"
  else
    echo "Health: DOWN"
  fi
}

# ── MaxScale config generator ─────────────────────────────
# Unified config with DC1 (servers 1-3) + DC2 (servers 4-6)

_generate_maxscale_config() {
  local out="$1"
  local ip1 ip2 ip3 ip4 ip5 ip6

  ip1=$(docker inspect -f '{{range $k,$v := .NetworkSettings.Networks}}{{$v.IPAddress}}{{end}}' mariadb1 2>/dev/null || echo "172.17.0.2")
  ip2=$(docker inspect -f '{{range $k,$v := .NetworkSettings.Networks}}{{$v.IPAddress}}{{end}}' mariadb2 2>/dev/null || echo "172.17.0.3")
  ip3=$(docker inspect -f '{{range $k,$v := .NetworkSettings.Networks}}{{$v.IPAddress}}{{end}}' mariadb3 2>/dev/null || echo "172.17.0.4")
  ip4=$(docker inspect -f '{{range $k,$v := .NetworkSettings.Networks}}{{$v.IPAddress}}{{end}}' mariadb4 2>/dev/null || echo "172.17.0.5")
  ip5=$(docker inspect -f '{{range $k,$v := .NetworkSettings.Networks}}{{$v.IPAddress}}{{end}}' mariadb5 2>/dev/null || echo "172.17.0.6")
  ip6=$(docker inspect -f '{{range $k,$v := .NetworkSettings.Networks}}{{$v.IPAddress}}{{end}}' mariadb6 2>/dev/null || echo "172.17.0.7")

  cat > "${out}" << EOF
[maxscale]
threads=auto
admin_host=0.0.0.0
admin_port=8989
admin_gui=true
admin_secure_gui=false

[server1]
type=server
address=${ip1}
port=3306
protocol=MariaDBBackend

[server2]
type=server
address=${ip2}
port=3306
protocol=MariaDBBackend

[server3]
type=server
address=${ip3}
port=3306
protocol=MariaDBBackend

[server4]
type=server
address=${ip4}
port=3306
protocol=MariaDBBackend

[server5]
type=server
address=${ip5}
port=3306
protocol=MariaDBBackend

[server6]
type=server
address=${ip6}
port=3306
protocol=MariaDBBackend

[HA-monitor]
type=monitor
module=mariadbmon
servers=server1,server2,server3,server4,server5,server6
user=${MARIADB_USER}
password=${MARIADB_PASS}
replication_user=repl
replication_password=repl123
monitor_interval=500ms
backend_connect_timeout=1s
backend_read_timeout=1s
failcount=1
verify_master_failure=false
master_failure_timeout=3000ms
auto_failover=true
auto_rejoin=true

[HA-RWS]
type=service
router=readwritesplit
servers=server1,server2,server3,server4,server5,server6
user=${MARIADB_USER}
password=${MARIADB_PASS}
enable_root_user=true

[HA-port]
type=listener
service=HA-RWS
protocol=MariaDBClient
port=4000
ssl=false
EOF
}

# ── Replication setup ─────────────────────────────────────
# Idempotent: skips if replication already running

_setup_replication() {
  echo "  [REPL] Setting up GTID replication..."

  local ip1 ip2 ip3 ip4 ip5 ip6
  ip1=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' mariadb1 2>/dev/null || echo "")
  ip2=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' mariadb2 2>/dev/null || echo "")
  ip3=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' mariadb3 2>/dev/null || echo "")
  ip4=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' mariadb4 2>/dev/null || echo "")
  ip5=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' mariadb5 2>/dev/null || echo "")
  ip6=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' mariadb6 2>/dev/null || echo "")

  _mariadb_exec() {
    local port="$1" sql="$2"
    mariadb -h 127.0.0.1 -P "${port}" -u root -p"${MARIADB_PASS}" --skip-ssl -e "${sql}" 2>/dev/null
  }

  _replication_running() {
    local port="$1"
    local status
    status=$(_mariadb_exec "${port}" "SHOW SLAVE STATUS\G" 2>/dev/null)
    echo "${status}" | grep -q "Slave_IO_Running: Yes" && \
    echo "${status}" | grep -q "Slave_SQL_Running: Yes"
  }

  # DC1: mariadb1 → mariadb2, mariadb3
  _mariadb_exec 3307 "CREATE USER IF NOT EXISTS 'repl'@'%' IDENTIFIED BY 'repl123'; GRANT REPLICATION SLAVE ON *.* TO 'repl'@'%'; FLUSH PRIVILEGES;"
  for port in 3308 3309; do
    if _replication_running "${port}"; then
      echo "  [SKIP] Replication already running on :${port}"
      continue
    fi
    _mariadb_exec "${port}" "STOP SLAVE;"
    _mariadb_exec "${port}" "CHANGE MASTER TO MASTER_HOST='${ip1}', MASTER_PORT=3306, MASTER_USER='repl', MASTER_PASSWORD='repl123', MASTER_USE_GTID=slave_pos; START SLAVE;"
    echo "  [OK] Replication set up on :${port} → mariadb1"
  done

  # DC2: mariadb1 → mariadb4, mariadb5, mariadb6 (cross-DC DR replicas)
  # Replication user already created on mariadb1 above
  for port in 3310 3311 3312; do
    if _replication_running "${port}"; then
      echo "  [SKIP] Replication already running on :${port}"
      continue
    fi
    _mariadb_exec "${port}" "STOP SLAVE;"
    _mariadb_exec "${port}" "CHANGE MASTER TO MASTER_HOST='${ip1}', MASTER_PORT=3306, MASTER_USER='repl', MASTER_PASSWORD='repl123', MASTER_USE_GTID=slave_pos; START SLAVE;"
    echo "  [OK] Replication set up on :${port} → mariadb1 (DC1 master)"
  done
}

# ── Dispatch ──────────────────────────────────────────────

case "${1:-}" in
  start)  cmd_start  ;;
  stop)   cmd_stop   ;;
  status) cmd_status ;;
  *)
    echo "Usage: $0 {start|stop|status}" >&2
    exit 1
    ;;
esac