#!/bin/bash
#
# Performs full and incremental hot backups of a MariaDB Galera Cluster node
# using mariadb-backup. Intended to be run on a single designated node via
# the mysql user's crontab.
#
# The script automatically decides whether to take a full or incremental
# backup based on FULL_BACKUP_CYCLE. If the latest full backup is older than
# FULL_BACKUP_CYCLE seconds, a new full backup is taken. Otherwise an
# incremental backup is created against the most recent full (or incremental).
#
# Old backups are pruned automatically based on FULL_BACKUP_CYCLE and KEEP.
#
# Usage: backup_galera_cluster_full.sh
#   No arguments. Configure via the variables at the top of this script.

# --- Configuration -----------------------------------------------------------

readonly MYSQL_USER='mariabackup'
readonly MYSQL_PASSWORD='your_secure_password'

readonly BACK_CMD='mariadb-backup'
readonly GZIP_CMD='gzip'
readonly STREAM_CMD='mbstream'

readonly BACK_DIR='/mariadb_backup'
readonly FULL_BACKUP_CYCLE=86400 # seconds between full backups (86400 = 1 day)
readonly KEEP=3                  # number of full backup cycles to retain

readonly LOCK_DIR='/tmp/mariabackup.lock'

# --- Functions ---------------------------------------------------------------

#######################################
# Print an error message to STDERR with a timestamp.
# Arguments:
#   Message string(s) to print.
# Outputs:
#   Writes timestamped message to STDERR.
#######################################
err() {
  echo "[$(date +'%Y-%m-%dT%H:%M:%S%z')]: $*" >&2
}

#######################################
# Release the lock directory and exit with the given code.
# Globals:
#   LOCK_DIR
# Arguments:
#   Exit code (integer).
#######################################
release_lock_and_exit() {
  local -r exit_code="$1"

  if rmdir "${LOCK_DIR}"; then
    echo 'Lock directory removed.'
  else
    err 'Could not remove lock directory.'
  fi
  exit "${exit_code}"
}

#######################################
# Acquire the lock directory or exit if another instance is running.
# Globals:
#   LOCK_DIR
# Arguments:
#   None
#######################################
get_lock_or_die() {
  if mkdir "${LOCK_DIR}"; then
    echo 'Lock acquired.'
  else
    err "Could not create lock directory: ${LOCK_DIR}"
    err 'Is another backup already running?'
    exit 1
  fi
}

#######################################
# Verify that all required external commands are available in PATH.
# Arguments:
#   None
# Returns:
#   Exits with code 1 if any required command is not found.
#######################################
check_dependencies() {
  local -r required_cmds=('mariadb-backup' 'mbstream' 'gzip' 'mariadb' 'mariadb-admin')
  local cmd

  for cmd in "${required_cmds[@]}"; do
    if ! command -v "${cmd}" >/dev/null 2>&1; then
      err "Required command not found: ${cmd}"
      err "Ensure it is installed and available in PATH."
      exit 1
    fi
  done
}

#######################################
# Verify that MariaDB is reachable with the configured credentials.
# Connects via the local unix socket; no TCP host or port is used.
# Globals:
#   MYSQL_USER, MYSQL_PASSWORD
# Arguments:
#   None
# Returns:
#   Exits with code 1 if MariaDB is not reachable or credentials are invalid.
#######################################
check_mariadb_connection() {
  local -r user_options=(
    "--user=${MYSQL_USER}"
    "--password=${MYSQL_PASSWORD}"
  )

  if [[ -z "$(mariadb-admin "${user_options[@]}" status | grep 'Uptime')" ]]; then
    err 'HALTED: MariaDB does not appear to be running.'
    exit 1
  fi

  if ! echo 'exit' | mariadb -s "${user_options[@]}"; then
    err 'HALTED: Supplied credentials appear to be incorrect (see script configuration).'
    exit 1
  fi
}

#######################################
# Ensure that a directory exists and is writable, creating it if necessary.
# Arguments:
#   Path to the directory.
# Returns:
#   Exits with code 1 if the directory cannot be created or is not writable.
#######################################
ensure_dir() {
  local -r dir="$1"

  if [[ ! -d "${dir}" ]]; then
    mkdir -p "${dir}"
  fi

  if [[ ! -d "${dir}" || ! -w "${dir}" ]]; then
    err "${dir} does not exist or is not writable."
    exit 1
  fi
}

#######################################
# Run a full backup of the MariaDB instance.
# Globals:
#   BACK_CMD, GZIP_CMD, STREAM_CMD, BACK_DIR, MYSQL_USER, MYSQL_PASSWORD
# Arguments:
#   None
#######################################
run_full_backup() {
  local target_dir
  target_dir="${BACK_DIR}/base/$(date +%F_%H-%M-%S)"
  mkdir -p "${target_dir}"

  local -r user_options=(
    "--user=${MYSQL_USER}"
    "--password=${MYSQL_PASSWORD}"
  )

  echo "Starting full backup to: ${target_dir}"

  "${BACK_CMD}" --backup \
    "${user_options[@]}" \
    --galera-info \
    --extra-lsndir="${target_dir}" \
    --stream="${STREAM_CMD}" |
    "${GZIP_CMD}" >"${target_dir}/backup.stream.gz"

  local -r pipe_status=("${PIPESTATUS[@]}")
  if ((pipe_status[0] != 0 || pipe_status[1] != 0)); then
    err "Full backup failed (mariadb-backup: ${pipe_status[0]}, gzip: ${pipe_status[1]})."
    rm -rf "${target_dir}"
    release_lock_and_exit 1
  fi

  echo "Full backup completed: ${target_dir}"
}

#######################################
# Run an incremental backup against the most recent base or incremental backup.
# Globals:
#   BACK_CMD, GZIP_CMD, STREAM_CMD, BACK_DIR, MYSQL_USER, MYSQL_PASSWORD
# Arguments:
#   latest_base: directory name of the latest full backup (not a full path).
#######################################
run_incremental_backup() {
  local -r latest_base="$1"
  local -r incr_base_dir="${BACK_DIR}/incr/${latest_base}"

  ensure_dir "${incr_base_dir}"

  # Chain incrementals: base against the latest incremental, or the full if
  # this is the first incremental in this cycle.
  local incr_source_dir
  local latest_incr
  latest_incr="$(find "${incr_base_dir}" -mindepth 1 -maxdepth 1 -type d |
    sort -r |
    head -1)"

  if [[ -n "${latest_incr}" ]]; then
    incr_source_dir="${latest_incr}"
  else
    incr_source_dir="${BACK_DIR}/base/${latest_base}"
  fi

  local target_dir
  target_dir="${incr_base_dir}/$(date +%F_%H-%M-%S)"
  mkdir -p "${target_dir}"

  local -r user_options=(
    "--user=${MYSQL_USER}"
    "--password=${MYSQL_PASSWORD}"
  )

  echo "Starting incremental backup to: ${target_dir}"
  echo "Incremental base: ${incr_source_dir}"

  "${BACK_CMD}" --backup \
    "${user_options[@]}" \
    --galera-info \
    --extra-lsndir="${target_dir}" \
    --incremental-basedir="${incr_source_dir}" \
    --stream="${STREAM_CMD}" |
    "${GZIP_CMD}" >"${target_dir}/backup.stream.gz"

  local -r pipe_status=("${PIPESTATUS[@]}")
  if ((pipe_status[0] != 0 || pipe_status[1] != 0)); then
    err "Incremental backup failed (mariadb-backup: ${pipe_status[0]}, gzip: ${pipe_status[1]})."
    rm -rf "${target_dir}"
    release_lock_and_exit 1
  fi

  echo "Incremental backup completed: ${target_dir}"
}

#######################################
# Delete full backups (and their incrementals) older than the retention window.
# Globals:
#   BACK_DIR, FULL_BACKUP_CYCLE, KEEP
# Arguments:
#   None
#######################################
prune_old_backups() {
  local -r retention_mins=$((FULL_BACKUP_CYCLE * (KEEP + 1) / 60))
  echo "Pruning backups older than ${retention_mins} minutes."

  local del
  while IFS= read -r del; do
    echo "Deleting old backup: ${del}"
    rm -rf "${BACK_DIR}/base/${del}"
    rm -rf "${BACK_DIR}/incr/${del}"
  done < <(find "${BACK_DIR}/base" \
    -mindepth 1 -maxdepth 1 \
    -type d \
    -mmin "+${retention_mins}" \
    -printf '%P\n')
}

#######################################
# Main entry point.
#######################################
main() {
  local -r start_time="$(date +%s)"

  local -r base_back_dir="${BACK_DIR}/base"
  local -r incr_back_dir="${BACK_DIR}/incr"

  check_dependencies

  ensure_dir "${base_back_dir}"
  ensure_dir "${incr_back_dir}"

  check_mariadb_connection

  get_lock_or_die
  # Release lock on unexpected exits (e.g. kill signal from cron timeout).
  trap 'err "Interrupted. Releasing lock."; rmdir "${LOCK_DIR}" 2>/dev/null; exit 1' \
    INT TERM

  # Determine whether to take a full or incremental backup.
  local latest
  latest="$(find "${base_back_dir}" -mindepth 1 -maxdepth 1 -type d \
    -printf '%P\n' |
    sort -r |
    head -1)"

  local backup_age=0
  if [[ -n "${latest}" ]]; then
    backup_age="$(stat -c %Y "${base_back_dir}/${latest}/backup.stream.gz" \
      2>/dev/null || echo 0)"
  fi

  if [[ -n "${latest}" ]] &&
    ((backup_age + FULL_BACKUP_CYCLE + 5 >= start_time)); then
    run_incremental_backup "${latest}"
  else
    run_full_backup
  fi

  prune_old_backups

  local -r elapsed=$(($(date +%s) - start_time))
  echo
  echo "Backup completed in ${elapsed} seconds at $(date)."

  release_lock_and_exit 0
}

main "$@"