Tenormis's Blog

Appearance

Docker Compose 部署 RabbitMQ 高可用集群 ​

1. RabbitMQ 集群 ​

1.1. 经典队列（镜像模式） ​

Tip：从 RabbitMQ 4.0 开始移除了镜像模式。

1.1.1. .env ​

Properties
RABBITMQ_VERSION=3.13.7-management

1.1.2. compose.yaml ​

YAML
name: rabbitmq-cluster

services:
  rabbitmq1:
    image: rabbitmq:${RABBITMQ_VERSION}
    container_name: rabbitmq1
    hostname: rabbitmq1
    networks:
      rabbitmq-cluster:
    ports:
      - "5672:5672"
      - "15672:15672"
    volumes:
      - "./rabbitmq1/data:/var/lib/rabbitmq"
      - "./rabbitmq1/conf/rabbitmq.conf:/etc/rabbitmq/rabbitmq.conf"
      - "./rabbitmq1/conf/definitions.json:/etc/rabbitmq/definitions.json"
      - "plugins:/opt/rabbitmq/plugins"
    restart: on-failure:3
    healthcheck:
      test: ["CMD", "rabbitmq-diagnostics", "check_local_alarms"]
      interval: 5s
      timeout: 10s
      retries: 10
      start_period: 20s

  rabbitmq2:
    image: rabbitmq:${RABBITMQ_VERSION}
    container_name: rabbitmq2
    hostname: rabbitmq2
    networks:
      rabbitmq-cluster:
    ports:
      - "5673:5672"
      - "15673:15672"
    volumes:
      - "./rabbitmq2/data:/var/lib/rabbitmq"
      - "./rabbitmq2/conf/rabbitmq.conf:/etc/rabbitmq/rabbitmq.conf"
      - "./rabbitmq2/conf/definitions.json:/etc/rabbitmq/definitions.json"
      - "plugins:/opt/rabbitmq/plugins"
    restart: on-failure:3
    healthcheck:
      test: ["CMD", "rabbitmq-diagnostics", "check_local_alarms"]
      interval: 5s
      timeout: 10s
      retries: 10
      start_period: 20s

  rabbitmq3:
    image: rabbitmq:${RABBITMQ_VERSION}
    container_name: rabbitmq3
    hostname: rabbitmq3
    networks:
      rabbitmq-cluster:
    ports:
      - "5674:5672"
      - "15674:15672"
    volumes:
      - "./rabbitmq3/data:/var/lib/rabbitmq"
      - "./rabbitmq3/conf/rabbitmq.conf:/etc/rabbitmq/rabbitmq.conf"
      - "./rabbitmq3/conf/definitions.json:/etc/rabbitmq/definitions.json"
      - "plugins:/opt/rabbitmq/plugins"
    restart: on-failure:3
    healthcheck:
      test: ["CMD", "rabbitmq-diagnostics", "check_local_alarms"]
      interval: 5s
      timeout: 10s
      retries: 10
      start_period: 20s

volumes:
  plugins:

networks:
  rabbitmq-cluster:

1.1.3. generate-config.sh ​

Bash
#!/bin/bash
set -eo pipefail

PARSED=$(getopt -o A:H: --long admin-password:,healthcheck-password: -n "${0##*/}" -- "$@") || {
  echo "Failed to parse options." >&2
  exit 1
}

eval set -- "$PARSED"

admin_password=""
healthcheck_password=""

while [ "$1" != "--" ]; do
  case "$1" in
    -A|--admin-password) admin_password="$2"; shift 2 ;;
    -H|--healthcheck-password) healthcheck_password="$2"; shift 2 ;;
    *) echo "Invalid option: $1" >&2; exit 1 ;;
  esac
done
shift # 跳过 --

[ -z "$admin_password" ] && { echo "'-A|--admin-password' option is required" >&2; exit 1; }
[ -z "$healthcheck_password" ] && { echo "'-H|--healthcheck-password' option is required" >&2; exit 1; }

# 导入环境变量
echo "Sourcing environments ..."
set -a
source .env
set +a

# RabbitMQ Hash Password
echo "Hashing passwords ..."
{
  read admin_password_hash
  read healthcheck_password_hash
} < <(docker run --rm rabbitmq:${RABBITMQ_VERSION} /bin/bash -c "$(cat << EOF
rabbitmqctl hash_password ${admin_password} | tail -n 1
rabbitmqctl hash_password ${healthcheck_password} | tail -n 1
EOF
)")

# 配置文件生成
for idx in $(seq 1 3); do
  echo "Creating RabbitMQ node ${idx} configuration files"

  mkdir -p rabbitmq${idx}/conf
  touch rabbitmq${idx}/conf/rabbitmq.conf
  touch rabbitmq${idx}/conf/definitions.json

  cat << EOF > rabbitmq${idx}/conf/rabbitmq.conf
definitions.import_backend = local_filesystem
definitions.local.path = /etc/rabbitmq/definitions.json
definitions.skip_if_unchanged = true
password_hashing_module = rabbit_password_hashing_sha256
cluster_partition_handling = pause_minority
EOF

  cat << EOF > rabbitmq${idx}/conf/definitions.json
{
  "vhosts": [
    {
      "name": "/",
      "description": "Default virtual host",
      "tags": [],
      "metadata": {
        "description": "Default virtual host",
        "tags": []
      }
    }
  ],
  "users": [
    {
      "name": "admin",
      "hashing_algorithm": "rabbit_password_hashing_sha256",
      "password_hash": "${admin_password_hash}",
      "tags": ["administrator"]
    },
    {
      "name": "healthcheck",
      "hashing_algorithm": "rabbit_password_hashing_sha256",
      "password_hash": "${healthcheck_password_hash}",
      "tags": ["monitoring"]
    }
  ],
  "permissions": [
    {
      "user": "admin",
      "vhost": "/",
      "configure": ".*",
      "write": ".*",
      "read": ".*"
    }
  ],
  "policies": [
    {
      "vhost": "/",
      "name": "ha-e2",
      "pattern": "^ha\\\\.",
      "apply-to": "all",
      "definition": {
        "ha-mode": "exactly",
        "ha-params": 2
      },
      "priority": 0
    }
  ]
}
EOF
done

echo "Configuration generated successfully."

1.1.4. build.sh ​

Bash
#!/bin/bash
set -eo pipefail

# 导入环境变量
echo "Sourcing environments ..."
set -a
source .env
set +a

wait_for_healthy() {
  for container in ${@:-$(docker compose ps --format '{{.Name}}' 2>/dev/null)}; do
    until [ "$(docker inspect -f '{{.State.Health.Status}}' "$container")" == "healthy" ]; do
      echo "Waiting for $container container to be healthy ..."
      sleep 1
    done

    # Or:
    # docker exec ${container} rabbitmqctl await_startup
  done
}

# 启动所有实例
echo "Staring services ..."
docker compose up -d

wait_for_healthy

# 停止 rabbitmq2 rabbitmq3
echo "Stopping rabbitmq2 rabbitmq3 ..."
docker compose stop rabbitmq2 rabbitmq3

# 复制 Erlang Cookie
echo "Copying erlang cookie ..."
cp -f ./rabbitmq1/data/.erlang.cookie ./rabbitmq2/data/.erlang.cookie
cp -f ./rabbitmq1/data/.erlang.cookie ./rabbitmq3/data/.erlang.cookie

# 启动 rabbitmq2 rabbitmq3
echo "Staring rabbitmq2 rabbitmq3 ..."
docker compose start rabbitmq2 rabbitmq3

wait_for_healthy

# 搭建集群
echo "Building cluster ..."
docker compose exec -T rabbitmq2 bash -c "rabbitmqctl stop_app && rabbitmqctl join_cluster rabbit@rabbitmq1 && rabbitmqctl start_app"
docker compose exec -T rabbitmq3 bash -c "rabbitmqctl stop_app && rabbitmqctl join_cluster rabbit@rabbitmq1 && rabbitmqctl start_app"

echo "Built successful."

1.1.5. 搭建集群 ​

Bash
$ # 生成 RabbitMQ 配置文件
  sudo ./generate-config.sh -A b8ekRBWp60CsefKvWpcn -H fQPzbPuxDxJmsfbfxxP0 && \
  # 搭建集群
  sudo ./build.sh

1.2. 仲裁队列 ​

1.2.1. .env ​

Properties
RABBITMQ_VERSION=4.0.5-management

1.2.2. compose.yaml ​

同经典队列 compose.yaml。

1.2.3. generate-config.sh ​

Bash
#!/bin/bash
set -eo pipefail

PARSED=$(getopt -o A:H: --long admin-password:,healthcheck-password: -n "${0##*/}" -- "$@") || {
  echo "Failed to parse options." >&2
  exit 1
}

eval set -- "$PARSED"

admin_password=""
healthcheck_password=""

while [ "$1" != "--" ]; do
  case "$1" in
    -A|--admin-password) admin_password="$2"; shift 2 ;;
    -H|--healthcheck-password) healthcheck_password="$2"; shift 2 ;;
    *) echo "Invalid option: $1" >&2; exit 1 ;;
  esac
done
shift # 跳过 --

[ -z "$admin_password" ] && { echo "'-A|--admin-password' option is required" >&2; exit 1; }
[ -z "$healthcheck_password" ] && { echo "'-H|--healthcheck-password' option is required" >&2; exit 1; }

# 导入环境变量
echo "Sourcing environments ..."
set -a
source .env
set +a

# RabbitMQ Hash Password
echo "Hashing passwords ..."
{
  read admin_password_hash
  read healthcheck_password_hash
} < <(docker run --rm rabbitmq:${RABBITMQ_VERSION} /bin/bash -c "$(cat << EOF
rabbitmqctl hash_password ${admin_password} | tail -n 1
rabbitmqctl hash_password ${healthcheck_password} | tail -n 1
EOF
)")

# 配置文件生成
for idx in $(seq 1 3); do
  echo "Creating RabbitMQ node ${idx} configuration files"

  mkdir -p rabbitmq${idx}/conf
  touch rabbitmq${idx}/conf/rabbitmq.conf
  touch rabbitmq${idx}/conf/definitions.json

  cat << EOF > rabbitmq${idx}/conf/rabbitmq.conf
definitions.import_backend = local_filesystem
definitions.local.path = /etc/rabbitmq/definitions.json
definitions.skip_if_unchanged = true
password_hashing_module = rabbit_password_hashing_sha256
EOF

  cat << EOF > rabbitmq${idx}/conf/definitions.json
{
  "vhosts": [
    {
      "name": "/",
      "description": "Default virtual host",
      "tags": [],
      "default_queue_type": "quorum",
      "metadata": {
        "description": "Default virtual host",
        "tags": [],
        "default_queue_type": "quorum"
      }
    }
  ],
  "users": [
    {
      "name": "admin",
      "hashing_algorithm": "rabbit_password_hashing_sha256",
      "password_hash": "${admin_password_hash}",
      "tags": ["administrator"]
    },
    {
      "name": "healthcheck",
      "hashing_algorithm": "rabbit_password_hashing_sha256",
      "password_hash": "${healthcheck_password_hash}",
      "tags": ["monitoring"]
    }
  ],
  "permissions": [
    {
      "user": "admin",
      "vhost": "/",
      "configure": ".*",
      "write": ".*",
      "read": ".*"
    }
  ]
}
EOF
done

echo "Configuration generated successfully."
差异对比
diff
 ...
 definitions.local.path = /etc/rabbitmq/definitions.json
 definitions.skip_if_unchanged = true
 password_hashing_module = rabbit_password_hashing_sha256
-cluster_partition_handling = pause_minority
 EOF

   cat << EOF > rabbitmq${idx}/conf/definitions.json
 ...
       "name": "/",
       "description": "Default virtual host",
       "tags": [],
+      "default_queue_type": "quorum",
       "metadata": {
         "description": "Default virtual host",
-        "tags": []
+        "tags": [],
+        "default_queue_type": "quorum"
       }
     }
   ],
 ...
       "write": ".*",
       "read": ".*"
     }
-  ],
-  "policies": [
-    {
-      "vhost": "/",
-      "name": "ha-e2",
-      "pattern": "^ha\\\\.",
-      "apply-to": "all",
-      "definition": {
-        "ha-mode": "exactly",
-        "ha-params": 2
-      },
-      "priority": 0
-    }
   ]
 }
 EOF
 ...

1.2.4. build.sh ​

同经典队列 build.sh。

1.2.5. 搭建集群 ​

同经典队列搭建集群。

2. HAProxy 负载均衡 ​

2.1. .env ​

在原 .env 中添加以下配置项：

Properties
HAPROXY_VERSION=3.1.3

2.2. compose.yaml ​

YAML
name: rabbitmq-cluster

services:
  rabbitmq1:
    image: rabbitmq:${RABBITMQ_VERSION}
    container_name: rabbitmq1
    hostname: rabbitmq1
    networks:
      rabbitmq-cluster:
    volumes:
      - "./rabbitmq1/data:/var/lib/rabbitmq"
      - "./rabbitmq1/conf/rabbitmq.conf:/etc/rabbitmq/rabbitmq.conf"
      - "./rabbitmq1/conf/definitions.json:/etc/rabbitmq/definitions.json"
      - "plugins:/opt/rabbitmq/plugins"
    restart: on-failure:3
    healthcheck:
      test: ["CMD", "rabbitmq-diagnostics", "check_local_alarms"]
      interval: 5s
      timeout: 10s
      retries: 10
      start_period: 20s

  rabbitmq2:
    image: rabbitmq:${RABBITMQ_VERSION}
    container_name: rabbitmq2
    hostname: rabbitmq2
    networks:
      rabbitmq-cluster:
    volumes:
      - "./rabbitmq2/data:/var/lib/rabbitmq"
      - "./rabbitmq2/conf/rabbitmq.conf:/etc/rabbitmq/rabbitmq.conf"
      - "./rabbitmq2/conf/definitions.json:/etc/rabbitmq/definitions.json"
      - "plugins:/opt/rabbitmq/plugins"
    restart: on-failure:3
    healthcheck:
      test: ["CMD", "rabbitmq-diagnostics", "check_local_alarms"]
      interval: 5s
      timeout: 10s
      retries: 10
      start_period: 20s

  rabbitmq3:
    image: rabbitmq:${RABBITMQ_VERSION}
    container_name: rabbitmq3
    hostname: rabbitmq3
    networks:
      rabbitmq-cluster:
    volumes:
      - "./rabbitmq3/data:/var/lib/rabbitmq"
      - "./rabbitmq3/conf/rabbitmq.conf:/etc/rabbitmq/rabbitmq.conf"
      - "./rabbitmq3/conf/definitions.json:/etc/rabbitmq/definitions.json"
      - "plugins:/opt/rabbitmq/plugins"
    restart: on-failure:3
    healthcheck:
      test: ["CMD", "rabbitmq-diagnostics", "check_local_alarms"]
      interval: 5s
      timeout: 10s
      retries: 10
      start_period: 20s

  rabbitmq-haproxy1:
    image: haproxy:${HAPROXY_VERSION}
    container_name: rabbitmq-haproxy1
    networks:
      rabbitmq-cluster:
    ports:
      - "1936:80"
      - "5672:5672"
      - "15672:15672"
    volumes:
      - "./rabbitmq-haproxy1/conf:/usr/local/etc/haproxy"
    sysctls:
      - net.ipv4.ip_unprivileged_port_start=0
    restart: on-failure:3
    depends_on:
      - rabbitmq1
      - rabbitmq2
      - rabbitmq3

  rabbitmq-haproxy2:
    image: haproxy:${HAPROXY_VERSION}
    container_name: rabbitmq-haproxy2
    networks:
      rabbitmq-cluster:
    ports:
      - "1937:80"
      - "5673:5672"
      - "15673:15672"
    volumes:
      - "./rabbitmq-haproxy2/conf:/usr/local/etc/haproxy"
    sysctls:
      - net.ipv4.ip_unprivileged_port_start=0
    restart: on-failure:3
    depends_on:
      - rabbitmq1
      - rabbitmq2
      - rabbitmq3

volumes:
  plugins:

networks:
  rabbitmq-cluster:
差异对比
diff
 ...
     hostname: rabbitmq1
     networks:
       rabbitmq-cluster:
-    ports:
-      - "5672:5672"
-      - "15672:15672"
     volumes:
       - "./rabbitmq1/data:/var/lib/rabbitmq"
       - "./rabbitmq1/conf/rabbitmq.conf:/etc/rabbitmq/rabbitmq.conf"
 ...
     hostname: rabbitmq2
     networks:
       rabbitmq-cluster:
-    ports:
-      - "5673:5672"
-      - "15673:15672"
     volumes:
       - "./rabbitmq2/data:/var/lib/rabbitmq"
       - "./rabbitmq2/conf/rabbitmq.conf:/etc/rabbitmq/rabbitmq.conf"
 ...
     hostname: rabbitmq3
     networks:
       rabbitmq-cluster:
-    ports:
-      - "5674:5672"
-      - "15674:15672"
     volumes:
       - "./rabbitmq3/data:/var/lib/rabbitmq"
       - "./rabbitmq3/conf/rabbitmq.conf:/etc/rabbitmq/rabbitmq.conf"
 ...
       retries: 10
       start_period: 20s

+  rabbitmq-haproxy1:
+    image: haproxy:${HAPROXY_VERSION}
+    container_name: rabbitmq-haproxy1
+    networks:
+      rabbitmq-cluster:
+    ports:
+      - "1936:80"
+      - "5672:5672"
+      - "15672:15672"
+    volumes:
+      - "./rabbitmq-haproxy1/conf:/usr/local/etc/haproxy"
+    sysctls:
+      - net.ipv4.ip_unprivileged_port_start=0
+    restart: on-failure:3
+    depends_on:
+      - rabbitmq1
+      - rabbitmq2
+      - rabbitmq3
+
+  rabbitmq-haproxy2:
+    image: haproxy:${HAPROXY_VERSION}
+    container_name: rabbitmq-haproxy2
+    networks:
+      rabbitmq-cluster:
+    ports:
+      - "1937:80"
+      - "5673:5672"
+      - "15673:15672"
+    volumes:
+      - "./rabbitmq-haproxy2/conf:/usr/local/etc/haproxy"
+    sysctls:
+      - net.ipv4.ip_unprivileged_port_start=0
+    restart: on-failure:3
+    depends_on:
+      - rabbitmq1
+      - rabbitmq2
+      - rabbitmq3
+
 volumes:
   plugins:

 ...

2.3. generate-haproxy-config.sh ​

Bash
#!/bin/bash
set -eo pipefail

PARSED=$(getopt -o S:H: --long stats-password:,healthcheck-password: -n "${0##*/}" -- "$@") || {
  echo "Failed to parse options." >&2
  exit 1
}

eval set -- "$PARSED"

stats_password=""
healthcheck_password=""

while [ "$1" != "--" ]; do
  case "$1" in
    -S|--stats-password) stats_password="$2"; shift 2 ;;
    -H|--healthcheck-password) healthcheck_password="$2"; shift 2 ;;
    *) echo "Invalid option: $1" >&2; exit 1 ;;
  esac
done
shift # 跳过 --

[ -z "$stats_password" ] && { echo "'-S|--stats-password' option is required" >&2; exit 1; }
[ -z "$healthcheck_password" ] && { echo "'-H|--healthcheck-password' option is required" >&2; exit 1; }

healthcheck_basic_auth=$(echo -n "healthcheck:${healthcheck_password}" | base64)

# HAProxy 配置文件生成
for idx in $(seq 1 2); do
  echo "Creating HAProxy node ${idx} configuration files"

  mkdir -p rabbitmq-haproxy${idx}/conf
  touch rabbitmq-haproxy${idx}/conf/haproxy.cfg

  cat << EOF > rabbitmq-haproxy${idx}/conf/haproxy.cfg
global
    log /dev/log local0 notice
    stats timeout 30s
    user haproxy
    group haproxy

defaults
    log global
    option dontlognull
    timeout connect 5000
    timeout client 50000
    timeout server 50000

frontend rabbitmq_frontend
    bind *:5672
    mode tcp
    option tcplog
    default_backend rabbitmq_backend

backend rabbitmq_backend
    mode tcp
    balance roundrobin
    option httpchk
    http-check send meth GET uri /api/health/checks/local-alarms hdr Authorization "Basic ${healthcheck_basic_auth}"
    http-check expect status 200
    server rabbitmq1 rabbitmq1:5672 check port 15672
    server rabbitmq2 rabbitmq2:5672 check port 15672
    server rabbitmq3 rabbitmq3:5672 check port 15672

frontend rabbitmq_management_frontend
    bind *:15672
    mode http
    default_backend rabbitmq_management_backend

backend rabbitmq_management_backend
    mode http
    balance leastconn
    server rabbitmq1 rabbitmq1:15672 check
    server rabbitmq2 rabbitmq2:15672 check
    server rabbitmq3 rabbitmq3:15672 check

listen stats
    bind *:80
    mode http
    stats enable
    stats uri /stats
    stats auth admin:${stats_password}
EOF
done

echo "Configuration generated successfully."

2.4. build.sh ​

Bash
#!/bin/bash
set -eo pipefail

# 导入环境变量
echo "Sourcing environments ..."
set -a
source .env
set +a

wait_for_healthy() {
  for container in ${@:-$(docker compose ps --format '{{.Name}}' 2>/dev/null)}; do
    until [ "$(docker inspect -f '{{.State.Health.Status}}' "$container")" == "healthy" ]; do
      echo "Waiting for $container container to be healthy ..."
      sleep 1
    done

    # Or:
    # docker exec ${container} rabbitmqctl await_startup
  done
}

# 启动 RabbitMQ
echo "Staring rabbitmq1 rabbitmq2 rabbitmq3 ..."
docker compose up -d rabbitmq1 rabbitmq2 rabbitmq3

wait_for_healthy

# 停止 rabbitmq2 rabbitmq3
echo "Stopping rabbitmq2 rabbitmq3 ..."
docker compose stop rabbitmq2 rabbitmq3

# 复制 Erlang Cookie
echo "Copying erlang cookie ..."
cp -f ./rabbitmq1/data/.erlang.cookie ./rabbitmq2/data/.erlang.cookie
cp -f ./rabbitmq1/data/.erlang.cookie ./rabbitmq3/data/.erlang.cookie

# 启动 rabbitmq2 rabbitmq3
echo "Staring rabbitmq2 rabbitmq3 ..."
docker compose start rabbitmq2 rabbitmq3

wait_for_healthy

# 搭建集群
echo "Building cluster ..."
docker compose exec -T rabbitmq2 bash -c "rabbitmqctl stop_app && rabbitmqctl join_cluster rabbit@rabbitmq1 && rabbitmqctl start_app"
docker compose exec -T rabbitmq3 bash -c "rabbitmqctl stop_app && rabbitmqctl join_cluster rabbit@rabbitmq1 && rabbitmqctl start_app"

# 启动 HAProxy
echo "Staring HAProxy ..."
docker compose up -d rabbitmq-haproxy1 rabbitmq-haproxy2

echo "Built successful."
差异对比
diff
 ...
   done
 }

-# 启动所有实例
-echo "Staring services ..."
-docker compose up -d
+# 启动 RabbitMQ
+echo "Staring rabbitmq1 rabbitmq2 rabbitmq3 ..."
+docker compose up -d rabbitmq1 rabbitmq2 rabbitmq3

 wait_for_healthy

 ...
 docker compose exec -T rabbitmq2 bash -c "rabbitmqctl stop_app && rabbitmqctl join_cluster rabbit@rabbitmq1 && rabbitmqctl start_app"
 docker compose exec -T rabbitmq3 bash -c "rabbitmqctl stop_app && rabbitmqctl join_cluster rabbit@rabbitmq1 && rabbitmqctl start_app"

+# 启动 HAProxy
+echo "Staring HAProxy ..."
+docker compose up -d rabbitmq-haproxy1 rabbitmq-haproxy2
+
 echo "Built successful."

2.5. 搭建集群 ​

Bash
$ # 生成 RabbitMQ 配置文件
  sudo ./generate-config.sh -A b8ekRBWp60CsefKvWpcn -H fQPzbPuxDxJmsfbfxxP0 && \
  # 生成 HAProxy 配置文件
  sudo ./generate-haproxy-config.sh -S 7y3yrBbgx2jnNq1KK6X1 -H fQPzbPuxDxJmsfbfxxP0 && \
  # 搭建集群
  sudo ./build.sh
差异对比
diff
 $ # 生成 RabbitMQ 配置文件
   sudo ./generate-config.sh -A b8ekRBWp60CsefKvWpcn -H fQPzbPuxDxJmsfbfxxP0 && \
+  # 生成 HAProxy 配置文件
+  sudo ./generate-haproxy-config.sh -S 7y3yrBbgx2jnNq1KK6X1 -H fQPzbPuxDxJmsfbfxxP0 && \
   # 搭建集群
   sudo ./build.sh

转载请注明作者和出处

Copyright © 2025-至今 Tenormis