PostgreSQL多节点部署分布式数据库之Citus
1. citus的架构
主要资源信息：
SCNP_prod_olap_manage_01 10.133.165.29 root / 密码
SCNP_prod_olap_manage_02 10.133.165.172 root / 密码
SCNP_prod_olap_data_01 10.133.165.135 root / 密码
SCNP_prod_olap_data_02 10.133.165.80 root / 密码
2.安装postgresSQL数据库，编译安装
1.安装初始化插件
# 安装插件
yum -y install readline readline-devel zlib zlib-devel gettext gettext-devel openssl openssl-devel pam pam-devel libxml2 libxml2-devel libxslt libxslt-devel perl perl-devel tcl-devel uuid-devel gcc gcc-c++ make flex bison perl-ExtUtils* libicu libicu-devel libcurl-devell libevent-devel lz4-devel#创建用户 注：在主机点上创建DBA用户
groupadd postgres -g 4000
useradd postgres -g 4000 -u 4000
#echo "Xhcw@2025"|passwd --stdin postgres#客户那边最限制了，密码长度要大--- echo "postgres:密码" | chpasswd
ALTER USER postgres WITH PASSWORD '密码';#直接使用这个也行
vim /etc/sudoers
postgres ALL=(ALL) NOPASSWD:ALL
echo "postgres ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers
2.挂在数据盘
#格式化磁盘mkfs.ext4 /dev/vdb# 临时挂载
mkdir /postgres
chown -R postgres:postgres /postgres
mount /dev/vdb /postgres# 设置开机自动挂载
# 获取分区 UUID
sudo blkid /dev/vdb/dev/vdb: UUID="3d337f61-6d99-4a4c-8168-d6d62ffa41f5" TYPE="ext4" # 编辑 /etc/fstab
echo "/dev/vdb /postgres ext4 defaults 0 0" | sudo tee -a /etc/fstab
3.创建数据目录
#每一台上都进行如下操作
mkdir /install
chmod -R 777 /install
mkdir -p /postgres/server /postgres/data/citus/24398
chown -R postgres:postgres /postgres /install
chmod 0775 /postgres
chmod 0700 /postgres/data4.进行离线安装
https://v5p9fgxcq6.feishu.cn/docx/KxKedI5QJobA95xFOZvcKQkin05#share-LSArdtKRio8zV4xDxFCcJ9JQnTf
#拉取代码
yum install -y lrzsz bzip2
rz
mv postgresql-15.13.tar.bz2 /install/
cd /install
tar -xvf postgresql-15.13.tar.bz2#编译安装
cd postgresql-15.13
./configure --prefix=/postgres/server --with-openssl
#安装编译文件
make && make install
5.创建软连接
su root
mkdir /postgresql
chmod -R 777 /postgresql
chown -R postgres:postgres /postgresql
cd /postgres
ln -s /postgres/server /postgresql
6.设置环境变量
vi /home/postgres/.bashrc#四个节点都需要一下的配置
export PGHOME=/postgres/server
export PATH=$PGHOME/bin:$PATH
export LD_LIBRARY_PATH=$PGHOME/lib:$LD_LIBRARY_PATH
export PGDATA=/postgres/data/citus/24398
export PGPORT=24398 # 所有节点使用相同端口# 主节点：
export PG_SERVER_NAME=coordinator# 从节点：
export PG_SERVER_NAME=standby# Worker节点1：
export PG_SERVER_NAME=worker01# Worker节点2：
export PG_SERVER_NAME=worker02#启动环境变量
source /home/postgres/.bashrc#四个节点都需要一下的配置
export PGHOME=/postgres/server
export PATH=$PGHOME/bin:$PATH
export LD_LIBRARY_PATH=$PGHOME/lib:$LD_LIBRARY_PATH
export PGDATA=/postgres/data/24398
export PGPORT=24398 # 所有节点使用相同端口# 主节点：
export PG_SERVER_NAME=pg01# 从节点：
export PG_SERVER_NAME=pg027.初始化数据库和修改监听端口
#初始化
su - postgres
/postgres/server/bin/initdb -D /postgres/data/citus/24398#修改家庭端口
echo "PORT=24398">> /postgres/data/citus/24398/postgresql.conf
echo "listen_addresses='*'">> /postgres/data/citus/24398/postgresql.conf
#检查配置是否成功
sudo tail -2 /postgres/data/citus/24398/postgresql.conf----->其他的部署
su - postgres
/postgres/server/bin/initdb -D /postgres/data/24398
echo "PORT=24398">> /postgres/data/24398/postgresql.conf
echo "listen_addresses='*'">> /postgres/data/24398/postgresql.con
sudo tail -2 /postgres/data/24398/postgresql.conf
8.创建日志存储目录和启动服务
#创建日志目录
sudo mkdir -p /postgres/data/citus/24398/log
sudo chown -R postgres:postgres /postgres/data/citus/---->其他的部署
sudo mkdir -p /postgres/data/24398/log
sudo chown -R postgres:postgres /postgres/data/
---->#启动服务
#su - postgres
/postgres/server/bin/pg_ctl start -D /postgres/data/citus/24398 -l /postgres/data/citus/24398/log/cnlog
/postgres/server/bin/pg_ctl start -D /postgres/data/citus/24398 -l /postgres/data/citus/24398/log/standbylog
/postgres/server/bin/pg_ctl start -D /postgres/data/citus/24398 -l /postgres/data/citus/24398/log/work01log
/postgres/server/bin/pg_ctl start -D /postgres/data/citus/24398 -l /postgres/data/citus/24398/log/work02log--->其他的集群配置
/postgres/server/bin/pg_ctl start -D /postgres/data/24398 -l /postgres/data/24398/log/pg01log
/postgres/server/bin/pg_ctl start -D /postgres/data/24398 -l /postgres/data/24398/log/pg02log
--->#检查端口是否起来
netstat -tlnp | grep 24398
3.单一的对postgres数据库进行主从复制实现高可用（这个是有需求在进行部署）
1.在主节点进行设置
1.同步密码
#进入数据库
su - postgres
psql -p 24398设置连接密码
CREATE USER postgres WITH REPLICATION LOGIN ENCRYPTED PASSWORD '密码';
SELECT pg_reload_conf();#修改密码
psql -p 24398
-- 在 psql 中执行
ALTER USER postgres WITH PASSWORD '密码';-- 或者使用 \password 命令
\password postgres#也可以不做，然后之间做免密登录
2.修改必要参数
#在主上进行操作
vim /postgres/data/24398/postgresql.confwal_level = replica
max_wal_senders = 10
wal_keep_size = 1024MB
hot_standby = on
max_replication_slots = 10 # 如果您使用复制槽重启数据库
pg_ctl -D /postgres/data/24398 restart
3.认证加密设置
#如果说做了加密，进行一下操作
# 复制连接
vi /postgres/data/24398/pg_hba.conf
host replication postgres 10.133.165.26/32 md5
host all postgres 127.0.0.1/32 md5然后在主节点建立认证
# 在主节点 (10.133.165.121) 和备节点 (10.133.165.26) 都执行
su - postgres
vim ~/.pgpass
10.133.165.121:24398:replication:replicator:your_password
10.133.165.26:24398:replication:replicator:your_password
localhost:24398:replication:replicator:your_password
127.0.0.1:24398:replication:replicator:your_password
#设置权限
chmod 600 ~/.pgpass#如果说不想使用密码认证使用下面的方式
vi /postgres/data/24398/pg_hba.conf
# 复制连接 - 使用trust免密10.133.165.26
host replication postgres 10.133.165.26/32 trust
host all postgres 127.0.0.1/32 trust#重启数据库
pg_ctl -D /postgres/data/24398 restart
#检查端口是否起来
netstat -tlnp | grep 24398
2.在从节点进行设置
1.停止数据库服务，然后清除数据
#停止服务
/postgres/server/bin/pg_ctl -D /postgres/data/24398 stop -l /postgres/data/24398/log/pg02log#清除数据
mv /postgres/data/24398 /postgres/data/24398_backup
mkdir -p /postgres/data/24398
chown postgres:postgres /postgres/data/24398
chmod 0750 /postgres/data/24398
2.使用pg_basebackup进行基础备份
su - postgres
/postgres/server/bin/pg_basebackup -h 10.133.165.121 -p 24398 -U postgres -D /postgres/data/24398 -Fp -Xs -P -R3.配置从节点postgresql.conf
vim /postgres/data/24398/postgresql.confhot_standby = on
max_standby_streaming_delay = 30s
wal_receiver_status_interval = 10s
hot_standby_feedback = onprimary_conninfo = 'host=10.133.165.121 port=24398 user=postgres application_name=pg02'#启动从库
/postgres/server/bin/pg_ctl -D /postgres/data/24398 start -l /postgres/data/24398/log/pg02log
4.主从同步状态查询
#在主节点检查复制状态
psql -p 24398 -c "SELECT client_addr, state, sync_state FROM pg_stat_replication;"#在从节点检查复制状态
psql -p 24398 -c "SELECT pg_is_in_recovery(), pg_last_wal_receive_lsn(), pg_last_wal_replay_lsn();"
5.验证主从同步
在主节点创建测试数据psql -p 24398 -c "CREATE DATABASE test_replication;"
psql -p 24398 -d test_replication -c "CREATE TABLE test_table (id serial PRIMARY KEY, data text);"
psql -p 24398 -d test_replication -c "INSERT INTO test_table (data) VALUES ('test data');" 在从节点验证数据psql -p 24398 -d test_replication -c "SELECT * FROM test_table;"
6.登录设置
#修改密码
psql -p 24398
-- 在 psql 中执行
ALTER USER postgres WITH PASSWORD '密码';在主节点上进行配置
[postgres@scnp-prod-oltp-01 ~]$ cat ~/.pgpass
10.133.165.121:24398:*:postgres:密码
localhost:24398:*:postgres:密码
127.0.0.1:24398:*:postgres:密码
10.133.165.26:24398:*:postgres:密码#然后在添加配置
vim /postgres/data/24398/pg_hba.conf# 允许本地 IP 连接
host all all 127.0.0.1/32 md5
host all all ::1/128 md5
host all all 10.133.165.121/32 md5# 如果需要，也添加备份服务器的 IP
host all all 10.133.165.26/32 md5
host all all 0.0.0.0/0 md5在备份节点进行配置
[postgres@scnp-prod-oltp-01 ~]$ cat ~/.pgpass
10.133.165.121:24398:*:postgres:密码
localhost:24398:*:postgres:密码
127.0.0.1:24398:*:postgres:密码
10.133.165.26:24398:*:postgres:密码#然后在添加配置
vim /postgres/data/24398/pg_hba.conf# 允许本地 IP 连接
host all all 127.0.0.1/32 md5
host all all ::1/128 md5
host all all 10.133.165.26/32 md5# 如果需要，也添加备份服务器的 IP
host all all 10.133.165.121/32 md5
host all all 0.0.0.0/0 md5#重启修改配置
pg_ctl -D /postgres/data/24398 restart#进行免密登录测试
psql -h 10.133.165.121 -p 24398 -U postgres -d postgres3.安装citus
https://v5p9fgxcq6.feishu.cn/docx/KxKedI5QJobA95xFOZvcKQkin05#share-Ff2fdpVLJoKxJ5xMoZ5cOGkbn6c
1.下载代码进行编译安装
#每一个节点都要安装cd /install
rz
unzip citus-11.3.0.zip
yum install -y autoconf automake libtool curl-devel
yum install -y libzstd-devel#进行编译安装cd citus-11.3.0/
./autogen.sh
./configure --prefix=/postgres/citus
make
make install#确认扩展安装成功
ls /postgres/citus/lib/postgresql/citus.so
2.修改配置
vim /postgres/data/citus/24398/postgresql.conf
shared_preload_libraries = 'citus'#进行重启
pg_ctl -D /postgres/data/citus/5000 restart#然后进入数据库然后进行配置
psql -p 24398
create extension citus;
3.配置认证机制
#在每一个节点进行添加，然后放置在此配置文件的最下面，目的是其他服务器进行访问的时候，需要密码认证
vim /postgres/data/citus/24398/pg_hba.conf
host all all 0.0.0.0/0 md5在worker 节点进行添加免密
配置免密（注意trust 规则必须在 scram-sha-256 或 md5 前面
vim /postgres/data/citus/24398/pg_hba.conf
host all postgres 10.133.165.29/32 trust
host all postgres 10.133.165.172/32 trust
#这个的目的是worker来master进行同步数据的时候进行免密#而我在环境中加的比较多
master
[root@scnp-prod-olap-manage-01 ~]# tail -10 /postgres/data/citus/24398/pg_hba.conf
# replication privilege.
local replication all trust
host replication all 127.0.0.1/32 trust
host replication all ::1/128 trust
# 允许备节点连接进行复制
host replication postgres 10.133.165.172/32 trust
host all postgres 10.133.165.135/32 trust
host all postgres 10.133.165.80/32 trust
host all all 0.0.0.0/0 md5 # 允许所有，md5密码standby
[root@scnp-prod-olap-manage-02 ~]# tail -10 /postgres/data/citus/24398/pg_hba.conf
# replication privilege.
local replication all trust
host replication all 127.0.0.1/32 trust
host replication all ::1/128 trust
# 允许主备节点连接
host replication postgres 10.133.165.29/32 trust
host all postgres 10.133.165.135/32 trust
host all postgres 10.133.165.80/32 trust
host all all 0.0.0.0/0 scram-sha-256worker01
[root@scnp-prod-olap-data-01 ~]# tail -10 /postgres/data/citus/24398/pg_hba.conf
# Allow replication connections from localhost, by a user with the
# replication privilege.
local replication all trust
local all postgres trust
host replication all 127.0.0.1/32 trust
host replication all ::1/128 trust
host all postgres 10.133.165.29/32 trust
host all postgres 10.133.165.172/32 trust
host all postgres 10.133.165.80/32 trust
host all all 0.0.0.0/0 scram-sha-256worker02
[root@scnp-prod-olap-data-02 ~]# tail -10 /postgres/data/citus/24398/pg_hba.conf
# IPv6 local connections:
host all all ::1/128 trust
# Allow replication connections from localhost, by a user with the
# replication privilege.
local replication all trust
host replication all 127.0.0.1/32 trust
host replication all ::1/128 trust
host all postgres 10.133.165.29/32 trust
host all postgres 10.133.165.172/32 trust
host all all 0.0.0.0/0 scram-sha-256我的目的是底层打通4.验证worker节点
#主节点
SELECT * from master_add_node('10.133.165.135', 24398);
SELECT * from master_add_node('10.133.165.80', 24398);#验证节点添加
postgres=# SELECT * FROM master_get_active_worker_nodes();node_name | node_port
----------------+-----------10.133.165.80 | 2439810.133.165.135 | 24398
(2 rows)
5.主备同步
在主节点配置
vim /postgres/data/citus/24398/pg_hba.conf
# 允许备节点连接进行复制
host replication postgres 10.133.165.172/32 trust
重启# 从主节点执行基础备份
/postgres/server/bin/pg_basebackup -h 10.133.165.29 -p 24398 -U postgres -D /postgres/data/citus/24398 -Fp -Xs -P -R#启动服务
/postgres/server/bin/pg_ctl start -D /postgres/data/citus/24398 -l /postgres/data/citus/24398/log/standbylog配置主备复制
在主节点进行操作
vim /postgres/data/citus/24398/postgresql.conf
wal_level = replica
max_wal_senders = 10
wal_keep_size = 1024MB
hot_standby = on
max_replication_slots = 10 # 如果您使用复制槽#重启
pg_ctl -D /postgres/data/citus/24398 restart
6.进行citus和worker验证
---->>
psql -p 24398
CREATE TABLE example_table (id INT UNIQUE NOT NULL,name VARCHAR(100));
ALTER TABLE example_table ADD PRIMARY KEY (id);
set citus.shard_count =2;
set citus.shard_replication_factor =2;
select create_distributed_table('example_table','id','hash');
insert into example_table select id,md5(random()::text) from generate_series(1,5000) as id;#查看select * from citus_tables;7.7.进行主备验证
#在节点进行验证备份节点postgres=# SELECT * FROM pg_stat_replication; pid | usesysid | usename | application_name | client_addr | client_hostname | client_port | backend_start | backend_xmin | state | sent_lsn | write_lsn | flush_lsn | replay_lsn | write_lag | flush_lag | replay_lag | sync_priority | sync_state | reply_time --------+----------+----------+------------------+----------------+-----------------+-------------+-------------------------------+--------------+--
---------+-----------+-----------+-----------+------------+-----------+-----------+------------+---------------+------------+-----------------------
--------158398 | 10 | postgres | walreceiver | 10.133.165.172 | | 20064 | 2025-09-22 16:20:28.189663+08 | | s
treaming | 0/305DD88 | 0/305DD88 | 0/305DD88 | 0/305DD88 | | | | 0 | async | 2025-09-22 16:39:25.59
2021+08
(1 row)
----->>此时能看到备份节点的ip检查 WAL 发送和接收
主节点进行发
SELECT * FROM pg_stat_replication;
备份节点进行收测试数据同步
主节点：
-- 在主节点上执行
CREATE TABLE IF NOT EXISTS replication_test (id SERIAL PRIMARY KEY,test_value TEXT,created_at TIMESTAMP DEFAULT NOW()
);INSERT INTO replication_test (test_value) VALUES ('测试主备复制-' || now()::text);备份节点；
-- 在备节点上执行
SELECT * FROM replication_test;
