Monit-基于非容器服务自恢复程序实践

1.需求：因为历史原因和软件程序原因，有上百台服务和所在服务未运行在容器中，需要在程序奔溃自动拉起(以Java Python C++为主)。

2.目的：能够非人为干预快速自动恢复，要求检测频率在10s一次

3.实现方式

3.1 根据不同语言自己开发脚本实现自动拉起和通知(不够标准化-弃用)

#!/bin/bash

# ========== 使用说明 ==========
# 1. 自定配置env: fat/prod等环境变量, nacos地址, lark webhook等
# 2. 确保每个Java服务有对应的启动脚本（如 deploy-xxx.sh）
# 3. 设置需要监控的Java服务列表（见下方SERVICES_PROCESS_ID等变量）
# 4. 将此脚本添加到crontab中定期执行，例如每1分钟检测一次monitor是否在运行，如果在运行不会重复运行，不在运行则会后台运行：
# */1 * * * * /bin/bash /path/to/this/monitor.sh >> /path/to/monitor.log 2>&1

# ========== 环境变量加载 ==========
# 在 crontab 中执行时，需要显式加载环境变量
# 按优先级加载多个可能的环境变量文件
ENV_FILES=(
    "/etc/profile"
    "/etc/bashrc"
    "/root/.bash_profile"
    "/root/.bashrc"
    "$HOME/.bash_profile"
    "$HOME/.bashrc"
)

for env_file in "${ENV_FILES[@]}"; do
    if [ -f "$env_file" ]; then
        echo "[$(date '+%Y-%m-%d %H:%M:%S')] 加载环境变量文件: $env_file" >> /tmp/monitor_env.log
        source "$env_file"
    fi
done

# 如果上述文件都没有加载到关键环境变量，则手动设置
if [ -z "$JAVA_HOME" ]; then
    export JAVA_HOME="/opt/jdk-17.0.8"
    export CLASSPATH=".:$JAVA_HOME/lib/tools.jar:$JAVA_HOME/lib/dt.jar"
    export PATH="$JAVA_HOME/bin:$PATH"
    echo "[$(date '+%Y-%m-%d %H:%M:%S')] 手动设置 JAVA_HOME: $JAVA_HOME" >> /tmp/monitor_env.log
fi

# 设置必要的环境变量
export env_nacos_address="nacos.test.com:8848"
export env_nacos_namespace="fat"
export ENV="fat"

# 记录环境变量状态（用于调试）
{
    echo "=== 环境变量检查 ==="
    echo "时间: $(date)"
    echo "JAVA_HOME: $JAVA_HOME"
    echo "PATH: $PATH"
    echo "ENV: $ENV"
    echo "env_nacos_address: $env_nacos_address"
    which java
    java -version 2>&1
    echo "=== 环境变量检查结束 ==="
} >> /tmp/monitor_env.log 2>&1

# ========== 脚本配置 ==========
target_folder="./"

LARK_WEBHOOK="https://open.larksuite.com/open-apis/bot/v2/hook/token"
LARK_LOG_FILE="$target_folder/lark.log"
MONITOR_LOG_FILE="/tmp/check.logs"

LARK_MAX_RETRY=3
LARK_RETRY_DELAY=10
LARK_ENV="fat"

# ========== 文件锁机制 ==========
LOCK_FILE="/tmp/service_monitor.lock"

# 尝试获取锁，如果失败则退出
exec 200>"$LOCK_FILE"
flock -n 200 || {
    echo "[$(date '+%Y-%m-%d %H:%M:%S')] 监控脚本已在运行，退出本次执行 (PID: $(cat $LOCK_FILE 2>/dev/null || echo '未知'))" >> "$MONITOR_LOG_FILE"
    exit 0
}

# 将当前PID写入锁文件
echo $$ > "$LOCK_FILE"

# 设置退出时清理锁文件
trap "rm -f $LOCK_FILE; exit" INT TERM EXIT

echo "[$(date '+%Y-%m-%d %H:%M:%S')] 启动服务监控进程 (PID: $$)" | tee -a "$MONITOR_LOG_FILE"

# ========== Lark 报警函数 ==========
send_lark_alert() {
    local message="$1"
    local timestamp=$(date "+%Y-%m-%d %H:%M:%S")
    
    local full_message="[$LARK_ENV][进程监控] $message\n"
    full_message+="服务器名: $(hostname)\n"
    full_message+="报警时间: $timestamp\n"
    #full_message+="<at id=ou_id1>Jame Mei</at>"
    #full_message+="<at id=ou_id2>Levi Li</at>"
    #full_message+="<at id=ou_id3>Bingbing Sun</at>"
    
    local json_data="{
        \"msg_type\": \"interactive\",
        \"card\": {
            \"elements\": [{
                \"tag\": \"div\",
                \"text\": {
                    \"content\": \"$full_message\",
                    \"tag\": \"lark_md\"
                }
            }]
        }
    }"
    
    local attempt=1
    while [[ $attempt -le $LARK_MAX_RETRY ]]; do
        http_status=$(curl -s -o /dev/null -w "%{http_code}" \
            -m 3 \
            -X POST "$LARK_WEBHOOK" \
            -H "Content-Type: application/json" \
            -d "$json_data")
        
        if [[ $http_status -eq 200 ]]; then
            echo "[$(date "+%F %T")] Lark报警发送成功" >> "$LARK_LOG_FILE"
            return 0
        else
            sleep $LARK_RETRY_DELAY
            ((attempt++))
        fi
    done
    echo "[$(date "+%F %T")] 错误: Lark报警发送失败,已达最大重试次数: $attempt" >> "$LARK_LOG_FILE"
    return 1
}

send_alert_async() {
    (
        send_lark_alert "$1"
    ) &> /dev/null &
}

# ========== 服务配置 ==========
WORK_DIR="/data/scripts"

# 服务列表
#SERVICES_PROCESS_ID[0]="pb-trading-engine-1.0-SNAPSHOT.jar"
#SERVICES_FRIENDLY_NAME[0]="交易引擎 (Trading Engine)"
#SERVICES_START_CMD[0]="./deploy-engine-new.sh start"

SERVICES_PROCESS_ID[1]="pb-trading-market-1.0-SNAPSHOT.jar"
SERVICES_FRIENDLY_NAME[1]="行情服务 (Trading Market)"
SERVICES_START_CMD[1]="./pb-trading-market.sh start"

SERVICES_PROCESS_ID[2]="rapidx-trading-query-realtime-1.0-SNAPSHOT.jar"
SERVICES_FRIENDLY_NAME[2]="实时查询 (Query Realtime)"
SERVICES_START_CMD[2]="./deploy-realtime-new.sh start"

SERVICES_PROCESS_ID[3]="router-server-1.0.0-SNAPSHOT.jar"
SERVICES_FRIENDLY_NAME[3]="路由服务 (Router Server)"
SERVICES_START_CMD[3]="./router-server.sh start"

SERVICES_PROCESS_ID[4]="pb-trading-query-1.0-SNAPSHOT.jar"
SERVICES_FRIENDLY_NAME[4]="交易查询 (Trading Query)"
SERVICES_START_CMD[4]="./pb-trading-query.sh start"

SERVICES_PROCESS_ID[5]="rapidx-trading-market-gateway-1.0-SNAPSHOT.jar"
SERVICES_FRIENDLY_NAME[5]="行情网关 (Market Gateway)"
SERVICES_START_CMD[5]="./rapidx-trading-market-gateway.sh start"

SERVICES_PROCESS_ID[6]="ltp-ems-1.0.0-master-SNAPSHOT.jar"
SERVICES_FRIENDLY_NAME[6]="订单管理 (EMS)"
SERVICES_START_CMD[6]="./deploy-ems-new.sh start"

SERVICES_PROCESS_ID[7]="pb-trading-push-0.0.1-SNAPSHOT.jar"
SERVICES_FRIENDLY_NAME[7]="推送服务 (Trading Push)"
SERVICES_START_CMD[7]="./pb-trading-push.sh start"

SERVICES_PROCESS_ID[8]="pb-trading-gateway-1.0-SNAPSHOT.jar"
SERVICES_FRIENDLY_NAME[8]="交易网关 (Trading Gateway)"
SERVICES_START_CMD[8]="./pb-trading-gateway.sh start"

SERVICES_PROCESS_ID[9]="exchange-data-server-1.0.0-SNAPSHOT.jar"
SERVICES_FRIENDLY_NAME[9]="exchange-data-server"
SERVICES_START_CMD[9]="./ltp-exchange-data-server.sh start"

SERVICES_PROCESS_ID[10]="rapidtrade-storage-1.0-SNAPSHOT.jar"
SERVICES_FRIENDLY_NAME[10]="rapidtrade-storage"
SERVICES_START_CMD[10]="./rapidtrade-storage.sh start"

SERVICES_PROCESS_ID[11]="rapidx-trading-algo-server-1.0-SNAPSHOT.jar"
SERVICES_FRIENDLY_NAME[11]="rapidx-trading-algo-server"
SERVICES_START_CMD[11]="./rapidx-trading-algo-server.sh start"

SERVICES_PROCESS_ID[12]="rapidtrade-mock-1.0-SNAPSHOT.jar"
SERVICES_FRIENDLY_NAME[12]="rapidtrade-mock"
SERVICES_START_CMD[12]="./rapidtrade-mock.sh start"

SERVICES_PROCESS_ID[13]="rapidx-ws-simulator-1.0-SNAPSHOT.jar"
SERVICES_FRIENDLY_NAME[13]="rapidx-ws-simulator"
SERVICES_START_CMD[13]="./rapidx-ws-simulator.sh start"

SERVICES_PROCESS_ID[14]="pb-trading-statistics-1.0-SNAPSHOT.jar"
SERVICES_FRIENDLY_NAME[14]="pb-trading-statistics"
SERVICES_START_CMD[14]="./pb-trading-statistics.sh start"

SERVICES_PROCESS_ID[15]="rapidx-trading-onezero-maker-1.0-SNAPSHOT.jar"
SERVICES_FRIENDLY_NAME[15]="rapidx-trading-onezero-maker"
SERVICES_START_CMD[15]="./rapidx-trading-onezero-maker.sh start"

SERVICES_PROCESS_ID[16]="pb-trading-transfer-1.0-SNAPSHOT.jar"
SERVICES_FRIENDLY_NAME[16]="pb-trading-transfer"
SERVICES_START_CMD[16]="./pb-trading-transfer.sh start"

SERVICES_PROCESS_ID[17]="rapidx-trading-clearing-1.0-SNAPSHOT.jar"
SERVICES_FRIENDLY_NAME[17]="rapidx-trading-clearing"
SERVICES_START_CMD[17]="./rapidx-trading-clearing.sh start"

SERVICES_PROCESS_ID[18]="pb-trading-monitor-1.0-SNAPSHOT.jar"
SERVICES_FRIENDLY_NAME[18]="pb-trading-monitor"
SERVICES_START_CMD[18]="./pb-trading-monitor.sh start"

SERVICES_PROCESS_ID[19]="rapidx-trading-query-persistent-1.0-SNAPSHOT.jar"
SERVICES_FRIENDLY_NAME[19]="rapidx-trading-query-persistent"
SERVICES_START_CMD[19]="./deploy-persistent-new.sh start"

SERVICES_PROCESS_ID[20]="rapidtrade-storage-1.0-SNAPSHOT.jar"
SERVICES_FRIENDLY_NAME[20]="rapidtrade-storage"
SERVICES_START_CMD[20]="./rapidtrade-storage.sh start"

SERVICES_PROCESS_ID[21]="bitu-trade-1.0-SNAPSHOT.jar"
SERVICES_FRIENDLY_NAME[21]="bitu-trade"
SERVICES_START_CMD[21]=bitu-trade.sh
#SERVICES_START_CMD[21]="./deploy.sh start"

SERVICES_PROCESS_ID[22]="ltp-data-integration-1.0-SNAPSHOT.jar"
SERVICES_FRIENDLY_NAME[22]="ltp-data-integration"
SERVICES_START_CMD[22]="./ltp-data-integration.sh start"

SERVICES_PROCESS_ID[23]="ingest-server-app-1.0-SNAPSHOT.jar"
SERVICES_FRIENDLY_NAME[23]="ingest-server-app"
SERVICES_START_CMD[23]="./data-ingest-server.sh start"  

SERVICES_PROCESS_ID[24]="ltp-data-visual-1.0-SNAPSHOT.jar"
SERVICES_FRIENDLY_NAME[24]="ltp-data-visual"
SERVICES_START_CMD[24]="./data-cam-visual.sh start" 


# ========== 主循环 ==========
mkdir -p "$(dirname "$MONITOR_LOG_FILE")"

while true; do
    for i in "${!SERVICES_PROCESS_ID[@]}"; do
        process_id="${SERVICES_PROCESS_ID[$i]}"
        friendly_name="${SERVICES_FRIENDLY_NAME[$i]}"
        start_cmd="${SERVICES_START_CMD[$i]}"

        if ! pgrep -f "$process_id" > /dev/null; then
            echo "[$(date '+%Y-%m-%d %H:%M:%S')] - 服务 [$friendly_name] (进程标识: $process_id) 未运行，正在重启..." | tee -a "$MONITOR_LOG_FILE"
            
            # 在子shell中启动服务，确保环境变量正确传递
            (
                # 再次加载环境变量确保子shell中有正确的环境
                for env_file in "${ENV_FILES[@]}"; do
                    if [ -f "$env_file" ]; then
                        source "$env_file"
                    fi
                done
                
                # 手动设置关键环境变量作为备用
                export JAVA_HOME="/opt/jdk-17.0.8"
                export CLASSPATH=".:$JAVA_HOME/lib/tools.jar:$JAVA_HOME/lib/dt.jar"
                export PATH="$JAVA_HOME/bin:$PATH"
                export env_nacos_address="nacos.test.com:8848"
                export env_nacos_namespace="fat"
                export ENV="fat"
                
                cd "$WORK_DIR"
                echo "[$(date '+%Y-%m-%d %H:%M:%S')] - 启动目录: $(pwd), JAVA_HOME: $JAVA_HOME" >> "$MONITOR_LOG_FILE"
                $start_cmd >> "$MONITOR_LOG_FILE" 2>&1
            ) &
            
            send_alert_async "服务 [$friendly_name] (进程标识: $process_id) 已停止运行！正在尝试重启。"
        else
            echo "[$(date '+%Y-%m-%d %H:%M:%S')] - 服务 [$friendly_name] 运行正常." >> "$MONITOR_LOG_FILE"
        fi
    done
    
    sleep 10
done

 

 

3.2 使用开源通用软件统一维护自动拉起(开发+运维都可以简单维护和使用)　　　　　　　　

#程序安装
dnf install -y gcc make openssl-devel bison flex zlib-devel
#apt install -y gcc make libssl-dev bison flex zlib1g-dev
#yum install -y gcc make openssl-devel bison flex zlib-devel
wget https://mmonit.com/monit/dist/monit-5.34.0.tar.gz
tar xf monit-5.34.0.tar.gz && cd monit-5.34.0/
./configure --prefix=/usr/local/monit --without-pam && make && make install
mkdir /usr/local/monit/etc -p && mkdir -p /usr/local/monit/etc/
cp monitrc /usr/local/monit/etc/
chmod 600 /usr/local/monit/etc/monitrc #配置文件定义检测时间，检测配置文件
ln -s /usr/local/monit/bin/monit /usr/sbin/monit
monit --version
mkdir /etc/monit/conf.d/ -p #所有进程检测配置文件，如启动脚本变动修改这里即可
cp /usr/local/monit/bin/monit  /usr/bin/


#systemd配置：vi /etc/systemd/system/monit.service
[Unit]
Description=Monit process monitor
Documentation=https://mmonit.com/monit/
After=network.target

[Service]
Type=forking
ExecStart=/usr/bin/monit -c  /usr/local/monit/etc/monitrc
ExecReload=/usr/bin/monit -c /usr/local/monit/etc/monitrc reload
ExecStop=/usr/bin/monit  -c  /usr/local/monit/etc/monitrc quit
PIDFile=/var/run/monit.pid
Restart=on-failure 
User=root
Group=root

[Install]
WantedBy=multi-user.target  # 多用户模式下开机自启

systemctl reload monit
systemctl enable monit
systemctl start monit

 

#进程配置方式vim /usr/local/monit/etc/monitrc：
set daemon 10
set logfile /var/log/monit.log

# 服务配置 /etc/monit/conf.d/rapidtrade-mock.conf
check process rapidtrade_mock matching "rapidtrade-mock"
    start program = "/data/scripts/rapidtrade-mock.sh start"
    stop program = "/data/scripts/rapidtrade-mock.sh stop"
    if does not exist then start



#port配置方式：
 check host rapidtrade_mock with address 127.0.0.1
    if failed 
        port 7040 
        type tcp 
        timeout 5 seconds 
        for 2 cycles
    then start
    start program = "/data/scripts/rapidtrade-mock.sh start" as uid root and gid root
    stop program = "/data/scripts/rapidtrade-mock.sh stop" as uid root and gid root
    if 3 restarts within 5 cycles then timeout




#健康监测端口+路径:经过测试有问题，无法启动服务并恢复正常。
check host my_web_service with address 127.0.0.1
    if failed
        port 80
        protocol http
        request "/actuator/prometheus" # 指定要检查的健康检查端点路径
        with timeout 10 seconds
        for 3 cycles
    then restart
    start program = "/usr/bin/systemctl start my-service"
    stop program = "/usr/bin/systemctl stop my-service"

 

 

4.测试和使用

这样基于传统服务，只需要改程序有对应的start.sh stop.sh脚本 就可以简单配置进程健康检测来维护服务自动拉起，简单高效不需要每个团队开发很多自己的脚本去维护。