init
This commit is contained in:
Chu Văn Hải committed
commit
fb6af905cd
30 files changed
+8653
No files matched your search
@@ -0,0 +1,182 @@
|
||||
#!/usr/bin/env bash
|
||||
# method 1: crontab with parallel block
|
||||
#* * * * * flock -n /var/lock/monitor-container-script.lock /www/server/scripts/monitor-container-script.sh >>/var/log/monitor-container-script.log 2>&1
|
||||
set -Eeuo pipefail
|
||||
|
||||
# method 2: crontab with parallel block
|
||||
# only use method 1 or method 2
|
||||
# Khóa tiến trình (FD 9)
|
||||
#~+~+~+~+~+~+~+~+~+~+~+~+~+~+~+~+~+~+~+~+~+~+~+~+~+~+~+~+~+~+~
|
||||
exec 9>/var/lock/monitor-container-script.lock
|
||||
if ! flock -n 9; then
|
||||
echo "[$(date '+%F %T')] ⚠️ Another run is in progress. Exit."
|
||||
exit 0
|
||||
fi
|
||||
trap 'flock -u 9' EXIT
|
||||
#~+~+~+~+~+~+~+~+~+~+~+~+~+~+~+~+~+~+~+~+~+~+~+~+~+~+~+~+~+~+~
|
||||
|
||||
# === Cấu hình chung ===
|
||||
CONTAINER_PREFIX="webapplication-production-"
|
||||
CONTAINER_PORT=3000 # cổng bên trong container (không cần dùng ở đây)
|
||||
HOST="localhost"
|
||||
TIMEOUT=3
|
||||
SLEEP_TIME=10
|
||||
COOLDOWN=180 # giây
|
||||
|
||||
CPU_LIMIT=80
|
||||
MEM_LIMIT=90
|
||||
MANAGE_SCRIPT="/home/forlifeci/scripts/manage-webapplication-production.sh restart_single"
|
||||
|
||||
CONTAINER_LOCK_FILE="/var/lib/container_maps.lock"
|
||||
LOCK_DIR="/tmp/container_restart_locks" && mkdir -p "$LOCK_DIR/logs"
|
||||
|
||||
remove_timestamp() { [[ $1 =~ ^(.*-[0-9]+)-([0-9]+)$ ]] && echo "${BASH_REMATCH[1]}" || echo "$1"; }
|
||||
|
||||
check_container_health() {
|
||||
local container=$1
|
||||
local cpu mem
|
||||
|
||||
echo "📡 $(date '+%Y-%m-%d %H:%M:%S') - Kiểm tra container... $container"
|
||||
|
||||
#cpu=$(docker stats --no-stream --format "{{.CPUPerc}}" "$container" | tr -d '%')
|
||||
#mem=$(docker stats --no-stream --format "{{.MemPerc}}" "$container" | tr -d '%')
|
||||
|
||||
#echo "Kiểm tra CPU / RAM quá giới hạn -> $container"
|
||||
#if (( $(echo "$cpu > $CPU_LIMIT" | bc -l) )) || (( $(echo "$mem > $MEM_LIMIT" | bc -l) )); then
|
||||
# echo "⚠️ $container quá tải (CPU: $cpu%, RAM: $mem%)"
|
||||
# return 1
|
||||
#fi
|
||||
|
||||
#echo "Kiểm tra port 3000 bên trong container -> $container"
|
||||
if ! docker exec "$container" sh -c "nc -z -w 2 127.0.0.1 3000" >/dev/null 2>&1; then
|
||||
echo "❌ $container không phản hồi trên port 3000"
|
||||
return 1
|
||||
fi
|
||||
|
||||
return 0
|
||||
}
|
||||
|
||||
# === Telegram Config ===
|
||||
TELEGRAM_BOT_TOKEN="7837210802:AAHFlwXOdm3PExgIbADJ_TstNoXwJ_Aq7g4"
|
||||
TELEGRAM_CHAT_ID="-1002468192435"
|
||||
THREAD_ID="10248"
|
||||
|
||||
# === Hàm gửi cảnh báo Telegram ===
|
||||
send_telegram_alert() {
|
||||
local container=$1
|
||||
local port=$2
|
||||
local fileName=${3:-"log"}
|
||||
local now=$(date '+%Y-%m-%d %H:%M:%S')
|
||||
local msg="🚨 *CẢNH BÁO* 🚨
|
||||
|
||||
Container: \`$container\`
|
||||
Port: \`$port\`
|
||||
Thời gian: *$now*
|
||||
Hành động: \`docker restart\`
|
||||
|
||||
➡️ Container không phản hồi, đã được khởi động lại.
|
||||
|
||||
http://42.118.103.30/logs/$container.restart.$fileName.html
|
||||
"
|
||||
|
||||
curl -s -X POST "https://api.telegram.org/bot$TELEGRAM_BOT_TOKEN/sendMessage" \
|
||||
-d chat_id="$TELEGRAM_CHAT_ID" \
|
||||
-d parse_mode="Markdown" \
|
||||
-d message_thread_id="$THREAD_ID" \
|
||||
-d text="$msg" > /dev/null
|
||||
}
|
||||
|
||||
# === Hàm kiểm tra port ===
|
||||
#check_port() {
|
||||
# local port=$1
|
||||
# nc -z -w $TIMEOUT $HOST $port
|
||||
#}
|
||||
|
||||
check_port() {
|
||||
local port=$1
|
||||
curl -s --max-time "$TIMEOUT" "http://$HOST:$port" -o /dev/null
|
||||
}
|
||||
|
||||
should_skip_check() {
|
||||
local container=$1
|
||||
local lock_file="$LOCK_DIR/$container.lock"
|
||||
|
||||
if [[ -f "$lock_file" ]]; then
|
||||
local last_restart
|
||||
last_restart=$(cat "$lock_file")
|
||||
local now
|
||||
now=$(date +%s)
|
||||
if (( now - last_restart < COOLDOWN )); then
|
||||
return 0 # bỏ qua
|
||||
else
|
||||
rm -f "$lock_file" # hết hạn cooldown
|
||||
fi
|
||||
fi
|
||||
return 1
|
||||
}
|
||||
|
||||
mark_restart() {
|
||||
local container=$1
|
||||
date +%s > "$LOCK_DIR/$container.lock"
|
||||
}
|
||||
|
||||
|
||||
get_docker_binding_port() {
|
||||
local container=$1
|
||||
echo $(docker port "$container" 3000/tcp | awk -F: '{print $2}' | xargs)
|
||||
}
|
||||
|
||||
containers=$(docker ps --format '{{.Names}}' | grep "^$CONTAINER_PREFIX")
|
||||
# === Vòng lặp kiểm tra ===
|
||||
#while true; do
|
||||
|
||||
for container in $containers; do
|
||||
if [[ -f $CONTAINER_LOCK_FILE ]]; then
|
||||
break
|
||||
fi
|
||||
|
||||
container_locked=$(remove_timestamp "$container")
|
||||
|
||||
# 🔹 Bỏ qua nếu container đang trong cooldown
|
||||
if should_skip_check "$container_locked"; then
|
||||
echo "⏩ Bỏ qua $container_locked (đang cooldown sau khi restart)"
|
||||
continue
|
||||
fi
|
||||
|
||||
host_port=$(get_docker_binding_port "$container")
|
||||
|
||||
if [ -z "$host_port" ]; then
|
||||
echo "⚠ $container không map port 3000/tcp, bỏ qua."
|
||||
continue
|
||||
fi
|
||||
|
||||
echo "";
|
||||
echo "👉 Kiểm tra $container tại $HOST:$host_port..."
|
||||
|
||||
if ! check_port $host_port; then
|
||||
echo "⚠️ $container KHÔNG phản hồi. Chờ 5s để xác nhận..."
|
||||
sleep 5
|
||||
if ! check_port $host_port; then
|
||||
echo "❌ $container vẫn không phản hồi! Restart & gửi Telegram..."
|
||||
logName=$(date +%s)
|
||||
mark_restart "$container_locked" # đánh dấu cooldown
|
||||
#docker restart "$container"
|
||||
restart=`bash $MANAGE_SCRIPT $container`
|
||||
logPath="$LOCK_DIR/logs/$container.restart.$logName.html"
|
||||
if [ ! -f "$logPath" ]; then
|
||||
echo "<!DOCTYPE html><html><meta charset=utf-8><style>pre{text-wrap: auto;}</style>" > "$logPath"
|
||||
fi
|
||||
echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~<br>" >> "$logPath"
|
||||
printf "%s" "<pre>$restart</pre>" >> "$logPath"
|
||||
send_telegram_alert "$container" "$host_port" "$logName"
|
||||
else
|
||||
echo "✅ $container đã phản hồi lại sau khi chờ."
|
||||
fi
|
||||
else
|
||||
echo "✅ $container phản hồi OK."
|
||||
fi
|
||||
done
|
||||
|
||||
# sleep $SLEEP_TIME
|
||||
#done
|
||||
#/home/forlifeci/scripts/manage-webapplication-production.sh restart_single $container
|
||||
Reference in new issue
Block a user