mirror of
https://github.com/NicolasBohn/NexQuant.git
synced 2026-07-27 15:37:44 +00:00
TPTBusiness
cbe1c52e00
Rename all source files, scripts, tests, documentation, and configuration from Predix/predix to NexQuant/nexquant across the entire codebase.
110 lines
2.9 KiB
Bash
Executable File
110 lines
2.9 KiB
Bash
Executable File
#!/bin/bash
|
|
# ============================================================================
|
|
# PREDIX Strategy Generator Watchdog
|
|
# Checks every 20min: is the generator running? If not, (re)start it.
|
|
# ============================================================================
|
|
|
|
SCRIPT_DIR="/home/nico/NexQuant"
|
|
GENERATOR="python ${SCRIPT_DIR}/nexquant_smart_strategy_gen.py"
|
|
TARGET_COUNT=3
|
|
LOGFILE="${SCRIPT_DIR}/results/logs/watchdog.log"
|
|
LOCKFILE="/tmp/nexquant_generator.lock"
|
|
MAX_ATTEMPTS=50 # Stop after this many attempts
|
|
PIDFILE="/tmp/nexquant_generator_attempt.pid"
|
|
|
|
mkdir -p "${SCRIPT_DIR}/results/logs"
|
|
|
|
log() {
|
|
echo "$(date '+%Y-%m-%d %H:%M:%S') - $1" >> "$LOGFILE"
|
|
}
|
|
|
|
# Get current attempt count
|
|
get_attempt_count() {
|
|
if [ -f "$PIDFILE" ]; then
|
|
cat "$PIDFILE"
|
|
else
|
|
echo "0"
|
|
fi
|
|
}
|
|
|
|
# Increment attempt count
|
|
increment_attempt() {
|
|
local current=$(get_attempt_count)
|
|
local next=$((current + 1))
|
|
echo "$next" > "$PIDFILE"
|
|
echo "$next"
|
|
}
|
|
|
|
# Check if generator is actually making progress
|
|
check_progress() {
|
|
local latest_log=$(ls -t ${SCRIPT_DIR}/results/logs/smart_strategy_gen_*.log 2>/dev/null | head -1)
|
|
if [ -n "$latest_log" ]; then
|
|
# Check if log was updated in last 5 minutes
|
|
local age=$(( $(date +%s) - $(stat -c %Y "$latest_log" 2>/dev/null || echo 0) ))
|
|
if [ $age -gt 300 ]; then
|
|
return 1 # Stale
|
|
fi
|
|
return 0 # Fresh
|
|
fi
|
|
return 1 # No log file
|
|
}
|
|
|
|
# Kill any existing generator processes
|
|
cleanup() {
|
|
pkill -9 -f "nexquant_smart_strategy_gen.py" 2>/dev/null
|
|
rm -f "$LOCKFILE"
|
|
log "Cleaned up old processes"
|
|
}
|
|
|
|
# Check if we've hit max attempts
|
|
if [ "$(get_attempt_count)" -ge "$MAX_ATTEMPTS" ]; then
|
|
log "MAX ATTEMPTS ($MAX_ATTEMPTS) reached. Stopping watchdog."
|
|
exit 0
|
|
fi
|
|
|
|
# Check if generator is running
|
|
if pgrep -f "nexquant_smart_strategy_gen.py" > /dev/null 2>&1; then
|
|
# Check if it's making progress
|
|
if check_progress; then
|
|
log "Generator is running and making progress. Exiting."
|
|
exit 0
|
|
else
|
|
log "Generator is running but appears stalled. Restarting..."
|
|
cleanup
|
|
fi
|
|
else
|
|
log "Generator is NOT running. Starting..."
|
|
cleanup
|
|
fi
|
|
|
|
# Increment attempt counter
|
|
ATTEMPT=$(increment_attempt)
|
|
log "=== Attempt $ATTEMPT / $MAX_ATTEMPTS ==="
|
|
|
|
# Create lock file
|
|
echo $$ > "$LOCKFILE"
|
|
|
|
# Start generator in background, capture PID
|
|
cd "$SCRIPT_DIR"
|
|
nohup $GENERATOR $TARGET_COUNT > /dev/null 2>&1 &
|
|
GEN_PID=$!
|
|
|
|
log "Started generator with PID $GEN_PID"
|
|
|
|
# Wait for process to finish (up to 20 min)
|
|
WAIT=0
|
|
while kill -0 $GEN_PID 2>/dev/null; do
|
|
sleep 10
|
|
WAIT=$((WAIT + 10))
|
|
if [ $WAIT -ge 1200 ]; then # 20 min timeout
|
|
log "Generator timed out after 20 min. Killing."
|
|
kill -9 $GEN_PID 2>/dev/null
|
|
break
|
|
fi
|
|
done
|
|
|
|
# Cleanup lock
|
|
rm -f "$LOCKFILE"
|
|
|
|
log "Generator finished (or was killed). Exit code: $?"
|