gavindiaz/PolyWeather
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add production cache warmer

Browse Source
This commit is contained in:
2569718930@qq.com
2026-06-14 04:23:15 +08:00
parent 4009a26431
commit 25408797f1
7 changed files with 539 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files
docker-compose.yml
+55 -5
View File
@@ -2,7 +2,7 @@ services:
polyweather_redis:
image: redis:7-alpine
container_name: polyweather_redis
command: redis-server --appendonly yes --maxmemory 128mb --maxmemory-policy noeviction
command: redis-server --appendonly yes --maxmemory ${POLYWEATHER_REDIS_MAXMEMORY:-512mb} --maxmemory-policy noeviction
restart: unless-stopped
healthcheck:
interval: 10s
@@ -100,9 +100,13 @@ services:
env_file: *id001
environment:
POLYWEATHER_REDIS_URL: ${POLYWEATHER_REDIS_URL:-redis://polyweather_redis:6379/0}
POLYWEATHER_EVENT_STORE: ${POLYWEATHER_EVENT_STORE:-redis}
POLYWEATHER_REDIS_REQUIRED: ${POLYWEATHER_REDIS_REQUIRED:-true}
POLYWEATHER_REDIS_STREAM_MAXLEN: ${POLYWEATHER_REDIS_STREAM_MAXLEN:-100000}
POLYWEATHER_SCAN_TERMINAL_REDIS_CACHE_ENABLED: ${POLYWEATHER_SCAN_TERMINAL_REDIS_CACHE_ENABLED:-true}
POLYWEATHER_COLLECTOR_PATCH_ENDPOINT: ''
POLYWEATHER_CITY_DETAIL_BATCH_CONCURRENCY: ${POLYWEATHER_CITY_DETAIL_BATCH_CONCURRENCY:-1}
POLYWEATHER_CITY_DETAIL_BATCH_GLOBAL_CONCURRENCY: ${POLYWEATHER_CITY_DETAIL_BATCH_GLOBAL_CONCURRENCY:-1}
POLYWEATHER_CITY_DETAIL_BATCH_CONCURRENCY: ${POLYWEATHER_CITY_DETAIL_BATCH_CONCURRENCY:-2}
POLYWEATHER_CITY_DETAIL_BATCH_GLOBAL_CONCURRENCY: ${POLYWEATHER_CITY_DETAIL_BATCH_GLOBAL_CONCURRENCY:-2}
POLYWEATHER_CITY_DETAIL_BATCH_QUEUE_WAIT_MS: ${POLYWEATHER_CITY_DETAIL_BATCH_QUEUE_WAIT_MS:-3000}
POLYWEATHER_CITY_DETAIL_BATCH_PARTIAL_TIMEOUT_MS: ${POLYWEATHER_CITY_DETAIL_BATCH_PARTIAL_TIMEOUT_MS:-8000}
POLYWEATHER_OBSERVATION_COLLECTOR_AMOS_SEC: ${POLYWEATHER_OBSERVATION_COLLECTOR_AMOS_SEC:-60}
@@ -114,8 +118,8 @@ services:
POLYWEATHER_OBSERVATION_COLLECTOR_MADIS_SEC: ${POLYWEATHER_OBSERVATION_COLLECTOR_MADIS_SEC:-300}
POLYWEATHER_OBSERVATION_COLLECTOR_TICK_SEC: ${POLYWEATHER_OBSERVATION_COLLECTOR_TICK_SEC:-30}
POLYWEATHER_SCAN_TERMINAL_BUILD_TIMEOUT_SEC: '30'
POLYWEATHER_SCAN_TERMINAL_MAX_WORKERS: ${POLYWEATHER_SCAN_TERMINAL_MAX_WORKERS:-1}
POLYWEATHER_SCAN_TERMINAL_PREWARM_ENABLED: 'false'
POLYWEATHER_SCAN_TERMINAL_MAX_WORKERS: ${POLYWEATHER_SCAN_TERMINAL_MAX_WORKERS:-4}
POLYWEATHER_SCAN_TERMINAL_PREWARM_ENABLED: ${POLYWEATHER_SCAN_TERMINAL_PREWARM_ENABLED:-false}
POLYWEATHER_SERVICE_ROLE: web
UVICORN_WORKERS: ${UVICORN_WORKERS:-2}
healthcheck:
@@ -164,6 +168,7 @@ services:
POLYWEATHER_OBSERVATION_COLLECTOR_INITIAL_DELAY_SEC: ${POLYWEATHER_OBSERVATION_COLLECTOR_INITIAL_DELAY_SEC:-15}
POLYWEATHER_OBSERVATION_COLLECTOR_MADIS_SEC: ${POLYWEATHER_OBSERVATION_COLLECTOR_MADIS_SEC:-300}
POLYWEATHER_OBSERVATION_COLLECTOR_TICK_SEC: ${POLYWEATHER_OBSERVATION_COLLECTOR_TICK_SEC:-30}
POLYWEATHER_OBSERVATION_COLLECTOR_CACHE_REFRESH_WORKERS: ${POLYWEATHER_OBSERVATION_COLLECTOR_CACHE_REFRESH_WORKERS:-2}
POLYWEATHER_REDIS_URL: ${POLYWEATHER_REDIS_URL:-redis://polyweather_redis:6379/0}
POLYWEATHER_SCAN_TERMINAL_PREWARM_ENABLED: 'false'
POLYWEATHER_SERVICE_ROLE: collector
@@ -186,6 +191,51 @@ services:
volumes:
- ${POLYWEATHER_RUNTIME_DATA_DIR:-/var/lib/polyweather}:/var/lib/polyweather
- ${POLYWEATHER_RUNTIME_DATA_DIR:-/var/lib/polyweather}:/app/data
polyweather_warmer:
command: python -m web.cache_warmer_worker
container_name: polyweather_warmer
depends_on:
polyweather_redis:
condition: service_healthy
polyweather_web:
condition: service_started
logging:
driver: "json-file"
options:
max-size: "50m"
max-file: "3"
cpus: ${POLYWEATHER_WARMER_CPUS:-0.75}
env_file: *id001
environment:
POLYWEATHER_EVENT_STORE: ${POLYWEATHER_EVENT_STORE:-redis}
POLYWEATHER_OBSERVATION_COLLECTOR_ENABLED: 'false'
POLYWEATHER_REDIS_URL: ${POLYWEATHER_REDIS_URL:-redis://polyweather_redis:6379/0}
POLYWEATHER_SCAN_TERMINAL_PREWARM_ENABLED: 'false'
POLYWEATHER_SERVICE_ROLE: warmer
POLYWEATHER_WARMER_CITY_BATCH_SIZE: ${POLYWEATHER_WARMER_CITY_BATCH_SIZE:-8}
POLYWEATHER_WARMER_CITY_INTERVAL_SEC: ${POLYWEATHER_WARMER_CITY_INTERVAL_SEC:-60}
POLYWEATHER_WARMER_ENABLED: ${POLYWEATHER_WARMER_ENABLED:-true}
POLYWEATHER_WARMER_SCAN_INTERVAL_SEC: ${POLYWEATHER_WARMER_SCAN_INTERVAL_SEC:-300}
POLYWEATHER_WARMER_TICK_SEC: ${POLYWEATHER_WARMER_TICK_SEC:-60}
healthcheck:
interval: 60s
retries: 3
test:
- CMD
- python
- -c
- import sqlite3; c=sqlite3.connect('/var/lib/polyweather/polyweather.db');
c.execute('SELECT 1'); c.close()
timeout: 10s
image: ghcr.io/yangyuan-zhen/polyweather-backend:${IMAGE_TAG:-latest}
mem_limit: ${POLYWEATHER_WARMER_MEM_LIMIT:-768m}
memswap_limit: ${POLYWEATHER_WARMER_MEMSWAP_LIMIT:-1g}
pids_limit: 256
restart: unless-stopped
user: ${UID:-1000}:${GID:-1000}
volumes:
- ${POLYWEATHER_RUNTIME_DATA_DIR:-/var/lib/polyweather}:/var/lib/polyweather
- ${POLYWEATHER_RUNTIME_DATA_DIR:-/var/lib/polyweather}:/app/data
x-polyweather-base:
env_file: *id001
image: ghcr.io/yangyuan-zhen/polyweather-backend:${IMAGE_TAG:-latest}