gavindiaz/DinQuant
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Refactor code for improved readability and consistency

Browse Source 
- Cleaned up whitespace and formatting in various files including http.py, language.py, logger.py, safe_exec.py, and SQL migration scripts.
- Consolidated import statements and removed unnecessary blank lines.
- Updated logging configuration for better clarity.
- Enhanced the safe execution code with improved error handling and logging.
- Removed commented-out code and unnecessary variables in backfill_zero_trades.py and other scripts.
- Added a pyproject.toml for Ruff and Vulture configuration.
- Introduced requirements-dev.txt for development dependencies.
- Removed commented-out stock entries in init.sql for cleaner migration scripts.
This commit is contained in:
dienakdz
2026-04-09 14:30:51 +07:00
parent 103055b3df
commit 87f2845483
157 changed files with 19026 additions and 17773 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend_api_python/app/utils/db_postgres.py
+69 -65
View File
@@ -4,19 +4,21 @@ PostgreSQL Database Connection Utility
Supports multi-user mode with connection pooling.
Provides placeholder conversion for backward compatibility with legacy code.
"""
import os
import threading
from typing import Optional, Any, List, Dict
from contextlib import contextmanager
from typing import Any, Dict, List, Optional
from app.utils.logger import get_logger
logger = get_logger(__name__)
# Try to import psycopg2
try:
import psycopg2
from psycopg2 import pool
from psycopg2.extras import RealDictCursor
HAS_PSYCOPG2 = True
except ImportError:
HAS_PSYCOPG2 = False
@@ -29,7 +31,7 @@ _pool_lock = threading.Lock()
def _get_database_url() -> str:
"""Get database connection URL from environment"""
return os.getenv('DATABASE_URL', '').strip()
return os.getenv("DATABASE_URL", "").strip()
def _parse_database_url(url: str) -> Dict[str, Any]:
@@ -38,131 +40,133 @@ def _parse_database_url(url: str) -> Dict[str, Any]:
"""
if not url:
return {}
# Remove protocol prefix
if url.startswith('postgresql://'):
if url.startswith("postgresql://"):
url = url[13:]
elif url.startswith('postgres://'):
elif url.startswith("postgres://"):
url = url[11:]
else:
return {}
result = {}
# Split user:password@host:port/dbname
if '@' in url:
auth, hostpart = url.rsplit('@', 1)
if ':' in auth:
result['user'], result['password'] = auth.split(':', 1)
if "@" in url:
auth, hostpart = url.rsplit("@", 1)
if ":" in auth:
result["user"], result["password"] = auth.split(":", 1)
else:
result['user'] = auth
result["user"] = auth
else:
hostpart = url
# Split host:port/dbname
if '/' in hostpart:
hostport, result['dbname'] = hostpart.split('/', 1)
if "/" in hostpart:
hostport, result["dbname"] = hostpart.split("/", 1)
else:
hostport = hostpart
if ':' in hostport:
result['host'], port_str = hostport.split(':', 1)
result['port'] = int(port_str)
if ":" in hostport:
result["host"], port_str = hostport.split(":", 1)
result["port"] = int(port_str)
else:
result['host'] = hostport
result['port'] = 5432
result["host"] = hostport
result["port"] = 5432
return result
def _get_connection_pool():
"""Get or create connection pool"""
global _connection_pool
if _connection_pool is not None:
return _connection_pool
with _pool_lock:
if _connection_pool is not None:
return _connection_pool
if not HAS_PSYCOPG2:
raise RuntimeError("psycopg2 is not installed. Cannot use PostgreSQL.")
db_url = _get_database_url()
if not db_url:
raise RuntimeError("DATABASE_URL environment variable is not set.")
params = _parse_database_url(db_url)
if not params:
raise RuntimeError(f"Invalid DATABASE_URL format: {db_url}")
try:
_connection_pool = pool.ThreadedConnectionPool(
minconn=2,
maxconn=20,
host=params.get('host', 'localhost'),
port=params.get('port', 5432),
user=params.get('user', 'quantdinger'),
password=params.get('password', ''),
dbname=params.get('dbname', 'quantdinger'),
host=params.get("host", "localhost"),
port=params.get("port", 5432),
user=params.get("user", "quantdinger"),
password=params.get("password", ""),
dbname=params.get("dbname", "quantdinger"),
connect_timeout=10,
)
logger.info(f"PostgreSQL connection pool created: {params.get('host')}:{params.get('port')}/{params.get('dbname')}")
logger.info(
f"PostgreSQL connection pool created: {params.get('host')}:{params.get('port')}/{params.get('dbname')}"
)
except Exception as e:
logger.error(f"Failed to create PostgreSQL connection pool: {e}")
raise
return _connection_pool
class PostgresCursor:
"""PostgreSQL cursor wrapper with placeholder conversion for backward compatibility"""
def __init__(self, cursor):
self._cursor = cursor
self._last_insert_id = None
def _convert_placeholders(self, query: str) -> str:
"""
Convert ? placeholders to PostgreSQL %s for backward compatibility.
Also handle some SQL syntax differences.
"""
# Replace ? -> %s
query = query.replace('?', '%s')
query = query.replace("?", "%s")
# INSERT OR IGNORE -> PostgreSQL: INSERT ... ON CONFLICT DO NOTHING
query = query.replace('INSERT OR IGNORE', 'INSERT')
query = query.replace("INSERT OR IGNORE", "INSERT")
return query
def execute(self, query: str, args: Any = None):
"""Execute SQL statement"""
query = self._convert_placeholders(query)
# Check if this is an INSERT and add RETURNING id if not present
is_insert = query.strip().upper().startswith('INSERT')
if is_insert and 'RETURNING' not in query.upper():
query = query.rstrip(';').rstrip() + ' RETURNING id'
is_insert = query.strip().upper().startswith("INSERT")
if is_insert and "RETURNING" not in query.upper():
query = query.rstrip(";").rstrip() + " RETURNING id"
if args:
if not isinstance(args, (tuple, list)):
args = (args,)
result = self._cursor.execute(query, args)
else:
result = self._cursor.execute(query)
# Capture last insert id for INSERT statements
if is_insert:
try:
row = self._cursor.fetchone()
if row and 'id' in row:
self._last_insert_id = row['id']
if row and "id" in row:
self._last_insert_id = row["id"]
except Exception:
pass
return result
def fetchone(self) -> Optional[Dict[str, Any]]:
"""Fetch single row"""
row = self._cursor.fetchone()
@@ -170,7 +174,7 @@ class PostgresCursor:
return None
# RealDictCursor already returns a dict, so return as-is
return row if isinstance(row, dict) else dict(row) if row else None
def fetchall(self) -> List[Dict[str, Any]]:
"""Fetch all rows"""
rows = self._cursor.fetchall()
@@ -178,16 +182,16 @@ class PostgresCursor:
return []
# RealDictCursor already returns dicts, so return as-is
return [row if isinstance(row, dict) else dict(row) for row in rows]
def close(self):
"""Close cursor"""
self._cursor.close()
@property
def lastrowid(self) -> Optional[int]:
"""Get last inserted row ID"""
return self._last_insert_id
@property
def rowcount(self) -> int:
"""Get affected row count"""
@@ -196,23 +200,23 @@ class PostgresCursor:
class PostgresConnection:
"""PostgreSQL connection wrapper"""
def __init__(self, conn):
self._conn = conn
self._pool = _get_connection_pool()
def cursor(self) -> PostgresCursor:
"""Create cursor"""
return PostgresCursor(self._conn.cursor(cursor_factory=RealDictCursor))
def commit(self):
"""Commit transaction"""
self._conn.commit()
def rollback(self):
"""Rollback transaction"""
self._conn.rollback()
def close(self):
"""Return connection to pool"""
if self._pool and self._conn:
@@ -280,7 +284,7 @@ def execute_sql(sql: str, params: tuple = None) -> List[Dict[str, Any]]:
with get_pg_connection() as conn:
cursor = conn.cursor()
cursor.execute(sql, params)
if sql.strip().upper().startswith('SELECT'):
if sql.strip().upper().startswith("SELECT"):
return cursor.fetchall()
conn.commit()
return []
@@ -290,11 +294,11 @@ def is_postgres_available() -> bool:
"""Check if PostgreSQL is available"""
if not HAS_PSYCOPG2:
return False
db_url = _get_database_url()
if not db_url:
return False
try:
with get_pg_connection() as conn:
cursor = conn.cursor()