refactor: Remove libs directory

This commit is contained in:
tukuaiai
2025-12-18 13:22:45 +08:00
parent 8487ece766
commit 21aa110e95
639 changed files with 1 additions and 110809 deletions
-69
View File
@@ -1,69 +0,0 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
r"""
配置模块 - 智能路径识别
支持: Linux 原生路径、WSL 路径 (\\wsl.localhost\Ubuntu\...)
"""
import os
import re
from dotenv import load_dotenv
# 项目目录
PROJECT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
OUTPUT_DIR = os.path.join(PROJECT_DIR, "output")
load_dotenv(os.path.join(PROJECT_DIR, ".env"))
def convert_wsl_path(path: str) -> str:
match = re.match(r'^\\\\wsl[.\$]?[^\\]*\\[^\\]+\\(.+)$', path, re.IGNORECASE)
if match:
return '/' + match.group(1).replace('\\', '/')
return path
def normalize_path(path: str) -> str:
path = path.strip()
path = convert_wsl_path(path)
return os.path.expanduser(path)
def get_paths(env_key: str) -> list:
val = os.getenv(env_key, "")
if not val:
return []
return [normalize_path(p) for p in val.split(",") if p.strip()]
def auto_detect_paths() -> dict:
home = os.path.expanduser("~")
kiro_db = os.path.join(home, ".local", "share", "kiro-cli")
candidates = {
"codex_paths": [os.path.join(home, ".codex", "sessions"), os.path.join(home, ".codex")],
"kiro_paths": [kiro_db] if os.path.exists(kiro_db) else [],
"gemini_paths": [os.path.join(home, ".gemini", "tmp"), os.path.join(home, ".gemini")],
"claude_paths": [os.path.join(home, ".claude")],
}
detected = {}
for key, paths in candidates.items():
for p in paths:
if os.path.exists(p):
detected[key] = [p]
break
if key not in detected:
detected[key] = []
return detected
def load_config() -> dict:
auto = auto_detect_paths()
os.makedirs(OUTPUT_DIR, exist_ok=True)
os.makedirs(os.path.join(OUTPUT_DIR, "logs"), exist_ok=True)
return {
"codex_paths": get_paths("CODEX_PATHS") or auto.get("codex_paths", []),
"kiro_paths": get_paths("KIRO_PATHS") or auto.get("kiro_paths", []),
"gemini_paths": get_paths("GEMINI_PATHS") or auto.get("gemini_paths", []),
"claude_paths": get_paths("CLAUDE_PATHS") or auto.get("claude_paths", []),
"output_dir": OUTPUT_DIR,
"log_dir": os.path.join(OUTPUT_DIR, "logs"),
"db_path": os.path.join(OUTPUT_DIR, "chat_history.db"),
}
CONFIG = load_config()
-42
View File
@@ -1,42 +0,0 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""日志模块 - 同时输出到控制台和文件"""
import logging
import os
from datetime import datetime
_logger = None
def setup_logger(log_dir: str = None) -> logging.Logger:
global _logger
if _logger:
return _logger
_logger = logging.getLogger('ai_chat_converter')
_logger.setLevel(logging.DEBUG)
_logger.handlers.clear()
fmt = logging.Formatter('%(asctime)s [%(levelname)s] %(message)s', datefmt='%Y-%m-%d %H:%M:%S')
# 控制台
ch = logging.StreamHandler()
ch.setLevel(logging.INFO)
ch.setFormatter(fmt)
_logger.addHandler(ch)
# 文件
if log_dir:
os.makedirs(log_dir, exist_ok=True)
log_file = os.path.join(log_dir, f"sync_{datetime.now().strftime('%Y%m%d')}.log")
fh = logging.FileHandler(log_file, encoding='utf-8')
fh.setLevel(logging.DEBUG)
fh.setFormatter(fmt)
_logger.addHandler(fh)
return _logger
def get_logger() -> logging.Logger:
global _logger
if not _logger:
_logger = setup_logger()
return _logger
-319
View File
@@ -1,319 +0,0 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
AI 聊天记录集中存储工具
命令:
python main.py # 同步一次
python main.py --watch # 持续监控
python main.py --prune # 清理孤立记录
python main.py --stats # 显示统计
python main.py --search <keyword> # 搜索
python main.py --export json|csv [--source codex|kiro|gemini|claude]
"""
import os
import sys
import subprocess
# 项目根目录
PROJECT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
VENV_DIR = os.path.join(PROJECT_DIR, '.venv')
REQUIREMENTS = os.path.join(PROJECT_DIR, 'requirements.txt')
def ensure_venv():
"""检测并创建虚拟环境,安装依赖"""
# 打包版本跳过
if getattr(sys, 'frozen', False):
return
# 已在虚拟环境中运行则跳过
if sys.prefix != sys.base_prefix:
return
# 检查 .venv 是否存在
venv_python = os.path.join(VENV_DIR, 'bin', 'python') if os.name != 'nt' else os.path.join(VENV_DIR, 'Scripts', 'python.exe')
if not os.path.exists(venv_python):
print("首次运行,创建虚拟环境...")
subprocess.run([sys.executable, '-m', 'venv', VENV_DIR], check=True)
print("安装依赖...")
pip = os.path.join(VENV_DIR, 'bin', 'pip') if os.name != 'nt' else os.path.join(VENV_DIR, 'Scripts', 'pip.exe')
subprocess.run([pip, 'install', '-r', REQUIREMENTS, '-q'], check=True)
print("环境准备完成,重新启动...\n")
# 使用虚拟环境重新执行
os.execv(venv_python, [venv_python] + sys.argv)
# 启动前检测虚拟环境
ensure_venv()
# 支持 PyInstaller 打包
if getattr(sys, 'frozen', False):
BASE_DIR = sys._MEIPASS
sys.path.insert(0, os.path.join(BASE_DIR, 'src'))
else:
BASE_DIR = os.path.dirname(os.path.abspath(__file__))
import argparse
from config import CONFIG
from parsers import CodexParser, GeminiParser, ClaudeParser, KiroParser
from storage import ChatStorage
from logger import setup_logger, get_logger
storage: ChatStorage = None
def main():
global storage
parser = argparse.ArgumentParser(description='AI Chat Converter')
parser.add_argument('-w', '--watch', action='store_true', help='持续监控模式')
parser.add_argument('--prune', action='store_true', help='清理孤立记录')
parser.add_argument('--stats', action='store_true', help='显示统计信息')
parser.add_argument('--search', type=str, help='搜索关键词')
parser.add_argument('--export', choices=['json', 'csv'], help='导出格式')
parser.add_argument('--source', choices=['codex', 'kiro', 'gemini', 'claude'], help='指定来源')
parser.add_argument('--output', type=str, help='导出文件路径')
args = parser.parse_args()
# 初始化
setup_logger(CONFIG["log_dir"])
log = get_logger()
storage = ChatStorage(CONFIG["db_path"])
# 命令分发
if args.prune:
cmd_prune()
elif args.stats:
cmd_stats()
elif args.search:
cmd_search(args.search, args.source)
elif args.export:
cmd_export(args.export, args.source, args.output)
elif args.watch:
cmd_sync()
cmd_watch()
else:
cmd_sync()
def cmd_sync():
log = get_logger()
log.info("=" * 50)
log.info("AI 聊天记录 → 集中存储")
log.info("=" * 50)
log.info(f"数据库: {CONFIG['db_path']}")
total_added, total_updated, total_skipped, total_errors = 0, 0, 0, 0
for cli, key, parser_cls in [
('codex', 'codex_paths', lambda: CodexParser('codex')),
('kiro', 'kiro_paths', KiroParser),
('gemini', 'gemini_paths', GeminiParser),
('claude', 'claude_paths', ClaudeParser),
]:
paths = CONFIG.get(key, [])
if not paths:
continue
parser = parser_cls()
if cli in ('claude', 'kiro'):
a, u, s, e = process_multi(parser, paths, cli)
else:
a, u, s, e = process(parser, paths)
log.info(f"[{cli.capitalize()}] 新增:{a} 更新:{u} 跳过:{s} 错误:{e}")
update_cli_meta(cli)
total_added += a
total_updated += u
total_skipped += s
total_errors += e
total = storage.get_total_stats()
storage.update_total_meta(total['sessions'], total['messages'], total['tokens'])
log.info("=" * 50)
log.info(f"总计: {total['sessions']} 会话, {total['messages']} 消息")
if total_errors > 0:
log.warning(f"错误: {total_errors} 个文件解析失败")
log.info("✓ 同步完成!")
print_token_stats()
def cmd_watch():
from watcher import ChatWatcher
from datetime import datetime
log = get_logger()
log.info("")
log.info("=" * 50)
log.info("实时监听模式 (watchdog)")
log.info("=" * 50)
watch_paths = []
path_source_map = {}
for cli, key in [('codex', 'codex_paths'), ('kiro', 'kiro_paths'),
('gemini', 'gemini_paths'), ('claude', 'claude_paths')]:
for p in CONFIG.get(key, []):
if os.path.isdir(p) or os.path.isfile(p):
watch_paths.append(p)
path_source_map[p] = cli
def on_change(file_path, event_type):
now = datetime.now().strftime('%H:%M:%S')
source = None
for p, s in path_source_map.items():
if file_path.startswith(p) or file_path == p:
source = s
break
if not source:
return
try:
if source == 'kiro':
parser = KiroParser()
for sess in parser.parse_file(file_path):
storage.upsert_session(sess.session_id, sess.source, sess.file_path, sess.cwd, sess.messages, int(sess.file_mtime))
log.info(f"[{now}] kiro 更新")
elif source == 'claude':
parser = ClaudeParser()
for sess in parser.parse_file(file_path):
fp = f"claude:{sess.session_id}"
storage.upsert_session(sess.session_id, sess.source, fp, sess.cwd, sess.messages, int(sess.file_mtime))
log.info(f"[{now}] claude 更新")
else:
parser = CodexParser(source) if source == 'codex' else GeminiParser()
sess = parser.parse_file(file_path)
fp = os.path.abspath(sess.file_path)
storage.upsert_session(sess.session_id, sess.source, fp, sess.cwd, sess.messages, int(sess.file_mtime))
log.info(f"[{now}] {source} {event_type}: {os.path.basename(file_path)}")
update_cli_meta(source)
total = storage.get_total_stats()
storage.update_total_meta(total['sessions'], total['messages'], total['tokens'])
except Exception as e:
log.error(f"[{now}] 处理失败 {file_path}: {e}")
log.info(f"监听目录: {len(watch_paths)}")
watcher = ChatWatcher(watch_paths, on_change)
watcher.start()
def cmd_prune():
log = get_logger()
log.info("清理孤立记录...")
removed = storage.prune()
total = sum(removed.values())
if total > 0:
for cli, count in removed.items():
if count > 0:
log.info(f" {cli}: 删除 {count}")
log.info(f"✓ 共清理 {total} 条孤立记录")
else:
log.info("✓ 无孤立记录")
def cmd_stats():
log = get_logger()
meta = storage.get_total_meta()
tokens = storage.get_token_stats()
log.info("=" * 50)
log.info("统计信息")
log.info("=" * 50)
log.info(f"数据库: {CONFIG['db_path']}")
log.info(f"总会话: {meta['total_sessions']}")
log.info(f"总消息: {meta['total_messages']}")
log.info(f"最后同步: {meta['last_sync']}")
log.info("")
log.info("Token 统计 (tiktoken):")
total_tokens = 0
for source in ['codex', 'kiro', 'gemini', 'claude']:
t = tokens.get(source, 0)
if t > 0:
log.info(f" {source}: {t:,}")
total_tokens += t
log.info(f" 总计: {total_tokens:,}")
def cmd_search(keyword: str, source: str = None):
log = get_logger()
results = storage.search(keyword, source)
log.info(f"搜索 '{keyword}' 找到 {len(results)} 个会话:")
for r in results[:20]:
log.info(f" [{r['source']}] {r['session_id']} - {r['cwd'] or 'N/A'}")
def cmd_export(fmt: str, source: str = None, output: str = None):
log = get_logger()
if not output:
output = os.path.join(CONFIG["output_dir"], f"export.{fmt}")
if fmt == 'json':
count = storage.export_json(output, source)
else:
count = storage.export_csv(output, source)
log.info(f"✓ 导出 {count} 条到 {output}")
def print_token_stats():
log = get_logger()
tokens = storage.get_token_stats()
log.info("")
log.info("=== Token 统计 (tiktoken) ===")
total = 0
for source in ['codex', 'kiro', 'gemini', 'claude']:
t = tokens.get(source, 0)
if t > 0:
log.info(f" {source}: {t:,} tokens")
total += t
log.info(f" 总计: {total:,} tokens")
def update_cli_meta(cli: str):
stats = storage.get_cli_stats(cli)
path = CONFIG.get(f"{cli}_paths", [""])[0] if CONFIG.get(f"{cli}_paths") else ""
storage.update_cli_meta(cli, path, stats['sessions'], stats['messages'], stats['tokens'])
def process(parser, paths) -> tuple:
log = get_logger()
added, updated, skipped, errors = 0, 0, 0, 0
for f in parser.find_files(paths):
try:
s = parser.parse_file(f)
file_path = os.path.abspath(s.file_path)
db_mtime = storage.get_file_mtime(file_path)
file_mtime = int(s.file_mtime)
if db_mtime == 0:
storage.upsert_session(s.session_id, s.source, file_path, s.cwd, s.messages, file_mtime)
added += 1
elif file_mtime > db_mtime:
storage.upsert_session(s.session_id, s.source, file_path, s.cwd, s.messages, file_mtime)
updated += 1
else:
skipped += 1
except Exception as e:
log.debug(f"解析失败 {f}: {e}")
errors += 1
return added, updated, skipped, errors
def process_multi(parser, paths, source: str) -> tuple:
"""处理返回多个会话的解析器(Claude/Kiro"""
log = get_logger()
added, updated, skipped, errors = 0, 0, 0, 0
for f in parser.find_files(paths):
try:
for s in parser.parse_file(f):
file_path = s.file_path # kiro:xxx 或 claude:xxx
db_mtime = storage.get_file_mtime(file_path)
file_mtime = int(s.file_mtime)
if db_mtime == 0:
storage.upsert_session(s.session_id, s.source, file_path, s.cwd, s.messages, file_mtime)
added += 1
elif file_mtime > db_mtime:
storage.upsert_session(s.session_id, s.source, file_path, s.cwd, s.messages, file_mtime)
updated += 1
else:
skipped += 1
except Exception as e:
log.debug(f"解析失败 {f}: {e}")
errors += 1
return added, updated, skipped, errors
if __name__ == '__main__':
main()
-7
View File
@@ -1,7 +0,0 @@
from .codex import CodexParser
from .gemini import GeminiParser
from .claude import ClaudeParser
from .kiro import KiroParser
from .base import SessionData
__all__ = ["CodexParser", "GeminiParser", "ClaudeParser", "KiroParser", "SessionData"]
-24
View File
@@ -1,24 +0,0 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
from abc import ABC, abstractmethod
from dataclasses import dataclass, field
from typing import List, Dict
@dataclass
class SessionData:
"""会话数据"""
session_id: str
source: str
file_path: str
file_mtime: float = 0
cwd: str = None
messages: List[Dict] = field(default_factory=list) # [{"time", "role", "content"}]
class BaseParser(ABC):
@abstractmethod
def find_files(self, paths: list) -> list:
pass
@abstractmethod
def parse_file(self, filepath: str) -> SessionData:
pass
-54
View File
@@ -1,54 +0,0 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
import os
import json
import hashlib
from datetime import datetime
from collections import defaultdict
from .base import BaseParser, SessionData
class ClaudeParser(BaseParser):
def find_files(self, paths: list) -> list:
files = []
for base in paths:
history = os.path.join(base, "history.jsonl")
if os.path.exists(history):
files.append(history)
return files
def parse_file(self, filepath: str) -> list:
"""返回多个 SessionData(按 project 分组)"""
projects = defaultdict(list)
file_mtime = os.path.getmtime(filepath)
with open(filepath, 'r', encoding='utf-8') as f:
for line in f:
line = line.strip()
if not line:
continue
data = json.loads(line)
content = data.get('display', '')
if not content:
continue
project = data.get('project', 'unknown')
ts_ms = data.get('timestamp', 0)
ts = datetime.fromtimestamp(ts_ms / 1000).isoformat() if ts_ms else ''
projects[project].append({
'time': ts,
'role': 'user',
'content': content
})
return [
SessionData(
session_id='claude-' + hashlib.md5(proj.encode()).hexdigest()[:12],
source='claude',
file_path=filepath,
file_mtime=file_mtime,
cwd=proj,
messages=msgs
)
for proj, msgs in projects.items()
]
-70
View File
@@ -1,70 +0,0 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
import os
import json
import re
from .base import BaseParser, SessionData
class CodexParser(BaseParser):
def __init__(self, source: str = 'codex'):
self.source = source
def find_files(self, paths: list) -> list:
files = []
for base in paths:
if not os.path.exists(base):
continue
for root, _, names in os.walk(base):
for f in names:
if f.endswith('.jsonl') and f != 'history.jsonl':
files.append(os.path.join(root, f))
return files
def _extract_id(self, filepath: str) -> str:
match = re.search(r'([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})',
os.path.basename(filepath))
return match.group(1) if match else os.path.basename(filepath).replace('.jsonl', '')
def parse_file(self, filepath: str) -> SessionData:
s = SessionData(
session_id=self._extract_id(filepath),
source=self.source,
file_path=filepath,
file_mtime=os.path.getmtime(filepath)
)
with open(filepath, 'r', encoding='utf-8', errors='ignore') as f:
for line in f:
line = line.strip()
if not line or line[0] != '{':
continue
try:
data = json.loads(line)
except json.JSONDecodeError:
continue
if data.get('type') == 'session_meta':
p = data.get('payload', {})
s.cwd = p.get('cwd')
s.session_id = p.get('id', s.session_id)
continue
if data.get('type') != 'response_item':
continue
payload = data.get('payload', {})
if payload.get('type') != 'message':
continue
role = payload.get('role')
if role not in ('user', 'assistant'):
continue
parts = [item.get('text', '') for item in payload.get('content', [])
if isinstance(item, dict) and item.get('type') in ('input_text', 'output_text', 'text')]
if parts:
s.messages.append({
'time': data.get('timestamp', ''),
'role': 'user' if role == 'user' else 'ai',
'content': ' '.join(parts)
})
return s
-40
View File
@@ -1,40 +0,0 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
import os
import glob
import json
from .base import BaseParser, SessionData
class GeminiParser(BaseParser):
def find_files(self, paths: list) -> list:
files = []
for base in paths:
if os.path.exists(base):
files.extend(glob.glob(os.path.join(base, "*", "chats", "*.json")))
return files
def parse_file(self, filepath: str) -> SessionData:
s = SessionData(
session_id=os.path.basename(filepath).replace('.json', ''),
source='gemini',
file_path=filepath,
file_mtime=os.path.getmtime(filepath)
)
with open(filepath, 'r', encoding='utf-8') as f:
data = json.load(f)
s.session_id = data.get('sessionId', s.session_id)
for msg in data.get('messages', []):
if msg.get('type') not in ('user', 'gemini'):
continue
content = msg.get('content', '')
if content:
s.messages.append({
'time': msg.get('timestamp', ''),
'role': 'user' if msg.get('type') == 'user' else 'ai',
'content': content
})
return s
-75
View File
@@ -1,75 +0,0 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""Kiro CLI 解析器 - 从 SQLite 数据库读取"""
import os
import json
import sqlite3
import hashlib
from datetime import datetime
from .base import BaseParser, SessionData
KIRO_DB = os.path.expanduser("~/.local/share/kiro-cli/data.sqlite3")
class KiroParser(BaseParser):
def find_files(self, paths: list) -> list:
"""返回数据库路径(如果存在)"""
if os.path.exists(KIRO_DB):
return [KIRO_DB]
return []
def parse_file(self, filepath: str) -> list:
"""解析 Kiro SQLite 数据库,返回多个 SessionData"""
sessions = []
file_mtime = os.path.getmtime(filepath)
conn = sqlite3.connect(filepath)
for row in conn.execute('SELECT key, value FROM conversations'):
cwd, value = row
try:
data = json.loads(value)
except json.JSONDecodeError:
continue
conv_id = data.get('conversation_id', hashlib.md5(cwd.encode()).hexdigest()[:12])
history = data.get('history', [])
messages = []
for item in history:
# 用户消息
if 'user' in item:
user = item['user']
content = user.get('content', {})
if isinstance(content, dict) and 'Prompt' in content:
prompt = content['Prompt'].get('prompt', '')
if prompt:
messages.append({
'time': '',
'role': 'user',
'content': prompt
})
# AI 回复
if 'assistant' in item:
assistant = item['assistant']
content = assistant.get('content', {})
if isinstance(content, dict) and 'Message' in content:
msg = content['Message'].get('message', '')
if msg:
messages.append({
'time': '',
'role': 'ai',
'content': msg
})
if messages:
sessions.append(SessionData(
session_id=f'kiro-{conv_id[:12]}',
source='kiro',
file_path=f'kiro:{conv_id}',
file_mtime=file_mtime,
cwd=cwd,
messages=messages
))
conn.close()
return sessions
-246
View File
@@ -1,246 +0,0 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""SQLite 存储模块 - 完整版"""
import sqlite3
import json
import os
import datetime
import tiktoken
SCHEMA_VERSION = 5
CLIS = ('codex', 'kiro', 'gemini', 'claude')
_encoder = tiktoken.get_encoding("cl100k_base")
def count_tokens(text: str) -> int:
return len(_encoder.encode(text)) if text else 0
class ChatStorage:
def __init__(self, db_path: str):
self.db_path = db_path
os.makedirs(os.path.dirname(db_path) or '.', exist_ok=True)
self._init_db()
def _conn(self):
return sqlite3.connect(self.db_path)
def _init_db(self):
with self._conn() as conn:
conn.execute('''CREATE TABLE IF NOT EXISTS meta (key TEXT PRIMARY KEY, value TEXT)''')
for cli in CLIS:
conn.execute(f'''CREATE TABLE IF NOT EXISTS meta_{cli} (key TEXT PRIMARY KEY, value TEXT)''')
conn.execute('''
CREATE TABLE IF NOT EXISTS sessions (
file_path TEXT PRIMARY KEY,
session_id TEXT,
source TEXT NOT NULL,
cwd TEXT,
messages TEXT,
file_mtime INTEGER,
start_time TEXT,
token_count INTEGER DEFAULT 0
)
''')
conn.execute('CREATE INDEX IF NOT EXISTS idx_source ON sessions(source)')
conn.execute('CREATE INDEX IF NOT EXISTS idx_session_id ON sessions(session_id)')
conn.execute('CREATE INDEX IF NOT EXISTS idx_start_time ON sessions(start_time)')
self._set_meta('meta', 'schema_version', str(SCHEMA_VERSION))
def _set_meta(self, table: str, key: str, value: str):
with self._conn() as conn:
conn.execute(f'INSERT OR REPLACE INTO {table} (key, value) VALUES (?, ?)', (key, value))
def _get_meta(self, table: str, key: str) -> str:
with self._conn() as conn:
row = conn.execute(f'SELECT value FROM {table} WHERE key = ?', (key,)).fetchone()
return row[0] if row else None
def update_cli_meta(self, cli: str, path: str, sessions: int, messages: int, tokens: int = None):
table = f'meta_{cli}'
now = datetime.datetime.now().isoformat()
# 顺序: path, sessions, messages, total_tokens, last_sync
self._set_meta(table, 'path', path)
self._set_meta(table, 'sessions', str(sessions))
self._set_meta(table, 'messages', str(messages))
self._set_meta(table, 'total_tokens', str(tokens or 0))
self._set_meta(table, 'last_sync', now)
def update_total_meta(self, sessions: int, messages: int, tokens: int = None):
now = datetime.datetime.now().isoformat()
self._set_meta('meta', 'total_sessions', str(sessions))
self._set_meta('meta', 'total_messages', str(messages))
if tokens is not None:
self._set_meta('meta', 'total_tokens', str(tokens))
self._set_meta('meta', 'last_sync', now)
def get_total_meta(self) -> dict:
return {
'schema_version': int(self._get_meta('meta', 'schema_version') or 0),
'total_sessions': int(self._get_meta('meta', 'total_sessions') or 0),
'total_messages': int(self._get_meta('meta', 'total_messages') or 0),
'last_sync': self._get_meta('meta', 'last_sync'),
}
def get_file_mtime(self, file_path: str) -> int:
with self._conn() as conn:
row = conn.execute('SELECT file_mtime FROM sessions WHERE file_path = ?', (file_path,)).fetchone()
return row[0] if row else 0
def upsert_session(self, session_id: str, source: str, file_path: str,
cwd: str, messages: list, file_mtime: int, start_time: str = None):
if file_path and not file_path.startswith('claude:') and not os.path.isabs(file_path):
file_path = os.path.abspath(file_path)
total_tokens = sum(count_tokens(msg.get('content', '')) for msg in messages)
if not start_time and messages:
start_time = messages[0].get('time')
messages_json = json.dumps(messages, ensure_ascii=False)
with self._conn() as conn:
conn.execute('''
INSERT INTO sessions (file_path, session_id, source, cwd, messages, file_mtime, start_time, token_count)
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
ON CONFLICT(file_path) DO UPDATE SET
session_id=excluded.session_id, messages=excluded.messages,
file_mtime=excluded.file_mtime, start_time=excluded.start_time, token_count=excluded.token_count
''', (file_path, session_id, source, cwd, messages_json, file_mtime, start_time, total_tokens))
def get_cli_stats(self, cli: str) -> dict:
with self._conn() as conn:
sessions = conn.execute('SELECT COUNT(*) FROM sessions WHERE source = ?', (cli,)).fetchone()[0]
row = conn.execute('SELECT SUM(json_array_length(messages)) FROM sessions WHERE source = ?', (cli,)).fetchone()
messages = row[0] or 0
tokens = conn.execute('SELECT SUM(token_count) FROM sessions WHERE source = ?', (cli,)).fetchone()[0] or 0
return {'sessions': sessions, 'messages': messages, 'tokens': tokens}
def get_total_stats(self) -> dict:
with self._conn() as conn:
sessions = conn.execute('SELECT COUNT(*) FROM sessions').fetchone()[0]
row = conn.execute('SELECT SUM(json_array_length(messages)) FROM sessions').fetchone()
messages = row[0] or 0
tokens = conn.execute('SELECT SUM(token_count) FROM sessions').fetchone()[0] or 0
return {'sessions': sessions, 'messages': messages, 'tokens': tokens}
def get_token_stats(self) -> dict:
with self._conn() as conn:
rows = conn.execute('SELECT source, SUM(token_count) FROM sessions GROUP BY source').fetchall()
return {r[0]: r[1] or 0 for r in rows}
# === 清理孤立记录 ===
def prune(self) -> dict:
"""删除源文件已不存在的记录"""
removed = {'codex': 0, 'kiro': 0, 'gemini': 0, 'claude': 0}
with self._conn() as conn:
rows = conn.execute('SELECT file_path, source FROM sessions').fetchall()
for fp, source in rows:
if fp.startswith('claude:'):
continue # Claude 使用虚拟路径
if not os.path.exists(fp):
conn.execute('DELETE FROM sessions WHERE file_path = ?', (fp,))
removed[source] = removed.get(source, 0) + 1
return removed
# === 查询 ===
def search(self, keyword: str, source: str = None, limit: int = 50) -> list:
"""搜索消息内容"""
sql = "SELECT file_path, session_id, source, cwd, messages, start_time FROM sessions WHERE messages LIKE ?"
params = [f'%{keyword}%']
if source:
sql += " AND source = ?"
params.append(source)
sql += f" ORDER BY start_time DESC LIMIT {limit}"
results = []
with self._conn() as conn:
for row in conn.execute(sql, params):
results.append({
'file_path': row[0], 'session_id': row[1], 'source': row[2],
'cwd': row[3], 'messages': json.loads(row[4]), 'start_time': row[5]
})
return results
def get_session(self, file_path: str) -> dict:
"""获取单个会话"""
with self._conn() as conn:
row = conn.execute(
'SELECT file_path, session_id, source, cwd, messages, start_time, token_count FROM sessions WHERE file_path = ?',
(file_path,)
).fetchone()
if not row:
return None
return {
'file_path': row[0], 'session_id': row[1], 'source': row[2], 'cwd': row[3],
'messages': json.loads(row[4]), 'start_time': row[5], 'token_count': row[6]
}
def list_sessions(self, source: str = None, limit: int = 100, offset: int = 0) -> list:
"""列出会话"""
sql = "SELECT file_path, session_id, source, cwd, start_time, token_count FROM sessions"
params = []
if source:
sql += " WHERE source = ?"
params.append(source)
sql += f" ORDER BY start_time DESC LIMIT {limit} OFFSET {offset}"
results = []
with self._conn() as conn:
for row in conn.execute(sql, params):
results.append({
'file_path': row[0], 'session_id': row[1], 'source': row[2],
'cwd': row[3], 'start_time': row[4], 'token_count': row[5]
})
return results
# === 导出 ===
def export_json(self, output_path: str, source: str = None):
"""导出为 JSON"""
sql = "SELECT file_path, session_id, source, cwd, messages, start_time, token_count FROM sessions"
params = []
if source:
sql += " WHERE source = ?"
params.append(source)
sql += " ORDER BY start_time"
data = []
with self._conn() as conn:
for row in conn.execute(sql, params):
data.append({
'file_path': row[0], 'session_id': row[1], 'source': row[2], 'cwd': row[3],
'messages': json.loads(row[4]), 'start_time': row[5], 'token_count': row[6]
})
with open(output_path, 'w', encoding='utf-8') as f:
json.dump(data, f, ensure_ascii=False, indent=2)
return len(data)
def export_csv(self, output_path: str, source: str = None):
"""导出为 CSV(扁平化消息)"""
import csv
sql = "SELECT session_id, source, cwd, messages, start_time FROM sessions"
params = []
if source:
sql += " WHERE source = ?"
params.append(source)
sql += " ORDER BY start_time"
count = 0
with open(output_path, 'w', encoding='utf-8', newline='') as f:
writer = csv.writer(f)
writer.writerow(['session_id', 'source', 'cwd', 'time', 'role', 'content'])
with self._conn() as conn:
for row in conn.execute(sql, params):
session_id, src, cwd, msgs_json, _ = row
for msg in json.loads(msgs_json):
writer.writerow([session_id, src, cwd, msg.get('time', ''), msg.get('role', ''), msg.get('content', '')])
count += 1
return count
# === 获取所有文件路径(用于 prune 检查) ===
def get_all_file_paths(self, source: str = None) -> set:
sql = "SELECT file_path FROM sessions"
params = []
if source:
sql += " WHERE source = ?"
params.append(source)
with self._conn() as conn:
return {row[0] for row in conn.execute(sql, params)}
-44
View File
@@ -1,44 +0,0 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""跨平台文件监控 (Linux/macOS/Windows)"""
from watchdog.observers import Observer
from watchdog.events import FileSystemEventHandler
import time
class ChatFileHandler(FileSystemEventHandler):
def __init__(self, callback, extensions):
self.callback = callback
self.extensions = extensions
def _check(self, event):
if event.is_directory:
return
path = event.src_path
if any(path.endswith(ext) for ext in self.extensions):
self.callback(path, event.event_type)
def on_created(self, event):
self._check(event)
def on_modified(self, event):
self._check(event)
class ChatWatcher:
def __init__(self, paths: list, callback, extensions=('.jsonl', '.json')):
self.observer = Observer()
handler = ChatFileHandler(callback, extensions)
for path in paths:
self.observer.schedule(handler, path, recursive=True)
def start(self):
self.observer.start()
try:
while True:
time.sleep(1)
except KeyboardInterrupt:
self.stop()
def stop(self):
self.observer.stop()
self.observer.join()