mirror of
https://github.com/tradecatlabs/vibe-coding-cn.git
synced 2026-08-12 18:38:04 +00:00
refactor: Remove libs directory
This commit is contained in:
-69
@@ -1,69 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
r"""
|
||||
配置模块 - 智能路径识别
|
||||
支持: Linux 原生路径、WSL 路径 (\\wsl.localhost\Ubuntu\...)
|
||||
"""
|
||||
import os
|
||||
import re
|
||||
from dotenv import load_dotenv
|
||||
|
||||
# 项目目录
|
||||
PROJECT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
OUTPUT_DIR = os.path.join(PROJECT_DIR, "output")
|
||||
|
||||
load_dotenv(os.path.join(PROJECT_DIR, ".env"))
|
||||
|
||||
def convert_wsl_path(path: str) -> str:
|
||||
match = re.match(r'^\\\\wsl[.\$]?[^\\]*\\[^\\]+\\(.+)$', path, re.IGNORECASE)
|
||||
if match:
|
||||
return '/' + match.group(1).replace('\\', '/')
|
||||
return path
|
||||
|
||||
def normalize_path(path: str) -> str:
|
||||
path = path.strip()
|
||||
path = convert_wsl_path(path)
|
||||
return os.path.expanduser(path)
|
||||
|
||||
def get_paths(env_key: str) -> list:
|
||||
val = os.getenv(env_key, "")
|
||||
if not val:
|
||||
return []
|
||||
return [normalize_path(p) for p in val.split(",") if p.strip()]
|
||||
|
||||
def auto_detect_paths() -> dict:
|
||||
home = os.path.expanduser("~")
|
||||
kiro_db = os.path.join(home, ".local", "share", "kiro-cli")
|
||||
candidates = {
|
||||
"codex_paths": [os.path.join(home, ".codex", "sessions"), os.path.join(home, ".codex")],
|
||||
"kiro_paths": [kiro_db] if os.path.exists(kiro_db) else [],
|
||||
"gemini_paths": [os.path.join(home, ".gemini", "tmp"), os.path.join(home, ".gemini")],
|
||||
"claude_paths": [os.path.join(home, ".claude")],
|
||||
}
|
||||
detected = {}
|
||||
for key, paths in candidates.items():
|
||||
for p in paths:
|
||||
if os.path.exists(p):
|
||||
detected[key] = [p]
|
||||
break
|
||||
if key not in detected:
|
||||
detected[key] = []
|
||||
return detected
|
||||
|
||||
def load_config() -> dict:
|
||||
auto = auto_detect_paths()
|
||||
|
||||
os.makedirs(OUTPUT_DIR, exist_ok=True)
|
||||
os.makedirs(os.path.join(OUTPUT_DIR, "logs"), exist_ok=True)
|
||||
|
||||
return {
|
||||
"codex_paths": get_paths("CODEX_PATHS") or auto.get("codex_paths", []),
|
||||
"kiro_paths": get_paths("KIRO_PATHS") or auto.get("kiro_paths", []),
|
||||
"gemini_paths": get_paths("GEMINI_PATHS") or auto.get("gemini_paths", []),
|
||||
"claude_paths": get_paths("CLAUDE_PATHS") or auto.get("claude_paths", []),
|
||||
"output_dir": OUTPUT_DIR,
|
||||
"log_dir": os.path.join(OUTPUT_DIR, "logs"),
|
||||
"db_path": os.path.join(OUTPUT_DIR, "chat_history.db"),
|
||||
}
|
||||
|
||||
CONFIG = load_config()
|
||||
-42
@@ -1,42 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
"""日志模块 - 同时输出到控制台和文件"""
|
||||
import logging
|
||||
import os
|
||||
from datetime import datetime
|
||||
|
||||
_logger = None
|
||||
|
||||
def setup_logger(log_dir: str = None) -> logging.Logger:
|
||||
global _logger
|
||||
if _logger:
|
||||
return _logger
|
||||
|
||||
_logger = logging.getLogger('ai_chat_converter')
|
||||
_logger.setLevel(logging.DEBUG)
|
||||
_logger.handlers.clear()
|
||||
|
||||
fmt = logging.Formatter('%(asctime)s [%(levelname)s] %(message)s', datefmt='%Y-%m-%d %H:%M:%S')
|
||||
|
||||
# 控制台
|
||||
ch = logging.StreamHandler()
|
||||
ch.setLevel(logging.INFO)
|
||||
ch.setFormatter(fmt)
|
||||
_logger.addHandler(ch)
|
||||
|
||||
# 文件
|
||||
if log_dir:
|
||||
os.makedirs(log_dir, exist_ok=True)
|
||||
log_file = os.path.join(log_dir, f"sync_{datetime.now().strftime('%Y%m%d')}.log")
|
||||
fh = logging.FileHandler(log_file, encoding='utf-8')
|
||||
fh.setLevel(logging.DEBUG)
|
||||
fh.setFormatter(fmt)
|
||||
_logger.addHandler(fh)
|
||||
|
||||
return _logger
|
||||
|
||||
def get_logger() -> logging.Logger:
|
||||
global _logger
|
||||
if not _logger:
|
||||
_logger = setup_logger()
|
||||
return _logger
|
||||
Vendored
-319
@@ -1,319 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
AI 聊天记录集中存储工具
|
||||
|
||||
命令:
|
||||
python main.py # 同步一次
|
||||
python main.py --watch # 持续监控
|
||||
python main.py --prune # 清理孤立记录
|
||||
python main.py --stats # 显示统计
|
||||
python main.py --search <keyword> # 搜索
|
||||
python main.py --export json|csv [--source codex|kiro|gemini|claude]
|
||||
"""
|
||||
import os
|
||||
import sys
|
||||
import subprocess
|
||||
|
||||
# 项目根目录
|
||||
PROJECT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
VENV_DIR = os.path.join(PROJECT_DIR, '.venv')
|
||||
REQUIREMENTS = os.path.join(PROJECT_DIR, 'requirements.txt')
|
||||
|
||||
def ensure_venv():
|
||||
"""检测并创建虚拟环境,安装依赖"""
|
||||
# 打包版本跳过
|
||||
if getattr(sys, 'frozen', False):
|
||||
return
|
||||
|
||||
# 已在虚拟环境中运行则跳过
|
||||
if sys.prefix != sys.base_prefix:
|
||||
return
|
||||
|
||||
# 检查 .venv 是否存在
|
||||
venv_python = os.path.join(VENV_DIR, 'bin', 'python') if os.name != 'nt' else os.path.join(VENV_DIR, 'Scripts', 'python.exe')
|
||||
|
||||
if not os.path.exists(venv_python):
|
||||
print("首次运行,创建虚拟环境...")
|
||||
subprocess.run([sys.executable, '-m', 'venv', VENV_DIR], check=True)
|
||||
print("安装依赖...")
|
||||
pip = os.path.join(VENV_DIR, 'bin', 'pip') if os.name != 'nt' else os.path.join(VENV_DIR, 'Scripts', 'pip.exe')
|
||||
subprocess.run([pip, 'install', '-r', REQUIREMENTS, '-q'], check=True)
|
||||
print("环境准备完成,重新启动...\n")
|
||||
|
||||
# 使用虚拟环境重新执行
|
||||
os.execv(venv_python, [venv_python] + sys.argv)
|
||||
|
||||
# 启动前检测虚拟环境
|
||||
ensure_venv()
|
||||
|
||||
# 支持 PyInstaller 打包
|
||||
if getattr(sys, 'frozen', False):
|
||||
BASE_DIR = sys._MEIPASS
|
||||
sys.path.insert(0, os.path.join(BASE_DIR, 'src'))
|
||||
else:
|
||||
BASE_DIR = os.path.dirname(os.path.abspath(__file__))
|
||||
|
||||
import argparse
|
||||
from config import CONFIG
|
||||
from parsers import CodexParser, GeminiParser, ClaudeParser, KiroParser
|
||||
from storage import ChatStorage
|
||||
from logger import setup_logger, get_logger
|
||||
|
||||
storage: ChatStorage = None
|
||||
|
||||
def main():
|
||||
global storage
|
||||
|
||||
parser = argparse.ArgumentParser(description='AI Chat Converter')
|
||||
parser.add_argument('-w', '--watch', action='store_true', help='持续监控模式')
|
||||
parser.add_argument('--prune', action='store_true', help='清理孤立记录')
|
||||
parser.add_argument('--stats', action='store_true', help='显示统计信息')
|
||||
parser.add_argument('--search', type=str, help='搜索关键词')
|
||||
parser.add_argument('--export', choices=['json', 'csv'], help='导出格式')
|
||||
parser.add_argument('--source', choices=['codex', 'kiro', 'gemini', 'claude'], help='指定来源')
|
||||
parser.add_argument('--output', type=str, help='导出文件路径')
|
||||
args = parser.parse_args()
|
||||
|
||||
# 初始化
|
||||
setup_logger(CONFIG["log_dir"])
|
||||
log = get_logger()
|
||||
|
||||
storage = ChatStorage(CONFIG["db_path"])
|
||||
|
||||
# 命令分发
|
||||
if args.prune:
|
||||
cmd_prune()
|
||||
elif args.stats:
|
||||
cmd_stats()
|
||||
elif args.search:
|
||||
cmd_search(args.search, args.source)
|
||||
elif args.export:
|
||||
cmd_export(args.export, args.source, args.output)
|
||||
elif args.watch:
|
||||
cmd_sync()
|
||||
cmd_watch()
|
||||
else:
|
||||
cmd_sync()
|
||||
|
||||
def cmd_sync():
|
||||
log = get_logger()
|
||||
log.info("=" * 50)
|
||||
log.info("AI 聊天记录 → 集中存储")
|
||||
log.info("=" * 50)
|
||||
log.info(f"数据库: {CONFIG['db_path']}")
|
||||
|
||||
total_added, total_updated, total_skipped, total_errors = 0, 0, 0, 0
|
||||
|
||||
for cli, key, parser_cls in [
|
||||
('codex', 'codex_paths', lambda: CodexParser('codex')),
|
||||
('kiro', 'kiro_paths', KiroParser),
|
||||
('gemini', 'gemini_paths', GeminiParser),
|
||||
('claude', 'claude_paths', ClaudeParser),
|
||||
]:
|
||||
paths = CONFIG.get(key, [])
|
||||
if not paths:
|
||||
continue
|
||||
|
||||
parser = parser_cls()
|
||||
if cli in ('claude', 'kiro'):
|
||||
a, u, s, e = process_multi(parser, paths, cli)
|
||||
else:
|
||||
a, u, s, e = process(parser, paths)
|
||||
|
||||
log.info(f"[{cli.capitalize()}] 新增:{a} 更新:{u} 跳过:{s} 错误:{e}")
|
||||
update_cli_meta(cli)
|
||||
total_added += a
|
||||
total_updated += u
|
||||
total_skipped += s
|
||||
total_errors += e
|
||||
|
||||
total = storage.get_total_stats()
|
||||
storage.update_total_meta(total['sessions'], total['messages'], total['tokens'])
|
||||
|
||||
log.info("=" * 50)
|
||||
log.info(f"总计: {total['sessions']} 会话, {total['messages']} 消息")
|
||||
if total_errors > 0:
|
||||
log.warning(f"错误: {total_errors} 个文件解析失败")
|
||||
log.info("✓ 同步完成!")
|
||||
|
||||
print_token_stats()
|
||||
|
||||
def cmd_watch():
|
||||
from watcher import ChatWatcher
|
||||
from datetime import datetime
|
||||
|
||||
log = get_logger()
|
||||
log.info("")
|
||||
log.info("=" * 50)
|
||||
log.info("实时监听模式 (watchdog)")
|
||||
log.info("=" * 50)
|
||||
|
||||
watch_paths = []
|
||||
path_source_map = {}
|
||||
|
||||
for cli, key in [('codex', 'codex_paths'), ('kiro', 'kiro_paths'),
|
||||
('gemini', 'gemini_paths'), ('claude', 'claude_paths')]:
|
||||
for p in CONFIG.get(key, []):
|
||||
if os.path.isdir(p) or os.path.isfile(p):
|
||||
watch_paths.append(p)
|
||||
path_source_map[p] = cli
|
||||
|
||||
def on_change(file_path, event_type):
|
||||
now = datetime.now().strftime('%H:%M:%S')
|
||||
source = None
|
||||
for p, s in path_source_map.items():
|
||||
if file_path.startswith(p) or file_path == p:
|
||||
source = s
|
||||
break
|
||||
if not source:
|
||||
return
|
||||
|
||||
try:
|
||||
if source == 'kiro':
|
||||
parser = KiroParser()
|
||||
for sess in parser.parse_file(file_path):
|
||||
storage.upsert_session(sess.session_id, sess.source, sess.file_path, sess.cwd, sess.messages, int(sess.file_mtime))
|
||||
log.info(f"[{now}] kiro 更新")
|
||||
elif source == 'claude':
|
||||
parser = ClaudeParser()
|
||||
for sess in parser.parse_file(file_path):
|
||||
fp = f"claude:{sess.session_id}"
|
||||
storage.upsert_session(sess.session_id, sess.source, fp, sess.cwd, sess.messages, int(sess.file_mtime))
|
||||
log.info(f"[{now}] claude 更新")
|
||||
else:
|
||||
parser = CodexParser(source) if source == 'codex' else GeminiParser()
|
||||
sess = parser.parse_file(file_path)
|
||||
fp = os.path.abspath(sess.file_path)
|
||||
storage.upsert_session(sess.session_id, sess.source, fp, sess.cwd, sess.messages, int(sess.file_mtime))
|
||||
log.info(f"[{now}] {source} {event_type}: {os.path.basename(file_path)}")
|
||||
|
||||
update_cli_meta(source)
|
||||
total = storage.get_total_stats()
|
||||
storage.update_total_meta(total['sessions'], total['messages'], total['tokens'])
|
||||
except Exception as e:
|
||||
log.error(f"[{now}] 处理失败 {file_path}: {e}")
|
||||
|
||||
log.info(f"监听目录: {len(watch_paths)} 个")
|
||||
watcher = ChatWatcher(watch_paths, on_change)
|
||||
watcher.start()
|
||||
|
||||
def cmd_prune():
|
||||
log = get_logger()
|
||||
log.info("清理孤立记录...")
|
||||
removed = storage.prune()
|
||||
total = sum(removed.values())
|
||||
if total > 0:
|
||||
for cli, count in removed.items():
|
||||
if count > 0:
|
||||
log.info(f" {cli}: 删除 {count} 条")
|
||||
log.info(f"✓ 共清理 {total} 条孤立记录")
|
||||
else:
|
||||
log.info("✓ 无孤立记录")
|
||||
|
||||
def cmd_stats():
|
||||
log = get_logger()
|
||||
meta = storage.get_total_meta()
|
||||
tokens = storage.get_token_stats()
|
||||
|
||||
log.info("=" * 50)
|
||||
log.info("统计信息")
|
||||
log.info("=" * 50)
|
||||
log.info(f"数据库: {CONFIG['db_path']}")
|
||||
log.info(f"总会话: {meta['total_sessions']}")
|
||||
log.info(f"总消息: {meta['total_messages']}")
|
||||
log.info(f"最后同步: {meta['last_sync']}")
|
||||
log.info("")
|
||||
log.info("Token 统计 (tiktoken):")
|
||||
total_tokens = 0
|
||||
for source in ['codex', 'kiro', 'gemini', 'claude']:
|
||||
t = tokens.get(source, 0)
|
||||
if t > 0:
|
||||
log.info(f" {source}: {t:,}")
|
||||
total_tokens += t
|
||||
log.info(f" 总计: {total_tokens:,}")
|
||||
|
||||
def cmd_search(keyword: str, source: str = None):
|
||||
log = get_logger()
|
||||
results = storage.search(keyword, source)
|
||||
log.info(f"搜索 '{keyword}' 找到 {len(results)} 个会话:")
|
||||
for r in results[:20]:
|
||||
log.info(f" [{r['source']}] {r['session_id']} - {r['cwd'] or 'N/A'}")
|
||||
|
||||
def cmd_export(fmt: str, source: str = None, output: str = None):
|
||||
log = get_logger()
|
||||
if not output:
|
||||
output = os.path.join(CONFIG["output_dir"], f"export.{fmt}")
|
||||
|
||||
if fmt == 'json':
|
||||
count = storage.export_json(output, source)
|
||||
else:
|
||||
count = storage.export_csv(output, source)
|
||||
|
||||
log.info(f"✓ 导出 {count} 条到 {output}")
|
||||
|
||||
def print_token_stats():
|
||||
log = get_logger()
|
||||
tokens = storage.get_token_stats()
|
||||
log.info("")
|
||||
log.info("=== Token 统计 (tiktoken) ===")
|
||||
total = 0
|
||||
for source in ['codex', 'kiro', 'gemini', 'claude']:
|
||||
t = tokens.get(source, 0)
|
||||
if t > 0:
|
||||
log.info(f" {source}: {t:,} tokens")
|
||||
total += t
|
||||
log.info(f" 总计: {total:,} tokens")
|
||||
|
||||
def update_cli_meta(cli: str):
|
||||
stats = storage.get_cli_stats(cli)
|
||||
path = CONFIG.get(f"{cli}_paths", [""])[0] if CONFIG.get(f"{cli}_paths") else ""
|
||||
storage.update_cli_meta(cli, path, stats['sessions'], stats['messages'], stats['tokens'])
|
||||
|
||||
def process(parser, paths) -> tuple:
|
||||
log = get_logger()
|
||||
added, updated, skipped, errors = 0, 0, 0, 0
|
||||
for f in parser.find_files(paths):
|
||||
try:
|
||||
s = parser.parse_file(f)
|
||||
file_path = os.path.abspath(s.file_path)
|
||||
db_mtime = storage.get_file_mtime(file_path)
|
||||
file_mtime = int(s.file_mtime)
|
||||
if db_mtime == 0:
|
||||
storage.upsert_session(s.session_id, s.source, file_path, s.cwd, s.messages, file_mtime)
|
||||
added += 1
|
||||
elif file_mtime > db_mtime:
|
||||
storage.upsert_session(s.session_id, s.source, file_path, s.cwd, s.messages, file_mtime)
|
||||
updated += 1
|
||||
else:
|
||||
skipped += 1
|
||||
except Exception as e:
|
||||
log.debug(f"解析失败 {f}: {e}")
|
||||
errors += 1
|
||||
return added, updated, skipped, errors
|
||||
|
||||
def process_multi(parser, paths, source: str) -> tuple:
|
||||
"""处理返回多个会话的解析器(Claude/Kiro)"""
|
||||
log = get_logger()
|
||||
added, updated, skipped, errors = 0, 0, 0, 0
|
||||
for f in parser.find_files(paths):
|
||||
try:
|
||||
for s in parser.parse_file(f):
|
||||
file_path = s.file_path # kiro:xxx 或 claude:xxx
|
||||
db_mtime = storage.get_file_mtime(file_path)
|
||||
file_mtime = int(s.file_mtime)
|
||||
if db_mtime == 0:
|
||||
storage.upsert_session(s.session_id, s.source, file_path, s.cwd, s.messages, file_mtime)
|
||||
added += 1
|
||||
elif file_mtime > db_mtime:
|
||||
storage.upsert_session(s.session_id, s.source, file_path, s.cwd, s.messages, file_mtime)
|
||||
updated += 1
|
||||
else:
|
||||
skipped += 1
|
||||
except Exception as e:
|
||||
log.debug(f"解析失败 {f}: {e}")
|
||||
errors += 1
|
||||
return added, updated, skipped, errors
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -1,7 +0,0 @@
|
||||
from .codex import CodexParser
|
||||
from .gemini import GeminiParser
|
||||
from .claude import ClaudeParser
|
||||
from .kiro import KiroParser
|
||||
from .base import SessionData
|
||||
|
||||
__all__ = ["CodexParser", "GeminiParser", "ClaudeParser", "KiroParser", "SessionData"]
|
||||
-24
@@ -1,24 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
from abc import ABC, abstractmethod
|
||||
from dataclasses import dataclass, field
|
||||
from typing import List, Dict
|
||||
|
||||
@dataclass
|
||||
class SessionData:
|
||||
"""会话数据"""
|
||||
session_id: str
|
||||
source: str
|
||||
file_path: str
|
||||
file_mtime: float = 0
|
||||
cwd: str = None
|
||||
messages: List[Dict] = field(default_factory=list) # [{"time", "role", "content"}]
|
||||
|
||||
class BaseParser(ABC):
|
||||
@abstractmethod
|
||||
def find_files(self, paths: list) -> list:
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def parse_file(self, filepath: str) -> SessionData:
|
||||
pass
|
||||
-54
@@ -1,54 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
import os
|
||||
import json
|
||||
import hashlib
|
||||
from datetime import datetime
|
||||
from collections import defaultdict
|
||||
from .base import BaseParser, SessionData
|
||||
|
||||
class ClaudeParser(BaseParser):
|
||||
def find_files(self, paths: list) -> list:
|
||||
files = []
|
||||
for base in paths:
|
||||
history = os.path.join(base, "history.jsonl")
|
||||
if os.path.exists(history):
|
||||
files.append(history)
|
||||
return files
|
||||
|
||||
def parse_file(self, filepath: str) -> list:
|
||||
"""返回多个 SessionData(按 project 分组)"""
|
||||
projects = defaultdict(list)
|
||||
file_mtime = os.path.getmtime(filepath)
|
||||
|
||||
with open(filepath, 'r', encoding='utf-8') as f:
|
||||
for line in f:
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
data = json.loads(line)
|
||||
content = data.get('display', '')
|
||||
if not content:
|
||||
continue
|
||||
|
||||
project = data.get('project', 'unknown')
|
||||
ts_ms = data.get('timestamp', 0)
|
||||
ts = datetime.fromtimestamp(ts_ms / 1000).isoformat() if ts_ms else ''
|
||||
|
||||
projects[project].append({
|
||||
'time': ts,
|
||||
'role': 'user',
|
||||
'content': content
|
||||
})
|
||||
|
||||
return [
|
||||
SessionData(
|
||||
session_id='claude-' + hashlib.md5(proj.encode()).hexdigest()[:12],
|
||||
source='claude',
|
||||
file_path=filepath,
|
||||
file_mtime=file_mtime,
|
||||
cwd=proj,
|
||||
messages=msgs
|
||||
)
|
||||
for proj, msgs in projects.items()
|
||||
]
|
||||
-70
@@ -1,70 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
import os
|
||||
import json
|
||||
import re
|
||||
from .base import BaseParser, SessionData
|
||||
|
||||
class CodexParser(BaseParser):
|
||||
def __init__(self, source: str = 'codex'):
|
||||
self.source = source
|
||||
|
||||
def find_files(self, paths: list) -> list:
|
||||
files = []
|
||||
for base in paths:
|
||||
if not os.path.exists(base):
|
||||
continue
|
||||
for root, _, names in os.walk(base):
|
||||
for f in names:
|
||||
if f.endswith('.jsonl') and f != 'history.jsonl':
|
||||
files.append(os.path.join(root, f))
|
||||
return files
|
||||
|
||||
def _extract_id(self, filepath: str) -> str:
|
||||
match = re.search(r'([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})',
|
||||
os.path.basename(filepath))
|
||||
return match.group(1) if match else os.path.basename(filepath).replace('.jsonl', '')
|
||||
|
||||
def parse_file(self, filepath: str) -> SessionData:
|
||||
s = SessionData(
|
||||
session_id=self._extract_id(filepath),
|
||||
source=self.source,
|
||||
file_path=filepath,
|
||||
file_mtime=os.path.getmtime(filepath)
|
||||
)
|
||||
|
||||
with open(filepath, 'r', encoding='utf-8', errors='ignore') as f:
|
||||
for line in f:
|
||||
line = line.strip()
|
||||
if not line or line[0] != '{':
|
||||
continue
|
||||
try:
|
||||
data = json.loads(line)
|
||||
except json.JSONDecodeError:
|
||||
continue
|
||||
|
||||
if data.get('type') == 'session_meta':
|
||||
p = data.get('payload', {})
|
||||
s.cwd = p.get('cwd')
|
||||
s.session_id = p.get('id', s.session_id)
|
||||
continue
|
||||
|
||||
if data.get('type') != 'response_item':
|
||||
continue
|
||||
payload = data.get('payload', {})
|
||||
if payload.get('type') != 'message':
|
||||
continue
|
||||
role = payload.get('role')
|
||||
if role not in ('user', 'assistant'):
|
||||
continue
|
||||
|
||||
parts = [item.get('text', '') for item in payload.get('content', [])
|
||||
if isinstance(item, dict) and item.get('type') in ('input_text', 'output_text', 'text')]
|
||||
if parts:
|
||||
s.messages.append({
|
||||
'time': data.get('timestamp', ''),
|
||||
'role': 'user' if role == 'user' else 'ai',
|
||||
'content': ' '.join(parts)
|
||||
})
|
||||
|
||||
return s
|
||||
-40
@@ -1,40 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
import os
|
||||
import glob
|
||||
import json
|
||||
from .base import BaseParser, SessionData
|
||||
|
||||
class GeminiParser(BaseParser):
|
||||
def find_files(self, paths: list) -> list:
|
||||
files = []
|
||||
for base in paths:
|
||||
if os.path.exists(base):
|
||||
files.extend(glob.glob(os.path.join(base, "*", "chats", "*.json")))
|
||||
return files
|
||||
|
||||
def parse_file(self, filepath: str) -> SessionData:
|
||||
s = SessionData(
|
||||
session_id=os.path.basename(filepath).replace('.json', ''),
|
||||
source='gemini',
|
||||
file_path=filepath,
|
||||
file_mtime=os.path.getmtime(filepath)
|
||||
)
|
||||
|
||||
with open(filepath, 'r', encoding='utf-8') as f:
|
||||
data = json.load(f)
|
||||
|
||||
s.session_id = data.get('sessionId', s.session_id)
|
||||
|
||||
for msg in data.get('messages', []):
|
||||
if msg.get('type') not in ('user', 'gemini'):
|
||||
continue
|
||||
content = msg.get('content', '')
|
||||
if content:
|
||||
s.messages.append({
|
||||
'time': msg.get('timestamp', ''),
|
||||
'role': 'user' if msg.get('type') == 'user' else 'ai',
|
||||
'content': content
|
||||
})
|
||||
|
||||
return s
|
||||
-75
@@ -1,75 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
"""Kiro CLI 解析器 - 从 SQLite 数据库读取"""
|
||||
import os
|
||||
import json
|
||||
import sqlite3
|
||||
import hashlib
|
||||
from datetime import datetime
|
||||
from .base import BaseParser, SessionData
|
||||
|
||||
KIRO_DB = os.path.expanduser("~/.local/share/kiro-cli/data.sqlite3")
|
||||
|
||||
class KiroParser(BaseParser):
|
||||
def find_files(self, paths: list) -> list:
|
||||
"""返回数据库路径(如果存在)"""
|
||||
if os.path.exists(KIRO_DB):
|
||||
return [KIRO_DB]
|
||||
return []
|
||||
|
||||
def parse_file(self, filepath: str) -> list:
|
||||
"""解析 Kiro SQLite 数据库,返回多个 SessionData"""
|
||||
sessions = []
|
||||
file_mtime = os.path.getmtime(filepath)
|
||||
|
||||
conn = sqlite3.connect(filepath)
|
||||
for row in conn.execute('SELECT key, value FROM conversations'):
|
||||
cwd, value = row
|
||||
try:
|
||||
data = json.loads(value)
|
||||
except json.JSONDecodeError:
|
||||
continue
|
||||
|
||||
conv_id = data.get('conversation_id', hashlib.md5(cwd.encode()).hexdigest()[:12])
|
||||
history = data.get('history', [])
|
||||
|
||||
messages = []
|
||||
for item in history:
|
||||
# 用户消息
|
||||
if 'user' in item:
|
||||
user = item['user']
|
||||
content = user.get('content', {})
|
||||
if isinstance(content, dict) and 'Prompt' in content:
|
||||
prompt = content['Prompt'].get('prompt', '')
|
||||
if prompt:
|
||||
messages.append({
|
||||
'time': '',
|
||||
'role': 'user',
|
||||
'content': prompt
|
||||
})
|
||||
|
||||
# AI 回复
|
||||
if 'assistant' in item:
|
||||
assistant = item['assistant']
|
||||
content = assistant.get('content', {})
|
||||
if isinstance(content, dict) and 'Message' in content:
|
||||
msg = content['Message'].get('message', '')
|
||||
if msg:
|
||||
messages.append({
|
||||
'time': '',
|
||||
'role': 'ai',
|
||||
'content': msg
|
||||
})
|
||||
|
||||
if messages:
|
||||
sessions.append(SessionData(
|
||||
session_id=f'kiro-{conv_id[:12]}',
|
||||
source='kiro',
|
||||
file_path=f'kiro:{conv_id}',
|
||||
file_mtime=file_mtime,
|
||||
cwd=cwd,
|
||||
messages=messages
|
||||
))
|
||||
|
||||
conn.close()
|
||||
return sessions
|
||||
-246
@@ -1,246 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
"""SQLite 存储模块 - 完整版"""
|
||||
import sqlite3
|
||||
import json
|
||||
import os
|
||||
import datetime
|
||||
import tiktoken
|
||||
|
||||
SCHEMA_VERSION = 5
|
||||
CLIS = ('codex', 'kiro', 'gemini', 'claude')
|
||||
|
||||
_encoder = tiktoken.get_encoding("cl100k_base")
|
||||
|
||||
def count_tokens(text: str) -> int:
|
||||
return len(_encoder.encode(text)) if text else 0
|
||||
|
||||
class ChatStorage:
|
||||
def __init__(self, db_path: str):
|
||||
self.db_path = db_path
|
||||
os.makedirs(os.path.dirname(db_path) or '.', exist_ok=True)
|
||||
self._init_db()
|
||||
|
||||
def _conn(self):
|
||||
return sqlite3.connect(self.db_path)
|
||||
|
||||
def _init_db(self):
|
||||
with self._conn() as conn:
|
||||
conn.execute('''CREATE TABLE IF NOT EXISTS meta (key TEXT PRIMARY KEY, value TEXT)''')
|
||||
for cli in CLIS:
|
||||
conn.execute(f'''CREATE TABLE IF NOT EXISTS meta_{cli} (key TEXT PRIMARY KEY, value TEXT)''')
|
||||
conn.execute('''
|
||||
CREATE TABLE IF NOT EXISTS sessions (
|
||||
file_path TEXT PRIMARY KEY,
|
||||
session_id TEXT,
|
||||
source TEXT NOT NULL,
|
||||
cwd TEXT,
|
||||
messages TEXT,
|
||||
file_mtime INTEGER,
|
||||
start_time TEXT,
|
||||
token_count INTEGER DEFAULT 0
|
||||
)
|
||||
''')
|
||||
conn.execute('CREATE INDEX IF NOT EXISTS idx_source ON sessions(source)')
|
||||
conn.execute('CREATE INDEX IF NOT EXISTS idx_session_id ON sessions(session_id)')
|
||||
conn.execute('CREATE INDEX IF NOT EXISTS idx_start_time ON sessions(start_time)')
|
||||
self._set_meta('meta', 'schema_version', str(SCHEMA_VERSION))
|
||||
|
||||
def _set_meta(self, table: str, key: str, value: str):
|
||||
with self._conn() as conn:
|
||||
conn.execute(f'INSERT OR REPLACE INTO {table} (key, value) VALUES (?, ?)', (key, value))
|
||||
|
||||
def _get_meta(self, table: str, key: str) -> str:
|
||||
with self._conn() as conn:
|
||||
row = conn.execute(f'SELECT value FROM {table} WHERE key = ?', (key,)).fetchone()
|
||||
return row[0] if row else None
|
||||
|
||||
def update_cli_meta(self, cli: str, path: str, sessions: int, messages: int, tokens: int = None):
|
||||
table = f'meta_{cli}'
|
||||
now = datetime.datetime.now().isoformat()
|
||||
# 顺序: path, sessions, messages, total_tokens, last_sync
|
||||
self._set_meta(table, 'path', path)
|
||||
self._set_meta(table, 'sessions', str(sessions))
|
||||
self._set_meta(table, 'messages', str(messages))
|
||||
self._set_meta(table, 'total_tokens', str(tokens or 0))
|
||||
self._set_meta(table, 'last_sync', now)
|
||||
|
||||
def update_total_meta(self, sessions: int, messages: int, tokens: int = None):
|
||||
now = datetime.datetime.now().isoformat()
|
||||
self._set_meta('meta', 'total_sessions', str(sessions))
|
||||
self._set_meta('meta', 'total_messages', str(messages))
|
||||
if tokens is not None:
|
||||
self._set_meta('meta', 'total_tokens', str(tokens))
|
||||
self._set_meta('meta', 'last_sync', now)
|
||||
|
||||
def get_total_meta(self) -> dict:
|
||||
return {
|
||||
'schema_version': int(self._get_meta('meta', 'schema_version') or 0),
|
||||
'total_sessions': int(self._get_meta('meta', 'total_sessions') or 0),
|
||||
'total_messages': int(self._get_meta('meta', 'total_messages') or 0),
|
||||
'last_sync': self._get_meta('meta', 'last_sync'),
|
||||
}
|
||||
|
||||
def get_file_mtime(self, file_path: str) -> int:
|
||||
with self._conn() as conn:
|
||||
row = conn.execute('SELECT file_mtime FROM sessions WHERE file_path = ?', (file_path,)).fetchone()
|
||||
return row[0] if row else 0
|
||||
|
||||
def upsert_session(self, session_id: str, source: str, file_path: str,
|
||||
cwd: str, messages: list, file_mtime: int, start_time: str = None):
|
||||
if file_path and not file_path.startswith('claude:') and not os.path.isabs(file_path):
|
||||
file_path = os.path.abspath(file_path)
|
||||
|
||||
total_tokens = sum(count_tokens(msg.get('content', '')) for msg in messages)
|
||||
if not start_time and messages:
|
||||
start_time = messages[0].get('time')
|
||||
|
||||
messages_json = json.dumps(messages, ensure_ascii=False)
|
||||
with self._conn() as conn:
|
||||
conn.execute('''
|
||||
INSERT INTO sessions (file_path, session_id, source, cwd, messages, file_mtime, start_time, token_count)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
|
||||
ON CONFLICT(file_path) DO UPDATE SET
|
||||
session_id=excluded.session_id, messages=excluded.messages,
|
||||
file_mtime=excluded.file_mtime, start_time=excluded.start_time, token_count=excluded.token_count
|
||||
''', (file_path, session_id, source, cwd, messages_json, file_mtime, start_time, total_tokens))
|
||||
|
||||
def get_cli_stats(self, cli: str) -> dict:
|
||||
with self._conn() as conn:
|
||||
sessions = conn.execute('SELECT COUNT(*) FROM sessions WHERE source = ?', (cli,)).fetchone()[0]
|
||||
row = conn.execute('SELECT SUM(json_array_length(messages)) FROM sessions WHERE source = ?', (cli,)).fetchone()
|
||||
messages = row[0] or 0
|
||||
tokens = conn.execute('SELECT SUM(token_count) FROM sessions WHERE source = ?', (cli,)).fetchone()[0] or 0
|
||||
return {'sessions': sessions, 'messages': messages, 'tokens': tokens}
|
||||
|
||||
def get_total_stats(self) -> dict:
|
||||
with self._conn() as conn:
|
||||
sessions = conn.execute('SELECT COUNT(*) FROM sessions').fetchone()[0]
|
||||
row = conn.execute('SELECT SUM(json_array_length(messages)) FROM sessions').fetchone()
|
||||
messages = row[0] or 0
|
||||
tokens = conn.execute('SELECT SUM(token_count) FROM sessions').fetchone()[0] or 0
|
||||
return {'sessions': sessions, 'messages': messages, 'tokens': tokens}
|
||||
|
||||
def get_token_stats(self) -> dict:
|
||||
with self._conn() as conn:
|
||||
rows = conn.execute('SELECT source, SUM(token_count) FROM sessions GROUP BY source').fetchall()
|
||||
return {r[0]: r[1] or 0 for r in rows}
|
||||
|
||||
# === 清理孤立记录 ===
|
||||
def prune(self) -> dict:
|
||||
"""删除源文件已不存在的记录"""
|
||||
removed = {'codex': 0, 'kiro': 0, 'gemini': 0, 'claude': 0}
|
||||
with self._conn() as conn:
|
||||
rows = conn.execute('SELECT file_path, source FROM sessions').fetchall()
|
||||
for fp, source in rows:
|
||||
if fp.startswith('claude:'):
|
||||
continue # Claude 使用虚拟路径
|
||||
if not os.path.exists(fp):
|
||||
conn.execute('DELETE FROM sessions WHERE file_path = ?', (fp,))
|
||||
removed[source] = removed.get(source, 0) + 1
|
||||
return removed
|
||||
|
||||
# === 查询 ===
|
||||
def search(self, keyword: str, source: str = None, limit: int = 50) -> list:
|
||||
"""搜索消息内容"""
|
||||
sql = "SELECT file_path, session_id, source, cwd, messages, start_time FROM sessions WHERE messages LIKE ?"
|
||||
params = [f'%{keyword}%']
|
||||
if source:
|
||||
sql += " AND source = ?"
|
||||
params.append(source)
|
||||
sql += f" ORDER BY start_time DESC LIMIT {limit}"
|
||||
|
||||
results = []
|
||||
with self._conn() as conn:
|
||||
for row in conn.execute(sql, params):
|
||||
results.append({
|
||||
'file_path': row[0], 'session_id': row[1], 'source': row[2],
|
||||
'cwd': row[3], 'messages': json.loads(row[4]), 'start_time': row[5]
|
||||
})
|
||||
return results
|
||||
|
||||
def get_session(self, file_path: str) -> dict:
|
||||
"""获取单个会话"""
|
||||
with self._conn() as conn:
|
||||
row = conn.execute(
|
||||
'SELECT file_path, session_id, source, cwd, messages, start_time, token_count FROM sessions WHERE file_path = ?',
|
||||
(file_path,)
|
||||
).fetchone()
|
||||
if not row:
|
||||
return None
|
||||
return {
|
||||
'file_path': row[0], 'session_id': row[1], 'source': row[2], 'cwd': row[3],
|
||||
'messages': json.loads(row[4]), 'start_time': row[5], 'token_count': row[6]
|
||||
}
|
||||
|
||||
def list_sessions(self, source: str = None, limit: int = 100, offset: int = 0) -> list:
|
||||
"""列出会话"""
|
||||
sql = "SELECT file_path, session_id, source, cwd, start_time, token_count FROM sessions"
|
||||
params = []
|
||||
if source:
|
||||
sql += " WHERE source = ?"
|
||||
params.append(source)
|
||||
sql += f" ORDER BY start_time DESC LIMIT {limit} OFFSET {offset}"
|
||||
|
||||
results = []
|
||||
with self._conn() as conn:
|
||||
for row in conn.execute(sql, params):
|
||||
results.append({
|
||||
'file_path': row[0], 'session_id': row[1], 'source': row[2],
|
||||
'cwd': row[3], 'start_time': row[4], 'token_count': row[5]
|
||||
})
|
||||
return results
|
||||
|
||||
# === 导出 ===
|
||||
def export_json(self, output_path: str, source: str = None):
|
||||
"""导出为 JSON"""
|
||||
sql = "SELECT file_path, session_id, source, cwd, messages, start_time, token_count FROM sessions"
|
||||
params = []
|
||||
if source:
|
||||
sql += " WHERE source = ?"
|
||||
params.append(source)
|
||||
sql += " ORDER BY start_time"
|
||||
|
||||
data = []
|
||||
with self._conn() as conn:
|
||||
for row in conn.execute(sql, params):
|
||||
data.append({
|
||||
'file_path': row[0], 'session_id': row[1], 'source': row[2], 'cwd': row[3],
|
||||
'messages': json.loads(row[4]), 'start_time': row[5], 'token_count': row[6]
|
||||
})
|
||||
|
||||
with open(output_path, 'w', encoding='utf-8') as f:
|
||||
json.dump(data, f, ensure_ascii=False, indent=2)
|
||||
return len(data)
|
||||
|
||||
def export_csv(self, output_path: str, source: str = None):
|
||||
"""导出为 CSV(扁平化消息)"""
|
||||
import csv
|
||||
sql = "SELECT session_id, source, cwd, messages, start_time FROM sessions"
|
||||
params = []
|
||||
if source:
|
||||
sql += " WHERE source = ?"
|
||||
params.append(source)
|
||||
sql += " ORDER BY start_time"
|
||||
|
||||
count = 0
|
||||
with open(output_path, 'w', encoding='utf-8', newline='') as f:
|
||||
writer = csv.writer(f)
|
||||
writer.writerow(['session_id', 'source', 'cwd', 'time', 'role', 'content'])
|
||||
with self._conn() as conn:
|
||||
for row in conn.execute(sql, params):
|
||||
session_id, src, cwd, msgs_json, _ = row
|
||||
for msg in json.loads(msgs_json):
|
||||
writer.writerow([session_id, src, cwd, msg.get('time', ''), msg.get('role', ''), msg.get('content', '')])
|
||||
count += 1
|
||||
return count
|
||||
|
||||
# === 获取所有文件路径(用于 prune 检查) ===
|
||||
def get_all_file_paths(self, source: str = None) -> set:
|
||||
sql = "SELECT file_path FROM sessions"
|
||||
params = []
|
||||
if source:
|
||||
sql += " WHERE source = ?"
|
||||
params.append(source)
|
||||
with self._conn() as conn:
|
||||
return {row[0] for row in conn.execute(sql, params)}
|
||||
-44
@@ -1,44 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
"""跨平台文件监控 (Linux/macOS/Windows)"""
|
||||
|
||||
from watchdog.observers import Observer
|
||||
from watchdog.events import FileSystemEventHandler
|
||||
import time
|
||||
|
||||
class ChatFileHandler(FileSystemEventHandler):
|
||||
def __init__(self, callback, extensions):
|
||||
self.callback = callback
|
||||
self.extensions = extensions
|
||||
|
||||
def _check(self, event):
|
||||
if event.is_directory:
|
||||
return
|
||||
path = event.src_path
|
||||
if any(path.endswith(ext) for ext in self.extensions):
|
||||
self.callback(path, event.event_type)
|
||||
|
||||
def on_created(self, event):
|
||||
self._check(event)
|
||||
|
||||
def on_modified(self, event):
|
||||
self._check(event)
|
||||
|
||||
class ChatWatcher:
|
||||
def __init__(self, paths: list, callback, extensions=('.jsonl', '.json')):
|
||||
self.observer = Observer()
|
||||
handler = ChatFileHandler(callback, extensions)
|
||||
for path in paths:
|
||||
self.observer.schedule(handler, path, recursive=True)
|
||||
|
||||
def start(self):
|
||||
self.observer.start()
|
||||
try:
|
||||
while True:
|
||||
time.sleep(1)
|
||||
except KeyboardInterrupt:
|
||||
self.stop()
|
||||
|
||||
def stop(self):
|
||||
self.observer.stop()
|
||||
self.observer.join()
|
||||
Reference in New Issue
Block a user