mirror of
https://github.com/wilsonfreitas/awesome-quant.git
synced 2026-07-27 20:07:50 +00:00
Add Codex workflow support
This commit is contained in:
Wilson Freitas
@@ -0,0 +1,117 @@
|
||||
"""Shared README entry parsing helpers for awesome-quant."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Iterable
|
||||
|
||||
ENTRY_RE = re.compile(r"^\s*- \[(.*)\]\((.*)\) - (.*)$")
|
||||
HEADING_RE = re.compile(r"^(#+) (.*)$")
|
||||
BADGE_RE = re.compile(r"\s*!\[[^\]]*\]\([^)]*\)\s*")
|
||||
LANGUAGE_PREFIX_RE = re.compile(r"^((?:`[^`]+`\s*)+)-\s*(.*)$")
|
||||
GITHUB_LINK_RE = re.compile(r"\[GitHub\]\((https://github\.com/[\w-]+/[-\w.]+)\)")
|
||||
MARKDOWN_URL_RE = re.compile(r"\[[^\]]+\]\(([^)]+)\)")
|
||||
|
||||
NO_LANGUAGE_REQUIRED_SECTIONS = {
|
||||
"Commercial & Proprietary Services",
|
||||
"Cross-Language Frameworks",
|
||||
"Reproducing Works, Training & Books",
|
||||
"Related Lists",
|
||||
}
|
||||
|
||||
VALID_SECTIONS = [
|
||||
"Numerical Libraries & Data Structures",
|
||||
"Financial Instruments & Pricing",
|
||||
"Technical Indicators",
|
||||
"Trading & Backtesting",
|
||||
"Portfolio Optimization & Risk Analysis",
|
||||
"Factor Analysis",
|
||||
"Sentiment Analysis & Alternative Data",
|
||||
"Time Series Analysis",
|
||||
"Market Data & Data Sources",
|
||||
"Prediction Markets",
|
||||
"Calendars & Market Hours",
|
||||
"Visualization",
|
||||
"Excel & Spreadsheet Integration",
|
||||
"Quant Research Environments",
|
||||
"Cross-Language Frameworks",
|
||||
"Reproducing Works, Training & Books",
|
||||
"Commercial & Proprietary Services",
|
||||
"Related Lists",
|
||||
]
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ReadmeEntry:
|
||||
line_number: int
|
||||
raw_line: str
|
||||
section: str
|
||||
name: str
|
||||
url: str
|
||||
tail: str
|
||||
languages: list[str]
|
||||
description: str
|
||||
|
||||
@property
|
||||
def markdown_urls(self) -> list[str]:
|
||||
return MARKDOWN_URL_RE.findall(self.raw_line)
|
||||
|
||||
@property
|
||||
def github_url(self) -> str:
|
||||
if "github.com" in self.url:
|
||||
return self.url
|
||||
match = GITHUB_LINK_RE.search(self.description)
|
||||
return match.group(1) if match else ""
|
||||
|
||||
|
||||
def slugify(text: str) -> str:
|
||||
"""Convert text to lowercase hyphen-separated slug."""
|
||||
text = text.lower().strip()
|
||||
text = re.sub(r"[&/]+", "-", text)
|
||||
text = re.sub(r"[^\w\s-]", "", text)
|
||||
text = re.sub(r"[\s_]+", "-", text)
|
||||
text = re.sub(r"-+", "-", text)
|
||||
return text.strip("-")
|
||||
|
||||
|
||||
def extract_languages(description: str) -> tuple[list[str], str]:
|
||||
"""Extract leading backtick language tags from an entry description."""
|
||||
match = LANGUAGE_PREFIX_RE.match(description)
|
||||
if not match:
|
||||
return [], description
|
||||
lang_str = match.group(1)
|
||||
clean_description = match.group(2)
|
||||
return re.findall(r"`([^`]+)`", lang_str), clean_description
|
||||
|
||||
|
||||
def iter_readme_entries(path: str | Path) -> Iterable[ReadmeEntry]:
|
||||
"""Yield parsed README entries with line numbers and current h2 section."""
|
||||
current_section = ""
|
||||
with Path(path).open("r", encoding="utf-8") as handle:
|
||||
for line_number, raw_line in enumerate(handle, 1):
|
||||
line = BADGE_RE.sub(" ", raw_line.rstrip("\n"))
|
||||
heading = HEADING_RE.match(line)
|
||||
if heading:
|
||||
level, title = heading.groups()
|
||||
if len(level) == 2 and title.strip() != "Contents":
|
||||
current_section = title.strip()
|
||||
continue
|
||||
|
||||
match = ENTRY_RE.match(line)
|
||||
if not match:
|
||||
continue
|
||||
|
||||
tail = match.group(3).strip()
|
||||
languages, description = extract_languages(tail)
|
||||
yield ReadmeEntry(
|
||||
line_number=line_number,
|
||||
raw_line=raw_line.rstrip("\n"),
|
||||
section=current_section,
|
||||
name=match.group(1).strip(),
|
||||
url=match.group(2).strip(),
|
||||
tail=tail,
|
||||
languages=languages,
|
||||
description=description.strip(),
|
||||
)
|
||||
Executable
+260
@@ -0,0 +1,260 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Validate awesome-quant README entries.
|
||||
|
||||
Default mode validates the full README and reports legacy duplicate/http URL
|
||||
issues as warnings. With --diff-from, only added README entry lines are checked
|
||||
and duplicate/url issues are errors for those additions.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
from collections import defaultdict
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
if str(ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(ROOT))
|
||||
|
||||
from scripts.readme_entries import (
|
||||
ENTRY_RE,
|
||||
GITHUB_LINK_RE,
|
||||
HEADING_RE,
|
||||
MARKDOWN_URL_RE,
|
||||
NO_LANGUAGE_REQUIRED_SECTIONS,
|
||||
VALID_SECTIONS,
|
||||
ReadmeEntry,
|
||||
extract_languages,
|
||||
iter_readme_entries,
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Issue:
|
||||
severity: str
|
||||
line_number: int
|
||||
code: str
|
||||
message: str
|
||||
|
||||
|
||||
def normalize_name(name: str) -> str:
|
||||
return " ".join(name.casefold().split())
|
||||
|
||||
|
||||
def normalize_url(url: str) -> str:
|
||||
return url.strip().rstrip("/").casefold()
|
||||
|
||||
|
||||
def read_added_line_numbers(diff_from: str, readme_path: Path) -> set[int]:
|
||||
path_arg = str(readme_path)
|
||||
commands = [
|
||||
["git", "diff", "--unified=0", f"{diff_from}...HEAD", "--", path_arg],
|
||||
["git", "diff", "--unified=0", diff_from, "--", path_arg],
|
||||
]
|
||||
last_error = ""
|
||||
for command in commands:
|
||||
proc = subprocess.run(command, text=True, capture_output=True, check=False)
|
||||
if proc.returncode == 0:
|
||||
return parse_added_lines_from_diff(proc.stdout)
|
||||
last_error = proc.stderr.strip()
|
||||
raise RuntimeError(f"git diff failed for {diff_from}: {last_error}")
|
||||
|
||||
|
||||
def parse_added_lines_from_diff(diff_text: str) -> set[int]:
|
||||
added: set[int] = set()
|
||||
new_line: int | None = None
|
||||
|
||||
for line in diff_text.splitlines():
|
||||
if line.startswith("@@ "):
|
||||
marker = line.split(" ")[2]
|
||||
start = marker.split(",", 1)[0].lstrip("+")
|
||||
new_line = int(start)
|
||||
continue
|
||||
|
||||
if new_line is None:
|
||||
continue
|
||||
|
||||
if line.startswith("+++") or line.startswith("---"):
|
||||
continue
|
||||
if line.startswith("+"):
|
||||
added.add(new_line)
|
||||
new_line += 1
|
||||
elif line.startswith("-"):
|
||||
continue
|
||||
else:
|
||||
new_line += 1
|
||||
|
||||
return added
|
||||
|
||||
|
||||
def build_duplicate_indexes(entries: list[ReadmeEntry]) -> tuple[dict[str, list[ReadmeEntry]], dict[str, list[ReadmeEntry]]]:
|
||||
names: dict[str, list[ReadmeEntry]] = defaultdict(list)
|
||||
urls: dict[str, list[ReadmeEntry]] = defaultdict(list)
|
||||
for entry in entries:
|
||||
names[normalize_name(entry.name)].append(entry)
|
||||
for url in entry.markdown_urls:
|
||||
urls[normalize_url(url)].append(entry)
|
||||
return names, urls
|
||||
|
||||
|
||||
def validate_entry(
|
||||
entry: ReadmeEntry,
|
||||
*,
|
||||
duplicate_names: dict[str, list[ReadmeEntry]],
|
||||
duplicate_urls: dict[str, list[ReadmeEntry]],
|
||||
strict_duplicates: bool,
|
||||
strict_urls: bool,
|
||||
strict_format: bool,
|
||||
) -> list[Issue]:
|
||||
issues: list[Issue] = []
|
||||
line = entry.line_number
|
||||
|
||||
if entry.section not in VALID_SECTIONS:
|
||||
issues.append(Issue("error", line, "section", f"entry is under unknown section {entry.section!r}"))
|
||||
|
||||
format_severity = "error" if strict_format else "warning"
|
||||
|
||||
if entry.section not in NO_LANGUAGE_REQUIRED_SECTIONS and not entry.languages:
|
||||
issues.append(Issue(format_severity, line, "language", "missing required backtick language tag prefix"))
|
||||
|
||||
if entry.languages:
|
||||
_languages, clean_description = extract_languages(entry.tail)
|
||||
if clean_description == entry.tail:
|
||||
issues.append(Issue(format_severity, line, "language", "language tags must be followed by ' - '"))
|
||||
|
||||
github_label_count = entry.description.count("[GitHub](")
|
||||
if github_label_count and not GITHUB_LINK_RE.search(entry.description):
|
||||
issues.append(Issue(format_severity, line, "github-link", "optional GitHub link must use [GitHub](https://github.com/owner/repo)"))
|
||||
|
||||
github_match = GITHUB_LINK_RE.search(entry.description)
|
||||
description_to_check = entry.description[: github_match.start()].rstrip() if github_match else entry.description.rstrip()
|
||||
# Allow trailing reference links after the sentence, e.g. [GitHub], [Website], or ([Demo](...)).
|
||||
previous = None
|
||||
while previous != description_to_check:
|
||||
previous = description_to_check
|
||||
description_to_check = re.sub(r"\s*\[[^\]]+\]\([^)]+\)\s*$", "", description_to_check).rstrip()
|
||||
description_to_check = re.sub(r"\s*\(\[[^\]]+\]\([^)]+\)\)\s*$", "", description_to_check).rstrip()
|
||||
if description_to_check and not description_to_check.endswith("."):
|
||||
issues.append(Issue(format_severity, line, "period", "description must end with a period before optional trailing link"))
|
||||
|
||||
for url in entry.markdown_urls:
|
||||
if not url.startswith("https://"):
|
||||
severity = "error" if strict_urls else "warning"
|
||||
issues.append(Issue(severity, line, "url", f"URL should use https://: {url}"))
|
||||
|
||||
duplicate_name_entries = [item for item in duplicate_names[normalize_name(entry.name)] if item.line_number != line]
|
||||
if duplicate_name_entries:
|
||||
severity = "error" if strict_duplicates else "warning"
|
||||
lines = ", ".join(str(item.line_number) for item in duplicate_name_entries)
|
||||
issues.append(Issue(severity, line, "duplicate-name", f"project name duplicates line(s): {lines}"))
|
||||
|
||||
seen_duplicate_url_lines: set[int] = set()
|
||||
for url in entry.markdown_urls:
|
||||
duplicate_url_entries = [item for item in duplicate_urls[normalize_url(url)] if item.line_number != line]
|
||||
for item in duplicate_url_entries:
|
||||
seen_duplicate_url_lines.add(item.line_number)
|
||||
if seen_duplicate_url_lines:
|
||||
severity = "error" if strict_duplicates else "warning"
|
||||
lines = ", ".join(str(line_number) for line_number in sorted(seen_duplicate_url_lines))
|
||||
issues.append(Issue(severity, line, "duplicate-url", f"URL duplicates line(s): {lines}"))
|
||||
|
||||
return issues
|
||||
|
||||
|
||||
def validate_malformed_added_lines(readme_path: Path, added_lines: set[int]) -> list[Issue]:
|
||||
issues: list[Issue] = []
|
||||
lines = readme_path.read_text(encoding="utf-8").splitlines()
|
||||
for line_number in sorted(added_lines):
|
||||
if line_number < 1 or line_number > len(lines):
|
||||
continue
|
||||
line = lines[line_number - 1]
|
||||
stripped = line.strip()
|
||||
if not stripped or HEADING_RE.match(stripped):
|
||||
continue
|
||||
if stripped.startswith("- ") and not ENTRY_RE.match(line):
|
||||
issues.append(Issue("error", line_number, "format", "added README bullet does not match entry regex"))
|
||||
return issues
|
||||
|
||||
|
||||
def format_issue(issue: Issue) -> str:
|
||||
location = f"line {issue.line_number}" if issue.line_number else "README"
|
||||
return f"{issue.severity.upper()} {location} [{issue.code}] {issue.message}"
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(description="Validate awesome-quant README entries.")
|
||||
parser.add_argument("--readme", default="README.md", help="README path to validate")
|
||||
parser.add_argument("--diff-from", help="validate only README lines added relative to this git ref")
|
||||
parser.add_argument("--strict-duplicates", action="store_true", help="treat duplicate names/URLs as errors in full validation")
|
||||
parser.add_argument("--strict-urls", action="store_true", help="treat non-https URLs as errors in full validation")
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def main() -> int:
|
||||
args = parse_args()
|
||||
readme_path = Path(args.readme)
|
||||
if not readme_path.exists():
|
||||
print(f"ERROR README not found: {readme_path}", file=sys.stderr)
|
||||
return 2
|
||||
|
||||
entries = list(iter_readme_entries(readme_path))
|
||||
duplicate_names, duplicate_urls = build_duplicate_indexes(entries)
|
||||
|
||||
added_lines: set[int] | None = None
|
||||
if args.diff_from:
|
||||
try:
|
||||
added_lines = read_added_line_numbers(args.diff_from, readme_path)
|
||||
except RuntimeError as exc:
|
||||
print(f"ERROR {exc}", file=sys.stderr)
|
||||
return 2
|
||||
selected_entries = [entry for entry in entries if entry.line_number in added_lines]
|
||||
strict_duplicates = True
|
||||
strict_urls = True
|
||||
strict_format = True
|
||||
else:
|
||||
selected_entries = entries
|
||||
strict_duplicates = args.strict_duplicates
|
||||
strict_urls = args.strict_urls
|
||||
strict_format = False
|
||||
|
||||
issues: list[Issue] = []
|
||||
if added_lines is not None:
|
||||
issues.extend(validate_malformed_added_lines(readme_path, added_lines))
|
||||
|
||||
for entry in selected_entries:
|
||||
issues.extend(
|
||||
validate_entry(
|
||||
entry,
|
||||
duplicate_names=duplicate_names,
|
||||
duplicate_urls=duplicate_urls,
|
||||
strict_duplicates=strict_duplicates,
|
||||
strict_urls=strict_urls,
|
||||
strict_format=strict_format,
|
||||
)
|
||||
)
|
||||
|
||||
errors = [issue for issue in issues if issue.severity == "error"]
|
||||
warnings = [issue for issue in issues if issue.severity == "warning"]
|
||||
|
||||
scope = f"added README lines relative to {args.diff_from}" if args.diff_from else "full README"
|
||||
print(f"Validated {len(selected_entries)} entries in {scope}.")
|
||||
for issue in sorted(issues, key=lambda item: (item.line_number, item.severity, item.code)):
|
||||
print(format_issue(issue))
|
||||
|
||||
if errors:
|
||||
print(f"Validation failed: {len(errors)} error(s), {len(warnings)} warning(s).")
|
||||
return 1
|
||||
|
||||
if warnings:
|
||||
print(f"Validation passed with {len(warnings)} warning(s).")
|
||||
else:
|
||||
print("Validation passed.")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Reference in New Issue
Block a user