FinDataHub/Scripts/bin/sync_bridge.py

#!/usr/bin/env python3
"""
sync_bridge.py — 统一同步桥接层

统一入口: sync_bridge.py <source> <type> [workers]

source: akshare | tushare
type: daily | weekly | monthly

tushare_fetcher 批量同步策略:
  daily   → [daily, market_sentiment]       — 日线、资金流、融资融券
  weekly  → [fundamental, static, event]     — 财报、基础信息、事件
  monthly → [index, extra, stock_driven]     — 指数、补充数据、股东
"""

import os, subprocess, sys, traceback
from pathlib import Path

HOME = Path.home()

# 优先使用 App 内部路径，再回退到外部源码
def _find_project(name: str) -> Path:
    candidates = []
    # 1. App 内部（通过环境变量传入，或从脚本位置推断）
    resources_dir = os.environ.get("APP_RESOURCES", "")
    if resources_dir:
        candidates.append(Path(resources_dir) / name)
    # 2. 从脚本位置推断（脚本在 Resources/bin/ 下）
    script_dir = Path(__file__).resolve().parent
    if script_dir.name == "bin" and script_dir.parent.name == "Resources":
        candidates.append(script_dir.parent / name)
    # 3. 外部源码
    candidates.append(HOME / "Projects/项目" / name)
    candidates.append(HOME / "Projects" / name)
    candidates.append(HOME / name)
    
    for p in candidates:
        if (p / "pyproject.toml").exists():
            return p
    return candidates[0]  # 返回第一个作为默认

PROJ = {
    "akshare": _find_project("akshare_fetcher"),
    "tushare": _find_project("tushare_fetcher"),
}

# tushare 批量同步 — 每个 sync_type 对应一组 fetch type
TUSHARE_BATCH = {
    "daily":   ["daily", "market_sentiment"],
    "weekly":  ["fundamental", "static", "event"],
    "monthly": ["index", "extra", "stock_driven"],
}

# 扩展的 fetch type → table list（含 CLI 里没有的 static/event/extra/stock_driven）
TYPE_TABLES = {
    "daily":   ["daily", "daily_basic", "weekly", "monthly", "adj_factor", "suspend"],
    "market_sentiment": ["moneyflow", "margin", "margin_detail", "top_list", "block_trade", "stk_holdernumber", "moneyflow_hsgt"],
    "fundamental": ["stock_basic", "income", "balancesheet", "cashflow", "fina_indicator", "fina_audit", "forecast", "express", "dividend"],
    "static": ["trade_cal", "index_basic", "index_classify", "stock_company", "bak_basic", "hs_const"],
    "event": ["new_share", "daily_info", "sz_daily_info", "fund_nav", "fund_share", "fund_manager"],
    "index": ["index_daily", "index_basic", "index_weight", "index_classify", "index_member_all", "index_dailybasic", "index_weekly", "index_monthly"],
    "extra": ["yearly", "stk_weekly_monthly", "stk_week_month_adj", "stk_limit", "report_rc", "cn_gdp", "cn_cpi", "cn_ppi", "cn_pmi", "cn_m"],
    "stock_driven": ["stk_rewards", "stk_holdernumber", "stk_holdertrade"],
}


def find_python(project: Path) -> str:
    for p in [project / ".venv/bin/python3", project / ".venv/bin/python",
              Path("/usr/bin/python3")]:
        if p.exists():
            return str(p)
    raise FileNotFoundError(f"找不到 python3 (from {project})")


def load_token(project: Path) -> str:
    env_file = project / ".env"
    if env_file.exists():
        for line in env_file.read_text().splitlines():
            if line.startswith("TUSHARE_TOKEN="):
                return line.split("=", 1)[1].strip().strip("'\"")
    return os.environ.get("TUSHARE_TOKEN", "")


def _pump_stream(fd, sink):
    """实时转发子进程输出流（块级读取，兼容 tqdm 用 \r 原地刷新的进度行）。"""
    while True:
        try:
            chunk = os.read(fd, 8192)
        except OSError:
            break
        if not chunk:
            break
        sink.write(chunk)
        sink.flush()


def run_akshare(project: Path, sync_type: str, workers: int, python: str) -> int:
    import threading
    env = os.environ.copy()
    env.update({"PYTHONPATH": f"{project}/src", "AKSHARE_HOME": str(project), "PYTHONUNBUFFERED": "1"})
    cmd = [python, "-m", "akshare_fetcher.cli", "sync", "--type", sync_type, "--workers", str(workers)]
    print(f"  {' '.join(cmd)}", flush=True)

    # 实时转发 stdout/stderr，不能 capture_output=True：
    # 子进程的 tqdm 进度写在 stderr，若被缓冲则 App 端收不到实时进度
    proc = subprocess.Popen(
        cmd, cwd=str(project), env=env,
        stdout=subprocess.PIPE, stderr=subprocess.PIPE,
    )
    t1 = threading.Thread(target=_pump_stream, args=(proc.stdout.fileno(), sys.stdout.buffer), daemon=True)
    t2 = threading.Thread(target=_pump_stream, args=(proc.stderr.fileno(), sys.stderr.buffer), daemon=True)
    t1.start(); t2.start()
    rc = proc.wait()
    t1.join(); t2.join()
    return rc


def run_tushare(project: Path, sync_type: str, workers: int, python: str) -> int:
    """tushare 批量同步 — 按策略拉取多组表"""
    token = load_token(project)
    if not token:
        print("  ❌ TUSHARE_TOKEN 未配置", flush=True)
        return 1

    fetch_types = TUSHARE_BATCH.get(sync_type, ["daily"])
    overall_rc = 0

    for ft in fetch_types:
        tables = TYPE_TABLES.get(ft, [ft])
        if not tables:
            print(f"  ⏭  {ft}: 无表定义", flush=True)
            continue

        print(f"\n  📡 [{ft}] → {len(tables)} 张表: {', '.join(tables[:5])}{'...' if len(tables) > 5 else ''}", flush=True)

        # 直接调用 DataFetcher，避免多次启动 Python 的开销
        env = os.environ.copy()
        env.update({
            "PYTHONPATH": f"{project}/src",
            "TUSHARE_HOME": str(project),
            "PYTHONUNBUFFERED": "1",
            "TUSHARE_TOKEN": token,
        })

        code = f"""
import sys
sys.path.insert(0, "{project}/src")
from tushare_fetcher.core.config import load_config
from tushare_fetcher.core.fetcher import DataFetcher
cfg = load_config()
fetcher = DataFetcher(cfg, max_workers={workers})
try:
    fetcher.run({tables})
except KeyboardInterrupt:
    print("\\n⚠  用户中断")
finally:
    fetcher.db.dispose()
"""
        cmd = [python, "-c", code]
        proc = subprocess.Popen(
            cmd, cwd=str(project), env=env,
            stdout=subprocess.PIPE, stderr=subprocess.PIPE,
        )
        import threading
        t1 = threading.Thread(target=_pump_stream, args=(proc.stdout.fileno(), sys.stdout.buffer), daemon=True)
        t2 = threading.Thread(target=_pump_stream, args=(proc.stderr.fileno(), sys.stderr.buffer), daemon=True)
        t1.start(); t2.start()
        rc = proc.wait()
        t1.join(); t2.join()
        if rc != 0:
            print(f"  ❌ [{ft}] 失败 (rc={rc})", flush=True)
            overall_rc = rc
        else:
            print(f"  ✅ [{ft}] 完成", flush=True)

    return overall_rc


if __name__ == "__main__":
    if len(sys.argv) < 3:
        print("用法: sync_bridge.py <source:akshare|tushare> <type:daily|weekly|monthly> [workers]")
        sys.exit(2)

    source = sys.argv[1]
    sync_type = sys.argv[2]
    workers = int(sys.argv[3]) if len(sys.argv) > 3 else 4

    proj = PROJ.get(source)
    if not proj or not proj.exists():
        print(f"❌ {source}_fetcher 项目不存在: {proj}")
        sys.exit(1)

    python = find_python(proj)
    print(f"  项目: {proj}", flush=True)
    print(f"  Python: {python}", flush=True)

    if source == "akshare":
        rc = run_akshare(proj, sync_type, workers, python)
    else:
        rc = run_tushare(proj, sync_type, workers, python)

    print(f"\n  退出码: {rc}", flush=True)
    sys.exit(rc)