FinDataHub
11d36d1c1e
问题 - 索引中该目录记为小写 `scripts/`，工作区实为 `Scripts/`。因 core.ignorecase=true， 本地 `git diff -- Scripts/bin/x.py` 会**静默返回空**（看似无改动）， 且在大小写敏感环境（Linux CI / Docker 挂载）会检出 `scripts/`， 与全仓所有 `Scripts/...` 引用不匹配。 - 该目录此前 10 个脚本（check_python_version.py、perf_snapshot.py、 install_collector_launchd.py、mysql_toggle.sh 等）从未入库， 而 CI 第 31 行正调用 `Scripts/bin/check_python_version.py --strict` —— 全新检出必然失败。 改动 - 索引路径统一为 `Scripts/`（工作区文件未移动，仅索引改名）。 - 补齐上述 10 个未跟踪脚本及其测试。 影响 - 修正前索引 35 条（全为小写路径），修正后 45 条。
264 lines
9.2 KiB
Python
264 lines
9.2 KiB
Python
#!/usr/bin/env python3
|
||
"""common.py — 公共配置与工具模块（统一 DB/路径/PID/端口管理）"""
|
||
|
||
import os
|
||
import signal
|
||
import sys
|
||
from pathlib import Path
|
||
|
||
# ── 数据库配置 ──────────────────────────────────────────────
|
||
MYSQL_CFG = dict(
|
||
host=os.environ.get("MYSQL_HOST", "localhost"),
|
||
port=int(os.environ.get("MYSQL_PORT", "3306")),
|
||
user=os.environ.get("MYSQL_USER", "root"),
|
||
password=os.environ.get("MYSQL_PASSWORD", ""),
|
||
database=os.environ.get("MYSQL_DATABASE", "news_flash"),
|
||
charset="utf8mb4",
|
||
)
|
||
|
||
MYSQL_CFG_AKSHARE = dict(
|
||
host=os.environ.get("MYSQL_HOST", "localhost"),
|
||
port=int(os.environ.get("MYSQL_PORT", "3306")),
|
||
user=os.environ.get("MYSQL_USER", "root"),
|
||
password=os.environ.get("MYSQL_PASSWORD", ""),
|
||
database=os.environ.get("MYSQL_DATABASE_AKSHARE", "akshare_data"),
|
||
charset="utf8mb4",
|
||
)
|
||
|
||
MYSQL_CMD = [
|
||
"mysql",
|
||
"-h", os.environ.get("MYSQL_HOST", "localhost"),
|
||
"-P", os.environ.get("MYSQL_PORT", "3306"),
|
||
"-u", os.environ.get("MYSQL_USER", "root"),
|
||
]
|
||
MYSQL_PWD = os.environ.get("MYSQL_PASSWORD", "")
|
||
DB_NEWS = os.environ.get("MYSQL_DATABASE", "news_flash")
|
||
DB_AKSHARE = os.environ.get("MYSQL_DATABASE_AKSHARE", "akshare_data")
|
||
|
||
|
||
def database_url(cfg=None):
|
||
"""构造 SQLAlchemy DATABASE_URL"""
|
||
from urllib.parse import quote_plus
|
||
c = cfg or MYSQL_CFG
|
||
pwd = quote_plus(c["password"])
|
||
return f"mysql+pymysql://{c['user']}:{pwd}@{c['host']}:{c['port']}/{c['database']}?charset=utf8mb4"
|
||
|
||
|
||
# ── 端口/服务地址 ────────────────────────────────────────────
|
||
BACKEND_PORT = 8100
|
||
OLLAMA_URL = "http://localhost:11434/api/chat"
|
||
RSS_HUB_URL = os.environ.get("RSS_HUB_URL", "http://localhost:1200")
|
||
|
||
BIN_DIR = str(Path(__file__).resolve().parent)
|
||
|
||
# 项目根目录：本文件位于 <root>/Scripts/bin/common.py
|
||
_PROJECT_ROOT = str(Path(BIN_DIR).parent.parent)
|
||
|
||
# 开发环境路径（仅未打包时使用；打包后由 _resolve_bundled_paths 提供）
|
||
# 由本文件位置动态推导，不要写死绝对路径——项目此前从 finance-terminal 改名为
|
||
# FinDataHub 后，旧常量仍指向 finance-terminal/backend/kuaixun-service，
|
||
# 导致开发环境直接跑脚本时 ModuleNotFoundError: No module named 'app.xxx'
|
||
_KX_HOME_DEV = str(Path(_PROJECT_ROOT) / "Services" / "kuaixun-service")
|
||
_KXS_HOME_DEV = _KX_HOME_DEV
|
||
|
||
KX_HOME = _KX_HOME_DEV
|
||
KXS_HOME = _KXS_HOME_DEV
|
||
|
||
#: 项目统一 Python 版本（见仓库根目录 .python-version）。
|
||
#: 各服务 pyproject.toml 的 requires-python 均锁定 >=3.12,<3.13。
|
||
PYTHON_TARGET_VERSION = "3.12"
|
||
|
||
|
||
def _detect_venv_site(home: str) -> str:
|
||
"""
|
||
动态探测 venv 的 site-packages 路径。
|
||
|
||
此前这里写死了某个具体小版本号，而实际 venv 用的是另一个版本，
|
||
属于悬空引用；更糟的是 Python 版本一变就要改代码。改为遍历
|
||
`.venv/lib/python3.X/` 并取版本号最大的那个（与 _resolve_bundled_paths 一致）。
|
||
"""
|
||
lib = Path(home) / ".venv" / "lib"
|
||
fallback = str(lib / f"python{PYTHON_TARGET_VERSION}" / "site-packages")
|
||
if not lib.is_dir():
|
||
return fallback
|
||
|
||
def _vkey(p: Path) -> tuple:
|
||
# "python3.12" -> (3, 12)
|
||
parts = p.parent.name.replace("python", "").split(".")
|
||
try:
|
||
return tuple(int(x) for x in parts)
|
||
except ValueError:
|
||
return (0, 0)
|
||
|
||
cands = [d / "site-packages" for d in lib.iterdir()
|
||
if d.name.startswith("python") and (d / "site-packages").exists()]
|
||
return str(max(cands, key=_vkey)) if cands else fallback
|
||
|
||
|
||
KX_VENV = _detect_venv_site(KX_HOME)
|
||
|
||
|
||
def _resolve_bundled_paths():
|
||
"""检测 app 内打包的资源路径（优先使用），找不到则回退到外部路径"""
|
||
resources_dir = Path(BIN_DIR).parent
|
||
bundled_kxs = resources_dir / "kuaixun-service"
|
||
bundled_venv = bundled_kxs / ".venv" / "lib"
|
||
|
||
# 找到 site-packages 路径
|
||
bundled_site_packages = None
|
||
if bundled_venv.exists():
|
||
for d in bundled_venv.iterdir():
|
||
if d.name.startswith("python") and d.is_dir():
|
||
sp = d / "site-packages"
|
||
if sp.exists():
|
||
bundled_site_packages = str(sp)
|
||
break
|
||
|
||
return str(bundled_kxs) if bundled_kxs.exists() else None, bundled_site_packages
|
||
|
||
|
||
BUNDLED_KXS_HOME, BUNDLED_VENV_SITE = _resolve_bundled_paths()
|
||
|
||
|
||
def setup_kuaixun_paths():
|
||
"""统一设置 kuaixun-service / venv / bin 的 sys.path，优先使用 app 内打包资源"""
|
||
import site
|
||
|
||
paths = []
|
||
site_dirs = []
|
||
|
||
if BUNDLED_KXS_HOME:
|
||
paths.append(BUNDLED_KXS_HOME)
|
||
if BUNDLED_VENV_SITE:
|
||
paths.append(BUNDLED_VENV_SITE)
|
||
site_dirs.append(BUNDLED_VENV_SITE)
|
||
|
||
paths.append(BIN_DIR)
|
||
|
||
if not BUNDLED_KXS_HOME:
|
||
paths.append(KX_HOME)
|
||
paths.append(KXS_HOME)
|
||
if not BUNDLED_VENV_SITE:
|
||
paths.append(KX_VENV)
|
||
site_dirs.append(KX_VENV)
|
||
|
||
for p in paths:
|
||
if p not in sys.path:
|
||
sys.path.insert(0, p)
|
||
|
||
for sd in site_dirs:
|
||
try:
|
||
site.addsitedir(sd)
|
||
except Exception:
|
||
pass
|
||
|
||
|
||
# ── .env 加载 ───────────────────────────────────────────────
|
||
def load_env(path):
|
||
"""加载 .env 文件到 os.environ（不覆盖已有变量）"""
|
||
if not os.path.exists(path):
|
||
return
|
||
with open(path) as f:
|
||
for line in f:
|
||
line = line.strip()
|
||
if not line or line.startswith("#") or "=" not in line:
|
||
continue
|
||
k, v = line.split("=", 1)
|
||
k = k.strip()
|
||
v = v.strip().strip('"').strip("'")
|
||
if k and k not in os.environ:
|
||
os.environ[k] = v
|
||
|
||
|
||
# ── PID 文件管理 ────────────────────────────────────────────
|
||
class PidManager:
|
||
"""PID 文件管理：write / status / stop"""
|
||
|
||
def __init__(self, pid_file: str):
|
||
self.pid_file = pid_file
|
||
|
||
def write(self):
|
||
from datetime import datetime
|
||
with open(self.pid_file, "w") as f:
|
||
f.write(f"{os.getpid()}:{datetime.now().isoformat()}")
|
||
|
||
def status(self) -> dict:
|
||
"""返回 {running, pid, started_at}"""
|
||
r = {"running": False, "pid": None, "started_at": None}
|
||
if not os.path.exists(self.pid_file):
|
||
return r
|
||
with open(self.pid_file) as f:
|
||
raw = f.read().strip()
|
||
# 必须 split(":", 1)：ISO 时间戳本身含冒号，用 split(":") 会把
|
||
# "2026-09-17T18:12:13.463481" 截成 "2026-09-17T18"，
|
||
# 导致 started_at 恒被截断到整点、(now-started)>600s 在每小时第 10 分钟后恒真。
|
||
parts = raw.split(":", 1)
|
||
pid = int(parts[0]) if parts and parts[0].isdigit() else None
|
||
started = parts[1] if len(parts) > 1 else None
|
||
if pid is not None:
|
||
try:
|
||
os.kill(pid, 0)
|
||
r["running"] = True
|
||
r["pid"] = pid
|
||
r["started_at"] = started
|
||
except (OSError, ProcessLookupError):
|
||
if os.path.exists(self.pid_file):
|
||
os.remove(self.pid_file)
|
||
return r
|
||
|
||
def stop(self) -> bool:
|
||
if not os.path.exists(self.pid_file):
|
||
return False
|
||
with open(self.pid_file) as f:
|
||
pid_str = f.read().strip().split(":")[0]
|
||
if pid_str.isdigit():
|
||
try:
|
||
os.kill(int(pid_str), signal.SIGTERM)
|
||
except (OSError, ProcessLookupError):
|
||
pass
|
||
if os.path.exists(self.pid_file):
|
||
os.remove(self.pid_file)
|
||
return True
|
||
return False
|
||
|
||
|
||
# ── Debug 日志工具 ──────────────────────────────────────────
|
||
# 通过 FDH_DEBUG 环境变量控制，值为 debug/info/warn/error/off
|
||
_DEBUG_LEVEL = os.environ.get("FDH_DEBUG", "info").lower()
|
||
_DEBUG_LEVELS = {"debug": 10, "info": 20, "warn": 30, "error": 40, "off": 99}
|
||
|
||
|
||
def _debug_level_val() -> int:
|
||
return _DEBUG_LEVELS.get(_DEBUG_LEVEL, 20)
|
||
|
||
|
||
def _debug_print(level: str, category: str, message: str) -> None:
|
||
"""统一格式的 debug 日志输出到 stderr"""
|
||
from datetime import datetime
|
||
ts = datetime.now().strftime("%H:%M:%S.%f")[:-3]
|
||
print(f"[{ts}] [{level}] [{category}] {message}", flush=True, file=sys.stderr)
|
||
|
||
|
||
def log_debug(category: str, message: str) -> None:
|
||
if _debug_level_val() <= 10:
|
||
_debug_print("DEBUG", category, message)
|
||
|
||
|
||
def log_info(category: str, message: str) -> None:
|
||
if _debug_level_val() <= 20:
|
||
_debug_print("INFO", category, message)
|
||
|
||
|
||
def log_warn(category: str, message: str) -> None:
|
||
if _debug_level_val() <= 30:
|
||
_debug_print("WARN", category, message)
|
||
|
||
|
||
def log_error(category: str, message: str) -> None:
|
||
if _debug_level_val() <= 40:
|
||
_debug_print("ERROR", category, message)
|
||
|
||
|
||
def is_debug_enabled() -> bool:
|
||
return _debug_level_val() <= 10
|