FinDataHub
11d36d1c1e
问题 - 索引中该目录记为小写 `scripts/`，工作区实为 `Scripts/`。因 core.ignorecase=true， 本地 `git diff -- Scripts/bin/x.py` 会**静默返回空**（看似无改动）， 且在大小写敏感环境（Linux CI / Docker 挂载）会检出 `scripts/`， 与全仓所有 `Scripts/...` 引用不匹配。 - 该目录此前 10 个脚本（check_python_version.py、perf_snapshot.py、 install_collector_launchd.py、mysql_toggle.sh 等）从未入库， 而 CI 第 31 行正调用 `Scripts/bin/check_python_version.py --strict` —— 全新检出必然失败。 改动 - 索引路径统一为 `Scripts/`（工作区文件未移动，仅索引改名）。 - 补齐上述 10 个未跟踪脚本及其测试。 影响 - 修正前索引 35 条（全为小写路径），修正后 45 条。
307 lines
10 KiB
Python
307 lines
10 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
FinDataHub 数据洞察
|
|
查询 MySQL 返回各数据库的汇总统计（总行数、表数、最后更新时间）
|
|
|
|
用法: insights.py [--mode overview|detail] [--database db_name]
|
|
overview: 查询所有数据库的汇总统计（默认）
|
|
detail: 查询单数据库的表级详情（兼容旧版）
|
|
"""
|
|
import argparse
|
|
import json
|
|
import os
|
|
import sys
|
|
from datetime import datetime
|
|
|
|
try:
|
|
import pymysql
|
|
except ImportError:
|
|
print(json.dumps({"status": "error", "message": "pymysql not available"}), file=sys.stderr)
|
|
sys.exit(1)
|
|
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument("--database", default=None, help="数据库名称（detail 模式使用）")
|
|
parser.add_argument("--mode", default="overview", choices=["overview", "detail"],
|
|
help="模式: overview=汇总, detail=表级详情")
|
|
args = parser.parse_args()
|
|
|
|
MYSQL_HOST = os.environ.get("MYSQL_HOST", "localhost")
|
|
MYSQL_PORT = int(os.environ.get("MYSQL_PORT", "3306"))
|
|
MYSQL_USER = os.environ.get("MYSQL_USER", "root")
|
|
MYSQL_PWD = os.environ.get("MYSQL_PASSWORD", "")
|
|
DB_FALLBACK = os.environ.get("MYSQL_DATABASE_AKSHARE", os.environ.get("MYSQL_DATABASE", "akshare_data"))
|
|
|
|
|
|
def get_conn(database):
|
|
return pymysql.connect(
|
|
host=MYSQL_HOST,
|
|
port=MYSQL_PORT,
|
|
user=MYSQL_USER,
|
|
password=MYSQL_PWD,
|
|
database=database,
|
|
charset="utf8mb4",
|
|
cursorclass=pymysql.cursors.DictCursor,
|
|
)
|
|
|
|
|
|
def parse_date(s):
|
|
if not s:
|
|
return None
|
|
if isinstance(s, datetime):
|
|
return s.strftime("%Y-%m-%d %H:%M:%S")
|
|
s = str(s).strip()
|
|
if len(s) == 8 and s.isdigit():
|
|
return f"{s[:4]}-{s[4:6]}-{s[6:8]}"
|
|
return s[:19] if len(s) >= 19 else (s[:10] if len(s) >= 10 else s)
|
|
|
|
|
|
# ─── overview 模式：多数据库汇总 ────────────────────────────────────────────
|
|
|
|
# 各数据库的日期探测配置
|
|
DB_DATE_PROBES = {
|
|
"news_flash": [
|
|
("news_flash", "FROM_UNIXTIME(MAX(published_at)/1000)"),
|
|
],
|
|
"akshare_data": [
|
|
("stock_daily", "MAX(trade_date)"),
|
|
("index_daily", "MAX(trade_date)"),
|
|
("fund_etf_daily", "MAX(trade_date)"),
|
|
],
|
|
"tushare_data": [
|
|
("stock_daily", "MAX(trade_date)"),
|
|
("index_daily", "MAX(trade_date)"),
|
|
],
|
|
}
|
|
|
|
SOURCE_META = {
|
|
"news_flash": {"display_name": "\u5feb\u8baf\u4e2d\u5fc3", "icon": "newspaper", "color": "#5E5CE6"},
|
|
"akshare_data": {"display_name": "AkShare", "icon": "chart.line.uptrend.xyaxis", "color": "#30D158"},
|
|
"tushare_data": {"display_name": "TuShare", "icon": "chart.bar", "color": "#FF9F0A"},
|
|
}
|
|
|
|
|
|
def overview_db(db_name):
|
|
"""查询单个数据库的汇总统计"""
|
|
conn = get_conn(db_name)
|
|
try:
|
|
# 获取所有基础表的行数和更新时间
|
|
with conn.cursor() as cur:
|
|
cur.execute(
|
|
"SELECT TABLE_NAME, TABLE_ROWS, UPDATE_TIME "
|
|
"FROM information_schema.TABLES "
|
|
"WHERE TABLE_SCHEMA = %s AND TABLE_TYPE = 'BASE TABLE'",
|
|
(db_name,),
|
|
)
|
|
tables = cur.fetchall()
|
|
|
|
table_names = [t["TABLE_NAME"] for t in tables]
|
|
total_rows = sum(int(t.get("TABLE_ROWS", 0) or 0) for t in tables)
|
|
total_tables = len(tables)
|
|
|
|
# 尝试从 UPDATE_TIME 获取最后更新时间
|
|
update_times = []
|
|
for t in tables:
|
|
ut = t.get("UPDATE_TIME")
|
|
if ut:
|
|
update_times.append(ut)
|
|
last_update = None
|
|
if update_times:
|
|
last_update = parse_date(max(update_times))
|
|
|
|
# 如果 UPDATE_TIME 不可用（InnoDB），查询日期列
|
|
if not last_update:
|
|
probes = DB_DATE_PROBES.get(db_name, [])
|
|
for tbl, col_expr in probes:
|
|
if tbl not in table_names:
|
|
continue
|
|
try:
|
|
sql = f"SELECT {col_expr} AS latest FROM `{tbl}`"
|
|
with conn.cursor() as cur:
|
|
cur.execute(sql)
|
|
row = cur.fetchone()
|
|
if row and row.get("latest"):
|
|
candidate = parse_date(row["latest"])
|
|
if candidate:
|
|
if not last_update or candidate > last_update:
|
|
last_update = candidate
|
|
except Exception:
|
|
pass
|
|
|
|
# 如果上述都查不到，尝试查第一张有数据的表
|
|
if not last_update:
|
|
for t in tables:
|
|
tn = t["TABLE_NAME"]
|
|
rows_count = int(t.get("TABLE_ROWS", 0) or 0)
|
|
if rows_count == 0:
|
|
continue
|
|
try:
|
|
sql = f"SELECT UPDATE_TIME FROM information_schema.TABLES WHERE TABLE_SCHEMA = %s AND TABLE_NAME = %s"
|
|
with conn.cursor() as cur:
|
|
cur.execute(sql, (db_name, tn))
|
|
r = cur.fetchone()
|
|
if r and r.get("UPDATE_TIME"):
|
|
last_update = parse_date(r["UPDATE_TIME"])
|
|
break
|
|
except Exception:
|
|
pass
|
|
|
|
meta = SOURCE_META.get(db_name, {"display_name": db_name, "icon": "tablecells", "color": "#888888"})
|
|
return {
|
|
"source": db_name,
|
|
"display_name": meta["display_name"],
|
|
"icon": meta["icon"],
|
|
"color": meta["color"],
|
|
"total_rows": total_rows,
|
|
"total_tables": total_tables,
|
|
"last_update": last_update,
|
|
}
|
|
except Exception as e:
|
|
meta = SOURCE_META.get(db_name, {"display_name": db_name, "icon": "tablecells", "color": "#888888"})
|
|
return {
|
|
"source": db_name,
|
|
"display_name": meta["display_name"],
|
|
"icon": meta["icon"],
|
|
"color": meta["color"],
|
|
"total_rows": 0,
|
|
"total_tables": 0,
|
|
"last_update": None,
|
|
"error": str(e),
|
|
}
|
|
finally:
|
|
conn.close()
|
|
|
|
|
|
def overview_mode():
|
|
"""查询所有三个数据库"""
|
|
databases = ["news_flash", "akshare_data", "tushare_data"]
|
|
results = []
|
|
for db_name in databases:
|
|
results.append(overview_db(db_name))
|
|
return json.dumps({"status": "ok", "sources": results, "mode": "overview"}, ensure_ascii=False)
|
|
|
|
|
|
# ─── detail 模式：单库全表动态查询 ──────────────────────────────────────
|
|
|
|
# 常见日期列名（按优先级排序）
|
|
DATE_COLUMN_CANDIDATES = [
|
|
"trade_date", "report_date", "nav_date", "created_at",
|
|
"published_at", "update_time", "f_ann_date", "ann_date",
|
|
"end_date", "datetime", "date", "time",
|
|
]
|
|
|
|
|
|
def query_all_tables(conn, database):
|
|
"""查询数据库中所有基础表的名称和行数"""
|
|
sql = (
|
|
"SELECT TABLE_NAME, TABLE_ROWS, UPDATE_TIME "
|
|
"FROM information_schema.TABLES "
|
|
"WHERE TABLE_SCHEMA = %s AND TABLE_TYPE = 'BASE TABLE'"
|
|
)
|
|
with conn.cursor() as cur:
|
|
cur.execute(sql, (database,))
|
|
return cur.fetchall()
|
|
|
|
|
|
def detect_date_column(conn, table_name):
|
|
"""自动探测表中的日期列"""
|
|
# 1) 先查 information_schema 找日期/时间类型列
|
|
sql = (
|
|
"SELECT COLUMN_NAME FROM information_schema.COLUMNS "
|
|
"WHERE TABLE_SCHEMA = %s AND TABLE_NAME = %s "
|
|
"AND DATA_TYPE IN ('date', 'datetime', 'timestamp', 'bigint') "
|
|
"ORDER BY ORDINAL_POSITION"
|
|
)
|
|
try:
|
|
with conn.cursor() as cur:
|
|
cur.execute(sql, (conn.db.decode() if isinstance(conn.db, bytes) else conn.db, table_name))
|
|
columns = [r["COLUMN_NAME"] for r in cur.fetchall()]
|
|
except Exception:
|
|
columns = []
|
|
|
|
# 2) 按候选名称优先级排序
|
|
scored = []
|
|
for col in columns:
|
|
col_lower = col.lower()
|
|
for i, candidate in enumerate(DATE_COLUMN_CANDIDATES):
|
|
if candidate in col_lower:
|
|
scored.append((i, col))
|
|
break
|
|
else:
|
|
# 其他日期类型列，优先级较低
|
|
scored.append((len(DATE_COLUMN_CANDIDATES), col))
|
|
scored.sort(key=lambda x: x[0])
|
|
|
|
return [col for _, col in scored]
|
|
|
|
|
|
def query_latest_date(conn, table_name, date_columns):
|
|
"""查询表中最新日期，尝试所有候选日期列"""
|
|
for col in date_columns:
|
|
try:
|
|
sql = f"SELECT `{col}` FROM `{table_name}` ORDER BY `{col}` DESC LIMIT 1"
|
|
with conn.cursor() as cur:
|
|
cur.execute(sql)
|
|
row = cur.fetchone()
|
|
if row and row.get(col):
|
|
return parse_date(row[col])
|
|
except Exception:
|
|
continue
|
|
return None
|
|
|
|
|
|
def detail_mode():
|
|
db = args.database or DB_FALLBACK
|
|
conn = get_conn(db)
|
|
try:
|
|
tables = query_all_tables(conn, db)
|
|
results = []
|
|
total_rows = 0
|
|
|
|
for t in tables:
|
|
tbl_name = t["TABLE_NAME"]
|
|
row_count = int(t.get("TABLE_ROWS", 0) or 0)
|
|
total_rows += row_count
|
|
|
|
info = {
|
|
"table": tbl_name,
|
|
"rows": row_count,
|
|
"latest": None,
|
|
"gap_days": None,
|
|
}
|
|
|
|
if row_count > 0:
|
|
# 自动探测日期列
|
|
date_cols = detect_date_column(conn, tbl_name)
|
|
latest = query_latest_date(conn, tbl_name, date_cols)
|
|
if latest:
|
|
info["latest"] = latest
|
|
try:
|
|
td = datetime.strptime(latest[:10], "%Y-%m-%d")
|
|
info["gap_days"] = (datetime.now() - td).days
|
|
except ValueError:
|
|
pass
|
|
|
|
results.append(info)
|
|
|
|
# 按行数降序排列，数据多的表在前
|
|
results.sort(key=lambda r: -r["rows"])
|
|
|
|
return json.dumps({
|
|
"status": "ok",
|
|
"tables": results,
|
|
"total_rows": total_rows,
|
|
"total_tables": len(results),
|
|
"db_size_gb": None,
|
|
}, ensure_ascii=False)
|
|
finally:
|
|
conn.close()
|
|
|
|
|
|
# ─── 入口 ─────────────────────────────────────────────────────────────────
|
|
|
|
if __name__ == "__main__":
|
|
if args.mode == "overview":
|
|
print(overview_mode())
|
|
else:
|
|
print(detail_mode())
|