FinDataHub
11d36d1c1e
问题 - 索引中该目录记为小写 `scripts/`，工作区实为 `Scripts/`。因 core.ignorecase=true， 本地 `git diff -- Scripts/bin/x.py` 会**静默返回空**（看似无改动）， 且在大小写敏感环境（Linux CI / Docker 挂载）会检出 `scripts/`， 与全仓所有 `Scripts/...` 引用不匹配。 - 该目录此前 10 个脚本（check_python_version.py、perf_snapshot.py、 install_collector_launchd.py、mysql_toggle.sh 等）从未入库， 而 CI 第 31 行正调用 `Scripts/bin/check_python_version.py --strict` —— 全新检出必然失败。 改动 - 索引路径统一为 `Scripts/`（工作区文件未移动，仅索引改名）。 - 补齐上述 10 个未跟踪脚本及其测试。 影响 - 修正前索引 35 条（全为小写路径），修正后 45 条。
283 lines
11 KiB
Python
283 lines
11 KiB
Python
#!/usr/bin/env python3
|
|
"""FinDataHub 数据库初始化脚本"""
|
|
import sys
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
|
|
MYSQL_HOST = os.environ.get("MYSQL_HOST", "localhost")
|
|
MYSQL_PORT = int(os.environ.get("MYSQL_PORT", "3306"))
|
|
MYSQL_USER = os.environ.get("MYSQL_USER", "root")
|
|
MYSQL_PASSWORD = os.environ.get("MYSQL_PASSWORD", "")
|
|
MYSQL_DATABASE = os.environ.get("MYSQL_DATABASE", "news_flash")
|
|
MYSQL_DATABASE_AKSHARE = os.environ.get("MYSQL_DATABASE_AKSHARE", "akshare_data")
|
|
MYSQL_DATABASE_TUSHARE = os.environ.get("MYSQL_DATABASE_TUSHARE", "tushare_data")
|
|
|
|
|
|
def connect(host, port, user, password, database=None):
|
|
import pymysql
|
|
return pymysql.connect(
|
|
host=host, port=port, user=user, password=password,
|
|
database=database, charset="utf8mb4",
|
|
connect_timeout=10
|
|
)
|
|
|
|
|
|
def execute_sql(conn, sql, params=None):
|
|
try:
|
|
with conn.cursor() as cur:
|
|
cur.execute(sql, params)
|
|
conn.commit()
|
|
return True
|
|
except Exception as e:
|
|
print(f"[SQL ERROR] {sql[:100]}...: {e}", file=sys.stderr)
|
|
return False
|
|
|
|
|
|
def init_news_flash(conn):
|
|
"""初始化 news_flash 数据库"""
|
|
steps = []
|
|
|
|
steps.append("创建 news_flash 数据库")
|
|
execute_sql(conn, f"""
|
|
CREATE DATABASE IF NOT EXISTS `{MYSQL_DATABASE}`
|
|
DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci
|
|
""")
|
|
|
|
conn.select_db(MYSQL_DATABASE)
|
|
|
|
steps.append("创建 news_sources 表")
|
|
execute_sql(conn, """
|
|
CREATE TABLE IF NOT EXISTS news_sources (
|
|
id INT AUTO_INCREMENT PRIMARY KEY,
|
|
name VARCHAR(50) NOT NULL COMMENT '数据源名称',
|
|
code VARCHAR(30) NOT NULL UNIQUE COMMENT '数据源代码',
|
|
base_url VARCHAR(255),
|
|
category VARCHAR(50) COMMENT '分类: macro/stock/tech/commodity',
|
|
priority TINYINT DEFAULT 0 COMMENT '0=P0核心, 1=P1重要, 2=P2补充',
|
|
is_active BOOLEAN DEFAULT TRUE,
|
|
fetch_interval INT DEFAULT 30 COMMENT '抓取间隔(秒)',
|
|
api_config JSON COMMENT 'API配置',
|
|
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
|
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP
|
|
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci
|
|
""")
|
|
|
|
steps.append("创建 news_flash 主表")
|
|
execute_sql(conn, """
|
|
CREATE TABLE IF NOT EXISTS news_flash (
|
|
id BIGINT AUTO_INCREMENT PRIMARY KEY,
|
|
source_id INT NOT NULL,
|
|
source_news_id VARCHAR(100) NOT NULL COMMENT '源站新闻ID',
|
|
title VARCHAR(500) NOT NULL,
|
|
content TEXT,
|
|
summary VARCHAR(1000),
|
|
url VARCHAR(500),
|
|
simhash BIGINT UNSIGNED COMMENT '64-bit SimHash',
|
|
category VARCHAR(50),
|
|
tags JSON,
|
|
importance TINYINT DEFAULT 0 COMMENT '0-5',
|
|
is_duplicate BOOLEAN DEFAULT FALSE,
|
|
duplicate_of BIGINT NULL,
|
|
dup_method VARCHAR(20) COMMENT 'exact/simhash/semantic',
|
|
is_pushed BOOLEAN DEFAULT FALSE,
|
|
pushed_at TIMESTAMP NULL,
|
|
push_channel VARCHAR(20) COMMENT 'bark/feishu',
|
|
published_at BIGINT NOT NULL COMMENT '源站时间戳(ms)',
|
|
fetched_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
|
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
|
UNIQUE KEY uk_source_news (source_id, source_news_id),
|
|
INDEX idx_published (published_at),
|
|
INDEX idx_simhash (simhash),
|
|
INDEX idx_category (category),
|
|
INDEX idx_is_pushed (is_pushed),
|
|
INDEX idx_fetched (fetched_at),
|
|
INDEX idx_importance_pushed (importance, is_pushed, is_duplicate),
|
|
INDEX idx_published_importance (published_at, importance),
|
|
FULLTEXT INDEX ft_title_content (title, content) WITH PARSER ngram
|
|
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci
|
|
""")
|
|
|
|
steps.append("创建 push_log 表")
|
|
execute_sql(conn, """
|
|
CREATE TABLE IF NOT EXISTS push_log (
|
|
id BIGINT AUTO_INCREMENT PRIMARY KEY,
|
|
news_id BIGINT NOT NULL,
|
|
channel VARCHAR(20) NOT NULL,
|
|
status VARCHAR(20) NOT NULL,
|
|
error_msg TEXT,
|
|
pushed_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
|
INDEX idx_news (news_id),
|
|
INDEX idx_status (status)
|
|
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4
|
|
""")
|
|
|
|
steps.append("创建 fetch_log 表")
|
|
execute_sql(conn, """
|
|
CREATE TABLE IF NOT EXISTS fetch_log (
|
|
id BIGINT AUTO_INCREMENT PRIMARY KEY,
|
|
source_id INT NOT NULL,
|
|
status VARCHAR(20) NOT NULL,
|
|
items_count INT DEFAULT 0,
|
|
new_count INT DEFAULT 0,
|
|
dup_count INT DEFAULT 0,
|
|
error_msg TEXT,
|
|
duration_ms INT,
|
|
fetched_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
|
INDEX idx_source_time (source_id, fetched_at)
|
|
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4
|
|
""")
|
|
|
|
steps.append("初始化数据源")
|
|
execute_sql(conn, """
|
|
INSERT IGNORE INTO news_sources (name, code, priority, fetch_interval, api_config) VALUES
|
|
('金十数据', 'jin10', 0, 10, '{"sdk": "jin10_flash", "categories": [1, 4]}'),
|
|
('财联社电报', 'cls', 0, 30, '{"url": "https://www.cls.cn/v1/roll/get_roll_list", "rn": 50}'),
|
|
('华尔街见闻', 'wallstreetcn', 0, 30, '{"url": "https://api-one.wallstcn.com/apiv1/content/lives", "channel": "global"}'),
|
|
('东方财富快讯', 'eastmoney', 1, 60, '{"url": "https://kuaixun.eastmoney.com/"}'),
|
|
('新浪财经快讯', 'sina', 1, 60, '{"url": "https://finance.sina.com.cn/"}'),
|
|
('36氪快讯', '36kr', 2, 60, '{"url": "https://36kr.com/api/newsflash"}'),
|
|
('GNews', 'gnews', 2, 120, '{"api_key": ""}'),
|
|
('NewsAPI', 'newsapi', 2, 120, '{"api_key": ""}'),
|
|
('格隆汇', 'gelonghui', 1, 60, '{"url": "https://www.gelonghui.com/"}'),
|
|
('腾讯新闻', 'tencent_news', 1, 60, '{"url": "https://news.qq.com/"}')
|
|
""")
|
|
|
|
return steps
|
|
|
|
|
|
def init_akshare_data(conn):
|
|
"""初始化 akshare_data 数据库（基础表）"""
|
|
steps = []
|
|
|
|
steps.append("创建 akshare_data 数据库")
|
|
execute_sql(conn, f"""
|
|
CREATE DATABASE IF NOT EXISTS `{MYSQL_DATABASE_AKSHARE}`
|
|
DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci
|
|
""")
|
|
|
|
conn.select_db(MYSQL_DATABASE_AKSHARE)
|
|
|
|
steps.append("创建 _table_meta 表")
|
|
execute_sql(conn, """
|
|
CREATE TABLE IF NOT EXISTS `_table_meta` (
|
|
`table_name` VARCHAR(64) PRIMARY KEY,
|
|
`last_update` DATE,
|
|
`row_count` BIGINT DEFAULT 0,
|
|
`status` VARCHAR(20) DEFAULT 'active',
|
|
`created_at` TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
|
`updated_at` TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP
|
|
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4
|
|
""")
|
|
|
|
steps.append("创建 stock_daily 表")
|
|
execute_sql(conn, """
|
|
CREATE TABLE IF NOT EXISTS `stock_daily` (
|
|
`ts_code` VARCHAR(16) NOT NULL COMMENT '股票代码',
|
|
`trade_date` VARCHAR(8) NOT NULL COMMENT '交易日期',
|
|
`open` DECIMAL(10,2) COMMENT '开盘价',
|
|
`high` DECIMAL(10,2) COMMENT '最高价',
|
|
`low` DECIMAL(10,2) COMMENT '最低价',
|
|
`close` DECIMAL(10,2) COMMENT '收盘价',
|
|
`pre_close` DECIMAL(10,2) COMMENT '昨收价',
|
|
`change` DECIMAL(10,2) COMMENT '涨跌额',
|
|
`pct_chg` DECIMAL(6,2) COMMENT '涨跌幅',
|
|
`vol` BIGINT COMMENT '成交量',
|
|
`amount` BIGINT COMMENT '成交额',
|
|
`created_at` TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
|
PRIMARY KEY (`ts_code`, `trade_date`),
|
|
INDEX `idx_trade_date` (`trade_date`)
|
|
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4
|
|
""")
|
|
|
|
steps.append("创建 index_daily 表")
|
|
execute_sql(conn, """
|
|
CREATE TABLE IF NOT EXISTS `index_daily` (
|
|
`ts_code` VARCHAR(16) NOT NULL COMMENT '指数代码',
|
|
`trade_date` VARCHAR(8) NOT NULL COMMENT '交易日期',
|
|
`open` DECIMAL(10,2) COMMENT '开盘价',
|
|
`high` DECIMAL(10,2) COMMENT '最高价',
|
|
`low` DECIMAL(10,2) COMMENT '最低价',
|
|
`close` DECIMAL(10,2) COMMENT '收盘价',
|
|
`pre_close` DECIMAL(10,2) COMMENT '昨收价',
|
|
`change` DECIMAL(10,2) COMMENT '涨跌额',
|
|
`pct_chg` DECIMAL(6,2) COMMENT '涨跌幅',
|
|
`vol` BIGINT COMMENT '成交量',
|
|
`amount` BIGINT COMMENT '成交额',
|
|
`created_at` TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
|
PRIMARY KEY (`ts_code`, `trade_date`),
|
|
INDEX `idx_trade_date` (`trade_date`)
|
|
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4
|
|
""")
|
|
|
|
return steps
|
|
|
|
|
|
def init_tushare_data(conn):
|
|
"""初始化 tushare_data 数据库（基础库，表由 tushare_fetcher 同步时创建）"""
|
|
steps = []
|
|
|
|
steps.append("创建 tushare_data 数据库")
|
|
execute_sql(conn, f"""
|
|
CREATE DATABASE IF NOT EXISTS `{MYSQL_DATABASE_TUSHARE}`
|
|
DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci
|
|
""")
|
|
|
|
conn.select_db(MYSQL_DATABASE_TUSHARE)
|
|
|
|
steps.append("创建 _table_meta 表")
|
|
execute_sql(conn, """
|
|
CREATE TABLE IF NOT EXISTS `_table_meta` (
|
|
`table_name` VARCHAR(64) PRIMARY KEY,
|
|
`last_update` DATE,
|
|
`row_count` BIGINT DEFAULT 0,
|
|
`status` VARCHAR(20) DEFAULT 'active',
|
|
`created_at` TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
|
`updated_at` TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP
|
|
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4
|
|
""")
|
|
|
|
return steps
|
|
|
|
|
|
def check_database(host, port, user, password, database):
|
|
"""检查数据库是否可连接"""
|
|
try:
|
|
conn = connect(host, port, user, password, database)
|
|
conn.close()
|
|
return True
|
|
except Exception as e:
|
|
print(f"[CHECK FAILED] {database}: {e}", file=sys.stderr)
|
|
return False
|
|
|
|
|
|
def main():
|
|
result = {
|
|
"success": True,
|
|
"steps": [],
|
|
"error": None
|
|
}
|
|
|
|
try:
|
|
conn = connect(MYSQL_HOST, MYSQL_PORT, MYSQL_USER, MYSQL_PASSWORD)
|
|
|
|
result["steps"].extend(init_news_flash(conn))
|
|
result["steps"].extend(init_akshare_data(conn))
|
|
result["steps"].extend(init_tushare_data(conn))
|
|
|
|
conn.close()
|
|
|
|
result["success"] = True
|
|
result["message"] = f"数据库初始化完成，共 {len(result['steps'])} 步"
|
|
|
|
except Exception as e:
|
|
result["success"] = False
|
|
result["error"] = str(e)
|
|
print(f"[INIT ERROR] {e}", file=sys.stderr)
|
|
|
|
print(json.dumps(result, ensure_ascii=False))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|