feat(data): 实现配置表 CRUD 与 Schema 初始化拆分
- 新增 data/db/ 数据库访问层：pool 管理、类型定义、Zod 校验、参数化 SQL 查询 - 新增 data/db/config-crud.ts：MonitoredSymbolsRepo / ExchangeConfigRepo / AppConfigRepo 三个 CRUD 服务类 - 新增 data/config.ts：中心化配置模块，零依赖 .env 解析 + Zod 校验 - 新增 data/schema/：klines.sql + config.sql 参考 DDL - 新增 data/exchanges/：交易所类型定义与 Binance WebSocket 封装 - 新增 data/run/：交易所连接启动入口 - 重构 data/init-db/：001_init.sql 仅保留 TimescaleDB + klines，配置表拆分至 002_config.sql - 更新 docker-compose.yml：挂载 init-db 初始化脚本
This commit is contained in:
Rekey
@@ -0,0 +1,177 @@
|
||||
-- ============================================================
|
||||
-- schema/config.sql — 配置表 DDL（参考副本）
|
||||
-- ============================================================
|
||||
-- 数据库：TimescaleDB (PostgreSQL 17)
|
||||
-- 说明：管理系统配置、监控标的、交易所连接参数
|
||||
--
|
||||
-- ⚠️ 权威初始化脚本已迁移至：data/init-db/001_init.sql
|
||||
-- 本文件保留作为 pg.initSchema() 非 Docker 部署的回退方案和文档参考。
|
||||
-- 修改表结构时请同步更新 001_init.sql。
|
||||
-- ============================================================
|
||||
|
||||
-- ============================================================
|
||||
-- 1. monitored_symbols — 监控交易对配置
|
||||
-- ============================================================
|
||||
-- 用途：声明数据采集模块需要订阅哪些交易对的 K 线流
|
||||
-- 消费方：WebSocket 行情采集服务启动时读取此表决定订阅列表
|
||||
-- ============================================================
|
||||
CREATE TABLE IF NOT EXISTS monitored_symbols (
|
||||
id SERIAL PRIMARY KEY,
|
||||
|
||||
-- ---- 标识维度（与 klines 表对齐） ----
|
||||
exchange TEXT NOT NULL, -- 交易所：binance / okx / bybit
|
||||
symbol TEXT NOT NULL, -- 交易对：BTCUSDT / ETHUSDT
|
||||
interval TEXT NOT NULL, -- K 线周期：1m / 5m / 15m / 1h / 4h / 1d
|
||||
|
||||
-- ---- 控制字段 ----
|
||||
enabled BOOLEAN NOT NULL DEFAULT TRUE, -- 是否启用采集
|
||||
priority SMALLINT NOT NULL DEFAULT 0, -- 优先级（数值越大越优先，用于限频时取舍）
|
||||
|
||||
-- ---- 备注 ----
|
||||
label TEXT, -- 人类可读标签，如 "BTC/USDT 1分钟线"
|
||||
notes TEXT, -- 备注说明
|
||||
|
||||
-- ---- 元数据 ----
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
||||
|
||||
-- 同一 (exchange, symbol, interval) 组合不可重复
|
||||
UNIQUE (exchange, symbol, interval)
|
||||
);
|
||||
|
||||
-- 索引：按启用状态快速筛选
|
||||
CREATE INDEX IF NOT EXISTS idx_monitored_symbols_enabled
|
||||
ON monitored_symbols (enabled, exchange, priority DESC);
|
||||
|
||||
-- 索引：按交易对查找所有周期
|
||||
CREATE INDEX IF NOT EXISTS idx_monitored_symbols_symbol
|
||||
ON monitored_symbols (symbol, interval);
|
||||
|
||||
COMMENT ON TABLE monitored_symbols IS '监控交易对配置表：声明哪些 (交易所,交易对,周期) 需要采集 K 线数据';
|
||||
COMMENT ON COLUMN monitored_symbols.exchange IS '交易所标识：binance / okx / bybit';
|
||||
COMMENT ON COLUMN monitored_symbols.symbol IS '交易对：BTCUSDT / ETHUSDT';
|
||||
COMMENT ON COLUMN monitored_symbols.interval IS 'K 线周期：1m / 5m / 15m / 1h / 4h / 1d';
|
||||
COMMENT ON COLUMN monitored_symbols.enabled IS '是否启用 WebSocket 订阅';
|
||||
COMMENT ON COLUMN monitored_symbols.priority IS '优先级（0-32767），限频时高优先级交易对优先保留';
|
||||
|
||||
|
||||
-- ============================================================
|
||||
-- 2. exchange_config — 交易所连接配置
|
||||
-- ============================================================
|
||||
-- 用途：存储各交易所的 API 端点、限频参数等连接级配置
|
||||
-- 安全提醒：API Key/Secret 不应明文存储于此表，
|
||||
-- 建议通过环境变量或 Vault 注入，此表仅存非敏感参数
|
||||
-- ============================================================
|
||||
CREATE TABLE IF NOT EXISTS exchange_config (
|
||||
id SERIAL PRIMARY KEY,
|
||||
|
||||
-- ---- 交易所标识 ----
|
||||
exchange TEXT NOT NULL UNIQUE, -- 交易所：binance / okx / bybit
|
||||
|
||||
-- ---- 连接参数 ----
|
||||
rest_url TEXT, -- REST API 基础 URL（留空则用 SDK 默认值）
|
||||
ws_url TEXT, -- WebSocket 基础 URL（留空则用 SDK 默认值）
|
||||
ws_ping_interval_ms INTEGER NOT NULL DEFAULT 30000, -- 心跳间隔（毫秒）
|
||||
|
||||
-- ---- 限频控制 ----
|
||||
rate_limit_per_sec REAL NOT NULL DEFAULT 20.0, -- 每秒最大请求数
|
||||
max_reconnect_attempts INT NOT NULL DEFAULT 10, -- 最大重连次数
|
||||
reconnect_delay_ms INT NOT NULL DEFAULT 3000, -- 重连延迟基数（指数退避）
|
||||
|
||||
-- ---- 开关 ----
|
||||
enabled BOOLEAN NOT NULL DEFAULT TRUE, -- 是否启用该交易所
|
||||
|
||||
-- ---- 备注 ----
|
||||
notes TEXT,
|
||||
|
||||
-- ---- 元数据 ----
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
|
||||
);
|
||||
|
||||
COMMENT ON TABLE exchange_config IS '交易所连接配置表：REST/WS 端点、限频、重连策略';
|
||||
COMMENT ON COLUMN exchange_config.rest_url IS 'REST API 地址，空则使用 SDK 默认';
|
||||
COMMENT ON COLUMN exchange_config.ws_url IS 'WebSocket 地址，空则使用 SDK 默认';
|
||||
COMMENT ON COLUMN exchange_config.rate_limit_per_sec IS '每秒最大请求数（Binance 默认 20/s）';
|
||||
COMMENT ON COLUMN exchange_config.max_reconnect_attempts IS 'WebSocket 断线最大重连次数';
|
||||
COMMENT ON COLUMN exchange_config.reconnect_delay_ms IS '重连退避基数（实际延迟 = 基数 × 2^attempts）';
|
||||
|
||||
|
||||
-- ============================================================
|
||||
-- 3. app_config — 全局应用配置（Key-Value）
|
||||
-- ============================================================
|
||||
-- 用途：存储不适合硬编码的运行时参数，如批量写入阈值
|
||||
-- ============================================================
|
||||
CREATE TABLE IF NOT EXISTS app_config (
|
||||
id SERIAL PRIMARY KEY,
|
||||
key TEXT NOT NULL UNIQUE, -- 配置键
|
||||
value TEXT NOT NULL, -- 配置值（统一存为文本，消费方自行解析类型）
|
||||
description TEXT, -- 说明
|
||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
|
||||
);
|
||||
|
||||
COMMENT ON TABLE app_config IS '全局应用配置（KV 结构），运行时参数集中管理';
|
||||
|
||||
-- 默认配置项
|
||||
INSERT INTO app_config (key, value, description) VALUES
|
||||
('batch_size', '500', '批量写入缓冲区条数阈值'),
|
||||
('flush_interval_ms', '1000', '缓冲区最大等待时间（毫秒）'),
|
||||
('log_level', 'info', '日志级别：trace / debug / info / warn / error'),
|
||||
('redis_publish_enabled', 'true', '是否启用 Redis 发布')
|
||||
ON CONFLICT (key) DO NOTHING;
|
||||
|
||||
|
||||
-- ============================================================
|
||||
-- 4. 初始数据：预置常见交易对的监控配置
|
||||
-- ============================================================
|
||||
-- 以下为建议的默认监控列表，可根据实际需求增删
|
||||
INSERT INTO monitored_symbols (exchange, symbol, interval, enabled, priority, label) VALUES
|
||||
-- Binance 主力交易对 — 1m
|
||||
('binance', 'BTCUSDT', '1m', TRUE, 10, 'BTC/USDT 1分钟线'),
|
||||
('binance', 'ETHUSDT', '1m', TRUE, 9, 'ETH/USDT 1分钟线'),
|
||||
('binance', 'SOLUSDT', '1m', TRUE, 8, 'SOL/USDT 1分钟线'),
|
||||
('binance', 'BNBUSDT', '1m', TRUE, 7, 'BNB/USDT 1分钟线'),
|
||||
|
||||
-- Binance 主力交易对 — 1h（策略用）
|
||||
('binance', 'BTCUSDT', '1h', TRUE, 10, 'BTC/USDT 1小时线'),
|
||||
('binance', 'ETHUSDT', '1h', TRUE, 9, 'ETH/USDT 1小时线'),
|
||||
('binance', 'SOLUSDT', '1h', TRUE, 8, 'SOL/USDT 1小时线'),
|
||||
|
||||
-- Binance 主力交易对 — 1d（日线）
|
||||
('binance', 'BTCUSDT', '1d', TRUE, 10, 'BTC/USDT 日线'),
|
||||
('binance', 'ETHUSDT', '1d', TRUE, 9, 'ETH/USDT 日线')
|
||||
ON CONFLICT (exchange, symbol, interval) DO NOTHING;
|
||||
|
||||
-- 预置交易所默认连接配置
|
||||
INSERT INTO exchange_config (exchange, rate_limit_per_sec, notes) VALUES
|
||||
('binance', 20.0, 'Binance 现货 — 默认权重限频 1200/min'),
|
||||
('okx', 10.0, 'OKX 现货 — 默认限频 10/s'),
|
||||
('bybit', 10.0, 'Bybit 现货 — 默认限频 10/s')
|
||||
ON CONFLICT (exchange) DO NOTHING;
|
||||
|
||||
|
||||
-- ============================================================
|
||||
-- 5. 常用查询示例
|
||||
-- ============================================================
|
||||
|
||||
-- 查询所有启用的监控标的（采集服务启动时使用）
|
||||
-- SELECT exchange, symbol, interval, priority
|
||||
-- FROM monitored_symbols
|
||||
-- WHERE enabled = TRUE
|
||||
-- ORDER BY exchange, priority DESC, symbol, interval;
|
||||
|
||||
-- 查询某交易所下所有交易对
|
||||
-- SELECT DISTINCT symbol
|
||||
-- FROM monitored_symbols
|
||||
-- WHERE exchange = 'binance' AND enabled = TRUE
|
||||
-- ORDER BY symbol;
|
||||
|
||||
-- 禁用某个交易对的采集
|
||||
-- UPDATE monitored_symbols SET enabled = FALSE, updated_at = NOW()
|
||||
-- WHERE exchange = 'binance' AND symbol = 'BTCUSDT' AND interval = '1m';
|
||||
|
||||
-- 新增监控标的（动态添加，无需重启）
|
||||
-- INSERT INTO monitored_symbols (exchange, symbol, interval, enabled, priority, label)
|
||||
-- VALUES ('binance', 'DOGEUSDT', '1m', TRUE, 5, 'DOGE/USDT 1分钟线')
|
||||
-- ON CONFLICT (exchange, symbol, interval) DO UPDATE
|
||||
-- SET enabled = TRUE, updated_at = NOW();
|
||||
@@ -0,0 +1,206 @@
|
||||
-- ============================================================
|
||||
-- schema/klines.sql — K 线表 DDL（参考副本）
|
||||
-- ============================================================
|
||||
-- 数据库：TimescaleDB (PostgreSQL 17 + timescaledb 2.x)
|
||||
-- 说明：存储全交易所 OHLCV 数据，按时间自动分区压缩
|
||||
--
|
||||
-- ⚠️ 权威初始化脚本：data/init-db/001_init.sql
|
||||
-- 本文件保留作为 pg.initSchema() 非 Docker 部署的回退方案和文档参考。
|
||||
-- 修改表结构时请同步更新 001_init.sql。
|
||||
-- ============================================================
|
||||
|
||||
-- ============================================================
|
||||
-- 1. klines — K 线主表（hypertable）
|
||||
-- ============================================================
|
||||
CREATE TABLE IF NOT EXISTS klines (
|
||||
-- ---- 时间维度 ----
|
||||
time TIMESTAMPTZ NOT NULL, -- K 线开盘时间（UTC）
|
||||
|
||||
-- ---- 标识维度 ----
|
||||
exchange TEXT NOT NULL, -- 交易所：binance / okx / bybit
|
||||
symbol TEXT NOT NULL, -- 交易对：BTCUSDT / ETHUSDT
|
||||
interval TEXT NOT NULL, -- 周期：1m / 5m / 15m / 1h / 4h / 1d
|
||||
|
||||
-- ---- OHLCV 核心数据 ----
|
||||
open NUMERIC(20,8) NOT NULL,
|
||||
high NUMERIC(20,8) NOT NULL,
|
||||
low NUMERIC(20,8) NOT NULL,
|
||||
close NUMERIC(20,8) NOT NULL,
|
||||
volume NUMERIC(20,8) NOT NULL DEFAULT 0, -- 成交量（基准币种）
|
||||
|
||||
-- ---- 扩展字段 ----
|
||||
quote_volume NUMERIC(20,8) DEFAULT 0, -- 成交额（计价币种）
|
||||
taker_buy_base_vol NUMERIC(20,8) DEFAULT 0, -- 主动买入量（基准币种）
|
||||
taker_buy_quote_vol NUMERIC(20,8) DEFAULT 0, -- 主动买入额（计价币种）
|
||||
trade_count INTEGER DEFAULT 0, -- 成交笔数
|
||||
is_closed BOOLEAN DEFAULT TRUE, -- K 线是否已闭合
|
||||
|
||||
-- ---- 元数据 ----
|
||||
created_at TIMESTAMPTZ DEFAULT NOW(),
|
||||
updated_at TIMESTAMPTZ DEFAULT NOW(),
|
||||
|
||||
-- 唯一约束：同一根 K 线不可重复
|
||||
UNIQUE (time, exchange, symbol, interval)
|
||||
);
|
||||
|
||||
-- ============================================================
|
||||
-- 2. 转换为 TimescaleDB hypertable
|
||||
-- ============================================================
|
||||
-- 按 time 列做 1 天分区，按 exchange 做 4 空间分区
|
||||
SELECT create_hypertable(
|
||||
'klines',
|
||||
'time',
|
||||
chunk_time_interval => INTERVAL '1 day',
|
||||
partitioning_column => 'exchange',
|
||||
number_partitions => 4,
|
||||
if_not_exists => TRUE
|
||||
);
|
||||
|
||||
-- ============================================================
|
||||
-- 3. 索引设计
|
||||
-- ============================================================
|
||||
|
||||
-- 主力查询索引（覆盖 95% 查询场景）
|
||||
-- 用途：按交易对+周期+时间范围查询最新 K 线
|
||||
CREATE INDEX IF NOT EXISTS idx_klines_lookup
|
||||
ON klines (exchange, symbol, interval, time DESC);
|
||||
|
||||
-- 回测专用索引
|
||||
-- 用途：按交易对+周期+时间正序遍历（策略回测）
|
||||
CREATE INDEX IF NOT EXISTS idx_klines_backtest
|
||||
ON klines (symbol, interval, time ASC);
|
||||
|
||||
-- 最新已闭合 K 线索引（部分索引，减小体积）
|
||||
-- 用途：获取已完成的最新 K 线（避免扫描未闭合数据）
|
||||
CREATE INDEX IF NOT EXISTS idx_klines_latest
|
||||
ON klines (exchange, symbol, interval, time DESC)
|
||||
WHERE is_closed = TRUE;
|
||||
|
||||
-- ============================================================
|
||||
-- 4. 压缩策略（列式压缩，压缩比约 90%）
|
||||
-- ============================================================
|
||||
ALTER TABLE klines SET (
|
||||
timescaledb.compress,
|
||||
timescaledb.compress_segmentby = 'exchange, symbol, interval',
|
||||
timescaledb.compress_orderby = 'time DESC'
|
||||
);
|
||||
|
||||
-- K 线闭合 7 天后自动触发压缩
|
||||
SELECT add_compression_policy('klines', INTERVAL '7 days', if_not_exists => TRUE);
|
||||
|
||||
-- ============================================================
|
||||
-- 5. 数据保留策略
|
||||
-- ============================================================
|
||||
-- 1m 粒度 K 线保留 90 天（更粗粒度由连续聚合视图覆盖）
|
||||
SELECT add_retention_policy('klines', INTERVAL '90 days', if_not_exists => TRUE);
|
||||
|
||||
-- ============================================================
|
||||
-- 6. 连续聚合视图（从 1m 自动派生高周期 K 线）
|
||||
-- ============================================================
|
||||
|
||||
-- ---- 5m K 线 ----
|
||||
CREATE MATERIALIZED VIEW IF NOT EXISTS klines_5m
|
||||
WITH (timescaledb.continuous) AS
|
||||
SELECT
|
||||
time_bucket('5 minutes', time) AS time,
|
||||
exchange,
|
||||
symbol,
|
||||
'5m'::TEXT AS interval,
|
||||
FIRST(open, time) AS open,
|
||||
MAX(high) AS high,
|
||||
MIN(low) AS low,
|
||||
LAST(close, time) AS close,
|
||||
SUM(volume) AS volume,
|
||||
SUM(quote_volume) AS quote_volume,
|
||||
SUM(taker_buy_base_vol) AS taker_buy_base_vol,
|
||||
SUM(taker_buy_quote_vol) AS taker_buy_quote_vol,
|
||||
SUM(trade_count) AS trade_count
|
||||
FROM klines
|
||||
WHERE interval = '1m'
|
||||
GROUP BY time_bucket('5 minutes', time), exchange, symbol;
|
||||
|
||||
-- ---- 15m K 线 ----
|
||||
CREATE MATERIALIZED VIEW IF NOT EXISTS klines_15m
|
||||
WITH (timescaledb.continuous) AS
|
||||
SELECT
|
||||
time_bucket('15 minutes', time) AS time,
|
||||
exchange,
|
||||
symbol,
|
||||
'15m'::TEXT AS interval,
|
||||
FIRST(open, time) AS open,
|
||||
MAX(high) AS high,
|
||||
MIN(low) AS low,
|
||||
LAST(close, time) AS close,
|
||||
SUM(volume) AS volume,
|
||||
SUM(quote_volume) AS quote_volume,
|
||||
SUM(taker_buy_base_vol) AS taker_buy_base_vol,
|
||||
SUM(taker_buy_quote_vol) AS taker_buy_quote_vol,
|
||||
SUM(trade_count) AS trade_count
|
||||
FROM klines
|
||||
WHERE interval = '1m'
|
||||
GROUP BY time_bucket('15 minutes', time), exchange, symbol;
|
||||
|
||||
-- ---- 1h K 线 ----
|
||||
CREATE MATERIALIZED VIEW IF NOT EXISTS klines_1h
|
||||
WITH (timescaledb.continuous) AS
|
||||
SELECT
|
||||
time_bucket('1 hour', time) AS time,
|
||||
exchange,
|
||||
symbol,
|
||||
'1h'::TEXT AS interval,
|
||||
FIRST(open, time) AS open,
|
||||
MAX(high) AS high,
|
||||
MIN(low) AS low,
|
||||
LAST(close, time) AS close,
|
||||
SUM(volume) AS volume,
|
||||
SUM(quote_volume) AS quote_volume,
|
||||
SUM(taker_buy_base_vol) AS taker_buy_base_vol,
|
||||
SUM(taker_buy_quote_vol) AS taker_buy_quote_vol,
|
||||
SUM(trade_count) AS trade_count
|
||||
FROM klines
|
||||
WHERE interval = '1m'
|
||||
GROUP BY time_bucket('1 hour', time), exchange, symbol;
|
||||
|
||||
-- ---- 1d K 线 ----
|
||||
CREATE MATERIALIZED VIEW IF NOT EXISTS klines_1d
|
||||
WITH (timescaledb.continuous) AS
|
||||
SELECT
|
||||
time_bucket('1 day', time) AS time,
|
||||
exchange,
|
||||
symbol,
|
||||
'1d'::TEXT AS interval,
|
||||
FIRST(open, time) AS open,
|
||||
MAX(high) AS high,
|
||||
MIN(low) AS low,
|
||||
LAST(close, time) AS close,
|
||||
SUM(volume) AS volume,
|
||||
SUM(quote_volume) AS quote_volume,
|
||||
SUM(taker_buy_base_vol) AS taker_buy_base_vol,
|
||||
SUM(taker_buy_quote_vol) AS taker_buy_quote_vol,
|
||||
SUM(trade_count) AS trade_count
|
||||
FROM klines
|
||||
WHERE interval = '1m'
|
||||
GROUP BY time_bucket('1 day', time), exchange, symbol;
|
||||
|
||||
-- 连续聚合视图也启用压缩
|
||||
ALTER MATERIALIZED VIEW klines_5m SET (timescaledb.compress = true);
|
||||
ALTER MATERIALIZED VIEW klines_15m SET (timescaledb.compress = true);
|
||||
ALTER MATERIALIZED VIEW klines_1h SET (timescaledb.compress = true);
|
||||
ALTER MATERIALIZED VIEW klines_1d SET (timescaledb.compress = true);
|
||||
|
||||
-- ============================================================
|
||||
-- 7. 常用查询示例
|
||||
-- ============================================================
|
||||
|
||||
-- 查询最新 N 根 1h K 线
|
||||
-- SELECT time, open, high, low, close, volume
|
||||
-- FROM klines_1h
|
||||
-- WHERE exchange = 'binance' AND symbol = 'BTCUSDT'
|
||||
-- ORDER BY time DESC LIMIT 100;
|
||||
|
||||
-- 查询某个时间范围内的原始 1m K 线
|
||||
-- SELECT time, open, high, low, close, volume
|
||||
-- FROM klines
|
||||
-- WHERE exchange = 'binance' AND symbol = 'ETHUSDT' AND interval = '1m'
|
||||
-- AND time BETWEEN '2026-06-01' AND '2026-06-06'
|
||||
-- ORDER BY time ASC;
|
||||
Reference in New Issue
Block a user