refactor: migrate API keys to config, extend Kline intervals, add DB extensions
Security: - Move hardcoded Binance API key/secret from rest.ts to env.yaml (exchange config segment) - Add ExchangeConfig validation in config/validators.ts - Export typed exchange config from config/index.ts - Update AGENTS/07-caveats.md to reflect the new policy Kline intervals (add 3m / 2h / 6h / 8h / 1mon): - TypeScript: update KlineInterval type, KLINE_INTERVAL_MS mapping, build_aggregates_sql refresh chain - Python: sync KlineInterval Literal type, INTERVAL_TO_TABLE and INTERVAL_MS mappings, db_test table list - SQL: add 5 continuous aggregate materialized views (klines_3m/2h/6h/8h/1mon) with indexes - SQL: extend default kline_intervals in trading_pairs table - SQL: add cross-sectional query indexes for klines_1d and klines_1w DB: - Enable pg_prewarm extension (backtest warmup) - Enable pg_stat_statements extension (slow query monitoring) Other: - data/run/exchange.ts: graceful pgsql shutdown after backfill completes - Config path: load from data/env.yaml (symlink) instead of project root
This commit is contained in:
Rekey
@@ -13,12 +13,18 @@
|
||||
-- - klines 基表由 02-init-tables.sql 创建为 TimescaleDB hypertable
|
||||
-- - 连续聚合视图由 03-continuous-aggregates.sql 创建
|
||||
-- - TypeORM 的 synchronize:true 与 SQL 脚本互为 fallback（开发/生产双路径）
|
||||
-- - 本脚本为 init-db 链的第一环，仅负责扩展启用
|
||||
-- - 本脚本为 init-db 链的第一环，负责扩展启用（TimescaleDB / pg_prewarm / pg_stat_statements）
|
||||
-- ============================================================
|
||||
|
||||
-- 启用 TimescaleDB 扩展（必须最先执行）
|
||||
CREATE EXTENSION IF NOT EXISTS timescaledb CASCADE;
|
||||
|
||||
-- 启用 pg_prewarm 扩展（回测预热，减少首轮查询延迟）
|
||||
CREATE EXTENSION IF NOT EXISTS pg_prewarm;
|
||||
|
||||
-- 启用 pg_stat_statements 扩展（慢查询监控，零成本）
|
||||
CREATE EXTENSION IF NOT EXISTS pg_stat_statements;
|
||||
|
||||
-- 验证扩展已启用
|
||||
DO $$
|
||||
BEGIN
|
||||
|
||||
@@ -100,7 +100,7 @@ CREATE TABLE IF NOT EXISTS trading_pairs (
|
||||
kline_interval VARCHAR(100) NOT NULL DEFAULT '1m',
|
||||
|
||||
-- K 线合成周期列表（逗号分隔，如 "1m,5m,15m,1h,4h,1d"）
|
||||
kline_intervals VARCHAR(100) NOT NULL DEFAULT '1m,5m,15m,1h,4h,1d',
|
||||
kline_intervals VARCHAR(100) NOT NULL DEFAULT '1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,1d,1w,1mon',
|
||||
|
||||
-- 历史 K 线最后补全时间（UTC）。默认 Unix epoch 起始，
|
||||
-- 新交易对从 epoch 起始时间开始全量补拉。
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
-- 03-continuous-aggregates.sql — K 线分层连续聚合视图
|
||||
-- ============================================================
|
||||
-- 从 klines（1m）基表创建分层连续聚合物化视图链：
|
||||
-- 1m → 5m → 15m → 30m → 1h → 4h → 1d → 1w
|
||||
-- 1m → 3m → 5m → 15m → 30m → 1h → 2h → 4h → 6h → 8h → 1d → 1w → 1mon
|
||||
--
|
||||
-- 执行前提：
|
||||
-- 1. klines hypertable 已创建（由 02-init-tables.sql 创建）
|
||||
@@ -36,6 +36,37 @@
|
||||
-- 3. 接入实时数据（模式 A 启用 policy / 模式 B 应用层触发）
|
||||
-- ============================================================
|
||||
|
||||
-- ============================================================
|
||||
-- 3m K 线（从 1m 基表聚合）
|
||||
-- ============================================================
|
||||
CREATE MATERIALIZED VIEW IF NOT EXISTS klines_3m
|
||||
WITH (timescaledb.continuous) AS
|
||||
SELECT
|
||||
time_bucket('3 minutes', time) AS time,
|
||||
exchange,
|
||||
symbol,
|
||||
'3m'::text AS interval,
|
||||
FIRST(open, time) AS open,
|
||||
MAX(high) AS high,
|
||||
MIN(low) AS low,
|
||||
LAST(close, time) AS close,
|
||||
SUM(volume) AS volume,
|
||||
SUM(quote_volume) AS quote_volume,
|
||||
SUM(taker_buy_base_vol) AS taker_buy_base_vol,
|
||||
SUM(taker_buy_quote_vol) AS taker_buy_quote_vol,
|
||||
SUM(trade_count)::integer AS trade_count
|
||||
FROM klines
|
||||
GROUP BY time_bucket('3 minutes', klines.time), exchange, symbol
|
||||
WITH NO DATA;
|
||||
|
||||
-- 【模式 A 用户】取消下面注释以启用定时调度刷新
|
||||
-- SELECT add_continuous_aggregate_policy('klines_3m',
|
||||
-- start_offset => INTERVAL '1 day',
|
||||
-- end_offset => INTERVAL '3 minutes',
|
||||
-- schedule_interval => INTERVAL '3 minutes',
|
||||
-- if_not_exists => TRUE
|
||||
-- );
|
||||
|
||||
-- ============================================================
|
||||
-- 5m K 线（从 1m 基表聚合）
|
||||
-- ============================================================
|
||||
@@ -160,6 +191,37 @@ WITH NO DATA;
|
||||
-- if_not_exists => TRUE
|
||||
-- );
|
||||
|
||||
-- ============================================================
|
||||
-- 2h K 线（从 1h 聚合，分层链）
|
||||
-- ============================================================
|
||||
CREATE MATERIALIZED VIEW IF NOT EXISTS klines_2h
|
||||
WITH (timescaledb.continuous) AS
|
||||
SELECT
|
||||
time_bucket('2 hours', time) AS time,
|
||||
exchange,
|
||||
symbol,
|
||||
'2h'::text AS interval,
|
||||
FIRST(open, time) AS open,
|
||||
MAX(high) AS high,
|
||||
MIN(low) AS low,
|
||||
LAST(close, time) AS close,
|
||||
SUM(volume) AS volume,
|
||||
SUM(quote_volume) AS quote_volume,
|
||||
SUM(taker_buy_base_vol) AS taker_buy_base_vol,
|
||||
SUM(taker_buy_quote_vol) AS taker_buy_quote_vol,
|
||||
SUM(trade_count)::integer AS trade_count
|
||||
FROM klines_1h
|
||||
GROUP BY time_bucket('2 hours', klines_1h.time), exchange, symbol
|
||||
WITH NO DATA;
|
||||
|
||||
-- 【模式 A 用户】取消下面注释以启用定时调度刷新
|
||||
-- SELECT add_continuous_aggregate_policy('klines_2h',
|
||||
-- start_offset => INTERVAL '10 days',
|
||||
-- end_offset => INTERVAL '2 hours',
|
||||
-- schedule_interval => INTERVAL '2 hours',
|
||||
-- if_not_exists => TRUE
|
||||
-- );
|
||||
|
||||
-- ============================================================
|
||||
-- 4h K 线（从 1h 聚合，分层链）
|
||||
-- ============================================================
|
||||
@@ -191,6 +253,68 @@ WITH NO DATA;
|
||||
-- if_not_exists => TRUE
|
||||
-- );
|
||||
|
||||
-- ============================================================
|
||||
-- 6h K 线（从 1h 聚合，分层链）
|
||||
-- ============================================================
|
||||
CREATE MATERIALIZED VIEW IF NOT EXISTS klines_6h
|
||||
WITH (timescaledb.continuous) AS
|
||||
SELECT
|
||||
time_bucket('6 hours', time) AS time,
|
||||
exchange,
|
||||
symbol,
|
||||
'6h'::text AS interval,
|
||||
FIRST(open, time) AS open,
|
||||
MAX(high) AS high,
|
||||
MIN(low) AS low,
|
||||
LAST(close, time) AS close,
|
||||
SUM(volume) AS volume,
|
||||
SUM(quote_volume) AS quote_volume,
|
||||
SUM(taker_buy_base_vol) AS taker_buy_base_vol,
|
||||
SUM(taker_buy_quote_vol) AS taker_buy_quote_vol,
|
||||
SUM(trade_count)::integer AS trade_count
|
||||
FROM klines_1h
|
||||
GROUP BY time_bucket('6 hours', klines_1h.time), exchange, symbol
|
||||
WITH NO DATA;
|
||||
|
||||
-- 【模式 A 用户】取消下面注释以启用定时调度刷新
|
||||
-- SELECT add_continuous_aggregate_policy('klines_6h',
|
||||
-- start_offset => INTERVAL '20 days',
|
||||
-- end_offset => INTERVAL '6 hours',
|
||||
-- schedule_interval => INTERVAL '6 hours',
|
||||
-- if_not_exists => TRUE
|
||||
-- );
|
||||
|
||||
-- ============================================================
|
||||
-- 8h K 线（从 4h 聚合，分层链）
|
||||
-- ============================================================
|
||||
CREATE MATERIALIZED VIEW IF NOT EXISTS klines_8h
|
||||
WITH (timescaledb.continuous) AS
|
||||
SELECT
|
||||
time_bucket('8 hours', time) AS time,
|
||||
exchange,
|
||||
symbol,
|
||||
'8h'::text AS interval,
|
||||
FIRST(open, time) AS open,
|
||||
MAX(high) AS high,
|
||||
MIN(low) AS low,
|
||||
LAST(close, time) AS close,
|
||||
SUM(volume) AS volume,
|
||||
SUM(quote_volume) AS quote_volume,
|
||||
SUM(taker_buy_base_vol) AS taker_buy_base_vol,
|
||||
SUM(taker_buy_quote_vol) AS taker_buy_quote_vol,
|
||||
SUM(trade_count)::integer AS trade_count
|
||||
FROM klines_4h
|
||||
GROUP BY time_bucket('8 hours', klines_4h.time), exchange, symbol
|
||||
WITH NO DATA;
|
||||
|
||||
-- 【模式 A 用户】取消下面注释以启用定时调度刷新
|
||||
-- SELECT add_continuous_aggregate_policy('klines_8h',
|
||||
-- start_offset => INTERVAL '30 days',
|
||||
-- end_offset => INTERVAL '8 hours',
|
||||
-- schedule_interval => INTERVAL '8 hours',
|
||||
-- if_not_exists => TRUE
|
||||
-- );
|
||||
|
||||
-- ============================================================
|
||||
-- 1d K 线（从 4h 聚合，分层链）
|
||||
-- ============================================================
|
||||
@@ -253,25 +377,75 @@ WITH NO DATA;
|
||||
-- if_not_exists => TRUE
|
||||
-- );
|
||||
|
||||
-- ============================================================
|
||||
-- 1mon K 线（从 1d 聚合，分层链）
|
||||
-- ============================================================
|
||||
CREATE MATERIALIZED VIEW IF NOT EXISTS klines_1mon
|
||||
WITH (timescaledb.continuous) AS
|
||||
SELECT
|
||||
time_bucket('1 month', time) AS time,
|
||||
exchange,
|
||||
symbol,
|
||||
'1mon'::text AS interval,
|
||||
FIRST(open, time) AS open,
|
||||
MAX(high) AS high,
|
||||
MIN(low) AS low,
|
||||
LAST(close, time) AS close,
|
||||
SUM(volume) AS volume,
|
||||
SUM(quote_volume) AS quote_volume,
|
||||
SUM(taker_buy_base_vol) AS taker_buy_base_vol,
|
||||
SUM(taker_buy_quote_vol) AS taker_buy_quote_vol,
|
||||
SUM(trade_count)::integer AS trade_count
|
||||
FROM klines_1d
|
||||
GROUP BY time_bucket('1 month', klines_1d.time), exchange, symbol
|
||||
WITH NO DATA;
|
||||
|
||||
-- 【模式 A 用户】取消下面注释以启用定时调度刷新
|
||||
-- SELECT add_continuous_aggregate_policy('klines_1mon',
|
||||
-- start_offset => INTERVAL '365 days',
|
||||
-- end_offset => INTERVAL '1 day',
|
||||
-- schedule_interval => INTERVAL '1 day',
|
||||
-- if_not_exists => TRUE
|
||||
-- );
|
||||
|
||||
-- ============================================================
|
||||
-- 推荐索引：加速按 symbol + time 的查询
|
||||
-- ============================================================
|
||||
CREATE INDEX IF NOT EXISTS idx_klines_3m_symbol_time ON klines_3m (exchange, symbol, time DESC);
|
||||
CREATE INDEX IF NOT EXISTS idx_klines_5m_symbol_time ON klines_5m (exchange, symbol, time DESC);
|
||||
CREATE INDEX IF NOT EXISTS idx_klines_15m_symbol_time ON klines_15m (exchange, symbol, time DESC);
|
||||
CREATE INDEX IF NOT EXISTS idx_klines_30m_symbol_time ON klines_30m (exchange, symbol, time DESC);
|
||||
CREATE INDEX IF NOT EXISTS idx_klines_1h_symbol_time ON klines_1h (exchange, symbol, time DESC);
|
||||
CREATE INDEX IF NOT EXISTS idx_klines_2h_symbol_time ON klines_2h (exchange, symbol, time DESC);
|
||||
CREATE INDEX IF NOT EXISTS idx_klines_4h_symbol_time ON klines_4h (exchange, symbol, time DESC);
|
||||
CREATE INDEX IF NOT EXISTS idx_klines_6h_symbol_time ON klines_6h (exchange, symbol, time DESC);
|
||||
CREATE INDEX IF NOT EXISTS idx_klines_8h_symbol_time ON klines_8h (exchange, symbol, time DESC);
|
||||
CREATE INDEX IF NOT EXISTS idx_klines_1d_symbol_time ON klines_1d (exchange, symbol, time DESC);
|
||||
CREATE INDEX IF NOT EXISTS idx_klines_1w_symbol_time ON klines_1w (exchange, symbol, time DESC);
|
||||
CREATE INDEX IF NOT EXISTS idx_klines_1mon_symbol_time ON klines_1mon (exchange, symbol, time DESC);
|
||||
|
||||
-- ============================================================
|
||||
-- 截面查询索引：加速同一时间点多品种回测查询
|
||||
-- 查询模式：WHERE exchange='binance' AND time='2024-01-01' AND symbol IN (…)
|
||||
-- 回测中跨品种截面查询最常见于日线和周线，因此只在这两层建额外索引。
|
||||
-- 如需其他周期（如 1h、4h）的截面查询，按同样模式扩展。
|
||||
-- ============================================================
|
||||
CREATE INDEX IF NOT EXISTS idx_klines_1d_exchange_time_symbol ON klines_1d (exchange, time DESC, symbol);
|
||||
CREATE INDEX IF NOT EXISTS idx_klines_1w_exchange_time_symbol ON klines_1w (exchange, time DESC, symbol);
|
||||
|
||||
-- ============================================================
|
||||
-- 首次创建后手动刷新所有视图（填充历史数据）
|
||||
-- 取消注释以下行执行：
|
||||
-- ============================================================
|
||||
-- CALL refresh_continuous_aggregate('klines_3m', NULL, NULL);
|
||||
-- CALL refresh_continuous_aggregate('klines_5m', NULL, NULL);
|
||||
-- CALL refresh_continuous_aggregate('klines_15m', NULL, NULL);
|
||||
-- CALL refresh_continuous_aggregate('klines_30m', NULL, NULL);
|
||||
-- CALL refresh_continuous_aggregate('klines_1h', NULL, NULL);
|
||||
-- CALL refresh_continuous_aggregate('klines_2h', NULL, NULL);
|
||||
-- CALL refresh_continuous_aggregate('klines_4h', NULL, NULL);
|
||||
-- CALL refresh_continuous_aggregate('klines_6h', NULL, NULL);
|
||||
-- CALL refresh_continuous_aggregate('klines_8h', NULL, NULL);
|
||||
-- CALL refresh_continuous_aggregate('klines_1d', NULL, NULL);
|
||||
-- CALL refresh_continuous_aggregate('klines_1w', NULL, NULL);
|
||||
-- CALL refresh_continuous_aggregate('klines_1mon', NULL, NULL);
|
||||
|
||||
Reference in New Issue
Block a user