Rekey
9351dec226
Security: - Move hardcoded Binance API key/secret from rest.ts to env.yaml (exchange config segment) - Add ExchangeConfig validation in config/validators.ts - Export typed exchange config from config/index.ts - Update AGENTS/07-caveats.md to reflect the new policy Kline intervals (add 3m / 2h / 6h / 8h / 1mon): - TypeScript: update KlineInterval type, KLINE_INTERVAL_MS mapping, build_aggregates_sql refresh chain - Python: sync KlineInterval Literal type, INTERVAL_TO_TABLE and INTERVAL_MS mappings, db_test table list - SQL: add 5 continuous aggregate materialized views (klines_3m/2h/6h/8h/1mon) with indexes - SQL: extend default kline_intervals in trading_pairs table - SQL: add cross-sectional query indexes for klines_1d and klines_1w DB: - Enable pg_prewarm extension (backtest warmup) - Enable pg_stat_statements extension (slow query monitoring) Other: - data/run/exchange.ts: graceful pgsql shutdown after backfill completes - Config path: load from data/env.yaml (symlink) instead of project root
452 lines
17 KiB
SQL
452 lines
17 KiB
SQL
-- ============================================================
|
|
-- 03-continuous-aggregates.sql — K 线分层连续聚合视图
|
|
-- ============================================================
|
|
-- 从 klines（1m）基表创建分层连续聚合物化视图链：
|
|
-- 1m → 3m → 5m → 15m → 30m → 1h → 2h → 4h → 6h → 8h → 1d → 1w → 1mon
|
|
--
|
|
-- 执行前提：
|
|
-- 1. klines hypertable 已创建（由 02-init-tables.sql 创建）
|
|
-- 2. klines 表中已有数据（至少一条，否则视图创建成功但无数据）
|
|
--
|
|
-- 执行方式：
|
|
-- psql -U trader -d trade -f 03-continuous-aggregates.sql
|
|
--
|
|
-- 幂等性：使用 IF NOT EXISTS，可重复执行
|
|
--
|
|
-- ============================================================
|
|
-- 聚合刷新模式选择（二选一）
|
|
-- ============================================================
|
|
-- 本脚本默认注释掉所有 add_continuous_aggregate_policy 调用。
|
|
-- 你需要根据部署场景选择一种刷新模式：
|
|
--
|
|
-- 【模式 A：定时调度刷新】（传统方式，适合简单场景）
|
|
-- 取消注释各节的 add_continuous_aggregate_policy 调用即可。
|
|
-- TimescaleDB Job Scheduler 按 schedule_interval 自动刷新。
|
|
-- 缺点：回填期间可能与 INSERT 竞争资源；聚合有调度延迟。
|
|
--
|
|
-- 【模式 B：应用层触发式刷新】（推荐，精细控制）
|
|
-- 保持 policy 注释状态。在应用层写入每条 1m K 线后，
|
|
-- 检测时间桶是否关闭，若关闭则调用 refresh_continuous_aggregate。
|
|
-- 优点：回填零干扰；聚合零延迟（桶关闭立即刷新）；无调度开销。
|
|
-- 应用层代码模板见 04-backfill-workflow.sql 末尾。
|
|
--
|
|
-- 回填工作流（两种模式通用）：
|
|
-- 1. 批量 INSERT 历史 K 线（policy 已注释，不冲突）
|
|
-- 2. 手动全量刷新所有视图（见文件末尾注释块）
|
|
-- 3. 接入实时数据（模式 A 启用 policy / 模式 B 应用层触发）
|
|
-- ============================================================
|
|
|
|
-- ============================================================
|
|
-- 3m K 线（从 1m 基表聚合）
|
|
-- ============================================================
|
|
CREATE MATERIALIZED VIEW IF NOT EXISTS klines_3m
|
|
WITH (timescaledb.continuous) AS
|
|
SELECT
|
|
time_bucket('3 minutes', time) AS time,
|
|
exchange,
|
|
symbol,
|
|
'3m'::text AS interval,
|
|
FIRST(open, time) AS open,
|
|
MAX(high) AS high,
|
|
MIN(low) AS low,
|
|
LAST(close, time) AS close,
|
|
SUM(volume) AS volume,
|
|
SUM(quote_volume) AS quote_volume,
|
|
SUM(taker_buy_base_vol) AS taker_buy_base_vol,
|
|
SUM(taker_buy_quote_vol) AS taker_buy_quote_vol,
|
|
SUM(trade_count)::integer AS trade_count
|
|
FROM klines
|
|
GROUP BY time_bucket('3 minutes', klines.time), exchange, symbol
|
|
WITH NO DATA;
|
|
|
|
-- 【模式 A 用户】取消下面注释以启用定时调度刷新
|
|
-- SELECT add_continuous_aggregate_policy('klines_3m',
|
|
-- start_offset => INTERVAL '1 day',
|
|
-- end_offset => INTERVAL '3 minutes',
|
|
-- schedule_interval => INTERVAL '3 minutes',
|
|
-- if_not_exists => TRUE
|
|
-- );
|
|
|
|
-- ============================================================
|
|
-- 5m K 线（从 1m 基表聚合）
|
|
-- ============================================================
|
|
CREATE MATERIALIZED VIEW IF NOT EXISTS klines_5m
|
|
WITH (timescaledb.continuous) AS
|
|
SELECT
|
|
time_bucket('5 minutes', time) AS time,
|
|
exchange,
|
|
symbol,
|
|
'5m'::text AS interval,
|
|
FIRST(open, time) AS open,
|
|
MAX(high) AS high,
|
|
MIN(low) AS low,
|
|
LAST(close, time) AS close,
|
|
SUM(volume) AS volume,
|
|
SUM(quote_volume) AS quote_volume,
|
|
SUM(taker_buy_base_vol) AS taker_buy_base_vol,
|
|
SUM(taker_buy_quote_vol) AS taker_buy_quote_vol,
|
|
SUM(trade_count)::integer AS trade_count
|
|
FROM klines
|
|
GROUP BY time_bucket('5 minutes', klines.time), exchange, symbol
|
|
WITH NO DATA;
|
|
|
|
-- 【模式 A 用户】取消下面注释以启用定时调度刷新
|
|
-- SELECT add_continuous_aggregate_policy('klines_5m',
|
|
-- start_offset => INTERVAL '1 day',
|
|
-- end_offset => INTERVAL '5 minutes',
|
|
-- schedule_interval => INTERVAL '5 minutes',
|
|
-- if_not_exists => TRUE
|
|
-- );
|
|
|
|
-- ============================================================
|
|
-- 15m K 线（从 5m 聚合，分层链）
|
|
-- ============================================================
|
|
CREATE MATERIALIZED VIEW IF NOT EXISTS klines_15m
|
|
WITH (timescaledb.continuous) AS
|
|
SELECT
|
|
time_bucket('15 minutes', time) AS time,
|
|
exchange,
|
|
symbol,
|
|
'15m'::text AS interval,
|
|
FIRST(open, time) AS open,
|
|
MAX(high) AS high,
|
|
MIN(low) AS low,
|
|
LAST(close, time) AS close,
|
|
SUM(volume) AS volume,
|
|
SUM(quote_volume) AS quote_volume,
|
|
SUM(taker_buy_base_vol) AS taker_buy_base_vol,
|
|
SUM(taker_buy_quote_vol) AS taker_buy_quote_vol,
|
|
SUM(trade_count)::integer AS trade_count
|
|
FROM klines_5m
|
|
GROUP BY time_bucket('15 minutes', klines_5m.time), exchange, symbol
|
|
WITH NO DATA;
|
|
|
|
-- 【模式 A 用户】取消下面注释以启用定时调度刷新
|
|
-- SELECT add_continuous_aggregate_policy('klines_15m',
|
|
-- start_offset => INTERVAL '2 days',
|
|
-- end_offset => INTERVAL '15 minutes',
|
|
-- schedule_interval => INTERVAL '15 minutes',
|
|
-- if_not_exists => TRUE
|
|
-- );
|
|
|
|
-- ============================================================
|
|
-- 30m K 线（从 15m 聚合，分层链）
|
|
-- ============================================================
|
|
CREATE MATERIALIZED VIEW IF NOT EXISTS klines_30m
|
|
WITH (timescaledb.continuous) AS
|
|
SELECT
|
|
time_bucket('30 minutes', time) AS time,
|
|
exchange,
|
|
symbol,
|
|
'30m'::text AS interval,
|
|
FIRST(open, time) AS open,
|
|
MAX(high) AS high,
|
|
MIN(low) AS low,
|
|
LAST(close, time) AS close,
|
|
SUM(volume) AS volume,
|
|
SUM(quote_volume) AS quote_volume,
|
|
SUM(taker_buy_base_vol) AS taker_buy_base_vol,
|
|
SUM(taker_buy_quote_vol) AS taker_buy_quote_vol,
|
|
SUM(trade_count)::integer AS trade_count
|
|
FROM klines_15m
|
|
GROUP BY time_bucket('30 minutes', klines_15m.time), exchange, symbol
|
|
WITH NO DATA;
|
|
|
|
-- 【模式 A 用户】取消下面注释以启用定时调度刷新
|
|
-- SELECT add_continuous_aggregate_policy('klines_30m',
|
|
-- start_offset => INTERVAL '3 days',
|
|
-- end_offset => INTERVAL '30 minutes',
|
|
-- schedule_interval => INTERVAL '30 minutes',
|
|
-- if_not_exists => TRUE
|
|
-- );
|
|
|
|
-- ============================================================
|
|
-- 1h K 线（从 30m 聚合，分层链）
|
|
-- ============================================================
|
|
CREATE MATERIALIZED VIEW IF NOT EXISTS klines_1h
|
|
WITH (timescaledb.continuous) AS
|
|
SELECT
|
|
time_bucket('1 hour', time) AS time,
|
|
exchange,
|
|
symbol,
|
|
'1h'::text AS interval,
|
|
FIRST(open, time) AS open,
|
|
MAX(high) AS high,
|
|
MIN(low) AS low,
|
|
LAST(close, time) AS close,
|
|
SUM(volume) AS volume,
|
|
SUM(quote_volume) AS quote_volume,
|
|
SUM(taker_buy_base_vol) AS taker_buy_base_vol,
|
|
SUM(taker_buy_quote_vol) AS taker_buy_quote_vol,
|
|
SUM(trade_count)::integer AS trade_count
|
|
FROM klines_30m
|
|
GROUP BY time_bucket('1 hour', klines_30m.time), exchange, symbol
|
|
WITH NO DATA;
|
|
|
|
-- 【模式 A 用户】取消下面注释以启用定时调度刷新
|
|
-- SELECT add_continuous_aggregate_policy('klines_1h',
|
|
-- start_offset => INTERVAL '7 days',
|
|
-- end_offset => INTERVAL '1 hour',
|
|
-- schedule_interval => INTERVAL '1 hour',
|
|
-- if_not_exists => TRUE
|
|
-- );
|
|
|
|
-- ============================================================
|
|
-- 2h K 线（从 1h 聚合，分层链）
|
|
-- ============================================================
|
|
CREATE MATERIALIZED VIEW IF NOT EXISTS klines_2h
|
|
WITH (timescaledb.continuous) AS
|
|
SELECT
|
|
time_bucket('2 hours', time) AS time,
|
|
exchange,
|
|
symbol,
|
|
'2h'::text AS interval,
|
|
FIRST(open, time) AS open,
|
|
MAX(high) AS high,
|
|
MIN(low) AS low,
|
|
LAST(close, time) AS close,
|
|
SUM(volume) AS volume,
|
|
SUM(quote_volume) AS quote_volume,
|
|
SUM(taker_buy_base_vol) AS taker_buy_base_vol,
|
|
SUM(taker_buy_quote_vol) AS taker_buy_quote_vol,
|
|
SUM(trade_count)::integer AS trade_count
|
|
FROM klines_1h
|
|
GROUP BY time_bucket('2 hours', klines_1h.time), exchange, symbol
|
|
WITH NO DATA;
|
|
|
|
-- 【模式 A 用户】取消下面注释以启用定时调度刷新
|
|
-- SELECT add_continuous_aggregate_policy('klines_2h',
|
|
-- start_offset => INTERVAL '10 days',
|
|
-- end_offset => INTERVAL '2 hours',
|
|
-- schedule_interval => INTERVAL '2 hours',
|
|
-- if_not_exists => TRUE
|
|
-- );
|
|
|
|
-- ============================================================
|
|
-- 4h K 线（从 1h 聚合，分层链）
|
|
-- ============================================================
|
|
CREATE MATERIALIZED VIEW IF NOT EXISTS klines_4h
|
|
WITH (timescaledb.continuous) AS
|
|
SELECT
|
|
time_bucket('4 hours', time) AS time,
|
|
exchange,
|
|
symbol,
|
|
'4h'::text AS interval,
|
|
FIRST(open, time) AS open,
|
|
MAX(high) AS high,
|
|
MIN(low) AS low,
|
|
LAST(close, time) AS close,
|
|
SUM(volume) AS volume,
|
|
SUM(quote_volume) AS quote_volume,
|
|
SUM(taker_buy_base_vol) AS taker_buy_base_vol,
|
|
SUM(taker_buy_quote_vol) AS taker_buy_quote_vol,
|
|
SUM(trade_count)::integer AS trade_count
|
|
FROM klines_1h
|
|
GROUP BY time_bucket('4 hours', klines_1h.time), exchange, symbol
|
|
WITH NO DATA;
|
|
|
|
-- 【模式 A 用户】取消下面注释以启用定时调度刷新
|
|
-- SELECT add_continuous_aggregate_policy('klines_4h',
|
|
-- start_offset => INTERVAL '14 days',
|
|
-- end_offset => INTERVAL '4 hours',
|
|
-- schedule_interval => INTERVAL '4 hours',
|
|
-- if_not_exists => TRUE
|
|
-- );
|
|
|
|
-- ============================================================
|
|
-- 6h K 线（从 1h 聚合，分层链）
|
|
-- ============================================================
|
|
CREATE MATERIALIZED VIEW IF NOT EXISTS klines_6h
|
|
WITH (timescaledb.continuous) AS
|
|
SELECT
|
|
time_bucket('6 hours', time) AS time,
|
|
exchange,
|
|
symbol,
|
|
'6h'::text AS interval,
|
|
FIRST(open, time) AS open,
|
|
MAX(high) AS high,
|
|
MIN(low) AS low,
|
|
LAST(close, time) AS close,
|
|
SUM(volume) AS volume,
|
|
SUM(quote_volume) AS quote_volume,
|
|
SUM(taker_buy_base_vol) AS taker_buy_base_vol,
|
|
SUM(taker_buy_quote_vol) AS taker_buy_quote_vol,
|
|
SUM(trade_count)::integer AS trade_count
|
|
FROM klines_1h
|
|
GROUP BY time_bucket('6 hours', klines_1h.time), exchange, symbol
|
|
WITH NO DATA;
|
|
|
|
-- 【模式 A 用户】取消下面注释以启用定时调度刷新
|
|
-- SELECT add_continuous_aggregate_policy('klines_6h',
|
|
-- start_offset => INTERVAL '20 days',
|
|
-- end_offset => INTERVAL '6 hours',
|
|
-- schedule_interval => INTERVAL '6 hours',
|
|
-- if_not_exists => TRUE
|
|
-- );
|
|
|
|
-- ============================================================
|
|
-- 8h K 线（从 4h 聚合，分层链）
|
|
-- ============================================================
|
|
CREATE MATERIALIZED VIEW IF NOT EXISTS klines_8h
|
|
WITH (timescaledb.continuous) AS
|
|
SELECT
|
|
time_bucket('8 hours', time) AS time,
|
|
exchange,
|
|
symbol,
|
|
'8h'::text AS interval,
|
|
FIRST(open, time) AS open,
|
|
MAX(high) AS high,
|
|
MIN(low) AS low,
|
|
LAST(close, time) AS close,
|
|
SUM(volume) AS volume,
|
|
SUM(quote_volume) AS quote_volume,
|
|
SUM(taker_buy_base_vol) AS taker_buy_base_vol,
|
|
SUM(taker_buy_quote_vol) AS taker_buy_quote_vol,
|
|
SUM(trade_count)::integer AS trade_count
|
|
FROM klines_4h
|
|
GROUP BY time_bucket('8 hours', klines_4h.time), exchange, symbol
|
|
WITH NO DATA;
|
|
|
|
-- 【模式 A 用户】取消下面注释以启用定时调度刷新
|
|
-- SELECT add_continuous_aggregate_policy('klines_8h',
|
|
-- start_offset => INTERVAL '30 days',
|
|
-- end_offset => INTERVAL '8 hours',
|
|
-- schedule_interval => INTERVAL '8 hours',
|
|
-- if_not_exists => TRUE
|
|
-- );
|
|
|
|
-- ============================================================
|
|
-- 1d K 线（从 4h 聚合，分层链）
|
|
-- ============================================================
|
|
CREATE MATERIALIZED VIEW IF NOT EXISTS klines_1d
|
|
WITH (timescaledb.continuous) AS
|
|
SELECT
|
|
time_bucket('1 day', time) AS time,
|
|
exchange,
|
|
symbol,
|
|
'1d'::text AS interval,
|
|
FIRST(open, time) AS open,
|
|
MAX(high) AS high,
|
|
MIN(low) AS low,
|
|
LAST(close, time) AS close,
|
|
SUM(volume) AS volume,
|
|
SUM(quote_volume) AS quote_volume,
|
|
SUM(taker_buy_base_vol) AS taker_buy_base_vol,
|
|
SUM(taker_buy_quote_vol) AS taker_buy_quote_vol,
|
|
SUM(trade_count)::integer AS trade_count
|
|
FROM klines_4h
|
|
GROUP BY time_bucket('1 day', klines_4h.time), exchange, symbol
|
|
WITH NO DATA;
|
|
|
|
-- 【模式 A 用户】取消下面注释以启用定时调度刷新
|
|
-- SELECT add_continuous_aggregate_policy('klines_1d',
|
|
-- start_offset => INTERVAL '30 days',
|
|
-- end_offset => INTERVAL '1 day',
|
|
-- schedule_interval => INTERVAL '1 day',
|
|
-- if_not_exists => TRUE
|
|
-- );
|
|
|
|
-- ============================================================
|
|
-- 1w K 线（从 1d 聚合，分层链）
|
|
-- ============================================================
|
|
CREATE MATERIALIZED VIEW IF NOT EXISTS klines_1w
|
|
WITH (timescaledb.continuous) AS
|
|
SELECT
|
|
time_bucket('1 week', time) AS time,
|
|
exchange,
|
|
symbol,
|
|
'1w'::text AS interval,
|
|
FIRST(open, time) AS open,
|
|
MAX(high) AS high,
|
|
MIN(low) AS low,
|
|
LAST(close, time) AS close,
|
|
SUM(volume) AS volume,
|
|
SUM(quote_volume) AS quote_volume,
|
|
SUM(taker_buy_base_vol) AS taker_buy_base_vol,
|
|
SUM(taker_buy_quote_vol) AS taker_buy_quote_vol,
|
|
SUM(trade_count)::integer AS trade_count
|
|
FROM klines_1d
|
|
GROUP BY time_bucket('1 week', klines_1d.time), exchange, symbol
|
|
WITH NO DATA;
|
|
|
|
-- 【模式 A 用户】取消下面注释以启用定时调度刷新
|
|
-- SELECT add_continuous_aggregate_policy('klines_1w',
|
|
-- start_offset => INTERVAL '90 days',
|
|
-- end_offset => INTERVAL '1 day',
|
|
-- schedule_interval => INTERVAL '1 day',
|
|
-- if_not_exists => TRUE
|
|
-- );
|
|
|
|
-- ============================================================
|
|
-- 1mon K 线（从 1d 聚合，分层链）
|
|
-- ============================================================
|
|
CREATE MATERIALIZED VIEW IF NOT EXISTS klines_1mon
|
|
WITH (timescaledb.continuous) AS
|
|
SELECT
|
|
time_bucket('1 month', time) AS time,
|
|
exchange,
|
|
symbol,
|
|
'1mon'::text AS interval,
|
|
FIRST(open, time) AS open,
|
|
MAX(high) AS high,
|
|
MIN(low) AS low,
|
|
LAST(close, time) AS close,
|
|
SUM(volume) AS volume,
|
|
SUM(quote_volume) AS quote_volume,
|
|
SUM(taker_buy_base_vol) AS taker_buy_base_vol,
|
|
SUM(taker_buy_quote_vol) AS taker_buy_quote_vol,
|
|
SUM(trade_count)::integer AS trade_count
|
|
FROM klines_1d
|
|
GROUP BY time_bucket('1 month', klines_1d.time), exchange, symbol
|
|
WITH NO DATA;
|
|
|
|
-- 【模式 A 用户】取消下面注释以启用定时调度刷新
|
|
-- SELECT add_continuous_aggregate_policy('klines_1mon',
|
|
-- start_offset => INTERVAL '365 days',
|
|
-- end_offset => INTERVAL '1 day',
|
|
-- schedule_interval => INTERVAL '1 day',
|
|
-- if_not_exists => TRUE
|
|
-- );
|
|
|
|
-- ============================================================
|
|
-- 推荐索引：加速按 symbol + time 的查询
|
|
-- ============================================================
|
|
CREATE INDEX IF NOT EXISTS idx_klines_3m_symbol_time ON klines_3m (exchange, symbol, time DESC);
|
|
CREATE INDEX IF NOT EXISTS idx_klines_5m_symbol_time ON klines_5m (exchange, symbol, time DESC);
|
|
CREATE INDEX IF NOT EXISTS idx_klines_15m_symbol_time ON klines_15m (exchange, symbol, time DESC);
|
|
CREATE INDEX IF NOT EXISTS idx_klines_30m_symbol_time ON klines_30m (exchange, symbol, time DESC);
|
|
CREATE INDEX IF NOT EXISTS idx_klines_1h_symbol_time ON klines_1h (exchange, symbol, time DESC);
|
|
CREATE INDEX IF NOT EXISTS idx_klines_2h_symbol_time ON klines_2h (exchange, symbol, time DESC);
|
|
CREATE INDEX IF NOT EXISTS idx_klines_4h_symbol_time ON klines_4h (exchange, symbol, time DESC);
|
|
CREATE INDEX IF NOT EXISTS idx_klines_6h_symbol_time ON klines_6h (exchange, symbol, time DESC);
|
|
CREATE INDEX IF NOT EXISTS idx_klines_8h_symbol_time ON klines_8h (exchange, symbol, time DESC);
|
|
CREATE INDEX IF NOT EXISTS idx_klines_1d_symbol_time ON klines_1d (exchange, symbol, time DESC);
|
|
CREATE INDEX IF NOT EXISTS idx_klines_1w_symbol_time ON klines_1w (exchange, symbol, time DESC);
|
|
CREATE INDEX IF NOT EXISTS idx_klines_1mon_symbol_time ON klines_1mon (exchange, symbol, time DESC);
|
|
|
|
-- ============================================================
|
|
-- 截面查询索引：加速同一时间点多品种回测查询
|
|
-- 查询模式：WHERE exchange='binance' AND time='2024-01-01' AND symbol IN (…)
|
|
-- 回测中跨品种截面查询最常见于日线和周线，因此只在这两层建额外索引。
|
|
-- 如需其他周期（如 1h、4h）的截面查询，按同样模式扩展。
|
|
-- ============================================================
|
|
CREATE INDEX IF NOT EXISTS idx_klines_1d_exchange_time_symbol ON klines_1d (exchange, time DESC, symbol);
|
|
CREATE INDEX IF NOT EXISTS idx_klines_1w_exchange_time_symbol ON klines_1w (exchange, time DESC, symbol);
|
|
|
|
-- ============================================================
|
|
-- 首次创建后手动刷新所有视图（填充历史数据）
|
|
-- 取消注释以下行执行：
|
|
-- ============================================================
|
|
-- CALL refresh_continuous_aggregate('klines_3m', NULL, NULL);
|
|
-- CALL refresh_continuous_aggregate('klines_5m', NULL, NULL);
|
|
-- CALL refresh_continuous_aggregate('klines_15m', NULL, NULL);
|
|
-- CALL refresh_continuous_aggregate('klines_30m', NULL, NULL);
|
|
-- CALL refresh_continuous_aggregate('klines_1h', NULL, NULL);
|
|
-- CALL refresh_continuous_aggregate('klines_2h', NULL, NULL);
|
|
-- CALL refresh_continuous_aggregate('klines_4h', NULL, NULL);
|
|
-- CALL refresh_continuous_aggregate('klines_6h', NULL, NULL);
|
|
-- CALL refresh_continuous_aggregate('klines_8h', NULL, NULL);
|
|
-- CALL refresh_continuous_aggregate('klines_1d', NULL, NULL);
|
|
-- CALL refresh_continuous_aggregate('klines_1w', NULL, NULL);
|
|
-- CALL refresh_continuous_aggregate('klines_1mon', NULL, NULL);
|