refactor: migrate API keys to config, extend Kline intervals, add DB extensions

Security:
- Move hardcoded Binance API key/secret from rest.ts to env.yaml (exchange config segment)
- Add ExchangeConfig validation in config/validators.ts
- Export typed exchange config from config/index.ts
- Update AGENTS/07-caveats.md to reflect the new policy

Kline intervals (add 3m / 2h / 6h / 8h / 1mon):
- TypeScript: update KlineInterval type, KLINE_INTERVAL_MS mapping, build_aggregates_sql refresh chain
- Python: sync KlineInterval Literal type, INTERVAL_TO_TABLE and INTERVAL_MS mappings, db_test table list
- SQL: add 5 continuous aggregate materialized views (klines_3m/2h/6h/8h/1mon) with indexes
- SQL: extend default kline_intervals in trading_pairs table
- SQL: add cross-sectional query indexes for klines_1d and klines_1w

DB:
- Enable pg_prewarm extension (backtest warmup)
- Enable pg_stat_statements extension (slow query monitoring)

Other:
- data/run/exchange.ts: graceful pgsql shutdown after backfill completes
- Config path: load from data/env.yaml (symlink) instead of project root
This commit is contained in:
Rekey
2026-06-14 18:45:01 +08:00
parent a9c45cce39
commit 9351dec226
15 changed files with 311 additions and 27 deletions
+2
View File
@@ -7,3 +7,5 @@ db/pgsql/
# Python # Python
__pycache__/ __pycache__/
.venv/ .venv/
.codegraph
+2 -2
View File
@@ -1,7 +1,7 @@
# 注意事项 # 注意事项
- **`data/exchanges/rest.ts` 包含硬编码的 Binance API Key**(第 105-106 行),不要提交到公开仓库 - **所有 API Key 统一在 `env.yaml` 中管理**,禁止在代码中硬编码。新增交易所时在 `exchange` 段添加对应子配置即可
- `env.yaml` 包含明文数据库密码且被 git 追踪,注意安全。 - `env.yaml` 包含明文密钥和数据库密码且被 git 追踪,注意安全,不要提交到公开仓库
- 未安装 Python 依赖(如 pydantic),`engine/` 目录有独立的 `.venv/` - 未安装 Python 依赖(如 pydantic),`engine/` 目录有独立的 `.venv/`
- `db/pgsql/``.gitignore` 中,这是 TimescaleDB 数据目录(Docker volume 映射)。 - `db/pgsql/``.gitignore` 中,这是 TimescaleDB 数据目录(Docker volume 映射)。
- `KLINE_INTERVAL_MS` 常量定义了两处:`data/exchanges/rest.ts``data/types/kline.ts` 的类型定义。新增周期需同步。 - `KLINE_INTERVAL_MS` 常量定义了两处:`data/exchanges/rest.ts``data/types/kline.ts` 的类型定义。新增周期需同步。
+26 -5
View File
@@ -11,7 +11,7 @@
// const ds = new DataSource({ ...pgsql }); // const ds = new DataSource({ ...pgsql });
// const redisClient = new Redis(redis.url); // const redisClient = new Redis(redis.url);
// //
// 配置文件位置:<project_root>/env.yaml // 配置文件位置:data/env.yaml(软链接 → 项目根目录 env.yaml)
// TypeScript / Python 模块共享同一份配置。 // TypeScript / Python 模块共享同一份配置。
// ============================================================ // ============================================================
@@ -39,12 +39,14 @@ function getProjectRoot(): string {
} }
/** /**
* 项目根目录读取 env.yaml 并解析为原始对象。 * 读取 data/env.yaml(软链接指向项目根目录 env.yaml并解析为原始对象。
* 文件不存在时抛出明确错误,不做静默降级。 * 文件不存在时抛出明确错误,不做静默降级。
*/ */
function loadYamlConfig(): unknown { function loadYamlConfig(): unknown {
const root = getProjectRoot(); const __filename = fileURLToPath(import.meta.url);
const yamlPath = resolve(root, "env.yaml"); const __dirname = dirname(__filename);
// config/index.ts → config/ → data/env.yaml
const yamlPath = resolve(__dirname, "../env.yaml");
let content: string; let content: string;
try { try {
@@ -52,7 +54,7 @@ function loadYamlConfig(): unknown {
} catch { } catch {
throw new Error( throw new Error(
`[config] 无法读取配置文件: ${yamlPath}\n` + `[config] 无法读取配置文件: ${yamlPath}\n` +
`请确保项目根目录存在 env.yaml(可参考 data/.env.example 的结构)`, `请确保 data/env.yaml 软链接指向项目根目录的 env.yaml`,
); );
} }
@@ -131,6 +133,19 @@ export const logging = {
pretty: rawConfig.logging.node_env === "development", pretty: rawConfig.logging.node_env === "development",
} as const; } as const;
/**
* 交易所 API 密钥配置
*
* 按交易所 ID 索引,目前仅 binance。
* 新增交易所时在 env.yaml 的 exchange 段添加对应子配置即可。
*/
export const exchange = {
binance: {
apiKey: rawConfig.exchange.binance.api_key,
apiSecret: rawConfig.exchange.binance.api_secret,
},
} as const;
// ============================================================ // ============================================================
// 4. 工具:运行时打印配置概要(不含敏感信息) // 4. 工具:运行时打印配置概要(不含敏感信息)
// ============================================================ // ============================================================
@@ -150,6 +165,12 @@ export function printConfigSummary(): void {
url: redis.url.replace(/\/\/.*@/, "//***@"), // 隐藏密码 url: redis.url.replace(/\/\/.*@/, "//***@"), // 隐藏密码
publishEnabled: redis.publishEnabled, publishEnabled: redis.publishEnabled,
}, },
exchange: {
binance: {
apiKey: exchange.binance.apiKey.slice(0, 6) + "***",
apiSecret: "***",
},
},
logging: { logging: {
level: logging.level, level: logging.level,
nodeEnv: logging.nodeEnv, nodeEnv: logging.nodeEnv,
+34
View File
@@ -15,6 +15,7 @@
export interface EnvConfig { export interface EnvConfig {
db: DbConfig; db: DbConfig;
redis: RedisConfig; redis: RedisConfig;
exchange: ExchangeConfig;
logging: LoggingConfig; logging: LoggingConfig;
} }
@@ -31,6 +32,18 @@ export interface RedisConfig {
publish_enabled: boolean; publish_enabled: boolean;
} }
/** 交易所 API 密钥配置(按交易所 ID 索引) */
export interface ExchangeConfig {
binance: ExchangeApiKeys;
// 未来扩展:okx、bybit 等
[exchangeId: string]: ExchangeApiKeys | undefined;
}
export interface ExchangeApiKeys {
api_key: string;
api_secret: string;
}
export interface LoggingConfig { export interface LoggingConfig {
level: "trace" | "debug" | "info" | "warn" | "error" | "fatal"; level: "trace" | "debug" | "info" | "warn" | "error" | "fatal";
node_env: "development" | "production" | "test"; node_env: "development" | "production" | "test";
@@ -77,6 +90,21 @@ export function validateConfig(raw: unknown): EnvConfig {
const redisUrl = assertString(redisObj["url"], "redis.url"); const redisUrl = assertString(redisObj["url"], "redis.url");
const redisPublishEnabled = assertBoolean(redisObj["publish_enabled"], "redis.publish_enabled"); const redisPublishEnabled = assertBoolean(redisObj["publish_enabled"], "redis.publish_enabled");
// --- exchange ---
const exchange = obj["exchange"];
if (typeof exchange !== "object" || exchange === null) {
throw new Error("[config] env.yaml 缺少 exchange 配置段");
}
const exObj = exchange as Record<string, unknown>;
const binance = exObj["binance"];
if (typeof binance !== "object" || binance === null) {
throw new Error("[config] env.yaml exchange 缺少 binance 配置");
}
const binanceObj = binance as Record<string, unknown>;
const binanceApiKey = assertString(binanceObj["api_key"], "exchange.binance.api_key");
const binanceApiSecret = assertString(binanceObj["api_secret"], "exchange.binance.api_secret");
// --- logging --- // --- logging ---
const logging = obj["logging"]; const logging = obj["logging"];
if (typeof logging !== "object" || logging === null) { if (typeof logging !== "object" || logging === null) {
@@ -99,6 +127,12 @@ export function validateConfig(raw: unknown): EnvConfig {
url: redisUrl, url: redisUrl,
publish_enabled: redisPublishEnabled, publish_enabled: redisPublishEnabled,
}, },
exchange: {
binance: {
api_key: binanceApiKey,
api_secret: binanceApiSecret,
},
},
logging: { logging: {
level: logLevel, level: logLevel,
node_env: nodeEnv, node_env: nodeEnv,
+7 -1
View File
@@ -13,12 +13,18 @@
-- - klines 基表由 02-init-tables.sql 创建为 TimescaleDB hypertable -- - klines 基表由 02-init-tables.sql 创建为 TimescaleDB hypertable
-- - 连续聚合视图由 03-continuous-aggregates.sql 创建 -- - 连续聚合视图由 03-continuous-aggregates.sql 创建
-- - TypeORM 的 synchronize:true 与 SQL 脚本互为 fallback(开发/生产双路径) -- - TypeORM 的 synchronize:true 与 SQL 脚本互为 fallback(开发/生产双路径)
-- - 本脚本为 init-db 链的第一环,负责扩展启用 -- - 本脚本为 init-db 链的第一环,负责扩展启用TimescaleDB / pg_prewarm / pg_stat_statements
-- ============================================================ -- ============================================================
-- 启用 TimescaleDB 扩展(必须最先执行) -- 启用 TimescaleDB 扩展(必须最先执行)
CREATE EXTENSION IF NOT EXISTS timescaledb CASCADE; CREATE EXTENSION IF NOT EXISTS timescaledb CASCADE;
-- 启用 pg_prewarm 扩展(回测预热,减少首轮查询延迟)
CREATE EXTENSION IF NOT EXISTS pg_prewarm;
-- 启用 pg_stat_statements 扩展(慢查询监控,零成本)
CREATE EXTENSION IF NOT EXISTS pg_stat_statements;
-- 验证扩展已启用 -- 验证扩展已启用
DO $$ DO $$
BEGIN BEGIN
+1 -1
View File
@@ -100,7 +100,7 @@ CREATE TABLE IF NOT EXISTS trading_pairs (
kline_interval VARCHAR(100) NOT NULL DEFAULT '1m', kline_interval VARCHAR(100) NOT NULL DEFAULT '1m',
-- K 线合成周期列表(逗号分隔,如 "1m,5m,15m,1h,4h,1d" -- K 线合成周期列表(逗号分隔,如 "1m,5m,15m,1h,4h,1d"
kline_intervals VARCHAR(100) NOT NULL DEFAULT '1m,5m,15m,1h,4h,1d', kline_intervals VARCHAR(100) NOT NULL DEFAULT '1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,1d,1w,1mon',
-- 历史 K 线最后补全时间(UTC)。默认 Unix epoch 起始, -- 历史 K 线最后补全时间(UTC)。默认 Unix epoch 起始,
-- 新交易对从 epoch 起始时间开始全量补拉。 -- 新交易对从 epoch 起始时间开始全量补拉。
+175 -1
View File
@@ -2,7 +2,7 @@
-- 03-continuous-aggregates.sql — K 线分层连续聚合视图 -- 03-continuous-aggregates.sql — K 线分层连续聚合视图
-- ============================================================ -- ============================================================
-- 从 klines(1m)基表创建分层连续聚合物化视图链: -- 从 klines(1m)基表创建分层连续聚合物化视图链:
-- 1m → 5m → 15m → 30m → 1h → 4h → 1d → 1w -- 1m → 3m → 5m → 15m → 30m → 1h → 2h → 4h → 6h → 8h → 1d → 1w → 1mon
-- --
-- 执行前提: -- 执行前提:
-- 1. klines hypertable 已创建(由 02-init-tables.sql 创建) -- 1. klines hypertable 已创建(由 02-init-tables.sql 创建)
@@ -36,6 +36,37 @@
-- 3. 接入实时数据(模式 A 启用 policy / 模式 B 应用层触发) -- 3. 接入实时数据(模式 A 启用 policy / 模式 B 应用层触发)
-- ============================================================ -- ============================================================
-- ============================================================
-- 3m K 线(从 1m 基表聚合)
-- ============================================================
CREATE MATERIALIZED VIEW IF NOT EXISTS klines_3m
WITH (timescaledb.continuous) AS
SELECT
time_bucket('3 minutes', time) AS time,
exchange,
symbol,
'3m'::text AS interval,
FIRST(open, time) AS open,
MAX(high) AS high,
MIN(low) AS low,
LAST(close, time) AS close,
SUM(volume) AS volume,
SUM(quote_volume) AS quote_volume,
SUM(taker_buy_base_vol) AS taker_buy_base_vol,
SUM(taker_buy_quote_vol) AS taker_buy_quote_vol,
SUM(trade_count)::integer AS trade_count
FROM klines
GROUP BY time_bucket('3 minutes', klines.time), exchange, symbol
WITH NO DATA;
-- 【模式 A 用户】取消下面注释以启用定时调度刷新
-- SELECT add_continuous_aggregate_policy('klines_3m',
-- start_offset => INTERVAL '1 day',
-- end_offset => INTERVAL '3 minutes',
-- schedule_interval => INTERVAL '3 minutes',
-- if_not_exists => TRUE
-- );
-- ============================================================ -- ============================================================
-- 5m K 线(从 1m 基表聚合) -- 5m K 线(从 1m 基表聚合)
-- ============================================================ -- ============================================================
@@ -160,6 +191,37 @@ WITH NO DATA;
-- if_not_exists => TRUE -- if_not_exists => TRUE
-- ); -- );
-- ============================================================
-- 2h K 线(从 1h 聚合,分层链)
-- ============================================================
CREATE MATERIALIZED VIEW IF NOT EXISTS klines_2h
WITH (timescaledb.continuous) AS
SELECT
time_bucket('2 hours', time) AS time,
exchange,
symbol,
'2h'::text AS interval,
FIRST(open, time) AS open,
MAX(high) AS high,
MIN(low) AS low,
LAST(close, time) AS close,
SUM(volume) AS volume,
SUM(quote_volume) AS quote_volume,
SUM(taker_buy_base_vol) AS taker_buy_base_vol,
SUM(taker_buy_quote_vol) AS taker_buy_quote_vol,
SUM(trade_count)::integer AS trade_count
FROM klines_1h
GROUP BY time_bucket('2 hours', klines_1h.time), exchange, symbol
WITH NO DATA;
-- 【模式 A 用户】取消下面注释以启用定时调度刷新
-- SELECT add_continuous_aggregate_policy('klines_2h',
-- start_offset => INTERVAL '10 days',
-- end_offset => INTERVAL '2 hours',
-- schedule_interval => INTERVAL '2 hours',
-- if_not_exists => TRUE
-- );
-- ============================================================ -- ============================================================
-- 4h K 线(从 1h 聚合,分层链) -- 4h K 线(从 1h 聚合,分层链)
-- ============================================================ -- ============================================================
@@ -191,6 +253,68 @@ WITH NO DATA;
-- if_not_exists => TRUE -- if_not_exists => TRUE
-- ); -- );
-- ============================================================
-- 6h K 线(从 1h 聚合,分层链)
-- ============================================================
CREATE MATERIALIZED VIEW IF NOT EXISTS klines_6h
WITH (timescaledb.continuous) AS
SELECT
time_bucket('6 hours', time) AS time,
exchange,
symbol,
'6h'::text AS interval,
FIRST(open, time) AS open,
MAX(high) AS high,
MIN(low) AS low,
LAST(close, time) AS close,
SUM(volume) AS volume,
SUM(quote_volume) AS quote_volume,
SUM(taker_buy_base_vol) AS taker_buy_base_vol,
SUM(taker_buy_quote_vol) AS taker_buy_quote_vol,
SUM(trade_count)::integer AS trade_count
FROM klines_1h
GROUP BY time_bucket('6 hours', klines_1h.time), exchange, symbol
WITH NO DATA;
-- 【模式 A 用户】取消下面注释以启用定时调度刷新
-- SELECT add_continuous_aggregate_policy('klines_6h',
-- start_offset => INTERVAL '20 days',
-- end_offset => INTERVAL '6 hours',
-- schedule_interval => INTERVAL '6 hours',
-- if_not_exists => TRUE
-- );
-- ============================================================
-- 8h K 线(从 4h 聚合,分层链)
-- ============================================================
CREATE MATERIALIZED VIEW IF NOT EXISTS klines_8h
WITH (timescaledb.continuous) AS
SELECT
time_bucket('8 hours', time) AS time,
exchange,
symbol,
'8h'::text AS interval,
FIRST(open, time) AS open,
MAX(high) AS high,
MIN(low) AS low,
LAST(close, time) AS close,
SUM(volume) AS volume,
SUM(quote_volume) AS quote_volume,
SUM(taker_buy_base_vol) AS taker_buy_base_vol,
SUM(taker_buy_quote_vol) AS taker_buy_quote_vol,
SUM(trade_count)::integer AS trade_count
FROM klines_4h
GROUP BY time_bucket('8 hours', klines_4h.time), exchange, symbol
WITH NO DATA;
-- 【模式 A 用户】取消下面注释以启用定时调度刷新
-- SELECT add_continuous_aggregate_policy('klines_8h',
-- start_offset => INTERVAL '30 days',
-- end_offset => INTERVAL '8 hours',
-- schedule_interval => INTERVAL '8 hours',
-- if_not_exists => TRUE
-- );
-- ============================================================ -- ============================================================
-- 1d K 线(从 4h 聚合,分层链) -- 1d K 线(从 4h 聚合,分层链)
-- ============================================================ -- ============================================================
@@ -253,25 +377,75 @@ WITH NO DATA;
-- if_not_exists => TRUE -- if_not_exists => TRUE
-- ); -- );
-- ============================================================
-- 1mon K 线(从 1d 聚合,分层链)
-- ============================================================
CREATE MATERIALIZED VIEW IF NOT EXISTS klines_1mon
WITH (timescaledb.continuous) AS
SELECT
time_bucket('1 month', time) AS time,
exchange,
symbol,
'1mon'::text AS interval,
FIRST(open, time) AS open,
MAX(high) AS high,
MIN(low) AS low,
LAST(close, time) AS close,
SUM(volume) AS volume,
SUM(quote_volume) AS quote_volume,
SUM(taker_buy_base_vol) AS taker_buy_base_vol,
SUM(taker_buy_quote_vol) AS taker_buy_quote_vol,
SUM(trade_count)::integer AS trade_count
FROM klines_1d
GROUP BY time_bucket('1 month', klines_1d.time), exchange, symbol
WITH NO DATA;
-- 【模式 A 用户】取消下面注释以启用定时调度刷新
-- SELECT add_continuous_aggregate_policy('klines_1mon',
-- start_offset => INTERVAL '365 days',
-- end_offset => INTERVAL '1 day',
-- schedule_interval => INTERVAL '1 day',
-- if_not_exists => TRUE
-- );
-- ============================================================ -- ============================================================
-- 推荐索引:加速按 symbol + time 的查询 -- 推荐索引:加速按 symbol + time 的查询
-- ============================================================ -- ============================================================
CREATE INDEX IF NOT EXISTS idx_klines_3m_symbol_time ON klines_3m (exchange, symbol, time DESC);
CREATE INDEX IF NOT EXISTS idx_klines_5m_symbol_time ON klines_5m (exchange, symbol, time DESC); CREATE INDEX IF NOT EXISTS idx_klines_5m_symbol_time ON klines_5m (exchange, symbol, time DESC);
CREATE INDEX IF NOT EXISTS idx_klines_15m_symbol_time ON klines_15m (exchange, symbol, time DESC); CREATE INDEX IF NOT EXISTS idx_klines_15m_symbol_time ON klines_15m (exchange, symbol, time DESC);
CREATE INDEX IF NOT EXISTS idx_klines_30m_symbol_time ON klines_30m (exchange, symbol, time DESC); CREATE INDEX IF NOT EXISTS idx_klines_30m_symbol_time ON klines_30m (exchange, symbol, time DESC);
CREATE INDEX IF NOT EXISTS idx_klines_1h_symbol_time ON klines_1h (exchange, symbol, time DESC); CREATE INDEX IF NOT EXISTS idx_klines_1h_symbol_time ON klines_1h (exchange, symbol, time DESC);
CREATE INDEX IF NOT EXISTS idx_klines_2h_symbol_time ON klines_2h (exchange, symbol, time DESC);
CREATE INDEX IF NOT EXISTS idx_klines_4h_symbol_time ON klines_4h (exchange, symbol, time DESC); CREATE INDEX IF NOT EXISTS idx_klines_4h_symbol_time ON klines_4h (exchange, symbol, time DESC);
CREATE INDEX IF NOT EXISTS idx_klines_6h_symbol_time ON klines_6h (exchange, symbol, time DESC);
CREATE INDEX IF NOT EXISTS idx_klines_8h_symbol_time ON klines_8h (exchange, symbol, time DESC);
CREATE INDEX IF NOT EXISTS idx_klines_1d_symbol_time ON klines_1d (exchange, symbol, time DESC); CREATE INDEX IF NOT EXISTS idx_klines_1d_symbol_time ON klines_1d (exchange, symbol, time DESC);
CREATE INDEX IF NOT EXISTS idx_klines_1w_symbol_time ON klines_1w (exchange, symbol, time DESC); CREATE INDEX IF NOT EXISTS idx_klines_1w_symbol_time ON klines_1w (exchange, symbol, time DESC);
CREATE INDEX IF NOT EXISTS idx_klines_1mon_symbol_time ON klines_1mon (exchange, symbol, time DESC);
-- ============================================================
-- 截面查询索引:加速同一时间点多品种回测查询
-- 查询模式:WHERE exchange='binance' AND time='2024-01-01' AND symbol IN (…)
-- 回测中跨品种截面查询最常见于日线和周线,因此只在这两层建额外索引。
-- 如需其他周期(如 1h、4h)的截面查询,按同样模式扩展。
-- ============================================================
CREATE INDEX IF NOT EXISTS idx_klines_1d_exchange_time_symbol ON klines_1d (exchange, time DESC, symbol);
CREATE INDEX IF NOT EXISTS idx_klines_1w_exchange_time_symbol ON klines_1w (exchange, time DESC, symbol);
-- ============================================================ -- ============================================================
-- 首次创建后手动刷新所有视图(填充历史数据) -- 首次创建后手动刷新所有视图(填充历史数据)
-- 取消注释以下行执行: -- 取消注释以下行执行:
-- ============================================================ -- ============================================================
-- CALL refresh_continuous_aggregate('klines_3m', NULL, NULL);
-- CALL refresh_continuous_aggregate('klines_5m', NULL, NULL); -- CALL refresh_continuous_aggregate('klines_5m', NULL, NULL);
-- CALL refresh_continuous_aggregate('klines_15m', NULL, NULL); -- CALL refresh_continuous_aggregate('klines_15m', NULL, NULL);
-- CALL refresh_continuous_aggregate('klines_30m', NULL, NULL); -- CALL refresh_continuous_aggregate('klines_30m', NULL, NULL);
-- CALL refresh_continuous_aggregate('klines_1h', NULL, NULL); -- CALL refresh_continuous_aggregate('klines_1h', NULL, NULL);
-- CALL refresh_continuous_aggregate('klines_2h', NULL, NULL);
-- CALL refresh_continuous_aggregate('klines_4h', NULL, NULL); -- CALL refresh_continuous_aggregate('klines_4h', NULL, NULL);
-- CALL refresh_continuous_aggregate('klines_6h', NULL, NULL);
-- CALL refresh_continuous_aggregate('klines_8h', NULL, NULL);
-- CALL refresh_continuous_aggregate('klines_1d', NULL, NULL); -- CALL refresh_continuous_aggregate('klines_1d', NULL, NULL);
-- CALL refresh_continuous_aggregate('klines_1w', NULL, NULL); -- CALL refresh_continuous_aggregate('klines_1w', NULL, NULL);
-- CALL refresh_continuous_aggregate('klines_1mon', NULL, NULL);
+8 -2
View File
@@ -1,19 +1,25 @@
import { MainClient, type Kline as BinanceRestKline } from "binance"; import { MainClient, type Kline as BinanceRestKline } from "binance";
import { logger } from "../utils/logger"; import { logger } from "../utils/logger";
import { exchange } from "../config";
import { BaseRestClient } from './base_rest'; import { BaseRestClient } from './base_rest';
import type { KlineInterval, Kline, MarketInfo } from '../types'; import type { KlineInterval, Kline, MarketInfo } from '../types';
/** K 线周期 → 毫秒数映射(用于时间桶计算) */ /** K 线周期 → 毫秒数映射(用于时间桶计算) */
export const KLINE_INTERVAL_MS: Record<KlineInterval, number> = { export const KLINE_INTERVAL_MS: Record<KlineInterval, number> = {
"1m": 60_000, "1m": 60_000,
"3m": 180_000,
"5m": 300_000, "5m": 300_000,
"15m": 900_000, "15m": 900_000,
"30m": 1_800_000, "30m": 1_800_000,
"1h": 3_600_000, "1h": 3_600_000,
"2h": 7_200_000,
"4h": 14_400_000, "4h": 14_400_000,
"6h": 21_600_000,
"8h": 28_800_000,
"1d": 86_400_000, "1d": 86_400_000,
"1w": 604_800_000, "1w": 604_800_000,
"1mon": 2_592_000_000,
}; };
// ============================================================ // ============================================================
@@ -102,8 +108,8 @@ async function fetchBinanceKlines(
limit = 500, limit = 500,
): Promise<Kline[]> { ): Promise<Kline[]> {
const client = new MainClient({ const client = new MainClient({
api_key: 'ONSJKIGRpDYLn6FdV17aAKfjclZ4I2LzamflhuMpsoRQA427lLKeyJlGtg2RZ7DH', api_key: exchange.binance.apiKey,
api_secret: '5Mfv4TgvDlRzCHbtl2nJL4mVHUvMm8pyjKiRjMoosBMxrhlqMw6CuQbg2qbS2Npd', api_secret: exchange.binance.apiSecret,
}, { }, {
timeout: 3000, timeout: 3000,
}); });
+21 -11
View File
@@ -3,8 +3,8 @@
// ============================================================ // ============================================================
// 用途: // 用途:
// 按月份粒度逐月刷新 klines 分层连续聚合物化视图链: // 按月份粒度逐月刷新 klines 分层连续聚合物化视图链:
// 5m → 15m → 30m → 1h → 4h → 1d → 1w // 基表 1m → 3m / 5m → 15m → 30m → 1h → 2h / 4h / 6h → 8h / 1d → 1w / 1mon
// 每层依赖下一层的数据,因此严格按从低到高顺序刷新。 // 严格按依赖顺序从低到高刷新。
// //
// 使用方式: // 使用方式:
// # 仅生成 SQL 不执行(dry-run,默认) // # 仅生成 SQL 不执行(dry-run,默认)
@@ -36,18 +36,26 @@ import { logger } from "../utils/logger";
/** /**
* 分层聚合视图链(按依赖顺序:低层级 → 高层级) * 分层聚合视图链(按依赖顺序:低层级 → 高层级)
* *
* 刷新顺序至关重要 * 实际依赖关系(来自于 03-continuous-aggregates.sql
* klines_5m 源数据来自 klines1m 基表) * 基表 1m
* klines_15m 源数据来自 klines_5m * ├── 3m (直接聚合 1m
* klines_30m 源数据来自 klines_15m * └── 5m (直接聚合 1m)→ 15m → 30m → 1h ──→ 2h
* klines_1h 源数据来自 klines_30m * ├── 4h ──→ 8h
* klines_4h 源数据来自 klines_1h * │ └── 1d ──→ 1w
* klines_1d 源数据来自 klines_4h * │ └── 1mon
* klines_1w 源数据来自 klines_1d * └── 6h
* *
* 必须严格按此顺序刷新,否则高层级聚合会缺少数据。 * 刷新时必须严格按此顺序,因为:
* - klines_3m / klines_5m 无聚合依赖,可最先刷新
* - klines_15m 依赖 klines_5m
* - klines_30m 依赖 klines_15m
* - klines_1h 依赖 klines_30m
* - klines_2h / klines_4h / klines_6h 依赖 klines_1h
* - klines_8h / klines_1d 依赖 klines_4h
* - klines_1w / klines_1mon 依赖 klines_1d
*/ */
const AGGREGATE_VIEWS = [ const AGGREGATE_VIEWS = [
"klines_3m",
"klines_5m", "klines_5m",
"klines_15m", "klines_15m",
"klines_30m", "klines_30m",
@@ -55,8 +63,10 @@ const AGGREGATE_VIEWS = [
"klines_2h", "klines_2h",
"klines_4h", "klines_4h",
"klines_6h", "klines_6h",
"klines_8h",
"klines_1d", "klines_1d",
"klines_1w", "klines_1w",
"klines_1mon",
] as const; ] as const;
/** 默认起始年月 */ /** 默认起始年月 */
+9
View File
@@ -42,3 +42,12 @@ for (const pair of allPairs) {
console.error("拉取失败:", err); console.error("拉取失败:", err);
} }
} }
// 所有币种回补完成以后等待1秒关闭pgsql连接退出此进程
await new Promise((resolve) => setTimeout(resolve, 1000));
const { AppDataSource } = await import("../db/data-source");
if (AppDataSource.isInitialized) {
await AppDataSource.destroy();
console.log("pgsql 连接已关闭");
}
process.exit(0);
+6 -1
View File
@@ -1,10 +1,15 @@
/** K 线周期枚举 */ /** K 线周期枚举 */
export type KlineInterval = export type KlineInterval =
| "1m" | "1m"
| "3m"
| "5m" | "5m"
| "15m" | "15m"
| "30m" | "30m"
| "1h" | "1h"
| "2h"
| "4h" | "4h"
| "6h"
| "8h"
| "1d" | "1d"
| "1w"; | "1w"
| "1mon";
+1 -1
View File
@@ -14,7 +14,7 @@ from pydantic import BaseModel, Field, field_validator
# K 线周期类型 # K 线周期类型
# ============================================================ # ============================================================
KlineInterval = Literal["1m", "5m", "15m", "30m", "1h", "2h", "4h", "6h", "1d", "1w"] KlineInterval = Literal["1m", "3m", "5m", "15m", "30m", "1h", "2h", "4h", "6h", "8h", "1d", "1w", "1mon"]
# ============================================================ # ============================================================
+6
View File
@@ -32,6 +32,7 @@ from ..common.models import Kline, KlineInterval
# ── 周期 → 表名映射 ── # ── 周期 → 表名映射 ──
INTERVAL_TO_TABLE: dict[KlineInterval, str] = { INTERVAL_TO_TABLE: dict[KlineInterval, str] = {
"1m": "klines", "1m": "klines",
"3m": "klines_3m",
"5m": "klines_5m", "5m": "klines_5m",
"15m": "klines_15m", "15m": "klines_15m",
"30m": "klines_30m", "30m": "klines_30m",
@@ -39,13 +40,16 @@ INTERVAL_TO_TABLE: dict[KlineInterval, str] = {
"2h": "klines_2h", "2h": "klines_2h",
"4h": "klines_4h", "4h": "klines_4h",
"6h": "klines_6h", "6h": "klines_6h",
"8h": "klines_8h",
"1d": "klines_1d", "1d": "klines_1d",
"1w": "klines_1w", "1w": "klines_1w",
"1mon": "klines_1mon",
} }
# ── 周期毫秒数 ── # ── 周期毫秒数 ──
INTERVAL_MS: dict[KlineInterval, int] = { INTERVAL_MS: dict[KlineInterval, int] = {
"1m": 60_000, "1m": 60_000,
"3m": 180_000,
"5m": 300_000, "5m": 300_000,
"15m": 900_000, "15m": 900_000,
"30m": 1_800_000, "30m": 1_800_000,
@@ -53,8 +57,10 @@ INTERVAL_MS: dict[KlineInterval, int] = {
"2h": 7_200_000, "2h": 7_200_000,
"4h": 14_400_000, "4h": 14_400_000,
"6h": 21_600_000, "6h": 21_600_000,
"8h": 28_800_000,
"1d": 86_400_000, "1d": 86_400_000,
"1w": 604_800_000, "1w": 604_800_000,
"1mon": 2_592_000_000,
} }
DEFAULT_BATCH_SIZE = 5000 DEFAULT_BATCH_SIZE = 5000
+5
View File
@@ -16,13 +16,18 @@ from common.config import config as app_config
# ── 各周期对应的表/视图 ── # ── 各周期对应的表/视图 ──
INTERVAL_TABLES: dict[str, str] = { INTERVAL_TABLES: dict[str, str] = {
"1m": "klines", "1m": "klines",
"3m": "klines_3m",
"5m": "klines_5m", "5m": "klines_5m",
"15m": "klines_15m", "15m": "klines_15m",
"30m": "klines_30m", "30m": "klines_30m",
"1h": "klines_1h", "1h": "klines_1h",
"2h": "klines_2h",
"4h": "klines_4h", "4h": "klines_4h",
"6h": "klines_6h",
"8h": "klines_8h",
"1d": "klines_1d", "1d": "klines_1d",
"1w": "klines_1w", "1w": "klines_1w",
"1mon": "klines_1mon",
} }
LIMIT = 5 LIMIT = 5
+6
View File
@@ -22,6 +22,12 @@ redis:
# 是否启用 Pub/Sub 行情发布(开发时可关闭以节省资源) # 是否启用 Pub/Sub 行情发布(开发时可关闭以节省资源)
publish_enabled: true publish_enabled: true
# --- 交易所 API 密钥 ---
exchange:
binance:
api_key: "ONSJKIGRpDYLn6FdV17aAKfjclZ4I2LzamflhuMpsoRQA427lLKeyJlGtg2RZ7DH"
api_secret: "5Mfv4TgvDlRzCHbtl2nJL4mVHUvMm8pyjKiRjMoosBMxrhlqMw6CuQbg2qbS2Npd"
# --- 日志 --- # --- 日志 ---
logging: logging:
# 日志级别:trace / debug / info / warn / error / fatal # 日志级别:trace / debug / info / warn / error / fatal