rekey/trade
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor: migrate API keys to config, extend Kline intervals, add DB extensions

Browse Source 
Security:
- Move hardcoded Binance API key/secret from rest.ts to env.yaml (exchange config segment)
- Add ExchangeConfig validation in config/validators.ts
- Export typed exchange config from config/index.ts
- Update AGENTS/07-caveats.md to reflect the new policy

Kline intervals (add 3m / 2h / 6h / 8h / 1mon):
- TypeScript: update KlineInterval type, KLINE_INTERVAL_MS mapping, build_aggregates_sql refresh chain
- Python: sync KlineInterval Literal type, INTERVAL_TO_TABLE and INTERVAL_MS mappings, db_test table list
- SQL: add 5 continuous aggregate materialized views (klines_3m/2h/6h/8h/1mon) with indexes
- SQL: extend default kline_intervals in trading_pairs table
- SQL: add cross-sectional query indexes for klines_1d and klines_1w

DB:
- Enable pg_prewarm extension (backtest warmup)
- Enable pg_stat_statements extension (slow query monitoring)

Other:
- data/run/exchange.ts: graceful pgsql shutdown after backfill completes
- Config path: load from data/env.yaml (symlink) instead of project root
This commit is contained in:
Rekey
2026-06-14 18:45:01 +08:00
parent a9c45cce39
commit 9351dec226
15 changed files with 311 additions and 27 deletions
Download Patch File Download Diff File Expand all files Collapse all files
data/config/index.ts
+26 -5
View File
@@ -11,7 +11,7 @@
// const ds = new DataSource({ ...pgsql });
// const redisClient = new Redis(redis.url);
//
// 配置文件位置：<project_root>/env.yaml
// 配置文件位置：data/env.yaml（软链接 → 项目根目录 env.yaml）
// TypeScript / Python 模块共享同一份配置。
// ============================================================
@@ -39,12 +39,14 @@ function getProjectRoot(): string {
}
/**
* 从项目根目录读取 env.yaml 并解析为原始对象。
* 读取 data/env.yaml（软链接指向项目根目录 env.yaml）并解析为原始对象。
* 文件不存在时抛出明确错误，不做静默降级。
*/
function loadYamlConfig(): unknown {
const root = getProjectRoot();
const yamlPath = resolve(root, "env.yaml");
const __filename = fileURLToPath(import.meta.url);
const __dirname = dirname(__filename);
// config/index.ts → config/ → data/env.yaml
const yamlPath = resolve(__dirname, "../env.yaml");
let content: string;
try {
@@ -52,7 +54,7 @@ function loadYamlConfig(): unknown {
} catch {
throw new Error(
`[config] 无法读取配置文件: ${yamlPath}\n` +
`请确保项目根目录存在 env.yaml（可参考 data/.env.example 的结构）。`,
`请确保 data/env.yaml 软链接指向项目根目录的 env.yaml。`,
);
}
@@ -131,6 +133,19 @@ export const logging = {
pretty: rawConfig.logging.node_env === "development",
} as const;
/**
* 交易所 API 密钥配置
*
* 按交易所 ID 索引，目前仅 binance。
* 新增交易所时在 env.yaml 的 exchange 段添加对应子配置即可。
*/
export const exchange = {
binance: {
apiKey: rawConfig.exchange.binance.api_key,
apiSecret: rawConfig.exchange.binance.api_secret,
},
} as const;
// ============================================================
// 4. 工具：运行时打印配置概要（不含敏感信息）
// ============================================================
@@ -150,6 +165,12 @@ export function printConfigSummary(): void {
url: redis.url.replace(/\/\/.*@/, "//***@"), // 隐藏密码
publishEnabled: redis.publishEnabled,
},
exchange: {
binance: {
apiKey: exchange.binance.apiKey.slice(0, 6) + "***",
apiSecret: "***",
},
},
logging: {
level: logging.level,
nodeEnv: logging.nodeEnv,
data/config/validators.ts
+34
View File
@@ -15,6 +15,7 @@
export interface EnvConfig {
db: DbConfig;
redis: RedisConfig;
exchange: ExchangeConfig;
logging: LoggingConfig;
}
@@ -31,6 +32,18 @@ export interface RedisConfig {
publish_enabled: boolean;
}
/** 交易所 API 密钥配置（按交易所 ID 索引） */
export interface ExchangeConfig {
binance: ExchangeApiKeys;
// 未来扩展：okx、bybit 等
[exchangeId: string]: ExchangeApiKeys | undefined;
}
export interface ExchangeApiKeys {
api_key: string;
api_secret: string;
}
export interface LoggingConfig {
level: "trace" | "debug" | "info" | "warn" | "error" | "fatal";
node_env: "development" | "production" | "test";
@@ -77,6 +90,21 @@ export function validateConfig(raw: unknown): EnvConfig {
const redisUrl = assertString(redisObj["url"], "redis.url");
const redisPublishEnabled = assertBoolean(redisObj["publish_enabled"], "redis.publish_enabled");
// --- exchange ---
const exchange = obj["exchange"];
if (typeof exchange !== "object" || exchange === null) {
throw new Error("[config] env.yaml 缺少 exchange 配置段");
}
const exObj = exchange as Record<string, unknown>;
const binance = exObj["binance"];
if (typeof binance !== "object" || binance === null) {
throw new Error("[config] env.yaml exchange 缺少 binance 配置");
}
const binanceObj = binance as Record<string, unknown>;
const binanceApiKey = assertString(binanceObj["api_key"], "exchange.binance.api_key");
const binanceApiSecret = assertString(binanceObj["api_secret"], "exchange.binance.api_secret");
// --- logging ---
const logging = obj["logging"];
if (typeof logging !== "object" || logging === null) {
@@ -99,6 +127,12 @@ export function validateConfig(raw: unknown): EnvConfig {
url: redisUrl,
publish_enabled: redisPublishEnabled,
},
exchange: {
binance: {
api_key: binanceApiKey,
api_secret: binanceApiSecret,
},
},
logging: {
level: logLevel,
node_env: nodeEnv,
data/db/init-db/01-timescaledb.sql
+7 -1
View File
@@ -13,12 +13,18 @@
-- - klines 基表由 02-init-tables.sql 创建为 TimescaleDB hypertable
-- - 连续聚合视图由 03-continuous-aggregates.sql 创建
-- - TypeORM 的 synchronize:true 与 SQL 脚本互为 fallback（开发/生产双路径）
-- - 本脚本为 init-db 链的第一环，仅负责扩展启用
-- - 本脚本为 init-db 链的第一环，负责扩展启用（TimescaleDB / pg_prewarm / pg_stat_statements）
-- ============================================================
-- 启用 TimescaleDB 扩展（必须最先执行）
CREATE EXTENSION IF NOT EXISTS timescaledb CASCADE;
-- 启用 pg_prewarm 扩展（回测预热，减少首轮查询延迟）
CREATE EXTENSION IF NOT EXISTS pg_prewarm;
-- 启用 pg_stat_statements 扩展（慢查询监控，零成本）
CREATE EXTENSION IF NOT EXISTS pg_stat_statements;
-- 验证扩展已启用
DO $$
BEGIN
data/db/init-db/02-init-tables.sql
+1 -1
View File
@@ -100,7 +100,7 @@ CREATE TABLE IF NOT EXISTS trading_pairs (
kline_interval VARCHAR(100) NOT NULL DEFAULT '1m',
-- K 线合成周期列表（逗号分隔，如 "1m,5m,15m,1h,4h,1d"）
kline_intervals VARCHAR(100) NOT NULL DEFAULT '1m,5m,15m,1h,4h,1d',
kline_intervals VARCHAR(100) NOT NULL DEFAULT '1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,1d,1w,1mon',
-- 历史 K 线最后补全时间（UTC）。默认 Unix epoch 起始，
-- 新交易对从 epoch 起始时间开始全量补拉。
data/db/init-db/03-continuous-aggregates.sql
+175 -1
View File
@@ -2,7 +2,7 @@
-- 03-continuous-aggregates.sql — K 线分层连续聚合视图
-- ============================================================
-- 从 klines（1m）基表创建分层连续聚合物化视图链：
-- 1m → 5m → 15m → 30m → 1h → 4h → 1d → 1w
-- 1m → 3m → 5m → 15m → 30m → 1h → 2h → 4h → 6h → 8h → 1d → 1w → 1mon
--
-- 执行前提：
-- 1. klines hypertable 已创建（由 02-init-tables.sql 创建）
@@ -36,6 +36,37 @@
-- 3. 接入实时数据（模式 A 启用 policy / 模式 B 应用层触发）
-- ============================================================
-- ============================================================
-- 3m K 线（从 1m 基表聚合）
-- ============================================================
CREATE MATERIALIZED VIEW IF NOT EXISTS klines_3m
WITH (timescaledb.continuous) AS
SELECT
time_bucket('3 minutes', time) AS time,
exchange,
symbol,
'3m'::text AS interval,
FIRST(open, time) AS open,
MAX(high) AS high,
MIN(low) AS low,
LAST(close, time) AS close,
SUM(volume) AS volume,
SUM(quote_volume) AS quote_volume,
SUM(taker_buy_base_vol) AS taker_buy_base_vol,
SUM(taker_buy_quote_vol) AS taker_buy_quote_vol,
SUM(trade_count)::integer AS trade_count
FROM klines
GROUP BY time_bucket('3 minutes', klines.time), exchange, symbol
WITH NO DATA;
-- 【模式 A 用户】取消下面注释以启用定时调度刷新
-- SELECT add_continuous_aggregate_policy('klines_3m',
-- start_offset => INTERVAL '1 day',
-- end_offset => INTERVAL '3 minutes',
-- schedule_interval => INTERVAL '3 minutes',
-- if_not_exists => TRUE
-- );
-- ============================================================
-- 5m K 线（从 1m 基表聚合）
-- ============================================================
@@ -160,6 +191,37 @@ WITH NO DATA;
-- if_not_exists => TRUE
-- );
-- ============================================================
-- 2h K 线（从 1h 聚合，分层链）
-- ============================================================
CREATE MATERIALIZED VIEW IF NOT EXISTS klines_2h
WITH (timescaledb.continuous) AS
SELECT
time_bucket('2 hours', time) AS time,
exchange,
symbol,
'2h'::text AS interval,
FIRST(open, time) AS open,
MAX(high) AS high,
MIN(low) AS low,
LAST(close, time) AS close,
SUM(volume) AS volume,
SUM(quote_volume) AS quote_volume,
SUM(taker_buy_base_vol) AS taker_buy_base_vol,
SUM(taker_buy_quote_vol) AS taker_buy_quote_vol,
SUM(trade_count)::integer AS trade_count
FROM klines_1h
GROUP BY time_bucket('2 hours', klines_1h.time), exchange, symbol
WITH NO DATA;
-- 【模式 A 用户】取消下面注释以启用定时调度刷新
-- SELECT add_continuous_aggregate_policy('klines_2h',
-- start_offset => INTERVAL '10 days',
-- end_offset => INTERVAL '2 hours',
-- schedule_interval => INTERVAL '2 hours',
-- if_not_exists => TRUE
-- );
-- ============================================================
-- 4h K 线（从 1h 聚合，分层链）
-- ============================================================
@@ -191,6 +253,68 @@ WITH NO DATA;
-- if_not_exists => TRUE
-- );
-- ============================================================
-- 6h K 线（从 1h 聚合，分层链）
-- ============================================================
CREATE MATERIALIZED VIEW IF NOT EXISTS klines_6h
WITH (timescaledb.continuous) AS
SELECT
time_bucket('6 hours', time) AS time,
exchange,
symbol,
'6h'::text AS interval,
FIRST(open, time) AS open,
MAX(high) AS high,
MIN(low) AS low,
LAST(close, time) AS close,
SUM(volume) AS volume,
SUM(quote_volume) AS quote_volume,
SUM(taker_buy_base_vol) AS taker_buy_base_vol,
SUM(taker_buy_quote_vol) AS taker_buy_quote_vol,
SUM(trade_count)::integer AS trade_count
FROM klines_1h
GROUP BY time_bucket('6 hours', klines_1h.time), exchange, symbol
WITH NO DATA;
-- 【模式 A 用户】取消下面注释以启用定时调度刷新
-- SELECT add_continuous_aggregate_policy('klines_6h',
-- start_offset => INTERVAL '20 days',
-- end_offset => INTERVAL '6 hours',
-- schedule_interval => INTERVAL '6 hours',
-- if_not_exists => TRUE
-- );
-- ============================================================
-- 8h K 线（从 4h 聚合，分层链）
-- ============================================================
CREATE MATERIALIZED VIEW IF NOT EXISTS klines_8h
WITH (timescaledb.continuous) AS
SELECT
time_bucket('8 hours', time) AS time,
exchange,
symbol,
'8h'::text AS interval,
FIRST(open, time) AS open,
MAX(high) AS high,
MIN(low) AS low,
LAST(close, time) AS close,
SUM(volume) AS volume,
SUM(quote_volume) AS quote_volume,
SUM(taker_buy_base_vol) AS taker_buy_base_vol,
SUM(taker_buy_quote_vol) AS taker_buy_quote_vol,
SUM(trade_count)::integer AS trade_count
FROM klines_4h
GROUP BY time_bucket('8 hours', klines_4h.time), exchange, symbol
WITH NO DATA;
-- 【模式 A 用户】取消下面注释以启用定时调度刷新
-- SELECT add_continuous_aggregate_policy('klines_8h',
-- start_offset => INTERVAL '30 days',
-- end_offset => INTERVAL '8 hours',
-- schedule_interval => INTERVAL '8 hours',
-- if_not_exists => TRUE
-- );
-- ============================================================
-- 1d K 线（从 4h 聚合，分层链）
-- ============================================================
@@ -253,25 +377,75 @@ WITH NO DATA;
-- if_not_exists => TRUE
-- );
-- ============================================================
-- 1mon K 线（从 1d 聚合，分层链）
-- ============================================================
CREATE MATERIALIZED VIEW IF NOT EXISTS klines_1mon
WITH (timescaledb.continuous) AS
SELECT
time_bucket('1 month', time) AS time,
exchange,
symbol,
'1mon'::text AS interval,
FIRST(open, time) AS open,
MAX(high) AS high,
MIN(low) AS low,
LAST(close, time) AS close,
SUM(volume) AS volume,
SUM(quote_volume) AS quote_volume,
SUM(taker_buy_base_vol) AS taker_buy_base_vol,
SUM(taker_buy_quote_vol) AS taker_buy_quote_vol,
SUM(trade_count)::integer AS trade_count
FROM klines_1d
GROUP BY time_bucket('1 month', klines_1d.time), exchange, symbol
WITH NO DATA;
-- 【模式 A 用户】取消下面注释以启用定时调度刷新
-- SELECT add_continuous_aggregate_policy('klines_1mon',
-- start_offset => INTERVAL '365 days',
-- end_offset => INTERVAL '1 day',
-- schedule_interval => INTERVAL '1 day',
-- if_not_exists => TRUE
-- );
-- ============================================================
-- 推荐索引：加速按 symbol + time 的查询
-- ============================================================
CREATE INDEX IF NOT EXISTS idx_klines_3m_symbol_time ON klines_3m (exchange, symbol, time DESC);
CREATE INDEX IF NOT EXISTS idx_klines_5m_symbol_time ON klines_5m (exchange, symbol, time DESC);
CREATE INDEX IF NOT EXISTS idx_klines_15m_symbol_time ON klines_15m (exchange, symbol, time DESC);
CREATE INDEX IF NOT EXISTS idx_klines_30m_symbol_time ON klines_30m (exchange, symbol, time DESC);
CREATE INDEX IF NOT EXISTS idx_klines_1h_symbol_time ON klines_1h (exchange, symbol, time DESC);
CREATE INDEX IF NOT EXISTS idx_klines_2h_symbol_time ON klines_2h (exchange, symbol, time DESC);
CREATE INDEX IF NOT EXISTS idx_klines_4h_symbol_time ON klines_4h (exchange, symbol, time DESC);
CREATE INDEX IF NOT EXISTS idx_klines_6h_symbol_time ON klines_6h (exchange, symbol, time DESC);
CREATE INDEX IF NOT EXISTS idx_klines_8h_symbol_time ON klines_8h (exchange, symbol, time DESC);
CREATE INDEX IF NOT EXISTS idx_klines_1d_symbol_time ON klines_1d (exchange, symbol, time DESC);
CREATE INDEX IF NOT EXISTS idx_klines_1w_symbol_time ON klines_1w (exchange, symbol, time DESC);
CREATE INDEX IF NOT EXISTS idx_klines_1mon_symbol_time ON klines_1mon (exchange, symbol, time DESC);
-- ============================================================
-- 截面查询索引：加速同一时间点多品种回测查询
-- 查询模式：WHERE exchange='binance' AND time='2024-01-01' AND symbol IN (…)
-- 回测中跨品种截面查询最常见于日线和周线，因此只在这两层建额外索引。
-- 如需其他周期（如 1h、4h）的截面查询，按同样模式扩展。
-- ============================================================
CREATE INDEX IF NOT EXISTS idx_klines_1d_exchange_time_symbol ON klines_1d (exchange, time DESC, symbol);
CREATE INDEX IF NOT EXISTS idx_klines_1w_exchange_time_symbol ON klines_1w (exchange, time DESC, symbol);
-- ============================================================
-- 首次创建后手动刷新所有视图（填充历史数据）
-- 取消注释以下行执行：
-- ============================================================
-- CALL refresh_continuous_aggregate('klines_3m', NULL, NULL);
-- CALL refresh_continuous_aggregate('klines_5m', NULL, NULL);
-- CALL refresh_continuous_aggregate('klines_15m', NULL, NULL);
-- CALL refresh_continuous_aggregate('klines_30m', NULL, NULL);
-- CALL refresh_continuous_aggregate('klines_1h', NULL, NULL);
-- CALL refresh_continuous_aggregate('klines_2h', NULL, NULL);
-- CALL refresh_continuous_aggregate('klines_4h', NULL, NULL);
-- CALL refresh_continuous_aggregate('klines_6h', NULL, NULL);
-- CALL refresh_continuous_aggregate('klines_8h', NULL, NULL);
-- CALL refresh_continuous_aggregate('klines_1d', NULL, NULL);
-- CALL refresh_continuous_aggregate('klines_1w', NULL, NULL);
-- CALL refresh_continuous_aggregate('klines_1mon', NULL, NULL);
data/exchanges/rest.ts
+8 -2
View File
@@ -1,19 +1,25 @@
import { MainClient, type Kline as BinanceRestKline } from "binance";
import { logger } from "../utils/logger";
import { exchange } from "../config";
import { BaseRestClient } from './base_rest';
import type { KlineInterval, Kline, MarketInfo } from '../types';
/** K 线周期 → 毫秒数映射（用于时间桶计算） */
export const KLINE_INTERVAL_MS: Record<KlineInterval, number> = {
"1m": 60_000,
"3m": 180_000,
"5m": 300_000,
"15m": 900_000,
"30m": 1_800_000,
"1h": 3_600_000,
"2h": 7_200_000,
"4h": 14_400_000,
"6h": 21_600_000,
"8h": 28_800_000,
"1d": 86_400_000,
"1w": 604_800_000,
"1mon": 2_592_000_000,
};
// ============================================================
@@ -102,8 +108,8 @@ async function fetchBinanceKlines(
limit = 500,
): Promise<Kline[]> {
const client = new MainClient({
api_key: 'ONSJKIGRpDYLn6FdV17aAKfjclZ4I2LzamflhuMpsoRQA427lLKeyJlGtg2RZ7DH',
api_secret: '5Mfv4TgvDlRzCHbtl2nJL4mVHUvMm8pyjKiRjMoosBMxrhlqMw6CuQbg2qbS2Npd',
api_key: exchange.binance.apiKey,
api_secret: exchange.binance.apiSecret,
}, {
timeout: 3000,
});
data/run/build_aggregates_sql.ts
+21 -11
View File
@@ -3,8 +3,8 @@
// ============================================================
// 用途：
// 按月份粒度逐月刷新 klines 分层连续聚合物化视图链：
// 5m → 15m → 30m → 1h → 4h → 1d → 1w
// 每层依赖下一层的数据，因此严格按从低到高顺序刷新。
// 基表 1m → 3m / 5m → 15m → 30m → 1h → 2h / 4h / 6h → 8h / 1d → 1w / 1mon
// 严格按依赖顺序从低到高刷新。
//
// 使用方式：
// # 仅生成 SQL 不执行（dry-run，默认）
@@ -36,18 +36,26 @@ import { logger } from "../utils/logger";
/**
* 分层聚合视图链（按依赖顺序：低层级 → 高层级）
*
* 刷新顺序至关重要：
* klines_5m 源数据来自 klines（1m 基表）
* klines_15m 源数据来自 klines_5m
* klines_30m 源数据来自 klines_15m
* klines_1h 源数据来自 klines_30m
* klines_4h 源数据来自 klines_1h
* klines_1d 源数据来自 klines_4h
* klines_1w 源数据来自 klines_1d
* 实际依赖关系（来自于 03-continuous-aggregates.sql）：
* 基表 1m
* ├── 3m （直接聚合 1m）
* └── 5m （直接聚合 1m）→ 15m → 30m → 1h ──→ 2h
* ├── 4h ──→ 8h
* │ └── 1d ──→ 1w
* │ └── 1mon
* └── 6h
*
* 必须严格按此顺序刷新，否则高层级聚合会缺少数据。
* 刷新时必须严格按此顺序，因为：
* - klines_3m / klines_5m 无聚合依赖，可最先刷新
* - klines_15m 依赖 klines_5m
* - klines_30m 依赖 klines_15m
* - klines_1h 依赖 klines_30m
* - klines_2h / klines_4h / klines_6h 依赖 klines_1h
* - klines_8h / klines_1d 依赖 klines_4h
* - klines_1w / klines_1mon 依赖 klines_1d
*/
const AGGREGATE_VIEWS = [
"klines_3m",
"klines_5m",
"klines_15m",
"klines_30m",
@@ -55,8 +63,10 @@ const AGGREGATE_VIEWS = [
"klines_2h",
"klines_4h",
"klines_6h",
"klines_8h",
"klines_1d",
"klines_1w",
"klines_1mon",
] as const;
/** 默认起始年月 */
data/run/exchange.ts
+10 -1
View File
@@ -41,4 +41,13 @@ for (const pair of allPairs) {
} catch (err) {
console.error("拉取失败:", err);
}
}
}
// 所有币种回补完成以后等待1秒关闭pgsql连接退出此进程
await new Promise((resolve) => setTimeout(resolve, 1000));
const { AppDataSource } = await import("../db/data-source");
if (AppDataSource.isInitialized) {
await AppDataSource.destroy();
console.log("pgsql 连接已关闭");
}
process.exit(0);
data/types/kline.ts
+6 -1
View File
@@ -1,10 +1,15 @@
/** K 线周期枚举 */
export type KlineInterval =
| "1m"
| "3m"
| "5m"
| "15m"
| "30m"
| "1h"
| "2h"
| "4h"
| "6h"
| "8h"
| "1d"
| "1w";
| "1w"
| "1mon";