refactor: migrate API keys to config, extend Kline intervals, add DB extensions
Security: - Move hardcoded Binance API key/secret from rest.ts to env.yaml (exchange config segment) - Add ExchangeConfig validation in config/validators.ts - Export typed exchange config from config/index.ts - Update AGENTS/07-caveats.md to reflect the new policy Kline intervals (add 3m / 2h / 6h / 8h / 1mon): - TypeScript: update KlineInterval type, KLINE_INTERVAL_MS mapping, build_aggregates_sql refresh chain - Python: sync KlineInterval Literal type, INTERVAL_TO_TABLE and INTERVAL_MS mappings, db_test table list - SQL: add 5 continuous aggregate materialized views (klines_3m/2h/6h/8h/1mon) with indexes - SQL: extend default kline_intervals in trading_pairs table - SQL: add cross-sectional query indexes for klines_1d and klines_1w DB: - Enable pg_prewarm extension (backtest warmup) - Enable pg_stat_statements extension (slow query monitoring) Other: - data/run/exchange.ts: graceful pgsql shutdown after backfill completes - Config path: load from data/env.yaml (symlink) instead of project root
This commit is contained in:
+3
-1
@@ -6,4 +6,6 @@ db/pgsql/
|
|||||||
|
|
||||||
# Python
|
# Python
|
||||||
__pycache__/
|
__pycache__/
|
||||||
.venv/
|
.venv/
|
||||||
|
|
||||||
|
.codegraph
|
||||||
@@ -1,7 +1,7 @@
|
|||||||
# 注意事项
|
# 注意事项
|
||||||
|
|
||||||
- **`data/exchanges/rest.ts` 包含硬编码的 Binance API Key**(第 105-106 行),不要提交到公开仓库。
|
- **所有 API Key 统一在 `env.yaml` 中管理**,禁止在代码中硬编码。新增交易所时在 `exchange` 段添加对应子配置即可。
|
||||||
- `env.yaml` 包含明文数据库密码且被 git 追踪,注意安全。
|
- `env.yaml` 包含明文密钥和数据库密码且被 git 追踪,注意安全,不要提交到公开仓库。
|
||||||
- 未安装 Python 依赖(如 pydantic),`engine/` 目录有独立的 `.venv/`。
|
- 未安装 Python 依赖(如 pydantic),`engine/` 目录有独立的 `.venv/`。
|
||||||
- `db/pgsql/` 在 `.gitignore` 中,这是 TimescaleDB 数据目录(Docker volume 映射)。
|
- `db/pgsql/` 在 `.gitignore` 中,这是 TimescaleDB 数据目录(Docker volume 映射)。
|
||||||
- `KLINE_INTERVAL_MS` 常量定义了两处:`data/exchanges/rest.ts` 和 `data/types/kline.ts` 的类型定义。新增周期需同步。
|
- `KLINE_INTERVAL_MS` 常量定义了两处:`data/exchanges/rest.ts` 和 `data/types/kline.ts` 的类型定义。新增周期需同步。
|
||||||
|
|||||||
+26
-5
@@ -11,7 +11,7 @@
|
|||||||
// const ds = new DataSource({ ...pgsql });
|
// const ds = new DataSource({ ...pgsql });
|
||||||
// const redisClient = new Redis(redis.url);
|
// const redisClient = new Redis(redis.url);
|
||||||
//
|
//
|
||||||
// 配置文件位置:<project_root>/env.yaml
|
// 配置文件位置:data/env.yaml(软链接 → 项目根目录 env.yaml)
|
||||||
// TypeScript / Python 模块共享同一份配置。
|
// TypeScript / Python 模块共享同一份配置。
|
||||||
// ============================================================
|
// ============================================================
|
||||||
|
|
||||||
@@ -39,12 +39,14 @@ function getProjectRoot(): string {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 从项目根目录读取 env.yaml 并解析为原始对象。
|
* 读取 data/env.yaml(软链接指向项目根目录 env.yaml)并解析为原始对象。
|
||||||
* 文件不存在时抛出明确错误,不做静默降级。
|
* 文件不存在时抛出明确错误,不做静默降级。
|
||||||
*/
|
*/
|
||||||
function loadYamlConfig(): unknown {
|
function loadYamlConfig(): unknown {
|
||||||
const root = getProjectRoot();
|
const __filename = fileURLToPath(import.meta.url);
|
||||||
const yamlPath = resolve(root, "env.yaml");
|
const __dirname = dirname(__filename);
|
||||||
|
// config/index.ts → config/ → data/env.yaml
|
||||||
|
const yamlPath = resolve(__dirname, "../env.yaml");
|
||||||
|
|
||||||
let content: string;
|
let content: string;
|
||||||
try {
|
try {
|
||||||
@@ -52,7 +54,7 @@ function loadYamlConfig(): unknown {
|
|||||||
} catch {
|
} catch {
|
||||||
throw new Error(
|
throw new Error(
|
||||||
`[config] 无法读取配置文件: ${yamlPath}\n` +
|
`[config] 无法读取配置文件: ${yamlPath}\n` +
|
||||||
`请确保项目根目录存在 env.yaml(可参考 data/.env.example 的结构)。`,
|
`请确保 data/env.yaml 软链接指向项目根目录的 env.yaml。`,
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -131,6 +133,19 @@ export const logging = {
|
|||||||
pretty: rawConfig.logging.node_env === "development",
|
pretty: rawConfig.logging.node_env === "development",
|
||||||
} as const;
|
} as const;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 交易所 API 密钥配置
|
||||||
|
*
|
||||||
|
* 按交易所 ID 索引,目前仅 binance。
|
||||||
|
* 新增交易所时在 env.yaml 的 exchange 段添加对应子配置即可。
|
||||||
|
*/
|
||||||
|
export const exchange = {
|
||||||
|
binance: {
|
||||||
|
apiKey: rawConfig.exchange.binance.api_key,
|
||||||
|
apiSecret: rawConfig.exchange.binance.api_secret,
|
||||||
|
},
|
||||||
|
} as const;
|
||||||
|
|
||||||
// ============================================================
|
// ============================================================
|
||||||
// 4. 工具:运行时打印配置概要(不含敏感信息)
|
// 4. 工具:运行时打印配置概要(不含敏感信息)
|
||||||
// ============================================================
|
// ============================================================
|
||||||
@@ -150,6 +165,12 @@ export function printConfigSummary(): void {
|
|||||||
url: redis.url.replace(/\/\/.*@/, "//***@"), // 隐藏密码
|
url: redis.url.replace(/\/\/.*@/, "//***@"), // 隐藏密码
|
||||||
publishEnabled: redis.publishEnabled,
|
publishEnabled: redis.publishEnabled,
|
||||||
},
|
},
|
||||||
|
exchange: {
|
||||||
|
binance: {
|
||||||
|
apiKey: exchange.binance.apiKey.slice(0, 6) + "***",
|
||||||
|
apiSecret: "***",
|
||||||
|
},
|
||||||
|
},
|
||||||
logging: {
|
logging: {
|
||||||
level: logging.level,
|
level: logging.level,
|
||||||
nodeEnv: logging.nodeEnv,
|
nodeEnv: logging.nodeEnv,
|
||||||
|
|||||||
@@ -15,6 +15,7 @@
|
|||||||
export interface EnvConfig {
|
export interface EnvConfig {
|
||||||
db: DbConfig;
|
db: DbConfig;
|
||||||
redis: RedisConfig;
|
redis: RedisConfig;
|
||||||
|
exchange: ExchangeConfig;
|
||||||
logging: LoggingConfig;
|
logging: LoggingConfig;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -31,6 +32,18 @@ export interface RedisConfig {
|
|||||||
publish_enabled: boolean;
|
publish_enabled: boolean;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** 交易所 API 密钥配置(按交易所 ID 索引) */
|
||||||
|
export interface ExchangeConfig {
|
||||||
|
binance: ExchangeApiKeys;
|
||||||
|
// 未来扩展:okx、bybit 等
|
||||||
|
[exchangeId: string]: ExchangeApiKeys | undefined;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface ExchangeApiKeys {
|
||||||
|
api_key: string;
|
||||||
|
api_secret: string;
|
||||||
|
}
|
||||||
|
|
||||||
export interface LoggingConfig {
|
export interface LoggingConfig {
|
||||||
level: "trace" | "debug" | "info" | "warn" | "error" | "fatal";
|
level: "trace" | "debug" | "info" | "warn" | "error" | "fatal";
|
||||||
node_env: "development" | "production" | "test";
|
node_env: "development" | "production" | "test";
|
||||||
@@ -77,6 +90,21 @@ export function validateConfig(raw: unknown): EnvConfig {
|
|||||||
const redisUrl = assertString(redisObj["url"], "redis.url");
|
const redisUrl = assertString(redisObj["url"], "redis.url");
|
||||||
const redisPublishEnabled = assertBoolean(redisObj["publish_enabled"], "redis.publish_enabled");
|
const redisPublishEnabled = assertBoolean(redisObj["publish_enabled"], "redis.publish_enabled");
|
||||||
|
|
||||||
|
// --- exchange ---
|
||||||
|
const exchange = obj["exchange"];
|
||||||
|
if (typeof exchange !== "object" || exchange === null) {
|
||||||
|
throw new Error("[config] env.yaml 缺少 exchange 配置段");
|
||||||
|
}
|
||||||
|
const exObj = exchange as Record<string, unknown>;
|
||||||
|
|
||||||
|
const binance = exObj["binance"];
|
||||||
|
if (typeof binance !== "object" || binance === null) {
|
||||||
|
throw new Error("[config] env.yaml exchange 缺少 binance 配置");
|
||||||
|
}
|
||||||
|
const binanceObj = binance as Record<string, unknown>;
|
||||||
|
const binanceApiKey = assertString(binanceObj["api_key"], "exchange.binance.api_key");
|
||||||
|
const binanceApiSecret = assertString(binanceObj["api_secret"], "exchange.binance.api_secret");
|
||||||
|
|
||||||
// --- logging ---
|
// --- logging ---
|
||||||
const logging = obj["logging"];
|
const logging = obj["logging"];
|
||||||
if (typeof logging !== "object" || logging === null) {
|
if (typeof logging !== "object" || logging === null) {
|
||||||
@@ -99,6 +127,12 @@ export function validateConfig(raw: unknown): EnvConfig {
|
|||||||
url: redisUrl,
|
url: redisUrl,
|
||||||
publish_enabled: redisPublishEnabled,
|
publish_enabled: redisPublishEnabled,
|
||||||
},
|
},
|
||||||
|
exchange: {
|
||||||
|
binance: {
|
||||||
|
api_key: binanceApiKey,
|
||||||
|
api_secret: binanceApiSecret,
|
||||||
|
},
|
||||||
|
},
|
||||||
logging: {
|
logging: {
|
||||||
level: logLevel,
|
level: logLevel,
|
||||||
node_env: nodeEnv,
|
node_env: nodeEnv,
|
||||||
|
|||||||
@@ -13,12 +13,18 @@
|
|||||||
-- - klines 基表由 02-init-tables.sql 创建为 TimescaleDB hypertable
|
-- - klines 基表由 02-init-tables.sql 创建为 TimescaleDB hypertable
|
||||||
-- - 连续聚合视图由 03-continuous-aggregates.sql 创建
|
-- - 连续聚合视图由 03-continuous-aggregates.sql 创建
|
||||||
-- - TypeORM 的 synchronize:true 与 SQL 脚本互为 fallback(开发/生产双路径)
|
-- - TypeORM 的 synchronize:true 与 SQL 脚本互为 fallback(开发/生产双路径)
|
||||||
-- - 本脚本为 init-db 链的第一环,仅负责扩展启用
|
-- - 本脚本为 init-db 链的第一环,负责扩展启用(TimescaleDB / pg_prewarm / pg_stat_statements)
|
||||||
-- ============================================================
|
-- ============================================================
|
||||||
|
|
||||||
-- 启用 TimescaleDB 扩展(必须最先执行)
|
-- 启用 TimescaleDB 扩展(必须最先执行)
|
||||||
CREATE EXTENSION IF NOT EXISTS timescaledb CASCADE;
|
CREATE EXTENSION IF NOT EXISTS timescaledb CASCADE;
|
||||||
|
|
||||||
|
-- 启用 pg_prewarm 扩展(回测预热,减少首轮查询延迟)
|
||||||
|
CREATE EXTENSION IF NOT EXISTS pg_prewarm;
|
||||||
|
|
||||||
|
-- 启用 pg_stat_statements 扩展(慢查询监控,零成本)
|
||||||
|
CREATE EXTENSION IF NOT EXISTS pg_stat_statements;
|
||||||
|
|
||||||
-- 验证扩展已启用
|
-- 验证扩展已启用
|
||||||
DO $$
|
DO $$
|
||||||
BEGIN
|
BEGIN
|
||||||
|
|||||||
@@ -100,7 +100,7 @@ CREATE TABLE IF NOT EXISTS trading_pairs (
|
|||||||
kline_interval VARCHAR(100) NOT NULL DEFAULT '1m',
|
kline_interval VARCHAR(100) NOT NULL DEFAULT '1m',
|
||||||
|
|
||||||
-- K 线合成周期列表(逗号分隔,如 "1m,5m,15m,1h,4h,1d")
|
-- K 线合成周期列表(逗号分隔,如 "1m,5m,15m,1h,4h,1d")
|
||||||
kline_intervals VARCHAR(100) NOT NULL DEFAULT '1m,5m,15m,1h,4h,1d',
|
kline_intervals VARCHAR(100) NOT NULL DEFAULT '1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,1d,1w,1mon',
|
||||||
|
|
||||||
-- 历史 K 线最后补全时间(UTC)。默认 Unix epoch 起始,
|
-- 历史 K 线最后补全时间(UTC)。默认 Unix epoch 起始,
|
||||||
-- 新交易对从 epoch 起始时间开始全量补拉。
|
-- 新交易对从 epoch 起始时间开始全量补拉。
|
||||||
|
|||||||
@@ -2,7 +2,7 @@
|
|||||||
-- 03-continuous-aggregates.sql — K 线分层连续聚合视图
|
-- 03-continuous-aggregates.sql — K 线分层连续聚合视图
|
||||||
-- ============================================================
|
-- ============================================================
|
||||||
-- 从 klines(1m)基表创建分层连续聚合物化视图链:
|
-- 从 klines(1m)基表创建分层连续聚合物化视图链:
|
||||||
-- 1m → 5m → 15m → 30m → 1h → 4h → 1d → 1w
|
-- 1m → 3m → 5m → 15m → 30m → 1h → 2h → 4h → 6h → 8h → 1d → 1w → 1mon
|
||||||
--
|
--
|
||||||
-- 执行前提:
|
-- 执行前提:
|
||||||
-- 1. klines hypertable 已创建(由 02-init-tables.sql 创建)
|
-- 1. klines hypertable 已创建(由 02-init-tables.sql 创建)
|
||||||
@@ -36,6 +36,37 @@
|
|||||||
-- 3. 接入实时数据(模式 A 启用 policy / 模式 B 应用层触发)
|
-- 3. 接入实时数据(模式 A 启用 policy / 模式 B 应用层触发)
|
||||||
-- ============================================================
|
-- ============================================================
|
||||||
|
|
||||||
|
-- ============================================================
|
||||||
|
-- 3m K 线(从 1m 基表聚合)
|
||||||
|
-- ============================================================
|
||||||
|
CREATE MATERIALIZED VIEW IF NOT EXISTS klines_3m
|
||||||
|
WITH (timescaledb.continuous) AS
|
||||||
|
SELECT
|
||||||
|
time_bucket('3 minutes', time) AS time,
|
||||||
|
exchange,
|
||||||
|
symbol,
|
||||||
|
'3m'::text AS interval,
|
||||||
|
FIRST(open, time) AS open,
|
||||||
|
MAX(high) AS high,
|
||||||
|
MIN(low) AS low,
|
||||||
|
LAST(close, time) AS close,
|
||||||
|
SUM(volume) AS volume,
|
||||||
|
SUM(quote_volume) AS quote_volume,
|
||||||
|
SUM(taker_buy_base_vol) AS taker_buy_base_vol,
|
||||||
|
SUM(taker_buy_quote_vol) AS taker_buy_quote_vol,
|
||||||
|
SUM(trade_count)::integer AS trade_count
|
||||||
|
FROM klines
|
||||||
|
GROUP BY time_bucket('3 minutes', klines.time), exchange, symbol
|
||||||
|
WITH NO DATA;
|
||||||
|
|
||||||
|
-- 【模式 A 用户】取消下面注释以启用定时调度刷新
|
||||||
|
-- SELECT add_continuous_aggregate_policy('klines_3m',
|
||||||
|
-- start_offset => INTERVAL '1 day',
|
||||||
|
-- end_offset => INTERVAL '3 minutes',
|
||||||
|
-- schedule_interval => INTERVAL '3 minutes',
|
||||||
|
-- if_not_exists => TRUE
|
||||||
|
-- );
|
||||||
|
|
||||||
-- ============================================================
|
-- ============================================================
|
||||||
-- 5m K 线(从 1m 基表聚合)
|
-- 5m K 线(从 1m 基表聚合)
|
||||||
-- ============================================================
|
-- ============================================================
|
||||||
@@ -160,6 +191,37 @@ WITH NO DATA;
|
|||||||
-- if_not_exists => TRUE
|
-- if_not_exists => TRUE
|
||||||
-- );
|
-- );
|
||||||
|
|
||||||
|
-- ============================================================
|
||||||
|
-- 2h K 线(从 1h 聚合,分层链)
|
||||||
|
-- ============================================================
|
||||||
|
CREATE MATERIALIZED VIEW IF NOT EXISTS klines_2h
|
||||||
|
WITH (timescaledb.continuous) AS
|
||||||
|
SELECT
|
||||||
|
time_bucket('2 hours', time) AS time,
|
||||||
|
exchange,
|
||||||
|
symbol,
|
||||||
|
'2h'::text AS interval,
|
||||||
|
FIRST(open, time) AS open,
|
||||||
|
MAX(high) AS high,
|
||||||
|
MIN(low) AS low,
|
||||||
|
LAST(close, time) AS close,
|
||||||
|
SUM(volume) AS volume,
|
||||||
|
SUM(quote_volume) AS quote_volume,
|
||||||
|
SUM(taker_buy_base_vol) AS taker_buy_base_vol,
|
||||||
|
SUM(taker_buy_quote_vol) AS taker_buy_quote_vol,
|
||||||
|
SUM(trade_count)::integer AS trade_count
|
||||||
|
FROM klines_1h
|
||||||
|
GROUP BY time_bucket('2 hours', klines_1h.time), exchange, symbol
|
||||||
|
WITH NO DATA;
|
||||||
|
|
||||||
|
-- 【模式 A 用户】取消下面注释以启用定时调度刷新
|
||||||
|
-- SELECT add_continuous_aggregate_policy('klines_2h',
|
||||||
|
-- start_offset => INTERVAL '10 days',
|
||||||
|
-- end_offset => INTERVAL '2 hours',
|
||||||
|
-- schedule_interval => INTERVAL '2 hours',
|
||||||
|
-- if_not_exists => TRUE
|
||||||
|
-- );
|
||||||
|
|
||||||
-- ============================================================
|
-- ============================================================
|
||||||
-- 4h K 线(从 1h 聚合,分层链)
|
-- 4h K 线(从 1h 聚合,分层链)
|
||||||
-- ============================================================
|
-- ============================================================
|
||||||
@@ -191,6 +253,68 @@ WITH NO DATA;
|
|||||||
-- if_not_exists => TRUE
|
-- if_not_exists => TRUE
|
||||||
-- );
|
-- );
|
||||||
|
|
||||||
|
-- ============================================================
|
||||||
|
-- 6h K 线(从 1h 聚合,分层链)
|
||||||
|
-- ============================================================
|
||||||
|
CREATE MATERIALIZED VIEW IF NOT EXISTS klines_6h
|
||||||
|
WITH (timescaledb.continuous) AS
|
||||||
|
SELECT
|
||||||
|
time_bucket('6 hours', time) AS time,
|
||||||
|
exchange,
|
||||||
|
symbol,
|
||||||
|
'6h'::text AS interval,
|
||||||
|
FIRST(open, time) AS open,
|
||||||
|
MAX(high) AS high,
|
||||||
|
MIN(low) AS low,
|
||||||
|
LAST(close, time) AS close,
|
||||||
|
SUM(volume) AS volume,
|
||||||
|
SUM(quote_volume) AS quote_volume,
|
||||||
|
SUM(taker_buy_base_vol) AS taker_buy_base_vol,
|
||||||
|
SUM(taker_buy_quote_vol) AS taker_buy_quote_vol,
|
||||||
|
SUM(trade_count)::integer AS trade_count
|
||||||
|
FROM klines_1h
|
||||||
|
GROUP BY time_bucket('6 hours', klines_1h.time), exchange, symbol
|
||||||
|
WITH NO DATA;
|
||||||
|
|
||||||
|
-- 【模式 A 用户】取消下面注释以启用定时调度刷新
|
||||||
|
-- SELECT add_continuous_aggregate_policy('klines_6h',
|
||||||
|
-- start_offset => INTERVAL '20 days',
|
||||||
|
-- end_offset => INTERVAL '6 hours',
|
||||||
|
-- schedule_interval => INTERVAL '6 hours',
|
||||||
|
-- if_not_exists => TRUE
|
||||||
|
-- );
|
||||||
|
|
||||||
|
-- ============================================================
|
||||||
|
-- 8h K 线(从 4h 聚合,分层链)
|
||||||
|
-- ============================================================
|
||||||
|
CREATE MATERIALIZED VIEW IF NOT EXISTS klines_8h
|
||||||
|
WITH (timescaledb.continuous) AS
|
||||||
|
SELECT
|
||||||
|
time_bucket('8 hours', time) AS time,
|
||||||
|
exchange,
|
||||||
|
symbol,
|
||||||
|
'8h'::text AS interval,
|
||||||
|
FIRST(open, time) AS open,
|
||||||
|
MAX(high) AS high,
|
||||||
|
MIN(low) AS low,
|
||||||
|
LAST(close, time) AS close,
|
||||||
|
SUM(volume) AS volume,
|
||||||
|
SUM(quote_volume) AS quote_volume,
|
||||||
|
SUM(taker_buy_base_vol) AS taker_buy_base_vol,
|
||||||
|
SUM(taker_buy_quote_vol) AS taker_buy_quote_vol,
|
||||||
|
SUM(trade_count)::integer AS trade_count
|
||||||
|
FROM klines_4h
|
||||||
|
GROUP BY time_bucket('8 hours', klines_4h.time), exchange, symbol
|
||||||
|
WITH NO DATA;
|
||||||
|
|
||||||
|
-- 【模式 A 用户】取消下面注释以启用定时调度刷新
|
||||||
|
-- SELECT add_continuous_aggregate_policy('klines_8h',
|
||||||
|
-- start_offset => INTERVAL '30 days',
|
||||||
|
-- end_offset => INTERVAL '8 hours',
|
||||||
|
-- schedule_interval => INTERVAL '8 hours',
|
||||||
|
-- if_not_exists => TRUE
|
||||||
|
-- );
|
||||||
|
|
||||||
-- ============================================================
|
-- ============================================================
|
||||||
-- 1d K 线(从 4h 聚合,分层链)
|
-- 1d K 线(从 4h 聚合,分层链)
|
||||||
-- ============================================================
|
-- ============================================================
|
||||||
@@ -253,25 +377,75 @@ WITH NO DATA;
|
|||||||
-- if_not_exists => TRUE
|
-- if_not_exists => TRUE
|
||||||
-- );
|
-- );
|
||||||
|
|
||||||
|
-- ============================================================
|
||||||
|
-- 1mon K 线(从 1d 聚合,分层链)
|
||||||
|
-- ============================================================
|
||||||
|
CREATE MATERIALIZED VIEW IF NOT EXISTS klines_1mon
|
||||||
|
WITH (timescaledb.continuous) AS
|
||||||
|
SELECT
|
||||||
|
time_bucket('1 month', time) AS time,
|
||||||
|
exchange,
|
||||||
|
symbol,
|
||||||
|
'1mon'::text AS interval,
|
||||||
|
FIRST(open, time) AS open,
|
||||||
|
MAX(high) AS high,
|
||||||
|
MIN(low) AS low,
|
||||||
|
LAST(close, time) AS close,
|
||||||
|
SUM(volume) AS volume,
|
||||||
|
SUM(quote_volume) AS quote_volume,
|
||||||
|
SUM(taker_buy_base_vol) AS taker_buy_base_vol,
|
||||||
|
SUM(taker_buy_quote_vol) AS taker_buy_quote_vol,
|
||||||
|
SUM(trade_count)::integer AS trade_count
|
||||||
|
FROM klines_1d
|
||||||
|
GROUP BY time_bucket('1 month', klines_1d.time), exchange, symbol
|
||||||
|
WITH NO DATA;
|
||||||
|
|
||||||
|
-- 【模式 A 用户】取消下面注释以启用定时调度刷新
|
||||||
|
-- SELECT add_continuous_aggregate_policy('klines_1mon',
|
||||||
|
-- start_offset => INTERVAL '365 days',
|
||||||
|
-- end_offset => INTERVAL '1 day',
|
||||||
|
-- schedule_interval => INTERVAL '1 day',
|
||||||
|
-- if_not_exists => TRUE
|
||||||
|
-- );
|
||||||
|
|
||||||
-- ============================================================
|
-- ============================================================
|
||||||
-- 推荐索引:加速按 symbol + time 的查询
|
-- 推荐索引:加速按 symbol + time 的查询
|
||||||
-- ============================================================
|
-- ============================================================
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_klines_3m_symbol_time ON klines_3m (exchange, symbol, time DESC);
|
||||||
CREATE INDEX IF NOT EXISTS idx_klines_5m_symbol_time ON klines_5m (exchange, symbol, time DESC);
|
CREATE INDEX IF NOT EXISTS idx_klines_5m_symbol_time ON klines_5m (exchange, symbol, time DESC);
|
||||||
CREATE INDEX IF NOT EXISTS idx_klines_15m_symbol_time ON klines_15m (exchange, symbol, time DESC);
|
CREATE INDEX IF NOT EXISTS idx_klines_15m_symbol_time ON klines_15m (exchange, symbol, time DESC);
|
||||||
CREATE INDEX IF NOT EXISTS idx_klines_30m_symbol_time ON klines_30m (exchange, symbol, time DESC);
|
CREATE INDEX IF NOT EXISTS idx_klines_30m_symbol_time ON klines_30m (exchange, symbol, time DESC);
|
||||||
CREATE INDEX IF NOT EXISTS idx_klines_1h_symbol_time ON klines_1h (exchange, symbol, time DESC);
|
CREATE INDEX IF NOT EXISTS idx_klines_1h_symbol_time ON klines_1h (exchange, symbol, time DESC);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_klines_2h_symbol_time ON klines_2h (exchange, symbol, time DESC);
|
||||||
CREATE INDEX IF NOT EXISTS idx_klines_4h_symbol_time ON klines_4h (exchange, symbol, time DESC);
|
CREATE INDEX IF NOT EXISTS idx_klines_4h_symbol_time ON klines_4h (exchange, symbol, time DESC);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_klines_6h_symbol_time ON klines_6h (exchange, symbol, time DESC);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_klines_8h_symbol_time ON klines_8h (exchange, symbol, time DESC);
|
||||||
CREATE INDEX IF NOT EXISTS idx_klines_1d_symbol_time ON klines_1d (exchange, symbol, time DESC);
|
CREATE INDEX IF NOT EXISTS idx_klines_1d_symbol_time ON klines_1d (exchange, symbol, time DESC);
|
||||||
CREATE INDEX IF NOT EXISTS idx_klines_1w_symbol_time ON klines_1w (exchange, symbol, time DESC);
|
CREATE INDEX IF NOT EXISTS idx_klines_1w_symbol_time ON klines_1w (exchange, symbol, time DESC);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_klines_1mon_symbol_time ON klines_1mon (exchange, symbol, time DESC);
|
||||||
|
|
||||||
|
-- ============================================================
|
||||||
|
-- 截面查询索引:加速同一时间点多品种回测查询
|
||||||
|
-- 查询模式:WHERE exchange='binance' AND time='2024-01-01' AND symbol IN (…)
|
||||||
|
-- 回测中跨品种截面查询最常见于日线和周线,因此只在这两层建额外索引。
|
||||||
|
-- 如需其他周期(如 1h、4h)的截面查询,按同样模式扩展。
|
||||||
|
-- ============================================================
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_klines_1d_exchange_time_symbol ON klines_1d (exchange, time DESC, symbol);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_klines_1w_exchange_time_symbol ON klines_1w (exchange, time DESC, symbol);
|
||||||
|
|
||||||
-- ============================================================
|
-- ============================================================
|
||||||
-- 首次创建后手动刷新所有视图(填充历史数据)
|
-- 首次创建后手动刷新所有视图(填充历史数据)
|
||||||
-- 取消注释以下行执行:
|
-- 取消注释以下行执行:
|
||||||
-- ============================================================
|
-- ============================================================
|
||||||
|
-- CALL refresh_continuous_aggregate('klines_3m', NULL, NULL);
|
||||||
-- CALL refresh_continuous_aggregate('klines_5m', NULL, NULL);
|
-- CALL refresh_continuous_aggregate('klines_5m', NULL, NULL);
|
||||||
-- CALL refresh_continuous_aggregate('klines_15m', NULL, NULL);
|
-- CALL refresh_continuous_aggregate('klines_15m', NULL, NULL);
|
||||||
-- CALL refresh_continuous_aggregate('klines_30m', NULL, NULL);
|
-- CALL refresh_continuous_aggregate('klines_30m', NULL, NULL);
|
||||||
-- CALL refresh_continuous_aggregate('klines_1h', NULL, NULL);
|
-- CALL refresh_continuous_aggregate('klines_1h', NULL, NULL);
|
||||||
|
-- CALL refresh_continuous_aggregate('klines_2h', NULL, NULL);
|
||||||
-- CALL refresh_continuous_aggregate('klines_4h', NULL, NULL);
|
-- CALL refresh_continuous_aggregate('klines_4h', NULL, NULL);
|
||||||
|
-- CALL refresh_continuous_aggregate('klines_6h', NULL, NULL);
|
||||||
|
-- CALL refresh_continuous_aggregate('klines_8h', NULL, NULL);
|
||||||
-- CALL refresh_continuous_aggregate('klines_1d', NULL, NULL);
|
-- CALL refresh_continuous_aggregate('klines_1d', NULL, NULL);
|
||||||
-- CALL refresh_continuous_aggregate('klines_1w', NULL, NULL);
|
-- CALL refresh_continuous_aggregate('klines_1w', NULL, NULL);
|
||||||
|
-- CALL refresh_continuous_aggregate('klines_1mon', NULL, NULL);
|
||||||
|
|||||||
@@ -1,19 +1,25 @@
|
|||||||
import { MainClient, type Kline as BinanceRestKline } from "binance";
|
import { MainClient, type Kline as BinanceRestKline } from "binance";
|
||||||
|
|
||||||
import { logger } from "../utils/logger";
|
import { logger } from "../utils/logger";
|
||||||
|
import { exchange } from "../config";
|
||||||
import { BaseRestClient } from './base_rest';
|
import { BaseRestClient } from './base_rest';
|
||||||
import type { KlineInterval, Kline, MarketInfo } from '../types';
|
import type { KlineInterval, Kline, MarketInfo } from '../types';
|
||||||
|
|
||||||
/** K 线周期 → 毫秒数映射(用于时间桶计算) */
|
/** K 线周期 → 毫秒数映射(用于时间桶计算) */
|
||||||
export const KLINE_INTERVAL_MS: Record<KlineInterval, number> = {
|
export const KLINE_INTERVAL_MS: Record<KlineInterval, number> = {
|
||||||
"1m": 60_000,
|
"1m": 60_000,
|
||||||
|
"3m": 180_000,
|
||||||
"5m": 300_000,
|
"5m": 300_000,
|
||||||
"15m": 900_000,
|
"15m": 900_000,
|
||||||
"30m": 1_800_000,
|
"30m": 1_800_000,
|
||||||
"1h": 3_600_000,
|
"1h": 3_600_000,
|
||||||
|
"2h": 7_200_000,
|
||||||
"4h": 14_400_000,
|
"4h": 14_400_000,
|
||||||
|
"6h": 21_600_000,
|
||||||
|
"8h": 28_800_000,
|
||||||
"1d": 86_400_000,
|
"1d": 86_400_000,
|
||||||
"1w": 604_800_000,
|
"1w": 604_800_000,
|
||||||
|
"1mon": 2_592_000_000,
|
||||||
};
|
};
|
||||||
|
|
||||||
// ============================================================
|
// ============================================================
|
||||||
@@ -102,8 +108,8 @@ async function fetchBinanceKlines(
|
|||||||
limit = 500,
|
limit = 500,
|
||||||
): Promise<Kline[]> {
|
): Promise<Kline[]> {
|
||||||
const client = new MainClient({
|
const client = new MainClient({
|
||||||
api_key: 'ONSJKIGRpDYLn6FdV17aAKfjclZ4I2LzamflhuMpsoRQA427lLKeyJlGtg2RZ7DH',
|
api_key: exchange.binance.apiKey,
|
||||||
api_secret: '5Mfv4TgvDlRzCHbtl2nJL4mVHUvMm8pyjKiRjMoosBMxrhlqMw6CuQbg2qbS2Npd',
|
api_secret: exchange.binance.apiSecret,
|
||||||
}, {
|
}, {
|
||||||
timeout: 3000,
|
timeout: 3000,
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -3,8 +3,8 @@
|
|||||||
// ============================================================
|
// ============================================================
|
||||||
// 用途:
|
// 用途:
|
||||||
// 按月份粒度逐月刷新 klines 分层连续聚合物化视图链:
|
// 按月份粒度逐月刷新 klines 分层连续聚合物化视图链:
|
||||||
// 5m → 15m → 30m → 1h → 4h → 1d → 1w
|
// 基表 1m → 3m / 5m → 15m → 30m → 1h → 2h / 4h / 6h → 8h / 1d → 1w / 1mon
|
||||||
// 每层依赖下一层的数据,因此严格按从低到高顺序刷新。
|
// 严格按依赖顺序从低到高刷新。
|
||||||
//
|
//
|
||||||
// 使用方式:
|
// 使用方式:
|
||||||
// # 仅生成 SQL 不执行(dry-run,默认)
|
// # 仅生成 SQL 不执行(dry-run,默认)
|
||||||
@@ -36,18 +36,26 @@ import { logger } from "../utils/logger";
|
|||||||
/**
|
/**
|
||||||
* 分层聚合视图链(按依赖顺序:低层级 → 高层级)
|
* 分层聚合视图链(按依赖顺序:低层级 → 高层级)
|
||||||
*
|
*
|
||||||
* 刷新顺序至关重要:
|
* 实际依赖关系(来自于 03-continuous-aggregates.sql):
|
||||||
* klines_5m 源数据来自 klines(1m 基表)
|
* 基表 1m
|
||||||
* klines_15m 源数据来自 klines_5m
|
* ├── 3m (直接聚合 1m)
|
||||||
* klines_30m 源数据来自 klines_15m
|
* └── 5m (直接聚合 1m)→ 15m → 30m → 1h ──→ 2h
|
||||||
* klines_1h 源数据来自 klines_30m
|
* ├── 4h ──→ 8h
|
||||||
* klines_4h 源数据来自 klines_1h
|
* │ └── 1d ──→ 1w
|
||||||
* klines_1d 源数据来自 klines_4h
|
* │ └── 1mon
|
||||||
* klines_1w 源数据来自 klines_1d
|
* └── 6h
|
||||||
*
|
*
|
||||||
* 必须严格按此顺序刷新,否则高层级聚合会缺少数据。
|
* 刷新时必须严格按此顺序,因为:
|
||||||
|
* - klines_3m / klines_5m 无聚合依赖,可最先刷新
|
||||||
|
* - klines_15m 依赖 klines_5m
|
||||||
|
* - klines_30m 依赖 klines_15m
|
||||||
|
* - klines_1h 依赖 klines_30m
|
||||||
|
* - klines_2h / klines_4h / klines_6h 依赖 klines_1h
|
||||||
|
* - klines_8h / klines_1d 依赖 klines_4h
|
||||||
|
* - klines_1w / klines_1mon 依赖 klines_1d
|
||||||
*/
|
*/
|
||||||
const AGGREGATE_VIEWS = [
|
const AGGREGATE_VIEWS = [
|
||||||
|
"klines_3m",
|
||||||
"klines_5m",
|
"klines_5m",
|
||||||
"klines_15m",
|
"klines_15m",
|
||||||
"klines_30m",
|
"klines_30m",
|
||||||
@@ -55,8 +63,10 @@ const AGGREGATE_VIEWS = [
|
|||||||
"klines_2h",
|
"klines_2h",
|
||||||
"klines_4h",
|
"klines_4h",
|
||||||
"klines_6h",
|
"klines_6h",
|
||||||
|
"klines_8h",
|
||||||
"klines_1d",
|
"klines_1d",
|
||||||
"klines_1w",
|
"klines_1w",
|
||||||
|
"klines_1mon",
|
||||||
] as const;
|
] as const;
|
||||||
|
|
||||||
/** 默认起始年月 */
|
/** 默认起始年月 */
|
||||||
|
|||||||
+10
-1
@@ -41,4 +41,13 @@ for (const pair of allPairs) {
|
|||||||
} catch (err) {
|
} catch (err) {
|
||||||
console.error("拉取失败:", err);
|
console.error("拉取失败:", err);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// 所有币种回补完成以后等待1秒关闭pgsql连接退出此进程
|
||||||
|
await new Promise((resolve) => setTimeout(resolve, 1000));
|
||||||
|
const { AppDataSource } = await import("../db/data-source");
|
||||||
|
if (AppDataSource.isInitialized) {
|
||||||
|
await AppDataSource.destroy();
|
||||||
|
console.log("pgsql 连接已关闭");
|
||||||
|
}
|
||||||
|
process.exit(0);
|
||||||
+6
-1
@@ -1,10 +1,15 @@
|
|||||||
/** K 线周期枚举 */
|
/** K 线周期枚举 */
|
||||||
export type KlineInterval =
|
export type KlineInterval =
|
||||||
| "1m"
|
| "1m"
|
||||||
|
| "3m"
|
||||||
| "5m"
|
| "5m"
|
||||||
| "15m"
|
| "15m"
|
||||||
| "30m"
|
| "30m"
|
||||||
| "1h"
|
| "1h"
|
||||||
|
| "2h"
|
||||||
| "4h"
|
| "4h"
|
||||||
|
| "6h"
|
||||||
|
| "8h"
|
||||||
| "1d"
|
| "1d"
|
||||||
| "1w";
|
| "1w"
|
||||||
|
| "1mon";
|
||||||
|
|||||||
@@ -14,7 +14,7 @@ from pydantic import BaseModel, Field, field_validator
|
|||||||
# K 线周期类型
|
# K 线周期类型
|
||||||
# ============================================================
|
# ============================================================
|
||||||
|
|
||||||
KlineInterval = Literal["1m", "5m", "15m", "30m", "1h", "2h", "4h", "6h", "1d", "1w"]
|
KlineInterval = Literal["1m", "3m", "5m", "15m", "30m", "1h", "2h", "4h", "6h", "8h", "1d", "1w", "1mon"]
|
||||||
|
|
||||||
|
|
||||||
# ============================================================
|
# ============================================================
|
||||||
|
|||||||
@@ -32,6 +32,7 @@ from ..common.models import Kline, KlineInterval
|
|||||||
# ── 周期 → 表名映射 ──
|
# ── 周期 → 表名映射 ──
|
||||||
INTERVAL_TO_TABLE: dict[KlineInterval, str] = {
|
INTERVAL_TO_TABLE: dict[KlineInterval, str] = {
|
||||||
"1m": "klines",
|
"1m": "klines",
|
||||||
|
"3m": "klines_3m",
|
||||||
"5m": "klines_5m",
|
"5m": "klines_5m",
|
||||||
"15m": "klines_15m",
|
"15m": "klines_15m",
|
||||||
"30m": "klines_30m",
|
"30m": "klines_30m",
|
||||||
@@ -39,13 +40,16 @@ INTERVAL_TO_TABLE: dict[KlineInterval, str] = {
|
|||||||
"2h": "klines_2h",
|
"2h": "klines_2h",
|
||||||
"4h": "klines_4h",
|
"4h": "klines_4h",
|
||||||
"6h": "klines_6h",
|
"6h": "klines_6h",
|
||||||
|
"8h": "klines_8h",
|
||||||
"1d": "klines_1d",
|
"1d": "klines_1d",
|
||||||
"1w": "klines_1w",
|
"1w": "klines_1w",
|
||||||
|
"1mon": "klines_1mon",
|
||||||
}
|
}
|
||||||
|
|
||||||
# ── 周期毫秒数 ──
|
# ── 周期毫秒数 ──
|
||||||
INTERVAL_MS: dict[KlineInterval, int] = {
|
INTERVAL_MS: dict[KlineInterval, int] = {
|
||||||
"1m": 60_000,
|
"1m": 60_000,
|
||||||
|
"3m": 180_000,
|
||||||
"5m": 300_000,
|
"5m": 300_000,
|
||||||
"15m": 900_000,
|
"15m": 900_000,
|
||||||
"30m": 1_800_000,
|
"30m": 1_800_000,
|
||||||
@@ -53,8 +57,10 @@ INTERVAL_MS: dict[KlineInterval, int] = {
|
|||||||
"2h": 7_200_000,
|
"2h": 7_200_000,
|
||||||
"4h": 14_400_000,
|
"4h": 14_400_000,
|
||||||
"6h": 21_600_000,
|
"6h": 21_600_000,
|
||||||
|
"8h": 28_800_000,
|
||||||
"1d": 86_400_000,
|
"1d": 86_400_000,
|
||||||
"1w": 604_800_000,
|
"1w": 604_800_000,
|
||||||
|
"1mon": 2_592_000_000,
|
||||||
}
|
}
|
||||||
|
|
||||||
DEFAULT_BATCH_SIZE = 5000
|
DEFAULT_BATCH_SIZE = 5000
|
||||||
|
|||||||
@@ -16,13 +16,18 @@ from common.config import config as app_config
|
|||||||
# ── 各周期对应的表/视图 ──
|
# ── 各周期对应的表/视图 ──
|
||||||
INTERVAL_TABLES: dict[str, str] = {
|
INTERVAL_TABLES: dict[str, str] = {
|
||||||
"1m": "klines",
|
"1m": "klines",
|
||||||
|
"3m": "klines_3m",
|
||||||
"5m": "klines_5m",
|
"5m": "klines_5m",
|
||||||
"15m": "klines_15m",
|
"15m": "klines_15m",
|
||||||
"30m": "klines_30m",
|
"30m": "klines_30m",
|
||||||
"1h": "klines_1h",
|
"1h": "klines_1h",
|
||||||
|
"2h": "klines_2h",
|
||||||
"4h": "klines_4h",
|
"4h": "klines_4h",
|
||||||
|
"6h": "klines_6h",
|
||||||
|
"8h": "klines_8h",
|
||||||
"1d": "klines_1d",
|
"1d": "klines_1d",
|
||||||
"1w": "klines_1w",
|
"1w": "klines_1w",
|
||||||
|
"1mon": "klines_1mon",
|
||||||
}
|
}
|
||||||
|
|
||||||
LIMIT = 5
|
LIMIT = 5
|
||||||
|
|||||||
@@ -22,6 +22,12 @@ redis:
|
|||||||
# 是否启用 Pub/Sub 行情发布(开发时可关闭以节省资源)
|
# 是否启用 Pub/Sub 行情发布(开发时可关闭以节省资源)
|
||||||
publish_enabled: true
|
publish_enabled: true
|
||||||
|
|
||||||
|
# --- 交易所 API 密钥 ---
|
||||||
|
exchange:
|
||||||
|
binance:
|
||||||
|
api_key: "ONSJKIGRpDYLn6FdV17aAKfjclZ4I2LzamflhuMpsoRQA427lLKeyJlGtg2RZ7DH"
|
||||||
|
api_secret: "5Mfv4TgvDlRzCHbtl2nJL4mVHUvMm8pyjKiRjMoosBMxrhlqMw6CuQbg2qbS2Npd"
|
||||||
|
|
||||||
# --- 日志 ---
|
# --- 日志 ---
|
||||||
logging:
|
logging:
|
||||||
# 日志级别:trace / debug / info / warn / error / fatal
|
# 日志级别:trace / debug / info / warn / error / fatal
|
||||||
|
|||||||
Reference in New Issue
Block a user