Release v1.3.0
This commit is contained in:
+979
@@ -0,0 +1,979 @@
|
||||
import argparse
|
||||
import asyncio
|
||||
import logging
|
||||
import os
|
||||
import sqlite3
|
||||
import time
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timezone
|
||||
from decimal import Decimal, InvalidOperation
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from zoneinfo import ZoneInfo
|
||||
|
||||
import aiohttp
|
||||
from dotenv import load_dotenv
|
||||
|
||||
|
||||
load_dotenv()
|
||||
|
||||
BASE_URL = os.getenv("BINANCE_FAPI_BASE_URL", "https://fapi.binance.com")
|
||||
DB_PATH = Path(os.getenv("DB_PATH", "data/app.db"))
|
||||
INTERVAL = "4h"
|
||||
INTERVAL_MS = 4 * 60 * 60 * 1000
|
||||
ATR_LENGTH = int(os.getenv("ATR_LENGTH", "14"))
|
||||
ATR_MULTIPLE = Decimal(os.getenv("ATR_MULTIPLE", "1.5"))
|
||||
CRYPTO_ATR_LENGTH = int(os.getenv("CRYPTO_ATR_LENGTH", str(ATR_LENGTH)))
|
||||
CRYPTO_ATR_MULTIPLE = Decimal(os.getenv("CRYPTO_ATR_MULTIPLE", str(ATR_MULTIPLE)))
|
||||
TRADFI_ATR_LENGTH = int(os.getenv("TRADFI_ATR_LENGTH", str(ATR_LENGTH)))
|
||||
TRADFI_ATR_MULTIPLE = Decimal(os.getenv("TRADFI_ATR_MULTIPLE", str(ATR_MULTIPLE)))
|
||||
INIT_KLINES_LIMIT = int(os.getenv("INIT_KLINES_LIMIT", "40"))
|
||||
CONCURRENCY = int(os.getenv("CONCURRENCY", "15"))
|
||||
HTTP_TIMEOUT = int(os.getenv("HTTP_TIMEOUT", "20"))
|
||||
MAX_RETRIES = int(os.getenv("MAX_RETRIES", "3"))
|
||||
RATE_LIMIT_BACKOFF = int(os.getenv("RATE_LIMIT_BACKOFF", "30"))
|
||||
BODY_RATIO_FILTER_ENABLED = os.getenv("BODY_RATIO_FILTER_ENABLED", "false").lower() == "true"
|
||||
MIN_BODY_RATIO = Decimal(os.getenv("MIN_BODY_RATIO", "0.5"))
|
||||
CRYPTO_BODY_RATIO_FILTER_ENABLED = (
|
||||
os.getenv("CRYPTO_BODY_RATIO_FILTER_ENABLED", str(BODY_RATIO_FILTER_ENABLED)).lower() == "true"
|
||||
)
|
||||
CRYPTO_MIN_BODY_RATIO = Decimal(os.getenv("CRYPTO_MIN_BODY_RATIO", str(MIN_BODY_RATIO)))
|
||||
TRADFI_BODY_RATIO_FILTER_ENABLED = (
|
||||
os.getenv("TRADFI_BODY_RATIO_FILTER_ENABLED", str(BODY_RATIO_FILTER_ENABLED)).lower() == "true"
|
||||
)
|
||||
TRADFI_MIN_BODY_RATIO = Decimal(os.getenv("TRADFI_MIN_BODY_RATIO", str(MIN_BODY_RATIO)))
|
||||
KLINES_RETENTION_PER_SYMBOL = int(os.getenv("KLINES_RETENTION_PER_SYMBOL", "500"))
|
||||
SIGNAL_RETENTION_DAYS = int(os.getenv("SIGNAL_RETENTION_DAYS", "90"))
|
||||
DISCORD_ENABLED = os.getenv("DISCORD_ENABLED", "false").lower() == "true"
|
||||
DISCORD_WEBHOOK_URL = os.getenv("DISCORD_WEBHOOK_URL", "").strip()
|
||||
|
||||
MARKET_CRYPTO = "CRYPTO"
|
||||
MARKET_TRADFI = "TRADFI"
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class MarketConfig:
|
||||
market_type: str
|
||||
label: str
|
||||
atr_length: int
|
||||
atr_multiple: Decimal
|
||||
body_ratio_filter_enabled: bool
|
||||
min_body_ratio: Decimal
|
||||
strict_time_grid: bool
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ScanResult:
|
||||
market_type: str
|
||||
label: str
|
||||
status: str
|
||||
symbols_total: int
|
||||
symbols_failed: int
|
||||
updated_klines: int
|
||||
signals_created: int
|
||||
elapsed_seconds: float
|
||||
error_summary: str | None = None
|
||||
|
||||
|
||||
MARKET_CONFIGS = {
|
||||
MARKET_CRYPTO: MarketConfig(
|
||||
market_type=MARKET_CRYPTO,
|
||||
label="Binance USDT 永续",
|
||||
atr_length=CRYPTO_ATR_LENGTH,
|
||||
atr_multiple=CRYPTO_ATR_MULTIPLE,
|
||||
body_ratio_filter_enabled=CRYPTO_BODY_RATIO_FILTER_ENABLED,
|
||||
min_body_ratio=CRYPTO_MIN_BODY_RATIO,
|
||||
strict_time_grid=True,
|
||||
),
|
||||
MARKET_TRADFI: MarketConfig(
|
||||
market_type=MARKET_TRADFI,
|
||||
label="Binance TradFi 永续",
|
||||
atr_length=TRADFI_ATR_LENGTH,
|
||||
atr_multiple=TRADFI_ATR_MULTIPLE,
|
||||
body_ratio_filter_enabled=TRADFI_BODY_RATIO_FILTER_ENABLED,
|
||||
min_body_ratio=TRADFI_MIN_BODY_RATIO,
|
||||
strict_time_grid=False,
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
logging.basicConfig(
|
||||
level=os.getenv("LOG_LEVEL", "INFO"),
|
||||
format="%(asctime)s %(levelname)s %(message)s",
|
||||
)
|
||||
logger = logging.getLogger("crypto-atr-scanner")
|
||||
|
||||
|
||||
def utc_now_ms() -> int:
|
||||
return int(datetime.now(timezone.utc).timestamp() * 1000)
|
||||
|
||||
|
||||
def latest_theoretical_closed_open_time(now_ms: int | None = None) -> int:
|
||||
now_ms = utc_now_ms() if now_ms is None else now_ms
|
||||
return (now_ms // INTERVAL_MS) * INTERVAL_MS - INTERVAL_MS
|
||||
|
||||
|
||||
def ms_to_iso(ms: int) -> str:
|
||||
return datetime.fromtimestamp(ms / 1000, timezone.utc).isoformat()
|
||||
|
||||
|
||||
def decimal_from_api(value: Any) -> Decimal:
|
||||
try:
|
||||
return Decimal(str(value))
|
||||
except (InvalidOperation, ValueError) as exc:
|
||||
raise ValueError(f"Invalid decimal value from API: {value!r}") from exc
|
||||
|
||||
|
||||
def db_connect(path: Path | str = DB_PATH) -> sqlite3.Connection:
|
||||
path = Path(path)
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
conn = sqlite3.connect(path)
|
||||
conn.row_factory = sqlite3.Row
|
||||
conn.execute("PRAGMA journal_mode = WAL;")
|
||||
conn.execute("PRAGMA busy_timeout = 5000;")
|
||||
return conn
|
||||
|
||||
|
||||
def init_db(conn: sqlite3.Connection) -> None:
|
||||
existing_symbols = conn.execute(
|
||||
"SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'symbols'"
|
||||
).fetchone()
|
||||
if existing_symbols:
|
||||
symbol_cols = [row["name"] for row in conn.execute("PRAGMA table_info(symbols)")]
|
||||
if "market_type" not in symbol_cols:
|
||||
migrate_v11_to_v12(conn)
|
||||
|
||||
conn.executescript(
|
||||
"""
|
||||
CREATE TABLE IF NOT EXISTS symbols (
|
||||
market_type TEXT NOT NULL DEFAULT 'CRYPTO',
|
||||
symbol TEXT NOT NULL,
|
||||
status TEXT NOT NULL,
|
||||
underlying_type TEXT,
|
||||
underlying_subtype TEXT,
|
||||
updated_at TEXT NOT NULL,
|
||||
PRIMARY KEY (market_type, symbol)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS klines (
|
||||
market_type TEXT NOT NULL DEFAULT 'CRYPTO',
|
||||
symbol TEXT NOT NULL,
|
||||
open_time INTEGER NOT NULL,
|
||||
open TEXT NOT NULL,
|
||||
high TEXT NOT NULL,
|
||||
low TEXT NOT NULL,
|
||||
close TEXT NOT NULL,
|
||||
close_time INTEGER NOT NULL,
|
||||
atr14 TEXT,
|
||||
PRIMARY KEY (market_type, symbol, open_time)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS signals (
|
||||
market_type TEXT NOT NULL DEFAULT 'CRYPTO',
|
||||
symbol TEXT NOT NULL,
|
||||
direction TEXT NOT NULL,
|
||||
multiple TEXT NOT NULL,
|
||||
range TEXT NOT NULL,
|
||||
atr14 TEXT NOT NULL,
|
||||
open_time INTEGER NOT NULL,
|
||||
created_at TEXT NOT NULL,
|
||||
PRIMARY KEY (market_type, symbol, open_time)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_klines_symbol_time
|
||||
ON klines (market_type, symbol, open_time DESC);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_signals_time
|
||||
ON signals (market_type, open_time DESC);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS scan_runs (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
market_type TEXT NOT NULL,
|
||||
started_at TEXT NOT NULL,
|
||||
finished_at TEXT NOT NULL,
|
||||
status TEXT NOT NULL,
|
||||
symbols_total INTEGER NOT NULL DEFAULT 0,
|
||||
symbols_success INTEGER NOT NULL DEFAULT 0,
|
||||
symbols_failed INTEGER NOT NULL DEFAULT 0,
|
||||
updated_klines INTEGER NOT NULL DEFAULT 0,
|
||||
signals_created INTEGER NOT NULL DEFAULT 0,
|
||||
elapsed_seconds TEXT NOT NULL,
|
||||
latest_target_open_time INTEGER,
|
||||
error_summary TEXT
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_scan_runs_market_finished
|
||||
ON scan_runs (market_type, finished_at DESC);
|
||||
"""
|
||||
)
|
||||
conn.commit()
|
||||
|
||||
|
||||
def migrate_v11_to_v12(conn: sqlite3.Connection) -> None:
|
||||
logger.info("Migrating database schema to v1.2 market_type layout")
|
||||
conn.executescript(
|
||||
"""
|
||||
ALTER TABLE symbols RENAME TO symbols_v11;
|
||||
ALTER TABLE klines RENAME TO klines_v11;
|
||||
ALTER TABLE signals RENAME TO signals_v11;
|
||||
|
||||
CREATE TABLE symbols (
|
||||
market_type TEXT NOT NULL DEFAULT 'CRYPTO',
|
||||
symbol TEXT NOT NULL,
|
||||
status TEXT NOT NULL,
|
||||
underlying_type TEXT,
|
||||
underlying_subtype TEXT,
|
||||
updated_at TEXT NOT NULL,
|
||||
PRIMARY KEY (market_type, symbol)
|
||||
);
|
||||
|
||||
CREATE TABLE klines (
|
||||
market_type TEXT NOT NULL DEFAULT 'CRYPTO',
|
||||
symbol TEXT NOT NULL,
|
||||
open_time INTEGER NOT NULL,
|
||||
open TEXT NOT NULL,
|
||||
high TEXT NOT NULL,
|
||||
low TEXT NOT NULL,
|
||||
close TEXT NOT NULL,
|
||||
close_time INTEGER NOT NULL,
|
||||
atr14 TEXT,
|
||||
PRIMARY KEY (market_type, symbol, open_time)
|
||||
);
|
||||
|
||||
CREATE TABLE signals (
|
||||
market_type TEXT NOT NULL DEFAULT 'CRYPTO',
|
||||
symbol TEXT NOT NULL,
|
||||
direction TEXT NOT NULL,
|
||||
multiple TEXT NOT NULL,
|
||||
range TEXT NOT NULL,
|
||||
atr14 TEXT NOT NULL,
|
||||
open_time INTEGER NOT NULL,
|
||||
created_at TEXT NOT NULL,
|
||||
PRIMARY KEY (market_type, symbol, open_time)
|
||||
);
|
||||
|
||||
INSERT INTO symbols (market_type, symbol, status, updated_at)
|
||||
SELECT 'CRYPTO', symbol, status, updated_at FROM symbols_v11;
|
||||
|
||||
INSERT INTO klines (market_type, symbol, open_time, open, high, low, close, close_time, atr14)
|
||||
SELECT 'CRYPTO', symbol, open_time, open, high, low, close, close_time, atr14 FROM klines_v11;
|
||||
|
||||
INSERT INTO signals (market_type, symbol, direction, multiple, range, atr14, open_time, created_at)
|
||||
SELECT 'CRYPTO', symbol, direction, multiple, range, atr14, open_time, created_at FROM signals_v11;
|
||||
|
||||
DROP TABLE symbols_v11;
|
||||
DROP TABLE klines_v11;
|
||||
DROP TABLE signals_v11;
|
||||
"""
|
||||
)
|
||||
conn.commit()
|
||||
|
||||
|
||||
async def fetch_json(
|
||||
session: aiohttp.ClientSession,
|
||||
path: str,
|
||||
params: dict[str, Any] | None = None,
|
||||
) -> Any:
|
||||
url = f"{BASE_URL}{path}"
|
||||
last_error: Exception | None = None
|
||||
for attempt in range(1, MAX_RETRIES + 1):
|
||||
try:
|
||||
async with session.get(url, params=params) as response:
|
||||
response.raise_for_status()
|
||||
return await response.json()
|
||||
except aiohttp.ClientResponseError as exc:
|
||||
last_error = exc
|
||||
if exc.status in {418, 429}:
|
||||
retry_after = exc.headers.get("Retry-After") if exc.headers else None
|
||||
delay = int(retry_after) if retry_after and retry_after.isdigit() else RATE_LIMIT_BACKOFF
|
||||
logger.warning(
|
||||
"Rate limited attempt=%s status=%s path=%s sleep=%ss",
|
||||
attempt,
|
||||
exc.status,
|
||||
path,
|
||||
delay,
|
||||
)
|
||||
else:
|
||||
delay = min(2**attempt, 10)
|
||||
logger.warning(
|
||||
"HTTP error attempt=%s status=%s path=%s error=%s",
|
||||
attempt,
|
||||
exc.status,
|
||||
path,
|
||||
exc,
|
||||
)
|
||||
await asyncio.sleep(delay)
|
||||
except Exception as exc: # noqa: BLE001 - log and retry all transport/API errors.
|
||||
last_error = exc
|
||||
delay = min(2**attempt, 10)
|
||||
logger.warning("Request failed attempt=%s path=%s error=%s", attempt, path, exc)
|
||||
await asyncio.sleep(delay)
|
||||
raise RuntimeError(f"Request failed after {MAX_RETRIES} attempts: {url}") from last_error
|
||||
|
||||
|
||||
def normalize_subtype(value: Any) -> str | None:
|
||||
if value is None:
|
||||
return None
|
||||
if isinstance(value, list):
|
||||
return ",".join(str(item) for item in value)
|
||||
return str(value)
|
||||
|
||||
|
||||
async def fetch_symbols(session: aiohttp.ClientSession, market_type: str) -> list[dict[str, Any]]:
|
||||
exchange_info = await fetch_json(session, "/fapi/v1/exchangeInfo")
|
||||
symbols: list[dict[str, Any]] = []
|
||||
for item in exchange_info.get("symbols", []):
|
||||
if item.get("status") != "TRADING" or item.get("quoteAsset") != "USDT":
|
||||
continue
|
||||
subtype = item.get("underlyingSubType") or []
|
||||
is_tradfi = item.get("contractType") == "TRADIFI_PERPETUAL" or "TradFi" in subtype
|
||||
is_crypto = item.get("contractType") == "PERPETUAL" and not is_tradfi
|
||||
if market_type == MARKET_CRYPTO and not is_crypto:
|
||||
continue
|
||||
if market_type == MARKET_TRADFI and not is_tradfi:
|
||||
continue
|
||||
symbols.append(
|
||||
{
|
||||
"symbol": item["symbol"],
|
||||
"underlying_type": item.get("underlyingType"),
|
||||
"underlying_subtype": normalize_subtype(subtype),
|
||||
}
|
||||
)
|
||||
return sorted(symbols, key=lambda row: row["symbol"])
|
||||
|
||||
|
||||
async def fetch_klines(
|
||||
session: aiohttp.ClientSession,
|
||||
symbol: str,
|
||||
limit: int = 2,
|
||||
start_time: int | None = None,
|
||||
end_time: int | None = None,
|
||||
) -> list[list[Any]]:
|
||||
params: dict[str, Any] = {"symbol": symbol, "interval": INTERVAL, "limit": limit}
|
||||
if start_time is not None:
|
||||
params["startTime"] = start_time
|
||||
if end_time is not None:
|
||||
params["endTime"] = end_time
|
||||
return await fetch_json(session, "/fapi/v1/klines", params)
|
||||
|
||||
|
||||
def closed_klines(raw_klines: list[list[Any]], now_ms: int) -> list[dict[str, Any]]:
|
||||
klines: list[dict[str, Any]] = []
|
||||
for item in raw_klines:
|
||||
close_time = int(item[6])
|
||||
if close_time > now_ms:
|
||||
continue
|
||||
klines.append(
|
||||
{
|
||||
"open_time": int(item[0]),
|
||||
"open": str(item[1]),
|
||||
"high": str(item[2]),
|
||||
"low": str(item[3]),
|
||||
"close": str(item[4]),
|
||||
"close_time": close_time,
|
||||
}
|
||||
)
|
||||
return sorted(klines, key=lambda row: row["open_time"])
|
||||
|
||||
|
||||
def upsert_symbols(conn: sqlite3.Connection, market_type: str, symbols: list[dict[str, Any]]) -> None:
|
||||
now = datetime.now(timezone.utc).isoformat()
|
||||
active = {row["symbol"] for row in symbols}
|
||||
for row in symbols:
|
||||
conn.execute(
|
||||
"""
|
||||
INSERT INTO symbols (
|
||||
market_type, symbol, status, underlying_type, underlying_subtype, updated_at
|
||||
)
|
||||
VALUES (?, ?, 'TRADING', ?, ?, ?)
|
||||
ON CONFLICT(market_type, symbol) DO UPDATE SET
|
||||
status = excluded.status,
|
||||
underlying_type = excluded.underlying_type,
|
||||
underlying_subtype = excluded.underlying_subtype,
|
||||
updated_at = excluded.updated_at
|
||||
""",
|
||||
(
|
||||
market_type,
|
||||
row["symbol"],
|
||||
row.get("underlying_type"),
|
||||
row.get("underlying_subtype"),
|
||||
now,
|
||||
),
|
||||
)
|
||||
|
||||
existing = conn.execute(
|
||||
"SELECT symbol FROM symbols WHERE market_type = ?",
|
||||
(market_type,),
|
||||
).fetchall()
|
||||
for row in existing:
|
||||
if row["symbol"] not in active:
|
||||
conn.execute(
|
||||
"""
|
||||
UPDATE symbols
|
||||
SET status = 'INACTIVE', updated_at = ?
|
||||
WHERE market_type = ? AND symbol = ?
|
||||
""",
|
||||
(now, market_type, row["symbol"]),
|
||||
)
|
||||
conn.commit()
|
||||
|
||||
|
||||
def latest_kline(conn: sqlite3.Connection, market_type: str, symbol: str) -> sqlite3.Row | None:
|
||||
return conn.execute(
|
||||
"""
|
||||
SELECT * FROM klines
|
||||
WHERE market_type = ? AND symbol = ?
|
||||
ORDER BY open_time DESC
|
||||
LIMIT 1
|
||||
""",
|
||||
(market_type, symbol),
|
||||
).fetchone()
|
||||
|
||||
|
||||
def active_symbols_from_db(conn: sqlite3.Connection, market_type: str) -> list[str]:
|
||||
rows = conn.execute(
|
||||
"""
|
||||
SELECT symbol FROM symbols
|
||||
WHERE market_type = ? AND status = 'TRADING'
|
||||
ORDER BY symbol ASC
|
||||
""",
|
||||
(market_type,),
|
||||
).fetchall()
|
||||
return [row["symbol"] for row in rows]
|
||||
|
||||
|
||||
def insert_klines(
|
||||
conn: sqlite3.Connection,
|
||||
market_type: str,
|
||||
symbol: str,
|
||||
klines: list[dict[str, Any]],
|
||||
) -> None:
|
||||
if not klines:
|
||||
return
|
||||
conn.executemany(
|
||||
"""
|
||||
INSERT INTO klines (
|
||||
market_type, symbol, open_time, open, high, low, close, close_time, atr14
|
||||
)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, NULL)
|
||||
ON CONFLICT(market_type, symbol, open_time) DO UPDATE SET
|
||||
open = excluded.open,
|
||||
high = excluded.high,
|
||||
low = excluded.low,
|
||||
close = excluded.close,
|
||||
close_time = excluded.close_time
|
||||
""",
|
||||
[
|
||||
(
|
||||
market_type,
|
||||
symbol,
|
||||
row["open_time"],
|
||||
row["open"],
|
||||
row["high"],
|
||||
row["low"],
|
||||
row["close"],
|
||||
row["close_time"],
|
||||
)
|
||||
for row in klines
|
||||
],
|
||||
)
|
||||
conn.commit()
|
||||
|
||||
|
||||
def recompute_atr_and_signals(
|
||||
conn: sqlite3.Connection,
|
||||
config: MarketConfig,
|
||||
symbol: str,
|
||||
candidate_open_times: set[int],
|
||||
) -> int:
|
||||
rows = conn.execute(
|
||||
"""
|
||||
SELECT * FROM klines
|
||||
WHERE market_type = ? AND symbol = ?
|
||||
ORDER BY open_time ASC
|
||||
""",
|
||||
(config.market_type, symbol),
|
||||
).fetchall()
|
||||
if not rows:
|
||||
return 0
|
||||
|
||||
prev_close: Decimal | None = None
|
||||
atr: Decimal | None = None
|
||||
tr_values: list[Decimal] = []
|
||||
signal_count = 0
|
||||
|
||||
for row in rows:
|
||||
high = decimal_from_api(row["high"])
|
||||
low = decimal_from_api(row["low"])
|
||||
close = decimal_from_api(row["close"])
|
||||
open_price = decimal_from_api(row["open"])
|
||||
|
||||
if prev_close is None:
|
||||
tr = high - low
|
||||
else:
|
||||
tr = max(high - low, abs(high - prev_close), abs(low - prev_close))
|
||||
|
||||
tr_values.append(tr)
|
||||
if len(tr_values) == config.atr_length:
|
||||
atr = sum(tr_values) / Decimal(config.atr_length)
|
||||
elif len(tr_values) > config.atr_length and atr is not None:
|
||||
atr = ((atr * Decimal(config.atr_length - 1)) + tr) / Decimal(config.atr_length)
|
||||
|
||||
atr_text = str(atr) if atr is not None else None
|
||||
conn.execute(
|
||||
"""
|
||||
UPDATE klines SET atr14 = ?
|
||||
WHERE market_type = ? AND symbol = ? AND open_time = ?
|
||||
""",
|
||||
(atr_text, config.market_type, symbol, row["open_time"]),
|
||||
)
|
||||
|
||||
if atr is not None and row["open_time"] in candidate_open_times:
|
||||
candle_range = high - low
|
||||
threshold = atr * config.atr_multiple
|
||||
if candle_range >= threshold and close != open_price:
|
||||
if config.body_ratio_filter_enabled:
|
||||
body_ratio = abs(close - open_price) / candle_range if candle_range else Decimal("0")
|
||||
if body_ratio < config.min_body_ratio:
|
||||
prev_close = close
|
||||
continue
|
||||
|
||||
direction = "Bullish" if close > open_price else "Bearish"
|
||||
multiple = candle_range / atr
|
||||
conn.execute(
|
||||
"""
|
||||
INSERT INTO signals (
|
||||
market_type, symbol, direction, multiple, range, atr14, open_time, created_at
|
||||
)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
|
||||
ON CONFLICT(market_type, symbol, open_time) DO UPDATE SET
|
||||
direction = excluded.direction,
|
||||
multiple = excluded.multiple,
|
||||
range = excluded.range,
|
||||
atr14 = excluded.atr14,
|
||||
created_at = excluded.created_at
|
||||
""",
|
||||
(
|
||||
config.market_type,
|
||||
symbol,
|
||||
direction,
|
||||
str(multiple),
|
||||
str(candle_range),
|
||||
str(atr),
|
||||
row["open_time"],
|
||||
datetime.now(timezone.utc).isoformat(),
|
||||
),
|
||||
)
|
||||
signal_count += 1
|
||||
|
||||
prev_close = close
|
||||
|
||||
conn.commit()
|
||||
return signal_count
|
||||
|
||||
|
||||
def cleanup_old_data(conn: sqlite3.Connection) -> tuple[int, int]:
|
||||
deleted_klines = 0
|
||||
deleted_signals = 0
|
||||
|
||||
if KLINES_RETENTION_PER_SYMBOL > 0:
|
||||
symbols = conn.execute("SELECT DISTINCT market_type, symbol FROM klines").fetchall()
|
||||
for row in symbols:
|
||||
cursor = conn.execute(
|
||||
"""
|
||||
DELETE FROM klines
|
||||
WHERE market_type = ?
|
||||
AND symbol = ?
|
||||
AND open_time NOT IN (
|
||||
SELECT open_time FROM klines
|
||||
WHERE market_type = ?
|
||||
AND symbol = ?
|
||||
ORDER BY open_time DESC
|
||||
LIMIT ?
|
||||
)
|
||||
""",
|
||||
(
|
||||
row["market_type"],
|
||||
row["symbol"],
|
||||
row["market_type"],
|
||||
row["symbol"],
|
||||
KLINES_RETENTION_PER_SYMBOL,
|
||||
),
|
||||
)
|
||||
deleted_klines += cursor.rowcount
|
||||
|
||||
if SIGNAL_RETENTION_DAYS > 0:
|
||||
cutoff = utc_now_ms() - (SIGNAL_RETENTION_DAYS * 24 * 60 * 60 * 1000)
|
||||
cursor = conn.execute("DELETE FROM signals WHERE open_time < ?", (cutoff,))
|
||||
deleted_signals = cursor.rowcount
|
||||
|
||||
conn.commit()
|
||||
return deleted_klines, deleted_signals
|
||||
|
||||
|
||||
def insert_scan_run(
|
||||
conn: sqlite3.Connection,
|
||||
market_type: str,
|
||||
started_at: str,
|
||||
status: str,
|
||||
symbols_total: int,
|
||||
symbols_success: int,
|
||||
symbols_failed: int,
|
||||
updated_klines: int,
|
||||
signals_created: int,
|
||||
elapsed_seconds: float,
|
||||
latest_target_open_time: int | None,
|
||||
error_summary: str | None = None,
|
||||
) -> None:
|
||||
conn.execute(
|
||||
"""
|
||||
INSERT INTO scan_runs (
|
||||
market_type, started_at, finished_at, status, symbols_total, symbols_success,
|
||||
symbols_failed, updated_klines, signals_created, elapsed_seconds,
|
||||
latest_target_open_time, error_summary
|
||||
)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
""",
|
||||
(
|
||||
market_type,
|
||||
started_at,
|
||||
datetime.now(timezone.utc).isoformat(),
|
||||
status,
|
||||
symbols_total,
|
||||
symbols_success,
|
||||
symbols_failed,
|
||||
updated_klines,
|
||||
signals_created,
|
||||
f"{elapsed_seconds:.1f}",
|
||||
latest_target_open_time,
|
||||
error_summary,
|
||||
),
|
||||
)
|
||||
conn.commit()
|
||||
|
||||
|
||||
async def fetch_missing_closed_klines(
|
||||
session: aiohttp.ClientSession,
|
||||
symbol: str,
|
||||
start_open_time: int,
|
||||
expected_open_time: int,
|
||||
now_ms: int,
|
||||
strict_time_grid: bool,
|
||||
) -> list[dict[str, Any]]:
|
||||
if start_open_time > expected_open_time:
|
||||
return []
|
||||
|
||||
needed = ((expected_open_time - start_open_time) // INTERVAL_MS) + 1
|
||||
raw = await fetch_klines(
|
||||
session,
|
||||
symbol,
|
||||
limit=max(2, min(1500, needed + 2)),
|
||||
start_time=start_open_time,
|
||||
end_time=expected_open_time + INTERVAL_MS - 1,
|
||||
)
|
||||
rows = closed_klines(raw, now_ms)
|
||||
by_open = {row["open_time"]: row for row in rows}
|
||||
missing: list[dict[str, Any]] = []
|
||||
missed_slots = 0
|
||||
cursor = start_open_time
|
||||
while cursor <= expected_open_time:
|
||||
row = by_open.get(cursor)
|
||||
if row is not None:
|
||||
missing.append(row)
|
||||
else:
|
||||
missed_slots += 1
|
||||
cursor += INTERVAL_MS
|
||||
if strict_time_grid and missed_slots:
|
||||
logger.warning(
|
||||
"%s missing %s expected 4H slots between %s and %s",
|
||||
symbol,
|
||||
missed_slots,
|
||||
ms_to_iso(start_open_time),
|
||||
ms_to_iso(expected_open_time),
|
||||
)
|
||||
return missing
|
||||
|
||||
|
||||
async def process_symbol(
|
||||
session: aiohttp.ClientSession,
|
||||
conn: sqlite3.Connection,
|
||||
config: MarketConfig,
|
||||
symbol: str,
|
||||
semaphore: asyncio.Semaphore,
|
||||
now_ms: int,
|
||||
expected_open_time: int,
|
||||
) -> tuple[str, int, int]:
|
||||
async with semaphore:
|
||||
latest = latest_kline(conn, config.market_type, symbol)
|
||||
new_rows: list[dict[str, Any]]
|
||||
|
||||
if latest is None:
|
||||
raw = await fetch_klines(session, symbol, limit=INIT_KLINES_LIMIT + 1)
|
||||
new_rows = closed_klines(raw, now_ms)[-INIT_KLINES_LIMIT:]
|
||||
if config.strict_time_grid:
|
||||
candidate_open_times = {expected_open_time}
|
||||
else:
|
||||
candidate_open_times = {new_rows[-1]["open_time"]} if new_rows else set()
|
||||
else:
|
||||
latest_open_time = int(latest["open_time"])
|
||||
if latest_open_time >= expected_open_time:
|
||||
raw = await fetch_klines(session, symbol, limit=2)
|
||||
rows = closed_klines(raw, now_ms)
|
||||
new_rows = [row for row in rows if row["open_time"] > latest_open_time]
|
||||
candidate_open_times = {row["open_time"] for row in new_rows}
|
||||
else:
|
||||
next_expected = latest_open_time + INTERVAL_MS
|
||||
new_rows = await fetch_missing_closed_klines(
|
||||
session,
|
||||
symbol,
|
||||
next_expected,
|
||||
expected_open_time,
|
||||
now_ms,
|
||||
config.strict_time_grid,
|
||||
)
|
||||
candidate_open_times = {row["open_time"] for row in new_rows}
|
||||
|
||||
insert_klines(conn, config.market_type, symbol, new_rows)
|
||||
signal_count = recompute_atr_and_signals(
|
||||
conn,
|
||||
config,
|
||||
symbol,
|
||||
candidate_open_times,
|
||||
)
|
||||
return symbol, len(new_rows), signal_count
|
||||
|
||||
|
||||
async def process_symbol_safe(
|
||||
session: aiohttp.ClientSession,
|
||||
conn: sqlite3.Connection,
|
||||
config: MarketConfig,
|
||||
symbol: str,
|
||||
semaphore: asyncio.Semaphore,
|
||||
now_ms: int,
|
||||
expected_open_time: int,
|
||||
) -> tuple[str, int, int, bool]:
|
||||
try:
|
||||
symbol, inserted, signal_count = await process_symbol(
|
||||
session,
|
||||
conn,
|
||||
config,
|
||||
symbol,
|
||||
semaphore,
|
||||
now_ms,
|
||||
expected_open_time,
|
||||
)
|
||||
return symbol, inserted, signal_count, True
|
||||
except Exception as exc: # noqa: BLE001 - one failed symbol must not block the scan.
|
||||
logger.warning("%s skipped error=%s", symbol, exc)
|
||||
return symbol, 0, 0, False
|
||||
|
||||
|
||||
async def run_scan(market_type: str = MARKET_CRYPTO) -> ScanResult:
|
||||
config = MARKET_CONFIGS[market_type]
|
||||
started_perf = time.perf_counter()
|
||||
started_at = datetime.now(timezone.utc).isoformat()
|
||||
conn = db_connect()
|
||||
init_db(conn)
|
||||
symbols: list[str] = []
|
||||
expected_open_time: int | None = None
|
||||
|
||||
timeout = aiohttp.ClientTimeout(total=HTTP_TIMEOUT)
|
||||
async with aiohttp.ClientSession(timeout=timeout) as session:
|
||||
try:
|
||||
symbol_rows = await fetch_symbols(session, market_type)
|
||||
upsert_symbols(conn, market_type, symbol_rows)
|
||||
symbols = [row["symbol"] for row in symbol_rows]
|
||||
except Exception as exc: # noqa: BLE001 - fallback keeps cron useful during API hiccups.
|
||||
symbols = active_symbols_from_db(conn, market_type)
|
||||
if not symbols:
|
||||
elapsed = time.perf_counter() - started_perf
|
||||
insert_scan_run(
|
||||
conn,
|
||||
market_type=market_type,
|
||||
started_at=started_at,
|
||||
status="failed",
|
||||
symbols_total=0,
|
||||
symbols_success=0,
|
||||
symbols_failed=0,
|
||||
updated_klines=0,
|
||||
signals_created=0,
|
||||
elapsed_seconds=elapsed,
|
||||
latest_target_open_time=None,
|
||||
error_summary=f"symbol refresh failed and no cached symbols: {exc}",
|
||||
)
|
||||
logger.error(
|
||||
"Scan aborted: symbol refresh failed and database has no cached symbols elapsed=%.1fs error=%s",
|
||||
elapsed,
|
||||
exc,
|
||||
)
|
||||
raise
|
||||
logger.warning(
|
||||
"Symbol refresh failed, using %s cached symbols error=%s",
|
||||
len(symbols),
|
||||
exc,
|
||||
)
|
||||
|
||||
now_ms = utc_now_ms()
|
||||
expected_open_time = latest_theoretical_closed_open_time(now_ms)
|
||||
semaphore = asyncio.Semaphore(CONCURRENCY)
|
||||
logger.info(
|
||||
"Scanning market=%s symbols=%s latest_closed_open=%s atr_length=%s atr_multiple=%s strict_time_grid=%s",
|
||||
market_type,
|
||||
len(symbols),
|
||||
ms_to_iso(expected_open_time),
|
||||
config.atr_length,
|
||||
config.atr_multiple,
|
||||
config.strict_time_grid,
|
||||
)
|
||||
|
||||
tasks = [
|
||||
process_symbol_safe(session, conn, config, symbol, semaphore, now_ms, expected_open_time)
|
||||
for symbol in symbols
|
||||
]
|
||||
updated = 0
|
||||
signals = 0
|
||||
failures = 0
|
||||
success = 0
|
||||
for task in asyncio.as_completed(tasks):
|
||||
symbol, inserted, signal_count, ok = await task
|
||||
if not ok:
|
||||
failures += 1
|
||||
continue
|
||||
success += 1
|
||||
updated += inserted
|
||||
signals += signal_count
|
||||
if inserted:
|
||||
logger.info("%s updated_klines=%s signals=%s", symbol, inserted, signal_count)
|
||||
|
||||
elapsed = time.perf_counter() - started_perf
|
||||
status = "success" if failures == 0 else "partial"
|
||||
insert_scan_run(
|
||||
conn,
|
||||
market_type=market_type,
|
||||
started_at=started_at,
|
||||
status=status,
|
||||
symbols_total=len(symbols),
|
||||
symbols_success=success,
|
||||
symbols_failed=failures,
|
||||
updated_klines=updated,
|
||||
signals_created=signals,
|
||||
elapsed_seconds=elapsed,
|
||||
latest_target_open_time=expected_open_time,
|
||||
)
|
||||
conn.close()
|
||||
speed = len(symbols) / elapsed if elapsed > 0 else 0
|
||||
logger.info(
|
||||
"Scan done market=%s symbols=%s updated_klines=%s signals=%s failures=%s elapsed=%.1fs speed=%.1f_symbols/s",
|
||||
market_type,
|
||||
len(symbols),
|
||||
updated,
|
||||
signals,
|
||||
failures,
|
||||
elapsed,
|
||||
speed,
|
||||
)
|
||||
logger.info("%.1f seconds scan complete", elapsed)
|
||||
|
||||
cleanup_conn = db_connect()
|
||||
deleted_klines, deleted_signals = cleanup_old_data(cleanup_conn)
|
||||
cleanup_conn.close()
|
||||
if deleted_klines or deleted_signals:
|
||||
logger.info(
|
||||
"Cleanup done deleted_klines=%s deleted_signals=%s",
|
||||
deleted_klines,
|
||||
deleted_signals,
|
||||
)
|
||||
return ScanResult(
|
||||
market_type=market_type,
|
||||
label=config.label,
|
||||
status=status,
|
||||
symbols_total=len(symbols),
|
||||
symbols_failed=failures,
|
||||
updated_klines=updated,
|
||||
signals_created=signals,
|
||||
elapsed_seconds=elapsed,
|
||||
)
|
||||
|
||||
|
||||
async def send_discord_summary(results: list[ScanResult], elapsed_seconds: float) -> None:
|
||||
if not DISCORD_ENABLED:
|
||||
return
|
||||
if not DISCORD_WEBHOOK_URL:
|
||||
logger.warning("Discord notification skipped: DISCORD_WEBHOOK_URL is empty")
|
||||
return
|
||||
|
||||
madrid_time = datetime.now(timezone.utc).astimezone(ZoneInfo("Europe/Madrid"))
|
||||
status_labels = {"success": "成功", "partial": "部分失败", "failed": "失败"}
|
||||
lines = ["**Crypto ATR Signal · 4H 扫描完成**", ""]
|
||||
for result in results:
|
||||
market_name = "Crypto" if result.market_type == MARKET_CRYPTO else "TradFi"
|
||||
status = status_labels.get(result.status, result.status)
|
||||
lines.append(
|
||||
f"{market_name}：{result.signals_created} 个信号 · {status} · "
|
||||
f"失败 {result.symbols_failed}"
|
||||
)
|
||||
lines.extend(
|
||||
[
|
||||
f"合计：{sum(result.signals_created for result in results)} 个信号",
|
||||
f"耗时：{elapsed_seconds:.1f} 秒",
|
||||
f"时间：{madrid_time.strftime('%Y-%m-%d %H:%M')} 马德里",
|
||||
]
|
||||
)
|
||||
payload = {"content": "\n".join(lines), "allowed_mentions": {"parse": []}}
|
||||
timeout = aiohttp.ClientTimeout(total=HTTP_TIMEOUT)
|
||||
try:
|
||||
async with aiohttp.ClientSession(timeout=timeout) as session:
|
||||
async with session.post(DISCORD_WEBHOOK_URL, json=payload) as response:
|
||||
if response.status not in {200, 204}:
|
||||
body = await response.text()
|
||||
logger.warning(
|
||||
"Discord notification failed status=%s body=%s",
|
||||
response.status,
|
||||
body[:300],
|
||||
)
|
||||
return
|
||||
logger.info("Discord aggregate notification sent")
|
||||
except Exception as exc: # noqa: BLE001 - notification must never fail the scan.
|
||||
logger.warning("Discord notification skipped error=%s", exc)
|
||||
|
||||
|
||||
async def run_all_markets() -> list[ScanResult]:
|
||||
started = time.perf_counter()
|
||||
results: list[ScanResult] = []
|
||||
for market_type in (MARKET_CRYPTO, MARKET_TRADFI):
|
||||
try:
|
||||
results.append(await run_scan(market_type))
|
||||
except Exception as exc: # noqa: BLE001 - one market must not block the other.
|
||||
config = MARKET_CONFIGS[market_type]
|
||||
logger.error("Market scan failed market=%s error=%s", market_type, exc)
|
||||
results.append(
|
||||
ScanResult(
|
||||
market_type=market_type,
|
||||
label=config.label,
|
||||
status="failed",
|
||||
symbols_total=0,
|
||||
symbols_failed=0,
|
||||
updated_klines=0,
|
||||
signals_created=0,
|
||||
elapsed_seconds=0,
|
||||
error_summary=str(exc),
|
||||
)
|
||||
)
|
||||
await send_discord_summary(results, time.perf_counter() - started)
|
||||
return results
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser(description="Scan Binance ATR signals")
|
||||
parser.add_argument(
|
||||
"--market",
|
||||
choices=["crypto", "tradfi", "all"],
|
||||
default="crypto",
|
||||
help="Market to scan: crypto, tradfi, or all",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
if args.market == "all":
|
||||
asyncio.run(run_all_markets())
|
||||
else:
|
||||
selected_market = MARKET_TRADFI if args.market == "tradfi" else MARKET_CRYPTO
|
||||
asyncio.run(run_scan(selected_market))
|
||||
Reference in New Issue
Block a user