kyu
6d3b0bacc0
- 19개 마이크로서비스 (news-collector, score-engine, ta-engine, dart-collector, aux-signal, us-market, graph-engine, telegram-bot, dashboard-api, kis-api 등) - 가치투자 스코어링 + 10공식 앙상블 보팅 (매직포뮬러/F-Score/Altman/PEG/ 모멘텀/Beneish/GP-A/G-Score/Amihud/BAB) - 뉴스 수집→형태소→임베딩→중복제거→AI분석 파이프라인 - 기술적분석 + GAT 그래프신경망 + 미증시 동조 시그널 Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
839 lines
37 KiB
Python
839 lines
37 KiB
Python
"""
|
||
US Market Sync Service (port 8383, 172.30.0.24)
|
||
|
||
미국증시 → 한국증시 동조 시그널 생성:
|
||
A) 섹터 ETF 동조 (SOXX, XBI, LIT 등 14개) → 한국 같은 섹터에 ±5점
|
||
B) 개별 페어 (NVDA↔SK하이닉스 등) 60일 회귀 베타 → ±10점
|
||
D) 자동 페어 발굴 (코스피200 × S&P500 60일 상관계수)
|
||
|
||
매일 KST 07:30 미국 정규장 마감 후 수집, 08:00 시그널 계산.
|
||
"""
|
||
import os
|
||
import asyncio
|
||
import json
|
||
from datetime import date, datetime, timedelta
|
||
from typing import Optional, List, Dict
|
||
|
||
import asyncpg
|
||
import orjson
|
||
import structlog
|
||
from fastapi import FastAPI, Query, BackgroundTasks
|
||
from apscheduler.schedulers.asyncio import AsyncIOScheduler
|
||
from apscheduler.triggers.cron import CronTrigger
|
||
from pytz import timezone
|
||
|
||
import httpx
|
||
import pandas as pd
|
||
import numpy as np
|
||
from scipy import stats
|
||
|
||
# ─────────────────────────────────────────────────────────────
|
||
# 설정
|
||
# ─────────────────────────────────────────────────────────────
|
||
PG = {
|
||
"host": os.getenv("POSTGRES_HOST", "postgres"),
|
||
"port": int(os.getenv("POSTGRES_PORT", 5432)),
|
||
"database": os.getenv("POSTGRES_DB", "trading_ai"),
|
||
"user": os.getenv("POSTGRES_USER", "kyu"),
|
||
"password": os.getenv("POSTGRES_PASSWORD", ""),
|
||
}
|
||
KST = timezone("Asia/Seoul")
|
||
FINNHUB_KEY = os.getenv("FINNHUB_API_KEY", "")
|
||
FINNHUB_BASE = "https://finnhub.io/api/v1"
|
||
AV_KEY = os.getenv("ALPHAVANTAGE_API_KEY", "")
|
||
AV_BASE = "https://www.alphavantage.co/query"
|
||
AV_DAILY_LIMIT = 25 # free tier: 25 calls/day
|
||
|
||
logger = structlog.get_logger()
|
||
app = FastAPI(title="US Market Sync")
|
||
pg_pool: Optional[asyncpg.Pool] = None
|
||
scheduler = AsyncIOScheduler(timezone=KST)
|
||
|
||
# ─────────────────────────────────────────────────────────────
|
||
# ETF → 한국 섹터 키워드 매핑 (dart_corps.sector LIKE 매칭용)
|
||
# 섹터 컬럼은 KSIC 한글 (예: "반도체 및 전자집적회로 제조업")
|
||
# ─────────────────────────────────────────────────────────────
|
||
SECTOR_ETFS: Dict[str, Dict] = {
|
||
"SOXX": {"keywords": ["반도체", "전자집적", "전자부품"],
|
||
"desc": "iShares Semiconductor"},
|
||
"SMH": {"keywords": ["반도체", "전자집적"],
|
||
"desc": "VanEck Semiconductor"},
|
||
"XLK": {"keywords": ["소프트웨어", "정보서비스", "컴퓨터"],
|
||
"desc": "Tech Select Sector"},
|
||
"QQQ": {"keywords": ["소프트웨어", "인터넷", "포털"],
|
||
"desc": "Nasdaq 100"},
|
||
"XBI": {"keywords": ["바이오", "생물의약", "의약품"],
|
||
"desc": "S&P Biotech"},
|
||
"IBB": {"keywords": ["바이오", "생물의약"],
|
||
"desc": "Nasdaq Biotech"},
|
||
"LIT": {"keywords": ["전지", "축전지", "이차전지"],
|
||
"desc": "Global Lithium"},
|
||
"XLE": {"keywords": ["석유", "정제", "가스", "원유"],
|
||
"desc": "Energy Select"},
|
||
"XLF": {"keywords": ["은행", "보험", "증권", "금융"],
|
||
"desc": "Financial Select"},
|
||
"XLV": {"keywords": ["의료", "병원", "의료용기기"],
|
||
"desc": "Health Care Select"},
|
||
"XLI": {"keywords": ["기계", "산업용", "건설"],
|
||
"desc": "Industrials Select"},
|
||
"XLP": {"keywords": ["식품", "음료", "가공식품"],
|
||
"desc": "Consumer Staples"},
|
||
"XLY": {"keywords": ["자동차", "여가", "의류", "백화점"],
|
||
"desc": "Consumer Discretionary"},
|
||
"ITA": {"keywords": ["항공", "방위", "조선"],
|
||
"desc": "Aerospace & Defense"},
|
||
}
|
||
|
||
# ─────────────────────────────────────────────────────────────
|
||
# 검증된 핵심 페어 (seed) — 학술/시장 통념상 강한 동조
|
||
# (us_ticker, kr_code, kr_name_hint) kr_name_hint는 검증용 코멘트
|
||
# ─────────────────────────────────────────────────────────────
|
||
SEED_PAIRS: List = [
|
||
# 반도체
|
||
("NVDA", "000660"), # SK하이닉스
|
||
("NVDA", "005930"), # 삼성전자
|
||
("AMD", "000660"),
|
||
("AMD", "005930"),
|
||
("MU", "000660"),
|
||
("INTC", "005930"),
|
||
("TSM", "000660"),
|
||
("TSM", "005930"),
|
||
# 2차전지
|
||
("TSLA", "373220"), # LG에너지솔루션
|
||
("TSLA", "247540"), # 에코프로비엠
|
||
("TSLA", "006400"), # 삼성SDI
|
||
("ALB", "006400"), # 알버말 (리튬)
|
||
("ALB", "373220"),
|
||
# 자동차
|
||
("F", "005380"), # 현대차
|
||
("GM", "000270"), # 기아
|
||
("TSLA", "012330"), # 현대모비스
|
||
("TM", "005380"), # 도요타→현대차
|
||
# 인터넷/IT
|
||
("GOOGL","035420"), # NAVER
|
||
("META", "035720"), # 카카오
|
||
("AAPL", "011070"), # LG이노텍
|
||
("AAPL", "009150"), # 삼성전기
|
||
# 바이오/제약
|
||
("PFE", "068270"), # 셀트리온
|
||
("MRK", "207940"), # 삼성바이오로직스
|
||
# 조선/방산
|
||
("LMT", "079550"), # LIG넥스원
|
||
("LMT", "329180"), # HD현대중공업
|
||
# 철강/소재
|
||
("NUE", "005490"), # POSCO홀딩스
|
||
# 화학
|
||
("DOW", "051910"), # LG화학
|
||
("LYB", "011170"), # 롯데케미칼
|
||
# 게임/엔터
|
||
("NTES", "036570"), # 엔씨소프트
|
||
("NTES", "251270"), # 넷마블
|
||
("DIS", "035250"), # 강원랜드 (엔터)
|
||
# 유통
|
||
("AMZN", "139480"), # 이마트
|
||
]
|
||
|
||
# 자동 발굴용 미국 종목 후보 (S&P500 대표 + 한국 영향 큰 종목)
|
||
DISCOVERY_US_TICKERS = [
|
||
# 반도체
|
||
"NVDA", "AMD", "MU", "INTC", "TSM", "AVGO", "QCOM", "TXN", "AMAT", "LRCX", "KLAC",
|
||
# 빅테크
|
||
"AAPL", "MSFT", "GOOGL", "AMZN", "META", "TSLA", "NFLX", "ORCL", "ADBE", "CRM",
|
||
# 금융
|
||
"JPM", "BAC", "GS", "MS", "WFC", "C", "BLK",
|
||
# 에너지/소재
|
||
"XOM", "CVX", "COP", "NUE", "FCX",
|
||
# 헬스/바이오
|
||
"JNJ", "PFE", "MRK", "ABBV", "LLY", "BMY", "GILD",
|
||
# 소비재/유통
|
||
"WMT", "COST", "HD", "MCD", "NKE", "SBUX", "DIS",
|
||
# 산업/방산
|
||
"BA", "LMT", "RTX", "GD", "CAT", "DE",
|
||
# 자동차
|
||
"F", "GM", "TM",
|
||
# 2차전지/리튬
|
||
"ALB", "RIVN",
|
||
# 화학
|
||
"DOW", "LYB",
|
||
]
|
||
|
||
# ─────────────────────────────────────────────────────────────
|
||
# DB 초기화
|
||
# ─────────────────────────────────────────────────────────────
|
||
DDL = """
|
||
CREATE TABLE IF NOT EXISTS us_market_daily (
|
||
ticker VARCHAR(20),
|
||
trade_date DATE,
|
||
open_price DOUBLE PRECISION,
|
||
close_price DOUBLE PRECISION,
|
||
prev_close DOUBLE PRECISION,
|
||
change_pct DOUBLE PRECISION,
|
||
volume BIGINT,
|
||
created_at TIMESTAMP DEFAULT NOW(),
|
||
PRIMARY KEY (ticker, trade_date)
|
||
);
|
||
CREATE INDEX IF NOT EXISTS idx_us_daily_ticker ON us_market_daily(ticker, trade_date DESC);
|
||
|
||
CREATE TABLE IF NOT EXISTS us_sector_etf_map (
|
||
etf_ticker VARCHAR(20) PRIMARY KEY,
|
||
sector_keywords TEXT[],
|
||
description TEXT,
|
||
updated_at TIMESTAMP DEFAULT NOW()
|
||
);
|
||
|
||
CREATE TABLE IF NOT EXISTS us_kr_pairs (
|
||
us_ticker VARCHAR(20),
|
||
kr_code VARCHAR(10),
|
||
beta_60d DOUBLE PRECISION,
|
||
correlation_60d DOUBLE PRECISION,
|
||
sample_size INTEGER,
|
||
source VARCHAR(20) DEFAULT 'seed',
|
||
updated_at TIMESTAMP DEFAULT NOW(),
|
||
PRIMARY KEY (us_ticker, kr_code)
|
||
);
|
||
CREATE INDEX IF NOT EXISTS idx_pairs_kr ON us_kr_pairs(kr_code);
|
||
|
||
CREATE TABLE IF NOT EXISTS us_overnight_signal (
|
||
kr_code VARCHAR(10),
|
||
signal_date DATE,
|
||
sector_adj DOUBLE PRECISION DEFAULT 0,
|
||
pair_adj DOUBLE PRECISION DEFAULT 0,
|
||
total_adj DOUBLE PRECISION DEFAULT 0,
|
||
contributing_pairs JSONB,
|
||
created_at TIMESTAMP DEFAULT NOW(),
|
||
PRIMARY KEY (kr_code, signal_date)
|
||
);
|
||
CREATE INDEX IF NOT EXISTS idx_overnight_date ON us_overnight_signal(signal_date DESC);
|
||
"""
|
||
|
||
# ─────────────────────────────────────────────────────────────
|
||
# 시작/종료
|
||
# ─────────────────────────────────────────────────────────────
|
||
@app.on_event("startup")
|
||
async def on_start():
|
||
global pg_pool
|
||
pg_pool = await asyncpg.create_pool(**PG, min_size=2, max_size=10)
|
||
async with pg_pool.acquire() as conn:
|
||
await conn.execute(DDL)
|
||
await seed_etfs_and_pairs()
|
||
|
||
scheduler.add_job(
|
||
collect_us_daily, CronTrigger(hour=7, minute=30),
|
||
id="us_collect", replace_existing=True)
|
||
scheduler.add_job(
|
||
calc_overnight_signals_all, CronTrigger(hour=8, minute=0),
|
||
id="overnight_calc", replace_existing=True)
|
||
scheduler.add_job(
|
||
recalc_pair_betas, CronTrigger(day_of_week="sun", hour=2, minute=0),
|
||
id="pair_beta", replace_existing=True)
|
||
scheduler.add_job(
|
||
discover_new_pairs, CronTrigger(day_of_week="sun", hour=3, minute=0),
|
||
id="pair_discover", replace_existing=True)
|
||
# 매일 06:00 Alpha Vantage 백필 (25 ticker/일, 60일+ 채워질 때까지)
|
||
scheduler.add_job(
|
||
backfill_yfinance, CronTrigger(hour=6, minute=0),
|
||
id="av_backfill", replace_existing=True)
|
||
scheduler.start()
|
||
logger.info("us-market.started")
|
||
|
||
|
||
@app.on_event("shutdown")
|
||
async def on_stop():
|
||
if scheduler.running:
|
||
scheduler.shutdown()
|
||
if pg_pool:
|
||
await pg_pool.close()
|
||
|
||
|
||
# ─────────────────────────────────────────────────────────────
|
||
# Seed: ETF + 핵심 페어 등록
|
||
# ─────────────────────────────────────────────────────────────
|
||
async def seed_etfs_and_pairs():
|
||
async with pg_pool.acquire() as conn:
|
||
for etf, meta in SECTOR_ETFS.items():
|
||
await conn.execute("""
|
||
INSERT INTO us_sector_etf_map (etf_ticker, sector_keywords, description)
|
||
VALUES ($1, $2, $3)
|
||
ON CONFLICT (etf_ticker) DO UPDATE
|
||
SET sector_keywords=$2, description=$3, updated_at=NOW()
|
||
""", etf, meta["keywords"], meta["desc"])
|
||
for us, kr in SEED_PAIRS:
|
||
await conn.execute("""
|
||
INSERT INTO us_kr_pairs (us_ticker, kr_code, source)
|
||
VALUES ($1, $2, 'seed')
|
||
ON CONFLICT (us_ticker, kr_code) DO NOTHING
|
||
""", us, kr)
|
||
logger.info("seed.done", etfs=len(SECTOR_ETFS), pairs=len(SEED_PAIRS))
|
||
|
||
|
||
# ─────────────────────────────────────────────────────────────
|
||
# Finnhub 헬퍼 — /quote (무료 OK, /stock/candle은 2024년부터 유료)
|
||
# 응답: c(current), pc(prev_close), dp(percent), o(open), h, l, t
|
||
# Free tier: 60 calls/분
|
||
# ─────────────────────────────────────────────────────────────
|
||
async def fetch_finnhub_quote(client: httpx.AsyncClient, ticker: str
|
||
) -> Optional[dict]:
|
||
"""Finnhub /quote 호출. 실패 시 None."""
|
||
if not FINNHUB_KEY:
|
||
return None
|
||
try:
|
||
r = await client.get(f"{FINNHUB_BASE}/quote", params={
|
||
"symbol": ticker, "token": FINNHUB_KEY,
|
||
}, timeout=15)
|
||
except Exception as e:
|
||
logger.warning("finnhub.req_err", ticker=ticker, err=str(e))
|
||
return None
|
||
if r.status_code == 429:
|
||
await asyncio.sleep(5)
|
||
return None
|
||
if r.status_code != 200:
|
||
return None
|
||
try:
|
||
j = r.json()
|
||
except Exception:
|
||
return None
|
||
# 휴장/잘못된 ticker 시 c=0
|
||
if not j or j.get("c", 0) <= 0:
|
||
return None
|
||
return j
|
||
|
||
|
||
# ─────────────────────────────────────────────────────────────
|
||
# 수집: /quote으로 미국 일간 종가 누적
|
||
# 매일 호출하면 us_market_daily에 시계열 점진 누적 → 베타 학습 가능
|
||
# ─────────────────────────────────────────────────────────────
|
||
async def collect_us_daily(days: int = 1):
|
||
"""매일 KST 07:30 호출. days 인자는 호환용(무시) — /quote은 단일 시점."""
|
||
if not FINNHUB_KEY:
|
||
logger.error("us.no_api_key")
|
||
return {"saved": 0, "err": "FINNHUB_API_KEY missing — set in .env"}
|
||
|
||
tickers = sorted(
|
||
set(SECTOR_ETFS.keys())
|
||
| {t for t, _ in SEED_PAIRS}
|
||
| set(DISCOVERY_US_TICKERS)
|
||
)
|
||
saved = 0
|
||
failed: List[str] = []
|
||
async with httpx.AsyncClient() as client:
|
||
for i, ticker in enumerate(tickers):
|
||
q = await fetch_finnhub_quote(client, ticker)
|
||
if not q:
|
||
failed.append(ticker)
|
||
else:
|
||
# t 타임스탬프(미국 ET 종가 시점) → trade_date
|
||
ts = q.get("t", 0)
|
||
trade_dt = (datetime.fromtimestamp(ts).date() if ts
|
||
else date.today() - timedelta(days=1))
|
||
async with pg_pool.acquire() as conn:
|
||
await conn.execute("""
|
||
INSERT INTO us_market_daily
|
||
(ticker, trade_date, open_price, close_price,
|
||
prev_close, change_pct, volume)
|
||
VALUES ($1, $2, $3, $4, $5, $6, $7)
|
||
ON CONFLICT (ticker, trade_date) DO UPDATE
|
||
SET open_price=$3, close_price=$4, prev_close=$5,
|
||
change_pct=$6
|
||
""", ticker, trade_dt,
|
||
float(q.get("o", 0) or 0), float(q["c"]),
|
||
float(q.get("pc", 0) or 0), float(q.get("dp", 0) or 0),
|
||
0)
|
||
saved += 1
|
||
# Rate limit: 60/분 → 1.1초 간격
|
||
if i < len(tickers) - 1:
|
||
await asyncio.sleep(1.1)
|
||
logger.info("us.collected", rows=saved, ok=len(tickers) - len(failed),
|
||
failed=len(failed))
|
||
return {"saved": saved, "tickers": len(tickers),
|
||
"ok": len(tickers) - len(failed), "failed": failed[:10]}
|
||
|
||
|
||
# ─────────────────────────────────────────────────────────────
|
||
# Alpha Vantage 백필 — TIME_SERIES_DAILY로 ticker당 100일 히스토리.
|
||
# Free tier 25 calls/day → 자동으로 매일 25개씩 분할 처리 (75 ticker × 3일).
|
||
# ─────────────────────────────────────────────────────────────
|
||
async def fetch_av_daily(client: httpx.AsyncClient, ticker: str
|
||
) -> Optional[List[dict]]:
|
||
"""Alpha Vantage TIME_SERIES_DAILY 호출. 실패/한도초과 시 None."""
|
||
try:
|
||
r = await client.get(AV_BASE, params={
|
||
"function": "TIME_SERIES_DAILY",
|
||
"symbol": ticker,
|
||
"outputsize": "compact", # 100일치 (full=20+년)
|
||
"apikey": AV_KEY,
|
||
}, timeout=20)
|
||
except Exception as e:
|
||
logger.warning("av.req_err", ticker=ticker, err=str(e))
|
||
return None
|
||
if r.status_code != 200:
|
||
return None
|
||
try:
|
||
j = r.json()
|
||
except Exception:
|
||
return None
|
||
if "Note" in j or "Information" in j or "Error Message" in j:
|
||
logger.warning("av.limit_or_err",
|
||
ticker=ticker, msg=str(j)[:200])
|
||
return None
|
||
ts = j.get("Time Series (Daily)")
|
||
if not ts:
|
||
return None
|
||
rows = []
|
||
for dt_str, ohlcv in ts.items():
|
||
try:
|
||
rows.append({
|
||
"trade_date": date.fromisoformat(dt_str),
|
||
"open": float(ohlcv["1. open"]),
|
||
"high": float(ohlcv["2. high"]),
|
||
"low": float(ohlcv["3. low"]),
|
||
"close": float(ohlcv["4. close"]),
|
||
"volume": int(float(ohlcv["5. volume"])),
|
||
})
|
||
except (KeyError, ValueError):
|
||
continue
|
||
rows.sort(key=lambda x: x["trade_date"])
|
||
return rows
|
||
|
||
|
||
async def backfill_yfinance(days: int = 180, max_tickers: int = 0):
|
||
"""Alpha Vantage로 히스토리 백필.
|
||
- max_tickers=0 (기본): AV_DAILY_LIMIT(=25)개만 처리 → 일일 한도 자동 준수
|
||
- max_tickers>0: 명시값 사용
|
||
- 이미 60일+ 데이터 있는 ticker는 건너뜀 → 3일치 분산 자동 진행
|
||
"""
|
||
if not AV_KEY:
|
||
return {"saved": 0, "err": "ALPHAVANTAGE_API_KEY missing — set in .env"}
|
||
all_tickers = sorted(
|
||
set(SECTOR_ETFS.keys())
|
||
| {t for t, _ in SEED_PAIRS}
|
||
| set(DISCOVERY_US_TICKERS)
|
||
)
|
||
# 이미 60일+ 누적된 ticker는 스킵
|
||
async with pg_pool.acquire() as conn:
|
||
rows = await conn.fetch(
|
||
"SELECT ticker, COUNT(*) AS n FROM us_market_daily "
|
||
"GROUP BY ticker HAVING COUNT(*) >= 60")
|
||
done = {r["ticker"] for r in rows}
|
||
pending = [t for t in all_tickers if t not in done]
|
||
limit = max_tickers if max_tickers > 0 else AV_DAILY_LIMIT
|
||
targets = pending[:limit]
|
||
|
||
saved = 0
|
||
failed: List[str] = []
|
||
async with httpx.AsyncClient() as client:
|
||
for i, ticker in enumerate(targets):
|
||
rows = await fetch_av_daily(client, ticker)
|
||
if not rows:
|
||
failed.append(ticker)
|
||
# 한도 초과면 즉시 중단
|
||
if i > 0 and len(failed) > 3 and len(failed) > i // 2:
|
||
logger.warning("av.likely_quota", processed=i)
|
||
break
|
||
else:
|
||
prev_close = None
|
||
async with pg_pool.acquire() as conn:
|
||
async with conn.transaction():
|
||
for row in rows:
|
||
pc = prev_close if prev_close is not None else 0.0
|
||
dp = ((row["close"] - pc) / pc * 100.0) if pc > 0 else 0.0
|
||
await conn.execute("""
|
||
INSERT INTO us_market_daily
|
||
(ticker, trade_date, open_price, close_price,
|
||
prev_close, change_pct, volume)
|
||
VALUES ($1, $2, $3, $4, $5, $6, $7)
|
||
ON CONFLICT (ticker, trade_date) DO UPDATE
|
||
SET open_price=$3, close_price=$4, prev_close=$5,
|
||
change_pct=$6, volume=$7
|
||
""", ticker, row["trade_date"], row["open"],
|
||
row["close"], pc, dp, row["volume"])
|
||
saved += 1
|
||
prev_close = row["close"]
|
||
# AV free tier 5 calls/min → 12초 간격
|
||
if i < len(targets) - 1:
|
||
await asyncio.sleep(12.5)
|
||
|
||
logger.info("us.av_backfill", saved=saved,
|
||
processed=len(targets) - len(failed),
|
||
failed=len(failed),
|
||
pending_remaining=len(pending) - len(targets) + len(failed))
|
||
return {"saved": saved, "processed": len(targets),
|
||
"ok": len(targets) - len(failed),
|
||
"failed": failed[:10],
|
||
"pending_after": max(0, len(pending) - len(targets) + len(failed)),
|
||
"source": "alphavantage"}
|
||
|
||
|
||
# ─────────────────────────────────────────────────────────────
|
||
# 시그널 계산: 한국 종목별 overnight 보정 점수
|
||
# ─────────────────────────────────────────────────────────────
|
||
async def calc_overnight_signals_all(target_date: Optional[date] = None):
|
||
"""오늘 자(=어제 미국장 마감) 보정 점수 계산.
|
||
|
||
A. 섹터 ETF 동조: ETF change_pct → 같은 sector 한국 종목에 ±5점
|
||
B. 페어 동조: 페어별 미국주 change_pct × beta → ±10점 (집계)
|
||
"""
|
||
target = target_date or date.today()
|
||
async with pg_pool.acquire() as conn:
|
||
# 1) 최신 미국 거래일의 ETF/주식 change_pct
|
||
us_rows = await conn.fetch("""
|
||
SELECT DISTINCT ON (ticker)
|
||
ticker, trade_date, change_pct
|
||
FROM us_market_daily
|
||
WHERE trade_date <= $1 AND change_pct IS NOT NULL
|
||
ORDER BY ticker, trade_date DESC
|
||
""", target)
|
||
us_chg = {r["ticker"]: float(r["change_pct"]) for r in us_rows}
|
||
if not us_chg:
|
||
logger.warning("overnight.no_us_data")
|
||
return {"saved": 0, "err": "no us data"}
|
||
|
||
# 2) 활성 한국 종목 + 섹터
|
||
kr_rows = await conn.fetch("""
|
||
SELECT stock_code, sector FROM dart_corps WHERE is_active=true
|
||
""")
|
||
|
||
# 3) 페어 매핑 (us_ticker → list of (kr_code, beta))
|
||
pair_rows = await conn.fetch("""
|
||
SELECT us_ticker, kr_code, beta_60d, correlation_60d
|
||
FROM us_kr_pairs
|
||
""")
|
||
pairs_by_kr: Dict[str, List] = {}
|
||
for r in pair_rows:
|
||
kr = r["kr_code"]
|
||
pairs_by_kr.setdefault(kr, []).append({
|
||
"us": r["us_ticker"],
|
||
"beta": float(r["beta_60d"]) if r["beta_60d"] else 1.0,
|
||
"corr": float(r["correlation_60d"]) if r["correlation_60d"] else 0.0,
|
||
})
|
||
|
||
# 4) ETF 매핑
|
||
etf_rows = await conn.fetch("SELECT etf_ticker, sector_keywords FROM us_sector_etf_map")
|
||
|
||
saved = 0
|
||
for kr in kr_rows:
|
||
code = kr["stock_code"]
|
||
sector = (kr["sector"] or "")
|
||
|
||
# A. 섹터 ETF 동조
|
||
sector_adj = 0.0
|
||
matched_etfs = []
|
||
for er in etf_rows:
|
||
kws = er["sector_keywords"] or []
|
||
if not sector or not kws:
|
||
continue
|
||
if any(kw and kw in sector for kw in kws):
|
||
pct = us_chg.get(er["etf_ticker"])
|
||
if pct is None:
|
||
continue
|
||
matched_etfs.append({"etf": er["etf_ticker"], "pct": pct})
|
||
if matched_etfs:
|
||
# 매칭된 ETF 평균 변동률 → ±5 클램프
|
||
avg_pct = sum(m["pct"] for m in matched_etfs) / len(matched_etfs)
|
||
sector_adj = max(-5.0, min(5.0, avg_pct * 1.5))
|
||
|
||
# B. 페어 베타 기반
|
||
pair_adj = 0.0
|
||
contributing = []
|
||
kr_pairs = pairs_by_kr.get(code, [])
|
||
for p in kr_pairs:
|
||
pct = us_chg.get(p["us"])
|
||
if pct is None:
|
||
continue
|
||
# 예상 갭 = 미국주 변동률 × 베타
|
||
exp_gap = pct * p["beta"]
|
||
# 상관계수 가중 (|corr|이 낮으면 신뢰도 ↓)
|
||
weight = max(0.3, abs(p["corr"])) if p["corr"] else 0.5
|
||
contrib = exp_gap * weight
|
||
pair_adj += contrib
|
||
contributing.append({
|
||
"us": p["us"], "pct": round(pct, 2),
|
||
"beta": round(p["beta"], 2),
|
||
"corr": round(p["corr"], 2),
|
||
"contribution": round(contrib, 2),
|
||
})
|
||
if contributing:
|
||
# 다중 페어 평균 + 클램프
|
||
pair_adj = max(-10.0, min(10.0, pair_adj / len(contributing) * 2.0))
|
||
|
||
total_adj = sector_adj + pair_adj
|
||
if abs(total_adj) < 0.1 and not matched_etfs and not contributing:
|
||
continue # 영향 없는 종목은 저장 스킵
|
||
|
||
await conn.execute("""
|
||
INSERT INTO us_overnight_signal
|
||
(kr_code, signal_date, sector_adj, pair_adj, total_adj, contributing_pairs)
|
||
VALUES ($1, $2, $3, $4, $5, $6)
|
||
ON CONFLICT (kr_code, signal_date) DO UPDATE
|
||
SET sector_adj=$3, pair_adj=$4, total_adj=$5,
|
||
contributing_pairs=$6, created_at=NOW()
|
||
""", code, target, sector_adj, pair_adj, total_adj,
|
||
json.dumps({"etfs": matched_etfs, "pairs": contributing}))
|
||
saved += 1
|
||
logger.info("overnight.calculated", saved=saved, date=str(target))
|
||
return {"saved": saved, "date": str(target)}
|
||
|
||
|
||
# ─────────────────────────────────────────────────────────────
|
||
# 페어 베타 재계산 (주 1회 일요일)
|
||
# ─────────────────────────────────────────────────────────────
|
||
async def recalc_pair_betas(window_days: int = 60):
|
||
"""등록된 페어에 대해 최근 N일 일간수익률로 회귀 → beta, correlation 갱신."""
|
||
since = date.today() - timedelta(days=window_days * 2) # 거래일 여유
|
||
|
||
async with pg_pool.acquire() as conn:
|
||
pairs = await conn.fetch("SELECT us_ticker, kr_code FROM us_kr_pairs")
|
||
updated, skipped = 0, 0
|
||
for p in pairs:
|
||
us_t, kr_c = p["us_ticker"], p["kr_code"]
|
||
# 미국 시계열
|
||
us_rows = await conn.fetch("""
|
||
SELECT trade_date, close_price FROM us_market_daily
|
||
WHERE ticker=$1 AND trade_date >= $2 ORDER BY trade_date
|
||
""", us_t, since)
|
||
if len(us_rows) < 30:
|
||
skipped += 1
|
||
continue
|
||
# 한국 시계열 (stock_prices.collected_at)
|
||
kr_rows = await conn.fetch("""
|
||
SELECT collected_at::date AS dt,
|
||
AVG(price)::float AS close
|
||
FROM stock_prices
|
||
WHERE stock_code=$1 AND collected_at::date >= $2
|
||
GROUP BY collected_at::date ORDER BY dt
|
||
""", kr_c, since)
|
||
if len(kr_rows) < 30:
|
||
skipped += 1
|
||
continue
|
||
|
||
us_df = pd.DataFrame([(r["trade_date"], r["close_price"]) for r in us_rows],
|
||
columns=["dt", "us"])
|
||
kr_df = pd.DataFrame([(r["dt"], r["close"]) for r in kr_rows],
|
||
columns=["dt", "kr"])
|
||
us_df["dt"] = pd.to_datetime(us_df["dt"])
|
||
kr_df["dt"] = pd.to_datetime(kr_df["dt"])
|
||
|
||
# 한국 종가는 미국 다음날에 영향 받음 → 미국 시계열을 +1일 시프트해서 매칭
|
||
us_df["dt"] = us_df["dt"] + pd.Timedelta(days=1)
|
||
merged = pd.merge(us_df, kr_df, on="dt", how="inner")
|
||
if len(merged) < window_days // 2:
|
||
skipped += 1
|
||
continue
|
||
|
||
us_ret = merged["us"].pct_change().dropna()
|
||
kr_ret = merged["kr"].pct_change().dropna()
|
||
n = min(len(us_ret), len(kr_ret))
|
||
if n < 20:
|
||
skipped += 1
|
||
continue
|
||
us_ret, kr_ret = us_ret.iloc[-n:].values, kr_ret.iloc[-n:].values
|
||
|
||
# 회귀: kr_ret = beta * us_ret + intercept
|
||
slope, _, r_val, _, _ = stats.linregress(us_ret, kr_ret)
|
||
beta = float(slope)
|
||
corr = float(r_val)
|
||
await conn.execute("""
|
||
UPDATE us_kr_pairs
|
||
SET beta_60d=$1, correlation_60d=$2, sample_size=$3, updated_at=NOW()
|
||
WHERE us_ticker=$4 AND kr_code=$5
|
||
""", beta, corr, n, us_t, kr_c)
|
||
updated += 1
|
||
logger.info("pair_beta.recalc", updated=updated, skipped=skipped)
|
||
return {"updated": updated, "skipped": skipped}
|
||
|
||
|
||
# ─────────────────────────────────────────────────────────────
|
||
# 자동 페어 발굴 (월 1회)
|
||
# 코스피200 시총상위 50개 × DISCOVERY_US 후보 → |corr|≥0.5인 것만 등록
|
||
# ─────────────────────────────────────────────────────────────
|
||
async def discover_new_pairs(min_abs_corr: float = 0.5, top_kr: int = 50,
|
||
window_days: int = 60):
|
||
since = date.today() - timedelta(days=window_days * 2)
|
||
async with pg_pool.acquire() as conn:
|
||
# 시총상위 한국 종목
|
||
top_rows = await conn.fetch("""
|
||
SELECT DISTINCT ON (stock_code) stock_code, market_cap
|
||
FROM stock_prices
|
||
WHERE collected_at::date >= CURRENT_DATE - INTERVAL '7 days'
|
||
AND market_cap > 0
|
||
ORDER BY stock_code, collected_at DESC
|
||
""")
|
||
kr_top = sorted(top_rows, key=lambda r: -(r["market_cap"] or 0))[:top_kr]
|
||
added = 0
|
||
for kr in kr_top:
|
||
kr_c = kr["stock_code"]
|
||
kr_rows = await conn.fetch("""
|
||
SELECT collected_at::date AS dt, AVG(price)::float AS close
|
||
FROM stock_prices
|
||
WHERE stock_code=$1 AND collected_at::date >= $2
|
||
GROUP BY collected_at::date ORDER BY dt
|
||
""", kr_c, since)
|
||
if len(kr_rows) < 30:
|
||
continue
|
||
kr_df = pd.DataFrame([(r["dt"], r["close"]) for r in kr_rows],
|
||
columns=["dt", "kr"])
|
||
kr_df["dt"] = pd.to_datetime(kr_df["dt"])
|
||
|
||
for us_t in DISCOVERY_US_TICKERS:
|
||
us_rows = await conn.fetch("""
|
||
SELECT trade_date, close_price FROM us_market_daily
|
||
WHERE ticker=$1 AND trade_date >= $2 ORDER BY trade_date
|
||
""", us_t, since)
|
||
if len(us_rows) < 30:
|
||
continue
|
||
us_df = pd.DataFrame([(r["trade_date"], r["close_price"]) for r in us_rows],
|
||
columns=["dt", "us"])
|
||
us_df["dt"] = pd.to_datetime(us_df["dt"]) + pd.Timedelta(days=1)
|
||
merged = pd.merge(us_df, kr_df, on="dt", how="inner")
|
||
if len(merged) < window_days // 2:
|
||
continue
|
||
us_ret = merged["us"].pct_change().dropna()
|
||
kr_ret = merged["kr"].pct_change().dropna()
|
||
n = min(len(us_ret), len(kr_ret))
|
||
if n < 20:
|
||
continue
|
||
slope, _, r_val, _, _ = stats.linregress(
|
||
us_ret.iloc[-n:].values, kr_ret.iloc[-n:].values)
|
||
if abs(r_val) < min_abs_corr:
|
||
continue
|
||
await conn.execute("""
|
||
INSERT INTO us_kr_pairs
|
||
(us_ticker, kr_code, beta_60d, correlation_60d, sample_size, source)
|
||
VALUES ($1, $2, $3, $4, $5, 'discovered')
|
||
ON CONFLICT (us_ticker, kr_code) DO UPDATE
|
||
SET beta_60d=$3, correlation_60d=$4, sample_size=$5, updated_at=NOW()
|
||
""", us_t, kr_c, float(slope), float(r_val), n)
|
||
added += 1
|
||
logger.info("pair_discover.done", added=added)
|
||
return {"added": added}
|
||
|
||
|
||
# ─────────────────────────────────────────────────────────────
|
||
# REST API
|
||
# ─────────────────────────────────────────────────────────────
|
||
@app.get("/health")
|
||
async def health():
|
||
return {"ok": True, "service": "us-market", "ts": datetime.now(KST).isoformat()}
|
||
|
||
|
||
@app.post("/collect")
|
||
async def manual_collect(days: int = Query(default=7, ge=1, le=365),
|
||
bg: BackgroundTasks = None):
|
||
"""수동 수집. days=7 일상, 백필은 180 권장."""
|
||
if bg:
|
||
bg.add_task(collect_us_daily, days)
|
||
return {"status": "queued", "days": days}
|
||
return await collect_us_daily(days)
|
||
|
||
|
||
@app.post("/collect/backfill")
|
||
async def backfill(days: int = Query(default=180, ge=30, le=365),
|
||
bg: BackgroundTasks = None):
|
||
"""대규모 백필 — 페어 베타 학습용 180일 권장. 76 ticker × 1.1초 ≈ 85초."""
|
||
if bg:
|
||
bg.add_task(collect_us_daily, days)
|
||
return {"status": "queued", "days": days}
|
||
return await collect_us_daily(days)
|
||
|
||
|
||
@app.post("/collect/yfinance-backfill")
|
||
async def yfinance_backfill_ep(days: int = Query(default=180, ge=30, le=730),
|
||
bg: BackgroundTasks = None):
|
||
"""yfinance로 일괄 히스토리 다운로드 (페어 베타 학습 초기 시드용)."""
|
||
if bg:
|
||
bg.add_task(backfill_yfinance, days)
|
||
return {"status": "queued", "days": days}
|
||
return await backfill_yfinance(days)
|
||
|
||
|
||
@app.post("/signal/calculate")
|
||
async def manual_signal(target: Optional[str] = None):
|
||
d = date.fromisoformat(target) if target else date.today()
|
||
return await calc_overnight_signals_all(d)
|
||
|
||
|
||
@app.get("/signal/{kr_code}")
|
||
async def get_signal(kr_code: str, days: int = Query(default=7, ge=1, le=90)):
|
||
async with pg_pool.acquire() as conn:
|
||
rows = await conn.fetch("""
|
||
SELECT signal_date, sector_adj, pair_adj, total_adj, contributing_pairs
|
||
FROM us_overnight_signal
|
||
WHERE kr_code=$1 AND signal_date >= CURRENT_DATE - $2::int
|
||
ORDER BY signal_date DESC
|
||
""", kr_code, days)
|
||
return [dict(r) for r in rows]
|
||
|
||
|
||
@app.get("/signal/latest")
|
||
async def latest_signals(limit: int = Query(default=50, le=500)):
|
||
async with pg_pool.acquire() as conn:
|
||
rows = await conn.fetch("""
|
||
SELECT DISTINCT ON (kr_code)
|
||
kr_code, signal_date, sector_adj, pair_adj, total_adj
|
||
FROM us_overnight_signal
|
||
ORDER BY kr_code, signal_date DESC
|
||
LIMIT $1
|
||
""", limit)
|
||
return [dict(r) for r in rows]
|
||
|
||
|
||
@app.get("/pairs")
|
||
async def list_pairs(kr_code: Optional[str] = None, us_ticker: Optional[str] = None):
|
||
async with pg_pool.acquire() as conn:
|
||
if kr_code:
|
||
rows = await conn.fetch(
|
||
"SELECT * FROM us_kr_pairs WHERE kr_code=$1 ORDER BY ABS(correlation_60d) DESC NULLS LAST",
|
||
kr_code)
|
||
elif us_ticker:
|
||
rows = await conn.fetch(
|
||
"SELECT * FROM us_kr_pairs WHERE us_ticker=$1 ORDER BY ABS(correlation_60d) DESC NULLS LAST",
|
||
us_ticker)
|
||
else:
|
||
rows = await conn.fetch(
|
||
"SELECT * FROM us_kr_pairs ORDER BY ABS(correlation_60d) DESC NULLS LAST LIMIT 200")
|
||
return [dict(r) for r in rows]
|
||
|
||
|
||
@app.post("/pairs/recalc-beta")
|
||
async def manual_recalc(window: int = Query(default=60, ge=20, le=250)):
|
||
return await recalc_pair_betas(window)
|
||
|
||
|
||
@app.post("/pairs/discover")
|
||
async def manual_discover(min_corr: float = Query(default=0.5, ge=0.1, le=0.9),
|
||
top_kr: int = Query(default=50, ge=10, le=200)):
|
||
return await discover_new_pairs(min_corr, top_kr)
|
||
|
||
|
||
@app.get("/etfs")
|
||
async def list_etfs():
|
||
async with pg_pool.acquire() as conn:
|
||
rows = await conn.fetch("SELECT * FROM us_sector_etf_map ORDER BY etf_ticker")
|
||
return [dict(r) for r in rows]
|
||
|
||
|
||
@app.get("/etfs/{etf}/latest")
|
||
async def etf_latest(etf: str):
|
||
async with pg_pool.acquire() as conn:
|
||
row = await conn.fetchrow("""
|
||
SELECT * FROM us_market_daily WHERE ticker=$1
|
||
ORDER BY trade_date DESC LIMIT 1
|
||
""", etf.upper())
|
||
return dict(row) if row else {"err": "no data"}
|
||
|
||
|
||
@app.get("/stats")
|
||
async def stats_endpoint():
|
||
async with pg_pool.acquire() as conn:
|
||
r1 = await conn.fetchrow(
|
||
"SELECT COUNT(*) AS rows, COUNT(DISTINCT ticker) AS tickers,"
|
||
" MIN(trade_date) AS earliest, MAX(trade_date) AS latest FROM us_market_daily")
|
||
r2 = await conn.fetchrow(
|
||
"SELECT COUNT(*) AS total, COUNT(*) FILTER (WHERE source='seed') AS seed,"
|
||
" COUNT(*) FILTER (WHERE source='discovered') AS discovered,"
|
||
" COUNT(*) FILTER (WHERE beta_60d IS NOT NULL) AS with_beta"
|
||
" FROM us_kr_pairs")
|
||
r3 = await conn.fetchrow(
|
||
"SELECT COUNT(*) AS rows, COUNT(DISTINCT kr_code) AS codes,"
|
||
" MAX(signal_date) AS latest FROM us_overnight_signal")
|
||
return {"us_daily": dict(r1), "pairs": dict(r2), "signals": dict(r3)}
|