kyu
6d3b0bacc0
- 19개 마이크로서비스 (news-collector, score-engine, ta-engine, dart-collector, aux-signal, us-market, graph-engine, telegram-bot, dashboard-api, kis-api 등) - 가치투자 스코어링 + 10공식 앙상블 보팅 (매직포뮬러/F-Score/Altman/PEG/ 모멘텀/Beneish/GP-A/G-Score/Amihud/BAB) - 뉴스 수집→형태소→임베딩→중복제거→AI분석 파이프라인 - 기술적분석 + GAT 그래프신경망 + 미증시 동조 시그널 Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
1298 lines
55 KiB
Python
1298 lines
55 KiB
Python
"""
|
|
DART 공시 수집기
|
|
- 전체 공시 실시간 수집 (10분마다)
|
|
- 재무제표 수집 (매일 1회)
|
|
- 기업 기본정보 (서버 시작 시 + 매일 갱신)
|
|
- 주요사항보고 수집
|
|
- 수집 즉시 AI 분석 파이프라인 연동
|
|
"""
|
|
|
|
import asyncio
|
|
import io
|
|
import json
|
|
import os
|
|
import re
|
|
import time
|
|
import zipfile
|
|
import xml.etree.ElementTree as ET
|
|
from datetime import datetime, timedelta
|
|
from typing import Optional
|
|
|
|
import asyncpg
|
|
import httpx
|
|
import redis.asyncio as aioredis
|
|
import structlog
|
|
import xmltodict
|
|
from apscheduler.schedulers.asyncio import AsyncIOScheduler
|
|
from fastapi import FastAPI, HTTPException, Query
|
|
from fastapi.responses import JSONResponse
|
|
from fastapi.middleware.cors import CORSMiddleware
|
|
|
|
structlog.configure(processors=[
|
|
structlog.processors.TimeStamper(fmt="iso"),
|
|
structlog.processors.add_log_level,
|
|
structlog.processors.JSONRenderer(),
|
|
])
|
|
logger = structlog.get_logger()
|
|
|
|
# ── 환경변수 ──────────────────────────────────────────────
|
|
DART_API_KEY = os.getenv("DART_API_KEY", "")
|
|
REDIS_HOST = os.getenv("REDIS_HOST", "redis")
|
|
REDIS_PASSWORD = os.getenv("REDIS_PASSWORD", "")
|
|
PG_HOST = os.getenv("POSTGRES_HOST", "postgres")
|
|
PG_PORT = int(os.getenv("POSTGRES_PORT", "5432"))
|
|
PG_DB = os.getenv("POSTGRES_DB", "trading_ai")
|
|
PG_USER = os.getenv("POSTGRES_USER", "kyu")
|
|
PG_PASS = os.getenv("POSTGRES_PASSWORD", "7895123")
|
|
BAREUN_API_URL = os.getenv("BAREUN_API_URL", "http://bareunaapi:5757")
|
|
OLLAMA_URL = os.getenv("OLLAMA_URL", "http://ollama:11434")
|
|
QDRANT_URL = os.getenv("QDRANT_URL", "http://qdrant:6333")
|
|
|
|
DART_BASE = "https://opendart.fss.or.kr/api"
|
|
|
|
# ── 전역 상태 ─────────────────────────────────────────────
|
|
pg_pool: Optional[asyncpg.Pool] = None
|
|
redis_client: Optional[aioredis.Redis] = None
|
|
corp_codes: dict[str, dict] = {} # {stock_code: {corp_code, corp_name, ...}}
|
|
scheduler = AsyncIOScheduler(timezone="Asia/Seoul")
|
|
|
|
class Stats:
|
|
disclosures: int = 0
|
|
financials: int = 0
|
|
corps_loaded: int = 0
|
|
last_run: str = ""
|
|
errors: int = 0
|
|
|
|
stats = Stats()
|
|
|
|
|
|
# ══════════════════════════════════════════════════════════
|
|
# 1. 기업 기본정보 (전체 상장 종목 코드)
|
|
# ══════════════════════════════════════════════════════════
|
|
|
|
async def load_corp_codes():
|
|
"""DART 고유번호 전체 다운로드 → 종목코드 매핑"""
|
|
global corp_codes
|
|
logger.info("corp_codes.loading")
|
|
|
|
async with httpx.AsyncClient(timeout=60) as client:
|
|
resp = await client.get(f"{DART_BASE}/corpCode.xml", params={"crtfc_key": DART_API_KEY})
|
|
|
|
if resp.status_code != 200:
|
|
logger.error("corp_codes.download_failed", status=resp.status_code)
|
|
return
|
|
|
|
# ZIP 파일 안에 XML
|
|
zf = zipfile.ZipFile(io.BytesIO(resp.content))
|
|
xml_data = zf.read(zf.namelist()[0])
|
|
root = ET.fromstring(xml_data)
|
|
|
|
new_map = {}
|
|
for item in root.findall(".//list"):
|
|
corp_code = item.findtext("corp_code", "")
|
|
corp_name = item.findtext("corp_name", "")
|
|
stock_code = item.findtext("stock_code", "").strip()
|
|
modify_date = item.findtext("modify_date", "")
|
|
|
|
if stock_code and len(stock_code) == 6:
|
|
new_map[stock_code] = {
|
|
"corp_code": corp_code,
|
|
"corp_name": corp_name,
|
|
"stock_code": stock_code,
|
|
"modify_date": modify_date,
|
|
}
|
|
|
|
corp_codes = new_map
|
|
stats.corps_loaded = len(corp_codes)
|
|
logger.info("corp_codes.loaded", count=len(corp_codes))
|
|
|
|
# DB에 저장
|
|
if pg_pool:
|
|
await save_corp_codes_to_db()
|
|
|
|
|
|
async def fetch_krx_active_codes() -> set:
|
|
"""KRX에서 현재 상장 중인 종목코드만 가져오기 (KRX 실패시 네이버 폴백)"""
|
|
import re as _re
|
|
active = set()
|
|
headers = {
|
|
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36",
|
|
"Referer": "http://data.krx.co.kr/contents/MDC/MDI/mdiLoader/index.cmd",
|
|
}
|
|
async with httpx.AsyncClient(timeout=30, headers=headers) as client:
|
|
for mkt_id in ("STK", "KSQ"):
|
|
try:
|
|
resp = await client.post(
|
|
"http://data.krx.co.kr/comm/bldAttend/getJsonData.cmd",
|
|
data={
|
|
"bld": "dbms/MDC/STAT/standard/MDCSTAT01901",
|
|
"locale": "ko_KR",
|
|
"mktId": mkt_id,
|
|
"share": "1",
|
|
"csvxls_is498": "false",
|
|
},
|
|
)
|
|
if resp.status_code == 200:
|
|
data = resp.json()
|
|
for item in data.get("OutBlock_1", []):
|
|
code = item.get("ISU_SRT_CD", "").strip()
|
|
if len(code) == 6:
|
|
active.add(code)
|
|
except Exception as e:
|
|
logger.warning("krx.active_fetch_failed", market=mkt_id, error=str(e))
|
|
|
|
# KRX 실패시 네이버 금융 폴백
|
|
if len(active) < 100:
|
|
logger.warning("krx.active_fallback_naver", krx_count=len(active))
|
|
try:
|
|
async with httpx.AsyncClient(timeout=15, headers={"User-Agent": "Mozilla/5.0"},
|
|
follow_redirects=True) as nav:
|
|
for sosok in [0, 1]:
|
|
for page in range(1, 50):
|
|
try:
|
|
r = await nav.get(
|
|
f"https://finance.naver.com/sise/sise_market_sum.naver"
|
|
f"?sosok={sosok}&page={page}")
|
|
text = r.content.decode("euc-kr", errors="ignore")
|
|
codes = _re.findall(r"main\.naver\?code=(\d{6})", text)
|
|
if not codes:
|
|
break
|
|
active.update(codes)
|
|
except Exception:
|
|
break
|
|
logger.info("naver.active_loaded", count=len(active))
|
|
except Exception as e:
|
|
logger.warning("naver.active_fetch_failed", error=str(e))
|
|
|
|
logger.info("krx.active_loaded", count=len(active))
|
|
return active
|
|
|
|
|
|
async def save_corp_codes_to_db():
|
|
"""기업 기본정보 DB 배치 저장 + KRX 현재 상장 여부 표시"""
|
|
active_codes = await fetch_krx_active_codes()
|
|
|
|
async with pg_pool.acquire() as conn:
|
|
await conn.execute("DELETE FROM dart_corps")
|
|
records = [
|
|
(v["stock_code"], v["corp_code"], v["corp_name"], v["modify_date"],
|
|
v["stock_code"] in active_codes)
|
|
for v in corp_codes.values()
|
|
]
|
|
await conn.executemany(
|
|
"INSERT INTO dart_corps (stock_code, corp_code, corp_name, modify_date, is_active)"
|
|
" VALUES ($1,$2,$3,$4,$5)",
|
|
records
|
|
)
|
|
active_saved = sum(1 for r in records if r[4])
|
|
logger.info("corp_codes.saved_to_db", total=len(corp_codes), active=active_saved)
|
|
|
|
async def fetch_krx_sectors() -> dict:
|
|
"""
|
|
KRX 산업분류 (전종목 KRX 산업코드/업종명) → {stock_code: sector_name}
|
|
bld: dbms/MDC/STAT/standard/MDCSTAT03901 (KRX 산업분류)
|
|
"""
|
|
out: dict = {}
|
|
headers = {
|
|
"User-Agent": "Mozilla/5.0",
|
|
"Referer": "http://data.krx.co.kr/contents/MDC/MDI/mdiLoader/index.cmd",
|
|
}
|
|
async with httpx.AsyncClient(timeout=30, headers=headers) as client:
|
|
for mkt_id in ("STK", "KSQ"):
|
|
try:
|
|
resp = await client.post(
|
|
"http://data.krx.co.kr/comm/bldAttend/getJsonData.cmd",
|
|
data={
|
|
"bld": "dbms/MDC/STAT/standard/MDCSTAT03901",
|
|
"locale": "ko_KR",
|
|
"mktId": mkt_id,
|
|
"trdDd": datetime.now().strftime("%Y%m%d"),
|
|
"money": "1",
|
|
"csvxls_is498": "false",
|
|
},
|
|
)
|
|
if resp.status_code == 200:
|
|
data = resp.json()
|
|
for item in data.get("block1", []):
|
|
code = (item.get("ISU_SRT_CD") or "").strip()
|
|
sec = (item.get("IDX_IND_NM") or "").strip()
|
|
if len(code) == 6 and sec:
|
|
out[code] = sec
|
|
except Exception as e:
|
|
logger.warning("krx.sectors_fetch_failed", market=mkt_id, error=str(e))
|
|
logger.info("krx.sectors_loaded", count=len(out))
|
|
return out
|
|
|
|
|
|
async def collect_sectors():
|
|
"""KRX 섹터 정보를 dart_corps.sector 컬럼에 저장"""
|
|
sectors = await fetch_krx_sectors()
|
|
if not sectors:
|
|
return 0
|
|
saved = 0
|
|
async with pg_pool.acquire() as conn:
|
|
for code, sec in sectors.items():
|
|
res = await conn.execute(
|
|
"UPDATE dart_corps SET sector=$1 WHERE stock_code=$2",
|
|
sec, code)
|
|
if "1" in res:
|
|
saved += 1
|
|
logger.info("sectors.saved", saved=saved)
|
|
return saved
|
|
|
|
|
|
def get_corp_code(stock_code: str) -> str:
|
|
"""종목코드 → DART 고유번호"""
|
|
info = corp_codes.get(stock_code)
|
|
return info["corp_code"] if info else ""
|
|
|
|
|
|
def get_corp_name(stock_code: str) -> str:
|
|
info = corp_codes.get(stock_code)
|
|
return info["corp_name"] if info else ""
|
|
|
|
|
|
# ══════════════════════════════════════════════════════════
|
|
# 2. 공시 전체 수집
|
|
# ══════════════════════════════════════════════════════════
|
|
|
|
async def fetch_disclosures(
|
|
bgn_de: str = "", end_de: str = "",
|
|
corp_code: str = "", pblntf_ty: str = "",
|
|
page_no: int = 1, page_count: int = 100,
|
|
) -> list[dict]:
|
|
"""DART 공시 목록 조회"""
|
|
if not bgn_de:
|
|
bgn_de = (datetime.now() - timedelta(days=1)).strftime("%Y%m%d")
|
|
if not end_de:
|
|
end_de = datetime.now().strftime("%Y%m%d")
|
|
|
|
params = {
|
|
"crtfc_key": DART_API_KEY,
|
|
"bgn_de": bgn_de,
|
|
"end_de": end_de,
|
|
"page_no": page_no,
|
|
"page_count": page_count,
|
|
}
|
|
if corp_code:
|
|
params["corp_code"] = corp_code
|
|
if pblntf_ty:
|
|
params["pblntf_ty"] = pblntf_ty # A:정기공시, B:주요사항, C:발행공시, D:지분공시, E:기타공시, F:외부감사, G:펀드, H:자산유동화, I:거래소, J:공정위, K:기타
|
|
|
|
async with httpx.AsyncClient(timeout=30) as client:
|
|
resp = await client.get(f"{DART_BASE}/list.json", params=params)
|
|
data = resp.json()
|
|
|
|
if data.get("status") != "000":
|
|
return []
|
|
|
|
return data.get("list", [])
|
|
|
|
|
|
async def collect_recent_disclosures():
|
|
"""최근 공시 수집 + AI 분석"""
|
|
logger.info("disclosures.collecting")
|
|
today = datetime.now().strftime("%Y%m%d")
|
|
yesterday = (datetime.now() - timedelta(days=1)).strftime("%Y%m%d")
|
|
|
|
all_disclosures = []
|
|
|
|
# 공시 유형별 수집
|
|
types = {
|
|
"A": "정기공시", # 사업보고서, 분기보고서
|
|
"B": "주요사항", # 유상증자, 합병, 분할 등
|
|
"C": "발행공시", # 채권, 주식 발행
|
|
"D": "지분공시", # 대량보유, 임원 지분변동
|
|
"E": "기타공시",
|
|
"I": "거래소공시", # 조회공시, 공시번복 등
|
|
}
|
|
|
|
for type_code, type_name in types.items():
|
|
try:
|
|
items = await fetch_disclosures(
|
|
bgn_de=yesterday, end_de=today, pblntf_ty=type_code
|
|
)
|
|
for item in items:
|
|
item["pblntf_ty_name"] = type_name
|
|
all_disclosures.extend(items)
|
|
await asyncio.sleep(0.5) # Rate limit
|
|
except Exception as e:
|
|
logger.warning("disclosures.type_error", type=type_name, error=str(e))
|
|
|
|
logger.info("disclosures.fetched", total=len(all_disclosures))
|
|
|
|
# 중복 제거 + DB 저장 + AI 분석
|
|
new_count = 0
|
|
for disc in all_disclosures:
|
|
rcept_no = disc.get("rcept_no", "")
|
|
|
|
# Redis 중복 체크
|
|
if redis_client:
|
|
exists = await redis_client.exists(f"dart:disc:{rcept_no}")
|
|
if exists:
|
|
continue
|
|
await redis_client.set(f"dart:disc:{rcept_no}", "1", ex=86400 * 7)
|
|
|
|
# DB 저장
|
|
if pg_pool:
|
|
await save_disclosure(disc)
|
|
|
|
# AI 분석 파이프라인
|
|
await analyze_disclosure(disc)
|
|
new_count += 1
|
|
await asyncio.sleep(0.3)
|
|
|
|
stats.disclosures += new_count
|
|
stats.last_run = datetime.now().isoformat()
|
|
logger.info("disclosures.done", new=new_count, total=len(all_disclosures))
|
|
|
|
|
|
async def save_disclosure(disc: dict):
|
|
"""공시 DB 저장"""
|
|
try:
|
|
async with pg_pool.acquire() as conn:
|
|
await conn.execute("""
|
|
INSERT INTO dart_disclosures
|
|
(rcept_no, rcept_dt, corp_code, corp_name, stock_code,
|
|
corp_cls, report_nm, flr_nm, pblntf_ty, pblntf_detail_ty)
|
|
VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10)
|
|
ON CONFLICT (rcept_no) DO NOTHING
|
|
""",
|
|
disc.get("rcept_no", ""),
|
|
disc.get("rcept_dt", ""),
|
|
disc.get("corp_code", ""),
|
|
disc.get("corp_name", ""),
|
|
disc.get("stock_code", ""),
|
|
disc.get("corp_cls", ""),
|
|
disc.get("report_nm", ""),
|
|
disc.get("flr_nm", ""),
|
|
disc.get("pblntf_ty", ""),
|
|
disc.get("pblntf_detail_ty", ""),
|
|
)
|
|
except Exception as e:
|
|
logger.warning("disclosure.save_error", error=str(e))
|
|
|
|
|
|
async def analyze_disclosure(disc: dict):
|
|
"""공시를 AI 분석 파이프라인으로 전달"""
|
|
corp_name = disc.get("corp_name", "")
|
|
stock_code = disc.get("stock_code", "")
|
|
report_nm = disc.get("report_nm", "")
|
|
pblntf_ty = disc.get("pblntf_ty_name", "")
|
|
|
|
title = f"[{pblntf_ty}] {corp_name}({stock_code}) - {report_nm}"
|
|
content = f"기업: {corp_name}, 종목코드: {stock_code}, 공시유형: {pblntf_ty}, 공시명: {report_nm}"
|
|
|
|
try:
|
|
async with httpx.AsyncClient(timeout=120) as client:
|
|
# vLLM AI 분석
|
|
prompt = (
|
|
f"DART 공시 분석:\n"
|
|
f"기업: {corp_name} ({stock_code})\n"
|
|
f"공시유형: {pblntf_ty}\n"
|
|
f"공시명: {report_nm}\n\n"
|
|
"위 공시가 해당 종목에 미치는 영향을 분석하여 JSON으로 응답:\n"
|
|
'{"sentiment":"호재 또는 악재 또는 중립",'
|
|
'"intensity":1에서5,'
|
|
f'"primary_stock":"{stock_code}",'
|
|
'"affected_stocks":[],'
|
|
'"reason":"판단근거 2문장",'
|
|
'"investment_action":"매수관심 또는 매도관심 또는 관망"}'
|
|
)
|
|
|
|
vllm_resp = await client.post(f"{OLLAMA_URL}/v1/chat/completions", json={
|
|
"model": "exaone3.5:7.8b",
|
|
"messages": [
|
|
{"role": "system", "content": "당신은 한국 주식 DART 공시 분석 전문가입니다. JSON으로만 응답하세요."},
|
|
{"role": "user", "content": prompt},
|
|
],
|
|
"max_tokens": 300, "temperature": 0.1,
|
|
})
|
|
|
|
analysis = {}
|
|
try:
|
|
c = vllm_resp.json()["choices"][0]["message"]["content"]
|
|
analysis = json.loads(re.sub(r"```json\n?|```", "", c).strip())
|
|
except Exception:
|
|
analysis = {"sentiment": "중립", "intensity": 0, "reason": "파싱실패",
|
|
"primary_stock": stock_code, "affected_stocks": [],
|
|
"investment_action": "관망"}
|
|
|
|
# news_analysis 테이블에 저장 (기존 파이프라인과 통합)
|
|
async with pg_pool.acquire() as conn:
|
|
h = disc.get("rcept_no", "")[:16]
|
|
escape = lambda s: (s or "").replace("'", "''")
|
|
await conn.execute(f"""
|
|
INSERT INTO news_analysis
|
|
(title, url, source, published_at, hash, sentiment, intensity,
|
|
primary_stock, affected_stocks, reason, investment_action,
|
|
keywords, stock_names, stock_codes, similar_count, analyzed_at)
|
|
VALUES (
|
|
'{escape(title[:500])}',
|
|
'https://dart.fss.or.kr/dsaf001/main.do?rcpNo={disc.get("rcept_no","")}',
|
|
'DART공시',
|
|
'{disc.get("rcept_dt", datetime.now().strftime("%Y%m%d"))}',
|
|
'{h}',
|
|
'{analysis.get("sentiment","중립")}',
|
|
{analysis.get("intensity", 0)},
|
|
'{escape(analysis.get("primary_stock",""))}',
|
|
'{json.dumps(analysis.get("affected_stocks",[]))}',
|
|
'{escape(analysis.get("reason",""))}',
|
|
'{analysis.get("investment_action","관망")}',
|
|
'{json.dumps([pblntf_ty, report_nm[:50]])}',
|
|
'{json.dumps([corp_name])}',
|
|
'{json.dumps([stock_code] if stock_code else [])}',
|
|
0,
|
|
'{datetime.now().isoformat()}'
|
|
) ON CONFLICT (hash) DO NOTHING
|
|
""")
|
|
except Exception as e:
|
|
logger.warning("disclosure.analyze_error", corp=corp_name, error=str(e))
|
|
|
|
|
|
# ══════════════════════════════════════════════════════════
|
|
# 3. 재무제표 수집
|
|
# ══════════════════════════════════════════════════════════
|
|
|
|
async def fetch_financial(
|
|
corp_code: str, bsns_year: str, reprt_code: str = "11011"
|
|
) -> list[dict]:
|
|
"""
|
|
재무제표 조회
|
|
reprt_code: 11013=1분기, 11012=반기, 11014=3분기, 11011=사업보고서
|
|
fs_div: CFS(연결) 우선, 없으면 OFS(별도) 폴백
|
|
"""
|
|
params = {
|
|
"crtfc_key": DART_API_KEY,
|
|
"corp_code": corp_code,
|
|
"bsns_year": bsns_year,
|
|
"reprt_code": reprt_code,
|
|
"fs_div": "CFS",
|
|
}
|
|
async with httpx.AsyncClient(timeout=30) as client:
|
|
resp = await client.get(f"{DART_BASE}/fnlttSinglAcntAll.json", params=params)
|
|
data = resp.json()
|
|
if data.get("status") == "000":
|
|
return data.get("list", [])
|
|
# CFS가 없는 경우(013) 별도재무제표로 폴백
|
|
if data.get("status") == "013":
|
|
params["fs_div"] = "OFS"
|
|
resp = await client.get(f"{DART_BASE}/fnlttSinglAcntAll.json", params=params)
|
|
data = resp.json()
|
|
if data.get("status") == "000":
|
|
return data.get("list", [])
|
|
# status 013 = 조회된 데이터 없음. 당해/직전 분기 보고서 미공시는 정상이므로 debug로 강등.
|
|
status = data.get("status")
|
|
if status == "013":
|
|
logger.debug("financial.no_data",
|
|
corp=corp_code, year=bsns_year, reprt=reprt_code,
|
|
msg=data.get("message"))
|
|
else:
|
|
logger.warning("financial.api_failed",
|
|
corp=corp_code, year=bsns_year, reprt=reprt_code,
|
|
status=status, msg=data.get("message"))
|
|
return []
|
|
|
|
|
|
def calc_financial_ratios(key_items: dict, prev_revenue: int = 0) -> dict:
|
|
"""버핏 스타일 재무 비율 계산"""
|
|
revenue = key_items.get("매출액", 0)
|
|
op_profit = key_items.get("영업이익", 0)
|
|
net_income = key_items.get("당기순이익", 0)
|
|
total_assets = key_items.get("자산총계", 0)
|
|
total_liabs = key_items.get("부채총계", 0)
|
|
total_equity = key_items.get("자본총계", 0)
|
|
op_cashflow = key_items.get("영업활동현금흐름", 0)
|
|
|
|
roe = round(net_income / total_equity * 100, 2) if total_equity > 0 else 0.0
|
|
operating_margin = round(op_profit / revenue * 100, 2) if revenue > 0 else 0.0
|
|
net_margin = round(net_income / revenue * 100, 2) if revenue > 0 else 0.0
|
|
debt_ratio = round(total_liabs / total_assets * 100, 2) if total_assets > 0 else 0.0
|
|
fcf_ratio = round(op_cashflow / revenue * 100, 2) if revenue > 0 else 0.0
|
|
revenue_growth = round((revenue - prev_revenue) / abs(prev_revenue) * 100, 2) \
|
|
if prev_revenue and prev_revenue != 0 else 0.0
|
|
return {
|
|
"roe": roe,
|
|
"operating_margin": operating_margin,
|
|
"net_margin": net_margin,
|
|
"debt_ratio": debt_ratio,
|
|
"fcf_ratio": fcf_ratio,
|
|
"revenue_growth": revenue_growth,
|
|
}
|
|
|
|
|
|
async def collect_financials_for_top_stocks(count: int = 300, years: int = 2, annual_only: bool = False):
|
|
"""상장 종목 재무제표 수집 (is_active 기준)
|
|
years: 최근 N년치 수집 (기본 2년)
|
|
annual_only: True면 사업보고서(11011)만 수집 → 다년 백필 시 호출 부담 ↓
|
|
"""
|
|
logger.info("financials.collecting", count=count, years=years, annual_only=annual_only)
|
|
|
|
cur_year = datetime.now().year
|
|
year_list = [str(cur_year - i) for i in reversed(range(years))] # ASC: [...,prev,cur]
|
|
if annual_only:
|
|
report_codes = [("11011", "사업보고서")]
|
|
else:
|
|
report_codes = [
|
|
("11011", "사업보고서"),
|
|
("11012", "반기보고서"),
|
|
("11013", "1분기보고서"),
|
|
("11014", "3분기보고서"),
|
|
]
|
|
|
|
# is_active 종목만 대상
|
|
if pg_pool:
|
|
async with pg_pool.acquire() as conn:
|
|
rows = await conn.fetch(
|
|
"SELECT stock_code FROM dart_corps WHERE is_active=true ORDER BY stock_code LIMIT $1", count)
|
|
active_codes = {r["stock_code"] for r in rows}
|
|
else:
|
|
active_codes = set(list(corp_codes.keys())[:count])
|
|
|
|
stock_list = [(sc, info) for sc, info in corp_codes.items() if sc in active_codes]
|
|
|
|
collected = 0
|
|
for stock_code, info in stock_list:
|
|
corp_code = info["corp_code"]
|
|
corp_name = info["corp_name"]
|
|
|
|
# 매출성장률용: 직전 연도 사업보고서 매출 (ASC 순회로 누적)
|
|
prev_revenue_annual = 0
|
|
for reprt_code, reprt_name in report_codes:
|
|
for yr in year_list:
|
|
try:
|
|
items = await fetch_financial(corp_code, yr, reprt_code)
|
|
if not items:
|
|
continue
|
|
|
|
# 자본변동표(SCE)는 '자본총계'/'당기순이익'이 시점별 8번 중복돼
|
|
# 마지막 값이 음수가 되면 ROE 계산이 깨짐 → 본표만 사용
|
|
valid_sj = ("재무상태표", "포괄손익계산서", "손익계산서", "현금흐름표")
|
|
aliases = {
|
|
"매출액": ("매출액", "수익(매출액)", "영업수익"),
|
|
"영업이익": ("영업이익", "영업이익(손실)"),
|
|
"당기순이익": ("당기순이익", "당기순이익(손실)",
|
|
"분기순이익", "분기순이익(손실)",
|
|
"반기순이익", "반기순이익(손실)"),
|
|
"자산총계": ("자산총계",),
|
|
"부채총계": ("부채총계",),
|
|
"자본총계": ("자본총계",),
|
|
"영업활동현금흐름": ("영업활동현금흐름", "영업활동으로 인한 현금흐름"),
|
|
}
|
|
key_items: dict = {}
|
|
for item in items:
|
|
if item.get("sj_nm", "") not in valid_sj:
|
|
continue
|
|
acnt_nm = item.get("account_nm", "")
|
|
amount = item.get("thstrm_amount", "")
|
|
for canonical, names in aliases.items():
|
|
if acnt_nm in names and canonical not in key_items:
|
|
try:
|
|
key_items[canonical] = int(amount.replace(",", ""))
|
|
except (ValueError, AttributeError):
|
|
pass
|
|
break
|
|
|
|
if key_items and pg_pool:
|
|
# 매출성장률: 11011 + 직전 연도 매출 있을 때만 의미 있음
|
|
prev_rev = prev_revenue_annual if reprt_code == "11011" else 0
|
|
ratios = calc_financial_ratios(key_items, prev_rev)
|
|
await save_financial(
|
|
stock_code, corp_code, corp_name,
|
|
yr, reprt_code, reprt_name, key_items, ratios
|
|
)
|
|
collected += 1
|
|
if reprt_code == "11011":
|
|
prev_revenue_annual = key_items.get("매출액", 0)
|
|
|
|
await asyncio.sleep(0.2)
|
|
except Exception as e:
|
|
logger.warning("financial.error", code=stock_code, error=str(e))
|
|
|
|
stats.financials += collected
|
|
logger.info("financials.done", collected=collected)
|
|
|
|
|
|
async def save_financial(
|
|
stock_code, corp_code, corp_name,
|
|
bsns_year, reprt_code, reprt_name, key_items, ratios: dict = None
|
|
):
|
|
"""재무제표 DB 저장 (비율 포함)"""
|
|
if ratios is None:
|
|
ratios = calc_financial_ratios(key_items)
|
|
try:
|
|
async with pg_pool.acquire() as conn:
|
|
await conn.execute("""
|
|
INSERT INTO dart_financials
|
|
(stock_code, corp_code, corp_name, bsns_year, reprt_code, reprt_name,
|
|
revenue, operating_profit, net_income, total_assets,
|
|
total_liabilities, total_equity, operating_cashflow,
|
|
roe, operating_margin, net_margin, debt_ratio, fcf_ratio, revenue_growth,
|
|
collected_at)
|
|
VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16,$17,$18,$19,$20)
|
|
ON CONFLICT (stock_code, bsns_year, reprt_code) DO UPDATE SET
|
|
revenue=$7, operating_profit=$8, net_income=$9,
|
|
total_assets=$10, total_liabilities=$11, total_equity=$12,
|
|
operating_cashflow=$13, roe=$14, operating_margin=$15,
|
|
net_margin=$16, debt_ratio=$17, fcf_ratio=$18, revenue_growth=$19,
|
|
collected_at=$20
|
|
""",
|
|
stock_code, corp_code, corp_name, bsns_year, reprt_code, reprt_name,
|
|
key_items.get("매출액", 0),
|
|
key_items.get("영업이익", 0),
|
|
key_items.get("당기순이익", 0),
|
|
key_items.get("자산총계", 0),
|
|
key_items.get("부채총계", 0),
|
|
key_items.get("자본총계", 0),
|
|
key_items.get("영업활동현금흐름", 0),
|
|
ratios.get("roe", 0.0),
|
|
ratios.get("operating_margin", 0.0),
|
|
ratios.get("net_margin", 0.0),
|
|
ratios.get("debt_ratio", 0.0),
|
|
ratios.get("fcf_ratio", 0.0),
|
|
ratios.get("revenue_growth", 0.0),
|
|
datetime.now(),
|
|
)
|
|
except Exception as e:
|
|
logger.warning("financial.save_error", code=stock_code, error=str(e))
|
|
|
|
|
|
# ══════════════════════════════════════════════════════════
|
|
# 4. 주요사항보고
|
|
# ══════════════════════════════════════════════════════════
|
|
|
|
async def collect_major_reports():
|
|
"""주요사항보고 수집 (유상증자, 합병, 자사주 등)"""
|
|
logger.info("major_reports.collecting")
|
|
|
|
today = datetime.now().strftime("%Y%m%d")
|
|
week_ago = (datetime.now() - timedelta(days=7)).strftime("%Y%m%d")
|
|
|
|
# B: 주요사항보고서
|
|
items = await fetch_disclosures(bgn_de=week_ago, end_de=today, pblntf_ty="B")
|
|
|
|
major_keywords = [
|
|
"유상증자", "무상증자", "합병", "분할", "자사주", "전환사채",
|
|
"신주인수권", "교환사채", "주식매수선택권", "배당", "감자",
|
|
"해산", "파산", "회생", "상장폐지", "거래정지",
|
|
]
|
|
|
|
important = []
|
|
for item in items:
|
|
report_nm = item.get("report_nm", "")
|
|
if any(kw in report_nm for kw in major_keywords):
|
|
item["is_major"] = True
|
|
important.append(item)
|
|
|
|
for item in important:
|
|
rcept_no = item.get("rcept_no", "")
|
|
if redis_client:
|
|
exists = await redis_client.exists(f"dart:major:{rcept_no}")
|
|
if exists:
|
|
continue
|
|
await redis_client.set(f"dart:major:{rcept_no}", "1", ex=86400 * 30)
|
|
|
|
item["pblntf_ty_name"] = "주요사항"
|
|
if pg_pool:
|
|
await save_disclosure(item)
|
|
await analyze_disclosure(item)
|
|
await asyncio.sleep(0.3)
|
|
|
|
logger.info("major_reports.done", total=len(items), important=len(important))
|
|
|
|
|
|
# ══════════════════════════════════════════════════════════
|
|
# 5. 배당 데이터 수집
|
|
# ══════════════════════════════════════════════════════════
|
|
|
|
async def collect_dividends(count: int = 200):
|
|
"""상위 종목 배당 데이터 수집 (DART 배당에 관한 사항)"""
|
|
logger.info("dividends.collecting", count=count)
|
|
saved = 0
|
|
async with pg_pool.acquire() as conn:
|
|
corps = await conn.fetch("""
|
|
SELECT d.stock_code, d.corp_code, d.corp_name,
|
|
f.revenue
|
|
FROM dart_corps d
|
|
JOIN dart_financials f ON f.stock_code = d.stock_code
|
|
WHERE d.is_active = true AND d.corp_code != ''
|
|
ORDER BY f.bsns_year DESC, f.revenue DESC NULLS LAST
|
|
LIMIT $1
|
|
""", count)
|
|
|
|
bsns_year = str(datetime.now().year - 1)
|
|
async with httpx.AsyncClient(timeout=30) as client:
|
|
for corp in corps:
|
|
try:
|
|
resp = await client.get(f"{DART_BASE}/alotMatter.json", params={
|
|
"crtfc_key": DART_API_KEY,
|
|
"corp_code": corp["corp_code"],
|
|
"bsns_year": bsns_year,
|
|
"reprt_code": "11011", # 사업보고서
|
|
})
|
|
data = resp.json()
|
|
if data.get("status") != "000" or not data.get("list"):
|
|
continue
|
|
|
|
# DART alotMatter 응답 파싱 (응답 키: thstrm/frmtrm, '-'는 0으로 처리)
|
|
def _to_int(v: str) -> int:
|
|
s = str(v or "").replace(",", "").strip()
|
|
if not s or s == "-":
|
|
return 0
|
|
try:
|
|
return int(float(s))
|
|
except: return 0
|
|
|
|
dps = dps_prev = total_div = 0
|
|
for item in data["list"]:
|
|
se = item.get("se", "")
|
|
th = item.get("thstrm", "")
|
|
fr = item.get("frmtrm", "")
|
|
if "주당 현금배당금" in se and dps == 0:
|
|
dps = _to_int(th)
|
|
dps_prev = _to_int(fr)
|
|
elif "현금배당금총액" in se and total_div == 0:
|
|
total_div = _to_int(th) * 1_000_000 # 백만원 → 원
|
|
if dps <= 0:
|
|
continue
|
|
async with pg_pool.acquire() as conn:
|
|
await conn.execute("""
|
|
INSERT INTO dart_dividends
|
|
(stock_code, corp_name, bsns_year, dps, dps_prev, total_dividend)
|
|
VALUES ($1,$2,$3,$4,$5,$6)
|
|
ON CONFLICT (stock_code, bsns_year)
|
|
DO UPDATE SET dps=$4, dps_prev=$5, total_dividend=$6, collected_at=NOW()
|
|
""", corp["stock_code"], corp["corp_name"], bsns_year,
|
|
dps, dps_prev, total_div)
|
|
saved += 1
|
|
await asyncio.sleep(0.15)
|
|
except Exception as e:
|
|
logger.warning("dividends.fetch_err", corp=corp["stock_code"], error=str(e))
|
|
|
|
logger.info("dividends.done", saved=saved)
|
|
return saved
|
|
|
|
|
|
# ══════════════════════════════════════════════════════════
|
|
# 6. 섹터 정보 수집 (KRX CSV)
|
|
# ══════════════════════════════════════════════════════════
|
|
|
|
_KSIC_MAP = {
|
|
"10":"식품","11":"음료","12":"담배","13":"섬유","14":"의복/모피","15":"가죽/신발",
|
|
"16":"목재","17":"펄프/종이","18":"인쇄","19":"석유정제","20":"화학",
|
|
"21":"의약품","22":"고무/플라스틱","23":"비금속광물","24":"1차금속","25":"금속가공",
|
|
"26":"전자/반도체","27":"의료광학","28":"전기장비","29":"기계","30":"자동차",
|
|
"31":"기타운송장비","32":"가구","33":"기타제조","34":"산업기계수리","35":"전기/가스","36":"수도",
|
|
"37":"하수처리","38":"폐기물","39":"환경복원","41":"건설(종합)","42":"건설(전문)",
|
|
"45":"자동차판매","46":"도매","47":"소매","49":"육상운송","50":"수상운송",
|
|
"51":"항공운송","52":"창고/물류","55":"숙박","56":"음식/음료","58":"출판",
|
|
"59":"영상/음악","60":"방송","61":"통신","62":"IT서비스","63":"정보서비스",
|
|
"64":"금융","65":"보험","66":"금융보조","68":"부동산","70":"연구개발",
|
|
"71":"전문서비스","72":"건축/엔지니어링","73":"기술시험","74":"기타전문",
|
|
"75":"수의업","76":"임대","77":"인력공급","78":"여행","79":"경비/청소",
|
|
"80":"콜센터","82":"기타사업","84":"공공행정","85":"교육","86":"보건",
|
|
"87":"사회복지","90":"창작/예술","91":"스포츠/여가","96":"기타서비스",
|
|
}
|
|
|
|
async def collect_sectors():
|
|
"""DART company API로 업종 수집 (재무데이터 보유 종목 대상)
|
|
sector_name (KSIC 한글) + sector_code (KSIC 2자리) + sector (score-engine 읽음)에 저장
|
|
"""
|
|
logger.info("sectors.collecting")
|
|
async with pg_pool.acquire() as conn:
|
|
corps = await conn.fetch("""
|
|
SELECT stock_code, corp_code FROM dart_corps
|
|
WHERE is_active = true AND corp_code != ''
|
|
AND (sector IS NULL OR sector = '')
|
|
ORDER BY stock_code LIMIT 5000
|
|
""")
|
|
updated = 0
|
|
async with httpx.AsyncClient(timeout=15) as client:
|
|
for corp in corps:
|
|
try:
|
|
resp = await client.get(f"{DART_BASE}/company.json", params={
|
|
"crtfc_key": DART_API_KEY, "corp_code": corp["corp_code"]
|
|
})
|
|
d = resp.json()
|
|
if d.get("status") != "000":
|
|
continue
|
|
code = str(d.get("induty_code", "") or "")
|
|
sector = _KSIC_MAP.get(code[:2], "")
|
|
if not sector and code:
|
|
sector = code # 매핑 없으면 코드 그대로
|
|
if sector:
|
|
async with pg_pool.acquire() as conn:
|
|
await conn.execute(
|
|
"UPDATE dart_corps SET sector_name=$1, sector_code=$2, sector=$1 WHERE stock_code=$3",
|
|
sector, code, corp["stock_code"])
|
|
updated += 1
|
|
await asyncio.sleep(0.1)
|
|
except Exception as e:
|
|
logger.debug("sectors.corp_err", corp=corp["stock_code"], error=str(e))
|
|
logger.info("sectors.done", updated=updated)
|
|
return updated
|
|
|
|
|
|
# ══════════════════════════════════════════════════════════
|
|
# 임원·대주주 매매 수집
|
|
# ══════════════════════════════════════════════════════════
|
|
|
|
def _parse_int_safe(v) -> int:
|
|
"""'1,330' / '-205,875' / '-' → int"""
|
|
if not v or v == "-":
|
|
return 0
|
|
try:
|
|
return int(str(v).replace(",", "").replace(" ", ""))
|
|
except Exception:
|
|
return 0
|
|
|
|
|
|
def _parse_float_safe(v) -> float:
|
|
if not v or v == "-":
|
|
return 0.0
|
|
try:
|
|
return float(str(v).replace(",", "").replace("%", "").replace(" ", ""))
|
|
except Exception:
|
|
return 0.0
|
|
|
|
|
|
async def fetch_insider_for_corp(client: httpx.AsyncClient, corp_code: str
|
|
) -> tuple[list, list]:
|
|
"""elestock(임원) + majorstock(대량보유) 동시 조회"""
|
|
base = "https://opendart.fss.or.kr/api"
|
|
out_elestock, out_major = [], []
|
|
try:
|
|
r1 = await client.get(f"{base}/elestock.json", params={
|
|
"crtfc_key": DART_API_KEY, "corp_code": corp_code}, timeout=15)
|
|
j = r1.json()
|
|
if j.get("status") == "000":
|
|
out_elestock = j.get("list") or []
|
|
except Exception as e:
|
|
logger.debug("insider.elestock_err", corp=corp_code, err=str(e))
|
|
try:
|
|
r2 = await client.get(f"{base}/majorstock.json", params={
|
|
"crtfc_key": DART_API_KEY, "corp_code": corp_code}, timeout=15)
|
|
j = r2.json()
|
|
if j.get("status") == "000":
|
|
out_major = j.get("list") or []
|
|
except Exception as e:
|
|
logger.debug("insider.major_err", corp=corp_code, err=str(e))
|
|
return out_elestock, out_major
|
|
|
|
|
|
async def collect_insider_trades(count: int = 500):
|
|
"""시총 상위 N개 종목의 임원·대주주 매매 보고 수집.
|
|
DART rate-limit 고려해 1초당 2건 (분당 120)."""
|
|
async with pg_pool.acquire() as conn:
|
|
# 시총 상위 종목 (stock_prices 최신)
|
|
rows = await conn.fetch("""
|
|
SELECT DISTINCT ON (d.stock_code) d.stock_code, d.corp_code,
|
|
COALESCE(p.market_cap, 0) AS mc
|
|
FROM dart_corps d
|
|
LEFT JOIN stock_prices p ON p.stock_code=d.stock_code
|
|
WHERE d.is_active=true
|
|
ORDER BY d.stock_code, p.collected_at DESC NULLS LAST
|
|
""")
|
|
top = sorted(rows, key=lambda r: -r["mc"])[:count]
|
|
logger.info("insider.collect_start", count=len(top))
|
|
|
|
saved_e, saved_m = 0, 0
|
|
async with httpx.AsyncClient() as client:
|
|
for i, row in enumerate(top):
|
|
ele, maj = await fetch_insider_for_corp(client, row["corp_code"])
|
|
for r in ele:
|
|
try:
|
|
await conn.execute("""
|
|
INSERT INTO dart_insider_trades
|
|
(rcept_no, stock_code, corp_code, rcept_dt,
|
|
repror, ofcps, main_shrholdr,
|
|
shares_change, shares_after, rate_after, source)
|
|
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, 'elestock')
|
|
ON CONFLICT (rcept_no) DO NOTHING
|
|
""", r.get("rcept_no"), row["stock_code"], row["corp_code"],
|
|
datetime.strptime(r["rcept_dt"], "%Y-%m-%d").date()
|
|
if r.get("rcept_dt") else None,
|
|
r.get("repror", ""), r.get("isu_exctv_ofcps", ""),
|
|
r.get("isu_main_shrholdr", ""),
|
|
_parse_int_safe(r.get("sp_stock_lmp_irds_cnt")),
|
|
_parse_int_safe(r.get("sp_stock_lmp_cnt")),
|
|
_parse_float_safe(r.get("sp_stock_lmp_rate")))
|
|
saved_e += 1
|
|
except Exception as e:
|
|
logger.debug("insider.ele_save_err", err=str(e))
|
|
for r in maj:
|
|
try:
|
|
await conn.execute("""
|
|
INSERT INTO dart_insider_trades
|
|
(rcept_no, stock_code, corp_code, rcept_dt,
|
|
repror, ofcps, main_shrholdr,
|
|
shares_change, shares_after, rate_after, source)
|
|
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, 'majorstock')
|
|
ON CONFLICT (rcept_no) DO NOTHING
|
|
""", r.get("rcept_no"), row["stock_code"], row["corp_code"],
|
|
datetime.strptime(r["rcept_dt"], "%Y-%m-%d").date()
|
|
if r.get("rcept_dt") else None,
|
|
r.get("repror", ""), r.get("report_tp", ""),
|
|
"5%이상",
|
|
_parse_int_safe(r.get("stkqy_irds")),
|
|
_parse_int_safe(r.get("stkqy")),
|
|
_parse_float_safe(r.get("stkrt")))
|
|
saved_m += 1
|
|
except Exception as e:
|
|
logger.debug("insider.maj_save_err", err=str(e))
|
|
# DART rate limit: 분당 1000건 OK. 0.5초 = 분당 120건 안전
|
|
if i < len(top) - 1:
|
|
await asyncio.sleep(0.5)
|
|
logger.info("insider.collect_done", elestock=saved_e, majorstock=saved_m)
|
|
return {"elestock": saved_e, "majorstock": saved_m}
|
|
|
|
|
|
# ══════════════════════════════════════════════════════════
|
|
# DB 테이블 초기화
|
|
# ══════════════════════════════════════════════════════════
|
|
|
|
async def init_db():
|
|
"""필요한 테이블 생성"""
|
|
async with pg_pool.acquire() as conn:
|
|
await conn.execute("""
|
|
CREATE TABLE IF NOT EXISTS dart_corps (
|
|
stock_code VARCHAR(10) PRIMARY KEY,
|
|
corp_code VARCHAR(20) NOT NULL,
|
|
corp_name VARCHAR(200) NOT NULL,
|
|
modify_date VARCHAR(20) DEFAULT '',
|
|
is_active BOOLEAN DEFAULT FALSE
|
|
)
|
|
""")
|
|
try:
|
|
await conn.execute(
|
|
"ALTER TABLE dart_corps ADD COLUMN is_active BOOLEAN DEFAULT FALSE")
|
|
except: pass
|
|
for col in ["sector_name VARCHAR(100) DEFAULT ''",
|
|
"sector_code VARCHAR(20) DEFAULT ''"]:
|
|
try:
|
|
await conn.execute(f"ALTER TABLE dart_corps ADD COLUMN {col}")
|
|
except: pass
|
|
await conn.execute("""
|
|
CREATE TABLE IF NOT EXISTS dart_disclosures (
|
|
id SERIAL,
|
|
rcept_no VARCHAR(20) PRIMARY KEY,
|
|
rcept_dt VARCHAR(10),
|
|
corp_code VARCHAR(20),
|
|
corp_name VARCHAR(200),
|
|
stock_code VARCHAR(10),
|
|
corp_cls VARCHAR(5),
|
|
report_nm TEXT,
|
|
flr_nm VARCHAR(200),
|
|
pblntf_ty VARCHAR(5),
|
|
pblntf_detail_ty VARCHAR(5),
|
|
collected_at TIMESTAMP DEFAULT NOW()
|
|
)
|
|
""")
|
|
await conn.execute("CREATE INDEX IF NOT EXISTS idx_disc_stock ON dart_disclosures(stock_code)")
|
|
await conn.execute("CREATE INDEX IF NOT EXISTS idx_disc_date ON dart_disclosures(rcept_dt DESC)")
|
|
await conn.execute("CREATE INDEX IF NOT EXISTS idx_disc_type ON dart_disclosures(pblntf_ty)")
|
|
# 임원·대주주 매매 보고
|
|
await conn.execute("""
|
|
CREATE TABLE IF NOT EXISTS dart_insider_trades (
|
|
id SERIAL PRIMARY KEY,
|
|
rcept_no VARCHAR(20) UNIQUE,
|
|
stock_code VARCHAR(10),
|
|
corp_code VARCHAR(20),
|
|
rcept_dt DATE,
|
|
repror VARCHAR(200),
|
|
ofcps VARCHAR(100) DEFAULT '',
|
|
main_shrholdr VARCHAR(50) DEFAULT '',
|
|
shares_change BIGINT DEFAULT 0,
|
|
shares_after BIGINT DEFAULT 0,
|
|
rate_after FLOAT DEFAULT 0,
|
|
source VARCHAR(20),
|
|
collected_at TIMESTAMP DEFAULT NOW()
|
|
)
|
|
""")
|
|
await conn.execute("CREATE INDEX IF NOT EXISTS idx_insider_stock ON dart_insider_trades(stock_code, rcept_dt DESC)")
|
|
await conn.execute("CREATE INDEX IF NOT EXISTS idx_insider_date ON dart_insider_trades(rcept_dt DESC)")
|
|
await conn.execute("""
|
|
CREATE TABLE IF NOT EXISTS dart_financials (
|
|
id SERIAL,
|
|
stock_code VARCHAR(10) NOT NULL,
|
|
corp_code VARCHAR(20),
|
|
corp_name VARCHAR(200),
|
|
bsns_year VARCHAR(4) NOT NULL,
|
|
reprt_code VARCHAR(10) NOT NULL,
|
|
reprt_name VARCHAR(50),
|
|
revenue BIGINT DEFAULT 0,
|
|
operating_profit BIGINT DEFAULT 0,
|
|
net_income BIGINT DEFAULT 0,
|
|
total_assets BIGINT DEFAULT 0,
|
|
total_liabilities BIGINT DEFAULT 0,
|
|
total_equity BIGINT DEFAULT 0,
|
|
operating_cashflow BIGINT DEFAULT 0,
|
|
collected_at TIMESTAMP DEFAULT NOW(),
|
|
UNIQUE(stock_code, bsns_year, reprt_code)
|
|
)
|
|
""")
|
|
await conn.execute("CREATE INDEX IF NOT EXISTS idx_fin_stock ON dart_financials(stock_code)")
|
|
await conn.execute("CREATE INDEX IF NOT EXISTS idx_fin_year ON dart_financials(bsns_year DESC)")
|
|
|
|
await conn.execute("""
|
|
CREATE TABLE IF NOT EXISTS dart_dividends (
|
|
id SERIAL PRIMARY KEY,
|
|
stock_code VARCHAR(10) NOT NULL,
|
|
corp_name VARCHAR(200) DEFAULT '',
|
|
bsns_year VARCHAR(4) NOT NULL,
|
|
dps BIGINT DEFAULT 0,
|
|
dps_prev BIGINT DEFAULT 0,
|
|
dividend_yield FLOAT DEFAULT 0,
|
|
total_dividend BIGINT DEFAULT 0,
|
|
collected_at TIMESTAMP DEFAULT NOW(),
|
|
UNIQUE(stock_code, bsns_year)
|
|
)
|
|
""")
|
|
await conn.execute(
|
|
"CREATE INDEX IF NOT EXISTS idx_div_stock ON dart_dividends(stock_code)")
|
|
logger.info("db.initialized")
|
|
|
|
# ══════════════════════════════════════════════════════════
|
|
# FastAPI 앱
|
|
# ══════════════════════════════════════════════════════════
|
|
|
|
app = FastAPI(title="DART 공시 수집기", version="1.0.0")
|
|
app.add_middleware(CORSMiddleware, allow_origins=["*"], allow_methods=["*"], allow_headers=["*"])
|
|
|
|
|
|
@app.on_event("startup")
|
|
async def startup():
|
|
global pg_pool, redis_client
|
|
|
|
# PostgreSQL
|
|
pg_pool = await asyncpg.create_pool(
|
|
host=PG_HOST, port=PG_PORT, database=PG_DB,
|
|
user=PG_USER, password=PG_PASS, min_size=2, max_size=10,
|
|
)
|
|
await init_db()
|
|
|
|
# Redis
|
|
redis_client = aioredis.Redis(
|
|
host=REDIS_HOST, port=6379, password=REDIS_PASSWORD,
|
|
db=5, decode_responses=True,
|
|
)
|
|
|
|
# 기업코드 로딩
|
|
await load_corp_codes()
|
|
|
|
# 스케줄러
|
|
# 공시 수집: 평일 08:00~18:00 / 10분마다
|
|
scheduler.add_job(collect_recent_disclosures, "cron",
|
|
day_of_week="mon-fri", hour="8-18", minute="*/10",
|
|
id="disclosures", replace_existing=True)
|
|
|
|
# 주요사항보고: 평일 09:00, 12:00, 15:00
|
|
scheduler.add_job(collect_major_reports, "cron",
|
|
day_of_week="mon-fri", hour="9,12,15",
|
|
id="major_reports", replace_existing=True)
|
|
|
|
# 재무제표: 매일 새벽 3시
|
|
scheduler.add_job(collect_financials_for_top_stocks, "cron",
|
|
hour=3, id="financials", replace_existing=True)
|
|
|
|
# 배당: 매주 일요일 04:00 (사업보고서 공시 직후 4월/연중 갱신)
|
|
scheduler.add_job(collect_dividends, "cron",
|
|
day_of_week="sun", hour=4, id="dividends", replace_existing=True)
|
|
|
|
# 섹터: 매주 일요일 05:00
|
|
scheduler.add_job(collect_sectors, "cron",
|
|
day_of_week="sun", hour=5, id="sectors", replace_existing=True)
|
|
|
|
# 기업코드 갱신: 매일 새벽 2시
|
|
scheduler.add_job(load_corp_codes, "cron",
|
|
hour=2, id="corp_codes", replace_existing=True)
|
|
|
|
# 임원·대주주 매매: 매일 새벽 6시 (시총상위 500개)
|
|
scheduler.add_job(collect_insider_trades, "cron",
|
|
hour=6, id="insider", replace_existing=True)
|
|
|
|
scheduler.start()
|
|
|
|
# 주간 섹터 잡(일 05:00)이 컨테이너 다운/재배포로 누락되면 catch-up 없음 →
|
|
# 섹터 노후/빈값이 score-engine 전 시장 4종목 붕괴를 유발. 실패모드(낮은 채움률)를
|
|
# 직접 겨냥: startup 시 활성종목 섹터 채움률<90%면 1회 백필 (deep_batch와 동일 패턴).
|
|
async def _sectors_catchup():
|
|
async with pg_pool.acquire() as conn:
|
|
fill = await conn.fetchval("""
|
|
SELECT COALESCE(
|
|
COUNT(*) FILTER (WHERE sector IS NOT NULL AND sector <> '')::float
|
|
/ NULLIF(COUNT(*), 0), 0)
|
|
FROM dart_corps WHERE is_active=true
|
|
""")
|
|
if fill < 0.9:
|
|
logger.info("sectors.catchup_start", fill=round(float(fill), 3))
|
|
await collect_sectors()
|
|
asyncio.create_task(_sectors_catchup())
|
|
|
|
logger.info("dart.started", corps=len(corp_codes))
|
|
|
|
|
|
@app.on_event("shutdown")
|
|
async def shutdown():
|
|
scheduler.shutdown()
|
|
if pg_pool: await pg_pool.close()
|
|
if redis_client: await redis_client.aclose()
|
|
|
|
|
|
@app.get("/health")
|
|
async def health():
|
|
return JSONResponse(content={
|
|
"status": "ok",
|
|
"dart_api_key": "set" if DART_API_KEY else "missing",
|
|
"corps_loaded": stats.corps_loaded,
|
|
"disclosures_collected": stats.disclosures,
|
|
"financials_collected": stats.financials,
|
|
"last_run": stats.last_run,
|
|
"errors": stats.errors,
|
|
})
|
|
|
|
|
|
@app.post("/collect/disclosures")
|
|
async def manual_disclosures():
|
|
asyncio.create_task(collect_recent_disclosures())
|
|
return {"status": "started"}
|
|
|
|
|
|
@app.post("/collect/financials")
|
|
async def manual_financials(
|
|
count: int = Query(default=50),
|
|
years: int = Query(default=2, ge=1, le=20),
|
|
annual_only: bool = Query(default=False),
|
|
):
|
|
asyncio.create_task(collect_financials_for_top_stocks(count, years, annual_only))
|
|
return {"status": "started", "count": count, "years": years, "annual_only": annual_only}
|
|
|
|
|
|
@app.post("/collect/sectors")
|
|
async def manual_sectors():
|
|
n = await collect_sectors()
|
|
return {"status": "done", "saved": n}
|
|
|
|
|
|
@app.post("/collect/major")
|
|
async def manual_major():
|
|
asyncio.create_task(collect_major_reports())
|
|
return {"status": "started"}
|
|
|
|
|
|
@app.post("/collect/corps")
|
|
async def manual_corps():
|
|
await load_corp_codes()
|
|
return {"status": "ok", "count": len(corp_codes)}
|
|
|
|
|
|
@app.post("/collect/insider")
|
|
async def manual_insider(count: int = Query(default=500, ge=10, le=2000)):
|
|
asyncio.create_task(collect_insider_trades(count))
|
|
return {"status": "started", "count": count}
|
|
|
|
|
|
@app.get("/insider/stats")
|
|
async def insider_stats():
|
|
async with pg_pool.acquire() as conn:
|
|
r = await conn.fetchrow("""
|
|
SELECT COUNT(*) AS total,
|
|
COUNT(DISTINCT stock_code) AS stocks,
|
|
SUM(CASE WHEN shares_change > 0 THEN 1 ELSE 0 END) AS buys,
|
|
SUM(CASE WHEN shares_change < 0 THEN 1 ELSE 0 END) AS sells,
|
|
MAX(rcept_dt) AS latest
|
|
FROM dart_insider_trades
|
|
""")
|
|
return dict(r) if r else {}
|
|
|
|
|
|
@app.get("/insider/{stock_code}")
|
|
async def get_insider(stock_code: str, days: int = Query(default=90, ge=1, le=365)):
|
|
async with pg_pool.acquire() as conn:
|
|
rows = await conn.fetch("""
|
|
SELECT rcept_dt, repror, ofcps, main_shrholdr,
|
|
shares_change, shares_after, rate_after, source
|
|
FROM dart_insider_trades
|
|
WHERE stock_code=$1 AND rcept_dt >= CURRENT_DATE - $2::int
|
|
ORDER BY rcept_dt DESC LIMIT 100
|
|
""", stock_code, days)
|
|
return [dict(r) for r in rows]
|
|
|
|
|
|
@app.get("/corps")
|
|
async def list_corps(limit: int = Query(default=50)):
|
|
items = list(corp_codes.values())[:limit]
|
|
return JSONResponse(content={"count": len(corp_codes), "items": items})
|
|
|
|
|
|
@app.get("/corps/{stock_code}")
|
|
async def get_corp(stock_code: str):
|
|
info = corp_codes.get(stock_code)
|
|
if not info:
|
|
raise HTTPException(404, "종목 없음")
|
|
return JSONResponse(content=info)
|
|
|
|
|
|
@app.get("/disclosures")
|
|
async def list_disclosures(
|
|
stock_code: str = Query(default=""),
|
|
days: int = Query(default=7),
|
|
limit: int = Query(default=50),
|
|
):
|
|
async with pg_pool.acquire() as conn:
|
|
if stock_code:
|
|
rows = await conn.fetch("""
|
|
SELECT * FROM dart_disclosures
|
|
WHERE stock_code = $1 AND rcept_dt >= $2
|
|
ORDER BY rcept_dt DESC LIMIT $3
|
|
""", stock_code, (datetime.now() - timedelta(days=days)).strftime("%Y%m%d"), limit)
|
|
else:
|
|
rows = await conn.fetch("""
|
|
SELECT * FROM dart_disclosures
|
|
WHERE rcept_dt >= $1
|
|
ORDER BY rcept_dt DESC LIMIT $2
|
|
""", (datetime.now() - timedelta(days=days)).strftime("%Y%m%d"), limit)
|
|
return [dict(r) for r in rows]
|
|
|
|
|
|
@app.get("/financials/{stock_code}")
|
|
async def get_financials(stock_code: str):
|
|
async with pg_pool.acquire() as conn:
|
|
rows = await conn.fetch("""
|
|
SELECT * FROM dart_financials
|
|
WHERE stock_code = $1
|
|
ORDER BY bsns_year DESC, reprt_code DESC
|
|
""", stock_code)
|
|
return [dict(r) for r in rows]
|
|
|
|
|
|
@app.post("/collect/dividends")
|
|
async def manual_dividends(count: int = Query(default=200)):
|
|
n = await collect_dividends(count)
|
|
return {"status": "done", "saved": n}
|
|
|
|
@app.post("/collect/sectors")
|
|
async def manual_sectors():
|
|
asyncio.create_task(collect_sectors())
|
|
return {"status": "started"}
|
|
|
|
@app.get("/dividends/{stock_code}")
|
|
async def get_dividends(stock_code: str):
|
|
async with pg_pool.acquire() as conn:
|
|
rows = await conn.fetch(
|
|
"SELECT * FROM dart_dividends WHERE stock_code=$1 ORDER BY bsns_year DESC", stock_code)
|
|
return [dict(r) for r in rows]
|
|
|
|
@app.get("/stats")
|
|
async def get_stats():
|
|
counts = {}
|
|
if pg_pool:
|
|
async with pg_pool.acquire() as conn:
|
|
counts["corps"] = await conn.fetchval("SELECT COUNT(*) FROM dart_corps")
|
|
counts["disclosures"] = await conn.fetchval("SELECT COUNT(*) FROM dart_disclosures")
|
|
counts["financials"] = await conn.fetchval("SELECT COUNT(*) FROM dart_financials")
|
|
counts["news_from_dart"] = await conn.fetchval(
|
|
"SELECT COUNT(*) FROM news_analysis WHERE source = 'DART공시'"
|
|
)
|
|
return JSONResponse(content=counts)
|