kyu
c2bed9102a
[소스 다변화] 라이브 검증(200+한글기사)된 10개 피드 추가: 연합인포맥스(채권·FX)·매경증권·한경 글로벌마켓·연합뉴스 산업·데일리안· ZDNet코리아·테크M·전자신문 반도체·오토헤럴드·철강금속신문. 마켓/FX·IT·반도체·자동차·철강 버티컬로 커버리지 확대. 검증: 37소스 라이브 크롤 360건, fetch 에러 0. [종목 귀속 정확성] extract_stocks의 한글명 부분일치(text.count) 오탐 차단. 일상어와 겹치는 모호 종목명 19개(대상·동양·동서·미래 등)는 본문에 주식 맥락 토큰(주가·실적·영업이익·코스피·㈜·반도체 등)이 있을 때만 인정. 보령·풍산·세방 등 고유명은 제외(누락 방지). 검증: '지원 대상으로'→무매칭, '대상 영업이익 증가'→대상 인정, 삼성전자 영향없음. 두 서비스 재빌드·재기동·라이브검증 완료. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
269 lines
11 KiB
Python
269 lines
11 KiB
Python
"""
|
|
바른 API FastAPI 서버 v2
|
|
- 서버 시작 시 KRX 전체 종목 동적 로딩
|
|
- 24시간마다 자동 갱신
|
|
- Bareun gRPC 연결
|
|
"""
|
|
import asyncio, hashlib, os, re, time
|
|
from contextlib import asynccontextmanager
|
|
from typing import Optional
|
|
import orjson, redis.asyncio as aioredis, structlog
|
|
from bareunpy import Tagger
|
|
from fastapi import FastAPI, HTTPException, Response
|
|
from fastapi.responses import JSONResponse
|
|
from fastapi.middleware.cors import CORSMiddleware
|
|
from prometheus_fastapi_instrumentator import Instrumentator
|
|
from pydantic import BaseModel, Field
|
|
from stock_loader import load_all_stocks, auto_refresh_stocks
|
|
from finance_dict import ALL_FINANCE_TERMS, HIGH_PRIORITY_TERMS, FINANCE_VERBS as FD_VERBS
|
|
|
|
structlog.configure(processors=[
|
|
structlog.processors.TimeStamper(fmt="iso"),
|
|
structlog.processors.add_log_level,
|
|
structlog.processors.JSONRenderer(),
|
|
])
|
|
logger = structlog.get_logger()
|
|
|
|
BAREUN_API_KEY = os.getenv("BAREUN_API_KEY", "")
|
|
BAREUN_SERVER_HOST = os.getenv("BAREUN_SERVER_HOST", "bareun")
|
|
BAREUN_SERVER_PORT = int(os.getenv("BAREUN_SERVER_PORT", "5656"))
|
|
REDIS_HOST = os.getenv("REDIS_HOST", "redis")
|
|
REDIS_PORT = int(os.getenv("REDIS_PORT", "6379"))
|
|
REDIS_PASSWORD = os.getenv("REDIS_PASSWORD", "")
|
|
REDIS_DB = int(os.getenv("REDIS_DB", "1"))
|
|
NEWS_DEDUP_TTL = int(os.getenv("NEWS_DEDUP_TTL", "86400"))
|
|
|
|
STOPWORDS = {
|
|
"것","수","등","및","또","또한","이","그","저","위","아래","관련","통해",
|
|
"위해","대해","따라","때문","이후","이전","현재","최근","지난","올해","내년",
|
|
"이번","오늘","어제","내일","이날","같은","다른","많은","더","가장","매우",
|
|
"이미","아직","모든","각","전체","일부","국내","해외","글로벌","세계","한국",
|
|
"미국","중국","일본","유럽","가운데","한편","다만","특히","실제","여전히",
|
|
"앞으로","지속","계속","자체","관계자","측","쪽","곳","점","경우","상황",
|
|
"때","중","간",
|
|
}
|
|
FINANCE_KEYWORDS = ALL_FINANCE_TERMS
|
|
FINANCE_VERBS = {
|
|
"급등","급락","폭등","폭락","상승","하락","반등","하락세","상승세",
|
|
"돌파","이탈","회복","저항","지지","돌파구","매집","매도","매수",
|
|
"초과달성","하회","상회","달성","부진","급성장","감소","증가","개선","악화",
|
|
} | FD_VERBS
|
|
START_TIME = time.time()
|
|
|
|
class AppState:
|
|
tagger: Optional[Tagger] = None
|
|
redis: Optional[aioredis.Redis] = None
|
|
stock_map: dict[str, str] = {}
|
|
stock_count: int = 0
|
|
refresh_task: Optional[asyncio.Task] = None
|
|
|
|
state = AppState()
|
|
|
|
@asynccontextmanager
|
|
async def lifespan(app: FastAPI):
|
|
state.stock_map = await load_all_stocks()
|
|
state.stock_count = len(state.stock_map)
|
|
logger.info("stocks.ready", count=state.stock_count)
|
|
state.refresh_task = asyncio.create_task(auto_refresh_stocks(state, 24))
|
|
try:
|
|
state.tagger = Tagger(BAREUN_API_KEY, BAREUN_SERVER_HOST, BAREUN_SERVER_PORT)
|
|
logger.info("tagger.ok")
|
|
except Exception as e:
|
|
logger.warning("tagger.failed", error=str(e))
|
|
try:
|
|
state.redis = aioredis.Redis(host=REDIS_HOST, port=REDIS_PORT,
|
|
password=REDIS_PASSWORD, db=REDIS_DB, decode_responses=True,
|
|
socket_connect_timeout=5, retry_on_timeout=True)
|
|
await state.redis.ping()
|
|
logger.info("redis.ok")
|
|
except Exception as e:
|
|
logger.error("redis.failed", error=str(e))
|
|
yield
|
|
if state.refresh_task: state.refresh_task.cancel()
|
|
if state.redis: await state.redis.aclose()
|
|
|
|
app = FastAPI(title="바른 API v2", version="2.0.0", lifespan=lifespan)
|
|
app.add_middleware(CORSMiddleware, allow_origins=["*"], allow_methods=["*"], allow_headers=["*"])
|
|
Instrumentator().instrument(app).expose(app, endpoint="/metrics")
|
|
|
|
class AnalyzeRequest(BaseModel):
|
|
title: str; content: str = ""; url: str = ""; source: str = ""; published_at: str = ""
|
|
|
|
class StockMention(BaseModel):
|
|
name: str; code: str; count: int
|
|
|
|
class AnalyzeResponse(BaseModel):
|
|
hash: str; is_duplicate: bool; stocks: list[StockMention]; keywords: list[str]
|
|
filtered_text: str; token_count: int; processing_time_ms: float
|
|
|
|
class BatchRequest(BaseModel):
|
|
items: list[AnalyzeRequest]
|
|
|
|
def news_hash(title, url):
|
|
return hashlib.sha256(f"{title.strip()}{url.strip()}".encode()).hexdigest()[:16]
|
|
|
|
async def is_duplicate(h):
|
|
if not state.redis: return False
|
|
try:
|
|
r = await state.redis.set(f"news:dedup:{h}", "1", ex=NEWS_DEDUP_TTL, nx=True)
|
|
return r is None
|
|
except: return False
|
|
|
|
def extract_morphemes(text):
|
|
if not state.tagger or not text.strip():
|
|
return [w for w in text.split() if len(w) >= 2 and w not in STOPWORDS]
|
|
try:
|
|
result = []
|
|
for t, p in state.tagger.pos(text):
|
|
if len(t) < 2 or t in STOPWORDS:
|
|
continue
|
|
if p in ("NNG", "NNP", "SL"):
|
|
result.append(t)
|
|
elif p in ("VV", "VA", "XR") and t in FINANCE_VERBS:
|
|
result.append(t)
|
|
return result
|
|
except:
|
|
return [w for w in text.split() if len(w) >= 2 and w not in STOPWORDS]
|
|
|
|
# 일상어와 철자가 겹쳐 오탐이 잦은 짧은 종목명 — 본문에 주식 맥락 토큰이 같이 있을 때만 인정.
|
|
# (예: "지원 대상으로 선정"→대상㈜ 오탐, "동양적 가치관"→동양 오탐 차단)
|
|
# 보령·대덕·풍산·세방처럼 일상어로 거의 안 쓰이는 고유 종목명은 넣지 않음(오히려 누락 유발).
|
|
AMBIGUOUS_NAMES = {
|
|
"대상", "동양", "동서", "미래", "대한", "고려", "한일", "서울", "부산",
|
|
"동국", "유성", "남성", "영원", "태양", "대성", "백산", "신성", "한창", "우성",
|
|
}
|
|
_STOCK_CTX = (
|
|
"주가", "주식", "증시", "상장", "상폐", "공시", "영업이익", "매출", "순이익",
|
|
"실적", "수주", "계약", "배당", "자사주", "유상증자", "무상증자", "목표주가",
|
|
"코스피", "코스닥", "급등", "급락", "상한가", "하한가", "㈜", "그룹", "지주",
|
|
"전자", "화학", "제약", "바이오", "반도체", "자동차", "건설", "증권", "철강",
|
|
)
|
|
|
|
def extract_stocks(text):
|
|
has_ctx = any(m in text for m in _STOCK_CTX)
|
|
found = {}
|
|
for name, code in state.stock_map.items():
|
|
if len(name) < 2:
|
|
continue # 1글자 종목명은 오탐 과다 → 제외
|
|
# 모호 종목명은 주식 맥락 토큰이 본문에 있을 때만 인정 (일상어 오탐 차단)
|
|
if name in AMBIGUOUS_NAMES and not has_ctx:
|
|
continue
|
|
if name.isascii():
|
|
# 영문/숫자 약칭(KT·SK·DB 등)은 단어경계 강제 (SKT·KTX 오탐 차단)
|
|
pat = rf"(?<![A-Za-z0-9]){re.escape(name)}(?![A-Za-z0-9])"
|
|
c = len(re.findall(pat, text))
|
|
else:
|
|
c = text.count(name) # 한글명: 교착어 특성상 부분일치 유지
|
|
if c > 0:
|
|
found[name] = StockMention(name=name, code=code, count=c)
|
|
# 더 긴 종목명에 포함된 짧은 종목명 제거 ('한국' ⊂ '한국전력', 'KT' ⊂ 'KT&G')
|
|
names = list(found)
|
|
for n in names:
|
|
if any(n != m and n in m for m in names):
|
|
found.pop(n, None)
|
|
return sorted(found.values(), key=lambda x: x.count, reverse=True)
|
|
|
|
def build_filtered(nouns, stocks):
|
|
sn = {s.name for s in stocks}
|
|
seen = set()
|
|
result = []
|
|
# 1순위: 고중요도 금융 이벤트 (어닝서프라이즈, 상한가 등)
|
|
for n in nouns:
|
|
if n in HIGH_PRIORITY_TERMS and n not in seen:
|
|
result.append(n); seen.add(n)
|
|
# 2순위: 종목명
|
|
for n in nouns:
|
|
if n in sn and n not in seen:
|
|
result.append(n); seen.add(n)
|
|
# 3순위: 전문 금융 용어 사전
|
|
for n in nouns:
|
|
if n in FINANCE_KEYWORDS and n not in seen:
|
|
result.append(n); seen.add(n)
|
|
# 4순위: 나머지 의미있는 명사
|
|
for n in nouns:
|
|
if n not in STOPWORDS and len(n) >= 2 and n not in seen:
|
|
result.append(n); seen.add(n)
|
|
return " ".join(result[:120])
|
|
|
|
def scan_finance_terms(text: str) -> list[str]:
|
|
"""형태소 분석 없이 원문에서 금융 전문 용어 직접 탐색 (복합어 대응)"""
|
|
found = []
|
|
for term in ALL_FINANCE_TERMS:
|
|
if term in text:
|
|
found.append(term)
|
|
return found
|
|
|
|
def _analyze(req):
|
|
text = f"{req.title} {req.content}".strip()
|
|
h = news_hash(req.title, req.url)
|
|
nouns = extract_morphemes(text)
|
|
# 원문 직접 스캔으로 형태소 분석이 놓친 복합어 추가
|
|
direct_terms = scan_finance_terms(text)
|
|
nouns = list(dict.fromkeys(nouns + direct_terms))
|
|
stocks = extract_stocks(text)
|
|
kw = list(dict.fromkeys(n for n in nouns if len(n) >= 2))[:50]
|
|
ft = build_filtered(nouns, stocks)
|
|
return h, stocks, kw, ft
|
|
|
|
@app.get("/health")
|
|
async def health():
|
|
tok = state.tagger is not None
|
|
rok = False
|
|
if state.redis:
|
|
try: await state.redis.ping(); rok = True
|
|
except: pass
|
|
return JSONResponse(content={"status": "ok" if tok else "degraded",
|
|
"tagger": "ok" if tok else "unavailable", "redis": "ok" if rok else "error",
|
|
"stocks_loaded": state.stock_count, "uptime": round(time.time()-START_TIME,1)})
|
|
|
|
@app.post("/analyze")
|
|
async def analyze(req: AnalyzeRequest):
|
|
t = time.perf_counter()
|
|
h, stocks, kw, ft = _analyze(req)
|
|
dup = await is_duplicate(h)
|
|
ms = round((time.perf_counter()-t)*1000, 2)
|
|
return Response(content=orjson.dumps(AnalyzeResponse(
|
|
hash=h, is_duplicate=dup, stocks=stocks, keywords=kw,
|
|
filtered_text=ft, token_count=len(ft.split()), processing_time_ms=ms
|
|
).model_dump()), media_type="application/json")
|
|
|
|
@app.post("/analyze/batch")
|
|
async def analyze_batch(req: BatchRequest):
|
|
if len(req.items) > 50: raise HTTPException(400, "최대 50개")
|
|
t = time.perf_counter()
|
|
results = []
|
|
for item in req.items:
|
|
try:
|
|
h, stocks, kw, ft = _analyze(item)
|
|
dup = await is_duplicate(h)
|
|
results.append({"title":item.title,"hash":h,"is_duplicate":dup,
|
|
"stocks":[s.model_dump() for s in stocks],"keywords":kw,
|
|
"filtered_text":ft,"token_count":len(ft.split())})
|
|
except Exception as e:
|
|
results.append({"title":item.title,"error":str(e),"is_duplicate":False})
|
|
ms = round((time.perf_counter()-t)*1000, 2)
|
|
dups = sum(1 for r in results if r.get("is_duplicate"))
|
|
return Response(content=orjson.dumps({"total":len(results),"duplicates":dups,
|
|
"processed":len(results)-dups,"elapsed_ms":ms,"results":results}),
|
|
media_type="application/json")
|
|
|
|
@app.get("/stocks")
|
|
async def stocks_list():
|
|
return JSONResponse(content={"count":len(state.stock_map),
|
|
"stocks":[{"name":k,"code":v} for k,v in list(state.stock_map.items())[:500]]})
|
|
|
|
@app.post("/stocks/refresh")
|
|
async def refresh():
|
|
m = await load_all_stocks()
|
|
if len(m) > 50:
|
|
state.stock_map = m; state.stock_count = len(m)
|
|
return JSONResponse(content={"status":"ok","count":len(m)})
|
|
raise HTTPException(500, "종목 로딩 실패")
|
|
|
|
@app.delete("/dedup/flush")
|
|
async def flush():
|
|
if not state.redis: raise HTTPException(503)
|
|
keys = await state.redis.keys("news:dedup:*")
|
|
if keys: await state.redis.delete(*keys)
|
|
return {"deleted": len(keys)}
|