Compare commits
3 Commits
8d50a345a9
...
f87756ee2e
| Author | SHA1 | Date | |
|---|---|---|---|
| f87756ee2e | |||
| da36454e21 | |||
| 3db047bade |
Vendored
+57
@@ -0,0 +1,57 @@
|
|||||||
|
pipeline {
|
||||||
|
agent any
|
||||||
|
|
||||||
|
options {
|
||||||
|
timestamps()
|
||||||
|
buildDiscarder(logRotator(numToKeepStr: '20'))
|
||||||
|
timeout(time: 10, unit: 'MINUTES')
|
||||||
|
}
|
||||||
|
|
||||||
|
stages {
|
||||||
|
stage('Checkout info') {
|
||||||
|
steps {
|
||||||
|
script {
|
||||||
|
echo "Branch: ${env.BRANCH_NAME ?: 'n/a'}"
|
||||||
|
echo "Commit: ${env.GIT_COMMIT ?: 'n/a'}"
|
||||||
|
echo "Build : #${env.BUILD_NUMBER} on ${env.NODE_NAME}"
|
||||||
|
}
|
||||||
|
sh 'git log -1 --pretty=fuller || true'
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
stage('Repo layout') {
|
||||||
|
steps {
|
||||||
|
sh '''
|
||||||
|
echo "=== top-level ==="
|
||||||
|
ls -la | head -30
|
||||||
|
echo
|
||||||
|
echo "=== compose files ==="
|
||||||
|
ls -la docker-compose*.yml 2>/dev/null || true
|
||||||
|
'''
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
stage('Validate compose') {
|
||||||
|
steps {
|
||||||
|
sh '''
|
||||||
|
if command -v docker >/dev/null 2>&1; then
|
||||||
|
echo "docker version:"
|
||||||
|
docker --version
|
||||||
|
echo
|
||||||
|
echo "=== compose config (syntax only, no apply) ==="
|
||||||
|
docker compose -f docker-compose.yml config --quiet \
|
||||||
|
&& echo "OK: docker-compose.yml is valid" \
|
||||||
|
|| echo "WARN: compose validation failed (non-fatal for now)"
|
||||||
|
else
|
||||||
|
echo "docker not available in this build environment, skipping"
|
||||||
|
fi
|
||||||
|
'''
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
post {
|
||||||
|
success { echo "SUCCESS — no deployment performed (by design)" }
|
||||||
|
failure { echo "FAILED — check logs" }
|
||||||
|
}
|
||||||
|
}
|
||||||
+11
-5
@@ -656,7 +656,8 @@ async def job_historical_raw(count: int = 0, max_pages: int = 100):
|
|||||||
|
|
||||||
|
|
||||||
async def job_process_raw(batch_size: int = 200):
|
async def job_process_raw(batch_size: int = 200):
|
||||||
"""news_raw의 미처리 행을 batch_size만큼 EXAONE 분석 → news_analysis로 이동"""
|
"""news_raw의 미처리 행을 batch_size만큼 EXAONE 분석 → news_analysis로 이동.
|
||||||
|
OLLAMA_NUM_PARALLEL=4와 매칭되는 4-동시 병렬 처리."""
|
||||||
async with pg_pool.acquire() as conn:
|
async with pg_pool.acquire() as conn:
|
||||||
rows = await conn.fetch("""
|
rows = await conn.fetch("""
|
||||||
SELECT id, stock_code, title, url, source, published_at_text
|
SELECT id, stock_code, title, url, source, published_at_text
|
||||||
@@ -666,8 +667,11 @@ async def job_process_raw(batch_size: int = 200):
|
|||||||
if not rows:
|
if not rows:
|
||||||
logger.info("process_raw.empty"); return 0
|
logger.info("process_raw.empty"); return 0
|
||||||
ok = 0
|
ok = 0
|
||||||
|
sem = asyncio.Semaphore(4)
|
||||||
async with httpx.AsyncClient() as c:
|
async with httpx.AsyncClient() as c:
|
||||||
for r in rows:
|
async def _one(r):
|
||||||
|
nonlocal ok
|
||||||
|
async with sem:
|
||||||
item = {
|
item = {
|
||||||
"title": r["title"], "url": r["url"],
|
"title": r["title"], "url": r["url"],
|
||||||
"source": r["source"] or "네이버금융",
|
"source": r["source"] or "네이버금융",
|
||||||
@@ -691,6 +695,7 @@ async def job_process_raw(batch_size: int = 200):
|
|||||||
except Exception as e:
|
except Exception as e:
|
||||||
stats.errors += 1
|
stats.errors += 1
|
||||||
logger.warning("process_raw.err", id=r["id"], error=str(e))
|
logger.warning("process_raw.err", id=r["id"], error=str(e))
|
||||||
|
await asyncio.gather(*(_one(r) for r in rows))
|
||||||
logger.info("process_raw.batch_done", batch=len(rows), processed_ok=ok)
|
logger.info("process_raw.batch_done", batch=len(rows), processed_ok=ok)
|
||||||
return ok
|
return ok
|
||||||
|
|
||||||
@@ -818,9 +823,10 @@ async def startup():
|
|||||||
scheduler.add_job(job_rss,"cron",day_of_week="sat,sun",hour="8-22",minute="*/15",id="rss_weekend",replace_existing=True)
|
scheduler.add_job(job_rss,"cron",day_of_week="sat,sun",hour="8-22",minute="*/15",id="rss_weekend",replace_existing=True)
|
||||||
# 평일 마켓 (네이버 금융): 9-17시 10분마다
|
# 평일 마켓 (네이버 금융): 9-17시 10분마다
|
||||||
scheduler.add_job(job_market,"cron",day_of_week="mon-fri",hour="9-17",minute="*/10",id="market",replace_existing=True)
|
scheduler.add_job(job_market,"cron",day_of_week="mon-fri",hour="9-17",minute="*/10",id="market",replace_existing=True)
|
||||||
# raw 뉴스 분석: 24시간 30분마다 batch 200건 처리 (백로그 소화용)
|
# raw 뉴스 분석: 30분마다 batch 200건 처리 (병렬화 4-동시로 30분 내 완주)
|
||||||
scheduler.add_job(job_process_raw,"cron",hour="*",minute="*/30",
|
scheduler.add_job(job_process_raw,"cron",minute="*/30",
|
||||||
id="process_raw",replace_existing=True,kwargs={"batch_size":200})
|
id="process_raw",replace_existing=True,
|
||||||
|
max_instances=1, kwargs={"batch_size":200})
|
||||||
# 매주 일요일 새벽 2시 raw 백필 (전체 활성종목)
|
# 매주 일요일 새벽 2시 raw 백필 (전체 활성종목)
|
||||||
scheduler.add_job(job_historical_raw,"cron",day_of_week="sun",hour="2",minute="0",
|
scheduler.add_job(job_historical_raw,"cron",day_of_week="sun",hour="2",minute="0",
|
||||||
id="historical_raw_weekly",replace_existing=True,
|
id="historical_raw_weekly",replace_existing=True,
|
||||||
|
|||||||
+899
-183
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user