rebuild(runtime): govern market operations and job truth
This commit is contained in:
leefer
@@ -0,0 +1,25 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
|
||||
from backend.features.operations.service import OperationsService
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
async def run_operations_scheduler(
|
||||
service: OperationsService, stop: asyncio.Event
|
||||
) -> None:
|
||||
while not stop.is_set():
|
||||
try:
|
||||
await asyncio.to_thread(service.tick)
|
||||
except Exception:
|
||||
logger.exception(
|
||||
"Operations scheduler tick failed",
|
||||
extra={"event": "operations.scheduler.failed"},
|
||||
)
|
||||
try:
|
||||
await asyncio.wait_for(stop.wait(), timeout=5)
|
||||
except TimeoutError:
|
||||
pass
|
||||
@@ -0,0 +1,154 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import sqlite3
|
||||
from datetime import datetime, timedelta
|
||||
from typing import Any
|
||||
|
||||
|
||||
class JobRepository:
|
||||
def begin(
|
||||
self,
|
||||
connection: sqlite3.Connection,
|
||||
*,
|
||||
kind: str,
|
||||
run_key: str,
|
||||
requested_date: str,
|
||||
trigger: str,
|
||||
started_at: datetime,
|
||||
stale_after_seconds: int,
|
||||
) -> sqlite3.Row:
|
||||
stale_before = (started_at - timedelta(seconds=stale_after_seconds)).isoformat(
|
||||
timespec="seconds"
|
||||
)
|
||||
connection.execute(
|
||||
"""
|
||||
UPDATE job_runs SET status = 'failed', finished_at = ?, duration_ms = ?,
|
||||
error_code = 'stale_job', error_message = '任务进程中断或超过最长运行时间'
|
||||
WHERE kind = ? AND status = 'running' AND started_at < ?
|
||||
""",
|
||||
(
|
||||
started_at.isoformat(timespec="seconds"),
|
||||
stale_after_seconds * 1000,
|
||||
kind,
|
||||
stale_before,
|
||||
),
|
||||
)
|
||||
attempt_row = connection.execute(
|
||||
"""
|
||||
SELECT COALESCE(MAX(attempt), 0) + 1 AS attempt
|
||||
FROM job_runs WHERE kind = ? AND run_key = ?
|
||||
""",
|
||||
(kind, run_key),
|
||||
).fetchone()
|
||||
attempt = int(attempt_row["attempt"] if attempt_row else 1)
|
||||
cursor = connection.execute(
|
||||
"""
|
||||
INSERT INTO job_runs (
|
||||
kind, run_key, requested_date, trigger, status, attempt, started_at
|
||||
) VALUES (?, ?, ?, ?, 'running', ?, ?)
|
||||
""",
|
||||
(
|
||||
kind,
|
||||
run_key,
|
||||
requested_date,
|
||||
trigger,
|
||||
attempt,
|
||||
started_at.isoformat(timespec="seconds"),
|
||||
),
|
||||
)
|
||||
return connection.execute(
|
||||
"SELECT * FROM job_runs WHERE id = ?", (cursor.lastrowid,)
|
||||
).fetchone()
|
||||
|
||||
def finish(
|
||||
self,
|
||||
connection: sqlite3.Connection,
|
||||
run_id: int,
|
||||
*,
|
||||
finished_at: datetime,
|
||||
duration_ms: int,
|
||||
coverage: float | None,
|
||||
source_set: list[str],
|
||||
output_version: str,
|
||||
payload: dict[str, Any],
|
||||
) -> None:
|
||||
connection.execute(
|
||||
"""
|
||||
UPDATE job_runs SET status = 'completed', finished_at = ?, duration_ms = ?,
|
||||
coverage = ?, source_set_json = ?, output_version = ?, payload_json = ?
|
||||
WHERE id = ? AND status = 'running'
|
||||
""",
|
||||
(
|
||||
finished_at.isoformat(timespec="seconds"),
|
||||
duration_ms,
|
||||
coverage,
|
||||
_json(source_set),
|
||||
output_version,
|
||||
_json(payload),
|
||||
run_id,
|
||||
),
|
||||
)
|
||||
|
||||
def fail(
|
||||
self,
|
||||
connection: sqlite3.Connection,
|
||||
run_id: int,
|
||||
*,
|
||||
finished_at: datetime,
|
||||
duration_ms: int,
|
||||
error_code: str,
|
||||
error_message: str,
|
||||
) -> None:
|
||||
connection.execute(
|
||||
"""
|
||||
UPDATE job_runs SET status = 'failed', finished_at = ?, duration_ms = ?,
|
||||
error_code = ?, error_message = ? WHERE id = ? AND status = 'running'
|
||||
""",
|
||||
(
|
||||
finished_at.isoformat(timespec="seconds"),
|
||||
duration_ms,
|
||||
error_code,
|
||||
error_message[:500],
|
||||
run_id,
|
||||
),
|
||||
)
|
||||
|
||||
def latest(self, connection: sqlite3.Connection, limit: int = 40) -> tuple[sqlite3.Row, ...]:
|
||||
return tuple(
|
||||
connection.execute(
|
||||
"SELECT * FROM job_runs ORDER BY started_at DESC, id DESC LIMIT ?",
|
||||
(limit,),
|
||||
).fetchall()
|
||||
)
|
||||
|
||||
def latest_for_kind(
|
||||
self, connection: sqlite3.Connection, kind: str
|
||||
) -> sqlite3.Row | None:
|
||||
return connection.execute(
|
||||
"SELECT * FROM job_runs WHERE kind = ? ORDER BY started_at DESC, id DESC LIMIT 1",
|
||||
(kind,),
|
||||
).fetchone()
|
||||
|
||||
def latest_success(
|
||||
self, connection: sqlite3.Connection, kind: str, requested_date: str = ""
|
||||
) -> sqlite3.Row | None:
|
||||
return connection.execute(
|
||||
"""
|
||||
SELECT * FROM job_runs
|
||||
WHERE kind = ? AND status = 'completed' AND (? = '' OR requested_date = ?)
|
||||
ORDER BY finished_at DESC, id DESC LIMIT 1
|
||||
""",
|
||||
(kind, requested_date, requested_date),
|
||||
).fetchone()
|
||||
|
||||
|
||||
def public_job(row: sqlite3.Row) -> dict[str, Any]:
|
||||
result = dict(row)
|
||||
result["source_set"] = json.loads(str(result.pop("source_set_json")))
|
||||
result["payload"] = json.loads(str(result.pop("payload_json")))
|
||||
return result
|
||||
|
||||
|
||||
def _json(value: Any) -> str:
|
||||
return json.dumps(value, ensure_ascii=False, separators=(",", ":"), sort_keys=True)
|
||||
@@ -1,33 +0,0 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
|
||||
from backend.features.screener.service import ScreenerError, ScreenerService
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
async def run_screener_scheduler(service: ScreenerService, stop: asyncio.Event) -> None:
|
||||
while not stop.is_set():
|
||||
try:
|
||||
result = await asyncio.to_thread(service.run_after_close)
|
||||
if result:
|
||||
logger.info(
|
||||
"Screener after-close run completed",
|
||||
extra={"event": "screener.completed", "context": result},
|
||||
)
|
||||
except ScreenerError as exc:
|
||||
logger.info(
|
||||
"Screener is waiting for complete market data",
|
||||
extra={"event": "screener.waiting", "context": {"reason": str(exc)}},
|
||||
)
|
||||
except Exception:
|
||||
logger.exception(
|
||||
"Screener scheduler failed",
|
||||
extra={"event": "screener.failed"},
|
||||
)
|
||||
try:
|
||||
await asyncio.wait_for(stop.wait(), timeout=300)
|
||||
except TimeoutError:
|
||||
pass
|
||||
@@ -0,0 +1,122 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import sqlite3
|
||||
from collections.abc import Callable
|
||||
from datetime import datetime
|
||||
from typing import Any
|
||||
from zoneinfo import ZoneInfo
|
||||
|
||||
from backend.database.connection import Database
|
||||
from backend.jobs.repository import JobRepository, public_job
|
||||
|
||||
SHANGHAI = ZoneInfo("Asia/Shanghai")
|
||||
|
||||
|
||||
class JobAlreadyRunning(RuntimeError):
|
||||
pass
|
||||
|
||||
|
||||
class JobService:
|
||||
def __init__(self, database: Database, repository: JobRepository) -> None:
|
||||
self._database = database
|
||||
self._repository = repository
|
||||
|
||||
def execute(
|
||||
self,
|
||||
*,
|
||||
kind: str,
|
||||
run_key: str,
|
||||
requested_date: str,
|
||||
trigger: str,
|
||||
operation: Callable[[], dict[str, Any]],
|
||||
stale_after_seconds: int,
|
||||
) -> dict[str, Any]:
|
||||
started = datetime.now(SHANGHAI)
|
||||
try:
|
||||
with self._database.transaction() as connection:
|
||||
row = self._repository.begin(
|
||||
connection,
|
||||
kind=kind,
|
||||
run_key=run_key,
|
||||
requested_date=requested_date,
|
||||
trigger=trigger,
|
||||
started_at=started,
|
||||
stale_after_seconds=stale_after_seconds,
|
||||
)
|
||||
except sqlite3.IntegrityError as exc:
|
||||
raise JobAlreadyRunning("同类任务正在运行") from exc
|
||||
run_id = int(row["id"])
|
||||
try:
|
||||
result = operation()
|
||||
except Exception as exc:
|
||||
finished = datetime.now(SHANGHAI)
|
||||
with self._database.transaction() as connection:
|
||||
self._repository.fail(
|
||||
connection,
|
||||
run_id,
|
||||
finished_at=finished,
|
||||
duration_ms=_duration(started, finished),
|
||||
error_code=type(exc).__name__,
|
||||
error_message=str(exc) or "任务执行失败",
|
||||
)
|
||||
raise
|
||||
finished = datetime.now(SHANGHAI)
|
||||
coverage = result.get("coverage")
|
||||
with self._database.transaction() as connection:
|
||||
self._repository.finish(
|
||||
connection,
|
||||
run_id,
|
||||
finished_at=finished,
|
||||
duration_ms=_duration(started, finished),
|
||||
coverage=float(coverage) if isinstance(coverage, (int, float)) else None,
|
||||
source_set=[str(value) for value in result.get("source_set") or []],
|
||||
output_version=str(result.get("output_version") or ""),
|
||||
payload=result,
|
||||
)
|
||||
return result
|
||||
|
||||
def latest(self, limit: int = 40) -> list[dict[str, Any]]:
|
||||
with self._database.read() as connection:
|
||||
return [public_job(row) for row in self._repository.latest(connection, limit)]
|
||||
|
||||
def latest_for_kind(self, kind: str) -> dict[str, Any] | None:
|
||||
with self._database.read() as connection:
|
||||
row = self._repository.latest_for_kind(connection, kind)
|
||||
return public_job(row) if row else None
|
||||
|
||||
def latest_success(self, kind: str, requested_date: str = "") -> dict[str, Any] | None:
|
||||
with self._database.read() as connection:
|
||||
row = self._repository.latest_success(connection, kind, requested_date)
|
||||
return public_job(row) if row else None
|
||||
|
||||
def ready_for_schedule(
|
||||
self,
|
||||
kind: str,
|
||||
*,
|
||||
now: datetime,
|
||||
completed_after_seconds: int,
|
||||
failed_after_seconds: int,
|
||||
) -> bool:
|
||||
"""Return whether a scheduler may start another run of this job kind."""
|
||||
latest = self.latest_for_kind(kind)
|
||||
if latest is None:
|
||||
return True
|
||||
if latest["status"] == "running":
|
||||
return False
|
||||
reference = latest.get("finished_at") or latest.get("started_at")
|
||||
if not reference:
|
||||
return True
|
||||
try:
|
||||
elapsed = (now - datetime.fromisoformat(str(reference))).total_seconds()
|
||||
except ValueError:
|
||||
return True
|
||||
cooldown = (
|
||||
completed_after_seconds
|
||||
if latest["status"] == "completed"
|
||||
else failed_after_seconds
|
||||
)
|
||||
return elapsed >= cooldown
|
||||
|
||||
|
||||
def _duration(started: datetime, finished: datetime) -> int:
|
||||
return max(0, round((finished - started).total_seconds() * 1000))
|
||||
Reference in New Issue
Block a user