leefer/caiwuzongzhang
1
0
Fork 0
Code Issues Pull Requests 4 Actions Packages Projects Releases Wiki Activity

B-42: 导入功能加固——逐工作表确认、流式上传与诊断证据

Browse Source
This commit is contained in:
腾讯WorkBuddy
2026-08-17 01:00:19 +08:00
parent 7f1a93f6a6
commit 486842963e
15 changed files with 2019 additions and 183 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/bank_importer/importing.py
+357 -13
View File
@@ -1,10 +1,16 @@
"""Immutable statement import pipeline.
"""Immutable statement import pipeline with per-worksheet review lifecycle.
Every uploaded file is hashed (SHA-256) and written once to content-addressed
storage before parsing. A repeated upload of identical bytes never creates a
second set of facts: it records a ``duplicate`` batch that points at the
original batch. Parse failures keep the batch and its diagnostics as an
``exception`` batch without producing any confirmed source rows.
Each worksheet is parsed independently into a :class:`SheetResult`
(``parsed`` / ``exception`` / ``ignored``) that is persisted in
``sheet_reviews`` as immutable evidence. Whether a parsed sheet may take part
in later matching and calculation is a separate, auditable human decision
(``review_status``): only ``confirmed`` sheets participate.
"""
from __future__ import annotations
@@ -14,11 +20,18 @@ import hashlib
import json
import os
from pathlib import Path
import shutil
import sqlite3
from . import auth
from .db import utc_now
from .models import StatementBatch
from .parser import StatementParseError, parse_statement
from .models import SheetResult, StatementBatch
from .parser import StatementParseError, analyze_workbook
from .reader import CorruptWorkbookError
class SheetReviewError(ValueError):
"""A review decision cannot be applied to the worksheet (mapped to 409)."""
@dataclass(frozen=True)
@@ -28,6 +41,7 @@ class ImportResult:
sha256: str
source_file_id: int
batches: tuple[StatementBatch, ...] = ()
sheets: tuple[SheetResult, ...] = ()
message: str | None = None
duplicate_same_company: bool = False # only meaningful when status == 'duplicate'
@@ -48,6 +62,78 @@ def import_statement(
return _record_duplicate(connection, existing_file["id"], sha256, company_id)
stored_path = _store_immutable(Path(storage_dir), original_filename, content, sha256)
try:
source_file_id, batch_id = _create_batch_records(
connection, sha256, original_filename, len(content), stored_path, company_id
)
except sqlite3.IntegrityError:
# Lost a concurrent-insert race on the sha256 UNIQUE constraint: the
# identical content was already persisted by another request, so this
# upload is a duplicate. The content-addressed file already exists.
existing = connection.execute(
"SELECT id FROM source_files WHERE sha256 = ?", (sha256,)
).fetchone()
if existing is not None:
return _record_duplicate(connection, existing["id"], sha256, company_id)
raise
return _parse_and_persist(
connection, stored_path, sha256, original_filename, source_file_id, batch_id
)
def import_statement_path(
connection: sqlite3.Connection,
storage_dir: str | Path,
original_filename: str,
upload_path: str | Path,
company_id: int | None = None,
) -> ImportResult:
"""Import from an already-downloaded upload file (streaming-friendly).
``upload_path`` names the temp file the multipart handler streamed to
disk; it is hashed incrementally and either discarded (duplicate) or
published into the content-addressed store. The upload file itself is
never modified.
"""
source = Path(upload_path)
sha256 = _file_sha256(source)
existing_file = connection.execute(
"SELECT id FROM source_files WHERE sha256 = ?", (sha256,)
).fetchone()
if existing_file is not None:
source.unlink(missing_ok=True)
return _record_duplicate(connection, existing_file["id"], sha256, company_id)
stored_path = _publish_immutable(Path(storage_dir), original_filename, source, sha256)
size = stored_path.stat().st_size
try:
source_file_id, batch_id = _create_batch_records(
connection, sha256, original_filename, size, stored_path, company_id
)
except sqlite3.IntegrityError:
# Lost a concurrent-insert race on the sha256 UNIQUE constraint: the
# identical content was already persisted by another request, so this
# upload is a duplicate. The content-addressed file already exists.
existing = connection.execute(
"SELECT id FROM source_files WHERE sha256 = ?", (sha256,)
).fetchone()
if existing is not None:
return _record_duplicate(connection, existing["id"], sha256, company_id)
raise
return _parse_and_persist(
connection, stored_path, sha256, original_filename, source_file_id, batch_id
)
def _create_batch_records(
connection: sqlite3.Connection,
sha256: str,
original_filename: str,
size_bytes: int,
stored_path: Path,
company_id: int | None,
) -> tuple[int, int]:
now = utc_now()
with connection:
cursor = connection.execute(
@@ -55,7 +141,7 @@ def import_statement(
INSERT INTO source_files (sha256, original_filename, size_bytes, storage_path, created_at)
VALUES (?, ?, ?, ?, ?)
""",
(sha256, original_filename, len(content), str(stored_path), now),
(sha256, original_filename, size_bytes, str(stored_path), now),
)
source_file_id = cursor.lastrowid
cursor = connection.execute(
@@ -66,27 +152,64 @@ def import_statement(
(source_file_id, company_id, now, now),
)
batch_id = cursor.lastrowid
return source_file_id, batch_id
def _parse_and_persist(
connection: sqlite3.Connection,
stored_path: Path,
sha256: str,
original_filename: str,
source_file_id: int,
batch_id: int,
) -> ImportResult:
try:
batches = parse_statement(stored_path)
except StatementParseError as exc:
sheets = analyze_workbook(stored_path)
except CorruptWorkbookError as exc:
message = _clean_message(str(exc), stored_path, original_filename)
with connection:
_insert_exception(connection, batch_id, "parse", message, original_filename)
_insert_exception(
connection, batch_id, "parse", message, original_filename,
diagnostics={"original_filename": original_filename},
)
_set_batch_status(connection, batch_id, "exception")
return ImportResult(batch_id, "exception", sha256, source_file_id, message=message)
except Exception as exc:
message = f"文件解析失败，请检查文件是否完整。（{type(exc).__name__}）"
with connection:
_insert_exception(connection, batch_id, "internal", message, original_filename)
_insert_exception(
connection, batch_id, "internal", message, original_filename,
diagnostics={"original_filename": original_filename},
)
_set_batch_status(connection, batch_id, "failed")
raise
with connection:
for batch in batches:
_insert_sheet_batch(connection, batch_id, batch)
parsed_batches: list[StatementBatch] = []
for sheet in sheets:
if sheet.outcome == "parsed" and sheet.batch is not None:
sheet_batch_id = _insert_sheet_batch(connection, batch_id, sheet.batch)
_insert_sheet_review(connection, batch_id, sheet, sheet_batch_id)
parsed_batches.append(sheet.batch)
else:
_insert_sheet_review(connection, batch_id, sheet, None)
if not parsed_batches:
message = _whole_file_message(original_filename, sheets)
with connection:
_insert_exception(
connection, batch_id, "parse", message, original_filename,
diagnostics=_exception_diagnostics(original_filename, sheets),
)
_set_batch_status(connection, batch_id, "exception")
return ImportResult(
batch_id, "exception", sha256, source_file_id,
message=message, sheets=sheets,
)
_set_batch_status(connection, batch_id, "parsed")
return ImportResult(batch_id, "parsed", sha256, source_file_id, batches=batches)
return ImportResult(
batch_id, "parsed", sha256, source_file_id,
batches=tuple(parsed_batches), sheets=sheets,
)
def _record_duplicate(
@@ -169,9 +292,39 @@ def _store_immutable(
return target
def _publish_immutable(
storage_dir: Path, original_filename: str, upload_path: Path, sha256: str
) -> Path:
suffix = Path(original_filename).suffix.lower()
target_dir = storage_dir / sha256[:2]
target_dir.mkdir(parents=True, exist_ok=True)
target = target_dir / f"{sha256}{suffix}"
temp = target_dir / f".{sha256}.tmp"
try:
# Copy keeps the same-filesystem guarantee even if the upload temp
# lives elsewhere; the hard link then publishes atomically.
shutil.copyfile(upload_path, temp)
try:
os.link(temp, target)
except FileExistsError:
pass
finally:
temp.unlink(missing_ok=True)
upload_path.unlink(missing_ok=True)
return target
def _file_sha256(path: Path) -> str:
digest = hashlib.sha256()
with path.open("rb") as handle:
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
digest.update(chunk)
return digest.hexdigest()
def _insert_sheet_batch(
connection: sqlite3.Connection, batch_id: int, batch: StatementBatch
) -> None:
) -> int:
now = utc_now()
cursor = connection.execute(
"""
@@ -226,6 +379,45 @@ def _insert_sheet_batch(
now,
),
)
return sheet_batch_id
def _insert_sheet_review(
connection: sqlite3.Connection,
batch_id: int,
sheet: SheetResult,
sheet_batch_id: int | None,
) -> None:
# Parse-side ignored sheets are terminal: there is nothing to confirm, so
# their review lifecycle is closed with a machine reason.
if sheet.outcome == "ignored":
review_status = "ignored"
review_reason = "空表或无有效内容，自动忽略。"
else:
review_status = "pending"
review_reason = None
connection.execute(
"""
INSERT INTO sheet_reviews (
import_batch_id, sheet_name, outcome, message, scanned_rows,
candidate_headers, sheet_batch_id, review_status, review_reason, created_at
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
""",
(
batch_id,
sheet.sheet_name,
sheet.outcome,
sheet.message,
sheet.scanned_rows,
json.dumps(list(sheet.candidate_headers), ensure_ascii=False)
if sheet.candidate_headers
else None,
sheet_batch_id,
review_status,
review_reason,
utc_now(),
),
)
def _insert_exception(
@@ -234,7 +426,12 @@ def _insert_exception(
stage: str,
message: str,
original_filename: str,
*,
diagnostics: dict[str, object] | None = None,
) -> None:
payload = {"original_filename": original_filename}
if diagnostics:
payload.update(diagnostics)
connection.execute(
"""
INSERT INTO import_exceptions (import_batch_id, stage, message, diagnostics, created_at)
@@ -244,7 +441,7 @@ def _insert_exception(
batch_id,
stage,
message,
json.dumps({"original_filename": original_filename}, ensure_ascii=False),
json.dumps(payload, ensure_ascii=False),
utc_now(),
),
)
@@ -257,7 +454,154 @@ def _set_batch_status(connection: sqlite3.Connection, batch_id: int, status: str
)
def _whole_file_message(original_filename: str, sheets: tuple[SheetResult, ...]) -> str:
detail = "；".join(
f"{sheet.sheet_name}：{sheet.message}" for sheet in sheets if sheet.message
)
return f"文件「{original_filename}」没有可确认的工作表。{detail}"
def _exception_diagnostics(
original_filename: str, sheets: tuple[SheetResult, ...]
) -> dict[str, object]:
return {
"original_filename": original_filename,
"sheets": [
{
"sheet_name": sheet.sheet_name,
"outcome": sheet.outcome,
"message": sheet.message,
"scanned_rows": sheet.scanned_rows,
"candidate_headers": list(sheet.candidate_headers),
}
for sheet in sheets
],
}
def _clean_message(message: str, stored_path: Path, original_filename: str) -> str:
return message.replace(str(stored_path), original_filename).replace(
stored_path.name, original_filename
)
# ---------------------------------------------------------------------------
# Per-sheet review lifecycle
# ---------------------------------------------------------------------------
def sheet_review_rows(connection: sqlite3.Connection, batch_id: int) -> list[sqlite3.Row]:
return connection.execute(
"""
SELECT r.id, r.sheet_name, r.outcome, r.message, r.scanned_rows,
r.candidate_headers, r.review_status, r.review_reason,
r.reviewed_by, r.reviewed_at,
s.bank_name, s.template_id, s.header_row, s.own_account, s.own_name,
s.period_start, s.period_end, s.transaction_count, s.warnings
FROM sheet_reviews r
LEFT JOIN sheet_batches s ON s.id = r.sheet_batch_id
WHERE r.import_batch_id = ?
ORDER BY r.id
""",
(batch_id,),
).fetchall()
def scoped_batch(
connection: sqlite3.Connection, batch_id: int, company_id: int | None
) -> sqlite3.Row | None:
"""Load a batch, scoping company users to their own tenant.
``company_id is None`` (admin) sees any batch; a company user only sees
batches owned by that company. Returns None when out of scope so callers
can answer 404 without leaking the batch's existence.
"""
if company_id is None:
return connection.execute(
"SELECT id, company_id FROM import_batches WHERE id = ?", (batch_id,)
).fetchone()
return connection.execute(
"SELECT id, company_id FROM import_batches WHERE id = ? AND company_id = ?",
(batch_id, company_id),
).fetchone()
def review_sheets(
connection: sqlite3.Connection,
batch_id: int,
sheet_names: list[str],
decision: str,
actor: sqlite3.Row | None,
company_id: int | None,
reason: str | None = None,
) -> dict[str, object]:
"""Apply ``confirm`` or ``ignore`` to whole worksheets atomically.
Multi-sheet writes run in one transaction; a failure rolls back every
sheet in the request. Idempotent repeats of the same decision succeed
without touching the audit log; changing a settled decision is a
conflict. Returns ``{"updated": [...], "already": [...]}``.
"""
if decision not in ("confirm", "ignore"):
raise ValueError("decision 必须是 confirm 或 ignore。")
names = [str(name).strip() for name in sheet_names]
if not names:
raise ValueError("必须至少指定一个工作表。")
reason = (reason or "").strip() or None
if decision == "ignore" and not reason:
raise ValueError("忽略工作表必须填写原因。")
batch = scoped_batch(connection, batch_id, company_id)
if batch is None:
raise LookupError("批次不存在。")
placeholders = ",".join("?" for _ in names)
rows = connection.execute(
f"""
SELECT * FROM sheet_reviews
WHERE import_batch_id = ? AND sheet_name IN ({placeholders})
""",
(batch_id, *names),
).fetchall()
found = {row["sheet_name"] for row in rows}
missing = [name for name in names if name not in found]
if missing:
raise ValueError(f"工作表不存在或不属于该批次：{'、'.join(missing)}")
target = "confirmed" if decision == "confirm" else "ignored"
now = utc_now()
updated: list[str] = []
already: list[str] = []
with connection:
for row in rows:
if row["review_status"] == target:
already.append(row["sheet_name"])
continue
if decision == "confirm" and row["outcome"] != "parsed":
raise SheetReviewError(
f"工作表「{row['sheet_name']}」没有可确认的交易，只能忽略或保留待处理。"
)
if row["review_status"] != "pending":
raise SheetReviewError(
f"工作表「{row['sheet_name']}」已处理，不能更改决定。"
)
cursor = connection.execute(
"""
UPDATE sheet_reviews
SET review_status = ?, review_reason = ?, reviewed_by = ?, reviewed_at = ?
WHERE id = ? AND review_status = 'pending'
""",
(target, reason, actor["id"] if actor is not None else None, now, row["id"]),
)
if cursor.rowcount:
updated.append(row["sheet_name"])
if updated:
auth.audit(
connection,
f"sheet_{decision}",
actor=actor,
target=f"batch:{batch_id}",
detail=f"sheets:{','.join(updated)}" + (f";reason:{reason}" if reason else ""),
)
return {"updated": updated, "already": already}