B-42: 导入功能加固——逐工作表确认、流式上传与诊断证据

This commit is contained in:
腾讯WorkBuddy
2026-08-17 01:00:19 +08:00
parent 7f1a93f6a6
commit 486842963e
15 changed files with 2019 additions and 183 deletions
+6
View File
@@ -13,3 +13,9 @@ server.out.log
server.err.log server.err.log
nul nul
# 本地协作工具运行目录
.multica/
.opencode/
.agent_context/
.kimi/
+217 -99
View File
@@ -10,9 +10,8 @@ from http.cookies import SimpleCookie
from http.server import SimpleHTTPRequestHandler, ThreadingHTTPServer from http.server import SimpleHTTPRequestHandler, ThreadingHTTPServer
from urllib.parse import parse_qs, urlparse from urllib.parse import parse_qs, urlparse
from bank_importer import auth, master_data from bank_importer import auth, importing, master_data, multipart
from bank_importer.db import connect, migrate, utc_now from bank_importer.db import connect, migrate, utc_now
from bank_importer.importing import import_statement
ROOT = Path(__file__).resolve().parent ROOT = Path(__file__).resolve().parent
@@ -49,6 +48,10 @@ class AppHandler(SimpleHTTPRequestHandler):
if rows_match: if rows_match:
self._handle_batch_rows(int(rows_match.group(1))) self._handle_batch_rows(int(rows_match.group(1)))
return return
sheets_match = re.fullmatch(r"/api/batches/(\d+)/sheets", path)
if sheets_match:
self._handle_batch_sheets(int(sheets_match.group(1)))
return
if path == "/api/export.csv": if path == "/api/export.csv":
self._handle_export_csv(query) self._handle_export_csv(query)
return return
@@ -119,6 +122,14 @@ class AppHandler(SimpleHTTPRequestHandler):
if alias_create: if alias_create:
self._handle_admin_add_alias(int(alias_create.group(1))) self._handle_admin_add_alias(int(alias_create.group(1)))
return return
confirm_match = re.fullmatch(r"/api/batches/(\d+)/confirm", path)
if confirm_match:
self._handle_batch_review(int(confirm_match.group(1)), "confirm")
return
ignore_match = re.fullmatch(r"/api/batches/(\d+)/ignore", path)
if ignore_match:
self._handle_batch_review(int(ignore_match.group(1)), "ignore")
return
self._send_json(404, {"status": "error", "message": "接口不存在。"}) self._send_json(404, {"status": "error", "message": "接口不存在。"})
# ------------------------------------------------------------------ # ------------------------------------------------------------------
@@ -308,15 +319,21 @@ class AppHandler(SimpleHTTPRequestHandler):
def _handle_parse(self) -> None: def _handle_parse(self) -> None:
connection = connect(DB_PATH) connection = connect(DB_PATH)
upload = None
try: try:
user = self._require_user(connection) user = self._require_user(connection)
if user is None: if user is None:
return return
try: try:
filename, content, fields = self._read_upload() fields, upload = self._receive_upload()
filename = os.path.basename(upload.filename)
suffix = Path(filename).suffix.lower() suffix = Path(filename).suffix.lower()
if suffix not in {".xls", ".xlsx"}: if suffix not in {".xls", ".xlsx"}:
raise ValueError("仅支持 .xls 或 .xlsx 银行流水文件。") raise ValueError("仅支持 .xls 或 .xlsx 银行流水文件。")
if not multipart.valid_file_signature(upload.path, suffix):
raise ValueError(
"文件内容与扩展名不符,可能已损坏或不是有效的 Excel 文件。"
)
if user["role"] == "company": if user["role"] == "company":
# Tenant binding comes from the session only; any # Tenant binding comes from the session only; any
@@ -325,12 +342,15 @@ class AppHandler(SimpleHTTPRequestHandler):
if not self._validate_upload_account(connection, user, fields): if not self._validate_upload_account(connection, user, fields):
return return
else: else:
company_id = self._parse_company_field(connection, fields.get("company_id")) company_id = self._parse_company_field(
connection, fields.get("company_id")
)
if company_id is None: if company_id is None:
return return
result = import_statement( result = importing.import_statement_path(
connection, STORAGE_DIR, filename, content, company_id=company_id connection, STORAGE_DIR, filename, upload.path,
company_id=company_id,
) )
auth.audit( auth.audit(
connection, connection,
@@ -356,8 +376,40 @@ class AppHandler(SimpleHTTPRequestHandler):
else: else:
self._send_json(200, payload) self._send_json(200, payload)
finally: finally:
if upload is not None:
upload.path.unlink(missing_ok=True)
connection.close() connection.close()
def _receive_upload(self) -> tuple[dict[str, str], multipart.UploadedFile]:
"""Stream the multipart body into a validated temp file.
The whole request is never buffered in memory; file bytes are written
to a temp file under the storage directory as they arrive and are
never trimmed. Only the first file part is accepted.
"""
raw_length = self.headers.get("Content-Length")
if raw_length is None:
raise ValueError("上传请求缺少 Content-Length。")
try:
content_length = int(raw_length)
except ValueError:
raise ValueError("上传请求的 Content-Length 无效。")
if content_length <= 0:
raise ValueError("文件为空或超过 20 MB 限制。")
if content_length > MAX_UPLOAD_BYTES + multipart.MAX_FIELD_BYTES:
raise ValueError("文件超过 20 MB 限制。")
try:
fields, upload = multipart.parse_upload(
self.rfile,
content_length,
self.headers.get("Content-Type", ""),
STORAGE_DIR / ".uploads",
max_file_bytes=MAX_UPLOAD_BYTES,
)
except multipart.MultipartError as exc:
raise ValueError(str(exc)) from exc
return fields, upload
def _validate_upload_account(self, connection, user, fields) -> bool: def _validate_upload_account(self, connection, user, fields) -> bool:
"""Validate the optional bank_account_id on a company upload. """Validate the optional bank_account_id on a company upload.
@@ -435,15 +487,40 @@ class AppHandler(SimpleHTTPRequestHandler):
f""" f"""
SELECT b.id, b.status, b.created_at, b.company_id, SELECT b.id, b.status, b.created_at, b.company_id,
f.original_filename, c.name AS company_name, f.original_filename, c.name AS company_name,
COALESCE(SUM(s.transaction_count), 0) AS transactions, COALESCE((
MIN(s.period_start) AS period_start, SELECT SUM(s.transaction_count) FROM sheet_batches s
MAX(s.period_end) AS period_end WHERE s.import_batch_id = b.id
), 0) AS transactions,
(SELECT s.bank_name FROM sheet_batches s
WHERE s.import_batch_id = b.id
ORDER BY s.id LIMIT 1) AS bank_name,
(SELECT MIN(s.period_start) FROM sheet_batches s
WHERE s.import_batch_id = b.id) AS period_start,
(SELECT MAX(s.period_end) FROM sheet_batches s
WHERE s.import_batch_id = b.id) AS period_end,
(SELECT COUNT(*) FROM source_rows r
JOIN sheet_batches s ON s.id = r.sheet_batch_id
JOIN sheet_reviews rv ON rv.sheet_batch_id = s.id
WHERE s.import_batch_id = b.id AND rv.review_status = 'confirmed')
AS confirmed_transactions,
(SELECT COUNT(*) FROM sheet_reviews r
WHERE r.import_batch_id = b.id AND r.review_status = 'confirmed')
AS confirmed_sheets,
(SELECT COUNT(*) FROM sheet_reviews r
WHERE r.import_batch_id = b.id
AND r.outcome = 'parsed' AND r.review_status = 'pending')
AS pending_sheets,
(SELECT COUNT(*) FROM sheet_reviews r
WHERE r.import_batch_id = b.id AND r.outcome = 'exception')
AS exception_sheets,
(SELECT COUNT(*) FROM sheet_reviews r
WHERE r.import_batch_id = b.id
AND (r.outcome = 'ignored' OR r.review_status = 'ignored'))
AS ignored_sheets
FROM import_batches b FROM import_batches b
JOIN source_files f ON f.id = b.source_file_id JOIN source_files f ON f.id = b.source_file_id
LEFT JOIN companies c ON c.id = b.company_id LEFT JOIN companies c ON c.id = b.company_id
LEFT JOIN sheet_batches s ON s.import_batch_id = b.id
{where} {where}
GROUP BY b.id
ORDER BY b.id DESC ORDER BY b.id DESC
LIMIT 500 LIMIT 500
""", """,
@@ -453,6 +530,89 @@ class AppHandler(SimpleHTTPRequestHandler):
finally: finally:
connection.close() connection.close()
def _handle_batch_sheets(self, batch_id: int) -> None:
connection = connect(DB_PATH)
try:
user = self._require_user(connection)
if user is None:
return
company_id = user["company_id"] if user["role"] == "company" else None
batch = importing.scoped_batch(connection, batch_id, company_id)
if batch is None:
self._send_json(404, {"status": "error", "message": "批次不存在。"})
return
file_row = connection.execute(
"""
SELECT f.original_filename FROM source_files f
JOIN import_batches b ON b.source_file_id = f.id
WHERE b.id = ?
""",
(batch_id,),
).fetchone()
rows = importing.sheet_review_rows(connection, batch_id)
self._send_json(
200,
{
"status": "ok",
"batch_id": batch_id,
"company_id": batch["company_id"],
"original_filename": file_row["original_filename"]
if file_row is not None
else None,
"sheets": [self._sheet_payload(row) for row in rows],
},
)
finally:
connection.close()
def _handle_batch_review(self, batch_id: int, decision: str) -> None:
connection = connect(DB_PATH)
try:
user = self._require_user(connection)
if user is None:
return
data = self._read_json_body()
if data is None:
return
sheets = data.get("sheets")
if not isinstance(sheets, list):
self._send_json(
400, {"status": "error", "message": "sheets 必须是工作表名称数组。"}
)
return
company_id = user["company_id"] if user["role"] == "company" else None
try:
outcome = importing.review_sheets(
connection,
batch_id,
sheets,
decision,
user,
company_id,
reason=str(data.get("reason") or "") if decision == "ignore" else None,
)
except LookupError as exc:
self._send_json(404, {"status": "error", "message": str(exc)})
return
except importing.SheetReviewError as exc:
self._send_json(409, {"status": "error", "message": str(exc)})
return
except ValueError as exc:
self._send_json(400, {"status": "error", "message": str(exc)})
return
rows = importing.sheet_review_rows(connection, batch_id)
self._send_json(
200,
{
"status": "ok",
"updated": outcome["updated"],
"already": outcome["already"],
"sheets": [self._sheet_payload(row) for row in rows],
},
)
finally:
connection.close()
def _handle_batch_rows(self, batch_id: int) -> None: def _handle_batch_rows(self, batch_id: int) -> None:
connection = connect(DB_PATH) connection = connect(DB_PATH)
try: try:
@@ -474,9 +634,11 @@ class AppHandler(SimpleHTTPRequestHandler):
SELECT r.id, r.sheet_batch_id, r.source_row, r.transaction_at, SELECT r.id, r.sheet_batch_id, r.source_row, r.transaction_at,
r.income, r.expense, r.balance, r.own_account, r.own_name, r.income, r.expense, r.balance, r.own_account, r.own_name,
r.counterparty_account, r.counterparty_name, r.counterparty_bank, r.counterparty_account, r.counterparty_name, r.counterparty_bank,
r.summary, r.purpose, r.reference, r.currency r.summary, r.purpose, r.reference, r.currency,
s.sheet_name, rv.review_status
FROM source_rows r FROM source_rows r
JOIN sheet_batches s ON s.id = r.sheet_batch_id JOIN sheet_batches s ON s.id = r.sheet_batch_id
JOIN sheet_reviews rv ON rv.sheet_batch_id = s.id
WHERE s.import_batch_id = ? WHERE s.import_batch_id = ?
ORDER BY s.id, r.source_row ORDER BY s.id, r.source_row
""", """,
@@ -529,6 +691,10 @@ class AppHandler(SimpleHTTPRequestHandler):
FROM source_rows r FROM source_rows r
JOIN sheet_batches s ON s.id = r.sheet_batch_id JOIN sheet_batches s ON s.id = r.sheet_batch_id
JOIN import_batches b ON b.id = s.import_batch_id JOIN import_batches b ON b.id = s.import_batch_id
-- Only worksheets confirmed by the cashier may leave the
-- ledger: parse success is not business confirmation.
JOIN sheet_reviews rv
ON rv.sheet_batch_id = s.id AND rv.review_status = 'confirmed'
{where} {where}
ORDER BY b.id, s.id, r.source_row ORDER BY b.id, s.id, r.source_row
""", """,
@@ -1117,8 +1283,37 @@ class AppHandler(SimpleHTTPRequestHandler):
return None return None
return data return data
@staticmethod
def _sheet_payload(row) -> dict[str, object]:
"""Serialize one persisted per-sheet review record."""
return {
"sheet_name": row["sheet_name"],
"outcome": row["outcome"],
"review_status": row["review_status"],
"message": row["message"],
"scanned_rows": row["scanned_rows"],
"candidate_headers": json.loads(row["candidate_headers"])
if row["candidate_headers"]
else [],
"review_reason": row["review_reason"],
"bank": row["bank_name"],
"template": row["template_id"],
"header_row": row["header_row"],
"period_start": row["period_start"],
"period_end": row["period_end"],
"transactions": row["transaction_count"] or 0,
"warnings": json.loads(row["warnings"]) if row["warnings"] else [],
}
@staticmethod @staticmethod
def _import_payload(connection, result) -> dict[str, object]: def _import_payload(connection, result) -> dict[str, object]:
"""Build the parse response from persisted state, not from memory.
All worksheet results are returned (no more ``batches[0]``). A
cross-company duplicate is opaque: only the generic status, the
uploader's own batch id and the sha256 are returned, never the other
company's batch id, summary or diagnostics.
"""
payload: dict[str, object] = { payload: dict[str, object] = {
"status": result.status, "status": result.status,
"batch_id": result.batch_id, "batch_id": result.batch_id,
@@ -1126,96 +1321,19 @@ class AppHandler(SimpleHTTPRequestHandler):
} }
if result.message: if result.message:
payload["message"] = result.message payload["message"] = result.message
if result.status == "duplicate" and not result.duplicate_same_company:
return payload
if result.status == "parsed": file_row = connection.execute(
batch = result.batches[0] "SELECT original_filename FROM source_files WHERE id = ?",
payload.update( (result.source_file_id,),
{ ).fetchone()
"bank": batch.bank_name, if file_row is not None:
"template": batch.template_id, payload["original_filename"] = file_row["original_filename"]
"header_row": batch.header_row, sheets = importing.sheet_review_rows(connection, result.batch_id)
"period_start": batch.period_start.isoformat() payload["sheets"] = [AppHandler._sheet_payload(row) for row in sheets]
if batch.period_start
else None,
"period_end": batch.period_end.isoformat()
if batch.period_end
else None,
"transactions": len(batch.transactions),
"warnings": list(batch.warnings),
}
)
elif result.status == "duplicate":
# Same-company duplicates keep their idempotent summary so the
# cashier sees the reused batch's bank/template/period/count.
# Cross-company duplicates are opaque: only the generic duplicate
# status and the uploader's own new batch id are returned, never
# the other company's batch id, summary or diagnostics.
if result.duplicate_same_company:
sheet = connection.execute(
"""
SELECT bank_name, template_id, header_row, period_start,
period_end, transaction_count, warnings
FROM sheet_batches
WHERE import_batch_id = ?
ORDER BY id LIMIT 1
""",
(result.batch_id,),
).fetchone()
if sheet is not None:
payload.update(
{
"bank": sheet["bank_name"],
"template": sheet["template_id"],
"header_row": sheet["header_row"],
"period_start": sheet["period_start"],
"period_end": sheet["period_end"],
"transactions": sheet["transaction_count"],
"warnings": json.loads(sheet["warnings"]),
}
)
return payload return payload
def _read_upload(self) -> tuple[str, bytes, dict[str, str]]:
"""Parse the multipart body into (filename, content, text fields)."""
content_length = int(self.headers.get("Content-Length", "0"))
if content_length <= 0 or content_length > MAX_UPLOAD_BYTES:
raise ValueError("文件为空或超过 20 MB 限制。")
content_type = self.headers.get("Content-Type", "")
boundary_match = re.search(r"boundary=(?:\"([^\"]+)\"|([^;]+))", content_type)
if not boundary_match:
raise ValueError("上传请求缺少文件边界。")
boundary = (boundary_match.group(1) or boundary_match.group(2)).encode()
body = self.rfile.read(content_length)
fields: dict[str, str] = {}
for part in body.split(b"--" + boundary):
header, separator, content = part.partition(b"\r\n\r\n")
if not separator:
continue
name_match = re.search(br'name="([^\"]+)"', header)
if not name_match:
continue
name = name_match.group(1).decode("utf-8", errors="replace")
if name == "file":
filename_match = re.search(br'filename="([^\"]+)"', header)
filename = (
filename_match.group(1).decode("utf-8", errors="replace")
if filename_match
else "statement.xlsx"
)
file_content = content.rstrip(b"\r\n")
elif b"filename=" not in header:
fields[name] = content.rstrip(b"\r\n").decode("utf-8", errors="replace")
try:
return os.path.basename(filename), file_content, fields
except UnboundLocalError:
raise ValueError("上传请求中没有找到文件。") from None
def _read_uploaded_file(self) -> tuple[str, bytes]:
filename, content, _fields = self._read_upload()
return filename, content
def _send_json( def _send_json(
self, self,
status: int, status: int,
+50 -1
View File
@@ -307,6 +307,53 @@ MIGRATIONS: tuple[Migration, ...] = (
ALTER TABLE bank_accounts_new RENAME TO bank_accounts; ALTER TABLE bank_accounts_new RENAME TO bank_accounts;
""", """,
), ),
Migration(
version=4,
name="0004_per_sheet_reviews",
# Per-worksheet lifecycle: the parse outcome (parsed/exception/ignored)
# is immutable evidence captured at import time; the human decision
# (pending/confirmed/ignored) is the audit-gated gate that lets a
# worksheet participate in later matching and calculation.
up="""
CREATE TABLE sheet_reviews (
id INTEGER PRIMARY KEY AUTOINCREMENT,
import_batch_id INTEGER NOT NULL REFERENCES import_batches (id),
sheet_name TEXT NOT NULL,
outcome TEXT NOT NULL
CHECK (outcome IN ('parsed', 'exception', 'ignored')),
message TEXT,
scanned_rows INTEGER,
candidate_headers TEXT,
sheet_batch_id INTEGER REFERENCES sheet_batches (id),
review_status TEXT NOT NULL DEFAULT 'pending'
CHECK (review_status IN ('pending', 'confirmed', 'ignored')),
review_reason TEXT,
reviewed_by INTEGER REFERENCES users (id),
reviewed_at TEXT,
created_at TEXT NOT NULL,
UNIQUE (import_batch_id, sheet_name)
);
CREATE INDEX idx_sheet_reviews_batch ON sheet_reviews (import_batch_id);
CREATE TRIGGER sheet_reviews_evidence_immutable BEFORE UPDATE ON sheet_reviews
BEGIN
SELECT RAISE (ABORT, 'sheet_reviews parse evidence is immutable')
WHERE OLD.outcome != NEW.outcome
OR OLD.message IS NOT NEW.message
OR OLD.scanned_rows IS NOT NEW.scanned_rows
OR OLD.candidate_headers IS NOT NEW.candidate_headers
OR OLD.sheet_batch_id IS NOT NEW.sheet_batch_id
OR OLD.import_batch_id IS NOT NEW.import_batch_id
OR OLD.sheet_name IS NOT NEW.sheet_name;
END;
""",
down="""
DROP TRIGGER IF EXISTS sheet_reviews_evidence_immutable;
DROP INDEX IF EXISTS idx_sheet_reviews_batch;
DROP TABLE IF EXISTS sheet_reviews;
""",
),
) )
@@ -314,7 +361,9 @@ def connect(path: str | Path) -> sqlite3.Connection:
db_path = Path(path) db_path = Path(path)
if str(db_path) != ":memory:": if str(db_path) != ":memory:":
db_path.parent.mkdir(parents=True, exist_ok=True) db_path.parent.mkdir(parents=True, exist_ok=True)
connection = sqlite3.connect(str(db_path)) # Long busy timeout so concurrent uploads/confirmations wait for the
# single SQLite writer instead of surfacing "database is locked" 500s.
connection = sqlite3.connect(str(db_path), timeout=30)
connection.row_factory = sqlite3.Row connection.row_factory = sqlite3.Row
connection.execute("PRAGMA foreign_keys = ON") connection.execute("PRAGMA foreign_keys = ON")
return connection return connection
+357 -13
View File
@@ -1,10 +1,16 @@
"""Immutable statement import pipeline. """Immutable statement import pipeline with per-worksheet review lifecycle.
Every uploaded file is hashed (SHA-256) and written once to content-addressed Every uploaded file is hashed (SHA-256) and written once to content-addressed
storage before parsing. A repeated upload of identical bytes never creates a storage before parsing. A repeated upload of identical bytes never creates a
second set of facts: it records a ``duplicate`` batch that points at the second set of facts: it records a ``duplicate`` batch that points at the
original batch. Parse failures keep the batch and its diagnostics as an original batch. Parse failures keep the batch and its diagnostics as an
``exception`` batch without producing any confirmed source rows. ``exception`` batch without producing any confirmed source rows.
Each worksheet is parsed independently into a :class:`SheetResult`
(``parsed`` / ``exception`` / ``ignored``) that is persisted in
``sheet_reviews`` as immutable evidence. Whether a parsed sheet may take part
in later matching and calculation is a separate, auditable human decision
(``review_status``): only ``confirmed`` sheets participate.
""" """
from __future__ import annotations from __future__ import annotations
@@ -14,11 +20,18 @@ import hashlib
import json import json
import os import os
from pathlib import Path from pathlib import Path
import shutil
import sqlite3 import sqlite3
from . import auth
from .db import utc_now from .db import utc_now
from .models import StatementBatch from .models import SheetResult, StatementBatch
from .parser import StatementParseError, parse_statement from .parser import StatementParseError, analyze_workbook
from .reader import CorruptWorkbookError
class SheetReviewError(ValueError):
"""A review decision cannot be applied to the worksheet (mapped to 409)."""
@dataclass(frozen=True) @dataclass(frozen=True)
@@ -28,6 +41,7 @@ class ImportResult:
sha256: str sha256: str
source_file_id: int source_file_id: int
batches: tuple[StatementBatch, ...] = () batches: tuple[StatementBatch, ...] = ()
sheets: tuple[SheetResult, ...] = ()
message: str | None = None message: str | None = None
duplicate_same_company: bool = False # only meaningful when status == 'duplicate' duplicate_same_company: bool = False # only meaningful when status == 'duplicate'
@@ -48,6 +62,78 @@ def import_statement(
return _record_duplicate(connection, existing_file["id"], sha256, company_id) return _record_duplicate(connection, existing_file["id"], sha256, company_id)
stored_path = _store_immutable(Path(storage_dir), original_filename, content, sha256) stored_path = _store_immutable(Path(storage_dir), original_filename, content, sha256)
try:
source_file_id, batch_id = _create_batch_records(
connection, sha256, original_filename, len(content), stored_path, company_id
)
except sqlite3.IntegrityError:
# Lost a concurrent-insert race on the sha256 UNIQUE constraint: the
# identical content was already persisted by another request, so this
# upload is a duplicate. The content-addressed file already exists.
existing = connection.execute(
"SELECT id FROM source_files WHERE sha256 = ?", (sha256,)
).fetchone()
if existing is not None:
return _record_duplicate(connection, existing["id"], sha256, company_id)
raise
return _parse_and_persist(
connection, stored_path, sha256, original_filename, source_file_id, batch_id
)
def import_statement_path(
connection: sqlite3.Connection,
storage_dir: str | Path,
original_filename: str,
upload_path: str | Path,
company_id: int | None = None,
) -> ImportResult:
"""Import from an already-downloaded upload file (streaming-friendly).
``upload_path`` names the temp file the multipart handler streamed to
disk; it is hashed incrementally and either discarded (duplicate) or
published into the content-addressed store. The upload file itself is
never modified.
"""
source = Path(upload_path)
sha256 = _file_sha256(source)
existing_file = connection.execute(
"SELECT id FROM source_files WHERE sha256 = ?", (sha256,)
).fetchone()
if existing_file is not None:
source.unlink(missing_ok=True)
return _record_duplicate(connection, existing_file["id"], sha256, company_id)
stored_path = _publish_immutable(Path(storage_dir), original_filename, source, sha256)
size = stored_path.stat().st_size
try:
source_file_id, batch_id = _create_batch_records(
connection, sha256, original_filename, size, stored_path, company_id
)
except sqlite3.IntegrityError:
# Lost a concurrent-insert race on the sha256 UNIQUE constraint: the
# identical content was already persisted by another request, so this
# upload is a duplicate. The content-addressed file already exists.
existing = connection.execute(
"SELECT id FROM source_files WHERE sha256 = ?", (sha256,)
).fetchone()
if existing is not None:
return _record_duplicate(connection, existing["id"], sha256, company_id)
raise
return _parse_and_persist(
connection, stored_path, sha256, original_filename, source_file_id, batch_id
)
def _create_batch_records(
connection: sqlite3.Connection,
sha256: str,
original_filename: str,
size_bytes: int,
stored_path: Path,
company_id: int | None,
) -> tuple[int, int]:
now = utc_now() now = utc_now()
with connection: with connection:
cursor = connection.execute( cursor = connection.execute(
@@ -55,7 +141,7 @@ def import_statement(
INSERT INTO source_files (sha256, original_filename, size_bytes, storage_path, created_at) INSERT INTO source_files (sha256, original_filename, size_bytes, storage_path, created_at)
VALUES (?, ?, ?, ?, ?) VALUES (?, ?, ?, ?, ?)
""", """,
(sha256, original_filename, len(content), str(stored_path), now), (sha256, original_filename, size_bytes, str(stored_path), now),
) )
source_file_id = cursor.lastrowid source_file_id = cursor.lastrowid
cursor = connection.execute( cursor = connection.execute(
@@ -66,27 +152,64 @@ def import_statement(
(source_file_id, company_id, now, now), (source_file_id, company_id, now, now),
) )
batch_id = cursor.lastrowid batch_id = cursor.lastrowid
return source_file_id, batch_id
def _parse_and_persist(
connection: sqlite3.Connection,
stored_path: Path,
sha256: str,
original_filename: str,
source_file_id: int,
batch_id: int,
) -> ImportResult:
try: try:
batches = parse_statement(stored_path) sheets = analyze_workbook(stored_path)
except StatementParseError as exc: except CorruptWorkbookError as exc:
message = _clean_message(str(exc), stored_path, original_filename) message = _clean_message(str(exc), stored_path, original_filename)
with connection: with connection:
_insert_exception(connection, batch_id, "parse", message, original_filename) _insert_exception(
connection, batch_id, "parse", message, original_filename,
diagnostics={"original_filename": original_filename},
)
_set_batch_status(connection, batch_id, "exception") _set_batch_status(connection, batch_id, "exception")
return ImportResult(batch_id, "exception", sha256, source_file_id, message=message) return ImportResult(batch_id, "exception", sha256, source_file_id, message=message)
except Exception as exc: except Exception as exc:
message = f"文件解析失败,请检查文件是否完整。({type(exc).__name__}" message = f"文件解析失败,请检查文件是否完整。({type(exc).__name__}"
with connection: with connection:
_insert_exception(connection, batch_id, "internal", message, original_filename) _insert_exception(
connection, batch_id, "internal", message, original_filename,
diagnostics={"original_filename": original_filename},
)
_set_batch_status(connection, batch_id, "failed") _set_batch_status(connection, batch_id, "failed")
raise raise
with connection: with connection:
for batch in batches: parsed_batches: list[StatementBatch] = []
_insert_sheet_batch(connection, batch_id, batch) for sheet in sheets:
if sheet.outcome == "parsed" and sheet.batch is not None:
sheet_batch_id = _insert_sheet_batch(connection, batch_id, sheet.batch)
_insert_sheet_review(connection, batch_id, sheet, sheet_batch_id)
parsed_batches.append(sheet.batch)
else:
_insert_sheet_review(connection, batch_id, sheet, None)
if not parsed_batches:
message = _whole_file_message(original_filename, sheets)
with connection:
_insert_exception(
connection, batch_id, "parse", message, original_filename,
diagnostics=_exception_diagnostics(original_filename, sheets),
)
_set_batch_status(connection, batch_id, "exception")
return ImportResult(
batch_id, "exception", sha256, source_file_id,
message=message, sheets=sheets,
)
_set_batch_status(connection, batch_id, "parsed") _set_batch_status(connection, batch_id, "parsed")
return ImportResult(batch_id, "parsed", sha256, source_file_id, batches=batches) return ImportResult(
batch_id, "parsed", sha256, source_file_id,
batches=tuple(parsed_batches), sheets=sheets,
)
def _record_duplicate( def _record_duplicate(
@@ -169,9 +292,39 @@ def _store_immutable(
return target return target
def _publish_immutable(
storage_dir: Path, original_filename: str, upload_path: Path, sha256: str
) -> Path:
suffix = Path(original_filename).suffix.lower()
target_dir = storage_dir / sha256[:2]
target_dir.mkdir(parents=True, exist_ok=True)
target = target_dir / f"{sha256}{suffix}"
temp = target_dir / f".{sha256}.tmp"
try:
# Copy keeps the same-filesystem guarantee even if the upload temp
# lives elsewhere; the hard link then publishes atomically.
shutil.copyfile(upload_path, temp)
try:
os.link(temp, target)
except FileExistsError:
pass
finally:
temp.unlink(missing_ok=True)
upload_path.unlink(missing_ok=True)
return target
def _file_sha256(path: Path) -> str:
digest = hashlib.sha256()
with path.open("rb") as handle:
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
digest.update(chunk)
return digest.hexdigest()
def _insert_sheet_batch( def _insert_sheet_batch(
connection: sqlite3.Connection, batch_id: int, batch: StatementBatch connection: sqlite3.Connection, batch_id: int, batch: StatementBatch
) -> None: ) -> int:
now = utc_now() now = utc_now()
cursor = connection.execute( cursor = connection.execute(
""" """
@@ -226,6 +379,45 @@ def _insert_sheet_batch(
now, now,
), ),
) )
return sheet_batch_id
def _insert_sheet_review(
connection: sqlite3.Connection,
batch_id: int,
sheet: SheetResult,
sheet_batch_id: int | None,
) -> None:
# Parse-side ignored sheets are terminal: there is nothing to confirm, so
# their review lifecycle is closed with a machine reason.
if sheet.outcome == "ignored":
review_status = "ignored"
review_reason = "空表或无有效内容,自动忽略。"
else:
review_status = "pending"
review_reason = None
connection.execute(
"""
INSERT INTO sheet_reviews (
import_batch_id, sheet_name, outcome, message, scanned_rows,
candidate_headers, sheet_batch_id, review_status, review_reason, created_at
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
""",
(
batch_id,
sheet.sheet_name,
sheet.outcome,
sheet.message,
sheet.scanned_rows,
json.dumps(list(sheet.candidate_headers), ensure_ascii=False)
if sheet.candidate_headers
else None,
sheet_batch_id,
review_status,
review_reason,
utc_now(),
),
)
def _insert_exception( def _insert_exception(
@@ -234,7 +426,12 @@ def _insert_exception(
stage: str, stage: str,
message: str, message: str,
original_filename: str, original_filename: str,
*,
diagnostics: dict[str, object] | None = None,
) -> None: ) -> None:
payload = {"original_filename": original_filename}
if diagnostics:
payload.update(diagnostics)
connection.execute( connection.execute(
""" """
INSERT INTO import_exceptions (import_batch_id, stage, message, diagnostics, created_at) INSERT INTO import_exceptions (import_batch_id, stage, message, diagnostics, created_at)
@@ -244,7 +441,7 @@ def _insert_exception(
batch_id, batch_id,
stage, stage,
message, message,
json.dumps({"original_filename": original_filename}, ensure_ascii=False), json.dumps(payload, ensure_ascii=False),
utc_now(), utc_now(),
), ),
) )
@@ -257,7 +454,154 @@ def _set_batch_status(connection: sqlite3.Connection, batch_id: int, status: str
) )
def _whole_file_message(original_filename: str, sheets: tuple[SheetResult, ...]) -> str:
detail = "".join(
f"{sheet.sheet_name}{sheet.message}" for sheet in sheets if sheet.message
)
return f"文件「{original_filename}」没有可确认的工作表。{detail}"
def _exception_diagnostics(
original_filename: str, sheets: tuple[SheetResult, ...]
) -> dict[str, object]:
return {
"original_filename": original_filename,
"sheets": [
{
"sheet_name": sheet.sheet_name,
"outcome": sheet.outcome,
"message": sheet.message,
"scanned_rows": sheet.scanned_rows,
"candidate_headers": list(sheet.candidate_headers),
}
for sheet in sheets
],
}
def _clean_message(message: str, stored_path: Path, original_filename: str) -> str: def _clean_message(message: str, stored_path: Path, original_filename: str) -> str:
return message.replace(str(stored_path), original_filename).replace( return message.replace(str(stored_path), original_filename).replace(
stored_path.name, original_filename stored_path.name, original_filename
) )
# ---------------------------------------------------------------------------
# Per-sheet review lifecycle
# ---------------------------------------------------------------------------
def sheet_review_rows(connection: sqlite3.Connection, batch_id: int) -> list[sqlite3.Row]:
return connection.execute(
"""
SELECT r.id, r.sheet_name, r.outcome, r.message, r.scanned_rows,
r.candidate_headers, r.review_status, r.review_reason,
r.reviewed_by, r.reviewed_at,
s.bank_name, s.template_id, s.header_row, s.own_account, s.own_name,
s.period_start, s.period_end, s.transaction_count, s.warnings
FROM sheet_reviews r
LEFT JOIN sheet_batches s ON s.id = r.sheet_batch_id
WHERE r.import_batch_id = ?
ORDER BY r.id
""",
(batch_id,),
).fetchall()
def scoped_batch(
connection: sqlite3.Connection, batch_id: int, company_id: int | None
) -> sqlite3.Row | None:
"""Load a batch, scoping company users to their own tenant.
``company_id is None`` (admin) sees any batch; a company user only sees
batches owned by that company. Returns None when out of scope so callers
can answer 404 without leaking the batch's existence.
"""
if company_id is None:
return connection.execute(
"SELECT id, company_id FROM import_batches WHERE id = ?", (batch_id,)
).fetchone()
return connection.execute(
"SELECT id, company_id FROM import_batches WHERE id = ? AND company_id = ?",
(batch_id, company_id),
).fetchone()
def review_sheets(
connection: sqlite3.Connection,
batch_id: int,
sheet_names: list[str],
decision: str,
actor: sqlite3.Row | None,
company_id: int | None,
reason: str | None = None,
) -> dict[str, object]:
"""Apply ``confirm`` or ``ignore`` to whole worksheets atomically.
Multi-sheet writes run in one transaction; a failure rolls back every
sheet in the request. Idempotent repeats of the same decision succeed
without touching the audit log; changing a settled decision is a
conflict. Returns ``{"updated": [...], "already": [...]}``.
"""
if decision not in ("confirm", "ignore"):
raise ValueError("decision 必须是 confirm 或 ignore。")
names = [str(name).strip() for name in sheet_names]
if not names:
raise ValueError("必须至少指定一个工作表。")
reason = (reason or "").strip() or None
if decision == "ignore" and not reason:
raise ValueError("忽略工作表必须填写原因。")
batch = scoped_batch(connection, batch_id, company_id)
if batch is None:
raise LookupError("批次不存在。")
placeholders = ",".join("?" for _ in names)
rows = connection.execute(
f"""
SELECT * FROM sheet_reviews
WHERE import_batch_id = ? AND sheet_name IN ({placeholders})
""",
(batch_id, *names),
).fetchall()
found = {row["sheet_name"] for row in rows}
missing = [name for name in names if name not in found]
if missing:
raise ValueError(f"工作表不存在或不属于该批次:{''.join(missing)}")
target = "confirmed" if decision == "confirm" else "ignored"
now = utc_now()
updated: list[str] = []
already: list[str] = []
with connection:
for row in rows:
if row["review_status"] == target:
already.append(row["sheet_name"])
continue
if decision == "confirm" and row["outcome"] != "parsed":
raise SheetReviewError(
f"工作表「{row['sheet_name']}」没有可确认的交易,只能忽略或保留待处理。"
)
if row["review_status"] != "pending":
raise SheetReviewError(
f"工作表「{row['sheet_name']}」已处理,不能更改决定。"
)
cursor = connection.execute(
"""
UPDATE sheet_reviews
SET review_status = ?, review_reason = ?, reviewed_by = ?, reviewed_at = ?
WHERE id = ? AND review_status = 'pending'
""",
(target, reason, actor["id"] if actor is not None else None, now, row["id"]),
)
if cursor.rowcount:
updated.append(row["sheet_name"])
if updated:
auth.audit(
connection,
f"sheet_{decision}",
actor=actor,
target=f"batch:{batch_id}",
detail=f"sheets:{','.join(updated)}" + (f";reason:{reason}" if reason else ""),
)
return {"updated": updated, "already": already}
+20
View File
@@ -40,3 +40,23 @@ class StatementBatch:
transactions: tuple[NormalizedTransaction, ...] transactions: tuple[NormalizedTransaction, ...]
warnings: tuple[str, ...] warnings: tuple[str, ...]
template_version: int = 1 template_version: int = 1
@dataclass(frozen=True)
class SheetResult:
"""Per-worksheet parse outcome with the four diagnostic evidence items.
``outcome`` is the deterministic parse verdict: ``parsed`` (transactions
normalized), ``exception`` (template/header/balance problems) or
``ignored`` (empty or single-cell sheets with no bank content). The human
decision about whether a parsed sheet may participate in calculations is
a separate lifecycle stage (``review_status``), persisted in
``sheet_reviews``; it is never decided here.
"""
sheet_name: str
outcome: str # parsed | exception | ignored
message: str | None = None
scanned_rows: int | None = None
candidate_headers: tuple[str, ...] = ()
batch: StatementBatch | None = None
+275
View File
@@ -0,0 +1,275 @@
"""Streaming multipart/form-data parser for the stdlib HTTP server.
The upload body is consumed in chunks from the request stream and never
assembled into memory as a whole. File parts are streamed straight to a temp
file on disk with an incremental size cap; text fields are capped at a small
field size. File bytes are preserved exactly (no trailing-byte trimming), the
first file part is the only one accepted, and both the extension and the file
signature are validated by the caller.
"""
from __future__ import annotations
from dataclasses import dataclass
from pathlib import Path
import re
from typing import BinaryIO
MAX_FIELD_BYTES = 16 * 1024
MAX_HEADER_BYTES = 32 * 1024
CHUNK_SIZE = 64 * 1024
# .xlsx files are ZIP containers (PK\x03\x04 local header or an empty-archive
# PK\x05\x06); .xls files are OLE2 compound documents.
XLSX_MAGIC = (b"PK\x03\x04", b"PK\x05\x06")
XLS_MAGIC = (b"\xd0\xcf\x11\xe0\xa1\xb1\x1a\xe1",)
class MultipartError(ValueError):
pass
@dataclass(frozen=True)
class UploadedFile:
filename: str
path: Path
size: int
def valid_file_signature(path: Path, suffix: str) -> bool:
"""True when the file's leading bytes match its declared extension."""
with path.open("rb") as handle:
head = handle.read(8)
if suffix == ".xlsx":
return any(head.startswith(magic) for magic in XLSX_MAGIC)
if suffix == ".xls":
return any(head.startswith(magic) for magic in XLS_MAGIC)
return False
class _MultipartStream:
def __init__(
self,
rfile: BinaryIO,
content_length: int,
content_type: str,
*,
max_body_bytes: int,
) -> None:
lowered = content_type.lower()
if not lowered.startswith("multipart/form-data"):
raise MultipartError("上传请求必须是 multipart/form-data。")
match = re.search(r"boundary=(?:\"([^\"]+)\"|([^;]+))", content_type)
if not match:
raise MultipartError("上传请求缺少文件边界。")
boundary = (match.group(1) or match.group(2)).strip().encode("utf-8")
if not boundary or len(boundary) > 200:
raise MultipartError("上传请求的文件边界无效。")
if content_length <= 0:
raise MultipartError("上传请求为空。")
if content_length > max_body_bytes:
raise MultipartError(f"上传请求超过 {max_body_bytes // (1024 * 1024)} MB 限制。")
self.rfile = rfile
self.remaining = content_length
self.total_read = 0
self.buffer = bytearray()
self.boundary = boundary
self._fill()
# ------------------------------------------------------------------
# Low-level stream helpers
# ------------------------------------------------------------------
def _fill(self) -> None:
if self.total_read >= self.remaining:
return
want = min(CHUNK_SIZE, self.remaining - self.total_read)
data = self.rfile.read(want)
if not data:
# Client closed early; clamp remaining so every helper sees EOF.
self.remaining = self.total_read
return
self.total_read += len(data)
self.buffer.extend(data)
def _take(self, count: int) -> bytes:
while len(self.buffer) < count:
if self.total_read >= self.remaining:
raise MultipartError("multipart 请求体不完整。")
self._fill()
out = bytes(self.buffer[:count])
del self.buffer[:count]
return out
def _read_line(self) -> bytes:
while True:
index = self.buffer.find(b"\n")
if index >= 0:
line = bytes(self.buffer[: index + 1])
del self.buffer[: index + 1]
return line
if self.total_read >= self.remaining:
if not self.buffer:
raise MultipartError("multipart 请求体不完整。")
line = bytes(self.buffer)
self.buffer.clear()
return line
self._fill()
def _skip_preamble(self) -> None:
marker = b"--" + self.boundary
while True:
index = self.buffer.find(marker)
if index >= 0:
del self.buffer[: index + len(marker)]
return
keep = len(marker) - 1
if len(self.buffer) > keep:
del self.buffer[: len(self.buffer) - keep]
if self.total_read >= self.remaining:
raise MultipartError("上传请求中没有找到文件。")
self._fill()
def _iter_content(self, boundary: bytes):
"""Yield content bytes up to (excluding) the ``\\r\\n--boundary`` marker."""
marker = b"\r\n--" + boundary
keep = len(marker) - 1
while True:
index = self.buffer.find(marker)
if index >= 0:
content = bytes(self.buffer[:index])
del self.buffer[:index]
if content:
yield content
return
if len(self.buffer) > keep:
safe = len(self.buffer) - keep
content = bytes(self.buffer[:safe])
del self.buffer[:safe]
if content:
yield content
if self.total_read >= self.remaining:
raise MultipartError("multipart 请求体缺少结束边界。")
self._fill()
def _after_content_boundary(self) -> str:
"""Consume the content-ending marker; return 'part' or 'end'."""
marker = b"\r\n--" + self.boundary
del self.buffer[: len(marker)]
indicator = self._take(2)
if indicator == b"--":
if self.buffer.startswith(b"\r\n"):
del self.buffer[:2]
elif self.buffer.startswith(b"\n"):
del self.buffer[:1]
return "end"
if indicator != b"\r\n":
raise MultipartError("上传请求格式无效。")
return "part"
# ------------------------------------------------------------------
# Part parsing
# ------------------------------------------------------------------
def _read_headers(self) -> bytes:
total = 0
lines: list[bytes] = []
while True:
line = self._read_line()
total += len(line)
if total > MAX_HEADER_BYTES:
raise MultipartError("multipart 头部过长。")
if line in (b"\r\n", b"\n"):
return b"".join(lines)
lines.append(line)
@staticmethod
def _parse_disposition(header: bytes) -> tuple[str | None, str | None]:
name = None
filename = None
name_match = re.search(br'name="([^"]*)"', header)
if name_match:
name = name_match.group(1).decode("utf-8", errors="replace")
filename_match = re.search(br'filename="([^"]*)"', header)
if filename_match:
filename = filename_match.group(1).decode("utf-8", errors="replace")
return name, filename
def parse(self, work_dir: Path) -> tuple[dict[str, str], UploadedFile | None]:
import tempfile
self._skip_preamble()
next_bytes = self._take(2)
if next_bytes == b"--":
# Empty multipart body: opening boundary immediately closes.
return {}, None
if next_bytes != b"\r\n":
raise MultipartError("上传请求格式无效。")
fields: dict[str, str] = {}
uploaded: UploadedFile | None = None
while True:
header = self._read_headers()
name, filename = self._parse_disposition(header)
if name is None:
raise MultipartError("上传请求缺少字段名。")
if filename is not None:
if uploaded is not None:
raise MultipartError("一次只能上传一个文件。")
with tempfile.NamedTemporaryFile(
dir=str(work_dir), prefix="upload-", delete=False
) as sink:
temp_path = Path(sink.name)
size = 0
try:
for chunk in self._iter_content(self.boundary):
size += len(chunk)
sink.write(chunk)
except Exception:
temp_path.unlink(missing_ok=True)
raise
uploaded = UploadedFile(
filename=filename, path=temp_path, size=size
)
if self._after_content_boundary() == "end":
break
else:
chunks: list[bytes] = []
total = 0
for chunk in self._iter_content(self.boundary):
total += len(chunk)
if total > MAX_FIELD_BYTES:
raise MultipartError("表单字段超过大小限制。")
chunks.append(chunk)
fields[name] = b"".join(chunks).decode(
"utf-8", errors="replace"
).strip()
if self._after_content_boundary() == "end":
break
return fields, uploaded
def parse_upload(
rfile: BinaryIO,
content_length: int,
content_type: str,
work_dir: str | Path,
*,
max_file_bytes: int,
) -> tuple[dict[str, str], UploadedFile | None]:
"""Stream a multipart upload; returns ``(fields, uploaded_file)``.
Raises :class:`MultipartError` for malformed or oversized bodies. The
caller owns the uploaded temp file and must remove it when done.
"""
work = Path(work_dir)
work.mkdir(parents=True, exist_ok=True)
stream = _MultipartStream(
rfile, content_length, content_type, max_body_bytes=max_file_bytes + MAX_FIELD_BYTES
)
fields, uploaded = stream.parse(work)
if uploaded is None:
raise MultipartError("上传请求中没有找到文件。")
if uploaded.size > max_file_bytes:
raise MultipartError(f"文件超过 {max_file_bytes // (1024 * 1024)} MB 限制。")
return fields, uploaded
+78 -22
View File
@@ -7,7 +7,7 @@ from pathlib import Path
import re import re
from typing import Any, Iterable from typing import Any, Iterable
from .models import NormalizedTransaction, StatementBatch from .models import NormalizedTransaction, SheetResult, StatementBatch
from .reader import RawSheet, read_workbook from .reader import RawSheet, read_workbook
from .templates import BankTemplate, TEMPLATES, normalize_header from .templates import BankTemplate, TEMPLATES, normalize_header
@@ -24,8 +24,11 @@ class AmbiguousTemplateError(StatementParseError):
pass pass
SCAN_LIMIT = 50
def detect_header( def detect_header(
rows: tuple[tuple[Any, ...], ...], scan_limit: int = 50 rows: tuple[tuple[Any, ...], ...], scan_limit: int = SCAN_LIMIT
) -> tuple[BankTemplate, int, dict[str, int]]: ) -> tuple[BankTemplate, int, dict[str, int]]:
candidates: list[tuple[int, BankTemplate, int, dict[str, int]]] = [] candidates: list[tuple[int, BankTemplate, int, dict[str, int]]] = []
for row_index, row in enumerate(rows[:scan_limit]): for row_index, row in enumerate(rows[:scan_limit]):
@@ -42,8 +45,8 @@ def detect_header(
if not candidates: if not candidates:
inspected = min(len(rows), scan_limit) inspected = min(len(rows), scan_limit)
candidate_rows = _header_candidate_summary(rows[:scan_limit]) candidate_rows = _header_candidate_rows(rows[:scan_limit])
detail = f";候选表头:{candidate_rows}" if candidate_rows else "" detail = f";候选表头:{''.join(candidate_rows)}" if candidate_rows else ""
raise UnknownTemplateError( raise UnknownTemplateError(
f"未识别到受支持的银行表头(已扫描前 {inspected} 行){detail}" f"未识别到受支持的银行表头(已扫描前 {inspected} 行){detail}"
) )
@@ -60,21 +63,69 @@ def detect_header(
def parse_statement(path: str | Path) -> tuple[StatementBatch, ...]: def parse_statement(path: str | Path) -> tuple[StatementBatch, ...]:
source = Path(path) batches = tuple(result.batch for result in analyze_workbook(path) if result.batch)
batches: list[StatementBatch] = []
errors: list[str] = []
for sheet in read_workbook(source):
if not any(any(_text(value) for value in row) for row in sheet.rows):
continue
try:
batches.append(_parse_sheet(source, sheet))
except UnknownTemplateError as exc:
errors.append(f"{sheet.name}: {exc}")
if not batches: if not batches:
detail = "; ".join(errors) or "Workbook contains no readable worksheets." raise UnknownTemplateError(f"{Path(path).name}: 工作簿中没有可解析的工作表。")
raise UnknownTemplateError(f"{source.name}: {detail}") return batches
return tuple(batches)
def analyze_workbook(path: str | Path) -> tuple[SheetResult, ...]:
"""Parse every worksheet into an independent result.
Each worksheet yields exactly one :class:`SheetResult` whose ``outcome``
is ``parsed``, ``exception`` or ``ignored``. Unreadable workbooks raise
``CorruptWorkbookError``; a workbook that reads but contains no parsable
sheet still returns one result per sheet so the UI can surface the
filename / sheet / scanned range / candidate headers evidence.
"""
source = Path(path)
return tuple(_analyze_sheet(source, sheet) for sheet in read_workbook(source))
def _analyze_sheet(source: Path, sheet: RawSheet) -> SheetResult:
rows = sheet.rows
scanned = min(len(rows), SCAN_LIMIT)
if not rows:
return SheetResult(
sheet_name=sheet.name,
outcome="ignored",
message=f"工作表「{sheet.name}」为空,已跳过。",
scanned_rows=0,
)
if not any(any(_text(value) for value in row) for row in rows):
return SheetResult(
sheet_name=sheet.name,
outcome="ignored",
message=f"工作表「{sheet.name}」无有效内容,已跳过。",
scanned_rows=scanned,
)
try:
batch = _parse_sheet(source, sheet)
except UnknownTemplateError as exc:
return SheetResult(
sheet_name=sheet.name,
outcome="exception",
message=_clean_sheet_message(str(exc), source),
scanned_rows=scanned,
candidate_headers=_header_candidate_rows(rows),
)
except (AmbiguousTemplateError, StatementParseError) as exc:
return SheetResult(
sheet_name=sheet.name,
outcome="exception",
message=_clean_sheet_message(str(exc), source),
scanned_rows=scanned,
)
return SheetResult(
sheet_name=sheet.name,
outcome="parsed",
scanned_rows=scanned,
batch=batch,
)
def _clean_sheet_message(message: str, source: Path) -> str:
return message.replace(str(source), source.name).replace(source.name, "本文件")
def parse_directory(path: str | Path) -> tuple[StatementBatch, ...]: def parse_directory(path: str | Path) -> tuple[StatementBatch, ...]:
@@ -90,9 +141,14 @@ def parse_directory(path: str | Path) -> tuple[StatementBatch, ...]:
return tuple(batch for file in files for batch in parse_statement(file)) return tuple(batch for file in files for batch in parse_statement(file))
def _header_candidate_summary( def _header_candidate_rows(
rows: tuple[tuple[Any, ...], ...], limit: int = 3 rows: tuple[tuple[Any, ...], ...], limit: int = 3
) -> str: ) -> tuple[str, ...]:
"""Preview up to ``limit`` plausible header rows.
Every non-empty row counts as a candidate, including single-cell rows, so
an ambiguous workbook always has explicit scan evidence for the UI.
"""
known_headers = { known_headers = {
alias alias
for template in TEMPLATES for template in TEMPLATES
@@ -102,7 +158,7 @@ def _header_candidate_summary(
candidates: list[tuple[int, int, int, tuple[str, ...]]] = [] candidates: list[tuple[int, int, int, tuple[str, ...]]] = []
for row_index, row in enumerate(rows): for row_index, row in enumerate(rows):
values = tuple(_text(value) for value in row if _text(value)) values = tuple(_text(value) for value in row if _text(value))
if len(values) < 2: if not values:
continue continue
matched = sum(normalize_header(value) in known_headers for value in values) matched = sum(normalize_header(value) in known_headers for value in values)
candidates.append((matched, len(values), row_index, values)) candidates.append((matched, len(values), row_index, values))
@@ -114,7 +170,7 @@ def _header_candidate_summary(
if len(values) > 8: if len(values) > 8:
preview += "……" preview += "……"
summaries.append(f"{row_index + 1} 行「{preview}") summaries.append(f"{row_index + 1} 行「{preview}")
return "".join(summaries) return tuple(summaries)
def _parse_sheet(source: Path, sheet: RawSheet) -> StatementBatch: def _parse_sheet(source: Path, sheet: RawSheet) -> StatementBatch:
+59 -13
View File
@@ -10,6 +10,23 @@ class UnsupportedWorkbookError(RuntimeError):
pass pass
class CorruptWorkbookError(RuntimeError):
"""The file cannot be opened as a valid Excel workbook.
Raised for corrupt/truncated files, wrong signatures and files that
exceed the resource limits. The message is user-safe: it never contains
server-side paths or the content-addressed storage filename.
"""
# Workbook resource guards (defense against decompression bombs and
# accidentally giant exports). Bank statements are small; these bounds are
# generous enough for real exports while keeping memory bounded.
MAX_SHEETS = 50
MAX_ROWS_PER_SHEET = 200_000
MAX_COLS_PER_SHEET = 64
@dataclass(frozen=True) @dataclass(frozen=True)
class RawSheet: class RawSheet:
name: str name: str
@@ -18,11 +35,18 @@ class RawSheet:
def read_workbook(path: Path) -> tuple[RawSheet, ...]: def read_workbook(path: Path) -> tuple[RawSheet, ...]:
suffix = path.suffix.lower() suffix = path.suffix.lower()
if suffix == ".xlsx": try:
return _read_xlsx(path) if suffix == ".xlsx":
if suffix == ".xls": return _read_xlsx(path)
return _read_xls(path) if suffix == ".xls":
raise UnsupportedWorkbookError(f"Unsupported workbook type: {suffix}") return _read_xls(path)
raise UnsupportedWorkbookError(f"Unsupported workbook type: {suffix}")
except (UnsupportedWorkbookError, CorruptWorkbookError):
raise
except Exception as exc:
raise CorruptWorkbookError(
"文件无法读取,可能已损坏或不是有效的 Excel 文件。"
) from exc
def _read_xlsx(path: Path) -> tuple[RawSheet, ...]: def _read_xlsx(path: Path) -> tuple[RawSheet, ...]:
@@ -35,13 +59,24 @@ def _read_xlsx(path: Path) -> tuple[RawSheet, ...]:
workbook = load_workbook(path, read_only=True, data_only=True) workbook = load_workbook(path, read_only=True, data_only=True)
try: try:
return tuple( sheets: list[RawSheet] = []
RawSheet( for worksheet in workbook.worksheets:
name=worksheet.title, rows: list[tuple[Any, ...]] = []
rows=tuple(tuple(row) for row in worksheet.iter_rows(values_only=True)), for row_index, row in enumerate(worksheet.iter_rows(values_only=True)):
) if row_index >= MAX_ROWS_PER_SHEET:
for worksheet in workbook.worksheets raise CorruptWorkbookError(
) f"工作表「{worksheet.title}」超过 {MAX_ROWS_PER_SHEET} 行,已拒绝读取。"
)
values = tuple(row)
if len(values) > MAX_COLS_PER_SHEET:
raise CorruptWorkbookError(
f"工作表「{worksheet.title}」列数超过 {MAX_COLS_PER_SHEET},已拒绝读取。"
)
rows.append(values)
sheets.append(RawSheet(name=worksheet.title, rows=tuple(rows)))
if len(sheets) > MAX_SHEETS:
raise CorruptWorkbookError(f"工作簿工作表数量超过 {MAX_SHEETS} 个,已拒绝读取。")
return tuple(sheets)
finally: finally:
workbook.close() workbook.close()
@@ -57,8 +92,19 @@ def _read_xls(path: Path) -> tuple[RawSheet, ...]:
workbook = xlrd.open_workbook(path, on_demand=True) workbook = xlrd.open_workbook(path, on_demand=True)
sheets: list[RawSheet] = [] sheets: list[RawSheet] = []
try: try:
for sheet_name in workbook.sheet_names(): names = workbook.sheet_names()
if len(names) > MAX_SHEETS:
raise CorruptWorkbookError(f"工作簿工作表数量超过 {MAX_SHEETS} 个,已拒绝读取。")
for sheet_name in names:
worksheet = workbook.sheet_by_name(sheet_name) worksheet = workbook.sheet_by_name(sheet_name)
if worksheet.nrows > MAX_ROWS_PER_SHEET:
raise CorruptWorkbookError(
f"工作表「{sheet_name}」超过 {MAX_ROWS_PER_SHEET} 行,已拒绝读取。"
)
if worksheet.ncols > MAX_COLS_PER_SHEET:
raise CorruptWorkbookError(
f"工作表「{sheet_name}」列数超过 {MAX_COLS_PER_SHEET},已拒绝读取。"
)
rows: list[tuple[Any, ...]] = [] rows: list[tuple[Any, ...]] = []
for row_index in range(worksheet.nrows): for row_index in range(worksheet.nrows):
values: list[Any] = [] values: list[Any] = []
+587
View File
@@ -0,0 +1,587 @@
"""HTTP integration tests for the hardened import API (B-42).
Covers streaming multipart limits, per-worksheet parse results with the four
diagnostic evidence items, the confirm/ignore review lifecycle with tenant
scoping and idempotency, unconfirmed-sheet export gating, and concurrent
duplicate uploads. Uses a real ``ThreadingHTTPServer`` like
``test_server_auth``.
"""
from __future__ import annotations
import io
import itertools
import json
import os
from pathlib import Path
import tempfile
import threading
import unittest
from openpyxl import Workbook
from bank_importer import auth
from bank_importer.db import connect, migrate
import server
from test_server_auth import Client, as_json
ROOT = Path(__file__).resolve().parents[1]
SAMPLES = ROOT / "流水模板"
CCB_SAMPLE = SAMPLES / "中国建设银行账户流水.xls"
CITIC_SAMPLE = SAMPLES / "中信银行账户流水.xlsx"
BOOTSTRAP_PASSWORD = "BootAdmin123"
ADMIN_PASSWORD = "AdminPass123"
# ---------------------------------------------------------------------------
# Fixtures
# ---------------------------------------------------------------------------
CCB_HEADER = [
"客户账号", "账户名称", "交易时间", "借方发生额(支取)", "贷方发生额(收入)",
"余额", "币种", "对方户名", "对方账号", "对方开户机构", "摘要", "备注",
]
CCB_DATA = [
["6228480000000000", "测试公司", "2026-01-05 10:00:00", "100.00", "", "99900.00", "RMB", "供应商", "1002003004", "某银行", "货款", ""],
["6228480000000000", "测试公司", "2026-01-06 11:00:00", "", "200.00", "100100.00", "RMB", "客户", "2003004005", "某银行", "收款", ""],
]
def workbook_bytes(sheets):
"""Build an xlsx in memory.
``sheets`` is a list of ``(name, rows)``; an empty ``rows`` list means an
empty worksheet.
"""
workbook = Workbook()
workbook.remove(workbook.active)
for name, rows in sheets:
worksheet = workbook.create_sheet(name)
for row in rows:
worksheet.append(row)
buffer = io.BytesIO()
workbook.save(buffer)
return buffer.getvalue()
# Workbook fixtures are content-hashed for dedupe, so every helper produces
# distinct bytes per call (each test uploads a genuinely new file).
_counter = itertools.count()
def multi_sheet_bytes():
"""Valid CCB sheet + unknown-header sheet + empty sheet in one workbook."""
return workbook_bytes(
[
("正常流水", [CCB_HEADER, *CCB_DATA]),
("未知模板", [["日期", "金额", "备注"], ["2026-01-01", next(_counter), "x"]]),
("空表", []),
]
)
def empty_workbook_bytes():
return workbook_bytes([("Sheet1", [])])
def single_cell_bytes():
return workbook_bytes([("候选", [[f"只有一个单元格 {next(_counter)}"]])])
def unknown_header_bytes():
return workbook_bytes([("流水", [["日期", "金额", "备注"], ["2026-01-01", next(_counter), "x"]])])
# ---------------------------------------------------------------------------
# Test server
# ---------------------------------------------------------------------------
class ImportApiServerTests(unittest.TestCase):
@classmethod
def setUpClass(cls) -> None:
cls.temp_dir = tempfile.TemporaryDirectory()
root = Path(cls.temp_dir.name)
cls.db_path = root / "app.db"
cls.storage = root / "files"
cls._old_db_path = server.DB_PATH
cls._old_storage = server.STORAGE_DIR
server.DB_PATH = cls.db_path
server.STORAGE_DIR = cls.storage
os.environ["APP_BOOTSTRAP_ADMIN_PASSWORD"] = BOOTSTRAP_PASSWORD
connection = connect(cls.db_path)
migrate(connection)
generated = server.ensure_bootstrap_admin(connection)
assert generated is None
connection.close()
class QuietHandler(server.AppHandler):
def log_message(self, *args) -> None:
pass
cls.httpd = server.ThreadingHTTPServer(("127.0.0.1", 0), QuietHandler)
cls.port = cls.httpd.server_address[1]
cls.thread = threading.Thread(target=cls.httpd.serve_forever, daemon=True)
cls.thread.start()
cls.admin = Client("127.0.0.1", cls.port)
status, _, data = cls.admin.post_json(
"/api/login",
{"username": "group-admin", "password": BOOTSTRAP_PASSWORD, "portal": "admin"},
)
assert status == 200, data
status, _, data = cls.admin.post_json(
"/api/password/change",
{"old_password": BOOTSTRAP_PASSWORD, "new_password": ADMIN_PASSWORD},
)
assert status == 200, data
status, _, data = cls.admin.post_json("/api/admin/companies", {"name": "甲公司"})
assert status == 200, data
cls.company_a = as_json(data)["company_id"]
status, _, data = cls.admin.post_json("/api/admin/companies", {"name": "乙公司"})
assert status == 200, data
cls.company_b = as_json(data)["company_id"]
cls.cashier_a, cls.cashier_a_password = cls.create_company_user(
cls.admin, cls.port, "cashier-a"
)
cls.cashier_b, cls.cashier_b_password = cls.create_company_user(
cls.admin, cls.port, "cashier-b"
)
@classmethod
def create_company_user(cls, admin, port, username):
status, _, data = admin.post_json(
"/api/admin/users", {"username": username, "company_id": cls.company_a}
)
assert status == 200, data
initial = as_json(data)["initial_password"]
client = Client("127.0.0.1", port)
status, _, data = client.post_json(
"/api/login",
{"username": username, "password": initial, "portal": "company"},
)
assert status == 200, data
new_password = "Changed456"
status, _, data = client.post_json(
"/api/password/change",
{"old_password": initial, "new_password": new_password},
)
assert status == 200, data
return client, new_password
@classmethod
def tearDownClass(cls) -> None:
cls.httpd.shutdown()
cls.httpd.server_close()
server.DB_PATH = cls._old_db_path
server.STORAGE_DIR = cls._old_storage
os.environ.pop("APP_BOOTSTRAP_ADMIN_PASSWORD", None)
cls.temp_dir.cleanup()
def fresh_client(self) -> Client:
return Client("127.0.0.1", self.port)
def upload(self, client, content: bytes, filename: str = "语句.xlsx", fields=None):
return client.post_multipart(
"/api/parse", fields or {}, filename, content
)
def upload_for_company(self, content: bytes, company_id: int, filename: str = "语句.xlsx"):
return self.admin.post_multipart(
"/api/parse", {"company_id": str(company_id)}, filename, content
)
def upload_for_a(self, content: bytes, filename: str = "语句.xlsx"):
return self.cashier_a.post_multipart("/api/parse", {}, filename, content)
def batch_sheets(self, client, batch_id: int):
status, _, data = client.get(f"/api/batches/{batch_id}/sheets")
self.assertEqual(200, status, data)
return as_json(data)["sheets"]
def confirm(self, client, batch_id: int, sheets, reason=None):
payload = {"sheets": sheets}
if reason is not None:
payload["reason"] = reason
return client.post_json(f"/api/batches/{batch_id}/confirm", payload)
def ignore(self, client, batch_id: int, sheets, reason):
return client.post_json(
f"/api/batches/{batch_id}/ignore", {"sheets": sheets, "reason": reason}
)
# ------------------------------------------------------------------
# Multipart validation
# ------------------------------------------------------------------
def test_oversized_body_returns_stable_error(self) -> None:
content = b"x" * (server.MAX_UPLOAD_BYTES + 1)
status, _, data = self.upload_for_a(content)
self.assertEqual(422, status)
payload = as_json(data)
self.assertIn("20 MB", payload["message"])
self.assertNotIn(server.STORAGE_DIR.as_posix(), payload["message"])
self.assertNotIn(str(self.storage), payload["message"])
def test_missing_boundary_returns_stable_error(self) -> None:
status, _, data = self.cashier_a.request(
"POST",
"/api/parse",
body=b"whatever",
headers={"Content-Type": "multipart/form-data"},
)
self.assertEqual(422, status)
self.assertIn("边界", as_json(data)["message"])
def test_wrong_content_type_returns_stable_error(self) -> None:
status, _, data = self.cashier_a.request(
"POST",
"/api/parse",
body=CCB_SAMPLE.read_bytes(),
headers={"Content-Type": "application/octet-stream"},
)
self.assertEqual(422, status)
def test_missing_content_length_returns_stable_error(self) -> None:
boundary = "----missingcl"
body = f"--{boundary}--\r\n".encode()
status, _, data = self.cashier_a.request(
"POST",
"/api/parse",
body=body,
headers={"Content-Type": f"multipart/form-data; boundary={boundary}"},
)
self.assertEqual(422, status)
def test_bad_extension_returns_stable_error(self) -> None:
status, _, data = self.upload_for_a(CCB_SAMPLE.read_bytes(), filename="statement.pdf")
self.assertEqual(422, status)
self.assertIn(".xls", as_json(data)["message"])
def test_signature_mismatch_returns_stable_error(self) -> None:
# Valid .xls content renamed with a .xlsx extension must be rejected.
status, _, data = self.upload_for_a(CCB_SAMPLE.read_bytes(), filename="fake.xlsx")
self.assertEqual(422, status)
self.assertIn("损坏", as_json(data)["message"])
def test_corrupt_file_returns_stable_error_without_temp_path(self) -> None:
status, _, data = self.upload_for_a(b"PK\x03\x04 not a real zip at all")
self.assertEqual(422, status)
payload = as_json(data)
text = json.dumps(payload, ensure_ascii=False)
self.assertNotIn(str(self.storage), text)
self.assertNotIn("temp", text.lower())
self.assertNotIn("data/files", text)
def test_two_file_parts_rejected(self) -> None:
boundary = "----twofilesboundary"
content = CCB_SAMPLE.read_bytes()
crlf = b"\r\n"
body = (
b"--" + boundary.encode() + crlf
+ b'Content-Disposition: form-data; name="file"; filename="a.xls"' + crlf
+ b"Content-Type: application/octet-stream" + crlf + crlf
+ content + crlf
+ b"--" + boundary.encode() + crlf
+ b'Content-Disposition: form-data; name="file"; filename="b.xls"' + crlf
+ b"Content-Type: application/octet-stream" + crlf + crlf
+ content + crlf
+ b"--" + boundary.encode() + b"--" + crlf
)
status, _, data = self.cashier_a.request(
"POST",
"/api/parse",
body=body,
headers={"Content-Type": f"multipart/form-data; boundary={boundary}"},
)
self.assertEqual(422, status)
def test_no_file_part_rejected(self) -> None:
boundary = "----nofile"
crlf = b"\r\n"
body = (
b"--" + boundary.encode() + crlf
+ b'Content-Disposition: form-data; name="company_id"' + crlf + crlf
+ b"1" + crlf
+ b"--" + boundary.encode() + b"--" + crlf
)
status, _, data = self.cashier_a.request(
"POST",
"/api/parse",
body=body,
headers={"Content-Type": f"multipart/form-data; boundary={boundary}"},
)
self.assertEqual(422, status)
self.assertIn("文件", as_json(data)["message"])
def test_upload_trailing_bytes_preserved_exactly(self) -> None:
# The stored source file must be byte-identical to what was uploaded
# even when it ends with CRLF/LF; the parser must never trim tail
# bytes, and the upload temp must not appear anywhere in the payload.
content = CCB_SAMPLE.read_bytes()
uploaded = content + b"\r\n\r\n\x00\x01"
status, _, data = self.upload_for_a(uploaded, filename="尾部字节.xls")
self.assertEqual(200, status, data)
batch_id = as_json(data)["batch_id"]
connection = connect(self.db_path)
try:
row = connection.execute(
"""
SELECT f.storage_path FROM source_files f
JOIN import_batches b ON b.source_file_id = f.id
WHERE b.id = ?
""",
(batch_id,),
).fetchone()
finally:
connection.close()
self.assertIsNotNone(row)
self.assertEqual(uploaded, Path(row["storage_path"]).read_bytes())
self.assertNotIn("upload-", json.dumps(as_json(data), ensure_ascii=False))
# ------------------------------------------------------------------
# Per-sheet results and four-item evidence
# ------------------------------------------------------------------
def test_multi_sheet_workbook_returns_every_sheet_result(self) -> None:
status, _, data = self.upload_for_a(multi_sheet_bytes(), filename="多工作表.xlsx")
self.assertEqual(200, status, data)
payload = as_json(data)
self.assertEqual("parsed", payload["status"])
self.assertEqual("多工作表.xlsx", payload.get("original_filename"))
sheets = payload["sheets"]
self.assertEqual(3, len(sheets))
by_name = {sheet["sheet_name"]: sheet for sheet in sheets}
self.assertEqual("parsed", by_name["正常流水"]["outcome"])
self.assertEqual("pending", by_name["正常流水"]["review_status"])
self.assertEqual("中国建设银行", by_name["正常流水"]["bank"])
self.assertEqual(2, by_name["正常流水"]["transactions"])
unknown = by_name["未知模板"]
self.assertEqual("exception", unknown["outcome"])
self.assertIn("未识别到受支持的银行表头", unknown["message"])
self.assertGreaterEqual(unknown["scanned_rows"], 2)
self.assertTrue(unknown["candidate_headers"])
self.assertIn("日期、金额、备注", "".join(unknown["candidate_headers"]))
empty = by_name["空表"]
self.assertEqual("ignored", empty["outcome"])
self.assertIn("为空", empty["message"])
self.assertEqual(0, empty["scanned_rows"])
def test_batch_detail_matches_parse_response(self) -> None:
status, _, data = self.upload_for_a(multi_sheet_bytes(), filename="详情.xlsx")
batch_id = as_json(data)["batch_id"]
sheets = self.batch_sheets(self.cashier_a, batch_id)
self.assertEqual(3, len(sheets))
status, _, data = self.cashier_a.get(f"/api/batches/{batch_id}/sheets")
self.assertEqual(200, status)
detail = as_json(data)
self.assertEqual("详情.xlsx", detail["original_filename"])
def test_unknown_header_file_returns_four_evidence_items(self) -> None:
status, _, data = self.upload_for_a(unknown_header_bytes(), filename="未知银行.xlsx")
self.assertEqual(422, status, data)
payload = as_json(data)
self.assertEqual("exception", payload["status"])
self.assertEqual("未知银行.xlsx", payload.get("original_filename"))
sheet = payload["sheets"][0]
self.assertEqual("exception", sheet["outcome"])
self.assertEqual("流水", sheet["sheet_name"])
self.assertGreaterEqual(sheet["scanned_rows"], 1)
self.assertTrue(sheet["candidate_headers"])
text = json.dumps(payload, ensure_ascii=False)
for evidence in (str(self.storage), "data/files", ".uploads", "upload-"):
self.assertNotIn(evidence, text)
def test_empty_workbook_returns_evidence(self) -> None:
status, _, data = self.upload_for_a(empty_workbook_bytes(), filename="空工作簿.xlsx")
self.assertEqual(422, status, data)
payload = as_json(data)
self.assertEqual("空工作簿.xlsx", payload.get("original_filename"))
sheet = payload["sheets"][0]
self.assertEqual("ignored", sheet["outcome"])
self.assertEqual("Sheet1", sheet["sheet_name"])
self.assertEqual(0, sheet["scanned_rows"])
def test_single_cell_sheet_returns_candidate_evidence(self) -> None:
status, _, data = self.upload_for_a(single_cell_bytes(), filename="单格.xlsx")
self.assertEqual(422, status, data)
sheet = as_json(data)["sheets"][0]
self.assertEqual("exception", sheet["outcome"])
self.assertEqual("候选", sheet["sheet_name"])
self.assertEqual(1, sheet["scanned_rows"])
self.assertTrue(sheet["candidate_headers"])
self.assertIn("只有一个单元格", "".join(sheet["candidate_headers"]))
# ------------------------------------------------------------------
# Confirm / ignore lifecycle
# ------------------------------------------------------------------
def test_confirm_and_ignore_lifecycle(self) -> None:
status, _, data = self.upload_for_a(multi_sheet_bytes(), filename="确认.xlsx")
payload = as_json(data)
batch_id = payload["batch_id"]
names = [s["sheet_name"] for s in payload["sheets"]]
self.assertIn("正常流水", names)
# Confirm the parsed sheet; exception/ignored cannot be confirmed.
status, _, data = self.confirm(self.cashier_a, batch_id, ["正常流水"])
self.assertEqual(200, status, data)
result = as_json(data)
self.assertEqual(["正常流水"], result["updated"])
sheet = next(s for s in result["sheets"] if s["sheet_name"] == "正常流水")
self.assertEqual("confirmed", sheet["review_status"])
# Confirming an exception sheet is a conflict.
status, _, data = self.confirm(self.cashier_a, batch_id, ["未知模板"])
self.assertEqual(409, status, data)
# Ignore the exception sheet with a reason.
status, _, data = self.ignore(self.cashier_a, batch_id, ["未知模板"], "模板待补充")
self.assertEqual(200, status, data)
sheet = next(s for s in as_json(data)["sheets"] if s["sheet_name"] == "未知模板")
self.assertEqual("ignored", sheet["review_status"])
self.assertEqual("模板待补充", sheet["review_reason"])
# Idempotent repeat of an already-applied decision.
status, _, data = self.confirm(self.cashier_a, batch_id, ["正常流水"])
self.assertEqual(200, status, data)
result = as_json(data)
self.assertEqual([], result["updated"])
self.assertEqual(["正常流水"], result["already"])
# Changing a settled decision is a conflict.
status, _, data = self.ignore(self.cashier_a, batch_id, ["正常流水"], "改主意")
self.assertEqual(409, status, data)
def test_ignore_requires_reason(self) -> None:
status, _, data = self.upload_for_a(multi_sheet_bytes(), filename="忽略原因.xlsx")
batch_id = as_json(data)["batch_id"]
status, _, data = self.cashier_a.post_json(
f"/api/batches/{batch_id}/ignore", {"sheets": ["未知模板"], "reason": " "}
)
self.assertEqual(400, status, data)
self.assertIn("原因", as_json(data)["message"])
def test_unknown_sheet_name_is_rejected_and_atomic(self) -> None:
status, _, data = self.upload_for_a(multi_sheet_bytes(), filename="原子.xlsx")
batch_id = as_json(data)["batch_id"]
# One valid + one unknown sheet in the same request must not leave a
# half-applied confirmation behind.
status, _, data = self.confirm(self.cashier_a, batch_id, ["正常流水", "不存在"])
self.assertEqual(400, status, data)
sheet = next(
s for s in self.batch_sheets(self.cashier_a, batch_id)
if s["sheet_name"] == "正常流水"
)
self.assertEqual("pending", sheet["review_status"])
def test_company_cannot_confirm_another_companys_batch(self) -> None:
status, _, data = self.upload_for_company(multi_sheet_bytes(), self.company_b)
batch_b = as_json(data)["batch_id"]
status, _, data = self.confirm(self.cashier_a, batch_b, ["正常流水"])
self.assertEqual(404, status, data)
status, _, data = self.cashier_a.get(f"/api/batches/{batch_b}/sheets")
self.assertEqual(404, status, data)
def test_export_excludes_unconfirmed_and_ignore_keeps_excluded(self) -> None:
def export_line_count() -> int:
status, _, data = self.cashier_a.get("/api/export.csv")
self.assertEqual(200, status)
return len([line for line in data.decode("utf-8-sig").splitlines() if line])
baseline = export_line_count()
# Unconfirmed upload: export stays unchanged (no downstream leak).
status, _, data = self.upload_for_a(CCB_SAMPLE.read_bytes(), filename="导出.xls")
batch_id = as_json(data)["batch_id"]
self.assertEqual("parsed", as_json(data)["status"])
self.assertEqual(baseline, export_line_count())
# Confirm → the sheet's rows enter the export.
sheets = self.batch_sheets(self.cashier_a, batch_id)
name = sheets[0]["sheet_name"]
status, _, data = self.confirm(self.cashier_a, batch_id, [name])
self.assertEqual(200, status, data)
self.assertGreater(export_line_count(), baseline)
# A confirmed sheet cannot be silently revoked: the settled decision
# is a conflict, not a silent data removal.
status, _, data = self.ignore(self.cashier_a, batch_id, [name], "取消")
self.assertEqual(409, status, data)
self.assertGreater(export_line_count(), baseline)
# A sheet that was only ever ignored never enters the export.
status, _, data = self.upload_for_a(multi_sheet_bytes(), filename="忽略.xlsx")
batch2 = as_json(data)["batch_id"]
parsed = [
sheet for sheet in self.batch_sheets(self.cashier_a, batch2)
if sheet["outcome"] == "parsed"
]
self.assertTrue(parsed)
before_ignore = export_line_count()
status, _, data = self.ignore(
self.cashier_a, batch2, [parsed[0]["sheet_name"]], "该表复核后不采用"
)
self.assertEqual(200, status, data)
self.assertEqual(before_ignore, export_line_count())
# ------------------------------------------------------------------
# Concurrency
# ------------------------------------------------------------------
def test_concurrent_duplicate_upload_keeps_one_fact_set(self) -> None:
content = CITIC_SAMPLE.read_bytes()
results: list[tuple[int, dict]] = []
errors: list[Exception] = []
barrier = threading.Barrier(2)
def upload() -> None:
try:
client = self.fresh_client()
client.cookies.update(self.cashier_a.cookies)
barrier.wait(timeout=10)
status, _, data = client.post_multipart(
"/api/parse", {}, "并发.xlsx", content
)
results.append((status, as_json(data)))
except Exception as exc: # pragma: no cover
errors.append(exc)
threads = [threading.Thread(target=upload) for _ in range(2)]
for thread in threads:
thread.start()
for thread in threads:
thread.join(timeout=30)
self.assertEqual([], errors)
self.assertEqual(2, len(results))
statuses = sorted(payload["status"] for _, payload in results)
self.assertEqual(["duplicate", "parsed"], statuses)
connection = connect(self.db_path)
try:
# The concurrent upload adds exactly one source file: identical
# bytes never create a second fact set.
count = connection.execute(
"SELECT COUNT(*) AS n FROM source_files WHERE original_filename = '并发.xlsx'"
).fetchone()["n"]
self.assertEqual(1, count)
duplicates = connection.execute(
"SELECT COUNT(*) AS n FROM import_batches WHERE status = 'duplicate'"
).fetchone()["n"]
self.assertGreaterEqual(duplicates, 1)
finally:
connection.close()
if __name__ == "__main__":
unittest.main()
+67 -1
View File
@@ -1,10 +1,19 @@
from __future__ import annotations from __future__ import annotations
from datetime import date from datetime import date
import io
from pathlib import Path from pathlib import Path
import tempfile
import unittest import unittest
from bank_importer.parser import UnknownTemplateError, detect_header, parse_directory from openpyxl import Workbook
from bank_importer.parser import (
UnknownTemplateError,
analyze_workbook,
detect_header,
parse_directory,
)
ROOT = Path(__file__).resolve().parents[1] ROOT = Path(__file__).resolve().parents[1]
@@ -83,5 +92,62 @@ class StatementParserTests(unittest.TestCase):
self.assertIn("日期、金额、备注", message) self.assertIn("日期、金额、备注", message)
class SheetResultTests(unittest.TestCase):
"""Per-worksheet outcomes: parsed / exception / ignored with evidence."""
def setUp(self) -> None:
self.temp_dir = tempfile.TemporaryDirectory()
self.addCleanup(self.temp_dir.cleanup)
def _write(self, sheets) -> Path:
workbook = Workbook()
workbook.remove(workbook.active)
for name, rows in sheets.items():
worksheet = workbook.create_sheet(name)
for row in rows:
worksheet.append(row)
path = Path(self.temp_dir.name) / "workbook.xlsx"
workbook.save(path)
return path
def test_every_sheet_has_an_independent_result(self) -> None:
path = self._write(
{
"正常": [
["客户账号", "账户名称", "交易时间", "借方发生额(支取)", "贷方发生额(收入)", "余额", "对方账号"],
["6228480000000000", "测试", "2026-01-05 10:00:00", "100.00", "", "99900.00", "1002003004"],
],
"未知": [["日期", "金额", "备注"], ["2026-01-01", "100", "x"]],
"空表": [],
}
)
results = analyze_workbook(path)
by_name = {result.sheet_name: result for result in results}
self.assertEqual({"正常", "未知", "空表"}, set(by_name))
self.assertEqual("parsed", by_name["正常"].outcome)
self.assertIsNotNone(by_name["正常"].batch)
self.assertEqual("exception", by_name["未知"].outcome)
self.assertIn("未识别到受支持的银行表头", by_name["未知"].message)
self.assertGreaterEqual(by_name["未知"].scanned_rows, 2)
self.assertTrue(any("日期、金额、备注" in row for row in by_name["未知"].candidate_headers))
self.assertEqual("ignored", by_name["空表"].outcome)
self.assertEqual(0, by_name["空表"].scanned_rows)
def test_single_cell_rows_are_candidate_evidence(self) -> None:
path = self._write({"候选": [["只有一个单元格"], ["又一格"]]})
(result,) = analyze_workbook(path)
self.assertEqual("exception", result.outcome)
self.assertEqual(2, result.scanned_rows)
self.assertTrue(result.candidate_headers)
self.assertIn("只有一个单元格", "".join(result.candidate_headers))
def test_messages_never_contain_stored_filename(self) -> None:
path = self._write({"流水": [["日期", "金额", "备注"]]})
(result,) = analyze_workbook(path)
self.assertEqual("exception", result.outcome)
self.assertNotIn(path.name, result.message)
self.assertNotIn(str(path), result.message)
if __name__ == "__main__": if __name__ == "__main__":
unittest.main() unittest.main()
+5 -4
View File
@@ -41,7 +41,7 @@ class PersistenceTestCase(unittest.TestCase):
class MigrationTests(PersistenceTestCase): class MigrationTests(PersistenceTestCase):
def test_migrate_creates_schema_and_is_idempotent(self) -> None: def test_migrate_creates_schema_and_is_idempotent(self) -> None:
first = applied_versions(self.connection) first = applied_versions(self.connection)
self.assertEqual([1, 2, 3], first) self.assertEqual([1, 2, 3, 4], first)
self.assertEqual([], migrate(self.connection)) self.assertEqual([], migrate(self.connection))
self.assertEqual(first, applied_versions(self.connection)) self.assertEqual(first, applied_versions(self.connection))
tables = { tables = {
@@ -58,6 +58,7 @@ class MigrationTests(PersistenceTestCase):
"source_files", "source_files",
"import_batches", "import_batches",
"sheet_batches", "sheet_batches",
"sheet_reviews",
"source_rows", "source_rows",
"import_exceptions", "import_exceptions",
"users", "users",
@@ -69,14 +70,14 @@ class MigrationTests(PersistenceTestCase):
self.assertIn(table, tables) self.assertIn(table, tables)
def test_rollback_removes_schema_and_forward_rebuilds_it(self) -> None: def test_rollback_removes_schema_and_forward_rebuilds_it(self) -> None:
self.assertEqual([3, 2, 1], rollback(self.connection, 0)) self.assertEqual([4, 3, 2, 1], rollback(self.connection, 0))
self.assertEqual([], applied_versions(self.connection)) self.assertEqual([], applied_versions(self.connection))
remaining = self.connection.execute( remaining = self.connection.execute(
"SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'source_rows'" "SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'source_rows'"
).fetchone() ).fetchone()
self.assertIsNone(remaining) self.assertIsNone(remaining)
self.assertEqual([1, 2, 3], migrate(self.connection)) self.assertEqual([1, 2, 3, 4], migrate(self.connection))
self.assertEqual([1, 2, 3], applied_versions(self.connection)) self.assertEqual([1, 2, 3, 4], applied_versions(self.connection))
class ImportPersistenceTests(PersistenceTestCase): class ImportPersistenceTests(PersistenceTestCase):
+23 -4
View File
@@ -209,12 +209,30 @@ class ServerAuthMatrixTests(unittest.TestCase):
) )
assert status == 200, data assert status == 200, data
cls.b_batch_id = as_json(data)["batch_id"] cls.b_batch_id = as_json(data)["batch_id"]
cls.confirm_all_sheets(cls.admin, cls.b_batch_id)
status, _, data = cls.cashier_a.post_multipart( status, _, data = cls.cashier_a.post_multipart(
"/api/parse", {}, CITIC_SAMPLE.name, CITIC_SAMPLE.read_bytes() "/api/parse", {}, CITIC_SAMPLE.name, CITIC_SAMPLE.read_bytes()
) )
assert status == 200, data assert status == 200, data
cls.a_batch_id = as_json(data)["batch_id"] cls.a_batch_id = as_json(data)["batch_id"]
cls.confirm_all_sheets(cls.cashier_a, cls.a_batch_id)
@classmethod
def confirm_all_sheets(cls, client, batch_id: int) -> None:
status, _, data = client.get(f"/api/batches/{batch_id}/sheets")
assert status == 200, data
names = [
sheet["sheet_name"]
for sheet in as_json(data)["sheets"]
if sheet["outcome"] == "parsed"
]
if not names:
return
status, _, data = client.post_json(
f"/api/batches/{batch_id}/confirm", {"sheets": names}
)
assert status == 200, data
@classmethod @classmethod
def tearDownClass(cls) -> None: def tearDownClass(cls) -> None:
@@ -459,10 +477,11 @@ class ServerAuthMatrixTests(unittest.TestCase):
payload = as_json(data) payload = as_json(data)
self.assertEqual("duplicate", payload["status"]) self.assertEqual("duplicate", payload["status"])
self.assertEqual(self.a_batch_id, payload["batch_id"]) self.assertEqual(self.a_batch_id, payload["batch_id"])
self.assertEqual("中信银行", payload.get("bank")) self.assertTrue(payload.get("sheets"))
self.assertTrue(payload.get("transactions", 0) > 0) self.assertEqual("中信银行", payload["sheets"][0].get("bank"))
self.assertIn("period_start", payload) self.assertTrue(payload["sheets"][0].get("transactions", 0) > 0)
self.assertIn("warnings", payload) self.assertIn("period_start", payload["sheets"][0])
self.assertIn("warnings", payload["sheets"][0])
# ------------------------------------------------------------------ # ------------------------------------------------------------------
# Disable / reset flows # Disable / reset flows
+260 -24
View File
@@ -1151,6 +1151,8 @@ function resetUpload() {
$("#uploadForm")?.reset(); $("#uploadForm")?.reset();
if ($("#filePreview")) $("#filePreview").hidden = true; if ($("#filePreview")) $("#filePreview").hidden = true;
if ($("#parseResult")) $("#parseResult").hidden = true; if ($("#parseResult")) $("#parseResult").hidden = true;
if ($("#sheetReview")) $("#sheetReview").hidden = true;
if ($("#sheetList")) $("#sheetList").replaceChildren();
if ($("#dropzone")) $("#dropzone").hidden = false; if ($("#dropzone")) $("#dropzone").hidden = false;
if ($("#parseButton")) { if ($("#parseButton")) {
$("#parseButton").disabled = true; $("#parseButton").disabled = true;
@@ -1203,41 +1205,265 @@ async function parseFile() {
result = { status: "error", message: "解析服务暂时不可用,请稍后重试。" }; result = { status: "error", message: "解析服务暂时不可用,请稍后重试。" };
} }
state.parseResult = result; state.parseResult = result;
const duplicated = result.status === "duplicate"; renderParseResult(result, parsed);
const opaqueDuplicate = duplicated && !result.bank;
const panel = $("#parseResult");
panel.classList.toggle("is-exception", !parsed);
$("use", panel).setAttribute("href", parsed ? "icons.svg#circle-check" : "icons.svg#circle-alert");
$("strong", panel).textContent = duplicated ? "文件已导入过" : parsed ? "文件解析完成" : "未识别到银行模板";
$("#parseSummary").textContent = parsed
? opaqueDuplicate
? "相同内容的文件已由其他公司导入,仅记录重复状态,不重复入账。"
: `${result.bank} · 表头第 ${result.header_row} 行 · ${result.transactions} 条明细 · ${result.warnings.length ? `${result.warnings.length} 项提示` : "校验通过"}${duplicated ? " · 重复上传,复用已有批次" : ""}`
: `${result.message} 系统不会猜测模板或自动入账。`;
panel.hidden = false;
$("#parseButton span").textContent = parsed ? "确认导入" : "提交异常";
$("#parseButton").disabled = false;
$("#parseButton").dataset.stage = "confirm";
} }
function confirmImport() { function renderParseResult(result, parsed) {
const row = document.createElement("tr"); const panel = $("#parseResult");
row.innerHTML = `<td><strong>IMP-260806-019</strong></td><td>${$("#accountSelect").value}</td><td>${state.parseResult?.period_start || "待确认"}${state.parseResult?.period_end || "待确认"}</td><td>${state.parseResult?.transactions ?? "—"}</td><td><span class="status warning">待计算</span></td><td><span class="status neutral">已解析</span></td><td>刚刚</td>`; if (!panel) return;
$("#importRows")?.prepend(row); const sheets = Array.isArray(result.sheets) ? result.sheets : [];
const duplicated = result.status === "duplicate";
const opaque = duplicated && !sheets.length;
panel.classList.toggle("is-exception", !parsed);
$("use", panel).setAttribute("href", parsed ? "icons.svg#circle-check" : "icons.svg#circle-alert");
$("#parseTitle").textContent = duplicated ? "文件已导入过" : parsed ? "文件解析完成" : "未识别到银行模板";
const pendingCount = sheets.filter((s) => s.outcome === "parsed" && s.review_status === "pending").length;
const exceptionCount = sheets.filter((s) => s.outcome === "exception").length;
const ignoredCount = sheets.filter((s) => s.outcome === "ignored" || s.review_status === "ignored").length;
let summary;
if (opaque) {
summary = "相同内容的文件已由其他公司导入,仅记录重复状态,不重复入账。";
} else if (sheets.length) {
summary = `${sheets.length} 个工作表:${pendingCount} 个待确认${exceptionCount ? `${exceptionCount} 个异常` : ""}${ignoredCount ? `${ignoredCount} 个忽略` : ""}。解析成功不等于业务确认。`;
} else {
summary = `${result.message || ""} 系统不会猜测模板或自动入账。`;
}
$("#parseSummary").textContent = summary;
panel.hidden = false;
renderSheetList(sheets, result.batch_id);
const button = $("#parseButton");
button.dataset.stage = "confirm";
if (!parsed) {
button.querySelector("span").textContent = "关闭";
} else if (opaque) {
button.querySelector("span").textContent = "完成";
} else if (sheets.length) {
button.querySelector("span").textContent = pendingCount ? `确认全部(${pendingCount} 个待确认)` : "完成";
} else {
button.querySelector("span").textContent = "完成";
}
}
function sheetStatusMeta(sheet) {
if (sheet.review_status === "confirmed") return { className: "success", label: "已确认" };
if (sheet.review_status === "ignored") return { className: "neutral", label: "已忽略" };
if (sheet.outcome === "exception") return { className: "danger", label: "异常待处理" };
if (sheet.outcome === "ignored") return { className: "neutral", label: "空表忽略" };
return { className: "warning", label: "待确认" };
}
function renderSheetList(sheets, batchId) {
const wrap = $("#sheetReview");
const list = $("#sheetList");
if (!wrap || !list || !sheets.length) {
if (wrap) wrap.hidden = true;
return;
}
wrap.hidden = false;
list.replaceChildren(...sheets.map((sheet) => buildSheetItem(sheet, batchId)));
}
function buildSheetItem(sheet, batchId) {
const item = document.createElement("article");
item.className = "sheet-item";
if (sheet.review_status === "pending") item.classList.add("is-pending");
const meta = sheetStatusMeta(sheet);
const head = document.createElement("div");
head.className = "sheet-item-head";
const name = document.createElement("strong");
name.textContent = sheet.sheet_name;
const badge = document.createElement("em");
badge.className = `status ${meta.className}`;
badge.textContent = meta.label;
head.append(name, badge);
const details = document.createElement("p");
details.className = "sheet-item-meta";
if (sheet.outcome === "parsed" && sheet.bank) {
const period = sheet.period_start ? ` · ${sheet.period_start}${sheet.period_end}` : "";
details.textContent = `${sheet.bank} · ${sheet.transactions} 条明细${period}`;
} else if (sheet.message) {
details.textContent = sheet.message;
} else {
details.textContent = "空工作表。";
}
if (sheet.review_reason) {
details.textContent += ` · 原因:${sheet.review_reason}`;
}
const body = document.createElement("div");
body.append(head, details);
const actions = document.createElement("div");
actions.className = "sheet-item-actions";
if (sheet.review_status === "pending") {
if (sheet.outcome === "parsed") {
const confirmButton = document.createElement("button");
confirmButton.type = "button";
confirmButton.className = "text-button";
confirmButton.textContent = "确认";
confirmButton.addEventListener("click", () => sheetReviewAction(batchId, sheet.sheet_name, "confirm"));
actions.append(confirmButton);
}
const ignoreButton = document.createElement("button");
ignoreButton.type = "button";
ignoreButton.className = "text-button";
ignoreButton.textContent = "忽略";
ignoreButton.addEventListener("click", () => sheetReviewAction(batchId, sheet.sheet_name, "ignore"));
actions.append(ignoreButton);
}
item.append(body, actions);
return item;
}
async function sheetReviewAction(batchId, sheetName, decision) {
const payload = { sheets: [sheetName] };
if (decision === "ignore") {
const reason = (window.prompt("请填写忽略原因(必填):", "") || "").trim();
if (!reason) {
showToast("忽略未提交", "必须填写忽略原因");
return;
}
payload.reason = reason;
}
const response = await fetch(`/api/batches/${batchId}/${decision}`, {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify(payload),
}).catch(() => null);
if (response?.status === 401 || response?.status === 403) {
window.location.href = "index.html";
return;
}
const result = await response?.json().catch(() => ({}));
if (!response || !response.ok) {
showToast("操作失败", result?.message || "请稍后重试");
return;
}
showToast(decision === "confirm" ? "工作表已确认" : "工作表已忽略", `${sheetName}`);
await refreshAfterSheetAction(result, batchId);
}
async function refreshAfterSheetAction(result, batchId) {
if (Array.isArray(result.sheets)) renderSheetList(result.sheets, batchId);
await loadImportBatches();
const button = $("#parseButton");
if (!button) return;
const pending = (result.sheets || []).filter((s) => s.outcome === "parsed" && s.review_status === "pending").length;
if (pending === 0 && (result.sheets || []).length) {
button.querySelector("span").textContent = "完成";
button.dataset.stage = "done";
} else {
button.querySelector("span").textContent = `确认全部(${pending} 个待确认)`;
}
}
async function confirmImport() {
const result = state.parseResult;
const batchId = result?.batch_id;
const sheets = Array.isArray(result?.sheets) ? result.sheets : [];
const pending = sheets
.filter((s) => s.outcome === "parsed" && s.review_status === "pending")
.map((s) => s.sheet_name);
if (!pending.length) {
$("#uploadDialog").close();
await loadImportBatches();
showView("upload");
return;
}
const button = $("#parseButton");
button.disabled = true;
button.querySelector("span").textContent = "正在确认...";
const response = await fetch(`/api/batches/${batchId}/confirm`, {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({ sheets: pending }),
}).catch(() => null);
if (response?.status === 401 || response?.status === 403) {
window.location.href = "index.html";
return;
}
const outcome = await response?.json().catch(() => ({}));
if (!response || !response.ok) {
button.disabled = false;
button.querySelector("span").textContent = "确认失败,点击重试";
showToast("确认失败", outcome?.message || "请稍后重试");
return;
}
button.disabled = false;
if (Array.isArray(outcome.sheets)) renderSheetList(outcome.sheets, batchId);
await loadImportBatches();
$("#uploadDialog").close(); $("#uploadDialog").close();
showView("upload"); showView("upload");
showToast("流水已进入归集队列", "原始文件和解析结果已保留"); showToast("流水已确认", `${outcome.updated.length} 个工作表已确认;未确认的工作表不参与计算`);
} }
function submitImportException() { function submitImportException() {
$("#uploadDialog").close(); $("#uploadDialog").close();
showView("upload"); showView("upload");
showToast("解析异常已提交", `${state.selectedFile?.name || "该文件"} 未入账,等待总账或模板维护人员处理`); showToast("解析异常未入账", `${state.selectedFile?.name || "该文件"} 不会进入匹配与计算,请核对模板后重新导出`);
}
function renderBatchRow(batch) {
const row = document.createElement("tr");
const idCell = document.createElement("td");
const id = document.createElement("strong");
id.textContent = `IMP-${String(batch.id).padStart(6, "0")}`;
const file = document.createElement("small");
file.textContent = batch.original_filename || "";
idCell.append(id, file);
const bank = document.createElement("td");
bank.textContent = batch.bank_name || "—";
const period = document.createElement("td");
period.textContent = batch.period_start && batch.period_end
? `${batch.period_start}${batch.period_end}`
: "—";
const count = document.createElement("td");
count.className = "number";
count.textContent = `${batch.confirmed_transactions ?? 0}`;
const coverage = document.createElement("td");
const coverageStatus = batch.status === "exception"
? { className: "danger", label: "未导入" }
: batch.pending_sheets > 0
? { className: "warning", label: `${batch.pending_sheets} 个待确认` }
: batch.confirmed_sheets > 0
? { className: "success", label: "已确认" }
: { className: "neutral", label: "待处理" };
coverage.innerHTML = `<span class="status ${coverageStatus.className}">${coverageStatus.label}</span>`;
const parseState = document.createElement("td");
const statusParts = [];
if (batch.exception_sheets > 0) statusParts.push(`${batch.exception_sheets} 个异常`);
if (batch.ignored_sheets > 0) statusParts.push(`${batch.ignored_sheets} 个忽略`);
parseState.textContent = statusParts.length ? statusParts.join("、") : "解析成功";
const time = document.createElement("td");
time.textContent = String(batch.created_at || "").slice(0, 16).replace("T", " ");
row.append(idCell, bank, period, count, coverage, parseState, time);
return row;
}
async function loadImportBatches() {
const tbody = $("#importRows");
if (!tbody) return;
const response = await fetch("/api/batches").catch(() => null);
if (response?.status === 401 || response?.status === 403) {
window.location.href = "index.html";
return;
}
const result = await response?.json().catch(() => ({}));
const batches = Array.isArray(result?.batches) ? result.batches : [];
if (batches.length) tbody.replaceChildren(...batches.map(renderBatchRow));
} }
function initCompany() { function initCompany() {
renderCompanyManualRecords(); renderCompanyManualRecords();
loadCompanyAccounts(); loadCompanyAccounts();
loadImportBatches();
const uploadDialog = $("#uploadDialog"); const uploadDialog = $("#uploadDialog");
$$('[data-open-upload]').forEach((button) => button.addEventListener("click", () => uploadDialog.showModal())); $$('[data-open-upload]').forEach((button) => button.addEventListener("click", () => uploadDialog.showModal()));
$$('[data-close-upload]').forEach((button) => button.addEventListener("click", () => uploadDialog.close())); $$('[data-close-upload]').forEach((button) => button.addEventListener("click", () => uploadDialog.close()));
@@ -1260,9 +1486,19 @@ function initCompany() {
} }
$("#uploadForm")?.addEventListener("submit", async (event) => { $("#uploadForm")?.addEventListener("submit", async (event) => {
event.preventDefault(); event.preventDefault();
if ($("#parseButton").dataset.stage === "confirm") { const stage = $("#parseButton").dataset.stage;
if (["parsed", "duplicate"].includes(state.parseResult?.status)) confirmImport(); if (stage === "confirm" || stage === "done") {
else submitImportException(); const result = state.parseResult;
const parsed = result && ["parsed", "duplicate"].includes(result.status);
if (parsed && stage === "confirm") {
await confirmImport();
} else if (parsed) {
$("#uploadDialog").close();
showView("upload");
await loadImportBatches();
} else {
submitImportException();
}
return; return;
} }
$("#parseButton").disabled = true; $("#parseButton").disabled = true;
+2 -2
View File
@@ -129,7 +129,7 @@
<section class="app-view" data-page="upload"> <section class="app-view" data-page="upload">
<header class="page-heading"><div><h1>流水导入</h1><p>上传 A公司银行账户流水,系统按表头识别银行模板</p></div><button class="button primary" data-open-upload><svg><use href="icons.svg#upload"/></svg>上传流水</button></header> <header class="page-heading"><div><h1>流水导入</h1><p>上传 A公司银行账户流水,系统按表头识别银行模板</p></div><button class="button primary" data-open-upload><svg><use href="icons.svg#upload"/></svg>上传流水</button></header>
<section class="panel"><div class="table-summary"><span>最近导入批次</span><span>原始文件与解析结果将永久保留</span></div><div class="table-scroll"><table class="data-table"><thead><tr><th>批次</th><th>银行账户</th><th>流水期间</th><th>明细数</th><th>覆盖状态</th><th>解析状态</th><th>上传时间</th></tr></thead><tbody id="importRows"><tr><td><strong>IMP-260806-018</strong></td><td>中信银行 · 5316</td><td>07.01—07.31</td><td>128</td><td><span class="status success">连续</span></td><td><span class="status success">已确认</span></td><td>今天 09:42</td></tr><tr><td><strong>IMP-260731-012</strong></td><td>工商银行 · 9481</td><td>06.01—06.30</td><td>96</td><td><span class="status danger">后续断档</span></td><td><span class="status success">已确认</span></td><td>07.31 16:18</td></tr></tbody></table></div></section> <section class="panel"><div class="table-summary"><span>最近导入批次</span><span>仅已确认工作表参与匹配与计算</span></div><div class="table-scroll"><table class="data-table"><thead><tr><th>批次</th><th>银行账户</th><th>流水期间</th><th>明细数</th><th>覆盖状态</th><th>解析状态</th><th>上传时间</th></tr></thead><tbody id="importRows"></tbody></table></div></section>
</section> </section>
<section class="app-view" data-page="manual"> <section class="app-view" data-page="manual">
@@ -184,7 +184,7 @@
<dialog id="uploadDialog" class="dialog upload-dialog"> <dialog id="uploadDialog" class="dialog upload-dialog">
<form id="uploadForm"> <form id="uploadForm">
<header><div><h2>上传银行流水</h2><p>A公司 · 系统将识别表头与银行模板</p></div><button type="button" class="icon-button" data-close-upload aria-label="关闭" title="关闭"><svg><use href="icons.svg#x"/></svg></button></header> <header><div><h2>上传银行流水</h2><p>A公司 · 系统将识别表头与银行模板</p></div><button type="button" class="icon-button" data-close-upload aria-label="关闭" title="关闭"><svg><use href="icons.svg#x"/></svg></button></header>
<div class="dialog-body"><label class="field"><span>银行账户</span><select id="accountSelect" required><option value="">请选择账户</option></select></label><label class="dropzone" id="dropzone"><input id="fileInput" type="file" accept=".xls,.xlsx" /><svg><use href="icons.svg#upload"/></svg><strong>选择或拖入银行流水文件</strong><span>支持 .xls 与 .xlsx,最大 20 MB</span></label><div class="file-preview" id="filePreview" hidden><span class="file-type"><svg><use href="icons.svg#file-spreadsheet"/></svg></span><span><strong id="fileName"></strong><small id="fileMeta"></small></span><button type="button" class="icon-button" id="removeFile" aria-label="移除文件" title="移除文件"><svg><use href="icons.svg#x"/></svg></button></div><div class="parse-result" id="parseResult" hidden><span class="notification-icon"><svg><use href="icons.svg#circle-check"/></svg></span><span><strong>文件解析完成</strong><p id="parseSummary"></p></span></div></div> <div class="dialog-body"><label class="field"><span>银行账户</span><select id="accountSelect" required><option value="">请选择账户</option></select></label><label class="dropzone" id="dropzone"><input id="fileInput" type="file" accept=".xls,.xlsx" /><svg><use href="icons.svg#upload"/></svg><strong>选择或拖入银行流水文件</strong><span>支持 .xls 与 .xlsx,最大 20 MB</span></label><div class="file-preview" id="filePreview" hidden><span class="file-type"><svg><use href="icons.svg#file-spreadsheet"/></svg></span><span><strong id="fileName"></strong><small id="fileMeta"></small></span><button type="button" class="icon-button" id="removeFile" aria-label="移除文件" title="移除文件"><svg><use href="icons.svg#x"/></svg></button></div><div class="parse-result" id="parseResult" hidden><span class="notification-icon"><svg><use href="icons.svg#circle-check"/></svg></span><span><strong id="parseTitle">文件解析完成</strong><p id="parseSummary"></p></span></div><div class="sheet-review" id="sheetReview" hidden><h3>工作表处理</h3><p class="sheet-review-hint">解析成功不等于业务确认:只有勾选并确认的工作表才会进入后续匹配与计算。</p><div class="sheet-list" id="sheetList" aria-live="polite"></div></div></div>
<footer><button type="button" class="button secondary" data-close-upload>取消</button><button class="button primary" id="parseButton" type="submit" disabled><span>开始解析</span></button></footer> <footer><button type="button" class="button secondary" data-close-upload>取消</button><button class="button primary" id="parseButton" type="submit" disabled><span>开始解析</span></button></footer>
</form> </form>
</dialog> </dialog>
+13
View File
@@ -412,6 +412,19 @@ main { width: min(1560px, 100%); margin: 0 auto; padding: 12px 32px 58px; }
.parse-result p { color: #9bd7b6; } .parse-result p { color: #9bd7b6; }
.parse-result.is-exception { border-color: rgba(255, 188, 82, 0.25); background: var(--color-warning-wash); } .parse-result.is-exception { border-color: rgba(255, 188, 82, 0.25); background: var(--color-warning-wash); }
.parse-result.is-exception p { color: #e7bd78; } .parse-result.is-exception p { color: #e7bd78; }
.sheet-review { margin-top: 12px; padding-top: 12px; border-top: 1px solid var(--color-line); }
.sheet-review h3 { font-size: 14px; }
.sheet-review-hint { margin-top: 4px; color: var(--color-ink-muted); font-size: 12px; line-height: 1.5; }
.sheet-list { display: grid; gap: 8px; margin-top: 10px; max-height: 220px; overflow-y: auto; }
.sheet-item { display: grid; grid-template-columns: minmax(0, 1fr) auto; gap: 8px; padding: 10px; border: 1px solid var(--color-line); border-radius: var(--radius-md); background: var(--color-surface-muted); }
.sheet-item.is-pending { border-color: rgba(255, 188, 82, 0.45); }
.sheet-item-head { display: flex; align-items: center; gap: 8px; flex-wrap: wrap; }
.sheet-item-head strong { font-size: 13px; }
.sheet-item-head small { color: var(--color-ink-muted); }
.sheet-item-meta { margin-top: 5px; color: var(--color-ink-muted); font-size: 12px; line-height: 1.5; }
.sheet-item-actions { display: flex; align-items: center; gap: 6px; }
.sheet-item-actions .text-button { font-size: 12px; padding: 4px 8px; }
.sheet-item-reason { margin-top: 6px; padding: 6px 8px; border: 1px solid var(--color-line); border-radius: var(--radius-sm); color: var(--color-ink-muted); font-size: 12px; }
.toast-region { position: fixed; right: 20px; bottom: 20px; z-index: 200; display: grid; gap: 8px; } .toast-region { position: fixed; right: 20px; bottom: 20px; z-index: 200; display: grid; gap: 8px; }
.toast { min-width: 280px; max-width: 390px; padding: 13px 15px; border: 1px solid var(--color-line-strong); border-radius: var(--radius-md); background: rgba(20, 20, 20, 0.96); color: var(--color-ink); box-shadow: var(--shadow-panel); backdrop-filter: blur(20px); } .toast { min-width: 280px; max-width: 390px; padding: 13px 15px; border: 1px solid var(--color-line-strong); border-radius: var(--radius-md); background: rgba(20, 20, 20, 0.96); color: var(--color-ink); box-shadow: var(--shadow-panel); backdrop-filter: blur(20px); }
.toast strong, .toast small { display: block; } .toast strong, .toast small { display: block; }