leefer/caiwuzongzhang
1
0
Fork 0
Code Issues Pull Requests 4 Actions Packages Projects Releases Wiki Activity

B-42: 导入功能加固——逐工作表确认、流式上传与诊断证据

Browse Source
This commit is contained in:
腾讯WorkBuddy
2026-08-17 01:00:19 +08:00
parent 7f1a93f6a6
commit 486842963e
15 changed files with 2019 additions and 183 deletions
Download Patch File Download Diff File Expand all files Collapse all files
server.py
+217 -99
View File
@@ -10,9 +10,8 @@ from http.cookies import SimpleCookie
from http.server import SimpleHTTPRequestHandler, ThreadingHTTPServer
from urllib.parse import parse_qs, urlparse
from bank_importer import auth, master_data
from bank_importer import auth, importing, master_data, multipart
from bank_importer.db import connect, migrate, utc_now
from bank_importer.importing import import_statement
ROOT = Path(__file__).resolve().parent
@@ -49,6 +48,10 @@ class AppHandler(SimpleHTTPRequestHandler):
if rows_match:
self._handle_batch_rows(int(rows_match.group(1)))
return
sheets_match = re.fullmatch(r"/api/batches/(\d+)/sheets", path)
if sheets_match:
self._handle_batch_sheets(int(sheets_match.group(1)))
return
if path == "/api/export.csv":
self._handle_export_csv(query)
return
@@ -119,6 +122,14 @@ class AppHandler(SimpleHTTPRequestHandler):
if alias_create:
self._handle_admin_add_alias(int(alias_create.group(1)))
return
confirm_match = re.fullmatch(r"/api/batches/(\d+)/confirm", path)
if confirm_match:
self._handle_batch_review(int(confirm_match.group(1)), "confirm")
return
ignore_match = re.fullmatch(r"/api/batches/(\d+)/ignore", path)
if ignore_match:
self._handle_batch_review(int(ignore_match.group(1)), "ignore")
return
self._send_json(404, {"status": "error", "message": "接口不存在。"})
# ------------------------------------------------------------------
@@ -308,15 +319,21 @@ class AppHandler(SimpleHTTPRequestHandler):
def _handle_parse(self) -> None:
connection = connect(DB_PATH)
upload = None
try:
user = self._require_user(connection)
if user is None:
return
try:
filename, content, fields = self._read_upload()
fields, upload = self._receive_upload()
filename = os.path.basename(upload.filename)
suffix = Path(filename).suffix.lower()
if suffix not in {".xls", ".xlsx"}:
raise ValueError("仅支持 .xls 或 .xlsx 银行流水文件。")
if not multipart.valid_file_signature(upload.path, suffix):
raise ValueError(
"文件内容与扩展名不符，可能已损坏或不是有效的 Excel 文件。"
)
if user["role"] == "company":
# Tenant binding comes from the session only; any
@@ -325,12 +342,15 @@ class AppHandler(SimpleHTTPRequestHandler):
if not self._validate_upload_account(connection, user, fields):
return
else:
company_id = self._parse_company_field(connection, fields.get("company_id"))
company_id = self._parse_company_field(
connection, fields.get("company_id")
)
if company_id is None:
return
result = import_statement(
connection, STORAGE_DIR, filename, content, company_id=company_id
result = importing.import_statement_path(
connection, STORAGE_DIR, filename, upload.path,
company_id=company_id,
)
auth.audit(
connection,
@@ -356,8 +376,40 @@ class AppHandler(SimpleHTTPRequestHandler):
else:
self._send_json(200, payload)
finally:
if upload is not None:
upload.path.unlink(missing_ok=True)
connection.close()
def _receive_upload(self) -> tuple[dict[str, str], multipart.UploadedFile]:
"""Stream the multipart body into a validated temp file.
The whole request is never buffered in memory; file bytes are written
to a temp file under the storage directory as they arrive and are
never trimmed. Only the first file part is accepted.
"""
raw_length = self.headers.get("Content-Length")
if raw_length is None:
raise ValueError("上传请求缺少 Content-Length。")
try:
content_length = int(raw_length)
except ValueError:
raise ValueError("上传请求的 Content-Length 无效。")
if content_length <= 0:
raise ValueError("文件为空或超过 20 MB 限制。")
if content_length > MAX_UPLOAD_BYTES + multipart.MAX_FIELD_BYTES:
raise ValueError("文件超过 20 MB 限制。")
try:
fields, upload = multipart.parse_upload(
self.rfile,
content_length,
self.headers.get("Content-Type", ""),
STORAGE_DIR / ".uploads",
max_file_bytes=MAX_UPLOAD_BYTES,
)
except multipart.MultipartError as exc:
raise ValueError(str(exc)) from exc
return fields, upload
def _validate_upload_account(self, connection, user, fields) -> bool:
"""Validate the optional bank_account_id on a company upload.
@@ -435,15 +487,40 @@ class AppHandler(SimpleHTTPRequestHandler):
f"""
SELECT b.id, b.status, b.created_at, b.company_id,
f.original_filename, c.name AS company_name,
COALESCE(SUM(s.transaction_count), 0) AS transactions,
MIN(s.period_start) AS period_start,
MAX(s.period_end) AS period_end
COALESCE((
SELECT SUM(s.transaction_count) FROM sheet_batches s
WHERE s.import_batch_id = b.id
), 0) AS transactions,
(SELECT s.bank_name FROM sheet_batches s
WHERE s.import_batch_id = b.id
ORDER BY s.id LIMIT 1) AS bank_name,
(SELECT MIN(s.period_start) FROM sheet_batches s
WHERE s.import_batch_id = b.id) AS period_start,
(SELECT MAX(s.period_end) FROM sheet_batches s
WHERE s.import_batch_id = b.id) AS period_end,
(SELECT COUNT(*) FROM source_rows r
JOIN sheet_batches s ON s.id = r.sheet_batch_id
JOIN sheet_reviews rv ON rv.sheet_batch_id = s.id
WHERE s.import_batch_id = b.id AND rv.review_status = 'confirmed')
AS confirmed_transactions,
(SELECT COUNT(*) FROM sheet_reviews r
WHERE r.import_batch_id = b.id AND r.review_status = 'confirmed')
AS confirmed_sheets,
(SELECT COUNT(*) FROM sheet_reviews r
WHERE r.import_batch_id = b.id
AND r.outcome = 'parsed' AND r.review_status = 'pending')
AS pending_sheets,
(SELECT COUNT(*) FROM sheet_reviews r
WHERE r.import_batch_id = b.id AND r.outcome = 'exception')
AS exception_sheets,
(SELECT COUNT(*) FROM sheet_reviews r
WHERE r.import_batch_id = b.id
AND (r.outcome = 'ignored' OR r.review_status = 'ignored'))
AS ignored_sheets
FROM import_batches b
JOIN source_files f ON f.id = b.source_file_id
LEFT JOIN companies c ON c.id = b.company_id
LEFT JOIN sheet_batches s ON s.import_batch_id = b.id
{where}
GROUP BY b.id
ORDER BY b.id DESC
LIMIT 500
""",
@@ -453,6 +530,89 @@ class AppHandler(SimpleHTTPRequestHandler):
finally:
connection.close()
def _handle_batch_sheets(self, batch_id: int) -> None:
connection = connect(DB_PATH)
try:
user = self._require_user(connection)
if user is None:
return
company_id = user["company_id"] if user["role"] == "company" else None
batch = importing.scoped_batch(connection, batch_id, company_id)
if batch is None:
self._send_json(404, {"status": "error", "message": "批次不存在。"})
return
file_row = connection.execute(
"""
SELECT f.original_filename FROM source_files f
JOIN import_batches b ON b.source_file_id = f.id
WHERE b.id = ?
""",
(batch_id,),
).fetchone()
rows = importing.sheet_review_rows(connection, batch_id)
self._send_json(
200,
{
"status": "ok",
"batch_id": batch_id,
"company_id": batch["company_id"],
"original_filename": file_row["original_filename"]
if file_row is not None
else None,
"sheets": [self._sheet_payload(row) for row in rows],
},
)
finally:
connection.close()
def _handle_batch_review(self, batch_id: int, decision: str) -> None:
connection = connect(DB_PATH)
try:
user = self._require_user(connection)
if user is None:
return
data = self._read_json_body()
if data is None:
return
sheets = data.get("sheets")
if not isinstance(sheets, list):
self._send_json(
400, {"status": "error", "message": "sheets 必须是工作表名称数组。"}
)
return
company_id = user["company_id"] if user["role"] == "company" else None
try:
outcome = importing.review_sheets(
connection,
batch_id,
sheets,
decision,
user,
company_id,
reason=str(data.get("reason") or "") if decision == "ignore" else None,
)
except LookupError as exc:
self._send_json(404, {"status": "error", "message": str(exc)})
return
except importing.SheetReviewError as exc:
self._send_json(409, {"status": "error", "message": str(exc)})
return
except ValueError as exc:
self._send_json(400, {"status": "error", "message": str(exc)})
return
rows = importing.sheet_review_rows(connection, batch_id)
self._send_json(
200,
{
"status": "ok",
"updated": outcome["updated"],
"already": outcome["already"],
"sheets": [self._sheet_payload(row) for row in rows],
},
)
finally:
connection.close()
def _handle_batch_rows(self, batch_id: int) -> None:
connection = connect(DB_PATH)
try:
@@ -474,9 +634,11 @@ class AppHandler(SimpleHTTPRequestHandler):
SELECT r.id, r.sheet_batch_id, r.source_row, r.transaction_at,
r.income, r.expense, r.balance, r.own_account, r.own_name,
r.counterparty_account, r.counterparty_name, r.counterparty_bank,
r.summary, r.purpose, r.reference, r.currency
r.summary, r.purpose, r.reference, r.currency,
s.sheet_name, rv.review_status
FROM source_rows r
JOIN sheet_batches s ON s.id = r.sheet_batch_id
JOIN sheet_reviews rv ON rv.sheet_batch_id = s.id
WHERE s.import_batch_id = ?
ORDER BY s.id, r.source_row
""",
@@ -529,6 +691,10 @@ class AppHandler(SimpleHTTPRequestHandler):
FROM source_rows r
JOIN sheet_batches s ON s.id = r.sheet_batch_id
JOIN import_batches b ON b.id = s.import_batch_id
-- Only worksheets confirmed by the cashier may leave the
-- ledger: parse success is not business confirmation.
JOIN sheet_reviews rv
ON rv.sheet_batch_id = s.id AND rv.review_status = 'confirmed'
{where}
ORDER BY b.id, s.id, r.source_row
""",
@@ -1117,8 +1283,37 @@ class AppHandler(SimpleHTTPRequestHandler):
return None
return data
@staticmethod
def _sheet_payload(row) -> dict[str, object]:
"""Serialize one persisted per-sheet review record."""
return {
"sheet_name": row["sheet_name"],
"outcome": row["outcome"],
"review_status": row["review_status"],
"message": row["message"],
"scanned_rows": row["scanned_rows"],
"candidate_headers": json.loads(row["candidate_headers"])
if row["candidate_headers"]
else [],
"review_reason": row["review_reason"],
"bank": row["bank_name"],
"template": row["template_id"],
"header_row": row["header_row"],
"period_start": row["period_start"],
"period_end": row["period_end"],
"transactions": row["transaction_count"] or 0,
"warnings": json.loads(row["warnings"]) if row["warnings"] else [],
}
@staticmethod
def _import_payload(connection, result) -> dict[str, object]:
"""Build the parse response from persisted state, not from memory.
All worksheet results are returned (no more ``batches[0]``). A
cross-company duplicate is opaque: only the generic status, the
uploader's own batch id and the sha256 are returned, never the other
company's batch id, summary or diagnostics.
"""
payload: dict[str, object] = {
"status": result.status,
"batch_id": result.batch_id,
@@ -1126,96 +1321,19 @@ class AppHandler(SimpleHTTPRequestHandler):
}
if result.message:
payload["message"] = result.message
if result.status == "duplicate" and not result.duplicate_same_company:
return payload
if result.status == "parsed":
batch = result.batches[0]
payload.update(
{
"bank": batch.bank_name,
"template": batch.template_id,
"header_row": batch.header_row,
"period_start": batch.period_start.isoformat()
if batch.period_start
else None,
"period_end": batch.period_end.isoformat()
if batch.period_end
else None,
"transactions": len(batch.transactions),
"warnings": list(batch.warnings),
}
)
elif result.status == "duplicate":
# Same-company duplicates keep their idempotent summary so the
# cashier sees the reused batch's bank/template/period/count.
# Cross-company duplicates are opaque: only the generic duplicate
# status and the uploader's own new batch id are returned, never
# the other company's batch id, summary or diagnostics.
if result.duplicate_same_company:
sheet = connection.execute(
"""
SELECT bank_name, template_id, header_row, period_start,
period_end, transaction_count, warnings
FROM sheet_batches
WHERE import_batch_id = ?
ORDER BY id LIMIT 1
""",
(result.batch_id,),
).fetchone()
if sheet is not None:
payload.update(
{
"bank": sheet["bank_name"],
"template": sheet["template_id"],
"header_row": sheet["header_row"],
"period_start": sheet["period_start"],
"period_end": sheet["period_end"],
"transactions": sheet["transaction_count"],
"warnings": json.loads(sheet["warnings"]),
}
)
file_row = connection.execute(
"SELECT original_filename FROM source_files WHERE id = ?",
(result.source_file_id,),
).fetchone()
if file_row is not None:
payload["original_filename"] = file_row["original_filename"]
sheets = importing.sheet_review_rows(connection, result.batch_id)
payload["sheets"] = [AppHandler._sheet_payload(row) for row in sheets]
return payload
def _read_upload(self) -> tuple[str, bytes, dict[str, str]]:
"""Parse the multipart body into (filename, content, text fields)."""
content_length = int(self.headers.get("Content-Length", "0"))
if content_length <= 0 or content_length > MAX_UPLOAD_BYTES:
raise ValueError("文件为空或超过 20 MB 限制。")
content_type = self.headers.get("Content-Type", "")
boundary_match = re.search(r"boundary=(?:\"([^\"]+)\"|([^;]+))", content_type)
if not boundary_match:
raise ValueError("上传请求缺少文件边界。")
boundary = (boundary_match.group(1) or boundary_match.group(2)).encode()
body = self.rfile.read(content_length)
fields: dict[str, str] = {}
for part in body.split(b"--" + boundary):
header, separator, content = part.partition(b"\r\n\r\n")
if not separator:
continue
name_match = re.search(br'name="([^\"]+)"', header)
if not name_match:
continue
name = name_match.group(1).decode("utf-8", errors="replace")
if name == "file":
filename_match = re.search(br'filename="([^\"]+)"', header)
filename = (
filename_match.group(1).decode("utf-8", errors="replace")
if filename_match
else "statement.xlsx"
)
file_content = content.rstrip(b"\r\n")
elif b"filename=" not in header:
fields[name] = content.rstrip(b"\r\n").decode("utf-8", errors="replace")
try:
return os.path.basename(filename), file_content, fields
except UnboundLocalError:
raise ValueError("上传请求中没有找到文件。") from None
def _read_uploaded_file(self) -> tuple[str, bytes]:
filename, content, _fields = self._read_upload()
return filename, content
def _send_json(
self,
status: int,