B-39: 数据库持久化与不可变导入
- 版本化 SQLite 迁移（向前/回滚）与连接助手。 - 导入管线：上传文件按 SHA-256 内容哈希不可变保存，文件/行/批次 幂等；重复上传返回 duplicate 并复用已有批次，不产生第二份事实。 - 每条规范化源行反查文件、工作表、原始行号与模板版本。 - 解析失败保留 exception/failed 批次与诊断，不产生已确认交易。 - 决策记录见 docs/decisions/002-persistence.md。
This commit is contained in:
腾讯WorkBuddy
@@ -0,0 +1,285 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from decimal import Decimal
|
||||
import hashlib
|
||||
import sqlite3
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
import unittest
|
||||
|
||||
from openpyxl import Workbook
|
||||
|
||||
from bank_importer.db import applied_versions, connect, migrate, rollback
|
||||
from bank_importer.importing import import_statement
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
SAMPLES = ROOT / "流水模板"
|
||||
SAMPLE_FILE = SAMPLES / "中国建设银行账户流水.xls"
|
||||
|
||||
|
||||
class PersistenceTestCase(unittest.TestCase):
|
||||
def setUp(self) -> None:
|
||||
self.temp_dir = tempfile.TemporaryDirectory()
|
||||
self.addCleanup(self.temp_dir.cleanup)
|
||||
root = Path(self.temp_dir.name)
|
||||
self.db_path = root / "app.db"
|
||||
self.storage = root / "files"
|
||||
self.connection = connect(self.db_path)
|
||||
self.addCleanup(self.connection.close)
|
||||
migrate(self.connection)
|
||||
|
||||
def import_sample(self) -> object:
|
||||
return import_statement(
|
||||
self.connection,
|
||||
self.storage,
|
||||
SAMPLE_FILE.name,
|
||||
SAMPLE_FILE.read_bytes(),
|
||||
)
|
||||
|
||||
|
||||
class MigrationTests(PersistenceTestCase):
|
||||
def test_migrate_creates_schema_and_is_idempotent(self) -> None:
|
||||
first = applied_versions(self.connection)
|
||||
self.assertEqual([1, 2, 3], first)
|
||||
self.assertEqual([], migrate(self.connection))
|
||||
self.assertEqual(first, applied_versions(self.connection))
|
||||
tables = {
|
||||
row["name"]
|
||||
for row in self.connection.execute(
|
||||
"SELECT name FROM sqlite_master WHERE type = 'table'"
|
||||
)
|
||||
}
|
||||
for table in (
|
||||
"companies",
|
||||
"bank_accounts",
|
||||
"account_aliases",
|
||||
"master_data_changes",
|
||||
"source_files",
|
||||
"import_batches",
|
||||
"sheet_batches",
|
||||
"source_rows",
|
||||
"import_exceptions",
|
||||
"users",
|
||||
"sessions",
|
||||
"login_attempts",
|
||||
"audit_log",
|
||||
"schema_migrations",
|
||||
):
|
||||
self.assertIn(table, tables)
|
||||
|
||||
def test_rollback_removes_schema_and_forward_rebuilds_it(self) -> None:
|
||||
self.assertEqual([3, 2, 1], rollback(self.connection, 0))
|
||||
self.assertEqual([], applied_versions(self.connection))
|
||||
remaining = self.connection.execute(
|
||||
"SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'source_rows'"
|
||||
).fetchone()
|
||||
self.assertIsNone(remaining)
|
||||
self.assertEqual([1, 2, 3], migrate(self.connection))
|
||||
self.assertEqual([1, 2, 3], applied_versions(self.connection))
|
||||
|
||||
|
||||
class ImportPersistenceTests(PersistenceTestCase):
|
||||
def test_import_persists_batch_sheets_and_rows(self) -> None:
|
||||
result = self.import_sample()
|
||||
self.assertEqual("parsed", result.status)
|
||||
|
||||
batch = self.connection.execute(
|
||||
"SELECT status FROM import_batches WHERE id = ?", (result.batch_id,)
|
||||
).fetchone()
|
||||
self.assertEqual("parsed", batch["status"])
|
||||
|
||||
sheets = self.connection.execute(
|
||||
"SELECT * FROM sheet_batches WHERE import_batch_id = ?", (result.batch_id,)
|
||||
).fetchall()
|
||||
self.assertEqual(1, len(sheets))
|
||||
self.assertEqual("中国建设银行", sheets[0]["bank_name"])
|
||||
self.assertEqual("ccb-account-detail-v1", sheets[0]["template_id"])
|
||||
self.assertEqual(1, sheets[0]["template_version"])
|
||||
|
||||
rows = self.connection.execute(
|
||||
"SELECT * FROM source_rows WHERE sheet_batch_id = ?", (sheets[0]["id"],)
|
||||
).fetchall()
|
||||
self.assertEqual(sheets[0]["transaction_count"], len(rows))
|
||||
|
||||
def test_amounts_keep_decimal_precision(self) -> None:
|
||||
result = self.import_sample()
|
||||
rows = self.connection.execute(
|
||||
"""
|
||||
SELECT income, expense FROM source_rows
|
||||
WHERE sheet_batch_id IN (
|
||||
SELECT id FROM sheet_batches WHERE import_batch_id = ?
|
||||
)
|
||||
""",
|
||||
(result.batch_id,),
|
||||
).fetchall()
|
||||
self.assertTrue(rows)
|
||||
for row in rows:
|
||||
self.assertEqual(row["income"], str(Decimal(row["income"])))
|
||||
self.assertEqual(row["expense"], str(Decimal(row["expense"])))
|
||||
|
||||
def test_every_row_traces_back_to_file_sheet_and_template(self) -> None:
|
||||
result = self.import_sample()
|
||||
row = self.connection.execute(
|
||||
"""
|
||||
SELECT r.source_row, s.sheet_name, s.template_id, s.template_version, f.sha256
|
||||
FROM source_rows r
|
||||
JOIN sheet_batches s ON s.id = r.sheet_batch_id
|
||||
JOIN import_batches b ON b.id = s.import_batch_id
|
||||
JOIN source_files f ON f.id = b.source_file_id
|
||||
WHERE b.id = ?
|
||||
LIMIT 1
|
||||
""",
|
||||
(result.batch_id,),
|
||||
).fetchone()
|
||||
self.assertIsNotNone(row)
|
||||
self.assertEqual(
|
||||
hashlib.sha256(SAMPLE_FILE.read_bytes()).hexdigest(), row["sha256"]
|
||||
)
|
||||
self.assertEqual("ccb-account-detail-v1", row["template_id"])
|
||||
self.assertEqual(1, row["template_version"])
|
||||
self.assertGreater(row["source_row"], 0)
|
||||
self.assertTrue(row["sheet_name"])
|
||||
|
||||
def test_source_file_is_stored_immutably_by_content_hash(self) -> None:
|
||||
self.import_sample()
|
||||
stored = self.connection.execute(
|
||||
"SELECT storage_path FROM source_files"
|
||||
).fetchone()
|
||||
stored_path = Path(stored["storage_path"])
|
||||
self.assertTrue(stored_path.is_file())
|
||||
self.assertEqual(SAMPLE_FILE.read_bytes(), stored_path.read_bytes())
|
||||
self.assertIn(hashlib.sha256(SAMPLE_FILE.read_bytes()).hexdigest(), stored_path.name)
|
||||
|
||||
def test_data_survives_reconnect(self) -> None:
|
||||
result = self.import_sample()
|
||||
self.connection.close()
|
||||
|
||||
reopened = connect(self.db_path)
|
||||
self.addCleanup(reopened.close)
|
||||
batch = reopened.execute(
|
||||
"SELECT status FROM import_batches WHERE id = ?", (result.batch_id,)
|
||||
).fetchone()
|
||||
self.assertEqual("parsed", batch["status"])
|
||||
files = reopened.execute("SELECT COUNT(*) AS n FROM source_files").fetchone()
|
||||
self.assertEqual(1, files["n"])
|
||||
rows = reopened.execute("SELECT COUNT(*) AS n FROM source_rows").fetchone()
|
||||
self.assertGreater(rows["n"], 0)
|
||||
|
||||
|
||||
class IdempotencyTests(PersistenceTestCase):
|
||||
def test_duplicate_upload_creates_no_second_facts(self) -> None:
|
||||
first = self.import_sample()
|
||||
second = self.import_sample()
|
||||
|
||||
self.assertEqual("parsed", first.status)
|
||||
self.assertEqual("duplicate", second.status)
|
||||
self.assertEqual(first.batch_id, second.batch_id)
|
||||
|
||||
files = self.connection.execute("SELECT COUNT(*) AS n FROM source_files").fetchone()
|
||||
self.assertEqual(1, files["n"])
|
||||
sheets = self.connection.execute("SELECT COUNT(*) AS n FROM sheet_batches").fetchone()
|
||||
self.assertEqual(1, sheets["n"])
|
||||
rows = self.connection.execute("SELECT COUNT(*) AS n FROM source_rows").fetchone()
|
||||
self.assertEqual(first.batches[0].transactions.__len__(), rows["n"])
|
||||
|
||||
duplicate = self.connection.execute(
|
||||
"SELECT status, duplicate_of_id FROM import_batches WHERE status = 'duplicate'"
|
||||
).fetchone()
|
||||
self.assertEqual(first.batch_id, duplicate["duplicate_of_id"])
|
||||
|
||||
def test_repeated_upload_under_a_different_filename_is_still_duplicate(self) -> None:
|
||||
first = self.import_sample()
|
||||
second = import_statement(
|
||||
self.connection,
|
||||
self.storage,
|
||||
"改名后的流水.xls",
|
||||
SAMPLE_FILE.read_bytes(),
|
||||
)
|
||||
self.assertEqual("duplicate", second.status)
|
||||
self.assertEqual(first.batch_id, second.batch_id)
|
||||
files = self.connection.execute("SELECT COUNT(*) AS n FROM source_files").fetchone()
|
||||
self.assertEqual(1, files["n"])
|
||||
|
||||
def test_immutability_triggers_block_updates_and_deletes(self) -> None:
|
||||
self.import_sample()
|
||||
for statement in (
|
||||
"UPDATE source_rows SET income = '0'",
|
||||
"DELETE FROM source_rows",
|
||||
"UPDATE source_files SET sha256 = 'x'",
|
||||
"DELETE FROM source_files",
|
||||
"UPDATE sheet_batches SET bank_name = 'x'",
|
||||
"DELETE FROM sheet_batches",
|
||||
):
|
||||
with self.subTest(statement=statement):
|
||||
with self.assertRaises(sqlite3.IntegrityError):
|
||||
self.connection.execute(statement)
|
||||
self.connection.rollback()
|
||||
|
||||
def test_row_level_unique_constraint_blocks_duplicate_rows(self) -> None:
|
||||
self.import_sample()
|
||||
row = self.connection.execute(
|
||||
"SELECT sheet_batch_id, source_row FROM source_rows LIMIT 1"
|
||||
).fetchone()
|
||||
with self.assertRaises(sqlite3.IntegrityError):
|
||||
self.connection.execute(
|
||||
"""
|
||||
INSERT INTO source_rows (
|
||||
sheet_batch_id, source_row, transaction_at, income, expense, created_at
|
||||
) VALUES (?, ?, '2026-01-01T00:00:00', '1', '0', '2026-01-01T00:00:00Z')
|
||||
""",
|
||||
(row["sheet_batch_id"], row["source_row"]),
|
||||
)
|
||||
self.connection.rollback()
|
||||
|
||||
|
||||
class FailedImportTests(PersistenceTestCase):
|
||||
def _unknown_template_bytes(self) -> bytes:
|
||||
workbook = Workbook()
|
||||
sheet = workbook.active
|
||||
sheet.title = "流水"
|
||||
sheet.append(["日期", "金额", "备注"])
|
||||
sheet.append(["2026-01-01", "100.00", "测试"])
|
||||
target = Path(self.temp_dir.name) / "unknown.xlsx"
|
||||
workbook.save(target)
|
||||
return target.read_bytes()
|
||||
|
||||
def test_parse_failure_keeps_exception_batch_without_rows(self) -> None:
|
||||
result = import_statement(
|
||||
self.connection, self.storage, "未知银行.xlsx", self._unknown_template_bytes()
|
||||
)
|
||||
self.assertEqual("exception", result.status)
|
||||
self.assertIsNotNone(result.message)
|
||||
self.assertIn("未知银行.xlsx", result.message)
|
||||
self.assertNotIn(result.sha256, result.message)
|
||||
|
||||
batch = self.connection.execute(
|
||||
"SELECT status FROM import_batches WHERE id = ?", (result.batch_id,)
|
||||
).fetchone()
|
||||
self.assertEqual("exception", batch["status"])
|
||||
|
||||
exceptions = self.connection.execute(
|
||||
"SELECT * FROM import_exceptions WHERE import_batch_id = ?",
|
||||
(result.batch_id,),
|
||||
).fetchall()
|
||||
self.assertEqual(1, len(exceptions))
|
||||
self.assertEqual("parse", exceptions[0]["stage"])
|
||||
|
||||
rows = self.connection.execute("SELECT COUNT(*) AS n FROM source_rows").fetchone()
|
||||
self.assertEqual(0, rows["n"])
|
||||
|
||||
# The source file is still preserved as evidence for later diagnosis.
|
||||
files = self.connection.execute("SELECT COUNT(*) AS n FROM source_files").fetchone()
|
||||
self.assertEqual(1, files["n"])
|
||||
|
||||
def test_reupload_after_failure_is_tracked_as_duplicate(self) -> None:
|
||||
content = self._unknown_template_bytes()
|
||||
first = import_statement(self.connection, self.storage, "未知银行.xlsx", content)
|
||||
second = import_statement(self.connection, self.storage, "未知银行.xlsx", content)
|
||||
self.assertEqual("exception", first.status)
|
||||
self.assertEqual("duplicate", second.status)
|
||||
self.assertEqual(first.batch_id, second.batch_id)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user