B-39: 数据库持久化与不可变导入
- 版本化 SQLite 迁移（向前/回滚）与连接助手。 - 导入管线：上传文件按 SHA-256 内容哈希不可变保存，文件/行/批次 幂等；重复上传返回 duplicate 并复用已有批次，不产生第二份事实。 - 每条规范化源行反查文件、工作表、原始行号与模板版本。 - 解析失败保留 exception/failed 批次与诊断，不产生已确认交易。 - 决策记录见 docs/decisions/002-persistence.md。
This commit is contained in:
腾讯WorkBuddy
@@ -0,0 +1,410 @@
|
||||
"""SQLite persistence: versioned migrations and connection helpers.
|
||||
|
||||
The database, migration runner and file storage choices are recorded in
|
||||
``docs/decisions/002-persistence.md``. Migrations are plain SQL applied in
|
||||
version order; each records itself in ``schema_migrations`` so re-running
|
||||
``migrate`` on an existing database is a no-op. Every migration ships a
|
||||
``down`` script so ``rollback`` can walk backwards for recovery and tests.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
import sqlite3
|
||||
|
||||
|
||||
DEFAULT_DB_PATH = Path("data/app.db")
|
||||
|
||||
|
||||
def utc_now() -> str:
|
||||
return datetime.now(timezone.utc).isoformat()
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Migration:
|
||||
version: int
|
||||
name: str
|
||||
up: str
|
||||
down: str
|
||||
|
||||
|
||||
MIGRATIONS: tuple[Migration, ...] = (
|
||||
Migration(
|
||||
version=1,
|
||||
name="0001_core_persistence",
|
||||
up="""
|
||||
CREATE TABLE companies (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
name TEXT NOT NULL UNIQUE,
|
||||
created_at TEXT NOT NULL,
|
||||
updated_at TEXT NOT NULL
|
||||
);
|
||||
|
||||
CREATE TABLE bank_accounts (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
company_id INTEGER REFERENCES companies (id),
|
||||
account_number TEXT NOT NULL UNIQUE,
|
||||
account_name TEXT,
|
||||
bank_name TEXT,
|
||||
status TEXT NOT NULL DEFAULT 'pending'
|
||||
CHECK (status IN ('pending', 'active', 'disabled')),
|
||||
created_at TEXT NOT NULL,
|
||||
updated_at TEXT NOT NULL
|
||||
);
|
||||
|
||||
CREATE TABLE source_files (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
sha256 TEXT NOT NULL UNIQUE,
|
||||
original_filename TEXT NOT NULL,
|
||||
size_bytes INTEGER NOT NULL,
|
||||
storage_path TEXT NOT NULL,
|
||||
created_at TEXT NOT NULL
|
||||
);
|
||||
|
||||
CREATE TABLE import_batches (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
source_file_id INTEGER NOT NULL REFERENCES source_files (id),
|
||||
status TEXT NOT NULL
|
||||
CHECK (status IN ('parsing', 'parsed', 'exception', 'failed', 'duplicate')),
|
||||
duplicate_of_id INTEGER REFERENCES import_batches (id),
|
||||
diagnostics TEXT,
|
||||
created_at TEXT NOT NULL,
|
||||
updated_at TEXT NOT NULL
|
||||
);
|
||||
|
||||
CREATE TABLE sheet_batches (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
import_batch_id INTEGER NOT NULL REFERENCES import_batches (id),
|
||||
sheet_name TEXT NOT NULL,
|
||||
bank_name TEXT NOT NULL,
|
||||
template_id TEXT NOT NULL,
|
||||
template_version INTEGER NOT NULL,
|
||||
header_row INTEGER NOT NULL,
|
||||
own_account TEXT,
|
||||
own_name TEXT,
|
||||
period_start TEXT,
|
||||
period_end TEXT,
|
||||
transaction_count INTEGER NOT NULL,
|
||||
warnings TEXT NOT NULL DEFAULT '[]',
|
||||
created_at TEXT NOT NULL,
|
||||
UNIQUE (import_batch_id, sheet_name)
|
||||
);
|
||||
|
||||
CREATE TABLE source_rows (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
sheet_batch_id INTEGER NOT NULL REFERENCES sheet_batches (id),
|
||||
source_row INTEGER NOT NULL,
|
||||
transaction_at TEXT NOT NULL,
|
||||
income TEXT NOT NULL,
|
||||
expense TEXT NOT NULL,
|
||||
balance TEXT,
|
||||
own_account TEXT,
|
||||
own_name TEXT,
|
||||
counterparty_account TEXT,
|
||||
counterparty_name TEXT,
|
||||
counterparty_bank TEXT,
|
||||
summary TEXT,
|
||||
purpose TEXT,
|
||||
reference TEXT,
|
||||
currency TEXT,
|
||||
created_at TEXT NOT NULL,
|
||||
UNIQUE (sheet_batch_id, source_row)
|
||||
);
|
||||
|
||||
CREATE TABLE import_exceptions (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
import_batch_id INTEGER NOT NULL REFERENCES import_batches (id),
|
||||
stage TEXT NOT NULL,
|
||||
message TEXT NOT NULL,
|
||||
diagnostics TEXT,
|
||||
created_at TEXT NOT NULL
|
||||
);
|
||||
|
||||
CREATE TRIGGER source_files_no_update BEFORE UPDATE ON source_files
|
||||
BEGIN SELECT RAISE (ABORT, 'source_files rows are immutable'); END;
|
||||
CREATE TRIGGER source_files_no_delete BEFORE DELETE ON source_files
|
||||
BEGIN SELECT RAISE (ABORT, 'source_files rows are immutable'); END;
|
||||
CREATE TRIGGER sheet_batches_no_update BEFORE UPDATE ON sheet_batches
|
||||
BEGIN SELECT RAISE (ABORT, 'sheet_batches rows are immutable'); END;
|
||||
CREATE TRIGGER sheet_batches_no_delete BEFORE DELETE ON sheet_batches
|
||||
BEGIN SELECT RAISE (ABORT, 'sheet_batches rows are immutable'); END;
|
||||
CREATE TRIGGER source_rows_no_update BEFORE UPDATE ON source_rows
|
||||
BEGIN SELECT RAISE (ABORT, 'source_rows rows are immutable'); END;
|
||||
CREATE TRIGGER source_rows_no_delete BEFORE DELETE ON source_rows
|
||||
BEGIN SELECT RAISE (ABORT, 'source_rows rows are immutable'); END;
|
||||
""",
|
||||
down="""
|
||||
DROP TRIGGER IF EXISTS source_rows_no_delete;
|
||||
DROP TRIGGER IF EXISTS source_rows_no_update;
|
||||
DROP TRIGGER IF EXISTS sheet_batches_no_delete;
|
||||
DROP TRIGGER IF EXISTS sheet_batches_no_update;
|
||||
DROP TRIGGER IF EXISTS source_files_no_delete;
|
||||
DROP TRIGGER IF EXISTS source_files_no_update;
|
||||
DROP TABLE IF EXISTS import_exceptions;
|
||||
DROP TABLE IF EXISTS source_rows;
|
||||
DROP TABLE IF EXISTS sheet_batches;
|
||||
DROP TABLE IF EXISTS import_batches;
|
||||
DROP TABLE IF EXISTS source_files;
|
||||
DROP TABLE IF EXISTS bank_accounts;
|
||||
DROP TABLE IF EXISTS companies;
|
||||
""",
|
||||
),
|
||||
Migration(
|
||||
version=2,
|
||||
name="0002_auth_and_sessions",
|
||||
up="""
|
||||
CREATE TABLE users (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
username TEXT NOT NULL UNIQUE,
|
||||
password_hash TEXT NOT NULL,
|
||||
role TEXT NOT NULL CHECK (role IN ('admin', 'company')),
|
||||
company_id INTEGER REFERENCES companies (id),
|
||||
status TEXT NOT NULL DEFAULT 'active' CHECK (status IN ('active', 'disabled')),
|
||||
must_change_password INTEGER NOT NULL DEFAULT 0,
|
||||
created_at TEXT NOT NULL,
|
||||
updated_at TEXT NOT NULL,
|
||||
CHECK (role != 'company' OR company_id IS NOT NULL),
|
||||
CHECK (role != 'admin' OR company_id IS NULL)
|
||||
);
|
||||
|
||||
CREATE TABLE sessions (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
token_hash TEXT NOT NULL UNIQUE,
|
||||
user_id INTEGER NOT NULL REFERENCES users (id),
|
||||
created_at TEXT NOT NULL,
|
||||
expires_at TEXT NOT NULL,
|
||||
revoked_at TEXT
|
||||
);
|
||||
|
||||
CREATE TABLE login_attempts (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
username TEXT NOT NULL,
|
||||
ip TEXT NOT NULL,
|
||||
success INTEGER NOT NULL,
|
||||
created_at TEXT NOT NULL
|
||||
);
|
||||
|
||||
CREATE TABLE audit_log (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
actor_user_id INTEGER REFERENCES users (id),
|
||||
actor_username TEXT,
|
||||
action TEXT NOT NULL,
|
||||
target TEXT,
|
||||
detail TEXT,
|
||||
ip TEXT,
|
||||
created_at TEXT NOT NULL
|
||||
);
|
||||
|
||||
ALTER TABLE import_batches ADD COLUMN company_id INTEGER REFERENCES companies (id);
|
||||
""",
|
||||
down="""
|
||||
DROP TABLE IF EXISTS audit_log;
|
||||
DROP TABLE IF EXISTS login_attempts;
|
||||
DROP TABLE IF EXISTS sessions;
|
||||
DROP TABLE IF EXISTS users;
|
||||
ALTER TABLE import_batches DROP COLUMN company_id;
|
||||
""",
|
||||
),
|
||||
Migration(
|
||||
version=3,
|
||||
name="0003_dynamic_master_data",
|
||||
# bank_accounts is rebuilt (SQLite cannot alter CHECK constraints):
|
||||
# status gains 'returned', and the account gains type, effective
|
||||
# interval and review fields. account_number stays UNIQUE and stores
|
||||
# the normalized digits-only form (see master_data.normalize).
|
||||
up="""
|
||||
CREATE TABLE bank_accounts_new (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
company_id INTEGER NOT NULL REFERENCES companies (id),
|
||||
account_number TEXT NOT NULL UNIQUE,
|
||||
account_name TEXT,
|
||||
bank_name TEXT NOT NULL DEFAULT '',
|
||||
account_type TEXT NOT NULL DEFAULT '一般户'
|
||||
CHECK (account_type IN ('基本户', '一般户', '专用户')),
|
||||
status TEXT NOT NULL DEFAULT 'pending'
|
||||
CHECK (status IN ('pending', 'active', 'returned', 'disabled')),
|
||||
effective_from TEXT,
|
||||
effective_to TEXT,
|
||||
submitted_by INTEGER REFERENCES users (id),
|
||||
reviewed_by INTEGER REFERENCES users (id),
|
||||
reviewed_at TEXT,
|
||||
review_reason TEXT,
|
||||
created_at TEXT NOT NULL,
|
||||
updated_at TEXT NOT NULL
|
||||
);
|
||||
INSERT INTO bank_accounts_new (
|
||||
id, company_id, account_number, account_name, bank_name,
|
||||
status, created_at, updated_at
|
||||
)
|
||||
SELECT id, company_id, account_number, account_name,
|
||||
COALESCE(bank_name, ''), status, created_at, updated_at
|
||||
FROM bank_accounts WHERE company_id IS NOT NULL;
|
||||
DROP TABLE bank_accounts;
|
||||
ALTER TABLE bank_accounts_new RENAME TO bank_accounts;
|
||||
|
||||
CREATE TABLE account_aliases (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
bank_account_id INTEGER NOT NULL REFERENCES bank_accounts (id),
|
||||
alias_kind TEXT NOT NULL CHECK (alias_kind IN ('name', 'account')),
|
||||
alias_value TEXT NOT NULL,
|
||||
priority INTEGER NOT NULL DEFAULT 100,
|
||||
effective_from TEXT,
|
||||
effective_to TEXT,
|
||||
created_by INTEGER REFERENCES users (id),
|
||||
created_at TEXT NOT NULL,
|
||||
UNIQUE (bank_account_id, alias_kind, alias_value)
|
||||
);
|
||||
|
||||
CREATE TABLE master_data_changes (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
entity_type TEXT NOT NULL
|
||||
CHECK (entity_type IN ('company', 'user', 'bank_account', 'account_alias')),
|
||||
entity_id INTEGER NOT NULL,
|
||||
action TEXT NOT NULL,
|
||||
before_json TEXT,
|
||||
after_json TEXT,
|
||||
reason TEXT,
|
||||
actor_user_id INTEGER REFERENCES users (id),
|
||||
actor_username TEXT,
|
||||
created_at TEXT NOT NULL
|
||||
);
|
||||
|
||||
ALTER TABLE companies ADD COLUMN credit_code TEXT;
|
||||
ALTER TABLE companies ADD COLUMN cashier_name TEXT;
|
||||
ALTER TABLE companies ADD COLUMN status TEXT NOT NULL DEFAULT 'active'
|
||||
CHECK (status IN ('active', 'preparing', 'disabled'));
|
||||
""",
|
||||
down="""
|
||||
ALTER TABLE companies DROP COLUMN status;
|
||||
ALTER TABLE companies DROP COLUMN cashier_name;
|
||||
ALTER TABLE companies DROP COLUMN credit_code;
|
||||
DROP TABLE IF EXISTS master_data_changes;
|
||||
DROP TABLE IF EXISTS account_aliases;
|
||||
|
||||
CREATE TABLE bank_accounts_new (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
company_id INTEGER REFERENCES companies (id),
|
||||
account_number TEXT NOT NULL UNIQUE,
|
||||
account_name TEXT,
|
||||
bank_name TEXT,
|
||||
status TEXT NOT NULL DEFAULT 'pending'
|
||||
CHECK (status IN ('pending', 'active', 'disabled')),
|
||||
created_at TEXT NOT NULL,
|
||||
updated_at TEXT NOT NULL
|
||||
);
|
||||
INSERT INTO bank_accounts_new (
|
||||
id, company_id, account_number, account_name, bank_name,
|
||||
status, created_at, updated_at
|
||||
)
|
||||
SELECT id, company_id, account_number, account_name, bank_name,
|
||||
CASE WHEN status = 'returned' THEN 'pending' ELSE status END,
|
||||
created_at, updated_at
|
||||
FROM bank_accounts;
|
||||
DROP TABLE bank_accounts;
|
||||
ALTER TABLE bank_accounts_new RENAME TO bank_accounts;
|
||||
""",
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def connect(path: str | Path) -> sqlite3.Connection:
|
||||
db_path = Path(path)
|
||||
if str(db_path) != ":memory:":
|
||||
db_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
connection = sqlite3.connect(str(db_path))
|
||||
connection.row_factory = sqlite3.Row
|
||||
connection.execute("PRAGMA foreign_keys = ON")
|
||||
return connection
|
||||
|
||||
|
||||
def applied_versions(connection: sqlite3.Connection) -> list[int]:
|
||||
exists = connection.execute(
|
||||
"SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'schema_migrations'"
|
||||
).fetchone()
|
||||
if not exists:
|
||||
return []
|
||||
rows = connection.execute(
|
||||
"SELECT version FROM schema_migrations ORDER BY version"
|
||||
).fetchall()
|
||||
return [row["version"] for row in rows]
|
||||
|
||||
|
||||
def migrate(connection: sqlite3.Connection) -> list[int]:
|
||||
"""Apply every pending migration; returns the versions applied now."""
|
||||
connection.execute(
|
||||
"""
|
||||
CREATE TABLE IF NOT EXISTS schema_migrations (
|
||||
version INTEGER PRIMARY KEY,
|
||||
name TEXT NOT NULL,
|
||||
applied_at TEXT NOT NULL
|
||||
)
|
||||
"""
|
||||
)
|
||||
applied = set(applied_versions(connection))
|
||||
newly_applied: list[int] = []
|
||||
for migration in MIGRATIONS:
|
||||
if migration.version in applied:
|
||||
continue
|
||||
with connection:
|
||||
connection.executescript(migration.up)
|
||||
connection.execute(
|
||||
"INSERT INTO schema_migrations (version, name, applied_at) VALUES (?, ?, ?)",
|
||||
(migration.version, migration.name, utc_now()),
|
||||
)
|
||||
newly_applied.append(migration.version)
|
||||
return newly_applied
|
||||
|
||||
|
||||
def rollback(connection: sqlite3.Connection, target_version: int = 0) -> list[int]:
|
||||
"""Reverse migrations above ``target_version``; returns reversed versions."""
|
||||
applied = applied_versions(connection)
|
||||
reversed_versions: list[int] = []
|
||||
for migration in sorted(MIGRATIONS, key=lambda item: item.version, reverse=True):
|
||||
if migration.version <= target_version or migration.version not in applied:
|
||||
continue
|
||||
with connection:
|
||||
connection.executescript(migration.down)
|
||||
connection.execute(
|
||||
"DELETE FROM schema_migrations WHERE version = ?",
|
||||
(migration.version,),
|
||||
)
|
||||
reversed_versions.append(migration.version)
|
||||
return reversed_versions
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="Apply or roll back database migrations.")
|
||||
parser.add_argument(
|
||||
"db_path",
|
||||
type=Path,
|
||||
nargs="?",
|
||||
default=DEFAULT_DB_PATH,
|
||||
help="SQLite database path (default: data/app.db)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--rollback-to",
|
||||
type=int,
|
||||
default=None,
|
||||
metavar="VERSION",
|
||||
help="Reverse migrations above VERSION instead of migrating forward",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
connection = connect(args.db_path)
|
||||
try:
|
||||
if args.rollback_to is None:
|
||||
applied = migrate(connection)
|
||||
print(f"applied migrations: {applied or 'none (already up to date)'}")
|
||||
else:
|
||||
reversed_versions = rollback(connection, args.rollback_to)
|
||||
print(f"reversed migrations: {reversed_versions or 'none'}")
|
||||
finally:
|
||||
connection.close()
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Reference in New Issue
Block a user