"""HEL-529 rework regressions: staging dedupe, anomaly convergence, source-catalog observation join (dataset-name ↔ interface-name), lineage update_freq. All read-only or within-batch fixes; none touch routing, the 8765 main site, or the 问天 frozen zone. """ from __future__ import annotations import tempfile import unittest from pathlib import Path from datahub.adapters.tushare import TushareAdapter from datahub.crypto import SecretVault from datahub.hub import Hub from datahub.pipeline import _dedupe_staging_rows from datahub.settings import Settings from tests.fixtures import fake_transport class StagingDedupeTests(unittest.TestCase): def test_popularity_within_batch_duplicates_collapse_keep_last(self) -> None: rows = [ {"ts_code": "600000.SH", "trade_date": "20260914", "source": "ths", "rank": 1}, {"ts_code": "000868.SZ", "trade_date": "20260914", "source": "dc", "rank": 2}, {"ts_code": "600000.SH", "trade_date": "20260914", "source": "dc", "rank": 3, "hot": 9.9}, {"ts_code": "600000.SH", "trade_date": "20260914", "source": "dc", "rank": 4, "hot": 8.8}, ] out = _dedupe_staging_rows("popularity", rows) self.assertEqual(len(out), 3) # (600000,ths) (000868,dc) (600000,dc) dup = [r for r in out if r["ts_code"] == "600000.SH" and r["source"] == "dc"][0] self.assertEqual(dup["rank"], 4) # keeps LAST occurrence self.assertEqual(out[0]["ts_code"], "600000.SH") # preserves first-seen order def test_dragon_tiger_seat_duplicates_collapse(self) -> None: rows = [ {"ts_code": "300010.SZ", "trade_date": "20260914", "hm_name": "T王", "buy_amount": 100}, {"ts_code": "300010.SZ", "trade_date": "20260914", "hm_name": "T王", "buy_amount": 200}, {"ts_code": "300010.SZ", "trade_date": "20260914", "hm_name": "T王", "buy_amount": 300}, ] out = _dedupe_staging_rows("dragon_tiger", rows) self.assertEqual(len(out), 1) self.assertEqual(out[0]["buy_amount"], 300) def test_different_sources_are_not_duplicates(self) -> None: rows = [ {"ts_code": "600000.SH", "trade_date": "20260914", "source": "ths"}, {"ts_code": "600000.SH", "trade_date": "20260914", "source": "dc"}, ] self.assertEqual(len(_dedupe_staging_rows("popularity", rows)), 2) def test_unknown_dataset_passthrough(self) -> None: rows = [{"a": 1}, {"a": 1}] self.assertEqual(_dedupe_staging_rows("calendar", rows), rows) class _Base(unittest.TestCase): def setUp(self) -> None: self.tmp = tempfile.TemporaryDirectory() settings = Settings( encryption_key=SecretVault.generate_key(), api_token="z" * 32, admin_password="StartPass1", tushare_token="real-tushare-token-abcdef", db_path=Path(self.tmp.name) / "hub.db", scheduler_enabled=False, ) self.hub = Hub(settings, adapter=TushareAdapter("real-tushare-token-abcdef", transport=fake_transport)) def tearDown(self) -> None: self.tmp.cleanup() class StagingDedupePublishTests(_Base): def test_duplicate_popularity_and_dragon_tiger_now_publish(self) -> None: """Replays the 2026-09-14 production failure: within-response duplicate keys used to abort the whole batch at the staging INSERT; with dedupe the same upstream payload publishes.""" db = self.hub.db trade_date = "20240902" # dc_hot returns 600000.SH twice within one response; hm_detail returns # the same (ts_code, hm_name) seat three times (mirrors live evidence). popularity_rows = [ {"ts_code": "600000.SH", "trade_date": trade_date, "source": "ths", "ts_name": "浦发银行", "rank": 1, "pct_change": 1.2, "current_price": 10.2, "hot": 90.0, "concept": "银行", "data_type": "热股"}, {"ts_code": "600000.SH", "trade_date": trade_date, "source": "dc", "ts_name": "浦发银行", "rank": 2, "pct_change": 1.2, "current_price": 10.2, "hot": 80.0, "concept": "银行", "data_type": "A股市场"}, {"ts_code": "600000.SH", "trade_date": trade_date, "source": "dc", "ts_name": "浦发银行", "rank": 3, "pct_change": 1.3, "current_price": 10.3, "hot": 81.0, "concept": "银行", "data_type": "A股市场"}, ] dragon_rows = [ {"trade_date": trade_date, "ts_code": "600000.SH", "ts_name": "浦发银行", "buy_amount": 100, "sell_amount": 200, "net_amount": -100, "hm_name": "测试游资", "hm_orgs": "某某营业部", "tag": "超买"}, {"trade_date": trade_date, "ts_code": "600000.SH", "ts_name": "浦发银行", "buy_amount": 300, "sell_amount": 0, "net_amount": 300, "hm_name": "测试游资", "hm_orgs": "某某营业部", "tag": "超买"}, ] self.hub.pipeline._stage("popularity", "b-dup-pop", popularity_rows) self.hub.pipeline._stage("dragon_tiger", "b-dup-dt", dragon_rows) pop = db.fetchall("SELECT * FROM staging_popularity WHERE batch_id = 'b-dup-pop'") dt = db.fetchall("SELECT * FROM staging_dragon_tiger WHERE batch_id = 'b-dup-dt'") self.assertEqual(len(pop), 2) self.assertEqual(len(dt), 1) kept = dt[0] self.assertEqual(kept["buy_amount"], 300) def test_hel562_popularity_collapsed_dups_publish_not_hard_fail(self) -> None: """HEL-562: staging dedupe alone is not enough — quality gate used to hard-fail on the same within-batch dups after they were already collapsed (live 20260915: duplicate keys: 3 → integrity_gate).""" trade_date = "20240902" # 3 within-batch dups on (ts_code, trade_date, source=dc) — mirrors # ths+dc merge where dc_hot repeats the same keys. rows = [ {"ts_code": "600000.SH", "trade_date": trade_date, "source": "ths", "ts_name": "浦发银行", "rank": 1, "pct_change": 1.2, "current_price": 10.2, "hot": 90.0, "concept": "银行", "data_type": "热股"}, {"ts_code": "000001.SZ", "trade_date": trade_date, "source": "dc", "ts_name": "平安银行", "rank": 1, "pct_change": 2.0, "current_price": 11.0, "hot": 88.0, "concept": "银行", "data_type": "A股市场"}, {"ts_code": "600000.SH", "trade_date": trade_date, "source": "dc", "ts_name": "浦发银行", "rank": 2, "pct_change": 1.2, "current_price": 10.2, "hot": 80.0, "concept": "银行", "data_type": "A股市场"}, {"ts_code": "600000.SH", "trade_date": trade_date, "source": "dc", "ts_name": "浦发银行", "rank": 3, "pct_change": 1.3, "current_price": 10.3, "hot": 81.0, "concept": "银行", "data_type": "A股市场"}, {"ts_code": "600000.SH", "trade_date": trade_date, "source": "dc", "ts_name": "浦发银行", "rank": 4, "pct_change": 1.4, "current_price": 10.4, "hot": 82.0, "concept": "银行", "data_type": "A股市场"}, ] report = self.hub.pipeline.validate("popularity", "b-gate", trade_date, rows) self.assertFalse(report["hard_fail"]) self.assertEqual(report["errors"], []) self.assertEqual(report["warnings"], ["duplicate keys: 2"]) result = self.hub.pipeline.run_dataset("popularity", trade_date, prepared_rows=rows) # warnings present → soft_fail → publication state is degraded (still served) self.assertEqual(result["state"], "degraded") self.assertFalse(result["quality"]["hard_fail"]) self.assertTrue(result["quality"]["soft_fail"]) self.assertIn("duplicate keys: 2", result["quality"]["warnings"]) eod = self.hub.db.fetchall( "SELECT * FROM eod_popularity WHERE trade_date = ? AND batch_id = ?", (trade_date, result["batch_id"]), ) # 5 raw → 3 unique keys after staging collapse (ths + two dc codes) self.assertEqual(len(eod), 3) pub = self.hub.db.fetchone( "SELECT * FROM publications WHERE dataset='popularity' AND trade_date=?", (trade_date,), ) self.assertEqual(pub["active_batch"], result["batch_id"]) self.assertEqual(pub["state"], "degraded") # Serving path accepts degraded the same as published (no DATASET_NOT_PUBLISHED) from datahub.serving import V1API api = V1API(self.hub.db, self.hub.pipeline, self.hub.settings) payload = api.handle("/v1/popularity", {"date": [trade_date]}) self.assertEqual(len(payload["data"]), 3) self.assertEqual(payload["meta"]["state"], "degraded") self.assertEqual(payload["meta"]["batch_id"], result["batch_id"]) def test_hel562_core_soft_still_hard_fails_on_duplicate_keys(self) -> None: """moneyflow/auction stay on the old soft gate: raw dups → hard_fail.""" trade_date = "20240902" rows = [ {"ts_code": "600000.SH", "trade_date": trade_date, "buy_sm_amount": 1, "sell_sm_amount": 1, "buy_md_amount": 1, "sell_md_amount": 1, "buy_lg_amount": 1, "sell_lg_amount": 1, "buy_elg_amount": 1, "sell_elg_amount": 1, "net_mf_amount": 0}, {"ts_code": "600000.SH", "trade_date": trade_date, "buy_sm_amount": 2, "sell_sm_amount": 2, "buy_md_amount": 2, "sell_md_amount": 2, "buy_lg_amount": 2, "sell_lg_amount": 2, "buy_elg_amount": 2, "sell_elg_amount": 2, "net_mf_amount": 0}, ] report = self.hub.pipeline.validate("moneyflow", "b-mf", trade_date, rows) self.assertTrue(report["hard_fail"]) self.assertIn("duplicate keys: 1", report["errors"]) self.assertEqual(report["warnings"], []) def test_hel562_popularity_date_mismatch_still_hard_fails(self) -> None: """Collapsed-dup carve-out must not weaken other soft integrity checks.""" rows = [ {"ts_code": "600000.SH", "trade_date": "20240901", "source": "ths", "ts_name": "浦发银行", "rank": 1, "pct_change": 1.2, "current_price": 10.2, "hot": 90.0, "concept": "银行", "data_type": "热股"}, ] report = self.hub.pipeline.validate("popularity", "b-bad-date", "20240902", rows) self.assertTrue(report["hard_fail"]) self.assertIn("date mismatch rows: 1", report["errors"]) class OverviewAnomalyConvergenceTests(_Base): def _seed_batches(self, today: str) -> None: db = self.hub.db rows = [ # resolved history: earlier failures/stalls, later success ("x-daily-001", "daily", "staged", "empty official batch", "2026-09-14T15:05:00+08:00"), ("x-daily-002", "daily", "failed", "release group not switched", "2026-09-14T16:10:00+08:00"), ("x-daily-006", "daily", "published", "", "2026-09-14T20:00:00+08:00"), # current unresolved faults ("x-pop-001", "popularity", "failed", "UNIQUE constraint failed: staging_popularity", "2026-09-14T22:40:00+08:00"), ("x-dt-001", "dragon_tiger", "failed", "UNIQUE constraint failed: staging_dragon_tiger", "2026-09-14T16:45:00+08:00"), ("x-dt-002", "dragon_tiger", "failed", "UNIQUE constraint failed: staging_dragon_tiger", "2026-09-14T21:48:00+08:00"), # staged-empty later published ("x-idx-001", "index_daily", "staged", "empty official batch", "2026-09-14T15:10:00+08:00"), ("x-idx-003", "index_daily", "published", "", "2026-09-14T16:10:00+08:00"), ] for batch_id, dataset, state, error, started in rows: db.execute( "INSERT INTO batches(batch_id, dataset, trade_date, state, attempt, rows_in, rows_out," " quality_json, started_at, finished_at, error) VALUES (?,?,?,?,?,?,?,?,?,?,?)", (batch_id, dataset, today, state, 1, None, None, None, started, started if state != "staged" else None, error or None), ) db.execute( "INSERT INTO publications(dataset, trade_date, active_batch, prev_batch, state, published_at)" " VALUES ('daily', ?, 'x-daily-006', 'x-daily-005', 'published', '2026-09-14T20:00:22+08:00')", (today,), ) db.execute( "INSERT INTO publications(dataset, trade_date, active_batch, prev_batch, state, published_at)" " VALUES ('index_daily', ?, 'x-idx-003', 'x-idx-002', 'published', '2026-09-14T16:10:12+08:00')", (today,), ) def test_anomalies_only_latest_unresolved(self) -> None: overview = self.hub.admin.overview() today = overview["trade_date"] self._seed_batches(today) anomalies = self.hub.admin.overview()["anomalies"] got = sorted((a["dataset"], a["batch_id"]) for a in anomalies) self.assertEqual( got, [ ("dragon_tiger", "x-dt-002"), # latest failed, never published ("popularity", "x-pop-001"), # latest failed, never published ], ) class SourceCatalogJoinTests(_Base): def test_observation_join_matches_dataset_named_health_rows(self) -> None: db = self.hub.db now = "2026-09-15T08:00:00+08:00" # tushare observability writes dataset names (real legacy behavior) for iface, state in [("valuation", "ok"), ("popularity", "ok"), ("stocks", "ok")]: db.execute( "INSERT INTO provider_health(provider, interface, state, last_ok_at, last_error," " last_fallback_reason, consec_failures, last_latency_ms, last_data_age_seconds, updated_at)" " VALUES ('tushare', ?, ?, ?, '', '', 0, 300, NULL, ?)", (iface, state, now, now), ) # realtime providers write interface names db.execute( "INSERT INTO provider_health(provider, interface, state, last_ok_at, last_error," " last_fallback_reason, consec_failures, last_latency_ms, last_data_age_seconds, updated_at)" " VALUES ('eastmoney', 'indices', 'ok', ?, '', '', 0, 153, NULL, ?)", (now, now), ) items = self.hub.admin.source_catalog()["items"] tushare = [i for i in items if i["provider"] == "tushare"][0] by_iface = {i["interface"]: i for i in tushare["interfaces"]} # daily_basic serves valuation → observed via dataset name self.assertTrue(by_iface["daily_basic"]["observed"]) self.assertEqual(by_iface["daily_basic"]["observed_basis"], "dataset") self.assertEqual(by_iface["daily_basic"]["observed_state"], "ok") # ths_hot + dc_hot serve popularity → observed via dataset name self.assertTrue(by_iface["ths_hot"]["observed"]) self.assertTrue(by_iface["dc_hot"]["observed"]) # stock_basic serves stocks self.assertTrue(by_iface["stock_basic"]["observed"]) # never-observed interface stays honestly unobserved (not "unconfigured") self.assertFalse(by_iface["trade_cal"]["observed"]) self.assertEqual(by_iface["trade_cal"]["observed_state"], "") # interfaces carry real batch groups groups = {i["interface"]: i["group"] for i in tushare["interfaces"]} self.assertEqual(groups["daily"], "盘后 A 批") self.assertEqual(groups["ths_hot"], "扩展软批") self.assertEqual(groups["index_daily"], "指数 B 批") eastmoney = [i for i in items if i["provider"] == "eastmoney"][0] em = {i["interface"]: i for i in eastmoney["interfaces"]} self.assertTrue(em["indices"]["observed"]) self.assertEqual(em["indices"]["observed_basis"], "interface") self.assertFalse(em["market_quotes"]["observed"]) def test_lineage_update_freq_present(self) -> None: items = self.hub.admin.lineage("20240902")["items"] self.assertTrue(items) for item in items: self.assertTrue(item.get("update_freq"), f"missing update_freq for {item['dataset']}") class LineageMentorIfindTests(_Base): """问师 → iFinD 血缘修正（第二轮返工）：不按名称猜关系，按真实调用代码。""" REPO = Path(__file__).resolve().parents[2] def test_no_mentor_dependency_on_ifind_wencai(self) -> None: items = self.hub.admin.lineage("20240902")["items"] wencai = [i for i in items if i["dataset"] == "ifind_wencai"] self.assertEqual(len(wencai), 1) consumers = wencai[0]["known_consumers"] for consumer in consumers: self.assertNotIn("问师", consumer, f"wencai consumer must not be 问师: {consumer}") all_consumers = " ".join(c for i in items for c in i["known_consumers"]) self.assertNotIn("问师(自然语言选股", all_consumers) def test_wencai_real_consumer_is_pools_enrichment_with_code_evidence(self) -> None: items = self.hub.admin.lineage("20240902")["items"] wencai = [i for i in items if i["dataset"] == "ifind_wencai"][0] self.assertTrue(any("股票池" in c for c in wencai["known_consumers"])) # Real call evidence in the main-site source tree: pools_src = (self.REPO / "backend" / "features" / "pools" / "service.py").read_text(encoding="utf-8") self.assertIn("ifind.wencai(", pools_src) self.assertIn("ifind_event_enrichment_v1", pools_src) # And 问师 itself never calls wencai: mentor_src = (self.REPO / "backend" / "features" / "mentor" / "service.py").read_text(encoding="utf-8") self.assertNotIn(".wencai(", mentor_src) def test_mentor_optional_ifind_history_subcapabilities(self) -> None: items = self.hub.admin.lineage("20240902")["items"] history = [i for i in items if i["dataset"] == "ifind_history"] self.assertEqual(len(history), 1) consumers = history[0]["known_consumers"] self.assertTrue(any("趋势思维模型" in c for c in consumers)) self.assertTrue(any("宏观思维模型" in c for c in consumers)) # every consumer must be a 问师 sub-capability, not the whole board for consumer in consumers: self.assertIn("·", consumer, f"not a sub-capability mapping: {consumer}") # Real call evidence: mentor builds market matrices via ifind.history mentor_src = (self.REPO / "backend" / "features" / "mentor" / "service.py").read_text(encoding="utf-8") self.assertIn("ifind.history(", mentor_src) self.assertIn("_mentor_market_matrix", mentor_src) self.assertIn("MENTOR_INDEX_UNIVERSE", mentor_src) self.assertIn("MENTOR_ETF_UNIVERSE", mentor_src) # Optional dependency: fails open when ifind is not configured self.assertIn("if not ifind or not ifind.configured", mentor_src) if __name__ == "__main__": unittest.main()