fix(HEL-562): 人气榜批内已合并重复键不再硬失败

扩展软数据集（popularity 等）在 staging 已按业务键去重后，
quality gate 仍把原始抓取的 duplicate keys 记为 hard_fail，
导致 20260915 人气榜 integrity_gate 拒发。现降为 warning（soft_fail/
degraded 仍可发布），核心七类与 moneyflow/auction 判重口径不变。

Co-authored-by: Cursor <cursoragent@cursor.com>
Co-authored-by: multica-agent <github@multica.ai>

xiaobai-datahub/datahub/pipeline.py

@@ -1491,8 +1491,20 @@ class Pipeline:
         else:
             keys = [(row.get("ts_code"), row.get("trade_date")) for row in rows]
         dup = row_n - len(set(keys))
+        # Extended soft datasets (popularity/dragon_tiger/…) already collapse
+        # within-batch dups in _dedupe_staging_rows before INSERT. Upstream
+        # ths/dc (and similar) routinely emit duplicate business keys; those
+        # collapsed dups must not hard-fail publish (HEL-562). Datasets without
+        # a staging business key — and all hard/core soft gates — still treat
+        # raw duplicate keys as errors.
+        staging_collapses_dups = (
+            dataset in EXTENDED_SOFT_DATASETS and dataset in STAGING_KEY_FIELDS
+        )
         if dup:
-            errors.append(f"duplicate keys: {dup}")
+            if staging_collapses_dups:
+                warnings.append(f"duplicate keys: {dup}")
+            else:
+                errors.append(f"duplicate keys: {dup}")
         bad_date = sum(1 for row in rows if str(row.get("trade_date")) != trade_date)
         if bad_date:
             errors.append(f"date mismatch rows: {bad_date}")
@@ -1511,7 +1523,8 @@ class Pipeline:
         field_report = self._field_gate(dataset, trade_date, rows, errors)
         if dataset in SOFT_DATASETS:
             allow_empty = dataset in {"popularity", "dragon_tiger", "moneyflow", "auction"}
-            hard_fail = bool(dup or bad_date or (empty and not allow_empty))
+            hard_dup = 0 if staging_collapses_dups else dup
+            hard_fail = bool(hard_dup or bad_date or (empty and not allow_empty))
         else:
             hard_fail = bool(errors) and (dataset in HARD_DATASETS or dataset == STOCKS_DATASET)
         report = {