P0 — Incident 状态机 + 分层自适应调查引擎#

-- incidents 表新增字段（ALTER TABLE ADD COLUMN IF NOT EXISTS 平滑升级）
ALTER TABLE incidents ADD COLUMN IF NOT EXISTS
    investigation_tier TEXT NOT NULL DEFAULT '';        -- 当前调查停在哪个 Tier (tier1/tier2/tier3/done)
ALTER TABLE incidents ADD COLUMN IF NOT EXISTS
    investigation_id TEXT;                              -- 关联 investigations 表
ALTER TABLE incidents ADD COLUMN IF NOT EXISTS
    rca_confidence DOUBLE PRECISION;                   -- 根因置信度
ALTER TABLE incidents ADD COLUMN IF NOT EXISTS
    resolved_count INTEGER NOT NULL DEFAULT 0;          -- 被拉回次数（防无限循环）
ALTER TABLE incidents ADD COLUMN IF NOT EXISTS
    escalation_reason TEXT NOT NULL DEFAULT '';          -- ESCALATED 原因

-- 状态迁移日志（审计 + 调试）
CREATE TABLE IF NOT EXISTS incident_transitions (
    id TEXT PRIMARY KEY,
    incident_id TEXT NOT NULL REFERENCES incidents(id) ON DELETE CASCADE,
    from_status TEXT NOT NULL,
    to_status TEXT NOT NULL,
    event TEXT NOT NULL,                                -- 触发事件名
    actor TEXT NOT NULL DEFAULT 'system',               -- system/human/<user_id>
    context JSONB NOT NULL DEFAULT '{}'::jsonb,         -- 事件上下文
    created_at TIMESTAMPTZ NOT NULL DEFAULT now()
);
CREATE INDEX IF NOT EXISTS idx_incident_transitions_incident
    ON incident_transitions(incident_id, created_at);

class AlertGroupReason(StrEnum):
    """告警聚合原因。"""
    ALERTMANAGER_GROUP = "alertmanager_group"   # Alertmanager 原生 groupKey
    TIME_WINDOW        = "time_window"          # 时间窗聚合（同 service + N秒内）
    TOPOLOGY           = "topology"             # 拓扑级联关联
    DEDUP              = "dedup"                # 重复告警去重

class TopologyRelation(BaseModel):
    """拓扑关联关系。"""
    source_service: str
    target_service: str
    relation_type: str = "depends_on"           # depends_on / same_node / same_namespace
    confidence: float = 1.0

class ImpactScore(BaseModel):
    """影响面评分（决定各 Tier 的置信门槛）。"""
    score: float = Field(ge=0.0, le=1.0)        # 0=低影响, 1=高影响
    factors: dict[str, float] = Field(default_factory=dict)  # 各因子权重明细
    # 因子示例: severity_weight, service_criticality, alert_count_weight, is_flapping_penalty

class IncidentContext(BaseModel):
    """L2 → L3 的产出。预处理层收敛后的根因候选集。"""
    incident_id: str
    source_alerts: list[NormalizedAlert]         # 聚合后的原始告警（≥1条）
    alert_group_reason: AlertGroupReason
    correlation_key: str
    root_cause_candidates: list[str] = Field(default_factory=list)  # L2 拓扑关联产出的候选组件
    impact_score: ImpactScore
    is_flapping: bool = False                    # 抖动检测结果
    topology_context: list[TopologyRelation] = Field(default_factory=list)
    matched_runbooks: list[str] = Field(default_factory=list)  # Tier 1 预匹配的 Runbook ID
    primary_service: str = ""
    primary_alertname: str = ""
    created_at: datetime

class PreprocessingConfig(BaseModel):
    """L2 预处理规则配置（对标 Alertmanager group_wait/group_interval/inhibit_rules）。"""

    # 时间窗聚合
    group_wait_sec: int = 30                     # 首条告警后等待 N 秒，聚合同 service 告警
    group_interval_sec: int = 300                # 同一 group 两次推送的最小间隔

    # 去重
    dedup_window_sec: int = 600                  # 同 fingerprint 告警在 N 秒内视为重复

    # 抑制规则（简化版 inhibit_rules）
    inhibit_rules: list[InhibitRule] = Field(default_factory=list)

    # 抖动检测
    flapping_threshold: int = 5                  # N 秒窗口内 firing→resolved 翻转 ≥ 此值判定抖动
    flapping_window_sec: int = 600

    # Impact 评分因子权重
    severity_weights: dict[str, float] = Field(
        default_factory=lambda: {"critical": 1.0, "warning": 0.5, "info": 0.1}
    )

class InhibitRule(BaseModel):
    """抑制规则：当高优先级告警存在时，抑制低优先级告警。"""
    source_match: dict[str, str]                 # 抑制源标签匹配 {"severity": "critical"}
    target_match: dict[str, str]                 # 被抑制目标标签匹配
    equal: list[str] = Field(default_factory=list)  # 要求两者相同的 label 列表

class RunbookMatchCondition(BaseModel):
    """Tier 1 匹配条件：告警指纹 + 前置条件校验。"""
    alertname_pattern: str                       # 正则匹配 alertname
    service_pattern: str = ".*"                  # 正则匹配 service
    label_matchers: dict[str, str] = Field(default_factory=dict)  # label 精确匹配
    severity_in: list[str] = Field(default_factory=lambda: ["critical", "warning"])

class RunbookStep(BaseModel):
    """Runbook 中的一个处置步骤。"""
    action_type: str                             # 资源域操作 (container.restart / file.write / ...)
    resource_pattern: str                        # 资源 URI 模板 (container://{service_name})
    params: dict[str, Any] = Field(default_factory=dict)
    description: str = ""
    timeout_sec: int = 60

class RunbookVerifyStep(BaseModel):
    """Runbook 的验证步骤（跑完 ≠ 解决）。"""
    gate: str                                    # command_success / service_health / metric_recovery
    params: dict[str, Any] = Field(default_factory=dict)
    timeout_sec: int = 60

class Runbook(BaseModel):
    """结构化 Runbook（Tier 1 确定性处置的单元）。"""
    id: str
    name: str
    display_name: str
    version: str = "v1"
    source: str = "seed"                         # seed(种子导入) / auto(Tier 3自增长) / manual(人工创建)
    status: str = "active"                       # active / disabled / pending_review

    # detect: 什么时候匹配
    match_conditions: list[RunbookMatchCondition]
    cooldown_sec: int = 300                      # 冷却期（同 Incident 命中后 N 秒内不再重复触发）

    # diagnose: 前置条件校验（可选，跑处置前确认环境）
    precondition_checks: list[RunbookStep] = Field(default_factory=list)

    # action: 处置步骤序列
    action_steps: list[RunbookStep]

    # verify: 验证步骤（跑完后验证是否真的解决了）
    verify_steps: list[RunbookVerifyStep]

    # escalate: 失败后降级策略
    escalate_to_human: bool = True               # verify 失败是否通知人工
    max_auto_retries: int = 0                    # 允许自动重试次数（0=不重试，直接递进 Tier 2）

    # rollback: 回滚步骤（失败时自动执行）
    rollback_steps: list[RunbookStep] = Field(default_factory=list)

    # 元数据
    description: str = ""
    tags: list[str] = Field(default_factory=list)
    created_at: datetime | None = None
    updated_at: datetime | None = None
    created_by: str = ""                         # seed/llm-auto/<user_id>
    hit_count: int = 0                           # 命中次数（统计用）
    success_count: int = 0                       # 命中后处置成功次数

class HypothesisStatus(StrEnum):
    PENDING              = "pending"              # 待验证
    INVESTIGATING        = "investigating"        # 取证中
    CONFIRMED            = "confirmed"            # 已确认（置信度达标）
    REFUTED              = "refuted"              # 已证伪
    INSUFFICIENT_EVIDENCE = "insufficient_evidence" # 证据不足

class Hypothesis(BaseModel):
    """Tier 3 假设驱动调查中的单条根因假设。"""
    id: str
    description: str                             # 假设描述（如"Redis 内存溢出导致服务不可用"）
    target_component: str                        # 涉及组件
    fault_domain: str                            # 故障域（host/container/network/datastore/...）
    prior_rank: int                              # 可能性排序（1=最可能）
    required_evidence_types: list[str]            # 需要哪类证据（log/metric/trace/config/...）
    status: HypothesisStatus = HypothesisStatus.PENDING
    confidence: float = 0.0                      # 置信度 [0, 1]
    evidence_chain: list[str] = Field(default_factory=list)  # evidence IDs
    refutation_reason: str = ""                  # 证伪原因
    verification_rounds: int = 0                 # 本假设已验证的轮次

class Investigation(BaseModel):
    """L3 → L4 的产出。调查层的完整结果。"""
    incident_id: str
    tier_completed: str                          # tier1 / tier2 / tier3
    hypotheses: list[Hypothesis] = Field(default_factory=list)  # Tier 3 时填充
    root_cause: RootCause | None = None
    total_iterations: int = 0
    evidence_ids: list[str] = Field(default_factory=list)
    tier1_result: dict[str, Any] | None = None   # Tier 1 命中时的 Runbook 执行结果
    tier2_result: Tier2Result | None = None       # Tier 2 的验证结果
    inherited_evidence: list[str] = Field(default_factory=list)  # 从前序 Tier 继承的证据

class RootCause(BaseModel):
    """结构化根因。"""
    component: str                               # 故障组件
    failure_mode: str                            # 故障模式
    description: str                             # 人读描述
    evidence_chain: list[str]                    # 支撑证据 ID 列表
    confidence: float                            # 置信度
    hypothesis_id: str = ""                      # 关联的假设 ID（Tier 3）

Investigation (Tier 3 产出)
   │
   ▼
LLM 提炼 Prompt:
  输入: Investigation.hypotheses[confirmed] + evidence_chain + remediation_result
  输出 schema: Runbook (§3.1 的 Pydantic model)
   │
   ▼
Runbook 草稿 (source="auto", status="pending_review")
   │
   ▼
审核队列:
  - 人工审核 (飞书通知 + 管理后台)
  - OR 自动化回归: 用同类历史告警回放，检查 Runbook 命中率 + 处置成功率
   │
   ▼
审核通过 → status="active", 入 Runbook Registry