feat(recommender): nightly ALS with offline metrics and CF candidates in the wave (M6)

- Implicit scores from play events and reactions per the signals table
  (90-day half-life, weaker skips outside the wave, dislikes override).
- numpy ALS for implicit feedback; time-split evaluation on the last
  7 days (NDCG@10, Recall@50) against a popularity baseline; a new model
  is activated only if NDCG@10 does not regress by more than 2%.
- Vectors are written to track_embeddings.cf and user_profiles_rec.taste_cf,
  versions to model_versions; NightlyTrainer runs at TRAIN_HOUR and on
  first start, `pipu-rec train` runs it manually.
- Wave: cf and item_cf candidate sources and a 0.10 cf term in the v1
  score; impressions are tagged heuristic-v1+cf.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

services/recommender/pipu_rec/train/__init__.py

@@ -0,0 +1 @@
+"""Ночное обучение: неявные оценки, ALS, офлайн-метрики (docs/recommendations.md#8)."""

services/recommender/pipu_rec/train/als.py

@@ -0,0 +1,111 @@
+"""ALS для неявной обратной связи (Hu, Koren, Volinsky, 2008) на numpy.
+
+На масштабе проекта (~100 пользователей × десятки тысяч треков) решение каждой строки — система 64×64:
+весь проход — секунды, без компилируемых зависимостей. Положительная оценка — предпочтение 1 с уверенностью
+c = 1 + α·score; отрицательная — предпочтение 0 с повышенной уверенностью (модель учит «не это»).
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+import numpy as np
+from numpy.typing import NDArray
+
+F32 = NDArray[np.float32]
+
+
+@dataclass
+class ALSModel:
+    users: list[str]
+    items: list[str]
+    user_factors: F32  # (n_users, k)
+    item_factors: F32  # (n_items, k)
+
+    def user_vec(self, user: str) -> F32 | None:
+        try:
+            i = self.users.index(user)
+        except ValueError:
+            return None
+        vec: F32 = self.user_factors[i]
+        return vec
+
+    def scores(self, user: str) -> NDArray[np.float64]:
+        u = self.user_vec(user)
+        if u is None:
+            return np.zeros(len(self.items))
+        return (self.item_factors @ u).astype(np.float64)
+
+
+def _solve_side(
+    fixed: F32,
+    rows: list[tuple[NDArray[np.intp], NDArray[np.float64], NDArray[np.float64]]],
+    reg: float,
+) -> F32:
+    """Пересчёт одной стороны: для каждой строки (индексы, уверенность, предпочтение) — закрытое решение
+    (YᵀY + Yᵀ(C−I)Y + λI) x = YᵀCp, где YᵀY общий для всех строк."""
+    k = fixed.shape[1]
+    yty = fixed.T @ fixed
+    out = np.zeros((len(rows), k), dtype=np.float32)
+    eye = reg * np.eye(k)
+    for r, (idx, conf, pref) in enumerate(rows):
+        if len(idx) == 0:
+            continue
+        y = fixed[idx].astype(np.float64)
+        a = yty + (y.T * (conf - 1.0)) @ y + eye
+        b = (y.T * conf) @ pref
+        out[r] = np.linalg.solve(a, b)
+    return out
+
+
+def train_als(
+    scores: dict[str, dict[str, float]],
+    factors: int = 64,
+    reg: float = 0.1,
+    alpha: float = 20.0,
+    iterations: int = 12,
+    seed: int = 0,
+) -> ALSModel | None:
+    """Обучает ALS по implicit_score. None — данных нет."""
+    users = sorted(u for u, row in scores.items() if row)
+    items = sorted({t for row in scores.values() for t in row})
+    if not users or not items:
+        return None
+    upos = {u: i for i, u in enumerate(users)}
+    ipos = {t: i for i, t in enumerate(items)}
+    by_user: list[list[tuple[int, float]]] = [[] for _ in users]
+    by_item: list[list[tuple[int, float]]] = [[] for _ in items]
+    for u, row in scores.items():
+        if u not in upos:
+            continue
+        for t, s in row.items():
+            by_user[upos[u]].append((ipos[t], s))
+            by_item[ipos[t]].append((upos[u], s))
+
+    def prep(
+        lists: list[list[tuple[int, float]]],
+    ) -> list[tuple[NDArray[np.intp], NDArray[np.float64], NDArray[np.float64]]]:
+        out = []
+        for lst in lists:
+            idx = np.array([i for i, _ in lst], dtype=np.intp)
+            s = np.array([v for _, v in lst], dtype=np.float64)
+            conf = 1.0 + alpha * np.abs(s) * np.where(s > 0, 1.0, 0.5)
+            pref = (s > 0).astype(np.float64)
+            out.append((idx, conf, pref))
+        return out
+
+    u_rows, i_rows = prep(by_user), prep(by_item)
+    k = min(factors, max(2, len(items) - 1))
+    rng = np.random.default_rng(seed)
+    x = (rng.standard_normal((len(users), k)) * 0.01).astype(np.float32)
+    y = (rng.standard_normal((len(items), k)) * 0.01).astype(np.float32)
+    for _ in range(iterations):
+        x = _solve_side(y, u_rows, reg)
+        y = _solve_side(x, i_rows, reg)
+    return ALSModel(users=users, items=items, user_factors=x, item_factors=y)
+
+
+def normalize(v: F32) -> F32:
+    """L2-нормировка строк (косинусное сходство в волне)."""
+    n = np.linalg.norm(v, axis=-1, keepdims=True)
+    return (v / np.where(n > 0, n, 1.0)).astype(np.float32)

services/recommender/pipu_rec/train/metrics.py

@@ -0,0 +1,39 @@
+"""Офлайн-метрики на отложенной по времени выборке: NDCG@10, Recall@50 (docs/recommendations.md#1)."""
+
+from __future__ import annotations
+
+import math
+from collections.abc import Callable, Iterable
+
+
+def ndcg_at_k(ranked: list[str], relevant: set[str], k: int = 10) -> float:
+    dcg = sum(1.0 / math.log2(i + 2) for i, t in enumerate(ranked[:k]) if t in relevant)
+    ideal = sum(1.0 / math.log2(i + 2) for i in range(min(k, len(relevant))))
+    return dcg / ideal if ideal > 0 else 0.0
+
+
+def recall_at_k(ranked: list[str], relevant: set[str], k: int = 50) -> float:
+    if not relevant:
+        return 0.0
+    return len(set(ranked[:k]) & relevant) / len(relevant)
+
+
+def evaluate(
+    users: Iterable[str],
+    relevant: dict[str, set[str]],
+    rank: Callable[[str], list[str]],
+) -> dict[str, float]:
+    """Средние метрики по пользователям, у которых в тесте есть хотя бы один релевантный трек."""
+    n = 0
+    ndcg = rec = 0.0
+    for u in users:
+        rel = relevant.get(u)
+        if not rel:
+            continue
+        ranked = rank(u)
+        ndcg += ndcg_at_k(ranked, rel, 10)
+        rec += recall_at_k(ranked, rel, 50)
+        n += 1
+    if n == 0:
+        return {"users": 0, "ndcg@10": 0.0, "recall@50": 0.0}
+    return {"users": n, "ndcg@10": round(ndcg / n, 4), "recall@50": round(rec / n, 4)}

services/recommender/pipu_rec/train/pipeline.py

@@ -0,0 +1,220 @@
+"""Ночной пайплайн (docs/recommendations.md#8): события → implicit_score → ALS → проверка метрик →
+векторы в track_embeddings.cf / user_profiles_rec.taste_cf и запись в model_versions."""
+
+from __future__ import annotations
+
+import datetime as dt
+import json
+import logging
+import threading
+import time
+from collections import Counter
+from typing import Any
+
+import numpy as np
+import psycopg
+
+from pipu_rec.train.als import ALSModel, normalize, train_als
+from pipu_rec.train.metrics import evaluate
+from pipu_rec.train.signals import DISLIKE, LIKE, Interaction, event_weight, implicit_scores
+
+log = logging.getLogger(__name__)
+
+HOLDOUT_DAYS = 7
+# Новая модель активируется, только если NDCG@10 не хуже активной более чем на 2%.
+MAX_REGRESSION = 0.02
+MIN_INTERACTIONS = 50
+ADVISORY_LOCK = 0x70697075  # «pipu»: один прогон обучения на кластер
+
+
+def load_interactions(conn: psycopg.Connection[Any]) -> list[Interaction]:
+    out: list[Interaction] = []
+    for user, track, ts, kind, played, duration, in_wave in conn.execute(
+        """
+        SELECT e.user_id::text, e.track_id::text, extract(epoch FROM e.occurred_at)::float8, e.type,
+            e.played_ms, t.duration_ms, (e.wave_session_id IS NOT NULL OR e.context_type = 'wave')
+        FROM play_events e JOIN tracks t ON t.id = e.track_id
+        WHERE e.type IN ('play_end', 'skip') AND e.occurred_at > now() - interval '365 days'
+        """
+    ):
+        w = event_weight(kind, played, duration, bool(in_wave))
+        if w is not None:
+            out.append(Interaction(user, track, float(ts), w))
+    for user, track, ts, reaction in conn.execute(
+        "SELECT user_id::text, track_id::text, extract(epoch FROM created_at)::float8, reaction::text "
+        "FROM track_reactions"
+    ):
+        out.append(Interaction(user, track, float(ts), LIKE if reaction == "like" else DISLIKE))
+    return out
+
+
+def holdout(
+    interactions: list[Interaction], now: float, days: int = HOLDOUT_DAYS
+) -> tuple[dict[str, dict[str, float]], dict[str, set[str]]]:
+    """Разбиение по времени: обучение — до отсечки, тест — треки, впервые понравившиеся после неё."""
+    cutoff = now - days * 86400
+    train = implicit_scores(interactions, now, before=cutoff)
+    after: dict[str, dict[str, float]] = {}
+    for it in interactions:
+        if it.ts >= cutoff:
+            row = after.setdefault(it.user, {})
+            row[it.track] = row.get(it.track, 0.0) + it.weight
+    relevant = {u: {t for t, w in c.items() if w > 0 and t not in train.get(u, {})} for u, c in after.items()}
+    return train, {u: r for u, r in relevant.items() if r}
+
+
+def offline_metrics(interactions: list[Interaction], now: float) -> dict[str, Any]:
+    """ALS против популярности на отложенной неделе."""
+    train, relevant = holdout(interactions, now)
+    model = train_als(train)
+    if model is None or not relevant:
+        return {"users": 0}
+    pop = Counter(t for row in train.values() for t, s in row.items() if s > 0)
+    popular = [t for t, _ in pop.most_common()]
+
+    def rank_als(u: str) -> list[str]:
+        seen = train.get(u, {})
+        order = np.argsort(-model.scores(u))
+        return [model.items[i] for i in order if model.items[i] not in seen][:50]
+
+    def rank_pop(u: str) -> list[str]:
+        seen = train.get(u, {})
+        return [t for t in popular if t not in seen][:50]
+
+    users = list(relevant)
+    return {"als": evaluate(users, relevant, rank_als), "popularity": evaluate(users, relevant, rank_pop)}
+
+
+def vec_text(v: np.ndarray) -> str:
+    return "[" + ",".join(f"{x:.6f}" for x in v.tolist()) + "]"
+
+
+def active_metrics(conn: psycopg.Connection[Any], kind: str) -> dict[str, Any] | None:
+    row = conn.execute(
+        "SELECT metrics FROM model_versions WHERE kind = %s AND is_active ORDER BY created_at DESC LIMIT 1",
+        (kind,),
+    ).fetchone()
+    return row[0] if row and row[0] else None
+
+
+def should_activate(new: dict[str, Any], current: dict[str, Any] | None) -> bool:
+    """Без активной модели или без тестовых данных — активируем; иначе NDCG@10 не хуже на > 2%."""
+    if current is None:
+        return True
+    new_ndcg = new.get("als", {}).get("ndcg@10")
+    cur_ndcg = (current.get("offline") or {}).get("als", {}).get("ndcg@10")
+    if new_ndcg is None or not cur_ndcg:
+        return True
+    return float(new_ndcg) >= float(cur_ndcg) * (1 - MAX_REGRESSION)
+
+
+def save_als(conn: psycopg.Connection[Any], model: ALSModel, version: str, metrics: dict[str, Any]) -> None:
+    items = normalize(model.item_factors)
+    users = normalize(model.user_factors)
+    with conn.transaction():
+        with conn.cursor() as cur:
+            cur.executemany(
+                """
+                INSERT INTO track_embeddings (track_id, cf, updated_at) VALUES (%s, %s::vector, now())
+                ON CONFLICT (track_id) DO UPDATE SET cf = EXCLUDED.cf, updated_at = now()
+                """,
+                [(t, vec_text(items[i])) for i, t in enumerate(model.items)],
+            )
+            cur.execute(
+                "UPDATE track_embeddings SET cf = NULL "
+                "WHERE cf IS NOT NULL AND NOT (track_id::text = ANY(%s))",
+                (model.items,),
+            )
+            cur.executemany(
+                """
+                INSERT INTO user_profiles_rec (user_id, taste_cf, updated_at) VALUES (%s, %s::vector, now())
+                ON CONFLICT (user_id) DO UPDATE SET taste_cf = EXCLUDED.taste_cf, updated_at = now()
+                """,
+                [(u, vec_text(users[i])) for i, u in enumerate(model.users)],
+            )
+        conn.execute("UPDATE model_versions SET is_active = false WHERE kind = 'als' AND is_active")
+        conn.execute(
+            "INSERT INTO model_versions (id, kind, metrics, is_active) VALUES (%s, 'als', %s, true)",
+            (version, json.dumps(metrics)),
+        )
+
+
+def train_all(dsn: str, now: float | None = None) -> dict[str, Any]:
+    """Один прогон обучения. Возвращает сводку (её же пишет в лог и model_versions)."""
+    now = now or time.time()
+    with psycopg.connect(dsn) as conn:
+        if not conn.execute("SELECT pg_try_advisory_lock(%s)", (ADVISORY_LOCK,)).fetchone()[0]:  # type: ignore[index]
+            return {"skipped": "another training is running"}
+        try:
+            interactions = load_interactions(conn)
+            if len(interactions) < MIN_INTERACTIONS:
+                return {"skipped": f"only {len(interactions)} interactions"}
+            started = time.monotonic()
+            offline = offline_metrics(interactions, now)
+            model = train_als(implicit_scores(interactions, now))
+            if model is None:
+                return {"skipped": "no data"}
+            version = "als-" + dt.datetime.fromtimestamp(now, dt.UTC).strftime("%Y%m%d%H%M")
+            metrics = {
+                "offline": offline,
+                "users": len(model.users),
+                "items": len(model.items),
+                "interactions": len(interactions),
+                "train_s": round(time.monotonic() - started, 1),
+            }
+            if not should_activate(offline, active_metrics(conn, "als")):
+                conn.execute(
+                    "INSERT INTO model_versions (id, kind, metrics, is_active) VALUES (%s, 'als', %s, false) "
+                    "ON CONFLICT (id) DO NOTHING",
+                    (version, json.dumps(metrics)),
+                )
+                log.warning("als %s not activated: offline metrics regressed %s", version, offline)
+                return {"version": version, "activated": False, **metrics}
+            save_als(conn, model, version, metrics)
+            log.info("als %s activated: %s", version, metrics)
+            return {"version": version, "activated": True, **metrics}
+        finally:
+            conn.execute("SELECT pg_advisory_unlock(%s)", (ADVISORY_LOCK,))
+
+
+class NightlyTrainer:
+    """Обучение раз в сутки в hour:00 по местному времени сервера; при первом запуске без модели — сразу.
+    on_done вызывается после успешной активации (горячая подмена каталога в памяти)."""
+
+    def __init__(self, dsn: str, hour: int = 3, on_done: Any = None) -> None:
+        self.dsn = dsn
+        self.hour = hour
+        self.on_done = on_done
+        self._stop = threading.Event()
+
+    def start(self) -> None:
+        threading.Thread(target=self._loop, name="nightly-trainer", daemon=True).start()
+
+    def stop(self) -> None:
+        self._stop.set()
+
+    def seconds_until_next(self, now: dt.datetime) -> float:
+        target = now.replace(hour=self.hour, minute=0, second=0, microsecond=0)
+        if target <= now:
+            target += dt.timedelta(days=1)
+        return (target - now).total_seconds()
+
+    def _run(self) -> None:
+        try:
+            result = train_all(self.dsn)
+            log.info("training finished: %s", result)
+            if result.get("activated") and self.on_done:
+                self.on_done()
+        except Exception:
+            log.exception("training failed")
+
+    def _loop(self) -> None:
+        try:
+            with psycopg.connect(self.dsn) as conn:
+                never = active_metrics(conn, "als") is None
+        except Exception:
+            never = False
+        if never:
+            self._run()
+        while not self._stop.wait(self.seconds_until_next(dt.datetime.now())):
+            self._run()

services/recommender/pipu_rec/train/signals.py

@@ -0,0 +1,64 @@
+"""Неявная оценка пары (пользователь, трек) из событий — таблица docs/recommendations.md#3."""
+
+from __future__ import annotations
+
+import math
+from collections import defaultdict
+from dataclasses import dataclass
+
+HALF_LIFE_DAYS = 90.0
+DAY_S = 86400.0
+
+LIKE = 3.0
+DISLIKE = -5.0
+
+
+@dataclass(frozen=True)
+class Interaction:
+    user: str
+    track: str
+    ts: float  # unix-время события
+    weight: float  # вклад до затухания
+
+
+def event_weight(kind: str, played_ms: int | None, duration_ms: int | None, in_wave: bool) -> float | None:
+    """Вклад одного прослушивания. Скип вне волны слабее (×0.5): человек мог просто листать альбом."""
+    played = played_ms or 0
+    if kind == "play_end":
+        return 1.0
+    if kind != "skip":
+        return None
+    if duration_ms and played >= 0.5 * duration_ms:
+        return 0.7  # засчитанное прослушивание без доигрывания
+    if played < 10_000:
+        w = -1.0
+    elif played < 30_000:
+        w = -0.7
+    else:
+        w = -0.2
+    return w if in_wave else w * 0.5
+
+
+def decay(ts: float, now: float) -> float:
+    """Экспоненциальное затухание с полупериодом 90 дней."""
+    return math.exp(-math.log(2) * max(now - ts, 0.0) / DAY_S / HALF_LIFE_DAYS)
+
+
+def implicit_scores(
+    interactions: list[Interaction], now: float, before: float | None = None
+) -> dict[str, dict[str, float]]:
+    """user → track → implicit_score; before — учитывать только события раньше этого момента.
+    Дизлайк перекрывает всё: оценка трека становится DISLIKE (исключение из рекомендаций)."""
+    out: dict[str, dict[str, float]] = defaultdict(dict)
+    disliked: set[tuple[str, str]] = set()
+    for it in interactions:
+        if before is not None and it.ts >= before:
+            continue
+        if it.weight <= DISLIKE:
+            disliked.add((it.user, it.track))
+            continue
+        row = out[it.user]
+        row[it.track] = row.get(it.track, 0.0) + it.weight * decay(it.ts, now)
+    for user, track in disliked:
+        out[user][track] = DISLIKE
+    return dict(out)