feat(rec): audio analyzer and My Wave v1 engine (M4)
- migration 00006: track features and embeddings (pgvector, HNSW), analysis queue filled by a trigger on new tracks, wave sessions and impressions, recommender profiles, model versions - analyzer basic-1 (ADR-014): ffmpeg decode, three 30 s excerpts, 256-d timbre/rhythm embedding, tempo by onset autocorrelation, key, loudness, energy, danceability, acousticness, intro length; worker with SKIP LOCKED claims, retries and re-analysis on version change - wave engine v1: candidates (session/taste similarity, fresh, liked artists, rediscover, popular, exploration, seed), filters (dislikes, played in 3 h, already served), heuristic score, activity and mood fit from contexts.yaml with relaxing thresholds, MMR with one track per artist, exploration share, session vector updated by reactions and a change of course after three skips - POST /internal/wave/batch with the catalog index cached in memory and the session kept in Redis Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
976ffb574b
commit
ef29b85320
20 files changed
+1530
-13
No files matched your search
Whitespace-only changes.
@@ -0,0 +1,73 @@
|
||||
"""Профили занятий и настроений: соответствие трека контексту (context_fit)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from importlib import resources
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
import yaml
|
||||
from numpy.typing import NDArray
|
||||
from pydantic import BaseModel, model_validator
|
||||
|
||||
FEATURES = ("bpm", "energy", "danceability", "valence", "acousticness")
|
||||
|
||||
|
||||
class Rule(BaseModel):
|
||||
feature: str
|
||||
min: float | None = None
|
||||
max: float | None = None
|
||||
sigma: float
|
||||
half_time: bool = False
|
||||
|
||||
@model_validator(mode="after")
|
||||
def check(self) -> Rule:
|
||||
if self.feature not in FEATURES:
|
||||
raise ValueError(f"unknown feature {self.feature!r}; known: {', '.join(FEATURES)}")
|
||||
if self.min is None and self.max is None:
|
||||
raise ValueError("rule needs min or max")
|
||||
if self.sigma <= 0:
|
||||
raise ValueError("sigma must be positive")
|
||||
return self
|
||||
|
||||
|
||||
class Context(BaseModel):
|
||||
icon: str
|
||||
rules: list[Rule]
|
||||
|
||||
|
||||
class Contexts(BaseModel):
|
||||
activities: dict[str, Context]
|
||||
moods: dict[str, Context]
|
||||
|
||||
|
||||
def load_contexts(path: str = "") -> Contexts:
|
||||
"""contexts.yaml из пути или встроенный; ошибка в файле — ошибка запуска (Pydantic)."""
|
||||
raw = (
|
||||
Path(path).read_text("utf-8")
|
||||
if path
|
||||
else resources.files(__package__).joinpath("contexts.yaml").read_text("utf-8")
|
||||
)
|
||||
return Contexts.model_validate(yaml.safe_load(raw))
|
||||
|
||||
|
||||
def _bell(x: NDArray[np.float64], rule: Rule) -> NDArray[np.float64]:
|
||||
lo = -np.inf if rule.min is None else rule.min
|
||||
hi = np.inf if rule.max is None else rule.max
|
||||
dist = np.where(x < lo, lo - x, np.where(x > hi, x - hi, 0.0))
|
||||
return np.asarray(np.exp(-0.5 * (dist / rule.sigma) ** 2))
|
||||
|
||||
|
||||
def fit(features: dict[str, NDArray[np.float64]], rules: list[Rule]) -> NDArray[np.float64]:
|
||||
"""Произведение «колоколов» по правилам; NaN (признака нет) — нейтрально (1).
|
||||
half_time: для темпа засчитывается и половинный (бег под 85 BPM — тот же шаг, что 170)."""
|
||||
n = len(next(iter(features.values())))
|
||||
out = np.ones(n)
|
||||
for rule in rules:
|
||||
x = features[rule.feature]
|
||||
known = ~np.isnan(x)
|
||||
b = _bell(np.nan_to_num(x), rule)
|
||||
if rule.half_time:
|
||||
b = np.maximum(b, _bell(np.nan_to_num(x) * 2, rule))
|
||||
out *= np.where(known, b, 1.0)
|
||||
return out
|
||||
@@ -0,0 +1,56 @@
|
||||
# Занятия и настроения «Моей волны» (docs/recommendations.md#64-занятия-и-настроения).
|
||||
# Каждое правило — «колокол» по признаку трека: внутри [min, max] — 1, снаружи спадает по Гауссу с sigma.
|
||||
# Признак, которого у трека нет (ещё не проанализирован), не штрафует.
|
||||
# Изменения применяются после перезапуска recommender, пересборка образа не нужна (CONTEXTS_PATH).
|
||||
activities:
|
||||
running:
|
||||
icon: directions_run
|
||||
rules:
|
||||
- {feature: bpm, min: 150, max: 185, sigma: 12, half_time: true}
|
||||
- {feature: energy, min: 0.7, sigma: 0.12}
|
||||
- {feature: danceability, min: 0.5, sigma: 0.2}
|
||||
workout:
|
||||
icon: fitness_center
|
||||
rules:
|
||||
- {feature: energy, min: 0.75, sigma: 0.1}
|
||||
- {feature: bpm, min: 110, max: 180, sigma: 20, half_time: true}
|
||||
waking_up:
|
||||
icon: wb_sunny
|
||||
rules:
|
||||
- {feature: energy, min: 0.3, max: 0.7, sigma: 0.12}
|
||||
- {feature: valence, min: 0.5, sigma: 0.15}
|
||||
focus:
|
||||
icon: headphones
|
||||
rules:
|
||||
- {feature: energy, min: 0.3, max: 0.6, sigma: 0.12}
|
||||
- {feature: acousticness, min: 0.3, sigma: 0.25}
|
||||
commute:
|
||||
icon: directions_car
|
||||
rules:
|
||||
- {feature: energy, min: 0.4, max: 0.8, sigma: 0.15}
|
||||
chill:
|
||||
icon: weekend
|
||||
rules:
|
||||
- {feature: energy, max: 0.5, sigma: 0.12}
|
||||
sleep:
|
||||
icon: bedtime
|
||||
rules:
|
||||
- {feature: energy, max: 0.35, sigma: 0.1}
|
||||
- {feature: bpm, max: 95, sigma: 12}
|
||||
moods:
|
||||
energetic:
|
||||
icon: bolt
|
||||
rules:
|
||||
- {feature: energy, min: 0.7, sigma: 0.12}
|
||||
happy:
|
||||
icon: sentiment_satisfied
|
||||
rules:
|
||||
- {feature: valence, min: 0.6, sigma: 0.12}
|
||||
calm:
|
||||
icon: spa
|
||||
rules:
|
||||
- {feature: energy, max: 0.45, sigma: 0.12}
|
||||
sad:
|
||||
icon: rainy
|
||||
rules:
|
||||
- {feature: valence, max: 0.4, sigma: 0.12}
|
||||
@@ -0,0 +1,306 @@
|
||||
"""Сборка пачки «Моей волны» (v1: эвристика). Чистые функции над массивами — без БД и сети."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
import numpy as np
|
||||
from numpy.typing import NDArray
|
||||
|
||||
from pipu_rec.wave.contexts import FEATURES, Rule, fit
|
||||
|
||||
MODEL_VER = "heuristic-v1"
|
||||
|
||||
Vec = NDArray[np.float32]
|
||||
|
||||
|
||||
@dataclass
|
||||
class CatalogIndex:
|
||||
"""Каталог в памяти: строка i — трек ids[i]."""
|
||||
|
||||
ids: list[str]
|
||||
artist: list[str] # основной исполнитель
|
||||
emb: Vec # (n, d), стандартизирован и нормирован; нулевой — нет анализа
|
||||
has_emb: NDArray[np.bool_]
|
||||
features: dict[str, NDArray[np.float64]] # NaN — нет признака
|
||||
popularity: NDArray[np.float64] # 0..1
|
||||
pos: dict[str, int] = field(default_factory=dict)
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
self.pos = {t: i for i, t in enumerate(self.ids)}
|
||||
|
||||
def __len__(self) -> int:
|
||||
return len(self.ids)
|
||||
|
||||
@staticmethod
|
||||
def build(
|
||||
ids: list[str],
|
||||
artist: list[str],
|
||||
raw_emb: list[Vec | None],
|
||||
features: dict[str, list[float | None]],
|
||||
plays: list[int],
|
||||
dim: int = 256,
|
||||
) -> CatalogIndex:
|
||||
n = len(ids)
|
||||
emb = np.zeros((n, dim), dtype=np.float32)
|
||||
has = np.zeros(n, dtype=bool)
|
||||
for i, e in enumerate(raw_emb):
|
||||
if e is not None:
|
||||
emb[i], has[i] = e, True
|
||||
if has.sum() >= 2:
|
||||
# Стандартизация по библиотеке: косинус сравнивает отличия треков, а не общий «звук библиотеки».
|
||||
mu = emb[has].mean(axis=0)
|
||||
sd = emb[has].std(axis=0) + 1e-6
|
||||
emb[has] = (emb[has] - mu) / sd
|
||||
emb[has] /= np.linalg.norm(emb[has], axis=1, keepdims=True) + 1e-9
|
||||
feats = {
|
||||
k: np.array([np.nan if v is None else v for v in features.get(k, [None] * n)], dtype=np.float64)
|
||||
for k in FEATURES
|
||||
}
|
||||
p = np.log1p(np.array(plays, dtype=np.float64))
|
||||
pop = p / p.max() if n and p.max() > 0 else np.zeros(n)
|
||||
return CatalogIndex(ids=ids, artist=artist, emb=emb, has_emb=has, features=feats, popularity=pop)
|
||||
|
||||
|
||||
@dataclass
|
||||
class Profile:
|
||||
"""Вкус пользователя: оценки треков и производные."""
|
||||
|
||||
scores: dict[str, float] # implicit_score > 0 — нравится
|
||||
disliked: set[str]
|
||||
disliked_artists: set[str]
|
||||
recent: set[str] # сыграны за последние 3 часа
|
||||
played: set[str] # слышал когда-либо
|
||||
liked_recent: set[str] # лайкнутые, игравшие за 24 ч — штраф
|
||||
rediscover: set[str] # любимые, не игравшие > 60 дней
|
||||
exploration: float = 0.15
|
||||
|
||||
|
||||
@dataclass
|
||||
class Session:
|
||||
"""Состояние сессии волны (Redis)."""
|
||||
|
||||
vec: list[float] | None = None
|
||||
served: list[str] = field(default_factory=list)
|
||||
recent_artists: list[str] = field(default_factory=list)
|
||||
skips_in_row: int = 0
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Pick:
|
||||
id: str
|
||||
source: str
|
||||
score: float
|
||||
|
||||
|
||||
# Вклад реакции в сессионный вектор: s ← normalize(s + η·r·e).
|
||||
REACTION_WEIGHT = {"complete": 1.0, "like": 2.0, "skip_early": -1.0, "dislike": -2.0, "skip": -0.5}
|
||||
ETA = 0.3
|
||||
|
||||
|
||||
def taste_centroids(index: CatalogIndex, profile: Profile, max_k: int = 6, seed: int = 0) -> Vec:
|
||||
"""Центроиды вкуса: k-means по эмбеддингам треков с положительной оценкой (веса = оценка).
|
||||
k растёт с историей: 1 центроид на ~8 любимых треков, не больше max_k."""
|
||||
rows = [(index.pos[t], s) for t, s in profile.scores.items() if s > 0 and t in index.pos]
|
||||
rows = [(i, s) for i, s in rows if index.has_emb[i]]
|
||||
if not rows:
|
||||
return np.zeros((0, index.emb.shape[1]), dtype=np.float32)
|
||||
x = index.emb[[i for i, _ in rows]]
|
||||
w = np.array([s for _, s in rows])
|
||||
k = int(min(max_k, max(1, len(rows) // 8)))
|
||||
rng = np.random.default_rng(seed)
|
||||
centers = x[rng.choice(len(x), size=k, replace=False)]
|
||||
for _ in range(10):
|
||||
assign = np.argmax(x @ centers.T, axis=1)
|
||||
for c in range(k):
|
||||
m = assign == c
|
||||
if m.any():
|
||||
v = (x[m] * w[m, None]).sum(axis=0)
|
||||
centers[c] = v / (np.linalg.norm(v) + 1e-9)
|
||||
return centers.astype(np.float32)
|
||||
|
||||
|
||||
def update_session(
|
||||
index: CatalogIndex, session: Session, feedback: list[tuple[str, str]], centroids: Vec
|
||||
) -> None:
|
||||
"""Обновляет сессионный вектор и счётчик скипов по реакциям на выданные треки.
|
||||
Три скипа подряд — «смена курса»: вектор переводится к наименее похожему центроиду вкуса."""
|
||||
d = index.emb.shape[1]
|
||||
vec = np.array(session.vec, dtype=np.float32) if session.vec else np.zeros(d, dtype=np.float32)
|
||||
for track_id, outcome in feedback:
|
||||
i = index.pos.get(track_id)
|
||||
r = REACTION_WEIGHT.get(outcome)
|
||||
if r is None:
|
||||
continue
|
||||
session.skips_in_row = session.skips_in_row + 1 if outcome in ("skip", "skip_early", "dislike") else 0
|
||||
if i is None or not index.has_emb[i]:
|
||||
continue
|
||||
vec = vec + ETA * r * index.emb[i]
|
||||
norm = float(np.linalg.norm(vec))
|
||||
vec = vec / norm if norm > 0 else vec
|
||||
if session.skips_in_row >= 3 and len(centroids):
|
||||
sims = centroids @ vec if np.any(vec) else np.zeros(len(centroids))
|
||||
vec = centroids[int(np.argmin(sims))].copy()
|
||||
session.skips_in_row = 0
|
||||
session.vec = vec.tolist() if np.any(vec) else None
|
||||
|
||||
|
||||
def _top(scores: NDArray[np.float64], mask: NDArray[np.bool_], k: int) -> NDArray[np.intp]:
|
||||
idx = np.nonzero(mask)[0]
|
||||
if len(idx) <= k:
|
||||
return idx
|
||||
part = idx[np.argpartition(-scores[idx], k)[:k]]
|
||||
return part
|
||||
|
||||
|
||||
def build_batch(
|
||||
index: CatalogIndex,
|
||||
profile: Profile,
|
||||
session: Session,
|
||||
rules: list[Rule],
|
||||
n: int = 5,
|
||||
exclude: set[str] | None = None,
|
||||
seed_ids: list[str] | None = None,
|
||||
rng: np.random.Generator | None = None,
|
||||
) -> list[Pick]:
|
||||
"""Кандидаты → фильтры → оценка → соответствие контексту → MMR с правилами разнообразия."""
|
||||
rng = rng or np.random.default_rng()
|
||||
size = len(index)
|
||||
if size == 0:
|
||||
return []
|
||||
centroids = taste_centroids(index, profile)
|
||||
svec = np.array(session.vec, dtype=np.float32) if session.vec else None
|
||||
|
||||
# 2. Фильтры.
|
||||
excluded = profile.disliked | profile.recent | set(session.served) | (exclude or set())
|
||||
allowed = np.array(
|
||||
[
|
||||
t not in excluded and a not in profile.disliked_artists
|
||||
for t, a in zip(index.ids, index.artist, strict=True)
|
||||
]
|
||||
)
|
||||
if not allowed.any():
|
||||
allowed = np.array([t not in profile.disliked for t in index.ids])
|
||||
if not allowed.any():
|
||||
return []
|
||||
|
||||
# Сходства со вкусом и сессией.
|
||||
sim_taste = (index.emb @ centroids.T).max(axis=1) if len(centroids) else np.zeros(size)
|
||||
sim_session = index.emb @ svec if svec is not None else np.zeros(size)
|
||||
sim_taste = np.where(index.has_emb, sim_taste, 0.0)
|
||||
sim_session = np.where(index.has_emb, sim_session, 0.0)
|
||||
|
||||
# Привязанность к исполнителю — сглаженное среднее оценок его треков.
|
||||
totals: dict[str, list[float]] = {}
|
||||
for t, s in profile.scores.items():
|
||||
i = index.pos.get(t)
|
||||
if i is not None:
|
||||
totals.setdefault(index.artist[i], []).append(s)
|
||||
affinity = {a: sum(v) / (len(v) + 2) for a, v in totals.items()}
|
||||
artist_aff = np.array([np.tanh(affinity.get(a, 0.0) / 3) for a in index.artist])
|
||||
|
||||
# 1. Источники кандидатов.
|
||||
sources: dict[int, str] = {}
|
||||
|
||||
def add(idxs: NDArray[np.intp] | list[int], source: str) -> None:
|
||||
for i in idxs:
|
||||
sources.setdefault(int(i), source)
|
||||
|
||||
if seed_ids:
|
||||
seeds = [index.pos[s] for s in seed_ids if s in index.pos and index.has_emb[index.pos[s]]]
|
||||
if seeds:
|
||||
q = index.emb[seeds].mean(axis=0)
|
||||
add(_top((index.emb @ q).astype(np.float64), allowed & index.has_emb, 100), "seed")
|
||||
query = np.maximum(sim_session, sim_taste) if svec is not None or len(centroids) else None
|
||||
if query is not None:
|
||||
# «Новое» помечаем раньше audio_sim: неслышанный трек рядом со вкусом считается исследованием.
|
||||
never = np.array([t not in profile.played for t in index.ids])
|
||||
add(_top(sim_taste + 0.3 * sim_session, allowed & index.has_emb & never, 50), "fresh")
|
||||
add(_top(query, allowed & index.has_emb, 150), "audio_sim")
|
||||
liked_artists = {a for a, v in affinity.items() if v > 0.3}
|
||||
if liked_artists:
|
||||
by_artist = [int(i) for i in np.nonzero(allowed)[0] if index.artist[i] in liked_artists]
|
||||
add(rng.permutation(by_artist)[:60].tolist() if by_artist else [], "artist")
|
||||
redis_ = [index.pos[t] for t in profile.rediscover if t in index.pos and allowed[index.pos[t]]]
|
||||
add(rng.permutation(redis_)[:30].tolist() if redis_ else [], "rediscover")
|
||||
add(_top(index.popularity + rng.random(size) * 0.3, allowed, 30), "popular")
|
||||
# Холодный старт и исследование: случайные треки из разных частей библиотеки.
|
||||
add(rng.permutation(np.nonzero(allowed)[0])[:40].tolist(), "explore")
|
||||
|
||||
cand = np.array(sorted(sources))
|
||||
# 3. Оценка v1.
|
||||
score = (
|
||||
0.35 * sim_session[cand]
|
||||
+ 0.25 * sim_taste[cand]
|
||||
+ 0.15 * artist_aff[cand]
|
||||
+ 0.05 * index.popularity[cand]
|
||||
+ 0.02 * rng.random(len(cand)) # разбивает ничьи, особенно при холодном старте
|
||||
)
|
||||
liked_recent = np.array([index.ids[i] in profile.liked_recent for i in cand])
|
||||
score = score - 0.2 * liked_recent
|
||||
|
||||
# 4. Соответствие контексту; если отсеялось слишком много — порог ослабляется.
|
||||
if rules:
|
||||
f = fit({k: v[cand] for k, v in index.features.items()}, rules)
|
||||
# Нужен запас для разнообразия, но не больше 50 и не больше четверти пула.
|
||||
enough = min(50, max(3 * n, len(cand) // 4))
|
||||
for threshold in (0.35, 0.15, 0.0):
|
||||
keep = f >= threshold
|
||||
if keep.sum() >= enough or threshold == 0.0:
|
||||
break
|
||||
cand, score, f = cand[keep], score[keep], f[keep]
|
||||
score = score + 0.5 * f # контекст важнее вкуса, когда выбран явно
|
||||
if len(cand) == 0:
|
||||
return []
|
||||
|
||||
# 5. MMR + правила: 1 трек исполнителя на пачку, не больше 2 на последние 10; доля «нового».
|
||||
picks: list[Pick] = []
|
||||
chosen: list[int] = []
|
||||
recent_artists = session.recent_artists[-10:]
|
||||
want_fresh = max(0, round(n * profile.exploration))
|
||||
order = list(range(len(cand)))
|
||||
lam = 0.75
|
||||
while len(picks) < n and order:
|
||||
best_j, best_v = -1, -np.inf
|
||||
for j in order:
|
||||
i = int(cand[j])
|
||||
a = index.artist[i]
|
||||
if any(index.artist[c] == a for c in chosen) or recent_artists.count(a) >= 2:
|
||||
continue
|
||||
redundancy = max((float(index.emb[i] @ index.emb[c]) for c in chosen), default=0.0)
|
||||
v = lam * float(score[j]) - (1 - lam) * redundancy
|
||||
if v > best_v:
|
||||
best_j, best_v = j, v
|
||||
if best_j < 0:
|
||||
# Правила разнообразия не дают набрать пачку — ослабляем их, но не повторяем треки.
|
||||
if recent_artists:
|
||||
recent_artists = []
|
||||
continue
|
||||
if chosen:
|
||||
chosen_artists = {index.artist[c] for c in chosen}
|
||||
rest = [j for j in order if index.artist[int(cand[j])] in chosen_artists]
|
||||
if not rest:
|
||||
break
|
||||
best_j = max(rest, key=lambda j: score[j])
|
||||
else:
|
||||
break
|
||||
i = int(cand[best_j])
|
||||
chosen.append(i)
|
||||
order.remove(best_j)
|
||||
picks.append(Pick(id=index.ids[i], source=sources[i], score=round(float(score[best_j]), 4)))
|
||||
|
||||
# Исследование: «новых» (неслышанных) меньше доли exploration — заменяем ими слабейшие выборы.
|
||||
fresh_left = sorted(
|
||||
(j for j in order if sources[int(cand[j])] == "fresh"), key=lambda j: score[j], reverse=True
|
||||
)
|
||||
while sum(p.source == "fresh" for p in picks) < want_fresh and fresh_left:
|
||||
j = fresh_left.pop(0)
|
||||
i = int(cand[j])
|
||||
others = [k for k, p in enumerate(picks) if p.source != "fresh"]
|
||||
if not others:
|
||||
break
|
||||
weakest = min(others, key=lambda k: picks[k].score)
|
||||
if any(index.artist[index.pos[p.id]] == index.artist[i] for k, p in enumerate(picks) if k != weakest):
|
||||
continue
|
||||
picks[weakest] = Pick(id=index.ids[i], source="fresh", score=round(float(score[j]), 4))
|
||||
return picks
|
||||
@@ -0,0 +1,93 @@
|
||||
"""Сервис «Моей волны»: собирает пачку из каталога, профиля, сессии и контекста."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Literal
|
||||
|
||||
import numpy as np
|
||||
import psycopg
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from pipu_rec.wave.contexts import Contexts, Rule
|
||||
from pipu_rec.wave.engine import MODEL_VER, Pick, build_batch, taste_centroids, update_session
|
||||
from pipu_rec.wave.store import IndexHolder, SessionStore, load_profile, seed_tracks
|
||||
|
||||
MAX_SERVED_MEMORY = 300
|
||||
|
||||
Outcome = Literal["complete", "skip", "skip_early", "like", "dislike"]
|
||||
|
||||
|
||||
class Feedback(BaseModel):
|
||||
track_id: str
|
||||
outcome: Outcome
|
||||
|
||||
|
||||
class Seed(BaseModel):
|
||||
type: Literal["track", "album", "artist"]
|
||||
id: str
|
||||
|
||||
|
||||
class BatchRequest(BaseModel):
|
||||
user_id: str
|
||||
session_id: str
|
||||
activity: str | None = None
|
||||
mood: str | None = None
|
||||
seed: Seed | None = None
|
||||
exclude_ids: list[str] = Field(default_factory=list, max_length=1000)
|
||||
feedback: list[Feedback] = Field(default_factory=list, max_length=200)
|
||||
n: int = Field(default=5, ge=1, le=20)
|
||||
|
||||
|
||||
class BatchTrack(BaseModel):
|
||||
id: str
|
||||
source: str
|
||||
score: float
|
||||
|
||||
|
||||
class BatchResponse(BaseModel):
|
||||
tracks: list[BatchTrack]
|
||||
model_ver: str
|
||||
|
||||
|
||||
class WaveService:
|
||||
def __init__(self, dsn: str, redis_url: str, contexts: Contexts) -> None:
|
||||
self.dsn = dsn
|
||||
self.index = IndexHolder(dsn)
|
||||
self.sessions = SessionStore(redis_url)
|
||||
self.contexts = contexts
|
||||
|
||||
def rules(self, activity: str | None, mood: str | None) -> list[Rule]:
|
||||
"""Занятие и настроение комбинируются — пересечение профилей; неизвестный ключ игнорируется."""
|
||||
rules: list[Rule] = []
|
||||
if activity and activity in self.contexts.activities:
|
||||
rules += self.contexts.activities[activity].rules
|
||||
if mood and mood in self.contexts.moods:
|
||||
rules += self.contexts.moods[mood].rules
|
||||
return rules
|
||||
|
||||
def batch(self, req: BatchRequest) -> BatchResponse:
|
||||
index = self.index.get()
|
||||
with psycopg.connect(self.dsn) as conn:
|
||||
profile = load_profile(conn, req.user_id)
|
||||
seeds = seed_tracks(conn, req.seed.type, req.seed.id) if req.seed else None
|
||||
session = self.sessions.get(req.session_id)
|
||||
update_session(
|
||||
index, session, [(f.track_id, f.outcome) for f in req.feedback], taste_centroids(index, profile)
|
||||
)
|
||||
picks: list[Pick] = build_batch(
|
||||
index,
|
||||
profile,
|
||||
session,
|
||||
self.rules(req.activity, req.mood),
|
||||
n=req.n,
|
||||
exclude=set(req.exclude_ids),
|
||||
seed_ids=seeds,
|
||||
rng=np.random.default_rng(),
|
||||
)
|
||||
session.served = (session.served + [p.id for p in picks])[-MAX_SERVED_MEMORY:]
|
||||
artists = [index.artist[index.pos[p.id]] for p in picks]
|
||||
session.recent_artists = (session.recent_artists + artists)[-10:]
|
||||
self.sessions.put(req.session_id, session)
|
||||
return BatchResponse(
|
||||
tracks=[BatchTrack(id=p.id, source=p.source, score=p.score) for p in picks], model_ver=MODEL_VER
|
||||
)
|
||||
@@ -0,0 +1,194 @@
|
||||
"""Загрузка каталога, профиля и сессии волны из PostgreSQL / Redis."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import math
|
||||
import threading
|
||||
import time
|
||||
from dataclasses import asdict
|
||||
from typing import Any, cast
|
||||
|
||||
import numpy as np
|
||||
import psycopg
|
||||
import redis
|
||||
|
||||
from pipu_rec.wave.engine import CatalogIndex, Profile, Session
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
INDEX_TTL_S = 300
|
||||
SESSION_TTL_S = 12 * 3600
|
||||
HALF_LIFE_DAYS = 90.0
|
||||
|
||||
|
||||
def parse_vector(text: str | None) -> np.ndarray | None:
|
||||
"""pgvector в текстовом виде «[0.1,0.2,...]»."""
|
||||
if not text:
|
||||
return None
|
||||
return np.array(text.strip("[]").split(","), dtype=np.float32)
|
||||
|
||||
|
||||
def load_index(conn: psycopg.Connection[Any]) -> CatalogIndex:
|
||||
"""Доступные треки: основной файл в подключённом хранилище и на месте."""
|
||||
rows = conn.execute(
|
||||
"""
|
||||
SELECT t.id::text,
|
||||
COALESCE((SELECT ta.artist_id::text FROM track_artists ta
|
||||
WHERE ta.track_id = t.id AND ta.role = 'main' ORDER BY ta.position LIMIT 1),
|
||||
al.album_artist_id::text),
|
||||
e.audio::text, f.bpm, f.energy, f.danceability, f.valence, f.acousticness,
|
||||
COALESCE((SELECT sum(play_count) FROM user_track_stats s WHERE s.track_id = t.id), 0)
|
||||
FROM tracks t
|
||||
JOIN albums al ON al.id = t.album_id
|
||||
JOIN media_files mf ON mf.id = t.primary_file_id AND mf.missing_since IS NULL
|
||||
JOIN library_sources src ON src.id = mf.source_id AND src.status = 'mounted'
|
||||
LEFT JOIN track_embeddings e ON e.track_id = t.id
|
||||
LEFT JOIN track_features f ON f.track_id = t.id
|
||||
"""
|
||||
).fetchall()
|
||||
feats: dict[str, list[float | None]] = {
|
||||
k: [] for k in ("bpm", "energy", "danceability", "valence", "acousticness")
|
||||
}
|
||||
for r in rows:
|
||||
for k, v in zip(feats, r[3:8], strict=True):
|
||||
feats[k].append(None if v is None else float(v))
|
||||
return CatalogIndex.build(
|
||||
ids=[r[0] for r in rows],
|
||||
artist=[r[1] for r in rows],
|
||||
raw_emb=[parse_vector(r[2]) for r in rows],
|
||||
features=feats,
|
||||
plays=[int(r[8]) for r in rows],
|
||||
)
|
||||
|
||||
|
||||
class IndexHolder:
|
||||
"""Каталог в памяти; перечитывается раз в INDEX_TTL_S в фоне, не блокируя чтение."""
|
||||
|
||||
def __init__(self, dsn: str) -> None:
|
||||
self.dsn = dsn
|
||||
self._index: CatalogIndex | None = None
|
||||
self._loaded = 0.0
|
||||
self._lock = threading.Lock()
|
||||
|
||||
def get(self) -> CatalogIndex:
|
||||
if self._index is None:
|
||||
self.refresh()
|
||||
elif time.monotonic() - self._loaded > INDEX_TTL_S and self._lock.acquire(blocking=False):
|
||||
threading.Thread(target=self._refresh_locked, daemon=True).start()
|
||||
assert self._index is not None
|
||||
return self._index
|
||||
|
||||
def refresh(self) -> None:
|
||||
with self._lock:
|
||||
self._load()
|
||||
|
||||
def _refresh_locked(self) -> None:
|
||||
try:
|
||||
self._load()
|
||||
finally:
|
||||
self._lock.release()
|
||||
|
||||
def _load(self) -> None:
|
||||
started = time.monotonic()
|
||||
with psycopg.connect(self.dsn) as conn:
|
||||
self._index = load_index(conn)
|
||||
self._loaded = time.monotonic()
|
||||
log.info(
|
||||
"catalog index loaded: tracks=%d analysed=%d in %.0f ms",
|
||||
len(self._index),
|
||||
int(self._index.has_emb.sum()),
|
||||
(self._loaded - started) * 1000,
|
||||
)
|
||||
|
||||
|
||||
def load_profile(conn: psycopg.Connection[Any], user_id: str) -> Profile:
|
||||
"""implicit_score из агрегатов с затуханием по давности; лайк +3, дизлайк — исключение."""
|
||||
now = time.time()
|
||||
scores: dict[str, float] = {}
|
||||
played: set[str] = set()
|
||||
last: dict[str, float] = {}
|
||||
for tid, plays, complete, skips, early, last_at in conn.execute(
|
||||
"""
|
||||
SELECT track_id::text, play_count, complete_count, skip_count, early_skip_count, last_played_at
|
||||
FROM user_track_stats WHERE user_id = %s
|
||||
""",
|
||||
(user_id,),
|
||||
):
|
||||
played.add(tid)
|
||||
raw = complete * 1.0 + max(plays - complete, 0) * 0.7 - early * 1.0 - max(skips - early, 0) * 0.2
|
||||
ts = last_at.timestamp() if last_at else now
|
||||
last[tid] = ts
|
||||
scores[tid] = raw * math.exp(-math.log(2) * (now - ts) / 86400 / HALF_LIFE_DAYS)
|
||||
disliked: set[str] = set()
|
||||
liked: set[str] = set()
|
||||
for tid, reaction in conn.execute(
|
||||
"SELECT track_id::text, reaction::text FROM track_reactions WHERE user_id = %s", (user_id,)
|
||||
):
|
||||
if reaction == "like":
|
||||
liked.add(tid)
|
||||
scores[tid] = scores.get(tid, 0.0) + 3.0
|
||||
else:
|
||||
disliked.add(tid)
|
||||
scores.pop(tid, None)
|
||||
disliked_artists = {
|
||||
r[0]
|
||||
for r in conn.execute(
|
||||
"SELECT artist_id::text FROM artist_reactions WHERE user_id = %s AND reaction = 'dislike'",
|
||||
(user_id,),
|
||||
)
|
||||
}
|
||||
recent = {
|
||||
r[0]
|
||||
for r in conn.execute(
|
||||
"""
|
||||
SELECT DISTINCT track_id::text FROM play_events
|
||||
WHERE user_id = %s AND type = 'play_start' AND occurred_at > now() - interval '3 hours'
|
||||
""",
|
||||
(user_id,),
|
||||
)
|
||||
}
|
||||
row = conn.execute("SELECT exploration FROM user_profiles_rec WHERE user_id = %s", (user_id,)).fetchone()
|
||||
exploration = float(row[0]) if row else 0.15
|
||||
if len(played) < 30:
|
||||
exploration = max(exploration, 0.4) # холодный старт: больше исследования на первых треках
|
||||
day = 86400
|
||||
return Profile(
|
||||
scores=scores,
|
||||
disliked=disliked,
|
||||
disliked_artists=disliked_artists,
|
||||
recent=recent,
|
||||
played=played,
|
||||
liked_recent={t for t in liked if now - last.get(t, 0) < day},
|
||||
rediscover={t for t, s in scores.items() if s > 1.5 and now - last.get(t, now) > 60 * day},
|
||||
exploration=exploration,
|
||||
)
|
||||
|
||||
|
||||
def seed_tracks(conn: psycopg.Connection[Any], seed_type: str, seed_id: str) -> list[str]:
|
||||
"""Треки, от которых запущена волна: трек, альбом или исполнитель."""
|
||||
queries = {
|
||||
"track": "SELECT %s::uuid::text",
|
||||
"album": "SELECT id::text FROM tracks WHERE album_id = %s",
|
||||
"artist": "SELECT track_id::text FROM track_artists WHERE artist_id = %s",
|
||||
}
|
||||
q = queries.get(seed_type)
|
||||
if not q:
|
||||
return []
|
||||
return [r[0] for r in conn.execute(q, (seed_id,))]
|
||||
|
||||
|
||||
class SessionStore:
|
||||
def __init__(self, url: str) -> None:
|
||||
self.rdb = redis.Redis.from_url(url, socket_connect_timeout=2, socket_timeout=2)
|
||||
|
||||
def get(self, session_id: str) -> Session:
|
||||
raw = cast(bytes | None, self.rdb.get(f"wave:session:{session_id}"))
|
||||
if not raw:
|
||||
return Session()
|
||||
data = json.loads(raw)
|
||||
return Session(**data)
|
||||
|
||||
def put(self, session_id: str, s: Session) -> None:
|
||||
self.rdb.set(f"wave:session:{session_id}", json.dumps(asdict(s)), ex=SESSION_TTL_S)
|
||||
Reference in new issue
Block a user