test: end-to-end user journey in CI; fix ALS vectors for small libraries

- newApp (extracted from main) is the exact production wiring; the
  `ci / db` job now also runs it on PostgreSQL, Redis and Meilisearch
  and walks the main user flows over HTTP: first admin sign-in and
  credential change, local source and scan, search, album / artist /
  top tracks, signed stream, events and history, likes, fallback wave,
  playback state, upload queue up to review.
- ALS on a small library produced fewer than 64 factors and the insert
  into vector(64) failed; vectors are now zero-padded to the column size.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

.gitea/workflows/ci-db.yml

@@ -1,5 +1,6 @@
 # Интеграционные проверки SQL на настоящем PostgreSQL + pgvector: миграции вверх / вниз / вверх,
-# запросы сканера, отпечатков, шаблонов и очереди (Go), загрузка каталога, профиля и обучение ALS (Python).
+# запросы сканера, отпечатков, шаблонов и очереди, сквозной сценарий пользователя через HTTP (Go),
+# загрузка каталога, профиля и обучение ALS (Python).
 name: ci / db
 
 on:
@@ -34,8 +35,18 @@ jobs:
           --health-interval 5s
           --health-timeout 5s
           --health-retries 20
+      redis:
+        image: redis:7.4-alpine
+      meilisearch:
+        image: getmeili/meilisearch:v1.11
+        env:
+          MEILI_MASTER_KEY: ci-master-key-0123456789
+          MEILI_ENV: development
     env:
       TEST_DATABASE_URL: postgres://pipu:pipu@postgres:5432/pipu?sslmode=disable
+      TEST_REDIS_URL: redis://redis:6379/0
+      TEST_MEILI_URL: http://meilisearch:7700
+      TEST_MEILI_KEY: ci-master-key-0123456789
     steps:
       - uses: actions/checkout@v4
       - uses: actions/setup-go@v5
@@ -43,6 +54,9 @@ jobs:
       - name: Go — миграции и запросы store
         working-directory: services/api
         run: go test -tags integration -count=1 -v ./internal/dbtest/
+      - name: Go — сквозной сценарий пользователя (newApp на Postgres, Redis, Meilisearch)
+        working-directory: services/api
+        run: go test -tags integration -count=1 -v -run TestUserJourney ./cmd/api/
       - uses: astral-sh/setup-uv@v6
         with: { version: "0.10.2", enable-cache: true, cache-dependency-glob: services/recommender/uv.lock }
       - name: Python — каталог, профиль, обучение ALS

services/api/cmd/api/e2e_test.go

@@ -0,0 +1,328 @@
+//go:build integration
+
+package main
+
+// Сквозная проверка основных функций пользователя на той же сборке, что в продакшене (newApp), поверх
+// настоящих PostgreSQL + pgvector, Redis и Meilisearch (CI-job `ci / db`): первый вход администратора,
+// подключение папки с музыкой и скан, поиск, альбом и исполнитель, стрим, события и история, лайки,
+// «Моя волна» (запасная, без рекомендателя), состояние плеера, очередь на добавление до рассмотрения.
+
+import (
+	"bytes"
+	"context"
+	"crypto/sha256"
+	"encoding/hex"
+	"encoding/json"
+	"fmt"
+	"io"
+	"log/slog"
+	"net/http"
+	"net/http/httptest"
+	"net/url"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/google/uuid"
+	"github.com/redis/go-redis/v9"
+	"github.com/stretchr/testify/require"
+
+	"pipu/api/internal/library"
+	"pipu/api/internal/library/meta"
+	"pipu/api/internal/platform/config"
+	"pipu/api/internal/platform/db"
+)
+
+type fakeProber struct{}
+
+// Probe: «Исполнитель - Название.flac» в папке альбома, 200 с.
+func (fakeProber) Probe(_ context.Context, abs string) (library.Probe, error) {
+	base := strings.TrimSuffix(filepath.Base(abs), filepath.Ext(abs))
+	artist, title, _ := strings.Cut(base, " - ")
+	return library.Probe{
+		Format: "flac", Codec: "flac", Bitrate: 900_000, SampleRate: 44100, BitDepth: 16, Channels: 2, DurationMs: 200_000,
+		Tags: meta.Fields{Artist: artist, Title: title, Album: filepath.Base(filepath.Dir(abs)), Year: "1988"},
+	}, nil
+}
+
+type client struct {
+	t     *testing.T
+	base  string
+	token string
+}
+
+// call — JSON-запрос; out == nil — тело не нужно. Возвращает статус.
+func (c *client) call(method, path string, body, out any) int {
+	c.t.Helper()
+	var rd io.Reader
+	if body != nil {
+		raw, err := json.Marshal(body)
+		require.NoError(c.t, err)
+		rd = bytes.NewReader(raw)
+	}
+	req, err := http.NewRequest(method, c.base+path, rd)
+	require.NoError(c.t, err)
+	req.Header.Set("Content-Type", "application/json")
+	if c.token != "" {
+		req.Header.Set("Authorization", "Bearer "+c.token)
+	}
+	res, err := http.DefaultClient.Do(req)
+	require.NoError(c.t, err)
+	defer func() { _ = res.Body.Close() }()
+	raw, _ := io.ReadAll(res.Body)
+	if res.StatusCode >= 400 {
+		c.t.Logf("%s %s → %d %s", method, path, res.StatusCode, raw)
+	}
+	if out != nil && res.StatusCode < 300 && len(raw) > 0 {
+		require.NoError(c.t, json.Unmarshal(raw, out), string(raw))
+	}
+	return res.StatusCode
+}
+
+func (c *client) ok(method, path string, body, out any) {
+	c.t.Helper()
+	st := c.call(method, path, body, out)
+	require.Less(c.t, st, 300, "%s %s → %d", method, path, st)
+}
+
+// eventually повторяет проверку до таймаута (скан, индексация поиска — фоновые).
+func eventually(t *testing.T, what string, fn func() bool) {
+	t.Helper()
+	deadline := time.Now().Add(30 * time.Second)
+	for time.Now().Before(deadline) {
+		if fn() {
+			return
+		}
+		time.Sleep(300 * time.Millisecond)
+	}
+	t.Fatalf("timeout: %s", what)
+}
+
+type track struct {
+	ID        string `json:"id"`
+	Title     string `json:"title"`
+	StreamURL string `json:"stream_url"`
+	Available bool   `json:"available"`
+	Reaction  string `json:"reaction"`
+	Album     struct {
+		ID string `json:"id"`
+	} `json:"album"`
+	Artists []struct {
+		ID string `json:"id"`
+	} `json:"artists"`
+}
+
+func TestUserJourney(t *testing.T) {
+	dbURL := os.Getenv("TEST_DATABASE_URL")
+	if dbURL == "" {
+		t.Skip("TEST_DATABASE_URL is not set")
+	}
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	host := t.TempDir()
+	album := filepath.Join(host, "music", "Кино", "Группа крови")
+	require.NoError(t, os.MkdirAll(album, 0o755))
+	for _, name := range []string{"Кино - Группа крови.flac", "Кино - Закрой за мной дверь.flac", "Кино - Война.flac"} {
+		require.NoError(t, os.WriteFile(filepath.Join(album, name), []byte("audio:"+name), 0o644))
+	}
+
+	t.Setenv("DATABASE_URL", dbURL)
+	t.Setenv("REDIS_URL", os.Getenv("TEST_REDIS_URL"))
+	t.Setenv("BOOTSTRAP_ADMIN_PASSWORD", "admin")
+	cfg, err := config.Load[config.API]()
+	require.NoError(t, err)
+	cfg.LibraryHostRoot = host
+	cfg.LibraryRoot = t.TempDir()
+	cfg.CacheDir = t.TempDir()
+	cfg.UploadsDir = t.TempDir()
+	cfg.UpdatesDir = t.TempDir()
+	cfg.XAccel = false
+	cfg.CookieSecure = false
+	cfg.RecommenderURL = "http://127.0.0.1:1" // рекомендатель недоступен — волна запасная
+	cfg.StorageAgentURL = "http://127.0.0.1:1"
+	if v := os.Getenv("TEST_MEILI_URL"); v != "" {
+		cfg.MeiliURL = v
+	}
+	cfg.MeiliMasterKey = os.Getenv("TEST_MEILI_KEY")
+
+	pool, err := db.Connect(ctx, cfg.DatabaseURL)
+	require.NoError(t, err)
+	defer pool.Close()
+	require.NoError(t, db.Migrate(ctx, pool))
+	opts, err := redis.ParseURL(cfg.RedisURL)
+	require.NoError(t, err)
+	rdb := redis.NewClient(opts)
+	defer func() { _ = rdb.Close() }()
+
+	handler, err := newApp(ctx, cfg, pool, rdb, slog.New(slog.DiscardHandler), fakeProber{})
+	require.NoError(t, err)
+	srv := httptest.NewServer(handler)
+	defer srv.Close()
+	c := &client{t: t, base: srv.URL}
+
+	// 1. Первый вход администратора: стандартные данные → обязательная смена.
+	var info struct {
+		BootstrapRequired bool `json:"bootstrap_required"`
+	}
+	c.ok("GET", "/api/v1/server-info", nil, &info)
+	var tokens struct {
+		AccessToken string `json:"access_token"`
+		User        struct {
+			ID                    string `json:"id"`
+			MustChangeCredentials bool   `json:"must_change_credentials"`
+		} `json:"user"`
+	}
+	c.ok("POST", "/api/v1/auth/login", map[string]any{"username": "admin", "password": "admin", "client_type": "web"}, &tokens)
+	c.token = tokens.AccessToken
+	if tokens.User.MustChangeCredentials {
+		c.ok("POST", "/api/v1/me/bootstrap-credentials",
+			map[string]any{"new_username": "owner", "new_password": "Str0ng-Passw0rd!"}, &tokens)
+		c.token = tokens.AccessToken
+	}
+
+	// 2. Папка с музыкой и скан.
+	var src struct {
+		ID     string `json:"id"`
+		Status string `json:"status"`
+	}
+	c.ok("POST", "/api/v1/admin/sources", map[string]any{"name": "Диск", "type": "local", "local_path": "music"}, &src)
+	require.Equal(t, "mounted", src.Status)
+	c.ok("POST", "/api/v1/admin/sources/"+src.ID+"/scan", map[string]any{"full": true}, nil)
+	var status struct {
+		Tracks int `json:"tracks"`
+	}
+	eventually(t, "scan", func() bool {
+		c.ok("GET", "/api/v1/admin/library/status", nil, &status)
+		return status.Tracks == 3
+	})
+
+	// 3. Поиск (Meilisearch индексирует в фоне).
+	var found struct {
+		Artists []struct {
+			ID string `json:"id"`
+		} `json:"artists"`
+		Albums []struct {
+			ID string `json:"id"`
+		} `json:"albums"`
+		Tracks []track `json:"tracks"`
+	}
+	eventually(t, "search index", func() bool {
+		return c.call("GET", "/api/v1/search?q="+url.QueryEscape("кино"), nil, &found) == 200 && len(found.Albums) == 1
+	})
+
+	// 4. Альбом, исполнитель, популярные треки.
+	var alb struct {
+		Tracks []track `json:"tracks"`
+	}
+	c.ok("GET", "/api/v1/albums/"+found.Albums[0].ID, nil, &alb)
+	require.Len(t, alb.Tracks, 3)
+	tr := alb.Tracks[0]
+	require.True(t, tr.Available)
+	artistID := tr.Artists[0].ID
+	c.ok("GET", "/api/v1/artists/"+artistID, nil, nil)
+	c.ok("GET", "/api/v1/artists/"+artistID+"/albums", nil, nil)
+	var top struct {
+		Items []track `json:"items"`
+	}
+	c.ok("GET", "/api/v1/artists/"+artistID+"/top-tracks", nil, &top)
+	require.Len(t, top.Items, 3)
+
+	// 5. Стрим по подписанной ссылке (без заголовка авторизации, как у <audio>).
+	res, err := http.Get(srv.URL + tr.StreamURL + "&q=original")
+	require.NoError(t, err)
+	body, _ := io.ReadAll(res.Body)
+	_ = res.Body.Close()
+	require.Equal(t, 200, res.StatusCode)
+	require.True(t, strings.HasPrefix(string(body), "audio:"), "streams the file")
+
+	// 6. События прослушивания → история.
+	now := time.Now().UTC()
+	ev := func(kind string, at time.Time, extra map[string]any) map[string]any {
+		e := map[string]any{"id": uuid.Must(uuid.NewV7()).String(), "type": kind, "track_id": tr.ID,
+			"occurred_at": at.Format(time.RFC3339Nano), "context_type": "album", "context_id": tr.Album.ID}
+		for k, v := range extra {
+			e[k] = v
+		}
+		return e
+	}
+	c.ok("POST", "/api/v1/events", map[string]any{"events": []any{
+		ev("play_start", now.Add(-200*time.Second), map[string]any{"position_ms": 0}),
+		ev("play_end", now, map[string]any{"position_ms": 200_000, "played_ms": 200_000}),
+	}}, nil)
+	var hist struct {
+		Items []struct {
+			Track track `json:"track"`
+		} `json:"items"`
+	}
+	c.ok("GET", "/api/v1/me/history", nil, &hist)
+	require.NotEmpty(t, hist.Items)
+
+	// 7. Лайк и коллекция.
+	c.ok("PUT", "/api/v1/me/reactions/tracks/"+tr.ID, map[string]any{"reaction": "like"}, nil)
+	var likes struct {
+		Total int `json:"total"`
+	}
+	c.ok("GET", "/api/v1/me/likes", nil, &likes)
+	require.Equal(t, 1, likes.Total)
+
+	// 8. «Моя волна»: рекомендатель недоступен — запасная волна из библиотеки.
+	var wave struct {
+		SessionID string  `json:"session_id"`
+		Tracks    []track `json:"tracks"`
+		Fallback  bool    `json:"fallback"`
+	}
+	c.ok("GET", "/api/v1/wave/options", nil, nil)
+	c.ok("POST", "/api/v1/wave/sessions", map[string]any{"activity": nil, "mood": nil}, &wave)
+	require.NotEmpty(t, wave.Tracks, "wave has tracks")
+	require.True(t, wave.Fallback)
+	c.ok("POST", "/api/v1/wave/sessions/"+wave.SessionID+"/next", map[string]any{"feedback": []any{}}, nil)
+
+	// 9. Состояние плеера (продолжить на другом устройстве).
+	c.ok("PUT", "/api/v1/me/playback-state", map[string]any{
+		"track_id": tr.ID, "position_ms": 1000, "queue": []string{tr.ID}, "queue_index": 0,
+	}, nil)
+	var st struct {
+		TrackID string `json:"track_id"`
+	}
+	c.ok("GET", "/api/v1/me/playback-state", nil, &st)
+	require.Equal(t, tr.ID, st.TrackID)
+
+	// 10. Очередь на добавление: резерв → tus → обработка до рассмотрения.
+	upload := []byte("audio:new upload")
+	sum := sha256.Sum256(upload)
+	var batch struct {
+		Files []struct {
+			TusURL   *string `json:"tus_url"`
+			Rejected *string `json:"rejected"`
+		} `json:"files"`
+	}
+	c.ok("GET", "/api/v1/uploads/quota", nil, nil)
+	c.ok("POST", "/api/v1/uploads", map[string]any{"files": []any{map[string]any{
+		"client_id": "1", "name": "Кино - Кукушка.flac", "rel_path": "Кино/Чёрный альбом/Кино - Кукушка.flac",
+		"size": len(upload), "sha256": hex.EncodeToString(sum[:]),
+	}}}, &batch)
+	require.Nil(t, batch.Files[0].Rejected)
+	req, err := http.NewRequest("PATCH", srv.URL+*batch.Files[0].TusURL, bytes.NewReader(upload))
+	require.NoError(t, err)
+	req.Header.Set("Authorization", "Bearer "+c.token)
+	req.Header.Set("Tus-Resumable", "1.0.0")
+	req.Header.Set("Content-Type", "application/offset+octet-stream")
+	req.Header.Set("Upload-Offset", "0")
+	res, err = http.DefaultClient.Do(req)
+	require.NoError(t, err)
+	_ = res.Body.Close()
+	require.Equal(t, 204, res.StatusCode)
+	var mine struct {
+		Items []struct {
+			Status string `json:"status"`
+		} `json:"items"`
+	}
+	eventually(t, "upload processed", func() bool {
+		c.ok("GET", "/api/v1/uploads/mine", nil, &mine)
+		return len(mine.Items) == 1 && mine.Items[0].Status == "pending_review"
+	})
+	fmt.Println("user journey ok")
+}

services/api/cmd/api/main.go

@@ -72,12 +72,25 @@ func run() error {
 	rdb := redis.NewClient(redisOpts)
 	defer func() { _ = rdb.Close() }()
 
-	authSvc, err := newAuthService(cfg, pool, rdb, log)
+	handler, err := newApp(ctx, cfg, pool, rdb, log, library.FFprobe{Bin: cfg.FFprobePath, Timeout: time.Minute})
 	if err != nil {
 		return err
 	}
+	srv := httpx.NewServer(cfg.HTTPAddr, handler)
+	log.Info("starting", "version", buildinfo.Version)
+	return httpx.Serve(ctx, log, srv, cfg.ShutdownTimeout)
+}
+
+// newApp собирает сервис: хранилища, сканер, плеер, волну, очередь и HTTP-роутер. Фоновые задачи живут до отмены ctx.
+// prober подменяется в сквозном тесте (там нет ffprobe).
+func newApp(ctx context.Context, cfg config.API, pool *pgxpool.Pool, rdb *redis.Client, log *slog.Logger,
+	prober library.Prober) (http.Handler, error) {
+	authSvc, err := newAuthService(cfg, pool, rdb, log)
+	if err != nil {
+		return nil, err
+	}
 	if err := authSvc.Bootstrap(ctx); err != nil {
-		return err
+		return nil, err
 	}
 
 	sys := system.NewHandler(system.Info{
@@ -92,7 +105,6 @@ func run() error {
 	})
 
 	libStore := library.NewStore(pool)
-	prober := library.FFprobe{Bin: cfg.FFprobePath, Timeout: time.Minute}
 	board := library.NewProgressBoard()
 	scanner := library.NewScanner(libStore, prober, board, log)
 	agent := library.NewHTTPAgent(cfg.StorageAgentURL, cfg.InternalToken)
@@ -102,7 +114,7 @@ func run() error {
 	indexer := search.NewIndexer(pool, meili, log)
 	signer, err := sign.New(cfg.SigningKey)
 	if err != nil {
-		return err
+		return nil, err
 	}
 	if cfg.SigningKey == "" {
 		log.Warn("STREAM_SIGNING_KEY is not set: cover and stream links will break on restart")
@@ -152,7 +164,7 @@ func run() error {
 		uploads.Config{StagingDir: cfg.UploadsDir, FFmpeg: cfg.FFmpegPath}, log)
 	upSvc.Run(ctx)
 
-	srv := httpx.NewServer(cfg.HTTPAddr, server.NewRouter(server.Deps{
+	return server.NewRouter(server.Deps{
 		Log:         log,
 		System:      sys,
 		Auth:        auth.NewHandler(authSvc, cfg.CookieSecure),
@@ -166,9 +178,7 @@ func run() error {
 			Dir: cfg.UpdatesDir, MinSupported: cfg.MinAndroidVersionCode}, log),
 		Uploads: uploads.NewHandler(upSvc),
 		TUS:     uploads.NewTUS(uploads.NewStore(pool), cfg.UploadsDir, upSvc.Complete, log),
-	}))
+	}), nil
-	log.Info("starting", "version", buildinfo.Version)
-	return httpx.Serve(ctx, log, srv, cfg.ShutdownTimeout)
 }
 
 func newAuthService(cfg config.API, pool *pgxpool.Pool, rdb *redis.Client, log *slog.Logger) (*auth.Service, error) {

services/recommender/pipu_rec/train/pipeline.py

@@ -108,9 +108,20 @@ def should_activate(new: dict[str, Any], current: dict[str, Any] | None) -> bool
     return float(new_ndcg) >= float(cur_ndcg) * (1 - MAX_REGRESSION)
 
 
+# Размерность колонок track_embeddings.cf / user_profiles_rec.taste_cf (vector(64)).
+CF_DIM = 64
+
+
+def pad(v: np.ndarray, dim: int = CF_DIM) -> np.ndarray:
+    """Маленькая библиотека даёт меньше факторов, чем колонка: добиваем нулями (косинус не меняется)."""
+    if v.shape[1] >= dim:
+        return v[:, :dim]
+    return np.pad(v, ((0, 0), (0, dim - v.shape[1])))
+
+
 def save_als(conn: psycopg.Connection[Any], model: ALSModel, version: str, metrics: dict[str, Any]) -> None:
-    items = normalize(model.item_factors)
+    items = pad(normalize(model.item_factors))
-    users = normalize(model.user_factors)
+    users = pad(normalize(model.user_factors))
     with conn.transaction():
         with conn.cursor() as cur:
             cur.executemany(

services/recommender/tests/test_train.py

@@ -108,3 +108,11 @@ def test_engine_uses_cf_source() -> None:
     assert len(picks) == 5
     assert sum(int(p.id[1:]) < 20 for p in picks) >= 4  # почти всё — из «своей» половины по ALS
     assert model_version(index, profile).endswith("+cf")
+
+
+def test_cf_vectors_are_padded_to_column_size() -> None:
+    from pipu_rec.train.pipeline import CF_DIM, pad
+
+    v = np.ones((3, 5), dtype=np.float32)
+    assert pad(v).shape == (3, CF_DIM)
+    assert pad(np.ones((2, 80), dtype=np.float32)).shape == (2, CF_DIM)