From 2285107d04b95e5b96bd5baabcfc4f560e47a118 Mon Sep 17 00:00:00 2001 From: mrleo1nid <43583385+mrleo1nid@users.noreply.github.com> Date: Tue, 20 Oct 2026 18:26:36 +0300 Subject: [PATCH] test: end-to-end user journey in CI; fix ALS vectors for small libraries - newApp (extracted from main) is the exact production wiring; the `ci / db` job now also runs it on PostgreSQL, Redis and Meilisearch and walks the main user flows over HTTP: first admin sign-in and credential change, local source and scan, search, album / artist / top tracks, signed stream, events and history, likes, fallback wave, playback state, upload queue up to review. - ALS on a small library produced fewer than 64 factors and the insert into vector(64) failed; vectors are now zero-padded to the column size. Co-Authored-By: Claude Opus 5.5 --- .gitea/workflows/ci-db.yml | 16 +- services/api/cmd/api/e2e_test.go | 328 ++++++++++++++++++ services/api/cmd/api/main.go | 26 +- .../recommender/pipu_rec/train/pipeline.py | 15 +- services/recommender/tests/test_train.py | 8 + 5 files changed, 382 insertions(+), 11 deletions(-) create mode 100644 services/api/cmd/api/e2e_test.go diff --git a/.gitea/workflows/ci-db.yml b/.gitea/workflows/ci-db.yml index 4fb0c29..0458749 100644 --- a/.gitea/workflows/ci-db.yml +++ b/.gitea/workflows/ci-db.yml @@ -1,5 +1,6 @@ # Интеграционные проверки SQL на настоящем PostgreSQL + pgvector: миграции вверх / вниз / вверх, -# запросы сканера, отпечатков, шаблонов и очереди (Go), загрузка каталога, профиля и обучение ALS (Python). +# запросы сканера, отпечатков, шаблонов и очереди, сквозной сценарий пользователя через HTTP (Go), +# загрузка каталога, профиля и обучение ALS (Python). name: ci / db on: @@ -34,8 +35,18 @@ jobs: --health-interval 5s --health-timeout 5s --health-retries 20 + redis: + image: redis:7.4-alpine + meilisearch: + image: getmeili/meilisearch:v1.11 + env: + MEILI_MASTER_KEY: ci-master-key-0123456789 + MEILI_ENV: development env: TEST_DATABASE_URL: postgres://pipu:pipu@postgres:5432/pipu?sslmode=disable + TEST_REDIS_URL: redis://redis:6379/0 + TEST_MEILI_URL: http://meilisearch:7700 + TEST_MEILI_KEY: ci-master-key-0123456789 steps: - uses: actions/checkout@v4 - uses: actions/setup-go@v5 @@ -43,6 +54,9 @@ jobs: - name: Go — миграции и запросы store working-directory: services/api run: go test -tags integration -count=1 -v ./internal/dbtest/ + - name: Go — сквозной сценарий пользователя (newApp на Postgres, Redis, Meilisearch) + working-directory: services/api + run: go test -tags integration -count=1 -v -run TestUserJourney ./cmd/api/ - uses: astral-sh/setup-uv@v6 with: { version: "0.10.2", enable-cache: true, cache-dependency-glob: services/recommender/uv.lock } - name: Python — каталог, профиль, обучение ALS diff --git a/services/api/cmd/api/e2e_test.go b/services/api/cmd/api/e2e_test.go new file mode 100644 index 0000000..e6c6401 --- /dev/null +++ b/services/api/cmd/api/e2e_test.go @@ -0,0 +1,328 @@ +//go:build integration + +package main + +// Сквозная проверка основных функций пользователя на той же сборке, что в продакшене (newApp), поверх +// настоящих PostgreSQL + pgvector, Redis и Meilisearch (CI-job `ci / db`): первый вход администратора, +// подключение папки с музыкой и скан, поиск, альбом и исполнитель, стрим, события и история, лайки, +// «Моя волна» (запасная, без рекомендателя), состояние плеера, очередь на добавление до рассмотрения. + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "io" + "log/slog" + "net/http" + "net/http/httptest" + "net/url" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/google/uuid" + "github.com/redis/go-redis/v9" + "github.com/stretchr/testify/require" + + "pipu/api/internal/library" + "pipu/api/internal/library/meta" + "pipu/api/internal/platform/config" + "pipu/api/internal/platform/db" +) + +type fakeProber struct{} + +// Probe: «Исполнитель - Название.flac» в папке альбома, 200 с. +func (fakeProber) Probe(_ context.Context, abs string) (library.Probe, error) { + base := strings.TrimSuffix(filepath.Base(abs), filepath.Ext(abs)) + artist, title, _ := strings.Cut(base, " - ") + return library.Probe{ + Format: "flac", Codec: "flac", Bitrate: 900_000, SampleRate: 44100, BitDepth: 16, Channels: 2, DurationMs: 200_000, + Tags: meta.Fields{Artist: artist, Title: title, Album: filepath.Base(filepath.Dir(abs)), Year: "1988"}, + }, nil +} + +type client struct { + t *testing.T + base string + token string +} + +// call — JSON-запрос; out == nil — тело не нужно. Возвращает статус. +func (c *client) call(method, path string, body, out any) int { + c.t.Helper() + var rd io.Reader + if body != nil { + raw, err := json.Marshal(body) + require.NoError(c.t, err) + rd = bytes.NewReader(raw) + } + req, err := http.NewRequest(method, c.base+path, rd) + require.NoError(c.t, err) + req.Header.Set("Content-Type", "application/json") + if c.token != "" { + req.Header.Set("Authorization", "Bearer "+c.token) + } + res, err := http.DefaultClient.Do(req) + require.NoError(c.t, err) + defer func() { _ = res.Body.Close() }() + raw, _ := io.ReadAll(res.Body) + if res.StatusCode >= 400 { + c.t.Logf("%s %s → %d %s", method, path, res.StatusCode, raw) + } + if out != nil && res.StatusCode < 300 && len(raw) > 0 { + require.NoError(c.t, json.Unmarshal(raw, out), string(raw)) + } + return res.StatusCode +} + +func (c *client) ok(method, path string, body, out any) { + c.t.Helper() + st := c.call(method, path, body, out) + require.Less(c.t, st, 300, "%s %s → %d", method, path, st) +} + +// eventually повторяет проверку до таймаута (скан, индексация поиска — фоновые). +func eventually(t *testing.T, what string, fn func() bool) { + t.Helper() + deadline := time.Now().Add(30 * time.Second) + for time.Now().Before(deadline) { + if fn() { + return + } + time.Sleep(300 * time.Millisecond) + } + t.Fatalf("timeout: %s", what) +} + +type track struct { + ID string `json:"id"` + Title string `json:"title"` + StreamURL string `json:"stream_url"` + Available bool `json:"available"` + Reaction string `json:"reaction"` + Album struct { + ID string `json:"id"` + } `json:"album"` + Artists []struct { + ID string `json:"id"` + } `json:"artists"` +} + +func TestUserJourney(t *testing.T) { + dbURL := os.Getenv("TEST_DATABASE_URL") + if dbURL == "" { + t.Skip("TEST_DATABASE_URL is not set") + } + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + host := t.TempDir() + album := filepath.Join(host, "music", "Кино", "Группа крови") + require.NoError(t, os.MkdirAll(album, 0o755)) + for _, name := range []string{"Кино - Группа крови.flac", "Кино - Закрой за мной дверь.flac", "Кино - Война.flac"} { + require.NoError(t, os.WriteFile(filepath.Join(album, name), []byte("audio:"+name), 0o644)) + } + + t.Setenv("DATABASE_URL", dbURL) + t.Setenv("REDIS_URL", os.Getenv("TEST_REDIS_URL")) + t.Setenv("BOOTSTRAP_ADMIN_PASSWORD", "admin") + cfg, err := config.Load[config.API]() + require.NoError(t, err) + cfg.LibraryHostRoot = host + cfg.LibraryRoot = t.TempDir() + cfg.CacheDir = t.TempDir() + cfg.UploadsDir = t.TempDir() + cfg.UpdatesDir = t.TempDir() + cfg.XAccel = false + cfg.CookieSecure = false + cfg.RecommenderURL = "http://127.0.0.1:1" // рекомендатель недоступен — волна запасная + cfg.StorageAgentURL = "http://127.0.0.1:1" + if v := os.Getenv("TEST_MEILI_URL"); v != "" { + cfg.MeiliURL = v + } + cfg.MeiliMasterKey = os.Getenv("TEST_MEILI_KEY") + + pool, err := db.Connect(ctx, cfg.DatabaseURL) + require.NoError(t, err) + defer pool.Close() + require.NoError(t, db.Migrate(ctx, pool)) + opts, err := redis.ParseURL(cfg.RedisURL) + require.NoError(t, err) + rdb := redis.NewClient(opts) + defer func() { _ = rdb.Close() }() + + handler, err := newApp(ctx, cfg, pool, rdb, slog.New(slog.DiscardHandler), fakeProber{}) + require.NoError(t, err) + srv := httptest.NewServer(handler) + defer srv.Close() + c := &client{t: t, base: srv.URL} + + // 1. Первый вход администратора: стандартные данные → обязательная смена. + var info struct { + BootstrapRequired bool `json:"bootstrap_required"` + } + c.ok("GET", "/api/v1/server-info", nil, &info) + var tokens struct { + AccessToken string `json:"access_token"` + User struct { + ID string `json:"id"` + MustChangeCredentials bool `json:"must_change_credentials"` + } `json:"user"` + } + c.ok("POST", "/api/v1/auth/login", map[string]any{"username": "admin", "password": "admin", "client_type": "web"}, &tokens) + c.token = tokens.AccessToken + if tokens.User.MustChangeCredentials { + c.ok("POST", "/api/v1/me/bootstrap-credentials", + map[string]any{"new_username": "owner", "new_password": "Str0ng-Passw0rd!"}, &tokens) + c.token = tokens.AccessToken + } + + // 2. Папка с музыкой и скан. + var src struct { + ID string `json:"id"` + Status string `json:"status"` + } + c.ok("POST", "/api/v1/admin/sources", map[string]any{"name": "Диск", "type": "local", "local_path": "music"}, &src) + require.Equal(t, "mounted", src.Status) + c.ok("POST", "/api/v1/admin/sources/"+src.ID+"/scan", map[string]any{"full": true}, nil) + var status struct { + Tracks int `json:"tracks"` + } + eventually(t, "scan", func() bool { + c.ok("GET", "/api/v1/admin/library/status", nil, &status) + return status.Tracks == 3 + }) + + // 3. Поиск (Meilisearch индексирует в фоне). + var found struct { + Artists []struct { + ID string `json:"id"` + } `json:"artists"` + Albums []struct { + ID string `json:"id"` + } `json:"albums"` + Tracks []track `json:"tracks"` + } + eventually(t, "search index", func() bool { + return c.call("GET", "/api/v1/search?q="+url.QueryEscape("кино"), nil, &found) == 200 && len(found.Albums) == 1 + }) + + // 4. Альбом, исполнитель, популярные треки. + var alb struct { + Tracks []track `json:"tracks"` + } + c.ok("GET", "/api/v1/albums/"+found.Albums[0].ID, nil, &alb) + require.Len(t, alb.Tracks, 3) + tr := alb.Tracks[0] + require.True(t, tr.Available) + artistID := tr.Artists[0].ID + c.ok("GET", "/api/v1/artists/"+artistID, nil, nil) + c.ok("GET", "/api/v1/artists/"+artistID+"/albums", nil, nil) + var top struct { + Items []track `json:"items"` + } + c.ok("GET", "/api/v1/artists/"+artistID+"/top-tracks", nil, &top) + require.Len(t, top.Items, 3) + + // 5. Стрим по подписанной ссылке (без заголовка авторизации, как у