diff --git a/.gitea/workflows/ci-db.yml b/.gitea/workflows/ci-db.yml index 3b8a6c4..d773a8e 100644 --- a/.gitea/workflows/ci-db.yml +++ b/.gitea/workflows/ci-db.yml @@ -75,6 +75,9 @@ jobs: # These suites share one database; dbtest also rolls all migrations down # and cleans orphan catalog rows. Do not run the packages concurrently. run: go test -p 1 -tags integration -count=1 -v ./internal/dbtest/ ./internal/social/ ./internal/uploads/ + - name: Go — релевантность поиска на Meilisearch + working-directory: services/api + run: go test -tags integration -count=1 -v ./internal/search/ - name: Go — сквозной сценарий пользователя (newApp на Postgres, Redis, Meilisearch) working-directory: services/api run: go test -tags integration -count=1 -v -run TestUserJourney ./cmd/api/ diff --git a/docs/library-and-storage.md b/docs/library-and-storage.md index ca0de58..02944aa 100644 --- a/docs/library-and-storage.md +++ b/docs/library-and-storage.md @@ -199,10 +199,17 @@ Meilisearch, индексы: | `albums` | `title`, `artist_name`, `title_translit` | `year`, `popularity` | | `tracks` | `title`, `artist_names`, `album_title`, `title_translit` | `genre`, `year`, `popularity`, `available` | -- `*_translit` — транслитерация (кириллица ↔ латиница), плюс поле с **исправлением раскладки** +- `*_translit` — варианты транслитерации (кириллица ↔ латиница), включая словарь + устоявшихся написаний (`Boulevard` → `бульвар`) наряду с побуквенным вариантом. + **Исправление раскладки** (`ghbdtn` → `привет`): при пустом результате клиентский запрос повторяется в альтернативной раскладке на стороне `api`. -- Ранжирование: стандартные правила Meilisearch + `popularity:desc` (глобальные засчитанные прослушивания за 90 дней). +- Запрос учитывает все слова (`matchingStrategy=all`), не отбрасывая последнее ради заполнения списка. + Опечатки и поиск по префиксу по-прежнему обрабатывает Meilisearch. +- Ранжирование: `words`, `typo`, `exactness`, `proximity`, `attribute`, `sort`, `popularity:desc`. + Точное имя исполнителя (включая вариант транслитерации) закрепляется перед составными именами + в подсказках и не опускается из-за персонализации. Новые варианты появляются после переиндексации; + API запускает её при старте. - `/search/suggest` использует `multi-search` (один запрос — три индекса), отдаёт ≤ 3/3/5. Клиент делает debounce 150 мс, отменяет предыдущий запрос (`AbortController`). - Персонализация: `suggest` берёт из Meilisearch втрое больше кандидатов и переранжирует их в `api` diff --git a/services/api/internal/search/indexer.go b/services/api/internal/search/indexer.go index 35138c1..f737778 100644 --- a/services/api/internal/search/indexer.go +++ b/services/api/internal/search/indexer.go @@ -27,7 +27,7 @@ type indexSpec struct { doc func(pgx.Rows) (map[string]any, error) } -var rankingRules = []string{"words", "typo", "proximity", "attribute", "sort", "exactness", "popularity:desc"} +var rankingRules = []string{"words", "typo", "exactness", "proximity", "attribute", "sort", "popularity:desc"} var specs = []indexSpec{ { @@ -43,7 +43,7 @@ var specs = []indexSpec{ if err := r.Scan(&id, &name); err != nil { return nil, err } - return map[string]any{"id": id, "name": name, "name_translit": Translit(name), "popularity": 0}, nil + return map[string]any{"id": id, "name": name, "name_translit": TranslitVariants(name), "popularity": 0}, nil }, }, { @@ -65,7 +65,7 @@ var specs = []indexSpec{ } return map[string]any{ "id": id, "title": title, "artist_name": artist, "year": year, "popularity": 0, - "title_translit": Translit(title), "artist_translit": Translit(artist), + "title_translit": TranslitVariants(title), "artist_translit": TranslitVariants(artist), }, nil }, }, @@ -97,12 +97,12 @@ var specs = []indexSpec{ } translit := make([]string, 0, len(artists)) for _, a := range artists { - translit = append(translit, Translit(a)) + translit = append(translit, TranslitVariants(a)...) } return map[string]any{ "id": id, "title": title, "album_title": album, "year": year, "artist_names": artists, "genres": genres, "available": available, "popularity": 0, - "title_translit": Translit(title), "artist_translit": translit, + "title_translit": TranslitVariants(title), "artist_translit": translit, }, nil }, }, diff --git a/services/api/internal/search/meili.go b/services/api/internal/search/meili.go index 1f50ef6..1c9fe42 100644 --- a/services/api/internal/search/meili.go +++ b/services/api/internal/search/meili.go @@ -141,13 +141,15 @@ type MultiQuery struct { Q string `json:"q"` Limit int `json:"limit"` Offset int `json:"offset,omitempty"` + MatchingStrategy string `json:"matchingStrategy,omitempty"` AttributesToRetrieve []string `json:"attributesToRetrieve"` } // MultiResult — ответ на один запрос: id найденных документов и оценка общего числа. type MultiResult struct { - IDs []string - Total int + IDs []string + Total int + ExactIDs []string } // MultiSearch выполняет несколько запросов за один вызов. @@ -155,7 +157,9 @@ func (m *Meili) MultiSearch(ctx context.Context, queries []MultiQuery) ([]MultiR var res struct { Results []struct { Hits []struct { - ID string `json:"id"` + ID string `json:"id"` + Name string `json:"name"` + NameTranslit any `json:"name_translit"` } `json:"hits"` EstimatedTotalHits int `json:"estimatedTotalHits"` } `json:"results"` @@ -167,8 +171,32 @@ func (m *Meili) MultiSearch(ctx context.Context, queries []MultiQuery) ([]MultiR for i, r := range res.Results { for _, h := range r.Hits { out[i].IDs = append(out[i].IDs, h.ID) + if exactArtistName(queries[i].Q, h.Name, h.NameTranslit) { + out[i].ExactIDs = append(out[i].ExactIDs, h.ID) + } } out[i].Total = r.EstimatedTotalHits } return out, nil } + +func exactArtistName(query, name string, translit any) bool { + key := searchKey(query) + if key == "" { + return false + } + if searchKey(name) == key { + return true + } + switch v := translit.(type) { + case string: // indexes built before transliteration alternatives + return searchKey(v) == key + case []any: + for _, item := range v { + if s, ok := item.(string); ok && searchKey(s) == key { + return true + } + } + } + return false +} diff --git a/services/api/internal/search/meili_test.go b/services/api/internal/search/meili_test.go index 1252f1b..5e6a63a 100644 --- a/services/api/internal/search/meili_test.go +++ b/services/api/internal/search/meili_test.go @@ -2,13 +2,49 @@ package search import ( "context" + "encoding/json" "errors" + "log/slog" "net/http" "net/http/httptest" "reflect" "testing" + + "github.com/stretchr/testify/require" ) +func TestQueryRequiresAllWordsAndProtectsExactArtist(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + var body struct { + Queries []MultiQuery `json:"queries"` + } + require.NoError(t, json.NewDecoder(r.Body).Decode(&body)) + require.Len(t, body.Queries, 3) + for _, q := range body.Queries { + require.Equal(t, "all", q.MatchingStrategy) + } + require.Contains(t, body.Queries[0].AttributesToRetrieve, "name_translit") + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"results":[{"hits":[{"id":"collab","name":"Boulevard Depo X 161"},{"id":"exact","name":"BOULEVARD DEPO","name_translit":["боулевард депо","бульвар депо"]}],"estimatedTotalHits":2},{"hits":[]},{"hits":[]}]}`)) + })) + defer srv.Close() + h := &Handler{meili: NewMeili(srv.URL, ""), log: slog.Default()} + f, err := h.query(context.Background(), "бульвар депо", limits{artists: 3, albums: 3, tracks: 5}) + require.NoError(t, err) + require.Equal(t, []string{"exact"}, f.exactArtists) + f = h.personal(context.Background(), f, limits{artists: 1}) + require.Equal(t, []string{"exact"}, f.artists) +} + +func TestExactArtistName(t *testing.T) { + require.True(t, exactArtistName("boulevard depo", "BOULEVARD DEPO", nil)) + require.True(t, exactArtistName("БУЛЬВАР ДЕПО", "BOULEVARD DEPO", []any{"бульвар депо"})) + require.True(t, exactArtistName("кино", "Kino", "кино")) + require.False(t, exactArtistName("boulevard depo", "Depo Boulevard", nil)) + require.False(t, exactArtistName("boulevard depo", "Boulevard Depo X 161", nil)) + require.False(t, exactArtistName("!", "", nil)) +} + func TestDeleteIndexIfExists(t *testing.T) { for _, tc := range []struct { name string diff --git a/services/api/internal/search/personal.go b/services/api/internal/search/personal.go index 5f888c9..9f099c6 100644 --- a/services/api/internal/search/personal.go +++ b/services/api/internal/search/personal.go @@ -12,6 +12,24 @@ import ( // suggestPool — во сколько раз больше кандидатов берётся из Meilisearch для переранжирования подсказок. const suggestPool = 3 +// Exact names stay ahead of related artists, regardless of listening history. +func personalizeArtists(found, exact []string, aff map[uuid.UUID]float64, n int) []string { + protected := make(map[string]bool, len(exact)) + for _, id := range exact { + protected[id] = true + } + var first, rest []string + for _, id := range found { + if protected[id] { + first = append(first, id) + } else { + rest = append(rest, id) + } + } + first = append(first, personalize(rest, aff, max(0, n-len(first)))...) + return first[:min(n, len(first))] +} + // Affinity — насколько пользователь «знает» исполнителей, альбомы и треки: прослушивания, доигрывания, // лайки и дизлайки. Отрицательное — не любит. Нужен для персональных подсказок поиска. type Affinity struct{ pool *pgxpool.Pool } diff --git a/services/api/internal/search/relevance_integration_test.go b/services/api/internal/search/relevance_integration_test.go new file mode 100644 index 0000000..3c3eeb6 --- /dev/null +++ b/services/api/internal/search/relevance_integration_test.go @@ -0,0 +1,55 @@ +//go:build integration + +package search + +import ( + "context" + "net/http" + "os" + "testing" + "time" + + "github.com/google/uuid" + "github.com/stretchr/testify/require" +) + +// Use a private index: never replace a running application's search indexes. +func TestArtistSearchRelevance(t *testing.T) { + url := os.Getenv("TEST_MEILI_URL") + if url == "" { + t.Skip("TEST_MEILI_URL is not set") + } + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + m := NewMeili(url, os.Getenv("TEST_MEILI_KEY")) + index := "search_test_" + uuid.New().String() + require.NoError(t, m.call(ctx, http.MethodPost, "/indexes", map[string]any{"uid": index, "primaryKey": "id"})) + t.Cleanup(func() { + ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) + defer cancel() + require.NoError(t, m.deleteIndexIfExists(ctx, index)) + }) + require.NoError(t, m.call(ctx, http.MethodPatch, "/indexes/"+index+"/settings", specs[0].settings)) + var docs []map[string]any + for id, name := range map[string]string{ + "exact": "BOULEVARD DEPO", "collaboration": "Boulevard Depo X 161", + "reversed": "Depo Boulevard", "unrelated": "Thoma Bulwer", + "partial": "Станция бульвар малина", + } { + docs = append(docs, map[string]any{"id": id, "name": name, "name_translit": TranslitVariants(name), "popularity": 0}) + } + require.NoError(t, m.call(ctx, http.MethodPost, "/indexes/"+index+"/documents", docs)) + for _, query := range []string{"boulevard depo", "бульвар депо", "боулевард депо"} { + t.Run(query, func(t *testing.T) { + results, err := m.MultiSearch(ctx, []MultiQuery{{IndexUID: index, Q: query, Limit: 10, + MatchingStrategy: "all", AttributesToRetrieve: []string{"id", "name", "name_translit"}}}) + require.NoError(t, err) + require.Len(t, results, 1) + require.NotEmpty(t, results[0].IDs) + require.Equal(t, "exact", results[0].IDs[0]) + require.NotContains(t, results[0].IDs, "unrelated") + require.NotContains(t, results[0].IDs, "partial") + require.Contains(t, results[0].ExactIDs, "exact") + }) + } +} diff --git a/services/api/internal/search/service.go b/services/api/internal/search/service.go index 886017b..a194443 100644 --- a/services/api/internal/search/service.go +++ b/services/api/internal/search/service.go @@ -43,6 +43,7 @@ type limits struct{ artists, albums, tracks, offset int } type found struct { artists, albums, tracks []string + exactArtists []string totals [3]int } @@ -50,7 +51,11 @@ func (h *Handler) query(ctx context.Context, q string, l limits) (found, error) var queries []MultiQuery add := func(index string, n int) { if n > 0 { - queries = append(queries, MultiQuery{IndexUID: index, Q: q, Limit: n, Offset: l.offset, AttributesToRetrieve: []string{"id"}}) + attrs := []string{"id"} + if index == IndexArtists { + attrs = append(attrs, "name", "name_translit") + } + queries = append(queries, MultiQuery{IndexUID: index, Q: q, Limit: n, Offset: l.offset, MatchingStrategy: "all", AttributesToRetrieve: attrs}) } } add(IndexArtists, l.artists) @@ -71,6 +76,7 @@ func (h *Handler) query(ctx context.Context, q string, l limits) (found, error) switch k { case 0: f.artists = res[i].IDs + f.exactArtists = res[i].ExactIDs case 1: f.albums = res[i].IDs case 2: @@ -181,7 +187,7 @@ func (h *Handler) personal(ctx context.Context, f found, want limits) found { h.log.WarnContext(ctx, "suggest personalization failed", "err", err) } } - f.artists = personalize(f.artists, aff, want.artists) + f.artists = personalizeArtists(f.artists, f.exactArtists, aff, want.artists) f.albums = personalize(f.albums, aff, want.albums) f.tracks = personalize(f.tracks, aff, want.tracks) return f diff --git a/services/api/internal/search/translit.go b/services/api/internal/search/translit.go index 4811d6b..da1d897 100644 --- a/services/api/internal/search/translit.go +++ b/services/api/internal/search/translit.go @@ -6,6 +6,36 @@ import ( "unicode" ) +// Established spellings cannot always be derived letter by letter. Keep these +// as whole-word alternatives, retaining the mechanical transliteration too. +var pronunciationWords = map[string]string{"boulevard": "бульвар"} + +func TranslitVariants(s string) []string { + out := []string{} + if mechanical := Translit(s); mechanical != "" { + out = append(out, mechanical) + } + words := strings.Fields(strings.ToLower(s)) + changed := false + for i, word := range words { + if spoken, ok := pronunciationWords[word]; ok { + words[i], changed = spoken, true + } else if translated := Translit(word); translated != "" { + words[i] = translated + } + } + if changed { + out = append(out, strings.Join(words, " ")) + } + return out +} + +func searchKey(s string) string { + return strings.Join(strings.FieldsFunc(strings.ToLower(s), func(r rune) bool { + return !unicode.IsLetter(r) && !unicode.IsDigit(r) + }), " ") +} + // Кириллица → латиница (упрощённая ГОСТ-подобная схема: «Кино» → «kino», «Щедрин» → «shchedrin»). var cyrToLat = map[rune]string{ 'а': "a", 'б': "b", 'в': "v", 'г': "g", 'д': "d", 'е': "e", 'ё': "e", 'ж': "zh", 'з': "z", 'и': "i", diff --git a/services/api/internal/search/translit_test.go b/services/api/internal/search/translit_test.go index 75d85b3..a25378f 100644 --- a/services/api/internal/search/translit_test.go +++ b/services/api/internal/search/translit_test.go @@ -16,6 +16,23 @@ func TestTranslit(t *testing.T) { assert.Equal(t, "", Translit("1984")) } +func TestTranslitVariants(t *testing.T) { + assert.Equal(t, []string{"боулевард депо", "бульвар депо"}, TranslitVariants("BOULEVARD DEPO")) + assert.Equal(t, []string{"боулевард депо кс 161", "бульвар депо кс 161"}, TranslitVariants("Boulevard Depo X 161")) + assert.Equal(t, []string{"kino"}, TranslitVariants("Кино")) + assert.Empty(t, TranslitVariants("1984")) + assert.Equal(t, []string{"боулевардиер"}, TranslitVariants("Boulevardier"), "whole words only") +} + +func TestPersonalizeKeepsExactArtistFirst(t *testing.T) { + exact, collaboration, reversed := uuid.NewString(), uuid.NewString(), uuid.NewString() + found := []string{collaboration, reversed, exact} + aff := map[uuid.UUID]float64{uuid.MustParse(collaboration): 10000, uuid.MustParse(exact): -100} + assert.Equal(t, []string{exact, collaboration}, personalizeArtists(found, []string{exact}, aff, 2)) + assert.Equal(t, []string{exact}, personalizeArtists(found, []string{exact}, aff, 1)) + assert.Equal(t, found[:2], personalizeArtists(found, nil, nil, 2)) +} + func TestSwitchLayout(t *testing.T) { assert.Equal(t, "привет", SwitchLayout("ghbdtn")) assert.Equal(t, "hello", SwitchLayout("руддщ"))