Improve search relevance and artist transliteration

.gitea/workflows/ci-db.yml

@@ -75,6 +75,9 @@ jobs:
         # These suites share one database; dbtest also rolls all migrations down
         # and cleans orphan catalog rows. Do not run the packages concurrently.
         run: go test -p 1 -tags integration -count=1 -v ./internal/dbtest/ ./internal/social/ ./internal/uploads/
+      - name: Go — релевантность поиска на Meilisearch
+        working-directory: services/api
+        run: go test -tags integration -count=1 -v ./internal/search/
       - name: Go — сквозной сценарий пользователя (newApp на Postgres, Redis, Meilisearch)
         working-directory: services/api
         run: go test -tags integration -count=1 -v -run TestUserJourney ./cmd/api/

docs/library-and-storage.md

@@ -199,10 +199,17 @@ Meilisearch, индексы:
 | `albums` | `title`, `artist_name`, `title_translit` | `year`, `popularity` |
 | `tracks` | `title`, `artist_names`, `album_title`, `title_translit` | `genre`, `year`, `popularity`, `available` |
 
-- `*_translit` — транслитерация (кириллица ↔ латиница), плюс поле с **исправлением раскладки**
+- `*_translit` — варианты транслитерации (кириллица ↔ латиница), включая словарь
+  устоявшихся написаний (`Boulevard` → `бульвар`) наряду с побуквенным вариантом.
+  **Исправление раскладки**
   (`ghbdtn` → `привет`): при пустом результате клиентский запрос повторяется в альтернативной раскладке
   на стороне `api`.
-- Ранжирование: стандартные правила Meilisearch + `popularity:desc` (глобальные засчитанные прослушивания за 90 дней).
+- Запрос учитывает все слова (`matchingStrategy=all`), не отбрасывая последнее ради заполнения списка.
+  Опечатки и поиск по префиксу по-прежнему обрабатывает Meilisearch.
+- Ранжирование: `words`, `typo`, `exactness`, `proximity`, `attribute`, `sort`, `popularity:desc`.
+  Точное имя исполнителя (включая вариант транслитерации) закрепляется перед составными именами
+  в подсказках и не опускается из-за персонализации. Новые варианты появляются после переиндексации;
+  API запускает её при старте.
 - `/search/suggest` использует `multi-search` (один запрос — три индекса), отдаёт ≤ 3/3/5. Клиент делает debounce 150 мс,
   отменяет предыдущий запрос (`AbortController`).
 - Персонализация: `suggest` берёт из Meilisearch втрое больше кандидатов и переранжирует их в `api`

services/api/internal/search/indexer.go

@@ -27,7 +27,7 @@ type indexSpec struct {
 	doc      func(pgx.Rows) (map[string]any, error)
 }
 
-var rankingRules = []string{"words", "typo", "proximity", "attribute", "sort", "exactness", "popularity:desc"}
+var rankingRules = []string{"words", "typo", "exactness", "proximity", "attribute", "sort", "popularity:desc"}
 
 var specs = []indexSpec{
 	{
@@ -43,7 +43,7 @@ var specs = []indexSpec{
 			if err := r.Scan(&id, &name); err != nil {
 				return nil, err
 			}
-			return map[string]any{"id": id, "name": name, "name_translit": Translit(name), "popularity": 0}, nil
+			return map[string]any{"id": id, "name": name, "name_translit": TranslitVariants(name), "popularity": 0}, nil
 		},
 	},
 	{
@@ -65,7 +65,7 @@ var specs = []indexSpec{
 			}
 			return map[string]any{
 				"id": id, "title": title, "artist_name": artist, "year": year, "popularity": 0,
-				"title_translit": Translit(title), "artist_translit": Translit(artist),
+				"title_translit": TranslitVariants(title), "artist_translit": TranslitVariants(artist),
 			}, nil
 		},
 	},
@@ -97,12 +97,12 @@ var specs = []indexSpec{
 			}
 			translit := make([]string, 0, len(artists))
 			for _, a := range artists {
-				translit = append(translit, Translit(a))
+				translit = append(translit, TranslitVariants(a)...)
 			}
 			return map[string]any{
 				"id": id, "title": title, "album_title": album, "year": year, "artist_names": artists,
 				"genres": genres, "available": available, "popularity": 0,
-				"title_translit": Translit(title), "artist_translit": translit,
+				"title_translit": TranslitVariants(title), "artist_translit": translit,
 			}, nil
 		},
 	},

services/api/internal/search/meili.go

@@ -141,13 +141,15 @@ type MultiQuery struct {
 	Q                    string   `json:"q"`
 	Limit                int      `json:"limit"`
 	Offset               int      `json:"offset,omitempty"`
+	MatchingStrategy     string   `json:"matchingStrategy,omitempty"`
 	AttributesToRetrieve []string `json:"attributesToRetrieve"`
 }
 
 // MultiResult — ответ на один запрос: id найденных документов и оценка общего числа.
 type MultiResult struct {
-	IDs   []string
+	IDs      []string
-	Total int
+	Total    int
+	ExactIDs []string
 }
 
 // MultiSearch выполняет несколько запросов за один вызов.
@@ -155,7 +157,9 @@ func (m *Meili) MultiSearch(ctx context.Context, queries []MultiQuery) ([]MultiR
 	var res struct {
 		Results []struct {
 			Hits []struct {
-				ID string `json:"id"`
+				ID           string `json:"id"`
+				Name         string `json:"name"`
+				NameTranslit any    `json:"name_translit"`
 			} `json:"hits"`
 			EstimatedTotalHits int `json:"estimatedTotalHits"`
 		} `json:"results"`
@@ -167,8 +171,32 @@ func (m *Meili) MultiSearch(ctx context.Context, queries []MultiQuery) ([]MultiR
 	for i, r := range res.Results {
 		for _, h := range r.Hits {
 			out[i].IDs = append(out[i].IDs, h.ID)
+			if exactArtistName(queries[i].Q, h.Name, h.NameTranslit) {
+				out[i].ExactIDs = append(out[i].ExactIDs, h.ID)
+			}
 		}
 		out[i].Total = r.EstimatedTotalHits
 	}
 	return out, nil
 }
+
+func exactArtistName(query, name string, translit any) bool {
+	key := searchKey(query)
+	if key == "" {
+		return false
+	}
+	if searchKey(name) == key {
+		return true
+	}
+	switch v := translit.(type) {
+	case string: // indexes built before transliteration alternatives
+		return searchKey(v) == key
+	case []any:
+		for _, item := range v {
+			if s, ok := item.(string); ok && searchKey(s) == key {
+				return true
+			}
+		}
+	}
+	return false
+}

services/api/internal/search/meili_test.go

@@ -2,13 +2,49 @@ package search
 
 import (
 	"context"
+	"encoding/json"
 	"errors"
+	"log/slog"
 	"net/http"
 	"net/http/httptest"
 	"reflect"
 	"testing"
+
+	"github.com/stretchr/testify/require"
 )
 
+func TestQueryRequiresAllWordsAndProtectsExactArtist(t *testing.T) {
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		var body struct {
+			Queries []MultiQuery `json:"queries"`
+		}
+		require.NoError(t, json.NewDecoder(r.Body).Decode(&body))
+		require.Len(t, body.Queries, 3)
+		for _, q := range body.Queries {
+			require.Equal(t, "all", q.MatchingStrategy)
+		}
+		require.Contains(t, body.Queries[0].AttributesToRetrieve, "name_translit")
+		w.Header().Set("Content-Type", "application/json")
+		_, _ = w.Write([]byte(`{"results":[{"hits":[{"id":"collab","name":"Boulevard Depo X 161"},{"id":"exact","name":"BOULEVARD DEPO","name_translit":["боулевард депо","бульвар депо"]}],"estimatedTotalHits":2},{"hits":[]},{"hits":[]}]}`))
+	}))
+	defer srv.Close()
+	h := &Handler{meili: NewMeili(srv.URL, ""), log: slog.Default()}
+	f, err := h.query(context.Background(), "бульвар депо", limits{artists: 3, albums: 3, tracks: 5})
+	require.NoError(t, err)
+	require.Equal(t, []string{"exact"}, f.exactArtists)
+	f = h.personal(context.Background(), f, limits{artists: 1})
+	require.Equal(t, []string{"exact"}, f.artists)
+}
+
+func TestExactArtistName(t *testing.T) {
+	require.True(t, exactArtistName("boulevard depo", "BOULEVARD DEPO", nil))
+	require.True(t, exactArtistName("БУЛЬВАР  ДЕПО", "BOULEVARD DEPO", []any{"бульвар депо"}))
+	require.True(t, exactArtistName("кино", "Kino", "кино"))
+	require.False(t, exactArtistName("boulevard depo", "Depo Boulevard", nil))
+	require.False(t, exactArtistName("boulevard depo", "Boulevard Depo X 161", nil))
+	require.False(t, exactArtistName("!", "", nil))
+}
+
 func TestDeleteIndexIfExists(t *testing.T) {
 	for _, tc := range []struct {
 		name       string

services/api/internal/search/personal.go

@@ -12,6 +12,24 @@ import (
 // suggestPool — во сколько раз больше кандидатов берётся из Meilisearch для переранжирования подсказок.
 const suggestPool = 3
 
+// Exact names stay ahead of related artists, regardless of listening history.
+func personalizeArtists(found, exact []string, aff map[uuid.UUID]float64, n int) []string {
+	protected := make(map[string]bool, len(exact))
+	for _, id := range exact {
+		protected[id] = true
+	}
+	var first, rest []string
+	for _, id := range found {
+		if protected[id] {
+			first = append(first, id)
+		} else {
+			rest = append(rest, id)
+		}
+	}
+	first = append(first, personalize(rest, aff, max(0, n-len(first)))...)
+	return first[:min(n, len(first))]
+}
+
 // Affinity — насколько пользователь «знает» исполнителей, альбомы и треки: прослушивания, доигрывания,
 // лайки и дизлайки. Отрицательное — не любит. Нужен для персональных подсказок поиска.
 type Affinity struct{ pool *pgxpool.Pool }

services/api/internal/search/relevance_integration_test.go

@@ -0,0 +1,55 @@
+//go:build integration
+
+package search
+
+import (
+	"context"
+	"net/http"
+	"os"
+	"testing"
+	"time"
+
+	"github.com/google/uuid"
+	"github.com/stretchr/testify/require"
+)
+
+// Use a private index: never replace a running application's search indexes.
+func TestArtistSearchRelevance(t *testing.T) {
+	url := os.Getenv("TEST_MEILI_URL")
+	if url == "" {
+		t.Skip("TEST_MEILI_URL is not set")
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
+	defer cancel()
+	m := NewMeili(url, os.Getenv("TEST_MEILI_KEY"))
+	index := "search_test_" + uuid.New().String()
+	require.NoError(t, m.call(ctx, http.MethodPost, "/indexes", map[string]any{"uid": index, "primaryKey": "id"}))
+	t.Cleanup(func() {
+		ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
+		defer cancel()
+		require.NoError(t, m.deleteIndexIfExists(ctx, index))
+	})
+	require.NoError(t, m.call(ctx, http.MethodPatch, "/indexes/"+index+"/settings", specs[0].settings))
+	var docs []map[string]any
+	for id, name := range map[string]string{
+		"exact": "BOULEVARD DEPO", "collaboration": "Boulevard Depo X 161",
+		"reversed": "Depo Boulevard", "unrelated": "Thoma Bulwer",
+		"partial": "Станция бульвар малина",
+	} {
+		docs = append(docs, map[string]any{"id": id, "name": name, "name_translit": TranslitVariants(name), "popularity": 0})
+	}
+	require.NoError(t, m.call(ctx, http.MethodPost, "/indexes/"+index+"/documents", docs))
+	for _, query := range []string{"boulevard depo", "бульвар депо", "боулевард депо"} {
+		t.Run(query, func(t *testing.T) {
+			results, err := m.MultiSearch(ctx, []MultiQuery{{IndexUID: index, Q: query, Limit: 10,
+				MatchingStrategy: "all", AttributesToRetrieve: []string{"id", "name", "name_translit"}}})
+			require.NoError(t, err)
+			require.Len(t, results, 1)
+			require.NotEmpty(t, results[0].IDs)
+			require.Equal(t, "exact", results[0].IDs[0])
+			require.NotContains(t, results[0].IDs, "unrelated")
+			require.NotContains(t, results[0].IDs, "partial")
+			require.Contains(t, results[0].ExactIDs, "exact")
+		})
+	}
+}

services/api/internal/search/service.go

@@ -43,6 +43,7 @@ type limits struct{ artists, albums, tracks, offset int }
 
 type found struct {
 	artists, albums, tracks []string
+	exactArtists            []string
 	totals                  [3]int
 }
 
@@ -50,7 +51,11 @@ func (h *Handler) query(ctx context.Context, q string, l limits) (found, error)
 	var queries []MultiQuery
 	add := func(index string, n int) {
 		if n > 0 {
-			queries = append(queries, MultiQuery{IndexUID: index, Q: q, Limit: n, Offset: l.offset, AttributesToRetrieve: []string{"id"}})
+			attrs := []string{"id"}
+			if index == IndexArtists {
+				attrs = append(attrs, "name", "name_translit")
+			}
+			queries = append(queries, MultiQuery{IndexUID: index, Q: q, Limit: n, Offset: l.offset, MatchingStrategy: "all", AttributesToRetrieve: attrs})
 		}
 	}
 	add(IndexArtists, l.artists)
@@ -71,6 +76,7 @@ func (h *Handler) query(ctx context.Context, q string, l limits) (found, error)
 		switch k {
 		case 0:
 			f.artists = res[i].IDs
+			f.exactArtists = res[i].ExactIDs
 		case 1:
 			f.albums = res[i].IDs
 		case 2:
@@ -181,7 +187,7 @@ func (h *Handler) personal(ctx context.Context, f found, want limits) found {
 			h.log.WarnContext(ctx, "suggest personalization failed", "err", err)
 		}
 	}
-	f.artists = personalize(f.artists, aff, want.artists)
+	f.artists = personalizeArtists(f.artists, f.exactArtists, aff, want.artists)
 	f.albums = personalize(f.albums, aff, want.albums)
 	f.tracks = personalize(f.tracks, aff, want.tracks)
 	return f

services/api/internal/search/translit.go

@@ -6,6 +6,36 @@ import (
 	"unicode"
 )
 
+// Established spellings cannot always be derived letter by letter. Keep these
+// as whole-word alternatives, retaining the mechanical transliteration too.
+var pronunciationWords = map[string]string{"boulevard": "бульвар"}
+
+func TranslitVariants(s string) []string {
+	out := []string{}
+	if mechanical := Translit(s); mechanical != "" {
+		out = append(out, mechanical)
+	}
+	words := strings.Fields(strings.ToLower(s))
+	changed := false
+	for i, word := range words {
+		if spoken, ok := pronunciationWords[word]; ok {
+			words[i], changed = spoken, true
+		} else if translated := Translit(word); translated != "" {
+			words[i] = translated
+		}
+	}
+	if changed {
+		out = append(out, strings.Join(words, " "))
+	}
+	return out
+}
+
+func searchKey(s string) string {
+	return strings.Join(strings.FieldsFunc(strings.ToLower(s), func(r rune) bool {
+		return !unicode.IsLetter(r) && !unicode.IsDigit(r)
+	}), " ")
+}
+
 // Кириллица → латиница (упрощённая ГОСТ-подобная схема: «Кино» → «kino», «Щедрин» → «shchedrin»).
 var cyrToLat = map[rune]string{
 	'а': "a", 'б': "b", 'в': "v", 'г': "g", 'д': "d", 'е': "e", 'ё': "e", 'ж': "zh", 'з': "z", 'и': "i",

services/api/internal/search/translit_test.go

@@ -16,6 +16,23 @@ func TestTranslit(t *testing.T) {
 	assert.Equal(t, "", Translit("1984"))
 }
 
+func TestTranslitVariants(t *testing.T) {
+	assert.Equal(t, []string{"боулевард депо", "бульвар депо"}, TranslitVariants("BOULEVARD DEPO"))
+	assert.Equal(t, []string{"боулевард депо кс 161", "бульвар депо кс 161"}, TranslitVariants("Boulevard Depo X 161"))
+	assert.Equal(t, []string{"kino"}, TranslitVariants("Кино"))
+	assert.Empty(t, TranslitVariants("1984"))
+	assert.Equal(t, []string{"боулевардиер"}, TranslitVariants("Boulevardier"), "whole words only")
+}
+
+func TestPersonalizeKeepsExactArtistFirst(t *testing.T) {
+	exact, collaboration, reversed := uuid.NewString(), uuid.NewString(), uuid.NewString()
+	found := []string{collaboration, reversed, exact}
+	aff := map[uuid.UUID]float64{uuid.MustParse(collaboration): 10000, uuid.MustParse(exact): -100}
+	assert.Equal(t, []string{exact, collaboration}, personalizeArtists(found, []string{exact}, aff, 2))
+	assert.Equal(t, []string{exact}, personalizeArtists(found, []string{exact}, aff, 1))
+	assert.Equal(t, found[:2], personalizeArtists(found, nil, nil, 2))
+}
+
 func TestSwitchLayout(t *testing.T) {
 	assert.Equal(t, "привет", SwitchLayout("ghbdtn"))
 	assert.Equal(t, "hello", SwitchLayout("руддщ"))