diff --git a/services/api/internal/genius/artist_images.go b/services/api/internal/genius/artist_images.go new file mode 100644 index 0000000..c34cc2a --- /dev/null +++ b/services/api/internal/genius/artist_images.go @@ -0,0 +1,102 @@ +package genius + +import ( + "context" + "errors" + "time" + + "github.com/google/uuid" +) + +const ( + artistImageBatch = 20 + artistImageRecheck = 30 * 24 * time.Hour +) + +// artistSource — то, что нужно проходу по аватаркам от Genius-клиента. +type artistSource interface { + Search(ctx context.Context, query string) ([]Song, error) + Artist(ctx context.Context, id int64) (Artist, error) +} + +// artistImage ищет исполнителя по имени: берёт песню, у которой primary_artist совпал с именем, +// и аватарку из неё либо из отдельного запроса артиста. Пустая строка — не нашли. +func artistImage(ctx context.Context, src artistSource, name string) (string, error) { + songs, err := src.Search(ctx, name) + if err != nil { + return "", err + } + for _, song := range songs { + if !sameArtist(song.Artist.Name, name) { + continue + } + if img := imageURL(song.Artist.ImageURL); img != "" { + return img, nil + } + if song.Artist.ID <= 0 { + continue + } + a, err := src.Artist(ctx, song.Artist.ID) + if err != nil { + var he *HTTPError + if errors.As(err, &he) && he.StatusCode == 404 { + continue + } + return "", err + } + return imageURL(a.ImageURL), nil + } + return "", nil +} + +// fillArtistImages проходит по исполнителям без аватарки; каждого проверяет раз в artistImageRecheck. +func (s *Service) fillArtistImages(ctx context.Context) error { + var active bool + if err := s.pool.QueryRow(ctx, `SELECT EXISTS(SELECT 1 FROM genius_state WHERE NOT paused AND cooldown_until<=now())`).Scan(&active); err != nil || !active { + return err + } + rows, err := s.pool.Query(ctx, `SELECT id, name FROM artists + WHERE image_path IS NULL AND genius_image_url IS NULL + AND (genius_image_checked_at IS NULL OR genius_image_checked_at < now()-$1::interval) + ORDER BY genius_image_checked_at NULLS FIRST LIMIT $2`, artistImageRecheck.String(), artistImageBatch) + if err != nil { + return err + } + type row struct { + id uuid.UUID + name string + } + var batch []row + for rows.Next() { + var r row + if err := rows.Scan(&r.id, &r.name); err != nil { + rows.Close() + return err + } + batch = append(batch, r) + } + rows.Close() + if err := rows.Err(); err != nil { + return err + } + for _, r := range batch { + if ctx.Err() != nil { + return ctx.Err() + } + reqCtx, cancel := context.WithTimeout(ctx, time.Minute) + img, err := artistImage(reqCtx, s.client, r.name) + cancel() + if err != nil { + var he *HTTPError + if errors.As(err, &he) && he.StatusCode == 429 { + _, _ = s.pool.Exec(ctx, `UPDATE genius_state SET cooldown_until=GREATEST(cooldown_until,$1)`, time.Now().Add(retryDelay(he.RetryAfter))) + } + return err + } + if _, err := s.pool.Exec(ctx, `UPDATE artists SET genius_image_checked_at=now(), + genius_image_url=COALESCE(genius_image_url, NULLIF($2,'')) WHERE id=$1`, r.id, img); err != nil { + return err + } + } + return nil +} diff --git a/services/api/internal/genius/metadata.go b/services/api/internal/genius/metadata.go index d19f459..6546f04 100644 --- a/services/api/internal/genius/metadata.go +++ b/services/api/internal/genius/metadata.go @@ -16,11 +16,23 @@ func normalize(s string) string { return strings.Join(strings.FieldsFunc(strings.ToLower(s), func(r rune) bool { return !unicode.IsLetter(r) && !unicode.IsNumber(r) }), " ") } +// sameArtist сравнивает имя на Genius с нашим: у Genius часто «имя (транслитерация)», +// поэтому скобочный хвост допустим. +func sameArtist(geniusName, name string) bool { + if normalize(geniusName) == normalize(name) { + return true + } + if i := strings.Index(geniusName, "("); i > 0 && strings.HasSuffix(strings.TrimSpace(geniusName), ")") { + return normalize(geniusName[:i]) == normalize(name) + } + return false +} + // Exact normalized identity; retain version qualifiers such as live/remix. func matchSong(songs []Song, artist, title string) (Song, bool) { var matched Song for _, s := range songs { - if normalize(s.Title) == normalize(title) && normalize(s.Artist.Name) == normalize(artist) { + if normalize(s.Title) == normalize(title) && sameArtist(s.Artist.Name, artist) { if matched.ID != 0 && matched.ID != s.ID { return Song{}, false } diff --git a/services/api/internal/genius/metadata_test.go b/services/api/internal/genius/metadata_test.go index bb907c3..ab430ad 100644 --- a/services/api/internal/genius/metadata_test.go +++ b/services/api/internal/genius/metadata_test.go @@ -95,3 +95,25 @@ func TestAnnotations(t *testing.T) { require.True(t, notes[0].Verified) require.Equal(t, "https://genius.com/annotations/1", notes[0].Url) } + +type fakeArtists struct { + songs []Song + artist Artist +} + +func (f fakeArtists) Search(context.Context, string) ([]Song, error) { return f.songs, nil } +func (f fakeArtists) Artist(context.Context, int64) (Artist, error) { return f.artist, nil } + +func TestArtistImage(t *testing.T) { + ctx := context.Background() + src := fakeArtists{songs: []Song{ + {Artist: Artist{ID: 1, Name: "Other", ImageURL: "https://images.genius.com/other.jpg"}}, + {Artist: Artist{ID: 2, Name: "Некич (Nekich)"}}, + }, artist: Artist{ID: 2, ImageURL: "https://images.genius.com/nekich.jpg"}} + img, err := artistImage(ctx, src, "некич") + require.NoError(t, err) + require.Equal(t, "https://images.genius.com/nekich.jpg", img) + img, err = artistImage(ctx, src, "Unknown") + require.NoError(t, err) + require.Empty(t, img) +} diff --git a/services/api/internal/genius/service.go b/services/api/internal/genius/service.go index 75cf37c..a726255 100644 --- a/services/api/internal/genius/service.go +++ b/services/api/internal/genius/service.go @@ -48,6 +48,9 @@ func (s *Service) Run(ctx context.Context) { if err := s.drain(ctx); err != nil && ctx.Err() == nil { s.log.Error("genius worker", "err", err) } + if err := s.fillArtistImages(ctx); err != nil && ctx.Err() == nil { + s.log.Error("genius artist images", "err", err) + } select { case <-ctx.Done(): return diff --git a/services/api/migrations/00024_artist_genius_check.sql b/services/api/migrations/00024_artist_genius_check.sql new file mode 100644 index 0000000..5450de4 --- /dev/null +++ b/services/api/migrations/00024_artist_genius_check.sql @@ -0,0 +1,4 @@ +-- +goose Up +ALTER TABLE artists ADD COLUMN genius_image_checked_at timestamptz; +-- +goose Down +ALTER TABLE artists DROP COLUMN genius_image_checked_at;