From fcfda042b7df73453fad033433473202c8afade7 Mon Sep 17 00:00:00 2001 From: Vo Van Nghia Date: Wed, 1 Jan 2025 10:52:45 +0100 Subject: [PATCH 01/12] add lyrics source --- .../down.sql | 4 +- .../down.sql | 8 +- .../down.sql | 33 +++++-- .../up.sql | 32 +++++-- nghe-backend/src/file/lyrics/mod.rs | 70 ++++++++++----- nghe-backend/src/orm/lyrics.rs | 88 +++++++++---------- .../media_retrieval/get_lyrics_by_song_id.rs | 18 ++-- nghe-backend/src/scan/scanner.rs | 3 +- nghe-backend/src/schema.rs | 11 +-- 9 files changed, 164 insertions(+), 103 deletions(-) diff --git a/nghe-backend/migrations/00000000000000_diesel_initial_setup/down.sql b/nghe-backend/migrations/00000000000000_diesel_initial_setup/down.sql index d1a359fc2..196f1c321 100644 --- a/nghe-backend/migrations/00000000000000_diesel_initial_setup/down.sql +++ b/nghe-backend/migrations/00000000000000_diesel_initial_setup/down.sql @@ -2,5 +2,5 @@ -- and other internal bookkeeping. This file is safe to edit, any future -- changes will be added to existing projects as new migrations. -drop function if exists diesel_manage_updated_at (_tbl regclass); -drop function if exists diesel_set_updated_at (); +drop function if exists diesel_manage_updated_at(_tbl regclass); +drop function if exists diesel_set_updated_at(); diff --git a/nghe-backend/migrations/2024-01-19-134746_create_updated_at_leave_scanned_at/down.sql b/nghe-backend/migrations/2024-01-19-134746_create_updated_at_leave_scanned_at/down.sql index 58838b648..eb84b0b47 100644 --- a/nghe-backend/migrations/2024-01-19-134746_create_updated_at_leave_scanned_at/down.sql +++ b/nghe-backend/migrations/2024-01-19-134746_create_updated_at_leave_scanned_at/down.sql @@ -1,8 +1,8 @@ -- This file should undo anything in `up.sql` -drop function add_updated_at (_tbl regclass); +drop function add_updated_at(_tbl regclass); -drop function set_updated_at (); +drop function set_updated_at(); -drop function add_updated_at_leave_scanned_at (_tbl regclass); +drop function add_updated_at_leave_scanned_at(_tbl regclass); -drop function set_updated_at_leave_scanned_at (); +drop function set_updated_at_leave_scanned_at(); diff --git a/nghe-backend/migrations/2024-12-24-040249_change_lyrics_primary_key/down.sql b/nghe-backend/migrations/2024-12-24-040249_change_lyrics_primary_key/down.sql index 1f2409d47..33e10fd27 100644 --- a/nghe-backend/migrations/2024-12-24-040249_change_lyrics_primary_key/down.sql +++ b/nghe-backend/migrations/2024-12-24-040249_change_lyrics_primary_key/down.sql @@ -1,6 +1,29 @@ -- This file should undo anything in `up.sql` -alter table lyrics -add column lyric_hash bigint not null, -add column lyric_size integer not null, -drop constraint lyrics_pkey, -add constraint lyrics_pkey primary key (song_id, description, language, external); +drop table lyrics; + +create table +lyrics ( + song_id uuid not null, + description text not null, + language text not null, + line_values text [] not null check ( + array_position(line_values, null) is null + ), + line_starts integer [] check ( + array_position(line_starts, null) is null + ), + lyric_hash bigint not null, + lyric_size integer not null, + external bool not null, + updated_at timestamptz not null default now(), + scanned_at timestamptz not null default now(), + check (line_starts is null or array_length(line_values, 1) = array_length(line_starts, 1)), + constraint lyrics_song_id_key foreign key (song_id) references songs ( + id + ) on delete cascade, + constraint lyrics_pkey primary key ( + song_id, description, language, external + ) +); + +select add_updated_at_leave_scanned_at('lyrics'); diff --git a/nghe-backend/migrations/2024-12-24-040249_change_lyrics_primary_key/up.sql b/nghe-backend/migrations/2024-12-24-040249_change_lyrics_primary_key/up.sql index 18eb10ee8..89fde7d9a 100644 --- a/nghe-backend/migrations/2024-12-24-040249_change_lyrics_primary_key/up.sql +++ b/nghe-backend/migrations/2024-12-24-040249_change_lyrics_primary_key/up.sql @@ -1,6 +1,28 @@ -- Your SQL goes here -alter table lyrics -drop column lyric_hash, -drop column lyric_size, -drop constraint lyrics_pkey, -add constraint lyrics_pkey primary key (song_id, description, external); +drop table lyrics; + +create table +lyrics ( + id uuid not null default gen_random_uuid() constraint lyrics_pkey primary key, + song_id uuid not null, + source text, + description text, + language text not null, + durations integer [] check (array_position(durations, null) is null), + texts text [] not null check (array_position(texts, null) is null), + updated_at timestamptz not null default now(), + scanned_at timestamptz not null default now(), + check (durations is null or array_length(durations, 1) = array_length(texts, 1)), + constraint lyrics_song_id_fkey foreign key (song_id) references songs ( + id + ) on delete cascade, + constraint lyrics_song_id_source_key unique (song_id, source) +); + +select add_updated_at_leave_scanned_at('lyrics'); + +create unique index lyrics_song_id_description_key on lyrics ( + song_id, description +) nulls not distinct where (source is null); + +create index lyrics_song_id_idx on lyrics (song_id); diff --git a/nghe-backend/src/file/lyrics/mod.rs b/nghe-backend/src/file/lyrics/mod.rs index 420863fed..0f41922a1 100644 --- a/nghe-backend/src/file/lyrics/mod.rs +++ b/nghe-backend/src/file/lyrics/mod.rs @@ -2,8 +2,7 @@ use core::str; use std::borrow::Cow; use alrc::AdvancedLrc; -use diesel::dsl::{exists, select}; -use diesel::{ExpressionMethods, QueryDsl}; +use diesel::{ExpressionMethods, OptionalExtension}; use diesel_async::RunQueryDsl; use isolang::Language; use lofty::id3::v2::{BinaryFrame, SynchronizedTextFrame, UnsynchronizedTextFrame}; @@ -12,7 +11,8 @@ use uuid::Uuid; use crate::database::Database; use crate::filesystem::Trait as _; -use crate::orm::{lyrics, songs}; +use crate::orm::lyrics; +use crate::orm::upsert::Insert; use crate::{Error, error, filesystem}; #[derive(Debug)] @@ -102,20 +102,35 @@ impl<'a> Lyrics<'a> { impl Lyrics<'_> { pub const EXTERNAL_EXTENSION: &'static str = "lrc"; - pub async fn upsert(&self, database: &Database, foreign: lyrics::Foreign) -> Result<(), Error> { - lyrics::Upsert { foreign, data: self.try_into()? }.upsert(database).await + pub async fn upsert( + &self, + database: &Database, + foreign: lyrics::Foreign, + source: Option>, + ) -> Result { + lyrics::Upsert { + foreign, + source: source.as_ref().map(AsRef::as_ref).map(Cow::Borrowed), + data: self.try_into()?, + } + .insert(database) + .await } - pub async fn query_external(database: &Database, song_id: Uuid) -> Result { - select(exists( - lyrics::table - .inner_join(songs::table) - .filter(lyrics::external) - .filter(songs::id.eq(song_id)), - )) - .get_result(&mut database.get().await?) - .await - .map_err(Error::from) + pub async fn set_source_scanned_at( + database: &Database, + song_id: Uuid, + source: impl AsRef, + ) -> Result, Error> { + diesel::update(lyrics::table) + .filter(lyrics::song_id.eq(song_id)) + .filter(lyrics::source.eq(source.as_ref())) + .set(lyrics::scanned_at.eq(crate::time::now().await)) + .returning(lyrics::id) + .get_result(&mut database.get().await?) + .await + .optional() + .map_err(Error::from) } pub async fn load( @@ -135,15 +150,22 @@ impl Lyrics<'_> { full: bool, song_id: Uuid, song_path: Utf8TypedPath<'_>, - ) -> Result<(), Error> { - if (full || !Self::query_external(database, song_id).await?) - && let Some(lyrics) = - Self::load(filesystem, song_path.with_extension(Self::EXTERNAL_EXTENSION).to_path()) - .await? - { - lyrics.upsert(database, lyrics::Foreign { song_id, external: true }).await?; - } - Ok(()) + ) -> Result, Error> { + let path = song_path.with_extension(Self::EXTERNAL_EXTENSION); + let path = path.to_path(); + + Ok( + if !full + && let Some(lyrics_id) = + Self::set_source_scanned_at(database, song_id, path).await? + { + Some(lyrics_id) + } else if let Some(lyrics) = Self::load(filesystem, path).await? { + Some(lyrics.upsert(database, lyrics::Foreign { song_id }, Some(path)).await?) + } else { + None + }, + ) } } diff --git a/nghe-backend/src/orm/lyrics.rs b/nghe-backend/src/orm/lyrics.rs index f754e1f99..1dcc1ddef 100644 --- a/nghe-backend/src/orm/lyrics.rs +++ b/nghe-backend/src/orm/lyrics.rs @@ -12,24 +12,17 @@ pub use crate::schema::lyrics::{self, *}; #[derive(Debug, Queryable, Selectable, Insertable, AsChangeset)] #[diesel(table_name = lyrics, check_for_backend(crate::orm::Type))] #[diesel(treat_none_as_null = true)] -pub struct Lyrics<'a> { +pub struct Data<'a> { + pub description: Option>, pub language: Cow<'a, str>, - #[diesel(select_expression = sql("lyrics.line_starts line_starts"))] + #[diesel(select_expression = sql("lyrics.durations durations"))] #[diesel(select_expression_type = SqlLiteral>> )] - pub line_starts: Option>, - #[diesel(select_expression = sql("lyrics.line_values line_values"))] + pub durations: Option>, + #[diesel(select_expression = sql("lyrics.texts texts"))] #[diesel(select_expression_type = SqlLiteral>)] - pub line_values: Vec>, -} - -#[derive(Debug, Insertable)] -#[diesel(table_name = lyrics)] -#[diesel(check_for_backend(diesel::pg::Pg))] -#[cfg_attr(test, derive(Queryable, Selectable))] -pub struct Key<'a> { - pub description: Cow<'a, str>, + pub texts: Vec>, } #[derive(Debug, Insertable)] @@ -37,18 +30,6 @@ pub struct Key<'a> { #[diesel(treat_none_as_null = true)] pub struct Foreign { pub song_id: Uuid, - pub external: bool, -} - -#[derive(Debug, Insertable)] -#[diesel(table_name = lyrics, check_for_backend(crate::orm::Type))] -#[diesel(treat_none_as_null = true)] -#[cfg_attr(test, derive(Queryable, Selectable))] -pub struct Data<'a> { - #[diesel(embed)] - pub key: Key<'a>, - #[diesel(embed)] - pub lyrics: Lyrics<'a>, } #[derive(Debug, Insertable)] @@ -57,33 +38,50 @@ pub struct Data<'a> { pub struct Upsert<'a> { #[diesel(embed)] pub foreign: Foreign, + pub source: Option>, #[diesel(embed)] pub data: Data<'a>, } mod upsert { - use diesel::ExpressionMethods; + use diesel::{DecoratableTarget, ExpressionMethods}; use diesel_async::RunQueryDsl; + use uuid::Uuid; use super::{Upsert, lyrics}; use crate::Error; use crate::database::Database; - impl Upsert<'_> { - pub async fn upsert(&self, database: &Database) -> Result<(), Error> { - diesel::insert_into(lyrics::table) - .values(self) - .on_conflict((lyrics::song_id, lyrics::external, lyrics::description)) - .do_update() - .set((&self.data.lyrics, lyrics::scanned_at.eq(crate::time::now().await))) - .execute(&mut database.get().await?) - .await?; - Ok(()) + impl crate::orm::upsert::Insert for Upsert<'_> { + async fn insert(&self, database: &Database) -> Result { + if self.source.is_some() { + diesel::insert_into(lyrics::table) + .values(self) + .on_conflict((lyrics::song_id, lyrics::source)) + .do_update() + .set((&self.data, lyrics::scanned_at.eq(crate::time::now().await))) + .returning(lyrics::id) + .get_result(&mut database.get().await?) + .await + } else { + diesel::insert_into(lyrics::table) + .values(self) + .on_conflict((lyrics::song_id, lyrics::description)) + .filter_target(lyrics::source.is_null()) + .do_update() + .set((&self.data, lyrics::scanned_at.eq(crate::time::now().await))) + .returning(lyrics::id) + .get_result(&mut database.get().await?) + .await + } + .map_err(Error::from) } } } mod convert { + use std::borrow::Cow; + use crate::Error; use crate::file::lyrics::{Lines, Lyrics}; use crate::orm::lyrics; @@ -92,7 +90,7 @@ mod convert { type Error = Error; fn try_from(value: &'a Lyrics<'_>) -> Result { - let (line_starts, line_values) = match &value.lines { + let (durations, texts) = match &value.lines { Lines::Unsync(lines) => { (None, lines.iter().map(|line| line.as_str().into()).collect()) } @@ -108,18 +106,12 @@ mod convert { (Some(durations), texts) } }; - let lyrics = lyrics::Lyrics { + Ok(Self { + description: value.description.as_deref().map(Cow::Borrowed), language: value.language.to_639_3().into(), - line_starts, - line_values, - }; - let key = lyrics::Key { - description: value - .description - .as_ref() - .map_or_else(|| "".into(), |description| description.as_str().into()), - }; - Ok(Self { key, lyrics }) + durations, + texts, + }) } } } diff --git a/nghe-backend/src/route/media_retrieval/get_lyrics_by_song_id.rs b/nghe-backend/src/route/media_retrieval/get_lyrics_by_song_id.rs index 89903fc4c..5f3de0b43 100644 --- a/nghe-backend/src/route/media_retrieval/get_lyrics_by_song_id.rs +++ b/nghe-backend/src/route/media_retrieval/get_lyrics_by_song_id.rs @@ -14,7 +14,7 @@ pub async fn handler(database: &Database, request: Request) -> Result Result Result<_, Error> { - if let Some(line_starts) = lyrics.line_starts { + if let Some(durations) = lyrics.durations { Ok(Lyrics { lang: lyrics.language.into_owned(), synced: true, - line: line_starts + line: durations .into_iter() - .zip_longest(lyrics.line_values.into_iter()) + .zip_longest(lyrics.texts.into_iter()) .map(|iter| { - if let EitherOrBoth::Both(start, value) = iter { + if let EitherOrBoth::Both(duration, text) = iter { Ok(Line { - start: Some(start.try_into()?), - value: value.into_owned(), + start: Some(duration.try_into()?), + value: text.into_owned(), }) } else { error::Kind::DatabaseCorruptionDetected.into() @@ -46,9 +46,9 @@ pub async fn handler(database: &Database, request: Request) -> Result Scanner<'db, 'fs, 'mf> { song_id: Uuid, song_path: Utf8TypedPath<'_>, ) -> Result<(), Error> { - lyrics::Lyrics::scan(&self.database, &self.filesystem, false, song_id, song_path).await + lyrics::Lyrics::scan(&self.database, &self.filesystem, false, song_id, song_path).await?; + Ok(()) } #[cfg_attr( diff --git a/nghe-backend/src/schema.rs b/nghe-backend/src/schema.rs index 8f828136d..44e49b92a 100644 --- a/nghe-backend/src/schema.rs +++ b/nghe-backend/src/schema.rs @@ -98,13 +98,14 @@ diesel::table! { use diesel::sql_types::*; use diesel_full_text_search::*; - lyrics (song_id, description, external) { + lyrics (id) { + id -> Uuid, song_id -> Uuid, - description -> Text, + source -> Nullable, + description -> Nullable, language -> Text, - line_values -> Array>, - line_starts -> Nullable>>, - external -> Bool, + durations -> Nullable>>, + texts -> Array>, updated_at -> Timestamptz, scanned_at -> Timestamptz, } From ada2d4bd6662e0741627a72116d438892f324fde Mon Sep 17 00:00:00 2001 From: Vo Van Nghia Date: Wed, 1 Jan 2025 12:44:24 +0100 Subject: [PATCH 02/12] add test for lyrics --- nghe-backend/src/file/lyrics/mod.rs | 16 +++++++ nghe-backend/src/orm/lyrics.rs | 27 +++++++++++ .../src/test/mock_impl/information.rs | 45 ++++++++++++++++--- .../src/test/mock_impl/music_folder.rs | 8 ++-- 4 files changed, 85 insertions(+), 11 deletions(-) diff --git a/nghe-backend/src/file/lyrics/mod.rs b/nghe-backend/src/file/lyrics/mod.rs index 0f41922a1..7a7fce559 100644 --- a/nghe-backend/src/file/lyrics/mod.rs +++ b/nghe-backend/src/file/lyrics/mod.rs @@ -174,10 +174,12 @@ impl Lyrics<'_> { mod test { use std::fmt::Display; + use diesel::{QueryDsl, SelectableHelper}; use fake::{Dummy, Fake, Faker}; use itertools::Itertools; use super::*; + use crate::test::Mock; impl FromIterator for Lines<'_> { fn from_iter>(iter: T) -> Self { @@ -219,6 +221,20 @@ mod test { } } + impl Lyrics<'static> { + pub async fn query_source(mock: &Mock, id: Uuid) -> Option { + lyrics::table + .filter(lyrics::song_id.eq(id)) + .filter(lyrics::source.is_not_null()) + .select(lyrics::Data::as_select()) + .get_result(&mut mock.get().await) + .await + .optional() + .unwrap() + .map(Self::from) + } + } + impl Dummy for Lyrics<'_> { fn dummy_with_rng(config: &Faker, rng: &mut R) -> Self { if config.fake_with_rng(rng) { Self::fake_sync() } else { Self::fake_unsync() } diff --git a/nghe-backend/src/orm/lyrics.rs b/nghe-backend/src/orm/lyrics.rs index 1dcc1ddef..37e5867f9 100644 --- a/nghe-backend/src/orm/lyrics.rs +++ b/nghe-backend/src/orm/lyrics.rs @@ -115,3 +115,30 @@ mod convert { } } } + +#[cfg(test)] +#[coverage(off)] +mod test { + use std::borrow::Cow; + + use crate::file::lyrics::Lyrics; + use crate::orm::lyrics; + + impl From> for Lyrics<'static> { + fn from(value: lyrics::Data<'_>) -> Self { + Self { + description: value.description.map(Cow::into_owned).map(Cow::Owned), + language: value.language.parse().unwrap(), + lines: if let Some(durations) = value.durations { + durations + .into_iter() + .zip(value.texts) + .map(|(duration, text)| (duration.try_into().unwrap(), text.into_owned())) + .collect() + } else { + value.texts.into_iter().map(Cow::into_owned).collect() + }, + } + } + } +} diff --git a/nghe-backend/src/test/mock_impl/information.rs b/nghe-backend/src/test/mock_impl/information.rs index bbe3c1921..b536fd990 100644 --- a/nghe-backend/src/test/mock_impl/information.rs +++ b/nghe-backend/src/test/mock_impl/information.rs @@ -7,21 +7,22 @@ use fake::{Fake, Faker}; use uuid::Uuid; use super::music_folder; -use crate::file::{self, audio, picture}; +use crate::file::{self, audio, lyrics, picture}; use crate::orm::{albums, songs}; use crate::test::assets; use crate::test::file::audio::dump::Metadata as _; use crate::test::filesystem::Trait as _; #[derive(Debug, Clone, PartialEq, Eq)] -pub struct Mock<'info, 'picture, 'path> { +pub struct Mock<'info, 'picture, 'lyrics, 'path> { pub information: audio::Information<'info>, pub dir_picture: Option>, + pub lyrics: Option>, pub relative_path: Cow<'path, str>, } #[bon::bon] -impl Mock<'static, 'static, 'static> { +impl Mock<'static, 'static, 'static, 'static> { pub async fn query_upsert(mock: &super::Mock, id: Uuid) -> songs::Upsert<'static> { songs::table .filter(songs::id.eq(id)) @@ -43,6 +44,7 @@ impl Mock<'static, 'static, 'static> { let genres = audio::Genres::query(mock, id).await; let picture = picture::Picture::query_song(mock, id).await; + let lyrics = lyrics::Lyrics::query_source(mock, id).await; let dir_picture = picture::Picture::query_album(mock, album_id).await; Self { @@ -57,6 +59,7 @@ impl Mock<'static, 'static, 'static> { property: upsert.data.property.try_into().unwrap(), file: upsert.data.file.try_into().unwrap(), }, + lyrics, dir_picture, relative_path: upsert.relative_path, } @@ -77,6 +80,7 @@ impl Mock<'static, 'static, 'static> { format: Option, file_property: Option>, property: Option, + lyrics: Option>>, dir_picture: Option>>, relative_path: Option>, ) -> Self { @@ -93,25 +97,29 @@ impl Mock<'static, 'static, 'static> { }); let property = property.unwrap_or_else(|| audio::Property::default(file.format)); + let lyrics = lyrics + .unwrap_or_else(|| if Faker.fake() { Some(lyrics::Lyrics::fake_sync()) } else { None }); let dir_picture = dir_picture.unwrap_or_else(|| Faker.fake()); let relative_path = relative_path.map_or_else(|| Faker.fake::().into(), std::convert::Into::into); Self { information: audio::Information { metadata, property, file }, + lyrics, dir_picture, relative_path, } } } -impl Mock<'_, '_, '_> { +impl Mock<'_, '_, '_, '_> { pub async fn upsert( &self, music_folder: &music_folder::Mock<'_>, song_id: impl Into>, ) -> Uuid { let database = music_folder.database(); + let dir_picture_id = if let Some(ref dir) = music_folder.config.cover_art.dir && let Some(ref picture) = self.dir_picture { @@ -120,7 +128,8 @@ impl Mock<'_, '_, '_> { None }; - self.information + let song_id = self + .information .upsert( database, &music_folder.config, @@ -132,7 +141,20 @@ impl Mock<'_, '_, '_> { song_id, ) .await - .unwrap() + .unwrap(); + + if let Some(lyrics) = self.lyrics.as_ref() { + lyrics + .upsert( + database, + crate::orm::lyrics::Foreign { song_id }, + Some(music_folder.absolutize(&self.relative_path)), + ) + .await + .unwrap(); + } + + song_id } pub async fn upsert_mock( @@ -167,6 +189,15 @@ impl Mock<'_, '_, '_> { let filesystem = &music_folder.to_impl(); filesystem.write(path, &data).await; + if let Some(lyrics) = self.lyrics.as_ref() { + filesystem + .write( + path.with_extension(lyrics::Lyrics::EXTERNAL_EXTENSION).to_path(), + lyrics.to_string().as_bytes(), + ) + .await; + } + let cover_art_config = &music_folder.config.cover_art; let parent = path.parent().unwrap(); let dir_picture = if let Some(picture) = @@ -196,7 +227,7 @@ impl Mock<'_, '_, '_> { } } -impl<'path> Mock<'_, '_, 'path> { +impl<'path> Mock<'_, '_, '_, 'path> { pub fn with_relative_path(self, relative_path: Cow<'path, str>) -> Self { Self { relative_path, ..self } } diff --git a/nghe-backend/src/test/mock_impl/music_folder.rs b/nghe-backend/src/test/mock_impl/music_folder.rs index 2b2653d27..0f2703593 100644 --- a/nghe-backend/src/test/mock_impl/music_folder.rs +++ b/nghe-backend/src/test/mock_impl/music_folder.rs @@ -24,8 +24,8 @@ use crate::test::filesystem::{self, Trait as _}; pub struct Mock<'a> { mock: &'a super::Mock, music_folder: music_folders::MusicFolder<'static>, - pub filesystem: IndexMap>, - pub database: IndexMap>, + pub filesystem: IndexMap>, + pub database: IndexMap>, pub config: scanner::Config, } @@ -300,7 +300,7 @@ impl<'a> Mock<'a> { pub async fn query_filesystem( &self, - ) -> IndexMap> { + ) -> IndexMap> { let song_ids: Vec<_> = stream::iter(0..self.filesystem.len()) .then(async |index| self.optional_song_id_filesystem(index).await) .filter_map(std::convert::identity) @@ -319,7 +319,7 @@ impl<'a> Mock<'a> { mod duration { use super::*; - impl audio::duration::Trait for IndexMap> { + impl audio::duration::Trait for IndexMap> { fn duration(&self) -> audio::Duration { self.values().map(|information| information.information.property.duration).sum() } From d1acaa0a786d3ce3de8c9aef9f9a88cb23c2e635 Mon Sep 17 00:00:00 2001 From: Vo Van Nghia Date: Wed, 1 Jan 2025 12:48:14 +0100 Subject: [PATCH 03/12] rename lyrics to lyrics --- nghe-backend/src/file/{lyrics => lyric}/mod.rs | 0 nghe-backend/src/file/mod.rs | 2 +- nghe-backend/src/orm/lyrics.rs | 4 ++-- nghe-backend/src/scan/scanner.rs | 4 ++-- nghe-backend/src/test/mock_impl/information.rs | 16 ++++++++-------- nghe-backend/src/test/mock_impl/music_folder.rs | 4 +++- 6 files changed, 16 insertions(+), 14 deletions(-) rename nghe-backend/src/file/{lyrics => lyric}/mod.rs (100%) diff --git a/nghe-backend/src/file/lyrics/mod.rs b/nghe-backend/src/file/lyric/mod.rs similarity index 100% rename from nghe-backend/src/file/lyrics/mod.rs rename to nghe-backend/src/file/lyric/mod.rs diff --git a/nghe-backend/src/file/mod.rs b/nghe-backend/src/file/mod.rs index db66423e0..d82f20f0a 100644 --- a/nghe-backend/src/file/mod.rs +++ b/nghe-backend/src/file/mod.rs @@ -1,5 +1,5 @@ pub mod audio; -pub mod lyrics; +pub mod lyric; pub mod picture; use std::num::{NonZero, NonZeroU32, NonZeroU64}; diff --git a/nghe-backend/src/orm/lyrics.rs b/nghe-backend/src/orm/lyrics.rs index 37e5867f9..7490bc45a 100644 --- a/nghe-backend/src/orm/lyrics.rs +++ b/nghe-backend/src/orm/lyrics.rs @@ -83,7 +83,7 @@ mod convert { use std::borrow::Cow; use crate::Error; - use crate::file::lyrics::{Lines, Lyrics}; + use crate::file::lyric::{Lines, Lyrics}; use crate::orm::lyrics; impl<'a> TryFrom<&'a Lyrics<'_>> for lyrics::Data<'a> { @@ -121,7 +121,7 @@ mod convert { mod test { use std::borrow::Cow; - use crate::file::lyrics::Lyrics; + use crate::file::lyric::Lyrics; use crate::orm::lyrics; impl From> for Lyrics<'static> { diff --git a/nghe-backend/src/scan/scanner.rs b/nghe-backend/src/scan/scanner.rs index dedebf057..3181eb639 100644 --- a/nghe-backend/src/scan/scanner.rs +++ b/nghe-backend/src/scan/scanner.rs @@ -15,7 +15,7 @@ use typed_path::Utf8TypedPath; use uuid::Uuid; use crate::database::Database; -use crate::file::{self, File, audio, lyrics, picture}; +use crate::file::{self, File, audio, lyric, picture}; use crate::filesystem::{self, Entry, Filesystem, Trait, entry}; use crate::integration::Informant; use crate::orm::{albums, music_folders, songs}; @@ -178,7 +178,7 @@ impl<'db, 'fs, 'mf> Scanner<'db, 'fs, 'mf> { song_id: Uuid, song_path: Utf8TypedPath<'_>, ) -> Result<(), Error> { - lyrics::Lyrics::scan(&self.database, &self.filesystem, false, song_id, song_path).await?; + lyric::Lyrics::scan(&self.database, &self.filesystem, false, song_id, song_path).await?; Ok(()) } diff --git a/nghe-backend/src/test/mock_impl/information.rs b/nghe-backend/src/test/mock_impl/information.rs index b536fd990..a90751cf2 100644 --- a/nghe-backend/src/test/mock_impl/information.rs +++ b/nghe-backend/src/test/mock_impl/information.rs @@ -7,8 +7,8 @@ use fake::{Fake, Faker}; use uuid::Uuid; use super::music_folder; -use crate::file::{self, audio, lyrics, picture}; -use crate::orm::{albums, songs}; +use crate::file::{self, audio, lyric, picture}; +use crate::orm::{albums, lyrics, songs}; use crate::test::assets; use crate::test::file::audio::dump::Metadata as _; use crate::test::filesystem::Trait as _; @@ -17,7 +17,7 @@ use crate::test::filesystem::Trait as _; pub struct Mock<'info, 'picture, 'lyrics, 'path> { pub information: audio::Information<'info>, pub dir_picture: Option>, - pub lyrics: Option>, + pub lyrics: Option>, pub relative_path: Cow<'path, str>, } @@ -44,7 +44,7 @@ impl Mock<'static, 'static, 'static, 'static> { let genres = audio::Genres::query(mock, id).await; let picture = picture::Picture::query_song(mock, id).await; - let lyrics = lyrics::Lyrics::query_source(mock, id).await; + let lyrics = lyric::Lyrics::query_source(mock, id).await; let dir_picture = picture::Picture::query_album(mock, album_id).await; Self { @@ -80,7 +80,7 @@ impl Mock<'static, 'static, 'static, 'static> { format: Option, file_property: Option>, property: Option, - lyrics: Option>>, + lyrics: Option>>, dir_picture: Option>>, relative_path: Option>, ) -> Self { @@ -98,7 +98,7 @@ impl Mock<'static, 'static, 'static, 'static> { let property = property.unwrap_or_else(|| audio::Property::default(file.format)); let lyrics = lyrics - .unwrap_or_else(|| if Faker.fake() { Some(lyrics::Lyrics::fake_sync()) } else { None }); + .unwrap_or_else(|| if Faker.fake() { Some(lyric::Lyrics::fake_sync()) } else { None }); let dir_picture = dir_picture.unwrap_or_else(|| Faker.fake()); let relative_path = relative_path.map_or_else(|| Faker.fake::().into(), std::convert::Into::into); @@ -147,7 +147,7 @@ impl Mock<'_, '_, '_, '_> { lyrics .upsert( database, - crate::orm::lyrics::Foreign { song_id }, + lyrics::Foreign { song_id }, Some(music_folder.absolutize(&self.relative_path)), ) .await @@ -192,7 +192,7 @@ impl Mock<'_, '_, '_, '_> { if let Some(lyrics) = self.lyrics.as_ref() { filesystem .write( - path.with_extension(lyrics::Lyrics::EXTERNAL_EXTENSION).to_path(), + path.with_extension(lyric::Lyrics::EXTERNAL_EXTENSION).to_path(), lyrics.to_string().as_bytes(), ) .await; diff --git a/nghe-backend/src/test/mock_impl/music_folder.rs b/nghe-backend/src/test/mock_impl/music_folder.rs index 0f2703593..37bccc7de 100644 --- a/nghe-backend/src/test/mock_impl/music_folder.rs +++ b/nghe-backend/src/test/mock_impl/music_folder.rs @@ -317,9 +317,11 @@ impl<'a> Mock<'a> { } mod duration { + use audio::duration::Trait; + use super::*; - impl audio::duration::Trait for IndexMap> { + impl Trait for IndexMap> { fn duration(&self) -> audio::Duration { self.values().map(|information| information.information.property.duration).sum() } From d7c5fca187bbe0eea269fc36f4f4025f139ae6a2 Mon Sep 17 00:00:00 2001 From: Vo Van Nghia Date: Wed, 1 Jan 2025 12:52:19 +0100 Subject: [PATCH 04/12] add lyrics cleanup one --- nghe-backend/src/file/audio/information.rs | 2 ++ nghe-backend/src/file/lyric/mod.rs | 16 +++++++++++++++- 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/nghe-backend/src/file/audio/information.rs b/nghe-backend/src/file/audio/information.rs index 78f76c993..b08aeb9a5 100644 --- a/nghe-backend/src/file/audio/information.rs +++ b/nghe-backend/src/file/audio/information.rs @@ -8,6 +8,7 @@ use uuid::Uuid; use super::{Album, Artists, Genres}; use crate::database::Database; +use crate::file::lyric; use crate::orm::upsert::Upsert as _; use crate::orm::{albums, songs}; use crate::scan::scanner; @@ -101,6 +102,7 @@ impl Information<'_> { ) -> Result<(), Error> { Artists::cleanup_one(database, started_at, song_id).await?; Genres::cleanup_one(database, started_at, song_id).await?; + lyric::Lyrics::cleanup_one(database, started_at, song_id).await?; Ok(()) } diff --git a/nghe-backend/src/file/lyric/mod.rs b/nghe-backend/src/file/lyric/mod.rs index 7a7fce559..8fbf997a6 100644 --- a/nghe-backend/src/file/lyric/mod.rs +++ b/nghe-backend/src/file/lyric/mod.rs @@ -117,7 +117,7 @@ impl Lyrics<'_> { .await } - pub async fn set_source_scanned_at( + async fn set_source_scanned_at( database: &Database, song_id: Uuid, source: impl AsRef, @@ -167,6 +167,20 @@ impl Lyrics<'_> { }, ) } + + pub async fn cleanup_one( + database: &Database, + started_at: time::OffsetDateTime, + song_id: Uuid, + ) -> Result<(), Error> { + // Delete all lyrics of a song which haven't been refreshed since timestamp. + diesel::delete(lyrics::table) + .filter(lyrics::song_id.eq(song_id)) + .filter(lyrics::scanned_at.lt(started_at)) + .execute(&mut database.get().await?) + .await?; + Ok(()) + } } #[cfg(test)] From 37ce8e1b5b1040d447c149deb72bbd32684cb03c Mon Sep 17 00:00:00 2001 From: Vo Van Nghia Date: Wed, 1 Jan 2025 12:55:53 +0100 Subject: [PATCH 05/12] rename lyrics to lyric --- nghe-backend/src/file/audio/information.rs | 2 +- nghe-backend/src/file/lyric/mod.rs | 38 +++++++++---------- nghe-backend/src/orm/lyrics.rs | 10 ++--- nghe-backend/src/scan/scanner.rs | 2 +- .../src/test/mock_impl/information.rs | 10 ++--- 5 files changed, 31 insertions(+), 31 deletions(-) diff --git a/nghe-backend/src/file/audio/information.rs b/nghe-backend/src/file/audio/information.rs index b08aeb9a5..5b6e42592 100644 --- a/nghe-backend/src/file/audio/information.rs +++ b/nghe-backend/src/file/audio/information.rs @@ -102,7 +102,7 @@ impl Information<'_> { ) -> Result<(), Error> { Artists::cleanup_one(database, started_at, song_id).await?; Genres::cleanup_one(database, started_at, song_id).await?; - lyric::Lyrics::cleanup_one(database, started_at, song_id).await?; + lyric::Lyric::cleanup_one(database, started_at, song_id).await?; Ok(()) } diff --git a/nghe-backend/src/file/lyric/mod.rs b/nghe-backend/src/file/lyric/mod.rs index 8fbf997a6..630cda6aa 100644 --- a/nghe-backend/src/file/lyric/mod.rs +++ b/nghe-backend/src/file/lyric/mod.rs @@ -24,7 +24,7 @@ pub enum Lines<'a> { #[derive(Debug)] #[cfg_attr(test, derive(PartialEq, Eq, Clone))] -pub struct Lyrics<'a> { +pub struct Lyric<'a> { pub description: Option>, pub language: Language, pub lines: Lines<'a>, @@ -42,7 +42,7 @@ impl FromIterator<(u32, String)> for Lines<'_> { } } -impl<'a> TryFrom<&'a UnsynchronizedTextFrame<'_>> for Lyrics<'a> { +impl<'a> TryFrom<&'a UnsynchronizedTextFrame<'_>> for Lyric<'a> { type Error = Error; fn try_from(frame: &'a UnsynchronizedTextFrame<'_>) -> Result { @@ -54,7 +54,7 @@ impl<'a> TryFrom<&'a UnsynchronizedTextFrame<'_>> for Lyrics<'a> { } } -impl<'a> TryFrom<&'a BinaryFrame<'_>> for Lyrics<'a> { +impl<'a> TryFrom<&'a BinaryFrame<'_>> for Lyric<'a> { type Error = Error; fn try_from(frame: &'a BinaryFrame<'_>) -> Result { @@ -67,7 +67,7 @@ impl<'a> TryFrom<&'a BinaryFrame<'_>> for Lyrics<'a> { } } -impl<'a> Lyrics<'a> { +impl<'a> Lyric<'a> { pub fn from_unsync_text(content: &'a str) -> Self { Self { description: None, @@ -99,7 +99,7 @@ impl<'a> Lyrics<'a> { } } -impl Lyrics<'_> { +impl Lyric<'_> { pub const EXTERNAL_EXTENSION: &'static str = "lrc"; pub async fn upsert( @@ -201,7 +201,7 @@ mod test { } } - impl Lyrics<'_> { + impl Lyric<'_> { pub fn is_sync(&self) -> bool { matches!(self.lines, Lines::Sync(_)) } @@ -235,7 +235,7 @@ mod test { } } - impl Lyrics<'static> { + impl Lyric<'static> { pub async fn query_source(mock: &Mock, id: Uuid) -> Option { lyrics::table .filter(lyrics::song_id.eq(id)) @@ -249,13 +249,13 @@ mod test { } } - impl Dummy for Lyrics<'_> { + impl Dummy for Lyric<'_> { fn dummy_with_rng(config: &Faker, rng: &mut R) -> Self { if config.fake_with_rng(rng) { Self::fake_sync() } else { Self::fake_unsync() } } } - impl Display for Lyrics<'_> { + impl Display for Lyric<'_> { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match &self.lines { Lines::Unsync(lines) => write!(f, "{}", lines.join("\n")), @@ -292,17 +292,17 @@ mod tests { #[rstest] fn test_lyrics_roundtrip(#[values(true, false)] sync: bool) { if sync { - let lyrics = Lyrics::fake_sync(); - assert_eq!(lyrics, Lyrics::from_sync_text(&lyrics.to_string()).unwrap()); + let lyrics = Lyric::fake_sync(); + assert_eq!(lyrics, Lyric::from_sync_text(&lyrics.to_string()).unwrap()); } else { - let lyrics = Lyrics::fake_unsync(); - assert_eq!(lyrics, Lyrics::from_unsync_text(&lyrics.to_string())); + let lyrics = Lyric::fake_unsync(); + assert_eq!(lyrics, Lyric::from_unsync_text(&lyrics.to_string())); } } #[rstest] - #[case("sync.lrc", Lyrics { - description: Some("Lyrics".to_owned().into()), + #[case("sync.lrc", Lyric { + description: Some("Lyric".to_owned().into()), language: Language::Eng, lines: vec![ (1020_u32, "Hello hi".to_owned()), @@ -312,7 +312,7 @@ mod tests { .into_iter() .collect() })] - #[case("unsync.txt", Lyrics { + #[case("unsync.txt", Lyric { description: None, language: Language::Und, lines: vec![ @@ -323,12 +323,12 @@ mod tests { .into_iter() .collect() })] - fn test_from_text(#[case] filename: &str, #[case] lyrics: Lyrics<'_>) { + fn test_from_text(#[case] filename: &str, #[case] lyrics: Lyric<'_>) { let content = std::fs::read_to_string(assets::dir().join("lyrics").join(filename)).unwrap(); let parsed = if lyrics.is_sync() { - Lyrics::from_sync_text(&content).unwrap() + Lyric::from_sync_text(&content).unwrap() } else { - Lyrics::from_unsync_text(&content) + Lyric::from_unsync_text(&content) }; assert_eq!(parsed, lyrics); } diff --git a/nghe-backend/src/orm/lyrics.rs b/nghe-backend/src/orm/lyrics.rs index 7490bc45a..9a8aac72a 100644 --- a/nghe-backend/src/orm/lyrics.rs +++ b/nghe-backend/src/orm/lyrics.rs @@ -83,13 +83,13 @@ mod convert { use std::borrow::Cow; use crate::Error; - use crate::file::lyric::{Lines, Lyrics}; + use crate::file::lyric::{Lines, Lyric}; use crate::orm::lyrics; - impl<'a> TryFrom<&'a Lyrics<'_>> for lyrics::Data<'a> { + impl<'a> TryFrom<&'a Lyric<'_>> for lyrics::Data<'a> { type Error = Error; - fn try_from(value: &'a Lyrics<'_>) -> Result { + fn try_from(value: &'a Lyric<'_>) -> Result { let (durations, texts) = match &value.lines { Lines::Unsync(lines) => { (None, lines.iter().map(|line| line.as_str().into()).collect()) @@ -121,10 +121,10 @@ mod convert { mod test { use std::borrow::Cow; - use crate::file::lyric::Lyrics; + use crate::file::lyric::Lyric; use crate::orm::lyrics; - impl From> for Lyrics<'static> { + impl From> for Lyric<'static> { fn from(value: lyrics::Data<'_>) -> Self { Self { description: value.description.map(Cow::into_owned).map(Cow::Owned), diff --git a/nghe-backend/src/scan/scanner.rs b/nghe-backend/src/scan/scanner.rs index 3181eb639..d1b2e0eac 100644 --- a/nghe-backend/src/scan/scanner.rs +++ b/nghe-backend/src/scan/scanner.rs @@ -178,7 +178,7 @@ impl<'db, 'fs, 'mf> Scanner<'db, 'fs, 'mf> { song_id: Uuid, song_path: Utf8TypedPath<'_>, ) -> Result<(), Error> { - lyric::Lyrics::scan(&self.database, &self.filesystem, false, song_id, song_path).await?; + lyric::Lyric::scan(&self.database, &self.filesystem, false, song_id, song_path).await?; Ok(()) } diff --git a/nghe-backend/src/test/mock_impl/information.rs b/nghe-backend/src/test/mock_impl/information.rs index a90751cf2..be4357a91 100644 --- a/nghe-backend/src/test/mock_impl/information.rs +++ b/nghe-backend/src/test/mock_impl/information.rs @@ -17,7 +17,7 @@ use crate::test::filesystem::Trait as _; pub struct Mock<'info, 'picture, 'lyrics, 'path> { pub information: audio::Information<'info>, pub dir_picture: Option>, - pub lyrics: Option>, + pub lyrics: Option>, pub relative_path: Cow<'path, str>, } @@ -44,7 +44,7 @@ impl Mock<'static, 'static, 'static, 'static> { let genres = audio::Genres::query(mock, id).await; let picture = picture::Picture::query_song(mock, id).await; - let lyrics = lyric::Lyrics::query_source(mock, id).await; + let lyrics = lyric::Lyric::query_source(mock, id).await; let dir_picture = picture::Picture::query_album(mock, album_id).await; Self { @@ -80,7 +80,7 @@ impl Mock<'static, 'static, 'static, 'static> { format: Option, file_property: Option>, property: Option, - lyrics: Option>>, + lyrics: Option>>, dir_picture: Option>>, relative_path: Option>, ) -> Self { @@ -98,7 +98,7 @@ impl Mock<'static, 'static, 'static, 'static> { let property = property.unwrap_or_else(|| audio::Property::default(file.format)); let lyrics = lyrics - .unwrap_or_else(|| if Faker.fake() { Some(lyric::Lyrics::fake_sync()) } else { None }); + .unwrap_or_else(|| if Faker.fake() { Some(lyric::Lyric::fake_sync()) } else { None }); let dir_picture = dir_picture.unwrap_or_else(|| Faker.fake()); let relative_path = relative_path.map_or_else(|| Faker.fake::().into(), std::convert::Into::into); @@ -192,7 +192,7 @@ impl Mock<'_, '_, '_, '_> { if let Some(lyrics) = self.lyrics.as_ref() { filesystem .write( - path.with_extension(lyric::Lyrics::EXTERNAL_EXTENSION).to_path(), + path.with_extension(lyric::Lyric::EXTERNAL_EXTENSION).to_path(), lyrics.to_string().as_bytes(), ) .await; From d613097901ebd633904a25ca88a4e001e13a578f Mon Sep 17 00:00:00 2001 From: Vo Van Nghia Date: Wed, 1 Jan 2025 13:13:21 +0100 Subject: [PATCH 06/12] add cleanup one external for lyric --- assets/test/lyrics/sync.lrc | 2 +- nghe-backend/src/file/audio/information.rs | 3 +-- nghe-backend/src/file/lyric/mod.rs | 5 +++-- nghe-backend/src/scan/scanner.rs | 12 ++++++++---- nghe-backend/src/test/mock_impl/information.rs | 2 +- 5 files changed, 14 insertions(+), 10 deletions(-) diff --git a/assets/test/lyrics/sync.lrc b/assets/test/lyrics/sync.lrc index c071e6e98..2f14480c5 100644 --- a/assets/test/lyrics/sync.lrc +++ b/assets/test/lyrics/sync.lrc @@ -1,4 +1,4 @@ -[desc:Lyrics] +[desc:Lyric] [lang:eng] [00:01.02]Hello hi diff --git a/nghe-backend/src/file/audio/information.rs b/nghe-backend/src/file/audio/information.rs index 5b6e42592..00f518394 100644 --- a/nghe-backend/src/file/audio/information.rs +++ b/nghe-backend/src/file/audio/information.rs @@ -8,7 +8,6 @@ use uuid::Uuid; use super::{Album, Artists, Genres}; use crate::database::Database; -use crate::file::lyric; use crate::orm::upsert::Upsert as _; use crate::orm::{albums, songs}; use crate::scan::scanner; @@ -102,7 +101,7 @@ impl Information<'_> { ) -> Result<(), Error> { Artists::cleanup_one(database, started_at, song_id).await?; Genres::cleanup_one(database, started_at, song_id).await?; - lyric::Lyric::cleanup_one(database, started_at, song_id).await?; + crate::file::lyric::Lyric::cleanup_one_external(database, started_at, song_id).await?; Ok(()) } diff --git a/nghe-backend/src/file/lyric/mod.rs b/nghe-backend/src/file/lyric/mod.rs index 630cda6aa..2a5d3f373 100644 --- a/nghe-backend/src/file/lyric/mod.rs +++ b/nghe-backend/src/file/lyric/mod.rs @@ -168,7 +168,7 @@ impl Lyric<'_> { ) } - pub async fn cleanup_one( + pub async fn cleanup_one_external( database: &Database, started_at: time::OffsetDateTime, song_id: Uuid, @@ -177,6 +177,7 @@ impl Lyric<'_> { diesel::delete(lyrics::table) .filter(lyrics::song_id.eq(song_id)) .filter(lyrics::scanned_at.lt(started_at)) + .filter(lyrics::source.is_not_null()) .execute(&mut database.get().await?) .await?; Ok(()) @@ -236,7 +237,7 @@ mod test { } impl Lyric<'static> { - pub async fn query_source(mock: &Mock, id: Uuid) -> Option { + pub async fn query_external(mock: &Mock, id: Uuid) -> Option { lyrics::table .filter(lyrics::song_id.eq(id)) .filter(lyrics::source.is_not_null()) diff --git a/nghe-backend/src/scan/scanner.rs b/nghe-backend/src/scan/scanner.rs index d1b2e0eac..2c7c800ed 100644 --- a/nghe-backend/src/scan/scanner.rs +++ b/nghe-backend/src/scan/scanner.rs @@ -173,12 +173,16 @@ impl<'db, 'fs, 'mf> Scanner<'db, 'fs, 'mf> { Ok(()) } - async fn update_external_lyrics( + async fn update_external_lyric( &self, + started_at: impl Into>, song_id: Uuid, song_path: Utf8TypedPath<'_>, ) -> Result<(), Error> { lyric::Lyric::scan(&self.database, &self.filesystem, false, song_id, song_path).await?; + if let Some(started_at) = started_at.into() { + lyric::Lyric::cleanup_one_external(&self.database, started_at, song_id).await?; + } Ok(()) } @@ -252,7 +256,7 @@ impl<'db, 'fs, 'mf> Scanner<'db, 'fs, 'mf> { // We also need to set album cover_art_id and external lyrics since it might be // added or removed after the previous scan. self.update_dir_picture(song_path.id, dir_picture_id).await?; - self.update_external_lyrics(song_path.id, absolute_path).await?; + self.update_external_lyric(started_at, song_path.id, absolute_path).await?; tracing::debug!("already scanned"); return Ok(song_path.id); } else if let Some(song_id) = song_id { @@ -317,7 +321,7 @@ impl<'db, 'fs, 'mf> Scanner<'db, 'fs, 'mf> { // We also need to set album cover_art_id and external lyrics since it might be // added or removed after the previous scan. self.update_dir_picture(song_path.id, dir_picture_id).await?; - self.update_external_lyrics(song_path.id, absolute_path).await?; + self.update_external_lyric(started_at, song_path.id, absolute_path).await?; tracing::warn!( old = %song_path.relative_path, new = %relative_path, "renamed duplication" ); @@ -343,7 +347,7 @@ impl<'db, 'fs, 'mf> Scanner<'db, 'fs, 'mf> { song_id, ) .await?; - self.update_external_lyrics(song_id, absolute_path).await?; + self.update_external_lyric(None, song_id, absolute_path).await?; audio::Information::cleanup_one(database, started_at, song_id).await?; Ok(song_id) diff --git a/nghe-backend/src/test/mock_impl/information.rs b/nghe-backend/src/test/mock_impl/information.rs index be4357a91..f7e89f10c 100644 --- a/nghe-backend/src/test/mock_impl/information.rs +++ b/nghe-backend/src/test/mock_impl/information.rs @@ -44,7 +44,7 @@ impl Mock<'static, 'static, 'static, 'static> { let genres = audio::Genres::query(mock, id).await; let picture = picture::Picture::query_song(mock, id).await; - let lyrics = lyric::Lyric::query_source(mock, id).await; + let lyrics = lyric::Lyric::query_external(mock, id).await; let dir_picture = picture::Picture::query_album(mock, album_id).await; Self { From 3df85031190b5efe1de9da567d58a00c086c7e15 Mon Sep 17 00:00:00 2001 From: Vo Van Nghia Date: Wed, 1 Jan 2025 16:06:48 +0100 Subject: [PATCH 07/12] fix filesystem test for lyrics --- nghe-backend/src/scan/scanner.rs | 41 +++++++++++++++---- .../src/test/mock_impl/information.rs | 24 ++++++----- .../src/test/mock_impl/music_folder.rs | 7 +++- 3 files changed, 53 insertions(+), 19 deletions(-) diff --git a/nghe-backend/src/scan/scanner.rs b/nghe-backend/src/scan/scanner.rs index 2c7c800ed..93738dabd 100644 --- a/nghe-backend/src/scan/scanner.rs +++ b/nghe-backend/src/scan/scanner.rs @@ -477,6 +477,7 @@ mod tests { async fn test_overwrite( #[future(awt)] mock: Mock, #[values(true, false)] same_album: bool, + #[values(true, false)] same_external_lyric: bool, ) { // Test a constraint with `album_id` and `relative_path`. let mut music_folder = mock.music_folder(0).await; @@ -489,6 +490,11 @@ mod tests { music_folder .add_audio_filesystem() .maybe_album(if same_album { Some(album) } else { None }) + .maybe_external_lyric(if same_external_lyric { + Some(database_audio[0].external_lyric.clone()) + } else { + None + }) .path("test") .format(database_audio[0].information.file.format) .call() @@ -500,14 +506,23 @@ mod tests { let database_audio = database_audio.shift_remove_index(0).unwrap().1; let filesystem_audio = music_folder.filesystem.shift_remove_index(0).unwrap().1; - if same_album { - assert_eq!( - database_audio.with_dir_picture(None), - filesystem_audio.with_dir_picture(None) - ); + + let (database_audio, filesystem_audio) = if same_external_lyric { + (database_audio, filesystem_audio) } else { - assert_eq!(database_audio, filesystem_audio); - } + ( + database_audio.with_external_lyric(None), + filesystem_audio.with_external_lyric(None), + ) + }; + + let (database_audio, filesystem_audio) = if same_album { + (database_audio.with_dir_picture(None), filesystem_audio.with_dir_picture(None)) + } else { + (database_audio, filesystem_audio) + }; + + assert_eq!(database_audio, filesystem_audio); } #[rstest] @@ -531,6 +546,7 @@ mod tests { async fn test_duplicate( #[future(awt)] mock: Mock, #[values(true, false)] same_dir: bool, + #[values(true, false)] same_external_lyric: bool, #[values(true, false)] full: bool, ) { let mut music_folder = mock.music_folder(0).await; @@ -540,6 +556,11 @@ mod tests { music_folder .add_audio_filesystem::<&str>() .metadata(audio.information.metadata.clone()) + .maybe_external_lyric(if same_external_lyric { + Some(audio.external_lyric.clone()) + } else { + None + }) .format(audio.information.file.format) .depth(if same_dir { 0 } else { (1..3).fake() }) .full(scan::start::Full { file: full, ..Default::default() }) @@ -558,6 +579,12 @@ mod tests { .unwrap(); assert_eq!(database_path, path); + let (database_audio, audio) = if same_external_lyric { + (database_audio, audio) + } else { + (database_audio.with_external_lyric(None), audio.with_external_lyric(None)) + }; + let (database_audio, audio) = if same_dir { (database_audio, audio) } else { diff --git a/nghe-backend/src/test/mock_impl/information.rs b/nghe-backend/src/test/mock_impl/information.rs index f7e89f10c..ec4f1ef66 100644 --- a/nghe-backend/src/test/mock_impl/information.rs +++ b/nghe-backend/src/test/mock_impl/information.rs @@ -17,7 +17,7 @@ use crate::test::filesystem::Trait as _; pub struct Mock<'info, 'picture, 'lyrics, 'path> { pub information: audio::Information<'info>, pub dir_picture: Option>, - pub lyrics: Option>, + pub external_lyric: Option>, pub relative_path: Cow<'path, str>, } @@ -44,7 +44,7 @@ impl Mock<'static, 'static, 'static, 'static> { let genres = audio::Genres::query(mock, id).await; let picture = picture::Picture::query_song(mock, id).await; - let lyrics = lyric::Lyric::query_external(mock, id).await; + let external_lyric = lyric::Lyric::query_external(mock, id).await; let dir_picture = picture::Picture::query_album(mock, album_id).await; Self { @@ -59,7 +59,7 @@ impl Mock<'static, 'static, 'static, 'static> { property: upsert.data.property.try_into().unwrap(), file: upsert.data.file.try_into().unwrap(), }, - lyrics, + external_lyric, dir_picture, relative_path: upsert.relative_path, } @@ -80,7 +80,7 @@ impl Mock<'static, 'static, 'static, 'static> { format: Option, file_property: Option>, property: Option, - lyrics: Option>>, + external_lyric: Option>>, dir_picture: Option>>, relative_path: Option>, ) -> Self { @@ -97,7 +97,7 @@ impl Mock<'static, 'static, 'static, 'static> { }); let property = property.unwrap_or_else(|| audio::Property::default(file.format)); - let lyrics = lyrics + let external_lyric = external_lyric .unwrap_or_else(|| if Faker.fake() { Some(lyric::Lyric::fake_sync()) } else { None }); let dir_picture = dir_picture.unwrap_or_else(|| Faker.fake()); let relative_path = @@ -105,7 +105,7 @@ impl Mock<'static, 'static, 'static, 'static> { Self { information: audio::Information { metadata, property, file }, - lyrics, + external_lyric, dir_picture, relative_path, } @@ -143,8 +143,8 @@ impl Mock<'_, '_, '_, '_> { .await .unwrap(); - if let Some(lyrics) = self.lyrics.as_ref() { - lyrics + if let Some(external_lyric) = self.external_lyric.as_ref() { + external_lyric .upsert( database, lyrics::Foreign { song_id }, @@ -189,11 +189,11 @@ impl Mock<'_, '_, '_, '_> { let filesystem = &music_folder.to_impl(); filesystem.write(path, &data).await; - if let Some(lyrics) = self.lyrics.as_ref() { + if let Some(external_lyric) = self.external_lyric.as_ref() { filesystem .write( path.with_extension(lyric::Lyric::EXTERNAL_EXTENSION).to_path(), - lyrics.to_string().as_bytes(), + external_lyric.to_string().as_bytes(), ) .await; } @@ -222,6 +222,10 @@ impl Mock<'_, '_, '_, '_> { } } + pub fn with_external_lyric(self, external_lyric: Option>) -> Self { + Self { external_lyric, ..self } + } + pub fn with_dir_picture(self, dir_picture: Option>) -> Self { Self { dir_picture, ..self } } diff --git a/nghe-backend/src/test/mock_impl/music_folder.rs b/nghe-backend/src/test/mock_impl/music_folder.rs index 37bccc7de..25c97b89e 100644 --- a/nghe-backend/src/test/mock_impl/music_folder.rs +++ b/nghe-backend/src/test/mock_impl/music_folder.rs @@ -14,8 +14,7 @@ use uuid::Uuid; use super::Information; use crate::database::Database; -use crate::file; -use crate::file::{File, audio, picture}; +use crate::file::{self, File, audio, lyric, picture}; use crate::filesystem::Trait as _; use crate::orm::{albums, music_folders, songs}; use crate::scan::scanner; @@ -108,6 +107,7 @@ impl<'a> Mock<'a> { genres: Option>, picture: Option>>, file_property: Option>, + external_lyric: Option>>, dir_picture: Option>>, relative_path: Option>, song_id: Option, @@ -121,6 +121,7 @@ impl<'a> Mock<'a> { .maybe_genres(genres) .maybe_picture(picture) .maybe_file_property(file_property) + .maybe_external_lyric(external_lyric) .maybe_dir_picture(dir_picture) .maybe_relative_path(relative_path); @@ -145,6 +146,7 @@ impl<'a> Mock<'a> { artists: Option>, genres: Option>, picture: Option>>, + external_lyric: Option>>, dir_picture: Option>>, #[builder(default = 1)] n_song: usize, #[builder(default = true)] scan: bool, @@ -158,6 +160,7 @@ impl<'a> Mock<'a> { .maybe_artists(artists) .maybe_genres(genres) .maybe_picture(picture) + .maybe_external_lyric(external_lyric) .maybe_dir_picture(dir_picture); for _ in 0..n_song { From 4a56c6c377c7e34810e3e2a355e30e6d75a5c981 Mon Sep 17 00:00:00 2001 From: Vo Van Nghia Date: Wed, 1 Jan 2025 16:39:04 +0100 Subject: [PATCH 08/12] add extract lyrics in metadata --- nghe-backend/src/config/parsing/id3v2/mod.rs | 6 ++++++ .../src/config/parsing/vorbis_comments.rs | 10 ++++++++++ nghe-backend/src/file/audio/extract/file.rs | 5 +++++ nghe-backend/src/file/audio/extract/mod.rs | 10 ++++++++++ nghe-backend/src/file/audio/extract/tag/id3v2.rs | 16 ++++++++++++++++ .../file/audio/extract/tag/vorbis_comments.rs | 9 +++++++++ nghe-backend/src/file/audio/metadata.rs | 2 ++ 7 files changed, 58 insertions(+) diff --git a/nghe-backend/src/config/parsing/id3v2/mod.rs b/nghe-backend/src/config/parsing/id3v2/mod.rs index a01041471..602c5c768 100644 --- a/nghe-backend/src/config/parsing/id3v2/mod.rs +++ b/nghe-backend/src/config/parsing/id3v2/mod.rs @@ -1,6 +1,7 @@ pub mod frame; use educe::Educe; +use lofty::id3; use serde::{Deserialize, Serialize}; use serde_with::serde_as; @@ -95,6 +96,11 @@ impl Artist { } } +impl Id3v2 { + pub const SYNC_LYRIC_FRAME_ID: id3::v2::FrameId<'static> = + id3::v2::FrameId::Valid(std::borrow::Cow::Borrowed("SYLT")); +} + #[cfg(test)] #[coverage(off)] mod test { diff --git a/nghe-backend/src/config/parsing/vorbis_comments.rs b/nghe-backend/src/config/parsing/vorbis_comments.rs index d468ea8ba..e315362d7 100644 --- a/nghe-backend/src/config/parsing/vorbis_comments.rs +++ b/nghe-backend/src/config/parsing/vorbis_comments.rs @@ -43,6 +43,15 @@ pub struct TrackDisc { pub disc_total: String, } +#[derive(Debug, Clone, Serialize, Deserialize, Educe)] +#[educe(Default)] +pub struct Lyric { + #[educe(Default(expression = "USNYNCEDLYRICS".into()))] + pub unsync: String, + #[educe(Default(expression = "SYNCEDLYRICS".into()))] + pub sync: String, +} + #[derive(Debug, Clone, Serialize, Deserialize, Educe)] #[educe(Default)] pub struct VorbisComments { @@ -58,6 +67,7 @@ pub struct VorbisComments { pub genres: String, #[educe(Default(expression = "COMPILATION".into()))] pub compilation: String, + pub lyric: Lyric, } impl Common { diff --git a/nghe-backend/src/file/audio/extract/file.rs b/nghe-backend/src/file/audio/extract/file.rs index 00785704e..186266b84 100644 --- a/nghe-backend/src/file/audio/extract/file.rs +++ b/nghe-backend/src/file/audio/extract/file.rs @@ -6,6 +6,7 @@ use lofty::ogg::VorbisComments; use super::{Metadata, Property}; use crate::file::audio::{self, Album, Artists, Genres, NameDateMbz, TrackDisc}; +use crate::file::lyric::Lyric; use crate::file::picture::Picture; use crate::{Error, config, error}; @@ -39,6 +40,10 @@ default impl<'a, T: Tag<'a>> Metadata<'a> for T { self.tag()?.genres(config) } + fn lyrics(&'a self, config: &'a config::Parsing) -> Result>, Error> { + self.tag()?.lyrics(config) + } + fn picture(&'a self) -> Result>, Error> { self.tag()?.picture() } diff --git a/nghe-backend/src/file/audio/extract/mod.rs b/nghe-backend/src/file/audio/extract/mod.rs index 4555d8c14..84c244213 100644 --- a/nghe-backend/src/file/audio/extract/mod.rs +++ b/nghe-backend/src/file/audio/extract/mod.rs @@ -4,6 +4,7 @@ mod tag; use isolang::Language; use super::{Album, Artists, File, Genres, NameDateMbz, TrackDisc}; +use crate::file::lyric::Lyric; use crate::file::picture::Picture; use crate::{Error, config}; @@ -14,6 +15,7 @@ pub trait Metadata<'a> { fn track_disc(&'a self, config: &'a config::Parsing) -> Result; fn languages(&'a self, config: &'a config::Parsing) -> Result, Error>; fn genres(&'a self, config: &'a config::Parsing) -> Result, Error>; + fn lyrics(&'a self, config: &'a config::Parsing) -> Result>, Error>; fn picture(&'a self) -> Result>, Error>; fn metadata(&'a self, config: &'a config::Parsing) -> Result, Error> { @@ -26,6 +28,7 @@ pub trait Metadata<'a> { album: self.album(config)?, artists: self.artists(config)?, genres: self.genres(config)?, + lyrics: self.lyrics(config)?, picture: self.picture()?, }) } @@ -78,6 +81,13 @@ impl<'a> Metadata<'a> for File { } } + fn lyrics(&'a self, config: &'a config::Parsing) -> Result>, Error> { + match self { + File::Flac { audio, .. } => audio.lyrics(config), + File::Mpeg { audio, .. } => audio.lyrics(config), + } + } + fn picture(&'a self) -> Result>, Error> { match self { File::Flac { audio, .. } => audio.picture(), diff --git a/nghe-backend/src/file/audio/extract/tag/id3v2.rs b/nghe-backend/src/file/audio/extract/tag/id3v2.rs index 6da7ce5f0..0cdb543d2 100644 --- a/nghe-backend/src/file/audio/extract/tag/id3v2.rs +++ b/nghe-backend/src/file/audio/extract/tag/id3v2.rs @@ -7,6 +7,7 @@ use uuid::Uuid; use crate::config::parsing::id3v2::frame; use crate::file::audio::{Album, Artist, Artists, Date, Genres, NameDateMbz, TrackDisc, extract}; +use crate::file::lyric::Lyric; use crate::file::picture::Picture; use crate::{Error, config, error}; @@ -139,6 +140,21 @@ impl<'a> extract::Metadata<'a> for Id3v2Tag { .unwrap_or_default()) } + fn lyrics(&'a self, _: &'a config::Parsing) -> Result>, Error> { + self.unsync_text() + .map(|frame| Ok(Lyric::from_unsync_text(&frame.content))) + .chain(self.into_iter().filter_map(|frame| { + if *frame.id() == config::parsing::id3v2::Id3v2::SYNC_LYRIC_FRAME_ID + && let Frame::Binary(frame) = frame + { + Some(frame.try_into()) + } else { + None + } + })) + .try_collect() + } + fn picture(&'a self) -> Result>, Error> { let mut iter = self.into_iter(); iter.find_map(|frame| { diff --git a/nghe-backend/src/file/audio/extract/tag/vorbis_comments.rs b/nghe-backend/src/file/audio/extract/tag/vorbis_comments.rs index eb5d5f408..f3bca3962 100644 --- a/nghe-backend/src/file/audio/extract/tag/vorbis_comments.rs +++ b/nghe-backend/src/file/audio/extract/tag/vorbis_comments.rs @@ -7,6 +7,7 @@ use lofty::ogg::{OggPictureStorage, VorbisComments}; use uuid::Uuid; use crate::file::audio::{Album, Artist, Artists, Date, Genres, NameDateMbz, TrackDisc, extract}; +use crate::file::lyric::Lyric; use crate::file::picture::Picture; use crate::{Error, config, error}; @@ -118,6 +119,14 @@ impl<'a> extract::Metadata<'a> for VorbisComments { Ok(self.get_all(&config.vorbis_comments.genres).collect()) } + fn lyrics(&'a self, config: &'a config::Parsing) -> Result>, Error> { + self.get(&config.vorbis_comments.lyric.unsync) + .map(|content| Ok(Lyric::from_unsync_text(content))) + .into_iter() + .chain(self.get_all(&config.vorbis_comments.lyric.sync).map(Lyric::from_sync_text)) + .try_collect() + } + fn picture(&'a self) -> Result>, Error> { Picture::extrat_ogg_picture_storage(self) } diff --git a/nghe-backend/src/file/audio/metadata.rs b/nghe-backend/src/file/audio/metadata.rs index 7cc6256fc..1e180f3e0 100644 --- a/nghe-backend/src/file/audio/metadata.rs +++ b/nghe-backend/src/file/audio/metadata.rs @@ -6,6 +6,7 @@ use itertools::Itertools; use o2o::o2o; use super::{Genres, artist, name_date_mbz, position}; +use crate::file::lyric::Lyric; use crate::file::picture::Picture; use crate::orm::songs; use crate::{Error, error}; @@ -40,6 +41,7 @@ pub struct Metadata<'a> { pub album: name_date_mbz::Album<'a>, pub artists: artist::Artists<'a>, pub genres: Genres<'a>, + pub lyrics: Vec>, pub picture: Option>, } From e9ac370adb43b5cd02c0005ef2c0c243dd79ea50 Mon Sep 17 00:00:00 2001 From: Vo Van Nghia Date: Wed, 1 Jan 2025 17:25:49 +0100 Subject: [PATCH 09/12] add dump lyrics in metadata --- nghe-backend/src/file/lyric/mod.rs | 47 +++++++++++++++++++ nghe-backend/src/test/file/audio/dump/file.rs | 6 +++ nghe-backend/src/test/file/audio/dump/mod.rs | 17 ++++++- .../src/test/file/audio/dump/tag/id3v2.rs | 8 ++++ .../file/audio/dump/tag/vorbis_comments.rs | 16 +++++++ .../src/test/mock_impl/information.rs | 8 ++++ 6 files changed, 101 insertions(+), 1 deletion(-) diff --git a/nghe-backend/src/file/lyric/mod.rs b/nghe-backend/src/file/lyric/mod.rs index 2a5d3f373..da6b5d877 100644 --- a/nghe-backend/src/file/lyric/mod.rs +++ b/nghe-backend/src/file/lyric/mod.rs @@ -192,6 +192,7 @@ mod test { use diesel::{QueryDsl, SelectableHelper}; use fake::{Dummy, Fake, Faker}; use itertools::Itertools; + use lofty::id3::v2::Frame; use super::*; use crate::test::Mock; @@ -237,6 +238,20 @@ mod test { } impl Lyric<'static> { + pub async fn query(mock: &Mock, id: Uuid) -> Vec { + lyrics::table + .filter(lyrics::song_id.eq(id)) + .filter(lyrics::source.is_null()) + .select(lyrics::Data::as_select()) + .order_by(lyrics::scanned_at) + .get_results(&mut mock.get().await) + .await + .unwrap() + .into_iter() + .map(Self::from) + .collect() + } + pub async fn query_external(mock: &Mock, id: Uuid) -> Option { lyrics::table .filter(lyrics::song_id.eq(id)) @@ -280,6 +295,38 @@ mod test { } } } + + impl From> for Frame<'static> { + fn from(value: Lyric<'_>) -> Self { + let language = value.language.to_639_3().as_bytes().try_into().unwrap(); + match value.lines { + Lines::Unsync(lines) => UnsynchronizedTextFrame::new( + lofty::TextEncoding::UTF8, + language, + value.description.map(Cow::into_owned).unwrap_or_default(), + lines.join("\n"), + ) + .into(), + Lines::Sync(lines) => BinaryFrame::new( + crate::config::parsing::id3v2::Id3v2::SYNC_LYRIC_FRAME_ID, + SynchronizedTextFrame::new( + lofty::TextEncoding::UTF8, + language, + lofty::id3::v2::TimestampFormat::MS, + lofty::id3::v2::SyncTextContentType::Lyrics, + value.description.map(Cow::into_owned), + lines + .into_iter() + .map(|(duration, text)| (duration, text.into_owned())) + .collect(), + ) + .as_bytes() + .unwrap(), + ) + .into(), + } + } + } } #[cfg(test)] diff --git a/nghe-backend/src/test/file/audio/dump/file.rs b/nghe-backend/src/test/file/audio/dump/file.rs index a574c4946..63e25c513 100644 --- a/nghe-backend/src/test/file/audio/dump/file.rs +++ b/nghe-backend/src/test/file/audio/dump/file.rs @@ -6,6 +6,7 @@ use lofty::ogg::{OggPictureStorage as _, VorbisComments}; use super::Metadata; use crate::config; use crate::file::audio::{Album, Artists, Genres, NameDateMbz, TrackDisc}; +use crate::file::lyric::Lyric; use crate::file::picture::Picture; trait TagMut { @@ -48,6 +49,11 @@ default impl Metadata for T { self } + fn dump_lyrics(&mut self, config: &config::Parsing, lyrics: Vec>) -> &mut Self { + self.tag_mut().dump_lyrics(config, lyrics); + self + } + fn dump_picture(&mut self, picture: Option>) -> &mut Self { self.tag_mut().dump_picture(picture); self diff --git a/nghe-backend/src/test/file/audio/dump/mod.rs b/nghe-backend/src/test/file/audio/dump/mod.rs index 1af5be2b0..a1c8dba63 100644 --- a/nghe-backend/src/test/file/audio/dump/mod.rs +++ b/nghe-backend/src/test/file/audio/dump/mod.rs @@ -5,6 +5,7 @@ use isolang::Language; use crate::config; use crate::file::audio::{self, Album, Artists, File, Genres, NameDateMbz, TrackDisc}; +use crate::file::lyric::Lyric; use crate::file::picture::Picture; pub trait Metadata { @@ -14,6 +15,7 @@ pub trait Metadata { fn dump_track_disc(&mut self, config: &config::Parsing, track_disc: TrackDisc) -> &mut Self; fn dump_languages(&mut self, config: &config::Parsing, languages: Vec) -> &mut Self; fn dump_genres(&mut self, config: &config::Parsing, genres: Genres<'_>) -> &mut Self; + fn dump_lyrics(&mut self, config: &config::Parsing, lyrics: Vec>) -> &mut Self; fn dump_picture(&mut self, picture: Option>) -> &mut Self; fn dump_metadata( @@ -21,7 +23,7 @@ pub trait Metadata { config: &config::Parsing, metadata: audio::Metadata<'_>, ) -> &mut Self { - let audio::Metadata { song, album, artists, genres, picture } = metadata; + let audio::Metadata { song, album, artists, genres, lyrics, picture } = metadata; let audio::Song { main, track_disc, languages } = song; self.dump_song(config, main) .dump_album(config, album) @@ -29,6 +31,7 @@ pub trait Metadata { .dump_track_disc(config, track_disc) .dump_languages(config, languages) .dump_genres(config, genres) + .dump_lyrics(config, lyrics) .dump_picture(picture) } } @@ -106,6 +109,18 @@ impl Metadata for File { self } + fn dump_lyrics(&mut self, config: &config::Parsing, lyrics: Vec>) -> &mut Self { + match self { + File::Flac { audio, .. } => { + audio.dump_lyrics(config, lyrics); + } + File::Mpeg { audio, .. } => { + audio.dump_lyrics(config, lyrics); + } + } + self + } + fn dump_picture(&mut self, picture: Option>) -> &mut Self { match self { File::Flac { audio, .. } => { diff --git a/nghe-backend/src/test/file/audio/dump/tag/id3v2.rs b/nghe-backend/src/test/file/audio/dump/tag/id3v2.rs index 10a423db1..b62241ffb 100644 --- a/nghe-backend/src/test/file/audio/dump/tag/id3v2.rs +++ b/nghe-backend/src/test/file/audio/dump/tag/id3v2.rs @@ -10,6 +10,7 @@ use crate::config; use crate::config::parsing::id3v2::frame; use crate::file::audio::position::Position; use crate::file::audio::{Album, Artist, Artists, Date, Genres, NameDateMbz, TrackDisc}; +use crate::file::lyric::Lyric; use crate::file::picture::Picture; use crate::test::file::audio::dump; @@ -139,6 +140,13 @@ impl dump::Metadata for Id3v2Tag { self } + fn dump_lyrics(&mut self, _: &config::Parsing, lyrics: Vec>) -> &mut Self { + for lyric in lyrics { + self.insert(lyric.into()); + } + self + } + fn dump_picture(&mut self, picture: Option>) -> &mut Self { if let Some(picture) = picture { self.insert_picture(picture.into()); diff --git a/nghe-backend/src/test/file/audio/dump/tag/vorbis_comments.rs b/nghe-backend/src/test/file/audio/dump/tag/vorbis_comments.rs index 29343872f..47f5c1aa0 100644 --- a/nghe-backend/src/test/file/audio/dump/tag/vorbis_comments.rs +++ b/nghe-backend/src/test/file/audio/dump/tag/vorbis_comments.rs @@ -6,6 +6,7 @@ use uuid::Uuid; use crate::config; use crate::file::audio::position::Position; use crate::file::audio::{Album, Artist, Artists, Date, Genres, NameDateMbz, TrackDisc}; +use crate::file::lyric::Lyric; use crate::file::picture::Picture; use crate::test::file::audio::dump; @@ -108,6 +109,21 @@ impl dump::Metadata for VorbisComments { self } + fn dump_lyrics(&mut self, config: &config::Parsing, lyrics: Vec>) -> &mut Self { + for lyric in lyrics { + self.push( + if lyric.is_sync() { + &config.vorbis_comments.lyric.sync + } else { + &config.vorbis_comments.lyric.unsync + } + .clone(), + lyric.to_string(), + ); + } + self + } + fn dump_picture(&mut self, picture: Option>) -> &mut Self { if let Some(picture) = picture { self.insert_picture(picture.into(), None).unwrap(); diff --git a/nghe-backend/src/test/mock_impl/information.rs b/nghe-backend/src/test/mock_impl/information.rs index ec4f1ef66..21ae3336f 100644 --- a/nghe-backend/src/test/mock_impl/information.rs +++ b/nghe-backend/src/test/mock_impl/information.rs @@ -42,6 +42,7 @@ impl Mock<'static, 'static, 'static, 'static> { let album = audio::Album::query_upsert(mock, upsert.foreign.album_id).await; let artists = audio::Artists::query(mock, id).await; let genres = audio::Genres::query(mock, id).await; + let lyrics = lyric::Lyric::query(mock, id).await; let picture = picture::Picture::query_song(mock, id).await; let external_lyric = lyric::Lyric::query_external(mock, id).await; @@ -54,6 +55,7 @@ impl Mock<'static, 'static, 'static, 'static> { album: album.data.try_into().unwrap(), artists, genres, + lyrics, picture, }, property: upsert.data.property.try_into().unwrap(), @@ -76,6 +78,7 @@ impl Mock<'static, 'static, 'static, 'static> { album: Option>, artists: Option>, genres: Option>, + lyrics: Option>>, picture: Option>>, format: Option, file_property: Option>, @@ -89,6 +92,11 @@ impl Mock<'static, 'static, 'static, 'static> { album: album.unwrap_or_else(|| Faker.fake()), artists: artists.unwrap_or_else(|| Faker.fake()), genres: genres.unwrap_or_else(|| Faker.fake()), + lyrics: lyrics.unwrap_or_else(|| { + let unsync = if Faker.fake() { Some(lyric::Lyric::fake_unsync()) } else { None }; + let sync = if Faker.fake() { Some(lyric::Lyric::fake_sync()) } else { None }; + unsync.into_iter().chain(sync).collect() + }), picture: picture.unwrap_or_else(|| Faker.fake()), }); let file = file_property.unwrap_or_else(|| file::Property { From f72e40d90f438afb15eac7a99238a23482cf824c Mon Sep 17 00:00:00 2001 From: Vo Van Nghia Date: Wed, 1 Jan 2025 18:33:18 +0100 Subject: [PATCH 10/12] use first line as description in unsync text --- nghe-backend/src/file/lyric/mod.rs | 28 +++++++++++++++++++--------- 1 file changed, 19 insertions(+), 9 deletions(-) diff --git a/nghe-backend/src/file/lyric/mod.rs b/nghe-backend/src/file/lyric/mod.rs index da6b5d877..918109df2 100644 --- a/nghe-backend/src/file/lyric/mod.rs +++ b/nghe-backend/src/file/lyric/mod.rs @@ -46,8 +46,9 @@ impl<'a> TryFrom<&'a UnsynchronizedTextFrame<'_>> for Lyric<'a> { type Error = Error; fn try_from(frame: &'a UnsynchronizedTextFrame<'_>) -> Result { + let description = frame.description.as_str(); Ok(Self { - description: Some(frame.description.as_str().into()), + description: if description.is_empty() { None } else { Some(description.into()) }, language: str::from_utf8(&frame.language)?.parse().map_err(error::Kind::from)?, lines: frame.content.lines().collect(), }) @@ -69,11 +70,15 @@ impl<'a> TryFrom<&'a BinaryFrame<'_>> for Lyric<'a> { impl<'a> Lyric<'a> { pub fn from_unsync_text(content: &'a str) -> Self { - Self { - description: None, - language: Language::Und, - lines: content.lines().filter(|text| !text.is_empty()).collect(), - } + let lines = content.lines().filter(|text| !text.is_empty()); + let (description, lines) = if cfg!(test) { + let mut lines = lines; + let description = lines.next().unwrap(); + (if description.is_empty() { None } else { Some(description.into()) }, lines.collect()) + } else { + (None, lines.collect()) + }; + Self { description, language: Language::Und, lines } } pub fn from_sync_text(content: &str) -> Result { @@ -230,9 +235,9 @@ mod test { pub fn fake_unsync() -> Self { Self { - description: None, + description: Faker.fake::>().map(Cow::Owned), language: Language::Und, - lines: fake::vec![String; 1..=5].into_iter().collect(), + lines: fake::vec![String; 2..=5].into_iter().collect(), } } } @@ -274,7 +279,12 @@ mod test { impl Display for Lyric<'_> { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match &self.lines { - Lines::Unsync(lines) => write!(f, "{}", lines.join("\n")), + Lines::Unsync(lines) => write!( + f, + "{}\n{}", + self.description.as_ref().map::<&str, _>(Cow::as_ref).unwrap_or_default(), + lines.join("\n") + ), Lines::Sync(lines) => { if let Some(description) = self.description.as_ref() { writeln!(f, "[desc:{description}]")?; From e60ce4bc9ae944505070d2bb499de2b39427506b Mon Sep 17 00:00:00 2001 From: Vo Van Nghia Date: Wed, 1 Jan 2025 19:31:55 +0100 Subject: [PATCH 11/12] add lyric query embedded --- nghe-backend/src/file/audio/information.rs | 12 ++++- nghe-backend/src/file/lyric/mod.rs | 51 ++++++++++++++----- nghe-backend/src/orm/lyrics.rs | 2 +- .../src/test/mock_impl/information.rs | 2 +- 4 files changed, 50 insertions(+), 17 deletions(-) diff --git a/nghe-backend/src/file/audio/information.rs b/nghe-backend/src/file/audio/information.rs index 00f518394..84c13dc14 100644 --- a/nghe-backend/src/file/audio/information.rs +++ b/nghe-backend/src/file/audio/information.rs @@ -8,8 +8,9 @@ use uuid::Uuid; use super::{Album, Artists, Genres}; use crate::database::Database; +use crate::file::lyric::Lyric; use crate::orm::upsert::Upsert as _; -use crate::orm::{albums, songs}; +use crate::orm::{albums, lyrics, songs}; use crate::scan::scanner; use crate::{Error, file}; @@ -48,6 +49,12 @@ impl Information<'_> { Genres::upsert_song(database, song_id, &genre_ids).await } + pub async fn upsert_lyrics(&self, database: &Database, song_id: Uuid) -> Result<(), Error> { + Lyric::upserts_embedded(database, lyrics::Foreign { song_id }, &self.metadata.lyrics) + .await?; + Ok(()) + } + pub async fn upsert_cover_art( &self, database: &Database, @@ -91,6 +98,7 @@ impl Information<'_> { let song_id = self.upsert_song(database, foreign, relative_path, song_id).await?; self.upsert_artists(database, &config.index.ignore_prefixes, song_id).await?; self.upsert_genres(database, song_id).await?; + self.upsert_lyrics(database, song_id).await?; Ok(song_id) } @@ -101,7 +109,7 @@ impl Information<'_> { ) -> Result<(), Error> { Artists::cleanup_one(database, started_at, song_id).await?; Genres::cleanup_one(database, started_at, song_id).await?; - crate::file::lyric::Lyric::cleanup_one_external(database, started_at, song_id).await?; + crate::file::lyric::Lyric::cleanup_one(database, started_at, song_id).await?; Ok(()) } diff --git a/nghe-backend/src/file/lyric/mod.rs b/nghe-backend/src/file/lyric/mod.rs index 918109df2..e99b5b0b9 100644 --- a/nghe-backend/src/file/lyric/mod.rs +++ b/nghe-backend/src/file/lyric/mod.rs @@ -4,6 +4,7 @@ use std::borrow::Cow; use alrc::AdvancedLrc; use diesel::{ExpressionMethods, OptionalExtension}; use diesel_async::RunQueryDsl; +use futures_lite::{StreamExt as _, stream}; use isolang::Language; use lofty::id3::v2::{BinaryFrame, SynchronizedTextFrame, UnsynchronizedTextFrame}; use typed_path::Utf8TypedPath; @@ -71,9 +72,9 @@ impl<'a> TryFrom<&'a BinaryFrame<'_>> for Lyric<'a> { impl<'a> Lyric<'a> { pub fn from_unsync_text(content: &'a str) -> Self { let lines = content.lines().filter(|text| !text.is_empty()); - let (description, lines) = if cfg!(test) { + let (description, lines) = if cfg!(test) && content.starts_with('#') { let mut lines = lines; - let description = lines.next().unwrap(); + let description = lines.next().unwrap().strip_prefix('#').unwrap(); (if description.is_empty() { None } else { Some(description.into()) }, lines.collect()) } else { (None, lines.collect()) @@ -122,7 +123,18 @@ impl Lyric<'_> { .await } - async fn set_source_scanned_at( + pub async fn upserts_embedded( + database: &Database, + foreign: lyrics::Foreign, + lyrics: &[Self], + ) -> Result, Error> { + stream::iter(lyrics) + .then(async |lyric| lyric.upsert(database, foreign, None::<&str>).await) + .try_collect() + .await + } + + async fn set_external_scanned_at( database: &Database, song_id: Uuid, source: impl AsRef, @@ -162,7 +174,7 @@ impl Lyric<'_> { Ok( if !full && let Some(lyrics_id) = - Self::set_source_scanned_at(database, song_id, path).await? + Self::set_external_scanned_at(database, song_id, path).await? { Some(lyrics_id) } else if let Some(lyrics) = Self::load(filesystem, path).await? { @@ -187,6 +199,20 @@ impl Lyric<'_> { .await?; Ok(()) } + + pub async fn cleanup_one( + database: &Database, + started_at: time::OffsetDateTime, + song_id: Uuid, + ) -> Result<(), Error> { + // Delete all lyrics of a song which haven't been refreshed since timestamp. + diesel::delete(lyrics::table) + .filter(lyrics::song_id.eq(song_id)) + .filter(lyrics::scanned_at.lt(started_at)) + .execute(&mut database.get().await?) + .await?; + Ok(()) + } } #[cfg(test)] @@ -243,7 +269,7 @@ mod test { } impl Lyric<'static> { - pub async fn query(mock: &Mock, id: Uuid) -> Vec { + pub async fn query_embedded(mock: &Mock, id: Uuid) -> Vec { lyrics::table .filter(lyrics::song_id.eq(id)) .filter(lyrics::source.is_null()) @@ -279,12 +305,12 @@ mod test { impl Display for Lyric<'_> { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match &self.lines { - Lines::Unsync(lines) => write!( - f, - "{}\n{}", - self.description.as_ref().map::<&str, _>(Cow::as_ref).unwrap_or_default(), - lines.join("\n") - ), + Lines::Unsync(lines) => { + if let Some(description) = self.description.as_ref() { + writeln!(f, "#{description}")?; + } + write!(f, "{}", lines.join("\n"))?; + } Lines::Sync(lines) => { if let Some(description) = self.description.as_ref() { writeln!(f, "[desc:{description}]")?; @@ -299,10 +325,9 @@ mod test { write!(f, "[{minutes:02}:{seconds:02}.{milliseconds:02}]")?; writeln!(f, "{text}")?; } - - Ok(()) } } + Ok(()) } } diff --git a/nghe-backend/src/orm/lyrics.rs b/nghe-backend/src/orm/lyrics.rs index 9a8aac72a..fbcf4e24c 100644 --- a/nghe-backend/src/orm/lyrics.rs +++ b/nghe-backend/src/orm/lyrics.rs @@ -25,7 +25,7 @@ pub struct Data<'a> { pub texts: Vec>, } -#[derive(Debug, Insertable)] +#[derive(Debug, Clone, Copy, Insertable)] #[diesel(table_name = lyrics, check_for_backend(crate::orm::Type))] #[diesel(treat_none_as_null = true)] pub struct Foreign { diff --git a/nghe-backend/src/test/mock_impl/information.rs b/nghe-backend/src/test/mock_impl/information.rs index 21ae3336f..d662438b6 100644 --- a/nghe-backend/src/test/mock_impl/information.rs +++ b/nghe-backend/src/test/mock_impl/information.rs @@ -42,7 +42,7 @@ impl Mock<'static, 'static, 'static, 'static> { let album = audio::Album::query_upsert(mock, upsert.foreign.album_id).await; let artists = audio::Artists::query(mock, id).await; let genres = audio::Genres::query(mock, id).await; - let lyrics = lyric::Lyric::query(mock, id).await; + let lyrics = lyric::Lyric::query_embedded(mock, id).await; let picture = picture::Picture::query_song(mock, id).await; let external_lyric = lyric::Lyric::query_external(mock, id).await; From cf6ca7a2d99af040132595e84bf27e43c81864a2 Mon Sep 17 00:00:00 2001 From: Vo Van Nghia Date: Wed, 1 Jan 2025 20:41:10 +0100 Subject: [PATCH 12/12] fix test for embedded lyric --- nghe-backend/src/file/audio/extract/tag/id3v2.rs | 2 +- nghe-backend/src/file/audio/metadata.rs | 1 + nghe-backend/src/file/lyric/mod.rs | 11 +++++++++-- nghe-backend/src/test/mock_impl/information.rs | 6 +----- 4 files changed, 12 insertions(+), 8 deletions(-) diff --git a/nghe-backend/src/file/audio/extract/tag/id3v2.rs b/nghe-backend/src/file/audio/extract/tag/id3v2.rs index 0cdb543d2..bdbad0724 100644 --- a/nghe-backend/src/file/audio/extract/tag/id3v2.rs +++ b/nghe-backend/src/file/audio/extract/tag/id3v2.rs @@ -142,7 +142,7 @@ impl<'a> extract::Metadata<'a> for Id3v2Tag { fn lyrics(&'a self, _: &'a config::Parsing) -> Result>, Error> { self.unsync_text() - .map(|frame| Ok(Lyric::from_unsync_text(&frame.content))) + .map(std::convert::TryInto::try_into) .chain(self.into_iter().filter_map(|frame| { if *frame.id() == config::parsing::id3v2::Id3v2::SYNC_LYRIC_FRAME_ID && let Frame::Binary(frame) = frame diff --git a/nghe-backend/src/file/audio/metadata.rs b/nghe-backend/src/file/audio/metadata.rs index 1e180f3e0..33b4c33aa 100644 --- a/nghe-backend/src/file/audio/metadata.rs +++ b/nghe-backend/src/file/audio/metadata.rs @@ -41,6 +41,7 @@ pub struct Metadata<'a> { pub album: name_date_mbz::Album<'a>, pub artists: artist::Artists<'a>, pub genres: Genres<'a>, + #[cfg_attr(test, dummy(expr = "Lyric::fake_vec()"))] pub lyrics: Vec>, pub picture: Option>, } diff --git a/nghe-backend/src/file/lyric/mod.rs b/nghe-backend/src/file/lyric/mod.rs index e99b5b0b9..a48828461 100644 --- a/nghe-backend/src/file/lyric/mod.rs +++ b/nghe-backend/src/file/lyric/mod.rs @@ -240,8 +240,9 @@ mod test { } pub fn fake_sync() -> Self { + // Force description as Some to avoid clash with unsync. Self { - description: Faker.fake::>().map(Cow::Owned), + description: Some(Faker.fake::().into()), language: Language::from_usize((0..=7915).fake()).unwrap(), lines: fake::vec![String; 1..=5] .into_iter() @@ -263,9 +264,15 @@ mod test { Self { description: Faker.fake::>().map(Cow::Owned), language: Language::Und, - lines: fake::vec![String; 2..=5].into_iter().collect(), + lines: fake::vec![String; 1..=5].into_iter().collect(), } } + + pub fn fake_vec() -> Vec { + let unsync = if Faker.fake() { Some(Self::fake_unsync()) } else { None }; + let sync = if Faker.fake() { Some(Self::fake_sync()) } else { None }; + unsync.into_iter().chain(sync).collect() + } } impl Lyric<'static> { diff --git a/nghe-backend/src/test/mock_impl/information.rs b/nghe-backend/src/test/mock_impl/information.rs index d662438b6..1999b9898 100644 --- a/nghe-backend/src/test/mock_impl/information.rs +++ b/nghe-backend/src/test/mock_impl/information.rs @@ -92,11 +92,7 @@ impl Mock<'static, 'static, 'static, 'static> { album: album.unwrap_or_else(|| Faker.fake()), artists: artists.unwrap_or_else(|| Faker.fake()), genres: genres.unwrap_or_else(|| Faker.fake()), - lyrics: lyrics.unwrap_or_else(|| { - let unsync = if Faker.fake() { Some(lyric::Lyric::fake_unsync()) } else { None }; - let sync = if Faker.fake() { Some(lyric::Lyric::fake_sync()) } else { None }; - unsync.into_iter().chain(sync).collect() - }), + lyrics: lyrics.unwrap_or_else(lyric::Lyric::fake_vec), picture: picture.unwrap_or_else(|| Faker.fake()), }); let file = file_property.unwrap_or_else(|| file::Property {