From 287df843828e106d4fef2661c796ffaa69742caf Mon Sep 17 00:00:00 2001 From: bytedream Date: Fri, 14 Jun 2024 00:17:08 +0200 Subject: [PATCH] Rework episode filtering --- Cargo.lock | 16 +- crunchy-cli-core/Cargo.toml | 2 +- crunchy-cli-core/src/archive/command.rs | 47 ++- crunchy-cli-core/src/archive/filter.rs | 466 ----------------------- crunchy-cli-core/src/archive/mod.rs | 1 - crunchy-cli-core/src/download/command.rs | 53 ++- crunchy-cli-core/src/download/filter.rs | 307 --------------- crunchy-cli-core/src/download/mod.rs | 1 - crunchy-cli-core/src/search/format.rs | 28 +- crunchy-cli-core/src/utils/filter.rs | 429 +++++++++++++++++++-- crunchy-cli-core/src/utils/format.rs | 2 + 11 files changed, 517 insertions(+), 835 deletions(-) delete mode 100644 crunchy-cli-core/src/archive/filter.rs delete mode 100644 crunchy-cli-core/src/download/filter.rs diff --git a/Cargo.lock b/Cargo.lock index 8400022a..7fdb3d9e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -400,9 +400,9 @@ dependencies = [ [[package]] name = "crunchyroll-rs" -version = "0.11.3" +version = "0.11.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d33b8d77c80dea79e66993cb67963b2171dcf0b8fbc87591c58f2dadfea8da2" +checksum = "d6e38c223aecf65c9c9bec50764beea5dc70b6c97cd7f767bf6860f2fc8e0a07" dependencies = [ "async-trait", "chrono", @@ -426,9 +426,9 @@ dependencies = [ [[package]] name = "crunchyroll-rs-internal" -version = "0.11.3" +version = "0.11.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa51945265f25c45f7d53bd70e5263dd023c0be45e38eaba886a971cb645d797" +checksum = "144a38040a21aaa456741a9f6749354527bb68ad3bb14210e0bbc40fbd95186c" dependencies = [ "darling", "quote", @@ -1967,9 +1967,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.37.0" +version = "1.38.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1adbebffeca75fcfd058afa480fb6c0b81e165a0323f9c9d39c9697e37c46787" +checksum = "ba4f4a02a7a80d6f274636f0aa95c7e383b912d41fe721a31f29e29698585a4a" dependencies = [ "backtrace", "bytes", @@ -1984,9 +1984,9 @@ dependencies = [ [[package]] name = "tokio-macros" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b8a1e28f2deaa14e508979454cb3a223b10b938b45af148bc0986de36f1923b" +checksum = "5f5ae998a069d4b5aba8ee9dad856af7d520c3699e6159b185c2acd48155d39a" dependencies = [ "proc-macro2", "quote", diff --git a/crunchy-cli-core/Cargo.toml b/crunchy-cli-core/Cargo.toml index 56964ec9..5c7b9017 100644 --- a/crunchy-cli-core/Cargo.toml +++ b/crunchy-cli-core/Cargo.toml @@ -16,7 +16,7 @@ anyhow = "1.0" async-speed-limit = "0.4" clap = { version = "4.5", features = ["derive", "string"] } chrono = "0.4" -crunchyroll-rs = { version = "0.11.3", features = ["experimental-stabilizations", "tower"] } +crunchyroll-rs = { version = "0.11.4", features = ["experimental-stabilizations", "tower"] } ctrlc = "3.4" dialoguer = { version = "0.11", default-features = false } dirs = "5.0" diff --git a/crunchy-cli-core/src/archive/command.rs b/crunchy-cli-core/src/archive/command.rs index d34c4b7d..0d1b3a49 100644 --- a/crunchy-cli-core/src/archive/command.rs +++ b/crunchy-cli-core/src/archive/command.rs @@ -1,10 +1,9 @@ -use crate::archive::filter::ArchiveFilter; use crate::utils::context::Context; use crate::utils::download::{ DownloadBuilder, DownloadFormat, DownloadFormatMetadata, MergeBehavior, }; use crate::utils::ffmpeg::FFmpegPreset; -use crate::utils::filter::Filter; +use crate::utils::filter::{Filter, FilterMediaScope}; use crate::utils::format::{Format, SingleFormat}; use crate::utils::locale::{all_locale_in_locales, resolve_locales, LanguageTagging}; use crate::utils::log::progress; @@ -284,9 +283,49 @@ impl Execute for Archive { for (i, (media_collection, url_filter)) in parsed_urls.into_iter().enumerate() { let progress_handler = progress!("Fetching series details"); - let single_format_collection = ArchiveFilter::new( + let single_format_collection = Filter::new( url_filter, - self.clone(), + self.audio.clone(), + self.subtitle.clone(), + |scope, locales| { + let audios = locales.into_iter().map(|l| l.to_string()).collect::>().join(", "); + match scope { + FilterMediaScope::Series(series) => warn!("Series {} is not available with {} audio", series.title, audios), + FilterMediaScope::Season(season) => warn!("Season {} is not available with {} audio", season.season_number, audios), + FilterMediaScope::Episode(episodes) => { + if episodes.len() == 1 { + warn!("Episode {} is not available with {} audio", episodes[0].sequence_number, audios) + } else if episodes.len() == 2 { + warn!("Season {} is only available with {} audio from episode {} to {}", episodes[0].season_number, audios, episodes[0].sequence_number, episodes[1].sequence_number) + } else { + unimplemented!() + } + } + } + Ok(true) + }, + |scope, locales| { + let subtitles = locales.into_iter().map(|l| l.to_string()).collect::>().join(", "); + match scope { + FilterMediaScope::Series(series) => warn!("Series {} is not available with {} subtitles", series.title, subtitles), + FilterMediaScope::Season(season) => warn!("Season {} is not available with {} subtitles", season.season_number, subtitles), + FilterMediaScope::Episode(episodes) => { + if episodes.len() == 1 { + warn!("Episode {} of season {} is not available with {} subtitles", episodes[0].sequence_number, episodes[0].season_title, subtitles) + } else if episodes.len() == 2 { + warn!("Season {} of season {} is only available with {} subtitles from episode {} to {}", episodes[0].season_number, episodes[0].season_title, subtitles, episodes[0].sequence_number, episodes[1].sequence_number) + } else { + unimplemented!() + } + } + } + Ok(true) + }, + |season| { + warn!("Skipping premium episodes in season {season}"); + Ok(()) + }, + Format::has_relative_fmt(&self.output), !self.yes, self.skip_specials, ctx.crunchy.premium().await, diff --git a/crunchy-cli-core/src/archive/filter.rs b/crunchy-cli-core/src/archive/filter.rs deleted file mode 100644 index b08fb6cf..00000000 --- a/crunchy-cli-core/src/archive/filter.rs +++ /dev/null @@ -1,466 +0,0 @@ -use crate::archive::command::Archive; -use crate::utils::filter::{real_dedup_vec, Filter}; -use crate::utils::format::{Format, SingleFormat, SingleFormatCollection}; -use crate::utils::interactive_select::{check_for_duplicated_seasons, get_duplicated_seasons}; -use crate::utils::parse::{fract, UrlFilter}; -use anyhow::Result; -use crunchyroll_rs::{Concert, Episode, Locale, Movie, MovieListing, MusicVideo, Season, Series}; -use log::{info, warn}; -use std::collections::{BTreeMap, HashMap}; -use std::ops::Not; - -enum Visited { - Series, - Season, - None, -} - -pub(crate) struct ArchiveFilter { - url_filter: UrlFilter, - archive: Archive, - interactive_input: bool, - skip_special: bool, - season_episodes: HashMap>, - season_subtitles_missing: Vec, - seasons_with_premium: Option>, - season_sorting: Vec, - visited: Visited, -} - -impl ArchiveFilter { - pub(crate) fn new( - url_filter: UrlFilter, - archive: Archive, - interactive_input: bool, - skip_special: bool, - is_premium: bool, - ) -> Self { - Self { - url_filter, - archive, - interactive_input, - skip_special, - season_episodes: HashMap::new(), - season_subtitles_missing: vec![], - seasons_with_premium: is_premium.not().then_some(vec![]), - season_sorting: vec![], - visited: Visited::None, - } - } -} - -impl Filter for ArchiveFilter { - type T = Vec; - type Output = SingleFormatCollection; - - async fn visit_series(&mut self, series: Series) -> Result> { - // `series.audio_locales` isn't always populated b/c of crunchyrolls api. so check if the - // audio is matching only if the field is populated - if !series.audio_locales.is_empty() { - let missing_audio = missing_locales(&series.audio_locales, &self.archive.audio); - if !missing_audio.is_empty() { - warn!( - "Series {} is not available with {} audio", - series.title, - missing_audio - .into_iter() - .map(|l| l.to_string()) - .collect::>() - .join(", ") - ) - } - let missing_subtitle = - missing_locales(&series.subtitle_locales, &self.archive.subtitle); - if !missing_subtitle.is_empty() { - warn!( - "Series {} is not available with {} subtitles", - series.title, - missing_subtitle - .into_iter() - .map(|l| l.to_string()) - .collect::>() - .join(", ") - ) - } - self.visited = Visited::Series - } - - let mut seasons = series.seasons().await?; - let mut remove_ids = vec![]; - for season in seasons.iter_mut() { - if !self.url_filter.is_season_valid(season.season_number) - || (!season - .audio_locales - .iter() - .any(|l| self.archive.audio.contains(l)) - && !season - .available_versions() - .await? - .iter() - .any(|l| self.archive.audio.contains(l))) - { - remove_ids.push(season.id.clone()); - } - } - - seasons.retain(|s| !remove_ids.contains(&s.id)); - - let duplicated_seasons = get_duplicated_seasons(&seasons); - if !duplicated_seasons.is_empty() { - if self.interactive_input { - check_for_duplicated_seasons(&mut seasons); - } else { - info!( - "Found duplicated seasons: {}", - duplicated_seasons - .iter() - .map(|d| d.to_string()) - .collect::>() - .join(", ") - ) - } - } - - Ok(seasons) - } - - async fn visit_season(&mut self, mut season: Season) -> Result> { - if !self.url_filter.is_season_valid(season.season_number) { - return Ok(vec![]); - } - - let mut seasons = season.version(self.archive.audio.clone()).await?; - if self - .archive - .audio - .iter() - .any(|l| season.audio_locales.contains(l)) - { - seasons.insert(0, season.clone()); - } - - if !matches!(self.visited, Visited::Series) { - let mut audio_locales: Vec = seasons - .iter() - .flat_map(|s| s.audio_locales.clone()) - .collect(); - real_dedup_vec(&mut audio_locales); - let missing_audio = missing_locales(&audio_locales, &self.archive.audio); - if !missing_audio.is_empty() { - warn!( - "Season {} is not available with {} audio", - season.season_number, - missing_audio - .into_iter() - .map(|l| l.to_string()) - .collect::>() - .join(", ") - ) - } - - let subtitle_locales: Vec = seasons - .iter() - .flat_map(|s| s.subtitle_locales.clone()) - .collect(); - let missing_subtitle = missing_locales(&subtitle_locales, &self.archive.subtitle); - if !missing_subtitle.is_empty() { - warn!( - "Season {} is not available with {} subtitles", - season.season_number, - missing_subtitle - .into_iter() - .map(|l| l.to_string()) - .collect::>() - .join(", ") - ) - } - self.visited = Visited::Season - } - - let mut episodes = vec![]; - for season in seasons { - self.season_sorting.push(season.id.clone()); - let season_locale = if season.audio_locales.len() < 2 { - Some( - season - .audio_locales - .first() - .cloned() - .unwrap_or(Locale::ja_JP), - ) - } else { - None - }; - let mut eps = season.episodes().await?; - let before_len = eps.len(); - - for mut ep in eps.clone() { - if let Some(l) = &season_locale { - if &ep.audio_locale == l { - continue; - } - eps.remove(eps.iter().position(|p| p.id == ep.id).unwrap()); - } else { - let mut requested_locales = self.archive.audio.clone(); - if let Some(idx) = requested_locales.iter().position(|p| p == &ep.audio_locale) - { - requested_locales.remove(idx); - } else { - eps.remove(eps.iter().position(|p| p.id == ep.id).unwrap()); - } - eps.extend(ep.version(self.archive.audio.clone()).await?); - } - } - if eps.len() < before_len { - if eps.is_empty() { - if matches!(self.visited, Visited::Series) { - warn!( - "Season {} is not available with {} audio", - season.season_number, - season_locale.unwrap_or(Locale::ja_JP) - ) - } - } else { - let last_episode = eps.last().unwrap(); - warn!( - "Season {} is only available with {} audio until episode {} ({})", - season.season_number, - season_locale.unwrap_or(Locale::ja_JP), - last_episode.sequence_number, - last_episode.title - ) - } - } - episodes.extend(eps) - } - - if Format::has_relative_fmt(&self.archive.output) { - for episode in episodes.iter() { - self.season_episodes - .entry(episode.season_id.clone()) - .or_default() - .push(episode.clone()) - } - } - - Ok(episodes) - } - - async fn visit_episode(&mut self, mut episode: Episode) -> Result> { - if !self - .url_filter - .is_episode_valid(episode.sequence_number, episode.season_number) - { - return Ok(None); - } - - // skip the episode if it's a special - if self.skip_special - && (episode.sequence_number == 0.0 || episode.sequence_number.fract() != 0.0) - { - return Ok(None); - } - - let mut episodes = vec![]; - if !matches!(self.visited, Visited::Series) && !matches!(self.visited, Visited::Season) { - if self.archive.audio.contains(&episode.audio_locale) { - episodes.push((episode.clone(), episode.subtitle_locales.clone())) - } - episodes.extend( - episode - .version(self.archive.audio.clone()) - .await? - .into_iter() - .map(|e| (e.clone(), e.subtitle_locales.clone())), - ); - let audio_locales: Vec = episodes - .iter() - .map(|(e, _)| e.audio_locale.clone()) - .collect(); - let missing_audio = missing_locales(&audio_locales, &self.archive.audio); - if !missing_audio.is_empty() { - warn!( - "Episode {} is not available with {} audio", - episode.sequence_number, - missing_audio - .into_iter() - .map(|l| l.to_string()) - .collect::>() - .join(", ") - ) - } - - let mut subtitle_locales: Vec = - episodes.iter().flat_map(|(_, s)| s.clone()).collect(); - real_dedup_vec(&mut subtitle_locales); - let missing_subtitles = missing_locales(&subtitle_locales, &self.archive.subtitle); - if !missing_subtitles.is_empty() - && !self - .season_subtitles_missing - .contains(&episode.season_number) - { - warn!( - "Episode {} is not available with {} subtitles", - episode.sequence_number, - missing_subtitles - .into_iter() - .map(|l| l.to_string()) - .collect::>() - .join(", ") - ); - self.season_subtitles_missing.push(episode.season_number) - } - } else { - episodes.push((episode.clone(), episode.subtitle_locales.clone())) - } - - if self.seasons_with_premium.is_some() { - let episode_len_before = episodes.len(); - episodes.retain(|(e, _)| !e.is_premium_only); - if episode_len_before < episodes.len() - && !self - .seasons_with_premium - .as_ref() - .unwrap() - .contains(&episode.season_number) - { - warn!( - "Skipping premium episodes in season {}", - episode.season_number - ); - self.seasons_with_premium - .as_mut() - .unwrap() - .push(episode.season_number) - } - - if episodes.is_empty() { - return Ok(None); - } - } - - let mut relative_episode_number = None; - let mut relative_sequence_number = None; - // get the relative episode number. only done if the output string has the pattern to include - // the relative episode number as this requires some extra fetching - if Format::has_relative_fmt(&self.archive.output) { - let season_eps = match self.season_episodes.get(&episode.season_id) { - Some(eps) => eps, - None => { - self.season_episodes.insert( - episode.season_id.clone(), - episode.season().await?.episodes().await?, - ); - self.season_episodes.get(&episode.season_id).unwrap() - } - }; - let mut non_integer_sequence_number_count = 0; - for (i, ep) in season_eps.iter().enumerate() { - if ep.sequence_number.fract() != 0.0 || ep.sequence_number == 0.0 { - non_integer_sequence_number_count += 1; - } - if ep.id == episode.id { - relative_episode_number = Some(i + 1); - relative_sequence_number = Some( - (i + 1 - non_integer_sequence_number_count) as f32 - + fract(ep.sequence_number), - ); - break; - } - } - if relative_episode_number.is_none() || relative_sequence_number.is_none() { - warn!( - "Failed to get relative episode number for episode {} ({}) of {} season {}", - episode.sequence_number, - episode.title, - episode.series_title, - episode.season_number, - ) - } - } - - Ok(Some( - episodes - .into_iter() - .map(|(e, s)| { - SingleFormat::new_from_episode( - e, - s, - relative_episode_number.map(|n| n as u32), - relative_sequence_number, - ) - }) - .collect(), - )) - } - - async fn visit_movie_listing(&mut self, movie_listing: MovieListing) -> Result> { - Ok(movie_listing.movies().await?) - } - - async fn visit_movie(&mut self, movie: Movie) -> Result> { - Ok(Some(vec![SingleFormat::new_from_movie(movie, vec![])])) - } - - async fn visit_music_video(&mut self, music_video: MusicVideo) -> Result> { - Ok(Some(vec![SingleFormat::new_from_music_video(music_video)])) - } - - async fn visit_concert(&mut self, concert: Concert) -> Result> { - Ok(Some(vec![SingleFormat::new_from_concert(concert)])) - } - - async fn finish(self, input: Vec) -> Result { - let flatten_input: Self::T = input.into_iter().flatten().collect(); - - let mut single_format_collection = SingleFormatCollection::new(); - - let mut pre_sorted: BTreeMap = BTreeMap::new(); - for data in flatten_input { - pre_sorted - .entry(data.identifier.clone()) - .or_insert(vec![]) - .push(data) - } - - let mut sorted: Vec<(String, Self::T)> = pre_sorted.into_iter().collect(); - sorted.sort_by(|(_, a), (_, b)| { - self.season_sorting - .iter() - .position(|p| p == &a.first().unwrap().season_id) - .unwrap() - .cmp( - &self - .season_sorting - .iter() - .position(|p| p == &b.first().unwrap().season_id) - .unwrap(), - ) - }); - - for (_, mut data) in sorted { - data.sort_by(|a, b| { - self.archive - .audio - .iter() - .position(|p| p == &a.audio) - .unwrap_or(usize::MAX) - .cmp( - &self - .archive - .audio - .iter() - .position(|p| p == &b.audio) - .unwrap_or(usize::MAX), - ) - }); - single_format_collection.add_single_formats(data) - } - - Ok(single_format_collection) - } -} - -fn missing_locales<'a>(available: &[Locale], searched: &'a [Locale]) -> Vec<&'a Locale> { - searched.iter().filter(|p| !available.contains(p)).collect() -} diff --git a/crunchy-cli-core/src/archive/mod.rs b/crunchy-cli-core/src/archive/mod.rs index c3544a44..670d0c2e 100644 --- a/crunchy-cli-core/src/archive/mod.rs +++ b/crunchy-cli-core/src/archive/mod.rs @@ -1,4 +1,3 @@ mod command; -mod filter; pub use command::Archive; diff --git a/crunchy-cli-core/src/download/command.rs b/crunchy-cli-core/src/download/command.rs index fcf069be..8e3794fc 100644 --- a/crunchy-cli-core/src/download/command.rs +++ b/crunchy-cli-core/src/download/command.rs @@ -1,8 +1,7 @@ -use crate::download::filter::DownloadFilter; use crate::utils::context::Context; use crate::utils::download::{DownloadBuilder, DownloadFormat, DownloadFormatMetadata}; use crate::utils::ffmpeg::{FFmpegPreset, SOFTSUB_CONTAINERS}; -use crate::utils::filter::Filter; +use crate::utils::filter::{Filter, FilterMediaScope}; use crate::utils::format::{Format, SingleFormat}; use crate::utils::locale::{resolve_locales, LanguageTagging}; use crate::utils::log::progress; @@ -14,7 +13,7 @@ use anyhow::bail; use anyhow::Result; use crunchyroll_rs::media::Resolution; use crunchyroll_rs::Locale; -use log::{debug, warn}; +use log::{debug, error, warn}; use std::collections::HashMap; use std::path::Path; @@ -250,9 +249,53 @@ impl Execute for Download { for (i, (media_collection, url_filter)) in parsed_urls.into_iter().enumerate() { let progress_handler = progress!("Fetching series details"); - let single_format_collection = DownloadFilter::new( + let single_format_collection = Filter::new( url_filter, - self.clone(), + vec![self.audio.clone()], + self.subtitle.as_ref().map_or(vec![], |s| vec![s.clone()]), + |scope, locales| { + match scope { + FilterMediaScope::Series(series) => bail!("Series {} is not available with {} audio", series.title, locales[0]), + FilterMediaScope::Season(season) => { + error!("Season {} is not available with {} audio", season.season_number, locales[0]); + Ok(false) + } + FilterMediaScope::Episode(episodes) => { + if episodes.len() == 1 { + warn!("Episode {} of season {} is not available with {} audio", episodes[0].sequence_number, episodes[0].season_title, locales[0]) + } else if episodes.len() == 2 { + warn!("Season {} is only available with {} audio from episode {} to {}", episodes[0].season_number, locales[0], episodes[0].sequence_number, episodes[1].sequence_number) + } else { + unimplemented!() + } + Ok(false) + } + } + }, + |scope, locales| { + match scope { + FilterMediaScope::Series(series) => bail!("Series {} is not available with {} subtitles", series.title, locales[0]), + FilterMediaScope::Season(season) => { + warn!("Season {} is not available with {} subtitles", season.season_number, locales[0]); + Ok(false) + }, + FilterMediaScope::Episode(episodes) => { + if episodes.len() == 1 { + warn!("Episode {} of season {} is not available with {} subtitles", episodes[0].sequence_number, episodes[0].season_title, locales[0]) + } else if episodes.len() == 2 { + warn!("Season {} is only available with {} subtitles from episode {} to {}", episodes[0].season_number, locales[0], episodes[0].sequence_number, episodes[1].sequence_number) + } else { + unimplemented!() + } + Ok(false) + } + } + }, + |season| { + warn!("Skipping premium episodes in season {season}"); + Ok(()) + }, + Format::has_relative_fmt(&self.output), !self.yes, self.skip_specials, ctx.crunchy.premium().await, diff --git a/crunchy-cli-core/src/download/filter.rs b/crunchy-cli-core/src/download/filter.rs deleted file mode 100644 index 1c62920e..00000000 --- a/crunchy-cli-core/src/download/filter.rs +++ /dev/null @@ -1,307 +0,0 @@ -use crate::download::Download; -use crate::utils::filter::Filter; -use crate::utils::format::{Format, SingleFormat, SingleFormatCollection}; -use crate::utils::interactive_select::{check_for_duplicated_seasons, get_duplicated_seasons}; -use crate::utils::parse::{fract, UrlFilter}; -use anyhow::{bail, Result}; -use crunchyroll_rs::{Concert, Episode, Movie, MovieListing, MusicVideo, Season, Series}; -use log::{error, info, warn}; -use std::collections::HashMap; -use std::ops::Not; - -pub(crate) struct DownloadFilter { - url_filter: UrlFilter, - download: Download, - interactive_input: bool, - skip_special: bool, - season_episodes: HashMap>, - season_subtitles_missing: Vec, - seasons_with_premium: Option>, - season_visited: bool, -} - -impl DownloadFilter { - pub(crate) fn new( - url_filter: UrlFilter, - download: Download, - interactive_input: bool, - skip_special: bool, - is_premium: bool, - ) -> Self { - Self { - url_filter, - download, - interactive_input, - skip_special, - season_episodes: HashMap::new(), - season_subtitles_missing: vec![], - seasons_with_premium: is_premium.not().then_some(vec![]), - season_visited: false, - } - } -} - -impl Filter for DownloadFilter { - type T = SingleFormat; - type Output = SingleFormatCollection; - - async fn visit_series(&mut self, series: Series) -> Result> { - // `series.audio_locales` isn't always populated b/c of crunchyrolls api. so check if the - // audio is matching only if the field is populated - if !series.audio_locales.is_empty() && !series.audio_locales.contains(&self.download.audio) - { - error!( - "Series {} is not available with {} audio", - series.title, self.download.audio - ); - return Ok(vec![]); - } - - let mut seasons = vec![]; - for mut season in series.seasons().await? { - if !self.url_filter.is_season_valid(season.season_number) { - continue; - } - - if !season - .audio_locales - .iter() - .any(|l| l == &self.download.audio) - { - if season - .available_versions() - .await? - .iter() - .any(|l| l == &self.download.audio) - { - season = season - .version(vec![self.download.audio.clone()]) - .await? - .remove(0) - } else { - error!( - "Season {} - '{}' is not available with {} audio", - season.season_number, - season.title, - self.download.audio.clone(), - ); - continue; - } - } - - seasons.push(season) - } - - let duplicated_seasons = get_duplicated_seasons(&seasons); - if !duplicated_seasons.is_empty() { - if self.interactive_input { - check_for_duplicated_seasons(&mut seasons); - } else { - info!( - "Found duplicated seasons: {}", - duplicated_seasons - .iter() - .map(|d| d.to_string()) - .collect::>() - .join(", ") - ) - } - } - - Ok(seasons) - } - - async fn visit_season(&mut self, season: Season) -> Result> { - self.season_visited = true; - - let mut episodes = season.episodes().await?; - - if Format::has_relative_fmt(&self.download.output) { - for episode in episodes.iter() { - self.season_episodes - .entry(episode.season_number) - .or_default() - .push(episode.clone()) - } - } - - episodes.retain(|e| { - self.url_filter - .is_episode_valid(e.sequence_number, season.season_number) - }); - - Ok(episodes) - } - - async fn visit_episode(&mut self, mut episode: Episode) -> Result> { - if !self - .url_filter - .is_episode_valid(episode.sequence_number, episode.season_number) - { - return Ok(None); - } - - // skip the episode if it's a special - if self.skip_special - && (episode.sequence_number == 0.0 || episode.sequence_number.fract() != 0.0) - { - return Ok(None); - } - - // check if the audio locale is correct. - // should only be incorrect if the console input was a episode url. otherwise - // `DownloadFilter::visit_season` returns the correct episodes with matching audio - if episode.audio_locale != self.download.audio { - // check if any other version (same episode, other language) of this episode is available - // with the requested audio. if not, return an error - if !episode - .available_versions() - .await? - .contains(&self.download.audio) - { - let error_message = format!( - "Episode {} ({}) of {} season {} is not available with {} audio", - episode.sequence_number, - episode.title, - episode.series_title, - episode.season_number, - self.download.audio - ); - // sometimes a series randomly has episode in an other language. if this is the case, - // only error if the input url was a episode url - if self.season_visited { - warn!("{}", error_message); - return Ok(None); - } else { - bail!("{}", error_message) - } - } - // overwrite the current episode with the other version episode - episode = episode - .version(vec![self.download.audio.clone()]) - .await? - .remove(0) - } - - // check if the subtitles are supported - if let Some(subtitle_locale) = &self.download.subtitle { - if !episode.subtitle_locales.contains(subtitle_locale) { - // if the episode doesn't have the requested subtitles, print a error. to print this - // error only once per season, it's checked if an error got printed before by looking - // up if the season id is present in `self.season_subtitles_missing`. if not, print - // the error and add the season id to `self.season_subtitles_missing`. if it is - // present, skip the error printing - if !self - .season_subtitles_missing - .contains(&episode.season_number) - { - self.season_subtitles_missing.push(episode.season_number); - error!( - "{} season {} is not available with {} subtitles", - episode.series_title, episode.season_number, subtitle_locale - ); - } - return Ok(None); - } - } - - if self.seasons_with_premium.is_some() && episode.is_premium_only { - if !self - .seasons_with_premium - .as_ref() - .unwrap() - .contains(&episode.season_number) - { - warn!( - "Skipping premium episodes in season {}", - episode.season_number - ); - self.seasons_with_premium - .as_mut() - .unwrap() - .push(episode.season_number) - } - - return Ok(None); - } - - let mut relative_episode_number = None; - let mut relative_sequence_number = None; - // get the relative episode number. only done if the output string has the pattern to include - // the relative episode number as this requires some extra fetching - if Format::has_relative_fmt(&self.download.output) { - let season_eps = match self.season_episodes.get(&episode.season_number) { - Some(eps) => eps, - None => { - self.season_episodes.insert( - episode.season_number, - episode.season().await?.episodes().await?, - ); - self.season_episodes.get(&episode.season_number).unwrap() - } - }; - let mut non_integer_sequence_number_count = 0; - for (i, ep) in season_eps.iter().enumerate() { - if ep.sequence_number.fract() != 0.0 || ep.sequence_number == 0.0 { - non_integer_sequence_number_count += 1; - } - if ep.id == episode.id { - relative_episode_number = Some(i + 1); - relative_sequence_number = Some( - (i + 1 - non_integer_sequence_number_count) as f32 - + fract(ep.sequence_number), - ); - break; - } - } - if relative_episode_number.is_none() || relative_sequence_number.is_none() { - warn!( - "Failed to get relative episode number for episode {} ({}) of {} season {}", - episode.sequence_number, - episode.title, - episode.series_title, - episode.season_number, - ) - } - } - - Ok(Some(SingleFormat::new_from_episode( - episode.clone(), - self.download.subtitle.clone().map_or(vec![], |s| { - if episode.subtitle_locales.contains(&s) { - vec![s] - } else { - vec![] - } - }), - relative_episode_number.map(|n| n as u32), - relative_sequence_number, - ))) - } - - async fn visit_movie_listing(&mut self, movie_listing: MovieListing) -> Result> { - Ok(movie_listing.movies().await?) - } - - async fn visit_movie(&mut self, movie: Movie) -> Result> { - Ok(Some(SingleFormat::new_from_movie(movie, vec![]))) - } - - async fn visit_music_video(&mut self, music_video: MusicVideo) -> Result> { - Ok(Some(SingleFormat::new_from_music_video(music_video))) - } - - async fn visit_concert(&mut self, concert: Concert) -> Result> { - Ok(Some(SingleFormat::new_from_concert(concert))) - } - - async fn finish(self, input: Vec) -> Result { - let mut single_format_collection = SingleFormatCollection::new(); - - for data in input { - single_format_collection.add_single_formats(vec![data]) - } - - Ok(single_format_collection) - } -} diff --git a/crunchy-cli-core/src/download/mod.rs b/crunchy-cli-core/src/download/mod.rs index 696872e3..47ca304c 100644 --- a/crunchy-cli-core/src/download/mod.rs +++ b/crunchy-cli-core/src/download/mod.rs @@ -1,4 +1,3 @@ mod command; -mod filter; pub use command::Download; diff --git a/crunchy-cli-core/src/search/format.rs b/crunchy-cli-core/src/search/format.rs index 10f46245..cf3c5bc9 100644 --- a/crunchy-cli-core/src/search/format.rs +++ b/crunchy-cli-core/src/search/format.rs @@ -241,14 +241,6 @@ macro_rules! must_match_if_true { }; } -macro_rules! self_and_versions { - ($var:expr => $audio:expr) => {{ - let mut items = vec![$var.clone()]; - items.extend($var.clone().version($audio).await?); - items - }}; -} - pub struct Format { pattern: Vec<(Range, Scope, String)>, pattern_count: HashMap, @@ -421,7 +413,15 @@ impl Format { }; let mut seasons = vec![]; for season in tmp_seasons { - seasons.extend(self_and_versions!(season => self.filter_options.audio.clone())) + seasons.push(season.clone()); + for version in season.versions { + if season.id == version.id { + continue; + } + if self.filter_options.audio.contains(&version.audio_locale) { + seasons.push(version.season().await?) + } + } } tree.extend( self.filter_options @@ -435,7 +435,15 @@ impl Format { if !episode_empty || !stream_empty { match &media_collection { MediaCollection::Episode(episode) => { - let episodes = self_and_versions!(episode => self.filter_options.audio.clone()); + let mut episodes = vec![episode.clone()]; + for version in &episode.versions { + if episode.id == version.id { + continue; + } + if self.filter_options.audio.contains(&version.audio_locale) { + episodes.push(version.episode().await?) + } + } tree.push(( Season::default(), episodes diff --git a/crunchy-cli-core/src/utils/filter.rs b/crunchy-cli-core/src/utils/filter.rs index 63fac9d6..b95596e1 100644 --- a/crunchy-cli-core/src/utils/filter.rs +++ b/crunchy-cli-core/src/utils/filter.rs @@ -1,24 +1,397 @@ +use crate::utils::format::{SingleFormat, SingleFormatCollection}; +use crate::utils::interactive_select::{check_for_duplicated_seasons, get_duplicated_seasons}; +use crate::utils::parse::{fract, UrlFilter}; use anyhow::Result; use crunchyroll_rs::{ - Concert, Episode, MediaCollection, Movie, MovieListing, MusicVideo, Season, Series, + Concert, Episode, Locale, MediaCollection, Movie, MovieListing, MusicVideo, Season, Series, }; +use log::{info, warn}; +use std::collections::{BTreeMap, HashMap}; +use std::ops::Not; -pub trait Filter { - type T: Send + Sized; - type Output: Send + Sized; - - async fn visit_series(&mut self, series: Series) -> Result>; - async fn visit_season(&mut self, season: Season) -> Result>; - async fn visit_episode(&mut self, episode: Episode) -> Result>; - async fn visit_movie_listing(&mut self, movie_listing: MovieListing) -> Result>; - async fn visit_movie(&mut self, movie: Movie) -> Result>; - async fn visit_music_video(&mut self, music_video: MusicVideo) -> Result>; - async fn visit_concert(&mut self, concert: Concert) -> Result>; - - async fn visit(mut self, media_collection: MediaCollection) -> Result - where - Self: Send + Sized, - { +pub(crate) enum FilterMediaScope<'a> { + Series(&'a Series), + Season(&'a Season), + /// Always contains 1 or 2 episodes. + /// - 1: The episode's audio is completely missing + /// - 2: The requested audio is only available from first entry to last entry + Episode(Vec<&'a Episode>), +} + +pub(crate) struct Filter { + url_filter: UrlFilter, + + skip_specials: bool, + interactive_input: bool, + + relative_episode_number: bool, + + audio_locales: Vec, + subtitle_locales: Vec, + + audios_missing: fn(FilterMediaScope, Vec<&Locale>) -> Result, + subtitles_missing: fn(FilterMediaScope, Vec<&Locale>) -> Result, + no_premium: fn(u32) -> Result<()>, + + is_premium: bool, + + series_visited: bool, + season_episodes: HashMap>, + season_with_premium: Option>, + season_sorting: Vec, +} + +impl Filter { + #[allow(clippy::too_many_arguments)] + pub(crate) fn new( + url_filter: UrlFilter, + audio_locales: Vec, + subtitle_locales: Vec, + audios_missing: fn(FilterMediaScope, Vec<&Locale>) -> Result, + subtitles_missing: fn(FilterMediaScope, Vec<&Locale>) -> Result, + no_premium: fn(u32) -> Result<()>, + relative_episode_number: bool, + interactive_input: bool, + skip_specials: bool, + is_premium: bool, + ) -> Self { + Self { + url_filter, + audio_locales, + subtitle_locales, + relative_episode_number, + interactive_input, + audios_missing, + subtitles_missing, + no_premium, + is_premium, + series_visited: false, + season_episodes: HashMap::new(), + skip_specials, + season_with_premium: is_premium.not().then_some(vec![]), + season_sorting: vec![], + } + } + + async fn visit_series(&mut self, series: Series) -> Result> { + // the audio locales field isn't always populated + if !series.audio_locales.is_empty() { + let missing_audios = missing_locales(&series.audio_locales, &self.audio_locales); + if !missing_audios.is_empty() + && !(self.audios_missing)(FilterMediaScope::Series(&series), missing_audios)? + { + return Ok(vec![]); + } + let missing_subtitles = + missing_locales(&series.subtitle_locales, &self.subtitle_locales); + if !missing_subtitles.is_empty() + && !(self.subtitles_missing)(FilterMediaScope::Series(&series), missing_subtitles)? + { + return Ok(vec![]); + } + } + + let mut seasons = vec![]; + for season in series.seasons().await? { + if !self.url_filter.is_season_valid(season.season_number) { + continue; + } + let missing_audios = missing_locales( + &season + .versions + .iter() + .map(|l| l.audio_locale.clone()) + .collect::>(), + &self.audio_locales, + ); + if !missing_audios.is_empty() + && !(self.audios_missing)(FilterMediaScope::Season(&season), missing_audios)? + { + return Ok(vec![]); + } + seasons.push(season) + } + + let duplicated_seasons = get_duplicated_seasons(&seasons); + if !duplicated_seasons.is_empty() { + if self.interactive_input { + check_for_duplicated_seasons(&mut seasons) + } else { + info!( + "Found duplicated seasons: {}", + duplicated_seasons + .iter() + .map(|d| d.to_string()) + .collect::>() + .join(", ") + ) + } + } + + self.series_visited = true; + + Ok(seasons) + } + + async fn visit_season(&mut self, season: Season) -> Result> { + if !self.url_filter.is_season_valid(season.season_number) { + return Ok(vec![]); + } + + let mut seasons = vec![]; + if self + .audio_locales + .iter() + .any(|l| season.audio_locales.contains(l)) + { + seasons.push(season.clone()) + } + for version in season.versions { + if season.id == version.id { + continue; + } + if self.audio_locales.contains(&version.audio_locale) { + seasons.push(version.season().await?) + } + } + + let mut episodes = vec![]; + for season in seasons { + self.season_sorting.push(season.id.clone()); + let mut eps = season.episodes().await?; + + // removes any episode that does not have the audio locale of the season. yes, this is + // the case sometimes + if season.audio_locales.len() < 2 { + let season_locale = season + .audio_locales + .first() + .cloned() + .unwrap_or(Locale::ja_JP); + eps.retain(|e| e.audio_locale == season_locale) + } + + if eps.len() < season.number_of_episodes as usize + && !(self.audios_missing)( + FilterMediaScope::Episode(vec![eps.first().unwrap(), eps.last().unwrap()]), + vec![&eps.first().unwrap().audio_locale], + )? + { + return Ok(vec![]); + } + + episodes.extend(eps) + } + + if self.relative_episode_number { + for episode in &episodes { + self.season_episodes + .entry(episode.season_id.clone()) + .or_default() + .push(episode.clone()) + } + } + + Ok(episodes) + } + + async fn visit_episode(&mut self, episode: Episode) -> Result> { + if !self + .url_filter + .is_episode_valid(episode.sequence_number, episode.season_number) + { + return Ok(vec![]); + } + + // skip the episode if it's a special + if self.skip_specials + && (episode.sequence_number == 0.0 || episode.sequence_number.fract() != 0.0) + { + return Ok(vec![]); + } + + let mut episodes = vec![]; + if !self.series_visited { + if self.audio_locales.contains(&episode.audio_locale) { + episodes.push(episode.clone()) + } + for version in &episode.versions { + // `episode` is also a version of itself. the if block above already adds the + // episode if it matches the requested audio, so it doesn't need to be requested + // here again + if version.id == episode.id { + continue; + } + if self.audio_locales.contains(&version.audio_locale) { + episodes.push(version.episode().await?) + } + } + + let audio_locales: Vec = + episodes.iter().map(|e| e.audio_locale.clone()).collect(); + let missing_audios = missing_locales(&audio_locales, &self.audio_locales); + if !missing_audios.is_empty() + && !(self.audios_missing)( + FilterMediaScope::Episode(vec![&episode]), + missing_audios, + )? + { + return Ok(vec![]); + } + + let mut subtitle_locales: Vec = episodes + .iter() + .flat_map(|e| e.subtitle_locales.clone()) + .collect(); + subtitle_locales.sort(); + subtitle_locales.dedup(); + let missing_subtitles = missing_locales(&subtitle_locales, &self.subtitle_locales); + if !missing_subtitles.is_empty() + && !(self.subtitles_missing)( + FilterMediaScope::Episode(vec![&episode]), + missing_subtitles, + )? + { + return Ok(vec![]); + } + } else { + episodes.push(episode.clone()) + } + + if let Some(seasons_with_premium) = &mut self.season_with_premium { + let episodes_len_before = episodes.len(); + episodes.retain(|e| !e.is_premium_only && !self.is_premium); + if episodes_len_before < episodes.len() + && !seasons_with_premium.contains(&episode.season_number) + { + (self.no_premium)(episode.season_number)?; + seasons_with_premium.push(episode.season_number) + } + + if episodes.is_empty() { + return Ok(vec![]); + } + } + + let mut relative_episode_number = None; + let mut relative_sequence_number = None; + if self.relative_episode_number { + let season_eps = match self.season_episodes.get(&episode.season_id) { + Some(eps) => eps, + None => { + self.season_episodes.insert( + episode.season_id.clone(), + episode.season().await?.episodes().await?, + ); + self.season_episodes.get(&episode.season_id).unwrap() + } + }; + let mut non_integer_sequence_number_count = 0; + for (i, ep) in season_eps.iter().enumerate() { + if ep.sequence_number != 0.0 || ep.sequence_number.fract() == 0.0 { + non_integer_sequence_number_count += 1 + } + if ep.id == episode.id { + relative_episode_number = Some(i + 1); + relative_sequence_number = Some( + (i + 1 - non_integer_sequence_number_count) as f32 + + fract(ep.sequence_number), + ); + break; + } + } + if relative_episode_number.is_none() || relative_sequence_number.is_none() { + warn!( + "Failed to get relative episode number for episode {} ({}) of {} season {}", + episode.sequence_number, + episode.title, + episode.series_title, + episode.season_number, + ) + } + } + + Ok(episodes + .into_iter() + .map(|e| { + SingleFormat::new_from_episode( + e.clone(), + e.subtitle_locales, + relative_episode_number.map(|n| n as u32), + relative_sequence_number, + ) + }) + .collect()) + } + + async fn visit_movie_listing(&mut self, movie_listing: MovieListing) -> Result> { + Ok(movie_listing.movies().await?) + } + + async fn visit_movie(&mut self, movie: Movie) -> Result> { + Ok(vec![SingleFormat::new_from_movie(movie, vec![])]) + } + + async fn visit_music_video(&mut self, music_video: MusicVideo) -> Result> { + Ok(vec![SingleFormat::new_from_music_video(music_video)]) + } + + async fn visit_concert(&mut self, concert: Concert) -> Result> { + Ok(vec![SingleFormat::new_from_concert(concert)]) + } + + async fn finish(self, input: Vec>) -> Result { + let flatten_input: Vec = input.into_iter().flatten().collect(); + + let mut single_format_collection = SingleFormatCollection::new(); + + let mut pre_sorted: BTreeMap> = BTreeMap::new(); + for data in flatten_input { + pre_sorted + .entry(data.identifier.clone()) + .or_default() + .push(data) + } + + let mut sorted: Vec<(String, Vec)> = pre_sorted.into_iter().collect(); + sorted.sort_by(|(_, a), (_, b)| { + self.season_sorting + .iter() + .position(|p| p == &a.first().unwrap().season_id) + .unwrap() + .cmp( + &self + .season_sorting + .iter() + .position(|p| p == &b.first().unwrap().season_id) + .unwrap(), + ) + }); + + for (_, mut data) in sorted { + data.sort_by(|a, b| { + self.audio_locales + .iter() + .position(|p| p == &a.audio) + .unwrap_or(usize::MAX) + .cmp( + &self + .audio_locales + .iter() + .position(|p| p == &b.audio) + .unwrap_or(usize::MAX), + ) + }); + single_format_collection.add_single_formats(data) + } + + Ok(single_format_collection) + } + + pub(crate) async fn visit( + mut self, + media_collection: MediaCollection, + ) -> Result { let mut items = vec![media_collection]; let mut result = vec![]; @@ -42,9 +415,7 @@ pub trait Filter { .collect::>(), ), MediaCollection::Episode(episode) => { - if let Some(t) = self.visit_episode(episode).await? { - result.push(t) - } + result.push(self.visit_episode(episode).await?) } MediaCollection::MovieListing(movie_listing) => new_items.extend( self.visit_movie_listing(movie_listing) @@ -53,20 +424,12 @@ pub trait Filter { .map(|m| m.into()) .collect::>(), ), - MediaCollection::Movie(movie) => { - if let Some(t) = self.visit_movie(movie).await? { - result.push(t) - } - } + MediaCollection::Movie(movie) => result.push(self.visit_movie(movie).await?), MediaCollection::MusicVideo(music_video) => { - if let Some(t) = self.visit_music_video(music_video).await? { - result.push(t) - } + result.push(self.visit_music_video(music_video).await?) } MediaCollection::Concert(concert) => { - if let Some(t) = self.visit_concert(concert).await? { - result.push(t) - } + result.push(self.visit_concert(concert).await?) } } } @@ -76,8 +439,10 @@ pub trait Filter { self.finish(result).await } +} - async fn finish(self, input: Vec) -> Result; +fn missing_locales<'a>(available: &[Locale], searched: &'a [Locale]) -> Vec<&'a Locale> { + searched.iter().filter(|p| !available.contains(p)).collect() } /// Remove all duplicates from a [`Vec`]. diff --git a/crunchy-cli-core/src/utils/format.rs b/crunchy-cli-core/src/utils/format.rs index f0a002c6..33ce2610 100644 --- a/crunchy-cli-core/src/utils/format.rs +++ b/crunchy-cli-core/src/utils/format.rs @@ -12,6 +12,7 @@ use std::collections::BTreeMap; use std::env; use std::path::{Path, PathBuf}; +#[allow(dead_code)] #[derive(Clone)] pub struct SingleFormat { pub identifier: String, @@ -347,6 +348,7 @@ impl Iterator for SingleFormatCollectionIterator { } } +#[allow(dead_code)] #[derive(Clone)] pub struct Format { pub title: String,