Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: 引入更健壮的新视频检测方法 #228

Merged
merged 4 commits into from
Jan 22, 2025
Merged

feat: 引入更健壮的新视频检测方法 #228

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
5177d86
feat: 为各个 video list 表添加 latest_row_at 字段
amtoaer Jan 22, 2025
e83344c
chore: 为 model 引入新增的字段
amtoaer Jan 22, 2025
908f35f
feat: 实现新版中断条件（待测试）
amtoaer Jan 22, 2025
c5203d6
test: 更新测试
amtoaer Jan 22, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
45 changes: 20 additions & 25 deletions crates/bili_sync/src/adapter/collection.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,3 @@
use std::collections::HashSet;
use std::path::Path;
use std::pin::Pin;

Expand All @@ -9,18 +8,14 @@ use futures::Stream;
use sea_orm::entity::prelude::*;
use sea_orm::sea_query::{IntoCondition, OnConflict};
use sea_orm::ActiveValue::Set;
use sea_orm::{DatabaseConnection, TransactionTrait};
use sea_orm::{DatabaseConnection, TransactionTrait, Unchanged};

use crate::adapter::{helper, VideoListModel};
use crate::bilibili::{self, BiliClient, Collection, CollectionItem, CollectionType, VideoInfo};
use crate::utils::status::Status;

#[async_trait]
impl VideoListModel for collection::Model {
async fn video_count(&self, connection: &DatabaseConnection) -> Result<u64> {
helper::count_videos(video::Column::CollectionId.eq(self.id).into_condition(), connection).await
}

async fn unfilled_videos(&self, connection: &DatabaseConnection) -> Result<Vec<video::Model>> {
helper::filter_videos(
video::Column::CollectionId
Expand Down Expand Up @@ -52,20 +47,6 @@ impl VideoListModel for collection::Model {
.await
}

async fn exist_labels(
&self,
videos_info: &[VideoInfo],
connection: &DatabaseConnection,
) -> Result<HashSet<String>> {
helper::video_keys(
video::Column::CollectionId.eq(self.id),
videos_info,
[video::Column::Bvid, video::Column::Pubtime],
connection,
)
.await
}

fn video_model_by_info(&self, video_info: &VideoInfo, base_model: Option<video::Model>) -> video::ActiveModel {
let mut video_model = video_info.to_model(base_model);
video_model.collection_id = Set(Some(self.id));
Expand All @@ -81,7 +62,7 @@ impl VideoListModel for collection::Model {
let info: Result<_> = async { Ok((video.get_tags().await?, video.get_view_info().await?)) }.await;
match info {
Ok((tags, view_info)) => {
let VideoInfo::View { pages, .. } = &view_info else {
let VideoInfo::Detail { pages, .. } = &view_info else {
unreachable!("view_info must be VideoInfo::View")
};
let txn = connection.begin().await?;
Expand All @@ -101,6 +82,21 @@ impl VideoListModel for collection::Model {
Ok(())
}

fn get_latest_row_at(&self) -> DateTime {
self.latest_row_at
}

async fn update_latest_row_at(&self, datetime: DateTime, connection: &DatabaseConnection) -> Result<()> {
collection::ActiveModel {
id: Unchanged(self.id),
latest_row_at: Set(datetime),
..Default::default()
}
.update(connection)
.await?;
Ok(())
}

fn log_fetch_video_start(&self) {
info!(
"开始获取{} {} - {} 的视频与分页信息...",
Expand Down Expand Up @@ -146,14 +142,13 @@ impl VideoListModel for collection::Model {
);
}

fn log_refresh_video_end(&self, got_count: usize, new_count: u64) {
fn log_refresh_video_end(&self, count: usize) {
info!(
"扫描{}: {} - {} 的新视频完成，获取了 {} 条新视频，其中有 {} 条新视频",
"扫描{}: {} - {} 的新视频完成，获取了 {} 条新视频",
CollectionType::from(self.r#type),
self.s_id,
self.name,
got_count,
new_count,
count,
);
}
}
Expand Down
42 changes: 19 additions & 23 deletions crates/bili_sync/src/adapter/favorite.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,3 @@
use std::collections::HashSet;
use std::path::Path;
use std::pin::Pin;

Expand All @@ -9,18 +8,14 @@ use futures::Stream;
use sea_orm::entity::prelude::*;
use sea_orm::sea_query::{IntoCondition, OnConflict};
use sea_orm::ActiveValue::Set;
use sea_orm::{DatabaseConnection, TransactionTrait};
use sea_orm::{DatabaseConnection, TransactionTrait, Unchanged};

use crate::adapter::{helper, VideoListModel};
use crate::bilibili::{self, BiliClient, FavoriteList, VideoInfo};
use crate::utils::status::Status;

#[async_trait]
impl VideoListModel for favorite::Model {
async fn video_count(&self, connection: &DatabaseConnection) -> Result<u64> {
helper::count_videos(video::Column::FavoriteId.eq(self.id).into_condition(), connection).await
}

async fn unfilled_videos(&self, connection: &DatabaseConnection) -> Result<Vec<video::Model>> {
helper::filter_videos(
video::Column::FavoriteId
Expand Down Expand Up @@ -52,20 +47,6 @@ impl VideoListModel for favorite::Model {
.await
}

async fn exist_labels(
&self,
videos_info: &[VideoInfo],
connection: &DatabaseConnection,
) -> Result<HashSet<String>> {
helper::video_keys(
video::Column::FavoriteId.eq(self.id),
videos_info,
[video::Column::Bvid, video::Column::Favtime],
connection,
)
.await
}

fn video_model_by_info(&self, video_info: &VideoInfo, base_model: Option<video::Model>) -> video::ActiveModel {
let mut video_model = video_info.to_model(base_model);
video_model.favorite_id = Set(Some(self.id));
Expand Down Expand Up @@ -98,6 +79,21 @@ impl VideoListModel for favorite::Model {
Ok(())
}

fn get_latest_row_at(&self) -> DateTime {
self.latest_row_at
}

async fn update_latest_row_at(&self, datetime: DateTime, connection: &DatabaseConnection) -> Result<()> {
favorite::ActiveModel {
id: Unchanged(self.id),
latest_row_at: Set(datetime),
..Default::default()
}
.update(connection)
.await?;
Ok(())
}

fn log_fetch_video_start(&self) {
info!("开始获取收藏夹 {} - {} 的视频与分页信息...", self.f_id, self.name);
}
Expand All @@ -118,10 +114,10 @@ impl VideoListModel for favorite::Model {
info!("开始扫描收藏夹: {} - {} 的新视频...", self.f_id, self.name);
}

fn log_refresh_video_end(&self, got_count: usize, new_count: u64) {
fn log_refresh_video_end(&self, count: usize) {
info!(
"扫描收藏夹: {} - {} 的新视频完成，获取了 {} 条新视频，其中有 {} 条新视频",
self.f_id, self.name, got_count, new_count
"扫描收藏夹: {} - {} 的新视频完成，获取了 {} 条新视频",
self.f_id, self.name, count
);
}
}
Expand Down
34 changes: 2 additions & 32 deletions crates/bili_sync/src/adapter/helper/mod.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,21 +1,14 @@
use std::collections::HashSet;
use std::path::Path;

use anyhow::Result;
use bili_sync_entity::*;
use sea_orm::entity::prelude::*;
use sea_orm::sea_query::{OnConflict, SimpleExpr};
use sea_orm::sea_query::OnConflict;
use sea_orm::ActiveValue::Set;
use sea_orm::{Condition, DatabaseTransaction, QuerySelect};
use sea_orm::{Condition, DatabaseTransaction};

use crate::bilibili::{BiliError, PageInfo, VideoInfo};
use crate::config::{PathSafeTemplate, TEMPLATE};
use crate::utils::id_time_key;

/// 使用 condition 筛选视频，返回视频数量
pub(super) async fn count_videos(condition: Condition, conn: &DatabaseConnection) -> Result<u64> {
Ok(video::Entity::find().filter(condition).count(conn).await?)
}

/// 使用 condition 筛选视频，返回视频列表
pub(super) async fn filter_videos(condition: Condition, conn: &DatabaseConnection) -> Result<Vec<video::Model>> {
Expand All @@ -34,29 +27,6 @@ pub(super) async fn filter_videos_with_pages(
.await?)
}

/// 返回 videos_info 存在于视频表里那部分对应的 key
pub(super) async fn video_keys(
expr: SimpleExpr,
videos_info: &[VideoInfo],
columns: [video::Column; 2],
conn: &DatabaseConnection,
) -> Result<HashSet<String>> {
Ok(video::Entity::find()
.filter(
video::Column::Bvid
.is_in(videos_info.iter().map(|v| v.bvid().to_string()))
.and(expr),
)
.select_only()
.columns(columns)
.into_tuple()
.all(conn)
.await?
.into_iter()
.map(|(bvid, time)| id_time_key(&bvid, &time))
.collect())
}

/// 返回设置了 path 的视频
pub(super) fn video_with_path(
mut video_model: video::ActiveModel,
Expand Down
16 changes: 7 additions & 9 deletions crates/bili_sync/src/adapter/mod.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,6 @@ mod helper;
mod submission;
mod watch_later;

use std::collections::HashSet;
use std::path::Path;
use std::pin::Pin;

Expand Down Expand Up @@ -43,9 +42,6 @@ pub async fn video_list_from<'a>(

#[async_trait]
pub trait VideoListModel {
/// 与视频列表关联的视频总数
async fn video_count(&self, connection: &DatabaseConnection) -> Result<u64>;

/// 未填充的视频
async fn unfilled_videos(&self, connection: &DatabaseConnection) -> Result<Vec<bili_sync_entity::video::Model>>;

Expand All @@ -55,10 +51,6 @@ pub trait VideoListModel {
connection: &DatabaseConnection,
) -> Result<Vec<(bili_sync_entity::video::Model, Vec<bili_sync_entity::page::Model>)>>;

/// 该批次视频的存在标记
async fn exist_labels(&self, videos_info: &[VideoInfo], connection: &DatabaseConnection)
-> Result<HashSet<String>>;

/// 视频信息对应的视频 model
fn video_model_by_info(
&self,
Expand All @@ -74,6 +66,12 @@ pub trait VideoListModel {
connection: &DatabaseConnection,
) -> Result<()>;

/// 获取视频 model 中记录的最新时间
fn get_latest_row_at(&self) -> DateTime;

/// 更新视频 model 中记录的最新时间
async fn update_latest_row_at(&self, datetime: DateTime, connection: &DatabaseConnection) -> Result<()>;

/// 开始获取视频
fn log_fetch_video_start(&self);

Expand All @@ -90,5 +88,5 @@ pub trait VideoListModel {
fn log_refresh_video_start(&self);

/// 结束刷新视频
fn log_refresh_video_end(&self, got_count: usize, new_count: u64);
fn log_refresh_video_end(&self, count: usize);
}
44 changes: 20 additions & 24 deletions crates/bili_sync/src/adapter/submission.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,3 @@
use std::collections::HashSet;
use std::path::Path;
use std::pin::Pin;

Expand All @@ -9,7 +8,7 @@ use futures::Stream;
use sea_orm::entity::prelude::*;
use sea_orm::sea_query::{IntoCondition, OnConflict};
use sea_orm::ActiveValue::Set;
use sea_orm::{DatabaseConnection, TransactionTrait};
use sea_orm::{DatabaseConnection, TransactionTrait, Unchanged};

use crate::adapter::helper::video_with_path;
use crate::adapter::{helper, VideoListModel};
Expand All @@ -18,10 +17,6 @@ use crate::utils::status::Status;

#[async_trait]
impl VideoListModel for submission::Model {
async fn video_count(&self, connection: &DatabaseConnection) -> Result<u64> {
helper::count_videos(video::Column::SubmissionId.eq(self.id).into_condition(), connection).await
}

async fn unfilled_videos(&self, connection: &DatabaseConnection) -> Result<Vec<video::Model>> {
helper::filter_videos(
video::Column::SubmissionId
Expand Down Expand Up @@ -53,20 +48,6 @@ impl VideoListModel for submission::Model {
.await
}

async fn exist_labels(
&self,
videos_info: &[VideoInfo],
connection: &DatabaseConnection,
) -> Result<HashSet<String>> {
helper::video_keys(
video::Column::SubmissionId.eq(self.id),
videos_info,
[video::Column::Bvid, video::Column::Ctime],
connection,
)
.await
}

fn video_model_by_info(&self, video_info: &VideoInfo, base_model: Option<video::Model>) -> video::ActiveModel {
let mut video_model = video_info.to_model(base_model);
video_model.submission_id = Set(Some(self.id));
Expand All @@ -82,7 +63,7 @@ impl VideoListModel for submission::Model {
let info: Result<_> = async { Ok((video.get_tags().await?, video.get_view_info().await?)) }.await;
match info {
Ok((tags, view_info)) => {
let VideoInfo::View { pages, .. } = &view_info else {
let VideoInfo::Detail { pages, .. } = &view_info else {
unreachable!("view_info must be VideoInfo::View")
};
let txn = connection.begin().await?;
Expand All @@ -102,6 +83,21 @@ impl VideoListModel for submission::Model {
Ok(())
}

fn get_latest_row_at(&self) -> DateTime {
self.latest_row_at
}

async fn update_latest_row_at(&self, datetime: DateTime, connection: &DatabaseConnection) -> Result<()> {
submission::ActiveModel {
id: Unchanged(self.id),
latest_row_at: Set(datetime),
..Default::default()
}
.update(connection)
.await?;
Ok(())
}

fn log_fetch_video_start(&self) {
info!(
"开始获取 UP 主 {} - {} 投稿的视频与分页信息...",
Expand Down Expand Up @@ -134,10 +130,10 @@ impl VideoListModel for submission::Model {
info!("开始扫描 UP 主 {} - {} 投稿的新视频...", self.upper_id, self.upper_name);
}

fn log_refresh_video_end(&self, got_count: usize, new_count: u64) {
fn log_refresh_video_end(&self, count: usize) {
info!(
"扫描 UP 主 {} - {} 投稿的新视频完成，获取了 {} 条新视频，其中有 {} 条新视频",
self.upper_id, self.upper_name, got_count, new_count,
"扫描 UP 主 {} - {} 投稿的新视频完成，获取了 {} 条新视频",
self.upper_id, self.upper_name, count,
);
}
}
Expand Down
Loading
Loading