From 49e57996c172ef21ee9f1a03d10884f2578d98ed Mon Sep 17 00:00:00 2001 From: orestonce Date: Thu, 22 Aug 2024 09:45:31 +0800 Subject: [PATCH] =?UTF-8?q?=E8=A7=84=E9=81=BF=20#bug:=20=E5=90=88=E5=B9=B6?= =?UTF-8?q?=E5=87=BA=E6=9D=A5=E7=9A=84mp4=E8=B6=85=E8=BF=874GB=E4=BC=9A?= =?UTF-8?q?=E5=87=BA=E9=94=99=20=E7=94=9F=E6=88=90=20ffmpeg=E8=87=AA?= =?UTF-8?q?=E8=A1=8C=E5=90=88=E5=B9=B6=E7=9A=84=E5=91=BD=E4=BB=A4=20?= =?UTF-8?q?=E8=AE=B0=E5=BD=95=E9=80=9A=E8=BF=87http.code=E8=B7=B3=E8=BF=87?= =?UTF-8?q?=E7=9A=84ts=E6=96=87=E4=BB=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- api.go | 68 ++++++++++++++++++++++++++++++++++++----------------- cmd/main.go | 8 ++++++- download.go | 38 ++++++++++++++++++++++++++++++ speed.go | 6 ++--- 4 files changed, 94 insertions(+), 26 deletions(-) diff --git a/api.go b/api.go index fcb85f3..3efb4cd 100644 --- a/api.go +++ b/api.go @@ -10,7 +10,6 @@ import ( "net/http" "net/url" "os" - "path" "path/filepath" "regexp" "sort" @@ -20,6 +19,8 @@ import ( "time" ) +const logFileName = `skip_by_http_code.txt` + func (this *DownloadEnv) StartDownload(req StartDownload_Req) (errMsg string) { this.status.Locker.Lock() defer this.status.Locker.Unlock() @@ -46,7 +47,7 @@ func (this *DownloadEnv) StartDownload(req StartDownload_Req) (errMsg string) { this.status.clearStatusNoLock() - this.status.progressBarShow = req.ProgressBarShow + this.status.ProgressBarShow = req.ProgressBarShow this.ctx, this.cancelFn = context.WithCancel(context.Background()) this.status.IsRunning = true go func() { @@ -266,9 +267,9 @@ func (this *DownloadEnv) runDownload(req StartDownload_Req, skipInfo SkipTsInfo) return } videoId := req.getVideoId() - videoDownloadDir := filepath.Join(req.SaveDir, "downloading", videoId) - if !isDirExists(videoDownloadDir) { - err = os.MkdirAll(videoDownloadDir, os.ModePerm) + tsSaveDir := filepath.Join(req.SaveDir, "downloading", videoId) + if !isDirExists(tsSaveDir) { + err = os.MkdirAll(tsSaveDir, os.ModePerm) if err != nil { this.setErrMsg("os.MkdirAll error: " + err.Error()) return @@ -278,7 +279,7 @@ func (this *DownloadEnv) runDownload(req StartDownload_Req, skipInfo SkipTsInfo) if this.logFile != nil { this.logFile.Sync() this.logFile.Close() - persistDebugFilePath := filepath.Join(videoDownloadDir, "debuglog.txt") + persistDebugFilePath := filepath.Join(tsSaveDir, "debuglog.txt") err = os.Rename(tempDebugFilePath, persistDebugFilePath) if err != nil { this.setErrMsg("os.Rename set persistDebugFilePath " + strconv.Quote(persistDebugFilePath) + " error : " + err.Error()) @@ -310,22 +311,10 @@ func (this *DownloadEnv) runDownload(req StartDownload_Req, skipInfo SkipTsInfo) this.setErrMsg("需要下载的文件为空") return } - if req.DebugLog { - buf := bytes.NewBuffer(nil) - buf.WriteString(">tsList\n") - for _, one := range tsList { - urlObj, _ := url.Parse(one.Url) - if urlObj == nil { - continue - } - fmt.Fprintf(buf, " %v : %v\n", one.Name, path.Base(urlObj.Path)) - } - this.logToFile(buf.String()) - } // 下载ts this.status.SetProgressBarTitle("[3/4]下载ts") this.status.SpeedResetBytes() - err = this.downloader(tsList, skipInfo, videoDownloadDir, encInfo, req.ThreadCount) + err = this.downloader(tsList, skipInfo, tsSaveDir, encInfo, req.ThreadCount) this.status.SpeedResetBytes() if err != nil { this.setErrMsg("下载ts文件错误: " + err.Error()) @@ -336,14 +325,49 @@ func (this *DownloadEnv) runDownload(req StartDownload_Req, skipInfo SkipTsInfo) return } var tsFileList []string + var skipByHttpCodeLog bytes.Buffer + var skipCount int + var expectMp4Bytes int64 for _, one := range tsList { if one.SkipByHttpCode { + skipCount++ + fmt.Fprintf(&skipByHttpCodeLog, "http.code=%v,filename=%v,url=%v\n", one.HttpCode, one.Name, one.Url) continue } - tsFileList = append(tsFileList, filepath.Join(videoDownloadDir, one.Name)) + fileNameFull := filepath.Join(tsSaveDir, one.Name) + var stat os.FileInfo + stat, err = os.Stat(fileNameFull) + if err != nil { + this.setErrMsg("ts文件" + one.Name + "分析失败: " + err.Error()) + return + } + expectMp4Bytes += stat.Size() + tsFileList = append(tsFileList, fileNameFull) + } + if skipByHttpCodeLog.Len() > 0 && false { + err = os.WriteFile(filepath.Join(tsSaveDir, logFileName), skipByHttpCodeLog.Bytes(), 0777) + if err != nil { + this.setErrMsg("写入" + logFileName + "失败, " + err.Error()) + return + } + if this.writeFfmpegCmd(tsSaveDir, tsList) == false { + return + } + this.setErrMsg("使用http.code跳过了" + strconv.Itoa(skipCount) + "条ts记录，请自行合并") + return + } + + //TODO: gomedia 暂未修复的bug,输出mp4超过4GB就会出错, 此处预估3.9GB + gbCount := float64(expectMp4Bytes) / 1024 / 1024 / 1024 + if gbCount > 3.9 { + if this.writeFfmpegCmd(tsSaveDir, tsList) == false { + return + } + this.setErrMsg("预期大小" + strconv.FormatFloat(gbCount, 'f', 2, 64) + "GB, 合并4GB以上的mp4可能出错， 请自行合并。") + return } var tmpOutputName string - tmpOutputName = filepath.Join(videoDownloadDir, "all.merge.mp4") + tmpOutputName = filepath.Join(tsSaveDir, "all.merge.mp4") this.status.SetProgressBarTitle("[4/4]合并ts为mp4") err = MergeTsFileListToSingleMp4(MergeTsFileListToSingleMp4_Req{ @@ -384,7 +408,7 @@ func (this *DownloadEnv) runDownload(req StartDownload_Req, skipInfo SkipTsInfo) } if req.SkipRemoveTs == false { this.logFileClose() - err = os.RemoveAll(videoDownloadDir) + err = os.RemoveAll(tsSaveDir) if err != nil { this.setErrMsg("删除下载目录失败: " + err.Error()) return diff --git a/cmd/main.go b/cmd/main.go index 98f2990..ff3dc23 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -102,11 +102,17 @@ var mergeCmd = &cobra.Command{ if gMergeReq.OutputMp4Name == "" { gMergeReq.OutputMp4Name = filepath.Join(gMergeReq.InputTsDir, "all.mp4") } + status := &m3u8d.SpeedStatus{ + IsRunning: true, + ProgressBarShow: true, + } + status.SetProgressBarTitle("合并ts") + status.ResetTotalBlockCount(len(tsFileList)) err = m3u8d.MergeTsFileListToSingleMp4(m3u8d.MergeTsFileListToSingleMp4_Req{ TsFileList: tsFileList, OutputMp4: gMergeReq.OutputMp4Name, Ctx: context.Background(), - Status: nil, + Status: status, }) if err != nil { log.Fatalln("合并失败", err) diff --git a/download.go b/download.go index e858ccd..2fcbee4 100644 --- a/download.go +++ b/download.go @@ -18,7 +18,9 @@ import ( "net/url" "os" "path" + "path/filepath" "regexp" + "runtime" "strconv" "strings" "sync" @@ -35,6 +37,7 @@ type TsInfo struct { Seq uint64 // 如果是aes加密并且没有iv, 这个seq需要充当iv Between_EXT_X_DISCONTINUITY bool SkipByHttpCode bool + HttpCode int } type GetStatus_Resp struct { @@ -197,6 +200,7 @@ func (this *DownloadEnv) downloadTsFile(ts *TsInfo, skipInfo SkipTsInfo, downloa if len(skipInfo.HttpCodeList) > 0 && isInIntSlice(httpCode, skipInfo.HttpCodeList) { this.status.SpeedAdd1Block(beginTime, 0) ts.SkipByHttpCode = true + ts.HttpCode = httpCode this.logToFile("skip ts " + strconv.Quote(ts.Name) + " byHttpCode: " + strconv.Itoa(httpCode)) return nil } @@ -562,6 +566,40 @@ func (this *DownloadEnv) logFileClose() { } } +func (this *DownloadEnv) writeFfmpegCmd(downloadingDir string, list []TsInfo) bool { + const listFileName = "filelist.txt" + + var fileListLog bytes.Buffer + for _, one := range list { + if one.SkipByHttpCode { + continue + } + fileListLog.WriteString("file " + one.Name + "\n") + } + err := os.WriteFile(filepath.Join(downloadingDir, listFileName), fileListLog.Bytes(), 0777) + if err != nil { + this.setErrMsg("写入" + listFileName + "失败, " + err.Error()) + return false + } + + var ffmpegCmdContent = "ffmpeg -f concat -i " + listFileName + " -c copy -y output.mp4" + var ffmpegCmdFileName = "merge-by-ffmpeg" + if runtime.GOOS == `windows` { + ffmpegCmdContent += "\r\npause" + ffmpegCmdFileName += ".bat" + } else { + ffmpegCmdFileName += ".sh" + } + + err = os.WriteFile(filepath.Join(downloadingDir, ffmpegCmdFileName), []byte(ffmpegCmdContent), 0777) + if err != nil { + this.setErrMsg("写入" + ffmpegCmdFileName + "失败, " + err.Error()) + return false + } + + return true +} + func IsContextCancel(ctx context.Context) bool { if ctx == nil { return false diff --git a/speed.go b/speed.go index bbb1a30..745f499 100644 --- a/speed.go +++ b/speed.go @@ -22,7 +22,7 @@ type SpeedStatus struct { progressPercent int progressBarTitle string - progressBarShow bool + ProgressBarShow bool lastDrawProgress time.Time errMsg string @@ -50,7 +50,7 @@ func (this *SpeedStatus) clearStatusNoLock() { this.progressPercent = 0 this.progressBarTitle = "" - this.progressBarShow = false + this.ProgressBarShow = false this.errMsg = "" this.saveFileTo = "" @@ -66,7 +66,7 @@ func (this *SpeedStatus) DrawProgressBar(total int, current int) { this.Locker.Lock() this.progressPercent = int(proportion * 100) title := this.progressBarTitle - if this.progressBarShow { + if this.ProgressBarShow { if this.lastDrawProgress.IsZero() || time.Now().Sub(this.lastDrawProgress).Milliseconds() > 100 { width := 50 pos := int(proportion * float32(width))