From f853c476c7b1da1a3dcd5232cd333c779221d393 Mon Sep 17 00:00:00 2001 From: orestonce Date: Thu, 21 Jul 2022 00:08:58 +0800 Subject: [PATCH] =?UTF-8?q?=E4=BF=AE=E5=A4=8Durl=E8=B7=AF=E5=BE=84?= =?UTF-8?q?=E8=AE=A1=E7=AE=97=E9=94=99=E8=AF=AF=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- download.go | 92 +++++++++++++++++++++++++---------------- download_test.go | 25 ++++++++++- export/main.go | 2 +- go.mod | 2 +- go.sum | 4 +- m3u8d-qt/m3u8d.cpp | 43 +++++++++++++------ m3u8d-qt/m3u8d.h | 2 + m3u8d-qt/mainwindow.cpp | 36 ++++++++-------- m3u8d-qt/version.rc | 6 +-- 9 files changed, 138 insertions(+), 74 deletions(-) diff --git a/download.go b/download.go index f4542b3..601eb53 100644 --- a/download.go +++ b/download.go @@ -15,11 +15,13 @@ import ( "net/http" "net/url" "os" + "path" "path/filepath" "regexp" "strconv" "strings" "sync" + "sync/atomic" "time" ) @@ -36,6 +38,7 @@ var gProgressPercentLocker sync.Mutex type GetProgress_Resp struct { Percent int Title string + SleepTh string } func GetProgress() (resp GetProgress_Resp) { @@ -46,6 +49,15 @@ func GetProgress() (resp GetProgress_Resp) { if resp.Title == "" { resp.Title = "正在下载" } + var sleepTh int32 + gOldEnvLocker.Lock() + if gOldEnv != nil { + sleepTh = atomic.LoadInt32(&gOldEnv.sleepTh) + } + gOldEnvLocker.Unlock() + if sleepTh > 0 { + resp.SleepTh = "有 " + strconv.Itoa(int(sleepTh)) + "个线程正在休眠." + } return resp } @@ -78,6 +90,7 @@ type downloadEnv struct { ctx context.Context client *http.Client header http.Header + sleepTh int32 } func (this *downloadEnv) RunDownload(req RunDownload_Req) (resp RunDownload_Resp) { @@ -114,7 +127,7 @@ func (this *downloadEnv) RunDownload(req RunDownload_Req) (resp RunDownload_Resp for key, valueList := range req.HeaderMap { this.header[key] = valueList } - SetProgressBarTitle("嗅探m3u8") + SetProgressBarTitle("[1/6]嗅探m3u8") var m3u8Body []byte req.M3u8Url, m3u8Body, err = this.sniffM3u8(req.M3u8Url) if err != nil { @@ -133,7 +146,7 @@ func (this *downloadEnv) RunDownload(req RunDownload_Req) (resp RunDownload_Resp return resp } if info != nil { - SetProgressBarTitle("检查是否已下载") + SetProgressBarTitle("[2/6]检查是否已下载") latestNameFullPath, found := info.SearchVideoInDir(req.SaveDir) if found { resp.IsSkipped = true @@ -166,15 +179,19 @@ func (this *downloadEnv) RunDownload(req RunDownload_Req) (resp RunDownload_Resp resp.IsCancel = this.GetIsCancel() return resp } - SetProgressBarTitle("获取ts列表") - tsList := getTsList(m3u8Host, string(m3u8Body)) + SetProgressBarTitle("[3/6]获取ts列表") + tsList, errMsg := getTsList(m3u8Host, string(m3u8Body)) + if errMsg != "" { + resp.ErrMsg = "获取ts列表错误: " + strconv.Quote(m3u8Host) + return resp + } if len(tsList) <= req.SkipTsCountFromHead { resp.ErrMsg = "需要下载的文件为空" return resp } tsList = tsList[req.SkipTsCountFromHead:] // 下载ts - SetProgressBarTitle("下载ts") + SetProgressBarTitle("[4/6]下载ts") err = this.downloader(tsList, downloadDir, ts_key) if err != nil { resp.ErrMsg = "下载ts文件错误: " + err.Error() @@ -190,7 +207,7 @@ func (this *downloadEnv) RunDownload(req RunDownload_Req) (resp RunDownload_Resp var contentHash string tmpOutputName = filepath.Join(downloadDir, "all.merge.mp4") - SetProgressBarTitle("合并ts为mp4") + SetProgressBarTitle("[5/6]合并ts为mp4") err = MergeTsFileListToSingleMp4(MergeTsFileListToSingleMp4_Req{ TsFileList: tsFileList, OutputMp4: tmpOutputName, @@ -200,7 +217,7 @@ func (this *downloadEnv) RunDownload(req RunDownload_Req) (resp RunDownload_Resp resp.ErrMsg = "合并错误: " + err.Error() return resp } - SetProgressBarTitle("计算文件hash") + SetProgressBarTitle("[6/6]计算文件hash") contentHash = getFileSha256(tmpOutputName) if contentHash == "" { resp.ErrMsg = "无法计算摘要信息: " + tmpOutputName @@ -243,7 +260,6 @@ func (this *downloadEnv) RunDownload(req RunDownload_Req) (resp RunDownload_Resp return resp } _ = os.Remove(filepath.Join(req.SaveDir, "downloading")) - SetProgressBarTitle("下载进度") return resp } @@ -294,7 +310,7 @@ func getHost(Url, ht string) (host string, err error) { } switch ht { case "apiv1": - return u.Scheme + "://" + u.Host + filepath.Dir(u.EscapedPath()), nil + return u.Scheme + "://" + u.Host + path.Dir(u.EscapedPath()), nil case "apiv2": return u.Scheme + "://" + u.Host, nil default: @@ -329,34 +345,27 @@ func (this *downloadEnv) getM3u8Key(host, html string) (key string, err error) { return "", nil } -func getTsList(host, body string) (tsList []TsInfo) { +func getTsList(host, body string) (tsList []TsInfo, errMsg string) { lines := strings.Split(body, "\n") index := 0 const TS_NAME_TEMPLATE = "%05d.ts" // ts视频片段命名规则 - var ts TsInfo for _, line := range lines { if !strings.HasPrefix(line, "#") && line != "" { - //有可能出现的二级嵌套格式的m3u8,请自行转换！ index++ - if strings.HasPrefix(line, "http") { - ts = TsInfo{ - Name: fmt.Sprintf(TS_NAME_TEMPLATE, index), - Url: line, - } - tsList = append(tsList, ts) - } else { - ts = TsInfo{ - Name: fmt.Sprintf(TS_NAME_TEMPLATE, index), - Url: fmt.Sprintf("%s/%s", host, line), - } - ts.Url = strings.ReplaceAll(ts.Url, `\`, `/`) - tsList = append(tsList, ts) + var after string + after, errMsg = resolveRefUrl(host, line) + if errMsg != "" { + return nil, errMsg } + tsList = append(tsList, TsInfo{ + Name: fmt.Sprintf(TS_NAME_TEMPLATE, index), + Url: after, + }) } } - return + return tsList, "" } // 下载ts文件 @@ -408,7 +417,7 @@ func (this *downloadEnv) SleepDur(d time.Duration) { } func (this *downloadEnv) downloader(tsList []TsInfo, downloadDir string, key string) (err error) { - task := gopool.NewThreadPool(8) + task := gopool.NewThreadPool(1) tsLen := len(tsList) downloadCount := 0 var locker sync.Mutex @@ -425,7 +434,9 @@ func (this *downloadEnv) downloader(tsList []TsInfo, downloadDir string, key str } locker.Unlock() if i > 0 { + atomic.AddInt32(&this.sleepTh, 1) this.SleepDur(time.Second * time.Duration(i)) + atomic.AddInt32(&this.sleepTh, -1) } lastErr = this.downloadTsFile(ts, downloadDir, key) if lastErr == nil { @@ -560,15 +571,11 @@ func (this *downloadEnv) sniffM3u8(urlS string) (afterUrl string, content []byte if m3u8Url == "" { return "", nil, errors.New("未发现m3u8资源_1") } - urlObj, err := url.Parse(urlS) - if err != nil { - return "", nil, err + var errMsg string + urlS, errMsg = resolveRefUrl(urlS, m3u8Url) + if errMsg != "" { + return "", nil, errors.New(errMsg) } - lineObj, err := url.Parse(m3u8Url) - if err != nil { - return "", nil, err - } - urlS = urlObj.ResolveReference(lineObj).String() continue } groups := regexp.MustCompile(`http[s]://[a-zA-Z0-9/\\.%_-]+.m3u8`).FindSubmatch(content) @@ -580,6 +587,18 @@ func (this *downloadEnv) sniffM3u8(urlS string) (afterUrl string, content []byte return "", nil, errors.New("未发现m3u8资源_3") } +func resolveRefUrl(baseUrl string, extUrl string) (after string, errMsg string) { + urlObj, err := url.Parse(baseUrl) + if err != nil { + return "", err.Error() + } + lineObj, err := url.Parse(extUrl) + if err != nil { + return "", err.Error() + } + return urlObj.ResolveReference(lineObj).String(), "" +} + func UrlHasSuffix(urlS string, suff string) bool { urlObj, err := url.Parse(urlS) if err != nil { @@ -605,6 +624,9 @@ func (this *downloadEnv) doGetRequest(urlS string) (data []byte, err error) { if err != nil { return nil, err } + if resp.StatusCode != 200 { + return content, errors.New("resp.Status: " + resp.Status) + } return content, nil } diff --git a/download_test.go b/download_test.go index 7a8d721..0496475 100644 --- a/download_test.go +++ b/download_test.go @@ -1,6 +1,8 @@ package m3u8d -import "testing" +import ( + "testing" +) func TestUrlHasSuffix(t *testing.T) { if UrlHasSuffix("/0001.ts", ".ts") == false { @@ -24,3 +26,24 @@ func TestUrlHasSuffix(t *testing.T) { return } } + +func TestGetTsList(t *testing.T) { + v, err := getHost(`https://example.com:65/3kb/hls/index.m3u8`, `apiv1`) + if err != nil { + panic(err) + } + if v != `https://example.com:65/3kb/hls` { + panic(v) + } + list, errMsg := getTsList(`https://example.com:65/3kb/hls`, `#EXTINF:3.753, +/3kb/hls/JJG.ts`) + if errMsg != "" { + panic(errMsg) + } + if len(list) != 1 { + panic(len(list)) + } + if list[0].Url != "https://example.com:65/3kb/hls/JJG.ts" { + panic(list[0].Url) + } +} diff --git a/export/main.go b/export/main.go index b3a15eb..fc2696c 100644 --- a/export/main.go +++ b/export/main.go @@ -13,7 +13,7 @@ import ( "strings" ) -const version = "1.5.1" +const version = "1.5.3" func main() { BuildCliBinary() // 编译二进制 diff --git a/go.mod b/go.mod index 293e0a6..b85c46e 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.17 require ( github.com/orestonce/cdb v0.0.0-20220528005855-d187c22240e2 - github.com/orestonce/go2cpp v0.0.0-20220625112310-a96256cb266b + github.com/orestonce/go2cpp v0.0.0-20220704224208-2d58769247a4 github.com/orestonce/gopool v0.0.0-20220508090328-d7d56d45b171 github.com/spf13/cobra v1.4.0 github.com/yapingcat/gomedia v0.0.0-20220623101430-02bb90c39484 diff --git a/go.sum b/go.sum index a8d8cf0..2eb7ab9 100644 --- a/go.sum +++ b/go.sum @@ -3,8 +3,8 @@ github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NH github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/orestonce/cdb v0.0.0-20220528005855-d187c22240e2 h1:AmGkuxSIOW0gA/jetY8d1DVV0cyQ08FMCxa1arkI6HQ= github.com/orestonce/cdb v0.0.0-20220528005855-d187c22240e2/go.mod h1:HMNNdA1LMQFJRwobtCzVevWcInFSo9rfs1fYeVYwU+c= -github.com/orestonce/go2cpp v0.0.0-20220625112310-a96256cb266b h1:fCPoY14Bm8725Ndki1OWKJ0wQrhFOMxeOEOEHpqMcVg= -github.com/orestonce/go2cpp v0.0.0-20220625112310-a96256cb266b/go.mod h1:1fsOAZftk08/dOTRqlp6f/MVwaEKOhrnPUg0RtWiSdY= +github.com/orestonce/go2cpp v0.0.0-20220704224208-2d58769247a4 h1:v6Y0pkcMIJdRgow+X9smChnYkC2v9Zqae/QjB7zzMoo= +github.com/orestonce/go2cpp v0.0.0-20220704224208-2d58769247a4/go.mod h1:1fsOAZftk08/dOTRqlp6f/MVwaEKOhrnPUg0RtWiSdY= github.com/orestonce/gopool v0.0.0-20220508090328-d7d56d45b171 h1:NnOl6HTrhrlTT7aaAybVOtq+fEztGFMoQtegckgwLdk= github.com/orestonce/gopool v0.0.0-20220508090328-d7d56d45b171/go.mod h1:MCQUrAPiG9/PTjHJuGqWLasKbIaG6z62KO6kfp90byM= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= diff --git a/m3u8d-qt/m3u8d.cpp b/m3u8d-qt/m3u8d.cpp index 4c71eec..a1787bb 100644 --- a/m3u8d-qt/m3u8d.cpp +++ b/m3u8d-qt/m3u8d.cpp @@ -244,23 +244,34 @@ GetProgress_Resp GetProgress(){ int outIdx = 0; { { - uint32_t tmp3 = uint32_t(uint8_t(out[outIdx+0]) << 24); - uint32_t tmp4 = uint32_t(uint8_t(out[outIdx+1]) << 16); - uint32_t tmp5 = uint32_t(uint8_t(out[outIdx+2]) << 8); - uint32_t tmp6 = uint32_t(uint8_t(out[outIdx+3]) << 0); - retValue.Percent = tmp3 | tmp4 | tmp5 | tmp6; + uint32_t tmp4 = uint32_t(uint8_t(out[outIdx+0]) << 24); + uint32_t tmp5 = uint32_t(uint8_t(out[outIdx+1]) << 16); + uint32_t tmp6 = uint32_t(uint8_t(out[outIdx+2]) << 8); + uint32_t tmp7 = uint32_t(uint8_t(out[outIdx+3]) << 0); + retValue.Percent = tmp4 | tmp5 | tmp6 | tmp7; outIdx+=4; } { - uint32_t tmp7 = 0; - uint32_t tmp8 = uint32_t(uint8_t(out[outIdx+0]) << 24); - uint32_t tmp9 = uint32_t(uint8_t(out[outIdx+1]) << 16); - uint32_t tmp10 = uint32_t(uint8_t(out[outIdx+2]) << 8); - uint32_t tmp11 = uint32_t(uint8_t(out[outIdx+3]) << 0); - tmp7 = tmp8 | tmp9 | tmp10 | tmp11; + uint32_t tmp8 = 0; + uint32_t tmp9 = uint32_t(uint8_t(out[outIdx+0]) << 24); + uint32_t tmp10 = uint32_t(uint8_t(out[outIdx+1]) << 16); + uint32_t tmp11 = uint32_t(uint8_t(out[outIdx+2]) << 8); + uint32_t tmp12 = uint32_t(uint8_t(out[outIdx+3]) << 0); + tmp8 = tmp9 | tmp10 | tmp11 | tmp12; outIdx+=4; - retValue.Title = std::string(out+outIdx, out+outIdx+tmp7); - outIdx+=tmp7; + retValue.Title = std::string(out+outIdx, out+outIdx+tmp8); + outIdx+=tmp8; + } + { + uint32_t tmp13 = 0; + uint32_t tmp14 = uint32_t(uint8_t(out[outIdx+0]) << 24); + uint32_t tmp15 = uint32_t(uint8_t(out[outIdx+1]) << 16); + uint32_t tmp16 = uint32_t(uint8_t(out[outIdx+2]) << 8); + uint32_t tmp17 = uint32_t(uint8_t(out[outIdx+3]) << 0); + tmp13 = tmp14 | tmp15 | tmp16 | tmp17; + outIdx+=4; + retValue.SleepTh = std::string(out+outIdx, out+outIdx+tmp13); + outIdx+=tmp13; } } if (out != NULL) { @@ -641,6 +652,12 @@ void RunOnUiThread::AddRunFnOn_UiThread(std::function fn) emit this->signal_newFn(); } +bool RunOnUiThread::Get_Done() +{ + QMutexLocker lk(&this->m_Mutex); + return this->m_done; +} + #include #include #include diff --git a/m3u8d-qt/m3u8d.h b/m3u8d-qt/m3u8d.h index b15d1ab..b575e83 100644 --- a/m3u8d-qt/m3u8d.h +++ b/m3u8d-qt/m3u8d.h @@ -30,6 +30,7 @@ void CloseOldEnv(); struct GetProgress_Resp{ int32_t Percent; std::string Title; + std::string SleepTh; GetProgress_Resp(): Percent(0){} }; GetProgress_Resp GetProgress(); @@ -59,6 +60,7 @@ public: // !!!注意,fn可能被调用,也可能由于RunOnUiThread被析构不被调用 // 依赖于在fn里delete回收内存, 关闭文件等操作可能造成内存泄露 void AddRunFnOn_UiThread(std::function fn); + bool Get_Done(); signals: void signal_newFn(); private slots: diff --git a/m3u8d-qt/mainwindow.cpp b/m3u8d-qt/mainwindow.cpp index 58675c4..9401c3f 100644 --- a/m3u8d-qt/mainwindow.cpp +++ b/m3u8d-qt/mainwindow.cpp @@ -17,6 +17,21 @@ MainWindow::MainWindow(QWidget *parent) : ui->lineEdit_SkipTsCountFromHead->setValidator(vd); ui->lineEdit_SkipTsCountFromHead->setPlaceholderText("[0,9999]"); ui->lineEdit_SaveDir->setPlaceholderText(QString::fromStdString(GetWd())); + m_syncUi.AddRunFnOn_OtherThread([this](){ + while(true) + { + QThread::msleep(50); + if (this->m_syncUi.Get_Done()) { + break; + } + m_syncUi.AddRunFnOn_UiThread([this](){ + GetProgress_Resp resp = GetProgress(); + ui->progressBar->setValue(resp.Percent); + ui->label_progressBar->setText(QString::fromStdString(resp.Title)); + ui->statusBar->showMessage(QString::fromStdString(resp.SleepTh), 5*1000); + }); + } + }); } MainWindow::~MainWindow() @@ -27,6 +42,9 @@ MainWindow::~MainWindow() void MainWindow::on_pushButton_RunDownload_clicked() { + if (ui->lineEdit_M3u8Url->isEnabled()==false) { + return; + } ui->lineEdit_M3u8Url->setEnabled(false); ui->lineEdit_SaveDir->setEnabled(false); ui->pushButton_SaveDir->setEnabled(false); @@ -40,24 +58,6 @@ void MainWindow::on_pushButton_RunDownload_clicked() ui->pushButton_curlMode->setEnabled(false); ui->pushButton_StopDownload->setEnabled(true); - m_syncUi.AddRunFnOn_OtherThread([this](){ - // isFinished被 other thread 和 ui thread共享 - std::shared_ptr isFinished = std::make_shared(false); - - while(isFinished->load() == false) - { - QThread::msleep(50); - // 可能以下闭包在运行前, other thread已经退出了, 所以isFinished需要使用shared_ptr - m_syncUi.AddRunFnOn_UiThread([this, isFinished](){ - GetProgress_Resp resp = GetProgress(); - ui->progressBar->setValue(resp.Percent); - ui->label_progressBar->setText(QString::fromStdString(resp.Title)); - if (ui->pushButton_RunDownload->isEnabled()) { - isFinished->store(true); - } - }); - } - }); RunDownload_Req req; req.M3u8Url = ui->lineEdit_M3u8Url->text().toStdString(); req.HostType = ui->comboBox_HostType->currentText().toStdString(); diff --git a/m3u8d-qt/version.rc b/m3u8d-qt/version.rc index 92de15a..bd356ab 100644 --- a/m3u8d-qt/version.rc +++ b/m3u8d-qt/version.rc @@ -7,8 +7,8 @@ IDI_ICON1 ICON "favicon.ico" #endif VS_VERSION_INFO VERSIONINFO - FILEVERSION 1,5,1,0 - PRODUCTVERSION 1,5,1,0 + FILEVERSION 1,5,3,0 + PRODUCTVERSION 1,5,3,0 FILEFLAGSMASK 0x3fL #ifdef _DEBUG FILEFLAGS VS_FF_DEBUG @@ -23,7 +23,7 @@ VS_VERSION_INFO VERSIONINFO BEGIN BLOCK "080404b0" BEGIN - VALUE "ProductVersion", "1.5.1.0\0" + VALUE "ProductVersion", "1.5.3.0\0" VALUE "ProductName", "m3u8Ƶع\0" VALUE "LegalCopyright", "https://github.com/orestonce/m3u8d\0" VALUE "FileDescription", "m3u8Ƶع\0"