root
/
m3u8d
mirror of https://gitee.com/zhang_1334717033/m3u8d.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

修复url路径计算错误问题

main
orestonce 2022-07-21 00:08:58 +08:00
parent ad19058ffd
commit f853c476c7
9 changed files with 138 additions and 74 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

92
download.go
View File

@ -15,11 +15,13 @@ import (
"net/http"
"net/url"
"os"
"path"
"path/filepath"
"regexp"
"strconv"
"strings"
"sync"
"sync/atomic"
"time"
)
@ -36,6 +38,7 @@ var gProgressPercentLocker sync.Mutex
type GetProgress_Resp struct {
Percent int
Title string
SleepTh string
}
func GetProgress() (resp GetProgress_Resp) {
@ -46,6 +49,15 @@ func GetProgress() (resp GetProgress_Resp) {
if resp.Title == "" {
resp.Title = "正在下载"
}
var sleepTh int32
gOldEnvLocker.Lock()
if gOldEnv != nil {
sleepTh = atomic.LoadInt32(&gOldEnv.sleepTh)
}
gOldEnvLocker.Unlock()
if sleepTh > 0 {
resp.SleepTh = "有 " + strconv.Itoa(int(sleepTh)) + "个线程正在休眠."
}
return resp
}
@ -78,6 +90,7 @@ type downloadEnv struct {
ctx context.Context
client *http.Client
header http.Header
sleepTh int32
}
func (this *downloadEnv) RunDownload(req RunDownload_Req) (resp RunDownload_Resp) {
@ -114,7 +127,7 @@ func (this *downloadEnv) RunDownload(req RunDownload_Req) (resp RunDownload_Resp
for key, valueList := range req.HeaderMap {
this.header[key] = valueList
}
SetProgressBarTitle("嗅探m3u8")
SetProgressBarTitle("[1/6]嗅探m3u8")
var m3u8Body []byte
req.M3u8Url, m3u8Body, err = this.sniffM3u8(req.M3u8Url)
if err != nil {
@ -133,7 +146,7 @@ func (this *downloadEnv) RunDownload(req RunDownload_Req) (resp RunDownload_Resp
return resp
}
if info != nil {
SetProgressBarTitle("检查是否已下载")
SetProgressBarTitle("[2/6]检查是否已下载")
latestNameFullPath, found := info.SearchVideoInDir(req.SaveDir)
if found {
resp.IsSkipped = true
@ -166,15 +179,19 @@ func (this *downloadEnv) RunDownload(req RunDownload_Req) (resp RunDownload_Resp
resp.IsCancel = this.GetIsCancel()
return resp
}
SetProgressBarTitle("获取ts列表")
tsList := getTsList(m3u8Host, string(m3u8Body))
SetProgressBarTitle("[3/6]获取ts列表")
tsList, errMsg := getTsList(m3u8Host, string(m3u8Body))
if errMsg != "" {
resp.ErrMsg = "获取ts列表错误: " + strconv.Quote(m3u8Host)
return resp
}
if len(tsList) <= req.SkipTsCountFromHead {
resp.ErrMsg = "需要下载的文件为空"
return resp
}
tsList = tsList[req.SkipTsCountFromHead:]
// 下载ts
SetProgressBarTitle("下载ts")
SetProgressBarTitle("[4/6]下载ts")
err = this.downloader(tsList, downloadDir, ts_key)
if err != nil {
resp.ErrMsg = "下载ts文件错误: " + err.Error()
@ -190,7 +207,7 @@ func (this *downloadEnv) RunDownload(req RunDownload_Req) (resp RunDownload_Resp
var contentHash string
tmpOutputName = filepath.Join(downloadDir, "all.merge.mp4")
SetProgressBarTitle("合并ts为mp4")
SetProgressBarTitle("[5/6]合并ts为mp4")
err = MergeTsFileListToSingleMp4(MergeTsFileListToSingleMp4_Req{
TsFileList: tsFileList,
OutputMp4: tmpOutputName,
@ -200,7 +217,7 @@ func (this *downloadEnv) RunDownload(req RunDownload_Req) (resp RunDownload_Resp
resp.ErrMsg = "合并错误: " + err.Error()
return resp
}
SetProgressBarTitle("计算文件hash")
SetProgressBarTitle("[6/6]计算文件hash")
contentHash = getFileSha256(tmpOutputName)
if contentHash == "" {
resp.ErrMsg = "无法计算摘要信息: " + tmpOutputName
@ -243,7 +260,6 @@ func (this *downloadEnv) RunDownload(req RunDownload_Req) (resp RunDownload_Resp
return resp
}
_ = os.Remove(filepath.Join(req.SaveDir, "downloading"))
SetProgressBarTitle("下载进度")
return resp
}
@ -294,7 +310,7 @@ func getHost(Url, ht string) (host string, err error) {
}
switch ht {
case "apiv1":
return u.Scheme + "://" + u.Host + filepath.Dir(u.EscapedPath()), nil
return u.Scheme + "://" + u.Host + path.Dir(u.EscapedPath()), nil
case "apiv2":
return u.Scheme + "://" + u.Host, nil
default:
@ -329,34 +345,27 @@ func (this *downloadEnv) getM3u8Key(host, html string) (key string, err error) {
return "", nil
}
func getTsList(host, body string) (tsList []TsInfo) {
func getTsList(host, body string) (tsList []TsInfo, errMsg string) {
lines := strings.Split(body, "\n")
index := 0
const TS_NAME_TEMPLATE = "%05d.ts" // ts视频片段命名规则
var ts TsInfo
for _, line := range lines {
if !strings.HasPrefix(line, "#") && line != "" {
//有可能出现的二级嵌套格式的m3u8,请自行转换！
index++
if strings.HasPrefix(line, "http") {
ts = TsInfo{
Name: fmt.Sprintf(TS_NAME_TEMPLATE, index),
Url: line,
}
tsList = append(tsList, ts)
} else {
ts = TsInfo{
Name: fmt.Sprintf(TS_NAME_TEMPLATE, index),
Url: fmt.Sprintf("%s/%s", host, line),
}
ts.Url = strings.ReplaceAll(ts.Url, `\`, `/`)
tsList = append(tsList, ts)
var after string
after, errMsg = resolveRefUrl(host, line)
if errMsg != "" {
return nil, errMsg
}
tsList = append(tsList, TsInfo{
Name: fmt.Sprintf(TS_NAME_TEMPLATE, index),
Url: after,
})
}
}
return
return tsList, ""
}
// 下载ts文件
@ -408,7 +417,7 @@ func (this *downloadEnv) SleepDur(d time.Duration) {
}
func (this *downloadEnv) downloader(tsList []TsInfo, downloadDir string, key string) (err error) {
task := gopool.NewThreadPool(8)
task := gopool.NewThreadPool(1)
tsLen := len(tsList)
downloadCount := 0
var locker sync.Mutex
@ -425,7 +434,9 @@ func (this *downloadEnv) downloader(tsList []TsInfo, downloadDir string, key str
}
locker.Unlock()
if i > 0 {
atomic.AddInt32(&this.sleepTh, 1)
this.SleepDur(time.Second * time.Duration(i))
atomic.AddInt32(&this.sleepTh, -1)
}
lastErr = this.downloadTsFile(ts, downloadDir, key)
if lastErr == nil {
@ -560,15 +571,11 @@ func (this *downloadEnv) sniffM3u8(urlS string) (afterUrl string, content []byte
if m3u8Url == "" {
return "", nil, errors.New("未发现m3u8资源_1")
}
urlObj, err := url.Parse(urlS)
if err != nil {
return "", nil, err
var errMsg string
urlS, errMsg = resolveRefUrl(urlS, m3u8Url)
if errMsg != "" {
return "", nil, errors.New(errMsg)
}
lineObj, err := url.Parse(m3u8Url)
if err != nil {
return "", nil, err
}
urlS = urlObj.ResolveReference(lineObj).String()
continue
}
groups := regexp.MustCompile(`http[s]://[a-zA-Z0-9/\\.%_-]+.m3u8`).FindSubmatch(content)
@ -580,6 +587,18 @@ func (this *downloadEnv) sniffM3u8(urlS string) (afterUrl string, content []byte
return "", nil, errors.New("未发现m3u8资源_3")
}
func resolveRefUrl(baseUrl string, extUrl string) (after string, errMsg string) {
urlObj, err := url.Parse(baseUrl)
if err != nil {
return "", err.Error()
}
lineObj, err := url.Parse(extUrl)
if err != nil {
return "", err.Error()
}
return urlObj.ResolveReference(lineObj).String(), ""
}
func UrlHasSuffix(urlS string, suff string) bool {
urlObj, err := url.Parse(urlS)
if err != nil {
@ -605,6 +624,9 @@ func (this *downloadEnv) doGetRequest(urlS string) (data []byte, err error) {
if err != nil {
return nil, err
}
if resp.StatusCode != 200 {
return content, errors.New("resp.Status: " + resp.Status)
}
return content, nil
}

25
download_test.go
View File

@ -1,6 +1,8 @@
package m3u8d
import "testing"
import (
"testing"
)
func TestUrlHasSuffix(t *testing.T) {
if UrlHasSuffix("/0001.ts", ".ts") == false {
@ -24,3 +26,24 @@ func TestUrlHasSuffix(t *testing.T) {
return
}
}
func TestGetTsList(t *testing.T) {
v, err := getHost(`https://example.com:65/3kb/hls/index.m3u8`, `apiv1`)
if err != nil {
panic(err)
}
if v != `https://example.com:65/3kb/hls` {
panic(v)
}
list, errMsg := getTsList(`https://example.com:65/3kb/hls`, `#EXTINF:3.753,
/3kb/hls/JJG.ts`)
if errMsg != "" {
panic(errMsg)
}
if len(list) != 1 {
panic(len(list))
}
if list[0].Url != "https://example.com:65/3kb/hls/JJG.ts" {
panic(list[0].Url)
}
}

2
export/main.go
View File

@ -13,7 +13,7 @@ import (
"strings"
)
const version = "1.5.1"
const version = "1.5.3"
func main() {
BuildCliBinary() // 编译二进制

2
go.mod
View File

@ -4,7 +4,7 @@ go 1.17
require (
github.com/orestonce/cdb v0.0.0-20220528005855-d187c22240e2
github.com/orestonce/go2cpp v0.0.0-20220625112310-a96256cb266b
github.com/orestonce/go2cpp v0.0.0-20220704224208-2d58769247a4
github.com/orestonce/gopool v0.0.0-20220508090328-d7d56d45b171
github.com/spf13/cobra v1.4.0
github.com/yapingcat/gomedia v0.0.0-20220623101430-02bb90c39484

4
go.sum
View File

@ -3,8 +3,8 @@ github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NH
github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
github.com/orestonce/cdb v0.0.0-20220528005855-d187c22240e2 h1:AmGkuxSIOW0gA/jetY8d1DVV0cyQ08FMCxa1arkI6HQ=
github.com/orestonce/cdb v0.0.0-20220528005855-d187c22240e2/go.mod h1:HMNNdA1LMQFJRwobtCzVevWcInFSo9rfs1fYeVYwU+c=
github.com/orestonce/go2cpp v0.0.0-20220625112310-a96256cb266b h1:fCPoY14Bm8725Ndki1OWKJ0wQrhFOMxeOEOEHpqMcVg=
github.com/orestonce/go2cpp v0.0.0-20220625112310-a96256cb266b/go.mod h1:1fsOAZftk08/dOTRqlp6f/MVwaEKOhrnPUg0RtWiSdY=
github.com/orestonce/go2cpp v0.0.0-20220704224208-2d58769247a4 h1:v6Y0pkcMIJdRgow+X9smChnYkC2v9Zqae/QjB7zzMoo=
github.com/orestonce/go2cpp v0.0.0-20220704224208-2d58769247a4/go.mod h1:1fsOAZftk08/dOTRqlp6f/MVwaEKOhrnPUg0RtWiSdY=
github.com/orestonce/gopool v0.0.0-20220508090328-d7d56d45b171 h1:NnOl6HTrhrlTT7aaAybVOtq+fEztGFMoQtegckgwLdk=
github.com/orestonce/gopool v0.0.0-20220508090328-d7d56d45b171/go.mod h1:MCQUrAPiG9/PTjHJuGqWLasKbIaG6z62KO6kfp90byM=
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=

43
m3u8d-qt/m3u8d.cpp
View File

@ -244,23 +244,34 @@ GetProgress_Resp GetProgress(){
int outIdx = 0;
{
{
uint32_t tmp3 = uint32_t(uint8_t(out[outIdx+0]) << 24);
uint32_t tmp4 = uint32_t(uint8_t(out[outIdx+1]) << 16);
uint32_t tmp5 = uint32_t(uint8_t(out[outIdx+2]) << 8);
uint32_t tmp6 = uint32_t(uint8_t(out[outIdx+3]) << 0);
retValue.Percent = tmp3 | tmp4 | tmp5 | tmp6;
uint32_t tmp4 = uint32_t(uint8_t(out[outIdx+0]) << 24);
uint32_t tmp5 = uint32_t(uint8_t(out[outIdx+1]) << 16);
uint32_t tmp6 = uint32_t(uint8_t(out[outIdx+2]) << 8);
uint32_t tmp7 = uint32_t(uint8_t(out[outIdx+3]) << 0);
retValue.Percent = tmp4 | tmp5 | tmp6 | tmp7;
outIdx+=4;
}
{
uint32_t tmp7 = 0;
uint32_t tmp8 = uint32_t(uint8_t(out[outIdx+0]) << 24);
uint32_t tmp9 = uint32_t(uint8_t(out[outIdx+1]) << 16);
uint32_t tmp10 = uint32_t(uint8_t(out[outIdx+2]) << 8);
uint32_t tmp11 = uint32_t(uint8_t(out[outIdx+3]) << 0);
tmp7 = tmp8 | tmp9 | tmp10 | tmp11;
uint32_t tmp8 = 0;
uint32_t tmp9 = uint32_t(uint8_t(out[outIdx+0]) << 24);
uint32_t tmp10 = uint32_t(uint8_t(out[outIdx+1]) << 16);
uint32_t tmp11 = uint32_t(uint8_t(out[outIdx+2]) << 8);
uint32_t tmp12 = uint32_t(uint8_t(out[outIdx+3]) << 0);
tmp8 = tmp9 | tmp10 | tmp11 | tmp12;
outIdx+=4;
retValue.Title = std::string(out+outIdx, out+outIdx+tmp7);
outIdx+=tmp7;
retValue.Title = std::string(out+outIdx, out+outIdx+tmp8);
outIdx+=tmp8;
}
{
uint32_t tmp13 = 0;
uint32_t tmp14 = uint32_t(uint8_t(out[outIdx+0]) << 24);
uint32_t tmp15 = uint32_t(uint8_t(out[outIdx+1]) << 16);
uint32_t tmp16 = uint32_t(uint8_t(out[outIdx+2]) << 8);
uint32_t tmp17 = uint32_t(uint8_t(out[outIdx+3]) << 0);
tmp13 = tmp14 | tmp15 | tmp16 | tmp17;
outIdx+=4;
retValue.SleepTh = std::string(out+outIdx, out+outIdx+tmp13);
outIdx+=tmp13;
}
}
if (out != NULL) {
@ -641,6 +652,12 @@ void RunOnUiThread::AddRunFnOn_UiThread(std::function<void ()> fn)
emit this->signal_newFn();
}
bool RunOnUiThread::Get_Done()
{
QMutexLocker lk(&this->m_Mutex);
return this->m_done;
}
#include <QTimer>
#include <QLabel>
#include <QWidget>

2
m3u8d-qt/m3u8d.h
View File

@ -30,6 +30,7 @@ void CloseOldEnv();
struct GetProgress_Resp{
int32_t Percent;
std::string Title;
std::string SleepTh;
GetProgress_Resp(): Percent(0){}
};
GetProgress_Resp GetProgress();
@ -59,6 +60,7 @@ public:
// !!!注意,fn可能被调用,也可能由于RunOnUiThread被析构不被调用
// 依赖于在fn里delete回收内存, 关闭文件等操作可能造成内存泄露
void AddRunFnOn_UiThread(std::function<void ()> fn);
bool Get_Done();
signals:
void signal_newFn();
private slots:

36
m3u8d-qt/mainwindow.cpp
View File

@ -17,6 +17,21 @@ MainWindow::MainWindow(QWidget *parent) :
ui->lineEdit_SkipTsCountFromHead->setValidator(vd);
ui->lineEdit_SkipTsCountFromHead->setPlaceholderText("[0,9999]");
ui->lineEdit_SaveDir->setPlaceholderText(QString::fromStdString(GetWd()));
m_syncUi.AddRunFnOn_OtherThread([this](){
while(true)
{
QThread::msleep(50);
if (this->m_syncUi.Get_Done()) {
break;
}
m_syncUi.AddRunFnOn_UiThread([this](){
GetProgress_Resp resp = GetProgress();
ui->progressBar->setValue(resp.Percent);
ui->label_progressBar->setText(QString::fromStdString(resp.Title));
ui->statusBar->showMessage(QString::fromStdString(resp.SleepTh), 5*1000);
});
}
});
}
MainWindow::~MainWindow()
@ -27,6 +42,9 @@ MainWindow::~MainWindow()
void MainWindow::on_pushButton_RunDownload_clicked()
{
if (ui->lineEdit_M3u8Url->isEnabled()==false) {
return;
}
ui->lineEdit_M3u8Url->setEnabled(false);
ui->lineEdit_SaveDir->setEnabled(false);
ui->pushButton_SaveDir->setEnabled(false);
@ -40,24 +58,6 @@ void MainWindow::on_pushButton_RunDownload_clicked()
ui->pushButton_curlMode->setEnabled(false);
ui->pushButton_StopDownload->setEnabled(true);
m_syncUi.AddRunFnOn_OtherThread([this](){
// isFinished被 other thread 和 ui thread共享
std::shared_ptr<std::atomic_bool> isFinished = std::make_shared<std::atomic_bool>(false);
while(isFinished->load() == false)
{
QThread::msleep(50);
// 可能以下闭包在运行前, other thread已经退出了, 所以isFinished需要使用shared_ptr
m_syncUi.AddRunFnOn_UiThread([this, isFinished](){
GetProgress_Resp resp = GetProgress();
ui->progressBar->setValue(resp.Percent);
ui->label_progressBar->setText(QString::fromStdString(resp.Title));
if (ui->pushButton_RunDownload->isEnabled()) {
isFinished->store(true);
}
});
}
});
RunDownload_Req req;
req.M3u8Url = ui->lineEdit_M3u8Url->text().toStdString();
req.HostType = ui->comboBox_HostType->currentText().toStdString();

6
m3u8d-qt/version.rc
View File

@ -7,8 +7,8 @@ IDI_ICON1 ICON "favicon.ico"
#endif
VS_VERSION_INFO VERSIONINFO
FILEVERSION 1,5,1,0
PRODUCTVERSION 1,5,1,0
FILEVERSION 1,5,3,0
PRODUCTVERSION 1,5,3,0
FILEFLAGSMASK 0x3fL
#ifdef _DEBUG
FILEFLAGS VS_FF_DEBUG
@ -23,7 +23,7 @@ VS_VERSION_INFO VERSIONINFO
BEGIN
BLOCK "080404b0"
BEGIN
VALUE "ProductVersion", "1.5.1.0\0"
VALUE "ProductVersion", "1.5.3.0\0"
VALUE "ProductName", "m3u8视频下载工具\0"
VALUE "LegalCopyright", "https://github.com/orestonce/m3u8d\0"
VALUE "FileDescription", "m3u8视频下载工具\0"