From 16e656fbed1c5090460fde1ea3e4196247996948 Mon Sep 17 00:00:00 2001 From: orestonce Date: Tue, 9 Aug 2022 08:05:32 +0800 Subject: [PATCH] =?UTF-8?q?=E4=BF=AE=E5=A4=8Dcurl=20-x=E8=AF=86=E5=88=AB?= =?UTF-8?q?=E9=94=99=E8=AF=AF=E9=97=AE=E9=A2=98=20=E4=BD=BF=E7=94=A8=20htt?= =?UTF-8?q?p.ProxyURL=20=E4=BB=A3=E6=9B=BF=E8=87=AA=E5=B7=B1=E5=AE=9E?= =?UTF-8?q?=E7=8E=B0=E7=9A=84=E4=BB=A3=E7=90=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 6 +-- curl.go | 18 ++++++++- download.go | 16 +++++--- go.sum | 2 - proxy.go | 107 ++++++++------------------------------------------ proxy_test.go | 24 +++++++++++ 6 files changed, 69 insertions(+), 104 deletions(-) create mode 100644 proxy_test.go diff --git a/README.md b/README.md index 7658abe..524da6c 100644 --- a/README.md +++ b/README.md @@ -28,11 +28,9 @@ * 支持从curl命令解析出需要的信息，正如 [cxjava/m3u8-downloader](https://github.com/cxjava/m3u8-downloader) 一样 * 显示下载速度、合并ts的速度 * 提供macos的图形化界面 + * 支持设置代理: http/socks5 + * http代理解释: 要访问的真实url是http协议, 使用代理服务器可见的GET形式; 如果要访问的真实url是https协议, 使用代理服务器不可见的CONNECT形式 ## TODO: - * [x] 支持设置代理 - * [x] 支持socks5代理 - * [x] 支持http CONNECT方法 - * [ ] 支持从 [proxy_pool](https://github.com/jhao104/proxy_pool) 得到的普通的http代理 * [ ] 支持多国语言 * [ ] 支持从一个txt里读取下载列表，批量下载 ## 二次开发操作手册: diff --git a/curl.go b/curl.go index 70c1c2f..f397439 100644 --- a/curl.go +++ b/curl.go @@ -73,6 +73,7 @@ func ParseCurl(cmdList []string) (resp ParseCurl_Resp) { isHeader := false isMethod := false + isProxy := false for idx := 0; idx < len(cmdList); idx++ { value := cmdList[idx] @@ -94,16 +95,28 @@ func ParseCurl(cmdList []string) (resp ParseCurl_Resp) { isMethod = false continue } + if isProxy { + resp.DownloadReq.SetProxy = value + isProxy = false + continue + } valueLow := strings.ToLower(value) switch valueLow { case "-h": isHeader = true case "--compressed": case "-x": - isMethod = true + if value == "-X" { + isMethod = true + } else { + isProxy = true + } case "-k", "--insecure": resp.DownloadReq.Insecure = true default: + if strings.HasPrefix(valueLow, "-") { // 不认识的flag, 跳过 + continue + } if resp.DownloadReq.M3u8Url != "" { resp.ErrMsg = "重复的url" return resp @@ -124,6 +137,9 @@ func RunDownload_Req_ToCurlStr(req RunDownload_Req) string { if req.Insecure { buf.WriteString(" \\\n -insecure") } + if req.SetProxy != "" { + buf.WriteString(" \\\n -X " + req.SetProxy) + } for key, vList := range req.HeaderMap { if len(vList) == 0 { continue diff --git a/download.go b/download.go index 81d8c7d..c82f05d 100644 --- a/download.go +++ b/download.go @@ -90,7 +90,7 @@ type RunDownload_Req struct { type downloadEnv struct { cancelFn func() ctx context.Context - client *http.Client + nowClient *http.Client header http.Header sleepTh int32 progressLocker sync.Mutex @@ -275,16 +275,20 @@ var gOldEnv *downloadEnv var gOldEnvLocker sync.Mutex func RunDownload(req RunDownload_Req) (resp RunDownload_Resp) { - req.SetProxy = strings.ToLower(req.SetProxy) + var proxyUrlObj *url.URL + req.SetProxy, proxyUrlObj, resp.ErrMsg = SetProxyFormat(req.SetProxy) + if resp.ErrMsg != "" { + return resp + } env := &downloadEnv{ - client: &http.Client{ + nowClient: &http.Client{ + Timeout: time.Second * 20, Transport: &http.Transport{ TLSClientConfig: &tls.Config{ InsecureSkipVerify: req.Insecure, }, - DialContext: newDialContext(req.SetProxy), + Proxy: http.ProxyURL(proxyUrlObj), }, - Timeout: time.Second * 10, }, speedBytesMap: map[time.Time]int64{}, } @@ -637,7 +641,7 @@ func (this *downloadEnv) doGetRequest(urlS string) (data []byte, err error) { } req = req.WithContext(this.ctx) req.Header = this.header - resp, err := this.client.Do(req) + resp, err := this.nowClient.Do(req) if err != nil { return nil, err } diff --git a/go.sum b/go.sum index ab7cb43..ec5203c 100644 --- a/go.sum +++ b/go.sum @@ -3,8 +3,6 @@ github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NH github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/orestonce/cdb v0.0.0-20220528005855-d187c22240e2 h1:AmGkuxSIOW0gA/jetY8d1DVV0cyQ08FMCxa1arkI6HQ= github.com/orestonce/cdb v0.0.0-20220528005855-d187c22240e2/go.mod h1:HMNNdA1LMQFJRwobtCzVevWcInFSo9rfs1fYeVYwU+c= -github.com/orestonce/go2cpp v0.0.0-20220730064838-feb9dd043919 h1:f8oUxbDjOgXrBhtDSaNWNAnZEPDTwrjpccdsrn4UCUs= -github.com/orestonce/go2cpp v0.0.0-20220730064838-feb9dd043919/go.mod h1:1fsOAZftk08/dOTRqlp6f/MVwaEKOhrnPUg0RtWiSdY= github.com/orestonce/go2cpp v0.0.0-20220802140809-b2a921a62a07 h1:nyt0GDiskod5Y9uNVrXgK7PZHeL7Ab1uVc3LhLg7/gk= github.com/orestonce/go2cpp v0.0.0-20220802140809-b2a921a62a07/go.mod h1:1fsOAZftk08/dOTRqlp6f/MVwaEKOhrnPUg0RtWiSdY= github.com/orestonce/gopool v0.0.0-20220508090328-d7d56d45b171 h1:NnOl6HTrhrlTT7aaAybVOtq+fEztGFMoQtegckgwLdk= diff --git a/proxy.go b/proxy.go index be64e06..90d4988 100644 --- a/proxy.go +++ b/proxy.go @@ -1,101 +1,26 @@ package m3u8d import ( - "bufio" - "bytes" - "context" - "crypto/tls" - "encoding/base64" - "errors" - "fmt" - "golang.org/x/net/proxy" - "net" - "net/http" "net/url" "strings" ) -func newDialContext(setProxy string) func(ctx context.Context, network, addr string) (net.Conn, error) { - return func(ctx context.Context, network, addr string) (net.Conn, error) { - if setProxy == "" { - return (&net.Dialer{}).DialContext(ctx, network, addr) - } else if strings.HasPrefix(setProxy, "http") { - return proxyByHttp(setProxy, ctx, addr) - } else { // socks5 - urlObj, err := url.Parse(setProxy) - if err != nil { - return nil, err - } - dialer, err := proxy.FromURL(urlObj, nil) - if err != nil { - return nil, err - } - return dialer.Dial(network, addr) +func SetProxyFormat(origin string) (after string, urlObj *url.URL, errMsg string) { + after = strings.ToLower(strings.TrimSpace(origin)) + if after == "" { + return after, nil, "" + } + if strings.Contains(after, "://") == false { + after = "http://" + after // 默认http + } + urlObj, err := url.Parse(after) + if err != nil { + return "", nil, "SetProxyFormat1: " + err.Error() + } + for _, vp := range []string{"http", "https", "socks5"} { + if urlObj.Scheme == vp { + return after, urlObj, "" } } - -} - -func proxyByHttp(setProxy string, ctx context.Context, to string) (net.Conn, error) { - // https://github.com/aidenesco/connect/blob/master/proxy.go - urlObj, err := url.Parse(setProxy) - if err != nil { - return nil, err - } - var tConn net.Conn - if strings.HasPrefix(setProxy, "https://") { - host := urlObj.Host - _, _, err = net.SplitHostPort(host) - if err != nil { - host = host + ":443" - } - tConn, err = (&tls.Dialer{}).DialContext(ctx, "tcp", host) - } else { - host := urlObj.Host - _, _, err = net.SplitHostPort(host) - if err != nil { - host = host + ":80" - } - tConn, err = (&net.Dialer{}).DialContext(ctx, "tcp", host) - } - - if err != nil { - return nil, err - } - defer func() { - if err != nil { - tConn.Close() - } - }() - - buf0 := bytes.NewBuffer(nil) - buf0.WriteString(`CONNECT ` + to + ` HTTP/1.1` + "\r\n") - - if u := urlObj.User; u != nil { - username := u.Username() - password, _ := u.Password() - buf0.WriteString("Proxy-Authorization: Basic " + base64.StdEncoding.EncodeToString([]byte(username+":"+password)) + "\r\n") - } - buf0.WriteString("\r\n") - _, err = tConn.Write(buf0.Bytes()) - if err != nil { - return nil, err - } - - bufr := bufio.NewReader(tConn) - - var response *http.Response - response, err = http.ReadResponse(bufr, nil) - if err != nil { - return nil, err - } - - switch response.StatusCode { - case http.StatusOK: - return tConn, nil - case http.StatusProxyAuthRequired: - return nil, errors.New("connect: invalid or missing \"Proxy-Authorization\" header") - default: - return nil, fmt.Errorf("connect: unexpected CONNECT response status \"%s\" (expect 200 OK)", response.Status) - } + return "", nil, "SetProxyFormat2: unknown schema " + urlObj.Scheme } diff --git a/proxy_test.go b/proxy_test.go new file mode 100644 index 0000000..dab0857 --- /dev/null +++ b/proxy_test.go @@ -0,0 +1,24 @@ +package m3u8d + +import ( + "testing" +) + +func TestSetProxyFormat(t *testing.T) { + runOne := func(origin string, expectAfter string) { + after, _, errMsg := SetProxyFormat(origin) + if errMsg != "" { + panic(errMsg) + } + if after != expectAfter { + panic(after) + } + } + runOne("httP://127.0.0.1:1234", "http://127.0.0.1:1234") + runOne("127.0.0.1:1234", "http://127.0.0.1:1234") + runOne("socKs5://127.0.0.1:1080", "socks5://127.0.0.1:1080") + _, _, errMsg := SetProxyFormat("htt://123.com") + if errMsg == "" { + t.Fatal("TestSetProxyFormat") + } +}