Skip to content

Commit

Permalink
🐛 修复了上传视频过大导致内存溢出的问题
Browse files Browse the repository at this point in the history 
⚡️增加了自定义上传线程数量
🎨 优化了精度条显示
  • Loading branch information
@XiaoMiku01
XiaoMiku01 committed Nov 18, 2022
1 parent 8fb36ff commit 64a41c8
Show file tree
Hide file tree
Showing 5 changed files with 101 additions and 43 deletions.
3 changes: 2 additions & 1 deletion README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,8 @@ func main() {
tid := 1 // 分区id
tag := "测试" // 标签 , 分割
source := "测试" // 来源 upType 为 2 时必填
threadNum := 10 // 上传线程数
// 上传视频
upload.NewUp(cookieFile).SetVideos(int64(tid), int64(upType), videoPath, coverPath, title, desc, tag, source).Up()
upload.NewUp(cookieFile,threadNum).SetVideos(int64(tid), int64(upType), videoPath, coverPath, title, desc, tag, source).Up()
}
```
3 changes: 2 additions & 1 deletion cmd/main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,12 +18,13 @@ func main() {
var tid = uploadCmd.Flag("tid", "分区id").Default("47").Int64()
var tag = uploadCmd.Flag("tags", "标签").Default("").String()
var source = uploadCmd.Flag("source", "来源 类型为转载时填写").Default("").String()
var threadNum = uploadCmd.Flag("thread", "上传线程数,默认 10").Default("10").Int()
kingpin.Parse()
switch kingpin.Parse() {
case loginCmd.FullCommand():
login.LoginBili()
case uploadCmd.FullCommand():
upload.NewUp(*cookie).
upload.NewUp(*cookie, *threadNum).
SetVideos(*tid, *upType, *videosPath, *coverPath, *videoTitle, *videoDesc, *tag, *source).
Up()
}
Expand Down
1 change: 1 addition & 0 deletions go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ go 1.19
require (
github.com/Baozisoftware/qrcode-terminal-go v0.0.0-20170407111555-c0650d8dff0f
github.com/imroc/req/v3 v3.24.0
github.com/panjf2000/ants/v2 v2.6.0
github.com/schollz/progressbar/v3 v3.11.0
github.com/tidwall/gjson v1.14.3
gopkg.in/alecthomas/kingpin.v2 v2.2.6
Expand Down
5 changes: 4 additions & 1 deletion go.sum
View file
Original file line number Diff line number Diff line change
Expand Up @@ -132,6 +132,8 @@ github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1y
github.com/onsi/gomega v1.13.0 h1:7lLHu94wT9Ij0o6EWWclhu0aOh32VxhkwEJvzuWPeak=
github.com/onsi/gomega v1.13.0/go.mod h1:lRk9szgn8TxENtWd0Tp4c3wjlRfMTMH27I+3Je41yGY=
github.com/openzipkin/zipkin-go v0.1.1/go.mod h1:NtoC/o8u3JlF1lSlyPNswIbeQH9bJTmOf0Erfk+hxe8=
github.com/panjf2000/ants/v2 v2.6.0 h1:xOSpw42m+BMiJ2I33we7h6fYzG4DAlpE1xyI7VS2gxU=
github.com/panjf2000/ants/v2 v2.6.0/go.mod h1:cU93usDlihJZ5CfRGNDYsiBYvoilLvBF5Qp/BT2GNRE=
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
Expand Down Expand Up @@ -176,8 +178,8 @@ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4=
github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
github.com/stretchr/testify v1.7.1 h1:5TQK59W5E3v0r2duFAb7P95B6hEeOyEnHRa8MjYSMTY=
github.com/tarm/serial v0.0.0-20180830185346-98f6abe2eb07/go.mod h1:kDXzergiv9cbyO7IOYJZWg1U88JhDg3PB6klq9Hg2pA=
github.com/tidwall/gjson v1.14.3 h1:9jvXn7olKEHU1S9vwoMGliaT8jq1vJ7IH/n9zD9Dnlw=
github.com/tidwall/gjson v1.14.3/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
Expand Down Expand Up @@ -342,6 +344,7 @@ gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo=
grpc.go4.org v0.0.0-20170609214715-11d0a25b4919/go.mod h1:77eQGdRu53HpSqPFJFmuJdjuHRquDANNeA4x7B8WQ9o=
honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
Expand Down
132 changes: 92 additions & 40 deletions upload/upload.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import (
"encoding/json"
"fmt"
"github.com/imroc/req/v3"
"github.com/panjf2000/ants/v2"
"github.com/schollz/progressbar/v3"
"github.com/tidwall/gjson"
"io"
Expand Down Expand Up @@ -36,6 +37,10 @@ type Up struct {
client *req.Client

upVideo *UpVideo

threadNum int
partChan chan Part
chunks int64
}

type UpVideo struct {
Expand All @@ -50,7 +55,9 @@ type UpVideo struct {
bizId int64
}

func NewUp(cookiePath string) *Up {
var wg sync.WaitGroup

func NewUp(cookiePath string, threadNum int) *Up {
var cookieinfo CookieInfo
loginInfo, err := os.ReadFile(cookiePath)
if err != nil || len(loginInfo) == 0 {
Expand Down Expand Up @@ -82,6 +89,7 @@ func NewUp(cookiePath string) *Up {
csrf: csrf,
client: client,
upVideo: &UpVideo{},
threadNum: threadNum,
}
}

Expand Down Expand Up @@ -192,14 +200,15 @@ func (u *Up) Up() {
LosslessMusic: 0,
Csrf: u.csrf,
}
//_ = addreq
_ = addreq
resp, _ := u.client.R().SetQueryParams(map[string]string{
"csrf": u.csrf,
}).SetBodyJsonMarshal(addreq).Post("https://member.bilibili.com/x/vu/web/add/v3")
log.Println(resp.String())
}

func (u *Up) upload() {
defer ants.Release()
var upinfo UpInfo
u.client.SetCommonHeader(
"X-Upos-Auth", u.upVideo.auth).R().
Expand All @@ -213,63 +222,40 @@ func (u *Up) upload() {
"meta_upos_uri": u.getMetaUposUri(),
}).SetResult(&upinfo).Post(u.upVideo.uploadBaseUrl)
u.upVideo.uploadId = upinfo.UploadId
chunks := int64(math.Ceil(float64(u.upVideo.videoSize) / float64(u.upVideo.chunkSize)))
u.chunks = int64(math.Ceil(float64(u.upVideo.videoSize) / float64(u.upVideo.chunkSize)))
var reqjson = new(ReqJson)
file, _ := os.Open(u.videosPath)
defer file.Close()
chunk := 0
start := 0
end := 0
bar := progressbar.Default(u.upVideo.videoSize/1024/1024, "视频上传中...")
var wg sync.WaitGroup
var partchan = make(chan Part, chunks)
bar := progressbar.NewOptions(int(u.upVideo.videoSize/1024/1024),
progressbar.OptionSetWriter(os.Stdout),
progressbar.OptionSetItsString("MB"),
progressbar.OptionSetDescription("视频上传中..."),
progressbar.OptionSetWidth(50),
progressbar.OptionShowIts(),
)
u.partChan = make(chan Part, u.chunks)
go func() {
for p := range partchan {
for p := range u.partChan {
reqjson.Parts = append(reqjson.Parts, p)
}
}()
//for i := 0; int64(i) < chunks; i++ {
p, _ := ants.NewPool(u.threadNum)
defer p.Release()
for {
buf := make([]byte, u.upVideo.chunkSize)
size, err := file.Read(buf)
if err != nil && err != io.EOF {
break
}
buf = buf[:size]
//size := n
if size > 0 {
wg.Add(1)
end += size
go func(chunk int, start, end, size int, buf []byte, bar *progressbar.ProgressBar) {
defer wg.Done()
resp, _ := u.client.R().SetHeaders(map[string]string{
"Content-Type": "application/octet-stream",
"Content-Length": strconv.Itoa(size),
}).SetQueryParams(map[string]string{
"partNumber": strconv.Itoa(chunk + 1),
"uploadId": u.upVideo.uploadId,
"chunk": strconv.Itoa(chunk),
"chunks": strconv.Itoa(int(chunks)),
"size": strconv.Itoa(size),
"start": strconv.Itoa(start),
"end": strconv.Itoa(end),
"total": strconv.FormatInt(u.upVideo.videoSize, 10),
}).SetBodyBytes(buf).SetRetryCount(5).AddRetryHook(func(resp *req.Response, err error) {
log.Println("重试发送分片", chunk)
return
}).
AddRetryCondition(func(resp *req.Response, err error) bool {
return err != nil || resp.StatusCode != 200
}).Put(u.upVideo.uploadBaseUrl)
bar.Add(len(buf) / 1024 / 1024)
if resp.StatusCode != 200 {
log.Println("分片", chunk, "上传失败", resp.StatusCode, "size", size)
}
partchan <- Part{
PartNumber: int64(chunk + 1),
ETag: "etag",
}
}(chunk, start, end, size, buf, bar)
_ = p.Submit(u.uploadPartWrapper(chunk, start, end, size, buf, bar))
buf = nil
start += size
chunk++
}
Expand All @@ -278,7 +264,7 @@ func (u *Up) upload() {
}
}
wg.Wait()
close(partchan)
close(u.partChan)
jsonString, _ := json.Marshal(&reqjson)
u.client.R().SetHeaders(map[string]string{
"Content-Type": "application/json",
Expand All @@ -299,6 +285,72 @@ func (u *Up) upload() {
}).Post(u.upVideo.uploadBaseUrl)
}

func (u *Up) uploadPart(chunk int, start, end, size int, buf []byte, bar *progressbar.ProgressBar) {
defer wg.Done()
resp, _ := u.client.R().SetHeaders(map[string]string{
"Content-Type": "application/octet-stream",
"Content-Length": strconv.Itoa(size),
}).SetQueryParams(map[string]string{
"partNumber": strconv.Itoa(chunk + 1),
"uploadId": u.upVideo.uploadId,
"chunk": strconv.Itoa(chunk),
"chunks": strconv.Itoa(int(u.chunks)),
"size": strconv.Itoa(size),
"start": strconv.Itoa(start),
"end": strconv.Itoa(end),
"total": strconv.FormatInt(u.upVideo.videoSize, 10),
}).SetBodyBytes(buf).SetRetryCount(5).AddRetryHook(func(resp *req.Response, err error) {
log.Println("重试发送分片", chunk)
return
}).
AddRetryCondition(func(resp *req.Response, err error) bool {
return err != nil || resp.StatusCode != 200
}).Put(u.upVideo.uploadBaseUrl)
bar.Add(len(buf) / 1024 / 1024)
if resp.StatusCode != 200 {
log.Println("分片", chunk, "上传失败", resp.StatusCode, "size", size)
}
u.partChan <- Part{
PartNumber: int64(chunk + 1),
ETag: "etag",
}
}

type taskFunc func()

func (u *Up) uploadPartWrapper(chunk int, start, end, size int, buf []byte, bar *progressbar.ProgressBar) taskFunc {
return func() {
defer wg.Done()
resp, _ := u.client.R().SetHeaders(map[string]string{
"Content-Type": "application/octet-stream",
"Content-Length": strconv.Itoa(size),
}).SetQueryParams(map[string]string{
"partNumber": strconv.Itoa(chunk + 1),
"uploadId": u.upVideo.uploadId,
"chunk": strconv.Itoa(chunk),
"chunks": strconv.Itoa(int(u.chunks)),
"size": strconv.Itoa(size),
"start": strconv.Itoa(start),
"end": strconv.Itoa(end),
"total": strconv.FormatInt(u.upVideo.videoSize, 10),
}).SetBodyBytes(buf).SetRetryCount(5).AddRetryHook(func(resp *req.Response, err error) {
log.Println("重试发送分片", chunk)
return
}).
AddRetryCondition(func(resp *req.Response, err error) bool {
return err != nil || resp.StatusCode != 200
}).Put(u.upVideo.uploadBaseUrl)
bar.Add(len(buf) / 1024 / 1024)
if resp.StatusCode != 200 {
log.Println("分片", chunk, "上传失败", resp.StatusCode, "size", size)
}
u.partChan <- Part{
PartNumber: int64(chunk + 1),
ETag: "etag",
}
}
}

func (u *Up) getMetaUposUri() string {
var metaUposUri PreUpInfo
u.client.R().SetQueryParams(map[string]string{
Expand Down

0 comments on commit 64a41c8

Please sign in to comment.