并发版断点续传
并发版就不能直接读取文件的大小了,需要一个文件来去存储下载的进度,在一开下载的时候读取内容,再根据内容去下载
完整代码,可能有一些瑕疵
package main
import (
"bufio"
"errors"
"fmt"
"io"
"math"
"net/http"
"os"
"path"
"strconv"
"strings"
"sync"
)
type fileContent struct {
taskId int
con []byte
start int
isEnd bool
}
var dwg sync.WaitGroup
//并发下载数
var maxGo = 2
var downInfo info
type info struct {
url string
maxGo int
schedule [][3]int
}
//读取配置
func (info *info) load(file *os.File) {
rd := bufio.NewReader(file)
//第一行
line, _ := rd.ReadString('\n')
line = strings.TrimSpace(line)
info.url = line
//第二行
line, _ = rd.ReadString('\n')
line = strings.TrimSpace(line)
maxGoArr := strings.Split(line, ":")
maxGo , _ := strconv.Atoi(maxGoArr[1])
info.maxGo = maxGo
//第三行
line, _ = rd.ReadString('\n')
line = strings.TrimSpace(line)
var tmpSchedule [][3]int
scheduleArr1 := strings.Split(line, ":")
scheduleArr1 = strings.Split(line, ",")
for _, scheduleArr := range scheduleArr1 {
scheduleOne := strings.Split(scheduleArr, "-")
start, _ := strconv.Atoi(scheduleOne[0])
len, _ := strconv.Atoi(scheduleOne[1])
down, _ := strconv.Atoi(scheduleOne[2])
tmpSchedule = append(tmpSchedule, [3]int{start, len, down})
}
info.schedule = tmpSchedule
}
//写入配置
func (info info) write(file *os.File) {
file.Seek(0, 0)
scheduleString := ""
for _, v := range info.schedule {
if len(scheduleString) != 0 {
scheduleString += ","
}
scheduleString += fmt.Sprintf("%d-%d-%d", v[0], v[1], v[2])
}
writeString := fmt.Sprintf("url:%s\nmaxGo:%d\nschedule:%s", info.url, info.maxGo, scheduleString)
file.Write([]byte(writeString))
}
func (info info) getDownloadSize() int {
downloadSize := 0
for _, v := range info.schedule {
downloadSize += v[2]
}
return downloadSize
}
func main() {
url := "http://ybook.iapi.im/header-leg.jpg"
request, err := http.NewRequest("HEAD", url, nil)
if err != nil {
panic(err)
}
client := http.Client{}
response, err := client.Do(request)
if err != nil {
panic(err)
}
fmt.Println(response.Header)
//map[Accept-Ranges:[bytes] Cache-Control:[max-age=2592000] Connection:[keep-alive] Content-Length:[572356] Content-Type:[image/jpeg] Date:[Tue, 23 Feb 2021 12:00:57 GMT] Etag:["6032775a-8bbc4"] Ex
//pires:[Thu, 25 Mar 2021 12:00:57 GMT] Last-Modified:[Sun, 21 Feb 2021 15:08:10 GMT] Server:[nginx]]
if response.Header.Get("Accept-Ranges") != "bytes" {
panic(errors.New("当前文件不支持下载"))
}
fileName := path.Base(url)
fileSize, _ := strconv.Atoi(response.Header.Get("Content-Length"))
fmt.Printf("文件名称:%s, 文件大小:%d \n", fileName, fileSize)
_, existErr := os.Stat("./" + fileName)
downloadFile, err := os.OpenFile("./"+fileName, os.O_CREATE|os.O_WRONLY, 0666)
if err != nil {
panic(err)
}
downloadConfigFile, err := os.OpenFile("./"+fileName+".jc", os.O_CREATE|os.O_RDWR, 0666)
if err != nil {
panic(err)
}
downloadSize := 0
if os.IsNotExist(existErr) {
downloadFile.Seek(int64(fileSize-1), 0)
downloadFile.Write([]byte{0})
downInfo.url = url;
downInfo.maxGo = maxGo
//每段平均下载数据量,最后一段会有偏差
avg := int(math.Ceil(float64(fileSize / maxGo)))
for i := 0; i < maxGo; i++ {
downloadStart := avg * i
downloadLen := avg
if fileSize < (downloadStart + avg) {
downloadLen = fileSize - downloadStart
}
downInfo.schedule = append(downInfo.schedule, [3]int{downloadStart, downloadLen, 0})
}
} else {
downInfo.load(downloadConfigFile)
downloadSize = downInfo.getDownloadSize()
}
contentChan := make(chan fileContent, downInfo.maxGo)
dwg.Add(downInfo.maxGo)
for i := 0; i < downInfo.maxGo; i++ {
go task(url, downInfo.schedule[i][0] + downInfo.schedule[i][2], downInfo.schedule[i][1] - downInfo.schedule[i][2], i, contentChan)
}
go writeFile(downloadFile, downloadConfigFile, contentChan, downloadSize)
dwg.Wait()
fmt.Println("\n下载完成")
}
func writeFile(downloadFile, downloadConfigFile *os.File, contentChan chan fileContent, downloadSize int) {
info, _ := downloadFile.Stat()
for {
select {
case c := <-contentChan:
downloadSize += len(c.con)
downloadFile.WriteAt(c.con, int64(c.start))
downInfo.schedule[c.taskId][2] += len(c.con)
downInfo.write(downloadConfigFile)
if c.isEnd {
dwg.Done()
}
fmt.Printf("\r当前下载量 %d/%d", downloadSize, info.Size())
}
}
}
func task(url string, downloadStart, downloadLen, taskId int, contentChan chan fileContent) {
request, err := http.NewRequest("GET", url, nil)
if err != nil {
panic(err)
}
request.Header.Add("Range", fmt.Sprintf("bytes=%d-%d", downloadStart, downloadStart+downloadLen-1))
client := http.Client{}
response, err := client.Do(request)
if err != nil {
panic(err)
}
defer response.Body.Close()
content := make([]byte, 1024*4)
start := downloadStart
isEnd := false
for {
read, err := response.Body.Read(content)
if err != nil {
if err != io.EOF {
panic(err)
}
isEnd = true
}
if read > 0 {
copyContent := make([]byte, read)
copy(copyContent, content[0:read])
contentChan <- fileContent{taskId, copyContent, start, isEnd}
start += read
}
if isEnd {
break
}
}
}
COMMENTS | NOTHING