golang 基于 Web 和 CLI 的下载管理器 三

发表于 2021-03-02  2k 次阅读


并发版断点续传

并发版就不能直接读取文件的大小了,需要一个文件来去存储下载的进度,在一开下载的时候读取内容,再根据内容去下载
完整代码,可能有一些瑕疵

package main

import (
    "bufio"
    "errors"
    "fmt"
    "io"
    "math"
    "net/http"
    "os"
    "path"
    "strconv"
    "strings"
    "sync"
)

type fileContent struct {
    taskId int
    con    []byte
    start  int
    isEnd  bool
}

var dwg sync.WaitGroup

//并发下载数
var maxGo = 2

var downInfo info

type info struct {
    url      string
    maxGo    int
    schedule [][3]int
}

//读取配置
func (info *info) load(file *os.File) {
    rd := bufio.NewReader(file)

    //第一行
    line, _ := rd.ReadString('\n')
    line = strings.TrimSpace(line)
    info.url = line

    //第二行
    line, _ = rd.ReadString('\n')
    line = strings.TrimSpace(line)
    maxGoArr := strings.Split(line, ":")
    maxGo , _ := strconv.Atoi(maxGoArr[1])
    info.maxGo = maxGo

    //第三行
    line, _ = rd.ReadString('\n')
    line = strings.TrimSpace(line)
    var tmpSchedule [][3]int
    scheduleArr1 := strings.Split(line, ":")
    scheduleArr1 = strings.Split(line, ",")
    for _, scheduleArr := range scheduleArr1 {
        scheduleOne := strings.Split(scheduleArr, "-")
        start, _ := strconv.Atoi(scheduleOne[0])
        len, _ := strconv.Atoi(scheduleOne[1])
        down, _ := strconv.Atoi(scheduleOne[2])
        tmpSchedule = append(tmpSchedule, [3]int{start, len, down})
    }
    info.schedule = tmpSchedule
}

//写入配置
func (info info) write(file *os.File) {
    file.Seek(0, 0)
    scheduleString := ""
    for _, v := range info.schedule {
        if len(scheduleString) != 0 {
            scheduleString += ","
        }
        scheduleString += fmt.Sprintf("%d-%d-%d", v[0], v[1], v[2])
    }
    writeString := fmt.Sprintf("url:%s\nmaxGo:%d\nschedule:%s", info.url, info.maxGo, scheduleString)
    file.Write([]byte(writeString))
}

func (info info) getDownloadSize() int {
    downloadSize := 0
    for _, v := range info.schedule {
        downloadSize += v[2]
    }
    return downloadSize
}

func main() {
    url := "http://ybook.iapi.im/header-leg.jpg"
    request, err := http.NewRequest("HEAD", url, nil)
    if err != nil {
        panic(err)
    }

    client := http.Client{}
    response, err := client.Do(request)
    if err != nil {
        panic(err)
    }

    fmt.Println(response.Header)
    //map[Accept-Ranges:[bytes] Cache-Control:[max-age=2592000] Connection:[keep-alive] Content-Length:[572356] Content-Type:[image/jpeg] Date:[Tue, 23 Feb 2021 12:00:57 GMT] Etag:["6032775a-8bbc4"] Ex
    //pires:[Thu, 25 Mar 2021 12:00:57 GMT] Last-Modified:[Sun, 21 Feb 2021 15:08:10 GMT] Server:[nginx]]
    if response.Header.Get("Accept-Ranges") != "bytes" {
        panic(errors.New("当前文件不支持下载"))
    }

    fileName := path.Base(url)
    fileSize, _ := strconv.Atoi(response.Header.Get("Content-Length"))

    fmt.Printf("文件名称:%s, 文件大小:%d \n", fileName, fileSize)

    _, existErr := os.Stat("./" + fileName)

    downloadFile, err := os.OpenFile("./"+fileName, os.O_CREATE|os.O_WRONLY, 0666)
    if err != nil {
        panic(err)
    }

    downloadConfigFile, err := os.OpenFile("./"+fileName+".jc", os.O_CREATE|os.O_RDWR, 0666)
    if err != nil {
        panic(err)
    }

    downloadSize := 0
    if os.IsNotExist(existErr) {
        downloadFile.Seek(int64(fileSize-1), 0)
        downloadFile.Write([]byte{0})
        downInfo.url = url;
        downInfo.maxGo = maxGo

        //每段平均下载数据量,最后一段会有偏差
        avg := int(math.Ceil(float64(fileSize / maxGo)))

        for i := 0; i < maxGo; i++ {
            downloadStart := avg * i
            downloadLen := avg
            if fileSize < (downloadStart + avg) {
                downloadLen = fileSize - downloadStart
            }
            downInfo.schedule = append(downInfo.schedule, [3]int{downloadStart, downloadLen, 0})
        }
    } else {
        downInfo.load(downloadConfigFile)
        downloadSize = downInfo.getDownloadSize()
    }

    contentChan := make(chan fileContent, downInfo.maxGo)

    dwg.Add(downInfo.maxGo)
    for i := 0; i < downInfo.maxGo; i++ {
        go task(url, downInfo.schedule[i][0] + downInfo.schedule[i][2], downInfo.schedule[i][1] - downInfo.schedule[i][2], i, contentChan)
    }

    go writeFile(downloadFile, downloadConfigFile, contentChan, downloadSize)

    dwg.Wait()
    fmt.Println("\n下载完成")
}

func writeFile(downloadFile, downloadConfigFile *os.File, contentChan chan fileContent, downloadSize int) {
    info, _ := downloadFile.Stat()
    for {
        select {
        case c := <-contentChan:
            downloadSize += len(c.con)
            downloadFile.WriteAt(c.con, int64(c.start))
            downInfo.schedule[c.taskId][2] += len(c.con)
            downInfo.write(downloadConfigFile)
            if c.isEnd {
                dwg.Done()
            }
            fmt.Printf("\r当前下载量 %d/%d", downloadSize, info.Size())
        }
    }
}

func task(url string, downloadStart, downloadLen, taskId int, contentChan chan fileContent) {
    request, err := http.NewRequest("GET", url, nil)
    if err != nil {
        panic(err)
    }

    request.Header.Add("Range", fmt.Sprintf("bytes=%d-%d", downloadStart, downloadStart+downloadLen-1))
    client := http.Client{}
    response, err := client.Do(request)
    if err != nil {
        panic(err)
    }

    defer response.Body.Close()

    content := make([]byte, 1024*4)
    start := downloadStart
    isEnd := false
    for {
        read, err := response.Body.Read(content)

        if err != nil {
            if err != io.EOF {
                panic(err)
            }
            isEnd = true
        }

        if read > 0 {
            copyContent := make([]byte, read)
            copy(copyContent, content[0:read])
            contentChan <- fileContent{taskId, copyContent, start, isEnd}
            start += read
        }

        if isEnd {
            break
        }
    }
}

本站文章基于国际协议BY-NA-SA 4.0协议共享;
如未特殊说明,本站文章皆为原创文章,请规范转载。

0

一盏灯 一座城 找一人 一路的颠沛流离