golang批量下载文件
package main
import (
"bufio"
"fmt"
"io/ioutil"
"log"
"net/http"
"os"
"path/filepath"
"strings"
"github.com/goodlucktou/sizedwaitgroup"
"strconv"
"encoding/json"
)
var (
downloadDestFolder = "./batch_download/"
)
func init() {
log.SetFlags(log.Lshortfile)
_ = os.MkdirAll(downloadDestFolder, 0777)
}
type configuration struct {
Type int
Url string
Fileext string
Start int
End int
Threadnum int
FilePath string
DestFolder string
}
func main() {
file, _ := os.Open("./config.json")
defer file.Close()
decoder := json.NewDecoder(file)
conf := configuration{}
err := decoder.Decode(&conf)
if err != nil {
fmt.Println("Error:", err)
}
downloadDestFolder = conf.DestFolder
fmt.Printf("%v \n",conf)
swg := sizedwaitgroup.New(conf.Threadnum)
if conf.Type ==1 {
//urlFromNum(conf,&swg)
url := conf.Url
start := conf.Start
end := conf.End
tmp := ""
for i:=start;i<=end;i++ {
tmp = strconv.Itoa(i)
url = strings.Replace(url,"{*}",tmp, 1)
filename := tmp+ conf.Fileext
//fmt.Println("url:", url)
swg.Add()
go func(url string, filename string) {
defer swg.Done()
download(url, filename)
}(url,filename)
}
}else if conf.Type ==2 {
//urlFromFile(conf)
fi, err := os.Open(conf.FilePath)
if err != nil {
fmt.Printf("Error: %s\n", err)
return
}
defer fi.Close()
br := bufio.NewReader(fi)
for {
line, _, err := br.ReadLine()
if err != nil {
log.Println("read url complete")
break
}
list := strings.Split(string(line), ",")
swg.Add()
go func(url string, filename string) {
defer swg.Done()
download(url, filename)
}(list[1],list[0])
}
}else{
fmt.Println("Error: no download type value")
}
swg.Wait()
}
func download(url string, filename string) {
res, err := http.Get(url)
if err != nil {
log.Printf("http.Get -> %v", err)
return
}
data, err := ioutil.ReadAll(res.Body)
if err != nil {
log.Printf("ioutil.ReadAll -> %s", err.Error())
return
}
defer res.Body.Close()
if err = ioutil.WriteFile(downloadDestFolder+string(filepath.Separator)+filename, data, 0777); err != nil {
log.Println("Error Saving:", filename, err)
} else {
log.Println("Saved:", filename)
}
}
config.json文件内容
{ "type":1, "Url":"https://cctv.com/mp4/mp{*}.ts","Fileext":".ts","Start":4000,"End":4355,"Threadnum":5,"FilePath":"./batch_download/file.txt","DestFolder":"./batch_download/"}
尊贵的董事大人
英文标题不为空时 视为本栏投稿
需要关键字 描述 英文标题