@@ -5,13 +5,13 @@ import (
5
5
"encoding/base64"
6
6
"fmt"
7
7
"io"
8
+ "io/ioutil"
8
9
"net/http"
9
10
"net/url"
10
11
"os"
11
12
"path"
12
13
"strconv"
13
-
14
- "github.com/gosuri/uiprogress"
14
+ "sync"
15
15
)
16
16
17
17
const (
@@ -39,6 +39,8 @@ type HTTPDownloader struct {
39
39
// PreStart returns false will don't continue
40
40
PreStart func (* http.Response ) bool
41
41
42
+ Thread int
43
+
42
44
Debug bool
43
45
RoundTripper http.RoundTripper
44
46
}
@@ -85,9 +87,6 @@ func (h *HTTPDownloader) fetchProxyFromEnv(scheme string) {
85
87
}
86
88
}
87
89
88
- //Range: bytes=10-
89
- //HTTP/1.1 206 Partial Content
90
-
91
90
// DownloadFile download a file with the progress
92
91
func (h * HTTPDownloader ) DownloadFile () error {
93
92
filepath , downloadURL , showProgress := h .TargetFilePath , h .URL , h .ShowProgress
@@ -175,52 +174,121 @@ func (h *HTTPDownloader) DownloadFile() error {
175
174
return err
176
175
}
177
176
178
- // ProgressIndicator hold the progress of io operation
179
- type ProgressIndicator struct {
180
- Writer io.Writer
181
- Reader io.Reader
182
- Title string
177
+ // DownloadFileWithMultipleThread downloads the files with multiple threads
178
+ func DownloadFileWithMultipleThread (targetURL , targetFilePath string , thread int , showProgress bool ) (err error ) {
179
+ // get the total size of the target file
180
+ var total int64
181
+ var rangeSupport bool
182
+ if total , rangeSupport , err = DetectSize (targetURL , targetFilePath , true ); err != nil {
183
+ return
184
+ }
185
+
186
+ if rangeSupport {
187
+ unit := total / int64 (thread )
188
+ offset := total - unit * int64 (thread )
189
+ var wg sync.WaitGroup
190
+
191
+ fmt .Printf ("start to download with %d threads, size: %d, unit: %d\n " , thread , total , unit )
192
+ for i := 0 ; i < thread ; i ++ {
193
+ wg .Add (1 )
194
+ go func (index int , wg * sync.WaitGroup ) {
195
+ defer wg .Done ()
196
+
197
+ end := unit * int64 (index + 1 ) - 1
198
+ if index == thread - 1 {
199
+ // this is the last part
200
+ end += offset
201
+ }
202
+ start := unit * int64 (index )
203
+
204
+ if downloadErr := DownloadWithContinue (targetURL , fmt .Sprintf ("%s-%d" , targetFilePath , index ), start , end , showProgress ); downloadErr != nil {
205
+ fmt .Println (downloadErr )
206
+ }
207
+ }(i , & wg )
208
+ }
183
209
184
- // bytes.Buffer
185
- Total float64
186
- count float64
187
- bar * uiprogress.Bar
210
+ wg .Wait ()
211
+
212
+ // concat all these partial files
213
+ var f * os.File
214
+ if f , err = os .OpenFile (targetFilePath , os .O_APPEND | os .O_CREATE | os .O_WRONLY , 0644 ); err == nil {
215
+ defer func () {
216
+ _ = f .Close ()
217
+ }()
218
+
219
+ for i := 0 ; i < thread ; i ++ {
220
+ partFile := fmt .Sprintf ("%s-%d" , targetFilePath , i )
221
+ if data , ferr := ioutil .ReadFile (partFile ); ferr == nil {
222
+ if _ , err = f .Write (data ); err != nil {
223
+ err = fmt .Errorf ("failed to write file: '%s'" , partFile )
224
+ break
225
+ } else {
226
+ _ = os .RemoveAll (partFile )
227
+ }
228
+ } else {
229
+ err = fmt .Errorf ("failed to read file: '%s'" , partFile )
230
+ break
231
+ }
232
+ }
233
+ }
234
+ } else {
235
+ fmt .Println ("cannot download it using multiple threads, failed to one" )
236
+ err = DownloadWithContinue (targetURL , targetFilePath , 0 , 0 , true )
237
+ }
238
+ return
188
239
}
189
240
190
- // Init set the default value for progress indicator
191
- func (i * ProgressIndicator ) Init () {
192
- uiprogress .Start () // start rendering
193
- i .bar = uiprogress .AddBar (100 ) // Add a new bar
241
+ // DownloadWithContinue downloads the files continuously
242
+ func DownloadWithContinue (targetURL , output string , continueAt , end int64 , showProgress bool ) (err error ) {
243
+ downloader := HTTPDownloader {
244
+ TargetFilePath : output ,
245
+ URL : targetURL ,
246
+ ShowProgress : showProgress ,
247
+ }
194
248
195
- // optionally, append and prepend completion and elapsed time
196
- i .bar .AppendCompleted ()
197
- // i.bar.PrependElapsed()
249
+ if continueAt >= 0 {
250
+ downloader .Header = make (map [string ]string , 1 )
198
251
199
- if i .Title != "" {
200
- i .bar .PrependFunc (func (_ * uiprogress.Bar ) string {
201
- return fmt .Sprintf ("%s: " , i .Title )
202
- })
252
+ if end > continueAt {
253
+ downloader .Header ["Range" ] = fmt .Sprintf ("bytes=%d-%d" , continueAt , end )
254
+ } else {
255
+ downloader .Header ["Range" ] = fmt .Sprintf ("bytes=%d-" , continueAt )
256
+ }
203
257
}
204
- }
205
258
206
- // Write writes the progress
207
- func (i * ProgressIndicator ) Write (p []byte ) (n int , err error ) {
208
- n , err = i .Writer .Write (p )
209
- i .setBar (n )
259
+ if err = downloader .DownloadFile (); err != nil {
260
+ err = fmt .Errorf ("cannot download from %s, error: %v" , targetURL , err )
261
+ }
210
262
return
211
263
}
212
264
213
- // Read reads the progress
214
- func (i * ProgressIndicator ) Read (p []byte ) (n int , err error ) {
215
- n , err = i .Reader .Read (p )
216
- i .setBar (n )
217
- return
218
- }
265
+ // DetectSize returns the size of target resource
266
+ func DetectSize (targetURL , output string , showProgress bool ) (total int64 , rangeSupport bool , err error ) {
267
+ downloader := HTTPDownloader {
268
+ TargetFilePath : output ,
269
+ URL : targetURL ,
270
+ ShowProgress : showProgress ,
271
+ }
272
+
273
+ var detectOffset int64
274
+ var lenErr error
275
+
276
+ detectOffset = 2
277
+ downloader .Header = make (map [string ]string , 1 )
278
+ downloader .Header ["Range" ] = fmt .Sprintf ("bytes=%d-" , detectOffset )
219
279
220
- func (i * ProgressIndicator ) setBar (n int ) {
221
- i .count += float64 (n )
280
+ downloader .PreStart = func (resp * http.Response ) bool {
281
+ rangeSupport = resp .StatusCode == http .StatusPartialContent
282
+ contentLen := resp .Header .Get ("Content-Length" )
283
+ if total , lenErr = strconv .ParseInt (contentLen , 10 , 0 ); lenErr == nil {
284
+ total += detectOffset
285
+ }
286
+ // always return false because we just want to get the header from response
287
+ return false
288
+ }
222
289
223
- if i . bar != nil {
224
- i . bar . Set (( int )( i . count * 100 / i . Total ) )
290
+ if err = downloader . DownloadFile (); err != nil || lenErr != nil {
291
+ err = fmt . Errorf ( "cannot download from %s, response error: %v, content length error: %v" , targetURL , err , lenErr )
225
292
}
293
+ return
226
294
}
0 commit comments