Add http download command (#2)

* Return a custom error for http download

* Finish the basic function of download file

Author: LinuxSuRen

.github/settings.yml

@@ -1,6 +1,6 @@
 repository:
   name: http-downloader
-  description: This is a template project
+  description: HTTP download tool
   homepage: https://github.com/linuxsuren
   private: false
   has_issues: true

.goreleaser.yml

@@ -3,7 +3,7 @@ project_name: http-downloader
 builds:
 - env:
   - CGO_ENABLED=0
-  binary: http-downloader
+  binary: hd
   goarch:
     - amd64
     - arm64
@@ -15,9 +15,9 @@ builds:
     post:
       - upx "{{ .Path }}"
   ldflags:
-    - -X github.com/linuxsuren/cgit/app.version={{.Version}}
-    - -X github.com/linuxsuren/cgit/app.commit={{.ShortCommit}}
-    - -X github.com/linuxsuren/cgit/app.date={{.Date}}
+    - -X github.com/linuxsuren/cobra-extension/version.version={{.Version}}
+    - -X github.com/linuxsuren/cobra-extension/version.commit={{.ShortCommit}}
+    - -X github.com/linuxsuren/cobra-extension/version.date={{.Date}}
     - -w
 dist: release
 archives:
@@ -46,20 +46,18 @@ changelog:
     - '^test:'
 brews:
   -
-    name: http-downloader
+    name: hd
     tap:
       owner: linuxsuren
       name: homebrew-linuxsuren
     folder: Formula
     homepage: "https://github.com/linuxsuren/http-downloader"
-    description: cgit is a tiny tool for Chinese developers.
+    description: HTTP download tool
     dependencies:
-      - name: vim
-        type: optional
       - name: bash-completion
         type: optional
     test: |
-      version_output = shell_output("#{bin}/http-downloader version")
+      version_output = shell_output("#{bin}/hd version")
       assert_match version.to_s, version_output
     install: |
       bin.install name
@@ -68,7 +66,7 @@ brews:
 nfpms:
   - file_name_template: "{{ .Binary }}-{{.Os}}-{{.Arch}}"
     homepage: https://github.com/linuxsuren/http-downloader
-    description: cgit is a tiny tool for Chinese developers.
+    description: HTTP download tool
     maintainer: rick <rick@jenkins-zh.cn>
     license: MIT
     vendor: Jenkins

Makefile

@@ -1,5 +1,5 @@
 build: fmt
-	CGO_ENABLE=0 go build -ldflags "-w -s" -o bin/github-go
+	CGO_ENABLE=0 go build -ldflags "-w -s" -o bin/hd
 
 run:
 	go run main.go
@@ -8,4 +8,4 @@ fmt:
 	go fmt ./...
 
 copy: build
-	sudo cp bin/github-go /usr/local/bin/github-go
+	sudo cp bin/hd /usr/local/bin/

cmd/root.go

@@ -0,0 +1,184 @@
+package cmd
+
+import (
+	"fmt"
+	"github.com/linuxsuren/http-downloader/pkg"
+	extver "github.com/linuxsuren/cobra-extension/version"
+	"github.com/spf13/cobra"
+	"io/ioutil"
+	"net/http"
+	"os"
+	"strconv"
+	"strings"
+	"sync"
+)
+
+// NewRoot returns the root command
+func NewRoot() (cmd *cobra.Command) {
+	opt := &downloadOption{}
+	cmd = &cobra.Command{
+		Use:     "hd",
+		Short:   "HTTP download tool",
+		PreRunE: opt.preRunE,
+		RunE:    opt.runE,
+	}
+
+	// set flags
+	flags := cmd.Flags()
+	flags.StringVarP(&opt.Output, "output", "o", "", "Write output to <file> instead of stdout.")
+	flags.BoolVarP(&opt.ShowProgress, "show-progress", "", true, "If show the progress of download")
+	flags.Int64VarP(&opt.ContinueAt, "continue-at", "", -1, "ContinueAt")
+	flags.IntVarP(&opt.Thread, "thread", "", 0, "")
+
+	cmd.AddCommand(
+		extver.NewVersionCmd("linuxsuren", "http-downloader", "hd", nil))
+	return
+}
+
+type downloadOption struct {
+	URL          string
+	Output       string
+	ShowProgress bool
+
+	ContinueAt int64
+
+	Thread int
+}
+
+func (o *downloadOption) preRunE(cmd *cobra.Command, args []string) (err error) {
+	if len(args) <= 0 {
+		return fmt.Errorf("no URL provided")
+	}
+
+	url := args[0]
+	if !strings.HasPrefix(url, "http://") && !strings.HasPrefix(url, "https://") {
+		err = fmt.Errorf("only http:// or https:// supported")
+	}
+	o.URL = url
+
+	if o.Output == "" {
+		err = fmt.Errorf("output cannot be empty")
+	}
+	return
+}
+
+func (o *downloadOption) runE(cmd *cobra.Command, args []string) (err error) {
+	if o.Thread <= 1 {
+		err = o.download(o.Output, o.ContinueAt, 0)
+	} else {
+		// get the total size of the target file
+		var total int64
+		var rangeSupport bool
+		if total, rangeSupport, err = o.detectSize(o.Output); err != nil {
+			return
+		}
+
+		if rangeSupport {
+			unit := total / int64(o.Thread)
+			offset := total - unit*int64(o.Thread)
+			var wg sync.WaitGroup
+
+			cmd.Printf("start to download with %d threads, size: %d, unit: %d\n", o.Thread, total, unit)
+			for i := 0; i < o.Thread; i++ {
+				wg.Add(1)
+				go func(index int, wg *sync.WaitGroup) {
+					defer wg.Done()
+
+					end := unit*int64(index+1) - 1
+					if index == o.Thread-1 {
+						// this is the last part
+						end += offset
+					}
+					start := unit * int64(index)
+
+					if downloadErr := o.download(fmt.Sprintf("%s-%d", o.Output, index), start, end); downloadErr != nil {
+						cmd.PrintErrln(downloadErr)
+					}
+				}(i, &wg)
+			}
+
+			wg.Wait()
+
+			// concat all these partial files
+			var f *os.File
+			if f, err = os.OpenFile(o.Output, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644); err == nil {
+				defer func() {
+					_ = f.Close()
+				}()
+
+				for i := 0; i < o.Thread; i++ {
+					partFile := fmt.Sprintf("%s-%d", o.Output, i)
+					if data, ferr := ioutil.ReadFile(partFile); ferr == nil {
+						if _, err = f.Write(data); err != nil {
+							err = fmt.Errorf("failed to write file: '%s'", partFile)
+							break
+						} else {
+							_ = os.RemoveAll(partFile)
+						}
+					} else {
+						err = fmt.Errorf("failed to read file: '%s'", partFile)
+						break
+					}
+				}
+			}
+		} else {
+			cmd.Println("cannot download it using multiple threads, failed to one")
+			err = o.download(o.Output, o.ContinueAt, 0)
+		}
+	}
+	return
+}
+
+func (o *downloadOption) detectSize(output string) (total int64, rangeSupport bool, err error) {
+	downloader := pkg.HTTPDownloader{
+		TargetFilePath: output,
+		URL:            o.URL,
+		ShowProgress:   o.ShowProgress,
+	}
+
+	var detectOffset int64
+	var lenErr error
+
+	detectOffset = 2
+	downloader.Header = make(map[string]string, 1)
+	downloader.Header["Range"] = fmt.Sprintf("bytes=%d-", detectOffset)
+
+	downloader.PreStart = func(resp *http.Response) bool {
+		rangeSupport = resp.StatusCode == http.StatusPartialContent
+		contentLen := resp.Header.Get("Content-Length")
+		if total, lenErr = strconv.ParseInt(contentLen, 10, 0); lenErr == nil {
+			total += detectOffset
+		}
+		//  always return false because we just want to get the header from response
+		return false
+	}
+
+	if err = downloader.DownloadFile(); err != nil || lenErr != nil {
+		err = fmt.Errorf("cannot download from %s, response error: %v, content length error: %v", o.URL, err, lenErr)
+	}
+	return
+}
+
+func (o *downloadOption) download(output string, continueAt, end int64) (err error) {
+	downloader := pkg.HTTPDownloader{
+		TargetFilePath: output,
+		URL:            o.URL,
+		ShowProgress:   o.ShowProgress,
+	}
+
+	if continueAt >= 0 {
+		downloader.Header = make(map[string]string, 1)
+
+		//fmt.Println("range", continueAt, end)
+		if end > continueAt {
+			downloader.Header["Range"] = fmt.Sprintf("bytes=%d-%d", continueAt, end)
+		} else {
+			downloader.Header["Range"] = fmt.Sprintf("bytes=%d-", continueAt)
+		}
+	}
+
+	if err = downloader.DownloadFile(); err != nil {
+		err = fmt.Errorf("cannot download from %s, error: %v", o.URL, err)
+	}
+	return
+}

go.mod

@@ -6,6 +6,7 @@ require (
 	github.com/golang/mock v1.4.4
 	github.com/gosuri/uiprogress v0.0.1
 	github.com/jenkins-zh/jenkins-cli v0.0.32
+	github.com/linuxsuren/cobra-extension v0.0.6
 	github.com/onsi/ginkgo v1.14.2
 	github.com/onsi/gomega v1.10.3
 	github.com/spf13/cobra v1.1.1

go.sum

@@ -155,6 +155,8 @@ github.com/kr/pty v1.1.4/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw=
 github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/linuxsuren/cobra-extension v0.0.6/go.mod h1:qcEJv7BbL0UpK6MbrTESP/nKf1+z1wQdMAnE1NBl3QQ=
+github.com/linuxsuren/http-downloader v0.0.2-0.20201207132639-19888a6beaec/go.mod h1:zRZY9FCDBuYNDxbI2Ny5suasZsMk7J6q9ecQ3V3PIqI=
 github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
 github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
 github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=

main.go

@@ -1,17 +1,12 @@
 package main
 
 import (
-	"github.com/spf13/cobra"
+	"github.com/linuxsuren/http-downloader/cmd"
+	"os"
 )
 
-func main()  {
-	cmd := &cobra.Command{
-		Use: "github-go",
-		Run: func(cmd *cobra.Command, args []string) {
-			cmd.Println("hello")
-		},
-	}
-	if err := cmd.Execute(); err != nil {
-		panic(err)
+func main() {
+	if err := cmd.NewRoot().Execute(); err != nil {
+		os.Exit(1)
 	}
 }

pkg/error.go

@@ -0,0 +1,14 @@
+package pkg
+
+import "fmt"
+
+// DownloadError represents the error of HTTP download
+type DownloadError struct {
+	StatusCode int
+	Message    string
+}
+
+// Error print the error message
+func (e *DownloadError) Error() string {
+	return fmt.Sprintf("%s: status code: %d", e.Message, e.StatusCode)
+}

pkg/http.go

@@ -5,7 +5,6 @@ import (
 	"encoding/base64"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"net/http"
 	"net/url"
 	"os"
@@ -35,6 +34,11 @@ type HTTPDownloader struct {
 	Proxy     string
 	ProxyAuth string
 
+	Header map[string]string
+
+	// PreStart returns false will don't continue
+	PreStart func(*http.Response) bool
+
 	Debug        bool
 	RoundTripper http.RoundTripper
 }
@@ -81,6 +85,9 @@ func (h *HTTPDownloader) fetchProxyFromEnv(scheme string) {
 	}
 }
 
+//Range: bytes=10-
+//HTTP/1.1 206 Partial Content
+
 // DownloadFile download a file with the progress
 func (h *HTTPDownloader) DownloadFile() error {
 	filepath, downloadURL, showProgress := h.TargetFilePath, h.URL, h.ShowProgress
@@ -90,6 +97,10 @@ func (h *HTTPDownloader) DownloadFile() error {
 		return err
 	}
 
+	for k, v := range h.Header {
+		req.Header.Set(k, v)
+	}
+
 	if h.UserName != "" && h.Password != "" {
 		req.SetBasicAuth(h.UserName, h.Password)
 	}
@@ -118,13 +129,16 @@ func (h *HTTPDownloader) DownloadFile() error {
 		return err
 	}
 
-	if resp.StatusCode != 200 {
-		if h.Debug {
-			if data, err := ioutil.ReadAll(resp.Body); err == nil {
-				ioutil.WriteFile("debug-download.html", data, 0664)
-			}
+	if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusPartialContent {
+		return &DownloadError{
+			Message:    fmt.Sprintf("failed to download from '%s'", downloadURL),
+			StatusCode: resp.StatusCode,
 		}
-		return fmt.Errorf("invalidate status code: %d", resp.StatusCode)
+	}
+
+	// pre-hook before get started to download file
+	if h.PreStart != nil && !h.PreStart(resp) {
+		return nil
 	}
 
 	writer := &ProgressIndicator{