package libgobuster import ( "context" "crypto/tls" "errors" "fmt" "io" "net/http" "net/url" "strings" ) // HTTPHeader holds a single key value pair of a HTTP header type HTTPHeader struct { Name string Value string } // HTTPClient represents a http object type HTTPClient struct { client *http.Client userAgent string defaultUserAgent string username string password string headers []HTTPHeader noCanonicalizeHeaders bool cookies string method string host string } // RequestOptions is used to pass options to a single individual request type RequestOptions struct { Host string Body io.Reader ReturnBody bool ModifiedHeaders []HTTPHeader UpdatedBasicAuthUsername string UpdatedBasicAuthPassword string } // NewHTTPClient returns a new HTTPClient func NewHTTPClient(opt *HTTPOptions) (*HTTPClient, error) { var proxyURLFunc func(*http.Request) (*url.URL, error) var client HTTPClient proxyURLFunc = http.ProxyFromEnvironment if opt == nil { return nil, fmt.Errorf("options is nil") } if opt.Proxy != "" { proxyURL, err := url.Parse(opt.Proxy) if err != nil { return nil, fmt.Errorf("proxy URL is invalid (%w)", err) } proxyURLFunc = http.ProxyURL(proxyURL) } var redirectFunc func(req *http.Request, via []*http.Request) error if !opt.FollowRedirect { redirectFunc = func(req *http.Request, via []*http.Request) error { return http.ErrUseLastResponse } } else { redirectFunc = nil } tlsConfig := tls.Config{ InsecureSkipVerify: opt.NoTLSValidation, // enable TLS1.0 and TLS1.1 support MinVersion: tls.VersionTLS10, } if opt.TLSCertificate != nil { tlsConfig.Certificates = []tls.Certificate{*opt.TLSCertificate} } client.client = &http.Client{ Timeout: opt.Timeout, CheckRedirect: redirectFunc, Transport: &http.Transport{ Proxy: proxyURLFunc, MaxIdleConns: 100, MaxIdleConnsPerHost: 100, TLSClientConfig: &tlsConfig, }} client.username = opt.Username client.password = opt.Password client.userAgent = opt.UserAgent client.defaultUserAgent = DefaultUserAgent() client.headers = opt.Headers client.noCanonicalizeHeaders = opt.NoCanonicalizeHeaders client.cookies = opt.Cookies client.method = opt.Method if client.method == "" { client.method = http.MethodGet } // Host header needs to be set separately for _, h := range opt.Headers { if h.Name == "Host" { client.host = h.Value break } } return &client, nil } // Request makes an http request and returns the status, the content length, the headers, the body and an error // if you want the body returned set the corresponding property inside RequestOptions func (client *HTTPClient) Request(ctx context.Context, fullURL string, opts RequestOptions) (int, int64, http.Header, []byte, error) { resp, err := client.makeRequest(ctx, fullURL, opts) if err != nil { // ignore context canceled errors if errors.Is(ctx.Err(), context.Canceled) { return 0, 0, nil, nil, nil } return 0, 0, nil, nil, err } defer resp.Body.Close() var body []byte var length int64 if opts.ReturnBody { body, err = io.ReadAll(resp.Body) if err != nil { return 0, 0, nil, nil, fmt.Errorf("could not read body %w", err) } length = int64(len(body)) } else { // DO NOT REMOVE! // absolutely needed so golang will reuse connections! length, err = io.Copy(io.Discard, resp.Body) if err != nil { return 0, 0, nil, nil, err } } return resp.StatusCode, length, resp.Header, body, nil } func (client *HTTPClient) makeRequest(ctx context.Context, fullURL string, opts RequestOptions) (*http.Response, error) { req, err := http.NewRequest(client.method, fullURL, opts.Body) if err != nil { return nil, err } // add the context so we can easily cancel out req = req.WithContext(ctx) if client.cookies != "" { req.Header.Set("Cookie", client.cookies) } // Use host for VHOST mode on a per request basis, otherwise the one provided from headers if opts.Host != "" { req.Host = opts.Host } else if client.host != "" { req.Host = client.host } if client.userAgent != "" { req.Header.Set("User-Agent", client.userAgent) } else { req.Header.Set("User-Agent", client.defaultUserAgent) } // add custom headers // if ModifiedHeaders are supplied use those, otherwise use the original ones // currently only relevant on fuzzing if len(opts.ModifiedHeaders) > 0 { for _, h := range opts.ModifiedHeaders { if client.noCanonicalizeHeaders { // https://stackoverflow.com/questions/26351716/how-to-keep-key-case-sensitive-in-request-header-using-golang req.Header[h.Name] = []string{h.Value} } else { req.Header.Set(h.Name, h.Value) } } } else { for _, h := range client.headers { if client.noCanonicalizeHeaders { // https://stackoverflow.com/questions/26351716/how-to-keep-key-case-sensitive-in-request-header-using-golang req.Header[h.Name] = []string{h.Value} } else { req.Header.Set(h.Name, h.Value) } } } if opts.UpdatedBasicAuthUsername != "" { req.SetBasicAuth(opts.UpdatedBasicAuthUsername, opts.UpdatedBasicAuthPassword) } else if client.username != "" { req.SetBasicAuth(client.username, client.password) } resp, err := client.client.Do(req) if err != nil { var ue *url.Error if errors.As(err, &ue) { if strings.HasPrefix(ue.Err.Error(), "x509") { return nil, fmt.Errorf("invalid certificate: %w", ue.Err) } } return nil, err } return resp, nil }