HTTP Client Configuration

7 minute read

HTTP Client Configuration Overview

The Go http.Client is highly configurable. Proper configuration ensures reliability, performance, and security.

Basic HTTP Client

Default Client

package client

import (
    "net/http"
    "time"
)

func NewClient(apiKey string) (*Client, error) {
    return &Client{
        apiKey: apiKey,
        httpClient: &http.Client{
            Timeout: 30 * time.Second,
        },
    }, nil
}

Problems with default:

No connection pooling control
Default TLS settings
No retry logic
Simple timeout

Timeouts Configuration

Types of Timeouts

func NewClient(apiKey string) (*Client, error) {
    transport := &http.Transport{
        // DNS resolution timeout
        DialContext: (&net.Dialer{
            Timeout:   10 * time.Second,
            KeepAlive: 30 * time.Second,
        }).DialContext,
        
        // TLS handshake timeout
        TLSHandshakeTimeout: 10 * time.Second,
        
        // Wait for response headers timeout
        ResponseHeaderTimeout: 20 * time.Second,
        
        // Wait for full response timeout
        ExpectContinueTimeout: 1 * time.Second,
    }
    
    return &Client{
        apiKey: apiKey,
        httpClient: &http.Client{
            // Overall request timeout
            Timeout:   60 * time.Second,
            Transport: transport,
        },
    }, nil
}

Timeout Hierarchy

Client.Timeout (60s) ← Overall limit
  ├─ Dial (10s) ← Connection establishment
  ├─ TLS Handshake (10s) ← TLS negotiation
  ├─ Response Headers (20s) ← Wait for headers
  └─ Body Read (remaining) ← Read response body

Context Timeout

func (c *Client) GetDatabase(ctx context.Context, id string) (*Database, error) {
    // Operation-specific timeout
    ctx, cancel := context.WithTimeout(ctx, 10*time.Second)
    defer cancel()
    
    req, _ := http.NewRequestWithContext(ctx, "GET", url, nil)
    
    resp, err := c.httpClient.Do(req)
    if err != nil {
        if ctx.Err() == context.DeadlineExceeded {
            return nil, fmt.Errorf("request timed out after 10s")
        }
        return nil, err
    }
    
    // ...
}

Connection Pooling

Configure Pools

func NewClient(apiKey string) (*Client, error) {
    transport := &http.Transport{
        // Maximum idle connections across all hosts
        MaxIdleConns: 100,
        
        // Maximum idle connections per host
        MaxIdleConnsPerHost: 10,
        
        // Maximum total connections per host
        MaxConnsPerHost: 50,
        
        // How long to keep idle connections
        IdleConnTimeout: 90 * time.Second,
        
        // Disable compression if transferring pre-compressed data
        DisableCompression: false,
        
        // Disable keep-alive
        DisableKeepAlives: false,
    }
    
    return &Client{
        httpClient: &http.Client{
            Transport: transport,
        },
    }, nil
}

Connection Reuse Benefits

// ✅ Good: Reuse connections
func BenchmarkWithConnectionPool(b *testing.B) {
    client := &http.Client{
        Transport: &http.Transport{
            MaxIdleConnsPerHost: 10,
        },
    }
    
    for i := 0; i < b.N; i++ {
        resp, _ := client.Get("https://api.example.com/test")
        resp.Body.Close()
    }
}

// ❌ Bad: New connection each time
func BenchmarkWithoutConnectionPool(b *testing.B) {
    for i := 0; i < b.N; i++ {
        resp, _ := http.Get("https://api.example.com/test")
        resp.Body.Close()
    }
}

// Results:
// WithConnectionPool:    1000 ns/op
// WithoutConnectionPool: 50000 ns/op  (50x slower!)

TLS Configuration

Custom TLS Config

import (
    "crypto/tls"
    "crypto/x509"
)

func NewClient(apiKey string, opts *ClientOptions) (*Client, error) {
    tlsConfig := &tls.Config{
        // Minimum TLS version
        MinVersion: tls.VersionTLS12,
        
        // Preferred cipher suites
        CipherSuites: []uint16{
            tls.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,
            tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,
        },
        
        // Certificate verification
        InsecureSkipVerify: false,
    }
    
    // Add custom CA certificates
    if opts.CACertPath != "" {
        caCert, err := os.ReadFile(opts.CACertPath)
        if err != nil {
            return nil, fmt.Errorf("failed to read CA cert: %w", err)
        }
        
        caCertPool := x509.NewCertPool()
        caCertPool.AppendCertsFromPEM(caCert)
        tlsConfig.RootCAs = caCertPool
    }
    
    transport := &http.Transport{
        TLSClientConfig: tlsConfig,
    }
    
    return &Client{
        httpClient: &http.Client{
            Transport: transport,
        },
    }, nil
}

Client Certificate Authentication

func NewClientWithCert(certFile, keyFile string) (*Client, error) {
    // Load client certificate
    cert, err := tls.LoadX509KeyPair(certFile, keyFile)
    if err != nil {
        return nil, fmt.Errorf("failed to load client cert: %w", err)
    }
    
    tlsConfig := &tls.Config{
        Certificates: []tls.Certificate{cert},
        MinVersion:   tls.VersionTLS12,
    }
    
    transport := &http.Transport{
        TLSClientConfig: tlsConfig,
    }
    
    return &Client{
        httpClient: &http.Client{
            Transport: transport,
        },
    }, nil
}

Skip TLS Verification (Development Only)

// ⚠️ WARNING: Only for development/testing
func NewInsecureClient(apiKey string) (*Client, error) {
    transport := &http.Transport{
        TLSClientConfig: &tls.Config{
            InsecureSkipVerify: true,  // Skip certificate verification
        },
    }
    
    return &Client{
        apiKey: apiKey,
        httpClient: &http.Client{
            Transport: transport,
        },
    }, nil
}

Custom Transport

Logging Transport

type LoggingTransport struct {
    Transport http.RoundTripper
    Logger    *log.Logger
}

func (t *LoggingTransport) RoundTrip(req *http.Request) (*http.Response, error) {
    // Log request
    t.Logger.Printf("[%s] %s", req.Method, req.URL.String())
    
    start := time.Now()
    
    // Make request
    resp, err := t.Transport.RoundTrip(req)
    
    duration := time.Since(start)
    
    if err != nil {
        t.Logger.Printf("Request failed: %v (took %v)", err, duration)
        return nil, err
    }
    
    // Log response
    t.Logger.Printf("[%d] %s (took %v)", resp.StatusCode, req.URL.String(), duration)
    
    return resp, nil
}

func NewClient(apiKey string) (*Client, error) {
    baseTransport := &http.Transport{
        MaxIdleConns: 100,
    }
    
    loggingTransport := &LoggingTransport{
        Transport: baseTransport,
        Logger:    log.New(os.Stdout, "HTTP: ", log.LstdFlags),
    }
    
    return &Client{
        httpClient: &http.Client{
            Transport: loggingTransport,
        },
    }, nil
}

Retry Transport

type RetryTransport struct {
    Transport  http.RoundTripper
    MaxRetries int
}

func (t *RetryTransport) RoundTrip(req *http.Request) (*http.Response, error) {
    var resp *http.Response
    var err error
    
    for attempt := 0; attempt <= t.MaxRetries; attempt++ {
        // Clone request body for retries
        var bodyBytes []byte
        if req.Body != nil {
            bodyBytes, _ = io.ReadAll(req.Body)
            req.Body = io.NopCloser(bytes.NewReader(bodyBytes))
        }
        
        resp, err = t.Transport.RoundTrip(req)
        
        // Success
        if err == nil && resp.StatusCode < 500 {
            return resp, nil
        }
        
        // Retry on 5xx or network errors
        if attempt < t.MaxRetries {
            waitTime := time.Duration(math.Pow(2, float64(attempt))) * time.Second
            time.Sleep(waitTime)
            
            // Reset body for retry
            if bodyBytes != nil {
                req.Body = io.NopCloser(bytes.NewReader(bodyBytes))
            }
        }
    }
    
    return resp, err
}

Chaining Transports

func NewClient(apiKey string) (*Client, error) {
    // Base transport
    baseTransport := &http.Transport{
        MaxIdleConns:        100,
        MaxIdleConnsPerHost: 10,
    }
    
    // Add retry logic
    retryTransport := &RetryTransport{
        Transport:  baseTransport,
        MaxRetries: 3,
    }
    
    // Add logging
    loggingTransport := &LoggingTransport{
        Transport: retryTransport,
        Logger:    log.New(os.Stdout, "HTTP: ", log.LstdFlags),
    }
    
    // Add authentication
    authTransport := &AuthTransport{
        Transport: loggingTransport,
        APIKey:    apiKey,
    }
    
    return &Client{
        httpClient: &http.Client{
            Transport: authTransport,
            Timeout:   60 * time.Second,
        },
    }, nil
}

// Request flow:
// Client → AuthTransport → LoggingTransport → RetryTransport → BaseTransport → API

Proxy Configuration

HTTP Proxy

func NewClientWithProxy(apiKey, proxyURL string) (*Client, error) {
    proxy, err := url.Parse(proxyURL)
    if err != nil {
        return nil, fmt.Errorf("invalid proxy URL: %w", err)
    }
    
    transport := &http.Transport{
        Proxy: http.ProxyURL(proxy),
    }
    
    return &Client{
        httpClient: &http.Client{
            Transport: transport,
        },
    }, nil
}

Proxy from Environment

func NewClient(apiKey string) (*Client, error) {
    transport := &http.Transport{
        // Automatically use HTTP_PROXY, HTTPS_PROXY, NO_PROXY env vars
        Proxy: http.ProxyFromEnvironment,
    }
    
    return &Client{
        httpClient: &http.Client{
            Transport: transport,
        },
    }, nil
}

Production Configuration

Complete Production Client

package client

import (
    "crypto/tls"
    "net"
    "net/http"
    "time"
)

type ClientConfig struct {
    APIKey         string
    BaseURL        string
    Timeout        time.Duration
    MaxRetries     int
    MaxIdleConns   int
    TLSMinVersion  uint16
    CACertPath     string
    SkipTLSVerify  bool
    ProxyURL       string
}

func NewClient(config *ClientConfig) (*Client, error) {
    // Validate config
    if config.APIKey == "" {
        return nil, fmt.Errorf("API key is required")
    }
    
    // Set defaults
    if config.Timeout == 0 {
        config.Timeout = 60 * time.Second
    }
    if config.MaxIdleConns == 0 {
        config.MaxIdleConns = 100
    }
    if config.TLSMinVersion == 0 {
        config.TLSMinVersion = tls.VersionTLS12
    }
    
    // Configure TLS
    tlsConfig := &tls.Config{
        MinVersion:         config.TLSMinVersion,
        InsecureSkipVerify: config.SkipTLSVerify,
    }
    
    // Add custom CA if provided
    if config.CACertPath != "" {
        caCert, err := os.ReadFile(config.CACertPath)
        if err != nil {
            return nil, fmt.Errorf("failed to read CA cert: %w", err)
        }
        
        caCertPool := x509.NewCertPool()
        caCertPool.AppendCertsFromPEM(caCert)
        tlsConfig.RootCAs = caCertPool
    }
    
    // Configure transport
    transport := &http.Transport{
        Proxy: http.ProxyFromEnvironment,
        DialContext: (&net.Dialer{
            Timeout:   10 * time.Second,
            KeepAlive: 30 * time.Second,
        }).DialContext,
        TLSHandshakeTimeout:   10 * time.Second,
        ResponseHeaderTimeout: 20 * time.Second,
        ExpectContinueTimeout: 1 * time.Second,
        MaxIdleConns:          config.MaxIdleConns,
        MaxIdleConnsPerHost:   10,
        MaxConnsPerHost:       50,
        IdleConnTimeout:       90 * time.Second,
        TLSClientConfig:       tlsConfig,
    }
    
    // Override proxy if specified
    if config.ProxyURL != "" {
        proxyURL, err := url.Parse(config.ProxyURL)
        if err != nil {
            return nil, fmt.Errorf("invalid proxy URL: %w", err)
        }
        transport.Proxy = http.ProxyURL(proxyURL)
    }
    
    // Create HTTP client
    httpClient := &http.Client{
        Transport: transport,
        Timeout:   config.Timeout,
    }
    
    return &Client{
        httpClient: httpClient,
        apiKey:     config.APIKey,
        baseURL:    config.BaseURL,
    }, nil
}

Usage Example

// Provider configuration
func (p *Provider) Configure(ctx context.Context, req provider.ConfigureRequest, resp *provider.ConfigureResponse) {
    var config ProviderModel
    resp.Diagnostics.Append(req.Config.Get(ctx, &config)...)
    
    clientConfig := &client.ClientConfig{
        APIKey:        config.APIKey.ValueString(),
        BaseURL:       config.Endpoint.ValueString(),
        Timeout:       time.Duration(config.Timeout.ValueInt64()) * time.Second,
        MaxRetries:    int(config.MaxRetries.ValueInt64()),
        SkipTLSVerify: config.SkipTLSVerify.ValueBool(),
    }
    
    client, err := client.NewClient(clientConfig)
    if err != nil {
        resp.Diagnostics.AddError("Failed to create client", err.Error())
        return
    }
    
    resp.DataSourceData = client
    resp.ResourceData = client
}

Testing HTTP Configuration

Test with Custom Transport

func TestClient_GetDatabase(t *testing.T) {
    // Create mock server
    server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
        w.WriteHeader(http.StatusOK)
        json.NewEncoder(w).Encode(Database{
            ID:   "db-123",
            Name: "test-db",
        })
    }))
    defer server.Close()
    
    // Create client with test server
    client, _ := NewClient(&ClientConfig{
        APIKey:  "test-key",
        BaseURL: server.URL,
        Timeout: 5 * time.Second,
    })
    
    db, err := client.GetDatabase(context.Background(), "db-123")
    
    assert.NoError(t, err)
    assert.Equal(t, "db-123", db.ID)
}

Best Practices

1. Configure Timeouts

// ✅ Good: Reasonable timeouts
httpClient := &http.Client{
    Timeout: 60 * time.Second,
    Transport: &http.Transport{
        DialContext: (&net.Dialer{
            Timeout: 10 * time.Second,
        }).DialContext,
    },
}

// ❌ Bad: No timeout
httpClient := &http.Client{}
// Can hang forever

2. Reuse Connections

// ✅ Good: Connection pooling
transport := &http.Transport{
    MaxIdleConns:        100,
    MaxIdleConnsPerHost: 10,
}

// ❌ Bad: No pooling
transport := &http.Transport{
    DisableKeepAlives: true,  // New connection every time
}

3. Secure TLS

// ✅ Good: Modern TLS
tlsConfig := &tls.Config{
    MinVersion: tls.VersionTLS12,
}

// ❌ Bad: Insecure
tlsConfig := &tls.Config{
    InsecureSkipVerify: true,  // Never in production!
}

Key Takeaways

✅ Always configure timeouts
✅ Use connection pooling
✅ Enforce minimum TLS version
✅ Support custom CA certificates
✅ Respect proxy environment variables
✅ Chain transports for middleware
✅ Test with mock servers
✅ Log HTTP operations
✅ Never skip TLS verification in production
✅ Configure operation-specific timeouts with context

What’s Next?

Learn how to implement authentication patterns including API keys, OAuth, and JWT.

Next: Authentication Patterns →

Previous: ← API Client Architecture

Last updated: July 2, 2026

Share on

Twitter Facebook LinkedIn