Eavesdrop on a Golang http.Client

The returned representation is an approximation only; some details of the initial request are lost while parsing it into an http.Request. In particular, the order and case of header field names are lost.

GET Request

t := &http.Transport{
ResponseHeaderTimeout: time.Second * 10 // don't wait forever
}

timeoutClient := &http.Client{
Transport: t,
}

resp, _ := timeoutClient.Get("http://example.com/")
// do something with resp

The Wrapper

// spyConnection wraps a net.Conn, all reads and writes are output to stderr, via WrapConnection().
type spyConnection struct {
net.Conn
io.Reader
io.Writer
}

// Read writes all data read from the underlying connection to sc.Writer.
func (sc *spyConnection) Read(b []byte) (int, error) {
return sc.Reader.Read(b)
}

// Write writes all data written to the underlying connection to sc.Writer.
func (sc *spyConnection) Write(b []byte) (int, error) {
return sc.Writer.Write(b)
}
// WrapConnection wraps an existing connection, all data read/written is written to w (os.Stderr if w == nil).
func WrapConnection(c net.Conn, output io.Writer) net.Conn {
if output == nil {
output = os.Stderr
}
return &spyConnection{
Conn: c,
Reader: io.TeeReader(c, output),
Writer: io.MultiWriter(output, c),
}
}

Dial

dialer := &net.Dialer{
Timeout: 30 * time.Second,
KeepAlive: 30 * time.Second,
}

dial := func(network, address string) (net.Conn, error) {
conn, err := dialer.Dial(network, address)
if err != nil {
return nil, err
}
return WrapConnection(conn, os.Stderr}, nil // return a wrapped net.Conn
}
t := &http.Transport{
ResponseHeaderTimeout: 10 * time.Second,
DisableCompression: true, // humans can't read a compressed response
Dial: dial, // use our new dial in the existing transport
}

TLS

dialTLS := func(network, address string) (net.Conn, error) {
plainConn, err := dialer.Dial(network, address)
if err != nil {
return nil, err
}

//Initiate TLS and check remote host name against certificate.
cfg := new(tls.Config)

// add https:// to satisfy url.Parse(), we won't use it
u, err := url.Parse(fmt.Sprintf("https://%s", address))
if err != nil {
return nil, err
}

serverName := u.Host[:strings.LastIndex(u.Host, ":")]
cfg.ServerName = serverName

tlsConn := tls.Client(plainConn, cfg)

errc := make(chan error, 2)
timer := time.AfterFunc(time.Second, func() {
errc <- errors.New("TLS handshake timeout")
})
go func() {
err := tlsConn.Handshake()
timer.Stop()
errc <- err
}()
if err := <-errc; err != nil {
plainConn.Close()
return nil, err
}
if !cfg.InsecureSkipVerify {
if err := tlsConn.VerifyHostname(cfg.ServerName); err != nil {
plainConn.Close()
return nil, err
}
}

return WrapConnection(tlsConn, os.Stderr, nil // wrap the resulting conn
}

Working code example

package main

import (
"crypto/tls"
"errors"
"fmt"
"io"
"io/ioutil"
"net"
"net/http"
"net/url"
"os"
"strings"
"time"
)

// WrapConnection wraps an existing connection, all data read/written is written to w (os.Stderr if w == nil).
func WrapConnection(c net.Conn, output io.Writer) net.Conn {
return &spyConnection{
Conn: c,
Reader: io.TeeReader(c, output),
Writer: io.MultiWriter(output, c),
}
}

// spyConnection wraps a net.Conn, all reads and writes are output to stderr, via WrapConnection().
type spyConnection struct {
net.Conn
io.Reader
io.Writer
}

// Read writes all data read from the underlying connection to sc.Writer.
func (sc *spyConnection) Read(b []byte) (int, error) {
return sc.Reader.Read(b)
}

// Write writes all data written to the underlying connection to sc.Writer.
func (sc *spyConnection) Write(b []byte) (int, error) {
return sc.Writer.Write(b)
}
func main() {
dialer := &net.Dialer{
Timeout: 30 * time.Second,
KeepAlive: 30 * time.Second,
}

dial := func(network, address string) (net.Conn, error) {
conn, err := dialer.Dial(network, address)
if err != nil {
return nil, err
}

fmt.Fprint(os.Stderr, fmt.Sprintf("\n%s\n\n", strings.Repeat("-", 80)))
return WrapConnection(conn, os.Stderr), nil // return a wrapped net.Conn
}

dialTLS := func(network, address string) (net.Conn, error) {
plainConn, err := dialer.Dial(network, address)
if err != nil {
return nil, err
}

//Initiate TLS and check remote host name against certificate.
cfg := new(tls.Config)

// add https:// to satisfy url.Parse(), we won't use it
u, err := url.Parse(fmt.Sprintf("https://%s", address))
if err != nil {
return nil, err
}

serverName := u.Host[:strings.LastIndex(u.Host, ":")]
cfg.ServerName = serverName

tlsConn := tls.Client(plainConn, cfg)

errc := make(chan error, 2)
timer := time.AfterFunc(time.Second, func() {
errc <- errors.New("TLS handshake timeout")
})
go func() {
err := tlsConn.Handshake()
timer.Stop()
errc <- err
}()
if err := <-errc; err != nil {
plainConn.Close()
return nil, err
}
if !cfg.InsecureSkipVerify {
if err := tlsConn.VerifyHostname(cfg.ServerName); err != nil {
plainConn.Close()
return nil, err
}
}

fmt.Fprint(os.Stderr, fmt.Sprintf("\n%s\n\n", strings.Repeat("-", 80)))
return WrapConnection(tlsConn, os.Stderr), nil // return a wrapped net.Conn
}

t := &http.Transport{
Dial: dial,
DialTLS: dialTLS,
DisableCompression: true, // humans can't read a compressed response
TLSHandshakeTimeout: 10 * time.Second,
ResponseHeaderTimeout: 10 * time.Second,
ExpectContinueTimeout: 1 * time.Second,
}

timeoutClient := &http.Client{
Transport: t,
}

// http
resp, _ := timeoutClient.Get("http://example.com/")
// reads are incomplete unless we force the body to be read
ioutil.ReadAll(resp.Body)
resp.Body.Close()

// https
resp, _ = timeoutClient.Get("https://example.com/")
ioutil.ReadAll(resp.Body)
resp.Body.Close()
time.Sleep(time.Second)
}

Package

Beyond http

Servers too

--

--

--

Love podcasts or audiobooks? Learn on the go with our new app.

Recommended from Medium

Does DevOps have a future?

Life is a journey of twists and turns, peaks and valleys, mountains to climb and oceans to explore.

Top 4 Reasons Why Online Python Course Is Amazing For Your Kid

How to use Geolocation, Geocoding and Reverse Geocoding in Ionic 4

How We Should Be Moving Forward

Business Process Management and Service Oriented Architecture

Getting Started With Adobe I/O Runtime (Project Firefly)

What are MTTx Metrics Good For? Let’s Find Out.

Get the Medium app

A button that says 'Download on the App Store', and if clicked it will lead you to the iOS App store
A button that says 'Get it on, Google Play', and if clicked it will lead you to the Google Play store
Peter Flood

Peter Flood

More from Medium

SOLID Principles of Object-Oriented Design in GoLang (Part-3)

[Crazy Go Day] Simple Golang Unit Test Implementation

TicTacGo — An Intro to Golang

Go(Golang) bood nabood…