Introduction to HTTP in Python3
What do TCP and UDP have anything to do with it all?
In this article we will dive into the world of web protocols, specifically HTTP, and explore how it relates to TCP and UDP in Python. It is a Python version of my last article, that introduced the same topics for Node.js. If this is more what you are after feel free to check it out here.
Starting with HTTP
HTTP (Hypertext Transfer Protocol) is the foundation of the web world. When you visit a website, your browser sends HTTP requests to a server, which then responds with web pages. It’s like a conversation between your browser and the server.
For instance, if you send a request that says “Joe” the server might reply with “Hi there, Joe”.
Below there is a basic example of how such HTTP server would work in Python3. We will use the http.client and the http.server built-in modules for that.
It’s a simple example so it doesn’t strictly follows HTTP standards and as the http.server module states, it is not recommended for production usage as it only implements basic security checks, but that’s fine for the time being.
import http.server
import http.client
PORT = 8001
class Store:
def __init__(self):
self.requestBody = ''
self.responseBody = ''
store = Store()
class MyHTTPRequestHandler(http.server.BaseHTTPRequestHandler):
def do_POST(self):
content_length = int(self.headers['Content-Length'])
content = self.rfile.read(content_length).decode('utf-8')
store.requestBody = content
response_content = f'Hi there, {content}'.encode('utf-8')
self.send_response(200)
self.send_header('Content-Type', 'text/plain')
self.send_header('Content-Length', len(response_content))
self.end_headers()
self.wfile.write(response_content)
def server_listen():
with http.server.HTTPServer(('localhost', PORT), MyHTTPRequestHandler) as server:
print(f'HTTP server listening on {PORT}')
http_request()
def http_request():
conn = http.client.HTTPConnection('localhost', PORT)
content = 'Joe'
headers = {
'Content-Type': 'text/plain',
'Content-Length': str(len(content))
}
conn.request('POST', '/greet', body=content, headers=headers)
response = conn.getresponse()
data = response.read().decode('utf-8')
store.responseBody = data
close_connections()
def close_connections():
server.server_close()
print(store.requestBody) # Joe
print(store.responseBody) # Hi there, Joe
server_listen()The TCP Connection
Now, let’s introduce TCP (Transmission Control Protocol). TCP is the underlying protocol that HTTP is built upon, as can be seen in the HTTP official specs.
Despite the fact we now know that, I’m going to ask you to pretend you don’t know that yet. Let’s prove it! Science, right? :)
In Python, we have a built-in modules called threading and socket that help us create TCP clients and servers.
It is worth knowing that TCP differs from HTTP by several aspects:
- Requests can’t be sent spontaneously. A connection must be established first.
- Once connected, messages can flow in both directions.
- An established connection must be manually closed.
Below is a simple implementation of a TCP client that wishes to be greeted by a server:
import socket
import threading
PORT = 8001
MAXIMUM_BYTES_RECEIVABLE = 1024
class Store:
def __init__(self):
self.requestBody = ''
self.responseBody = ''
store = Store()
def handle_client(client_socket):
request_data = client_socket.recv(MAXIMUM_BYTES_RECEIVABLE).decode('utf-8')
store.requestBody = request_data
response_data = f'Hi there, {request_data}'.encode('utf-8')
client_socket.send(response_data)
response = client_socket.recv(MAXIMUM_BYTES_RECEIVABLE).decode('utf-8')
store.responseBody = response
client_socket.close()
def server_listen():
server = socket.socket(socket.AF_INET, socket.SOCK_STREAM) # When the socket type is socket.SOCK_STREAM the protocol being used is TCP by default.
server.bind(('0.0.0.0', PORT))
server.listen(5)
print(f'TCP server listening on {PORT}')
while True:
client_socket, addr = server.accept() # Blocks execution and waits for an incoming connection.
client_handler = threading.Thread(target=handle_client, args=(client_socket,))
client_handler.start()
def http_request():
client = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
client.connect(('localhost', PORT))
content = 'Joe'
client.send(content.encode('utf-8'))
client.shutdown(socket.SHUT_WR)
response = client.recv(MAXIMUM_BYTES_RECEIVABLE).decode('utf-8')
store.responseBody = response
client.close()
close_connections()
def close_connections():
server.close()
print(store.requestBody) # Joe
print(store.responseBody) # Hi there, Joe
if __name__ == '__main__':
server_listen()
http_request()Now, imagine having a TCP proxy that can pass messages between HTTP clients and servers. Even though this proxy doesn’t understand HTTP, it can still transmit requests and responses.
Here’s how the implementation for this will look like:
import socket
import http.client
import threading
HTTP_PORT = 8001
PROXY_TCP_PORT = 8002
MAXIMUM_BYTES_RECEIVABLE = 1024
class Store:
def __init__(self):
self.requestBody = ''
self.responseBody = ''
store = Store()
def proxy_handler(local_socket):
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as remote_socket:
remote_socket.connect(('localhost', HTTP_PORT))
def forward(src, dst):
while True:
data = src.recv(MAXIMUM_BYTES_RECEIVABLE)
if not data:
break
dst.send(data)
threading.Thread(target=forward, args=(local_socket, remote_socket)).start()
threading.Thread(target=forward, args=(remote_socket, local_socket)).start()
def http_server_handler(client_socket):
data = client_socket.recv(MAXIMUM_BYTES_RECEIVABLE).decode('utf-8')
store.requestBody = data
response_data = f'Hi there, {data}'.encode('utf-8')
client_socket.send(response_data)
response = client_socket.recv(MAXIMUM_BYTES_RECEIVABLE).decode('utf-8')
store.responseBody = response
def http_server_listen():
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as server:
server.bind(('0.0.0.0', HTTP_PORT)
server.listen(5)
print(f'HTTP server listening on {HTTP_PORT}')
while True:
client_socket, addr = server.accept()
threading.Thread(target=http_server_handler, args=(client_socket,)).start()
def proxy_listen():
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as proxy_server:
proxy_server.bind(('0.0.0.0', PROXY_TCP_PORT))
proxy_server.listen(5)
print(f'TCP proxy listening on {PROXY_TCP_PORT}')
while True:
local_socket, addr = proxy_server.accept()
threading.Thread(target=proxy_handler, args=(local_socket,)).start()
def http_request():
conn = http.client.HTTPConnection('localhost', PROXY_TCP_PORT)
content = 'Joe'
headers = {
'Content-Type': 'text/plain',
'Content-Length': str(len(content))
}
conn.request('POST', '/greet', body=content, headers=headers)
response = conn.getresponse()
data = response.read().decode('utf-8')
close_connections()
def close_connections():
http_server_listen_thread.join()
proxy_listen_thread.join()
print(store.requestBody) # Joe
print(store.responseBody) # Hi there, Joe
if __name__ == '__main__':
http_server_listen_thread = threading.Thread(target=http_server_listen)
proxy_listen_thread = threading.Thread(target=proxy_listen)
http_server_listen_thread.start()
http_server_listen_thread.join()
proxy_listen_thread.start()
http_request()As mentioned before, it can be seen that although the TCP proxy server does not know what HTTP is the requests and responses completely go through.
Understanding TCP’s Characteristics
Before we go any further, a couple of facts on TCP:
- It’s reliable, meaning it ensures that messages are acknowledged, retransmitted if needed, and timed out if necessary.
- It guarantees that data arrives in the correct order.
This is quite amazing. No wonder TCP is so common.
However….
You knew it’s coming, right?
TCP can be a bit heavy — it requires setting up three packets for a socket connection to be able to allow data sending.
In the world of HTTP, this means that parallel HTTP/1.1 requests require multiple TCP connections, which can be resource-intensive.
HTTP/2 tries to improve this by handling parallel requests over a single connection. However, it faces challenges when a single packet times out or arrives out of order, causing all requests to stall.
Now, imagine there’s an alternative to TCP that allows parallel HTTP messages without these collective consequences. Sounds good, right?
That alternative is called UDP (User Datagram Protocol).
The UDP Connection
Let’s begin with how UDP differs from TCP:
- There’s no concept of a connection; you send data, and you hope someone receives it.
- You can only send small chunks of data which do not necessarily represent a whole message (read more about MTU), and there are no built-in delimiters unless explicitly included.
- As a result, creating even a basic request/response mechanism is more complex (but still possible).
Let’s dive in into an example of a UDP client that wants to communicate with a server.
This time, we will define our socket to be of a SOCK_DGRAM type:
import socket
PORT = 8001
EOS = b'\0' # End of stream
MAXIMUM_BYTES_RECEIVABLE = 1024
class Store:
def __init__(self):
self.requestBody = ''
self.responseBody = ''
store = Store()
def slice_but_last(data, encoding='utf-8'):
return data[:-1].decode(encoding)
def server_listen():
sender = None
with socket.socket(socket.AF_INET, socket.SOCK_DGRAM) as server:
server.bind(('0.0.0.0', PORT))
print(f'UDP server listening on {PORT}')
while True:
chunk, addr = server.recvfrom(MAXIMUM_BYTES_RECEIVABLE)
sender = addr if sender is None else sender
store.requestBody += slice_but_last(chunk)
if chunk[-1:] == EOS:
response_data = f'Hi there, {store.requestBody}'.encode('utf-8') + EOS
server.sendto(response_data, sender)
# Note: You can choose to close the server here if needed
break
def http_request():
with socket.socket(socket.AF_INET, socket.SOCK_DGRAM) as client:
content = 'Joe'.encode('utf-8') + EOS
client.sendto(content, ('localhost', PORT))
response_data, _ = client.recvfrom(MAXIMUM_BYTES_RECEIVABLE)
store.responseBody = slice_but_last(response_data)
close_connections()
def close_connections():
print(store.requestBody) # Joe
print(store.responseBody) # Hi there, Joe
if __name__ == '__main__':
server_listen()
http_request()So given that we have an HTTP parser (http-parser, for example), this is how an HTTP solution can be implemented over UDP:
import socket
from http_parser.parser import HttpParser
PORT = 8001
CRLF = '\r\n'
MAXIMUM_BYTES_RECEIVABLE = 1024
class Store:
def __init__(self):
self.requestBody = ''
self.responseBody = ''
store = Store()
def server_listen():
parser = HttpParser()
def on_body(data):
store.requestBody += data
def on_message_complete():
content = f'Hi there, {store.requestBody}'
response = f'HTTP/1.1 200 OK{CRLF}' \
f'Content-Type: text/plain{CRLF}' \
f'Content-Length: {len(content)}{CRLF}' \
f'{CRLF}' \
f'{content}'
server.sendto(response.encode('utf-8'), sender)
parser.on_body = on_body
parser.on_message_complete = on_message_complete
sender = None
with socket.socket(socket.AF_INET, socket.SOCK_DGRAM) as server:
server.bind(('0.0.0.0', PORT))
print(f'UDP server listening on {PORT}')
while True:
chunk, sender = server.recvfrom(MAXIMUM_BYTES_RECEIVABLE)
parser.execute(chunk)
def http_request():
parser = HttpParser()
def on_body(data):
store.responseBody += data
def on_message_complete():
close_connections()
parser.on_body = on_body
parser.on_message_complete = on_message_complete
content = 'Joe'
request = f'POST /greet HTTP/1.1{CRLF}' \
f'Content-Type: text/plain{CRLF}' \
f'Content-Length: {len(content)}{CRLF}' \
f'{CRLF}' \
f'{content}'
client.sendto(request.encode('utf-8'), ('localhost', PORT))
def close_connections():
server.close()
client.close()
print(store.requestBody) # Joe
print(store.responseBody) # Hi there, Joe
if __name__ == '__main__':
client = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
server_listen()
http_request()Okay… looks good. We have a full implementation using UDP.
But we shouldn’t get too excited about UDP just yet. It has some significant drawbacks:
- It’s unreliable, meaning you can’t be sure if your message will reach its destination.
- Packets don’t arrive in order, so you can’t guarantee the order of messages.
The Rise of QUIC and HTTP/3
To address UDP’s limitations, a new protocol called QUIC was created. QUIC is built on top of UDP and uses clever algorithms to make it:
- Reliable
- Ensure ordered packet delivery
- Keep things lightweight
This leads us straight into HTTP/3, which is still relatively new and experimental. It uses QUIC to fix the issues presented by HTTP/2. In HTTP/3, there are no connections, so sessions aren’t affected by each other.
HTTP/3 is a promising advancement in web protocols, taking advantage of QUIC and UDP’s strengths.
While there is no built-in support for the QUIC protocol, one can use the aioquic module that supports both QUIC and HTTP/3 implementations.
Example using the QUIC protocol
See this simple example for a server using QUIC:
import asyncio
import ssl
from aioquic.asyncio import connect, connect_udp, Connection, serve
from aioquic.asyncio.protocol import BaseProtocol, DatagramProtocol
from aioquic.asyncio.protocol.stream import DataReceived
class HTTPServerProtocol(BaseProtocol):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
async def data_received(self, data):
await super().data_received(data)
if isinstance(self._quic, Connection):
for stream_id, buffer in self._quic._events[DataReceived]:
data = buffer.read()
response = f'HTTP/1.1 200 OK\r\nContent-Length: {len(data)}\r\n\r\n{data.decode("utf-8")}'
self._quic.send_stream_data(stream_id, response.encode('utf-8'))
async def main():
loop = asyncio.get_event_loop()
# Create QUIC server context
quic_server = await loop.create_server(HTTPServerProtocol, 'localhost', 8001)
async with quic_server:
await quic_server.serve_forever()
if __name__ == '__main__':
loop = asyncio.get_event_loop()
loop.run_until_complete(main())And this is the client:
import asyncio
import ssl
from aioquic.asyncio import connect, connect_udp, Connection
from aioquic.asyncio.protocol import BaseProtocol
class HTTPClientProtocol(BaseProtocol):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.connected_event = asyncio.Event()
def quic_event_received(self, event):
super().quic_event_received(event)
if event.matches('connected'):
self.connected_event.set()
async def request(self, path, data=None):
stream_id = self._quic.get_next_available_stream_id()
self._quic.send_stream_data(stream_id, data)
await self.connected_event.wait()
response = await self._quic.receive_data(stream_id)
return response
async def main():
loop = asyncio.get_event_loop()
# Create QUIC client context
quic = connect('localhost', 8001)
async with quic as protocol:
client_protocol = HTTPClientProtocol(quic, protocol._session_id, None)
await client_protocol.connected_event.wait()
data = 'Hello, Joe!'
response = await client_protocol.request('/greet', data.encode('utf-8'))
print(response.decode('utf-8'))
if __name__ == '__main__':
loop = asyncio.get_event_loop()
loop.run_until_complete(main())Example using the HTTP/3 protocol
And just to have a complete picture of all examples, here is an example using the HTTP/3 protocol (with the help of the aioquic module).
The server:
import asyncio
from aioquic.asyncio.protocol import connect, connect_udp, serve, QuicProtocol
from aioquic.asyncio.protocol.stream import DataReceived
from h11 import Response, Connection
from h11._events import Data
class HTTP3ServerProtocol(QuicProtocol):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.conn = Connection()
def quic_event_received(self, event):
super().quic_event_received(event)
if event.matches('handshake_completed'):
self.conn.initiate_upgrade_for_http2()
async def data_received(self, data):
await super().data_received(data)
if isinstance(self._quic, QuicProtocol):
for stream_id, buffer in self._quic._events[DataReceived]:
data = buffer.read()
response = Response(status_code=200, headers=[('content-length', str(len(data)))], content=data)
data = self.conn.send(response)
self._quic.transmit_data(stream_id, data)
async def main():
loop = asyncio.get_event_loop()
# Create QUIC server context
quic_server = await loop.create_server(HTTP3ServerProtocol, 'localhost', 8001)
async with quic_server:
await quic_server.serve_forever()
if __name__ == '__main__':
loop = asyncio.get_event_loop()
loop.run_until_complete(main())And the client:
import asyncio
from aioquic.asyncio.protocol import connect, connect_udp, QuicProtocol
from h11 import Request, Response, Connection
from h11._events import Data
class HTTP3ClientProtocol(QuicProtocol):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.conn = Connection()
async def request(self, path, data=None):
stream_id = self._quic.get_next_available_stream_id()
request = Request(method='POST', target=path, headers=[('content-length', str(len(data)))]
if data else [])
data = self.conn.send(request)
self._quic.transmit_data(stream_id, data)
while True:
event = self.conn.next_event()
if isinstance(event, Data):
self._quic.transmit_data(stream_id, event.data)
elif event == h11.EndOfMessage():
break
response = await self._quic.receive_data(stream_id)
return response
async def main():
loop = asyncio.get_event_loop()
# Create QUIC client context
quic = connect('localhost', 8001)
async with quic as protocol:
client_protocol = HTTP3ClientProtocol(quic, protocol._session_id, None)
data = 'Hello, Joe!'
response = await client_protocol.request('/greet', data.encode('utf-8'))
print(response.content.decode('utf-8'))
if __name__ == '__main__':
loop = asyncio.get_event_loop()
loop.run_until_complete(main())Final words
This evolution raises the question of whether we might say goodbye to TCP in the future, but that’s a topic for another article and maybe for the future to say.
And that’s a wrap on our journey through HTTP, TCP, and UDP in Python3! It might seem complex, but it’s a fascinating world of web communication beneath the surface of every website you visit.
