Create a Video Chat/Video Steaming App using Python

Due to the pandemic the only way to stay connected through the internet. But due to such a huge activity in Advertisement department, data leak and data privacy is a big issue. To overcome the data privacy issues, let us create our own video chat app in order to stay connected with our dear ones and avoid tracking by any of the advertising companies. Previously we created a text chat app now let’s take a step forward and create a video chat app.

Refer to my chat app before proceeding: Creating a Chat App in Python using UDP | by Gursimar Singh | Python in Plain English

Let us look at the code,

We will be using CV2 module of OpenCV library for capturing the video.

Learn more about CV2: Image Processing using OpenCV in Python | by Gursimar Singh | Jun, 2021 | Python in Plain English

It is recommended to create a separate environment to install the required libraries so that nothing interferes with the default environment in case of any errors

Server.py

from pyfiglet import Figlet
os.system("clear")
pyf = Figlet(font='puffy')
a = pyf.renderText("Video Chat App without Multi-Threading")
b = pyf.renderText("Server")
os.system("tput setaf 3")
print(a)
import socket, cv2, pickle,struct
# Socket Create
server_socket = socket.socket(socket.AF_INET,socket.SOCK_STREAM)
host_name  = socket.gethostname()
host_ip = socket.gethostbyname(host_name)
print('HOST IP:',host_ip)
port = 9999
socket_address = (host_ip,port)
# Socket Bind
server_socket.bind(socket_address)
# Socket Listen
server_socket.listen(1)
print("Listening at:",socket_address)
# Socket Accept
while True:
 client_socket,addr = server_socket.accept()
 print('Connected to:',addr)
 if client_socket:
  vid = cv2.VideoCapture(0)
  
  while(vid.isOpened()):
   ret,image = vid.read()
   img_serialize = pickle.dumps(image)
   message = struct.pack("Q",len(img_serialize))+img_serialize
   client_socket.sendall(message)
   
   cv2.imshow('Video from Server',image)
   key = cv2.waitKey(10) 
   if key ==13:
    client_socket.close()

Client.py

from pyfiglet import Figlet
os.system("clear")
pyf = Figlet(font='puffy')
a = pyf.renderText("Video Chat App without Multi-Threading")
b = pyf.renderText("Client")
os.system("tput setaf 3")
print(a)
import socket,cv2, pickle,struct
# create socket
client_socket = socket.socket(socket.AF_INET,socket.SOCK_STREAM)
#  server ip address here
host_ip = '<IP>' 
port = 9999
client_socket.connect((host_ip,port)) 
data = b""
metadata_size = struct.calcsize("Q")
while True:
 while len(data) < metadata_size:
  packet = client_socket.recv(4*1024) 
  if not packet: break
  data += packet
 packed_msg_size = data[:metadata_size]
 data = data[metadata_size:]
 msg_size = struct.unpack("Q",packed_msg_size)[0]
 
 while len(data) < msg_size:
  data += client_socket.recv(4*1024)
  frame_data = data[:msg_size]
  data  = data[msg_size:]
  frame = pickle.loads(frame_data)
  cv2.imshow("Receiving Video ",frame)
  key = cv2.waitKey(10) 
  if key  == 13:
   break
client_socket.close()

We run the server.py and after it we try to run client.py, pictures are transferred blazingly fast hence creating a video streaming.

Here we are first clicking picture and then converting it from array to bytes format using pickle module

#Converting this image to bytes format 
import pickle
photo_serialize=pickle.dumps(photo)

Now we need to provide the buffer size so that the client can receive the data. For this we use the struct module in Python. Here “Q” is the value we can pass in the arguments so that it provides 8 bytes.

We pack the data using the pack() function available in struct module, the first 8 bytes corresponds to the size of the file.

We use the unpack function to unpack the packed data.

import struct
size = struct.calcsize("Q")
message = struct.pack("Q",len(img_serialize))+img_serialize
msg_size = struct.unpack("Q",packed_msg_size)[0]

Instead, we can also use the flatten() and tostring() functions.

def recordVideo():
    time.sleep(5)    
    while True:
        ret, frame = cap.read()
        d = frame.flatten()
        video = d.tostring()
        c.sendall(video)
        time.sleep(0.2)
def rcvVideo():
     while True:
          data, addr = s.recvfrom(230400)
          frames = ""
          frames += data
          if len(frames) == (230400):
frame = numpy.fromstring (frames,dtype=numpy.uint8)
              frame = frame.reshape (240,320,3)
              cv2.imshow('frame',frame)
              frames=""
              cv2.waitKey(1)
          else:
              frames=""

Now, let us use multi-threading and create a smooth video chat app. We will be starting 4 threads in Client A and Client B each simultaneously(2 threads for video streaming and receiving and 2 audio streaming and listening). One thread will send the video and other will receive the video.

We will be using PyAudio library for audio transfer. Now, installing it can be tricky so install it using the following command only or else you might end up breaking your head and still wont be able to find a stable version.

$ conda install -c anaconda pyaudio

For this we need Anaconda distribution to be installed. Anaconda has an updated bug free version of PyAudio library.

Client A

import os
from pyfiglet import Figletos.system("clear")
pyf = Figlet(font='puffy')
a = pyf.renderText("UDP Chat App with Multi-Threading")
os.system("tput setaf 3")
print(a)
import socket, cv2, pickle, struct, threading, time
# Socket Create
s = socket.socket(socket.AF_INET,socket.SOCK_STREAM)
# Socket Accept
def sender():
 time.sleep(15)
 host_name  = socket.gethostname()
 host_ip = socket.gethostbyname(host_name)
 print('Host IP:',host_ip)
 port = 9999
 socket_address = (host_ip,port)
# Socket Bind
 s.bind(socket_address)
# Socket Listen
 s.listen(5)
 print("Listening at:",socket_address)
 while True:
  client_socket,addr = s.accept()
  print('Connection to:',addr)
  if client_socket:
   vid = cv2.VideoCapture(0)
  
   while(vid.isOpened()):
    ret,image = vid.read()
    img_serialize = pickle.dumps(image)
    message = struct.pack("Q",len(img_serialize))+img_serialize
    client_socket.sendall(message)
   
    cv2.imshow('Video from server', image)
    key = cv2.waitKey(10) 
    if key ==13:
     client_socket.close()
#Audio
chunk = 1024
FORMAT = pyaudio.paInt16
CHANNELS = 1
RATE = 44100
p = pyaudio.PyAudio()
stream = p.open(format = FORMAT,
                channels = CHANNELS,
                rate = RATE,
                input = True,
                frames_per_buffer = chunk)
#Audio Socket Initialization
audioSocket = socket.socket()
port1 = 5000
audioSocket.bind((<IP>,port1))
audioSocket.listen(5)
cAudio, addr = audioSocket.accept()
def recordAudio():
    time.sleep(5)
    while True:
        data = stream.read(chunk)
        if data:
            cAudio.sendall(data)
def rcvAudio():
     while True:
          audioData = audioSocket.recv(size)
          stream.write(audioData)
def connect_server():
 host_ip = '<IP>' 
 port = 1234
 s.connect((host_ip,port)) 
 data = b""
 metadata_size = struct.calcsize("Q")
 while True:
  while len(data) < metadata_size:
   packet = s.recv(4*1024) 
   if not packet: break
   data+=packet
  packed_msg_size = data[:metadata_size]
  data = data[metadata_size:]
  msg_size = struct.unpack("Q",packed_msg_size)[0]
 
  while len(data) < msg_size:
   data += s.recv(4*1024)
  frame_data = data[:msg_size]
  data  = data[msg_size:]
  frame = pickle.loads(frame_data)
  cv2.imshow("Receiving Video",frame)
  key = cv2.waitKey(10) 
  if key  == 13:
   break
 s.close()
x1 = threading.Thread(target = sender)
x2 = threading.Thread(target = connect_server)
x3 = threading.Thread(target = recordAudio)
x4 = threading.Thread(target = rcvAudio)
# start a thread
x1.start()
x2.start()
x3.start()
x4.start()

Client B

import os
from pyfiglet import Figletos.system("clear")
pyf = Figlet(font='puffy')
a = pyf.renderText("UDP Chat App with Multi-Threading")
os.system("tput setaf 3")
print(a)
import socket, cv2, pickle, struct, time, threading
# create socket
s = socket.socket(socket.AF_INET,socket.SOCK_STREAM)
def connect_server():
 time.sleep(15)
 host_ip = '192.168.99.1' 
 port = 9999
 s.connect((host_ip,port)) 
 data = b""
 metadata_size = struct.calcsize("Q")
 while True:
  while len(data) < metadata_size:
   packet = s.recv(4*1024) 
   if not packet: break
   data+=packet
  packed_msg_size = data[:metadata_size]
  data = data[metadata_size:]
  msg_size = struct.unpack("Q",packed_msg_size)[0]
 
  while len(data) < msg_size:
   data += s.recv(4*1024)
   frame_data = data[:msg_size]
   data  = data[msg_size:]
   frame = pickle.loads(frame_data)
   cv2.imshow("Receiving Video", frame)
   key = cv2.waitKey(10) 
   if key  == 13:
    break
 s.close()
def sender():
 host_name  = socket.gethostname()
 host_ip = socket.gethostbyname(host_name)
 print('Host IP:',host_ip)
 port = 1234
 socket_address = (host_ip,port)
 # Socket Bind
 s.bind(socket_address)
 # Socket Listen
 s.listen(5)
 print("Listening at:",socket_address)
 while True:
  client_socket,addr = s.accept()
  print('Connected to:',addr)
  if client_socket:
   vid = cv2.VideoCapture(1)
  
   while(vid.isOpened()):
    ret,image = vid.read()
    img_serialize = pickle.dumps(image)
    message = struct.pack("Q",len(img_serialize))+img_serialize
    client_socket.sendall(message)
   
    cv2.imshow('Video from server',image)
    key = cv2.waitKey(10) 
    if key ==13:
     client_socket.close()
#Audio
chunk = 1024
FORMAT = pyaudio.paInt16
CHANNELS = 1
RATE = 44100
p = pyaudio.PyAudio()
stream = p.open(format = FORMAT,
                channels = CHANNELS,
                rate = RATE,
                input = True,
                frames_per_buffer = chunk)
#Audio Socket Initialization
audioSocket = socket.socket()
port1 = 5000
audioSocket.bind((<IP>,port1))
audioSocket.listen(5)
cAudio, addr = audioSocket.accept()
def recordAudio():
    time.sleep(5)
    while True:
        data = stream.read(chunk)
        if data:
            cAudio.sendall(data)
def rcvAudio():
     while True:
          audioData = audioSocket.recv(size)
          stream.write(audioData)
x1 = threading.Thread(target = connect_server)
x2 = threading.Thread(target = sender)
x3 = threading.Thread(target = recordAudio)
x4 = threading.Thread(target = rcvAudio)
x1.start()
x2.start()
x3.start()
x4.start()

Here in PyAudio library, we need to specify the format, channels, rate, input and frames_per_buffer.

We create a separate socket for the audio so that it doesn’t intervene with the video stream.

Now we can run the ClientA.py and ClientB.py on different computers and have fun.