1.环境html
操做系统为 Ubuntu 16.04python
python 3.5
opencv-python 3.4.1.15
numpy 1.14.5
PyAudio 0.2.11服务器
2. 内容简介
本实验实现简易的视频通讯工具
在视频通讯的基础上加入语音
用户能够选择通讯的质量，即画质、停顿等参数
支持IPv6网络
3.安装环境app
$ sudo pip3 install numpy
$ sudo pip3 install opencv_python
这一步下载了咱们须要的opencv-python和numpy两个包。框架
剩下的PyAudio，因为本虚拟环境的部分问题，咱们单独分开下载。socket
$ sudo apt-get install portaudio19-dev python-all-dev python3-all-dev
$ sudo pip3 install pyaudio==0.2.11
如今，咱们的实验环境就搭好了。ide
4. 实验原理
实验实现了简易的视频通讯工具，基于 OpenCV 和 PyAudio，使用 TCP 协议通讯，通讯双方创建双向 CS 链接，
双方均维护一个客户端和一个服务器端。在捕获视频信息后，根据用户指定的参数对画面作压缩并传输。函数
实验步骤
接下来咱们分步骤讲解本实验。工具
4.1 实现双向 C/S 链接
先为双方的通讯设计 Server 类和 Client类，两个类均继承 threading.Thread，只须要分别实现 __init__、__del__和run方法，
以后对象调用.start()方法便可在独立线程中执行run方法中的内容。首先Client类须要存储远端的IP地址和端口，
而Server类须要存储本地服务器监听的端口号。用户还应当能够指定通讯双方使用的协议版本，即基于IPv4 仍是IPv6 的TCP链接。
所以Server类的初始化须要传入两个参数（端口、版本），Client类的初始化须要三个参数（远端IP、端口、版本）。新建文件vchat.py，
在其中定义基础的两个类以下。
1 from socket import *
2 import threading
3 class Video_Server(threading.Thread):
4 def __init__(self, port, version) :
5 threading.Thread.__init__(self)
6 self.setDaemon(True)
7 self.ADDR = ('', port)
8 if version == 4:
9 self.sock = socket(AF_INET ,SOCK_STREAM)
10 else:
11 self.sock = socket(AF_INET6 ,SOCK_STREAM)
12 def __del__(self):
13 self.sock.close()
14 # TODO
15 def run(self):
16 print("server starts...")
17 self.sock.bind(self.ADDR)
18 self.sock.listen(1)
19 conn, addr = self.sock.accept()
20 print("remote client success connected...")
21 # TODO
22
23 class Video_Client(threading.Thread):
24 def __init__(self ,ip, port, version):
25 threading.Thread.__init__(self)
26 self.setDaemon(True)
27 self.ADDR = (ip, port)
28 if version == 4:
29 self.sock = socket(AF_INET, SOCK_STREAM)
30 else:
31 self.sock = socket(AF_INET6, SOCK_STREAM)
32 def __del__(self) :
33 self.sock.close()
34 # TODO
35 def run(self):
36 print("client starts...")
37 while True:
38 try:
39 self.sock.connect(self.ADDR)
40 break
41 except:
42 time.sleep(3)
43 continue
44 print("client connected...")
45 # TODO
复制代码
4.2 实现摄像头数据流捕获
OpenCV 为 Python 提供的接口很是简单而且易于理解。捕获视频流的任务应当由Client类完成，
下面完善Client的run函数。在下面的代码中，咱们为类添加了一个成员变量cap，它用来捕获默认摄像头的输出。
复制代码
1 class Video_Client(threading.Thread):
2 def __init__(self ,ip, port, version):
3 threading.Thread.__init__(self)
4 self.setDaemon(True)
5 self.ADDR = (ip, port)
6 if version == 4:
7 self.sock = socket(AF_INET, SOCK_STREAM)
8 else:
9 self.sock = socket(AF_INET6, SOCK_STREAM)
10 self.cap = cv2.VideoCapture(0)
11 def __del__(self) :
12 self.sock.close()
13 self.cap.release()
14 def run(self):
15 print("client starts...")
16 while True:
17 try:
18 self.sock.connect(self.ADDR)
19 break
20 except:
21 time.sleep(3)
22 continue
23 print("client connected...")
24 while self.cap.isOpened():
25 ret, frame = self.cap.read()
26 # TODO
复制代码
4.3 发送捕获到的数据到服务器
已经捕获到数据，接下来要发送字节流。首先咱们继续编写Client，为其添加发送数据功能的实现。这里只改动了run方法。
在捕获到帧后，咱们使用pickle.dumps方法对其打包，并用sock.sendall方法发送。
注意发送过程当中咱们用struct.pack方法为每批数据加了一个头，用于接收方确认接受数据的长度。
复制代码
1 def run(self):
2 while True:
3 try:
4 self.sock.connect(self.ADDR)
5 break
6 except:
7 time.sleep(3)
8 continue
9 print("client connected...")
10 while self.cap.isOpened():
11 ret, frame = self.cap.read()
12 data = pickle.dumps(frame)
13 try:
14 self.sock.sendall(struct.pack("L", len(data)) + data)
15 except:
16 break
复制代码
下面编写Server，在服务器端链接成功后，应当建立一个窗口用于显示接收到的视频。由于链接不必定建立成功，
所以cv.destroyAllWindows()被放在一个try..catch块中防止出现错误。在接收数据过程当中，
咱们使用payload_size记录当前从缓冲区读入的数据长度，这个长度经过struct.calcsize('L')来读取。
使用该变量的意义在于缓冲区中读出的数据可能不足一个帧，也可能由多个帧构成。为了准确提取每一帧，咱们用payload_size区分帧的边界。
在从缓冲区读出的数据流长度超过payload_size时，剩余部分和下一次读出的数据流合并，
不足payload_size时将合并下一次读取的数据流到当前帧中。在接收完完整的一帧后，显示在建立的窗口中。同时咱们为窗口建立一个键盘响应，
当按下Esc 或 q键时退出程序。
复制代码
class Video_Server(threading.Thread):
def __init__(self, port, version) :
threading.Thread.__init__(self)
self.setDaemon(True)
self.ADDR = ('', port)
if version == 4:
self.sock = socket(AF_INET ,SOCK_STREAM)
else:
self.sock = socket(AF_INET6 ,SOCK_STREAM)
def __del__(self):
self.sock.close()
try:
cv2.destroyAllWindows()
except:
pass
def run(self):
print("server starts...")
self.sock.bind(self.ADDR)
self.sock.listen(1)
conn, addr = self.sock.accept()
print("remote client success connected...")
data = "".encode("utf-8")
payload_size = struct.calcsize("L")
cv2.namedWindow('Remote', cv2.WINDOW_NORMAL)
while True:
while len(data) < payload_size:
data += conn.recv(81920)
packed_size = data[:payload_size]
data = data[payload_size:]
msg_size = struct.unpack("L", packed_size)[0]
while len(data) < msg_size:
data += conn.recv(81920)
zframe_data = data[:msg_size]
data = data[msg_size:]
frame_data = zlib.decompress(zframe_data)
frame = pickle.loads(frame_data)
cv2.imshow('Remote', frame)
if cv2.waitKey(1) & 0xFF == 27:
break
复制代码
4.4 视频缩放和数据压缩
如今的服务器和客户端已经能够运行，你能够在代码中建立一个Client类实例和一个Server类实例，并将IP地址设为127.0.0.1，
端口设为任意合法的（0-65535）且不冲突的值，版本设为IPv4。执行代码等同于本身和本身通讯。若是网络情况很差，
你也许会发现本身和本身的通讯也有卡顿现象。为了使画面质量、延迟可以和现实网络情况相匹配，咱们须要容许用户指定通讯中画面的质量，
同时咱们的代码应当自己具备压缩数据的能力，以尽量利用带宽。
当用户指定使用低画质通讯，咱们应当对原始数据作变换，最简单的方式即将捕获的每一帧按比例缩放，同时下降传输的帧速，
在代码中体现为resize，该函数的第二个参数为缩放中心，后两个参数为缩放比例，而且根据用户指定的等级，再也不传输捕获的每一帧，
而是间隔几帧传输一帧。为了防止用户指定的画质过差，代码中限制了最坏状况下的缩放比例为0.3，最大帧间隔为3。此外，
咱们在发送每一帧的数据前使用zlib.compress对其压缩，尽可能下降带宽负担。
复制代码
1 class Video_Client(threading.Thread):
2 def __init__(self ,ip, port, level, version):
3 threading.Thread.__init__(self)
4 self.setDaemon(True)
5 self.ADDR = (ip, port)
6 if level <= 3:
7 self.interval = level
8 else:
9 self.interval = 3
10 self.fx = 1 / (self.interval + 1)
11 if self.fx < 0.3:
12 self.fx = 0.3
13 if version == 4:
14 self.sock = socket(AF_INET, SOCK_STREAM)
15 else:
16 self.sock = socket(AF_INET6, SOCK_STREAM)
17 self.cap = cv2.VideoCapture(0)
18 def __del__(self) :
19 self.sock.close()
20 self.cap.release()
21 def run(self):
22 print("VEDIO client starts...")
23 while True:
24 try:
25 self.sock.connect(self.ADDR)
26 break
27 except:
28 time.sleep(3)
29 continue
30 print("VEDIO client connected...")
31 while self.cap.isOpened():
32 ret, frame = self.cap.read()
33 sframe = cv2.resize(frame, (0,0), fx=self.fx, fy=self.fx)
34 data = pickle.dumps(sframe)
35 zdata = zlib.compress(data, zlib.Z_BEST_COMPRESSION)
36 try:
37 self.sock.sendall(struct.pack("L", len(zdata)) + zdata)
38 except:
39 break
40 for i in range(self.interval):
41 self.cap.read()
复制代码
服务器端最终代码以下，增长了对接收到数据的解压缩处理。
复制代码
1 class Video_Server(threading.Thread):
2 def __init__(self, port, version) :
3 threading.Thread.__init__(self)
4 self.setDaemon(True)
5 self.ADDR = ('', port)
6 if version == 4:
7 self.sock = socket(AF_INET ,SOCK_STREAM)
8 else:
9 self.sock = socket(AF_INET6 ,SOCK_STREAM)
10 def __del__(self):
11 self.sock.close()
12 try:
13 cv2.destroyAllWindows()
14 except:
15 pass
16 def run(self):
17 print("VEDIO server starts...")
18 self.sock.bind(self.ADDR)
19 self.sock.listen(1)
20 conn, addr = self.sock.accept()
21 print("remote VEDIO client success connected...")
22 data = "".encode("utf-8")
23 payload_size = struct.calcsize("L")
24 cv2.namedWindow('Remote', cv2.WINDOW_NORMAL)
25 while True:
26 while len(data) < payload_size:
27 data += conn.recv(81920)
28 packed_size = data[:payload_size]
29 data = data[payload_size:]
30 msg_size = struct.unpack("L", packed_size)[0]
31 while len(data) < msg_size:
32 data += conn.recv(81920)
33 zframe_data = data[:msg_size]
34 data = data[msg_size:]
35 frame_data = zlib.decompress(zframe_data)
36 frame = pickle.loads(frame_data)
37 cv2.imshow('Remote', frame)
38 if cv2.waitKey(1) & 0xFF == 27:
39 break
复制代码
4.5 加入音频的捕获和传输
在完成视频通讯的基础上，总体框架对于音频通讯能够直接挪用，只须要修改其中捕获视频/音频的代码和服务器解码播放的部分。
这里咱们使用 PyAudio 库处理音频，在 Linux 下你也能够选择 sounddevice。关于sounddevice这里不作过多介绍，
将vchat.py复制一份，重命名为achat.py，简单修改几处，最终音频捕获、传输的完整代码以下。
我将上面代码中的Server和Client分别加上Video和Audio前缀以区分，同时显示给用户的print输出语句也作了必定修改，
对于视频加上VIDEO前缀，音频加上AUDIO前缀。若是你对代码中使用到的 PyAudio 提供的库函数有所疑问，
复制代码
1 class Audio_Server(threading.Thread):
2 def __init__(self, port, version) :
3 threading.Thread.__init__(self)
4 self.setDaemon(True)
5 self.ADDR = ('', port)
6 if version == 4:
7 self.sock = socket(AF_INET ,SOCK_STREAM)
8 else:
9 self.sock = socket(AF_INET6 ,SOCK_STREAM)
10 self.p = pyaudio.PyAudio()
11 self.stream = None
12 def __del__(self):
13 self.sock.close()
14 if self.stream is not None:
15 self.stream.stop_stream()
16 self.stream.close()
17 self.p.terminate()
18 def run(self):
19 print("AUDIO server starts...")
20 self.sock.bind(self.ADDR)
21 self.sock.listen(1)
22 conn, addr = self.sock.accept()
23 print("remote AUDIO client success connected...")
24 data = "".encode("utf-8")
25 payload_size = struct.calcsize("L")
26 self.stream = self.p.open(format=FORMAT,
27 channels=CHANNELS,
28 rate=RATE,
29 output=True,
30 frames_per_buffer = CHUNK
31 )
32 while True:
33 while len(data) < payload_size:
34 data += conn.recv(81920)
35 packed_size = data[:payload_size]
36 data = data[payload_size:]
37 msg_size = struct.unpack("L", packed_size)[0]
38 while len(data) < msg_size:
39 data += conn.recv(81920)
40 frame_data = data[:msg_size]
41 data = data[msg_size:]
42 frames = pickle.loads(frame_data)
43 for frame in frames:
44 self.stream.write(frame, CHUNK)
45
46 class Audio_Client(threading.Thread):
47 def __init__(self ,ip, port, version):
48 threading.Thread.__init__(self)
49 self.setDaemon(True)
50 self.ADDR = (ip, port)
51 if version == 4:
52 self.sock = socket(AF_INET, SOCK_STREAM)
53 else:
54 self.sock = socket(AF_INET6, SOCK_STREAM)
55 self.p = pyaudio.PyAudio()
56 self.stream = None
57 def __del__(self) :
58 self.sock.close()
59 if self.stream is not None:
60 self.stream.stop_stream()
61 self.stream.close()
62 self.p.terminate()
63 def run(self):
64 print("AUDIO client starts...")
65 while True:
66 try:
67 self.sock.connect(self.ADDR)
68 break
69 except:
70 time.sleep(3)
71 continue
72 print("AUDIO client connected...")
73 self.stream = self.p.open(format=FORMAT,
74 channels=CHANNELS,
75 rate=RATE,
76 input=True,
77 frames_per_buffer=CHUNK)
78 while self.stream.is_active():
79 frames = []
80 for i in range(0, int(RATE / CHUNK * RECORD_SECONDS)):
81 data = self.stream.read(CHUNK)
82 frames.append(data)
83 senddata = pickle.dumps(frames)
84 try:
85 self.sock.sendall(struct.pack("L", len(senddata)) + senddata)
86 except:
87 break
复制代码
至此咱们完成了 vchat.py 的编写。
4.6 编写程序入口 main.py
为了提供用户参数解析，代码使用了argparse。你可能对此前几个类中初始化方法的self.setDaemon(True)有疑惑。
这个方法的调用使每一个线程在主线程结束以后自动退出，保证程序不会出现崩溃且没法销毁的状况。在main.py中，
咱们经过每隔1s作一次线程的保活检查，若是视频/音频中出现阻塞/故障，主线程会终止。
复制代码
1 import sys
2 import time
3 import argparse
4 from vchat import Video_Server, Video_Client
5 from achat import Audio_Server, Audio_Client
6
7 parser = argparse.ArgumentParser()
8
9 parser.add_argument('--host', type=str, default='127.0.0.1')
10 parser.add_argument('--port', type=int, default=10087)
11 parser.add_argument('--level', type=int, default=1)
12 parser.add_argument('-v', '--version', type=int, default=4)
13
14 args = parser.parse_args()
15
16 IP = args.host
17 PORT = args.port
18 VERSION = args.version
19 LEVEL = args.level
20
21 if __name__ == '__main__':
22 vclient = Video_Client(IP, PORT, LEVEL, VERSION)
23 vserver = Video_Server(PORT, VERSION)
24 aclient = Audio_Client(IP, PORT+1, VERSION)
25 aserver = Audio_Server(PORT+1, VERSION)
26 vclient.start()
27 aclient.start()
28 time.sleep(1) # make delay to start server
29 vserver.start()
30 aserver.start()
31 while True:
32 time.sleep(1)
33 if not vserver.isAlive() or not vclient.isAlive():
34 print("Video connection lost...")
35 sys.exit(0)
36 if not aserver.isAlive() or not aclient.isAlive():
37 print("Audio connection lost...")
38 sys.exit(0)
复制代码
最终能够实现和本身视频聊天
最终代码：三个文件
代码一：achat.py
1 from socket import *
2 import threading
3 import cv2
4 import re
5 import sys
6 import os
7 import time
8 import pyaudio
9 import struct
10 import pickle
11 import zlib
12 import wave
13
14
15 CHUNK = 1024
16 FORMAT = pyaudio.paInt16
17 CHANNELS = 2
18 RATE = 44100
19 RECORD_SECONDS = 5
20 WAVE_OUTPUT_FILENAME = "output.wav"
21
22
23
24 class Audio_Server(threading.Thread):
25 def __init__(self, port, version) :
26 threading.Thread.__init__(self)
27 self.setDaemon(True)
28 self.ADDR = ('', port)
29 if version == 4:
30 self.sock = socket(AF_INET ,SOCK_STREAM)
31 else:
32 self.sock = socket(AF_INET6 ,SOCK_STREAM)
33 self.p = pyaudio.PyAudio()
34 self.stream = None
35 def __del__(self):
36 self.sock.close()
37 if self.stream is not None:
38 self.stream.stop_stream()
39 self.stream.close()
40 self.p.terminate()
41
42 def run(self):
43 print("AUDIO server starts...")
44 self.sock.bind(self.ADDR)
45 self.sock.listen(1)
46 conn, addr = self.sock.accept()
47 print("remote AUDIO client success connected...")
48 data = "".encode("utf-8")
49 payload_size = struct.calcsize("L")
50 self.stream = self.p.open(format=FORMAT,
51 channels=CHANNELS,
52 rate=RATE,
53 output=True,
54 frames_per_buffer = CHUNK
55 )
56
57 while True:
58 while len(data) < payload_size:
59 data += conn.recv(81920)
60 packed_size = data[:payload_size]
61 data = data[payload_size:]
62 msg_size = struct.unpack("L", packed_size)[0]
63 while len(data) < msg_size:
64 data += conn.recv(81920)
65 frame_data = data[:msg_size]
66 data = data[msg_size:]
67 frames = pickle.loads(frame_data)
68 for frame in frames:
69 self.stream.write(frame, CHUNK)
70
71 class Audio_Client(threading.Thread):
72 def __init__(self ,ip, port, version):
73 threading.Thread.__init__(self)
74 self.setDaemon(True)
75 self.ADDR = (ip, port)
76 if version == 4:
77 self.sock = socket(AF_INET, SOCK_STREAM)
78 else:
79 self.sock = socket(AF_INET6, SOCK_STREAM)
80 self.p = pyaudio.PyAudio()
81 self.stream = None
82 def __del__(self) :
83 self.sock.close()
84 if self.stream is not None:
85 self.stream.stop_stream()
86 self.stream.close()
87 self.p.terminate()
88 def run(self):
89 print("AUDIO client starts...")
90 while True:
91 try:
92 self.sock.connect(self.ADDR)
93 break
94 except:
95 time.sleep(3)
96 continue
97 print("AUDIO client connected...")
98 self.stream = self.p.open(format=FORMAT,
99 channels=CHANNELS,
100 rate=RATE,
101 input=True,
102 frames_per_buffer=CHUNK)
103 while self.stream.is_active():
104 frames = []
105 for i in range(0, int(RATE / CHUNK * RECORD_SECONDS)):
106 data = self.stream.read(CHUNK)
107 frames.append(data)
108 senddata = pickle.dumps(frames)
109 try:
110 self.sock.sendall(struct.pack("L", len(senddata)) + senddata)
111 except:
112 break
代码二：vchat.py
from socket import *
import threading
import cv2
import re
import time
import sys
import os
import struct
import pickle
import zlib
import wave
class Video_Client(threading.Thread):
def __init__(self ,ip, port, level, version):
threading.Thread.__init__(self)
self.setDaemon(True)
self.ADDR = (ip, port)
if level <= 3:
self.interval = level
else:
self.interval = 3
self.fx = 1 / (self.interval + 1)
if self.fx < 0.3:
self.fx = 0.3
if version == 4:
self.sock = socket(AF_INET, SOCK_STREAM)
else:
self.sock = socket(AF_INET6, SOCK_STREAM)
self.cap = cv2.VideoCapture(0)
def __del__(self) :
self.sock.close()
self.cap.release()
def run(self):
print("VEDIO client starts...")
while True:
try:
self.sock.connect(self.ADDR)
break
except:
time.sleep(3)
continue
print("VEDIO client connected...")
while self.cap.isOpened():
ret, frame = self.cap.read()
sframe = cv2.resize(frame, (0,0), fx=self.fx, fy=self.fx)
data = pickle.dumps(sframe)
zdata = zlib.compress(data, zlib.Z_BEST_COMPRESSION)
try:
self.sock.sendall(struct.pack("L", len(zdata)) + zdata)
except:
break
for i in range(self.interval):
self.cap.read()
# 服务器端最终代码以下，增长了对接收到数据的解压缩处理。
class Video_Server(threading.Thread):
def __init__(self, port, version) :
threading.Thread.__init__(self)
self.setDaemon(True)
self.ADDR = ('', port)
if version == 4:
self.sock = socket(AF_INET ,SOCK_STREAM)
else:
self.sock = socket(AF_INET6 ,SOCK_STREAM)
def __del__(self):
self.sock.close()
try:
cv2.destroyAllWindows()
except:
pass
def run(self):
print("VEDIO server starts...")
self.sock.bind(self.ADDR)
self.sock.listen(1)
conn, addr = self.sock.accept()
print("remote VEDIO client success connected...")
data = "".encode("utf-8")
payload_size = struct.calcsize("L")
cv2.namedWindow('Remote', cv2.WINDOW_NORMAL)
while True:
while len(data) < payload_size:
data += conn.recv(81920)
packed_size = data[:payload_size]
data = data[payload_size:]
msg_size = struct.unpack("L", packed_size)[0]
while len(data) < msg_size:
data += conn.recv(81920)
zframe_data = data[:msg_size]
data = data[msg_size:]
frame_data = zlib.decompress(zframe_data)
frame = pickle.loads(frame_data)
cv2.imshow('Remote', frame)
if cv2.waitKey(1) & 0xFF == 27:
break
代码三：main.py
import sys
import time
import argparse
import cv2
import re
import pyaudio
import pickle
import os
import struct
import zlib
import wave
from vchat import Video_Server, Video_Client
from achat import Audio_Server, Audio_Client
parser = argparse.ArgumentParser()
parser.add_argument('--host', type=str, default='127.0.0.1')
parser.add_argument('--port', type=int, default=10087)
parser.add_argument('--level', type=int, default=1)
parser.add_argument('-v', '--version', type=int, default=4)
args = parser.parse_args()
IP = args.host
PORT = args.port
VERSION = args.version
LEVEL = args.level
if __name__ == '__main__':
vclient = Video_Client(IP, PORT, LEVEL, VERSION)
vserver = Video_Server(PORT, VERSION)
aclient = Audio_Client(IP, PORT+1, VERSION)
aserver = Audio_Server(PORT+1, VERSION)
vclient.start()
aclient.start()
time.sleep(1) # make delay to start server
vserver.start()
aserver.start()
while True:
time.sleep(1)
if not vserver.isAlive() or not vclient.isAlive():
print("Video connection lost...")
sys.exit(0)
if not aserver.isAlive() or not aclient.isAlive():
print("Audio connection lost...")
sys.exit(0)
新人学习python，多有不足，谢谢你们观看。