# 为何会出现黏包现象?
# 首先只有在TCP协议中才会出现黏包现象,
# 是由于TCP协议是面向流的协议
# 在发送的数据传输的过程当中还有缓存机制来避免数据丢失
# 所以 在连续发送小数据的时候 以及接收大小不符的时候都容易出现黏包现象
# 本质仍是由于咱们在接收数据的时候不知道发送的数据的长短
# 解决黏包问题
# 在传输大量数据以前先告诉接收端要发送的数据大小
# 若是想更漂亮的解决问题,能够经过struct模块来定制协议
html
1,解决方案一python
问题的根源在于,接收端不知道发送端将要传送的字节流的长度,因此解决粘包的方法就是围绕,如何让发送端在发送数据前,把本身将要发送的字节流总大小让接收端知晓,而后接收端来一个死循环接收完全部数据。算法
#_*_coding:utf-8_*_
import socket,subprocess ip_port=('127.0.0.1',8080) s=socket.socket(socket.AF_INET,socket.SOCK_STREAM) s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) s.bind(ip_port) s.listen(5) while True: conn,addr=s.accept() print('客户端',addr) while True: msg=conn.recv(1024) if not msg:break res=subprocess.Popen(msg.decode('utf-8'),shell=True,\ stdin=subprocess.PIPE,\ stderr=subprocess.PIPE,\ stdout=subprocess.PIPE) err=res.stderr.read() if err: ret=err else: ret=res.stdout.read() data_length=len(ret) conn.send(str(data_length).encode('utf-8')) data=conn.recv(1024).decode('utf-8') if data == 'recv_ready': conn.sendall(ret) conn.close()
#_*_coding:utf-8_*_
import socket,time s=socket.socket(socket.AF_INET,socket.SOCK_STREAM) res=s.connect_ex(('127.0.0.1',8080)) while True: msg=input('>>: ').strip() if len(msg) == 0:continue
if msg == 'quit':break s.send(msg.encode('utf-8')) length=int(s.recv(1024).decode('utf-8')) s.send('recv_ready'.encode('utf-8')) send_size=0 recv_size=0 data=b''
while recv_size < length: data+=s.recv(1024) recv_size+=len(data)
存在的问题:
程序的运行速度远快于网络传输速度,因此在发送一段字节前,先用send去发送该字节流长度,这种方式会放大网络延迟带来的性能损耗shell
2,解决方案进阶json
咱们能够借助一个模块,这个模块能够把要发送的数据长度转换成固定长度的字节。这样客户端每次接收消息以前只要先接受这个固定长度字节的内容看一看接下来要接收的信息大小,那么最终接受的数据只要达到这个值就中止,就能恰好很少很多的接收完整的数据了。windows
2.1,struct模块缓存
该模块能够把一个类型,如数字,转成固定长度的bytes服务器
>>> struct.pack('i',1111111111111)
struct.error: 'i' format requires -2147483648 <= number <= 2147483647 #这个是范围
import json,struct
#假设经过客户端上传1T:1073741824000的文件a.txt
#为避免粘包,必须自定制报头
header={'file_size':1073741824000,'file_name':'/a/b/c/d/e/a.txt','md5':'8f6fbf8347faa4924a76856701edb0f3'} #1T数据,文件路径和md5值
#为了该报头能传送,须要序列化而且转为bytes
head_bytes=bytes(json.dumps(header),encoding='utf-8') #序列化并转成bytes,用于传输
#为了让客户端知道报头的长度,用struck将报头长度这个数字转成固定长度:4个字节
head_len_bytes=struct.pack('i',len(head_bytes)) #这4个字节里只包含了一个数字,该数字是报头的长度
#客户端开始发送
conn.send(head_len_bytes) #先发报头的长度,4个bytes
conn.send(head_bytes) #再发报头的字节格式
conn.sendall(文件内容) #而后发真实内容的字节格式
#服务端开始接收
head_len_bytes=s.recv(4) #先收报头4个bytes,获得报头长度的字节格式
x=struct.unpack('i',head_len_bytes)[0] #提取报头的长度
head_bytes=s.recv(x) #按照报头长度x,收取报头的bytes格式
header=json.loads(json.dumps(header)) #提取报头
#最后根据报头的内容提取真实的数据,好比
real_data_len=s.recv(header['file_size'])
s.recv(real_data_len)
#_*_coding:utf-8_*_ #http://www.cnblogs.com/coser/archive/2011/12/17/2291160.html
__author__ = 'Linhaifeng'
import struct import binascii import ctypes values1 = (1, 'abc'.encode('utf-8'), 2.7) values2 = ('defg'.encode('utf-8'),101) s1 = struct.Struct('I3sf') s2 = struct.Struct('4sI') print(s1.size,s2.size) prebuffer=ctypes.create_string_buffer(s1.size+s2.size) print('Before : ',binascii.hexlify(prebuffer)) # t=binascii.hexlify('asdfaf'.encode('utf-8')) # print(t)
s1.pack_into(prebuffer,0,*values1) s2.pack_into(prebuffer,s1.size,*values2) print('After pack',binascii.hexlify(prebuffer)) print(s1.unpack_from(prebuffer,0)) print(s2.unpack_from(prebuffer,s1.size)) s3=struct.Struct('ii') s3.pack_into(prebuffer,0,123,123) print('After pack',binascii.hexlify(prebuffer)) print(s3.unpack_from(prebuffer,0))
2.2,使用struct解决黏包 网络
借助struct模块,咱们知道长度数字能够被转换成一个标准大小的4字节数字。所以能够利用这个特色来预先发送数据长度。app
发送时 | 接收时 |
先发送struct转换好的数据长度4字节 | 先接受4个字节使用struct转换成数字来获取要接收的数据长度 |
再发送数据 | 再按照长度接收数据 |
import socket,struct,json import subprocess phone=socket.socket(socket.AF_INET,socket.SOCK_STREAM) phone.setsockopt(socket.SOL_SOCKET,socket.SO_REUSEADDR,1) #就是它,在bind前加
phone.bind(('127.0.0.1',8080)) phone.listen(5) while True: conn,addr=phone.accept() while True: cmd=conn.recv(1024) if not cmd:break
print('cmd: %s' %cmd) res=subprocess.Popen(cmd.decode('utf-8'), shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) err=res.stderr.read() print(err) if err: back_msg=err else: back_msg=res.stdout.read() conn.send(struct.pack('i',len(back_msg))) #先发back_msg的长度
conn.sendall(back_msg) #在发真实的内容
conn.close()
#_*_coding:utf-8_*_
import socket,time,struct s=socket.socket(socket.AF_INET,socket.SOCK_STREAM) res=s.connect_ex(('127.0.0.1',8080)) while True: msg=input('>>: ').strip() if len(msg) == 0:continue
if msg == 'quit':break s.send(msg.encode('utf-8')) l=s.recv(4) x=struct.unpack('i',l)[0] print(type(x),x) # print(struct.unpack('I',l))
r_s=0 data=b''
while r_s < x: r_d=s.recv(1024) data+=r_d r_s+=len(r_d) # print(data.decode('utf-8'))
print(data.decode('gbk')) #windows默认gbk编码
发送时 | 接收时 |
先发报头长度 |
先收报头长度,用struct取出来 |
再编码报头内容而后发送 | 根据取出的长度收取报头内容,而后解码,反序列化 |
最后发真实内容 | 从反序列化的结果中取出待取数据的详细信息,而后去取真实的数据内容 |
import socket,struct,json import subprocess phone=socket.socket(socket.AF_INET,socket.SOCK_STREAM) phone.setsockopt(socket.SOL_SOCKET,socket.SO_REUSEADDR,1) #就是它,在bind前加
phone.bind(('127.0.0.1',8080)) phone.listen(5) while True: conn,addr=phone.accept() while True: cmd=conn.recv(1024) if not cmd:break
print('cmd: %s' %cmd) res=subprocess.Popen(cmd.decode('utf-8'), shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) err=res.stderr.read() print(err) if err: back_msg=err else: back_msg=res.stdout.read() headers={'data_size':len(back_msg)} head_json=json.dumps(headers) head_json_bytes=bytes(head_json,encoding='utf-8') conn.send(struct.pack('i',len(head_json_bytes))) #先发报头的长度
conn.send(head_json_bytes) #再发报头
conn.sendall(back_msg) #在发真实的内容
conn.close()
from socket import *
import struct,json ip_port=('127.0.0.1',8080) client=socket(AF_INET,SOCK_STREAM) client.connect(ip_port) while True: cmd=input('>>: ') if not cmd:continue client.send(bytes(cmd,encoding='utf-8')) head=client.recv(4) head_json_len=struct.unpack('i',head)[0] head_json=json.loads(client.recv(head_json_len).decode('utf-8')) data_len=head_json['data_size'] recv_size=0 recv_data=b''
while recv_size < data_len: recv_data+=client.recv(1024) recv_size+=len(recv_data) print(recv_data.decode('utf-8')) #print(recv_data.decode('gbk')) #windows默认gbk编码
FTP做业:上传下载文件
import socket import struct import json import subprocess import os class MYTCPServer: address_family = socket.AF_INET socket_type = socket.SOCK_STREAM allow_reuse_address = False max_packet_size = 8192 coding='utf-8' request_queue_size = 5 server_dir='file_upload'
def __init__(self, server_address, bind_and_activate=True): """Constructor. May be extended, do not override.""" self.server_address=server_address self.socket = socket.socket(self.address_family, self.socket_type) if bind_and_activate: try: self.server_bind() self.server_activate() except: self.server_close() raise
def server_bind(self): """Called by constructor to bind the socket. """
if self.allow_reuse_address: self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) self.socket.bind(self.server_address) self.server_address = self.socket.getsockname() def server_activate(self): """Called by constructor to activate the server. """ self.socket.listen(self.request_queue_size) def server_close(self): """Called to clean-up the server. """ self.socket.close() def get_request(self): """Get the request and client address from the socket. """
return self.socket.accept() def close_request(self, request): """Called to clean up an individual request.""" request.close() def run(self): while True: self.conn,self.client_addr=self.get_request() print('from client ',self.client_addr) while True: try: head_struct = self.conn.recv(4) if not head_struct:break head_len = struct.unpack('i', head_struct)[0] head_json = self.conn.recv(head_len).decode(self.coding) head_dic = json.loads(head_json) print(head_dic) #head_dic={'cmd':'put','filename':'a.txt','filesize':123123}
cmd=head_dic['cmd'] if hasattr(self,cmd): func=getattr(self,cmd) func(head_dic) except Exception: break
def put(self,args): file_path=os.path.normpath(os.path.join( self.server_dir, args['filename'] )) filesize=args['filesize'] recv_size=0 print('----->',file_path) with open(file_path,'wb') as f: while recv_size < filesize: recv_data=self.conn.recv(self.max_packet_size) f.write(recv_data) recv_size+=len(recv_data) print('recvsize:%s filesize:%s' %(recv_size,filesize)) tcpserver1=MYTCPServer(('127.0.0.1',8080)) tcpserver1.run() #下列代码与本题无关
class MYUDPServer: """UDP server class.""" address_family = socket.AF_INET socket_type = socket.SOCK_DGRAM allow_reuse_address = False max_packet_size = 8192 coding='utf-8'
def get_request(self): data, client_addr = self.socket.recvfrom(self.max_packet_size) return (data, self.socket), client_addr def server_activate(self): # No need to call listen() for UDP.
pass
def shutdown_request(self, request): # No need to shutdown anything.
self.close_request(request) def close_request(self, request): # No need to close anything.
pass
import socket import struct import json import os class MYTCPClient: address_family = socket.AF_INET socket_type = socket.SOCK_STREAM allow_reuse_address = False max_packet_size = 8192 coding='utf-8' request_queue_size = 5
def __init__(self, server_address, connect=True): self.server_address=server_address self.socket = socket.socket(self.address_family, self.socket_type) if connect: try: self.client_connect() except: self.client_close() raise
def client_connect(self): self.socket.connect(self.server_address) def client_close(self): self.socket.close() def run(self): while True: inp=input(">>: ").strip() if not inp:continue l=inp.split() cmd=l[0] if hasattr(self,cmd): func=getattr(self,cmd) func(l) def put(self,args): cmd=args[0] filename=args[1] if not os.path.isfile(filename): print('file:%s is not exists' %filename) return
else: filesize=os.path.getsize(filename) head_dic={'cmd':cmd,'filename':os.path.basename(filename),'filesize':filesize} print(head_dic) head_json=json.dumps(head_dic) head_json_bytes=bytes(head_json,encoding=self.coding) head_struct=struct.pack('i',len(head_json_bytes)) self.socket.send(head_struct) self.socket.send(head_json_bytes) send_size=0 with open(filename,'rb') as f: for line in f: self.socket.send(line) send_size+=len(line) print(send_size) else: print('upload successful') client=MYTCPClient(('127.0.0.1',8080)) client.run()
3,socket的更多方法介绍
服务端套接字函数
s.bind() 绑定(主机,端口号)到套接字
s.listen() 开始TCP监听
s.accept() 被动接受TCP客户的链接,(阻塞式)等待链接的到来
客户端套接字函数
s.connect() 主动初始化TCP服务器链接
s.connect_ex() connect()函数的扩展版本,出错时返回出错码,而不是抛出异常
公共用途的套接字函数
s.recv() 接收TCP数据
s.send() 发送TCP数据
s.sendall() 发送TCP数据
s.recvfrom() 接收UDP数据
s.sendto() 发送UDP数据
s.getpeername() 链接到当前套接字的远端的地址
s.getsockname() 当前套接字的地址
s.getsockopt() 返回指定套接字的参数
s.setsockopt() 设置指定套接字的参数
s.close() 关闭套接字
面向锁的套接字方法
s.setblocking() 设置套接字的阻塞与非阻塞模式
s.settimeout() 设置阻塞套接字操做的超时时间
s.gettimeout() 获得阻塞套接字操做的超时时间
面向文件的套接字的函数
s.fileno() 套接字的文件描述符
s.makefile() 建立一个与该套接字相关的文件
官方文档对socket模块下的socket.send()和socket.sendall()解释以下: socket.send(string[, flags]) Send data to the socket. The socket must be connected to a remote socket. The optional flags argument has the same meaning as for recv() above. Returns the number of bytes sent. Applications are responsible for checking that all data has been sent; if only some of the data was transmitted, the application needs to attempt delivery of the remaining data. send()的返回值是发送的字节数量,这个数量值可能小于要发送的string的字节数,也就是说可能没法发送string中全部的数据。若是有错误则会抛出异常。 – socket.sendall(string[, flags]) Send data to the socket. The socket must be connected to a remote socket. The optional flags argument has the same meaning as for recv() above. Unlike send(), this method continues to send data from string until either all data has been sent or an error occurs. None is returned on success. On error, an exception is raised, and there is no way to determine how much data, if any, was successfully sent. 尝试发送string的全部数据,成功则返回None,失败则抛出异常。 故,下面两段代码是等价的: #sock.sendall('Hello world\n')
#buffer = 'Hello world\n' #while buffer: # bytes = sock.send(buffer) # buffer = buffer[bytes:]
4,验证客户端连接的合法性
# sever端
import os
import hmac
import socket
secret_key = b'egg' # 密钥
sk = socket.socket()
sk.bind(('127.0.0.1',8080))
sk.listen()
def check_conn(conn):
msg = os.urandom(32) # 用os模块的urandom随机生成一个32位的字节
conn.send(msg) # 将要加密内容发送给客户端
h = hmac.new(secret_key,msg) # 建立一个对象 # 传入的密钥,要加密的内容
digest = h.digest() # 加密以后的内容
client_digest = conn.recv(1024) # 接收客户端加密后的验证内容
return hmac.compare_digest(digest,client_digest) # 对比两个加密内容是否同样
conn,addr = sk.accept()
res = check_conn(conn)
if res:
print('合法的客户端')
conn.close()
else:
print('不合法的客户端')
conn.close()
sk.close()
# client端
import hmac
import socket
secret_key = b'egg'
sk = socket.socket()
sk.connect(('127.0.0.1',8080))
msg = sk.recv(1024) # 接收服务端验证合法性时要加密的内容
h = hmac.new(secret_key,msg)
digest = h.digest()
sk.send(digest)
sk.close()
若是你想在分布式系统中实现一个简单的客户端连接认证功能,又不像SSL那么复杂,那么利用hmac+加盐的方式来实现
#_*_coding:utf-8_*_
from socket import *
import hmac,os secret_key=b'linhaifeng bang bang bang'
def conn_auth(conn): ''' 认证客户端连接 :param conn: :return: '''
print('开始验证新连接的合法性') msg=os.urandom(32) conn.sendall(msg) h=hmac.new(secret_key,msg) digest=h.digest() respone=conn.recv(len(digest)) return hmac.compare_digest(respone,digest) def data_handler(conn,bufsize=1024): if not conn_auth(conn): print('该连接不合法,关闭') conn.close() return
print('连接合法,开始通讯') while True: data=conn.recv(bufsize) if not data:break conn.sendall(data.upper()) def server_handler(ip_port,bufsize,backlog=5): ''' 只处理连接 :param ip_port: :return: ''' tcp_socket_server=socket(AF_INET,SOCK_STREAM) tcp_socket_server.bind(ip_port) tcp_socket_server.listen(backlog) while True: conn,addr=tcp_socket_server.accept() print('新链接[%s:%s]' %(addr[0],addr[1])) data_handler(conn,bufsize) if __name__ == '__main__': ip_port=('127.0.0.1',9999) bufsize=1024 server_handler(ip_port,bufsize)
#_*_coding:utf-8_*_
__author__ = 'Linhaifeng'
from socket import *
import hmac,os secret_key=b'linhaifeng bang bang bang'
def conn_auth(conn): ''' 验证客户端到服务器的连接 :param conn: :return: ''' msg=conn.recv(32) h=hmac.new(secret_key,msg) digest=h.digest() conn.sendall(digest) def client_handler(ip_port,bufsize=1024): tcp_socket_client=socket(AF_INET,SOCK_STREAM) tcp_socket_client.connect(ip_port) conn_auth(tcp_socket_client) while True: data=input('>>: ').strip() if not data:continue
if data == 'quit':break tcp_socket_client.sendall(data.encode('utf-8')) respone=tcp_socket_client.recv(bufsize) print(respone.decode('utf-8')) tcp_socket_client.close() if __name__ == '__main__': ip_port=('127.0.0.1',9999) bufsize=1024 client_handler(ip_port,bufsize)
#_*_coding:utf-8_*_
__author__ = 'Linhaifeng'
from socket import *
def client_handler(ip_port,bufsize=1024): tcp_socket_client=socket(AF_INET,SOCK_STREAM) tcp_socket_client.connect(ip_port) while True: data=input('>>: ').strip() if not data:continue
if data == 'quit':break tcp_socket_client.sendall(data.encode('utf-8')) respone=tcp_socket_client.recv(bufsize) print(respone.decode('utf-8')) tcp_socket_client.close() if __name__ == '__main__': ip_port=('127.0.0.1',9999) bufsize=1024 client_handler(ip_port,bufsize)
#_*_coding:utf-8_*_
__author__ = 'Linhaifeng'
from socket import *
import hmac,os secret_key=b'linhaifeng bang bang bang1111'
def conn_auth(conn): ''' 验证客户端到服务器的连接 :param conn: :return: ''' msg=conn.recv(32) h=hmac.new(secret_key,msg) digest=h.digest() conn.sendall(digest) def client_handler(ip_port,bufsize=1024): tcp_socket_client=socket(AF_INET,SOCK_STREAM) tcp_socket_client.connect(ip_port) conn_auth(tcp_socket_client) while True: data=input('>>: ').strip() if not data:continue
if data == 'quit':break tcp_socket_client.sendall(data.encode('utf-8')) respone=tcp_socket_client.recv(bufsize) print(respone.decode('utf-8')) tcp_socket_client.close() if __name__ == '__main__': ip_port=('127.0.0.1',9999) bufsize=1024 client_handler(ip_port,bufsize)
5,socketserver
import socketserver class Myserver(socketserver.BaseRequestHandler): def handle(self): self.data = self.request.recv(1024).strip() print("{} wrote:".format(self.client_address[0])) print(self.data) self.request.sendall(self.data.upper()) if __name__ == "__main__": HOST, PORT = "127.0.0.1", 9999
# 设置allow_reuse_address容许服务器重用地址
socketserver.TCPServer.allow_reuse_address = True # 建立一个server, 将服务地址绑定到127.0.0.1:9999
server = socketserver.TCPServer((HOST, PORT),Myserver) # 让server永远运行下去,除非强制中止程序
server.serve_forever()
import socket HOST, PORT = "127.0.0.1", 9999 data = "hello"
# 建立一个socket连接,SOCK_STREAM表明使用TCP协议
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock: sock.connect((HOST, PORT)) # 连接到客户端
sock.sendall(bytes(data + "\n", "utf-8")) # 向服务端发送数据
received = str(sock.recv(1024), "utf-8")# 从服务端接收数据
print("Sent: {}".format(data)) print("Received: {}".format(received))
# socket 创建一个tcp,在同一时间只能链接一个客户端
# socketserver 创建一个tcp,在同一时间能链接多个客户端
import socketserver class MyServer(socketserver.BaseRequestHandler): def handle(self): # self.request 就至关于一个conn,就是拿到了conn(地址)
while True: print(self.client_address) msg = self.request.recv(1024).decode('utf-8') if msg == 'q':break
print(msg) info = input('%s>>>'%msg[:2]) self.request.send(info.encode('utf-8')) if __name__ == '__main__': server = socketserver.ThreadingTCPServer(('127.0.0.1',8080),MyServer) # thread 线程
server.allow_reuse_address = True server.serve_forever()
import socket sk = socket.socket() sk.connect(('127.0.0.1',8080)) while True: msg = input('>>>') if msg == 'q': sk.send(b'q') break sk.send(('美团 :'+msg).encode('utf-8')) ret = sk.recv(1024).decode('utf-8') print(ret) sk.close()
import socket sk = socket.socket() sk.connect(('127.0.0.1',8080)) while True: msg = input('>>>') if msg == 'q': sk.send(b'q') break sk.send(('大众点评 :'+msg).encode('utf-8')) ret = sk.recv(1024).decode('utf-8') print(ret) sk.close()
6,setblocking() 将套接字设为非阻塞
import socket
sk = socket.socket()
sk.setblocking(False) # 设置套接字对象,
#设置在接收时若是没有接收到内容就直接过去,不会在等。
sk.bind(('127.0.0.1',8080))
sk.listen()
try:
conn,addr = sk.accept() # 通过设置后,当执行到这没有接收到内容时,
#不会像之前写的程序(没有设置的程序)同样,
#程序会停在此处等待接收内容,而是直接过去,
#可是两个变量没有被赋值会报错。
except BlockingIOError:
pass
print('+++++++')
7,
# 多个send 小的数据连在一块儿,会发生黏包现象,是tcp协议内部的优化算法形成的# 多个接收 只有一个接收到,其余的没有接收到,当断开连接时, windows 高版本会发送一个空消息 ,低版本会直接报错