从小白到入门webrtc音视频通话-编程知识

0. 写在前面

先会骑车，再研究为什么这么骑，才是我认为学习技术的思路，底部付了demo例子，根据例子上面的介绍即可运行。

1. 音视频通话要用到的技术简介

websocket
- 介绍：1. 服务器可以向浏览器推送信息；2. 一次握手成功，可持续互相发送信息
- 在音视频通话钟的作用：1. 作为音视频两个通话终端的桥梁，传递彼此上下线、网络环境等消息，因此他们都叫websocket为“信令服务器”
coturn
- 介绍：1. 包含stun服务和turn服务，stun可实现两个终端点对点语音通话；turn服务在无法点对点通话时，用作中转音视频流。
webrtc
- 介绍：1. 开源项目；2. 用于音视频实时互动、游戏、即时通讯、文件传输。

2. webrtc音视频通话开发思路

2.1. webrtc调用时序图

下图简化了B客户端创建PeerConnection，具体流程要看下面“调用时序图介绍”

2.2. 调用时序图介绍

上图名词介绍
1. client A：客户端A
2. Stun Server：穿透服务器，也就是coturn服务器中的Stun
3. Signal Server：信令服务器，也就是web socket搭建的服务器
4. client B：客户端B
5. PeerConnection（WebRtc的接口）
流程介绍
1. A客户端先发送信息到信令服务器，信令服务器存储A客户端信息，等待其他客户端加入。
2. B客户端再发送信息到信令服务器，信令服务器存储B客户端信息，并告知A已有新客户端加入。
3. A客户端创建 PeerConnection（WebRtc的接口），用于获取本地的网络信息、以及保存对方的网络信息、传递音视频流、监听通话过程状态。（B客户端后面也需要创建PeerConnection）
4. AddStreams：A客户端添加本地音视频流到PeerConnection
5. CreateOffer：A客户端创建Offer并发送给信令服务器，由信令服务器转给B客户端。Offer中包含本地的网络信息（SDP）。
6. CreateAnswer：B客户端收到Offer后，创建放到自己的PeerConnection中，并获取自己的网络信息（SDP），通过发送给信令服务器，由信令服务器转发给A客户端。
7. 上面步骤进行完毕后，开始通过信令服务器（coturn），双方客户端获取自己的地址作为候选人“candidate”，然后通过websocket发送给对方。彼此拿到候选地址后，互相进行访问测试，建立链接。
8. OnAddStream：获取对方音视频流，PeerConnection有ontrack监听器能拿到对方视频流数据。

2. 搭建WebSocket服务器

看例子中代码，使用nodejs启动

3. 搭建Coturn音视频穿透服务器

公司内网虚拟机中穿透服务器Coturn的搭建

4. 遇到的问题

后面再慢慢补吧，问题有点多

5. 例子

客户端代码使用html+js编写
WebSocket代码使用js编写使用nodejs运行
android端代码请下载：WebRtcAndroidDemo

5.1 客户端代码

引入adapter-latest.js文件，此文件如果过期了就自己百度找找吧。
将ws://192.168.1.60:9001/ws改为自己websocket服务所在电脑的ip地址，在本地启动则是本机地址
将iceServers中的ip改为coturn服务器所在ip地址

<html><head><title>Voice WebRTC demo</title></head><h1>WebRTC demo 1v1</h1><div id="buttons"><input id="zero-roomId" type="text" placeholder="请输入房间ID" maxlength="40"/><button id="joinBtn" type="button">加入</button><button id="leaveBtn" type="button">离开</button>    </div><div id="videos"><video id="localVideo" autoplay muted playsinline>本地窗口</video><video id="remoteVideo" autoplay playsinline>远端窗口</video></div><script src="js/main.js"></script><!-- 可直接引入在线js:https://webrtc.github.io/adapter/adapter-latest.js  --><script src="js/adapter-latest.js"></script>
</html>

'use strict';// join 主动加入房间
// leave 主动离开房间
// new-peer 有人加入房间，通知已经在房间的人
// peer-leave 有人离开房间，通知已经在房间的人
// offer 发送offer给对端peer
// answer发送offer给对端peer
// candidate 发送candidate给对端peer
const SIGNAL_TYPE_JOIN = "join";
const SIGNAL_TYPE_RESP_JOIN = "resp-join";  // 告知加入者对方是谁
const SIGNAL_TYPE_LEAVE = "leave";
const SIGNAL_TYPE_NEW_PEER = "new-peer";
const SIGNAL_TYPE_PEER_LEAVE = "peer-leave";
const SIGNAL_TYPE_OFFER = "offer";
const SIGNAL_TYPE_ANSWER = "answer";
const SIGNAL_TYPE_CANDIDATE = "candidate";var localUserId = Math.random().toString(36).substr(2); // 本地uid
var remoteUserId = -1;      // 对端
var roomId = 0;var localVideo = document.querySelector('#localVideo');
var remoteVideo = document.querySelector('#remoteVideo');
var localStream = null;
var remoteStream = null;
var pc = null;var zeroRTCEngine;function handleIceCandidate(event) {console.info("handleIceCandidate");if (event.candidate) {var candidateJson = {'label': event.candidate.sdpMLineIndex,'id': event.candidate.sdpMid,'candidate': event.candidate.candidate};var jsonMsg = {'cmd': SIGNAL_TYPE_CANDIDATE,'roomId': roomId,'uid': localUserId,'remoteUid':remoteUserId,'msg': JSON.stringify(candidateJson) };var message = JSON.stringify(jsonMsg);zeroRTCEngine.sendMessage(message);console.info("handleIceCandidate message: " + message);console.info("send candidate message");} else {console.warn("End of candidates");}
}function handleRemoteStreamAdd(event) {console.info("handleRemoteStreamAdd");remoteStream = event.streams[0];// 视频轨道// let videoTracks = remoteStream.getVideoTracks()// 音频轨道// let audioTracks = remoteStream.getAudioTracks()remoteVideo.srcObject = remoteStream;
}function handleConnectionStateChange() {if(pc != null) {console.info("ConnectionState -> " + pc.connectionState);}
}function handleIceConnectionStateChange() {if(pc != null) {console.info("IceConnectionState -> " + pc.iceConnectionState);}
}function createPeerConnection() {var defaultConfiguration = {  bundlePolicy: "max-bundle",rtcpMuxPolicy: "require",iceTransportPolicy:"all",//relay 或者 all// 修改ice数组测试效果，需要进行封装iceServers: [{"urls": ["turn:192.168.1.173:3478?transport=udp","turn:192.168.1.173:3478?transport=tcp"       // 可以插入多个进行备选],"username": "lqf","credential": "123456"},{"urls": ["stun:192.168.1.173:3478"]}]};pc = new RTCPeerConnection(defaultConfiguration); // 音视频通话的核心类pc.onicecandidate = handleIceCandidate;pc.ontrack = handleRemoteStreamAdd;pc.onconnectionstatechange = handleConnectionStateChange;pc.oniceconnectionstatechange = handleIceConnectionStateChangelocalStream.getTracks().forEach((track) => pc.addTrack(track, localStream)); // 把本地流设置给RTCPeerConnection
}function createOfferAndSendMessage(session) {pc.setLocalDescription(session).then(function () {var jsonMsg = {'cmd': 'offer','roomId': roomId,'uid': localUserId,'remoteUid': remoteUserId,'msg': JSON.stringify(session)};var message = JSON.stringify(jsonMsg);zeroRTCEngine.sendMessage(message);// console.info("send offer message: " + message);console.info("send offer message");}).catch(function (error) {console.error("offer setLocalDescription failed: " + error);});}function handleCreateOfferError(error) {console.error("handleCreateOfferError: " + error);
}function createAnswerAndSendMessage(session) {pc.setLocalDescription(session).then(function () {var jsonMsg = {'cmd': 'answer','roomId': roomId,'uid': localUserId,'remoteUid': remoteUserId,'msg': JSON.stringify(session)};var message = JSON.stringify(jsonMsg);zeroRTCEngine.sendMessage(message);// console.info("send answer message: " + message);console.info("send answer message");}).catch(function (error) {console.error("answer setLocalDescription failed: " + error);});}function handleCreateAnswerError(error) {console.error("handleCreateAnswerError: " + error);
}var ZeroRTCEngine = function (wsUrl) {this.init(wsUrl);zeroRTCEngine = this;return this;
}ZeroRTCEngine.prototype.init = function (wsUrl) {// 设置websocket  urlthis.wsUrl = wsUrl;/** websocket对象 */this.signaling = null;
}ZeroRTCEngine.prototype.createWebsocket = function () {zeroRTCEngine = this;zeroRTCEngine.signaling = new WebSocket(this.wsUrl);zeroRTCEngine.signaling.onopen = function () {zeroRTCEngine.onOpen();}zeroRTCEngine.signaling.onmessage = function (ev) {zeroRTCEngine.onMessage(ev);}zeroRTCEngine.signaling.onerror = function (ev) {zeroRTCEngine.onError(ev);}zeroRTCEngine.signaling.onclose = function (ev) {zeroRTCEngine.onClose(ev);}
}ZeroRTCEngine.prototype.onOpen = function () {console.log("websocket打开");
}
ZeroRTCEngine.prototype.onMessage = function (event) {console.log("websocket收到信息: " + event.data);var jsonMsg = null;try {jsonMsg = JSON.parse(event.data);} catch(e) {console.warn("onMessage parse Json failed:" + e);return;}switch (jsonMsg.cmd) {case SIGNAL_TYPE_NEW_PEER:handleRemoteNewPeer(jsonMsg);break;case SIGNAL_TYPE_RESP_JOIN:handleResponseJoin(jsonMsg);break;case SIGNAL_TYPE_PEER_LEAVE:handleRemotePeerLeave(jsonMsg);break;case SIGNAL_TYPE_OFFER:handleRemoteOffer(jsonMsg);break;case SIGNAL_TYPE_ANSWER:handleRemoteAnswer(jsonMsg);break;case SIGNAL_TYPE_CANDIDATE:handleRemoteCandidate(jsonMsg);break;}
}ZeroRTCEngine.prototype.onError = function (event) {console.log("onError: " + event.data);
}ZeroRTCEngine.prototype.onClose = function (event) {console.log("onClose -> code: " + event.code + ", reason:" + EventTarget.reason);
}ZeroRTCEngine.prototype.sendMessage = function (message) {this.signaling.send(message);
}function handleResponseJoin(message) {console.info("handleResponseJoin, remoteUid: " + message.remoteUid);remoteUserId = message.remoteUid;// doOffer();
}function handleRemotePeerLeave(message) {console.info("handleRemotePeerLeave, remoteUid: " + message.remoteUid);remoteVideo.srcObject = null;if(pc != null) {pc.close();pc = null;}
}function handleRemoteNewPeer(message) {console.info("处理远端新加入链接，并发送offer, remoteUid: " + message.remoteUid);remoteUserId = message.remoteUid;doOffer();
}function handleRemoteOffer(message) {console.info("handleRemoteOffer");if(pc == null) {createPeerConnection();}var desc = JSON.parse(message.msg);pc.setRemoteDescription(desc);doAnswer();
}function handleRemoteAnswer(message) {console.info("handleRemoteAnswer");var desc = JSON.parse(message.msg);pc.setRemoteDescription(desc);
}function handleRemoteCandidate(message) {console.info("handleRemoteCandidate");var jsonMsg = message.msg;if(typeof message.msg === "string"){jsonMsg = JSON.parse(message.msg);}var candidateMsg = {'sdpMLineIndex': jsonMsg.label,'sdpMid': jsonMsg.id,'candidate': jsonMsg.candidate};var candidate = new RTCIceCandidate(candidateMsg);pc.addIceCandidate(candidate).catch(e => {console.error("addIceCandidate failed:" + e.name);});
}function doOffer() {// 创建RTCPeerConnectionif (pc == null) {createPeerConnection();}// let options = {offerToReceiveVideo:true}// pc.createOffer(options).then(createOfferAndSendMessage).catch(handleCreateOfferError);pc.createOffer().then(createOfferAndSendMessage).catch(handleCreateOfferError);
}function doAnswer() {pc.createAnswer().then(createAnswerAndSendMessage).catch(handleCreateAnswerError);
}function doJoin(roomId) {var jsonMsg = {'cmd': 'join','roomId': roomId,'uid': localUserId,};var message = JSON.stringify(jsonMsg);zeroRTCEngine.sendMessage(message);console.info("doJoin message: " + message);
}function doLeave() {var jsonMsg = {'cmd': 'leave','roomId': roomId,'uid': localUserId,};var message = JSON.stringify(jsonMsg);zeroRTCEngine.sendMessage(message);console.info("doLeave message: " + message);hangup();
}function hangup() {localVideo.srcObject = null; // 0.关闭自己的本地显示remoteVideo.srcObject = null; // 1.不显示对方closeLocalStream(); // 2. 关闭本地流if(pc != null) {pc.close(); // 3.关闭RTCPeerConnectionpc = null;}
}function closeLocalStream() {if(localStream != null) {localStream.getTracks().forEach((track) => {track.stop();});}
}function openLocalStream(stream) {console.log('Open local stream');doJoin(roomId);localVideo.srcObject = stream;      // 显示画面localStream = stream;   // 保存本地流的句柄
}function initLocalStream() {navigator.mediaDevices.getUserMedia({audio: true,video: true}).then(openLocalStream).catch(function (e) {alert("getUserMedia() error: " + e.name);});
}
// zeroRTCEngine = new ZeroRTCEngine("wss://192.168.1.60:80/ws");
zeroRTCEngine = new ZeroRTCEngine("ws://192.168.1.60:9001/ws");
zeroRTCEngine.createWebsocket();document.getElementById('joinBtn').onclick = function () {roomId = document.getElementById('zero-roomId').value;if (roomId == "" || roomId == "请输入房间ID") {alert("请输入房间ID");return;}console.log("第一步：加入按钮被点击, roomId: " + roomId);// 初始化本地码流initLocalStream();
}document.getElementById('leaveBtn').onclick = function () {console.log("离开按钮被点击");doLeave();
}

5.2. 编写websocket服务

使用nodejs启动

var ws = require("nodejs-websocket")
var prort = 9001;// join 主动加入房间
// leave 主动离开房间
// new-peer 有人加入房间，通知已经在房间的人
// peer-leave 有人离开房间，通知已经在房间的人
// offer 发送offer给对端peer
// answer发送offer给对端peer
// candidate 发送candidate给对端peer
const SIGNAL_TYPE_JOIN = "join";
const SIGNAL_TYPE_RESP_JOIN = "resp-join";  // 告知加入者对方是谁
const SIGNAL_TYPE_LEAVE = "leave";
const SIGNAL_TYPE_NEW_PEER = "new-peer";
const SIGNAL_TYPE_PEER_LEAVE = "peer-leave";
const SIGNAL_TYPE_OFFER = "offer";
const SIGNAL_TYPE_ANSWER = "answer";
const SIGNAL_TYPE_CANDIDATE = "candidate";/** ----- ZeroRTCMap ----- */
var ZeroRTCMap = function () {this._entrys = new Array();this.put = function (key, value) {if (key == null || key == undefined) {return;}var index = this._getIndex(key);if (index == -1) {var entry = new Object();entry.key = key;entry.value = value;this._entrys[this._entrys.length] = entry;} else {this._entrys[index].value = value;}};this.get = function (key) {var index = this._getIndex(key);return (index != -1) ? this._entrys[index].value : null;};this.remove = function (key) {var index = this._getIndex(key);if (index != -1) {this._entrys.splice(index, 1);}};this.clear = function () {this._entrys.length = 0;};this.contains = function (key) {var index = this._getIndex(key);return (index != -1) ? true : false;};this.size = function () {return this._entrys.length;};this.getEntrys = function () {return this._entrys;};this._getIndex = function (key) {if (key == null || key == undefined) {return -1;}var _length = this._entrys.length;for (var i = 0; i < _length; i++) {var entry = this._entrys[i];if (entry == null || entry == undefined) {continue;}if (entry.key === key) {// equalreturn i;}}return -1;};
}var roomTableMap = new ZeroRTCMap();function Client(uid, conn, roomId) {this.uid = uid;     // 用户所属的idthis.conn = conn;   // uid对应的websocket连接this.roomId = roomId;
}function handleJoin(message, conn) {var roomId = message.roomId;var uid = message.uid;console.info("uid: " + uid + "try to join room " + roomId);var roomMap = roomTableMap.get(roomId);if (roomMap == null) {roomMap = new  ZeroRTCMap();        // 如果房间没有创建，则新创建一个房间roomTableMap.put(roomId, roomMap);}if(roomMap.size() >= 2) {console.error("roomId:" + roomId + " 已经有两人存在，请使用其他房间");// 加信令通知客户端，房间已满return null;}var client = new Client(uid, conn, roomId);roomMap.put(uid, client);if(roomMap.size() > 1) {// 房间里面已经有人了，加上新进来的人，那就是>=2了，所以要通知对方var clients = roomMap.getEntrys();for(var i in clients) {var remoteUid = clients[i].key;if (remoteUid != uid) {var jsonMsg = {'cmd': SIGNAL_TYPE_NEW_PEER,'remoteUid': uid};var msg = JSON.stringify(jsonMsg);var remoteClient =roomMap.get(remoteUid);console.info("new-peer: " + msg);remoteClient.conn.sendText(msg);jsonMsg = {'cmd':SIGNAL_TYPE_RESP_JOIN,'remoteUid': remoteUid};msg = JSON.stringify(jsonMsg);console.info("resp-join: " + msg);conn.sendText(msg);}}}return client;
}function handleLeave(message) {var roomId = message.roomId;var uid = message.uid;var roomMap = roomTableMap.get(roomId);if (roomMap == null) {console.error("handleLeave can't find then roomId " + roomId);return;}if (!roomMap.contains(uid)) {console.info("uid: " + uid +" have leave roomId " + roomId);return;}console.info("uid: " + uid + " leave room " + roomId);roomMap.remove(uid);        // 删除发送者if(roomMap.size() >= 1) {var clients = roomMap.getEntrys();for(var i in clients) {var jsonMsg = {'cmd': 'peer-leave','remoteUid': uid // 谁离开就填写谁};var msg = JSON.stringify(jsonMsg);var remoteUid = clients[i].key;var remoteClient = roomMap.get(remoteUid);if(remoteClient) {console.info("notify peer:" + remoteClient.uid + ", uid:" + uid + " leave");remoteClient.conn.sendText(msg);}}}
}function handleForceLeave(client) {var roomId = client.roomId;var uid = client.uid;// 1. 先查找房间号var roomMap = roomTableMap.get(roomId);if (roomMap == null) {console.warn("handleForceLeave can't find then roomId " + roomId);return;}// 2. 判别uid是否在房间if (!roomMap.contains(uid)) {console.info("uid: " + uid +" have leave roomId " + roomId);return;}// 3.走到这一步，说明客户端没有正常离开，所以我们要执行离开程序console.info("uid: " + uid + " force leave room " + roomId);roomMap.remove(uid);        // 删除发送者if(roomMap.size() >= 1) {var clients = roomMap.getEntrys();for(var i in clients) {var jsonMsg = {'cmd': 'peer-leave','remoteUid': uid // 谁离开就填写谁};var msg = JSON.stringify(jsonMsg);var remoteUid = clients[i].key;var remoteClient = roomMap.get(remoteUid);if(remoteClient) {console.info("notify peer:" + remoteClient.uid + ", uid:" + uid + " leave");remoteClient.conn.sendText(msg);}}}
}function handleOffer(message) {var roomId = message.roomId;var uid = message.uid;var remoteUid = message.remoteUid;console.info("handleOffer uid: " + uid + "transfer  offer  to remoteUid" + remoteUid);var roomMap = roomTableMap.get(roomId);if (roomMap == null) {console.error("handleOffer can't find then roomId " + roomId);return;}if(roomMap.get(uid) == null) {console.error("handleOffer can't find then uid " + uid);return;}var remoteClient = roomMap.get(remoteUid);if(remoteClient) {var msg = JSON.stringify(message);remoteClient.conn.sendText(msg);    //把数据发送给对方} else {console.error("can't find remoteUid： " + remoteUid);}
}function handleAnswer(message) {var roomId = message.roomId;var uid = message.uid;var remoteUid = message.remoteUid;console.info("handleAnswer uid: " + uid + "transfer answer  to remoteUid" + remoteUid);var roomMap = roomTableMap.get(roomId);if (roomMap == null) {console.error("handleAnswer can't find then roomId " + roomId);return;}if(roomMap.get(uid) == null) {console.error("handleAnswer can't find then uid " + uid);return;}var remoteClient = roomMap.get(remoteUid);if(remoteClient) {var msg = JSON.stringify(message);remoteClient.conn.sendText(msg);} else {console.error("can't find remoteUid： " + remoteUid);}
}function handleCandidate(message) {var roomId = message.roomId;var uid = message.uid;var remoteUid = message.remoteUid;console.info("处理Candidate uid: " + uid + "transfer candidate  to remoteUid" + remoteUid);var roomMap = roomTableMap.get(roomId);if (roomMap == null) {console.error("handleCandidate can't find then roomId " + roomId);return;}if(roomMap.get(uid) == null) {console.error("handleCandidate can't find then uid " + uid);return;}var remoteClient = roomMap.get(remoteUid);if(remoteClient) {var msg = JSON.stringify(message);remoteClient.conn.sendText(msg);} else {console.error("can't find remoteUid： " + remoteUid);}
}
// 创建监听9001端口webSocket服务
var server = ws.createServer(function(conn){console.log("创建一个新的连接--------")conn.client = null; // 对应的客户端信息// conn.sendText("我收到你的连接了....");conn.on("text", function(str) {// console.info("recv msg:" + str);var jsonMsg = JSON.parse(str);switch (jsonMsg.cmd) {case SIGNAL_TYPE_JOIN:conn.client = handleJoin(jsonMsg, conn);break;case SIGNAL_TYPE_LEAVE:handleLeave(jsonMsg);break;case SIGNAL_TYPE_OFFER:handleOffer(jsonMsg);break;   case SIGNAL_TYPE_ANSWER:handleAnswer(jsonMsg);break; case SIGNAL_TYPE_CANDIDATE:handleCandidate(jsonMsg);break;      }});conn.on("close", function(code, reason) {console.info("连接关闭 code: " + code + ", reason: " + reason);if(conn.client != null) {// 强制让客户端从房间退出handleForceLeave(conn.client);}});conn.on("error", function(err) {console.info("监听到错误:" + err);});
}).listen(prort);