纯前端页面实现的智能语音实时听写、大模型答复、语音实时合成功能。

<template><div class="Model-container" style="padding: 10px;margin-bottom:50px; "><!--聊天窗口开始 --><el-row><el-col :span="24"><divstyle="width: 1200px;margin: 0 auto;background-color: white;border-radius: 5px;box-shadow: 0 0 10px #cccccc"><div style="text-align: center;line-height: 50px;">大模型智能问答</div><!--展示会话窗口--><div ref="scrollContainer" style="height: 530px;overflow: auto;border-top:1px solid #ccc"v-html="content"></div><div style="height: 150px;"><textarea v-model="text" @keydown.enter.prevent="sendAndAsk"style="height: 160px;width: 100%;padding: 20px; border: none;border-top: 1px solid #ccc;border-bottom: 1px solid #ccc;outline: none"></textarea><div style="text-align: left;padding-right: 10px;"><el-button type="primary" size="medium" @click="sendAndAsk">发送咨询</el-button><el-button type="success" size="medium" @click="voiceSend"><i class="el-icon-microphone"></i>语音输入</el-button><el-button type="danger" size="medium" @click="stopVoice">停止朗读</el-button><el-button type="danger" size="medium" @click="clearHistory">清空历史</el-button></div></div></div></el-col></el-row><!--聊天窗口结束 --></div>
</template><script>// 初始化录音工具,注意目录
let recorder = new Recorder("../../recorder")
recorder.onStart = () => {console.log("开始录音了")
}
recorder.onStop = () => {console.log("结束录音了")
}
// 发送中间帧和最后一帧
recorder.onFrameRecorded = ({isLastFrame, frameBuffer}) => {if (!isLastFrame && wsFlag) { // 发送中间帧const params = {data: {status: 1,format: "audio/L16;rate=16000",encoding: "raw",audio: toBase64(frameBuffer),},};wsTask.send(JSON.stringify(params)) // 执行发送} else {if (wsFlag) {const params = {data: {status: 2,format: "audio/L16;rate=16000",encoding: "raw",audio: "",},};console.log("发送最后一帧", params, wsFlag)wsTask.send(JSON.stringify(params)) // 执行发送}}
}let wsFlag = false;
let wsTask = {};
let wsFlagModel = false;
let wsTaskModel = {};
const audioPlayer = new AudioPlayer("../../player"); // 播放器
export default {name: "Model",data() {return {dialogWidth: window.screen.width >= 1920 ? window.screen.width * 0.77 + "px" : window.screen.width * 0.9 + "px",wangHeight: window.screen.width >= 1920 ? window.screen.height * 0.541 + "px" : window.screen.height * 0.6 + "px",user: localStorage.getItem("user") ? JSON.parse(localStorage.getItem("user")) : {}, // 获取本地存储用户text: "",sendTextForMySql: "",scrollFlag: false,URL: 'wss://iat-api.xfyun.cn/v2/iat',URL_MODEL: 'wss://spark-api.xf-yun.com/v4.0/chat', // 大模型地址resultText: "",resultTextTemp: "",textList: [],// 大模型历史会话记录messageList: [],modelRes: "",content: '', // 现在用到的变量都需要提前定义needInsertFlag: false,ttsText: ""}},created() {// 请求分页数据this.initUserQuestion()},methods: {stopVoice() {window.location.reload()},doWsWork() {let bgs = this.bgMusic ? 1 : 0;const url = this.getWebSocketUrlTts(atob(this.user.apikey), atob(this.user.apisecret));if ("WebSocket" in window) {this.ttsWS = new WebSocket(url);} else if ("MozWebSocket" in window) {this.ttsWS = new MozWebSocket(url);} else {alert("浏览器不支持WebSocket");return;}this.ttsWS.onopen = (e) => {console.log("链接成功...")audioPlayer.start({autoPlay: true,sampleRate: 16000,resumePlayDuration: 1000});let text = this.ttsText;let tte = document.getElementById("tte") ? "unicode" : "UTF8";let params = {common: {app_id: atob(this.user.appid),},business: {aue: "raw",auf: "audio/L16;rate=16000",vcn: "x4_panting",bgs: bgs,tte,},data: {status: 2,text: this.encodeText(text, tte),},};this.ttsWS.send(JSON.stringify(params));console.log("发送成功...")};this.ttsWS.onmessage = (e) => {let jsonData = JSON.parse(e.data);// console.log("合成返回的数据" + JSON.stringify(jsonData));// 合成失败if (jsonData.code !== 0) {console.error(jsonData);return;}audioPlayer.postMessage({type: "base64",data: jsonData.data.audio,isLastData: jsonData.data.status === 2,});if (jsonData.code === 0 && jsonData.data.status === 2) {this.ttsWS.close();}};this.ttsWS.onerror = (e) => {console.error(e);};this.ttsWS.onclose = (e) => {console.log(e + "链接已关闭");};},
// 文本编码encodeText(text, type) {if (type === "unicode") {let buf = new ArrayBuffer(text.length * 4);let bufView = new Uint16Array(buf);for (let i = 0, strlen = text.length; i < strlen; i++) {bufView[i] = text.charCodeAt(i);}let binary = "";let bytes = new Uint8Array(buf);let len = bytes.byteLength;for (let i = 0; i < len; i++) {binary += String.fromCharCode(bytes[i]);}return window.btoa(binary);} else {return base64.encode(text);}},
// 鉴权方法getWebSocketUrlTts(apiKey, apiSecret) {let url = "wss://tts-api.xfyun.cn/v2/tts";let host = location.host;let date = new Date().toGMTString();let algorithm = "hmac-sha256";let headers = "host date request-line";let signatureOrigin = `host: ${host}\ndate: ${date}\nGET /v2/tts HTTP/1.1`;let signatureSha = CryptoJS.HmacSHA256(signatureOrigin, apiSecret);let signature = CryptoJS.enc.Base64.stringify(signatureSha);let authorizationOrigin = `api_key="${apiKey}", algorithm="${algorithm}", headers="${headers}", signature="${signature}"`;let authorization = btoa(authorizationOrigin);url = `${url}?authorization=${authorization}&date=${date}&host=${host}`;return url;},clearHistory() {this.$http.post("/model/list_delete_by_send_user", {sendUser: this.user.name}).then(res => {if (res.data.code === "200") {this.$message.success('清空历史成功')window.location.reload()} else {this.$message.error('清空历史对话失败，' + res.data.message)}})},initUserQuestion() { // 从数据库查询数据，展示用户问答记录this.needInsertFlag = false;this.$http.post("/model/list_page", {sendUser: this.user.name}).then(res => {console.log(res.data)if (res.data.code === "200") {//  this.$message.success('查询历史对话成功')this.messageList = res.data.object.data;// alert("执行")this.messageList.forEach(item => {this.createContent(null, item.sendUser, item.sendContent)let temp = {"role": "user","content": item.sendContent}this.textList.push(temp);this.createContent(null, "大模型", item.modelAnswer)temp = {"role": "assistant","content": item.modelAnswer}this.textList.push(temp);})// alert("执行结束")console.log(JSON.stringify(this.textList))} else {this.$message.error('查询历史对话失败，' + res.data.message)}})},voiceSend() { // 开始语音识别要做的动作// 首先要调用扣费APIthis.user.ability = "语音听写能力" // 标记能力this.$http.post("/big/consume_balance", this.user).then(res => {if (res.data.code === "200") {// 触发父级更新user方法this.$emit("person_fff_user", res.data.object)this.resultText = "";this.resultTextTemp = "";this.wsInit();} else {this.$message.error(res.data.message)return false // 这个必须要做}})// 调用扣费API结束},stopRecorder() { // 听写可以用到的方法recorder.stop();_this.$message.success("实时听写停止！")},
// 建立ws连接async wsInitModel() {let _this = this;if (typeof (WebSocket) == 'undefined') {console.log('您的浏览器不支持ws...')} else {console.log('您的浏览器支持ws！！！')let reqeustUrl = await _this.getWebSocketUrlModel()wsTaskModel = new WebSocket(reqeustUrl);// ws的几个事件，在vue中定义wsTaskModel.onopen = function () {_this.modelRes = " " // 每次清空上次结果console.log('ws已经打开...')let tempUserInfo = {role: "user",content: _this.text}_this.textList.push(tempUserInfo) // 添加最新问题，历史问题从数据库查询wsFlagModel = truelet params = {"header": {"app_id": atob(_this.user.appid),"uid": "fd3f47e4-d"}, "parameter": {"chat": {"domain": "4.0Ultra","temperature": 0.01,"max_tokens": 8192}}, "payload": {"message": {"text": _this.textList/* "text": [{"role": "user", "content": "中国第一个皇帝是谁？"}, {"role": "assistant", "content": "秦始皇"}, {"role": "user", "content": "秦始皇修的长城吗"}, {"role": "assistant", "content": "是的"}, {"role": "user", "content": _this.text}]*/}}};console.log("发送第一帧数据...")wsTaskModel.send(JSON.stringify(params)) // 执行发送_this.sendTextForMySql = _this.text // 记录一份用于存储_this.text = ""; // 清空文本}wsTaskModel.onmessage = function (message) { // 调用第二个API 自动把语音转成文本// console.log('收到数据===' + message.data)let jsonData = JSON.parse(message.data);// console.log(jsonData)let tempList = jsonData.payload.choices.text;for (let i = 0; i < tempList.length; i++) {_this.modelRes = _this.modelRes + tempList[i].content;// console.log(tempList[i].content)}// 检测到结束或异常关闭if (jsonData.header.code === 0 && jsonData.header.status === 2) { // 拿到最终的听写文本后，我们会调用大模型wsTaskModel.close();wsFlagModel = false_this.createContent(null, "大模型", _this.modelRes);_this.ttsText = _this.modelRes_this.doWsWork()}if (jsonData.header.code !== 0) {wsTaskModel.close();wsFlagModel = falseconsole.error(jsonData);}}wsTaskModel.onclose = function () {console.log('ws已关闭...')}wsTaskModel.onerror = function () {console.log('发生错误...')}}},
// 获取鉴权地址与参数getWebSocketUrlModel() {return new Promise((resolve, reject) => {// 请求地址根据语种不同变化var url = this.URL_MODEL;var host = "spark-api.xf-yun.com";var apiKeyName = "api_key";var date = new Date().toGMTString();var algorithm = "hmac-sha256";var headers = "host date request-line";var signatureOrigin = `host: ${host}\ndate: ${date}\nGET /v4.0/chat HTTP/1.1`;var signatureSha = CryptoJS.HmacSHA256(signatureOrigin, atob(this.user.apisecret));var signature = CryptoJS.enc.Base64.stringify(signatureSha);var authorizationOrigin =`${apiKeyName}="${atob(this.user.apikey)}", algorithm="${algorithm}", headers="${headers}", signature="${signature}"`;var authorization = base64.encode(authorizationOrigin);url = `${url}?authorization=${authorization}&date=${encodeURI(date)}&host=${host}`;console.log(url)resolve(url); // 主要是返回地址});},
// 建立ws连接async wsInit() {//  this.iat = "";this.$message.success("请您说出提问内容~")let _this = this;if (typeof (WebSocket) == 'undefined') {console.log('您的浏览器不支持ws...')} else {console.log('您的浏览器支持ws！！！')let reqeustUrl = await _this.getWebSocketUrl()wsTask = new WebSocket(reqeustUrl);// ws的几个事件，在vue中定义wsTask.onopen = function () {console.log('ws已经打开...')wsFlag = truelet params = { // 第一帧数据common: {app_id: atob(_this.user.appid),},business: {language: "zh_cn",domain: "iat",accent: "mandarin",vad_eos: 2000,dwa: "wpgs",},data: {status: 0,format: "audio/L16;rate=16000",encoding: "raw",},};console.log("发送第一帧数据...")wsTask.send(JSON.stringify(params)) // 执行发送// 下面就可以循环发送中间帧了// 开始录音console.log("开始录音")recorder.start({sampleRate: 16000,frameSize: 1280,});}wsTask.onmessage = function (message) { // 调用第二个API 自动把语音转成文本// console.log('收到数据===' + message.data)let jsonData = JSON.parse(message.data);if (jsonData.data && jsonData.data.result) {let data = jsonData.data.result;let str = "";let ws = data.ws;for (let i = 0; i < ws.length; i++) {str = str + ws[i].cw[0].w;}if (data.pgs) {if (data.pgs === "apd") {// 将resultTextTemp同步给resultText_this.resultText = _this.resultTextTemp;}// 将结果存储在resultTextTemp中_this.resultTextTemp = _this.resultText + str;} else {_this.resultText = _this.resultText + str;}_this.text = _this.resultTextTemp || _this.resultText || "";}// 检测到结束或异常关闭if (jsonData.code === 0 && jsonData.data.status === 2) { // 拿到最终的听写文本后，我们会调用大模型// alert("执行了")recorder.stop();_this.$message.success("检测到您2秒没说话，自动结束识别！")wsTask.close();wsFlag = false}if (jsonData.code !== 0) {wsTask.close();wsFlag = falseconsole.error(jsonData);}}// 关闭事件wsTask.onclose = function () {console.log('ws已关闭...')}wsTask.onerror = function () {console.log('发生错误...')}}},
// 获取鉴权地址与参数getWebSocketUrl() {return new Promise((resolve, reject) => {// 请求地址根据语种不同变化var url = this.URL;var host = "iat-api.xfyun.cn";var apiKeyName = "api_key";var date = new Date().toGMTString();var algorithm = "hmac-sha256";var headers = "host date request-line";var signatureOrigin = `host: ${host}\ndate: ${date}\nGET /v2/iat HTTP/1.1`;var signatureSha = CryptoJS.HmacSHA256(signatureOrigin, atob(this.user.apisecret));var signature = CryptoJS.enc.Base64.stringify(signatureSha);var authorizationOrigin =`${apiKeyName}="${atob(this.user.apikey)}", algorithm="${algorithm}", headers="${headers}", signature="${signature}"`;var authorization = base64.encode(authorizationOrigin);url = `${url}?authorization=${authorization}&date=${encodeURI(date)}&host=${host}`;console.log(url)resolve(url); // 主要是返回地址});},sendAndAsk() { // 用户发送消息// 首先要调用扣费APIif (this.text == "") {this.$message.error("发送消息不能为空")return false}this.needInsertFlag = truethis.user.ability = "大模型问答" // 标记能力this.$http.post("/big/consume_balance", this.user).then(res => {if (res.data.code === "200") {// 触发父级更新user方法this.$emit("person_fff_user", res.data.object)if (!wsFlag) {// console.log("我打印的" + this.user.name)this.createContent(null, this.user.name, this.text);// 调用大模型this.wsInitModel();} else {this.$message.warning("听写工作中，请稍后再发送...")}} else {this.$message.error(res.data.message)return false // 这个必须要做}})// 调用扣费API结束},createContent(remoteUser, nowUser, text) {  // 这个方法是用来将 json的聊天消息数据转换成 html的。if (text == "") {this.$message.error("发送消息不能为空")return false}let html// alert("执行了")// 当前用户消息if (nowUser == this.user.name) { // nowUser 表示是否显示当前用户发送的聊天消息，绿色气泡html = "<div class=\"el-row\" style=\"padding: 5px 0;\">\n" +"  <div class=\"el-col el-col-22\" style=\"text-align: right; padding-right: 10px\">\n" +"    <div class=\"tip left myLeft\">" + text + "</div>\n" +"  </div>\n" +"  <div class=\"el-col el-col-2\">\n" +"  <span class=\"el-avatar el-avatar--circle\" style=\"height: 40px; width: 40px; line-height: 40px;\">\n" +"    <img src=\"" + this.user.avatar + "\" style=\"object-fit: cover;\">\n" +"  </span>\n" +"  </div>\n" +"</div>";} else {   // 其他表示大模型的答复，蓝色的气泡html = "<div class=\"el-row\" style=\"padding: 5px 0;width: 760px;\">\n" +"  <div class=\"el-col el-col-2\" style=\"text-align: right\">\n" +"  <span class=\"el-avatar el-avatar--circle\" style=\"height: 40px; width: 40px; line-height: 40px;\">\n" +"    <img src=\"" + `https://wdfgdzx.top:3333/document/cd39af3e175b4524890c267e07298f5b.png` + "\" style=\"object-fit: cover;\">\n" +"  </span>\n" +"  </div>\n" +"  <div class=\"el-col el-col-22\" style=\"text-align: left; padding-left: 10px\">\n" +"    <div class=\"tip right myLeft\">" + text + "</div>\n" +"  </div>\n" +"</div>";// 大模型答复完毕应该插入数据库记录let modelEntity = {sendUser: this.user.name,sendContent: this.sendTextForMySql,modelAnswer: text,type: "大模型文本问答" // 固定值}if (this.needInsertFlag) {this.$http.post("/model/insertOrUpdate", modelEntity).then(res => {if (res.data.code == "200") {// 执行成功} else {this.$message.error(res.data.message)}})}}console.log(html)this.content += html;// 滚动到底部setTimeout(this.scrollToDown, 300) // 延迟滚动才有效果},scrollToDown() {this.scrollFlag = true;if (this.scrollFlag) { // 滚动到底部let container = this.$refs.scrollContainer;container.scrollTop = 5000;}}}
}
</script><!--scoped 不能加-->
<style>
.tip {color: white;border-radius: 10px;font-family: sans-serif;padding: 10px;width: auto;display: inline-block !important;display: inline;
}.right {background-color: deepskyblue;
}.myLeft {text-align: left;
}.myRight {text-align: right;
}.myCenter {text-align: center;
}.left {background-color: forestgreen;
}
</style>

package com.black.controller;import cn.hutool.poi.excel.ExcelReader;
import cn.hutool.poi.excel.ExcelUtil;
import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
import com.black.mapper.ModelMapper;
import com.black.mapper.UserMapper;
import com.black.pojo.Model;
import com.black.pojo.User;
import com.black.util.Constants;
import com.black.util.MyUtils;
import com.black.util.Res;
import org.springframework.web.bind.annotation.*;
import org.springframework.web.multipart.MultipartFile;import javax.annotation.Resource;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
import java.util.List;@RestController
@RequestMapping("model")
public class ModelController {@ResourceModelMapper modelMapper;@PostMapping("/insertOrUpdate")public Res insertOrUpdate(@RequestBody Model model) throws Exception { // @RequestBody很重要if (model.getId() != null) { // 存在则更新modelMapper.updateById(model);} else {try {model.setSendTime(new Date());modelMapper.insert(model);} catch (Exception e) {e.printStackTrace();return Res.error(Constants.CODE_500, "系统错误");}}return Res.success(null);}@PostMapping("/delete")public Res delete(@RequestBody Model model) {modelMapper.deleteById(model);return Res.success(null);}@PostMapping("/select")public Res select(@RequestBody Model model) {Model existModel = modelMapper.selectById(model.getId());return Res.success(existModel);// 需要返回对象}@PostMapping("/list_page")public Res list_page(@RequestBody Model model) {// 1、查询条件QueryWrapper<Model> queryWrapper = new QueryWrapper<>();if (!MyUtils.blankFlag(model.getSendUser())) { // 如果非空，执行模糊查询queryWrapper.eq("send_user", model.getSendUser());}List<Model> dataList = modelMapper.selectList(queryWrapper); // 进行分页数据查询// 3、构建分页查询，返回给前端HashMap<Object, Object> hashMap = new HashMap<>();hashMap.put("data", dataList);return Res.success(hashMap);}@PostMapping("/list_delete_by_send_user")public Res list_delete_by_send_user(@RequestBody Model model) {QueryWrapper<Model> queryWrapper = new QueryWrapper<>();if (!MyUtils.blankFlag(model.getSendUser())) { // 如果非空，执行模糊查询queryWrapper.eq("send_user", model.getSendUser());}modelMapper.delete(queryWrapper);return Res.success(null);}@PostMapping("/list_delete")public Res list_delete(@RequestBody Model model) {modelMapper.deleteBatchIds(model.getRemoveIdList());return Res.success(null);}@PostMapping("/list_model") // 0、查询所有public Res list_model() {return Res.success(modelMapper.selectList(null));}@RequestMapping("/list_import") // 1、一般用不到导入方法public Res list_import(@RequestParam("multipartFile") MultipartFile multipartFile, @RequestParam("token") String token) throws Exception {ExcelReader excelReader12 = ExcelUtil.getReader(multipartFile.getInputStream(), 0);List<List<Object>> rowList12 = excelReader12.read();int nameIndex = 0;for (List<Object> row : rowList12) {if (nameIndex >= 1 && row.size() == 18 && row.get(3) != null && row.get(3) != "") {QueryWrapper<Model> modelQueryWrapper = new QueryWrapper<>();modelQueryWrapper.eq("name", row.get(3).toString());Model model = modelMapper.selectOne(modelQueryWrapper);if (model == null) { // 不存在则插入} else { // 存在则更新}}nameIndex++;}return null;}
}