From 14b803609ed4af820f9a210fd052f6e553a08e7c Mon Sep 17 00:00:00 2001 From: wangxiangshun Date: Mon, 27 Oct 2025 21:11:45 +0800 Subject: [PATCH] =?UTF-8?q?AI=20=E6=8F=90=E9=97=AE=E8=BF=BD=E9=97=AE?= =?UTF-8?q?=E7=8E=AF=E5=A2=83demo,=E4=BB=A5=E5=8F=8A=E8=AF=84=E5=88=86?= =?UTF-8?q?=E5=A4=84=E7=90=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../vetti/socket/ChatWebSocketHandler.java | 63 +++++++++++++++---- .../web/controller/ai/AiCommonController.java | 6 +- 2 files changed, 53 insertions(+), 16 deletions(-) diff --git a/vetti-admin/src/main/java/com/vetti/socket/ChatWebSocketHandler.java b/vetti-admin/src/main/java/com/vetti/socket/ChatWebSocketHandler.java index bcd5860..191ec7e 100644 --- a/vetti-admin/src/main/java/com/vetti/socket/ChatWebSocketHandler.java +++ b/vetti-admin/src/main/java/com/vetti/socket/ChatWebSocketHandler.java @@ -133,7 +133,6 @@ public class ChatWebSocketHandler { Map mapResult = JSONUtil.toBean(JSONUtil.parseObj(message), Map.class); String resultFlag = mapResult.get("msg"); if ("done".equals(resultFlag)) { - log.info("1、开始处理时间:{}", System.currentTimeMillis() / 1000); //开始合并语音流 //发送消息 WebSocket webSocket = cacheWebSocket.get(session.getId()); @@ -198,7 +197,6 @@ public class ChatWebSocketHandler { //获取完问答数据,直接清空缓存数据 cacheClientTts.put(clientId,""); cacheReplyFlag.put(session.getId(),""); - log.info("1、开始进行AI回答时间:{}", System.currentTimeMillis() / 1000); //把提问的文字发送给CPT(流式处理) OpenAiStreamClient aiStreamClient = SpringUtils.getBean(OpenAiStreamClient.class); log.info("AI提示词为:{}",promptJson); @@ -215,7 +213,6 @@ public class ChatWebSocketHandler { cacheQuestionResult.put(session.getId(),questionResult); // 实时输出内容 //开始进行语音输出-流式持续输出 - log.info("2、开始进行AI回答时间:{}", System.currentTimeMillis() / 1000); //把结果文字转成语音文件 //生成文件 //生成唯一文件名 @@ -223,7 +220,6 @@ public class ChatWebSocketHandler { String resultPathUrl = RuoYiConfig.getProfile() + VOICE_STORAGE_RESULT_DIR + resultFileName; ElevenLabsClient elevenLabsClient = SpringUtils.getBean(ElevenLabsClient.class); elevenLabsClient.handleTextToVoice(content, resultPathUrl); - log.info("3、开始进行AI回答时间:{}", System.currentTimeMillis() / 1000); //持续返回数据流给客户端 try { //文件转换成文件流 @@ -231,7 +227,6 @@ public class ChatWebSocketHandler { //发送文件流数据 session.getBasicRemote().sendBinary(outByteBuffer); // 发送响应确认 - log.info("4、开始进行AI回答时间:{}", System.currentTimeMillis() / 1000); } catch (IOException e) { e.printStackTrace(); } @@ -262,9 +257,59 @@ public class ChatWebSocketHandler { } catch (Exception e) { throw new RuntimeException(e); } - log.info("5、结束进行AI回答时间:{}", System.currentTimeMillis() / 1000); } + @Override + public void onError(Throwable throwable) { + throwable.printStackTrace(); + } + }); + }else if("end".equals(resultFlag)){ + //发送面试官结束语音流 + String openingPathUrl = RuoYiConfig.getProfile() + VOICE_SYSTEM_DIR + "end.wav"; + try { + //文件转换成文件流 + ByteBuffer outByteBuffer = convertFileToByteBuffer(openingPathUrl); + //发送文件流数据 + session.getBasicRemote().sendBinary(outByteBuffer); + // 发送响应确认 + log.info("结束返回面试官语音信息:{}", System.currentTimeMillis() / 1000); + } catch (IOException e) { + e.printStackTrace(); + } + //返回文本评分 + //处理模型提问逻辑 + String promptJson = ""; + //获取缓存记录 + String msgMapData = cacheMsgMapData.get(session.getId()); + if(StrUtil.isNotEmpty(msgMapData)){ + List list = JSONUtil.toList(msgMapData, Map.class); + //获取最后一条数据记录 + Map mapEntity = list.get(0); + //更新问题记录 + mapEntity.put("role","system"); + mapEntity.put("content","你是建筑行业面试专家，对Construction Labourer候选人回答进行1-5分评分。"); + promptJson = JSONUtil.toJsonStr(list); + //结束回答要清空问答数据 + cacheMsgMapData.put(session.getId(),""); + } + log.info("结束AI提示词为:{}",promptJson); + OpenAiStreamClient aiStreamClient = SpringUtils.getBean(OpenAiStreamClient.class); + aiStreamClient.streamChat(promptJson, new OpenAiStreamListenerService() { + @Override + public void onMessage(String content) { + log.info("返回AI结果：{}", content); + try { + //发送文件流数据 + session.getBasicRemote().sendText(content); + }catch (Exception e){ + e.printStackTrace(); + } + } + + @Override + public void onComplete() { + } @Override public void onError(Throwable throwable) { throwable.printStackTrace(); @@ -280,20 +325,16 @@ public class ChatWebSocketHandler { // 接收二进制消息（流数据） @OnMessage public void onBinaryMessage(Session session, @PathParam("clientId") String clientId, ByteBuffer byteBuffer) { - log.info("1、开始接收数据流时间:{}", System.currentTimeMillis() / 1000); log.info("客户端ID为:{}", clientId); // 处理二进制流数据 byte[] bytes = new byte[byteBuffer.remaining()]; //从缓冲区中读取数据并存储到指定的字节数组中 byteBuffer.get(bytes); - log.info("2、开始接收数据流时间:{}", System.currentTimeMillis() / 1000); // 生成唯一文件名 String fileName = clientId + "_" + System.currentTimeMillis() + ".wav"; String pathUrl = RuoYiConfig.getProfile() + VOICE_STORAGE_DIR + fileName; log.info("文件路径为:{}", pathUrl); - log.info("3、开始接收数据流时间:{}", System.currentTimeMillis() / 1000); try { - log.info("文件流的大小为:{}", bytes.length); saveAsWebM(bytes, pathUrl); //接收到数据流后直接就进行SST处理 //语音格式转换 @@ -306,7 +347,6 @@ public class ChatWebSocketHandler { if (webSocket != null) { // 1. 启动音频缓冲 // webSocket.send("{\"type\": \"input_audio_buffer.start\"}"); - log.info("3.1 开始发送数据音频流啦"); File outputFile = new File(pathOutUrl); // 输出PCM格式文件 ByteBuffer buffer = ByteBuffer.wrap(FileUtils.readFileToByteArray(outputFile)); byte[] outBytes = new byte[buffer.remaining()]; @@ -315,7 +355,6 @@ public class ChatWebSocketHandler { String base64Audio = Base64.getEncoder().encodeToString(outBytes); String message = "{ \"type\": \"input_audio_buffer.append\", \"audio\": \"" + base64Audio + "\" }"; webSocket.send(message); - log.info("4、开始接收数据流时间:{}", System.currentTimeMillis() / 1000); // 3. 提交音频并请求转录 // webSocket.send("{\"type\": \"input_audio_buffer.commit\"}"); // webSocket.send("{\"type\": \"response.create\"}"); diff --git a/vetti-admin/src/main/java/com/vetti/web/controller/ai/AiCommonController.java b/vetti-admin/src/main/java/com/vetti/web/controller/ai/AiCommonController.java index aed585c..acd17ab 100644 --- a/vetti-admin/src/main/java/com/vetti/web/controller/ai/AiCommonController.java +++ b/vetti-admin/src/main/java/com/vetti/web/controller/ai/AiCommonController.java @@ -40,10 +40,8 @@ public class AiCommonController extends BaseController public AjaxResult handleTextToVice() { //你好,我是本次的面试官Vetti,请点击开始按钮后,做一段自我介绍. - elevenLabsClient.handleTextToVoice("Hello, I am Vetti, the interviewer for this interview. Please click the start button and give a self introduction","/Users/wangxiangshun/Desktop/临时文件/opening2.wav"); - - elevenLabsClient.handleTextToVoice("Good","/Users/wangxiangshun/Desktop/临时文件/good.wav"); - + //本轮面试结束,谢谢您的配合,面试结果将稍后通知 + elevenLabsClient.handleTextToVoice("This round of interview is over. Thank you for your cooperation. The interview results will be notified later","/Users/wangxiangshun/Desktop/临时文件/end.wav"); return success(); }