AI 提问追问环境demo,以及评分处理

vetti-admin/src/main/java/com/vetti/socket/ChatWebSocketHandler.java

@@ -133,7 +133,6 @@ public class ChatWebSocketHandler {
                 Map<String, String> mapResult = JSONUtil.toBean(JSONUtil.parseObj(message), Map.class);
                 String resultFlag = mapResult.get("msg");
                 if ("done".equals(resultFlag)) {
-                    log.info("1、开始处理时间:{}", System.currentTimeMillis() / 1000);
                     //开始合并语音流
                     //发送消息
                     WebSocket webSocket = cacheWebSocket.get(session.getId());
@@ -198,7 +197,6 @@ public class ChatWebSocketHandler {
                     //获取完问答数据,直接清空缓存数据
                     cacheClientTts.put(clientId,"");
                     cacheReplyFlag.put(session.getId(),"");
-                    log.info("1、开始进行AI回答时间:{}", System.currentTimeMillis() / 1000);
                     //把提问的文字发送给CPT(流式处理)
                     OpenAiStreamClient aiStreamClient = SpringUtils.getBean(OpenAiStreamClient.class);
                     log.info("AI提示词为:{}",promptJson);
@@ -215,7 +213,6 @@ public class ChatWebSocketHandler {
                             cacheQuestionResult.put(session.getId(),questionResult);
                             // 实时输出内容
                             //开始进行语音输出-流式持续输出
-                            log.info("2、开始进行AI回答时间:{}", System.currentTimeMillis() / 1000);
                             //把结果文字转成语音文件
                             //生成文件
                             //生成唯一文件名
@@ -223,7 +220,6 @@ public class ChatWebSocketHandler {
                             String resultPathUrl = RuoYiConfig.getProfile() + VOICE_STORAGE_RESULT_DIR + resultFileName;
                             ElevenLabsClient elevenLabsClient = SpringUtils.getBean(ElevenLabsClient.class);
                             elevenLabsClient.handleTextToVoice(content, resultPathUrl);
-                            log.info("3、开始进行AI回答时间:{}", System.currentTimeMillis() / 1000);
                             //持续返回数据流给客户端
                             try {
                                 //文件转换成文件流
@@ -231,7 +227,6 @@ public class ChatWebSocketHandler {
                                 //发送文件流数据
                                 session.getBasicRemote().sendBinary(outByteBuffer);
                                 // 发送响应确认
-                                log.info("4、开始进行AI回答时间:{}", System.currentTimeMillis() / 1000);
                             } catch (IOException e) {
                                 e.printStackTrace();
                             }
@@ -262,9 +257,59 @@ public class ChatWebSocketHandler {
                             } catch (Exception e) {
                                 throw new RuntimeException(e);
                             }
-                            log.info("5、结束进行AI回答时间:{}", System.currentTimeMillis() / 1000);
                         }
 
+                        @Override
+                        public void onError(Throwable throwable) {
+                            throwable.printStackTrace();
+                        }
+                    });
+                }else if("end".equals(resultFlag)){
+                    //发送面试官结束语音流
+                    String openingPathUrl = RuoYiConfig.getProfile() + VOICE_SYSTEM_DIR + "end.wav";
+                    try {
+                        //文件转换成文件流
+                        ByteBuffer outByteBuffer = convertFileToByteBuffer(openingPathUrl);
+                        //发送文件流数据
+                        session.getBasicRemote().sendBinary(outByteBuffer);
+                        // 发送响应确认
+                        log.info("结束返回面试官语音信息:{}", System.currentTimeMillis() / 1000);
+                    } catch (IOException e) {
+                        e.printStackTrace();
+                    }
+                    //返回文本评分
+                    //处理模型提问逻辑
+                    String promptJson = "";
+                    //获取缓存记录
+                    String msgMapData = cacheMsgMapData.get(session.getId());
+                    if(StrUtil.isNotEmpty(msgMapData)){
+                        List<Map> list = JSONUtil.toList(msgMapData, Map.class);
+                        //获取最后一条数据记录
+                        Map<String,String> mapEntity = list.get(0);
+                        //更新问题记录
+                        mapEntity.put("role","system");
+                        mapEntity.put("content","你是建筑行业面试专家，对Construction Labourer候选人回答进行1-5分评分。");
+                        promptJson = JSONUtil.toJsonStr(list);
+                        //结束回答要清空问答数据
+                        cacheMsgMapData.put(session.getId(),"");
+                    }
+                    log.info("结束AI提示词为:{}",promptJson);
+                    OpenAiStreamClient aiStreamClient = SpringUtils.getBean(OpenAiStreamClient.class);
+                    aiStreamClient.streamChat(promptJson, new OpenAiStreamListenerService() {
+                        @Override
+                        public void onMessage(String content) {
+                            log.info("返回AI结果：{}", content);
+                            try {
+                                //发送文件流数据
+                                session.getBasicRemote().sendText(content);
+                            }catch (Exception e){
+                                e.printStackTrace();
+                            }
+                        }
+
+                        @Override
+                        public void onComplete() {
+                        }
                         @Override
                         public void onError(Throwable throwable) {
                             throwable.printStackTrace();
@@ -280,20 +325,16 @@ public class ChatWebSocketHandler {
     // 接收二进制消息（流数据）
     @OnMessage
     public void onBinaryMessage(Session session, @PathParam("clientId") String clientId, ByteBuffer byteBuffer) {
-        log.info("1、开始接收数据流时间:{}", System.currentTimeMillis() / 1000);
         log.info("客户端ID为:{}", clientId);
         // 处理二进制流数据
         byte[] bytes = new byte[byteBuffer.remaining()];
         //从缓冲区中读取数据并存储到指定的字节数组中
         byteBuffer.get(bytes);
-        log.info("2、开始接收数据流时间:{}", System.currentTimeMillis() / 1000);
         // 生成唯一文件名
         String fileName = clientId + "_" + System.currentTimeMillis() + ".wav";
         String pathUrl = RuoYiConfig.getProfile() + VOICE_STORAGE_DIR + fileName;
         log.info("文件路径为:{}", pathUrl);
-        log.info("3、开始接收数据流时间:{}", System.currentTimeMillis() / 1000);
         try {
-            log.info("文件流的大小为:{}", bytes.length);
             saveAsWebM(bytes, pathUrl);
             //接收到数据流后直接就进行SST处理
             //语音格式转换
@@ -306,7 +347,6 @@ public class ChatWebSocketHandler {
             if (webSocket != null) {
 //                 1. 启动音频缓冲
 //                webSocket.send("{\"type\": \"input_audio_buffer.start\"}");
-                log.info("3.1 开始发送数据音频流啦");
                 File outputFile = new File(pathOutUrl);  // 输出PCM格式文件
                 ByteBuffer buffer = ByteBuffer.wrap(FileUtils.readFileToByteArray(outputFile));
                 byte[] outBytes = new byte[buffer.remaining()];
@@ -315,7 +355,6 @@ public class ChatWebSocketHandler {
                 String base64Audio = Base64.getEncoder().encodeToString(outBytes);
                 String message = "{ \"type\": \"input_audio_buffer.append\", \"audio\": \"" + base64Audio + "\" }";
                 webSocket.send(message);
-                log.info("4、开始接收数据流时间:{}", System.currentTimeMillis() / 1000);
                 // 3. 提交音频并请求转录
 //                webSocket.send("{\"type\": \"input_audio_buffer.commit\"}");
 //                webSocket.send("{\"type\": \"response.create\"}");

vetti-admin/src/main/java/com/vetti/web/controller/ai/AiCommonController.java

@@ -40,10 +40,8 @@ public class AiCommonController extends BaseController
     public AjaxResult handleTextToVice()
     {
         //你好,我是本次的面试官Vetti,请点击开始按钮后,做一段自我介绍.
-        elevenLabsClient.handleTextToVoice("Hello, I am Vetti, the interviewer for this interview. Please click the start button and give a self introduction","/Users/wangxiangshun/Desktop/临时文件/opening2.wav");
+        //本轮面试结束,谢谢您的配合,面试结果将稍后通知
-
+        elevenLabsClient.handleTextToVoice("This round of interview is over. Thank you for your cooperation. The interview results will be notified later","/Users/wangxiangshun/Desktop/临时文件/end.wav");
-        elevenLabsClient.handleTextToVoice("Good","/Users/wangxiangshun/Desktop/临时文件/good.wav");
-
 
         return success();
     }