新模型业务逻辑调整,以及衔接语更新

vetti-admin/src/main/java/com/vetti/socket/ChatWebSocketHandler.java

@@ -2,6 +2,7 @@ package com.vetti.socket;
 
 import cn.hutool.core.collection.CollectionUtil;
 import cn.hutool.core.date.DateUtil;
+import cn.hutool.core.util.ObjectUtil;
 import cn.hutool.core.util.StrUtil;
 import cn.hutool.json.JSONUtil;
 import com.vetti.common.ai.elevenLabs.ElevenLabsClient;
@@ -167,77 +168,44 @@ public class ChatWebSocketHandler {
                             promptJson = JSONUtil.toJsonStr(list);
                             cacheMsgMapData.put(session.getId(), promptJson);
                         }
+                        //开始返回衔接语
+                        String openingPathUrl = RuoYiConfig.getProfile() + VOICE_SYSTEM_DIR + "good.wav";
+                        sendVoiceBuffer(openingPathUrl, session);
+
                         //开始使用模型进行追问
-                        //把提问的文字发送给CPT(流式处理)
+                        //把提问的文字发送给GPT
-                        OpenAiStreamClient aiStreamClient = SpringUtils.getBean(OpenAiStreamClient.class);
+                        ChatGPTClient chatGPTClient = SpringUtils.getBean(ChatGPTClient.class);
                         log.info("AI提示词为:{}", promptJson);
-                        //先获取回答的评分,是否符合要求
+                        String resultMsg = chatGPTClient.handleAiChat(promptJson,"QA");
-                        Boolean isEndFlag = getInterviewScore(clientId,promptJson, session, "");
+                        if(StrUtil.isNotEmpty(resultMsg)) {
-                        if(isEndFlag){
+                            //开始解析返回结果
-                            log.info("面试回答符合条件规则,继续追问啦!!!!!");
+                            Map mapResultData = JSONUtil.toBean(resultMsg,Map.class);
-                            final int[] resultNum = {(int) (Math.random() * 2) + 1};
+                            //获取评分
-                            aiStreamClient.streamChat(promptJson, new OpenAiStreamListenerService() {
+                            //验证是否触发对应的规则
-                                @Override
+                            Boolean isEndFlag = getInterviewScore(resultMsg, session);
-                                public void onMessage(String content) {
+                            if(isEndFlag){
-                                    log.info("返回AI结果：{}", content.replaceAll("\n", ""));
+                                log.info("面试回答符合条件规则,继续追问啦!!!!!");
-                                    //获取1和2的随机数
+                                int resultNum = (int) (Math.random() * 2);
-                                    if(resultNum[0] == 1){
+                                List<String> questions = JSONUtil.toList(mapResultData.get("follow_up_questions").toString(), String.class);
-                                        content = "";
+                                String questionStr = questions.get(resultNum);
-                                    }
+                                if (StrUtil.isNotEmpty(questionStr)) {
-                                    resultNum[0] = resultNum[0] +1;
+                                    //开始进行语音输出-流式持续输出
-                                    log.info("提问的问题:{}",content);
+                                    sendTTSBuffer(clientId, questionStr, session);
-//                                    String contentData = content.replaceAll("\n", "");
+                                    // 实时输出内容
-                                    //返回是追问的问题
-                                    //获取的是追问的问题
-                                    if (StrUtil.isNotEmpty(content)) {
-                                        //对问题进行数据缓存
-                                        cacheQuestionResult.put(session.getId(), content);
-                                        //开始进行语音输出-流式持续输出
-                                        sendTTSBuffer(clientId, content, session);
-                                        // 实时输出内容
-                                        try {
-                                            //把文本也给前端返回去
-                                            Map<String, String> dataText = new HashMap<>();
-                                            dataText.put("type", "question");
-                                            dataText.put("content", content);
-                                            log.info("提问的问题文本发送啦:{}",JSONUtil.toJsonStr(dataText));
-                                            session.getBasicRemote().sendText(JSONUtil.toJsonStr(dataText));
-                                        } catch (Exception e) {
-                                            e.printStackTrace();
-                                        }
-                                    }
-                                }
-
-                                @Override
-                                public void onComplete() {
                                     try {
-                                        //开始往缓存中记录提问的问题
+                                        //把文本也给前端返回去
-                                        String questionResult = cacheQuestionResult.get(session.getId());
+                                        Map<String, String> dataText = new HashMap<>();
-                                        if (StrUtil.isNotEmpty(questionResult)) {
+                                        dataText.put("type", "question");
-                                            //获取缓存记录
+                                        dataText.put("content", questionStr);
-                                            String msgMapData = cacheMsgMapData.get(session.getId());
+                                        log.info("提问的问题文本发送啦:{}",JSONUtil.toJsonStr(dataText));
-                                            if (StrUtil.isNotEmpty(msgMapData)) {
+                                        session.getBasicRemote().sendText(JSONUtil.toJsonStr(dataText));
-                                                List<Map> list = JSONUtil.toList(msgMapData, Map.class);
-                                                Map<String, String> mapEntity = new HashMap<>();
-                                                mapEntity.put("role", "user");
-                                                mapEntity.put("content", "Question：" + questionResult + "\\nCandidate Answer：{}");
-                                                list.add(mapEntity);
-                                                cacheMsgMapData.put(session.getId(), JSONUtil.toJsonStr(list));
-                                            }
-                                        }
-                                        //清空问题
-                                        cacheQuestionResult.put(session.getId(), "");
-
                                     } catch (Exception e) {
-                                        throw new RuntimeException(e);
+                                        e.printStackTrace();
                                     }
+                                    //开始对问题进行缓存
+                                    recordQuestion(questionStr,session);
                                 }
-
+                            }
-                                @Override
-                                public void onError(Throwable throwable) {
-                                    throwable.printStackTrace();
-                                }
-                            });
                         }
                     }
                 } else if ("end".equals(resultFlag)) {
@@ -347,7 +315,7 @@ public class ChatWebSocketHandler {
             List<Map<String, String>> list = new LinkedList();
             Map<String, String> mapEntity = new HashMap<>();
             mapEntity.put("role", "system");
-            mapEntity.put("content", "You are an interviewer. Generate in-depth follow-up questions based on candidate responses.");
+            mapEntity.put("content", "You are a construction industry interview expert. Evaluate candidate responses and provide scores (1-5) and follow-up questions when needed. Always respond in JSON format.");
             list.add(mapEntity);
             //获取预设问题-直接TTS转换返回语音结果
             IHotakeProblemBaseInfoService problemBaseInfoService = SpringUtils.getBean(IHotakeProblemBaseInfoService.class);
@@ -358,7 +326,7 @@ public class ChatWebSocketHandler {
             if (CollectionUtil.isNotEmpty(baseInfoList)) {
                 HotakeProblemBaseInfo baseInfo = baseInfoList.get(0);
                 if (StrUtil.isNotEmpty(baseInfo.getContents())) {
-                    String[] qStrs = baseInfo.getContents().split(",");
+                    String[] qStrs = baseInfo.getContents().split("#AA#");
                     int random_index = (int) (Math.random() * qStrs.length);
                     //获取问题文本
                     String question = qStrs[random_index];
@@ -435,13 +403,9 @@ public class ChatWebSocketHandler {
         resultEntity.put("content", resultMsg);
         resultEntity.put("type", "score");
         try{
-            //返回评分语音
-//            sendTTSBuffer(clientId,resultMsg,session);
-
             //返回最终的评分结构
             log.info("返回最终的评分结构:{}",JSONUtil.toJsonStr(resultEntity));
             session.getBasicRemote().sendText(JSONUtil.toJsonStr(resultEntity));
-
         }catch (Exception e){
             e.printStackTrace();
         }
@@ -458,12 +422,12 @@ public class ChatWebSocketHandler {
      * @param content
      * @param session return false 立即结束面试
      */
-    private Boolean handleScoreRecord(String content, Session session) {
+    private Boolean handleScoreRecord(Object content, Session session) {
         Map<String, Integer> scoreRecordMap = cacheScoreResult.get(session.getId());
         log.info("获取评分结果:{}",content);
         //对评分进行处理
-        if (StrUtil.isNotEmpty(content)) {
+        if (ObjectUtil.isNotEmpty(content)) {
-            String[] strs = content.split("/");
+            String[] strs = content.toString().split("/");
             //取第一个数就是对应的评分
             log.info("获取的数据为:{}",strs[0]);
             BigDecimal score = new BigDecimal(strs[0].trim());
@@ -504,57 +468,28 @@ public class ChatWebSocketHandler {
     }
 
     /**
-     * 获取面试回答评分,并且校验是否结束面试
+     * 校验是否结束面试,结束后直接返回评分
      *
-     * @param promptJson 提示词数据json
+     * @param resultMsg 问答AI返回的结果数据
      * @param session    客户端会话
-     * @param position   职位
      */
-    private Boolean getInterviewScore(String clientId,String promptJson, Session session, String position) {
+    private Boolean getInterviewScore(String resultMsg, Session session) {
         //返回文本评分
-        //获取缓存记录
+        //开始解析返回结果
-        String msgMapData = cacheMsgMapData.get(session.getId());
+        Map mapResultData = JSONUtil.toBean(resultMsg,Map.class);
-        if (StrUtil.isNotEmpty(msgMapData)) {
+        //获取评分
-            List<Map> list = JSONUtil.toList(msgMapData, Map.class);
+        Object scoreStr = mapResultData.get("score");
-            //获取第一条数据记录
+        Object assessment = mapResultData.get("assessment");
-            Map<String, String> mapEntity = list.get(0);
-            //更新问题记录
-            mapEntity.put("role", "system");
-            mapEntity.put("content", "You are a construction industry interview expert. Rate candidate responses on a 1-5 scale and analyze key signals.");
-            //每个回答的内容前面要加上候选人的职位
-            if (StrUtil.isNotEmpty(position)) {
-                for (Map map : list) {
-                    if ("user".equals(map.get("role").toString())) {
-                        map.put("content", "Position: " + position + "\\n" + map.get("content"));
-                    }
-                }
-            }
-            promptJson = JSONUtil.toJsonStr(list);
-        }
-        log.info("评分AI提示词为:{}", promptJson);
-        ChatGPTClient gptClient = SpringUtils.getBean(ChatGPTClient.class);
-        String resultMsg = gptClient.handleAiChat(promptJson, "QA");
-        //评论格式为: Score: 3/5\nAssessment: Basically correct answer but lacks detail
-        String resultScore = "";
-        String scoreText = resultMsg;
-        if (StrUtil.isNotEmpty(resultMsg)) {
-            resultMsg = resultMsg.replaceAll("\n","#AA#");
-            String[] resultMsgs = resultMsg.split("#AA#");
-            resultScore = resultMsgs[0].replaceAll(SCORE_FLAG, "");
-        }
         //校验面试是否结束
-        Boolean flag = handleScoreRecord(resultScore, session);
+        Boolean flag = handleScoreRecord(scoreStr, session);
         try {
             if (!flag) {
                 //发送面试官结束语音流
                 String openingPathUrl = RuoYiConfig.getProfile() + VOICE_SYSTEM_DIR + "end.wav";
                 sendVoiceBuffer(openingPathUrl, session);
 
-                //返回评分语音
-//                sendTTSBuffer(clientId,scoreText,session);
-
                 Map<String, String> resultEntity = new HashMap<>();
-                resultEntity.put("content", scoreText);
+                resultEntity.put("content", scoreStr +"\n"+assessment);
                 resultEntity.put("type", "score");
                 //返回评分结果
                 log.info("返回最终的评分结果:{}",JSONUtil.toJsonStr(resultEntity));
@@ -567,5 +502,25 @@ public class ChatWebSocketHandler {
         return flag;
     }
 
+    /**
+     * 记录问题
+     * @param questionResult
+     * @param session
+     */
+    private void recordQuestion(String questionResult,Session session) {
+        if (StrUtil.isNotEmpty(questionResult)) {
+            //获取缓存记录
+            String msgMapData = cacheMsgMapData.get(session.getId());
+            if (StrUtil.isNotEmpty(msgMapData)) {
+                List<Map> list = JSONUtil.toList(msgMapData, Map.class);
+                Map<String, String> mapEntity = new HashMap<>();
+                mapEntity.put("role", "user");
+                mapEntity.put("content", "Question：" + questionResult + "\\nCandidate Answer：{}");
+                list.add(mapEntity);
+                cacheMsgMapData.put(session.getId(), JSONUtil.toJsonStr(list));
+            }
+        }
+    }
+
 }
 

vetti-admin/src/main/java/com/vetti/socket/ChatWebSocketHandler2.java

@@ -0,0 +1,571 @@
+package com.vetti.socket;
+
+import cn.hutool.core.collection.CollectionUtil;
+import cn.hutool.core.date.DateUtil;
+import cn.hutool.core.util.StrUtil;
+import cn.hutool.json.JSONUtil;
+import com.vetti.common.ai.elevenLabs.ElevenLabsClient;
+import com.vetti.common.ai.gpt.ChatGPTClient;
+import com.vetti.common.ai.gpt.OpenAiStreamClient;
+import com.vetti.common.ai.gpt.service.OpenAiStreamListenerService;
+import com.vetti.common.config.RuoYiConfig;
+import com.vetti.common.utils.spring.SpringUtils;
+import com.vetti.hotake.domain.HotakeProblemBaseInfo;
+import com.vetti.hotake.service.IHotakeProblemBaseInfoService;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.io.FileUtils;
+import org.springframework.stereotype.Component;
+
+import javax.websocket.*;
+import javax.websocket.server.PathParam;
+import javax.websocket.server.ServerEndpoint;
+import java.io.File;
+import java.io.IOException;
+import java.math.BigDecimal;
+import java.nio.ByteBuffer;
+import java.util.HashMap;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+
+/**
+ * 语音面试 web处理器
+ */
+@Slf4j
+@ServerEndpoint("/voice-websocket222222/{clientId}")
+@Component
+public class ChatWebSocketHandler2 {
+
+    /**
+     * 评分标记
+     */
+    private final String SCORE_FLAG = "Score:";
+
+
+    /**
+     * 缓存客户端流式解析的语音文本数据
+     */
+    private final Map<String, String> cacheClientTts = new ConcurrentHashMap<>();
+
+    /**
+     * 缓存客户端,标记是否是自我介绍后的初次问答
+     */
+    private final Map<String, String> cacheReplyFlag = new ConcurrentHashMap<>();
+
+    /**
+     * 缓存客户端,面试回答信息
+     */
+    private final Map<String, String> cacheMsgMapData = new ConcurrentHashMap<>();
+
+    /**
+     * 缓存客户端,AI提问的问题结果信息
+     */
+    private final Map<String, String> cacheQuestionResult = new ConcurrentHashMap<>();
+
+    /**
+     * 缓存客户端,得分结果记录
+     */
+    private final Map<String, Map<String, Integer>> cacheScoreResult = new ConcurrentHashMap<>();
+
+    // 语音文件保存目录
+    private static final String VOICE_STORAGE_DIR = "/voice_files/";
+
+    // 语音结果文件保存目录
+    private static final String VOICE_STORAGE_RESULT_DIR = "/voice_result_files/";
+
+    // 系统语音目录
+    private static final String VOICE_SYSTEM_DIR = "/system_files/";
+
+    public ChatWebSocketHandler2() {
+        // 初始化存储目录
+        File dir = new File(RuoYiConfig.getProfile() + VOICE_STORAGE_DIR);
+        if (!dir.exists()) {
+            dir.mkdirs();
+        }
+
+        File resultDir = new File(RuoYiConfig.getProfile() + VOICE_STORAGE_RESULT_DIR);
+        if (!resultDir.exists()) {
+            resultDir.mkdirs();
+        }
+    }
+
+    // 连接建立时调用
+    @OnOpen
+    public void onOpen(Session session, @PathParam("clientId") String clientId) {
+        log.info("WebSocket 链接已建立:{}", clientId);
+        log.info("WebSocket session 链接已建立:{}", session.getId());
+        cacheClientTts.put(clientId, new String());
+        //是初次自我介绍后的问答环节
+        cacheReplyFlag.put(session.getId(), "YES");
+        //初始化面试回答数据记录
+        cacheMsgMapData.put(session.getId(), "");
+        //初始化面试问题
+        cacheQuestionResult.put(session.getId(), "");
+        //初始化得分结果记录
+        Map<String, Integer> scoreResultData = new HashMap<>();
+        scoreResultData.put("0-1", 0);
+        scoreResultData.put("4-5", 0);
+        scoreResultData.put("2-3", 0);
+        scoreResultData.put("2-5", 0);
+        cacheScoreResult.put(session.getId(), scoreResultData);
+        //发送初始化面试官语音流
+        String openingPathUrl = RuoYiConfig.getProfile() + VOICE_SYSTEM_DIR + "opening.wav";
+        sendVoiceBuffer(openingPathUrl, session);
+    }
+
+    /**
+     * 接收文本消息
+     *
+     * @param session  客户端会话
+     * @param message  消息
+     *                 如:
+     *                 {
+     *                 "type": "start | done | end",
+     *                 "content": "内容"
+     *                 }
+     * @param clientId 用户ID
+     */
+    @OnMessage
+    public void onTextMessage(Session session, String message, @PathParam("clientId") String clientId) {
+        log.info("我是接收文本消息:{}", message);
+        try {
+            //处理文本结果
+            if (StrUtil.isNotEmpty(message)) {
+                Map<String, String> mapResult = JSONUtil.toBean(JSONUtil.parseObj(message), Map.class);
+                String resultFlag = mapResult.get("type");
+                if ("done".equals(resultFlag)) {
+                    //开始合并语音流
+                    String startFlag = cacheReplyFlag.get(session.getId());
+                    //语音结束,开始进行回答解析
+                    log.info("开始文本处理,客户端ID为:{}", clientId);
+                    String cacheResultText = mapResult.get("content");
+                    log.info("开始文本处理,面试者回答信息为:{}", cacheResultText);
+                    if (StrUtil.isEmpty(cacheResultText)) {
+                        cacheResultText = "";
+                    }
+
+                    //这是初次处理的逻辑
+                    if ("YES".equals(startFlag)) {
+                        //初始化-不走大模型-直接对候选人进行提问
+                        initializationQuestion(clientId, session);
+                        //发送完第一次消息后,直接删除标记,开始进行正常的面试问答流程
+                        cacheReplyFlag.put(session.getId(), "");
+                    } else {
+                        //开始根据面试者回答的问题,进行追问回答
+                        //获取面试者回答信息
+                        //获取缓存记录
+                        String promptJson = "";
+                        String msgMapData = cacheMsgMapData.get(session.getId());
+                        if (StrUtil.isNotEmpty(msgMapData)) {
+                            List<Map> list = JSONUtil.toList(msgMapData, Map.class);
+                            //获取最后一条数据记录
+                            Map<String, String> mapEntity = list.get(list.size() - 1);
+                            //更新问题记录
+                            String content = mapEntity.get("content");
+                            mapEntity.put("content", StrUtil.format(content, cacheResultText));
+                            promptJson = JSONUtil.toJsonStr(list);
+                            cacheMsgMapData.put(session.getId(), promptJson);
+                        }
+                        //开始使用模型进行追问
+                        //把提问的文字发送给CPT(流式处理)
+                        OpenAiStreamClient aiStreamClient = SpringUtils.getBean(OpenAiStreamClient.class);
+                        log.info("AI提示词为:{}", promptJson);
+                        //先获取回答的评分,是否符合要求
+                        Boolean isEndFlag = getInterviewScore(clientId,promptJson, session, "");
+                        if(isEndFlag){
+                            log.info("面试回答符合条件规则,继续追问啦!!!!!");
+                            final int[] resultNum = {(int) (Math.random() * 2) + 1};
+                            aiStreamClient.streamChat(promptJson, new OpenAiStreamListenerService() {
+                                @Override
+                                public void onMessage(String content) {
+                                    log.info("返回AI结果：{}", content.replaceAll("\n", ""));
+                                    //获取1和2的随机数
+                                    if(resultNum[0] == 1){
+                                        content = "";
+                                    }
+                                    resultNum[0] = resultNum[0] +1;
+                                    log.info("提问的问题:{}",content);
+//                                    String contentData = content.replaceAll("\n", "");
+                                    //返回是追问的问题
+                                    //获取的是追问的问题
+                                    if (StrUtil.isNotEmpty(content)) {
+                                        //对问题进行数据缓存
+                                        cacheQuestionResult.put(session.getId(), content);
+                                        //开始进行语音输出-流式持续输出
+                                        sendTTSBuffer(clientId, content, session);
+                                        // 实时输出内容
+                                        try {
+                                            //把文本也给前端返回去
+                                            Map<String, String> dataText = new HashMap<>();
+                                            dataText.put("type", "question");
+                                            dataText.put("content", content);
+                                            log.info("提问的问题文本发送啦:{}",JSONUtil.toJsonStr(dataText));
+                                            session.getBasicRemote().sendText(JSONUtil.toJsonStr(dataText));
+                                        } catch (Exception e) {
+                                            e.printStackTrace();
+                                        }
+                                    }
+                                }
+
+                                @Override
+                                public void onComplete() {
+                                    try {
+                                        //开始往缓存中记录提问的问题
+                                        String questionResult = cacheQuestionResult.get(session.getId());
+                                        if (StrUtil.isNotEmpty(questionResult)) {
+                                            //获取缓存记录
+                                            String msgMapData = cacheMsgMapData.get(session.getId());
+                                            if (StrUtil.isNotEmpty(msgMapData)) {
+                                                List<Map> list = JSONUtil.toList(msgMapData, Map.class);
+                                                Map<String, String> mapEntity = new HashMap<>();
+                                                mapEntity.put("role", "user");
+                                                mapEntity.put("content", "Question：" + questionResult + "\\nCandidate Answer：{}");
+                                                list.add(mapEntity);
+                                                cacheMsgMapData.put(session.getId(), JSONUtil.toJsonStr(list));
+                                            }
+                                        }
+                                        //清空问题
+                                        cacheQuestionResult.put(session.getId(), "");
+
+                                    } catch (Exception e) {
+                                        throw new RuntimeException(e);
+                                    }
+                                }
+
+                                @Override
+                                public void onError(Throwable throwable) {
+                                    throwable.printStackTrace();
+                                }
+                            });
+                        }
+                    }
+                } else if ("end".equals(resultFlag)) {
+                    log.info("面试结束啦!!!!!");
+                    handleInterviewEnd(clientId,session,"");
+                }
+            }
+        } catch (Exception e) {
+            e.printStackTrace();
+        }
+    }
+
+    // 接收二进制消息（流数据）
+    @OnMessage
+    public void onBinaryMessage(Session session, @PathParam("clientId") String clientId, ByteBuffer byteBuffer) {
+        log.info("我是接受二进制流的-客户端ID为:{}", clientId);
+    }
+
+    // 连接关闭时调用
+    @OnClose
+    public void onClose(Session session, CloseReason reason) {
+        System.out.println("WebSocket连接已关闭: " + session.getId() + ", 原因: " + reason.getReasonPhrase());
+        //链接关闭,清空内存
+        //是初次自我介绍后的问答环节
+        cacheReplyFlag.put(session.getId(), "");
+        //初始化面试回答数据记录
+        cacheMsgMapData.put(session.getId(), "");
+        //初始化面试问题
+        cacheQuestionResult.put(session.getId(), "");
+
+        cacheScoreResult.put(session.getId(), null);
+    }
+
+    // 发生错误时调用
+    @OnError
+    public void onError(Session session, Throwable throwable) {
+        System.err.println("WebSocket错误发生: " + throwable.getMessage());
+        throwable.printStackTrace();
+    }
+
+    /**
+     * File 转换成 ByteBuffer
+     *
+     * @param fileUrl 文件路径
+     * @return
+     */
+    private ByteBuffer convertFileToByteBuffer(String fileUrl) {
+        File file = new File(fileUrl);
+        try {
+            return ByteBuffer.wrap(FileUtils.readFileToByteArray(file));
+        } catch (Exception e) {
+            e.printStackTrace();
+        }
+        return null;
+    }
+
+    /**
+     * 发送语音流给前端
+     *
+     * @param pathUrl 语音文件地址
+     * @param session 客户端会话
+     */
+    private void sendVoiceBuffer(String pathUrl, Session session) {
+        try {
+            //文件转换成文件流
+            ByteBuffer outByteBuffer = convertFileToByteBuffer(pathUrl);
+            //发送文件流数据
+            session.getBasicRemote().sendBinary(outByteBuffer);
+            // 发送响应确认
+            log.info("已经成功发送了语音流给前端:{}", DateUtil.now());
+        } catch (IOException e) {
+            e.printStackTrace();
+        }
+    }
+
+    /**
+     * 发送文本转语音,发送语音流给前端
+     *
+     * @param clientId 用户ID
+     * @param content  文本内容
+     * @param session  客户端会话ID
+     */
+    private void sendTTSBuffer(String clientId, String content, Session session) {
+        String resultFileName = clientId + "_" + System.currentTimeMillis() + ".wav";
+        String resultPathUrl = RuoYiConfig.getProfile() + VOICE_STORAGE_RESULT_DIR + resultFileName;
+        ElevenLabsClient elevenLabsClient = SpringUtils.getBean(ElevenLabsClient.class);
+        elevenLabsClient.handleTextToVoice(content, resultPathUrl);
+        //持续返回数据流给客户端
+        log.info("发送语音流成功啦!!!!!!!");
+        sendVoiceBuffer(resultPathUrl, session);
+    }
+
+    /**
+     * 对候选者初次进行提问业务逻辑处理(初始化系统随机获取第一个问题)
+     *
+     * @param clientId 用户ID
+     * @param session  客户端会话
+     */
+    private void initializationQuestion(String clientId, Session session) {
+        try {
+            log.info("开始获取到clientid :{}",clientId);
+            //自我介绍结束后马上返回一个Good
+            //发送初始化面试官语音流
+            String openingPathUrl = RuoYiConfig.getProfile() + VOICE_SYSTEM_DIR + "good.wav";
+            sendVoiceBuffer(openingPathUrl, session);
+            //初始化面试流程的提问
+            List<Map<String, String>> list = new LinkedList();
+            Map<String, String> mapEntity = new HashMap<>();
+            mapEntity.put("role", "system");
+            mapEntity.put("content", "You are an interviewer. Generate in-depth follow-up questions based on candidate responses.");
+            list.add(mapEntity);
+            //获取预设问题-直接TTS转换返回语音结果
+            IHotakeProblemBaseInfoService problemBaseInfoService = SpringUtils.getBean(IHotakeProblemBaseInfoService.class);
+            HotakeProblemBaseInfo queryPro = new HotakeProblemBaseInfo();
+            queryPro.setUserId(Long.valueOf(clientId));
+            List<HotakeProblemBaseInfo> baseInfoList = problemBaseInfoService.selectHotakeProblemBaseInfoList(queryPro);
+            log.info("准备进行第一个问题的提问:{}",JSONUtil.toJsonStr(baseInfoList));
+            if (CollectionUtil.isNotEmpty(baseInfoList)) {
+                HotakeProblemBaseInfo baseInfo = baseInfoList.get(0);
+                if (StrUtil.isNotEmpty(baseInfo.getContents())) {
+                    String[] qStrs = baseInfo.getContents().split(",");
+                    int random_index = (int) (Math.random() * qStrs.length);
+                    //获取问题文本
+                    String question = qStrs[random_index];
+                    Map<String, String> mapEntityQ = new HashMap<>();
+                    mapEntityQ.put("role", "user");
+                    mapEntityQ.put("content", "Question：" + question + "\\nCandidate Answer：{}");
+                    list.add(mapEntityQ);
+                    log.info("开始提问啦:{}",JSONUtil.toJsonStr(list));
+                    //直接对该问题进行转换处理返回语音流
+                    log.info("第一个问题为:{}",question);
+                    sendTTSBuffer(clientId, question, session);
+                    //发送问题文本
+                    try {
+                        //把文本也给前端返回去
+                        Map<String, String> dataText = new HashMap<>();
+                        dataText.put("type", "question");
+                        dataText.put("content", question);
+                        log.info("提问的问题文本发送啦:{}",JSONUtil.toJsonStr(dataText));
+                        session.getBasicRemote().sendText(JSONUtil.toJsonStr(dataText));
+                    } catch (Exception e) {
+                        e.printStackTrace();
+                    }
+                }
+            }
+            //初始化记录提示词数据到-缓存中
+            cacheMsgMapData.put(session.getId(), JSONUtil.toJsonStr(list));
+        } catch (Exception e) {
+            e.printStackTrace();
+            log.error("面试流程初始化失败:{}", e.getMessage());
+        }
+    }
+
+    /**
+     * 处理面试结束业务逻辑
+     *
+     * @param session 客户端会话
+     * @param position 职位
+     */
+    private void handleInterviewEnd(String clientId,Session session,String position) {
+        //暂时的业务逻辑
+        //发送面试官结束语音流
+        String openingPathUrl = RuoYiConfig.getProfile() + VOICE_SYSTEM_DIR + "end.wav";
+        sendVoiceBuffer(openingPathUrl, session);
+
+        //返回文本评分
+        //处理模型提问逻辑
+        //获取缓存记录
+        String msgMapData = cacheMsgMapData.get(session.getId());
+        String promptJson = "";
+        if (StrUtil.isNotEmpty(msgMapData)) {
+            List<Map> list = JSONUtil.toList(msgMapData, Map.class);
+            //获取第一条数据记录
+            Map<String, String> mapEntity = list.get(0);
+            //更新问题记录
+            mapEntity.put("role", "system");
+            mapEntity.put("content", "You are a construction industry interview expert. Rate candidate responses on a 1-5 scale and analyze key signals.");
+            //每个回答的内容前面要加上候选人的职位
+            if (StrUtil.isNotEmpty(position)) {
+                for (Map map : list) {
+                    if ("user".equals(map.get("role").toString())) {
+                        map.put("content", "Position: " + position + "\\n" + map.get("content"));
+                    }
+                }
+            }
+            promptJson = JSONUtil.toJsonStr(list);
+
+            //结束回答要清空问答数据
+            cacheMsgMapData.put(session.getId(), "");
+        }
+        log.info("结束AI提示词为:{}", promptJson);
+        ChatGPTClient gptClient = SpringUtils.getBean(ChatGPTClient.class);
+        String resultMsg = gptClient.handleAiChat(promptJson, "QA");
+        Map<String, String> resultEntity = new HashMap<>();
+        resultEntity.put("content", resultMsg);
+        resultEntity.put("type", "score");
+        try{
+            //返回评分语音
+//            sendTTSBuffer(clientId,resultMsg,session);
+
+            //返回最终的评分结构
+            log.info("返回最终的评分结构:{}",JSONUtil.toJsonStr(resultEntity));
+            session.getBasicRemote().sendText(JSONUtil.toJsonStr(resultEntity));
+
+        }catch (Exception e){
+            e.printStackTrace();
+        }
+    }
+
+    /**
+     * 处理评分记录
+     * 触发规则:
+     * 1、获得 0-1 分 大于1次  立即结束面试
+     * 2、获取 4-5 分 大于3次  立即结束面试
+     * 3、获取 2-3 分 大于3次  立即结束面试
+     * 4、获取 2-5 分 大于4次  立即结束面试
+     *
+     * @param content
+     * @param session return false 立即结束面试
+     */
+    private Boolean handleScoreRecord(String content, Session session) {
+        Map<String, Integer> scoreRecordMap = cacheScoreResult.get(session.getId());
+        log.info("获取评分结果:{}",content);
+        //对评分进行处理
+        if (StrUtil.isNotEmpty(content)) {
+            String[] strs = content.split("/");
+            //取第一个数就是对应的评分
+            log.info("获取的数据为:{}",strs[0]);
+            BigDecimal score = new BigDecimal(strs[0].trim());
+            //记录Key为1
+            if (BigDecimal.ZERO.compareTo(score) <= 0 && BigDecimal.ONE.compareTo(score) >= 0) {
+                Integer n1 = scoreRecordMap.get("0-1") + 1;
+                scoreRecordMap.put("0-1", n1);
+                if (n1 > 1) {
+                    return false;
+                }
+            }
+            //记录Key为2
+            if (new BigDecimal(4).compareTo(score) <= 0 && new BigDecimal(5).compareTo(score) >= 0) {
+                Integer n1 = scoreRecordMap.get("4-5") + 1;
+                scoreRecordMap.put("4-5", n1);
+                if (n1 > 3) {
+                    return false;
+                }
+            }
+            //记录Key为3
+            if (new BigDecimal(2).compareTo(score) <= 0 && new BigDecimal(3).compareTo(score) >= 0) {
+                Integer n1 = scoreRecordMap.get("2-3") + 1;
+                scoreRecordMap.put("2-3", n1);
+                if (n1 > 3) {
+                    return false;
+                }
+            }
+            //记录Key为4
+            if (new BigDecimal(2).compareTo(score) <= 0 && new BigDecimal(5).compareTo(score) >= 0) {
+                Integer n1 = scoreRecordMap.get("2-5") + 1;
+                scoreRecordMap.put("2-5", n1);
+                if (n1 > 4) {
+                    return false;
+                }
+            }
+        }
+        return true;
+    }
+
+    /**
+     * 获取面试回答评分,并且校验是否结束面试
+     *
+     * @param promptJson 提示词数据json
+     * @param session    客户端会话
+     * @param position   职位
+     */
+    private Boolean getInterviewScore(String clientId,String promptJson, Session session, String position) {
+        //返回文本评分
+        //获取缓存记录
+        String msgMapData = cacheMsgMapData.get(session.getId());
+        if (StrUtil.isNotEmpty(msgMapData)) {
+            List<Map> list = JSONUtil.toList(msgMapData, Map.class);
+            //获取第一条数据记录
+            Map<String, String> mapEntity = list.get(0);
+            //更新问题记录
+            mapEntity.put("role", "system");
+            mapEntity.put("content", "You are a construction industry interview expert. Rate candidate responses on a 1-5 scale and analyze key signals.");
+            //每个回答的内容前面要加上候选人的职位
+            if (StrUtil.isNotEmpty(position)) {
+                for (Map map : list) {
+                    if ("user".equals(map.get("role").toString())) {
+                        map.put("content", "Position: " + position + "\\n" + map.get("content"));
+                    }
+                }
+            }
+            promptJson = JSONUtil.toJsonStr(list);
+        }
+        log.info("评分AI提示词为:{}", promptJson);
+        ChatGPTClient gptClient = SpringUtils.getBean(ChatGPTClient.class);
+        String resultMsg = gptClient.handleAiChat(promptJson, "QA");
+        //评论格式为: Score: 3/5\nAssessment: Basically correct answer but lacks detail
+        String resultScore = "";
+        String scoreText = resultMsg;
+        if (StrUtil.isNotEmpty(resultMsg)) {
+            resultMsg = resultMsg.replaceAll("\n","#AA#");
+            String[] resultMsgs = resultMsg.split("#AA#");
+            resultScore = resultMsgs[0].replaceAll(SCORE_FLAG, "");
+        }
+        //校验面试是否结束
+        Boolean flag = handleScoreRecord(resultScore, session);
+        try {
+            if (!flag) {
+                //发送面试官结束语音流
+                String openingPathUrl = RuoYiConfig.getProfile() + VOICE_SYSTEM_DIR + "end.wav";
+                sendVoiceBuffer(openingPathUrl, session);
+
+                //返回评分语音
+//                sendTTSBuffer(clientId,scoreText,session);
+
+                Map<String, String> resultEntity = new HashMap<>();
+                resultEntity.put("content", scoreText);
+                resultEntity.put("type", "score");
+                //返回评分结果
+                log.info("返回最终的评分结果:{}",JSONUtil.toJsonStr(resultEntity));
+                session.getBasicRemote().sendText(JSONUtil.toJsonStr(resultEntity));
+
+            }
+        } catch (Exception e) {
+            e.printStackTrace();
+        }
+        return flag;
+    }
+
+}
+

vetti-admin/src/main/java/com/vetti/web/controller/ai/AiCommonController.java

@@ -53,7 +53,7 @@ public class AiCommonController extends BaseController
         //你好,我是本次的面试官Vetti,请点击开始按钮后,做一段自我介绍.
         //你好,我是本次的面试官Vetti,请在三秒后,开始做一段自我介绍.
         //本轮面试结束,谢谢您的配合,面试结果将稍后通知
-        elevenLabsClient.handleTextToVoice("Hello, I am Vetti, the interviewer for this interview. Please begin a self introduction in three seconds","/Users/wangxiangshun/Desktop/临时文件/opening1.wav");
+        elevenLabsClient.handleTextToVoice("Ok, I have received your reply.","/Users/wangxiangshun/Desktop/临时文件/good.wav");
 
         return success();
     }

vetti-admin/src/main/resources/application-druid.yml

@@ -169,7 +169,7 @@ whisper:
 chatGpt:
     apiKey: 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
     apiUrl: https://api.openai.com/v1/chat/completions
-    model: ft:gpt-3.5-turbo-0125:vetti::CYl9OBMN
+    model: ft:gpt-3.5-turbo-0125:vetti:interview-unified:CaGyCXOr
     modelCV: ft:gpt-3.5-turbo-0125:vetti:vetti-resume-full:CYT0C8JG
     role: system
 

vetti-admin/target/classes/application-druid.yml

@@ -169,7 +169,7 @@ whisper:
 chatGpt:
     apiKey: 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
     apiUrl: https://api.openai.com/v1/chat/completions
-    model: ft:gpt-3.5-turbo-0125:vetti::CYl9OBMN
+    model: ft:gpt-3.5-turbo-0125:vetti:interview-unified:CaGyCXOr
     modelCV: ft:gpt-3.5-turbo-0125:vetti:vetti-resume-full:CYT0C8JG
     role: system
 

vetti-common/src/main/java/com/vetti/common/ai/elevenLabs/ElevenLabsClient.java

@@ -57,13 +57,11 @@ public class ElevenLabsClient {
         Gson gson = new Gson();
         StringEntity entity = new StringEntity(gson.toJson(payload), ContentType.APPLICATION_JSON);
         httpPost.setEntity(entity);
-
         try (CloseableHttpResponse response = httpClient.execute(httpPost)) {
             HttpEntity responseEntity = response.getEntity();
             if (responseEntity != null) {
                 try (InputStream inputStream = responseEntity.getContent();
                      FileOutputStream outputStream = new FileOutputStream(outputFilePath)) {
-
                     byte[] buffer = new byte[4096];
                     int bytesRead;
                     while ((bytesRead = inputStream.read(buffer)) != -1) {