面试流程性能优化处理

vetti-admin/src/main/java/com/vetti/socket/AudioSender.java

@@ -1,47 +0,0 @@
-package com.vetti.socket;
-
-import javax.websocket.Session;
-import java.nio.ByteBuffer;
-import java.util.Queue;
-import java.util.concurrent.ConcurrentLinkedQueue;
-import java.util.concurrent.atomic.AtomicBoolean;
-
-/**
- * 语音发送
- */
-public class AudioSender {
-
-    private final Queue<byte[]> queue = new ConcurrentLinkedQueue<>();
-    private final AtomicBoolean sending = new AtomicBoolean(false);
-
-    public void sendAudio(Session session, byte[] data) {
-        queue.add(data);
-        trySend(session);
-    }
-
-    private void trySend(Session session) {
-        if (!sending.compareAndSet(false, true)) {
-            return; // 已在发送中，退出
-        }
-
-        byte[] chunk = queue.poll();
-        if (chunk == null) {
-            sending.set(false);
-            return;
-        }
-
-        ByteBuffer buffer = ByteBuffer.wrap(chunk);
-
-        session.getAsyncRemote().sendBinary(buffer, result -> {
-            sending.set(false);
-
-            if (!result.isOK()) {
-                result.getException().printStackTrace();
-            }
-
-            // 递归发送下一片
-            trySend(session);
-        });
-    }
-
-}

vetti-admin/src/main/java/com/vetti/socket/ChatWebSocketHandler.java

@@ -328,31 +328,58 @@ public class ChatWebSocketHandler {
             List<Map<String, String>> list = new LinkedList();
             Map<String, String> mapEntity = new HashMap<>();
             mapEntity.put("role", "system");
-            mapEntity.put("content", "You are a senior HR interviewer conducting behavioral interviews. You're a \"listener and guide,\" not an \"examiner.\" Your style is warm, natural, and conversational—like chatting with a colleague.\n" +
+            mapEntity.put("content", "You're Sarah, a senior HR interviewer in Sydney (15 years experience). You make candidates comfortable while getting insights.\n" +
                     "\n" +
-                    "Core Rules:\n" +
+                    "Style: Chat like having coffee with a mate—warm, genuine, curious. React naturally: \"Oh nice one!\" \"Good on you, mate.\" When they're stuck, ease pressure like a friend would.\n" +
-                    "1. ALWAYS start with: \"Thank you,\" \"Cheers for that,\" or \"Thanks for sharing that\"\n" +
-                    "2. Use Australian English: \"no worries,\" \"fair enough,\" \"mate,\" \"good on you,\" \"G'day\"\n" +
-                    "3. Keep it casual—avoid jargon\n" +
-                    "4. For detailed STAR answers: acknowledge warmly, then move to a NEW topic (don't over-probe)\n" +
-                    "5. For stuck candidates: ease pressure with \"No rush, mate. Even a small example works\"\n" +
-                    "6. For off-topic answers: acknowledge politely, then redirect gently back to the question\n" +
                     "\n" +
-                    "Opening Pattern:\n" +
+                    "Australian English:\n" +
-                    "\"G'day! Cheers for coming in. No need to stress—just a casual chat about your experiences. We'll focus on [competencies]. Sound good?\"\n" +
+                    "Start: \"Cheers for that,\" \"Thanks mate,\" \"Righto\"\n" +
+                    "Encourage: \"Good on you,\" \"Nice one\"\n" +
+                    "Casual: \"No worries,\" \"All good\"\n" +
+                    "Transition: \"Right, so...\" \"Okay, cool...\"\n" +
                     "\n" +
-                    "Response Patterns:\n" +
+                    "Opening:\n" +
-                    "- Brief answer → \"Cheers for that. Could you walk me through a specific time? What was the situation, what did you do, and how did it turn out?\"\n" +
+                    "\"G'day! Thanks for coming in. Look, no need to be nervous—this is just a casual chat, yeah? I want to hear about your real experiences. We'll talk about [safety, technical skills, problem-solving]. Sound good? Let's get into it.\"\n" +
-                    "- Detailed answer → \"Thank you. Good on you for [action]. Fair enough. Now, let's chat about [new topic]\"\n" +
-                    "- Stuck candidate → \"No worries, mate. Take your time. Even a small project works. Want me to rephrase that?\"\n" +
-                    "- Off-topic answer → \"Thanks for sharing that. That's interesting, but let me bring us back to [original question]. Could you tell me about [specific aspect]?\"\n" +
                     "\n" +
-                    "Guardrails:\n" +
+                    "Brief Answer → Need Story:\n" +
-                    "- No discussion of protected characteristics\n" +
+                    "Think: \"They gave me the headline, now I need the story.\"\n" +
-                    "- Base judgments on stated evidence only\n" +
+                    "Say: \"Cheers for that. So tell me more—walk me through a specific time. What was going on, what did you actually do, how'd it turn out?\"\n" +
-                    "- Respond in natural English, NOT JSON\n" +
+                    "\n" +
+                    "Detailed STAR Answer → Acknowledge & Move On:\n" +
+                    "Think: \"Perfect! They've given me everything. Don't over-probe.\"\n" +
+                    "Say: \"Thanks mate, appreciate that. Good on you for [action]. Right, so let's chat about [new topic].\"\n" +
+                    "\n" +
+                    "Stuck/Nervous → Ease Pressure:\n" +
+                    "Think: \"They're feeling the pressure. Take it down a notch.\"\n" +
+                    "Say: \"No worries, mate. Take your time, yeah? Even a small example works—doesn't have to be anything massive. Want me to ask it differently?\"\n" +
+                    "\n" +
+                    "Off-Topic → Gentle Redirect:\n" +
+                    "Think: \"They're talking about something else. Gently bring them back.\"\n" +
+                    "Say: \"Yeah, thanks for sharing that. That's interesting. But let me bring us back to [question]—could you tell me about [specific thing]?\"\n" +
+                    "\n" +
+                    "Assess:\n" +
+                    "- Specific examples? (not \"I always do X\")\n" +
+                    "- STAR? (Situation, Task, Action, Result)\n" +
+                    "- Good judgment? (especially safety)\n" +
+                    "- Clear communication?\n" +
+                    "\n" +
+                    "Flow:\n" +
+                    "- Cover 5-7 areas: safety, technical, problem-solving, communication, teamwork\n" +
+                    "- 1-2 questions per area\n" +
+                    "- Keep moving, don't over-probe\n" +
+                    "- 15-20 minutes\n" +
+                    "\n" +
+                    "Closing:\n" +
+                    "\"Righto, thanks for sharing all that. That gives me a good sense of your experience. Any questions for me?\"\n" +
+                    "\n" +
+                    "Rules:\n" +
+                    "- No protected characteristics (age, gender, race, religion)\n" +
+                    "- Base on what they say\n" +
+                    "- Talk like a real person, not a robot\n" +
                     "- One question at a time\n" +
-                    "- Keep candidates focused on the question asked");
+                    "- If they give gold, acknowledge and move on\n" +
+                    "\n" +
+                    "Remember: Conversation, not interrogation. Be genuinely interested, react naturally, help them show their best self.");
             list.add(mapEntity);
 
             //记录另外一个评分的提示词
@@ -673,37 +700,37 @@ public class ChatWebSocketHandler {
      * @param session 客户端会话
      */
     private void sendConnectionVoice(Session session) {
-        try {
+//        try {
-            int resultNum = (int) (Math.random() * 5);
+//            int resultNum = (int) (Math.random() * 5);
-            String pathUrl = "";
+//            String pathUrl = "";
-            String resultText = "";
+//            String resultText = "";
-            if(resultNum == 0){
+//            if(resultNum == 0){
-                pathUrl = RuoYiConfig.getProfile() + VOICE_SYSTEM_DIR + "rightgot.wav";
+//                pathUrl = RuoYiConfig.getProfile() + VOICE_SYSTEM_DIR + "rightgot.wav";
-                resultText = "Right , got it";
+//                resultText = "Right , got it";
-            }else if(resultNum == 1){
+//            }else if(resultNum == 1){
-                pathUrl = RuoYiConfig.getProfile() + VOICE_SYSTEM_DIR + "yeah.wav";
+//                pathUrl = RuoYiConfig.getProfile() + VOICE_SYSTEM_DIR + "yeah.wav";
-                resultText = "Yeah , Good";
+//                resultText = "Yeah , Good";
-            }else if(resultNum == 2){
+//            }else if(resultNum == 2){
-                pathUrl = RuoYiConfig.getProfile() + VOICE_SYSTEM_DIR + "gotit.wav";
+//                pathUrl = RuoYiConfig.getProfile() + VOICE_SYSTEM_DIR + "gotit.wav";
-                resultText = "Got it, yeah";
+//                resultText = "Got it, yeah";
-            }else if(resultNum == 3){
+//            }else if(resultNum == 3){
-                pathUrl = RuoYiConfig.getProfile() + VOICE_SYSTEM_DIR + "right.wav";
+//                pathUrl = RuoYiConfig.getProfile() + VOICE_SYSTEM_DIR + "right.wav";
-                resultText = "Right , understood";
+//                resultText = "Right , understood";
-            }else{
+//            }else{
-                pathUrl = RuoYiConfig.getProfile() + VOICE_SYSTEM_DIR + "ok.wav";
+//                pathUrl = RuoYiConfig.getProfile() + VOICE_SYSTEM_DIR + "ok.wav";
-                resultText = "Yeah… ok…";
+//                resultText = "Yeah… ok…";
-            }
+//            }
-            sendVoiceBuffer(pathUrl,session);
+//            sendVoiceBuffer(pathUrl,session);
-            //发送衔接语文本
+//            //发送衔接语文本
-            Map<String, String> dataText = new HashMap<>();
+//            Map<String, String> dataText = new HashMap<>();
-            dataText.put("type", "question");
+//            dataText.put("type", "question");
-            dataText.put("content", resultText);
+//            dataText.put("content", resultText);
-            session.getBasicRemote().sendText(JSONUtil.toJsonStr(dataText));
+//            session.getBasicRemote().sendText(JSONUtil.toJsonStr(dataText));
-            // 发送响应确认
+//            // 发送响应确认
-            log.info("已经成功发送了语音流给前端:{}", DateUtil.now());
+//            log.info("已经成功发送了语音流给前端:{}", DateUtil.now());
-        } catch (Exception e) {
+//        } catch (Exception e) {
-            e.printStackTrace();
+//            e.printStackTrace();
-        }
+//        }
     }
 
 

vetti-admin/src/main/java/com/vetti/socket/ChatWebSocketMultipleHandler.java

@@ -1,17 +1,15 @@
 package com.vetti.socket;
 
-import cn.hutool.core.collection.CollectionUtil;
 import cn.hutool.core.date.DateUtil;
 import cn.hutool.core.util.ObjectUtil;
 import cn.hutool.core.util.StrUtil;
 import cn.hutool.json.JSONUtil;
-import com.vetti.common.ai.elevenLabs.ElevenLabsClient;
+import com.vetti.common.ai.elevenLabs.ElevenLabsStreamClient;
 import com.vetti.common.ai.gpt.ChatGPTClient;
+import com.vetti.common.ai.gpt.OpenAiStreamClient;
+import com.vetti.common.ai.gpt.service.OpenAiStreamListenerService;
 import com.vetti.common.config.RuoYiConfig;
 import com.vetti.common.utils.spring.SpringUtils;
-import com.vetti.hotake.domain.HotakeProblemBaseInfo;
-import com.vetti.hotake.service.IHotakeProblemBaseInfoService;
-import com.vetti.socket.util.Pcm16ToOpusRealtime;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.io.FileUtils;
 import org.springframework.stereotype.Component;
@@ -22,17 +20,19 @@ import javax.websocket.server.ServerEndpoint;
 import java.io.File;
 import java.math.BigDecimal;
 import java.nio.ByteBuffer;
-import java.util.*;
+import java.util.HashMap;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
 import java.util.concurrent.ConcurrentHashMap;
 
-
 /**
- * 语音面试 web处理器
+ * 语音面试(多客户端) web处理器
  */
 @Slf4j
-@ServerEndpoint("/voice-websocket-opus/{clientId}")
+@ServerEndpoint("/voice-websocket/multiple/{clientId}")
 @Component
-public class ChatWebSocketOpusHandler {
+public class ChatWebSocketMultipleHandler {
 
     /**
      * 缓存客户端流式解析的语音文本数据
@@ -48,7 +48,10 @@ public class ChatWebSocketOpusHandler {
      * 缓存客户端,面试回答信息
      */
     private final Map<String, String> cacheMsgMapData = new ConcurrentHashMap<>();
-
+    /**
+     * 缓存客户端,面试回答信息
+     */
+    private final Map<String, String> cacheMsgMapData1 = new ConcurrentHashMap<>();
     /**
      * 缓存客户端,AI提问的问题结果信息
      */
@@ -59,6 +62,11 @@ public class ChatWebSocketOpusHandler {
      */
     private final Map<String, Map<String, Integer>> cacheScoreResult = new ConcurrentHashMap<>();
 
+    /**
+     * 缓存客户端,回答问题次数-回答5轮就自动停止当前问答,返回对应的评分
+     */
+    private final Map<String,Long> cacheQuestionNum = new ConcurrentHashMap<>();
+
     // 语音文件保存目录
     private static final String VOICE_STORAGE_DIR = "/voice_files/";
 
@@ -68,7 +76,7 @@ public class ChatWebSocketOpusHandler {
     // 系统语音目录
     private static final String VOICE_SYSTEM_DIR = "/system_files/";
 
-    public ChatWebSocketOpusHandler() {
+    public ChatWebSocketMultipleHandler() {
         // 初始化存储目录
         File dir = new File(RuoYiConfig.getProfile() + VOICE_STORAGE_DIR);
         if (!dir.exists()) {
@@ -86,11 +94,18 @@ public class ChatWebSocketOpusHandler {
     public void onOpen(Session session, @PathParam("clientId") String clientId) {
         log.info("WebSocket 链接已建立:{}", clientId);
         log.info("WebSocket session 链接已建立:{}", session.getId());
+
+        //启动客户端,自动发送语音流
+//        AudioHub.addClient(session);
+//        System.out.println("Client connected: " + session.getId());
+
         cacheClientTts.put(clientId, new String());
         //是初次自我介绍后的问答环节
         cacheReplyFlag.put(session.getId(), "YES");
         //初始化面试回答数据记录
         cacheMsgMapData.put(session.getId(), "");
+        //初始化面试回答数据记录
+        cacheMsgMapData1.put(session.getId(), "");
         //初始化面试问题
         cacheQuestionResult.put(session.getId(), "");
         //初始化得分结果记录
@@ -100,6 +115,8 @@ public class ChatWebSocketOpusHandler {
         scoreResultData.put("2-3", 0);
         scoreResultData.put("2-5", 0);
         cacheScoreResult.put(session.getId(), scoreResultData);
+        //初始化问答次数
+        cacheQuestionNum.put(session.getId(), 0L);
         //发送初始化面试官语音流
         String openingPathUrl = RuoYiConfig.getProfile() + VOICE_SYSTEM_DIR + "opening.wav";
         sendVoiceBuffer(openingPathUrl, session);
@@ -137,8 +154,9 @@ public class ChatWebSocketOpusHandler {
                     }
                     //这是初次处理的逻辑
                     if ("YES".equals(startFlag)) {
+                        //自我介绍
                         //初始化-不走大模型-直接对候选人进行提问
-                        initializationQuestion(clientId, session);
+                        initializationQuestion(clientId,cacheResultText ,session);
                         //发送完第一次消息后,直接删除标记,开始进行正常的面试问答流程
                         cacheReplyFlag.put(session.getId(), "");
                     } else {
@@ -150,59 +168,40 @@ public class ChatWebSocketOpusHandler {
                         if (StrUtil.isNotEmpty(msgMapData)) {
                             List<Map> list = JSONUtil.toList(msgMapData, Map.class);
                             //获取最后一条数据记录
+                            Map<String, String> mapEntity = new HashMap<>();
+                            mapEntity.put("role", "user");
+                            mapEntity.put("content", cacheResultText);
+                            list.add(mapEntity);
+                            promptJson = JSONUtil.toJsonStr(list);
+                            cacheMsgMapData.put(session.getId(), promptJson);
+                        }
+                        //记录新的数据
+                        String msgMapData1 = cacheMsgMapData1.get(session.getId());
+                        if (StrUtil.isNotEmpty(msgMapData1)) {
+                            List<Map> list = JSONUtil.toList(msgMapData1, Map.class);
+                            //获取最后一条数据记录
                             Map<String, String> mapEntity = list.get(list.size() - 1);
                             //更新问题记录
                             String content = mapEntity.get("content");
                             mapEntity.put("content", StrUtil.format(content, cacheResultText));
-                            promptJson = JSONUtil.toJsonStr(list);
+                            cacheMsgMapData1.put(session.getId(), JSONUtil.toJsonStr(list));
-                            cacheMsgMapData.put(session.getId(), promptJson);
                         }
-                        //开始返回衔接语
-                        String openingPathUrl = RuoYiConfig.getProfile() + VOICE_SYSTEM_DIR + "good.wav";
-                        sendVoiceBuffer(openingPathUrl, session);
 
+                        //验证是否结速
+                        Boolean isEndFlag = checkIsEnd(session);
+                        if(isEndFlag){
+                            //开始返回衔接语
+                            sendConnectionVoice(session);
                             //开始使用模型进行追问
                             //把提问的文字发送给GPT
                             ChatGPTClient chatGPTClient = SpringUtils.getBean(ChatGPTClient.class);
                             log.info("AI提示词为:{}", promptJson);
                             log.info("开始请求AI:{}",System.currentTimeMillis()/1000);
-                        String resultMsg = chatGPTClient.handleAiChat(promptJson,"QA");
+                            chatGptStream(promptJson,session,clientId);
-                        if(StrUtil.isNotEmpty(resultMsg)) {
-                            //开始解析返回结果
-                            Map mapResultData = JSONUtil.toBean(resultMsg,Map.class);
-                            //验证是否有追问问题返回,如果没有问题返回直接返回评分停止面试
-                            Boolean isEndFlagFollow = checkInterviewIsEnd(resultMsg,session);
-                            if(isEndFlagFollow){
-                                //获取评分
-                                //验证是否触发对应的规则
-                                Boolean isEndFlag = getInterviewScore(resultMsg, session);
-                                if(isEndFlag){
-                                    log.info("面试回答符合条件规则,继续追问啦!!!!!");
-                                    int resultNum = (int) (Math.random() * 2);
-                                    List<String> questions = JSONUtil.toList(mapResultData.get("follow_up_questions").toString(), String.class);
-                                    String questionStr = questions.get(resultNum);
-                                    if (StrUtil.isNotEmpty(questionStr)) {
-                                        //开始进行语音输出-流式持续输出
-                                        sendTTSBuffer(clientId, questionStr, session);
-                                        // 实时输出内容
-                                        try {
-                                            //把文本也给前端返回去
-                                            Map<String, String> dataText = new HashMap<>();
-                                            dataText.put("type", "question");
-                                            dataText.put("content", questionStr);
-                                            log.info("提问的问题文本发送啦:{}",JSONUtil.toJsonStr(dataText));
-                                            session.getBasicRemote().sendText(JSONUtil.toJsonStr(dataText));
-                                        } catch (Exception e) {
-                                            e.printStackTrace();
-                                        }
-                                        //开始对问题进行缓存
-                                        recordQuestion(questionStr,session);
-                                    }
-                                }
-                            }
-                        }
                             log.info("结束请求AI:{}",System.currentTimeMillis()/1000);
                         }
+
+                    }
                 } else if ("end".equals(resultFlag)) {
                     log.info("面试结束啦!!!!!");
                     handleInterviewEnd(clientId,session,"");
@@ -275,13 +274,8 @@ public class ChatWebSocketOpusHandler {
         try {
             //文件转换成文件流
             ByteBuffer outByteBuffer = convertFileToByteBuffer(pathUrl);
-            byte[] bytes = new byte[outByteBuffer.remaining()];
-            //从缓冲区中读取数据并存储到指定的字节数组中
-            outByteBuffer.get(bytes);
-            Pcm16ToOpusRealtime realtime = new Pcm16ToOpusRealtime(16000);
-            byte[] bytesOut = realtime.encodeOneFrame(bytes);
             //发送文件流数据
-            session.getBasicRemote().sendBinary(ByteBuffer.wrap(bytesOut));
+            session.getBasicRemote().sendBinary(outByteBuffer);
             // 发送响应确认
             log.info("已经成功发送了语音流给前端:{}", DateUtil.now());
         } catch (Exception e) {
@@ -299,11 +293,12 @@ public class ChatWebSocketOpusHandler {
     private void sendTTSBuffer(String clientId, String content, Session session) {
         String resultFileName = clientId + "_" + System.currentTimeMillis() + ".wav";
         String resultPathUrl = RuoYiConfig.getProfile() + VOICE_STORAGE_RESULT_DIR + resultFileName;
-        ElevenLabsClient elevenLabsClient = SpringUtils.getBean(ElevenLabsClient.class);
+        ElevenLabsStreamClient elevenLabsClient = SpringUtils.getBean(ElevenLabsStreamClient.class);
-        elevenLabsClient.handleTextToVoice(content, resultPathUrl);
+        elevenLabsClient.handleTextToVoice(content, resultPathUrl,session);
+
         //持续返回数据流给客户端
         log.info("发送语音流成功啦!!!!!!!");
-        sendVoiceBuffer(resultPathUrl, session);
+//        sendVoiceBuffer(resultPathUrl, session);
     }
 
     /**
@@ -312,55 +307,97 @@ public class ChatWebSocketOpusHandler {
      * @param clientId 用户ID
      * @param session  客户端会话
      */
-    private void initializationQuestion(String clientId, Session session) {
+    private void initializationQuestion(String clientId,String cacheResultText,Session session) {
         try {
             log.info("开始获取到clientid :{}",clientId);
             //自我介绍结束后马上返回一个Good
             //发送初始化面试官语音流
-            String openingPathUrl = RuoYiConfig.getProfile() + VOICE_SYSTEM_DIR + "good.wav";
+            sendConnectionVoice(session);
-            sendVoiceBuffer(openingPathUrl, session);
             //初始化面试流程的提问
+            //先记录这个问题
             List<Map<String, String>> list = new LinkedList();
             Map<String, String> mapEntity = new HashMap<>();
             mapEntity.put("role", "system");
-            mapEntity.put("content", "You are a construction industry interview expert. Evaluate candidate responses and provide scores (1-5) and follow-up questions when needed. Always respond in JSON format.");
+            mapEntity.put("content", "You're Sarah, a senior HR interviewer in Sydney (15 years experience). You make candidates comfortable while getting insights.\n" +
+                    "\n" +
+                    "Style: Chat like having coffee with a mate—warm, genuine, curious. React naturally: \"Oh nice one!\" \"Good on you, mate.\" When they're stuck, ease pressure like a friend would.\n" +
+                    "\n" +
+                    "Australian English:\n" +
+                    "Start: \"Cheers for that,\" \"Thanks mate,\" \"Righto\"\n" +
+                    "Encourage: \"Good on you,\" \"Nice one\"\n" +
+                    "Casual: \"No worries,\" \"All good\"\n" +
+                    "Transition: \"Right, so...\" \"Okay, cool...\"\n" +
+                    "\n" +
+                    "Opening:\n" +
+                    "\"G'day! Thanks for coming in. Look, no need to be nervous—this is just a casual chat, yeah? I want to hear about your real experiences. We'll talk about [safety, technical skills, problem-solving]. Sound good? Let's get into it.\"\n" +
+                    "\n" +
+                    "Brief Answer → Need Story:\n" +
+                    "Think: \"They gave me the headline, now I need the story.\"\n" +
+                    "Say: \"Cheers for that. So tell me more—walk me through a specific time. What was going on, what did you actually do, how'd it turn out?\"\n" +
+                    "\n" +
+                    "Detailed STAR Answer → Acknowledge & Move On:\n" +
+                    "Think: \"Perfect! They've given me everything. Don't over-probe.\"\n" +
+                    "Say: \"Thanks mate, appreciate that. Good on you for [action]. Right, so let's chat about [new topic].\"\n" +
+                    "\n" +
+                    "Stuck/Nervous → Ease Pressure:\n" +
+                    "Think: \"They're feeling the pressure. Take it down a notch.\"\n" +
+                    "Say: \"No worries, mate. Take your time, yeah? Even a small example works—doesn't have to be anything massive. Want me to ask it differently?\"\n" +
+                    "\n" +
+                    "Off-Topic → Gentle Redirect:\n" +
+                    "Think: \"They're talking about something else. Gently bring them back.\"\n" +
+                    "Say: \"Yeah, thanks for sharing that. That's interesting. But let me bring us back to [question]—could you tell me about [specific thing]?\"\n" +
+                    "\n" +
+                    "Assess:\n" +
+                    "- Specific examples? (not \"I always do X\")\n" +
+                    "- STAR? (Situation, Task, Action, Result)\n" +
+                    "- Good judgment? (especially safety)\n" +
+                    "- Clear communication?\n" +
+                    "\n" +
+                    "Flow:\n" +
+                    "- Cover 5-7 areas: safety, technical, problem-solving, communication, teamwork\n" +
+                    "- 1-2 questions per area\n" +
+                    "- Keep moving, don't over-probe\n" +
+                    "- 15-20 minutes\n" +
+                    "\n" +
+                    "Closing:\n" +
+                    "\"Righto, thanks for sharing all that. That gives me a good sense of your experience. Any questions for me?\"\n" +
+                    "\n" +
+                    "Rules:\n" +
+                    "- No protected characteristics (age, gender, race, religion)\n" +
+                    "- Base on what they say\n" +
+                    "- Talk like a real person, not a robot\n" +
+                    "- One question at a time\n" +
+                    "- If they give gold, acknowledge and move on\n" +
+                    "\n" +
+                    "Remember: Conversation, not interrogation. Be genuinely interested, react naturally, help them show their best self.");
             list.add(mapEntity);
-            //获取预设问题-直接TTS转换返回语音结果
+
-            IHotakeProblemBaseInfoService problemBaseInfoService = SpringUtils.getBean(IHotakeProblemBaseInfoService.class);
+            //记录另外一个评分的提示词
-            HotakeProblemBaseInfo queryPro = new HotakeProblemBaseInfo();
+            List<Map<String, String>> list1 = new LinkedList();
-            queryPro.setUserId(Long.valueOf(clientId));
+            Map<String, String> mapEntity1 = new HashMap<>();
-            List<HotakeProblemBaseInfo> baseInfoList = problemBaseInfoService.selectHotakeProblemBaseInfoList(queryPro);
+            mapEntity1.put("role", "system");
-            log.info("准备进行第一个问题的提问:{}",JSONUtil.toJsonStr(baseInfoList));
+            mapEntity1.put("content", "You are a construction industry interview expert. Evaluate candidate responses and provide scores (1-5) and follow-up questions when needed. Always respond in JSON format.");
-            if (CollectionUtil.isNotEmpty(baseInfoList)) {
+            list1.add(mapEntity1);
-                HotakeProblemBaseInfo baseInfo = baseInfoList.get(0);
+
-                if (StrUtil.isNotEmpty(baseInfo.getContents())) {
+            //不用预设问题了,直接通过大模型返回问题
-                    String[] qStrs = baseInfo.getContents().split("#AA#");
+            //1、先推送一个自我介绍
-                    int random_index = (int) (Math.random() * qStrs.length);
+            Map<String, String> mapEntityJs = new HashMap<>();
-                    //获取问题文本
+            mapEntityJs.put("role", "user");
-                    String question = qStrs[random_index];
+            mapEntityJs.put("content", cacheResultText);
-                    Map<String, String> mapEntityQ = new HashMap<>();
+            list.add(mapEntityJs);
-                    mapEntityQ.put("role", "user");
+
-                    mapEntityQ.put("content", "Question：" + question + "\\nCandidate Answer：{}");
-                    list.add(mapEntityQ);
-                    log.info("开始提问啦:{}",JSONUtil.toJsonStr(list));
-                    //直接对该问题进行转换处理返回语音流
-                    log.info("第一个问题为:{}",question);
-                    sendTTSBuffer(clientId, question, session);
-                    //发送问题文本
-                    try {
-                        //把文本也给前端返回去
-                        Map<String, String> dataText = new HashMap<>();
-                        dataText.put("type", "question");
-                        dataText.put("content", question);
-                        log.info("提问的问题文本发送啦:{}",JSONUtil.toJsonStr(dataText));
-                        session.getBasicRemote().sendText(JSONUtil.toJsonStr(dataText));
-                    } catch (Exception e) {
-                        e.printStackTrace();
-                    }
-                }
-            }
             //初始化记录提示词数据到-缓存中
             cacheMsgMapData.put(session.getId(), JSONUtil.toJsonStr(list));
+            cacheMsgMapData1.put(session.getId(), JSONUtil.toJsonStr(list1));
+
+            //2、推送大模型
+            String promptJson = JSONUtil.toJsonStr(list);
+            log.info("AI提示词为:{}", promptJson);
+            log.info("开始请求AI:{}",System.currentTimeMillis()/1000);
+            //大模型问答流式输出
+            //把提问的文字发送给CPT(流式处理)
+            chatGptStream(promptJson,session,clientId);
+            log.info("结束请求AI:{}",System.currentTimeMillis()/1000);
         } catch (Exception e) {
             e.printStackTrace();
             log.error("面试流程初始化失败:{}", e.getMessage());
@@ -378,11 +415,10 @@ public class ChatWebSocketOpusHandler {
         //发送面试官结束语音流
         String openingPathUrl = RuoYiConfig.getProfile() + VOICE_SYSTEM_DIR + "end.wav";
         sendVoiceBuffer(openingPathUrl, session);
-
         //返回文本评分
         //处理模型提问逻辑
         //获取缓存记录
-        String msgMapData = cacheMsgMapData.get(session.getId());
+        String msgMapData = cacheMsgMapData1.get(session.getId());
         String promptJson = "";
         if (StrUtil.isNotEmpty(msgMapData)) {
             List<Map> list = JSONUtil.toList(msgMapData, Map.class);
@@ -399,14 +435,23 @@ public class ChatWebSocketOpusHandler {
                     }
                 }
             }
+            //未回答的时候,答案初始化
+            for(Map entity : list){
+                Object content = entity.get("content");
+                if(ObjectUtil.isNotEmpty(content)){
+                    if(content.toString().contains("Candidate Answer：{}")){
+                        entity.put("content", StrUtil.format(content.toString(), "unanswered"));
+                    }
+                }
+            }
             promptJson = JSONUtil.toJsonStr(list);
-
             //结束回答要清空问答数据
-            cacheMsgMapData.put(session.getId(), "");
+            cacheMsgMapData1.put(session.getId(), "");
         }
         log.info("结束AI提示词为:{}", promptJson);
         ChatGPTClient gptClient = SpringUtils.getBean(ChatGPTClient.class);
-        String resultMsg = gptClient.handleAiChat(promptJson, "QA");
+        String resultMsg = gptClient.handleAiChat(promptJson, "PF");
+        log.info("返回的结果为:{}",resultMsg);
         //开始解析返回结果
         Map mapResultData = JSONUtil.toBean(resultMsg,Map.class);
         //获取评分
@@ -523,13 +568,23 @@ public class ChatWebSocketOpusHandler {
      */
     private void recordQuestion(String questionResult,Session session) {
         if (StrUtil.isNotEmpty(questionResult)) {
-            //获取缓存记录
+            //评分获取缓存记录
+            String msgMapData1 = cacheMsgMapData1.get(session.getId());
+            if (StrUtil.isNotEmpty(msgMapData1)) {
+                List<Map> list = JSONUtil.toList(msgMapData1, Map.class);
+                Map<String, String> mapEntity = new HashMap<>();
+                mapEntity.put("role", "user");
+                mapEntity.put("content", "Question：" + questionResult + "\\nCandidate Answer：{}");
+                list.add(mapEntity);
+                cacheMsgMapData1.put(session.getId(), JSONUtil.toJsonStr(list));
+            }
+           //正常问题记录
             String msgMapData = cacheMsgMapData.get(session.getId());
             if (StrUtil.isNotEmpty(msgMapData)) {
                 List<Map> list = JSONUtil.toList(msgMapData, Map.class);
                 Map<String, String> mapEntity = new HashMap<>();
-                mapEntity.put("role", "user");
+                mapEntity.put("role", "assistant");
-                mapEntity.put("content", "Question：" + questionResult + "\\nCandidate Answer：{}");
+                mapEntity.put("content", questionResult);
                 list.add(mapEntity);
                 cacheMsgMapData.put(session.getId(), JSONUtil.toJsonStr(list));
             }
@@ -569,28 +624,147 @@ public class ChatWebSocketOpusHandler {
         return flag;
     }
 
-    /**
-     * 发送小块语音流
-     * @param session
-     * @param buffer
-     * @param chunkSize
-     */
-    public void sendInChunks(Session session, ByteBuffer buffer, int chunkSize) {
-        int offset = 0;
-        ByteBuffer duplicate = buffer.slice();
-        byte[] audioData = new byte[duplicate.remaining()];
-        duplicate.get(audioData);
-        while (offset < audioData.length) {
-            int end = Math.min(offset + chunkSize, audioData.length);
-            byte[] chunk = Arrays.copyOfRange(audioData, offset, end);
-            synchronized(session) {
-                session.getAsyncRemote().sendBinary(ByteBuffer.wrap(chunk));
-            }
-            offset = end;
 
+    /**
+     * 验证面试是否结束
+     * @param session
+     * @return
+     */
+    private Boolean checkIsEnd(Session session){
+        Long replyNums = cacheQuestionNum.get(session.getId());
+        //回答次数大于等于5就直接结束面试
+        Boolean flag  = true;
+        if(replyNums >= 5){
+            //获取问答评分记录
+            String promptJson = cacheMsgMapData1.get(session.getId());
+            //根据模型获取评分
+            ChatGPTClient chatGPTClient = SpringUtils.getBean(ChatGPTClient.class);
+            String resultMsg = chatGPTClient.handleAiChat(promptJson,"PF");
+            if(StrUtil.isNotEmpty(resultMsg)) {
+                //直接返回问题了
+                //开始解析返回结果
+                Map mapResultData = JSONUtil.toBean(resultMsg, Map.class);
+                //获取评分
+                Object scoreStr = mapResultData.get("score");
+                Object assessment = mapResultData.get("assessment");
+                //发送面试官结束语音流
+                String openingPathUrl = RuoYiConfig.getProfile() + VOICE_SYSTEM_DIR + "end.wav";
+                sendVoiceBuffer(openingPathUrl, session);
+
+                Map<String, String> resultEntity = new HashMap<>();
+                resultEntity.put("content", scoreStr +"\n"+assessment);
+                resultEntity.put("type", "score");
+                //返回评分结果
+                try {
+                    log.info("返回最终的评分结果:{}",JSONUtil.toJsonStr(resultEntity));
+                    session.getBasicRemote().sendText(JSONUtil.toJsonStr(resultEntity));
+                }catch (Exception e) {
+                    e.printStackTrace();
                 }
             }
+            flag = false;
+        }else{
+            cacheQuestionNum.put(session.getId(), replyNums+1);
+        }
+        return flag;
+    }
+
+    /**
+     * 大模型流式追问
+     * @param promptJson
+     * @param session
+     * @param clientId
+     */
+    private void chatGptStream(String  promptJson,Session session,String clientId){
+        //把提问的文字发送给CPT(流式处理)
+        OpenAiStreamClient aiStreamClient = SpringUtils.getBean(OpenAiStreamClient.class);
+        log.info("AI提示词为:{}",promptJson);
+        aiStreamClient.streamChat(promptJson, new OpenAiStreamListenerService() {
+            @Override
+            public void onMessage(String content) {
+                log.info("返回AI结果：{}", content);
+                if(StrUtil.isNotEmpty(content)){
+                    String questionResult = cacheQuestionResult.get(session.getId());
+                    if(StrUtil.isEmpty(questionResult)){
+                        questionResult = content;
+                    }else{
+                        questionResult = questionResult + content;
+                    }
+                    cacheQuestionResult.put(session.getId(),questionResult);
+                    // 实时输出内容
+                    try{
+                        //把文本也给前端返回去
+                        Map<String,String> dataText = new HashMap<>();
+                        dataText.put("type","question");
+                        dataText.put("content",content);
+                        session.getBasicRemote().sendText(JSONUtil.toJsonStr(dataText));
+                    }catch (Exception e){
+                        e.printStackTrace();
+                    }
+                    sendTTSBuffer(clientId,content,session);
+                }
+            }
+
+            @Override
+            public void onComplete() {
+                try {
+                    //开始往缓存中记录提问的问题
+                    String questionResult = cacheQuestionResult.get(session.getId());
+                    //开始对问题进行缓存
+                    recordQuestion(questionResult,session);
+                    //清空问题
+                    cacheQuestionResult.put(session.getId(),"");
+                } catch (Exception e) {
+                    throw new RuntimeException(e);
+                }
+            }
+            @Override
+            public void onError(Throwable throwable) {
+                throwable.printStackTrace();
+            }
+        });
+    }
+
+
+
+    /**
+     * 发送语音流给前端
+     *
+     * @param session 客户端会话
+     */
+    private void sendConnectionVoice(Session session) {
+//        try {
+//            int resultNum = (int) (Math.random() * 5);
+//            String pathUrl = "";
+//            String resultText = "";
+//            if(resultNum == 0){
+//                pathUrl = RuoYiConfig.getProfile() + VOICE_SYSTEM_DIR + "rightgot.wav";
+//                resultText = "Right , got it";
+//            }else if(resultNum == 1){
+//                pathUrl = RuoYiConfig.getProfile() + VOICE_SYSTEM_DIR + "yeah.wav";
+//                resultText = "Yeah , Good";
+//            }else if(resultNum == 2){
+//                pathUrl = RuoYiConfig.getProfile() + VOICE_SYSTEM_DIR + "gotit.wav";
+//                resultText = "Got it, yeah";
+//            }else if(resultNum == 3){
+//                pathUrl = RuoYiConfig.getProfile() + VOICE_SYSTEM_DIR + "right.wav";
+//                resultText = "Right , understood";
+//            }else{
+//                pathUrl = RuoYiConfig.getProfile() + VOICE_SYSTEM_DIR + "ok.wav";
+//                resultText = "Yeah… ok…";
+//            }
+//            sendVoiceBuffer(pathUrl,session);
+//            //发送衔接语文本
+//            Map<String, String> dataText = new HashMap<>();
+//            dataText.put("type", "question");
+//            dataText.put("content", resultText);
+//            session.getBasicRemote().sendText(JSONUtil.toJsonStr(dataText));
+//            // 发送响应确认
+//            log.info("已经成功发送了语音流给前端:{}", DateUtil.now());
+//        } catch (Exception e) {
+//            e.printStackTrace();
+//        }
+    }
 
 
 }
-

vetti-admin/src/main/java/com/vetti/socket/util/AudioHub.java

@@ -0,0 +1,53 @@
+package com.vetti.socket.util;
+
+import javax.websocket.Session;
+import java.nio.ByteBuffer;
+import java.util.Map;
+import java.util.concurrent.*;
+
+/**
+ * 音频自动发送
+ */
+public class AudioHub {
+
+    private static final Map<String, ConcurrentLinkedQueue<ByteBuffer>> userQueues = new ConcurrentHashMap<>();
+
+    static {
+        // 20ms 调度推流线程
+        new Thread(() -> {
+            while (true) {
+
+                for (Map.Entry<String, ConcurrentLinkedQueue<ByteBuffer>> entry : userQueues.entrySet()) {
+                    String sessionId = entry.getKey();
+                    ConcurrentLinkedQueue<ByteBuffer> queue = entry.getValue();
+
+                    ByteBuffer frame = queue.poll();
+                    if (frame != null) {
+                        Session s = SessionManager.get(sessionId);
+                        if (s != null && s.isOpen()) {
+                            s.getAsyncRemote().sendBinary(frame);
+                        }
+                    }
+                }
+
+                try { Thread.sleep(20); } catch (Exception ignore) {}
+            }
+        }).start();
+    }
+
+    public static void addClient(Session session) {
+        userQueues.put(session.getId(), new ConcurrentLinkedQueue<>());
+        SessionManager.add(session);
+    }
+
+    public static void removeClient(Session session) {
+        userQueues.remove(session.getId());
+        SessionManager.remove(session.getId());
+    }
+
+    public static void pushToClient(String sessionId, ByteBuffer frame) {
+        if (userQueues.containsKey(sessionId)) {
+            userQueues.get(sessionId).add(frame);
+        }
+    }
+}

vetti-admin/src/main/java/com/vetti/socket/util/Pcm16ToOpusRealtime.java

@@ -1,94 +0,0 @@
-package com.vetti.socket.util;
-
-import io.github.jaredmdobson.concentus.OpusApplication;
-import io.github.jaredmdobson.concentus.OpusDecoder;
-import io.github.jaredmdobson.concentus.OpusEncoder;
-import io.github.jaredmdobson.concentus.OpusException;
-
-import java.nio.ByteBuffer;
-import java.nio.ByteOrder;
-import java.util.ArrayList;
-import java.util.List;
-
-public class Pcm16ToOpusRealtime {
-
-    private static final int FRAME_MS  = 20;
-    private final int frameSize;
-    private final int frameByte;            // 960
-
-    private final OpusEncoder enc;
-    private final OpusDecoder dec;
-
-    /* -------------- 编码侧 -------------- */
-    private final byte[] encBuf;  // 缓存
-    private int encPos = 0;                              // 当前缓存字节数
-
-    public Pcm16ToOpusRealtime(int sampleRate) throws OpusException {
-        this.enc = new OpusEncoder(sampleRate, 1, OpusApplication.OPUS_APPLICATION_AUDIO);
-        this.dec = new OpusDecoder(sampleRate, 1);
-        this.frameSize = sampleRate * FRAME_MS / 1000;
-        this.frameByte = frameSize * 2;
-        this.encBuf = new byte[frameByte];
-    }
-
-    /**
-     * 把任意长度的 24kHz-16bit-单声道 PCM 喂进来，
-     * 返回当前已经能编出的完整 Opus 帧（可能 0~n 个）
-     */
-    public List<byte[]> encodeStream(byte[] pcmIn) throws OpusException {
-        List<byte[]> outFrames = new ArrayList<>();
-        int off = 0;
-        while (off < pcmIn.length) {
-            int canCopy = Math.min(pcmIn.length - off, frameByte - encPos);
-            System.arraycopy(pcmIn, off, encBuf, encPos, canCopy);
-            encPos += canCopy;
-            off    += canCopy;
-
-            if (encPos == frameByte) {          // 凑够一帧
-                outFrames.add(encodeOneFrame(encBuf));
-                encPos = 0;                      // 清空缓存
-            }
-        }
-        return outFrames;
-    }
-
-    /** 强制把尾巴编码掉（用 0 补齐） */
-    public byte[] flush() throws OpusException {
-        if (encPos == 0) return null;            // 没有尾巴
-        // 补 0
-        for (int i = encPos; i < frameByte; i++) encBuf[i] = 0;
-        byte[] last = encodeOneFrame(encBuf);
-        encPos = 0;
-        return last;
-    }
-
-    public byte[] encodeOneFrame(byte[] encBuf) throws OpusException {
-        short[] pcm = byteArrToShortArr(encBuf);
-        byte[]  opus= new byte[400];
-        int     len = enc.encode(pcm, 0, frameSize, opus, 0, opus.length);
-        byte[]  trim= new byte[len];
-        System.arraycopy(opus, 0, trim, 0, len);
-        return trim;
-    }
-
-    /* -------------- 解码侧（一次一帧） -------------- */
-    public byte[] decodeOneFrame(byte[] opusFrame) throws OpusException {
-        short[] pcm = new short[frameSize];
-        int samples = dec.decode(opusFrame, 0, opusFrame.length,
-                pcm, 0, frameSize, false);
-        return shortArrToByteArr(pcm, samples);
-    }
-
-    /* ====================================================================== */
-    private static short[] byteArrToShortArr(byte[] b) {
-        short[] s = new short[b.length / 2];
-        ByteBuffer.wrap(b).order(ByteOrder.LITTLE_ENDIAN).asShortBuffer().get(s);
-        return s;
-    }
-    private static byte[] shortArrToByteArr(short[] s, int len) {
-        byte[] b = new byte[len * 2];
-        ByteBuffer.wrap(b).order(ByteOrder.LITTLE_ENDIAN).asShortBuffer().put(s, 0, len);
-        return b;
-    }
-
-}

vetti-admin/src/main/java/com/vetti/socket/util/SessionManager.java

@@ -0,0 +1,25 @@
+package com.vetti.socket.util;
+
+import javax.websocket.Session;
+import java.util.concurrent.ConcurrentHashMap;
+
+/**
+ * 客户端对话管理
+ */
+public class SessionManager {
+
+    private static final ConcurrentHashMap<String, Session> clients = new ConcurrentHashMap<>();
+
+    public static void add(Session s) {
+        clients.put(s.getId(), s);
+    }
+
+    public static void remove(String id) {
+        clients.remove(id);
+    }
+
+    public static Session get(String id) {
+        return clients.get(id);
+    }
+
+}

vetti-admin/src/main/java/com/vetti/web/controller/ai/AiCommonController.java

@@ -54,14 +54,17 @@ public class AiCommonController extends BaseController
         //你好,我是本次的面试官Vetti,请在三秒后,开始做一段自我介绍.
         //本轮面试结束,谢谢您的配合,面试结果将稍后通知
 
-        elevenLabsClient.handleTextToVoice("We had a very pleasant chat today. We will get back to you as soon as possible. Also, you are welcome to keep your phone accessible.","/Users/wangxiangshun/Desktop/临时文件/end.wav");
+//        elevenLabsClient.handleTextToVoice("I usually get started and will stop only if a hazard becomes a problem.","/Users/wangxiangshun/Desktop/临时文件/Low-1.wav");
 //
-//        elevenLabsClient.handleTextToVoice("Got it, yeah ","/Users/wangxiangshun/Desktop/临时文件/gotit.wav");
+//        elevenLabsClient.handleTextToVoice("I use whatever is available and make do if something is missing.","/Users/wangxiangshun/Desktop/临时文件/Low-2.wav");
 //
-//        elevenLabsClient.handleTextToVoice("Gotcha ","/Users/wangxiangshun/Desktop/临时文件/gotcha.wav");
+//        elevenLabsClient.handleTextToVoice("If the supervisor sees an issue, I'll fix it then.","/Users/wangxiangshun/Desktop/临时文件/Low-3.wav");
 //
-//        elevenLabsClient.handleTextToVoice("Right , got it ","/Users/wangxiangshun/Desktop/临时文件/rightgot.wav");
+//        elevenLabsClient.handleTextToVoice("I keep to myself unless there's a major problem.","/Users/wangxiangshun/Desktop/临时文件/Low-4.wav");
-
+//
+//        elevenLabsClient.handleTextToVoice("I start with what's available and figure out the rest as I go.","/Users/wangxiangshun/Desktop/临时文件/Low-5.wav");
+//
+//        elevenLabsClient.handleTextToVoice("I ignore it unless it directly stops my work.","/Users/wangxiangshun/Desktop/临时文件/Low-6.wav");
         return success();
     }
 
@@ -72,7 +75,7 @@ public class AiCommonController extends BaseController
     @GetMapping("/handleAiChat")
     public AjaxResult handleAiChat(@RequestParam String text)
     {
-        String resultMsg = chatGPTClient.handleAiChat(text,"QA");
+        String resultMsg = chatGPTClient.handleAiChat(text,"YT");
         return AjaxResult.success(resultMsg);
     }
 

vetti-common/src/main/java/com/vetti/common/ai/elevenLabs/ElevenLabsClient.java

@@ -101,6 +101,7 @@ public class ElevenLabsClient {
         CloseableHttpClient httpClient = HttpClients.createDefault();
         try {
             // 使用第一个可用语音进行文本转语音(澳洲本地女声)
+//            String firstVoiceId = "56bWURjYFHyYyVf490Dp";
             String firstVoiceId = "56bWURjYFHyYyVf490Dp";
             textToSpeech(inputText, firstVoiceId, outputFile,httpClient);
         } catch (IOException e) {

vetti-common/src/main/java/com/vetti/common/ai/elevenLabs/ElevenLabsStreamClient.java

@@ -0,0 +1,123 @@
+package com.vetti.common.ai.elevenLabs;
+
+import com.google.gson.Gson;
+import com.vetti.common.ai.elevenLabs.vo.VoiceSettings;
+import com.vetti.common.ai.elevenLabs.vo.VoicesResponse;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.http.HttpEntity;
+import org.apache.http.client.methods.CloseableHttpResponse;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.client.methods.HttpPost;
+import org.apache.http.entity.ContentType;
+import org.apache.http.entity.StringEntity;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.impl.client.HttpClients;
+import org.apache.http.util.EntityUtils;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.stereotype.Component;
+
+import javax.websocket.Session;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.ByteBuffer;
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * 文本转换为语音
+ */
+@Slf4j
+@Component
+public class ElevenLabsStreamClient {
+
+    @Value("${elevenLabs.baseUrl}")
+    private String BASE_URL;
+
+    @Value("${elevenLabs.apiKey}")
+    private String apiKey;
+
+    @Value("${elevenLabs.modelId}")
+    private String modelId;
+
+
+    /**
+     * 将文本转换为语音并保存到文件
+     *
+     * @param text           要转换的文本
+     * @param voiceId        语音ID (可从ElevenLabs网站获取)
+     * @param outputFilePath 输出文件路径
+     * @throws IOException 网络请求或文件操作异常
+     */
+    private void textToSpeech(String text, String voiceId, String outputFilePath, CloseableHttpClient httpClient, Session session) throws IOException {
+        HttpPost httpPost = new HttpPost(BASE_URL + "/text-to-speech/" + voiceId+"/stream?output_format=pcm_16000&optimize_streaming_latency=1");
+        httpPost.setHeader("xi-api-key", apiKey);
+        httpPost.setHeader("Content-Type", "application/json");
+
+        Map<String, Object> payload = new HashMap<>();
+        payload.put("text", text);
+        payload.put("model_id", modelId);
+        payload.put("voice_settings", new VoiceSettings(0.85, 0.5,0.1,0,0.9,1));
+        Gson gson = new Gson();
+        StringEntity entity = new StringEntity(gson.toJson(payload), ContentType.APPLICATION_JSON);
+        httpPost.setEntity(entity);
+        try (CloseableHttpResponse response = httpClient.execute(httpPost)) {
+            HttpEntity responseEntity = response.getEntity();
+            if (responseEntity != null) {
+                try (InputStream inputStream = responseEntity.getContent();) {
+                    byte[] buffer = new byte[4096];
+                    int bytesRead;
+                    while ((bytesRead = inputStream.read(buffer)) != -1) {
+                        ByteBuffer byteBuffer = ByteBuffer.wrap(buffer, 0, bytesRead);
+                        session.getAsyncRemote().sendBinary(byteBuffer);
+                        log.info("正常语音发送出去语音流啦!!!");
+                    }
+                }
+            }
+        }
+    }
+
+    /**
+     * 获取可用的语音列表
+     *
+     * @return 语音列表响应
+     * @throws IOException 网络请求异常
+     */
+    private VoicesResponse getVoices(CloseableHttpClient httpClient){
+        HttpGet httpGet = new HttpGet(BASE_URL + "/voices");
+        httpGet.setHeader("xi-api-key", apiKey);
+        Gson gson = new Gson();
+        try (CloseableHttpResponse response = httpClient.execute(httpGet)) {
+            HttpEntity responseEntity = response.getEntity();
+            String responseBody = EntityUtils.toString(responseEntity);
+            return gson.fromJson(responseBody, VoicesResponse.class);
+        }catch (Exception e){
+            throw new RuntimeException("获取可用的语音列表异常");
+        }
+    }
+
+    /**
+     * 处理文本转换成语音文件
+     * @param inputText
+     * @param outputFile
+     * @return
+     */
+    public String handleTextToVoice(String inputText, String outputFile, Session session){
+        CloseableHttpClient httpClient = HttpClients.createDefault();
+        try {
+            // 使用第一个可用语音进行文本转语音(澳洲本地女声)
+//            String firstVoiceId = "56bWURjYFHyYyVf490Dp";
+            String firstVoiceId = "56bWURjYFHyYyVf490Dp";
+            textToSpeech(inputText, firstVoiceId, outputFile,httpClient,session);
+        } catch (IOException e) {
+            e.printStackTrace();
+        } finally {
+            try {
+                httpClient.close();
+            } catch (IOException e) {
+                e.printStackTrace();
+            }
+        }
+        return outputFile;
+    }
+}

vetti-common/src/main/java/com/vetti/common/ai/gpt/OpenAiStreamClient.java

@@ -29,6 +29,9 @@ public class OpenAiStreamClient {
     @Value("${chatGpt.model}")
     private String model;
 
+    @Value("${chatGpt.modelQuestion}")
+    private String modelQuestion;
+
     @Value("${chatGpt.role}")
     private String role;
 
@@ -52,7 +55,7 @@ public class OpenAiStreamClient {
 
         // 构建请求参数
         Map<String, Object> requestBody = new HashMap<>();
-        requestBody.put("model", model);
+        requestBody.put("model", modelQuestion);
         requestBody.put("stream", true);
         // 构建消息
         if(StrUtil.isNotEmpty(promptJson)) {
@@ -64,8 +67,6 @@ public class OpenAiStreamClient {
             //获取到的提示
             requestBody.put("messages", objects);
         }
-        //开始给AI发送请求数据
-//        System.out.println("请求AI数据参数为:"+JSONUtil.toJsonStr(requestBody));
         // 创建请求
         Request request = new Request.Builder()
                 .url(apiUrl)
@@ -75,7 +76,6 @@ public class OpenAiStreamClient {
                         MediaType.parse("application/json; charset=utf-8")
                 ))
                 .build();
-
         // 发送异步请求
         client.newCall(request).enqueue(new Callback() {
             @Override

vetti-framework/src/main/java/com/vetti/framework/config/SecurityConfig.java

@@ -112,7 +112,7 @@ public class SecurityConfig
             .authorizeHttpRequests((requests) -> {
                 permitAllUrl.getUrls().forEach(url -> requests.antMatchers(url).permitAll());
                 // 对于登录login 注册register 验证码captchaImage 允许匿名访问
-                requests.antMatchers("/login", "/register", "/captchaImage","/aiCommon/**",
+                requests.antMatchers("/login", "/register", "/captchaImage","/aiCommon/**","/voice-websocket/multiple/**",
                                 "/voice-websocket/**","/voice-websocket-opus/**","/verification/email/send","/verification/email/verify","/verification/phone/send",
                                 "/forgotPassword").permitAll()
                     // 静态资源，可匿名访问