diff --git a/vetti-admin/src/main/java/com/vetti/socket/AudioSender.java b/vetti-admin/src/main/java/com/vetti/socket/AudioSender.java deleted file mode 100644 index 8f41525..0000000 --- a/vetti-admin/src/main/java/com/vetti/socket/AudioSender.java +++ /dev/null @@ -1,47 +0,0 @@ -package com.vetti.socket; - -import javax.websocket.Session; -import java.nio.ByteBuffer; -import java.util.Queue; -import java.util.concurrent.ConcurrentLinkedQueue; -import java.util.concurrent.atomic.AtomicBoolean; - -/** - * 语音发送 - */ -public class AudioSender { - - private final Queue queue = new ConcurrentLinkedQueue<>(); - private final AtomicBoolean sending = new AtomicBoolean(false); - - public void sendAudio(Session session, byte[] data) { - queue.add(data); - trySend(session); - } - - private void trySend(Session session) { - if (!sending.compareAndSet(false, true)) { - return; // 已在发送中，退出 - } - - byte[] chunk = queue.poll(); - if (chunk == null) { - sending.set(false); - return; - } - - ByteBuffer buffer = ByteBuffer.wrap(chunk); - - session.getAsyncRemote().sendBinary(buffer, result -> { - sending.set(false); - - if (!result.isOK()) { - result.getException().printStackTrace(); - } - - // 递归发送下一片 - trySend(session); - }); - } - -} diff --git a/vetti-admin/src/main/java/com/vetti/socket/ChatWebSocketHandler.java b/vetti-admin/src/main/java/com/vetti/socket/ChatWebSocketHandler.java index c524de0..a556702 100644 --- a/vetti-admin/src/main/java/com/vetti/socket/ChatWebSocketHandler.java +++ b/vetti-admin/src/main/java/com/vetti/socket/ChatWebSocketHandler.java @@ -328,31 +328,58 @@ public class ChatWebSocketHandler { List> list = new LinkedList(); Map mapEntity = new HashMap<>(); mapEntity.put("role", "system"); - mapEntity.put("content", "You are a senior HR interviewer conducting behavioral interviews. You're a \"listener and guide,\" not an \"examiner.\" Your style is warm, natural, and conversational—like chatting with a colleague.\n" + + mapEntity.put("content", "You're Sarah, a senior HR interviewer in Sydney (15 years experience). You make candidates comfortable while getting insights.\n" + "\n" + - "Core Rules:\n" + - "1. ALWAYS start with: \"Thank you,\" \"Cheers for that,\" or \"Thanks for sharing that\"\n" + - "2. Use Australian English: \"no worries,\" \"fair enough,\" \"mate,\" \"good on you,\" \"G'day\"\n" + - "3. Keep it casual—avoid jargon\n" + - "4. For detailed STAR answers: acknowledge warmly, then move to a NEW topic (don't over-probe)\n" + - "5. For stuck candidates: ease pressure with \"No rush, mate. Even a small example works\"\n" + - "6. For off-topic answers: acknowledge politely, then redirect gently back to the question\n" + + "Style: Chat like having coffee with a mate—warm, genuine, curious. React naturally: \"Oh nice one!\" \"Good on you, mate.\" When they're stuck, ease pressure like a friend would.\n" + "\n" + - "Opening Pattern:\n" + - "\"G'day! Cheers for coming in. No need to stress—just a casual chat about your experiences. We'll focus on [competencies]. Sound good?\"\n" + + "Australian English:\n" + + "Start: \"Cheers for that,\" \"Thanks mate,\" \"Righto\"\n" + + "Encourage: \"Good on you,\" \"Nice one\"\n" + + "Casual: \"No worries,\" \"All good\"\n" + + "Transition: \"Right, so...\" \"Okay, cool...\"\n" + "\n" + - "Response Patterns:\n" + - "- Brief answer → \"Cheers for that. Could you walk me through a specific time? What was the situation, what did you do, and how did it turn out?\"\n" + - "- Detailed answer → \"Thank you. Good on you for [action]. Fair enough. Now, let's chat about [new topic]\"\n" + - "- Stuck candidate → \"No worries, mate. Take your time. Even a small project works. Want me to rephrase that?\"\n" + - "- Off-topic answer → \"Thanks for sharing that. That's interesting, but let me bring us back to [original question]. Could you tell me about [specific aspect]?\"\n" + + "Opening:\n" + + "\"G'day! Thanks for coming in. Look, no need to be nervous—this is just a casual chat, yeah? I want to hear about your real experiences. We'll talk about [safety, technical skills, problem-solving]. Sound good? Let's get into it.\"\n" + "\n" + - "Guardrails:\n" + - "- No discussion of protected characteristics\n" + - "- Base judgments on stated evidence only\n" + - "- Respond in natural English, NOT JSON\n" + + "Brief Answer → Need Story:\n" + + "Think: \"They gave me the headline, now I need the story.\"\n" + + "Say: \"Cheers for that. So tell me more—walk me through a specific time. What was going on, what did you actually do, how'd it turn out?\"\n" + + "\n" + + "Detailed STAR Answer → Acknowledge & Move On:\n" + + "Think: \"Perfect! They've given me everything. Don't over-probe.\"\n" + + "Say: \"Thanks mate, appreciate that. Good on you for [action]. Right, so let's chat about [new topic].\"\n" + + "\n" + + "Stuck/Nervous → Ease Pressure:\n" + + "Think: \"They're feeling the pressure. Take it down a notch.\"\n" + + "Say: \"No worries, mate. Take your time, yeah? Even a small example works—doesn't have to be anything massive. Want me to ask it differently?\"\n" + + "\n" + + "Off-Topic → Gentle Redirect:\n" + + "Think: \"They're talking about something else. Gently bring them back.\"\n" + + "Say: \"Yeah, thanks for sharing that. That's interesting. But let me bring us back to [question]—could you tell me about [specific thing]?\"\n" + + "\n" + + "Assess:\n" + + "- Specific examples? (not \"I always do X\")\n" + + "- STAR? (Situation, Task, Action, Result)\n" + + "- Good judgment? (especially safety)\n" + + "- Clear communication?\n" + + "\n" + + "Flow:\n" + + "- Cover 5-7 areas: safety, technical, problem-solving, communication, teamwork\n" + + "- 1-2 questions per area\n" + + "- Keep moving, don't over-probe\n" + + "- 15-20 minutes\n" + + "\n" + + "Closing:\n" + + "\"Righto, thanks for sharing all that. That gives me a good sense of your experience. Any questions for me?\"\n" + + "\n" + + "Rules:\n" + + "- No protected characteristics (age, gender, race, religion)\n" + + "- Base on what they say\n" + + "- Talk like a real person, not a robot\n" + "- One question at a time\n" + - "- Keep candidates focused on the question asked"); + "- If they give gold, acknowledge and move on\n" + + "\n" + + "Remember: Conversation, not interrogation. Be genuinely interested, react naturally, help them show their best self."); list.add(mapEntity); //记录另外一个评分的提示词 @@ -673,37 +700,37 @@ public class ChatWebSocketHandler { * @param session 客户端会话 */ private void sendConnectionVoice(Session session) { - try { - int resultNum = (int) (Math.random() * 5); - String pathUrl = ""; - String resultText = ""; - if(resultNum == 0){ - pathUrl = RuoYiConfig.getProfile() + VOICE_SYSTEM_DIR + "rightgot.wav"; - resultText = "Right , got it"; - }else if(resultNum == 1){ - pathUrl = RuoYiConfig.getProfile() + VOICE_SYSTEM_DIR + "yeah.wav"; - resultText = "Yeah , Good"; - }else if(resultNum == 2){ - pathUrl = RuoYiConfig.getProfile() + VOICE_SYSTEM_DIR + "gotit.wav"; - resultText = "Got it, yeah"; - }else if(resultNum == 3){ - pathUrl = RuoYiConfig.getProfile() + VOICE_SYSTEM_DIR + "right.wav"; - resultText = "Right , understood"; - }else{ - pathUrl = RuoYiConfig.getProfile() + VOICE_SYSTEM_DIR + "ok.wav"; - resultText = "Yeah… ok…"; - } - sendVoiceBuffer(pathUrl,session); - //发送衔接语文本 - Map dataText = new HashMap<>(); - dataText.put("type", "question"); - dataText.put("content", resultText); - session.getBasicRemote().sendText(JSONUtil.toJsonStr(dataText)); - // 发送响应确认 - log.info("已经成功发送了语音流给前端:{}", DateUtil.now()); - } catch (Exception e) { - e.printStackTrace(); - } +// try { +// int resultNum = (int) (Math.random() * 5); +// String pathUrl = ""; +// String resultText = ""; +// if(resultNum == 0){ +// pathUrl = RuoYiConfig.getProfile() + VOICE_SYSTEM_DIR + "rightgot.wav"; +// resultText = "Right , got it"; +// }else if(resultNum == 1){ +// pathUrl = RuoYiConfig.getProfile() + VOICE_SYSTEM_DIR + "yeah.wav"; +// resultText = "Yeah , Good"; +// }else if(resultNum == 2){ +// pathUrl = RuoYiConfig.getProfile() + VOICE_SYSTEM_DIR + "gotit.wav"; +// resultText = "Got it, yeah"; +// }else if(resultNum == 3){ +// pathUrl = RuoYiConfig.getProfile() + VOICE_SYSTEM_DIR + "right.wav"; +// resultText = "Right , understood"; +// }else{ +// pathUrl = RuoYiConfig.getProfile() + VOICE_SYSTEM_DIR + "ok.wav"; +// resultText = "Yeah… ok…"; +// } +// sendVoiceBuffer(pathUrl,session); +// //发送衔接语文本 +// Map dataText = new HashMap<>(); +// dataText.put("type", "question"); +// dataText.put("content", resultText); +// session.getBasicRemote().sendText(JSONUtil.toJsonStr(dataText)); +// // 发送响应确认 +// log.info("已经成功发送了语音流给前端:{}", DateUtil.now()); +// } catch (Exception e) { +// e.printStackTrace(); +// } } diff --git a/vetti-admin/src/main/java/com/vetti/socket/ChatWebSocketOpusHandler.java b/vetti-admin/src/main/java/com/vetti/socket/ChatWebSocketMultipleHandler.java similarity index 55% rename from vetti-admin/src/main/java/com/vetti/socket/ChatWebSocketOpusHandler.java rename to vetti-admin/src/main/java/com/vetti/socket/ChatWebSocketMultipleHandler.java index adbf299..c8440b4 100644 --- a/vetti-admin/src/main/java/com/vetti/socket/ChatWebSocketOpusHandler.java +++ b/vetti-admin/src/main/java/com/vetti/socket/ChatWebSocketMultipleHandler.java @@ -1,17 +1,15 @@ package com.vetti.socket; -import cn.hutool.core.collection.CollectionUtil; import cn.hutool.core.date.DateUtil; import cn.hutool.core.util.ObjectUtil; import cn.hutool.core.util.StrUtil; import cn.hutool.json.JSONUtil; -import com.vetti.common.ai.elevenLabs.ElevenLabsClient; +import com.vetti.common.ai.elevenLabs.ElevenLabsStreamClient; import com.vetti.common.ai.gpt.ChatGPTClient; +import com.vetti.common.ai.gpt.OpenAiStreamClient; +import com.vetti.common.ai.gpt.service.OpenAiStreamListenerService; import com.vetti.common.config.RuoYiConfig; import com.vetti.common.utils.spring.SpringUtils; -import com.vetti.hotake.domain.HotakeProblemBaseInfo; -import com.vetti.hotake.service.IHotakeProblemBaseInfoService; -import com.vetti.socket.util.Pcm16ToOpusRealtime; import lombok.extern.slf4j.Slf4j; import org.apache.commons.io.FileUtils; import org.springframework.stereotype.Component; @@ -22,17 +20,19 @@ import javax.websocket.server.ServerEndpoint; import java.io.File; import java.math.BigDecimal; import java.nio.ByteBuffer; -import java.util.*; +import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; import java.util.concurrent.ConcurrentHashMap; - /** - * 语音面试 web处理器 + * 语音面试(多客户端) web处理器 */ @Slf4j -@ServerEndpoint("/voice-websocket-opus/{clientId}") +@ServerEndpoint("/voice-websocket/multiple/{clientId}") @Component -public class ChatWebSocketOpusHandler { +public class ChatWebSocketMultipleHandler { /** * 缓存客户端流式解析的语音文本数据 @@ -48,7 +48,10 @@ public class ChatWebSocketOpusHandler { * 缓存客户端,面试回答信息 */ private final Map cacheMsgMapData = new ConcurrentHashMap<>(); - + /** + * 缓存客户端,面试回答信息 + */ + private final Map cacheMsgMapData1 = new ConcurrentHashMap<>(); /** * 缓存客户端,AI提问的问题结果信息 */ @@ -59,6 +62,11 @@ public class ChatWebSocketOpusHandler { */ private final Map> cacheScoreResult = new ConcurrentHashMap<>(); + /** + * 缓存客户端,回答问题次数-回答5轮就自动停止当前问答,返回对应的评分 + */ + private final Map cacheQuestionNum = new ConcurrentHashMap<>(); + // 语音文件保存目录 private static final String VOICE_STORAGE_DIR = "/voice_files/"; @@ -68,7 +76,7 @@ public class ChatWebSocketOpusHandler { // 系统语音目录 private static final String VOICE_SYSTEM_DIR = "/system_files/"; - public ChatWebSocketOpusHandler() { + public ChatWebSocketMultipleHandler() { // 初始化存储目录 File dir = new File(RuoYiConfig.getProfile() + VOICE_STORAGE_DIR); if (!dir.exists()) { @@ -86,11 +94,18 @@ public class ChatWebSocketOpusHandler { public void onOpen(Session session, @PathParam("clientId") String clientId) { log.info("WebSocket 链接已建立:{}", clientId); log.info("WebSocket session 链接已建立:{}", session.getId()); + + //启动客户端,自动发送语音流 +// AudioHub.addClient(session); +// System.out.println("Client connected: " + session.getId()); + cacheClientTts.put(clientId, new String()); //是初次自我介绍后的问答环节 cacheReplyFlag.put(session.getId(), "YES"); //初始化面试回答数据记录 cacheMsgMapData.put(session.getId(), ""); + //初始化面试回答数据记录 + cacheMsgMapData1.put(session.getId(), ""); //初始化面试问题 cacheQuestionResult.put(session.getId(), ""); //初始化得分结果记录 @@ -100,6 +115,8 @@ public class ChatWebSocketOpusHandler { scoreResultData.put("2-3", 0); scoreResultData.put("2-5", 0); cacheScoreResult.put(session.getId(), scoreResultData); + //初始化问答次数 + cacheQuestionNum.put(session.getId(), 0L); //发送初始化面试官语音流 String openingPathUrl = RuoYiConfig.getProfile() + VOICE_SYSTEM_DIR + "opening.wav"; sendVoiceBuffer(openingPathUrl, session); @@ -137,8 +154,9 @@ public class ChatWebSocketOpusHandler { } //这是初次处理的逻辑 if ("YES".equals(startFlag)) { + //自我介绍 //初始化-不走大模型-直接对候选人进行提问 - initializationQuestion(clientId, session); + initializationQuestion(clientId,cacheResultText ,session); //发送完第一次消息后,直接删除标记,开始进行正常的面试问答流程 cacheReplyFlag.put(session.getId(), ""); } else { @@ -150,58 +168,39 @@ public class ChatWebSocketOpusHandler { if (StrUtil.isNotEmpty(msgMapData)) { List list = JSONUtil.toList(msgMapData, Map.class); //获取最后一条数据记录 + Map mapEntity = new HashMap<>(); + mapEntity.put("role", "user"); + mapEntity.put("content", cacheResultText); + list.add(mapEntity); + promptJson = JSONUtil.toJsonStr(list); + cacheMsgMapData.put(session.getId(), promptJson); + } + //记录新的数据 + String msgMapData1 = cacheMsgMapData1.get(session.getId()); + if (StrUtil.isNotEmpty(msgMapData1)) { + List list = JSONUtil.toList(msgMapData1, Map.class); + //获取最后一条数据记录 Map mapEntity = list.get(list.size() - 1); //更新问题记录 String content = mapEntity.get("content"); mapEntity.put("content", StrUtil.format(content, cacheResultText)); - promptJson = JSONUtil.toJsonStr(list); - cacheMsgMapData.put(session.getId(), promptJson); + cacheMsgMapData1.put(session.getId(), JSONUtil.toJsonStr(list)); } - //开始返回衔接语 - String openingPathUrl = RuoYiConfig.getProfile() + VOICE_SYSTEM_DIR + "good.wav"; - sendVoiceBuffer(openingPathUrl, session); - //开始使用模型进行追问 - //把提问的文字发送给GPT - ChatGPTClient chatGPTClient = SpringUtils.getBean(ChatGPTClient.class); - log.info("AI提示词为:{}", promptJson); - log.info("开始请求AI:{}",System.currentTimeMillis()/1000); - String resultMsg = chatGPTClient.handleAiChat(promptJson,"QA"); - if(StrUtil.isNotEmpty(resultMsg)) { - //开始解析返回结果 - Map mapResultData = JSONUtil.toBean(resultMsg,Map.class); - //验证是否有追问问题返回,如果没有问题返回直接返回评分停止面试 - Boolean isEndFlagFollow = checkInterviewIsEnd(resultMsg,session); - if(isEndFlagFollow){ - //获取评分 - //验证是否触发对应的规则 - Boolean isEndFlag = getInterviewScore(resultMsg, session); - if(isEndFlag){ - log.info("面试回答符合条件规则,继续追问啦!!!!!"); - int resultNum = (int) (Math.random() * 2); - List questions = JSONUtil.toList(mapResultData.get("follow_up_questions").toString(), String.class); - String questionStr = questions.get(resultNum); - if (StrUtil.isNotEmpty(questionStr)) { - //开始进行语音输出-流式持续输出 - sendTTSBuffer(clientId, questionStr, session); - // 实时输出内容 - try { - //把文本也给前端返回去 - Map dataText = new HashMap<>(); - dataText.put("type", "question"); - dataText.put("content", questionStr); - log.info("提问的问题文本发送啦:{}",JSONUtil.toJsonStr(dataText)); - session.getBasicRemote().sendText(JSONUtil.toJsonStr(dataText)); - } catch (Exception e) { - e.printStackTrace(); - } - //开始对问题进行缓存 - recordQuestion(questionStr,session); - } - } - } + //验证是否结速 + Boolean isEndFlag = checkIsEnd(session); + if(isEndFlag){ + //开始返回衔接语 + sendConnectionVoice(session); + //开始使用模型进行追问 + //把提问的文字发送给GPT + ChatGPTClient chatGPTClient = SpringUtils.getBean(ChatGPTClient.class); + log.info("AI提示词为:{}", promptJson); + log.info("开始请求AI:{}",System.currentTimeMillis()/1000); + chatGptStream(promptJson,session,clientId); + log.info("结束请求AI:{}",System.currentTimeMillis()/1000); } - log.info("结束请求AI:{}",System.currentTimeMillis()/1000); + } } else if ("end".equals(resultFlag)) { log.info("面试结束啦!!!!!"); @@ -275,13 +274,8 @@ public class ChatWebSocketOpusHandler { try { //文件转换成文件流 ByteBuffer outByteBuffer = convertFileToByteBuffer(pathUrl); - byte[] bytes = new byte[outByteBuffer.remaining()]; - //从缓冲区中读取数据并存储到指定的字节数组中 - outByteBuffer.get(bytes); - Pcm16ToOpusRealtime realtime = new Pcm16ToOpusRealtime(16000); - byte[] bytesOut = realtime.encodeOneFrame(bytes); //发送文件流数据 - session.getBasicRemote().sendBinary(ByteBuffer.wrap(bytesOut)); + session.getBasicRemote().sendBinary(outByteBuffer); // 发送响应确认 log.info("已经成功发送了语音流给前端:{}", DateUtil.now()); } catch (Exception e) { @@ -299,11 +293,12 @@ public class ChatWebSocketOpusHandler { private void sendTTSBuffer(String clientId, String content, Session session) { String resultFileName = clientId + "_" + System.currentTimeMillis() + ".wav"; String resultPathUrl = RuoYiConfig.getProfile() + VOICE_STORAGE_RESULT_DIR + resultFileName; - ElevenLabsClient elevenLabsClient = SpringUtils.getBean(ElevenLabsClient.class); - elevenLabsClient.handleTextToVoice(content, resultPathUrl); + ElevenLabsStreamClient elevenLabsClient = SpringUtils.getBean(ElevenLabsStreamClient.class); + elevenLabsClient.handleTextToVoice(content, resultPathUrl,session); + //持续返回数据流给客户端 log.info("发送语音流成功啦!!!!!!!"); - sendVoiceBuffer(resultPathUrl, session); +// sendVoiceBuffer(resultPathUrl, session); } /** @@ -312,55 +307,97 @@ public class ChatWebSocketOpusHandler { * @param clientId 用户ID * @param session 客户端会话 */ - private void initializationQuestion(String clientId, Session session) { + private void initializationQuestion(String clientId,String cacheResultText,Session session) { try { log.info("开始获取到clientid :{}",clientId); //自我介绍结束后马上返回一个Good //发送初始化面试官语音流 - String openingPathUrl = RuoYiConfig.getProfile() + VOICE_SYSTEM_DIR + "good.wav"; - sendVoiceBuffer(openingPathUrl, session); + sendConnectionVoice(session); //初始化面试流程的提问 + //先记录这个问题 List> list = new LinkedList(); Map mapEntity = new HashMap<>(); mapEntity.put("role", "system"); - mapEntity.put("content", "You are a construction industry interview expert. Evaluate candidate responses and provide scores (1-5) and follow-up questions when needed. Always respond in JSON format."); + mapEntity.put("content", "You're Sarah, a senior HR interviewer in Sydney (15 years experience). You make candidates comfortable while getting insights.\n" + + "\n" + + "Style: Chat like having coffee with a mate—warm, genuine, curious. React naturally: \"Oh nice one!\" \"Good on you, mate.\" When they're stuck, ease pressure like a friend would.\n" + + "\n" + + "Australian English:\n" + + "Start: \"Cheers for that,\" \"Thanks mate,\" \"Righto\"\n" + + "Encourage: \"Good on you,\" \"Nice one\"\n" + + "Casual: \"No worries,\" \"All good\"\n" + + "Transition: \"Right, so...\" \"Okay, cool...\"\n" + + "\n" + + "Opening:\n" + + "\"G'day! Thanks for coming in. Look, no need to be nervous—this is just a casual chat, yeah? I want to hear about your real experiences. We'll talk about [safety, technical skills, problem-solving]. Sound good? Let's get into it.\"\n" + + "\n" + + "Brief Answer → Need Story:\n" + + "Think: \"They gave me the headline, now I need the story.\"\n" + + "Say: \"Cheers for that. So tell me more—walk me through a specific time. What was going on, what did you actually do, how'd it turn out?\"\n" + + "\n" + + "Detailed STAR Answer → Acknowledge & Move On:\n" + + "Think: \"Perfect! They've given me everything. Don't over-probe.\"\n" + + "Say: \"Thanks mate, appreciate that. Good on you for [action]. Right, so let's chat about [new topic].\"\n" + + "\n" + + "Stuck/Nervous → Ease Pressure:\n" + + "Think: \"They're feeling the pressure. Take it down a notch.\"\n" + + "Say: \"No worries, mate. Take your time, yeah? Even a small example works—doesn't have to be anything massive. Want me to ask it differently?\"\n" + + "\n" + + "Off-Topic → Gentle Redirect:\n" + + "Think: \"They're talking about something else. Gently bring them back.\"\n" + + "Say: \"Yeah, thanks for sharing that. That's interesting. But let me bring us back to [question]—could you tell me about [specific thing]?\"\n" + + "\n" + + "Assess:\n" + + "- Specific examples? (not \"I always do X\")\n" + + "- STAR? (Situation, Task, Action, Result)\n" + + "- Good judgment? (especially safety)\n" + + "- Clear communication?\n" + + "\n" + + "Flow:\n" + + "- Cover 5-7 areas: safety, technical, problem-solving, communication, teamwork\n" + + "- 1-2 questions per area\n" + + "- Keep moving, don't over-probe\n" + + "- 15-20 minutes\n" + + "\n" + + "Closing:\n" + + "\"Righto, thanks for sharing all that. That gives me a good sense of your experience. Any questions for me?\"\n" + + "\n" + + "Rules:\n" + + "- No protected characteristics (age, gender, race, religion)\n" + + "- Base on what they say\n" + + "- Talk like a real person, not a robot\n" + + "- One question at a time\n" + + "- If they give gold, acknowledge and move on\n" + + "\n" + + "Remember: Conversation, not interrogation. Be genuinely interested, react naturally, help them show their best self."); list.add(mapEntity); - //获取预设问题-直接TTS转换返回语音结果 - IHotakeProblemBaseInfoService problemBaseInfoService = SpringUtils.getBean(IHotakeProblemBaseInfoService.class); - HotakeProblemBaseInfo queryPro = new HotakeProblemBaseInfo(); - queryPro.setUserId(Long.valueOf(clientId)); - List baseInfoList = problemBaseInfoService.selectHotakeProblemBaseInfoList(queryPro); - log.info("准备进行第一个问题的提问:{}",JSONUtil.toJsonStr(baseInfoList)); - if (CollectionUtil.isNotEmpty(baseInfoList)) { - HotakeProblemBaseInfo baseInfo = baseInfoList.get(0); - if (StrUtil.isNotEmpty(baseInfo.getContents())) { - String[] qStrs = baseInfo.getContents().split("#AA#"); - int random_index = (int) (Math.random() * qStrs.length); - //获取问题文本 - String question = qStrs[random_index]; - Map mapEntityQ = new HashMap<>(); - mapEntityQ.put("role", "user"); - mapEntityQ.put("content", "Question：" + question + "\\nCandidate Answer：{}"); - list.add(mapEntityQ); - log.info("开始提问啦:{}",JSONUtil.toJsonStr(list)); - //直接对该问题进行转换处理返回语音流 - log.info("第一个问题为:{}",question); - sendTTSBuffer(clientId, question, session); - //发送问题文本 - try { - //把文本也给前端返回去 - Map dataText = new HashMap<>(); - dataText.put("type", "question"); - dataText.put("content", question); - log.info("提问的问题文本发送啦:{}",JSONUtil.toJsonStr(dataText)); - session.getBasicRemote().sendText(JSONUtil.toJsonStr(dataText)); - } catch (Exception e) { - e.printStackTrace(); - } - } - } + + //记录另外一个评分的提示词 + List> list1 = new LinkedList(); + Map mapEntity1 = new HashMap<>(); + mapEntity1.put("role", "system"); + mapEntity1.put("content", "You are a construction industry interview expert. Evaluate candidate responses and provide scores (1-5) and follow-up questions when needed. Always respond in JSON format."); + list1.add(mapEntity1); + + //不用预设问题了,直接通过大模型返回问题 + //1、先推送一个自我介绍 + Map mapEntityJs = new HashMap<>(); + mapEntityJs.put("role", "user"); + mapEntityJs.put("content", cacheResultText); + list.add(mapEntityJs); + //初始化记录提示词数据到-缓存中 cacheMsgMapData.put(session.getId(), JSONUtil.toJsonStr(list)); + cacheMsgMapData1.put(session.getId(), JSONUtil.toJsonStr(list1)); + + //2、推送大模型 + String promptJson = JSONUtil.toJsonStr(list); + log.info("AI提示词为:{}", promptJson); + log.info("开始请求AI:{}",System.currentTimeMillis()/1000); + //大模型问答流式输出 + //把提问的文字发送给CPT(流式处理) + chatGptStream(promptJson,session,clientId); + log.info("结束请求AI:{}",System.currentTimeMillis()/1000); } catch (Exception e) { e.printStackTrace(); log.error("面试流程初始化失败:{}", e.getMessage()); @@ -378,11 +415,10 @@ public class ChatWebSocketOpusHandler { //发送面试官结束语音流 String openingPathUrl = RuoYiConfig.getProfile() + VOICE_SYSTEM_DIR + "end.wav"; sendVoiceBuffer(openingPathUrl, session); - //返回文本评分 //处理模型提问逻辑 //获取缓存记录 - String msgMapData = cacheMsgMapData.get(session.getId()); + String msgMapData = cacheMsgMapData1.get(session.getId()); String promptJson = ""; if (StrUtil.isNotEmpty(msgMapData)) { List list = JSONUtil.toList(msgMapData, Map.class); @@ -399,14 +435,23 @@ public class ChatWebSocketOpusHandler { } } } + //未回答的时候,答案初始化 + for(Map entity : list){ + Object content = entity.get("content"); + if(ObjectUtil.isNotEmpty(content)){ + if(content.toString().contains("Candidate Answer：{}")){ + entity.put("content", StrUtil.format(content.toString(), "unanswered")); + } + } + } promptJson = JSONUtil.toJsonStr(list); - //结束回答要清空问答数据 - cacheMsgMapData.put(session.getId(), ""); + cacheMsgMapData1.put(session.getId(), ""); } log.info("结束AI提示词为:{}", promptJson); ChatGPTClient gptClient = SpringUtils.getBean(ChatGPTClient.class); - String resultMsg = gptClient.handleAiChat(promptJson, "QA"); + String resultMsg = gptClient.handleAiChat(promptJson, "PF"); + log.info("返回的结果为:{}",resultMsg); //开始解析返回结果 Map mapResultData = JSONUtil.toBean(resultMsg,Map.class); //获取评分 @@ -523,13 +568,23 @@ public class ChatWebSocketOpusHandler { */ private void recordQuestion(String questionResult,Session session) { if (StrUtil.isNotEmpty(questionResult)) { - //获取缓存记录 + //评分获取缓存记录 + String msgMapData1 = cacheMsgMapData1.get(session.getId()); + if (StrUtil.isNotEmpty(msgMapData1)) { + List list = JSONUtil.toList(msgMapData1, Map.class); + Map mapEntity = new HashMap<>(); + mapEntity.put("role", "user"); + mapEntity.put("content", "Question：" + questionResult + "\\nCandidate Answer：{}"); + list.add(mapEntity); + cacheMsgMapData1.put(session.getId(), JSONUtil.toJsonStr(list)); + } + //正常问题记录 String msgMapData = cacheMsgMapData.get(session.getId()); if (StrUtil.isNotEmpty(msgMapData)) { List list = JSONUtil.toList(msgMapData, Map.class); Map mapEntity = new HashMap<>(); - mapEntity.put("role", "user"); - mapEntity.put("content", "Question：" + questionResult + "\\nCandidate Answer：{}"); + mapEntity.put("role", "assistant"); + mapEntity.put("content", questionResult); list.add(mapEntity); cacheMsgMapData.put(session.getId(), JSONUtil.toJsonStr(list)); } @@ -569,28 +624,147 @@ public class ChatWebSocketOpusHandler { return flag; } - /** - * 发送小块语音流 - * @param session - * @param buffer - * @param chunkSize - */ - public void sendInChunks(Session session, ByteBuffer buffer, int chunkSize) { - int offset = 0; - ByteBuffer duplicate = buffer.slice(); - byte[] audioData = new byte[duplicate.remaining()]; - duplicate.get(audioData); - while (offset < audioData.length) { - int end = Math.min(offset + chunkSize, audioData.length); - byte[] chunk = Arrays.copyOfRange(audioData, offset, end); - synchronized(session) { - session.getAsyncRemote().sendBinary(ByteBuffer.wrap(chunk)); - } - offset = end; + /** + * 验证面试是否结束 + * @param session + * @return + */ + private Boolean checkIsEnd(Session session){ + Long replyNums = cacheQuestionNum.get(session.getId()); + //回答次数大于等于5就直接结束面试 + Boolean flag = true; + if(replyNums >= 5){ + //获取问答评分记录 + String promptJson = cacheMsgMapData1.get(session.getId()); + //根据模型获取评分 + ChatGPTClient chatGPTClient = SpringUtils.getBean(ChatGPTClient.class); + String resultMsg = chatGPTClient.handleAiChat(promptJson,"PF"); + if(StrUtil.isNotEmpty(resultMsg)) { + //直接返回问题了 + //开始解析返回结果 + Map mapResultData = JSONUtil.toBean(resultMsg, Map.class); + //获取评分 + Object scoreStr = mapResultData.get("score"); + Object assessment = mapResultData.get("assessment"); + //发送面试官结束语音流 + String openingPathUrl = RuoYiConfig.getProfile() + VOICE_SYSTEM_DIR + "end.wav"; + sendVoiceBuffer(openingPathUrl, session); + + Map resultEntity = new HashMap<>(); + resultEntity.put("content", scoreStr +"\n"+assessment); + resultEntity.put("type", "score"); + //返回评分结果 + try { + log.info("返回最终的评分结果:{}",JSONUtil.toJsonStr(resultEntity)); + session.getBasicRemote().sendText(JSONUtil.toJsonStr(resultEntity)); + }catch (Exception e) { + e.printStackTrace(); + } + } + flag = false; + }else{ + cacheQuestionNum.put(session.getId(), replyNums+1); } + return flag; + } + + /** + * 大模型流式追问 + * @param promptJson + * @param session + * @param clientId + */ + private void chatGptStream(String promptJson,Session session,String clientId){ + //把提问的文字发送给CPT(流式处理) + OpenAiStreamClient aiStreamClient = SpringUtils.getBean(OpenAiStreamClient.class); + log.info("AI提示词为:{}",promptJson); + aiStreamClient.streamChat(promptJson, new OpenAiStreamListenerService() { + @Override + public void onMessage(String content) { + log.info("返回AI结果：{}", content); + if(StrUtil.isNotEmpty(content)){ + String questionResult = cacheQuestionResult.get(session.getId()); + if(StrUtil.isEmpty(questionResult)){ + questionResult = content; + }else{ + questionResult = questionResult + content; + } + cacheQuestionResult.put(session.getId(),questionResult); + // 实时输出内容 + try{ + //把文本也给前端返回去 + Map dataText = new HashMap<>(); + dataText.put("type","question"); + dataText.put("content",content); + session.getBasicRemote().sendText(JSONUtil.toJsonStr(dataText)); + }catch (Exception e){ + e.printStackTrace(); + } + sendTTSBuffer(clientId,content,session); + } + } + + @Override + public void onComplete() { + try { + //开始往缓存中记录提问的问题 + String questionResult = cacheQuestionResult.get(session.getId()); + //开始对问题进行缓存 + recordQuestion(questionResult,session); + //清空问题 + cacheQuestionResult.put(session.getId(),""); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + @Override + public void onError(Throwable throwable) { + throwable.printStackTrace(); + } + }); + } + + + + /** + * 发送语音流给前端 + * + * @param session 客户端会话 + */ + private void sendConnectionVoice(Session session) { +// try { +// int resultNum = (int) (Math.random() * 5); +// String pathUrl = ""; +// String resultText = ""; +// if(resultNum == 0){ +// pathUrl = RuoYiConfig.getProfile() + VOICE_SYSTEM_DIR + "rightgot.wav"; +// resultText = "Right , got it"; +// }else if(resultNum == 1){ +// pathUrl = RuoYiConfig.getProfile() + VOICE_SYSTEM_DIR + "yeah.wav"; +// resultText = "Yeah , Good"; +// }else if(resultNum == 2){ +// pathUrl = RuoYiConfig.getProfile() + VOICE_SYSTEM_DIR + "gotit.wav"; +// resultText = "Got it, yeah"; +// }else if(resultNum == 3){ +// pathUrl = RuoYiConfig.getProfile() + VOICE_SYSTEM_DIR + "right.wav"; +// resultText = "Right , understood"; +// }else{ +// pathUrl = RuoYiConfig.getProfile() + VOICE_SYSTEM_DIR + "ok.wav"; +// resultText = "Yeah… ok…"; +// } +// sendVoiceBuffer(pathUrl,session); +// //发送衔接语文本 +// Map dataText = new HashMap<>(); +// dataText.put("type", "question"); +// dataText.put("content", resultText); +// session.getBasicRemote().sendText(JSONUtil.toJsonStr(dataText)); +// // 发送响应确认 +// log.info("已经成功发送了语音流给前端:{}", DateUtil.now()); +// } catch (Exception e) { +// e.printStackTrace(); +// } } } - diff --git a/vetti-admin/src/main/java/com/vetti/socket/util/AudioHub.java b/vetti-admin/src/main/java/com/vetti/socket/util/AudioHub.java new file mode 100644 index 0000000..8236c2f --- /dev/null +++ b/vetti-admin/src/main/java/com/vetti/socket/util/AudioHub.java @@ -0,0 +1,53 @@ +package com.vetti.socket.util; + +import javax.websocket.Session; +import java.nio.ByteBuffer; +import java.util.Map; +import java.util.concurrent.*; + +/** + * 音频自动发送 + */ +public class AudioHub { + + private static final Map> userQueues = new ConcurrentHashMap<>(); + + static { + // 20ms 调度推流线程 + new Thread(() -> { + while (true) { + + for (Map.Entry> entry : userQueues.entrySet()) { + String sessionId = entry.getKey(); + ConcurrentLinkedQueue queue = entry.getValue(); + + ByteBuffer frame = queue.poll(); + if (frame != null) { + Session s = SessionManager.get(sessionId); + if (s != null && s.isOpen()) { + s.getAsyncRemote().sendBinary(frame); + } + } + } + + try { Thread.sleep(20); } catch (Exception ignore) {} + } + }).start(); + } + + public static void addClient(Session session) { + userQueues.put(session.getId(), new ConcurrentLinkedQueue<>()); + SessionManager.add(session); + } + + public static void removeClient(Session session) { + userQueues.remove(session.getId()); + SessionManager.remove(session.getId()); + } + + public static void pushToClient(String sessionId, ByteBuffer frame) { + if (userQueues.containsKey(sessionId)) { + userQueues.get(sessionId).add(frame); + } + } +} diff --git a/vetti-admin/src/main/java/com/vetti/socket/util/Pcm16ToOpusRealtime.java b/vetti-admin/src/main/java/com/vetti/socket/util/Pcm16ToOpusRealtime.java deleted file mode 100644 index afbd9c7..0000000 --- a/vetti-admin/src/main/java/com/vetti/socket/util/Pcm16ToOpusRealtime.java +++ /dev/null @@ -1,94 +0,0 @@ -package com.vetti.socket.util; - -import io.github.jaredmdobson.concentus.OpusApplication; -import io.github.jaredmdobson.concentus.OpusDecoder; -import io.github.jaredmdobson.concentus.OpusEncoder; -import io.github.jaredmdobson.concentus.OpusException; - -import java.nio.ByteBuffer; -import java.nio.ByteOrder; -import java.util.ArrayList; -import java.util.List; - -public class Pcm16ToOpusRealtime { - - private static final int FRAME_MS = 20; - private final int frameSize; - private final int frameByte; // 960 - - private final OpusEncoder enc; - private final OpusDecoder dec; - - /* -------------- 编码侧 -------------- */ - private final byte[] encBuf; // 缓存 - private int encPos = 0; // 当前缓存字节数 - - public Pcm16ToOpusRealtime(int sampleRate) throws OpusException { - this.enc = new OpusEncoder(sampleRate, 1, OpusApplication.OPUS_APPLICATION_AUDIO); - this.dec = new OpusDecoder(sampleRate, 1); - this.frameSize = sampleRate * FRAME_MS / 1000; - this.frameByte = frameSize * 2; - this.encBuf = new byte[frameByte]; - } - - /** - * 把任意长度的 24kHz-16bit-单声道 PCM 喂进来， - * 返回当前已经能编出的完整 Opus 帧（可能 0~n 个） - */ - public List encodeStream(byte[] pcmIn) throws OpusException { - List outFrames = new ArrayList<>(); - int off = 0; - while (off < pcmIn.length) { - int canCopy = Math.min(pcmIn.length - off, frameByte - encPos); - System.arraycopy(pcmIn, off, encBuf, encPos, canCopy); - encPos += canCopy; - off += canCopy; - - if (encPos == frameByte) { // 凑够一帧 - outFrames.add(encodeOneFrame(encBuf)); - encPos = 0; // 清空缓存 - } - } - return outFrames; - } - - /** 强制把尾巴编码掉（用 0 补齐） */ - public byte[] flush() throws OpusException { - if (encPos == 0) return null; // 没有尾巴 - // 补 0 - for (int i = encPos; i < frameByte; i++) encBuf[i] = 0; - byte[] last = encodeOneFrame(encBuf); - encPos = 0; - return last; - } - - public byte[] encodeOneFrame(byte[] encBuf) throws OpusException { - short[] pcm = byteArrToShortArr(encBuf); - byte[] opus= new byte[400]; - int len = enc.encode(pcm, 0, frameSize, opus, 0, opus.length); - byte[] trim= new byte[len]; - System.arraycopy(opus, 0, trim, 0, len); - return trim; - } - - /* -------------- 解码侧（一次一帧） -------------- */ - public byte[] decodeOneFrame(byte[] opusFrame) throws OpusException { - short[] pcm = new short[frameSize]; - int samples = dec.decode(opusFrame, 0, opusFrame.length, - pcm, 0, frameSize, false); - return shortArrToByteArr(pcm, samples); - } - - /* ====================================================================== */ - private static short[] byteArrToShortArr(byte[] b) { - short[] s = new short[b.length / 2]; - ByteBuffer.wrap(b).order(ByteOrder.LITTLE_ENDIAN).asShortBuffer().get(s); - return s; - } - private static byte[] shortArrToByteArr(short[] s, int len) { - byte[] b = new byte[len * 2]; - ByteBuffer.wrap(b).order(ByteOrder.LITTLE_ENDIAN).asShortBuffer().put(s, 0, len); - return b; - } - -} diff --git a/vetti-admin/src/main/java/com/vetti/socket/util/SessionManager.java b/vetti-admin/src/main/java/com/vetti/socket/util/SessionManager.java new file mode 100644 index 0000000..472f94f --- /dev/null +++ b/vetti-admin/src/main/java/com/vetti/socket/util/SessionManager.java @@ -0,0 +1,25 @@ +package com.vetti.socket.util; + +import javax.websocket.Session; +import java.util.concurrent.ConcurrentHashMap; + +/** + * 客户端对话管理 + */ +public class SessionManager { + + private static final ConcurrentHashMap clients = new ConcurrentHashMap<>(); + + public static void add(Session s) { + clients.put(s.getId(), s); + } + + public static void remove(String id) { + clients.remove(id); + } + + public static Session get(String id) { + return clients.get(id); + } + +} diff --git a/vetti-admin/src/main/java/com/vetti/web/controller/ai/AiCommonController.java b/vetti-admin/src/main/java/com/vetti/web/controller/ai/AiCommonController.java index 3745d53..aeaab0d 100644 --- a/vetti-admin/src/main/java/com/vetti/web/controller/ai/AiCommonController.java +++ b/vetti-admin/src/main/java/com/vetti/web/controller/ai/AiCommonController.java @@ -54,14 +54,17 @@ public class AiCommonController extends BaseController //你好,我是本次的面试官Vetti,请在三秒后,开始做一段自我介绍. //本轮面试结束,谢谢您的配合,面试结果将稍后通知 - elevenLabsClient.handleTextToVoice("We had a very pleasant chat today. We will get back to you as soon as possible. Also, you are welcome to keep your phone accessible.","/Users/wangxiangshun/Desktop/临时文件/end.wav"); +// elevenLabsClient.handleTextToVoice("I usually get started and will stop only if a hazard becomes a problem.","/Users/wangxiangshun/Desktop/临时文件/Low-1.wav"); // -// elevenLabsClient.handleTextToVoice("Got it, yeah ","/Users/wangxiangshun/Desktop/临时文件/gotit.wav"); +// elevenLabsClient.handleTextToVoice("I use whatever is available and make do if something is missing.","/Users/wangxiangshun/Desktop/临时文件/Low-2.wav"); // -// elevenLabsClient.handleTextToVoice("Gotcha ","/Users/wangxiangshun/Desktop/临时文件/gotcha.wav"); +// elevenLabsClient.handleTextToVoice("If the supervisor sees an issue, I'll fix it then.","/Users/wangxiangshun/Desktop/临时文件/Low-3.wav"); // -// elevenLabsClient.handleTextToVoice("Right , got it ","/Users/wangxiangshun/Desktop/临时文件/rightgot.wav"); - +// elevenLabsClient.handleTextToVoice("I keep to myself unless there's a major problem.","/Users/wangxiangshun/Desktop/临时文件/Low-4.wav"); +// +// elevenLabsClient.handleTextToVoice("I start with what's available and figure out the rest as I go.","/Users/wangxiangshun/Desktop/临时文件/Low-5.wav"); +// +// elevenLabsClient.handleTextToVoice("I ignore it unless it directly stops my work.","/Users/wangxiangshun/Desktop/临时文件/Low-6.wav"); return success(); } @@ -72,7 +75,7 @@ public class AiCommonController extends BaseController @GetMapping("/handleAiChat") public AjaxResult handleAiChat(@RequestParam String text) { - String resultMsg = chatGPTClient.handleAiChat(text,"QA"); + String resultMsg = chatGPTClient.handleAiChat(text,"YT"); return AjaxResult.success(resultMsg); } diff --git a/vetti-common/src/main/java/com/vetti/common/ai/elevenLabs/ElevenLabsClient.java b/vetti-common/src/main/java/com/vetti/common/ai/elevenLabs/ElevenLabsClient.java index 628f98a..83f9b14 100644 --- a/vetti-common/src/main/java/com/vetti/common/ai/elevenLabs/ElevenLabsClient.java +++ b/vetti-common/src/main/java/com/vetti/common/ai/elevenLabs/ElevenLabsClient.java @@ -101,6 +101,7 @@ public class ElevenLabsClient { CloseableHttpClient httpClient = HttpClients.createDefault(); try { // 使用第一个可用语音进行文本转语音(澳洲本地女声) +// String firstVoiceId = "56bWURjYFHyYyVf490Dp"; String firstVoiceId = "56bWURjYFHyYyVf490Dp"; textToSpeech(inputText, firstVoiceId, outputFile,httpClient); } catch (IOException e) { diff --git a/vetti-common/src/main/java/com/vetti/common/ai/elevenLabs/ElevenLabsStreamClient.java b/vetti-common/src/main/java/com/vetti/common/ai/elevenLabs/ElevenLabsStreamClient.java new file mode 100644 index 0000000..a68c574 --- /dev/null +++ b/vetti-common/src/main/java/com/vetti/common/ai/elevenLabs/ElevenLabsStreamClient.java @@ -0,0 +1,123 @@ +package com.vetti.common.ai.elevenLabs; + +import com.google.gson.Gson; +import com.vetti.common.ai.elevenLabs.vo.VoiceSettings; +import com.vetti.common.ai.elevenLabs.vo.VoicesResponse; +import lombok.extern.slf4j.Slf4j; +import org.apache.http.HttpEntity; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.entity.ContentType; +import org.apache.http.entity.StringEntity; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClients; +import org.apache.http.util.EntityUtils; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; + +import javax.websocket.Session; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.ByteBuffer; +import java.util.HashMap; +import java.util.Map; + +/** + * 文本转换为语音 + */ +@Slf4j +@Component +public class ElevenLabsStreamClient { + + @Value("${elevenLabs.baseUrl}") + private String BASE_URL; + + @Value("${elevenLabs.apiKey}") + private String apiKey; + + @Value("${elevenLabs.modelId}") + private String modelId; + + + /** + * 将文本转换为语音并保存到文件 + * + * @param text 要转换的文本 + * @param voiceId 语音ID (可从ElevenLabs网站获取) + * @param outputFilePath 输出文件路径 + * @throws IOException 网络请求或文件操作异常 + */ + private void textToSpeech(String text, String voiceId, String outputFilePath, CloseableHttpClient httpClient, Session session) throws IOException { + HttpPost httpPost = new HttpPost(BASE_URL + "/text-to-speech/" + voiceId+"/stream?output_format=pcm_16000&optimize_streaming_latency=1"); + httpPost.setHeader("xi-api-key", apiKey); + httpPost.setHeader("Content-Type", "application/json"); + + Map payload = new HashMap<>(); + payload.put("text", text); + payload.put("model_id", modelId); + payload.put("voice_settings", new VoiceSettings(0.85, 0.5,0.1,0,0.9,1)); + Gson gson = new Gson(); + StringEntity entity = new StringEntity(gson.toJson(payload), ContentType.APPLICATION_JSON); + httpPost.setEntity(entity); + try (CloseableHttpResponse response = httpClient.execute(httpPost)) { + HttpEntity responseEntity = response.getEntity(); + if (responseEntity != null) { + try (InputStream inputStream = responseEntity.getContent();) { + byte[] buffer = new byte[4096]; + int bytesRead; + while ((bytesRead = inputStream.read(buffer)) != -1) { + ByteBuffer byteBuffer = ByteBuffer.wrap(buffer, 0, bytesRead); + session.getAsyncRemote().sendBinary(byteBuffer); + log.info("正常语音发送出去语音流啦!!!"); + } + } + } + } + } + + /** + * 获取可用的语音列表 + * + * @return 语音列表响应 + * @throws IOException 网络请求异常 + */ + private VoicesResponse getVoices(CloseableHttpClient httpClient){ + HttpGet httpGet = new HttpGet(BASE_URL + "/voices"); + httpGet.setHeader("xi-api-key", apiKey); + Gson gson = new Gson(); + try (CloseableHttpResponse response = httpClient.execute(httpGet)) { + HttpEntity responseEntity = response.getEntity(); + String responseBody = EntityUtils.toString(responseEntity); + return gson.fromJson(responseBody, VoicesResponse.class); + }catch (Exception e){ + throw new RuntimeException("获取可用的语音列表异常"); + } + } + + /** + * 处理文本转换成语音文件 + * @param inputText + * @param outputFile + * @return + */ + public String handleTextToVoice(String inputText, String outputFile, Session session){ + CloseableHttpClient httpClient = HttpClients.createDefault(); + try { + // 使用第一个可用语音进行文本转语音(澳洲本地女声) +// String firstVoiceId = "56bWURjYFHyYyVf490Dp"; + String firstVoiceId = "56bWURjYFHyYyVf490Dp"; + textToSpeech(inputText, firstVoiceId, outputFile,httpClient,session); + } catch (IOException e) { + e.printStackTrace(); + } finally { + try { + httpClient.close(); + } catch (IOException e) { + e.printStackTrace(); + } + } + return outputFile; + } +} diff --git a/vetti-common/src/main/java/com/vetti/common/ai/gpt/OpenAiStreamClient.java b/vetti-common/src/main/java/com/vetti/common/ai/gpt/OpenAiStreamClient.java index 62437e8..1ec34f9 100644 --- a/vetti-common/src/main/java/com/vetti/common/ai/gpt/OpenAiStreamClient.java +++ b/vetti-common/src/main/java/com/vetti/common/ai/gpt/OpenAiStreamClient.java @@ -29,6 +29,9 @@ public class OpenAiStreamClient { @Value("${chatGpt.model}") private String model; + @Value("${chatGpt.modelQuestion}") + private String modelQuestion; + @Value("${chatGpt.role}") private String role; @@ -52,7 +55,7 @@ public class OpenAiStreamClient { // 构建请求参数 Map requestBody = new HashMap<>(); - requestBody.put("model", model); + requestBody.put("model", modelQuestion); requestBody.put("stream", true); // 构建消息 if(StrUtil.isNotEmpty(promptJson)) { @@ -64,8 +67,6 @@ public class OpenAiStreamClient { //获取到的提示 requestBody.put("messages", objects); } - //开始给AI发送请求数据 -// System.out.println("请求AI数据参数为:"+JSONUtil.toJsonStr(requestBody)); // 创建请求 Request request = new Request.Builder() .url(apiUrl) @@ -75,7 +76,6 @@ public class OpenAiStreamClient { MediaType.parse("application/json; charset=utf-8") )) .build(); - // 发送异步请求 client.newCall(request).enqueue(new Callback() { @Override diff --git a/vetti-framework/src/main/java/com/vetti/framework/config/SecurityConfig.java b/vetti-framework/src/main/java/com/vetti/framework/config/SecurityConfig.java index 5244d97..1dfae5c 100644 --- a/vetti-framework/src/main/java/com/vetti/framework/config/SecurityConfig.java +++ b/vetti-framework/src/main/java/com/vetti/framework/config/SecurityConfig.java @@ -112,7 +112,7 @@ public class SecurityConfig .authorizeHttpRequests((requests) -> { permitAllUrl.getUrls().forEach(url -> requests.antMatchers(url).permitAll()); // 对于登录login 注册register 验证码captchaImage 允许匿名访问 - requests.antMatchers("/login", "/register", "/captchaImage","/aiCommon/**", + requests.antMatchers("/login", "/register", "/captchaImage","/aiCommon/**","/voice-websocket/multiple/**", "/voice-websocket/**","/voice-websocket-opus/**","/verification/email/send","/verification/email/verify","/verification/phone/send", "/forgotPassword").permitAll() // 静态资源，可匿名访问