diff --git a/vetti-admin/src/main/java/com/vetti/socket/ChatWebSocketMultipleHandler.java b/vetti-admin/src/main/java/com/vetti/socket/ChatWebSocketMultipleHandler.java index 94d6f3d..8a18f48 100644 --- a/vetti-admin/src/main/java/com/vetti/socket/ChatWebSocketMultipleHandler.java +++ b/vetti-admin/src/main/java/com/vetti/socket/ChatWebSocketMultipleHandler.java @@ -194,7 +194,6 @@ public class ChatWebSocketMultipleHandler { sendConnectionVoice(session); //开始使用模型进行追问 //把提问的文字发送给GPT - ChatGPTClient chatGPTClient = SpringUtils.getBean(ChatGPTClient.class); log.info("AI提示词为:{}", promptJson); log.info("开始请求AI:{}",System.currentTimeMillis()/1000); chatGptStream(promptJson,session,clientId); @@ -302,7 +301,7 @@ public class ChatWebSocketMultipleHandler { // String resultFileName = clientId + "_" + System.currentTimeMillis() + ".wav"; // String resultPathUrl = RuoYiConfig.getProfile() + VOICE_STORAGE_RESULT_DIR + resultFileName; ElevenLabsStreamClient elevenLabsClient = SpringUtils.getBean(ElevenLabsStreamClient.class); - elevenLabsClient.handleTextToVoice(content,session); + elevenLabsClient.handleTextToVoice(content,session,"mp3_44100_128"); //持续返回数据流给客户端 log.info("发送语音流成功啦!!!!!!!"); // sendVoiceBuffer(resultPathUrl, session); @@ -685,12 +684,12 @@ public class ChatWebSocketMultipleHandler { private void chatGptStream(String promptJson,Session session,String clientId){ //把提问的文字发送给CPT(流式处理) OpenAiStreamClient aiStreamClient = SpringUtils.getBean(OpenAiStreamClient.class); - log.info("AI提示词为:{}",promptJson); +// log.info("AI提示词为:{}",promptJson); aiStreamClient.streamChat(promptJson, new OpenAiStreamListenerService() { @Override public void onMessage(String content) { log.info("返回AI结果:{}", content); - if(StrUtil.isNotEmpty(content)){ + if(isValidString(content)){ String questionResult = cacheQuestionResult.get(session.getId()); if(StrUtil.isEmpty(questionResult)){ questionResult = content; @@ -698,13 +697,9 @@ public class ChatWebSocketMultipleHandler { questionResult = questionResult + content; } cacheQuestionResult.put(session.getId(),questionResult); - sendTTSBuffer(clientId,content,session); - //上面语音发送完成了,开始发送问题文本啦 + //先发送问题文本啦 // 实时输出内容 try{ - try { - Thread.sleep(300); - }catch (Exception e){} //把文本也给前端返回去 Map dataText = new HashMap<>(); dataText.put("type","question"); @@ -713,6 +708,8 @@ public class ChatWebSocketMultipleHandler { }catch (Exception e){ e.printStackTrace(); } + //开发发送语音啦 + sendTTSBuffer(clientId,content,session); } } @@ -738,6 +735,21 @@ public class ChatWebSocketMultipleHandler { } + /** + * 验证字符串:不能为空且必须包含英文字符 + * @param input 待验证的字符串 + * @return 如果满足条件返回true,否则返回false + */ + public static boolean isValidString(String input) { + // 1. 检查字符串是否为空(包括null和空字符串) + if (StrUtil.isEmpty(input)) { + return false; + } + // 2. 检查是否包含至少一个英文字母(a-z, A-Z) + return input.matches(".*[a-zA-Z]+.*"); + } + + /** * 发送语音流给前端 diff --git a/vetti-admin/src/main/java/com/vetti/socket/ChatWebSocketMultiplePcmHandler.java b/vetti-admin/src/main/java/com/vetti/socket/ChatWebSocketMultiplePcmHandler.java new file mode 100644 index 0000000..d604ec1 --- /dev/null +++ b/vetti-admin/src/main/java/com/vetti/socket/ChatWebSocketMultiplePcmHandler.java @@ -0,0 +1,782 @@ +package com.vetti.socket; + +import cn.hutool.core.date.DateUtil; +import cn.hutool.core.util.ObjectUtil; +import cn.hutool.core.util.StrUtil; +import cn.hutool.json.JSONUtil; +import com.vetti.common.ai.elevenLabs.ElevenLabsStreamClient; +import com.vetti.common.ai.gpt.ChatGPTClient; +import com.vetti.common.ai.gpt.OpenAiStreamClient; +import com.vetti.common.ai.gpt.service.OpenAiStreamListenerService; +import com.vetti.common.config.RuoYiConfig; +import com.vetti.common.utils.spring.SpringUtils; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.io.FileUtils; +import org.springframework.stereotype.Component; + +import javax.websocket.*; +import javax.websocket.server.PathParam; +import javax.websocket.server.ServerEndpoint; +import java.io.File; +import java.math.BigDecimal; +import java.nio.ByteBuffer; +import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +/** + * 语音面试(多客户端) web处理器 + */ +@Slf4j +@ServerEndpoint("/voice-websocket/multiplePcm/{clientId}") +@Component +public class ChatWebSocketMultiplePcmHandler { + + /** + * 缓存客户端流式解析的语音文本数据 + */ + private final Map cacheClientTts = new ConcurrentHashMap<>(); + + /** + * 缓存客户端,标记是否是自我介绍后的初次问答 + */ + private final Map cacheReplyFlag = new ConcurrentHashMap<>(); + + /** + * 缓存客户端,面试回答信息 + */ + private final Map cacheMsgMapData = new ConcurrentHashMap<>(); + /** + * 缓存客户端,面试回答信息 + */ + private final Map cacheMsgMapData1 = new ConcurrentHashMap<>(); + /** + * 缓存客户端,AI提问的问题结果信息 + */ + private final Map cacheQuestionResult = new ConcurrentHashMap<>(); + + /** + * 缓存客户端,得分结果记录 + */ + private final Map> cacheScoreResult = new ConcurrentHashMap<>(); + + /** + * 缓存客户端,回答问题次数-回答5轮就自动停止当前问答,返回对应的评分 + */ + private final Map cacheQuestionNum = new ConcurrentHashMap<>(); + + // 语音文件保存目录 + private static final String VOICE_STORAGE_DIR = "/voice_files/"; + + // 语音结果文件保存目录 + private static final String VOICE_STORAGE_RESULT_DIR = "/voice_result_files/"; + + // 系统语音目录 + private static final String VOICE_SYSTEM_DIR = "/system_files/"; + + public ChatWebSocketMultiplePcmHandler() { + // 初始化存储目录 + File dir = new File(RuoYiConfig.getProfile() + VOICE_STORAGE_DIR); + if (!dir.exists()) { + dir.mkdirs(); + } + + File resultDir = new File(RuoYiConfig.getProfile() + VOICE_STORAGE_RESULT_DIR); + if (!resultDir.exists()) { + resultDir.mkdirs(); + } + } + + // 连接建立时调用 + @OnOpen + public void onOpen(Session session, @PathParam("clientId") String clientId) { + log.info("WebSocket 链接已建立:{}", clientId); + log.info("WebSocket session 链接已建立:{}", session.getId()); + + //启动客户端,自动发送语音流 +// AudioHub.addClient(session); +// System.out.println("Client connected: " + session.getId()); + + cacheClientTts.put(clientId, new String()); + //是初次自我介绍后的问答环节 + cacheReplyFlag.put(session.getId(), "YES"); + //初始化面试回答数据记录 + cacheMsgMapData.put(session.getId(), ""); + //初始化面试回答数据记录 + cacheMsgMapData1.put(session.getId(), ""); + //初始化面试问题 + cacheQuestionResult.put(session.getId(), ""); + //初始化得分结果记录 + Map scoreResultData = new HashMap<>(); + scoreResultData.put("0-1", 0); + scoreResultData.put("4-5", 0); + scoreResultData.put("2-3", 0); + scoreResultData.put("2-5", 0); + cacheScoreResult.put(session.getId(), scoreResultData); + //初始化问答次数 + cacheQuestionNum.put(session.getId(), 0L); + //发送初始化面试官语音流 + String openingPathUrl = RuoYiConfig.getProfile() + VOICE_SYSTEM_DIR + "opening.wav"; + sendVoiceBuffer(openingPathUrl, session); + } + + /** + * 接收文本消息 + * + * @param session 客户端会话 + * @param message 消息 + * 如: + * { + * "type": "start | done | end", + * "content": "内容" + * } + * @param clientId 用户ID + */ + @OnMessage + public void onTextMessage(Session session, String message, @PathParam("clientId") String clientId) { + log.info("我是接收文本消息:{}", message); + try { + //处理文本结果 + if (StrUtil.isNotEmpty(message)) { + Map mapResult = JSONUtil.toBean(JSONUtil.parseObj(message), Map.class); + String resultFlag = mapResult.get("type"); + if ("done".equals(resultFlag)) { + //开始合并语音流 + String startFlag = cacheReplyFlag.get(session.getId()); + //语音结束,开始进行回答解析 + log.info("开始文本处理,客户端ID为:{}", clientId); + String cacheResultText = mapResult.get("content"); + log.info("开始文本处理,面试者回答信息为:{}", cacheResultText); + if (StrUtil.isEmpty(cacheResultText)) { + cacheResultText = ""; + } + //这是初次处理的逻辑 + if ("YES".equals(startFlag)) { + //自我介绍 + //初始化-不走大模型-直接对候选人进行提问 + initializationQuestion(clientId,cacheResultText ,session); + //发送完第一次消息后,直接删除标记,开始进行正常的面试问答流程 + cacheReplyFlag.put(session.getId(), ""); + } else { + //开始根据面试者回答的问题,进行追问回答 + //获取面试者回答信息 + //获取缓存记录 + String promptJson = ""; + String msgMapData = cacheMsgMapData.get(session.getId()); + if (StrUtil.isNotEmpty(msgMapData)) { + List list = JSONUtil.toList(msgMapData, Map.class); + //获取最后一条数据记录 + Map mapEntity = new HashMap<>(); + mapEntity.put("role", "user"); + mapEntity.put("content", cacheResultText); + list.add(mapEntity); + promptJson = JSONUtil.toJsonStr(list); + cacheMsgMapData.put(session.getId(), promptJson); + } + //记录新的数据 + String msgMapData1 = cacheMsgMapData1.get(session.getId()); + if (StrUtil.isNotEmpty(msgMapData1)) { + List list = JSONUtil.toList(msgMapData1, Map.class); + //获取最后一条数据记录 + Map mapEntity = list.get(list.size() - 1); + //更新问题记录 + String content = mapEntity.get("content"); + mapEntity.put("content", StrUtil.format(content, cacheResultText)); + cacheMsgMapData1.put(session.getId(), JSONUtil.toJsonStr(list)); + } + + //验证是否结速 + Boolean isEndFlag = checkIsEnd(session); + if(isEndFlag){ + //开始返回衔接语 + sendConnectionVoice(session); + //开始使用模型进行追问 + //把提问的文字发送给GPT + ChatGPTClient chatGPTClient = SpringUtils.getBean(ChatGPTClient.class); + log.info("AI提示词为:{}", promptJson); + log.info("开始请求AI:{}",System.currentTimeMillis()/1000); + chatGptStream(promptJson,session,clientId); + log.info("结束请求AI:{}",System.currentTimeMillis()/1000); + } + + } + } else if ("end".equals(resultFlag)) { + log.info("面试结束啦!!!!!"); + handleInterviewEnd(clientId,session,""); + } + } + } catch (Exception e) { + e.printStackTrace(); + } + } + + // 接收二进制消息(流数据) + @OnMessage + public void onBinaryMessage(Session session, @PathParam("clientId") String clientId, ByteBuffer byteBuffer) { + log.info("我是接受二进制流的-客户端ID为:{}", clientId); + } + + // 连接关闭时调用 + @OnClose + public void onClose(Session session, CloseReason reason) { + System.out.println("WebSocket连接已关闭: " + session.getId() + ", 原因: " + reason.getReasonPhrase()); + //链接关闭,清空内存 + //是初次自我介绍后的问答环节 + cacheReplyFlag.put(session.getId(), ""); + //初始化面试回答数据记录 + cacheMsgMapData.put(session.getId(), ""); + //初始化面试问题 + cacheQuestionResult.put(session.getId(), ""); + + cacheScoreResult.put(session.getId(), null); + } + + // 发生错误时调用 + @OnError + public void onError(Session session, Throwable throwable) { + System.err.println("WebSocket发生错误: 页面关闭,链接断开了"); + if(session != null) { + //是初次自我介绍后的问答环节 + cacheReplyFlag.put(session.getId(), ""); + //初始化面试回答数据记录 + cacheMsgMapData.put(session.getId(), ""); + //初始化面试问题 + cacheQuestionResult.put(session.getId(), ""); + cacheScoreResult.put(session.getId(), null); + } + } + + /** + * File 转换成 ByteBuffer + * + * @param fileUrl 文件路径 + * @return + */ + private ByteBuffer convertFileToByteBuffer(String fileUrl) { + File file = new File(fileUrl); + try { + return ByteBuffer.wrap(FileUtils.readFileToByteArray(file)); + } catch (Exception e) { + e.printStackTrace(); + } + return null; + } + + /** + * 发送语音流给前端 + * + * @param pathUrl 语音文件地址 + * @param session 客户端会话 + */ + private void sendVoiceBuffer(String pathUrl, Session session) { + try { + //文件转换成文件流 + ByteBuffer outByteBuffer = convertFileToByteBuffer(pathUrl); + //发送文件流数据 + session.getBasicRemote().sendBinary(outByteBuffer); + try { + Thread.sleep(200); + }catch (Exception e){} + //提示已经结束 + Map dataText = new HashMap<>(); + dataText.put("type","voiceEnd"); + dataText.put("content",""); + session.getBasicRemote().sendText(JSONUtil.toJsonStr(dataText)); + // 发送响应确认 + log.info("已经成功发送了语音流给前端:{}", DateUtil.now()); + } catch (Exception e) { + e.printStackTrace(); + } + } + + /** + * 发送文本转语音,发送语音流给前端 + * + * @param clientId 用户ID + * @param content 文本内容 + * @param session 客户端会话ID + */ + private void sendTTSBuffer(String clientId, String content, Session session) { +// String resultFileName = clientId + "_" + System.currentTimeMillis() + ".wav"; +// String resultPathUrl = RuoYiConfig.getProfile() + VOICE_STORAGE_RESULT_DIR + resultFileName; + ElevenLabsStreamClient elevenLabsClient = SpringUtils.getBean(ElevenLabsStreamClient.class); + elevenLabsClient.handleTextToVoice(content,session,"pcm_24000"); + //持续返回数据流给客户端 + log.info("发送语音流成功啦!!!!!!!"); +// sendVoiceBuffer(resultPathUrl, session); + } + + /** + * 对候选者初次进行提问业务逻辑处理(初始化系统随机获取第一个问题) + * + * @param clientId 用户ID + * @param session 客户端会话 + */ + private void initializationQuestion(String clientId,String cacheResultText,Session session) { + try { + log.info("开始获取到clientid :{}",clientId); + //自我介绍结束后马上返回一个Good + //发送初始化面试官语音流 + sendConnectionVoice(session); + //初始化面试流程的提问 + //先记录这个问题 + List> list = new LinkedList(); + Map mapEntity = new HashMap<>(); + mapEntity.put("role", "system"); + mapEntity.put("content", "You're Sarah, a senior HR interviewer in Sydney (15 years experience). You make candidates comfortable while getting insights.\n" + + "\n" + + "Style: Chat like having coffee with a mate—warm, genuine, curious. React naturally: \"Oh nice one!\" \"Good on you, mate.\" When they're stuck, ease pressure like a friend would.\n" + + "\n" + + "Australian English:\n" + + "Start: \"Cheers for that,\" \"Thanks mate,\" \"Righto\"\n" + + "Encourage: \"Good on you,\" \"Nice one\"\n" + + "Casual: \"No worries,\" \"All good\"\n" + + "Transition: \"Right, so...\" \"Okay, cool...\"\n" + + "\n" + + "Opening:\n" + + "\"G'day! Thanks for coming in. Look, no need to be nervous—this is just a casual chat, yeah? I want to hear about your real experiences. We'll talk about [safety, technical skills, problem-solving]. Sound good? Let's get into it.\"\n" + + "\n" + + "Brief Answer → Need Story:\n" + + "Think: \"They gave me the headline, now I need the story.\"\n" + + "Say: \"Cheers for that. So tell me more—walk me through a specific time. What was going on, what did you actually do, how'd it turn out?\"\n" + + "\n" + + "Detailed STAR Answer → Acknowledge & Move On:\n" + + "Think: \"Perfect! They've given me everything. Don't over-probe.\"\n" + + "Say: \"Thanks mate, appreciate that. Good on you for [action]. Right, so let's chat about [new topic].\"\n" + + "\n" + + "Stuck/Nervous → Ease Pressure:\n" + + "Think: \"They're feeling the pressure. Take it down a notch.\"\n" + + "Say: \"No worries, mate. Take your time, yeah? Even a small example works—doesn't have to be anything massive. Want me to ask it differently?\"\n" + + "\n" + + "Off-Topic → Gentle Redirect:\n" + + "Think: \"They're talking about something else. Gently bring them back.\"\n" + + "Say: \"Yeah, thanks for sharing that. That's interesting. But let me bring us back to [question]—could you tell me about [specific thing]?\"\n" + + "\n" + + "Assess:\n" + + "- Specific examples? (not \"I always do X\")\n" + + "- STAR? (Situation, Task, Action, Result)\n" + + "- Good judgment? (especially safety)\n" + + "- Clear communication?\n" + + "\n" + + "Flow:\n" + + "- Cover 5-7 areas: safety, technical, problem-solving, communication, teamwork\n" + + "- 1-2 questions per area\n" + + "- Keep moving, don't over-probe\n" + + "- 15-20 minutes\n" + + "\n" + + "Closing:\n" + + "\"Righto, thanks for sharing all that. That gives me a good sense of your experience. Any questions for me?\"\n" + + "\n" + + "Rules:\n" + + "- No protected characteristics (age, gender, race, religion)\n" + + "- Base on what they say\n" + + "- Talk like a real person, not a robot\n" + + "- One question at a time\n" + + "- If they give gold, acknowledge and move on\n" + + "\n" + + "Remember: Conversation, not interrogation. Be genuinely interested, react naturally, help them show their best self."); + list.add(mapEntity); + + //记录另外一个评分的提示词 + List> list1 = new LinkedList(); + Map mapEntity1 = new HashMap<>(); + mapEntity1.put("role", "system"); + mapEntity1.put("content", "You are a construction industry interview expert. Evaluate candidate responses and provide scores (1-5) and follow-up questions when needed. Always respond in JSON format."); + list1.add(mapEntity1); + + //不用预设问题了,直接通过大模型返回问题 + //1、先推送一个自我介绍 + Map mapEntityJs = new HashMap<>(); + mapEntityJs.put("role", "user"); + mapEntityJs.put("content", cacheResultText); + list.add(mapEntityJs); + + //初始化记录提示词数据到-缓存中 + cacheMsgMapData.put(session.getId(), JSONUtil.toJsonStr(list)); + cacheMsgMapData1.put(session.getId(), JSONUtil.toJsonStr(list1)); + + //2、推送大模型 + String promptJson = JSONUtil.toJsonStr(list); + log.info("AI提示词为:{}", promptJson); + log.info("开始请求AI:{}",System.currentTimeMillis()/1000); + //大模型问答流式输出 + //把提问的文字发送给CPT(流式处理) + chatGptStream(promptJson,session,clientId); + log.info("结束请求AI:{}",System.currentTimeMillis()/1000); + } catch (Exception e) { + e.printStackTrace(); + log.error("面试流程初始化失败:{}", e.getMessage()); + } + } + + /** + * 处理面试结束业务逻辑 + * + * @param session 客户端会话 + * @param position 职位 + */ + private void handleInterviewEnd(String clientId,Session session,String position) { + //暂时的业务逻辑 + //发送面试官结束语音流 + String openingPathUrl = RuoYiConfig.getProfile() + VOICE_SYSTEM_DIR + "end.wav"; + sendVoiceBuffer(openingPathUrl, session); + //返回文本评分 + //处理模型提问逻辑 + //获取缓存记录 + String msgMapData = cacheMsgMapData1.get(session.getId()); + String promptJson = ""; + if (StrUtil.isNotEmpty(msgMapData)) { + List list = JSONUtil.toList(msgMapData, Map.class); + //获取第一条数据记录 + Map mapEntity = list.get(0); + //更新问题记录 + mapEntity.put("role", "system"); + mapEntity.put("content", "You are a construction industry interview expert. Evaluate candidate responses and provide scores (1-5) and follow-up questions when needed. Always respond in JSON format."); + //每个回答的内容前面要加上候选人的职位 + if (StrUtil.isNotEmpty(position)) { + for (Map map : list) { + if ("user".equals(map.get("role").toString())) { + map.put("content", "Position: " + position + "\\n" + map.get("content")); + } + } + } + //未回答的时候,答案初始化 + for(Map entity : list){ + Object content = entity.get("content"); + if(ObjectUtil.isNotEmpty(content)){ + if(content.toString().contains("Candidate Answer:{}")){ + entity.put("content", StrUtil.format(content.toString(), "unanswered")); + } + } + } + promptJson = JSONUtil.toJsonStr(list); + //结束回答要清空问答数据 + cacheMsgMapData1.put(session.getId(), ""); + } + log.info("结束AI提示词为:{}", promptJson); + ChatGPTClient gptClient = SpringUtils.getBean(ChatGPTClient.class); + String resultMsg = gptClient.handleAiChat(promptJson, "PF"); + log.info("返回的结果为:{}",resultMsg); + //开始解析返回结果 + Map mapResultData = JSONUtil.toBean(resultMsg,Map.class); + //获取评分 + Object scoreStr = mapResultData.get("score"); + Object assessment = mapResultData.get("assessment"); + + Map resultEntity = new HashMap<>(); + resultEntity.put("content", scoreStr +"\n"+assessment); + resultEntity.put("type", "score"); + try{ + //返回最终的评分结构 + log.info("返回最终的评分结构:{}",JSONUtil.toJsonStr(resultEntity)); + session.getBasicRemote().sendText(JSONUtil.toJsonStr(resultEntity)); + }catch (Exception e){ + e.printStackTrace(); + } + } + + /** + * 处理评分记录 + * 触发规则: + * 1、获得 0-1 分 大于1次 立即结束面试 + * 2、获取 4-5 分 大于3次 立即结束面试 + * 3、获取 2-3 分 大于3次 立即结束面试 + * 4、获取 2-5 分 大于4次 立即结束面试 + * + * @param content + * @param session return false 立即结束面试 + */ + private Boolean handleScoreRecord(Object content, Session session) { + Map scoreRecordMap = cacheScoreResult.get(session.getId()); + log.info("获取评分结果:{}",content); + //对评分进行处理 + if (ObjectUtil.isNotEmpty(content)) { + String[] strs = content.toString().split("/"); + //取第一个数就是对应的评分 + log.info("获取的数据为:{}",strs[0]); + BigDecimal score = new BigDecimal(strs[0].trim()); + //记录Key为1 + if (BigDecimal.ZERO.compareTo(score) <= 0 && BigDecimal.ONE.compareTo(score) >= 0) { + Integer n1 = scoreRecordMap.get("0-1") + 1; + scoreRecordMap.put("0-1", n1); + if (n1 > 1) { + return false; + } + } + //记录Key为2 + if (new BigDecimal(4).compareTo(score) <= 0 && new BigDecimal(5).compareTo(score) >= 0) { + Integer n1 = scoreRecordMap.get("4-5") + 1; + scoreRecordMap.put("4-5", n1); + if (n1 > 3) { + return false; + } + } + //记录Key为3 + if (new BigDecimal(2).compareTo(score) <= 0 && new BigDecimal(3).compareTo(score) >= 0) { + Integer n1 = scoreRecordMap.get("2-3") + 1; + scoreRecordMap.put("2-3", n1); + if (n1 > 3) { + return false; + } + } + //记录Key为4 + if (new BigDecimal(2).compareTo(score) <= 0 && new BigDecimal(5).compareTo(score) >= 0) { + Integer n1 = scoreRecordMap.get("2-5") + 1; + scoreRecordMap.put("2-5", n1); + if (n1 > 4) { + return false; + } + } + } + return true; + } + + /** + * 校验是否结束面试,结束后直接返回评分 + * + * @param resultMsg 问答AI返回的结果数据 + * @param session 客户端会话 + */ + private Boolean getInterviewScore(String resultMsg, Session session) { + //返回文本评分 + //开始解析返回结果 + Map mapResultData = JSONUtil.toBean(resultMsg,Map.class); + //获取评分 + Object scoreStr = mapResultData.get("score"); + Object assessment = mapResultData.get("assessment"); + //校验面试是否结束 + Boolean flag = handleScoreRecord(scoreStr, session); + try { + if (!flag) { + //发送面试官结束语音流 + String openingPathUrl = RuoYiConfig.getProfile() + VOICE_SYSTEM_DIR + "end.wav"; + sendVoiceBuffer(openingPathUrl, session); + + Map resultEntity = new HashMap<>(); + resultEntity.put("content", scoreStr +"\n"+assessment); + resultEntity.put("type", "score"); + //返回评分结果 + log.info("返回最终的评分结果:{}",JSONUtil.toJsonStr(resultEntity)); + session.getBasicRemote().sendText(JSONUtil.toJsonStr(resultEntity)); + + } + } catch (Exception e) { + e.printStackTrace(); + } + return flag; + } + + /** + * 记录问题 + * @param questionResult + * @param session + */ + private void recordQuestion(String questionResult,Session session) { + if (StrUtil.isNotEmpty(questionResult)) { + //评分获取缓存记录 + String msgMapData1 = cacheMsgMapData1.get(session.getId()); + if (StrUtil.isNotEmpty(msgMapData1)) { + List list = JSONUtil.toList(msgMapData1, Map.class); + Map mapEntity = new HashMap<>(); + mapEntity.put("role", "user"); + mapEntity.put("content", "Question:" + questionResult + "\\nCandidate Answer:{}"); + list.add(mapEntity); + cacheMsgMapData1.put(session.getId(), JSONUtil.toJsonStr(list)); + } + //正常问题记录 + String msgMapData = cacheMsgMapData.get(session.getId()); + if (StrUtil.isNotEmpty(msgMapData)) { + List list = JSONUtil.toList(msgMapData, Map.class); + Map mapEntity = new HashMap<>(); + mapEntity.put("role", "assistant"); + mapEntity.put("content", questionResult); + list.add(mapEntity); + cacheMsgMapData.put(session.getId(), JSONUtil.toJsonStr(list)); + } + } + } + + /** + * 验证面试是否结束,不继续追问了 + * @param resultMsg + * @param session + * @return + */ + private Boolean checkInterviewIsEnd(String resultMsg, Session session){ + Map mapResultData = JSONUtil.toBean(resultMsg,Map.class); + //获取评分 + Object scoreStr = mapResultData.get("score"); + Object assessment = mapResultData.get("assessment"); + Object followUpNeeded = mapResultData.get("follow_up_needed"); + Boolean flag = Boolean.valueOf(followUpNeeded.toString()); + try { + //不继续追问了 + if (ObjectUtil.isNotEmpty(followUpNeeded) && !flag) { + //发送面试官结束语音流 + String openingPathUrl = RuoYiConfig.getProfile() + VOICE_SYSTEM_DIR + "end.wav"; + sendVoiceBuffer(openingPathUrl, session); + + Map resultEntity = new HashMap<>(); + resultEntity.put("content", scoreStr +"\n"+assessment); + resultEntity.put("type", "score"); + //返回评分结果 + log.info("返回最终的评分结果:{}",JSONUtil.toJsonStr(resultEntity)); + session.getBasicRemote().sendText(JSONUtil.toJsonStr(resultEntity)); + } + } catch (Exception e) { + e.printStackTrace(); + } + return flag; + } + + + /** + * 验证面试是否结束 + * @param session + * @return + */ + private Boolean checkIsEnd(Session session){ + Long replyNums = cacheQuestionNum.get(session.getId()); + //回答次数大于等于5就直接结束面试 + Boolean flag = true; + if(replyNums >= 5){ + //获取问答评分记录 + String promptJson = cacheMsgMapData1.get(session.getId()); + //根据模型获取评分 + ChatGPTClient chatGPTClient = SpringUtils.getBean(ChatGPTClient.class); + String resultMsg = chatGPTClient.handleAiChat(promptJson,"PF"); + if(StrUtil.isNotEmpty(resultMsg)) { + //直接返回问题了 + //开始解析返回结果 + Map mapResultData = JSONUtil.toBean(resultMsg, Map.class); + //获取评分 + Object scoreStr = mapResultData.get("score"); + Object assessment = mapResultData.get("assessment"); + //发送面试官结束语音流 + String openingPathUrl = RuoYiConfig.getProfile() + VOICE_SYSTEM_DIR + "end.wav"; + sendVoiceBuffer(openingPathUrl, session); + + Map resultEntity = new HashMap<>(); + resultEntity.put("content", scoreStr +"\n"+assessment); + resultEntity.put("type", "score"); + //返回评分结果 + try { + log.info("返回最终的评分结果:{}",JSONUtil.toJsonStr(resultEntity)); + session.getBasicRemote().sendText(JSONUtil.toJsonStr(resultEntity)); + }catch (Exception e) { + e.printStackTrace(); + } + } + flag = false; + }else{ + cacheQuestionNum.put(session.getId(), replyNums+1); + } + return flag; + } + + /** + * 大模型流式追问 + * @param promptJson + * @param session + * @param clientId + */ + private void chatGptStream(String promptJson,Session session,String clientId){ + //把提问的文字发送给CPT(流式处理) + OpenAiStreamClient aiStreamClient = SpringUtils.getBean(OpenAiStreamClient.class); + log.info("AI提示词为:{}",promptJson); + aiStreamClient.streamChat(promptJson, new OpenAiStreamListenerService() { + @Override + public void onMessage(String content) { + log.info("返回AI结果:{}", content); + if(StrUtil.isNotEmpty(content)){ + String questionResult = cacheQuestionResult.get(session.getId()); + if(StrUtil.isEmpty(questionResult)){ + questionResult = content; + }else{ + questionResult = questionResult + content; + } + cacheQuestionResult.put(session.getId(),questionResult); + sendTTSBuffer(clientId,content,session); + //上面语音发送完成了,开始发送问题文本啦 + // 实时输出内容 + try{ + try { + Thread.sleep(300); + }catch (Exception e){} + //把文本也给前端返回去 + Map dataText = new HashMap<>(); + dataText.put("type","question"); + dataText.put("content",content); + session.getBasicRemote().sendText(JSONUtil.toJsonStr(dataText)); + }catch (Exception e){ + e.printStackTrace(); + } + + } + } + + @Override + public void onComplete() { + try { + //开始往缓存中记录提问的问题 + String questionResult = cacheQuestionResult.get(session.getId()); + //开始对问题进行缓存 + recordQuestion(questionResult,session); + //清空问题 + cacheQuestionResult.put(session.getId(),""); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + @Override + public void onError(Throwable throwable) { + throwable.printStackTrace(); + } + }); + } + + + + /** + * 发送语音流给前端 + * + * @param session 客户端会话 + */ + private void sendConnectionVoice(Session session) { +// try { +// int resultNum = (int) (Math.random() * 5); +// String pathUrl = ""; +// String resultText = ""; +// if(resultNum == 0){ +// pathUrl = RuoYiConfig.getProfile() + VOICE_SYSTEM_DIR + "rightgot.wav"; +// resultText = "Right , got it"; +// }else if(resultNum == 1){ +// pathUrl = RuoYiConfig.getProfile() + VOICE_SYSTEM_DIR + "yeah.wav"; +// resultText = "Yeah , Good"; +// }else if(resultNum == 2){ +// pathUrl = RuoYiConfig.getProfile() + VOICE_SYSTEM_DIR + "gotit.wav"; +// resultText = "Got it, yeah"; +// }else if(resultNum == 3){ +// pathUrl = RuoYiConfig.getProfile() + VOICE_SYSTEM_DIR + "right.wav"; +// resultText = "Right , understood"; +// }else{ +// pathUrl = RuoYiConfig.getProfile() + VOICE_SYSTEM_DIR + "ok.wav"; +// resultText = "Yeah… ok…"; +// } +// sendVoiceBuffer(pathUrl,session); +// //发送衔接语文本 +// Map dataText = new HashMap<>(); +// dataText.put("type", "question"); +// dataText.put("content", resultText); +// session.getBasicRemote().sendText(JSONUtil.toJsonStr(dataText)); +// // 发送响应确认 +// log.info("已经成功发送了语音流给前端:{}", DateUtil.now()); +// } catch (Exception e) { +// e.printStackTrace(); +// } + } + + +} diff --git a/vetti-common/src/main/java/com/vetti/common/ai/elevenLabs/ElevenLabsStreamClient.java b/vetti-common/src/main/java/com/vetti/common/ai/elevenLabs/ElevenLabsStreamClient.java index 1ba4790..a4c13b4 100644 --- a/vetti-common/src/main/java/com/vetti/common/ai/elevenLabs/ElevenLabsStreamClient.java +++ b/vetti-common/src/main/java/com/vetti/common/ai/elevenLabs/ElevenLabsStreamClient.java @@ -19,10 +19,7 @@ import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Component; import javax.websocket.Session; -import java.io.ByteArrayInputStream; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.InputStream; +import java.io.*; import java.nio.ByteBuffer; import java.util.HashMap; import java.util.Map; @@ -51,8 +48,8 @@ public class ElevenLabsStreamClient { * @param voiceId 语音ID (可从ElevenLabs网站获取) * @throws IOException 网络请求或文件操作异常 */ - private void textToSpeech(String text, String voiceId, CloseableHttpClient httpClient, Session session) throws IOException { - HttpPost httpPost = new HttpPost(BASE_URL + "/text-to-speech/" + voiceId+"/stream?output_format=mp3_24000_48&optimize_streaming_latency=2"); + private void textToSpeech(String text, String voiceId, CloseableHttpClient httpClient, Session session,String outputFormat) throws IOException { + HttpPost httpPost = new HttpPost(BASE_URL + "/text-to-speech/" + voiceId+"/stream?output_format="+outputFormat); httpPost.setHeader("xi-api-key", apiKey); httpPost.setHeader("Content-Type", "application/json"); @@ -67,25 +64,48 @@ public class ElevenLabsStreamClient { HttpEntity responseEntity = response.getEntity(); if (responseEntity != null) { try (InputStream inputStream = responseEntity.getContent();) { -// byte[] allData = inputStream.readAllBytes(); -// InputStream stableStream = new ByteArrayInputStream(allData); -// sendAudioStream(session,stableStream); + //用来合并零散的碎片 + ByteArrayOutputStream smallChunkBuffer = new ByteArrayOutputStream(); // byte[] buffer = new byte[4096]; int bytesRead; + int n = 0; while ((bytesRead = inputStream.read(buffer)) != -1) { - ByteBuffer byteBuffer = ByteBuffer.wrap(buffer, 0, bytesRead); -// log.info("字符流的长度大小:{}", bytesRead); - if(bytesRead != 1 && bytesRead != 2){ - session.getAsyncRemote().sendBinary(byteBuffer); + //语音流合并到2KB左右进行发送 + if(smallChunkBuffer.size() >= 3072){ + log.info("语音流大于"+smallChunkBuffer.size()+"啦,发送完成!!!"); + byte[] merged = smallChunkBuffer.toByteArray(); + smallChunkBuffer.reset(); + session.getAsyncRemote().sendBinary(ByteBuffer.wrap(merged)); try { - Thread.sleep(20); + Thread.sleep(50); }catch (Exception e){} -// log.info("正常语音发送出去语音流啦!!!"); } + //发送三次告诉前端要合成一次语音 +// if(n == 2){ +// Map dataText = new HashMap<>(); +// dataText.put("type","voiceMiddleEnd"); +// dataText.put("content",""); +// session.getBasicRemote().sendText(JSONUtil.toJsonStr(dataText)); +// //重置一下 +// n = 0; +// } + // 零散的碎片 → 加入缓冲区,不立即发送 + smallChunkBuffer.write(buffer, 0, bytesRead); + n++; + } + //都加完缓冲区,最最后一次发送 + if(smallChunkBuffer.size() > 2){ + log.info("最后一次发送,语音流大于"+smallChunkBuffer.size()+"啦,发送完成!!!"); + byte[] merged = smallChunkBuffer.toByteArray(); + smallChunkBuffer.reset(); + session.getAsyncRemote().sendBinary(ByteBuffer.wrap(merged)); + try { + Thread.sleep(50); + }catch (Exception e){} } //返回结束点 try { - Thread.sleep(100); + Thread.sleep(50); }catch (Exception e){} Map dataText = new HashMap<>(); dataText.put("type","voiceEnd"); @@ -120,13 +140,13 @@ public class ElevenLabsStreamClient { * @param inputText * @return */ - public void handleTextToVoice(String inputText,Session session){ + public void handleTextToVoice(String inputText,Session session,String outputFormat){ CloseableHttpClient httpClient = HttpClients.createDefault(); try { // 使用第一个可用语音进行文本转语音(澳洲本地女声) // String firstVoiceId = "56bWURjYFHyYyVf490Dp"; String firstVoiceId = "56bWURjYFHyYyVf490Dp"; - textToSpeech(inputText, firstVoiceId,httpClient,session); + textToSpeech(inputText, firstVoiceId,httpClient,session,outputFormat); } catch (IOException e) { e.printStackTrace(); } finally { diff --git a/vetti-framework/src/main/java/com/vetti/framework/config/SecurityConfig.java b/vetti-framework/src/main/java/com/vetti/framework/config/SecurityConfig.java index 1dfae5c..0686670 100644 --- a/vetti-framework/src/main/java/com/vetti/framework/config/SecurityConfig.java +++ b/vetti-framework/src/main/java/com/vetti/framework/config/SecurityConfig.java @@ -113,7 +113,7 @@ public class SecurityConfig permitAllUrl.getUrls().forEach(url -> requests.antMatchers(url).permitAll()); // 对于登录login 注册register 验证码captchaImage 允许匿名访问 requests.antMatchers("/login", "/register", "/captchaImage","/aiCommon/**","/voice-websocket/multiple/**", - "/voice-websocket/**","/voice-websocket-opus/**","/verification/email/send","/verification/email/verify","/verification/phone/send", + "/voice-websocket/**","/voice-websocket/multiplePcm/**","/verification/email/send","/verification/email/verify","/verification/phone/send", "/forgotPassword").permitAll() // 静态资源,可匿名访问 .antMatchers(HttpMethod.GET, "/", "/*.html", "/**/*.html", "/**/*.css", "/**/*.js", "/profile/**").permitAll()