diff --git a/vetti-admin/src/main/java/com/vetti/socket/ChatWebSocketHandler.java b/vetti-admin/src/main/java/com/vetti/socket/ChatWebSocketHandler.java index 085e5a7..fd14ecc 100644 --- a/vetti-admin/src/main/java/com/vetti/socket/ChatWebSocketHandler.java +++ b/vetti-admin/src/main/java/com/vetti/socket/ChatWebSocketHandler.java @@ -7,8 +7,6 @@ import cn.hutool.core.util.StrUtil; import cn.hutool.json.JSONUtil; import com.vetti.common.ai.elevenLabs.ElevenLabsClient; import com.vetti.common.ai.gpt.ChatGPTClient; -import com.vetti.common.ai.gpt.OpenAiStreamClient; -import com.vetti.common.ai.gpt.service.OpenAiStreamListenerService; import com.vetti.common.config.RuoYiConfig; import com.vetti.common.utils.spring.SpringUtils; import com.vetti.hotake.domain.HotakeProblemBaseInfo; @@ -145,7 +143,6 @@ public class ChatWebSocketHandler { if (StrUtil.isEmpty(cacheResultText)) { cacheResultText = ""; } - //这是初次处理的逻辑 if ("YES".equals(startFlag)) { //初始化-不走大模型-直接对候选人进行提问 @@ -176,37 +173,43 @@ public class ChatWebSocketHandler { //把提问的文字发送给GPT ChatGPTClient chatGPTClient = SpringUtils.getBean(ChatGPTClient.class); log.info("AI提示词为:{}", promptJson); + log.info("开始请求AI:{}",System.currentTimeMillis()/1000); String resultMsg = chatGPTClient.handleAiChat(promptJson,"QA"); if(StrUtil.isNotEmpty(resultMsg)) { //开始解析返回结果 Map mapResultData = JSONUtil.toBean(resultMsg,Map.class); - //获取评分 - //验证是否触发对应的规则 - Boolean isEndFlag = getInterviewScore(resultMsg, session); - if(isEndFlag){ - log.info("面试回答符合条件规则,继续追问啦!!!!!"); - int resultNum = (int) (Math.random() * 2); - List questions = JSONUtil.toList(mapResultData.get("follow_up_questions").toString(), String.class); - String questionStr = questions.get(resultNum); - if (StrUtil.isNotEmpty(questionStr)) { - //开始进行语音输出-流式持续输出 - sendTTSBuffer(clientId, questionStr, session); - // 实时输出内容 - try { - //把文本也给前端返回去 - Map dataText = new HashMap<>(); - dataText.put("type", "question"); - dataText.put("content", questionStr); - log.info("提问的问题文本发送啦:{}",JSONUtil.toJsonStr(dataText)); - session.getBasicRemote().sendText(JSONUtil.toJsonStr(dataText)); - } catch (Exception e) { - e.printStackTrace(); + //验证是否有追问问题返回,如果没有问题返回直接返回评分停止面试 + Boolean isEndFlagFollow = checkInterviewIsEnd(resultMsg,session); + if(isEndFlagFollow){ + //获取评分 + //验证是否触发对应的规则 + Boolean isEndFlag = getInterviewScore(resultMsg, session); + if(isEndFlag){ + log.info("面试回答符合条件规则,继续追问啦!!!!!"); + int resultNum = (int) (Math.random() * 2); + List questions = JSONUtil.toList(mapResultData.get("follow_up_questions").toString(), String.class); + String questionStr = questions.get(resultNum); + if (StrUtil.isNotEmpty(questionStr)) { + //开始进行语音输出-流式持续输出 + sendTTSBuffer(clientId, questionStr, session); + // 实时输出内容 + try { + //把文本也给前端返回去 + Map dataText = new HashMap<>(); + dataText.put("type", "question"); + dataText.put("content", questionStr); + log.info("提问的问题文本发送啦:{}",JSONUtil.toJsonStr(dataText)); + session.getBasicRemote().sendText(JSONUtil.toJsonStr(dataText)); + } catch (Exception e) { + e.printStackTrace(); + } + //开始对问题进行缓存 + recordQuestion(questionStr,session); } - //开始对问题进行缓存 - recordQuestion(questionStr,session); } } } + log.info("结束请求AI:{}",System.currentTimeMillis()/1000); } } else if ("end".equals(resultFlag)) { log.info("面试结束啦!!!!!"); @@ -242,8 +245,16 @@ public class ChatWebSocketHandler { // 发生错误时调用 @OnError public void onError(Session session, Throwable throwable) { - System.err.println("WebSocket错误发生: " + throwable.getMessage()); - throwable.printStackTrace(); + System.err.println("WebSocket发生错误: 页面关闭,链接断开了"); + if(session != null) { + //是初次自我介绍后的问答环节 + cacheReplyFlag.put(session.getId(), ""); + //初始化面试回答数据记录 + cacheMsgMapData.put(session.getId(), ""); + //初始化面试问题 + cacheQuestionResult.put(session.getId(), ""); + cacheScoreResult.put(session.getId(), null); + } } /** @@ -528,5 +539,38 @@ public class ChatWebSocketHandler { } } + /** + * 验证面试是否结束,不继续追问了 + * @param resultMsg + * @param session + * @return + */ + private Boolean checkInterviewIsEnd(String resultMsg, Session session){ + Map mapResultData = JSONUtil.toBean(resultMsg,Map.class); + //获取评分 + Object scoreStr = mapResultData.get("score"); + Object assessment = mapResultData.get("assessment"); + Object followUpNeeded = mapResultData.get("follow_up_needed"); + Boolean flag = Boolean.valueOf(followUpNeeded.toString()); + try { + //不继续追问了 + if (ObjectUtil.isNotEmpty(followUpNeeded) && !flag) { + //发送面试官结束语音流 + String openingPathUrl = RuoYiConfig.getProfile() + VOICE_SYSTEM_DIR + "end.wav"; + sendVoiceBuffer(openingPathUrl, session); + + Map resultEntity = new HashMap<>(); + resultEntity.put("content", scoreStr +"\n"+assessment); + resultEntity.put("type", "score"); + //返回评分结果 + log.info("返回最终的评分结果:{}",JSONUtil.toJsonStr(resultEntity)); + session.getBasicRemote().sendText(JSONUtil.toJsonStr(resultEntity)); + } + } catch (Exception e) { + e.printStackTrace(); + } + return flag; + } + } diff --git a/vetti-admin/src/main/java/com/vetti/web/controller/ai/AiCommonController.java b/vetti-admin/src/main/java/com/vetti/web/controller/ai/AiCommonController.java index 7361050..7de79e3 100644 --- a/vetti-admin/src/main/java/com/vetti/web/controller/ai/AiCommonController.java +++ b/vetti-admin/src/main/java/com/vetti/web/controller/ai/AiCommonController.java @@ -53,7 +53,7 @@ public class AiCommonController extends BaseController //你好,我是本次的面试官Vetti,请点击开始按钮后,做一段自我介绍. //你好,我是本次的面试官Vetti,请在三秒后,开始做一段自我介绍. //本轮面试结束,谢谢您的配合,面试结果将稍后通知 - elevenLabsClient.handleTextToVoice("Ok, I have received your reply.","/Users/wangxiangshun/Desktop/临时文件/good.wav"); + elevenLabsClient.handleTextToVoice("Yeah","/Users/wangxiangshun/Desktop/临时文件/Yeah.wav"); return success(); } diff --git a/vetti-admin/src/main/resources/application-dev.yml b/vetti-admin/src/main/resources/application-dev.yml index 2c0cd20..45db8c0 100644 --- a/vetti-admin/src/main/resources/application-dev.yml +++ b/vetti-admin/src/main/resources/application-dev.yml @@ -156,7 +156,7 @@ whisper: apiKey: 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 language: en apiClientTokenUrl: https://api.openai.com/v1/realtime/sessions - prompt: This is a clear conversation in English. The user is speaking naturally through their device microphone. Please transcribe accurately with proper punctuation and natural speech patterns. + prompt: You are a translator. Detect when the user stops speaking. When a full sentence is complete, send recognized text as type:\'transcript\', and send its English translation as type:\'translation\'. # AI 聊天 diff --git a/vetti-admin/src/main/resources/application-druid.yml b/vetti-admin/src/main/resources/application-druid.yml index 2cfecce..5de3c89 100644 --- a/vetti-admin/src/main/resources/application-druid.yml +++ b/vetti-admin/src/main/resources/application-druid.yml @@ -163,7 +163,7 @@ whisper: apiKey: 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 language: en apiClientTokenUrl: https://api.openai.com/v1/realtime/sessions - prompt: This is a clear conversation in English. The user is speaking naturally through their device microphone. Please transcribe accurately with proper punctuation and natural speech patterns. + prompt: You are a translator. Detect when the user stops speaking. When a full sentence is complete, send recognized text as type:\'transcript\', and send its English translation as type:\'translation\'. # AI 聊天 chatGpt: diff --git a/vetti-admin/target/classes/application-druid.yml b/vetti-admin/target/classes/application-druid.yml index 2cfecce..5de3c89 100644 --- a/vetti-admin/target/classes/application-druid.yml +++ b/vetti-admin/target/classes/application-druid.yml @@ -163,7 +163,7 @@ whisper: apiKey: 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 language: en apiClientTokenUrl: https://api.openai.com/v1/realtime/sessions - prompt: This is a clear conversation in English. The user is speaking naturally through their device microphone. Please transcribe accurately with proper punctuation and natural speech patterns. + prompt: You are a translator. Detect when the user stops speaking. When a full sentence is complete, send recognized text as type:\'transcript\', and send its English translation as type:\'translation\'. # AI 聊天 chatGpt: