diff --git a/vetti-admin/src/main/java/com/vetti/socket/ChatWebSocketHandler.java b/vetti-admin/src/main/java/com/vetti/socket/ChatWebSocketHandler.java index 60df77c..c524de0 100644 --- a/vetti-admin/src/main/java/com/vetti/socket/ChatWebSocketHandler.java +++ b/vetti-admin/src/main/java/com/vetti/socket/ChatWebSocketHandler.java @@ -1,6 +1,5 @@ package com.vetti.socket; -import cn.hutool.core.collection.CollectionUtil; import cn.hutool.core.date.DateUtil; import cn.hutool.core.util.ObjectUtil; import cn.hutool.core.util.StrUtil; @@ -9,8 +8,6 @@ import com.vetti.common.ai.elevenLabs.ElevenLabsClient; import com.vetti.common.ai.gpt.ChatGPTClient; import com.vetti.common.config.RuoYiConfig; import com.vetti.common.utils.spring.SpringUtils; -import com.vetti.hotake.domain.HotakeProblemBaseInfo; -import com.vetti.hotake.service.IHotakeProblemBaseInfoService; import lombok.extern.slf4j.Slf4j; import org.apache.commons.io.FileUtils; import org.springframework.stereotype.Component; @@ -21,7 +18,10 @@ import javax.websocket.server.ServerEndpoint; import java.io.File; import java.math.BigDecimal; import java.nio.ByteBuffer; -import java.util.*; +import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; import java.util.concurrent.ConcurrentHashMap; @@ -148,8 +148,9 @@ public class ChatWebSocketHandler { } //这是初次处理的逻辑 if ("YES".equals(startFlag)) { + //自我介绍 //初始化-不走大模型-直接对候选人进行提问 - initializationQuestion(clientId, session); + initializationQuestion(clientId,cacheResultText ,session); //发送完第一次消息后,直接删除标记,开始进行正常的面试问答流程 cacheReplyFlag.put(session.getId(), ""); } else { @@ -164,6 +165,7 @@ public class ChatWebSocketHandler { Map mapEntity = new HashMap<>(); mapEntity.put("role", "user"); mapEntity.put("content", cacheResultText); + list.add(mapEntity); promptJson = JSONUtil.toJsonStr(list); cacheMsgMapData.put(session.getId(), promptJson); } @@ -183,8 +185,7 @@ public class ChatWebSocketHandler { Boolean isEndFlag = checkIsEnd(session); if(isEndFlag){ //开始返回衔接语 - String openingPathUrl = RuoYiConfig.getProfile() + VOICE_SYSTEM_DIR + "good.wav"; - sendVoiceBuffer(openingPathUrl, session); + sendConnectionVoice(session); //开始使用模型进行追问 //把提问的文字发送给GPT ChatGPTClient chatGPTClient = SpringUtils.getBean(ChatGPTClient.class); @@ -284,10 +285,8 @@ public class ChatWebSocketHandler { try { //文件转换成文件流 ByteBuffer outByteBuffer = convertFileToByteBuffer(pathUrl); -// sendInChunks(session, outByteBuffer, 2048); //发送文件流数据 session.getBasicRemote().sendBinary(outByteBuffer); - // 发送响应确认 log.info("已经成功发送了语音流给前端:{}", DateUtil.now()); } catch (Exception e) { @@ -318,47 +317,44 @@ public class ChatWebSocketHandler { * @param clientId 用户ID * @param session 客户端会话 */ - private void initializationQuestion(String clientId, Session session) { + private void initializationQuestion(String clientId,String cacheResultText,Session session) { try { log.info("开始获取到clientid :{}",clientId); //自我介绍结束后马上返回一个Good //发送初始化面试官语音流 - String openingPathUrl = RuoYiConfig.getProfile() + VOICE_SYSTEM_DIR + "good.wav"; - sendVoiceBuffer(openingPathUrl, session); + sendConnectionVoice(session); //初始化面试流程的提问 //先记录这个问题 List> list = new LinkedList(); Map mapEntity = new HashMap<>(); mapEntity.put("role", "system"); - mapEntity.put("content", "You are an expert HR interviewer and behavioural assessment analyst. You conduct structured, unbiased interviews while maintaining a natural, warm, and conversational speaking style. You are calm, friendly, and professional. You never express personal opinions or emotions. You rely strictly on the candidate's spoken words, evidence, and globally accepted HR competency standards.\n" + + mapEntity.put("content", "You are a senior HR interviewer conducting behavioral interviews. You're a \"listener and guide,\" not an \"examiner.\" Your style is warm, natural, and conversational—like chatting with a colleague.\n" + "\n" + - "Environment:\n" + - "You are running a live job interview on behalf of an employer. The candidate cannot see you. Your entire understanding comes from what they say aloud. You must guide them through the interview with clear questions, natural conversational pacing, and psychological safety. You operate under strict global HR compliance rules. All evaluations must be based only on job-relevant behaviours and never on assumptions.\n" + + "Core Rules:\n" + + "1. ALWAYS start with: \"Thank you,\" \"Cheers for that,\" or \"Thanks for sharing that\"\n" + + "2. Use Australian English: \"no worries,\" \"fair enough,\" \"mate,\" \"good on you,\" \"G'day\"\n" + + "3. Keep it casual—avoid jargon\n" + + "4. For detailed STAR answers: acknowledge warmly, then move to a NEW topic (don't over-probe)\n" + + "5. For stuck candidates: ease pressure with \"No rush, mate. Even a small example works\"\n" + + "6. For off-topic answers: acknowledge politely, then redirect gently back to the question\n" + "\n" + - "Tone:\n" + - "Your tone is warm, human, and conversational while remaining professional and precise. You sound like a real interviewer, not a script. Your phrasing is simple, clear, and spoken naturally. You acknowledge candidate responses with brief, human phrases such as \"Thank you,\" \"I understand,\" or \"That makes sense.\" You never overwhelm the candidate with long questions. You speak in Australian English.\n" + + "Opening Pattern:\n" + + "\"G'day! Cheers for coming in. No need to stress—just a casual chat about your experiences. We'll focus on [competencies]. Sound good?\"\n" + "\n" + - "Goal:\n" + - "Your primary goal is to conduct a structured, fair, and evidence-based behavioural interview while maintaining a natural conversational flow.\n" + - "\n" + - "Process:\n" + - "1. Begin by welcoming the candidate and briefly explaining the competencies you will be assessing\n" + - "2. Guide the candidate through behavioural questions one at a time\n" + - "3. Encourage them to share real examples using STAR structure (Situation, Task, Action, Result)\n" + - "4. Ask probing follow-ups ONLY when necessary to clarify their personal role, actions, decisions, or results\n" + - "5. Focus strictly on what they describe, never speculating or assuming missing details\n" + - "6. Maintain a psychologically safe and supportive conversational environment\n" + + "Response Patterns:\n" + + "- Brief answer → \"Cheers for that. Could you walk me through a specific time? What was the situation, what did you do, and how did it turn out?\"\n" + + "- Detailed answer → \"Thank you. Good on you for [action]. Fair enough. Now, let's chat about [new topic]\"\n" + + "- Stuck candidate → \"No worries, mate. Take your time. Even a small project works. Want me to rephrase that?\"\n" + + "- Off-topic answer → \"Thanks for sharing that. That's interesting, but let me bring us back to [original question]. Could you tell me about [specific aspect]?\"\n" + "\n" + "Guardrails:\n" + - "- Stay strictly within HR compliance and anti-discrimination standards\n" + - "- Be specific and concrete about the candidate's stated actions, experience, and results\n" + - "- Do NOT discuss or evaluate protected characteristics (age, gender, ethnicity, religion, disability, health, family status)\n" + - "- Do NOT guess or infer details that were not stated\n" + - "- Do NOT ask leading questions or hypotheticals unless explicitly required for the role\n" + - "- Do NOT provide legal, medical, financial, or immigration advice\n" + - "- When speaking to the candidate, do NOT use bullet points, lists, or prefixes\n" + - "- Respond in natural conversational English, NOT in JSON format"); + "- No discussion of protected characteristics\n" + + "- Base judgments on stated evidence only\n" + + "- Respond in natural English, NOT JSON\n" + + "- One question at a time\n" + + "- Keep candidates focused on the question asked"); list.add(mapEntity); + //记录另外一个评分的提示词 List> list1 = new LinkedList(); Map mapEntity1 = new HashMap<>(); @@ -366,53 +362,41 @@ public class ChatWebSocketHandler { mapEntity1.put("content", "You are a construction industry interview expert. Evaluate candidate responses and provide scores (1-5) and follow-up questions when needed. Always respond in JSON format."); list1.add(mapEntity1); + //不用预设问题了,直接通过大模型返回问题 + //1、先推送一个自我介绍 + Map mapEntityJs = new HashMap<>(); + mapEntityJs.put("role", "user"); + mapEntityJs.put("content", cacheResultText); + list.add(mapEntityJs); - //获取预设问题-直接TTS转换返回语音结果 - IHotakeProblemBaseInfoService problemBaseInfoService = SpringUtils.getBean(IHotakeProblemBaseInfoService.class); - HotakeProblemBaseInfo queryPro = new HotakeProblemBaseInfo(); - queryPro.setUserId(Long.valueOf(clientId)); - List baseInfoList = problemBaseInfoService.selectHotakeProblemBaseInfoList(queryPro); - log.info("准备进行第一个问题的提问:{}",JSONUtil.toJsonStr(baseInfoList)); - if (CollectionUtil.isNotEmpty(baseInfoList)) { - HotakeProblemBaseInfo baseInfo = baseInfoList.get(0); - if (StrUtil.isNotEmpty(baseInfo.getContents())) { - String[] qStrs = baseInfo.getContents().split("#AA#"); - int random_index = (int) (Math.random() * qStrs.length); - //获取问题文本 - String question = qStrs[random_index]; - //面试者提问问题了 - Map mapEntityQ = new HashMap<>(); - mapEntityQ.put("role", "assistant"); - mapEntityQ.put("content", question); - list.add(mapEntityQ); - - //开始记录评分问题 - Map mapEntityQ1 = new HashMap<>(); - mapEntityQ1.put("role", "user"); - mapEntityQ1.put("content", "Question：" + question + "\\nCandidate Answer：{}"); - list1.add(mapEntityQ1); - - - log.info("开始提问啦:{}",JSONUtil.toJsonStr(list)); - //直接对该问题进行转换处理返回语音流 - log.info("第一个问题为:{}",question); - sendTTSBuffer(clientId, question, session); - //发送问题文本 - try { - //把文本也给前端返回去 - Map dataText = new HashMap<>(); - dataText.put("type", "question"); - dataText.put("content", question); - log.info("提问的问题文本发送啦:{}",JSONUtil.toJsonStr(dataText)); - session.getBasicRemote().sendText(JSONUtil.toJsonStr(dataText)); - } catch (Exception e) { - e.printStackTrace(); - } - } - } //初始化记录提示词数据到-缓存中 cacheMsgMapData.put(session.getId(), JSONUtil.toJsonStr(list)); - cacheMsgMapData1.put(session.getId(), JSONUtil.toJsonStr(list)); + cacheMsgMapData1.put(session.getId(), JSONUtil.toJsonStr(list1)); + + //2、推送大模型 + String promptJson = JSONUtil.toJsonStr(list); + ChatGPTClient chatGPTClient = SpringUtils.getBean(ChatGPTClient.class); + log.info("AI提示词为:{}", promptJson); + log.info("开始请求AI:{}",System.currentTimeMillis()/1000); + String resultMsg = chatGPTClient.handleAiChat(promptJson,"QA"); + if(StrUtil.isNotEmpty(resultMsg)) { + //直接返回问题 + //开始进行语音输出-流式持续输出 + sendTTSBuffer(clientId, resultMsg, session); + // 实时输出内容 + try { + //把文本也给前端返回去 + Map dataText = new HashMap<>(); + dataText.put("type", "question"); + dataText.put("content", resultMsg); + log.info("提问的问题文本发送啦:{}",JSONUtil.toJsonStr(dataText)); + session.getBasicRemote().sendText(JSONUtil.toJsonStr(dataText)); + } catch (Exception e) { + e.printStackTrace(); + } + //开始对问题进行缓存 + recordQuestion(resultMsg,session); + } } catch (Exception e) { e.printStackTrace(); log.error("面试流程初始化失败:{}", e.getMessage()); @@ -430,11 +414,10 @@ public class ChatWebSocketHandler { //发送面试官结束语音流 String openingPathUrl = RuoYiConfig.getProfile() + VOICE_SYSTEM_DIR + "end.wav"; sendVoiceBuffer(openingPathUrl, session); - //返回文本评分 //处理模型提问逻辑 //获取缓存记录 - String msgMapData = cacheMsgMapData.get(session.getId()); + String msgMapData = cacheMsgMapData1.get(session.getId()); String promptJson = ""; if (StrUtil.isNotEmpty(msgMapData)) { List list = JSONUtil.toList(msgMapData, Map.class); @@ -451,14 +434,23 @@ public class ChatWebSocketHandler { } } } + //未回答的时候,答案初始化 + for(Map entity : list){ + Object content = entity.get("content"); + if(ObjectUtil.isNotEmpty(content)){ + if(content.toString().contains("Candidate Answer：{}")){ + entity.put("content", StrUtil.format(content.toString(), "unanswered")); + } + } + } promptJson = JSONUtil.toJsonStr(list); - //结束回答要清空问答数据 - cacheMsgMapData.put(session.getId(), ""); + cacheMsgMapData1.put(session.getId(), ""); } log.info("结束AI提示词为:{}", promptJson); ChatGPTClient gptClient = SpringUtils.getBean(ChatGPTClient.class); - String resultMsg = gptClient.handleAiChat(promptJson, "QA"); + String resultMsg = gptClient.handleAiChat(promptJson, "PF"); + log.info("返回的结果为:{}",resultMsg); //开始解析返回结果 Map mapResultData = JSONUtil.toBean(resultMsg,Map.class); //获取评分 @@ -675,6 +667,45 @@ public class ChatWebSocketHandler { return flag; } + /** + * 发送语音流给前端 + * + * @param session 客户端会话 + */ + private void sendConnectionVoice(Session session) { + try { + int resultNum = (int) (Math.random() * 5); + String pathUrl = ""; + String resultText = ""; + if(resultNum == 0){ + pathUrl = RuoYiConfig.getProfile() + VOICE_SYSTEM_DIR + "rightgot.wav"; + resultText = "Right , got it"; + }else if(resultNum == 1){ + pathUrl = RuoYiConfig.getProfile() + VOICE_SYSTEM_DIR + "yeah.wav"; + resultText = "Yeah , Good"; + }else if(resultNum == 2){ + pathUrl = RuoYiConfig.getProfile() + VOICE_SYSTEM_DIR + "gotit.wav"; + resultText = "Got it, yeah"; + }else if(resultNum == 3){ + pathUrl = RuoYiConfig.getProfile() + VOICE_SYSTEM_DIR + "right.wav"; + resultText = "Right , understood"; + }else{ + pathUrl = RuoYiConfig.getProfile() + VOICE_SYSTEM_DIR + "ok.wav"; + resultText = "Yeah… ok…"; + } + sendVoiceBuffer(pathUrl,session); + //发送衔接语文本 + Map dataText = new HashMap<>(); + dataText.put("type", "question"); + dataText.put("content", resultText); + session.getBasicRemote().sendText(JSONUtil.toJsonStr(dataText)); + // 发送响应确认 + log.info("已经成功发送了语音流给前端:{}", DateUtil.now()); + } catch (Exception e) { + e.printStackTrace(); + } + } + } diff --git a/vetti-admin/src/main/java/com/vetti/web/controller/ai/AiCommonController.java b/vetti-admin/src/main/java/com/vetti/web/controller/ai/AiCommonController.java index 7de79e3..3745d53 100644 --- a/vetti-admin/src/main/java/com/vetti/web/controller/ai/AiCommonController.java +++ b/vetti-admin/src/main/java/com/vetti/web/controller/ai/AiCommonController.java @@ -53,7 +53,14 @@ public class AiCommonController extends BaseController //你好,我是本次的面试官Vetti,请点击开始按钮后,做一段自我介绍. //你好,我是本次的面试官Vetti,请在三秒后,开始做一段自我介绍. //本轮面试结束,谢谢您的配合,面试结果将稍后通知 - elevenLabsClient.handleTextToVoice("Yeah","/Users/wangxiangshun/Desktop/临时文件/Yeah.wav"); + + elevenLabsClient.handleTextToVoice("We had a very pleasant chat today. We will get back to you as soon as possible. Also, you are welcome to keep your phone accessible.","/Users/wangxiangshun/Desktop/临时文件/end.wav"); +// +// elevenLabsClient.handleTextToVoice("Got it, yeah ","/Users/wangxiangshun/Desktop/临时文件/gotit.wav"); +// +// elevenLabsClient.handleTextToVoice("Gotcha ","/Users/wangxiangshun/Desktop/临时文件/gotcha.wav"); +// +// elevenLabsClient.handleTextToVoice("Right , got it ","/Users/wangxiangshun/Desktop/临时文件/rightgot.wav"); return success(); } diff --git a/vetti-admin/src/main/resources/application-druid.yml b/vetti-admin/src/main/resources/application-druid.yml index 1013820..aa620e5 100644 --- a/vetti-admin/src/main/resources/application-druid.yml +++ b/vetti-admin/src/main/resources/application-druid.yml @@ -152,7 +152,7 @@ verification: # 文本转语音 elevenLabs: baseUrl: https://api.elevenlabs.io/v1 - apiKey: sk_5240d8f56cb1eb5225fffcf903f62479884d1af5b3de6812 + apiKey: sk_dfe2b45e19bf8ad93a71d3a0faa61619a91e817df549d116 # apiKey: sk_88f5a560e1bbde0e5b8b6b6eb1812163a98bfb98554acbec modelId: eleven_turbo_v2_5 @@ -163,7 +163,7 @@ whisper: apiKey: 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 language: en apiClientTokenUrl: https://api.openai.com/v1/realtime/sessions - prompt: You are a translator. Detect when the user stops speaking. When a full sentence is complete, send recognized text as type:\'transcript\', and send its English translation as type:\'translation\'. + prompt: You are an English translator. Detect when the user stops speaking. When a full sentence is complete, send recognized text as type:\'transcript\', and send its English translation as type:\'translation\'. The final translation result should all be returned to English. # AI 聊天 chatGpt: diff --git a/vetti-admin/target/classes/application-druid.yml b/vetti-admin/target/classes/application-druid.yml index 05fa870..aa620e5 100644 --- a/vetti-admin/target/classes/application-druid.yml +++ b/vetti-admin/target/classes/application-druid.yml @@ -152,7 +152,7 @@ verification: # 文本转语音 elevenLabs: baseUrl: https://api.elevenlabs.io/v1 - apiKey: sk_5240d8f56cb1eb5225fffcf903f62479884d1af5b3de6812 + apiKey: sk_dfe2b45e19bf8ad93a71d3a0faa61619a91e817df549d116 # apiKey: sk_88f5a560e1bbde0e5b8b6b6eb1812163a98bfb98554acbec modelId: eleven_turbo_v2_5 @@ -163,13 +163,14 @@ whisper: apiKey: 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 language: en apiClientTokenUrl: https://api.openai.com/v1/realtime/sessions - prompt: You are a translator. Detect when the user stops speaking. When a full sentence is complete, send recognized text as type:\'transcript\', and send its English translation as type:\'translation\'. + prompt: You are an English translator. Detect when the user stops speaking. When a full sentence is complete, send recognized text as type:\'transcript\', and send its English translation as type:\'translation\'. The final translation result should all be returned to English. # AI 聊天 chatGpt: apiKey: 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 apiUrl: https://api.openai.com/v1/chat/completions model: ft:gpt-3.5-turbo-0125:vetti:interview-unified:CaGyCXOr + modelQuestion: gpt-4o-mini modelCV: ft:gpt-3.5-turbo-0125:vetti:vetti-resume-full:CYT0C8JG role: system diff --git a/vetti-common/src/main/java/com/vetti/common/ai/elevenLabs/ElevenLabsClient.java b/vetti-common/src/main/java/com/vetti/common/ai/elevenLabs/ElevenLabsClient.java index 2829c03..628f98a 100644 --- a/vetti-common/src/main/java/com/vetti/common/ai/elevenLabs/ElevenLabsClient.java +++ b/vetti-common/src/main/java/com/vetti/common/ai/elevenLabs/ElevenLabsClient.java @@ -101,7 +101,7 @@ public class ElevenLabsClient { CloseableHttpClient httpClient = HttpClients.createDefault(); try { // 使用第一个可用语音进行文本转语音(澳洲本地女声) - String firstVoiceId = "LwSYl3oLKw4IEbIEei6q"; + String firstVoiceId = "56bWURjYFHyYyVf490Dp"; textToSpeech(inputText, firstVoiceId, outputFile,httpClient); } catch (IOException e) { e.printStackTrace(); diff --git a/vetti-common/src/main/java/com/vetti/common/ai/elevenLabs/vo/VoiceSettings.java b/vetti-common/src/main/java/com/vetti/common/ai/elevenLabs/vo/VoiceSettings.java index 67ffb8e..db25199 100644 --- a/vetti-common/src/main/java/com/vetti/common/ai/elevenLabs/vo/VoiceSettings.java +++ b/vetti-common/src/main/java/com/vetti/common/ai/elevenLabs/vo/VoiceSettings.java @@ -19,6 +19,10 @@ public class VoiceSettings { private double clarity; private double speed; +// +// private double style; +// +// private Boolean use_speaker_boost; public VoiceSettings(double stability, double similarity_boost, double rate,double start_time,double clarity, double speed) { this.stability = stability; @@ -27,6 +31,8 @@ public class VoiceSettings { this.start_time = start_time; this.clarity = clarity; this.speed = speed; +// this.style = style; +// this.use_speaker_boost = use_speaker_boost; } // getter方法 @@ -53,4 +59,11 @@ public class VoiceSettings { return speed; } +// public double getStyle() { +// return style; +// } +// +// public Boolean getUse_speaker_boost() { +// return use_speaker_boost; +// } }