From 2fa0d46d483ad939bcd5a382ae14304b1d004a60 Mon Sep 17 00:00:00 2001 From: wangxiangshun Date: Mon, 10 Nov 2025 22:51:39 +0800 Subject: [PATCH] =?UTF-8?q?=E6=96=B0=E6=A8=A1=E5=9E=8B=E4=B8=9A=E5=8A=A1?= =?UTF-8?q?=E9=80=BB=E8=BE=91=E8=B0=83=E6=95=B4,=E4=BB=A5=E5=8F=8A?= =?UTF-8?q?=E8=A1=94=E6=8E=A5=E8=AF=AD=E6=9B=B4=E6=96=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../vetti/socket/ChatWebSocketHandler.java | 181 +++--- .../vetti/socket/ChatWebSocketHandler2.java | 571 ++++++++++++++++++ .../web/controller/ai/AiCommonController.java | 2 +- .../src/main/resources/application-druid.yml | 2 +- .../target/classes/application-druid.yml | 2 +- .../ai/elevenLabs/ElevenLabsClient.java | 2 - 6 files changed, 642 insertions(+), 118 deletions(-) create mode 100644 vetti-admin/src/main/java/com/vetti/socket/ChatWebSocketHandler2.java diff --git a/vetti-admin/src/main/java/com/vetti/socket/ChatWebSocketHandler.java b/vetti-admin/src/main/java/com/vetti/socket/ChatWebSocketHandler.java index 73bffd2..79ffdb1 100644 --- a/vetti-admin/src/main/java/com/vetti/socket/ChatWebSocketHandler.java +++ b/vetti-admin/src/main/java/com/vetti/socket/ChatWebSocketHandler.java @@ -2,6 +2,7 @@ package com.vetti.socket; import cn.hutool.core.collection.CollectionUtil; import cn.hutool.core.date.DateUtil; +import cn.hutool.core.util.ObjectUtil; import cn.hutool.core.util.StrUtil; import cn.hutool.json.JSONUtil; import com.vetti.common.ai.elevenLabs.ElevenLabsClient; @@ -167,77 +168,44 @@ public class ChatWebSocketHandler { promptJson = JSONUtil.toJsonStr(list); cacheMsgMapData.put(session.getId(), promptJson); } + //开始返回衔接语 + String openingPathUrl = RuoYiConfig.getProfile() + VOICE_SYSTEM_DIR + "good.wav"; + sendVoiceBuffer(openingPathUrl, session); + //开始使用模型进行追问 - //把提问的文字发送给CPT(流式处理) - OpenAiStreamClient aiStreamClient = SpringUtils.getBean(OpenAiStreamClient.class); + //把提问的文字发送给GPT + ChatGPTClient chatGPTClient = SpringUtils.getBean(ChatGPTClient.class); log.info("AI提示词为:{}", promptJson); - //先获取回答的评分,是否符合要求 - Boolean isEndFlag = getInterviewScore(clientId,promptJson, session, ""); - if(isEndFlag){ - log.info("面试回答符合条件规则,继续追问啦!!!!!"); - final int[] resultNum = {(int) (Math.random() * 2) + 1}; - aiStreamClient.streamChat(promptJson, new OpenAiStreamListenerService() { - @Override - public void onMessage(String content) { - log.info("返回AI结果：{}", content.replaceAll("\n", "")); - //获取1和2的随机数 - if(resultNum[0] == 1){ - content = ""; - } - resultNum[0] = resultNum[0] +1; - log.info("提问的问题:{}",content); -// String contentData = content.replaceAll("\n", ""); - //返回是追问的问题 - //获取的是追问的问题 - if (StrUtil.isNotEmpty(content)) { - //对问题进行数据缓存 - cacheQuestionResult.put(session.getId(), content); - //开始进行语音输出-流式持续输出 - sendTTSBuffer(clientId, content, session); - // 实时输出内容 - try { - //把文本也给前端返回去 - Map dataText = new HashMap<>(); - dataText.put("type", "question"); - dataText.put("content", content); - log.info("提问的问题文本发送啦:{}",JSONUtil.toJsonStr(dataText)); - session.getBasicRemote().sendText(JSONUtil.toJsonStr(dataText)); - } catch (Exception e) { - e.printStackTrace(); - } - } - } - - @Override - public void onComplete() { + String resultMsg = chatGPTClient.handleAiChat(promptJson,"QA"); + if(StrUtil.isNotEmpty(resultMsg)) { + //开始解析返回结果 + Map mapResultData = JSONUtil.toBean(resultMsg,Map.class); + //获取评分 + //验证是否触发对应的规则 + Boolean isEndFlag = getInterviewScore(resultMsg, session); + if(isEndFlag){ + log.info("面试回答符合条件规则,继续追问啦!!!!!"); + int resultNum = (int) (Math.random() * 2); + List questions = JSONUtil.toList(mapResultData.get("follow_up_questions").toString(), String.class); + String questionStr = questions.get(resultNum); + if (StrUtil.isNotEmpty(questionStr)) { + //开始进行语音输出-流式持续输出 + sendTTSBuffer(clientId, questionStr, session); + // 实时输出内容 try { - //开始往缓存中记录提问的问题 - String questionResult = cacheQuestionResult.get(session.getId()); - if (StrUtil.isNotEmpty(questionResult)) { - //获取缓存记录 - String msgMapData = cacheMsgMapData.get(session.getId()); - if (StrUtil.isNotEmpty(msgMapData)) { - List list = JSONUtil.toList(msgMapData, Map.class); - Map mapEntity = new HashMap<>(); - mapEntity.put("role", "user"); - mapEntity.put("content", "Question：" + questionResult + "\\nCandidate Answer：{}"); - list.add(mapEntity); - cacheMsgMapData.put(session.getId(), JSONUtil.toJsonStr(list)); - } - } - //清空问题 - cacheQuestionResult.put(session.getId(), ""); - + //把文本也给前端返回去 + Map dataText = new HashMap<>(); + dataText.put("type", "question"); + dataText.put("content", questionStr); + log.info("提问的问题文本发送啦:{}",JSONUtil.toJsonStr(dataText)); + session.getBasicRemote().sendText(JSONUtil.toJsonStr(dataText)); } catch (Exception e) { - throw new RuntimeException(e); + e.printStackTrace(); } + //开始对问题进行缓存 + recordQuestion(questionStr,session); } - - @Override - public void onError(Throwable throwable) { - throwable.printStackTrace(); - } - }); + } } } } else if ("end".equals(resultFlag)) { @@ -347,7 +315,7 @@ public class ChatWebSocketHandler { List> list = new LinkedList(); Map mapEntity = new HashMap<>(); mapEntity.put("role", "system"); - mapEntity.put("content", "You are an interviewer. Generate in-depth follow-up questions based on candidate responses."); + mapEntity.put("content", "You are a construction industry interview expert. Evaluate candidate responses and provide scores (1-5) and follow-up questions when needed. Always respond in JSON format."); list.add(mapEntity); //获取预设问题-直接TTS转换返回语音结果 IHotakeProblemBaseInfoService problemBaseInfoService = SpringUtils.getBean(IHotakeProblemBaseInfoService.class); @@ -358,7 +326,7 @@ public class ChatWebSocketHandler { if (CollectionUtil.isNotEmpty(baseInfoList)) { HotakeProblemBaseInfo baseInfo = baseInfoList.get(0); if (StrUtil.isNotEmpty(baseInfo.getContents())) { - String[] qStrs = baseInfo.getContents().split(","); + String[] qStrs = baseInfo.getContents().split("#AA#"); int random_index = (int) (Math.random() * qStrs.length); //获取问题文本 String question = qStrs[random_index]; @@ -435,13 +403,9 @@ public class ChatWebSocketHandler { resultEntity.put("content", resultMsg); resultEntity.put("type", "score"); try{ - //返回评分语音 -// sendTTSBuffer(clientId,resultMsg,session); - //返回最终的评分结构 log.info("返回最终的评分结构:{}",JSONUtil.toJsonStr(resultEntity)); session.getBasicRemote().sendText(JSONUtil.toJsonStr(resultEntity)); - }catch (Exception e){ e.printStackTrace(); } @@ -458,12 +422,12 @@ public class ChatWebSocketHandler { * @param content * @param session return false 立即结束面试 */ - private Boolean handleScoreRecord(String content, Session session) { + private Boolean handleScoreRecord(Object content, Session session) { Map scoreRecordMap = cacheScoreResult.get(session.getId()); log.info("获取评分结果:{}",content); //对评分进行处理 - if (StrUtil.isNotEmpty(content)) { - String[] strs = content.split("/"); + if (ObjectUtil.isNotEmpty(content)) { + String[] strs = content.toString().split("/"); //取第一个数就是对应的评分 log.info("获取的数据为:{}",strs[0]); BigDecimal score = new BigDecimal(strs[0].trim()); @@ -504,57 +468,28 @@ public class ChatWebSocketHandler { } /** - * 获取面试回答评分,并且校验是否结束面试 + * 校验是否结束面试,结束后直接返回评分 * - * @param promptJson 提示词数据json + * @param resultMsg 问答AI返回的结果数据 * @param session 客户端会话 - * @param position 职位 */ - private Boolean getInterviewScore(String clientId,String promptJson, Session session, String position) { + private Boolean getInterviewScore(String resultMsg, Session session) { //返回文本评分 - //获取缓存记录 - String msgMapData = cacheMsgMapData.get(session.getId()); - if (StrUtil.isNotEmpty(msgMapData)) { - List list = JSONUtil.toList(msgMapData, Map.class); - //获取第一条数据记录 - Map mapEntity = list.get(0); - //更新问题记录 - mapEntity.put("role", "system"); - mapEntity.put("content", "You are a construction industry interview expert. Rate candidate responses on a 1-5 scale and analyze key signals."); - //每个回答的内容前面要加上候选人的职位 - if (StrUtil.isNotEmpty(position)) { - for (Map map : list) { - if ("user".equals(map.get("role").toString())) { - map.put("content", "Position: " + position + "\\n" + map.get("content")); - } - } - } - promptJson = JSONUtil.toJsonStr(list); - } - log.info("评分AI提示词为:{}", promptJson); - ChatGPTClient gptClient = SpringUtils.getBean(ChatGPTClient.class); - String resultMsg = gptClient.handleAiChat(promptJson, "QA"); - //评论格式为: Score: 3/5\nAssessment: Basically correct answer but lacks detail - String resultScore = ""; - String scoreText = resultMsg; - if (StrUtil.isNotEmpty(resultMsg)) { - resultMsg = resultMsg.replaceAll("\n","#AA#"); - String[] resultMsgs = resultMsg.split("#AA#"); - resultScore = resultMsgs[0].replaceAll(SCORE_FLAG, ""); - } + //开始解析返回结果 + Map mapResultData = JSONUtil.toBean(resultMsg,Map.class); + //获取评分 + Object scoreStr = mapResultData.get("score"); + Object assessment = mapResultData.get("assessment"); //校验面试是否结束 - Boolean flag = handleScoreRecord(resultScore, session); + Boolean flag = handleScoreRecord(scoreStr, session); try { if (!flag) { //发送面试官结束语音流 String openingPathUrl = RuoYiConfig.getProfile() + VOICE_SYSTEM_DIR + "end.wav"; sendVoiceBuffer(openingPathUrl, session); - //返回评分语音 -// sendTTSBuffer(clientId,scoreText,session); - Map resultEntity = new HashMap<>(); - resultEntity.put("content", scoreText); + resultEntity.put("content", scoreStr +"\n"+assessment); resultEntity.put("type", "score"); //返回评分结果 log.info("返回最终的评分结果:{}",JSONUtil.toJsonStr(resultEntity)); @@ -567,5 +502,25 @@ public class ChatWebSocketHandler { return flag; } + /** + * 记录问题 + * @param questionResult + * @param session + */ + private void recordQuestion(String questionResult,Session session) { + if (StrUtil.isNotEmpty(questionResult)) { + //获取缓存记录 + String msgMapData = cacheMsgMapData.get(session.getId()); + if (StrUtil.isNotEmpty(msgMapData)) { + List list = JSONUtil.toList(msgMapData, Map.class); + Map mapEntity = new HashMap<>(); + mapEntity.put("role", "user"); + mapEntity.put("content", "Question：" + questionResult + "\\nCandidate Answer：{}"); + list.add(mapEntity); + cacheMsgMapData.put(session.getId(), JSONUtil.toJsonStr(list)); + } + } + } + } diff --git a/vetti-admin/src/main/java/com/vetti/socket/ChatWebSocketHandler2.java b/vetti-admin/src/main/java/com/vetti/socket/ChatWebSocketHandler2.java new file mode 100644 index 0000000..a3f99a0 --- /dev/null +++ b/vetti-admin/src/main/java/com/vetti/socket/ChatWebSocketHandler2.java @@ -0,0 +1,571 @@ +package com.vetti.socket; + +import cn.hutool.core.collection.CollectionUtil; +import cn.hutool.core.date.DateUtil; +import cn.hutool.core.util.StrUtil; +import cn.hutool.json.JSONUtil; +import com.vetti.common.ai.elevenLabs.ElevenLabsClient; +import com.vetti.common.ai.gpt.ChatGPTClient; +import com.vetti.common.ai.gpt.OpenAiStreamClient; +import com.vetti.common.ai.gpt.service.OpenAiStreamListenerService; +import com.vetti.common.config.RuoYiConfig; +import com.vetti.common.utils.spring.SpringUtils; +import com.vetti.hotake.domain.HotakeProblemBaseInfo; +import com.vetti.hotake.service.IHotakeProblemBaseInfoService; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.io.FileUtils; +import org.springframework.stereotype.Component; + +import javax.websocket.*; +import javax.websocket.server.PathParam; +import javax.websocket.server.ServerEndpoint; +import java.io.File; +import java.io.IOException; +import java.math.BigDecimal; +import java.nio.ByteBuffer; +import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +/** + * 语音面试 web处理器 + */ +@Slf4j +@ServerEndpoint("/voice-websocket222222/{clientId}") +@Component +public class ChatWebSocketHandler2 { + + /** + * 评分标记 + */ + private final String SCORE_FLAG = "Score:"; + + + /** + * 缓存客户端流式解析的语音文本数据 + */ + private final Map cacheClientTts = new ConcurrentHashMap<>(); + + /** + * 缓存客户端,标记是否是自我介绍后的初次问答 + */ + private final Map cacheReplyFlag = new ConcurrentHashMap<>(); + + /** + * 缓存客户端,面试回答信息 + */ + private final Map cacheMsgMapData = new ConcurrentHashMap<>(); + + /** + * 缓存客户端,AI提问的问题结果信息 + */ + private final Map cacheQuestionResult = new ConcurrentHashMap<>(); + + /** + * 缓存客户端,得分结果记录 + */ + private final Map> cacheScoreResult = new ConcurrentHashMap<>(); + + // 语音文件保存目录 + private static final String VOICE_STORAGE_DIR = "/voice_files/"; + + // 语音结果文件保存目录 + private static final String VOICE_STORAGE_RESULT_DIR = "/voice_result_files/"; + + // 系统语音目录 + private static final String VOICE_SYSTEM_DIR = "/system_files/"; + + public ChatWebSocketHandler2() { + // 初始化存储目录 + File dir = new File(RuoYiConfig.getProfile() + VOICE_STORAGE_DIR); + if (!dir.exists()) { + dir.mkdirs(); + } + + File resultDir = new File(RuoYiConfig.getProfile() + VOICE_STORAGE_RESULT_DIR); + if (!resultDir.exists()) { + resultDir.mkdirs(); + } + } + + // 连接建立时调用 + @OnOpen + public void onOpen(Session session, @PathParam("clientId") String clientId) { + log.info("WebSocket 链接已建立:{}", clientId); + log.info("WebSocket session 链接已建立:{}", session.getId()); + cacheClientTts.put(clientId, new String()); + //是初次自我介绍后的问答环节 + cacheReplyFlag.put(session.getId(), "YES"); + //初始化面试回答数据记录 + cacheMsgMapData.put(session.getId(), ""); + //初始化面试问题 + cacheQuestionResult.put(session.getId(), ""); + //初始化得分结果记录 + Map scoreResultData = new HashMap<>(); + scoreResultData.put("0-1", 0); + scoreResultData.put("4-5", 0); + scoreResultData.put("2-3", 0); + scoreResultData.put("2-5", 0); + cacheScoreResult.put(session.getId(), scoreResultData); + //发送初始化面试官语音流 + String openingPathUrl = RuoYiConfig.getProfile() + VOICE_SYSTEM_DIR + "opening.wav"; + sendVoiceBuffer(openingPathUrl, session); + } + + /** + * 接收文本消息 + * + * @param session 客户端会话 + * @param message 消息 + * 如: + * { + * "type": "start | done | end", + * "content": "内容" + * } + * @param clientId 用户ID + */ + @OnMessage + public void onTextMessage(Session session, String message, @PathParam("clientId") String clientId) { + log.info("我是接收文本消息:{}", message); + try { + //处理文本结果 + if (StrUtil.isNotEmpty(message)) { + Map mapResult = JSONUtil.toBean(JSONUtil.parseObj(message), Map.class); + String resultFlag = mapResult.get("type"); + if ("done".equals(resultFlag)) { + //开始合并语音流 + String startFlag = cacheReplyFlag.get(session.getId()); + //语音结束,开始进行回答解析 + log.info("开始文本处理,客户端ID为:{}", clientId); + String cacheResultText = mapResult.get("content"); + log.info("开始文本处理,面试者回答信息为:{}", cacheResultText); + if (StrUtil.isEmpty(cacheResultText)) { + cacheResultText = ""; + } + + //这是初次处理的逻辑 + if ("YES".equals(startFlag)) { + //初始化-不走大模型-直接对候选人进行提问 + initializationQuestion(clientId, session); + //发送完第一次消息后,直接删除标记,开始进行正常的面试问答流程 + cacheReplyFlag.put(session.getId(), ""); + } else { + //开始根据面试者回答的问题,进行追问回答 + //获取面试者回答信息 + //获取缓存记录 + String promptJson = ""; + String msgMapData = cacheMsgMapData.get(session.getId()); + if (StrUtil.isNotEmpty(msgMapData)) { + List list = JSONUtil.toList(msgMapData, Map.class); + //获取最后一条数据记录 + Map mapEntity = list.get(list.size() - 1); + //更新问题记录 + String content = mapEntity.get("content"); + mapEntity.put("content", StrUtil.format(content, cacheResultText)); + promptJson = JSONUtil.toJsonStr(list); + cacheMsgMapData.put(session.getId(), promptJson); + } + //开始使用模型进行追问 + //把提问的文字发送给CPT(流式处理) + OpenAiStreamClient aiStreamClient = SpringUtils.getBean(OpenAiStreamClient.class); + log.info("AI提示词为:{}", promptJson); + //先获取回答的评分,是否符合要求 + Boolean isEndFlag = getInterviewScore(clientId,promptJson, session, ""); + if(isEndFlag){ + log.info("面试回答符合条件规则,继续追问啦!!!!!"); + final int[] resultNum = {(int) (Math.random() * 2) + 1}; + aiStreamClient.streamChat(promptJson, new OpenAiStreamListenerService() { + @Override + public void onMessage(String content) { + log.info("返回AI结果：{}", content.replaceAll("\n", "")); + //获取1和2的随机数 + if(resultNum[0] == 1){ + content = ""; + } + resultNum[0] = resultNum[0] +1; + log.info("提问的问题:{}",content); +// String contentData = content.replaceAll("\n", ""); + //返回是追问的问题 + //获取的是追问的问题 + if (StrUtil.isNotEmpty(content)) { + //对问题进行数据缓存 + cacheQuestionResult.put(session.getId(), content); + //开始进行语音输出-流式持续输出 + sendTTSBuffer(clientId, content, session); + // 实时输出内容 + try { + //把文本也给前端返回去 + Map dataText = new HashMap<>(); + dataText.put("type", "question"); + dataText.put("content", content); + log.info("提问的问题文本发送啦:{}",JSONUtil.toJsonStr(dataText)); + session.getBasicRemote().sendText(JSONUtil.toJsonStr(dataText)); + } catch (Exception e) { + e.printStackTrace(); + } + } + } + + @Override + public void onComplete() { + try { + //开始往缓存中记录提问的问题 + String questionResult = cacheQuestionResult.get(session.getId()); + if (StrUtil.isNotEmpty(questionResult)) { + //获取缓存记录 + String msgMapData = cacheMsgMapData.get(session.getId()); + if (StrUtil.isNotEmpty(msgMapData)) { + List list = JSONUtil.toList(msgMapData, Map.class); + Map mapEntity = new HashMap<>(); + mapEntity.put("role", "user"); + mapEntity.put("content", "Question：" + questionResult + "\\nCandidate Answer：{}"); + list.add(mapEntity); + cacheMsgMapData.put(session.getId(), JSONUtil.toJsonStr(list)); + } + } + //清空问题 + cacheQuestionResult.put(session.getId(), ""); + + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + @Override + public void onError(Throwable throwable) { + throwable.printStackTrace(); + } + }); + } + } + } else if ("end".equals(resultFlag)) { + log.info("面试结束啦!!!!!"); + handleInterviewEnd(clientId,session,""); + } + } + } catch (Exception e) { + e.printStackTrace(); + } + } + + // 接收二进制消息（流数据） + @OnMessage + public void onBinaryMessage(Session session, @PathParam("clientId") String clientId, ByteBuffer byteBuffer) { + log.info("我是接受二进制流的-客户端ID为:{}", clientId); + } + + // 连接关闭时调用 + @OnClose + public void onClose(Session session, CloseReason reason) { + System.out.println("WebSocket连接已关闭: " + session.getId() + ", 原因: " + reason.getReasonPhrase()); + //链接关闭,清空内存 + //是初次自我介绍后的问答环节 + cacheReplyFlag.put(session.getId(), ""); + //初始化面试回答数据记录 + cacheMsgMapData.put(session.getId(), ""); + //初始化面试问题 + cacheQuestionResult.put(session.getId(), ""); + + cacheScoreResult.put(session.getId(), null); + } + + // 发生错误时调用 + @OnError + public void onError(Session session, Throwable throwable) { + System.err.println("WebSocket错误发生: " + throwable.getMessage()); + throwable.printStackTrace(); + } + + /** + * File 转换成 ByteBuffer + * + * @param fileUrl 文件路径 + * @return + */ + private ByteBuffer convertFileToByteBuffer(String fileUrl) { + File file = new File(fileUrl); + try { + return ByteBuffer.wrap(FileUtils.readFileToByteArray(file)); + } catch (Exception e) { + e.printStackTrace(); + } + return null; + } + + /** + * 发送语音流给前端 + * + * @param pathUrl 语音文件地址 + * @param session 客户端会话 + */ + private void sendVoiceBuffer(String pathUrl, Session session) { + try { + //文件转换成文件流 + ByteBuffer outByteBuffer = convertFileToByteBuffer(pathUrl); + //发送文件流数据 + session.getBasicRemote().sendBinary(outByteBuffer); + // 发送响应确认 + log.info("已经成功发送了语音流给前端:{}", DateUtil.now()); + } catch (IOException e) { + e.printStackTrace(); + } + } + + /** + * 发送文本转语音,发送语音流给前端 + * + * @param clientId 用户ID + * @param content 文本内容 + * @param session 客户端会话ID + */ + private void sendTTSBuffer(String clientId, String content, Session session) { + String resultFileName = clientId + "_" + System.currentTimeMillis() + ".wav"; + String resultPathUrl = RuoYiConfig.getProfile() + VOICE_STORAGE_RESULT_DIR + resultFileName; + ElevenLabsClient elevenLabsClient = SpringUtils.getBean(ElevenLabsClient.class); + elevenLabsClient.handleTextToVoice(content, resultPathUrl); + //持续返回数据流给客户端 + log.info("发送语音流成功啦!!!!!!!"); + sendVoiceBuffer(resultPathUrl, session); + } + + /** + * 对候选者初次进行提问业务逻辑处理(初始化系统随机获取第一个问题) + * + * @param clientId 用户ID + * @param session 客户端会话 + */ + private void initializationQuestion(String clientId, Session session) { + try { + log.info("开始获取到clientid :{}",clientId); + //自我介绍结束后马上返回一个Good + //发送初始化面试官语音流 + String openingPathUrl = RuoYiConfig.getProfile() + VOICE_SYSTEM_DIR + "good.wav"; + sendVoiceBuffer(openingPathUrl, session); + //初始化面试流程的提问 + List> list = new LinkedList(); + Map mapEntity = new HashMap<>(); + mapEntity.put("role", "system"); + mapEntity.put("content", "You are an interviewer. Generate in-depth follow-up questions based on candidate responses."); + list.add(mapEntity); + //获取预设问题-直接TTS转换返回语音结果 + IHotakeProblemBaseInfoService problemBaseInfoService = SpringUtils.getBean(IHotakeProblemBaseInfoService.class); + HotakeProblemBaseInfo queryPro = new HotakeProblemBaseInfo(); + queryPro.setUserId(Long.valueOf(clientId)); + List baseInfoList = problemBaseInfoService.selectHotakeProblemBaseInfoList(queryPro); + log.info("准备进行第一个问题的提问:{}",JSONUtil.toJsonStr(baseInfoList)); + if (CollectionUtil.isNotEmpty(baseInfoList)) { + HotakeProblemBaseInfo baseInfo = baseInfoList.get(0); + if (StrUtil.isNotEmpty(baseInfo.getContents())) { + String[] qStrs = baseInfo.getContents().split(","); + int random_index = (int) (Math.random() * qStrs.length); + //获取问题文本 + String question = qStrs[random_index]; + Map mapEntityQ = new HashMap<>(); + mapEntityQ.put("role", "user"); + mapEntityQ.put("content", "Question：" + question + "\\nCandidate Answer：{}"); + list.add(mapEntityQ); + log.info("开始提问啦:{}",JSONUtil.toJsonStr(list)); + //直接对该问题进行转换处理返回语音流 + log.info("第一个问题为:{}",question); + sendTTSBuffer(clientId, question, session); + //发送问题文本 + try { + //把文本也给前端返回去 + Map dataText = new HashMap<>(); + dataText.put("type", "question"); + dataText.put("content", question); + log.info("提问的问题文本发送啦:{}",JSONUtil.toJsonStr(dataText)); + session.getBasicRemote().sendText(JSONUtil.toJsonStr(dataText)); + } catch (Exception e) { + e.printStackTrace(); + } + } + } + //初始化记录提示词数据到-缓存中 + cacheMsgMapData.put(session.getId(), JSONUtil.toJsonStr(list)); + } catch (Exception e) { + e.printStackTrace(); + log.error("面试流程初始化失败:{}", e.getMessage()); + } + } + + /** + * 处理面试结束业务逻辑 + * + * @param session 客户端会话 + * @param position 职位 + */ + private void handleInterviewEnd(String clientId,Session session,String position) { + //暂时的业务逻辑 + //发送面试官结束语音流 + String openingPathUrl = RuoYiConfig.getProfile() + VOICE_SYSTEM_DIR + "end.wav"; + sendVoiceBuffer(openingPathUrl, session); + + //返回文本评分 + //处理模型提问逻辑 + //获取缓存记录 + String msgMapData = cacheMsgMapData.get(session.getId()); + String promptJson = ""; + if (StrUtil.isNotEmpty(msgMapData)) { + List list = JSONUtil.toList(msgMapData, Map.class); + //获取第一条数据记录 + Map mapEntity = list.get(0); + //更新问题记录 + mapEntity.put("role", "system"); + mapEntity.put("content", "You are a construction industry interview expert. Rate candidate responses on a 1-5 scale and analyze key signals."); + //每个回答的内容前面要加上候选人的职位 + if (StrUtil.isNotEmpty(position)) { + for (Map map : list) { + if ("user".equals(map.get("role").toString())) { + map.put("content", "Position: " + position + "\\n" + map.get("content")); + } + } + } + promptJson = JSONUtil.toJsonStr(list); + + //结束回答要清空问答数据 + cacheMsgMapData.put(session.getId(), ""); + } + log.info("结束AI提示词为:{}", promptJson); + ChatGPTClient gptClient = SpringUtils.getBean(ChatGPTClient.class); + String resultMsg = gptClient.handleAiChat(promptJson, "QA"); + Map resultEntity = new HashMap<>(); + resultEntity.put("content", resultMsg); + resultEntity.put("type", "score"); + try{ + //返回评分语音 +// sendTTSBuffer(clientId,resultMsg,session); + + //返回最终的评分结构 + log.info("返回最终的评分结构:{}",JSONUtil.toJsonStr(resultEntity)); + session.getBasicRemote().sendText(JSONUtil.toJsonStr(resultEntity)); + + }catch (Exception e){ + e.printStackTrace(); + } + } + + /** + * 处理评分记录 + * 触发规则: + * 1、获得 0-1 分 大于1次 立即结束面试 + * 2、获取 4-5 分 大于3次 立即结束面试 + * 3、获取 2-3 分 大于3次 立即结束面试 + * 4、获取 2-5 分 大于4次 立即结束面试 + * + * @param content + * @param session return false 立即结束面试 + */ + private Boolean handleScoreRecord(String content, Session session) { + Map scoreRecordMap = cacheScoreResult.get(session.getId()); + log.info("获取评分结果:{}",content); + //对评分进行处理 + if (StrUtil.isNotEmpty(content)) { + String[] strs = content.split("/"); + //取第一个数就是对应的评分 + log.info("获取的数据为:{}",strs[0]); + BigDecimal score = new BigDecimal(strs[0].trim()); + //记录Key为1 + if (BigDecimal.ZERO.compareTo(score) <= 0 && BigDecimal.ONE.compareTo(score) >= 0) { + Integer n1 = scoreRecordMap.get("0-1") + 1; + scoreRecordMap.put("0-1", n1); + if (n1 > 1) { + return false; + } + } + //记录Key为2 + if (new BigDecimal(4).compareTo(score) <= 0 && new BigDecimal(5).compareTo(score) >= 0) { + Integer n1 = scoreRecordMap.get("4-5") + 1; + scoreRecordMap.put("4-5", n1); + if (n1 > 3) { + return false; + } + } + //记录Key为3 + if (new BigDecimal(2).compareTo(score) <= 0 && new BigDecimal(3).compareTo(score) >= 0) { + Integer n1 = scoreRecordMap.get("2-3") + 1; + scoreRecordMap.put("2-3", n1); + if (n1 > 3) { + return false; + } + } + //记录Key为4 + if (new BigDecimal(2).compareTo(score) <= 0 && new BigDecimal(5).compareTo(score) >= 0) { + Integer n1 = scoreRecordMap.get("2-5") + 1; + scoreRecordMap.put("2-5", n1); + if (n1 > 4) { + return false; + } + } + } + return true; + } + + /** + * 获取面试回答评分,并且校验是否结束面试 + * + * @param promptJson 提示词数据json + * @param session 客户端会话 + * @param position 职位 + */ + private Boolean getInterviewScore(String clientId,String promptJson, Session session, String position) { + //返回文本评分 + //获取缓存记录 + String msgMapData = cacheMsgMapData.get(session.getId()); + if (StrUtil.isNotEmpty(msgMapData)) { + List list = JSONUtil.toList(msgMapData, Map.class); + //获取第一条数据记录 + Map mapEntity = list.get(0); + //更新问题记录 + mapEntity.put("role", "system"); + mapEntity.put("content", "You are a construction industry interview expert. Rate candidate responses on a 1-5 scale and analyze key signals."); + //每个回答的内容前面要加上候选人的职位 + if (StrUtil.isNotEmpty(position)) { + for (Map map : list) { + if ("user".equals(map.get("role").toString())) { + map.put("content", "Position: " + position + "\\n" + map.get("content")); + } + } + } + promptJson = JSONUtil.toJsonStr(list); + } + log.info("评分AI提示词为:{}", promptJson); + ChatGPTClient gptClient = SpringUtils.getBean(ChatGPTClient.class); + String resultMsg = gptClient.handleAiChat(promptJson, "QA"); + //评论格式为: Score: 3/5\nAssessment: Basically correct answer but lacks detail + String resultScore = ""; + String scoreText = resultMsg; + if (StrUtil.isNotEmpty(resultMsg)) { + resultMsg = resultMsg.replaceAll("\n","#AA#"); + String[] resultMsgs = resultMsg.split("#AA#"); + resultScore = resultMsgs[0].replaceAll(SCORE_FLAG, ""); + } + //校验面试是否结束 + Boolean flag = handleScoreRecord(resultScore, session); + try { + if (!flag) { + //发送面试官结束语音流 + String openingPathUrl = RuoYiConfig.getProfile() + VOICE_SYSTEM_DIR + "end.wav"; + sendVoiceBuffer(openingPathUrl, session); + + //返回评分语音 +// sendTTSBuffer(clientId,scoreText,session); + + Map resultEntity = new HashMap<>(); + resultEntity.put("content", scoreText); + resultEntity.put("type", "score"); + //返回评分结果 + log.info("返回最终的评分结果:{}",JSONUtil.toJsonStr(resultEntity)); + session.getBasicRemote().sendText(JSONUtil.toJsonStr(resultEntity)); + + } + } catch (Exception e) { + e.printStackTrace(); + } + return flag; + } + +} + diff --git a/vetti-admin/src/main/java/com/vetti/web/controller/ai/AiCommonController.java b/vetti-admin/src/main/java/com/vetti/web/controller/ai/AiCommonController.java index 1386cd4..7361050 100644 --- a/vetti-admin/src/main/java/com/vetti/web/controller/ai/AiCommonController.java +++ b/vetti-admin/src/main/java/com/vetti/web/controller/ai/AiCommonController.java @@ -53,7 +53,7 @@ public class AiCommonController extends BaseController //你好,我是本次的面试官Vetti,请点击开始按钮后,做一段自我介绍. //你好,我是本次的面试官Vetti,请在三秒后,开始做一段自我介绍. //本轮面试结束,谢谢您的配合,面试结果将稍后通知 - elevenLabsClient.handleTextToVoice("Hello, I am Vetti, the interviewer for this interview. Please begin a self introduction in three seconds","/Users/wangxiangshun/Desktop/临时文件/opening1.wav"); + elevenLabsClient.handleTextToVoice("Ok, I have received your reply.","/Users/wangxiangshun/Desktop/临时文件/good.wav"); return success(); } diff --git a/vetti-admin/src/main/resources/application-druid.yml b/vetti-admin/src/main/resources/application-druid.yml index 2a13167..2cfecce 100644 --- a/vetti-admin/src/main/resources/application-druid.yml +++ b/vetti-admin/src/main/resources/application-druid.yml @@ -169,7 +169,7 @@ whisper: chatGpt: apiKey: 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 apiUrl: https://api.openai.com/v1/chat/completions - model: ft:gpt-3.5-turbo-0125:vetti::CYl9OBMN + model: ft:gpt-3.5-turbo-0125:vetti:interview-unified:CaGyCXOr modelCV: ft:gpt-3.5-turbo-0125:vetti:vetti-resume-full:CYT0C8JG role: system diff --git a/vetti-admin/target/classes/application-druid.yml b/vetti-admin/target/classes/application-druid.yml index 2a13167..2cfecce 100644 --- a/vetti-admin/target/classes/application-druid.yml +++ b/vetti-admin/target/classes/application-druid.yml @@ -169,7 +169,7 @@ whisper: chatGpt: apiKey: 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 apiUrl: https://api.openai.com/v1/chat/completions - model: ft:gpt-3.5-turbo-0125:vetti::CYl9OBMN + model: ft:gpt-3.5-turbo-0125:vetti:interview-unified:CaGyCXOr modelCV: ft:gpt-3.5-turbo-0125:vetti:vetti-resume-full:CYT0C8JG role: system diff --git a/vetti-common/src/main/java/com/vetti/common/ai/elevenLabs/ElevenLabsClient.java b/vetti-common/src/main/java/com/vetti/common/ai/elevenLabs/ElevenLabsClient.java index 7a28941..4055240 100644 --- a/vetti-common/src/main/java/com/vetti/common/ai/elevenLabs/ElevenLabsClient.java +++ b/vetti-common/src/main/java/com/vetti/common/ai/elevenLabs/ElevenLabsClient.java @@ -57,13 +57,11 @@ public class ElevenLabsClient { Gson gson = new Gson(); StringEntity entity = new StringEntity(gson.toJson(payload), ContentType.APPLICATION_JSON); httpPost.setEntity(entity); - try (CloseableHttpResponse response = httpClient.execute(httpPost)) { HttpEntity responseEntity = response.getEntity(); if (responseEntity != null) { try (InputStream inputStream = responseEntity.getContent(); FileOutputStream outputStream = new FileOutputStream(outputFilePath)) { - byte[] buffer = new byte[4096]; int bytesRead; while ((bytesRead = inputStream.read(buffer)) != -1) {