diff --git a/vetti-admin/src/main/java/com/vetti/socket/ChatWebSocketHandler.java b/vetti-admin/src/main/java/com/vetti/socket/ChatWebSocketHandler.java index a0a52ca..306d34c 100644 --- a/vetti-admin/src/main/java/com/vetti/socket/ChatWebSocketHandler.java +++ b/vetti-admin/src/main/java/com/vetti/socket/ChatWebSocketHandler.java @@ -1,7 +1,8 @@ package com.vetti.socket; +import cn.hutool.core.collection.CollectionUtil; +import cn.hutool.core.date.DateUtil; import cn.hutool.core.util.StrUtil; -import cn.hutool.json.JSONObject; import cn.hutool.json.JSONUtil; import com.vetti.common.ai.elevenLabs.ElevenLabsClient; import com.vetti.common.ai.gpt.ChatGPTClient; @@ -9,20 +10,19 @@ import com.vetti.common.ai.gpt.OpenAiStreamClient; import com.vetti.common.ai.gpt.service.OpenAiStreamListenerService; import com.vetti.common.config.RuoYiConfig; import com.vetti.common.utils.spring.SpringUtils; +import com.vetti.hotake.domain.HotakeProblemBaseInfo; +import com.vetti.hotake.service.IHotakeProblemBaseInfoService; +import io.swagger.models.auth.In; import lombok.extern.slf4j.Slf4j; -import okhttp3.*; import org.apache.commons.io.FileUtils; import org.springframework.stereotype.Component; -import javax.sound.sampled.AudioFormat; -import javax.sound.sampled.AudioInputStream; -import javax.sound.sampled.AudioSystem; import javax.websocket.*; import javax.websocket.server.PathParam; import javax.websocket.server.ServerEndpoint; import java.io.File; -import java.io.FileOutputStream; import java.io.IOException; +import java.math.BigDecimal; import java.nio.ByteBuffer; import java.util.HashMap; import java.util.LinkedList; @@ -38,44 +38,41 @@ import java.util.concurrent.ConcurrentHashMap; @Component public class ChatWebSocketHandler { - // @Value("${whisper.apiUrl}") - private String API_URL = "wss://api.openai.com/v1/realtime?intent=transcription"; + /** + * 追问问题标记 + */ + private final String QUESTION_FLAG = "FOLLOW-UP:"; - // @Value("${whisper.model}") - private String MODEL = "gpt-4o-mini-transcribe"; + /** + * 评分标记 + */ + private final String SCORE_FLAG = "Score:"; - // @Value("${whisper.apiKey}") - private String apiKey = "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"; - - // @Value("${whisper.language}") - private String language = "en"; /** * 缓存客户端流式解析的语音文本数据 */ private final Map cacheClientTts = new ConcurrentHashMap<>(); - /** - * 缓存客户端调用OpenAi中的websocket-STT 流式传输数据 - */ - private final Map cacheWebSocket = new ConcurrentHashMap<>(); - /** * 缓存客户端,标记是否是自我介绍后的初次问答 */ - private final Map cacheReplyFlag = new ConcurrentHashMap<>(); + private final Map cacheReplyFlag = new ConcurrentHashMap<>(); /** * 缓存客户端,面试回答信息 */ - private final Map cacheMsgMapData = new ConcurrentHashMap<>(); + private final Map cacheMsgMapData = new ConcurrentHashMap<>(); /** * 缓存客户端,AI提问的问题结果信息 */ - private final Map cacheQuestionResult = new ConcurrentHashMap<>(); + private final Map cacheQuestionResult = new ConcurrentHashMap<>(); -// private final Map cacheOpeningResult = new ConcurrentHashMap<>(); + /** + * 缓存客户端,得分结果记录 + */ + private final Map> cacheScoreResult = new ConcurrentHashMap<>(); // 语音文件保存目录 private static final String VOICE_STORAGE_DIR = "/voice_files/"; @@ -105,184 +102,183 @@ public class ChatWebSocketHandler { log.info("WebSocket 链接已建立:{}", clientId); log.info("WebSocket session 链接已建立:{}", session.getId()); cacheClientTts.put(clientId, new String()); - //初始化STT流式语音转换文本的socket链接 -// createWhisperRealtimeSocket(session.getId()); //是初次自我介绍后的问答环节 - cacheReplyFlag.put(session.getId(),"YES"); + cacheReplyFlag.put(session.getId(), "YES"); //初始化面试回答数据记录 - cacheMsgMapData.put(session.getId(),""); + cacheMsgMapData.put(session.getId(), ""); //初始化面试问题 - cacheQuestionResult.put(session.getId(),""); - //开场白控制 -// String flag = cacheOpeningResult.get(clientId); -// if(StrUtil.isEmpty(flag)){ - //发送初始化面试官语音流 - String openingPathUrl = RuoYiConfig.getProfile() + VOICE_SYSTEM_DIR + "opening.wav"; - try { - //文件转换成文件流 - ByteBuffer outByteBuffer = convertFileToByteBuffer(openingPathUrl); - //发送文件流数据 - session.getBasicRemote().sendBinary(outByteBuffer); -// cacheOpeningResult.put(clientId,"YES"); - // 发送响应确认 - log.info("初始化返回面试官语音信息:{}", System.currentTimeMillis() / 1000); - } catch (IOException e) { - e.printStackTrace(); - } -// } + cacheQuestionResult.put(session.getId(), ""); + //初始化得分结果记录 + Map scoreResultData = new HashMap<>(); + scoreResultData.put("0-1", 0); + scoreResultData.put("4-5", 0); + scoreResultData.put("2-3", 0); + scoreResultData.put("2-5", 0); + cacheScoreResult.put(session.getId(), scoreResultData); + //发送初始化面试官语音流 + String openingPathUrl = RuoYiConfig.getProfile() + VOICE_SYSTEM_DIR + "opening.wav"; + sendVoiceBuffer(openingPathUrl, session); } - // 接收文本消息 + /** + * 接收文本消息 + * + * @param session 客户端会话 + * @param message 消息 + * 如: + * { + * "type": "start | done | end", + * "content": "内容" + * } + * @param clientId 用户ID + */ @OnMessage public void onTextMessage(Session session, String message, @PathParam("clientId") String clientId) { - System.out.println("接收到文本消息: " + message); + log.info("我是接收文本消息:{}", message); try { -// { -// "type": "start | done | end", -// "content": "内容" -// } //处理文本结果 if (StrUtil.isNotEmpty(message)) { Map mapResult = JSONUtil.toBean(JSONUtil.parseObj(message), Map.class); String resultFlag = mapResult.get("type"); if ("done".equals(resultFlag)) { //开始合并语音流 - //发送消息 -// WebSocket webSocket = cacheWebSocket.get(session.getId()); -// if (webSocket != null) { -// webSocket.send("{\"type\": \"input_audio_buffer.commit\"}"); -// webSocket.send("{\"type\": \"response.create\"}"); -// } String startFlag = cacheReplyFlag.get(session.getId()); //语音结束,开始进行回答解析 - log.info("开始文本处理,客户端ID为:{}",clientId); -// String cacheResultText = cacheClientTts.get(session.getId()); + log.info("开始文本处理,客户端ID为:{}", clientId); String cacheResultText = mapResult.get("content"); log.info("开始文本处理,面试者回答信息为:{}", cacheResultText); if (StrUtil.isEmpty(cacheResultText)) { - cacheResultText = "I first check the forklift's logbook for recent issues, inspect tires and brakes, verify the load capacity matches today's task, and confirm my licence is current—all per SWMS requirements."; + cacheResultText = ""; } - String promptJson = ""; - if("YES".equals(startFlag)) { - //自我介绍结束后马上返回一个Good - //发送初始化面试官语音流 - String openingPathUrl = RuoYiConfig.getProfile() + VOICE_SYSTEM_DIR + "good.wav"; - try { - //文件转换成文件流 - ByteBuffer outByteBuffer = convertFileToByteBuffer(openingPathUrl); - //发送文件流数据 - session.getBasicRemote().sendBinary(outByteBuffer); - // 发送响应确认 - log.info("初始化返回面试官语音信息:{}", System.currentTimeMillis() / 1000); - } catch (IOException e) { - e.printStackTrace(); - } - Map mapEntity = new HashMap<>(); - mapEntity.put("role","system"); - mapEntity.put("content","You are an interviewer. Generate follow-up questions based on Construction Labourer candidate responses.Only return one question and do not repeat the previously returned questions \\n MPORTANT: Do not ask the same question again if the answer is incorrect"); - List> list = new LinkedList(); - list.add(mapEntity); - promptJson = JSONUtil.toJsonStr(list); - //记录缓存中 - cacheMsgMapData.put(session.getId(),promptJson); - }else{ + + //这是初次处理的逻辑 + if ("YES".equals(startFlag)) { + //初始化-不走大模型-直接对候选人进行提问 + initializationQuestion(clientId, session); + //发送完第一次消息后,直接删除标记,开始进行正常的面试问答流程 + cacheReplyFlag.put(session.getId(), ""); + } else { //开始根据面试者回答的问题,进行追问回答 -// { -// role: "system", -// content: "你是面试官，根据Construction Labourer候选人回答生成追问。" -// }, -// { -// role: "user", -// content: `问题：${question}\n候选人回答：${answer}` -// } //获取面试者回答信息 //获取缓存记录 + String promptJson = ""; String msgMapData = cacheMsgMapData.get(session.getId()); - if(StrUtil.isNotEmpty(msgMapData)){ + if (StrUtil.isNotEmpty(msgMapData)) { List list = JSONUtil.toList(msgMapData, Map.class); //获取最后一条数据记录 - Map mapEntity = list.get(list.size()-1); + Map mapEntity = list.get(list.size() - 1); //更新问题记录 String content = mapEntity.get("content"); mapEntity.put("content", StrUtil.format(content, cacheResultText)); promptJson = JSONUtil.toJsonStr(list); - cacheMsgMapData.put(session.getId(),promptJson); + cacheMsgMapData.put(session.getId(), promptJson); } - } - //获取完问答数据,直接清空缓存数据 - cacheClientTts.put(session.getId(),""); - cacheReplyFlag.put(session.getId(),""); - //把提问的文字发送给CPT(流式处理) - OpenAiStreamClient aiStreamClient = SpringUtils.getBean(OpenAiStreamClient.class); - log.info("AI提示词为:{}",promptJson); - aiStreamClient.streamChat(promptJson, new OpenAiStreamListenerService() { - @Override - public void onMessage(String content) { - log.info("返回AI结果：{}", content); - if(StrUtil.isNotEmpty(content)){ - String questionResult = cacheQuestionResult.get(session.getId()); - if(StrUtil.isEmpty(questionResult)){ - questionResult = content; - }else{ - questionResult = questionResult + content; + //开始使用模型进行追问 + //把提问的文字发送给CPT(流式处理) + OpenAiStreamClient aiStreamClient = SpringUtils.getBean(OpenAiStreamClient.class); + log.info("AI提示词为:{}", promptJson); + //Score: 返回的是评分,后面只会跟着一个 + //FOLLOW-UP: 返回的是问题,后面的每一行都是问题 + aiStreamClient.streamChat(promptJson, new OpenAiStreamListenerService() { + String isScore = "0"; + //是否结束面试 + Boolean flag = true; + String resultText = ""; + String resultEvaluate = ""; + //是否遇到问题记录 + Boolean isFlow = false; + @Override + public void onMessage(String content) { + log.info("返回AI结果：{}", content.replaceAll("\n", "")); + if (StrUtil.isEmpty(resultText)) { + resultText = content; + } else { + resultText = resultText + content; } - cacheQuestionResult.put(session.getId(),questionResult); - // 实时输出内容 - //开始进行语音输出-流式持续输出 - //把结果文字转成语音文件 - //生成文件 - //生成唯一文件名 - String resultFileName = clientId + "_" + System.currentTimeMillis() + ".wav"; - String resultPathUrl = RuoYiConfig.getProfile() + VOICE_STORAGE_RESULT_DIR + resultFileName; - ElevenLabsClient elevenLabsClient = SpringUtils.getBean(ElevenLabsClient.class); - elevenLabsClient.handleTextToVoice(content, resultPathUrl); - //持续返回数据流给客户端 + String contentData = content.replaceAll("\n", ""); + //记录获取的分数,并且验证分数是否完成 + //获取评分 + if (contentData.contains(SCORE_FLAG)) { + isScore = "1"; + } + if ("1".equals(isScore)) { + //获取的是评分,并且记录评分 + flag = handleScoreRecord(content, session); + } + if (contentData.contains(QUESTION_FLAG)) { + isFlow = true; + } + if (flag) { + //返回是追问的问题 + if (contentData.contains(QUESTION_FLAG)) { + //获取的是追问的问题 + contentData = contentData.replace(QUESTION_FLAG, ""); + if (StrUtil.isNotEmpty(contentData)) { + //对问题进行数据缓存 + cacheQuestionResult.put(session.getId(), contentData); + //开始进行语音输出-流式持续输出 + sendTTSBuffer(clientId, contentData, session); + // 实时输出内容 + try { + //把文本也给前端返回去 + Map dataText = new HashMap<>(); + dataText.put("type", "question"); + dataText.put("content", contentData); + session.getBasicRemote().sendText(JSONUtil.toJsonStr(dataText)); + } catch (Exception e) { + e.printStackTrace(); + } + + } + } + if(!isFlow){ + if (StrUtil.isEmpty(resultEvaluate)) { + resultEvaluate = content; + } else { + resultEvaluate = resultEvaluate + content; + } + } + } + } + + @Override + public void onComplete() { try { - //文件转换成文件流 - ByteBuffer outByteBuffer = convertFileToByteBuffer(resultPathUrl); - //发送文件流数据 - session.getBasicRemote().sendBinary(outByteBuffer); - // 发送响应确认 - } catch (IOException e) { - e.printStackTrace(); + //开始往缓存中记录提问的问题 + String questionResult = cacheQuestionResult.get(session.getId()); + if(StrUtil.isNotEmpty(questionResult)){ + //获取缓存记录 + String msgMapData = cacheMsgMapData.get(session.getId()); + if (StrUtil.isNotEmpty(msgMapData)) { + List list = JSONUtil.toList(msgMapData, Map.class); + Map mapEntity = new HashMap<>(); + mapEntity.put("role", "user"); + mapEntity.put("content", "Question：" + questionResult + "\\nCandidate Answer：{}"); + list.add(mapEntity); + cacheMsgMapData.put(session.getId(), JSONUtil.toJsonStr(list)); + } + } + //清空问题 + cacheQuestionResult.put(session.getId(), ""); + if(!flag || !isFlow){ + //理解结束面试 + //发送面试结束的通知已经评分 + handleInterviewEnd(session, resultEvaluate); + } + } catch (Exception e) { + throw new RuntimeException(e); } } - } - @Override - public void onComplete() { - try { - //开始往缓存中记录提问的问题 - String questionResult = cacheQuestionResult.get(session.getId()); - //获取缓存记录 - String msgMapData = cacheMsgMapData.get(session.getId()); - if(StrUtil.isNotEmpty(msgMapData)){ - List list = JSONUtil.toList(msgMapData, Map.class); - Map mapEntity = new HashMap<>(); - mapEntity.put("role","user"); - mapEntity.put("content","Question："+questionResult+"\\nCandidate Answer：{}"); - list.add(mapEntity); - cacheMsgMapData.put(session.getId(),JSONUtil.toJsonStr(list)); - } - //清空问题 - cacheQuestionResult.put(session.getId(),""); - - Map resultEntity = new HashMap<>(); - resultEntity.put("msg", "done"); - //发送通知告诉客户端已经回答结束了 - session.getBasicRemote().sendText(JSONUtil.toJsonStr(resultEntity)); - } catch (Exception e) { - throw new RuntimeException(e); + @Override + public void onError(Throwable throwable) { + throwable.printStackTrace(); } - } + }); + } + } else if ("end".equals(resultFlag)) { + //暂时的业务逻辑 - @Override - public void onError(Throwable throwable) { - throwable.printStackTrace(); - } - }); - }else if("end".equals(resultFlag)){ -// cacheOpeningResult.put(clientId,""); //发送面试官结束语音流 String openingPathUrl = RuoYiConfig.getProfile() + VOICE_SYSTEM_DIR + "end.wav"; try { @@ -300,23 +296,23 @@ public class ChatWebSocketHandler { String promptJson = ""; //获取缓存记录 String msgMapData = cacheMsgMapData.get(session.getId()); - if(StrUtil.isNotEmpty(msgMapData)){ + if (StrUtil.isNotEmpty(msgMapData)) { List list = JSONUtil.toList(msgMapData, Map.class); //获取最后一条数据记录 - Map mapEntity = list.get(0); + Map mapEntity = list.get(0); //更新问题记录 - mapEntity.put("role","system"); - mapEntity.put("content","You are a construction industry interview expert. Rate Construction Labourer candidate responses on a 1-5 scale. IMPORTANT: If the answer is completely unrelated, contains technical errors, system messages, or is nonsensical, give it a score of 0/5 and explain why it's invalid."); + mapEntity.put("role", "system"); + mapEntity.put("content", "You are a construction industry interview expert. Rate Construction Labourer candidate responses on a 1-5 scale. IMPORTANT: If the answer is completely unrelated, contains technical errors, system messages, or is nonsensical, give it a score of 0/5 and explain why it's invalid."); promptJson = JSONUtil.toJsonStr(list); //结束回答要清空问答数据 - cacheMsgMapData.put(session.getId(),""); + cacheMsgMapData.put(session.getId(), ""); } - log.info("结束AI提示词为:{}",promptJson); + log.info("结束AI提示词为:{}", promptJson); ChatGPTClient gptClient = SpringUtils.getBean(ChatGPTClient.class); - String resultMsg = gptClient.handleAiChat(promptJson,"QA"); + String resultMsg = gptClient.handleAiChat(promptJson, "QA"); Map resultEntity = new HashMap<>(); resultEntity.put("msg", resultMsg); - resultEntity.put("dataType","score"); + resultEntity.put("dataType", "score"); session.getBasicRemote().sendText(JSONUtil.toJsonStr(resultEntity)); } } @@ -328,54 +324,20 @@ public class ChatWebSocketHandler { // 接收二进制消息（流数据） @OnMessage public void onBinaryMessage(Session session, @PathParam("clientId") String clientId, ByteBuffer byteBuffer) { -// log.info("客户端ID为:{}", clientId); -// // 处理二进制流数据 -// byte[] bytes = new byte[byteBuffer.remaining()]; -// //从缓冲区中读取数据并存储到指定的字节数组中 -// byteBuffer.get(bytes); -// // 生成唯一文件名 -// String fileName = clientId + "_" + System.currentTimeMillis() + ".wav"; -// String pathUrl = RuoYiConfig.getProfile() + VOICE_STORAGE_DIR + fileName; -// log.info("文件路径为:{}", pathUrl); -// try { -// saveAsWebM(bytes, pathUrl); -// //接收到数据流后直接就进行SST处理 -// //语音格式转换 -// String fileOutName = clientId + "_" + System.currentTimeMillis() + ".pcm"; -// String pathOutUrl = RuoYiConfig.getProfile() + VOICE_STORAGE_DIR + fileOutName; -// handleAudioToPCM(pathUrl, pathOutUrl); -// //发送消息 -// WebSocket webSocket = cacheWebSocket.get(session.getId()); -// log.info("获取的socket对象为:{}", webSocket); -// if (webSocket != null) { -//// 1. 启动音频缓冲 -//// webSocket.send("{\"type\": \"input_audio_buffer.start\"}"); -// File outputFile = new File(pathOutUrl); // 输出PCM格式文件 -// ByteBuffer buffer = ByteBuffer.wrap(FileUtils.readFileToByteArray(outputFile)); -// byte[] outBytes = new byte[buffer.remaining()]; -// //从缓冲区中读取数据并存储到指定的字节数组中 -// buffer.get(outBytes); -// String base64Audio = Base64.getEncoder().encodeToString(outBytes); -// String message = "{ \"type\": \"input_audio_buffer.append\", \"audio\": \"" + base64Audio + "\" }"; -// webSocket.send(message); -// // 3. 提交音频并请求转录 -//// webSocket.send("{\"type\": \"input_audio_buffer.commit\"}"); -//// webSocket.send("{\"type\": \"response.create\"}"); -// } -// } catch (Exception e) { -// e.printStackTrace(); -// } - + log.info("我是接受二进制流的-客户端ID为:{}", clientId); } // 连接关闭时调用 @OnClose public void onClose(Session session, CloseReason reason) { System.out.println("WebSocket连接已关闭: " + session.getId() + ", 原因: " + reason.getReasonPhrase()); -// WebSocket webSocket = cacheWebSocket.get(session.getId()); -// if (webSocket != null) { -// webSocket.close(1000, null); -// } + //链接关闭,清空内存 + //是初次自我介绍后的问答环节 + cacheReplyFlag.put(session.getId(), ""); + //初始化面试回答数据记录 + cacheMsgMapData.put(session.getId(), ""); + //初始化面试问题 + cacheQuestionResult.put(session.getId(), ""); } // 发生错误时调用 @@ -385,45 +347,6 @@ public class ChatWebSocketHandler { throwable.printStackTrace(); } - /** - * 将字节数组保存为WebM文件 - * - * @param byteData 包含WebM数据的字节数组 - * @param filePath 目标文件路径 - * @return 操作是否成功 - */ - private boolean saveAsWebM(byte[] byteData, String filePath) { - // 检查输入参数 - if (byteData == null || byteData.length == 0) { - System.err.println("字节数组为空，无法生成WebM文件"); - return false; - } - if (filePath == null || filePath.trim().isEmpty()) { - System.err.println("文件路径不能为空"); - return false; - } - FileOutputStream fos = null; - try { - fos = new FileOutputStream(filePath); - fos.write(byteData); - fos.flush(); - System.out.println("WebM文件已成功生成: " + filePath); - return true; - } catch (IOException e) { - System.err.println("写入文件时发生错误: " + e.getMessage()); - e.printStackTrace(); - } finally { - if (fos != null) { - try { - fos.close(); - } catch (IOException e) { - e.printStackTrace(); - } - } - } - return false; - } - /** * File 转换成 ByteBuffer * @@ -441,130 +364,181 @@ public class ChatWebSocketHandler { } /** - * 创建STT WebSocket 客户端链接 + * 发送语音流给前端 * - * @param clientId 客户端ID + * @param pathUrl 语音文件地址 + * @param session 客户端会话 */ - private void createWhisperRealtimeSocket(String clientId) { + private void sendVoiceBuffer(String pathUrl, Session session) { try { - OkHttpClient client = new OkHttpClient(); - // 设置 WebSocket 请求 - Request request = new Request.Builder() - .url(API_URL) - .addHeader("Authorization", "Bearer " + apiKey) - .addHeader("OpenAI-Beta", "realtime=v1") - .build(); - client.newWebSocket(request, new WebSocketListener() { - @Override - public void onOpen(WebSocket webSocket, Response response) { - System.out.println("✅ WebSocket 连接成功"); - //发送配置 - JSONObject config = new JSONObject(); - JSONObject sessionConfig = new JSONObject(); - JSONObject transcription = new JSONObject(); - JSONObject turnDetection = new JSONObject(); - // 配置转录参数 - transcription.put("model", MODEL); - transcription.put("language", language); // 中文 - // 配置断句检测 - turnDetection.put("type", "server_vad"); - turnDetection.put("prefix_padding_ms", 300); - turnDetection.put("silence_duration_ms", 10); - // 组装完整配置 - sessionConfig.put("input_audio_transcription", transcription); - sessionConfig.put("turn_detection", turnDetection); - config.put("type", "transcription_session.update"); - config.put("session", sessionConfig); - webSocket.send(config.toString()); + //文件转换成文件流 + ByteBuffer outByteBuffer = convertFileToByteBuffer(pathUrl); + //发送文件流数据 + session.getBasicRemote().sendBinary(outByteBuffer); + // 发送响应确认 + log.info("已经成功发送了语音流给前端:{}", DateUtil.now()); + } catch (IOException e) { + e.printStackTrace(); + } + } - // 1. 启动音频缓冲 -// webSocket.send("{\"type\": \"input_audio_buffer.start\"}"); + /** + * 发送文本转语音,发送语音流给前端 + * + * @param clientId 用户ID + * @param content 文本内容 + * @param session 客户端会话ID + */ + private void sendTTSBuffer(String clientId, String content, Session session) { + String resultFileName = clientId + "_" + System.currentTimeMillis() + ".wav"; + String resultPathUrl = RuoYiConfig.getProfile() + VOICE_STORAGE_RESULT_DIR + resultFileName; + ElevenLabsClient elevenLabsClient = SpringUtils.getBean(ElevenLabsClient.class); + elevenLabsClient.handleTextToVoice(content, resultPathUrl); + //持续返回数据流给客户端 + sendVoiceBuffer(resultPathUrl, session); + } - //存储客户端webSocket对象,对数据进行隔离处理 - cacheWebSocket.put(clientId, webSocket); + /** + * 对候选者初次进行提问业务逻辑处理(初始化系统随机获取第一个问题) + * + * @param clientId 用户ID + * @param session 客户端会话 + */ + private void initializationQuestion(String clientId, Session session) { + try { + //自我介绍结束后马上返回一个Good + //发送初始化面试官语音流 + String openingPathUrl = RuoYiConfig.getProfile() + VOICE_SYSTEM_DIR + "good.wav"; + sendVoiceBuffer(openingPathUrl, session); + //初始化面试流程的提问 + List> list = new LinkedList(); + Map mapEntity = new HashMap<>(); + mapEntity.put("role", "system"); + mapEntity.put("content", "You are a construction industry interview expert. You MUST provide both an evaluation and follow-up questions for every response. Use this exact format:\\n\\nEVALUATION:\\n[Rate 0-5 and provide detailed assessment. If answer is unrelated, contains technical errors, system messages, or is nonsensical, give 0/5]\\n\\nFOLLOW-UP:\\n[Always generate 1-2 relevant follow-up questions. If answer was invalid, ask for clarification or repeat the question]\\n\\nIMPORTANT: You must include both EVALUATION and FOLLOW-UP sections in every response."); + list.add(mapEntity); + //获取预设问题-直接TTS转换返回语音结果 + IHotakeProblemBaseInfoService problemBaseInfoService = SpringUtils.getBean(IHotakeProblemBaseInfoService.class); + HotakeProblemBaseInfo queryPro = new HotakeProblemBaseInfo(); + queryPro.setUserId(Long.valueOf(clientId)); + List baseInfoList = problemBaseInfoService.selectHotakeProblemBaseInfoList(queryPro); + if (CollectionUtil.isNotEmpty(baseInfoList)) { + HotakeProblemBaseInfo baseInfo = baseInfoList.get(0); + if (StrUtil.isNotEmpty(baseInfo.getContents())) { + String[] qStrs = baseInfo.getContents().split(","); + int random_index = (int) (Math.random() * qStrs.length); + //获取问题文本 + String question = qStrs[random_index]; + Map mapEntityQ = new HashMap<>(); + mapEntityQ.put("role", "user"); + mapEntityQ.put("content", "Question：" + question + "\\nCandidate Answer：{}"); + list.add(mapEntityQ); + //直接对该问题进行转换处理返回语音流 + sendTTSBuffer(clientId, question, session); } + } + //初始化记录提示词数据到-缓存中 + cacheMsgMapData.put(session.getId(), JSONUtil.toJsonStr(list)); + } catch (Exception e) { + e.printStackTrace(); + log.error("面试流程初始化失败:{}", e.getMessage()); + } + } - @Override - public void onMessage(WebSocket webSocket, String text) { -// System.out.println("📩 收到转录结果: " + text); - //对数据进行解析 - if (StrUtil.isNotEmpty(text)) { - Map mapResultData = JSONUtil.toBean(text, Map.class); - if ("conversation.item.input_audio_transcription.delta".equals(mapResultData.get("type"))) { - String resultText = mapResultData.get("delta"); - //进行客户端文本数据存储 - String cacheString = cacheClientTts.get(clientId); - if (StrUtil.isNotEmpty(cacheString)) { - cacheString = cacheString + resultText; - } else { - cacheString = resultText; - } - log.info("收到转录结果:{}","客户端ID为:"+clientId+",转录结果:"+resultText); - cacheClientTts.put(clientId, cacheString); - } - } - } + /** + * 处理面试结束业务逻辑 + * 触发规则: + * 1、获得 0-1 分 大于1次 立即结束面试 + * 2、获取 4-5 分 大于3次 立即结束面试 + * 3、获取 2-3 分 大于3次 立即结束面试 + * 4、获取 2-5 分 大于4次 立即结束面试 + * 5、没有 FOLLOW-UP 理解结束面试 + * + * @param session 客户端会话 + * @param content 追问内容 + */ + private void handleInterviewEnd(Session session, String content) { + //验证是否触发面试结束逻辑 + try { + //发送面试官结束语音流 + String pathUrl = RuoYiConfig.getProfile() + VOICE_SYSTEM_DIR + "end.wav"; + sendVoiceBuffer(pathUrl, session); + //返回文本评分 + //结束回答要清空问答数据 + cacheMsgMapData.put(session.getId(), ""); + Map resultEntity = new HashMap<>(); + resultEntity.put("msg", content); + resultEntity.put("dataType", "score"); + session.getBasicRemote().sendText(JSONUtil.toJsonStr(resultEntity)); - @Override - public void onFailure(WebSocket webSocket, Throwable t, Response response) { - System.err.println("❌ 连接失败: " + t.getMessage()); -// latch.countDown(); - } - - @Override - public void onClosing(WebSocket webSocket, int code, String reason) { - System.out.println("⚠️ 连接即将关闭: " + reason); - webSocket.close(1000, null); -// latch.countDown(); - } - }); } catch (Exception e) { e.printStackTrace(); } } /** - * 语音流文件格式转换 + * 解析AI结果,获取随机问题 * - * @param pathUrl - * @param outPathUrl + * @param content + * @return */ - private void handleAudioToPCM(String pathUrl, String outPathUrl) { - File inputFile = new File(pathUrl); // 输入音频文件 - File outputFile = new File(outPathUrl); // 输出PCM格式文件 - try { - // 读取音频文件 - AudioInputStream inputAudioStream = AudioSystem.getAudioInputStream(inputFile); - // 获取音频文件的格式信息 - AudioFormat sourceFormat = inputAudioStream.getFormat(); - System.out.println("Input Audio Format: " + sourceFormat); - // 设置目标PCM格式 (可以是16-bit, 8kHz, Mono, Linear PCM) - AudioFormat pcmFormat = new AudioFormat( - AudioFormat.Encoding.PCM_SIGNED, - sourceFormat.getSampleRate(), - 16, // 16-bit samples - 1, // 单声道 - 2, // 每个样本2字节（16位） - sourceFormat.getSampleRate(), - false // 大端模式 - ); - // 获取PCM格式的音频流 - AudioInputStream pcmAudioStream = AudioSystem.getAudioInputStream(pcmFormat, inputAudioStream); - // 创建输出文件流 - FileOutputStream fos = new FileOutputStream(outputFile); - byte[] buffer = new byte[1024]; - int bytesRead; - // 将PCM音频数据写入输出文件 - while ((bytesRead = pcmAudioStream.read(buffer)) != -1) { - fos.write(buffer, 0, bytesRead); + private String handleReturnQuestion(String content) { + + + return ""; + } + + /** + * 处理评分记录 + * 触发规则: + * 1、获得 0-1 分 大于1次 立即结束面试 + * 2、获取 4-5 分 大于3次 立即结束面试 + * 3、获取 2-3 分 大于3次 立即结束面试 + * 4、获取 2-5 分 大于4次 立即结束面试 + * + * @param content + * @param session return false 立即结束面试 + */ + private Boolean handleScoreRecord(String content, Session session) { + Map scoreRecordMap = cacheScoreResult.get(session.getId()); + //对评分进行处理 + if (StrUtil.isNotEmpty(content)) { + String[] strs = content.split("\\\\"); + //取第一个数就是对应的评分 + BigDecimal score = new BigDecimal(strs[0]); + //记录Key为1 + if (BigDecimal.ZERO.compareTo(score) <= 0 && BigDecimal.ONE.compareTo(score) >= 0) { + Integer n1 = scoreRecordMap.get("0-1") + 1; + scoreRecordMap.put("0-1", n1); + if (n1 > 1) { + return false; + } + } + //记录Key为2 + if (new BigDecimal(4).compareTo(score) <= 0 && new BigDecimal(5).compareTo(score) >= 0) { + Integer n1 = scoreRecordMap.get("4-5") + 1; + scoreRecordMap.put("4-5", n1); + if (n1 > 3) { + return false; + } + } + //记录Key为3 + if (new BigDecimal(2).compareTo(score) <= 0 && new BigDecimal(3).compareTo(score) >= 0) { + Integer n1 = scoreRecordMap.get("2-3") + 1; + scoreRecordMap.put("2-3", n1); + if (n1 > 3) { + return false; + } + } + //记录Key为4 + if (new BigDecimal(2).compareTo(score) <= 0 && new BigDecimal(5).compareTo(score) >= 0) { + Integer n1 = scoreRecordMap.get("2-5") + 1; + scoreRecordMap.put("2-5", n1); + if (n1 > 4) { + return false; + } } - // 关闭流 - pcmAudioStream.close(); - fos.close(); - System.out.println("Audio has been converted to PCM format and saved at: " + outputFile.getAbsolutePath()); - } catch (Exception e) { - e.printStackTrace(); } + return true; } } diff --git a/vetti-admin/src/main/java/com/vetti/socket/ChatWebSocketHandler1.java b/vetti-admin/src/main/java/com/vetti/socket/ChatWebSocketHandler1.java new file mode 100644 index 0000000..a72a145 --- /dev/null +++ b/vetti-admin/src/main/java/com/vetti/socket/ChatWebSocketHandler1.java @@ -0,0 +1,589 @@ +package com.vetti.socket; + +import cn.hutool.core.collection.CollectionUtil; +import cn.hutool.core.util.StrUtil; +import cn.hutool.json.JSONObject; +import cn.hutool.json.JSONUtil; +import com.vetti.common.ai.elevenLabs.ElevenLabsClient; +import com.vetti.common.ai.gpt.ChatGPTClient; +import com.vetti.common.ai.gpt.OpenAiStreamClient; +import com.vetti.common.ai.gpt.service.OpenAiStreamListenerService; +import com.vetti.common.config.RuoYiConfig; +import com.vetti.common.utils.spring.SpringUtils; +import com.vetti.hotake.domain.HotakeProblemBaseInfo; +import com.vetti.hotake.service.IHotakeProblemBaseInfoService; +import lombok.extern.slf4j.Slf4j; +import okhttp3.*; +import org.apache.commons.io.FileUtils; +import org.springframework.stereotype.Component; + +import javax.sound.sampled.AudioFormat; +import javax.sound.sampled.AudioInputStream; +import javax.sound.sampled.AudioSystem; +import javax.websocket.*; +import javax.websocket.server.PathParam; +import javax.websocket.server.ServerEndpoint; +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +/** + * 语音面试 web处理器(无用的只是记录使用) + */ +@Slf4j +@ServerEndpoint("/voice-websocket1111111111/{clientId}") +@Component +public class ChatWebSocketHandler1 { + + // @Value("${whisper.apiUrl}") + private String API_URL = "wss://api.openai.com/v1/realtime?intent=transcription"; + + // @Value("${whisper.model}") + private String MODEL = "gpt-4o-mini-transcribe"; + + // @Value("${whisper.apiKey}") + private String apiKey = "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"; + + // @Value("${whisper.language}") + private String language = "en"; + + /** + * 缓存客户端流式解析的语音文本数据 + */ + private final Map cacheClientTts = new ConcurrentHashMap<>(); + + /** + * 缓存客户端调用OpenAi中的websocket-STT 流式传输数据 + */ + private final Map cacheWebSocket = new ConcurrentHashMap<>(); + + /** + * 缓存客户端,标记是否是自我介绍后的初次问答 + */ + private final Map cacheReplyFlag = new ConcurrentHashMap<>(); + + /** + * 缓存客户端,面试回答信息 + */ + private final Map cacheMsgMapData = new ConcurrentHashMap<>(); + + /** + * 缓存客户端,AI提问的问题结果信息 + */ + private final Map cacheQuestionResult = new ConcurrentHashMap<>(); + +// private final Map cacheOpeningResult = new ConcurrentHashMap<>(); + + // 语音文件保存目录 + private static final String VOICE_STORAGE_DIR = "/voice_files/"; + + // 语音结果文件保存目录 + private static final String VOICE_STORAGE_RESULT_DIR = "/voice_result_files/"; + + // 系统语音目录 + private static final String VOICE_SYSTEM_DIR = "/system_files/"; + + public ChatWebSocketHandler1() { + // 初始化存储目录 + File dir = new File(RuoYiConfig.getProfile() + VOICE_STORAGE_DIR); + if (!dir.exists()) { + dir.mkdirs(); + } + + File resultDir = new File(RuoYiConfig.getProfile() + VOICE_STORAGE_RESULT_DIR); + if (!resultDir.exists()) { + resultDir.mkdirs(); + } + } + + // 连接建立时调用 + @OnOpen + public void onOpen(Session session, @PathParam("clientId") String clientId) { + log.info("WebSocket 链接已建立:{}", clientId); + log.info("WebSocket session 链接已建立:{}", session.getId()); + cacheClientTts.put(clientId, new String()); + //初始化STT流式语音转换文本的socket链接 +// createWhisperRealtimeSocket(session.getId()); + //是初次自我介绍后的问答环节 + cacheReplyFlag.put(session.getId(),"YES"); + //初始化面试回答数据记录 + cacheMsgMapData.put(session.getId(),""); + //初始化面试问题 + cacheQuestionResult.put(session.getId(),""); + //开场白控制 +// String flag = cacheOpeningResult.get(clientId); +// if(StrUtil.isEmpty(flag)){ + //发送初始化面试官语音流 + String openingPathUrl = RuoYiConfig.getProfile() + VOICE_SYSTEM_DIR + "opening.wav"; + try { + //文件转换成文件流 + ByteBuffer outByteBuffer = convertFileToByteBuffer(openingPathUrl); + //发送文件流数据 + session.getBasicRemote().sendBinary(outByteBuffer); +// cacheOpeningResult.put(clientId,"YES"); + // 发送响应确认 + log.info("初始化返回面试官语音信息:{}", System.currentTimeMillis() / 1000); + } catch (IOException e) { + e.printStackTrace(); + } +// } + } + + // 接收文本消息 + @OnMessage + public void onTextMessage(Session session, String message, @PathParam("clientId") String clientId) { + System.out.println("接收到文本消息: " + message); + try { +// { +// "type": "start | done | end", +// "content": "内容" +// } + //处理文本结果 + if (StrUtil.isNotEmpty(message)) { + Map mapResult = JSONUtil.toBean(JSONUtil.parseObj(message), Map.class); + String resultFlag = mapResult.get("type"); + if ("done".equals(resultFlag)) { + //开始合并语音流 + String startFlag = cacheReplyFlag.get(session.getId()); + //语音结束,开始进行回答解析 + log.info("开始文本处理,客户端ID为:{}",clientId); + String cacheResultText = mapResult.get("content"); + log.info("开始文本处理,面试者回答信息为:{}", cacheResultText); + if (StrUtil.isEmpty(cacheResultText)) { + cacheResultText = "I first check the forklift's logbook for recent issues, inspect tires and brakes, verify the load capacity matches today's task, and confirm my licence is current—all per SWMS requirements."; + } + String promptJson = ""; + //这是初次处理的逻辑 + if("YES".equals(startFlag)) { + //自我介绍结束后马上返回一个Good + //发送初始化面试官语音流 + String openingPathUrl = RuoYiConfig.getProfile() + VOICE_SYSTEM_DIR + "good.wav"; + try { + //文件转换成文件流 + ByteBuffer outByteBuffer = convertFileToByteBuffer(openingPathUrl); + //发送文件流数据 + session.getBasicRemote().sendBinary(outByteBuffer); + // 发送响应确认 + log.info("初始化返回面试官语音信息:{}", System.currentTimeMillis() / 1000); + } catch (IOException e) { + e.printStackTrace(); + } + List> list = new LinkedList(); + Map mapEntity = new HashMap<>(); + mapEntity.put("role","system"); + mapEntity.put("content","You are an interviewer. Generate follow-up questions based on Construction Labourer candidate responses.Only return one question and do not repeat the previously returned questions \\n MPORTANT: Do not ask the same question again if the answer is incorrect"); + list.add(mapEntity); + //获取预设问题-直接TTS转换返回语音结果 + IHotakeProblemBaseInfoService problemBaseInfoService = SpringUtils.getBean(IHotakeProblemBaseInfoService.class); + HotakeProblemBaseInfo queryPro = new HotakeProblemBaseInfo(); + queryPro.setUserId(Long.valueOf(clientId)); + List baseInfoList = problemBaseInfoService.selectHotakeProblemBaseInfoList(queryPro); + if(CollectionUtil.isNotEmpty(baseInfoList)) { + HotakeProblemBaseInfo baseInfo = baseInfoList.get(0); + if(StrUtil.isNotEmpty(baseInfo.getContents())){ + String[] qStrs = baseInfo.getContents().split(","); + int random_index = (int) (Math.random()*qStrs.length); + //获取问题文本 + String question = qStrs[random_index]; + Map mapEntityQ = new HashMap<>(); + mapEntityQ.put("role","user"); + mapEntityQ.put("content","Question："+question+"\\nCandidate Answer：{}"); + list.add(mapEntityQ); + } + } + promptJson = JSONUtil.toJsonStr(list); + //记录缓存中 + cacheMsgMapData.put(session.getId(),promptJson); + //直接对该问题进行转换处理返回语音流 + + }else{ + //开始根据面试者回答的问题,进行追问回答 + //获取面试者回答信息 + //获取缓存记录 + String msgMapData = cacheMsgMapData.get(session.getId()); + if(StrUtil.isNotEmpty(msgMapData)){ + List list = JSONUtil.toList(msgMapData, Map.class); + //获取最后一条数据记录 + Map mapEntity = list.get(list.size()-1); + //更新问题记录 + String content = mapEntity.get("content"); + mapEntity.put("content", StrUtil.format(content, cacheResultText)); + promptJson = JSONUtil.toJsonStr(list); + cacheMsgMapData.put(session.getId(),promptJson); + } + } + //获取完问答数据,直接清空缓存数据 + cacheClientTts.put(session.getId(),""); + cacheReplyFlag.put(session.getId(),""); + //把提问的文字发送给CPT(流式处理) + OpenAiStreamClient aiStreamClient = SpringUtils.getBean(OpenAiStreamClient.class); + log.info("AI提示词为:{}",promptJson); + aiStreamClient.streamChat(promptJson, new OpenAiStreamListenerService() { + @Override + public void onMessage(String content) { + log.info("返回AI结果：{}", content); + if(StrUtil.isNotEmpty(content)){ + String questionResult = cacheQuestionResult.get(session.getId()); + if(StrUtil.isEmpty(questionResult)){ + questionResult = content; + }else{ + questionResult = questionResult + content; + } + cacheQuestionResult.put(session.getId(),questionResult); + // 实时输出内容 + try{ + //把文本也给前端返回去 + Map dataText = new HashMap<>(); + dataText.put("type","question"); + dataText.put("content",content); + session.getBasicRemote().sendText(JSONUtil.toJsonStr(dataText)); + }catch (Exception e){ + e.printStackTrace(); + } + //开始进行语音输出-流式持续输出 + //把结果文字转成语音文件 + //生成文件 + //生成唯一文件名 + String resultFileName = clientId + "_" + System.currentTimeMillis() + ".wav"; + String resultPathUrl = RuoYiConfig.getProfile() + VOICE_STORAGE_RESULT_DIR + resultFileName; + ElevenLabsClient elevenLabsClient = SpringUtils.getBean(ElevenLabsClient.class); + elevenLabsClient.handleTextToVoice(content, resultPathUrl); + //持续返回数据流给客户端 + try { + //文件转换成文件流 + ByteBuffer outByteBuffer = convertFileToByteBuffer(resultPathUrl); + //发送文件流数据 + session.getBasicRemote().sendBinary(outByteBuffer); + // 发送响应确认 + } catch (IOException e) { + e.printStackTrace(); + } + } + } + + @Override + public void onComplete() { + try { + //开始往缓存中记录提问的问题 + String questionResult = cacheQuestionResult.get(session.getId()); + //获取缓存记录 + String msgMapData = cacheMsgMapData.get(session.getId()); + if(StrUtil.isNotEmpty(msgMapData)){ + List list = JSONUtil.toList(msgMapData, Map.class); + Map mapEntity = new HashMap<>(); + mapEntity.put("role","user"); + mapEntity.put("content","Question："+questionResult+"\\nCandidate Answer：{}"); + list.add(mapEntity); + cacheMsgMapData.put(session.getId(),JSONUtil.toJsonStr(list)); + } + //清空问题 + cacheQuestionResult.put(session.getId(),""); + + Map resultEntity = new HashMap<>(); + resultEntity.put("msg", "done"); + //发送通知告诉客户端已经回答结束了 + session.getBasicRemote().sendText(JSONUtil.toJsonStr(resultEntity)); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + @Override + public void onError(Throwable throwable) { + throwable.printStackTrace(); + } + }); + }else if("end".equals(resultFlag)){ +// cacheOpeningResult.put(clientId,""); + //发送面试官结束语音流 + String openingPathUrl = RuoYiConfig.getProfile() + VOICE_SYSTEM_DIR + "end.wav"; + try { + //文件转换成文件流 + ByteBuffer outByteBuffer = convertFileToByteBuffer(openingPathUrl); + //发送文件流数据 + session.getBasicRemote().sendBinary(outByteBuffer); + // 发送响应确认 + log.info("结束返回面试官语音信息:{}", System.currentTimeMillis() / 1000); + } catch (IOException e) { + e.printStackTrace(); + } + //返回文本评分 + //处理模型提问逻辑 + String promptJson = ""; + //获取缓存记录 + String msgMapData = cacheMsgMapData.get(session.getId()); + if(StrUtil.isNotEmpty(msgMapData)){ + List list = JSONUtil.toList(msgMapData, Map.class); + //获取最后一条数据记录 + Map mapEntity = list.get(0); + //更新问题记录 + mapEntity.put("role","system"); + mapEntity.put("content","You are a construction industry interview expert. Rate Construction Labourer candidate responses on a 1-5 scale. IMPORTANT: If the answer is completely unrelated, contains technical errors, system messages, or is nonsensical, give it a score of 0/5 and explain why it's invalid."); + promptJson = JSONUtil.toJsonStr(list); + //结束回答要清空问答数据 + cacheMsgMapData.put(session.getId(),""); + } + log.info("结束AI提示词为:{}",promptJson); + ChatGPTClient gptClient = SpringUtils.getBean(ChatGPTClient.class); + String resultMsg = gptClient.handleAiChat(promptJson,"QA"); + Map resultEntity = new HashMap<>(); + resultEntity.put("msg", resultMsg); + resultEntity.put("dataType","score"); + session.getBasicRemote().sendText(JSONUtil.toJsonStr(resultEntity)); + } + } + } catch (Exception e) { + e.printStackTrace(); + } + } + + // 接收二进制消息（流数据） + @OnMessage + public void onBinaryMessage(Session session, @PathParam("clientId") String clientId, ByteBuffer byteBuffer) { +// log.info("客户端ID为:{}", clientId); +// // 处理二进制流数据 +// byte[] bytes = new byte[byteBuffer.remaining()]; +// //从缓冲区中读取数据并存储到指定的字节数组中 +// byteBuffer.get(bytes); +// // 生成唯一文件名 +// String fileName = clientId + "_" + System.currentTimeMillis() + ".wav"; +// String pathUrl = RuoYiConfig.getProfile() + VOICE_STORAGE_DIR + fileName; +// log.info("文件路径为:{}", pathUrl); +// try { +// saveAsWebM(bytes, pathUrl); +// //接收到数据流后直接就进行SST处理 +// //语音格式转换 +// String fileOutName = clientId + "_" + System.currentTimeMillis() + ".pcm"; +// String pathOutUrl = RuoYiConfig.getProfile() + VOICE_STORAGE_DIR + fileOutName; +// handleAudioToPCM(pathUrl, pathOutUrl); +// //发送消息 +// WebSocket webSocket = cacheWebSocket.get(session.getId()); +// log.info("获取的socket对象为:{}", webSocket); +// if (webSocket != null) { +//// 1. 启动音频缓冲 +//// webSocket.send("{\"type\": \"input_audio_buffer.start\"}"); +// File outputFile = new File(pathOutUrl); // 输出PCM格式文件 +// ByteBuffer buffer = ByteBuffer.wrap(FileUtils.readFileToByteArray(outputFile)); +// byte[] outBytes = new byte[buffer.remaining()]; +// //从缓冲区中读取数据并存储到指定的字节数组中 +// buffer.get(outBytes); +// String base64Audio = Base64.getEncoder().encodeToString(outBytes); +// String message = "{ \"type\": \"input_audio_buffer.append\", \"audio\": \"" + base64Audio + "\" }"; +// webSocket.send(message); +// // 3. 提交音频并请求转录 +//// webSocket.send("{\"type\": \"input_audio_buffer.commit\"}"); +//// webSocket.send("{\"type\": \"response.create\"}"); +// } +// } catch (Exception e) { +// e.printStackTrace(); +// } + + } + + // 连接关闭时调用 + @OnClose + public void onClose(Session session, CloseReason reason) { + System.out.println("WebSocket连接已关闭: " + session.getId() + ", 原因: " + reason.getReasonPhrase()); +// WebSocket webSocket = cacheWebSocket.get(session.getId()); +// if (webSocket != null) { +// webSocket.close(1000, null); +// } + } + + // 发生错误时调用 + @OnError + public void onError(Session session, Throwable throwable) { + System.err.println("WebSocket错误发生: " + throwable.getMessage()); + throwable.printStackTrace(); + } + + /** + * 将字节数组保存为WebM文件 + * + * @param byteData 包含WebM数据的字节数组 + * @param filePath 目标文件路径 + * @return 操作是否成功 + */ + private boolean saveAsWebM(byte[] byteData, String filePath) { + // 检查输入参数 + if (byteData == null || byteData.length == 0) { + System.err.println("字节数组为空，无法生成WebM文件"); + return false; + } + if (filePath == null || filePath.trim().isEmpty()) { + System.err.println("文件路径不能为空"); + return false; + } + FileOutputStream fos = null; + try { + fos = new FileOutputStream(filePath); + fos.write(byteData); + fos.flush(); + System.out.println("WebM文件已成功生成: " + filePath); + return true; + } catch (IOException e) { + System.err.println("写入文件时发生错误: " + e.getMessage()); + e.printStackTrace(); + } finally { + if (fos != null) { + try { + fos.close(); + } catch (IOException e) { + e.printStackTrace(); + } + } + } + return false; + } + + /** + * File 转换成 ByteBuffer + * + * @param fileUrl 文件路径 + * @return + */ + private ByteBuffer convertFileToByteBuffer(String fileUrl) { + File file = new File(fileUrl); + try { + return ByteBuffer.wrap(FileUtils.readFileToByteArray(file)); + } catch (Exception e) { + e.printStackTrace(); + } + return null; + } + + /** + * 创建STT WebSocket 客户端链接 + * + * @param clientId 客户端ID + */ + private void createWhisperRealtimeSocket(String clientId) { + try { + OkHttpClient client = new OkHttpClient(); + // 设置 WebSocket 请求 + Request request = new Request.Builder() + .url(API_URL) + .addHeader("Authorization", "Bearer " + apiKey) + .addHeader("OpenAI-Beta", "realtime=v1") + .build(); + client.newWebSocket(request, new WebSocketListener() { + @Override + public void onOpen(WebSocket webSocket, Response response) { + System.out.println("✅ WebSocket 连接成功"); + //发送配置 + JSONObject config = new JSONObject(); + JSONObject sessionConfig = new JSONObject(); + JSONObject transcription = new JSONObject(); + JSONObject turnDetection = new JSONObject(); + // 配置转录参数 + transcription.put("model", MODEL); + transcription.put("language", language); // 中文 + // 配置断句检测 + turnDetection.put("type", "server_vad"); + turnDetection.put("prefix_padding_ms", 300); + turnDetection.put("silence_duration_ms", 10); + // 组装完整配置 + sessionConfig.put("input_audio_transcription", transcription); + sessionConfig.put("turn_detection", turnDetection); + config.put("type", "transcription_session.update"); + config.put("session", sessionConfig); + webSocket.send(config.toString()); + + // 1. 启动音频缓冲 +// webSocket.send("{\"type\": \"input_audio_buffer.start\"}"); + + //存储客户端webSocket对象,对数据进行隔离处理 + cacheWebSocket.put(clientId, webSocket); + } + + @Override + public void onMessage(WebSocket webSocket, String text) { +// System.out.println("📩 收到转录结果: " + text); + //对数据进行解析 + if (StrUtil.isNotEmpty(text)) { + Map mapResultData = JSONUtil.toBean(text, Map.class); + if ("conversation.item.input_audio_transcription.delta".equals(mapResultData.get("type"))) { + String resultText = mapResultData.get("delta"); + //进行客户端文本数据存储 + String cacheString = cacheClientTts.get(clientId); + if (StrUtil.isNotEmpty(cacheString)) { + cacheString = cacheString + resultText; + } else { + cacheString = resultText; + } + log.info("收到转录结果:{}","客户端ID为:"+clientId+",转录结果:"+resultText); + cacheClientTts.put(clientId, cacheString); + } + } + } + + @Override + public void onFailure(WebSocket webSocket, Throwable t, Response response) { + System.err.println("❌ 连接失败: " + t.getMessage()); +// latch.countDown(); + } + + @Override + public void onClosing(WebSocket webSocket, int code, String reason) { + System.out.println("⚠️ 连接即将关闭: " + reason); + webSocket.close(1000, null); +// latch.countDown(); + } + }); + } catch (Exception e) { + e.printStackTrace(); + } + } + + /** + * 语音流文件格式转换 + * + * @param pathUrl + * @param outPathUrl + */ + private void handleAudioToPCM(String pathUrl, String outPathUrl) { + File inputFile = new File(pathUrl); // 输入音频文件 + File outputFile = new File(outPathUrl); // 输出PCM格式文件 + try { + // 读取音频文件 + AudioInputStream inputAudioStream = AudioSystem.getAudioInputStream(inputFile); + // 获取音频文件的格式信息 + AudioFormat sourceFormat = inputAudioStream.getFormat(); + System.out.println("Input Audio Format: " + sourceFormat); + // 设置目标PCM格式 (可以是16-bit, 8kHz, Mono, Linear PCM) + AudioFormat pcmFormat = new AudioFormat( + AudioFormat.Encoding.PCM_SIGNED, + sourceFormat.getSampleRate(), + 16, // 16-bit samples + 1, // 单声道 + 2, // 每个样本2字节（16位） + sourceFormat.getSampleRate(), + false // 大端模式 + ); + // 获取PCM格式的音频流 + AudioInputStream pcmAudioStream = AudioSystem.getAudioInputStream(pcmFormat, inputAudioStream); + // 创建输出文件流 + FileOutputStream fos = new FileOutputStream(outputFile); + byte[] buffer = new byte[1024]; + int bytesRead; + // 将PCM音频数据写入输出文件 + while ((bytesRead = pcmAudioStream.read(buffer)) != -1) { + fos.write(buffer, 0, bytesRead); + } + // 关闭流 + pcmAudioStream.close(); + fos.close(); + System.out.println("Audio has been converted to PCM format and saved at: " + outputFile.getAbsolutePath()); + } catch (Exception e) { + e.printStackTrace(); + } + } + +} + diff --git a/vetti-admin/src/main/java/com/vetti/web/controller/ai/AiCommonController.java b/vetti-admin/src/main/java/com/vetti/web/controller/ai/AiCommonController.java index f55cda7..30de200 100644 --- a/vetti-admin/src/main/java/com/vetti/web/controller/ai/AiCommonController.java +++ b/vetti-admin/src/main/java/com/vetti/web/controller/ai/AiCommonController.java @@ -2,6 +2,8 @@ package com.vetti.web.controller.ai; import com.vetti.common.ai.elevenLabs.ElevenLabsClient; import com.vetti.common.ai.gpt.ChatGPTClient; +import com.vetti.common.ai.gpt.OpenAiStreamClient; +import com.vetti.common.ai.gpt.service.OpenAiStreamListenerService; import com.vetti.common.ai.whisper.WhisperClient; import com.vetti.common.core.controller.BaseController; import com.vetti.common.core.domain.AjaxResult; @@ -35,6 +37,9 @@ public class AiCommonController extends BaseController @Autowired private WhisperClient whisperClient; + @Autowired + private OpenAiStreamClient streamClient; + @Autowired private ICommonService commonService; @@ -63,6 +68,35 @@ public class AiCommonController extends BaseController return AjaxResult.success(resultMsg); } + /** + * AI 流式聊天 + */ + @ApiOperation("AI 流式聊天") + @GetMapping("/stream/handleAiChat") + public AjaxResult handleAiStreamChat(@RequestParam String text) + { + streamClient.streamChat(text, new OpenAiStreamListenerService() { + @Override + public void onMessage(String content) { + System.out.println("AI 返回数据: "+content); + } + + @Override + public void onComplete() { + + } + + @Override + public void onError(Throwable throwable) { + throwable.printStackTrace(); + } + }); + + + String resultMsg = chatGPTClient.handleAiChat(text,"QA"); + return AjaxResult.success(resultMsg); + } + /** diff --git a/vetti-admin/src/main/java/com/vetti/web/controller/system/SysProfileController.java b/vetti-admin/src/main/java/com/vetti/web/controller/system/SysProfileController.java index 46e2583..0ed2a59 100644 --- a/vetti-admin/src/main/java/com/vetti/web/controller/system/SysProfileController.java +++ b/vetti-admin/src/main/java/com/vetti/web/controller/system/SysProfileController.java @@ -4,11 +4,15 @@ import java.util.List; import java.util.Map; import cn.hutool.core.bean.BeanUtil; +import cn.hutool.core.collection.CollectionUtil; import cn.hutool.json.JSONUtil; import com.vetti.common.core.domain.R; +import com.vetti.common.enums.IsInterviewEnum; import com.vetti.common.enums.UserOperStepsEnum; import com.vetti.hotake.domain.HotakeCvInfo; +import com.vetti.hotake.domain.HotakeProblemBaseInfo; import com.vetti.hotake.service.IHotakeCvInfoService; +import com.vetti.hotake.service.IHotakeProblemBaseInfoService; import com.vetti.web.entity.dto.SysUserDto; import io.swagger.annotations.Api; import io.swagger.annotations.ApiOperation; @@ -56,6 +60,9 @@ public class SysProfileController extends BaseController @Autowired private IHotakeCvInfoService cvInfoService; + @Autowired + private IHotakeProblemBaseInfoService problemBaseInfoService; + /** * 个人信息 */ @@ -71,6 +78,16 @@ public class SysProfileController extends BaseController query.setUserId(user.getUserId()); List cvInfoList = cvInfoService.selectHotakeCvInfoList(query); dto.setCvInfoList(cvInfoList); + + //是否可以参加面试 + HotakeProblemBaseInfo queryProblemBaseInfo = new HotakeProblemBaseInfo(); + queryProblemBaseInfo.setUserId(user.getUserId()); + List problemBaseInfoList = problemBaseInfoService.selectHotakeProblemBaseInfoList(queryProblemBaseInfo); + if(CollectionUtil.isNotEmpty(problemBaseInfoList)){ + dto.setIsInterview(IsInterviewEnum.INTERVIEW_1.getCode()); + }else{ + dto.setIsInterview(IsInterviewEnum.INTERVIEW_0.getCode()); + } // ajax.put("roleGroup", userService.selectUserRoleGroup(loginUser.getUsername())); // ajax.put("postGroup", userService.selectUserPostGroup(loginUser.getUsername())); return R.ok(dto); diff --git a/vetti-admin/src/main/java/com/vetti/web/entity/dto/SysUserDto.java b/vetti-admin/src/main/java/com/vetti/web/entity/dto/SysUserDto.java index bc592f5..204e17e 100644 --- a/vetti-admin/src/main/java/com/vetti/web/entity/dto/SysUserDto.java +++ b/vetti-admin/src/main/java/com/vetti/web/entity/dto/SysUserDto.java @@ -18,6 +18,9 @@ import java.util.List; @Accessors(chain = true) public class SysUserDto extends SysUser { + @ApiModelProperty("是否可以参加面试标识(0:不可以,1:可以)") + private String isInterview; + @ApiModelProperty("简历数据集合") private List cvInfoList; } diff --git a/vetti-admin/src/main/resources/application-druid.yml b/vetti-admin/src/main/resources/application-druid.yml index ebc82af..ce69a14 100644 --- a/vetti-admin/src/main/resources/application-druid.yml +++ b/vetti-admin/src/main/resources/application-druid.yml @@ -169,7 +169,7 @@ whisper: chatGpt: apiKey: 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 apiUrl: https://api.openai.com/v1/chat/completions - model: ft:gpt-3.5-turbo-0125:vetti:construction-labourer-test:CTIvLD5n + model: ft:gpt-3.5-turbo-0125:vetti:construction-labourer-test:CWKBNvE2 modelCV: ft:gpt-3.5-turbo-0125:vetti:vetti-resume-test:CWPinJQq role: system diff --git a/vetti-admin/target/classes/application-druid.yml b/vetti-admin/target/classes/application-druid.yml index ebc82af..ce69a14 100644 --- a/vetti-admin/target/classes/application-druid.yml +++ b/vetti-admin/target/classes/application-druid.yml @@ -169,7 +169,7 @@ whisper: chatGpt: apiKey: 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 apiUrl: https://api.openai.com/v1/chat/completions - model: ft:gpt-3.5-turbo-0125:vetti:construction-labourer-test:CTIvLD5n + model: ft:gpt-3.5-turbo-0125:vetti:construction-labourer-test:CWKBNvE2 modelCV: ft:gpt-3.5-turbo-0125:vetti:vetti-resume-test:CWPinJQq role: system diff --git a/vetti-common/src/main/java/com/vetti/common/ai/gpt/OpenAiStreamClient.java b/vetti-common/src/main/java/com/vetti/common/ai/gpt/OpenAiStreamClient.java index 2f4219f..62437e8 100644 --- a/vetti-common/src/main/java/com/vetti/common/ai/gpt/OpenAiStreamClient.java +++ b/vetti-common/src/main/java/com/vetti/common/ai/gpt/OpenAiStreamClient.java @@ -33,7 +33,7 @@ public class OpenAiStreamClient { private String role; // 定义作为分割点的标点符号集合 - private final String punctuationStr = "。,？,！,；,\\.,\\?,!,;"; + private final String punctuationStr = "。？！；.?!;"; /** * 发送流式请求 diff --git a/vetti-common/src/main/java/com/vetti/common/enums/IsInterviewEnum.java b/vetti-common/src/main/java/com/vetti/common/enums/IsInterviewEnum.java new file mode 100644 index 0000000..64e1800 --- /dev/null +++ b/vetti-common/src/main/java/com/vetti/common/enums/IsInterviewEnum.java @@ -0,0 +1,30 @@ +package com.vetti.common.enums; + +/** + * 是否可以参加面试 + */ +public enum IsInterviewEnum { + + INTERVIEW_0("0", "不可以"), + INTERVIEW_1("1", "可以"), + ; + + private final String code; + private final String info; + + IsInterviewEnum(String code, String info) + { + this.code = code; + this.info = info; + } + + public String getCode() + { + return code; + } + + public String getInfo() + { + return info; + } +} diff --git a/vetti-system/target/classes/mapper/system/SysUserMapper.xml b/vetti-system/target/classes/mapper/system/SysUserMapper.xml index 7b9217f..129aec1 100644 --- a/vetti-system/target/classes/mapper/system/SysUserMapper.xml +++ b/vetti-system/target/classes/mapper/system/SysUserMapper.xml @@ -25,6 +25,7 @@ PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" + @@ -66,7 +67,7 @@ PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" select u.user_id, u.dept_id, u.user_name, u.nick_name, u.email, u.avatar, u.phonenumber, u.password, u.sex, u.status, u.del_flag, u.login_ip, u.login_date, u.pwd_update_date, u.create_by, u.create_time, u.remark, d.dept_id, d.parent_id, d.ancestors, d.dept_name, d.order_num, d.leader, d.status as dept_status, r.role_id, r.role_name, r.role_key, r.role_sort, r.data_scope, r.status as role_status,u.sys_user_type - ,u.steps,u.job_position,u.experience,u.cv_url,u.location,u.job_type,u.relocate,u.best_side_json,u.address,u.user_flag + ,u.steps,u.job_position,u.experience,u.cv_url,u.location,u.job_type,u.relocate,u.best_side_json,u.address,u.user_flag,u.user_set_json from sys_user u left join sys_dept d on u.dept_id = d.dept_id left join sys_user_role ur on u.user_id = ur.user_id @@ -75,7 +76,7 @@ PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" select distinct u.user_id, u.dept_id, u.user_name, u.nick_name, u.email, u.phonenumber, u.status, u.create_time,u.sys_user_type,u.steps,u.job_position,u.experience,u.cv_url,u.location, - u.job_type,u.relocate,u.best_side_json,u.address,u.user_flag + u.job_type,u.relocate,u.best_side_json,u.address,u.user_flag,u.user_set_json from sys_user u left join sys_dept d on u.dept_id = d.dept_id left join sys_user_role ur on u.user_id = ur.user_id @@ -127,7 +128,7 @@ PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN"