From 9a2c8d7de1a94321ad2cf624e8c0f691e6db43c4 Mon Sep 17 00:00:00 2001 From: wangxiangshun Date: Sat, 25 Oct 2025 11:59:24 +0800 Subject: [PATCH] =?UTF-8?q?=E6=A8=A1=E5=9E=8B=E8=B0=83=E6=95=B4=E4=BB=A5?= =?UTF-8?q?=E5=8F=8A=E4=B8=9A=E5=8A=A1=E6=B5=81=E7=A8=8B=E5=A4=84=E7=90=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../vetti/socket/ChatWebSocketHandler.java | 149 +++++++++++------- .../src/main/resources/application-druid.yml | 2 +- .../target/classes/application-druid.yml | 2 +- 3 files changed, 90 insertions(+), 63 deletions(-) diff --git a/vetti-admin/src/main/java/com/vetti/socket/ChatWebSocketHandler.java b/vetti-admin/src/main/java/com/vetti/socket/ChatWebSocketHandler.java index 838210a..ce7d12a 100644 --- a/vetti-admin/src/main/java/com/vetti/socket/ChatWebSocketHandler.java +++ b/vetti-admin/src/main/java/com/vetti/socket/ChatWebSocketHandler.java @@ -33,28 +33,33 @@ import java.util.concurrent.ConcurrentHashMap; @Component public class ChatWebSocketHandler { -// @Value("${whisper.apiUrl}") + // @Value("${whisper.apiUrl}") private String API_URL = "wss://api.openai.com/v1/realtime?intent=transcription"; -// @Value("${whisper.model}") + // @Value("${whisper.model}") private String MODEL = "gpt-4o-mini-transcribe"; -// @Value("${whisper.apiKey}") + // @Value("${whisper.apiKey}") private String apiKey = "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"; -// @Value("${whisper.language}") + // @Value("${whisper.language}") private String language = "en"; /** * 缓存客户端流式解析的语音文本数据 */ - private final Map cacheClientTts = new ConcurrentHashMap<>(); + private final Map cacheClientTts = new ConcurrentHashMap<>(); /** * 缓存客户端调用OpenAi中的websocket-STT 流式传输数据 */ private final Map cacheWebSocket = new ConcurrentHashMap<>(); + /** + * 缓存客户端,标记是否是自我介绍后的初次问答 + */ + private final Map cacheReplyFlag = new ConcurrentHashMap<>(); + // 语音文件保存目录 private static final String VOICE_STORAGE_DIR = "/voice_files/"; @@ -82,9 +87,11 @@ public class ChatWebSocketHandler { public void onOpen(Session session, @PathParam("clientId") String clientId) { log.info("WebSocket 链接已建立:{}", clientId); log.info("WebSocket session 链接已建立:{}", session.getId()); - cacheClientTts.put(clientId,new String()); + cacheClientTts.put(clientId, new String()); //初始化STT流式语音转换文本的socket链接 - createWhisperRealtimeSocket(clientId); + createWhisperRealtimeSocket(session.getId()); + //是初次自我介绍后的问答环节 + cacheReplyFlag.put(session.getId(),"YES"); //发送初始化面试官语音流 String openingPathUrl = RuoYiConfig.getProfile() + VOICE_SYSTEM_DIR + "opening.wav"; try { @@ -93,7 +100,7 @@ public class ChatWebSocketHandler { //发送文件流数据 session.getBasicRemote().sendBinary(outByteBuffer); // 发送响应确认 - log.info("初始化返回面试官语音信息:{}",System.currentTimeMillis()/1000); + log.info("初始化返回面试官语音信息:{}", System.currentTimeMillis() / 1000); } catch (IOException e) { e.printStackTrace(); } @@ -102,40 +109,58 @@ public class ChatWebSocketHandler { // 接收文本消息 @OnMessage - public void onTextMessage(Session session, String message,@PathParam("clientId") String clientId) { + public void onTextMessage(Session session, String message, @PathParam("clientId") String clientId) { System.out.println("接收到文本消息: " + message); try { //处理文本结果 - if(StrUtil.isNotEmpty(message)){ - Map mapResult = JSONUtil.toBean(JSONUtil.parseObj(message),Map.class); + if (StrUtil.isNotEmpty(message)) { + Map mapResult = JSONUtil.toBean(JSONUtil.parseObj(message), Map.class); String resultFlag = mapResult.get("msg"); - if("done".equals(resultFlag)){ - log.info("1、开始处理时间:{}",System.currentTimeMillis()/1000); -// //开始合并语音流 + if ("done".equals(resultFlag)) { + log.info("1、开始处理时间:{}", System.currentTimeMillis() / 1000); + //开始合并语音流 //发送消息 - WebSocket webSocket = cacheWebSocket.get(clientId); - if(webSocket != null){ - + WebSocket webSocket = cacheWebSocket.get(session.getId()); + if (webSocket != null) { + webSocket.send("{\"type\": \"input_audio_buffer.commit\"}"); + webSocket.send("{\"type\": \"response.create\"}"); } - webSocket.send("{\"type\": \"input_audio_buffer.commit\"}"); - webSocket.send("{\"type\": \"response.create\"}"); - + String startFlag = cacheReplyFlag.get(session.getId()); //语音结束,开始进行回答解析 String cacheResultText = cacheClientTts.get(clientId); - log.info("返回的结果为:{}",cacheResultText); - if(StrUtil.isEmpty(cacheResultText)){ - cacheResultText = "Hello , How are you?"; + log.info("返回的结果为:{}", cacheResultText); + if (StrUtil.isEmpty(cacheResultText)) { + cacheResultText = "Hi."; } - log.info("1、开始进行AI回答时间:{}",System.currentTimeMillis()/1000); + if("YES".equals(startFlag)) { + //自我介绍结束后马上返回一个Good + //发送初始化面试官语音流 + String openingPathUrl = RuoYiConfig.getProfile() + VOICE_SYSTEM_DIR + "good.wav"; + try { + //文件转换成文件流 + ByteBuffer outByteBuffer = convertFileToByteBuffer(openingPathUrl); + //发送文件流数据 + session.getBasicRemote().sendBinary(outByteBuffer); + // 发送响应确认 + log.info("初始化返回面试官语音信息:{}", System.currentTimeMillis() / 1000); + } catch (IOException e) { + e.printStackTrace(); + } + cacheResultText = "你是面试官，根据Construction Labourer候选人回答生成追问。只要一个问题"; + } + //获取完问答数据,直接清空缓存数据 + cacheClientTts.put(clientId,""); + cacheReplyFlag.put(session.getId(),""); + log.info("1、开始进行AI回答时间:{}", System.currentTimeMillis() / 1000); //把提问的文字发送给CPT(流式处理) OpenAiStreamClient aiStreamClient = SpringUtils.getBean(OpenAiStreamClient.class); aiStreamClient.streamChat(cacheResultText, new OpenAiStreamListenerService() { @Override public void onMessage(String content) { - log.info("返回AI结果：{}",content); + log.info("返回AI结果：{}", content); // 实时输出内容 //开始进行语音输出-流式持续输出 - log.info("2、开始进行AI回答时间:{}",System.currentTimeMillis()/1000); + log.info("2、开始进行AI回答时间:{}", System.currentTimeMillis() / 1000); //把结果文字转成语音文件 //生成文件 //生成唯一文件名 @@ -143,7 +168,7 @@ public class ChatWebSocketHandler { String resultPathUrl = RuoYiConfig.getProfile() + VOICE_STORAGE_RESULT_DIR + resultFileName; ElevenLabsClient elevenLabsClient = SpringUtils.getBean(ElevenLabsClient.class); elevenLabsClient.handleTextToVoice(content, resultPathUrl); - log.info("3、开始进行AI回答时间:{}",System.currentTimeMillis()/1000); + log.info("3、开始进行AI回答时间:{}", System.currentTimeMillis() / 1000); //持续返回数据流给客户端 try { //文件转换成文件流 @@ -151,7 +176,7 @@ public class ChatWebSocketHandler { //发送文件流数据 session.getBasicRemote().sendBinary(outByteBuffer); // 发送响应确认 - log.info("4、开始进行AI回答时间:{}",System.currentTimeMillis()/1000); + log.info("4、开始进行AI回答时间:{}", System.currentTimeMillis() / 1000); } catch (IOException e) { e.printStackTrace(); } @@ -160,14 +185,14 @@ public class ChatWebSocketHandler { @Override public void onComplete() { try { - Map resultEntity = new HashMap<>(); - resultEntity.put("msg","done"); + Map resultEntity = new HashMap<>(); + resultEntity.put("msg", "done"); //发送通知告诉客户端已经回答结束了 session.getBasicRemote().sendText(JSONUtil.toJsonStr(resultEntity)); } catch (Exception e) { throw new RuntimeException(e); } - log.info("5、结束进行AI回答时间:{}",System.currentTimeMillis()/1000); + log.info("5、结束进行AI回答时间:{}", System.currentTimeMillis() / 1000); } @Override @@ -185,30 +210,30 @@ public class ChatWebSocketHandler { // 接收二进制消息（流数据） @OnMessage public void onBinaryMessage(Session session, @PathParam("clientId") String clientId, ByteBuffer byteBuffer) { - log.info("1、开始接收数据流时间:{}",System.currentTimeMillis()/1000); + log.info("1、开始接收数据流时间:{}", System.currentTimeMillis() / 1000); log.info("客户端ID为:{}", clientId); // 处理二进制流数据 byte[] bytes = new byte[byteBuffer.remaining()]; //从缓冲区中读取数据并存储到指定的字节数组中 byteBuffer.get(bytes); - log.info("2、开始接收数据流时间:{}",System.currentTimeMillis()/1000); + log.info("2、开始接收数据流时间:{}", System.currentTimeMillis() / 1000); // 生成唯一文件名 String fileName = clientId + "_" + System.currentTimeMillis() + ".wav"; - String pathUrl = RuoYiConfig.getProfile()+VOICE_STORAGE_DIR + fileName; + String pathUrl = RuoYiConfig.getProfile() + VOICE_STORAGE_DIR + fileName; log.info("文件路径为:{}", pathUrl); - log.info("3、开始接收数据流时间:{}",System.currentTimeMillis()/1000); - try{ - log.info("文件流的大小为:{}",bytes.length); - saveAsWebM(bytes,pathUrl); + log.info("3、开始接收数据流时间:{}", System.currentTimeMillis() / 1000); + try { + log.info("文件流的大小为:{}", bytes.length); + saveAsWebM(bytes, pathUrl); //接收到数据流后直接就进行SST处理 //语音格式转换 String fileOutName = clientId + "_" + System.currentTimeMillis() + ".pcm"; - String pathOutUrl = RuoYiConfig.getProfile()+VOICE_STORAGE_DIR + fileOutName; - handleAudioToPCM(pathUrl,pathOutUrl); + String pathOutUrl = RuoYiConfig.getProfile() + VOICE_STORAGE_DIR + fileOutName; + handleAudioToPCM(pathUrl, pathOutUrl); //发送消息 - WebSocket webSocket = cacheWebSocket.get(clientId); - log.info("获取的socket对象为:{}",webSocket); - if(webSocket != null){ + WebSocket webSocket = cacheWebSocket.get(session.getId()); + log.info("获取的socket对象为:{}", webSocket); + if (webSocket != null) { // 1. 启动音频缓冲 // webSocket.send("{\"type\": \"input_audio_buffer.start\"}"); log.info("3.1 开始发送数据音频流啦"); @@ -220,12 +245,12 @@ public class ChatWebSocketHandler { String base64Audio = Base64.getEncoder().encodeToString(outBytes); String message = "{ \"type\": \"input_audio_buffer.append\", \"audio\": \"" + base64Audio + "\" }"; webSocket.send(message); - log.info("4、开始接收数据流时间:{}",System.currentTimeMillis()/1000); + log.info("4、开始接收数据流时间:{}", System.currentTimeMillis() / 1000); // 3. 提交音频并请求转录 // webSocket.send("{\"type\": \"input_audio_buffer.commit\"}"); // webSocket.send("{\"type\": \"response.create\"}"); } - }catch (Exception e){ + } catch (Exception e) { e.printStackTrace(); } @@ -235,10 +260,10 @@ public class ChatWebSocketHandler { @OnClose public void onClose(Session session, CloseReason reason) { System.out.println("WebSocket连接已关闭: " + session.getId() + ", 原因: " + reason.getReasonPhrase()); -// WebSocket webSocket = cacheWebSocket.get(clientId); -// if(webSocket != null){ -// webSocket.close(1000,null); -// } + WebSocket webSocket = cacheWebSocket.get(session.getId()); + if (webSocket != null) { + webSocket.close(1000, null); + } } // 发生错误时调用 @@ -305,10 +330,11 @@ public class ChatWebSocketHandler { /** * 创建STT WebSocket 客户端链接 + * * @param clientId 客户端ID */ - private void createWhisperRealtimeSocket(String clientId){ - try{ + private void createWhisperRealtimeSocket(String clientId) { + try { OkHttpClient client = new OkHttpClient(); // 设置 WebSocket 请求 Request request = new Request.Builder() @@ -343,25 +369,25 @@ public class ChatWebSocketHandler { // webSocket.send("{\"type\": \"input_audio_buffer.start\"}"); //存储客户端webSocket对象,对数据进行隔离处理 - cacheWebSocket.put(clientId,webSocket); + cacheWebSocket.put(clientId, webSocket); } @Override public void onMessage(WebSocket webSocket, String text) { System.out.println("📩 收到转录结果: " + text); //对数据进行解析 - if(StrUtil.isNotEmpty(text)){ - Map mapResultData = JSONUtil.toBean(text,Map.class); - if("conversation.item.input_audio_transcription.delta".equals(mapResultData.get("type"))){ + if (StrUtil.isNotEmpty(text)) { + Map mapResultData = JSONUtil.toBean(text, Map.class); + if ("conversation.item.input_audio_transcription.delta".equals(mapResultData.get("type"))) { String resultText = mapResultData.get("delta"); //进行客户端文本数据存储 String cacheString = cacheClientTts.get(clientId); - if(StrUtil.isNotEmpty(cacheString)){ - cacheString = cacheString+resultText; - }else { + if (StrUtil.isNotEmpty(cacheString)) { + cacheString = cacheString + resultText; + } else { cacheString = resultText; } - cacheClientTts.put(clientId,cacheString); + cacheClientTts.put(clientId, cacheString); } } } @@ -379,17 +405,18 @@ public class ChatWebSocketHandler { // latch.countDown(); } }); - }catch (Exception e){ + } catch (Exception e) { e.printStackTrace(); } } /** * 语音流文件格式转换 + * * @param pathUrl * @param outPathUrl */ - private void handleAudioToPCM(String pathUrl,String outPathUrl){ + private void handleAudioToPCM(String pathUrl, String outPathUrl) { File inputFile = new File(pathUrl); // 输入音频文件 File outputFile = new File(outPathUrl); // 输出PCM格式文件 try { diff --git a/vetti-admin/src/main/resources/application-druid.yml b/vetti-admin/src/main/resources/application-druid.yml index 34556a7..511c901 100644 --- a/vetti-admin/src/main/resources/application-druid.yml +++ b/vetti-admin/src/main/resources/application-druid.yml @@ -165,7 +165,7 @@ chatGpt: apiKey: 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 apiUrl: https://api.openai.com/v1/chat/completions model: ft:gpt-3.5-turbo-0125:vetti:construction-labourer-test:CTIvLD5n - role: user + role: system http: diff --git a/vetti-admin/target/classes/application-druid.yml b/vetti-admin/target/classes/application-druid.yml index 34556a7..511c901 100644 --- a/vetti-admin/target/classes/application-druid.yml +++ b/vetti-admin/target/classes/application-druid.yml @@ -165,7 +165,7 @@ chatGpt: apiKey: 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 apiUrl: https://api.openai.com/v1/chat/completions model: ft:gpt-3.5-turbo-0125:vetti:construction-labourer-test:CTIvLD5n - role: user + role: system http: