sxc/Vetti-Service-new
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

模型调整以及业务流程处理

Browse Source
This commit is contained in:
wangxiangshun
2025-10-25 11:59:24 +08:00
parent b51a69f3ea
commit 9a2c8d7de1
3 changed files with 90 additions and 63 deletions
Download Patch File Download Diff File Expand all files Collapse all files

147
vetti-admin/src/main/java/com/vetti/socket/ChatWebSocketHandler.java
View File

@@ -33,28 +33,33 @@ import java.util.concurrent.ConcurrentHashMap;
@Component
public class ChatWebSocketHandler {
// @Value("${whisper.apiUrl}")
// @Value("${whisper.apiUrl}")
private String API_URL = "wss://api.openai.com/v1/realtime?intent=transcription";
// @Value("${whisper.model}")
// @Value("${whisper.model}")
private String MODEL = "gpt-4o-mini-transcribe";
// @Value("${whisper.apiKey}")
// @Value("${whisper.apiKey}")
private String apiKey = "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";
// @Value("${whisper.language}")
// @Value("${whisper.language}")
private String language = "en";
/**
* 缓存客户端流式解析的语音文本数据
*/
private final Map<String,String> cacheClientTts = new ConcurrentHashMap<>();
private final Map<String, String> cacheClientTts = new ConcurrentHashMap<>();
/**
* 缓存客户端调用OpenAi中的websocket-STT 流式传输数据
*/
private final Map<String, WebSocket> cacheWebSocket = new ConcurrentHashMap<>();
/**
* 缓存客户端,标记是否是自我介绍后的初次问答
*/
private final Map<String,String> cacheReplyFlag = new ConcurrentHashMap<>();
// 语音文件保存目录
private static final String VOICE_STORAGE_DIR = "/voice_files/";
@@ -82,9 +87,11 @@ public class ChatWebSocketHandler {
public void onOpen(Session session, @PathParam("clientId") String clientId) {
log.info("WebSocket 链接已建立:{}", clientId);
log.info("WebSocket session 链接已建立:{}", session.getId());
cacheClientTts.put(clientId,new String());
cacheClientTts.put(clientId, new String());
//初始化STT流式语音转换文本的socket链接
createWhisperRealtimeSocket(clientId);
createWhisperRealtimeSocket(session.getId());
//是初次自我介绍后的问答环节
cacheReplyFlag.put(session.getId(),"YES");
//发送初始化面试官语音流
String openingPathUrl = RuoYiConfig.getProfile() + VOICE_SYSTEM_DIR + "opening.wav";
try {
@@ -93,7 +100,7 @@ public class ChatWebSocketHandler {
//发送文件流数据
session.getBasicRemote().sendBinary(outByteBuffer);
// 发送响应确认
log.info("初始化返回面试官语音信息:{}",System.currentTimeMillis()/1000);
log.info("初始化返回面试官语音信息:{}", System.currentTimeMillis() / 1000);
} catch (IOException e) {
e.printStackTrace();
}
@@ -102,40 +109,58 @@ public class ChatWebSocketHandler {
// 接收文本消息
@OnMessage
public void onTextMessage(Session session, String message,@PathParam("clientId") String clientId) {
public void onTextMessage(Session session, String message, @PathParam("clientId") String clientId) {
System.out.println("接收到文本消息: " + message);
try {
//处理文本结果
if(StrUtil.isNotEmpty(message)){
Map<String,String> mapResult = JSONUtil.toBean(JSONUtil.parseObj(message),Map.class);
if (StrUtil.isNotEmpty(message)) {
Map<String, String> mapResult = JSONUtil.toBean(JSONUtil.parseObj(message), Map.class);
String resultFlag = mapResult.get("msg");
if("done".equals(resultFlag)){
log.info("1、开始处理时间:{}",System.currentTimeMillis()/1000);
// //开始合并语音流
if ("done".equals(resultFlag)) {
log.info("1、开始处理时间:{}", System.currentTimeMillis() / 1000);
//开始合并语音流
//发送消息
WebSocket webSocket = cacheWebSocket.get(clientId);
if(webSocket != null){
}
WebSocket webSocket = cacheWebSocket.get(session.getId());
if (webSocket != null) {
webSocket.send("{\"type\": \"input_audio_buffer.commit\"}");
webSocket.send("{\"type\": \"response.create\"}");
}
String startFlag = cacheReplyFlag.get(session.getId());
//语音结束,开始进行回答解析
String cacheResultText = cacheClientTts.get(clientId);
log.info("返回的结果为:{}",cacheResultText);
if(StrUtil.isEmpty(cacheResultText)){
cacheResultText = "Hello , How are you?";
log.info("返回的结果为:{}", cacheResultText);
if (StrUtil.isEmpty(cacheResultText)) {
cacheResultText = "Hi.";
}
log.info("1、开始进行AI回答时间:{}",System.currentTimeMillis()/1000);
if("YES".equals(startFlag)) {
//自我介绍结束后马上返回一个Good
//发送初始化面试官语音流
String openingPathUrl = RuoYiConfig.getProfile() + VOICE_SYSTEM_DIR + "good.wav";
try {
//文件转换成文件流
ByteBuffer outByteBuffer = convertFileToByteBuffer(openingPathUrl);
//发送文件流数据
session.getBasicRemote().sendBinary(outByteBuffer);
// 发送响应确认
log.info("初始化返回面试官语音信息:{}", System.currentTimeMillis() / 1000);
} catch (IOException e) {
e.printStackTrace();
}
cacheResultText = "你是面试官，根据Construction Labourer候选人回答生成追问。只要一个问题";
}
//获取完问答数据,直接清空缓存数据
cacheClientTts.put(clientId,"");
cacheReplyFlag.put(session.getId(),"");
log.info("1、开始进行AI回答时间:{}", System.currentTimeMillis() / 1000);
//把提问的文字发送给CPT(流式处理)
OpenAiStreamClient aiStreamClient = SpringUtils.getBean(OpenAiStreamClient.class);
aiStreamClient.streamChat(cacheResultText, new OpenAiStreamListenerService() {
@Override
public void onMessage(String content) {
log.info("返回AI结果：{}",content);
log.info("返回AI结果：{}", content);
// 实时输出内容
//开始进行语音输出-流式持续输出
log.info("2、开始进行AI回答时间:{}",System.currentTimeMillis()/1000);
log.info("2、开始进行AI回答时间:{}", System.currentTimeMillis() / 1000);
//把结果文字转成语音文件
//生成文件
//生成唯一文件名
@@ -143,7 +168,7 @@ public class ChatWebSocketHandler {
String resultPathUrl = RuoYiConfig.getProfile() + VOICE_STORAGE_RESULT_DIR + resultFileName;
ElevenLabsClient elevenLabsClient = SpringUtils.getBean(ElevenLabsClient.class);
elevenLabsClient.handleTextToVoice(content, resultPathUrl);
log.info("3、开始进行AI回答时间:{}",System.currentTimeMillis()/1000);
log.info("3、开始进行AI回答时间:{}", System.currentTimeMillis() / 1000);
//持续返回数据流给客户端
try {
//文件转换成文件流
@@ -151,7 +176,7 @@ public class ChatWebSocketHandler {
//发送文件流数据
session.getBasicRemote().sendBinary(outByteBuffer);
// 发送响应确认
log.info("4、开始进行AI回答时间:{}",System.currentTimeMillis()/1000);
log.info("4、开始进行AI回答时间:{}", System.currentTimeMillis() / 1000);
} catch (IOException e) {
e.printStackTrace();
}
@@ -160,14 +185,14 @@ public class ChatWebSocketHandler {
@Override
public void onComplete() {
try {
Map<String,String> resultEntity = new HashMap<>();
resultEntity.put("msg","done");
Map<String, String> resultEntity = new HashMap<>();
resultEntity.put("msg", "done");
//发送通知告诉客户端已经回答结束了
session.getBasicRemote().sendText(JSONUtil.toJsonStr(resultEntity));
} catch (Exception e) {
throw new RuntimeException(e);
}
log.info("5、结束进行AI回答时间:{}",System.currentTimeMillis()/1000);
log.info("5、结束进行AI回答时间:{}", System.currentTimeMillis() / 1000);
}
@Override
@@ -185,30 +210,30 @@ public class ChatWebSocketHandler {
// 接收二进制消息（流数据）
@OnMessage
public void onBinaryMessage(Session session, @PathParam("clientId") String clientId, ByteBuffer byteBuffer) {
log.info("1、开始接收数据流时间:{}",System.currentTimeMillis()/1000);
log.info("1、开始接收数据流时间:{}", System.currentTimeMillis() / 1000);
log.info("客户端ID为:{}", clientId);
// 处理二进制流数据
byte[] bytes = new byte[byteBuffer.remaining()];
//从缓冲区中读取数据并存储到指定的字节数组中
byteBuffer.get(bytes);
log.info("2、开始接收数据流时间:{}",System.currentTimeMillis()/1000);
log.info("2、开始接收数据流时间:{}", System.currentTimeMillis() / 1000);
// 生成唯一文件名
String fileName = clientId + "_" + System.currentTimeMillis() + ".wav";
String pathUrl = RuoYiConfig.getProfile()+VOICE_STORAGE_DIR + fileName;
String pathUrl = RuoYiConfig.getProfile() + VOICE_STORAGE_DIR + fileName;
log.info("文件路径为:{}", pathUrl);
log.info("3、开始接收数据流时间:{}",System.currentTimeMillis()/1000);
try{
log.info("文件流的大小为:{}",bytes.length);
saveAsWebM(bytes,pathUrl);
log.info("3、开始接收数据流时间:{}", System.currentTimeMillis() / 1000);
try {
log.info("文件流的大小为:{}", bytes.length);
saveAsWebM(bytes, pathUrl);
//接收到数据流后直接就进行SST处理
//语音格式转换
String fileOutName = clientId + "_" + System.currentTimeMillis() + ".pcm";
String pathOutUrl = RuoYiConfig.getProfile()+VOICE_STORAGE_DIR + fileOutName;
handleAudioToPCM(pathUrl,pathOutUrl);
String pathOutUrl = RuoYiConfig.getProfile() + VOICE_STORAGE_DIR + fileOutName;
handleAudioToPCM(pathUrl, pathOutUrl);
//发送消息
WebSocket webSocket = cacheWebSocket.get(clientId);
log.info("获取的socket对象为:{}",webSocket);
if(webSocket != null){
WebSocket webSocket = cacheWebSocket.get(session.getId());
log.info("获取的socket对象为:{}", webSocket);
if (webSocket != null) {
// 1. 启动音频缓冲
// webSocket.send("{\"type\": \"input_audio_buffer.start\"}");
log.info("3.1 开始发送数据音频流啦");
@@ -220,12 +245,12 @@ public class ChatWebSocketHandler {
String base64Audio = Base64.getEncoder().encodeToString(outBytes);
String message = "{ \"type\": \"input_audio_buffer.append\", \"audio\": \"" + base64Audio + "\" }";
webSocket.send(message);
log.info("4、开始接收数据流时间:{}",System.currentTimeMillis()/1000);
log.info("4、开始接收数据流时间:{}", System.currentTimeMillis() / 1000);
// 3. 提交音频并请求转录
// webSocket.send("{\"type\": \"input_audio_buffer.commit\"}");
// webSocket.send("{\"type\": \"response.create\"}");
}
}catch (Exception e){
} catch (Exception e) {
e.printStackTrace();
}
@@ -235,10 +260,10 @@ public class ChatWebSocketHandler {
@OnClose
public void onClose(Session session, CloseReason reason) {
System.out.println("WebSocket连接已关闭: " + session.getId() + ", 原因: " + reason.getReasonPhrase());
// WebSocket webSocket = cacheWebSocket.get(clientId);
// if(webSocket != null){
// webSocket.close(1000,null);
// }
WebSocket webSocket = cacheWebSocket.get(session.getId());
if (webSocket != null) {
webSocket.close(1000, null);
}
}
// 发生错误时调用
@@ -305,10 +330,11 @@ public class ChatWebSocketHandler {
/**
* 创建STT WebSocket 客户端链接
*
* @param clientId 客户端ID
*/
private void createWhisperRealtimeSocket(String clientId){
try{
private void createWhisperRealtimeSocket(String clientId) {
try {
OkHttpClient client = new OkHttpClient();
// 设置 WebSocket 请求
Request request = new Request.Builder()
@@ -343,25 +369,25 @@ public class ChatWebSocketHandler {
// webSocket.send("{\"type\": \"input_audio_buffer.start\"}");
//存储客户端webSocket对象,对数据进行隔离处理
cacheWebSocket.put(clientId,webSocket);
cacheWebSocket.put(clientId, webSocket);
}
@Override
public void onMessage(WebSocket webSocket, String text) {
System.out.println("📩 收到转录结果: " + text);
//对数据进行解析
if(StrUtil.isNotEmpty(text)){
Map<String,String> mapResultData = JSONUtil.toBean(text,Map.class);
if("conversation.item.input_audio_transcription.delta".equals(mapResultData.get("type"))){
if (StrUtil.isNotEmpty(text)) {
Map<String, String> mapResultData = JSONUtil.toBean(text, Map.class);
if ("conversation.item.input_audio_transcription.delta".equals(mapResultData.get("type"))) {
String resultText = mapResultData.get("delta");
//进行客户端文本数据存储
String cacheString = cacheClientTts.get(clientId);
if(StrUtil.isNotEmpty(cacheString)){
cacheString = cacheString+resultText;
}else {
if (StrUtil.isNotEmpty(cacheString)) {
cacheString = cacheString + resultText;
} else {
cacheString = resultText;
}
cacheClientTts.put(clientId,cacheString);
cacheClientTts.put(clientId, cacheString);
}
}
}
@@ -379,17 +405,18 @@ public class ChatWebSocketHandler {
// latch.countDown();
}
});
}catch (Exception e){
} catch (Exception e) {
e.printStackTrace();
}
}
/**
* 语音流文件格式转换
*
* @param pathUrl
* @param outPathUrl
*/
private void handleAudioToPCM(String pathUrl,String outPathUrl){
private void handleAudioToPCM(String pathUrl, String outPathUrl) {
File inputFile = new File(pathUrl); // 输入音频文件
File outputFile = new File(outPathUrl); // 输出PCM格式文件
try {

2
vetti-admin/src/main/resources/application-druid.yml
View File

@@ -165,7 +165,7 @@ chatGpt:
apiKey: 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
apiUrl: https://api.openai.com/v1/chat/completions
model: ft:gpt-3.5-turbo-0125:vetti:construction-labourer-test:CTIvLD5n
role: user
role: system
http:

2
vetti-admin/target/classes/application-druid.yml
View File

@@ -165,7 +165,7 @@ chatGpt:
apiKey: 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
apiUrl: https://api.openai.com/v1/chat/completions
model: ft:gpt-3.5-turbo-0125:vetti:construction-labourer-test:CTIvLD5n
role: user
role: system
http: