sxc/Vetti-Service-new
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

模型调整以及业务流程处理

Browse Source
This commit is contained in:
wangxiangshun
2025-10-25 11:59:24 +08:00
parent b51a69f3ea
commit 9a2c8d7de1
3 changed files with 90 additions and 63 deletions
Download Patch File Download Diff File Expand all files Collapse all files

149
vetti-admin/src/main/java/com/vetti/socket/ChatWebSocketHandler.java
View File

@@ -33,28 +33,33 @@ import java.util.concurrent.ConcurrentHashMap;
@Component @Component
public class ChatWebSocketHandler { public class ChatWebSocketHandler {
// @Value("${whisper.apiUrl}") // @Value("${whisper.apiUrl}")
private String API_URL = "wss://api.openai.com/v1/realtime?intent=transcription"; private String API_URL = "wss://api.openai.com/v1/realtime?intent=transcription";
// @Value("${whisper.model}") // @Value("${whisper.model}")
private String MODEL = "gpt-4o-mini-transcribe"; private String MODEL = "gpt-4o-mini-transcribe";
// @Value("${whisper.apiKey}") // @Value("${whisper.apiKey}")
private String apiKey = "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"; private String apiKey = "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";
// @Value("${whisper.language}") // @Value("${whisper.language}")
private String language = "en"; private String language = "en";
/** /**
* 缓存客户端流式解析的语音文本数据 * 缓存客户端流式解析的语音文本数据
*/ */
private final Map<String,String> cacheClientTts = new ConcurrentHashMap<>(); private final Map<String, String> cacheClientTts = new ConcurrentHashMap<>();
/** /**
* 缓存客户端调用OpenAi中的websocket-STT 流式传输数据 * 缓存客户端调用OpenAi中的websocket-STT 流式传输数据
*/ */
private final Map<String, WebSocket> cacheWebSocket = new ConcurrentHashMap<>(); private final Map<String, WebSocket> cacheWebSocket = new ConcurrentHashMap<>();
/**
* 缓存客户端,标记是否是自我介绍后的初次问答
*/
private final Map<String,String> cacheReplyFlag = new ConcurrentHashMap<>();
// 语音文件保存目录 // 语音文件保存目录
private static final String VOICE_STORAGE_DIR = "/voice_files/"; private static final String VOICE_STORAGE_DIR = "/voice_files/";
@@ -82,9 +87,11 @@ public class ChatWebSocketHandler {
public void onOpen(Session session, @PathParam("clientId") String clientId) { public void onOpen(Session session, @PathParam("clientId") String clientId) {
log.info("WebSocket 链接已建立:{}", clientId); log.info("WebSocket 链接已建立:{}", clientId);
log.info("WebSocket session 链接已建立:{}", session.getId()); log.info("WebSocket session 链接已建立:{}", session.getId());
cacheClientTts.put(clientId,new String()); cacheClientTts.put(clientId, new String());
//初始化STT流式语音转换文本的socket链接 //初始化STT流式语音转换文本的socket链接
createWhisperRealtimeSocket(clientId); createWhisperRealtimeSocket(session.getId());
//是初次自我介绍后的问答环节
cacheReplyFlag.put(session.getId(),"YES");
//发送初始化面试官语音流 //发送初始化面试官语音流
String openingPathUrl = RuoYiConfig.getProfile() + VOICE_SYSTEM_DIR + "opening.wav"; String openingPathUrl = RuoYiConfig.getProfile() + VOICE_SYSTEM_DIR + "opening.wav";
try { try {
@@ -93,7 +100,7 @@ public class ChatWebSocketHandler {
//发送文件流数据 //发送文件流数据
session.getBasicRemote().sendBinary(outByteBuffer); session.getBasicRemote().sendBinary(outByteBuffer);
// 发送响应确认 // 发送响应确认
log.info("初始化返回面试官语音信息:{}",System.currentTimeMillis()/1000); log.info("初始化返回面试官语音信息:{}", System.currentTimeMillis() / 1000);
} catch (IOException e) { } catch (IOException e) {
e.printStackTrace(); e.printStackTrace();
} }
@@ -102,40 +109,58 @@ public class ChatWebSocketHandler {
// 接收文本消息 // 接收文本消息
@OnMessage @OnMessage
public void onTextMessage(Session session, String message,@PathParam("clientId") String clientId) { public void onTextMessage(Session session, String message, @PathParam("clientId") String clientId) {
System.out.println("接收到文本消息: " + message); System.out.println("接收到文本消息: " + message);
try { try {
//处理文本结果 //处理文本结果
if(StrUtil.isNotEmpty(message)){ if (StrUtil.isNotEmpty(message)) {
Map<String,String> mapResult = JSONUtil.toBean(JSONUtil.parseObj(message),Map.class); Map<String, String> mapResult = JSONUtil.toBean(JSONUtil.parseObj(message), Map.class);
String resultFlag = mapResult.get("msg"); String resultFlag = mapResult.get("msg");
if("done".equals(resultFlag)){ if ("done".equals(resultFlag)) {
log.info("1、开始处理时间:{}",System.currentTimeMillis()/1000); log.info("1、开始处理时间:{}", System.currentTimeMillis() / 1000);
// //开始合并语音流 //开始合并语音流
//发送消息 //发送消息
WebSocket webSocket = cacheWebSocket.get(clientId); WebSocket webSocket = cacheWebSocket.get(session.getId());
if(webSocket != null){ if (webSocket != null) {
webSocket.send("{\"type\": \"input_audio_buffer.commit\"}");
webSocket.send("{\"type\": \"response.create\"}");
} }
webSocket.send("{\"type\": \"input_audio_buffer.commit\"}"); String startFlag = cacheReplyFlag.get(session.getId());
webSocket.send("{\"type\": \"response.create\"}");
//语音结束,开始进行回答解析 //语音结束,开始进行回答解析
String cacheResultText = cacheClientTts.get(clientId); String cacheResultText = cacheClientTts.get(clientId);
log.info("返回的结果为:{}",cacheResultText); log.info("返回的结果为:{}", cacheResultText);
if(StrUtil.isEmpty(cacheResultText)){ if (StrUtil.isEmpty(cacheResultText)) {
cacheResultText = "Hello , How are you?"; cacheResultText = "Hi.";
} }
log.info("1、开始进行AI回答时间:{}",System.currentTimeMillis()/1000); if("YES".equals(startFlag)) {
//自我介绍结束后马上返回一个Good
//发送初始化面试官语音流
String openingPathUrl = RuoYiConfig.getProfile() + VOICE_SYSTEM_DIR + "good.wav";
try {
//文件转换成文件流
ByteBuffer outByteBuffer = convertFileToByteBuffer(openingPathUrl);
//发送文件流数据
session.getBasicRemote().sendBinary(outByteBuffer);
// 发送响应确认
log.info("初始化返回面试官语音信息:{}", System.currentTimeMillis() / 1000);
} catch (IOException e) {
e.printStackTrace();
}
cacheResultText = "你是面试官，根据Construction Labourer候选人回答生成追问。只要一个问题";
}
//获取完问答数据,直接清空缓存数据
cacheClientTts.put(clientId,"");
cacheReplyFlag.put(session.getId(),"");
log.info("1、开始进行AI回答时间:{}", System.currentTimeMillis() / 1000);
//把提问的文字发送给CPT(流式处理) //把提问的文字发送给CPT(流式处理)
OpenAiStreamClient aiStreamClient = SpringUtils.getBean(OpenAiStreamClient.class); OpenAiStreamClient aiStreamClient = SpringUtils.getBean(OpenAiStreamClient.class);
aiStreamClient.streamChat(cacheResultText, new OpenAiStreamListenerService() { aiStreamClient.streamChat(cacheResultText, new OpenAiStreamListenerService() {
@Override @Override
public void onMessage(String content) { public void onMessage(String content) {
log.info("返回AI结果：{}",content); log.info("返回AI结果：{}", content);
// 实时输出内容 // 实时输出内容
//开始进行语音输出-流式持续输出 //开始进行语音输出-流式持续输出
log.info("2、开始进行AI回答时间:{}",System.currentTimeMillis()/1000); log.info("2、开始进行AI回答时间:{}", System.currentTimeMillis() / 1000);
//把结果文字转成语音文件 //把结果文字转成语音文件
//生成文件 //生成文件
//生成唯一文件名 //生成唯一文件名
@@ -143,7 +168,7 @@ public class ChatWebSocketHandler {
String resultPathUrl = RuoYiConfig.getProfile() + VOICE_STORAGE_RESULT_DIR + resultFileName; String resultPathUrl = RuoYiConfig.getProfile() + VOICE_STORAGE_RESULT_DIR + resultFileName;
ElevenLabsClient elevenLabsClient = SpringUtils.getBean(ElevenLabsClient.class); ElevenLabsClient elevenLabsClient = SpringUtils.getBean(ElevenLabsClient.class);
elevenLabsClient.handleTextToVoice(content, resultPathUrl); elevenLabsClient.handleTextToVoice(content, resultPathUrl);
log.info("3、开始进行AI回答时间:{}",System.currentTimeMillis()/1000); log.info("3、开始进行AI回答时间:{}", System.currentTimeMillis() / 1000);
//持续返回数据流给客户端 //持续返回数据流给客户端
try { try {
//文件转换成文件流 //文件转换成文件流
@@ -151,7 +176,7 @@ public class ChatWebSocketHandler {
//发送文件流数据 //发送文件流数据
session.getBasicRemote().sendBinary(outByteBuffer); session.getBasicRemote().sendBinary(outByteBuffer);
// 发送响应确认 // 发送响应确认
log.info("4、开始进行AI回答时间:{}",System.currentTimeMillis()/1000); log.info("4、开始进行AI回答时间:{}", System.currentTimeMillis() / 1000);
} catch (IOException e) { } catch (IOException e) {
e.printStackTrace(); e.printStackTrace();
} }
@@ -160,14 +185,14 @@ public class ChatWebSocketHandler {
@Override @Override
public void onComplete() { public void onComplete() {
try { try {
Map<String,String> resultEntity = new HashMap<>(); Map<String, String> resultEntity = new HashMap<>();
resultEntity.put("msg","done"); resultEntity.put("msg", "done");
//发送通知告诉客户端已经回答结束了 //发送通知告诉客户端已经回答结束了
session.getBasicRemote().sendText(JSONUtil.toJsonStr(resultEntity)); session.getBasicRemote().sendText(JSONUtil.toJsonStr(resultEntity));
} catch (Exception e) { } catch (Exception e) {
throw new RuntimeException(e); throw new RuntimeException(e);
} }
log.info("5、结束进行AI回答时间:{}",System.currentTimeMillis()/1000); log.info("5、结束进行AI回答时间:{}", System.currentTimeMillis() / 1000);
} }
@Override @Override
@@ -185,30 +210,30 @@ public class ChatWebSocketHandler {
// 接收二进制消息（流数据） // 接收二进制消息（流数据）
@OnMessage @OnMessage
public void onBinaryMessage(Session session, @PathParam("clientId") String clientId, ByteBuffer byteBuffer) { public void onBinaryMessage(Session session, @PathParam("clientId") String clientId, ByteBuffer byteBuffer) {
log.info("1、开始接收数据流时间:{}",System.currentTimeMillis()/1000); log.info("1、开始接收数据流时间:{}", System.currentTimeMillis() / 1000);
log.info("客户端ID为:{}", clientId); log.info("客户端ID为:{}", clientId);
// 处理二进制流数据 // 处理二进制流数据
byte[] bytes = new byte[byteBuffer.remaining()]; byte[] bytes = new byte[byteBuffer.remaining()];
//从缓冲区中读取数据并存储到指定的字节数组中 //从缓冲区中读取数据并存储到指定的字节数组中
byteBuffer.get(bytes); byteBuffer.get(bytes);
log.info("2、开始接收数据流时间:{}",System.currentTimeMillis()/1000); log.info("2、开始接收数据流时间:{}", System.currentTimeMillis() / 1000);
// 生成唯一文件名 // 生成唯一文件名
String fileName = clientId + "_" + System.currentTimeMillis() + ".wav"; String fileName = clientId + "_" + System.currentTimeMillis() + ".wav";
String pathUrl = RuoYiConfig.getProfile()+VOICE_STORAGE_DIR + fileName; String pathUrl = RuoYiConfig.getProfile() + VOICE_STORAGE_DIR + fileName;
log.info("文件路径为:{}", pathUrl); log.info("文件路径为:{}", pathUrl);
log.info("3、开始接收数据流时间:{}",System.currentTimeMillis()/1000); log.info("3、开始接收数据流时间:{}", System.currentTimeMillis() / 1000);
try{ try {
log.info("文件流的大小为:{}",bytes.length); log.info("文件流的大小为:{}", bytes.length);
saveAsWebM(bytes,pathUrl); saveAsWebM(bytes, pathUrl);
//接收到数据流后直接就进行SST处理 //接收到数据流后直接就进行SST处理
//语音格式转换 //语音格式转换
String fileOutName = clientId + "_" + System.currentTimeMillis() + ".pcm"; String fileOutName = clientId + "_" + System.currentTimeMillis() + ".pcm";
String pathOutUrl = RuoYiConfig.getProfile()+VOICE_STORAGE_DIR + fileOutName; String pathOutUrl = RuoYiConfig.getProfile() + VOICE_STORAGE_DIR + fileOutName;
handleAudioToPCM(pathUrl,pathOutUrl); handleAudioToPCM(pathUrl, pathOutUrl);
//发送消息 //发送消息
WebSocket webSocket = cacheWebSocket.get(clientId); WebSocket webSocket = cacheWebSocket.get(session.getId());
log.info("获取的socket对象为:{}",webSocket); log.info("获取的socket对象为:{}", webSocket);
if(webSocket != null){ if (webSocket != null) {
// 1. 启动音频缓冲 // 1. 启动音频缓冲
// webSocket.send("{\"type\": \"input_audio_buffer.start\"}"); // webSocket.send("{\"type\": \"input_audio_buffer.start\"}");
log.info("3.1 开始发送数据音频流啦"); log.info("3.1 开始发送数据音频流啦");
@@ -220,12 +245,12 @@ public class ChatWebSocketHandler {
String base64Audio = Base64.getEncoder().encodeToString(outBytes); String base64Audio = Base64.getEncoder().encodeToString(outBytes);
String message = "{ \"type\": \"input_audio_buffer.append\", \"audio\": \"" + base64Audio + "\" }"; String message = "{ \"type\": \"input_audio_buffer.append\", \"audio\": \"" + base64Audio + "\" }";
webSocket.send(message); webSocket.send(message);
log.info("4、开始接收数据流时间:{}",System.currentTimeMillis()/1000); log.info("4、开始接收数据流时间:{}", System.currentTimeMillis() / 1000);
// 3. 提交音频并请求转录 // 3. 提交音频并请求转录
// webSocket.send("{\"type\": \"input_audio_buffer.commit\"}"); // webSocket.send("{\"type\": \"input_audio_buffer.commit\"}");
// webSocket.send("{\"type\": \"response.create\"}"); // webSocket.send("{\"type\": \"response.create\"}");
} }
}catch (Exception e){ } catch (Exception e) {
e.printStackTrace(); e.printStackTrace();
} }
@@ -235,10 +260,10 @@ public class ChatWebSocketHandler {
@OnClose @OnClose
public void onClose(Session session, CloseReason reason) { public void onClose(Session session, CloseReason reason) {
System.out.println("WebSocket连接已关闭: " + session.getId() + ", 原因: " + reason.getReasonPhrase()); System.out.println("WebSocket连接已关闭: " + session.getId() + ", 原因: " + reason.getReasonPhrase());
// WebSocket webSocket = cacheWebSocket.get(clientId); WebSocket webSocket = cacheWebSocket.get(session.getId());
// if(webSocket != null){ if (webSocket != null) {
// webSocket.close(1000,null); webSocket.close(1000, null);
// } }
} }
// 发生错误时调用 // 发生错误时调用
@@ -305,10 +330,11 @@ public class ChatWebSocketHandler {
/** /**
* 创建STT WebSocket 客户端链接 * 创建STT WebSocket 客户端链接
*
* @param clientId 客户端ID * @param clientId 客户端ID
*/ */
private void createWhisperRealtimeSocket(String clientId){ private void createWhisperRealtimeSocket(String clientId) {
try{ try {
OkHttpClient client = new OkHttpClient(); OkHttpClient client = new OkHttpClient();
// 设置 WebSocket 请求 // 设置 WebSocket 请求
Request request = new Request.Builder() Request request = new Request.Builder()
@@ -343,25 +369,25 @@ public class ChatWebSocketHandler {
// webSocket.send("{\"type\": \"input_audio_buffer.start\"}"); // webSocket.send("{\"type\": \"input_audio_buffer.start\"}");
//存储客户端webSocket对象,对数据进行隔离处理 //存储客户端webSocket对象,对数据进行隔离处理
cacheWebSocket.put(clientId,webSocket); cacheWebSocket.put(clientId, webSocket);
} }
@Override @Override
public void onMessage(WebSocket webSocket, String text) { public void onMessage(WebSocket webSocket, String text) {
System.out.println("📩 收到转录结果: " + text); System.out.println("📩 收到转录结果: " + text);
//对数据进行解析 //对数据进行解析
if(StrUtil.isNotEmpty(text)){ if (StrUtil.isNotEmpty(text)) {
Map<String,String> mapResultData = JSONUtil.toBean(text,Map.class); Map<String, String> mapResultData = JSONUtil.toBean(text, Map.class);
if("conversation.item.input_audio_transcription.delta".equals(mapResultData.get("type"))){ if ("conversation.item.input_audio_transcription.delta".equals(mapResultData.get("type"))) {
String resultText = mapResultData.get("delta"); String resultText = mapResultData.get("delta");
//进行客户端文本数据存储 //进行客户端文本数据存储
String cacheString = cacheClientTts.get(clientId); String cacheString = cacheClientTts.get(clientId);
if(StrUtil.isNotEmpty(cacheString)){ if (StrUtil.isNotEmpty(cacheString)) {
cacheString = cacheString+resultText; cacheString = cacheString + resultText;
}else { } else {
cacheString = resultText; cacheString = resultText;
} }
cacheClientTts.put(clientId,cacheString); cacheClientTts.put(clientId, cacheString);
} }
} }
} }
@@ -379,17 +405,18 @@ public class ChatWebSocketHandler {
// latch.countDown(); // latch.countDown();
} }
}); });
}catch (Exception e){ } catch (Exception e) {
e.printStackTrace(); e.printStackTrace();
} }
} }
/** /**
* 语音流文件格式转换 * 语音流文件格式转换
*
* @param pathUrl * @param pathUrl
* @param outPathUrl * @param outPathUrl
*/ */
private void handleAudioToPCM(String pathUrl,String outPathUrl){ private void handleAudioToPCM(String pathUrl, String outPathUrl) {
File inputFile = new File(pathUrl); // 输入音频文件 File inputFile = new File(pathUrl); // 输入音频文件
File outputFile = new File(outPathUrl); // 输出PCM格式文件 File outputFile = new File(outPathUrl); // 输出PCM格式文件
try { try {

2
vetti-admin/src/main/resources/application-druid.yml
View File

@@ -165,7 +165,7 @@ chatGpt:
apiKey: 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 apiKey: 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
apiUrl: https://api.openai.com/v1/chat/completions apiUrl: https://api.openai.com/v1/chat/completions
model: ft:gpt-3.5-turbo-0125:vetti:construction-labourer-test:CTIvLD5n model: ft:gpt-3.5-turbo-0125:vetti:construction-labourer-test:CTIvLD5n
role: user role: system
http: http:

2
vetti-admin/target/classes/application-druid.yml
View File

@@ -165,7 +165,7 @@ chatGpt:
apiKey: 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 apiKey: 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
apiUrl: https://api.openai.com/v1/chat/completions apiUrl: https://api.openai.com/v1/chat/completions
model: ft:gpt-3.5-turbo-0125:vetti:construction-labourer-test:CTIvLD5n model: ft:gpt-3.5-turbo-0125:vetti:construction-labourer-test:CTIvLD5n
role: user role: system
http: http: