sxc/Vetti-Service-new
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

AI 提问追问环境demo,以及评分处理

Browse Source
This commit is contained in:
wangxiangshun
2025-10-27 21:11:45 +08:00
parent fb05e6ca37
commit 14b803609e
2 changed files with 53 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files

63
vetti-admin/src/main/java/com/vetti/socket/ChatWebSocketHandler.java
View File

@@ -133,7 +133,6 @@ public class ChatWebSocketHandler {
Map<String, String> mapResult = JSONUtil.toBean(JSONUtil.parseObj(message), Map.class);
String resultFlag = mapResult.get("msg");
if ("done".equals(resultFlag)) {
log.info("1、开始处理时间:{}", System.currentTimeMillis() / 1000);
//开始合并语音流
//发送消息
WebSocket webSocket = cacheWebSocket.get(session.getId());
@@ -198,7 +197,6 @@ public class ChatWebSocketHandler {
//获取完问答数据,直接清空缓存数据
cacheClientTts.put(clientId,"");
cacheReplyFlag.put(session.getId(),"");
log.info("1、开始进行AI回答时间:{}", System.currentTimeMillis() / 1000);
//把提问的文字发送给CPT(流式处理)
OpenAiStreamClient aiStreamClient = SpringUtils.getBean(OpenAiStreamClient.class);
log.info("AI提示词为:{}",promptJson);
@@ -215,7 +213,6 @@ public class ChatWebSocketHandler {
cacheQuestionResult.put(session.getId(),questionResult);
// 实时输出内容
//开始进行语音输出-流式持续输出
log.info("2、开始进行AI回答时间:{}", System.currentTimeMillis() / 1000);
//把结果文字转成语音文件
//生成文件
//生成唯一文件名
@@ -223,7 +220,6 @@ public class ChatWebSocketHandler {
String resultPathUrl = RuoYiConfig.getProfile() + VOICE_STORAGE_RESULT_DIR + resultFileName;
ElevenLabsClient elevenLabsClient = SpringUtils.getBean(ElevenLabsClient.class);
elevenLabsClient.handleTextToVoice(content, resultPathUrl);
log.info("3、开始进行AI回答时间:{}", System.currentTimeMillis() / 1000);
//持续返回数据流给客户端
try {
//文件转换成文件流
@@ -231,7 +227,6 @@ public class ChatWebSocketHandler {
//发送文件流数据
session.getBasicRemote().sendBinary(outByteBuffer);
// 发送响应确认
log.info("4、开始进行AI回答时间:{}", System.currentTimeMillis() / 1000);
} catch (IOException e) {
e.printStackTrace();
}
@@ -262,9 +257,59 @@ public class ChatWebSocketHandler {
} catch (Exception e) {
throw new RuntimeException(e);
}
log.info("5、结束进行AI回答时间:{}", System.currentTimeMillis() / 1000);
}
@Override
public void onError(Throwable throwable) {
throwable.printStackTrace();
}
});
}else if("end".equals(resultFlag)){
//发送面试官结束语音流
String openingPathUrl = RuoYiConfig.getProfile() + VOICE_SYSTEM_DIR + "end.wav";
try {
//文件转换成文件流
ByteBuffer outByteBuffer = convertFileToByteBuffer(openingPathUrl);
//发送文件流数据
session.getBasicRemote().sendBinary(outByteBuffer);
// 发送响应确认
log.info("结束返回面试官语音信息:{}", System.currentTimeMillis() / 1000);
} catch (IOException e) {
e.printStackTrace();
}
//返回文本评分
//处理模型提问逻辑
String promptJson = "";
//获取缓存记录
String msgMapData = cacheMsgMapData.get(session.getId());
if(StrUtil.isNotEmpty(msgMapData)){
List<Map> list = JSONUtil.toList(msgMapData, Map.class);
//获取最后一条数据记录
Map<String,String> mapEntity = list.get(0);
//更新问题记录
mapEntity.put("role","system");
mapEntity.put("content","你是建筑行业面试专家，对Construction Labourer候选人回答进行1-5分评分。");
promptJson = JSONUtil.toJsonStr(list);
//结束回答要清空问答数据
cacheMsgMapData.put(session.getId(),"");
}
log.info("结束AI提示词为:{}",promptJson);
OpenAiStreamClient aiStreamClient = SpringUtils.getBean(OpenAiStreamClient.class);
aiStreamClient.streamChat(promptJson, new OpenAiStreamListenerService() {
@Override
public void onMessage(String content) {
log.info("返回AI结果：{}", content);
try {
//发送文件流数据
session.getBasicRemote().sendText(content);
}catch (Exception e){
e.printStackTrace();
}
}
@Override
public void onComplete() {
}
@Override
public void onError(Throwable throwable) {
throwable.printStackTrace();
@@ -280,20 +325,16 @@ public class ChatWebSocketHandler {
// 接收二进制消息（流数据）
@OnMessage
public void onBinaryMessage(Session session, @PathParam("clientId") String clientId, ByteBuffer byteBuffer) {
log.info("1、开始接收数据流时间:{}", System.currentTimeMillis() / 1000);
log.info("客户端ID为:{}", clientId);
// 处理二进制流数据
byte[] bytes = new byte[byteBuffer.remaining()];
//从缓冲区中读取数据并存储到指定的字节数组中
byteBuffer.get(bytes);
log.info("2、开始接收数据流时间:{}", System.currentTimeMillis() / 1000);
// 生成唯一文件名
String fileName = clientId + "_" + System.currentTimeMillis() + ".wav";
String pathUrl = RuoYiConfig.getProfile() + VOICE_STORAGE_DIR + fileName;
log.info("文件路径为:{}", pathUrl);
log.info("3、开始接收数据流时间:{}", System.currentTimeMillis() / 1000);
try {
log.info("文件流的大小为:{}", bytes.length);
saveAsWebM(bytes, pathUrl);
//接收到数据流后直接就进行SST处理
//语音格式转换
@@ -306,7 +347,6 @@ public class ChatWebSocketHandler {
if (webSocket != null) {
// 1. 启动音频缓冲
// webSocket.send("{\"type\": \"input_audio_buffer.start\"}");
log.info("3.1 开始发送数据音频流啦");
File outputFile = new File(pathOutUrl); // 输出PCM格式文件
ByteBuffer buffer = ByteBuffer.wrap(FileUtils.readFileToByteArray(outputFile));
byte[] outBytes = new byte[buffer.remaining()];
@@ -315,7 +355,6 @@ public class ChatWebSocketHandler {
String base64Audio = Base64.getEncoder().encodeToString(outBytes);
String message = "{ \"type\": \"input_audio_buffer.append\", \"audio\": \"" + base64Audio + "\" }";
webSocket.send(message);
log.info("4、开始接收数据流时间:{}", System.currentTimeMillis() / 1000);
// 3. 提交音频并请求转录
// webSocket.send("{\"type\": \"input_audio_buffer.commit\"}");
// webSocket.send("{\"type\": \"response.create\"}");

6
vetti-admin/src/main/java/com/vetti/web/controller/ai/AiCommonController.java
View File

@@ -40,10 +40,8 @@ public class AiCommonController extends BaseController
public AjaxResult handleTextToVice()
{
//你好,我是本次的面试官Vetti,请点击开始按钮后,做一段自我介绍.
elevenLabsClient.handleTextToVoice("Hello, I am Vetti, the interviewer for this interview. Please click the start button and give a self introduction","/Users/wangxiangshun/Desktop/临时文件/opening2.wav");
elevenLabsClient.handleTextToVoice("Good","/Users/wangxiangshun/Desktop/临时文件/good.wav");
//本轮面试结束,谢谢您的配合,面试结果将稍后通知
elevenLabsClient.handleTextToVoice("This round of interview is over. Thank you for your cooperation. The interview results will be notified later","/Users/wangxiangshun/Desktop/临时文件/end.wav");
return success();
}