sxc/Vetti-Service-new
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

自动面试流程优化以及修改

Browse Source
This commit is contained in:
wangxiangshun
2025-11-06 00:41:21 +08:00
parent 6f2689fd1e
commit e1db50003e
10 changed files with 1017 additions and 366 deletions
Download Patch File Download Diff File Expand all files Collapse all files

692
vetti-admin/src/main/java/com/vetti/socket/ChatWebSocketHandler.java
View File

@@ -1,7 +1,8 @@
package com.vetti.socket;
import cn.hutool.core.collection.CollectionUtil;
import cn.hutool.core.date.DateUtil;
import cn.hutool.core.util.StrUtil;
import cn.hutool.json.JSONObject;
import cn.hutool.json.JSONUtil;
import com.vetti.common.ai.elevenLabs.ElevenLabsClient;
import com.vetti.common.ai.gpt.ChatGPTClient;
@@ -9,20 +10,19 @@ import com.vetti.common.ai.gpt.OpenAiStreamClient;
import com.vetti.common.ai.gpt.service.OpenAiStreamListenerService;
import com.vetti.common.config.RuoYiConfig;
import com.vetti.common.utils.spring.SpringUtils;
import com.vetti.hotake.domain.HotakeProblemBaseInfo;
import com.vetti.hotake.service.IHotakeProblemBaseInfoService;
import io.swagger.models.auth.In;
import lombok.extern.slf4j.Slf4j;
import okhttp3.*;
import org.apache.commons.io.FileUtils;
import org.springframework.stereotype.Component;
import javax.sound.sampled.AudioFormat;
import javax.sound.sampled.AudioInputStream;
import javax.sound.sampled.AudioSystem;
import javax.websocket.*;
import javax.websocket.server.PathParam;
import javax.websocket.server.ServerEndpoint;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.math.BigDecimal;
import java.nio.ByteBuffer;
import java.util.HashMap;
import java.util.LinkedList;
@@ -38,44 +38,41 @@ import java.util.concurrent.ConcurrentHashMap;
@Component
public class ChatWebSocketHandler {
// @Value("${whisper.apiUrl}")
private String API_URL = "wss://api.openai.com/v1/realtime?intent=transcription";
/**
* 追问问题标记
*/
private final String QUESTION_FLAG = "FOLLOW-UP:";
// @Value("${whisper.model}")
private String MODEL = "gpt-4o-mini-transcribe";
/**
* 评分标记
*/
private final String SCORE_FLAG = "Score:";
// @Value("${whisper.apiKey}")
private String apiKey = "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";
// @Value("${whisper.language}")
private String language = "en";
/**
* 缓存客户端流式解析的语音文本数据
*/
private final Map<String, String> cacheClientTts = new ConcurrentHashMap<>();
/**
* 缓存客户端调用OpenAi中的websocket-STT 流式传输数据
*/
private final Map<String, WebSocket> cacheWebSocket = new ConcurrentHashMap<>();
/**
* 缓存客户端,标记是否是自我介绍后的初次问答
*/
private final Map<String,String> cacheReplyFlag = new ConcurrentHashMap<>();
private final Map<String, String> cacheReplyFlag = new ConcurrentHashMap<>();
/**
* 缓存客户端,面试回答信息
*/
private final Map<String,String> cacheMsgMapData = new ConcurrentHashMap<>();
private final Map<String, String> cacheMsgMapData = new ConcurrentHashMap<>();
/**
* 缓存客户端,AI提问的问题结果信息
*/
private final Map<String,String> cacheQuestionResult = new ConcurrentHashMap<>();
private final Map<String, String> cacheQuestionResult = new ConcurrentHashMap<>();
// private final Map<String,String> cacheOpeningResult = new ConcurrentHashMap<>();
/**
* 缓存客户端,得分结果记录
*/
private final Map<String, Map<String, Integer>> cacheScoreResult = new ConcurrentHashMap<>();
// 语音文件保存目录
private static final String VOICE_STORAGE_DIR = "/voice_files/";
@@ -105,184 +102,183 @@ public class ChatWebSocketHandler {
log.info("WebSocket 链接已建立:{}", clientId);
log.info("WebSocket session 链接已建立:{}", session.getId());
cacheClientTts.put(clientId, new String());
//初始化STT流式语音转换文本的socket链接
// createWhisperRealtimeSocket(session.getId());
//是初次自我介绍后的问答环节
cacheReplyFlag.put(session.getId(),"YES");
cacheReplyFlag.put(session.getId(), "YES");
//初始化面试回答数据记录
cacheMsgMapData.put(session.getId(),"");
cacheMsgMapData.put(session.getId(), "");
//初始化面试问题
cacheQuestionResult.put(session.getId(),"");
//开场白控制
// String flag = cacheOpeningResult.get(clientId);
// if(StrUtil.isEmpty(flag)){
//发送初始化面试官语音流
String openingPathUrl = RuoYiConfig.getProfile() + VOICE_SYSTEM_DIR + "opening.wav";
try {
//文件转换成文件流
ByteBuffer outByteBuffer = convertFileToByteBuffer(openingPathUrl);
//发送文件流数据
session.getBasicRemote().sendBinary(outByteBuffer);
// cacheOpeningResult.put(clientId,"YES");
// 发送响应确认
log.info("初始化返回面试官语音信息:{}", System.currentTimeMillis() / 1000);
} catch (IOException e) {
e.printStackTrace();
}
// }
cacheQuestionResult.put(session.getId(), "");
//初始化得分结果记录
Map<String, Integer> scoreResultData = new HashMap<>();
scoreResultData.put("0-1", 0);
scoreResultData.put("4-5", 0);
scoreResultData.put("2-3", 0);
scoreResultData.put("2-5", 0);
cacheScoreResult.put(session.getId(), scoreResultData);
//发送初始化面试官语音流
String openingPathUrl = RuoYiConfig.getProfile() + VOICE_SYSTEM_DIR + "opening.wav";
sendVoiceBuffer(openingPathUrl, session);
}
// 接收文本消息
/**
* 接收文本消息
*
* @param session 客户端会话
* @param message 消息
* 如:
* {
* "type": "start | done | end",
* "content": "内容"
* }
* @param clientId 用户ID
*/
@OnMessage
public void onTextMessage(Session session, String message, @PathParam("clientId") String clientId) {
System.out.println("接收到文本消息: " + message);
log.info("我是接收文本消息:{}", message);
try {
// {
// "type": "start | done | end",
// "content": "内容"
// }
//处理文本结果
if (StrUtil.isNotEmpty(message)) {
Map<String, String> mapResult = JSONUtil.toBean(JSONUtil.parseObj(message), Map.class);
String resultFlag = mapResult.get("type");
if ("done".equals(resultFlag)) {
//开始合并语音流
//发送消息
// WebSocket webSocket = cacheWebSocket.get(session.getId());
// if (webSocket != null) {
// webSocket.send("{\"type\": \"input_audio_buffer.commit\"}");
// webSocket.send("{\"type\": \"response.create\"}");
// }
String startFlag = cacheReplyFlag.get(session.getId());
//语音结束,开始进行回答解析
log.info("开始文本处理,客户端ID为:{}",clientId);
// String cacheResultText = cacheClientTts.get(session.getId());
log.info("开始文本处理,客户端ID为:{}", clientId);
String cacheResultText = mapResult.get("content");
log.info("开始文本处理,面试者回答信息为:{}", cacheResultText);
if (StrUtil.isEmpty(cacheResultText)) {
cacheResultText = "I first check the forklift's logbook for recent issues, inspect tires and brakes, verify the load capacity matches today's task, and confirm my licence is current—all per SWMS requirements.";
cacheResultText = "";
}
String promptJson = "";
if("YES".equals(startFlag)) {
//自我介绍结束后马上返回一个Good
//发送初始化面试官语音流
String openingPathUrl = RuoYiConfig.getProfile() + VOICE_SYSTEM_DIR + "good.wav";
try {
//文件转换成文件流
ByteBuffer outByteBuffer = convertFileToByteBuffer(openingPathUrl);
//发送文件流数据
session.getBasicRemote().sendBinary(outByteBuffer);
// 发送响应确认
log.info("初始化返回面试官语音信息:{}", System.currentTimeMillis() / 1000);
} catch (IOException e) {
e.printStackTrace();
}
Map<String,String> mapEntity = new HashMap<>();
mapEntity.put("role","system");
mapEntity.put("content","You are an interviewer. Generate follow-up questions based on Construction Labourer candidate responses.Only return one question and do not repeat the previously returned questions \\n MPORTANT: Do not ask the same question again if the answer is incorrect");
List<Map<String,String>> list = new LinkedList();
list.add(mapEntity);
promptJson = JSONUtil.toJsonStr(list);
//记录缓存中
cacheMsgMapData.put(session.getId(),promptJson);
}else{
//这是初次处理的逻辑
if ("YES".equals(startFlag)) {
//初始化-不走大模型-直接对候选人进行提问
initializationQuestion(clientId, session);
//发送完第一次消息后,直接删除标记,开始进行正常的面试问答流程
cacheReplyFlag.put(session.getId(), "");
} else {
//开始根据面试者回答的问题,进行追问回答
// {
// role: "system",
// content: "你是面试官，根据Construction Labourer候选人回答生成追问。"
// },
// {
// role: "user",
// content: `问题：${question}\n候选人回答：${answer}`
// }
//获取面试者回答信息
//获取缓存记录
String promptJson = "";
String msgMapData = cacheMsgMapData.get(session.getId());
if(StrUtil.isNotEmpty(msgMapData)){
if (StrUtil.isNotEmpty(msgMapData)) {
List<Map> list = JSONUtil.toList(msgMapData, Map.class);
//获取最后一条数据记录
Map<String,String> mapEntity = list.get(list.size()-1);
Map<String, String> mapEntity = list.get(list.size() - 1);
//更新问题记录
String content = mapEntity.get("content");
mapEntity.put("content", StrUtil.format(content, cacheResultText));
promptJson = JSONUtil.toJsonStr(list);
cacheMsgMapData.put(session.getId(),promptJson);
cacheMsgMapData.put(session.getId(), promptJson);
}
}
//获取完问答数据,直接清空缓存数据
cacheClientTts.put(session.getId(),"");
cacheReplyFlag.put(session.getId(),"");
//把提问的文字发送给CPT(流式处理)
OpenAiStreamClient aiStreamClient = SpringUtils.getBean(OpenAiStreamClient.class);
log.info("AI提示词为:{}",promptJson);
aiStreamClient.streamChat(promptJson, new OpenAiStreamListenerService() {
@Override
public void onMessage(String content) {
log.info("返回AI结果：{}", content);
if(StrUtil.isNotEmpty(content)){
String questionResult = cacheQuestionResult.get(session.getId());
if(StrUtil.isEmpty(questionResult)){
questionResult = content;
}else{
questionResult = questionResult + content;
//开始使用模型进行追问
//把提问的文字发送给CPT(流式处理)
OpenAiStreamClient aiStreamClient = SpringUtils.getBean(OpenAiStreamClient.class);
log.info("AI提示词为:{}", promptJson);
//Score: 返回的是评分,后面只会跟着一个
//FOLLOW-UP: 返回的是问题,后面的每一行都是问题
aiStreamClient.streamChat(promptJson, new OpenAiStreamListenerService() {
String isScore = "0";
//是否结束面试
Boolean flag = true;
String resultText = "";
String resultEvaluate = "";
//是否遇到问题记录
Boolean isFlow = false;
@Override
public void onMessage(String content) {
log.info("返回AI结果：{}", content.replaceAll("\n", ""));
if (StrUtil.isEmpty(resultText)) {
resultText = content;
} else {
resultText = resultText + content;
}
cacheQuestionResult.put(session.getId(),questionResult);
// 实时输出内容
//开始进行语音输出-流式持续输出
//把结果文字转成语音文件
//生成文件
//生成唯一文件名
String resultFileName = clientId + "_" + System.currentTimeMillis() + ".wav";
String resultPathUrl = RuoYiConfig.getProfile() + VOICE_STORAGE_RESULT_DIR + resultFileName;
ElevenLabsClient elevenLabsClient = SpringUtils.getBean(ElevenLabsClient.class);
elevenLabsClient.handleTextToVoice(content, resultPathUrl);
//持续返回数据流给客户端
String contentData = content.replaceAll("\n", "");
//记录获取的分数,并且验证分数是否完成
//获取评分
if (contentData.contains(SCORE_FLAG)) {
isScore = "1";
}
if ("1".equals(isScore)) {
//获取的是评分,并且记录评分
flag = handleScoreRecord(content, session);
}
if (contentData.contains(QUESTION_FLAG)) {
isFlow = true;
}
if (flag) {
//返回是追问的问题
if (contentData.contains(QUESTION_FLAG)) {
//获取的是追问的问题
contentData = contentData.replace(QUESTION_FLAG, "");
if (StrUtil.isNotEmpty(contentData)) {
//对问题进行数据缓存
cacheQuestionResult.put(session.getId(), contentData);
//开始进行语音输出-流式持续输出
sendTTSBuffer(clientId, contentData, session);
// 实时输出内容
try {
//把文本也给前端返回去
Map<String, String> dataText = new HashMap<>();
dataText.put("type", "question");
dataText.put("content", contentData);
session.getBasicRemote().sendText(JSONUtil.toJsonStr(dataText));
} catch (Exception e) {
e.printStackTrace();
}
}
}
if(!isFlow){
if (StrUtil.isEmpty(resultEvaluate)) {
resultEvaluate = content;
} else {
resultEvaluate = resultEvaluate + content;
}
}
}
}
@Override
public void onComplete() {
try {
//文件转换成文件流
ByteBuffer outByteBuffer = convertFileToByteBuffer(resultPathUrl);
//发送文件流数据
session.getBasicRemote().sendBinary(outByteBuffer);
// 发送响应确认
} catch (IOException e) {
e.printStackTrace();
//开始往缓存中记录提问的问题
String questionResult = cacheQuestionResult.get(session.getId());
if(StrUtil.isNotEmpty(questionResult)){
//获取缓存记录
String msgMapData = cacheMsgMapData.get(session.getId());
if (StrUtil.isNotEmpty(msgMapData)) {
List<Map> list = JSONUtil.toList(msgMapData, Map.class);
Map<String, String> mapEntity = new HashMap<>();
mapEntity.put("role", "user");
mapEntity.put("content", "Question：" + questionResult + "\\nCandidate Answer：{}");
list.add(mapEntity);
cacheMsgMapData.put(session.getId(), JSONUtil.toJsonStr(list));
}
}
//清空问题
cacheQuestionResult.put(session.getId(), "");
if(!flag || !isFlow){
//理解结束面试
//发送面试结束的通知已经评分
handleInterviewEnd(session, resultEvaluate);
}
} catch (Exception e) {
throw new RuntimeException(e);
}
}
}
@Override
public void onComplete() {
try {
//开始往缓存中记录提问的问题
String questionResult = cacheQuestionResult.get(session.getId());
//获取缓存记录
String msgMapData = cacheMsgMapData.get(session.getId());
if(StrUtil.isNotEmpty(msgMapData)){
List<Map> list = JSONUtil.toList(msgMapData, Map.class);
Map<String,String> mapEntity = new HashMap<>();
mapEntity.put("role","user");
mapEntity.put("content","Question："+questionResult+"\\nCandidate Answer：{}");
list.add(mapEntity);
cacheMsgMapData.put(session.getId(),JSONUtil.toJsonStr(list));
}
//清空问题
cacheQuestionResult.put(session.getId(),"");
Map<String, String> resultEntity = new HashMap<>();
resultEntity.put("msg", "done");
//发送通知告诉客户端已经回答结束了
session.getBasicRemote().sendText(JSONUtil.toJsonStr(resultEntity));
} catch (Exception e) {
throw new RuntimeException(e);
@Override
public void onError(Throwable throwable) {
throwable.printStackTrace();
}
}
});
}
} else if ("end".equals(resultFlag)) {
//暂时的业务逻辑
@Override
public void onError(Throwable throwable) {
throwable.printStackTrace();
}
});
}else if("end".equals(resultFlag)){
// cacheOpeningResult.put(clientId,"");
//发送面试官结束语音流
String openingPathUrl = RuoYiConfig.getProfile() + VOICE_SYSTEM_DIR + "end.wav";
try {
@@ -300,23 +296,23 @@ public class ChatWebSocketHandler {
String promptJson = "";
//获取缓存记录
String msgMapData = cacheMsgMapData.get(session.getId());
if(StrUtil.isNotEmpty(msgMapData)){
if (StrUtil.isNotEmpty(msgMapData)) {
List<Map> list = JSONUtil.toList(msgMapData, Map.class);
//获取最后一条数据记录
Map<String,String> mapEntity = list.get(0);
Map<String, String> mapEntity = list.get(0);
//更新问题记录
mapEntity.put("role","system");
mapEntity.put("content","You are a construction industry interview expert. Rate Construction Labourer candidate responses on a 1-5 scale. IMPORTANT: If the answer is completely unrelated, contains technical errors, system messages, or is nonsensical, give it a score of 0/5 and explain why it's invalid.");
mapEntity.put("role", "system");
mapEntity.put("content", "You are a construction industry interview expert. Rate Construction Labourer candidate responses on a 1-5 scale. IMPORTANT: If the answer is completely unrelated, contains technical errors, system messages, or is nonsensical, give it a score of 0/5 and explain why it's invalid.");
promptJson = JSONUtil.toJsonStr(list);
//结束回答要清空问答数据
cacheMsgMapData.put(session.getId(),"");
cacheMsgMapData.put(session.getId(), "");
}
log.info("结束AI提示词为:{}",promptJson);
log.info("结束AI提示词为:{}", promptJson);
ChatGPTClient gptClient = SpringUtils.getBean(ChatGPTClient.class);
String resultMsg = gptClient.handleAiChat(promptJson,"QA");
String resultMsg = gptClient.handleAiChat(promptJson, "QA");
Map<String, String> resultEntity = new HashMap<>();
resultEntity.put("msg", resultMsg);
resultEntity.put("dataType","score");
resultEntity.put("dataType", "score");
session.getBasicRemote().sendText(JSONUtil.toJsonStr(resultEntity));
}
}
@@ -328,54 +324,20 @@ public class ChatWebSocketHandler {
// 接收二进制消息（流数据）
@OnMessage
public void onBinaryMessage(Session session, @PathParam("clientId") String clientId, ByteBuffer byteBuffer) {
// log.info("客户端ID为:{}", clientId);
// // 处理二进制流数据
// byte[] bytes = new byte[byteBuffer.remaining()];
// //从缓冲区中读取数据并存储到指定的字节数组中
// byteBuffer.get(bytes);
// // 生成唯一文件名
// String fileName = clientId + "_" + System.currentTimeMillis() + ".wav";
// String pathUrl = RuoYiConfig.getProfile() + VOICE_STORAGE_DIR + fileName;
// log.info("文件路径为:{}", pathUrl);
// try {
// saveAsWebM(bytes, pathUrl);
// //接收到数据流后直接就进行SST处理
// //语音格式转换
// String fileOutName = clientId + "_" + System.currentTimeMillis() + ".pcm";
// String pathOutUrl = RuoYiConfig.getProfile() + VOICE_STORAGE_DIR + fileOutName;
// handleAudioToPCM(pathUrl, pathOutUrl);
// //发送消息
// WebSocket webSocket = cacheWebSocket.get(session.getId());
// log.info("获取的socket对象为:{}", webSocket);
// if (webSocket != null) {
//// 1. 启动音频缓冲
//// webSocket.send("{\"type\": \"input_audio_buffer.start\"}");
// File outputFile = new File(pathOutUrl); // 输出PCM格式文件
// ByteBuffer buffer = ByteBuffer.wrap(FileUtils.readFileToByteArray(outputFile));
// byte[] outBytes = new byte[buffer.remaining()];
// //从缓冲区中读取数据并存储到指定的字节数组中
// buffer.get(outBytes);
// String base64Audio = Base64.getEncoder().encodeToString(outBytes);
// String message = "{ \"type\": \"input_audio_buffer.append\", \"audio\": \"" + base64Audio + "\" }";
// webSocket.send(message);
// // 3. 提交音频并请求转录
//// webSocket.send("{\"type\": \"input_audio_buffer.commit\"}");
//// webSocket.send("{\"type\": \"response.create\"}");
// }
// } catch (Exception e) {
// e.printStackTrace();
// }
log.info("我是接受二进制流的-客户端ID为:{}", clientId);
}
// 连接关闭时调用
@OnClose
public void onClose(Session session, CloseReason reason) {
System.out.println("WebSocket连接已关闭: " + session.getId() + ", 原因: " + reason.getReasonPhrase());
// WebSocket webSocket = cacheWebSocket.get(session.getId());
// if (webSocket != null) {
// webSocket.close(1000, null);
// }
//链接关闭,清空内存
//是初次自我介绍后的问答环节
cacheReplyFlag.put(session.getId(), "");
//初始化面试回答数据记录
cacheMsgMapData.put(session.getId(), "");
//初始化面试问题
cacheQuestionResult.put(session.getId(), "");
}
// 发生错误时调用
@@ -385,45 +347,6 @@ public class ChatWebSocketHandler {
throwable.printStackTrace();
}
/**
* 将字节数组保存为WebM文件
*
* @param byteData 包含WebM数据的字节数组
* @param filePath 目标文件路径
* @return 操作是否成功
*/
private boolean saveAsWebM(byte[] byteData, String filePath) {
// 检查输入参数
if (byteData == null || byteData.length == 0) {
System.err.println("字节数组为空，无法生成WebM文件");
return false;
}
if (filePath == null || filePath.trim().isEmpty()) {
System.err.println("文件路径不能为空");
return false;
}
FileOutputStream fos = null;
try {
fos = new FileOutputStream(filePath);
fos.write(byteData);
fos.flush();
System.out.println("WebM文件已成功生成: " + filePath);
return true;
} catch (IOException e) {
System.err.println("写入文件时发生错误: " + e.getMessage());
e.printStackTrace();
} finally {
if (fos != null) {
try {
fos.close();
} catch (IOException e) {
e.printStackTrace();
}
}
}
return false;
}
/**
* File 转换成 ByteBuffer
*
@@ -441,130 +364,181 @@ public class ChatWebSocketHandler {
}
/**
* 创建STT WebSocket 客户端链接
* 发送语音流给前端
*
* @param clientId 客户端ID
* @param pathUrl 语音文件地址
* @param session 客户端会话
*/
private void createWhisperRealtimeSocket(String clientId) {
private void sendVoiceBuffer(String pathUrl, Session session) {
try {
OkHttpClient client = new OkHttpClient();
// 设置 WebSocket 请求
Request request = new Request.Builder()
.url(API_URL)
.addHeader("Authorization", "Bearer " + apiKey)
.addHeader("OpenAI-Beta", "realtime=v1")
.build();
client.newWebSocket(request, new WebSocketListener() {
@Override
public void onOpen(WebSocket webSocket, Response response) {
System.out.println("✅ WebSocket 连接成功");
//发送配置
JSONObject config = new JSONObject();
JSONObject sessionConfig = new JSONObject();
JSONObject transcription = new JSONObject();
JSONObject turnDetection = new JSONObject();
// 配置转录参数
transcription.put("model", MODEL);
transcription.put("language", language); // 中文
// 配置断句检测
turnDetection.put("type", "server_vad");
turnDetection.put("prefix_padding_ms", 300);
turnDetection.put("silence_duration_ms", 10);
// 组装完整配置
sessionConfig.put("input_audio_transcription", transcription);
sessionConfig.put("turn_detection", turnDetection);
config.put("type", "transcription_session.update");
config.put("session", sessionConfig);
webSocket.send(config.toString());
//文件转换成文件流
ByteBuffer outByteBuffer = convertFileToByteBuffer(pathUrl);
//发送文件流数据
session.getBasicRemote().sendBinary(outByteBuffer);
// 发送响应确认
log.info("已经成功发送了语音流给前端:{}", DateUtil.now());
} catch (IOException e) {
e.printStackTrace();
}
}
// 1. 启动音频缓冲
// webSocket.send("{\"type\": \"input_audio_buffer.start\"}");
/**
* 发送文本转语音,发送语音流给前端
*
* @param clientId 用户ID
* @param content 文本内容
* @param session 客户端会话ID
*/
private void sendTTSBuffer(String clientId, String content, Session session) {
String resultFileName = clientId + "_" + System.currentTimeMillis() + ".wav";
String resultPathUrl = RuoYiConfig.getProfile() + VOICE_STORAGE_RESULT_DIR + resultFileName;
ElevenLabsClient elevenLabsClient = SpringUtils.getBean(ElevenLabsClient.class);
elevenLabsClient.handleTextToVoice(content, resultPathUrl);
//持续返回数据流给客户端
sendVoiceBuffer(resultPathUrl, session);
}
//存储客户端webSocket对象,对数据进行隔离处理
cacheWebSocket.put(clientId, webSocket);
/**
* 对候选者初次进行提问业务逻辑处理(初始化系统随机获取第一个问题)
*
* @param clientId 用户ID
* @param session 客户端会话
*/
private void initializationQuestion(String clientId, Session session) {
try {
//自我介绍结束后马上返回一个Good
//发送初始化面试官语音流
String openingPathUrl = RuoYiConfig.getProfile() + VOICE_SYSTEM_DIR + "good.wav";
sendVoiceBuffer(openingPathUrl, session);
//初始化面试流程的提问
List<Map<String, String>> list = new LinkedList();
Map<String, String> mapEntity = new HashMap<>();
mapEntity.put("role", "system");
mapEntity.put("content", "You are a construction industry interview expert. You MUST provide both an evaluation and follow-up questions for every response. Use this exact format:\\n\\nEVALUATION:\\n[Rate 0-5 and provide detailed assessment. If answer is unrelated, contains technical errors, system messages, or is nonsensical, give 0/5]\\n\\nFOLLOW-UP:\\n[Always generate 1-2 relevant follow-up questions. If answer was invalid, ask for clarification or repeat the question]\\n\\nIMPORTANT: You must include both EVALUATION and FOLLOW-UP sections in every response.");
list.add(mapEntity);
//获取预设问题-直接TTS转换返回语音结果
IHotakeProblemBaseInfoService problemBaseInfoService = SpringUtils.getBean(IHotakeProblemBaseInfoService.class);
HotakeProblemBaseInfo queryPro = new HotakeProblemBaseInfo();
queryPro.setUserId(Long.valueOf(clientId));
List<HotakeProblemBaseInfo> baseInfoList = problemBaseInfoService.selectHotakeProblemBaseInfoList(queryPro);
if (CollectionUtil.isNotEmpty(baseInfoList)) {
HotakeProblemBaseInfo baseInfo = baseInfoList.get(0);
if (StrUtil.isNotEmpty(baseInfo.getContents())) {
String[] qStrs = baseInfo.getContents().split(",");
int random_index = (int) (Math.random() * qStrs.length);
//获取问题文本
String question = qStrs[random_index];
Map<String, String> mapEntityQ = new HashMap<>();
mapEntityQ.put("role", "user");
mapEntityQ.put("content", "Question：" + question + "\\nCandidate Answer：{}");
list.add(mapEntityQ);
//直接对该问题进行转换处理返回语音流
sendTTSBuffer(clientId, question, session);
}
}
//初始化记录提示词数据到-缓存中
cacheMsgMapData.put(session.getId(), JSONUtil.toJsonStr(list));
} catch (Exception e) {
e.printStackTrace();
log.error("面试流程初始化失败:{}", e.getMessage());
}
}
@Override
public void onMessage(WebSocket webSocket, String text) {
// System.out.println("📩 收到转录结果: " + text);
//对数据进行解析
if (StrUtil.isNotEmpty(text)) {
Map<String, String> mapResultData = JSONUtil.toBean(text, Map.class);
if ("conversation.item.input_audio_transcription.delta".equals(mapResultData.get("type"))) {
String resultText = mapResultData.get("delta");
//进行客户端文本数据存储
String cacheString = cacheClientTts.get(clientId);
if (StrUtil.isNotEmpty(cacheString)) {
cacheString = cacheString + resultText;
} else {
cacheString = resultText;
}
log.info("收到转录结果:{}","客户端ID为:"+clientId+",转录结果:"+resultText);
cacheClientTts.put(clientId, cacheString);
}
}
}
/**
* 处理面试结束业务逻辑
* 触发规则:
* 1、获得 0-1 分 大于1次 立即结束面试
* 2、获取 4-5 分 大于3次 立即结束面试
* 3、获取 2-3 分 大于3次 立即结束面试
* 4、获取 2-5 分 大于4次 立即结束面试
* 5、没有 FOLLOW-UP 理解结束面试
*
* @param session 客户端会话
* @param content 追问内容
*/
private void handleInterviewEnd(Session session, String content) {
//验证是否触发面试结束逻辑
try {
//发送面试官结束语音流
String pathUrl = RuoYiConfig.getProfile() + VOICE_SYSTEM_DIR + "end.wav";
sendVoiceBuffer(pathUrl, session);
//返回文本评分
//结束回答要清空问答数据
cacheMsgMapData.put(session.getId(), "");
Map<String, String> resultEntity = new HashMap<>();
resultEntity.put("msg", content);
resultEntity.put("dataType", "score");
session.getBasicRemote().sendText(JSONUtil.toJsonStr(resultEntity));
@Override
public void onFailure(WebSocket webSocket, Throwable t, Response response) {
System.err.println("❌ 连接失败: " + t.getMessage());
// latch.countDown();
}
@Override
public void onClosing(WebSocket webSocket, int code, String reason) {
System.out.println("⚠️ 连接即将关闭: " + reason);
webSocket.close(1000, null);
// latch.countDown();
}
});
} catch (Exception e) {
e.printStackTrace();
}
}
/**
* 语音流文件格式转换
* 解析AI结果,获取随机问题
*
* @param pathUrl
* @param outPathUrl
* @param content
* @return
*/
private void handleAudioToPCM(String pathUrl, String outPathUrl) {
File inputFile = new File(pathUrl); // 输入音频文件
File outputFile = new File(outPathUrl); // 输出PCM格式文件
try {
// 读取音频文件
AudioInputStream inputAudioStream = AudioSystem.getAudioInputStream(inputFile);
// 获取音频文件的格式信息
AudioFormat sourceFormat = inputAudioStream.getFormat();
System.out.println("Input Audio Format: " + sourceFormat);
// 设置目标PCM格式 (可以是16-bit, 8kHz, Mono, Linear PCM)
AudioFormat pcmFormat = new AudioFormat(
AudioFormat.Encoding.PCM_SIGNED,
sourceFormat.getSampleRate(),
16, // 16-bit samples
1, // 单声道
2, // 每个样本2字节（16位）
sourceFormat.getSampleRate(),
false // 大端模式
);
// 获取PCM格式的音频流
AudioInputStream pcmAudioStream = AudioSystem.getAudioInputStream(pcmFormat, inputAudioStream);
// 创建输出文件流
FileOutputStream fos = new FileOutputStream(outputFile);
byte[] buffer = new byte[1024];
int bytesRead;
// 将PCM音频数据写入输出文件
while ((bytesRead = pcmAudioStream.read(buffer)) != -1) {
fos.write(buffer, 0, bytesRead);
private String handleReturnQuestion(String content) {
return "";
}
/**
* 处理评分记录
* 触发规则:
* 1、获得 0-1 分 大于1次 立即结束面试
* 2、获取 4-5 分 大于3次 立即结束面试
* 3、获取 2-3 分 大于3次 立即结束面试
* 4、获取 2-5 分 大于4次 立即结束面试
*
* @param content
* @param session return false 立即结束面试
*/
private Boolean handleScoreRecord(String content, Session session) {
Map<String, Integer> scoreRecordMap = cacheScoreResult.get(session.getId());
//对评分进行处理
if (StrUtil.isNotEmpty(content)) {
String[] strs = content.split("\\\\");
//取第一个数就是对应的评分
BigDecimal score = new BigDecimal(strs[0]);
//记录Key为1
if (BigDecimal.ZERO.compareTo(score) <= 0 && BigDecimal.ONE.compareTo(score) >= 0) {
Integer n1 = scoreRecordMap.get("0-1") + 1;
scoreRecordMap.put("0-1", n1);
if (n1 > 1) {
return false;
}
}
//记录Key为2
if (new BigDecimal(4).compareTo(score) <= 0 && new BigDecimal(5).compareTo(score) >= 0) {
Integer n1 = scoreRecordMap.get("4-5") + 1;
scoreRecordMap.put("4-5", n1);
if (n1 > 3) {
return false;
}
}
//记录Key为3
if (new BigDecimal(2).compareTo(score) <= 0 && new BigDecimal(3).compareTo(score) >= 0) {
Integer n1 = scoreRecordMap.get("2-3") + 1;
scoreRecordMap.put("2-3", n1);
if (n1 > 3) {
return false;
}
}
//记录Key为4
if (new BigDecimal(2).compareTo(score) <= 0 && new BigDecimal(5).compareTo(score) >= 0) {
Integer n1 = scoreRecordMap.get("2-5") + 1;
scoreRecordMap.put("2-5", n1);
if (n1 > 4) {
return false;
}
}
// 关闭流
pcmAudioStream.close();
fos.close();
System.out.println("Audio has been converted to PCM format and saved at: " + outputFile.getAbsolutePath());
} catch (Exception e) {
e.printStackTrace();
}
return true;
}
}

589
vetti-admin/src/main/java/com/vetti/socket/ChatWebSocketHandler1.java Normal file
View File

@@ -0,0 +1,589 @@
package com.vetti.socket;
import cn.hutool.core.collection.CollectionUtil;
import cn.hutool.core.util.StrUtil;
import cn.hutool.json.JSONObject;
import cn.hutool.json.JSONUtil;
import com.vetti.common.ai.elevenLabs.ElevenLabsClient;
import com.vetti.common.ai.gpt.ChatGPTClient;
import com.vetti.common.ai.gpt.OpenAiStreamClient;
import com.vetti.common.ai.gpt.service.OpenAiStreamListenerService;
import com.vetti.common.config.RuoYiConfig;
import com.vetti.common.utils.spring.SpringUtils;
import com.vetti.hotake.domain.HotakeProblemBaseInfo;
import com.vetti.hotake.service.IHotakeProblemBaseInfoService;
import lombok.extern.slf4j.Slf4j;
import okhttp3.*;
import org.apache.commons.io.FileUtils;
import org.springframework.stereotype.Component;
import javax.sound.sampled.AudioFormat;
import javax.sound.sampled.AudioInputStream;
import javax.sound.sampled.AudioSystem;
import javax.websocket.*;
import javax.websocket.server.PathParam;
import javax.websocket.server.ServerEndpoint;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
/**
* 语音面试 web处理器(无用的只是记录使用)
*/
@Slf4j
@ServerEndpoint("/voice-websocket1111111111/{clientId}")
@Component
public class ChatWebSocketHandler1 {
// @Value("${whisper.apiUrl}")
private String API_URL = "wss://api.openai.com/v1/realtime?intent=transcription";
// @Value("${whisper.model}")
private String MODEL = "gpt-4o-mini-transcribe";
// @Value("${whisper.apiKey}")
private String apiKey = "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";
// @Value("${whisper.language}")
private String language = "en";
/**
* 缓存客户端流式解析的语音文本数据
*/
private final Map<String, String> cacheClientTts = new ConcurrentHashMap<>();
/**
* 缓存客户端调用OpenAi中的websocket-STT 流式传输数据
*/
private final Map<String, WebSocket> cacheWebSocket = new ConcurrentHashMap<>();
/**
* 缓存客户端,标记是否是自我介绍后的初次问答
*/
private final Map<String,String> cacheReplyFlag = new ConcurrentHashMap<>();
/**
* 缓存客户端,面试回答信息
*/
private final Map<String,String> cacheMsgMapData = new ConcurrentHashMap<>();
/**
* 缓存客户端,AI提问的问题结果信息
*/
private final Map<String,String> cacheQuestionResult = new ConcurrentHashMap<>();
// private final Map<String,String> cacheOpeningResult = new ConcurrentHashMap<>();
// 语音文件保存目录
private static final String VOICE_STORAGE_DIR = "/voice_files/";
// 语音结果文件保存目录
private static final String VOICE_STORAGE_RESULT_DIR = "/voice_result_files/";
// 系统语音目录
private static final String VOICE_SYSTEM_DIR = "/system_files/";
public ChatWebSocketHandler1() {
// 初始化存储目录
File dir = new File(RuoYiConfig.getProfile() + VOICE_STORAGE_DIR);
if (!dir.exists()) {
dir.mkdirs();
}
File resultDir = new File(RuoYiConfig.getProfile() + VOICE_STORAGE_RESULT_DIR);
if (!resultDir.exists()) {
resultDir.mkdirs();
}
}
// 连接建立时调用
@OnOpen
public void onOpen(Session session, @PathParam("clientId") String clientId) {
log.info("WebSocket 链接已建立:{}", clientId);
log.info("WebSocket session 链接已建立:{}", session.getId());
cacheClientTts.put(clientId, new String());
//初始化STT流式语音转换文本的socket链接
// createWhisperRealtimeSocket(session.getId());
//是初次自我介绍后的问答环节
cacheReplyFlag.put(session.getId(),"YES");
//初始化面试回答数据记录
cacheMsgMapData.put(session.getId(),"");
//初始化面试问题
cacheQuestionResult.put(session.getId(),"");
//开场白控制
// String flag = cacheOpeningResult.get(clientId);
// if(StrUtil.isEmpty(flag)){
//发送初始化面试官语音流
String openingPathUrl = RuoYiConfig.getProfile() + VOICE_SYSTEM_DIR + "opening.wav";
try {
//文件转换成文件流
ByteBuffer outByteBuffer = convertFileToByteBuffer(openingPathUrl);
//发送文件流数据
session.getBasicRemote().sendBinary(outByteBuffer);
// cacheOpeningResult.put(clientId,"YES");
// 发送响应确认
log.info("初始化返回面试官语音信息:{}", System.currentTimeMillis() / 1000);
} catch (IOException e) {
e.printStackTrace();
}
// }
}
// 接收文本消息
@OnMessage
public void onTextMessage(Session session, String message, @PathParam("clientId") String clientId) {
System.out.println("接收到文本消息: " + message);
try {
// {
// "type": "start | done | end",
// "content": "内容"
// }
//处理文本结果
if (StrUtil.isNotEmpty(message)) {
Map<String, String> mapResult = JSONUtil.toBean(JSONUtil.parseObj(message), Map.class);
String resultFlag = mapResult.get("type");
if ("done".equals(resultFlag)) {
//开始合并语音流
String startFlag = cacheReplyFlag.get(session.getId());
//语音结束,开始进行回答解析
log.info("开始文本处理,客户端ID为:{}",clientId);
String cacheResultText = mapResult.get("content");
log.info("开始文本处理,面试者回答信息为:{}", cacheResultText);
if (StrUtil.isEmpty(cacheResultText)) {
cacheResultText = "I first check the forklift's logbook for recent issues, inspect tires and brakes, verify the load capacity matches today's task, and confirm my licence is current—all per SWMS requirements.";
}
String promptJson = "";
//这是初次处理的逻辑
if("YES".equals(startFlag)) {
//自我介绍结束后马上返回一个Good
//发送初始化面试官语音流
String openingPathUrl = RuoYiConfig.getProfile() + VOICE_SYSTEM_DIR + "good.wav";
try {
//文件转换成文件流
ByteBuffer outByteBuffer = convertFileToByteBuffer(openingPathUrl);
//发送文件流数据
session.getBasicRemote().sendBinary(outByteBuffer);
// 发送响应确认
log.info("初始化返回面试官语音信息:{}", System.currentTimeMillis() / 1000);
} catch (IOException e) {
e.printStackTrace();
}
List<Map<String,String>> list = new LinkedList();
Map<String,String> mapEntity = new HashMap<>();
mapEntity.put("role","system");
mapEntity.put("content","You are an interviewer. Generate follow-up questions based on Construction Labourer candidate responses.Only return one question and do not repeat the previously returned questions \\n MPORTANT: Do not ask the same question again if the answer is incorrect");
list.add(mapEntity);
//获取预设问题-直接TTS转换返回语音结果
IHotakeProblemBaseInfoService problemBaseInfoService = SpringUtils.getBean(IHotakeProblemBaseInfoService.class);
HotakeProblemBaseInfo queryPro = new HotakeProblemBaseInfo();
queryPro.setUserId(Long.valueOf(clientId));
List<HotakeProblemBaseInfo> baseInfoList = problemBaseInfoService.selectHotakeProblemBaseInfoList(queryPro);
if(CollectionUtil.isNotEmpty(baseInfoList)) {
HotakeProblemBaseInfo baseInfo = baseInfoList.get(0);
if(StrUtil.isNotEmpty(baseInfo.getContents())){
String[] qStrs = baseInfo.getContents().split(",");
int random_index = (int) (Math.random()*qStrs.length);
//获取问题文本
String question = qStrs[random_index];
Map<String,String> mapEntityQ = new HashMap<>();
mapEntityQ.put("role","user");
mapEntityQ.put("content","Question："+question+"\\nCandidate Answer：{}");
list.add(mapEntityQ);
}
}
promptJson = JSONUtil.toJsonStr(list);
//记录缓存中
cacheMsgMapData.put(session.getId(),promptJson);
//直接对该问题进行转换处理返回语音流
}else{
//开始根据面试者回答的问题,进行追问回答
//获取面试者回答信息
//获取缓存记录
String msgMapData = cacheMsgMapData.get(session.getId());
if(StrUtil.isNotEmpty(msgMapData)){
List<Map> list = JSONUtil.toList(msgMapData, Map.class);
//获取最后一条数据记录
Map<String,String> mapEntity = list.get(list.size()-1);
//更新问题记录
String content = mapEntity.get("content");
mapEntity.put("content", StrUtil.format(content, cacheResultText));
promptJson = JSONUtil.toJsonStr(list);
cacheMsgMapData.put(session.getId(),promptJson);
}
}
//获取完问答数据,直接清空缓存数据
cacheClientTts.put(session.getId(),"");
cacheReplyFlag.put(session.getId(),"");
//把提问的文字发送给CPT(流式处理)
OpenAiStreamClient aiStreamClient = SpringUtils.getBean(OpenAiStreamClient.class);
log.info("AI提示词为:{}",promptJson);
aiStreamClient.streamChat(promptJson, new OpenAiStreamListenerService() {
@Override
public void onMessage(String content) {
log.info("返回AI结果：{}", content);
if(StrUtil.isNotEmpty(content)){
String questionResult = cacheQuestionResult.get(session.getId());
if(StrUtil.isEmpty(questionResult)){
questionResult = content;
}else{
questionResult = questionResult + content;
}
cacheQuestionResult.put(session.getId(),questionResult);
// 实时输出内容
try{
//把文本也给前端返回去
Map<String,String> dataText = new HashMap<>();
dataText.put("type","question");
dataText.put("content",content);
session.getBasicRemote().sendText(JSONUtil.toJsonStr(dataText));
}catch (Exception e){
e.printStackTrace();
}
//开始进行语音输出-流式持续输出
//把结果文字转成语音文件
//生成文件
//生成唯一文件名
String resultFileName = clientId + "_" + System.currentTimeMillis() + ".wav";
String resultPathUrl = RuoYiConfig.getProfile() + VOICE_STORAGE_RESULT_DIR + resultFileName;
ElevenLabsClient elevenLabsClient = SpringUtils.getBean(ElevenLabsClient.class);
elevenLabsClient.handleTextToVoice(content, resultPathUrl);
//持续返回数据流给客户端
try {
//文件转换成文件流
ByteBuffer outByteBuffer = convertFileToByteBuffer(resultPathUrl);
//发送文件流数据
session.getBasicRemote().sendBinary(outByteBuffer);
// 发送响应确认
} catch (IOException e) {
e.printStackTrace();
}
}
}
@Override
public void onComplete() {
try {
//开始往缓存中记录提问的问题
String questionResult = cacheQuestionResult.get(session.getId());
//获取缓存记录
String msgMapData = cacheMsgMapData.get(session.getId());
if(StrUtil.isNotEmpty(msgMapData)){
List<Map> list = JSONUtil.toList(msgMapData, Map.class);
Map<String,String> mapEntity = new HashMap<>();
mapEntity.put("role","user");
mapEntity.put("content","Question："+questionResult+"\\nCandidate Answer：{}");
list.add(mapEntity);
cacheMsgMapData.put(session.getId(),JSONUtil.toJsonStr(list));
}
//清空问题
cacheQuestionResult.put(session.getId(),"");
Map<String, String> resultEntity = new HashMap<>();
resultEntity.put("msg", "done");
//发送通知告诉客户端已经回答结束了
session.getBasicRemote().sendText(JSONUtil.toJsonStr(resultEntity));
} catch (Exception e) {
throw new RuntimeException(e);
}
}
@Override
public void onError(Throwable throwable) {
throwable.printStackTrace();
}
});
}else if("end".equals(resultFlag)){
// cacheOpeningResult.put(clientId,"");
//发送面试官结束语音流
String openingPathUrl = RuoYiConfig.getProfile() + VOICE_SYSTEM_DIR + "end.wav";
try {
//文件转换成文件流
ByteBuffer outByteBuffer = convertFileToByteBuffer(openingPathUrl);
//发送文件流数据
session.getBasicRemote().sendBinary(outByteBuffer);
// 发送响应确认
log.info("结束返回面试官语音信息:{}", System.currentTimeMillis() / 1000);
} catch (IOException e) {
e.printStackTrace();
}
//返回文本评分
//处理模型提问逻辑
String promptJson = "";
//获取缓存记录
String msgMapData = cacheMsgMapData.get(session.getId());
if(StrUtil.isNotEmpty(msgMapData)){
List<Map> list = JSONUtil.toList(msgMapData, Map.class);
//获取最后一条数据记录
Map<String,String> mapEntity = list.get(0);
//更新问题记录
mapEntity.put("role","system");
mapEntity.put("content","You are a construction industry interview expert. Rate Construction Labourer candidate responses on a 1-5 scale. IMPORTANT: If the answer is completely unrelated, contains technical errors, system messages, or is nonsensical, give it a score of 0/5 and explain why it's invalid.");
promptJson = JSONUtil.toJsonStr(list);
//结束回答要清空问答数据
cacheMsgMapData.put(session.getId(),"");
}
log.info("结束AI提示词为:{}",promptJson);
ChatGPTClient gptClient = SpringUtils.getBean(ChatGPTClient.class);
String resultMsg = gptClient.handleAiChat(promptJson,"QA");
Map<String, String> resultEntity = new HashMap<>();
resultEntity.put("msg", resultMsg);
resultEntity.put("dataType","score");
session.getBasicRemote().sendText(JSONUtil.toJsonStr(resultEntity));
}
}
} catch (Exception e) {
e.printStackTrace();
}
}
// 接收二进制消息（流数据）
@OnMessage
public void onBinaryMessage(Session session, @PathParam("clientId") String clientId, ByteBuffer byteBuffer) {
// log.info("客户端ID为:{}", clientId);
// // 处理二进制流数据
// byte[] bytes = new byte[byteBuffer.remaining()];
// //从缓冲区中读取数据并存储到指定的字节数组中
// byteBuffer.get(bytes);
// // 生成唯一文件名
// String fileName = clientId + "_" + System.currentTimeMillis() + ".wav";
// String pathUrl = RuoYiConfig.getProfile() + VOICE_STORAGE_DIR + fileName;
// log.info("文件路径为:{}", pathUrl);
// try {
// saveAsWebM(bytes, pathUrl);
// //接收到数据流后直接就进行SST处理
// //语音格式转换
// String fileOutName = clientId + "_" + System.currentTimeMillis() + ".pcm";
// String pathOutUrl = RuoYiConfig.getProfile() + VOICE_STORAGE_DIR + fileOutName;
// handleAudioToPCM(pathUrl, pathOutUrl);
// //发送消息
// WebSocket webSocket = cacheWebSocket.get(session.getId());
// log.info("获取的socket对象为:{}", webSocket);
// if (webSocket != null) {
//// 1. 启动音频缓冲
//// webSocket.send("{\"type\": \"input_audio_buffer.start\"}");
// File outputFile = new File(pathOutUrl); // 输出PCM格式文件
// ByteBuffer buffer = ByteBuffer.wrap(FileUtils.readFileToByteArray(outputFile));
// byte[] outBytes = new byte[buffer.remaining()];
// //从缓冲区中读取数据并存储到指定的字节数组中
// buffer.get(outBytes);
// String base64Audio = Base64.getEncoder().encodeToString(outBytes);
// String message = "{ \"type\": \"input_audio_buffer.append\", \"audio\": \"" + base64Audio + "\" }";
// webSocket.send(message);
// // 3. 提交音频并请求转录
//// webSocket.send("{\"type\": \"input_audio_buffer.commit\"}");
//// webSocket.send("{\"type\": \"response.create\"}");
// }
// } catch (Exception e) {
// e.printStackTrace();
// }
}
// 连接关闭时调用
@OnClose
public void onClose(Session session, CloseReason reason) {
System.out.println("WebSocket连接已关闭: " + session.getId() + ", 原因: " + reason.getReasonPhrase());
// WebSocket webSocket = cacheWebSocket.get(session.getId());
// if (webSocket != null) {
// webSocket.close(1000, null);
// }
}
// 发生错误时调用
@OnError
public void onError(Session session, Throwable throwable) {
System.err.println("WebSocket错误发生: " + throwable.getMessage());
throwable.printStackTrace();
}
/**
* 将字节数组保存为WebM文件
*
* @param byteData 包含WebM数据的字节数组
* @param filePath 目标文件路径
* @return 操作是否成功
*/
private boolean saveAsWebM(byte[] byteData, String filePath) {
// 检查输入参数
if (byteData == null || byteData.length == 0) {
System.err.println("字节数组为空，无法生成WebM文件");
return false;
}
if (filePath == null || filePath.trim().isEmpty()) {
System.err.println("文件路径不能为空");
return false;
}
FileOutputStream fos = null;
try {
fos = new FileOutputStream(filePath);
fos.write(byteData);
fos.flush();
System.out.println("WebM文件已成功生成: " + filePath);
return true;
} catch (IOException e) {
System.err.println("写入文件时发生错误: " + e.getMessage());
e.printStackTrace();
} finally {
if (fos != null) {
try {
fos.close();
} catch (IOException e) {
e.printStackTrace();
}
}
}
return false;
}
/**
* File 转换成 ByteBuffer
*
* @param fileUrl 文件路径
* @return
*/
private ByteBuffer convertFileToByteBuffer(String fileUrl) {
File file = new File(fileUrl);
try {
return ByteBuffer.wrap(FileUtils.readFileToByteArray(file));
} catch (Exception e) {
e.printStackTrace();
}
return null;
}
/**
* 创建STT WebSocket 客户端链接
*
* @param clientId 客户端ID
*/
private void createWhisperRealtimeSocket(String clientId) {
try {
OkHttpClient client = new OkHttpClient();
// 设置 WebSocket 请求
Request request = new Request.Builder()
.url(API_URL)
.addHeader("Authorization", "Bearer " + apiKey)
.addHeader("OpenAI-Beta", "realtime=v1")
.build();
client.newWebSocket(request, new WebSocketListener() {
@Override
public void onOpen(WebSocket webSocket, Response response) {
System.out.println("✅ WebSocket 连接成功");
//发送配置
JSONObject config = new JSONObject();
JSONObject sessionConfig = new JSONObject();
JSONObject transcription = new JSONObject();
JSONObject turnDetection = new JSONObject();
// 配置转录参数
transcription.put("model", MODEL);
transcription.put("language", language); // 中文
// 配置断句检测
turnDetection.put("type", "server_vad");
turnDetection.put("prefix_padding_ms", 300);
turnDetection.put("silence_duration_ms", 10);
// 组装完整配置
sessionConfig.put("input_audio_transcription", transcription);
sessionConfig.put("turn_detection", turnDetection);
config.put("type", "transcription_session.update");
config.put("session", sessionConfig);
webSocket.send(config.toString());
// 1. 启动音频缓冲
// webSocket.send("{\"type\": \"input_audio_buffer.start\"}");
//存储客户端webSocket对象,对数据进行隔离处理
cacheWebSocket.put(clientId, webSocket);
}
@Override
public void onMessage(WebSocket webSocket, String text) {
// System.out.println("📩 收到转录结果: " + text);
//对数据进行解析
if (StrUtil.isNotEmpty(text)) {
Map<String, String> mapResultData = JSONUtil.toBean(text, Map.class);
if ("conversation.item.input_audio_transcription.delta".equals(mapResultData.get("type"))) {
String resultText = mapResultData.get("delta");
//进行客户端文本数据存储
String cacheString = cacheClientTts.get(clientId);
if (StrUtil.isNotEmpty(cacheString)) {
cacheString = cacheString + resultText;
} else {
cacheString = resultText;
}
log.info("收到转录结果:{}","客户端ID为:"+clientId+",转录结果:"+resultText);
cacheClientTts.put(clientId, cacheString);
}
}
}
@Override
public void onFailure(WebSocket webSocket, Throwable t, Response response) {
System.err.println("❌ 连接失败: " + t.getMessage());
// latch.countDown();
}
@Override
public void onClosing(WebSocket webSocket, int code, String reason) {
System.out.println("⚠️ 连接即将关闭: " + reason);
webSocket.close(1000, null);
// latch.countDown();
}
});
} catch (Exception e) {
e.printStackTrace();
}
}
/**
* 语音流文件格式转换
*
* @param pathUrl
* @param outPathUrl
*/
private void handleAudioToPCM(String pathUrl, String outPathUrl) {
File inputFile = new File(pathUrl); // 输入音频文件
File outputFile = new File(outPathUrl); // 输出PCM格式文件
try {
// 读取音频文件
AudioInputStream inputAudioStream = AudioSystem.getAudioInputStream(inputFile);
// 获取音频文件的格式信息
AudioFormat sourceFormat = inputAudioStream.getFormat();
System.out.println("Input Audio Format: " + sourceFormat);
// 设置目标PCM格式 (可以是16-bit, 8kHz, Mono, Linear PCM)
AudioFormat pcmFormat = new AudioFormat(
AudioFormat.Encoding.PCM_SIGNED,
sourceFormat.getSampleRate(),
16, // 16-bit samples
1, // 单声道
2, // 每个样本2字节（16位）
sourceFormat.getSampleRate(),
false // 大端模式
);
// 获取PCM格式的音频流
AudioInputStream pcmAudioStream = AudioSystem.getAudioInputStream(pcmFormat, inputAudioStream);
// 创建输出文件流
FileOutputStream fos = new FileOutputStream(outputFile);
byte[] buffer = new byte[1024];
int bytesRead;
// 将PCM音频数据写入输出文件
while ((bytesRead = pcmAudioStream.read(buffer)) != -1) {
fos.write(buffer, 0, bytesRead);
}
// 关闭流
pcmAudioStream.close();
fos.close();
System.out.println("Audio has been converted to PCM format and saved at: " + outputFile.getAbsolutePath());
} catch (Exception e) {
e.printStackTrace();
}
}
}

34
vetti-admin/src/main/java/com/vetti/web/controller/ai/AiCommonController.java
View File

@@ -2,6 +2,8 @@ package com.vetti.web.controller.ai;
import com.vetti.common.ai.elevenLabs.ElevenLabsClient;
import com.vetti.common.ai.gpt.ChatGPTClient;
import com.vetti.common.ai.gpt.OpenAiStreamClient;
import com.vetti.common.ai.gpt.service.OpenAiStreamListenerService;
import com.vetti.common.ai.whisper.WhisperClient;
import com.vetti.common.core.controller.BaseController;
import com.vetti.common.core.domain.AjaxResult;
@@ -35,6 +37,9 @@ public class AiCommonController extends BaseController
@Autowired
private WhisperClient whisperClient;
@Autowired
private OpenAiStreamClient streamClient;
@Autowired
private ICommonService commonService;
@@ -63,6 +68,35 @@ public class AiCommonController extends BaseController
return AjaxResult.success(resultMsg);
}
/**
* AI 流式聊天
*/
@ApiOperation("AI 流式聊天")
@GetMapping("/stream/handleAiChat")
public AjaxResult handleAiStreamChat(@RequestParam String text)
{
streamClient.streamChat(text, new OpenAiStreamListenerService() {
@Override
public void onMessage(String content) {
System.out.println("AI 返回数据: "+content);
}
@Override
public void onComplete() {
}
@Override
public void onError(Throwable throwable) {
throwable.printStackTrace();
}
});
String resultMsg = chatGPTClient.handleAiChat(text,"QA");
return AjaxResult.success(resultMsg);
}
/**

17
vetti-admin/src/main/java/com/vetti/web/controller/system/SysProfileController.java
View File

@@ -4,11 +4,15 @@ import java.util.List;
import java.util.Map;
import cn.hutool.core.bean.BeanUtil;
import cn.hutool.core.collection.CollectionUtil;
import cn.hutool.json.JSONUtil;
import com.vetti.common.core.domain.R;
import com.vetti.common.enums.IsInterviewEnum;
import com.vetti.common.enums.UserOperStepsEnum;
import com.vetti.hotake.domain.HotakeCvInfo;
import com.vetti.hotake.domain.HotakeProblemBaseInfo;
import com.vetti.hotake.service.IHotakeCvInfoService;
import com.vetti.hotake.service.IHotakeProblemBaseInfoService;
import com.vetti.web.entity.dto.SysUserDto;
import io.swagger.annotations.Api;
import io.swagger.annotations.ApiOperation;
@@ -56,6 +60,9 @@ public class SysProfileController extends BaseController
@Autowired
private IHotakeCvInfoService cvInfoService;
@Autowired
private IHotakeProblemBaseInfoService problemBaseInfoService;
/**
* 个人信息
*/
@@ -71,6 +78,16 @@ public class SysProfileController extends BaseController
query.setUserId(user.getUserId());
List<HotakeCvInfo> cvInfoList = cvInfoService.selectHotakeCvInfoList(query);
dto.setCvInfoList(cvInfoList);
//是否可以参加面试
HotakeProblemBaseInfo queryProblemBaseInfo = new HotakeProblemBaseInfo();
queryProblemBaseInfo.setUserId(user.getUserId());
List<HotakeProblemBaseInfo> problemBaseInfoList = problemBaseInfoService.selectHotakeProblemBaseInfoList(queryProblemBaseInfo);
if(CollectionUtil.isNotEmpty(problemBaseInfoList)){
dto.setIsInterview(IsInterviewEnum.INTERVIEW_1.getCode());
}else{
dto.setIsInterview(IsInterviewEnum.INTERVIEW_0.getCode());
}
// ajax.put("roleGroup", userService.selectUserRoleGroup(loginUser.getUsername()));
// ajax.put("postGroup", userService.selectUserPostGroup(loginUser.getUsername()));
return R.ok(dto);

3
vetti-admin/src/main/java/com/vetti/web/entity/dto/SysUserDto.java
View File

@@ -18,6 +18,9 @@ import java.util.List;
@Accessors(chain = true)
public class SysUserDto extends SysUser {
@ApiModelProperty("是否可以参加面试标识(0:不可以,1:可以)")
private String isInterview;
@ApiModelProperty("简历数据集合")
private List<HotakeCvInfo> cvInfoList;
}

2
vetti-admin/src/main/resources/application-druid.yml
View File

@@ -169,7 +169,7 @@ whisper:
chatGpt:
apiKey: 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
apiUrl: https://api.openai.com/v1/chat/completions
model: ft:gpt-3.5-turbo-0125:vetti:construction-labourer-test:CTIvLD5n
model: ft:gpt-3.5-turbo-0125:vetti:construction-labourer-test:CWKBNvE2
modelCV: ft:gpt-3.5-turbo-0125:vetti:vetti-resume-test:CWPinJQq
role: system

2
vetti-admin/target/classes/application-druid.yml
View File

@@ -169,7 +169,7 @@ whisper:
chatGpt:
apiKey: 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
apiUrl: https://api.openai.com/v1/chat/completions
model: ft:gpt-3.5-turbo-0125:vetti:construction-labourer-test:CTIvLD5n
model: ft:gpt-3.5-turbo-0125:vetti:construction-labourer-test:CWKBNvE2
modelCV: ft:gpt-3.5-turbo-0125:vetti:vetti-resume-test:CWPinJQq
role: system

2
vetti-common/src/main/java/com/vetti/common/ai/gpt/OpenAiStreamClient.java
View File

@@ -33,7 +33,7 @@ public class OpenAiStreamClient {
private String role;
// 定义作为分割点的标点符号集合
private final String punctuationStr = "。,？,！,；,\\.,\\?,!,;";
private final String punctuationStr = "。？！；.?!;";
/**
* 发送流式请求

30
vetti-common/src/main/java/com/vetti/common/enums/IsInterviewEnum.java Normal file
View File

@@ -0,0 +1,30 @@
package com.vetti.common.enums;
/**
* 是否可以参加面试
*/
public enum IsInterviewEnum {
INTERVIEW_0("0", "不可以"),
INTERVIEW_1("1", "可以"),
;
private final String code;
private final String info;
IsInterviewEnum(String code, String info)
{
this.code = code;
this.info = info;
}
public String getCode()
{
return code;
}
public String getInfo()
{
return info;
}
}

12
vetti-system/target/classes/mapper/system/SysUserMapper.xml
View File

@@ -25,6 +25,7 @@ PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN"
<result property="updateTime" column="update_time" />
<result property="remark" column="remark" />
<result property="sysUserType" column="sys_user_type" />
<result property="userSetJson" column="user_set_json" />
<result property="steps" column="steps" />
<result property="jobPosition" column="job_position" />
@@ -66,7 +67,7 @@ PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN"
select u.user_id, u.dept_id, u.user_name, u.nick_name, u.email, u.avatar, u.phonenumber, u.password, u.sex, u.status, u.del_flag, u.login_ip, u.login_date, u.pwd_update_date, u.create_by, u.create_time, u.remark,
d.dept_id, d.parent_id, d.ancestors, d.dept_name, d.order_num, d.leader, d.status as dept_status,
r.role_id, r.role_name, r.role_key, r.role_sort, r.data_scope, r.status as role_status,u.sys_user_type
,u.steps,u.job_position,u.experience,u.cv_url,u.location,u.job_type,u.relocate,u.best_side_json,u.address,u.user_flag
,u.steps,u.job_position,u.experience,u.cv_url,u.location,u.job_type,u.relocate,u.best_side_json,u.address,u.user_flag,u.user_set_json
from sys_user u
left join sys_dept d on u.dept_id = d.dept_id
left join sys_user_role ur on u.user_id = ur.user_id
@@ -75,7 +76,7 @@ PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN"
<select id="selectUserList" parameterType="SysUser" resultMap="SysUserResult">
select u.user_id, u.dept_id, u.nick_name, u.user_name, u.email, u.avatar, u.phonenumber, u.sex, u.status, u.del_flag, u.login_ip, u.login_date, u.create_by, u.create_time, u.remark,
d.dept_name, d.leader,u.sys_user_type,u.steps,u.job_position,u.experience,u.cv_url,u.location,u.job_type,u.relocate,u.best_side_json,u.address,u.user_flag from sys_user u
d.dept_name, d.leader,u.sys_user_type,u.steps,u.job_position,u.experience,u.cv_url,u.location,u.job_type,u.relocate,u.best_side_json,u.address,u.user_flag,u.user_set_json from sys_user u
left join sys_dept d on u.dept_id = d.dept_id
where u.del_flag = '0'
<if test="userId != null and userId != 0">
@@ -109,7 +110,7 @@ PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN"
<select id="selectAllocatedList" parameterType="SysUser" resultMap="SysUserResult">
select distinct u.user_id, u.dept_id, u.user_name, u.nick_name, u.email, u.phonenumber, u.status,
u.create_time,u.sys_user_type,u.steps,u.job_position,u.experience,u.cv_url,u.location,
u.job_type,u.relocate,u.best_side_json,u.address,u.user_flag
u.job_type,u.relocate,u.best_side_json,u.address,u.user_flag,u.user_set_json
from sys_user u
left join sys_dept d on u.dept_id = d.dept_id
left join sys_user_role ur on u.user_id = ur.user_id
@@ -127,7 +128,7 @@ PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN"
<select id="selectUnallocatedList" parameterType="SysUser" resultMap="SysUserResult">
select distinct u.user_id, u.dept_id, u.user_name, u.nick_name, u.email, u.phonenumber, u.status, u.create_time,u.sys_user_type,
u.steps,u.job_position,u.experience,u.cv_url,u.location,u.job_type,u.relocate,u.best_side_json,u.address,u.user_flag
u.steps,u.job_position,u.experience,u.cv_url,u.location,u.job_type,u.relocate,u.best_side_json,u.address,u.user_flag,u.user_set_json
from sys_user u
left join sys_dept d on u.dept_id = d.dept_id
left join sys_user_role ur on u.user_id = ur.user_id
@@ -193,6 +194,7 @@ PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN"
<if test="bestSideJson != null and bestSideJson != ''">best_side_json,</if>
<if test="address != null and address != ''">address,</if>
<if test="userFlag != null and userFlag != ''">user_flag,</if>
<if test="userSetJson != null and userSetJson != ''">user_set_json,</if>
create_time
)values(
@@ -221,6 +223,7 @@ PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN"
<if test="bestSideJson != null and bestSideJson != ''">#{bestSideJson},</if>
<if test="address != null and address != ''">#{address},</if>
<if test="userFlag != null and userFlag != ''">#{userFlag},</if>
<if test="userSetJson != null and userSetJson != ''">#{userSetJson},</if>
sysdate()
)
@@ -253,6 +256,7 @@ PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN"
<if test="bestSideJson != null and bestSideJson != ''">best_side_json = #{bestSideJson},</if>
<if test="address != null and address != ''">address = #{address},</if>
<if test="userFlag != null and userFlag != ''">user_flag = #{userFlag},</if>
<if test="userSetJson != null and userSetJson != ''">user_set_json = #{userSetJson},</if>
update_time = sysdate()
</set>