新模型业务逻辑调整,以及衔接语更新
This commit is contained in:
@@ -2,6 +2,7 @@ package com.vetti.socket;
|
||||
|
||||
import cn.hutool.core.collection.CollectionUtil;
|
||||
import cn.hutool.core.date.DateUtil;
|
||||
import cn.hutool.core.util.ObjectUtil;
|
||||
import cn.hutool.core.util.StrUtil;
|
||||
import cn.hutool.json.JSONUtil;
|
||||
import com.vetti.common.ai.elevenLabs.ElevenLabsClient;
|
||||
@@ -167,77 +168,44 @@ public class ChatWebSocketHandler {
|
||||
promptJson = JSONUtil.toJsonStr(list);
|
||||
cacheMsgMapData.put(session.getId(), promptJson);
|
||||
}
|
||||
//开始返回衔接语
|
||||
String openingPathUrl = RuoYiConfig.getProfile() + VOICE_SYSTEM_DIR + "good.wav";
|
||||
sendVoiceBuffer(openingPathUrl, session);
|
||||
|
||||
//开始使用模型进行追问
|
||||
//把提问的文字发送给CPT(流式处理)
|
||||
OpenAiStreamClient aiStreamClient = SpringUtils.getBean(OpenAiStreamClient.class);
|
||||
//把提问的文字发送给GPT
|
||||
ChatGPTClient chatGPTClient = SpringUtils.getBean(ChatGPTClient.class);
|
||||
log.info("AI提示词为:{}", promptJson);
|
||||
//先获取回答的评分,是否符合要求
|
||||
Boolean isEndFlag = getInterviewScore(clientId,promptJson, session, "");
|
||||
if(isEndFlag){
|
||||
log.info("面试回答符合条件规则,继续追问啦!!!!!");
|
||||
final int[] resultNum = {(int) (Math.random() * 2) + 1};
|
||||
aiStreamClient.streamChat(promptJson, new OpenAiStreamListenerService() {
|
||||
@Override
|
||||
public void onMessage(String content) {
|
||||
log.info("返回AI结果：{}", content.replaceAll("\n", ""));
|
||||
//获取1和2的随机数
|
||||
if(resultNum[0] == 1){
|
||||
content = "";
|
||||
}
|
||||
resultNum[0] = resultNum[0] +1;
|
||||
log.info("提问的问题:{}",content);
|
||||
// String contentData = content.replaceAll("\n", "");
|
||||
//返回是追问的问题
|
||||
//获取的是追问的问题
|
||||
if (StrUtil.isNotEmpty(content)) {
|
||||
//对问题进行数据缓存
|
||||
cacheQuestionResult.put(session.getId(), content);
|
||||
//开始进行语音输出-流式持续输出
|
||||
sendTTSBuffer(clientId, content, session);
|
||||
// 实时输出内容
|
||||
try {
|
||||
//把文本也给前端返回去
|
||||
Map<String, String> dataText = new HashMap<>();
|
||||
dataText.put("type", "question");
|
||||
dataText.put("content", content);
|
||||
log.info("提问的问题文本发送啦:{}",JSONUtil.toJsonStr(dataText));
|
||||
session.getBasicRemote().sendText(JSONUtil.toJsonStr(dataText));
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void onComplete() {
|
||||
String resultMsg = chatGPTClient.handleAiChat(promptJson,"QA");
|
||||
if(StrUtil.isNotEmpty(resultMsg)) {
|
||||
//开始解析返回结果
|
||||
Map mapResultData = JSONUtil.toBean(resultMsg,Map.class);
|
||||
//获取评分
|
||||
//验证是否触发对应的规则
|
||||
Boolean isEndFlag = getInterviewScore(resultMsg, session);
|
||||
if(isEndFlag){
|
||||
log.info("面试回答符合条件规则,继续追问啦!!!!!");
|
||||
int resultNum = (int) (Math.random() * 2);
|
||||
List<String> questions = JSONUtil.toList(mapResultData.get("follow_up_questions").toString(), String.class);
|
||||
String questionStr = questions.get(resultNum);
|
||||
if (StrUtil.isNotEmpty(questionStr)) {
|
||||
//开始进行语音输出-流式持续输出
|
||||
sendTTSBuffer(clientId, questionStr, session);
|
||||
// 实时输出内容
|
||||
try {
|
||||
//开始往缓存中记录提问的问题
|
||||
String questionResult = cacheQuestionResult.get(session.getId());
|
||||
if (StrUtil.isNotEmpty(questionResult)) {
|
||||
//获取缓存记录
|
||||
String msgMapData = cacheMsgMapData.get(session.getId());
|
||||
if (StrUtil.isNotEmpty(msgMapData)) {
|
||||
List<Map> list = JSONUtil.toList(msgMapData, Map.class);
|
||||
Map<String, String> mapEntity = new HashMap<>();
|
||||
mapEntity.put("role", "user");
|
||||
mapEntity.put("content", "Question：" + questionResult + "\\nCandidate Answer：{}");
|
||||
list.add(mapEntity);
|
||||
cacheMsgMapData.put(session.getId(), JSONUtil.toJsonStr(list));
|
||||
}
|
||||
}
|
||||
//清空问题
|
||||
cacheQuestionResult.put(session.getId(), "");
|
||||
|
||||
//把文本也给前端返回去
|
||||
Map<String, String> dataText = new HashMap<>();
|
||||
dataText.put("type", "question");
|
||||
dataText.put("content", questionStr);
|
||||
log.info("提问的问题文本发送啦:{}",JSONUtil.toJsonStr(dataText));
|
||||
session.getBasicRemote().sendText(JSONUtil.toJsonStr(dataText));
|
||||
} catch (Exception e) {
|
||||
throw new RuntimeException(e);
|
||||
e.printStackTrace();
|
||||
}
|
||||
//开始对问题进行缓存
|
||||
recordQuestion(questionStr,session);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void onError(Throwable throwable) {
|
||||
throwable.printStackTrace();
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
} else if ("end".equals(resultFlag)) {
|
||||
@@ -347,7 +315,7 @@ public class ChatWebSocketHandler {
|
||||
List<Map<String, String>> list = new LinkedList();
|
||||
Map<String, String> mapEntity = new HashMap<>();
|
||||
mapEntity.put("role", "system");
|
||||
mapEntity.put("content", "You are an interviewer. Generate in-depth follow-up questions based on candidate responses.");
|
||||
mapEntity.put("content", "You are a construction industry interview expert. Evaluate candidate responses and provide scores (1-5) and follow-up questions when needed. Always respond in JSON format.");
|
||||
list.add(mapEntity);
|
||||
//获取预设问题-直接TTS转换返回语音结果
|
||||
IHotakeProblemBaseInfoService problemBaseInfoService = SpringUtils.getBean(IHotakeProblemBaseInfoService.class);
|
||||
@@ -358,7 +326,7 @@ public class ChatWebSocketHandler {
|
||||
if (CollectionUtil.isNotEmpty(baseInfoList)) {
|
||||
HotakeProblemBaseInfo baseInfo = baseInfoList.get(0);
|
||||
if (StrUtil.isNotEmpty(baseInfo.getContents())) {
|
||||
String[] qStrs = baseInfo.getContents().split(",");
|
||||
String[] qStrs = baseInfo.getContents().split("#AA#");
|
||||
int random_index = (int) (Math.random() * qStrs.length);
|
||||
//获取问题文本
|
||||
String question = qStrs[random_index];
|
||||
@@ -435,13 +403,9 @@ public class ChatWebSocketHandler {
|
||||
resultEntity.put("content", resultMsg);
|
||||
resultEntity.put("type", "score");
|
||||
try{
|
||||
//返回评分语音
|
||||
// sendTTSBuffer(clientId,resultMsg,session);
|
||||
|
||||
//返回最终的评分结构
|
||||
log.info("返回最终的评分结构:{}",JSONUtil.toJsonStr(resultEntity));
|
||||
session.getBasicRemote().sendText(JSONUtil.toJsonStr(resultEntity));
|
||||
|
||||
}catch (Exception e){
|
||||
e.printStackTrace();
|
||||
}
|
||||
@@ -458,12 +422,12 @@ public class ChatWebSocketHandler {
|
||||
* @param content
|
||||
* @param session return false 立即结束面试
|
||||
*/
|
||||
private Boolean handleScoreRecord(String content, Session session) {
|
||||
private Boolean handleScoreRecord(Object content, Session session) {
|
||||
Map<String, Integer> scoreRecordMap = cacheScoreResult.get(session.getId());
|
||||
log.info("获取评分结果:{}",content);
|
||||
//对评分进行处理
|
||||
if (StrUtil.isNotEmpty(content)) {
|
||||
String[] strs = content.split("/");
|
||||
if (ObjectUtil.isNotEmpty(content)) {
|
||||
String[] strs = content.toString().split("/");
|
||||
//取第一个数就是对应的评分
|
||||
log.info("获取的数据为:{}",strs[0]);
|
||||
BigDecimal score = new BigDecimal(strs[0].trim());
|
||||
@@ -504,57 +468,28 @@ public class ChatWebSocketHandler {
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取面试回答评分,并且校验是否结束面试
|
||||
* 校验是否结束面试,结束后直接返回评分
|
||||
*
|
||||
* @param promptJson 提示词数据json
|
||||
* @param resultMsg 问答AI返回的结果数据
|
||||
* @param session 客户端会话
|
||||
* @param position 职位
|
||||
*/
|
||||
private Boolean getInterviewScore(String clientId,String promptJson, Session session, String position) {
|
||||
private Boolean getInterviewScore(String resultMsg, Session session) {
|
||||
//返回文本评分
|
||||
//获取缓存记录
|
||||
String msgMapData = cacheMsgMapData.get(session.getId());
|
||||
if (StrUtil.isNotEmpty(msgMapData)) {
|
||||
List<Map> list = JSONUtil.toList(msgMapData, Map.class);
|
||||
//获取第一条数据记录
|
||||
Map<String, String> mapEntity = list.get(0);
|
||||
//更新问题记录
|
||||
mapEntity.put("role", "system");
|
||||
mapEntity.put("content", "You are a construction industry interview expert. Rate candidate responses on a 1-5 scale and analyze key signals.");
|
||||
//每个回答的内容前面要加上候选人的职位
|
||||
if (StrUtil.isNotEmpty(position)) {
|
||||
for (Map map : list) {
|
||||
if ("user".equals(map.get("role").toString())) {
|
||||
map.put("content", "Position: " + position + "\\n" + map.get("content"));
|
||||
}
|
||||
}
|
||||
}
|
||||
promptJson = JSONUtil.toJsonStr(list);
|
||||
}
|
||||
log.info("评分AI提示词为:{}", promptJson);
|
||||
ChatGPTClient gptClient = SpringUtils.getBean(ChatGPTClient.class);
|
||||
String resultMsg = gptClient.handleAiChat(promptJson, "QA");
|
||||
//评论格式为: Score: 3/5\nAssessment: Basically correct answer but lacks detail
|
||||
String resultScore = "";
|
||||
String scoreText = resultMsg;
|
||||
if (StrUtil.isNotEmpty(resultMsg)) {
|
||||
resultMsg = resultMsg.replaceAll("\n","#AA#");
|
||||
String[] resultMsgs = resultMsg.split("#AA#");
|
||||
resultScore = resultMsgs[0].replaceAll(SCORE_FLAG, "");
|
||||
}
|
||||
//开始解析返回结果
|
||||
Map mapResultData = JSONUtil.toBean(resultMsg,Map.class);
|
||||
//获取评分
|
||||
Object scoreStr = mapResultData.get("score");
|
||||
Object assessment = mapResultData.get("assessment");
|
||||
//校验面试是否结束
|
||||
Boolean flag = handleScoreRecord(resultScore, session);
|
||||
Boolean flag = handleScoreRecord(scoreStr, session);
|
||||
try {
|
||||
if (!flag) {
|
||||
//发送面试官结束语音流
|
||||
String openingPathUrl = RuoYiConfig.getProfile() + VOICE_SYSTEM_DIR + "end.wav";
|
||||
sendVoiceBuffer(openingPathUrl, session);
|
||||
|
||||
//返回评分语音
|
||||
// sendTTSBuffer(clientId,scoreText,session);
|
||||
|
||||
Map<String, String> resultEntity = new HashMap<>();
|
||||
resultEntity.put("content", scoreText);
|
||||
resultEntity.put("content", scoreStr +"\n"+assessment);
|
||||
resultEntity.put("type", "score");
|
||||
//返回评分结果
|
||||
log.info("返回最终的评分结果:{}",JSONUtil.toJsonStr(resultEntity));
|
||||
@@ -567,5 +502,25 @@ public class ChatWebSocketHandler {
|
||||
return flag;
|
||||
}
|
||||
|
||||
/**
|
||||
* 记录问题
|
||||
* @param questionResult
|
||||
* @param session
|
||||
*/
|
||||
private void recordQuestion(String questionResult,Session session) {
|
||||
if (StrUtil.isNotEmpty(questionResult)) {
|
||||
//获取缓存记录
|
||||
String msgMapData = cacheMsgMapData.get(session.getId());
|
||||
if (StrUtil.isNotEmpty(msgMapData)) {
|
||||
List<Map> list = JSONUtil.toList(msgMapData, Map.class);
|
||||
Map<String, String> mapEntity = new HashMap<>();
|
||||
mapEntity.put("role", "user");
|
||||
mapEntity.put("content", "Question：" + questionResult + "\\nCandidate Answer：{}");
|
||||
list.add(mapEntity);
|
||||
cacheMsgMapData.put(session.getId(), JSONUtil.toJsonStr(list));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,571 @@
|
||||
package com.vetti.socket;
|
||||
|
||||
import cn.hutool.core.collection.CollectionUtil;
|
||||
import cn.hutool.core.date.DateUtil;
|
||||
import cn.hutool.core.util.StrUtil;
|
||||
import cn.hutool.json.JSONUtil;
|
||||
import com.vetti.common.ai.elevenLabs.ElevenLabsClient;
|
||||
import com.vetti.common.ai.gpt.ChatGPTClient;
|
||||
import com.vetti.common.ai.gpt.OpenAiStreamClient;
|
||||
import com.vetti.common.ai.gpt.service.OpenAiStreamListenerService;
|
||||
import com.vetti.common.config.RuoYiConfig;
|
||||
import com.vetti.common.utils.spring.SpringUtils;
|
||||
import com.vetti.hotake.domain.HotakeProblemBaseInfo;
|
||||
import com.vetti.hotake.service.IHotakeProblemBaseInfoService;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.commons.io.FileUtils;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
import javax.websocket.*;
|
||||
import javax.websocket.server.PathParam;
|
||||
import javax.websocket.server.ServerEndpoint;
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.math.BigDecimal;
|
||||
import java.nio.ByteBuffer;
|
||||
import java.util.HashMap;
|
||||
import java.util.LinkedList;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
|
||||
/**
|
||||
* 语音面试 web处理器
|
||||
*/
|
||||
@Slf4j
|
||||
@ServerEndpoint("/voice-websocket222222/{clientId}")
|
||||
@Component
|
||||
public class ChatWebSocketHandler2 {
|
||||
|
||||
/**
|
||||
* 评分标记
|
||||
*/
|
||||
private final String SCORE_FLAG = "Score:";
|
||||
|
||||
|
||||
/**
|
||||
* 缓存客户端流式解析的语音文本数据
|
||||
*/
|
||||
private final Map<String, String> cacheClientTts = new ConcurrentHashMap<>();
|
||||
|
||||
/**
|
||||
* 缓存客户端,标记是否是自我介绍后的初次问答
|
||||
*/
|
||||
private final Map<String, String> cacheReplyFlag = new ConcurrentHashMap<>();
|
||||
|
||||
/**
|
||||
* 缓存客户端,面试回答信息
|
||||
*/
|
||||
private final Map<String, String> cacheMsgMapData = new ConcurrentHashMap<>();
|
||||
|
||||
/**
|
||||
* 缓存客户端,AI提问的问题结果信息
|
||||
*/
|
||||
private final Map<String, String> cacheQuestionResult = new ConcurrentHashMap<>();
|
||||
|
||||
/**
|
||||
* 缓存客户端,得分结果记录
|
||||
*/
|
||||
private final Map<String, Map<String, Integer>> cacheScoreResult = new ConcurrentHashMap<>();
|
||||
|
||||
// 语音文件保存目录
|
||||
private static final String VOICE_STORAGE_DIR = "/voice_files/";
|
||||
|
||||
// 语音结果文件保存目录
|
||||
private static final String VOICE_STORAGE_RESULT_DIR = "/voice_result_files/";
|
||||
|
||||
// 系统语音目录
|
||||
private static final String VOICE_SYSTEM_DIR = "/system_files/";
|
||||
|
||||
public ChatWebSocketHandler2() {
|
||||
// 初始化存储目录
|
||||
File dir = new File(RuoYiConfig.getProfile() + VOICE_STORAGE_DIR);
|
||||
if (!dir.exists()) {
|
||||
dir.mkdirs();
|
||||
}
|
||||
|
||||
File resultDir = new File(RuoYiConfig.getProfile() + VOICE_STORAGE_RESULT_DIR);
|
||||
if (!resultDir.exists()) {
|
||||
resultDir.mkdirs();
|
||||
}
|
||||
}
|
||||
|
||||
// 连接建立时调用
|
||||
@OnOpen
|
||||
public void onOpen(Session session, @PathParam("clientId") String clientId) {
|
||||
log.info("WebSocket 链接已建立:{}", clientId);
|
||||
log.info("WebSocket session 链接已建立:{}", session.getId());
|
||||
cacheClientTts.put(clientId, new String());
|
||||
//是初次自我介绍后的问答环节
|
||||
cacheReplyFlag.put(session.getId(), "YES");
|
||||
//初始化面试回答数据记录
|
||||
cacheMsgMapData.put(session.getId(), "");
|
||||
//初始化面试问题
|
||||
cacheQuestionResult.put(session.getId(), "");
|
||||
//初始化得分结果记录
|
||||
Map<String, Integer> scoreResultData = new HashMap<>();
|
||||
scoreResultData.put("0-1", 0);
|
||||
scoreResultData.put("4-5", 0);
|
||||
scoreResultData.put("2-3", 0);
|
||||
scoreResultData.put("2-5", 0);
|
||||
cacheScoreResult.put(session.getId(), scoreResultData);
|
||||
//发送初始化面试官语音流
|
||||
String openingPathUrl = RuoYiConfig.getProfile() + VOICE_SYSTEM_DIR + "opening.wav";
|
||||
sendVoiceBuffer(openingPathUrl, session);
|
||||
}
|
||||
|
||||
/**
|
||||
* 接收文本消息
|
||||
*
|
||||
* @param session 客户端会话
|
||||
* @param message 消息
|
||||
* 如:
|
||||
* {
|
||||
* "type": "start | done | end",
|
||||
* "content": "内容"
|
||||
* }
|
||||
* @param clientId 用户ID
|
||||
*/
|
||||
@OnMessage
|
||||
public void onTextMessage(Session session, String message, @PathParam("clientId") String clientId) {
|
||||
log.info("我是接收文本消息:{}", message);
|
||||
try {
|
||||
//处理文本结果
|
||||
if (StrUtil.isNotEmpty(message)) {
|
||||
Map<String, String> mapResult = JSONUtil.toBean(JSONUtil.parseObj(message), Map.class);
|
||||
String resultFlag = mapResult.get("type");
|
||||
if ("done".equals(resultFlag)) {
|
||||
//开始合并语音流
|
||||
String startFlag = cacheReplyFlag.get(session.getId());
|
||||
//语音结束,开始进行回答解析
|
||||
log.info("开始文本处理,客户端ID为:{}", clientId);
|
||||
String cacheResultText = mapResult.get("content");
|
||||
log.info("开始文本处理,面试者回答信息为:{}", cacheResultText);
|
||||
if (StrUtil.isEmpty(cacheResultText)) {
|
||||
cacheResultText = "";
|
||||
}
|
||||
|
||||
//这是初次处理的逻辑
|
||||
if ("YES".equals(startFlag)) {
|
||||
//初始化-不走大模型-直接对候选人进行提问
|
||||
initializationQuestion(clientId, session);
|
||||
//发送完第一次消息后,直接删除标记,开始进行正常的面试问答流程
|
||||
cacheReplyFlag.put(session.getId(), "");
|
||||
} else {
|
||||
//开始根据面试者回答的问题,进行追问回答
|
||||
//获取面试者回答信息
|
||||
//获取缓存记录
|
||||
String promptJson = "";
|
||||
String msgMapData = cacheMsgMapData.get(session.getId());
|
||||
if (StrUtil.isNotEmpty(msgMapData)) {
|
||||
List<Map> list = JSONUtil.toList(msgMapData, Map.class);
|
||||
//获取最后一条数据记录
|
||||
Map<String, String> mapEntity = list.get(list.size() - 1);
|
||||
//更新问题记录
|
||||
String content = mapEntity.get("content");
|
||||
mapEntity.put("content", StrUtil.format(content, cacheResultText));
|
||||
promptJson = JSONUtil.toJsonStr(list);
|
||||
cacheMsgMapData.put(session.getId(), promptJson);
|
||||
}
|
||||
//开始使用模型进行追问
|
||||
//把提问的文字发送给CPT(流式处理)
|
||||
OpenAiStreamClient aiStreamClient = SpringUtils.getBean(OpenAiStreamClient.class);
|
||||
log.info("AI提示词为:{}", promptJson);
|
||||
//先获取回答的评分,是否符合要求
|
||||
Boolean isEndFlag = getInterviewScore(clientId,promptJson, session, "");
|
||||
if(isEndFlag){
|
||||
log.info("面试回答符合条件规则,继续追问啦!!!!!");
|
||||
final int[] resultNum = {(int) (Math.random() * 2) + 1};
|
||||
aiStreamClient.streamChat(promptJson, new OpenAiStreamListenerService() {
|
||||
@Override
|
||||
public void onMessage(String content) {
|
||||
log.info("返回AI结果：{}", content.replaceAll("\n", ""));
|
||||
//获取1和2的随机数
|
||||
if(resultNum[0] == 1){
|
||||
content = "";
|
||||
}
|
||||
resultNum[0] = resultNum[0] +1;
|
||||
log.info("提问的问题:{}",content);
|
||||
// String contentData = content.replaceAll("\n", "");
|
||||
//返回是追问的问题
|
||||
//获取的是追问的问题
|
||||
if (StrUtil.isNotEmpty(content)) {
|
||||
//对问题进行数据缓存
|
||||
cacheQuestionResult.put(session.getId(), content);
|
||||
//开始进行语音输出-流式持续输出
|
||||
sendTTSBuffer(clientId, content, session);
|
||||
// 实时输出内容
|
||||
try {
|
||||
//把文本也给前端返回去
|
||||
Map<String, String> dataText = new HashMap<>();
|
||||
dataText.put("type", "question");
|
||||
dataText.put("content", content);
|
||||
log.info("提问的问题文本发送啦:{}",JSONUtil.toJsonStr(dataText));
|
||||
session.getBasicRemote().sendText(JSONUtil.toJsonStr(dataText));
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void onComplete() {
|
||||
try {
|
||||
//开始往缓存中记录提问的问题
|
||||
String questionResult = cacheQuestionResult.get(session.getId());
|
||||
if (StrUtil.isNotEmpty(questionResult)) {
|
||||
//获取缓存记录
|
||||
String msgMapData = cacheMsgMapData.get(session.getId());
|
||||
if (StrUtil.isNotEmpty(msgMapData)) {
|
||||
List<Map> list = JSONUtil.toList(msgMapData, Map.class);
|
||||
Map<String, String> mapEntity = new HashMap<>();
|
||||
mapEntity.put("role", "user");
|
||||
mapEntity.put("content", "Question：" + questionResult + "\\nCandidate Answer：{}");
|
||||
list.add(mapEntity);
|
||||
cacheMsgMapData.put(session.getId(), JSONUtil.toJsonStr(list));
|
||||
}
|
||||
}
|
||||
//清空问题
|
||||
cacheQuestionResult.put(session.getId(), "");
|
||||
|
||||
} catch (Exception e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void onError(Throwable throwable) {
|
||||
throwable.printStackTrace();
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
} else if ("end".equals(resultFlag)) {
|
||||
log.info("面试结束啦!!!!!");
|
||||
handleInterviewEnd(clientId,session,"");
|
||||
}
|
||||
}
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
}
|
||||
|
||||
// 接收二进制消息（流数据）
|
||||
@OnMessage
|
||||
public void onBinaryMessage(Session session, @PathParam("clientId") String clientId, ByteBuffer byteBuffer) {
|
||||
log.info("我是接受二进制流的-客户端ID为:{}", clientId);
|
||||
}
|
||||
|
||||
// 连接关闭时调用
|
||||
@OnClose
|
||||
public void onClose(Session session, CloseReason reason) {
|
||||
System.out.println("WebSocket连接已关闭: " + session.getId() + ", 原因: " + reason.getReasonPhrase());
|
||||
//链接关闭,清空内存
|
||||
//是初次自我介绍后的问答环节
|
||||
cacheReplyFlag.put(session.getId(), "");
|
||||
//初始化面试回答数据记录
|
||||
cacheMsgMapData.put(session.getId(), "");
|
||||
//初始化面试问题
|
||||
cacheQuestionResult.put(session.getId(), "");
|
||||
|
||||
cacheScoreResult.put(session.getId(), null);
|
||||
}
|
||||
|
||||
// 发生错误时调用
|
||||
@OnError
|
||||
public void onError(Session session, Throwable throwable) {
|
||||
System.err.println("WebSocket错误发生: " + throwable.getMessage());
|
||||
throwable.printStackTrace();
|
||||
}
|
||||
|
||||
/**
|
||||
* File 转换成 ByteBuffer
|
||||
*
|
||||
* @param fileUrl 文件路径
|
||||
* @return
|
||||
*/
|
||||
private ByteBuffer convertFileToByteBuffer(String fileUrl) {
|
||||
File file = new File(fileUrl);
|
||||
try {
|
||||
return ByteBuffer.wrap(FileUtils.readFileToByteArray(file));
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* 发送语音流给前端
|
||||
*
|
||||
* @param pathUrl 语音文件地址
|
||||
* @param session 客户端会话
|
||||
*/
|
||||
private void sendVoiceBuffer(String pathUrl, Session session) {
|
||||
try {
|
||||
//文件转换成文件流
|
||||
ByteBuffer outByteBuffer = convertFileToByteBuffer(pathUrl);
|
||||
//发送文件流数据
|
||||
session.getBasicRemote().sendBinary(outByteBuffer);
|
||||
// 发送响应确认
|
||||
log.info("已经成功发送了语音流给前端:{}", DateUtil.now());
|
||||
} catch (IOException e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 发送文本转语音,发送语音流给前端
|
||||
*
|
||||
* @param clientId 用户ID
|
||||
* @param content 文本内容
|
||||
* @param session 客户端会话ID
|
||||
*/
|
||||
private void sendTTSBuffer(String clientId, String content, Session session) {
|
||||
String resultFileName = clientId + "_" + System.currentTimeMillis() + ".wav";
|
||||
String resultPathUrl = RuoYiConfig.getProfile() + VOICE_STORAGE_RESULT_DIR + resultFileName;
|
||||
ElevenLabsClient elevenLabsClient = SpringUtils.getBean(ElevenLabsClient.class);
|
||||
elevenLabsClient.handleTextToVoice(content, resultPathUrl);
|
||||
//持续返回数据流给客户端
|
||||
log.info("发送语音流成功啦!!!!!!!");
|
||||
sendVoiceBuffer(resultPathUrl, session);
|
||||
}
|
||||
|
||||
/**
|
||||
* 对候选者初次进行提问业务逻辑处理(初始化系统随机获取第一个问题)
|
||||
*
|
||||
* @param clientId 用户ID
|
||||
* @param session 客户端会话
|
||||
*/
|
||||
private void initializationQuestion(String clientId, Session session) {
|
||||
try {
|
||||
log.info("开始获取到clientid :{}",clientId);
|
||||
//自我介绍结束后马上返回一个Good
|
||||
//发送初始化面试官语音流
|
||||
String openingPathUrl = RuoYiConfig.getProfile() + VOICE_SYSTEM_DIR + "good.wav";
|
||||
sendVoiceBuffer(openingPathUrl, session);
|
||||
//初始化面试流程的提问
|
||||
List<Map<String, String>> list = new LinkedList();
|
||||
Map<String, String> mapEntity = new HashMap<>();
|
||||
mapEntity.put("role", "system");
|
||||
mapEntity.put("content", "You are an interviewer. Generate in-depth follow-up questions based on candidate responses.");
|
||||
list.add(mapEntity);
|
||||
//获取预设问题-直接TTS转换返回语音结果
|
||||
IHotakeProblemBaseInfoService problemBaseInfoService = SpringUtils.getBean(IHotakeProblemBaseInfoService.class);
|
||||
HotakeProblemBaseInfo queryPro = new HotakeProblemBaseInfo();
|
||||
queryPro.setUserId(Long.valueOf(clientId));
|
||||
List<HotakeProblemBaseInfo> baseInfoList = problemBaseInfoService.selectHotakeProblemBaseInfoList(queryPro);
|
||||
log.info("准备进行第一个问题的提问:{}",JSONUtil.toJsonStr(baseInfoList));
|
||||
if (CollectionUtil.isNotEmpty(baseInfoList)) {
|
||||
HotakeProblemBaseInfo baseInfo = baseInfoList.get(0);
|
||||
if (StrUtil.isNotEmpty(baseInfo.getContents())) {
|
||||
String[] qStrs = baseInfo.getContents().split(",");
|
||||
int random_index = (int) (Math.random() * qStrs.length);
|
||||
//获取问题文本
|
||||
String question = qStrs[random_index];
|
||||
Map<String, String> mapEntityQ = new HashMap<>();
|
||||
mapEntityQ.put("role", "user");
|
||||
mapEntityQ.put("content", "Question：" + question + "\\nCandidate Answer：{}");
|
||||
list.add(mapEntityQ);
|
||||
log.info("开始提问啦:{}",JSONUtil.toJsonStr(list));
|
||||
//直接对该问题进行转换处理返回语音流
|
||||
log.info("第一个问题为:{}",question);
|
||||
sendTTSBuffer(clientId, question, session);
|
||||
//发送问题文本
|
||||
try {
|
||||
//把文本也给前端返回去
|
||||
Map<String, String> dataText = new HashMap<>();
|
||||
dataText.put("type", "question");
|
||||
dataText.put("content", question);
|
||||
log.info("提问的问题文本发送啦:{}",JSONUtil.toJsonStr(dataText));
|
||||
session.getBasicRemote().sendText(JSONUtil.toJsonStr(dataText));
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
}
|
||||
}
|
||||
//初始化记录提示词数据到-缓存中
|
||||
cacheMsgMapData.put(session.getId(), JSONUtil.toJsonStr(list));
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace();
|
||||
log.error("面试流程初始化失败:{}", e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 处理面试结束业务逻辑
|
||||
*
|
||||
* @param session 客户端会话
|
||||
* @param position 职位
|
||||
*/
|
||||
private void handleInterviewEnd(String clientId,Session session,String position) {
|
||||
//暂时的业务逻辑
|
||||
//发送面试官结束语音流
|
||||
String openingPathUrl = RuoYiConfig.getProfile() + VOICE_SYSTEM_DIR + "end.wav";
|
||||
sendVoiceBuffer(openingPathUrl, session);
|
||||
|
||||
//返回文本评分
|
||||
//处理模型提问逻辑
|
||||
//获取缓存记录
|
||||
String msgMapData = cacheMsgMapData.get(session.getId());
|
||||
String promptJson = "";
|
||||
if (StrUtil.isNotEmpty(msgMapData)) {
|
||||
List<Map> list = JSONUtil.toList(msgMapData, Map.class);
|
||||
//获取第一条数据记录
|
||||
Map<String, String> mapEntity = list.get(0);
|
||||
//更新问题记录
|
||||
mapEntity.put("role", "system");
|
||||
mapEntity.put("content", "You are a construction industry interview expert. Rate candidate responses on a 1-5 scale and analyze key signals.");
|
||||
//每个回答的内容前面要加上候选人的职位
|
||||
if (StrUtil.isNotEmpty(position)) {
|
||||
for (Map map : list) {
|
||||
if ("user".equals(map.get("role").toString())) {
|
||||
map.put("content", "Position: " + position + "\\n" + map.get("content"));
|
||||
}
|
||||
}
|
||||
}
|
||||
promptJson = JSONUtil.toJsonStr(list);
|
||||
|
||||
//结束回答要清空问答数据
|
||||
cacheMsgMapData.put(session.getId(), "");
|
||||
}
|
||||
log.info("结束AI提示词为:{}", promptJson);
|
||||
ChatGPTClient gptClient = SpringUtils.getBean(ChatGPTClient.class);
|
||||
String resultMsg = gptClient.handleAiChat(promptJson, "QA");
|
||||
Map<String, String> resultEntity = new HashMap<>();
|
||||
resultEntity.put("content", resultMsg);
|
||||
resultEntity.put("type", "score");
|
||||
try{
|
||||
//返回评分语音
|
||||
// sendTTSBuffer(clientId,resultMsg,session);
|
||||
|
||||
//返回最终的评分结构
|
||||
log.info("返回最终的评分结构:{}",JSONUtil.toJsonStr(resultEntity));
|
||||
session.getBasicRemote().sendText(JSONUtil.toJsonStr(resultEntity));
|
||||
|
||||
}catch (Exception e){
|
||||
e.printStackTrace();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 处理评分记录
|
||||
* 触发规则:
|
||||
* 1、获得 0-1 分 大于1次 立即结束面试
|
||||
* 2、获取 4-5 分 大于3次 立即结束面试
|
||||
* 3、获取 2-3 分 大于3次 立即结束面试
|
||||
* 4、获取 2-5 分 大于4次 立即结束面试
|
||||
*
|
||||
* @param content
|
||||
* @param session return false 立即结束面试
|
||||
*/
|
||||
private Boolean handleScoreRecord(String content, Session session) {
|
||||
Map<String, Integer> scoreRecordMap = cacheScoreResult.get(session.getId());
|
||||
log.info("获取评分结果:{}",content);
|
||||
//对评分进行处理
|
||||
if (StrUtil.isNotEmpty(content)) {
|
||||
String[] strs = content.split("/");
|
||||
//取第一个数就是对应的评分
|
||||
log.info("获取的数据为:{}",strs[0]);
|
||||
BigDecimal score = new BigDecimal(strs[0].trim());
|
||||
//记录Key为1
|
||||
if (BigDecimal.ZERO.compareTo(score) <= 0 && BigDecimal.ONE.compareTo(score) >= 0) {
|
||||
Integer n1 = scoreRecordMap.get("0-1") + 1;
|
||||
scoreRecordMap.put("0-1", n1);
|
||||
if (n1 > 1) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
//记录Key为2
|
||||
if (new BigDecimal(4).compareTo(score) <= 0 && new BigDecimal(5).compareTo(score) >= 0) {
|
||||
Integer n1 = scoreRecordMap.get("4-5") + 1;
|
||||
scoreRecordMap.put("4-5", n1);
|
||||
if (n1 > 3) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
//记录Key为3
|
||||
if (new BigDecimal(2).compareTo(score) <= 0 && new BigDecimal(3).compareTo(score) >= 0) {
|
||||
Integer n1 = scoreRecordMap.get("2-3") + 1;
|
||||
scoreRecordMap.put("2-3", n1);
|
||||
if (n1 > 3) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
//记录Key为4
|
||||
if (new BigDecimal(2).compareTo(score) <= 0 && new BigDecimal(5).compareTo(score) >= 0) {
|
||||
Integer n1 = scoreRecordMap.get("2-5") + 1;
|
||||
scoreRecordMap.put("2-5", n1);
|
||||
if (n1 > 4) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取面试回答评分,并且校验是否结束面试
|
||||
*
|
||||
* @param promptJson 提示词数据json
|
||||
* @param session 客户端会话
|
||||
* @param position 职位
|
||||
*/
|
||||
private Boolean getInterviewScore(String clientId,String promptJson, Session session, String position) {
|
||||
//返回文本评分
|
||||
//获取缓存记录
|
||||
String msgMapData = cacheMsgMapData.get(session.getId());
|
||||
if (StrUtil.isNotEmpty(msgMapData)) {
|
||||
List<Map> list = JSONUtil.toList(msgMapData, Map.class);
|
||||
//获取第一条数据记录
|
||||
Map<String, String> mapEntity = list.get(0);
|
||||
//更新问题记录
|
||||
mapEntity.put("role", "system");
|
||||
mapEntity.put("content", "You are a construction industry interview expert. Rate candidate responses on a 1-5 scale and analyze key signals.");
|
||||
//每个回答的内容前面要加上候选人的职位
|
||||
if (StrUtil.isNotEmpty(position)) {
|
||||
for (Map map : list) {
|
||||
if ("user".equals(map.get("role").toString())) {
|
||||
map.put("content", "Position: " + position + "\\n" + map.get("content"));
|
||||
}
|
||||
}
|
||||
}
|
||||
promptJson = JSONUtil.toJsonStr(list);
|
||||
}
|
||||
log.info("评分AI提示词为:{}", promptJson);
|
||||
ChatGPTClient gptClient = SpringUtils.getBean(ChatGPTClient.class);
|
||||
String resultMsg = gptClient.handleAiChat(promptJson, "QA");
|
||||
//评论格式为: Score: 3/5\nAssessment: Basically correct answer but lacks detail
|
||||
String resultScore = "";
|
||||
String scoreText = resultMsg;
|
||||
if (StrUtil.isNotEmpty(resultMsg)) {
|
||||
resultMsg = resultMsg.replaceAll("\n","#AA#");
|
||||
String[] resultMsgs = resultMsg.split("#AA#");
|
||||
resultScore = resultMsgs[0].replaceAll(SCORE_FLAG, "");
|
||||
}
|
||||
//校验面试是否结束
|
||||
Boolean flag = handleScoreRecord(resultScore, session);
|
||||
try {
|
||||
if (!flag) {
|
||||
//发送面试官结束语音流
|
||||
String openingPathUrl = RuoYiConfig.getProfile() + VOICE_SYSTEM_DIR + "end.wav";
|
||||
sendVoiceBuffer(openingPathUrl, session);
|
||||
|
||||
//返回评分语音
|
||||
// sendTTSBuffer(clientId,scoreText,session);
|
||||
|
||||
Map<String, String> resultEntity = new HashMap<>();
|
||||
resultEntity.put("content", scoreText);
|
||||
resultEntity.put("type", "score");
|
||||
//返回评分结果
|
||||
log.info("返回最终的评分结果:{}",JSONUtil.toJsonStr(resultEntity));
|
||||
session.getBasicRemote().sendText(JSONUtil.toJsonStr(resultEntity));
|
||||
|
||||
}
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
return flag;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -53,7 +53,7 @@ public class AiCommonController extends BaseController
|
||||
//你好,我是本次的面试官Vetti,请点击开始按钮后,做一段自我介绍.
|
||||
//你好,我是本次的面试官Vetti,请在三秒后,开始做一段自我介绍.
|
||||
//本轮面试结束,谢谢您的配合,面试结果将稍后通知
|
||||
elevenLabsClient.handleTextToVoice("Hello, I am Vetti, the interviewer for this interview. Please begin a self introduction in three seconds","/Users/wangxiangshun/Desktop/临时文件/opening1.wav");
|
||||
elevenLabsClient.handleTextToVoice("Ok, I have received your reply.","/Users/wangxiangshun/Desktop/临时文件/good.wav");
|
||||
|
||||
return success();
|
||||
}
|
||||
|
||||
@@ -169,7 +169,7 @@ whisper:
|
||||
chatGpt:
|
||||
apiKey: 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
|
||||
apiUrl: https://api.openai.com/v1/chat/completions
|
||||
model: ft:gpt-3.5-turbo-0125:vetti::CYl9OBMN
|
||||
model: ft:gpt-3.5-turbo-0125:vetti:interview-unified:CaGyCXOr
|
||||
modelCV: ft:gpt-3.5-turbo-0125:vetti:vetti-resume-full:CYT0C8JG
|
||||
role: system
|
||||
|
||||
|
||||
@@ -169,7 +169,7 @@ whisper:
|
||||
chatGpt:
|
||||
apiKey: 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
|
||||
apiUrl: https://api.openai.com/v1/chat/completions
|
||||
model: ft:gpt-3.5-turbo-0125:vetti::CYl9OBMN
|
||||
model: ft:gpt-3.5-turbo-0125:vetti:interview-unified:CaGyCXOr
|
||||
modelCV: ft:gpt-3.5-turbo-0125:vetti:vetti-resume-full:CYT0C8JG
|
||||
role: system
|
||||
|
||||
|
||||
@@ -57,13 +57,11 @@ public class ElevenLabsClient {
|
||||
Gson gson = new Gson();
|
||||
StringEntity entity = new StringEntity(gson.toJson(payload), ContentType.APPLICATION_JSON);
|
||||
httpPost.setEntity(entity);
|
||||
|
||||
try (CloseableHttpResponse response = httpClient.execute(httpPost)) {
|
||||
HttpEntity responseEntity = response.getEntity();
|
||||
if (responseEntity != null) {
|
||||
try (InputStream inputStream = responseEntity.getContent();
|
||||
FileOutputStream outputStream = new FileOutputStream(outputFilePath)) {
|
||||
|
||||
byte[] buffer = new byte[4096];
|
||||
int bytesRead;
|
||||
while ((bytesRead = inputStream.read(buffer)) != -1) {
|
||||
|
||||
Reference in New Issue
Block a user