Agent 业务逻辑完善
This commit is contained in:
@@ -0,0 +1,200 @@
|
||||
package com.vetti.socket.agents;
|
||||
|
||||
import javax.websocket.*;
|
||||
import javax.websocket.server.ServerEndpoint;
|
||||
|
||||
import cn.hutool.core.date.DateUtil;
|
||||
import cn.hutool.json.JSONUtil;
|
||||
import com.vetti.common.config.RuoYiConfig;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.commons.io.FileUtils;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.net.URI;
|
||||
import java.nio.ByteBuffer;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Base64;
|
||||
import java.util.concurrent.Executors;
|
||||
import java.util.concurrent.ScheduledExecutorService;
|
||||
import java.util.concurrent.ScheduledFuture;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
|
||||
@Slf4j
|
||||
@ServerEndpoint("/voice-websocket/elevenLabsAgent/{clientId}")
|
||||
@Component
|
||||
public class VoiceBridgeEndpoint {
|
||||
|
||||
// 语音文件保存目录
|
||||
private static final String VOICE_STORAGE_DIR = "/voice_files/";
|
||||
|
||||
// 语音结果文件保存目录
|
||||
private static final String VOICE_STORAGE_RESULT_DIR = "/voice_result_files/";
|
||||
|
||||
// 系统语音目录
|
||||
private static final String VOICE_SYSTEM_DIR = "/system_files/";
|
||||
|
||||
|
||||
private Session frontendSession;
|
||||
|
||||
private ElevenLabsAgentEndpoint agentClient;
|
||||
|
||||
private final ScheduledExecutorService scheduler = Executors.newSingleThreadScheduledExecutor();
|
||||
|
||||
private ScheduledFuture<?> pendingCommit;
|
||||
|
||||
private final long timeoutMs = 600;
|
||||
|
||||
private static final String AGENT_URL =
|
||||
"wss://api.elevenlabs.io/v1/convai/conversation" +
|
||||
"?agent_id=agent_9401kd09yfjnes2vddz1n29wev2t";
|
||||
|
||||
@OnOpen
|
||||
public void onOpen(Session session) throws Exception {
|
||||
log.info("已经有客户端链接啦:{}",session.getId());
|
||||
this.frontendSession = session;
|
||||
ClientEndpointConfig clientConfig =
|
||||
ClientEndpointConfig.Builder.create()
|
||||
.configurator(new ClientEndpointConfig.Configurator() {
|
||||
@Override
|
||||
public void beforeRequest(
|
||||
Map<String, List<String>> headers) {
|
||||
headers.put(
|
||||
"Authorization",
|
||||
List.of("Bearer "+"sk_dfe2b45e19bf8ad93a71d3a0faa61619a91e817df549d116" )
|
||||
);
|
||||
}
|
||||
})
|
||||
.build();
|
||||
WebSocketContainer container =
|
||||
ContainerProvider.getWebSocketContainer();
|
||||
agentClient = new ElevenLabsAgentEndpoint(msg -> {
|
||||
if (msg instanceof String) {
|
||||
frontendSession.getAsyncRemote()
|
||||
.sendText((String) msg);
|
||||
} else if (msg instanceof ByteBuffer) {
|
||||
frontendSession.getAsyncRemote()
|
||||
.sendBinary((ByteBuffer) msg);
|
||||
}
|
||||
});
|
||||
// ✅ 完全匹配的方法签名
|
||||
container.connectToServer(
|
||||
agentClient, // Endpoint 子类
|
||||
clientConfig,
|
||||
URI.create(AGENT_URL)
|
||||
);
|
||||
log.info("我开始准备发送启动的提示语音啦");
|
||||
//链接成功啦
|
||||
//发送初始化面试官语音流
|
||||
// String openingPathUrl = RuoYiConfig.getProfile() + VOICE_SYSTEM_DIR + "opening.wav";
|
||||
// sendVoiceBuffer(openingPathUrl, session);
|
||||
log.info("发送完毕啦");
|
||||
}
|
||||
|
||||
/** 前端 → ElevenLabs（JSON 控制） */
|
||||
@OnMessage
|
||||
public void onText(String message) {
|
||||
log.info("我收到前端发送过来的文本啦:{}",message);
|
||||
agentClient.sendText(message);
|
||||
}
|
||||
|
||||
/** 前端 → ElevenLabs（语音 PCM Binary） */
|
||||
@OnMessage
|
||||
public void onBinary(ByteBuffer buffer) {
|
||||
log.info("我收到前端发送过来的PCM语音流啦");
|
||||
//处理语音流,base64推送过去
|
||||
String bufferBase64 = convertByteBufferToBase64Pcm16k(buffer);
|
||||
Map<String,String> binaryMap = new HashMap<>();
|
||||
binaryMap.put("type","input_audio_buffer.append");
|
||||
binaryMap.put("audio",bufferBase64);
|
||||
String jsonStr = JSONUtil.toJsonStr(binaryMap);
|
||||
log.info("记录Agent对象是不是为空:{}",jsonStr);
|
||||
agentClient.sendBinaryText(jsonStr);
|
||||
|
||||
//发送结束的语音流-进行语音流提交
|
||||
// 重置 commit 定时器
|
||||
if (pendingCommit != null) {
|
||||
pendingCommit.cancel(false);
|
||||
}
|
||||
|
||||
pendingCommit = scheduler.schedule(() -> {
|
||||
log.info("No audio received, commit()");
|
||||
agentClient.commit();
|
||||
}, timeoutMs, TimeUnit.MILLISECONDS);
|
||||
|
||||
}
|
||||
|
||||
@OnClose
|
||||
public void onClose() throws IOException {
|
||||
agentClient.close();
|
||||
}
|
||||
|
||||
@OnError
|
||||
public void onError(Throwable t) {
|
||||
t.printStackTrace();
|
||||
}
|
||||
|
||||
/**
|
||||
* 发送语音流给前端
|
||||
*
|
||||
* @param pathUrl 语音文件地址
|
||||
* @param session 客户端会话
|
||||
*/
|
||||
private void sendVoiceBuffer(String pathUrl, Session session) {
|
||||
try {
|
||||
//文件转换成文件流
|
||||
ByteBuffer outByteBuffer = convertFileToByteBuffer(pathUrl);
|
||||
//发送文件流数据
|
||||
session.getAsyncRemote().sendBinary(outByteBuffer);
|
||||
try {
|
||||
Thread.sleep(200);
|
||||
}catch (Exception e){}
|
||||
//提示已经结束
|
||||
Map<String,String> dataText = new HashMap<>();
|
||||
dataText.put("type","voiceEnd");
|
||||
dataText.put("content","");
|
||||
session.getAsyncRemote().sendText(JSONUtil.toJsonStr(dataText));
|
||||
// 发送响应确认
|
||||
log.info("已经成功发送了语音流给前端:{}", DateUtil.now());
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* File 转换成 ByteBuffer
|
||||
*
|
||||
* @param fileUrl 文件路径
|
||||
* @return
|
||||
*/
|
||||
private ByteBuffer convertFileToByteBuffer(String fileUrl) {
|
||||
File file = new File(fileUrl);
|
||||
try {
|
||||
return ByteBuffer.wrap(FileUtils.readFileToByteArray(file));
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* 核心方法：ByteBuffer 转 Base64 编码的 16K PCM
|
||||
* @param buffer 原始16K PCM音频的ByteBuffer
|
||||
* @return Base64字符串（16K PCM格式）
|
||||
*/
|
||||
private String convertByteBufferToBase64Pcm16k(ByteBuffer buffer) {
|
||||
// 1. 从ByteBuffer提取字节数组（关键：避免越界）
|
||||
byte[] audioBytes = new byte[buffer.remaining()];
|
||||
buffer.get(audioBytes); // 读取数据到字节数组，buffer指针会移动
|
||||
buffer.rewind(); // 重置buffer指针（可选，便于后续复用）
|
||||
|
||||
// 2. 编码为Base64字符串（Java 8+ 原生支持）
|
||||
return Base64.getEncoder().encodeToString(audioBytes);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user