package com.vetti.socket.agents; import javax.websocket.*; import javax.websocket.server.ServerEndpoint; import cn.hutool.core.date.DateUtil; import cn.hutool.json.JSONUtil; import com.vetti.common.config.RuoYiConfig; import lombok.extern.slf4j.Slf4j; import org.apache.commons.io.FileUtils; import org.springframework.stereotype.Component; import java.io.File; import java.io.IOException; import java.net.URI; import java.nio.ByteBuffer; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Base64; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.ScheduledFuture; import java.util.concurrent.TimeUnit; @Slf4j @ServerEndpoint("/voice-websocket/elevenLabsAgent/{clientId}") @Component public class VoiceBridgeEndpoint { // 语音文件保存目录 private static final String VOICE_STORAGE_DIR = "/voice_files/"; // 语音结果文件保存目录 private static final String VOICE_STORAGE_RESULT_DIR = "/voice_result_files/"; // 系统语音目录 private static final String VOICE_SYSTEM_DIR = "/system_files/"; private Session frontendSession; private ElevenLabsAgentEndpoint agentClient; private final ScheduledExecutorService scheduler = Executors.newSingleThreadScheduledExecutor(); private ScheduledFuture pendingCommit; private final long timeoutMs = 600; private static final String AGENT_URL = "wss://api.elevenlabs.io/v1/convai/conversation" + "?agent_id=agent_9401kd09yfjnes2vddz1n29wev2t"; @OnOpen public void onOpen(Session session) throws Exception { log.info("已经有客户端链接啦:{}",session.getId()); this.frontendSession = session; ClientEndpointConfig clientConfig = ClientEndpointConfig.Builder.create() .configurator(new ClientEndpointConfig.Configurator() { @Override public void beforeRequest( Map> headers) { headers.put( "Authorization", List.of("Bearer "+"sk_dfe2b45e19bf8ad93a71d3a0faa61619a91e817df549d116" ) ); } }) .build(); WebSocketContainer container = ContainerProvider.getWebSocketContainer(); agentClient = new ElevenLabsAgentEndpoint(msg -> { if (msg instanceof String) { frontendSession.getAsyncRemote() .sendText((String) msg); } else if (msg instanceof ByteBuffer) { frontendSession.getAsyncRemote() .sendBinary((ByteBuffer) msg); } }); // ✅ 完全匹配的方法签名 container.connectToServer( agentClient, // Endpoint 子类 clientConfig, URI.create(AGENT_URL) ); log.info("我开始准备发送启动的提示语音啦"); //链接成功啦 //发送初始化面试官语音流 // String openingPathUrl = RuoYiConfig.getProfile() + VOICE_SYSTEM_DIR + "opening.wav"; // sendVoiceBuffer(openingPathUrl, session); log.info("发送完毕啦"); } /** 前端 → ElevenLabs(JSON 控制) */ @OnMessage public void onText(String message) { log.info("我收到前端发送过来的文本啦:{}",message); agentClient.sendText(message); } /** 前端 → ElevenLabs(语音 PCM Binary) */ @OnMessage public void onBinary(ByteBuffer buffer) { log.info("我收到前端发送过来的PCM语音流啦"); //处理语音流,base64推送过去 String bufferBase64 = convertByteBufferToBase64Pcm16k(buffer); Map binaryMap = new HashMap<>(); binaryMap.put("type","input_audio_buffer.append"); binaryMap.put("audio",bufferBase64); String jsonStr = JSONUtil.toJsonStr(binaryMap); log.info("记录Agent对象是不是为空:{}",jsonStr); agentClient.sendBinaryText(jsonStr); //发送结束的语音流-进行语音流提交 // 重置 commit 定时器 if (pendingCommit != null) { pendingCommit.cancel(false); } pendingCommit = scheduler.schedule(() -> { log.info("No audio received, commit()"); agentClient.commit(); }, timeoutMs, TimeUnit.MILLISECONDS); } @OnClose public void onClose() throws IOException { agentClient.close(); } @OnError public void onError(Throwable t) { t.printStackTrace(); } /** * 发送语音流给前端 * * @param pathUrl 语音文件地址 * @param session 客户端会话 */ private void sendVoiceBuffer(String pathUrl, Session session) { try { //文件转换成文件流 ByteBuffer outByteBuffer = convertFileToByteBuffer(pathUrl); //发送文件流数据 session.getAsyncRemote().sendBinary(outByteBuffer); try { Thread.sleep(200); }catch (Exception e){} //提示已经结束 Map dataText = new HashMap<>(); dataText.put("type","voiceEnd"); dataText.put("content",""); session.getAsyncRemote().sendText(JSONUtil.toJsonStr(dataText)); // 发送响应确认 log.info("已经成功发送了语音流给前端:{}", DateUtil.now()); } catch (Exception e) { e.printStackTrace(); } } /** * File 转换成 ByteBuffer * * @param fileUrl 文件路径 * @return */ private ByteBuffer convertFileToByteBuffer(String fileUrl) { File file = new File(fileUrl); try { return ByteBuffer.wrap(FileUtils.readFileToByteArray(file)); } catch (Exception e) { e.printStackTrace(); } return null; } /** * 核心方法:ByteBuffer 转 Base64 编码的 16K PCM * @param buffer 原始16K PCM音频的ByteBuffer * @return Base64字符串(16K PCM格式) */ private String convertByteBufferToBase64Pcm16k(ByteBuffer buffer) { // 1. 从ByteBuffer提取字节数组(关键:避免越界) byte[] audioBytes = new byte[buffer.remaining()]; buffer.get(audioBytes); // 读取数据到字节数组,buffer指针会移动 buffer.rewind(); // 重置buffer指针(可选,便于后续复用) // 2. 编码为Base64字符串(Java 8+ 原生支持) return Base64.getEncoder().encodeToString(audioBytes); } }