diff --git a/.idea/compiler.xml b/.idea/compiler.xml index 03667d8..7883a6b 100644 --- a/.idea/compiler.xml +++ b/.idea/compiler.xml @@ -11,8 +11,9 @@ - + + diff --git a/.idea/encodings.xml b/.idea/encodings.xml index 25bc32b..bb396d2 100644 --- a/.idea/encodings.xml +++ b/.idea/encodings.xml @@ -5,6 +5,8 @@ + + diff --git a/pom.xml b/pom.xml index fe16a91..7f87c0a 100644 --- a/pom.xml +++ b/pom.xml @@ -80,6 +80,13 @@ import + + + org.springframework.boot + spring-boot-starter-websocket + 2.5.15 + + ch.qos.logback @@ -254,6 +261,13 @@ ${vetti.version} + + + com.vetti + vetti-ai + ${vetti.version} + + com.squareup.okhttp3 @@ -300,6 +314,7 @@ vetti-quartz vetti-generator vetti-common + vetti-ai pom diff --git a/vetti-admin/pom.xml b/vetti-admin/pom.xml index 532d24f..6e12b32 100644 --- a/vetti-admin/pom.xml +++ b/vetti-admin/pom.xml @@ -25,6 +25,12 @@ true + + + org.springframework.boot + spring-boot-starter-websocket + + io.springfox @@ -69,7 +75,15 @@ vetti-generator - + + + com.vetti + vetti-ai + + + org.springframework.boot + spring-boot-starter-websocket + diff --git a/vetti-admin/src/main/java/com/vetti/socket/VoiceHandshakeInterceptor.java b/vetti-admin/src/main/java/com/vetti/socket/VoiceHandshakeInterceptor.java new file mode 100644 index 0000000..d2ba28e --- /dev/null +++ b/vetti-admin/src/main/java/com/vetti/socket/VoiceHandshakeInterceptor.java @@ -0,0 +1,36 @@ +package com.vetti.socket; + +import org.springframework.http.server.ServerHttpRequest; +import org.springframework.http.server.ServerHttpResponse; +import org.springframework.http.server.ServletServerHttpRequest; +import org.springframework.stereotype.Component; +import org.springframework.web.socket.WebSocketHandler; +import org.springframework.web.socket.server.HandshakeInterceptor; + +import java.util.Map; + +@Component +public class VoiceHandshakeInterceptor implements HandshakeInterceptor { + + @Override + public boolean beforeHandshake(ServerHttpRequest request, ServerHttpResponse response, + WebSocketHandler wsHandler, Map attributes) throws Exception { + // 从请求参数中获取客户端ID + if (request instanceof ServletServerHttpRequest) { + ServletServerHttpRequest servletRequest = (ServletServerHttpRequest) request; + String clientId = servletRequest.getServletRequest().getParameter("clientId"); + if (clientId != null && !clientId.isEmpty()) { + attributes.put("clientId", clientId); + System.out.println("客户端连接: " + clientId); + } + } + return true; + } + + @Override + public void afterHandshake(ServerHttpRequest request, ServerHttpResponse response, + WebSocketHandler wsHandler, Exception exception) { + // 握手后操作，可留空 + } +} + diff --git a/vetti-admin/src/main/java/com/vetti/socket/VoiceWebSocketHandler.java b/vetti-admin/src/main/java/com/vetti/socket/VoiceWebSocketHandler.java new file mode 100644 index 0000000..d9e24ca --- /dev/null +++ b/vetti-admin/src/main/java/com/vetti/socket/VoiceWebSocketHandler.java @@ -0,0 +1,156 @@ +package com.vetti.socket; + +import com.vetti.socket.vo.VoicePartMessage; +import org.springframework.stereotype.Component; +import org.springframework.web.socket.CloseStatus; +import org.springframework.web.socket.TextMessage; +import org.springframework.web.socket.WebSocketSession; +import org.springframework.web.socket.handler.TextWebSocketHandler; +import com.fasterxml.jackson.databind.ObjectMapper; +import java.io.*; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.*; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.locks.ReentrantLock; +import java.util.Base64; + +@Component +public class VoiceWebSocketHandler extends TextWebSocketHandler { + + // 存储每个客户端的语音分片，key: clientId, value: 分片映射 + private final Map> clientVoiceParts = new ConcurrentHashMap<>(); + // 存储每个客户端的总分片数，key: clientId + private final Map clientTotalParts = new ConcurrentHashMap<>(); + // 用于并发控制的锁 + private final Map clientLocks = new ConcurrentHashMap<>(); + // JSON序列化工具 + private final ObjectMapper objectMapper = new ObjectMapper(); + // 语音文件保存目录 + private static final String VOICE_STORAGE_DIR = "voice_files/"; + + public VoiceWebSocketHandler() { + // 初始化存储目录 + File dir = new File(VOICE_STORAGE_DIR); + if (!dir.exists()) { + dir.mkdirs(); + } + } + + @Override + public void afterConnectionEstablished(WebSocketSession session) throws Exception { + String clientId = getClientId(session); + if (clientId != null) { + // 初始化客户端数据结构 + clientVoiceParts.put(clientId, new TreeMap<>()); // TreeMap保证分片有序 + clientLocks.putIfAbsent(clientId, new ReentrantLock()); + System.out.println("客户端连接建立: " + clientId); + } + } + + @Override + protected void handleTextMessage(WebSocketSession session, TextMessage message) throws Exception { + String clientId = getClientId(session); + if (clientId == null) { + System.err.println("无法获取客户端ID"); + return; + } + + try { + // 解析前端发送的JSON消息 + VoicePartMessage voiceMessage = objectMapper.readValue(message.getPayload(), VoicePartMessage.class); + + // 处理语音分片 + if ("voice_part".equals(voiceMessage.getType())) { + processVoicePart(clientId, voiceMessage, session); + } + } catch (Exception e) { + System.err.println("处理消息出错: " + e.getMessage()); + e.printStackTrace(); + } + } + + @Override + public void afterConnectionClosed(WebSocketSession session, CloseStatus status) throws Exception { + String clientId = getClientId(session); + if (clientId != null) { + // 清理客户端资源 + clientVoiceParts.remove(clientId); + clientTotalParts.remove(clientId); + clientLocks.remove(clientId); + System.out.println("客户端连接关闭: " + clientId); + } + } + + /** + * 处理语音分片 + */ + private void processVoicePart(String clientId, VoicePartMessage message, WebSocketSession session) throws Exception { + ReentrantLock lock = clientLocks.get(clientId); + lock.lock(); // 加锁确保线程安全 + try { + // 保存总分片数 + clientTotalParts.put(clientId, message.getTotalParts()); + + // 解码Base64数据并存储分片 + byte[] voiceData = Base64.getDecoder().decode(message.getData()); + clientVoiceParts.get(clientId).put(message.getPartNumber(), voiceData); + + System.out.printf("接收客户端 %s 的分片 %d/%d%n", + clientId, message.getPartNumber() + 1, message.getTotalParts()); + + // 检查是否所有分片都已接收 + checkAndMergeParts(clientId, session); + } finally { + lock.unlock(); // 释放锁 + } + } + + /** + * 检查是否所有分片都已接收，如果是则合并 + */ + private void checkAndMergeParts(String clientId, WebSocketSession session) throws Exception { + Map parts = clientVoiceParts.get(clientId); + Integer totalParts = clientTotalParts.get(clientId); + + if (parts == null || totalParts == null) { + return; + } + + // 所有分片都已接收 + if (parts.size() == totalParts) { + System.out.println("所有分片接收完成，开始合并: " + clientId); + + // 生成唯一文件名 + String fileName = clientId + "_" + System.currentTimeMillis() + ".wav"; + Path outputPath = Paths.get(VOICE_STORAGE_DIR + fileName); + + // 合并分片 + try (FileOutputStream fos = new FileOutputStream(outputPath.toFile())) { + for (byte[] part : parts.values()) { + fos.write(part); + } + } + + System.out.println("语音文件合并完成，保存路径: " + outputPath); + + // 向客户端发送处理完成消息 + Map response = new HashMap<>(); + response.put("type", "complete"); + response.put("message", "语音接收完成"); + response.put("fileName", fileName); + session.sendMessage(new TextMessage(objectMapper.writeValueAsString(response))); + + // 清理已合并的分片数据 + clientVoiceParts.get(clientId).clear(); + } + } + + /** + * 从会话中获取客户端ID + */ + private String getClientId(WebSocketSession session) { + return (String) session.getAttributes().get("clientId"); + } +} diff --git a/vetti-admin/src/main/java/com/vetti/socket/WebSocketConfig.java b/vetti-admin/src/main/java/com/vetti/socket/WebSocketConfig.java new file mode 100644 index 0000000..0b52f1c --- /dev/null +++ b/vetti-admin/src/main/java/com/vetti/socket/WebSocketConfig.java @@ -0,0 +1,31 @@ +package com.vetti.socket; + +import org.springframework.context.annotation.Configuration; +import org.springframework.web.socket.config.annotation.EnableWebSocket; +import org.springframework.web.socket.config.annotation.WebSocketConfigurer; +import org.springframework.web.socket.config.annotation.WebSocketHandlerRegistry; + + +@Configuration +@EnableWebSocket +public class WebSocketConfig implements WebSocketConfigurer { + + private final VoiceWebSocketHandler voiceWebSocketHandler; + + private final VoiceHandshakeInterceptor voiceHandshakeInterceptor; + + // 构造函数注入 + public WebSocketConfig(VoiceWebSocketHandler voiceWebSocketHandler, + VoiceHandshakeInterceptor interceptor) { + this.voiceWebSocketHandler = voiceWebSocketHandler; + this.voiceHandshakeInterceptor = interceptor; + } + + @Override + public void registerWebSocketHandlers(WebSocketHandlerRegistry registry) { + // 注册WebSocket处理器，设置路径和允许跨域 + registry.addHandler(voiceWebSocketHandler, "/voice-websocket") + .addInterceptors(voiceHandshakeInterceptor) + .setAllowedOrigins("*"); // 生产环境应指定具体域名而非* + } +} diff --git a/vetti-admin/src/main/java/com/vetti/socket/vo/VoicePartMessage.java b/vetti-admin/src/main/java/com/vetti/socket/vo/VoicePartMessage.java new file mode 100644 index 0000000..47da4c1 --- /dev/null +++ b/vetti-admin/src/main/java/com/vetti/socket/vo/VoicePartMessage.java @@ -0,0 +1,47 @@ +package com.vetti.socket.vo; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; + +/** + * 语音分片消息实体类，对应前端发送的JSON结构 + */ +@JsonIgnoreProperties(ignoreUnknown = true) +public class VoicePartMessage { + private String type; // 消息类型，如"voice_part" + private int partNumber; // 分片编号，从0开始 + private int totalParts; // 总分片数 + private String data; // Base64编码的分片数据 + + // getter和setter + public String getType() { + return type; + } + + public void setType(String type) { + this.type = type; + } + + public int getPartNumber() { + return partNumber; + } + + public void setPartNumber(int partNumber) { + this.partNumber = partNumber; + } + + public int getTotalParts() { + return totalParts; + } + + public void setTotalParts(int totalParts) { + this.totalParts = totalParts; + } + + public String getData() { + return data; + } + + public void setData(String data) { + this.data = data; + } +} diff --git a/vetti-admin/src/main/resources/application-dev.yml b/vetti-admin/src/main/resources/application-dev.yml index df0fd10..8813b7c 100644 --- a/vetti-admin/src/main/resources/application-dev.yml +++ b/vetti-admin/src/main/resources/application-dev.yml @@ -142,11 +142,13 @@ verification: length: 5 # 验证码过期时间（分钟） expiration-minutes: 10 + # 文本转语音 elevenLabs: baseUrl: https://api.elevenlabs.io/v1 apiKey: sk_5240d8f56cb1eb5225fffcf903f62479884d1af5b3de6812 modelId: eleven_monolingual_v1 + # 语音转文本 whisper: apiUrl: https://api.openai.com/v1/audio/transcriptions diff --git a/vetti-ai/pom.xml b/vetti-ai/pom.xml new file mode 100644 index 0000000..573e42a --- /dev/null +++ b/vetti-ai/pom.xml @@ -0,0 +1,37 @@ + + + + + vetti-service + com.vetti + 3.9.0 + + + 4.0.0 + + vetti-ai + + + + + + io.swagger + swagger-models + + + + org.projectlombok + lombok + + + + + com.vetti + vetti-common + + + + + \ No newline at end of file diff --git a/vetti-ai/src/main/java/com/vetti/ai/service/ChatCommonService.java b/vetti-ai/src/main/java/com/vetti/ai/service/ChatCommonService.java new file mode 100644 index 0000000..d8d5f79 --- /dev/null +++ b/vetti-ai/src/main/java/com/vetti/ai/service/ChatCommonService.java @@ -0,0 +1,13 @@ +package com.vetti.ai.service; + +/** + * 面试聊天共通 服务层 + */ +public interface ChatCommonService { + + + /** + * 处理面试聊天语音结果数据 + */ + public void handleChatVoiceData(); +} diff --git a/vetti-ai/src/main/java/com/vetti/ai/service/impl/ChatCommonServiceImpl.java b/vetti-ai/src/main/java/com/vetti/ai/service/impl/ChatCommonServiceImpl.java new file mode 100644 index 0000000..ffa634e --- /dev/null +++ b/vetti-ai/src/main/java/com/vetti/ai/service/impl/ChatCommonServiceImpl.java @@ -0,0 +1,35 @@ +package com.vetti.ai.service.impl; + +import com.vetti.ai.service.ChatCommonService; +import com.vetti.common.ai.elevenLabs.ElevenLabsClient; +import com.vetti.common.ai.gpt.ChatGPTClient; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + + +/** + * 聊天面试共通 服务层实现 + */ +@Service +public class ChatCommonServiceImpl implements ChatCommonService { + + @Autowired + private ElevenLabsClient elevenLabsClient; + + @Autowired + private ChatGPTClient chatGPTClient; + + @Override + public void handleChatVoiceData() { + //1、获取面试传输的语音文件 + + //2、语音文件转换成文本字符串 + + //3、把文本传输到GPT中,等待回复 + + //4、GPT返回的结果,文本转成语音文件 + + //5、返回最终的语音文件 + + } +}