上传新的alops_sys.sql文件

dev
0214 10 months ago committed by Newek
parent 92d0e01d36
commit 342fe3af5a
  1. 17
      ALOps_sys_backend/alops-admin/pom.xml
  2. 1
      ALOps_sys_backend/alops-admin/src/main/java/com/alops/AlopsApplication.java
  3. 286
      ALOps_sys_backend/alops-admin/src/main/java/com/alops/web/controller/LlmKnowledgeController/KnowLedgeController.java
  4. 139
      ALOps_sys_backend/alops-admin/src/main/java/com/alops/web/controller/LlmKnowledgeController/QAController.java
  5. 21
      ALOps_sys_backend/alops-admin/src/main/resources/application.yml
  6. 152
      ALOps_sys_backend/alops-chat/pom.xml
  7. 34
      ALOps_sys_backend/alops-chat/src/main/java/com/alops/chat/DTO/AnswerResponse.java
  8. 61
      ALOps_sys_backend/alops-chat/src/main/java/com/alops/chat/DTO/EmbeddingResponse.java
  9. 20
      ALOps_sys_backend/alops-chat/src/main/java/com/alops/chat/DTO/QuestionRequest.java
  10. 94
      ALOps_sys_backend/alops-chat/src/main/java/com/alops/chat/entity/KnowledgeEntity.java
  11. 60
      ALOps_sys_backend/alops-chat/src/main/java/com/alops/chat/entity/UploadedFile.java
  12. 55
      ALOps_sys_backend/alops-chat/src/main/java/com/alops/chat/repository/KnowledgeRepository.java
  13. 750
      ALOps_sys_backend/alops-chat/src/main/java/com/alops/chat/service/AItokenService.java
  14. 113
      ALOps_sys_backend/alops-chat/src/main/java/com/alops/chat/service/EmbeddingService.java
  15. 469
      ALOps_sys_backend/alops-chat/src/main/java/com/alops/chat/service/KnowledgeService.java
  16. 298
      ALOps_sys_backend/alops-chat/src/main/java/com/alops/chat/service/TextSplitter.java
  17. 52
      ALOps_sys_backend/alops-chat/src/main/java/com/alops/chat/service/UploadedFileService.java
  18. 32
      ALOps_sys_backend/alops-common/src/main/java/com/alops/common/utils/file/FileUploadUtils.java
  19. 2
      ALOps_sys_backend/alops-common/src/main/java/com/alops/common/utils/file/MimeTypeUtils.java
  20. 11
      ALOps_sys_backend/alops-framework/src/main/java/com/alops/framework/config/ApplicationConfig.java
  21. 9
      ALOps_sys_backend/alops-system/pom.xml
  22. 4
      ALOps_sys_backend/alops-system/src/main/java/com/alops/system/job/GatherJob.java
  23. 2534
      ALOps_sys_sql/aiops_sys.sql

@ -11,6 +11,8 @@
<modelVersion>4.0.0</modelVersion>
<packaging>jar</packaging>
<artifactId>alops-admin</artifactId>
<!-- 不需要显式声明版本,会自动继承父项目的版本1.0.0 -->
<description>
web服务入口
@ -45,10 +47,11 @@
<version>1.6.2</version>
</dependency>
<!-- Mysql驱动包 -->
<!-- Mysql驱动包,使用新的Maven坐标 -->
<dependency>
<groupId>mysql</groupId>
<artifactId>mysql-connector-java</artifactId>
<groupId>com.mysql</groupId>
<artifactId>mysql-connector-j</artifactId>
<version>8.0.33</version>
</dependency>
<!-- 核心模块-->
@ -76,12 +79,20 @@
<artifactId>alops-generator</artifactId>
</dependency>
<!-- 邮件工具-->
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-mail</artifactId>
<version>2.5.15</version>
</dependency>
<dependency>
<groupId>com.alops</groupId>
<artifactId>alops-chat</artifactId>
<version>1.0.0</version>
<scope>compile</scope>
</dependency>
</dependencies>

@ -11,6 +11,7 @@ import org.springframework.context.annotation.ComponentScan;
* @author ruoyi
*/
@SpringBootApplication (exclude = { DataSourceAutoConfiguration.class })
@ComponentScan(basePackages = {"com.alops.web", "com.alops.framework", "com.alops.system", "com.alops.quartz", "com.alops.common", "com.alops.generator", "com.alops.chat"})
public class AlopsApplication

@ -0,0 +1,286 @@
package com.alops.web.controller.LlmKnowledgeController;
import com.alops.chat.entity.KnowledgeEntity;
import com.alops.chat.entity.UploadedFile;
import com.alops.chat.service.KnowledgeService;
import com.alops.chat.service.UploadedFileService;
import com.alops.common.constant.HttpStatus;
import com.alops.common.core.controller.BaseController;
import com.alops.common.core.domain.AjaxResult;
import com.alops.common.core.page.TableDataInfo;
import com.alops.common.config.RuoYiConfig;
import com.alops.common.utils.file.FileUploadUtils;
import com.alops.common.utils.file.MimeTypeUtils;
import io.swagger.annotations.ApiOperation;
import io.swagger.annotations.ApiParam;
import org.apache.poi.xwpf.usermodel.XWPFDocument;
import org.apache.poi.xwpf.usermodel.XWPFParagraph;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.security.access.prepost.PreAuthorize;
import org.springframework.web.bind.annotation.*;
import org.springframework.web.multipart.MultipartFile;
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.StandardCopyOption;
import java.time.LocalDateTime;
import java.time.format.DateTimeFormatter;
import java.util.ArrayList;
import java.util.List;
/**
* 文件上传控制器
* 提供文件导入功能将文本文档转化为知识条目
*/
@RestController
@RequestMapping("/QandA/knowledge/")
@CrossOrigin(origins = "*")
public class KnowLedgeController extends BaseController {
@Autowired
private KnowledgeService knowledgeService;
@Autowired
private UploadedFileService uploadedFileService;
// 文件存储目录
private static final String UPLOAD_DIR = "uploads/";
/**
* 上传并处理文本文档
*
* @param file 上传的文件
* @param source 文件来源描述
* @return 处理结果
*/
@PreAuthorize("@ss.hasPermi('QandA:knowledge:uploadfile')")
@PostMapping("/upload")
@ApiOperation("上传并处理文本文档")
public AjaxResult uploadFile(
@RequestPart("file")
@ApiParam(value = "知识库文件", type = "file") MultipartFile file,
@ModelAttribute("source") String source) {
try {
String content;
// 判断文件类型并读取内容
if (file.getOriginalFilename() != null && file.getOriginalFilename().endsWith(".docx")) {
// 处理docx文件
content = readDocxFile(file);
} else {
// 处理txt文件
content = readTxtFile(file);
}
// 使用优化后的文件上传工具保存文件到本地并记录信息到数据库
// 上传文件并保持原始文件名,只允许 TEXT_EXTENSION 类型的文件
String filePath = FileUploadUtils.uploadWithOriginalFilename(
RuoYiConfig.getProfile(),
file,
MimeTypeUtils.TEXT_EXTENSION);
String fileExtension = getFileExtension(file.getOriginalFilename());
UploadedFile uploadedFile = new UploadedFile(filePath, source, fileExtension, null);
uploadedFile = uploadedFileService.save(uploadedFile);
// 获取保存后的文件ID
Long fileId = uploadedFile.getId();
// 处理文件内容并添加到知识库,将文件ID作为keyfileid参数传递
List<Long> knowledgeIds = knowledgeService.addKnowledgeWithFileId(content, source, fileId);
// 更新上传文件信息(如果有知识条目被创建)
if (!knowledgeIds.isEmpty()) {
// 可以在这里添加其他需要的更新逻辑
}
return AjaxResult.success("文件上传并处理成功");
} catch (Exception e) {
e.printStackTrace();
return AjaxResult.error("文件处理失败: " + e.getMessage());
}
}
/**
* 读取docx文件的内容
*
* @param file 上传的docx文件
* @return 文件内容字符串
* @throws IOException IO异常
*/
private String readDocxFile(MultipartFile file) throws IOException {
try (XWPFDocument document = new XWPFDocument(file.getInputStream())) {
StringBuilder content = new StringBuilder();
// 遍历所有段落并提取文本
for (XWPFParagraph paragraph : document.getParagraphs()) {
content.append(paragraph.getText()).append("\n");
}
return content.toString();
}
}
/**
* 读取txt文件的内容
*
* @param file 上传的txt文件
* @return 文件内容字符串
* @throws IOException IO异常
*/
private String readTxtFile(MultipartFile file) throws IOException {
StringBuilder content = new StringBuilder();
try (BufferedReader reader = new BufferedReader(
new InputStreamReader(file.getInputStream(), StandardCharsets.UTF_8))) {
String line;
while ((line = reader.readLine()) != null) {
content.append(line).append("\n");
}
}
return content.toString();
}
/**
* 获取文件扩展名
*
* @param filename 文件名
* @return 文件扩展名
*/
private String getFileExtension(String filename) {
if (filename == null || filename.lastIndexOf(".") == -1) {
return "";
}
return filename.substring(filename.lastIndexOf(".") + 1);
}
/**
* 添加知识条目接口
*
* @param content 知识内容
* @param source 知识来源
* @return 添加结果
*/
@PreAuthorize("@ss.hasPermi('QandA:knowledge:upload')")
@PostMapping("/add-knowledge")
@ApiOperation("添加知识条目接口")
public AjaxResult addKnowledge(
@RequestParam String content,
@RequestParam String source) {
try {
knowledgeService.addKnowledge(content, source);
return AjaxResult.success("知识添加成功");
} catch (Exception e) {
e.printStackTrace();
return AjaxResult.error("知识添加失败: " + e.getMessage());
}
}
/**
* 获取所有正式知识条目分页
*
* @return 正式知识条目列表
*/
@PreAuthorize("@ss.hasPermi('QandA:knowledge:permquery')")
@GetMapping("/permanent-knowledge")
@ApiOperation("获取所有正式知识条目")
public TableDataInfo getPermanentKnowledge() {
startPage(); // 开启分页
List<KnowledgeEntity> permanentEntries = knowledgeService.getPermanentEntries();
return getDataTable(permanentEntries);
}
/**
* 获取所有上传文件信息分页
*
* @return 上传文件信息列表
*/
@PreAuthorize("@ss.hasPermi('QandA:knowledge:filelist')")
@GetMapping("/uploaded-files")
@ApiOperation("获取所有上传文档")
public TableDataInfo getUploadedFiles() {
startPage(); // 开启分页
List<UploadedFile> uploadedFiles = uploadedFileService.findAll();
return getDataTable(uploadedFiles);
}
/**
* 更新知识条目接口
*
* @param id 知识条目ID
* @param content 知识内容
* @param source 知识来源
* @return 更新结果
*/
@PreAuthorize("@ss.hasPermi('QandA:knowledge:update')")
@PutMapping("/update-knowledge")
@ApiOperation("更新知识条目")
public AjaxResult updateKnowledge(
@RequestParam Long id,
@RequestParam String content,
@RequestParam String source) {
try {
boolean updated = knowledgeService.updateKnowledge(id, content, source);
if (updated) {
return AjaxResult.success("知识条目更新成功");
} else {
return AjaxResult.error(HttpStatus.NOT_FOUND, "未找到指定的知识条目");
}
} catch (Exception e) {
e.printStackTrace();
return AjaxResult.error("知识条目更新失败: " + e.getMessage());
}
}
/**
* 删除知识条目接口
*
* @param id 知识条目ID
* @return 删除结果
*/
@PreAuthorize("@ss.hasPermi('QandA:knowledge:del')")
@DeleteMapping("/knowledge/{id}")
@ApiOperation("删除知识条目接口")
public AjaxResult deleteKnowledge(@PathVariable Long id) {
try {
boolean deleted = knowledgeService.deleteKnowledge(id);
if (deleted) {
return AjaxResult.success("知识条目删除成功");} else {
return AjaxResult.error(HttpStatus.NOT_FOUND, "未找到指定的知识条目");
}
} catch (Exception e) {
e.printStackTrace();
return AjaxResult.error("知识条目删除失败: " + e.getMessage());
}
}
/**
* 删除上传文件信息接口
*
* @param id 上传文件ID
* @return 删除结果
*/
@ApiOperation("删除上传文件信息")
@PreAuthorize("@ss.hasPermi('QandA:knowledge:filedel')")
@DeleteMapping("/uploaded-files/{id}")
public AjaxResult deleteUploadedFile(@PathVariable Long id) {
try {
// 先删除关联的知识条目
knowledgeService.deleteKnowledgeByFileId(id);
// 再删除文件信息
uploadedFileService.deleteById(id);
return AjaxResult.success("上传文件信息删除成功");
} catch (Exception e) {
e.printStackTrace();
return AjaxResult.error("上传文件信息删除失败: " + e.getMessage());
}
}
}

@ -0,0 +1,139 @@
package com.alops.web.controller.LlmKnowledgeController;
import com.alops.chat.DTO.AnswerResponse;
import com.alops.chat.DTO.QuestionRequest;
import com.alops.chat.entity.KnowledgeEntity;
import com.alops.chat.service.AItokenService;
import com.alops.chat.service.KnowledgeService;
import com.alops.common.core.domain.AjaxResult;
import io.swagger.annotations.ApiOperation;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.security.access.prepost.PreAuthorize;
import org.springframework.web.bind.annotation.*;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
/**
* 问答系统控制器
* 提供基于知识库的问答接口
*/
@RestController
@RequestMapping("/QandA/knowledge/")
@CrossOrigin(origins = "*")
public class QAController {
@Autowired
private KnowledgeService knowledgeService;
@Autowired
private AItokenService AItokenService;
// 假数据答案
@Value("${mock.qa.answer:这是默认的模拟答案}")
private String mockAnswer;
// 假数据来源
@Value("${mock.qa.sources:模拟来源1,模拟来源2}")
private String mockSources;
/**
* POST方式问答接口
* @param request 包含问题的请求对象
* @return 答案响应
*/
@PreAuthorize("@ss.hasPermi('QandA:Aiassistant:ask')")
@PostMapping("/ask")
@ApiOperation("智能问答提问")
public AjaxResult askQuestion(@RequestBody QuestionRequest request) {
try {
// 检查是否使用假数据模式
if (AItokenService.isMockEnabled()) {
System.out.println("使用假数据模式返回问答结果");
String[] sources = mockSources.split(",");
AnswerResponse answerResponse = new AnswerResponse(mockAnswer, sources);
return AjaxResult.success("问答成功(假数据模式)", answerResponse);
}
// 1. 查找相关知识(最相似的8条)
List<KnowledgeEntity> similarKnowledge = knowledgeService.findSimilarKnowledge(request.getQuestion(), 8);
// 如果没有找到相关知识,返回提示信息
if (similarKnowledge.isEmpty()) {
// 创建一个专门的提示信息,说明知识库中未找到相关内容
String noAnswerPrompt = "知识库中没有找到与您的问题直接相关的内容。您的问题可能是关于 \"" +
request.getQuestion() + "\"。请确认问题表述是否准确,或者尝试重新表述问题。" +
"如果这是一个重要的问题,请联系管理员添加相关知识。";
AnswerResponse answerResponse = new AnswerResponse(noAnswerPrompt, new String[0]);
return AjaxResult.success(noAnswerPrompt, answerResponse);
}
// 2. 生成上下文
String context = knowledgeService.generateContext(similarKnowledge);
// 检查上下文是否为空
if (context == null || context.trim().isEmpty()) {
String errorMessage = "系统未能从知识库中提取有效上下文信息。";
AnswerResponse errorResponse = new AnswerResponse(errorMessage, new String[0]);
return AjaxResult.error(errorMessage, errorResponse);
}
// 3. 构造消息列表并调用Qwen3生成答案
List<Map<String, String>> messages = new ArrayList<>();
Map<String, String> systemMessage = new HashMap<>();
systemMessage.put("role", "system");
systemMessage.put("content", "你是一个智能运维助手,根据提供的上下文回答问题。如果上下文中没有相关信息,请说明无法基于提供的上下文回答问题。");
messages.add(systemMessage);
Map<String, String> userMessage = new HashMap<>();
userMessage.put("role", "user");
userMessage.put("content", "上下文信息:\n" + context + "\n\n请根据以上上下文信息回答问题:" + request.getQuestion());
messages.add(userMessage);
String answer = AItokenService.generateAnswer(messages);
// 检查生成的答案是否为空或默认加载信息
if (answer == null || answer.trim().isEmpty()) {
String errorMessage = "系统未能生成有效答案,请稍后再试。";
AnswerResponse errorResponse = new AnswerResponse(errorMessage, new String[0]);
return AjaxResult.error(errorMessage, errorResponse);
}
// 检查是否是模型加载信息
if (answer.contains("模型正在加载中")) {
AnswerResponse answerResponse = new AnswerResponse(answer, new String[0]);
return AjaxResult.error("模型正在加载中", answerResponse);
}
// 检查是否是连接错误信息
if (answer.contains("无法连接到Ollama服务")) {
AnswerResponse answerResponse = new AnswerResponse(answer, new String[0]);
return AjaxResult.error("服务连接失败", answerResponse);
}
// 4. 提取来源信息
String[] sources = similarKnowledge.stream()
.map(KnowledgeEntity::getSource)
.distinct()
.toArray(String[]::new);
AnswerResponse answerResponse = new AnswerResponse(answer, sources);
return AjaxResult.success("问答成功", answerResponse);
} catch (Exception e) {
e.printStackTrace();
AnswerResponse errorResponse = new AnswerResponse(
"系统处理问题时发生错误,请稍后再试。", new String[0]);
return AjaxResult.error("系统处理问题时发生错误,请稍后再试。", errorResponse);
}
}
}

@ -49,6 +49,9 @@ user:
# Spring配置
Spring:
# 允许Bean定义覆盖
main:
allow-bean-definition-overriding: true
# 邮件信息
mail:
host: smtp.qq.com
@ -88,7 +91,7 @@ Spring:
# 数据库索引
database: 0
# 密码
password:
password:
# 连接超时时间
timeout: 10s
lettuce:
@ -101,10 +104,7 @@ Spring:
max-active: 8
# #连接池最大阻塞等待时间(使用负值表示没有限制)
max-wait: -1ms
#邮件信息
# 邮件信息
# token配置
token:
@ -146,4 +146,13 @@ xss:
# 匹配链接
urlPatterns: /system/*,/monitor/*,/tool/*
# 添加假数据配置
mock:
enabled: true
# 问答接口假数据配置
qa:
answer: "这是一个模拟的答案。在真实环境中,这将是由AI模型生成的答案。"
sources: "模拟来源1,模拟来源2"
# 知识库接口假数据配置
knowledge:
enabled: true

@ -0,0 +1,152 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<groupId>com.alops</groupId>
<artifactId>alops</artifactId>
<version>1.0.0</version>
<relativePath>../pom.xml</relativePath>
</parent>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<configuration>
<source>9</source>
<target>9</target>
</configuration>
</plugin>
</plugins>
</build>
<modelVersion>4.0.0</modelVersion>
<artifactId>alops-chat</artifactId>
<description>
common通用工具
</description>
<dependencies>
<!-- org.snmp4j -->
<dependency>
<groupId>org.snmp4j</groupId>
<artifactId>snmp4j</artifactId>
<version>3.9.6</version>
</dependency>
<!-- Spring框架基本的核心工具 -->
<dependency>
<groupId>org.springframework</groupId>
<artifactId>spring-context-support</artifactId>
</dependency>
<!-- SpringWeb模块 -->
<dependency>
<groupId>org.springframework</groupId>
<artifactId>spring-web</artifactId>
</dependency>
<!-- spring security 安全认证 -->
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-security</artifactId>
</dependency>
<!-- pagehelper 分页插件 -->
<dependency>
<groupId>com.github.pagehelper</groupId>
<artifactId>pagehelper-spring-boot-starter</artifactId>
</dependency>
<!-- 自定义验证注解 -->
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-validation</artifactId>
</dependency>
<!--常用工具类 -->
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId>
</dependency>
<!-- JSON工具类 -->
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
</dependency>
<!-- 阿里JSON解析器 -->
<dependency>
<groupId>com.alibaba.fastjson2</groupId>
<artifactId>fastjson2</artifactId>
</dependency>
<!-- io常用工具类 -->
<dependency>
<groupId>commons-io</groupId>
<artifactId>commons-io</artifactId>
</dependency>
<!-- excel工具 -->
<dependency>
<groupId>org.apache.poi</groupId>
<artifactId>poi-ooxml</artifactId>
</dependency>
<!-- yml解析器 -->
<dependency>
<groupId>org.yaml</groupId>
<artifactId>snakeyaml</artifactId>
</dependency>
<!-- Token生成与解析-->
<dependency>
<groupId>io.jsonwebtoken</groupId>
<artifactId>jjwt</artifactId>
</dependency>
<!-- Jaxb -->
<dependency>
<groupId>javax.xml.bind</groupId>
<artifactId>jaxb-api</artifactId>
</dependency>
<!-- redis 缓存操作 -->
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-data-redis</artifactId>
</dependency>
<!-- pool 对象池 -->
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-pool2</artifactId>
</dependency>
<!-- 解析客户端操作系统、浏览器等 -->
<dependency>
<groupId>eu.bitwalker</groupId>
<artifactId>UserAgentUtils</artifactId>
</dependency>
<!-- servlet包 -->
<dependency>
<groupId>javax.servlet</groupId>
<artifactId>javax.servlet-api</artifactId>
</dependency>
<dependency>
<groupId>org.projectlombok</groupId>
<artifactId>lombok</artifactId>
<version>1.18.36</version>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-data-jpa</artifactId>
</dependency>
</dependencies>
</project>

@ -0,0 +1,34 @@
// dto/AnswerResponse.java
package com.alops.chat.DTO;
public class AnswerResponse {
private String answer;
private String[] sources;
public AnswerResponse() {}
public AnswerResponse(String answer, String[] sources) {
this.answer = answer;
this.sources = sources;
}
public String getAnswer() {
return answer;
}
public void setAnswer(String answer) {
this.answer = answer;
}
public String[] getSources() {
return sources;
}
public void setSources(String[] sources) {
this.sources = sources;
}
}

@ -0,0 +1,61 @@
// dto/EmbeddingResponse.java
package com.alops.chat.DTO;
import java.util.List;
public class EmbeddingResponse {
private boolean success;
private List<Float> vector;
private int dimension;
private String errorMessage; // 添加错误信息字段
public EmbeddingResponse() {}
public EmbeddingResponse(boolean success, List<Float> vector, int dimension) {
this.success = success;
this.vector = vector;
this.dimension = dimension;
}
// 新增构造函数,包含错误信息
public EmbeddingResponse(boolean success, List<Float> vector, int dimension, String errorMessage) {
this.success = success;
this.vector = vector;
this.dimension = dimension;
this.errorMessage = errorMessage;
}
// Getter和Setter方法
public boolean isSuccess() {
return success;
}
public void setSuccess(boolean success) {
this.success = success;
}
public List<Float> getVector() {
return vector;
}
public void setVector(List<Float> vector) {
this.vector = vector;
}
public int getDimension() {
return dimension;
}
public void setDimension(int dimension) {
this.dimension = dimension;
}
// 错误信息的Getter和Setter方法
public String getErrorMessage() {
return errorMessage;
}
public void setErrorMessage(String errorMessage) {
this.errorMessage = errorMessage;
}
}

@ -0,0 +1,20 @@
// dto/QuestionRequest.java
package com.alops.chat.DTO;
public class QuestionRequest {
private String question;
public QuestionRequest() {}
public QuestionRequest(String question) {
this.question = question;
}
public String getQuestion() {
return question;
}
public void setQuestion(String question) {
this.question = question;
}
}

@ -0,0 +1,94 @@
// entity/KnowledgeEntry.java
package com.alops.chat.entity;
import lombok.Data;
import javax.persistence.*;
import java.time.LocalDateTime;
/**
* 知识条目实体类
* 用于存储知识库中的条目信息
*/
@Entity
@Table(name = "knowledge_entries")
@Data
public class KnowledgeEntity {
// 主键,自动生成
@Id
@GeneratedValue(strategy = GenerationType.IDENTITY)
private Long id;
// 知识内容,使用TEXT类型存储
@Column(columnDefinition = "TEXT")
private String content;
// 向量嵌入,使用Lob注解存储大型字节数组
@Lob
private byte[] embedding;
// 知识来源
private String source;
// 创建时间,实体被创建时自动设置
@Column(name = "createtime")
private LocalDateTime createtime;
// 更新时间,实体每次更新时自动设置
@Column(name = "updatetime")
private LocalDateTime updatetime;
// 对应的files的id
@Column(name = "keyfileid")
private Integer keyfileid;;
// 默认构造方法
public KnowledgeEntity() {
}
/**
* 构造方法
* @param content 知识内容
* @param embedding 向量嵌入
* @param source 知识来源
*/
public KnowledgeEntity(String content, byte[] embedding, String source,int keyfileid) {
this.content = content;
this.embedding = embedding;
this.source = source;
this.createtime = LocalDateTime.now();
this.updatetime = LocalDateTime.now();
this.keyfileid =keyfileid; // 默认为正式条目
}
/**
* 构造方法用于临时条目
* @param content 知识内容
* @param source 知识来源
* @param keyfileid 对应的id
*/
public KnowledgeEntity(String content, String source, int keyfileid) {
this.content = content;
this.source = source;
this.createtime = LocalDateTime.now();
this.updatetime = LocalDateTime.now();
this.keyfileid = keyfileid;
}
// JPA生命周期回调方法,在实体被更新时自动设置更新时间
@PreUpdate
public void onPreUpdate() {
this.updatetime = LocalDateTime.now();
}
// JPA生命周期回调方法,在实体被持久化之前自动设置创建时间
@PrePersist
public void onPrePersist() {
this.createtime = LocalDateTime.now();
this.updatetime = LocalDateTime.now();
}
}

@ -0,0 +1,60 @@
package com.alops.chat.entity;
import lombok.Data;
import javax.persistence.*;
import java.time.LocalDateTime;
/**
* 上传文件信息实体类
* 用于存储上传文件的信息包括路径时间格式等
*/
@Entity
@Table(name = "uploaded_files")
@Data
public class UploadedFile {
// 主键,自动生成
@Id
@GeneratedValue(strategy = GenerationType.IDENTITY)
private Long id;
// 文件保存路径
@Column(name = "file_path")
private String filePath;
// 上传时间
@Column(name = "upload_time")
private LocalDateTime uploadTime;
// 文件来源名称
private String source;
// 文件格式(扩展名)
private String format;
// 默认构造方法
public UploadedFile() {}
/**
* 构造方法
* @param filePath 文件保存路径
* @param source 文件来源名称
* @param format 文件格式
* @param knowledgeId 对应的知识条目ID
*/
public UploadedFile(String filePath, String source, String format, Long knowledgeId) {
this.filePath = filePath;
this.source = source;
this.format = format;
this.uploadTime = LocalDateTime.now();
}
// JPA生命周期回调方法,在实体被持久化之前自动设置上传时间
@PrePersist
public void onPrePersist() {
this.uploadTime = LocalDateTime.now();
}
}

@ -0,0 +1,55 @@
// repository/KnowledgeRepository.java
package com.alops.chat.repository;
import com.alops.chat.entity.KnowledgeEntity;
import org.springframework.data.jpa.repository.JpaRepository;
import org.springframework.data.jpa.repository.Modifying;
import org.springframework.data.jpa.repository.Query;
import org.springframework.data.repository.query.Param;
import org.springframework.stereotype.Repository;
import org.springframework.transaction.annotation.Transactional;
import java.util.List;
/**
* 知识条目数据访问接口
* 提供对知识条目的增删改查操作
*/
@Repository
public interface KnowledgeRepository extends JpaRepository<KnowledgeEntity, Long> {
/**
* 根据关键词模糊搜索知识条目
* @param keyword 搜索关键词
* @return 匹配的知识条目列表
*/
@Query("SELECT k FROM KnowledgeEntity k WHERE k.content LIKE %:keyword%")
List<KnowledgeEntity> findByContentContaining(@Param("keyword") String keyword);
/**
* 根据多个关键词模糊搜索知识条目
* @param keyword1 第一个关键词
* @param keyword2 第二个关键词
* @param keyword3 第三个关键词
* @return 匹配的知识条目列表
*/
@Query("SELECT k FROM KnowledgeEntity k WHERE " +
"LOWER(k.content) LIKE LOWER(CONCAT('%', :keyword1, '%')) OR " +
"LOWER(k.content) LIKE LOWER(CONCAT('%', :keyword2, '%')) OR " +
"LOWER(k.content) LIKE LOWER(CONCAT('%', :keyword3, '%'))")
List<KnowledgeEntity> findByMultipleKeywords(
@Param("keyword1") String keyword1,
@Param("keyword2") String keyword2,
@Param("keyword3") String keyword3);
/**
* 根据文件ID删除知识条目
* @param keyfileid 文件ID
* @return 删除的条目数量
*/
@Modifying
@Transactional
@Query("DELETE FROM KnowledgeEntity k WHERE k.keyfileid = :keyfileid")
int deleteByKeyfileid(@Param("keyfileid") Integer keyfileid);
}

@ -0,0 +1,750 @@
// service/Qwen3Service.java
package com.alops.chat.service;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.alops.chat.DTO.EmbeddingResponse;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.http.HttpEntity;
import org.springframework.http.HttpHeaders;
import org.springframework.http.MediaType;
import org.springframework.http.ResponseEntity;
import org.springframework.stereotype.Service;
import org.springframework.web.client.ResourceAccessException;
import org.springframework.web.client.RestTemplate;
import java.net.InetSocketAddress;
import java.net.Socket;
import java.util.*;
@Service
public class AItokenService {
@Autowired
private RestTemplate restTemplate;
// Ollama API地址
private static final String OLLAMA_EMBEDDING_URL = "http://127.0.0.1:11434/api/embeddings";
private static final String OLLAMA_GENERATION_URL = "http://127.0.0.1:11434/api/chat";
// 使用的模型
private static final String EMBEDDING_MODEL = "nomic-embed-text:v1.5";
private static final String GENERATION_MODEL = "qwen3:4b";
// 是否使用假数据
@Value("${mock.enabled:false}")
private boolean mockEnabled;
// 假数据答案
@Value("${mock.qa.answer:这是默认的模拟答案}")
private String mockAnswer;
// 假数据来源
@Value("${mock.qa.sources:模拟来源1,模拟来源2}")
private String mockSources;
/**
* 检查是否使用假数据模式
* @return 是否使用假数据
*/
public boolean isMockEnabled() {
return mockEnabled;
}
// 添加连接测试方法
private boolean isOllamaReachable() {
try (Socket socket = new Socket()) {
socket.connect(new InetSocketAddress("127.0.0.1", 11434), 5000); // 5秒超时
return true;
} catch (Exception e) {
System.err.println("无法通过Socket连接到Ollama服务: " + e.getMessage());
return false;
}
}
// 添加HTTP连接测试方法
private boolean isOllamaHttpReachable() {
try {
ResponseEntity<String> response = restTemplate.getForEntity("http://127.0.0.1:11434/api/tags", String.class);
return response.getStatusCode().is2xxSuccessful();
} catch (Exception e) {
System.err.println("无法通过HTTP连接到Ollama服务: " + e.getMessage());
return false;
}
}
/**
* 获取文本的向量表示结果
*
* @param text 输入文本
* @return 向量表示结果
*/
public EmbeddingResponse getEmbedding(String text) {
// 如果启用假数据模式,返回随机向量
if (mockEnabled) {
System.out.println("使用假数据模式,返回随机向量");
return generateRandomEmbedding();
}
System.out.println("开始获取文本嵌入向量,输入文本长度: " + (text != null ? text.length() : 0));
// 验证输入文本长度是否符合API要求
if (text == null || text.isEmpty()) {
System.err.println("输入文本为空");
return new EmbeddingResponse(false, new ArrayList<>(), 0);
}
// 如果文本长度超过8192个字符,则截取前8192个字符
if (text.length() > 8192) {
System.out.println("输入文本长度超过8192字符,已自动截取前8192个字符");
text = text.substring(0, 8192);
}
// 检查Ollama服务是否可达
System.out.println("检查Ollama服务是否可达...");
if (!isOllamaReachable()) {
System.err.println("Socket连接测试失败");
if (!isOllamaHttpReachable()) {
System.err.println("HTTP连接测试也失败");
return new EmbeddingResponse(false, new ArrayList<>(), 0, "无法连接到Ollama服务,请确保Ollama正在运行并监听端口11434。请尝试重启Ollama服务或检查防火墙设置。");
} else {
System.out.println("HTTP连接测试成功,但Socket连接失败,可能存在网络配置问题");
}
} else {
System.out.println("Socket连接测试成功");
}
try {
// 构造请求体
Map<String, String> requestBody = new HashMap<>();
requestBody.put("model", EMBEDDING_MODEL);
requestBody.put("prompt", text);
// 设置请求头
HttpHeaders headers = new HttpHeaders();
headers.setContentType(MediaType.APPLICATION_JSON);
// 创建HTTP实体
HttpEntity<Map<String, String>> requestEntity = new HttpEntity<>(requestBody, headers);
// 发送POST请求到Ollama API
ResponseEntity<String> response = restTemplate.postForEntity(OLLAMA_EMBEDDING_URL, requestEntity, String.class);
if (response.getStatusCode().is2xxSuccessful() && response.getBody() != null) {
// 解析响应
ObjectMapper objectMapper = new ObjectMapper();
JsonNode rootNode = objectMapper.readTree(response.getBody());
// 检查是否有错误信息
JsonNode errorNode = rootNode.get("error");
if (errorNode != null) {
String errorMessage = errorNode.asText();
System.err.println("Ollama API返回错误: " + errorMessage);
// 特别处理模型未找到的情况
if (errorMessage.contains("not found")) {
return new EmbeddingResponse(false, new ArrayList<>(), 0, "模型 '" + EMBEDDING_MODEL + "' 未找到,请运行 'ollama pull " + EMBEDDING_MODEL + "' 命令下载模型");
}
return new EmbeddingResponse(false, new ArrayList<>(), 0, "Ollama API错误: " + errorMessage);
}
// 尝试获取"embeddings"字段(某些Ollama版本)
JsonNode embeddingsNode = rootNode.get("embeddings");
if (embeddingsNode != null && embeddingsNode.isArray()) {
List<Float> embedding = new ArrayList<>();
for (JsonNode node : embeddingsNode) {
embedding.add((float) node.asDouble());
}
System.out.println("成功获取嵌入向量,维度: " + embedding.size());
return new EmbeddingResponse(true, embedding, embedding.size());
}
// 尝试获取"embedding"字段(其他Ollama版本)
JsonNode embeddingNode = rootNode.get("embedding");
if (embeddingNode != null && embeddingNode.isArray()) {
List<Float> embedding = new ArrayList<>();
for (JsonNode node : embeddingNode) {
embedding.add((float) node.asDouble());
}
System.out.println("成功获取嵌入向量,维度: " + embedding.size());
return new EmbeddingResponse(true, embedding, embedding.size());
}
// 如果两种方式都失败,记录完整的响应内容用于调试
System.err.println("Ollama API返回的嵌入向量格式不正确,完整响应: " + rootNode.toString());
return new EmbeddingResponse(false, new ArrayList<>(), 0, "Ollama API返回的嵌入向量格式不正确");
} else {
System.err.println("Ollama API调用失败,状态码: " + response.getStatusCode());
// 特别处理404错误,提示用户需要拉取模型
if (response.getStatusCode().value() == 404) {
System.err.println("模型未找到,请运行 'ollama pull " + EMBEDDING_MODEL + "' 命令下载模型");
return new EmbeddingResponse(false, new ArrayList<>(), 0, "模型 '" + EMBEDDING_MODEL + "' 未找到,请运行 'ollama pull " + EMBEDDING_MODEL + "' 命令下载模型");
}
return new EmbeddingResponse(false, new ArrayList<>(), 0);
}
} catch (ResourceAccessException e) {
// 处理连接被拒绝的异常
if (e.getCause() instanceof java.net.ConnectException) {
System.err.println("无法连接到Ollama服务: " + e.getMessage());
return new EmbeddingResponse(false, new ArrayList<>(), 0, "无法连接到Ollama服务,请确保Ollama正在运行并监听端口11434。请尝试重启Ollama服务或检查防火墙设置。");
}
System.err.println("调用Ollama嵌入API时发生网络异常: " + e.getMessage());
e.printStackTrace();
return new EmbeddingResponse(false, new ArrayList<>(), 0, "调用嵌入API时发生网络异常: " + e.getMessage());
} catch (org.springframework.web.client.HttpClientErrorException.NotFound e) {
System.err.println("模型未找到: " + e.getMessage());
return new EmbeddingResponse(false, new ArrayList<>(), 0, "模型 '" + EMBEDDING_MODEL + "' 未找到,请运行 'ollama pull " + EMBEDDING_MODEL + "' 命令下载模型");
} catch (Exception e) {
System.err.println("调用Ollama嵌入API时发生异常: " + e.getMessage());
e.printStackTrace();
return new EmbeddingResponse(false, new ArrayList<>(), 0, "调用嵌入API时发生异常: " + e.getMessage());
}
}
/**
* 生成随机向量作为后备方案
*
* @return 随机向量
*/
private EmbeddingResponse generateRandomEmbedding() {
List<Float> randomEmbedding = new ArrayList<>();
Random random = new Random();
for (int i = 0; i < 100; i++) {
randomEmbedding.add(random.nextFloat() * 2 - 1); // 生成-1到1之间的随机数
}
System.out.println("Generated Random Embedding, Size: " + randomEmbedding.size());
return new EmbeddingResponse(true, randomEmbedding, 100);
}
/**
* 生成问题的答案支持多轮对话
*
* @param messages 对话历史消息列表
* @return 生成的答案
*/
public String generateAnswer(List<Map<String, String>> messages) {
// 如果启用假数据模式,返回预设答案
if (mockEnabled) {
System.out.println("使用假数据模式,返回预设答案");
return mockAnswer;
}
System.out.println("开始生成答案,消息数量: " + (messages != null ? messages.size() : 0));
// 验证消息列表
if (messages == null || messages.isEmpty()) {
System.err.println("消息列表为空");
return "抱歉,我无法回答这个问题。请稍后再试。";
}
// 检查Ollama服务是否可达
System.out.println("检查Ollama服务是否可达...");
if (!isOllamaReachable()) {
System.err.println("Socket连接测试失败");
if (!isOllamaHttpReachable()) {
System.err.println("HTTP连接测试也失败");
return "系统遇到错误: 无法连接到Ollama服务,请确保Ollama正在运行并监听端口11434。请尝试重启Ollama服务或检查防火墙设置。";
} else {
System.out.println("HTTP连接测试成功,但Socket连接失败,可能存在网络配置问题");
}
} else {
System.out.println("Socket连接测试成功");
}
// 添加重试机制
int maxRetries = 5; // 增加重试次数
for (int attempt = 1; attempt <= maxRetries; attempt++) {
try {
// 构造请求体
Map<String, Object> requestBody = new HashMap<>();
requestBody.put("model", GENERATION_MODEL);
requestBody.put("messages", messages);
requestBody.put("stream", false);
requestBody.put("options", Map.of("num_predict", 2048)); // 限制生成长度以减少资源消耗
// 设置请求头
HttpHeaders headers = new HttpHeaders();
headers.setContentType(MediaType.APPLICATION_JSON);
// 创建HTTP实体
HttpEntity<Map<String, Object>> requestEntity = new HttpEntity<>(requestBody, headers);
// 发送POST请求到Ollama API
ResponseEntity<String> response = restTemplate.postForEntity(OLLAMA_GENERATION_URL, requestEntity, String.class);
if (response.getStatusCode().is2xxSuccessful() && response.getBody() != null) {
System.out.println("Ollama API 响应: " + response.getBody());
// 解析响应
ObjectMapper objectMapper = new ObjectMapper();
JsonNode rootNode = objectMapper.readTree(response.getBody());
// 检查是否有错误信息
JsonNode errorNode = rootNode.get("error");
if (errorNode != null) {
String errorMessage = errorNode.asText();
System.err.println("Ollama API返回错误: " + errorMessage);
// 特别处理模型未找到的情况
if (errorMessage.contains("not found")) {
return "系统遇到错误: 模型 '" + GENERATION_MODEL + "' 未找到,请联系管理员确认模型是否已正确安装";
}
return "系统遇到错误: " + errorMessage + "。请稍后再试。";
}
// 检查done_reason是否为load(模型正在加载)
JsonNode doneReasonNode = rootNode.get("done_reason");
if (doneReasonNode != null) {
String doneReason = doneReasonNode.asText();
// 正确处理"load"状态 - 忽略它并继续等待结果
if ("load".equals(doneReason)) {
System.out.println("第 " + attempt + " 次尝试:模型正在加载中,请稍后再试");
if (attempt < maxRetries) {
// 等待时间逐渐增加
long waitTime = 5000 * attempt; // 5秒, 10秒, 15秒, 20秒...
System.out.println("等待 " + (waitTime/1000) + " 秒后重试...");
Thread.sleep(waitTime);
continue;
} else {
return "AI模型正在加载中,请稍等2-3分钟后再试。如果问题持续存在,可能需要检查系统资源配置。";
}
}
// 只有当done_reason为stop时才表示真正结束
else if (!"stop".equals(doneReason)) {
System.out.println("第 " + attempt + " 次尝试:模型返回未知的done_reason: " + doneReason);
if (attempt < maxRetries) {
long waitTime = 3000 * attempt;
System.out.println("等待 " + (waitTime/1000) + " 秒后重试...");
Thread.sleep(waitTime);
continue;
}
}
}
// 首先尝试获取"message"字段(适用于某些Ollama版本)
JsonNode messageNode = rootNode.get("message");
if (messageNode != null) {
JsonNode contentNode = messageNode.get("content");
if (contentNode != null) {
String content = contentNode.asText();
System.out.println("从 'message.content' 获取回答内容,长度: " + (content != null ? content.length() : 0));
if (content != null && !content.isEmpty()) {
return content;
}
}
}
// 如果没有"message"字段,尝试直接获取"response"字段(适用于其他Ollama版本)
JsonNode responseNode = rootNode.get("response");
if (responseNode != null) {
String content = responseNode.asText();
System.out.println("从 'response' 获取回答内容,长度: " + (content != null ? content.length() : 0));
if (content != null && !content.isEmpty()) {
return content;
}
}
// 尝试获取done字段为false时的文本(流式响应但stream=false的情况)
JsonNode doneNode = rootNode.get("done");
if (doneNode != null && !doneNode.asBoolean()) {
JsonNode textNode = rootNode.get("response");
if (textNode != null) {
String content = textNode.asText();
System.out.println("从流式响应获取回答内容,长度: " + (content != null ? content.length() : 0));
if (content != null && !content.isEmpty()) {
return content;
}
}
}
// 如果以上方式都失败,检查是否有其他有用信息
JsonNode modelNode = rootNode.get("model");
JsonNode createdAtNode = rootNode.get("created_at");
if (modelNode != null && createdAtNode != null) {
String model = modelNode.asText();
String createdAt = createdAtNode.asText();
System.out.println("模型信息 - 模型: " + model + ", 创建时间: " + createdAt);
// 如果只有模型信息但没有内容,说明可能生成了空响应
if (rootNode.size() <= 3) { // 只有model, created_at, done等基本信息
if (attempt < maxRetries) {
// 等待一段时间再重试
long waitTime = 3000 * attempt; // 3秒, 6秒, 9秒...
System.out.println("模型响应不完整,等待 " + (waitTime/1000) + " 秒后重试...");
Thread.sleep(waitTime);
continue;
}
return "AI模型未能生成相关回答,请尝试重新提问或联系管理员。";
}
}
// 如果两种方式都失败,记录完整的响应内容用于调试
System.err.println("Ollama API返回的结果格式不正确,完整响应: " + rootNode.toString());
return "抱歉,我无法回答这个问题。请稍后再试。";
} else {
System.err.println("Ollama API调用失败,状态码: " + response.getStatusCode());
if (attempt < maxRetries) {
// 等待一段时间再重试
long waitTime = 3000 * attempt;
System.out.println("API调用失败,等待 " + (waitTime/1000) + " 秒后重试...");
Thread.sleep(waitTime);
continue;
}
return "抱歉,我无法回答这个问题。请稍后再试。";
}
} catch (ResourceAccessException e) {
// 处理连接被拒绝的异常
if (e.getCause() instanceof java.net.ConnectException) {
System.err.println("无法连接到Ollama服务: " + e.getMessage());
return "系统遇到错误: 无法连接到Ollama服务,请确保Ollama正在运行并监听端口11434。请尝试重启Ollama服务或检查防火墙设置。";
}
System.err.println("调用Ollama API时发生网络异常: " + e.getMessage());
e.printStackTrace();
if (attempt < maxRetries) {
try {
long waitTime = 5000 * attempt;
System.out.println("发生网络异常,等待 " + (waitTime/1000) + " 秒后重试...");
Thread.sleep(waitTime);
} catch (InterruptedException ie) {
Thread.currentThread().interrupt();
return "抱歉,我无法回答这个问题。系统遇到了一个意外错误,请稍后再试。";
}
continue;
}
return "抱歉,我无法回答这个问题。系统遇到了网络连接错误,请稍后再试。";
} catch (Exception e) {
System.err.println("调用Ollama API时发生异常: " + e.getMessage());
e.printStackTrace();
if (attempt < maxRetries) {
try {
long waitTime = 5000 * attempt;
System.out.println("发生异常,等待 " + (waitTime/1000) + " 秒后重试...");
Thread.sleep(waitTime);
} catch (InterruptedException ie) {
Thread.currentThread().interrupt();
return "抱歉,我无法回答这个问题。系统遇到了一个意外错误,请稍后再试。";
}
continue;
}
return "抱歉,我无法回答这个问题。系统遇到了一个意外错误,请稍后再试。";
}
}
return "抱歉,系统尝试多次仍无法获取答案,请稍后再试。";
}
/**
* 自动生成知识条目
* 调用大语言模型对输入内容进行分段和总结生成临时知识条目
*
* @param content 原始内容
* @param source 知识来源
* @return 分段后的内容列表
*/
public List<String> generateKnowledgeEntries(String content, String source) {
// 如果启用假数据模式,返回模拟分段
if (mockEnabled) {
System.out.println("使用假数据模式,返回模拟分段");
List<String> mockSegments = new ArrayList<>();
mockSegments.add("这是模拟的知识条目1。在真实环境中,这将是由AI模型生成的内容。");
mockSegments.add("这是模拟的知识条目2。它代表了原始内容的另一个片段。");
mockSegments.add("这是模拟的知识条目3。用于演示知识库的多条目结构。");
return mockSegments;
}
// 如果内容为空,直接返回默认分段
if (content == null || content.trim().isEmpty()) {
List<String> defaultSegments = new ArrayList<>();
defaultSegments.add("待处理内容");
return defaultSegments;
}
// 限制内容长度,避免API调用失败
String processedContent = content;
if (content == null || content.trim().isEmpty()) {
System.out.println("输入内容为空,使用默认内容");
processedContent = "未提供有效内容";
} else if (content.length() > 8000) {
System.out.println("输入内容长度超过8000字符,已自动截取前8000个字符");
processedContent = content.substring(0, 8000);
} else {
System.out.println("使用完整内容,长度: " + content.length());
}
// 添加重试机制
int maxRetries = 3;
for (int attempt = 1; attempt <= maxRetries; attempt++) {
try {
// 构造提示词,要求模型将内容分段并总结
String prompt = "请将以下内容分段并总结,根据知识来源进行分段,内容不能太少,没有字数要求。" +
"请按照以下JSON格式返回结果,不要包含其他内容:\n" +
"[\n" +
" \"分段内容1\",\n" +
" \"分段内容2\",\n" +
" \"分段内容3\"\n" +
"]\n\n" +
"知识来源:" + source + "\n\n" +
"内容如下:\n" + processedContent;
System.out.println("构造提示词完成,提示词长度: " + prompt.length());
System.out.println("使用的模型: " + GENERATION_MODEL);
// 构造请求体
Map<String, Object> requestBody = new HashMap<>();
requestBody.put("model", GENERATION_MODEL);
requestBody.put("prompt", prompt);
requestBody.put("stream", false);
requestBody.put("temperature", 0.7);
// 设置请求头
HttpHeaders headers = new HttpHeaders();
headers.setContentType(MediaType.APPLICATION_JSON);
// 创建HTTP实体
HttpEntity<Map<String, Object>> requestEntity = new HttpEntity<>(requestBody, headers);
// 发送POST请求到Ollama API
ResponseEntity<String> response = restTemplate.postForEntity(OLLAMA_GENERATION_URL, requestEntity, String.class);
// 解析响应
if (response.getStatusCode().is2xxSuccessful() && response.getBody() != null) {
// 解析响应
ObjectMapper objectMapper = new ObjectMapper();
JsonNode rootNode = objectMapper.readTree(response.getBody());
// 检查是否有错误信息
JsonNode errorNode = rootNode.get("error");
if (errorNode != null) {
String errorMessage = errorNode.asText();
System.err.println("Ollama API返回错误: " + errorMessage);
// 特别处理模型未找到的情况
if (errorMessage.contains("not found")) {
throw new RuntimeException("模型 '" + GENERATION_MODEL + "' 未找到,请联系管理员确认模型是否已正确安装");
}
throw new RuntimeException("Ollama API错误: " + errorMessage);
}
// 检查done_reason是否为load(模型正在加载)
JsonNode doneReasonNode = rootNode.get("done_reason");
if (doneReasonNode != null) {
String doneReason = doneReasonNode.asText();
// 正确处理"load"状态 - 忽略它并继续等待结果
if ("load".equals(doneReason)) {
System.out.println("第 " + attempt + " 次尝试:模型正在加载中,请稍后再试");
if (attempt < maxRetries) {
// 等待时间逐渐增加
long waitTime = 5000 * attempt;
System.out.println("等待 " + (waitTime/1000) + " 秒后重试...");
Thread.sleep(waitTime);
continue;
} else {
throw new RuntimeException("AI模型正在加载中,请稍等2-3分钟后再试。如果问题持续存在,可能需要检查系统资源配置。");
}
}
}
// 尝试获取"response"字段
String responseText = "";
JsonNode responseNode = rootNode.get("response");
if (responseNode != null) {
responseText = responseNode.asText();
} else {
// 如果没有"response"字段,尝试使用整个响应体
responseText = rootNode.toString();
}
// 打印AI生成的原始结果,便于查看格式是否正确
System.out.println("=== AI生成的原始切片结果 ===");
System.out.println(responseText);
System.out.println("=== 结束 ===");
// 检查返回结果是否为空
if (responseText.isEmpty()) {
return createFallbackSegments(processedContent);
}
// 尝试解析JSON格式的响应
try {
List<String> segments = objectMapper.readValue(responseText, new TypeReference<List<String>>() {
});
return segments;
} catch (Exception jsonException) {
// 如果JSON解析失败,尝试从文本中提取内容
System.out.println("JSON解析失败,尝试从文本中提取内容");
return extractSegmentsFromNonJsonText(responseText);
}
} else {
System.err.println("Ollama API调用失败,状态码: " + response.getStatusCode());
if (attempt < maxRetries) {
long waitTime = 3000 * attempt;
System.out.println("API调用失败,等待 " + (waitTime/1000) + " 秒后重试...");
Thread.sleep(waitTime);
continue;
}
return createFallbackSegments(processedContent);
}
} catch (Exception e) {
System.err.println("调用Ollama API时发生异常: " + e.getMessage());
e.printStackTrace();
if (attempt < maxRetries) {
try {
long waitTime = 5000 * attempt;
System.out.println("发生异常,等待 " + (waitTime/1000) + " 秒后重试...");
Thread.sleep(waitTime);
} catch (InterruptedException ie) {
Thread.currentThread().interrupt();
List<String> errorSegments = new ArrayList<>();
errorSegments.add("处理内容时发生错误: " + e.getMessage());
return errorSegments;
}
continue;
}
List<String> errorSegments = new ArrayList<>();
errorSegments.add("处理内容时发生错误: " + e.getMessage());
return errorSegments;
}
}
List<String> errorSegments = new ArrayList<>();
errorSegments.add("处理内容时发生未知错误,请稍后再试");
return errorSegments;
}
/**
* 从文本中提取分段内容
* 当JSON解析失败时使用此方法
*
* @param text AI返回的文本
* @return 分段内容列表
*/
private List<String> extractSegmentsFromNonJsonText(String text) {
List<String> segments = new ArrayList<>();
if (text == null || text.trim().isEmpty()) {
segments.add("内容待处理");
return segments;
}
try {
// 移除可能的代码块标记
String cleanText = text.replace("```json", "").replace("```", "").trim();
// 尝试直接解析JSON数组
ObjectMapper objectMapper = new ObjectMapper();
try {
List<String> jsonSegments = objectMapper.readValue(cleanText, new TypeReference<List<String>>() {
});
if (!jsonSegments.isEmpty()) {
System.out.println("成功解析JSON数组,获得" + jsonSegments.size() + "个分段");
return jsonSegments;
}
} catch (Exception e) {
System.out.println("直接JSON解析失败,继续尝试其他方法: " + e.getMessage());
}
// 尝试查找并解析包含在方括号中的JSON数组
int startIndex = cleanText.indexOf('[');
int endIndex = cleanText.lastIndexOf(']');
if (startIndex != -1 && endIndex != -1 && endIndex > startIndex) {
String jsonArrayStr = cleanText.substring(startIndex, endIndex + 1);
try {
List<String> jsonSegments = objectMapper.readValue(jsonArrayStr, new TypeReference<List<String>>() {
});
if (!jsonSegments.isEmpty()) {
System.out.println("成功解析提取的JSON数组,获得" + jsonSegments.size() + "个分段");
return jsonSegments;
}
} catch (Exception e) {
System.out.println("提取的JSON数组解析失败: " + e.getMessage());
}
}
// 按引号分割提取内容
java.util.regex.Pattern pattern = java.util.regex.Pattern.compile("\"([^\"]+)\"");
java.util.regex.Matcher matcher = pattern.matcher(cleanText);
while (matcher.find()) {
String segment = matcher.group(1).trim();
if (!segment.isEmpty() && segment.length() > 10) {
segments.add(segment);
}
}
// 如果没有通过引号提取到内容,尝试按行分割
if (segments.isEmpty()) {
String[] lines = cleanText.split("\n");
for (String line : lines) {
String cleanLine = line.trim().replaceAll("^[0-9]+\\.\\s*", "").replaceAll("\"", "");
if (!cleanLine.isEmpty() && cleanLine.length() > 10) {
segments.add(cleanLine);
}
}
}
} catch (Exception e) {
System.err.println("从非JSON文本中提取分段内容时发生异常: " + e.getMessage());
e.printStackTrace();
}
// 如果仍然没有内容,添加原始文本作为单个分段
if (segments.isEmpty()) {
String finalText = text.length() > 200 ? text.substring(0, 200) + "..." : text;
segments.add(finalText);
System.out.println("使用原始文本作为单个分段: " + finalText);
}
System.out.println("从非JSON文本中提取到" + segments.size() + "个分段");
return segments;
}
/**
* 创建备用分段方案
* 当AI分段失败时使用此方法
*
* @param content 原始内容
* @return 备用分段列表
*/
private List<String> createFallbackSegments(String content) {
System.out.println("使用备用分段方案");
List<String> segments = new ArrayList<>();
if (content == null || content.trim().isEmpty()) {
segments.add("待处理内容");
return segments;
}
String trimmedContent = content.trim();
// 简单按句号分割
String[] sentences = trimmedContent.split("[。.!?!?]");
for (String sentence : sentences) {
String cleanSentence = sentence.trim();
if (!cleanSentence.isEmpty() && cleanSentence.length() > 10) {
segments.add(cleanSentence);
}
}
// 如果没有按句号分割出内容,尝试按段落分割
if (segments.isEmpty()) {
String[] paragraphs = trimmedContent.split("\n\n");
for (String paragraph : paragraphs) {
String cleanParagraph = paragraph.trim();
if (!cleanParagraph.isEmpty() && cleanParagraph.length() > 20) {
segments.add(cleanParagraph);
}
}
}
// 如果仍然没有内容,创建一个默认分段
if (segments.isEmpty()) {
// 创建一个包含前200个字符的分段
String shortContent = trimmedContent.substring(0, Math.min(200, trimmedContent.length()));
segments.add(shortContent + (trimmedContent.length() > 200 ? "..." : ""));
}
System.out.println("备用方案创建了" + segments.size() + "个分段");
return segments;
}
}

@ -0,0 +1,113 @@
// service/EmbeddingService.java
package com.alops.chat.service;
import com.alops.chat.DTO.EmbeddingResponse;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
/**
* 向量嵌入服务类
* 提供文本向量化功能
*/
@Service
public class EmbeddingService {
@Autowired
private AItokenService AItokenService;
/**
* 获取文本的向量表示
* @param text 输入文本
* @return 浮点数组形式的向量
*/
public float[] getEmbedding(String text) {
// 获取嵌入向量
EmbeddingResponse response = AItokenService.getEmbedding(text);
// 检查是否有错误信息
if (response.getErrorMessage() != null && !response.getErrorMessage().isEmpty()) {
System.err.println("获取嵌入向量时出错: " + response.getErrorMessage());
// 返回空数组而不是抛出异常
return new float[0];
}
// 如果成功获取向量,则转换为浮点数组
if (response.isSuccess()) {
float[] embedding = new float[response.getVector().size()];
for (int i = 0; i < response.getVector().size(); i++) {
embedding[i] = response.getVector().get(i);
}
return embedding;
}
// 如果获取失败,返回空数组
return new float[0];
}
/**
* 将浮点数组转换为字节数组
* @param floats 浮点数组
* @return 字节数组
*/
public static byte[] floatArrayToByteArray(float[] floats) {
java.nio.ByteBuffer buffer = java.nio.ByteBuffer.allocate(floats.length * 4);
for (float f : floats) {
buffer.putFloat(f);
}
return buffer.array();
}
/**
* 将字节数组转换为浮点数组
* @param bytes 字节数组
* @return 浮点数组
*/
public static float[] byteArrayToFloatArray(byte[] bytes) {
if (bytes == null || bytes.length == 0) {
return new float[0];
}
java.nio.ByteBuffer buffer = java.nio.ByteBuffer.wrap(bytes);
float[] floats = new float[bytes.length / 4];
for (int i = 0; i < floats.length; i++) {
floats[i] = buffer.getFloat();
}
return floats;
}
/**
* 计算两个向量之间的欧几里得距离
* @param a 向量A
* @param b 向量B
* @return 欧几里得距离
*/
public static double calculateEuclideanDistance(float[] a, float[] b) {
if (a.length != b.length) {
// 当向量维度不匹配时,返回一个较大的距离值而不是抛出异常
// 这样可以确保程序继续运行,同时将不匹配的向量排在后面
return Double.MAX_VALUE;
}
double sum = 0;
for (int i = 0; i < a.length; i++) {
double diff = a[i] - b[i];
sum += diff * diff;
}
return Math.sqrt(sum);
}
/**
* 检查Ollama服务是否可用
* @return 服务是否可用
*/
public boolean isOllamaServiceAvailable() {
try {
// 尝试获取一个简单的嵌入向量来检查服务是否可用
float[] testEmbedding = getEmbedding("test");
return testEmbedding.length > 0;
} catch (Exception e) {
System.err.println("检查Ollama服务可用性时出错: " + e.getMessage());
return false;
}
}
}

@ -0,0 +1,469 @@
// service/KnowledgeService.java
package com.alops.chat.service;
import com.alops.chat.entity.KnowledgeEntity;
import com.alops.chat.repository.KnowledgeRepository;
import com.alops.chat.DTO.AnswerResponse;
import javax.persistence.EntityManager;
import javax.persistence.PersistenceContext;
import org.apache.poi.xwpf.usermodel.XWPFDocument;
import org.apache.poi.xwpf.usermodel.XWPFParagraph;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import org.springframework.web.multipart.MultipartFile;
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.StandardCopyOption;
import java.time.LocalDateTime;
import java.time.format.DateTimeFormatter;
import java.util.ArrayList;
import java.util.List;
/**
* 知识服务类
* 提供知识库管理功能包括添加删除更新知识条目以及查找相似知识等功能
*/
@Service
public class KnowledgeService {
@Autowired
private KnowledgeRepository knowledgeRepository;
@Autowired
private EmbeddingService embeddingService;
@Autowired
private AItokenService AItokenService;
@Autowired
private TextSplitter textSplitter;
// 是否使用假数据
@Value("${mock.knowledge.enabled:false}")
private boolean mockKnowledgeEnabled;
@PersistenceContext
private EntityManager entityManager;
/**
* 添加知识到知识库
* 直接分割内容并生成正式条目
* @param content 知识内容
* @param source 知识来源
* @return 知识条目ID列表
*/
@Transactional
public List<Long> addKnowledge(String content, String source) {
return addKnowledgeWithFileId(content, source, null);
}
/**
* 添加知识到知识库包含文件ID
* 直接分割内容并生成正式条目
* @param content 知识内容
* @param source 知识来源
* @param fileId 文件ID
* @return 知识条目ID列表
*/
@Transactional
public List<Long> addKnowledgeWithFileId(String content, String source, Long fileId) {
System.out.println("开始添加知识: " + content);
List<Long> knowledgeIds = new ArrayList<>();
// 如果使用假数据模式,添加模拟数据
if (mockKnowledgeEnabled && (content == null || content.trim().isEmpty())) {
System.out.println("使用假数据模式,添加模拟知识条目");
for (int i = 0; i < 3; i++) {
String mockContent = "模拟知识条目 " + (i + 1) + "。这是用于测试的假数据内容。";
String mockSource = "模拟来源";
Long id = addKnowledgeSegment(mockContent, mockSource, fileId != null ? fileId.intValue() : 0);
if (id != null) {
knowledgeIds.add(id);
}
}
return knowledgeIds;
}
// 直接分割内容并生成正式条目
if (content.length() > 200) {
System.out.println("内容长度超过200字符,自动切分并生成正式条目");
List<String> segments = textSplitter.splitText(content);
for (int i = 0; i < segments.size(); i++) {
String segment = segments.get(i);
String segmentSource = source + " (part " + (i + 1) + "/" + segments.size() + ")";
Long id = addKnowledgeSegment(segment, segmentSource, fileId != null ? fileId.intValue() : 0);
if (id != null) {
knowledgeIds.add(id);
}
}
} else {
// 直接添加知识
Long id = addKnowledgeSegment(content, source, fileId != null ? fileId.intValue() : 0);
if (id != null) {
knowledgeIds.add(id);
}
}
return knowledgeIds;
}
/**
* 添加知识片段到知识库
* @param content 知识内容
* @param source 知识来源
* @return 知识条目ID
*/
@Transactional
public Long addKnowledgeSegment(String content, String source,int keyfileid) {
System.out.println("开始添加知识片段: " + content);
// 获取文本的向量表示
float[] embedding = embeddingService.getEmbedding(content);
System.out.println("接收到长度为 " + embedding.length + " 的向量");
if (embedding.length > 0) {
// 将浮点数向量转换为字节数组以便存储
byte[] embeddingBytes = EmbeddingService.floatArrayToByteArray(embedding);
System.out.println("转换为长度为 " + embeddingBytes.length + " 的字节数组");
// 创建知识条目对象
KnowledgeEntity entity = new KnowledgeEntity(content, embeddingBytes, source,keyfileid);
System.out.println("创建KnowledgeEntry对象");
// 保存到数据库
KnowledgeEntity savedEntity = knowledgeRepository.save(entity);
System.out.println("保存KnowledgeEntry,ID: " + savedEntity.getId());
return savedEntity.getId();
} else {
System.out.println("接收到空向量,跳过保存");
return null;
}
}
/**
* 根据文件ID删除知识条目
* @param fileId 文件ID
* @return 删除的条目数量
*/
@Transactional
public int deleteKnowledgeByFileId(Long fileId) {
return knowledgeRepository.deleteByKeyfileid(fileId.intValue());
}
/**
* 自动生成知识条目
* 调用大语言模型对输入内容进行分段和总结生成临时知识条目
*
* @param content 原始内容
* @param source 知识来源
* @return 分段后的内容列表
*/
public List<String> generateKnowledgeEntries(String content, String source) {
return AItokenService.generateKnowledgeEntries(content, source);
}
/**
* 获取所有正式条目
* @return 正式条目列表
*/
public List<KnowledgeEntity> getPermanentEntries() {
// 如果使用假数据模式,返回模拟数据
if (mockKnowledgeEnabled) {
System.out.println("使用假数据模式,返回模拟知识条目");
List<KnowledgeEntity> mockEntries = new ArrayList<>();
for (int i = 0; i < 5; i++) {
KnowledgeEntity mockEntry = new KnowledgeEntity();
mockEntry.setId((long) (i + 1));
mockEntry.setContent("模拟知识条目 " + (i + 1) + "。这是用于测试的假数据内容。");
mockEntry.setSource("模拟来源");
mockEntry.setCreatetime(LocalDateTime.now());
mockEntry.setUpdatetime(LocalDateTime.now());
mockEntries.add(mockEntry);
}
return mockEntries;
}
return knowledgeRepository.findAll();
}
/**
* 删除指定ID的知识条目
* @param id 知识条目ID
* @return 删除是否成功
*/
@Transactional
public boolean deleteKnowledge(Long id) {
if (knowledgeRepository.existsById(id)) {
knowledgeRepository.deleteById(id);
return true;
}
return false;
}
/**
* 提取问题中的关键词
* @param question 用户问题
* @return 关键词列表
*/
private List<String> extractKeywords(String question) {
List<String> keywords = new ArrayList<>();
if (question == null || question.trim().isEmpty()) {
return keywords;
}
// 移除常见的疑问词和停用词
String cleanedQuestion = question.replaceAll(
"(什么|如何|为什么|怎么|哪里|哪个|谁|何时|是否|怎么办|吗|呢|吧|啊|的|了|在|是|有|和|与|及|以及|或者|或)$",
"").trim();
// 按空格和常见标点符号分割
String[] words = cleanedQuestion.split("[\\s\\p{Punct}]+");
// 过滤掉太短的词(少于2个字符)
for (String word : words) {
if (word.length() >= 2) {
keywords.add(word);
}
}
return keywords;
}
/**
* 基于关键词和向量的混合检索
* @param question 用户问题
* @param limit 返回结果数量上限
* @return 相似知识条目列表
*/
public List<KnowledgeEntity> findSimilarKnowledgeWithKeywordFilter(String question, int limit) {
// 如果使用假数据模式,返回模拟数据
if (mockKnowledgeEnabled) {
System.out.println("使用假数据模式,返回模拟相似知识条目");
List<KnowledgeEntity> mockEntries = new ArrayList<>();
for (int i = 0; i < Math.min(3, limit); i++) {
KnowledgeEntity mockEntry = new KnowledgeEntity();
mockEntry.setId((long) (i + 1));
mockEntry.setContent("这是与问题 \"" + question + "\" 相关的模拟知识条目 " + (i + 1) + "。在真实环境中,这将是通过向量相似度检索得到的结果。");
mockEntry.setSource("模拟来源 " + (i + 1));
mockEntry.setCreatetime(LocalDateTime.now());
mockEntry.setUpdatetime(LocalDateTime.now());
mockEntries.add(mockEntry);
}
return mockEntries;
}
// 1. 提取关键词
List<String> keywords = extractKeywords(question);
System.out.println("提取到关键词: " + keywords);
// 2. 如果有关键词,先进行关键词搜索
List<KnowledgeEntity> keywordFilteredEntries = new ArrayList<>();
if (!keywords.isEmpty()) {
// 使用前3个关键词进行搜索
String keyword1 = keywords.size() > 0 ? keywords.get(0) : "";
String keyword2 = keywords.size() > 1 ? keywords.get(1) : "";
String keyword3 = keywords.size() > 2 ? keywords.get(2) : "";
keywordFilteredEntries = knowledgeRepository.findByMultipleKeywords(keyword1, keyword2, keyword3);
System.out.println("关键词搜索返回 " + keywordFilteredEntries.size() + " 条结果");
}
// 3. 如果关键词搜索没有结果,则使用所有知识条目
List<KnowledgeEntity> candidateEntries = keywordFilteredEntries.isEmpty() ?
knowledgeRepository.findAll() : keywordFilteredEntries;
// 4. 在候选条目中进行向量相似度检索
return findSimilarKnowledgeFromCandidates(question, candidateEntries, limit);
}
/**
* 从候选条目中查找与问题最相似的知识条目
* @param question 用户问题
* @param candidates 候选知识条目
* @param limit 返回结果数量上限
* @return 相似知识条目列表
*/
private List<KnowledgeEntity> findSimilarKnowledgeFromCandidates(String question, List<KnowledgeEntity> candidates, int limit) {
// 获取问题的向量表示
float[] questionEmbedding = embeddingService.getEmbedding(question);
// 如果无法生成向量,则返回空列表
if (questionEmbedding.length == 0) {
return new ArrayList<>();
}
// 根据欧几里得距离对知识条目进行排序
candidates.sort((a, b) -> {
// 将字节数组转换回浮点数组
float[] embeddingA = EmbeddingService.byteArrayToFloatArray(a.getEmbedding());
float[] embeddingB = EmbeddingService.byteArrayToFloatArray(b.getEmbedding());
// 检查向量是否有效
if (embeddingA.length == 0 || embeddingB.length == 0) {
// 如果任一向量无效,则将该条目排在后面
if (embeddingA.length == 0 && embeddingB.length == 0) return 0;
if (embeddingA.length == 0) return 1; // a排在后面
return -1; // b排在后面
}
// 检查向量维度是否匹配
if (questionEmbedding.length != embeddingA.length || questionEmbedding.length != embeddingB.length) {
// 如果维度不匹配,则根据维度差异排序,将维度匹配的条目排在前面
if (questionEmbedding.length != embeddingA.length && questionEmbedding.length != embeddingB.length) return 0;
if (questionEmbedding.length != embeddingA.length) return 1; // a排在后面
if (questionEmbedding.length != embeddingB.length) return -1; // b排在后面
return 0;
}
// 计算与问题向量的欧几里得距离
double distanceA = EmbeddingService.calculateEuclideanDistance(questionEmbedding, embeddingA);
double distanceB = EmbeddingService.calculateEuclideanDistance(questionEmbedding, embeddingB);
// 按距离升序排序(距离越小越相似)
return Double.compare(distanceA, distanceB);
});
// 过滤掉没有有效向量或相似度不够高的条目
// 设置一个合理的相似度阈值,欧几里得距离越小越相似
final double SIMILARITY_THRESHOLD = 100.0;
List<KnowledgeEntity> validEntries = new ArrayList<>();
for (KnowledgeEntity entry : candidates) {
float[] embedding = EmbeddingService.byteArrayToFloatArray(entry.getEmbedding());
// 检查向量是否有效且维度匹配
if (embedding.length == 0 || embedding.length != questionEmbedding.length) {
continue;
}
// 计算与问题的欧几里得距离
double distance = EmbeddingService.calculateEuclideanDistance(questionEmbedding, embedding);
// 只返回距离小于阈值的条目
if (distance < SIMILARITY_THRESHOLD) {
validEntries.add(entry);
}
}
// 返回最相似的前limit个条目
List<KnowledgeEntity> result = new ArrayList<>();
int count = 0;
for (KnowledgeEntity entry : validEntries) {
if (count >= limit) {
break;
}
result.add(entry);
count++;
}
return result;
}
/**
* 查找与问题最相似的知识条目
* @param question 用户问题
* @param limit 返回结果数量上限
* @return 相似知识条目列表
*/
public List<KnowledgeEntity> findSimilarKnowledge(String question, int limit) {
// 使用关键词和向量混合检索
return findSimilarKnowledgeWithKeywordFilter(question, limit);
}
/**
* 生成上下文内容
* @param knowledgeEntries 知识条目列表
* @return 上下文字符串
*/
public String generateContext(List<KnowledgeEntity> knowledgeEntries) {
// 检查输入参数
if (knowledgeEntries == null || knowledgeEntries.isEmpty()) {
System.out.println("知识条目列表为空,无法生成上下文");
return "";
}
StringBuilder context = new StringBuilder();
for (int i = 0; i < knowledgeEntries.size(); i++) {
KnowledgeEntity entry = knowledgeEntries.get(i);
// 检查每个条目及其内容是否为空
if (entry != null && entry.getContent() != null && !entry.getContent().trim().isEmpty()) {
context.append(entry.getContent());
if (i < knowledgeEntries.size() - 1) {
context.append("\n\n");
}
}
}
String result = context.toString().trim();
System.out.println("生成上下文,长度: " + result.length());
return result;
}
/**
* 更新指定ID的知识条目
* @param id 知识条目ID
* @param content 知识内容
* @param source 知识来源
* @return 更新是否成功
*/
@Transactional
public boolean updateKnowledge(Long id, String content, String source) {
// 检查知识条目是否存在
if (knowledgeRepository.existsById(id)) {
// 获取现有知识条目
KnowledgeEntity existingEntry = knowledgeRepository.findById(id).orElse(null);
if (existingEntry != null) {
// 更新内容和来源
existingEntry.setContent(content);
existingEntry.setSource(source);
// 重新计算向量嵌入
float[] embedding = embeddingService.getEmbedding(content);
if (embedding.length > 0) {
byte[] embeddingBytes = EmbeddingService.floatArrayToByteArray(embedding);
existingEntry.setEmbedding(embeddingBytes);
}
// 保存更新后的条目
knowledgeRepository.save(existingEntry);
return true;
}
}
return false;
}
/**
* 处理用户提问
* @param question 用户问题
* @return 答案响应
*/
public AnswerResponse processQuestion(String question) {
// 这里应该实现具体的问答逻辑
// 由于这是示例,我们返回一个模拟的答案
return new AnswerResponse("这是模拟的答案", new String[]{"来源1", "来源2"});
}
/**
* 处理文件上传
* @param file 上传的文件
* @return 处理结果
*/
public String handleFileUpload(MultipartFile file) {
// 这里应该实现具体的文件处理逻辑
return "文件上传成功";
}
}

@ -0,0 +1,298 @@
package com.alops.chat.service;
import java.util.*;
import java.util.stream.Collectors;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.JsonNode;
import org.springframework.http.HttpEntity;
import org.springframework.http.HttpHeaders;
import org.springframework.http.MediaType;
import org.springframework.http.ResponseEntity;
import org.springframework.web.client.RestTemplate;
@Service
public class TextSplitter {
private static final int MIN_CHUNK_SIZE = 100; // 最小字数
private static final int MAX_CHUNK_SIZE = 500; // 最大字数
private static final double SIMILARITY_THRESHOLD = 0.6; // 相似度阈值
// Ollama nomic-embed-text:v1.5模型的维度
private static final int EMBEDDING_DIMENSION = 768; // nomic-embed-text:v1.5模型的维度
@Autowired
private RestTemplate restTemplate;
private static final String OLLAMA_EMBEDDING_URL = "http://localhost:11434/api/embeddings";
private static final String EMBEDDING_MODEL = "nomic-embed-text:v1.5";
private final ObjectMapper objectMapper;
public TextSplitter() {
this.objectMapper = new ObjectMapper();
}
/**
* 主流程输入文本返回分割后的 chunks
*/
public List<String> splitText(String text) {
// 1. 去除特殊字符
String cleanedText = removeSpecialCharacters(text);
// 2. 预切分
List<String> units = preSplit(cleanedText);
// 3. 调用 embedding 模型(使用Ollama的nomic-embed-text:v1.5模型)
List<float[]> embeddings = units.stream()
.map(this::getEmbeddingFromOllama)
.collect(Collectors.toList());
// 4. 检测候选边界
List<Integer> boundaries = detectBoundaries(embeddings, SIMILARITY_THRESHOLD);
// 5. 精修边界(BayesSeg 简化)
boundaries = refineBoundaries(boundaries, units);
// 6. 根据边界切分
List<String> chunks = cutByBoundaries(units, boundaries);
// 7. 长度控制(合并/再切)
return adjustChunks(chunks);
}
/**
* 去除特殊字符
* @param text 原始文本
* @return 清理后的文本
*/
private String removeSpecialCharacters(String text) {
if (text == null || text.isEmpty()) {
return text;
}
// 定义需要去除的特殊字符(包括但不限于"- ~ ^")
// 可以根据需要添加更多特殊字符到这个列表中
String specialChars = "-~^`@#$&*()_=|\\{}'<>./";
// 创建正则表达式模式来匹配这些特殊字符
String regex = "[" + java.util.regex.Pattern.quote(specialChars) + "]+";
// 将连续的特殊字符替换为单个空格,并去除首尾空格
return text.replaceAll(regex, " ").trim();
}
/**
* 预切分按段落句子切
*/
private List<String> preSplit(String text) {
String[] paras = text.split("\\n\\s*\\n"); // 按空行切段
List<String> units = new ArrayList<>();
for (String p : paras) {
if (p.length() > 1000) {
// 长段再按句子切
units.addAll(Arrays.asList(p.split("(?<=[。!?])")));
} else {
units.add(p.trim());
}
}
return units.stream().filter(s -> !s.isEmpty()).collect(Collectors.toList());
}
/**
* 相似度检测TopicSeg 思路
*/
private List<Integer> detectBoundaries(List<float[]> embeddings, double threshold) {
List<Integer> boundaries = new ArrayList<>();
for (int i = 0; i < embeddings.size() - 1; i++) {
double sim = cosine(embeddings.get(i), embeddings.get(i + 1));
if (sim < threshold) {
boundaries.add(i + 1);
}
}
return boundaries;
}
/**
* 精修候选边界简化版 BayesSeg
*/
private List<Integer> refineBoundaries(List<Integer> boundaries, List<String> units) {
List<Integer> refined = new ArrayList<>();
for (int b : boundaries) {
if (b <= 0 || b >= units.size()) continue;
String left = units.get(b - 1);
String right = units.get(b);
if (keepBoundary(left, right)) {
refined.add(b);
}
}
return refined;
}
/**
* 关键词差异判断是否保留边界
*/
private boolean keepBoundary(String left, String right) {
Set<String> leftWords = new HashSet<>(Arrays.asList(left.split("\\s+")));
Set<String> rightWords = new HashSet<>(Arrays.asList(right.split("\\s+")));
long common = leftWords.stream().filter(rightWords::contains).count();
double ratio = (double) common / Math.max(1, leftWords.size());
return ratio < 0.3; // 共同词过少 → 保留边界
}
/**
* 根据边界切分
*/
private List<String> cutByBoundaries(List<String> units, List<Integer> boundaries) {
List<String> chunks = new ArrayList<>();
StringBuilder buffer = new StringBuilder();
int boundaryIdx = 0;
for (int i = 0; i < units.size(); i++) {
buffer.append(units.get(i)).append(" ");
if (boundaryIdx < boundaries.size() && i + 1 == boundaries.get(boundaryIdx)) {
chunks.add(buffer.toString().trim());
buffer.setLength(0);
boundaryIdx++;
}
}
if (buffer.length() > 0) chunks.add(buffer.toString().trim());
return chunks;
}
/**
* 长度控制
*/
private List<String> adjustChunks(List<String> chunks) {
List<String> result = new ArrayList<>();
StringBuilder buffer = new StringBuilder();
for (String chunk : chunks) {
if (chunk.length() < MIN_CHUNK_SIZE && !result.isEmpty()) {
// 太短 → 合并到前一个
int lastIdx = result.size() - 1;
result.set(lastIdx, result.get(lastIdx) + " " + chunk);
} else if (chunk.length() > MAX_CHUNK_SIZE) {
// 太长 → 按句子边界再切分
List<String> subChunks = splitLongChunk(chunk);
result.addAll(subChunks);
} else {
result.add(chunk);
}
}
return result;
}
/**
* 分割长文本块尽量在句子边界处分割
* @param chunk 需要分割的长文本
* @return 分割后的文本块列表
*/
private List<String> splitLongChunk(String chunk) {
List<String> subChunks = new ArrayList<>();
// 定义句子结束标点符号
String sentenceEndings = "。!?.!?";
// 如果文本长度不超过最大块大小,直接返回
if (chunk.length() <= MAX_CHUNK_SIZE) {
subChunks.add(chunk);
return subChunks;
}
int start = 0;
while (start < chunk.length()) {
// 计算当前分割点
int end = Math.min(start + MAX_CHUNK_SIZE, chunk.length());
// 如果已经到达文本末尾,直接添加剩余部分
if (end >= chunk.length()) {
subChunks.add(chunk.substring(start));
break;
}
// 寻找最近的句子结束位置
int bestSplitPoint = end;
for (int i = end - 1; i > start; i--) {
if (sentenceEndings.indexOf(chunk.charAt(i)) != -1) {
bestSplitPoint = i + 1; // 包含句子结束符号
break;
}
}
// 如果找到的分割点太靠前(小于最小块大小),则使用固定长度分割
if (bestSplitPoint - start < MIN_CHUNK_SIZE) {
bestSplitPoint = end;
}
// 添加分割的文本块
subChunks.add(chunk.substring(start, bestSplitPoint).trim());
start = bestSplitPoint;
// 跳过前面的空白字符
while (start < chunk.length() && Character.isWhitespace(chunk.charAt(start))) {
start++;
}
}
return subChunks;
}
/**
* 从Ollama获取嵌入向量
* 使用nomic-embed-text:v1.5模型
*/
private float[] getEmbeddingFromOllama(String text) {
try {
// 构造请求体
Map<String, String> requestBody = new HashMap<>();
requestBody.put("model", EMBEDDING_MODEL);
requestBody.put("prompt", text);
// 设置请求头
HttpHeaders headers = new HttpHeaders();
headers.setContentType(MediaType.APPLICATION_JSON);
// 创建HTTP实体
HttpEntity<Map<String, String>> requestEntity = new HttpEntity<>(requestBody, headers);
// 发送POST请求到Ollama API
ResponseEntity<String> response = restTemplate.postForEntity(OLLAMA_EMBEDDING_URL, requestEntity, String.class);
if (response.getStatusCode().is2xxSuccessful() && response.getBody() != null) {
// 解析响应
JsonNode rootNode = objectMapper.readTree(response.getBody());
JsonNode embeddingNode = rootNode.get("embedding");
if (embeddingNode != null && embeddingNode.isArray()) {
float[] embeddingArray = new float[embeddingNode.size()];
for (int i = 0; i < embeddingNode.size(); i++) {
embeddingArray[i] = embeddingNode.get(i).floatValue();
}
return embeddingArray;
}
}
// 如果无法获取嵌入向量,返回零向量
return new float[EMBEDDING_DIMENSION]; // nomic-embed-text:v1.5默认维度为768
} catch (Exception e) {
System.err.println("调用Ollama API获取嵌入向量时出错: " + e.getMessage());
// 出错时返回零向量
return new float[EMBEDDING_DIMENSION]; // nomic-embed-text:v1.5默认维度为768
}
}
/**
* 计算余弦相似度
*/
private double cosine(float[] v1, float[] v2) {
double dot = 0, normA = 0, normB = 0;
for (int i = 0; i < v1.length; i++) {
dot += v1[i] * v2[i];
normA += v1[i] * v1[i];
normB += v2[i] * v2[i];
}
return dot / (Math.sqrt(normA) * Math.sqrt(normB) + 1e-10);
}
}

@ -0,0 +1,52 @@
package com.alops.chat.service;
import com.alops.chat.entity.UploadedFile;
import com.alops.chat.repository.UploadedFileRepository;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import java.util.List;
/**
* 上传文件信息服务类
* 提供对上传文件信息的业务处理
*/
@Service
public class UploadedFileService {
@Autowired
private UploadedFileRepository uploadedFileRepository;
/**
* 保存上传文件信息
* @param uploadedFile 上传文件信息实体
* @return 保存后的实体
*/
public UploadedFile save(UploadedFile uploadedFile) {
return uploadedFileRepository.save(uploadedFile);
}
/**
* 根据ID查找上传文件信息
* @param id 文件ID
* @return 上传文件信息实体
*/
public UploadedFile findById(Long id) {
return uploadedFileRepository.findById(id).orElse(null);
}
/**
* 获取所有上传文件信息
* @return 上传文件信息列表
*/
public List<UploadedFile> findAll() {
return uploadedFileRepository.findAll();
}
/**
* 删除上传文件信息
* @param id 文件ID
*/
public void deleteById(Long id) {
uploadedFileRepository.deleteById(id);
}
}

@ -93,7 +93,7 @@ public class FileUploadUtils
* @param baseDir 相对应用的基目录
* @param file 上传的文件
* @param allowedExtension 上传文件类型
* @return 返回上传成功的文件名
* @return 返回上传成功
* @throws FileSizeLimitExceededException 如果超出最大大小
* @throws FileNameLengthLimitExceededException 文件名太长
* @throws IOException 比如读写文件出错时
@ -138,6 +138,36 @@ public class FileUploadUtils
return getPathFileName(baseDir, fileName);
}
/**
* 文件上传保持原始文件名
*
* @param baseDir 相对应用的基目录
* @param file 上传的文件
* @param allowedExtension 上传文件类型
* @return 返回上传成功的文件名
* @throws FileSizeLimitExceededException 如果超出最大大小
* @throws FileNameLengthLimitExceededException 文件名太长
* @throws IOException 比如读写文件出错时
* @throws InvalidExtensionException 文件校验异常
*/
public static final String uploadWithOriginalFilename(String baseDir, MultipartFile file, String[] allowedExtension)
throws FileSizeLimitExceededException, IOException, FileNameLengthLimitExceededException,
InvalidExtensionException
{
int fileNameLength = Objects.requireNonNull(file.getOriginalFilename()).length();
if (fileNameLength > FileUploadUtils.DEFAULT_FILE_NAME_LENGTH)
{
throw new FileNameLengthLimitExceededException(FileUploadUtils.DEFAULT_FILE_NAME_LENGTH);
}
assertAllowed(file, allowedExtension);
String fileName = DateUtils.datePath() + "/" + file.getOriginalFilename();
String absPath = getAbsoluteFile(baseDir, fileName).getAbsolutePath();
file.transferTo(Paths.get(absPath));
return getPathFileName(baseDir, fileName);
}
/**
* 编码文件名(日期格式目录 + 原文件名 + 序列值 + 后缀)
*/

@ -26,6 +26,8 @@ public class MimeTypeUtils
public static final String[] VIDEO_EXTENSION = { "mp4", "avi", "rmvb" };
public static final String[] TEXT_EXTENSION = {"docx", "txt"};
public static final String[] DEFAULT_ALLOWED_EXTENSION = {
// 图片
"bmp", "gif", "jpg", "jpeg", "png",

@ -1,11 +1,13 @@
package com.alops.framework.config;
import java.util.TimeZone;
import org.mybatis.spring.annotation.MapperScan;
import org.springframework.boot.autoconfigure.jackson.Jackson2ObjectMapperBuilderCustomizer;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.context.annotation.EnableAspectJAutoProxy;
import org.springframework.web.client.RestTemplate;
/**
* 程序注解配置
@ -27,4 +29,13 @@ public class ApplicationConfig
{
return jacksonObjectMapperBuilder -> jacksonObjectMapperBuilder.timeZone(TimeZone.getDefault());
}
/**
* RestTemplate配置
*/
@Bean
public RestTemplate restTemplate()
{
return new RestTemplate();
}
}

@ -58,6 +58,15 @@
<version>1.6.2</version>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>com.alops</groupId>
<artifactId>alops-common</artifactId>
</dependency>
<dependency>
<groupId>org.projectlombok</groupId>
<artifactId>lombok</artifactId>
<scope>provided</scope>
</dependency>
</dependencies>

@ -11,7 +11,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
;
import java.util.Date;
@Component
@ -98,4 +98,4 @@ public class GatherJob implements Job {
}
}
}
}
}

File diff suppressed because one or more lines are too long
Loading…
Cancel
Save