|
|
@@ -3,6 +3,7 @@ package com.takai.ai.service.impl;
|
|
|
import com.alibaba.fastjson2.JSON;
|
|
|
import com.alibaba.fastjson2.JSONArray;
|
|
|
import com.alibaba.fastjson2.JSONObject;
|
|
|
+import com.github.pagehelper.PageHelper;
|
|
|
import com.takai.ai.domain.TakaiSysOss;
|
|
|
import com.takai.ai.domain.dto.*;
|
|
|
import com.takai.ai.domain.entity.*;
|
|
|
@@ -17,6 +18,7 @@ import com.takai.common.config.DeepseekConfig;
|
|
|
import com.takai.common.core.domain.entity.SysDictData;
|
|
|
import com.takai.common.core.redis.RedisCache;
|
|
|
import com.takai.common.enums.DataSourceType;
|
|
|
+import com.takai.common.utils.SecurityUtils;
|
|
|
import com.takai.common.utils.StringUtils;
|
|
|
import com.takai.common.utils.uuid.IdUtils;
|
|
|
import com.takai.framework.web.service.SysPermissionService;
|
|
|
@@ -42,12 +44,12 @@ import org.springframework.web.multipart.MultipartFile;
|
|
|
import org.springframework.web.servlet.mvc.method.annotation.SseEmitter;
|
|
|
import javax.validation.constraints.NotNull;
|
|
|
import java.io.IOException;
|
|
|
+import java.nio.charset.StandardCharsets;
|
|
|
import java.time.LocalDate;
|
|
|
import java.time.LocalDateTime;
|
|
|
import java.time.temporal.ChronoUnit;
|
|
|
import java.util.*;
|
|
|
import java.util.concurrent.TimeUnit;
|
|
|
-import java.util.function.Function;
|
|
|
import java.util.stream.Collectors;
|
|
|
|
|
|
/**
|
|
|
@@ -591,57 +593,48 @@ public class TakaiAiServiceImpl implements ITakaiAiService {
|
|
|
|
|
|
if (jsonObject != null && jsonObject.containsKey("code") && jsonObject.getInteger("code") == 200) {
|
|
|
for (UploadDocumentParams vo : params) {
|
|
|
- TakaiKnowledge info = takaiKnowledgeMapper.selectTargetKnowledge(TakaiKnowledge.builder().knowledgeId(knowledgeId).build());
|
|
|
- if(info != null){
|
|
|
- // 更新知识库文件大小,总字符数, 文件总数
|
|
|
- JSONObject docInfo = jsonObject.getJSONObject("doc_info");
|
|
|
- if(docInfo != null){
|
|
|
- List<TakaiDocument> documentList = takaiDocumentMapper.selectDocumentList(TakaiDocument.builder().knowledgeId(knowledgeId).build());
|
|
|
- Integer fileLen = docInfo.getInteger("file_size");
|
|
|
- Integer wordNum = docInfo.getInteger("total_char_len");
|
|
|
- Integer sliceTotal = docInfo.getInteger("slice_num");
|
|
|
- TakaiKnowledge paramInfo = TakaiKnowledge.builder().knowledgeId(knowledgeId)
|
|
|
- .length(info.getLength() == null ? fileLen : info.getLength() + fileLen ) // 文件大小
|
|
|
- .wordNum(info.getWordNum() == null ? wordNum : info.getWordNum() + wordNum) // 总字符数
|
|
|
- .documentSize(documentList == null || documentList.size() == 0 ? 1 : documentList.size() + 1) // 文件总数
|
|
|
- .build();
|
|
|
- takaiKnowledgeMapper.updateKnowledge(paramInfo);
|
|
|
-
|
|
|
- for (TakaiSysOss oss : result) {
|
|
|
- // oss保存到数据库
|
|
|
- takaiSysOssMapper.insertSysOss(oss);
|
|
|
- }
|
|
|
+ // 更新知识库文件大小,总字符数, 文件总数
|
|
|
+ JSONObject docInfo = jsonObject.getJSONObject("doc_info");
|
|
|
+ if(docInfo != null){
|
|
|
+ Integer fileLen = docInfo.getInteger("file_size");
|
|
|
+ Integer wordNum = docInfo.getInteger("total_char_len");
|
|
|
+ Integer sliceTotal = docInfo.getInteger("slice_num");
|
|
|
+
|
|
|
+ for (TakaiSysOss oss : result) {
|
|
|
+ // oss保存到数据库
|
|
|
+ takaiSysOssMapper.insertSysOss(oss);
|
|
|
+ }
|
|
|
|
|
|
- // 保存知识信息
|
|
|
- TakaiDocument document = TakaiDocument.builder()
|
|
|
- .documentId(vo.getDocument_id())
|
|
|
- .knowledgeId(knowledgeId)
|
|
|
- .customSeparator(String.format("[\"%s\"", "\\n") + "]")
|
|
|
- .sentenceSize("300")
|
|
|
- .name(vo.getName())
|
|
|
- .url(vo.getUrl())
|
|
|
- .sliceTotal(sliceTotal)
|
|
|
- .length(fileLen)
|
|
|
- .wordNum(wordNum)
|
|
|
- .build();
|
|
|
- int documentIdInsert = takaiDocumentMapper.insertDocument(document);
|
|
|
-
|
|
|
- if (documentIdInsert > 0) {
|
|
|
- // 保存知识设置信息
|
|
|
- SnowflakeDigitGenerator snowflakeDigitGenerator = new SnowflakeDigitGenerator(1, 1);
|
|
|
- long id = snowflakeDigitGenerator.nextId();
|
|
|
- TakaiDocumentSettings dSettings = new TakaiDocumentSettings();
|
|
|
- dSettings.setId(String.valueOf(id));
|
|
|
- dSettings.setKnowledgeId(knowledgeId);
|
|
|
- dSettings.setDocumentId(vo.getDocument_id());
|
|
|
- dSettings.setSetSlice("0"); // 默认 按标题段落切片
|
|
|
- dSettings.setSetAnalyze("1"); // 默认 图片转换成标识符
|
|
|
- dSettings.setSetTable("0"); // 默认 ttable转图片
|
|
|
- takaiDocumentSettingsMapper.insertDocumentSettings(dSettings);
|
|
|
- }
|
|
|
+ // 保存知识信息
|
|
|
+ TakaiDocument document = TakaiDocument.builder()
|
|
|
+ .documentId(vo.getDocument_id())
|
|
|
+ .knowledgeId(knowledgeId)
|
|
|
+ .customSeparator(String.format("[\"%s\"", "\\n") + "]")
|
|
|
+ .sentenceSize("300")
|
|
|
+ .name(vo.getName())
|
|
|
+ .url(vo.getUrl())
|
|
|
+ .sliceTotal(sliceTotal)
|
|
|
+ .length(fileLen)
|
|
|
+ .wordNum(wordNum)
|
|
|
+ .build();
|
|
|
+ int documentIdInsert = takaiDocumentMapper.insertDocument(document);
|
|
|
+
|
|
|
+ if (documentIdInsert > 0) {
|
|
|
+ // 保存知识设置信息
|
|
|
+ SnowflakeDigitGenerator snowflakeDigitGenerator = new SnowflakeDigitGenerator(1, 1);
|
|
|
+ long id = snowflakeDigitGenerator.nextId();
|
|
|
+ TakaiDocumentSettings dSettings = new TakaiDocumentSettings();
|
|
|
+ dSettings.setId(String.valueOf(id));
|
|
|
+ dSettings.setKnowledgeId(knowledgeId);
|
|
|
+ dSettings.setDocumentId(vo.getDocument_id());
|
|
|
+ dSettings.setSetSlice("0"); // 默认 按标题段落切片
|
|
|
+ dSettings.setSetAnalyze("1"); // 默认 图片转换成标识符
|
|
|
+ dSettings.setSetTable("0"); // 默认 ttable转图片
|
|
|
+ takaiDocumentSettingsMapper.insertDocumentSettings(dSettings);
|
|
|
}
|
|
|
}
|
|
|
}
|
|
|
+ updateKnowledgeSize(knowledgeId);
|
|
|
setRedisCache(knowledgeId);
|
|
|
return 1;
|
|
|
}
|
|
|
@@ -649,6 +642,46 @@ public class TakaiAiServiceImpl implements ITakaiAiService {
|
|
|
return 0;
|
|
|
}
|
|
|
|
|
|
+ private void updateKnowledgeSize(String knowledgeId) {
|
|
|
+ TakaiKnowledge info = takaiKnowledgeMapper.selectTargetKnowledge(TakaiKnowledge.builder().knowledgeId(knowledgeId).build());
|
|
|
+ List<TakaiDocument> documentList = takaiDocumentMapper.selectDocumentList(TakaiDocument.builder().knowledgeId(knowledgeId).build());
|
|
|
+ int totalLen = documentList.stream()
|
|
|
+ .mapToInt(doc -> doc.getLength() != null ? doc.getLength() : 0)
|
|
|
+ .sum();
|
|
|
+ int totalWordNum = documentList.stream()
|
|
|
+ .mapToInt(doc -> doc.getWordNum() != null ? doc.getWordNum() : 0)
|
|
|
+ .sum();
|
|
|
+ int docSize = documentList == null || documentList.size() == 0 ? 0 : documentList.size();
|
|
|
+ TakaiKnowledge paramInfo = TakaiKnowledge.builder().knowledgeId(knowledgeId)
|
|
|
+ .length(totalLen) // 文件大小
|
|
|
+ .wordNum(totalWordNum) // 总字符数
|
|
|
+ .documentSize(docSize) // 文件总数
|
|
|
+ .build();
|
|
|
+ takaiKnowledgeMapper.updateKnowledge(paramInfo);
|
|
|
+ }
|
|
|
+
|
|
|
+ private void updateDocumentSize(String knowledgeId,String documentId) {
|
|
|
+ List<TakaiSliceInfo> sliceList = selectByDocumentId(documentId,null);
|
|
|
+ //字符数
|
|
|
+ int totalLen = sliceList.stream()
|
|
|
+ .mapToInt(slice -> slice.getSliceText() != null ? slice.getSliceText().length() : 0)
|
|
|
+ .sum();
|
|
|
+ //文件大小
|
|
|
+ int totalSize = sliceList.stream()
|
|
|
+ .mapToInt(slice -> slice.getSliceText() != null ? slice.getSliceText().getBytes(StandardCharsets.UTF_8).length : 0)
|
|
|
+ .sum();
|
|
|
+ TakaiDocument doc = TakaiDocument.builder()
|
|
|
+ .documentId(documentId)
|
|
|
+ .length(totalSize)
|
|
|
+ .wordNum(totalLen)
|
|
|
+ .build();
|
|
|
+ String updateBy = SecurityUtils.getLoginUser() == null ? "":SecurityUtils.getLoginUser().getUsername();
|
|
|
+ doc.setUpdateBy(updateBy);
|
|
|
+ doc.setUpdateTime(new Date());
|
|
|
+ takaiDocumentMapper.updateDocument(doc);
|
|
|
+ updateKnowledgeSize(knowledgeId);
|
|
|
+ }
|
|
|
+
|
|
|
@Override
|
|
|
public int updateDocument(TakaiDocumentParams documentParams, String documentId) {
|
|
|
int i = takaiDocumentMapper.updateDocument(TakaiDocument.builder()
|
|
|
@@ -692,17 +725,8 @@ public class TakaiAiServiceImpl implements ITakaiAiService {
|
|
|
int i = takaiDocumentMapper.delDocument(documentId);
|
|
|
|
|
|
if(i > 0){
|
|
|
- TakaiKnowledge info = takaiKnowledgeMapper.selectTargetKnowledge(TakaiKnowledge.builder().knowledgeId(vo.getKnowledgeId()).build());
|
|
|
- if(info != null){
|
|
|
- // 更新知识库文件大小,总字符数, 文件总数
|
|
|
- TakaiKnowledge paramInfo = TakaiKnowledge.builder().knowledgeId(vo.getKnowledgeId())
|
|
|
- .length(vo.getLength() == null ? info.getLength() == null ? 0 : info.getLength() : info.getLength() - vo.getLength()) // 文件大小
|
|
|
- .wordNum(vo.getWordNum() == null ? info.getWordNum() == null ? 0 : info.getWordNum() : info.getWordNum() - vo.getWordNum()) // 总字符数
|
|
|
- .documentSize(info.getDocumentSize() == null ? 0 : info.getDocumentSize() - 1) // 文件总数
|
|
|
- .build();
|
|
|
- takaiKnowledgeMapper.updateKnowledge(paramInfo);
|
|
|
- }
|
|
|
-
|
|
|
+ //更新知识库
|
|
|
+ updateKnowledgeSize(vo.getKnowledgeId());
|
|
|
// 删除minio文件
|
|
|
try {
|
|
|
logger.info("删除minio文件");
|
|
|
@@ -970,12 +994,14 @@ public class TakaiAiServiceImpl implements ITakaiAiService {
|
|
|
Integer code = obj.getInteger("code");
|
|
|
logger.info("删除切片调用python接口成功,返回结果:{}", body);
|
|
|
if (code == 200) {
|
|
|
+ updateDocumentSize(knowledgeId,documentId);
|
|
|
return 1;
|
|
|
}
|
|
|
}
|
|
|
} catch (IOException e) {
|
|
|
logger.error("删除切片调用python接口失败", e.getMessage());
|
|
|
}
|
|
|
+
|
|
|
return 0;
|
|
|
}
|
|
|
|
|
|
@@ -1000,6 +1026,34 @@ public class TakaiAiServiceImpl implements ITakaiAiService {
|
|
|
return null;
|
|
|
}
|
|
|
|
|
|
+ @Override
|
|
|
+ public List<String> uploadSliceImage(MultipartFile[] files, String knowledgeId, String documentId) {
|
|
|
+ List<TakaiSysOss> resultList = null;
|
|
|
+ try {
|
|
|
+ resultList = minioUtil.uploadSliceImages(files,knowledgeId,documentId);
|
|
|
+ } catch (Exception e) {
|
|
|
+ logger.error("上传文件失败", e.getMessage());
|
|
|
+ }
|
|
|
+ //保存图片信息
|
|
|
+ List<String> originTextList = new ArrayList<>();
|
|
|
+ for (TakaiSysOss result : resultList) {
|
|
|
+ TakaiMediaReplacement takaiMediaReplacement =
|
|
|
+ TakaiMediaReplacement.builder()
|
|
|
+ .knowledgeId(knowledgeId)
|
|
|
+ .documentId(documentId)
|
|
|
+ .originText(result.getOriginText())
|
|
|
+ .mediaType("image")
|
|
|
+ .mediaUrl(result.getUrl())
|
|
|
+ .build();
|
|
|
+ Date currentDate = new Date();
|
|
|
+ takaiMediaReplacement.setCreateTime(currentDate);
|
|
|
+ takaiMediaReplacement.setUpdateTime(currentDate);
|
|
|
+ takaiMediaReplacementMapper.insertMedia(takaiMediaReplacement);
|
|
|
+ originTextList.add(result.getOriginText());
|
|
|
+ }
|
|
|
+ return originTextList;
|
|
|
+ }
|
|
|
+
|
|
|
@Override
|
|
|
public int updateSliceInfo(TakaiSliceUpdateParams params) {
|
|
|
String url = deepseekConfig.getBaseurl() + deepseekConfig.getUpdateSlice();
|
|
|
@@ -1019,12 +1073,14 @@ public class TakaiAiServiceImpl implements ITakaiAiService {
|
|
|
JSONObject obj = JSON.parseObject(body);
|
|
|
Integer code = obj.getInteger("code");
|
|
|
if (code == 200) {
|
|
|
+ updateDocumentSize(params.getKnowledgeId(),params.getDocumentId());
|
|
|
return 1;
|
|
|
}
|
|
|
}
|
|
|
} catch (Exception e) {
|
|
|
logger.error("更新切片信息调用python接口失败", e.getMessage());
|
|
|
}
|
|
|
+
|
|
|
return 0;
|
|
|
}
|
|
|
|
|
|
@@ -1060,6 +1116,7 @@ public class TakaiAiServiceImpl implements ITakaiAiService {
|
|
|
JSONObject obj = JSON.parseObject(body);
|
|
|
Integer code = obj.getInteger("code");
|
|
|
if (code == 200) {
|
|
|
+ updateDocumentSize(params.getKnowledgeId(),params.getDocumentId());
|
|
|
return 1;
|
|
|
}
|
|
|
}
|
|
|
@@ -1070,9 +1127,10 @@ public class TakaiAiServiceImpl implements ITakaiAiService {
|
|
|
return 0;
|
|
|
}
|
|
|
|
|
|
+
|
|
|
@Override
|
|
|
- public List<TakaiSliceInfo> selectByDocumentId(String documentId) {
|
|
|
- return takaiSliceInfoMapper.selectByDocumentId(documentId);
|
|
|
+ public List<TakaiSliceInfo> selectByDocumentId(String documentId,String sliceText) {
|
|
|
+ return takaiSliceInfoMapper.selectByDocumentId(documentId,sliceText);
|
|
|
}
|
|
|
|
|
|
@Override
|
|
|
@@ -1111,13 +1169,16 @@ public class TakaiAiServiceImpl implements ITakaiAiService {
|
|
|
List<Object> result = new ArrayList();
|
|
|
if (dictDataList!= null && dictDataList.size() > 0) {
|
|
|
for (SysDictData dictData : dictDataList){
|
|
|
+ PageHelper.startPage(1, 3, "sort is null asc,sort");
|
|
|
Map<String, Object> map = new HashMap();
|
|
|
- TakaiApplication paramsVo = TakaiApplication.builder().approver(userId).flag("0").build();
|
|
|
+ TakaiApplication paramsVo = TakaiApplication.builder().approver(userId).flag("0").addOrderBy("0").build();
|
|
|
+ //项目应用
|
|
|
if(dictData.getDictCode() == 41){
|
|
|
Map<String, Object> l = searchAppTypeGroupList(paramsVo, dictData);
|
|
|
if(!MapUtils.isEmpty(l)){
|
|
|
result.add(l);
|
|
|
}
|
|
|
+ //其他分类应用
|
|
|
}else{
|
|
|
paramsVo.setTypeId(Long.valueOf(dictData.getDictCode()));
|
|
|
List<TakaiApplicationResult> list = takaiApplicationMapper.selectApplicationList(paramsVo);
|
|
|
@@ -1443,7 +1504,7 @@ public class TakaiAiServiceImpl implements ITakaiAiService {
|
|
|
return new OkHttpClient.Builder()
|
|
|
.connectTimeout(10, TimeUnit.SECONDS)
|
|
|
.writeTimeout(50, TimeUnit.SECONDS)
|
|
|
- .readTimeout(10, TimeUnit.MINUTES)
|
|
|
+ .readTimeout(20, TimeUnit.MINUTES)
|
|
|
.build();
|
|
|
}
|
|
|
|