MinioService.py 21 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600
  1. import base64
  2. import os
  3. from datetime import timedelta, datetime
  4. from io import BytesIO
  5. import chardet
  6. import fitz
  7. import unicodedata
  8. from cfgv import ValidationError
  9. from django.core.files.base import ContentFile
  10. from django.db import transaction
  11. from minio import Minio, S3Error
  12. from minio.commonconfig import CopySource
  13. from DCbackend import settings
  14. from DCbackend.utils.common import success, fail
  15. from backend.models import Admin, Knowledgebase, File, DocumentKbm, File2document
  16. from pypinyin import lazy_pinyin, Style
  17. import re
  18. # 初始化 MinIO 客户端
  19. minio_client = Minio(
  20. settings.MINIO_ENDPOINT,
  21. access_key=settings.MINIO_ACCESS_KEY,
  22. secret_key=settings.MINIO_SECRET_KEY,
  23. secure=settings.MINIO_SECURE
  24. )
  25. def contains_chinese(text):
  26. """检查字符串是否包含中文字符"""
  27. return any('\u4e00' <= char <= '\u9fff' for char in text)
  28. def convert_to_pinyin(text):
  29. """
  30. 将中文字符转换为不带声调的拼音,保留字母和数字,其他字符替换为连字符,
  31. 并确保生成的名称符合 bucket 命名规则
  32. """
  33. result = []
  34. for char in text:
  35. if '\u4e00' <= char <= '\u9fff':
  36. # 中文字符
  37. pinyin = lazy_pinyin(char, style=Style.NORMAL)
  38. result.extend(pinyin)
  39. elif char.isalnum():
  40. # 字母和数字
  41. result.append(char.lower())
  42. else:
  43. # 其他字符替换为连字符
  44. result.append('-')
  45. # 合并字符并去除声调
  46. bucket_name = ''.join(result)
  47. bucket_name = unicodedata.normalize('NFKD', bucket_name).encode('ASCII', 'ignore').decode('ASCII')
  48. # 合并连续的连字符
  49. bucket_name = re.sub(r'-+', '-', bucket_name)
  50. # 确保名称以字母或数字开头和结尾
  51. bucket_name = bucket_name.strip('-')
  52. # 如果名称为空,使用默认名称
  53. if not bucket_name:
  54. bucket_name = 'default-bucket'
  55. # 确保名称长度在 3-63 之间
  56. if len(bucket_name) < 3:
  57. bucket_name = bucket_name.ljust(3, 'a')
  58. elif len(bucket_name) > 63:
  59. bucket_name = bucket_name[:63]
  60. return bucket_name.lower()
  61. class MinioService:
  62. @staticmethod
  63. def listBuckets(request):
  64. #查询所有bucket信息
  65. buckets = minio_client.list_buckets()
  66. # 将 Bucket 对象转换为可序列化的字典,包含所有可用信息
  67. serializable_buckets = []
  68. for bucket in buckets:
  69. # 处理创建日期
  70. if isinstance(bucket.creation_date, datetime):
  71. # 添加8小时
  72. adjusted_date = bucket.creation_date + timedelta(hours=8)
  73. formatted_date = adjusted_date.strftime("%Y-%m-%d %H:%M:%S")
  74. else:
  75. formatted_date = str(bucket.creation_date)
  76. bucket_info = {
  77. 'bucket_name': bucket.name,
  78. 'creation_date': formatted_date,
  79. }
  80. #根据桶名查询数量
  81. try:
  82. objects = minio_client.list_objects(bucket.name, recursive=True)
  83. file_count = sum(1 for _ in objects)
  84. bucket_info['file_count'] = file_count
  85. except Exception as e:
  86. bucket_info['file_count'] = f"Error: {str(e)}"
  87. serializable_buckets.append(bucket_info)
  88. return success(serializable_buckets)
  89. @staticmethod
  90. def createBucket(request):
  91. """创建新的 bucket"""
  92. try:
  93. user_id = request.POST.get("user_id")
  94. # 从请求中获取 bucket 名称
  95. bucket_name = request.POST.get('bucket_name')
  96. description = request.POST.get('description', "")
  97. if not bucket_name:
  98. return fail('Bucket name is required')
  99. minio_name = bucket_name
  100. if contains_chinese(bucket_name):
  101. minio_name = convert_to_pinyin(bucket_name)
  102. # 检查 bucket 是否已存在
  103. if minio_client.bucket_exists(minio_name):
  104. return fail(f'Bucket "{minio_name}" 已存在')
  105. # 创建 bucket
  106. minio_client.make_bucket(minio_name)
  107. # 同步数据库
  108. return MinioService.saveBucketDb(bucket_name, user_id, minio_name, description)
  109. except S3Error as e:
  110. return fail(f'Failed to create bucket: {str(e)}')
  111. except Exception as e:
  112. return fail(f'An unexpected error occurred: {str(e)}')
  113. @staticmethod
  114. def saveBucketDb(bucket_name, user_id,minio_name,description):
  115. try:
  116. admin = Admin.objects.get(id=user_id)
  117. except Admin.DoesNotExist:
  118. return fail("用户信息不存在")
  119. try:
  120. # 假设 Knowledgebase 是您的模型类名
  121. db = Knowledgebase(
  122. role_id=admin.role_id,
  123. name=bucket_name,
  124. location=minio_name,
  125. description=description,
  126. created_by=user_id
  127. )
  128. db.save()
  129. file = File.objects.filter(
  130. role_id=admin.role_id,
  131. created_by=user_id,
  132. name='/',
  133. source_type=''
  134. ).exclude(status=4).first()
  135. if file is None:
  136. # 如果记录不存在,创建新记录
  137. file = File.objects.create(
  138. role_id=admin.role_id,
  139. name='/',
  140. source_type='',
  141. created_by= user_id,
  142. location= '',
  143. type= "folder",
  144. status= 5,
  145. )
  146. # 创建后更新parent_id
  147. file.parent_id = file.id
  148. file.save()
  149. kbm = File.objects.create(
  150. role_id=admin.role_id,
  151. name='.knowledgebase', # 根据需要修改名称
  152. source_type='knowledgebase',
  153. created_by=user_id,
  154. location='',
  155. type="folder",
  156. status=5,
  157. parent_id=file.id # 设置parent_id为主记录的id
  158. )
  159. bucket = File.objects.create(
  160. role_id=admin.role_id,
  161. name=bucket_name, # 根据需要修改名称
  162. source_type='knowledgebase',
  163. created_by=user_id,
  164. location='',
  165. type="folder",
  166. status=5,
  167. parent_id=kbm.id
  168. )
  169. else:
  170. kbm = File.objects.get(
  171. role_id=admin.role_id,
  172. created_by=user_id,
  173. name='.knowledgebase',
  174. type='folder',
  175. source_type='knowledgebase'
  176. )
  177. bucket = File.objects.create(
  178. role_id=admin.role_id,
  179. name=bucket_name, # 根据需要修改名称
  180. source_type='knowledgebase',
  181. created_by=user_id,
  182. location='',
  183. type="folder",
  184. status=5,
  185. parent_id=kbm.id
  186. )
  187. except ValidationError as e:
  188. return fail(f"验证错误: {e}")
  189. except Exception as e:
  190. return fail(f"保存失败: {str(e)}")
  191. return success(f'Bucket "{bucket_name}" 创建成功')
  192. @staticmethod
  193. def is_valid_bucket_name(bucket_name):
  194. """检查 bucket 名称是否合法"""
  195. import re
  196. # bucket 名称必须在 3-63 个字符之间,只能包含小写字母、数字和连字符
  197. pattern = r'^[a-z0-9][a-z0-9\-]{1,61}[a-z0-9]$'
  198. return re.match(pattern, bucket_name) is not None
  199. #获取指定buck内文件信息
  200. @staticmethod
  201. def getBucketContents(request):
  202. """获取指定 bucket 内的所有文件信息"""
  203. try:
  204. # 从请求中获取 bucket 名称
  205. bucket_name = request.POST.get('bucket_name')
  206. # 使用 get 方法获取 page 和 page_size,如果不存在则使用默认值
  207. page = int(request.POST.get('page', 1))
  208. page_size = int(request.POST.get('page_size', 10))
  209. print(page, page_size)
  210. if not bucket_name:
  211. return fail('请求参数为空')
  212. # 检查 bucket 是否存在
  213. if not minio_client.bucket_exists(bucket_name):
  214. return fail(f'Bucket "{bucket_name}" 不存在')
  215. # 获取 bucket 内的所有对象
  216. objects = minio_client.list_objects(bucket_name, recursive=True)
  217. # 整理文件信息
  218. file_info = list(objects) # 转换为列表以获得准确的长度
  219. # 计算总数和总页数
  220. total_count = len(file_info)
  221. total_pages = (total_count + page_size - 1) // page_size
  222. # 确保页码在有效范围内
  223. page = max(1, min(page, total_pages))
  224. # 计算切片的起始和结束索引
  225. start_index = (page - 1) * page_size
  226. end_index = min(start_index + page_size, total_count)
  227. # 获取当前页的数据
  228. paginated_files = [
  229. {
  230. 'object_name': obj.object_name,
  231. 'size': obj.size,
  232. 'last_modified': obj.last_modified,
  233. 'version_id': obj.version_id,
  234. 'etag': obj.etag
  235. }
  236. for obj in file_info[start_index:end_index]
  237. ]
  238. return success({
  239. 'bucket_name': bucket_name,
  240. 'files': paginated_files,
  241. 'page': page,
  242. 'page_size': page_size,
  243. 'total_pages': total_pages,
  244. 'total_count': total_count
  245. })
  246. except S3Error as e:
  247. return fail(f'Failed to get bucket contents: {str(e)}')
  248. except Exception as e:
  249. return fail(f'An unexpected error occurred: {str(e)}')
  250. @staticmethod
  251. @transaction.atomic()
  252. def post(request):
  253. """上传文件"""
  254. uploaded_file = request.FILES['file'] # 获取上传的文件
  255. bucket_id = request.POST.get('bucket_id') # BUCKET的名称
  256. user_id = request.POST.get('user_id') # BUCKET的名称
  257. file_path = request.POST.get('file_path', '')
  258. doc_type_id = request.POST.get('doc_type_id',0)
  259. if not uploaded_file:
  260. return fail('没有需要上传的文件')
  261. if not bucket_id:
  262. return fail('bucket_id为空')
  263. # 使用 file_path 和文件名构造对象名称
  264. object_name = f"{file_path.strip('/')}/{uploaded_file.name}".lstrip('/')
  265. count = DocumentKbm.objects.filter(name=object_name, kb_id=bucket_id).exclude(status=4).count()
  266. if count > 0:
  267. return fail("已有重复文件,请删除后重试")
  268. try:
  269. # 读取文件内容
  270. file_content = uploaded_file.read()
  271. # 使用 MinIO 客户端上传文件
  272. knowledgebase = Knowledgebase.objects.get(id=bucket_id)
  273. bucket_name = knowledgebase.location
  274. minio_client.put_object(
  275. bucket_name,
  276. object_name,
  277. ContentFile(file_content),
  278. length=len(file_content),
  279. content_type=uploaded_file.content_type
  280. )
  281. MinioService.saveDocumentKbm(knowledgebase, uploaded_file, user_id, object_name,doc_type_id)
  282. return success("保存成功")
  283. except Exception as e:
  284. return fail(str(e))
  285. def is_image_file(extension):
  286. image_extensions = [
  287. 'jpg', 'jpeg', 'png', 'gif', 'bmp', 'tiff', 'webp',
  288. 'svg', 'raw', 'heif', 'heic', 'indd',
  289. 'ai', 'eps', 'psd', 'xcf', 'cr2', 'nef', 'orf', 'sr2',
  290. 'jfif', 'exif', 'ico', 'tga'
  291. ]
  292. return extension.lower().strip('.') in image_extensions
  293. def is_text_file(extension):
  294. text_extensions = [
  295. 'txt', 'pdf', 'doc', 'docx', 'rtf', 'odt', 'xls', 'xlsx',
  296. 'csv', 'tsv', 'json', 'xml', 'html', 'htm', 'md', 'markdown',
  297. 'tex', 'log', 'ini', 'cfg', 'conf', 'py', 'js', 'css', 'scss',
  298. 'less', 'sql', 'php', 'java', 'c', 'cpp', 'h', 'hpp', 'sh',
  299. 'bat', 'ps1', 'rb', 'yaml', 'yml', 'toml', 'rst', 'asciidoc',
  300. 'ppt', 'pptx', 'odp', 'key', 'pages', 'numbers'
  301. ]
  302. return extension.lower().strip('.') in text_extensions
  303. @staticmethod
  304. @transaction.atomic
  305. def saveDocumentKbm(knowledgebase,uploaded_file,user_id,object_name,doc_type_id):
  306. try:
  307. size = uploaded_file.size
  308. _, file_extension = os.path.splitext(uploaded_file.name)
  309. # file_extension 现在包含了文件的后缀名,包括点号(例如 ".txt")
  310. # 如果您不想要点号,可以这样做:
  311. file_extension = file_extension[1:] if file_extension else ''
  312. # 根据文件类型设置 parser_id
  313. if MinioService.is_image_file(file_extension):
  314. parser_id = 'picture'
  315. elif MinioService.is_text_file(file_extension):
  316. parser_id = 'naive'
  317. documentKbm = DocumentKbm.objects.create(
  318. kb_id=knowledgebase.id,
  319. parser_id=parser_id,
  320. parser_config='{"pages": [[1, 1000000]]}',
  321. type = file_extension,
  322. created_by=user_id,
  323. name=object_name,
  324. location=object_name,
  325. size=size,
  326. doc_type_id=doc_type_id
  327. )
  328. doc_id = documentKbm.id
  329. admin = Admin.objects.get(id=user_id)
  330. file = File.objects.filter(
  331. role_id=admin.role_id,
  332. created_by=user_id,
  333. name=knowledgebase.name,
  334. source_type='knowledgebase',
  335. type='folder'
  336. ).exclude(status=4).first()
  337. fileDB = File.objects.create(
  338. role_id=admin.role_id,
  339. name=object_name,
  340. source_type='knowledgebase',
  341. created_by=user_id,
  342. location=object_name,
  343. type=file_extension,
  344. status=5,
  345. size=size,
  346. parent_id= file.id
  347. )
  348. file_id = fileDB.id
  349. file2document = File2document.objects.create(
  350. file_id=file_id,
  351. document_id=doc_id
  352. )
  353. child_count = File.objects.filter(parent_id=file.id).exclude(status=4).count()
  354. knowledgebase.doc_num = child_count
  355. knowledgebase.save()
  356. except Exception as e:
  357. return fail(str(e))
  358. # 根据名称获取地址
  359. @staticmethod
  360. def nameGetUrl(request):
  361. object_name = request.POST.get('object_name')
  362. bucket_name = request.POST.get('bucket_name')
  363. return MinioService.geturl(object_name,bucket_name)
  364. @staticmethod
  365. def geturl(name,bucket_name):
  366. """获取文件地址"""
  367. object_name = name
  368. if not object_name:
  369. return fail('Object name is required')
  370. try:
  371. # 生成一个预签名 URL,有效期为1小时
  372. url = minio_client.presigned_get_object(
  373. bucket_name,
  374. object_name,
  375. expires=timedelta(hours=1)
  376. )
  377. return success({'url': url})
  378. except Exception as e:
  379. return fail(str(e))
  380. @staticmethod
  381. def deleteFile(request):
  382. """根据名称删除文件"""
  383. bucket_name = request.POST.get('bucket_name')
  384. object_name = request.POST.get('object_name')
  385. if not object_name:
  386. return fail('需要删除的文件未找到')
  387. try:
  388. minio_client.remove_object(bucket_name, object_name)
  389. return success('删除成功')
  390. except S3Error as e:
  391. return fail(f'Failed to delete file: {str(e)}')
  392. except Exception as e:
  393. return fail(f'An unexpected error occurred: {str(e)}')
  394. @staticmethod
  395. def renameFile(request):
  396. """重命名 bucket 中的文件"""
  397. bucket_name = request.POST.get('bucket_name')
  398. old_name = request.POST.get('object_name')
  399. new_name = request.POST.get('new_name')
  400. if not old_name or not new_name:
  401. return fail('原文件名或新文件名未提供')
  402. # 检查新文件名是否包含后缀,如果没有则添加原文件的后缀
  403. if '.' not in new_name:
  404. old_extension = old_name.split('.')[-1] if '.' in old_name else ''
  405. new_name = f"{new_name}.{old_extension}" if old_extension else new_name
  406. try:
  407. # 复制对象到新名称
  408. result = minio_client.copy_object(
  409. bucket_name, new_name,
  410. CopySource(bucket_name, old_name)
  411. )
  412. # 如果复制成功,删除原对象
  413. if result:
  414. minio_client.remove_object(bucket_name, old_name)
  415. return success("修改昵称成功")
  416. else:
  417. return fail('文件重命名失败')
  418. except S3Error as e:
  419. return fail(f'重命名文件失败: {str(e)}')
  420. except Exception as e:
  421. return fail(f'发生意外错误: {str(e)}')
  422. @staticmethod
  423. def delete_bucket(request):
  424. """删除 bucket"""
  425. bucket_name = request.POST.get('bucket_name')
  426. if not bucket_name:
  427. return fail({'bucket_name 为空'})
  428. try:
  429. # 开始事务
  430. with transaction.atomic():
  431. # 1. 从 MinIO 删除 bucket
  432. try:
  433. # 首先删除 bucket 中的所有对象
  434. objects = minio_client.list_objects(bucket_name, recursive=True)
  435. for obj in objects:
  436. minio_client.remove_object(bucket_name, obj.object_name)
  437. # 然后删除 bucket
  438. minio_client.remove_bucket(bucket_name)
  439. except S3Error as e:
  440. return fail(f'MinIO 错误: {str(e)}')
  441. # 2. 从数据库中删除相应记录
  442. try:
  443. bucket = Knowledgebase.objects.get(name=bucket_name)
  444. bucket.delete()
  445. except Knowledgebase.DoesNotExist:
  446. return fail('数据库中不存在该 bucket 记录')
  447. return fail(f'Bucket "{bucket_name}" 已成功删除')
  448. except Exception as e:
  449. return fail(f'删除 bucket 时发生错误: {str(e)}')
  450. @staticmethod
  451. def deleteBucket(request):
  452. return MinioService.delete_bucket(request)
  453. #切片
  454. @staticmethod
  455. def readPdfSlice(request):
  456. """读取 PDF 切片,包括文本和图片"""
  457. bucket_name = request.POST.get('bucket_name')
  458. object_name = request.POST.get('object_name')
  459. start_page = int(request.POST.get('start_page', 1))
  460. end_page = int(request.POST.get('end_page', -1))
  461. if not bucket_name or not object_name:
  462. return fail('bucket_name 或 object_name 为空')
  463. try:
  464. # 从 MinIO 获取 PDF 文件
  465. response = minio_client.get_object(bucket_name, object_name)
  466. pdf_content = BytesIO(response.read())
  467. # 使用 PyMuPDF 读取 PDF
  468. doc = fitz.open(stream=pdf_content, filetype="pdf")
  469. total_pages = len(doc)
  470. # 调整页面范围
  471. start_page = max(1, start_page) - 1 # 转换为从 0 开始的索引
  472. end_page = min(total_pages, end_page if end_page > 0 else total_pages)
  473. # 读取指定页面范围的内容
  474. result = []
  475. for page_num in range(start_page, end_page):
  476. page = doc[page_num]
  477. text = page.get_text()
  478. # 提取图片
  479. images = []
  480. for img in page.get_images():
  481. xref = img[0]
  482. base_image = doc.extract_image(xref)
  483. image_data = base_image["image"]
  484. image_format = base_image["ext"]
  485. image_base64 = base64.b64encode(image_data).decode('utf-8')
  486. images.append({
  487. 'format': image_format,
  488. 'data': image_base64
  489. })
  490. result.append({
  491. 'page_number': page_num + 1,
  492. 'content': text,
  493. 'images': images
  494. })
  495. # 关闭连接
  496. doc.close()
  497. pdf_content.close()
  498. response.close()
  499. info = {
  500. 'total_pages': total_pages,
  501. 'sliced_content': result
  502. }
  503. return success(info)
  504. except Exception as e:
  505. return fail(str(e))