feat: 修改了upload_file消息格式，添加了腾讯云COS服务相关的操作

.dockerignore

@@ -23,3 +23,4 @@
 **/values.dev.yaml
 LICENSE
 README.md
+AicsKnowledgeBase_file/files/

.gitignore

@@ -3,3 +3,4 @@ obj/
 /packages/
 riderModule.iml
 /_ReSharper.Caches/
+AicsKnowledgeBase_file/files/

AicsKnowledgeBase_file/AicsKnowledgeBase_file.csproj

@@ -10,9 +10,10 @@
 
     <ItemGroup>
         <PackageReference Include="Confluent.Kafka" Version="2.1.1" />
-        <PackageReference Include="Confluent.SchemaRegistry.Serdes.Json" Version="2.1.1" />
         <PackageReference Include="Microsoft.AspNetCore.OpenApi" Version="7.0.2" />
+        <PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
         <PackageReference Include="Swashbuckle.AspNetCore" Version="6.4.0" />
+        <PackageReference Include="Tencent.QCloud.Cos.Sdk" Version="5.4.34" />
     </ItemGroup>
 
     <ItemGroup>

AicsKnowledgeBase_file/Controllers/FileController.cs

@@ -8,9 +8,11 @@ namespace AicsKnowledgeBase_file.Controllers;
 [Route("[controller]/{fileId}")]
 public class FileController : ControllerBase {
     private readonly KnowledgeFileHandler _knowledgeFileHandler;
+    private readonly TencentCosHandler _tencentCosHandler;
 
-    public FileController(KnowledgeFileHandler knowledgeFileHandler) {
+    public FileController(KnowledgeFileHandler knowledgeFileHandler, TencentCosHandler tencentCosHandler) {
         _knowledgeFileHandler = knowledgeFileHandler;
+        _tencentCosHandler = tencentCosHandler;
     }
 
     [HttpGet("status")]
@@ -39,23 +41,6 @@ public class FileController : ControllerBase {
         }
     }
 
-    // [HttpPost("metadata")]
-    /*public async Task<IActionResult> PostFileMetadata(string fileId, [FromBody] ClientFileMetadata metadata) {
-        var currentData = await _knowledgeFileHandler.GetFileMetadata(fileId);
-        if (currentData.Ticket != metadata.Ticket) {
-            return this.ProblemFromCode(ErrorCodes.TicketMismatch);
-        }
-
-        if (currentData.Md5 != null || currentData.Size != null) {
-            return this.ProblemFromCode(ErrorCodes.FileMetadataConflict);
-        }
-
-        currentData.Md5 = metadata.Md5;
-        currentData.Size = metadata.Size;
-        await _knowledgeFileHandler.SaveFileMetadata(fileId, currentData);
-        return Ok();
-    }*/
-
     [HttpPost]
     public async Task<IActionResult> UploadFilePart(string fileId, [FromForm] UploadFileDto dto) {
         var metadata = await _knowledgeFileHandler.GetFileMetadata(fileId);
@@ -80,16 +65,13 @@ public class FileController : ControllerBase {
             return Problem(statusCode: StatusCodes.Status400BadRequest, title: e.Message);
         }
 
+        if (metadata.ExternalPreviewable) {
+            await _tencentCosHandler.UploadFile(fileId, metadata.Suffix);
+        }
         return Ok();
     }
 }
 
-/*public record ClientFileMetadata(
-    string Ticket,
-    string Md5,
-    ulong Size
-);*/
-
 public record UploadFileDto(
     string Ticket,
     ulong RangeStart,

AicsKnowledgeBase_file/Handlers/KnowledgeFileHandler.cs

@@ -38,6 +38,10 @@ public class KnowledgeFileHandler {
         return $"{FilePath}/{fileId}.part.{rangeStart}-{rangeEnd}";
     }
 
+    public string GetFileName(string fileId) {
+        return $"{FilePath}/{fileId}";
+    }
+
     public async Task SaveFileSlice(string fileId, IFormFile file, ulong rangeStart, ulong rangeEnd) {
         await file.CopyToAsync(File.Create(GetSliceFileName(fileId, rangeStart, rangeEnd)));
     }

AicsKnowledgeBase_file/Handlers/TencentCosHandler.cs

@@ -0,0 +1,35 @@
+using AicsKnowledgeBase_file.Models;
+using COSXML;
+using COSXML.Auth;
+using COSXML.Transfer;
+using Microsoft.Extensions.Options;
+
+namespace AicsKnowledgeBase_file.Handlers; 
+
+public class TencentCosHandler {
+    private readonly TencentCosOptions _tencentCosOptions;
+    private readonly CosXml _cosXml;
+    private readonly TransferManager _transferManager;
+    private readonly KnowledgeFileHandler _knowledgeFileHandler;
+
+    public TencentCosHandler(IOptions<TencentCosOptions> tencentCosOptions, KnowledgeFileHandler knowledgeFileHandler) {
+        _tencentCosOptions = tencentCosOptions.Value;
+        _knowledgeFileHandler = knowledgeFileHandler;
+        _cosXml = new CosXmlServer(
+            new CosXmlConfig.Builder()
+                .IsHttps(true)
+                .SetRegion(_tencentCosOptions.Region)
+                .Build(),
+            new DefaultQCloudCredentialProvider(_tencentCosOptions.SecretId, _tencentCosOptions.SecretKey, 600)
+        );
+        _transferManager = new TransferManager(_cosXml, new TransferConfig {
+            SliceSizeForUpload = 3 * 1024 * 1024
+        });
+    }
+
+    public async Task UploadFile(string fileId, string fileSuffix) {
+        var uploadTask = new COSXMLUploadTask(_tencentCosOptions.Bucket, $"{fileId}.{fileSuffix}");
+        uploadTask.SetSrcPath(_knowledgeFileHandler.GetFileName(fileId));
+        await _transferManager.UploadAsync(uploadTask);
+    }
+}

AicsKnowledgeBase_file/Handlers/UpdateFileMessageConsumerHandler.cs

@@ -1,7 +1,6 @@
 using AicsKnowledgeBase_file.Models;
+using AicsKnowledgeBase_file.Utilities;
 using Confluent.Kafka;
-using Confluent.Kafka.SyncOverAsync;
-using Confluent.SchemaRegistry.Serdes;
 using Microsoft.Extensions.Options;
 using Newtonsoft.Json;
 
@@ -32,20 +31,21 @@ public class UpdateFileMessageConsumerHandler : BackgroundService {
 
     protected override Task ExecuteAsync(CancellationToken stoppingToken) {
         return Task.Run(async () => {
-            using var consumer = new ConsumerBuilder<string, FileTicket>(_consumerConfig)
+            using var consumer = new ConsumerBuilder<string, string>(_consumerConfig)
-                .SetValueDeserializer(new JsonDeserializer<FileTicket>().AsSyncOverAsync())
                 .Build();
             consumer.Subscribe(TopicName);
             try {
                 while (true) {
                     try {
                         var result = consumer.Consume(stoppingToken);
-                        var msg = result.Message.Value!;
+                        var msg = JsonUtils.Deserialize<FileTicket>(result.Message.Value!)!;
                         await _knowledgeFileHandler.SaveFileMetadata(msg.Id, new FileMetadata {
                             IsCompleted = false,
                             Ticket = msg.Ticket,
                             Md5 = msg.Md5,
-                            Size = msg.Size
+                            Size = msg.Size,
+                            ExternalPreviewable = msg.ExternalPreviewable,
+                            Suffix = msg.Suffix
                         });
                     } catch (ConsumeException e) {
                         _logger.LogError("Consume Exception: {Exception}", e);
@@ -63,5 +63,7 @@ public record FileTicket(
     [JsonProperty("ticket")] string Ticket,
     [JsonProperty("id")] string Id,
     [JsonProperty("md5")] string Md5,
-    [JsonProperty("size")] ulong Size
+    [JsonProperty("size")] ulong Size,
+    [JsonProperty("externalPreviewable")] bool ExternalPreviewable,
+    [JsonProperty("suffix")] string Suffix
 );

AicsKnowledgeBase_file/Models/FileMetadata.cs

@@ -5,4 +5,6 @@ public class FileMetadata {
     public string Ticket { get; set; } = null!;
     public string Md5 { get; set; } = null!;
     public ulong Size { get; set; } = 0;
+    public bool ExternalPreviewable { get; set; } = false;
+    public string Suffix { get; set; } = null!;
 }

AicsKnowledgeBase_file/Models/TencentCosOptions.cs

@@ -0,0 +1,9 @@
+namespace AicsKnowledgeBase_file.Models; 
+
+public class TencentCosOptions {
+    public const string SectionName = "Tencent:Cos";
+    public string SecretId { get; set; } = null!;
+    public string SecretKey { get; set; } = null!;
+    public string Bucket { get; set; } = null!;
+    public string Region { get; set; } = null!;
+}

AicsKnowledgeBase_file/Program.cs

@@ -3,6 +3,8 @@ using AicsKnowledgeBase_file.Handlers;
 using AicsKnowledgeBase_file.Models;
 using Microsoft.AspNetCore.Server.Kestrel.Core;
 
+Directory.CreateDirectory("./files");
+
 var builder = WebApplication.CreateBuilder(args);
 builder.WebHost.UseUrls("https://*:8081");
 builder.WebHost.UseKestrel((context, options) => {
@@ -15,11 +17,13 @@ builder.WebHost.UseKestrel((context, options) => {
 
 // Add services to the container.
 builder.Services.Configure<KafkaOptions>(builder.Configuration.GetSection(KafkaOptions.SectionName));
+builder.Services.Configure<TencentCosOptions>(builder.Configuration.GetSection(TencentCosOptions.SectionName));
 builder.Services.Configure<RouteOptions>(options => { options.LowercaseUrls = true; });
 builder.Services.AddControllers().AddJsonOptions(options => {
     options.JsonSerializerOptions.PropertyNamingPolicy = JsonNamingPolicy.CamelCase;
     options.JsonSerializerOptions.PropertyNameCaseInsensitive = true;
 });
+builder.Services.AddSingleton<TencentCosHandler>();
 builder.Services.AddSingleton<KnowledgeFileHandler>();
 builder.Services.AddHostedService<UpdateFileMessageConsumerHandler>();
 // Learn more about configuring Swagger/OpenAPI at https://aka.ms/aspnetcore/swashbuckle

AicsKnowledgeBase_file/appsettings.json

@@ -2,6 +2,13 @@
   "Logging": {
     "LogLevel": {
       "Default": "Warning"
+    },
+    "Console": {
+      "FormatterName": "simple",
+      "FormatterOptions": {
+        "SingleLine": true,
+        "TimestampFormat": "MM-dd HH:mm:ss "
+      }
     }
   },
   "AllowedHosts": "*",
@@ -11,5 +18,13 @@
   },
   "Ssl": {
     "UseKeyFile": true
+  },
+  "Tencent": {
+    "Cos": {
+      "SecretId": "AKIDSlvvhEYfYBetvYzCBvhJrDLGwNbcR2B7",
+      "SecretKey": "kHZigS3UkqlY8WiuypXM3ZwCHA0iHepp",
+      "Bucket": "aics-1300085057",
+      "Region": "ap-nanjing"
+    }
   }
 }