From 85f47330bd153e2832a7d867859fd4b222cf1419 Mon Sep 17 00:00:00 2001 From: ArgonarioD Date: Wed, 5 Jul 2023 22:46:02 +0800 Subject: [PATCH] =?UTF-8?q?feat:=20=E4=BF=AE=E6=94=B9=E4=BA=86upload=5Ffil?= =?UTF-8?q?e=E6=B6=88=E6=81=AF=E6=A0=BC=E5=BC=8F=EF=BC=8C=E6=B7=BB?= =?UTF-8?q?=E5=8A=A0=E4=BA=86=E8=85=BE=E8=AE=AF=E4=BA=91COS=E6=9C=8D?= =?UTF-8?q?=E5=8A=A1=E7=9B=B8=E5=85=B3=E7=9A=84=E6=93=8D=E4=BD=9C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .dockerignore | 3 +- .gitignore | 3 +- .../AicsKnowledgeBase_file.csproj | 3 +- .../Controllers/FileController.cs | 30 ++++------------ .../Handlers/KnowledgeFileHandler.cs | 4 +++ .../Handlers/TencentCosHandler.cs | 35 +++++++++++++++++++ .../UpdateFileMessageConsumerHandler.cs | 16 +++++---- AicsKnowledgeBase_file/Models/FileMetadata.cs | 2 ++ .../Models/TencentCosOptions.cs | 9 +++++ AicsKnowledgeBase_file/Program.cs | 4 +++ AicsKnowledgeBase_file/appsettings.json | 15 ++++++++ 11 files changed, 90 insertions(+), 34 deletions(-) create mode 100644 AicsKnowledgeBase_file/Handlers/TencentCosHandler.cs create mode 100644 AicsKnowledgeBase_file/Models/TencentCosOptions.cs diff --git a/.dockerignore b/.dockerignore index cd967fc..2bc15c6 100644 --- a/.dockerignore +++ b/.dockerignore @@ -22,4 +22,5 @@ **/secrets.dev.yaml **/values.dev.yaml LICENSE -README.md \ No newline at end of file +README.md +AicsKnowledgeBase_file/files/ \ No newline at end of file diff --git a/.gitignore b/.gitignore index add57be..95b20af 100644 --- a/.gitignore +++ b/.gitignore @@ -2,4 +2,5 @@ bin/ obj/ /packages/ riderModule.iml -/_ReSharper.Caches/ \ No newline at end of file +/_ReSharper.Caches/ +AicsKnowledgeBase_file/files/ \ No newline at end of file diff --git a/AicsKnowledgeBase_file/AicsKnowledgeBase_file.csproj b/AicsKnowledgeBase_file/AicsKnowledgeBase_file.csproj index e655dad..fced843 100644 --- a/AicsKnowledgeBase_file/AicsKnowledgeBase_file.csproj +++ b/AicsKnowledgeBase_file/AicsKnowledgeBase_file.csproj @@ -10,9 +10,10 @@ - + + diff --git a/AicsKnowledgeBase_file/Controllers/FileController.cs b/AicsKnowledgeBase_file/Controllers/FileController.cs index 53ab54b..cb43b03 100644 --- a/AicsKnowledgeBase_file/Controllers/FileController.cs +++ b/AicsKnowledgeBase_file/Controllers/FileController.cs @@ -8,9 +8,11 @@ namespace AicsKnowledgeBase_file.Controllers; [Route("[controller]/{fileId}")] public class FileController : ControllerBase { private readonly KnowledgeFileHandler _knowledgeFileHandler; + private readonly TencentCosHandler _tencentCosHandler; - public FileController(KnowledgeFileHandler knowledgeFileHandler) { + public FileController(KnowledgeFileHandler knowledgeFileHandler, TencentCosHandler tencentCosHandler) { _knowledgeFileHandler = knowledgeFileHandler; + _tencentCosHandler = tencentCosHandler; } [HttpGet("status")] @@ -39,23 +41,6 @@ public class FileController : ControllerBase { } } - // [HttpPost("metadata")] - /*public async Task PostFileMetadata(string fileId, [FromBody] ClientFileMetadata metadata) { - var currentData = await _knowledgeFileHandler.GetFileMetadata(fileId); - if (currentData.Ticket != metadata.Ticket) { - return this.ProblemFromCode(ErrorCodes.TicketMismatch); - } - - if (currentData.Md5 != null || currentData.Size != null) { - return this.ProblemFromCode(ErrorCodes.FileMetadataConflict); - } - - currentData.Md5 = metadata.Md5; - currentData.Size = metadata.Size; - await _knowledgeFileHandler.SaveFileMetadata(fileId, currentData); - return Ok(); - }*/ - [HttpPost] public async Task UploadFilePart(string fileId, [FromForm] UploadFileDto dto) { var metadata = await _knowledgeFileHandler.GetFileMetadata(fileId); @@ -80,16 +65,13 @@ public class FileController : ControllerBase { return Problem(statusCode: StatusCodes.Status400BadRequest, title: e.Message); } + if (metadata.ExternalPreviewable) { + await _tencentCosHandler.UploadFile(fileId, metadata.Suffix); + } return Ok(); } } -/*public record ClientFileMetadata( - string Ticket, - string Md5, - ulong Size -);*/ - public record UploadFileDto( string Ticket, ulong RangeStart, diff --git a/AicsKnowledgeBase_file/Handlers/KnowledgeFileHandler.cs b/AicsKnowledgeBase_file/Handlers/KnowledgeFileHandler.cs index 0437729..3c981e5 100644 --- a/AicsKnowledgeBase_file/Handlers/KnowledgeFileHandler.cs +++ b/AicsKnowledgeBase_file/Handlers/KnowledgeFileHandler.cs @@ -38,6 +38,10 @@ public class KnowledgeFileHandler { return $"{FilePath}/{fileId}.part.{rangeStart}-{rangeEnd}"; } + public string GetFileName(string fileId) { + return $"{FilePath}/{fileId}"; + } + public async Task SaveFileSlice(string fileId, IFormFile file, ulong rangeStart, ulong rangeEnd) { await file.CopyToAsync(File.Create(GetSliceFileName(fileId, rangeStart, rangeEnd))); } diff --git a/AicsKnowledgeBase_file/Handlers/TencentCosHandler.cs b/AicsKnowledgeBase_file/Handlers/TencentCosHandler.cs new file mode 100644 index 0000000..a36b357 --- /dev/null +++ b/AicsKnowledgeBase_file/Handlers/TencentCosHandler.cs @@ -0,0 +1,35 @@ +using AicsKnowledgeBase_file.Models; +using COSXML; +using COSXML.Auth; +using COSXML.Transfer; +using Microsoft.Extensions.Options; + +namespace AicsKnowledgeBase_file.Handlers; + +public class TencentCosHandler { + private readonly TencentCosOptions _tencentCosOptions; + private readonly CosXml _cosXml; + private readonly TransferManager _transferManager; + private readonly KnowledgeFileHandler _knowledgeFileHandler; + + public TencentCosHandler(IOptions tencentCosOptions, KnowledgeFileHandler knowledgeFileHandler) { + _tencentCosOptions = tencentCosOptions.Value; + _knowledgeFileHandler = knowledgeFileHandler; + _cosXml = new CosXmlServer( + new CosXmlConfig.Builder() + .IsHttps(true) + .SetRegion(_tencentCosOptions.Region) + .Build(), + new DefaultQCloudCredentialProvider(_tencentCosOptions.SecretId, _tencentCosOptions.SecretKey, 600) + ); + _transferManager = new TransferManager(_cosXml, new TransferConfig { + SliceSizeForUpload = 3 * 1024 * 1024 + }); + } + + public async Task UploadFile(string fileId, string fileSuffix) { + var uploadTask = new COSXMLUploadTask(_tencentCosOptions.Bucket, $"{fileId}.{fileSuffix}"); + uploadTask.SetSrcPath(_knowledgeFileHandler.GetFileName(fileId)); + await _transferManager.UploadAsync(uploadTask); + } +} \ No newline at end of file diff --git a/AicsKnowledgeBase_file/Handlers/UpdateFileMessageConsumerHandler.cs b/AicsKnowledgeBase_file/Handlers/UpdateFileMessageConsumerHandler.cs index b38b725..1166d24 100644 --- a/AicsKnowledgeBase_file/Handlers/UpdateFileMessageConsumerHandler.cs +++ b/AicsKnowledgeBase_file/Handlers/UpdateFileMessageConsumerHandler.cs @@ -1,7 +1,6 @@ using AicsKnowledgeBase_file.Models; +using AicsKnowledgeBase_file.Utilities; using Confluent.Kafka; -using Confluent.Kafka.SyncOverAsync; -using Confluent.SchemaRegistry.Serdes; using Microsoft.Extensions.Options; using Newtonsoft.Json; @@ -32,20 +31,21 @@ public class UpdateFileMessageConsumerHandler : BackgroundService { protected override Task ExecuteAsync(CancellationToken stoppingToken) { return Task.Run(async () => { - using var consumer = new ConsumerBuilder(_consumerConfig) - .SetValueDeserializer(new JsonDeserializer().AsSyncOverAsync()) + using var consumer = new ConsumerBuilder(_consumerConfig) .Build(); consumer.Subscribe(TopicName); try { while (true) { try { var result = consumer.Consume(stoppingToken); - var msg = result.Message.Value!; + var msg = JsonUtils.Deserialize(result.Message.Value!)!; await _knowledgeFileHandler.SaveFileMetadata(msg.Id, new FileMetadata { IsCompleted = false, Ticket = msg.Ticket, Md5 = msg.Md5, - Size = msg.Size + Size = msg.Size, + ExternalPreviewable = msg.ExternalPreviewable, + Suffix = msg.Suffix }); } catch (ConsumeException e) { _logger.LogError("Consume Exception: {Exception}", e); @@ -63,5 +63,7 @@ public record FileTicket( [JsonProperty("ticket")] string Ticket, [JsonProperty("id")] string Id, [JsonProperty("md5")] string Md5, - [JsonProperty("size")] ulong Size + [JsonProperty("size")] ulong Size, + [JsonProperty("externalPreviewable")] bool ExternalPreviewable, + [JsonProperty("suffix")] string Suffix ); \ No newline at end of file diff --git a/AicsKnowledgeBase_file/Models/FileMetadata.cs b/AicsKnowledgeBase_file/Models/FileMetadata.cs index 6c08695..984915b 100644 --- a/AicsKnowledgeBase_file/Models/FileMetadata.cs +++ b/AicsKnowledgeBase_file/Models/FileMetadata.cs @@ -5,4 +5,6 @@ public class FileMetadata { public string Ticket { get; set; } = null!; public string Md5 { get; set; } = null!; public ulong Size { get; set; } = 0; + public bool ExternalPreviewable { get; set; } = false; + public string Suffix { get; set; } = null!; } \ No newline at end of file diff --git a/AicsKnowledgeBase_file/Models/TencentCosOptions.cs b/AicsKnowledgeBase_file/Models/TencentCosOptions.cs new file mode 100644 index 0000000..41b10ca --- /dev/null +++ b/AicsKnowledgeBase_file/Models/TencentCosOptions.cs @@ -0,0 +1,9 @@ +namespace AicsKnowledgeBase_file.Models; + +public class TencentCosOptions { + public const string SectionName = "Tencent:Cos"; + public string SecretId { get; set; } = null!; + public string SecretKey { get; set; } = null!; + public string Bucket { get; set; } = null!; + public string Region { get; set; } = null!; +} \ No newline at end of file diff --git a/AicsKnowledgeBase_file/Program.cs b/AicsKnowledgeBase_file/Program.cs index 5a270d3..c066dc7 100644 --- a/AicsKnowledgeBase_file/Program.cs +++ b/AicsKnowledgeBase_file/Program.cs @@ -3,6 +3,8 @@ using AicsKnowledgeBase_file.Handlers; using AicsKnowledgeBase_file.Models; using Microsoft.AspNetCore.Server.Kestrel.Core; +Directory.CreateDirectory("./files"); + var builder = WebApplication.CreateBuilder(args); builder.WebHost.UseUrls("https://*:8081"); builder.WebHost.UseKestrel((context, options) => { @@ -15,11 +17,13 @@ builder.WebHost.UseKestrel((context, options) => { // Add services to the container. builder.Services.Configure(builder.Configuration.GetSection(KafkaOptions.SectionName)); +builder.Services.Configure(builder.Configuration.GetSection(TencentCosOptions.SectionName)); builder.Services.Configure(options => { options.LowercaseUrls = true; }); builder.Services.AddControllers().AddJsonOptions(options => { options.JsonSerializerOptions.PropertyNamingPolicy = JsonNamingPolicy.CamelCase; options.JsonSerializerOptions.PropertyNameCaseInsensitive = true; }); +builder.Services.AddSingleton(); builder.Services.AddSingleton(); builder.Services.AddHostedService(); // Learn more about configuring Swagger/OpenAPI at https://aka.ms/aspnetcore/swashbuckle diff --git a/AicsKnowledgeBase_file/appsettings.json b/AicsKnowledgeBase_file/appsettings.json index 06f37d8..f5a6c93 100644 --- a/AicsKnowledgeBase_file/appsettings.json +++ b/AicsKnowledgeBase_file/appsettings.json @@ -2,6 +2,13 @@ "Logging": { "LogLevel": { "Default": "Warning" + }, + "Console": { + "FormatterName": "simple", + "FormatterOptions": { + "SingleLine": true, + "TimestampFormat": "MM-dd HH:mm:ss " + } } }, "AllowedHosts": "*", @@ -11,5 +18,13 @@ }, "Ssl": { "UseKeyFile": true + }, + "Tencent": { + "Cos": { + "SecretId": "AKIDSlvvhEYfYBetvYzCBvhJrDLGwNbcR2B7", + "SecretKey": "kHZigS3UkqlY8WiuypXM3ZwCHA0iHepp", + "Bucket": "aics-1300085057", + "Region": "ap-nanjing" + } } }