feat: 修改了upload_file消息格式,添加了腾讯云COS服务相关的操作

master
ArgonarioD 2023-07-05 22:46:02 +08:00
parent 94e28b5725
commit 85f47330bd
11 changed files with 90 additions and 34 deletions

View File

@ -23,3 +23,4 @@
**/values.dev.yaml **/values.dev.yaml
LICENSE LICENSE
README.md README.md
AicsKnowledgeBase_file/files/

1
.gitignore vendored
View File

@ -3,3 +3,4 @@ obj/
/packages/ /packages/
riderModule.iml riderModule.iml
/_ReSharper.Caches/ /_ReSharper.Caches/
AicsKnowledgeBase_file/files/

View File

@ -10,9 +10,10 @@
<ItemGroup> <ItemGroup>
<PackageReference Include="Confluent.Kafka" Version="2.1.1" /> <PackageReference Include="Confluent.Kafka" Version="2.1.1" />
<PackageReference Include="Confluent.SchemaRegistry.Serdes.Json" Version="2.1.1" />
<PackageReference Include="Microsoft.AspNetCore.OpenApi" Version="7.0.2" /> <PackageReference Include="Microsoft.AspNetCore.OpenApi" Version="7.0.2" />
<PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
<PackageReference Include="Swashbuckle.AspNetCore" Version="6.4.0" /> <PackageReference Include="Swashbuckle.AspNetCore" Version="6.4.0" />
<PackageReference Include="Tencent.QCloud.Cos.Sdk" Version="5.4.34" />
</ItemGroup> </ItemGroup>
<ItemGroup> <ItemGroup>

View File

@ -8,9 +8,11 @@ namespace AicsKnowledgeBase_file.Controllers;
[Route("[controller]/{fileId}")] [Route("[controller]/{fileId}")]
public class FileController : ControllerBase { public class FileController : ControllerBase {
private readonly KnowledgeFileHandler _knowledgeFileHandler; private readonly KnowledgeFileHandler _knowledgeFileHandler;
private readonly TencentCosHandler _tencentCosHandler;
public FileController(KnowledgeFileHandler knowledgeFileHandler) { public FileController(KnowledgeFileHandler knowledgeFileHandler, TencentCosHandler tencentCosHandler) {
_knowledgeFileHandler = knowledgeFileHandler; _knowledgeFileHandler = knowledgeFileHandler;
_tencentCosHandler = tencentCosHandler;
} }
[HttpGet("status")] [HttpGet("status")]
@ -39,23 +41,6 @@ public class FileController : ControllerBase {
} }
} }
// [HttpPost("metadata")]
/*public async Task<IActionResult> PostFileMetadata(string fileId, [FromBody] ClientFileMetadata metadata) {
var currentData = await _knowledgeFileHandler.GetFileMetadata(fileId);
if (currentData.Ticket != metadata.Ticket) {
return this.ProblemFromCode(ErrorCodes.TicketMismatch);
}
if (currentData.Md5 != null || currentData.Size != null) {
return this.ProblemFromCode(ErrorCodes.FileMetadataConflict);
}
currentData.Md5 = metadata.Md5;
currentData.Size = metadata.Size;
await _knowledgeFileHandler.SaveFileMetadata(fileId, currentData);
return Ok();
}*/
[HttpPost] [HttpPost]
public async Task<IActionResult> UploadFilePart(string fileId, [FromForm] UploadFileDto dto) { public async Task<IActionResult> UploadFilePart(string fileId, [FromForm] UploadFileDto dto) {
var metadata = await _knowledgeFileHandler.GetFileMetadata(fileId); var metadata = await _knowledgeFileHandler.GetFileMetadata(fileId);
@ -80,16 +65,13 @@ public class FileController : ControllerBase {
return Problem(statusCode: StatusCodes.Status400BadRequest, title: e.Message); return Problem(statusCode: StatusCodes.Status400BadRequest, title: e.Message);
} }
if (metadata.ExternalPreviewable) {
await _tencentCosHandler.UploadFile(fileId, metadata.Suffix);
}
return Ok(); return Ok();
} }
} }
/*public record ClientFileMetadata(
string Ticket,
string Md5,
ulong Size
);*/
public record UploadFileDto( public record UploadFileDto(
string Ticket, string Ticket,
ulong RangeStart, ulong RangeStart,

View File

@ -38,6 +38,10 @@ public class KnowledgeFileHandler {
return $"{FilePath}/{fileId}.part.{rangeStart}-{rangeEnd}"; return $"{FilePath}/{fileId}.part.{rangeStart}-{rangeEnd}";
} }
public string GetFileName(string fileId) {
return $"{FilePath}/{fileId}";
}
public async Task SaveFileSlice(string fileId, IFormFile file, ulong rangeStart, ulong rangeEnd) { public async Task SaveFileSlice(string fileId, IFormFile file, ulong rangeStart, ulong rangeEnd) {
await file.CopyToAsync(File.Create(GetSliceFileName(fileId, rangeStart, rangeEnd))); await file.CopyToAsync(File.Create(GetSliceFileName(fileId, rangeStart, rangeEnd)));
} }

View File

@ -0,0 +1,35 @@
using AicsKnowledgeBase_file.Models;
using COSXML;
using COSXML.Auth;
using COSXML.Transfer;
using Microsoft.Extensions.Options;
namespace AicsKnowledgeBase_file.Handlers;
public class TencentCosHandler {
private readonly TencentCosOptions _tencentCosOptions;
private readonly CosXml _cosXml;
private readonly TransferManager _transferManager;
private readonly KnowledgeFileHandler _knowledgeFileHandler;
public TencentCosHandler(IOptions<TencentCosOptions> tencentCosOptions, KnowledgeFileHandler knowledgeFileHandler) {
_tencentCosOptions = tencentCosOptions.Value;
_knowledgeFileHandler = knowledgeFileHandler;
_cosXml = new CosXmlServer(
new CosXmlConfig.Builder()
.IsHttps(true)
.SetRegion(_tencentCosOptions.Region)
.Build(),
new DefaultQCloudCredentialProvider(_tencentCosOptions.SecretId, _tencentCosOptions.SecretKey, 600)
);
_transferManager = new TransferManager(_cosXml, new TransferConfig {
SliceSizeForUpload = 3 * 1024 * 1024
});
}
public async Task UploadFile(string fileId, string fileSuffix) {
var uploadTask = new COSXMLUploadTask(_tencentCosOptions.Bucket, $"{fileId}.{fileSuffix}");
uploadTask.SetSrcPath(_knowledgeFileHandler.GetFileName(fileId));
await _transferManager.UploadAsync(uploadTask);
}
}

View File

@ -1,7 +1,6 @@
using AicsKnowledgeBase_file.Models; using AicsKnowledgeBase_file.Models;
using AicsKnowledgeBase_file.Utilities;
using Confluent.Kafka; using Confluent.Kafka;
using Confluent.Kafka.SyncOverAsync;
using Confluent.SchemaRegistry.Serdes;
using Microsoft.Extensions.Options; using Microsoft.Extensions.Options;
using Newtonsoft.Json; using Newtonsoft.Json;
@ -32,20 +31,21 @@ public class UpdateFileMessageConsumerHandler : BackgroundService {
protected override Task ExecuteAsync(CancellationToken stoppingToken) { protected override Task ExecuteAsync(CancellationToken stoppingToken) {
return Task.Run(async () => { return Task.Run(async () => {
using var consumer = new ConsumerBuilder<string, FileTicket>(_consumerConfig) using var consumer = new ConsumerBuilder<string, string>(_consumerConfig)
.SetValueDeserializer(new JsonDeserializer<FileTicket>().AsSyncOverAsync())
.Build(); .Build();
consumer.Subscribe(TopicName); consumer.Subscribe(TopicName);
try { try {
while (true) { while (true) {
try { try {
var result = consumer.Consume(stoppingToken); var result = consumer.Consume(stoppingToken);
var msg = result.Message.Value!; var msg = JsonUtils.Deserialize<FileTicket>(result.Message.Value!)!;
await _knowledgeFileHandler.SaveFileMetadata(msg.Id, new FileMetadata { await _knowledgeFileHandler.SaveFileMetadata(msg.Id, new FileMetadata {
IsCompleted = false, IsCompleted = false,
Ticket = msg.Ticket, Ticket = msg.Ticket,
Md5 = msg.Md5, Md5 = msg.Md5,
Size = msg.Size Size = msg.Size,
ExternalPreviewable = msg.ExternalPreviewable,
Suffix = msg.Suffix
}); });
} catch (ConsumeException e) { } catch (ConsumeException e) {
_logger.LogError("Consume Exception: {Exception}", e); _logger.LogError("Consume Exception: {Exception}", e);
@ -63,5 +63,7 @@ public record FileTicket(
[JsonProperty("ticket")] string Ticket, [JsonProperty("ticket")] string Ticket,
[JsonProperty("id")] string Id, [JsonProperty("id")] string Id,
[JsonProperty("md5")] string Md5, [JsonProperty("md5")] string Md5,
[JsonProperty("size")] ulong Size [JsonProperty("size")] ulong Size,
[JsonProperty("externalPreviewable")] bool ExternalPreviewable,
[JsonProperty("suffix")] string Suffix
); );

View File

@ -5,4 +5,6 @@ public class FileMetadata {
public string Ticket { get; set; } = null!; public string Ticket { get; set; } = null!;
public string Md5 { get; set; } = null!; public string Md5 { get; set; } = null!;
public ulong Size { get; set; } = 0; public ulong Size { get; set; } = 0;
public bool ExternalPreviewable { get; set; } = false;
public string Suffix { get; set; } = null!;
} }

View File

@ -0,0 +1,9 @@
namespace AicsKnowledgeBase_file.Models;
public class TencentCosOptions {
public const string SectionName = "Tencent:Cos";
public string SecretId { get; set; } = null!;
public string SecretKey { get; set; } = null!;
public string Bucket { get; set; } = null!;
public string Region { get; set; } = null!;
}

View File

@ -3,6 +3,8 @@ using AicsKnowledgeBase_file.Handlers;
using AicsKnowledgeBase_file.Models; using AicsKnowledgeBase_file.Models;
using Microsoft.AspNetCore.Server.Kestrel.Core; using Microsoft.AspNetCore.Server.Kestrel.Core;
Directory.CreateDirectory("./files");
var builder = WebApplication.CreateBuilder(args); var builder = WebApplication.CreateBuilder(args);
builder.WebHost.UseUrls("https://*:8081"); builder.WebHost.UseUrls("https://*:8081");
builder.WebHost.UseKestrel((context, options) => { builder.WebHost.UseKestrel((context, options) => {
@ -15,11 +17,13 @@ builder.WebHost.UseKestrel((context, options) => {
// Add services to the container. // Add services to the container.
builder.Services.Configure<KafkaOptions>(builder.Configuration.GetSection(KafkaOptions.SectionName)); builder.Services.Configure<KafkaOptions>(builder.Configuration.GetSection(KafkaOptions.SectionName));
builder.Services.Configure<TencentCosOptions>(builder.Configuration.GetSection(TencentCosOptions.SectionName));
builder.Services.Configure<RouteOptions>(options => { options.LowercaseUrls = true; }); builder.Services.Configure<RouteOptions>(options => { options.LowercaseUrls = true; });
builder.Services.AddControllers().AddJsonOptions(options => { builder.Services.AddControllers().AddJsonOptions(options => {
options.JsonSerializerOptions.PropertyNamingPolicy = JsonNamingPolicy.CamelCase; options.JsonSerializerOptions.PropertyNamingPolicy = JsonNamingPolicy.CamelCase;
options.JsonSerializerOptions.PropertyNameCaseInsensitive = true; options.JsonSerializerOptions.PropertyNameCaseInsensitive = true;
}); });
builder.Services.AddSingleton<TencentCosHandler>();
builder.Services.AddSingleton<KnowledgeFileHandler>(); builder.Services.AddSingleton<KnowledgeFileHandler>();
builder.Services.AddHostedService<UpdateFileMessageConsumerHandler>(); builder.Services.AddHostedService<UpdateFileMessageConsumerHandler>();
// Learn more about configuring Swagger/OpenAPI at https://aka.ms/aspnetcore/swashbuckle // Learn more about configuring Swagger/OpenAPI at https://aka.ms/aspnetcore/swashbuckle

View File

@ -2,6 +2,13 @@
"Logging": { "Logging": {
"LogLevel": { "LogLevel": {
"Default": "Warning" "Default": "Warning"
},
"Console": {
"FormatterName": "simple",
"FormatterOptions": {
"SingleLine": true,
"TimestampFormat": "MM-dd HH:mm:ss "
}
} }
}, },
"AllowedHosts": "*", "AllowedHosts": "*",
@ -11,5 +18,13 @@
}, },
"Ssl": { "Ssl": {
"UseKeyFile": true "UseKeyFile": true
},
"Tencent": {
"Cos": {
"SecretId": "AKIDSlvvhEYfYBetvYzCBvhJrDLGwNbcR2B7",
"SecretKey": "kHZigS3UkqlY8WiuypXM3ZwCHA0iHepp",
"Bucket": "aics-1300085057",
"Region": "ap-nanjing"
}
} }
} }