feat: 修改了upload_file消息格式,添加了腾讯云COS服务相关的操作
parent
94e28b5725
commit
85f47330bd
|
@ -23,3 +23,4 @@
|
||||||
**/values.dev.yaml
|
**/values.dev.yaml
|
||||||
LICENSE
|
LICENSE
|
||||||
README.md
|
README.md
|
||||||
|
AicsKnowledgeBase_file/files/
|
|
@ -3,3 +3,4 @@ obj/
|
||||||
/packages/
|
/packages/
|
||||||
riderModule.iml
|
riderModule.iml
|
||||||
/_ReSharper.Caches/
|
/_ReSharper.Caches/
|
||||||
|
AicsKnowledgeBase_file/files/
|
|
@ -10,9 +10,10 @@
|
||||||
|
|
||||||
<ItemGroup>
|
<ItemGroup>
|
||||||
<PackageReference Include="Confluent.Kafka" Version="2.1.1" />
|
<PackageReference Include="Confluent.Kafka" Version="2.1.1" />
|
||||||
<PackageReference Include="Confluent.SchemaRegistry.Serdes.Json" Version="2.1.1" />
|
|
||||||
<PackageReference Include="Microsoft.AspNetCore.OpenApi" Version="7.0.2" />
|
<PackageReference Include="Microsoft.AspNetCore.OpenApi" Version="7.0.2" />
|
||||||
|
<PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
|
||||||
<PackageReference Include="Swashbuckle.AspNetCore" Version="6.4.0" />
|
<PackageReference Include="Swashbuckle.AspNetCore" Version="6.4.0" />
|
||||||
|
<PackageReference Include="Tencent.QCloud.Cos.Sdk" Version="5.4.34" />
|
||||||
</ItemGroup>
|
</ItemGroup>
|
||||||
|
|
||||||
<ItemGroup>
|
<ItemGroup>
|
||||||
|
|
|
@ -8,9 +8,11 @@ namespace AicsKnowledgeBase_file.Controllers;
|
||||||
[Route("[controller]/{fileId}")]
|
[Route("[controller]/{fileId}")]
|
||||||
public class FileController : ControllerBase {
|
public class FileController : ControllerBase {
|
||||||
private readonly KnowledgeFileHandler _knowledgeFileHandler;
|
private readonly KnowledgeFileHandler _knowledgeFileHandler;
|
||||||
|
private readonly TencentCosHandler _tencentCosHandler;
|
||||||
|
|
||||||
public FileController(KnowledgeFileHandler knowledgeFileHandler) {
|
public FileController(KnowledgeFileHandler knowledgeFileHandler, TencentCosHandler tencentCosHandler) {
|
||||||
_knowledgeFileHandler = knowledgeFileHandler;
|
_knowledgeFileHandler = knowledgeFileHandler;
|
||||||
|
_tencentCosHandler = tencentCosHandler;
|
||||||
}
|
}
|
||||||
|
|
||||||
[HttpGet("status")]
|
[HttpGet("status")]
|
||||||
|
@ -39,23 +41,6 @@ public class FileController : ControllerBase {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// [HttpPost("metadata")]
|
|
||||||
/*public async Task<IActionResult> PostFileMetadata(string fileId, [FromBody] ClientFileMetadata metadata) {
|
|
||||||
var currentData = await _knowledgeFileHandler.GetFileMetadata(fileId);
|
|
||||||
if (currentData.Ticket != metadata.Ticket) {
|
|
||||||
return this.ProblemFromCode(ErrorCodes.TicketMismatch);
|
|
||||||
}
|
|
||||||
|
|
||||||
if (currentData.Md5 != null || currentData.Size != null) {
|
|
||||||
return this.ProblemFromCode(ErrorCodes.FileMetadataConflict);
|
|
||||||
}
|
|
||||||
|
|
||||||
currentData.Md5 = metadata.Md5;
|
|
||||||
currentData.Size = metadata.Size;
|
|
||||||
await _knowledgeFileHandler.SaveFileMetadata(fileId, currentData);
|
|
||||||
return Ok();
|
|
||||||
}*/
|
|
||||||
|
|
||||||
[HttpPost]
|
[HttpPost]
|
||||||
public async Task<IActionResult> UploadFilePart(string fileId, [FromForm] UploadFileDto dto) {
|
public async Task<IActionResult> UploadFilePart(string fileId, [FromForm] UploadFileDto dto) {
|
||||||
var metadata = await _knowledgeFileHandler.GetFileMetadata(fileId);
|
var metadata = await _knowledgeFileHandler.GetFileMetadata(fileId);
|
||||||
|
@ -80,16 +65,13 @@ public class FileController : ControllerBase {
|
||||||
return Problem(statusCode: StatusCodes.Status400BadRequest, title: e.Message);
|
return Problem(statusCode: StatusCodes.Status400BadRequest, title: e.Message);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (metadata.ExternalPreviewable) {
|
||||||
|
await _tencentCosHandler.UploadFile(fileId, metadata.Suffix);
|
||||||
|
}
|
||||||
return Ok();
|
return Ok();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/*public record ClientFileMetadata(
|
|
||||||
string Ticket,
|
|
||||||
string Md5,
|
|
||||||
ulong Size
|
|
||||||
);*/
|
|
||||||
|
|
||||||
public record UploadFileDto(
|
public record UploadFileDto(
|
||||||
string Ticket,
|
string Ticket,
|
||||||
ulong RangeStart,
|
ulong RangeStart,
|
||||||
|
|
|
@ -38,6 +38,10 @@ public class KnowledgeFileHandler {
|
||||||
return $"{FilePath}/{fileId}.part.{rangeStart}-{rangeEnd}";
|
return $"{FilePath}/{fileId}.part.{rangeStart}-{rangeEnd}";
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public string GetFileName(string fileId) {
|
||||||
|
return $"{FilePath}/{fileId}";
|
||||||
|
}
|
||||||
|
|
||||||
public async Task SaveFileSlice(string fileId, IFormFile file, ulong rangeStart, ulong rangeEnd) {
|
public async Task SaveFileSlice(string fileId, IFormFile file, ulong rangeStart, ulong rangeEnd) {
|
||||||
await file.CopyToAsync(File.Create(GetSliceFileName(fileId, rangeStart, rangeEnd)));
|
await file.CopyToAsync(File.Create(GetSliceFileName(fileId, rangeStart, rangeEnd)));
|
||||||
}
|
}
|
||||||
|
|
|
@ -0,0 +1,35 @@
|
||||||
|
using AicsKnowledgeBase_file.Models;
|
||||||
|
using COSXML;
|
||||||
|
using COSXML.Auth;
|
||||||
|
using COSXML.Transfer;
|
||||||
|
using Microsoft.Extensions.Options;
|
||||||
|
|
||||||
|
namespace AicsKnowledgeBase_file.Handlers;
|
||||||
|
|
||||||
|
public class TencentCosHandler {
|
||||||
|
private readonly TencentCosOptions _tencentCosOptions;
|
||||||
|
private readonly CosXml _cosXml;
|
||||||
|
private readonly TransferManager _transferManager;
|
||||||
|
private readonly KnowledgeFileHandler _knowledgeFileHandler;
|
||||||
|
|
||||||
|
public TencentCosHandler(IOptions<TencentCosOptions> tencentCosOptions, KnowledgeFileHandler knowledgeFileHandler) {
|
||||||
|
_tencentCosOptions = tencentCosOptions.Value;
|
||||||
|
_knowledgeFileHandler = knowledgeFileHandler;
|
||||||
|
_cosXml = new CosXmlServer(
|
||||||
|
new CosXmlConfig.Builder()
|
||||||
|
.IsHttps(true)
|
||||||
|
.SetRegion(_tencentCosOptions.Region)
|
||||||
|
.Build(),
|
||||||
|
new DefaultQCloudCredentialProvider(_tencentCosOptions.SecretId, _tencentCosOptions.SecretKey, 600)
|
||||||
|
);
|
||||||
|
_transferManager = new TransferManager(_cosXml, new TransferConfig {
|
||||||
|
SliceSizeForUpload = 3 * 1024 * 1024
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
public async Task UploadFile(string fileId, string fileSuffix) {
|
||||||
|
var uploadTask = new COSXMLUploadTask(_tencentCosOptions.Bucket, $"{fileId}.{fileSuffix}");
|
||||||
|
uploadTask.SetSrcPath(_knowledgeFileHandler.GetFileName(fileId));
|
||||||
|
await _transferManager.UploadAsync(uploadTask);
|
||||||
|
}
|
||||||
|
}
|
|
@ -1,7 +1,6 @@
|
||||||
using AicsKnowledgeBase_file.Models;
|
using AicsKnowledgeBase_file.Models;
|
||||||
|
using AicsKnowledgeBase_file.Utilities;
|
||||||
using Confluent.Kafka;
|
using Confluent.Kafka;
|
||||||
using Confluent.Kafka.SyncOverAsync;
|
|
||||||
using Confluent.SchemaRegistry.Serdes;
|
|
||||||
using Microsoft.Extensions.Options;
|
using Microsoft.Extensions.Options;
|
||||||
using Newtonsoft.Json;
|
using Newtonsoft.Json;
|
||||||
|
|
||||||
|
@ -32,20 +31,21 @@ public class UpdateFileMessageConsumerHandler : BackgroundService {
|
||||||
|
|
||||||
protected override Task ExecuteAsync(CancellationToken stoppingToken) {
|
protected override Task ExecuteAsync(CancellationToken stoppingToken) {
|
||||||
return Task.Run(async () => {
|
return Task.Run(async () => {
|
||||||
using var consumer = new ConsumerBuilder<string, FileTicket>(_consumerConfig)
|
using var consumer = new ConsumerBuilder<string, string>(_consumerConfig)
|
||||||
.SetValueDeserializer(new JsonDeserializer<FileTicket>().AsSyncOverAsync())
|
|
||||||
.Build();
|
.Build();
|
||||||
consumer.Subscribe(TopicName);
|
consumer.Subscribe(TopicName);
|
||||||
try {
|
try {
|
||||||
while (true) {
|
while (true) {
|
||||||
try {
|
try {
|
||||||
var result = consumer.Consume(stoppingToken);
|
var result = consumer.Consume(stoppingToken);
|
||||||
var msg = result.Message.Value!;
|
var msg = JsonUtils.Deserialize<FileTicket>(result.Message.Value!)!;
|
||||||
await _knowledgeFileHandler.SaveFileMetadata(msg.Id, new FileMetadata {
|
await _knowledgeFileHandler.SaveFileMetadata(msg.Id, new FileMetadata {
|
||||||
IsCompleted = false,
|
IsCompleted = false,
|
||||||
Ticket = msg.Ticket,
|
Ticket = msg.Ticket,
|
||||||
Md5 = msg.Md5,
|
Md5 = msg.Md5,
|
||||||
Size = msg.Size
|
Size = msg.Size,
|
||||||
|
ExternalPreviewable = msg.ExternalPreviewable,
|
||||||
|
Suffix = msg.Suffix
|
||||||
});
|
});
|
||||||
} catch (ConsumeException e) {
|
} catch (ConsumeException e) {
|
||||||
_logger.LogError("Consume Exception: {Exception}", e);
|
_logger.LogError("Consume Exception: {Exception}", e);
|
||||||
|
@ -63,5 +63,7 @@ public record FileTicket(
|
||||||
[JsonProperty("ticket")] string Ticket,
|
[JsonProperty("ticket")] string Ticket,
|
||||||
[JsonProperty("id")] string Id,
|
[JsonProperty("id")] string Id,
|
||||||
[JsonProperty("md5")] string Md5,
|
[JsonProperty("md5")] string Md5,
|
||||||
[JsonProperty("size")] ulong Size
|
[JsonProperty("size")] ulong Size,
|
||||||
|
[JsonProperty("externalPreviewable")] bool ExternalPreviewable,
|
||||||
|
[JsonProperty("suffix")] string Suffix
|
||||||
);
|
);
|
|
@ -5,4 +5,6 @@ public class FileMetadata {
|
||||||
public string Ticket { get; set; } = null!;
|
public string Ticket { get; set; } = null!;
|
||||||
public string Md5 { get; set; } = null!;
|
public string Md5 { get; set; } = null!;
|
||||||
public ulong Size { get; set; } = 0;
|
public ulong Size { get; set; } = 0;
|
||||||
|
public bool ExternalPreviewable { get; set; } = false;
|
||||||
|
public string Suffix { get; set; } = null!;
|
||||||
}
|
}
|
|
@ -0,0 +1,9 @@
|
||||||
|
namespace AicsKnowledgeBase_file.Models;
|
||||||
|
|
||||||
|
public class TencentCosOptions {
|
||||||
|
public const string SectionName = "Tencent:Cos";
|
||||||
|
public string SecretId { get; set; } = null!;
|
||||||
|
public string SecretKey { get; set; } = null!;
|
||||||
|
public string Bucket { get; set; } = null!;
|
||||||
|
public string Region { get; set; } = null!;
|
||||||
|
}
|
|
@ -3,6 +3,8 @@ using AicsKnowledgeBase_file.Handlers;
|
||||||
using AicsKnowledgeBase_file.Models;
|
using AicsKnowledgeBase_file.Models;
|
||||||
using Microsoft.AspNetCore.Server.Kestrel.Core;
|
using Microsoft.AspNetCore.Server.Kestrel.Core;
|
||||||
|
|
||||||
|
Directory.CreateDirectory("./files");
|
||||||
|
|
||||||
var builder = WebApplication.CreateBuilder(args);
|
var builder = WebApplication.CreateBuilder(args);
|
||||||
builder.WebHost.UseUrls("https://*:8081");
|
builder.WebHost.UseUrls("https://*:8081");
|
||||||
builder.WebHost.UseKestrel((context, options) => {
|
builder.WebHost.UseKestrel((context, options) => {
|
||||||
|
@ -15,11 +17,13 @@ builder.WebHost.UseKestrel((context, options) => {
|
||||||
|
|
||||||
// Add services to the container.
|
// Add services to the container.
|
||||||
builder.Services.Configure<KafkaOptions>(builder.Configuration.GetSection(KafkaOptions.SectionName));
|
builder.Services.Configure<KafkaOptions>(builder.Configuration.GetSection(KafkaOptions.SectionName));
|
||||||
|
builder.Services.Configure<TencentCosOptions>(builder.Configuration.GetSection(TencentCosOptions.SectionName));
|
||||||
builder.Services.Configure<RouteOptions>(options => { options.LowercaseUrls = true; });
|
builder.Services.Configure<RouteOptions>(options => { options.LowercaseUrls = true; });
|
||||||
builder.Services.AddControllers().AddJsonOptions(options => {
|
builder.Services.AddControllers().AddJsonOptions(options => {
|
||||||
options.JsonSerializerOptions.PropertyNamingPolicy = JsonNamingPolicy.CamelCase;
|
options.JsonSerializerOptions.PropertyNamingPolicy = JsonNamingPolicy.CamelCase;
|
||||||
options.JsonSerializerOptions.PropertyNameCaseInsensitive = true;
|
options.JsonSerializerOptions.PropertyNameCaseInsensitive = true;
|
||||||
});
|
});
|
||||||
|
builder.Services.AddSingleton<TencentCosHandler>();
|
||||||
builder.Services.AddSingleton<KnowledgeFileHandler>();
|
builder.Services.AddSingleton<KnowledgeFileHandler>();
|
||||||
builder.Services.AddHostedService<UpdateFileMessageConsumerHandler>();
|
builder.Services.AddHostedService<UpdateFileMessageConsumerHandler>();
|
||||||
// Learn more about configuring Swagger/OpenAPI at https://aka.ms/aspnetcore/swashbuckle
|
// Learn more about configuring Swagger/OpenAPI at https://aka.ms/aspnetcore/swashbuckle
|
||||||
|
|
|
@ -2,6 +2,13 @@
|
||||||
"Logging": {
|
"Logging": {
|
||||||
"LogLevel": {
|
"LogLevel": {
|
||||||
"Default": "Warning"
|
"Default": "Warning"
|
||||||
|
},
|
||||||
|
"Console": {
|
||||||
|
"FormatterName": "simple",
|
||||||
|
"FormatterOptions": {
|
||||||
|
"SingleLine": true,
|
||||||
|
"TimestampFormat": "MM-dd HH:mm:ss "
|
||||||
|
}
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"AllowedHosts": "*",
|
"AllowedHosts": "*",
|
||||||
|
@ -11,5 +18,13 @@
|
||||||
},
|
},
|
||||||
"Ssl": {
|
"Ssl": {
|
||||||
"UseKeyFile": true
|
"UseKeyFile": true
|
||||||
|
},
|
||||||
|
"Tencent": {
|
||||||
|
"Cos": {
|
||||||
|
"SecretId": "AKIDSlvvhEYfYBetvYzCBvhJrDLGwNbcR2B7",
|
||||||
|
"SecretKey": "kHZigS3UkqlY8WiuypXM3ZwCHA0iHepp",
|
||||||
|
"Bucket": "aics-1300085057",
|
||||||
|
"Region": "ap-nanjing"
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in New Issue