auto
/
AicsKnowledgeBase_file
5
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

feat: 修改了upload_file消息格式，添加了腾讯云COS服务相关的操作

master
ArgonarioD 2023-07-05 22:46:02 +08:00
parent 94e28b5725
commit 85f47330bd
11 changed files with 90 additions and 34 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
.dockerignore
View File

@ -22,4 +22,5 @@
**/secrets.dev.yaml
**/values.dev.yaml
LICENSE
README.md
README.md
AicsKnowledgeBase_file/files/

3
.gitignore vendored
View File

@ -2,4 +2,5 @@ bin/
obj/
/packages/
riderModule.iml
/_ReSharper.Caches/
/_ReSharper.Caches/
AicsKnowledgeBase_file/files/

3
AicsKnowledgeBase_file/AicsKnowledgeBase_file.csproj
View File

@ -10,9 +10,10 @@
<ItemGroup>
<PackageReference Include="Confluent.Kafka" Version="2.1.1" />
<PackageReference Include="Confluent.SchemaRegistry.Serdes.Json" Version="2.1.1" />
<PackageReference Include="Microsoft.AspNetCore.OpenApi" Version="7.0.2" />
<PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
<PackageReference Include="Swashbuckle.AspNetCore" Version="6.4.0" />
<PackageReference Include="Tencent.QCloud.Cos.Sdk" Version="5.4.34" />
</ItemGroup>
<ItemGroup>

30
AicsKnowledgeBase_file/Controllers/FileController.cs
View File

@ -8,9 +8,11 @@ namespace AicsKnowledgeBase_file.Controllers;
[Route("[controller]/{fileId}")]
public class FileController : ControllerBase {
private readonly KnowledgeFileHandler _knowledgeFileHandler;
private readonly TencentCosHandler _tencentCosHandler;
public FileController(KnowledgeFileHandler knowledgeFileHandler) {
public FileController(KnowledgeFileHandler knowledgeFileHandler, TencentCosHandler tencentCosHandler) {
_knowledgeFileHandler = knowledgeFileHandler;
_tencentCosHandler = tencentCosHandler;
}
[HttpGet("status")]
@ -39,23 +41,6 @@ public class FileController : ControllerBase {
}
}
// [HttpPost("metadata")]
/*public async Task<IActionResult> PostFileMetadata(string fileId, [FromBody] ClientFileMetadata metadata) {
var currentData = await _knowledgeFileHandler.GetFileMetadata(fileId);
if (currentData.Ticket != metadata.Ticket) {
return this.ProblemFromCode(ErrorCodes.TicketMismatch);
}
if (currentData.Md5 != null || currentData.Size != null) {
return this.ProblemFromCode(ErrorCodes.FileMetadataConflict);
}
currentData.Md5 = metadata.Md5;
currentData.Size = metadata.Size;
await _knowledgeFileHandler.SaveFileMetadata(fileId, currentData);
return Ok();
}*/
[HttpPost]
public async Task<IActionResult> UploadFilePart(string fileId, [FromForm] UploadFileDto dto) {
var metadata = await _knowledgeFileHandler.GetFileMetadata(fileId);
@ -80,16 +65,13 @@ public class FileController : ControllerBase {
return Problem(statusCode: StatusCodes.Status400BadRequest, title: e.Message);
}
if (metadata.ExternalPreviewable) {
await _tencentCosHandler.UploadFile(fileId, metadata.Suffix);
}
return Ok();
}
}
/*public record ClientFileMetadata(
string Ticket,
string Md5,
ulong Size
);*/
public record UploadFileDto(
string Ticket,
ulong RangeStart,

4
AicsKnowledgeBase_file/Handlers/KnowledgeFileHandler.cs
View File

@ -38,6 +38,10 @@ public class KnowledgeFileHandler {
return $"{FilePath}/{fileId}.part.{rangeStart}-{rangeEnd}";
}
public string GetFileName(string fileId) {
return $"{FilePath}/{fileId}";
}
public async Task SaveFileSlice(string fileId, IFormFile file, ulong rangeStart, ulong rangeEnd) {
await file.CopyToAsync(File.Create(GetSliceFileName(fileId, rangeStart, rangeEnd)));
}

35
AicsKnowledgeBase_file/Handlers/TencentCosHandler.cs Normal file
View File

@ -0,0 +1,35 @@
using AicsKnowledgeBase_file.Models;
using COSXML;
using COSXML.Auth;
using COSXML.Transfer;
using Microsoft.Extensions.Options;
namespace AicsKnowledgeBase_file.Handlers;
public class TencentCosHandler {
private readonly TencentCosOptions _tencentCosOptions;
private readonly CosXml _cosXml;
private readonly TransferManager _transferManager;
private readonly KnowledgeFileHandler _knowledgeFileHandler;
public TencentCosHandler(IOptions<TencentCosOptions> tencentCosOptions, KnowledgeFileHandler knowledgeFileHandler) {
_tencentCosOptions = tencentCosOptions.Value;
_knowledgeFileHandler = knowledgeFileHandler;
_cosXml = new CosXmlServer(
new CosXmlConfig.Builder()
.IsHttps(true)
.SetRegion(_tencentCosOptions.Region)
.Build(),
new DefaultQCloudCredentialProvider(_tencentCosOptions.SecretId, _tencentCosOptions.SecretKey, 600)
);
_transferManager = new TransferManager(_cosXml, new TransferConfig {
SliceSizeForUpload = 3 * 1024 * 1024
});
}
public async Task UploadFile(string fileId, string fileSuffix) {
var uploadTask = new COSXMLUploadTask(_tencentCosOptions.Bucket, $"{fileId}.{fileSuffix}");
uploadTask.SetSrcPath(_knowledgeFileHandler.GetFileName(fileId));
await _transferManager.UploadAsync(uploadTask);
}
}

16
AicsKnowledgeBase_file/Handlers/UpdateFileMessageConsumerHandler.cs
View File

@ -1,7 +1,6 @@
using AicsKnowledgeBase_file.Models;
using AicsKnowledgeBase_file.Utilities;
using Confluent.Kafka;
using Confluent.Kafka.SyncOverAsync;
using Confluent.SchemaRegistry.Serdes;
using Microsoft.Extensions.Options;
using Newtonsoft.Json;
@ -32,20 +31,21 @@ public class UpdateFileMessageConsumerHandler : BackgroundService {
protected override Task ExecuteAsync(CancellationToken stoppingToken) {
return Task.Run(async () => {
using var consumer = new ConsumerBuilder<string, FileTicket>(_consumerConfig)
.SetValueDeserializer(new JsonDeserializer<FileTicket>().AsSyncOverAsync())
using var consumer = new ConsumerBuilder<string, string>(_consumerConfig)
.Build();
consumer.Subscribe(TopicName);
try {
while (true) {
try {
var result = consumer.Consume(stoppingToken);
var msg = result.Message.Value!;
var msg = JsonUtils.Deserialize<FileTicket>(result.Message.Value!)!;
await _knowledgeFileHandler.SaveFileMetadata(msg.Id, new FileMetadata {
IsCompleted = false,
Ticket = msg.Ticket,
Md5 = msg.Md5,
Size = msg.Size
Size = msg.Size,
ExternalPreviewable = msg.ExternalPreviewable,
Suffix = msg.Suffix
});
} catch (ConsumeException e) {
_logger.LogError("Consume Exception: {Exception}", e);
@ -63,5 +63,7 @@ public record FileTicket(
[JsonProperty("ticket")] string Ticket,
[JsonProperty("id")] string Id,
[JsonProperty("md5")] string Md5,
[JsonProperty("size")] ulong Size
[JsonProperty("size")] ulong Size,
[JsonProperty("externalPreviewable")] bool ExternalPreviewable,
[JsonProperty("suffix")] string Suffix
);

2
AicsKnowledgeBase_file/Models/FileMetadata.cs
View File

@ -5,4 +5,6 @@ public class FileMetadata {
public string Ticket { get; set; } = null!;
public string Md5 { get; set; } = null!;
public ulong Size { get; set; } = 0;
public bool ExternalPreviewable { get; set; } = false;
public string Suffix { get; set; } = null!;
}

9
AicsKnowledgeBase_file/Models/TencentCosOptions.cs Normal file
View File

@ -0,0 +1,9 @@
namespace AicsKnowledgeBase_file.Models;
public class TencentCosOptions {
public const string SectionName = "Tencent:Cos";
public string SecretId { get; set; } = null!;
public string SecretKey { get; set; } = null!;
public string Bucket { get; set; } = null!;
public string Region { get; set; } = null!;
}

4
AicsKnowledgeBase_file/Program.cs
View File

@ -3,6 +3,8 @@ using AicsKnowledgeBase_file.Handlers;
using AicsKnowledgeBase_file.Models;
using Microsoft.AspNetCore.Server.Kestrel.Core;
Directory.CreateDirectory("./files");
var builder = WebApplication.CreateBuilder(args);
builder.WebHost.UseUrls("https://*:8081");
builder.WebHost.UseKestrel((context, options) => {
@ -15,11 +17,13 @@ builder.WebHost.UseKestrel((context, options) => {
// Add services to the container.
builder.Services.Configure<KafkaOptions>(builder.Configuration.GetSection(KafkaOptions.SectionName));
builder.Services.Configure<TencentCosOptions>(builder.Configuration.GetSection(TencentCosOptions.SectionName));
builder.Services.Configure<RouteOptions>(options => { options.LowercaseUrls = true; });
builder.Services.AddControllers().AddJsonOptions(options => {
options.JsonSerializerOptions.PropertyNamingPolicy = JsonNamingPolicy.CamelCase;
options.JsonSerializerOptions.PropertyNameCaseInsensitive = true;
});
builder.Services.AddSingleton<TencentCosHandler>();
builder.Services.AddSingleton<KnowledgeFileHandler>();
builder.Services.AddHostedService<UpdateFileMessageConsumerHandler>();
// Learn more about configuring Swagger/OpenAPI at https://aka.ms/aspnetcore/swashbuckle

15
AicsKnowledgeBase_file/appsettings.json
View File

@ -2,6 +2,13 @@
"Logging": {
"LogLevel": {
"Default": "Warning"
},
"Console": {
"FormatterName": "simple",
"FormatterOptions": {
"SingleLine": true,
"TimestampFormat": "MM-dd HH:mm:ss "
}
}
},
"AllowedHosts": "*",
@ -11,5 +18,13 @@
},
"Ssl": {
"UseKeyFile": true
},
"Tencent": {
"Cos": {
"SecretId": "AKIDSlvvhEYfYBetvYzCBvhJrDLGwNbcR2B7",
"SecretKey": "kHZigS3UkqlY8WiuypXM3ZwCHA0iHepp",
"Bucket": "aics-1300085057",
"Region": "ap-nanjing"
}
}
}