diff --git a/.dockerignore b/.dockerignore
index cd967fc..2bc15c6 100644
--- a/.dockerignore
+++ b/.dockerignore
@@ -22,4 +22,5 @@
**/secrets.dev.yaml
**/values.dev.yaml
LICENSE
-README.md
\ No newline at end of file
+README.md
+AicsKnowledgeBase_file/files/
\ No newline at end of file
diff --git a/.gitignore b/.gitignore
index add57be..95b20af 100644
--- a/.gitignore
+++ b/.gitignore
@@ -2,4 +2,5 @@ bin/
obj/
/packages/
riderModule.iml
-/_ReSharper.Caches/
\ No newline at end of file
+/_ReSharper.Caches/
+AicsKnowledgeBase_file/files/
\ No newline at end of file
diff --git a/AicsKnowledgeBase_file/AicsKnowledgeBase_file.csproj b/AicsKnowledgeBase_file/AicsKnowledgeBase_file.csproj
index e655dad..fced843 100644
--- a/AicsKnowledgeBase_file/AicsKnowledgeBase_file.csproj
+++ b/AicsKnowledgeBase_file/AicsKnowledgeBase_file.csproj
@@ -10,9 +10,10 @@
-
+
+
diff --git a/AicsKnowledgeBase_file/Controllers/FileController.cs b/AicsKnowledgeBase_file/Controllers/FileController.cs
index 53ab54b..cb43b03 100644
--- a/AicsKnowledgeBase_file/Controllers/FileController.cs
+++ b/AicsKnowledgeBase_file/Controllers/FileController.cs
@@ -8,9 +8,11 @@ namespace AicsKnowledgeBase_file.Controllers;
[Route("[controller]/{fileId}")]
public class FileController : ControllerBase {
private readonly KnowledgeFileHandler _knowledgeFileHandler;
+ private readonly TencentCosHandler _tencentCosHandler;
- public FileController(KnowledgeFileHandler knowledgeFileHandler) {
+ public FileController(KnowledgeFileHandler knowledgeFileHandler, TencentCosHandler tencentCosHandler) {
_knowledgeFileHandler = knowledgeFileHandler;
+ _tencentCosHandler = tencentCosHandler;
}
[HttpGet("status")]
@@ -39,23 +41,6 @@ public class FileController : ControllerBase {
}
}
- // [HttpPost("metadata")]
- /*public async Task PostFileMetadata(string fileId, [FromBody] ClientFileMetadata metadata) {
- var currentData = await _knowledgeFileHandler.GetFileMetadata(fileId);
- if (currentData.Ticket != metadata.Ticket) {
- return this.ProblemFromCode(ErrorCodes.TicketMismatch);
- }
-
- if (currentData.Md5 != null || currentData.Size != null) {
- return this.ProblemFromCode(ErrorCodes.FileMetadataConflict);
- }
-
- currentData.Md5 = metadata.Md5;
- currentData.Size = metadata.Size;
- await _knowledgeFileHandler.SaveFileMetadata(fileId, currentData);
- return Ok();
- }*/
-
[HttpPost]
public async Task UploadFilePart(string fileId, [FromForm] UploadFileDto dto) {
var metadata = await _knowledgeFileHandler.GetFileMetadata(fileId);
@@ -80,16 +65,13 @@ public class FileController : ControllerBase {
return Problem(statusCode: StatusCodes.Status400BadRequest, title: e.Message);
}
+ if (metadata.ExternalPreviewable) {
+ await _tencentCosHandler.UploadFile(fileId, metadata.Suffix);
+ }
return Ok();
}
}
-/*public record ClientFileMetadata(
- string Ticket,
- string Md5,
- ulong Size
-);*/
-
public record UploadFileDto(
string Ticket,
ulong RangeStart,
diff --git a/AicsKnowledgeBase_file/Handlers/KnowledgeFileHandler.cs b/AicsKnowledgeBase_file/Handlers/KnowledgeFileHandler.cs
index 0437729..3c981e5 100644
--- a/AicsKnowledgeBase_file/Handlers/KnowledgeFileHandler.cs
+++ b/AicsKnowledgeBase_file/Handlers/KnowledgeFileHandler.cs
@@ -38,6 +38,10 @@ public class KnowledgeFileHandler {
return $"{FilePath}/{fileId}.part.{rangeStart}-{rangeEnd}";
}
+ public string GetFileName(string fileId) {
+ return $"{FilePath}/{fileId}";
+ }
+
public async Task SaveFileSlice(string fileId, IFormFile file, ulong rangeStart, ulong rangeEnd) {
await file.CopyToAsync(File.Create(GetSliceFileName(fileId, rangeStart, rangeEnd)));
}
diff --git a/AicsKnowledgeBase_file/Handlers/TencentCosHandler.cs b/AicsKnowledgeBase_file/Handlers/TencentCosHandler.cs
new file mode 100644
index 0000000..a36b357
--- /dev/null
+++ b/AicsKnowledgeBase_file/Handlers/TencentCosHandler.cs
@@ -0,0 +1,35 @@
+using AicsKnowledgeBase_file.Models;
+using COSXML;
+using COSXML.Auth;
+using COSXML.Transfer;
+using Microsoft.Extensions.Options;
+
+namespace AicsKnowledgeBase_file.Handlers;
+
+public class TencentCosHandler {
+ private readonly TencentCosOptions _tencentCosOptions;
+ private readonly CosXml _cosXml;
+ private readonly TransferManager _transferManager;
+ private readonly KnowledgeFileHandler _knowledgeFileHandler;
+
+ public TencentCosHandler(IOptions tencentCosOptions, KnowledgeFileHandler knowledgeFileHandler) {
+ _tencentCosOptions = tencentCosOptions.Value;
+ _knowledgeFileHandler = knowledgeFileHandler;
+ _cosXml = new CosXmlServer(
+ new CosXmlConfig.Builder()
+ .IsHttps(true)
+ .SetRegion(_tencentCosOptions.Region)
+ .Build(),
+ new DefaultQCloudCredentialProvider(_tencentCosOptions.SecretId, _tencentCosOptions.SecretKey, 600)
+ );
+ _transferManager = new TransferManager(_cosXml, new TransferConfig {
+ SliceSizeForUpload = 3 * 1024 * 1024
+ });
+ }
+
+ public async Task UploadFile(string fileId, string fileSuffix) {
+ var uploadTask = new COSXMLUploadTask(_tencentCosOptions.Bucket, $"{fileId}.{fileSuffix}");
+ uploadTask.SetSrcPath(_knowledgeFileHandler.GetFileName(fileId));
+ await _transferManager.UploadAsync(uploadTask);
+ }
+}
\ No newline at end of file
diff --git a/AicsKnowledgeBase_file/Handlers/UpdateFileMessageConsumerHandler.cs b/AicsKnowledgeBase_file/Handlers/UpdateFileMessageConsumerHandler.cs
index b38b725..1166d24 100644
--- a/AicsKnowledgeBase_file/Handlers/UpdateFileMessageConsumerHandler.cs
+++ b/AicsKnowledgeBase_file/Handlers/UpdateFileMessageConsumerHandler.cs
@@ -1,7 +1,6 @@
using AicsKnowledgeBase_file.Models;
+using AicsKnowledgeBase_file.Utilities;
using Confluent.Kafka;
-using Confluent.Kafka.SyncOverAsync;
-using Confluent.SchemaRegistry.Serdes;
using Microsoft.Extensions.Options;
using Newtonsoft.Json;
@@ -32,20 +31,21 @@ public class UpdateFileMessageConsumerHandler : BackgroundService {
protected override Task ExecuteAsync(CancellationToken stoppingToken) {
return Task.Run(async () => {
- using var consumer = new ConsumerBuilder(_consumerConfig)
- .SetValueDeserializer(new JsonDeserializer().AsSyncOverAsync())
+ using var consumer = new ConsumerBuilder(_consumerConfig)
.Build();
consumer.Subscribe(TopicName);
try {
while (true) {
try {
var result = consumer.Consume(stoppingToken);
- var msg = result.Message.Value!;
+ var msg = JsonUtils.Deserialize(result.Message.Value!)!;
await _knowledgeFileHandler.SaveFileMetadata(msg.Id, new FileMetadata {
IsCompleted = false,
Ticket = msg.Ticket,
Md5 = msg.Md5,
- Size = msg.Size
+ Size = msg.Size,
+ ExternalPreviewable = msg.ExternalPreviewable,
+ Suffix = msg.Suffix
});
} catch (ConsumeException e) {
_logger.LogError("Consume Exception: {Exception}", e);
@@ -63,5 +63,7 @@ public record FileTicket(
[JsonProperty("ticket")] string Ticket,
[JsonProperty("id")] string Id,
[JsonProperty("md5")] string Md5,
- [JsonProperty("size")] ulong Size
+ [JsonProperty("size")] ulong Size,
+ [JsonProperty("externalPreviewable")] bool ExternalPreviewable,
+ [JsonProperty("suffix")] string Suffix
);
\ No newline at end of file
diff --git a/AicsKnowledgeBase_file/Models/FileMetadata.cs b/AicsKnowledgeBase_file/Models/FileMetadata.cs
index 6c08695..984915b 100644
--- a/AicsKnowledgeBase_file/Models/FileMetadata.cs
+++ b/AicsKnowledgeBase_file/Models/FileMetadata.cs
@@ -5,4 +5,6 @@ public class FileMetadata {
public string Ticket { get; set; } = null!;
public string Md5 { get; set; } = null!;
public ulong Size { get; set; } = 0;
+ public bool ExternalPreviewable { get; set; } = false;
+ public string Suffix { get; set; } = null!;
}
\ No newline at end of file
diff --git a/AicsKnowledgeBase_file/Models/TencentCosOptions.cs b/AicsKnowledgeBase_file/Models/TencentCosOptions.cs
new file mode 100644
index 0000000..41b10ca
--- /dev/null
+++ b/AicsKnowledgeBase_file/Models/TencentCosOptions.cs
@@ -0,0 +1,9 @@
+namespace AicsKnowledgeBase_file.Models;
+
+public class TencentCosOptions {
+ public const string SectionName = "Tencent:Cos";
+ public string SecretId { get; set; } = null!;
+ public string SecretKey { get; set; } = null!;
+ public string Bucket { get; set; } = null!;
+ public string Region { get; set; } = null!;
+}
\ No newline at end of file
diff --git a/AicsKnowledgeBase_file/Program.cs b/AicsKnowledgeBase_file/Program.cs
index 5a270d3..c066dc7 100644
--- a/AicsKnowledgeBase_file/Program.cs
+++ b/AicsKnowledgeBase_file/Program.cs
@@ -3,6 +3,8 @@ using AicsKnowledgeBase_file.Handlers;
using AicsKnowledgeBase_file.Models;
using Microsoft.AspNetCore.Server.Kestrel.Core;
+Directory.CreateDirectory("./files");
+
var builder = WebApplication.CreateBuilder(args);
builder.WebHost.UseUrls("https://*:8081");
builder.WebHost.UseKestrel((context, options) => {
@@ -15,11 +17,13 @@ builder.WebHost.UseKestrel((context, options) => {
// Add services to the container.
builder.Services.Configure(builder.Configuration.GetSection(KafkaOptions.SectionName));
+builder.Services.Configure(builder.Configuration.GetSection(TencentCosOptions.SectionName));
builder.Services.Configure(options => { options.LowercaseUrls = true; });
builder.Services.AddControllers().AddJsonOptions(options => {
options.JsonSerializerOptions.PropertyNamingPolicy = JsonNamingPolicy.CamelCase;
options.JsonSerializerOptions.PropertyNameCaseInsensitive = true;
});
+builder.Services.AddSingleton();
builder.Services.AddSingleton();
builder.Services.AddHostedService();
// Learn more about configuring Swagger/OpenAPI at https://aka.ms/aspnetcore/swashbuckle
diff --git a/AicsKnowledgeBase_file/appsettings.json b/AicsKnowledgeBase_file/appsettings.json
index 06f37d8..f5a6c93 100644
--- a/AicsKnowledgeBase_file/appsettings.json
+++ b/AicsKnowledgeBase_file/appsettings.json
@@ -2,6 +2,13 @@
"Logging": {
"LogLevel": {
"Default": "Warning"
+ },
+ "Console": {
+ "FormatterName": "simple",
+ "FormatterOptions": {
+ "SingleLine": true,
+ "TimestampFormat": "MM-dd HH:mm:ss "
+ }
}
},
"AllowedHosts": "*",
@@ -11,5 +18,13 @@
},
"Ssl": {
"UseKeyFile": true
+ },
+ "Tencent": {
+ "Cos": {
+ "SecretId": "AKIDSlvvhEYfYBetvYzCBvhJrDLGwNbcR2B7",
+ "SecretKey": "kHZigS3UkqlY8WiuypXM3ZwCHA0iHepp",
+ "Bucket": "aics-1300085057",
+ "Region": "ap-nanjing"
+ }
}
}