日常如果是上传一些小文件，在程序实现中，我们都是直接上传，一般都没什么问题。如果针对大文件上传的业务中，就会面临着：

1、网速问题，导致文件上传超时，而导致失败。

2、效率问题，上传大文件等待时间过长，如果是需要上传多个，就会更慢。

3、体验问题，用户无法预知上传还需花费的时间，系统没有及时反馈，用户无法判断文件是否还在上传，还是断开。

这时候就需要采用分布式文件上传系统。

项目简介

这是一个基于.Net Core构建的简单、跨平台分布式文件上传系统，支持分块上传、多个项目同时上传、接口权限控制采用JWT机制。

技术架构

1、跨平台：这是基于.Net Core开发的系统，可以部署在Docker, Windows, Linux, Mac。

2、.Net 2.1 + Jwt + simple-uploader

项目结构

项目分为分块上传与一般上传Demo，Web、控制台上传Demo。ufs项目是分布式文件上传的统一接口，ufs会根据配置把上传的文件发到ufs.node节点，ufs.node会把上传成功路径返回给ufs并存储，用户访问的时候，ufs会访问对应节点返回资源。

UploadServer为一般文件上传接口，UploadServer.FrontEndDemo为Web上传文件Demo。

**使用
**

1、配置

配置允许上传域名、服务接口地址、允许的文件格式、文件大小、存储路径等信息。

{  "AllowedHosts": "*",  "urls": "http://localhost:6001",  "uploadServer": {    "rootUrl": "http://localhost:6001",    "entryPoint1": "/upload",    "entryPoint2": "/chunkUpload",    "virtualPath": "",    "physicalPath": "/Users/loogn/Desktop/uploader",    "appendMimes": ".htm3:text/html;",    "responseCache": 604800,    "jwtSecret": "1234561234",    "limitSize": "20mb",    "allowExts": ".txt;.jpg;.jpeg;.png;.doc;.docx;.xls;.xlsx;.ppt;.pptx;.pdf",    "apps": {      "default": {        "allowOrigins": "",        "enableThumbnail": true,        "limitExts": ".exe;",        "thumbnailExts": ".jpg;.jpeg;.png;"      },      "app1": {        "allowOrigins": "*"      }    }  }}

2、前端

一般上传代码

$("#file1").change(function () {$.ajaxFileUpload({fileElementId: 'file1',url: 'http://localhost:6001/upload',dataType: 'text',//success: function (data) {console.log("上传成功：", data);},data: {"jwt": jwt}});});

分块上传

var uploader = new Uploader({target: 'http://localhost:6001/chunkupload',headers: {jwt: jwt}});uploader.assignBrowse(document.getElementById('browseButton'));//uploader.assignBrowse(document.getElementById('folderButton'), true);//// 文件添加 单个文件uploader.on('fileAdded', function (file, event) {console.log("fileAdded:", file, event)});// 单个文件上传成功uploader.on('fileSuccess', function (rootFile, file, message) {console.log("fileSuccess：", rootFile, file, message)});// 根下的单个文件（文件夹）上传完成uploader.on('fileComplete', function (rootFile) {console.log("fileComplete:", rootFile)});// 某个文件上传失败了uploader.on('fileError', function (rootFile, file, message) {console.log("fileError:", rootFile, file, message)});

3、后端

一般上传

 public async Task InvokeAsync(HttpContext context, RequestDelegate next){context.Response.Headers.Add("Access-Control-Allow-Origin", "*");context.Response.Headers.Add("Access-Control-Allow-Headers", "content-type,jwt,origin");if (context.Request.Method.Equals(HttpMethods.Options, StringComparison.OrdinalIgnoreCase)){context.Response.StatusCode = (int) HttpStatusCode.OK;}else if (context.Request.Method.Equals(HttpMethods.Post, StringComparison.OrdinalIgnoreCase)){//验证jwtstring token = null;if (context.Request.Headers.TryGetValue("jwt", out StringValues jwt)){token = jwt.ToString();}else if (context.Request.Form.TryGetValue("jwt", out jwt)){token = jwt.ToString();}else{await context.Response.WriteAsync(new UploadResult(){msg = "No JWT in the header and form"}.toJson());return;}try{var payload = new JwtBuilder().WithSecret(_config.JWTSecret).MustVerifySignature().Decode<JwtPayload>(token);var msg = payload.validate();if (msg != null){await context.Response.WriteAsync(new UploadResult(){msg = msg}.toJson());return;}//特定的配置var appConfig = _config.GetAppConfig(payload.app);//跨域context.Request.Headers.TryGetValue("Origin", out var origins);var origin = origins.ToString();if (!string.IsNullOrEmpty(origin) && appConfig.IsAllowOrigin(origin)){context.Response.Headers.Add("Access-Control-Allow-Origin", origin);}//获取上传的文件var file = context.Request.Form.Files.FirstOrDefault();if (file == null || file.Length == 0){await context.Response.WriteAsync(new UploadResult(){msg = "There is no file data"}.toJson());return;}//大小验证if (file.Length > (payload.GetByteSize() ?? _config.GetByteSize())){await context.Response.WriteAsync(new UploadResult(){msg = "The file is too big"}.toJson());return;}//后缀验证var ext = Path.GetExtension(file.FileName);if (!(payload.exts + _config.AllowExts).Contains(ext, StringComparison.OrdinalIgnoreCase)|| appConfig.LimitExts.Contains(ext, StringComparison.OrdinalIgnoreCase)){await context.Response.WriteAsync(new UploadResult(){msg = "File extension is not allowed"}.toJson());return;}//上传逻辑var now = DateTime.Now;var yy = now.ToString("yyyy");var mm = now.ToString("MM");var dd = now.ToString("dd");var fileName = Guid.NewGuid().ToString("n") + ext;var folder = Path.Combine(_config.PhysicalPath, payload.app, yy, mm, dd);if (!Directory.Exists(folder)){Directory.CreateDirectory(folder);}var filePath = Path.Combine(folder, fileName);using (var fileStream = new FileStream(filePath, FileMode.Create, FileAccess.Write)){file.CopyTo(fileStream);fileStream.Flush(true);}var fileUrl = _config.RootUrl + "/" + payload.app + "/" + yy + "/" + mm +"/" +dd +"/" + fileName;await context.Response.WriteAsync(new UploadResult(){ok = true,url = fileUrl}.toJson());}catch (TokenExpiredException){await context.Response.WriteAsync(new UploadResult(){msg = "Token has expired"}.toJson());}catch (SignatureVerificationException){await context.Response.WriteAsync(new UploadResult(){msg = "Token has invalid signature"}.toJson());}}else{await context.Response.WriteAsync(new UploadResult(){msg = $"Request method '{context.Request.Method}' is not supported"}.toJson());}}

分块上传

public async Task InvokeAsync(HttpContext context, RequestDelegate next){context.Response.Headers.Add("Access-Control-Allow-Origin", "*");context.Response.Headers.Add("Access-Control-Allow-Headers", "content-type,jwt,origin");if (context.Request.Method.Equals(HttpMethods.Options, StringComparison.OrdinalIgnoreCase)){context.Response.StatusCode = (int)HttpStatusCode.OK;}else if (context.Request.Method.Equals(HttpMethods.Get, StringComparison.OrdinalIgnoreCase)){//简单实现context.Request.Query.TryGetValue("chunkNumber", out var chunkNumbers);int.TryParse(chunkNumbers.ToString(), out var chunkNumber);context.Request.Query.TryGetValue("identifier", out var identifiers);if (chunkNumber == 0 || string.IsNullOrEmpty(identifiers)){context.Response.StatusCode = 204;}else{var chunkFilename = getChunkFilename(_config.PhysicalPath, chunkNumber, identifiers);if (File.Exists(chunkFilename)){await context.Response.WriteAsync("found");}else{context.Response.StatusCode = 204;}}}else if (context.Request.Method.Equals(HttpMethods.Post, StringComparison.OrdinalIgnoreCase)){//验证jwtstring token = null;if (context.Request.Headers.TryGetValue("jwt", out StringValues jwt)){token = jwt.ToString();}else if (context.Request.Form.TryGetValue("jwt", out jwt)){token = jwt.ToString();}else{await context.Response.WriteAsync(new UploadResult(){msg = "No JWT in the header and form"}.toJson());return;}try{var payload = new JwtBuilder().WithSecret(_config.JWTSecret).MustVerifySignature().Decode<JwtPayload>(token);var msg = payload.validate();if (msg != null){await context.Response.WriteAsync(new UploadResult(){msg = msg}.toJson());return;}//特定的配置var appConfig = _config.GetAppConfig(payload.app);//跨域context.Request.Headers.TryGetValue("Origin", out var origins);var origin = origins.ToString();if (!string.IsNullOrEmpty(origin) && appConfig.IsAllowOrigin(origin)){context.Response.Headers.Add("Access-Control-Allow-Origin", origin);}//获取上传的文件分片var file = context.Request.Form.Files.FirstOrDefault();if (file == null || file.Length == 0){await context.Response.WriteAsync(new UploadResult(){msg = "There is no file data"}.toJson());return;}//后缀验证var ext = Path.GetExtension(file.FileName);if (!(payload.exts + _config.AllowExts).Contains(ext, StringComparison.OrdinalIgnoreCase)|| appConfig.LimitExts.Contains(ext, StringComparison.OrdinalIgnoreCase)){await context.Response.WriteAsync(new UploadResult(){msg = "File extension is not allowed"}.toJson());return;}//获取参数                    getParams(context, out var chunkNumber, out var chunkSize, out var totalSize, out string identifier,out string filename, out int totalChunks);//验证参数var validMsg = validateRequest(chunkNumber, chunkSize, totalSize, identifier, filename, file.Length, totalChunks, payload.GetByteSize() ?? _config.GetByteSize());if (validMsg != null){await context.Response.WriteAsync(new UploadResult(){msg = validMsg}.toJson());return;}else{var chunkFilename = getChunkFilename(_config.PhysicalPath, chunkNumber, identifier);try{using (var fileStream = File.OpenWrite(chunkFilename)){var stream = file.OpenReadStream();stream.CopyTo(fileStream);fileStream.Flush(true);countDict.AddOrUpdate(identifier, 1, (key, oldValue) => oldValue + 1);}if (chunkNumber == totalChunks){//验证块的完整性while (true){if (countDict.GetValueOrDefault(identifier) < totalChunks){await Task.Delay(TimeSpan.FromMilliseconds(500));}else{countDict.Remove(identifier, out _);break;}}//merge file;string[] chunkFiles = Directory.GetFiles(Path.Combine(_config.PhysicalPath, temporaryFolder),"uploader-" + identifier + ".*",SearchOption.TopDirectoryOnly);var fileUrl = await MergeChunkFiles(payload, ext, chunkFiles);await context.Response.WriteAsync(new UploadResult(){ok = true,url = fileUrl}.toJson());}else{await context.Response.WriteAsync("partly_done");return;}}catch (Exception exp){await context.Response.WriteAsync(new UploadResult(){msg = exp.Message}.toJson());return;}}}catch (TokenExpiredException){await context.Response.WriteAsync(new UploadResult(){msg = "Token has expired"}.toJson());}catch (SignatureVerificationException){await context.Response.WriteAsync(new UploadResult(){msg = "Token has invalid signature"}.toJson());}}else{context.Response.StatusCode = (int)HttpStatusCode.MethodNotAllowed;await context.Response.WriteAsync($"Request method '{context.Request.Method}' is not supported");}}

4、上传结果

上传成功

{"ok":true,"msg":null,"url":"http://localhost:6001/test/2019/06/17/abcd.jpg"}

上传失败

{"ok":false,"msg":"The file is too big","url":null}

Gitee：https://gitee.com/loogn/UploadServer

- End -

推荐阅读

• 推荐一款基于 .NET Core开源的小程序商城系统

• 推荐基于.Net6+Furion +iView开发的一套极简的进销存管理系统

• Sampler：可视化数据库监控警报工具

• 一个基于.Net Core+Vue+Element Ui开发的OA系统

• **推荐一个.Net常用代码集合，助你高效完成业务
  **

专注分享编程知识、热门有用有趣的开源项目