From 17c4c219ac7283e1eafe3f5adb4456c5863bea9a Mon Sep 17 00:00:00 2001 From: zhangmeng <1334717033@qq.com> Date: Fri, 15 Sep 2023 18:26:19 +0800 Subject: [PATCH] =?UTF-8?q?2023=E5=B9=B49=E6=9C=8815=E6=97=A518:26:10?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitignore | 1 + pom.xml | 19 ++ .../controller/BigFileUploadController.java | 30 ++ .../com/zhangmeng/tools/utils/MinioUtils.java | 265 ++++++++++++++++++ 4 files changed, 315 insertions(+) create mode 100644 src/main/java/com/zhangmeng/tools/controller/BigFileUploadController.java create mode 100644 src/main/java/com/zhangmeng/tools/utils/MinioUtils.java diff --git a/.gitignore b/.gitignore index cc29507..889dbe7 100644 --- a/.gitignore +++ b/.gitignore @@ -37,3 +37,4 @@ build/ .vscode/ /logs/ +/bin/ diff --git a/pom.xml b/pom.xml index a2973f2..ebb1820 100644 --- a/pom.xml +++ b/pom.xml @@ -412,6 +412,25 @@ richtextfx 0.10.0 + + + io.minio + minio + 8.4.3 + + + + + com.j256.simplemagic + simplemagic + 1.17 + + + + com.squareup.okhttp3 + okhttp + 4.8.1 + diff --git a/src/main/java/com/zhangmeng/tools/controller/BigFileUploadController.java b/src/main/java/com/zhangmeng/tools/controller/BigFileUploadController.java new file mode 100644 index 0000000..8638e2a --- /dev/null +++ b/src/main/java/com/zhangmeng/tools/controller/BigFileUploadController.java @@ -0,0 +1,30 @@ +package com.zhangmeng.tools.controller; + + +import com.zhangmeng.tools.utils.MinioUtils; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.codec.digest.DigestUtils; +import org.apache.commons.io.FileUtils; +import org.apache.commons.io.FilenameUtils; +import org.apache.commons.io.IOUtils; + +import java.io.*; +import java.nio.ByteBuffer; +import java.nio.channels.FileChannel; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; + +import static com.zhangmeng.tools.utils.MinioUtils.test; + +/** + * 大文件上传 + * + * @author zhangmeng + * @version 1.0 + * @date 2023年9月15日15:19:58 + */ +@Slf4j +public class BigFileUploadController { + + +} diff --git a/src/main/java/com/zhangmeng/tools/utils/MinioUtils.java b/src/main/java/com/zhangmeng/tools/utils/MinioUtils.java new file mode 100644 index 0000000..01c94ec --- /dev/null +++ b/src/main/java/com/zhangmeng/tools/utils/MinioUtils.java @@ -0,0 +1,265 @@ +package com.zhangmeng.tools.utils; + +import io.minio.GetObjectArgs; +import io.minio.MinioClient; +import io.minio.PutObjectArgs; +import io.minio.UploadObjectArgs; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.codec.digest.DigestUtils; +import org.apache.commons.io.FilenameUtils; +import org.apache.commons.io.IOUtils; + +import java.io.*; +import java.nio.ByteBuffer; +import java.nio.channels.FileChannel; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; + +@Slf4j +public class MinioUtils { + + public static void getInstance(String endpoint, String accessKey, String secretKey) { + minioClient(endpoint, accessKey, secretKey); + } + + public static class FileChunkReader implements AutoCloseable { + + private final FileChannel channel; + private final ByteBuffer buffer; + private long chunkCount; + private final int CHUNK_SIZE; + private String name; + private File file; + private String extension; + private String md5; + + public FileChunkReader(Path filePath, int chunkSize) throws IOException { + this.channel = FileChannel.open(filePath, StandardOpenOption.READ); + this.buffer = ByteBuffer.allocate(chunkSize); + this.CHUNK_SIZE = chunkSize; + long totalSize = channel.size(); // total size of the file in bytes + this.chunkCount = (totalSize + CHUNK_SIZE - 1) / CHUNK_SIZE; + this.file = filePath.toFile(); + this.name = this.file.getName(); + extension();//拓展名 + md5(); + } + + public void md5() throws IOException { + this.md5 = DigestUtils.md5Hex(new FileInputStream(this.file)); + } + + public void extension() { + this.extension = FilenameUtils.getExtension(file.getName()); + } + + public byte[] readNextChunk() throws IOException { + buffer.clear(); + int bytesRead = channel.read(buffer); + + if (bytesRead == -1) { // 文件结束 + return null; + } + + buffer.flip(); // 准备从缓冲区读取数据 + + byte[] chunk = new byte[bytesRead]; + buffer.get(chunk); + return chunk; + } + + public String getExtension() { + return extension; + } + + public void setExtension(String extension) { + this.extension = extension; + } + + public String getMd5() { + return md5; + } + + public void setMd5(String md5) { + this.md5 = md5; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public FileChannel getChannel() { + return channel; + } + + public ByteBuffer getBuffer() { + return buffer; + } + + public long getChunkCount() { + return chunkCount; + } + + public void setChunkCount(long chunkCount) { + this.chunkCount = chunkCount; + } + + public int getCHUNK_SIZE() { + return CHUNK_SIZE; + } + + public File getFile() { + return file; + } + + public void setFile(File file) { + this.file = file; + } + + public void close() throws IOException { + channel.close(); + } + + } + + private static MinioClient minioClient; + + public static MinioClient getMinioClient() { + return minioClient; + } + + private static void minioClient(String endpoint, String accessKey, String secretKey) { + + minioClient = MinioClient.builder() + .endpoint(endpoint) + .credentials(accessKey, secretKey) + .build(); + } + + public static void addMediaFilesToMinIO(byte[] bytes, String bucket, String objectName, String contentType) { + //转为流 + ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(bytes); + + try { + PutObjectArgs putObjectArgs = PutObjectArgs.builder().bucket(bucket).object(objectName) + //-1表示文件分片按5M(不小于5M,不大于5T),分片数量最大10000， + .stream(byteArrayInputStream, byteArrayInputStream.available(), -1) + .contentType(contentType) + .build(); + + minioClient.putObject(putObjectArgs); + } catch (Exception e) { + e.printStackTrace(); + } + } + + //将文件上传到minIO，传入文件绝对路径 + public static void addMediaFilesToMinIO(String filePath, String bucket, String objectName) { + try { + minioClient.uploadObject( + UploadObjectArgs.builder() + .bucket(bucket) + .object(objectName) + .filename(filePath) + .build()); + } catch (Exception e) { + e.printStackTrace(); + log.error("上传文件到文件系统出错"); + } + } + + public static File downloadFileFromMinIO(File file, String bucket, String objectName) { + + InputStream fileInputStream = null; + OutputStream fileOutputStream = null; + try { + fileInputStream = minioClient.getObject( + GetObjectArgs.builder() + .bucket(bucket) + .object(objectName) + .build()); + try { + fileOutputStream = new FileOutputStream(file); + IOUtils.copy(fileInputStream, fileOutputStream); + + } catch (IOException e) { + log.error("下载文件" + objectName + "出错"); + } + } catch (Exception e) { + e.printStackTrace(); + log.error("文件不存在" + objectName); + } finally { + if (fileInputStream != null) { + try { + fileInputStream.close(); + } catch (IOException e) { + e.printStackTrace(); + } + } + if (fileOutputStream != null) { + try { + fileOutputStream.close(); + } catch (IOException e) { + e.printStackTrace(); + } + } + } + return file; + } + + //得到分块文件的目录 + public static String getChunkFileFolderPath(String fileMd5) { + return fileMd5.substring(0, 1) + "/" + fileMd5.substring(1, 2) + "/" + fileMd5 + "/" + "chunk" + "/"; + } + + public static final int chunkSize = 1024 * 1024 * 2; + + public static void test(String path){ + + try (MinioUtils.FileChunkReader reader = new MinioUtils.FileChunkReader(Path.of(path), chunkSize);) { // 1MB chunk size + byte[] chunk; + //扩展名 + String extName = reader.getExtension(); + + //合并为一个文件 + //创建临时文件作为合并文件 + File mergeFile = null; + try { + mergeFile = File.createTempFile(reader.getMd5(), "." + extName); + } catch (IOException e) { + e.printStackTrace(); + log.error("合并文件过程中创建临时文件出错"); + } + + byte[] b = new byte[1024 * 1024]; + RandomAccessFile raf_write = new RandomAccessFile(mergeFile, "rw"); + while ((chunk = reader.readNextChunk()) != null) { + // process the chunk + String md5Hex = DigestUtils.md5Hex(chunk); + System.out.println("--------------" + md5Hex); + System.out.println("--------------" + reader.getChunkCount()); + + //开始合并 + InputStream inputStream = new ByteArrayInputStream(chunk); + int len = -1; + while ((len = inputStream.read(b)) != -1) { + //向合并后的文件写 + raf_write.write(b, 0, len); + } + } + System.out.println(mergeFile); + + } catch (IOException e) { + e.printStackTrace(); + } + + } + + public static void main(String[] args) { + test("G:\\FXGL基础入门教程 Java游戏引擎教程_1简介与游戏演示\\1-1 简介与游戏演示-1080P 高清-AVC.mp4"); + } +}