diff --git a/.gitignore b/.gitignore index 9154f4c..01aced2 100644 --- a/.gitignore +++ b/.gitignore @@ -24,3 +24,7 @@ hs_err_pid* replay_pid* +/huawei-obs-springboot-starter.iml +/huawei-obs-springboot-starter.ipr +/huawei-obs-springboot-starter.iws +/target/ diff --git a/.idea/.gitignore b/.idea/.gitignore new file mode 100644 index 0000000..35410ca --- /dev/null +++ b/.idea/.gitignore @@ -0,0 +1,8 @@ +# 默认忽略的文件 +/shelf/ +/workspace.xml +# 基于编辑器的 HTTP 客户端请求 +/httpRequests/ +# Datasource local storage ignored files +/dataSources/ +/dataSources.local.xml diff --git a/pom.xml b/pom.xml new file mode 100644 index 0000000..8568f26 --- /dev/null +++ b/pom.xml @@ -0,0 +1,74 @@ + + + 4.0.0 + + com.njcn + minioss-springboot-starter + 1.0.0 + + + + nexus-releases + Nexus Release Repository + http://192.168.1.13:8001/nexus/content/repositories/releases/ + + + nexus-snapshots + Nexus Snapshot Repository + http://192.168.1.13:8001/nexus/content/repositories/snapshots/ + + + + + 8 + 8 + UTF-8 + 0.5.3 + 4.8.1 + 8.2.1 + + 灿能minioss组件提取的starter模块 + jar + + + + + com.njcn + common-core + 1.0.0 + + + + me.tongfei + progressbar + ${progressbar.version} + + + com.squareup.okhttp3 + okhttp + ${okhttp.version} + + + io.minio + minio + ${minio.version} + + + com.squareup.okhttp3 + okhttp + + + + + + org.springframework.boot + spring-boot-configuration-processor + true + 2.3.12.RELEASE + + + + + \ No newline at end of file diff --git a/src/main/java/com/njcn/minioss/bo/MinIoUploadResDTO.java b/src/main/java/com/njcn/minioss/bo/MinIoUploadResDTO.java new file mode 100644 index 0000000..ceda954 --- /dev/null +++ b/src/main/java/com/njcn/minioss/bo/MinIoUploadResDTO.java @@ -0,0 +1,24 @@ +package com.njcn.minioss.bo; + +import lombok.Data; + +import java.io.Serializable; + +/** + * @author hongawen + * @version 1.0.0 + * @date 2022年10月16日 18:40 + */ +@Data +public class MinIoUploadResDTO implements Serializable { + + private static final long serialVersionUID = 475040120689218785L; + private String minFileName; + private String minFileUrl; + + public MinIoUploadResDTO(String minFileName, String minFileUrl) { + this.minFileName = minFileName; + this.minFileUrl = minFileUrl; + } + +} diff --git a/src/main/java/com/njcn/minioss/config/MinIoProperties.java b/src/main/java/com/njcn/minioss/config/MinIoProperties.java new file mode 100644 index 0000000..d189102 --- /dev/null +++ b/src/main/java/com/njcn/minioss/config/MinIoProperties.java @@ -0,0 +1,43 @@ +package com.njcn.minioss.config; + +import io.minio.MinioClient; +import lombok.Data; +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.context.annotation.Bean; +import org.springframework.stereotype.Component; + +/** + * @author hongawen + * @version 1.0.0 + * @date 2022年10月16日 18:37 + */ +@Data +@ConfigurationProperties(prefix = "min.io") +public class MinIoProperties { + + /** + * Minio 服务端ip + */ + private String endpoint; + + /** + * Minio 访问通行key + */ + private String accessKey; + + /** + * Minio 访问秘钥key + */ + private String secretKey; + + /** + * Minio 桶名称 + */ + private String bucket; + + @Bean + public MinioClient getMinioClient() { + return MinioClient.builder() + .endpoint(endpoint).credentials(accessKey, secretKey).build(); + } +} diff --git a/src/main/java/com/njcn/minioss/util/MinIoUtils.java b/src/main/java/com/njcn/minioss/util/MinIoUtils.java new file mode 100644 index 0000000..51631a1 --- /dev/null +++ b/src/main/java/com/njcn/minioss/util/MinIoUtils.java @@ -0,0 +1,606 @@ +package com.njcn.minioss.util; + +import com.njcn.minioss.bo.MinIoUploadResDTO; +import com.njcn.minioss.config.MinIoProperties; +import io.minio.*; +import io.minio.http.Method; +import io.minio.messages.Bucket; +import io.minio.messages.DeleteError; +import io.minio.messages.DeleteObject; +import io.minio.messages.Item; +import lombok.SneakyThrows; +import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.context.annotation.Configuration; +import org.springframework.web.multipart.MultipartFile; + +import javax.annotation.Resource; +import java.io.ByteArrayInputStream; +import java.io.InputStream; +import java.util.*; + + +@Configuration +@EnableConfigurationProperties({MinIoProperties.class}) +public class MinIoUtils { + + @Resource + private MinioClient instance; + + private static final String SEPARATOR_DOT = "."; + + private static final String SEPARATOR_ACROSS = "-"; + + private static final String SEPARATOR_STR = ""; + + // 存储桶名称 + private static final String chunkBucKet = "minio_bucket"; + + /** + * 不排序 + */ + public final static boolean NOT_SORT = false; + + /** + * 排序 + */ + public final static boolean SORT = true; + + /** + * 默认过期时间(分钟) + */ + private final static Integer DEFAULT_EXPIRY = 60; + + /** + * 删除分片 + */ + public final static boolean DELETE_CHUNK_OBJECT = true; + /** + * 不删除分片 + */ + public final static boolean NOT_DELETE_CHUNK_OBJECT = false; + + /** + * 判断桶是否存在 + * + * @param bucketName 桶名 + * @return boolean + */ + public boolean bucketExists(String bucketName) { + try { + return instance.bucketExists(BucketExistsArgs.builder().bucket(bucketName).build()); + } catch (Exception e) { + e.printStackTrace(); + } + return false; + } + + + /** + * 创建存储桶 + * 创建 bucket + * + * @param bucketName 桶名 + */ + public void makeBucket(String bucketName) { + try { + boolean isExist = bucketExists(bucketName); + if (!isExist) { + instance.makeBucket(MakeBucketArgs.builder().bucket(bucketName).build()); + } + } catch (Exception e) { + e.printStackTrace(); + } + } + + /** + * @return java.util.List + * @Description 获取文件存储服务的所有存储桶名称 + */ + public List listBucketNames() { + List bucketList = listBuckets(); + List bucketListName = new ArrayList<>(); + for (Bucket bucket : bucketList) { + bucketListName.add(bucket.name()); + } + return bucketListName; + } + + /** + * @return java.util.List + * @Description 列出所有存储桶 + */ + @SneakyThrows + private List listBuckets() { + return instance.listBuckets(); + } + + + /** + * 获取对象文件名称列表 + * + * @param bucketName 存储桶名称 + * @param prefix 对象名称前缀(文件夹 /xx/xx/xxx.jpg 中的 /xx/xx/) + * @return objectNames + */ + public List listObjectNames(String bucketName, String prefix) { + return listObjectNames(bucketName, prefix, NOT_SORT); + } + + + /** + * 获取对象文件名称列表 + * + * @param bucketName 存储桶名称 + * @param prefix 对象名称前缀(文件夹 /xx/xx/xxx.jpg 中的 /xx/xx/) + * @param sort 是否排序(升序) + * @return objectNames + */ + @SneakyThrows + public List listObjectNames(String bucketName, String prefix, Boolean sort) { + boolean flag = bucketExists(bucketName); + if (flag) { + ListObjectsArgs listObjectsArgs; + if (null == prefix) { + listObjectsArgs = ListObjectsArgs.builder() + .bucket(bucketName) + .recursive(true) + .build(); + } else { + listObjectsArgs = ListObjectsArgs.builder() + .bucket(bucketName) + .prefix(prefix) + .recursive(true) + .build(); + } + Iterable> chunks = instance.listObjects(listObjectsArgs); + List chunkPaths = new ArrayList<>(); + for (Result item : chunks) { + chunkPaths.add(item.get().objectName()); + } + if (sort) { + chunkPaths.sort(new Str2IntComparator(false)); + } + return chunkPaths; + } + return new ArrayList<>(); + } + + /** + * 在桶下创建文件夹,文件夹层级结构根据参数决定 + * + * @param bucket 桶名称 + * @param WotDir 格式为 xxx/xxx/xxx/ + */ + @SneakyThrows + public String createDirectory(String bucket, String WotDir) { + if (!this.bucketExists(bucket)) { + return null; + } + instance.putObject(PutObjectArgs.builder().bucket(bucket).object(WotDir).stream( + new ByteArrayInputStream(new byte[]{}), 0, -1) + .build()); + return WotDir; + } + + + /** + * 删除一个文件 + * + * @param bucketName 桶名称 + * @param objectName /xx/xx/xxx.jpg + */ + @SneakyThrows + public boolean removeObject(String bucketName, String objectName) { + + if (!bucketExists(bucketName)) { + return false; + } + instance.removeObject( + RemoveObjectArgs.builder() + .bucket(bucketName) + .object(objectName) + .build()); + return true; + } + + /** + * @param bucketName 桶名称 + * @param objectNames /xx/xx/xxx.jpg + * @return java.util.List + * @Description 删除指定桶的多个文件对象, 返回删除错误的对象列表，全部删除成功，返回空列表 + */ + @SneakyThrows + public List removeObjects(String bucketName, List objectNames) { + if (!bucketExists(bucketName)) { + return new ArrayList<>(); + } + List deleteObjects = new ArrayList<>(objectNames.size()); + for (String objectName : objectNames) { + deleteObjects.add(new DeleteObject(objectName)); + } + List deleteErrorNames = new ArrayList<>(); + Iterable> results = instance.removeObjects( + RemoveObjectsArgs.builder() + .bucket(bucketName) + .objects(deleteObjects) + .build()); + for (Result result : results) { + DeleteError error = result.get(); + deleteErrorNames.add(error.objectName()); + } + return deleteErrorNames; + } + + + /** + * 获取访问对象的外链地址 + * 获取文件的下载url + * + * @param bucketName 存储桶名称 + * @param objectName 对象名称 + * @param expiry 过期时间(分钟) 最大为7天 超过7天则默认最大值 + * @return viewUrl + */ + @SneakyThrows + public String getObjectUrl(String bucketName, String objectName, Integer expiry) { + expiry = expiryHandle(expiry); + return instance.getPresignedObjectUrl( + GetPresignedObjectUrlArgs.builder() + .method(Method.GET) + .bucket(bucketName) + .object(objectName) + .expiry(expiry) + .build() + ); + } + + + /** + * 创建上传文件对象的外链 + * + * @param bucketName 存储桶名称 + * @param objectName 欲上传文件对象的名称 + * @return uploadUrl + */ + public String createUploadUrl(String bucketName, String objectName) { + return createUploadUrl(bucketName, objectName, DEFAULT_EXPIRY); + } + + /** + * 创建上传文件对象的外链 + * + * @param bucketName 存储桶名称 + * @param objectName 欲上传文件对象的名称 + * @param expiry 过期时间(分钟) 最大为7天 超过7天则默认最大值 + * @return uploadUrl + */ + @SneakyThrows + public String createUploadUrl(String bucketName, String objectName, Integer expiry) { + expiry = expiryHandle(expiry); + return instance.getPresignedObjectUrl( + GetPresignedObjectUrlArgs.builder() + .method(Method.PUT) + .bucket(bucketName) + .object(objectName) + .expiry(expiry) + .build() + ); + } + + +// /** +// * 批量下载 +// * +// * @param directory +// * @return +// */ +// @SneakyThrows +// public List downLoadMore(String bucket, String directory) { +// Iterable> objs = instance.listObjects(ListObjectsArgs.builder().bucket(bucket).prefix(directory).useUrlEncodingType(false).build()); +// List list = new ArrayList<>(); +// for (io.minio.Result result : objs) { +// String objectName = null; +// objectName = result.get().objectName(); +// ObjectStat statObject = instance.statObject(StatObjectArgs.builder().bucket(bucket).object(objectName).build()); +// if (statObject != null && statObject.length() > 0) { +// String fileurl = instance.getPresignedObjectUrl(GetPresignedObjectUrlArgs.builder().bucket(bucket).object(statObject.name()).method(Method.GET).build()); +// list.add(fileurl); +// } +// } +// return list; +// } +// + + /** + * @param multipartFile 文件 + * @param bucketName 桶名 + * @param directory image/ + * @return java.lang.String + * @Description 文件上传 + */ + public MinIoUploadResDTO upload(MultipartFile multipartFile, String bucketName, String directory) throws Exception { + if (!this.bucketExists(bucketName)) { + this.makeBucket(bucketName); + } + InputStream inputStream = multipartFile.getInputStream(); + directory = Optional.ofNullable(directory).orElse(""); + String minFileName = directory + minFileName(multipartFile.getOriginalFilename()); + //上传文件到指定目录 + instance.putObject(PutObjectArgs.builder() + .bucket(bucketName) + .object(minFileName) + .contentType(multipartFile.getContentType()) + .stream(inputStream, inputStream.available(), -1) + .build()); + inputStream.close(); + // 返回生成文件名、访问路径 + return new MinIoUploadResDTO(minFileName, getObjectUrl(bucketName, minFileName, DEFAULT_EXPIRY)); + } + + /** + * 文件流上传 + * @param inputStream 文件流 + * @param bucketName 桶名 + * @param directory image/ + * @return java.lang.String + */ + public MinIoUploadResDTO uploadStream(InputStream inputStream, String bucketName, String directory, String fileName) throws Exception { + if (!this.bucketExists(bucketName)) { + this.makeBucket(bucketName); + } + directory = Optional.ofNullable(directory).orElse(""); + String minFileName = directory + minFileName(fileName); + //上传文件到指定目录 + instance.putObject(PutObjectArgs.builder() + .bucket(bucketName) + .object(minFileName) + .stream(inputStream, inputStream.available(), -1) + .build()); + inputStream.close(); + // 返回生成文件名、访问路径 + return new MinIoUploadResDTO(minFileName, getObjectUrl(bucketName, minFileName, DEFAULT_EXPIRY)); + } + + + /** + * @param response + * @return java.lang.String + * @Description 下载文件 + */ +// public void download(HttpServletResponse response, String bucketName, String minFileName) throws Exception { +// InputStream fileInputStream = instance.getObject(GetObjectArgs.builder() +// .bucket(bucketName) +// .object(minFileName).build()); +// response.setHeader("Content-Disposition", "attachment;filename=" + minFileName); +// response.setContentType("application/force-download"); +// response.setCharacterEncoding("UTF-8"); +// IOUtils.copy(fileInputStream, response.getOutputStream()); +// } + + + /** + * 批量创建分片上传外链 + * + * @param bucketName 存储桶名称 + * @param objectMD5 欲上传分片文件主文件的MD5 + * @param chunkCount 分片数量 + * @return uploadChunkUrls + */ + public List createUploadChunkUrlList(String bucketName, String objectMD5, Integer chunkCount) { + if (null == bucketName) { + bucketName = chunkBucKet; + } + if (null == objectMD5) { + return null; + } + objectMD5 += "/"; + if (null == chunkCount || 0 == chunkCount) { + return null; + } + List urlList = new ArrayList<>(chunkCount); + for (int i = 1; i <= chunkCount; i++) { + String objectName = objectMD5 + i + ".chunk"; + urlList.add(createUploadUrl(bucketName, objectName, DEFAULT_EXPIRY)); + } + return urlList; + } + + /** + * 创建指定序号的分片文件上传外链 + * + * @param bucketName 存储桶名称 + * @param objectMD5 欲上传分片文件主文件的MD5 + * @param partNumber 分片序号 + * @return uploadChunkUrl + */ + public String createUploadChunkUrl(String bucketName, String objectMD5, Integer partNumber) { + if (null == bucketName) { + bucketName = chunkBucKet; + } + if (null == objectMD5) { + return null; + } + objectMD5 += "/" + partNumber + ".chunk"; + return createUploadUrl(bucketName, objectMD5, DEFAULT_EXPIRY); + } + + + /** + * 获取分片文件名称列表 + * + * @param bucketName 存储桶名称 + * @param ObjectMd5 对象Md5 + * @return objectChunkNames + */ + public List listChunkObjectNames(String bucketName, String ObjectMd5) { + if (null == bucketName) { + bucketName = chunkBucKet; + } + if (null == ObjectMd5) { + return null; + } + return listObjectNames(bucketName, ObjectMd5, SORT); + } + + /** + * 获取分片名称地址HashMap key=分片序号 value=分片文件地址 + * + * @param bucketName 存储桶名称 + * @param ObjectMd5 对象Md5 + * @return objectChunkNameMap + */ + public Map mapChunkObjectNames(String bucketName, String ObjectMd5) { + if (null == bucketName) { + bucketName = chunkBucKet; + } + if (null == ObjectMd5) { + return null; + } + List chunkPaths = listObjectNames(bucketName, ObjectMd5); + if (null == chunkPaths || chunkPaths.size() == 0) { + return null; + } + Map chunkMap = new HashMap<>(chunkPaths.size()); + for (String chunkName : chunkPaths) { + Integer partNumber = Integer.parseInt(chunkName.substring(chunkName.indexOf("/") + 1, chunkName.lastIndexOf("."))); + chunkMap.put(partNumber, chunkName); + } + return chunkMap; + } + + + /** + * 合并分片文件成对象文件 + * + * @param chunkBucKetName 分片文件所在存储桶名称 + * @param composeBucketName 合并后的对象文件存储的存储桶名称 + * @param chunkNames 分片文件名称集合 + * @param objectName 合并后的对象文件名称 + * @return true/false + */ + @SneakyThrows + public boolean composeObject(String chunkBucKetName, String composeBucketName, List chunkNames, String objectName, boolean isDeleteChunkObject) { + if (null == chunkBucKetName) { + chunkBucKetName = chunkBucKet; + } + List sourceObjectList = new ArrayList<>(chunkNames.size()); + for (String chunk : chunkNames) { + sourceObjectList.add( + ComposeSource.builder() + .bucket(chunkBucKetName) + .object(chunk) + .build() + ); + } + instance.composeObject( + ComposeObjectArgs.builder() + .bucket(composeBucketName) + .object(objectName) + .sources(sourceObjectList) + .build() + ); + if (isDeleteChunkObject) { + removeObjects(chunkBucKetName, chunkNames); + } + return true; + } + + /** + * 合并分片文件成对象文件 + * + * @param bucketName 存储桶名称 + * @param chunkNames 分片文件名称集合 + * @param objectName 合并后的对象文件名称 + * @return true/false + */ + public boolean composeObject(String bucketName, List chunkNames, String objectName) { + return composeObject(chunkBucKet, bucketName, chunkNames, objectName, NOT_DELETE_CHUNK_OBJECT); + } + + /** + * 合并分片文件成对象文件 + * + * @param bucketName 存储桶名称 + * @param chunkNames 分片文件名称集合 + * @param objectName 合并后的对象文件名称 + * @return true/false + */ + public boolean composeObject(String bucketName, List chunkNames, String objectName, boolean isDeleteChunkObject) { + return composeObject(chunkBucKet, bucketName, chunkNames, objectName, isDeleteChunkObject); + } + + /** + * 合并分片文件，合并成功后删除分片文件 + * + * @param bucketName 存储桶名称 + * @param chunkNames 分片文件名称集合 + * @param objectName 合并后的对象文件名称 + * @return true/false + */ + public boolean composeObjectAndRemoveChunk(String bucketName, List chunkNames, String objectName) { + return composeObject(chunkBucKet, bucketName, chunkNames, objectName, DELETE_CHUNK_OBJECT); + } + + + /** + * @param originalFileName 原始名称 + * @return java.lang.String + * @Description 生成上传文件名 + */ + public String minFileName(String originalFileName) { + String suffix = originalFileName; + if (originalFileName.contains(SEPARATOR_DOT)) { + suffix = originalFileName.substring(originalFileName.lastIndexOf(SEPARATOR_DOT)); + } + return UUID.randomUUID().toString().replace(SEPARATOR_ACROSS, SEPARATOR_STR).toUpperCase() + suffix; + } + + + /** + * 将分钟数转换为秒数 + * + * @param expiry 过期时间(分钟数) + * @return expiry + */ + private static int expiryHandle(Integer expiry) { + expiry = expiry * 60; + if (expiry > 604800) { + return 604800; + } + return expiry; + } + + static class Str2IntComparator implements Comparator { + private final boolean reverseOrder; // 是否倒序 + + public Str2IntComparator(boolean reverseOrder) { + this.reverseOrder = reverseOrder; + } + + @Override + public int compare(String arg0, String arg1) { + Integer intArg0 = Integer.parseInt(arg0.substring(arg0.indexOf("/") + 1, arg0.lastIndexOf("."))); + Integer intArg1 = Integer.parseInt(arg1.substring(arg1.indexOf("/") + 1, arg1.lastIndexOf("."))); + if (reverseOrder) { + return intArg1 - intArg0; + } else { + return intArg0 - intArg1; + } + } + } + + /*** + * 根据url地址获取对象名称 + * @author hongawen + * @date 2022/10/17 20:05 + * @param objectUrl 对象地址 + * @return String 对象名称 + */ + public static String getObjectNameByUrl(String objectUrl) { + if (objectUrl.indexOf("?") < 0) { + return "unknownFile"; + } + String objectName = objectUrl.substring(0, objectUrl.indexOf("?")); + return objectName.substring(objectName.lastIndexOf("/") + 1); + } +} diff --git a/src/main/resources/META-INF/spring.factories b/src/main/resources/META-INF/spring.factories new file mode 100644 index 0000000..2e2535f --- /dev/null +++ b/src/main/resources/META-INF/spring.factories @@ -0,0 +1,5 @@ +org.springframework.boot.autoconfigure.EnableAutoConfiguration=\ + com.njcn.minioss.config.MinIoProperties,\ + com.njcn.minioss.util.MinIoUtils + +