From 08c8384ae7bda5f05b6951b4d4b96e041b95fe31 Mon Sep 17 00:00:00 2001 From: masc <59968873@qq.com> Date: Sun, 14 Jul 2024 22:39:34 +0800 Subject: [PATCH 1/2] =?UTF-8?q?=E4=BA=8C=E5=BC=80=E5=8A=A0=E5=85=A5Minio,?= =?UTF-8?q?=E6=9A=82=E5=81=9C=E4=BD=BF=E7=94=A8ES?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pom.xml | 20 +- .../teriteri/backend/config/OSSConfig.java | 32 -- .../config/properties/MinioProperties.java | 50 +++ .../config/properties/OssProperties.java | 40 ++ .../backend/controller/TestControlelr.java | 33 ++ .../teriteri/backend/enums/FilePathEnum.java | 57 +++ .../backend/enums/FileUploadModeEnum.java | 48 +++ .../backend/pojo/dto/UploadFileDto.java | 81 ++++ .../service/impl/user/UserServiceImpl.java | 10 +- .../impl/video/DirectVideoUploadConsumer.java | 6 +- .../service/impl/video/VideoServiceImpl.java | 8 +- .../impl/video/VideoUploadServiceImpl.java | 9 +- .../backend/strategy/UploadStrategy.java | 18 + .../context/UploadStrategyContext.java | 38 ++ .../impl/login/LoginStrategyImpl.java | 9 + .../impl/upload/AbstractUploadStrategy.java | 76 ++++ .../impl/upload/MinioUploadStrategyImpl.java | 131 ++++++ .../impl/upload/OssUploadStrategyImpl.java | 117 +++++ .../com/teriteri/backend/utils/ESUtil.java | 51 +-- .../backend/utils/FileUploadUtil.java | 406 ++++++++++++++++++ .../com/teriteri/backend/utils/OssUtil.java | 246 ----------- .../com/teriteri/backend/utils/RedisUtil.java | 2 +- .../backend/BackendApplicationTests.java | 17 +- ...plication => "\344\272\214\345\274\200.md" | 77 +++- 24 files changed, 1236 insertions(+), 346 deletions(-) delete mode 100644 src/main/java/com/teriteri/backend/config/OSSConfig.java create mode 100644 src/main/java/com/teriteri/backend/config/properties/MinioProperties.java create mode 100644 src/main/java/com/teriteri/backend/config/properties/OssProperties.java create mode 100644 src/main/java/com/teriteri/backend/controller/TestControlelr.java create mode 100644 src/main/java/com/teriteri/backend/enums/FilePathEnum.java create mode 100644 src/main/java/com/teriteri/backend/enums/FileUploadModeEnum.java create mode 100644 src/main/java/com/teriteri/backend/pojo/dto/UploadFileDto.java create mode 100644 src/main/java/com/teriteri/backend/strategy/UploadStrategy.java create mode 100644 src/main/java/com/teriteri/backend/strategy/context/UploadStrategyContext.java create mode 100644 src/main/java/com/teriteri/backend/strategy/impl/login/LoginStrategyImpl.java create mode 100644 src/main/java/com/teriteri/backend/strategy/impl/upload/AbstractUploadStrategy.java create mode 100644 src/main/java/com/teriteri/backend/strategy/impl/upload/MinioUploadStrategyImpl.java create mode 100644 src/main/java/com/teriteri/backend/strategy/impl/upload/OssUploadStrategyImpl.java create mode 100644 src/main/java/com/teriteri/backend/utils/FileUploadUtil.java delete mode 100644 src/main/java/com/teriteri/backend/utils/OssUtil.java rename src/main/resources/application => "\344\272\214\345\274\200.md" (71%) diff --git a/pom.xml b/pom.xml index c7c7106..5134890 100644 --- a/pom.xml +++ b/pom.xml @@ -21,7 +21,19 @@ 7.17.16 - + + + + io.minio + minio + 8.2.2 + + + + cn.hutool + hutool-all + 5.8.18 + com.alibaba @@ -216,8 +228,12 @@ spring-boot-starter-test test + + org.springframework + spring-test + - + diff --git a/src/main/java/com/teriteri/backend/config/OSSConfig.java b/src/main/java/com/teriteri/backend/config/OSSConfig.java deleted file mode 100644 index 87ba6be..0000000 --- a/src/main/java/com/teriteri/backend/config/OSSConfig.java +++ /dev/null @@ -1,32 +0,0 @@ -package com.teriteri.backend.config; - -import com.aliyun.oss.ClientBuilderConfiguration; -import com.aliyun.oss.OSS; -import com.aliyun.oss.OSSClientBuilder; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; - -@Configuration -public class OSSConfig { - @Value("${oss.endpoint}") - private String OSS_ENDPOINT; - - @Value("${oss.keyId}") - private String ACCESS_KEY_ID; - - @Value("${oss.keySecret}") - private String ACCESS_KEY_SECRET; - - @Value("${oss.idleTimeout}") - private long IDLE_TIMEOUT; - - @Bean(destroyMethod = "shutdown") - public OSS ossClient() { - ClientBuilderConfiguration conf = new ClientBuilderConfiguration(); - //连接空闲超时时间，超时则关闭 - conf.setIdleConnectionTime(IDLE_TIMEOUT); - // 创建OSSClient实例 - return new OSSClientBuilder().build(OSS_ENDPOINT, ACCESS_KEY_ID, ACCESS_KEY_SECRET, conf); - } -} diff --git a/src/main/java/com/teriteri/backend/config/properties/MinioProperties.java b/src/main/java/com/teriteri/backend/config/properties/MinioProperties.java new file mode 100644 index 0000000..2b46e94 --- /dev/null +++ b/src/main/java/com/teriteri/backend/config/properties/MinioProperties.java @@ -0,0 +1,50 @@ +package com.teriteri.backend.config.properties; + +import lombok.Data; +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.context.annotation.Configuration; + +import java.util.List; + +/** + * @author 码头薯条Pro + * @date 2024/7/14 11:01 + *

+ */ +@Data +@Configuration +@ConfigurationProperties(prefix = MinioProperties.PREFIX) +public class MinioProperties { + /** + * 配置前缀 + */ + public static final String PREFIX = "upload.minio"; + + /** + * 对象存储服务的URL + */ + private String endpoint; + + /** + * Access key 账户ID + */ + private String accessKey; + + /** + * Secret key 密码 + */ + private String secretKey; + + /** + * 默认的存储桶名称 + */ + private String bucketName; + /** + * 分片存储的临时桶 + */ + private String bucketNameSlice; + /** + * 可上传的文件后缀名 + */ + private List fileExt; +} diff --git a/src/main/java/com/teriteri/backend/config/properties/OssProperties.java b/src/main/java/com/teriteri/backend/config/properties/OssProperties.java new file mode 100644 index 0000000..43b2338 --- /dev/null +++ b/src/main/java/com/teriteri/backend/config/properties/OssProperties.java @@ -0,0 +1,40 @@ +package com.teriteri.backend.config.properties; + +import lombok.Data; +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.context.annotation.Configuration; + +/** + * @author 码头薯条Pro + * @date 2024/7/14 11:00 + *

oss配置属性

+ */ +@Data +@Configuration +@ConfigurationProperties(prefix = "upload.oss") +public class OssProperties { + /** + * oss域名 + */ + private String url; + + /** + * 终点 + */ + private String endpoint; + + /** + * 访问密钥id + */ + private String accessKeyId; + + /** + * 访问密钥密码 + */ + private String accessKeySecret; + + /** + * bucket名称 + */ + private String bucketName; +} diff --git a/src/main/java/com/teriteri/backend/controller/TestControlelr.java b/src/main/java/com/teriteri/backend/controller/TestControlelr.java new file mode 100644 index 0000000..ae8c3c0 --- /dev/null +++ b/src/main/java/com/teriteri/backend/controller/TestControlelr.java @@ -0,0 +1,33 @@ +package com.teriteri.backend.controller; + +import com.teriteri.backend.utils.FileUploadUtil; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RestController; + +import java.io.IOException; +import java.util.Map; + +/** + * @author 码头薯条Pro + * @date 2024/7/13 19:36 + *

map) throws IOException { + String hash = map.get("hash"); + // 合并到OSS，并返回URL地址 + String url = fileUploadUtil.appendUploadVideo(hash); + + return url; + } + + +} diff --git a/src/main/java/com/teriteri/backend/enums/FilePathEnum.java b/src/main/java/com/teriteri/backend/enums/FilePathEnum.java new file mode 100644 index 0000000..c9ca886 --- /dev/null +++ b/src/main/java/com/teriteri/backend/enums/FilePathEnum.java @@ -0,0 +1,57 @@ +package com.teriteri.backend.enums; + +import lombok.AllArgsConstructor; +import lombok.Getter; + +/** + * @author 码头薯条Pro + * @date 2024/7/14 10:54 + *

文件路径枚举

+ */ +@Getter +@AllArgsConstructor +public enum FilePathEnum { + + /** + * 头像路径 + */ + AVATAR("/avatar/", "/avatar", "头像路径"), + + /** + * 文章图片路径 + */ + ARTICLE("/article/", "/article", "文章图片路径"), + + /** + * 配置图片路径 + */ + CONFIG("/config/", "/config", "配置图片路径"), + + /** + * 说说图片路径 + */ + TALK("/talk/", "/talk", "说说图片路径"), + + /** + * 照片路径 + */ + PHOTO("/photo/", "/photo", "相册路径"), + /** + * 媒体 + */ + VIDEO("/media/", "/media", "相册路径"); + /** + * 路径 + */ + private final String path; + + /** + * 文件路径 + */ + private final String filePath; + + /** + * 描述 + */ + private final String description; +} diff --git a/src/main/java/com/teriteri/backend/enums/FileUploadModeEnum.java b/src/main/java/com/teriteri/backend/enums/FileUploadModeEnum.java new file mode 100644 index 0000000..e8c235d --- /dev/null +++ b/src/main/java/com/teriteri/backend/enums/FileUploadModeEnum.java @@ -0,0 +1,48 @@ +package com.teriteri.backend.enums; + +import lombok.AllArgsConstructor; +import lombok.Getter; + +/** + * @author 码头薯条Pro + * @date 2024/7/14 10:43 + *

上传模式枚举

+ */ +@Getter +@AllArgsConstructor +public enum FileUploadModeEnum { + /** + * oss + */ + OSS("oss", "ossUploadStrategyImpl"), + + /** + * minio + */ + MINIO("minio", "minioUploadStrategyImpl"); + + /** + * 模式 + */ + private final String mode; + + /** + * 策略 + */ + private final String strategy; + + /** + * 获取策略 + * + * @param mode 模式 + * @return 搜索策略 + */ + public static String getStrategy(String mode) { + for (FileUploadModeEnum value : FileUploadModeEnum.values()) { + if (value.getMode().equals(mode)) { + return value.getStrategy(); + } + } + return null; + } +} diff --git a/src/main/java/com/teriteri/backend/pojo/dto/UploadFileDto.java b/src/main/java/com/teriteri/backend/pojo/dto/UploadFileDto.java new file mode 100644 index 0000000..8b50f4e --- /dev/null +++ b/src/main/java/com/teriteri/backend/pojo/dto/UploadFileDto.java @@ -0,0 +1,81 @@ +package com.teriteri.backend.pojo.dto; + +import lombok.Data; +import org.springframework.web.multipart.MultipartFile; + +import java.io.File; +import java.io.InputStream; +import java.io.Serializable; + +/** + * @author 码头薯条Pro + * @date 2024/7/14 10:18 + *

上传文件DTO

+ */ +@Data +public class UploadFileDto implements Serializable { + /** + * 标识列 + */ + private String uuid; + /** + * 需要上传的文件 + */ + private File localFile; + /** + * 需要上传的文件 + */ + private MultipartFile file; + /** + * 需要上传的文件 inputStream + */ + private InputStream fileInputStream; + /** + * 文件Md5 + */ + private String fileMd5; + /** + * 文件原始名称 + */ + private String originalFileName; + /** + * 文件存储地址 上传路径 + */ + private String uploadPath; + /** + * 需要上传的文件名 + */ + private String fileName; + /** + * 0不需要分片 1需要分片 + */ + private Integer needSlice; + /** + * 第几个分片 + */ + private Integer indexSlice; + /** + * 总分片 + */ + private Integer totalSlice; + /** + * 分片追加的位置 + */ + private long position; + /** + * 分片的大小 + */ + private Double sliceFileSize; + /** + * 文件大小 + */ + private Double fileSize; + /** + * 桶名称 + */ + private String bucketName; + /** + * 是否上传;0：未上传，1：已上传 + */ + private Integer isUploaded; +} diff --git a/src/main/java/com/teriteri/backend/service/impl/user/UserServiceImpl.java b/src/main/java/com/teriteri/backend/service/impl/user/UserServiceImpl.java index 3b1e936..ec97037 100644 --- a/src/main/java/com/teriteri/backend/service/impl/user/UserServiceImpl.java +++ b/src/main/java/com/teriteri/backend/service/impl/user/UserServiceImpl.java @@ -10,7 +10,7 @@ import com.teriteri.backend.service.user.UserService; import com.teriteri.backend.service.video.VideoStatsService; import com.teriteri.backend.utils.ESUtil; -import com.teriteri.backend.utils.OssUtil; +import com.teriteri.backend.utils.FileUploadUtil; import com.teriteri.backend.utils.RedisUtil; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; @@ -45,9 +45,9 @@ public class UserServiceImpl implements UserService { private ESUtil esUtil; @Autowired - private OssUtil ossUtil; + private FileUploadUtil fileUploadUtil; - @Value("${oss.bucketUrl}") + @Value("${upload.oss.bucketName}") private String OSS_BUCKET_URL; @Autowired @@ -227,7 +227,7 @@ public CustomResponse updateUserInfo(Integer uid, String nickname, String desc, @Override public CustomResponse updateUserAvatar(Integer uid, MultipartFile file) throws IOException { // 保存封面到OSS，返回URL - String avatar_url = ossUtil.uploadImage(file, "avatar"); + String avatar_url = fileUploadUtil.uploadImage(file, "avatar"); // 查旧的头像地址 User user = userMapper.selectById(uid); // 先更新数据库 @@ -240,7 +240,7 @@ public CustomResponse updateUserAvatar(Integer uid, MultipartFile file) throws I if (user.getAvatar().startsWith(OSS_BUCKET_URL)) { String filename = user.getAvatar().substring(OSS_BUCKET_URL.length()); // System.out.println("要删除的源文件：" + filename); - ossUtil.deleteFiles(filename); + fileUploadUtil.deleteFiles(filename); } }, taskExecutor); return new CustomResponse(200, "OK", avatar_url); diff --git a/src/main/java/com/teriteri/backend/service/impl/video/DirectVideoUploadConsumer.java b/src/main/java/com/teriteri/backend/service/impl/video/DirectVideoUploadConsumer.java index 1f90eb5..516cbc4 100644 --- a/src/main/java/com/teriteri/backend/service/impl/video/DirectVideoUploadConsumer.java +++ b/src/main/java/com/teriteri/backend/service/impl/video/DirectVideoUploadConsumer.java @@ -7,7 +7,7 @@ import com.teriteri.backend.pojo.VideoStats; import com.teriteri.backend.pojo.dto.VideoUploadInfoDTO; import com.teriteri.backend.utils.ESUtil; -import com.teriteri.backend.utils.OssUtil; +import com.teriteri.backend.utils.FileUploadUtil; import com.teriteri.backend.utils.RedisUtil; import lombok.extern.slf4j.Slf4j; import org.springframework.amqp.rabbit.annotation.RabbitHandler; @@ -45,7 +45,7 @@ public class DirectVideoUploadConsumer { private RedisUtil redisUtil; @Autowired - private OssUtil ossUtil; + private FileUploadUtil fileUploadUtil; @Autowired private ESUtil esUtil; @@ -108,7 +108,7 @@ public void handleMergeChunks(String jsonPayload) throws IOException { // } // 合并到OSS，并返回URL地址 - url = ossUtil.appendUploadVideo(vui.getHash()); + url = fileUploadUtil.appendUploadVideo(vui.getHash()); if (url == null) { return; } diff --git a/src/main/java/com/teriteri/backend/service/impl/video/VideoServiceImpl.java b/src/main/java/com/teriteri/backend/service/impl/video/VideoServiceImpl.java index a1125aa..46788b3 100644 --- a/src/main/java/com/teriteri/backend/service/impl/video/VideoServiceImpl.java +++ b/src/main/java/com/teriteri/backend/service/impl/video/VideoServiceImpl.java @@ -13,7 +13,7 @@ import com.teriteri.backend.service.video.VideoService; import com.teriteri.backend.service.video.VideoStatsService; import com.teriteri.backend.utils.ESUtil; -import com.teriteri.backend.utils.OssUtil; +import com.teriteri.backend.utils.FileUploadUtil; import com.teriteri.backend.utils.RedisUtil; import lombok.extern.slf4j.Slf4j; import org.apache.ibatis.session.ExecutorType; @@ -58,7 +58,7 @@ public class VideoServiceImpl implements VideoService { private RedisUtil redisUtil; @Autowired - private OssUtil ossUtil; + private FileUploadUtil fileUploadUtil; @Autowired private ESUtil esUtil; @@ -442,8 +442,8 @@ public CustomResponse updateVideoStatus(Integer vid, Integer status) throws IOEx redisUtil.delValue("danmu_idset:" + vid); // 删除该视频的弹幕 redisUtil.zsetDelMember("user_video_upload:" + video.getUid(), video.getVid()); // 搞个异步线程去删除OSS的源文件 - CompletableFuture.runAsync(() -> ossUtil.deleteFiles(videoPrefix), taskExecutor); - CompletableFuture.runAsync(() -> ossUtil.deleteFiles(coverPrefix), taskExecutor); + CompletableFuture.runAsync(() -> fileUploadUtil.deleteFiles(videoPrefix), taskExecutor); + CompletableFuture.runAsync(() -> fileUploadUtil.deleteFiles(coverPrefix), taskExecutor); // 批量删除该视频下的全部评论缓存 CompletableFuture.runAsync(() -> { Set set = redisUtil.zReverange("comment_video:" + vid, 0, -1); diff --git a/src/main/java/com/teriteri/backend/service/impl/video/VideoUploadServiceImpl.java b/src/main/java/com/teriteri/backend/service/impl/video/VideoUploadServiceImpl.java index a765e5e..01ec879 100644 --- a/src/main/java/com/teriteri/backend/service/impl/video/VideoUploadServiceImpl.java +++ b/src/main/java/com/teriteri/backend/service/impl/video/VideoUploadServiceImpl.java @@ -1,6 +1,5 @@ package com.teriteri.backend.service.impl.video; -import com.fasterxml.jackson.core.JsonProcessingException; import com.teriteri.backend.mapper.VideoMapper; import com.teriteri.backend.mapper.VideoStatsMapper; import com.teriteri.backend.pojo.CustomResponse; @@ -10,7 +9,7 @@ import com.teriteri.backend.service.utils.CurrentUser; import com.teriteri.backend.service.video.VideoUploadService; import com.teriteri.backend.utils.ESUtil; -import com.teriteri.backend.utils.OssUtil; +import com.teriteri.backend.utils.FileUploadUtil; import com.teriteri.backend.utils.RedisUtil; import lombok.extern.slf4j.Slf4j; import org.springframework.amqp.rabbit.core.RabbitTemplate; @@ -54,7 +53,7 @@ public class VideoUploadServiceImpl implements VideoUploadService { private RabbitTemplate rabbitTemplate; @Autowired - private OssUtil ossUtil; + private FileUploadUtil fileUploadUtil; @Autowired private ESUtil esUtil; @@ -200,7 +199,7 @@ public CustomResponse addVideo(MultipartFile cover, VideoUploadInfoDTO videoUplo // } // 保存封面到OSS，返回URL - String coverUrl = ossUtil.uploadImage(cover, "cover"); + String coverUrl = fileUploadUtil.uploadImage(cover, "cover"); // 将投稿信息封装 videoUploadInfoDTO.setCoverUrl(coverUrl); @@ -277,7 +276,7 @@ public void mergeChunks(VideoUploadInfoDTO vui) throws IOException { // } // 合并到OSS，并返回URL地址 - url = ossUtil.appendUploadVideo(vui.getHash()); + url = fileUploadUtil.appendUploadVideo(vui.getHash()); if (url == null) { return; } diff --git a/src/main/java/com/teriteri/backend/strategy/UploadStrategy.java b/src/main/java/com/teriteri/backend/strategy/UploadStrategy.java new file mode 100644 index 0000000..c578c75 --- /dev/null +++ b/src/main/java/com/teriteri/backend/strategy/UploadStrategy.java @@ -0,0 +1,18 @@ +package com.teriteri.backend.strategy; + +import com.teriteri.backend.pojo.dto.UploadFileDto; + +/** + * @author 码头薯条Pro + * @date 2024/7/14 10:22 + *

上传策略

+ */ +public interface UploadStrategy { + /** + * 上传文件 + * + * @param uploadInfo 需要上传文件的详情 + * @return {@link String} 文件地址 + */ + String uploadFile(UploadFileDto uploadInfo); +} diff --git a/src/main/java/com/teriteri/backend/strategy/context/UploadStrategyContext.java b/src/main/java/com/teriteri/backend/strategy/context/UploadStrategyContext.java new file mode 100644 index 0000000..053fc02 --- /dev/null +++ b/src/main/java/com/teriteri/backend/strategy/context/UploadStrategyContext.java @@ -0,0 +1,38 @@ +package com.teriteri.backend.strategy.context; + +import com.teriteri.backend.enums.FileUploadModeEnum; +import com.teriteri.backend.pojo.dto.UploadFileDto; +import com.teriteri.backend.strategy.UploadStrategy; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Service; +import org.springframework.web.multipart.MultipartFile; + +import java.util.Map; + +/** + * @author 码头薯条Pro + * @date 2024/7/14 10:33 + *

上传策略上下文

+ */ +@Service +public class UploadStrategyContext { + /** + * 上传模式 + */ + @Value("${upload.strategy}") + private String uploadStrategy; + + @Autowired + private Map uploadStrategyMap; + + /** + * 上传文件 + * + * @param uploadInfo 文件上传信息 + * @return {@link String} 文件地址 + */ + public String executeUploadStrategy(UploadFileDto uploadInfo) { + return uploadStrategyMap.get(FileUploadModeEnum.getStrategy(uploadStrategy)).uploadFile(uploadInfo); + } +} diff --git a/src/main/java/com/teriteri/backend/strategy/impl/login/LoginStrategyImpl.java b/src/main/java/com/teriteri/backend/strategy/impl/login/LoginStrategyImpl.java new file mode 100644 index 0000000..362c44d --- /dev/null +++ b/src/main/java/com/teriteri/backend/strategy/impl/login/LoginStrategyImpl.java @@ -0,0 +1,9 @@ +package com.teriteri.backend.strategy.impl.login; + +/** + * @author 码头薯条Pro + * @date 2024/7/14 10:37 + *

登录策略

TODO QQ gitee github 等社交账户登录 + */ +public class LoginStrategyImpl { +} diff --git a/src/main/java/com/teriteri/backend/strategy/impl/upload/AbstractUploadStrategy.java b/src/main/java/com/teriteri/backend/strategy/impl/upload/AbstractUploadStrategy.java new file mode 100644 index 0000000..6a0bc69 --- /dev/null +++ b/src/main/java/com/teriteri/backend/strategy/impl/upload/AbstractUploadStrategy.java @@ -0,0 +1,76 @@ +package com.teriteri.backend.strategy.impl.upload; + +import com.aliyun.oss.ServiceException; +import com.teriteri.backend.pojo.dto.UploadFileDto; +import com.teriteri.backend.strategy.UploadStrategy; +import org.springframework.stereotype.Service; + +import java.io.IOException; +import java.io.InputStream; +import java.util.List; + +/** + * @author 码头薯条Pro + * @date 2024/7/14 10:36 + *

抽象上传模板

+ */ +@Service +public abstract class AbstractUploadStrategy implements UploadStrategy { + @Override + public String uploadFile(UploadFileDto uploadInfo) { + try { + //根据信息做处理 + if (0 == uploadInfo.getNeedSlice()) { + //不分片 上传完整文件 + upload(uploadInfo.getUploadPath(), uploadInfo.getFileName(), uploadInfo.getFileInputStream()); + } else if (1 == uploadInfo.getNeedSlice()) { + //分片上传并合并 + uploadAndComposeFile(uploadInfo); + } + return getFileAccessUrl(uploadInfo.getUploadPath() + "" + uploadInfo.getFileName()); + } catch (Exception e) { + e.printStackTrace(); + throw new ServiceException("文件上传失败"); + } + } + + /** + * 判断文件是否存在 + * + * @param filePath 文件路径 + * @return {@link Boolean} + */ + public abstract Boolean exists(String filePath); + + /** + * 上传 + * + * @param path 路径 + * @param fileName 文件名 + * @param inputStream 输入流 + * @throws IOException io异常 + */ + public abstract void upload(String path, String fileName, InputStream inputStream); + + /** + * 上传 然后 合并分片 + * + * @param uploadInfo 文件上传信息 + */ + public abstract void uploadAndComposeFile(UploadFileDto uploadInfo); + + /** + * 删除文件 + * + * @param fileNameList 删除文件的路径 + */ + public abstract void removeFiles(List fileNameList); + + /** + * 获取文件访问url + * + * @param filePath 文件路径 + * @return {@link String} 文件url + */ + public abstract String getFileAccessUrl(String filePath); +} diff --git a/src/main/java/com/teriteri/backend/strategy/impl/upload/MinioUploadStrategyImpl.java b/src/main/java/com/teriteri/backend/strategy/impl/upload/MinioUploadStrategyImpl.java new file mode 100644 index 0000000..b095468 --- /dev/null +++ b/src/main/java/com/teriteri/backend/strategy/impl/upload/MinioUploadStrategyImpl.java @@ -0,0 +1,131 @@ +package com.teriteri.backend.strategy.impl.upload; + +import com.aliyun.oss.OSS; +import com.teriteri.backend.config.properties.MinioProperties; +import com.teriteri.backend.pojo.dto.UploadFileDto; +import io.minio.*; +import io.minio.messages.DeleteError; +import io.minio.messages.DeleteObject; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Service; +import javax.annotation.Resource; +import java.io.InputStream; +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +/** + * @author 码头薯条Pro + * @date 2024/7/14 10:48 + *

Minio上传策略

+ */ +@Slf4j +@Service("minioUploadStrategyImpl") +public class MinioUploadStrategyImpl extends AbstractUploadStrategy { + @Resource + private MinioProperties minioProperties; + + /** + * 获取ossClient + * + * @return {@link OSS} ossClient + */ + private MinioClient getMinioClient() { + MinioClient minioClient = MinioClient.builder() + .endpoint(minioProperties.getEndpoint()) + .credentials(minioProperties.getAccessKey(), minioProperties.getSecretKey()) + .build(); + return minioClient; + } + + @Override + public Boolean exists(String filePath) { + return null; + } + + @Override + public void upload(String path, String fileName, InputStream stream) { + MinioClient minioClient = getMinioClient(); + try { + // 写入文件 + minioClient.putObject(PutObjectArgs.builder().bucket(minioProperties.getBucketName()) + .object(path + fileName) + .stream(stream, stream.available(), -1).contentType("application/octet" + "-stream").build()); + log.info("上传到minio文件|uploadFile|参数：bucketName：{}，路径：{} " + , minioProperties.getBucketName(), path + fileName); + } catch (Exception e) { + log.error("文件上传到Minio异常|参数：bucketName:{},路径:{},|异常:{}", minioProperties.getBucketName(), path + fileName, e); + } + } + + @Override + public void uploadAndComposeFile(UploadFileDto uploadInfo) { + MinioClient minioClient = getMinioClient(); + try { + String tempPath = uploadInfo.getFileMd5().concat("/").concat(Integer.toString(uploadInfo.getIndexSlice())); + // 写入临时文件夹 + minioClient.putObject(PutObjectArgs.builder() + .bucket(minioProperties.getBucketName()) + .object(tempPath) + .stream(uploadInfo.getFile().getInputStream(), uploadInfo.getFile().getSize(), -1) + .contentType(uploadInfo.getFile().getContentType()) + .build()); + log.info("上传分片到minio文件|路径：{}，分片下标：{}", tempPath, uploadInfo.getIndexSlice()); + //如果分片是最后一片 那就合并 + if ((uploadInfo.getTotalSlice() - 1) == uploadInfo.getIndexSlice()) { + log.info("合并Minio文件开始：path：{}，分片下标：{}", tempPath, uploadInfo.getIndexSlice()); + List sourceObjectList = Stream.iterate(0, i -> ++i) + .limit(uploadInfo.getTotalSlice()) + .map(i -> ComposeSource.builder() + .bucket(minioProperties.getBucketName()) + .object(uploadInfo.getFileMd5().concat("/").concat(Integer.toString(i))) // 临时文件们的路径 + .build()) + .collect(Collectors.toList()); + ObjectWriteResponse response = minioClient.composeObject( + ComposeObjectArgs.builder() + .bucket(minioProperties.getBucketName()) + .object(uploadInfo.getUploadPath() + uploadInfo.getFileName()) //新指定的路径 + .sources(sourceObjectList) + .build()); + log.info("合并Minio文件结束：path：{}，分片下标：{}", uploadInfo.getUploadPath() + uploadInfo.getFileName(), uploadInfo.getIndexSlice()); + //删除临时分片 + List fileNameList = new ArrayList<>(); + for (int i = 0; i < uploadInfo.getTotalSlice(); i++) { + fileNameList.add(uploadInfo.getFileMd5().concat("/").concat(Integer.toString(i))); + } + removeFiles(fileNameList); + } + } catch (Exception e) { + log.error("上传分片到minio文件异常|路径：{}，分片下标：{} 异常:{}", uploadInfo.getUploadPath() + uploadInfo.getFileName(), uploadInfo.getIndexSlice(), e); + } + } + + @Override + public void removeFiles(List fileNameList) { + MinioClient minioClient = getMinioClient(); + try { + //拼接需要删除的文件入参 + List deleteObjectList = new ArrayList<>(); + for (String fileName : fileNameList) { + DeleteObject temp = new DeleteObject(fileName); + deleteObjectList.add(temp); + } + //调用删除 + Iterable> results = + minioClient.removeObjects( + RemoveObjectsArgs.builder().bucket(minioProperties.getBucketName()).objects(deleteObjectList).build()); + log.info("Minio多个文件删除完成!|参数：fileNameList:{}", fileNameList); + } catch (Exception e) { + log.error("Minio多个文件删除异常!|参数：fileNameList:{}|异常:{}", fileNameList, e); + } + } + + @Override + public String getFileAccessUrl(String filePath) { + //获取文件的访问路径URL + String url = minioProperties.getEndpoint() + "/" + minioProperties.getBucketName() + "/" + filePath; + log.info("上传到minio文件|访问路径：{} ", url); + return url; + } +} diff --git a/src/main/java/com/teriteri/backend/strategy/impl/upload/OssUploadStrategyImpl.java b/src/main/java/com/teriteri/backend/strategy/impl/upload/OssUploadStrategyImpl.java new file mode 100644 index 0000000..5ad768c --- /dev/null +++ b/src/main/java/com/teriteri/backend/strategy/impl/upload/OssUploadStrategyImpl.java @@ -0,0 +1,117 @@ +package com.teriteri.backend.strategy.impl.upload; + +import com.aliyun.oss.ClientException; +import com.aliyun.oss.OSS; +import com.aliyun.oss.OSSClientBuilder; +import com.aliyun.oss.OSSException; +import com.aliyun.oss.model.AppendObjectRequest; +import com.aliyun.oss.model.AppendObjectResult; +import com.aliyun.oss.model.ObjectMetadata; +import com.teriteri.backend.config.properties.OssProperties; +import com.teriteri.backend.pojo.dto.UploadFileDto; +import com.teriteri.backend.utils.RedisUtil; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import java.io.*; +import java.util.List; +import java.util.concurrent.TimeUnit; + +/** + * @author 码头薯条Pro + * @date 2024/7/14 10:47 + *

oss上传策略

+ */ +@Slf4j +@Service("ossUploadStrategyImpl") +public class OssUploadStrategyImpl extends AbstractUploadStrategy { + + @Autowired + private OssProperties ossProperties; + @Autowired + private RedisUtil redisUtil; + /** + * 获取ossClient + * + * @return {@link OSS} ossClient + */ + private OSS getOssClient() { + return new OSSClientBuilder().build(ossProperties.getEndpoint(), ossProperties.getAccessKeyId(), ossProperties.getAccessKeySecret()); + } + + @Override + public Boolean exists(String filePath) { + return getOssClient().doesObjectExist(ossProperties.getBucketName(), filePath); + } + + @Override + public void upload(String path, String fileName, InputStream inputStream) { + OSS ossClient = getOssClient(); + try { + // 调用oss方法上传 + ossClient.putObject(ossProperties.getBucketName(), path + fileName, inputStream); + } catch (OSSException oe) { + log.error("Error Message:" + oe.getErrorMessage()); + log.error("Error Code:" + oe.getErrorCode()); + log.info("Request ID:" + oe.getRequestId()); + log.info("Host ID:" + oe.getHostId()); + } catch (ClientException ce) { + log.error("Caught an ClientException, Error Message:" + ce.getMessage()); + } finally { + if (ossClient != null) { + ossClient.shutdown(); + } + } + } + + @Override + public void uploadAndComposeFile(UploadFileDto uploadInfo) { + OSS ossClient = getOssClient(); + try { + String s = redisUtil.getValue(uploadInfo.getUploadPath() + uploadInfo.getFileName()).toString(); + Long position = Long.parseLong(s); + if (-1L == position) { + position = 0L; + } + ObjectMetadata meta = new ObjectMetadata(); + meta.setContentType("application/octet" + "-stream"); + File localFile = uploadInfo.getLocalFile(); + // 读取分片数据 + FileInputStream fis = new FileInputStream(localFile); + byte[] buffer = new byte[(int) localFile.length()]; + fis.read(buffer); + fis.close(); + // 追加上传分片数据 + AppendObjectRequest appendObjectRequest = new AppendObjectRequest(ossProperties.getBucketName(), uploadInfo.getUploadPath() + uploadInfo.getFileName(), new ByteArrayInputStream(buffer), meta); + appendObjectRequest.setPosition(position); + AppendObjectResult appendObjectResult = ossClient.appendObject(appendObjectRequest); + position = appendObjectResult.getNextPosition(); + redisUtil.setExValue(uploadInfo.getUploadPath() + uploadInfo.getFileName(), position, 5 * 60, TimeUnit.SECONDS); + } catch (OSSException oe) { + log.error("OSS出错了:" + oe.getErrorMessage()); + throw oe; + } catch (ClientException ce) { + log.error("OSS连接出错了:" + ce.getMessage()); + throw ce; + } catch (FileNotFoundException e) { + e.printStackTrace(); + } catch (IOException e) { + e.printStackTrace(); + } + } + + @Override + public void removeFiles(List fileNameList) { + + } + + @Override + public String getFileAccessUrl(String filePath) { + String url = ossProperties.getUrl() + "/" + filePath; + log.info("上传到OSS文件|访问路径：{} ", url); + return url; + } + + +} diff --git a/src/main/java/com/teriteri/backend/utils/ESUtil.java b/src/main/java/com/teriteri/backend/utils/ESUtil.java index f0618cf..e14a5f7 100644 --- a/src/main/java/com/teriteri/backend/utils/ESUtil.java +++ b/src/main/java/com/teriteri/backend/utils/ESUtil.java @@ -29,13 +29,13 @@ public class ESUtil { * @param video */ public void addVideo(Video video) throws IOException { - try { + /* try { ESVideo esVideo = new ESVideo(video.getVid(), video.getUid(), video.getTitle(), video.getMcId(), video.getScId(), video.getTags(), video.getStatus()); client.index(i -> i.index("video").id(esVideo.getVid().toString()).document(esVideo)); } catch (IOException e) { log.error("添加视频文档到ElasticSearch时出错了：" + e); throw e; - } + }*/ } /** @@ -43,12 +43,12 @@ public void addVideo(Video video) throws IOException { * @param vid */ public void deleteVideo(Integer vid) throws IOException { - try { + /* try { client.delete(d -> d.index("video").id(vid.toString())); } catch (IOException e) { log.error("删除ElasticSearch视频文档时失败了：" + e); throw e; - } + }*/ } /** @@ -56,13 +56,13 @@ public void deleteVideo(Integer vid) throws IOException { * @param video */ public void updateVideo(Video video) throws IOException { - try { + /* try { ESVideo esVideo = new ESVideo(video.getVid(), video.getUid(), video.getTitle(), video.getMcId(), video.getScId(), video.getTags(), video.getStatus()); client.update(u -> u.index("video").id(video.getVid().toString()).doc(esVideo), ESVideo.class); } catch (IOException e) { log.error("更新ElasticSearch视频文档时出错了：" + e); throw e; - } + }*/ } /** @@ -72,7 +72,7 @@ public void updateVideo(Video video) throws IOException { * @return */ public Long getVideoCount(String keyword, boolean onlyPass) { - try { + /*try { Query query = Query.of(q -> q.multiMatch(m -> m.fields("title", "tags").query(keyword))); Query query1 = Query.of(q -> q.constantScore(c -> c.filter(f -> f.term(t -> t.field("status").value(1))))); Query bool = Query.of(q -> q.bool(b -> b.must(query1).must(query))); @@ -87,7 +87,7 @@ public Long getVideoCount(String keyword, boolean onlyPass) { } catch (IOException e) { log.error("查询ES相关视频数量时出错了：" + e); return 0L; - } + }*/return 0L; } /** @@ -98,7 +98,7 @@ public Long getVideoCount(String keyword, boolean onlyPass) { * @return 包含查到的数据id列表，按匹配分数排序 */ public List searchVideosByKeyword(String keyword, Integer page, Integer size, boolean onlyPass) { - try { + /* try { List list = new ArrayList<>(); Query query = Query.of(q -> q.multiMatch(m -> m.fields("title", "tags").query(keyword))); Query query1 = Query.of(q -> q.constantScore(c -> c.filter(f -> f.term(t -> t.field("status").value(1))))); @@ -117,7 +117,8 @@ public List searchVideosByKeyword(String keyword, Integer page, Integer } catch (IOException e) { log.error("查询ES相关视频文档时出错了：" + e); return Collections.emptyList(); - } + }*/ + return Collections.emptyList(); } @@ -126,13 +127,13 @@ public List searchVideosByKeyword(String keyword, Integer page, Integer * @param user */ public void addUser(User user) throws IOException { - try { + /* try { ESUser esUser = new ESUser(user.getUid(), user.getNickname()); client.index(i -> i.index("user").id(esUser.getUid().toString()).document(esUser)); } catch (IOException e) { log.error("添加用户文档到elasticsearch时出错了：" + e); throw e; - } + }*/ } /** @@ -140,12 +141,12 @@ public void addUser(User user) throws IOException { * @param uid */ public void deleteUser(Integer uid) throws IOException { - try { + /* try { client.delete(d -> d.index("user").id(uid.toString())); } catch (IOException e) { log.error("删除ElasticSearch用户文档时失败了：" + e); throw e; - } + }*/ } /** @@ -153,13 +154,13 @@ public void deleteUser(Integer uid) throws IOException { * @param user */ public void updateUser(User user) throws IOException { - try { + /*try { ESUser esUser = new ESUser(user.getUid(), user.getNickname()); client.update(u -> u.index("user").id(user.getUid().toString()).doc(esUser), ESUser.class); } catch (IOException e) { log.error("更新ElasticSearch用户文档时出错了：" + e); throw e; - } + }*/ } /** @@ -168,7 +169,7 @@ public void updateUser(User user) throws IOException { * @return */ public Long getUserCount(String keyword) { - try { + /* try { Query query = Query.of(q -> q.simpleQueryString(s -> s.fields("nickname").query(keyword).defaultOperator(Operator.And))); CountRequest countRequest = new CountRequest.Builder().index("user").query(query).build(); CountResponse countResponse = client.count(countRequest); @@ -176,7 +177,8 @@ public Long getUserCount(String keyword) { } catch (IOException e) { log.error("查询ES相关用户数量时出错了：" + e); return 0L; - } + }*/ + return 0L; } /** @@ -187,7 +189,7 @@ public Long getUserCount(String keyword) { * @return 包含查到的数据id列表，按匹配分数排序 */ public List searchUsersByKeyword(String keyword, Integer page, Integer size) { - try { + /* try { List list = new ArrayList<>(); Query query = Query.of(q -> q.simpleQueryString(s -> s.fields("nickname").query(keyword).defaultOperator(Operator.And))); SearchRequest searchRequest = new SearchRequest.Builder().index("user").query(query).from((page - 1) * size).size(size).build(); @@ -199,7 +201,7 @@ public List searchUsersByKeyword(String keyword, Integer page, Integer } catch (IOException e) { log.error("查询ES相关用户文档时出错了：" + e); return Collections.emptyList(); - } + }*/ return Collections.emptyList(); } @@ -210,12 +212,12 @@ public List searchUsersByKeyword(String keyword, Integer page, Integer * @param text */ public void addSearchWord(String text) { - try { + /* try { ESSearchWord esSearchWord = new ESSearchWord(text); client.index(i -> i.index("search_word").document(esSearchWord)); } catch (IOException e) { log.error("添加搜索词文档到elasticsearch时出错了：" + e); - } + }*/ } /** @@ -224,7 +226,7 @@ public void addSearchWord(String text) { * @return */ public List getMatchingWord(String text) { - try { + /* try { List list = new ArrayList<>(); Query query = Query.of(q -> q.simpleQueryString(s -> s.fields("content").query(text).defaultOperator(Operator.And))); // 关键词全匹配 Query query1 = Query.of(q -> q.prefix(p -> p.field("content").value(text))); @@ -238,6 +240,7 @@ public List getMatchingWord(String text) { } catch (IOException e) { log.error("获取ES搜索提示词时出错了：" + e); return Collections.emptyList(); - } + }*/ + return Collections.emptyList(); } } diff --git a/src/main/java/com/teriteri/backend/utils/FileUploadUtil.java b/src/main/java/com/teriteri/backend/utils/FileUploadUtil.java new file mode 100644 index 0000000..4f60f11 --- /dev/null +++ b/src/main/java/com/teriteri/backend/utils/FileUploadUtil.java @@ -0,0 +1,406 @@ +package com.teriteri.backend.utils; + +import com.aliyun.oss.ClientException; +import com.aliyun.oss.OSS; +import com.aliyun.oss.OSSException; +import com.aliyun.oss.model.*; +import com.teriteri.backend.enums.FilePathEnum; +import com.teriteri.backend.pojo.dto.UploadFileDto; +import com.teriteri.backend.strategy.context.UploadStrategyContext; +import lombok.NonNull; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.io.FilenameUtils; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.mock.web.MockMultipartFile; +import org.springframework.stereotype.Component; +import org.springframework.web.multipart.MultipartFile; + +import java.io.*; +import java.net.URLDecoder; +import java.text.SimpleDateFormat; +import java.util.*; +import java.util.concurrent.TimeUnit; + + +@Slf4j +@Component +public class FileUploadUtil { + @Value("${directory.chunk}") + private String CHUNK_DIRECTORY; // 分片存储目录 + @Autowired + private RedisUtil redisUtil; + @Autowired + private UploadStrategyContext uploadStrategyContext; + + /** + * 往阿里云对象存储上传单张图片 + * + * @param file 图片文件 + * @param type 图片分类，如 cover、carousel、other等，不允许空字符串，这里没有做判断了，自己注意就好 + * @return 图片的URL地址 + * @throws IOException + */ + public String uploadImage(@NonNull MultipartFile file, @NonNull String type) throws IOException { + // 生成文件名 + String originalFilename = file.getOriginalFilename(); // 获取原文件名 + String ext = "." + FilenameUtils.getExtension(originalFilename); // 获取文件后缀 + String uuid = System.currentTimeMillis() + UUID.randomUUID().toString().replace("-", ""); + String fileName = uuid + ext; + // 完整路径名 +// String date = new SimpleDateFormat("yyyy-MM-dd").format(new Date()).replace("-", ""); +// String filePathName = date + "/img/" + type + "/" + fileName; + String folderName = FilePathEnum.PHOTO.getFilePath() + "/picture/" + type + "/"; + UploadFileDto uploadInfo = new UploadFileDto(); + uploadInfo.setUploadPath(folderName); + uploadInfo.setFileName(fileName); + uploadInfo.setNeedSlice(0); +// uploadInfo.setFileMd5(); + uploadInfo.setFile(file); + uploadInfo.setFileInputStream(file.getInputStream()); + String url = uploadStrategyContext.executeUploadStrategy(uploadInfo); + +/* if (uploadStrategyConfig.equals("minio")) { + try { + //路径 + + OssFile ossFile = ossTemplate.upLoadFile(folderName, fileName, file); + log.info(String.valueOf(ossFile)); + } catch (OSSException oe) { + log.error("OSS出错了:" + oe.getErrorMessage()); + throw oe; + } catch (ClientException ce) { + log.error("OSS连接出错了:" + ce.getMessage()); + throw ce; + } + //前端可访问的资源路径 + String url = minioEndpoint + "/" + minioBucketName + "/" + filePathName; + log.info("前端可访问的资源路径,{}",url); + return url; + } + if (uploadStrategyConfig.equals("oss")) { + try { + ossClient.putObject( + OSS_BUCKET, // 仓库名 + filePathName, // 文件名（含路径） + file.getInputStream() // 数据流 + ); + } catch (OSSException oe) { + log.error("OSS出错了:" + oe.getErrorMessage()); + throw oe; + } catch (ClientException ce) { + log.error("OSS连接出错了:" + ce.getMessage()); + throw ce; + } + return OSS_BUCKET_URL + filePathName; + }*/ + return url; + } + + /** + * 往阿里云对象存储上传单个视频，简单上传 + * + * @param file 视频文件 + * @return 视频的URL地址 + * @throws IOException + */ + /* public String uploadVideo(@NonNull MultipartFile file) throws IOException { + // 生成文件名 + String originalFilename = file.getOriginalFilename(); // 获取原文件名 + String ext = "." + FilenameUtils.getExtension(originalFilename); // 获取文件后缀 + String uuid = System.currentTimeMillis() + UUID.randomUUID().toString().replace("-", ""); + String fileName = uuid + ext; + // 完整路径名 + String date = new SimpleDateFormat("yyyy-MM-dd").format(new Date()).replace("-", ""); + String filePathName = date + "/video/" + fileName; + try { + ossClient.putObject( + OSS_BUCKET, // 仓库名 + filePathName, // 文件名（含路径） + file.getInputStream() // 数据流 + ); + } catch (OSSException oe) { + log.error("OSS出错了:" + oe.getErrorMessage()); + throw oe; + } catch (ClientException ce) { + log.error("OSS连接出错了:" + ce.getMessage()); + throw ce; + } + return OSS_BUCKET_URL + filePathName; + }*/ + + /** + * 将本地的视频分片文件追加合并上传到OSS + * + * @param hash 视频的hash值，用于检索对应分片 + * @return 视频在OSS的URL地址 + * @throws IOException + */ + public String appendUploadVideo(@NonNull String hash) throws IOException { + // 生成文件名 +// String uuid = System.currentTimeMillis() + UUID.randomUUID().toString().replace("-", ""); + String fileName = hash + ".mp4"; +// String date = new SimpleDateFormat("yyyy-MM-dd").format(new Date()).replace("-", ""); +// String filePathName = date + "/video/" + fileName; + String folderName = FilePathEnum.VIDEO.getFilePath() + "/video/"; + int totalSlice = 0;//总分片数量 + while (true) { + File chunkFile = new File(CHUNK_DIRECTORY + hash + "-" + totalSlice); + if (!chunkFile.exists()) { + if (totalSlice == 0) { + log.error("没找到任何相关分片文件"); + return null; + } + break; + } + totalSlice++; + } + int chunkIndex = 0; + Long position = 0L; // 追加位置 + String url = ""; + while (true) { + Object s = redisUtil.getValue(folderName + fileName); + if (null == s || -1 == Long.parseLong(s.toString())) { + redisUtil.setExValue(folderName + fileName, position.toString(), 5 * 60, TimeUnit.SECONDS); + } + File chunkFile = new File(CHUNK_DIRECTORY + hash + "-" + chunkIndex); + if (!chunkFile.exists()) { + if (chunkIndex == 0) { + log.error("没找到任何相关分片文件"); + return null; + } + break; + } + FileInputStream input = new FileInputStream(chunkFile); + MultipartFile multipartFile = new MockMultipartFile("file", chunkFile.getName(), "application/octet" + "-stream", input); + // 上传分片文件 + UploadFileDto uploadInfo = new UploadFileDto(); + uploadInfo.setUploadPath(folderName); + uploadInfo.setFileName(fileName); + uploadInfo.setNeedSlice(1); + uploadInfo.setIndexSlice(chunkIndex); + uploadInfo.setTotalSlice(totalSlice); + uploadInfo.setPosition(position); + uploadInfo.setFileMd5(hash); + uploadInfo.setLocalFile(chunkFile); + uploadInfo.setFile(multipartFile); + uploadInfo.setFileInputStream(multipartFile.getInputStream()); + url = uploadStrategyContext.executeUploadStrategy(uploadInfo); + chunkFile.delete(); // 上传完后删除分片 + chunkIndex++; + } + //阿里云专用，需要传每次的位置去合并（下一次合并依赖上一次的位置） + redisUtil.setExValue(folderName + fileName, -1L, 5 * 60, TimeUnit.SECONDS); + return url; + } + +/* + private String appendUploadVideoAliYunOss(String hash) throws IOException { + // 生成文件名 + String uuid = System.currentTimeMillis() + UUID.randomUUID().toString().replace("-", ""); + String fileName = uuid + ".mp4"; + // 完整路径名 + String date = new SimpleDateFormat("yyyy-MM-dd").format(new Date()).replace("-", ""); + String filePathName = date + "/video/" + fileName; + ObjectMetadata meta = new ObjectMetadata(); + // 设置内容类型为MP4视频 + meta.setContentType("video/mp4"); + int chunkIndex = 0; + long position = 0; // 追加位置 + while (true) { + File chunkFile = new File(CHUNK_DIRECTORY + hash + "-" + chunkIndex); + if (!chunkFile.exists()) { + if (chunkIndex == 0) { + log.error("没找到任何相关分片文件"); + return null; + } + break; + } + // 读取分片数据 + FileInputStream fis = new FileInputStream(chunkFile); + byte[] buffer = new byte[(int) chunkFile.length()]; + fis.read(buffer); + fis.close(); + // 追加上传分片数据 + try { + AppendObjectRequest appendObjectRequest = new AppendObjectRequest(OSS_BUCKET, filePathName, new ByteArrayInputStream(buffer), meta); + appendObjectRequest.setPosition(position); + AppendObjectResult appendObjectResult = ossClient.appendObject(appendObjectRequest); + position = appendObjectResult.getNextPosition(); + } catch (OSSException oe) { + log.error("OSS出错了:" + oe.getErrorMessage()); + throw oe; + } catch (ClientException ce) { + log.error("OSS连接出错了:" + ce.getMessage()); + throw ce; + } + chunkFile.delete(); // 上传完后删除分片 + chunkIndex++; + } + return OSS_BUCKET_URL + filePathName; + } +*/ + +/* + private String appendUploadVideoMinioOss(String hash)throws IOException { + // 生成文件名 + String uuid = System.currentTimeMillis() + UUID.randomUUID().toString().replace("-", ""); + String fileName = uuid + ".mp4"; + // 完整路径名 + String date = new SimpleDateFormat("yyyy-MM-dd").format(new Date()).replace("-", ""); + String filePathName = date + "/video/" + fileName; + ObjectMetadata meta = new ObjectMetadata(); + // 设置内容类型为MP4视频 + meta.setContentType("video/mp4"); + int chunkIndex = 0; + long position = 0; // 追加位置 + while (true) { + File chunkFile = new File(CHUNK_DIRECTORY + hash + "-" + chunkIndex); + if (!chunkFile.exists()) { + if (chunkIndex == 0) { + log.error("没找到任何相关分片文件"); + return null; + } + break; + } + String folderName = date + "/video/"; + FileInputStream input = new FileInputStream(chunkFile); + MultipartFile multipartFile = new MockMultipartFile("file", + chunkFile.getName(), "application/octet" + "-stream", input); + // 上传分片文件 + Boolean isUpload = ossTemplate.uploadFileNew("teriteri", hash, chunkIndex, multipartFile); + + // 读取分片数据 + InputStream fis = new FileInputStream(chunkFile); + byte[] buffer = new byte[(int) chunkFile.length()]; + fis.read(buffer); + fis.close(); + // 追加上传分片数据 + try { + InputStream byteArrayInputStream = new ByteArrayInputStream(buffer); + //路径 + String folderName = date + "/video/"; + OssFile ossFile = ossTemplate.upLoadFile(folderName, fileName, null, byteArrayInputStream); + log.error("视频上传MINIO{}", ossFile); + } catch (Exception oe) { + log.error("视频上传MINIO出错了:" + oe); + throw oe; + } + + chunkFile.delete(); // 上传完后删除分片 + chunkIndex++; + } + //合并 + List sourceObjectList = Stream.iterate(0, i -> ++i) + .limit(chunkIndex) + .map(i -> ComposeSource.builder() + .bucket("teriteri") + .object(hash.concat("/").concat(Integer.toString(i))) + .build()) + .collect(Collectors.toList()); + ossTemplate.composeFileNew("teriteri", filePathName, sourceObjectList); + //删除临时分片 + + return minioEndpoint + "/" + minioBucketName + "/" + filePathName; + } + +*/ + + /** + * 往阿里云对象存储上传单个视频分片文件，在阿里云上设置了3天的生命周期，超过三天未合并使用的分片将被自动删除 + * 建议只在分布式系统使用，单体系统还是存在本地更好存取以及节省流量 + * + * @param file 分片文件 + * @param name 分片名，以 hash-index 命名 + * @return 是否上传成功，一般返回false是因为分片已存在 + * @throws IOException + */ + /*public boolean uploadChunk(@NonNull MultipartFile file, @NonNull String name) throws IOException { + String fileName = "chunk/" + name; // 分片文件在OSS的存储路径名 + boolean success = false; + try { + // 判断文件是否存在 + boolean found = ossClient.doesObjectExist(OSS_BUCKET, fileName); + if (!found) { + // 不存在才上传 + ossClient.putObject(OSS_BUCKET, fileName, file.getInputStream()); + success = true; // 上传成功 + } + } catch (OSSException oe) { + log.error("OSS出错了:" + oe.getErrorMessage()); + throw oe; + } catch (ClientException ce) { + log.error("OSS连接出错了:" + ce.getMessage()); + throw ce; + } + return success; + }*/ + + /** + * 查询指定目录下，指定前缀的文件数量，阿里云限制了单次查询最多100条 + * + * @param prefix 要筛选的文件名前缀，包括目录路径，如果为空字符串，则查询全部文件 + * @return 指定目录下，指定前缀的文件数量 + */ + public int countFiles(@NonNull String prefix) { + int count = 0; + /* try { + ListObjectsRequest listObjectsRequest = new ListObjectsRequest(OSS_BUCKET); + listObjectsRequest.setPrefix(prefix); + ObjectListing objectListing = ossClient.listObjects(listObjectsRequest); + List objectSummaries = objectListing.getObjectSummaries(); + count = objectSummaries.size(); + } catch (OSSException oe) { + log.error("OSS出错了:" + oe.getErrorMessage()); + throw oe; + } catch (ClientException ce) { + log.error("OSS连接出错了:" + ce.getMessage()); + throw ce; + }*/ + return count; + } + + /** + * 删除指定目录下指定前缀的所有文件，不允许目录和前缀都是空字符串 + * + * @param prefix 要筛选的文件名前缀，包括目录路径，不允许为空字符串 + */ + public void deleteFiles(@NonNull String prefix) { + /* if (prefix.equals("")) { + log.warn("你正试图删除整个bucket，已拒绝该危险操作"); + return; + } + try { + // 列举所有包含指定前缀的文件并删除。 + String nextMarker = null; + ObjectListing objectListing; + do { + ListObjectsRequest listObjectsRequest = new ListObjectsRequest(OSS_BUCKET).withPrefix(prefix).withMarker(nextMarker); + objectListing = ossClient.listObjects(listObjectsRequest); + if (objectListing.getObjectSummaries().size() > 0) { + List keys = new ArrayList<>(); + for (OSSObjectSummary s : objectListing.getObjectSummaries()) { +// System.out.println("key name: " + s.getKey()); + keys.add(s.getKey()); + } + DeleteObjectsRequest deleteObjectsRequest = new DeleteObjectsRequest(OSS_BUCKET).withKeys(keys).withEncodingType("url"); + DeleteObjectsResult deleteObjectsResult = ossClient.deleteObjects(deleteObjectsRequest); + List deletedObjects = deleteObjectsResult.getDeletedObjects(); + try { + for (String obj : deletedObjects) { + String deleteObj = URLDecoder.decode(obj, "UTF-8"); +// log.info("删除文件：" + deleteObj); + } + } catch (UnsupportedEncodingException e) { + e.printStackTrace(); + } + } + nextMarker = objectListing.getNextMarker(); + } while (objectListing.isTruncated()); + } catch (OSSException oe) { + log.error("OSS出错了:" + oe.getErrorMessage()); + } catch (ClientException ce) { + log.error("OSS连接出错了:" + ce.getMessage()); + }*/ + } +} diff --git a/src/main/java/com/teriteri/backend/utils/OssUtil.java b/src/main/java/com/teriteri/backend/utils/OssUtil.java deleted file mode 100644 index cddeaaf..0000000 --- a/src/main/java/com/teriteri/backend/utils/OssUtil.java +++ /dev/null @@ -1,246 +0,0 @@ -package com.teriteri.backend.utils; - -import com.aliyun.oss.ClientException; -import com.aliyun.oss.OSS; -import com.aliyun.oss.OSSException; -import com.aliyun.oss.model.*; -import lombok.NonNull; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.io.FilenameUtils; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.stereotype.Component; -import org.springframework.web.multipart.MultipartFile; - -import java.io.*; -import java.net.URLDecoder; -import java.text.SimpleDateFormat; -import java.util.ArrayList; -import java.util.Date; -import java.util.List; -import java.util.UUID; - -@Slf4j -@Component -public class OssUtil { - @Value("${oss.bucket}") - private String OSS_BUCKET; - - @Value("${oss.bucketUrl}") - private String OSS_BUCKET_URL; - - @Value("${directory.chunk}") - private String CHUNK_DIRECTORY; // 分片存储目录 - - @Autowired - private OSS ossClient; - - /** - * 往阿里云对象存储上传单张图片 - * @param file 图片文件 - * @param type 图片分类，如 cover、carousel、other等，不允许空字符串，这里没有做判断了，自己注意就好 - * @return 图片的URL地址 - * @throws IOException - */ - public String uploadImage(@NonNull MultipartFile file, @NonNull String type) throws IOException { - // 生成文件名 - String originalFilename = file.getOriginalFilename(); // 获取原文件名 - String ext = "." + FilenameUtils.getExtension(originalFilename); // 获取文件后缀 - String uuid = System.currentTimeMillis() + UUID.randomUUID().toString().replace("-", ""); - String fileName = uuid + ext; - // 完整路径名 - String date = new SimpleDateFormat("yyyy-MM-dd").format(new Date()).replace("-", ""); - String filePathName = date + "/img/" + type + "/" + fileName; - try { - ossClient.putObject( - OSS_BUCKET, // 仓库名 - filePathName, // 文件名（含路径） - file.getInputStream() // 数据流 - ); - } catch (OSSException oe) { - log.error("OSS出错了:" + oe.getErrorMessage()); - throw oe; - } catch (ClientException ce) { - log.error("OSS连接出错了:" + ce.getMessage()); - throw ce; - } - return OSS_BUCKET_URL + filePathName; - } - - /** - * 往阿里云对象存储上传单个视频，简单上传 - * @param file 视频文件 - * @return 视频的URL地址 - * @throws IOException - */ - public String uploadVideo(@NonNull MultipartFile file) throws IOException { - // 生成文件名 - String originalFilename = file.getOriginalFilename(); // 获取原文件名 - String ext = "." + FilenameUtils.getExtension(originalFilename); // 获取文件后缀 - String uuid = System.currentTimeMillis() + UUID.randomUUID().toString().replace("-", ""); - String fileName = uuid + ext; - // 完整路径名 - String date = new SimpleDateFormat("yyyy-MM-dd").format(new Date()).replace("-", ""); - String filePathName = date + "/video/" + fileName; - try { - ossClient.putObject( - OSS_BUCKET, // 仓库名 - filePathName, // 文件名（含路径） - file.getInputStream() // 数据流 - ); - } catch (OSSException oe) { - log.error("OSS出错了:" + oe.getErrorMessage()); - throw oe; - } catch (ClientException ce) { - log.error("OSS连接出错了:" + ce.getMessage()); - throw ce; - } - return OSS_BUCKET_URL + filePathName; - } - - /** - * 将本地的视频分片文件追加合并上传到OSS - * @param hash 视频的hash值，用于检索对应分片 - * @return 视频在OSS的URL地址 - * @throws IOException - */ - public String appendUploadVideo(@NonNull String hash) throws IOException { - // 生成文件名 - String uuid = System.currentTimeMillis() + UUID.randomUUID().toString().replace("-", ""); - String fileName = uuid + ".mp4"; - // 完整路径名 - String date = new SimpleDateFormat("yyyy-MM-dd").format(new Date()).replace("-", ""); - String filePathName = date + "/video/" + fileName; - ObjectMetadata meta = new ObjectMetadata(); - // 设置内容类型为MP4视频 - meta.setContentType("video/mp4"); - int chunkIndex = 0; - long position = 0; // 追加位置 - while (true) { - File chunkFile = new File(CHUNK_DIRECTORY + hash + "-" + chunkIndex); - if (!chunkFile.exists()) { - if (chunkIndex == 0) { - log.error("没找到任何相关分片文件"); - return null; - } - break; - } - // 读取分片数据 - FileInputStream fis = new FileInputStream(chunkFile); - byte[] buffer = new byte[(int) chunkFile.length()]; - fis.read(buffer); - fis.close(); - // 追加上传分片数据 - try { - AppendObjectRequest appendObjectRequest = new AppendObjectRequest(OSS_BUCKET, filePathName, new ByteArrayInputStream(buffer), meta); - appendObjectRequest.setPosition(position); - AppendObjectResult appendObjectResult = ossClient.appendObject(appendObjectRequest); - position = appendObjectResult.getNextPosition(); - } catch (OSSException oe) { - log.error("OSS出错了:" + oe.getErrorMessage()); - throw oe; - } catch (ClientException ce) { - log.error("OSS连接出错了:" + ce.getMessage()); - throw ce; - } - chunkFile.delete(); // 上传完后删除分片 - chunkIndex++; - } - return OSS_BUCKET_URL + filePathName; - } - - /** - * 往阿里云对象存储上传单个视频分片文件，在阿里云上设置了3天的生命周期，超过三天未合并使用的分片将被自动删除 - * 建议只在分布式系统使用，单体系统还是存在本地更好存取以及节省流量 - * @param file 分片文件 - * @param name 分片名，以 hash-index 命名 - * @return 是否上传成功，一般返回false是因为分片已存在 - * @throws IOException - */ - public boolean uploadChunk(@NonNull MultipartFile file, @NonNull String name) throws IOException { - String fileName = "chunk/" + name; // 分片文件在OSS的存储路径名 - boolean success = false; - try { - // 判断文件是否存在 - boolean found = ossClient.doesObjectExist(OSS_BUCKET, fileName); - if (!found) { - // 不存在才上传 - ossClient.putObject(OSS_BUCKET, fileName, file.getInputStream()); - success = true; // 上传成功 - } - } catch (OSSException oe) { - log.error("OSS出错了:" + oe.getErrorMessage()); - throw oe; - } catch (ClientException ce) { - log.error("OSS连接出错了:" + ce.getMessage()); - throw ce; - } - return success; - } - - /** - * 查询指定目录下，指定前缀的文件数量，阿里云限制了单次查询最多100条 - * @param prefix 要筛选的文件名前缀，包括目录路径，如果为空字符串，则查询全部文件 - * @return 指定目录下，指定前缀的文件数量 - */ - public int countFiles(@NonNull String prefix) { - int count = 0; - try { - ListObjectsRequest listObjectsRequest = new ListObjectsRequest(OSS_BUCKET); - listObjectsRequest.setPrefix(prefix); - ObjectListing objectListing = ossClient.listObjects(listObjectsRequest); - List objectSummaries = objectListing.getObjectSummaries(); - count = objectSummaries.size(); - } catch (OSSException oe) { - log.error("OSS出错了:" + oe.getErrorMessage()); - throw oe; - } catch (ClientException ce) { - log.error("OSS连接出错了:" + ce.getMessage()); - throw ce; - } - return count; - } - - /** - * 删除指定目录下指定前缀的所有文件，不允许目录和前缀都是空字符串 - * @param prefix 要筛选的文件名前缀，包括目录路径，不允许为空字符串 - */ - public void deleteFiles(@NonNull String prefix) { - if (prefix.equals("")) { - log.warn("你正试图删除整个bucket，已拒绝该危险操作"); - return; - } - try { - // 列举所有包含指定前缀的文件并删除。 - String nextMarker = null; - ObjectListing objectListing; - do { - ListObjectsRequest listObjectsRequest = new ListObjectsRequest(OSS_BUCKET).withPrefix(prefix).withMarker(nextMarker); - objectListing = ossClient.listObjects(listObjectsRequest); - if (objectListing.getObjectSummaries().size() > 0) { - List keys = new ArrayList<>(); - for (OSSObjectSummary s : objectListing.getObjectSummaries()) { -// System.out.println("key name: " + s.getKey()); - keys.add(s.getKey()); - } - DeleteObjectsRequest deleteObjectsRequest = new DeleteObjectsRequest(OSS_BUCKET).withKeys(keys).withEncodingType("url"); - DeleteObjectsResult deleteObjectsResult = ossClient.deleteObjects(deleteObjectsRequest); - List deletedObjects = deleteObjectsResult.getDeletedObjects(); - try { - for(String obj : deletedObjects) { - String deleteObj = URLDecoder.decode(obj, "UTF-8"); -// log.info("删除文件：" + deleteObj); - } - } catch (UnsupportedEncodingException e) { - e.printStackTrace(); - } - } - nextMarker = objectListing.getNextMarker(); - } while (objectListing.isTruncated()); - } catch (OSSException oe) { - log.error("OSS出错了:" + oe.getErrorMessage()); - } catch (ClientException ce) { - log.error("OSS连接出错了:" + ce.getMessage()); - } - } -} diff --git a/src/main/java/com/teriteri/backend/utils/RedisUtil.java b/src/main/java/com/teriteri/backend/utils/RedisUtil.java index b755964..d4bfb0f 100644 --- a/src/main/java/com/teriteri/backend/utils/RedisUtil.java +++ b/src/main/java/com/teriteri/backend/utils/RedisUtil.java @@ -674,4 +674,4 @@ public void setExValueForWeekend(String key, Object value) { redisTemplate.opsForValue().set(key, value, remainTime, TimeUnit.SECONDS); } -} \ No newline at end of file +} diff --git a/src/test/java/com/teriteri/backend/BackendApplicationTests.java b/src/test/java/com/teriteri/backend/BackendApplicationTests.java index cefd8aa..ba8dc81 100644 --- a/src/test/java/com/teriteri/backend/BackendApplicationTests.java +++ b/src/test/java/com/teriteri/backend/BackendApplicationTests.java @@ -1,10 +1,8 @@ package com.teriteri.backend; import co.elastic.clients.elasticsearch.ElasticsearchClient; -import co.elastic.clients.elasticsearch._types.query_dsl.Query; import co.elastic.clients.elasticsearch.core.*; import co.elastic.clients.elasticsearch.core.bulk.BulkOperation; -import co.elastic.clients.elasticsearch.core.search.Hit; import co.elastic.clients.elasticsearch.indices.CreateIndexRequest; import co.elastic.clients.elasticsearch.indices.CreateIndexResponse; import com.alibaba.druid.pool.DruidDataSource; @@ -14,14 +12,13 @@ import com.teriteri.backend.pojo.*; import com.teriteri.backend.service.video.VideoStatsService; import com.teriteri.backend.utils.ESUtil; -import com.teriteri.backend.utils.OssUtil; +import com.teriteri.backend.utils.FileUploadUtil; import com.teriteri.backend.utils.RedisUtil; import org.junit.jupiter.api.Test; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.context.SpringBootTest; import org.springframework.data.redis.core.RedisTemplate; import org.springframework.mock.web.MockMultipartFile; -import org.springframework.security.core.parameters.P; import org.springframework.web.multipart.MultipartFile; import javax.sql.DataSource; @@ -36,7 +33,7 @@ import java.util.List; import java.util.Set; -@SpringBootTest(properties = {"spring.profiles.active=test"}) +@SpringBootTest//(properties = {"spring.profiles.active=test"}) class ApplicationTests { @Autowired DataSource dataSource; @@ -57,7 +54,7 @@ class ApplicationTests { private VideoStatsService videoStatsService; @Autowired - private OssUtil ossUtil; + private FileUploadUtil fileUploadUtil; @Autowired private ElasticsearchClient client; @@ -117,18 +114,18 @@ void ossUploadImg() throws IOException { FileInputStream fileInputStream = new FileInputStream(file); MultipartFile multipartFile = new MockMultipartFile(file.getName(), file.getName(), "application/sql", fileInputStream); - String url = ossUtil.uploadImage(multipartFile, "cover"); + String url = fileUploadUtil.uploadImage(multipartFile, "cover"); System.out.println(url); } @Test void ossCountFiles() { - System.out.println(ossUtil.countFiles("img/cover/1696")); + System.out.println(fileUploadUtil.countFiles("img/cover/1696")); } @Test void ossdeleteFiles() { - ossUtil.deleteFiles("img/cover/1696"); + fileUploadUtil.deleteFiles("img/cover/1696"); } @Test @@ -243,4 +240,4 @@ void testFormatString() { System.out.println(formattedString); System.out.println(countChineseAndLetters(formattedString)); } -} \ No newline at end of file +} diff --git a/src/main/resources/application "b/\344\272\214\345\274\200.md" similarity index 71% rename from src/main/resources/application rename to "\344\272\214\345\274\200.md" index d2482ef..75b86e9 100644 --- a/src/main/resources/application +++ "b/\344\272\214\345\274\200.md" @@ -1,3 +1,13 @@ +新增Minio的上传 + +文件分片上传参考教程 +https://www.bilibili.com/video/BV1vC4y1X7UR/?spm_id_from=333.337.search-card.all.click&vd_source=cdf8e8dced2322e34a0f11dac70ec316 + +demo +https://github.com/dufGIT/minio-file/ + +配置文件 +```sql server: port: 7070 tomcat: @@ -13,9 +23,9 @@ spring: type: com.alibaba.druid.pool.DruidDataSource driver-class-name: com.mysql.cj.jdbc.Driver # 使用Unicode字符集、指定字符编码为UTF-8、禁用SSL连接、允许多个查询在一次请求中执行 - url: jdbc:mysql://***.***.***.***:3306/teriteri?serverTimezone=Asia/Shanghai&useUnicode=true&characterEncoding=utf-8&useSSL=false&allowMultiQueries=true - username: *** - password: *** + url: jdbc:mysql://82.16.167.47:3306/teriteri?serverTimezone=Asia/Shanghai&useUnicode=true&characterEncoding=utf-8&useSSL=false&allowMultiQueries=true + username: 1234 + password: 1234 # Spring Boot 默认是不注入这些属性值的，需要自己绑定 # druid 数据源专有配置 # 初始建立的连接数，因为连接远程数据库久不使用会失效，所以按回默认的初始0条就好了 @@ -61,10 +71,10 @@ spring: # Redis数据库索引（默认为0） database: 0 # Redis服务器地址 - host: ***.***.***.*** + host: 192.168.56.10 # Redis服务器连接端口 port: 6379 - password: *** + password: jedis: # 连接池是为了避免频繁地创建和销毁Redis连接，以提高性能 pool: @@ -109,21 +119,60 @@ directory: oss: # 对象存储桶的名字 - bucket: *** + bucket: Bucket域名 # 外网访问的域名，记得最后面的"/" - bucketUrl: https://***.oss-cn-***.aliyuncs.com/ + bucketUrl: https://Bucket域名.oss-cn-beijing.aliyuncs.com/ # 地域节点 - endpoint: http://oss-cn-***.aliyuncs.com + endpoint: http://oss-cn-beijing.aliyuncs.com # 有访问权限的用户的 access-key-id - keyId: ****** + keyId: **** # 有访问权限的用户的 access-key-Secret - keySecret: ****** + keySecret: **** # ossClient实例维持的空闲时间，单位毫秒，超过会自动关闭释放资源 idleTimeout: 10000 +upload: + strategy: minio + minio: + name: minio + endpoint: http://127.0.0.1:9005 #对象存储服务的URL http://127.0.0.1:9005/20240713/img/avatar/17208644874846c336bcce7984f7691b39f349281abd9.jpg + accessKey: oSfIPZ0GyfQIQAAez29T #Access key账户 写账号也可以 + secretKey: UPL6JBJHgUsy3sEvtPkKqnwT0Le8svEblAJlLtmP #Secret key密码 + bucketName: teriteri # 桶名称 + bucketNameSlice: teriteri # 分片存储的临时桶 + expire: 7200 # 过期时间 + local: + # nginx映射本地文件路径 + url: http://xxxx.top:8099/ + # 本地文件存储路径（用docker启动的话，路径为docker内部的路径） + path: /usr/local/upload + # oss存储 https://file.oss-cn-beijing.aliyuncs.com/Default.jpg + oss: + # url: http://Bucket域名 + url: https://file.oss-cn-beijing.aliyuncs.com + # endpoint: OSS配置endpoint，需要去掉后缀 + endpoint: oss-cn-beijing.aliyuncs.com + # bucketName: OSS配置bucketName + bucketName: **** + # accessKeyId: OSS配置accessKeyId + accessKeyId: **** + # accesskeySecret: OSS配置accesskeySecret + accesskeySecret: ****** + # cos存储 + cos: + url: https://Bucket域名 + secretId: COS配置secretId + secretKey: COS配置secretKey + region: COS配置region + bucketName: COS配置bucketName + + elasticsearch: - # elasticsearch服务地址 - host: ***.***.***.*** + # elasticsearch服务地址 可以不需要，因为我把es的都注释掉了 + host: 192.168.56.1011 port: 9200 - username: *** - password: *** + username: + password: + + +``` From 7a0046d3160ba73a105c14eeabce3afeeee92539 Mon Sep 17 00:00:00 2001 From: masc <59968873@qq.com> Date: Thu, 25 Jul 2024 01:06:33 +0800 Subject: [PATCH 2/2] =?UTF-8?q?minio=E5=9C=B0=E5=9D=80=E5=8F=AF=E5=8F=98?= =?UTF-8?q?=E6=94=AF=E6=8C=81=E9=85=8D=E7=BD=AE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../backend/controller/TestControlelr.java | 12 +++-- .../backend/mapper/SiteConfigMapper.java | 15 ++++++ .../com/teriteri/backend/pojo/SiteConfig.java | 21 ++++++++ .../service/comment/SiteConfigService.java | 15 ++++++ .../impl/comment/SiteConfigServiceImpl.java | 51 +++++++++++++++++++ .../impl/user/UserAccountServiceImpl.java | 6 ++- .../service/impl/user/UserServiceImpl.java | 8 ++- .../impl/video/FavoriteServiceImpl.java | 6 +++ .../service/impl/video/VideoServiceImpl.java | 26 +++++++++- .../backend/service/utils/CurrentUser.java | 6 ++- .../impl/upload/MinioUploadStrategyImpl.java | 3 +- 11 files changed, 160 insertions(+), 9 deletions(-) create mode 100644 src/main/java/com/teriteri/backend/mapper/SiteConfigMapper.java create mode 100644 src/main/java/com/teriteri/backend/pojo/SiteConfig.java create mode 100644 src/main/java/com/teriteri/backend/service/comment/SiteConfigService.java create mode 100644 src/main/java/com/teriteri/backend/service/impl/comment/SiteConfigServiceImpl.java diff --git a/src/main/java/com/teriteri/backend/controller/TestControlelr.java b/src/main/java/com/teriteri/backend/controller/TestControlelr.java index ae8c3c0..e61f77a 100644 --- a/src/main/java/com/teriteri/backend/controller/TestControlelr.java +++ b/src/main/java/com/teriteri/backend/controller/TestControlelr.java @@ -1,5 +1,7 @@ package com.teriteri.backend.controller; +import com.teriteri.backend.pojo.SiteConfig; +import com.teriteri.backend.service.comment.SiteConfigService; import com.teriteri.backend.utils.FileUploadUtil; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.web.bind.annotation.PostMapping; @@ -19,14 +21,18 @@ public class TestControlelr { @Autowired private FileUploadUtil fileUploadUtil; + @Autowired + private SiteConfigService siteConfigService; @PostMapping("/test/testUpload") - public String testController(@RequestBody Map map) throws IOException { + public Object testController(@RequestBody Map map) throws IOException { String hash = map.get("hash"); // 合并到OSS，并返回URL地址 - String url = fileUploadUtil.appendUploadVideo(hash); +// String url = fileUploadUtil.appendUploadVideo(hash); + + String sysConfig = siteConfigService.getFileServiceDomain(); - return url; + return sysConfig; } diff --git a/src/main/java/com/teriteri/backend/mapper/SiteConfigMapper.java b/src/main/java/com/teriteri/backend/mapper/SiteConfigMapper.java new file mode 100644 index 0000000..fa83e9f --- /dev/null +++ b/src/main/java/com/teriteri/backend/mapper/SiteConfigMapper.java @@ -0,0 +1,15 @@ +package com.teriteri.backend.mapper; + +import com.baomidou.mybatisplus.core.mapper.BaseMapper; +import com.teriteri.backend.pojo.SiteConfig; +import org.apache.ibatis.annotations.Mapper; + +/** + * @author 码头薯条Pro + * @date 2024/7/24 23:40 + *

系统配置参数 + */ +@Mapper +public interface SiteConfigMapper extends BaseMapper { + +} diff --git a/src/main/java/com/teriteri/backend/pojo/SiteConfig.java b/src/main/java/com/teriteri/backend/pojo/SiteConfig.java new file mode 100644 index 0000000..7f4da42 --- /dev/null +++ b/src/main/java/com/teriteri/backend/pojo/SiteConfig.java @@ -0,0 +1,21 @@ +package com.teriteri.backend.pojo; + +import com.baomidou.mybatisplus.annotation.IdType; +import com.baomidou.mybatisplus.annotation.TableId; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +/** + * @author 码头薯条Pro + * @date 2024/7/24 23:35 + *

+ */ +@Data +@AllArgsConstructor +@NoArgsConstructor +public class SiteConfig { + @TableId(type = IdType.AUTO) + private Integer id; // 唯一标识 + private String minio;//minio +} diff --git a/src/main/java/com/teriteri/backend/service/comment/SiteConfigService.java b/src/main/java/com/teriteri/backend/service/comment/SiteConfigService.java new file mode 100644 index 0000000..bfd92f0 --- /dev/null +++ b/src/main/java/com/teriteri/backend/service/comment/SiteConfigService.java @@ -0,0 +1,15 @@ +package com.teriteri.backend.service.comment; + +import com.teriteri.backend.pojo.SiteConfig; + +/** + * @author 码头薯条Pro + * @date 2024/7/24 23:43 + *

公告配置service

+ */ +public interface SiteConfigService { + + SiteConfig getSysConfig(); + + String getFileServiceDomain(); +} diff --git a/src/main/java/com/teriteri/backend/service/impl/comment/SiteConfigServiceImpl.java b/src/main/java/com/teriteri/backend/service/impl/comment/SiteConfigServiceImpl.java new file mode 100644 index 0000000..d6cb8e2 --- /dev/null +++ b/src/main/java/com/teriteri/backend/service/impl/comment/SiteConfigServiceImpl.java @@ -0,0 +1,51 @@ +package com.teriteri.backend.service.impl.comment; + +import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; +import com.teriteri.backend.mapper.SiteConfigMapper; +import com.teriteri.backend.pojo.Category; +import com.teriteri.backend.pojo.SiteConfig; +import com.teriteri.backend.service.comment.SiteConfigService; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Service; + +import java.util.List; + +/** + * @author 码头薯条Pro + * @date 2024/7/24 23:45 + *

公告配置service

+ */ +@Slf4j +@Service +public class SiteConfigServiceImpl implements SiteConfigService { + @Autowired + private SiteConfigMapper siteConfigMapper; + /** + * 上传模式 + */ + @Value("${upload.strategy}") + private String uploadStrategy; + + @Override + public SiteConfig getSysConfig() { + QueryWrapper queryWrapper = new QueryWrapper<>(); + List configList = siteConfigMapper.selectList(queryWrapper); + return configList.get(0); + } + + @Override + public String getFileServiceDomain() { + if ("oss".equals(uploadStrategy)) { + return ""; + } + if ("minio".equals(uploadStrategy)) { + QueryWrapper queryWrapper = new QueryWrapper<>(); + List configList = siteConfigMapper.selectList(queryWrapper); + SiteConfig siteConfig = configList.get(0); + return siteConfig.getMinio(); + } + return null; + } +} diff --git a/src/main/java/com/teriteri/backend/service/impl/user/UserAccountServiceImpl.java b/src/main/java/com/teriteri/backend/service/impl/user/UserAccountServiceImpl.java index 4517c9c..2613f2e 100644 --- a/src/main/java/com/teriteri/backend/service/impl/user/UserAccountServiceImpl.java +++ b/src/main/java/com/teriteri/backend/service/impl/user/UserAccountServiceImpl.java @@ -11,6 +11,7 @@ import com.teriteri.backend.pojo.MsgUnread; import com.teriteri.backend.pojo.User; import com.teriteri.backend.pojo.dto.UserDTO; +import com.teriteri.backend.service.comment.SiteConfigService; import com.teriteri.backend.service.user.UserAccountService; import com.teriteri.backend.service.user.UserService; import com.teriteri.backend.service.utils.CurrentUser; @@ -67,7 +68,8 @@ public class UserAccountServiceImpl implements UserAccountService { @Autowired private AuthenticationProvider authenticationProvider; - + @Autowired + private SiteConfigService siteConfigService; @Autowired @Qualifier("taskExecutor") private Executor taskExecutor; @@ -343,6 +345,8 @@ public CustomResponse adminPersonalInfo() { // 如果redis中没有user数据，就从mysql中获取并更新到redis if (user == null) { user = userMapper.selectById(LoginUserId); + String fileServiceDomain = siteConfigService.getFileServiceDomain(); + user.setAvatar(fileServiceDomain + "" + user.getAvatar()); User finalUser = user; CompletableFuture.runAsync(() -> { redisUtil.setExObjectValue("user:" + finalUser.getUid(), finalUser); // 默认存活1小时 diff --git a/src/main/java/com/teriteri/backend/service/impl/user/UserServiceImpl.java b/src/main/java/com/teriteri/backend/service/impl/user/UserServiceImpl.java index ec97037..e5f6dcd 100644 --- a/src/main/java/com/teriteri/backend/service/impl/user/UserServiceImpl.java +++ b/src/main/java/com/teriteri/backend/service/impl/user/UserServiceImpl.java @@ -7,6 +7,7 @@ import com.teriteri.backend.pojo.User; import com.teriteri.backend.pojo.VideoStats; import com.teriteri.backend.pojo.dto.UserDTO; +import com.teriteri.backend.service.comment.SiteConfigService; import com.teriteri.backend.service.user.UserService; import com.teriteri.backend.service.video.VideoStatsService; import com.teriteri.backend.utils.ESUtil; @@ -49,7 +50,8 @@ public class UserServiceImpl implements UserService { @Value("${upload.oss.bucketName}") private String OSS_BUCKET_URL; - + @Autowired + private SiteConfigService siteConfigService; @Autowired @Qualifier("taskExecutor") private Executor taskExecutor; @@ -66,6 +68,8 @@ public UserDTO getUserById(Integer id) { // 如果redis中没有user数据，就从mysql中获取并更新到redis if (user == null) { user = userMapper.selectById(id); + String fileServiceDomain = siteConfigService.getFileServiceDomain(); + user.setAvatar(fileServiceDomain + "" + user.getAvatar()); if (user == null) { return null; // 如果uid不存在则返回空 } @@ -230,6 +234,8 @@ public CustomResponse updateUserAvatar(Integer uid, MultipartFile file) throws I String avatar_url = fileUploadUtil.uploadImage(file, "avatar"); // 查旧的头像地址 User user = userMapper.selectById(uid); + String fileServiceDomain = siteConfigService.getFileServiceDomain(); + user.setAvatar(fileServiceDomain + "" + user.getAvatar()); // 先更新数据库 UpdateWrapper updateWrapper = new UpdateWrapper<>(); updateWrapper.eq("uid", uid).set("avatar", avatar_url); diff --git a/src/main/java/com/teriteri/backend/service/impl/video/FavoriteServiceImpl.java b/src/main/java/com/teriteri/backend/service/impl/video/FavoriteServiceImpl.java index 1c913ce..b8edc88 100644 --- a/src/main/java/com/teriteri/backend/service/impl/video/FavoriteServiceImpl.java +++ b/src/main/java/com/teriteri/backend/service/impl/video/FavoriteServiceImpl.java @@ -7,6 +7,7 @@ import com.teriteri.backend.mapper.VideoMapper; import com.teriteri.backend.pojo.Favorite; import com.teriteri.backend.pojo.Video; +import com.teriteri.backend.service.comment.SiteConfigService; import com.teriteri.backend.service.video.FavoriteService; import com.teriteri.backend.utils.RedisUtil; import org.apache.ibatis.session.ExecutorType; @@ -38,6 +39,8 @@ public class FavoriteServiceImpl implements FavoriteService { @Autowired @Qualifier("taskExecutor") private Executor taskExecutor; + @Autowired + private SiteConfigService siteConfigService; @Override public List getFavorites(Integer uid, boolean isOwner) { @@ -69,6 +72,9 @@ public List getFavorites(Integer uid, boolean isOwner) { if (set != null && set.size() > 0) { Integer vid = (Integer) set.iterator().next(); Video video = videoMapper.selectById(vid); + String fileServiceDomain = siteConfigService.getFileServiceDomain(); + video.setVideoUrl(fileServiceDomain + "" + video.getVideoUrl()); + video.setCoverUrl(fileServiceDomain + "" + video.getCoverUrl()); favorite.setCover(video.getCoverUrl()); } } diff --git a/src/main/java/com/teriteri/backend/service/impl/video/VideoServiceImpl.java b/src/main/java/com/teriteri/backend/service/impl/video/VideoServiceImpl.java index 46788b3..16bd416 100644 --- a/src/main/java/com/teriteri/backend/service/impl/video/VideoServiceImpl.java +++ b/src/main/java/com/teriteri/backend/service/impl/video/VideoServiceImpl.java @@ -8,6 +8,7 @@ import com.teriteri.backend.pojo.Video; import com.teriteri.backend.pojo.VideoStats; import com.teriteri.backend.service.category.CategoryService; +import com.teriteri.backend.service.comment.SiteConfigService; import com.teriteri.backend.service.user.UserService; import com.teriteri.backend.service.utils.CurrentUser; import com.teriteri.backend.service.video.VideoService; @@ -69,7 +70,8 @@ public class VideoServiceImpl implements VideoService { @Autowired @Qualifier("taskExecutor") private Executor taskExecutor; - + @Autowired + private SiteConfigService siteConfigService; /** * 根据id分页获取视频信息，包括用户和分区信息 * @param set 要查询的视频id集合 @@ -111,6 +113,9 @@ public List> getVideosWithDataByIds(Set set, Integer // long start = System.currentTimeMillis(); // System.out.println("================ 开始查询 " + video.getVid() + " 号视频相关信息 =============== 当前时间 " + start); Map map = new HashMap<>(); + String fileServiceDomain = siteConfigService.getFileServiceDomain(); + video.setVideoUrl(fileServiceDomain + "" + video.getVideoUrl()); + video.setCoverUrl(fileServiceDomain + "" + video.getCoverUrl()); map.put("video", video); CompletableFuture userFuture = CompletableFuture.runAsync(() -> { @@ -159,6 +164,9 @@ public List> getVideosWithDataByIdsOrderByDesc(List .findFirst() .orElse(null); if (video == null) return Stream.empty(); // 跳过该项 + String fileServiceDomain = siteConfigService.getFileServiceDomain(); + video.setVideoUrl(fileServiceDomain + "" + video.getVideoUrl()); + video.setCoverUrl(fileServiceDomain + "" + video.getCoverUrl()); if (video.getStatus() == 3) { // 视频已删除 Video video1 = new Video(); @@ -270,6 +278,9 @@ public Map getVideoWithDataById(Integer vid) { queryWrapper.eq("vid", vid).ne("status", 3); video = videoMapper.selectOne(queryWrapper); if (video != null) { + String fileServiceDomain = siteConfigService.getFileServiceDomain(); + video.setVideoUrl(fileServiceDomain + "" + video.getVideoUrl()); + video.setCoverUrl(fileServiceDomain + "" + video.getCoverUrl()); Video finalVideo1 = video; CompletableFuture.runAsync(() -> { redisUtil.setExObjectValue("video:" + vid, finalVideo1); // 异步更新到redis @@ -316,10 +327,12 @@ public List> getVideosWithDataByIdList(List list) { .filter(v -> Objects.equals(v.getVid(), vid)) .findFirst() .orElse(null); - if (video == null) { return Stream.empty(); // 跳过该项 } + String fileServiceDomain = siteConfigService.getFileServiceDomain(); + video.setVideoUrl(fileServiceDomain + "" + video.getVideoUrl()); + video.setCoverUrl(fileServiceDomain + "" + video.getCoverUrl()); map.put("video", video); CompletableFuture userFuture = CompletableFuture.runAsync(() -> { @@ -366,6 +379,9 @@ public CustomResponse updateVideoStatus(Integer vid, Integer status) throws IOEx customResponse.setMessage("视频不见了QAQ"); return customResponse; } + String fileServiceDomain = siteConfigService.getFileServiceDomain(); + video.setVideoUrl(fileServiceDomain + "" + video.getVideoUrl()); + video.setCoverUrl(fileServiceDomain + "" + video.getCoverUrl()); Integer lastStatus = video.getStatus(); video.setStatus(1); UpdateWrapper