1.审计日志问题排查

pqs-common/common-oss/src/main/java/com/njcn/oss/utils/FileStorageUtil.java

@@ -1,5 +1,6 @@
 package com.njcn.oss.utils;
 
+import cn.hutool.core.io.FileUtil;
 import com.njcn.common.config.GeneralInfo;
 import com.njcn.common.pojo.exception.BusinessException;
 import com.njcn.huawei.obs.util.OBSUtil;
@@ -7,6 +8,7 @@ import com.njcn.minioss.bo.MinIoUploadResDTO;
 import com.njcn.minioss.config.MinIossProperties;
 import com.njcn.minioss.util.MinIoUtils;
 import com.njcn.oss.constant.GeneralConstant;
+import com.njcn.oss.constant.OssPath;
 import com.njcn.oss.enums.OssResponseEnum;
 import io.minio.*;
 import io.minio.errors.*;
@@ -21,9 +23,7 @@ import org.springframework.web.multipart.MultipartFile;
 
 import javax.annotation.Resource;
 import javax.servlet.http.HttpServletResponse;
-import java.io.BufferedOutputStream;
+import java.io.*;
-import java.io.InputStream;
-import java.io.OutputStream;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Optional;
@@ -92,7 +92,7 @@ public class FileStorageUtil {
      * @param dir 服务器文件存放路径
      * @param flag 是否保留原始文件名
      */
-    public String uploadMultipart(MultipartFile multipartFile, String dir,boolean flag) {
+    public String uploadMultipart(MultipartFile multipartFile, String dir, boolean flag) {
         String filePath;
         if (generalInfo.getBusinessFileStorage() == GeneralConstant.HUAWEI_OBS) {
             filePath = dir + minIoUtils.minFileName(multipartFile.getOriginalFilename());
@@ -100,7 +100,7 @@ public class FileStorageUtil {
         } else {
             try {
                 //把名称存入数据
-                MinIoUploadResDTO minIoUploadResDTO = minIoUtils.upload(multipartFile, minIossProperties.getBucket(), dir,flag);
+                MinIoUploadResDTO minIoUploadResDTO = minIoUtils.upload(multipartFile, minIossProperties.getBucket(), dir, flag);
                 filePath = minIoUploadResDTO.getMinFileName();
             } catch (Exception e) {
                 throw new BusinessException(OssResponseEnum.UPLOAD_FILE_ERROR);
@@ -136,6 +136,7 @@ public class FileStorageUtil {
 
     /**
      * 上传InputStream流,并指定文件的名称
+     *
      * @author xuyang
      */
     public String uploadStreamSpecifyName(InputStream inputStream, String dir, String fileName) {
@@ -211,7 +212,7 @@ public class FileStorageUtil {
         } catch (Exception exception) {
             throw new BusinessException(OssResponseEnum.DOWNLOAD_FILE_STREAM_ERROR);
         }
-        String fileType = filePath.substring(filePath.lastIndexOf('.')+1).toLowerCase();
+        String fileType = filePath.substring(filePath.lastIndexOf('.') + 1).toLowerCase();
         switch (fileType) {
             case "jpg":
             case "jpeg":
@@ -264,18 +265,19 @@ public class FileStorageUtil {
         }
         directory = (String) Optional.ofNullable(directory).orElse("");
         String minFileName = directory + fileName;
-        this.instance.putObject((PutObjectArgs)((io.minio.PutObjectArgs.Builder)((io.minio.PutObjectArgs.Builder)PutObjectArgs.builder().bucket(bucketName)).object(minFileName)).stream(inputStream, (long)inputStream.available(), -1L).build());
+        this.instance.putObject((PutObjectArgs) ((io.minio.PutObjectArgs.Builder) ((io.minio.PutObjectArgs.Builder) PutObjectArgs.builder().bucket(bucketName)).object(minFileName)).stream(inputStream, (long) inputStream.available(), -1L).build());
         inputStream.close();
         return new MinIoUploadResDTO(minFileName, minIoUtils.getObjectUrl(bucketName, minFileName, 60));
     }
+
     /**
      * @Description: getAllFile 获取minio指定文件夹下所有文件
      * @Param:
-    * @return: java.lang.Iterable<io.minio.Result<io.minio.messages.Item>>
+     * @return: java.lang.Iterable<io.minio.Result < io.minio.messages.Item>>
      * @Author: clam
      * @Date: 2024/9/27
      */
-    public List<MultipartFile> getAllFile(String bucketName, String directory) throws Exception{
+    public List<MultipartFile> getAllFile(String bucketName, String directory) throws Exception {
         List<MultipartFile> files = new ArrayList<>();
         Iterable<Result<Item>> results = this.instance.listObjects(
                 ListObjectsArgs.builder().bucket(bucketName).prefix(directory).recursive(true).build());
@@ -288,7 +290,6 @@ public class FileStorageUtil {
                     GetObjectArgs.builder().bucket(bucketName).object(item.objectName()).build());
 
 
-
             // 假设你有一个方法可以将InputStream转换为MultipartFile
             MultipartFile multipartFile = new MockMultipartFile(item.objectName(), item.objectName(), "text/plain", stream);
             files.add(multipartFile);
@@ -297,4 +298,38 @@ public class FileStorageUtil {
     }
 
 
+    /**
+     * 目前只给审计日志使用
+     *
+     * @author cdf
+     * @date 2024/10/15
+     */
+    public File getLastFile(String bucketName, String directory) throws Exception {
+        File file = null;
+        if (generalInfo.getBusinessFileStorage() == GeneralConstant.MINIO_OSS) {
+            Iterable<Result<Item>> results = this.instance.listObjects(
+                    ListObjectsArgs.builder().bucket(bucketName).prefix(directory).recursive(true).build());
+            Item lastItem = null;
+            for (Result<Item> result : results) {
+                Item item = result.get();
+                if (lastItem == null) {
+                    lastItem = item;
+                } else {
+                    if (item.lastModified().isAfter(lastItem.lastModified())) {
+                        lastItem = item;
+                    }
+                }
+            }
+            InputStream stream = this.instance.getObject(
+                    GetObjectArgs.builder().bucket(bucketName).object(lastItem.objectName()).build());
+
+            file = File.createTempFile(lastItem.objectName(), "xlsx");
+            FileUtil.writeFromStream(stream, file);
+        }else if(generalInfo.getBusinessFileStorage() == GeneralConstant.HUAWEI_OBS){
+            file = obsUtil.getLastFile(OssPath.LOGBAK);
+
+        }
+        return file;
+    }
+
 }

pqs-common/common-web/src/main/java/com/njcn/web/utils/GwSendUtil.java

@@ -136,8 +136,6 @@ public class GwSendUtil {
             HttpParameters.Builder builder = HttpParameters.newBuilder();
 
             builder.requestURL(sendEnum.getNewUrl()) // 设置请求的URL,可以拼接URL请求参数
-                    .api("zongbuSync") // 设置服务名
-                    .version("1.0.0") // 设置版本号
                     .method("post") // 设置调用方式, 必须为 post
                     .contentType("application/json; charset=UTF-8"); //设置请求content-type
 

pqs-system/system-boot/src/main/java/com/njcn/system/handler/MqttMessageHandler.java

@@ -6,6 +6,7 @@ import cn.hutool.core.util.StrUtil;
 import cn.hutool.json.JSONObject;
 import com.github.tocrhz.mqtt.annotation.MqttSubscribe;
 import com.github.tocrhz.mqtt.annotation.Payload;
+import com.njcn.common.pojo.constant.LogInfo;
 import com.njcn.common.pojo.constant.PatternRegex;
 import com.njcn.common.pojo.dto.LogInfoDTO;
 import com.njcn.common.utils.PubUtils;
@@ -63,7 +64,7 @@ public class MqttMessageHandler {
         }
 
         LogInfoDTO logInfoDTO = PubUtils.json2obj(new String(message.getPayload(), StandardCharsets.UTF_8),LogInfoDTO.class);
-
+        if(!logInfoDTO.getLoginName().equals(LogInfo.UNKNOWN_USER) && !logInfoDTO.getOperate().equals(LogInfo.UNKNOWN_OPERATE) && !logInfoDTO.getIp().equals(LogInfo.UNKNOWN_IP)){
             if("注销".equals(logInfoDTO.getOperateType()) || "认证".equals(logInfoDTO.getOperateType())){
                 logInfoDTO.setType(1);
             }else {
@@ -80,6 +81,7 @@ public class MqttMessageHandler {
             }
             userLogService.addUserLog(logInfoDTO);
         }
+    }
 
     /**
      * 订阅审计日志的记录，并进行入库操作

pqs-system/system-boot/src/main/java/com/njcn/system/service/impl/AuditServiceImpl.java

@@ -34,7 +34,9 @@ import com.njcn.common.pojo.enums.response.CommonResponseEnum;
 import com.njcn.common.pojo.exception.BusinessException;
 import com.njcn.common.utils.PubUtils;
 import com.njcn.db.constant.DbConstant;
+import com.njcn.oss.constant.GeneralConstant;
 import com.njcn.oss.constant.OssPath;
+import com.njcn.oss.utils.FileStorageUtil;
 import com.njcn.redis.utils.RedisUtil;
 import com.njcn.system.enums.SystemResponseEnum;
 import com.njcn.system.excel.UserLogExcel;
@@ -57,8 +59,7 @@ import org.springframework.stereotype.Service;
 import org.springframework.transaction.annotation.Transactional;
 
 import javax.servlet.http.HttpServletRequest;
-import java.io.File;
+import java.io.*;
-import java.io.FileInputStream;
 import java.text.ParseException;
 import java.time.LocalDateTime;
 import java.time.format.DateTimeFormatter;
@@ -94,6 +95,8 @@ public class AuditServiceImpl extends ServiceImpl<UserLogMapper, UserLog> implem
 
     private final MqttPublisher publisher;
 
+    private final FileStorageUtil fileStorageUtil;
+
     @Override
     public Page<AuditLogVO> getAuditLog(AuditParam auditParam) {
         List<AuditLogVO> auditLogVOS = new ArrayList<>();
@@ -206,11 +209,14 @@ public class AuditServiceImpl extends ServiceImpl<UserLogMapper, UserLog> implem
     public void logFileWriter() {
         TimeInterval timeInterval = new TimeInterval();
         //限制多次点击
+        redisUtil.delete("logFileWriter");
         String logFileWriter = redisUtil.getStringByKey("logFileWriter");
         if (StrUtil.isNotBlank(logFileWriter) || ObjectUtil.equals(logFileWriter, "1")) {
             throw new BusinessException(SystemResponseEnum.MULTIPLE_CLICKS_LOG_FILE_WRITER);
         }
         redisUtil.saveByKeyWithExpire("logFileWriter", "1", 600L);
+
+
         Config config = iConfigService.getOne(new LambdaQueryWrapper<Config>().eq(Config::getState, DataStateEnum.ENABLE.getCode()));
         LocalDateTime nowDate = LocalDateTime.now();
         LocalDateTime agoDate = nowDate.minusMonths(config.getLogTime()).with(TemporalAdjusters.firstDayOfMonth());
@@ -223,12 +229,29 @@ public class AuditServiceImpl extends ServiceImpl<UserLogMapper, UserLog> implem
         aa.ge("sys_user_log.update_time", date);
         aa.le("sys_user_log.update_time", endTime);
         String nowTime = date + "至" + endTime;
-        File file = new File(generalInfo.getBusinessTempPath() + File.separator + OssPath.LOGBAK);
+
+        //必须放到循环外，否则会刷新流
+        ExcelWriter excelWriter =null;
+        FileInputStream fileInputStream;
+        File file = null;
+        try {
+            if (generalInfo.getBusinessFileStorage() == GeneralConstant.LOCAL_DISK) {
+                file = new File(generalInfo.getBusinessTempPath() + File.separator + OssPath.LOGBAK);
                 if (!file.exists() && !file.isDirectory()) {
                     file.mkdir();
                 }
-        //必须放到循环外，否则会刷新流
+                excelWriter = EasyExcel.write(generalInfo.getBusinessTempPath() + File.separator + OssPath.LOGBAK + nowTime + ExcelTypeEnum.XLSX.getValue(), UserLogExcel.class).build();
-        ExcelWriter excelWriter = EasyExcel.write(generalInfo.getBusinessTempPath() + File.separator + OssPath.LOGBAK + nowTime + ExcelTypeEnum.XLSX.getValue(), UserLogExcel.class).build();
+            } else if (generalInfo.getBusinessFileStorage() == GeneralConstant.HUAWEI_OBS || generalInfo.getBusinessFileStorage() == GeneralConstant.MINIO_OSS) {
+                file = File.createTempFile(date+"_"+endTime,".xlsx");
+                excelWriter = EasyExcel.write(file, UserLogExcel.class).build();
+            }
+        }catch (Exception e){
+            log.error("获取文件服务器异常");
+            redisUtil.delete("logFileWriter");
+            e.printStackTrace();
+        }
+
+
         try {
             //记录总数:实际中需要根据查询条件进行统计即可:一共多少条
             int totalCount = this.count(aa);
@@ -257,13 +280,24 @@ public class AuditServiceImpl extends ServiceImpl<UserLogMapper, UserLog> implem
                     excelWriter.write(list, writeSheet);
                 }
             }
-
         } catch (Exception e) {
             redisUtil.delete("logFileWriter");
             e.printStackTrace();
         } finally {
             excelWriter.finish();
         }
+
+        if (generalInfo.getBusinessFileStorage() == GeneralConstant.HUAWEI_OBS || generalInfo.getBusinessFileStorage() == GeneralConstant.MINIO_OSS) {
+            try {
+                fileInputStream = new FileInputStream(file);
+                fileStorageUtil.uploadStream(fileInputStream,OssPath.LOGBAK,file.getName());
+            } catch (FileNotFoundException e) {
+                throw new RuntimeException(e);
+            }finally {
+                file.delete();
+            }
+        }
+
         redisUtil.delete("logFileWriter");
         System.out.println("日志备份结束，共花费时间" + timeInterval.intervalSecond() + "S");
     }
@@ -276,12 +310,13 @@ public class AuditServiceImpl extends ServiceImpl<UserLogMapper, UserLog> implem
             throw new BusinessException(SystemResponseEnum.MULTIPLE_CLICKS_LOG_FILE_WRITER);
         }
         redisUtil.saveByKeyWithExpire("recoverLogFile", "1", 600L);
-        File newFile = getLastFile();
+        FileInputStream in = null;
+        File newFile = null;
+        try {
+            newFile = getLastFile();
             if (FileUtil.isEmpty(newFile)) {
                 throw new BusinessException(SystemResponseEnum.NOT_FIND_FILE);
             }
-        FileInputStream in = null;
-        try {
             in = new FileInputStream(newFile);
             Set<String> logIndex = new HashSet<>(this.baseMapper.getAllIndex());
             List<UserLog> userLogs = new ArrayList<>();
@@ -307,6 +342,10 @@ public class AuditServiceImpl extends ServiceImpl<UserLogMapper, UserLog> implem
             throw new BusinessException(SystemResponseEnum.LOG_EXCEPTION);
         } finally {
             IOUtils.closeQuietly(in);
+
+            if(generalInfo.getBusinessFileStorage() == GeneralConstant.HUAWEI_OBS || generalInfo.getBusinessFileStorage() == GeneralConstant.MINIO_OSS) {
+                newFile.deleteOnExit();
+            }
         }
         redisUtil.delete("recoverLogFile");
     }
@@ -430,7 +469,16 @@ public class AuditServiceImpl extends ServiceImpl<UserLogMapper, UserLog> implem
      * 返回一个最新修改日期的文件
      */
     public File getLastFile() {
-        File parentFile = new File(generalInfo.getBusinessTempPath() + File.separator + OssPath.LOGBAK);
+        File parentFile = null;
+        if(generalInfo.getBusinessFileStorage() == GeneralConstant.LOCAL_DISK){
+            parentFile = new File(generalInfo.getBusinessTempPath() + File.separator + OssPath.LOGBAK);
+        }else if(generalInfo.getBusinessFileStorage() == GeneralConstant.HUAWEI_OBS || generalInfo.getBusinessFileStorage() == GeneralConstant.MINIO_OSS){
+            try {
+                parentFile = fileStorageUtil.getLastFile("excelreport",OssPath.LOGBAK);
+            } catch (Exception e) {
+                throw new RuntimeException(e);
+            }
+        }
         //文件夹下的所有子文件数组
         File[] files = parentFile.listFiles();
         List<Date> list;
@@ -439,7 +487,7 @@ public class AuditServiceImpl extends ServiceImpl<UserLogMapper, UserLog> implem
             list.sort(Comparator.reverseOrder());
             return Arrays.stream(files).filter(tempFile -> FileUtil.lastModifiedTime(tempFile).equals(list.get(0))).collect(Collectors.toList()).get(0);
         }
-        return null;
+        return parentFile;
     }
 
     /**

pqs-system/system-boot/src/main/java/com/njcn/system/timer/tasks/PqTypicalSourceTaskRunner.java

@@ -4,6 +4,8 @@ import cn.hutool.core.date.DatePattern;
 import cn.hutool.core.date.DateUtil;
 import com.njcn.common.pojo.exception.BusinessException;
 import com.njcn.harmonic.api.PqTypicalSourceFeignClient;
+import com.njcn.harmonic.api.UploadGwDataFeignClient;
+import com.njcn.harmonic.pojo.param.UploadDataParam;
 import com.njcn.harmonic.pojo.param.UploadParam;
 import com.njcn.system.timer.TimerTaskRunner;
 import lombok.RequiredArgsConstructor;
@@ -20,15 +22,19 @@ public class PqTypicalSourceTaskRunner implements TimerTaskRunner {
 
     private final PqTypicalSourceFeignClient pqTypicalSourceFeignClient;
 
+    private final UploadGwDataFeignClient uploadGwDataFeignClient;
+
 
     @Override
     public void action(String date) {
         UploadParam param=new UploadParam();
-        param.setTime(DateUtil.yesterday().toString(DatePattern.NORM_DATE_PATTERN));
+        String dateDeal =DateUtil.yesterday().toString(DatePattern.NORM_DATE_PATTERN);
-        String code = pqTypicalSourceFeignClient.send(param).getCode();
+        param.setTime(dateDeal);
-        if(code.equals("A0002")){
+        pqTypicalSourceFeignClient.send(param).getCode();
-            throw new BusinessException("失败");
+
-        }
+        UploadDataParam uploadDataParam = new UploadDataParam();
+        uploadDataParam.setSearchBeginTime(dateDeal);
+        uploadGwDataFeignClient.upGwCommPoint(uploadDataParam);
 
     }
 }