Microservice/pqs
5
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

1.审计日志问题排查

Browse Source
This commit is contained in:
cdf
2024-10-16 09:35:13 +08:00
parent 5676c736a5
commit bd153dc7b9
5 changed files with 139 additions and 50 deletions
Download Patch File Download Diff File Expand all files Collapse all files

55
pqs-common/common-oss/src/main/java/com/njcn/oss/utils/FileStorageUtil.java
View File

@@ -1,5 +1,6 @@
package com.njcn.oss.utils;
import cn.hutool.core.io.FileUtil;
import com.njcn.common.config.GeneralInfo;
import com.njcn.common.pojo.exception.BusinessException;
import com.njcn.huawei.obs.util.OBSUtil;
@@ -7,6 +8,7 @@ import com.njcn.minioss.bo.MinIoUploadResDTO;
import com.njcn.minioss.config.MinIossProperties;
import com.njcn.minioss.util.MinIoUtils;
import com.njcn.oss.constant.GeneralConstant;
import com.njcn.oss.constant.OssPath;
import com.njcn.oss.enums.OssResponseEnum;
import io.minio.*;
import io.minio.errors.*;
@@ -21,9 +23,7 @@ import org.springframework.web.multipart.MultipartFile;
import javax.annotation.Resource;
import javax.servlet.http.HttpServletResponse;
import java.io.BufferedOutputStream;
import java.io.InputStream;
import java.io.OutputStream;
import java.io.*;
import java.util.ArrayList;
import java.util.List;
import java.util.Optional;
@@ -92,7 +92,7 @@ public class FileStorageUtil {
* @param dir 服务器文件存放路径
* @param flag 是否保留原始文件名
*/
public String uploadMultipart(MultipartFile multipartFile, String dir,boolean flag) {
public String uploadMultipart(MultipartFile multipartFile, String dir, boolean flag) {
String filePath;
if (generalInfo.getBusinessFileStorage() == GeneralConstant.HUAWEI_OBS) {
filePath = dir + minIoUtils.minFileName(multipartFile.getOriginalFilename());
@@ -100,7 +100,7 @@ public class FileStorageUtil {
} else {
try {
//把名称存入数据
MinIoUploadResDTO minIoUploadResDTO = minIoUtils.upload(multipartFile, minIossProperties.getBucket(), dir,flag);
MinIoUploadResDTO minIoUploadResDTO = minIoUtils.upload(multipartFile, minIossProperties.getBucket(), dir, flag);
filePath = minIoUploadResDTO.getMinFileName();
} catch (Exception e) {
throw new BusinessException(OssResponseEnum.UPLOAD_FILE_ERROR);
@@ -136,6 +136,7 @@ public class FileStorageUtil {
/**
* 上传InputStream流,并指定文件的名称
*
* @author xuyang
*/
public String uploadStreamSpecifyName(InputStream inputStream, String dir, String fileName) {
@@ -211,7 +212,7 @@ public class FileStorageUtil {
} catch (Exception exception) {
throw new BusinessException(OssResponseEnum.DOWNLOAD_FILE_STREAM_ERROR);
}
String fileType = filePath.substring(filePath.lastIndexOf('.')+1).toLowerCase();
String fileType = filePath.substring(filePath.lastIndexOf('.') + 1).toLowerCase();
switch (fileType) {
case "jpg":
case "jpeg":
@@ -264,18 +265,19 @@ public class FileStorageUtil {
}
directory = (String) Optional.ofNullable(directory).orElse("");
String minFileName = directory + fileName;
this.instance.putObject((PutObjectArgs)((io.minio.PutObjectArgs.Builder)((io.minio.PutObjectArgs.Builder)PutObjectArgs.builder().bucket(bucketName)).object(minFileName)).stream(inputStream, (long)inputStream.available(), -1L).build());
this.instance.putObject((PutObjectArgs) ((io.minio.PutObjectArgs.Builder) ((io.minio.PutObjectArgs.Builder) PutObjectArgs.builder().bucket(bucketName)).object(minFileName)).stream(inputStream, (long) inputStream.available(), -1L).build());
inputStream.close();
return new MinIoUploadResDTO(minFileName, minIoUtils.getObjectUrl(bucketName, minFileName, 60));
}
/**
* @Description: getAllFile 获取minio指定文件夹下所有文件
* @Param:
* @return: java.lang.Iterable<io.minio.Result<io.minio.messages.Item>>
* @return: java.lang.Iterable<io.minio.Result < io.minio.messages.Item>>
* @Author: clam
* @Date: 2024/9/27
*/
public List<MultipartFile> getAllFile(String bucketName, String directory) throws Exception{
public List<MultipartFile> getAllFile(String bucketName, String directory) throws Exception {
List<MultipartFile> files = new ArrayList<>();
Iterable<Result<Item>> results = this.instance.listObjects(
ListObjectsArgs.builder().bucket(bucketName).prefix(directory).recursive(true).build());
@@ -288,7 +290,6 @@ public class FileStorageUtil {
GetObjectArgs.builder().bucket(bucketName).object(item.objectName()).build());
// 假设你有一个方法可以将InputStream转换为MultipartFile
MultipartFile multipartFile = new MockMultipartFile(item.objectName(), item.objectName(), "text/plain", stream);
files.add(multipartFile);
@@ -297,4 +298,38 @@ public class FileStorageUtil {
}
/**
* 目前只给审计日志使用
*
* @author cdf
* @date 2024/10/15
*/
public File getLastFile(String bucketName, String directory) throws Exception {
File file = null;
if (generalInfo.getBusinessFileStorage() == GeneralConstant.MINIO_OSS) {
Iterable<Result<Item>> results = this.instance.listObjects(
ListObjectsArgs.builder().bucket(bucketName).prefix(directory).recursive(true).build());
Item lastItem = null;
for (Result<Item> result : results) {
Item item = result.get();
if (lastItem == null) {
lastItem = item;
} else {
if (item.lastModified().isAfter(lastItem.lastModified())) {
lastItem = item;
}
}
}
InputStream stream = this.instance.getObject(
GetObjectArgs.builder().bucket(bucketName).object(lastItem.objectName()).build());
file = File.createTempFile(lastItem.objectName(), "xlsx");
FileUtil.writeFromStream(stream, file);
}else if(generalInfo.getBusinessFileStorage() == GeneralConstant.HUAWEI_OBS){
file = obsUtil.getLastFile(OssPath.LOGBAK);
}
return file;
}
}

2
pqs-common/common-web/src/main/java/com/njcn/web/utils/GwSendUtil.java
View File

@@ -136,8 +136,6 @@ public class GwSendUtil {
HttpParameters.Builder builder = HttpParameters.newBuilder();
builder.requestURL(sendEnum.getNewUrl()) // 设置请求的URL,可以拼接URL请求参数
.api("zongbuSync") // 设置服务名
.version("1.0.0") // 设置版本号
.method("post") // 设置调用方式, 必须为 post
.contentType("application/json; charset=UTF-8"); //设置请求content-type

4
pqs-system/system-boot/src/main/java/com/njcn/system/handler/MqttMessageHandler.java
View File

@@ -6,6 +6,7 @@ import cn.hutool.core.util.StrUtil;
import cn.hutool.json.JSONObject;
import com.github.tocrhz.mqtt.annotation.MqttSubscribe;
import com.github.tocrhz.mqtt.annotation.Payload;
import com.njcn.common.pojo.constant.LogInfo;
import com.njcn.common.pojo.constant.PatternRegex;
import com.njcn.common.pojo.dto.LogInfoDTO;
import com.njcn.common.utils.PubUtils;
@@ -63,7 +64,7 @@ public class MqttMessageHandler {
}
LogInfoDTO logInfoDTO = PubUtils.json2obj(new String(message.getPayload(), StandardCharsets.UTF_8),LogInfoDTO.class);
if(!logInfoDTO.getLoginName().equals(LogInfo.UNKNOWN_USER) && !logInfoDTO.getOperate().equals(LogInfo.UNKNOWN_OPERATE) && !logInfoDTO.getIp().equals(LogInfo.UNKNOWN_IP)){
if("注销".equals(logInfoDTO.getOperateType()) || "认证".equals(logInfoDTO.getOperateType())){
logInfoDTO.setType(1);
}else {
@@ -80,6 +81,7 @@ public class MqttMessageHandler {
}
userLogService.addUserLog(logInfoDTO);
}
}
/**
* 订阅审计日志的记录，并进行入库操作

70
pqs-system/system-boot/src/main/java/com/njcn/system/service/impl/AuditServiceImpl.java
View File

@@ -34,7 +34,9 @@ import com.njcn.common.pojo.enums.response.CommonResponseEnum;
import com.njcn.common.pojo.exception.BusinessException;
import com.njcn.common.utils.PubUtils;
import com.njcn.db.constant.DbConstant;
import com.njcn.oss.constant.GeneralConstant;
import com.njcn.oss.constant.OssPath;
import com.njcn.oss.utils.FileStorageUtil;
import com.njcn.redis.utils.RedisUtil;
import com.njcn.system.enums.SystemResponseEnum;
import com.njcn.system.excel.UserLogExcel;
@@ -57,8 +59,7 @@ import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import javax.servlet.http.HttpServletRequest;
import java.io.File;
import java.io.FileInputStream;
import java.io.*;
import java.text.ParseException;
import java.time.LocalDateTime;
import java.time.format.DateTimeFormatter;
@@ -94,6 +95,8 @@ public class AuditServiceImpl extends ServiceImpl<UserLogMapper, UserLog> implem
private final MqttPublisher publisher;
private final FileStorageUtil fileStorageUtil;
@Override
public Page<AuditLogVO> getAuditLog(AuditParam auditParam) {
List<AuditLogVO> auditLogVOS = new ArrayList<>();
@@ -206,11 +209,14 @@ public class AuditServiceImpl extends ServiceImpl<UserLogMapper, UserLog> implem
public void logFileWriter() {
TimeInterval timeInterval = new TimeInterval();
//限制多次点击
redisUtil.delete("logFileWriter");
String logFileWriter = redisUtil.getStringByKey("logFileWriter");
if (StrUtil.isNotBlank(logFileWriter) || ObjectUtil.equals(logFileWriter, "1")) {
throw new BusinessException(SystemResponseEnum.MULTIPLE_CLICKS_LOG_FILE_WRITER);
}
redisUtil.saveByKeyWithExpire("logFileWriter", "1", 600L);
Config config = iConfigService.getOne(new LambdaQueryWrapper<Config>().eq(Config::getState, DataStateEnum.ENABLE.getCode()));
LocalDateTime nowDate = LocalDateTime.now();
LocalDateTime agoDate = nowDate.minusMonths(config.getLogTime()).with(TemporalAdjusters.firstDayOfMonth());
@@ -223,12 +229,29 @@ public class AuditServiceImpl extends ServiceImpl<UserLogMapper, UserLog> implem
aa.ge("sys_user_log.update_time", date);
aa.le("sys_user_log.update_time", endTime);
String nowTime = date + "至" + endTime;
File file = new File(generalInfo.getBusinessTempPath() + File.separator + OssPath.LOGBAK);
//必须放到循环外，否则会刷新流
ExcelWriter excelWriter =null;
FileInputStream fileInputStream;
File file = null;
try {
if (generalInfo.getBusinessFileStorage() == GeneralConstant.LOCAL_DISK) {
file = new File(generalInfo.getBusinessTempPath() + File.separator + OssPath.LOGBAK);
if (!file.exists() && !file.isDirectory()) {
file.mkdir();
}
//必须放到循环外，否则会刷新流
ExcelWriter excelWriter = EasyExcel.write(generalInfo.getBusinessTempPath() + File.separator + OssPath.LOGBAK + nowTime + ExcelTypeEnum.XLSX.getValue(), UserLogExcel.class).build();
excelWriter = EasyExcel.write(generalInfo.getBusinessTempPath() + File.separator + OssPath.LOGBAK + nowTime + ExcelTypeEnum.XLSX.getValue(), UserLogExcel.class).build();
} else if (generalInfo.getBusinessFileStorage() == GeneralConstant.HUAWEI_OBS || generalInfo.getBusinessFileStorage() == GeneralConstant.MINIO_OSS) {
file = File.createTempFile(date+"_"+endTime,".xlsx");
excelWriter = EasyExcel.write(file, UserLogExcel.class).build();
}
}catch (Exception e){
log.error("获取文件服务器异常");
redisUtil.delete("logFileWriter");
e.printStackTrace();
}
try {
//记录总数:实际中需要根据查询条件进行统计即可:一共多少条
int totalCount = this.count(aa);
@@ -257,13 +280,24 @@ public class AuditServiceImpl extends ServiceImpl<UserLogMapper, UserLog> implem
excelWriter.write(list, writeSheet);
}
}
} catch (Exception e) {
redisUtil.delete("logFileWriter");
e.printStackTrace();
} finally {
excelWriter.finish();
}
if (generalInfo.getBusinessFileStorage() == GeneralConstant.HUAWEI_OBS || generalInfo.getBusinessFileStorage() == GeneralConstant.MINIO_OSS) {
try {
fileInputStream = new FileInputStream(file);
fileStorageUtil.uploadStream(fileInputStream,OssPath.LOGBAK,file.getName());
} catch (FileNotFoundException e) {
throw new RuntimeException(e);
}finally {
file.delete();
}
}
redisUtil.delete("logFileWriter");
System.out.println("日志备份结束，共花费时间" + timeInterval.intervalSecond() + "S");
}
@@ -276,12 +310,13 @@ public class AuditServiceImpl extends ServiceImpl<UserLogMapper, UserLog> implem
throw new BusinessException(SystemResponseEnum.MULTIPLE_CLICKS_LOG_FILE_WRITER);
}
redisUtil.saveByKeyWithExpire("recoverLogFile", "1", 600L);
File newFile = getLastFile();
FileInputStream in = null;
File newFile = null;
try {
newFile = getLastFile();
if (FileUtil.isEmpty(newFile)) {
throw new BusinessException(SystemResponseEnum.NOT_FIND_FILE);
}
FileInputStream in = null;
try {
in = new FileInputStream(newFile);
Set<String> logIndex = new HashSet<>(this.baseMapper.getAllIndex());
List<UserLog> userLogs = new ArrayList<>();
@@ -307,6 +342,10 @@ public class AuditServiceImpl extends ServiceImpl<UserLogMapper, UserLog> implem
throw new BusinessException(SystemResponseEnum.LOG_EXCEPTION);
} finally {
IOUtils.closeQuietly(in);
if(generalInfo.getBusinessFileStorage() == GeneralConstant.HUAWEI_OBS || generalInfo.getBusinessFileStorage() == GeneralConstant.MINIO_OSS) {
newFile.deleteOnExit();
}
}
redisUtil.delete("recoverLogFile");
}
@@ -430,7 +469,16 @@ public class AuditServiceImpl extends ServiceImpl<UserLogMapper, UserLog> implem
* 返回一个最新修改日期的文件
*/
public File getLastFile() {
File parentFile = new File(generalInfo.getBusinessTempPath() + File.separator + OssPath.LOGBAK);
File parentFile = null;
if(generalInfo.getBusinessFileStorage() == GeneralConstant.LOCAL_DISK){
parentFile = new File(generalInfo.getBusinessTempPath() + File.separator + OssPath.LOGBAK);
}else if(generalInfo.getBusinessFileStorage() == GeneralConstant.HUAWEI_OBS || generalInfo.getBusinessFileStorage() == GeneralConstant.MINIO_OSS){
try {
parentFile = fileStorageUtil.getLastFile("excelreport",OssPath.LOGBAK);
} catch (Exception e) {
throw new RuntimeException(e);
}
}
//文件夹下的所有子文件数组
File[] files = parentFile.listFiles();
List<Date> list;
@@ -439,7 +487,7 @@ public class AuditServiceImpl extends ServiceImpl<UserLogMapper, UserLog> implem
list.sort(Comparator.reverseOrder());
return Arrays.stream(files).filter(tempFile -> FileUtil.lastModifiedTime(tempFile).equals(list.get(0))).collect(Collectors.toList()).get(0);
}
return null;
return parentFile;
}
/**

16
pqs-system/system-boot/src/main/java/com/njcn/system/timer/tasks/PqTypicalSourceTaskRunner.java
View File

@@ -4,6 +4,8 @@ import cn.hutool.core.date.DatePattern;
import cn.hutool.core.date.DateUtil;
import com.njcn.common.pojo.exception.BusinessException;
import com.njcn.harmonic.api.PqTypicalSourceFeignClient;
import com.njcn.harmonic.api.UploadGwDataFeignClient;
import com.njcn.harmonic.pojo.param.UploadDataParam;
import com.njcn.harmonic.pojo.param.UploadParam;
import com.njcn.system.timer.TimerTaskRunner;
import lombok.RequiredArgsConstructor;
@@ -20,15 +22,19 @@ public class PqTypicalSourceTaskRunner implements TimerTaskRunner {
private final PqTypicalSourceFeignClient pqTypicalSourceFeignClient;
private final UploadGwDataFeignClient uploadGwDataFeignClient;
@Override
public void action(String date) {
UploadParam param=new UploadParam();
param.setTime(DateUtil.yesterday().toString(DatePattern.NORM_DATE_PATTERN));
String code = pqTypicalSourceFeignClient.send(param).getCode();
if(code.equals("A0002")){
throw new BusinessException("失败");
}
String dateDeal =DateUtil.yesterday().toString(DatePattern.NORM_DATE_PATTERN);
param.setTime(dateDeal);
pqTypicalSourceFeignClient.send(param).getCode();
UploadDataParam uploadDataParam = new UploadDataParam();
uploadDataParam.setSearchBeginTime(dateDeal);
uploadGwDataFeignClient.upGwCommPoint(uploadDataParam);
}
}