Microservice/pqs
5
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

日志记录改造由mqtt异步改造为redis list实现，TPS满足：150-200条/秒

Browse Source
This commit is contained in:
hongawen
2026-01-16 15:14:16 +08:00
parent c9301d1f8f
commit 23c4a78383
10 changed files with 551 additions and 36 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
pqs-common/common-oss/pom.xml
View File

@@ -63,6 +63,13 @@
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>com.aliyun.oss</groupId>
<artifactId>aliyun-sdk-oss</artifactId>
<version>3.18.0</version>
<optional>false</optional>
</dependency>
</dependencies>
</project>

17
pqs-common/common-redis/src/main/java/com/njcn/redis/pojo/enums/RedisKeyEnum.java
View File

@@ -45,7 +45,22 @@ public enum RedisKeyEnum {
/**
* 云前置心跳
*/
CLD_HEART_BEAT_KEY("CLD_HEART_BEAT:", 180L);
CLD_HEART_BEAT_KEY("CLD_HEART_BEAT:", 180L),
/**
* 用户日志队列
*/
USER_LOG_QUEUE("USER_LOG_QUEUE", -1L),
/**
* 用户日志邮件推送队列
*/
USER_LOG_EMAIL_QUEUE("USER_LOG_EMAIL_QUEUE", -1L),
/**
* 终端日志
*/
DEVICE_LOG_QUEUE("DEVICE_LOG_QUEUE", -1L);
private final String key;

66
pqs-common/common-redis/src/main/java/com/njcn/redis/utils/RedisMessageQueueUtil.java Normal file
View File

@@ -0,0 +1,66 @@
package com.njcn.redis.utils;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.data.redis.core.RedisTemplate;
import org.springframework.stereotype.Component;
import java.util.concurrent.TimeUnit;
/**
* Redis 消息队列工具类
*/
@Slf4j
@Component
@RequiredArgsConstructor
public class RedisMessageQueueUtil {
private final RedisTemplate<String, Object> redisTemplate;
/**
* 推送消息到队列（左推）
*
* @param queueKey 队列Key
* @param message 消息内容（JSON字符串）
*/
public void pushMessage(String queueKey, String message) {
try {
redisTemplate.opsForList().leftPush(queueKey, message);
} catch (Exception e) {
log.error("推送消息到队列失败，queueKey={}", queueKey, e);
throw e;
}
}
/**
* 阻塞式弹出消息（右弹）
*
* @param queueKey 队列Key
* @param timeout 超时时间（秒）
* @return 消息内容，超时返回null
*/
public String popMessage(String queueKey, long timeout) {
try {
Object result = redisTemplate.opsForList().rightPop(queueKey, timeout, TimeUnit.SECONDS);
return result != null ? result.toString() : null;
} catch (Exception e) {
log.error("弹出消息失败，queueKey={}", queueKey, e);
return null;
}
}
/**
* 获取队列长度（用于监控）
*
* @param queueKey 队列Key
* @return 队列长度
*/
public Long getQueueSize(String queueKey) {
try {
return redisTemplate.opsForList().size(queueKey);
} catch (Exception e) {
log.error("获取队列长度失败，queueKey={}", queueKey, e);
return 0L;
}
}
}

3
pqs-common/common-web/src/main/java/com/njcn/web/config/FeignConfig.java
View File

@@ -47,6 +47,7 @@ public class FeignConfig {
@Bean
public Decoder feignDecoder() {
return (response, type) -> {
String bodyStr = Util.toString(response.body().asReader(Util.UTF_8));
//对结果进行转换
HttpResult<Object> result = PubUtils.json2obj(bodyStr, type);
@@ -60,6 +61,8 @@ public class FeignConfig {
}
switch (commonResponseEnum) {
case SUCCESS:
// case NO_DATA:
// case FAIL:
return result;
default:
throw new BusinessException(result);

56
pqs-common/common-web/src/main/java/com/njcn/web/service/impl/LogServiceImpl.java
View File

@@ -1,8 +1,6 @@
package com.njcn.web.service.impl;
import com.github.tocrhz.mqtt.publisher.MqttPublisher;
import com.njcn.common.config.GeneralInfo;
import com.njcn.common.pojo.annotation.OperateInfo;
import com.njcn.common.pojo.constant.LogInfo;
import com.njcn.common.pojo.dto.DeviceLogDTO;
import com.njcn.common.pojo.dto.LogInfoDTO;
@@ -10,6 +8,8 @@ import com.njcn.common.pojo.enums.response.CommonResponseEnum;
import com.njcn.common.pojo.response.HttpResult;
import com.njcn.common.utils.PubUtils;
import com.njcn.common.utils.ReflectCommonUtil;
import com.njcn.redis.pojo.enums.RedisKeyEnum;
import com.njcn.redis.utils.RedisMessageQueueUtil;
import com.njcn.web.advice.DeviceLog;
import com.njcn.web.service.ILogService;
import com.njcn.web.utils.RequestUtil;
@@ -46,8 +46,15 @@ public class LogServiceImpl implements ILogService {
private final GeneralInfo generalInfo;
/**
* mqtt处理日志异步发布
*/
// private final MqttPublisher publisher;
private final MqttPublisher publisher;
/**
* redis队列处理日志异步发布
*/
private final RedisMessageQueueUtil redisMessageQueueUtil;
/**
@@ -82,14 +89,27 @@ public class LogServiceImpl implements ILogService {
String operateType = ReflectCommonUtil.getOperateTypeByMethod(returnType.getMethod());
Integer severity = levelStringToNumber(level);
LogInfoDTO logInfoDTO = new LogInfoDTO(loginName, userName, ip, methodDescribe, operateType, result.equalsIgnoreCase("失败") ? 0 : 1, "", severity, type.equalsIgnoreCase("业务事件") ? 0 : 1, generalInfo.getMicroServiceName(), userIndex, LocalDateTime.now());
publisher.send("/userLog", PubUtils.obj2json(logInfoDTO), 2, false);
// publisher.send("/userLog", PubUtils.obj2json(logInfoDTO), 2, false);
redisMessageQueueUtil.pushMessage(
RedisKeyEnum.USER_LOG_QUEUE.getKey(),
PubUtils.obj2json(logInfoDTO)
);
//推送审计消息功能
if (severity != 0) {
if (!logInfoDTO.getLoginName().equals(LogInfo.UNKNOWN_USER)) {
publisher.send("/userLogPush", PubUtils.obj2json(logInfoDTO), 2, false);
/**注意，此处是给前端推送的，如果没有了mqtt协议，此处前端就无消息可消费了*/
// publisher.send("/userLogPush", PubUtils.obj2json(logInfoDTO), 2, false);
//发送邮箱功能
if (severity == 2 && logInfoDTO.getResult() == 0) {
//publisher.send("/userLogEmailPush", PubUtils.obj2json(logInfoDTO), 2, false);
// redisMessageQueueUtil.pushMessage(
// RedisKeyEnum.USER_LOG_EMAIL_QUEUE.getKey(),
// PubUtils.obj2json(logInfoDTO)
// );
}
}
}
@@ -97,8 +117,11 @@ public class LogServiceImpl implements ILogService {
if (Objects.nonNull((returnType.getMethod())) && (returnType.getMethod()).isAnnotationPresent(DeviceLog.class)) {
String deviceOperate = returnType.getMethod().getAnnotation(DeviceLog.class).operateType();
DeviceLogDTO deviceLogDTO = new DeviceLogDTO(userName, deviceOperate, result.equalsIgnoreCase("失败") ? 0 : 1, "", loginName, userIndex);
publisher.send("/deviceLog", PubUtils.obj2json(deviceLogDTO), 2, false);
// publisher.send("/deviceLog", PubUtils.obj2json(deviceLogDTO), 2, false);
redisMessageQueueUtil.pushMessage(
RedisKeyEnum.DEVICE_LOG_QUEUE.getKey(),
PubUtils.obj2json(deviceLogDTO)
);
}
}
@@ -135,7 +158,11 @@ public class LogServiceImpl implements ILogService {
String operateType = ReflectCommonUtil.getOperateTypeByMethod(method);
Integer severity = levelStringToNumber(level);
LogInfoDTO logInfoDTO = new LogInfoDTO(tempLogInfo.getLoginName(), tempLogInfo.getUserName(), tempLogInfo.getIp(), ReflectCommonUtil.getMethodDescribeByMethod(method), operateType, result.equalsIgnoreCase("失败") ? 0 : 1, message, severity, type.equalsIgnoreCase("业务事件") ? 0 : 1, generalInfo.getMicroServiceName(), userIndex, LocalDateTime.now());
publisher.send("/userLog", PubUtils.obj2json(logInfoDTO), 1, false);
// publisher.send("/userLog", PubUtils.obj2json(logInfoDTO), 1, false);
redisMessageQueueUtil.pushMessage(
RedisKeyEnum.USER_LOG_QUEUE.getKey(),
PubUtils.obj2json(logInfoDTO)
);
auditPush(severity, logInfoDTO);
}
@@ -163,7 +190,11 @@ public class LogServiceImpl implements ILogService {
Integer severity = levelStringToNumber(level);
String operateType = ReflectCommonUtil.getOperateTypeByMethod(method);
LogInfoDTO logInfoDTO = new LogInfoDTO(loginName, "", ip, ReflectCommonUtil.getMethodDescribeByMethod(method), operateType, result.equalsIgnoreCase("失败") ? 0 : 1, message, severity, type.equalsIgnoreCase("业务事件") ? 0 : 1, generalInfo.getMicroServiceName(), loginName, LocalDateTime.now());
publisher.send("/userLog", PubUtils.obj2json(logInfoDTO), 1, false);
// publisher.send("/userLog", PubUtils.obj2json(logInfoDTO), 1, false);
redisMessageQueueUtil.pushMessage(
RedisKeyEnum.USER_LOG_QUEUE.getKey(),
PubUtils.obj2json(logInfoDTO)
);
auditPush(severity, logInfoDTO);
}
@@ -171,7 +202,12 @@ public class LogServiceImpl implements ILogService {
//推送审计消息功能
if (severity != 0) {
if (!logInfoDTO.getLoginName().equals(LogInfo.UNKNOWN_USER)) {
publisher.send("/userLogPush", PubUtils.obj2json(logInfoDTO), 2, false);
/**注意，此处是给前端推送的，如果没有了mqtt协议，此处前端就无消息可消费了*/
// publisher.send("/userLogPush", PubUtils.obj2json(logInfoDTO), 2, false);
// redisMessageQueueUtil.pushMessage(
// RedisKeyEnum.USER_LOG_QUEUE.getKey(),
// PubUtils.obj2json(logInfoDTO)
// );
//发送邮箱功能
if (severity == 2 && logInfoDTO.getResult() == 0) {
//publisher.send("/userLogEmailPush", PubUtils.obj2json(logInfoDTO), 2, false);