Microservice/pqs
5
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

代码调整

Browse Source
This commit is contained in:
hongawen
2023-05-31 18:56:37 +08:00
parent d2a1694242
commit ee31e8438b
9 changed files with 181 additions and 43 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
pqs-common/common-db/src/main/java/com/njcn/db/config/MybatisConfig.java
View File

@@ -4,6 +4,7 @@ import com.baomidou.mybatisplus.annotation.DbType;
import com.baomidou.mybatisplus.extension.plugins.MybatisPlusInterceptor;
import com.baomidou.mybatisplus.extension.plugins.inner.PaginationInnerInterceptor;
import com.njcn.db.handler.AutoFillValueHandler;
import com.njcn.db.handler.BatchInjector;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
@@ -34,4 +35,17 @@ public class MybatisConfig {
return new AutoFillValueHandler();
}
/***
* 自定义注册器，处理批量插入
* @author hongawen
* @date 2023/5/30 14:53
* @return BatchInjector
*/
@Bean
public BatchInjector sqlInjector() {
return new BatchInjector();
}
}

25
pqs-common/common-db/src/main/java/com/njcn/db/handler/BatchInjector.java Normal file
View File

@@ -0,0 +1,25 @@
package com.njcn.db.handler;
import com.baomidou.mybatisplus.annotation.FieldFill;
import com.baomidou.mybatisplus.core.injector.AbstractMethod;
import com.baomidou.mybatisplus.core.injector.DefaultSqlInjector;
import com.baomidou.mybatisplus.extension.injector.methods.InsertBatchSomeColumn;
import java.util.List;
/**
* @author hongawen
* @version 1.0.0
* @date 2023年05月30日 14:51
*/
public class BatchInjector extends DefaultSqlInjector {
@Override
public List<AbstractMethod> getMethodList(Class<?> mapperClass) {
List<AbstractMethod> methodList = super.getMethodList(mapperClass);
//更新时自动填充的字段，不用插入值
methodList.add(new InsertBatchSomeColumn(i -> i.getFieldFill() != FieldFill.UPDATE));
return methodList;
}
}

19
pqs-common/common-db/src/main/java/com/njcn/db/mapper/BatchMapper.java Normal file
View File

@@ -0,0 +1,19 @@
package com.njcn.db.mapper;
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
import java.util.List;
/**
* @author hongawen
* @version 1.0.0
* @date 2023年05月30日 14:54
*/
public interface BatchMapper<T> extends BaseMapper<T> {
/**
* 真正的批量插入
*/
int insertBatchSomeColumn(List<T> entityList);
}

5
pqs-system/system-api/pom.xml
View File

@@ -43,6 +43,11 @@
<groupId>com.github.jeffreyning</groupId>
<artifactId>mybatisplus-plus</artifactId>
</dependency>
<dependency>
<groupId>com.alibaba</groupId>
<artifactId>easyexcel</artifactId>
<version>${easyexcel}</version>
</dependency>
</dependencies>
</project>

17
pqs-system/system-api/src/main/java/com/njcn/system/pojo/po/UserLog.java
View File

@@ -1,5 +1,6 @@
package com.njcn.system.pojo.po;
import com.alibaba.excel.annotation.ExcelProperty;
import com.baomidou.mybatisplus.annotation.FieldFill;
import com.baomidou.mybatisplus.annotation.FieldStrategy;
import com.baomidou.mybatisplus.annotation.TableField;
@@ -23,70 +24,83 @@ public class UserLog {
/**
* 事件日志Id
*/
@ExcelProperty(value = "id主键")
private String id;
/**
* 登录名
*/
@TableField(insertStrategy = FieldStrategy.IGNORED)
@ExcelProperty(value = "登录名")
private String loginName;
/**
* 用户已登录：用户名
*/
@TableField(insertStrategy = FieldStrategy.IGNORED)
@ExcelProperty(value = "用户名")
private String userName;
/**
* 操作Ip
*/
@ExcelProperty(value = "操作ip")
private String ip;
/**
* 操作内容
*/
@ExcelProperty(value = "操作内容")
private String operate;
/**
* 操作类型 比如：查询、新增、删除等等
*/
@ExcelProperty(value = "操作类型")
private String operateType;
/**
* 操作结果 0.失败 1.成功
*/
@ExcelProperty(value = "操作结果")
private Integer result;
/**
* 失败原因
*/
@TableField(insertStrategy = FieldStrategy.IGNORED)
@ExcelProperty(value = "失败原因")
private String failReason;
/**
* 严重度 0.普通 1.中等 2.严重
*/
@ExcelProperty(value = "时间等级(严重度)")
private Integer level;
/**
* 事件类型 0.业务事件 1.系统事件
*/
@ExcelProperty(value = "事件类型")
private Integer type;
/**
* 模块名
*/
@ExcelProperty(value = "模块名称")
private String serviceName;
/**
* 告警标志 0.未告警 1.已告警
*/
@ExcelProperty(value = "告警标志")
private Integer state;
/**
* 创建用户
*/
@TableField(fill = FieldFill.INSERT, insertStrategy = FieldStrategy.IGNORED)
@ExcelProperty(value = "创建用户")
private String createBy;
@@ -95,12 +109,14 @@ public class UserLog {
*/
@TableField(fill = FieldFill.INSERT, insertStrategy = FieldStrategy.IGNORED)
@JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss")
@ExcelProperty(value = "创建时间" )
private LocalDateTime createTime;
/**
* 更新用户
*/
@TableField(fill = FieldFill.INSERT_UPDATE, insertStrategy = FieldStrategy.IGNORED)
@ExcelProperty(value = "更新用户")
private String updateBy;
/**
@@ -108,6 +124,7 @@ public class UserLog {
*/
@TableField(fill = FieldFill.INSERT_UPDATE, insertStrategy = FieldStrategy.IGNORED)
@JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss")
@ExcelProperty(value = "更新时间" )
private LocalDateTime updateTime;

6
pqs-system/system-boot/pom.xml
View File

@@ -58,11 +58,7 @@
<version>1.0.0</version>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>com.alibaba</groupId>
<artifactId>easyexcel</artifactId>
<version>${easyexcel}</version>
</dependency>
</dependencies>
<build>

15
pqs-system/system-boot/src/main/java/com/njcn/system/mapper/UserLogMapper.java
View File

@@ -1,11 +1,14 @@
package com.njcn.system.mapper;
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
import com.njcn.db.mapper.BatchMapper;
import com.njcn.system.excel.UserLogExcel;
import com.njcn.system.pojo.po.UserLog;
import org.apache.ibatis.annotations.Param;
import java.util.HashSet;
import java.util.List;
/**
* <p>
* Mapper 接口
@@ -14,7 +17,7 @@ import java.util.List;
* @author hongawen
* @since 2021-12-13
*/
public interface UserLogMapper extends BaseMapper<UserLog> {
public interface UserLogMapper extends BatchMapper<UserLog> {
/**
@@ -36,4 +39,12 @@ public interface UserLogMapper extends BaseMapper<UserLog> {
* @return
*/
Float getMemoInfo(@Param("schema") String schema);
/***
* 查询表中所有索引
* @author hongawen
* @date 2023/5/31 15:56
* @return List<String>
*/
List<String> getAllIndex();
}

7
pqs-system/system-boot/src/main/java/com/njcn/system/mapper/mapping/UserLogMapper.xml
View File

@@ -57,4 +57,11 @@
where table_schema = #{schema} and table_name="sys_user_log"
</select>
<select id="getAllIndex" resultType="String">
SELECT
id
FROM
sys_user_log
</select>
</mapper>

74
pqs-system/system-boot/src/main/java/com/njcn/system/service/impl/AuditServiceImpl.java
View File

@@ -5,12 +5,15 @@ import cn.hutool.core.bean.BeanUtil;
import cn.hutool.core.collection.CollUtil;
import cn.hutool.core.date.DateTime;
import cn.hutool.core.date.DateUtil;
import cn.hutool.core.date.StopWatch;
import cn.hutool.core.io.FileUtil;
import cn.hutool.core.util.ObjectUtil;
import cn.hutool.core.util.StrUtil;
import cn.hutool.json.JSONUtil;
import com.alibaba.excel.EasyExcel;
import com.alibaba.excel.ExcelWriter;
import com.alibaba.excel.context.AnalysisContext;
import com.alibaba.excel.event.AnalysisEventListener;
import com.alibaba.excel.support.ExcelTypeEnum;
import com.alibaba.excel.write.metadata.WriteSheet;
import com.alibaba.excel.write.style.column.LongestMatchColumnWidthStyleStrategy;
@@ -43,6 +46,7 @@ import org.apache.commons.fileupload.disk.DiskFileItemFactory;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.http.MediaType;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import org.springframework.web.multipart.MultipartFile;
import org.springframework.web.multipart.commons.CommonsMultipartFile;
@@ -164,12 +168,12 @@ public class AuditServiceImpl extends ServiceImpl<UserLogMapper, UserLog> implem
aa.ge("sys_user_log.update_time", date);
aa.le("sys_user_log.update_time", endTime);
String nowTime = date + "至" + endTime;
File file=new File(generalInfo.getBusinessTempPath() + "/" + OssPath.LOGBAK);
File file = new File(generalInfo.getBusinessTempPath() + File.separator + OssPath.LOGBAK);
if (!file.exists() && !file.isDirectory()) {
file.mkdir();
}
//必须放到循环外，否则会刷新流
ExcelWriter excelWriter = EasyExcel.write(generalInfo.getBusinessTempPath() + "/"+ OssPath.LOGBAK + nowTime + ExcelTypeEnum.XLSX.getValue(), UserLogExcel.class)
ExcelWriter excelWriter = EasyExcel.write(generalInfo.getBusinessTempPath() + File.separator + OssPath.LOGBAK + nowTime + ExcelTypeEnum.XLSX.getValue(), UserLogExcel.class)
// .excelType(ExcelTypeEnum.CSV)
.build();
@@ -207,6 +211,7 @@ public class AuditServiceImpl extends ServiceImpl<UserLogMapper, UserLog> implem
excelWriter.write(list, writeSheet);
}
}
} catch (Exception e) {
redisUtil.delete("recoverLogFile");
e.printStackTrace();
@@ -233,6 +238,7 @@ public class AuditServiceImpl extends ServiceImpl<UserLogMapper, UserLog> implem
}
@Override
@Transactional(rollbackFor = {Exception.class})
public void recoverLogFile() {
String logFileWriter = redisUtil.getStringByKey("recoverLogFile");
if (StrUtil.isNotBlank(logFileWriter) || ObjectUtil.equals(logFileWriter, "1")) {
@@ -246,20 +252,23 @@ public class AuditServiceImpl extends ServiceImpl<UserLogMapper, UserLog> implem
FileInputStream in = null;
try {
in = new FileInputStream(newFile);
String name = newFile.getName();
String[] split = name.split(ExcelTypeEnum.XLSX.getValue());
String[] splitTime = split[0].split("至");
if (StrUtil.isBlank(splitTime[0]) || StrUtil.isBlank(splitTime[1])) {
throw new BusinessException(AuditLogEnum.LOG_EXCEPTIONTIME);
Set<String> logIndex = new HashSet<>(this.baseMapper.getAllIndex());
List<UserLog> userLogs = new ArrayList<>();
EasyExcel.read(newFile.getPath(), UserLog.class, new AnalysisEventListener<UserLog>() {
@Override
public void invoke(UserLog userLog, AnalysisContext analysisContext) {
// 将读取到的每一行存入reportDetails集合中
if (!logIndex.contains(userLog.getId())) {
userLogs.add(userLog);
}
}
LambdaQueryWrapper<UserLog> le = new LambdaQueryWrapper<UserLog>()
.ge(UserLog::getCreateTime, DateUtil.beginOfDay(DateUtil.parse(splitTime[0])))
.le(UserLog::getCreateTime, splitTime[1].replace("_", ":"));
this.remove(le);
EasyExcel.read(newFile, UserLogExcel.class, new DataListener(this.getBaseMapper()))
.excelType(ExcelTypeEnum.XLSX).doReadAll();
@Override
public void doAfterAllAnalysed(AnalysisContext analysisContext) {
}
}).sheet().doRead();
// 执行批量插入
saveLogsBatch(userLogs, 1500);
} catch (Exception e) {
redisUtil.delete("recoverLogFile");
e.printStackTrace();
@@ -271,6 +280,41 @@ public class AuditServiceImpl extends ServiceImpl<UserLogMapper, UserLog> implem
}
/***
* 批量插入日志
* @author hongawen
* @date 2023/5/30 15:56
* @return boolean
*/
public boolean saveLogsBatch(Collection<UserLog> userLogs, int batchSize) {
try {
int size = userLogs.size();
int idxLimit = Math.min(batchSize, size);
List<UserLog> userLogTemp = new ArrayList<>(userLogs);
//保存单批提交的数据集合
if (idxLimit == batchSize) {
int times = size / idxLimit + 1;
for (int i = 1; i <= times; i++) {
if (size > idxLimit) {
List<UserLog> temp = userLogTemp.subList(0, idxLimit);
this.baseMapper.insertBatchSomeColumn(temp);
temp.clear();
size = size - idxLimit;
} else {
this.baseMapper.insertBatchSomeColumn(userLogTemp);
}
}
} else {
this.baseMapper.insertBatchSomeColumn(userLogTemp);
}
} catch (Exception e) {
log.error("saveBatch fail", e);
return false;
}
return true;
}
@Override
public Page<AuditLogCusVO> censusAuditLog(AuditParam auditParam) {
//待分页数据总量
@@ -350,7 +394,7 @@ public class AuditServiceImpl extends ServiceImpl<UserLogMapper, UserLog> implem
* 返回一个最新修改日期的文件
*/
public File getLastFile() {
File parentFile = new File(generalInfo.getBusinessTempPath()+"/"+ OssPath.LOGBAK);
File parentFile = new File(generalInfo.getBusinessTempPath() + File.separator + OssPath.LOGBAK);
//文件夹下的所有子文件数组
File[] files = parentFile.listFiles();