diff --git a/pqs-common/common-db/src/main/java/com/njcn/db/config/MybatisConfig.java b/pqs-common/common-db/src/main/java/com/njcn/db/config/MybatisConfig.java index 7520c7d94..a0517065b 100644 --- a/pqs-common/common-db/src/main/java/com/njcn/db/config/MybatisConfig.java +++ b/pqs-common/common-db/src/main/java/com/njcn/db/config/MybatisConfig.java @@ -4,6 +4,7 @@ import com.baomidou.mybatisplus.annotation.DbType; import com.baomidou.mybatisplus.extension.plugins.MybatisPlusInterceptor; import com.baomidou.mybatisplus.extension.plugins.inner.PaginationInnerInterceptor; import com.njcn.db.handler.AutoFillValueHandler; +import com.njcn.db.handler.BatchInjector; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; @@ -34,4 +35,17 @@ public class MybatisConfig { return new AutoFillValueHandler(); } + + /*** + * 自定义注册器，处理批量插入 + * @author hongawen + * @date 2023/5/30 14:53 + * @return BatchInjector + */ + @Bean + public BatchInjector sqlInjector() { + return new BatchInjector(); + } + + } diff --git a/pqs-common/common-db/src/main/java/com/njcn/db/handler/BatchInjector.java b/pqs-common/common-db/src/main/java/com/njcn/db/handler/BatchInjector.java new file mode 100644 index 000000000..94788f3e2 --- /dev/null +++ b/pqs-common/common-db/src/main/java/com/njcn/db/handler/BatchInjector.java @@ -0,0 +1,25 @@ +package com.njcn.db.handler; + +import com.baomidou.mybatisplus.annotation.FieldFill; +import com.baomidou.mybatisplus.core.injector.AbstractMethod; +import com.baomidou.mybatisplus.core.injector.DefaultSqlInjector; +import com.baomidou.mybatisplus.extension.injector.methods.InsertBatchSomeColumn; + +import java.util.List; + +/** + * @author hongawen + * @version 1.0.0 + * @date 2023年05月30日 14:51 + */ +public class BatchInjector extends DefaultSqlInjector { + + @Override + public List getMethodList(Class mapperClass) { + List methodList = super.getMethodList(mapperClass); + //更新时自动填充的字段，不用插入值 + methodList.add(new InsertBatchSomeColumn(i -> i.getFieldFill() != FieldFill.UPDATE)); + return methodList; + } + +} diff --git a/pqs-common/common-db/src/main/java/com/njcn/db/mapper/BatchMapper.java b/pqs-common/common-db/src/main/java/com/njcn/db/mapper/BatchMapper.java new file mode 100644 index 000000000..a4687c5c9 --- /dev/null +++ b/pqs-common/common-db/src/main/java/com/njcn/db/mapper/BatchMapper.java @@ -0,0 +1,19 @@ +package com.njcn.db.mapper; + +import com.baomidou.mybatisplus.core.mapper.BaseMapper; + +import java.util.List; + +/** + * @author hongawen + * @version 1.0.0 + * @date 2023年05月30日 14:54 + */ +public interface BatchMapper extends BaseMapper { + + /** + * 真正的批量插入 + */ + int insertBatchSomeColumn(List entityList); + +} diff --git a/pqs-system/system-api/pom.xml b/pqs-system/system-api/pom.xml index 2b29a6086..d32937665 100644 --- a/pqs-system/system-api/pom.xml +++ b/pqs-system/system-api/pom.xml @@ -43,6 +43,11 @@ com.github.jeffreyning mybatisplus-plus + + com.alibaba + easyexcel + ${easyexcel} + \ No newline at end of file diff --git a/pqs-system/system-api/src/main/java/com/njcn/system/pojo/po/UserLog.java b/pqs-system/system-api/src/main/java/com/njcn/system/pojo/po/UserLog.java index 2f5892e62..6e1395e96 100644 --- a/pqs-system/system-api/src/main/java/com/njcn/system/pojo/po/UserLog.java +++ b/pqs-system/system-api/src/main/java/com/njcn/system/pojo/po/UserLog.java @@ -1,5 +1,6 @@ package com.njcn.system.pojo.po; +import com.alibaba.excel.annotation.ExcelProperty; import com.baomidou.mybatisplus.annotation.FieldFill; import com.baomidou.mybatisplus.annotation.FieldStrategy; import com.baomidou.mybatisplus.annotation.TableField; @@ -23,70 +24,83 @@ public class UserLog { /** * 事件日志Id */ + @ExcelProperty(value = "id主键") private String id; /** * 登录名 */ @TableField(insertStrategy = FieldStrategy.IGNORED) + @ExcelProperty(value = "登录名") private String loginName; /** * 用户已登录：用户名 */ @TableField(insertStrategy = FieldStrategy.IGNORED) + @ExcelProperty(value = "用户名") private String userName; /** * 操作Ip */ + @ExcelProperty(value = "操作ip") private String ip; /** * 操作内容 */ + @ExcelProperty(value = "操作内容") private String operate; /** * 操作类型 比如：查询、新增、删除等等 */ + @ExcelProperty(value = "操作类型") private String operateType; /** * 操作结果 0.失败 1.成功 */ + @ExcelProperty(value = "操作结果") private Integer result; /** * 失败原因 */ @TableField(insertStrategy = FieldStrategy.IGNORED) + @ExcelProperty(value = "失败原因") private String failReason; /** * 严重度 0.普通 1.中等 2.严重 */ + @ExcelProperty(value = "时间等级(严重度)") private Integer level; /** * 事件类型 0.业务事件 1.系统事件 */ + @ExcelProperty(value = "事件类型") private Integer type; /** * 模块名 */ + @ExcelProperty(value = "模块名称") private String serviceName; /** * 告警标志 0.未告警 1.已告警 */ + @ExcelProperty(value = "告警标志") private Integer state; /** * 创建用户 */ @TableField(fill = FieldFill.INSERT, insertStrategy = FieldStrategy.IGNORED) + @ExcelProperty(value = "创建用户") private String createBy; @@ -95,12 +109,14 @@ public class UserLog { */ @TableField(fill = FieldFill.INSERT, insertStrategy = FieldStrategy.IGNORED) @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss") + @ExcelProperty(value = "创建时间" ) private LocalDateTime createTime; /** * 更新用户 */ @TableField(fill = FieldFill.INSERT_UPDATE, insertStrategy = FieldStrategy.IGNORED) + @ExcelProperty(value = "更新用户") private String updateBy; /** @@ -108,6 +124,7 @@ public class UserLog { */ @TableField(fill = FieldFill.INSERT_UPDATE, insertStrategy = FieldStrategy.IGNORED) @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss") + @ExcelProperty(value = "更新时间" ) private LocalDateTime updateTime; diff --git a/pqs-system/system-boot/pom.xml b/pqs-system/system-boot/pom.xml index 6aff32ecc..a07357835 100644 --- a/pqs-system/system-boot/pom.xml +++ b/pqs-system/system-boot/pom.xml @@ -58,11 +58,7 @@ 1.0.0 compile - - com.alibaba - easyexcel - ${easyexcel} - + diff --git a/pqs-system/system-boot/src/main/java/com/njcn/system/mapper/UserLogMapper.java b/pqs-system/system-boot/src/main/java/com/njcn/system/mapper/UserLogMapper.java index 29749b077..9bc5b2520 100644 --- a/pqs-system/system-boot/src/main/java/com/njcn/system/mapper/UserLogMapper.java +++ b/pqs-system/system-boot/src/main/java/com/njcn/system/mapper/UserLogMapper.java @@ -1,11 +1,14 @@ package com.njcn.system.mapper; -import com.baomidou.mybatisplus.core.mapper.BaseMapper; +import com.njcn.db.mapper.BatchMapper; import com.njcn.system.excel.UserLogExcel; import com.njcn.system.pojo.po.UserLog; import org.apache.ibatis.annotations.Param; + +import java.util.HashSet; import java.util.List; + /** *

* Mapper 接口 @@ -14,7 +17,7 @@ import java.util.List; * @author hongawen * @since 2021-12-13 */ -public interface UserLogMapper extends BaseMapper { +public interface UserLogMapper extends BatchMapper { /** @@ -36,4 +39,12 @@ public interface UserLogMapper extends BaseMapper { * @return */ Float getMemoInfo(@Param("schema") String schema); + + /*** + * 查询表中所有索引 + * @author hongawen + * @date 2023/5/31 15:56 + * @return List + */ + List getAllIndex(); } diff --git a/pqs-system/system-boot/src/main/java/com/njcn/system/mapper/mapping/UserLogMapper.xml b/pqs-system/system-boot/src/main/java/com/njcn/system/mapper/mapping/UserLogMapper.xml index b4ea6e956..183de516f 100644 --- a/pqs-system/system-boot/src/main/java/com/njcn/system/mapper/mapping/UserLogMapper.xml +++ b/pqs-system/system-boot/src/main/java/com/njcn/system/mapper/mapping/UserLogMapper.xml @@ -57,4 +57,11 @@ where table_schema = #{schema} and table_name="sys_user_log" + + diff --git a/pqs-system/system-boot/src/main/java/com/njcn/system/service/impl/AuditServiceImpl.java b/pqs-system/system-boot/src/main/java/com/njcn/system/service/impl/AuditServiceImpl.java index f7b07391a..5001046e1 100644 --- a/pqs-system/system-boot/src/main/java/com/njcn/system/service/impl/AuditServiceImpl.java +++ b/pqs-system/system-boot/src/main/java/com/njcn/system/service/impl/AuditServiceImpl.java @@ -5,12 +5,15 @@ import cn.hutool.core.bean.BeanUtil; import cn.hutool.core.collection.CollUtil; import cn.hutool.core.date.DateTime; import cn.hutool.core.date.DateUtil; +import cn.hutool.core.date.StopWatch; import cn.hutool.core.io.FileUtil; import cn.hutool.core.util.ObjectUtil; import cn.hutool.core.util.StrUtil; import cn.hutool.json.JSONUtil; import com.alibaba.excel.EasyExcel; import com.alibaba.excel.ExcelWriter; +import com.alibaba.excel.context.AnalysisContext; +import com.alibaba.excel.event.AnalysisEventListener; import com.alibaba.excel.support.ExcelTypeEnum; import com.alibaba.excel.write.metadata.WriteSheet; import com.alibaba.excel.write.style.column.LongestMatchColumnWidthStyleStrategy; @@ -43,6 +46,7 @@ import org.apache.commons.fileupload.disk.DiskFileItemFactory; import org.springframework.beans.factory.annotation.Value; import org.springframework.http.MediaType; import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; import org.springframework.web.multipart.MultipartFile; import org.springframework.web.multipart.commons.CommonsMultipartFile; @@ -79,34 +83,34 @@ public class AuditServiceImpl extends ServiceImpl implem Page info = this.page(new Page<>(auditParam.getPageNum(), auditParam.getPageSize()), new LambdaQueryWrapper() .like(StrUtil.isNotBlank(auditParam.getLoginName()), UserLog::getLoginName, auditParam.getLoginName()) - .ne(UserLog::getLoginName,"unknown user") - .ne(UserLog::getLoginName,"") - .ne(UserLog::getOperate,"unknown user") + .ne(UserLog::getLoginName, "unknown user") + .ne(UserLog::getLoginName, "") + .ne(UserLog::getOperate, "unknown user") .eq(auditParam.getType() != null, UserLog::getType, auditParam.getType()) .eq(StrUtil.isNotBlank(auditParam.getOperateType()), UserLog::getOperateType, auditParam.getOperateType()) .ge(StrUtil.isNotBlank(auditParam.getSearchBeginTime()), UserLog::getCreateTime, DateUtil.beginOfDay(DateUtil.parse(auditParam.getSearchBeginTime()))) .le(StrUtil.isNotBlank(auditParam.getSearchEndTime()), UserLog::getCreateTime, DateUtil.endOfDay(DateUtil.parse(auditParam.getSearchEndTime()))) - .or(StrUtil.isNotBlank(auditParam.getLoginName()),qw -> + .or(StrUtil.isNotBlank(auditParam.getLoginName()), qw -> qw.like(StrUtil.isNotBlank(auditParam.getLoginName()), UserLog::getIp, auditParam.getLoginName()) - .ne(UserLog::getLoginName,"unknown user") - .ne(UserLog::getLoginName,"") - .ne(UserLog::getOperate,"unknown user") - .eq(auditParam.getType() != null, UserLog::getType, auditParam.getType()) - .eq(StrUtil.isNotBlank(auditParam.getOperateType()), UserLog::getOperateType, auditParam.getOperateType()) - .ge(StrUtil.isNotBlank(auditParam.getSearchBeginTime()), UserLog::getCreateTime, DateUtil.beginOfDay(DateUtil.parse(auditParam.getSearchBeginTime()))) - .le(StrUtil.isNotBlank(auditParam.getSearchEndTime()), UserLog::getCreateTime, DateUtil.endOfDay(DateUtil.parse(auditParam.getSearchEndTime()))) - ) - .or(StrUtil.isNotBlank(auditParam.getLoginName()),qw -> - qw.like(StrUtil.isNotBlank(auditParam.getLoginName()), UserLog::getCreateTime, auditParam.getLoginName()) - .ne(UserLog::getLoginName,"unknown user") - .ne(UserLog::getLoginName,"") - .ne(UserLog::getOperate,"unknown user") + .ne(UserLog::getLoginName, "unknown user") + .ne(UserLog::getLoginName, "") + .ne(UserLog::getOperate, "unknown user") .eq(auditParam.getType() != null, UserLog::getType, auditParam.getType()) .eq(StrUtil.isNotBlank(auditParam.getOperateType()), UserLog::getOperateType, auditParam.getOperateType()) .ge(StrUtil.isNotBlank(auditParam.getSearchBeginTime()), UserLog::getCreateTime, DateUtil.beginOfDay(DateUtil.parse(auditParam.getSearchBeginTime()))) .le(StrUtil.isNotBlank(auditParam.getSearchEndTime()), UserLog::getCreateTime, DateUtil.endOfDay(DateUtil.parse(auditParam.getSearchEndTime()))) ) - .orderByDesc( UserLog::getCreateTime) + .or(StrUtil.isNotBlank(auditParam.getLoginName()), qw -> + qw.like(StrUtil.isNotBlank(auditParam.getLoginName()), UserLog::getCreateTime, auditParam.getLoginName()) + .ne(UserLog::getLoginName, "unknown user") + .ne(UserLog::getLoginName, "") + .ne(UserLog::getOperate, "unknown user") + .eq(auditParam.getType() != null, UserLog::getType, auditParam.getType()) + .eq(StrUtil.isNotBlank(auditParam.getOperateType()), UserLog::getOperateType, auditParam.getOperateType()) + .ge(StrUtil.isNotBlank(auditParam.getSearchBeginTime()), UserLog::getCreateTime, DateUtil.beginOfDay(DateUtil.parse(auditParam.getSearchBeginTime()))) + .le(StrUtil.isNotBlank(auditParam.getSearchEndTime()), UserLog::getCreateTime, DateUtil.endOfDay(DateUtil.parse(auditParam.getSearchEndTime()))) + ) + .orderByDesc(UserLog::getCreateTime) ); Page page = BeanUtil.copyProperties(info, Page.class); if (CollUtil.isNotEmpty(info.getRecords())) { @@ -164,12 +168,12 @@ public class AuditServiceImpl extends ServiceImpl implem aa.ge("sys_user_log.update_time", date); aa.le("sys_user_log.update_time", endTime); String nowTime = date + "至" + endTime; - File file=new File(generalInfo.getBusinessTempPath() + "/" + OssPath.LOGBAK); - if(!file.exists() && !file .isDirectory()){ + File file = new File(generalInfo.getBusinessTempPath() + File.separator + OssPath.LOGBAK); + if (!file.exists() && !file.isDirectory()) { file.mkdir(); } //必须放到循环外，否则会刷新流 - ExcelWriter excelWriter = EasyExcel.write(generalInfo.getBusinessTempPath() + "/"+ OssPath.LOGBAK + nowTime + ExcelTypeEnum.XLSX.getValue(), UserLogExcel.class) + ExcelWriter excelWriter = EasyExcel.write(generalInfo.getBusinessTempPath() + File.separator + OssPath.LOGBAK + nowTime + ExcelTypeEnum.XLSX.getValue(), UserLogExcel.class) // .excelType(ExcelTypeEnum.CSV) .build(); @@ -207,6 +211,7 @@ public class AuditServiceImpl extends ServiceImpl implem excelWriter.write(list, writeSheet); } } + } catch (Exception e) { redisUtil.delete("recoverLogFile"); e.printStackTrace(); @@ -233,6 +238,7 @@ public class AuditServiceImpl extends ServiceImpl implem } @Override + @Transactional(rollbackFor = {Exception.class}) public void recoverLogFile() { String logFileWriter = redisUtil.getStringByKey("recoverLogFile"); if (StrUtil.isNotBlank(logFileWriter) || ObjectUtil.equals(logFileWriter, "1")) { @@ -246,20 +252,23 @@ public class AuditServiceImpl extends ServiceImpl implem FileInputStream in = null; try { in = new FileInputStream(newFile); - String name = newFile.getName(); - String[] split = name.split(ExcelTypeEnum.XLSX.getValue()); - - String[] splitTime = split[0].split("至"); - if (StrUtil.isBlank(splitTime[0]) || StrUtil.isBlank(splitTime[1])) { - throw new BusinessException(AuditLogEnum.LOG_EXCEPTIONTIME); - } - LambdaQueryWrapper le = new LambdaQueryWrapper() - .ge(UserLog::getCreateTime, DateUtil.beginOfDay(DateUtil.parse(splitTime[0]))) - .le(UserLog::getCreateTime, splitTime[1].replace("_", ":")); - this.remove(le); - EasyExcel.read(newFile, UserLogExcel.class, new DataListener(this.getBaseMapper())) - .excelType(ExcelTypeEnum.XLSX).doReadAll(); + Set logIndex = new HashSet<>(this.baseMapper.getAllIndex()); + List userLogs = new ArrayList<>(); + EasyExcel.read(newFile.getPath(), UserLog.class, new AnalysisEventListener() { + @Override + public void invoke(UserLog userLog, AnalysisContext analysisContext) { + // 将读取到的每一行存入reportDetails集合中 + if (!logIndex.contains(userLog.getId())) { + userLogs.add(userLog); + } + } + @Override + public void doAfterAllAnalysed(AnalysisContext analysisContext) { + } + }).sheet().doRead(); + // 执行批量插入 + saveLogsBatch(userLogs, 1500); } catch (Exception e) { redisUtil.delete("recoverLogFile"); e.printStackTrace(); @@ -271,6 +280,41 @@ public class AuditServiceImpl extends ServiceImpl implem } + /*** + * 批量插入日志 + * @author hongawen + * @date 2023/5/30 15:56 + * @return boolean + */ + public boolean saveLogsBatch(Collection userLogs, int batchSize) { + try { + int size = userLogs.size(); + int idxLimit = Math.min(batchSize, size); + List userLogTemp = new ArrayList<>(userLogs); + //保存单批提交的数据集合 + if (idxLimit == batchSize) { + int times = size / idxLimit + 1; + for (int i = 1; i <= times; i++) { + if (size > idxLimit) { + List temp = userLogTemp.subList(0, idxLimit); + this.baseMapper.insertBatchSomeColumn(temp); + temp.clear(); + size = size - idxLimit; + } else { + this.baseMapper.insertBatchSomeColumn(userLogTemp); + } + } + } else { + this.baseMapper.insertBatchSomeColumn(userLogTemp); + } + } catch (Exception e) { + log.error("saveBatch fail", e); + return false; + } + return true; + } + + @Override public Page censusAuditLog(AuditParam auditParam) { //待分页数据总量 @@ -314,7 +358,7 @@ public class AuditServiceImpl extends ServiceImpl implem for (User datum : data) { vo = BeanUtil.copyProperties(datum, OnlineUsersVO.class); Long integer = time.get(datum.getId()); - DateTime date = DateUtil.date(integer*1000); + DateTime date = DateUtil.date(integer * 1000); vo.setTime(date.toDateStr()); info.add(vo); } @@ -350,7 +394,7 @@ public class AuditServiceImpl extends ServiceImpl implem * 返回一个最新修改日期的文件 */ public File getLastFile() { - File parentFile = new File(generalInfo.getBusinessTempPath()+"/"+ OssPath.LOGBAK); + File parentFile = new File(generalInfo.getBusinessTempPath() + File.separator + OssPath.LOGBAK); //文件夹下的所有子文件数组 File[] files = parentFile.listFiles();