Merge remote-tracking branch 'origin/master'

pqs-common/common-db/src/main/java/com/njcn/db/config/MybatisConfig.java

@@ -4,6 +4,7 @@ import com.baomidou.mybatisplus.annotation.DbType;
 import com.baomidou.mybatisplus.extension.plugins.MybatisPlusInterceptor;
 import com.baomidou.mybatisplus.extension.plugins.inner.PaginationInnerInterceptor;
 import com.njcn.db.handler.AutoFillValueHandler;
+import com.njcn.db.handler.BatchInjector;
 import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.Configuration;
 
@@ -34,4 +35,17 @@ public class MybatisConfig {
         return new AutoFillValueHandler();
     }
 
+
+    /***
+     * 自定义注册器，处理批量插入
+     * @author hongawen
+     * @date 2023/5/30 14:53
+     * @return BatchInjector
+     */
+    @Bean
+    public BatchInjector sqlInjector() {
+        return new BatchInjector();
+    }
+
+
 }

pqs-common/common-db/src/main/java/com/njcn/db/handler/BatchInjector.java

@@ -0,0 +1,25 @@
+package com.njcn.db.handler;
+
+import com.baomidou.mybatisplus.annotation.FieldFill;
+import com.baomidou.mybatisplus.core.injector.AbstractMethod;
+import com.baomidou.mybatisplus.core.injector.DefaultSqlInjector;
+import com.baomidou.mybatisplus.extension.injector.methods.InsertBatchSomeColumn;
+
+import java.util.List;
+
+/**
+ * @author hongawen
+ * @version 1.0.0
+ * @date 2023年05月30日 14:51
+ */
+public class BatchInjector extends DefaultSqlInjector {
+
+    @Override
+    public List<AbstractMethod> getMethodList(Class<?> mapperClass) {
+        List<AbstractMethod> methodList = super.getMethodList(mapperClass);
+        //更新时自动填充的字段，不用插入值
+        methodList.add(new InsertBatchSomeColumn(i -> i.getFieldFill() != FieldFill.UPDATE));
+        return methodList;
+    }
+
+}

pqs-common/common-db/src/main/java/com/njcn/db/mapper/BatchMapper.java

@@ -0,0 +1,19 @@
+package com.njcn.db.mapper;
+
+import com.baomidou.mybatisplus.core.mapper.BaseMapper;
+
+import java.util.List;
+
+/**
+ * @author hongawen
+ * @version 1.0.0
+ * @date 2023年05月30日 14:54
+ */
+public interface BatchMapper<T> extends BaseMapper<T> {
+
+    /**
+     * 真正的批量插入
+     */
+    int insertBatchSomeColumn(List<T> entityList);
+
+}

pqs-system/system-api/pom.xml

@@ -43,6 +43,11 @@
             <groupId>com.github.jeffreyning</groupId>
             <artifactId>mybatisplus-plus</artifactId>
         </dependency>
+        <dependency>
+            <groupId>com.alibaba</groupId>
+            <artifactId>easyexcel</artifactId>
+            <version>${easyexcel}</version>
+        </dependency>
     </dependencies>
 
 </project>

pqs-system/system-api/src/main/java/com/njcn/system/pojo/po/UserLog.java

@@ -1,5 +1,6 @@
 package com.njcn.system.pojo.po;
 
+import com.alibaba.excel.annotation.ExcelProperty;
 import com.baomidou.mybatisplus.annotation.FieldFill;
 import com.baomidou.mybatisplus.annotation.FieldStrategy;
 import com.baomidou.mybatisplus.annotation.TableField;
@@ -23,70 +24,83 @@ public class UserLog {
     /**
      * 事件日志Id
      */
+    @ExcelProperty(value = "id主键")
     private String id;
 
     /**
      * 登录名
      */
     @TableField(insertStrategy = FieldStrategy.IGNORED)
+    @ExcelProperty(value = "登录名")
     private String loginName;
 
     /**
      * 用户已登录：用户名
      */
     @TableField(insertStrategy = FieldStrategy.IGNORED)
+    @ExcelProperty(value = "用户名")
     private String userName;
 
     /**
      * 操作Ip
      */
+    @ExcelProperty(value = "操作ip")
     private String ip;
 
     /**
      * 操作内容
      */
+    @ExcelProperty(value = "操作内容")
     private String operate;
 
     /**
      * 操作类型 比如：查询、新增、删除等等
      */
+    @ExcelProperty(value = "操作类型")
     private String operateType;
 
     /**
      * 操作结果 0.失败 1.成功
      */
+    @ExcelProperty(value = "操作结果")
     private Integer result;
 
     /**
      * 失败原因
      */
     @TableField(insertStrategy = FieldStrategy.IGNORED)
+    @ExcelProperty(value = "失败原因")
     private String failReason;
 
     /**
      * 严重度 0.普通 1.中等 2.严重
      */
+    @ExcelProperty(value = "时间等级(严重度)")
     private Integer level;
 
     /**
      * 事件类型 0.业务事件 1.系统事件
      */
+    @ExcelProperty(value = "事件类型")
     private Integer type;
 
     /**
      * 模块名
      */
+    @ExcelProperty(value = "模块名称")
     private String serviceName;
 
     /**
      * 告警标志 0.未告警 1.已告警
      */
+    @ExcelProperty(value = "告警标志")
     private Integer state;
 
     /**
      * 创建用户
      */
     @TableField(fill = FieldFill.INSERT, insertStrategy = FieldStrategy.IGNORED)
+    @ExcelProperty(value = "创建用户")
     private String createBy;
 
 
@@ -95,12 +109,14 @@ public class UserLog {
      */
     @TableField(fill = FieldFill.INSERT, insertStrategy = FieldStrategy.IGNORED)
     @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss")
+    @ExcelProperty(value = "创建时间" )
     private LocalDateTime createTime;
 
     /**
      * 更新用户
      */
     @TableField(fill = FieldFill.INSERT_UPDATE, insertStrategy = FieldStrategy.IGNORED)
+    @ExcelProperty(value = "更新用户")
     private String updateBy;
 
     /**
@@ -108,6 +124,7 @@ public class UserLog {
      */
     @TableField(fill = FieldFill.INSERT_UPDATE, insertStrategy = FieldStrategy.IGNORED)
     @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss")
+    @ExcelProperty(value = "更新时间" )
     private LocalDateTime updateTime;
 
 

pqs-system/system-boot/pom.xml

@@ -58,11 +58,7 @@
             <version>1.0.0</version>
             <scope>compile</scope>
         </dependency>
-        <dependency>
+
-            <groupId>com.alibaba</groupId>
-            <artifactId>easyexcel</artifactId>
-            <version>${easyexcel}</version>
-        </dependency>
     </dependencies>
 
     <build>

pqs-system/system-boot/src/main/java/com/njcn/system/mapper/UserLogMapper.java

@@ -1,11 +1,14 @@
 package com.njcn.system.mapper;
 
-import com.baomidou.mybatisplus.core.mapper.BaseMapper;
+import com.njcn.db.mapper.BatchMapper;
 import com.njcn.system.excel.UserLogExcel;
 import com.njcn.system.pojo.po.UserLog;
 import org.apache.ibatis.annotations.Param;
+
+import java.util.HashSet;
 import java.util.List;
 
+
 /**
  * <p>
  *  Mapper 接口
@@ -14,7 +17,7 @@ import java.util.List;
  * @author hongawen
  * @since 2021-12-13
  */
-public interface UserLogMapper extends BaseMapper<UserLog> {
+public interface UserLogMapper extends BatchMapper<UserLog> {
 
 
     /**
@@ -36,4 +39,12 @@ public interface UserLogMapper extends BaseMapper<UserLog> {
      * @return
      */
     Float getMemoInfo(@Param("schema") String schema);
+
+    /***
+     * 查询表中所有索引
+     * @author hongawen
+     * @date 2023/5/31 15:56
+     * @return List<String>
+     */
+    List<String> getAllIndex();
 }

pqs-system/system-boot/src/main/java/com/njcn/system/mapper/mapping/UserLogMapper.xml

@@ -57,4 +57,11 @@
         where table_schema = #{schema}  and table_name="sys_user_log"
     </select>
 
+    <select id="getAllIndex" resultType="String">
+        SELECT
+            id
+        FROM
+            sys_user_log
+    </select>
+
 </mapper>

pqs-system/system-boot/src/main/java/com/njcn/system/service/impl/AuditServiceImpl.java

@@ -5,12 +5,15 @@ import cn.hutool.core.bean.BeanUtil;
 import cn.hutool.core.collection.CollUtil;
 import cn.hutool.core.date.DateTime;
 import cn.hutool.core.date.DateUtil;
+import cn.hutool.core.date.StopWatch;
 import cn.hutool.core.io.FileUtil;
 import cn.hutool.core.util.ObjectUtil;
 import cn.hutool.core.util.StrUtil;
 import cn.hutool.json.JSONUtil;
 import com.alibaba.excel.EasyExcel;
 import com.alibaba.excel.ExcelWriter;
+import com.alibaba.excel.context.AnalysisContext;
+import com.alibaba.excel.event.AnalysisEventListener;
 import com.alibaba.excel.support.ExcelTypeEnum;
 import com.alibaba.excel.write.metadata.WriteSheet;
 import com.alibaba.excel.write.style.column.LongestMatchColumnWidthStyleStrategy;
@@ -43,6 +46,7 @@ import org.apache.commons.fileupload.disk.DiskFileItemFactory;
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.http.MediaType;
 import org.springframework.stereotype.Service;
+import org.springframework.transaction.annotation.Transactional;
 import org.springframework.web.multipart.MultipartFile;
 import org.springframework.web.multipart.commons.CommonsMultipartFile;
 
@@ -79,34 +83,34 @@ public class AuditServiceImpl extends ServiceImpl<UserLogMapper, UserLog> implem
 
         Page<UserLog> info = this.page(new Page<>(auditParam.getPageNum(), auditParam.getPageSize()), new LambdaQueryWrapper<UserLog>()
                 .like(StrUtil.isNotBlank(auditParam.getLoginName()), UserLog::getLoginName, auditParam.getLoginName())
-                .ne(UserLog::getLoginName,"unknown user")
+                .ne(UserLog::getLoginName, "unknown user")
-                .ne(UserLog::getLoginName,"")
+                .ne(UserLog::getLoginName, "")
-                .ne(UserLog::getOperate,"unknown user")
+                .ne(UserLog::getOperate, "unknown user")
                 .eq(auditParam.getType() != null, UserLog::getType, auditParam.getType())
                 .eq(StrUtil.isNotBlank(auditParam.getOperateType()), UserLog::getOperateType, auditParam.getOperateType())
                 .ge(StrUtil.isNotBlank(auditParam.getSearchBeginTime()), UserLog::getCreateTime, DateUtil.beginOfDay(DateUtil.parse(auditParam.getSearchBeginTime())))
                 .le(StrUtil.isNotBlank(auditParam.getSearchEndTime()), UserLog::getCreateTime, DateUtil.endOfDay(DateUtil.parse(auditParam.getSearchEndTime())))
-                .or(StrUtil.isNotBlank(auditParam.getLoginName()),qw ->
+                .or(StrUtil.isNotBlank(auditParam.getLoginName()), qw ->
                         qw.like(StrUtil.isNotBlank(auditParam.getLoginName()), UserLog::getIp, auditParam.getLoginName())
-                        .ne(UserLog::getLoginName,"unknown user")
+                                .ne(UserLog::getLoginName, "unknown user")
-                        .ne(UserLog::getLoginName,"")
+                                .ne(UserLog::getLoginName, "")
-                        .ne(UserLog::getOperate,"unknown user")
+                                .ne(UserLog::getOperate, "unknown user")
-                        .eq(auditParam.getType() != null, UserLog::getType, auditParam.getType())
-                        .eq(StrUtil.isNotBlank(auditParam.getOperateType()), UserLog::getOperateType, auditParam.getOperateType())
-                        .ge(StrUtil.isNotBlank(auditParam.getSearchBeginTime()), UserLog::getCreateTime, DateUtil.beginOfDay(DateUtil.parse(auditParam.getSearchBeginTime())))
-                        .le(StrUtil.isNotBlank(auditParam.getSearchEndTime()), UserLog::getCreateTime, DateUtil.endOfDay(DateUtil.parse(auditParam.getSearchEndTime())))
-                )
-                .or(StrUtil.isNotBlank(auditParam.getLoginName()),qw ->
-                        qw.like(StrUtil.isNotBlank(auditParam.getLoginName()), UserLog::getCreateTime, auditParam.getLoginName())
-                                .ne(UserLog::getLoginName,"unknown user")
-                                .ne(UserLog::getLoginName,"")
-                                .ne(UserLog::getOperate,"unknown user")
                                 .eq(auditParam.getType() != null, UserLog::getType, auditParam.getType())
                                 .eq(StrUtil.isNotBlank(auditParam.getOperateType()), UserLog::getOperateType, auditParam.getOperateType())
                                 .ge(StrUtil.isNotBlank(auditParam.getSearchBeginTime()), UserLog::getCreateTime, DateUtil.beginOfDay(DateUtil.parse(auditParam.getSearchBeginTime())))
                                 .le(StrUtil.isNotBlank(auditParam.getSearchEndTime()), UserLog::getCreateTime, DateUtil.endOfDay(DateUtil.parse(auditParam.getSearchEndTime())))
                 )
-                .orderByDesc( UserLog::getCreateTime)
+                .or(StrUtil.isNotBlank(auditParam.getLoginName()), qw ->
+                        qw.like(StrUtil.isNotBlank(auditParam.getLoginName()), UserLog::getCreateTime, auditParam.getLoginName())
+                                .ne(UserLog::getLoginName, "unknown user")
+                                .ne(UserLog::getLoginName, "")
+                                .ne(UserLog::getOperate, "unknown user")
+                                .eq(auditParam.getType() != null, UserLog::getType, auditParam.getType())
+                                .eq(StrUtil.isNotBlank(auditParam.getOperateType()), UserLog::getOperateType, auditParam.getOperateType())
+                                .ge(StrUtil.isNotBlank(auditParam.getSearchBeginTime()), UserLog::getCreateTime, DateUtil.beginOfDay(DateUtil.parse(auditParam.getSearchBeginTime())))
+                                .le(StrUtil.isNotBlank(auditParam.getSearchEndTime()), UserLog::getCreateTime, DateUtil.endOfDay(DateUtil.parse(auditParam.getSearchEndTime())))
+                )
+                .orderByDesc(UserLog::getCreateTime)
         );
         Page<AuditLogVO> page = BeanUtil.copyProperties(info, Page.class);
         if (CollUtil.isNotEmpty(info.getRecords())) {
@@ -164,12 +168,12 @@ public class AuditServiceImpl extends ServiceImpl<UserLogMapper, UserLog> implem
         aa.ge("sys_user_log.update_time", date);
         aa.le("sys_user_log.update_time", endTime);
         String nowTime = date + "至" + endTime;
-        File file=new File(generalInfo.getBusinessTempPath() + "/" + OssPath.LOGBAK);
+        File file = new File(generalInfo.getBusinessTempPath() + File.separator + OssPath.LOGBAK);
-        if(!file.exists() && !file .isDirectory()){
+        if (!file.exists() && !file.isDirectory()) {
             file.mkdir();
         }
         //必须放到循环外，否则会刷新流
-        ExcelWriter excelWriter = EasyExcel.write(generalInfo.getBusinessTempPath() + "/"+ OssPath.LOGBAK + nowTime + ExcelTypeEnum.XLSX.getValue(), UserLogExcel.class)
+        ExcelWriter excelWriter = EasyExcel.write(generalInfo.getBusinessTempPath() + File.separator + OssPath.LOGBAK + nowTime + ExcelTypeEnum.XLSX.getValue(), UserLogExcel.class)
 //                .excelType(ExcelTypeEnum.CSV)
                 .build();
 
@@ -207,6 +211,7 @@ public class AuditServiceImpl extends ServiceImpl<UserLogMapper, UserLog> implem
                     excelWriter.write(list, writeSheet);
                 }
             }
+
         } catch (Exception e) {
             redisUtil.delete("recoverLogFile");
             e.printStackTrace();
@@ -233,6 +238,7 @@ public class AuditServiceImpl extends ServiceImpl<UserLogMapper, UserLog> implem
     }
 
     @Override
+    @Transactional(rollbackFor = {Exception.class})
     public void recoverLogFile() {
         String logFileWriter = redisUtil.getStringByKey("recoverLogFile");
         if (StrUtil.isNotBlank(logFileWriter) || ObjectUtil.equals(logFileWriter, "1")) {
@@ -246,20 +252,23 @@ public class AuditServiceImpl extends ServiceImpl<UserLogMapper, UserLog> implem
         FileInputStream in = null;
         try {
             in = new FileInputStream(newFile);
-            String name = newFile.getName();
+            Set<String> logIndex = new HashSet<>(this.baseMapper.getAllIndex());
-            String[] split = name.split(ExcelTypeEnum.XLSX.getValue());
+            List<UserLog> userLogs = new ArrayList<>();
-
+            EasyExcel.read(newFile.getPath(), UserLog.class, new AnalysisEventListener<UserLog>() {
-            String[] splitTime = split[0].split("至");
+                @Override
-            if (StrUtil.isBlank(splitTime[0]) || StrUtil.isBlank(splitTime[1])) {
+                public void invoke(UserLog userLog, AnalysisContext analysisContext) {
-                throw new BusinessException(AuditLogEnum.LOG_EXCEPTIONTIME);
+                    // 将读取到的每一行存入reportDetails集合中
-            }
+                    if (!logIndex.contains(userLog.getId())) {
-            LambdaQueryWrapper<UserLog> le = new LambdaQueryWrapper<UserLog>()
+                        userLogs.add(userLog);
-                    .ge(UserLog::getCreateTime, DateUtil.beginOfDay(DateUtil.parse(splitTime[0])))
+                    }
-                    .le(UserLog::getCreateTime, splitTime[1].replace("_", ":"));
+                }
-            this.remove(le);
-            EasyExcel.read(newFile, UserLogExcel.class, new DataListener(this.getBaseMapper()))
-                    .excelType(ExcelTypeEnum.XLSX).doReadAll();
 
+                @Override
+                public void doAfterAllAnalysed(AnalysisContext analysisContext) {
+                }
+            }).sheet().doRead();
+            // 执行批量插入
+            saveLogsBatch(userLogs, 1500);
         } catch (Exception e) {
             redisUtil.delete("recoverLogFile");
             e.printStackTrace();
@@ -271,6 +280,41 @@ public class AuditServiceImpl extends ServiceImpl<UserLogMapper, UserLog> implem
     }
 
 
+    /***
+     * 批量插入日志
+     * @author hongawen
+     * @date 2023/5/30 15:56
+     * @return boolean
+     */
+    public boolean saveLogsBatch(Collection<UserLog> userLogs, int batchSize) {
+        try {
+            int size = userLogs.size();
+            int idxLimit = Math.min(batchSize, size);
+            List<UserLog> userLogTemp = new ArrayList<>(userLogs);
+            //保存单批提交的数据集合
+            if (idxLimit == batchSize) {
+                int times = size / idxLimit + 1;
+                for (int i = 1; i <= times; i++) {
+                    if (size > idxLimit) {
+                        List<UserLog> temp = userLogTemp.subList(0, idxLimit);
+                        this.baseMapper.insertBatchSomeColumn(temp);
+                        temp.clear();
+                        size = size - idxLimit;
+                    } else {
+                        this.baseMapper.insertBatchSomeColumn(userLogTemp);
+                    }
+                }
+            } else {
+                this.baseMapper.insertBatchSomeColumn(userLogTemp);
+            }
+        } catch (Exception e) {
+            log.error("saveBatch fail", e);
+            return false;
+        }
+        return true;
+    }
+
+
     @Override
     public Page<AuditLogCusVO> censusAuditLog(AuditParam auditParam) {
         //待分页数据总量
@@ -314,7 +358,7 @@ public class AuditServiceImpl extends ServiceImpl<UserLogMapper, UserLog> implem
             for (User datum : data) {
                 vo = BeanUtil.copyProperties(datum, OnlineUsersVO.class);
                 Long integer = time.get(datum.getId());
-                DateTime date = DateUtil.date(integer*1000);
+                DateTime date = DateUtil.date(integer * 1000);
                 vo.setTime(date.toDateStr());
                 info.add(vo);
             }
@@ -350,7 +394,7 @@ public class AuditServiceImpl extends ServiceImpl<UserLogMapper, UserLog> implem
      * 返回一个最新修改日期的文件
      */
     public File getLastFile() {
-        File parentFile = new File(generalInfo.getBusinessTempPath()+"/"+ OssPath.LOGBAK);
+        File parentFile = new File(generalInfo.getBusinessTempPath() + File.separator + OssPath.LOGBAK);
 
         //文件夹下的所有子文件数组
         File[] files = parentFile.listFiles();