Other/data-migration
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

添加数据同步代码

Browse Source
This commit is contained in:
hzj
2024-01-10 10:40:02 +08:00
parent 4702c266a4
commit ff80efa279
8 changed files with 94 additions and 51 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
oracle-data/oracle-source/src/main/java/com/njcn/oracle/bo/param/ServiceTypeEnum.java
View File

@@ -57,9 +57,12 @@ public enum ServiceTypeEnum {
* @return
*/
public static List<String> getExecutableTypes() {
return Arrays.stream(ServiceTypeEnum.values()).map(ServiceTypeEnum::name).collect(Collectors.toList());
return Arrays.stream(ServiceTypeEnum.values()).map(tmep->{
return tmep.code;
}).collect(Collectors.toList());
}
/**
* 通过code获取枚举值
* @param code

32
oracle-data/oracle-source/src/main/java/com/njcn/oracle/bo/po/JobDetail.java
View File

@@ -9,7 +9,9 @@ import com.github.jeffreyning.mybatisplus.anno.MppMultiId;
import lombok.Data;
import java.math.BigDecimal;
import java.time.LocalDate;
import java.time.LocalDateTime;
import java.util.Objects;
/**
*
@@ -34,22 +36,14 @@ public class JobDetail {
* 起始时间
*/
@JsonFormat(
pattern = "yyyy-MM-dd HH:mm:ss"
pattern = "yyyy-MM-dd"
)
@MppMultiId(value = "START_TIME")
private LocalDateTime startTime;
@MppMultiId(value = "EXCUTE_DATE")
private LocalDate excuteDate;
/**
* 截止时间
*/
@JsonFormat(
pattern = "yyyy-MM-dd HH:mm:ss"
)
@MppMultiId(value = "END_TIME")
private LocalDateTime endTime;
/**
* 状态（0-执行中、1-成功、2-失败）
* 状态（0-执行中、1-成功、2-失败,3未执行）
*/
@TableField(value = "STATE")
private Integer state;
@@ -65,5 +59,17 @@ public class JobDetail {
@TableField(value = "UPDATE_TIME")
private LocalDateTime updateTime;
@Override
public boolean equals(Object o) {
if (this == o) {
return true;
}
if (o == null || getClass() != o.getClass()) {
return false;
}
JobDetail jobDetail = (JobDetail) o;
return Objects.equals(tableName, jobDetail.tableName) && Objects.equals(excuteDate, jobDetail.excuteDate);
}
}

7
oracle-data/oracle-source/src/main/java/com/njcn/oracle/mapper/mapping/JobDetailMapper.xml
View File

@@ -5,8 +5,7 @@
<!--@mbg.generated-->
<!--@Table JOB_DETAIL-->
<id column="TABLE_NAME" jdbcType="VARCHAR" property="tableName" />
<id column="START_TIME" jdbcType="TIMESTAMP" property="startTime" />
<id column="END_TIME" jdbcType="TIMESTAMP" property="endTime" />
<id column="EXCUTE_DATE" jdbcType="TIMESTAMP" property="excuteDate" />
<id column="STATE" jdbcType="DECIMAL" property="state" />
<result column="ROW_COUNT" jdbcType="DECIMAL" property="rowCount" />
<result column="UPDATE_TIME" jdbcType="TIMESTAMP" property="updateTime" />
@@ -19,7 +18,7 @@
<select id="query" resultMap="BaseResultMap">
select * from ( select row_.*, rownum rownum_ from (
select * from JOB_DETAIL a where 1=1
and a.START_TIME between #{jobQueryParam.startTime,jdbcType=DATE} and #{jobQueryParam.endTime,jdbcType=DATE}
and a.EXCUTE_DATE between #{jobQueryParam.startTime,jdbcType=DATE} and #{jobQueryParam.endTime,jdbcType=DATE}
<if test="jobQueryParam.states!=null and jobQueryParam.states.size()!=0">
AND a.STATE IN
<foreach collection="jobQueryParam.states" item="item" open="(" close=")" separator=",">
@@ -32,7 +31,7 @@
#{item}
</foreach>
</if>
order by a.START_TIME desc
order by a.EXCUTE_DATE desc
) row_
where rownum &lt;= #{jobQueryParam.currentPage} * #{jobQueryParam.pageSize} ) where rownum_ &gt; (#{jobQueryParam.currentPage} - 1) * #{jobQueryParam.pageSize}
</select>

5
oracle-data/oracle-source/src/main/java/com/njcn/oracle/service/DataSyncService.java
View File

@@ -1,8 +1,9 @@
package com.njcn.oracle.service;
import com.njcn.oracle.bo.param.MigrationParam;
import com.njcn.oracle.mybatis.service.IReplenishMybatisService;
import java.time.LocalDate;
/**
* Description:
* Date: 2024/1/8 16:16【需求编号】
@@ -11,5 +12,5 @@ import com.njcn.oracle.mybatis.service.IReplenishMybatisService;
* @version V1.0.0
*/
public interface DataSyncService {
void dataSync(IReplenishMybatisService executor, String temp, MigrationParam migration);
void dataSync(IReplenishMybatisService executor, String temp, LocalDate migration);
}

15
oracle-data/oracle-source/src/main/java/com/njcn/oracle/service/impl/DataSyncServiceImpl.java
View File

@@ -1,6 +1,8 @@
package com.njcn.oracle.service.impl;
import cn.hutool.core.collection.CollectionUtil;
import cn.hutool.core.date.DatePattern;
import cn.hutool.core.date.LocalDateTimeUtil;
import com.njcn.oracle.bo.param.MigrationParam;
import com.njcn.oracle.bo.po.JobDetail;
import com.njcn.oracle.mybatis.service.IReplenishMybatisService;
@@ -10,8 +12,8 @@ import lombok.RequiredArgsConstructor;
import org.springframework.scheduling.annotation.Async;
import org.springframework.stereotype.Service;
import java.time.LocalDate;
import java.time.LocalDateTime;
import java.time.temporal.ChronoUnit;
import java.util.List;
import java.util.Objects;
@@ -29,16 +31,21 @@ public class DataSyncServiceImpl implements DataSyncService {
@Override
@Async
public void dataSync(IReplenishMybatisService executor, String temp, MigrationParam migration) {
public void dataSync(IReplenishMybatisService executor, String temp, LocalDate date) {
LocalDateTime localDateTime = date.atStartOfDay();
LocalDateTime tempStartTime = LocalDateTimeUtil.beginOfDay(localDateTime);
LocalDateTime tempEndTime = LocalDateTimeUtil.endOfDay(localDateTime);
MigrationParam migration = new MigrationParam();
migration.setStartTime(tempStartTime);
migration.setEndTime(tempEndTime);
List list = executor.queryData(migration);
JobDetail jobDetail = new JobDetail();
jobDetail.setTableName(temp);
jobDetail.setStartTime(migration.getStartTime());
jobDetail.setEndTime(migration.getEndTime());
jobDetail.setExcuteDate(date);
JobDetail select = jobDetailService.select(jobDetail);
if (Objects.nonNull(select)){

47
oracle-data/oracle-source/src/main/java/com/njcn/oracle/service/impl/JobDetailServiceImpl.java
View File

@@ -7,10 +7,16 @@ import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
import com.github.jeffreyning.mybatisplus.service.MppServiceImpl;
import com.njcn.oracle.bo.param.JobQueryParam;
import com.njcn.oracle.bo.param.ServiceTypeEnum;
import com.njcn.oracle.util.LocalDateUtil;
import org.apache.commons.lang3.StringUtils;
import org.springframework.stereotype.Service;
import javax.annotation.Resource;
import java.time.LocalDate;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.List;
import java.util.stream.Collectors;
import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
import com.njcn.oracle.mybatis.mapper.JobDetailMapper;
import com.njcn.oracle.bo.po.JobDetail;
@@ -33,8 +39,7 @@ public class JobDetailServiceImpl extends MppServiceImpl<JobDetailMapper, JobDet
@Override
public JobDetail select(JobDetail jobDetail) {
JobDetail jobDetail1 = this.lambdaQuery().eq(JobDetail::getTableName,jobDetail.getTableName()).
eq(JobDetail::getStartTime,jobDetail.getStartTime()).
eq(JobDetail::getEndTime,jobDetail.getEndTime()).one();
eq(JobDetail::getExcuteDate,jobDetail.getExcuteDate()).one();
return jobDetail1;
}
@@ -42,16 +47,41 @@ public class JobDetailServiceImpl extends MppServiceImpl<JobDetailMapper, JobDet
public IPage<JobDetail> selectByParam(JobQueryParam jobQueryParam) {
IPage<JobDetail> page = new Page<>(jobQueryParam.getCurrentPage(), jobQueryParam.getPageSize());
QueryWrapper<JobDetail> queryWrapper = new QueryWrapper<>();
queryWrapper.lambda().between(JobDetail::getStartTime, jobQueryParam.getStartTime(), jobQueryParam.getEndTime()).
queryWrapper.lambda().between(JobDetail::getExcuteDate, jobQueryParam.getStartTime(), jobQueryParam.getEndTime()).
in(!CollectionUtils.isEmpty(jobQueryParam.getStates()), JobDetail::getState, jobQueryParam.getStates()).
in(!CollectionUtils.isEmpty(jobQueryParam.getTableNames()), JobDetail::getTableName, jobQueryParam.getTableNames()).orderByDesc(JobDetail::getStartTime);
in(!CollectionUtils.isEmpty(jobQueryParam.getTableNames()), JobDetail::getTableName, jobQueryParam.getTableNames());
Integer integer = this.getBaseMapper().selectCount(queryWrapper);
List<JobDetail> jobDetails = this.getBaseMapper().selectList(queryWrapper);
List<String> tableNames = new ArrayList<>();
if(CollectionUtils.isEmpty(jobQueryParam.getTableNames())){
tableNames = ServiceTypeEnum.getExecutableTypes();
}else {
tableNames = jobQueryParam.getTableNames();
}
List<LocalDate> dateList = LocalDateUtil.getDateList(jobQueryParam.getStartTime(), jobQueryParam.getEndTime());
List<String> finalTableNames = tableNames;
dateList.stream().forEach(localDate -> {
finalTableNames.stream().forEach(temp->{
JobDetail jobDetail = new JobDetail();
jobDetail.setTableName(temp);
jobDetail.setExcuteDate(localDate);
if (jobDetails.contains(jobDetail)){
return;
}
jobDetail.setTableName_CN(ServiceTypeEnum.getValueByCode(temp));
jobDetail.setState(3);
jobDetails.add(jobDetail);
});
});
List<JobDetail> jobDetailIPage = this.getBaseMapper().query(jobQueryParam);
List<JobDetail> jobDetailIPage= jobDetails.stream().sorted(Comparator.comparing(JobDetail::getExcuteDate).reversed()).
skip((jobQueryParam.getCurrentPage() - 1) * jobQueryParam.getPageSize()).
limit(jobQueryParam.getPageSize()).collect(Collectors.toList());
jobDetailIPage.stream().forEach(temp->temp.setTableName_CN(ServiceTypeEnum.getValueByCode(temp.getTableName())));
page.setRecords(jobDetailIPage);
page.setTotal(Long.parseLong(integer+""));
page.setTotal(Long.parseLong(jobDetails.size()+""));
return page;
}
@@ -59,8 +89,7 @@ public class JobDetailServiceImpl extends MppServiceImpl<JobDetailMapper, JobDet
public boolean jobRemove(JobDetail jobDetail) {
QueryWrapper<JobDetail> queryWrapper = new QueryWrapper<>();
queryWrapper.lambda().eq(JobDetail::getTableName,jobDetail.getTableName()).
eq(JobDetail::getStartTime,jobDetail.getStartTime()).
eq(JobDetail::getEndTime,jobDetail.getEndTime());
eq(JobDetail::getExcuteDate,jobDetail.getExcuteDate());
boolean remove = this.remove(queryWrapper);
return remove;
}

12
oracle-data/oracle-target/src/main/java/com/njcn/oracle/controller/DataSyncController.java
View File

@@ -64,20 +64,10 @@ public class DataSyncController {
throw new RuntimeException(e);
}
List<LocalDate> dateList = LocalDateUtil.getDateList(dataAsynParam.getStartTime(), dataAsynParam.getEndTime());
LocalDateTime startTime = LocalDateTimeUtil.beginOfDay(dataAsynParam.getStartTime().atStartOfDay());
LocalDateTime endTime = LocalDateTimeUtil.endOfDay(dataAsynParam.getEndTime().atStartOfDay());
IReplenishMybatisService finalExecutor = executor;
dateList.forEach(date->{
LocalDateTime localDateTime = date.atStartOfDay();
LocalDateTime tempStartTime = LocalDateTimeUtil.beginOfDay(localDateTime);
LocalDateTime tempEndTime = LocalDateTimeUtil.endOfDay(localDateTime);
tempStartTime =LocalDateTimeUtil.parse(tempStartTime.format(formatter),DatePattern.NORM_DATETIME_PATTERN);
tempEndTime =LocalDateTimeUtil.parse(tempEndTime.format(formatter),DatePattern.NORM_DATETIME_PATTERN);
MigrationParam migration = new MigrationParam();
migration.setStartTime(tempStartTime);
migration.setEndTime(tempEndTime);
dataSyncService.dataSync(finalExecutor,temp,migration);
dataSyncService.dataSync(finalExecutor,temp,date);
});

22
oracle-data/oracle-target/src/main/resources/static/index.html
View File

@@ -70,6 +70,7 @@
<el-option label="执行中" value="0"></el-option>
<el-option label="成功" value="1"></el-option>
<el-option label="失败" value="2"></el-option>
<el-option label="未执行" value="3"></el-option>
</el-select>
</el-form-item>
<el-form-item>
@@ -80,8 +81,8 @@
<el-table :data="tableData" style="width: 100%; height: calc(100vh - 120px)" stripe align="center" v-loading="formInline.loading">
<el-table-column prop="tableName" label="指标名"></el-table-column>
<el-table-column prop="tableName_CN" label="表名"></el-table-column>
<el-table-column prop="startTime" label="日期(年月日)">
<template #default="scope"> {{scope.row.startTime&&scope.row.startTime.split(' ')[0]}} </template>
<el-table-column prop="excuteDate" label="日期(年月日)">
<!-- <template #default="scope"> {{scope.row.startTime&&scope.row.startTime.split(' ')[0]}} </template>-->
</el-table-column>
<el-table-column prop="rowCount" label="条目数"></el-table-column>
<el-table-column prop="updateTime" label="执行时间"></el-table-column>
@@ -90,6 +91,7 @@
<el-tag effect="dark" type="danger" v-if="scope.row.state === 2">失败</el-tag>
<el-tag effect="dark" type="primary" v-else-if="scope.row.state === 0">执行中</el-tag>
<el-tag effect="dark" type="success" v-else-if="scope.row.state === 1">成功</el-tag>
<el-tag effect="dark" type="danger" v-else-if="scope.row.state === 3">未执行</el-tag>
</template>
</el-table-column>
<el-table-column prop="name" label="操作">
@@ -99,7 +101,7 @@
<el-button type="danger" text v-if="scope.row.state === 1">删除</el-button>
</template>
</el-popconfirm>
<el-button type="primary" text v-if="scope.row.state === 2" @click="move(scope.row)">迁移</el-button>
<el-button type="primary" text v-if="scope.row.state === 2 || scope.row.state === 3" @click="move(scope.row)">迁移</el-button>
</template>
</el-table-column>
</el-table>
@@ -198,6 +200,13 @@
})
.then((res) => {
console.log(res)
res.records.forEach(item=>{
for(let key in item){
if(!item[key] && item[key] !== 0){
item[key] = '/'
}
}
})
tableData.value = res.records
formInline.total = res.total
})
@@ -222,8 +231,8 @@
if (row) {
obj = {
tableNames: [row.tableName],
startTime: row.startTime.split(' ')[0],
endTime: row.startTime.split(' ')[0],
startTime: row.excuteDate,
endTime: row.excuteDate,
}
}
fetch('/data/dataSync', {
@@ -253,8 +262,7 @@
},
body: JSON.stringify({
tableName: row.tableName,
startTime: row.startTime,
endTime: row.endTime,
excuteDate: row.excuteDate,
}),
})
.then((res) => {