SyncTool/jb_syncdata
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

微调

Browse Source
This commit is contained in:
hongawen
2023-10-19 11:15:01 +08:00
parent 51c7dbede7
commit 24fa9db4d4
4 changed files with 162 additions and 110 deletions
Download Patch File Download Diff File Expand all files Collapse all files

132
src/main/java/com/njcn/jbsyncdata/controller/DisPhotovoltaicController.java
View File

@@ -1,6 +1,9 @@
package com.njcn.jbsyncdata.controller;
import cn.hutool.core.collection.CollectionUtil;
import cn.hutool.core.date.DatePattern;
import cn.hutool.core.date.DateTime;
import cn.hutool.core.date.DateUtil;
import cn.hutool.core.io.file.FileReader;
import cn.hutool.core.text.StrPool;
import cn.hutool.core.util.StrUtil;
@@ -41,7 +44,10 @@ public class DisPhotovoltaicController {
private final IBusinessService businessService;
private final DisPhotovoltaicService disPhotovoltaicService;
@ApiOperation(value = "查询所有用户的遥测数据")
/**
* @param date yyyy-MM-dd
*/
@ApiOperation(value = "查询指定日期所有用户的遥测数据")
@PostMapping("/queryTelemetryData")
public void queryTelemetryData(String date) {
try {
@@ -51,6 +57,30 @@ public class DisPhotovoltaicController {
}
}
/**
* @param startTime 起始时间 yyyy-MM-dd
* @param endTime 截止时间 yyyy-MM-dd
*/
@ApiOperation(value = "查询日期范围内所有用户的遥测数据")
@PostMapping("/queryTelemetryDataTimeRange")
public void queryTelemetryDataTimeRange(String startTime, String endTime) {
try {
DateTime startDate = DateUtil.parse(startTime, DatePattern.NORM_DATE_FORMAT);
DateTime endDate = DateUtil.parse(endTime, DatePattern.NORM_DATE_FORMAT);
long betweenDay = DateUtil.betweenDay(startDate, endDate, true);
//第一天
businessService.queryTelemetryData(DateUtil.format(startDate, DatePattern.NORM_DATE_PATTERN));
//后续递增1
for (int i = 0; i < betweenDay; i++) {
startDate = DateUtil.offsetDay(startDate, 1);
businessService.queryTelemetryData(DateUtil.format(startDate, DatePattern.NORM_DATE_PATTERN));
}
} catch (Exception exception) {
exception.printStackTrace();
}
}
@ApiOperation(value = "导入10kv分布式光伏接入情况", produces = MediaType.APPLICATION_OCTET_STREAM_VALUE)
@PostMapping("/import10KV")
public void import10(MultipartFile file, HttpServletResponse response) throws Exception {
@@ -84,106 +114,6 @@ public class DisPhotovoltaicController {
}
public static void main(String[] args) throws IOException {
//读取10kV的数据
List<DisPhotovoltaic10Excel> list10kV = EasyExcel.read("D:\\temp\\基础表.xlsx")
.excelType(ExcelTypeEnum.XLSX)
.head(DisPhotovoltaic10Excel.class)
.headRowNumber(2)
.sheet(0).doReadSync();
list10kV = list10kV.stream()
.filter(t -> StrUtil.isNotBlank(t.getGenerationUserID()))
.filter(t -> StrUtil.isNotBlank(t.getStageID()))
.filter(StreamUtil.distinctByKey(DisPhotovoltaic10Excel::getGenerationUserID))
.collect(Collectors.toList());
//读取380V的数据
List<DisPhotovoltaic380Excel> list380v = EasyExcel.read("D:\\temp\\基础表.xlsx")
.excelType(ExcelTypeEnum.XLSX)
.head(DisPhotovoltaic380Excel.class)
.headRowNumber(2)
.sheet(1).doReadSync();
list380v = list380v.stream()
.filter(t -> StrUtil.isNotBlank(t.getGenerationUserID()))
.filter(t -> StrUtil.isNotBlank(t.getStageID()))
.filter(StreamUtil.distinctByKey(DisPhotovoltaic380Excel::getGenerationUserID))
.collect(Collectors.toList());
//读取所有没数据的用户号
FileReader fileReader = new FileReader("D:\\temp\\all.txt");
List<String> noData = fileReader.readLines();
Set<String> noDataSet = new HashSet<>(noData);
FileReader fileReader2 = new FileReader("D:\\temp\\part.txt");
List<String> noPartData = fileReader2.readLines();
Set<String> noPartDataSet = new HashSet<>(noPartData);
long millis = System.currentTimeMillis();
//梳理10kV全部没有数据的
List<DisPhotovoltaic10Excel> collect = list10kV.stream().filter(t -> noDataSet.contains(t.getGenerationUserID())).collect(Collectors.toList());
EasyExcel.write("D:\\temp\\10kV全部没有数据的.xlsx", DisPhotovoltaic10Excel.class).sheet("10kV全部没有数据的").doWrite(collect);
long millis1 = System.currentTimeMillis();
System.out.println("10kV全部没有数据的耗时：" + (millis1 - millis));
//10kV部分没有数据的
Map<String, List<String>> noPartDataMap = noPartDataSet.stream().collect(Collectors.groupingBy(t -> t.substring(0, t.indexOf(StrPool.COMMA))));
Map<String, List<DisPhotovoltaic10Excel>> all10kVMap = list10kV.stream().collect(Collectors.groupingBy(DisPhotovoltaic10Excel::getGenerationUserID));
Set<String> keyedSet1 = all10kVMap.keySet();
List<DisPhotovoltaic10Excel> final10kVUserData = new LinkedList<>();
String info;
for (String userId : keyedSet1) {
List<String> infoList = noPartDataMap.get(userId);
if (CollectionUtil.isNotEmpty(infoList)) {
info = infoList.get(0);
DisPhotovoltaic10Excel disPhotovoltaic10Excel = all10kVMap.get(userId).get(0);
info = info.substring(info.indexOf(StrPool.COMMA) + 1);
info = info.replaceAll(StrPool.AT, "||")
.replaceAll("A_phsA", "A相电流")
.replaceAll("A_phsB", "B相电流")
.replaceAll("A_phsC", "C相电流")
.replaceAll("PhV_phsA", "A相电压")
.replaceAll("PhV_phsB", "B相电压")
.replaceAll("PhV_phsC", "C相电压")
.replaceAll("TotW", "有功")
.replaceAll("TotVar", "无功");
disPhotovoltaic10Excel.setTypes(info);
final10kVUserData.add(disPhotovoltaic10Excel);
}
}
EasyExcel.write("D:\\temp\\10kV部分没有数据的.xlsx", DisPhotovoltaic10Excel.class).sheet("10kV部分没有数据的").doWrite(final10kVUserData);
long millis2 = System.currentTimeMillis();
System.out.println("10kV部分没有数据的：" + (millis2 - millis1));
//梳理380V全部没有数据的
List<DisPhotovoltaic380Excel> collect4 = list380v.stream().filter(t -> noDataSet.contains(t.getGenerationUserID())).collect(Collectors.toList());
EasyExcel.write("D:\\temp\\380V全部没有数据的.xlsx", DisPhotovoltaic380Excel.class).sheet("380V全部没有数据的").doWrite(collect4);
long millis3 = System.currentTimeMillis();
System.out.println("380V全部没有数据的：" + (millis3 - millis2));
//380V部分没有数据的
Map<String, List<DisPhotovoltaic380Excel>> all380VMap = list380v.stream().collect(Collectors.groupingBy(DisPhotovoltaic380Excel::getGenerationUserID));
Set<String> keyedSet = noPartDataMap.keySet();
List<DisPhotovoltaic380Excel> final380VUserData = new LinkedList<>();
List<DisPhotovoltaic380Excel> disPhotovoltaic380ExcelList;
DisPhotovoltaic380Excel disPhotovoltaic380Excel;
for (String userId : keyedSet) {
disPhotovoltaic380ExcelList = all380VMap.get(userId);
if (CollectionUtil.isNotEmpty(disPhotovoltaic380ExcelList)) {
disPhotovoltaic380Excel = disPhotovoltaic380ExcelList.get(0);
info = noPartDataMap.get(userId).get(0);
info = info.substring(info.indexOf(StrPool.COMMA) + 1);
info = info.replaceAll(StrPool.AT, "||")
.replaceAll("A_phsA", "A相电流")
.replaceAll("A_phsB", "B相电流")
.replaceAll("A_phsC", "C相电流")
.replaceAll("PhV_phsA", "A相电压")
.replaceAll("PhV_phsB", "B相电压")
.replaceAll("PhV_phsC", "C相电压")
.replaceAll("TotW", "有功")
.replaceAll("TotVar", "无功");
disPhotovoltaic380Excel.setTypes(info);
final380VUserData.add(disPhotovoltaic380Excel);
}
}
EasyExcel.write("D:\\temp\\380V部分没有数据的.xlsx", DisPhotovoltaic380Excel.class).sheet("380V部分没有数据的").doWrite(final380VUserData);
long millis4 = System.currentTimeMillis();
System.out.println("380V部分没有数据的：" + (millis4 - millis3));
}
@ApiOperation(value = "将用户数据导入到配网表中")
@PostMapping("/insertDistributionMonitor")

17
src/main/java/com/njcn/jbsyncdata/service/impl/BusinessServiceImpl.java
View File

@@ -69,7 +69,6 @@ public class BusinessServiceImpl implements IBusinessService {
DateTime dateTemp = DateUtil.parse(date, DatePattern.NORM_DATE_FORMAT);
DateTime beginOfDay = DateUtil.beginOfDay(dateTemp);
DateTime endOfDay = DateUtil.endOfDay(dateTemp);
//有指标返回，但是指标的遥测整体数据为null
RestTemplateUtil restTemplateUtil = new RestTemplateUtil();
TokenResult tokenWithRestTemplate = tokenComponent.getTokenWithRestTemplate();
if (null == tokenWithRestTemplate) {
@@ -115,7 +114,11 @@ public class BusinessServiceImpl implements IBusinessService {
JSONArray jsonArray = JSONUtil.createArray();
jsonArray.add(jsonObjectSub);
jsonObject.set("filters", jsonArray);
responseEntities.add(restTemplateUtil.post(tokenComponent.getUrl().concat("/realMeasCenter/telemetry/commonQuery"), headers, jsonObject, String.class));
try {
responseEntities.add(restTemplateUtil.post(tokenComponent.getUrl().concat("/realMeasCenter/telemetry/commonQuery"), headers, jsonObject, String.class));
} catch (Exception exception) {
log.error("远程调用接口异常，异常为：" + exception);
}
}
//开始解析数据
Set<String> userIdConcatMeasType = new HashSet<>();
@@ -234,7 +237,7 @@ public class BusinessServiceImpl implements IBusinessService {
* @param typeData 远程根据用户编号获取的数据 Map</表名/String, List<Map</属性名/String,/数值/String>>> typeData = new HashMap<>();
*/
private void batchInsertData(Map<String, List<String>> typeData) {
log.error("总计有{}条记录入库，以5000作为基数分片插入influxdb", typeData.size());
log.error("总计有{}条记录入库，以20000作为基数分片插入influxdb", typeData.size());
List<String> sqlList = new ArrayList<>();
Set<String> tableNames = typeData.keySet();
String[] datas;
@@ -266,10 +269,16 @@ public class BusinessServiceImpl implements IBusinessService {
List<List<String>> subSqlList = ListUtils.partition(sqlList, 20000);
int count = 1;
for (List<String> sql : subSqlList) {
influxDbUtils.batchInsert(influxDbUtils.getDbName(), "autogen", InfluxDB.ConsistencyLevel.ALL, TimeUnit.MILLISECONDS, sql);
try {
influxDbUtils.batchInsert(influxDbUtils.getDbName(), "autogen", InfluxDB.ConsistencyLevel.ALL, TimeUnit.MILLISECONDS, sql);
} catch (Exception exception) {
log.error("数据批量入库异常，异常为：{}",exception.toString());
exception.printStackTrace();
}
log.error("已经入库{}条记录!", count * 20000);
count++;
}
log.error("当前批次所有数据，{}条均已入库！", sqlList.size());
}
}

113
src/main/java/com/njcn/jbsyncdata/util/PubUtils.java
View File

@@ -1,13 +1,22 @@
package com.njcn.jbsyncdata.util;
import cn.hutool.core.collection.CollectionUtil;
import cn.hutool.core.io.file.FileReader;
import cn.hutool.core.text.StrPool;
import cn.hutool.core.util.RandomUtil;
import cn.hutool.core.util.StrUtil;
import com.alibaba.excel.EasyExcel;
import com.alibaba.excel.support.ExcelTypeEnum;
import com.fasterxml.jackson.databind.DeserializationFeature;
import com.fasterxml.jackson.databind.JavaType;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.SerializationFeature;
import com.fasterxml.jackson.databind.type.TypeFactory;
import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
import com.njcn.jbsyncdata.pojo.DisPhotovoltaic10Excel;
import com.njcn.jbsyncdata.pojo.DisPhotovoltaic380Excel;
import io.swagger.annotations.ApiOperation;
import org.springframework.web.bind.annotation.PostMapping;
import java.io.IOException;
import java.lang.reflect.Method;
@@ -25,6 +34,7 @@ import java.util.*;
import java.util.concurrent.TimeUnit;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
import static java.lang.Integer.parseInt;
@@ -440,4 +450,107 @@ public class PubUtils {
return Double.longBitsToDouble(value);
}
//***************************************************添加结束********************************************************
public static void main(String[] args) throws IOException {
//读取10kV的数据
List<DisPhotovoltaic10Excel> list10kV = EasyExcel.read("D:\\temp\\基础表.xlsx")
.excelType(ExcelTypeEnum.XLSX)
.head(DisPhotovoltaic10Excel.class)
.headRowNumber(2)
.sheet(0).doReadSync();
list10kV = list10kV.stream()
.filter(t -> StrUtil.isNotBlank(t.getGenerationUserID()))
.filter(t -> StrUtil.isNotBlank(t.getStageID()))
.filter(StreamUtil.distinctByKey(DisPhotovoltaic10Excel::getGenerationUserID))
.collect(Collectors.toList());
//读取380V的数据
List<DisPhotovoltaic380Excel> list380v = EasyExcel.read("D:\\temp\\基础表.xlsx")
.excelType(ExcelTypeEnum.XLSX)
.head(DisPhotovoltaic380Excel.class)
.headRowNumber(2)
.sheet(1).doReadSync();
list380v = list380v.stream()
.filter(t -> StrUtil.isNotBlank(t.getGenerationUserID()))
.filter(t -> StrUtil.isNotBlank(t.getStageID()))
.filter(StreamUtil.distinctByKey(DisPhotovoltaic380Excel::getGenerationUserID))
.collect(Collectors.toList());
//读取所有没数据的用户号
FileReader fileReader = new FileReader("D:\\temp\\all.txt");
List<String> noData = fileReader.readLines();
Set<String> noDataSet = new HashSet<>(noData);
FileReader fileReader2 = new FileReader("D:\\temp\\part.txt");
List<String> noPartData = fileReader2.readLines();
Set<String> noPartDataSet = new HashSet<>(noPartData);
long millis = System.currentTimeMillis();
//梳理10kV全部没有数据的
List<DisPhotovoltaic10Excel> collect = list10kV.stream().filter(t -> noDataSet.contains(t.getGenerationUserID())).collect(Collectors.toList());
EasyExcel.write("D:\\temp\\10kV全部没有数据的.xlsx", DisPhotovoltaic10Excel.class).sheet("10kV全部没有数据的").doWrite(collect);
long millis1 = System.currentTimeMillis();
System.out.println("10kV全部没有数据的耗时：" + (millis1 - millis));
//10kV部分没有数据的
Map<String, List<String>> noPartDataMap = noPartDataSet.stream().collect(Collectors.groupingBy(t -> t.substring(0, t.indexOf(StrPool.COMMA))));
Map<String, List<DisPhotovoltaic10Excel>> all10kVMap = list10kV.stream().collect(Collectors.groupingBy(DisPhotovoltaic10Excel::getGenerationUserID));
Set<String> keyedSet1 = all10kVMap.keySet();
List<DisPhotovoltaic10Excel> final10kVUserData = new LinkedList<>();
String info;
for (String userId : keyedSet1) {
List<String> infoList = noPartDataMap.get(userId);
if (CollectionUtil.isNotEmpty(infoList)) {
info = infoList.get(0);
DisPhotovoltaic10Excel disPhotovoltaic10Excel = all10kVMap.get(userId).get(0);
info = info.substring(info.indexOf(StrPool.COMMA) + 1);
info = info.replaceAll(StrPool.AT, "||")
.replaceAll("A_phsA", "A相电流")
.replaceAll("A_phsB", "B相电流")
.replaceAll("A_phsC", "C相电流")
.replaceAll("PhV_phsA", "A相电压")
.replaceAll("PhV_phsB", "B相电压")
.replaceAll("PhV_phsC", "C相电压")
.replaceAll("TotW", "有功")
.replaceAll("TotVar", "无功");
disPhotovoltaic10Excel.setTypes(info);
final10kVUserData.add(disPhotovoltaic10Excel);
}
}
EasyExcel.write("D:\\temp\\10kV部分没有数据的.xlsx", DisPhotovoltaic10Excel.class).sheet("10kV部分没有数据的").doWrite(final10kVUserData);
long millis2 = System.currentTimeMillis();
System.out.println("10kV部分没有数据的：" + (millis2 - millis1));
//梳理380V全部没有数据的
List<DisPhotovoltaic380Excel> collect4 = list380v.stream().filter(t -> noDataSet.contains(t.getGenerationUserID())).collect(Collectors.toList());
EasyExcel.write("D:\\temp\\380V全部没有数据的.xlsx", DisPhotovoltaic380Excel.class).sheet("380V全部没有数据的").doWrite(collect4);
long millis3 = System.currentTimeMillis();
System.out.println("380V全部没有数据的：" + (millis3 - millis2));
//380V部分没有数据的
Map<String, List<DisPhotovoltaic380Excel>> all380VMap = list380v.stream().collect(Collectors.groupingBy(DisPhotovoltaic380Excel::getGenerationUserID));
Set<String> keyedSet = noPartDataMap.keySet();
List<DisPhotovoltaic380Excel> final380VUserData = new LinkedList<>();
List<DisPhotovoltaic380Excel> disPhotovoltaic380ExcelList;
DisPhotovoltaic380Excel disPhotovoltaic380Excel;
for (String userId : keyedSet) {
disPhotovoltaic380ExcelList = all380VMap.get(userId);
if (CollectionUtil.isNotEmpty(disPhotovoltaic380ExcelList)) {
disPhotovoltaic380Excel = disPhotovoltaic380ExcelList.get(0);
info = noPartDataMap.get(userId).get(0);
info = info.substring(info.indexOf(StrPool.COMMA) + 1);
info = info.replaceAll(StrPool.AT, "||")
.replaceAll("A_phsA", "A相电流")
.replaceAll("A_phsB", "B相电流")
.replaceAll("A_phsC", "C相电流")
.replaceAll("PhV_phsA", "A相电压")
.replaceAll("PhV_phsB", "B相电压")
.replaceAll("PhV_phsC", "C相电压")
.replaceAll("TotW", "有功")
.replaceAll("TotVar", "无功");
disPhotovoltaic380Excel.setTypes(info);
final380VUserData.add(disPhotovoltaic380Excel);
}
}
EasyExcel.write("D:\\temp\\380V部分没有数据的.xlsx", DisPhotovoltaic380Excel.class).sheet("380V部分没有数据的").doWrite(final380VUserData);
long millis4 = System.currentTimeMillis();
System.out.println("380V部分没有数据的：" + (millis4 - millis3));
}
}

10
src/main/resources/application.yml
View File

@@ -9,12 +9,12 @@ spring:
datasource:
druid:
driver-class-name: com.mysql.cj.jdbc.Driver
url: jdbc:mysql://10.118.135.128:13306/pqsinfo_pms_jb?useUnicode=true&characterEncoding=utf-8&useSSL=true&serverTimezone=CTT&rewriteBatchedStatements=true
username: root
password: njcnpqs
# url: jdbc:mysql://101.132.73.63:13306/pqsinfo_pms_jb?useUnicode=true&characterEncoding=utf-8&useSSL=false&serverTimezone=CTT
# url: jdbc:mysql://10.118.135.128:13306/pqsinfo_pms_jb?useUnicode=true&characterEncoding=utf-8&useSSL=true&serverTimezone=CTT&rewriteBatchedStatements=true
# username: root
# password: '*#Bg20230711'
# password: njcnpqs
url: jdbc:mysql://101.132.73.63:13306/pqsinfo_pms_jb?useUnicode=true&characterEncoding=utf-8&useSSL=false&serverTimezone=CTT
username: root
password: '*#Bg20230711'
#初始化建立物理连接的个数、最小、最大连接数
initial-size: 5
min-idle: 5