SyncTool/jb_syncdata
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

1.代码调整

Browse Source
This commit is contained in:
wr
2023-11-24 15:02:35 +08:00
parent e73aeabf46
commit 71ed4ffe12
7 changed files with 157 additions and 48 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
src/main/java/com/njcn/jbsyncdata/controller/DisPhotovoltaicController.java
View File

@@ -156,9 +156,10 @@ public class DisPhotovoltaicController {
.sheet(0).doReadSync();
list = list.stream()
.filter(t -> !"#N/A".equals(t.getPmsID()))
.filter(t -> StrUtil.isNotBlank(t.getPmsID()))
.collect(Collectors.toList());
disPhotovoltaicService.SavaArea(list,response);
System.out.println();
}
@ApiOperation(value = "导入张家口供电公司台区数据")
@@ -171,6 +172,7 @@ public class DisPhotovoltaicController {
.filter(t -> StrUtil.isNotBlank(t.getId()))
.filter(StreamUtil.distinctByKey(ZhangDistributionAreaExcel::getId))
.filter(t -> !"#N/A".equals(t.getPmsID()))
.filter(t -> StrUtil.isNotBlank(t.getPmsID()))
.collect(Collectors.toList());
disPhotovoltaicService.SavaZhangArea(list,response);
System.out.println();

4
src/main/java/com/njcn/jbsyncdata/pojo/DistributionAreaExcel.java
View File

@@ -21,8 +21,6 @@ public class DistributionAreaExcel implements Serializable {
private static final long serialVersionUID = 1L;
@ExcelProperty(value = "错误信息")
private String errorMessage;
/**
* 台区编号
@@ -54,5 +52,7 @@ public class DistributionAreaExcel implements Serializable {
@ExcelProperty(value = "供电所")
private String powerSupply;
@ExcelProperty(value = "无数据的指标")
private String types;
}

2
src/main/java/com/njcn/jbsyncdata/pojo/ZhangDistributionAreaExcel.java
View File

@@ -110,4 +110,6 @@ public class ZhangDistributionAreaExcel implements Serializable {
private String apLoadType;
@ExcelProperty(value = "无数据的指标")
private String types;
}

5
src/main/java/com/njcn/jbsyncdata/service/impl/BusinessServiceImpl.java
View File

@@ -3,6 +3,7 @@ package com.njcn.jbsyncdata.service.impl;
import cn.hutool.core.date.DatePattern;
import cn.hutool.core.date.DateTime;
import cn.hutool.core.date.DateUtil;
import cn.hutool.core.util.StrUtil;
import cn.hutool.json.JSONObject;
import cn.hutool.json.JSONUtil;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
@@ -97,6 +98,7 @@ public class BusinessServiceImpl implements IBusinessService {
}
@Override
@Async("asyncExecutor")
public void queryTelemetryAreaData(String date) {
DateTime dateTemp = DateUtil.parse(date, DatePattern.NORM_DATE_FORMAT);
DateTime beginOfDay = DateUtil.beginOfDay(dateTemp);
@@ -130,6 +132,9 @@ public class BusinessServiceImpl implements IBusinessService {
.select(PmsPowerDistributionarea::getId, PmsPowerDistributionarea::getPmsID)
.isNotNull(PmsPowerDistributionarea::getPmsID)
);
list = list.stream()
.filter(t -> StrUtil.isNotBlank(t.getPmsID()))
.collect(Collectors.toList());
List<List<PmsPowerDistributionarea>> singleQueryDataUserId = ListUtils.partition(list, 20000);
for (int k = 0; k < singleQueryDataUserId.size(); k++) {
areaDataProcessing.asyncInfluxDb(tokenComponent,date, restTemplateUtil, typeList, jsonObject, jsonObjectSub, headers, singleQueryDataUserId, k);

2
src/main/java/com/njcn/jbsyncdata/service/impl/DisPhotovoltaicServiceImpl.java
View File

@@ -398,7 +398,7 @@ public class DisPhotovoltaicServiceImpl implements DisPhotovoltaicService {
String substring = excel.getPCapacity().substring(excel.getPCapacity().lastIndexOf("-")+1, excel.getPCapacity().indexOf("/") );
area.setPCapacity(Float.valueOf(substring));
}else{
area.setPCapacity(0.0f);
area.setPCapacity(100.0f);
}
area.setRegionalism("");
area.setDevRegionalism("");

4
src/main/java/com/njcn/jbsyncdata/util/AreaDataProcessing.java
View File

@@ -200,9 +200,9 @@ public class AreaDataProcessing {
excalationData.add(key);
}
}
FileWriter lackDataWriter = FileWriter.create(new File("/usr/local/syncData/lackData" + date + k + ".txt"));
FileWriter lackDataWriter = FileWriter.create(new File("/usr/local/syncData/lackData" + date + k + "台区.txt"));
lackDataWriter.writeLines(lackData);
FileWriter excalationDataWriter = FileWriter.create(new File("/usr/local/syncData/excalationData" + date + k + ".txt"));
FileWriter excalationDataWriter = FileWriter.create(new File("/usr/local/syncData/excalationData" + date + k + "台区.txt"));
excalationDataWriter.writeLines(excalationData);
}
log.error("用户有指标没有数据的长度为：{}", userIdConcatMeasType.size());

184
src/main/java/com/njcn/jbsyncdata/util/PubUtils.java
View File

@@ -15,6 +15,8 @@ import com.fasterxml.jackson.databind.type.TypeFactory;
import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
import com.njcn.jbsyncdata.pojo.DisPhotovoltaic10Excel;
import com.njcn.jbsyncdata.pojo.DisPhotovoltaic380Excel;
import com.njcn.jbsyncdata.pojo.DistributionAreaExcel;
import com.njcn.jbsyncdata.pojo.ZhangDistributionAreaExcel;
import io.swagger.annotations.ApiOperation;
import org.springframework.web.bind.annotation.PostMapping;
@@ -452,55 +454,152 @@ public class PubUtils {
//***************************************************添加结束********************************************************
// public static void main(String[] args) throws IOException {
// //读取10kV的数据
// List<DisPhotovoltaic10Excel> list10kV = EasyExcel.read("D:\\temp\\基础表.xlsx")
// .excelType(ExcelTypeEnum.XLSX)
// .head(DisPhotovoltaic10Excel.class)
// .headRowNumber(2)
// .sheet(0).doReadSync();
// list10kV = list10kV.stream()
// .filter(t -> StrUtil.isNotBlank(t.getGenerationUserID()))
// .filter(t -> StrUtil.isNotBlank(t.getStageID()))
// .filter(StreamUtil.distinctByKey(DisPhotovoltaic10Excel::getGenerationUserID))
// .collect(Collectors.toList());
// //读取380V的数据
// List<DisPhotovoltaic380Excel> list380v = EasyExcel.read("D:\\temp\\基础表.xlsx")
// .excelType(ExcelTypeEnum.XLSX)
// .head(DisPhotovoltaic380Excel.class)
// .headRowNumber(2)
// .sheet(1).doReadSync();
// list380v = list380v.stream()
// .filter(t -> StrUtil.isNotBlank(t.getGenerationUserID()))
// .filter(t -> StrUtil.isNotBlank(t.getStageID()))
// .filter(StreamUtil.distinctByKey(DisPhotovoltaic380Excel::getGenerationUserID))
// .collect(Collectors.toList());
// //读取所有没数据的用户号
// FileReader fileReader = new FileReader("D:\\temp\\all.txt");
// List<String> noData = fileReader.readLines();
// Set<String> noDataSet = new HashSet<>(noData);
// FileReader fileReader2 = new FileReader("D:\\temp\\part.txt");
// List<String> noPartData = fileReader2.readLines();
// Set<String> noPartDataSet = new HashSet<>(noPartData);
//
// long millis = System.currentTimeMillis();
// //梳理10kV全部没有数据的
// List<DisPhotovoltaic10Excel> collect = list10kV.stream().filter(t -> noDataSet.contains(t.getGenerationUserID())).collect(Collectors.toList());
// EasyExcel.write("D:\\temp\\10kV全部没有数据的.xlsx", DisPhotovoltaic10Excel.class).sheet("10kV全部没有数据的").doWrite(collect);
// long millis1 = System.currentTimeMillis();
// System.out.println("10kV全部没有数据的耗时：" + (millis1 - millis));
// //10kV部分没有数据的
// Map<String, List<String>> noPartDataMap = noPartDataSet.stream().collect(Collectors.groupingBy(t -> t.substring(0, t.indexOf(StrPool.COMMA))));
// Map<String, List<DisPhotovoltaic10Excel>> all10kVMap = list10kV.stream().collect(Collectors.groupingBy(DisPhotovoltaic10Excel::getGenerationUserID));
// Set<String> keyedSet1 = all10kVMap.keySet();
// List<DisPhotovoltaic10Excel> final10kVUserData = new LinkedList<>();
// String info;
// for (String userId : keyedSet1) {
// List<String> infoList = noPartDataMap.get(userId);
// if (CollectionUtil.isNotEmpty(infoList)) {
// info = infoList.get(0);
// DisPhotovoltaic10Excel disPhotovoltaic10Excel = all10kVMap.get(userId).get(0);
// info = info.substring(info.indexOf(StrPool.COMMA) + 1);
// info = info.replaceAll(StrPool.AT, "||")
// .replaceAll("A_phsA", "A相电流")
// .replaceAll("A_phsB", "B相电流")
// .replaceAll("A_phsC", "C相电流")
// .replaceAll("PhV_phsA", "A相电压")
// .replaceAll("PhV_phsB", "B相电压")
// .replaceAll("PhV_phsC", "C相电压")
// .replaceAll("TotW", "有功")
// .replaceAll("TotVar", "无功");
// disPhotovoltaic10Excel.setTypes(info);
// final10kVUserData.add(disPhotovoltaic10Excel);
// }
// }
// EasyExcel.write("D:\\temp\\10kV部分没有数据的.xlsx", DisPhotovoltaic10Excel.class).sheet("10kV部分没有数据的").doWrite(final10kVUserData);
// long millis2 = System.currentTimeMillis();
// System.out.println("10kV部分没有数据的：" + (millis2 - millis1));
// //梳理380V全部没有数据的
// List<DisPhotovoltaic380Excel> collect4 = list380v.stream().filter(t -> noDataSet.contains(t.getGenerationUserID())).collect(Collectors.toList());
// EasyExcel.write("D:\\temp\\380V全部没有数据的.xlsx", DisPhotovoltaic380Excel.class).sheet("380V全部没有数据的").doWrite(collect4);
// long millis3 = System.currentTimeMillis();
// System.out.println("380V全部没有数据的：" + (millis3 - millis2));
// //380V部分没有数据的
// Map<String, List<DisPhotovoltaic380Excel>> all380VMap = list380v.stream().collect(Collectors.groupingBy(DisPhotovoltaic380Excel::getGenerationUserID));
// Set<String> keyedSet = noPartDataMap.keySet();
// List<DisPhotovoltaic380Excel> final380VUserData = new LinkedList<>();
// List<DisPhotovoltaic380Excel> disPhotovoltaic380ExcelList;
// DisPhotovoltaic380Excel disPhotovoltaic380Excel;
// for (String userId : keyedSet) {
// disPhotovoltaic380ExcelList = all380VMap.get(userId);
// if (CollectionUtil.isNotEmpty(disPhotovoltaic380ExcelList)) {
// disPhotovoltaic380Excel = disPhotovoltaic380ExcelList.get(0);
// info = noPartDataMap.get(userId).get(0);
// info = info.substring(info.indexOf(StrPool.COMMA) + 1);
// info = info.replaceAll(StrPool.AT, "||")
// .replaceAll("A_phsA", "A相电流")
// .replaceAll("A_phsB", "B相电流")
// .replaceAll("A_phsC", "C相电流")
// .replaceAll("PhV_phsA", "A相电压")
// .replaceAll("PhV_phsB", "B相电压")
// .replaceAll("PhV_phsC", "C相电压")
// .replaceAll("TotW", "有功")
// .replaceAll("TotVar", "无功");
// disPhotovoltaic380Excel.setTypes(info);
// final380VUserData.add(disPhotovoltaic380Excel);
// }
// }
// EasyExcel.write("D:\\temp\\380V部分没有数据的.xlsx", DisPhotovoltaic380Excel.class).sheet("380V部分没有数据的").doWrite(final380VUserData);
// long millis4 = System.currentTimeMillis();
// System.out.println("380V部分没有数据的：" + (millis4 - millis3));
// }
public static void main(String[] args) throws IOException {
//读取10kV的数据
List<DisPhotovoltaic10Excel> list10kV = EasyExcel.read("D:\\temp\\基础表.xlsx")
.excelType(ExcelTypeEnum.XLSX)
.head(DisPhotovoltaic10Excel.class)
.headRowNumber(2)
List<DistributionAreaExcel> list = EasyExcel.read("D:\\test\\唐山.xlsx")
.head(DistributionAreaExcel.class)
.sheet(0).doReadSync();
list10kV = list10kV.stream()
.filter(t -> StrUtil.isNotBlank(t.getGenerationUserID()))
.filter(t -> StrUtil.isNotBlank(t.getStageID()))
.filter(StreamUtil.distinctByKey(DisPhotovoltaic10Excel::getGenerationUserID))
list = list.stream()
.filter(t -> !"#N/A".equals(t.getPmsID()))
.collect(Collectors.toList());
//读取380V的数据
List<DisPhotovoltaic380Excel> list380v = EasyExcel.read("D:\\temp\\基础表.xlsx")
.excelType(ExcelTypeEnum.XLSX)
.head(DisPhotovoltaic380Excel.class)
.headRowNumber(2)
.sheet(1).doReadSync();
list380v = list380v.stream()
.filter(t -> StrUtil.isNotBlank(t.getGenerationUserID()))
.filter(t -> StrUtil.isNotBlank(t.getStageID()))
.filter(StreamUtil.distinctByKey(DisPhotovoltaic380Excel::getGenerationUserID))
.collect(Collectors.toList());
//读取所有没数据的用户号
FileReader fileReader = new FileReader("D:\\temp\\all.txt");
List<ZhangDistributionAreaExcel> listZhang = EasyExcel.read("D:\\test\\张家口.xlsx")
.head(ZhangDistributionAreaExcel.class)
.doReadAllSync();
listZhang = listZhang.stream()
.filter(t -> StrUtil.isNotBlank(t.getId()))
.filter(StreamUtil.distinctByKey(ZhangDistributionAreaExcel::getId))
.filter(t -> !"#N/A".equals(t.getPmsID()))
.filter(t -> StrUtil.isNotBlank(t.getPmsID()))
.collect(Collectors.toList());
//读取所有没数据的用户号
FileReader fileReader = new FileReader("D:\\test\\all.txt");
List<String> noData = fileReader.readLines();
Set<String> noDataSet = new HashSet<>(noData);
FileReader fileReader2 = new FileReader("D:\\temp\\part.txt");
FileReader fileReader2 = new FileReader("D:\\test\\part.txt");
List<String> noPartData = fileReader2.readLines();
Set<String> noPartDataSet = new HashSet<>(noPartData);
long millis = System.currentTimeMillis();
//梳理10kV全部没有数据的
List<DisPhotovoltaic10Excel> collect = list10kV.stream().filter(t -> noDataSet.contains(t.getGenerationUserID())).collect(Collectors.toList());
EasyExcel.write("D:\\temp\\10kV全部没有数据的.xlsx", DisPhotovoltaic10Excel.class).sheet("10kV全部没有数据的").doWrite(collect);
//唐山没有数据的信息
List<DistributionAreaExcel> collect = list.stream().filter(t -> noDataSet.contains(t.getPmsID())).collect(Collectors.toList());
EasyExcel.write("D:\\test\\唐山全部没有数据的.xlsx", DistributionAreaExcel.class).sheet("唐山全部没有数据的").doWrite(collect);
long millis1 = System.currentTimeMillis();
System.out.println("10kV全部没有数据的耗时：" + (millis1 - millis));
//10kV部分没有数据的
System.out.println("唐山全部没有数据的耗时：" + (millis1 - millis));
//唐山部分没有数据的
Map<String, List<String>> noPartDataMap = noPartDataSet.stream().collect(Collectors.groupingBy(t -> t.substring(0, t.indexOf(StrPool.COMMA))));
Map<String, List<DisPhotovoltaic10Excel>> all10kVMap = list10kV.stream().collect(Collectors.groupingBy(DisPhotovoltaic10Excel::getGenerationUserID));
Map<String, List<DistributionAreaExcel>> all10kVMap = list.stream().collect(Collectors.groupingBy(DistributionAreaExcel::getPmsID));
Set<String> keyedSet1 = all10kVMap.keySet();
List<DisPhotovoltaic10Excel> final10kVUserData = new LinkedList<>();
List<DistributionAreaExcel> final10kVUserData = new LinkedList<>();
String info;
for (String userId : keyedSet1) {
List<String> infoList = noPartDataMap.get(userId);
if (CollectionUtil.isNotEmpty(infoList)) {
info = infoList.get(0);
DisPhotovoltaic10Excel disPhotovoltaic10Excel = all10kVMap.get(userId).get(0);
DistributionAreaExcel disPhotovoltaic10Excel = all10kVMap.get(userId).get(0);
info = info.substring(info.indexOf(StrPool.COMMA) + 1);
info = info.replaceAll(StrPool.AT, "||")
.replaceAll("A_phsA", "A相电流")
@@ -515,20 +614,22 @@ public class PubUtils {
final10kVUserData.add(disPhotovoltaic10Excel);
}
}
EasyExcel.write("D:\\temp\\10kV部分没有数据的.xlsx", DisPhotovoltaic10Excel.class).sheet("10kV部分没有数据的").doWrite(final10kVUserData);
EasyExcel.write("D:\\test\\唐山部分没有数据的.xlsx", DistributionAreaExcel.class).sheet("唐山部分没有数据的").doWrite(final10kVUserData);
long millis2 = System.currentTimeMillis();
System.out.println("10kV部分没有数据的：" + (millis2 - millis1));
System.out.println("唐山部分没有数据的：" + (millis2 - millis1));
//梳理380V全部没有数据的
List<DisPhotovoltaic380Excel> collect4 = list380v.stream().filter(t -> noDataSet.contains(t.getGenerationUserID())).collect(Collectors.toList());
EasyExcel.write("D:\\temp\\380V全部没有数据的.xlsx", DisPhotovoltaic380Excel.class).sheet("380V全部没有数据的").doWrite(collect4);
List<ZhangDistributionAreaExcel> collect4 = listZhang.stream().filter(t -> noDataSet.contains(t.getPmsID())).collect(Collectors.toList());
EasyExcel.write("D:\\test\\张家口全部没有数据的.xlsx", ZhangDistributionAreaExcel.class).sheet("张家口全部没有数据的").doWrite(collect4);
long millis3 = System.currentTimeMillis();
System.out.println("380V全部没有数据的：" + (millis3 - millis2));
System.out.println("张家口全部没有数据的：" + (millis3 - millis2));
//380V部分没有数据的
Map<String, List<DisPhotovoltaic380Excel>> all380VMap = list380v.stream().collect(Collectors.groupingBy(DisPhotovoltaic380Excel::getGenerationUserID));
Map<String, List<ZhangDistributionAreaExcel>> all380VMap = listZhang.stream().collect(Collectors.groupingBy(ZhangDistributionAreaExcel::getPmsID));
Set<String> keyedSet = noPartDataMap.keySet();
List<DisPhotovoltaic380Excel> final380VUserData = new LinkedList<>();
List<DisPhotovoltaic380Excel> disPhotovoltaic380ExcelList;
DisPhotovoltaic380Excel disPhotovoltaic380Excel;
List<ZhangDistributionAreaExcel> final380VUserData = new LinkedList<>();
List<ZhangDistributionAreaExcel> disPhotovoltaic380ExcelList;
ZhangDistributionAreaExcel disPhotovoltaic380Excel;
for (String userId : keyedSet) {
disPhotovoltaic380ExcelList = all380VMap.get(userId);
if (CollectionUtil.isNotEmpty(disPhotovoltaic380ExcelList)) {
@@ -548,9 +649,8 @@ public class PubUtils {
final380VUserData.add(disPhotovoltaic380Excel);
}
}
EasyExcel.write("D:\\temp\\380V部分没有数据的.xlsx", DisPhotovoltaic380Excel.class).sheet("380V部分没有数据的").doWrite(final380VUserData);
EasyExcel.write("D:\\test\\张家口部分没有数据的.xlsx", ZhangDistributionAreaExcel.class).sheet("张家口部分没有数据的").doWrite(final380VUserData);
long millis4 = System.currentTimeMillis();
System.out.println("380V部分没有数据的：" + (millis4 - millis3));
System.out.println("张家口部分没有数据的：" + (millis4 - millis3));
}
}