diff --git a/src/main/java/com/njcn/jbsyncdata/controller/DisPhotovoltaicController.java b/src/main/java/com/njcn/jbsyncdata/controller/DisPhotovoltaicController.java index 54c567c..5886cc1 100644 --- a/src/main/java/com/njcn/jbsyncdata/controller/DisPhotovoltaicController.java +++ b/src/main/java/com/njcn/jbsyncdata/controller/DisPhotovoltaicController.java @@ -1,6 +1,9 @@ package com.njcn.jbsyncdata.controller; import cn.hutool.core.collection.CollectionUtil; +import cn.hutool.core.date.DatePattern; +import cn.hutool.core.date.DateTime; +import cn.hutool.core.date.DateUtil; import cn.hutool.core.io.file.FileReader; import cn.hutool.core.text.StrPool; import cn.hutool.core.util.StrUtil; @@ -41,7 +44,10 @@ public class DisPhotovoltaicController { private final IBusinessService businessService; private final DisPhotovoltaicService disPhotovoltaicService; - @ApiOperation(value = "查询所有用户的遥测数据") + /** + * @param date yyyy-MM-dd + */ + @ApiOperation(value = "查询指定日期所有用户的遥测数据") @PostMapping("/queryTelemetryData") public void queryTelemetryData(String date) { try { @@ -51,6 +57,30 @@ public class DisPhotovoltaicController { } } + /** + * @param startTime 起始时间 yyyy-MM-dd + * @param endTime 截止时间 yyyy-MM-dd + */ + @ApiOperation(value = "查询日期范围内所有用户的遥测数据") + @PostMapping("/queryTelemetryDataTimeRange") + public void queryTelemetryDataTimeRange(String startTime, String endTime) { + try { + DateTime startDate = DateUtil.parse(startTime, DatePattern.NORM_DATE_FORMAT); + DateTime endDate = DateUtil.parse(endTime, DatePattern.NORM_DATE_FORMAT); + long betweenDay = DateUtil.betweenDay(startDate, endDate, true); + //第一天 + businessService.queryTelemetryData(DateUtil.format(startDate, DatePattern.NORM_DATE_PATTERN)); + //后续递增1 + for (int i = 0; i < betweenDay; i++) { + startDate = DateUtil.offsetDay(startDate, 1); + businessService.queryTelemetryData(DateUtil.format(startDate, DatePattern.NORM_DATE_PATTERN)); + } + } catch (Exception exception) { + exception.printStackTrace(); + } + } + + @ApiOperation(value = "导入10kv分布式光伏接入情况", produces = MediaType.APPLICATION_OCTET_STREAM_VALUE) @PostMapping("/import10KV") public void import10(MultipartFile file, HttpServletResponse response) throws Exception { @@ -84,106 +114,6 @@ public class DisPhotovoltaicController { } - public static void main(String[] args) throws IOException { - //读取10kV的数据 - List list10kV = EasyExcel.read("D:\\temp\\基础表.xlsx") - .excelType(ExcelTypeEnum.XLSX) - .head(DisPhotovoltaic10Excel.class) - .headRowNumber(2) - .sheet(0).doReadSync(); - list10kV = list10kV.stream() - .filter(t -> StrUtil.isNotBlank(t.getGenerationUserID())) - .filter(t -> StrUtil.isNotBlank(t.getStageID())) - .filter(StreamUtil.distinctByKey(DisPhotovoltaic10Excel::getGenerationUserID)) - .collect(Collectors.toList()); - //读取380V的数据 - List list380v = EasyExcel.read("D:\\temp\\基础表.xlsx") - .excelType(ExcelTypeEnum.XLSX) - .head(DisPhotovoltaic380Excel.class) - .headRowNumber(2) - .sheet(1).doReadSync(); - list380v = list380v.stream() - .filter(t -> StrUtil.isNotBlank(t.getGenerationUserID())) - .filter(t -> StrUtil.isNotBlank(t.getStageID())) - .filter(StreamUtil.distinctByKey(DisPhotovoltaic380Excel::getGenerationUserID)) - .collect(Collectors.toList()); - //读取所有没数据的用户号 - - FileReader fileReader = new FileReader("D:\\temp\\all.txt"); - List noData = fileReader.readLines(); - Set noDataSet = new HashSet<>(noData); - FileReader fileReader2 = new FileReader("D:\\temp\\part.txt"); - List noPartData = fileReader2.readLines(); - Set noPartDataSet = new HashSet<>(noPartData); - - long millis = System.currentTimeMillis(); - //梳理10kV全部没有数据的 - List collect = list10kV.stream().filter(t -> noDataSet.contains(t.getGenerationUserID())).collect(Collectors.toList()); - EasyExcel.write("D:\\temp\\10kV全部没有数据的.xlsx", DisPhotovoltaic10Excel.class).sheet("10kV全部没有数据的").doWrite(collect); - long millis1 = System.currentTimeMillis(); - System.out.println("10kV全部没有数据的耗时：" + (millis1 - millis)); - //10kV部分没有数据的 - Map> noPartDataMap = noPartDataSet.stream().collect(Collectors.groupingBy(t -> t.substring(0, t.indexOf(StrPool.COMMA)))); - Map> all10kVMap = list10kV.stream().collect(Collectors.groupingBy(DisPhotovoltaic10Excel::getGenerationUserID)); - Set keyedSet1 = all10kVMap.keySet(); - List final10kVUserData = new LinkedList<>(); - String info; - for (String userId : keyedSet1) { - List infoList = noPartDataMap.get(userId); - if (CollectionUtil.isNotEmpty(infoList)) { - info = infoList.get(0); - DisPhotovoltaic10Excel disPhotovoltaic10Excel = all10kVMap.get(userId).get(0); - info = info.substring(info.indexOf(StrPool.COMMA) + 1); - info = info.replaceAll(StrPool.AT, "||") - .replaceAll("A_phsA", "A相电流") - .replaceAll("A_phsB", "B相电流") - .replaceAll("A_phsC", "C相电流") - .replaceAll("PhV_phsA", "A相电压") - .replaceAll("PhV_phsB", "B相电压") - .replaceAll("PhV_phsC", "C相电压") - .replaceAll("TotW", "有功") - .replaceAll("TotVar", "无功"); - disPhotovoltaic10Excel.setTypes(info); - final10kVUserData.add(disPhotovoltaic10Excel); - } - } - EasyExcel.write("D:\\temp\\10kV部分没有数据的.xlsx", DisPhotovoltaic10Excel.class).sheet("10kV部分没有数据的").doWrite(final10kVUserData); - long millis2 = System.currentTimeMillis(); - System.out.println("10kV部分没有数据的：" + (millis2 - millis1)); - //梳理380V全部没有数据的 - List collect4 = list380v.stream().filter(t -> noDataSet.contains(t.getGenerationUserID())).collect(Collectors.toList()); - EasyExcel.write("D:\\temp\\380V全部没有数据的.xlsx", DisPhotovoltaic380Excel.class).sheet("380V全部没有数据的").doWrite(collect4); - long millis3 = System.currentTimeMillis(); - System.out.println("380V全部没有数据的：" + (millis3 - millis2)); - //380V部分没有数据的 - Map> all380VMap = list380v.stream().collect(Collectors.groupingBy(DisPhotovoltaic380Excel::getGenerationUserID)); - Set keyedSet = noPartDataMap.keySet(); - List final380VUserData = new LinkedList<>(); - List disPhotovoltaic380ExcelList; - DisPhotovoltaic380Excel disPhotovoltaic380Excel; - for (String userId : keyedSet) { - disPhotovoltaic380ExcelList = all380VMap.get(userId); - if (CollectionUtil.isNotEmpty(disPhotovoltaic380ExcelList)) { - disPhotovoltaic380Excel = disPhotovoltaic380ExcelList.get(0); - info = noPartDataMap.get(userId).get(0); - info = info.substring(info.indexOf(StrPool.COMMA) + 1); - info = info.replaceAll(StrPool.AT, "||") - .replaceAll("A_phsA", "A相电流") - .replaceAll("A_phsB", "B相电流") - .replaceAll("A_phsC", "C相电流") - .replaceAll("PhV_phsA", "A相电压") - .replaceAll("PhV_phsB", "B相电压") - .replaceAll("PhV_phsC", "C相电压") - .replaceAll("TotW", "有功") - .replaceAll("TotVar", "无功"); - disPhotovoltaic380Excel.setTypes(info); - final380VUserData.add(disPhotovoltaic380Excel); - } - } - EasyExcel.write("D:\\temp\\380V部分没有数据的.xlsx", DisPhotovoltaic380Excel.class).sheet("380V部分没有数据的").doWrite(final380VUserData); - long millis4 = System.currentTimeMillis(); - System.out.println("380V部分没有数据的：" + (millis4 - millis3)); - } @ApiOperation(value = "将用户数据导入到配网表中") @PostMapping("/insertDistributionMonitor") diff --git a/src/main/java/com/njcn/jbsyncdata/service/impl/BusinessServiceImpl.java b/src/main/java/com/njcn/jbsyncdata/service/impl/BusinessServiceImpl.java index e428f67..35fd5ea 100644 --- a/src/main/java/com/njcn/jbsyncdata/service/impl/BusinessServiceImpl.java +++ b/src/main/java/com/njcn/jbsyncdata/service/impl/BusinessServiceImpl.java @@ -69,7 +69,6 @@ public class BusinessServiceImpl implements IBusinessService { DateTime dateTemp = DateUtil.parse(date, DatePattern.NORM_DATE_FORMAT); DateTime beginOfDay = DateUtil.beginOfDay(dateTemp); DateTime endOfDay = DateUtil.endOfDay(dateTemp); - //有指标返回，但是指标的遥测整体数据为null RestTemplateUtil restTemplateUtil = new RestTemplateUtil(); TokenResult tokenWithRestTemplate = tokenComponent.getTokenWithRestTemplate(); if (null == tokenWithRestTemplate) { @@ -115,7 +114,11 @@ public class BusinessServiceImpl implements IBusinessService { JSONArray jsonArray = JSONUtil.createArray(); jsonArray.add(jsonObjectSub); jsonObject.set("filters", jsonArray); - responseEntities.add(restTemplateUtil.post(tokenComponent.getUrl().concat("/realMeasCenter/telemetry/commonQuery"), headers, jsonObject, String.class)); + try { + responseEntities.add(restTemplateUtil.post(tokenComponent.getUrl().concat("/realMeasCenter/telemetry/commonQuery"), headers, jsonObject, String.class)); + } catch (Exception exception) { + log.error("远程调用接口异常，异常为：" + exception); + } } //开始解析数据 Set userIdConcatMeasType = new HashSet<>(); @@ -234,7 +237,7 @@ public class BusinessServiceImpl implements IBusinessService { * @param typeData 远程根据用户编号获取的数据 Map>> typeData = new HashMap<>(); */ private void batchInsertData(Map> typeData) { - log.error("总计有{}条记录入库，以5000作为基数分片插入influxdb", typeData.size()); + log.error("总计有{}条记录入库，以20000作为基数分片插入influxdb", typeData.size()); List sqlList = new ArrayList<>(); Set tableNames = typeData.keySet(); String[] datas; @@ -266,10 +269,16 @@ public class BusinessServiceImpl implements IBusinessService { List> subSqlList = ListUtils.partition(sqlList, 20000); int count = 1; for (List sql : subSqlList) { - influxDbUtils.batchInsert(influxDbUtils.getDbName(), "autogen", InfluxDB.ConsistencyLevel.ALL, TimeUnit.MILLISECONDS, sql); + try { + influxDbUtils.batchInsert(influxDbUtils.getDbName(), "autogen", InfluxDB.ConsistencyLevel.ALL, TimeUnit.MILLISECONDS, sql); + } catch (Exception exception) { + log.error("数据批量入库异常，异常为：{}",exception.toString()); + exception.printStackTrace(); + } log.error("已经入库{}条记录!", count * 20000); count++; } + log.error("当前批次所有数据，{}条均已入库！", sqlList.size()); } } diff --git a/src/main/java/com/njcn/jbsyncdata/util/PubUtils.java b/src/main/java/com/njcn/jbsyncdata/util/PubUtils.java index c0a2744..ceacb4e 100644 --- a/src/main/java/com/njcn/jbsyncdata/util/PubUtils.java +++ b/src/main/java/com/njcn/jbsyncdata/util/PubUtils.java @@ -1,13 +1,22 @@ package com.njcn.jbsyncdata.util; +import cn.hutool.core.collection.CollectionUtil; +import cn.hutool.core.io.file.FileReader; +import cn.hutool.core.text.StrPool; import cn.hutool.core.util.RandomUtil; import cn.hutool.core.util.StrUtil; +import com.alibaba.excel.EasyExcel; +import com.alibaba.excel.support.ExcelTypeEnum; import com.fasterxml.jackson.databind.DeserializationFeature; import com.fasterxml.jackson.databind.JavaType; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.SerializationFeature; import com.fasterxml.jackson.databind.type.TypeFactory; import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; +import com.njcn.jbsyncdata.pojo.DisPhotovoltaic10Excel; +import com.njcn.jbsyncdata.pojo.DisPhotovoltaic380Excel; +import io.swagger.annotations.ApiOperation; +import org.springframework.web.bind.annotation.PostMapping; import java.io.IOException; import java.lang.reflect.Method; @@ -25,6 +34,7 @@ import java.util.*; import java.util.concurrent.TimeUnit; import java.util.regex.Matcher; import java.util.regex.Pattern; +import java.util.stream.Collectors; import static java.lang.Integer.parseInt; @@ -440,4 +450,107 @@ public class PubUtils { return Double.longBitsToDouble(value); } //***************************************************添加结束******************************************************** + + + public static void main(String[] args) throws IOException { + //读取10kV的数据 + List list10kV = EasyExcel.read("D:\\temp\\基础表.xlsx") + .excelType(ExcelTypeEnum.XLSX) + .head(DisPhotovoltaic10Excel.class) + .headRowNumber(2) + .sheet(0).doReadSync(); + list10kV = list10kV.stream() + .filter(t -> StrUtil.isNotBlank(t.getGenerationUserID())) + .filter(t -> StrUtil.isNotBlank(t.getStageID())) + .filter(StreamUtil.distinctByKey(DisPhotovoltaic10Excel::getGenerationUserID)) + .collect(Collectors.toList()); + //读取380V的数据 + List list380v = EasyExcel.read("D:\\temp\\基础表.xlsx") + .excelType(ExcelTypeEnum.XLSX) + .head(DisPhotovoltaic380Excel.class) + .headRowNumber(2) + .sheet(1).doReadSync(); + list380v = list380v.stream() + .filter(t -> StrUtil.isNotBlank(t.getGenerationUserID())) + .filter(t -> StrUtil.isNotBlank(t.getStageID())) + .filter(StreamUtil.distinctByKey(DisPhotovoltaic380Excel::getGenerationUserID)) + .collect(Collectors.toList()); + //读取所有没数据的用户号 + + FileReader fileReader = new FileReader("D:\\temp\\all.txt"); + List noData = fileReader.readLines(); + Set noDataSet = new HashSet<>(noData); + FileReader fileReader2 = new FileReader("D:\\temp\\part.txt"); + List noPartData = fileReader2.readLines(); + Set noPartDataSet = new HashSet<>(noPartData); + + long millis = System.currentTimeMillis(); + //梳理10kV全部没有数据的 + List collect = list10kV.stream().filter(t -> noDataSet.contains(t.getGenerationUserID())).collect(Collectors.toList()); + EasyExcel.write("D:\\temp\\10kV全部没有数据的.xlsx", DisPhotovoltaic10Excel.class).sheet("10kV全部没有数据的").doWrite(collect); + long millis1 = System.currentTimeMillis(); + System.out.println("10kV全部没有数据的耗时：" + (millis1 - millis)); + //10kV部分没有数据的 + Map> noPartDataMap = noPartDataSet.stream().collect(Collectors.groupingBy(t -> t.substring(0, t.indexOf(StrPool.COMMA)))); + Map> all10kVMap = list10kV.stream().collect(Collectors.groupingBy(DisPhotovoltaic10Excel::getGenerationUserID)); + Set keyedSet1 = all10kVMap.keySet(); + List final10kVUserData = new LinkedList<>(); + String info; + for (String userId : keyedSet1) { + List infoList = noPartDataMap.get(userId); + if (CollectionUtil.isNotEmpty(infoList)) { + info = infoList.get(0); + DisPhotovoltaic10Excel disPhotovoltaic10Excel = all10kVMap.get(userId).get(0); + info = info.substring(info.indexOf(StrPool.COMMA) + 1); + info = info.replaceAll(StrPool.AT, "||") + .replaceAll("A_phsA", "A相电流") + .replaceAll("A_phsB", "B相电流") + .replaceAll("A_phsC", "C相电流") + .replaceAll("PhV_phsA", "A相电压") + .replaceAll("PhV_phsB", "B相电压") + .replaceAll("PhV_phsC", "C相电压") + .replaceAll("TotW", "有功") + .replaceAll("TotVar", "无功"); + disPhotovoltaic10Excel.setTypes(info); + final10kVUserData.add(disPhotovoltaic10Excel); + } + } + EasyExcel.write("D:\\temp\\10kV部分没有数据的.xlsx", DisPhotovoltaic10Excel.class).sheet("10kV部分没有数据的").doWrite(final10kVUserData); + long millis2 = System.currentTimeMillis(); + System.out.println("10kV部分没有数据的：" + (millis2 - millis1)); + //梳理380V全部没有数据的 + List collect4 = list380v.stream().filter(t -> noDataSet.contains(t.getGenerationUserID())).collect(Collectors.toList()); + EasyExcel.write("D:\\temp\\380V全部没有数据的.xlsx", DisPhotovoltaic380Excel.class).sheet("380V全部没有数据的").doWrite(collect4); + long millis3 = System.currentTimeMillis(); + System.out.println("380V全部没有数据的：" + (millis3 - millis2)); + //380V部分没有数据的 + Map> all380VMap = list380v.stream().collect(Collectors.groupingBy(DisPhotovoltaic380Excel::getGenerationUserID)); + Set keyedSet = noPartDataMap.keySet(); + List final380VUserData = new LinkedList<>(); + List disPhotovoltaic380ExcelList; + DisPhotovoltaic380Excel disPhotovoltaic380Excel; + for (String userId : keyedSet) { + disPhotovoltaic380ExcelList = all380VMap.get(userId); + if (CollectionUtil.isNotEmpty(disPhotovoltaic380ExcelList)) { + disPhotovoltaic380Excel = disPhotovoltaic380ExcelList.get(0); + info = noPartDataMap.get(userId).get(0); + info = info.substring(info.indexOf(StrPool.COMMA) + 1); + info = info.replaceAll(StrPool.AT, "||") + .replaceAll("A_phsA", "A相电流") + .replaceAll("A_phsB", "B相电流") + .replaceAll("A_phsC", "C相电流") + .replaceAll("PhV_phsA", "A相电压") + .replaceAll("PhV_phsB", "B相电压") + .replaceAll("PhV_phsC", "C相电压") + .replaceAll("TotW", "有功") + .replaceAll("TotVar", "无功"); + disPhotovoltaic380Excel.setTypes(info); + final380VUserData.add(disPhotovoltaic380Excel); + } + } + EasyExcel.write("D:\\temp\\380V部分没有数据的.xlsx", DisPhotovoltaic380Excel.class).sheet("380V部分没有数据的").doWrite(final380VUserData); + long millis4 = System.currentTimeMillis(); + System.out.println("380V部分没有数据的：" + (millis4 - millis3)); + } + } diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml index bec9f93..0593fed 100644 --- a/src/main/resources/application.yml +++ b/src/main/resources/application.yml @@ -9,12 +9,12 @@ spring: datasource: druid: driver-class-name: com.mysql.cj.jdbc.Driver - url: jdbc:mysql://10.118.135.128:13306/pqsinfo_pms_jb?useUnicode=true&characterEncoding=utf-8&useSSL=true&serverTimezone=CTT&rewriteBatchedStatements=true - username: root - password: njcnpqs -# url: jdbc:mysql://101.132.73.63:13306/pqsinfo_pms_jb?useUnicode=true&characterEncoding=utf-8&useSSL=false&serverTimezone=CTT +# url: jdbc:mysql://10.118.135.128:13306/pqsinfo_pms_jb?useUnicode=true&characterEncoding=utf-8&useSSL=true&serverTimezone=CTT&rewriteBatchedStatements=true # username: root -# password: '*#Bg20230711' +# password: njcnpqs + url: jdbc:mysql://101.132.73.63:13306/pqsinfo_pms_jb?useUnicode=true&characterEncoding=utf-8&useSSL=false&serverTimezone=CTT + username: root + password: '*#Bg20230711' #初始化建立物理连接的个数、最小、最大连接数 initial-size: 5 min-idle: 5