Skip to content

O
operating-vehicle
Commit 0b820297 by 杜发飞
Options

1

parent 27c4f5c8
Showing with 243 additions and 434 deletions
pom.xml
...@@ -87,6 +87,11 @@ ...@@ -87,6 +87,11 @@
</dependency> </dependency>
<dependency> <dependency>
<groupId>org.apache.ignite</groupId> <groupId>org.apache.ignite</groupId>
<artifactId>ignite-log4j2</artifactId>
<version>${ignite.version}</version>
</dependency>
<dependency>
<groupId>org.apache.ignite</groupId>
<artifactId>ignite-spring</artifactId> <artifactId>ignite-spring</artifactId>
<version>${ignite.version}</version> <version>${ignite.version}</version>
</dependency> </dependency>
......
src/main/java/com/hikcreate/ignite/domain/ErrorMsg.java 0 → 100644
package com.hikcreate.ignite.domain;
import org.apache.ignite.cache.query.annotations.QuerySqlField;
import java.io.Serializable;
public class ErrorMsg implements Serializable {
private static final long serialVersionUID = 1L;
@QuerySqlField
private String data; //数据
@QuerySqlField
private String error; //错误原因
public ErrorMsg(String data, String error) {
this.data = data;
this.error = error;
}
}
src/main/java/com/hikcreate/ignite/domain/alarm/DailyAlarmDeal.java
...@@ -4,7 +4,7 @@ import org.apache.ignite.cache.query.annotations.QuerySqlField; ...@@ -4,7 +4,7 @@ import org.apache.ignite.cache.query.annotations.QuerySqlField;
import java.io.Serializable; import java.io.Serializable;
/** /**
* 每日报警处理处 * 每日报警处理
*/ */
public class DailyAlarmDeal implements Serializable { public class DailyAlarmDeal implements Serializable {
...@@ -17,9 +17,6 @@ public class DailyAlarmDeal implements Serializable { ...@@ -17,9 +17,6 @@ public class DailyAlarmDeal implements Serializable {
private String vehicleColor; //车牌颜色 private String vehicleColor; //车牌颜色
@QuerySqlField @QuerySqlField
private String useNature; //使用性质 公交
@QuerySqlField
private String supervisionId; //报警督办 ID private String supervisionId; //报警督办 ID
@QuerySqlField @QuerySqlField
...@@ -28,10 +25,9 @@ public class DailyAlarmDeal implements Serializable { ...@@ -28,10 +25,9 @@ public class DailyAlarmDeal implements Serializable {
@QuerySqlField @QuerySqlField
private Boolean isDeal; //是否处理 private Boolean isDeal; //是否处理
public DailyAlarmDeal(String vehicleNo, String vehicleColor, String useNature, String supervisionId, String warnTime, Boolean isDeal) { public DailyAlarmDeal(String vehicleNo, String vehicleColor, String supervisionId, String warnTime, Boolean isDeal) {
this.vehicleNo = vehicleNo; this.vehicleNo = vehicleNo;
this.vehicleColor = vehicleColor; this.vehicleColor = vehicleColor;
this.useNature = useNature;
this.supervisionId = supervisionId; this.supervisionId = supervisionId;
this.warnTime = warnTime; this.warnTime = warnTime;
this.isDeal = isDeal; this.isDeal = isDeal;
......
src/main/resources/conf.properties
kafka.bootstrap.servers=39.100.49.76:9092 kafka.bootstrap.servers=39.100.49.76:9092
#kafka.bootstrap.servers=10.197.236.154:9092
kafka.zookerper.servers=10.197.236.154:2181
#,10.197.236.169:2181,10.197.236.184:2181/kafka
#kafka.zookerper.servers=172.26.111.183:2181,172.26.111.178:2181,172.26.111.186:2181/tbd_kafka #kafka.zookerper.servers=172.26.111.183:2181,172.26.111.178:2181,172.26.111.186:2181/tbd_kafka
kafka.zookerper.servers=10.197.236.211:2181
application.kafka.topic=tbd-transport-data-gathering application.kafka.topic=tbd-transport-data-gathering
basicsInfo.kafka.topic=transport_basedata_operation basicsInfo.kafka.topic=transport_basedata_operation
hive.group.id=hive hive.group.id=hive
ignite.group.id=ignite ignite.group.id=ignite3
basics.group.id=basics basics.group.id=basics2
hive.driver=org.apache.hive.jdbc.HiveDriver hive.driver=org.apache.hive.jdbc.HiveDriver
hive.url=jdbc:hive2://hadoop02:10000/ods hive.url=jdbc:hive2://hadoop02:10000/ods
......
src/main/scala/com/hikcreate/data/client/IgniteClient.scala
package com.hikcreate.data.client package com.hikcreate.data.client
import java.util.concurrent.TimeUnit import java.util.concurrent.TimeUnit
import com.hikcreate.ignite.domain.PrimaryKey
import com.hikcreate.ignite.domain.{ErrorMsg, PrimaryKey}
import com.hikcreate.ignite.domain.alarm._ import com.hikcreate.ignite.domain.alarm._
import com.hikcreate.ignite.domain.alarm.processor.{DailyAlarmDealUpdate, DailyAlarmUpdate} import com.hikcreate.ignite.domain.alarm.processor.{DailyAlarmDealUpdate, DailyAlarmUpdate}
import com.hikcreate.ignite.domain.basic._ import com.hikcreate.ignite.domain.basic._
...@@ -11,6 +12,7 @@ import org.apache.ignite.binary.BinaryObject ...@@ -11,6 +12,7 @@ import org.apache.ignite.binary.BinaryObject
import org.apache.ignite.cache.CacheMode import org.apache.ignite.cache.CacheMode
import org.apache.ignite.configuration.CacheConfiguration import org.apache.ignite.configuration.CacheConfiguration
import org.apache.ignite.{Ignite, IgniteCache, Ignition} import org.apache.ignite.{Ignite, IgniteCache, Ignition}
import scala.collection.JavaConversions.mapAsJavaMap import scala.collection.JavaConversions.mapAsJavaMap
import scala.collection.JavaConverters._ import scala.collection.JavaConverters._
/** /**
...@@ -31,15 +33,6 @@ object IgniteClient { ...@@ -31,15 +33,6 @@ object IgniteClient {
binary.build() binary.build()
} }
/*********************************基础信息表****************************************************************/ /*********************************基础信息表****************************************************************/
//平台基础信息表
lazy val basicPlatformInfo: IgniteCache[Long, PlatformInfo] = ignite.getOrCreateCache(
new CacheConfiguration[Long,PlatformInfo]()
.setSqlSchema("BasicInfo")
.setName("PlatformInfo")
.setDataRegionName("500MB_Region")
.setCacheMode(CacheMode.REPLICATED)
.setIndexedTypes(classOf[Long],classOf[PlatformInfo])
)
//企业基础信息表--企业接入情况 //企业基础信息表--企业接入情况
lazy val basicEnterpriseInfo: IgniteCache[Long, EnterpriseInfo] = ignite.getOrCreateCache( lazy val basicEnterpriseInfo: IgniteCache[Long, EnterpriseInfo] = ignite.getOrCreateCache(
new CacheConfiguration[Long,EnterpriseInfo]() new CacheConfiguration[Long,EnterpriseInfo]()
...@@ -67,7 +60,6 @@ object IgniteClient { ...@@ -67,7 +60,6 @@ object IgniteClient {
.setCacheMode(CacheMode.REPLICATED) .setCacheMode(CacheMode.REPLICATED)
.setIndexedTypes(classOf[Long],classOf[AlarmTypeInfo]) .setIndexedTypes(classOf[Long],classOf[AlarmTypeInfo])
) )
/*********************************营运车辆监测****************************************************************/ /*********************************营运车辆监测****************************************************************/
/** /**
* 今日车辆在线情况 累计行驶 累计安全行驶里程 * 今日车辆在线情况 累计行驶 累计安全行驶里程
...@@ -207,8 +199,18 @@ object IgniteClient { ...@@ -207,8 +199,18 @@ object IgniteClient {
//.setExpiryPolicyFactory(CreatedExpiryPolicy.factoryOf(new Duration(TimeUnit.HOURS,24))) //.setExpiryPolicyFactory(CreatedExpiryPolicy.factoryOf(new Duration(TimeUnit.HOURS,24)))
) )
/*********************************错误数据****************************************************************/
lazy val errorMsgInfo: IgniteCache[String,ErrorMsg] = ignite.getOrCreateCache(
new CacheConfiguration[String,ErrorMsg]()
.setSqlSchema("Error")
.setName("ErrorMsg")
.setDataRegionName("500MB_Region")
.setCacheMode(CacheMode.REPLICATED)
.setIndexedTypes(classOf[String],classOf[ErrorMsg])
)
def main(args: Array[String]): Unit = { def main(args: Array[String]): Unit = {
ignite.cacheNames().asScala.foreach(x=>ignite.getOrCreateCache(x).clear()) ignite.cacheNames().asScala.foreach(println(_))
ignite.close() ignite.close()
} }
} }
src/main/scala/com/hikcreate/data/common/Sparking.scala
...@@ -15,8 +15,8 @@ trait Sparking { ...@@ -15,8 +15,8 @@ trait Sparking {
.set("spark.serializer","org.apache.spark.serializer.KryoSerializer") .set("spark.serializer","org.apache.spark.serializer.KryoSerializer")
.set("hive.exec.dynamici.partition","true") .set("hive.exec.dynamici.partition","true")
.set("hive.exec.dynamic.partition.mode","nonstrict") .set("hive.exec.dynamic.partition.mode","nonstrict")
//.setAppName("test") .setAppName("test")
//.setMaster("local[*]") .setMaster("local[*]")
def getKafkaParams(servers:String,groupId: String):Map[String,Object] = { def getKafkaParams(servers:String,groupId: String):Map[String,Object] = {
Map[String,Object]( Map[String,Object](
......
src/main/scala/com/hikcreate/data/sync/SyncBasic.scala
...@@ -6,10 +6,9 @@ import com.hikcreate.data.client.IgniteClient ...@@ -6,10 +6,9 @@ import com.hikcreate.data.client.IgniteClient
import com.hikcreate.data.common.{Logging, Sparking} import com.hikcreate.data.common.{Logging, Sparking}
import com.hikcreate.data.constant.Const import com.hikcreate.data.constant.Const
import com.hikcreate.data.util.{Tools, ZkManager} import com.hikcreate.data.util.{Tools, ZkManager}
import com.hikcreate.ignite.domain.basic.{EnterpriseInfo, VehicleInfo, PlatformInfo} import com.hikcreate.ignite.domain.basic.{AlarmTypeInfo, EnterpriseInfo, VehicleInfo}
import org.apache.spark.streaming.kafka010._ import org.apache.spark.streaming.kafka010._
import org.apache.spark.streaming.{Seconds, StreamingContext} import org.apache.spark.streaming.{Seconds, StreamingContext}
import scala.collection.mutable.ArrayBuffer import scala.collection.mutable.ArrayBuffer
object SyncBasic extends Sparking with Logging{ object SyncBasic extends Sparking with Logging{
...@@ -20,8 +19,7 @@ object SyncBasic extends Sparking with Logging{ ...@@ -20,8 +19,7 @@ object SyncBasic extends Sparking with Logging{
val offsets = zkManager.getBeginOffset(Const.basicsInfoTopic,Const.basicsGroupId) val offsets = zkManager.getBeginOffset(Const.basicsInfoTopic,Const.basicsGroupId)
val offsetRanges = new ArrayBuffer[OffsetRange]() val offsetRanges = new ArrayBuffer[OffsetRange]()
val ssc = new StreamingContext(conf,Seconds(1)) val ssc = new StreamingContext(conf,Seconds(1))
val inputStream = KafkaUtils.createDirectStream[String,String]( val inputStream = KafkaUtils.createDirectStream[String,String](ssc,
ssc,
LocationStrategies.PreferConsistent, LocationStrategies.PreferConsistent,
ConsumerStrategies.Subscribe[String,String](Const.basicsInfoTopic,kafkaParams,offsets)) ConsumerStrategies.Subscribe[String,String](Const.basicsInfoTopic,kafkaParams,offsets))
inputStream.transform{ rdd => inputStream.transform{ rdd =>
...@@ -41,52 +39,50 @@ object SyncBasic extends Sparking with Logging{ ...@@ -41,52 +39,50 @@ object SyncBasic extends Sparking with Logging{
def processRow(iterator:Iterator[String]): Unit = { def processRow(iterator:Iterator[String]): Unit = {
iterator.foreach{ x => iterator.foreach{ x =>
val json = JSON.parseObject(x) val json = JSON.parseObject(x)
println(x)
val tableName = json.getString("dataType") val tableName = json.getString("dataType")
val operation = json.getString("operationType") val operation = json.getString("operationType")
json.remove("dataType") json.remove("dataType")
json.remove("operationType") json.remove("operationType")
val str = JSON.toJSONString(json,SerializerFeature.WriteMapNullValue)
if(tableName.equals("baseIntoPlatformInfo")){ //平台
operation match {
case "add" =>
IgniteClient.basicPlatformInfo.withKeepBinary().put(json.getLong("id"),JSON.parseObject(str,classOf[PlatformInfo]))
case "update" =>
IgniteClient.basicPlatformInfo.withKeepBinary().put(json.getLong("id"),JSON.parseObject(str,classOf[PlatformInfo]))
case "delete" =>
IgniteClient.basicPlatformInfo.withKeepBinary().remove(json.getLong("id"))
}
}
if(tableName.equals("baseIntoEnterpriseInfo")){ //企业 if(tableName.equals("baseIntoEnterpriseInfo")){ //企业
val str = JSON.toJSONString(json,SerializerFeature.WriteMapNullValue)
operation match { operation match {
case "add" => case "add" =>
IgniteClient.basicEnterpriseInfo.withKeepBinary().put(json.getLong("id"),JSON.parseObject(str,classOf[EnterpriseInfo])) IgniteClient.basicEnterpriseInfo.withKeepBinary().put(json.getLong("id"),
JSON.parseObject(str,classOf[EnterpriseInfo]))
case "update" => case "update" =>
IgniteClient.basicEnterpriseInfo.withKeepBinary().put(json.getLong("id"),JSON.parseObject(str,classOf[EnterpriseInfo])) IgniteClient.basicEnterpriseInfo.withKeepBinary().put(json.getLong("id"),
JSON.parseObject(str,classOf[EnterpriseInfo]))
case "delete" => case "delete" =>
IgniteClient.basicEnterpriseInfo.withKeepBinary().remove(json.getLong("id")) IgniteClient.basicEnterpriseInfo.withKeepBinary().remove(json.getLong("id"))
} }
} }
if(tableName.equals("baseIntoVehicleInfo")){ //车辆 if(tableName.equals("baseIntoVehicleInfo")){ //车辆
val companyInfo = Tools.getEnterpriseInfo(json.getString("enterpriseCode"))
if(companyInfo.isDefined){
json.put("province",companyInfo.get.getProvince)
json.put("city",companyInfo.get.getCity)
json.put("area",companyInfo.get.getArea)
}
val str = JSON.toJSONString(json,SerializerFeature.WriteMapNullValue)
operation match { operation match {
case "add" => case "add" =>
val companyInfo = Tools.getEnterpriseInfo(json.getString("enterpriseCode")) IgniteClient.basicVehicleInfo.withKeepBinary().put(json.getLong("id"),
if(companyInfo.isDefined){ JSON.parseObject(str,classOf[VehicleInfo]))
//json.put("province",companyInfo.get.getProvince)
//json.put("city",companyInfo.get.getCity)
}
IgniteClient.basicVehicleInfo.withKeepBinary().put(json.getLong("id"),JSON.parseObject(str,classOf[VehicleInfo]))
case "update" => case "update" =>
IgniteClient.basicVehicleInfo.withKeepBinary().put(json.getLong("id"),JSON.parseObject(str,classOf[VehicleInfo])) IgniteClient.basicVehicleInfo.withKeepBinary().put(json.getLong("id"),
JSON.parseObject(str,classOf[VehicleInfo]))
case "delete" => case "delete" =>
IgniteClient.basicVehicleInfo.withKeepBinary().remove(json.getLong("id")) IgniteClient.basicVehicleInfo.withKeepBinary().remove(json.getLong("id"))
} }
} }
if(tableName.equals("baseWarningType")){ if(tableName.equals("baseWarningType")){ //报警类型
val str = JSON.toJSONString(json,SerializerFeature.WriteMapNullValue)
operation match { operation match {
case "add" => case "add" =>
IgniteClient.basicAlarmTypeInfo.withKeepBinary().put(json.getLong("id"),JSON.parseObject(str,classOf[VehicleInfo])) IgniteClient.basicAlarmTypeInfo.withKeepBinary().put(json.getLong("id"),JSON.parseObject(str,classOf[AlarmTypeInfo]))
case "update" => case "update" =>
IgniteClient.basicAlarmTypeInfo.withKeepBinary().put(json.getLong("id"),JSON.parseObject(str,classOf[VehicleInfo])) IgniteClient.basicAlarmTypeInfo.withKeepBinary().put(json.getLong("id"),JSON.parseObject(str,classOf[AlarmTypeInfo]))
case "delete" => case "delete" =>
IgniteClient.basicAlarmTypeInfo.withKeepBinary().remove(json.getLong("id")) IgniteClient.basicAlarmTypeInfo.withKeepBinary().remove(json.getLong("id"))
} }
......
src/main/scala/com/hikcreate/data/sync/SyncIgnite.scala
...@@ -7,10 +7,10 @@ import com.hikcreate.data.constant.Const ...@@ -7,10 +7,10 @@ import com.hikcreate.data.constant.Const
import com.hikcreate.data.model.TableKey import com.hikcreate.data.model.TableKey
import com.hikcreate.data.util.{Tools, ZkManager} import com.hikcreate.data.util.{Tools, ZkManager}
import com.hikcreate.ignite.domain.PrimaryKey import com.hikcreate.ignite.domain.PrimaryKey
import com.hikcreate.ignite.domain.alarm._ import com.hikcreate.ignite.domain.alarm.{AttachmentInfo, DailyAlarm, DailyAlarmDeal, DailyAlarmDetail}
import com.hikcreate.ignite.domain.vehicles._ import com.hikcreate.ignite.domain.vehicles.{AlarmNumber, DailyTravel, DriverNumber, VehicleNumber}
import org.apache.spark.streaming.kafka010._
import org.apache.spark.streaming.{Seconds, StreamingContext} import org.apache.spark.streaming.{Seconds, StreamingContext}
import org.apache.spark.streaming.kafka010._
import org.joda.time.DateTime import org.joda.time.DateTime
import org.joda.time.format.DateTimeFormat import org.joda.time.format.DateTimeFormat
import scala.collection.mutable.ArrayBuffer import scala.collection.mutable.ArrayBuffer
...@@ -22,8 +22,7 @@ object SyncIgnite extends Sparking with Logging{ ...@@ -22,8 +22,7 @@ object SyncIgnite extends Sparking with Logging{
val kafkaParams = getKafkaParams(Const.bootstrap, Const.igniteGroupId) val kafkaParams = getKafkaParams(Const.bootstrap, Const.igniteGroupId)
val offsets = zkManager.getBeginOffset(Const.applicationTopic, Const.igniteGroupId) val offsets = zkManager.getBeginOffset(Const.applicationTopic, Const.igniteGroupId)
val offsetRanges = new ArrayBuffer[OffsetRange]() val offsetRanges = new ArrayBuffer[OffsetRange]()
val ssc = new StreamingContext(conf, Seconds(2)) val ssc = new StreamingContext(conf, Seconds(1))
//ssc.sparkContext.setLogLevel("WARN")
val inputStream = KafkaUtils.createDirectStream[String, String](ssc, val inputStream = KafkaUtils.createDirectStream[String, String](ssc,
LocationStrategies.PreferConsistent, LocationStrategies.PreferConsistent,
ConsumerStrategies.Subscribe[String, String](Const.applicationTopic,kafkaParams,offsets)) ConsumerStrategies.Subscribe[String, String](Const.applicationTopic,kafkaParams,offsets))
...@@ -33,170 +32,175 @@ object SyncIgnite extends Sparking with Logging{ ...@@ -33,170 +32,175 @@ object SyncIgnite extends Sparking with Logging{
rdd rdd
}.map(x => x.value()).foreachRDD{ rdd => }.map(x => x.value()).foreachRDD{ rdd =>
if (!rdd.isEmpty()) { if (!rdd.isEmpty()) {
rdd.foreachPartition(iterator => processRow(iterator)) rdd.map(JSON.parseObject)
zkManager.saveEndOffset(offsetRanges,Const.igniteGroupId) .groupBy(json=>TableKey(Option(json.getString("msgId")),Option(json.getString("dataType"))))
} .foreachPartition( x=>
} x.foreach{ x =>
ssc.start() try{
ssc.awaitTermination() x._1 match {
} case TableKey(Some("0x1200"),Some("0x1202")) => //车辆定位消息
val value = x._2.toList.grouped(20)
value.foreach{ sub =>
val codes = Tools.getLocationCodes(sub.map(x=>(x.getDouble("lon")/1000000D,x.getDouble("lat")/1000000D)))
sub.zip(codes).foreach{ x=>
val json = x._1
val code = x._2
val vehicleNo = json.getString("vehicleNo")
val vehicleColor = json.getString("vehicleColor")
val lon = json.getDouble("lon")/1000000D
val lat = json.getDouble("lat")/1000000D
val dateTime = DateTime.parse(json.getString("dateTime"),DateTimeFormat.forPattern("yyyy-MM-dd HH:mm:ss"))//定位时间
val time = dateTime.toString("yyyy-MM-dd HH:mm:ss")
val date = dateTime.toLocalDate.toString("yyyy-MM-dd")
val vehicleInfoOptional = Tools.getVehicleInfo(vehicleNo,vehicleColor)
def processRow(iterator:Iterator[String]): Unit = { val vehicleProvince = vehicleInfoOptional.map(x=>x.getProvince).getOrElse("无")
iterator.foreach{ x => val vehicleCity = vehicleInfoOptional.map(x=>x.getCity).getOrElse("无")
try { val vehicleArea = vehicleInfoOptional.map(x=>x.getArea).getOrElse("无")
val json = JSON.parseObject(x) val useNature = vehicleInfoOptional.map(x=>x.getUseNature).getOrElse("无")
TableKey(Option(json.getString("msgId")), Option(json.getString("dataType"))) match { //累计行驶车辆数
//车辆定位消息 val vehicleNumberKey = IgniteClient.getBinaryObject(new PrimaryKey(code._1,code._2,code._3,vehicleNo,vehicleColor))
case tableKey if tableKey == TableKey(Some("0x1200"),Some("0x1202")) => val vehicleNumberValue = new VehicleNumber(code._1,code._2,code._3,useNature)
val vehicleNo = json.getString("vehicleNo") IgniteClient.vehicleNumberCache.withKeepBinary().put(vehicleNumberKey,vehicleNumberValue)
val vehicleColor = json.getString("vehicleColor") //累计行驶 累计安全行驶里程 今日车辆在线情况
val lon = json.getDouble("lon")/1000000 val mileageKey = IgniteClient.getBinaryObject(new PrimaryKey(vehicleNo,vehicleColor))
val lat = json.getDouble("lat")/1000000 val mileageValue = new DailyTravel(vehicleProvince,vehicleCity,vehicleArea,useNature,date,lat,lon,time,0D,0L,code._1,code._2,code._3)
val code = Tools.getLocationCode(lon,lat) if(!IgniteClient.dailyTravelCache.withKeepBinary().putIfAbsent(mileageKey,mileageValue)){
val dateTime = DateTime.parse(json.getString("dateTime"),DateTimeFormat.forPattern("yyyy-MM-dd HH:mm:ss"))//定位时间 IgniteClient.updateMileageCache(mileageKey,lat,lon,time)
val time = dateTime.toString("yyyy-MM-dd HH:mm:ss") }
val date = dateTime.toLocalDate.toString("yyyy-MM-dd") }
val vehicleInfoOptional = Tools.getVehicleInfo(vehicleNo,vehicleColor) }
val vehicleProvince = vehicleInfoOptional.map(x=>x.getProvince).getOrElse("无") case TableKey(Some("0x1200"),Some("0x1203")) => //车辆定位消息补报
val vehicleCity = vehicleInfoOptional.map(x=>x.getCity).getOrElse("无") val flat = x._2.flatMap(x=>Tools.addLocation(x))
val vehicleArea = vehicleInfoOptional.map(x=>x.getArea).getOrElse("无") val value = flat.toList.grouped(20)
val useNature = vehicleInfoOptional.map(x=>x.getUseNature).getOrElse("无") value.foreach{ sub =>
//累计行驶车辆数 val codes = Tools.getLocationCodes(sub.map(x=>(x.getDouble("lon")/1000000D,x.getDouble("lat")/1000000D)))
val vehicleNumberKey = IgniteClient.getBinaryObject(new PrimaryKey(code._1,code._2,code._3,vehicleNo,vehicleColor)) sub.zip(codes).foreach{ x=>
val vehicleNumberValue = new VehicleNumber(code._1,code._2,code._3,useNature) val json = x._1
IgniteClient.vehicleNumberCache.withKeepBinary().put(vehicleNumberKey,vehicleNumberValue) val code = x._2
//累计行驶 累计安全行驶里程 今日车辆在线情况 val vehicleNo = json.getString("vehicleNo")
val mileageKey = IgniteClient.getBinaryObject(new PrimaryKey(vehicleNo,vehicleColor)) val vehicleColor = json.getString("vehicleColor")
val mileageValue = new DailyTravel(vehicleProvince,vehicleCity,vehicleArea,useNature,date,lat,lon,time, val vehicleInfoOptional = Tools.getVehicleInfo(vehicleNo,vehicleColor)
0D,0L,code._1,code._2,code._3) val useNature = vehicleInfoOptional.map(x=>x.getUseNature).getOrElse("无")
if(!IgniteClient.dailyTravelCache.withKeepBinary().putIfAbsent(mileageKey,mileageValue)){ //累计行驶车辆数
IgniteClient.updateMileageCache(mileageKey,lat,lon,time) val vehicleNumberKey = IgniteClient.getBinaryObject(new PrimaryKey(code._1,code._2,code._3,vehicleNo,vehicleColor))
} val vehicleNumberValue = new VehicleNumber(code._1,code._2,code._3,useNature)
//车辆定位消息补报 IgniteClient.vehicleNumberCache.withKeepBinary().put(vehicleNumberKey,vehicleNumberValue)
case tableKey if tableKey == TableKey(Some("0x1200"),Some("0x1203")) => }
Tools.addLocation(json).foreach{ x => }
val vehicleNo = x.getString("vehicleNo") case tableKey if tableKey == TableKey(Some("0x1D00"),Some("0x1d02")) => //定时上传驾驶员身份识别信息
val vehicleColor = x.getString("vehicleColor") x._2.foreach{ x=>
val lon = x.getDouble("lon")/1000000 val vehicleNo = x.getString("vehicleNo")
val lat = x.getDouble("lat")/1000000 val vehicleColor = x.getString("vehicleColor")
val code = Tools.getLocationCode(lon,lat) val driverId = x.getString("driverId")
val vehicleInfoOptional = Tools.getVehicleInfo(vehicleNo,vehicleColor) val vehicleInfoOptional = Tools.getVehicleInfo(vehicleNo,vehicleColor)
val useNature = vehicleInfoOptional.map(x=>x.getUseNature).getOrElse("无") val vehicleProvince = vehicleInfoOptional.map(x=>x.getProvince).getOrElse("无")
//累计行驶车辆数 val vehicleCity = vehicleInfoOptional.map(x=>x.getCity).getOrElse("无")
val vehicleNumberKey = IgniteClient.getBinaryObject(new PrimaryKey(code._1,code._2,code._3,vehicleNo,vehicleColor)) val vehicleArea = vehicleInfoOptional.map(x=>x.getArea).getOrElse("无")
val vehicleNumberValue = new VehicleNumber(code._1,code._2,code._3,useNature) val useNature = vehicleInfoOptional.map(x=>x.getUseNature).getOrElse("无")
IgniteClient.vehicleNumberCache.withKeepBinary().put(vehicleNumberKey,vehicleNumberValue) val key = IgniteClient.getBinaryObject(new PrimaryKey(driverId))
} val driverNumber = new DriverNumber(vehicleProvince,vehicleCity,vehicleArea,useNature)
//定时上传驾驶员身份识别信息 IgniteClient.driverNumberCache.withKeepBinary().put(key,driverNumber)
case tableKey if tableKey == TableKey(Some("0x1D00"),Some("0x1d02")) => }
val vehicleNo = json.getString("vehicleNo") case TableKey(Some("0x1C00"),Some("0x1c02")) => //智能视频报警附件上传结果上报
val vehicleColor = json.getString("vehicleColor") x._2.foreach{ x=>
val driverId = json.getString("driverId") x.remove("msgId")
val vehicleInfoOptional = Tools.getVehicleInfo(vehicleNo,vehicleColor) x.remove("dataType")
val vehicleProvince = vehicleInfoOptional.map(x=>x.getProvince).getOrElse("无") val timestamp = x.getLong("warnTime") * 1000
val vehicleCity = vehicleInfoOptional.map(x=>x.getCity).getOrElse("无") val dateTime = new DateTime(timestamp)
val vehicleArea = vehicleInfoOptional.map(x=>x.getArea).getOrElse("无") val warnTime = dateTime.toString("yyyy-MM-dd HH:mm:ss")
val useNature = vehicleInfoOptional.map(x=>x.getUseNature).getOrElse("无") x.put("warnTime",warnTime)
val key = IgniteClient.getBinaryObject(new PrimaryKey(driverId)) val key = IgniteClient.getBinaryObject(
val driverNumber = new DriverNumber(vehicleProvince,vehicleCity,vehicleArea,useNature) new PrimaryKey(x.getString("vehicleNo"),
IgniteClient.driverNumberCache.withKeepBinary().put(key,driverNumber)//接入驾驶人数 x.getString("vehicleColor"),
//智能视频报警附件上传结果上报 x.getString("deviceId"),
case tableKey if tableKey == TableKey(Some("0x1C00"),Some("0x1c02")) => x.getString("warnTime"),
json.remove("msgId") x.getString("fileIndex"))
json.remove("dataType") )
val timestamp = json.getLong("warnTime") * 1000 val value = JSON.parseObject(x.toJSONString,classOf[AttachmentInfo])
val dateTime = new DateTime(timestamp) IgniteClient.attachmentCache.withKeepBinary().put(key,value)
val warnTime = dateTime.toString("yyyy-MM-dd HH:mm:ss") }
json.put("warnTime",warnTime) case TableKey(Some("0x1400"), Some("0x1402")) => //上报报警信息消息
val key = IgniteClient.getBinaryObject( val value = x._2.filter(x=>Const.warnTypes.contains(x.getString("warnType"))).toList.grouped(20)
new PrimaryKey( value.foreach{ sub=>
json.getString("vehicleNo"), val codes = Tools.getLocationCodes(sub.map{ x=>
json.getString("vehicleColor"), val infoStr = x.getString("infoContent")
json.getString("deviceId"), val infoJson = Tools.getInfoContentJsonobj(infoStr)
json.getString("warnTime"), (infoJson.getDouble("LONGITUDE")/1000000D,infoJson.getDouble("LATITUDE")/1000000D)
json.getString("fileIndex")) })
) sub.zip(codes).foreach{ x=>
val value = JSON.parseObject(json.toJSONString,classOf[AttachmentInfo]) val json = x._1
IgniteClient.attachmentCache.withKeepBinary().put(key,value) val code = x._2
//上报报警信息消息 val vehicleNo = json.getString("vehicleNo")
case tableKey if tableKey == TableKey(Some("0x1400"), Some("0x1402")) => val vehicleColor = json.getString("vehicleColor")
val vehicleNo = json.getString("vehicleNo") val warnType = json.getString("warnType")
val vehicleColor = json.getString("vehicleColor") val timestamp = json.getLong("warnTime") * 1000
val warnType = json.getString("warnType") val time = new DateTime(timestamp)
val timestamp = json.getLong("warnTime") * 1000 val warnTime = time.toString("yyyy-MM-dd HH:mm:ss")
if(Const.warnTypes.contains(warnType)){ val date = time.toString("yyyy-MM-dd")
val infoStr = json.getString("infoContent")
val infoJson = Tools.getInfoContentJsonobj(infoStr) val infoStr = json.getString("infoContent")
val deviceId = infoJson.getString("DEVICE_ID") val infoJson = Tools.getInfoContentJsonobj(infoStr)
val lon = infoJson.getDouble("LONGITUDE")/1000000 val eventType = infoJson.getString("EVENT_TYPE")
val lat = infoJson.getDouble("LATITUDE")/1000000 val deviceId = infoJson.getString("DEVICE_ID")
val code = Tools.getLocationCode(lon,lat) val lon = infoJson.getDouble("LONGITUDE")/1000000D
val time = new DateTime(timestamp) val lat = infoJson.getDouble("LATITUDE")/1000000D
val warnTime = time.toString("yyyy-MM-dd HH:mm:ss") val vehicleInfoOptional = Tools.getVehicleInfo(vehicleNo,vehicleColor)
val date = time.toString("yyyy-MM-dd") val useNature = vehicleInfoOptional.map(x=>x.getUseNature).getOrElse("无")
val eventType = infoJson.getString("EVENT_TYPE") val alarmKey = IgniteClient.getBinaryObject(new PrimaryKey(code._1,code._2,code._3,useNature))
val vehicleInfoOptional = Tools.getVehicleInfo(vehicleNo,vehicleColor) val alarmNumber = new AlarmNumber(code._1,code._2,code._3,useNature,1L) //累计行驶报警数
val alarmInfoOptional = Tools.getAlarmInfo(warnType,eventType) if(!IgniteClient.alarmNumberCache.withKeepBinary().putIfAbsent(alarmKey,alarmNumber)){
if(alarmInfoOptional.isDefined){ //是否关联到报警信息基础表 IgniteClient.updateAlarmNumberCache(alarmKey)
val useNature = vehicleInfoOptional.map(x=>x.getUseNature).getOrElse("没有关联到车辆性质") }
val alarmKey = IgniteClient.getBinaryObject(new PrimaryKey(code._1,code._2,code._3,useNature)) val dailyAlarmNumber = new DailyAlarm(code._1,code._2,code._3,useNature,date,1L) //今日报警数
//累计行驶报警数 if(!IgniteClient.dailyAlarmNumberCache.withKeepBinary().putIfAbsent(alarmKey,dailyAlarmNumber)){
val alarmNumber = new AlarmNumber(code._1,code._2,code._3,useNature,1L) IgniteClient.updateDailyAlarmNumberCache(alarmKey,warnTime)
if(!IgniteClient.alarmNumberCache.withKeepBinary().putIfAbsent(alarmKey,alarmNumber)){ }
IgniteClient.updateAlarmNumberCache(alarmKey) val alarmInfoOptional = Tools.getAlarmInfo(warnType,eventType)
} if(alarmInfoOptional.isDefined){ //是否关联到报警信息基础表
//今日报警数 val alarmType = alarmInfoOptional.get.getWarningLevelCode //今日报警详情
val dailyAlarmNumber = new DailyAlarm(code._1,code._2,code._3,useNature,date,1L) val alarmContent = alarmInfoOptional.get.getWarningTypeName
if(!IgniteClient.dailyAlarmNumberCache.withKeepBinary().putIfAbsent(alarmKey,dailyAlarmNumber)){ val alarmDetailKey = IgniteClient.getBinaryObject(new PrimaryKey(vehicleNo,vehicleColor,warnTime))
IgniteClient.updateDailyAlarmNumberCache(alarmKey,warnTime) val dailyAlarmDetail = new DailyAlarmDetail(code._1,code._2,code._3,vehicleNo,vehicleColor,useNature,deviceId,warnTime,lon,lat,alarmType,alarmContent)
IgniteClient.dailyAlarmDetailCache.withKeepBinary().put(alarmDetailKey,dailyAlarmDetail)
}
}
}
case TableKey(Some("0x9400"),Some("0x9401")) => //报警督办请求信息
x._2.foreach{ x=>
val vehicleNo = x.getString("vehicleNo")
val vehicleColor = x.getString("vehicleColor")
val superVisionId = x.getString("superVisionId")
val timestamp = x.getLong("warnTime") * 1000
val warnTime = new DateTime(timestamp).toString("yyyy-MM-dd HH:mm:ss")
val key = IgniteClient.getBinaryObject(new PrimaryKey(vehicleNo,vehicleColor,superVisionId))
val value = new DailyAlarmDeal(vehicleNo,vehicleColor,superVisionId,warnTime,false)
if(!IgniteClient.dailyAlarmDealCache.withKeepBinary().putIfAbsent(key,value)){
IgniteClient.updateDailyAlarmDealCache(key,Map("superVisionId"->superVisionId,"warnTime"->warnTime))
}
}
case tableKey if tableKey == TableKey(Some("0x1400"),Some("0x1401")) => //报警督办应答消息
x._2.foreach{ x=>
val vehicleNo = x.getString("vehicleNo")
val vehicleColor = x.getString("vehicleColor")
val superVisionId = x.getString("supervisionId")
val key = IgniteClient.getBinaryObject(new PrimaryKey(vehicleNo,vehicleColor,superVisionId))
val value = new DailyAlarmDeal(true)
if(!IgniteClient.dailyAlarmDealCache.withKeepBinary().putIfAbsent(key,value)){
IgniteClient.updateDailyAlarmDealCache(key,Map("isDeal"-> (true:java.lang.Boolean)))
}
}
case _ =>
} }
//今日报警详情 }catch {
val alarmType = alarmInfoOptional.get.getWarningLevelCode case e:Exception => e.printStackTrace()
val alarmContent = alarmInfoOptional.get.getWarningTypeName
val alarmDetailKey = IgniteClient.getBinaryObject(new PrimaryKey(vehicleNo,vehicleColor,warnTime))
val dailyAlarmDetail = new DailyAlarmDetail(
code._1,code._2,code._3,vehicleNo,vehicleColor,useNature,
deviceId,warnTime,lon,lat,alarmType,alarmContent)
IgniteClient.dailyAlarmDetailCache.withKeepBinary().put(alarmDetailKey,dailyAlarmDetail)
} }
} }
//报警督办请求信息 )
case tableKey if tableKey == TableKey(Some("0x9400"),Some("0x9401")) => zkManager.saveEndOffset(offsetRanges,Const.igniteGroupId)
val vehicleNo = json.getString("vehicleNo")
val vehicleColor = json.getString("vehicleColor")
val superVisionId = json.getString("superVisionId")
val timestamp = json.getLong("warnTime") * 1000
val time = new DateTime(timestamp)
val warnTime = time.toString("yyyy-MM-dd HH:mm:ss")
val vehicleInfoOptional = Tools.getVehicleInfo(vehicleNo,vehicleColor)
val useNature = vehicleInfoOptional.map(x=>x.getUseNature).getOrElse("无")
val key = IgniteClient.getBinaryObject(new PrimaryKey(vehicleNo,vehicleColor,superVisionId))
val value = new DailyAlarmDeal(vehicleNo,vehicleColor,useNature,superVisionId,warnTime,false)
if(!IgniteClient.dailyAlarmDealCache.withKeepBinary().putIfAbsent(key,value)){
IgniteClient.updateDailyAlarmDealCache(key,Map(
"useNature"->useNature,
"superVisionId"->superVisionId,
"warnTime"->warnTime)
)
}
//报警督办应答消息
case tableKey if tableKey == TableKey(Some("0x1400"),Some("0x1401")) =>
val vehicleNo = json.getString("vehicleNo")
val vehicleColor = json.getString("vehicleColor")
val superVisionId = json.getString("supervisionId")
val key = IgniteClient.getBinaryObject(new PrimaryKey(vehicleNo,vehicleColor,superVisionId))
val value = new DailyAlarmDeal(true)
if(!IgniteClient.dailyAlarmDealCache.withKeepBinary().putIfAbsent(key,value)){
IgniteClient.updateDailyAlarmDealCache(key,Map("isDeal"-> (true:java.lang.Boolean)))
}
case _ =>
}
}catch {
case e:Exception =>
println(x)
println(e.getMessage)
e.printStackTrace()
} }
} }
ssc.start()
ssc.awaitTermination()
} }
} }
src/main/scala/com/hikcreate/data/sync/SyncIgnite1.scala deleted 100644 → 0
package com.hikcreate.data.sync
import com.alibaba.fastjson.JSON
import com.hikcreate.data.client.IgniteClient
import com.hikcreate.data.common.{Logging, Sparking}
import com.hikcreate.data.constant.Const
import com.hikcreate.data.model.TableKey
import com.hikcreate.data.util.{Tools, ZkManager}
import com.hikcreate.ignite.domain.PrimaryKey
import com.hikcreate.ignite.domain.alarm.{AttachmentInfo, DailyAlarm, DailyAlarmDeal, DailyAlarmDetail}
import com.hikcreate.ignite.domain.vehicles.{AlarmNumber, DailyTravel, DriverNumber, VehicleNumber}
import org.apache.spark.streaming.{Seconds, StreamingContext}
import org.apache.spark.streaming.kafka010._
import org.joda.time.DateTime
import org.joda.time.format.DateTimeFormat
import scala.collection.mutable.ArrayBuffer
object SyncIgnite1 extends Sparking with Logging{
def main(args: Array[String]): Unit = {
val zkManager = ZkManager(Const.zkKafka)
val kafkaParams = getKafkaParams(Const.bootstrap, Const.igniteGroupId)
val offsets = zkManager.getBeginOffset(Const.applicationTopic, Const.igniteGroupId)
val offsetRanges = new ArrayBuffer[OffsetRange]()
val ssc = new StreamingContext(conf, Seconds(1))
val inputStream = KafkaUtils.createDirectStream[String, String](ssc,
LocationStrategies.PreferConsistent,
ConsumerStrategies.Subscribe[String, String](Const.applicationTopic,kafkaParams,offsets))
inputStream.transform { rdd =>
offsetRanges.clear()
offsetRanges.append(rdd.asInstanceOf[HasOffsetRanges].offsetRanges: _*)
rdd
}.map(x => x.value()).foreachRDD{ rdd =>
if (!rdd.isEmpty()) {
rdd.map(JSON.parseObject)
.groupBy(json=>TableKey(Option(json.getString("msgId")),Option(json.getString("dataType"))))
.foreachPartition( x=>
x.foreach{ x =>
try{
x._1 match {
case TableKey(Some("0x1200"),Some("0x1202")) => //车辆定位消息
val value = x._2.toList.grouped(20)
value.foreach{ sub =>
val codes = Tools.getLocationCodes(sub.map(x=>(x.getDouble("lon")/1000000,x.getDouble("lat")/1000000)))
sub.zip(codes).foreach{ x=>
val json = x._1
val code = x._2
val vehicleNo = json.getString("vehicleNo")
val vehicleColor = json.getString("vehicleColor")
val lon = json.getDouble("lon")/1000000
val lat = json.getDouble("lat")/1000000
val dateTime = DateTime.parse(json.getString("dateTime"),DateTimeFormat.forPattern("yyyy-MM-dd HH:mm:ss"))//定位时间
val time = dateTime.toString("yyyy-MM-dd HH:mm:ss")
val date = dateTime.toLocalDate.toString("yyyy-MM-dd")
val vehicleInfoOptional = Tools.getVehicleInfo(vehicleNo,vehicleColor)
val vehicleProvince = vehicleInfoOptional.map(x=>x.getProvince).getOrElse("无")
val vehicleCity = vehicleInfoOptional.map(x=>x.getCity).getOrElse("无")
val vehicleArea = vehicleInfoOptional.map(x=>x.getArea).getOrElse("无")
val useNature = vehicleInfoOptional.map(x=>x.getUseNature).getOrElse("无")
//累计行驶车辆数
val vehicleNumberKey = IgniteClient.getBinaryObject(new PrimaryKey(code._1,code._2,code._3,vehicleNo,vehicleColor))
val vehicleNumberValue = new VehicleNumber(code._1,code._2,code._3,useNature)
IgniteClient.vehicleNumberCache.withKeepBinary().put(vehicleNumberKey,vehicleNumberValue)
//累计行驶 累计安全行驶里程 今日车辆在线情况
val mileageKey = IgniteClient.getBinaryObject(new PrimaryKey(vehicleNo,vehicleColor))
val mileageValue = new DailyTravel(vehicleProvince,vehicleCity,vehicleArea,useNature,
date, lat,lon,time, 0D,0L,code._1,code._2,code._3)
if(!IgniteClient.dailyTravelCache.withKeepBinary().putIfAbsent(mileageKey,mileageValue)){
IgniteClient.updateMileageCache(mileageKey,lat,lon,time)
}
}
}
case TableKey(Some("0x1200"),Some("0x1203")) => //车辆定位消息补报
val flat = x._2.flatMap(x=>Tools.addLocation(x))
val value = flat.toList.grouped(20)
value.foreach{ sub =>
val codes = Tools.getLocationCodes(sub.map(x=>(x.getDouble("lon")/1000000,x.getDouble("lat")/1000000)))
sub.zip(codes).foreach{ x=>
val json = x._1
val code = x._2
val vehicleNo = json.getString("vehicleNo")
val vehicleColor = json.getString("vehicleColor")
val vehicleInfoOptional = Tools.getVehicleInfo(vehicleNo,vehicleColor)
val useNature = vehicleInfoOptional.map(x=>x.getUseNature).getOrElse("无")
//累计行驶车辆数
val vehicleNumberKey = IgniteClient.getBinaryObject(new PrimaryKey(code._1,code._2,code._3,vehicleNo,vehicleColor))
val vehicleNumberValue = new VehicleNumber(code._1,code._2,code._3,useNature)
IgniteClient.vehicleNumberCache.withKeepBinary().put(vehicleNumberKey,vehicleNumberValue)
}
}
case tableKey if tableKey == TableKey(Some("0x1D00"),Some("0x1d02")) => //定时上传驾驶员身份识别信息
x._2.foreach{ x=>
val vehicleNo = x.getString("vehicleNo")
val vehicleColor = x.getString("vehicleColor")
val driverId = x.getString("driverId")
val vehicleInfoOptional = Tools.getVehicleInfo(vehicleNo,vehicleColor)
val vehicleProvince = vehicleInfoOptional.map(x=>x.getProvince).getOrElse("无")
val vehicleCity = vehicleInfoOptional.map(x=>x.getCity).getOrElse("无")
val vehicleArea = vehicleInfoOptional.map(x=>x.getArea).getOrElse("无")
val useNature = vehicleInfoOptional.map(x=>x.getUseNature).getOrElse("无")
val key = IgniteClient.getBinaryObject(new PrimaryKey(driverId))
val driverNumber = new DriverNumber(vehicleProvince,vehicleCity,vehicleArea,useNature)
IgniteClient.driverNumberCache.withKeepBinary().put(key,driverNumber)
}
case TableKey(Some("0x1C00"),Some("0x1c02")) => //智能视频报警附件上传结果上报
x._2.foreach{ x=>
x.remove("msgId")
x.remove("dataType")
val timestamp = x.getLong("warnTime") * 1000
val dateTime = new DateTime(timestamp)
val warnTime = dateTime.toString("yyyy-MM-dd HH:mm:ss")
x.put("warnTime",warnTime)
val key = IgniteClient.getBinaryObject(
new PrimaryKey(x.getString("vehicleNo"),
x.getString("vehicleColor"),
x.getString("deviceId"),
x.getString("warnTime"),
x.getString("fileIndex"))
)
val value = JSON.parseObject(x.toJSONString,classOf[AttachmentInfo])
IgniteClient.attachmentCache.withKeepBinary().put(key,value)
}
case TableKey(Some("0x1400"), Some("0x1402")) => //上报报警信息消息
val value = x._2.filter(x=>Const.warnTypes.contains(x.getString("warnType"))).toList.grouped(20)
value.foreach{ sub=>
val codes = Tools.getLocationCodes(sub
.map{ x=>
val infoStr = x.getString("infoContent")
val infoJson = Tools.getInfoContentJsonobj(infoStr)
(infoJson.getDouble("LONGITUDE")/1000000,infoJson.getDouble("LATITUDE")/1000000)
})
sub.zip(codes).foreach{ x=>
val json = x._1
val code = x._2
val vehicleNo = json.getString("vehicleNo")
val vehicleColor = json.getString("vehicleColor")
val warnType = json.getString("warnType")
val timestamp = json.getLong("warnTime") * 1000
val time = new DateTime(timestamp)
val warnTime = time.toString("yyyy-MM-dd HH:mm:ss")
val date = time.toString("yyyy-MM-dd")
val infoStr = json.getString("infoContent")
val infoJson = Tools.getInfoContentJsonobj(infoStr)
val eventType = infoJson.getString("EVENT_TYPE")
val deviceId = infoJson.getString("DEVICE_ID")
val lon = infoJson.getDouble("LONGITUDE")/1000000
val lat = infoJson.getDouble("LATITUDE")/1000000
val vehicleInfoOptional = Tools.getVehicleInfo(vehicleNo,vehicleColor)
val alarmInfoOptional = Tools.getAlarmInfo(warnType,eventType)
if(alarmInfoOptional.isDefined){ //是否关联到报警信息基础表
val useNature = vehicleInfoOptional.map(x=>x.getUseNature).getOrElse("没有关联到车辆性质")
val alarmKey = IgniteClient.getBinaryObject(new PrimaryKey(code._1,code._2,code._3,useNature))
//累计行驶报警数
val alarmNumber = new AlarmNumber(code._1,code._2,code._3,useNature,1L)
if(!IgniteClient.alarmNumberCache.withKeepBinary().putIfAbsent(alarmKey,alarmNumber)){
IgniteClient.updateAlarmNumberCache(alarmKey)
}
//今日报警数
val dailyAlarmNumber = new DailyAlarm(code._1,code._2,code._3,useNature,date,1L)
if(!IgniteClient.dailyAlarmNumberCache.withKeepBinary().putIfAbsent(alarmKey,dailyAlarmNumber)){
IgniteClient.updateDailyAlarmNumberCache(alarmKey,warnTime)
}
//今日报警详情
val alarmType = alarmInfoOptional.get.getWarningLevelCode
val alarmContent = alarmInfoOptional.get.getWarningTypeName
val alarmDetailKey = IgniteClient.getBinaryObject(new PrimaryKey(vehicleNo,vehicleColor,warnTime))
val dailyAlarmDetail = new DailyAlarmDetail(code._1,code._2,code._3,vehicleNo,vehicleColor,useNature,
deviceId,warnTime,lon,lat,alarmType,alarmContent)
IgniteClient.dailyAlarmDetailCache.withKeepBinary().put(alarmDetailKey,dailyAlarmDetail)
}
}
}
//报警督办请求信息
case TableKey(Some("0x9400"),Some("0x9401")) =>
x._2.foreach{ x=>
val vehicleNo = x.getString("vehicleNo")
val vehicleColor = x.getString("vehicleColor")
val superVisionId = x.getString("superVisionId")
val timestamp = x.getLong("warnTime") * 1000
val warnTime = new DateTime(timestamp).toString("yyyy-MM-dd HH:mm:ss")
val vehicleInfoOptional = Tools.getVehicleInfo(vehicleNo,vehicleColor)
val useNature = vehicleInfoOptional.map(x=>x.getUseNature).getOrElse("无")
val key = IgniteClient.getBinaryObject(new PrimaryKey(vehicleNo,vehicleColor,superVisionId))
val value = new DailyAlarmDeal(vehicleNo,vehicleColor,useNature,superVisionId,warnTime,false)
if(!IgniteClient.dailyAlarmDealCache.withKeepBinary().putIfAbsent(key,value)){
IgniteClient.updateDailyAlarmDealCache(key,Map(
"useNature"->useNature,
"superVisionId"->superVisionId,
"warnTime"->warnTime)
)
}
}
//报警督办应答消息
case tableKey if tableKey == TableKey(Some("0x1400"),Some("0x1401")) =>
x._2.foreach{ x=>
val vehicleNo = x.getString("vehicleNo")
val vehicleColor = x.getString("vehicleColor")
val superVisionId = x.getString("supervisionId")
val key = IgniteClient.getBinaryObject(new PrimaryKey(vehicleNo,vehicleColor,superVisionId))
val value = new DailyAlarmDeal(true)
if(!IgniteClient.dailyAlarmDealCache.withKeepBinary().putIfAbsent(key,value)){
IgniteClient.updateDailyAlarmDealCache(key,Map("isDeal"-> (true:java.lang.Boolean)))
}
}
case _ =>
}
}catch {
case e:Exception => e.printStackTrace()
}
}
)
zkManager.saveEndOffset(offsetRanges,Const.igniteGroupId)
}
}
ssc.start()
ssc.awaitTermination()
}
}
src/main/scala/com/hikcreate/data/util/Tools.scala
...@@ -36,13 +36,14 @@ object Tools extends Logging{ ...@@ -36,13 +36,14 @@ object Tools extends Logging{
lonAndLat.put("latitude",lat) lonAndLat.put("latitude",lat)
arr.add(lonAndLat) arr.add(lonAndLat)
json.put("locations",arr) json.put("locations",arr)
val starttime = DateTime.now() println(json.toJSONString)
//val starttime = DateTime.now()
val response = Http(Const.areaCodeAndAddressUrl).postData(json.toJSONString) val response = Http(Const.areaCodeAndAddressUrl).postData(json.toJSONString)
.header("content-type","application/json") .header("content-type","application/json")
//.timeout(connTimeoutMs = 1000,readTimeoutMs = 1000) //.timeout(connTimeoutMs = 1000,readTimeoutMs = 1000)
.asString .asString
val endtime = DateTime.now() val endtime = DateTime.now()
println("http请求时间："+new Duration(starttime,endtime).getMillis) //println("http请求时间："+new Duration(starttime,endtime).getMillis)
val body = JSON.parseObject(response.body) val body = JSON.parseObject(response.body)
val item = body.getJSONObject("result").getJSONArray("regeoItems").getJSONObject(0) val item = body.getJSONObject("result").getJSONArray("regeoItems").getJSONObject(0)
val address = item.getString("formattedAddress") val address = item.getString("formattedAddress")
...@@ -78,6 +79,7 @@ object Tools extends Logging{ ...@@ -78,6 +79,7 @@ object Tools extends Logging{
//val endTime = DateTime.now() //val endTime = DateTime.now()
//println("http请求时间："+new Duration(startTime,endTime).getMillis/1000) //println("http请求时间："+new Duration(startTime,endTime).getMillis/1000)
val body = JSON.parseObject(response.body) val body = JSON.parseObject(response.body)
println(response.body)
val items = body.getJSONObject("result").getJSONArray("regeoItems") val items = body.getJSONObject("result").getJSONArray("regeoItems")
(0 until items.size()).map{ index => (0 until items.size()).map{ index =>
val item = items.getJSONObject(index) val item = items.getJSONObject(index)
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment