Commit 26a16293 by 杜发飞

1

parent aa830172
...@@ -132,7 +132,7 @@ ...@@ -132,7 +132,7 @@
<build> <build>
<plugins> <plugins>
<plugin> <!--<plugin>
<artifactId>maven-dependency-plugin</artifactId> <artifactId>maven-dependency-plugin</artifactId>
<executions> <executions>
<execution> <execution>
...@@ -146,12 +146,17 @@ ...@@ -146,12 +146,17 @@
<includeScope>runtime</includeScope> <includeScope>runtime</includeScope>
<excludeTransitive>false</excludeTransitive> <excludeTransitive>false</excludeTransitive>
</configuration> </configuration>
</plugin> </plugin>-->
<plugin> <plugin>
<groupId>org.scala-tools</groupId> <groupId>org.scala-tools</groupId>
<artifactId>maven-scala-plugin</artifactId> <artifactId>maven-scala-plugin</artifactId>
<version>2.15.2</version> <version>2.15.2</version>
<executions><execution><goals><goal>compile</goal></goals></execution> <executions>
<execution>
<goals>
<goal>compile</goal>
</goals>
</execution>
</executions> </executions>
</plugin> </plugin>
</plugins> </plugins>
......
...@@ -208,7 +208,7 @@ object IgniteClient { ...@@ -208,7 +208,7 @@ object IgniteClient {
) )
def main(args: Array[String]): Unit = { def main(args: Array[String]): Unit = {
//ignite.cacheNames().asScala.foreach(x=>ignite.destroyCache(x)) ignite.cacheNames().asScala.foreach(x=>ignite.getOrCreateCache(x).clear())
ignite.close() ignite.close()
} }
} }
package com.hikcreate.data.common package com.hikcreate.data.common
import org.apache.log4j.{Level, Logger}
import org.apache.kafka.common.serialization.StringDeserializer import org.apache.kafka.common.serialization.StringDeserializer
import org.apache.spark.SparkConf import org.apache.spark.SparkConf
trait Sparking { trait Sparking {
// 屏蔽不必要的日志 ,在终端上显示需要的日志
Logger.getLogger("org.apache.spark").setLevel(Level.WARN)
Logger.getLogger("org.eclipse.jetty.server").setLevel(Level.WARN)
Logger.getLogger("org.apache.kafka.clients.consumer").setLevel(Level.WARN)
val conf: SparkConf = new SparkConf() val conf: SparkConf = new SparkConf()
.set("spark.serializer","org.apache.spark.serializer.KryoSerializer") .set("spark.serializer","org.apache.spark.serializer.KryoSerializer")
.set("hive.exec.dynamici.partition","true") .set("hive.exec.dynamici.partition","true")
.set("hive.exec.dynamic.partition.mode","nonstrict") .set("hive.exec.dynamic.partition.mode","nonstrict")
.setAppName("test") //.setAppName("test")
.setMaster("local[*]") //.setMaster("local[*]")
def getKafkaParams(servers:String,groupId: String):Map[String,Object] = { def getKafkaParams(servers:String,groupId: String):Map[String,Object] = {
Map[String,Object]( Map[String,Object](
......
...@@ -22,7 +22,8 @@ object SyncIgnite extends Sparking with Logging{ ...@@ -22,7 +22,8 @@ object SyncIgnite extends Sparking with Logging{
val kafkaParams = getKafkaParams(Const.bootstrap, Const.igniteGroupId) val kafkaParams = getKafkaParams(Const.bootstrap, Const.igniteGroupId)
val offsets = zkManager.getBeginOffset(Const.applicationTopic, Const.igniteGroupId) val offsets = zkManager.getBeginOffset(Const.applicationTopic, Const.igniteGroupId)
val offsetRanges = new ArrayBuffer[OffsetRange]() val offsetRanges = new ArrayBuffer[OffsetRange]()
val ssc = new StreamingContext(conf, Seconds(1)) val ssc = new StreamingContext(conf, Seconds(2))
//ssc.sparkContext.setLogLevel("WARN")
val inputStream = KafkaUtils.createDirectStream[String, String](ssc, val inputStream = KafkaUtils.createDirectStream[String, String](ssc,
LocationStrategies.PreferConsistent, LocationStrategies.PreferConsistent,
ConsumerStrategies.Subscribe[String, String](Const.applicationTopic,kafkaParams,offsets)) ConsumerStrategies.Subscribe[String, String](Const.applicationTopic,kafkaParams,offsets))
...@@ -43,7 +44,6 @@ object SyncIgnite extends Sparking with Logging{ ...@@ -43,7 +44,6 @@ object SyncIgnite extends Sparking with Logging{
def processRow(iterator:Iterator[String]): Unit = { def processRow(iterator:Iterator[String]): Unit = {
iterator.foreach{ x => iterator.foreach{ x =>
try { try {
println(x)
val json = JSON.parseObject(x) val json = JSON.parseObject(x)
TableKey(Option(json.getString("msgId")), Option(json.getString("dataType"))) match { TableKey(Option(json.getString("msgId")), Option(json.getString("dataType"))) match {
//车辆定位消息 //车辆定位消息
...@@ -137,7 +137,7 @@ object SyncIgnite extends Sparking with Logging{ ...@@ -137,7 +137,7 @@ object SyncIgnite extends Sparking with Logging{
val eventType = infoJson.getString("EVENT_TYPE") val eventType = infoJson.getString("EVENT_TYPE")
val vehicleInfoOptional = Tools.getVehicleInfo(vehicleNo,vehicleColor) val vehicleInfoOptional = Tools.getVehicleInfo(vehicleNo,vehicleColor)
val alarmInfoOptional = Tools.getAlarmInfo(warnType,eventType) val alarmInfoOptional = Tools.getAlarmInfo(warnType,eventType)
if(alarmInfoOptional.isDefined){//是否关联到报警信息基础表 if(alarmInfoOptional.isDefined){ //是否关联到报警信息基础表
val useNature = vehicleInfoOptional.map(x=>x.getUseNature).getOrElse("没有关联到车辆性质") val useNature = vehicleInfoOptional.map(x=>x.getUseNature).getOrElse("没有关联到车辆性质")
val alarmKey = IgniteClient.getBinaryObject(new PrimaryKey(code._1,code._2,code._3,useNature)) val alarmKey = IgniteClient.getBinaryObject(new PrimaryKey(code._1,code._2,code._3,useNature))
//累计行驶报警数 //累计行驶报警数
...@@ -183,7 +183,7 @@ object SyncIgnite extends Sparking with Logging{ ...@@ -183,7 +183,7 @@ object SyncIgnite extends Sparking with Logging{
case tableKey if tableKey == TableKey(Some("0x1400"),Some("0x1401")) => case tableKey if tableKey == TableKey(Some("0x1400"),Some("0x1401")) =>
val vehicleNo = json.getString("vehicleNo") val vehicleNo = json.getString("vehicleNo")
val vehicleColor = json.getString("vehicleColor") val vehicleColor = json.getString("vehicleColor")
val superVisionId = json.getString("superVisionId") val superVisionId = json.getString("supervisionId")
val key = IgniteClient.getBinaryObject(new PrimaryKey(vehicleNo,vehicleColor,superVisionId)) val key = IgniteClient.getBinaryObject(new PrimaryKey(vehicleNo,vehicleColor,superVisionId))
val value = new DailyAlarmDeal(true) val value = new DailyAlarmDeal(true)
if(!IgniteClient.dailyAlarmDealCache.withKeepBinary().putIfAbsent(key,value)){ if(!IgniteClient.dailyAlarmDealCache.withKeepBinary().putIfAbsent(key,value)){
...@@ -193,6 +193,8 @@ object SyncIgnite extends Sparking with Logging{ ...@@ -193,6 +193,8 @@ object SyncIgnite extends Sparking with Logging{
} }
}catch { }catch {
case e:Exception => case e:Exception =>
println(x)
println(e.getMessage)
e.printStackTrace() e.printStackTrace()
} }
} }
......
...@@ -8,8 +8,11 @@ import com.hikcreate.data.common.Logging ...@@ -8,8 +8,11 @@ import com.hikcreate.data.common.Logging
import com.hikcreate.data.constant.Const import com.hikcreate.data.constant.Const
import com.hikcreate.ignite.domain.basic.{AlarmTypeInfo, EnterpriseInfo, VehicleInfo} import com.hikcreate.ignite.domain.basic.{AlarmTypeInfo, EnterpriseInfo, VehicleInfo}
import org.apache.ignite.cache.query.SqlQuery import org.apache.ignite.cache.query.SqlQuery
import org.joda.time.{DateTime, Duration}
import scalaj.http.Http import scalaj.http.Http
import scala.collection.mutable.ArrayBuffer
object Tools extends Logging{ object Tools extends Logging{
def addLocation(json:JSONObject): Array[JSONObject] = { def addLocation(json:JSONObject): Array[JSONObject] = {
...@@ -25,25 +28,29 @@ object Tools extends Logging{ ...@@ -25,25 +28,29 @@ object Tools extends Logging{
} }
def getAddressAndLocationCode(lon:Double,lat:Double):(String,String) = { def getAddressAndLocationCode(lon:Double,lat:Double):(String,String) = {
val json = new JSONObject() try{
val arr = new JSONArray() val json = new JSONObject()
val lonAndLat = new JSONObject() val arr = new JSONArray()
lonAndLat.put("longitude",lon) val lonAndLat = new JSONObject()
lonAndLat.put("latitude",lat) lonAndLat.put("longitude",lon)
arr.add(lonAndLat) lonAndLat.put("latitude",lat)
json.put("locations",arr) arr.add(lonAndLat)
val response = Http(Const.areaCodeAndAddressUrl).postData(json.toJSONString) json.put("locations",arr)
.header("content-type","application/json")//.charset("ISO-8859-1") val starttime = DateTime.now()
.timeout(connTimeoutMs = 8000, readTimeoutMs = 8000) val response = Http(Const.areaCodeAndAddressUrl).postData(json.toJSONString)
.asString .header("content-type","application/json")
if(response.code == 200){ //.timeout(connTimeoutMs = 1000,readTimeoutMs = 1000)
.asString
val endtime = DateTime.now()
println("http请求时间:"+new Duration(starttime,endtime).getMillis)
val body = JSON.parseObject(response.body) val body = JSON.parseObject(response.body)
val item = body.getJSONObject("result").getJSONArray("regeoItems").getJSONObject(0) val item = body.getJSONObject("result").getJSONArray("regeoItems").getJSONObject(0)
val address = item.getString("formattedAddress") val address = item.getString("formattedAddress")
val locationCode = item.getJSONObject("addressComponent").getString("adcode") val locationCode = item.getJSONObject("addressComponent").getString("adcode")
(address,locationCode) (address,locationCode)
}else{ }catch{
throw new RuntimeException("http请求城市区编码出错") case e:Exception =>
throw new RuntimeException(e)
} }
} }
...@@ -56,6 +63,39 @@ object Tools extends Logging{ ...@@ -56,6 +63,39 @@ object Tools extends Logging{
(provinceCode,cityCode,areaCode) (provinceCode,cityCode,areaCode)
} }
def getAddressAndLocationCodes(buffer:List[(Double,Double)]): IndexedSeq[(String,String)] = {
val json = new JSONObject()
val arr = new JSONArray()
buffer.foreach{x =>
val lonAndLat = new JSONObject()
lonAndLat.put("longitude",x._1)
lonAndLat.put("latitude",x._2)
arr.add(lonAndLat)
}
json.put("locations",arr)
val startTime = DateTime.now()
val response = Http(Const.areaCodeAndAddressUrl).postData(json.toJSONString).header("content-type","application/json").asString
val endTime = DateTime.now()
println("http请求时间:"+new Duration(startTime,endTime).getMillis)
val body = JSON.parseObject(response.body)
val items = body.getJSONObject("result").getJSONArray("regeoItems")
(0 until items.size()).map{ index =>
val item = items.getJSONObject(index)
val address = item.getString("formattedAddress")
val locationCode = item.getJSONObject("addressComponent").getString("adcode")
(address,locationCode)
}
}
def getLocationCodes(buffer:List[(Double,Double)]): IndexedSeq[(String,String,String)] = {
getAddressAndLocationCodes(buffer).map(x=>x._2).map{ locationCode =>
val provinceCode = locationCode.substring(0,2)
val cityCode = locationCode.substring(2,4)
val areaCode = locationCode.substring(4,6)
(provinceCode,cityCode,areaCode)
}
}
def getInfoContentJsonobj(infoStr:String):JSONObject = { def getInfoContentJsonobj(infoStr:String):JSONObject = {
val jsonStr=("{\""+infoStr.replace(":=","\":\"").replace(";","\",\"")+"\"}").replace(",\"\"}","}") val jsonStr=("{\""+infoStr.replace(":=","\":\"").replace(";","\",\"")+"\"}").replace(",\"\"}","}")
val jSONObject = JSON.parseObject(jsonStr) val jSONObject = JSON.parseObject(jsonStr)
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment