Skip to content
Projects
Groups
Snippets
Help
This project
Loading...
Sign in / Register
Toggle navigation
O
operating-vehicle
Overview
Overview
Details
Activity
Cycle Analytics
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Charts
Issues
0
Issues
0
List
Board
Labels
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Charts
Wiki
Wiki
Snippets
Snippets
Members
Collapse sidebar
Close sidebar
Activity
Graph
Charts
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
杜发飞
operating-vehicle
Commits
26a16293
Commit
26a16293
authored
Oct 26, 2019
by
杜发飞
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
1
parent
aa830172
Expand all
Hide whitespace changes
Inline
Side-by-side
Showing
6 changed files
with
77 additions
and
24 deletions
+77
-24
pom.xml
+8
-3
src/main/scala/com/hikcreate/data/client/IgniteClient.scala
+1
-1
src/main/scala/com/hikcreate/data/common/Sparking.scala
+8
-2
src/main/scala/com/hikcreate/data/sync/SyncIgnite.scala
+6
-4
src/main/scala/com/hikcreate/data/sync/SyncIgnite1.scala
+0
-0
src/main/scala/com/hikcreate/data/util/Tools.scala
+54
-14
No files found.
pom.xml
View file @
26a16293
...
@@ -132,7 +132,7 @@
...
@@ -132,7 +132,7 @@
<build>
<build>
<plugins>
<plugins>
<plugin>
<
!--<
plugin>
<artifactId>maven-dependency-plugin</artifactId>
<artifactId>maven-dependency-plugin</artifactId>
<executions>
<executions>
<execution>
<execution>
...
@@ -146,12 +146,17 @@
...
@@ -146,12 +146,17 @@
<includeScope>runtime</includeScope>
<includeScope>runtime</includeScope>
<excludeTransitive>false</excludeTransitive>
<excludeTransitive>false</excludeTransitive>
</configuration>
</configuration>
</plugin>
</plugin>
-->
<plugin>
<plugin>
<groupId>
org.scala-tools
</groupId>
<groupId>
org.scala-tools
</groupId>
<artifactId>
maven-scala-plugin
</artifactId>
<artifactId>
maven-scala-plugin
</artifactId>
<version>
2.15.2
</version>
<version>
2.15.2
</version>
<executions><execution><goals><goal>
compile
</goal></goals></execution>
<executions>
<execution>
<goals>
<goal>
compile
</goal>
</goals>
</execution>
</executions>
</executions>
</plugin>
</plugin>
</plugins>
</plugins>
...
...
src/main/scala/com/hikcreate/data/client/IgniteClient.scala
View file @
26a16293
...
@@ -208,7 +208,7 @@ object IgniteClient {
...
@@ -208,7 +208,7 @@ object IgniteClient {
)
)
def
main
(
args
:
Array
[
String
])
:
Unit
=
{
def
main
(
args
:
Array
[
String
])
:
Unit
=
{
//ignite.cacheNames().asScala.foreach(x=>ignite.destroyCache(x
))
ignite
.
cacheNames
().
asScala
.
foreach
(
x
=>
ignite
.
getOrCreateCache
(
x
).
clear
(
))
ignite
.
close
()
ignite
.
close
()
}
}
}
}
src/main/scala/com/hikcreate/data/common/Sparking.scala
View file @
26a16293
package
com.hikcreate.data.common
package
com.hikcreate.data.common
import
org.apache.log4j.
{
Level
,
Logger
}
import
org.apache.kafka.common.serialization.StringDeserializer
import
org.apache.kafka.common.serialization.StringDeserializer
import
org.apache.spark.SparkConf
import
org.apache.spark.SparkConf
trait
Sparking
{
trait
Sparking
{
// 屏蔽不必要的日志 ,在终端上显示需要的日志
Logger
.
getLogger
(
"org.apache.spark"
).
setLevel
(
Level
.
WARN
)
Logger
.
getLogger
(
"org.eclipse.jetty.server"
).
setLevel
(
Level
.
WARN
)
Logger
.
getLogger
(
"org.apache.kafka.clients.consumer"
).
setLevel
(
Level
.
WARN
)
val
conf
:
SparkConf
=
new
SparkConf
()
val
conf
:
SparkConf
=
new
SparkConf
()
.
set
(
"spark.serializer"
,
"org.apache.spark.serializer.KryoSerializer"
)
.
set
(
"spark.serializer"
,
"org.apache.spark.serializer.KryoSerializer"
)
.
set
(
"hive.exec.dynamici.partition"
,
"true"
)
.
set
(
"hive.exec.dynamici.partition"
,
"true"
)
.
set
(
"hive.exec.dynamic.partition.mode"
,
"nonstrict"
)
.
set
(
"hive.exec.dynamic.partition.mode"
,
"nonstrict"
)
.
setAppName
(
"test"
)
//
.setAppName("test")
.
setMaster
(
"local[*]"
)
//
.setMaster("local[*]")
def
getKafkaParams
(
servers
:
String
,
groupId
:
String
)
:
Map
[
String
,
Object
]
=
{
def
getKafkaParams
(
servers
:
String
,
groupId
:
String
)
:
Map
[
String
,
Object
]
=
{
Map
[
String
,
Object
](
Map
[
String
,
Object
](
...
...
src/main/scala/com/hikcreate/data/sync/SyncIgnite.scala
View file @
26a16293
...
@@ -22,7 +22,8 @@ object SyncIgnite extends Sparking with Logging{
...
@@ -22,7 +22,8 @@ object SyncIgnite extends Sparking with Logging{
val
kafkaParams
=
getKafkaParams
(
Const
.
bootstrap
,
Const
.
igniteGroupId
)
val
kafkaParams
=
getKafkaParams
(
Const
.
bootstrap
,
Const
.
igniteGroupId
)
val
offsets
=
zkManager
.
getBeginOffset
(
Const
.
applicationTopic
,
Const
.
igniteGroupId
)
val
offsets
=
zkManager
.
getBeginOffset
(
Const
.
applicationTopic
,
Const
.
igniteGroupId
)
val
offsetRanges
=
new
ArrayBuffer
[
OffsetRange
]()
val
offsetRanges
=
new
ArrayBuffer
[
OffsetRange
]()
val
ssc
=
new
StreamingContext
(
conf
,
Seconds
(
1
))
val
ssc
=
new
StreamingContext
(
conf
,
Seconds
(
2
))
//ssc.sparkContext.setLogLevel("WARN")
val
inputStream
=
KafkaUtils
.
createDirectStream
[
String
,
String
](
ssc
,
val
inputStream
=
KafkaUtils
.
createDirectStream
[
String
,
String
](
ssc
,
LocationStrategies
.
PreferConsistent
,
LocationStrategies
.
PreferConsistent
,
ConsumerStrategies
.
Subscribe
[
String
,
String
](
Const
.
applicationTopic
,
kafkaParams
,
offsets
))
ConsumerStrategies
.
Subscribe
[
String
,
String
](
Const
.
applicationTopic
,
kafkaParams
,
offsets
))
...
@@ -43,7 +44,6 @@ object SyncIgnite extends Sparking with Logging{
...
@@ -43,7 +44,6 @@ object SyncIgnite extends Sparking with Logging{
def
processRow
(
iterator
:
Iterator
[
String
])
:
Unit
=
{
def
processRow
(
iterator
:
Iterator
[
String
])
:
Unit
=
{
iterator
.
foreach
{
x
=>
iterator
.
foreach
{
x
=>
try
{
try
{
println
(
x
)
val
json
=
JSON
.
parseObject
(
x
)
val
json
=
JSON
.
parseObject
(
x
)
TableKey
(
Option
(
json
.
getString
(
"msgId"
)),
Option
(
json
.
getString
(
"dataType"
)))
match
{
TableKey
(
Option
(
json
.
getString
(
"msgId"
)),
Option
(
json
.
getString
(
"dataType"
)))
match
{
//车辆定位消息
//车辆定位消息
...
@@ -137,7 +137,7 @@ object SyncIgnite extends Sparking with Logging{
...
@@ -137,7 +137,7 @@ object SyncIgnite extends Sparking with Logging{
val
eventType
=
infoJson
.
getString
(
"EVENT_TYPE"
)
val
eventType
=
infoJson
.
getString
(
"EVENT_TYPE"
)
val
vehicleInfoOptional
=
Tools
.
getVehicleInfo
(
vehicleNo
,
vehicleColor
)
val
vehicleInfoOptional
=
Tools
.
getVehicleInfo
(
vehicleNo
,
vehicleColor
)
val
alarmInfoOptional
=
Tools
.
getAlarmInfo
(
warnType
,
eventType
)
val
alarmInfoOptional
=
Tools
.
getAlarmInfo
(
warnType
,
eventType
)
if
(
alarmInfoOptional
.
isDefined
){
//是否关联到报警信息基础表
if
(
alarmInfoOptional
.
isDefined
){
//是否关联到报警信息基础表
val
useNature
=
vehicleInfoOptional
.
map
(
x
=>
x
.
getUseNature
).
getOrElse
(
"没有关联到车辆性质"
)
val
useNature
=
vehicleInfoOptional
.
map
(
x
=>
x
.
getUseNature
).
getOrElse
(
"没有关联到车辆性质"
)
val
alarmKey
=
IgniteClient
.
getBinaryObject
(
new
PrimaryKey
(
code
.
_1
,
code
.
_2
,
code
.
_3
,
useNature
))
val
alarmKey
=
IgniteClient
.
getBinaryObject
(
new
PrimaryKey
(
code
.
_1
,
code
.
_2
,
code
.
_3
,
useNature
))
//累计行驶报警数
//累计行驶报警数
...
@@ -183,7 +183,7 @@ object SyncIgnite extends Sparking with Logging{
...
@@ -183,7 +183,7 @@ object SyncIgnite extends Sparking with Logging{
case
tableKey
if
tableKey
==
TableKey
(
Some
(
"0x1400"
),
Some
(
"0x1401"
))
=>
case
tableKey
if
tableKey
==
TableKey
(
Some
(
"0x1400"
),
Some
(
"0x1401"
))
=>
val
vehicleNo
=
json
.
getString
(
"vehicleNo"
)
val
vehicleNo
=
json
.
getString
(
"vehicleNo"
)
val
vehicleColor
=
json
.
getString
(
"vehicleColor"
)
val
vehicleColor
=
json
.
getString
(
"vehicleColor"
)
val
superVisionId
=
json
.
getString
(
"super
V
isionId"
)
val
superVisionId
=
json
.
getString
(
"super
v
isionId"
)
val
key
=
IgniteClient
.
getBinaryObject
(
new
PrimaryKey
(
vehicleNo
,
vehicleColor
,
superVisionId
))
val
key
=
IgniteClient
.
getBinaryObject
(
new
PrimaryKey
(
vehicleNo
,
vehicleColor
,
superVisionId
))
val
value
=
new
DailyAlarmDeal
(
true
)
val
value
=
new
DailyAlarmDeal
(
true
)
if
(!
IgniteClient
.
dailyAlarmDealCache
.
withKeepBinary
().
putIfAbsent
(
key
,
value
)){
if
(!
IgniteClient
.
dailyAlarmDealCache
.
withKeepBinary
().
putIfAbsent
(
key
,
value
)){
...
@@ -193,6 +193,8 @@ object SyncIgnite extends Sparking with Logging{
...
@@ -193,6 +193,8 @@ object SyncIgnite extends Sparking with Logging{
}
}
}
catch
{
}
catch
{
case
e
:
Exception
=>
case
e
:
Exception
=>
println
(
x
)
println
(
e
.
getMessage
)
e
.
printStackTrace
()
e
.
printStackTrace
()
}
}
}
}
...
...
src/main/scala/com/hikcreate/data/sync/SyncIgnite1.scala
0 → 100644
View file @
26a16293
This diff is collapsed.
Click to expand it.
src/main/scala/com/hikcreate/data/util/Tools.scala
View file @
26a16293
...
@@ -8,8 +8,11 @@ import com.hikcreate.data.common.Logging
...
@@ -8,8 +8,11 @@ import com.hikcreate.data.common.Logging
import
com.hikcreate.data.constant.Const
import
com.hikcreate.data.constant.Const
import
com.hikcreate.ignite.domain.basic.
{
AlarmTypeInfo
,
EnterpriseInfo
,
VehicleInfo
}
import
com.hikcreate.ignite.domain.basic.
{
AlarmTypeInfo
,
EnterpriseInfo
,
VehicleInfo
}
import
org.apache.ignite.cache.query.SqlQuery
import
org.apache.ignite.cache.query.SqlQuery
import
org.joda.time.
{
DateTime
,
Duration
}
import
scalaj.http.Http
import
scalaj.http.Http
import
scala.collection.mutable.ArrayBuffer
object
Tools
extends
Logging
{
object
Tools
extends
Logging
{
def
addLocation
(
json
:
JSONObject
)
:
Array
[
JSONObject
]
=
{
def
addLocation
(
json
:
JSONObject
)
:
Array
[
JSONObject
]
=
{
...
@@ -25,25 +28,29 @@ object Tools extends Logging{
...
@@ -25,25 +28,29 @@ object Tools extends Logging{
}
}
def
getAddressAndLocationCode
(
lon
:
Double
,
lat
:
Double
)
:
(
String
,
String
)
=
{
def
getAddressAndLocationCode
(
lon
:
Double
,
lat
:
Double
)
:
(
String
,
String
)
=
{
val
json
=
new
JSONObject
()
try
{
val
arr
=
new
JSONArray
()
val
json
=
new
JSONObject
()
val
lonAndLat
=
new
JSONObject
()
val
arr
=
new
JSONArray
()
lonAndLat
.
put
(
"longitude"
,
lon
)
val
lonAndLat
=
new
JSONObject
()
lonAndLat
.
put
(
"latitude"
,
lat
)
lonAndLat
.
put
(
"longitude"
,
lon
)
arr
.
add
(
lonAndLat
)
lonAndLat
.
put
(
"latitude"
,
lat
)
json
.
put
(
"locations"
,
arr
)
arr
.
add
(
lonAndLat
)
val
response
=
Http
(
Const
.
areaCodeAndAddressUrl
).
postData
(
json
.
toJSONString
)
json
.
put
(
"locations"
,
arr
)
.
header
(
"content-type"
,
"application/json"
)
//.charset("ISO-8859-1")
val
starttime
=
DateTime
.
now
()
.
timeout
(
connTimeoutMs
=
8000
,
readTimeoutMs
=
8000
)
val
response
=
Http
(
Const
.
areaCodeAndAddressUrl
).
postData
(
json
.
toJSONString
)
.
asString
.
header
(
"content-type"
,
"application/json"
)
if
(
response
.
code
==
200
){
//.timeout(connTimeoutMs = 1000,readTimeoutMs = 1000)
.
asString
val
endtime
=
DateTime
.
now
()
println
(
"http请求时间:"
+
new
Duration
(
starttime
,
endtime
).
getMillis
)
val
body
=
JSON
.
parseObject
(
response
.
body
)
val
body
=
JSON
.
parseObject
(
response
.
body
)
val
item
=
body
.
getJSONObject
(
"result"
).
getJSONArray
(
"regeoItems"
).
getJSONObject
(
0
)
val
item
=
body
.
getJSONObject
(
"result"
).
getJSONArray
(
"regeoItems"
).
getJSONObject
(
0
)
val
address
=
item
.
getString
(
"formattedAddress"
)
val
address
=
item
.
getString
(
"formattedAddress"
)
val
locationCode
=
item
.
getJSONObject
(
"addressComponent"
).
getString
(
"adcode"
)
val
locationCode
=
item
.
getJSONObject
(
"addressComponent"
).
getString
(
"adcode"
)
(
address
,
locationCode
)
(
address
,
locationCode
)
}
else
{
}
catch
{
throw
new
RuntimeException
(
"http请求城市区编码出错"
)
case
e
:
Exception
=>
throw
new
RuntimeException
(
e
)
}
}
}
}
...
@@ -56,6 +63,39 @@ object Tools extends Logging{
...
@@ -56,6 +63,39 @@ object Tools extends Logging{
(
provinceCode
,
cityCode
,
areaCode
)
(
provinceCode
,
cityCode
,
areaCode
)
}
}
def
getAddressAndLocationCodes
(
buffer
:
List
[(
Double
,
Double
)])
:
IndexedSeq
[(
String
,
String
)]
=
{
val
json
=
new
JSONObject
()
val
arr
=
new
JSONArray
()
buffer
.
foreach
{
x
=>
val
lonAndLat
=
new
JSONObject
()
lonAndLat
.
put
(
"longitude"
,
x
.
_1
)
lonAndLat
.
put
(
"latitude"
,
x
.
_2
)
arr
.
add
(
lonAndLat
)
}
json
.
put
(
"locations"
,
arr
)
val
startTime
=
DateTime
.
now
()
val
response
=
Http
(
Const
.
areaCodeAndAddressUrl
).
postData
(
json
.
toJSONString
).
header
(
"content-type"
,
"application/json"
).
asString
val
endTime
=
DateTime
.
now
()
println
(
"http请求时间:"
+
new
Duration
(
startTime
,
endTime
).
getMillis
)
val
body
=
JSON
.
parseObject
(
response
.
body
)
val
items
=
body
.
getJSONObject
(
"result"
).
getJSONArray
(
"regeoItems"
)
(
0
until
items
.
size
()).
map
{
index
=>
val
item
=
items
.
getJSONObject
(
index
)
val
address
=
item
.
getString
(
"formattedAddress"
)
val
locationCode
=
item
.
getJSONObject
(
"addressComponent"
).
getString
(
"adcode"
)
(
address
,
locationCode
)
}
}
def
getLocationCodes
(
buffer
:
List
[(
Double
,
Double
)])
:
IndexedSeq
[(
String
,
String
,
String
)]
=
{
getAddressAndLocationCodes
(
buffer
).
map
(
x
=>
x
.
_2
).
map
{
locationCode
=>
val
provinceCode
=
locationCode
.
substring
(
0
,
2
)
val
cityCode
=
locationCode
.
substring
(
2
,
4
)
val
areaCode
=
locationCode
.
substring
(
4
,
6
)
(
provinceCode
,
cityCode
,
areaCode
)
}
}
def
getInfoContentJsonobj
(
infoStr
:
String
)
:
JSONObject
=
{
def
getInfoContentJsonobj
(
infoStr
:
String
)
:
JSONObject
=
{
val
jsonStr
=(
"{\""
+
infoStr
.
replace
(
":="
,
"\":\""
).
replace
(
";"
,
"\",\""
)+
"\"}"
).
replace
(
",\"\"}"
,
"}"
)
val
jsonStr
=(
"{\""
+
infoStr
.
replace
(
":="
,
"\":\""
).
replace
(
";"
,
"\",\""
)+
"\"}"
).
replace
(
",\"\"}"
,
"}"
)
val
jSONObject
=
JSON
.
parseObject
(
jsonStr
)
val
jSONObject
=
JSON
.
parseObject
(
jsonStr
)
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment