Skip to content

G
gy_shell
Commit e92d2f9c by 李辅翼
Options

信用概览修改

parent 1c5e829a
Showing with 396 additions and 0 deletions
信用分相关/credit_activate_stat/credit_activate_stat2mysql.sh 0 → 100644
#!/bin/bash
#!/bin/bash
/opt/cloudera/parcels/CDH-5.15.1-1.cdh5.15.1.p0.4/bin/sqoop export --connect jdbc:mysql://172.16.25.43:3306/db_gy_dmsmp --username root --password 'mysql!@Hik' -m 1 \
--table credit_activate_stat --input-fields-terminated-by "\t" --input-lines-terminated-by "\n" --update-key "date_time" --update-mode allowinsert \
--columns "total_driver, add_driver,date_time " \
--export-dir /data/traffic/credit_activate_stat/000*
信用分相关/credit_activate_stat/statistics_everyday_credit_activate_stat.sh 0 → 100644
#!/bin/bash
#!/bin/bash
##每天向date_table插入当天的日期数据
/opt/cloudera/parcels/CDH-5.15.1-1.cdh5.15.1.p0.4/bin/hive -e "insert into date_table SELECT current_date(),pmod(datediff(current_date(),'2018-01-01') + 1,7),substr(current_date(),1,7),substr(current_date(),1,4)"
##交通信用激活人数（两部分，一部分是昨日新申领驾照的，另一部分是注册APP的，需要去重：昨日新申领驾照的+昨日注册app中没有驾照的,app注册用户直接写入hdfs）
##昨日新申领驾照的
/opt/cloudera/parcels/CDH-5.15.1-1.cdh5.15.1.p0.4/bin/hive -e "insert overwrite table activate_credit_day select sfzmhm,cclzrq,to_date(cclzrq),SUBSTR(cclzrq,1,7) from drivinglicense_gy_day_removal"
##昨日app注册的用户（去除申领了驾照的）
/opt/cloudera/parcels/CDH-5.15.1-1.cdh5.15.1.p0.4/bin/hive -e "insert into activate_credit_day select a.* from app_activate_credit a
left outer join drivinglicense_gy_all b on a.id_card = b.sfzmhm and a.day = date_add(current_date,-1) where b.sfzmhm is null"
/opt/cloudera/parcels/CDH-5.15.1-1.cdh5.15.1.p0.4/bin/hive -e "insert into activate_credit_all partition(month) select * from activate_credit_day" -hiveconf hive.exec.dynamic.partition.mode=nonstrict --hiveconf hive.exec.dynamic.partition=true --hiveconf hive.exec.max.dynamic.partitions=36600 --hiveconf hive.exec.max.dynamic.partitions.pernode=36600
##交通信用激活人数统计计全表
/opt/cloudera/parcels/CDH-5.15.1-1.cdh5.15.1.p0.4/bin/hive -e "insert overwrite table credit_activate_stat
select sum(b.num),max(a.num), a.day,current_timestamp from
(select day,count(1) as num from activate_credit_all group by day) a
inner join
(select day,count(day) as num from activate_credit_all group by day) b
on 1=1
where a.day>=b.day
group by a.day
order by a.day desc"
##统计全表日期补全
/opt/cloudera/parcels/CDH-5.15.1-1.cdh5.15.1.p0.4/bin/hive -e "insert overwrite table credit_activate_stat select sum(d.add_driver),max(c.add_driver),c.day,current_timestamp from
(select case when b.date_time is null then 0 else b.add_driver end as add_driver,
a.date_time as day from date_table a left outer join credit_activate_stat b on a.date_time=b.date_time) c
inner join
(select case when b.date_time is null then 0 else b.add_driver end as add_driver,
a.date_time as day from date_table a left outer join credit_activate_stat b on a.date_time=b.date_time) d
on 1=1 where c.day>=d.day group by c.day order by c.day desc"
##全表数据抽出天表数据
/opt/cloudera/parcels/CDH-5.15.1-1.cdh5.15.1.p0.4/bin/hive -e "insert overwrite table credit_activate_stat_day select * from credit_activate_stat where date_time=date_add(current_date,-1)"
信用分相关/credit_change_analysis/credit_change_analysis2mysql.sh 0 → 100644
#!/bin/bash
#!/bin/bash
/opt/cloudera/parcels/CDH-5.15.1-1.cdh5.15.1.p0.4/bin/sqoop export -m 5 --connect jdbc:mysql://172.16.25.43:3306/db_gy_dmsmp --username root --password 'mysql!@Hik' -m 3 --table credit_change_analysis --input-fields-terminated-by "\t" --input-lines-terminated-by "\n" --columns "item_name,item_type,num,date_time,dimension,update_time" --update-key "item_name,date_time" --update-mode allowinsert --export-dir /data/credit/credit_change_analysis_day/
信用分相关/credit_change_analysis/statistics_everyday_credit_change_analysis.sh 0 → 100644
#!/bin/bash
#!/bin/bash
/opt/cloudera/parcels/CDH-5.15.1-1.cdh5.15.1.p0.4/bin/hive -f /var/lib/hadoop-hdfs/work/credit/statistics_everyday_credit_change_analysis.sql
信用分相关/credit_change_analysis/statistics_everyday_credit_change_analysis.sql 0 → 100644
INSERT overwrite table credit_change_analysis_day
INSERT overwrite table credit_change_analysis_day
SELECT act_type2,act_type1,sum(times) total,date_sub(eval_day,1) cal_date,score_type,current_timestamp() from credit_action_times_coe
WHERE eval_day=current_date() GROUP BY act_type2,act_type1,score_type,eval_day ;
信用分相关/credit_level_analysis/credit_level_analysis2mysql.sh 0 → 100644
#!/bin/bash
#!/bin/bash
/opt/cloudera/parcels/CDH-5.15.1-1.cdh5.15.1.p0.4/bin/sqoop export --connect jdbc:mysql://172.16.25.43:3306/db_gy_dmsmp --username root --password 'mysql!@Hik' -m 1 \
--table credit_level_analysis --input-fields-terminated-by "\t" --input-lines-terminated-by "\n" \
--columns "credit_type,credit_name,type_count,date_time" \
--export-dir /data/credit/credit_level_analysis_day/000*
信用分相关/credit_level_analysis/statistics_everymonth_credit_level_analysis.sh 0 → 100644
#!/bin/bash
#!/bin/bash
##--信用分等级分析
/opt/cloudera/parcels/CDH-5.15.1-1.cdh5.15.1.p0.4/bin/hive -e "insert overwrite table credit_level_analysis_day
select
a.big_level ,
case
when a.big_level=1 then '优'
when a.big_level=2 then '良'
when a.big_level=3 then '中'
when a.big_level=4 then '差'
else 'other' end,
a.num ,
trunc(current_date,'MM'),
current_timestamp()
from
(select big_level ,count(1) as num from credit_score_mining_out
where eval_day= trunc(current_date,'MM') group by big_level ) a "
##天表数据导入到全表
/opt/cloudera/parcels/CDH-5.15.1-1.cdh5.15.1.p0.4/bin/hive -e "insert into credit_level_analysis select * from credit_level_analysis_day"
sh /var/lib/hadoop-hdfs/work/blacklist/sqoop2mysql/credit_level_analysis2mysql.sh>>/home/logs/blacklist/credit_level_analysis2mysql_$(date +\%Y\%m\%d).log 2>&1
信用分相关/credit_month_average/credit_month_average2mysql.sh 0 → 100644
#!/bin/bash
#!/bin/bash
/opt/cloudera/parcels/CDH-5.15.1-1.cdh5.15.1.p0.4/bin/sqoop export --connect jdbc:mysql://172.16.25.43:3306/db_gy_dmsmp --username root --password 'mysql!@Hik' -m 1 \
--table credit_month_average --input-fields-terminated-by "\t" --input-lines-terminated-by "\n" \
--columns "date_month,avg_credit" \
--export-dir /data/credit/credit_month_average/000*
信用分相关/credit_month_average/statistics_everymonth_credit_month_average.sh 0 → 100644
#!/bin/bash
#!/bin/bash
##月平均分记录
#SUBSTR(DATE_SUB(current_date,DAYOFMONTH(current_date)),1,7)
/opt/cloudera/parcels/CDH-5.15.1-1.cdh5.15.1.p0.4/bin/hive -e "insert overwrite table credit_month_average_month select
current_date,
avg(b_score),
current_timestamp
from credit_score_mining_out where eval_day= trunc(current_date(),'MM')"
##月表插入到总表
/opt/cloudera/parcels/CDH-5.15.1-1.cdh5.15.1.p0.4/bin/hive -e "insert into credit_month_average select * from credit_month_average_month"
echo "结果导出"
sh /var/lib/hadoop-hdfs/work/blacklist/sqoop2mysql/credit_month_average2mysql.sh>>/home/logs/blacklist/credit_month_average2mysql_$(date +\%Y\%m\%d).log 2>&1
信用分相关/credit_radar/credit_radar2mysql.sh 0 → 100644
#!/bin/bash
#!/bin/bash
##信用雷达
/opt/cloudera/parcels/CDH-5.15.1-1.cdh5.15.1.p0.4/bin/sqoop export --connect jdbc:mysql://172.16.25.43:3306/db_gy_dmsmp --username root --password 'mysql!@Hik' -m 1 \
--table credit_radar --input-fields-terminated-by "\t" --input-lines-terminated-by "\n" --update-key "indicator_type,date_time" --update-mode allowinsert \
--columns "indicator_type,indicator_name,indicator_score,date_time" \
--export-dir /data/credit/credit_radar/000*
信用分相关/credit_radar/statistics_everyday_credit_radar.sh 0 → 100644
#!/bin/bash
#!/bin/bash
##雷达信用
/opt/cloudera/parcels/CDH-5.15.1-1.cdh5.15.1.p0.4/bin/hive -e"insert overwrite table credit_radar select 1,'违法行为',avg(crime_percent_rank),date_add(eval_day,-1),current_date
from credit_score_mining_out where eval_day=current_date group by eval_day"
/opt/cloudera/parcels/CDH-5.15.1-1.cdh5.15.1.p0.4/bin/hive -e "insert into credit_radar select 2,'安全行驶',avg(accident_percent_rank),date_add(eval_day,-1),current_date
from credit_score_mining_out where eval_day=current_date group by eval_day "
/opt/cloudera/parcels/CDH-5.15.1-1.cdh5.15.1.p0.4/bin/hive -e "insert into credit_radar select 3,'基础信息',avg(per_info_rank),date_add(eval_day,-1), current_date
from credit_score_mining_out where eval_day=current_date group by eval_day "
/opt/cloudera/parcels/CDH-5.15.1-1.cdh5.15.1.p0.4/bin/hive -e " insert into credit_radar select 4,'文明活动',avg(good_act_rank),date_add(eval_day,-1),current_date
from credit_score_mining_out where eval_day=current_date group by eval_day"
/opt/cloudera/parcels/CDH-5.15.1-1.cdh5.15.1.p0.4/bin/hive -e "insert into credit_radar select 5,'信用履约',avg(task_percent_rank),date_add(eval_day,-1) ,current_date
from credit_score_mining_out where eval_day=current_date group by eval_day"
/opt/cloudera/parcels/CDH-5.15.1-1.cdh5.15.1.p0.4/bin/hive -e "insert into credit_radar_his partition(day) select * , date_time from credit_radar" -hiveconf hive.exec.dynamic.partition.mode=nonstrict --hiveconf hive.exec.dynamic.partition=true --hiveconf hive.exec.max.dynamic.partitions=36600 --hiveconf hive.exec.max.dynamic.partitions.pernode=36600
信用分相关/credit_total/credit_activate_stat2mysql.sh 0 → 100644
#!/bin/bash
#!/bin/bash
/opt/cloudera/parcels/CDH-5.15.1-1.cdh5.15.1.p0.4/bin/sqoop export --connect jdbc:mysql://172.16.25.43:3306/db_gy_dmsmp --username root --password 'mysql!@Hik' -m 1 \
--table credit_activate_stat --input-fields-terminated-by "\t" --input-lines-terminated-by "\n" --update-key "date_time" --update-mode allowinsert \
--columns "total_driver, add_driver,date_time " \
--export-dir /data/traffic/credit_activate_stat/000*
信用分相关/credit_total/statistics_everyday_credit_total.sh 0 → 100644
#!/bin/bash
#!/bin/bash
##信用数据总计表插入数据
/opt/cloudera/parcels/CDH-5.15.1-1.cdh5.15.1.p0.4/bin/hive -e "insert overwrite table credit_total
select
i.date_time as date_time,
case when i.total_driver_hb is null then 0.00 else i.total_driver_hb end as total_driver_hb,
case when j.total_driver_tb is null then 0.00 else j.total_driver_tb end as total_driver_tb,
i.total_driver as total_driver,
case when i.add_driver is null then 0 else i.add_driver end as add_driver,
case when j.add_driver_tb is null then 0.00 else j.add_driver_tb end as add_driver_tb,
case when i.add_driver_hb is null then 0.00 else i.add_driver_hb end as add_driver_hb,
case when k.add_blacklist is null then 0 else k.add_blacklist end as add_blacklist,
case when k.add_blacklist_hb is null then 0.00 else k.add_blacklist_hb end as add_blacklist_hb,
case when l.add_blacklist_tb is null then 0.00 else l.add_blacklist_tb end as add_blacklist_tb,
current_timestamp
from
(
select
a.date_time as date_time,
a.total_driver as total_driver,
a.add_driver as add_driver,
round((a.total_driver-b.total_driver)/b.total_driver,6)*100 as total_driver_hb,
round((a.add_driver-b.add_driver)/b.add_driver,6)*100 as add_driver_hb
from (select *,row_number() over(order by date_time desc ) xh from credit_activate_stat) a
inner join (select *,row_number() over(order by date_time desc ) xh from credit_activate_stat) b on a.xh=b.xh-1
) i left outer join
(
select
c.date_time as date_time,
round((c.total_driver-d.total_driver)/d.total_driver,6)*100 as total_driver_tb,
round((c.add_driver-d.add_driver)/d.add_driver,6)*100 as add_driver_tb
from credit_activate_stat c inner join credit_activate_stat d on c.date_time=date_add(d.date_time,30)
) j on (i.date_time=j.date_time) left outer join
(
select
e.date_time as date_time,
e.num as add_blacklist,
round((e.num-f.num)/f.num,6)*100 as add_blacklist_hb
from
(select SUBSTR(startTime,1,10) as date_time,count(1) as num from blacklist_total where sfhmd=1 group by SUBSTR(startTime,1,10) ) e
inner join
(select SUBSTR(startTime,1,10) as date_time,count(1) as num from blacklist_total where sfhmd=1 group by SUBSTR(startTime,1,10)) f
on e.date_time = date_add(f.date_time,1)
) k on (j.date_time=k.date_time) left outer join
(
select
g.date_time as date_time,
round((g.num-h.num)/h.num,6)*100 as add_blacklist_tb
from
(select SUBSTR(startTime,1,10) as date_time,count(1) as num from blacklist_total where sfhmd=1 group by SUBSTR(startTime,1,10)) g
inner join
(select SUBSTR(startTime,1,10) as date_time,count(1) as num from blacklist_total where sfhmd=1 group by SUBSTR(startTime,1,10)) h
on g.date_time = date_add(h.date_time,30)
) l on (k.date_time=l.date_time) order by date_time desc"
/opt/cloudera/parcels/CDH-5.15.1-1.cdh5.15.1.p0.4/bin/hive -e "insert overwrite table credit_total_day select * from credit_total where date_time=date_add(current_date,-1)"
信用分相关/建表语句/credit.txt 0 → 100644
-------------------credit_total-------------------------------
-------------------credit_total-------------------------------
create table IF NOT EXISTS app.credit_total(
date_time string,
total_driver_hb decimal(10,2),
total_driver_tb decimal(10,2),
total_driver_driver int ,
add_driver int ,
add_driver_tb decimal(10,2),
add_driver_hb decimal(10,2),
add_blacklist int ,
add_blacklist_hb decimal(10,2),
add_blacklist_tb decimal(10,2),
active_total_user int,
active_total_user_hb decimal(10,2),
active_total_user_tb decimal(10,2),
active_black_user int,
active_black_user_hb decimal(10,2),
active_black_user_tb decimal(10,2),
blacklist int,
update_time string
)
ROW FORMAT DELIMITED
FIELDS TERMINATED BY '\t'
LINES TERMINATED BY '\n'
location '/app/traffic/credit_total';
create table IF NOT EXISTS app.credit_active_total(
date_time string,
active_total_user int,
active_total_user_hb decimal(10,2),
active_total_user_tb decimal(10,2),
active_black_user int,
active_black_user_hb decimal(10,2),
active_black_user_tb decimal(10,2),
blacklist int
)
ROW FORMAT DELIMITED
FIELDS TERMINATED BY '\t'
LINES TERMINATED BY '\n'
location '/app/traffic/credit_active_total';
------------------------------------------------credit_activate_stat--------------------------------------------------------
create table IF NOT EXISTS app.credit_activate_stat (
total_driver int COMMENT'总数',
add_driver int COMMENT'当日新增总数',
date_time string COMMENT'日期',
update_time string COMMENT'更新时间'
)
ROW FORMAT DELIMITED
FIELDS TERMINATED BY '\t'
LINES TERMINATED BY '\n'
location '/app/traffic/credit_activate_stat';
create table IF NOT EXISTS app.credit_activate_stat_his (
total_driver int COMMENT'总数',
add_driver int COMMENT'当日新增总数',
date_time string COMMENT'日期',
update_time string COMMENT'更新时间'
)
ROW FORMAT DELIMITED
FIELDS TERMINATED BY '\t'
LINES TERMINATED BY '\n'
location '/app/traffic/credit_activate_stat_his';
-------------------------------------------active_black------------------------------------------
create table IF NOT EXISTS app.active_black (
idcard string COMMENT'身份证号',
fromtable string COMMENT'来源表',
starttime string COMMENT'开始时间,default.blacklist_total.starttime',
endtime string COMMENT'结束时间',
startday string COMMENT'开始日期',
reasoncode string COMMENT'原因代码',
sfhmd string COMMENT'是否黑名单'
)
ROW FORMAT DELIMITED
FIELDS TERMINATED BY '\t'
LINES TERMINATED BY '\n'
location '/app/traffic/active_black';
create table IF NOT EXISTS app.active_black_hiscount (
date_time string COMMENT'日期',
all_num string COMMENT'累计人数',
add_num string COMMENT'新增人数'
)
ROW FORMAT DELIMITED
FIELDS TERMINATED BY '\t'
LINES TERMINATED BY '\n'
location '/app/traffic/active_black_hiscount';
create table IF NOT EXISTS app.total_black_hiscount (
date_time string COMMENT'日期',
all_num string COMMENT'累计人数',
add_num string COMMENT'新增人数'
)
ROW FORMAT DELIMITED
FIELDS TERMINATED BY '\t'
LINES TERMINATED BY '\n'
location '/app/traffic/total_black_hiscount';
-------------------------------------------------------app.credit_level_analysis_day------------------------------------------------------------
create table IF NOT EXISTS app.credit_level_analysis_day (
credit_type string COMMENT'等级类型',
credit_name string COMMENT'等级名称',
type_count int COMMENT'该级别人数',
date_time string COMMENT'日期',
update_time string COMMENT'更新时间'
)
ROW FORMAT DELIMITED
FIELDS TERMINATED BY '\t'
LINES TERMINATED BY '\n'
location '/app/traffic/credit_level_analysis_day';
create table IF NOT EXISTS app.credit_level_analysis_all (
credit_type string COMMENT'等级类型',
credit_name string COMMENT'等级名称',
type_count int COMMENT'该级别人数',
date_time string COMMENT'日期',
update_time string COMMENT'更新时间',
type string COMMENT'1为激活，0为所有'
)
ROW FORMAT DELIMITED
FIELDS TERMINATED BY '\t'
LINES TERMINATED BY '\n'
location '/app/traffic/credit_level_analysis_all';
--------------------------------------------------------------------app.credit_month_average_month----------------------------------------------------------------------------------
create table IF NOT EXISTS app.credit_month_average_month_all (
date_month string COMMENT'月份',
avg_credit string COMMENT'平均分',
update_time string COMMENT'更新时间',
type string COMMENT'1为激活，0为所有'
)
ROW FORMAT DELIMITED
FIELDS TERMINATED BY '\t'
LINES TERMINATED BY '\n'
location '/app/traffic/credit_month_average_month_all';
---------------------------------------------------------------------------------app.credit_change_analysis_day--------------------------------------------------------------------------------------------------------
create table IF NOT EXISTS app.credit_change_analysis_day (
item_name string comment '行为小类',
item_type string comment '行为大类',
num int comment '次数',
date_time date comment '计算日期',
dimension int comment '计算维度:1加分,0减分',
update_time string comment '更新时间',
type string COMMENT'1为激活，0为所有'
)
ROW FORMAT DELIMITED
FIELDS TERMINATED BY '\t'
LINES TERMINATED BY '\n'
location '/app/traffic/credit_change_analysis_day';
-------------------------------------------------------------------------------------app.credit_radar-----------------------------------------------------------------------------------------------------------------
create table IF NOT EXISTS app.credit_radar (
indicator_type string,
indicator_name string,
indicator_score double,
date_time string,
update_time string,
type string COMMENT'1为激活，0为所有'
)
ROW FORMAT DELIMITED
FIELDS TERMINATED BY '\t'
LINES TERMINATED BY '\n'
location '/app/traffic/credit_radar';
---------------------------------------------------------------------------------------------------app.credit_score_record----------------------------------------------------------------------------------------------------------
create table IF NOT EXISTS app.credit_score_record(
id_card string COMMENT '身份证号',
gmt_evaluate string COMMENT '评估时间',
month_evaluate string COMMENT '评估月份',
credit_hide_score int COMMENT '用户隐藏信用分',
credit_display_score int COMMENT '用户展示分',
credit_big_level_code int COMMENT '大等级',
credit_small_level_code int COMMENT '小等级',
credit_score_rate string COMMENT '展示分排名占比. 2710表示: 27.1%',
increase_score int COMMENT '加分的分数',
deduction_score int COMMENT '扣分的分数',
traffic_compliance double COMMENT '交通守法',
safe_driving double COMMENT '安全行驶',
base_info double COMMENT '基础信息',
civilized_activity double COMMENT '文明活动',
credit_compliance double COMMENT '信用履约',
credit_display_score_rank int COMMENT '展示分排名',
credit_hide_score_rank int COMMENT '隐藏分排名',
credit_hide_score_rate string COMMENT '隐藏分排名占比',
type string COMMENT'1为激活，0为所有'
)
ROW FORMAT DELIMITED
FIELDS TERMINATED BY '\t'
LINES TERMINATED BY '\n'
location '/app/credit/credit_score_record';
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment