Skip to content

G
gy_shell
Commit 7ec1e418 by 李辅翼
Options

信用概览重构

parent d59d089e
Showing with 101 additions and 16 deletions
信用分相关/2mysql/day/overview.txt
#!/bin/bash #!/bin/bash
...@@ -2,14 +2,17 @@ ...@@ -2,14 +2,17 @@
##credit_score_record ##credit_score_record
/opt/cloudera/parcels/CDH-5.15.1-1.cdh5.15.1.p0.4/bin/sqoop export --connect jdbc:mysql://172.16.25.43:3306/db_base_service --username root --password 'mysql!@Hik' -m 1 --table credit_score_record_1023 --input-fields-terminated-by "\t" --input-lines-terminated-by "\n" --update-key "id_card,month_evaluate,data_type" --update-mode allowinsert --input-null-string '\\N' --input-null-non-string '\\N' --columns "id_card, gmt_evaluate, month_evaluate, credit_hide_score, credit_display_score, credit_big_level_code, credit_small_level_code, credit_score_rate, increase_score, deduction_score, traffic_compliance, safe_driving, base_info, civilized_activity, credit_compliance, credit_display_score_rank, credit_hide_score_rank, credit_hide_score_rate,data_type" --export-dir /app/credit/credit_score_record/ /opt/cloudera/parcels/CDH-5.15.1-1.cdh5.15.1.p0.4/bin/sqoop export --connect jdbc:mysql://172.16.25.43:3306/db_base_service --username root --password 'mysql!@Hik' -m 1 --table credit_score_record_1023 --input-fields-terminated-by "\t" --input-lines-terminated-by "\n" --update-key "id_card,month_evaluate,data_type" --update-mode allowinsert --input-null-string '\\N' --input-null-non-string '\\N' --columns "id_card, gmt_evaluate, month_evaluate, credit_hide_score, credit_display_score, credit_big_level_code, credit_small_level_code, credit_score_rate, increase_score, deduction_score, traffic_compliance, safe_driving, base_info, civilized_activity, credit_compliance, credit_display_score_rank, credit_hide_score_rank, credit_hide_score_rate,data_type" --export-dir /app/credit/credit_score_record/
##credit_radar ##credit_radar
/opt/cloudera/parcels/CDH-5.15.1-1.cdh5.15.1.p0.4/bin/sqoop export --connect jdbc:mysql://172.16.25.43:3306/db_gy_dmsmp --username root --password 'mysql!@Hik' -m 1 --table credit_radar_1023 --input-fields-terminated-by "\t" --input-lines-terminated-by "\n" --update-key "date_time,indicator_type,data_type" --update-mode allowinsert --input-null-string '\\N' --input-null-non-string '\\N' --columns "indicator_type, indicator_name, indicator_score, date_time,update_time,data_type" --export-dir /app/traffic/credit_radar/ /opt/cloudera/parcels/CDH-5.15.1-1.cdh5.15.1.p0.4/bin/sqoop export --connect jdbc:mysql://172.16.25.43:3306/db_gy_dmsmp --username root --password 'mysql!@Hik' -m 1 --table credit_radar --input-fields-terminated-by "\t" --input-lines-terminated-by "\n" --update-key "date_time,indicator_type,data_type" --update-mode allowinsert --input-null-string '\\N' --input-null-non-string '\\N' --columns "indicator_type, indicator_name, indicator_score, date_time,update_time,data_type" --export-dir /app/traffic/credit_radar/
##credit_change_analysis ##credit_change_analysis
/opt/cloudera/parcels/CDH-5.15.1-1.cdh5.15.1.p0.4/bin/sqoop export --connect jdbc:mysql://172.16.25.43:3306/db_gy_dmsmp --username root --password 'mysql!@Hik' -m 1 --table credit_change_analysis_1023 --input-fields-terminated-by "\t" --input-lines-terminated-by "\n" --update-key "item_name,date_time,data_type" --update-mode allowinsert --input-null-string '\\N' --input-null-non-string '\\N' --columns "item_name, item_type, num, date_time, dimension, update_time, data_type" --export-dir /app/traffic/credit_change_analysis_day/ /opt/cloudera/parcels/CDH-5.15.1-1.cdh5.15.1.p0.4/bin/sqoop export --connect jdbc:mysql://172.16.25.43:3306/db_gy_dmsmp --username root --password 'mysql!@Hik' -m 1 --table credit_change_analysis --input-fields-terminated-by "\t" --input-lines-terminated-by "\n" --update-key "item_name,date_time,data_type" --update-mode allowinsert --input-null-string '\\N' --input-null-non-string '\\N' --columns "item_name, item_type, num, date_time, dimension, update_time, data_type" --export-dir /app/traffic/credit_change_analysis_day/
##credit_total ##credit_total
/opt/cloudera/parcels/CDH-5.15.1-1.cdh5.15.1.p0.4/bin/sqoop export --connect jdbc:mysql://172.16.25.43:3306/db_gy_dmsmp --username root --password 'mysql!@Hik' -m 1 --table credit_total_1023 --input-fields-terminated-by "\t" --input-lines-terminated-by "\n" --update-key "item_name,date_time,data_type" --update-mode allowinsert --input-null-string '\\N' --input-null-non-string '\\N' --columns "date_time, total_driver_hb, total_driver_tb, total_driver, add_driver, add_driver_tb, add_driver_hb, add_blacklist, add_blacklist_hb,add_blacklist_tb, active_total_user, active_total_user_hb, active_total_user_tb, active_black_user, active_black_user_hb, active_black_user_tb,blacklist, blacklist_hb, blacklist_tb,update_time" --export-dir /app/traffic/credit_total/ /opt/cloudera/parcels/CDH-5.15.1-1.cdh5.15.1.p0.4/bin/sqoop export --connect jdbc:mysql://172.16.25.43:3306/db_gy_dmsmp --username root --password 'mysql!@Hik' -m 1 --table credit_total --input-fields-terminated-by "\t" --input-lines-terminated-by "\n" --update-key "item_name,date_time,data_type" --update-mode allowinsert --input-null-string '\\N' --input-null-non-string '\\N' --columns "date_time, total_driver_hb, total_driver_tb, total_driver, add_driver, add_driver_tb, add_driver_hb, add_blacklist, add_blacklist_hb,add_blacklist_tb, active_total_user, active_total_user_hb, active_total_user_tb, active_black_user, active_black_user_hb, active_black_user_tb,blacklist, blacklist_hb, blacklist_tb,update_time" --export-dir /app/traffic/credit_total/
##credit_activate_stat ##credit_activate_stat
#/opt/cloudera/parcels/CDH-5.15.1-1.cdh5.15.1.p0.4/bin/sqoop export --connect jdbc:mysql://172.16.25.43:3306/db_gy_dmsmp --username root --password 'mysql!@Hik' -m 1 --table credit_activate_stat --input-fields-terminated-by "\t" --input-lines-terminated-by "\n" --update-key "date_time" --update-mode allowinsert --input-null-string '\\N' --input-null-non-string '\\N' --columns "total_driver, add_driver,date_time,update_time" --export-dir /app/traffic/credit_activate_stat/ /opt/cloudera/parcels/CDH-5.15.1-1.cdh5.15.1.p0.4/bin/sqoop export --connect jdbc:mysql://172.16.25.43:3306/db_gy_dmsmp --username root --password 'mysql!@Hik' -m 1 --table credit_activate_stat --input-fields-terminated-by "\t" --input-lines-terminated-by "\n" --update-key "date_time" --update-mode allowinsert --input-null-string '\\N' --input-null-non-string '\\N' --columns "total_driver, add_driver,date_time,update_time" --export-dir /app/traffic/credit_activate_stat_his/
##驾驶员类别
/opt/cloudera/parcels/CDH-5.15.1-1.cdh5.15.1.p0.4/bin/sqoop export -m 1 --connect jdbc:mysql://172.16.25.43:3306/db_gy_dmsmp --username root --password 'mysql!@Hik' --table credit_driver_type_stat --columns "driver_type,driver_type_name,total_driver,date_time" --input-fields-terminated-by '^' --update-key "driver_type,date_time" --update-mode allowinsert --export-dir /data/traffic/driver_type_stat/
##排名
/opt/cloudera/parcels/CDH-5.15.1-1.cdh5.15.1.p0.4/bin/sqoop export -m 1 --connect jdbc:mysql://172.16.25.43:3306/db_gy_dmsmp --username root --password 'mysql!@Hik' --table credit_ranking --columns "ranking,driver_name,id_card,sex,driver_type,credit_big_level_code,credit_small_level_code,credit_score,credit_update,ranking_change,update_time,data_type,month_evaluate" --input-fields-terminated-by '\t' --update-key "id_card,data_type,month_evaluate" --update-mode allowinsert --export-dir /app/credit/credit_ranking/
......
信用分相关/2mysql/month/overview.txt
##credit_month_average ##credit_month_average
##credit_month_average ##credit_month_average
/opt/cloudera/parcels/CDH-5.15.1-1.cdh5.15.1.p0.4/bin/sqoop export --connect jdbc:mysql://172.16.25.43:3306/db_gy_dmsmp --username root --password 'mysql!@Hik' -m 1 --table credit_month_average_1023 --input-fields-terminated-by "\t" --input-lines-terminated-by "\n" --update-key "date_month,data_type" --update-mode allowinsert --input-null-string '\\N' --input-null-non-string '\\N' --columns "date_month, avg_credit, update_time, data_type" --export-dir /app/traffic/credit_month_average_month_all/ /opt/cloudera/parcels/CDH-5.15.1-1.cdh5.15.1.p0.4/bin/sqoop export --connect jdbc:mysql://172.16.25.43:3306/db_gy_dmsmp --username root --password 'mysql!@Hik' -m 1 --table credit_month_average --input-fields-terminated-by "\t" --input-lines-terminated-by "\n" --update-key "date_month,data_type" --update-mode allowinsert --input-null-string '\\N' --input-null-non-string '\\N' --columns "date_month, avg_credit, update_time, data_type" --export-dir /app/traffic/credit_month_average_month_all/
##credit_level_analysis ##credit_level_analysis
/opt/cloudera/parcels/CDH-5.15.1-1.cdh5.15.1.p0.4/bin/sqoop export --connect jdbc:mysql://172.16.25.43:3306/db_gy_dmsmp --username root --password 'mysql!@Hik' -m 1 --table credit_level_analysis_1023 --input-fields-terminated-by "\t" --input-lines-terminated-by "\n" --update-key "credit_type,date_time,data_type" --update-mode allowinsert --input-null-string '\\N' --input-null-non-string '\\N' --columns "credit_type, credit_name, type_count, date_time, update_time, data_type" --export-dir /app/traffic/credit_level_analysis_all/ /opt/cloudera/parcels/CDH-5.15.1-1.cdh5.15.1.p0.4/bin/sqoop export --connect jdbc:mysql://172.16.25.43:3306/db_gy_dmsmp --username root --password 'mysql!@Hik' -m 1 --table credit_level_analysis --input-fields-terminated-by "\t" --input-lines-terminated-by "\n" --update-key "credit_type,date_time,data_type" --update-mode allowinsert --input-null-string '\\N' --input-null-non-string '\\N' --columns "credit_type, credit_name, type_count, date_time, update_time, data_type" --export-dir /app/traffic/credit_level_analysis_all/
信用分相关/schedual.sh 0 → 100644
#!/bin/bash
#!/bin/bash
source /etc/profile
time=$(date "+%Y-%m-%d")
DAY=`date +%d`
#使用python脚本导入源数据
rm -rf /home/sqooptask/lifuyi_py/log/data2hive.log
/usr/local/anaconda3/bin/python3 /home/sqooptask/lifuyi_py/automaticComdata_threading.py
file="/home/sqooptask/lifuyi_py/log/data2hive.log"
#仰正
ssh root@172.16.25.25 "/usr/local/anaconda3/bin/python3 /home/data_mining/guiyang_traffic/code_in_use/credit_per_info_zyz_online.py $time"
#黑名单
sh /var/lib/hadoop-hdfs/work/blacklist/blacklist_everyday_merge_script.sh
#信用分计算
echo '开始计算信用分'
ssh root@172.16.25.25 "/usr/local/anaconda3/bin/python3 /home/data_mining/guiyang_traffic/code_in_use/credit_action_times_yy_online.py $time '2019-09-06'"
ssh root@172.16.25.25 "/usr/local/anaconda3/bin/python3 /home/data_mining/guiyang_traffic/code_in_use/credit_score_out_fsc_online.py $time"
#激活人数统计表 每天调度,新版会弃用
sh /var/lib/hadoop-hdfs/work/credit/statistics_everyday_credit_activate_stat.sh
#sh /var/lib/hadoop-hdfs/work/blacklist/sqoop2mysql/credit_activate_stat2mysql.sh >> /var/lib/hadoop-hdfs/work/blacklist/sqoop2mysql/credit_activate_stat2mysql_$(date +\%Y\%m\%d).log 2>&1 &
#信用数据总计统计表 每天调度，新版需要保留部分
sh /var/lib/hadoop-hdfs/work/credit/statistics_everyday_credit_total.sh
#sh /var/lib/hadoop-hdfs/work/blacklist/sqoop2mysql/credit_total2mysql.sh >> /var/lib/hadoop-hdfs/work/blacklist/sqoop2mysql/credit_total2mysql_$(date +\%Y\%m\%d).log 2>&1 &
#驾驶员信息表 每天调度
sh /var/lib/hadoop-hdfs/work/credit/statistics_everyday_credit_driver_message.sh
sh /var/lib/hadoop-hdfs/work/blacklist/sqoop2mysql/credit_driver_info2mysql.sh >> /var/lib/hadoop-hdfs/work/blacklist/sqoop2mysql/credit_driver_info2mysql_$(date +\%Y\%m\%d).log 2>&1 &
#信用预警表 每天调度
sh /var/lib/hadoop-hdfs/work/credit/statistics_everyday_credit_score_warn.sh
sh /var/lib/hadoop-hdfs/work/blacklist/sqoop2mysql/credit_score_warn2mysql.sh >> /var/lib/hadoop-hdfs/work/blacklist/sqoop2mysql/credit_score_warn2mysql_$(date +\%Y\%m\%d).log 2>&1 &
#信用分变更分析，新版弃用
sh /var/lib/hadoop-hdfs/work/credit/statistics_everyday_credit_change_analysis.sh
#sh /var/lib/hadoop-hdfs/work/blacklist/sqoop2mysql/credit_change_analysis2mysql.sh >> /var/lib/hadoop-hdfs/work/blacklist/sqoop2mysql/credit_change_analysis2mysql_$(date +\%Y\%m\%d).log 2>&1 &
#信用分月变更调度 每天调度，新版保留月调度即可
if [ "$DAY" = 01 ]; then
sh /home/credict/tomysql/credit_score_record_month.sh >> /home/credict/tomysql/credit_score_record_month_$(date +\%Y\%m\%d).log 2>&1 &
fi
sh /home/credict/tomysql/credit_score_record_everyday.sh >> /home/credict/tomysql/credit_score_record_everyday_$(date +\%Y\%m\%d).log 2>&1 &
#sh /var/lib/hadoop-hdfs/work/credit/statistics_everyday_credit_score_record.sh
#sh /var/lib/hadoop-hdfs/work/blacklist/sqoop2mysql/credit_score_record2mysql.sh >> /var/lib/hadoop-hdfs/work/blacklist/sqoop2mysql/credit_score_record2mysql_$(date +\%Y\%m\%d).log 2>&1 &
#信用雷达，新版弃用
sh /var/lib/hadoop-hdfs/work/credit/statistics_everyday_credit_radar.sh
#sh /var/lib/hadoop-hdfs/work/blacklist/sqoop2mysql/credit_radar2mysql.sh >> /var/lib/hadoop-hdfs/work/blacklist/sqoop2mysql/credit_radar2mysql_$(date +\%Y\%m\%d).log 2>&1 &
##变更详情
sh /var/lib/hadoop-hdfs/work/credit/statistics_everyday_credit_score_detail.sh
sh /var/lib/hadoop-hdfs/work/blacklist/sqoop2mysql/credit_score_detail2mysql.sh >> /var/lib/hadoop-hdfs/work/blacklist/sqoop2mysql/credit_score_detail2mysql_$(date +\%Y\%m\%d).log 2>&1 &
##驾驶员类别
##sh /home/sqooptask/driver_type/tomsql.sh
##新版测试sql
/opt/cloudera/parcels/CDH-5.15.1-1.cdh5.15.1.p0.4/bin/hive -f /home/credict/hive_cal/day/overview.sql
sh /home/credict/tomysql/day/credit_day.sh>>/home/credict/tomysql/day/credit_day_$(date +\%Y\%m\%d).log 2>&1 &
if [ "$DAY" = 01 ]; then
sh /home/credict/tomysql/month/credit_month.sh>>/home/credict/tomysql/month/credit_month_$(date +\%Y\%m\%d).log 2>&1 &
fi
信用分相关/建表语句/credit.txt
-------------------credit_total------------------------------- -------------------credit_total-------------------------------
...@@ -178,8 +178,35 @@ ROW FORMAT DELIMITED ...@@ -178,8 +178,35 @@ ROW FORMAT DELIMITED
FIELDS TERMINATED BY '\t' FIELDS TERMINATED BY '\t'
LINES TERMINATED BY '\n' LINES TERMINATED BY '\n'
location '/app/credit/credit_score_record'; location '/app/credit/credit_score_record';
------------------------------------------------------------------------------------------------------app.driver_gy_tongji----------------------------------------------------------
create table IF NOT EXISTS app.driver_gy_tongji (
date_time string COMMENT'日期',
add_user string COMMENT'新增驾驶人',
all_user string COMMENT'累计驾驶人'
)
ROW FORMAT DELIMITED
FIELDS TERMINATED BY '\t'
LINES TERMINATED BY '\n'
location '/app/traffic/driver_gy_tongji';
-------------------------------------------------------------------------------------------------------排名前10--------------------------------------------------------------------------
create table IF NOT EXISTS app.credit_ranking (
ranking int ,
driver_name string,
id_card string,
sex string,
driver_type string,
credit_big_level_code string,
credit_small_level_code string,
credit_score int ,
credit_update int,
ranking_change int,
update_time string,
type string
)
ROW FORMAT DELIMITED
FIELDS TERMINATED BY '\t'
LINES TERMINATED BY '\n'
location '/app/credit/credit_ranking';
......
信用分相关/建表语句/脚本.txt
--credit_activate_stat天表 --credit_activate_stat天表
...@@ -31,7 +31,7 @@ select date_time,sum(add_num) over(partition by sfhmd order by date_time) all_nu ...@@ -31,7 +31,7 @@ select date_time,sum(add_num) over(partition by sfhmd order by date_time) all_nu
insert overwrite table app.total_black_hiscount insert overwrite table app.total_black_hiscount
select date_time,sum(add_num) over(partition by sfhmd order by date_time) all_num,add_num from select date_time,sum(add_num) over(partition by sfhmd order by date_time) all_num,add_num from
(select a.date_time date_time,case when b.startday is null then 0 else b.num end as add_num,'1' sfhmd from default.date_table a left outer join (select a.date_time date_time,case when b.startday is null then 0 else b.num end as add_num,'1' sfhmd from default.date_table a left outer join
(select day startday,count(1) num from (select idcard,min(substr(starttime,0,10)) day from default.blacklist_total group by idcard)tb group by day) (select day startday,count(1) num from (select idcard,min(substr(starttime,0,10)) day from default.blacklist_total WHERE sfhmd='1' group by idcard)tb group by day)
b on a.date_time=b.startday) tab; b on a.date_time=b.startday) tab;
--credit_active_total:激活环比同比表 --credit_active_total:激活环比同比表
insert overwrite table app.credit_active_total insert overwrite table app.credit_active_total
...@@ -84,9 +84,10 @@ app.total_black_hiscount a2 inner join app.total_black_hiscount b2 on a2.date_ti ...@@ -84,9 +84,10 @@ app.total_black_hiscount a2 inner join app.total_black_hiscount b2 on a2.date_ti
)n on (m.date_time=n.date_time) order by date_time desc; )n on (m.date_time=n.date_time) order by date_time desc;
--app.credit_total ：信用数据总计表 --app.credit_total ：信用数据总计表
INSERT OVERWRITE table app.credit_total INSERT OVERWRITE table app.credit_total
SELECT a.date_time,a.total_driver_hb,a.total_driver_tb,a.total_driver_driver,a.add_driver,a.add_driver_tb,a.add_blacklist_hb,a.add_blacklist,a.add_blacklist_hb,a.add_blacklist_tb, SELECT a.date_time,a.total_driver_hb,a.total_driver_tb,tab1.num,a.add_driver,a.add_driver_tb,a.add_blacklist_hb,a.add_blacklist,a.add_blacklist_hb,a.add_blacklist_tb,
b.active_total_user,b.active_total_user_hb,b.active_total_user_tb,b.active_black_user,b.active_black_user_hb,b.active_black_user_tb,b.blacklist,b.blacklist_hb,b.blacklist_tb,current_timestamp() b.active_total_user,b.active_total_user_hb,b.active_total_user_tb,b.active_black_user,b.active_black_user_hb,b.active_black_user_tb,b.blacklist,b.blacklist_hb,b.blacklist_tb,current_timestamp()
from default.credit_total a INNER JOIN app.credit_active_total b on a.date_time=b.date_time; from default.credit_total a INNER JOIN app.credit_active_total b on a.date_time=b.date_time left outer join (SELECT count(DISTINCT sfzmhm) num
from default.drivinglicense_gy_all WHERE fzjg='贵A')tab1 on 1=1;
--app.credit_level_analysis_day:等级分析表-------------------------------------------------------月 --app.credit_level_analysis_day:等级分析表-------------------------------------------------------月
insert overwrite table app.credit_level_analysis_day insert overwrite table app.credit_level_analysis_day
select select
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment