Skip to content

G
gy_shell
Commit 1c5e829a by 李辅翼
Options

图片

parent f63e0ea5
Showing with 33 additions and 0 deletions
信用分相关/score_record/credit_score_record_day.sh 0 → 100644
#!/bin/bash
#!/bin/bash
/opt/cloudera/parcels/CDH-5.15.1-1.cdh5.15.1.p0.4/bin/hive -e "insert overwrite table app.credit_score_record_day SELECT idcard_no,eval_day,substr(eval_day,1,7),b_score,per_info_score+total_no_bad_score+total_task_add_score+total_good_act_score,abs(total_crime_score+total_acci_score+total_task_m_score),crime_percent_rank,accident_percent_rank,per_info_rank,good_act_rank,task_percent_rank,b_rank,cast((tb.num-b_rank)/tb.num as decimal(38,6))*10000 FROM default.credit_score_mining_out a INNER JOIN (SELECT count(1) num from default.credit_score_mining_out WHERE eval_day=current_date() ) tb on 1=1 where a.eval_day=current_date()"
/opt/cloudera/parcels/CDH-5.15.1-1.cdh5.15.1.p0.4/bin/sqoop export --connect jdbc:mysql://172.16.25.43:3306/db_base_service --username root --password 'mysql!@Hik' -m 1 --table credit_score_record --input-fields-terminated-by "\t" --input-lines-terminated-by "\n" --input-null-string '\\N' --input-null-non-string '\\N' --columns "id_card,gmt_evaluate,month_evaluate,credit_hide_score,increase_score,deduction_score,traffic_compliance,safe_driving,base_info,civilized_activity,credit_compliance,credit_hide_score_rank,credit_hide_score_rate" --update-key "id_card,month_evaluate" --update-mode allowinsert --export-dir /app/credit/credit_score_record_day
/opt/cloudera/parcels/CDH-5.15.1-1.cdh5.15.1.p0.4/bin/hive -e "insert overwrite table app.credit_score_record_day_new SELECT idcard_no,substr(eval_day,1,7),a_score,big_level,small_level,cast((tb.num-a_rank)/tb.num as decimal(38,6))*10000,a_rank FROM (select idcard_no,eval_day,a_score,big_level,small_level,a_rank from (select b1.*,row_number() over(partition by idcard_no order by eval_day asc) evalrank from (select a0.* from (select idcard_no,eval_day,a_score,big_level,small_level,a_rank from default.credit_score_mining_out where eval_day>trunc(current_date(),'MM')) a0 left outer join (select idcard_no from default.credit_score_mining_out where eval_day=trunc(current_date(),'MM')) a1 on a0.idcard_no=a1.idcard_no where a1.idcard_no is null)b1 ) b2 where evalrank=1) a INNER JOIN (SELECT count(1) num from default.credit_score_mining_out WHERE eval_day=current_date()) tb on 1=1 "
/opt/cloudera/parcels/CDH-5.15.1-1.cdh5.15.1.p0.4/bin/sqoop export --connect jdbc:mysql://172.16.25.43:3306/db_base_service --username root --password 'mysql!@Hik' -m 1 --table credit_score_record --input-fields-terminated-by "\t" --input-lines-terminated-by "\n" --input-null-string '\\N' --input-null-non-string '\\N' --columns "id_card,month_evaluate,credit_display_score,credit_big_level_code,credit_small_level_code,credit_score_rate,credit_display_score_rank" --update-key "id_card,month_evaluate" --update-mode allowinsert --export-dir /app/credit/credit_score_record_day_new
信用分相关/score_record/credit_score_record_everyday.sh 0 → 100644
#!/bin/bash
#!/bin/bash
#每天得不包含展示分部分
/opt/cloudera/parcels/CDH-5.15.1-1.cdh5.15.1.p0.4/bin/hive -e "insert overwrite table app.credit_score_record_day SELECT idcard_no,eval_day,substr(eval_day,1,7),b_score,per_info_score+total_no_bad_score+total_task_add_score+total_good_act_score,abs(total_crime_score+total_acci_score+total_task_m_score),crime_percent_rank,accident_percent_rank,per_info_rank,good_act_rank,task_percent_rank,b_rank,cast((tb.num-b_rank)/tb.num as decimal(38,6))*10000 FROM (select * from default.credit_score_mining_out where eval_day=current_date()) a INNER JOIN (SELECT count(1) num from default.credit_score_mining_out WHERE eval_day=current_date() ) tb on 1=1"
#每月展示分
#/opt/cloudera/parcels/CDH-5.15.1-1.cdh5.15.1.p0.4/bin/hive -e "insert overwrite table app.credit_score_record_month SELECT idcard_no,substr(eval_day,1,7),a_score,big_level,small_level,cast((tb.num-a_rank)/tb.num as decimal(38,6))*10000,a_rank FROM default.credit_score_mining_out a INNER JOIN (SELECT count(1) num from default.credit_score_mining_out WHERE eval_day=trunc(current_date(),'MM')) tb on 1=1 where a.eval_day=trunc(current_date(),'MM')"
#每天中新算出来得人员得展示分
/opt/cloudera/parcels/CDH-5.15.1-1.cdh5.15.1.p0.4/bin/hive -e "insert overwrite table app.credit_score_record_day_new SELECT idcard_no,substr(eval_day,1,7),a_score,big_level,small_level,cast((tb.num-a_rank)/tb.num as decimal(38,6))*10000,a_rank FROM (select idcard_no,eval_day,a_score,big_level,small_level,a_rank from (select b1.*,row_number() over(partition by idcard_no order by eval_day asc) evalrank from (select a0.* from (select idcard_no,eval_day,a_score,big_level,small_level,a_rank from default.credit_score_mining_out where eval_day>trunc(current_date(),'MM')) a0 left outer join (select idcard_no from default.credit_score_mining_out where eval_day=trunc(current_date(),'MM')) a1 on a0.idcard_no=a1.idcard_no where a1.idcard_no is null)b1 ) b2 where evalrank=1) a INNER JOIN (SELECT count(1) num from default.credit_score_mining_out WHERE eval_day=current_date()) tb on 1=1 "
#合并
/opt/cloudera/parcels/CDH-5.15.1-1.cdh5.15.1.p0.4/bin/hive -e "INSERT OVERWRITE TABLE app.credit_score_record_everyday SELECT a.id_card,a.gmt_evaluate,a.month_evaluate,a.credit_hide_score,b.credit_display_score,b.credit_big_level_code,b.credit_small_level_code,b.credit_score_rate,a.increase_score,a.deduction_score,a.traffic_compliance,a.safe_driving,a.base_info,a.civilized_activity,a.credit_compliance,b.credit_display_score_rank,a.credit_hide_score_rank,a.credit_hide_score_rate from app.credit_score_record_day a INNER JOIN app.credit_score_record_month b on a.id_card=b.id_card"
/opt/cloudera/parcels/CDH-5.15.1-1.cdh5.15.1.p0.4/bin/hive -e "insert into app.credit_score_record_everyday SELECT a.id_card,a.gmt_evaluate,a.month_evaluate,a.credit_hide_score,b.credit_display_score,b.credit_big_level_code,b.credit_small_level_code,b.credit_score_rate,a.increase_score,a.deduction_score,a.traffic_compliance,a.safe_driving,a.base_info,a.civilized_activity,a.credit_compliance,b.credit_display_score_rank,a.credit_hide_score_rank,a.credit_hide_score_rate from app.credit_score_record_day a INNER JOIN app.credit_score_record_day_new b on a.id_card=b.id_card"
/opt/cloudera/parcels/CDH-5.15.1-1.cdh5.15.1.p0.4/bin/sqoop export --connect jdbc:mysql://172.16.25.43:3306/db_base_service --username root --password 'mysql!@Hik' -m 1 --table credit_score_record --input-fields-terminated-by "\t" --input-lines-terminated-by "\n" --input-null-string '\\N' --input-null-non-string '\\N' --columns "id_card,gmt_evaluate,month_evaluate,credit_hide_score,credit_display_score,credit_big_level_code,credit_small_level_code,credit_score_rate,increase_score,deduction_score,traffic_compliance,safe_driving,base_info,civilized_activity,credit_compliance,credit_display_score_rank,credit_hide_score_rank,credit_hide_score_rate" --update-key "id_card,month_evaluate" --update-mode allowinsert --export-dir /app/credit/credit_score_record_everyday/
信用分相关/score_record/credit_score_record_month.sh 0 → 100644
#!/bin/bash
#!/bin/bash
/opt/cloudera/parcels/CDH-5.15.1-1.cdh5.15.1.p0.4/bin/hive -e "insert overwrite table app.credit_score_record_month SELECT idcard_no,substr(eval_day,1,7),a_score,big_level,small_level,cast((tb.num-a_rank)/tb.num as decimal(38,6))*10000,a_rank FROM default.credit_score_mining_out a INNER JOIN (SELECT count(1) num from default.credit_score_mining_out WHERE eval_day=trunc(current_date(),'MM')) tb on 1=1 where a.eval_day=trunc(current_date(),'MM')"
违法图片/vio_pic/vio_picurl_day.sh 0 → 100644
#!/bin/bash
#!/bin/bash
/opt/cloudera/parcels/CDH-5.15.1-1.cdh5.15.1.p0.4/bin/sqoop import --connect jdbc:oracle:thin:@193.1.100.1:1521:hiatmpdb --username haikang --password haikang -m 3 --target-dir /kakou/ods/VIO_VIOLATION_PIC_DAY_ODS --delete-target-dir --fields-terminated-by "^" --lines-terminated-by "\n" --hive-drop-import-delims --split-by CADDRESSCODE --where "1=1" --query """SELECT result.*, to_char(result.export_time,'yyyy-MM-dd') as day ,to_char(result.export_time,'yyyy-MM') as month FROM wf_haikang result where result.dillegaldate between sysdate-20 and sysdate and result.export_time between to_date(to_char(sysdate-1,'yyyy-mm-dd'), 'yyyy-mm-dd') and to_date(to_char(sysdate,'yyyy-mm-dd'), 'yyyy-mm-dd') and \$CONDITIONS"""
/opt/cloudera/parcels/CDH-5.15.1-1.cdh5.15.1.p0.4/bin/hive -e "LOAD DATA INPATH '/kakou/ods/VIO_VIOLATION_PIC_DAY_ODS' INTO TABLE kakou.vio_violation_pic_day_ods"
/opt/cloudera/parcels/CDH-5.15.1-1.cdh5.15.1.p0.4/bin/hive -e "INSERT into ods.hbase_viopic SELECT concat_ws('#',ccarnumber,clicensetype,substr(dillegaldate,0,19),coffense),ccarnumber,clicensetype,caddresscode,cillegaladdress,coffense,dillegaldate,cpic1path,cpic2path,cpic3path,export_time,month,day from kakou.vio_violation_pic_day_ods"
/opt/cloudera/parcels/CDH-5.15.1-1.cdh5.15.1.p0.4/bin/hive -e "insert into kakou.vio_violation_pic_his_ods PARTITION(day) SELECT CCARNUMBER,CLICENSETYPE,CADDRESSCODE,CILLEGALADDRESS,COFFENSE,DILLEGALDATE,CPIC1PATH,CPIC2PATH,CPIC3PATH,export_time,month,day from kakou.vio_violation_pic_day_ods DISTRIBUTE BY rand()" --hiveconf hive.exec.dynamic.partition.mode=nonstrict --hiveconf hive.exec.dynamic.partition=true --hiveconf hive.exec.max.dynamic.partitions=36600 --hiveconf hive.exec.max.dynamic.partitions.pernode=36600
违法图片/vio_pic/vio_picurl_hand.sh 0 → 100644
#!/bin/bash
#!/bin/bash
/opt/cloudera/parcels/CDH-5.15.1-1.cdh5.15.1.p0.4/bin/sqoop import --connect jdbc:oracle:thin:@193.1.100.1:1521:hiatmpdb --username haikang --password haikang -m 1 --target-dir /kakou/ods/VIO_VIOLATION_PIC_DAY_ODS --delete-target-dir --fields-terminated-by "^" --lines-terminated-by "\n" --hive-drop-import-delims --split-by CADDRESSCODE --where "1=1" --query """SELECT result.*, to_char(result.export_time,'yyyy-MM-dd') as day ,to_char(result.export_time,'yyyy-MM') as month FROM wf_haikang result where result.dillegaldate between sysdate-970 and sysdate-850 and result.export_time between to_date(to_char(sysdate-950,'yyyy-mm-dd'), 'yyyy-mm-dd') and to_date(to_char(sysdate-850,'yyyy-mm-dd'), 'yyyy-mm-dd') and \$CONDITIONS"""
/opt/cloudera/parcels/CDH-5.15.1-1.cdh5.15.1.p0.4/bin/hive -e "LOAD DATA INPATH '/kakou/ods/VIO_VIOLATION_PIC_DAY_ODS' INTO TABLE kakou.vio_violation_pic_day_ods"
/opt/cloudera/parcels/CDH-5.15.1-1.cdh5.15.1.p0.4/bin/hive -e "INSERT into ods.hbase_viopic SELECT concat_ws('#',ccarnumber,clicensetype,substr(dillegaldate,0,19),coffense),ccarnumber,clicensetype,caddresscode,cillegaladdress,coffense,dillegaldate,cpic1path,cpic2path,cpic3path,export_time,month,day from kakou.vio_violation_pic_day_ods"
/opt/cloudera/parcels/CDH-5.15.1-1.cdh5.15.1.p0.4/bin/hive -e "insert into kakou.vio_violation_pic_his_ods PARTITION(day) SELECT CCARNUMBER,CLICENSETYPE,CADDRESSCODE,CILLEGALADDRESS,COFFENSE,DILLEGALDATE,CPIC1PATH,CPIC2PATH,CPIC3PATH,export_time,month,day from kakou.vio_violation_pic_day_ods DISTRIBUTE BY rand()" --hiveconf hive.exec.dynamic.partition.mode=nonstrict --hiveconf hive.exec.dynamic.partition=true --hiveconf hive.exec.max.dynamic.partitions=36600 --hiveconf hive.exec.max.dynamic.partitions.pernode=36600
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment