Commit 1c5e829a by 李辅翼

图片

parent f63e0ea5
#!/bin/bash
#!/bin/bash
/opt/cloudera/parcels/CDH-5.15.1-1.cdh5.15.1.p0.4/bin/hive -e "insert overwrite table app.credit_score_record_day SELECT idcard_no,eval_day,substr(eval_day,1,7),b_score,per_info_score+total_no_bad_score+total_task_add_score+total_good_act_score,abs(total_crime_score+total_acci_score+total_task_m_score),crime_percent_rank,accident_percent_rank,per_info_rank,good_act_rank,task_percent_rank,b_rank,cast((tb.num-b_rank)/tb.num as decimal(38,6))*10000 FROM default.credit_score_mining_out a INNER JOIN (SELECT count(1) num from default.credit_score_mining_out WHERE eval_day=current_date() ) tb on 1=1 where a.eval_day=current_date()"
/opt/cloudera/parcels/CDH-5.15.1-1.cdh5.15.1.p0.4/bin/sqoop export --connect jdbc:mysql://172.16.25.43:3306/db_base_service --username root --password 'mysql!@Hik' -m 1 --table credit_score_record --input-fields-terminated-by "\t" --input-lines-terminated-by "\n" --input-null-string '\\N' --input-null-non-string '\\N' --columns "id_card,gmt_evaluate,month_evaluate,credit_hide_score,increase_score,deduction_score,traffic_compliance,safe_driving,base_info,civilized_activity,credit_compliance,credit_hide_score_rank,credit_hide_score_rate" --update-key "id_card,month_evaluate" --update-mode allowinsert --export-dir /app/credit/credit_score_record_day
/opt/cloudera/parcels/CDH-5.15.1-1.cdh5.15.1.p0.4/bin/hive -e "insert overwrite table app.credit_score_record_day_new SELECT idcard_no,substr(eval_day,1,7),a_score,big_level,small_level,cast((tb.num-a_rank)/tb.num as decimal(38,6))*10000,a_rank FROM (select idcard_no,eval_day,a_score,big_level,small_level,a_rank from (select b1.*,row_number() over(partition by idcard_no order by eval_day asc) evalrank from (select a0.* from (select idcard_no,eval_day,a_score,big_level,small_level,a_rank from default.credit_score_mining_out where eval_day>trunc(current_date(),'MM')) a0 left outer join (select idcard_no from default.credit_score_mining_out where eval_day=trunc(current_date(),'MM')) a1 on a0.idcard_no=a1.idcard_no where a1.idcard_no is null)b1 ) b2 where evalrank=1) a INNER JOIN (SELECT count(1) num from default.credit_score_mining_out WHERE eval_day=current_date()) tb on 1=1 "
/opt/cloudera/parcels/CDH-5.15.1-1.cdh5.15.1.p0.4/bin/sqoop export --connect jdbc:mysql://172.16.25.43:3306/db_base_service --username root --password 'mysql!@Hik' -m 1 --table credit_score_record --input-fields-terminated-by "\t" --input-lines-terminated-by "\n" --input-null-string '\\N' --input-null-non-string '\\N' --columns "id_card,month_evaluate,credit_display_score,credit_big_level_code,credit_small_level_code,credit_score_rate,credit_display_score_rank" --update-key "id_card,month_evaluate" --update-mode allowinsert --export-dir /app/credit/credit_score_record_day_new
#!/bin/bash
#!/bin/bash
#每天得不包含展示分部分
/opt/cloudera/parcels/CDH-5.15.1-1.cdh5.15.1.p0.4/bin/hive -e "insert overwrite table app.credit_score_record_day SELECT idcard_no,eval_day,substr(eval_day,1,7),b_score,per_info_score+total_no_bad_score+total_task_add_score+total_good_act_score,abs(total_crime_score+total_acci_score+total_task_m_score),crime_percent_rank,accident_percent_rank,per_info_rank,good_act_rank,task_percent_rank,b_rank,cast((tb.num-b_rank)/tb.num as decimal(38,6))*10000 FROM (select * from default.credit_score_mining_out where eval_day=current_date()) a INNER JOIN (SELECT count(1) num from default.credit_score_mining_out WHERE eval_day=current_date() ) tb on 1=1"
#每月展示分
#/opt/cloudera/parcels/CDH-5.15.1-1.cdh5.15.1.p0.4/bin/hive -e "insert overwrite table app.credit_score_record_month SELECT idcard_no,substr(eval_day,1,7),a_score,big_level,small_level,cast((tb.num-a_rank)/tb.num as decimal(38,6))*10000,a_rank FROM default.credit_score_mining_out a INNER JOIN (SELECT count(1) num from default.credit_score_mining_out WHERE eval_day=trunc(current_date(),'MM')) tb on 1=1 where a.eval_day=trunc(current_date(),'MM')"
#每天中新算出来得人员得展示分
/opt/cloudera/parcels/CDH-5.15.1-1.cdh5.15.1.p0.4/bin/hive -e "insert overwrite table app.credit_score_record_day_new SELECT idcard_no,substr(eval_day,1,7),a_score,big_level,small_level,cast((tb.num-a_rank)/tb.num as decimal(38,6))*10000,a_rank FROM (select idcard_no,eval_day,a_score,big_level,small_level,a_rank from (select b1.*,row_number() over(partition by idcard_no order by eval_day asc) evalrank from (select a0.* from (select idcard_no,eval_day,a_score,big_level,small_level,a_rank from default.credit_score_mining_out where eval_day>trunc(current_date(),'MM')) a0 left outer join (select idcard_no from default.credit_score_mining_out where eval_day=trunc(current_date(),'MM')) a1 on a0.idcard_no=a1.idcard_no where a1.idcard_no is null)b1 ) b2 where evalrank=1) a INNER JOIN (SELECT count(1) num from default.credit_score_mining_out WHERE eval_day=current_date()) tb on 1=1 "
#合并
/opt/cloudera/parcels/CDH-5.15.1-1.cdh5.15.1.p0.4/bin/hive -e "INSERT OVERWRITE TABLE app.credit_score_record_everyday SELECT a.id_card,a.gmt_evaluate,a.month_evaluate,a.credit_hide_score,b.credit_display_score,b.credit_big_level_code,b.credit_small_level_code,b.credit_score_rate,a.increase_score,a.deduction_score,a.traffic_compliance,a.safe_driving,a.base_info,a.civilized_activity,a.credit_compliance,b.credit_display_score_rank,a.credit_hide_score_rank,a.credit_hide_score_rate from app.credit_score_record_day a INNER JOIN app.credit_score_record_month b on a.id_card=b.id_card"
/opt/cloudera/parcels/CDH-5.15.1-1.cdh5.15.1.p0.4/bin/hive -e "insert into app.credit_score_record_everyday SELECT a.id_card,a.gmt_evaluate,a.month_evaluate,a.credit_hide_score,b.credit_display_score,b.credit_big_level_code,b.credit_small_level_code,b.credit_score_rate,a.increase_score,a.deduction_score,a.traffic_compliance,a.safe_driving,a.base_info,a.civilized_activity,a.credit_compliance,b.credit_display_score_rank,a.credit_hide_score_rank,a.credit_hide_score_rate from app.credit_score_record_day a INNER JOIN app.credit_score_record_day_new b on a.id_card=b.id_card"
/opt/cloudera/parcels/CDH-5.15.1-1.cdh5.15.1.p0.4/bin/sqoop export --connect jdbc:mysql://172.16.25.43:3306/db_base_service --username root --password 'mysql!@Hik' -m 1 --table credit_score_record --input-fields-terminated-by "\t" --input-lines-terminated-by "\n" --input-null-string '\\N' --input-null-non-string '\\N' --columns "id_card,gmt_evaluate,month_evaluate,credit_hide_score,credit_display_score,credit_big_level_code,credit_small_level_code,credit_score_rate,increase_score,deduction_score,traffic_compliance,safe_driving,base_info,civilized_activity,credit_compliance,credit_display_score_rank,credit_hide_score_rank,credit_hide_score_rate" --update-key "id_card,month_evaluate" --update-mode allowinsert --export-dir /app/credit/credit_score_record_everyday/
#!/bin/bash
#!/bin/bash
/opt/cloudera/parcels/CDH-5.15.1-1.cdh5.15.1.p0.4/bin/hive -e "insert overwrite table app.credit_score_record_month SELECT idcard_no,substr(eval_day,1,7),a_score,big_level,small_level,cast((tb.num-a_rank)/tb.num as decimal(38,6))*10000,a_rank FROM default.credit_score_mining_out a INNER JOIN (SELECT count(1) num from default.credit_score_mining_out WHERE eval_day=trunc(current_date(),'MM')) tb on 1=1 where a.eval_day=trunc(current_date(),'MM')"
// ORM class for table 'null'
// ORM class for table 'null'
// WARNING: This class is AUTO-GENERATED. Modify at your own risk.
//
// Debug information:
// Generated date: Wed May 15 05:57:43 GMT 2019
// For connector: org.apache.sqoop.manager.OracleManager
import org.apache.hadoop.io.BytesWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.mapred.lib.db.DBWritable;
import com.cloudera.sqoop.lib.JdbcWritableBridge;
import com.cloudera.sqoop.lib.DelimiterSet;
import com.cloudera.sqoop.lib.FieldFormatter;
import com.cloudera.sqoop.lib.RecordParser;
import com.cloudera.sqoop.lib.BooleanParser;
import com.cloudera.sqoop.lib.BlobRef;
import com.cloudera.sqoop.lib.ClobRef;
import com.cloudera.sqoop.lib.LargeObjectLoader;
import com.cloudera.sqoop.lib.SqoopRecord;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.nio.CharBuffer;
import java.sql.Date;
import java.sql.Time;
import java.sql.Timestamp;
import java.util.Arrays;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.HashMap;
public class QueryResult extends SqoopRecord implements DBWritable, Writable {
private final int PROTOCOL_VERSION = 3;
public int getClassFormatVersion() { return PROTOCOL_VERSION; }
public static interface FieldSetterCommand { void setField(Object value); } protected ResultSet __cur_result_set;
private Map<String, FieldSetterCommand> setters = new HashMap<String, FieldSetterCommand>();
private void init0() {
setters.put("CCARNUMBER", new FieldSetterCommand() {
@Override
public void setField(Object value) {
CCARNUMBER = (String)value;
}
});
setters.put("CLICENSETYPE", new FieldSetterCommand() {
@Override
public void setField(Object value) {
CLICENSETYPE = (String)value;
}
});
setters.put("CADDRESSCODE", new FieldSetterCommand() {
@Override
public void setField(Object value) {
CADDRESSCODE = (String)value;
}
});
setters.put("CILLEGALADDRESS", new FieldSetterCommand() {
@Override
public void setField(Object value) {
CILLEGALADDRESS = (String)value;
}
});
setters.put("COFFENSE", new FieldSetterCommand() {
@Override
public void setField(Object value) {
COFFENSE = (String)value;
}
});
setters.put("DILLEGALDATE", new FieldSetterCommand() {
@Override
public void setField(Object value) {
DILLEGALDATE = (java.sql.Timestamp)value;
}
});
setters.put("CPIC1PATH", new FieldSetterCommand() {
@Override
public void setField(Object value) {
CPIC1PATH = (String)value;
}
});
setters.put("CPIC2PATH", new FieldSetterCommand() {
@Override
public void setField(Object value) {
CPIC2PATH = (String)value;
}
});
setters.put("CPIC3PATH", new FieldSetterCommand() {
@Override
public void setField(Object value) {
CPIC3PATH = (String)value;
}
});
setters.put("DAY", new FieldSetterCommand() {
@Override
public void setField(Object value) {
DAY = (String)value;
}
});
setters.put("MONTH", new FieldSetterCommand() {
@Override
public void setField(Object value) {
MONTH = (String)value;
}
});
}
public QueryResult() {
init0();
}
private String CCARNUMBER;
public String get_CCARNUMBER() {
return CCARNUMBER;
}
public void set_CCARNUMBER(String CCARNUMBER) {
this.CCARNUMBER = CCARNUMBER;
}
public QueryResult with_CCARNUMBER(String CCARNUMBER) {
this.CCARNUMBER = CCARNUMBER;
return this;
}
private String CLICENSETYPE;
public String get_CLICENSETYPE() {
return CLICENSETYPE;
}
public void set_CLICENSETYPE(String CLICENSETYPE) {
this.CLICENSETYPE = CLICENSETYPE;
}
public QueryResult with_CLICENSETYPE(String CLICENSETYPE) {
this.CLICENSETYPE = CLICENSETYPE;
return this;
}
private String CADDRESSCODE;
public String get_CADDRESSCODE() {
return CADDRESSCODE;
}
public void set_CADDRESSCODE(String CADDRESSCODE) {
this.CADDRESSCODE = CADDRESSCODE;
}
public QueryResult with_CADDRESSCODE(String CADDRESSCODE) {
this.CADDRESSCODE = CADDRESSCODE;
return this;
}
private String CILLEGALADDRESS;
public String get_CILLEGALADDRESS() {
return CILLEGALADDRESS;
}
public void set_CILLEGALADDRESS(String CILLEGALADDRESS) {
this.CILLEGALADDRESS = CILLEGALADDRESS;
}
public QueryResult with_CILLEGALADDRESS(String CILLEGALADDRESS) {
this.CILLEGALADDRESS = CILLEGALADDRESS;
return this;
}
private String COFFENSE;
public String get_COFFENSE() {
return COFFENSE;
}
public void set_COFFENSE(String COFFENSE) {
this.COFFENSE = COFFENSE;
}
public QueryResult with_COFFENSE(String COFFENSE) {
this.COFFENSE = COFFENSE;
return this;
}
private java.sql.Timestamp DILLEGALDATE;
public java.sql.Timestamp get_DILLEGALDATE() {
return DILLEGALDATE;
}
public void set_DILLEGALDATE(java.sql.Timestamp DILLEGALDATE) {
this.DILLEGALDATE = DILLEGALDATE;
}
public QueryResult with_DILLEGALDATE(java.sql.Timestamp DILLEGALDATE) {
this.DILLEGALDATE = DILLEGALDATE;
return this;
}
private String CPIC1PATH;
public String get_CPIC1PATH() {
return CPIC1PATH;
}
public void set_CPIC1PATH(String CPIC1PATH) {
this.CPIC1PATH = CPIC1PATH;
}
public QueryResult with_CPIC1PATH(String CPIC1PATH) {
this.CPIC1PATH = CPIC1PATH;
return this;
}
private String CPIC2PATH;
public String get_CPIC2PATH() {
return CPIC2PATH;
}
public void set_CPIC2PATH(String CPIC2PATH) {
this.CPIC2PATH = CPIC2PATH;
}
public QueryResult with_CPIC2PATH(String CPIC2PATH) {
this.CPIC2PATH = CPIC2PATH;
return this;
}
private String CPIC3PATH;
public String get_CPIC3PATH() {
return CPIC3PATH;
}
public void set_CPIC3PATH(String CPIC3PATH) {
this.CPIC3PATH = CPIC3PATH;
}
public QueryResult with_CPIC3PATH(String CPIC3PATH) {
this.CPIC3PATH = CPIC3PATH;
return this;
}
private String DAY;
public String get_DAY() {
return DAY;
}
public void set_DAY(String DAY) {
this.DAY = DAY;
}
public QueryResult with_DAY(String DAY) {
this.DAY = DAY;
return this;
}
private String MONTH;
public String get_MONTH() {
return MONTH;
}
public void set_MONTH(String MONTH) {
this.MONTH = MONTH;
}
public QueryResult with_MONTH(String MONTH) {
this.MONTH = MONTH;
return this;
}
public boolean equals(Object o) {
if (this == o) {
return true;
}
if (!(o instanceof QueryResult)) {
return false;
}
QueryResult that = (QueryResult) o;
boolean equal = true;
equal = equal && (this.CCARNUMBER == null ? that.CCARNUMBER == null : this.CCARNUMBER.equals(that.CCARNUMBER));
equal = equal && (this.CLICENSETYPE == null ? that.CLICENSETYPE == null : this.CLICENSETYPE.equals(that.CLICENSETYPE));
equal = equal && (this.CADDRESSCODE == null ? that.CADDRESSCODE == null : this.CADDRESSCODE.equals(that.CADDRESSCODE));
equal = equal && (this.CILLEGALADDRESS == null ? that.CILLEGALADDRESS == null : this.CILLEGALADDRESS.equals(that.CILLEGALADDRESS));
equal = equal && (this.COFFENSE == null ? that.COFFENSE == null : this.COFFENSE.equals(that.COFFENSE));
equal = equal && (this.DILLEGALDATE == null ? that.DILLEGALDATE == null : this.DILLEGALDATE.equals(that.DILLEGALDATE));
equal = equal && (this.CPIC1PATH == null ? that.CPIC1PATH == null : this.CPIC1PATH.equals(that.CPIC1PATH));
equal = equal && (this.CPIC2PATH == null ? that.CPIC2PATH == null : this.CPIC2PATH.equals(that.CPIC2PATH));
equal = equal && (this.CPIC3PATH == null ? that.CPIC3PATH == null : this.CPIC3PATH.equals(that.CPIC3PATH));
equal = equal && (this.DAY == null ? that.DAY == null : this.DAY.equals(that.DAY));
equal = equal && (this.MONTH == null ? that.MONTH == null : this.MONTH.equals(that.MONTH));
return equal;
}
public boolean equals0(Object o) {
if (this == o) {
return true;
}
if (!(o instanceof QueryResult)) {
return false;
}
QueryResult that = (QueryResult) o;
boolean equal = true;
equal = equal && (this.CCARNUMBER == null ? that.CCARNUMBER == null : this.CCARNUMBER.equals(that.CCARNUMBER));
equal = equal && (this.CLICENSETYPE == null ? that.CLICENSETYPE == null : this.CLICENSETYPE.equals(that.CLICENSETYPE));
equal = equal && (this.CADDRESSCODE == null ? that.CADDRESSCODE == null : this.CADDRESSCODE.equals(that.CADDRESSCODE));
equal = equal && (this.CILLEGALADDRESS == null ? that.CILLEGALADDRESS == null : this.CILLEGALADDRESS.equals(that.CILLEGALADDRESS));
equal = equal && (this.COFFENSE == null ? that.COFFENSE == null : this.COFFENSE.equals(that.COFFENSE));
equal = equal && (this.DILLEGALDATE == null ? that.DILLEGALDATE == null : this.DILLEGALDATE.equals(that.DILLEGALDATE));
equal = equal && (this.CPIC1PATH == null ? that.CPIC1PATH == null : this.CPIC1PATH.equals(that.CPIC1PATH));
equal = equal && (this.CPIC2PATH == null ? that.CPIC2PATH == null : this.CPIC2PATH.equals(that.CPIC2PATH));
equal = equal && (this.CPIC3PATH == null ? that.CPIC3PATH == null : this.CPIC3PATH.equals(that.CPIC3PATH));
equal = equal && (this.DAY == null ? that.DAY == null : this.DAY.equals(that.DAY));
equal = equal && (this.MONTH == null ? that.MONTH == null : this.MONTH.equals(that.MONTH));
return equal;
}
public void readFields(ResultSet __dbResults) throws SQLException {
this.__cur_result_set = __dbResults;
this.CCARNUMBER = JdbcWritableBridge.readString(1, __dbResults);
this.CLICENSETYPE = JdbcWritableBridge.readString(2, __dbResults);
this.CADDRESSCODE = JdbcWritableBridge.readString(3, __dbResults);
this.CILLEGALADDRESS = JdbcWritableBridge.readString(4, __dbResults);
this.COFFENSE = JdbcWritableBridge.readString(5, __dbResults);
this.DILLEGALDATE = JdbcWritableBridge.readTimestamp(6, __dbResults);
this.CPIC1PATH = JdbcWritableBridge.readString(7, __dbResults);
this.CPIC2PATH = JdbcWritableBridge.readString(8, __dbResults);
this.CPIC3PATH = JdbcWritableBridge.readString(9, __dbResults);
this.DAY = JdbcWritableBridge.readString(10, __dbResults);
this.MONTH = JdbcWritableBridge.readString(11, __dbResults);
}
public void readFields0(ResultSet __dbResults) throws SQLException {
this.CCARNUMBER = JdbcWritableBridge.readString(1, __dbResults);
this.CLICENSETYPE = JdbcWritableBridge.readString(2, __dbResults);
this.CADDRESSCODE = JdbcWritableBridge.readString(3, __dbResults);
this.CILLEGALADDRESS = JdbcWritableBridge.readString(4, __dbResults);
this.COFFENSE = JdbcWritableBridge.readString(5, __dbResults);
this.DILLEGALDATE = JdbcWritableBridge.readTimestamp(6, __dbResults);
this.CPIC1PATH = JdbcWritableBridge.readString(7, __dbResults);
this.CPIC2PATH = JdbcWritableBridge.readString(8, __dbResults);
this.CPIC3PATH = JdbcWritableBridge.readString(9, __dbResults);
this.DAY = JdbcWritableBridge.readString(10, __dbResults);
this.MONTH = JdbcWritableBridge.readString(11, __dbResults);
}
public void loadLargeObjects(LargeObjectLoader __loader)
throws SQLException, IOException, InterruptedException {
}
public void loadLargeObjects0(LargeObjectLoader __loader)
throws SQLException, IOException, InterruptedException {
}
public void write(PreparedStatement __dbStmt) throws SQLException {
write(__dbStmt, 0);
}
public int write(PreparedStatement __dbStmt, int __off) throws SQLException {
JdbcWritableBridge.writeString(CCARNUMBER, 1 + __off, -9, __dbStmt);
JdbcWritableBridge.writeString(CLICENSETYPE, 2 + __off, 12, __dbStmt);
JdbcWritableBridge.writeString(CADDRESSCODE, 3 + __off, 12, __dbStmt);
JdbcWritableBridge.writeString(CILLEGALADDRESS, 4 + __off, 12, __dbStmt);
JdbcWritableBridge.writeString(COFFENSE, 5 + __off, 12, __dbStmt);
JdbcWritableBridge.writeTimestamp(DILLEGALDATE, 6 + __off, 93, __dbStmt);
JdbcWritableBridge.writeString(CPIC1PATH, 7 + __off, 12, __dbStmt);
JdbcWritableBridge.writeString(CPIC2PATH, 8 + __off, 12, __dbStmt);
JdbcWritableBridge.writeString(CPIC3PATH, 9 + __off, 12, __dbStmt);
JdbcWritableBridge.writeString(DAY, 10 + __off, 12, __dbStmt);
JdbcWritableBridge.writeString(MONTH, 11 + __off, 12, __dbStmt);
return 11;
}
public void write0(PreparedStatement __dbStmt, int __off) throws SQLException {
JdbcWritableBridge.writeString(CCARNUMBER, 1 + __off, -9, __dbStmt);
JdbcWritableBridge.writeString(CLICENSETYPE, 2 + __off, 12, __dbStmt);
JdbcWritableBridge.writeString(CADDRESSCODE, 3 + __off, 12, __dbStmt);
JdbcWritableBridge.writeString(CILLEGALADDRESS, 4 + __off, 12, __dbStmt);
JdbcWritableBridge.writeString(COFFENSE, 5 + __off, 12, __dbStmt);
JdbcWritableBridge.writeTimestamp(DILLEGALDATE, 6 + __off, 93, __dbStmt);
JdbcWritableBridge.writeString(CPIC1PATH, 7 + __off, 12, __dbStmt);
JdbcWritableBridge.writeString(CPIC2PATH, 8 + __off, 12, __dbStmt);
JdbcWritableBridge.writeString(CPIC3PATH, 9 + __off, 12, __dbStmt);
JdbcWritableBridge.writeString(DAY, 10 + __off, 12, __dbStmt);
JdbcWritableBridge.writeString(MONTH, 11 + __off, 12, __dbStmt);
}
public void readFields(DataInput __dataIn) throws IOException {
this.readFields0(__dataIn); }
public void readFields0(DataInput __dataIn) throws IOException {
if (__dataIn.readBoolean()) {
this.CCARNUMBER = null;
} else {
this.CCARNUMBER = Text.readString(__dataIn);
}
if (__dataIn.readBoolean()) {
this.CLICENSETYPE = null;
} else {
this.CLICENSETYPE = Text.readString(__dataIn);
}
if (__dataIn.readBoolean()) {
this.CADDRESSCODE = null;
} else {
this.CADDRESSCODE = Text.readString(__dataIn);
}
if (__dataIn.readBoolean()) {
this.CILLEGALADDRESS = null;
} else {
this.CILLEGALADDRESS = Text.readString(__dataIn);
}
if (__dataIn.readBoolean()) {
this.COFFENSE = null;
} else {
this.COFFENSE = Text.readString(__dataIn);
}
if (__dataIn.readBoolean()) {
this.DILLEGALDATE = null;
} else {
this.DILLEGALDATE = new Timestamp(__dataIn.readLong());
this.DILLEGALDATE.setNanos(__dataIn.readInt());
}
if (__dataIn.readBoolean()) {
this.CPIC1PATH = null;
} else {
this.CPIC1PATH = Text.readString(__dataIn);
}
if (__dataIn.readBoolean()) {
this.CPIC2PATH = null;
} else {
this.CPIC2PATH = Text.readString(__dataIn);
}
if (__dataIn.readBoolean()) {
this.CPIC3PATH = null;
} else {
this.CPIC3PATH = Text.readString(__dataIn);
}
if (__dataIn.readBoolean()) {
this.DAY = null;
} else {
this.DAY = Text.readString(__dataIn);
}
if (__dataIn.readBoolean()) {
this.MONTH = null;
} else {
this.MONTH = Text.readString(__dataIn);
}
}
public void write(DataOutput __dataOut) throws IOException {
if (null == this.CCARNUMBER) {
__dataOut.writeBoolean(true);
} else {
__dataOut.writeBoolean(false);
Text.writeString(__dataOut, CCARNUMBER);
}
if (null == this.CLICENSETYPE) {
__dataOut.writeBoolean(true);
} else {
__dataOut.writeBoolean(false);
Text.writeString(__dataOut, CLICENSETYPE);
}
if (null == this.CADDRESSCODE) {
__dataOut.writeBoolean(true);
} else {
__dataOut.writeBoolean(false);
Text.writeString(__dataOut, CADDRESSCODE);
}
if (null == this.CILLEGALADDRESS) {
__dataOut.writeBoolean(true);
} else {
__dataOut.writeBoolean(false);
Text.writeString(__dataOut, CILLEGALADDRESS);
}
if (null == this.COFFENSE) {
__dataOut.writeBoolean(true);
} else {
__dataOut.writeBoolean(false);
Text.writeString(__dataOut, COFFENSE);
}
if (null == this.DILLEGALDATE) {
__dataOut.writeBoolean(true);
} else {
__dataOut.writeBoolean(false);
__dataOut.writeLong(this.DILLEGALDATE.getTime());
__dataOut.writeInt(this.DILLEGALDATE.getNanos());
}
if (null == this.CPIC1PATH) {
__dataOut.writeBoolean(true);
} else {
__dataOut.writeBoolean(false);
Text.writeString(__dataOut, CPIC1PATH);
}
if (null == this.CPIC2PATH) {
__dataOut.writeBoolean(true);
} else {
__dataOut.writeBoolean(false);
Text.writeString(__dataOut, CPIC2PATH);
}
if (null == this.CPIC3PATH) {
__dataOut.writeBoolean(true);
} else {
__dataOut.writeBoolean(false);
Text.writeString(__dataOut, CPIC3PATH);
}
if (null == this.DAY) {
__dataOut.writeBoolean(true);
} else {
__dataOut.writeBoolean(false);
Text.writeString(__dataOut, DAY);
}
if (null == this.MONTH) {
__dataOut.writeBoolean(true);
} else {
__dataOut.writeBoolean(false);
Text.writeString(__dataOut, MONTH);
}
}
public void write0(DataOutput __dataOut) throws IOException {
if (null == this.CCARNUMBER) {
__dataOut.writeBoolean(true);
} else {
__dataOut.writeBoolean(false);
Text.writeString(__dataOut, CCARNUMBER);
}
if (null == this.CLICENSETYPE) {
__dataOut.writeBoolean(true);
} else {
__dataOut.writeBoolean(false);
Text.writeString(__dataOut, CLICENSETYPE);
}
if (null == this.CADDRESSCODE) {
__dataOut.writeBoolean(true);
} else {
__dataOut.writeBoolean(false);
Text.writeString(__dataOut, CADDRESSCODE);
}
if (null == this.CILLEGALADDRESS) {
__dataOut.writeBoolean(true);
} else {
__dataOut.writeBoolean(false);
Text.writeString(__dataOut, CILLEGALADDRESS);
}
if (null == this.COFFENSE) {
__dataOut.writeBoolean(true);
} else {
__dataOut.writeBoolean(false);
Text.writeString(__dataOut, COFFENSE);
}
if (null == this.DILLEGALDATE) {
__dataOut.writeBoolean(true);
} else {
__dataOut.writeBoolean(false);
__dataOut.writeLong(this.DILLEGALDATE.getTime());
__dataOut.writeInt(this.DILLEGALDATE.getNanos());
}
if (null == this.CPIC1PATH) {
__dataOut.writeBoolean(true);
} else {
__dataOut.writeBoolean(false);
Text.writeString(__dataOut, CPIC1PATH);
}
if (null == this.CPIC2PATH) {
__dataOut.writeBoolean(true);
} else {
__dataOut.writeBoolean(false);
Text.writeString(__dataOut, CPIC2PATH);
}
if (null == this.CPIC3PATH) {
__dataOut.writeBoolean(true);
} else {
__dataOut.writeBoolean(false);
Text.writeString(__dataOut, CPIC3PATH);
}
if (null == this.DAY) {
__dataOut.writeBoolean(true);
} else {
__dataOut.writeBoolean(false);
Text.writeString(__dataOut, DAY);
}
if (null == this.MONTH) {
__dataOut.writeBoolean(true);
} else {
__dataOut.writeBoolean(false);
Text.writeString(__dataOut, MONTH);
}
}
private static final DelimiterSet __outputDelimiters = new DelimiterSet((char) 94, (char) 10, (char) 0, (char) 0, false);
public String toString() {
return toString(__outputDelimiters, true);
}
public String toString(DelimiterSet delimiters) {
return toString(delimiters, true);
}
public String toString(boolean useRecordDelim) {
return toString(__outputDelimiters, useRecordDelim);
}
public String toString(DelimiterSet delimiters, boolean useRecordDelim) {
StringBuilder __sb = new StringBuilder();
char fieldDelim = delimiters.getFieldsTerminatedBy();
// special case for strings hive, droppingdelimiters \n,\r,\01 from strings
__sb.append(FieldFormatter.hiveStringDropDelims(CCARNUMBER==null?"null":CCARNUMBER, delimiters));
__sb.append(fieldDelim);
// special case for strings hive, droppingdelimiters \n,\r,\01 from strings
__sb.append(FieldFormatter.hiveStringDropDelims(CLICENSETYPE==null?"null":CLICENSETYPE, delimiters));
__sb.append(fieldDelim);
// special case for strings hive, droppingdelimiters \n,\r,\01 from strings
__sb.append(FieldFormatter.hiveStringDropDelims(CADDRESSCODE==null?"null":CADDRESSCODE, delimiters));
__sb.append(fieldDelim);
// special case for strings hive, droppingdelimiters \n,\r,\01 from strings
__sb.append(FieldFormatter.hiveStringDropDelims(CILLEGALADDRESS==null?"null":CILLEGALADDRESS, delimiters));
__sb.append(fieldDelim);
// special case for strings hive, droppingdelimiters \n,\r,\01 from strings
__sb.append(FieldFormatter.hiveStringDropDelims(COFFENSE==null?"null":COFFENSE, delimiters));
__sb.append(fieldDelim);
__sb.append(FieldFormatter.escapeAndEnclose(DILLEGALDATE==null?"null":"" + DILLEGALDATE, delimiters));
__sb.append(fieldDelim);
// special case for strings hive, droppingdelimiters \n,\r,\01 from strings
__sb.append(FieldFormatter.hiveStringDropDelims(CPIC1PATH==null?"null":CPIC1PATH, delimiters));
__sb.append(fieldDelim);
// special case for strings hive, droppingdelimiters \n,\r,\01 from strings
__sb.append(FieldFormatter.hiveStringDropDelims(CPIC2PATH==null?"null":CPIC2PATH, delimiters));
__sb.append(fieldDelim);
// special case for strings hive, droppingdelimiters \n,\r,\01 from strings
__sb.append(FieldFormatter.hiveStringDropDelims(CPIC3PATH==null?"null":CPIC3PATH, delimiters));
__sb.append(fieldDelim);
// special case for strings hive, droppingdelimiters \n,\r,\01 from strings
__sb.append(FieldFormatter.hiveStringDropDelims(DAY==null?"null":DAY, delimiters));
__sb.append(fieldDelim);
// special case for strings hive, droppingdelimiters \n,\r,\01 from strings
__sb.append(FieldFormatter.hiveStringDropDelims(MONTH==null?"null":MONTH, delimiters));
if (useRecordDelim) {
__sb.append(delimiters.getLinesTerminatedBy());
}
return __sb.toString();
}
public void toString0(DelimiterSet delimiters, StringBuilder __sb, char fieldDelim) {
// special case for strings hive, droppingdelimiters \n,\r,\01 from strings
__sb.append(FieldFormatter.hiveStringDropDelims(CCARNUMBER==null?"null":CCARNUMBER, delimiters));
__sb.append(fieldDelim);
// special case for strings hive, droppingdelimiters \n,\r,\01 from strings
__sb.append(FieldFormatter.hiveStringDropDelims(CLICENSETYPE==null?"null":CLICENSETYPE, delimiters));
__sb.append(fieldDelim);
// special case for strings hive, droppingdelimiters \n,\r,\01 from strings
__sb.append(FieldFormatter.hiveStringDropDelims(CADDRESSCODE==null?"null":CADDRESSCODE, delimiters));
__sb.append(fieldDelim);
// special case for strings hive, droppingdelimiters \n,\r,\01 from strings
__sb.append(FieldFormatter.hiveStringDropDelims(CILLEGALADDRESS==null?"null":CILLEGALADDRESS, delimiters));
__sb.append(fieldDelim);
// special case for strings hive, droppingdelimiters \n,\r,\01 from strings
__sb.append(FieldFormatter.hiveStringDropDelims(COFFENSE==null?"null":COFFENSE, delimiters));
__sb.append(fieldDelim);
__sb.append(FieldFormatter.escapeAndEnclose(DILLEGALDATE==null?"null":"" + DILLEGALDATE, delimiters));
__sb.append(fieldDelim);
// special case for strings hive, droppingdelimiters \n,\r,\01 from strings
__sb.append(FieldFormatter.hiveStringDropDelims(CPIC1PATH==null?"null":CPIC1PATH, delimiters));
__sb.append(fieldDelim);
// special case for strings hive, droppingdelimiters \n,\r,\01 from strings
__sb.append(FieldFormatter.hiveStringDropDelims(CPIC2PATH==null?"null":CPIC2PATH, delimiters));
__sb.append(fieldDelim);
// special case for strings hive, droppingdelimiters \n,\r,\01 from strings
__sb.append(FieldFormatter.hiveStringDropDelims(CPIC3PATH==null?"null":CPIC3PATH, delimiters));
__sb.append(fieldDelim);
// special case for strings hive, droppingdelimiters \n,\r,\01 from strings
__sb.append(FieldFormatter.hiveStringDropDelims(DAY==null?"null":DAY, delimiters));
__sb.append(fieldDelim);
// special case for strings hive, droppingdelimiters \n,\r,\01 from strings
__sb.append(FieldFormatter.hiveStringDropDelims(MONTH==null?"null":MONTH, delimiters));
}
private static final DelimiterSet __inputDelimiters = new DelimiterSet((char) 94, (char) 10, (char) 0, (char) 0, false);
private RecordParser __parser;
public void parse(Text __record) throws RecordParser.ParseError {
if (null == this.__parser) {
this.__parser = new RecordParser(__inputDelimiters);
}
List<String> __fields = this.__parser.parseRecord(__record);
__loadFromFields(__fields);
}
public void parse(CharSequence __record) throws RecordParser.ParseError {
if (null == this.__parser) {
this.__parser = new RecordParser(__inputDelimiters);
}
List<String> __fields = this.__parser.parseRecord(__record);
__loadFromFields(__fields);
}
public void parse(byte [] __record) throws RecordParser.ParseError {
if (null == this.__parser) {
this.__parser = new RecordParser(__inputDelimiters);
}
List<String> __fields = this.__parser.parseRecord(__record);
__loadFromFields(__fields);
}
public void parse(char [] __record) throws RecordParser.ParseError {
if (null == this.__parser) {
this.__parser = new RecordParser(__inputDelimiters);
}
List<String> __fields = this.__parser.parseRecord(__record);
__loadFromFields(__fields);
}
public void parse(ByteBuffer __record) throws RecordParser.ParseError {
if (null == this.__parser) {
this.__parser = new RecordParser(__inputDelimiters);
}
List<String> __fields = this.__parser.parseRecord(__record);
__loadFromFields(__fields);
}
public void parse(CharBuffer __record) throws RecordParser.ParseError {
if (null == this.__parser) {
this.__parser = new RecordParser(__inputDelimiters);
}
List<String> __fields = this.__parser.parseRecord(__record);
__loadFromFields(__fields);
}
private void __loadFromFields(List<String> fields) {
Iterator<String> __it = fields.listIterator();
String __cur_str = null;
try {
__cur_str = __it.next();
if (__cur_str.equals("null")) { this.CCARNUMBER = null; } else {
this.CCARNUMBER = __cur_str;
}
__cur_str = __it.next();
if (__cur_str.equals("null")) { this.CLICENSETYPE = null; } else {
this.CLICENSETYPE = __cur_str;
}
__cur_str = __it.next();
if (__cur_str.equals("null")) { this.CADDRESSCODE = null; } else {
this.CADDRESSCODE = __cur_str;
}
__cur_str = __it.next();
if (__cur_str.equals("null")) { this.CILLEGALADDRESS = null; } else {
this.CILLEGALADDRESS = __cur_str;
}
__cur_str = __it.next();
if (__cur_str.equals("null")) { this.COFFENSE = null; } else {
this.COFFENSE = __cur_str;
}
__cur_str = __it.next();
if (__cur_str.equals("null") || __cur_str.length() == 0) { this.DILLEGALDATE = null; } else {
this.DILLEGALDATE = java.sql.Timestamp.valueOf(__cur_str);
}
__cur_str = __it.next();
if (__cur_str.equals("null")) { this.CPIC1PATH = null; } else {
this.CPIC1PATH = __cur_str;
}
__cur_str = __it.next();
if (__cur_str.equals("null")) { this.CPIC2PATH = null; } else {
this.CPIC2PATH = __cur_str;
}
__cur_str = __it.next();
if (__cur_str.equals("null")) { this.CPIC3PATH = null; } else {
this.CPIC3PATH = __cur_str;
}
__cur_str = __it.next();
if (__cur_str.equals("null")) { this.DAY = null; } else {
this.DAY = __cur_str;
}
__cur_str = __it.next();
if (__cur_str.equals("null")) { this.MONTH = null; } else {
this.MONTH = __cur_str;
}
} catch (RuntimeException e) { throw new RuntimeException("Can't parse input data: '" + __cur_str + "'", e); } }
private void __loadFromFields0(Iterator<String> __it) {
String __cur_str = null;
try {
__cur_str = __it.next();
if (__cur_str.equals("null")) { this.CCARNUMBER = null; } else {
this.CCARNUMBER = __cur_str;
}
__cur_str = __it.next();
if (__cur_str.equals("null")) { this.CLICENSETYPE = null; } else {
this.CLICENSETYPE = __cur_str;
}
__cur_str = __it.next();
if (__cur_str.equals("null")) { this.CADDRESSCODE = null; } else {
this.CADDRESSCODE = __cur_str;
}
__cur_str = __it.next();
if (__cur_str.equals("null")) { this.CILLEGALADDRESS = null; } else {
this.CILLEGALADDRESS = __cur_str;
}
__cur_str = __it.next();
if (__cur_str.equals("null")) { this.COFFENSE = null; } else {
this.COFFENSE = __cur_str;
}
__cur_str = __it.next();
if (__cur_str.equals("null") || __cur_str.length() == 0) { this.DILLEGALDATE = null; } else {
this.DILLEGALDATE = java.sql.Timestamp.valueOf(__cur_str);
}
__cur_str = __it.next();
if (__cur_str.equals("null")) { this.CPIC1PATH = null; } else {
this.CPIC1PATH = __cur_str;
}
__cur_str = __it.next();
if (__cur_str.equals("null")) { this.CPIC2PATH = null; } else {
this.CPIC2PATH = __cur_str;
}
__cur_str = __it.next();
if (__cur_str.equals("null")) { this.CPIC3PATH = null; } else {
this.CPIC3PATH = __cur_str;
}
__cur_str = __it.next();
if (__cur_str.equals("null")) { this.DAY = null; } else {
this.DAY = __cur_str;
}
__cur_str = __it.next();
if (__cur_str.equals("null")) { this.MONTH = null; } else {
this.MONTH = __cur_str;
}
} catch (RuntimeException e) { throw new RuntimeException("Can't parse input data: '" + __cur_str + "'", e); } }
public Object clone() throws CloneNotSupportedException {
QueryResult o = (QueryResult) super.clone();
o.DILLEGALDATE = (o.DILLEGALDATE != null) ? (java.sql.Timestamp) o.DILLEGALDATE.clone() : null;
return o;
}
public void clone0(QueryResult o) throws CloneNotSupportedException {
o.DILLEGALDATE = (o.DILLEGALDATE != null) ? (java.sql.Timestamp) o.DILLEGALDATE.clone() : null;
}
public Map<String, Object> getFieldMap() {
Map<String, Object> __sqoop$field_map = new HashMap<String, Object>();
__sqoop$field_map.put("CCARNUMBER", this.CCARNUMBER);
__sqoop$field_map.put("CLICENSETYPE", this.CLICENSETYPE);
__sqoop$field_map.put("CADDRESSCODE", this.CADDRESSCODE);
__sqoop$field_map.put("CILLEGALADDRESS", this.CILLEGALADDRESS);
__sqoop$field_map.put("COFFENSE", this.COFFENSE);
__sqoop$field_map.put("DILLEGALDATE", this.DILLEGALDATE);
__sqoop$field_map.put("CPIC1PATH", this.CPIC1PATH);
__sqoop$field_map.put("CPIC2PATH", this.CPIC2PATH);
__sqoop$field_map.put("CPIC3PATH", this.CPIC3PATH);
__sqoop$field_map.put("DAY", this.DAY);
__sqoop$field_map.put("MONTH", this.MONTH);
return __sqoop$field_map;
}
public void getFieldMap0(Map<String, Object> __sqoop$field_map) {
__sqoop$field_map.put("CCARNUMBER", this.CCARNUMBER);
__sqoop$field_map.put("CLICENSETYPE", this.CLICENSETYPE);
__sqoop$field_map.put("CADDRESSCODE", this.CADDRESSCODE);
__sqoop$field_map.put("CILLEGALADDRESS", this.CILLEGALADDRESS);
__sqoop$field_map.put("COFFENSE", this.COFFENSE);
__sqoop$field_map.put("DILLEGALDATE", this.DILLEGALDATE);
__sqoop$field_map.put("CPIC1PATH", this.CPIC1PATH);
__sqoop$field_map.put("CPIC2PATH", this.CPIC2PATH);
__sqoop$field_map.put("CPIC3PATH", this.CPIC3PATH);
__sqoop$field_map.put("DAY", this.DAY);
__sqoop$field_map.put("MONTH", this.MONTH);
}
public void setField(String __fieldName, Object __fieldVal) {
if (!setters.containsKey(__fieldName)) {
throw new RuntimeException("No such field:"+__fieldName);
}
setters.get(__fieldName).setField(__fieldVal);
}
}
#!/bin/bash
#!/bin/bash
/opt/cloudera/parcels/CDH-5.15.1-1.cdh5.15.1.p0.4/bin/sqoop import --connect jdbc:oracle:thin:@193.1.100.1:1521:hiatmpdb --username haikang --password haikang -m 3 --target-dir /kakou/ods/VIO_VIOLATION_PIC_DAY_ODS --delete-target-dir --fields-terminated-by "^" --lines-terminated-by "\n" --hive-drop-import-delims --split-by CADDRESSCODE --where "1=1" --query """SELECT result.*, to_char(result.export_time,'yyyy-MM-dd') as day ,to_char(result.export_time,'yyyy-MM') as month FROM wf_haikang result where result.dillegaldate between sysdate-20 and sysdate and result.export_time between to_date(to_char(sysdate-1,'yyyy-mm-dd'), 'yyyy-mm-dd') and to_date(to_char(sysdate,'yyyy-mm-dd'), 'yyyy-mm-dd') and \$CONDITIONS"""
/opt/cloudera/parcels/CDH-5.15.1-1.cdh5.15.1.p0.4/bin/hive -e "LOAD DATA INPATH '/kakou/ods/VIO_VIOLATION_PIC_DAY_ODS' INTO TABLE kakou.vio_violation_pic_day_ods"
/opt/cloudera/parcels/CDH-5.15.1-1.cdh5.15.1.p0.4/bin/hive -e "INSERT into ods.hbase_viopic SELECT concat_ws('#',ccarnumber,clicensetype,substr(dillegaldate,0,19),coffense),ccarnumber,clicensetype,caddresscode,cillegaladdress,coffense,dillegaldate,cpic1path,cpic2path,cpic3path,export_time,month,day from kakou.vio_violation_pic_day_ods"
/opt/cloudera/parcels/CDH-5.15.1-1.cdh5.15.1.p0.4/bin/hive -e "insert into kakou.vio_violation_pic_his_ods PARTITION(day) SELECT CCARNUMBER,CLICENSETYPE,CADDRESSCODE,CILLEGALADDRESS,COFFENSE,DILLEGALDATE,CPIC1PATH,CPIC2PATH,CPIC3PATH,export_time,month,day from kakou.vio_violation_pic_day_ods DISTRIBUTE BY rand()" --hiveconf hive.exec.dynamic.partition.mode=nonstrict --hiveconf hive.exec.dynamic.partition=true --hiveconf hive.exec.max.dynamic.partitions=36600 --hiveconf hive.exec.max.dynamic.partitions.pernode=36600
#!/bin/bash
#!/bin/bash
/opt/cloudera/parcels/CDH-5.15.1-1.cdh5.15.1.p0.4/bin/sqoop import --connect jdbc:oracle:thin:@193.1.100.1:1521:hiatmpdb --username haikang --password haikang -m 1 --target-dir /kakou/ods/VIO_VIOLATION_PIC_DAY_ODS --delete-target-dir --fields-terminated-by "^" --lines-terminated-by "\n" --hive-drop-import-delims --split-by CADDRESSCODE --where "1=1" --query """SELECT result.*, to_char(result.export_time,'yyyy-MM-dd') as day ,to_char(result.export_time,'yyyy-MM') as month FROM wf_haikang result where result.dillegaldate between sysdate-970 and sysdate-850 and result.export_time between to_date(to_char(sysdate-950,'yyyy-mm-dd'), 'yyyy-mm-dd') and to_date(to_char(sysdate-850,'yyyy-mm-dd'), 'yyyy-mm-dd') and \$CONDITIONS"""
/opt/cloudera/parcels/CDH-5.15.1-1.cdh5.15.1.p0.4/bin/hive -e "LOAD DATA INPATH '/kakou/ods/VIO_VIOLATION_PIC_DAY_ODS' INTO TABLE kakou.vio_violation_pic_day_ods"
/opt/cloudera/parcels/CDH-5.15.1-1.cdh5.15.1.p0.4/bin/hive -e "INSERT into ods.hbase_viopic SELECT concat_ws('#',ccarnumber,clicensetype,substr(dillegaldate,0,19),coffense),ccarnumber,clicensetype,caddresscode,cillegaladdress,coffense,dillegaldate,cpic1path,cpic2path,cpic3path,export_time,month,day from kakou.vio_violation_pic_day_ods"
/opt/cloudera/parcels/CDH-5.15.1-1.cdh5.15.1.p0.4/bin/hive -e "insert into kakou.vio_violation_pic_his_ods PARTITION(day) SELECT CCARNUMBER,CLICENSETYPE,CADDRESSCODE,CILLEGALADDRESS,COFFENSE,DILLEGALDATE,CPIC1PATH,CPIC2PATH,CPIC3PATH,export_time,month,day from kakou.vio_violation_pic_day_ods DISTRIBUTE BY rand()" --hiveconf hive.exec.dynamic.partition.mode=nonstrict --hiveconf hive.exec.dynamic.partition=true --hiveconf hive.exec.max.dynamic.partitions=36600 --hiveconf hive.exec.max.dynamic.partitions.pernode=36600
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment