Skip to content

F
ftp_pic
Commit 1a74f06d by 李辅翼
Options

v4

parent ce2a82c6
Showing with 31 additions and 26 deletions
src/main/java/com/hikcreate/drv_photo_pic/impl/VioPicImpl.java
...@@ -53,13 +53,14 @@ public class VioPicImpl implements VioPic { ...@@ -53,13 +53,14 @@ public class VioPicImpl implements VioPic {
@Autowired @Autowired
private static FileService fileService; private FileService fileService;
private String rowkey; private String rowkey;
private static TableName vioTableName; private static TableName vioTableName;
private static Table vioTable; private static Table vioTable;
private static Configuration hbaseConf; private static Configuration hbaseConf;
private static org.apache.hadoop.hbase.client.Connection hbaseConn; private static org.apache.hadoop.hbase.client.Connection hbaseConn;
private static boolean isPut = false;
private static String hphm; private static String hphm;
private static String hpzl; private static String hpzl;
...@@ -83,7 +84,7 @@ public class VioPicImpl implements VioPic { ...@@ -83,7 +84,7 @@ public class VioPicImpl implements VioPic {
try { try {
String JDBC_DRIVER = "org.apache.hive.jdbc.HiveDriver"; String JDBC_DRIVER = "org.apache.hive.jdbc.HiveDriver";
Class.forName(JDBC_DRIVER); Class.forName(JDBC_DRIVER);
logger.info(hiveUrl + "-----------" + hivePassWord + "--------" + hiveUser); // logger.info(hiveUrl + "-----------" + hivePassWord + "--------" + hiveUser);
Connection connection = DriverManager.getConnection(hiveUrl, hiveUser, hivePassWord); Connection connection = DriverManager.getConnection(hiveUrl, hiveUser, hivePassWord);
PreparedStatement pstm = connection.prepareStatement(incrementVioPicSql); PreparedStatement pstm = connection.prepareStatement(incrementVioPicSql);
pstm.setString(1, endDay); pstm.setString(1, endDay);
...@@ -93,23 +94,23 @@ public class VioPicImpl implements VioPic { ...@@ -93,23 +94,23 @@ public class VioPicImpl implements VioPic {
// "INNER JOIN (SELECT * from default.vio_surveil_all WHERE clsj='null' or clsj='' or clsj is null) b WHERE a.ccarnumber=b.hphm " + // "INNER JOIN (SELECT * from default.vio_surveil_all WHERE clsj='null' or clsj='' or clsj is null) b WHERE a.ccarnumber=b.hphm " +
// "and a.clicensetype=b.hpzl and substr(a.dillegaldate,0,16)=substr(b.wfsj,0,16)"; // "and a.clicensetype=b.hpzl and substr(a.dillegaldate,0,16)=substr(b.wfsj,0,16)";
// PreparedStatement pstm = connection.prepareStatement(incrementVioPicSql); // PreparedStatement pstm = connection.prepareStatement(incrementVioPicSql);
logger.info(pstm.toString()); // logger.info(pstm.toString());
logger.info(startDay); // logger.info(startDay);
logger.info(endDay); // logger.info(endDay);
ResultSet resultSet = pstm.executeQuery(); ResultSet resultSet = pstm.executeQuery();
hbaseConf = HBaseConfiguration.create(); hbaseConf = HBaseConfiguration.create();
hbaseConf.set("hbase.zookeeper.property.clientPort", hbaseAddress); hbaseConf.set("hbase.zookeeper.property.clientPort", hbaseAddress);
hbaseConf.set("hbase.zookeeper.quorum", hbaseZkQuorum); hbaseConf.set("hbase.zookeeper.quorum", hbaseZkQuorum);
hbaseConf.set("hbase.master", hbaseMaster); // hbaseConf.set("hbase.master", hbaseMaster);
hbaseConn = ConnectionFactory.createConnection(hbaseConf); hbaseConn = ConnectionFactory.createConnection(hbaseConf);
logger.info("success connection --------------------------"); // logger.info("success connection --------------------------");
vioTableName = TableName.valueOf(vioTableStr); vioTableName = TableName.valueOf(vioTableStr);
vioTable = hbaseConn.getTable(vioTableName); vioTable = hbaseConn.getTable(vioTableName);
restTemplate = new RestTemplate(); restTemplate = new RestTemplate();
headers = new HttpHeaders(); headers = new HttpHeaders();
method = HttpMethod.POST; method = HttpMethod.POST;
headers.setContentType(MediaType.APPLICATION_FORM_URLENCODED); headers.setContentType(MediaType.APPLICATION_FORM_URLENCODED);
logger.info("dgewhdjdddddddddddddd-------------"); // logger.info("dgewhdjdddddddddddddd-------------");
loop1: loop1:
while (resultSet.next()) { while (resultSet.next()) {
logger.info("数据库查询出东西了------------"); logger.info("数据库查询出东西了------------");
...@@ -149,7 +150,7 @@ public class VioPicImpl implements VioPic { ...@@ -149,7 +150,7 @@ public class VioPicImpl implements VioPic {
colValue[5] = value; colValue[5] = value;
} }
} }
logger.info("查询get结果:"+colValue.toString()); logger.info("查询get结果:" + colValue.toString());
//插入数据操作 //插入数据操作
//url1 //url1
if (url1 != null && !"".equals(url1)) { if (url1 != null && !"".equals(url1)) {
...@@ -159,11 +160,11 @@ public class VioPicImpl implements VioPic { ...@@ -159,11 +160,11 @@ public class VioPicImpl implements VioPic {
} }
if (colValue[0] == null || "".equals(colValue[0])) { if (colValue[0] == null || "".equals(colValue[0])) {
//url1插入ftpurl1 //url1插入ftpurl1
putData("ftpUrl1", "fastDfsUrl1", url1, ftpUrl); putData("ftpUrl1", "fastDfsUrl1", url1, ftpUrl, fileService);
} else { } else {
if (colValue[3] == null || "".equals(colValue[3])) { if (colValue[3] == null || "".equals(colValue[3])) {
//url1插入ftpurl4 //url1插入ftpurl4
putData("ftpUrl4", "fastDfsUrl4", url1, ftpUrl); putData("ftpUrl4", "fastDfsUrl4", url1, ftpUrl, fileService);
} }
} }
} }
...@@ -173,11 +174,11 @@ public class VioPicImpl implements VioPic { ...@@ -173,11 +174,11 @@ public class VioPicImpl implements VioPic {
} }
if (colValue[1] == null || "".equals(colValue[1])) { if (colValue[1] == null || "".equals(colValue[1])) {
//url1插入ftpurl1 //url1插入ftpurl1
putData("ftpUrl2", "fastDfsUrl2", url2, ftpUrl); putData("ftpUrl2", "fastDfsUrl2", url2, ftpUrl, fileService);
} else { } else {
if (colValue[4] == null || "".equals(colValue[4])) { if (colValue[4] == null || "".equals(colValue[4])) {
//url1插入ftpurl4 //url1插入ftpurl4
putData("ftpUrl5", "fastDfsUrl5", url2, ftpUrl); putData("ftpUrl5", "fastDfsUrl5", url2, ftpUrl, fileService);
} }
} }
} }
...@@ -187,15 +188,18 @@ public class VioPicImpl implements VioPic { ...@@ -187,15 +188,18 @@ public class VioPicImpl implements VioPic {
} }
if (colValue[2] == null || "".equals(colValue[2])) { if (colValue[2] == null || "".equals(colValue[2])) {
//url1插入ftpurl1 //url1插入ftpurl1
putData("ftpUrl3", "fastDfsUrl3", url3, ftpUrl); putData("ftpUrl3", "fastDfsUrl3", url3, ftpUrl, fileService);
} else { } else {
if (colValue[5] == null || "".equals(colValue[5])) { if (colValue[5] == null || "".equals(colValue[5])) {
//url1插入ftpurl4 //url1插入ftpurl4
putData("ftpUrl6", "fastDfsUrl6", url3, ftpUrl); putData("ftpUrl6", "fastDfsUrl6", url3, ftpUrl, fileService);
} }
} }
} }
vioTable.put(put); if (isPut) {
vioTable.put(put);
isPut = false;
}
} }
resultSet.close(); resultSet.close();
pstm.close(); pstm.close();
...@@ -225,26 +229,27 @@ public class VioPicImpl implements VioPic { ...@@ -225,26 +229,27 @@ public class VioPicImpl implements VioPic {
} }
private static void putData(String hbaseFtpCol, String hbaseFastCol, String url, String ftpUrl) { private static void putData(String hbaseFtpCol, String hbaseFastCol, String url, String ftpUrl, FileService fileService) {
put.addColumn("useful".getBytes(), hbaseFtpCol.getBytes(), url.getBytes());
//存fastDfs //存fastDfs
params = new LinkedMultiValueMap<String, String>(); params = new LinkedMultiValueMap<String, String>();
params.add("urls", url); params.add("urls", url);
requestEntity = new HttpEntity<>(params, headers); requestEntity = new HttpEntity<>(params, headers);
ResponseEntity<PicByte> resp = restTemplate.exchange(ftpUrl, method, requestEntity, PicByte.class); ResponseEntity<PicByte> resp = restTemplate.exchange(ftpUrl, method, requestEntity, PicByte.class);
Map<String, byte[]> map = resp.getBody().getMap(); Map<String, byte[]> map = resp.getBody().getMap();
if(map.size()>0){ if (map != null && map.size() > 0) {
for (Map.Entry<String, byte[]> entry : map.entrySet()) { for (Map.Entry<String, byte[]> entry : map.entrySet()) {
String[] arr = url.split("/"); String[] arr = url.split("/");
// logger.info("数组++++"+arr.length+"======"+url+"*****"+map.size()+"***"+arr[arr.length - 1]);
// logger.info("进入循环"+fileService);
String fastDfsUrl = fileService.uploadFile(entry.getValue(), arr[arr.length - 1]); String fastDfsUrl = fileService.uploadFile(entry.getValue(), arr[arr.length - 1]);
logger.info("fastdfs地址"+fastDfsUrl); logger.info("fastdfs地址" + fastDfsUrl);
put.addColumn("useful".getBytes(), hbaseFastCol.getBytes(), fastDfsUrl.getBytes()); put.addColumn("useful".getBytes(), hbaseFastCol.getBytes(), fastDfsUrl.getBytes());
put.addColumn("useful".getBytes(), hbaseFtpCol.getBytes(), url.getBytes());
isPut = true;
} }
}else { } else {
logger.info("此ftp地址数据为空：=========================="+url); logger.info("此ftp地址数据为空：==========================" + url);
} }
} }
} }
src/main/java/com/hikcreate/service/fdfs/service/impl/FileServiceImpl.java
...@@ -14,7 +14,7 @@ import javax.annotation.Resource; ...@@ -14,7 +14,7 @@ import javax.annotation.Resource;
* @author 赵东 * @author 赵东
* @create 2019/3/22 13:45 * @create 2019/3/22 13:45
*/ */
@Service @Service("fileServiceImpl")
public class FileServiceImpl implements FileService { public class FileServiceImpl implements FileService {
private static Logger logger = LoggerFactory.getLogger(FileServiceImpl.class); private static Logger logger = LoggerFactory.getLogger(FileServiceImpl.class);
@Resource @Resource
......
src/main/resources/application.properties
...@@ -29,7 +29,7 @@ hbase.driverlicense.table=drv:drivinglicense ...@@ -29,7 +29,7 @@ hbase.driverlicense.table=drv:drivinglicense
hbase.vio.table=vio:vio_violation hbase.vio.table=vio:vio_violation
#hive----sql #hive----sql
increment.vio.pic.sql=SELECT a.ccarnumber hphm,a.clicensetype hpzl,b.wfsj wfsj,a.cpic1path url1,a.cpic2path url2,a.cpic3path url3 from (SELECT * from kakou.vio_violation_pic_his_ods WHERE substr(export_time,0,10)<=? and substr(export_time,0,10)>?) a INNER JOIN (SELECT * from default.vio_surveil_all WHERE clsj='null' or clsj='' or clsj is null) b WHERE a.ccarnumber=b.hphm and a.clicensetype=b.hpzl and substr(a.dillegaldate,0,16)=substr(b.wfsj,0,16) increment.vio.pic.sql=SELECT a.ccarnumber hphm,a.clicensetype hpzl,b.wfsj wfsj,a.cpic1path url1,a.cpic2path url2,a.cpic3path url3 from (SELECT * from kakou.vio_violation_pic_his_ods WHERE substr(export_time,0,10)<=? and substr(export_time,0,10)>?) a INNER JOIN (SELECT * from default.vio_surveil_all WHERE clbj='0') b WHERE a.ccarnumber=b.hphm and a.clicensetype=b.hpzl and substr(a.dillegaldate,0,16)=substr(b.wfsj,0,16)
pastDay=10 pastDay=10
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment