數據存放的環境

系統版本：CentOS 7.5

數據庫：MySQL 5.7.23

?

數據庫設計

設計概述

ODS表為原始數據，數據從客戶業務系統拉取，包含中文數據；

DW表去掉中文后的數據，數據從ODS表獲取；

DM為展示的數據，即系統最后需求的數據；

T為基礎表。

銀行表：t_bank

原始數據表：

ods_bankdata_gs,ods_bankdata_ny,ods_bankdata_zg,ods_bankdata_js……

ods_bankdata_gs_history,ods_bankdata_ny_history……..

DW數據表：

dw_bankdata_gs,dw_bankdata_ny,dw_bankdata_zg,dw_bankdata_js……

dw_bankdata_gs_history,dw_bankdata_ny_hisroty……..

DM展示表：dm_bankfor

?

具體設計

#創建數據庫

create database dbBank;

?

/*

需求：

拉取過來的數據(比如浦發，興業，農行等)，格式如：

興業：

手機號,姓名,開通信用卡功能(10萬條)

浦發：

手機號,姓名,開通信用卡功能(10萬條)

......

?

將上面的數據合并，因為每個身份證號有可能在多家銀行開有信用卡，需要將20萬條數據，假如這時只有18萬不重復身份證號的數據

*/

Base

#################################################

##################??? BASE???? ##################

#################################################

?

/*銀行表：t_bank

序號,代碼,銀行名,銀行簡寫

bankId

bankCode

bankName

bankAD

*/

create table t_bank(bankId int primary key auto_increment comment '主鍵自增',bankCode varchar(50) comment '銀行代碼',bankName varchar(200) not null unique comment '銀行名',bankAD varchar(20) not null unique comment '銀行簡寫')comment='銀行表';insert into t_bank values(1,'','工商銀行','gs');insert into t_bank values(2,'','農業銀行','ny');insert into t_bank values(3,'','中國銀行','zg');insert into t_bank values(4,'','建設銀行','js');insert into t_bank values(5,'','交通銀行','jt');insert into t_bank values(6,'','進出口銀行','jck');insert into t_bank values(7,'','國家開發銀行','kf');insert into t_bank values(8,'','農業發展銀行','nf');insert into t_bank values(9,'','招商銀行','zs');insert into t_bank values(10,'','浦發銀行','pf');insert into t_bank values(11,'','中信銀行','zx');insert into t_bank values(12,'','光大銀行','gd');insert into t_bank values(13,'','華夏銀行','hx');insert into t_bank values(14,'','民生銀行','ms');insert into t_bank values(15,'','廣發銀行','gf');insert into t_bank values(16,'','興業銀行','xy');insert into t_bank values(17,'','平安銀行','pa');insert into t_bank values(18,'','浙商銀行','zhes');insert into t_bank values(19,'','恒豐銀行','hf');insert into t_bank values(20,'','渤海銀行','bh');insert into t_bank values(21,'','廣西農村信用社','gxnxs');insert into t_bank values(22,'','郵政儲蓄銀行','yz');commit;

ods

##################################################

##################??? ODS???? ####################

##################################################

?

/*原始數據表：根據銀行類型分表：ods_bankdata_gs,ods_bankdata_ny,ods_bankdata_zg,ods_bankdata_js,

序號,姓名,手機號,身份證號,銀行名,插入時間,備注

id

name

phone

idNumber

bankName

insTime

notes

*/

?

#--工商銀行_ods

drop table ods_bankdata_gs;create table ods_bankdata_gs(id bigint primary key auto_increment comment '序號',name varchar(50) comment '姓名',phone bigint comment '手機號',idnumber varchar(20) comment '身份證號',bankname varchar(100) comment '銀行名',instime datetime comment '插入時間',notes varchar(200) comment '備注')comment='工商銀行原始表';insert into ods_bankdata_gs values(1,'王正初',18502461576,'450204198004293000','工商銀行',sysdate(),'');insert into ods_bankdata_gs values(2,'李娟麗',18845873946,'653101198412151000','工商銀行',sysdate(),'');insert into ods_bankdata_gs values(3,'趙語山',19957322330,'320621197702127000','工商銀行',sysdate(),'');insert into ods_bankdata_gs values(4,'張善和',16322872701,'150722198112018000','工商銀行',sysdate(),'');insert into ods_bankdata_gs values(5,'張南煙',12850907936,'52262819870913206x','工商銀行',sysdate(),'');insert into ods_bankdata_gs values(6,'王夢桃',15816875148,'440403199007149000','工商銀行',sysdate(),'');insert into ods_bankdata_gs values(7,'王清涵',15875793602,'430923197805092000','工商銀行',sysdate(),'');insert into ods_bankdata_gs values(8,'李春嬌',12610912365,'130500199411174000','工商銀行',sysdate(),'');insert into ods_bankdata_gs values(9,'王冰凡',14968798213,'141026198706191000','工商銀行',sysdate(),'');insert into ods_bankdata_gs values(10,'王鈴語',19781652609,'533323198803306000','工商銀行',sysdate(),'');insert into ods_bankdata_gs values(11,'李博裕',11897455113,'340826198707281000','工商銀行',sysdate(),'');insert into ods_bankdata_gs values(12,'李昊然',16237208490,'530181198712230000','工商銀行',sysdate(),'');insert into ods_bankdata_gs values(13,'李沛凝',18044968073,'150100198704294000','工商銀行',sysdate(),'');insert into ods_bankdata_gs values(14,'張詩蕾',16534073687,'610327198502213000','工商銀行',sysdate(),'');insert into ods_bankdata_gs values(15,'張懷玉',18873064922,'210181198503183000','工商銀行',sysdate(),'');insert into ods_bankdata_gs values(16,'張晨希',14113640518,'150105199306017000','工商銀行',sysdate(),'');insert into ods_bankdata_gs values(17,'趙安露',13071845241,'441825198011015000','工商銀行',sysdate(),'');insert into ods_bankdata_gs values(18,'趙凝芙',18809594871,'433126199306095000','工商銀行',sysdate(),'');insert into ods_bankdata_gs values(19,'羅初晴',19594219248,'500108199111039000','工商銀行',sysdate(),'');insert into ods_bankdata_gs values(20,'羅飛昂',17786947697,'411724197606164000','工商銀行',sysdate(),'');commit;

#--農業銀行_ods

drop table ods_bankdata_ny;create table ods_bankdata_ny(id bigint primary key auto_increment comment '序號',name varchar(50) comment '姓名',phone bigint comment '手機號',idnumber varchar(20) comment '身份證號',bankname varchar(100) comment '銀行名',instime datetime comment '插入時間',notes varchar(200) comment '備注')comment='農業銀行原始表';insert into ods_bankdata_ny values(1,'張善和',16322872701,'150722198112018000','農業銀行',sysdate(),'');insert into ods_bankdata_ny values(2,'李昊然',16237208490,'530181198712230000','農業銀行',sysdate(),'');insert into ods_bankdata_ny values(3,'趙安露',13071845241,'441825198011015000','農業銀行',sysdate(),'');insert into ods_bankdata_ny values(4,'羅飛昂',17786947697,'411724197606164000','農業銀行',sysdate(),'');insert into ods_bankdata_ny values(5,'趙凝芙',18809594871,'433126199306095000','農業銀行',sysdate(),'');insert into ods_bankdata_ny values(6,'陳弘揚',18492438399,'231283197705204000','農業銀行',sysdate(),'');insert into ods_bankdata_ny values(7,'陳晶晶',12552681742,'15010019870622882x','農業銀行',sysdate(),'');insert into ods_bankdata_ny values(8,'陳奇思',18725476234,'441601199201178000','農業銀行',sysdate(),'');insert into ods_bankdata_ny values(9,'陳凝遠',13876150809,'621224198305040000','農業銀行',sysdate(),'');insert into ods_bankdata_ny values(10,'李學',16162095706,'150102198008201000','農業銀行',sysdate(),'');insert into ods_bankdata_ny values(11,'張戈',17573568191,'140524199110122000','農業銀行',sysdate(),'');insert into ods_bankdata_ny values(12,'卜洪',12939298114,'370826198206133000','農業銀行',sysdate(),'');insert into ods_bankdata_ny values(13,'羅明',18581723138,'411224197509134000','農業銀行',sysdate(),'');insert into ods_bankdata_ny values(14,'歐陽偉大',18972010816,'610822198907180000','農業銀行',sysdate(),'');insert into ods_bankdata_ny values(15,'錢懷柔',18910192175,'513426199302189000','農業銀行',sysdate(),'');insert into ods_bankdata_ny values(16,'錢采波',13645247225,'451221199203159000','農業銀行',sysdate(),'');insert into ods_bankdata_ny values(17,'錢秋芳',18550469783,'321324198203030000','農業銀行',sysdate(),'');insert into ods_bankdata_ny values(18,'朱天青',13827526397,'450901198007240000','農業銀行',sysdate(),'');insert into ods_bankdata_ny values(19,'朱思凡',18075584434,'450981199210025000','農業銀行',sysdate(),'');insert into ods_bankdata_ny values(20,'朱天薇',17391130746,'451122198505270000','農業銀行',sysdate(),'');commit;

?

#----ods歷史表，用于存儲歷史數據----#

?

#--工商銀行_ods_history

drop table ods_bankdata_gs_history;create table ods_bankdata_gs_history(id bigint primary key auto_increment comment '序號',name varchar(50) comment '姓名',phone bigint comment '手機號',idnumber varchar(20) comment '身份證號',bankname varchar(100) comment '銀行名',instime datetime comment '插入時間',notes varchar(200) comment '備注')comment='工商銀行原始表_歷史表';

?

?

#--農業銀行_ods_history

drop table ods_bankdata_ny_history;create table ods_bankdata_ny_history(id bigint primary key auto_increment comment '序號',name varchar(50) comment '姓名',phone bigint comment '手機號',idnumber varchar(20) comment '身份證號',bankname varchar(100) comment '銀行名',instime datetime comment '插入時間',notes varchar(200) comment '備注')comment='農業銀行原始表歷史表';

dw

################################################

################??? DW???? #####################

################################################

?

/*DW表：根據銀行類型分表： dw_bankdata_gs,dw_bankdata_ny,dw_bankdata_zg,dw_bankdata_js,

序號,姓名,手機號,身份證號,銀行名,銀行簡寫,插入時間

id

idnumber

phone

bankad

batch

instime

*/

#--工商銀行_dw

drop table dw_bankdata_gs;create table dw_bankdata_gs(id bigint primary key auto_increment comment '序號',idnumber varchar(20) comment '身份證號',phone bigint comment '手機號',bankad varchar(100) comment '銀行簡寫',batch_gs int comment '工商批次',instime datetime comment '插入時間')comment='工商銀行dw';

?

#--農業銀行_dw

drop table dw_bankdata_ny;create table dw_bankdata_ny(id bigint primary key auto_increment comment '序號',idnumber varchar(20) comment '身份證號',phone bigint comment '手機號',bankad varchar(100) comment '銀行簡寫',batch_ny int comment '農業批次',instime datetime comment '插入時間')comment='農業銀行dw';

?

?

#######--------歷史表，用于存儲歷史數據

#--工商銀行_dw_history

drop table dw_bankdata_gs_history;create table dw_bankdata_gs_history(id bigint primary key auto_increment comment '序號',idnumber varchar(20) comment '身份證號',phone bigint comment '手機號',bankad varchar(100) comment '銀行簡寫',batch_gs int comment '工商批次',instime date comment '插入時間')comment='工商銀行dw';

?

#--農業銀行_dw_history

drop table dw_bankdata_ny_history;create table dw_bankdata_ny_history(id bigint primary key auto_increment comment '序號',idnumber varchar(20) comment '身份證號',phone bigint comment '手機號',bankad varchar(100) comment '銀行簡寫',batch_ny int comment '農業批次',instime datetime comment '插入時間')comment='農業銀行dw';

dm

##################################

########??? DM???? ###############

##################################

?

/*

**************************

實際需求的表--數據表：dm_bankfor

序號,身份證號,手機號,銀行簡寫,插入時間

id

idnumber

phone

bankad

instime

batch_gs

instime_gs

batch_ny

instime_ny

……..

**************************

*/

drop table dm_bankfor;create table dm_bankfor(id bigint primary key auto_increment comment '序號',idnumber varchar(20) not null unique comment '身份證號',phone bigint comment '手機號',bankad varchar(100) comment '銀行簡寫匯總',batch_gs int comment '工商批次',instime_gs datetime comment '工商插入時間',batch_ny int comment '農業批次',instime_ny datetime comment '農業插入時間',batch_zg int comment '中國銀行批次',instime_zg datetime comment '中國銀行插入時間',batch_js int comment '建設批次',instime_js datetime comment '建設插入時間',batch_jt int comment '交通批次',instime_jt datetime comment '交通插入時間',batch_jck int comment '進出口批次',instime_jck datetime comment '進出口插入時間',batch_kf int comment '國家開發批次',instime_kf datetime comment '國家開發插入時間',batch_nf int comment '農業發展批次',instime_nf datetime comment '農業發展插入時間',batch_zs int comment '招商批次',instime_zs datetime comment '招商插入時間',batch_pf int comment '浦發批次',instime_pf datetime comment '浦發插入時間',batch_zx int comment '中信批次',instime_zx datetime comment '中信插入時間',batch_gd int comment '光大批次',instime_gd datetime comment '光大插入時間',batch_hx int comment '華夏批次',instime_hx datetime comment '華夏插入時間',batch_ms int comment '民生批次',instime_ms datetime comment '民生插入時間',batch_gf int comment '廣發批次',instime_gf datetime comment '廣發插入時間',batch_xy int comment '興業批次',instime_xy datetime comment '興業插入時間',batch_pa int comment '平安批次',instime_pa datetime comment '平安插入時間',batch_zhes int comment '浙商批次',instime_zhes datetime comment '浙商插入時間',batch_hf int comment '恒豐批次',instime_hf datetime comment '恒豐插入時間',batch_bh int comment '渤海批次',instime_bh datetime comment '渤海插入時間',batch_gxnxs int comment '廣西農村信用社批次',instime_gxnxs datetime comment '廣西農村信用社插入時間',batch_yz int comment '郵政儲蓄銀行批次',instime_yz datetime comment '郵政儲蓄銀行插入時間')comment='信用卡開通數據表';

存儲過程

##################################################

################??? 存儲過程???? #################

##################################################

/*

**************************

ods到dw，工商銀行

**************************

*/

?

delimiter //drop procedure pro_dw_gs;create procedure pro_dw_gs()begin#聲明一個標志done,用來判斷游標是否遍歷完成declare v_done int default false;#特別注意這里的名字不能與由游標中使用的列明相同，否則得到的數據都是nulldeclare v_idnumber varchar ( 255 ) default null;declare v_phone bigint default null;declare v_instime date default null;declare v_batch_gs int;#定義一個判斷dw_bankdata_gs表是否有數據declare v_dw_gs bigint;#聲明游標對應的 sql 語句declare cur cursor for select idnumber,phone,instime from ods_bankdata_gs;#在游標循環到最后會將 done 設置為 truedeclare continue handler for not found set v_done = true;#假如dw_bankdata_gs不為空，插入數據前將dw_bankdata_gs的數據插入dw_bankdata_gs_history歷史表select count(*) into v_dw_gs from dw_bankdata_gs;if v_dw_gs <> 0 theninsert into dw_bankdata_gs_history select * from dw_bankdata_gs;delete from dw_bankdata_gs;commit;end if;#batch_gs的值select count(distinct batch_gs) into v_batch_gs from dw_bankdata_gs_history;if v_batch_gs = 0 thenset v_batch_gs = 1;elseselect max(distinct batch_gs)+1 into v_batch_gs from dw_bankdata_gs_history;end if;#執行查詢open cur;#遍歷游標每一行read_loop :loop#提取游標里的數據fetch cur into v_idnumber,v_phone,v_instime;#聲明結束的時候if v_done thenleave read_loop;end if;insert into dw_bankdata_gs ( idnumber, phone, bankad, batch_gs, instime )values( v_idnumber, v_phone, 'gs', v_batch_gs, v_instime );end loop;#關閉游標close cur;end//delimiter ;

#執行存儲過程

call pro_dw_gs();

#刪除存儲過程

#drop procedure pro_dw_gs;

?

/*

**************************

ods到dw，農業銀行

**************************

*/

?

delimiter //create procedure pro_dw_ny()begin#聲明一個標志done,用來判斷游標是否遍歷完成declare v_done int default false;#特別注意這里的名字不能與由游標中使用的列明相同，否則得到的數據都是nulldeclare v_idnumber varchar ( 255 ) default null;declare v_phone bigint default null;declare v_instime date default null;declare v_batch_ny int;declare v_dw_ny bigint;#聲明游標對應的 sql 語句declare cur cursor for select idnumber,phone,instime from ods_bankdata_ny;#在游標循環到最后會將 done 設置為 truedeclare continue handler for not found set v_done = true;#假如dw_bankdata_gs不為空，插入數據前將dw_bankdata_gs的數據插入dw_bankdata_gs_history歷史表select count(*) into v_dw_ny from dw_bankdata_ny;if v_dw_ny <> 0 theninsert into dw_bankdata_ny_history select * from dw_bankdata_ny;delete from dw_bankdata_ny;commit;end if;#batch_ny的值select count(distinct batch_ny) into v_batch_ny from dw_bankdata_ny_history;if v_batch_ny = 0 thenset v_batch_ny = 1;elseselect max(distinct batch_ny)+1 into v_batch_ny from dw_bankdata_ny_history;end if;#執行查詢open cur;#遍歷游標每一行read_loop :loop#提取游標里的數據fetch cur into v_idnumber,v_phone,v_instime;#聲明結束的時候if v_done thenleave read_loop;end if;insert into dw_bankdata_ny ( idnumber, phone, bankad, batch_ny, instime )values( v_idnumber, v_phone, 'ny', v_batch_ny, v_instime );end loop;#關閉游標close cur;end//delimiter ;

#執行存儲過程

call pro_dw_ny();

#刪除存儲過程

#drop procedure pro_dw_ny;

/*

**************************

DM 匯總數據_工商銀行

**************************

*/

?

delimiter //#drop procedure if exists pro_dm_all_gs;create procedure pro_dm_all_gs()begin#聲明一個標志done,用來判斷游標是否遍歷完成declare v_done int default false;#特別注意這里的名字不能與由游標中使用的列明相同，否則得到的數據都是nulldeclare v_idnumber varchar ( 255 ) default null;declare v_phone bigint default null;declare v_bankad varchar(200) default null;declare v_batch_gs int default null;declare v_instime_gs date default null;#定義兩個判斷符declare v_exists varchar(10) default null;declare v_exists_gs varchar(10) default null;#聲明游標對應的 sql 語句declare cur cursor for select idnumber,phone,bankad,batch_gs,instime from dw_bankdata_gs;#在游標循環到最后會將 done 設置為 truedeclare continue handler for not found set v_done = true;#執行查詢open cur;#遍歷游標每一行read_loop :loop#提取游標里的數據fetch cur into v_idnumber,v_phone,v_bankad,v_batch_gs,v_instime_gs;#聲明結束的時候if v_done thenleave read_loop;end if;#判斷是否有這個身份證號碼select count(*) into v_exists from dm_bankfor where idnumber = v_idnumber;if v_exists = 0 then#dm_bankfor表里沒有這個身份證的數據，直接插入數據insert into dm_bankfor ( idnumber, phone, bankad, batch_gs, instime_gs )values( v_idnumber, v_phone, v_bankad, v_batch_gs, v_instime_gs );else#dm_bankfor表有這個身份證的數據，更新update dm_bankfor set bankad='gs'where idnumber = v_idnumber and length(bankad) = 0;#update dm_bankfor set bankad=concat(bankad,',',v_bankad)where idnumber = v_idnumber and bankad not like ('%gs%') and length(bankad) <> 0;#更新batchupdate dm_bankfor set batch_gs = v_batch_gs where idnumber = v_idnumber and batch_gs is null;#更新instime_gsupdate dm_bankfor set instime_gs = v_instime_gs where idnumber = v_idnumber and instime_gs is null;end if;end loop;#關閉游標close cur;end//delimiter ;

call pro_dm_all_gs();

#刪除存儲過程

#drop procedure pro_dm_all_gs;

?

?

/*

**************************

DM 匯總數據_農業銀行

**************************

*/

?

delimiter //drop procedure if exists pro_dm_all_ny;create procedure pro_dm_all_ny()begin#聲明一個標志done,用來判斷游標是否遍歷完成declare v_done int default false;#特別注意這里的名字不能與由游標中使用的列明相同，否則得到的數據都是nulldeclare v_idnumber varchar ( 255 ) default null;declare v_phone bigint default null;declare v_bankad varchar(200) default null;declare v_batch_ny int default null;declare v_instime_ny date default null;#定義兩個判斷符declare v_exists varchar(10) default null;declare v_exists_ny varchar(10) default null;#聲明游標對應的 sql 語句declare cur cursor for select idnumber,phone,bankad,batch_ny,instime from dw_bankdata_ny;#在游標循環到最后會將 done 設置為 truedeclare continue handler for not found set v_done = true;#執行查詢open cur;#遍歷游標每一行read_loop :loop#提取游標里的數據fetch cur into v_idnumber,v_phone,v_bankad,v_batch_ny,v_instime_ny;#聲明結束的時候if v_done thenleave read_loop;end if;#判斷是否有這個身份證號碼select count(*) into v_exists from dm_bankfor where idnumber = v_idnumber;if v_exists = 0 then#dm_bankfor表里沒有這個身份證的數據，直接插入數據insert into dm_bankfor ( idnumber, phone, bankad, batch_ny, instime_ny )values( v_idnumber, v_phone, v_bankad, v_batch_ny, v_instime_ny );else#dm_bankfor表有這個身份證的數據，更新update dm_bankfor set bankad='ny'where idnumber = v_idnumber and length(bankad) = 0;#update dm_bankfor set bankad=concat(bankad,',',v_bankad)where idnumber = v_idnumber and bankad not like ('%ny%') and length(bankad) <> 0;#更新batchupdate dm_bankfor set batch_ny = v_batch_ny where idnumber = v_idnumber and batch_ny is null;#更新instime_nyupdate dm_bankfor set instime_ny = v_instime_ny where idnumber = v_idnumber and instime_ny is null;end if;end loop;#關閉游標close cur;end//delimiter ;

call pro_dm_all_ny();

#刪除存儲過程

#drop procedure pro_dm_all_ny;

?

Python導數

1、將excle表格數據導入到mysql數據庫下ods表

#!/usr/bin/env python3# -*- coding: utf-8 -*-import pymysql #支持Python3.0##讀取excel使用(支持03)import xlrdfrom datetime import datetimefrom xlrd import xldate_as_tuplefrom builtins import int##打開數據庫conn = pymysql.connect(host="192.168.8.31",port=3306,user="root",passwd="123",db="dbBank",charset="utf8")##打開游標cur = conn.cursor()##將excel文件導入mysql中def importExcelToMysql(path):#把ods表數據插入到history表cur.execute("insert into ods_bankdata_gs_history select * from ods_bankdata_gs")#刪除表數據cur.execute("delete from ods_bankdata_gs")#根據Excel路徑讀取Excelworkbook = xlrd.open_workbook(path)sheets = workbook.sheet_names()#根據sheet名稱獲取sheet,sheets[0]為第一個表格名稱worksheet = workbook.sheet_by_name(sheets[0])##遍歷行for i in range(1, worksheet.nrows):row = worksheet.row(i)##初始化數組sqlstr = []##遍歷列for j in range(0, worksheet.ncols):##構造數組sqlstr.append(worksheet.cell_value(i, j))##插入數據庫valuestr = [str(sqlstr[0]), int(sqlstr[1]), str(sqlstr[2]), str(sqlstr[3])]##執行sql語句cur.execute("insert into ods_bankdata_gs(name,phone,idnumber,bankname,instime) " +"values(%s,%s,%s,%s,sysdate())", valuestr)#關閉游標cur.close()#提交conn.commit()#關閉連接conn.close()# 打印信息rows = str(worksheet.nrows)print("成功導入: %s 條數據"%rows)#excel文件路徑read03path = r"D:\0data\python\juns\juns01.xlsx";#開始執行時間starttime = datetime.now()print(starttime)##調用函數importExcelToMysql(read03path)#結束時間endtime = datetime.now()print(endtime)

?

總結

以上是生活随笔為你收集整理的【juns项目】信用卡数据项目2-2：数据设计的全部內容，希望文章能夠幫你解決所遇到的問題。

如果覺得生活随笔網站內容還不錯，歡迎將生活随笔推薦給好友。