-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathgen_json_ora.py
300 lines (279 loc) · 12.8 KB
/
gen_json_ora.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
#!/usr/bin/env python
# -*- coding:utf-8 -*-
__author__ = 'wangrong'
import json
import os
import re
import chardet
import zipfile
# class config_json:
# def __init__(self):
# '''
# 初始化字典
# '''
# self.ExecCfg = {}
# self.ExecCfg['fileSeq'] = ''
# self.ExecCfg['dataSourceType'] = ''
# self.ExecCfg['url'] = ''
# self.ExecCfg['username'] = ''
# self.ExecCfg['password'] = ''
# self.ExecCfg['sqlFileList'] = list()
#
# def setfileseq(self, fileseq):
# self.ExecCfg['fileSeq'] = fileseq
#
# def setdatasourcetype(self, datasourcetype):
# self.ExecCfg['dataSourceType'] = datasourcetype
#
# def seturl(self, url):
# self.ExecCfg['url'] = url
#
# def setusername(self, username):
# self.ExecCfg['username'] = username
#
# def setpassword(self, password):
# self.ExecCfg['password'] = password
#
# def setsqlfilelist(self, sqlfilelist):
# self.ExecCfg['sqlFileList'].append(sqlfilelist)
#
# def GetDbInstance(self, dbuser):
# pass
# # return dbschema,dbport
class GenJson4Ora():
def __init__(self, script_home, db_con_dns):
'''
:param script_home: script_home = r'E:/Python3Project/testdir'
:param db_con_dns: db_con_dns = '172.16.80.41:11521:cc'
'''
script_home = script_home.replace('\\', '/')
if script_home[-1] == '/':
self.ScriptHome = script_home[:-1]
else:
self.ScriptHome = script_home
if db_con_dns is None:
self.DbConDnsInfo = {
'instance': {
'cc': ['apig', 'ftf', 'cic', 'cpc', 'cpc_flow', 'custc', 'oc', 'drm', 'ebc', 'pos', 'sa', 'src'],
'pmt': ['ab', 'bc', 'pcc'],
'rb': ['pcb', 'inv', 'rb'],
'sett': ['med', 'sett'],
'stbp': ['stbp', 'etl', 'dap']},
'dburl': {'cc': ['172.16.80.11', '11521'],
'pmt': ['172.16.80.12', '11521'],
'rb': ['172.16.80.13', '11521'],
'sett': ['172.16.80.14', '11521'],
'stbp': ['172.16.80.15', '11521']},
'password': ''
}
else:
self.DbConDnsInfo = db_con_dns
def GetDnsString(self, dbuser):
for one_instance in self.DbConDnsInfo['instance']:
if isinstance(self.DbConDnsInfo['instance'][one_instance], list):
for one_value in self.DbConDnsInfo['instance'][one_instance]:
if one_value == dbuser:
db_instance = one_instance
db_string = 'jdbc:oracle:thin:@' + self.DbConDnsInfo['dburl'][db_instance][0] + ':' + \
self.DbConDnsInfo['dburl'][db_instance][1] + ':' + db_instance
return db_string
else:
continue
def GetUsernameFromFile(self, file_name, split_filter='_'):
try:
temp_username = os.path.splitext(file_name)[0].split(split_filter)[-3] + '_' + \
os.path.splitext(file_name)[0].split(split_filter)[-2]
except IndexError:
temp_username = 'NA'
if temp_username == 'cpc_flow':
username = 'cpc_flow'
elif temp_username == 'NA':
username = 'NA'
else:
username = os.path.splitext(file_name)[0].split(split_filter)[-2]
return username
def UpdateDictValues(self, dict, keys, input_values):
temp_values = dict[keys]
if isinstance(temp_values, (str, int, float, bool)):
temp_values = input_values
elif isinstance(temp_values, list):
if isinstance(input_values, (str, int, float, bool)):
temp_values.append(input_values)
else:
temp_values = temp_values + input_values
dict[keys] = temp_values
return dict
def CheckFileEncoding(self, fullpath_file_name):
'''
:param fullpath_file_name: 文件的全路径 'E:/Python3Project/testdir/balc/sql/balc_product_init_balc_bc_oracle.sql'
:return:
'''
f = open(fullpath_file_name, 'rb')
data = f.read()
code_result = chardet.detect(data)
if code_result['encoding'] != 'utf-8':
print(
"\033[0;31m[{}] Character Set Error\033[0m,current Character set is \033[1;31;42m{}\033[0m,Expected \033[1;31;42mUTF-8\033[0m character set ".format(
fullpath_file_name, code_result['encoding']))
def GetDictFromDir(self, fullpath_dir):
user_dir_dict = {}
no_user_file = []
last_db_username = ''
for dirpath, dirname, filenames in os.walk(fullpath_dir):
for file_name in filenames:
if os.path.splitext(file_name)[1] == '.sql':
# print('process file:{}'.format(file_name))
if re.search('qmdb', file_name.lower()):
continue
else:
fullpath_file = fullpath_dir + '/sql/' + file_name
# self.CheckFileEncoding(fullpath_file)
db_username = self.GetUsernameFromFile(file_name)
if db_username == 'NA':
no_user_file.append(file_name)
continue
if db_username in user_dir_dict.keys():
user_dir_dict = self.UpdateDictValues(user_dir_dict, db_username, file_name)
else:
user_dir_dict[db_username] = file_name.split()
last_db_username = db_username
if no_user_file:
user_dir_dict = self.UpdateDictValues(user_dir_dict, last_db_username, no_user_file)
return user_dir_dict
def SortList(self, input_list):
'''
:param input_list: 文件列表['balc_product_init_balc_bc_oracle.sql','balc_product_balc_bc_oracle.sql']
:return:
'''
# 定义5个临时变量，用来存储5种条件过滤出来的内容
tmpfile1 = []
tmpfile2 = []
tmpfile3 = []
tmpfile4 = []
tmpfile5 = []
tmpfile6 = []
tmpfile7 = []
tmpfile8 = []
for onefile in input_list:
if re.search('core', onefile.lower()) and not re.search('init', onefile.lower()):
tmpfile1.append(onefile)
elif re.search('core', onefile.lower()) and re.search('init', onefile.lower()):
tmpfile2.append(onefile)
elif re.search('prod', onefile.lower()) and re.search('tf', onefile.lower()):
tmpfile3.append(onefile)
elif re.search('prod', onefile.lower()) and re.search('bf', onefile.lower()):
tmpfile4.append(onefile)
elif re.search('prod', onefile.lower()) and not re.search('init', onefile.lower()):
tmpfile5.append(onefile)
elif re.search('prod', onefile.lower()) and re.search('init', onefile.lower()):
tmpfile6.append(onefile)
else:
tmpfile7.append(onefile)
## core >> core_init >> product >> product_init >> others
tmpfile = tmpfile1 + tmpfile2 + tmpfile3 + tmpfile4 + tmpfile5 + tmpfile6 + tmpfile7
return tmpfile
def SortFileList(self, sql_file_list):
sorted_file_list = self.SortList(sql_file_list)
sqlFileList = []
seqnum = 1
for one_sql_file in sorted_file_list:
one_dict = {}
one_dict['fileName'] = one_sql_file
one_dict['seqnum'] = seqnum
seqnum += 1
sqlFileList.append(one_dict)
return sqlFileList
def WriteJsonConfigFile(self, json_file_path, json_data):
with open(json_file_path, "w", encoding='utf-8') as f:
json.dump(json_data, f, indent=1)
print("\033[1;32m[2]\033[0mDump json to file {} \033[1;32msuccess\033[0m".format(json_file_path))
def ZipProductionSqlDir(self, zipdir, zip_filename='sql.zip', skip_keyword=['qmdb'], skip_postfix=['.zip']):
## formate zipdir path,remove '/'
if zipdir[-1] == '/':
zipdir = zipdir[:-1]
zip_filename = zipdir + '/' + zip_filename
zip_handle = zipfile.ZipFile(zip_filename, 'w', zipfile.ZIP_DEFLATED)
for dirpath, dirnames, filenames in os.walk(zipdir):
fpath = dirpath.replace(zipdir, '') ##remove the parent dir path
for filename in filenames:
if os.path.splitext(filename)[1] not in skip_postfix:
if skip_keyword:
for skip_key in skip_keyword:
if not re.search(skip_key, filename.lower()):
zip_handle.write(os.path.join(dirpath, filename), fpath + '/' + filename)
else:
zip_handle.write(os.path.join(dirpath, filename), fpath + '/' + filename)
print('\033[1;32m[3]\033[0mCompress file [{}] \033[1;32msuccess\033[0m'.format(zip_filename))
zip_handle.close()
def MainProcess(self):
for production_dir in os.listdir(self.ScriptHome):
fileSeq = 1
fullpath_dir = self.ScriptHome + '/' + production_dir
print('\033[1;32m[1]\033[0mProcess production \033[1;32m[directory]\033[0m:{}'.format(fullpath_dir))
if not os.path.isdir(fullpath_dir):
print("\033[1;33m[Warning]Skip file\033[0m {}".format(fullpath_dir))
continue
user_file_dict = self.GetDictFromDir(fullpath_dir)
# print(user_file_dict,fullpath_dir,len(user_file_dict))
if not len(user_file_dict):
print("\033[1;33m[Warning]Skip product\033[0m[{}]".format(production_dir))
continue
config_json = []
for one_user_name in user_file_dict:
# print (one_user_name)
# print (user_file_dict[one_user_name])
tmp_config_json = {}
#调整etlsys数据库用户的执行的顺序在etl,etlbdp之前
if one_user_name.lower() == 'etlsys':
tmp_etl_seq=fileSeq
for oneseq in range(len(config_json)):
if config_json[oneseq]['username'] in ['etl','etlbdp']:
tmp_seq=config_json[oneseq]['fileSeq']
if tmp_seq < fileSeq:
config_json[oneseq]['fileSeq']=fileSeq
tmp_etl_seq=tmp_seq
tmp_config_json["fileSeq"] = tmp_etl_seq
else:
tmp_config_json["fileSeq"] = fileSeq
tmp_config_json["dataSourceType"] = 'oracle'
# print ('one_user_name=',one_user_name,type(one_user_name))
dbstring = self.GetDnsString(one_user_name)
if not dbstring:
print("dbconnect is incorrect!")
exit()
tmp_config_json["url"] = dbstring
##根据需要修改密码，默认密码和用户名相同
tmp_config_json["username"] = one_user_name
if self.DbConDnsInfo['password']:
tmp_config_json["password"] = self.DbConDnsInfo['password']
else:
tmp_config_json["password"] = one_user_name
# print (user_file_dict[one_user_name])
sqlFileList = self.SortFileList(user_file_dict[one_user_name])
tmp_config_json["sqlFileList"] = sqlFileList
config_json.append(tmp_config_json)
fileSeq += 1
## write json string to file
json_file_path = self.ScriptHome + '/' + production_dir + '/sql/config.json'
self.WriteJsonConfigFile(json_file_path, config_json)
##compress to sql.zip
compress_dir = self.ScriptHome + '/' + production_dir + '/sql'
self.ZipProductionSqlDir(compress_dir)
if __name__ == '__main__':
script_home = r'E:/Workspace/鲲鹏项目/一键部署server/临时版本/20180809-01-演示'
db_con_dns = {
'instance': {
'cc': ['apig', 'ftf', 'cic', 'cpc', 'cpc_flow', 'custc', 'oc', 'drm', 'ebc', 'pos', 'sa', 'src'],
'pmt': ['ab', 'bc', 'pcc'],
'rb': ['pcb', 'inv', 'rb'],
'sett': ['med', 'sett'],
'stbp': ['stbp', 'etl', 'dap']},
'dburl': {'cc': ['172.16.80.11', '11521'],
'pmt': ['172.16.80.12', '11521'],
'rb': ['172.16.80.13', '11521'],
'sett': ['172.16.80.14', '11521'],
'stbp': ['172.16.80.15', '11521']},
'password': ''
}
gen = GenJson4Ora(script_home, db_con_dns)
gen.MainProcess()