Python dataformat.py通用數據格式轉化腳本

需求：在進行Hadoop測試時，需要造大量數據，例如某個表存在56列，但實際程序邏輯只適用到某幾列，我們造的數據 也只需要某幾列

構造幾列數據，轉化為對應數據表格式

涉及模塊：os,getopt,sys

輸入：源格式，文本文件

輸出：目標格式，文本文件

1. #!/usr/bin/python
2. # -*- coding: utf-8 -*-
3. #dataformat.py
4. #this script change data from your source to the dest data format
5. #2011-08-05 created version0.1
6. #2011-10-29 add row-row mapping ,default row value .rebuild all functions. version0.2
7. #next:add data auto generate by re expression
9. import os,getopt,sys
11. #讀入文件,返回所有行
12. def read_file(path):
13. f = open(path, "r")
14. lines = f.readlines()
15. f.close()
16. return lines
18. #處理一行，轉為目標格式，返回目標行
19. def one_line_proc(parts, total, ft_map, outsp, empty_fill):
20. toindex = 0
21. outline = ""
22. keys = ft_map.keys()
23. for i in range(1, total+1):
24. if i in keys:
25. fill_index = ft_map[i]
26. if fill_index.startswith("d"):
27. outline += fill_index[1:]
28. else:
29. outline += parts[int(fill_index)-1]
30. else:
31. outline += empty_fill
32. if i !=total:
33. outline += outsp
34. #TODO:加入使用默認值列 若是以d開頭，後面是默認，否則取文件對應列 done
35. #TODO:這裡根據這個判斷長度也需要換掉 done
36. return outline
38. #處理入口，讀文件，循環處理每一行，寫出
39. #輸入數據分隔符默認\t,輸出數據默認分隔符\t
40. def process(inpath, total, to, outpath, insp="\t", outsp="\t", empty_fill=""):
41. #TODO:這裡將to轉為映射格式 done
42. ft_map = {}
43. in_count = 0
44. used_row = []
45. for to_row in to:
46. if r"\:" not in to_row and len(to_row.split(":"))==2:
47. used_row.append(int(to_row.split(":")[1]))
48. if r"\=" not in str(to_row) and len(str(to_row).split("="))==2:
49. pass
50. else:
51. in_count += 1
53. for to_row in to:
54. if r"\=" not in str(to_row) and len(str(to_row).split("="))==2:
55. ft_map.update({int(to_row.split("=")[0]):"d"+to_row.split("=")[1]})
56. continue
57. elif r"\:" not in to_row and len(to_row.split(":"))==2:
58. ft_map.update({int(to_row.split(":")[0]):to_row.split(":")[1]})
59. continue
60. else:
61. to_index = 0
62. for i in range(1, 100):
63. if i not in used_row:
64. to_index = i
65. break
66. ft_map.update({int(to_row):str(to_index)})
67. used_row.append(to_index)
69. lines = read_file(inpath)
70. f = open(outpath,"w")
71. result=[]
72. for line in lines:
73. parts = line.strip("\n").split(insp)
74. #TODO:這裡判斷長度必須換掉 done
75. if len(parts) >= in_count:
76. outline = one_line_proc(parts, total, ft_map, outsp, empty_fill)
77. result.append(outline+"\n")
78. f.writelines(result)
79. f.close()
81. #打印幫助信息
82. def help_msg():
83. print("功能：原數據文件轉為目標數據格式")
84. print("選項:")
85. print("\t -i inputfilepath [必輸，原文件路徑]")
86. print("\t -t n [必輸，n為數字，目標數據總的域個數]")
87. print("\t -a '1,3,4' [必輸，域編號字符串，逗號分隔。指定域用原數據字段填充，未指定用'0'填充]")
88. print("\t -o outputfilepath [可選，默認為 inputfilepath.dist ]")
89. print("\t -F 'FS' [可選，原文件域分隔符，默認為\\t ]")
90. print("\t -P 'OFS' [可選，輸出文件的域分隔符，默認為\\t ]")
91. sys.exit(0)
93. #程序入口，讀入參數，執行
94. def main():
95. try:
96. opts,args = getopt.getopt(sys.argv[1:],"F:P:t:a:i:o:f:h")
98. for op,value in opts:
99. if op in ("-h","-H","--help"):
100. help_msg()
101. if op == "-i":
102. inpath = value
103. elif op == "-o":
104. outpath = value
105. elif op == "-t":
106. total = int(value)
107. elif op == "-a":
108. to = value.split(",")
109. elif op == "-F":
110. insp = value.decode("string_escape")
111. elif op == "-P":
112. outsp = value.decode("string_escape")
113. elif op == "-f":
114. empty_fill = value
115. #考慮下這邊放在神馬地方合適
116. if len(opts) < 3:
117. print(sys.argv[0]+" : the amount of params must great equal than 3")
118. sys.exit(1)
120. except getopt.GetoptError:
121. print(sys.argv[0]+" : params are not defined well!")
123. if 'inpath' not in dir():
124. print(sys.argv[0]+" : -i param is needed,input file path must define!")
125. sys.exit(1)
127. if 'total' not in dir():
128. print(sys.argv[0]+" : -t param is needed,the fields of result file must define!")
129. sys.exit(1)
131. if 'to' not in dir():
132. print(sys.argv[0]+" : -a param is needed,must assign the field to put !")
133. sys.exit(1)
135. if not os.path.exists(inpath):
136. print(sys.argv[0]+" file : %s is not exists"%inpath)
137. sys.exit(1)
139. if 'empty_fill' not in dir():
140. empty_fill = ''
142. tmp=[]
143. for st in to:
144. tmp.append(str(st))
145. to=tmp
147. if 'outpath' not in dir():
148. outpath = inpath+".dist"
150. if 'insp' in dir() and 'outsp' in dir():
151. process(inpath,total,to,outpath,insp,outsp,empty_fill=empty_fill)
152. elif 'insp' in dir():
153. process(inpath,total,to,outpath,insp,empty_fill=empty_fill)
154. elif 'outsp' in dir():
155. process(inpath,total,to,outpath,outsp=outsp,empty_fill=empty_fill)
156. else:
157. process(inpath,total,to,outpath,empty_fill=empty_fill)
159. if __name__ =="__main__":
160. main()