1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20 import os.path, sys, re, string, copy, difflib, logging
21
22 from pysys import log
23 from pysys.constants import *
24 from pysys.exceptions import *
25
26
27 -def trimContents(contents, expressions, exclude=True):
28 """Reduce a list of strings based by including/excluding lines which match any of a set of regular expressions, returning the processed list.
29
30 The method reduces an input list of strings based on whether each string matches or does not match a
31 list of regular expressions.
32
33 @param contents: The input list of strings to trim based on matches to regular expressions
34 @param expressions: The input list of regular expressions
35 @param exclude: If true matches to the regular expressions exclude the line, if false matches include the line
36 @return: The processed list
37 @rtype: list
38 """
39 if len(expressions) == 0:
40 return contents
41
42 regexp = []
43 for i in range(0, len(expressions)):
44 regexp.append(re.compile(expressions[i]))
45
46 list = copy.deepcopy(contents)
47 for i in range(0, len(contents)):
48 for j in range(0, len(regexp)):
49 if (exclude and regexp[j].search(contents[i]) is not None) or (not exclude and regexp[j].search(contents[i]) is None):
50 list.remove(contents[i])
51 break
52
53 return list
54
55
56
58 """Replace all occurrences of keyword values in a list of strings with a set value, returning the processed list.
59
60 The replacementList parameter should contain a list of tuples to use in the replacement, e.g.
61 [('foo', 'bar'), ('swim', 'swam')].
62
63 @param list: The input list of strings to performed the replacement on
64 @param replacementList: A list of tuples (key, value) where matches to key are replaced with value
65 @return: The processed list
66 @rtype: list
67
68 """
69 for pair in replacementList:
70 regexp = re.compile(pair[0])
71 for j in range(0, len(list)):
72 list[j] = re.sub(regexp, pair[1], list[j])
73
74 return list
75
76
77
78 -def logContents(message, list):
79 """Log a list of strings, prepending the line number to each line in the log output.
80
81 @param list: The list of strings to log
82 """
83 count = 0
84 log.debug(message)
85 for line in list:
86 count = count + 1
87 log.debug(" Line %-5d: %s" % (count, line))
88
89
90
91 -def filediff(file1, file2, ignore=[], sort=True, replacementList=[], include=[], unifiedDiffOutput=None):
92 """Perform a file comparison between two (preprocessed) input files, returning true if the files are equivalent.
93
94 The method reads in the files and loads the contents of each as a list of strings. The two files are
95 said to be equal if the two lists are equal. The method allows for preprocessing of the string lists
96 to trim down their contents prior to the comparison being performed. Preprocessing is either to remove
97 entries from the lists which match any entry in a set of regular expressions, include only lines which
98 match any entry in a set of regular expressions, replace certain keywords in the string values of each list
99 with a set value (e.g. to replace time stamps etc), or to sort the lists before the comparison (e.g. where
100 determinism may not exist). Verbose logging of the method occurs at DEBUG level showing the contents of the
101 processed lists prior to the comparison being performed.
102
103 @param file1: The full path to the first file to use in the comparison
104 @param file2: The full path to the second file to use in the comparison, typically a reference file
105 @param ignore: A list of regular expressions which remove entries in the input file contents before making the comparison
106 @param sort: Boolean to sort the input file contents before making the comparison
107 @param replacementList: A list of tuples (key, value) where matches to key are replaced with value in the input file contents before making the comparison
108 @param include: A list of regular expressions used to select lines from the input file contents to use in the comparison
109 @param unifiedDiffOutput: If specified, indicates the full path of a file to which unified diff output will be written,
110 if the diff fails.
111 @return: success (True / False)
112 @rtype: boolean
113 @raises FileNotFoundException: Raised if either of the files do not exist
114
115 """
116 for file in file1, file2:
117 if not os.path.exists(file):
118 raise FileNotFoundException, "unable to find file %s" % (os.path.basename(file))
119 else:
120 list1 = []
121 list2 = []
122
123 with open(file1, 'r') as f:
124 for i in f: list1.append(i.strip())
125
126 with open(file2, 'r') as f:
127 for i in f: list2.append(i.strip())
128
129 list1 = trimContents(list1, ignore, exclude=True)
130 list2 = trimContents(list2, ignore, exclude=True)
131 list1 = trimContents(list1, include, exclude=False)
132 list2 = trimContents(list2, include, exclude=False)
133 list1 = replace(list1, replacementList)
134 list2 = replace(list2, replacementList)
135 if sort:
136 list1.sort()
137 list2.sort()
138
139 logContents("Contents of %s after pre-processing;" % os.path.basename(file1), list1)
140 logContents("Contents of %s after pre-processing;" % os.path.basename(file2), list2)
141
142 if list1 != list2:
143 log.debug("Unified diff between pre-processed input files;")
144 l1 = []
145 l2 = []
146 for i in list1: l1.append("%s\n"%i)
147 for i in list2: l2.append("%s\n"%i)
148
149
150 diff = ''.join(difflib.unified_diff(l2, l1,
151 fromfile='%s (%d lines)'%(os.path.basename(file2), len(l2)),
152 tofile='%s (%d lines)'%(os.path.basename(file1), len(l1)),
153 ))
154 if unifiedDiffOutput:
155 with open(unifiedDiffOutput, 'w') as f:
156 f.write(diff)
157 for line in diff.split('\n'): log.debug(" %s", line)
158
159 if list1 == list2: return True
160 return False
161
162
163
164
165 if __name__ == "__main__":
166 if len(sys.argv) < 3:
167 print "Usage: filediff.py <file1> <file2> [regexpr1 [regexp2]...]"
168 sys.exit()
169 else:
170 ignore = []
171 for i in range(3,len(sys.argv)):
172 ignore.append(sys.argv[i])
173
174 try:
175 status = filediff(sys.argv[1], sys.argv[2], ignore)
176 except FileNotFoundException, value:
177 print "caught %s: %s" % (sys.exc_info()[0], value)
178 print "unable to diff files... exiting"
179 else:
180 if status == True:
181 print "No differences detected"
182 else:
183 print "Differences detected"
184