Package pysys :: Package utils :: Module filediff
[hide private]
[frames] | no frames]

Source Code for Module pysys.utils.filediff

  1  #!/usr/bin/env python 
  2  # PySys System Test Framework, Copyright (C) 2006-2016  M.B.Grieve 
  3   
  4  # This library is free software; you can redistribute it and/or 
  5  # modify it under the terms of the GNU Lesser General Public 
  6  # License as published by the Free Software Foundation; either 
  7  # version 2.1 of the License, or (at your option) any later version. 
  8   
  9  # This library is distributed in the hope that it will be useful, 
 10  # but WITHOUT ANY WARRANTY; without even the implied warranty of 
 11  # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU 
 12  # Lesser General Public License for more details. 
 13   
 14  # You should have received a copy of the GNU Lesser General Public 
 15  # License along with this library; if not, write to the Free Software 
 16  # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA 
 17   
 18  # Contact: moraygrieve@users.sourceforge.net 
 19   
 20  import os.path, sys, re, string, copy, difflib, logging 
 21   
 22  from pysys import log 
 23  from pysys.constants import * 
 24  from pysys.exceptions import * 
 25   
 26   
27 -def trimContents(contents, expressions, exclude=True):
28 """Reduce a list of strings based by including/excluding lines which match any of a set of regular expressions, returning the processed list. 29 30 The method reduces an input list of strings based on whether each string matches or does not match a 31 list of regular expressions. 32 33 @param contents: The input list of strings to trim based on matches to regular expressions 34 @param expressions: The input list of regular expressions 35 @param exclude: If true matches to the regular expressions exclude the line, if false matches include the line 36 @return: The processed list 37 @rtype: list 38 """ 39 if len(expressions) == 0: 40 return contents 41 42 regexp = [] 43 for i in range(0, len(expressions)): 44 regexp.append(re.compile(expressions[i])) 45 46 list = copy.deepcopy(contents) 47 for i in range(0, len(contents)): 48 for j in range(0, len(regexp)): 49 if (exclude and regexp[j].search(contents[i]) is not None) or (not exclude and regexp[j].search(contents[i]) is None): 50 list.remove(contents[i]) 51 break 52 53 return list
54 55 56
57 -def replace(list, replacementList):
58 """Replace all occurrences of keyword values in a list of strings with a set value, returning the processed list. 59 60 The replacementList parameter should contain a list of tuples to use in the replacement, e.g. 61 [('foo', 'bar'), ('swim', 'swam')]. 62 63 @param list: The input list of strings to performed the replacement on 64 @param replacementList: A list of tuples (key, value) where matches to key are replaced with value 65 @return: The processed list 66 @rtype: list 67 68 """ 69 for pair in replacementList: 70 regexp = re.compile(pair[0]) 71 for j in range(0, len(list)): 72 list[j] = re.sub(regexp, pair[1], list[j]) 73 74 return list
75 76 77
78 -def logContents(message, list):
79 """Log a list of strings, prepending the line number to each line in the log output. 80 81 @param list: The list of strings to log 82 """ 83 count = 0 84 log.debug(message) 85 for line in list: 86 count = count + 1 87 log.debug(" Line %-5d: %s" % (count, line))
88 89 90
91 -def filediff(file1, file2, ignore=[], sort=True, replacementList=[], include=[], unifiedDiffOutput=None):
92 """Perform a file comparison between two (preprocessed) input files, returning true if the files are equivalent. 93 94 The method reads in the files and loads the contents of each as a list of strings. The two files are 95 said to be equal if the two lists are equal. The method allows for preprocessing of the string lists 96 to trim down their contents prior to the comparison being performed. Preprocessing is either to remove 97 entries from the lists which match any entry in a set of regular expressions, include only lines which 98 match any entry in a set of regular expressions, replace certain keywords in the string values of each list 99 with a set value (e.g. to replace time stamps etc), or to sort the lists before the comparison (e.g. where 100 determinism may not exist). Verbose logging of the method occurs at DEBUG level showing the contents of the 101 processed lists prior to the comparison being performed. 102 103 @param file1: The full path to the first file to use in the comparison 104 @param file2: The full path to the second file to use in the comparison, typically a reference file 105 @param ignore: A list of regular expressions which remove entries in the input file contents before making the comparison 106 @param sort: Boolean to sort the input file contents before making the comparison 107 @param replacementList: A list of tuples (key, value) where matches to key are replaced with value in the input file contents before making the comparison 108 @param include: A list of regular expressions used to select lines from the input file contents to use in the comparison 109 @param unifiedDiffOutput: If specified, indicates the full path of a file to which unified diff output will be written, 110 if the diff fails. 111 @return: success (True / False) 112 @rtype: boolean 113 @raises FileNotFoundException: Raised if either of the files do not exist 114 115 """ 116 for file in file1, file2: 117 if not os.path.exists(file): 118 raise FileNotFoundException, "unable to find file %s" % (os.path.basename(file)) 119 else: 120 list1 = [] 121 list2 = [] 122 123 with open(file1, 'r') as f: 124 for i in f: list1.append(i.strip()) 125 126 with open(file2, 'r') as f: 127 for i in f: list2.append(i.strip()) 128 129 list1 = trimContents(list1, ignore, exclude=True) 130 list2 = trimContents(list2, ignore, exclude=True) 131 list1 = trimContents(list1, include, exclude=False) 132 list2 = trimContents(list2, include, exclude=False) 133 list1 = replace(list1, replacementList) 134 list2 = replace(list2, replacementList) 135 if sort: 136 list1.sort() 137 list2.sort() 138 139 logContents("Contents of %s after pre-processing;" % os.path.basename(file1), list1) 140 logContents("Contents of %s after pre-processing;" % os.path.basename(file2), list2) 141 142 if list1 != list2: 143 log.debug("Unified diff between pre-processed input files;") 144 l1 = [] 145 l2 = [] 146 for i in list1: l1.append("%s\n"%i) 147 for i in list2: l2.append("%s\n"%i) 148 149 # nb: have to switch 1 and 2 around to get the right diff for a typical output,ref file pair 150 diff = ''.join(difflib.unified_diff(l2, l1, 151 fromfile='%s (%d lines)'%(os.path.basename(file2), len(l2)), 152 tofile='%s (%d lines)'%(os.path.basename(file1), len(l1)), 153 )) 154 if unifiedDiffOutput: 155 with open(unifiedDiffOutput, 'w') as f: 156 f.write(diff) 157 for line in diff.split('\n'): log.debug(" %s", line) 158 159 if list1 == list2: return True 160 return False
161 162 163 164 # entry point for running the script as an executable 165 if __name__ == "__main__": 166 if len(sys.argv) < 3: 167 print "Usage: filediff.py <file1> <file2> [regexpr1 [regexp2]...]" 168 sys.exit() 169 else: 170 ignore = [] 171 for i in range(3,len(sys.argv)): 172 ignore.append(sys.argv[i]) 173 174 try: 175 status = filediff(sys.argv[1], sys.argv[2], ignore) 176 except FileNotFoundException, value: 177 print "caught %s: %s" % (sys.exc_info()[0], value) 178 print "unable to diff files... exiting" 179 else: 180 if status == True: 181 print "No differences detected" 182 else: 183 print "Differences detected" 184