Source code for pysys.config.project

#!/usr/bin/env python
# PySys System Test Framework, Copyright (C) 2006-2022 M.B. Grieve

# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2.1 of the License, or (at your option) any later version.

# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Lesser General Public License for more details.

# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA

"""
The `Project <pysys.config.project.Project>` class holds the ``pysysproject.xml`` project configuration, including all 
user-defined project properties. 

"""

__all__ = ['Project'] # Project is the only member we expose/document from this module

import os.path, logging, xml.dom.minidom, collections, codecs, time
import platform
import locale
import getpass

import pysys
import pysys.utils.misc
from pysys.constants import *
from pysys import __version__
from importlib import import_module
from pysys.utils.logutils import ColorLogFormatter, BaseLogFormatter
from pysys.utils.fileutils import mkdir, loadProperties, toLongPathSafe
from pysys.utils.pycompat import openfile, makeReadOnlyDict
from pysys.exceptions import UserError

log = logging.getLogger('pysys.config.project')

class _XMLProjectParser(object):
	"""
	:meta private: Not public API. 
	"""
	def __init__(self, dirname, file, outdir):
		self.dirname = toLongPathSafe(dirname, onlyIfNeeded=True) # main reason for this is to capitalize (normalize) drive letter on windows to match other toLongPathSafe invocations
		self.xmlfile = os.path.join(dirname, file)
		log.debug('Loading project file: %s', self.xmlfile)
		self.environment = 'env'
		
		# project load time is a reasonable proxy for test start time, 
		# and we might want to substitute the date/time into property values
		self.startTimestamp = time.time()
		
		try:
			username = os.getenv('PYSYS_USERNAME') or getpass.getuser().lower() # getpass throws if no env var is set to help with this
		except Exception as ex:
			username = 'UNKNOWN_USER'
		
		self.properties = {
			'testRootDir':self.dirname,
			
			'outDirName':os.path.basename(outdir),
			
			'startDate':time.strftime('%Y-%m-%d', time.localtime(self.startTimestamp)),
			'startTime':time.strftime('%H.%M.%S', time.localtime(self.startTimestamp)),
			'startTimeSecs':'%0.3f'%self.startTimestamp,

			'hostname':HOSTNAME.lower().split('.')[0],
			'os':platform.system().lower(), # e.g. 'windows', 'linux', 'darwin'; a more modern alternative to OSFAMILY
			'osfamily':OSFAMILY, # windows or unix
			
			'pysysTemplatesDir': os.path.dirname(__file__)+os.sep+'templates',
			'username': username,
			
			'/': os.sep, # so people can write strings like foo${/}bar and get a forward or back-slash depending on platform

			# old names
			'root':self.dirname, # old name for testRootDir
		}
		
		if not os.path.exists(self.xmlfile):
			raise Exception("Unable to find supplied project file \"%s\"" % self.xmlfile)
		
		try:
			self.doc = xml.dom.minidom.parse(self.xmlfile)
		except Exception:
			raise Exception(sys.exc_info()[1])
		else:
			if self.doc.getElementsByTagName('pysysproject') == []:
				raise Exception("No <pysysproject> element supplied in project file")
			else:
				self.root = self.doc.getElementsByTagName('pysysproject')[0]
		
		extraProjectXMLs = os.getenv('PYSYS_PROJECT_APPEND', '')
		if extraProjectXMLs:
			for extraXMLFile in extraProjectXMLs.split(','):
				extraXMLFile = extraXMLFile.strip()
				log.info('Loading additional project file: %s', extraXMLFile)
				extra = xml.dom.minidom.parse(extraXMLFile).getElementsByTagName('pysysproject')
				assert extra, 'Cannot find <pysysproject> in %s'%extraXMLFile

				for extraNode in list(extra[0].childNodes):
					self.root.appendChild(extraNode)

		self.__pathProperties = set() # set set of properties which are known to contain a path

	def checkVersions(self):
		requirespython = self.root.getElementsByTagName('requires-python')
		if requirespython and requirespython[0].firstChild: 
			requirespython = requirespython[0].firstChild.nodeValue
			if requirespython:
				if list(sys.version_info) < list(map(int, requirespython.split('.'))):
					raise UserError('This test project requires Python version %s or greater, but this is version %s (from %s)'%(requirespython, '.'.join([str(x) for x in sys.version_info[:3]]), sys.executable))

		requirespysys = self.root.getElementsByTagName('requires-pysys')
		if requirespysys and requirespysys[0].firstChild: 
			requirespysys = requirespysys[0].firstChild.nodeValue
			if requirespysys:
				thisversion = __version__
				if pysys.utils.misc.compareVersions(requirespysys, thisversion) > 0:
					raise UserError('This test project requires PySys version %s or greater, but this is version %s'%(requirespysys, thisversion))


	def unlink(self):
		if self.doc: self.doc.unlink()	


	def getProperties(self):
		propertyNodeList = [element for element in self.root.getElementsByTagName('property') if element.parentNode == self.root]

		for propertyNode in propertyNodeList:
			permittedAttributes = None
			# use of these options for customizing the property names of env/root/osfamily is no longer encouraged; just kept for compat
			if propertyNode.hasAttribute("environment"):
				self.environment = propertyNode.getAttribute("environment")
			elif propertyNode.hasAttribute("root"): 
				propname = propertyNode.getAttribute("root")
				self.properties[propname] = self.dirname
				log.debug('Setting project property %s="%s"', propname, self.dirname)
			elif propertyNode.hasAttribute("osfamily"): # just for older configs, better to use ${os} now
				propname = propertyNode.getAttribute("osfamily")
				self.properties[propname] = OSFAMILY
				log.debug('Setting project property %s="%s"', propname, OSFAMILY)
					
			elif propertyNode.hasAttribute("file"): 
				file = self.expandProperties(propertyNode.getAttribute("file"), default=propertyNode, name='properties file reading')
				self.getPropertiesFromFile(os.path.normpath(os.path.join(self.dirname, file)) if file else '', 
					pathMustExist=(propertyNode.getAttribute("pathMustExist") or '').lower()=='true',
					includes=propertyNode.getAttribute("includes"),
					excludes=propertyNode.getAttribute("excludes"),
					prefix=propertyNode.getAttribute("prefix") or '',
					)
				permittedAttributes = {'name', 'file', 'default', 'pathMustExist', 'includes', 'excludes', 'prefix'}

			elif propertyNode.hasAttribute("name"):
				name = propertyNode.getAttribute("name") 
				value = self.expandProperties(
						propertyNode.getAttribute("value") or propertyNode.getAttribute("path")
						or '\n'.join(n.data for n in propertyNode.childNodes 
							if (n.nodeType in {n.TEXT_NODE,n.CDATA_SECTION_NODE}) and n.data), 
					default=propertyNode, name=name)
				if name in self.properties:
					raise UserError('Cannot set project property "%s" as it is already set'%name)

				ispath = False
				if (propertyNode.getAttribute("pathMustExist") or '').lower()=='true':
					if not (value and os.path.exists(os.path.join(self.dirname, value))):
						raise UserError('Cannot find path referenced in project property "%s": "%s"'%(
							name, '' if not value else os.path.normpath(os.path.join(self.dirname, value))))
					ispath = True
				elif propertyNode.getAttribute("path"): ispath = True
				
				if ispath: # since we know it's a path, make it a nice one
					value = os.path.normpath(value)
					self.__pathProperties.add(name)

				self.properties[name] = value
				log.debug('Setting project property %s="%s"', name, value)

				permittedAttributes = {'name', 'value', 'path', 'default', 'pathMustExist'}
			else:
				raise UserError('Found <property> with no name= or file=')
			
			if permittedAttributes is not None:
				for att in range(propertyNode.attributes.length):
					attName = propertyNode.attributes.item(att).name
					if attName not in permittedAttributes: 
						# not an error, to allow for adding new ones in future pysys versions, but worth warning about
						log.warning('Unknown <property> attribute "%s" in project configuration'%attName)


		# default is to support IDEs like VSCode
		self.properties.setdefault('pysysLogAbsolutePaths', str( os.getenv('TERM_PROGRAM','')=='vscode' ) )
		# always allow local env var override regardless of project property
		if os.getenv('PYSYS_LOG_ABSOLUTE_PATHS'): self.properties['pysysLogAbsolutePaths'] = os.getenv('PYSYS_LOG_ABSOLUTE_PATHS','').lower()=='true'

		return self.properties



	def getPropertiesFromFile(self, file, pathMustExist=False, includes=None, excludes=None, prefix=''):
		if not os.path.isfile(file):
			if pathMustExist:
				raise UserError('Cannot find properties file referenced in %s: "%s"'%(
					self.xmlfile, file))

			log.debug('Skipping project properties file which not exist: "%s"', file)
			return

		try:
			rawProps = loadProperties(file) # since PySys 1.6.0 this is UTF-8 by default
		except UnicodeDecodeError:
			# fall back to ISO8859-1 if not valid UTF-8 (matching Java 9+ behaviour)
			rawProps = loadProperties(file, encoding='iso8859-1')
		
		props = collections.OrderedDict()
		for name, value in rawProps.items():
			if includes and not re.match(includes, name): continue
			if excludes and re.match(excludes, name): continue
			props[prefix+name] = value
				
		for name, value in props.items():
			# when loading properties files it's not so helpful to give errors (and there's nowhere else to put an empty value) so default to empty string
			value = self.expandProperties(value, default='', name=name)	
			
			if name in self.__pathProperties:
				value = os.path.normpath(value)
			
			if name in self.properties and value != self.properties[name]:
				# Whereas we want a hard error for duplicate <property name=".../> entries, for properties files 
				# there's a good case to allow overwriting of properties, but log it at INFO
				if name in self.__pathProperties and os.path.normcase(value)==os.path.normcase(self.properties[name]):
					pass
				else:
					log.info('Overwriting previous value of project property "%s" with new value "%s" from "%s"'%(name, value, os.path.basename(file)))

			self.properties[name] = value
			log.debug('Setting project property %s="%s" (from %s)', name, self.properties[name], file)

	def expandProperties(self, value, default, name=None):
		"""
		Expand any ${...} project properties or env vars, with ${$} for escaping.
		The "default" is expanded and used if value contains some undefined variables. 
		If default=None then an error is raised instead. If default is a node, its "default" attribute is used
		
		The "name" is used to generate more informative error messages
		"""
		envprefix = self.environment+'.'
		errorprefix = ('Error setting project property "%s": '%name) if name else ''
		
		if hasattr(default, 'getAttribute'):
			default = default.getAttribute("default") if default.hasAttribute("default") else None

		def expandProperty(m):
			m = m.group(1)
			if m == '$': return '$'
			try:
				if m.startswith(envprefix): 
					return os.environ[m[len(envprefix):]]
				if m.startswith('env:'): # for consistency with eval: also support this syntax
					return os.environ[m[4:]]
			except KeyError as ex:
				raise KeyError(errorprefix+'cannot find environment variable "%s"'%m[len(envprefix):])
			
			if m.startswith('eval:'):
				props = dict(self.properties)
				props.pop('os', None) # remove this to avoid hiding the os.path module
				props['properties'] = self.properties
				try:
					v = pysys.utils.safeeval.safeEval(m[5:], extraNamespace=props, 
						errorMessage='Failed to evaluate Python eval() string "{expr}" during property expansion due to {error}')
					return str(v)
				except Exception as ex:
					raise UserError(str(ex))

			if m in self.properties:
				return self.properties[m]
			else:
				raise KeyError(errorprefix+'PySys project property ${%s} is not defined, please check your pysysproject.xml file'%m)
		try:
			return re.sub(r'[$][{]([^}]+)[}]', expandProperty, value)
		except KeyError as ex:
			if default is None: raise UserError('%s; if this is intended to be an optional property please add a default="..." value'%ex)
			log.debug('Failed to resolve value "%s" of property "%s", so falling back to default value', value, name or '<unknown>')
			return re.sub(r'[$][{]([^}]+)[}]', expandProperty, default)

	def getRunnerDetails(self):
		nodes = self.root.getElementsByTagName('runner')
		if not nodes: return DEFAULT_RUNNER
		classname, propertiesdict = self._parseClassAndConfigDict(nodes[0], None, returnClassAsName=True)
		assert not propertiesdict, 'Properties are not supported under <runner>'
		return classname

	def getCollectTestOutputDetails(self):
		r = []
		for n in self.root.getElementsByTagName('collect-test-output'):
			x = {
				'pattern':n.getAttribute('pattern'),
				'outputDir':self.expandProperties(n.getAttribute('outputDir'), default=None, name='collect-test-output outputDir'),
				'outputPattern':n.getAttribute('outputPattern'),
			}
			assert 'pattern' in x, x
			assert 'outputDir' in x, x
			assert 'outputPattern' in x, x
			assert '@UNIQUE@' in x['outputPattern'], 'collect-test-output outputPattern must include @UNIQUE@'
			r.append(x)
		return r


	def getPerformanceReporterDetails(self):
		nodeList = self.root.getElementsByTagName('performance-reporter')
		results = []
		for n in nodeList:
			cls, optionsDict = self._parseClassAndConfigDict(n, 'pysys.perf.reporters.CSVPerformanceReporter')
			optionsDict['summaryfile'] = self.expandProperties(optionsDict.get('summaryfile', ''), default=None, name='performance-reporter summaryfile')
			results.append( (cls, optionsDict) )
		
		if not results: # add the defaults
			results.append( self._parseClassAndConfigDict(None, 'pysys.perf.reporters.CSVPerformanceReporter') )
			results.append( self._parseClassAndConfigDict(None, 'pysys.perf.reporters.PrintSummaryPerformanceReporter') )
			
		return results

	def getProjectHelp(self):
		help = ''
		for e in self.root.getElementsByTagName('project-help'):
			for n in e.childNodes:
				if (n.nodeType in {e.TEXT_NODE,e.CDATA_SECTION_NODE}) and n.data:
					help += n.data
		return help

	def getDescriptorLoaderClass(self):
		nodeList = self.root.getElementsByTagName('descriptor-loader')
		cls, optionsDict = self._parseClassAndConfigDict(nodeList[0] if nodeList else None, 'pysys.config.descriptor.DescriptorLoader')
		
		if optionsDict: raise UserError('Unexpected descriptor-loader attribute(s): '+', '.join(list(optionsDict.keys())))
		
		return cls

	def getTestPlugins(self):
		plugins = []
		for node in self.root.getElementsByTagName('test-plugin'):
			cls, optionsDict = self._parseClassAndConfigDict(node, None)
			alias = optionsDict.pop('alias', None)
			plugins.append( (cls, alias, optionsDict) )
		return plugins
		
	def getRunnerPlugins(self):
		plugins = []
		for node in self.root.getElementsByTagName('runner-plugin'):
			cls, optionsDict = self._parseClassAndConfigDict(node, None)
			alias = optionsDict.pop('alias', None)
			plugins.append( (cls, alias, optionsDict) )
		return plugins

	def getDescriptorLoaderPlugins(self):
		for node in self.root.getElementsByTagName('descriptor-loader-plugin'):
			if node.parentNode == self.root:
				# Eventually will be an exception but for now we just log a warning to allow the same project to work with both new and old versions simultaneously for easy migration
				# raise UserError(...)
				log.warning('From PySys version 2.2, <descriptor-loader-plugin> is no longer supported at the project level - please move it to <pysysdirconfig> instead')

	def getMakerDetails(self):
		nodes = self.root.getElementsByTagName('maker')
		if not nodes: return DEFAULT_MAKER
		classname, propertiesdict = self._parseClassAndConfigDict(nodes[0], None, returnClassAsName=True)
		assert not propertiesdict, 'Properties are not supported under <maker>'
		return classname
	
	def createFormatters(self):
		stdout = runlog = None
		
		formattersNodeList = self.root.getElementsByTagName('formatters')
		if formattersNodeList:
			formattersNodeList = formattersNodeList[0].getElementsByTagName('formatter')
		if formattersNodeList:
			for formatterNode in formattersNodeList:
				fname = formatterNode.getAttribute('name')
				if fname not in ['stdout', 'runlog']:
					raise UserError('Formatter "%s" is invalid - must be stdout or runlog'%fname)

				if fname == 'stdout':
					cls, options = self._parseClassAndConfigDict(formatterNode, 'pysys.utils.logutils.ColorLogFormatter')
					options['__formatterName'] = 'stdout'
					stdout = cls(options)
				else:
					cls, options = self._parseClassAndConfigDict(formatterNode, 'pysys.utils.logutils.BaseLogFormatter')
					options['__formatterName'] = 'runlog'
					runlog = cls(options)
		return stdout, runlog

	def getDefaultFileEncodings(self):
		result = []
		for n in self.root.getElementsByTagName('default-file-encoding'):
			pattern = (n.getAttribute('pattern') or '').strip().replace('\\','/')
			encoding = (n.getAttribute('encoding') or '').strip()
			if not pattern: raise UserError('<default-file-encoding> element must include both a pattern= attribute')
			if encoding: 
				codecs.lookup(encoding) # give an exception if an invalid encoding is specified
			else:
				encoding=None
			result.append({'pattern':pattern, 'encoding':encoding})
		return result

	def getExecutionOrderHints(self):
		result = []
		secondaryModesHintDelta = None
		
		def makeregex(s):
			if not s: return None
			if s.startswith('!'): raise UserError('Exclusions such as !xxx are not permitted in execution-order configuration')
			
			# make a regex that will match either the entire expression as a literal 
			# or the entire expression as a regex
			s = s.rstrip('$')
			try:
				#return re.compile('(%s|%s)$'%(re.escape(s), s))
				return re.compile('%s$'%(s))
			except Exception as ex:
				raise UserError('Invalid regular expression in execution-order "%s": %s'%(s, ex))
		
		for parent in self.root.getElementsByTagName('execution-order'):
			if parent.getAttribute('secondaryModesHintDelta'):
				secondaryModesHintDelta = float(parent.getAttribute('secondaryModesHintDelta'))
			for n in parent.getElementsByTagName('execution-order'):
				moderegex = makeregex(n.getAttribute('forMode'))
				groupregex = makeregex(n.getAttribute('forGroup'))
				if not (moderegex or groupregex): raise UserError('Must specify either forMode, forGroup or both')
				
				hintmatcher = lambda groups, mode, moderegex=moderegex, groupregex=groupregex: (
					(moderegex is None or moderegex.match(mode or '')) and
					(groupregex is None or any(groupregex.match(group) for group in groups))
					)
				
				result.append( 
					(float(n.getAttribute('hint')), hintmatcher )
					)
		if secondaryModesHintDelta is None: 
			secondaryModesHintDelta = +100.0 # default value
		return result, secondaryModesHintDelta

	def getWriterDetails(self):
		# writers can optionally be under a 'writers' parent node but this is now optional, to facilitate 
		# a third party plugin vendor providing a snippet of a few consecutive lines to paste into the project config 
		# to enable new functionality
		writers = []
		writerNodeList = self.root.getElementsByTagName('writer')
		if not writerNodeList: return []
		for writerNode in writerNodeList:
			pythonclassconstructor, propertiesdict = self._parseClassAndConfigDict(writerNode, None)
			writers.append( (pythonclassconstructor, propertiesdict) )
		return writers

	def addToPath(self):		
		for elementname in ['path', 'pythonpath']:
			pathNodeList = self.root.getElementsByTagName(elementname)

			for pathNode in pathNodeList:
					value = self.expandProperties(pathNode.getAttribute("value"), default=None, name='pythonpath')
					if not value: 
						raise UserError('Cannot add directory to the pythonpath: "%s"'%value)

					# we ignore the "relative" option and always make it relative to the testrootdir if not already absolute
					value = os.path.join(self.dirname, value)
					value = os.path.normpath(value)
					if not os.path.isdir(value): 
						raise UserError('Cannot add non-existent directory to the python <path>: "%s"'%value)
					else:
						log.debug('Adding value to path: %s', value)
						sys.path.append(value)


	def writeXml(self):
		f = open(self.xmlfile, 'w')
		f.write(self.doc.toxml())
		f.close()

	def _parseClassAndConfigDict(self, node, defaultClass, **kwargs):
		""" See `_parseClassAndConfigDictImpl` """
		return self._parseClassAndConfigDictImpl(self.expandProperties, node, defaultClass, **kwargs)

	@staticmethod
	def _parseClassAndConfigDictImpl(expandPropertiesImpl, node, defaultClass, returnClassAsName=False):
		"""Parses a dictionary of arbitrary options and a python class out of the specified XML node.

		This is a static method as it is used internally from DescriptorLoader. 

		The node may optionally contain classname and module (if not specified as a separate attribute,
		module will be extracted from the first part of classname); any other attributes will be returned in
		the optionsDict, as will <property name=""></property> child elements.

		:param node: The node, may be None
		:param defaultClass: a string specifying the name of the default fully-qualified class, if any
		:return: a tuple of (pythonclassconstructor, propertiesdict), or if returnClassAsName (classname, propertiesDict)
		"""
		optionsDict = {}
		if node:
			for att in range(node.attributes.length):
				name = node.attributes.item(att).name.strip()
				if name in optionsDict: raise UserError('Duplicate property "%s" in <%s> configuration'%(name, node.tagName))
				optionsDict[name] = expandPropertiesImpl(node.attributes.item(att).value, default=None, name=name)
			for tag in node.getElementsByTagName('property'):
				name = tag.getAttribute('name')
				assert name
				if name in optionsDict: raise UserError('Duplicate property "%s" in <%s> configuration'%(name, node.tagName))
				optionsDict[name] = expandPropertiesImpl(
					tag.getAttribute("value") or '\n'.join(n.data for n in tag.childNodes 
							if (n.nodeType in {n.TEXT_NODE,n.CDATA_SECTION_NODE}) and n.data),
					default=tag, name=name)
			
		classname = optionsDict.pop('classname', defaultClass)
		if not classname: raise UserError('Missing require attribute "classname=" for <%s>'%node.tagName)
		mod = optionsDict.pop('module', '.'.join(classname.split('.')[:-1]))
		classname = classname.split('.')[-1]

		if returnClassAsName:
			return (mod+'.'+classname).strip('.'), optionsDict

		# defer importing the module until we actually need to instantiate the 
		# class, to avoid introducing tricky module import order problems, given 
		# that the project itself needs loading very early
		def classConstructor(*args, **kwargs):
			module = import_module(mod)
			cls = getattr(module, classname)
			try:
				return cls(*args, **kwargs) # invoke the constructor for this class
			except Exception as ex:
				# The classname is not available in the higher-level exception handler, so useful to log it here in case it's not obvious
				log.error('Failed to instantiate "%s" class due to %s: %s', cls.__name__, type(ex).__name__, ex)
				raise
		return classConstructor, optionsDict

[docs]def getProjectConfigTemplates(): """Get a list of available templates that can be used for creating a new project configuration. :return: A dict, where each value is an absolute path to an XML template file and each key is the display name for that template. """ templatedir = os.path.dirname(__file__)+'/templates/project' templates = { t.replace('.xml',''): templatedir+'/'+t for t in os.listdir(templatedir) if t.endswith('.xml')} assert templates, 'No project templates found in %s'%templatedir return templates
[docs]def createProjectConfig(targetdir, templatepath=None): """Create a new project configuration file in the specified targetdir. """ if not templatepath: templatepath = getProjectConfigTemplates()['default'] mkdir(targetdir) # using ascii ensures we don't unintentionally add weird characters to the default (utf-8) file with openfile(templatepath, encoding='ascii') as src: with openfile(os.path.abspath(targetdir+'/'+DEFAULT_PROJECTFILE[0]), 'w', encoding='ascii') as target: for l in src: l = l.replace('@PYTHON_VERSION@', '%s.%s.%s'%sys.version_info[0:3]) l = l.replace('@PYSYS_VERSION@', '.'.join(__version__.split('.')[0:2])) target.write(l)
[docs]class Project(object): """Contains settings for the entire test project, as defined by the ``pysysproject.xml`` project configuration file; see :doc:`/pysys/ProjectConfiguration`. To get a reference to the current `Project` instance, use the `pysys.basetest.BaseTest.project` (or `pysys.process.user.ProcessUser.project`) field. All project properties are strings. If you need to get a project property value that's a a bool/int/float/list it is recommended to use `getProperty()` which will automatically perform the conversion. For string properties you can just use ``project.propName`` or ``project.properties['propName']``. :ivar dict(str,str) ~.properties: The resolved values of all project properties defined in the configuration file. In addition, each of these is set as an attribute onto the `Project` instance itself. :ivar str ~.root: Full path to the project root directory, as specified by the first PySys project file encountered when walking up the directory tree from the start directory. If no project file was found, this is just the start directory PySys was run from. :ivar str ~.projectFile: Full path to the project file. """ __INSTANCE = None __frozen = False def __init__(self, root, projectFile, outdir=None): assert projectFile self.root = root if not outdir: outdir = DEFAULT_OUTDIR if not os.path.exists(os.path.join(root, projectFile)): raise UserError("Project file not found: %s" % os.path.normpath(os.path.join(root, projectFile))) try: parser = _XMLProjectParser(root, projectFile, outdir=outdir) except UserError: raise except Exception as e: raise Exception("Error parsing project file \"%s\": %s" % (os.path.join(root, projectFile),sys.exc_info()[1])) parser.checkVersions() self.projectFile = os.path.join(root, projectFile) self.startTimestamp = parser.startTimestamp # get the properties properties = parser.getProperties() keys = list(properties.keys()) keys.sort() for key in keys: if not hasattr(self, key): # don't overwrite existing props; people will have to use .getProperty() to access them setattr(self, key, properties[key]) self.properties = dict(properties) # add to the python path parser.addToPath() # get the runner if specified self.runnerClassname = parser.getRunnerDetails() # get the maker if specified self.makerClassname = parser.getMakerDetails() self.writers = parser.getWriterDetails() self.testPlugins = parser.getTestPlugins() self.runnerPlugins = parser.getRunnerPlugins() parser.getDescriptorLoaderPlugins() # just to check there are none self.perfReporterConfig = parser.getPerformanceReporterDetails() self.descriptorLoaderClass = parser.getDescriptorLoaderClass() # get the stdout and runlog formatters stdoutformatter, runlogformatter = parser.createFormatters() self.defaultFileEncodings = parser.getDefaultFileEncodings() self.executionOrderHints, self.executionOrderSecondaryModesHintDelta = parser.getExecutionOrderHints() self.collectTestOutput = parser.getCollectTestOutputDetails() self.projectHelp = parser.getProjectHelp() self.projectHelp = parser.expandProperties(self.projectHelp, default=None, name='project-help') self._defaultDirConfig = None # this field is not public API e = parser.root.getElementsByTagName('pysysdirconfig') assert len(e) <= 1, 'Cannot have more than one pysysdirconfig element in pysysproject.xml' if e: self._defaultDirConfig = pysys.config.descriptor._XMLDescriptorParser.parse(self.projectFile, istest=False, project=self, xmlRootElement=e[0]) # set the data attributes parser.unlink() if not stdoutformatter: stdoutformatter = ColorLogFormatter({'__formatterName':'stdout'}) if not runlogformatter: runlogformatter = BaseLogFormatter({'__formatterName':'runlog'}) PySysFormatters = collections.namedtuple('PySysFormatters', ['stdout', 'runlog']) self.formatters = PySysFormatters(stdoutformatter, runlogformatter) # for safety (test independence, and thread-safety), make it hard for people to accidentally edit project properties later self.properties = makeReadOnlyDict(self.properties) self.__frozen = True def __setattr__(self, name, value): if self.__frozen: raise Exception('Project cannot be modified after it has been loaded (use the runner to store global state if needed)') object.__setattr__(self, name, value)
[docs] def expandProperties(self, value): """ Expand any ${...} project properties in the specified string. An exception is thrown if any property is missing. This method is only for expanding project properties and ``${eval:xxx})`` strings, so ``${env.*}`` syntax is not permitted (if you need to use an environment variable, put it into a project property first). .. versionadded:: 1.6.0 :param str value: The string in which any properties will be expanded. ${$} can be used for escaping a literal $ if needed. :return str: The value with properties expanded, or None if value=None. """ if (not value) or ('${' not in value): return value def expandProperty(m): m = m.group(1) if m == '$': return '$' if m.startswith('eval:'): props = dict(self.properties) props.pop('os', None) # remove this to avoid hiding the os.path module props['properties'] = self.properties try: v = pysys.utils.safeeval.safeEval(m[5:], extraNamespace=props, errorMessage='{error}') return str(v) except Exception as ex: raise Exception('Error resolving ${%s} eval() string: %s'%(m, ex)) return self.properties[m] try: return re.sub(r'[$][{]([^}]+)[}]', expandProperty, value) except KeyError as ex: # A more informative error, but not a UserError since we don't have the context of where it was called from raise Exception('Cannot resolve project property %s in: %s'%(ex, value))
[docs] def getProperty(self, key, default): """ Get the specified project property value, or a default if it is not defined, with type conversion from string to int/float/bool/list[str] (matching the default's type; for list[str], comma-separated input is assumed). .. versionadded:: 1.6.0 :param str key: The name of the property. :param bool/int/float/str/list[str] default: The default value to return if the property is not set. The type of the default parameter will be used to convert the property value from a string if it is provided. An exception will be raised if the value is non-empty but cannot be converted to the indicated type. """ return pysys.utils.misc.getTypedValueOrDefault(key, self.properties.get(key, None), default)
[docs] @staticmethod def getInstance(): """ Provides access to the singleton instance of Project. Raises an exception if the project has not yet been loaded. Use ``self.project`` to get access to the project instance where possible, for example from a `pysys.basetest.BaseTest` or `pysys.baserunner.BaseRunner` class. This attribute is for use in internal functions and classes that do not have a ``self.project``. """ if Project.__INSTANCE: return Project.__INSTANCE if 'doctest' in sys.argv[0]: return None # special-case for doctesting raise Exception('Cannot call Project.getInstance() as the project has not been loaded yet')
[docs] @staticmethod def findAndLoadProject(startdir=None, outdir=None): """Find and load a project file, starting from the specified directory. If this fails an error is logged and the process is terminated. The method walks up the directory tree from the supplied path until the PySys project file is found. The location of the project file defines the project root location. The contents of the project file determine project specific constants as specified by property elements in the xml project file. To ensure that all loaded modules have a pre-initialised projects instance, any launching application should first import the loadproject file, and then make a call to it prior to importing all names within the constants module. :param st rstartdir: The initial path to start from when trying to locate the project file :param str outdir: The output directory specified on the command line. Some project properties may depend on this. """ projectFile = os.getenv('PYSYS_PROJECTFILE', None) search = startdir or os.getcwd() if not projectFile: projectFileSet = set(DEFAULT_PROJECTFILE) drive, path = os.path.splitdrive(search) while (not search == drive): intersection = projectFileSet & set(os.listdir(search)) if intersection : projectFile = intersection.pop() break else: search, drop = os.path.split(search) if not drop: search = drive if not projectFile or not os.path.exists(os.path.join(search, projectFile)): # pragma: no cover if os.getenv('PYSYS_PERMIT_NO_PROJECTFILE','').lower()=='true': sys.stderr.write('FATAL ERROR: The PYSYS_PERMIT_NO_PROJECTFILE environment variable is no longer supported - you must create a pysysproject.xml file for your project') sys.exit(1) else: sys.stderr.write('\n'.join([ # | "WARNING: No PySys test project file exists in this directory (or its parents):", " - If you wish to start a new project, begin by running 'pysys makeproject'.", " - If you are trying to use an existing project, change directory to a ", " location under the directory that contains your project file.", "" ])) sys.exit(1) try: project = Project(search, projectFile, outdir=outdir) stdoutHandler.setFormatter(project.formatters.stdout) import pysys.constants pysys.constants.PROJECT = project # for compatibility for old tests Project.__INSTANCE = project # set singleton return project except UserError as e: sys.stderr.write("ERROR: Failed to load project - %s"%e) sys.exit(1) except Exception as e: sys.stderr.write("ERROR: Failed to load project due to %s - %s\n"%(e.__class__.__name__, e)) traceback.print_exc() sys.exit(1)
pysys.constants.Project = Project # for compatibility's sake' need to do this early import pysys.utils.safeeval # down here to break circular dependency import pysys.config.descriptor