0001
0002
0003
0004
0005
0006
0007
0008
0009"""tools for automatically triggering test suites.
0010
0011... for example, a continuous integeration test triggered
0012from a `subversion post-commit hook`_.
0013
0014Since repositories in real life contain several packages that most likely
0015will not be tested in the exact same way, there is no generic script
0016provided here to trigger test suites.
0017
0018Defining your own bgtest script is easy though.
0019Here is an example with custom packages ::
0020
0021 from testtools.bgtests import TestablePackage, run_packages
0022 class MyPackage(TestablePackage):
0023 svn_path = 'file://%s/widget/trunk'
0024 ex = 'nosetests -v'
0025 def recognizes_file(self, filepath, code=None):
0026 return filepath.startswith('widget/trunk')
0027
0028 run_packages( [MyPackage()], '/path/to/repos', notify='egg@yourface' )
0029
0030You can fire something like this off very nicely inside a `subversion post-commit hook`_
0031by passing through the repos_root and revision.
0032
0033The default test command is `nosetests`_, a discovery test runner for python,
0034but any command can be used here.
0035
0036If you want to run your tests on a machine separate from that which holds your
0037repository, then you can run a SimpleXMLRPCServer using the `PackageTestHandler`
0038(see its docstring for an example).
0039
0040.. _subversion post-commit hook: http://svnbook.red-bean.com/nightly/en/svn-book.html#svn.reposadmin.create.hooks
0041.. _nosetests: http://nose.python-hosting.com/
0042
0043"""
0044
0045import os
0046import sys
0047import sre
0048import optparse
0049import inspect
0050from tempfile import mkdtemp
0051from datetime import datetime
0052from subprocess import Popen, PIPE, STDOUT
0053import smtplib
0054import logging
0055from testtools.io import TempIO
0056try:
0057 from cStringIO import StringIO
0058except ImportError:
0059 from StringIO import StringIO
0060from SimpleXMLRPCServer import SimpleXMLRPCServer
0061from warnings import warn
0062
0063
0064class TestablePackage(object):
0065 """encapsulates a package that can be extracted and tested.
0066
0067 """
0068 ex = 'nosetests'
0069 svnpath = None
0070
0071 def __init__(self, **kw):
0072 [setattr(self,k,v) for k,v in kw.items()]
0073 if self.svnpath is None:
0074 raise ValueError, "svnpath cannot be None"
0075
0076 def get_name(self):
0077 """get the name of this package.
0078
0079 looks for self.__name__ or else uses the class name of self.
0080
0081 """
0082 if hasattr(self,'__name__'):
0083 return self.__name__
0084 else:
0085 return self.__class__.__name__
0086
0087 def skip_notify(self, buffer):
0088 """when True, notification is skipped.
0089
0090 it's only called on error.
0091 buffer is the output of the failing test(s)
0092
0093 """
0094 return False
0095
0096 def recognizes_file(self, filepath, code=None):
0097 """should return True if filepath from repos affects this package.
0098
0099 filepath is grabbed from the output of svnlook.
0100 it starts with the first dir relative to the repository root
0101 and will not contain a forward slash.
0102
0103 """
0104 return False
0105
0106 def runtests( self, rev=None, stream=sys.stdout,
0107 notify=None, header='', tmp_dir=None):
0108 """runs tests for this package.
0109
0110 returns True if tests passed, false otherwise.
0111
0112 Keyword Arguments:
0113 ------------------
0114 - rev -- optional revision to export (head otherwise)
0115 - stream -- filelike object for reporter, defaults to sys.stdout
0116 - notify -- if not empty, results are emailed here
0117 - header -- string to prepend onto test suite output
0118 - tmp_dir -- directory to export into and run tests (i.e. /tmp, the default)
0119
0120 """
0121 reporter = Reporter(stream=stream, notify=notify,
0122 header=header, skip_notify=self.skip_notify )
0123 res = run( self.svnpath, reporter, rev = rev, test_cmd = self.ex,
0124 tmp_prefix=self.get_name()+'_', tmp_dir=tmp_dir,
0125 setup = self.setup )
0126
0127 return res
0128
0129 def setup(self):
0130 """hook for package to setup environment, etc. before running tests.
0131
0132 """
0133 pass
0134
0135class Reporter(object):
0136 """reports the test results.
0137
0138 outputs to a log and/or email address.
0139
0140 Keyword Arguments:
0141 ------------------
0142 - stream -- filelike object for reporter
0143 - notify -- if not empty, results are emailed here
0144 - export -- svn path we are reporting about
0145 - revision -- revision we are reporting about
0146 - header -- gets added to buffer
0147 - skip_notify -- callback that accepts buffer contents on error and \
0148 returns True if notification should be skipped
0149 - em_from -- email From address
0150
0151 """
0152 def __init__( self, stream=sys.stdout, notify=None,
0153 export='unknown', revision='unknown', header='',
0154 skip_notify=None, em_from='bgtests@localhost'):
0155 self.export = export
0156 self.revision = revision
0157 self.stream = stream
0158 self.buffer = StringIO()
0159 if header:
0160 self.write(header)
0161 self.notify = notify
0162 self.em_from = em_from
0163
0164 if callable(skip_notify):
0165 self._skip_notify = skip_notify
0166 else:
0167 def sk(buf): return False
0168 self._skip_notify = sk
0169
0170 def finish(self, tests_passed=False):
0171 if not tests_passed and self.notify:
0172 output = self.buffer.getvalue()
0173 if self._skip_notify(output):
0174 print >>self, " ** notification skipped"
0175 return
0176
0177 print >>self, " ** emailing error report to %s" % self.notify
0178
0179 s = smtplib.SMTP('localhost')
0180 msg = "\n".join([ "From: %s" % self.em_from, "To: %s" % self.notify,
0181 "Subject: bgtests FAILED: %s:%s" % ( self.export,
0182 self.revision),
0183 "", output])
0184
0185 s.sendmail(self.em_from, self.notify, msg)
0186 s.quit()
0187
0188 def start(self, export=None, revision=None, tester=None):
0189 def override(k,v):
0190 if v is not None: setattr(self, k,v)
0191
0192 override('export', export)
0193 override('revision', revision)
0194 override('tester', tester)
0195
0196 print >>self, '[start: %s]' % datetime.now()
0197 print >>self, '[svn_path: %s]' % self.export
0198 print >>self, "[svn_revision: %s]" % self.revision
0199 print >>self, "[tester: %s]" % tester
0200
0201 def write(self, chunk):
0202 self.stream.write(chunk)
0203 self.buffer.write(chunk)
0204
0205class PackageTestHandler(object):
0206 """runs test packages via XMLRPC.
0207
0208 a handler designed for `SimpleXMLRPCServer`.
0209
0210 Example of starting the server::
0211
0212 pkg = TestablePackage(svnpath = 'http://svn.myrepo/')
0213 handler = PackageTestHandler([pkg])
0214 serv = SimpleXMLRPCServer(("localhost", 8000))
0215 serv.register_introspection_functions()
0216 serv.register_instance(handler)
0217 serv.serve_forever()
0218
0219 You send a request to the server and call a method to trigger tests.
0220 see `trigger_from_svnlook` for a concrete example of client/server interaction.
0221
0222 Arguments:
0223 ----------
0224 - packages -- list of `TestablePackage` objects
0225
0226 Keyword Arguments:
0227 ------------------
0228 - stream -- filelike object for reporter, defaults to sys.stdout
0229 - notify -- if not empty, results are emailed here
0230 - tmp_dir -- directory to export into and run tests
0231
0232 """
0233 def __init__( self, packages, stream=sys.stdout,
0234 notify=None, tmp_dir=None ):
0235 self.packages = packages
0236 self.stream = stream
0237 self.notify = notify
0238 self.tmp_dir = tmp_dir
0239
0240 def trigger_from_svnlook(self, svnlook_info, svnlook_changed, revision):
0241 """analyzes input from `svnlook changed` and runs tests affected by the change.
0242
0243 this method is designed to support the scenario where your subversion \
0244 repository is located on one machine but you want to run the tests on another.
0245
0246 A client that calls this from within a `subversion post-commit hook`_
0247 might look like this::
0248
0249 from testtools.bgtests import SvnInspector
0250 from xmlrpclib import ServerProxy
0251 meta = SvnInspector(repos_root, revision=revision)
0252 s = ServerProxy('http://%s:%s' % (host, port))
0253 passed = s.trigger_from_svnlook( meta.info(), "\\n".join(meta.changed()), revision )
0254
0255 NOTE: the handler assumes the svnlook output is from the same repository
0256 that the packages reside in and currently doesn't not support mixing packages that
0257 come from different repositories.
0258
0259 returns True if all tests passed (or if none ran), False otherwise.
0260
0261 .. _subversion post-commit hook: http://svnbook.red-bean.com/nightly/en/svn-book.html#svn.reposadmin.create.hooks
0262
0263 """
0264 class repos_meta:
0265 info = StringIO(svnlook_info).getvalue
0266 changed = StringIO(svnlook_changed).readlines
0267
0268 passed = run_packages( self.packages, rev=revision, repos_meta=repos_meta,
0269 stream=self.stream, notify=self.notify, tmp_dir=self.tmp_dir)
0270 self.stream.flush()
0271 return passed
0272
0273class SvnInspector:
0274 """encapsulates meta data for a repository.
0275
0276 currently this is subversion specific.
0277
0278 Arguments:
0279 ----------
0280 - repos_root -- abs path to base of repository (cannot be a URL)
0281
0282 Keyword Arguments:
0283 ------------------
0284 - revision -- inspect this revision (defaults to head)
0285
0286 """
0287 def __init__(self, repos_root, revision=None):
0288 self.repos_root = repos_root
0289 self.revision = revision
0290
0291 def changed(self):
0292 """yields each line in the `svnlook changed` command.
0293
0294 returns an iterable.
0295
0296 """
0297 cmd = 'svnlook changed %s' % self.repos_root
0298 if self.revision is not None:
0299 cmd = cmd + ' --revision %s' % self.revision
0300
0301 cmd = Popen(cmd, stdout=PIPE, shell=True, env=os.environ)
0302 for line in cmd.stdout.readlines():
0303 yield line
0304
0305 r = cmd.wait()
0306 if r != 0:
0307 raise RuntimeError, '`svnlook changed` exited: %s' % r
0308
0309 def info(self):
0310 """get `svnlook info` command output.
0311
0312 returns a string.
0313
0314 """
0315 cmd = ['svnlook','info']
0316 if self.revision is not None:
0317 cmd.extend( ['--revision', str(self.revision)] )
0318
0319 cmd.append(self.repos_root)
0320 return Popen( cmd, stdout=PIPE, stderr=STDOUT).communicate()[0]
0321
0322def run(svn_path, out=None, rev=None, test_cmd='nosetests', stream=sys.stdout,
0323 notify=None, tmp_dir=None, tmp_prefix='tmp_bgtest_', setup=None):
0324 """runs tests in the background.
0325
0326 exports svn_path (URL to package) into a temp dir and runs test_cmd.
0327
0328 Keyword Arguments:
0329 ------------------
0330 - out -- reporter object, will be created otherwise
0331 - rev -- optional revision to export (head otherwise)
0332 - test_cmd -- command to run tests with
0333 - stream -- filelike object for reporter, defaults to sys.stdout
0334 - notify -- if not empty, results are emailed here
0335 - tmp_dir -- directory to export into and run tests
0336 - tmp_prefix -- a unique temporary directory is created with this prefix
0337 - setup -- a callback to do any setup work once you are *in* the dir of your project
0338
0339 returns True if test passed, false otherwise.
0340
0341 """
0342 if out is None:
0343 out = Reporter( stream=stream, notify=notify )
0344
0345 pkg_dir = TempIO(dir=tmp_dir, prefix=tmp_prefix)
0346 cwd = os.getcwd()
0347 try:
0348
0349 os.chdir(pkg_dir.root)
0350
0351
0352 cmd = 'svn export "%s" . --force -q' % (svn_path)
0353 if rev is not None:
0354 cmd = cmd + ' -r %s' % rev
0355
0356 svn = Popen(cmd, env=os.environ, shell=True)
0357 svn.wait()
0358 assert svn.returncode == 0, 'svn exited %s; cmd="%s"' % (svn.returncode, cmd)
0359
0360 if setup is not None:
0361 setup()
0362
0363 out.start(export=svn_path, revision=rev, tester=test_cmd)
0364
0365
0366
0367 tester = Popen(test_cmd, stdout=PIPE, stderr=PIPE,
0368 shell=True, env=os.environ)
0369
0370 for line in tester.stdout.readlines():
0371 out.write(line)
0372 for line in tester.stderr.readlines():
0373 out.write(line)
0374
0375 r = tester.wait()
0376 test_passed = (r == 0)
0377 out.finish( test_passed )
0378
0379 finally:
0380 os.chdir(cwd)
0381 del pkg_dir
0382
0383 return test_passed
0384
0385def run_packages( packages, repos_root=None, rev=None, stream=sys.stdout,
0386 notify=None, tmp_dir=None, repos_meta=None):
0387 """runs tests for packages who were affected by the last change in a repository.
0388
0389 gets last changed paths from a repos_meta object to ask
0390 each package if it was affected by any one of the file paths.
0391 If so, the package's test suite is `run` in the manner defined by the package.
0392
0393 Arguments:
0394 ----------
0395 - packages -- list of `TestablePackage` objects
0396
0397 Keyword Arguments:
0398 ------------------
0399 - repos_root -- abs path to base of repository (cannot be a URL); \
0400 may be omitted in lieu of a repos_meta object
0401 - rev -- revision to inspect, defaults to None (head of repos)
0402 - stream -- filelike object for reporter, defaults to sys.stdout
0403 - notify -- if not empty, results are emailed here
0404 - tmp_dir -- directory to export into and run tests
0405 - repos_meta -- instance that provides the interface of `SvnInspector`
0406
0407 see `run` for how svn exporting works.
0408
0409 returns True if all tests passed (or none ran), False otherwise.
0410
0411 """
0412 if repos_meta is None and repos_root is None:
0413 raise ValueError, "repos_root cannot be None in absence of a repos_meta object"
0414 if repos_meta is None:
0415 repos_meta = SvnInspector(repos_root = repos_root, revision = rev)
0416
0417
0418 header = repos_meta.info()
0419
0420 svnpath = None
0421 statline = sre.compile(r'([^\s]+)\s+(.*)')
0422 pkgs_to_test = []
0423
0424 for line in repos_meta.changed():
0425 line = line.strip()
0426 if line == '':
0427 continue
0428 m = statline.match(line)
0429 if m is None:
0430
0431 print >>stream, " ** unparsable line in `svnlook changed`: '%s'" % line
0432 continue
0433 else:
0434 code, file = m.groups()
0435
0436 for pk in packages:
0437 if pk.recognizes_file(file, code=code):
0438 if pk not in pkgs_to_test:
0439 pkgs_to_test.append(pk)
0440
0441 if len(pkgs_to_test) == 0:
0442 print >>stream, " ** no tests to run for repos=%s, revision=%s" % (repos_root, rev)
0443 return True
0444
0445 all_passed = True
0446 for pkg in pkgs_to_test:
0447 passed = pkg.runtests( rev=rev, stream=stream, notify=notify,
0448 header=header, tmp_dir=tmp_dir)
0449 if not passed:
0450 all_passed = False
0451
0452 return all_passed