# Copyright (c) 2004 DoCoMo Euro-Labs GmbH (Munich, Germany). # Copyright (c) 2001-2004 LOGILAB S.A. (Paris, FRANCE). # # http://www.docomolab-euro.com/ -- mailto:tarlano@docomolab-euro.com # http://www.logilab.fr/ -- mailto:contact@logilab.fr # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public # License as published by the Free Software Foundation; either # version 2.1 of the License, or (at your option) any later version. # # This library is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU # Lesser General Public License for more details. # # You should have received a copy of the GNU Lesser General Public # License along with this library; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA from __future__ import generators """Apycot_ extenstions for narval acceptance tests .. _Apycot: http://www.logilab.org/projects/apycot :version: $Revision:$ :author: Logilab :copyright: 2001-2004 LOGILAB S.A. (Paris, FRANCE) 2004 DoCoMo Euro-Labs GmbH (Munich, Germany) :contact: http://www.logilab.fr/ -- mailto:contact@logilab.fr http://www.docomolab-euro.com/ -- mailto:tarlano@docomolab-euro.com :var KERNEL_TEST_RGX: regular expression matching kernel test :var STEP_TEST_RGX: regular expression matching actions / recipes test :var ENGINE: path to the main interpreter file :var TEST_DESCR_EXPR: match expression used to get the test description element """ __revision__ = '$Id: autotest.py,v 1.2 2002/08/14 11:47:37 syt Exp $' __docformat__ = "restructuredtext en" import sys import re import traceback from os import spawnv, P_WAIT, listdir from os.path import join from tester import register, IChecker from tester.utils import SimpleOptionsManagerMixIn from narval import init_log, bibal from narval.tools import main from narval.reader import REGISTRY, multi_match_expression from narval.config import get_share, DEFAULT_CONFIGURATOR from narval.narvalrc import NarvalRC from narval import testutils ENGINE = main.__file__ #MAXTIME = '120' # delay until we consider narval is blocked (in secondes) KERNEL_TEST_RGX = re.compile('^(\d\d\d)(bis)?\.xml$') STEP_TEST_RGX = re.compile('^((\w+)\.(-|\w)+)(\d)?\.xml$') TEST_DESCR_EXPR = 'isinstance(elmt, TestDescriptionElement)' def get_test_cases(home, regex=None): """return acceptance test cases matching a regular expression :type home: path :param home: narval home directory :type regex: str :param regex: optional regular expression used to filter cases to execute :rtype: iterator :return: an iterator on tests matching the given regular expression (all if None), wher each test is defined by a 2-uple (test prefix, rc file name) """ regex = re.compile(regex or '.*') for fname in listdir(join(home, 'tests', 'cases')): if regex.match(fname) is None: continue match = KERNEL_TEST_RGX.match(fname) if match: ext = match.group(2) or '' yield match.group(1) + ext, 'selftestrc.xml' continue match = STEP_TEST_RGX.match(fname) if match: ext = match.group(4) or '' yield match.group(1) + ext, '%src.xml' % match.group(2).lower() def run_test_case(tmp, name, rcfile, home, debug=False): """execute a single test case by spawning narval :type tmp: str :param tmp: the temporary directory used to put files generated by the test :type name: str :param name: the test name :type rcfile: str :param rcfile: path to the narval rc file :type home: str :param home: the narval home directory :type debug: bool :param debug: flag indicating whether we should start narval in debug mode :rtype: str :return: the interpreter's memory file, written at the end of the test execution """ rcfile = join(home, 'tests', 'rc', rcfile) memfile = join(home, 'tests', 'cases', '%s.xml' % name) tmpfile = join(tmp, 'mem-%s.xml' % name) args = ('python', ENGINE, '--quit-when-done', '--home', home, '--memory-file', memfile, '--rc-file', rcfile, # '--quit-after', MAXTIME, '--save-mem-on-quit', tmpfile) if debug: args += ('--debug',) print ' '.join(args) else: args += ('--quiet',) if spawnv(P_WAIT, sys.executable, args): raise Exception('''Error while spawning python Tried with following parameters: %s %s --home %s --memory-file %s --save-mem-on-quit %s\ --quit-when-done''' % ( sys.executable, ENGINE, home, memfile, tmpfile)) return tmpfile def validate_test(memory_file, validation_file, writer, debug=False) : """validate result of a test by loading the memory file and applying the validation file :type memory_file: str :param memory_file: path to the memory file :type validation_file: str :param validation_file: path to the validation file :type writer: tester.interfaces.IWriter :param writer: apycot's writer instance :type debug: bool :param debug: flag indicating whether we should start narval in debug mode :rtype: bool :return: true if the test succeed """ try: elmts = REGISTRY.from_stream(open(memory_file), 1) except Exception, ex: if debug: traceback.print_exc() writer.log(ERROR, memory_file, None, 'while loading memory file: %s' % ex) return False test_descr = multi_match_expression(TEST_DESCR_EXPR, elmts).next() category = '%s:%s' % (test_descr.category, test_descr.recipe) test_context = vars(testutils) test_context['memory'] = elmts try: exec open(validation_file) in REGISTRY.global_context, test_context writer.log(INFO, memory_file, None, '%s: succeed' % category) return True except Exception, ex: if debug: traceback.print_exc() _, _, tb = sys.exc_info() where = traceback.format_tb(tb, 2)[-1] sep = '\n' + '-' * 30 + '\n' writer.log(ERROR, memory_file, None, '%s: failed%s%s(%s: %s)%s' % ( category, sep, where, ex.__class__.__name__, ex, sep)) return False class NarvalChecker(SimpleOptionsManagerMixIn): """run acceptance tests for narval from the apycot framework """ __implements__ = IChecker __name__ = 'narval_acceptance' def run(self, test, writer): """run the check on the given test object :type test: tester.Test.Test :param test: the test object :type writer: tester.interfaces.IWriter :param writer: apycot's writer instance :rtype: bool :return: true if the check succeed Options: - **command**: space separated regular expressions to match a subset of available test cases, default to 'all' (short cut for '.*') - **debug**: give additional debug information """ command = self.get_option('command', 'all') debug = self.get_option('debug', False) # FIXME: need correct handling of narval home / test home # currently that's a dirty hack to look for test in the share # directory... test_home = get_share() narval_home = self.get_option('narval_home', test_home) if narval_home is not None: DEFAULT_CONFIGURATOR.set_home_from_command_line(narval_home) if not debug: init_log(LOG_ERR, sid='narval-apycot') else: init_log(LOG_NOTICE, sid='narval-apycot') # instantiate narval rc with the correct rc file NarvalRC('whatever') if command == 'all': commands = [None] else: commands = command.split() # initialize the registry with library elements writer.log(INFO, test.repo.path, None, 'initializing engine') from narval.engine import Narval engine = Narval('') bibal.load_interfaces(REGISTRY) bibal.load_elements(REGISTRY) bibal.load_protocol_handlers(engine) # [number of failed test, number of succeed test] results = [0, 0] for expr in commands: for name, rcfile in get_test_cases(test_home, expr): # execute test writer.log(INFO, test.repo.path, None, 'running test case %s (rc file: %s)' % (name, rcfile)) try: outfile = run_test_case(test.tmpdir, name, rcfile, test_home, debug) except Exception, ex: writer.log(ERROR, test.repo.path, None, str(ex)) results[False] += 1 continue valfile = join(test_home, 'tests', 'validate', 'val-%s.py' % name) try: result = validate_test(outfile, valfile, writer, debug) except Exception, ex: writer.log(ERROR, test.repo.path, None, str(ex)) results[False] += 1 continue results[result] += 1 if results == [0, 0]: raise Exception('No test matched %r' % command) writer.raw('failures', results[0]) writer.raw('success', results[1]) return not results[False] register('checker', NarvalChecker)