536 lines
23 KiB
Python
536 lines
23 KiB
Python
# This Source Code Form is subject to the terms of the Mozilla Public
|
|
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
|
|
|
from __future__ import absolute_import, unicode_literals
|
|
|
|
import cPickle as pickle
|
|
import os
|
|
import sys
|
|
|
|
import mozpack.path as mozpath
|
|
|
|
from mozpack.copier import FileCopier
|
|
from mozpack.manifests import InstallManifest
|
|
|
|
from .base import MozbuildObject
|
|
from .util import OrderedDefaultDict
|
|
from collections import defaultdict
|
|
|
|
import manifestparser
|
|
|
|
def rewrite_test_base(test, new_base, honor_install_to_subdir=False):
|
|
"""Rewrite paths in a test to be under a new base path.
|
|
|
|
This is useful for running tests from a separate location from where they
|
|
were defined.
|
|
|
|
honor_install_to_subdir and the underlying install-to-subdir field are a
|
|
giant hack intended to work around the restriction where the mochitest
|
|
runner can't handle single test files with multiple configurations. This
|
|
argument should be removed once the mochitest runner talks manifests
|
|
(bug 984670).
|
|
"""
|
|
test['here'] = mozpath.join(new_base, test['dir_relpath'])
|
|
|
|
if honor_install_to_subdir and test.get('install-to-subdir'):
|
|
manifest_relpath = mozpath.relpath(test['path'],
|
|
mozpath.dirname(test['manifest']))
|
|
test['path'] = mozpath.join(new_base, test['dir_relpath'],
|
|
test['install-to-subdir'], manifest_relpath)
|
|
else:
|
|
test['path'] = mozpath.join(new_base, test['file_relpath'])
|
|
|
|
return test
|
|
|
|
|
|
class TestMetadata(object):
|
|
"""Holds information about tests.
|
|
|
|
This class provides an API to query tests active in the build
|
|
configuration.
|
|
"""
|
|
|
|
def __init__(self, all_tests, test_defaults=None):
|
|
self._tests_by_path = OrderedDefaultDict(list)
|
|
self._tests_by_flavor = defaultdict(set)
|
|
self._test_dirs = set()
|
|
|
|
with open(all_tests, 'rb') as fh:
|
|
test_data = pickle.load(fh)
|
|
defaults = None
|
|
if test_defaults:
|
|
with open(test_defaults, 'rb') as fh:
|
|
defaults = pickle.load(fh)
|
|
for path, tests in test_data.items():
|
|
for metadata in tests:
|
|
if defaults:
|
|
manifest = metadata['manifest']
|
|
manifest_defaults = defaults.get(manifest)
|
|
if manifest_defaults:
|
|
metadata = manifestparser.combine_fields(manifest_defaults,
|
|
metadata)
|
|
self._tests_by_path[path].append(metadata)
|
|
self._test_dirs.add(os.path.dirname(path))
|
|
flavor = metadata.get('flavor')
|
|
self._tests_by_flavor[flavor].add(path)
|
|
|
|
def tests_with_flavor(self, flavor):
|
|
"""Obtain all tests having the specified flavor.
|
|
|
|
This is a generator of dicts describing each test.
|
|
"""
|
|
|
|
for path in sorted(self._tests_by_flavor.get(flavor, [])):
|
|
yield self._tests_by_path[path]
|
|
|
|
def resolve_tests(self, paths=None, flavor=None, subsuite=None, under_path=None,
|
|
tags=None):
|
|
"""Resolve tests from an identifier.
|
|
|
|
This is a generator of dicts describing each test.
|
|
|
|
``paths`` can be an iterable of values to use to identify tests to run.
|
|
If an entry is a known test file, tests associated with that file are
|
|
returned (there may be multiple configurations for a single file). If
|
|
an entry is a directory, or a prefix of a directory containing tests,
|
|
all tests in that directory are returned. If the string appears in a
|
|
known test file, that test file is considered. If the path contains
|
|
a wildcard pattern, tests matching that pattern are returned.
|
|
|
|
If ``under_path`` is a string, it will be used to filter out tests that
|
|
aren't in the specified path prefix relative to topsrcdir or the
|
|
test's installed dir.
|
|
|
|
If ``flavor`` is a string, it will be used to filter returned tests
|
|
to only be the flavor specified. A flavor is something like
|
|
``xpcshell``.
|
|
|
|
If ``subsuite`` is a string, it will be used to filter returned tests
|
|
to only be in the subsuite specified.
|
|
|
|
If ``tags`` are specified, they will be used to filter returned tests
|
|
to only those with a matching tag.
|
|
"""
|
|
if tags:
|
|
tags = set(tags)
|
|
|
|
def fltr(tests):
|
|
for test in tests:
|
|
if flavor:
|
|
if (flavor == 'devtools' and test.get('flavor') != 'browser-chrome') or \
|
|
(flavor != 'devtools' and test.get('flavor') != flavor):
|
|
continue
|
|
|
|
if subsuite and test.get('subsuite') != subsuite:
|
|
continue
|
|
|
|
if tags and not (tags & set(test.get('tags', '').split())):
|
|
continue
|
|
|
|
if under_path \
|
|
and not test['file_relpath'].startswith(under_path):
|
|
continue
|
|
|
|
# Make a copy so modifications don't change the source.
|
|
yield dict(test)
|
|
|
|
paths = paths or []
|
|
paths = [mozpath.normpath(p) for p in paths]
|
|
if not paths:
|
|
paths = [None]
|
|
|
|
candidate_paths = set()
|
|
|
|
for path in sorted(paths):
|
|
if path is None:
|
|
candidate_paths |= set(self._tests_by_path.keys())
|
|
continue
|
|
|
|
if '*' in path:
|
|
candidate_paths |= {p for p in self._tests_by_path
|
|
if mozpath.match(p, path)}
|
|
continue
|
|
|
|
# If the path is a directory, or the path is a prefix of a directory
|
|
# containing tests, pull in all tests in that directory.
|
|
if (path in self._test_dirs or
|
|
any(p.startswith(path) for p in self._tests_by_path)):
|
|
candidate_paths |= {p for p in self._tests_by_path
|
|
if p.startswith(path)}
|
|
continue
|
|
|
|
# If it's a test file, add just that file.
|
|
candidate_paths |= {p for p in self._tests_by_path if path in p}
|
|
|
|
for p in sorted(candidate_paths):
|
|
tests = self._tests_by_path[p]
|
|
|
|
for test in fltr(tests):
|
|
yield test
|
|
|
|
|
|
class TestResolver(MozbuildObject):
|
|
"""Helper to resolve tests from the current environment to test files."""
|
|
|
|
def __init__(self, *args, **kwargs):
|
|
MozbuildObject.__init__(self, *args, **kwargs)
|
|
|
|
# If installing tests is going to result in re-generating the build
|
|
# backend, we need to do this here, so that the updated contents of
|
|
# all-tests.pkl make it to the set of tests to run.
|
|
self._run_make(target='run-tests-deps', pass_thru=True,
|
|
print_directory=False)
|
|
|
|
self._tests = TestMetadata(os.path.join(self.topobjdir,
|
|
'all-tests.pkl'),
|
|
test_defaults=os.path.join(self.topobjdir,
|
|
'test-defaults.pkl'))
|
|
|
|
self._test_rewrites = {
|
|
'a11y': os.path.join(self.topobjdir, '_tests', 'testing',
|
|
'mochitest', 'a11y'),
|
|
'browser-chrome': os.path.join(self.topobjdir, '_tests', 'testing',
|
|
'mochitest', 'browser'),
|
|
'jetpack-package': os.path.join(self.topobjdir, '_tests', 'testing',
|
|
'mochitest', 'jetpack-package'),
|
|
'jetpack-addon': os.path.join(self.topobjdir, '_tests', 'testing',
|
|
'mochitest', 'jetpack-addon'),
|
|
'chrome': os.path.join(self.topobjdir, '_tests', 'testing',
|
|
'mochitest', 'chrome'),
|
|
'mochitest': os.path.join(self.topobjdir, '_tests', 'testing',
|
|
'mochitest', 'tests'),
|
|
'web-platform-tests': os.path.join(self.topobjdir, '_tests', 'testing',
|
|
'web-platform'),
|
|
'xpcshell': os.path.join(self.topobjdir, '_tests', 'xpcshell'),
|
|
}
|
|
|
|
def resolve_tests(self, cwd=None, **kwargs):
|
|
"""Resolve tests in the context of the current environment.
|
|
|
|
This is a more intelligent version of TestMetadata.resolve_tests().
|
|
|
|
This function provides additional massaging and filtering of low-level
|
|
results.
|
|
|
|
Paths in returned tests are automatically translated to the paths in
|
|
the _tests directory under the object directory.
|
|
|
|
If cwd is defined, we will limit our results to tests under the
|
|
directory specified. The directory should be defined as an absolute
|
|
path under topsrcdir or topobjdir for it to work properly.
|
|
"""
|
|
rewrite_base = None
|
|
|
|
if cwd:
|
|
norm_cwd = mozpath.normpath(cwd)
|
|
norm_srcdir = mozpath.normpath(self.topsrcdir)
|
|
norm_objdir = mozpath.normpath(self.topobjdir)
|
|
|
|
reldir = None
|
|
|
|
if norm_cwd.startswith(norm_objdir):
|
|
reldir = norm_cwd[len(norm_objdir)+1:]
|
|
elif norm_cwd.startswith(norm_srcdir):
|
|
reldir = norm_cwd[len(norm_srcdir)+1:]
|
|
|
|
result = self._tests.resolve_tests(under_path=reldir,
|
|
**kwargs)
|
|
|
|
else:
|
|
result = self._tests.resolve_tests(**kwargs)
|
|
|
|
for test in result:
|
|
rewrite_base = self._test_rewrites.get(test['flavor'], None)
|
|
|
|
if rewrite_base:
|
|
yield rewrite_test_base(test, rewrite_base,
|
|
honor_install_to_subdir=True)
|
|
else:
|
|
yield test
|
|
|
|
# These definitions provide a single source of truth for modules attempting
|
|
# to get a view of all tests for a build. Used by the emitter to figure out
|
|
# how to read/install manifests and by test dependency annotations in Files()
|
|
# entries to enumerate test flavors.
|
|
|
|
# While there are multiple test manifests, the behavior is very similar
|
|
# across them. We enforce this by having common handling of all
|
|
# manifests and outputting a single class type with the differences
|
|
# described inside the instance.
|
|
#
|
|
# Keys are variable prefixes and values are tuples describing how these
|
|
# manifests should be handled:
|
|
#
|
|
# (flavor, install_root, install_subdir, package_tests)
|
|
#
|
|
# flavor identifies the flavor of this test.
|
|
# install_root is the path prefix to install the files starting from the root
|
|
# directory and not as specified by the manifest location. (bug 972168)
|
|
# install_subdir is the path of where to install the files in
|
|
# the tests directory.
|
|
# package_tests indicates whether to package test files into the test
|
|
# package; suites that compile the test files should not install
|
|
# them into the test package.
|
|
#
|
|
TEST_MANIFESTS = dict(
|
|
A11Y=('a11y', 'testing/mochitest', 'a11y', True),
|
|
BROWSER_CHROME=('browser-chrome', 'testing/mochitest', 'browser', True),
|
|
ANDROID_INSTRUMENTATION=('instrumentation', 'instrumentation', '.', False),
|
|
JETPACK_PACKAGE=('jetpack-package', 'testing/mochitest', 'jetpack-package', True),
|
|
JETPACK_ADDON=('jetpack-addon', 'testing/mochitest', 'jetpack-addon', False),
|
|
FIREFOX_UI_FUNCTIONAL=('firefox-ui-functional', 'firefox-ui', '.', False),
|
|
FIREFOX_UI_UPDATE=('firefox-ui-update', 'firefox-ui', '.', False),
|
|
PUPPETEER_FIREFOX=('firefox-ui-functional', 'firefox-ui', '.', False),
|
|
|
|
# marionette tests are run from the srcdir
|
|
# TODO(ato): make packaging work as for other test suites
|
|
MARIONETTE=('marionette', 'marionette', '.', False),
|
|
MARIONETTE_UNIT=('marionette', 'marionette', '.', False),
|
|
MARIONETTE_WEBAPI=('marionette', 'marionette', '.', False),
|
|
|
|
METRO_CHROME=('metro-chrome', 'testing/mochitest', 'metro', True),
|
|
MOCHITEST=('mochitest', 'testing/mochitest', 'tests', True),
|
|
MOCHITEST_CHROME=('chrome', 'testing/mochitest', 'chrome', True),
|
|
WEBRTC_SIGNALLING_TEST=('steeplechase', 'steeplechase', '.', True),
|
|
XPCSHELL_TESTS=('xpcshell', 'xpcshell', '.', True),
|
|
)
|
|
|
|
# Reftests have their own manifest format and are processed separately.
|
|
REFTEST_FLAVORS = ('crashtest', 'reftest')
|
|
|
|
# Web platform tests have their own manifest format and are processed separately.
|
|
WEB_PLATFORM_TESTS_FLAVORS = ('web-platform-tests',)
|
|
|
|
def all_test_flavors():
|
|
return ([v[0] for v in TEST_MANIFESTS.values()] +
|
|
list(REFTEST_FLAVORS) +
|
|
list(WEB_PLATFORM_TESTS_FLAVORS) +
|
|
['python'])
|
|
|
|
class TestInstallInfo(object):
|
|
def __init__(self):
|
|
self.seen = set()
|
|
self.pattern_installs = []
|
|
self.installs = []
|
|
self.external_installs = set()
|
|
self.deferred_installs = set()
|
|
|
|
def __ior__(self, other):
|
|
self.pattern_installs.extend(other.pattern_installs)
|
|
self.installs.extend(other.installs)
|
|
self.external_installs |= other.external_installs
|
|
self.deferred_installs |= other.deferred_installs
|
|
return self
|
|
|
|
class SupportFilesConverter(object):
|
|
"""Processes a "support-files" entry from a test object, either from
|
|
a parsed object from a test manifests or its representation in
|
|
moz.build and returns the installs to perform for this test object.
|
|
|
|
Processing the same support files multiple times will not have any further
|
|
effect, and the structure of the parsed objects from manifests will have a
|
|
lot of repeated entries, so this class takes care of memoizing.
|
|
"""
|
|
def __init__(self):
|
|
self._fields = (('head', set()),
|
|
('tail', set()),
|
|
('support-files', set()),
|
|
('generated-files', set()))
|
|
|
|
def convert_support_files(self, test, install_root, manifest_dir, out_dir):
|
|
# Arguments:
|
|
# test - The test object to process.
|
|
# install_root - The directory under $objdir/_tests that will contain
|
|
# the tests for this harness (examples are "testing/mochitest",
|
|
# "xpcshell").
|
|
# manifest_dir - Absoulute path to the (srcdir) directory containing the
|
|
# manifest that included this test
|
|
# out_dir - The path relative to $objdir/_tests used as the destination for the
|
|
# test, based on the relative path to the manifest in the srcdir,
|
|
# the install_root, and 'install-to-subdir', if present in the manifest.
|
|
info = TestInstallInfo()
|
|
for field, seen in self._fields:
|
|
value = test.get(field, '')
|
|
for pattern in value.split():
|
|
|
|
# We track uniqueness locally (per test) where duplicates are forbidden,
|
|
# and globally, where they are permitted. If a support file appears multiple
|
|
# times for a single test, there are unnecessary entries in the manifest. But
|
|
# many entries will be shared across tests that share defaults.
|
|
# We need to memoize on the basis of both the path and the output
|
|
# directory for the benefit of tests specifying 'install-to-subdir'.
|
|
key = field, pattern, out_dir
|
|
if key in info.seen:
|
|
raise ValueError("%s appears multiple times in a test manifest under a %s field,"
|
|
" please omit the duplicate entry." % (pattern, field))
|
|
info.seen.add(key)
|
|
if key in seen:
|
|
continue
|
|
seen.add(key)
|
|
|
|
if field == 'generated-files':
|
|
info.external_installs.add(mozpath.normpath(mozpath.join(out_dir, pattern)))
|
|
# '!' indicates our syntax for inter-directory support file
|
|
# dependencies. These receive special handling in the backend.
|
|
elif pattern[0] == '!':
|
|
info.deferred_installs.add(pattern)
|
|
# We only support globbing on support-files because
|
|
# the harness doesn't support * for head and tail.
|
|
elif '*' in pattern and field == 'support-files':
|
|
info.pattern_installs.append((manifest_dir, pattern, out_dir))
|
|
# "absolute" paths identify files that are to be
|
|
# placed in the install_root directory (no globs)
|
|
elif pattern[0] == '/':
|
|
full = mozpath.normpath(mozpath.join(manifest_dir,
|
|
mozpath.basename(pattern)))
|
|
info.installs.append((full, mozpath.join(install_root, pattern[1:])))
|
|
else:
|
|
full = mozpath.normpath(mozpath.join(manifest_dir, pattern))
|
|
dest_path = mozpath.join(out_dir, pattern)
|
|
|
|
# If the path resolves to a different directory
|
|
# tree, we take special behavior depending on the
|
|
# entry type.
|
|
if not full.startswith(manifest_dir):
|
|
# If it's a support file, we install the file
|
|
# into the current destination directory.
|
|
# This implementation makes installing things
|
|
# with custom prefixes impossible. If this is
|
|
# needed, we can add support for that via a
|
|
# special syntax later.
|
|
if field == 'support-files':
|
|
dest_path = mozpath.join(out_dir,
|
|
os.path.basename(pattern))
|
|
# If it's not a support file, we ignore it.
|
|
# This preserves old behavior so things like
|
|
# head files doesn't get installed multiple
|
|
# times.
|
|
else:
|
|
continue
|
|
info.installs.append((full, mozpath.normpath(dest_path)))
|
|
return info
|
|
|
|
def _resolve_installs(paths, topobjdir, manifest):
|
|
"""Using the given paths as keys, find any unresolved installs noted
|
|
by the build backend corresponding to those keys, and add them
|
|
to the given manifest.
|
|
"""
|
|
filename = os.path.join(topobjdir, 'test-installs.pkl')
|
|
with open(filename, 'rb') as fh:
|
|
resolved_installs = pickle.load(fh)
|
|
|
|
for path in paths:
|
|
path = path[2:]
|
|
if path not in resolved_installs:
|
|
raise Exception('A cross-directory support file path noted in a '
|
|
'test manifest does not appear in any other manifest.\n "%s" '
|
|
'must appear in another test manifest to specify an install '
|
|
'for "!/%s".' % (path, path))
|
|
installs = resolved_installs[path]
|
|
for install_info in installs:
|
|
try:
|
|
if len(install_info) == 3:
|
|
manifest.add_pattern_symlink(*install_info)
|
|
if len(install_info) == 2:
|
|
manifest.add_symlink(*install_info)
|
|
except ValueError:
|
|
# A duplicate value here is pretty likely when running
|
|
# multiple directories at once, and harmless.
|
|
pass
|
|
|
|
def install_test_files(topsrcdir, topobjdir, tests_root, test_objs):
|
|
"""Installs the requested test files to the objdir. This is invoked by
|
|
test runners to avoid installing tens of thousands of test files when
|
|
only a few tests need to be run.
|
|
"""
|
|
flavor_info = {flavor: (root, prefix, install)
|
|
for (flavor, root, prefix, install) in TEST_MANIFESTS.values()}
|
|
objdir_dest = mozpath.join(topobjdir, tests_root)
|
|
|
|
converter = SupportFilesConverter()
|
|
install_info = TestInstallInfo()
|
|
for o in test_objs:
|
|
flavor = o['flavor']
|
|
if flavor not in flavor_info:
|
|
# This is a test flavor that isn't installed by the build system.
|
|
continue
|
|
root, prefix, install = flavor_info[flavor]
|
|
if not install:
|
|
# This flavor isn't installed to the objdir.
|
|
continue
|
|
|
|
manifest_path = o['manifest']
|
|
manifest_dir = mozpath.dirname(manifest_path)
|
|
|
|
out_dir = mozpath.join(root, prefix, manifest_dir[len(topsrcdir) + 1:])
|
|
file_relpath = o['file_relpath']
|
|
source = mozpath.join(topsrcdir, file_relpath)
|
|
dest = mozpath.join(root, prefix, file_relpath)
|
|
if 'install-to-subdir' in o:
|
|
out_dir = mozpath.join(out_dir, o['install-to-subdir'])
|
|
manifest_relpath = mozpath.relpath(source, mozpath.dirname(manifest_path))
|
|
dest = mozpath.join(out_dir, manifest_relpath)
|
|
|
|
install_info.installs.append((source, dest))
|
|
install_info |= converter.convert_support_files(o, root,
|
|
manifest_dir,
|
|
out_dir)
|
|
|
|
manifest = InstallManifest()
|
|
|
|
for source, dest in set(install_info.installs):
|
|
if dest in install_info.external_installs:
|
|
continue
|
|
manifest.add_symlink(source, dest)
|
|
for base, pattern, dest in install_info.pattern_installs:
|
|
manifest.add_pattern_symlink(base, pattern, dest)
|
|
|
|
_resolve_installs(install_info.deferred_installs, topobjdir, manifest)
|
|
|
|
# Harness files are treated as a monolith and installed each time we run tests.
|
|
# Fortunately there are not very many.
|
|
manifest |= InstallManifest(mozpath.join(topobjdir,
|
|
'_build_manifests',
|
|
'install', tests_root))
|
|
copier = FileCopier()
|
|
manifest.populate_registry(copier)
|
|
copier.copy(objdir_dest,
|
|
remove_unaccounted=False)
|
|
|
|
|
|
# Convenience methods for test manifest reading.
|
|
def read_manifestparser_manifest(context, manifest_path):
|
|
path = mozpath.normpath(mozpath.join(context.srcdir, manifest_path))
|
|
return manifestparser.TestManifest(manifests=[path], strict=True,
|
|
rootdir=context.config.topsrcdir,
|
|
finder=context._finder,
|
|
handle_defaults=False)
|
|
|
|
def read_reftest_manifest(context, manifest_path):
|
|
import reftest
|
|
path = mozpath.normpath(mozpath.join(context.srcdir, manifest_path))
|
|
manifest = reftest.ReftestManifest(finder=context._finder)
|
|
manifest.load(path)
|
|
return manifest
|
|
|
|
def read_wpt_manifest(context, paths):
|
|
manifest_path, tests_root = paths
|
|
full_path = mozpath.normpath(mozpath.join(context.srcdir, manifest_path))
|
|
old_path = sys.path[:]
|
|
try:
|
|
# Setup sys.path to include all the dependencies required to import
|
|
# the web-platform-tests manifest parser. web-platform-tests provides
|
|
# a the localpaths.py to do the path manipulation, which we load,
|
|
# providing the __file__ variable so it can resolve the relative
|
|
# paths correctly.
|
|
paths_file = os.path.join(context.config.topsrcdir, "testing",
|
|
"web-platform", "tests", "tools", "localpaths.py")
|
|
_globals = {"__file__": paths_file}
|
|
execfile(paths_file, _globals)
|
|
import manifest as wptmanifest
|
|
finally:
|
|
sys.path = old_path
|
|
f = context._finder.get(full_path)
|
|
return wptmanifest.manifest.load(tests_root, f)
|