""" Shared setup file for simple python packages. Uses a setup.cfg that is the same as the distutils2 project, unless noted otherwise. It exists for two reasons: 1) This makes it easier to reuse setup.py code between my own projects 2) Easier migration to distutils2 when that catches on. Additional functionality: * Section metadata: requires-test: Same as 'tests_require' option for setuptools. """ import sys import os import re import platform from fnmatch import fnmatch import os import sys import time import tempfile import tarfile try: import urllib.request as urllib except ImportError: import urllib from distutils import log try: from hashlib import md5 except ImportError: from md5 import md5 if sys.version_info[0] == 2: from ConfigParser import RawConfigParser, NoOptionError, NoSectionError else: from configparser import RawConfigParser, NoOptionError, NoSectionError ROOTDIR = os.path.dirname(os.path.abspath(__file__)) # # # # Parsing the setup.cfg and converting it to something that can be # used by setuptools.setup() # # # def eval_marker(value): """ Evaluate an distutils2 environment marker. This code is unsafe when used with hostile setup.cfg files, but that's not a problem for our own files. """ value = value.strip() class M: def __init__(self, **kwds): for k, v in kwds.items(): setattr(self, k, v) variables = { 'python_version': '%d.%d'%(sys.version_info[0], sys.version_info[1]), 'python_full_version': sys.version.split()[0], 'os': M( name=os.name, ), 'sys': M( platform=sys.platform, ), 'platform': M( version=platform.version(), machine=platform.machine(), ), } return bool(eval(value, variables, variables)) return True def _opt_value(cfg, into, section, key, transform = None): try: v = cfg.get(section, key) if transform != _as_lines and ';' in v: v, marker = v.rsplit(';', 1) if not eval_marker(marker): return v = v.strip() if v: if transform: into[key] = transform(v.strip()) else: into[key] = v.strip() except (NoOptionError, NoSectionError): pass def _as_bool(value): if value.lower() in ('y', 'yes', 'on'): return True elif value.lower() in ('n', 'no', 'off'): return False elif value.isdigit(): return bool(int(value)) else: raise ValueError(value) def _as_list(value): return value.split() def _as_lines(value): result = [] for v in value.splitlines(): if ';' in v: v, marker = v.rsplit(';', 1) if not eval_marker(marker): continue v = v.strip() if v: result.append(v) else: result.append(v) return result def _map_requirement(value): m = re.search(r'(\S+)\s*(?:\((.*)\))?', value) name = m.group(1) version = m.group(2) if version is None: return name else: mapped = [] for v in version.split(','): v = v.strip() if v[0].isdigit(): # Checks for a specific version prefix m = v.rsplit('.', 1) mapped.append('>=%s,<%s.%s'%( v, m[0], int(m[1])+1)) else: mapped.append(v) return '%s %s'%(name, ','.join(mapped),) def _as_requires(value): requires = [] for req in value.splitlines(): if ';' in req: req, marker = v.rsplit(';', 1) if not eval_marker(marker): continue req = req.strip() if not req: continue requires.append(_map_requirement(req)) return requires def parse_setup_cfg(): cfg = RawConfigParser() r = cfg.read([os.path.join(ROOTDIR, 'setup.cfg')]) if len(r) != 1: print("Cannot read 'setup.cfg'") sys.exit(1) metadata = dict( name = cfg.get('metadata', 'name'), version = cfg.get('metadata', 'version'), description = cfg.get('metadata', 'description'), ) _opt_value(cfg, metadata, 'metadata', 'license') _opt_value(cfg, metadata, 'metadata', 'maintainer') _opt_value(cfg, metadata, 'metadata', 'maintainer_email') _opt_value(cfg, metadata, 'metadata', 'author') _opt_value(cfg, metadata, 'metadata', 'author_email') _opt_value(cfg, metadata, 'metadata', 'url') _opt_value(cfg, metadata, 'metadata', 'download_url') _opt_value(cfg, metadata, 'metadata', 'classifiers', _as_lines) _opt_value(cfg, metadata, 'metadata', 'platforms', _as_list) _opt_value(cfg, metadata, 'metadata', 'packages', _as_list) _opt_value(cfg, metadata, 'metadata', 'keywords', _as_list) try: v = cfg.get('metadata', 'requires-dist') except (NoOptionError, NoSectionError): pass else: requires = _as_requires(v) if requires: metadata['install_requires'] = requires try: v = cfg.get('metadata', 'requires-test') except (NoOptionError, NoSectionError): pass else: requires = _as_requires(v) if requires: metadata['tests_require'] = requires try: v = cfg.get('metadata', 'long_description_file') except (NoOptionError, NoSectionError): pass else: parts = [] for nm in v.split(): fp = open(nm, 'rU') parts.append(fp.read()) fp.close() metadata['long_description'] = '\n\n'.join(parts) try: v = cfg.get('metadata', 'zip-safe') except (NoOptionError, NoSectionError): pass else: metadata['zip_safe'] = _as_bool(v) try: v = cfg.get('metadata', 'console_scripts') except (NoOptionError, NoSectionError): pass else: if 'entry_points' not in metadata: metadata['entry_points'] = {} metadata['entry_points']['console_scripts'] = v.splitlines() if sys.version_info[:2] <= (2,6): try: metadata['tests_require'] += ", unittest2" except KeyError: metadata['tests_require'] = "unittest2" return metadata # # # # Bootstrapping setuptools/distribute, based on # a heavily modified version of distribute_setup.py # # # SETUPTOOLS_PACKAGE='setuptools' try: import subprocess def _python_cmd(*args): args = (sys.executable,) + args return subprocess.call(args) == 0 except ImportError: def _python_cmd(*args): args = (sys.executable,) + args new_args = [] for a in args: new_args.append(a.replace("'", "'\"'\"'")) os.system(' '.join(new_args)) == 0 try: import json def get_pypi_src_download(package): url = 'https://pypi.python.org/pypi/%s/json'%(package,) fp = urllib.urlopen(url) try: try: data = fp.read() finally: fp.close() except urllib.error: raise RuntimeError("Cannot determine download link for %s"%(package,)) pkgdata = json.loads(data.decode('utf-8')) if 'urls' not in pkgdata: raise RuntimeError("Cannot determine download link for %s"%(package,)) for info in pkgdata['urls']: if info['packagetype'] == 'sdist' and info['url'].endswith('tar.gz'): return (info.get('md5_digest'), info['url']) raise RuntimeError("Cannot determine downlink link for %s"%(package,)) except ImportError: # Python 2.5 compatibility, no JSON in stdlib but luckily JSON syntax is # simular enough to Python's syntax to be able to abuse the Python compiler import _ast as ast def get_pypi_src_download(package): url = 'https://pypi.python.org/pypi/%s/json'%(package,) fp = urllib.urlopen(url) try: try: data = fp.read() finally: fp.close() except urllib.error: raise RuntimeError("Cannot determine download link for %s"%(package,)) a = compile(data, '-', 'eval', ast.PyCF_ONLY_AST) if not isinstance(a, ast.Expression): raise RuntimeError("Cannot determine download link for %s"%(package,)) a = a.body if not isinstance(a, ast.Dict): raise RuntimeError("Cannot determine download link for %s"%(package,)) for k, v in zip(a.keys, a.values): if not isinstance(k, ast.Str): raise RuntimeError("Cannot determine download link for %s"%(package,)) k = k.s if k == 'urls': a = v break else: raise RuntimeError("PyPI JSON for %s doesn't contain URLs section"%(package,)) if not isinstance(a, ast.List): raise RuntimeError("Cannot determine download link for %s"%(package,)) for info in v.elts: if not isinstance(info, ast.Dict): raise RuntimeError("Cannot determine download link for %s"%(package,)) url = None packagetype = None chksum = None for k, v in zip(info.keys, info.values): if not isinstance(k, ast.Str): raise RuntimeError("Cannot determine download link for %s"%(package,)) if k.s == 'url': if not isinstance(v, ast.Str): raise RuntimeError("Cannot determine download link for %s"%(package,)) url = v.s elif k.s == 'packagetype': if not isinstance(v, ast.Str): raise RuntimeError("Cannot determine download link for %s"%(package,)) packagetype = v.s elif k.s == 'md5_digest': if not isinstance(v, ast.Str): raise RuntimeError("Cannot determine download link for %s"%(package,)) chksum = v.s if url is not None and packagetype == 'sdist' and url.endswith('.tar.gz'): return (chksum, url) raise RuntimeError("Cannot determine download link for %s"%(package,)) def _build_egg(egg, tarball, to_dir): # extracting the tarball tmpdir = tempfile.mkdtemp() log.warn('Extracting in %s', tmpdir) old_wd = os.getcwd() try: os.chdir(tmpdir) tar = tarfile.open(tarball) _extractall(tar) tar.close() # going in the directory subdir = os.path.join(tmpdir, os.listdir(tmpdir)[0]) os.chdir(subdir) log.warn('Now working in %s', subdir) # building an egg log.warn('Building a %s egg in %s', egg, to_dir) _python_cmd('setup.py', '-q', 'bdist_egg', '--dist-dir', to_dir) finally: os.chdir(old_wd) # returning the result log.warn(egg) if not os.path.exists(egg): raise IOError('Could not build the egg.') def _do_download(to_dir, packagename=SETUPTOOLS_PACKAGE): tarball = download_setuptools(packagename, to_dir) version = tarball.split('-')[-1][:-7] egg = os.path.join(to_dir, '%s-%s-py%d.%d.egg' % (packagename, version, sys.version_info[0], sys.version_info[1])) if not os.path.exists(egg): _build_egg(egg, tarball, to_dir) sys.path.insert(0, egg) import setuptools setuptools.bootstrap_install_from = egg def use_setuptools(): # making sure we use the absolute path return _do_download(os.path.abspath(os.curdir)) def download_setuptools(packagename, to_dir): # making sure we use the absolute path to_dir = os.path.abspath(to_dir) try: from urllib.request import urlopen except ImportError: from urllib2 import urlopen chksum, url = get_pypi_src_download(packagename) tgz_name = os.path.basename(url) saveto = os.path.join(to_dir, tgz_name) src = dst = None if not os.path.exists(saveto): # Avoid repeated downloads try: log.warn("Downloading %s", url) src = urlopen(url) # Read/write all in one block, so we don't create a corrupt file # if the download is interrupted. data = src.read() if chksum is not None: data_sum = md5(data).hexdigest() if data_sum != chksum: raise RuntimeError("Downloading %s failed: corrupt checksum"%(url,)) dst = open(saveto, "wb") dst.write(data) finally: if src: src.close() if dst: dst.close() return os.path.realpath(saveto) def _extractall(self, path=".", members=None): """Extract all members from the archive to the current working directory and set owner, modification time and permissions on directories afterwards. `path' specifies a different directory to extract to. `members' is optional and must be a subset of the list returned by getmembers(). """ import copy import operator from tarfile import ExtractError directories = [] if members is None: members = self for tarinfo in members: if tarinfo.isdir(): # Extract directories with a safe mode. directories.append(tarinfo) tarinfo = copy.copy(tarinfo) tarinfo.mode = 448 # decimal for oct 0700 self.extract(tarinfo, path) # Reverse sort directories. if sys.version_info < (2, 4): def sorter(dir1, dir2): return cmp(dir1.name, dir2.name) directories.sort(sorter) directories.reverse() else: directories.sort(key=operator.attrgetter('name'), reverse=True) # Set correct owner, mtime and filemode on directories. for tarinfo in directories: dirpath = os.path.join(path, tarinfo.name) try: self.chown(tarinfo, dirpath) self.utime(tarinfo, dirpath) self.chmod(tarinfo, dirpath) except ExtractError: e = sys.exc_info()[1] if self.errorlevel > 1: raise else: self._dbg(1, "tarfile: %s" % e) # # # # Definitions of custom commands # # # try: import setuptools except ImportError: use_setuptools() from setuptools import setup try: from distutils.core import PyPIRCCommand except ImportError: PyPIRCCommand = None # Ancient python version from distutils.core import Command from distutils.errors import DistutilsError from distutils import log if PyPIRCCommand is None: class upload_docs (Command): description = "upload sphinx documentation" user_options = [] def initialize_options(self): pass def finalize_options(self): pass def run(self): raise DistutilsError("not supported on this version of python") else: class upload_docs (PyPIRCCommand): description = "upload sphinx documentation" user_options = PyPIRCCommand.user_options def initialize_options(self): PyPIRCCommand.initialize_options(self) self.username = '' self.password = '' def finalize_options(self): PyPIRCCommand.finalize_options(self) config = self._read_pypirc() if config != {}: self.username = config['username'] self.password = config['password'] def run(self): import subprocess import shutil import zipfile import os import urllib import StringIO from base64 import standard_b64encode import httplib import urlparse # Extract the package name from distutils metadata meta = self.distribution.metadata name = meta.get_name() # Run sphinx if os.path.exists('doc/_build'): shutil.rmtree('doc/_build') os.mkdir('doc/_build') p = subprocess.Popen(['make', 'html'], cwd='doc') exit = p.wait() if exit != 0: raise DistutilsError("sphinx-build failed") # Collect sphinx output if not os.path.exists('dist'): os.mkdir('dist') zf = zipfile.ZipFile('dist/%s-docs.zip'%(name,), 'w', compression=zipfile.ZIP_DEFLATED) for toplevel, dirs, files in os.walk('doc/_build/html'): for fn in files: fullname = os.path.join(toplevel, fn) relname = os.path.relpath(fullname, 'doc/_build/html') print ("%s -> %s"%(fullname, relname)) zf.write(fullname, relname) zf.close() # Upload the results, this code is based on the distutils # 'upload' command. content = open('dist/%s-docs.zip'%(name,), 'rb').read() data = { ':action': 'doc_upload', 'name': name, 'content': ('%s-docs.zip'%(name,), content), } auth = "Basic " + standard_b64encode(self.username + ":" + self.password) boundary = '--------------GHSKFJDLGDS7543FJKLFHRE75642756743254' sep_boundary = '\n--' + boundary end_boundary = sep_boundary + '--' body = StringIO.StringIO() for key, value in data.items(): if not isinstance(value, list): value = [value] for value in value: if isinstance(value, tuple): fn = ';filename="%s"'%(value[0]) value = value[1] else: fn = '' body.write(sep_boundary) body.write('\nContent-Disposition: form-data; name="%s"'%key) body.write(fn) body.write("\n\n") body.write(value) body.write(end_boundary) body.write('\n') body = body.getvalue() self.announce("Uploading documentation to %s"%(self.repository,), log.INFO) schema, netloc, url, params, query, fragments = \ urlparse.urlparse(self.repository) if schema == 'http': http = httplib.HTTPConnection(netloc) elif schema == 'https': http = httplib.HTTPSConnection(netloc) else: raise AssertionError("unsupported schema "+schema) data = '' loglevel = log.INFO try: http.connect() http.putrequest("POST", url) http.putheader('Content-type', 'multipart/form-data; boundary=%s'%boundary) http.putheader('Content-length', str(len(body))) http.putheader('Authorization', auth) http.endheaders() http.send(body) except socket.error: e = socket.exc_info()[1] self.announce(str(e), log.ERROR) return r = http.getresponse() if r.status in (200, 301): self.announce('Upload succeeded (%s): %s' % (r.status, r.reason), log.INFO) else: self.announce('Upload failed (%s): %s' % (r.status, r.reason), log.ERROR) print ('-'*75) print (r.read()) print ('-'*75) def recursiveGlob(root, pathPattern): """ Recursively look for files matching 'pathPattern'. Return a list of matching files/directories. """ result = [] for rootpath, dirnames, filenames in os.walk(root): for fn in filenames: if fnmatch(fn, pathPattern): result.append(os.path.join(rootpath, fn)) return result def importExternalTestCases(unittest, pathPattern="test_*.py", root=".", package=None): """ Import all unittests in the PyObjC tree starting at 'root' """ testFiles = recursiveGlob(root, pathPattern) testModules = map(lambda x:x[len(root)+1:-3].replace('/', '.'), testFiles) if package is not None: testModules = [(package + '.' + m) for m in testModules] suites = [] for modName in testModules: try: module = __import__(modName) except ImportError: print("SKIP %s: %s"%(modName, sys.exc_info()[1])) continue if '.' in modName: for elem in modName.split('.')[1:]: module = getattr(module, elem) s = unittest.defaultTestLoader.loadTestsFromModule(module) suites.append(s) return unittest.TestSuite(suites) class test (Command): description = "run test suite" user_options = [ ('verbosity=', None, "print what tests are run"), ] def initialize_options(self): self.verbosity='1' def finalize_options(self): if isinstance(self.verbosity, str): self.verbosity = int(self.verbosity) def cleanup_environment(self): ei_cmd = self.get_finalized_command('egg_info') egg_name = ei_cmd.egg_name.replace('-', '_') to_remove = [] for dirname in sys.path: bn = os.path.basename(dirname) if bn.startswith(egg_name + "-"): to_remove.append(dirname) for dirname in to_remove: log.info("removing installed %r from sys.path before testing"%( dirname,)) sys.path.remove(dirname) def add_project_to_sys_path(self): from pkg_resources import normalize_path, add_activation_listener from pkg_resources import working_set, require self.reinitialize_command('egg_info') self.run_command('egg_info') self.reinitialize_command('build_ext', inplace=1) self.run_command('build_ext') # Check if this distribution is already on sys.path # and remove that version, this ensures that the right # copy of the package gets tested. self.__old_path = sys.path[:] self.__old_modules = sys.modules.copy() ei_cmd = self.get_finalized_command('egg_info') sys.path.insert(0, normalize_path(ei_cmd.egg_base)) sys.path.insert(1, os.path.dirname(__file__)) # Strip the namespace packages defined in this distribution # from sys.modules, needed to reset the search path for # those modules. nspkgs = getattr(self.distribution, 'namespace_packages') if nspkgs is not None: for nm in nspkgs: del sys.modules[nm] # Reset pkg_resources state: add_activation_listener(lambda dist: dist.activate()) working_set.__init__() require('%s==%s'%(ei_cmd.egg_name, ei_cmd.egg_version)) def remove_from_sys_path(self): from pkg_resources import working_set sys.path[:] = self.__old_path sys.modules.clear() sys.modules.update(self.__old_modules) working_set.__init__() def run(self): import unittest # Ensure that build directory is on sys.path (py3k) self.cleanup_environment() self.add_project_to_sys_path() try: meta = self.distribution.metadata name = meta.get_name() test_pkg = name + "_tests" suite = importExternalTestCases(unittest, "test_*.py", test_pkg, test_pkg) runner = unittest.TextTestRunner(verbosity=self.verbosity) result = runner.run(suite) # Print out summary. This is a structured format that # should make it easy to use this information in scripts. summary = dict( count=result.testsRun, fails=len(result.failures), errors=len(result.errors), xfails=len(getattr(result, 'expectedFailures', [])), xpass=len(getattr(result, 'expectedSuccesses', [])), skip=len(getattr(result, 'skipped', [])), ) print("SUMMARY: %s"%(summary,)) finally: self.remove_from_sys_path() # # # # And finally run the setuptools main entry point. # # # metadata = parse_setup_cfg() setup( cmdclass=dict( upload_docs=upload_docs, test=test, ), **metadata )