dh_python2

#! /usr/bin/python
# -*- coding: UTF-8 -*- vim: et ts=4 sw=4

# Copyright © 2010 Piotr Ożarowski <piotr@debian.org>
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.

from __future__ import with_statement
import logging
import os
import re
import sys
from filecmp import dircmp, cmpfiles
from optparse import OptionParser, SUPPRESS_HELP
from os.path import isdir, islink, exists, join
from shutil import rmtree, copy as fcopy
from stat import ST_MODE, S_IXUSR, S_IXGRP, S_IXOTH
sys.path.insert(1, '/usr/share/python/')
from debpython.debhelper import DebHelper
from debpython.version import SUPPORTED, DEFAULT, \
    debsorted, getver, vrepr, parse_pycentral_vrange, \
    get_requested_versions, parse_vrange, vrange_str
from debpython.pydist import guess_dependency, validate as validate_pydist
from debpython.tools import sitedir, relative_symlink, \
                            shebang2pyver
from debpython.option import Option

# initialize script
logging.basicConfig(format='%(levelname).1s: %(module)s:%(lineno)d: '
                           '%(message)s')
log = logging.getLogger('dh_python')
os.umask(022)

EGGnPTH_RE = re.compile(r'(.*?)(-py\d\.\d+)?(.*?)(\.egg-info|\.pth)$')
PUBLIC_DIR_RE = re.compile(r'.*?/usr/lib/python(\d.\d+)/(site|dist)-packages')

"""TODO: move it to manpage
Examples:
    dh_python
    dh_python -V 2.4- # public files only, Python >= 2.4
    dh_python -p python-foo -X 'bar.*' /usr/lib/baz/ # private files in
                                                       python-foo package
"""

# naming conventions used in the file:
# * version - tuple of integers
# * ver - string representation of version
# * vrange - version range, pair of max and min versions
# * fn - file name (without path)
# * fpath - file path


### FILES ######################################################
def fix_locations(package):
    """Move files to the right location."""
    found_versions = {}
    for version in SUPPORTED:
        ver = vrepr(version)
        to_check = [i % ver for i in (\
                    'usr/local/lib/python%s/site-packages',
                    'usr/local/lib/python%s/dist-packages',
                    'var/lib/python-support/python%s',
                    'usr/lib/pymodules/python%s')]
        if version >= (2, 6):
            to_check.append("usr/lib/python%s/site-packages" % ver)
        dstdir = sitedir(version, package)

        for location in to_check:
            srcdir = "debian/%s/%s" % (package, location)
            if isdir(srcdir):
                if ver in found_versions:
                    log.error('files for version %s '
                              'found in two locations:\n %s\n %s',
                              ver, location, found_versions[ver])
                    exit(2)
                log.info('Python %s should install files in %s. '
                         'Did you forget "--install-layout=deb"?',
                         ver, sitedir(version))
                if not isdir(dstdir):
                    os.makedirs(dstdir)
                # TODO: what about relative symlinks?
                log.debug('moving files from %s to %s', srcdir, dstdir)
                os.renames(srcdir, dstdir)
                found_versions[ver] = location

        # do the same with debug locations
        dbg_to_check = ['usr/lib/debug/%s' % i for i in to_check]
        dbg_to_check.append("usr/lib/debug/usr/lib/pyshared/python%s" % ver)
        dstdir = sitedir(version, package, gdb=True)

        for location in to_check:
            srcdir = "debian/%s/%s" % (package, location)
            if isdir(srcdir):
                if not isdir(dstdir):
                    os.makedirs(dstdir)
                log.debug('moving files from %s to %s', srcdir, dstdir)
                os.renames(srcdir, dstdir)


### SHARING FILES ##############################################
def share(package, stats, options):
    """Move files to /usr/share/pyshared/ if possible."""
    if package.endswith('-dbg'):
        # nothing to share in debug packages
        return
    pubvers = debsorted(i for i in stats['public_vers'] if i[0] == 2)
    if len(pubvers) > 1:
        for pos, version1 in enumerate(pubvers):
            dir1 = sitedir(version1, package)
            for version2 in pubvers[pos + 1:]:
                dir2 = sitedir(version2, package)
                dc = dircmp(dir1, dir2)
                share_2x(dir1, dir2, dc)
    elif len(pubvers) == 1:
        # TODO: remove this once file conflicts will not be needed anymore
        move_to_pyshared(sitedir(pubvers[0], package))

    for version in stats['public_ext']:
        create_ext_links(sitedir(version, package))

    if options.guess_versions and pubvers:
        versions = get_requested_versions(options.vrange)
        for version in (i for i in versions if i[0] == 2):
            if version not in pubvers:
                log.debug('guessing files for Python %s', vrepr(version))
                versions_without_ext = debsorted(set(pubvers) -\
                                                 stats['public_ext'])
                if not versions_without_ext:
                    log.error('you most probably have to build extension '
                              'for python%s.', vrepr(version))
                    exit(12)
                srcver = versions_without_ext[0]
                if srcver in stats['public_vers']:
                    stats['public_vers'].add(version)
                share_2x(sitedir(srcver, package), sitedir(version, package))


def move_to_pyshared(dir1):
    # dir1 starts with debian/packagename/usr/lib/pythonX.Y/*-packages/
    debian, package, path = dir1.split('/', 2)
    dstdir = join(debian, package, 'usr/share/pyshared/', \
                  '/'.join(dir1.split('/')[6:]))

    fext = lambda fname: fname.rsplit('.', 1)[-1]

    for i in os.listdir(dir1):
        fpath1 = join(dir1, i)
        if isdir(fpath1):
            if any(fn for fn in os.listdir(fpath1) if fext(fn) != 'so'):
                # at least one file that is not an extension
                move_to_pyshared(join(dir1, i))
        else:
            if fext(i) == 'so':
                continue
            fpath2 = join(dstdir, i)
            if not exists(fpath2):
                if not exists(dstdir):
                    os.makedirs(dstdir)
                os.rename(fpath1, fpath2)
                relative_symlink(fpath2, fpath1)


def create_ext_links(dir1):
    """Create extension symlinks in /usr/lib/pyshared/pythonX.Y.

    These symlinks are used to let dpkg detect file conflicts with
    python-support and python-central packages.
    """

    debian, package, path = dir1.split('/', 2)
    python, _, module_subpath = path[8:].split('/', 2)
    dstdir = join(debian, package, 'usr/lib/pyshared/', python, module_subpath)

    for i in os.listdir(dir1):
        fpath1 = join(dir1, i)
        if isdir(fpath1):
            create_ext_links(fpath1)
        elif i.rsplit('.', 1)[-1] == 'so':
            fpath2 = join(dstdir, i)
            if exists(fpath2):
                continue
            if not exists(dstdir):
                os.makedirs(dstdir)
            relative_symlink(fpath1, join(dstdir, i))


def share_2x(dir1, dir2, dc=None):
    """Move common files to pyshared and create symlinks in original
    locations."""
    debian, package, path = dir2.split('/', 2)
    # dir1 starts with debian/packagename/usr/lib/pythonX.Y/*-packages/
    dstdir = join(debian, package, 'usr/share/pyshared/', \
                  '/'.join(dir1.split('/')[6:]))
    if not exists(dstdir):
        os.makedirs(dstdir)
    if dc is None:  # guess/copy mode
        if not exists(dir2):
            os.makedirs(dir2)
        common_dirs = []
        common_files = []
        for i in os.listdir(dir1):
            if isdir(join(dir1, i)):
                common_dirs.append([i, None])
            else:
                # directories with .so files will be blocked earlier
                common_files.append(i)
    else:
        common_dirs = dc.subdirs.iteritems()
        common_files = dc.common_files
        # dircmp returns common names only, lets check files more carefully...
        common_files = cmpfiles(dir1, dir2, common_files)[0]

    for fn in common_files:
        fpath1 = join(dir1, fn)
        fpath2 = join(dir2, fn)
        fpath3 = join(dstdir, fn)
        # do not touch symlinks created by previous loop or other tools
        if dc and not islink(fpath1):
            # replace with a link to pyshared
            os.rename(fpath1, fpath3)
            relative_symlink(fpath3, fpath1)
        if dc is None:  # guess/copy mode
            if islink(fpath1):
                # ralative links will work as well, it's always the same level
                os.symlink(os.readlink(fpath1), fpath2)
            else:
                if exists(fpath3):
                    # cannot share it, pyshared contains another copy
                    fcopy(fpath1, fpath2)
                else:
                    # replace with a link to pyshared
                    os.rename(fpath1, fpath3)
                    relative_symlink(fpath3, fpath1)
                    relative_symlink(fpath3, fpath2)
        else:
            os.remove(fpath2)
            relative_symlink(fpath3, fpath2)
    for dn, dc in common_dirs:
        share_2x(join(dir1, dn), join(dir2, dn), dc)


### PACKAGE DETAILS ############################################
def scan(package, dname=None):
    """Gather statistics about Python files in given package."""
    r = {'requires.txt': set(),
         'shebangs': set(),
         'public_vers': set(),
         'private_dirs': {},
         'compile': False,
         'public_ext': set()}

    dbg_package = package.endswith('-dbg')

    if not dname:
        proot = "debian/%s" % package
        if dname is False:
            private_to_check = []
        else:
            private_to_check = [i % package for i in
                                ('usr/lib/%s', 'usr/lib/games/%s',
                                'usr/share/%s', 'usr/share/games/%s')]
    else:
        proot = join('debian', package, dname.strip('/'))
        private_to_check = [dname[1:]]

    for root, dirs, file_names in os.walk(proot):
        # ignore Python 3.X locations
        if '/usr/lib/python3' in root or\
           '/usr/local/lib/python3' in root:
            # warn only once
            warn = root[root.find('/lib/python'):].count('/') == 2
            if warn:
                log.warning('Python 3.x location detected, '
                            'please use dh_python3: %s', root)
            continue

        bin_dir = private_dir = None
        public_dir = PUBLIC_DIR_RE.match(root)
        if public_dir:
            version = getver(public_dir.group(1))
            if root.endswith('-packages'):
                r['public_vers'].add(version)
        else:
            version = False
            for i in private_to_check:
                if root.startswith(join('debian', package, i)):
                    private_dir = '/' + i
                    break
            else:  # i.e. not public_dir and not private_dir
                if len(root.split('/', 6)) < 6 and (\
                   root.endswith('/bin') or root.endswith('/usr/games')):
                   # /bin or /usr/bin or /usr/games
                    bin_dir = root

        # handle some EGG related data (.egg-info dirs)
        for name in dirs:
            match = EGGnPTH_RE.match(name)
            if match:
                if dbg_package:
                    rmtree(join(root, name))
                    dirs.pop(dirs.index(name))
                    continue
                if match.group(2) is not None:
                    new_name = ''.join(match.group(1, 3, 4))
                    log.debug('renaming %s to %s', name, new_name)
                    os.rename(join(root, name), join(root, new_name))
        if root.endswith('.egg-info') and 'requires.txt' in file_names:
            r['requires.txt'].add(join(root, 'requires.txt'))
            continue

        # check files
        for fn in file_names:
            fext = fn.rsplit('.', 1)[-1]
            if fext in ('pyc', 'pyo'):
                os.remove(join(root, fn))
                continue
            if public_dir:
                if dbg_package and fext not in ('so', 'h'):
                    os.remove(join(root, fn))
                    continue

            elif private_dir:
                mode = os.stat(join(root, fn))[ST_MODE]
                if mode is S_IXUSR or mode is S_IXGRP or mode is S_IXOTH:
                    res = shebang2pyver(join(root, fn))
                    if res:
                        r['private_dirs'].setdefault(private_dir, {})\
                            .setdefault('shebangs', set()).add(res)

            if public_dir or private_dir:
                if fext == 'so':
                    (r if public_dir else
                     r['private_dirs'].setdefault(private_dir, {}))\
                    ['public_ext'].add(version)
                    continue
                elif fext == 'py':
                    (r if public_dir else
                     r['private_dirs'].setdefault(private_dir, {}))\
                    ['compile'] = True
                    continue

            # .egg-info files
            match = EGGnPTH_RE.match(fn)
            if match:
                if match.group(2) is not None:
                    new_name = ''.join(match.group(1, 3, 4))
                    log.debug('renaming %s to %s', fn, new_name)
                    os.rename(join(root, fn), join(root, new_name))
                continue
            # search for scripts in bin dirs
            if bin_dir:
                fpath = join(root, fn)
                res = shebang2pyver(fpath)
                if res:
                    r['shebangs'].add(res)

    if dbg_package:
        # remove empty directories in -dbg packages
        proot = proot + '/usr/lib'
        for root, dirs, file_names in os.walk(proot, topdown=False):
            if '-packages/' in root and not file_names:
                try:
                    os.rmdir(root)
                except:
                    pass

    log.debug("package %s details = %s", package, r)
    return r


def dependencies(package, stats, options):
    log.debug('generating dependencies for package %s', package)
    depends = []
    recommends = []
    suggests = []
    enhances = []
    rtscripts = []

    tpl = 'python%d.%d-dbg' if package.endswith('-dbg') else 'python%d.%d'
    dep = ' | '.join(tpl % i for i in debsorted(stats['public_vers']))
    if dep:
        depends.append(dep)

    # make sure pycompile binary is available
    if stats['compile']:
        depends.append("python (>= 2.6.5-2~)")

    for interpreter, version in stats['shebangs']:
        if interpreter not in depends:
            depends.append(interpreter)

    for private_dir, details in stats['private_dirs'].iteritems():
        versions = list(v for i, v in details.get('shebangs', []) if v)
        if len(versions) > 1:
            log.error('more than one Python dependency from shebangs'
                      '(%s shebang versions: %s', private_dir, versions)
            exit(13)
        elif len(versions) == 1:  # one hardcoded version
            depends.append("python%d.%d" % versions[0])
            # TODO: if versions[0] not in requested_versions: FTBFS
        elif details.get('compile', False):
            # no hardcoded versions, but there's something to compile
            args = ''
            vr = options.vrange
            if vr:
                args += "-V %s" % vrange_str(vr)
                if vr[0]:  # minumum version specified
                    depends.append("python (>= %s)" % vrepr(vr[0]))
                if vr[1]:  # maximum version specified
                    depends.append("python (<< %s)" % vrepr(vr[1]))

            for pattern in options.regexpr or []:
                args += " -X '%s'" % pattern.replace("'", r"\'")
            rtscripts.append((private_dir, args))

    if options.guess_deps:
        for fn in stats['requires.txt']:
            for i in parse_pydep(fn):
                depends.append(i)

    log.debug('D=%s; R=%s; S=%s; E=%s, RT=%s', depends, recommends, \
              suggests, enhances, rtscripts)
    return depends, recommends, suggests, enhances, rtscripts


def parse_pydep(fname):
    public_dir = PUBLIC_DIR_RE.match(fname)
    if public_dir:
        ver = public_dir.group(1)
    else:
        ver = None

    result = []
    with open(fname, 'r') as fp:
        for line in fp:
            line = line.strip()
            # ignore all optional sections
            if line.startswith('['):
                break
            if line:
                dependency = guess_dependency(line, ver)
                if dependency:
                    result.append(dependency)
    return result


################################################################
def main():
    usage = '%prog -p PACKAGE [-V [X.Y][-][A.B]] DIR_OR_FILE [-X REGEXPR]\n'
    parser = OptionParser(usage, version='%prog 2.0~beta1',
                                   option_class=Option)
    parser.add_option('--no-guessing-versions', action='store_false',
        dest='guess_versions', default=True,
        help='disables guessing other supported Python versions')
    parser.add_option('--no-guessing-deps', action='store_false',
        dest='guess_deps', default=True, help='disables guessing dependencies')
    parser.add_option('--skip-private', action='store_true',
        dest='skip_private', default=False,
        help='don\'t check private directories')
    parser.add_option('-v', '--verbose', action='store_true', dest='verbose',
        default=False, help='turn verbose mode one')
    # arch=False->arch:all only, arch=True->arch:any only, None->all of them
    parser.add_option('-i', '--indep', action='store_false',
        dest='arch', default=None,
        help='Act on architecture independent packages')
    parser.add_option('-a', '--arch', action='store_true',
        dest='arch', help='Act on architecture dependent packages')
    parser.add_option('-q', '--quiet', action='store_false', dest='verbose',
        default=True, help='be quiet')
    parser.add_option('-p', '--package', action='append', dest='package',
        help='Act on the package named PACKAGE')
    parser.add_option('-N', '--no-package', action='append', dest='no_package',
        help='Do not act on the specified package')
    parser.add_option('-V', type='version_range', dest='vrange',
        help='specify list of supported Python versions. ' +\
             'See pycompile(1) for examples')
    parser.add_option('-X', '--exclude', action='append', dest='regexpr',
        help='exclude items that match given REGEXPR. You may use this option'
             'multiple times to build up a list of things to exclude.')
    # ignore some debhelper options:
    parser.add_option('-O', help=SUPPRESS_HELP)

    (options, args) = parser.parse_args()
    # regexpr option type is not used so lets check patterns here
    for pattern in options.regexpr or []:
        # fail now rather than at runtime
        try:
            pattern = re.compile(pattern)
        except:
            log.error('regular expression is not valid: %s', pattern)
            exit(1)

    if not options.vrange and exists('debian/pyversions'):
        log.debug('parsing version range from debian/pyversions')
        with open('debian/pyversions') as fp:
            for line in fp:
                line = line.strip()
                if line:
                    options.vrange = parse_vrange(line)
                    break

    private_dir = None if not args else args[0]
    # TODO: support more than one private dir at the same time (see :meth:scan)
    if options.skip_private:
        private_dir = False

    if options.verbose:
        log.setLevel(logging.INFO)
    if os.environ.get('DH_VERBOSE') == '1':
        log.setLevel(logging.DEBUG)
        log.debug('argv: %s', sys.argv)
        log.debug('options: %s', options)
        log.debug('args: %s', args)

    dh = DebHelper(options.package, options.no_package)
    if not options.vrange and dh.python_version:
        options.vrange = parse_pycentral_vrange(dh.python_version)

    for package, pdetails in dh.packages.iteritems():
        if options.arch is False and pdetails['arch'] != 'all' or \
           options.arch is True and pdetails['arch'] == 'all':
            continue
        log.debug('processing package %s...', package)
        fix_locations(package)
        stats = scan(package, private_dir)
        share(package, stats, options)

        dep, rec, sug, enh, rts = dependencies(package, stats, options)
        for i in dep:
            dh.addsubstvar(package, 'python:Depends', i)
        for i in rec:
            dh.addsubstvar(package, 'python:Recommends', i)
        for i in sug:
            dh.addsubstvar(package, 'python:Suggests', i)
        for i in enh:
            dh.addsubstvar(package, 'python:Enhances', i)
        for i in rts:
            dh.add_rtupdate(package, i)

        if stats['public_vers']:
            dh.addsubstvar(package, 'python:Versions', \
                           ', '.join(sorted(vrepr(stats['public_vers']))))
            ps = package.split('-', 1)
            if len(ps) > 1 and ps[0] == 'python':
                dh.addsubstvar(package, 'python:Provides', \
                           ', '.join("python%s-%s" % (i, ps[1])\
                           for i in sorted(vrepr(stats['public_vers']))))

        pyclean_added = False  # invoke pyclean only once in maintainer script
        if stats['compile']:
            dh.autoscript(package, 'postinst', 'postinst-pycompile', '')
            dh.autoscript(package, 'prerm', 'prerm-pyclean', '')
            pyclean_added = True

        for pdir, details in stats['private_dirs'].iteritems():
            if not details.get('compile'):
                continue
            if not pyclean_added:
                dh.autoscript(package, 'prerm', 'prerm-pyclean', '')
                pyclean_added = True

            args = pdir

            ext_for = details.get('public_ext')
            if ext_for is None:  # no extension
                if options.vrange:
                    args += " -V %s" % vrange_str(options.vrange)
            elif ext_for is False:  # extension's version not detected
                if options.vrange and '-' not in vrange_str(options.vrange):
                    ver = vrange_str(options.vrange)
                else:  # try shebang or default Python version
                    ver = (list(v for i, v in details.get('shebangs', [])
                           if v) or [None])[0] or DEFAULT
                args += " -V %s" % vrepr(ver)
            else:
                args += " -V %s" % vrepr(ext_for.pop())

            for pattern in options.regexpr or []:
                args += " -X '%s'" % pattern.replace("'", r"\'")

            dh.autoscript(package, 'postinst', 'postinst-pycompile', args)

        pydist_file = join('debian', "%s.pydist" % package)
        if exists(pydist_file):
            if not validate_pydist(pydist_file, True):
                log.warning("%s.pydist file is invalid", package)
            else:
                dstdir = join('debian', package, 'usr/share/python/dist/')
                if not exists(dstdir):
                    os.makedirs(dstdir)
                fcopy(pydist_file, join(dstdir, package))

    dh.save()

if __name__ == '__main__':
    main()