Skip to content
Snippets Groups Projects
Commit a0b54dbb authored by Piotr Ożarowski's avatar Piotr Ożarowski
Browse files

dh_python2, pycompile, pyclean: add "namespace" feature:

dh_python2 parses Egg's namespace_packages.txt files (in addition to
--namespace command line argument(s)) and drops empty __init__.py files
from binary package. pycompile will regenerates them at install time and
pyclean will remove them at uninstall time (if they're no longer used in
installed packages
parent 3a743e71
No related branches found
No related tags found
No related merge requests found
......@@ -2,6 +2,12 @@ python-defaults (2.6.6-14) UNRELEASED; urgency=low
* python.mk: add py_builddir macro.
$(call py_builddir, 2.6) returns "build/lib.linux-x86_64-2.6" on amd64
* dh_python2, pycompile, pyclean: add "namespace" feature:
dh_python2 parses Egg's namespace_packages.txt files (in addition to
--namespace command line argument(s)) and drops empty __init__.py files
from binary package. pycompile will regenerates them at install time and
pyclean will remove them at uninstall time (if they're no longer used in
installed packages
* Remove myself from Uploaders
-- Piotr Ożarowski <piotr@debian.org> Sun, 27 Mar 2011 16:29:05 +0200
......
# -*- coding: UTF-8 -*-
# Copyright © 2010 Piotr Ożarowski <piotr@debian.org>
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.
import logging
from os import walk
from os.path import abspath, isfile, join
from subprocess import Popen, PIPE
log = logging.getLogger(__name__)
def from_directory(dname, extensions=('.py',)):
"""Generate *.py file names available in given directory."""
extensions = tuple(extensions) # .endswith doesn't like list
if isinstance(dname, (list, tuple)):
for item in dname:
for fn in from_directory(item):
yield fn
elif isfile(dname) and dname.endswith(extensions):
yield dname
else:
for root, dirs, file_names in walk(abspath(dname)):
for fn in file_names:
if fn.endswith(extensions):
yield join(root, fn)
def from_package(package_name, extensions=('.py',)):
"""Generate *.py file names available in given package."""
extensions = tuple(extensions) # .endswith doesn't like list
process = Popen("/usr/bin/dpkg -L %s" % package_name,\
shell=True, stdout=PIPE)
stdout, stderr = process.communicate()
if process.returncode != 0:
raise Exception("cannot get content of %s" % package_name)
for line in stdout.splitlines():
if line.endswith(extensions):
yield line
def filter_directory(files, dname):
"""Generate *.py file names that match given directory."""
for fn in files:
if fn.startswith(dname):
yield fn
def filter_public(files, versions):
"""Generate *.py file names that match given versions."""
versions_str = set("%d.%d" % i for i in versions)
for fn in files:
if fn.startswith('/usr/lib/python') and \
fn[15:18] in versions_str:
yield fn
def filter_out_ext(files, extensions):
"""Removes files with matching extensions from given generator."""
for fn in files:
if not fn.endswith(extensions):
yield fn
# -*- coding: UTF-8 -*-
# Copyright © 2011 Piotr Ożarowski <piotr@debian.org>
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.
from __future__ import with_statement
import logging
from os import environ, listdir, remove, rmdir
from os.path import dirname, exists, join, getsize
from subprocess import Popen, PIPE
from debpython.pydist import PUBLIC_DIR_RE
from debpython.tools import memoize, sitedir
log = logging.getLogger(__name__)
def parse(fpaths, other=None):
"""Parse namespace_packages.txt files."""
result = set(other or [])
for fpath in fpaths:
with open(fpath, 'r') as fp:
for line in fp:
if line:
result.add(line.strip())
return result
@memoize
def load(package=None):
"""Return a set of namespaces to regenerate/clean.
:param package: limit namespaces to the ones needed by given package
"""
fpaths = None
# DESTDIR is used in tests
nsdir = "%s/usr/share/python/ns/" % environ.get('DESTDIR', '')
if package:
# only one package is processed, no need to load all files
fpath = join(nsdir, package)
if exists(fpath):
fpaths = [fpath]
else:
# load all files
if exists(nsdir):
fpaths = [join(nsdir, i) for i in listdir(nsdir)]
if fpaths:
result = set(i.replace('.', '/') for i in parse(fpaths))
else:
result = set()
return result
def add_namespace_files(files, package=None, action=None):
"""Add __init__.py files to given generator."""
if action is not None:
namespaces = set("/%s" % i for i in load(package))
already_processed = set()
removal_candidates = set()
for fn in files:
yield fn
if action is None:
continue
dpath = dirname(fn)
if dpath not in already_processed:
already_processed.add(dpath)
if PUBLIC_DIR_RE.match(dpath):
for ns in namespaces:
fpath = join(dpath, '__init__.py')
if dpath.endswith(ns):
if action is True:
try:
open(fpath, 'a').close()
except:
log.error('cannot create %s', fpath)
else:
yield fpath
else: # action is False
# postpone it due to dpkg -S call
removal_candidates.add(fpath)
# now deal with to-be-removed namespace candidates (dpkg -S is expensive)
# dpgk -S is used just to be safe (in case some other package is providing
# __init__.py file although it's in /usr/share/python/ns dir)
if action is False and removal_candidates:
process = Popen("/usr/bin/dpkg -S %s 2>/dev/null" % \
' '.join(removal_candidates), shell=True, stdout=PIPE)
# FIXME: len(search_string) > 131072
stdout, stderr = process.communicate()
for line in stdout.splitlines():
ns = line.split(': ', 1)[1]
if ns in removal_candidates:
removal_candidates.remove(ns)
for fpath in removal_candidates:
try:
remove(fpath)
except (IOError, OSError), e:
log.error('cannot remove %s', fpath)
log.debug(e)
else:
yield fpath
def remove_from_package(package, namespaces, versions):
"""Remove empty __init__.py files for requested namespaces."""
if not isinstance(namespaces, set):
namespaces = set(namespaces)
keep = set()
for ns in namespaces:
for version in versions:
fpath = join(sitedir(version, package), *ns.split('.'))
fpath = join(fpath, '__init__.py')
if not exists(fpath):
continue
if getsize(fpath) != 0:
log.warning('file not empty, cannot share %s namespace', ns)
keep.add(ns)
break
# return a set of namespaces that should be handled by pycompile/pyclean
result = namespaces - keep
# remove empty __init__.py files, if available
for ns in result:
for version in versions:
dpath = join(sitedir(version, package), *ns.split('.'))
fpath = join(dpath, '__init__.py')
if exists(fpath):
remove(fpath)
if not listdir(dpath):
rmdir(dpath)
# clean pyshared dir as well
dpath = join('debian', package, 'usr/share/pyshared', *ns.split('.'))
fpath = join(dpath, '__init__.py')
if exists(fpath):
remove(fpath)
if not listdir(dpath):
rmdir(dpath)
return result
......@@ -37,7 +37,7 @@ from debpython.depends import Dependencies
from debpython.version import SUPPORTED, DEFAULT, \
debsorted, getver, vrepr, parse_pycentral_vrange, \
get_requested_versions, parse_vrange, vrange_str
from debpython.namespace import parse as parse_nsp
import debpython.namespace as ns
from debpython.pydist import validate as validate_pydist, \
PUBLIC_DIR_RE
from debpython.tools import sitedir, relative_symlink, \
......@@ -144,10 +144,10 @@ def share(package, stats, options):
stats['public_ext'])
if not versions_without_ext:
log.error('extension for python%s is missing. '
'Build extensions for all supported Python '
'versions (`pyversions -vr`) or adjust '
'X-Python-Version field or pass '
'--no-guessing-versions to dh_python2',
'Build extensions for all supported Python '
'versions (`pyversions -vr`) or adjust '
'X-Python-Version field or pass '
'--no-guessing-versions to dh_python2',
vrepr(version))
exit(3)
srcver = versions_without_ext[0]
......@@ -599,13 +599,19 @@ def main():
fcopy(pydist_file, join(dstdir, package))
# namespace feature - recreate __init__.py files at install time
nsp = parse_nsp(pdetails['nsp.txt'], options.namespaces)
# TODO: skip non-empty __init__.py files, remove empty ones
nsp = ns.parse(stats['nsp.txt'], options.namespaces)
# note that pycompile/pyclean is already added to maintainer scripts
# and it should remain there even if __init__.py was the only .py file
try:
nsp = ns.remove_from_package(package, nsp, stats['public_vers'])
except (IOError, OSError), e:
log.error('cannot remove __init__.py from package: %s', e)
exit(6)
if nsp:
dstdir = join('debian', package, 'usr/share/python/ns/')
if not exists(dstdir):
os.makedirs(dstdir)
with open(join(dstdir, package), 'w') as fp:
with open(join(dstdir, package), 'a') as fp:
fp.writelines("%s\n" % i for i in nsp)
dh.save()
......
......@@ -49,6 +49,12 @@ to override it. If you want dh_python2 to generate more strict dependencies
/usr/share/doc/python-doc/README.PyDist (provided by python-doc package) for
more information.
dh_python2 parses Egg's namespace_packages.txt files (in addition to
--namespace command line argument(s)) and drops empty __init__.py files from
binary package. pycompile will regenerates them at install time and pyclean
will remove them at uninstall time (if they're no longer used in installed
packages.
OPTIONS
=======
--version show program's version number and exit
......@@ -88,6 +94,9 @@ OPTIONS
--suggests=SUGGESTS translate given requirements into Debian dependencies
and add them to ${python:Suggests}
--namespace use this option (multiple time if necessary) if
namespace_packages.txt is not complete
SEE ALSO
========
* /usr/share/doc/python/python-policy.txt.gz
......
......@@ -2,7 +2,7 @@
# -*- coding: UTF-8 -*-
# vim: et ts=4 sw=4
# Copyright © 2010 Piotr Ożarowski <piotr@debian.org>
# Copyright © 2010-2011 Piotr Ożarowski <piotr@debian.org>
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
......@@ -24,11 +24,12 @@
import logging
import optparse
import sys
from os import environ, remove, walk
from os.path import exists, isdir, isfile, join
from subprocess import Popen, PIPE
from os import environ, remove
from os.path import exists
from sys import argv
from debpython import files as dpf
from debpython.namespace import add_namespace_files
# initialize script
logging.basicConfig(format='%(levelname).1s: %(module)s:%(lineno)d: '
......@@ -66,34 +67,9 @@ def destroyer(): # ;-)
log.info("removed files: %s", counter)
def get_files(items):
for item in items:
if isfile(item) and item.endswith('.py'):
yield item
elif isdir(item):
for root, dirs, files in walk(item):
#for fn in glob1(root, '*.py'):
# yield join(root, fn)
for fn in files:
if fn.endswith('.py'):
yield join(root, fn)
def get_package_files(package_name):
process = Popen("/usr/bin/dpkg -L %s" % package_name,\
shell=True, stdout=PIPE)
stdout, stderr = process.communicate()
if process.returncode != 0:
log.error('cannot get content of %s', package_name)
sys.exit(2)
for line in stdout.split('\n'):
if line.endswith('.py'):
yield line
def main():
usage = '%prog [-p PACKAGE | DIR_OR_FILE]'
parser = optparse.OptionParser(usage, version='%prog 0.9')
parser = optparse.OptionParser(usage, version='%prog 1.0')
parser.add_option('-v', '--verbose', action='store_true', dest='verbose',
help='turn verbose more one')
parser.add_option('-q', '--quiet', action='store_false', dest='verbose',
......@@ -101,11 +77,11 @@ def main():
parser.add_option('-p', '--package',
help='specify Debian package name to clean')
(options, args) = parser.parse_args()
options, args = parser.parse_args()
if options.verbose or environ.get('PYCLEAN_DEBUG') == '1':
log.setLevel(logging.DEBUG)
log.debug('argv: %s', sys.argv)
log.debug('argv: %s', argv)
log.debug('options: %s', options)
log.debug('args: %s', args)
else:
......@@ -120,15 +96,21 @@ def main():
if options.package:
log.info('cleaning package %s', options.package)
for filename in get_package_files(options.package):
files = dpf.from_package(options.package, extensions=('.py', '.so'))
files = add_namespace_files(files, options.package, action=False)
files = dpf.filter_out_ext(files, ('.so',))
for filename in files:
d.send(filename)
elif args:
log.info('cleaning directories: %s', args)
for filename in get_files(args):
files = dpf.from_directory(args, extensions=('.py', '.so'))
files = add_namespace_files(files, action=False)
files = dpf.filter_out_ext(files, ('.so',))
for filename in files:
d.send(filename)
else:
parser.print_usage()
sys.exit(1)
exit(1)
if __name__ == '__main__':
main()
......@@ -2,7 +2,7 @@
# -*- coding: utf-8 -*-
# vim: et ts=4 sw=4
# Copyright © 2010 Piotr Ożarowski <piotr@debian.org>
# Copyright © 2010-2011 Piotr Ożarowski <piotr@debian.org>
# Copyright © 2010 Canonical Ltd
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
......@@ -28,12 +28,15 @@ import logging
import optparse
import os
import sys
from os import environ, listdir, walk
from os.path import abspath, exists, isdir, isfile, islink, join
from os import environ, listdir
from os.path import exists, isdir, islink, join
from subprocess import PIPE, STDOUT, Popen
sys.path.insert(1, '/usr/share/python/')
from debpython.version import SUPPORTED, debsorted, vrepr, \
get_requested_versions, parse_vrange, getver
from debpython import files as dpf
from debpython.namespace import add_namespace_files
from debpython.option import Option, compile_regexpr
from debpython.pydist import PUBLIC_DIR_RE
from debpython.tools import memoize
......@@ -55,50 +58,6 @@ Examples:
"""
### FILES ######################################################
def get_directory_files(dname):
"""Generate *.py file names available in given directory."""
if isfile(dname) and dname.endswith('.py'):
yield dname
else:
for root, dirs, file_names in walk(abspath(dname)):
#if root != dname and not exists(join(root, '__init__.py')):
# del dirs[:]
# continue
for fn in file_names:
if fn.endswith('.py'):
yield join(root, fn)
def get_package_files(package_name):
"""Generate *.py file names available in given package."""
process = Popen("/usr/bin/dpkg -L %s" % package_name,\
shell=True, stdout=PIPE)
stdout, stderr = process.communicate()
if process.returncode != 0:
log.error('cannot get content of %s', package_name)
exit(2)
for line in stdout.split('\n'):
if line.endswith('.py'):
yield line
def get_private_files(files, dname):
"""Generate *.py file names that match given directory."""
for fn in files:
if fn.startswith(dname):
yield fn
def get_public_files(files, versions):
"""Generate *.py file names that match given versions."""
versions_str = set("%d.%d" % i for i in versions)
for fn in files:
if fn.startswith('/usr/lib/python') and \
fn[15:18] in versions_str:
yield fn
### EXCLUDES ###################################################
@memoize
def get_exclude_patterns_from_dir(name='/usr/share/python/bcep/'):
......@@ -224,7 +183,7 @@ def compile(files, versions, force, optimize, e_patterns=None):
def main():
usage = '%prog [-V [X.Y][-][A.B]] DIR_OR_FILE [-X REGEXPR]\n' + \
' %prog -p PACKAGE'
parser = optparse.OptionParser(usage, version='%prog 0.9',
parser = optparse.OptionParser(usage, version='%prog 1.0',
option_class=Option)
parser.add_option('-v', '--verbose', action='store_true', dest='verbose',
help='turn verbose mode on')
......@@ -280,7 +239,7 @@ multiple times to build up a list of things to exclude.')
compile_versions = debsorted(versions)[:1]
log.debug('compile versions: %s', versions)
pkg_files = tuple(get_package_files(options.package))
pkg_files = tuple(dpf.from_package(options.package))
for item in args:
e_patterns = get_exclude_patterns(item, options.regexpr, \
compile_versions)
......@@ -289,22 +248,28 @@ multiple times to build up a list of things to exclude.')
else:
log.debug('byte compiling %s using Python %s',
item, compile_versions)
files = get_private_files(pkg_files, item)
files = dpf.filter_directory(pkg_files, item)
compile(files, compile_versions, options.force,
options.optimize, e_patterns)
elif options.package: # package's public modules
# no need to limit versions here, it's either pyr mode or version is
# hardcoded in path / via -V option
# no need to limit versions here, version is hardcoded in path or
# via -V option
e_patterns = get_exclude_patterns()
files = get_package_files(options.package)
files = get_public_files(files, versions)
files = dpf.from_package(options.package, extensions=('.py', '.so'))
files = dpf.filter_public(files, versions)
files = add_namespace_files(files, options.package, action=True)
files = dpf.filter_out_ext(files, ('.so',))
compile(files, versions,
options.force, options.optimize, e_patterns)
elif args: # other directories/files (public ones mostly)
versions = debsorted(versions)[:1]
for item in args:
e_patterns = get_exclude_patterns(item, options.regexpr, versions)
files = get_directory_files(item)
files = dpf.from_directory(item, extensions=('.py', '.so'))
files = list(files); print files
files = add_namespace_files(files, action=True)
files = list(files); print files
files = dpf.filter_out_ext(files, ('.so',))
compile(files, versions,
options.force, options.optimize, e_patterns)
else:
......@@ -314,12 +279,14 @@ multiple times to build up a list of things to exclude.')
# wait for all processes to finish
rv = 0
for process in WORKERS.itervalues():
(child_output, child_unused) = process.communicate()
child_output, child_unused = process.communicate()
if process.returncode not in (None, 0):
# FIXME: find out the package the file belongs to
sys.stderr.write(child_output)
rv = process.returncode
sys.exit(rv)
if rv != 0:
rv += 100
exit(rv)
if __name__ == '__main__':
main()
#!/usr/bin/make -f
# enable or disable tests here:
TESTS := test1 test2 test3 test4
TESTS := test1 test2 test3 test4 test5
all: $(TESTS)
......
#!/usr/bin/make -f
all: run check
run: clean
dpkg-buildpackage -b -us -uc
check:
# test dh_python2
test -f debian/python-foo/usr/share/pyshared/keep_this_one/__init__.py
test ! -f debian/python-foo/usr/share/pyshared/remove_this_one/__init__.py
test ! -f debian/python-foo/usr/share/pyshared/foo/__init__.py
grep -q remove_this_one debian/python-foo/usr/share/python/ns/python-foo
grep -q foo debian/python-foo/usr/share/python/ns/python-foo
grep -q bar.baz debian/python-foo/usr/share/python/ns/python-foo
grep -q keep_this_one debian/python-foo/usr/share/python/ns/python-foo && false || true
grep -q "pycompile -p python-foo" debian/python-foo/DEBIAN/postinst
grep -q "pyclean -p python-foo" debian/python-foo/DEBIAN/prerm
# test pycompile
DESTDIR=debian/python-foo/ ../../pycompile -v debian/python-foo/usr/lib/
[ `ls debian/python-foo/usr/lib/python2.*/*-packages/remove_this_one/__init__.py | wc -l` != '0' ]
[ `ls debian/python-foo/usr/lib/python2.*/*-packages/remove_this_one/__init__.pyc | wc -l` != '0' ]
# test pyclean
DESTDIR=debian/python-foo/ ../../pyclean -v debian/python-foo/usr/lib/
[ `ls debian/python-foo/usr/lib/python2.*/*-packages/remove_this_one/__init__.py 2>/dev/null || true | wc -l` = 0 ]
[ `ls debian/python-foo/usr/lib/python2.*/*-packages/remove_this_one/__init__.pyc 2>/dev/null || true | wc -l` = 0 ]
clean:
./debian/rules clean
foo (0.1.1) unstable; urgency=low
* Initial release
-- Piotr Ożarowski <piotr@debian.org> Sun, 27 Mar 2011 21:09:27 +0200
7
Source: foo
Section: python
Priority: optional
Maintainer: Piotr Ożarowski <piotr@debian.org>
Build-Depends: debhelper (>= 7.0.50~)
Build-Depends-Indep: python
Standards-Version: 3.9.1
Package: python-foo
Architecture: all
Depends: ${python:Depends}, ${misc:Depends}
Description: example 5 - namespace feature
exemple package #5 - dropping __init__.py file from binary package and
recreating it at install time (and removing at in remove time)
The Debian packaging is © 2011, Piotr Ożarowski <piotr@debian.org> and
is licensed under the MIT License.
/usr/share/pyshared/foo.egg-info
/usr/share/pyshared/keep_this_one
/usr/share/pyshared/remove_this_one
#!/usr/bin/make -f
%:
dh $@ --buildsystem=python_distutils
override_dh_auto_build:
override_dh_auto_install:
set -e;\
cd debian/python-foo/usr/share/pyshared/;\
echo "keep_this_one\nremove_this_one" > foo.egg-info/namespace_packages.txt;\
echo "True" > keep_this_one/__init__.py;\
touch remove_this_one/__init__.py remove_this_one/foo.py
override_dh_pysupport:
DH_VERBOSE=1 ../../dh_python2 --namespace foo --namespace bar.baz
clean:
dh_clean
3.0 (native)
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment