Skip to content
Snippets Groups Projects
Commit 648eba4f authored by Ryan Gonzalez's avatar Ryan Gonzalez
Browse files

Speed up source package downloads when one is unavailable

Instead of falling back to doing each download individually, this keeps
halving the downloads until it finds the broken one. It generally
results in around 1/4th of the amount of download commands required,
speeding up the build process.
parent fe67ba4a
No related branches found
No related tags found
1 merge request!14Add several features used for the reference runtime
......@@ -179,6 +179,49 @@ def read_built_using(path):
return ret
def install_source_packages(in_chroot, sources, included, missing_sources):
try:
subprocess.check_call(in_chroot + [
'sh', '-euc',
'dir="$1"; shift; mkdir -p "$dir"; cd "$dir"; "$@"',
'sh', # argv[0]
'/src/files', # working directory
'apt-get', '-y', '--download-only', '-q', '-q',
'-oAPT::Get::Only-Source=true', 'source',
] + [s.get_source for s in sources])
except subprocess.CalledProcessError:
if len(sources) == 1:
s = sources[0]
logger.warning('Unable to download source code for %s',
s.get_source)
missing_sources.add(s)
subprocess.call(in_chroot + [
'apt-cache', 'showsrc', s.source,
])
return 1
else:
logger.warning(
'Unable to download %d source(s) as a batch, trying '
'to split them in half', len(sources))
pivot = len(sources) // 2
attempts = 1
for half in sources[:pivot], sources[pivot:]:
if half:
attempts += install_source_packages(
in_chroot, half, included, missing_sources
)
return attempts
else:
included |= set(sources)
return 1
def main():
# type: (...) -> None
parser = argparse.ArgumentParser(
......@@ -271,43 +314,10 @@ def main():
sources.add(s)
get_source.append(s.get_source)
try:
subprocess.check_call(in_chroot + [
'sh', '-euc',
'dir="$1"; shift; mkdir -p "$dir"; cd "$dir"; "$@"',
'sh', # argv[0]
'/src/files', # working directory
'apt-get', '-y', '--download-only', '-q', '-q',
'-oAPT::Get::Only-Source=true', 'source',
] + get_source)
except subprocess.CalledProcessError:
logger.warning(
'Unable to download some sources as a batch, trying '
'to download sources individually')
for s in sources:
try:
subprocess.check_call(in_chroot + [
'sh', '-euc',
'dir="$1"; shift; mkdir -p "$dir"; cd "$dir"; "$@"',
'sh', # argv[0]
'/src/files', # working directory
'apt-get', '-y', '--download-only', '-q', '-q',
'-oAPT::Get::Only-Source=true', 'source',
s.get_source,
])
except subprocess.CalledProcessError:
# Non-fatal for now
logger.warning(
'Unable to get source code for %s', s.source)
missing_sources.add(s)
subprocess.call(in_chroot + [
'apt-cache', 'showsrc', s.source,
])
else:
included.add(s)
else:
included = set(sources)
attempts = install_source_packages(
in_chroot, list(sources), included, missing_sources)
logger.info('Downloaded %d source packages after %d attempt(s)',
len(included), attempts)
parent = args.output or os.path.join(args.sysroot, 'src', 'files')
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment