diff --git a/tools/Makefile b/tools/Makefile
index cea70b11f1f43c86c7666fc6b8acb2bc76a6e7b6..10a239b6190b0118e1c52c7f04fba479c06c4925 100644
--- a/tools/Makefile
+++ b/tools/Makefile
@@ -1,8 +1,10 @@
INSTALL ?= install
PREFIX ?= /usr
-ROOT_TOOLDIR ?= $(DESTDIR)$(PREFIX)/bin
+bindir = $(PREFIX)/bin
+ROOT_TOOLDIR ?= $(DESTDIR)$(bindir)
-TOOLS =
+TOOLS = \
+ ade
all:
diff --git a/tools/ade b/tools/ade
new file mode 100755
index 0000000000000000000000000000000000000000..b7d10b01985361f0091cf59a1cf9017ec9d75617
--- /dev/null
+++ b/tools/ade
@@ -0,0 +1,655 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# PYTHON_ARGCOMPLETE_OK
+#
+# Copyright © 2016 Collabora Ltd.
+#
+# SPDX-License-Identifier: MPL-2.0
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+#
+
+import argparse
+import argcomplete
+import configparser
+import glob
+import logging
+import os
+import pathlib
+import re
+import shutil
+import struct
+import sys
+import tarfile
+import tempfile
+
+from urllib.error import URLError
+from urllib.request import urlopen, urlretrieve
+
+HARD_FLOAT_FLAG = 0x00000400
+
+
+class InvalidSysrootArchiveError(Exception):
+ def __init__(self, message):
+ self.message = message
+
+ def __str__(self):
+ return self.message
+
+
+class NotInstalledError(Exception):
+ def __init__(self):
+ pass
+
+
+class NotSupportedError(Exception):
+ def __init__(self):
+ pass
+
+class Colors:
+ HEADER = '\033[95m'
+ OKBLUE = '\033[94m'
+ OKGREEN = '\033[92m'
+ WARNING = '\033[93m'
+ FAIL = '\033[91m'
+ ENDC = '\033[0m'
+
+ force_disable = False
+
+ @classmethod
+ def disable(cls):
+ cls.HEADER = ''
+ cls.OKBLUE = ''
+ cls.OKGREEN = ''
+ cls.WARNING = ''
+ cls.FAIL = ''
+ cls.ENDC = ''
+
+ @classmethod
+ def enable(cls):
+ if cls.force_disable:
+ return
+
+ cls.HEADER = '\033[95m'
+ cls.OKBLUE = '\033[94m'
+ cls.OKGREEN = '\033[92m'
+ cls.WARNING = '\033[93m'
+ cls.FAIL = '\033[91m'
+ cls.ENDC = '\033[0m'
+
+
+class ELFHeader:
+
+ def __init__(self, data):
+ fields = struct.unpack('<4sBBBBB7sHHI', data[0:24])
+
+ # ELF Magic
+ if fields[0] != b'\x7fELF':
+ print("NO MAGIC!")
+ raise ValueError
+
+ # Address size (32 or 64 bits)
+ if fields[1] == 1:
+ self.addr_size = 32
+ elif fields[1] == 2:
+ self.addr_size = 64
+ else:
+ print("Weird address size {0}".format(fields[1]))
+ raise ValueError
+
+ # Instruction Set Architecture
+ if fields[8] == 0x28:
+ self.machine = 'arm'
+ else:
+ print("Not supported machine: {0}".format(fields[8]))
+ raise ValueError
+
+ if self.addr_size == 32:
+ self.flags, = struct.unpack('<I', data[36:40])
+ elif self.addr_size == 64:
+ self.flags, = struct.unpack('<I', data[48:52])
+
+ def get_architecture(self):
+ if self.machine == 'arm' and self.addr_size == 32:
+ if self.flags & HARD_FLOAT_FLAG:
+ return 'armhf'
+ else:
+ return 'armel'
+ elif self.machine == 'arm' and self.addre_size == 64:
+ return 'arm64'
+ else:
+ raise ValueError
+
+
+class SysrootVersion:
+
+ def __init__(self, string=None, url=None, path=None):
+ self.distro = ''
+ self.release = ''
+ self.arch = None
+ self.date = ''
+ self.build = 0
+ self.author = ''
+ self.url = url
+ if string:
+ self.parse_string(string)
+ if path:
+ self.parse_path(path)
+
+ def parse_string(self, string):
+ p = re.compile("^\s*(\w*)\s*(\d\d\.\d\d) (\d{8,8})\.(\d+)\s*(\w*)\s*$")
+ m = p.match(string)
+ if not m:
+ raise ValueError
+ self.distro = m.groups()[0]
+ self.release = m.groups()[1]
+ self.date = m.groups()[2]
+ self.build = int(m.groups()[3], 10)
+ self.author = m.groups()[4]
+
+ def parse_path(self, path):
+ try:
+ with open(os.path.join(path, 'etc', 'image_version')) as f:
+ self.parse_string(f.read())
+ with open(os.path.join(path, 'bin', 'ls'), 'rb') as f:
+ self.arch = ELFHeader(f.read(64)).get_architecture()
+ except FileNotFoundError:
+ if not os.path.exists(path) or not os.listdir(path):
+ raise NotInstalledError
+
+ def __str__(self):
+ return "{0} {1} - {2}.{3} ({4})".format(self.distro,
+ self.release,
+ self.date,
+ self.build,
+ self.arch)
+
+ def is_compatible(self, other):
+ return self.distro == other.distro and \
+ self.release == other.release and \
+ self.arch == other.arch
+
+ def __eq__(self, other):
+ if not other:
+ return False
+ return self.release == other.release and \
+ self.date == other.date and \
+ self.build == other.build
+
+ def __lt__(self, other):
+ if not other:
+ return False
+ if self.release != other.release:
+ return self.release < other.release
+ if self.date != other.date:
+ return self.date < other.date
+ if self.build != other.build:
+ return self.build < other.build
+ return False
+
+ def __gt__(self, other):
+ if not other:
+ return True
+ if self.release != other.release:
+ return self.release > other.release
+ if self.date != other.date:
+ return self.date > other.date
+ if self.build != other.build:
+ return self.build > other.build
+ return False
+
+
+class SysrootArchive:
+
+ def __init__(self, filename):
+ self.filename = filename
+ self.verify()
+
+ def verify(self):
+ with tarfile.open(self.filename, errorlevel=0) as f:
+ # Extract version file and determine sysroot version
+ version_file = None
+ for path in ['etc/image_version', 'binary/etc/image_version']:
+ try:
+ version_file = f.getmember(path)
+ break
+ except KeyError:
+ continue
+ if not version_file:
+ raise InvalidSysrootArchiveError('Missing image_version file in archive')
+
+ try:
+ with f.extractfile(version_file) as reader:
+ self.version = SysrootVersion(reader.read().decode('utf-8'))
+ except Exception as e:
+ print(e)
+ raise InvalidSysrootArchiveError('Invalid image_version file in archive')
+
+ # Extract /bin/ls and determine sysroot architecture
+ ls_file = None
+ for path in ['bin/ls', 'binary/bin/ls']:
+ try:
+ ls_file = f.getmember(path)
+ break
+ except KeyError:
+ continue
+ if not ls_file:
+ raise InvalidSysrootArchiveError('Missing ls executable in archive')
+
+ try:
+ with f.extractfile(ls_file) as reader:
+ header = ELFHeader(reader.read(64))
+ self.version.arch = header.get_architecture()
+ except NotSupportedError:
+ raise InvalidSysrootArchiveError('Architecture is not supported')
+ except:
+ raise InvalidSysrootArchiveError('Invalid ELF header for /bin/ls executable')
+
+ def extract(self, path):
+ with tarfile.open(self.filename, errorlevel=0) as f:
+ f.extractall(path)
+
+
+def print_progress(count, blockSize, total):
+ barLength = 50
+ status = count * blockSize / total
+ percents = "{0:.0f}".format(100 * status)
+ filledLength = int(round(barLength * status))
+ bar = '=' * filledLength + ' ' * (barLength - filledLength)
+ sys.stdout.write('\r%s |%s| %s%s %s' % ("sysroot.tar.gz", bar, percents, '%', "")),
+ if status >= 1.0:
+ sys.stdout.write('\n')
+ sys.stdout.flush()
+
+class Ade:
+
+ def __init__(self):
+ self.command = ''
+ self.subcommand = ''
+
+ self.url = None
+ self.path = None
+ self.file = None
+
+ self.distro = None
+ self.release = None
+ self.arch = None
+
+ self.force = False
+
+ def get_host_distro(self):
+ try:
+ f = open('/etc/image_version')
+ version = SysrootVersion(f.read())
+ if not version.distro:
+ self.die("No distribution name found in version string")
+ return (version.distro, version.release)
+ except FileNotFoundError:
+ self.die("Couldn't find file /etc/image_version")
+ except:
+ self.die("Version file isn't matching expected format")
+
+ def validate_url(self):
+ if not self.url:
+ try:
+ config = configparser.ConfigParser(interpolation=None)
+ config.read('/etc/sysroot.conf')
+ self.url = config[self.distro]['url']
+ except (FileNotFoundError, KeyError):
+ self.die("No URL given to retrieve {0} sysroot" .format(self.distro))
+
+ self.url = self.url.replace('%(distro)', self.distro) \
+ .replace('%(release)', self.release) \
+ .replace('%(arch)', self.arch)
+
+ def validate_destination(self):
+ if not self.dest:
+ self.dest = os.getcwd()
+
+ def validate_install_path(self):
+ if not self.path:
+ try:
+ config = configparser.ConfigParser()
+ config.read('/etc/sysroot.conf')
+ self.path = config['general']['path']
+ except:
+ self.path = '/opt/sysroot/'
+
+ def validate_sysroot_id(self):
+ # XXX If only URL is specified, try to obtain distro and arch from it.
+ # XXX Allow project-specific config file to specify these info
+
+ if not self.distro:
+ print("* No distribution specified, defaulting to host distribution")
+ self.distro, release = self.get_host_distro()
+ if not self.release:
+ print("* No release version specified, defaulting to host release version")
+ distro, self.release = self.get_host_distro()
+ if distro != self.distro:
+ self.die("Mismatch between host distro and specified distro")
+ if not self.arch:
+ print("* No architecture specified, defaulting to 'armhf'")
+ self.arch = 'armhf'
+
+ def parse_version_file(self, content):
+ try:
+ mod_content = "[sysroot]\n" + content
+ config = configparser.ConfigParser()
+ config.read_string(mod_content)
+ return SysrootVersion(config['sysroot']['version'], config['sysroot']['url'])
+ except configparser.ParsingError:
+ self.die("Invalid syntax for sysroot version file")
+ except KeyError:
+ self.die("Missing version property in sysroot version file")
+ except ValueError:
+ self.die("Malformed version property in sysroot version file")
+
+ def get_latest_version(self):
+ print("* Checking latest version available for {0}{1} {2} ({3}){4}" \
+ .format(Colors.OKBLUE, self.distro, self.release, self.arch, Colors.ENDC))
+ try:
+ print("* Downloading version file from: {0}".format(self.url))
+ resp = urlopen(self.url)
+ version = self.parse_version_file(resp.read().decode('utf-8'))
+
+ # Add distro and arch details if unknown
+ if not version.distro:
+ version.distro = self.distro
+ if not version.arch:
+ version.arch = self.arch
+
+ print("* Retrieved latest version: {0}{1}{2}" \
+ .format(Colors.OKGREEN, version, Colors.ENDC))
+
+ return version
+ except URLError as e:
+ self.die("Couldn't retrieve sysroot version file: {0}".format(e.reason))
+ except UnicodeDecodeError:
+ self.die("Invalid sysroot version file")
+
+ def get_installed_version(self):
+ print("* Checking currently installed version")
+ try:
+ p = os.path.join(self.path, self.distro, self.release, self.arch)
+ return SysrootVersion(path=p)
+ except NotInstalledError:
+ return None
+ except ValueError:
+ self.die("Invalid sysroot installation at '{0}'".format(p))
+
+ def download_sysroot(self, version):
+ try:
+ print("* Downloading sysroot from: {0}".format(version.url))
+ filename, headers = urlretrieve(version.url, reporthook=print_progress)
+ except URLError as e:
+ try:
+ os.remove(filename)
+ except:
+ pass
+ self.die("Error while downloading sysroot: {0}".format(e.reason))
+
+ try:
+ print("* Verifying downloaded sysroot version")
+ f = SysrootArchive(filename)
+ except InvalidSysrootArchiveError as e:
+ self.die("Invalid sysroot archive: {0}".format(e.message))
+
+ if not version.is_compatible(f.version):
+ self.die("Mismatch between downloaded sysroot ({0}) and expected one".format(f.version))
+
+ # Don't fail on this check as it's broken for current images
+ if f.version != version:
+ logging.warning("Mismatch between downloaded version and expected one")
+
+ return f
+
+ def verify_sysroot(self, path):
+ print("* Verifying sysroot archive '{0}'".format(self.file))
+
+ try:
+ archive = SysrootArchive(self.file)
+ new_version = archive.version
+ print("* Sysroot archive has version: {0}{1}{2}".format(Colors.OKGREEN, new_version, Colors.ENDC))
+ except Exception as e:
+ self.die("Invalid sysroot archive: {0}".format(e))
+
+ return archive
+
+ def install_sysroot(self, archive):
+ try:
+ print("* Creating install directory")
+ path = os.path.join(self.path, self.distro, self.release, self.arch)
+ if not os.path.isdir(path):
+ # XXX Do earlier in case permission is denied or allow restart
+ os.makedirs(path)
+ except Exception as e:
+ self.die("Couldn't create install directory: {0}".format(e))
+
+ try:
+ print("* Extracting sysroot to '{0}'".format(path))
+ archive.extract(path)
+ except Exception as e:
+ shutil.rmtree(path)
+ # FIXME remove all empty directory if possible
+ self.die("Couldn't extract sysroot to install path: {0}".format(e))
+
+ bindir = os.path.join(path, 'binary')
+ if os.path.isdir(bindir):
+ print("* Moving sysroot files out of the 'binary' directory")
+ for filename in os.listdir(bindir):
+ os.rename(os.path.join(bindir, filename), os.path.join(path, filename))
+ os.rmdir(bindir)
+
+ installed_version = self.get_installed_version()
+ if not installed_version.is_compatible(archive.version):
+ self.die("Mismatch between installed sysroot ({0}) and expected one".format(archive.version))
+
+ # Don't fail on this check as it's broken for current images
+ if installed_version != archive.version:
+ logging.warning("Mismatch between installed version and expected one")
+
+ def uninstall_sysroot(self, version):
+ path = os.path.join(self.path, version.distro, version.release, version.arch)
+ print("* Removing directory '{0}'".format(path))
+ shutil.rmtree(path)
+
+ def do_sysroot_list(self):
+ self.validate_install_path()
+
+ found = False
+ sysroots = glob.glob(os.path.join(self.path, '*', '*', '*'))
+ for sysroot in sysroots:
+ try:
+ version = SysrootVersion(path=sysroot)
+ found = True
+ print("* {0}".format(version))
+ except Exception as e:
+ pass
+ if not found:
+ print("{0}No sysroot installed in directory {1}.{2}".format(Colors.WARNING, self.path, Colors.ENDC))
+
+ def do_sysroot_latest(self):
+ self.validate_sysroot_id()
+ self.validate_url()
+ self.get_latest_version()
+
+ def do_sysroot_download(self):
+ self.validate_sysroot_id()
+ self.validate_url()
+ self.validate_destination()
+
+ version = self.get_latest_version()
+ f = self.download_sysroot(version)
+
+ template = os.path.join(self.dest, "sysroot-{0}-{1}-{2}_{3}.{4}{5}.tar.gz")
+ tfilename = template.format(version.distro, version.release, version.arch,
+ version.date, version.build, "{0}")
+
+ filename = tfilename.format("")
+ if os.path.exists(filename):
+ for i in range(1, 1000):
+ filename = tfilename.format('(' + str(i) + ')')
+ if not os.path.exists(filename):
+ break
+
+ print("* Moving downloaded sysroot to '{0}'".format(filename))
+ shutil.move(f.filename, filename)
+
+ def do_sysroot_verify(self):
+ self.verify_sysroot(self.file)
+
+ def do_sysroot_install(self):
+ archive = None
+ if not self.file:
+ self.validate_sysroot_id()
+ self.validate_url()
+ new_version = self.get_latest_version()
+ else:
+ if self.distro or self.release or self.arch:
+ self.die("Incompatible arguments given: --file and --distro/--release/--arch")
+ archive = self.verify_sysroot(self.file)
+ new_version = archive.version
+ self.distro = new_version.distro
+ self.release = new_version.release
+ self.arch = new_version.arch
+
+ self.validate_install_path()
+ installed_version = self.get_installed_version()
+
+ if not installed_version:
+ print("* Installing version {0}{1}{2}".format(Colors.OKGREEN, new_version, Colors.ENDC))
+ elif self.force:
+ prefix = ''
+ if new_version == installed_version:
+ prefix = 're'
+ print("* Forcing {0}installation of version {1}{2}{3}".format(prefix, Colors.OKGREEN, new_version, Colors.ENDC))
+ else:
+ color = Colors.WARNING
+ if new_version == installed_version:
+ color = Colors.OKGREEN
+ print("* Sysroot {0}{1}{2} is already installed" \
+ .format(color, installed_version, Colors.ENDC))
+ return
+
+ try:
+ if not self.file:
+ archive = self.download_sysroot(new_version)
+ if installed_version:
+ self.uninstall_sysroot(installed_version)
+ self.install_sysroot(archive)
+ finally:
+ if archive and not self.file:
+ os.remove(archive.filename)
+
+ print("* Installation has been completed")
+
+ def do_sysroot_update(self):
+ self.validate_sysroot_id()
+ self.validate_url()
+ new_version = self.get_latest_version()
+
+ self.validate_install_path()
+ installed_version = self.get_installed_version()
+
+ if not installed_version:
+ print("* No sysroot currently installed for {0}{1} {2} ({3}){4}" \
+ .format(Colors.OKBLUE, self.distro, self.release, self.arch, Colors.ENDC))
+ return
+ elif installed_version < new_version:
+ print("* Upgrading from {0}{1}{2} to version {3}{4}{5}" \
+ .format(Colors.WARNING, installed_version, Colors.ENDC,
+ Colors.OKGREEN, new_version, Colors.ENDC))
+ elif self.force:
+ print("* Forcing {0}installation of version {1}{2}{3}" \
+ .format(Colors.OKBLUE, new_version, Colors.ENDC, prefix))
+ elif installed_version == new_version:
+ print("* Installed version {0}{1}{2} is already up-to-date" \
+ .format(Colors.OKGREEN, new_version, Colors.ENDC))
+ return
+ elif installed_version > new_version:
+ print("* Installed version {0}{1}{2} is more recent than {3}{4}{5}" \
+ .format(Colors.WARNING, installed_version, Colors.ENDC,
+ Colors.OKGREEN, new_version, Colors.ENDC))
+ return
+
+ try:
+ archive = self.download_sysroot(new_version)
+ self.uninstall_sysroot(installed_version)
+ self.install_sysroot(archive)
+ finally:
+ if archive:
+ os.remove(archive.filename)
+
+ print("* Update has been completed")
+
+ def do_sysroot_uninstall(self):
+ self.validate_sysroot_id()
+ self.validate_install_path()
+ installed_version = self.get_installed_version()
+
+ if not installed_version:
+ print("* No sysroot currently installed for {0}{1} {2} ({3}){4}" \
+ .format(Colors.OKBLUE, self.distro, self.release, self.arch, Colors.ENDC))
+ return
+ else:
+ print("* Uninstalling sysroot {0}{1}{2}" \
+ .format(Colors.WARNING, installed_version, Colors.ENDC))
+
+ self.uninstall_sysroot(installed_version)
+ print("* Sysroot for {0}{1} {2} ({3}){4} has been uninstalled" \
+ .format(Colors.OKBLUE, self.distro, self.release, self.arch, Colors.ENDC))
+
+ def die(self, message):
+ logging.error(message)
+ sys.exit(1)
+
+ def run(self):
+ method = 'do_' + self.command.replace('-', '_') + '_' + self.subcommand.replace('-', '_')
+ getattr(self, method)()
+
+
+if __name__ == '__main__':
+ root_parser = argparse.ArgumentParser(description='ADE - Apertis Development Environment')
+ argcomplete.autocomplete(root_parser)
+ subparsers = root_parser.add_subparsers(dest='command')
+ subparsers.required = True
+
+ # Sysroot parser
+ sysroot_parser = subparsers.add_parser('sysroot', help='Sysroot related commands')
+ sysroot_parser.add_argument('--path', help="Sysroot installation directory")
+ sysroot_subparsers = sysroot_parser.add_subparsers(dest='subcommand')
+ sysroot_subparsers.required = True
+
+ # Common sysroot parsers
+ sysroot_id_parser = argparse.ArgumentParser(add_help=False)
+ sysroot_id_parser.add_argument('--distro', help="Distribution name (e.g. apertis)")
+ sysroot_id_parser.add_argument('--release', help="Distribution release version (e.g. 16.09)")
+ sysroot_id_parser.add_argument('--arch', help="Sysroot architecture", choices=['armhf', 'arm64'])
+ sysroot_url_parser = argparse.ArgumentParser(add_help=False)
+ sysroot_url_parser.add_argument('--url', help="Sysroot download URL")
+
+ # Sysroot subcommands
+ parser = sysroot_subparsers.add_parser('list', help='List all installed sysroots')
+ parser = sysroot_subparsers.add_parser('latest', help='Retrieve version of latest available sysroot',
+ parents=[sysroot_id_parser, sysroot_url_parser])
+ parser = sysroot_subparsers.add_parser('download', help='Download latest sysroot archive',
+ parents=[sysroot_id_parser, sysroot_url_parser])
+ parser.add_argument('--dest', help="Download destination directory")
+ parser = sysroot_subparsers.add_parser('verify', help='Check sysroot archive for validity')
+ parser.add_argument('--file', required=True, help='Path to the archive file to verify')
+ parser = sysroot_subparsers.add_parser('install', help='Install new sysroot',
+ parents=[sysroot_id_parser, sysroot_url_parser])
+ parser.add_argument('--file', help="Path to (already downloaded) archive to install")
+ parser.add_argument('--force', help="Force sysroot installation", action='store_true')
+ parser = sysroot_subparsers.add_parser('update', help='Update sysroot to latest version',
+ parents=[sysroot_id_parser, sysroot_url_parser])
+ parser.add_argument('--force', help="Force sysroot update", action='store_true')
+ parser = sysroot_subparsers.add_parser('uninstall', help='Uninstall sysroot',
+ parents=[sysroot_id_parser])
+
+ argcomplete.autocomplete(root_parser)
+
+ obj = Ade()
+ root_parser.parse_args(namespace=obj)
+ obj.run()