diff --git a/tools/ade b/tools/ade index 38c63a78971fbfe2ac26f2b935707a5e9b6eebac..3fcdc829376ca7dcf3ee9f9929b142e72b191de8 100755 --- a/tools/ade +++ b/tools/ade @@ -33,6 +33,26 @@ from urllib.request import urlopen, urlretrieve HARD_FLOAT_FLAG = 0x00000400 +def print_progress(count, blockSize, total): + barLength = 50 + status = count * blockSize / total + percents = "{0:.0f}".format(100 * status) + filledLength = int(round(barLength * status)) + bar = '=' * filledLength + ' ' * (barLength - filledLength) + sys.stdout.write('\r%s |%s| %s%s %s' % ("sysroot.tar.gz", bar, percents, '%', "")), + if status >= 1.0: + sys.stdout.write('\n') + sys.stdout.flush() + + +class SysrootManagerError(Exception): + def __init__(self, message): + self.message = message + + def __str__(self): + return self.message + + class InvalidSysrootArchiveError(Exception): def __init__(self, message): self.message = message @@ -102,28 +122,32 @@ class TargetTriplet: class SysrootVersion: - def __init__(self, string, url=None): - self.distro = '' - self.release = '' - self.arch = None - self.date = '' - self.build = 0 - self.author = '' + def __init__(self, distro, release, arch, date=None, build=None, author=None, url=None): + self.distro = distro + self.release = release + self.arch = arch + self.date = date + self.build = build + self.author = author self.url = url - self.parse_string(string) + + def set_url(self, url): if url and not is_valid_url(url): raise ValueError("'url'") + self.url = url - def parse_string(self, string): + def from_string(string): p = re.compile("^\s*(\w*)\s*(\d\d\.\d\d) (\d{8,8})\.(\d+)\s*(\w*)\s*$") m = p.match(string) if not m: raise ValueError("'version'") - self.distro = m.groups()[0] - self.release = m.groups()[1] - self.date = m.groups()[2] - self.build = int(m.groups()[3], 10) - self.author = m.groups()[4] + return SysrootVersion(m.groups()[0], m.groups()[1], None, \ + m.groups()[2], int(m.groups()[3], 10), m.groups()[4]) + + def get_name(self): + return "{0} - {1} ({2})".format(self.distro, + self.release, + self.arch) def get_tag(self): return "{0}-{1}-{2}_{3}.{4}".format(self.distro, @@ -191,7 +215,7 @@ class Sysroot: def parse_path(self, path): try: with open(os.path.join(path, 'etc', 'image_version')) as f: - self.version = SysrootVersion(f.read()) + self.version = SysrootVersion.from_string(f.read()) with open(os.path.join(path, 'usr', 'lib', 'pkg-config.multiarch')) as f: self.version.arch = TargetTriplet(f.read().strip()).arch except FileNotFoundError as e: @@ -225,7 +249,7 @@ class SysrootArchive: try: with f.extractfile(version_file) as reader: - self.version = SysrootVersion(reader.read().decode('utf-8')) + self.version = SysrootVersion.from_string(reader.read().decode('utf-8')) except Exception as e: raise InvalidSysrootArchiveError('Invalid image_version file in archive') @@ -256,192 +280,183 @@ class SysrootArchive: f.extractall(path) -def print_progress(count, blockSize, total): - barLength = 50 - status = count * blockSize / total - percents = "{0:.0f}".format(100 * status) - filledLength = int(round(barLength * status)) - bar = '=' * filledLength + ' ' * (barLength - filledLength) - sys.stdout.write('\r%s |%s| %s%s %s' % ("sysroot.tar.gz", bar, percents, '%', "")), - if status >= 1.0: - sys.stdout.write('\n') - sys.stdout.flush() +class SysrootManager: -class Ade: + def __init__(self, path, url, user, password, config=None): + self.path = path + self.url = url + self.user = user + self.password = password + self.config = config - def __init__(self): - self.command = '' - self.subcommand = '' + if not self.path: + try: + parser = configparser.ConfigParser() + parser.read(config) + self.path = os.path.expanduser(parser['general']['path']) + except: + self.path = '/opt/sysroot/' - self.config = None - self.url = None - self.path = None - self.file = None - self.dest = None + self.password_mgr = None + if self.user: + self.password_mgr = urllib.request.HTTPPasswordMgrWithDefaultRealm() + auth_handler = urllib.request.HTTPBasicAuthHandler(self.password_mgr) + opener = urllib.request.build_opener(auth_handler) + urllib.request.install_opener(opener) - self.distro = None - self.release = None - self.arch = None + # Local operations - self.user = None - self.password = '' - self.password_mgr = None + def get_versions(self): + sysroots = glob.glob(os.path.join(self.path, '*', '[0-9][0-9].[0-9][0-9]', '*')) + versions = [] + for path in sysroots: + try: + sysroot = Sysroot(path) + versions.append(sysroot.version) + except Exception as e: + pass + return versions - self.force = False + def get_installed(self, version): + try: + p = os.path.join(self.path, version.distro, version.release, version.arch) + return Sysroot(p) + except NotInstalledError: + return None + except (FileNotFoundError, ValueError): + raise SysrootManagerError("Invalid sysroot installation at '{0}'".format(p)) - def get_host_distro(self): + def install(self, archive): + version = archive.version try: - f = open('/etc/image_version') - version = SysrootVersion(f.read()) - if not version.distro: - self.die("No distribution name found in version string") - return (version.distro, version.release) - except FileNotFoundError: - self.die("Couldn't find file /etc/image_version") - except: - self.die("Version file isn't matching expected format") + path = os.path.join(self.path, version.distro, version.release, version.arch) + if not os.path.isdir(path): + # XXX Do earlier in case permission is denied or allow restart + os.makedirs(path) + except Exception as e: + raise SysrootManagerError("Couldn't create install directory: {0}".format(e)) - def find_configuration(self): - if not self.config: - self.config = xdg.BaseDirectory.load_first_config('ade', 'sysroot.conf') + try: + archive.extract(path) + except Exception as e: + shutil.rmtree(path) + # FIXME remove all empty directory if possible + raise SysrootManagerError("Couldn't extract sysroot to install path: {0}".format(e)) - def validate_url(self): - if not self.url: - try: - parser = configparser.ConfigParser(interpolation=None) - parser.read(self.config) - try: - self.url = parser[self.distro]['url'] - except KeyError: - self.url = parser['general']['url'] - except (FileNotFoundError, KeyError): - self.die("No URL given to retrieve {0} sysroot" .format(self.distro)) + bindir = os.path.join(path, 'binary') + if os.path.isdir(bindir): + for filename in os.listdir(bindir): + os.rename(os.path.join(bindir, filename), os.path.join(path, filename)) + os.rmdir(bindir) - self.url = self.url.replace('%(distro)', self.distro) \ - .replace('%(release)', self.release) \ - .replace('%(arch)', self.arch) - if not is_valid_url(self.url): - self.die("Invalid URL format: {0}".format(self.url)) + installed = self.get_installed(version) + if not installed.version.is_compatible(version): + raise SysrootManagerError("Mismatch between installed sysroot ({0}) and expected one".format(installed.version)) + # Don't fail on this check as it's broken for current images + if installed.version != version: + logging.warning("Mismatch between installed version and expected one") - def validate_destination(self): - if not self.dest: - self.validate_install_path() - self.dest = os.path.join(self.path, 'downloads') + return installed - def validate_install_path(self): - if not self.path: + def uninstall(self, sysroot): + shutil.rmtree(sysroot.path) + + # Remote operations + + def _add_url(self, url): + if self.password_mgr: + self.password_mgr.add_password(None, url, self.user, self.password) + + def _get_url(self, version): + url = self.url + if not url: try: - parser = configparser.ConfigParser() + parser = configparser.ConfigParser(interpolation=None) parser.read(self.config) - self.path = os.path.expanduser(parser['general']['path']) - except: - self.path = '/opt/sysroot/' + try: + url = parser[version.distro]['url'] + except KeyError: + url = parser['general']['url'] + except (FileNotFoundError, KeyError): + raise SysrootManagerError("No URL given to retrieve {0} sysroot" .format(version.distro)) - def validate_sysroot_id(self): - # XXX If only URL is specified, try to obtain distro and arch from it. - # XXX Allow project-specific config file to specify these info + url = url.replace('%(distro)', version.distro) \ + .replace('%(release)', version.release) \ + .replace('%(arch)', version.arch) + if not is_valid_url(url): + raise SysrootManagerError("Invalid URL format: {0}".format(url)) - if not self.distro: - self.info("* No distribution specified, defaulting to host distribution") - self.distro, release = self.get_host_distro() - if not self.release: - self.info("* No release version specified, defaulting to host release version") - distro, self.release = self.get_host_distro() - if distro != self.distro: - self.die("Mismatch between host distro and specified distro") - if not self.arch: - self.info("* No architecture specified, defaulting to 'armhf'") - self.arch = 'armhf' + self._add_url(url) - def validate_auth(self): - if self.user: - self.password_mgr = urllib.request.HTTPPasswordMgrWithDefaultRealm() - auth_handler = urllib.request.HTTPBasicAuthHandler(self.password_mgr) - opener = urllib.request.build_opener(auth_handler) - urllib.request.install_opener(opener) + return url - def parse_version_file(self, content): + def _parse_version_file(self, content): try: mod_content = "[sysroot]\n" + content parser = configparser.ConfigParser() parser.read_string(mod_content) - return SysrootVersion(parser['sysroot']['version'], parser['sysroot']['url']) + version = SysrootVersion.from_string(parser['sysroot']['version']) + version.set_url(parser['sysroot']['url']) + return version except configparser.ParsingError: - self.die("Invalid syntax for sysroot version file") + raise SysrootManagerError("Invalid syntax for sysroot version file") except KeyError as e: - self.die("Missing {0} property in sysroot version file".format(e)) + raise SysrootManagerError("Missing {0} property in sysroot version file".format(e)) except ValueError as e: - self.die("Malformed {0} property in sysroot version file".format(e)) + raise SysrootManagerError("Malformed {0} property in sysroot version file".format(e)) - def get_latest_version(self): - self.info("* Checking latest version available for {0}{1} {2} ({3}){4}" \ - .format(Colors.OKBLUE, self.distro, self.release, self.arch, Colors.ENDC)) + def get_download_dir(self): + return os.path.join(self.path, 'downloads') + + def get_latest(self, version): try: - self.info("* Downloading version file from: {0}".format(self.url)) - if self.password_mgr: - self.password_mgr.add_password(None, self.url, self.user, self.password) - resp = urlopen(self.url) - version = self.parse_version_file(resp.read().decode('utf-8')) + resp = urlopen(self._get_url(version)) + latest_version = self._parse_version_file(resp.read().decode('utf-8')) # Add distro and arch details if unknown - if not version.distro: - version.distro = self.distro - if not version.arch: - version.arch = self.arch + if not latest_version.distro: + latest_version.distro = version.distro + if not latest_version.arch: + latest_version.arch = version.arch - self.info("* Retrieved latest version: {0}{1}{2}" \ - .format(Colors.OKGREEN, version, Colors.ENDC)) - self.info("* Download URL: {0}".format(version.url)) - - return version + return latest_version except URLError as e: - self.die("Couldn't retrieve sysroot version file: {0}".format(e.reason)) + raise SysrootManagerError("Couldn't retrieve sysroot version file: {0}".format(e.reason)) except UnicodeDecodeError: - self.die("Invalid sysroot version file") + raise SysrootManagerError("Invalid sysroot version file") - def get_installed_sysroot(self): - self.info("* Checking currently installed version") + def download(self, version, dest=None, progress=False): + if not dest: + dest = self.get_download_dir() try: - p = os.path.join(self.path, self.distro, self.release, self.arch) - return Sysroot(p) - except NotInstalledError: - return None - except (FileNotFoundError, ValueError): - self.die("Invalid sysroot installation at '{0}'".format(p)) - - def download_sysroot(self, version): - try: - os.makedirs(self.dest, exist_ok=True) - filename = tempfile.NamedTemporaryFile(dir=self.dest).name + os.makedirs(dest, exist_ok=True) + filename = tempfile.NamedTemporaryFile(dir=dest).name except Exception as e: - self.die("Couldn't create download directory: {0}".format(e)) + raise("Couldn't create download directory: {0}".format(e)) try: - self.info("* Downloading sysroot from: {0}".format(version.url)) hook = None - if self.format == 'friendly': + if progress: hook = print_progress - if self.password_mgr: - self.password_mgr.add_password(None, version.url, self.user, self.password) + self._add_url(version.url) _, headers = urlretrieve(version.url, filename=filename, reporthook=hook) except URLError as e: try: os.remove(filename) except: pass - self.die("Error while downloading sysroot: {0}".format(e.reason)) + raise SysrootManagerError("Error while downloading sysroot: {0}".format(e.reason)) try: - self.info("* Verifying downloaded sysroot version") f = SysrootArchive(filename) except InvalidSysrootArchiveError as e: os.remove(filename) - self.die("Invalid sysroot archive: {0}".format(e.message)) + raise SysrootManagerError("Invalid sysroot archive: {0}".format(e.message)) if not version.is_compatible(f.version): os.remove(filename) - self.die("Mismatch between downloaded sysroot ({0}) and expected one".format(f.version)) + raise SysrootManagerError("Mismatch between downloaded sysroot ({0}) and expected one".format(f.version)) # Don't fail on this check as it's broken for current images if f.version != version: @@ -449,70 +464,73 @@ class Ade: return f - def verify_sysroot(self, path): - self.info("* Verifying sysroot archive '{0}'".format(self.file)) - try: - archive = SysrootArchive(self.file) - new_version = archive.version - self.info("* Sysroot archive has version: {0}{1}{2}".format(Colors.OKGREEN, new_version, Colors.ENDC)) - except Exception as e: - self.die("Invalid sysroot archive: {0}".format(e)) +class Ade: - return archive + def __init__(self): + self.command = '' + self.subcommand = '' - def install_sysroot(self, archive): - try: - self.info("* Creating install directory") - path = os.path.join(self.path, self.distro, self.release, self.arch) - if not os.path.isdir(path): - # XXX Do earlier in case permission is denied or allow restart - os.makedirs(path) - except Exception as e: - self.die("Couldn't create install directory: {0}".format(e)) + self.config = None + self.url = None + self.path = None + self.file = None + self.dest = None + self.sysroot_version = None + self.distro = None + self.release = None + self.arch = None + + self.user = None + self.password = '' + + self.force = False + + def get_host_distro(self): try: - self.info("* Extracting sysroot to '{0}'".format(path)) - archive.extract(path) - except Exception as e: - shutil.rmtree(path) - # FIXME remove all empty directory if possible - self.die("Couldn't extract sysroot to install path: {0}".format(e)) + f = open('/etc/image_version') + version = SysrootVersion.from_string(f.read()) + if not version.distro: + self.die("No distribution name found in version string") + return (version.distro, version.release) + except FileNotFoundError: + self.die("Couldn't find file /etc/image_version") + except: + self.die("Version file isn't matching expected format") - bindir = os.path.join(path, 'binary') - if os.path.isdir(bindir): - self.info("* Moving sysroot files out of the 'binary' directory") - for filename in os.listdir(bindir): - os.rename(os.path.join(bindir, filename), os.path.join(path, filename)) - os.rmdir(bindir) + def find_configuration(self): + if not self.config: + self.config = xdg.BaseDirectory.load_first_config('ade', 'sysroot.conf') - installed = self.get_installed_sysroot() - if not installed.version.is_compatible(archive.version): - self.die("Mismatch between installed sysroot ({0}) and expected one".format(archive.version)) + def validate_sysroot_version(self): + # XXX If only URL is specified, try to obtain distro and arch from it. + # XXX Allow project-specific config file to specify these info - # Don't fail on this check as it's broken for current images - if installed.version != archive.version: - logging.warning("Mismatch between installed version and expected one") + if not self.distro: + self.info("* No distribution specified, defaulting to host distribution") + self.distro, release = self.get_host_distro() + if not self.release: + self.info("* No release version specified, defaulting to host release version") + distro, self.release = self.get_host_distro() + if distro != self.distro: + self.die("Mismatch between host distro and specified distro") + if not self.arch: + self.info("* No architecture specified, defaulting to 'armhf'") + self.arch = 'armhf' - return installed + self.sysroot_version = SysrootVersion(self.distro, self.release, self.arch) - def uninstall_sysroot(self, sysroot): - self.info("* Removing directory '{0}'".format(sysroot.path)) - shutil.rmtree(sysroot.path) + def get_sysroot_manager(self): + return SysrootManager(self.path, self.url, self.user, self.password, config=self.config) def do_sysroot_list(self): - self.validate_install_path() + manager = self.get_sysroot_manager() + versions = manager.get_versions() - sysroots = glob.glob(os.path.join(self.path, '*', '[0-9][0-9].[0-9][0-9]', '*')) - versions = [] - for path in sysroots: - try: - sysroot = Sysroot(path) - versions.append(sysroot.version) - except Exception as e: - pass if not versions: - self.info("{0}No sysroot installed in directory {1}.{2}".format(Colors.WARNING, self.path, Colors.ENDC)) + self.info("{0}No sysroot installed in directory {1}.{2}" + .format(Colors.WARNING, manager.path, Colors.ENDC)) else: versions.sort() for version in versions: @@ -522,35 +540,42 @@ class Ade: print('InstalledSysroots:' + ';'.join(l)) def do_sysroot_installed(self): - self.validate_install_path() - self.validate_sysroot_id() - sysroot = self.get_installed_sysroot() + self.validate_sysroot_version() + + manager = self.get_sysroot_manager() + sysroot = manager.get_installed(self.sysroot_version) if sysroot: self.info("* Retrieved current version: {0}{1}{2}".format(Colors.WARNING, sysroot.version, Colors.ENDC)) if self.format == 'parseable': print('InstalledVersion:' + sysroot.version.get_tag()) else: - self.info("* Sysroot {0}{1} {2} ({3}){4} is not currently installed" - .format(Colors.OKBLUE, self.distro, self.release, self.arch, Colors.ENDC)) + self.info("* Sysroot {0}{1}{2} is not currently installed" + .format(Colors.OKBLUE, self.sysroot_version.get_name(), Colors.ENDC)) def do_sysroot_latest(self): - self.validate_sysroot_id() - self.validate_url() - self.validate_auth() + self.validate_sysroot_version() + + self.info("* Checking latest version available for {0}{1}{2}" \ + .format(Colors.OKBLUE, self.sysroot_version.get_name(), Colors.ENDC)) + manager = self.get_sysroot_manager() + version = manager.get_latest(self.sysroot_version) - version = self.get_latest_version() + self.info("* Retrieved latest version: {0}{1}{2}" \ + .format(Colors.OKGREEN, version, Colors.ENDC)) + self.info("* Download URL: {0}".format(version.url)) if self.format == 'parseable': print('LatestVersion:' + version.get_tag()) print('LatestURL:' + version.url) def do_sysroot_download(self): - self.validate_sysroot_id() - self.validate_url() - self.validate_auth() - self.validate_destination() + self.validate_sysroot_version() - version = self.get_latest_version() - f = self.download_sysroot(version) + manager = self.get_sysroot_manager() + version = manager.get_latest(self.sysroot_version) + f = manager.download(version, self.dest, progress=(self.format == 'friendly')) + + if not self.dest: + self.dest = manager.get_download_dir() template = os.path.join(self.dest, "sysroot-{0}-{1}-{2}_{3}.{4}{5}.tar.gz") tfilename = template.format(version.distro, version.release, version.arch, @@ -569,30 +594,36 @@ class Ade: if self.format == 'parseable': print('DownloadedArchive:' + filename) + def _verify_sysroot_archive(self, path): + self.info("* Verifying sysroot archive '{0}'".format(self.file)) + + try: + archive = SysrootArchive(self.file) + new_version = archive.version + self.info("* Sysroot archive has version: {0}{1}{2}".format(Colors.OKGREEN, new_version, Colors.ENDC)) + except Exception as e: + self.die("Invalid sysroot archive: {0}".format(e)) + + return archive + def do_sysroot_verify(self): - archive = self.verify_sysroot(self.file) + archive = self._verify_sysroot_archive(self.file) if self.format == 'parseable': print('VerifiedVersion:' + archive.version.get_tag()) def do_sysroot_install(self): archive = None + manager = self.get_sysroot_manager() if not self.file: - self.validate_sysroot_id() - self.validate_url() - self.validate_auth() - self.validate_destination() - new_version = self.get_latest_version() + self.validate_sysroot_version() + new_version = manager.get_latest(self.sysroot_version) else: if self.distro or self.release or self.arch: self.die("Incompatible arguments given: --file and --distro/--release/--arch") - archive = self.verify_sysroot(self.file) + archive = self._verify_sysroot_archive(self.file) new_version = archive.version - self.distro = new_version.distro - self.release = new_version.release - self.arch = new_version.arch - self.validate_install_path() - installed = self.get_installed_sysroot() + installed = manager.get_installed(new_version) if not installed: self.info("* Installing version {0}{1}{2}".format(Colors.OKGREEN, new_version, Colors.ENDC)) @@ -609,10 +640,10 @@ class Ade: try: if not self.file: - archive = self.download_sysroot(new_version) + archive = manager.download(new_version, progress=(self.format == 'friendly')) if installed: - self.uninstall_sysroot(installed) - installed = self.install_sysroot(archive) + manager.uninstall(installed) + installed = manager.install(archive) finally: if archive and not self.file: os.remove(archive.filename) @@ -622,18 +653,15 @@ class Ade: print('InstalledVersion:' + installed.version.get_tag()) def do_sysroot_update(self): - self.validate_sysroot_id() - self.validate_url() - self.validate_auth() - self.validate_destination() - new_version = self.get_latest_version() + self.validate_sysroot_version() - self.validate_install_path() - installed = self.get_installed_sysroot() + manager = self.get_sysroot_manager() + new_version = manager.get_latest(self.sysroot_version) + installed = manager.get_installed(self.sysroot_version) if not installed: - self.die("No sysroot currently installed for {0}{1} {2} ({3}){4}" \ - .format(Colors.OKBLUE, self.distro, self.release, self.arch, Colors.ENDC)) + self.die("No sysroot currently installed for {0}{1}{2}" \ + .format(Colors.OKBLUE, self.sysroot_version.get_name(), Colors.ENDC)) elif installed.version < new_version: self.info("* Upgrading from {0}{1}{2} to version {3}{4}{5}" \ .format(Colors.WARNING, installed.version, Colors.ENDC, @@ -653,9 +681,9 @@ class Ade: Colors.OKGREEN, new_version, Colors.ENDC)) try: - archive = self.download_sysroot(new_version) - self.uninstall_sysroot(installed) - installed = self.install_sysroot(archive) + archive = manager.download(new_version, progress=(self.format == 'friendly')) + manager.uninstall(installed) + installed = manager.install(archive) finally: if archive: os.remove(archive.filename) @@ -665,21 +693,21 @@ class Ade: print('InstalledVersion:' + installed.version.get_tag()) def do_sysroot_uninstall(self): - self.validate_sysroot_id() - self.validate_install_path() - installed = self.get_installed_sysroot() + self.validate_sysroot_version() + + manager = self.get_sysroot_manager() + installed = manager.get_installed(self.sysroot_version) if not installed: - self.die("No sysroot currently installed for {0}{1} {2} ({3}){4}" \ - .format(Colors.OKBLUE, self.distro, self.release, self.arch, Colors.ENDC)) - return + self.die("No sysroot currently installed for {0}{1}{2}" \ + .format(Colors.OKBLUE, self.sysroot_version.get_name(), Colors.ENDC)) else: self.info("* Uninstalling sysroot {0}{1}{2}" \ .format(Colors.WARNING, installed.version, Colors.ENDC)) - self.uninstall_sysroot(installed) - self.info("* Sysroot for {0}{1} {2} ({3}){4} has been uninstalled" \ - .format(Colors.OKBLUE, self.distro, self.release, self.arch, Colors.ENDC)) + manager.uninstall(installed) + self.info("* Sysroot for {0}{1}{2} has been uninstalled" \ + .format(Colors.OKBLUE, self.sysroot_version.get_name(), Colors.ENDC)) def info(self, message): if self.format == 'friendly':