Add support for authentication when retrieving remote data

New --user and --password options for latest, download, install
and update subcommands.

Signed-off-by: Louis-Francis Ratté-Boulianne <lfrb@collabora.com>
Reviewed-by: Justin Kim <justin.kim@collabora.com>
Differential Revision: https://phabricator.apertis.org/D4920

doc/man/ade-sysroot.1

@@ -54,6 +54,11 @@ these to select the sysroot\&.
 With \-\-url option, it downloads the sysroot version file from <url> and
 get the sysroot information from that file\&.
 .sp
+.SH AUTHENTICATION
+For commands retrieving data from a remote server (\fBlatest\fR, \fBdownload\fR,
+\fBinstall\fR and \fBupdate\fR), the \-\-user and \-\-password options can be
+used to define login information for that server\&.
+.sp
 .SH OPTIONS
 .sp
 .PP

tests/test-download

@@ -15,6 +15,7 @@ import tempfile
 from test_util import SysrootServer
 from test_util import templatedconfig
 from test_util import should_succeed, should_fail
+from test_util import add_auth_params, should_succeed, should_fail
 from test_util import BASE_ARCHIVE, BASE_CONFIG, \
                       SYSROOTS, LATEST_SYSROOTS, \
                       CONFIG_FILES, BAD_URLS
@@ -36,6 +37,7 @@ for sysroot in SYSROOTS:
     with tempfile.TemporaryDirectory() as tmpdir:
         params = ['--url', server.base_url().format(*sysroot), '--distro', sysroot[0],
                   '--release', sysroot[1], '--arch', sysroot[2]]
+        add_auth_params(params, sysroot)
         should_succeed('download', *params, path=tmpdir,
                        check=lambda x: check_archive(x, sysroot))
 
@@ -44,6 +46,7 @@ for sysroot in SYSROOTS:
     with tempfile.TemporaryDirectory() as tmpdir:
         with templatedconfig(server, BASE_CONFIG.format(*sysroot)) as config:
             params = ['--distro', sysroot[0], '--release', sysroot[1], '--arch', sysroot[2]]
+            add_auth_params(params, sysroot)
             should_succeed('download', *params, config=config, path=tmpdir,
                             check=lambda x: check_archive(x, sysroot))
 
@@ -53,6 +56,7 @@ for config_file in CONFIG_FILES:
         for sysroot in LATEST_SYSROOTS:
             with tempfile.TemporaryDirectory() as tmpdir:
                 params = ['--distro', sysroot[0], '--release', sysroot[1], '--arch', sysroot[2]]
+                add_auth_params(params, sysroot)
                 should_succeed('download', *params, config=config, path=tmpdir,
                                check=lambda x: check_archive(x, sysroot))
 

tests/test-install

@@ -15,6 +15,7 @@ import tempfile
 from test_util import SysrootServer
 from test_util import templatedconfig
 from test_util import should_succeed, should_fail
+from test_util import add_auth_params, should_succeed, should_fail
 from test_util import BASE_TAG, BASE_ARCHIVE, BASE_CONFIG, \
                       SYSROOTS, LATEST_SYSROOTS, SYSROOT1, SYSROOT2, \
                       CONFIG_FILES, BAD_URLS
@@ -42,6 +43,7 @@ for sysroot in SYSROOTS:
     with tempfile.TemporaryDirectory() as tmpdir:
         params = ['--url', server.base_url().format(*sysroot), '--distro', sysroot[0],
                   '--release', sysroot[1], '--arch', sysroot[2]]
+        add_auth_params(params, sysroot)
         should_succeed('install', *params, path=tmpdir,
                        check=lambda x: check_sysroot(x, tmpdir, sysroot))
 
@@ -56,6 +58,7 @@ for sysroot in SYSROOTS:
     with tempfile.TemporaryDirectory() as tmpdir:
         with templatedconfig(server, BASE_CONFIG.format(*sysroot)) as config:
             params = ['--distro', sysroot[0], '--release', sysroot[1], '--arch', sysroot[2]]
+            add_auth_params(params, sysroot)
             should_succeed('install', *params, config=config, path=tmpdir,
                            check=lambda x: check_sysroot(x, tmpdir, sysroot))
 
@@ -65,6 +68,7 @@ for config_file in CONFIG_FILES:
         for sysroot in LATEST_SYSROOTS:
             with tempfile.TemporaryDirectory() as tmpdir:
                 params = ['--distro', sysroot[0], '--release', sysroot[1], '--arch', sysroot[2]]
+                add_auth_params(params, sysroot)
                 should_succeed('install', *params, config=config, path=tmpdir,
                                check=lambda x: check_sysroot(x, tmpdir, sysroot))
 

tests/test-latest

@@ -14,6 +14,7 @@ import os
 from test_util import SysrootServer
 from test_util import templatedconfig
 from test_util import should_succeed, should_fail
+from test_util import add_auth_params, should_succeed, should_fail
 from test_util import BASE_TAG, BASE_ARCHIVE, BASE_CONFIG, \
                       SYSROOTS, LATEST_SYSROOTS, \
                       CONFIG_FILES, BAD_VERSION_URLS
@@ -32,6 +33,7 @@ def check_version(result, sysroot):
 for sysroot in SYSROOTS:
     params = ['--url', server.base_url().format(*sysroot), '--distro', sysroot[0],
               '--release', sysroot[1], '--arch', sysroot[2]]
+    add_auth_params(params, sysroot)
     should_succeed('latest', *params,
                    check=lambda x: check_version(x, sysroot))
 
@@ -39,6 +41,7 @@ for sysroot in SYSROOTS:
 for sysroot in SYSROOTS:
     with templatedconfig(server, BASE_CONFIG.format(*sysroot)) as config:
         params = ['--distro', sysroot[0], '--release', sysroot[1], '--arch', sysroot[2]]
+        add_auth_params(params, sysroot)
         should_succeed('latest', *params, config=config,
                        check=lambda x: check_version(x, sysroot))
 
@@ -47,6 +50,7 @@ for config_file in CONFIG_FILES:
     with templatedconfig(server, config_file) as config:
         for sysroot in LATEST_SYSROOTS:
             params = ['--distro', sysroot[0], '--release', sysroot[1], '--arch', sysroot[2]]
+            add_auth_params(params, sysroot)
             should_succeed('latest', *params, config=config,
                            check=lambda x: check_version(x, sysroot))
 

tests/test-update

@@ -15,6 +15,7 @@ import tempfile
 from test_util import SysrootServer
 from test_util import templatedconfig
 from test_util import should_succeed, should_fail
+from test_util import add_auth_params, should_succeed, should_fail
 from test_util import BASE_TAG, BASE_ARCHIVE, BASE_CONFIG, \
                       SYSROOTS, LATEST_SYSROOTS, SYSROOT1, SYSROOT2, \
                       SYSROOT_VERSIONS, CONFIG_FILES, BAD_URLS
@@ -48,6 +49,7 @@ with tempfile.TemporaryDirectory() as tmpdir:
     for sysroot in SYSROOT_VERSIONS:
         params = ['--url', server.base_url().format(*sysroot),
                   '--distro', sysroot[0], '--release', sysroot[1], '--arch', sysroot[2]]
+        add_auth_params(params, sysroot)
         should_succeed('update', *params, path=tmpdir,
                        check=lambda x: check_sysroot(x, tmpdir, sysroot))
 
@@ -56,6 +58,7 @@ with tempfile.TemporaryDirectory() as tmpdir:
     with templatedconfig(server, BASE_CONFIG.format(*SYSROOT2)) as config:
         install_initial_release(tmpdir)
         params = ['--distro', SYSROOT2[0], '--release', SYSROOT2[1], '--arch', SYSROOT2[2]]
+        add_auth_params(params, sysroot)
         should_succeed('update', *params, config=config, path=tmpdir,
                        check=lambda x: check_sysroot(x, tmpdir, SYSROOT2))
 
@@ -65,6 +68,7 @@ for config_file in CONFIG_FILES:
         install_initial_release(tmpdir)
         with templatedconfig(server, config_file) as config:
             params = ['--distro', SYSROOT2[0], '--release', SYSROOT2[1], '--arch', SYSROOT2[2]]
+            add_auth_params(params, sysroot)
             should_succeed('update', *params, config=config, path=tmpdir,
                            check=lambda x: check_sysroot(x, tmpdir, SYSROOT2))
 

tests/test_util.py

@@ -8,6 +8,7 @@
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 #
 
+import base64
 import http.server
 import os
 import socket
@@ -24,6 +25,10 @@ BASE_ARCHIVE = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'files'
 BASE_CONFIG  = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'files', 'configs', "{0}.conf".format(BASE_TAG))
 BASE_INSTALL  = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'files', 'install')
 
+FERMIS_USER = "fermis"
+FERMIS_PASSWORD = "secret"
+FERMIS_KEY = base64.b64encode("{0}:{1}".format(FERMIS_USER, FERMIS_PASSWORD).encode('ascii'))
+
 # All available sysroot versions
 SYSROOTS = [
     ('apertis',   '16.09', 'armhf', '20161031.0'),
@@ -46,6 +51,11 @@ LATEST_SYSROOTS = [
     ('fermis',    '16.12', 'armhf', '20161101.0')
 ]
 
+# Sysroots that need authentication
+PROTECTED_SYSROOTS = [
+    ('fermis',    '16.12', 'armhf', '20161101.0')
+]
+
 # Incrementing version to test updates
 SYSROOT1 = ('apertis', '16.09', 'armhf', '20161031.0')  #First
 SYSROOT2 = ('apertis', '16.09', 'armhf', '20161101.10') #Last
@@ -74,6 +84,8 @@ BAD_VERSION_URLS = [
     'invalidformat',
     'http://HOST/not_found',
     'http://HOST/versions/empty',
+    'http://HOST/fermis/unauthorized',
+    'http://HOST/versions/empty',
     'http://HOST/versions/missing_version',
     'http://HOST/versions/missing_url',
     'http://HOST/versions/invalid_version'
@@ -96,6 +108,10 @@ def run_cmd(command, *args, **kwargs):
     cmd += [command, *args]
     return subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
 
+def add_auth_params(params, sysroot):
+    if sysroot in PROTECTED_SYSROOTS:
+        params += ['--user', FERMIS_USER, '--password', FERMIS_PASSWORD]
+
 def should_succeed(command, *args, check=None, **kwargs):
     result = run_cmd(command, *args, **kwargs)
     if result.returncode:
@@ -128,7 +144,21 @@ def split_elements(string):
 
 # Setup HTTP server
 class SysrootHTTPRequestHandler(http.server.SimpleHTTPRequestHandler):
+    def needs_authentication(self):
+        return 'fermis' in self.path
+
+    def do_AUTHHEAD(self):
+        self.send_response(401)
+        self.send_header('WWW-Authenticate', 'Basic realm=\"Test\"')
+        self.send_header('Content-type', 'text/html')
+        self.end_headers()
+
     def do_GET(self, *args):
+        if self.needs_authentication():
+            key = self.headers.get('Authorization')
+            if not key or key.encode('ascii') != b'Basic ' + FERMIS_KEY:
+                self.do_AUTHHEAD()
+                pass
         super().do_GET(*args)
 
     def log_message(self, *args):

tools/ade

@@ -23,6 +23,7 @@ import struct
 import sys
 import tarfile
 import tempfile
+import urllib
 import xdg.BaseDirectory
 
 from urllib.error import URLError
@@ -281,6 +282,10 @@ class Ade:
         self.release = None
         self.arch = None
 
+        self.user = None
+        self.password = ''
+        self.password_mgr = None
+
         self.force = False
 
     def get_host_distro(self):
@@ -348,6 +353,13 @@ class Ade:
             self.info("* No architecture specified, defaulting to 'armhf'")
             self.arch = 'armhf'
 
+    def validate_auth(self):
+        if self.user:
+            self.password_mgr = urllib.request.HTTPPasswordMgrWithDefaultRealm()
+            auth_handler = urllib.request.HTTPBasicAuthHandler(self.password_mgr)
+            opener = urllib.request.build_opener(auth_handler)
+            urllib.request.install_opener(opener)
+
     def parse_version_file(self, content):
         try:
             mod_content = "[sysroot]\n" + content
@@ -366,6 +378,8 @@ class Ade:
               .format(Colors.OKBLUE, self.distro, self.release, self.arch, Colors.ENDC))
         try:
             self.info("* Downloading version file from: {0}".format(self.url))
+            if self.password_mgr:
+                self.password_mgr.add_password(None, self.url, self.user, self.password)
             resp = urlopen(self.url)
             version = self.parse_version_file(resp.read().decode('utf-8'))
 
@@ -407,6 +421,8 @@ class Ade:
             hook = None
             if self.format == 'friendly':
                 hook = print_progress
+            if self.password_mgr:
+                self.password_mgr.add_password(None, version.url, self.user, self.password)
             _, headers = urlretrieve(version.url, filename=filename, reporthook=hook)
         except URLError as e:
             try:
@@ -520,6 +536,8 @@ class Ade:
     def do_sysroot_latest(self):
         self.validate_sysroot_id()
         self.validate_url()
+        self.validate_auth()
+
         version = self.get_latest_version()
         if self.format == 'parseable':
             print('LatestVersion:' + version.get_tag())
@@ -528,6 +546,7 @@ class Ade:
     def do_sysroot_download(self):
         self.validate_sysroot_id()
         self.validate_url()
+        self.validate_auth()
         self.validate_destination()
 
         version = self.get_latest_version()
@@ -560,6 +579,7 @@ class Ade:
         if not self.file:
             self.validate_sysroot_id()
             self.validate_url()
+            self.validate_auth()
             self.validate_destination()
             new_version = self.get_latest_version()
         else:
@@ -604,6 +624,7 @@ class Ade:
     def do_sysroot_update(self):
         self.validate_sysroot_id()
         self.validate_url()
+        self.validate_auth()
         self.validate_destination()
         new_version = self.get_latest_version()
 
@@ -698,24 +719,27 @@ if __name__ == '__main__':
     sysroot_id_parser.add_argument('--arch', help="Sysroot architecture", choices=['armhf', 'arm64'])
     sysroot_url_parser = argparse.ArgumentParser(add_help=False)
     sysroot_url_parser.add_argument('--url', help="Sysroot download URL")
+    sysroot_auth_parser = argparse.ArgumentParser(add_help=False)
+    sysroot_auth_parser.add_argument('--user', help="Sysroot remote server user")
+    sysroot_auth_parser.add_argument('--password', help="Sysroot remote server password")
 
     # Sysroot subcommands
     parser = sysroot_subparsers.add_parser('list', help='List all installed sysroots')
     parser = sysroot_subparsers.add_parser('installed', help='Retrieve version of currently installed sysroot',
                                            parents=[sysroot_id_parser])
     parser = sysroot_subparsers.add_parser('latest', help='Retrieve version of latest available sysroot',
-                                           parents=[sysroot_id_parser, sysroot_url_parser])
+                                           parents=[sysroot_id_parser, sysroot_url_parser, sysroot_auth_parser])
     parser = sysroot_subparsers.add_parser('download', help='Download latest sysroot archive',
-                                           parents=[sysroot_id_parser, sysroot_url_parser])
+                                           parents=[sysroot_id_parser, sysroot_url_parser, sysroot_auth_parser])
     parser.add_argument('--dest',  help="Download destination directory")
     parser = sysroot_subparsers.add_parser('verify', help='Check sysroot archive for validity')
     parser.add_argument('--file',  required=True, help='Path to the archive file to verify')
     parser = sysroot_subparsers.add_parser('install', help='Install new sysroot',
-                                           parents=[sysroot_id_parser, sysroot_url_parser])
+                                           parents=[sysroot_id_parser, sysroot_url_parser, sysroot_auth_parser])
     parser.add_argument('--file',  help="Path to (already downloaded) archive to install")
     parser.add_argument('--force', help="Force sysroot installation", action='store_true')
     parser = sysroot_subparsers.add_parser('update', help='Update sysroot to latest version',
-                                           parents=[sysroot_id_parser, sysroot_url_parser])
+                                           parents=[sysroot_id_parser, sysroot_url_parser, sysroot_auth_parser])
     parser.add_argument('--force', help="Force sysroot update", action='store_true')
     parser = sysroot_subparsers.add_parser('uninstall', help='Uninstall sysroot',
                                            parents=[sysroot_id_parser])