From 2f576bfb3631b66d4d307dbd1d1523171e029111 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Louis-Francis=20Ratt=C3=A9-Boulianne?= <lfrb@collabora.com>
Date: Thu, 17 Nov 2016 09:59:57 -0500
Subject: [PATCH] Add support for authentication when retrieving remote data
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
New --user and --password options for latest, download, install
and update subcommands.
Signed-off-by: Louis-Francis Ratté-Boulianne <lfrb@collabora.com>
Reviewed-by: Justin Kim <justin.kim@collabora.com>
Differential Revision: https://phabricator.apertis.org/D4920
---
doc/man/ade-sysroot.1 | 5 +++++
tests/test-download | 4 ++++
tests/test-install | 4 ++++
tests/test-latest | 4 ++++
tests/test-update | 4 ++++
tests/test_util.py | 30 ++++++++++++++++++++++++++++++
tools/ade | 32 ++++++++++++++++++++++++++++----
7 files changed, 79 insertions(+), 4 deletions(-)
diff --git a/doc/man/ade-sysroot.1 b/doc/man/ade-sysroot.1
index 5a02c60..76c77d4 100644
--- a/doc/man/ade-sysroot.1
+++ b/doc/man/ade-sysroot.1
@@ -54,6 +54,11 @@ these to select the sysroot\&.
With \-\-url option, it downloads the sysroot version file from <url> and
get the sysroot information from that file\&.
.sp
+.SH AUTHENTICATION
+For commands retrieving data from a remote server (\fBlatest\fR, \fBdownload\fR,
+\fBinstall\fR and \fBupdate\fR), the \-\-user and \-\-password options can be
+used to define login information for that server\&.
+.sp
.SH OPTIONS
.sp
.PP
diff --git a/tests/test-download b/tests/test-download
index eb64a32..b70192e 100755
--- a/tests/test-download
+++ b/tests/test-download
@@ -15,6 +15,7 @@ import tempfile
from test_util import SysrootServer
from test_util import templatedconfig
from test_util import should_succeed, should_fail
+from test_util import add_auth_params, should_succeed, should_fail
from test_util import BASE_ARCHIVE, BASE_CONFIG, \
SYSROOTS, LATEST_SYSROOTS, \
CONFIG_FILES, BAD_URLS
@@ -36,6 +37,7 @@ for sysroot in SYSROOTS:
with tempfile.TemporaryDirectory() as tmpdir:
params = ['--url', server.base_url().format(*sysroot), '--distro', sysroot[0],
'--release', sysroot[1], '--arch', sysroot[2]]
+ add_auth_params(params, sysroot)
should_succeed('download', *params, path=tmpdir,
check=lambda x: check_archive(x, sysroot))
@@ -44,6 +46,7 @@ for sysroot in SYSROOTS:
with tempfile.TemporaryDirectory() as tmpdir:
with templatedconfig(server, BASE_CONFIG.format(*sysroot)) as config:
params = ['--distro', sysroot[0], '--release', sysroot[1], '--arch', sysroot[2]]
+ add_auth_params(params, sysroot)
should_succeed('download', *params, config=config, path=tmpdir,
check=lambda x: check_archive(x, sysroot))
@@ -53,6 +56,7 @@ for config_file in CONFIG_FILES:
for sysroot in LATEST_SYSROOTS:
with tempfile.TemporaryDirectory() as tmpdir:
params = ['--distro', sysroot[0], '--release', sysroot[1], '--arch', sysroot[2]]
+ add_auth_params(params, sysroot)
should_succeed('download', *params, config=config, path=tmpdir,
check=lambda x: check_archive(x, sysroot))
diff --git a/tests/test-install b/tests/test-install
index b4aa011..4dddaf4 100755
--- a/tests/test-install
+++ b/tests/test-install
@@ -15,6 +15,7 @@ import tempfile
from test_util import SysrootServer
from test_util import templatedconfig
from test_util import should_succeed, should_fail
+from test_util import add_auth_params, should_succeed, should_fail
from test_util import BASE_TAG, BASE_ARCHIVE, BASE_CONFIG, \
SYSROOTS, LATEST_SYSROOTS, SYSROOT1, SYSROOT2, \
CONFIG_FILES, BAD_URLS
@@ -42,6 +43,7 @@ for sysroot in SYSROOTS:
with tempfile.TemporaryDirectory() as tmpdir:
params = ['--url', server.base_url().format(*sysroot), '--distro', sysroot[0],
'--release', sysroot[1], '--arch', sysroot[2]]
+ add_auth_params(params, sysroot)
should_succeed('install', *params, path=tmpdir,
check=lambda x: check_sysroot(x, tmpdir, sysroot))
@@ -56,6 +58,7 @@ for sysroot in SYSROOTS:
with tempfile.TemporaryDirectory() as tmpdir:
with templatedconfig(server, BASE_CONFIG.format(*sysroot)) as config:
params = ['--distro', sysroot[0], '--release', sysroot[1], '--arch', sysroot[2]]
+ add_auth_params(params, sysroot)
should_succeed('install', *params, config=config, path=tmpdir,
check=lambda x: check_sysroot(x, tmpdir, sysroot))
@@ -65,6 +68,7 @@ for config_file in CONFIG_FILES:
for sysroot in LATEST_SYSROOTS:
with tempfile.TemporaryDirectory() as tmpdir:
params = ['--distro', sysroot[0], '--release', sysroot[1], '--arch', sysroot[2]]
+ add_auth_params(params, sysroot)
should_succeed('install', *params, config=config, path=tmpdir,
check=lambda x: check_sysroot(x, tmpdir, sysroot))
diff --git a/tests/test-latest b/tests/test-latest
index 57f7df8..f9d4c99 100755
--- a/tests/test-latest
+++ b/tests/test-latest
@@ -14,6 +14,7 @@ import os
from test_util import SysrootServer
from test_util import templatedconfig
from test_util import should_succeed, should_fail
+from test_util import add_auth_params, should_succeed, should_fail
from test_util import BASE_TAG, BASE_ARCHIVE, BASE_CONFIG, \
SYSROOTS, LATEST_SYSROOTS, \
CONFIG_FILES, BAD_VERSION_URLS
@@ -32,6 +33,7 @@ def check_version(result, sysroot):
for sysroot in SYSROOTS:
params = ['--url', server.base_url().format(*sysroot), '--distro', sysroot[0],
'--release', sysroot[1], '--arch', sysroot[2]]
+ add_auth_params(params, sysroot)
should_succeed('latest', *params,
check=lambda x: check_version(x, sysroot))
@@ -39,6 +41,7 @@ for sysroot in SYSROOTS:
for sysroot in SYSROOTS:
with templatedconfig(server, BASE_CONFIG.format(*sysroot)) as config:
params = ['--distro', sysroot[0], '--release', sysroot[1], '--arch', sysroot[2]]
+ add_auth_params(params, sysroot)
should_succeed('latest', *params, config=config,
check=lambda x: check_version(x, sysroot))
@@ -47,6 +50,7 @@ for config_file in CONFIG_FILES:
with templatedconfig(server, config_file) as config:
for sysroot in LATEST_SYSROOTS:
params = ['--distro', sysroot[0], '--release', sysroot[1], '--arch', sysroot[2]]
+ add_auth_params(params, sysroot)
should_succeed('latest', *params, config=config,
check=lambda x: check_version(x, sysroot))
diff --git a/tests/test-update b/tests/test-update
index 01785c0..4d3e64c 100755
--- a/tests/test-update
+++ b/tests/test-update
@@ -15,6 +15,7 @@ import tempfile
from test_util import SysrootServer
from test_util import templatedconfig
from test_util import should_succeed, should_fail
+from test_util import add_auth_params, should_succeed, should_fail
from test_util import BASE_TAG, BASE_ARCHIVE, BASE_CONFIG, \
SYSROOTS, LATEST_SYSROOTS, SYSROOT1, SYSROOT2, \
SYSROOT_VERSIONS, CONFIG_FILES, BAD_URLS
@@ -48,6 +49,7 @@ with tempfile.TemporaryDirectory() as tmpdir:
for sysroot in SYSROOT_VERSIONS:
params = ['--url', server.base_url().format(*sysroot),
'--distro', sysroot[0], '--release', sysroot[1], '--arch', sysroot[2]]
+ add_auth_params(params, sysroot)
should_succeed('update', *params, path=tmpdir,
check=lambda x: check_sysroot(x, tmpdir, sysroot))
@@ -56,6 +58,7 @@ with tempfile.TemporaryDirectory() as tmpdir:
with templatedconfig(server, BASE_CONFIG.format(*SYSROOT2)) as config:
install_initial_release(tmpdir)
params = ['--distro', SYSROOT2[0], '--release', SYSROOT2[1], '--arch', SYSROOT2[2]]
+ add_auth_params(params, sysroot)
should_succeed('update', *params, config=config, path=tmpdir,
check=lambda x: check_sysroot(x, tmpdir, SYSROOT2))
@@ -65,6 +68,7 @@ for config_file in CONFIG_FILES:
install_initial_release(tmpdir)
with templatedconfig(server, config_file) as config:
params = ['--distro', SYSROOT2[0], '--release', SYSROOT2[1], '--arch', SYSROOT2[2]]
+ add_auth_params(params, sysroot)
should_succeed('update', *params, config=config, path=tmpdir,
check=lambda x: check_sysroot(x, tmpdir, SYSROOT2))
diff --git a/tests/test_util.py b/tests/test_util.py
index a1f8fba..1155180 100644
--- a/tests/test_util.py
+++ b/tests/test_util.py
@@ -8,6 +8,7 @@
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
#
+import base64
import http.server
import os
import socket
@@ -24,6 +25,10 @@ BASE_ARCHIVE = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'files'
BASE_CONFIG = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'files', 'configs', "{0}.conf".format(BASE_TAG))
BASE_INSTALL = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'files', 'install')
+FERMIS_USER = "fermis"
+FERMIS_PASSWORD = "secret"
+FERMIS_KEY = base64.b64encode("{0}:{1}".format(FERMIS_USER, FERMIS_PASSWORD).encode('ascii'))
+
# All available sysroot versions
SYSROOTS = [
('apertis', '16.09', 'armhf', '20161031.0'),
@@ -46,6 +51,11 @@ LATEST_SYSROOTS = [
('fermis', '16.12', 'armhf', '20161101.0')
]
+# Sysroots that need authentication
+PROTECTED_SYSROOTS = [
+ ('fermis', '16.12', 'armhf', '20161101.0')
+]
+
# Incrementing version to test updates
SYSROOT1 = ('apertis', '16.09', 'armhf', '20161031.0') #First
SYSROOT2 = ('apertis', '16.09', 'armhf', '20161101.10') #Last
@@ -74,6 +84,8 @@ BAD_VERSION_URLS = [
'invalidformat',
'http://HOST/not_found',
'http://HOST/versions/empty',
+ 'http://HOST/fermis/unauthorized',
+ 'http://HOST/versions/empty',
'http://HOST/versions/missing_version',
'http://HOST/versions/missing_url',
'http://HOST/versions/invalid_version'
@@ -96,6 +108,10 @@ def run_cmd(command, *args, **kwargs):
cmd += [command, *args]
return subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+def add_auth_params(params, sysroot):
+ if sysroot in PROTECTED_SYSROOTS:
+ params += ['--user', FERMIS_USER, '--password', FERMIS_PASSWORD]
+
def should_succeed(command, *args, check=None, **kwargs):
result = run_cmd(command, *args, **kwargs)
if result.returncode:
@@ -128,7 +144,21 @@ def split_elements(string):
# Setup HTTP server
class SysrootHTTPRequestHandler(http.server.SimpleHTTPRequestHandler):
+ def needs_authentication(self):
+ return 'fermis' in self.path
+
+ def do_AUTHHEAD(self):
+ self.send_response(401)
+ self.send_header('WWW-Authenticate', 'Basic realm=\"Test\"')
+ self.send_header('Content-type', 'text/html')
+ self.end_headers()
+
def do_GET(self, *args):
+ if self.needs_authentication():
+ key = self.headers.get('Authorization')
+ if not key or key.encode('ascii') != b'Basic ' + FERMIS_KEY:
+ self.do_AUTHHEAD()
+ pass
super().do_GET(*args)
def log_message(self, *args):
diff --git a/tools/ade b/tools/ade
index 0aa3fd1..629f952 100755
--- a/tools/ade
+++ b/tools/ade
@@ -23,6 +23,7 @@ import struct
import sys
import tarfile
import tempfile
+import urllib
import xdg.BaseDirectory
from urllib.error import URLError
@@ -281,6 +282,10 @@ class Ade:
self.release = None
self.arch = None
+ self.user = None
+ self.password = ''
+ self.password_mgr = None
+
self.force = False
def get_host_distro(self):
@@ -348,6 +353,13 @@ class Ade:
self.info("* No architecture specified, defaulting to 'armhf'")
self.arch = 'armhf'
+ def validate_auth(self):
+ if self.user:
+ self.password_mgr = urllib.request.HTTPPasswordMgrWithDefaultRealm()
+ auth_handler = urllib.request.HTTPBasicAuthHandler(self.password_mgr)
+ opener = urllib.request.build_opener(auth_handler)
+ urllib.request.install_opener(opener)
+
def parse_version_file(self, content):
try:
mod_content = "[sysroot]\n" + content
@@ -366,6 +378,8 @@ class Ade:
.format(Colors.OKBLUE, self.distro, self.release, self.arch, Colors.ENDC))
try:
self.info("* Downloading version file from: {0}".format(self.url))
+ if self.password_mgr:
+ self.password_mgr.add_password(None, self.url, self.user, self.password)
resp = urlopen(self.url)
version = self.parse_version_file(resp.read().decode('utf-8'))
@@ -407,6 +421,8 @@ class Ade:
hook = None
if self.format == 'friendly':
hook = print_progress
+ if self.password_mgr:
+ self.password_mgr.add_password(None, version.url, self.user, self.password)
_, headers = urlretrieve(version.url, filename=filename, reporthook=hook)
except URLError as e:
try:
@@ -520,6 +536,8 @@ class Ade:
def do_sysroot_latest(self):
self.validate_sysroot_id()
self.validate_url()
+ self.validate_auth()
+
version = self.get_latest_version()
if self.format == 'parseable':
print('LatestVersion:' + version.get_tag())
@@ -528,6 +546,7 @@ class Ade:
def do_sysroot_download(self):
self.validate_sysroot_id()
self.validate_url()
+ self.validate_auth()
self.validate_destination()
version = self.get_latest_version()
@@ -560,6 +579,7 @@ class Ade:
if not self.file:
self.validate_sysroot_id()
self.validate_url()
+ self.validate_auth()
self.validate_destination()
new_version = self.get_latest_version()
else:
@@ -604,6 +624,7 @@ class Ade:
def do_sysroot_update(self):
self.validate_sysroot_id()
self.validate_url()
+ self.validate_auth()
self.validate_destination()
new_version = self.get_latest_version()
@@ -698,24 +719,27 @@ if __name__ == '__main__':
sysroot_id_parser.add_argument('--arch', help="Sysroot architecture", choices=['armhf', 'arm64'])
sysroot_url_parser = argparse.ArgumentParser(add_help=False)
sysroot_url_parser.add_argument('--url', help="Sysroot download URL")
+ sysroot_auth_parser = argparse.ArgumentParser(add_help=False)
+ sysroot_auth_parser.add_argument('--user', help="Sysroot remote server user")
+ sysroot_auth_parser.add_argument('--password', help="Sysroot remote server password")
# Sysroot subcommands
parser = sysroot_subparsers.add_parser('list', help='List all installed sysroots')
parser = sysroot_subparsers.add_parser('installed', help='Retrieve version of currently installed sysroot',
parents=[sysroot_id_parser])
parser = sysroot_subparsers.add_parser('latest', help='Retrieve version of latest available sysroot',
- parents=[sysroot_id_parser, sysroot_url_parser])
+ parents=[sysroot_id_parser, sysroot_url_parser, sysroot_auth_parser])
parser = sysroot_subparsers.add_parser('download', help='Download latest sysroot archive',
- parents=[sysroot_id_parser, sysroot_url_parser])
+ parents=[sysroot_id_parser, sysroot_url_parser, sysroot_auth_parser])
parser.add_argument('--dest', help="Download destination directory")
parser = sysroot_subparsers.add_parser('verify', help='Check sysroot archive for validity')
parser.add_argument('--file', required=True, help='Path to the archive file to verify')
parser = sysroot_subparsers.add_parser('install', help='Install new sysroot',
- parents=[sysroot_id_parser, sysroot_url_parser])
+ parents=[sysroot_id_parser, sysroot_url_parser, sysroot_auth_parser])
parser.add_argument('--file', help="Path to (already downloaded) archive to install")
parser.add_argument('--force', help="Force sysroot installation", action='store_true')
parser = sysroot_subparsers.add_parser('update', help='Update sysroot to latest version',
- parents=[sysroot_id_parser, sysroot_url_parser])
+ parents=[sysroot_id_parser, sysroot_url_parser, sysroot_auth_parser])
parser.add_argument('--force', help="Force sysroot update", action='store_true')
parser = sysroot_subparsers.add_parser('uninstall', help='Uninstall sysroot',
parents=[sysroot_id_parser])
--
GitLab