From 2b744bfdbb50fa2e86f3ac26a700c7918d5bbff0 Mon Sep 17 00:00:00 2001 From: Brent Baude Date: Mon, 3 Apr 2017 13:50:40 -0500 Subject: [PATCH] Honor proxy usage If HTTP[S]_PROXY is defined, honor it in python requests usage as well as pass it on to skopeo. If http[s]_proxy is defined in atomic.conf, use it; however, environment variables will override these if defined. Added --insecure to Atomic push so the user can override the logic (or lack thereof) around deducing if a registry is insecure. Also needed for integration tests. Closes: #964 Approved by: rhatdan --- Atomic/atomic.py | 1 + Atomic/pulp.py | 12 +++-- Atomic/push.py | 8 ++- Atomic/satellite.py | 20 ++++--- Atomic/satellite_new.py.test | 20 ++++--- Atomic/trust.py | 3 +- Atomic/util.py | 97 +++++++++++++++------------------- atomic.conf | 6 +++ atomic_dbus.py | 6 ++- bash/atomic | 1 + docs/atomic-push.1.md | 4 ++ tests/integration/test_dbus.py | 10 ++-- 12 files changed, 105 insertions(+), 83 deletions(-) diff --git a/Atomic/atomic.py b/Atomic/atomic.py index 715fe4a..2b1e3a5 100644 --- a/Atomic/atomic.py +++ b/Atomic/atomic.py @@ -63,6 +63,7 @@ class Atomic(object): self.run_opts = None self.atomic_config = util.get_atomic_config() self.local_tokens = {} + util.set_proxy() def __enter__(self): return self diff --git a/Atomic/pulp.py b/Atomic/pulp.py index f484488..5638f7b 100644 --- a/Atomic/pulp.py +++ b/Atomic/pulp.py @@ -72,21 +72,25 @@ class PulpServer(object): self._chunk_size = 1048576 # 1 MB per upload call def _call_pulp(self, url, req_type='get', payload=None): + proxies = util.get_proxy() if req_type == 'get': r = requests.get(url, auth=(self._username, self._password), - verify=self._verify_ssl) + verify=self._verify_ssl, + proxies=proxies) elif req_type == 'post': r = requests.post(url, auth=(self._username, self._password), data=json.dumps(payload), - verify=self._verify_ssl) + verify=self._verify_ssl, + proxies=proxies) elif req_type == 'put': # some calls pass in binary data so we don't log payload data or # json encode it here r = requests.put(url, auth=(self._username, self._password), - data=payload, verify=self._verify_ssl) + data=payload, verify=self._verify_ssl, + proxies=proxies) elif req_type == 'delete': r = requests.delete(url, auth=(self._username, self._password), - verify=self._verify_ssl) + verify=self._verify_ssl, proxies=proxies) else: raise ValueError('Invalid value of "req_type" parameter: {0}' ''.format(req_type)) diff --git a/Atomic/push.py b/Atomic/push.py index 01f44ee..4ed5d07 100644 --- a/Atomic/push.py +++ b/Atomic/push.py @@ -69,6 +69,8 @@ def cli(subparser): "use an alternate user's GPG keyring for signing. " "Useful when running with sudo, " "e.g. set to '~/.gnupg'.")) + pushp.add_argument("--insecure", dest="insecure", default=False, + action='store_true', help=_("Do not check registry certificates")) # pushp.add_argument("--activation_key_name", # default=None, # dest="activation_key_name", @@ -180,8 +182,10 @@ class Push(Atomic): if sign and self.args.debug: util.write_out("\nSigning with '{}'\n".format(self.args.sign_by)) - insecure = True if util.is_insecure_registry(self.d.info()['RegistryConfig'], util.strip_port(reg)) else False - + if self.args.insecure: + insecure = True + else: + insecure = True if util.is_insecure_registry(self.d.info()['RegistryConfig'], util.strip_port(reg)) else False # We must push the file to the registry first prior to performing a # local signature because the manifest file must be on the registry return_code = util.skopeo_copy(local_image, remote_image, debug=self.args.debug, diff --git a/Atomic/satellite.py b/Atomic/satellite.py index 8a41000..564814a 100644 --- a/Atomic/satellite.py +++ b/Atomic/satellite.py @@ -70,11 +70,12 @@ class SatelliteServer(object): def _call_satellite(self, url, req_type='get', payload=None): """This function handles requests to the Satellite Server""" + proxies = util.get_proxy() if req_type == 'get': if (self._debug): print('Calling Satellite URL "{0}"'.format(url)) r = requests.get(url, auth=(self._username, self._password), - verify=self._verify_ssl) + verify=self._verify_ssl, proxies=proxies) elif req_type == 'post': if (self._debug): print('Posting to Satellite URL "{0}"'.format(url)) @@ -83,7 +84,8 @@ class SatelliteServer(object): json.dumps(payload, indent=2))) r = requests.post(url, auth=(self._username, self._password), data=json.dumps(payload), - verify=self._verify_ssl) + verify=self._verify_ssl, + proxies=proxies) elif req_type == 'post-nodata': if (self._debug): print('Posting to Satellite URL "{0}". No data sent.'.format( @@ -91,12 +93,14 @@ class SatelliteServer(object): header = {'Content-Type': 'application/json'} r = requests.post(url, auth=(self._username, self._password), headers=header, data=json.dumps(payload), - verify=self._verify_ssl) + verify=self._verify_ssl, + proxies=proxies) elif req_type == 'put': if self._debug: print('Putting to Satellite URL "{0}"'.format(url)) r = requests.put(url, auth=(self._username, self._password), - data=payload, verify=self._verify_ssl) + data=payload, verify=self._verify_ssl, + proxies=proxies) elif req_type == 'put-jsonHead': if self._debug: print('Putting with json header to Satellite URL "{0}"' @@ -104,7 +108,7 @@ class SatelliteServer(object): header = {'Content-Type': 'application/json'} r = requests.put(url, auth=(self._username, self._password), headers=header, data=json.dumps(payload), - verify=self._verify_ssl) + verify=self._verify_ssl, proxies=proxies) elif req_type == 'put-multi-part': if self._debug: print('Multi-Part Putting to Satellite URL "{0}"'.format(url)) @@ -115,13 +119,15 @@ class SatelliteServer(object): } r = requests.put(url, auth=(self._username, self._password), headers=header, data=payload, - verify=self._verify_ssl) + verify=self._verify_ssl, + proxies=proxies) elif req_type == 'delete': if self._debug: print('Delete call to Satellite URL "{0}"'.format(url)) header = {'Content-Type': 'application/json'} r = requests.delete(url, auth=(self._username, self._password), - headers=header, verify=self._verify_ssl) + headers=header, verify=self._verify_ssl, + proxies=proxies) else: raise IOError('Invalid value of "req_type" parameter: {0}' .format(req_type)) diff --git a/Atomic/satellite_new.py.test b/Atomic/satellite_new.py.test index 70ba7b2..ae35806 100644 --- a/Atomic/satellite_new.py.test +++ b/Atomic/satellite_new.py.test @@ -103,11 +103,13 @@ class SatelliteServer(object): def _call_satellite(self, url, req_type='get', payload=None, filePayload=None): """This function handles requests to the Satellite Server""" + proxies = util.get_proxy() if req_type == 'get': if (self._debug): print('Calling Satellite URL "{0}"'.format(url)) r = requests.get(url, auth=(self._username, self._password), - verify=self._verify_ssl) + verify=self._verify_ssl, + proxies=proxies) elif req_type == 'post': if (self._debug): print('Posting to Satellite URL "{0}"'.format(url)) @@ -116,7 +118,8 @@ class SatelliteServer(object): json.dumps(payload, indent=2))) r = requests.post(url, auth=(self._username, self._password), data=json.dumps(payload), - verify=self._verify_ssl) + verify=self._verify_ssl, + proxies=proxies) elif req_type == 'post-nodata': if (self._debug): print('Posting to Satellite URL "{0}". No data sent.'.format( @@ -124,12 +127,14 @@ class SatelliteServer(object): header = {'Content-Type': 'application/json'} r = requests.post(url, auth=(self._username, self._password), headers=header, data=json.dumps(payload), - verify=self._verify_ssl) + verify=self._verify_ssl, + proxies=proxies) elif req_type == 'put': if self._debug: print('Putting to Satellite URL "{0}"'.format(url)) r = requests.put(url, auth=(self._username, self._password), - data=payload, verify=self._verify_ssl) + data=payload, verify=self._verify_ssl, + proxies=proxies) elif req_type == 'put-jsonHead': if self._debug: print('Putting with json header to Satellite URL "{0}"' @@ -137,7 +142,7 @@ class SatelliteServer(object): header = {'Content-Type': 'application/json'} r = requests.put(url, auth=(self._username, self._password), headers=header, data=json.dumps(payload), - verify=self._verify_ssl) + verify=self._verify_ssl, proxies=proxies) elif req_type == 'put-multi-part': if self._debug: print('Multi-Part Putting to Satellite URL "{0}"'.format(url)) @@ -148,13 +153,14 @@ class SatelliteServer(object): } r = requests.put(url, auth=(self._username, self._password), headers=header, data=payload, - verify=self._verify_ssl) + verify=self._verify_ssl, + proxies=proxies) elif req_type == 'delete': if self._debug: print('Delete call to Satellite URL "{0}"'.format(url)) header = {'Content-Type': 'application/json'} r = requests.delete(url, auth=(self._username, self._password), - headers=header, verify=self._verify_ssl) + headers=header, verify=self._verify_ssl, proxies=proxies) else: raise ValueError('Invalid value of "req_type" parameter: {0}' .format(req_type)) diff --git a/Atomic/trust.py b/Atomic/trust.py index 418229a..c8dc465 100644 --- a/Atomic/trust.py +++ b/Atomic/trust.py @@ -243,7 +243,8 @@ class Trust(Atomic): raise ValueError("Aborting 'trust add' due to insecure download of public key from %s." % key_reference) if self.args.debug: util.write_out("Downloading key from %s" % key_reference) - r = requests.get(key_reference) + proxies = util.get_proxy() + r = requests.get(key_reference, proxies=proxies) if r.status_code == 200: keydata = r.content else: diff --git a/Atomic/util.py b/Atomic/util.py index fff8717..a3dfecc 100644 --- a/Atomic/util.py +++ b/Atomic/util.py @@ -18,7 +18,6 @@ import tempfile import shutil import re import requests -import ipaddress import socket from Atomic.backends._docker_errors import NoDockerDaemon import fcntl @@ -124,7 +123,8 @@ def subp(cmd, cwd=None, newline=False): proc = subprocess.Popen(cmd, cwd=cwd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, close_fds=True, - universal_newlines=newline) + universal_newlines=newline, + env=os.environ) out, err = proc.communicate() return ReturnTuple(proc.returncode, stdout=out, stderr=err) @@ -338,7 +338,7 @@ def skopeo_standalone_sign(image, manifest_file_name, fingerprint, signature_pat fingerprint, "-o", signature_path] if debug: write_out("Executing: {}".format(" ".join(cmd))) - return check_call(cmd) + return check_call(cmd, env=os.environ) def skopeo_manifest_digest(manifest_file, debug=False): cmd = [SKOPEO_PATH] @@ -371,7 +371,7 @@ def skopeo_copy(source, destination, debug=False, sign_by=None, insecure=False, cmd = cmd + [source, destination] if debug: write_out("Executing: {}".format(" ".join(cmd))) - return check_call(cmd) + return check_call(cmd, env=os.environ) @@ -408,9 +408,12 @@ def get_atomic_config_item(config_items, atomic_config=None, default=None): yaml_struct = atomic_config try: for i in items: - yaml_struct = yaml_struct[i] + yaml_struct = yaml_struct[i.lower()] except KeyError: - return None + try: + yaml_struct = yaml_struct[i.upper()] + except KeyError: + return None return yaml_struct if atomic_config is None: atomic_config = get_atomic_config() @@ -693,56 +696,12 @@ def is_insecure_registry(registry_config, registry): raise ValueError("Registry value cannot be blank") if is_python2 and not isinstance(registry, unicode): #pylint: disable=undefined-variable,unicode-builtin registry = unicode(registry) #pylint: disable=unicode-builtin,undefined-variable + insecure_registries = [x for x in registry_config['IndexConfigs'] if registry_config['IndexConfigs'][x]['Secure'] is False] - ip_registries = [] - ipv4_regs = [] - ipv6_regs = [] - registry_ips = [] - - def is_ipv4(_ip): - if is_python2 and not isinstance(_ip, unicode): #pylint: disable=unicode-builtin,undefined-variable - _ip = unicode(_ip) #pylint: disable=unicode-builtin,undefined-variable - if ipaddress.ip_address(_ip).version == 4: - return True - return False - - def get_ips_from_host(_host): - return list(set([x[4][0] for x in socket.getaddrinfo(_host, None)])) - - try: - ipaddress.ip_address(registry) - registry_ips.append(registry) - except ValueError: - for r in get_ips_from_host(registry): - registry_ips.append(r) - - insecure_cidrs = registry_config['InsecureRegistryCIDRs'] - insecure_registries = [strip_port(v['Name']) for _, v in registry_config['IndexConfigs'].items() if not v['Secure']] - for i in insecure_registries: - try: - ipaddress.ip_address(i) - ip_registries.append(i) - except ValueError: - for j in get_ips_from_host(i): - ip_registries.append(j) - - for ip in ip_registries: - if is_ipv4(ip): - ipv4_regs.append(ip) - else: - ipv6_regs.append(ip) - - # Everything is now in IP notation or CIDR - for registry_ip in registry_ips: - # Check IP addresses associated with known insecure registries - if is_python2 and not isinstance(registry_ip, unicode): #pylint: disable=unicode-builtin, undefined-variable - registry_ip = unicode(registry_ip) #pylint: disable=unicode-builtin, undefined-variable - if registry_ip in ipv4_regs or registry_ip in ipv6_regs: - return True - # Check if the IP falls in the the CIDR notation - for cidr_subnet in insecure_cidrs: - if ipaddress.ip_address(registry_ip ) in ipaddress.ip_network(cidr_subnet): - return True + # Be only as good as docker + if registry in insecure_registries: + return True + return False def is_valid_image_uri(uri, qualifying=None): ''' @@ -938,6 +897,8 @@ class Decompose(object): try: socket.gethostbyname(strip_port(_input)) except socket.gaierror: + if _input in [x['hostname'] for x in get_registries()]: + return True return False return True @@ -1034,3 +995,29 @@ def write_template(inputfilename, data, values, destination): outfile.write(result) return result return None + +def get_proxy(): + """ + Returns proxy information from environment variables as a dict + """ + def _get_envs_capped(): + return {k.upper(): v for k,v in os.environ.items()} + + proxies = {} + envs = _get_envs_capped() + + # Environment variables should override configuration items + proxies['http'] = get_atomic_config_item(['HTTP_PROXY']) if 'HTTP_PROXY' not in envs else envs['HTTP_PROXY'] + proxies['https'] = get_atomic_config_item(['HTTPS_PROXY']) if 'HTTPS_PROXY' not in envs else envs['HTTPS_PROXY'] + return proxies + +def set_proxy(): + """ + Sets proxy as environment variable if not set already + """ + proxies = get_proxy() + if proxies['http'] and 'HTTP_PROXY' not in os.environ: + os.environ['HTTP_PROXY'] = proxies['http'] + if proxies['https'] and 'HTTPS_PROXY' not in os.environ: + os.environ['HTTPS_PROXY'] = proxies['https'] + return proxies diff --git a/atomic.conf b/atomic.conf index 484de8b..0789a25 100644 --- a/atomic.conf +++ b/atomic.conf @@ -17,3 +17,9 @@ sigstore_metadata_image: sigstore # default_signer: # Absolute path to GPG keyring. Value set as environment variable GNUPGHOME #gnupg_homedir: /home/USER/.gnupg +# +# To always use a proxy with atomic, you can uncomment and fill out +# below. +# +#http_proxy= +#https_proxy= diff --git a/atomic_dbus.py b/atomic_dbus.py index 4d3a2ca..75ecb7d 100755 --- a/atomic_dbus.py +++ b/atomic_dbus.py @@ -70,6 +70,7 @@ class atomic_dbus(slip.dbus.service.Object): self.ignore = False self.image = None self.images = [] + self.insecure = False self.import_location = None self.json = True self.keytype = None @@ -299,15 +300,16 @@ class atomic_dbus(slip.dbus.service.Object): # The ImagePush method will push the specific image to a registry @slip.dbus.polkit.require_auth("org.atomic.readwrite") - @dbus.service.method("org.atomic", in_signature='sbbbssssssss', out_signature='i') + @dbus.service.method("org.atomic", in_signature='sbbbssssssssb', out_signature='i') def ImagePush(self, image, pulp, satellite, verify_ssl, url, username, password, - activation_key, repo_id, registry_type, sign_by, gnupghome): + activation_key, repo_id, registry_type, sign_by, gnupghome, insecure): p = Push() args = self.Args() args.image = image args.pulp = pulp args.satellite = satellite args.verify_ssl = verify_ssl + args.insecure = insecure args.url = None if not url else url args.username = None if not username else username args.password = None if not password else password diff --git a/bash/atomic b/bash/atomic index 739eebd..8432e80 100644 --- a/bash/atomic +++ b/bash/atomic @@ -429,6 +429,7 @@ _atomic_push() { --pulp --satellite --verify_ssl + --insecure --debug --all -a --force -f diff --git a/docs/atomic-push.1.md b/docs/atomic-push.1.md index ef6dfec..a7a0699 100644 --- a/docs/atomic-push.1.md +++ b/docs/atomic-push.1.md @@ -9,6 +9,7 @@ atomic-push - push Image to repository [**-a**][**--activation_key**[=*ACTIVATION_KEY*]] [**--debug**] [**-h**|**--help**] +[**--insecure**] [**--pulp**] [**-p**][**--password**[=*PASSWORD*]] [**-r**][**--repository_id**[=*REPOSITORY_ID*]] @@ -32,6 +33,9 @@ atomic-push - push Image to repository **-h** **--help** Print usage statement +**--insecure** + Indicate that the regsitry does not require HTTPS or certificate verification. + **-p PASSWORD** **--password PASSWORD** Password for remote registry diff --git a/tests/integration/test_dbus.py b/tests/integration/test_dbus.py index 04f2705..31ff1b0 100755 --- a/tests/integration/test_dbus.py +++ b/tests/integration/test_dbus.py @@ -162,13 +162,13 @@ class TestDBus(): TestDBus.add_cleanup_cmd('docker rmi docker.io/library/registry:2') TestDBus.add_cleanup_cmd('docker rmi docker.io/alpine:latest') TestDBus.add_cleanup_cmd('docker rmi localhost:5000/alpine:latest') - results = self.dbus_object.ImagePush("localhost:5000/alpine:latest", False, False, False, "", "foo", "bar", "", "", "", "", "") + results = self.dbus_object.ImagePush("localhost:5000/alpine:latest", False, False, False, "", "foo", "bar", "", "", "", "", "", True) assert(results == 0) @integration_test def test_push_no_password(self): try: - self.dbus_object.ImagePush("localhost:5000/alpine:latest", False, False, False, "", "foo", "", "", "", "", "", "") + self.dbus_object.ImagePush("localhost:5000/alpine:latest", False, False, False, "", "foo", "", "", "", "", "", "", True) raise ValueError("Expected an exception to be raised and was not.") except dbus.DBusException: pass @@ -176,7 +176,7 @@ class TestDBus(): @integration_test def test_push_no_username(self): try: - self.dbus_object.ImagePush("localhost:5000/alpine:latest", False, False, False, "", "", "", "", "", "", "", "") + self.dbus_object.ImagePush("localhost:5000/alpine:latest", False, False, False, "", "", "", "", "", "", "", "", True) raise ValueError("Expected an exception to be raised and was not.") except dbus.DBusException: pass @@ -184,7 +184,7 @@ class TestDBus(): @integration_test def test_push_pulp_no_username(self): try: - self.dbus_object.ImagePush("localhost:5000/alpine:latest", True, False, False, "url", "", "", "", "", "", "", "") + self.dbus_object.ImagePush("localhost:5000/alpine:latest", True, False, False, "url", "", "", "", "", "", "", "", True) raise ValueError("Expected an exception to be raised and was not.") except dbus.DBusException: pass @@ -192,7 +192,7 @@ class TestDBus(): @integration_test def test_push_pulp_no_url(self): try: - self.dbus_object.ImagePush("localhost:5000/alpine:latest", True, False, False, "", "foo", "bar", "", "", "", "", "") + self.dbus_object.ImagePush("localhost:5000/alpine:latest", True, False, False, "", "foo", "bar", "", "", "", "", "", True) raise ValueError("Expected an exception to be raised and was not.") except dbus.DBusException: pass