projectatomic/atomic
1
0
Fork 0
mirror of https://github.com/projectatomic/atomic.git synced 2026-02-05 18:45:01 +01:00
Code Issues Releases Activity

Honor proxy usage

Browse Source 
If HTTP[S]_PROXY is defined, honor it in python requests usage
as well as pass it on to skopeo.

If http[s]_proxy is defined in atomic.conf, use it; however, environment
variables will override these if defined.

Added --insecure to Atomic push so the user can override the logic
(or lack thereof) around deducing if a registry is insecure.  Also
needed for integration tests.

Closes: #964
Approved by: rhatdan
This commit is contained in:
Brent Baude
2017-04-03 13:50:40 -05:00
committed by Atomic Bot
parent cf5d5bc8e8
commit 2b744bfdbb
12 changed files with 105 additions and 83 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
Atomic/atomic.py
View File

@@ -63,6 +63,7 @@ class Atomic(object):
self.run_opts = None
self.atomic_config = util.get_atomic_config()
self.local_tokens = {}
util.set_proxy()
def __enter__(self):
return self

12
Atomic/pulp.py
View File

@@ -72,21 +72,25 @@ class PulpServer(object):
self._chunk_size = 1048576 # 1 MB per upload call
def _call_pulp(self, url, req_type='get', payload=None):
proxies = util.get_proxy()
if req_type == 'get':
r = requests.get(url, auth=(self._username, self._password),
verify=self._verify_ssl)
verify=self._verify_ssl,
proxies=proxies)
elif req_type == 'post':
r = requests.post(url, auth=(self._username, self._password),
data=json.dumps(payload),
verify=self._verify_ssl)
verify=self._verify_ssl,
proxies=proxies)
elif req_type == 'put':
# some calls pass in binary data so we don't log payload data or
# json encode it here
r = requests.put(url, auth=(self._username, self._password),
data=payload, verify=self._verify_ssl)
data=payload, verify=self._verify_ssl,
proxies=proxies)
elif req_type == 'delete':
r = requests.delete(url, auth=(self._username, self._password),
verify=self._verify_ssl)
verify=self._verify_ssl, proxies=proxies)
else:
raise ValueError('Invalid value of "req_type" parameter: {0}'
''.format(req_type))

8
Atomic/push.py
View File

@@ -69,6 +69,8 @@ def cli(subparser):
"use an alternate user's GPG keyring for signing. "
"Useful when running with sudo, "
"e.g. set to '~/.gnupg'."))
pushp.add_argument("--insecure", dest="insecure", default=False,
action='store_true', help=_("Do not check registry certificates"))
# pushp.add_argument("--activation_key_name",
# default=None,
# dest="activation_key_name",
@@ -180,8 +182,10 @@ class Push(Atomic):
if sign and self.args.debug:
util.write_out("\nSigning with '{}'\n".format(self.args.sign_by))
insecure = True if util.is_insecure_registry(self.d.info()['RegistryConfig'], util.strip_port(reg)) else False
if self.args.insecure:
insecure = True
else:
insecure = True if util.is_insecure_registry(self.d.info()['RegistryConfig'], util.strip_port(reg)) else False
# We must push the file to the registry first prior to performing a
# local signature because the manifest file must be on the registry
return_code = util.skopeo_copy(local_image, remote_image, debug=self.args.debug,

20
Atomic/satellite.py
View File

@@ -70,11 +70,12 @@ class SatelliteServer(object):
def _call_satellite(self, url, req_type='get', payload=None):
"""This function handles requests to the Satellite Server"""
proxies = util.get_proxy()
if req_type == 'get':
if (self._debug):
print('Calling Satellite URL "{0}"'.format(url))
r = requests.get(url, auth=(self._username, self._password),
verify=self._verify_ssl)
verify=self._verify_ssl, proxies=proxies)
elif req_type == 'post':
if (self._debug):
print('Posting to Satellite URL "{0}"'.format(url))
@@ -83,7 +84,8 @@ class SatelliteServer(object):
json.dumps(payload, indent=2)))
r = requests.post(url, auth=(self._username, self._password),
data=json.dumps(payload),
verify=self._verify_ssl)
verify=self._verify_ssl,
proxies=proxies)
elif req_type == 'post-nodata':
if (self._debug):
print('Posting to Satellite URL "{0}". No data sent.'.format(
@@ -91,12 +93,14 @@ class SatelliteServer(object):
header = {'Content-Type': 'application/json'}
r = requests.post(url, auth=(self._username, self._password),
headers=header, data=json.dumps(payload),
verify=self._verify_ssl)
verify=self._verify_ssl,
proxies=proxies)
elif req_type == 'put':
if self._debug:
print('Putting to Satellite URL "{0}"'.format(url))
r = requests.put(url, auth=(self._username, self._password),
data=payload, verify=self._verify_ssl)
data=payload, verify=self._verify_ssl,
proxies=proxies)
elif req_type == 'put-jsonHead':
if self._debug:
print('Putting with json header to Satellite URL "{0}"'
@@ -104,7 +108,7 @@ class SatelliteServer(object):
header = {'Content-Type': 'application/json'}
r = requests.put(url, auth=(self._username, self._password),
headers=header, data=json.dumps(payload),
verify=self._verify_ssl)
verify=self._verify_ssl, proxies=proxies)
elif req_type == 'put-multi-part':
if self._debug:
print('Multi-Part Putting to Satellite URL "{0}"'.format(url))
@@ -115,13 +119,15 @@ class SatelliteServer(object):
}
r = requests.put(url, auth=(self._username, self._password),
headers=header, data=payload,
verify=self._verify_ssl)
verify=self._verify_ssl,
proxies=proxies)
elif req_type == 'delete':
if self._debug:
print('Delete call to Satellite URL "{0}"'.format(url))
header = {'Content-Type': 'application/json'}
r = requests.delete(url, auth=(self._username, self._password),
headers=header, verify=self._verify_ssl)
headers=header, verify=self._verify_ssl,
proxies=proxies)
else:
raise IOError('Invalid value of "req_type" parameter: {0}'
.format(req_type))

20
Atomic/satellite_new.py.test
View File

@@ -103,11 +103,13 @@ class SatelliteServer(object):
def _call_satellite(self, url, req_type='get', payload=None,
filePayload=None):
"""This function handles requests to the Satellite Server"""
proxies = util.get_proxy()
if req_type == 'get':
if (self._debug):
print('Calling Satellite URL "{0}"'.format(url))
r = requests.get(url, auth=(self._username, self._password),
verify=self._verify_ssl)
verify=self._verify_ssl,
proxies=proxies)
elif req_type == 'post':
if (self._debug):
print('Posting to Satellite URL "{0}"'.format(url))
@@ -116,7 +118,8 @@ class SatelliteServer(object):
json.dumps(payload, indent=2)))
r = requests.post(url, auth=(self._username, self._password),
data=json.dumps(payload),
verify=self._verify_ssl)
verify=self._verify_ssl,
proxies=proxies)
elif req_type == 'post-nodata':
if (self._debug):
print('Posting to Satellite URL "{0}". No data sent.'.format(
@@ -124,12 +127,14 @@ class SatelliteServer(object):
header = {'Content-Type': 'application/json'}
r = requests.post(url, auth=(self._username, self._password),
headers=header, data=json.dumps(payload),
verify=self._verify_ssl)
verify=self._verify_ssl,
proxies=proxies)
elif req_type == 'put':
if self._debug:
print('Putting to Satellite URL "{0}"'.format(url))
r = requests.put(url, auth=(self._username, self._password),
data=payload, verify=self._verify_ssl)
data=payload, verify=self._verify_ssl,
proxies=proxies)
elif req_type == 'put-jsonHead':
if self._debug:
print('Putting with json header to Satellite URL "{0}"'
@@ -137,7 +142,7 @@ class SatelliteServer(object):
header = {'Content-Type': 'application/json'}
r = requests.put(url, auth=(self._username, self._password),
headers=header, data=json.dumps(payload),
verify=self._verify_ssl)
verify=self._verify_ssl, proxies=proxies)
elif req_type == 'put-multi-part':
if self._debug:
print('Multi-Part Putting to Satellite URL "{0}"'.format(url))
@@ -148,13 +153,14 @@ class SatelliteServer(object):
}
r = requests.put(url, auth=(self._username, self._password),
headers=header, data=payload,
verify=self._verify_ssl)
verify=self._verify_ssl,
proxies=proxies)
elif req_type == 'delete':
if self._debug:
print('Delete call to Satellite URL "{0}"'.format(url))
header = {'Content-Type': 'application/json'}
r = requests.delete(url, auth=(self._username, self._password),
headers=header, verify=self._verify_ssl)
headers=header, verify=self._verify_ssl, proxies=proxies)
else:
raise ValueError('Invalid value of "req_type" parameter: {0}'
.format(req_type))

3
Atomic/trust.py
View File

@@ -243,7 +243,8 @@ class Trust(Atomic):
raise ValueError("Aborting 'trust add' due to insecure download of public key from %s." % key_reference)
if self.args.debug:
util.write_out("Downloading key from %s" % key_reference)
r = requests.get(key_reference)
proxies = util.get_proxy()
r = requests.get(key_reference, proxies=proxies)
if r.status_code == 200:
keydata = r.content
else:

97
Atomic/util.py
View File

@@ -18,7 +18,6 @@ import tempfile
import shutil
import re
import requests
import ipaddress
import socket
from Atomic.backends._docker_errors import NoDockerDaemon
import fcntl
@@ -124,7 +123,8 @@ def subp(cmd, cwd=None, newline=False):
proc = subprocess.Popen(cmd, cwd=cwd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE, close_fds=True,
universal_newlines=newline)
universal_newlines=newline,
env=os.environ)
out, err = proc.communicate()
return ReturnTuple(proc.returncode, stdout=out, stderr=err)
@@ -338,7 +338,7 @@ def skopeo_standalone_sign(image, manifest_file_name, fingerprint, signature_pat
fingerprint, "-o", signature_path]
if debug:
write_out("Executing: {}".format(" ".join(cmd)))
return check_call(cmd)
return check_call(cmd, env=os.environ)
def skopeo_manifest_digest(manifest_file, debug=False):
cmd = [SKOPEO_PATH]
@@ -371,7 +371,7 @@ def skopeo_copy(source, destination, debug=False, sign_by=None, insecure=False,
cmd = cmd + [source, destination]
if debug:
write_out("Executing: {}".format(" ".join(cmd)))
return check_call(cmd)
return check_call(cmd, env=os.environ)
@@ -408,9 +408,12 @@ def get_atomic_config_item(config_items, atomic_config=None, default=None):
yaml_struct = atomic_config
try:
for i in items:
yaml_struct = yaml_struct[i]
yaml_struct = yaml_struct[i.lower()]
except KeyError:
return None
try:
yaml_struct = yaml_struct[i.upper()]
except KeyError:
return None
return yaml_struct
if atomic_config is None:
atomic_config = get_atomic_config()
@@ -693,56 +696,12 @@ def is_insecure_registry(registry_config, registry):
raise ValueError("Registry value cannot be blank")
if is_python2 and not isinstance(registry, unicode): #pylint: disable=undefined-variable,unicode-builtin
registry = unicode(registry) #pylint: disable=unicode-builtin,undefined-variable
insecure_registries = [x for x in registry_config['IndexConfigs'] if registry_config['IndexConfigs'][x]['Secure'] is False]
ip_registries = []
ipv4_regs = []
ipv6_regs = []
registry_ips = []
def is_ipv4(_ip):
if is_python2 and not isinstance(_ip, unicode): #pylint: disable=unicode-builtin,undefined-variable
_ip = unicode(_ip) #pylint: disable=unicode-builtin,undefined-variable
if ipaddress.ip_address(_ip).version == 4:
return True
return False
def get_ips_from_host(_host):
return list(set([x[4][0] for x in socket.getaddrinfo(_host, None)]))
try:
ipaddress.ip_address(registry)
registry_ips.append(registry)
except ValueError:
for r in get_ips_from_host(registry):
registry_ips.append(r)
insecure_cidrs = registry_config['InsecureRegistryCIDRs']
insecure_registries = [strip_port(v['Name']) for _, v in registry_config['IndexConfigs'].items() if not v['Secure']]
for i in insecure_registries:
try:
ipaddress.ip_address(i)
ip_registries.append(i)
except ValueError:
for j in get_ips_from_host(i):
ip_registries.append(j)
for ip in ip_registries:
if is_ipv4(ip):
ipv4_regs.append(ip)
else:
ipv6_regs.append(ip)
# Everything is now in IP notation or CIDR
for registry_ip in registry_ips:
# Check IP addresses associated with known insecure registries
if is_python2 and not isinstance(registry_ip, unicode): #pylint: disable=unicode-builtin, undefined-variable
registry_ip = unicode(registry_ip) #pylint: disable=unicode-builtin, undefined-variable
if registry_ip in ipv4_regs or registry_ip in ipv6_regs:
return True
# Check if the IP falls in the the CIDR notation
for cidr_subnet in insecure_cidrs:
if ipaddress.ip_address(registry_ip ) in ipaddress.ip_network(cidr_subnet):
return True
# Be only as good as docker
if registry in insecure_registries:
return True
return False
def is_valid_image_uri(uri, qualifying=None):
'''
@@ -938,6 +897,8 @@ class Decompose(object):
try:
socket.gethostbyname(strip_port(_input))
except socket.gaierror:
if _input in [x['hostname'] for x in get_registries()]:
return True
return False
return True
@@ -1034,3 +995,29 @@ def write_template(inputfilename, data, values, destination):
outfile.write(result)
return result
return None
def get_proxy():
"""
Returns proxy information from environment variables as a dict
"""
def _get_envs_capped():
return {k.upper(): v for k,v in os.environ.items()}
proxies = {}
envs = _get_envs_capped()
# Environment variables should override configuration items
proxies['http'] = get_atomic_config_item(['HTTP_PROXY']) if 'HTTP_PROXY' not in envs else envs['HTTP_PROXY']
proxies['https'] = get_atomic_config_item(['HTTPS_PROXY']) if 'HTTPS_PROXY' not in envs else envs['HTTPS_PROXY']
return proxies
def set_proxy():
"""
Sets proxy as environment variable if not set already
"""
proxies = get_proxy()
if proxies['http'] and 'HTTP_PROXY' not in os.environ:
os.environ['HTTP_PROXY'] = proxies['http']
if proxies['https'] and 'HTTPS_PROXY' not in os.environ:
os.environ['HTTPS_PROXY'] = proxies['https']
return proxies

6
atomic.conf
View File

@@ -17,3 +17,9 @@ sigstore_metadata_image: sigstore
# default_signer:
# Absolute path to GPG keyring. Value set as environment variable GNUPGHOME
#gnupg_homedir: /home/USER/.gnupg
#
# To always use a proxy with atomic, you can uncomment and fill out
# below.
#
#http_proxy=
#https_proxy=

6
atomic_dbus.py
View File

@@ -70,6 +70,7 @@ class atomic_dbus(slip.dbus.service.Object):
self.ignore = False
self.image = None
self.images = []
self.insecure = False
self.import_location = None
self.json = True
self.keytype = None
@@ -299,15 +300,16 @@ class atomic_dbus(slip.dbus.service.Object):
# The ImagePush method will push the specific image to a registry
@slip.dbus.polkit.require_auth("org.atomic.readwrite")
@dbus.service.method("org.atomic", in_signature='sbbbssssssss', out_signature='i')
@dbus.service.method("org.atomic", in_signature='sbbbssssssssb', out_signature='i')
def ImagePush(self, image, pulp, satellite, verify_ssl, url, username, password,
activation_key, repo_id, registry_type, sign_by, gnupghome):
activation_key, repo_id, registry_type, sign_by, gnupghome, insecure):
p = Push()
args = self.Args()
args.image = image
args.pulp = pulp
args.satellite = satellite
args.verify_ssl = verify_ssl
args.insecure = insecure
args.url = None if not url else url
args.username = None if not username else username
args.password = None if not password else password

1
bash/atomic
View File

@@ -429,6 +429,7 @@ _atomic_push() {
--pulp
--satellite
--verify_ssl
--insecure
--debug
--all -a
--force -f

4
docs/atomic-push.1.md
View File

@@ -9,6 +9,7 @@ atomic-push - push Image to repository
[**-a**][**--activation_key**[=*ACTIVATION_KEY*]]
[**--debug**]
[**-h**|**--help**]
[**--insecure**]
[**--pulp**]
[**-p**][**--password**[=*PASSWORD*]]
[**-r**][**--repository_id**[=*REPOSITORY_ID*]]
@@ -32,6 +33,9 @@ atomic-push - push Image to repository
**-h** **--help**
Print usage statement
**--insecure**
Indicate that the regsitry does not require HTTPS or certificate verification.
**-p PASSWORD** **--password PASSWORD**
Password for remote registry

10
tests/integration/test_dbus.py
View File

@@ -162,13 +162,13 @@ class TestDBus():
TestDBus.add_cleanup_cmd('docker rmi docker.io/library/registry:2')
TestDBus.add_cleanup_cmd('docker rmi docker.io/alpine:latest')
TestDBus.add_cleanup_cmd('docker rmi localhost:5000/alpine:latest')
results = self.dbus_object.ImagePush("localhost:5000/alpine:latest", False, False, False, "", "foo", "bar", "", "", "", "", "")
results = self.dbus_object.ImagePush("localhost:5000/alpine:latest", False, False, False, "", "foo", "bar", "", "", "", "", "", True)
assert(results == 0)
@integration_test
def test_push_no_password(self):
try:
self.dbus_object.ImagePush("localhost:5000/alpine:latest", False, False, False, "", "foo", "", "", "", "", "", "")
self.dbus_object.ImagePush("localhost:5000/alpine:latest", False, False, False, "", "foo", "", "", "", "", "", "", True)
raise ValueError("Expected an exception to be raised and was not.")
except dbus.DBusException:
pass
@@ -176,7 +176,7 @@ class TestDBus():
@integration_test
def test_push_no_username(self):
try:
self.dbus_object.ImagePush("localhost:5000/alpine:latest", False, False, False, "", "", "", "", "", "", "", "")
self.dbus_object.ImagePush("localhost:5000/alpine:latest", False, False, False, "", "", "", "", "", "", "", "", True)
raise ValueError("Expected an exception to be raised and was not.")
except dbus.DBusException:
pass
@@ -184,7 +184,7 @@ class TestDBus():
@integration_test
def test_push_pulp_no_username(self):
try:
self.dbus_object.ImagePush("localhost:5000/alpine:latest", True, False, False, "url", "", "", "", "", "", "", "")
self.dbus_object.ImagePush("localhost:5000/alpine:latest", True, False, False, "url", "", "", "", "", "", "", "", True)
raise ValueError("Expected an exception to be raised and was not.")
except dbus.DBusException:
pass
@@ -192,7 +192,7 @@ class TestDBus():
@integration_test
def test_push_pulp_no_url(self):
try:
self.dbus_object.ImagePush("localhost:5000/alpine:latest", True, False, False, "", "foo", "bar", "", "", "", "", "")
self.dbus_object.ImagePush("localhost:5000/alpine:latest", True, False, False, "", "foo", "bar", "", "", "", "", "", True)
raise ValueError("Expected an exception to be raised and was not.")
except dbus.DBusException:
pass