Adapt rest API to full hyperlinks

Adapt to rest API change from https://github.com/ansible/galaxy/pull/1761 https://github.com/ansible/galaxy/issues/1757 Require 'requests-mock' Use api.get_object instead of api.get_collection_detail() Need to separate rest_api.GalaxyAPI into http client stuff and a object->url map layer, but for now start with this use of GalaxyApi.get_collection_detail()
2026-02-05 12:45:17 +01:00 · 2019-04-25 17:06:25 -04:00
parent c6c3f87935
commit 83e45584e2
4 changed files with 81 additions and 69 deletions
--- a/ansible_galaxy/fetch/galaxy_url.py
+++ b/ansible_galaxy/fetch/galaxy_url.py
@@ -1,7 +1,7 @@
-
 import logging

 import semantic_version
+from six.moves.urllib.parse import quote as urlquote

 # mv details of this here
 from ansible_galaxy import exceptions
@@ -53,11 +53,6 @@ class GalaxyUrlFetch(base.BaseFetch):
        log.debug('requirement_spec: %s', requirement_spec)
        # log.debug('Validate TLS certificates: %s', self.validate_certs)

-    def build_full_download_url(self, rel_download_url):
-        api_server = self.galaxy_context.server['url']
-        full_download_url = '{api_server}{rel_download_url}'.format(api_server=api_server, rel_download_url=rel_download_url)
-        return full_download_url
-
    def find(self):
        '''Find a collection

@@ -81,11 +76,19 @@ class GalaxyUrlFetch(base.BaseFetch):
        # TODO: extract parsing of cli content sorta-url thing and add better tests

        # FIXME: Remove? We kind of need the actual Collection detail yet (ever?)
-        # collection_detail_data = api.get_collection_detail(namespace, collection_name)
+        collection_detail_url = '{base_api_url}/v2/collections/{namespace}/{name}'.format(base_api_url=api.base_api_url,
+                                                                                          namespace=urlquote(namespace),
+                                                                                          name=urlquote(collection_name))

-        # if not collection_detail_data:
-        #     raise exceptions.GalaxyClientError("- sorry, %s was not found on %s." % (self.requirement_spec.label,
-        #                                                                              api.api_server))
+        log.debug('collection_detail_url: %s', collection_detail_url)
+
+        collection_detail_data = api.get_object(href=collection_detail_url)
+
+        if not collection_detail_data:
+            raise exceptions.GalaxyClientError("- sorry, %s was not found on %s." % (self.requirement_spec.label,
+                                                                                     api.api_server))
+
+        versions_url = collection_detail_data.get('versions_url', None)

        # TODO: if versions ends up with a 'related' we could follow it instead of specific
        #       get_collection_version_list()
@@ -98,15 +101,18 @@ class GalaxyUrlFetch(base.BaseFetch):
        #   "version": "1.2.3",
        #   "href": "/api/v2/collections/ansible/k8s/versions/1.2.3/",
        #  }]
-        log.debug('Getting collectionversions for %s.%s', namespace, collection_name)
-        collection_version_list_data = api.get_collection_version_list(namespace, collection_name)
+        log.debug('Getting collectionversions for %s.%s from %s',
+                  namespace, collection_name, versions_url)
+
+        collection_version_list_data = api._get_paginated_list(versions_url)
+
+        log.debug('collectionvertlist data:\n%s', collection_version_list_data)
+
        if not collection_version_list_data:
            raise exceptions.GalaxyClientError("- sorry, %s was not found on %s." %
                                               (self.requirement_spec.label,
                                                api.api_server))

-        log.debug('collectionvertlist data:\n%s', collection_version_list_data)
-
        collection_version_strings = [a.get('version') for a in collection_version_list_data if a.get('version', None)]

        # a Version() component of the CollectionVersion
@@ -134,16 +140,13 @@ class GalaxyUrlFetch(base.BaseFetch):
                                                                    self.requirement_spec.label,
                                                                    requirement_spec=self.requirement_spec)

-        best_collectionversion_detail_data = api.get_href(href=best_collectionversion.get('href', None))
-        rel_download_url = best_collectionversion_detail_data.get('download_url', None)
+        best_collectionversion_detail_data = api.get_object(href=best_collectionversion.get('href', None))

-        log.debug('rel_download_url for %s.%s: %s', namespace, collection_name, rel_download_url)
+        download_url = best_collectionversion_detail_data.get('download_url', None)

-        full_download_url = self.build_full_download_url(rel_download_url)
+        log.debug('download_url for %s.%s: %s', namespace, collection_name, download_url)

-        log.debug('full_download_url for %s.%s: %s', namespace, collection_name, full_download_url)
-
-        if not full_download_url:
+        if not download_url:
            raise exceptions.GalaxyError('no external_url info on the Repository object from %s' % self.requirement_spec.label)

        # collectionversion_metadata = best_collectionversion_detail_data.get('metadata', None)
@@ -154,7 +157,7 @@ class GalaxyUrlFetch(base.BaseFetch):
        results = {'content': {'galaxy_namespace': namespace,
                               'repo_name': collection_name,
                               'version': best_version},
-                   'custom': {'download_url': full_download_url},
+                   'custom': {'download_url': download_url},
                   }

        return results
--- a/ansible_galaxy/rest_api.py
+++ b/ansible_galaxy/rest_api.py
@@ -239,8 +239,12 @@ class GalaxyAPI(object):
        self.log.debug('related_url=%s', list_url)

        page_size = page_size or 50
-        params = urlencode({'page_size': 50})
-        url = '%s?%s' % (list_url, params)
+        # param_dict = {'page_size': 50}
+        param_dict = {}
+        params = urlencode(param_dict)
+        url = list_url
+        if params:
+            url = '%s?%s' % (list_url, params)
        log.debug('url: %s params: %s', url, params)

        # can raise a GalaxyClientError
@@ -273,20 +277,10 @@ class GalaxyAPI(object):
        return data

    @g_connect
-    def get_collection_version_list(self, namespace, name):
-        namespace = urlquote(namespace)
-        name = urlquote(name)
-        # TODO: in theory, this url isn't fixed, but based on the 'versions' field of CollectionDetail
-        url = "%s%s" % (self.base_api_url,
-                        '/v2/collections/{namespace}/{name}/versions/'.format(namespace=namespace, name=name))
-
-        data = self._get_paginated_list(url)
-        return data
-
-    @g_connect
-    def get_href(self, href=None):
-        # hrefs start from '/api/...'
-        url = "%s%s" % (self.api_server, href)
+    def get_object(self, href=None):
+        '''Get a full url and return deserialized results'''
+        url = href
+        # url = "%s%s" % (self.api_server, href)

        data = self.__call_galaxy(url, http_method='GET')
        return data
--- a/requirements_test.txt
+++ b/requirements_test.txt
@@ -7,6 +7,7 @@ pytest-django>=3.4.2
 pytest-cov
 pytest-runner
 pytest-mock
+requests-mock
 tox
 # until test_galaxy_upstream tests are ported to pytest
 unittest2 ; python_version < '2.7'
--- a/tests/ansible_galaxy/fetch/test_galaxy_url.py
+++ b/tests/ansible_galaxy/fetch/test_galaxy_url.py
@@ -4,6 +4,7 @@ import pytest

 from ansible_galaxy import exceptions
 from ansible_galaxy.fetch import galaxy_url
+from ansible_galaxy.models.context import GalaxyContext
 from ansible_galaxy.models.requirement_spec import RequirementSpec

 log = logging.getLogger(__name__)
@@ -45,39 +46,53 @@ EXAMPLE_REPO_VERSIONS_LIST = \


@pytest.fixture
-def galaxy_url_fetch(galaxy_context):
+def galaxy_context_example_invalid(galaxy_context):
+    context = GalaxyContext(content_path=galaxy_context.content_path,
+                            server={'url': 'http://example.invalid',
+                                    'ignore_certs': False})
+    return context
+
+
+@pytest.fixture
+def galaxy_url_fetch(galaxy_context_example_invalid):
    req_spec = RequirementSpec(namespace='some_namespace',
                               name='some_name',
                               version_spec='==9.3.245')

-    galaxy_url_fetch = galaxy_url.GalaxyUrlFetch(requirement_spec=req_spec, galaxy_context=galaxy_context)
+    galaxy_url_fetch = galaxy_url.GalaxyUrlFetch(requirement_spec=req_spec, galaxy_context=galaxy_context_example_invalid)
    log.debug('galaxy_url_fetch: %s', galaxy_url_fetch)

    return galaxy_url_fetch


-def test_galaxy_url_fetch_find(galaxy_url_fetch, mocker):
-    MockedGalaxyAPI = mocker.patch('ansible_galaxy.fetch.galaxy_url.GalaxyAPI', autospec=True)
+def test_galaxy_url_fetch_find(galaxy_url_fetch, requests_mock):
+    download_url = 'http://example.invalid/api/v2/collections/some_ns/some_name/versions/9.3.245/artifact'

-    download_url = '/some/path/some_ns/some_name/versions/9.3.245/artifact'
+    requests_mock.get('http://example.invalid/api/',
+                      json={'current_version': 'v2'})

-    instance = MockedGalaxyAPI.return_value
-    instance._api_server = mocker.Mock(return_value='http://example.invalid/')
-    instance.get_collection_detail.return_value = {'related': {'versions': 'http://example.invalid/foo'},
-                                                   'download_url': download_url}
-    instance.get_collection_version_list.return_value = [{'version': '1.2.3',
-                                                          'href': 'http://example.invalid/api/v2/collections/some_ns/some_name/versions/1.2.3/'},
-                                                         {'version': '9.3.245',
-                                                          'href': 'http://example.invalid/api/v2/collections/some_ns/some_name/versions/9.3.245/'}]
+    # get CollectionVersionList
+    requests_mock.get('http://example.invalid/api/v2/collections/some_ns/some_name/versions/',
+                      json={
+                          'count': 2,
+                          'next': None,
+                          'previous': None,
+                          'results':
+                          [{'version': '1.2.3',
+                            'href': 'http://example.invalid/api/v2/collections/some_ns/some_name/versions/1.2.3/'},
+                           {'version': '9.3.245',
+                            'href': 'http://example.invalid/api/v2/collections/some_ns/some_name/versions/9.3.245/'}]})

    # The request to get the CollectionVersion detail via href from CollectionVersion list
-    instance.get_href.return_value = {'download_url': download_url,
-                                      'metadata': '',
-                                      'version': '1.2.3'}
+    requests_mock.get('http://example.invalid/api/v2/collections/some_ns/some_name/versions/9.3.245/',
+                      json={'download_url': download_url,
+                            'metadata': {},
+                            'version': '9.3.245'})
+
+    # The Collection detail
+    requests_mock.get('http://example.invalid/api/v2/collections/some_namespace/some_name',
+                      json={'versions_url': 'http://example.invalid/api/v2/collections/some_ns/some_name/versions/'})

-    # FIXME: Remove this when we get download_url from CollectionVersion detail instead of building it from server url
-    # mocker.patch('ansible_galaxy.fetch.galaxy_url.GalaxyUrlFetch.galaxy_context.server',
-    #             return_value={'url': 'http://example.invalid/'})
    res = galaxy_url_fetch.find()

    log.debug('res:%s', res)
@@ -85,33 +100,32 @@ def test_galaxy_url_fetch_find(galaxy_url_fetch, mocker):
    assert res['content']['galaxy_namespace'] == 'some_namespace'
    assert res['content']['repo_name'] == 'some_name'

-    assert res['custom']['download_url'] == "http://localhost:8000%s" % download_url
+    assert res['custom']['download_url'] == download_url


-def test_galaxy_url_fetch_find_no_repo_data(galaxy_url_fetch, mocker):
-    MockedGalaxyAPI = mocker.patch('ansible_galaxy.fetch.galaxy_url.GalaxyAPI', autospec=True)
+def test_galaxy_url_fetch_find_no_repo_data(galaxy_url_fetch, galaxy_context, requests_mock):
+    requests_mock.get('http://example.invalid/api/',
+                      json={'current_version': 'v2'})
+    requests_mock.get('http://example.invalid/api/v2/collections/some_namespace/some_name',
+                      json={})

-    instance = MockedGalaxyAPI.return_value
-    instance.get_collection_detail.return_value = {}
-    instance.get_collection_version_list.return_value = []
-
-    faux_server_url = 'http://galaxy.invalid/'
-    instance.api_server = faux_server_url
+    # galaxy_url_fetch = galaxy_url.GalaxyUrlFetch(requirement_spec=req_spec, galaxy_context=context)
    # - sorry, some_namespace.some_name (version_spec: ==9.3.245) was not found on http://galaxy.invalid/.
    with pytest.raises(exceptions.GalaxyClientError,
-                       match='- sorry, some_namespace.some_name.*version_spec.*was not found on %s' % faux_server_url) as exc_info:
+                       match='- sorry, some_namespace.some_name.*version_spec.*was not found on http://example.invalid') as exc_info:
        galaxy_url_fetch.find()

    log.debug('exc_info:%s', exc_info)


-def test_galaxy_url_fetch_fetch(galaxy_url_fetch, mocker):
+def test_galaxy_url_fetch_fetch(galaxy_url_fetch, mocker, requests_mock):
+    download_url = 'http://example.invalid/api/v2/collections/some_ns/some_name/versions/9.3.245/artifact'
    collection_path = '/dev/null/path/to/collection.tar.gz'

    mocked_download_fetch_url = mocker.patch('ansible_galaxy.fetch.galaxy_url.download.fetch_url', autospec=True)
    mocked_download_fetch_url.return_value = collection_path

-    download_url = 'http://example.invalid/invalid/whatever'
+    # download_url = 'http://example.invalid/invalid/whatever'
    find_results = {'content': {'galaxy_namespace': 'some_namespace',
                                'repo_name': 'some_name'},
                    'custom': {'repo_data': {},