Introduce tox for testing and add e2e framework

This commit refactors the testing infrastructure to use tox,
centralizing the configuration in pyproject.toml.

Key changes:
- Makefile test targets now delegate to tox.
- A new pytest-based e2e testing framework is added.
- An initial e2e test for the version command is included.

Signed-off-by: Roberto Majadas <rmajadas@redhat.com>

.gitignore

@@ -16,8 +16,10 @@ venv/
 __pycache__/
 .aider*
 .coverage
+coverage/
 coverage.*
 htmlcov/
 .idea/
 .hypothesis/
 uv.lock
+.tox

Makefile

@@ -10,8 +10,7 @@ PATH := $(PATH):$(HOME)/.local/bin
 MYPIP ?= pip
 IMAGE ?= ramalama
 PROJECT_DIR:=$(shell dirname $(realpath $(lastword $(MAKEFILE_LIST))))
-PYTHON_SCRIPTS := $(shell grep -lEr "^\#\!\s*/usr/bin/(env +)?python(3)?(\s|$$)" --exclude-dir={.venv,venv} $(PROJECT_DIR) || true)
-PYTEST_COMMON_CMD ?= PYTHONPATH=. pytest test/unit/ -vv
+PYTHON_SCRIPTS := $(shell grep -lEr "^\#\!\s*/usr/bin/(env +)?python(3)?(\s|$$)" --exclude-dir={.venv,venv,.tox} $(PROJECT_DIR) || true)
 BATS_IMAGE ?= localhost/bats:latest
 
 default: help
@@ -43,13 +42,6 @@ help:
 	@echo "  - make clean"
 	@echo
 
-install-detailed-cov-requirements:
-	${MYPIP} install ".[cov-detailed]"
-
-.PHONY: install-cov-requirements
-install-cov-requirements:
-	${MYPIP} install ".[cov]"
-
 .PHONY: install-uv
 install-uv:
 	./install-uv.sh
@@ -193,30 +185,30 @@ bats-in-container: extra-opts = --security-opt unmask=/proc/* --device /dev/net/
 ci:
 	test/ci.sh
 
+.PHONY: requires-tox
+requires-tox:
+	@command -v tox >/dev/null 2>&1 || ${MYPIP} install tox
+
 .PHONY: unit-tests
-unit-tests:
-	$(PYTEST_COMMON_CMD)
+unit-tests: requires-tox
+	tox
 
 .PHONY: unit-tests-verbose
-unit-tests-verbose:
-	$(PYTEST_COMMON_CMD) --full-trace --capture=tee-sys
-
-.PHONY: cov-run
-cov-run: install-cov-requirements
-	PYTHONPATH=. coverage run -m pytest test/unit/
+unit-tests-verbose: requires-tox
+	tox -- --full-trace --capture=tee-sys
 
 .PHONY: cov-tests
-cov-tests: cov-run
-	PYTHONPATH=. coverage report
+cov-tests: requires-tox
+	tox -- --cov
 
 .PHONY: detailed-cov-tests
-detailed-cov-tests: install-detailed-cov-requirements cov-run
-	PYTHONPATH=. coverage report -m
-	PYTHONPATH=. coverage html
-	PYTHONPATH=. coverage json
-	PYTHONPATH=. coverage lcov
-	PYTHONPATH=. coverage xml
+detailed-cov-tests: requires-tox
+	tox -e coverage
 
+.PHONY: e2e-tests
+e2e-tests: requires-tox
+	# This makefile target runs the new e2e-tests pytest based
+	tox -e e2e
 
 .PHONY: end-to-end-tests
 end-to-end-tests: validate bats bats-nocontainer ci

pyproject.toml

@@ -45,6 +45,7 @@ dev = [
     "wheel~=0.45.0",
     "mypy",
     "pyyaml",
+    "tox"
 ]
 
 cov = [
@@ -87,10 +88,10 @@ extend-exclude = "(.history|build|dist|docs|hack)"
 [tool.isort]
 profile = "black"
 line_length = 120
-skip = [".venv", "venv"]
+skip = [".venv", "venv", ".tox"]
 
 [tool.codespell]
-skip = ["build", "ramalama.egg-info", "logos", ".git", "venv", ".venv"]
+skip = ["build", "ramalama.egg-info", "logos", ".git", "venv", ".venv", ".tox"]
 dictionary = ".codespelldict"
 ignore-words-list = [
   "assertin",
@@ -124,13 +125,16 @@ exclude = [
 preview = true
 quote-style = "preserve"
 
-[tool.pytest.ini_options]
-testpaths = ["."]
-log_cli = true
-log_cli_level = "WARN"
-log_cli_format = "%(asctime)s [%(levelname)8s] %(message)s (%(filename)s:%(lineno)s)"
-log_cli_date_format = "%Y-%m-%d %H:%M:%S"
+[tool.coverage]
+html.directory = "coverage/html"
+xml.output = "coverage/coverage.xml"
+json.output = "coverage/coverage.json"
+lcov.output = "coverage/coverage.lcov"
 
+[tool.pytest.ini_options]
+testpaths = ["test"]
+markers = ["e2e", "distro_integration"]
+addopts = "-m 'not e2e' --color=yes"
 
 [tool.setuptools.packages.find]
 include = ["ramalama", "ramalama.*"]
@@ -144,3 +148,50 @@ include = ["ramalama", "ramalama.*"]
 "share/zsh/site-functions" = ["completions/zsh/site-functions/*"]
 "share/fish/vendor_completions.d" = ["completions/fish/vendor_completions.d/*"]
 
+[tool.tox]
+
+[tool.tox.env_run_base]
+package = "wheel"
+extras = ["dev", "cov"]
+pass_env = [
+  "GITHUB_ACTIONS",
+]
+commands = [
+  [
+    "pytest",
+    "-vvv",
+    "--tb=short",
+    "--basetemp={envtmpdir}",
+    { replace = "posargs", extend = true },
+  ],
+]
+
+[tool.tox.env.e2e]
+commands = [
+  [
+    "pytest",
+    "-m", "e2e",
+    "-vvv",
+    "--basetemp={envtmpdir}",
+    "--tb=short",
+    { replace = "posargs", extend = true },
+  ],
+]
+
+[tool.tox.env.coverage]
+extras = ["dev", "cov", "cov-detailed"]
+commands = [
+  [
+    "pytest",
+    "-vvv",
+    "--cov",
+    "--cov-report=xml",
+    "--cov-report=html",
+    "--cov-report=json",
+    "--cov-report=lcov",
+    "--cov-report=term",
+    "--tb=short",
+    "--basetemp={envtmpdir}",
+    { replace = "posargs", extend = true },
+  ],
+]

test/conftest.py

@@ -0,0 +1,67 @@
+import os
+import platform
+import shutil
+
+import pytest
+
+ramalama_container = True
+ramalama_container_engine = "podman"
+
+
+def pytest_addoption(parser):
+    container_group = parser.getgroup("container")
+    container_group.addoption(
+        "--container",
+        action="store_true",
+        dest="container",
+        default=True,
+        help="Enable container mode",
+    )
+    container_group.addoption(
+        "--no-container",
+        action="store_false",
+        dest="container",
+        help="Disable container mode",
+    )
+    container_group.addoption(
+        "--container-engine",
+        action="store",
+        choices=["podman", "docker"],
+        default="podman",
+        help="Container engine to use",
+    )
+
+
+def pytest_configure(config):
+    global ramalama_container
+    global ramalama_container_engine
+    ramalama_container = config.option.container
+    ramalama_container_engine = config.option.container_engine
+
+
+@pytest.fixture()
+def is_container(request):
+    return ramalama_container
+
+
+@pytest.fixture()
+def container_engine(request):
+    return ramalama_container_engine
+
+
+skip_if_no_container = pytest.mark.skipif("not config.option.container", reason="no container mode is enabled")
+skip_if_container = pytest.mark.skipif("config.option.container", reason="container mode is enabled")
+skip_if_docker = pytest.mark.skipif(
+    "config.option.container_engine == 'docker'", reason="docker is the container engine"
+)
+
+skip_if_darwin = pytest.mark.skipif(platform.system() == "Darwin", reason="Darwin operating system")
+skip_if_not_darwin = pytest.mark.skipif(platform.system() != "Darwin", reason="not Darwin operating system")
+skip_if_gh_actions_darwin = pytest.mark.skipif(
+    platform.system() == "Darwin" and os.getenv("GITHUB_ACTIONS") == "true",
+    reason="GitHub Actions Darwin has not container support",
+)
+
+skip_if_no_huggingface_cli = pytest.mark.skipif(
+    shutil.which("huggingface-cli") is None, reason="huggingface-cli not installed"
+)

test/e2e/test_basic.py

@@ -0,0 +1,11 @@
+import re
+
+import pytest
+from utils import check_output
+
+
+@pytest.mark.e2e
+def test_version_line_output():
+    result = check_output(["ramalama", "version"])
+
+    assert re.match(r"ramalama version \d+\.\d+.\d+", result)

test/e2e/utils.py

@@ -0,0 +1,84 @@
+import os
+import shutil
+import subprocess
+import tempfile
+from pathlib import Path
+from test.conftest import ramalama_container, ramalama_container_engine
+
+
+class RamalamaExecWorkspace:
+    def __init__(
+        self,
+        isolated: bool = True,
+        config: str = None,
+        env_vars: dict = None,
+        container_engine_discover=True,
+        container_discover=True,
+    ):
+        self.isolated = isolated
+        self.config = config
+        self.environ = os.environ.copy()
+        self.workspace_dir = tempfile.mkdtemp() if self.isolated or self.config else None
+        self.storage_dir = None
+        self.__prev_working_dir = None
+
+        # Create ramalama.conf for the workspace if config is provided
+        if self.workspace_dir and self.config:
+            config_path = Path(self.workspace_dir) / "ramalama.conf"
+            with config_path.open("w") as f:
+                f.write(self.config.format(workspace_dir=self.workspace_dir))
+                self.environ["RAMALAMA_CONFIG"] = config_path.as_posix()
+
+        # Create storage directory
+        if self.isolated or self.config:
+            storage_dir = Path(self.workspace_dir) / ".storage"
+            storage_dir.mkdir()
+            self.storage_dir = storage_dir.as_posix()
+
+        # Enable env variables from pytest addoption parameters
+        if container_discover:
+            self.environ["RAMALAMA_IN_CONTAINER"] = "True" if ramalama_container else "False"
+        if container_engine_discover:
+            self.environ["RAMALAMA_CONTAINER_ENGINE"] = ramalama_container_engine
+
+        # Update the environ with the extra env vars provided if any
+        if env_vars:
+            self.environ |= env_vars
+
+    def __enter__(self):
+        self.__prev_working_dir = os.getcwd()
+        os.chdir(self.workspace_dir)
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        os.chdir(self.__prev_working_dir)
+        self.close()
+
+    def close(self):
+        if self.workspace_dir and os.path.exists(self.workspace_dir):
+            shutil.rmtree(self.workspace_dir)
+
+    def _prepare_kwargs(self, kwargs):
+        env = self.environ.copy()
+        if 'env' in kwargs:
+            env |= kwargs['env']
+        kwargs['env'] = env
+        return kwargs
+
+    def check_output(self, *args, **kwargs):
+        kwargs = self._prepare_kwargs(kwargs)
+        return subprocess.check_output(*args, **kwargs).decode("utf-8")
+
+    def check_call(self, *args, **kwargs):
+        kwargs = self._prepare_kwargs(kwargs)
+        return subprocess.check_call(*args, **kwargs)
+
+
+def check_output(*args, **kwargs):
+    with RamalamaExecWorkspace() as ctx:
+        return ctx.check_output(*args, **kwargs)
+
+
+def check_call(*args, **kwargs):
+    with RamalamaExecWorkspace() as ctx:
+        return ctx.check_call(*args, **kwargs)