ansible-test - Update venv management and sanity requirements (#83729)

* ansible-test - Update venv management
* Upgrade from pip 24.0 to 24.2
* Omit `wheel` and `setuptools` from ansible-test managed virtual environments
* Drop pre-release hacks
* Update mypy requirements
* Freeze sanity test requirements
* Update sanity test configuration
* Update sanity ignores

changelogs/fragments/ansible-test-sanity-test-requirements.yml

@@ -0,0 +1,2 @@
+minor_changes:
+  - ansible-test - Updated the frozen requirements for all sanity tests.

changelogs/fragments/ansible-test-venv-bootstrap.yml

@@ -0,0 +1,3 @@
+minor_changes:
+  - ansible-test - Virtual environments created by ansible-test no longer include the ``wheel`` or ``setuptools`` packages.
+  - ansible-test - Upgrade ``pip`` used in ansible-test managed virtual environments from version 24.0 to 24.2.

hacking/update-sanity-requirements.py

@@ -52,7 +52,6 @@ class SanityTest:
             if pip_freeze.stdout:
                 raise Exception(f'Initial virtual environment is not empty:\n{pip_freeze.stdout}')
 
-            subprocess.run(pip + ['install', 'wheel'], env=env, check=True)  # make bdist_wheel available during pip install
             subprocess.run(pip + ['install', '-r', self.source_path], env=env, check=True)
 
             freeze_options = ['--all']

test/lib/ansible_test/_data/requirements/constraints.txt

@@ -8,5 +8,3 @@ requests-ntlm >= 1.1.0 # message encryption support
 requests-credssp >= 0.1.0 # message encryption support
 mock >= 2.0.0 # needed for features backported from Python 3.6 unittest.mock (assert_called, assert_called_once...)
 pytest-mock >= 1.4.0 # needed for mock_use_standalone_module pytest option
-cffi == 1.17.0rc1 ; python_version >= '3.13' # temporary hack to support Python 3.13
-pyyaml == 6.0.2rc1 ; python_version >= '3.13' # temporary hack to support Python 3.13

test/lib/ansible_test/_data/requirements/sanity.ansible-doc.txt

@@ -1,5 +1,5 @@
 # edit "sanity.ansible-doc.in" and generate with: hacking/update-sanity-requirements.py --test ansible-doc
-Jinja2==3.1.3
+Jinja2==3.1.4
 MarkupSafe==2.1.5
-packaging==24.0
-PyYAML==6.0.1
+packaging==24.1
+PyYAML==6.0.2

test/lib/ansible_test/_data/requirements/sanity.changelog.txt

@@ -1,9 +1,9 @@
 # edit "sanity.changelog.in" and generate with: hacking/update-sanity-requirements.py --test changelog
-antsibull-changelog==0.26.0
+antsibull-changelog==0.29.0
 docutils==0.18.1
-packaging==24.0
-PyYAML==6.0.1
+packaging==24.1
+PyYAML==6.0.2
 rstcheck==5.0.0
 semantic-version==2.10.0
 types-docutils==0.18.3
-typing_extensions==4.10.0
+typing_extensions==4.12.2

test/lib/ansible_test/_data/requirements/sanity.import.plugin.txt

@@ -1,4 +1,4 @@
 # edit "sanity.import.plugin.in" and generate with: hacking/update-sanity-requirements.py --test import.plugin
-Jinja2==3.1.3
+Jinja2==3.1.4
 MarkupSafe==2.1.5
-PyYAML==6.0.1
+PyYAML==6.0.2

test/lib/ansible_test/_data/requirements/sanity.import.txt

@@ -1,2 +1,2 @@
 # edit "sanity.import.in" and generate with: hacking/update-sanity-requirements.py --test import
-PyYAML==6.0.1
+PyYAML==6.0.2

test/lib/ansible_test/_data/requirements/sanity.integration-aliases.txt

@@ -1,2 +1,2 @@
 # edit "sanity.integration-aliases.in" and generate with: hacking/update-sanity-requirements.py --test integration-aliases
-PyYAML==6.0.1
+PyYAML==6.0.2

test/lib/ansible_test/_data/requirements/sanity.mypy.in

@@ -2,9 +2,10 @@ mypy
 cryptography  # type stubs not published separately
 jinja2  # type stubs not published separately
 packaging  # type stubs not published separately
+tomli  # type stubs not published separately, required for toml inventory plugin
 types-backports
 types-paramiko
 types-pyyaml
 types-requests
-types-setuptools
+types-setuptools  # required for the pkg_resources import in the pip module
 types-toml

test/lib/ansible_test/_data/requirements/sanity.mypy.txt

@@ -1,18 +1,18 @@
 # edit "sanity.mypy.in" and generate with: hacking/update-sanity-requirements.py --test mypy
-cffi==1.16.0
-cryptography==42.0.5
-Jinja2==3.1.3
+cffi==1.17.0
+cryptography==43.0.0
+Jinja2==3.1.4
 MarkupSafe==2.1.5
-mypy==1.9.0
+mypy==1.11.1
 mypy-extensions==1.0.0
-packaging==24.0
-pycparser==2.21
+packaging==24.1
+pycparser==2.22
 tomli==2.0.1
 types-backports==0.1.3
-types-paramiko==3.4.0.20240311
-types-PyYAML==6.0.12.20240311
-types-requests==2.31.0.20240311
-types-setuptools==69.2.0.20240317
+types-paramiko==3.4.0.20240423
+types-PyYAML==6.0.12.20240724
+types-requests==2.32.0.20240712
+types-setuptools==71.1.0.20240806
 types-toml==0.10.8.20240310
-typing_extensions==4.10.0
-urllib3==2.2.1
+typing_extensions==4.12.2
+urllib3==2.2.2

test/lib/ansible_test/_data/requirements/sanity.pep8.txt

@@ -1,2 +1,2 @@
 # edit "sanity.pep8.in" and generate with: hacking/update-sanity-requirements.py --test pep8
-pycodestyle==2.11.1
+pycodestyle==2.12.1

test/lib/ansible_test/_data/requirements/sanity.pylint.txt

@@ -1,11 +1,9 @@
 # edit "sanity.pylint.in" and generate with: hacking/update-sanity-requirements.py --test pylint
-astroid==3.1.0
+astroid==3.2.4
 dill==0.3.8
 isort==5.13.2
 mccabe==0.7.0
-platformdirs==4.2.0
-pylint==3.1.0
-PyYAML==6.0.1
-tomli==2.0.1
-tomlkit==0.12.4
-typing_extensions==4.10.0
+platformdirs==4.2.2
+pylint==3.2.6
+PyYAML==6.0.2
+tomlkit==0.13.0

test/lib/ansible_test/_data/requirements/sanity.runtime-metadata.txt

@@ -1,3 +1,3 @@
 # edit "sanity.runtime-metadata.in" and generate with: hacking/update-sanity-requirements.py --test runtime-metadata
-PyYAML==6.0.1
-voluptuous==0.14.2
+PyYAML==6.0.2
+voluptuous==0.15.2

test/lib/ansible_test/_data/requirements/sanity.validate-modules.txt

@@ -1,6 +1,6 @@
 # edit "sanity.validate-modules.in" and generate with: hacking/update-sanity-requirements.py --test validate-modules
 antsibull-docs-parser==1.0.0
-Jinja2==3.1.3
+Jinja2==3.1.4
 MarkupSafe==2.1.5
-PyYAML==6.0.1
-voluptuous==0.14.2
+PyYAML==6.0.2
+voluptuous==0.15.2

test/lib/ansible_test/_internal/python_requirements.py

@@ -112,6 +112,8 @@ class PipBootstrap(PipCommand):
 
     pip_version: str
     packages: list[str]
+    setuptools: bool
+    wheel: bool
 
 
 # Entry Points
@@ -177,6 +179,8 @@ def collect_bootstrap(python: PythonConfig) -> list[PipCommand]:
     bootstrap = PipBootstrap(
         pip_version=pip_version,
         packages=packages,
+        setuptools=False,
+        wheel=False,
     )
 
     return [bootstrap]
@@ -218,17 +222,6 @@ def collect_requirements(
         # removing them reduces the size of environments cached in containers
         uninstall_packages = list(get_venv_packages(python))
 
-        if not minimize:
-            # installed packages may have run-time dependencies on setuptools
-            uninstall_packages.remove('setuptools')
-
-        # hack to allow the package-data sanity test to keep wheel in the venv
-        install_commands = [command for command in commands if isinstance(command, PipInstall)]
-        install_wheel = any(install.has_package('wheel') for install in install_commands)
-
-        if install_wheel:
-            uninstall_packages.remove('wheel')
-
         commands.extend(collect_uninstall(packages=uninstall_packages))
 
     return commands
@@ -412,9 +405,7 @@ def get_venv_packages(python: PythonConfig) -> dict[str, str]:
     #       See: https://github.com/ansible/base-test-container/blob/main/files/installer.py
 
     default_packages = dict(
-        pip='24.0',
-        setuptools='70.0.0',
-        wheel='0.43.0',
+        pip='24.2',
     )
 
     override_packages: dict[str, dict[str, str]] = {

test/lib/ansible_test/_util/controller/sanity/mypy/ansible-core.ini

@@ -8,7 +8,8 @@
 strict_optional = False
 # There are ~70 errors reported in ansible-core when checking attributes.
 # Until the number of occurrences are reduced, it's better to disable the check.
-disable_error_code = attr-defined
+# The safe-super rule is disabled because it reports false positives on methods which return None.
+disable_error_code = attr-defined,safe-super
 
 [mypy-ansible.module_utils.six.moves.*]
 ignore_missing_imports = True

test/lib/ansible_test/_util/controller/sanity/mypy/ansible-test.ini

@@ -9,7 +9,8 @@ strict_optional = False
 # There are ~13 type-abstract errors reported in ansible-test.
 # This is due to assumptions mypy makes about Type and abstract types.
 # See: https://discuss.python.org/t/add-abstracttype-to-the-typing-module/21996/13
-disable_error_code = type-abstract
+# The safe-super rule is disabled because it reports false positives on methods which return None.
+disable_error_code = type-abstract,safe-super
 
 [mypy-argcomplete]
 ignore_missing_imports = True

test/lib/ansible_test/_util/controller/sanity/pylint/config/collection.cfg

@@ -69,6 +69,7 @@ disable=
     pointless-statement,
     pointless-string-statement,
     possibly-unused-variable,
+    possibly-used-before-assignment,
     protected-access,
     raise-missing-from,  # Python 2.x does not support raise from
     redefined-argument-from-local,

test/lib/ansible_test/_util/controller/sanity/pylint/config/default.cfg

@@ -63,6 +63,7 @@ disable=
     not-an-iterable,
     not-callable,
     possibly-unused-variable,
+    possibly-used-before-assignment,
     protected-access,
     raise-missing-from,  # Python 2.x does not support raise from
     redefined-argument-from-local,

test/lib/ansible_test/_util/target/setup/requirements.py

@@ -66,6 +66,8 @@ def bootstrap(pip, options):  # type: (str, t.Dict[str, t.Any]) -> None
     """Bootstrap pip and related packages in an empty virtual environment."""
     pip_version = options['pip_version']
     packages = options['packages']
+    setuptools = options['setuptools']
+    wheel = options['wheel']
 
     url = 'https://ci-files.testing.ansible.com/ansible-test/get-pip-%s.py' % pip_version
     cache_path = os.path.expanduser('~/.ansible/test/cache/get_pip_%s.py' % pip_version.replace(".", "_"))
@@ -101,6 +103,12 @@ https://github.com/ansible/ansible/issues/77304
     options = common_pip_options()
     options.extend(packages)
 
+    if not setuptools:
+        options.append('--no-setuptools')
+
+    if not wheel:
+        options.append('--no-wheel')
+
     command = [sys.executable, pip] + options
 
     execute_command(command, env=env)

test/sanity/code-smell/deprecated-config.requirements.txt

@@ -1,4 +1,4 @@
 # edit "deprecated-config.requirements.in" and generate with: hacking/update-sanity-requirements.py --test deprecated-config
-Jinja2==3.1.3
+Jinja2==3.1.4
 MarkupSafe==2.1.5
-PyYAML==6.0.1
+PyYAML==6.0.2

test/sanity/code-smell/package-data.requirements.in

@@ -1,8 +1,6 @@
 build  # required to build sdist
-wheel  # required to build wheel
 jinja2
 pyyaml
 resolvelib < 1.1.0
 rstcheck < 6  # newer versions have too many dependencies
 antsibull-changelog
-setuptools == 66.1.0  # minimum supported setuptools

test/sanity/code-smell/package-data.requirements.txt

@@ -1,17 +1,14 @@
 # edit "package-data.requirements.in" and generate with: hacking/update-sanity-requirements.py --test package-data
-antsibull-changelog==0.26.0
-build==1.1.1
+antsibull-changelog==0.29.0
+build==1.2.1
 docutils==0.18.1
-Jinja2==3.1.3
+Jinja2==3.1.4
 MarkupSafe==2.1.5
-packaging==24.0
-pyproject_hooks==1.0.0
-PyYAML==6.0.1
+packaging==24.1
+pyproject_hooks==1.1.0
+PyYAML==6.0.2
 resolvelib==1.0.1
 rstcheck==5.0.0
 semantic-version==2.10.0
-setuptools==66.1.0
-tomli==2.0.1
 types-docutils==0.18.3
-typing_extensions==4.10.0
-wheel==0.43.0
+typing_extensions==4.12.2

test/sanity/code-smell/pymarkdown.requirements.txt

@@ -1,9 +1,9 @@
 # edit "pymarkdown.requirements.in" and generate with: hacking/update-sanity-requirements.py --test pymarkdown
 application_properties==0.8.2
 Columnar==1.4.1
-pymarkdownlnt==0.9.18
-PyYAML==6.0.1
+pymarkdownlnt==0.9.22
+PyYAML==6.0.2
 tomli==2.0.1
 toolz==0.12.1
-typing_extensions==4.10.0
+typing_extensions==4.12.2
 wcwidth==0.2.13

test/sanity/code-smell/update-bundled.requirements.txt

@@ -1,2 +1,2 @@
 # edit "update-bundled.requirements.in" and generate with: hacking/update-sanity-requirements.py --test update-bundled
-packaging==24.0
+packaging==24.1

test/sanity/ignore.txt

@@ -160,3 +160,10 @@ test/integration/targets/find/files/hello_world.gbk no-smart-quotes
 test/integration/targets/find/files/hello_world.gbk no-unwanted-characters
 lib/ansible/plugins/action/__init__.py pylint:ansible-deprecated-version  # 2.18 deprecation
 lib/ansible/template/__init__.py pylint:ansible-deprecated-version  # 2.18 deprecation
+lib/ansible/module_utils/facts/hardware/aix.py pylint:used-before-assignment
+lib/ansible/modules/rpm_key.py pylint:used-before-assignment
+lib/ansible/modules/service.py pylint:used-before-assignment
+lib/ansible/modules/user.py pylint:used-before-assignment
+lib/ansible/plugins/action/copy.py pylint:undefined-variable
+test/integration/targets/module_utils/library/test_optional.py pylint:used-before-assignment
+test/support/windows-integration/plugins/action/win_copy.py pylint:undefined-variable