[CI][Python] Upgrade deps assuming python 3.9+ (#40323)

- Regen requirements.bazel.lock with Python 3.9
- bump isort to 6.0.1 (except in pylint, which needs to be updated separately)
- fix python version specifiers for black, isort and pylint, typeguard
- fix default ignore patterns for isort and pylint
- consistent debug info: python version, pip list
- consistent virtualenv naming: `.venv-ci-*`
- bazel: bump typeguard to 4.4.2
- bazel: bumped gevent to `25.9.1`, greenlet to `3.2.4` to support Python 3.13, closes #40685
- bazel: bump pyyaml for python 3.14 support
- bazel: take care of temporary pins to support 3.8-based CIs

Bazel RBE CIs upgraded in the following changelists, and currently run Python 3.10:
- cl/845778848
- cl/845816768

Relevant testing was done in #41239.

Closes #40323

PiperOrigin-RevId: 846423001

diff --git a/.pylintrc b/.pylintrc
index 006796e54ed70c786b961de6cdd97e28b0b06821..17e0ac9b065c032a54064e6347897810d2a87fd1 100644 (file)
--- a/.pylintrc
+++ b/.pylintrc
@@ -1,4 +1,5 @@
 [MAIN]
+py-version = 3.9
 load-plugins=
        pylint.extensions.no_self_use,
        pylint.extensions.comparison_placement,
@@ -10,6 +11,9 @@ ignore=
        src/python/grpcio/grpc/framework/common,
        src/python/grpcio/grpc/framework/foundation,
        src/python/grpcio/grpc/framework/interfaces,
+ignore-patterns=
+       .*pb2\.py,
+       .*pb2_grpc\.py,
 
 extension-pkg-whitelist=grpc._cython.cygrpc
 

diff --git a/.pylintrc-examples b/.pylintrc-examples
index 6690ae3d30472b2251437929442982d5804be1c4..722412c85391f138a65729ed7234fe7a57c995f0 100644 (file)
--- a/.pylintrc-examples
+++ b/.pylintrc-examples
@@ -1,4 +1,5 @@
 [MAIN]
+py-version = 3.9
 load-plugins=
        pylint.extensions.no_self_use,
        pylint.extensions.comparison_placement,
@@ -10,6 +11,9 @@ ignore=
        src/python/grpcio/grpc/framework/common,
        src/python/grpcio/grpc/framework/foundation,
        src/python/grpcio/grpc/framework/interfaces,
+ignore-patterns=
+       .*pb2\.py,
+       .*pb2_grpc\.py,
 
 [VARIABLES]
 

diff --git a/.pylintrc-tests b/.pylintrc-tests
index 794e776c6534780d3e5a7b738710b87b5cb8bf03..23a525535fe0ecc9ed119aa24dc9a79995f0f96a 100644 (file)
--- a/.pylintrc-tests
+++ b/.pylintrc-tests
@@ -1,4 +1,5 @@
 [MAIN]
+py-version = 3.9
 load-plugins=
        pylint.extensions.no_self_use,
        pylint.extensions.comparison_placement,
@@ -9,6 +10,9 @@ ignore=
        src/python/grpcio_tests/tests/unit/framework,
        src/python/grpcio_tests/tests/unit/framework/common,
        src/python/grpcio_tests/tests/unit/framework/foundation,
+ignore-patterns=
+       .*pb2\.py,
+       .*pb2_grpc\.py,
 
 [VARIABLES]
 

diff --git a/grpc-style-config.toml b/grpc-style-config.toml
index bd6b50d9112be17e4a103c4c8c74db6886814337..862d66e6023f4a9d5753ac297cbe126dc32f6cd0 100644 (file)
--- a/grpc-style-config.toml
+++ b/grpc-style-config.toml
@@ -4,6 +4,8 @@ target-version = [
   "py39",
   "py310",
   "py311",
+  "py312",
+  "py313",
 ]
 extend-exclude = '''
 # A regex preceded with ^/ will apply only to files and directories
@@ -29,6 +31,9 @@ extend-exclude = '''
 
 [tool.isort]
 profile = "black"
+py_version = 39
+follow_links = false
+
 line_length = 80
 src_paths = [
     "examples/python/data_transmission",
@@ -48,6 +53,7 @@ skip_glob = [
     "*pb2*.pyi",
     "**/site-packages/**/*",
     "tools/distrib/python/xds_protos/*",
+    "**/.venv-*",
 ]
 single_line_exclusions = ["typing"]
 force_single_line = true

diff --git a/requirements.bazel.lock b/requirements.bazel.lock
index d25ab148e58e1ea9aa4ec9c64b7503661b3633f3..29c94976210314daa4c0a05c5410a4493c964bc5 100644 (file)
--- a/requirements.bazel.lock
+++ b/requirements.bazel.lock
@@ -1,5 +1,5 @@
 #
-# This file is autogenerated by pip-compile with Python 3.8
+# This file is autogenerated by pip-compile with Python 3.9
 # by the following command:
 #
 #    pip-compile --allow-unsafe --output-file=requirements.bazel.lock requirements.bazel.txt
@@ -26,13 +26,13 @@ deprecated==1.2.18
     # via
     #   opentelemetry-api
     #   opentelemetry-semantic-conventions
-gevent==24.2.1
+gevent==25.9.1
     # via -r requirements.bazel.txt
 google-auth==2.38.0
     # via -r requirements.bazel.txt
 googleapis-common-protos==1.66.0
     # via -r requirements.bazel.txt
-greenlet==3.1.1
+greenlet==3.2.4
     # via gevent
 hyperlink==21.0.0
     # via twisted
@@ -77,7 +77,7 @@ pyasn1==0.6.1
     #   rsa
 pyasn1-modules==0.4.1
     # via google-auth
-pyyaml==6.0.2
+pyyaml==6.0.3
     # via -r requirements.bazel.txt
 requests==2.32.3
     # via
@@ -89,7 +89,7 @@ tomli==2.2.1
     # via incremental
 twisted==24.11.0
     # via -r requirements.bazel.txt
-typeguard==4.2.1
+typeguard==4.4.2
     # via -r requirements.bazel.txt
 typing-extensions==4.12.2
     # via
@@ -115,8 +115,7 @@ zope-interface==7.2
     #   twisted
 
 # The following packages are considered to be unsafe in a requirements file:
-setuptools==77.0.1 ; python_version > "3.8" # Manually added for python > 3.8, until Bazel RBE jobs support Python >= 3.9 (Tech debt: b/427881645)
-setuptools==75.3.2 ; python_version <= "3.8"
+setuptools==77.0.1
     # via
     #   -r requirements.bazel.txt
     #   incremental

diff --git a/requirements.bazel.txt b/requirements.bazel.txt
index 21fb346156b82c9f29d553c7f9fdacde04738907..eca2c0c01fafaa74ac27eaf30bdc07e2aaed2802 100644 (file)
--- a/requirements.bazel.txt
+++ b/requirements.bazel.txt
@@ -16,7 +16,7 @@
 # Instructions for generation follow:
 # Use the oldest supported version of Python
 #
-#   $ docker run -it --rm -v $(pwd):/grpc python:3.8 /bin/bash
+#   $ docker run -it --rm -v $(pwd):/grpc python:3.9 /bin/bash
 #   # cd grpc
 #   # pip install pip-tools
 #   # pip-compile --allow-unsafe requirements.bazel.txt -o requirements.bazel.lock
@@ -26,17 +26,21 @@
 # When installing a new package, sometimes it's desirable to avoid unnecessary
 # package upgrades. In such case, consider the following flow instead.
 #
-#   $ python3.8 -m venv .venv-bazel
+#   $ python3.9 -m venv .venv-bazel
 #   $ source .venv-bazel/bin/activate
 #   $ pip install pip-tools
-#   $ pip install -r requirements.bazel.lock # make the change to version in requirements.bazel.txt after this
+#   $ pip install -r requirements.bazel.lock
+#
+#   Add the new package to requirements.bazel.txt
 #   $ pip-compile --no-upgrade --allow-unsafe requirements.bazel.txt -o requirements.bazel.lock
-#   $ deactivate
+#
+#   To update a single package, change its version spec requirements.bazel.txt
+#   $ pip-compile --no-upgrade --allow-unsafe requirements.bazel.txt -o requirements.bazel.lock -P package_name
 
 absl-py
 certifi
 chardet
-gevent
+gevent~=25.8
 google-auth
 googleapis-common-protos
 idna
@@ -47,11 +51,8 @@ opentelemetry-sdk
 protobuf
 pyyaml  # for DNS test
 requests
-setuptools>=77.0.1; python_version > "3.8"
-setuptools>=75.3.2; python_version <= "3.8"
-# Currently our CI uses Python < 3.8, hence <4.4.1 was added
-# TODO(asheshvidyut): remove the <4.4.1, when CI uses python >= 3.9
-typeguard~=4.2.0,<4.4.1
+setuptools>=77.0.1
+typeguard~=4.2
 typing-extensions==4.12.2
 twisted  # for DNS test
 urllib3

diff --git a/tools/distrib/black_code.sh b/tools/distrib/black_code.sh
index 7f72662917b69e22cc248cfbdd3a8cc3b2ab4b6d..c8c1f6d4b03d8fcf33fdeacf748bc8e8120481d4 100755 (executable)
--- a/tools/distrib/black_code.sh
+++ b/tools/distrib/black_code.sh
@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 # Copyright 2015 gRPC authors.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -13,7 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-set -ex
+set -eux
 
 ACTION="${1:-}"
 [[ $ACTION == '' ]] || [[ $ACTION == '--diff' ]] || [[ $ACTION == '--check' ]]
@@ -29,9 +29,12 @@ DIRS=(
     'setup.py'
 )
 
-VIRTUALENV=venv_black_code
-python3 -m virtualenv $VIRTUALENV
-source $VIRTUALENV/bin/activate
+VIRTUALENV=".venv-ci-black"
+python3 -m virtualenv "${VIRTUALENV}"
+source "${VIRTUALENV}/bin/activate"
+python -VV
 
-python3 -m pip install black==25.1.0
-python3 -m black --config=grpc-style-config.toml $ACTION "${DIRS[@]}"
+pip install black==25.1.0
+pip list
+
+exec black --config=grpc-style-config.toml $ACTION "${DIRS[@]}"

diff --git a/tools/distrib/check_pytype.sh b/tools/distrib/check_pytype.sh
index aa17b54068c255285b56364eeef15a8cd067fdd3..cfd497d8f2b4cdac1a91ae96c02949993bad52be 100755 (executable)
--- a/tools/distrib/check_pytype.sh
+++ b/tools/distrib/check_pytype.sh
@@ -1,4 +1,4 @@
-#! /bin/bash -ex
+#!/usr/bin/env bash
 # Copyright 2025 The gRPC Authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -13,9 +13,14 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-VIRTUALENV=.venv_check_pytype
-python3.11 -m virtualenv $VIRTUALENV
-source $VIRTUALENV/bin/activate
+set -eux
+
+VIRTUALENV=".venv-ci-pytype"
+python3.11 -m virtualenv "${VIRTUALENV}"
+source "${VIRTUALENV}/bin/activate"
+python -VV
 
 pip install pytype==2024.10.11
+pip list
+
 pytype --output=~/.cache/pytype --config=grpc-style-config.toml

diff --git a/tools/distrib/isort_code.sh b/tools/distrib/isort_code.sh
index 7fde72a37f26db7482ba81ba78fd67d51150262a..e8e16d0701ca8b81bd3b440a6bb628bc58fa096a 100755 (executable)
--- a/tools/distrib/isort_code.sh
+++ b/tools/distrib/isort_code.sh
@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 # Copyright 2021 The gRPC Authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -13,7 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-set -ex
+set -eux
 
 ACTION=${1:---overwrite-in-place}
 [[ $ACTION == '--overwrite-in-place' ]] || [[ $ACTION == '--diff' ]]
@@ -33,9 +33,12 @@ DIRS=(
     'setup.py'
 )
 
-VIRTUALENV=venv_isort_code
-python3 -m virtualenv $VIRTUALENV
-source $VIRTUALENV/bin/activate
+VIRTUALENV=".venv-ci-isort"
+python3 -m virtualenv "${VIRTUALENV}"
+source "${VIRTUALENV}/bin/activate"
+python -VV
 
-python3 -m pip install isort==5.9.2
-python3 -m isort $ACTION --settings-path=grpc-style-config.toml --dont-follow-links "${DIRS[@]}"
+pip install isort==6.0.1
+pip list
+
+isort $ACTION --settings-path=grpc-style-config.toml "${DIRS[@]}"

diff --git a/tools/distrib/pylint_code.sh b/tools/distrib/pylint_code.sh
index a2bd55c18e914dc04199fbd244ea3acf52dd99e1..c9794b6a50813687b984aa49e6b083efa20893bb 100755 (executable)
--- a/tools/distrib/pylint_code.sh
+++ b/tools/distrib/pylint_code.sh
@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 # Copyright 2017 The gRPC Authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -13,10 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-set -ex
-
-# NOTE(rbellevi): We ignore generated code.
-IGNORE_PATTERNS=--ignore-patterns='.*pb2\.py,.*pb2_grpc\.py'
+set -eux
 
 # change to root directory
 cd "$(dirname "$0")/../.."
@@ -37,34 +34,25 @@ TEST_DIRS=(
     'src/python/grpcio_tests/tests_gevent'
 )
 
-VIRTUALENV=".venv-pylint"
-
-# When running locally, allow to reuse existing venv.
-if [[ ! -d "${VIRTUALENV}" ]]; then
-  python3.11 -m virtualenv "${VIRTUALENV}"
-fi
-
+VIRTUALENV=".venv-ci-pylint"
+python3.11 -m virtualenv "${VIRTUALENV}"
 source "${VIRTUALENV}/bin/activate"
-python3 --version
+python -VV
 
 # TODO(https://github.com/grpc/grpc/issues/23394): Update Pylint.
-pip install --upgrade astroid==2.15.8 \
-  pylint==2.17.7 \
-  "isort~=5.11"
+pip install --upgrade astroid==2.15.8 pylint==2.17.7 "isort~=5.11"
+pip list
 
 EXIT=0
 for dir in "${DIRS[@]}"; do
-  python3 -m pylint --rcfile=.pylintrc -rn "$dir" ${IGNORE_PATTERNS}  || EXIT=1
+  pylint --rcfile=.pylintrc -rn "$dir" || EXIT=1
 done
 
 for dir in "${TEST_DIRS[@]}"; do
-  python3 -m pylint --rcfile=.pylintrc-tests -rn "$dir" ${IGNORE_PATTERNS} || EXIT=1
+  pylint --rcfile=.pylintrc-tests -rn "$dir" || EXIT=1
 done
 
-find examples/python \
-  -iname "*.py" \
-  -not -name "*_pb2.py" \
-  -not -name "*_pb2_grpc.py" \
-  | xargs python3 -m pylint --rcfile=.pylintrc-examples -rn ${IGNORE_PATTERNS}
+find examples/python -iname "*.py" -not -name "*_pb2*.py" \
+  | xargs pylint --rcfile=.pylintrc-examples -rn
 
 exit $EXIT

diff --git a/tools/distrib/ruff_code.sh b/tools/distrib/ruff_code.sh
index 0fa2a87f54dc0513d1d70943ab2ef664b07d5a61..b6dd615b9ad5813c50906783f6b20f4a0c57a59b 100755 (executable)
--- a/tools/distrib/ruff_code.sh
+++ b/tools/distrib/ruff_code.sh
@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 # Copyright 2025 The gRPC Authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -13,7 +13,10 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-set -ex
+set -eux
+
+ACTION="${1:-}"
+[[ $ACTION == '' ]] || [[ $ACTION == '--fix' ]]
 
 # change to root directory
 cd "$(dirname "$0")/../.."
@@ -32,17 +35,12 @@ DIRS=(
    examples/python
 )
 
-VIRTUALENV=.venv-ruff
-python3 -m virtualenv $VIRTUALENV
-source $VIRTUALENV/bin/activate
-python3 --version
+VIRTUALENV=".venv-ci-ruff"
+python3 -m virtualenv "${VIRTUALENV}"
+source "${VIRTUALENV}/bin/activate"
+python -VV
 
 pip install ruff==0.12.2
+pip list
 
-# Check if --fix flag is provided
-RUFF_COMMAND="ruff check"
-if [[ "$1" == "--fix" ]]; then
-    RUFF_COMMAND="ruff check --fix"
-fi
-
-exec $RUFF_COMMAND "${DIRS[@]}"
+exec ruff check $ACTION "${DIRS[@]}"

diff --git a/tools/internal_ci/helper_scripts/prepare_build_linux_rc b/tools/internal_ci/helper_scripts/prepare_build_linux_rc
index 26e8a859aaeff63e8ffdb693c399c24c84d620d1..65b44f0c90ac031fcc7bcd7893aa1d991f03ee2d 100644 (file)
--- a/tools/internal_ci/helper_scripts/prepare_build_linux_rc
+++ b/tools/internal_ci/helper_scripts/prepare_build_linux_rc
@@ -22,7 +22,8 @@ echo "Kokoro pool: ${KOKORO_JOB_POOL:-undefined}"
 if [[ -f /GRPC_IMAGE_NAME ]]; then
   echo "gRPC custom VM image: $(cat /GRPC_IMAGE_NAME)"
 fi
-echo -e "System: $(uname -a)\n"
+echo -e "System: $(uname -a)"
+echo -e "Python: $(python3 -VV)\n"
 echo -e "-- OS --\n$(lsb_release -a 2>/dev/null)\n"
 echo -e "-- CPU --\n$(lscpu)\n"
 echo -e "-- Memory --\n$(lsmem --summary)\n$(free -h --si)\n"