From 75731fd2a7ee44ce83e89136db00a2c6582e3b11 Mon Sep 17 00:00:00 2001 From: Sergii Tkachenko Date: Thu, 18 Dec 2025 14:48:35 -0800 Subject: [PATCH] [CI][Python] Upgrade deps assuming python 3.9+ (#40323) - Regen requirements.bazel.lock with Python 3.9 - bump isort to 6.0.1 (except in pylint, which needs to be updated separately) - fix python version specifiers for black, isort and pylint, typeguard - fix default ignore patterns for isort and pylint - consistent debug info: python version, pip list - consistent virtualenv naming: `.venv-ci-*` - bazel: bump typeguard to 4.4.2 - bazel: bumped gevent to `25.9.1`, greenlet to `3.2.4` to support Python 3.13, closes #40685 - bazel: bump pyyaml for python 3.14 support - bazel: take care of temporary pins to support 3.8-based CIs Bazel RBE CIs upgraded in the following changelists, and currently run Python 3.10: - cl/845778848 - cl/845816768 Relevant testing was done in #41239. Closes #40323 PiperOrigin-RevId: 846423001 --- .pylintrc | 4 +++ .pylintrc-examples | 4 +++ .pylintrc-tests | 4 +++ grpc-style-config.toml | 6 ++++ requirements.bazel.lock | 13 ++++--- requirements.bazel.txt | 21 ++++++------ tools/distrib/black_code.sh | 17 ++++++---- tools/distrib/check_pytype.sh | 13 ++++--- tools/distrib/isort_code.sh | 17 ++++++---- tools/distrib/pylint_code.sh | 34 ++++++------------- tools/distrib/ruff_code.sh | 24 ++++++------- .../helper_scripts/prepare_build_linux_rc | 3 +- 12 files changed, 88 insertions(+), 72 deletions(-) diff --git a/.pylintrc b/.pylintrc index 006796e54e..17e0ac9b06 100644 --- a/.pylintrc +++ b/.pylintrc @@ -1,4 +1,5 @@ [MAIN] +py-version = 3.9 load-plugins= pylint.extensions.no_self_use, pylint.extensions.comparison_placement, @@ -10,6 +11,9 @@ ignore= src/python/grpcio/grpc/framework/common, src/python/grpcio/grpc/framework/foundation, src/python/grpcio/grpc/framework/interfaces, +ignore-patterns= + .*pb2\.py, + .*pb2_grpc\.py, extension-pkg-whitelist=grpc._cython.cygrpc diff --git a/.pylintrc-examples b/.pylintrc-examples index 6690ae3d30..722412c853 100644 --- a/.pylintrc-examples +++ b/.pylintrc-examples @@ -1,4 +1,5 @@ [MAIN] +py-version = 3.9 load-plugins= pylint.extensions.no_self_use, pylint.extensions.comparison_placement, @@ -10,6 +11,9 @@ ignore= src/python/grpcio/grpc/framework/common, src/python/grpcio/grpc/framework/foundation, src/python/grpcio/grpc/framework/interfaces, +ignore-patterns= + .*pb2\.py, + .*pb2_grpc\.py, [VARIABLES] diff --git a/.pylintrc-tests b/.pylintrc-tests index 794e776c65..23a525535f 100644 --- a/.pylintrc-tests +++ b/.pylintrc-tests @@ -1,4 +1,5 @@ [MAIN] +py-version = 3.9 load-plugins= pylint.extensions.no_self_use, pylint.extensions.comparison_placement, @@ -9,6 +10,9 @@ ignore= src/python/grpcio_tests/tests/unit/framework, src/python/grpcio_tests/tests/unit/framework/common, src/python/grpcio_tests/tests/unit/framework/foundation, +ignore-patterns= + .*pb2\.py, + .*pb2_grpc\.py, [VARIABLES] diff --git a/grpc-style-config.toml b/grpc-style-config.toml index bd6b50d911..862d66e602 100644 --- a/grpc-style-config.toml +++ b/grpc-style-config.toml @@ -4,6 +4,8 @@ target-version = [ "py39", "py310", "py311", + "py312", + "py313", ] extend-exclude = ''' # A regex preceded with ^/ will apply only to files and directories @@ -29,6 +31,9 @@ extend-exclude = ''' [tool.isort] profile = "black" +py_version = 39 +follow_links = false + line_length = 80 src_paths = [ "examples/python/data_transmission", @@ -48,6 +53,7 @@ skip_glob = [ "*pb2*.pyi", "**/site-packages/**/*", "tools/distrib/python/xds_protos/*", + "**/.venv-*", ] single_line_exclusions = ["typing"] force_single_line = true diff --git a/requirements.bazel.lock b/requirements.bazel.lock index d25ab148e5..29c9497621 100644 --- a/requirements.bazel.lock +++ b/requirements.bazel.lock @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with Python 3.8 +# This file is autogenerated by pip-compile with Python 3.9 # by the following command: # # pip-compile --allow-unsafe --output-file=requirements.bazel.lock requirements.bazel.txt @@ -26,13 +26,13 @@ deprecated==1.2.18 # via # opentelemetry-api # opentelemetry-semantic-conventions -gevent==24.2.1 +gevent==25.9.1 # via -r requirements.bazel.txt google-auth==2.38.0 # via -r requirements.bazel.txt googleapis-common-protos==1.66.0 # via -r requirements.bazel.txt -greenlet==3.1.1 +greenlet==3.2.4 # via gevent hyperlink==21.0.0 # via twisted @@ -77,7 +77,7 @@ pyasn1==0.6.1 # rsa pyasn1-modules==0.4.1 # via google-auth -pyyaml==6.0.2 +pyyaml==6.0.3 # via -r requirements.bazel.txt requests==2.32.3 # via @@ -89,7 +89,7 @@ tomli==2.2.1 # via incremental twisted==24.11.0 # via -r requirements.bazel.txt -typeguard==4.2.1 +typeguard==4.4.2 # via -r requirements.bazel.txt typing-extensions==4.12.2 # via @@ -115,8 +115,7 @@ zope-interface==7.2 # twisted # The following packages are considered to be unsafe in a requirements file: -setuptools==77.0.1 ; python_version > "3.8" # Manually added for python > 3.8, until Bazel RBE jobs support Python >= 3.9 (Tech debt: b/427881645) -setuptools==75.3.2 ; python_version <= "3.8" +setuptools==77.0.1 # via # -r requirements.bazel.txt # incremental diff --git a/requirements.bazel.txt b/requirements.bazel.txt index 21fb346156..eca2c0c01f 100644 --- a/requirements.bazel.txt +++ b/requirements.bazel.txt @@ -16,7 +16,7 @@ # Instructions for generation follow: # Use the oldest supported version of Python # -# $ docker run -it --rm -v $(pwd):/grpc python:3.8 /bin/bash +# $ docker run -it --rm -v $(pwd):/grpc python:3.9 /bin/bash # # cd grpc # # pip install pip-tools # # pip-compile --allow-unsafe requirements.bazel.txt -o requirements.bazel.lock @@ -26,17 +26,21 @@ # When installing a new package, sometimes it's desirable to avoid unnecessary # package upgrades. In such case, consider the following flow instead. # -# $ python3.8 -m venv .venv-bazel +# $ python3.9 -m venv .venv-bazel # $ source .venv-bazel/bin/activate # $ pip install pip-tools -# $ pip install -r requirements.bazel.lock # make the change to version in requirements.bazel.txt after this +# $ pip install -r requirements.bazel.lock +# +# Add the new package to requirements.bazel.txt # $ pip-compile --no-upgrade --allow-unsafe requirements.bazel.txt -o requirements.bazel.lock -# $ deactivate +# +# To update a single package, change its version spec requirements.bazel.txt +# $ pip-compile --no-upgrade --allow-unsafe requirements.bazel.txt -o requirements.bazel.lock -P package_name absl-py certifi chardet -gevent +gevent~=25.8 google-auth googleapis-common-protos idna @@ -47,11 +51,8 @@ opentelemetry-sdk protobuf pyyaml # for DNS test requests -setuptools>=77.0.1; python_version > "3.8" -setuptools>=75.3.2; python_version <= "3.8" -# Currently our CI uses Python < 3.8, hence <4.4.1 was added -# TODO(asheshvidyut): remove the <4.4.1, when CI uses python >= 3.9 -typeguard~=4.2.0,<4.4.1 +setuptools>=77.0.1 +typeguard~=4.2 typing-extensions==4.12.2 twisted # for DNS test urllib3 diff --git a/tools/distrib/black_code.sh b/tools/distrib/black_code.sh index 7f72662917..c8c1f6d4b0 100755 --- a/tools/distrib/black_code.sh +++ b/tools/distrib/black_code.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # Copyright 2015 gRPC authors. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -set -ex +set -eux ACTION="${1:-}" [[ $ACTION == '' ]] || [[ $ACTION == '--diff' ]] || [[ $ACTION == '--check' ]] @@ -29,9 +29,12 @@ DIRS=( 'setup.py' ) -VIRTUALENV=venv_black_code -python3 -m virtualenv $VIRTUALENV -source $VIRTUALENV/bin/activate +VIRTUALENV=".venv-ci-black" +python3 -m virtualenv "${VIRTUALENV}" +source "${VIRTUALENV}/bin/activate" +python -VV -python3 -m pip install black==25.1.0 -python3 -m black --config=grpc-style-config.toml $ACTION "${DIRS[@]}" +pip install black==25.1.0 +pip list + +exec black --config=grpc-style-config.toml $ACTION "${DIRS[@]}" diff --git a/tools/distrib/check_pytype.sh b/tools/distrib/check_pytype.sh index aa17b54068..cfd497d8f2 100755 --- a/tools/distrib/check_pytype.sh +++ b/tools/distrib/check_pytype.sh @@ -1,4 +1,4 @@ -#! /bin/bash -ex +#!/usr/bin/env bash # Copyright 2025 The gRPC Authors # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -13,9 +13,14 @@ # See the License for the specific language governing permissions and # limitations under the License. -VIRTUALENV=.venv_check_pytype -python3.11 -m virtualenv $VIRTUALENV -source $VIRTUALENV/bin/activate +set -eux + +VIRTUALENV=".venv-ci-pytype" +python3.11 -m virtualenv "${VIRTUALENV}" +source "${VIRTUALENV}/bin/activate" +python -VV pip install pytype==2024.10.11 +pip list + pytype --output=~/.cache/pytype --config=grpc-style-config.toml diff --git a/tools/distrib/isort_code.sh b/tools/distrib/isort_code.sh index 7fde72a37f..e8e16d0701 100755 --- a/tools/distrib/isort_code.sh +++ b/tools/distrib/isort_code.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # Copyright 2021 The gRPC Authors # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -set -ex +set -eux ACTION=${1:---overwrite-in-place} [[ $ACTION == '--overwrite-in-place' ]] || [[ $ACTION == '--diff' ]] @@ -33,9 +33,12 @@ DIRS=( 'setup.py' ) -VIRTUALENV=venv_isort_code -python3 -m virtualenv $VIRTUALENV -source $VIRTUALENV/bin/activate +VIRTUALENV=".venv-ci-isort" +python3 -m virtualenv "${VIRTUALENV}" +source "${VIRTUALENV}/bin/activate" +python -VV -python3 -m pip install isort==5.9.2 -python3 -m isort $ACTION --settings-path=grpc-style-config.toml --dont-follow-links "${DIRS[@]}" +pip install isort==6.0.1 +pip list + +isort $ACTION --settings-path=grpc-style-config.toml "${DIRS[@]}" diff --git a/tools/distrib/pylint_code.sh b/tools/distrib/pylint_code.sh index a2bd55c18e..c9794b6a50 100755 --- a/tools/distrib/pylint_code.sh +++ b/tools/distrib/pylint_code.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # Copyright 2017 The gRPC Authors # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -13,10 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -set -ex - -# NOTE(rbellevi): We ignore generated code. -IGNORE_PATTERNS=--ignore-patterns='.*pb2\.py,.*pb2_grpc\.py' +set -eux # change to root directory cd "$(dirname "$0")/../.." @@ -37,34 +34,25 @@ TEST_DIRS=( 'src/python/grpcio_tests/tests_gevent' ) -VIRTUALENV=".venv-pylint" - -# When running locally, allow to reuse existing venv. -if [[ ! -d "${VIRTUALENV}" ]]; then - python3.11 -m virtualenv "${VIRTUALENV}" -fi - +VIRTUALENV=".venv-ci-pylint" +python3.11 -m virtualenv "${VIRTUALENV}" source "${VIRTUALENV}/bin/activate" -python3 --version +python -VV # TODO(https://github.com/grpc/grpc/issues/23394): Update Pylint. -pip install --upgrade astroid==2.15.8 \ - pylint==2.17.7 \ - "isort~=5.11" +pip install --upgrade astroid==2.15.8 pylint==2.17.7 "isort~=5.11" +pip list EXIT=0 for dir in "${DIRS[@]}"; do - python3 -m pylint --rcfile=.pylintrc -rn "$dir" ${IGNORE_PATTERNS} || EXIT=1 + pylint --rcfile=.pylintrc -rn "$dir" || EXIT=1 done for dir in "${TEST_DIRS[@]}"; do - python3 -m pylint --rcfile=.pylintrc-tests -rn "$dir" ${IGNORE_PATTERNS} || EXIT=1 + pylint --rcfile=.pylintrc-tests -rn "$dir" || EXIT=1 done -find examples/python \ - -iname "*.py" \ - -not -name "*_pb2.py" \ - -not -name "*_pb2_grpc.py" \ - | xargs python3 -m pylint --rcfile=.pylintrc-examples -rn ${IGNORE_PATTERNS} +find examples/python -iname "*.py" -not -name "*_pb2*.py" \ + | xargs pylint --rcfile=.pylintrc-examples -rn exit $EXIT diff --git a/tools/distrib/ruff_code.sh b/tools/distrib/ruff_code.sh index 0fa2a87f54..b6dd615b9a 100755 --- a/tools/distrib/ruff_code.sh +++ b/tools/distrib/ruff_code.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # Copyright 2025 The gRPC Authors # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -13,7 +13,10 @@ # See the License for the specific language governing permissions and # limitations under the License. -set -ex +set -eux + +ACTION="${1:-}" +[[ $ACTION == '' ]] || [[ $ACTION == '--fix' ]] # change to root directory cd "$(dirname "$0")/../.." @@ -32,17 +35,12 @@ DIRS=( examples/python ) -VIRTUALENV=.venv-ruff -python3 -m virtualenv $VIRTUALENV -source $VIRTUALENV/bin/activate -python3 --version +VIRTUALENV=".venv-ci-ruff" +python3 -m virtualenv "${VIRTUALENV}" +source "${VIRTUALENV}/bin/activate" +python -VV pip install ruff==0.12.2 +pip list -# Check if --fix flag is provided -RUFF_COMMAND="ruff check" -if [[ "$1" == "--fix" ]]; then - RUFF_COMMAND="ruff check --fix" -fi - -exec $RUFF_COMMAND "${DIRS[@]}" +exec ruff check $ACTION "${DIRS[@]}" diff --git a/tools/internal_ci/helper_scripts/prepare_build_linux_rc b/tools/internal_ci/helper_scripts/prepare_build_linux_rc index 26e8a859aa..65b44f0c90 100644 --- a/tools/internal_ci/helper_scripts/prepare_build_linux_rc +++ b/tools/internal_ci/helper_scripts/prepare_build_linux_rc @@ -22,7 +22,8 @@ echo "Kokoro pool: ${KOKORO_JOB_POOL:-undefined}" if [[ -f /GRPC_IMAGE_NAME ]]; then echo "gRPC custom VM image: $(cat /GRPC_IMAGE_NAME)" fi -echo -e "System: $(uname -a)\n" +echo -e "System: $(uname -a)" +echo -e "Python: $(python3 -VV)\n" echo -e "-- OS --\n$(lsb_release -a 2>/dev/null)\n" echo -e "-- CPU --\n$(lscpu)\n" echo -e "-- Memory --\n$(lsmem --summary)\n$(free -h --si)\n" -- 2.43.0