Migrate Common Linting Tooling to Rust-Based Packages (#38107)

* Swap out pylint and flake8 with ruff

* time the mypy run

* consolidate mypy checks into linting

* Migrate pre-commit hook, pin versions in setup.py

* fix new breakages

* lint/format adjustments

* review comments, fix SKILL configs

Author: jrmccluskey

.agent/skills/python-development/SKILL.md

@@ -41,9 +41,9 @@ description: Guides Python SDK development in Apache Beam, including environment
 - `pyproject.toml` - Build configuration
 - `tox.ini` - Test automation
 - `pytest.ini` - Pytest configuration
-- `.pylintrc` - Linting rules
+- `ruff.toml` - Linting rules
 - `.isort.cfg` - Import sorting
-- `mypy.ini` - Type checking
+- `pyrefly.toml` - Type checking
 
 ## Environment Setup
 
@@ -176,10 +176,10 @@ Use `--requirements_file=requirements.txt` or custom containers.
 ## Code Quality Tools
 ```bash
 # Linting
-pylint apache_beam/
+ruff check apache_beam/
 
 # Type checking
-mypy apache_beam/
+pyrefly check apache_beam/
 
 # Formatting (via yapf)
 yapf -i apache_beam/file.py

.pre-commit-config.yaml

@@ -31,12 +31,11 @@ repos:
                 sdks/python/apache_beam/portability/api/.*pb2.*.py
             )$
 
-  - repo: https://github.com/pycqa/pylint
-    # this rev is a release tag in the repo above and corresponds with a pylint
-    # version. make sure this matches the version of pylint in tox.ini.
-    rev: v4.0.2
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    # this rev is a release tag in the repo above and corresponds with a ruff
+    # version. make sure this matches the version of ruff in setup.py
+    rev: v0.15.7
     hooks:
-      - id: pylint
-        args: ["--rcfile=sdks/python/.pylintrc"]
+      - id: ruff-check
         files: ^sdks/python/apache_beam/
-        exclude: *exclude
+        args: ["--config=sdks/python/ruff.toml"]

sdks/python/apache_beam/io/gcp/bigquery_change_history_test.py

@@ -43,6 +43,7 @@
 
 def _ts(*args, **kwargs) -> Timestamp:
   """Create a UTC datetime and return a Beam Timestamp."""
+  # pyrefly: ignore[bad-keyword-argument]
   dt = datetime.datetime(*args, tzinfo=datetime.timezone.utc, **kwargs)
   return Timestamp(dt.timestamp())
 

sdks/python/apache_beam/ml/inference/agent_development_kit.py

@@ -237,6 +237,7 @@ def run_inference(
 
       # Wrap plain strings in a Content object
       if isinstance(element, str):
+        # pyrefly: ignore[bad-instantiation]
         message = genai_Content(role="user", parts=[genai_Part(text=element)])
       else:
         # Assume the caller has already constructed a types.Content object

sdks/python/apache_beam/typehints/typehints_test.py

@@ -712,7 +712,8 @@ def test_type_checks_not_dict(self):
 
   def test_type_check_collection(self):
     hint = typehints.Dict[str, int]
-    l = collections.defaultdict(list[("blue", 2)])
+    element = ("blue", 2)
+    l = collections.defaultdict(list[element])
     self.assertIsNone(hint.type_check(l))
 
   def test_type_check_invalid_key_type(self):

sdks/python/ruff.toml

@@ -0,0 +1,96 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+exclude = [
+    ".bzr",
+    ".direnv",
+    ".eggs",
+    ".git",
+    ".git-rewrite",
+    ".hg",
+    ".ipynb_checkpoints",
+    ".mypy_cache",
+    ".nox",
+    ".pants.d",
+    ".pyenv",
+    ".pytest_cache",
+    ".pytype",
+    ".ruff_cache",
+    ".svn",
+    ".tox",
+    ".venv",
+    ".vscode",
+    "__pypackages__",
+    "_build",
+    "buck-out",
+    "build",
+    "dist",
+    "node_modules",
+    "site-packages",
+    "venv",
+    "*.pxd",
+    "*.pyx",
+    "*pb2*.py",
+    "**/examples/**/*.py",
+    "**/examples/**/*.ipynb",
+    "**/portability/api/**/*.py",
+    "**/portability/api/__init__.py",
+]
+
+target-version = "py310"
+
+src = ["apache_beam"]
+
+[lint]
+select = ["E9", "PL", "F821", "F822", "F823"]
+ignore = [
+    # Ignored Pylint Checks
+    "PLC0415",  # import-outside-toplevel
+    "PLR2004",  # magic-value-comparison
+    "PLR0913",  # too-many-arguments
+    "PLR0912",  # too-many-branches
+    "PLW0108",  # unnecessary-lambda
+    "PLW2901",  # redefined-loop-name
+    "PLR0915",  # too-many-statements
+    "PLR1714",  # repeated-equality-comparison
+    "PLR0911",  # too-many-return-statements
+    "PLR5501",  # collapsible-else-if
+    "PLW0603",  # global-statement
+    "PLR1730",  # if-stmt-min-max
+    "PLW1641",  # eq-without-hash
+    "PLW0602",  # global-variable-not-assigned
+    "PLC1802",  # len-test
+    "PLC3002",  # unnecessary-direct-lambda-call
+    "PLW0642",  # self-or-cls-assignment
+    "PLR1733",  # unnecessary-dict-index-lookup
+    "PLR0402",  # manual-from-import
+    "PLC0132",  # type-param-name-mismatch
+    "PLC0206",  # dict-index-missing-items
+    "PLC0207",  # missing-maxsplit-arg
+    "PLR1704",  # redefined-argument-from-local
+    "PLR1711",  # useless-return
+    "PLW0406",  # import-self
+    "PLW3301",  # nested-min-max
+    "PLR2044",  # empty-comment
+]
+
+# Allow fix for all enabled rules (when `--fix`) is provided.
+fixable = ["ALL"]
+unfixable = []
+
+# Allow unused variables when underscore-prefixed.
+dummy-variable-rgx = "^(_+|(_+[a-zA-Z0-9_]*[a-zA-Z0-9]+?))$"

sdks/python/scripts/run_pylint.sh

@@ -78,11 +78,8 @@ done
 echo -e "Skipping lint for files:\n${FILES_TO_IGNORE}"
 echo -e "Linting modules:\n${MODULE}"
 
-echo "Running pylint..."
-pylint -j8 ${MODULE} --ignore-patterns="$FILES_TO_IGNORE"
-echo "Running flake8..."
-flake8 ${MODULE} --count --select=E9,F821,F822,F823 --show-source --statistics \
-  --exclude="${FILES_TO_IGNORE}"
+echo "Running ruff..."
+ruff check ${MODULE} --extend-exclude="$FILES_TO_IGNORE"
 
 echo "Running isort..."
 # Skip files where isort is behaving weirdly

sdks/python/setup.py

@@ -49,7 +49,7 @@ def normalize_path(filename):
   return os.path.normcase(os.path.realpath(os.path.normpath(filename)))
 
 
-class mypy(Command):
+class pyrefly(Command):
   user_options = []
 
   def initialize_options(self):
@@ -71,10 +71,10 @@ def get_project_path(self):
     return os.path.join(project_path, to_filename(ei_cmd.egg_name))
 
   def run(self):
-    args = ['mypy', self.get_project_path()]
+    args = ['pyrefly', 'check', self.get_project_path()]
     result = subprocess.call(args)
     if result != 0:
-      raise DistutilsError("mypy exited with status %d" % result)
+      raise DistutilsError("pyrefly exited with status %d" % result)
 
 
 def get_version():
@@ -446,6 +446,12 @@ def get_portability_package_data():
       python_requires=python_requires,
       # BEAM-8840: Do NOT use tests_require or setup_requires.
       extras_require={
+          'dev': [
+            'isort==7.0.0',
+            'pyrefly==0.54.0',
+            'ruff==0.15.7',
+            'yapf==0.43.0',
+          ],
           'dill': [
               # Dill doesn't have forwards-compatibility guarantees within minor
               # version. Pickles created with a new version of dill may not
@@ -683,6 +689,6 @@ def get_portability_package_data():
       license='Apache License, Version 2.0',
       keywords=PACKAGE_KEYWORDS,
       cmdclass={
-          'mypy': mypy,
+          'pyrefly': pyrefly,
       },
   )

sdks/python/test-suites/tox/pycommon/build.gradle

@@ -40,6 +40,3 @@ check.dependsOn formatter
 
 toxTask "lint", "lint", "${posargs}"
 linter.dependsOn lint
-
-toxTask "mypy", "mypy", "${posargs}"
-linter.dependsOn mypy

sdks/python/tox.ini

@@ -17,7 +17,7 @@
 
 [tox]
 # new environments will be excluded by default unless explicitly added to envlist.
-envlist = py310,py311,py312,py313,py314,py310-{cloud,cloudcoverage,dask},py311-{cloud,dask},py312-{cloud,dask},py313-{cloud,dask},py314-{cloud,dask},docs,lint,mypy,whitespacelint
+envlist = py310,py311,py312,py313,py314,py310-{cloud,cloudcoverage,dask},py311-{cloud,dask},py312-{cloud,dask},py313-{cloud,dask},py314-{cloud,dask},docs,lint,whitespacelint
 toxworkdir = {toxinidir}/target/{env:ENV_NAME:.tox}
 
 [pycodestyle]
@@ -190,16 +190,17 @@ commands =
 [testenv:lint]
 # Don't set TMPDIR to avoid "AF_UNIX path too long" errors in pylint.
 setenv =
-# keep the version of pylint in sync with the 'rev' in .pre-commit-config.yaml
 deps =
-  astroid<4.1.0,>=4.0.1
-  pycodestyle==2.8.0
-  pylint==4.0.2
-  isort==7.0.0
-  flake8==4.0.1
+  dask==2022.01.0
+  distributed==2022.01.0
+extras =
+  gcp
+  dev
 commands =
-  pylint --version
+  ruff --version
   time {toxinidir}/scripts/run_pylint.sh
+  pyrefly --version
+  time python setup.py pyrefly
 
 [testenv:whitespacelint]
 setenv =
@@ -208,18 +209,6 @@ deps =
 commands =
   time {toxinidir}/scripts/run_whitespacelint.sh
 
-[testenv:mypy]
-deps =
-  mypy==1.13.0
-  dask==2022.01.0
-  distributed==2022.01.0
-# make extras available in case any of these libs are typed
-extras =
-  gcp
-commands =
-  mypy --version
-  python setup.py mypy
-
 
 [testenv:docs]
 extras = test,gcp,docs,interactive,dataframe,dask